From d0e4af8a88dc7a0377677000d0c92104ff215347 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 15 Apr 2025 11:19:19 -0400
Subject: [PATCH 001/710] Silence -Wcast-function-type warnings on idiomatic
 Windows code (#135660)

On Windows, GetProcAddress() is the API used to dynamically load
function pointers (similar to dlsym on Linux). This API returns a
function pointer (a typedef named FARPROC), which means that casting
from the call to the eventual correct type is technically a function
type mismatch on the cast. However, because this is idiomatic code on
Windows, we should accept it unless -Wcast-function-type-strict is
passed.

This was brought up in post-commit review feedback on
https://github.com/llvm/llvm-project/pull/86131
---
 clang/docs/ReleaseNotes.rst                   | 10 ++++++
 clang/lib/Sema/SemaCast.cpp                   | 23 ++++++++++++
 clang/test/Sema/warn-cast-function-type-win.c | 36 +++++++++++++++++++
 3 files changed, 69 insertions(+)
 create mode 100644 clang/test/Sema/warn-cast-function-type-win.c

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 6d1daaa84caaa..2d2606085998c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -358,6 +358,16 @@ Improvements to Clang's diagnostics
 
 - Now correctly diagnose a tentative definition of an array with static
   storage duration in pedantic mode in C. (#GH50661)
+- No longer diagnosing idiomatic function pointer casts on Windows under
+  ``-Wcast-function-type-mismatch`` (which is enabled by ``-Wextra``). Clang
+  would previously warn on this construct, but will no longer do so on Windows:
+
+  .. code-block:: c
+
+    typedef void (WINAPI *PGNSI)(LPSYSTEM_INFO);
+    HMODULE Lib = LoadLibrary("kernel32");
+    PGNSI FnPtr = (PGNSI)GetProcAddress(Lib, "GetNativeSystemInfo");
+
 
 - An error is now emitted when a ``musttail`` call is made to a function marked with the ``not_tail_called`` attribute. (#GH133509).
 
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 2824dfce1572c..1591075ff05d8 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -1153,10 +1153,33 @@ static unsigned int checkCastFunctionType(Sema &Self, const ExprResult &SrcExpr,
     return false;
   };
 
+  auto IsFarProc = [](const FunctionType *T) {
+    // The definition of FARPROC depends on the platform in terms of its return
+    // type, which could be int, or long long, etc. We'll look for a source
+    // signature for: <integer type> (*)() and call that "close enough" to
+    // FARPROC to be sufficient to silence the diagnostic. This is similar to
+    // how we allow casts between function pointers and void * for supporting
+    // dlsym.
+    // Note: we could check for __stdcall on the function pointer as well, but
+    // that seems like splitting hairs.
+    if (!T->getReturnType()->isIntegerType())
+      return false;
+    if (const auto *PT = T->getAs<FunctionProtoType>())
+      return !PT->isVariadic() && PT->getNumParams() == 0;
+    return true;
+  };
+
   // Skip if either function type is void(*)(void)
   if (IsVoidVoid(SrcFTy) || IsVoidVoid(DstFTy))
     return 0;
 
+  // On Windows, GetProcAddress() returns a FARPROC, which is a typedef for a
+  // function pointer type (with no prototype, in C). We don't want to diagnose
+  // this case so we don't diagnose idiomatic code on Windows.
+  if (Self.getASTContext().getTargetInfo().getTriple().isOSWindows() &&
+      IsFarProc(SrcFTy))
+    return 0;
+
   // Check return type.
   if (!argTypeIsABIEquivalent(SrcFTy->getReturnType(), DstFTy->getReturnType(),
                               Self.Context))
diff --git a/clang/test/Sema/warn-cast-function-type-win.c b/clang/test/Sema/warn-cast-function-type-win.c
new file mode 100644
index 0000000000000..4e7ba33b258d8
--- /dev/null
+++ b/clang/test/Sema/warn-cast-function-type-win.c
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 %s -triple x86_64-windows -fsyntax-only -Wcast-function-type -Wno-cast-function-type-strict -verify=windows
+// RUN: %clang_cc1 %s -triple x86_64-windows -fsyntax-only -Wcast-function-type -Wno-cast-function-type-strict -x c++ -verify=windows
+// RUN: %clang_cc1 %s -triple x86_64-pc-linux -fsyntax-only -Wcast-function-type -Wno-cast-function-type-strict -verify=linux
+// RUN: %clang_cc1 %s -triple x86_64-pc-linux -fsyntax-only -Wcast-function-type -Wno-cast-function-type-strict -x c++ -verify=linux,linux-cpp
+// RUN: %clang_cc1 %s -triple x86_64-windows -fsyntax-only -Wcast-function-type -Wcast-function-type-strict -x c++ -verify=strict
+// windows-no-diagnostics
+
+// On Windows targets, this is expected to compile fine, and on non-Windows
+// targets, this should diagnose the mismatch. This is to allow for idiomatic
+// use of GetProcAddress, similar to what we do for dlsym. On non-Windows
+// targets, this should be diagnosed.
+typedef int (*FARPROC1)();
+typedef unsigned long long (*FARPROC2)();
+
+FARPROC1 GetProcAddress1(void);
+FARPROC2 GetProcAddress2(void);
+
+typedef int (*test1_type)(int);
+typedef float(*test2_type)();
+
+void test(void) {
+  // This does not diagnose on Linux in C mode because FARPROC1 has a matching
+  // return type to test1_type, but FARPROC1 has no prototype and so checking
+  // is disabled for further compatibility issues. In C++ mode, all functions
+  // have a prototype and so the check happens.
+  test1_type t1 = (test1_type)GetProcAddress1();
+  // linux-cpp-warning@-1 {{cast from 'FARPROC1' (aka 'int (*)()') to 'test1_type' (aka 'int (*)(int)') converts to incompatible function type}}
+  // strict-warning@-2 {{cast from 'FARPROC1' (aka 'int (*)()') to 'test1_type' (aka 'int (*)(int)') converts to incompatible function type}}
+  
+  // This case is diagnosed in both C and C++ modes on Linux because the return
+  // type of FARPROC2 does not match the return type of test2_type.
+  test2_type t2 = (test2_type)GetProcAddress2();
+  // linux-warning@-1 {{cast from 'FARPROC2' (aka 'unsigned long long (*)()') to 'test2_type' (aka 'float (*)()') converts to incompatible function type}}
+  // strict-warning@-2 {{cast from 'FARPROC2' (aka 'unsigned long long (*)()') to 'test2_type' (aka 'float (*)()') converts to incompatible function type}}
+}
+

From 88b6229dc3f65876b3f627616a024cd5dbcadcb0 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 15 Apr 2025 08:41:39 -0700
Subject: [PATCH 002/710] [TableGen] Remove unhelpful error messages from
 PseudoLoweringEmitter. (#135747)

All of the notes using the location of ResultInst will just print the
location inside of the PseudoInstExpansion class.

There was one note using the location of DI->getDef(), but knowing where
one of the two mismatched types is defined isn't helpful. The operand
types need to be the same, so the mismatch message we already printed
should be enough.
---
 llvm/utils/TableGen/PseudoLoweringEmitter.cpp | 67 +++++++------------
 1 file changed, 25 insertions(+), 42 deletions(-)

diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index 56b9e499cb4aa..7f67c13c0bbbd 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -90,14 +90,12 @@ unsigned PseudoLoweringEmitter::addDagOperandMapping(
       // problem.
       // FIXME: We probably shouldn't ever get a non-zero BaseIdx here.
       assert(BaseIdx == 0 && "Named subargument in pseudo expansion?!");
-      if (DI->getDef() != Insn.Operands[BaseIdx + i].Rec) {
-        PrintError(Rec, "In pseudo instruction '" + Rec->getName() +
-                            "', operand type '" + DI->getDef()->getName() +
-                            "' does not match expansion operand type '" +
-                            Insn.Operands[BaseIdx + i].Rec->getName() + "'");
-        PrintFatalNote(DI->getDef(),
-                       "Value was assigned at the following location:");
-      }
+      if (DI->getDef() != Insn.Operands[BaseIdx + i].Rec)
+        PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                                 "', operand type '" + DI->getDef()->getName() +
+                                 "' does not match expansion operand type '" +
+                                 Insn.Operands[BaseIdx + i].Rec->getName() +
+                                 "'");
       // Source operand maps to destination operand. The Data element
       // will be filled in later, just set the Kind for now. Do it
       // for each corresponding MachineInstr operand, not just the first.
@@ -138,38 +136,26 @@ void PseudoLoweringEmitter::evaluateExpansion(const Record *Rec) {
   LLVM_DEBUG(dbgs() << "  Result: " << *Dag << "\n");
 
   const DefInit *OpDef = dyn_cast<DefInit>(Dag->getOperator());
-  if (!OpDef) {
-    PrintError(Rec, "In pseudo instruction '" + Rec->getName() +
-                        "', result operator is not a record");
-    PrintFatalNote(Rec->getValue("ResultInst"),
-                   "Result was assigned at the following location:");
-  }
+  if (!OpDef)
+    PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                             "', result operator is not a record");
   const Record *Operator = OpDef->getDef();
-  if (!Operator->isSubClassOf("Instruction")) {
-    PrintError(Rec, "In pseudo instruction '" + Rec->getName() +
-                        "', result operator '" + Operator->getName() +
-                        "' is not an instruction");
-    PrintFatalNote(Rec->getValue("ResultInst"),
-                   "Result was assigned at the following location:");
-  }
+  if (!Operator->isSubClassOf("Instruction"))
+    PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                             "', result operator '" + Operator->getName() +
+                             "' is not an instruction");
 
   CodeGenInstruction Insn(Operator);
 
-  if (Insn.isCodeGenOnly || Insn.isPseudo) {
-    PrintError(Rec, "In pseudo instruction '" + Rec->getName() +
-                        "', result operator '" + Operator->getName() +
-                        "' cannot be a pseudo instruction");
-    PrintFatalNote(Rec->getValue("ResultInst"),
-                   "Result was assigned at the following location:");
-  }
+  if (Insn.isCodeGenOnly || Insn.isPseudo)
+    PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                             "', result operator '" + Operator->getName() +
+                             "' cannot be a pseudo instruction");
 
-  if (Insn.Operands.size() != Dag->getNumArgs()) {
-    PrintError(Rec, "In pseudo instruction '" + Rec->getName() +
-                        "', result operator '" + Operator->getName() +
-                        "' has the wrong number of operands");
-    PrintFatalNote(Rec->getValue("ResultInst"),
-                   "Result was assigned at the following location:");
-  }
+  if (Insn.Operands.size() != Dag->getNumArgs())
+    PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                             "', result operator '" + Operator->getName() +
+                             "' has the wrong number of operands");
 
   unsigned NumMIOperands = 0;
   for (const auto &Op : Insn.Operands)
@@ -202,13 +188,10 @@ void PseudoLoweringEmitter::evaluateExpansion(const Record *Rec) {
       continue;
     StringMap<unsigned>::iterator SourceOp =
         SourceOperands.find(Dag->getArgNameStr(i));
-    if (SourceOp == SourceOperands.end()) {
-      PrintError(Rec, "In pseudo instruction '" + Rec->getName() +
-                          "', output operand '" + Dag->getArgNameStr(i) +
-                          "' has no matching source operand");
-      PrintFatalNote(Rec->getValue("ResultInst"),
-                     "Value was assigned at the following location:");
-    }
+    if (SourceOp == SourceOperands.end())
+      PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                               "', output operand '" + Dag->getArgNameStr(i) +
+                               "' has no matching source operand");
     // Map the source operand to the destination operand index for each
     // MachineInstr operand.
     for (unsigned I = 0, E = Insn.Operands[i].MINumOperands; I != E; ++I)

From 7b5a459611212b650e863c0ad6a9fa49c07e29df Mon Sep 17 00:00:00 2001
From: Paul Bowen-Huggett <paulhuggett@mac.com>
Date: Tue, 15 Apr 2025 17:52:00 +0200
Subject: [PATCH 003/710] [RISCV] Just reporting an error shouldn't generate a
 crash diagnostic (#134040)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Wanting to examine some of generated code, I tried MCA with the command:
~~~bash
llvm-mca -mtriple=riscv32-unknown-unknown -mcpu=rocket -iterations=300
core_list_join.s
~~~
I was greeted with the following error message:
~~~
LLVM ERROR: RV32 target requires an RV32 CPU
PLEASE submit a bug report to
https://github.com/llvm/llvm-project/issues/ and include the crash
backtrace.
Stack dump:
…
~~~
On beginning to investigate the “bug”, I discovered that the code was
simply attempting to report a user error. It used report_fatal_error()
to do so but with the “bool GenCrashDiag” argument enabled (the
default). This tiny change adds a wrapper function which calls
report_fatal_error() as before but with GenCrashDiag disabled.
---
 .../RISCV/MCTargetDesc/RISCVBaseInfo.cpp       | 18 +++++++++++++-----
 llvm/test/MC/RISCV/target-abi-invalid.s        |  6 +++---
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
index 3b3460c308d7e..6d2659aa1236e 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
@@ -51,6 +51,14 @@ namespace RISCV {
 #include "RISCVGenSearchableTables.inc"
 } // namespace RISCV
 
+// Report an error but don't ask the user to report a bug.
+[[noreturn]] static void reportError(const char *Reason) {
+  report_fatal_error(Reason, /*gen_crash_diag=*/false);
+}
+[[noreturn]] static void reportError(Error Err) {
+  report_fatal_error(std::move(Err), /*gen_crash_diag=*/false);
+}
+
 namespace RISCVABI {
 ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits,
                      StringRef ABIName) {
@@ -87,7 +95,7 @@ ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits,
   if ((TargetABI == RISCVABI::ABI::ABI_ILP32E ||
        (TargetABI == ABI_Unknown && IsRVE && !IsRV64)) &&
       FeatureBits[RISCV::FeatureStdExtD])
-    report_fatal_error("ILP32E cannot be used with the D ISA extension");
+    reportError("ILP32E cannot be used with the D ISA extension");
 
   if (TargetABI != ABI_Unknown)
     return TargetABI;
@@ -95,7 +103,7 @@ ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits,
   // If no explicit ABI is given, try to compute the default ABI.
   auto ISAInfo = RISCVFeatures::parseFeatureBits(IsRV64, FeatureBits);
   if (!ISAInfo)
-    report_fatal_error(ISAInfo.takeError());
+    reportError(ISAInfo.takeError());
   return getTargetABI((*ISAInfo)->computeDefaultABI());
 }
 
@@ -127,12 +135,12 @@ namespace RISCVFeatures {
 
 void validate(const Triple &TT, const FeatureBitset &FeatureBits) {
   if (TT.isArch64Bit() && !FeatureBits[RISCV::Feature64Bit])
-    report_fatal_error("RV64 target requires an RV64 CPU");
+    reportError("RV64 target requires an RV64 CPU");
   if (!TT.isArch64Bit() && !FeatureBits[RISCV::Feature32Bit])
-    report_fatal_error("RV32 target requires an RV32 CPU");
+    reportError("RV32 target requires an RV32 CPU");
   if (FeatureBits[RISCV::Feature32Bit] &&
       FeatureBits[RISCV::Feature64Bit])
-    report_fatal_error("RV32 and RV64 can't be combined");
+    reportError("RV32 and RV64 can't be combined");
 }
 
 llvm::Expected<std::unique_ptr<RISCVISAInfo>>
diff --git a/llvm/test/MC/RISCV/target-abi-invalid.s b/llvm/test/MC/RISCV/target-abi-invalid.s
index f78b1481b1e48..253af3f1a395a 100644
--- a/llvm/test/MC/RISCV/target-abi-invalid.s
+++ b/llvm/test/MC/RISCV/target-abi-invalid.s
@@ -30,7 +30,7 @@
 # RUN:   | FileCheck -check-prefix=RV32E-LP64 %s
 # RUN: llvm-mc -triple=riscv32 -mattr=+e,+f -target-abi lp64f < %s 2>&1 \
 # RUN:   | FileCheck -check-prefix=RV32EF-LP64F %s
-# RUN: not --crash llvm-mc -triple=riscv32 -mattr=+e,+d -target-abi lp64f < %s 2>&1 \
+# RUN: not llvm-mc -triple=riscv32 -mattr=+e,+d -target-abi lp64f < %s 2>&1 \
 # RUN:   | FileCheck -check-prefix=RV32EFD-LP64D %s
 # RUN: llvm-mc -triple=riscv32 -mattr=+e -target-abi lp64e %s 2>&1 \
 # RUN:   | FileCheck -check-prefix=RV32E-LP64E %s
@@ -70,9 +70,9 @@
 # RUN:   | FileCheck -check-prefix=RV32EF-ILP32F %s
 # RUN: llvm-mc -triple=riscv32 -mattr=+e,+f -target-abi ilp32f < %s 2>&1 \
 # RUN:   | FileCheck -check-prefix=RV32EF-ILP32F %s
-# RUN: not --crash llvm-mc -triple=riscv32 -mattr=+e,+d -target-abi ilp32f < %s 2>&1 \
+# RUN: not llvm-mc -triple=riscv32 -mattr=+e,+d -target-abi ilp32f < %s 2>&1 \
 # RUN:   | FileCheck -check-prefix=RV32EFD-ILP32F %s
-# RUN: not --crash llvm-mc -triple=riscv32 -mattr=+e,+d -target-abi ilp32d < %s 2>&1 \
+# RUN: not llvm-mc -triple=riscv32 -mattr=+e,+d -target-abi ilp32d < %s 2>&1 \
 # RUN:   | FileCheck -check-prefix=RV32EFD-ILP32D %s
 
 # RV32E-ILP32: Only the ilp32e ABI is supported for RV32E (ignoring target-abi)

From a399c6926a8701083c767cbb041e22ff92e9d717 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Tue, 15 Apr 2025 13:00:49 -0300
Subject: [PATCH 004/710] [CI] monolithic-linux improvements (#135499)

Some improvements to monolithic-linux CI:

1) Add correct configuration and dependencies for LLDB testing which
   is actually relevant for clang changes.
2) Skip clang installation and separate configuration for runtimes.
   They will be built with the just built clang either way.
   This avoids building the runtimes twice when LLDB is also tested.
3) Make sure any generated clang reproducers end up as artifacts.
4) Set up llvm-symbolizer environment variable so that its preferred
over
any symbolizer just built, as it can be much slower when built for
debugging.
5) Add all projects as dependencies of `.ci`, to make sure everything is
   tested when it changes.
---
 .ci/compute_projects.py      | 10 +++--
 .ci/compute_projects_test.py | 13 +++++++
 .ci/monolithic-linux.sh      | 72 ++++++++++++++++--------------------
 lldb/test/requirements.txt   |  2 +
 4 files changed, 53 insertions(+), 44 deletions(-)

diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py
index ff43547c9bbe5..17a2136a270d5 100644
--- a/.ci/compute_projects.py
+++ b/.ci/compute_projects.py
@@ -52,6 +52,9 @@
     "clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
     "clang-tools-extra": {"libc"},
     "mlir": {"flang"},
+    # Test everything if ci scripts are changed.
+    # FIXME: Figure out what is missing and add here.
+    ".ci": {"llvm", "clang", "lld", "lldb"},
 }
 
 DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
@@ -130,12 +133,11 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
 def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     projects_to_test = set()
     for modified_project in modified_projects:
-        # Skip all projects where we cannot run tests.
-        if modified_project not in PROJECT_CHECK_TARGETS:
-            continue
         if modified_project in RUNTIMES:
             continue
-        projects_to_test.add(modified_project)
+        # Skip all projects where we cannot run tests.
+        if modified_project in PROJECT_CHECK_TARGETS:
+            projects_to_test.add(modified_project)
         if modified_project not in DEPENDENTS_TO_TEST:
             continue
         for dependent_project in DEPENDENTS_TO_TEST[modified_project]:
diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py
index e787fd8133c86..1ab1c82498932 100644
--- a/.ci/compute_projects_test.py
+++ b/.ci/compute_projects_test.py
@@ -188,6 +188,19 @@ def test_exclude_gn(self):
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
 
+     def test_ci(self):
+        env_variables = compute_projects.get_env_variables(
+            [".ci/compute_projects.py"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"],
+                         "clang;lld;llvm;lldb")
+        self.assertEqual(env_variables["project_check_targets"], "check-clang
+                         check-lld check-llvm check-lldb")
+        self.assertEqual(env_variables["runtimes_to_build"],
+                         "libcxx;libcxxabi;libunwind")
+        self.assertEqual(env_variables["runtimes_check_targets"], "check-cxx
+                         check-cxxabi check-unwind")
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 6461c9d40ad59..f81a14cca6cb3 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -18,7 +18,6 @@ set -o pipefail
 
 MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
 BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
-INSTALL_DIR="${BUILD_DIR}/install"
 rm -rf "${BUILD_DIR}"
 
 ccache --zero-stats
@@ -28,10 +27,14 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
   ccache --clear
 fi
 
+mkdir -p artifacts/reproducers
+
+# Make sure any clang reproducers will end up as artifacts.
+export CLANG_CRASH_DIAGNOSTICS_DIR=`realpath artifacts/reproducers`
+
 function at-exit {
   retcode=$?
 
-  mkdir -p artifacts
   ccache --print-stats > artifacts/ccache_stats.txt
   cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
 
@@ -50,17 +53,28 @@ trap at-exit EXIT
 
 projects="${1}"
 targets="${2}"
+runtimes="${3}"
 
 lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
 
 echo "--- cmake"
+
 export PIP_BREAK_SYSTEM_PACKAGES=1
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
+
+# Set the system llvm-symbolizer as preferred.
+export LLVM_SYMBOLIZER_PATH=`which llvm-symbolizer`
+[[ ! -f "${LLVM_SYMBOLIZER_PATH}" ]] && echo "llvm-symbolizer not found!"
+
+# Set up all runtimes either way. libcxx is a dependency of LLDB.
+# If it ends up being unused, not much harm.
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
+      -D LLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
       -G Ninja \
+      -D CMAKE_PREFIX_PATH="${HOME}/.local" \
       -D CMAKE_BUILD_TYPE=Release \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
@@ -69,69 +83,47 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_LLD=ON \
       -D CMAKE_CXX_FLAGS=-gmlt \
       -D LLVM_CCACHE_BUILD=ON \
+      -D LIBCXX_CXX_ABI=libcxxabi \
       -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
+      -D LLDB_ENABLE_PYTHON=ON \
+      -D LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=ON
 
 echo "--- ninja"
+
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
 
-runtimes="${3}"
 runtime_targets="${4}"
 
-# Compiling runtimes with just-built Clang and running their tests
-# as an additional testing for Clang.
+# Run runtimes tests.
+# We don't need to do a clean separate build of runtimes, because runtimes
+# will be built against just built clang, and because LIBCXX_TEST_PARAMS
+# and LIBCXXABI_TEST_PARAMS only affect lit configuration, which successfully
+# propagates without a clean build. Other that those two variables, builds
+# are supposed to be the same.
 if [[ "${runtimes}" != "" ]]; then
   if [[ "${runtime_targets}" == "" ]]; then
     echo "Runtimes to build are specified, but targets are not."
     exit 1
   fi
 
-  echo "--- ninja install-clang"
-
-  ninja -C ${BUILD_DIR} install-clang install-clang-resource-headers
-
-  RUNTIMES_BUILD_DIR="${MONOREPO_ROOT}/build-runtimes"
-  INSTALL_DIR="${BUILD_DIR}/install"
-  mkdir -p ${RUNTIMES_BUILD_DIR}
-
   echo "--- cmake runtimes C++26"
 
-  rm -rf "${RUNTIMES_BUILD_DIR}"
-  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
-      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
-      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
-      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
-      -D LIBCXX_CXX_ABI=libcxxabi \
-      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+  cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LIBCXX_TEST_PARAMS="std=c++26" \
-      -D LIBCXXABI_TEST_PARAMS="std=c++26" \
-      -D LLVM_LIT_ARGS="${lit_args}"
+      -D LIBCXXABI_TEST_PARAMS="std=c++26"
 
   echo "--- ninja runtimes C++26"
 
-  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
+  ninja -vC "${BUILD_DIR}" ${runtime_targets}
 
   echo "--- cmake runtimes clang modules"
 
-  # We don't need to do a clean build of runtimes, because LIBCXX_TEST_PARAMS
-  # and LIBCXXABI_TEST_PARAMS only affect lit configuration, which successfully
-  # propagates without a clean build. Other that those two variables, builds
-  # are supposed to be the same.
-
-  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
-      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
-      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
-      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
-      -D LIBCXX_CXX_ABI=libcxxabi \
-      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+  cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
-      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
-      -D LLVM_LIT_ARGS="${lit_args}"
+      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang"
 
   echo "--- ninja runtimes clang modules"
 
-  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
+  ninja -vC "${BUILD_DIR}" ${runtime_targets}
 fi
diff --git a/lldb/test/requirements.txt b/lldb/test/requirements.txt
index 708f2327a6a04..c187180697d0b 100644
--- a/lldb/test/requirements.txt
+++ b/lldb/test/requirements.txt
@@ -5,3 +5,5 @@ psutil>=5.9.4
 # See llvm.org/pr22274.
 pexpect>=4.9.0; sys_platform != 'win32'
 packaging
+# Required for python tests
+swig

From 289baf1f42c8b5773271b611cd235d4ab94bb4e8 Mon Sep 17 00:00:00 2001
From: Nathan Ridge <zeratul976@hotmail.com>
Date: Tue, 15 Apr 2025 12:04:35 -0400
Subject: [PATCH 005/710] [clang][AST] Handle implicit first argument in
 CallExpr::getBeginLoc() (#135757)

Fixes https://github.com/llvm/llvm-project/issues/135522
---
 clang/docs/ReleaseNotes.rst                | 2 ++
 clang/lib/AST/Expr.cpp                     | 7 +++++--
 clang/test/SemaCXX/cxx2b-deducing-this.cpp | 7 +++++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 2d2606085998c..c106148855436 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -411,6 +411,8 @@ Bug Fixes in This Version
 - ``#embed`` directive now diagnoses use of a non-character file (device file)
   such as ``/dev/urandom`` as an error. This restriction may be relaxed in the
   future. See (#GH126629).
+- Fixed a clang 20 regression where diagnostics attached to some calls to member functions
+  using C++23 "deducing this" did not have a diagnostic location (#GH135522)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 5fab2c73f214b..59c0e47c7c195 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1652,8 +1652,11 @@ SourceLocation CallExpr::getBeginLoc() const {
   if (!isTypeDependent()) {
     if (const auto *Method =
             dyn_cast_if_present<const CXXMethodDecl>(getCalleeDecl());
-        Method && Method->isExplicitObjectMemberFunction())
-      return getArg(0)->getBeginLoc();
+        Method && Method->isExplicitObjectMemberFunction()) {
+      if (auto FirstArgLoc = getArg(0)->getBeginLoc(); FirstArgLoc.isValid()) {
+        return FirstArgLoc;
+      }
+    }
   }
 
   SourceLocation begin = getCallee()->getBeginLoc();
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index 6f17ce7275456..7e392213710a4 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -1134,3 +1134,10 @@ struct S {
 static_assert((S{} << 11) == a);
 // expected-error@-1 {{use of undeclared identifier 'a'}}
 }
+
+namespace GH135522 {
+struct S {
+  auto f(this auto) -> S;
+  bool g() { return f(); } // expected-error {{no viable conversion from returned value of type 'S' to function return type 'bool'}}
+};
+}

From d41e517748e2dbb51e27bed217f3dd7a4c4fb86f Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen@sifive.com>
Date: Wed, 16 Apr 2025 00:08:59 +0800
Subject: [PATCH 006/710] [SLP] Make getSameOpcode support interchangeable
 instructions. (#135797)

We use the term "interchangeable instructions" to refer to different
operators that have the same meaning (e.g., `add x, 0` is equivalent to
`mul x, 1`).
Non-constant values are not supported, as they may incur high costs with
little benefit.

---------

Co-authored-by: Alexey Bataev <a.bataev@gmx.com>
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 462 ++++++++++++++++--
 .../SLPVectorizer/AArch64/vec3-base.ll        |   8 +-
 ...reversed-strided-node-with-external-ptr.ll |   7 +-
 .../SLPVectorizer/RISCV/vec3-base.ll          |   8 +-
 .../X86/BinOpSameOpcodeHelper.ll              |  36 ++
 .../SLPVectorizer/X86/barriercall.ll          |   4 +-
 .../X86/bottom-to-top-reorder.ll              |  11 +-
 .../buildvector-postpone-for-dependency.ll    |   8 +-
 .../SLPVectorizer/X86/bv-shuffle-mask.ll      |   4 +-
 .../X86/extract-scalar-from-undef.ll          |  28 +-
 .../SLPVectorizer/X86/extractcost.ll          |   4 +-
 ...gathered-delayed-nodes-with-reused-user.ll |  34 +-
 .../X86/minbitwidth-drop-wrapping-flags.ll    |   4 +-
 .../X86/multi-extracts-bv-combined.ll         |   6 +-
 .../non-scheduled-inst-reused-as-last-inst.ll |  44 +-
 .../SLPVectorizer/X86/propagate_ir_flags.ll   |  12 +-
 .../reduced-val-vectorized-in-transform.ll    |   6 +-
 .../X86/reorder_diamond_match.ll              |   4 +-
 .../X86/shuffle-mask-emission.ll              |   8 +-
 .../Transforms/SLPVectorizer/X86/vec3-base.ll |  19 +-
 .../X86/vect_copyable_in_binops.ll            |   8 +-
 .../alternate-opcode-sindle-bv.ll             |  35 +-
 .../Transforms/SLPVectorizer/bbi-106161.ll    |  19 +
 .../Transforms/SLPVectorizer/isOpcodeOrAlt.ll |  61 +++
 .../resized-alt-shuffle-after-minbw.ll        |   4 +-
 .../SLPVectorizer/shuffle-mask-resized.ll     |   4 +-
 26 files changed, 647 insertions(+), 201 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/BinOpSameOpcodeHelper.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/bbi-106161.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/isOpcodeOrAlt.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cc775e4b260dc..253933a2438cd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -599,6 +599,28 @@ static std::optional<unsigned> getElementIndex(const Value *Inst,
   return Index;
 }
 
+/// \returns true if all of the values in \p VL use the same opcode.
+/// For comparison instructions, also checks if predicates match.
+/// PoisonValues are considered matching.
+/// Interchangeable instructions are not considered.
+static bool allSameOpcode(ArrayRef<Value *> VL) {
+  auto *It = find_if(VL, IsaPred<Instruction>);
+  if (It == VL.end())
+    return true;
+  Instruction *MainOp = cast<Instruction>(*It);
+  unsigned Opcode = MainOp->getOpcode();
+  bool IsCmpOp = isa<CmpInst>(MainOp);
+  CmpInst::Predicate BasePred = IsCmpOp ? cast<CmpInst>(MainOp)->getPredicate()
+                                        : CmpInst::BAD_ICMP_PREDICATE;
+  return std::all_of(It, VL.end(), [&](Value *V) {
+    if (auto *CI = dyn_cast<CmpInst>(V))
+      return BasePred == CI->getPredicate();
+    if (auto *I = dyn_cast<Instruction>(V))
+      return I->getOpcode() == Opcode;
+    return isa<PoisonValue>(V);
+  });
+}
+
 namespace {
 /// Specifies the way the mask should be analyzed for undefs/poisonous elements
 /// in the shuffle mask.
@@ -814,6 +836,272 @@ static std::optional<unsigned> getExtractIndex(const Instruction *E) {
 }
 
 namespace {
+/// \returns true if \p Opcode is allowed as part of the main/alternate
+/// instruction for SLP vectorization.
+///
+/// Example of unsupported opcode is SDIV that can potentially cause UB if the
+/// "shuffled out" lane would result in division by zero.
+bool isValidForAlternation(unsigned Opcode) {
+  return !Instruction::isIntDivRem(Opcode);
+}
+
+/// Helper class that determines VL can use the same opcode.
+/// Alternate instruction is supported. In addition, it supports interchangeable
+/// instruction. An interchangeable instruction is an instruction that can be
+/// converted to another instruction with same semantics. For example, x << 1 is
+/// equal to x * 2. x * 1 is equal to x | 0.
+class BinOpSameOpcodeHelper {
+  using MaskType = std::uint_fast16_t;
+  /// Sort SupportedOp because it is used by binary_search.
+  constexpr static std::initializer_list<unsigned> SupportedOp = {
+      Instruction::Add,  Instruction::Sub, Instruction::Mul, Instruction::Shl,
+      Instruction::AShr, Instruction::And, Instruction::Or,  Instruction::Xor};
+  enum : MaskType {
+    ShlBIT = 0b1,
+    AShrBIT = 0b10,
+    MulBIT = 0b100,
+    AddBIT = 0b1000,
+    SubBIT = 0b10000,
+    AndBIT = 0b100000,
+    OrBIT = 0b1000000,
+    XorBIT = 0b10000000,
+    MainOpBIT = 0b100000000,
+    LLVM_MARK_AS_BITMASK_ENUM(MainOpBIT)
+  };
+  /// Return a non-nullptr if either operand of I is a ConstantInt.
+  /// The second return value represents the operand position. We check the
+  /// right-hand side first (1). If the right hand side is not a ConstantInt and
+  /// the instruction is neither Sub, Shl, nor AShr, we then check the left hand
+  /// side (0).
+  static std::pair<ConstantInt *, unsigned>
+  isBinOpWithConstantInt(const Instruction *I) {
+    unsigned Opcode = I->getOpcode();
+    assert(binary_search(SupportedOp, Opcode) && "Unsupported opcode.");
+    (void)SupportedOp;
+    auto *BinOp = cast<BinaryOperator>(I);
+    if (auto *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)))
+      return {CI, 1};
+    if (Opcode == Instruction::Sub || Opcode == Instruction::Shl ||
+        Opcode == Instruction::AShr)
+      return {nullptr, 0};
+    if (auto *CI = dyn_cast<ConstantInt>(BinOp->getOperand(0)))
+      return {CI, 0};
+    return {nullptr, 0};
+  }
+  struct InterchangeableInfo {
+    const Instruction *I = nullptr;
+    /// The bit it sets represents whether MainOp can be converted to.
+    MaskType Mask = MainOpBIT | XorBIT | OrBIT | AndBIT | SubBIT | AddBIT |
+                    MulBIT | AShrBIT | ShlBIT;
+    /// We cannot create an interchangeable instruction that does not exist in
+    /// VL. For example, VL [x + 0, y * 1] can be converted to [x << 0, y << 0],
+    /// but << does not exist in VL. In the end, we convert VL to [x * 1, y *
+    /// 1]. SeenBefore is used to know what operations have been seen before.
+    MaskType SeenBefore = 0;
+    InterchangeableInfo(const Instruction *I) : I(I) {}
+    /// Return false allows BinOpSameOpcodeHelper to find an alternate
+    /// instruction. Directly setting the mask will destroy the mask state,
+    /// preventing us from determining which instruction it should convert to.
+    bool trySet(MaskType OpcodeInMaskForm, MaskType InterchangeableMask) {
+      if (Mask & InterchangeableMask) {
+        SeenBefore |= OpcodeInMaskForm;
+        Mask &= InterchangeableMask;
+        return true;
+      }
+      return false;
+    }
+    bool equal(unsigned Opcode) {
+      if (Opcode == I->getOpcode())
+        return trySet(MainOpBIT, MainOpBIT);
+      return false;
+    }
+    unsigned getOpcode() const {
+      MaskType Candidate = Mask & SeenBefore;
+      if (Candidate & MainOpBIT)
+        return I->getOpcode();
+      if (Candidate & ShlBIT)
+        return Instruction::Shl;
+      if (Candidate & AShrBIT)
+        return Instruction::AShr;
+      if (Candidate & MulBIT)
+        return Instruction::Mul;
+      if (Candidate & AddBIT)
+        return Instruction::Add;
+      if (Candidate & SubBIT)
+        return Instruction::Sub;
+      if (Candidate & AndBIT)
+        return Instruction::And;
+      if (Candidate & OrBIT)
+        return Instruction::Or;
+      if (Candidate & XorBIT)
+        return Instruction::Xor;
+      llvm_unreachable("Cannot find interchangeable instruction.");
+    }
+    SmallVector<Value *> getOperand(const Instruction *To) const {
+      unsigned ToOpcode = To->getOpcode();
+      unsigned FromOpcode = I->getOpcode();
+      if (FromOpcode == ToOpcode)
+        return SmallVector<Value *>(I->operands());
+      assert(binary_search(SupportedOp, ToOpcode) && "Unsupported opcode.");
+      auto [CI, Pos] = isBinOpWithConstantInt(I);
+      const APInt &FromCIValue = CI->getValue();
+      unsigned FromCIValueBitWidth = FromCIValue.getBitWidth();
+      APInt ToCIValue;
+      switch (FromOpcode) {
+      case Instruction::Shl:
+        if (ToOpcode == Instruction::Mul) {
+          ToCIValue = APInt::getOneBitSet(FromCIValueBitWidth,
+                                          FromCIValue.getZExtValue());
+        } else {
+          assert(FromCIValue.isZero() && "Cannot convert the instruction.");
+          ToCIValue = ToOpcode == Instruction::And
+                          ? APInt::getAllOnes(FromCIValueBitWidth)
+                          : APInt::getZero(FromCIValueBitWidth);
+        }
+        break;
+      case Instruction::Mul:
+        assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
+        if (ToOpcode == Instruction::Shl) {
+          ToCIValue = APInt(FromCIValueBitWidth, FromCIValue.logBase2());
+        } else {
+          assert(FromCIValue.isOne() && "Cannot convert the instruction.");
+          ToCIValue = ToOpcode == Instruction::And
+                          ? APInt::getAllOnes(FromCIValueBitWidth)
+                          : APInt::getZero(FromCIValueBitWidth);
+        }
+        break;
+      case Instruction::Add:
+      case Instruction::Sub:
+        if (FromCIValue.isZero()) {
+          ToCIValue = APInt::getZero(FromCIValueBitWidth);
+        } else {
+          assert(is_contained({Instruction::Add, Instruction::Sub}, ToOpcode) &&
+                 "Cannot convert the instruction.");
+          ToCIValue = FromCIValue;
+          ToCIValue.negate();
+        }
+        break;
+      case Instruction::And:
+        assert(FromCIValue.isAllOnes() && "Cannot convert the instruction.");
+        ToCIValue = ToOpcode == Instruction::Mul
+                        ? APInt::getOneBitSet(FromCIValueBitWidth, 0)
+                        : APInt::getZero(FromCIValueBitWidth);
+        break;
+      default:
+        assert(FromCIValue.isZero() && "Cannot convert the instruction.");
+        ToCIValue = APInt::getZero(FromCIValueBitWidth);
+        break;
+      }
+      Value *LHS = I->getOperand(1 - Pos);
+      Constant *RHS =
+          ConstantInt::get(I->getOperand(Pos)->getType(), ToCIValue);
+      if (Pos == 1)
+        return SmallVector<Value *>({LHS, RHS});
+      return SmallVector<Value *>({RHS, LHS});
+    }
+  };
+  InterchangeableInfo MainOp;
+  InterchangeableInfo AltOp;
+  bool isValidForAlternation(const Instruction *I) const {
+    return ::isValidForAlternation(MainOp.I->getOpcode()) &&
+           ::isValidForAlternation(I->getOpcode());
+  }
+  bool initializeAltOp(const Instruction *I) {
+    if (AltOp.I)
+      return true;
+    if (!isValidForAlternation(I))
+      return false;
+    AltOp.I = I;
+    return true;
+  }
+
+public:
+  BinOpSameOpcodeHelper(const Instruction *MainOp,
+                        const Instruction *AltOp = nullptr)
+      : MainOp(MainOp), AltOp(AltOp) {
+    assert(is_sorted(SupportedOp) && "SupportedOp is not sorted.");
+  }
+  bool add(const Instruction *I) {
+    assert(isa<BinaryOperator>(I) &&
+           "BinOpSameOpcodeHelper only accepts BinaryOperator.");
+    unsigned Opcode = I->getOpcode();
+    MaskType OpcodeInMaskForm;
+    // Prefer Shl, AShr, Mul, Add, Sub, And, Or and Xor over MainOp.
+    switch (Opcode) {
+    case Instruction::Shl:
+      OpcodeInMaskForm = ShlBIT;
+      break;
+    case Instruction::AShr:
+      OpcodeInMaskForm = AShrBIT;
+      break;
+    case Instruction::Mul:
+      OpcodeInMaskForm = MulBIT;
+      break;
+    case Instruction::Add:
+      OpcodeInMaskForm = AddBIT;
+      break;
+    case Instruction::Sub:
+      OpcodeInMaskForm = SubBIT;
+      break;
+    case Instruction::And:
+      OpcodeInMaskForm = AndBIT;
+      break;
+    case Instruction::Or:
+      OpcodeInMaskForm = OrBIT;
+      break;
+    case Instruction::Xor:
+      OpcodeInMaskForm = XorBIT;
+      break;
+    default:
+      return MainOp.equal(Opcode) ||
+             (initializeAltOp(I) && AltOp.equal(Opcode));
+    }
+    MaskType InterchangeableMask = OpcodeInMaskForm;
+    ConstantInt *CI = isBinOpWithConstantInt(I).first;
+    if (CI) {
+      constexpr MaskType CanBeAll =
+          XorBIT | OrBIT | AndBIT | SubBIT | AddBIT | MulBIT | AShrBIT | ShlBIT;
+      const APInt &CIValue = CI->getValue();
+      switch (Opcode) {
+      case Instruction::Shl:
+        if (CIValue.ult(CIValue.getBitWidth()))
+          InterchangeableMask = CIValue.isZero() ? CanBeAll : MulBIT | ShlBIT;
+        break;
+      case Instruction::Mul:
+        if (CIValue.isOne()) {
+          InterchangeableMask = CanBeAll;
+          break;
+        }
+        if (CIValue.isPowerOf2())
+          InterchangeableMask = MulBIT | ShlBIT;
+        break;
+      case Instruction::Add:
+      case Instruction::Sub:
+        InterchangeableMask = CIValue.isZero() ? CanBeAll : SubBIT | AddBIT;
+        break;
+      case Instruction::And:
+        if (CIValue.isAllOnes())
+          InterchangeableMask = CanBeAll;
+        break;
+      default:
+        if (CIValue.isZero())
+          InterchangeableMask = CanBeAll;
+        break;
+      }
+    }
+    return MainOp.trySet(OpcodeInMaskForm, InterchangeableMask) ||
+           (initializeAltOp(I) &&
+            AltOp.trySet(OpcodeInMaskForm, InterchangeableMask));
+  }
+  unsigned getMainOpcode() const { return MainOp.getOpcode(); }
+  bool hasAltOp() const { return AltOp.I; }
+  unsigned getAltOpcode() const {
+    return hasAltOp() ? AltOp.getOpcode() : getMainOpcode();
+  }
+  SmallVector<Value *> getOperand(const Instruction *I) const {
+    return MainOp.getOperand(I);
+  }
+};
 
 /// Main data required for vectorization of instructions.
 class InstructionsState {
@@ -861,9 +1149,27 @@ class InstructionsState {
   /// Some of the instructions in the list have alternate opcodes.
   bool isAltShuffle() const { return getMainOp() != getAltOp(); }
 
-  bool isOpcodeOrAlt(Instruction *I) const {
-    unsigned CheckedOpcode = I->getOpcode();
-    return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
+  /// Checks if the instruction matches either the main or alternate opcode.
+  /// \returns
+  /// - MainOp if \param I matches MainOp's opcode directly or can be converted
+  /// to it
+  /// - AltOp if \param I matches AltOp's opcode directly or can be converted to
+  /// it
+  /// - nullptr if \param I cannot be matched or converted to either opcode
+  Instruction *getMatchingMainOpOrAltOp(Instruction *I) const {
+    assert(MainOp && "MainOp cannot be nullptr.");
+    if (I->getOpcode() == MainOp->getOpcode())
+      return MainOp;
+    // Prefer AltOp instead of interchangeable instruction of MainOp.
+    assert(AltOp && "AltOp cannot be nullptr.");
+    if (I->getOpcode() == AltOp->getOpcode())
+      return AltOp;
+    if (!I->isBinaryOp())
+      return nullptr;
+    BinOpSameOpcodeHelper Converter(MainOp);
+    if (Converter.add(I) && Converter.add(MainOp) && !Converter.hasAltOp())
+      return MainOp;
+    return AltOp;
   }
 
   /// Checks if main/alt instructions are shift operations.
@@ -913,23 +1219,41 @@ class InstructionsState {
   static InstructionsState invalid() { return {nullptr, nullptr}; }
 };
 
-} // end anonymous namespace
-
-/// \returns true if \p Opcode is allowed as part of the main/alternate
-/// instruction for SLP vectorization.
-///
-/// Example of unsupported opcode is SDIV that can potentially cause UB if the
-/// "shuffled out" lane would result in division by zero.
-static bool isValidForAlternation(unsigned Opcode) {
-  if (Instruction::isIntDivRem(Opcode))
-    return false;
-
-  return true;
+std::pair<Instruction *, SmallVector<Value *>>
+convertTo(Instruction *I, const InstructionsState &S) {
+  Instruction *SelectedOp = S.getMatchingMainOpOrAltOp(I);
+  assert(SelectedOp && "Cannot convert the instruction.");
+  if (I->isBinaryOp()) {
+    BinOpSameOpcodeHelper Converter(I);
+    return std::make_pair(SelectedOp, Converter.getOperand(SelectedOp));
+  }
+  return std::make_pair(SelectedOp, SmallVector<Value *>(I->operands()));
 }
 
+} // end anonymous namespace
+
 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
                                        const TargetLibraryInfo &TLI);
 
+/// Find an instruction with a specific opcode in VL.
+/// \param VL Array of values to search through. Must contain only Instructions
+///           and PoisonValues.
+/// \param Opcode The instruction opcode to search for
+/// \returns
+/// - The first instruction found with matching opcode
+/// - nullptr if no matching instruction is found
+Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL, unsigned Opcode) {
+  for (Value *V : VL) {
+    if (isa<PoisonValue>(V))
+      continue;
+    assert(isa<Instruction>(V) && "Only accepts PoisonValue and Instruction.");
+    auto *Inst = cast<Instruction>(V);
+    if (Inst->getOpcode() == Opcode)
+      return Inst;
+  }
+  return nullptr;
+}
+
 /// Checks if the provided operands of 2 cmp instructions are compatible, i.e.
 /// compatible instructions or constants, or just some other regular values.
 static bool areCompatibleCmpOps(Value *BaseOp0, Value *BaseOp1, Value *Op0,
@@ -993,6 +1317,7 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
   unsigned Opcode = MainOp->getOpcode();
   unsigned AltOpcode = Opcode;
 
+  BinOpSameOpcodeHelper BinOpHelper(MainOp);
   bool SwappedPredsCompatible = IsCmpOp && [&]() {
     SetVector<unsigned> UniquePreds, UniqueNonSwappedPreds;
     UniquePreds.insert(BasePred);
@@ -1039,14 +1364,8 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
       return InstructionsState::invalid();
     unsigned InstOpcode = I->getOpcode();
     if (IsBinOp && isa<BinaryOperator>(I)) {
-      if (InstOpcode == Opcode || InstOpcode == AltOpcode)
+      if (BinOpHelper.add(I))
         continue;
-      if (Opcode == AltOpcode && isValidForAlternation(InstOpcode) &&
-          isValidForAlternation(Opcode)) {
-        AltOpcode = InstOpcode;
-        AltOp = I;
-        continue;
-      }
     } else if (IsCastOp && isa<CastInst>(I)) {
       Value *Op0 = MainOp->getOperand(0);
       Type *Ty0 = Op0->getType();
@@ -1147,7 +1466,22 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
     return InstructionsState::invalid();
   }
 
-  return InstructionsState(MainOp, AltOp);
+  if (IsBinOp) {
+    MainOp = findInstructionWithOpcode(VL, BinOpHelper.getMainOpcode());
+    assert(MainOp && "Cannot find MainOp with Opcode from BinOpHelper.");
+    AltOp = findInstructionWithOpcode(VL, BinOpHelper.getAltOpcode());
+    assert(MainOp && "Cannot find AltOp with Opcode from BinOpHelper.");
+  }
+  assert((MainOp == AltOp || !allSameOpcode(VL)) &&
+         "Incorrect implementation of allSameOpcode.");
+  InstructionsState S(MainOp, AltOp);
+  assert(all_of(VL,
+                [&](Value *V) {
+                  return isa<PoisonValue>(V) ||
+                         S.getMatchingMainOpOrAltOp(cast<Instruction>(V));
+                }) &&
+         "Invalid InstructionsState.");
+  return S;
 }
 
 /// \returns true if all of the values in \p VL have the same type or false
@@ -2560,11 +2894,11 @@ class BoUpSLP {
         // Since operand reordering is performed on groups of commutative
         // operations or alternating sequences (e.g., +, -), we can safely tell
         // the inverse operations by checking commutativity.
-        bool IsInverseOperation = !isCommutative(cast<Instruction>(V));
+        auto [SelectedOp, Ops] = convertTo(cast<Instruction>(VL[Lane]), S);
+        bool IsInverseOperation = !isCommutative(SelectedOp);
         for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
           bool APO = (OpIdx == 0) ? false : IsInverseOperation;
-          OpsVec[OpIdx][Lane] = {cast<Instruction>(V)->getOperand(OpIdx), APO,
-                                 false};
+          OpsVec[OpIdx][Lane] = {Ops[OpIdx], APO, false};
         }
       }
     }
@@ -3542,14 +3876,16 @@ class BoUpSLP {
     /// Some of the instructions in the list have alternate opcodes.
     bool isAltShuffle() const { return S.isAltShuffle(); }
 
-    bool isOpcodeOrAlt(Instruction *I) const { return S.isOpcodeOrAlt(I); }
+    Instruction *getMatchingMainOpOrAltOp(Instruction *I) const {
+      return S.getMatchingMainOpOrAltOp(I);
+    }
 
     /// Chooses the correct key for scheduling data. If \p Op has the same (or
     /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is
     /// \p OpValue.
     Value *isOneOf(Value *Op) const {
       auto *I = dyn_cast<Instruction>(Op);
-      if (I && isOpcodeOrAlt(I))
+      if (I && getMatchingMainOpOrAltOp(I))
         return Op;
       return S.getMainOp();
     }
@@ -8428,11 +8764,15 @@ static std::pair<size_t, size_t> generateKeySubkey(
   return std::make_pair(Key, SubKey);
 }
 
+/// Checks if the specified instruction \p I is an main operation for the given
+/// \p MainOp and \p AltOp instructions.
+static bool isMainInstruction(Instruction *I, Instruction *MainOp,
+                              Instruction *AltOp, const TargetLibraryInfo &TLI);
+
 /// Checks if the specified instruction \p I is an alternate operation for
 /// the given \p MainOp and \p AltOp instructions.
-static bool isAlternateInstruction(const Instruction *I,
-                                   const Instruction *MainOp,
-                                   const Instruction *AltOp,
+static bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
+                                   Instruction *AltOp,
                                    const TargetLibraryInfo &TLI);
 
 bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
@@ -9245,7 +9585,8 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
       continue;
     }
     if ((LocalState.getAltOpcode() != LocalState.getOpcode() &&
-         I->getOpcode() == LocalState.getOpcode()) ||
+         isMainInstruction(I, LocalState.getMainOp(), LocalState.getAltOp(),
+                           *TLI)) ||
         (LocalState.getAltOpcode() == LocalState.getOpcode() &&
          !isAlternateInstruction(I, LocalState.getMainOp(),
                                  LocalState.getAltOp(), *TLI))) {
@@ -10344,9 +10685,14 @@ void BoUpSLP::TreeEntry::buildAltOpShuffleMask(
   }
 }
 
-static bool isAlternateInstruction(const Instruction *I,
-                                   const Instruction *MainOp,
-                                   const Instruction *AltOp,
+static bool isMainInstruction(Instruction *I, Instruction *MainOp,
+                              Instruction *AltOp,
+                              const TargetLibraryInfo &TLI) {
+  return InstructionsState(MainOp, AltOp).getMatchingMainOpOrAltOp(I) == MainOp;
+}
+
+static bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
+                                   Instruction *AltOp,
                                    const TargetLibraryInfo &TLI) {
   if (auto *MainCI = dyn_cast<CmpInst>(MainOp)) {
     auto *AltCI = cast<CmpInst>(AltOp);
@@ -10366,7 +10712,7 @@ static bool isAlternateInstruction(const Instruction *I,
            "their swap.");
     return MainP != P && MainP != SwappedP;
   }
-  return I->getOpcode() == AltOp->getOpcode();
+  return InstructionsState(MainOp, AltOp).getMatchingMainOpOrAltOp(I) == AltOp;
 }
 
 TTI::OperandValueInfo BoUpSLP::getOperandInfo(ArrayRef<Value *> Ops) {
@@ -11129,7 +11475,9 @@ void BoUpSLP::transformNodes() {
       // same opcode and same parent block or all constants.
       if (VL.size() <= 2 || LoadEntriesToVectorize.contains(Idx) ||
           !(!E.hasState() || E.getOpcode() == Instruction::Load ||
-            E.isAltShuffle() || !allSameBlock(VL)) ||
+            // We use allSameOpcode instead of isAltShuffle because we don't
+            // want to use interchangeable instruction here.
+            !allSameOpcode(VL) || !allSameBlock(VL)) ||
           allConstant(VL) || isSplat(VL))
         continue;
       if (ForceLoadGather && E.hasState() && E.getOpcode() == Instruction::Load)
@@ -11174,7 +11522,7 @@ void BoUpSLP::transformNodes() {
             if (IsSplat)
               continue;
             InstructionsState S = getSameOpcode(Slice, *TLI);
-            if (!S || S.isAltShuffle() || !allSameBlock(Slice) ||
+            if (!S || !allSameOpcode(Slice) || !allSameBlock(Slice) ||
                 (S.getOpcode() == Instruction::Load &&
                  areKnownNonVectorizableLoads(Slice)) ||
                 (S.getOpcode() != Instruction::Load &&
@@ -12974,14 +13322,22 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       if (isa<PoisonValue>(UniqueValues[Idx]))
         return InstructionCost(TTI::TCC_Free);
 
-      auto *VI = cast<Instruction>(UniqueValues[Idx]);
-      unsigned OpIdx = isa<UnaryOperator>(VI) ? 0 : 1;
-      TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(VI->getOperand(0));
-      TTI::OperandValueInfo Op2Info =
-          TTI::getOperandInfo(VI->getOperand(OpIdx));
-      SmallVector<const Value *> Operands(VI->operand_values());
+      // We cannot retrieve the operand from UniqueValues[Idx] because an
+      // interchangeable instruction may be used. The order and the actual
+      // operand might differ from what is retrieved from UniqueValues[Idx].
+      Value *Op1 = E->getOperand(0)[Idx];
+      Value *Op2;
+      SmallVector<const Value *, 2> Operands(1, Op1);
+      if (isa<UnaryOperator>(UniqueValues[Idx])) {
+        Op2 = Op1;
+      } else {
+        Op2 = E->getOperand(1)[Idx];
+        Operands.push_back(Op2);
+      }
+      TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(Op1);
+      TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(Op2);
       return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind,
-                                         Op1Info, Op2Info, Operands, VI);
+                                         Op1Info, Op2Info, Operands);
     };
     auto GetVectorCost = [=](InstructionCost CommonCost) {
       if (ShuffleOrOp == Instruction::And && It != MinBWs.end()) {
@@ -13211,7 +13567,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
         return InstructionCost(TTI::TCC_Free);
 
       auto *VI = cast<Instruction>(UniqueValues[Idx]);
-      assert(E->isOpcodeOrAlt(VI) && "Unexpected main/alternate opcode");
+      assert(E->getMatchingMainOpOrAltOp(VI) &&
+             "Unexpected main/alternate opcode");
       (void)E;
       return TTI->getInstructionCost(VI, CostKind);
     };
@@ -13279,7 +13636,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       SmallVector<int> Mask;
       E->buildAltOpShuffleMask(
           [&](Instruction *I) {
-            assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
+            assert(E->getMatchingMainOpOrAltOp(I) &&
+                   "Unexpected main/alternate opcode");
             return isAlternateInstruction(I, E->getMainOp(), E->getAltOp(),
                                           *TLI);
           },
@@ -15441,7 +15799,8 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
                        !isa<GetElementPtrInst>(V))
                      return true;
                    auto *I = dyn_cast<Instruction>(V);
-                   return !I || !E->isOpcodeOrAlt(I) || I->getParent() == BB ||
+                   return !I || !E->getMatchingMainOpOrAltOp(I) ||
+                          I->getParent() == BB ||
                           isVectorLikeInstWithConstOps(I);
                  })) &&
          "Expected gathered loads or GEPs or instructions from same basic "
@@ -17585,7 +17944,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       Value *V = Builder.CreateBinOp(
           static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS,
           RHS);
-      propagateIRFlags(V, E->Scalars, VL0, It == MinBWs.end());
+      propagateIRFlags(V, E->Scalars, nullptr, It == MinBWs.end());
       if (auto *I = dyn_cast<Instruction>(V)) {
         V = ::propagateMetadata(I, E->Scalars);
         // Drop nuw flags for abs(sub(commutative), true).
@@ -18005,7 +18364,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         SmallVector<int> Mask;
         E->buildAltOpShuffleMask(
             [E, this](Instruction *I) {
-              assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
+              assert(E->getMatchingMainOpOrAltOp(I) &&
+                     "Unexpected main/alternate opcode");
               return isAlternateInstruction(I, E->getMainOp(), E->getAltOp(),
                                             *TLI);
             },
@@ -21796,7 +22156,7 @@ class HorizontalReduction {
         // Also check if the instruction was folded to constant/other value.
         auto *Inst = dyn_cast<Instruction>(RdxVal);
         if ((Inst && isVectorLikeInstWithConstOps(Inst) &&
-             (!S || !S.isOpcodeOrAlt(Inst))) ||
+             (!S || !S.getMatchingMainOpOrAltOp(Inst))) ||
             (S && !Inst))
           continue;
         Candidates.push_back(RdxVal);
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
index feb4ad865f314..d527d38adbee3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
@@ -314,10 +314,10 @@ define void @store_try_reorder(ptr %dst) {
 ;
 ; POW2-ONLY-LABEL: @store_try_reorder(
 ; POW2-ONLY-NEXT:  entry:
-; POW2-ONLY-NEXT:    [[ADD:%.*]] = add i32 0, 0
-; POW2-ONLY-NEXT:    store i32 [[ADD]], ptr [[DST:%.*]], align 4
-; POW2-ONLY-NEXT:    [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1
-; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4
+; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887:%.*]], align 4
+; POW2-ONLY-NEXT:    [[ADD216:%.*]] = sub i32 0, 0
+; POW2-ONLY-NEXT:    [[ARRAYIDX_I1891:%.*]] = getelementptr i32, ptr [[ARRAYIDX_I1887]], i64 2
+; POW2-ONLY-NEXT:    store i32 [[ADD216]], ptr [[ARRAYIDX_I1891]], align 4
 ; POW2-ONLY-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reversed-strided-node-with-external-ptr.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reversed-strided-node-with-external-ptr.ll
index fd3d4ab80b29c..ff897180cc9b7 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reversed-strided-node-with-external-ptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reversed-strided-node-with-external-ptr.ll
@@ -7,13 +7,12 @@ define void @test(ptr %a, i64 %0) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[A]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[TMP0]], i32 0
 ; CHECK-NEXT:    br label %[[BB:.*]]
 ; CHECK:       [[BB]]:
-; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[TMP4]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = or disjoint <2 x i64> [[TMP3]], <i64 1, i64 0>
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, <2 x ptr> [[TMP2]], <2 x i64> [[TMP5]]
-; CHECK-NEXT:    [[ARRAYIDX17_I28_1:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[ARRAYIDX17_I28_1:%.*]] = extractelement <2 x ptr> [[TMP6]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> [[TMP6]], i32 8, <2 x i1> splat (i1 true), <2 x double> poison)
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x double>, ptr [[A]], align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x double>, ptr [[A]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
index 7ab5e4d6cb787..481d586e6658a 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
@@ -324,10 +324,10 @@ define void @store_try_reorder(ptr %dst) {
 ;
 ; POW2-ONLY-LABEL: @store_try_reorder(
 ; POW2-ONLY-NEXT:  entry:
-; POW2-ONLY-NEXT:    [[ADD:%.*]] = add i32 0, 0
-; POW2-ONLY-NEXT:    store i32 [[ADD]], ptr [[DST:%.*]], align 4
-; POW2-ONLY-NEXT:    [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1
-; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4
+; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887:%.*]], align 4
+; POW2-ONLY-NEXT:    [[ADD216:%.*]] = sub i32 0, 0
+; POW2-ONLY-NEXT:    [[ARRAYIDX_I1891:%.*]] = getelementptr i32, ptr [[ARRAYIDX_I1887]], i64 2
+; POW2-ONLY-NEXT:    store i32 [[ADD216]], ptr [[ARRAYIDX_I1891]], align 4
 ; POW2-ONLY-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/BinOpSameOpcodeHelper.ll b/llvm/test/Transforms/SLPVectorizer/X86/BinOpSameOpcodeHelper.ll
new file mode 100644
index 0000000000000..6f27555aeb3f1
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/BinOpSameOpcodeHelper.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=slp-vectorizer -S %s | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i64 0, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 0, 1
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 0
+; CHECK-NEXT:    [[UMIN120:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP0]], i64 [[TMP2]])
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 0, 0
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 0
+; CHECK-NEXT:    [[UMIN122:%.*]] = call i64 @llvm.umin.i64(i64 [[UMIN120]], i64 [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 0, 1
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP5]], 0
+; CHECK-NEXT:    [[UMIN123:%.*]] = call i64 @llvm.umin.i64(i64 [[UMIN122]], i64 [[TMP6]])
+; CHECK-NEXT:    [[UMIN124:%.*]] = call i64 @llvm.umin.i64(i64 [[UMIN123]], i64 0)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = mul i64 0, 0
+  %1 = lshr i64 %0, 0
+  %2 = sub i64 0, 1
+  %3 = lshr i64 %2, 0
+  %umin120 = call i64 @llvm.umin.i64(i64 %1, i64 %3)
+  %4 = sub i64 0, 0
+  %5 = lshr i64 %4, 0
+  %umin122 = call i64 @llvm.umin.i64(i64 %umin120, i64 %5)
+  %6 = add i64 0, 1
+  %7 = lshr i64 %6, 0
+  %umin123 = call i64 @llvm.umin.i64(i64 %umin122, i64 %7)
+  %umin124 = call i64 @llvm.umin.i64(i64 %umin123, i64 0)
+  ret void
+}
+
+declare i64 @llvm.umin.i64(i64, i64)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll b/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
index f46a5d84a86cc..a39e602e2da71 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
@@ -10,9 +10,7 @@ define i32 @foo(ptr nocapture %A, i32 %n) {
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (...) @bar()
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i32 0
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 3, i32 10>
-; CHECK-NEXT:    [[TMP2:%.*]] = shl <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 3, i32 10>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 8, i32 10>
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], splat (i32 9)
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[A:%.*]], align 4
 ; CHECK-NEXT:    ret i32 undef
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
index 889f5a95c81d6..299677ca80b34 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
@@ -10,15 +10,10 @@ define void @test(ptr %0, ptr %1, ptr %2) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = sub <4 x i32> <i32 0, i32 0, i32 undef, i32 0>, [[TMP8]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub <4 x i32> [[TMP11]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP6]]
-; CHECK-NEXT:    [[TMP14:%.*]] = add <4 x i32> [[TMP13]], <i32 0, i32 0, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP15:%.*]] = sub <4 x i32> [[TMP13]], <i32 0, i32 0, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> [[TMP15]], <4 x i32> <i32 2, i32 0, i32 1, i32 7>
+; CHECK-NEXT:    [[TMP16:%.*]] = add <4 x i32> <i32 0, i32 0, i32 1, i32 0>, [[TMP13]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = add <4 x i32> [[TMP16]], zeroinitializer
-; CHECK-NEXT:    [[TMP18:%.*]] = sub <4 x i32> [[TMP16]], zeroinitializer
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP17]], <4 x i32> [[TMP18]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP20:%.*]] = add <4 x i32> [[TMP19]], zeroinitializer
-; CHECK-NEXT:    [[TMP21:%.*]] = sub <4 x i32> [[TMP19]], zeroinitializer
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP20]], <4 x i32> [[TMP21]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = add <4 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    store <4 x i32> [[TMP22]], ptr [[TMP2:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
index 43c42c1ea2bfb..03a89e54e4212 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
@@ -8,15 +8,13 @@ define void @test() {
 ; CHECK:       [[BB1:.*]]:
 ; CHECK-NEXT:    br label %[[BB2:.*]]
 ; CHECK:       [[BB2]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi <4 x i32> [ poison, %[[BB1]] ], [ [[TMP5:%.*]], %[[BB6]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <4 x i32> [ poison, %[[BB1]] ], [ [[TMP4:%.*]], %[[BB6]] ]
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[BB6]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP8:%.*]], %[[BB6]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 5, i32 4>
-; CHECK-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i32> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP5]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 5, i32 4>
+; CHECK-NEXT:    [[TMP4]] = mul <4 x i32> [[TMP3]], zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 poison>, <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[TMP8]] = mul <2 x i32> zeroinitializer, [[TMP7]]
 ; CHECK-NEXT:    br i1 false, label %[[BB2]], label %[[BB6]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
index 766916fe71f35..c4ddc5d63cc04 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
@@ -7,14 +7,12 @@ define i16 @test(i16 %v1, i16 %v2) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i16> <i16 0, i16 0, i16 0, i16 poison>, i16 [[V2]], i32 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> <i16 0, i16 0, i16 0, i16 poison>, i16 [[V1]], i32 3
-; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i16> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and <4 x i16> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 poison, i32 3>
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i16> [[TMP5]], i16 [[V1]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i16> [[TMP6]], <2 x i16> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i16> [[TMP7]], zeroinitializer
-; CHECK-NEXT:    [[TMP9:%.*]] = and <4 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = and <4 x i16> [[TMP3]], zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = and <4 x i16> [[TMP9]], zeroinitializer
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <4 x i16> [[TMP10]], zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i1> [[TMP11]], zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
index 1c62e57edfc46..514d5f974cb16 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
@@ -4,17 +4,13 @@
 define i64 @foo(i32 %tmp7) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[TMP5:%.*]], i32 2
-; CHECK-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 undef, 0
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 undef, i32 0>, i32 [[TMP24]], i32 4
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> [[TMP0]], i32 0, i32 5
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 undef, i32 poison, i32 undef>, i32 [[TMP24]], i32 6
-; CHECK-NEXT:    [[TMP12:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> poison, <4 x i32> [[TMP3]], i64 0)
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP12]], <8 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 poison, i32 2, i32 3, i32 poison, i32 14, i32 poison>
-; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <8 x i32> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 4, i32 5, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison>, i32 [[TMP8:%.*]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP0]], <i32 0, i32 0, i32 poison, i32 0, i32 0, i32 poison, i32 0, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 0>, <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 14, i32 poison, i32 poison, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x i32> [[TMP2]], i32 0, i32 5
+; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw <8 x i32> [[TMP13]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <8 x i32> [[TMP13]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP6]], <8 x i32> <i32 8, i32 1, i32 2, i32 3, i32 12, i32 13, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP8:%.*]] = add <8 x i32> zeroinitializer, [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i32> [[TMP8]], zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP9]])
@@ -29,7 +25,7 @@ bb:
   %tmp4 = xor i32 %tmp3, 0
   %tmp6 = sub i32 0, 0
   %tmp8 = sub i32 %tmp7, 0
-  %tmp9 = sub nsw i32 0, undef
+  %tmp9 = sub nsw i32 0, poison
   %tmp10 = add nsw i32 0, %tmp6
   %tmp11 = sub nsw i32 0, %tmp8
   %tmp12 = add i32 0, %tmp10
@@ -44,10 +40,10 @@ bb:
   %tmp21 = add i32 %tmp20, %tmp17
   %tmp22 = sub i32 0, 0
   %tmp23 = add i32 0, 0
-  %tmp24 = sub i32 undef, 0
-  %tmp25 = add nsw i32 %tmp23, undef
+  %tmp24 = sub i32 poison, 0
+  %tmp25 = add nsw i32 %tmp23, poison
   %tmp26 = add nsw i32 %tmp24, %tmp22
-  %tmp27 = sub nsw i32 undef, %tmp24
+  %tmp27 = sub nsw i32 poison, %tmp24
   %tmp28 = add i32 0, %tmp25
   %tmp29 = xor i32 %tmp28, 0
   %tmp30 = add i32 0, %tmp26
@@ -58,7 +54,7 @@ bb:
   %tmp35 = add i32 %tmp34, %tmp29
   %tmp36 = add i32 %tmp35, 0
   %tmp37 = add i32 %tmp36, %tmp33
-  %tmp38 = sub nsw i32 0, undef
+  %tmp38 = sub nsw i32 0, poison
   %tmp39 = add i32 0, %tmp38
   %tmp40 = xor i32 %tmp39, 0
   %tmp41 = add i32 0, %tmp37
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
index 02c3173adc654..c6f5308cf54aa 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
@@ -9,9 +9,7 @@ define i32 @foo(ptr nocapture %A, i32 %n, i32 %m) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i32 0
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 3, i32 10>
-; CHECK-NEXT:    [[TMP2:%.*]] = shl <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 3, i32 10>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 8, i32 10>
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], splat (i32 9)
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[A:%.*]], align 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
index b39480b12496b..5a9ea0d292fa0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
@@ -6,26 +6,26 @@
 define i64 @foo() {
 ; CHECK-LABEL: define i64 @foo() {
 ; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[ADD7:%.*]] = add i64 0, 0
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[ADD:%.*]], [[BB3]] ]
-; CHECK-NEXT:    [[PHI2:%.*]] = phi i64 [ [[TMP9:%.*]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i64> [ [[TMP5:%.*]], [[BB3]] ]
 ; CHECK-NEXT:    ret i64 0
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[PHI5:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ 0, [[BB3]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, [[BB]] ], [ [[TMP7:%.*]], [[BB3]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[ADD]] = add i64 [[TMP3]], [[TMP2]]
-; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 0
-; CHECK-NEXT:    [[TMP9]] = or i64 [[PHI5]], 0
-; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP7]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[ADD]], i32 0
+; CHECK-NEXT:    [[PHI4:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ 0, [[BB3]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, [[BB]] ], [ [[TMP3:%.*]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> <i64 poison, i64 0>, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[PHI4]], i32 0
+; CHECK-NEXT:    [[TMP3]] = add <2 x i64> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5]] = add <2 x i64> [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ADD7]]
+; CHECK-NEXT:    [[OR:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[OR]], 0
 ; CHECK-NEXT:    br i1 false, label [[BB3]], label [[BB1:%.*]]
 ;
 ; FORCED-LABEL: define i64 @foo() {
 ; FORCED-NEXT:  bb:
-; FORCED-NEXT:    [[TMP8:%.*]] = add i64 0, 0
+; FORCED-NEXT:    [[ADD7:%.*]] = add i64 0, 0
 ; FORCED-NEXT:    br label [[BB3:%.*]]
 ; FORCED:       bb1:
 ; FORCED-NEXT:    [[TMP0:%.*]] = phi <2 x i64> [ [[TMP5:%.*]], [[BB3]] ]
@@ -36,12 +36,10 @@ define i64 @foo() {
 ; FORCED-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> <i64 poison, i64 0>, <2 x i32> <i32 0, i32 3>
 ; FORCED-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[PHI5]], i32 0
 ; FORCED-NEXT:    [[TMP7]] = add <2 x i64> [[TMP6]], [[TMP2]]
-; FORCED-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; FORCED-NEXT:    [[TMP4:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
-; FORCED-NEXT:    [[TMP5]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i32> <i32 0, i32 3>
-; FORCED-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[TMP8]]
-; FORCED-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
-; FORCED-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[TMP9]], 0
+; FORCED-NEXT:    [[TMP5]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; FORCED-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ADD7]]
+; FORCED-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; FORCED-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[TMP8]], 0
 ; FORCED-NEXT:    br i1 false, label [[BB3]], label [[BB1:%.*]]
 ;
 bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-drop-wrapping-flags.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-drop-wrapping-flags.ll
index 2a5bfa7390770..0198b1c5cb846 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-drop-wrapping-flags.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-drop-wrapping-flags.ll
@@ -9,9 +9,7 @@ define i32 @test() {
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i8> poison, i8 [[A_PROMOTED]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i16> [[TMP5]], <i16 0, i16 -1, i16 0, i16 0>
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext i16 [[TMP7]] to i32
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-extracts-bv-combined.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-extracts-bv-combined.ll
index e6a166c27ac49..230e165e43edc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/multi-extracts-bv-combined.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-extracts-bv-combined.ll
@@ -8,10 +8,8 @@ define i32 @foo() {
 ; CHECK-NEXT:    [[D:%.*]] = load i32, ptr null, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 undef, i32 1, i32 0>, i32 [[D]], i32 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 2, i32 3, i32 1, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i32> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr getelementptr inbounds ([64 x i32], ptr null, i64 0, i64 15), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add <8 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    store <8 x i32> [[TMP2]], ptr getelementptr inbounds ([64 x i32], ptr null, i64 0, i64 15), align 4
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
index 1163c8219dabe..034fe82862950 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
@@ -4,6 +4,24 @@
 ; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
 
 define void @foo() {
+; CHECK-LABEL: define void @foo() {
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 0, i32 0
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB1]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ [[TMP2]], [[BB4]] ]
+; CHECK-NEXT:    ret void
+;
 ; FORCED-LABEL: define void @foo() {
 ; FORCED-NEXT:  bb:
 ; FORCED-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 0, i32 0
@@ -11,9 +29,7 @@ define void @foo() {
 ; FORCED:       bb1:
 ; FORCED-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
 ; FORCED-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], [[TMP0]]
-; FORCED-NEXT:    [[TMP3:%.*]] = or <2 x i32> [[TMP1]], [[TMP0]]
-; FORCED-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; FORCED-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; FORCED-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
 ; FORCED-NEXT:    [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
 ; FORCED-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
 ; FORCED-NEXT:    [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
@@ -21,29 +37,9 @@ define void @foo() {
 ; FORCED:       bb4:
 ; FORCED-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB1]]
 ; FORCED:       bb5:
-; FORCED-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ [[TMP4]], [[BB4]] ]
+; FORCED-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ [[TMP2]], [[BB4]] ]
 ; FORCED-NEXT:    ret void
 ;
-; CHECK-LABEL: define void @foo() {
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    br label [[BB1:%.*]]
-; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[SHL]], i32 0
-; CHECK-NEXT:    [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
-; CHECK-NEXT:    br label [[BB4]]
-; CHECK:       bb4:
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP6]], i32 1
-; CHECK-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB1]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[PHI6:%.*]] = phi i32 [ [[SHL]], [[BB4]] ]
-; CHECK-NEXT:    [[PHI7:%.*]] = phi i32 [ [[TMP8]], [[BB4]] ]
-; CHECK-NEXT:    ret void
-;
 bb:
   br label %bb1
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll b/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
index cb02f4d10923c..ad8e905a8ca02 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
@@ -330,9 +330,7 @@ define void @only_arcp(ptr %x) {
 define void @addsub_all_nsw(ptr %x) {
 ; CHECK-LABEL: @addsub_all_nsw(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 1, i32 -1, i32 1, i32 -1>
 ; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr [[X]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -361,9 +359,7 @@ define void @addsub_all_nsw(ptr %x) {
 define void @addsub_some_nsw(ptr %x) {
 ; CHECK-LABEL: @addsub_some_nsw(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 -1, i32 1, i32 -1>
 ; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr [[X]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -392,9 +388,7 @@ define void @addsub_some_nsw(ptr %x) {
 define void @addsub_no_nsw(ptr %x) {
 ; CHECK-LABEL: @addsub_no_nsw(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 -1, i32 1, i32 -1>
 ; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr [[X]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll
index 81f3bf99f3fd8..7fe6941d52da7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll
@@ -9,16 +9,16 @@ define i32 @test(i1 %cond) {
 ; CHECK:       [[BB]]:
 ; CHECK-NEXT:    [[P1:%.*]] = phi i32 [ [[OR92:%.*]], %[[BB]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ [[TMP8:%.*]], %[[BB]] ], [ zeroinitializer, %[[ENTRY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 1, 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> <i32 poison, i32 poison, i32 0, i32 0>, <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[P1]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> zeroinitializer, [[TMP4]]
 ; CHECK-NEXT:    [[OR92]] = or i32 1, 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[OR92]], i32 0
+; CHECK-NEXT:    [[TMP8]] = xor <2 x i32> [[TMP9]], [[TMP7]]
 ; CHECK-NEXT:    [[OP_RDX:%.*]] = xor i32 [[TMP6]], [[OR92]]
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[OP_RDX]], i32 0
-; CHECK-NEXT:    [[TMP8]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP1]], i32 1
 ; CHECK-NEXT:    br i1 [[COND]], label %[[EXIT:.*]], label %[[BB]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 [[OP_RDX]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
index cda88620ab88a..fff2b72df613e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
@@ -14,10 +14,8 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i16> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = sub <4 x i16> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i16> [[TMP9]], <4 x i16> [[TMP10]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-NEXT:    [[TMP12:%.*]] = add <4 x i16> zeroinitializer, [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = sub <4 x i16> zeroinitializer, [[TMP11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i16> [[TMP12]], <4 x i16> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP15:%.*]] = sext <4 x i16> [[TMP14]] to <4 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = sext <4 x i16> [[TMP13]] to <4 x i32>
 ; CHECK-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP2]], align 16
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll b/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
index fcc295de62adf..a17ccb4b46ef9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
@@ -6,11 +6,9 @@ define i1 @test() {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[H_PROMOTED118_I_FR:%.*]] = freeze i32 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[H_PROMOTED118_I_FR]], i32 2
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> zeroinitializer, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> zeroinitializer, [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP3]], <4 x i32> <i32 2, i32 2, i32 7, i32 2>
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> zeroinitializer, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <4 x i32> <i32 2, i32 2, i32 7, i32 2>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP5]], <i32 0, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[TMP6]], <i32 1, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
index 6e2a43ac5f9f1..15dd6756cd7db 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
@@ -242,13 +242,18 @@ exit:
 }
 
 define void @store_try_reorder(ptr %dst) {
-; CHECK-LABEL: @store_try_reorder(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 0, 0
-; CHECK-NEXT:    store i32 [[ADD]], ptr [[DST:%.*]], align 4
-; CHECK-NEXT:    [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1
-; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @store_try_reorder(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    store <3 x i32> zeroinitializer, ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @store_try_reorder(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[DST:%.*]], align 4
+; POW2-ONLY-NEXT:    [[ADD216:%.*]] = sub i32 0, 0
+; POW2-ONLY-NEXT:    [[ARRAYIDX_I1891:%.*]] = getelementptr i32, ptr [[DST]], i64 2
+; POW2-ONLY-NEXT:    store i32 [[ADD216]], ptr [[ARRAYIDX_I1891]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %add = add i32 0, 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
index 869a9d1aee80e..4f3d551e21122 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
@@ -192,9 +192,7 @@ define void @addsub0(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
 ; CHECK-NEXT:    store i32 [[TMP1]], ptr [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 -2, i32 -3>
-; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <2 x i32> [[TMP2]], <i32 -2, i32 -3>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 -2, i32 3>
 ; CHECK-NEXT:    store <2 x i32> [[TMP5]], ptr [[INCDEC_PTR3]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -225,9 +223,7 @@ define void @addsub1(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 2
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[TMP0]], splat (i32 -1)
-; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw <2 x i32> [[TMP0]], splat (i32 -1)
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP0]], <i32 -1, i32 1>
 ; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/alternate-opcode-sindle-bv.ll b/llvm/test/Transforms/SLPVectorizer/alternate-opcode-sindle-bv.ll
index c250029519590..9b6511d0d8284 100644
--- a/llvm/test/Transforms/SLPVectorizer/alternate-opcode-sindle-bv.ll
+++ b/llvm/test/Transforms/SLPVectorizer/alternate-opcode-sindle-bv.ll
@@ -1,18 +1,29 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=X86 %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=AARCH64 %}
 
 define <2 x i32> @test(i32 %arg) {
-; CHECK-LABEL: define <2 x i32> @test(
-; CHECK-SAME: i32 [[ARG:%.*]]) {
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[ARG]], 0
-; CHECK-NEXT:    [[MUL:%.*]] = mul i32 0, 1
-; CHECK-NEXT:    [[MUL1:%.*]] = mul i32 [[OR]], [[MUL]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 0, [[MUL1]]
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[OR]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[MUL]], i32 1
-; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+; X86-LABEL: define <2 x i32> @test(
+; X86-SAME: i32 [[ARG:%.*]]) {
+; X86-NEXT:  bb:
+; X86-NEXT:    [[OR:%.*]] = or i32 [[ARG]], 0
+; X86-NEXT:    [[MUL:%.*]] = mul i32 0, 1
+; X86-NEXT:    [[MUL1:%.*]] = mul i32 [[OR]], [[MUL]]
+; X86-NEXT:    [[CMP:%.*]] = icmp ugt i32 0, [[MUL1]]
+; X86-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[OR]], i32 0
+; X86-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[MUL]], i32 1
+; X86-NEXT:    ret <2 x i32> [[TMP1]]
+;
+; AARCH64-LABEL: define <2 x i32> @test(
+; AARCH64-SAME: i32 [[ARG:%.*]]) {
+; AARCH64-NEXT:  bb:
+; AARCH64-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[ARG]], i32 0
+; AARCH64-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[TMP0]], zeroinitializer
+; AARCH64-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; AARCH64-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; AARCH64-NEXT:    [[MUL1:%.*]] = mul i32 [[TMP2]], [[TMP3]]
+; AARCH64-NEXT:    [[CMP:%.*]] = icmp ugt i32 0, [[MUL1]]
+; AARCH64-NEXT:    ret <2 x i32> [[TMP1]]
 ;
 bb:
   %or = or i32 %arg, 0
diff --git a/llvm/test/Transforms/SLPVectorizer/bbi-106161.ll b/llvm/test/Transforms/SLPVectorizer/bbi-106161.ll
new file mode 100644
index 0000000000000..29339c8a8aac8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/bbi-106161.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -S %s | FileCheck %s
+
+define i128 @f_768_3162(i16 %0) {
+; CHECK-LABEL: @f_768_3162(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHL22:%.*]] = shl i16 0, -1
+; CHECK-NEXT:    [[MUL23:%.*]] = mul i16 0, [[TMP0:%.*]]
+; CHECK-NEXT:    [[MUL24:%.*]] = mul i16 [[SHL22]], [[MUL23]]
+; CHECK-NEXT:    store i16 [[MUL24]], ptr null, align 1
+; CHECK-NEXT:    ret i128 0
+;
+entry:
+  %shl22 = shl i16 0, -1
+  %mul23 = mul i16 0, %0
+  %mul24 = mul i16 %shl22, %mul23
+  store i16 %mul24, ptr null, align 1
+  ret i128 0
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/isOpcodeOrAlt.ll b/llvm/test/Transforms/SLPVectorizer/isOpcodeOrAlt.ll
new file mode 100644
index 0000000000000..414997fe8e6f7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/isOpcodeOrAlt.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -S -slp-max-reg-size=1024 %s | FileCheck %s
+
+define void @test1(ptr %a, ptr %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 0
+; CHECK-NEXT:    [[GEP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[GEP0]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[TMP0]], <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[GEP4]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %gep0 = getelementptr inbounds i32, ptr %a, i64 0
+  %gep1 = getelementptr inbounds i32, ptr %a, i64 1
+  %gep2 = getelementptr inbounds i32, ptr %a, i64 2
+  %gep3 = getelementptr inbounds i32, ptr %a, i64 3
+  %0 = load i32, ptr %gep0, align 4
+  %1 = load i32, ptr %gep1, align 4
+  %2 = load i32, ptr %gep2, align 4
+  %3 = load i32, ptr %gep3, align 4
+  %op0 = shl i32 %0, 1
+  %op1 = add i32 %1, zeroinitializer
+  %op2 = mul i32 %2, 2
+  %op3 = shl i32 %3, zeroinitializer
+  %gep4 = getelementptr inbounds i32, ptr %b, i64 0
+  %gep5 = getelementptr inbounds i32, ptr %b, i64 1
+  %gep6 = getelementptr inbounds i32, ptr %b, i64 2
+  %gep7 = getelementptr inbounds i32, ptr %b, i64 3
+  store i32 %op0, ptr %gep4, align 4
+  store i32 %op1, ptr %gep5, align 4
+  store i32 %op2, ptr %gep6, align 4
+  store i32 %op3, ptr %gep7, align 4
+  ret void
+}
+
+define void @test2(i64 %_xstride) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL3:%.*]] = mul i64 [[_XSTRIDE:%.*]], 1
+; CHECK-NEXT:    [[MUL5:%.*]] = mul i64 0, 0
+; CHECK-NEXT:    [[MUL9:%.*]] = sub i64 0, [[_XSTRIDE]]
+; CHECK-NEXT:    [[MUL12:%.*]] = shl i64 [[_XSTRIDE]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i8, ptr null, i64 [[MUL3]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr i8, ptr null, i64 [[MUL5]]
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr i8, ptr null, i64 [[MUL9]]
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr i8, ptr null, i64 [[MUL12]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul3 = mul i64 %_xstride, 1
+  %mul5 = mul i64 0, 0
+  %mul9 = sub i64 0, %_xstride
+  %mul12 = shl i64 %_xstride, 1
+  %arrayidx = getelementptr i8, ptr null, i64 %mul3
+  %arrayidx6 = getelementptr i8, ptr null, i64 %mul5
+  %arrayidx10 = getelementptr i8, ptr null, i64 %mul9
+  %arrayidx13 = getelementptr i8, ptr null, i64 %mul12
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll b/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
index 056b6222cae72..caca410f056c1 100644
--- a/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
+++ b/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
@@ -6,11 +6,9 @@ define void @func(i32 %0) {
 ; CHECK-SAME: i32 [[TMP0:%.*]]) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 0, i32 0>, i32 [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i32 [[TMP0]], 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <32 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <32 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP6]] to i64
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[TMP9]], 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP9]] to i32
diff --git a/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll b/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
index 732b50396a460..cf5927bf58327 100644
--- a/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
@@ -12,9 +12,7 @@ define i32 @test() {
 ; CHECK-NEXT:    br i1 false, label [[BB4:%.*]], label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> <i32 0, i32 poison>, <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i32> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP5]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP5]] = add <2 x i32> zeroinitializer, [[TMP2]]
 ; CHECK-NEXT:    br label [[BB1]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[TMP6:%.*]] = phi <8 x i32> [ [[TMP1]], [[BB1]] ]

From c8121b99a99fe1785add732aa062039b7c5fdd32 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 15 Apr 2025 09:10:04 -0700
Subject: [PATCH 007/710] [RISCV] Xqcilb: remove RISCVMCExpr::VK_QC_E_JUMP_PLT
 and drop `@plt` parsing

Follow-up to the just landed #135044 . Remove `@plt` parsing (only
needed by legacy `call foo@plt`). MCParser's `@` parsing is problematic.
Supporting target variations like (`foo+2@plt foo@plt+2 (foo+2)@plt`)
involves messy hacks. We should refrain from adding new `@` uses.

Remove unneeded `RISCVMCExpr::VK_QC_E_JUMP_PLT` (should only be used
when an instruction might have multiple reasonable relocations
https://maskray.me/blog/2025-03-16-relocation-generation-in-assemblers).

---

GCC's initial initial RISC-V port made a mistake by having both `call
foo` (non-PIC) and `call foo@plt` (PIC), likely misled by x86/SystemZ.
It was determined that the `@plt` was not needed. Since R_RISCV_CALL had
questionable undefined weak semantics in GNU ld (which has been removed
then), we kept R_RISCV_CALL_PLT and deprecated R_RISCV_CALL.

For RISC-V instructions, we only keep `@` in call/jump for backward
compatibility and discourage it for all other instructions.

(
There is disagreement about whether `PLT` in `JUMP_PLT` is useful or
misleading.
MaskRay's opnion: For new branch relocations with procedure call
semantics, use `_CALL` and avoid `_PLT` in the relocation name.
`_PLT` should only be used in data directives (e.g. R_RISCV_PLT32) to
indicate that the address of a function is not significant.
)

Pull Request: https://github.com/llvm/llvm-project/pull/135507
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 32 ++--------------
 .../RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp | 34 +----------------
 .../Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp |  5 +--
 .../Target/RISCV/MCTargetDesc/RISCVMCExpr.h   |  1 -
 llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td   | 38 ++++---------------
 llvm/test/MC/RISCV/xqcilb-invalid.s           |  6 ++-
 llvm/test/MC/RISCV/xqcilb-relocations.s       | 10 -----
 7 files changed, 18 insertions(+), 108 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 2f8e002f291db..013944787ff2d 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -587,17 +587,6 @@ struct RISCVOperand final : public MCParsedAsmOperand {
            (VK == RISCVMCExpr::VK_CALL || VK == RISCVMCExpr::VK_CALL_PLT);
   }
 
-  bool isPseudoQCJumpSymbol() const {
-    int64_t Imm;
-    // Must be of 'immediate' type but not a constant.
-    if (!isImm() || evaluateConstantImm(getImm(), Imm))
-      return false;
-
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
-    return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
-           VK == RISCVMCExpr::VK_QC_E_JUMP_PLT;
-  }
-
   bool isPseudoJumpSymbol() const {
     int64_t Imm;
     // Must be of 'immediate' type but not a constant.
@@ -1598,11 +1587,11 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return generateImmOutOfRangeError(Operands, ErrorInfo,
                                       std::numeric_limits<int32_t>::min(),
                                       std::numeric_limits<uint32_t>::max());
-  case Match_InvalidSImm32Lsb0:
+  case Match_InvalidBareSImm32Lsb0:
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, std::numeric_limits<int32_t>::min(),
         std::numeric_limits<int32_t>::max() - 1,
-        "operand must be a multiple of 2 bytes in the range ");
+        "operand must be a multiple of 2 bytes in the range");
   case Match_InvalidRnumArg: {
     return generateImmOutOfRangeError(Operands, ErrorInfo, 0, 10);
   }
@@ -2151,25 +2140,10 @@ ParseStatus RISCVAsmParser::parsePseudoQCJumpSymbol(OperandVector &Operands) {
 
   std::string Identifier(getTok().getIdentifier());
   SMLoc E = getTok().getEndLoc();
-
-  if (getLexer().peekTok().is(AsmToken::At)) {
-    Lex();
-    Lex();
-    SMLoc PLTLoc = getLoc();
-    StringRef PLT;
-    E = getTok().getEndLoc();
-    if (getParser().parseIdentifier(PLT) || PLT != "plt")
-      return Error(PLTLoc,
-                   "'@plt' is the only valid operand for this instruction");
-  } else {
-    Lex();
-  }
-
-  RISCVMCExpr::Specifier Kind = RISCVMCExpr::VK_QC_E_JUMP_PLT;
+  Lex();
 
   MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
   Res = MCSymbolRefExpr::create(Sym, getContext());
-  Res = RISCVMCExpr::create(Res, Kind, getContext());
   Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64()));
   return ParseStatus::Success;
 }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index f324907d49fd9..6283f1d120aaa 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -56,10 +56,6 @@ class RISCVMCCodeEmitter : public MCCodeEmitter {
                           SmallVectorImpl<MCFixup> &Fixups,
                           const MCSubtargetInfo &STI) const;
 
-  void expandQCJump(const MCInst &MI, SmallVectorImpl<char> &CB,
-                    SmallVectorImpl<MCFixup> &Fixups,
-                    const MCSubtargetInfo &STI) const;
-
   void expandTLSDESCCall(const MCInst &MI, SmallVectorImpl<char> &CB,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const;
@@ -173,26 +169,6 @@ void RISCVMCCodeEmitter::expandFunctionCall(const MCInst &MI,
   support::endian::write(CB, Binary, llvm::endianness::little);
 }
 
-void RISCVMCCodeEmitter::expandQCJump(const MCInst &MI,
-                                      SmallVectorImpl<char> &CB,
-                                      SmallVectorImpl<MCFixup> &Fixups,
-                                      const MCSubtargetInfo &STI) const {
-  MCOperand Func = MI.getOperand(0);
-  assert(Func.isExpr() && "Expected expression");
-
-  auto Opcode =
-      (MI.getOpcode() == RISCV::PseudoQC_E_J) ? RISCV::QC_E_J : RISCV::QC_E_JAL;
-  MCInst Jump = MCInstBuilder(Opcode).addExpr(Func.getExpr());
-
-  uint64_t Bits = getBinaryCodeForInstr(Jump, Fixups, STI) & 0xffff'ffff'ffffu;
-  SmallVector<char, 8> Encoding;
-  support::endian::write(Encoding, Bits, llvm::endianness::little);
-  assert(Encoding[6] == 0 && Encoding[7] == 0 &&
-         "Unexpected encoding for 48-bit instruction");
-  Encoding.truncate(6);
-  CB.append(Encoding);
-}
-
 void RISCVMCCodeEmitter::expandTLSDESCCall(const MCInst &MI,
                                            SmallVectorImpl<char> &CB,
                                            SmallVectorImpl<MCFixup> &Fixups,
@@ -464,11 +440,6 @@ void RISCVMCCodeEmitter::encodeInstruction(const MCInst &MI,
     expandTLSDESCCall(MI, CB, Fixups, STI);
     MCNumEmitted += 1;
     return;
-  case RISCV::PseudoQC_E_J:
-  case RISCV::PseudoQC_E_JAL:
-    expandQCJump(MI, CB, Fixups, STI);
-    MCNumEmitted += 1;
-    return;
   }
 
   switch (Size) {
@@ -685,9 +656,6 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
     case RISCVMCExpr::VK_QC_ABS20:
       FixupKind = RISCV::fixup_riscv_qc_abs20_u;
       break;
-    case RISCVMCExpr::VK_QC_E_JUMP_PLT:
-      FixupKind = RISCV::fixup_riscv_qc_e_jump_plt;
-      break;
     }
   } else if (Kind == MCExpr::SymbolRef || Kind == MCExpr::Binary) {
     // FIXME: Sub kind binary exprs have chance of underflow.
@@ -705,6 +673,8 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       FixupKind = RISCV::fixup_riscv_qc_e_branch;
     } else if (MIFrm == RISCVII::InstFormatQC_EAI) {
       FixupKind = RISCV::fixup_riscv_qc_e_32;
+    } else if (MIFrm == RISCVII::InstFormatQC_EJ) {
+      FixupKind = RISCV::fixup_riscv_qc_e_jump_plt;
     }
   }
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
index 99f72620f97ed..d6650e156c8b3 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
@@ -34,8 +34,7 @@ const RISCVMCExpr *RISCVMCExpr::create(const MCExpr *Expr, Specifier S,
 
 void RISCVMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   Specifier S = getSpecifier();
-  bool HasVariant = ((S != VK_None) && (S != VK_CALL) && (S != VK_CALL_PLT) &&
-                     (S != VK_QC_E_JUMP_PLT));
+  bool HasVariant = ((S != VK_None) && (S != VK_CALL) && (S != VK_CALL_PLT));
 
   if (HasVariant)
     OS << '%' << getSpecifierName(S) << '(';
@@ -168,8 +167,6 @@ StringRef RISCVMCExpr::getSpecifierName(Specifier S) {
     return "pltpcrel";
   case VK_QC_ABS20:
     return "qc.abs20";
-  case VK_QC_E_JUMP_PLT:
-    return "qc_e_jump_plt";
   }
   llvm_unreachable("Invalid ELF symbol kind");
 }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
index d60879d34dc17..e0aa7ff244521 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
@@ -44,7 +44,6 @@ class RISCVMCExpr : public MCTargetExpr {
     VK_TLSDESC_ADD_LO,
     VK_TLSDESC_CALL,
     VK_QC_ABS20,
-    VK_QC_E_JUMP_PLT
   };
 
 private:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index 4ac17c8283866..8eaa5e394a91c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -139,32 +139,20 @@ def bare_simm32 : RISCVOp<XLenVT> {
 }
 
 // A 32-bit signed immediate where the least significant bit is zero.
-def simm32_lsb0 : Operand<OtherVT> {
-  let ParserMatchClass = SImmAsmOperand<32, "Lsb0">;
+def bare_simm32_lsb0 : Operand<OtherVT> {
+  let ParserMatchClass = BareSImmNLsb0AsmOperand<32>;
   let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getImmOpValueAsr1";
   let DecoderMethod = "decodeSImmOperandAndLsl1<32>";
   let MCOperandPredicate = [{
     int64_t Imm;
-    if (!MCOp.evaluateAsConstantImm(Imm))
-      return false;
-    return isShiftedInt<31, 1>(Imm);
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedInt<31, 1>(Imm);
+    return MCOp.isBareSymbolRef();
   }];
   let OperandType = "OPERAND_PCREL";
 }
 
-def PseudoQCJumpSymbol : AsmOperandClass {
-  let Name = "PseudoQCJumpSymbol";
-  let RenderMethod = "addImmOperands";
-  let DiagnosticType = "InvalidPseudoQCJumpSymbol";
-  let DiagnosticString = "operand must be a valid jump target";
-  let ParserMethod = "parsePseudoQCJumpSymbol";
-}
-
-def pseudo_qc_jump_symbol : Operand<XLenVT> {
-  let ParserMatchClass = PseudoQCJumpSymbol;
-}
-
 //===----------------------------------------------------------------------===//
 // Instruction Formats
 //===----------------------------------------------------------------------===//
@@ -313,7 +301,7 @@ def InsnQC_EJ : DirectiveInsnQC_EJ<(outs),
                                         uimm3:$func3,
                                         uimm2:$func2,
                                         uimm5:$func5,
-                                        simm32_lsb0:$imm31),
+                                        bare_simm32_lsb0:$imm31),
                                    "$opcode, $func3, $func2, $func5, $imm31">;
 def InsnQC_ES : DirectiveInsnQC_ES<(outs),
                                    (ins uimm7_opcode:$opcode,
@@ -365,7 +353,7 @@ def : InstAlias<".insn_qc.ej $opcode, $func3, $func2, $func5, $imm31",
                             uimm3:$func3,
                             uimm2:$func2,
                             uimm5:$func5,
-                            simm32_lsb0:$imm31)>;
+                            bare_simm32_lsb0:$imm31)>;
 def : InstAlias<".insn_qc.es $opcode, $func3, $func2, $rs2, ${imm26}(${rs1})",
                  (InsnQC_ES uimm7_opcode:$opcode,
                             uimm3:$func3,
@@ -719,7 +707,7 @@ class QCIRVInstEI<bits<3> funct3, bits<2> funct2, string opcodestr>
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class QCIRVInst48EJ<bits<2> func2, string opcodestr>
-    : RVInst48<(outs), (ins simm32_lsb0:$imm31),
+    : RVInst48<(outs), (ins bare_simm32_lsb0:$imm31),
                opcodestr, "$imm31", [], InstFormatQC_EJ> {
   bits<31> imm31;
 
@@ -1231,16 +1219,6 @@ def PseudoQC_E_SH : PseudoStore<"qc.e.sh">;
 def PseudoQC_E_SW : PseudoStore<"qc.e.sw">;
 } // Predicates = [HasVendorXqcilo, IsRV32]
 
-let isCall = 0, isBarrier = 1, isTerminator = 1,
-    isCodeGenOnly = 0, hasSideEffects = 0, mayStore = 0, mayLoad = 0 in
-def PseudoQC_E_J : Pseudo<(outs), (ins pseudo_qc_jump_symbol:$func), [],
-                          "qc.e.j", "$func">;
-
-let isCall = 1, Defs = [X1], isCodeGenOnly = 0, hasSideEffects = 0,
-    mayStore = 0, mayLoad = 0 in
-def PseudoQC_E_JAL: Pseudo<(outs), (ins pseudo_qc_jump_symbol:$func), [],
-                           "qc.e.jal", "$func">;
-
 //===----------------------------------------------------------------------===//
 // Code Gen Patterns
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/RISCV/xqcilb-invalid.s b/llvm/test/MC/RISCV/xqcilb-invalid.s
index 10d456c8ac0aa..1c584da890dd3 100644
--- a/llvm/test/MC/RISCV/xqcilb-invalid.s
+++ b/llvm/test/MC/RISCV/xqcilb-invalid.s
@@ -13,6 +13,8 @@ qc.e.j  -2147483649
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcilb' (Qualcomm uC Long Branch Extension)
 qc.e.j  -2147483648
 
+# CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcilb' (Qualcomm uC Long Branch Extension)
+qc.e.j  foo
 
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.e.jal
@@ -23,5 +25,5 @@ qc.e.jal 2147483649
 # CHECK-MINUS: :[[@LINE+1]]:1: error: instruction requires the following: 'Xqcilb' (Qualcomm uC Long Branch Extension)
 qc.e.jal 2147483640
 
-# CHECK: :[[@LINE+1]]:12: error: '@plt' is the only valid operand for this instruction
-qc.e.j foo@rlt
+# CHECK: :[[@LINE+1]]:11: error: unexpected token
+qc.e.j foo@plt
diff --git a/llvm/test/MC/RISCV/xqcilb-relocations.s b/llvm/test/MC/RISCV/xqcilb-relocations.s
index a475cde3f6bfd..92e543d94f6b6 100644
--- a/llvm/test/MC/RISCV/xqcilb-relocations.s
+++ b/llvm/test/MC/RISCV/xqcilb-relocations.s
@@ -15,16 +15,6 @@ qc.e.j foo
 # INSTR: qc.e.j foo
 # FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_qc_e_jump_plt
 
-qc.e.j foo@plt
-# RELOC: R_RISCV_CUSTOM195 foo 0x0
-# INSTR: qc.e.j foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_qc_e_jump_plt
-
-qc.e.jal foo@plt
-# RELOC: R_RISCV_CUSTOM195 foo 0x0
-# INSTR: qc.e.jal foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_qc_e_jump_plt
-
 qc.e.jal foo
 # RELOC: R_RISCV_CUSTOM195 foo 0x0
 # INSTR: qc.e.jal foo

From 98534eee847dfa80ff88b213a628a6149f8754d9 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 15 Apr 2025 09:10:19 -0700
Subject: [PATCH 008/710] [mlir] Migrate away from PointerUnion::dyn_cast (NFC)
 (#135770)

Note that PointerUnion::dyn_cast has been soft deprecated in
PointerUnion.h:

  // FIXME: Replace the uses of is(), get() and dyn_cast() with
  //        isa<T>, cast<T> and the llvm::dyn_cast<T>

Literal migration would result in dyn_cast_if_present (see the
definition of PointerUnion::dyn_cast), but this patch uses dyn_cast
because we have a call to dyn_cast earlier in the function, implying
that attrOrProp is nonnull.
---
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 2431807ce463d..edcb2f507ae50 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -1828,7 +1828,7 @@ void OpEmitter::genPropertiesSupportForBytecode(
           name, tgfmt(namedProperty->prop.getWriteToMlirBytecodeCall(), &fctx));
       continue;
     }
-    const auto *namedAttr = attrOrProp.dyn_cast<const AttributeMetadata *>();
+    const auto *namedAttr = dyn_cast<const AttributeMetadata *>(attrOrProp);
     StringRef name = namedAttr->attrName;
     if (namedAttr->isRequired) {
       readPropertiesMethod << formatv(R"(

From ece10a64cb180ba931b60cbd939d80412973eada Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 15 Apr 2025 12:21:04 -0400
Subject: [PATCH 009/710] Allow some attributes on declarations after
 definitions (#135791)

The deprecated, maybe_unused, and nodiscard standard attributes may all
be applied to a redeclaration after a definition has already appeared.
We were previously dropping the attribute in that case, now we retain
the attribute after the redeclaration.

Note: someday we may want to tablegen this as part of information from
Attr.td. We may also want to relax the restriction here so that the
syntax used does not matter. This is an intentionally conservative fix.

Fixes #135481
---
 clang/docs/ReleaseNotes.rst                   |  5 +++
 clang/lib/Sema/SemaDecl.cpp                   | 15 +++++++
 .../Sema/attr-decl-after-definition-std.c     | 41 +++++++++++++++++++
 3 files changed, 61 insertions(+)
 create mode 100644 clang/test/Sema/attr-decl-after-definition-std.c

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index c106148855436..e63de32a0b2aa 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -439,6 +439,11 @@ Bug Fixes to Attribute Support
 - No longer crashing on ``__attribute__((align_value(N)))`` during template
   instantiation when the function parameter type is not a pointer or reference.
   (#GH26612)
+- Now allowing the ``[[deprecated]]``, ``[[maybe_unused]]``, and
+  ``[[nodiscard]]`` to be applied to a redeclaration after a definition in both
+  C and C++ mode for the standard spellings (other spellings, such as
+  ``__attribute__((unused))`` are still ignored after the definition, though
+  this behavior may be relaxed in the future). (#GH135481)
 
 Bug Fixes to C++ Support
 ^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index e9805c345b6af..240ce5391af81 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2996,6 +2996,21 @@ static void checkNewAttributesAfterDef(Sema &S, Decl *New, const Decl *Old) {
       // msvc will allow a subsequent definition to add an uuid to a class
       ++I;
       continue;
+    } else if (isa<DeprecatedAttr, WarnUnusedResultAttr, UnusedAttr>(
+                   NewAttribute) &&
+               NewAttribute->isStandardAttributeSyntax()) {
+      // C++14 [dcl.attr.deprecated]p3: A name or entity declared without the
+      // deprecated attribute can later be re-declared with the attribute and
+      // vice-versa.
+      // C++17 [dcl.attr.unused]p4: A name or entity declared without the
+      // maybe_unused attribute can later be redeclared with the attribute and
+      // vice versa.
+      // C++20 [dcl.attr.nodiscard]p2: A name or entity declared without the
+      // nodiscard attribute can later be redeclared with the attribute and
+      // vice-versa.
+      // C23 6.7.13.3p3, 6.7.13.4p3. and 6.7.13.5p5 give the same allowances.
+      ++I;
+      continue;
     } else if (const AlignedAttr *AA = dyn_cast<AlignedAttr>(NewAttribute)) {
       if (AA->isAlignas()) {
         // C++11 [dcl.align]p6:
diff --git a/clang/test/Sema/attr-decl-after-definition-std.c b/clang/test/Sema/attr-decl-after-definition-std.c
new file mode 100644
index 0000000000000..bab52b4dd97ad
--- /dev/null
+++ b/clang/test/Sema/attr-decl-after-definition-std.c
@@ -0,0 +1,41 @@
+// RUN: %clang_cc1 -fsyntax-only -Wignored-attributes -verify -std=c23 %s
+// RUN: %clang_cc1 -fsyntax-only -Wignored-attributes -verify -x c++ %s
+// RUN: %clang_cc1 -fsyntax-only -ast-dump %s | FileCheck %s
+
+inline int frob(int x) { return x; }
+
+[[deprecated]] int frob(int); // expected-note 2 {{'frob' has been explicitly marked deprecated here}}
+
+void use1() {
+  // Using this should give a deprecation warning, but not a nodiscard warning.	
+  frob(0); // expected-warning {{'frob' is deprecated}}
+}
+
+[[nodiscard]] int frob(int);
+
+void use2() {
+  // This should give both warnings.
+  frob(0); // expected-warning {{'frob' is deprecated}} \
+              expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+}
+
+[[maybe_unused]] int frob(int);
+
+// Currently, this is only allowed for the standard spelling of the attributes.
+void blob() {}                           // expected-note {{previous definition is here}}
+__attribute__((deprecated)) void blob(); // expected-warning {{attribute declaration must precede definition}}
+
+// CHECK: FunctionDecl {{.*}} frob
+
+// CHECK: FunctionDecl {{.*}} prev {{.*}} frob
+// CHECK:  DeprecatedAttr
+
+// CHECK: FunctionDecl {{.*}} prev {{.*}} frob
+// CHECK:  DeprecatedAttr
+// CHECK:  WarnUnusedResultAttr
+
+// CHECK: FunctionDecl {{.*}} prev {{.*}} frob
+// CHECK:  DeprecatedAttr
+// CHECK:  WarnUnusedResultAttr
+// CHECK:  UnusedAttr
+

From a9553990fb6de8b4d99d05c95fe949deef6357e1 Mon Sep 17 00:00:00 2001
From: alx32 <103613512+alx32@users.noreply.github.com>
Date: Tue, 15 Apr 2025 09:31:03 -0700
Subject: [PATCH 010/710] [DWARFLinker] Update `stmt-seq-macho.test` to use
 `update_test_body.py` (#133363)

In this change we update DWARFLinker test
`llvm/test/tools/dsymutil/ARM/stmt-seq-macho.test` to be self-contained
and easy to regenerate via `update_test_body.py`. As relating to [this
comment](https://github.com/llvm/llvm-project/pull/132875/files#r2012471834)
- this would be approach nr.2

Updating the test can be done via:
```
PATH=/path/to/llvm/bin:$PATH llvm/utils/update_test_body.py llvm/test/tools/dsymutil/ARM/stmt-seq-macho.test
```
---
 .../tools/dsymutil/ARM/stmt-seq-macho.test    | 1862 ++++++++++++++++-
 .../private/tmp/stmt_seq/stmt_seq_macho.exe   |  Bin 17360 -> 0 bytes
 .../private/tmp/stmt_seq/stmt_seq_macho.o     |  Bin 4704 -> 0 bytes
 3 files changed, 1842 insertions(+), 20 deletions(-)
 delete mode 100755 llvm/test/tools/dsymutil/Inputs/private/tmp/stmt_seq/stmt_seq_macho.exe
 delete mode 100644 llvm/test/tools/dsymutil/Inputs/private/tmp/stmt_seq/stmt_seq_macho.o

diff --git a/llvm/test/tools/dsymutil/ARM/stmt-seq-macho.test b/llvm/test/tools/dsymutil/ARM/stmt-seq-macho.test
index 1dd1f61f1f7fb..1e08bc07e14cb 100644
--- a/llvm/test/tools/dsymutil/ARM/stmt-seq-macho.test
+++ b/llvm/test/tools/dsymutil/ARM/stmt-seq-macho.test
@@ -1,5 +1,10 @@
-RUN: dsymutil --flat -oso-prepend-path %p/../Inputs %p/../Inputs/private/tmp/stmt_seq/stmt_seq_macho.exe -o %t.stmt_seq_macho.dSYM
-RUN: llvm-dwarfdump --debug-info --debug-line -v %t.stmt_seq_macho.dSYM | sort | FileCheck %s -check-prefix=CHECK_DSYM
+## Test that verifies DW_AT_LLVM_stmt_sequence attributes are correctly patched in the dSYM
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: yaml2obj %t/stmt_seq_macho.exe.yaml -o %t/stmt_seq_macho.exe
+# RUN: yaml2obj %t/stmt_seq_macho.o.yaml   -o %t/stmt_seq_macho.o
+# RUN: dsymutil --flat --verify-dwarf=none -oso-prepend-path %t %t/stmt_seq_macho.exe -o %t/stmt_seq_macho.dSYM
+# RUN: llvm-dwarfdump --debug-info --debug-line -v %t/stmt_seq_macho.dSYM | sort | FileCheck %s -check-prefix=CHECK_DSYM
 
 # CHECK_DSYM: DW_AT_LLVM_stmt_sequence [DW_FORM_sec_offset] ([[OFFSET1:(0x[0-9a-f]+)]])
 # CHECK_DSYM: DW_AT_LLVM_stmt_sequence [DW_FORM_sec_offset] ([[OFFSET2:(0x[0-9a-f]+)]])
@@ -11,27 +16,18 @@ RUN: llvm-dwarfdump --debug-info --debug-line -v %t.stmt_seq_macho.dSYM | sort |
 # CHECK_DSYM: [[OFFSET3]]: 00 DW_LNE_set_address
 # CHECK_DSYM: [[OFFSET4]]: 00 DW_LNE_set_address
 
-
-########  Generate stmt_seq_macho.exe & stmt_seq_macho.o via script:  ##########
-# ------------------------------------------------------------------------------
-#!/bin/bash
-TOOLCHAIN=/path/to/llvm/bin
-
-# ------------------------------------------------------------------------------
-# Create the stmt_seq_macho.cpp source file
-# ------------------------------------------------------------------------------
-cat > stmt_seq_macho.cpp << 'EOF'
+#--- stmt_seq_macho.cpp
 #define ATTRIB extern "C" __attribute__((noinline))
 
 ATTRIB int function3_copy1(int a) {
     int b = a + 3;
     return b + 1;
 }
- 
+
 ATTRIB int function2_copy1(int a) {
     return a - 22;
 }
- 
+
 ATTRIB int function3_copy2(int a) {
     int b = a + 3;
     return b + 1;
@@ -45,11 +41,11 @@ ATTRIB int function2_copy2(int a) {
 struct logic_error {
     logic_error(const char* s) {}
 };
- 
+
 struct length_error : public logic_error {
     __attribute__((noinline)) explicit length_error(const char* s) : logic_error(s) {}
 };
- 
+
 int main() {
     int sum = 0;
     sum += function2_copy2(3);
@@ -58,11 +54,12 @@ int main() {
     length_error e("test");
     return sum;
 }
-EOF
 
-"$TOOLCHAIN/clang" \
-  --target=arm64-apple-macos11 \
+#--- gen
+# Compile to an object file
+clang --target=arm64-apple-macos11 \
   -c \
+  -fdebug-compilation-dir=/private/tmp/stmt_seq \
   -g \
   -gdwarf-4 \
   -fno-unwind-tables \
@@ -73,11 +70,1836 @@ EOF
   stmt_seq_macho.cpp \
   -o stmt_seq_macho.o
 
-"$TOOLCHAIN/ld64.lld" \
+# Link into an executable
+ld64.lld \
   -arch arm64 \
   -platform_version macos 11.0.0 11.0.0 \
   -o stmt_seq_macho.exe \
   stmt_seq_macho.o \
   -dead_strip \
   --icf=all \
+  -oso_prefix $(pwd)/ \
   --keep-icf-stabs
+
+# Convert executable to YAML for the test
+echo "#--- stmt_seq_macho.o.yaml"
+obj2yaml stmt_seq_macho.o
+echo ""
+echo "#--- stmt_seq_macho.exe.yaml"
+obj2yaml stmt_seq_macho.exe
+
+#--- stmt-seq-macho.yaml
+#--- stmt_seq_macho.o.yaml
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x100000C
+  cpusubtype:      0x0
+  filetype:        0x1
+  ncmds:           5
+  sizeofcmds:      1176
+  flags:           0x2000
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         1032
+    segname:         ''
+    vmaddr:          0
+    vmsize:          2793
+    fileoff:         1208
+    filesize:        2793
+    maxprot:         7
+    initprot:        7
+    nsects:          12
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0
+        size:            128
+        offset:          0x4B8
+        align:           2
+        reloff:          0xFA8
+        nreloc:          7
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         00100011C0035FD600580051C0035FD600100011C0035FD600580051C0035FD6FFC300D1F44F01A9FD7B02A9FD8300916000805200000094F30300AA20058052000000941400130B6001805200000094F30300AA0100009021000091E03F0091000000948002130BFD7B42A9F44F41A9FFC30091C0035FD600000014C0035FD6
+        relocations:
+          - address:         0x78
+            symbolnum:       4
+            pcrel:           true
+            length:          2
+            extern:          true
+            type:            2
+            scattered:       false
+            value:           0
+          - address:         0x60
+            symbolnum:       3
+            pcrel:           true
+            length:          2
+            extern:          true
+            type:            2
+            scattered:       false
+            value:           0
+          - address:         0x58
+            symbolnum:       1
+            pcrel:           false
+            length:          2
+            extern:          true
+            type:            4
+            scattered:       false
+            value:           0
+          - address:         0x54
+            symbolnum:       1
+            pcrel:           true
+            length:          2
+            extern:          true
+            type:            3
+            scattered:       false
+            value:           0
+          - address:         0x4C
+            symbolnum:       5
+            pcrel:           true
+            length:          2
+            extern:          true
+            type:            2
+            scattered:       false
+            value:           0
+          - address:         0x40
+            symbolnum:       8
+            pcrel:           true
+            length:          2
+            extern:          true
+            type:            2
+            scattered:       false
+            value:           0
+          - address:         0x34
+            symbolnum:       6
+            pcrel:           true
+            length:          2
+            extern:          true
+            type:            2
+            scattered:       false
+            value:           0
+      - sectname:        __cstring
+        segname:         __TEXT
+        addr:            0x80
+        size:            5
+        offset:          0x538
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '7465737400'
+      - sectname:        __debug_loc
+        segname:         __DWARF
+        addr:            0x85
+        size:            412
+        offset:          0x53D
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         00000000000000000400000000000000010050040000000000000008000000000000000400A301509F0000000000000000000000000000000000000000000000000400000000000000030070039F0000000000000000000000000000000008000000000000000C000000000000000100500C0000000000000010000000000000000400A301509F0000000000000000000000000000000010000000000000001400000000000000010050140000000000000018000000000000000400A301509F0000000000000000000000000000000010000000000000001400000000000000030070039F0000000000000000000000000000000018000000000000001C000000000000000100501C0000000000000020000000000000000400A301509F000000000000000000000000000000001C0000000000000020000000000000000100500000000000000000000000000000000030000000000000003C00000000000000030011009F3C0000000000000048000000000000000100634800000000000000540000000000000001006400000000000000000000000000000000
+      - sectname:        __debug_abbrev
+        segname:         __DWARF
+        addr:            0x221
+        size:            359
+        offset:          0x6D9
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+      - sectname:        __debug_info
+        segname:         __DWARF
+        addr:            0x388
+        size:            686
+        offset:          0x840
+        align:           0
+        reloff:          0xFE0
+        nreloc:          14
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        relocations:
+          - address:         0x26A
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x251
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x216
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x1B8
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x1A5
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x191
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x17E
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x140
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x104
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0xC8
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x9B
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x5F
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x37
+            symbolnum:       2
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x22
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+      - sectname:        __debug_str
+        segname:         __DWARF
+        addr:            0x636
+        size:            239
+        offset:          0xAEE
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+      - sectname:        __apple_names
+        segname:         __DWARF
+        addr:            0x725
+        size:            260
+        offset:          0xBDD
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         485341480100000008000000080000000C000000000000000100000001000600000000000200000005000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF90D9F86F88CB36CF4908311CD1125E5389CB36CF4A08311C522B70536A7F9A7C8000000094000000A4000000B4000000C4000000D4000000E4000000F40000008A0000000200000015020000690200000000000055000000010000009A0000000000000045000000010000005E00000000000000A3000000010000001502000000000000750000000100000003010000000000006500000001000000C700000000000000BB00000001000000690200000000000085000000010000003F01000000000000
+      - sectname:        __apple_objc
+        segname:         __DWARF
+        addr:            0x829
+        size:            36
+        offset:          0xCE1
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         485341480100000001000000000000000C000000000000000100000001000600FFFFFFFF
+      - sectname:        __apple_namespac
+        segname:         __DWARF
+        addr:            0x84D
+        size:            36
+        offset:          0xD05
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         485341480100000001000000000000000C000000000000000100000001000600FFFFFFFF
+      - sectname:        __apple_types
+        segname:         __DWARF
+        addr:            0x871
+        size:            195
+        offset:          0xD29
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         48534148010000000500000005000000140000000000000003000000010006000300050004000B000000000002000000FFFFFFFF03000000040000007CA8F05D90D9F86F5B738CDC3080880B6320957C64000000770000008A0000009D000000B00000009700000001000000EA010000130000000000008A00000001000000C80100001300000000000031000000010000005700000024000000000000D300000001000000A1020000240000000000002C000000010000005000000024000000000000
+      - sectname:        __debug_frame
+        segname:         __DWARF
+        addr:            0x938
+        size:            208
+        offset:          0xDF0
+        align:           3
+        reloff:          0x1050
+        nreloc:          7
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         14000000FFFFFFFF0400080001781E0C1F00000000000000140000000000000000000000000000000800000000000000140000000000000008000000000000000800000000000000140000000000000010000000000000000800000000000000140000000000000018000000000000000800000000000000240000000000000020000000000000005800000000000000500C1D109E019D02930394040000000014000000000000007800000000000000040000000000000014000000000000007C000000000000000400000000000000
+        relocations:
+          - address:         0xC0
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0xA8
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x80
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x68
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x50
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x38
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x20
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+      - sectname:        __debug_line
+        segname:         __DWARF
+        addr:            0xA08
+        size:            225
+        offset:          0xEC0
+        align:           0
+        reloff:          0x1088
+        nreloc:          7
+        flags:           0x2000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        relocations:
+          - address:         0xD1
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0xBD
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x92
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x7E
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x66
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x50
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+          - address:         0x3A
+            symbolnum:       1
+            pcrel:           false
+            length:          3
+            extern:          false
+            type:            0
+            scattered:       false
+            value:           0
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         24
+    platform:        1
+    minos:           720896
+    sdk:             0
+    ntools:          0
+  - cmd:             LC_LINKER_OPTIMIZATION_HINT
+    cmdsize:         16
+    dataoff:         4288
+    datasize:        8
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          4296
+    nsyms:           10
+    stroff:          4456
+    strsize:         144
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       3
+    iextdefsym:      3
+    nextdefsym:      7
+    iundefsym:       10
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:
+  NameList:
+    - n_strx:          138
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          1
+      n_type:          0xE
+      n_sect:          2
+      n_desc:          0
+      n_value:         128
+    - n_strx:          132
+      n_type:          0xE
+      n_sect:          2
+      n_desc:          0
+      n_value:         128
+    - n_strx:          39
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          192
+      n_value:         120
+    - n_strx:          14
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          192
+      n_value:         124
+    - n_strx:          115
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         8
+    - n_strx:          81
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         24
+    - n_strx:          98
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          64
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         16
+    - n_strx:          8
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         32
+  StringTable:
+    - ''
+    - l_.str
+    - _main
+    - __ZN12length_errorC2EPKc
+    - __ZN12length_errorC1EPKc
+    - _function3_copy2
+    - _function2_copy2
+    - _function3_copy1
+    - _function2_copy1
+    - ltmp1
+    - ltmp0
+DWARF:
+  debug_str:
+    - ''
+    - stmt_seq_macho.cpp
+    - '/'
+    - '/private/tmp/stmt_seq'
+    - char
+    - __ARRAY_SIZE_TYPE__
+    - function3_copy1
+    - function2_copy1
+    - function3_copy2
+    - function2_copy2
+    - main
+    - length_error
+    - logic_error
+    - _ZN12length_errorC1EPKc
+    - _ZN12length_errorC2EPKc
+    - int
+    - a
+    - b
+    - result
+    - e
+    - sum
+    - this
+    - s
+  debug_abbrev:
+    - ID:              0
+      Table:
+        - Code:            0x1
+          Tag:             DW_TAG_compile_unit
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_producer
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_language
+              Form:            DW_FORM_data2
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_LLVM_sysroot
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_stmt_list
+              Form:            DW_FORM_sec_offset
+            - Attribute:       DW_AT_comp_dir
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_APPLE_optimized
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_data4
+        - Code:            0x2
+          Tag:             DW_TAG_variable
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_exprloc
+        - Code:            0x3
+          Tag:             DW_TAG_array_type
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0x4
+          Tag:             DW_TAG_subrange_type
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_count
+              Form:            DW_FORM_data1
+        - Code:            0x5
+          Tag:             DW_TAG_const_type
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0x6
+          Tag:             DW_TAG_base_type
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_encoding
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_byte_size
+              Form:            DW_FORM_data1
+        - Code:            0x7
+          Tag:             DW_TAG_base_type
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_byte_size
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_encoding
+              Form:            DW_FORM_data1
+        - Code:            0x8
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_data4
+            - Attribute:       DW_AT_APPLE_omit_frame_ptr
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_LLVM_stmt_sequence
+              Form:            DW_FORM_sec_offset
+            - Attribute:       DW_AT_frame_base
+              Form:            DW_FORM_exprloc
+            - Attribute:       DW_AT_call_all_calls
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_external
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_APPLE_optimized
+              Form:            DW_FORM_flag_present
+        - Code:            0x9
+          Tag:             DW_TAG_formal_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_sec_offset
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0xA
+          Tag:             DW_TAG_variable
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_sec_offset
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0xB
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_data4
+            - Attribute:       DW_AT_LLVM_stmt_sequence
+              Form:            DW_FORM_sec_offset
+            - Attribute:       DW_AT_frame_base
+              Form:            DW_FORM_exprloc
+            - Attribute:       DW_AT_call_all_calls
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_external
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_APPLE_optimized
+              Form:            DW_FORM_flag_present
+        - Code:            0xC
+          Tag:             DW_TAG_variable
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_exprloc
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0xD
+          Tag:             DW_TAG_call_site
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_call_origin
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_call_return_pc
+              Form:            DW_FORM_addr
+        - Code:            0xE
+          Tag:             DW_TAG_call_site_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_exprloc
+            - Attribute:       DW_AT_call_value
+              Form:            DW_FORM_exprloc
+        - Code:            0xF
+          Tag:             DW_TAG_structure_type
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_calling_convention
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_byte_size
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+        - Code:            0x10
+          Tag:             DW_TAG_inheritance
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_data_member_location
+              Form:            DW_FORM_data1
+        - Code:            0x11
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_declaration
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_external
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_APPLE_optimized
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_explicit
+              Form:            DW_FORM_flag_present
+        - Code:            0x12
+          Tag:             DW_TAG_formal_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_artificial
+              Form:            DW_FORM_flag_present
+        - Code:            0x13
+          Tag:             DW_TAG_formal_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0x14
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_declaration
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_external
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_APPLE_optimized
+              Form:            DW_FORM_flag_present
+        - Code:            0x15
+          Tag:             DW_TAG_pointer_type
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0x16
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_data4
+            - Attribute:       DW_AT_APPLE_omit_frame_ptr
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_LLVM_stmt_sequence
+              Form:            DW_FORM_sec_offset
+            - Attribute:       DW_AT_frame_base
+              Form:            DW_FORM_exprloc
+            - Attribute:       DW_AT_object_pointer
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_call_all_calls
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_linkage_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_specification
+              Form:            DW_FORM_ref4
+        - Code:            0x17
+          Tag:             DW_TAG_formal_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_exprloc
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_artificial
+              Form:            DW_FORM_flag_present
+        - Code:            0x18
+          Tag:             DW_TAG_formal_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_exprloc
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_decl_file
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_decl_line
+              Form:            DW_FORM_data1
+            - Attribute:       DW_AT_type
+              Form:            DW_FORM_ref4
+        - Code:            0x19
+          Tag:             DW_TAG_call_site
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_call_origin
+              Form:            DW_FORM_ref4
+            - Attribute:       DW_AT_call_tail_call
+              Form:            DW_FORM_flag_present
+            - Attribute:       DW_AT_call_pc
+              Form:            DW_FORM_addr
+  debug_info:
+    - Length:          0x2AA
+      Version:         4
+      AbbrevTableID:   0
+      AbbrOffset:      0x0
+      AddrSize:        8
+      Entries:
+        - AbbrCode:        0x1
+          Values:
+            - Value:           0x0
+            - Value:           0x21
+            - Value:           0x1
+            - Value:           0x14
+            - Value:           0x0
+            - Value:           0x16
+            - Value:           0x1
+            - Value:           0x0
+            - Value:           0x80
+        - AbbrCode:        0x2
+          Values:
+            - Value:           0x3F
+            - Value:           0x1
+            - Value:           0x23
+            - Value:           0x9
+              BlockData:       [ 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                                 0x0 ]
+        - AbbrCode:        0x3
+          Values:
+            - Value:           0x4B
+        - AbbrCode:        0x4
+          Values:
+            - Value:           0x57
+            - Value:           0x5
+        - AbbrCode:        0x0
+        - AbbrCode:        0x5
+          Values:
+            - Value:           0x50
+        - AbbrCode:        0x6
+          Values:
+            - Value:           0x2C
+            - Value:           0x6
+            - Value:           0x1
+        - AbbrCode:        0x7
+          Values:
+            - Value:           0x31
+            - Value:           0x8
+            - Value:           0x7
+        - AbbrCode:        0x8
+          Values:
+            - Value:           0x0
+            - Value:           0x8
+            - Value:           0x1
+            - Value:           0x34
+            - Value:           0x1
+              BlockData:       [ 0x6F ]
+            - Value:           0x1
+            - Value:           0x45
+            - Value:           0x1
+            - Value:           0x3
+            - Value:           0x2A1
+            - Value:           0x1
+            - Value:           0x1
+        - AbbrCode:        0x9
+          Values:
+            - Value:           0x0
+            - Value:           0xD7
+            - Value:           0x1
+            - Value:           0x3
+            - Value:           0x2A1
+        - AbbrCode:        0xA
+          Values:
+            - Value:           0x39
+            - Value:           0xD9
+            - Value:           0x1
+            - Value:           0x4
+            - Value:           0x2A1
+        - AbbrCode:        0x0
+        - AbbrCode:        0x8
+          Values:
+            - Value:           0x8
+            - Value:           0x8
+            - Value:           0x1
+            - Value:           0x4A
+            - Value:           0x1
+              BlockData:       [ 0x6F ]
+            - Value:           0x1
+            - Value:           0x55
+            - Value:           0x1
+            - Value:           0x8
+            - Value:           0x2A1
+            - Value:           0x1
+            - Value:           0x1
+        - AbbrCode:        0x9
+          Values:
+            - Value:           0x5E
+            - Value:           0xD7
+            - Value:           0x1
+            - Value:           0x8
+            - Value:           0x2A1
+        - AbbrCode:        0x0
+        - AbbrCode:        0x8
+          Values:
+            - Value:           0x10
+            - Value:           0x8
+            - Value:           0x1
+            - Value:           0x60
+            - Value:           0x1
+              BlockData:       [ 0x6F ]
+            - Value:           0x1
+            - Value:           0x65
+            - Value:           0x1
+            - Value:           0xC
+            - Value:           0x2A1
+            - Value:           0x1
+            - Value:           0x1
+        - AbbrCode:        0x9
+          Values:
+            - Value:           0x97
+            - Value:           0xD7
+            - Value:           0x1
+            - Value:           0xC
+            - Value:           0x2A1
+        - AbbrCode:        0xA
+          Values:
+            - Value:           0xD0
+            - Value:           0xD9
+            - Value:           0x1
+            - Value:           0xD
+            - Value:           0x2A1
+        - AbbrCode:        0x0
+        - AbbrCode:        0x8
+          Values:
+            - Value:           0x18
+            - Value:           0x8
+            - Value:           0x1
+            - Value:           0x78
+            - Value:           0x1
+              BlockData:       [ 0x6F ]
+            - Value:           0x1
+            - Value:           0x75
+            - Value:           0x1
+            - Value:           0x11
+            - Value:           0x2A1
+            - Value:           0x1
+            - Value:           0x1
+        - AbbrCode:        0x9
+          Values:
+            - Value:           0xF5
+            - Value:           0xD7
+            - Value:           0x1
+            - Value:           0x11
+            - Value:           0x2A1
+        - AbbrCode:        0xA
+          Values:
+            - Value:           0x12E
+            - Value:           0xDB
+            - Value:           0x1
+            - Value:           0x12
+            - Value:           0x2A1
+        - AbbrCode:        0x0
+        - AbbrCode:        0xB
+          Values:
+            - Value:           0x20
+            - Value:           0x58
+            - Value:           0x8F
+            - Value:           0x1
+              BlockData:       [ 0x6D ]
+            - Value:           0x1
+            - Value:           0x85
+            - Value:           0x1
+            - Value:           0x1E
+            - Value:           0x2A1
+            - Value:           0x1
+            - Value:           0x1
+        - AbbrCode:        0xC
+          Values:
+            - Value:           0x2
+              BlockData:       [ 0x8F, 0xF ]
+            - Value:           0xE2
+            - Value:           0x1
+            - Value:           0x23
+            - Value:           0x1C8
+        - AbbrCode:        0xA
+          Values:
+            - Value:           0x151
+            - Value:           0xE4
+            - Value:           0x1
+            - Value:           0x1F
+            - Value:           0x2A1
+        - AbbrCode:        0xD
+          Values:
+            - Value:           0x103
+            - Value:           0x38
+        - AbbrCode:        0xE
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x50 ]
+            - Value:           0x1
+              BlockData:       [ 0x33 ]
+        - AbbrCode:        0x0
+        - AbbrCode:        0xD
+          Values:
+            - Value:           0xC7
+            - Value:           0x44
+        - AbbrCode:        0xE
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x50 ]
+            - Value:           0x2
+              BlockData:       [ 0x10, 0x29 ]
+        - AbbrCode:        0x0
+        - AbbrCode:        0xD
+          Values:
+            - Value:           0x9A
+            - Value:           0x50
+        - AbbrCode:        0xE
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x50 ]
+            - Value:           0x1
+              BlockData:       [ 0x3B ]
+        - AbbrCode:        0x0
+        - AbbrCode:        0xD
+          Values:
+            - Value:           0x215
+            - Value:           0x64
+        - AbbrCode:        0xE
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x50 ]
+            - Value:           0x2
+              BlockData:       [ 0x8F, 0xF ]
+        - AbbrCode:        0x0
+        - AbbrCode:        0x0
+        - AbbrCode:        0xF
+          Values:
+            - Value:           0x5
+            - Value:           0x8A
+            - Value:           0x1
+            - Value:           0x1
+            - Value:           0x1A
+        - AbbrCode:        0x10
+          Values:
+            - Value:           0x1EA
+            - Value:           0x0
+        - AbbrCode:        0x11
+          Values:
+            - Value:           0x8A
+            - Value:           0x1
+            - Value:           0x1B
+            - Value:           0x1
+            - Value:           0x1
+            - Value:           0x1
+            - Value:           0x1
+        - AbbrCode:        0x12
+          Values:
+            - Value:           0x210
+            - Value:           0x1
+        - AbbrCode:        0x13
+          Values:
+            - Value:           0x20B
+        - AbbrCode:        0x0
+        - AbbrCode:        0x0
+        - AbbrCode:        0xF
+          Values:
+            - Value:           0x5
+            - Value:           0x97
+            - Value:           0x1
+            - Value:           0x1
+            - Value:           0x16
+        - AbbrCode:        0x14
+          Values:
+            - Value:           0x97
+            - Value:           0x1
+            - Value:           0x17
+            - Value:           0x1
+            - Value:           0x1
+            - Value:           0x1
+        - AbbrCode:        0x12
+          Values:
+            - Value:           0x206
+            - Value:           0x1
+        - AbbrCode:        0x13
+          Values:
+            - Value:           0x20B
+        - AbbrCode:        0x0
+        - AbbrCode:        0x0
+        - AbbrCode:        0x15
+          Values:
+            - Value:           0x1EA
+        - AbbrCode:        0x15
+          Values:
+            - Value:           0x4B
+        - AbbrCode:        0x15
+          Values:
+            - Value:           0x1C8
+        - AbbrCode:        0x16
+          Values:
+            - Value:           0x78
+            - Value:           0x4
+            - Value:           0x1
+            - Value:           0xB7
+            - Value:           0x1
+              BlockData:       [ 0x6F ]
+            - Value:           0x234
+            - Value:           0x1
+            - Value:           0xA3
+            - Value:           0x1D7
+        - AbbrCode:        0x17
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x50 ]
+            - Value:           0xE8
+            - Value:           0x2A8
+            - Value:           0x1
+        - AbbrCode:        0x18
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x51 ]
+            - Value:           0xED
+            - Value:           0x1
+            - Value:           0x1B
+            - Value:           0x20B
+        - AbbrCode:        0x19
+          Values:
+            - Value:           0x269
+            - Value:           0x1
+            - Value:           0x78
+        - AbbrCode:        0xE
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x50 ]
+            - Value:           0x3
+              BlockData:       [ 0xA3, 0x1, 0x50 ]
+        - AbbrCode:        0xE
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x51 ]
+            - Value:           0x3
+              BlockData:       [ 0xA3, 0x1, 0x51 ]
+        - AbbrCode:        0x0
+        - AbbrCode:        0x0
+        - AbbrCode:        0x16
+          Values:
+            - Value:           0x7C
+            - Value:           0x4
+            - Value:           0x1
+            - Value:           0xCB
+            - Value:           0x1
+              BlockData:       [ 0x6F ]
+            - Value:           0x288
+            - Value:           0x1
+            - Value:           0xBB
+            - Value:           0x1D7
+        - AbbrCode:        0x17
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x50 ]
+            - Value:           0xE8
+            - Value:           0x2A8
+            - Value:           0x1
+        - AbbrCode:        0x18
+          Values:
+            - Value:           0x1
+              BlockData:       [ 0x51 ]
+            - Value:           0xED
+            - Value:           0x1
+            - Value:           0x1B
+            - Value:           0x20B
+        - AbbrCode:        0x0
+        - AbbrCode:        0x6
+          Values:
+            - Value:           0xD3
+            - Value:           0x5
+            - Value:           0x4
+        - AbbrCode:        0x15
+          Values:
+            - Value:           0x1C8
+        - AbbrCode:        0x0
+  debug_line:
+    - Length:          221
+      Version:         4
+      PrologueLength:  42
+      MinInstLength:   1
+      MaxOpsPerInst:   1
+      DefaultIsStmt:   1
+      LineBase:        251
+      LineRange:       14
+      OpcodeBase:      13
+      StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
+      Files:
+        - Name:            stmt_seq_macho.cpp
+          DirIdx:          0
+          ModTime:         0
+          Length:          0
+      Opcodes:
+        - Opcode:          DW_LNS_set_column
+          Data:            14
+        - Opcode:          DW_LNS_set_prologue_end
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          9
+          SubOpcode:       DW_LNE_set_address
+          Data:            0
+        - Opcode:          0x16
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            5
+        - Opcode:          DW_LNS_negate_stmt
+          Data:            0
+        - Opcode:          0x4A
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            14
+        - Opcode:          DW_LNS_set_prologue_end
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          9
+          SubOpcode:       DW_LNE_set_address
+          Data:            8
+        - Opcode:          0x1A
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            5
+        - Opcode:          DW_LNS_negate_stmt
+          Data:            0
+        - Opcode:          0x4A
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            14
+        - Opcode:          DW_LNS_set_prologue_end
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          9
+          SubOpcode:       DW_LNE_set_address
+          Data:            16
+        - Opcode:          DW_LNS_advance_line
+          SData:           13
+          Data:            0
+        - Opcode:          DW_LNS_copy
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            5
+        - Opcode:          DW_LNS_negate_stmt
+          Data:            0
+        - Opcode:          0x4A
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            20
+        - Opcode:          DW_LNS_set_prologue_end
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          9
+          SubOpcode:       DW_LNE_set_address
+          Data:            24
+        - Opcode:          DW_LNS_advance_line
+          SData:           17
+          Data:            0
+        - Opcode:          DW_LNS_copy
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            5
+        - Opcode:          0x4B
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          9
+          SubOpcode:       DW_LNE_set_address
+          Data:            32
+        - Opcode:          DW_LNS_advance_line
+          SData:           29
+          Data:            0
+        - Opcode:          DW_LNS_copy
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            12
+        - Opcode:          DW_LNS_set_prologue_end
+          Data:            0
+        - Opcode:          0xF4
+          Data:            0
+        - Opcode:          0xBB
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            9
+        - Opcode:          DW_LNS_negate_stmt
+          Data:            0
+        - Opcode:          0x82
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            12
+        - Opcode:          DW_LNS_negate_stmt
+          Data:            0
+        - Opcode:          0x4B
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            18
+        - Opcode:          0xBB
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            9
+        - Opcode:          0xF1
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            5
+        - Opcode:          DW_LNS_set_epilogue_begin
+          Data:            0
+        - Opcode:          0x4C
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            85
+        - Opcode:          DW_LNS_set_prologue_end
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          9
+          SubOpcode:       DW_LNE_set_address
+          Data:            120
+        - Opcode:          DW_LNS_advance_line
+          SData:           26
+          Data:            0
+        - Opcode:          DW_LNS_copy
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+        - Opcode:          DW_LNS_set_column
+          Data:            86
+        - Opcode:          DW_LNS_set_prologue_end
+          Data:            0
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          9
+          SubOpcode:       DW_LNE_set_address
+          Data:            124
+        - Opcode:          DW_LNS_advance_line
+          SData:           26
+          Data:            0
+        - Opcode:          DW_LNS_copy
+          Data:            0
+        - Opcode:          DW_LNS_advance_pc
+          Data:            4
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+...
+
+#--- stmt_seq_macho.exe.yaml
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x100000C
+  cpusubtype:      0x0
+  filetype:        0x2
+  ncmds:           13
+  sizeofcmds:      688
+  flags:           0x200085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          4294967296
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __TEXT
+    vmaddr:          4294967296
+    vmsize:          16384
+    fileoff:         0
+    filesize:        16384
+    maxprot:         5
+    initprot:        5
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x1000002F0
+        size:            112
+        offset:          0x2F0
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         00580051C0035FD600100011C0035FD6FFC300D1F44F01A9FD7B02A9FD83009160008052F7FFFF97F30300AA20058052F6FFFF971400130B60018052F1FFFF97F30300AA610100101F2003D5E03F0091060000948002130BFD7B42A9F44F41A9FFC30091C0035FD601000014C0035FD6
+      - sectname:        __cstring
+        segname:         __TEXT
+        addr:            0x100000360
+        size:            5
+        offset:          0x360
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '7465737400'
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          4294983680
+    vmsize:          960
+    fileoff:         16384
+    filesize:        960
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      0
+    rebase_size:     0
+    bind_off:        0
+    bind_size:       0
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      16384
+    export_size:     96
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          16488
+    nsyms:           22
+    stroff:          16840
+    strsize:         192
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       17
+    iextdefsym:      17
+    nextdefsym:      5
+    iundefsym:       22
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         32
+    name:            12
+    Content:         '/usr/lib/dyld'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            4C4C4480-5555-3144-A138-E5DA50CC68DB
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           720896
+    sdk:             720896
+    ntools:          1
+    Tools:
+      - tool:            4
+        version:         1376256
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        768
+    stacksize:       0
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         16480
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         16488
+    datasize:        0
+  - cmd:             LC_CODE_SIGNATURE
+    cmdsize:         16
+    dataoff:         17040
+    datasize:        304
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      43
+            Name:            _mh_execute_header
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      47
+            Name:            main
+            Flags:           0x0
+            Address:         0x300
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    0
+            NodeOffset:      52
+            Name:            function
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    0
+                NodeOffset:      71
+                Name:            2_copy
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    3
+                    NodeOffset:      79
+                    Name:            '1'
+                    Flags:           0x0
+                    Address:         0x2F0
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    3
+                    NodeOffset:      84
+                    Name:            '2'
+                    Flags:           0x0
+                    Address:         0x2F0
+                    Other:           0x0
+                    ImportName:      ''
+              - TerminalSize:    3
+                NodeOffset:      89
+                Name:            3_copy2
+                Flags:           0x0
+                Address:         0x2F8
+                Other:           0x0
+                ImportName:      ''
+  NameList:
+    - n_strx:          129
+      n_type:          0x64
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+    - n_strx:          170
+      n_type:          0x66
+      n_sect:          0
+      n_desc:          1
+      n_value:         0
+    - n_strx:          59
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968152
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         4
+    - n_strx:          84
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968156
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         4
+    - n_strx:          2
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968064
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         88
+    - n_strx:          8
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968048
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         8
+    - n_strx:          25
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968056
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         8
+    - n_strx:          42
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968048
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         8
+    - n_strx:          1
+      n_type:          0x64
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          59
+      n_type:          0x1E
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968152
+    - n_strx:          84
+      n_type:          0x1E
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968156
+    - n_strx:          2
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968064
+    - n_strx:          8
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968048
+    - n_strx:          25
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968056
+    - n_strx:          42
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294968048
+    - n_strx:          109
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          16
+      n_value:         4294967296
+  StringTable:
+    - ' '
+    - _main
+    - _function2_copy1
+    - _function3_copy2
+    - _function2_copy2
+    - __ZN12length_errorC1EPKc
+    - __ZN12length_errorC2EPKc
+    - __mh_execute_header
+    - '/private/tmp/stmt_seq/stmt_seq_macho.cpp'
+    - stmt_seq_macho.o
+    - ''
+    - ''
+    - ''
+    - ''
+    - ''
+  FunctionStarts:  [ 0x2F0, 0x2F8, 0x300, 0x358, 0x35C ]
+...
diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/stmt_seq/stmt_seq_macho.exe b/llvm/test/tools/dsymutil/Inputs/private/tmp/stmt_seq/stmt_seq_macho.exe
deleted file mode 100755
index 4dd4ee8deb0b4f5dfed1c148d9cc302e466a4059..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 17360
zcmeI4T}V_>5P)azY6)g#T3J$UA<_T3TLe+jYA$9<Yi6)oaJ2R6F1bJLUP~)zO~j%n
zDF~tmdn!dEqNgAbqUf!BC?hO-=%J`!E9no^X70Vmb}KRJbq401Gc)JR&iT&W$GH!`
zzaO6kU;z{bH4F6yPCCJcGk`^?yHJIwEGl)^x~n|oO%;7}s_Qq4^1MbBqT21PP7Uk(
zU8z1W?R0dLQf(wcXlkow%nds=Uji3WzKFSmY`Ir=r9s>b>4Olym=+C$noQ@f_0?I8
zd{hGo*~yZU8v{RiRg~>1cbAmOv$~8tkK}%|Nu*6RCv#i@5HHkMO!IJIO}K>W$V2tJ
z02bi*pa|e8j+v;Hs3h5@(phNDzsPLdWJASw2J9`dsJ%IG$llP_+<-PoJ}T8wQBiVq
z|9(fwm3uR{m2@pU??j&s)i@(nm8s9ua0W0RR^fSRA4onlM_GwF7q!j>kb&cB)24Hh
zEJ3AsPccBDk(}I9ix>6|TE!FGluhHrT^JhmSbE1!D7|AH&|L@dsxK207e85{&z2sq
z`i%2j$jPj;#H&7-{(1|{U1qaBez^m>X8>G?D><2CCyIMVJw?5kubc8>DY-OiYAoq(
zow_WJFaajO1egF5U;<2l2`~XBzyz286JP>NfC(@GCcp%k025#WOn?b60Vco%m;e)C
z0!)AjFaajO1egF5U;<2l2`~XBzyz286JP>NfC(@GCcp%k025#WOn?b60Vco%z#_1}
z-YSBAp|+~N7EKj?wZ1`(!rEYcAO!ZtmXJ>ighQ~2TFk8w$Bt*kDSY8b8<cL;M<>*P
z;sB2ms;!W~Y{ug&^_eHqGcvr`<lT;Q*&|NW`%oLrZg8@<q84D&v={rosg0Yo0M+OZ
z2cR~3iXWUJPQ_82`p1*^d(lNgz2g*@-v^*Il%M=0)VELZub3!#zl^Uj<5TUYJq`3Q
zh0e1SZFwH5pKMt_)jNmQoAuJ~kS+65J!H#z==YQ3LEWa#ccWe#2>rf<yf5R595CsS
zcQxrSRygsK3blJ2&So{#q~S{tjfSH;9q!6<-=6|bGw}QCVvj@vN9#4!t_36Zm=@GT
zOg%C=!Qy=W@CIKb0)MqItRD;?-(Soc#1j^p$0WD@nE>?DP33`tZ-}CQ{-70O^1AWW
zEAFu=`ezDg&ottaaf__nEExY2GGvTrR=z#Zy!`2*@!@wZ&vO@VY47mg?pp9|_4Vba
zM=lLt&mTC|f72Vj|E6&;ZFp$tu;THiPn^7U@_6U7JLh(tU3c|=<8ua&%}Zo&>R<n=
bYxkFrr>|b#mo-pOQ+ln_*D<m=F}nQ+VXQf7

diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/stmt_seq/stmt_seq_macho.o b/llvm/test/tools/dsymutil/Inputs/private/tmp/stmt_seq/stmt_seq_macho.o
deleted file mode 100644
index 76fba6580055bb63e95693ec1cefcb7d19118233..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 4704
zcmb7HZERat89v9?*Ky*+iQ}Y++ir7NVI3$=?1m=oKANzFC2iT9G_Z6NyR{uBapkYs
zPRX{=h5}MSlh!~hs=*itklOt~AR$qNkU9;CN&B%@LZC@Zq7o7k5@-_zAy665bM86$
za9pjt%6s1L=bm%!d(L&<{`X)1+bV<<APzS8q8CFH42ppLv~NR`0XJ|(=NP*QZZfbH
zIZiFRRMQ58<4ljuOt0HUThR*fJkUEjTpZyQCZsYPCsVCea>aR_Uyo;!xgOH6QCw4D
zYgGxyna!pb=bd~hqwPnZ9G&{~TFO_o_8qlTLOfK5hcKtRMg}jHPFJ!gx!%oqyL9%v
z#(T%NR%(=nc)8+Si3#<1XDzqv$GU37@7m+ozo4r&#KR`4Xg%IOJ)fub{QlPBf@~P;
z@KWV+KI;@yg=`f`^?0jn29Q_dO_;`T-5eX@mC`5F3#rGupx3us*MFvM9Y|{p@zngv
zsZ2fIFY54~y+gd(bLEEXeOKeXp(k_UL-E}Ho2y_0QByy^?=`zX*EHS?UMgSt6>~ne
z?4R!ET#;~F@XGpq-q3h^+Fj+n+u$V-?ghKAPak^NF)&KRo)GQ{NNzgw24M3M*C5Y9
zvLXp4ao%LM#MZWg^G>Kkgsxi7@5GFlRI<+g;0<y8<`eSrtrt9(Z+%&;9v3T9fIa_r
zOS~4fUA;?$1IOjMA8R|eONiA!9u%w0UGaniw_f<v<(p58UIyN(dX5lXl(lSi{WjBg
zA2ugxRQ>vKZT&zdU%lOeOtvM;)|Mm#VNMFrjmjM-HAXklE;otOGAdsU>COrpPRl^-
zbdiW|)TnkfMz7C8q0V0u16q4fOG|{rtHv1DPAM`*H6IWuX4f!GR!Dv{7`Cn8ndM+d
z&yL`Y(~*$u#I5ET6cgbCfd>K}ti+OXB8=_W1>X>Q)czP0A$+@q6?`}lfTxLSAn-6$
ze@qhM&!;0VF87S|J|D4y#F+>mi~wDTW?OiAj1R3nxXBE-)s0JAk=wgo+BPmvhrft^
z3{Jp*3KMM)%Y6Y(gipkg+lhG&LkY#CTlQc?%_4(&blM0GLJ3nr!`9^p(cP{B+cqy~
zHvD{~7<>#BJy;2*<D&GshPzb!BH-N4$jFx@?}FEG;N33v(-gZz0pki)&>2^#aRsC2
zAO?AFvqghs9me2&5`!&lwuA2?2if!rgulrg0d=(u!X-;T<!F5$k5xC~BY3K7w(Lh_
zjHZ`S;Wx#Uhgem5ougmXJ_A+NDpUB)nPKvArj@R9Y*oqMf~u00*>Vl0dQhADAaf#E
zyTq!}4UQpI`wyt9R+%j^Oz<&IvQyOtq6m{_z%PMi;eC#GGurBTsr_;0-uouTmPrgr
zn)fi+W<ecBE-5IJa<6Fn4b(@KL%BR1_lvfdVRK_iOnX4Ib)(#z7r5{g^q2ilBuKfv
z<1fG!A!Y68>_BQbfPoo&qLkaZl<evBt;f1q)XjI<&8cqVE|Az6KL^XwAd=e=`aQTu
zCZ{1T!PP4#k3p3?_>4#ngDJ(tEL@#IISDZd%w+<Ci1`*+md+ymz7N3^zJtiHy=q<}
zlTQ`rqbIYKYOYj_CgQPy*uZ?Q7TvQ@tCgz<`ukxlET&_bQlUSUU)r1K&*x7T`pT8k
ziEO48W9t5){lf$CR5qR1H#?Y045fz$6T_*@P<&|b?Ce}TJve}jq5Yq{UsP*_np4d_
z>l9L%g;Fe2E{lHAU#{d%rfS*#TA|!;GDKz}Rlz5EbZTn!m~(XEsWE5zSaQs9#N1*r
zQ^Q>MI+;@Wx%ir$sI%R8qAs2gg;cI6^4a2iZNbS_Dy53Zm*#UBtvgRW9#3qHJ`x{G
z9?poZ{)F=9iZziEX;H~m7xOid71hOps4e8GqAJFZj*ere)k)<+FrO;=q;P*t;(e)q
zFg|EN=id2O>Fitk-k$KscV6#&=IGa;AA&yhiSp4Cr(a%PLG5|)55YeJ{~G*T@Sngp
z!E9LKv$@$vq}RcGKF<J2-5roT{3JFc<E$aw0MllJlhhRe$#X?wvoU1w)lr4b79*KG
z(B_ss;IjXp9R80%?jksdKbp9g7Q9~dlgeE)aT(*$Sibb$=ed_pS6}?Yz{=S`CVF9c
z7XB}QzYgZb`V+7^#E?9>xbaEq%tP`BYZtS*8s5;m!5}V{E?z>~brB|Ayl1qFQ|7e1
zXb8D<ck4Z{NoFao<+jU4KU;@nq{;7a-CO*<b$<PL5!L<~Et9Qxb$nO8;`yd^-b<Ik
zUD9#o+~D7M+c4!_uvdH>qm=SL!8Qsm87v!}bcBQ*Y!S`6wQzyiw(k(iT(4Q*_Dx>1
zN~=v8e-|;#Dy<OwhXDd#l#c*ecS*ao<>oK#X5SgR)pywLg!;a12afPzpJlmOq_thr
z_c{8^0^##`iRcnju@`;EA<<HOGmwZKnRewvM<CIXeIt-qL*F1I4tL)PtP^>@NiDZ&
zc|~oI@cl^3RV`0xIie;1FT|hkP2f>}TT4gFAxJ>_KBDcbT3*s}Mau;(*^0y&)-tN4
zKGMElY5N;mp3-toOH+S9+fANnZiPBRK>!<~8pOm_wBsvv_I<G1<<<4inzqgMSCzfq
z?~~f@g(RiHyrv9|K%s3q1@A!FQAkC3C&vHj@zKG}@7(StZxGN||K=}FLO90ZesVTA
R%r~hnBQEmzwz=bg_#YC=MLqxk


From 40050888a11a92a2fbf4da5dda5956ca9ecb6b55 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Tue, 15 Apr 2025 09:59:23 -0700
Subject: [PATCH 011/710] [clang][depscan] Centralize logic for populating
 StableDirs, NFC (#135704)

Pass a reference to `StableDirs` when creating ModuleDepCollector. This
avoids needing to create one from the same ScanInstance for each call to
`handleTopLevelModule` & reduces the amount of potential downstream
changes needed for handling StableDirs.
---
 .../DependencyScanning/ModuleDepCollector.h   |  6 +++-
 .../DependencyScanningWorker.cpp              | 29 ++++++++++---------
 .../DependencyScanning/ModuleDepCollector.cpp | 20 +++++--------
 3 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
index ce5e67d2624d9..d2d0d56e5212c 100644
--- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
+++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
@@ -282,7 +282,8 @@ class ModuleDepCollector final : public DependencyCollector {
                      CompilerInstance &ScanInstance, DependencyConsumer &C,
                      DependencyActionController &Controller,
                      CompilerInvocation OriginalCI,
-                     const PrebuiltModulesAttrsMap PrebuiltModulesASTMap);
+                     const PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
+                     const ArrayRef<StringRef> StableDirs);
 
   void attachToPreprocessor(Preprocessor &PP) override;
   void attachToASTReader(ASTReader &R) override;
@@ -305,6 +306,9 @@ class ModuleDepCollector final : public DependencyCollector {
   /// Mapping from prebuilt AST filepaths to their attributes referenced during
   /// dependency collecting.
   const PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
+  /// Directory paths known to be stable through an active development and build
+  /// cycle.
+  const ArrayRef<StringRef> StableDirs;
   /// Path to the main source file.
   std::string MainFile;
   /// Hash identifying the compilation conditions of the current TU.
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 6595f8ff5dc55..bae436afe0897 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -100,7 +100,7 @@ class PrebuiltModuleListener : public ASTReaderListener {
                          PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
                          const HeaderSearchOptions &HSOpts,
                          const LangOptions &LangOpts, DiagnosticsEngine &Diags,
-                         const llvm::SmallVector<StringRef> &StableDirs)
+                         const ArrayRef<StringRef> StableDirs)
       : PrebuiltModuleFiles(PrebuiltModuleFiles),
         NewModuleFiles(NewModuleFiles),
         PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
@@ -199,7 +199,7 @@ class PrebuiltModuleListener : public ASTReaderListener {
   const LangOptions &ExistingLangOpts;
   DiagnosticsEngine &Diags;
   std::string CurrentFile;
-  const llvm::SmallVector<StringRef> &StableDirs;
+  const ArrayRef<StringRef> StableDirs;
 };
 
 /// Visit the given prebuilt module and collect all of the modules it
@@ -208,16 +208,8 @@ static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
                                 CompilerInstance &CI,
                                 PrebuiltModuleFilesT &ModuleFiles,
                                 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
-                                DiagnosticsEngine &Diags) {
-
-  // Gather the set of stable directories to use as transitive dependencies are
-  // discovered.
-  llvm::SmallVector<StringRef> StableDirs;
-  std::string SysrootToUse(CI.getHeaderSearchOpts().Sysroot);
-  if (!SysrootToUse.empty() &&
-      (llvm::sys::path::root_directory(SysrootToUse) != SysrootToUse))
-    StableDirs = {SysrootToUse, CI.getHeaderSearchOpts().ResourceDir};
-
+                                DiagnosticsEngine &Diags,
+                                const ArrayRef<StringRef> StableDirs) {
   // List of module files to be processed.
   llvm::SmallVector<std::string> Worklist;
 
@@ -448,6 +440,15 @@ class DependencyScanningAction : public tooling::ToolAction {
     auto *FileMgr = ScanInstance.createFileManager(FS);
     ScanInstance.createSourceManager(*FileMgr);
 
+    // Create a collection of stable directories derived from the ScanInstance
+    // for determining whether module dependencies would fully resolve from
+    // those directories.
+    llvm::SmallVector<StringRef> StableDirs;
+    const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
+    if (!Sysroot.empty() &&
+        (llvm::sys::path::root_directory(Sysroot) != Sysroot))
+      StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
+
     // Store a mapping of prebuilt module files and their properties like header
     // search options. This will prevent the implicit build to create duplicate
     // modules and will force reuse of the existing prebuilt module files
@@ -459,7 +460,7 @@ class DependencyScanningAction : public tooling::ToolAction {
               ScanInstance.getPreprocessorOpts().ImplicitPCHInclude,
               ScanInstance,
               ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
-              PrebuiltModulesASTMap, ScanInstance.getDiagnostics()))
+              PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs))
         return false;
 
     // Create the dependency collector that will collect the produced
@@ -489,7 +490,7 @@ class DependencyScanningAction : public tooling::ToolAction {
     case ScanningOutputFormat::Full:
       MDC = std::make_shared<ModuleDepCollector>(
           Service, std::move(Opts), ScanInstance, Consumer, Controller,
-          OriginalInvocation, std::move(PrebuiltModulesASTMap));
+          OriginalInvocation, std::move(PrebuiltModulesASTMap), StableDirs);
       ScanInstance.addDependencyCollector(MDC);
       break;
     }
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index ebd392fbfa7d6..429bf823616da 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -763,14 +763,9 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   MD.IsSystem = M->IsSystem;
 
   // Start off with the assumption that this module is shareable when there
-  // is a sysroot provided. As more dependencies are discovered, check if those
-  // come from the provided shared directories.
-  const llvm::SmallVector<StringRef> StableDirs = {
-      MDC.ScanInstance.getHeaderSearchOpts().Sysroot,
-      MDC.ScanInstance.getHeaderSearchOpts().ResourceDir};
-  MD.IsInStableDirectories =
-      !StableDirs[0].empty() &&
-      (llvm::sys::path::root_directory(StableDirs[0]) != StableDirs[0]);
+  // are stable directories. As more dependencies are discovered, check if those
+  // come from the provided directories.
+  MD.IsInStableDirectories = !MDC.StableDirs.empty();
 
   // For modules which use export_as link name, the linked product that of the
   // corresponding export_as-named module.
@@ -817,7 +812,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
           auto FullFilePath = ASTReader::ResolveImportedPath(
               PathBuf, IFI.UnresolvedImportedFilename, MF->BaseDirectory);
           MD.IsInStableDirectories =
-              isPathInStableDir(StableDirs, *FullFilePath);
+              isPathInStableDir(MDC.StableDirs, *FullFilePath);
         }
         if (!(IFI.TopLevel && IFI.ModuleMap))
           return;
@@ -864,7 +859,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   // IsInStableDirectories.
   if (MD.IsInStableDirectories)
     MD.IsInStableDirectories =
-        areOptionsInStableDir(StableDirs, CI.getHeaderSearchOpts());
+        areOptionsInStableDir(MDC.StableDirs, CI.getHeaderSearchOpts());
 
   MDC.associateWithContextHash(CI, IgnoreCWD, MD);
 
@@ -978,11 +973,12 @@ ModuleDepCollector::ModuleDepCollector(
     std::unique_ptr<DependencyOutputOptions> Opts,
     CompilerInstance &ScanInstance, DependencyConsumer &C,
     DependencyActionController &Controller, CompilerInvocation OriginalCI,
-    const PrebuiltModulesAttrsMap PrebuiltModulesASTMap)
+    const PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
+    const ArrayRef<StringRef> StableDirs)
     : Service(Service), ScanInstance(ScanInstance), Consumer(C),
       Controller(Controller),
       PrebuiltModulesASTMap(std::move(PrebuiltModulesASTMap)),
-      Opts(std::move(Opts)),
+      StableDirs(StableDirs), Opts(std::move(Opts)),
       CommonInvocation(
           makeCommonInvocationForModuleBuild(std::move(OriginalCI))) {}
 

From 3192ecfa89a48b5f56ff36956abe7e84327ced5d Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Tue, 15 Apr 2025 10:12:18 -0700
Subject: [PATCH 012/710] [CodeComplete] Don't drop ArrayToPointerDecay when
 doing member completion (#134951)

Fixes https://github.com/llvm/llvm-project/issues/123146.

rdar://138851576
---
 clang/lib/Sema/SemaCodeComplete.cpp       | 5 ++++-
 clang/test/CodeCompletion/member-access.c | 9 +++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 793ffb6a00b86..f6ec4cb0f069e 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -5726,7 +5726,10 @@ class ConceptInfo {
 QualType getApproximateType(const Expr *E, HeuristicResolver &Resolver) {
   if (E->getType().isNull())
     return QualType();
-  E = E->IgnoreParenImpCasts();
+  // Don't drop implicit cast if it's an array decay.
+  if (auto *ICE = dyn_cast<ImplicitCastExpr>(E);
+      !ICE || ICE->getCastKind() != CK_ArrayToPointerDecay)
+    E = E->IgnoreParenImpCasts();
   QualType Unresolved = E->getType();
   // Resolve DependentNameType
   if (const auto *DNT = Unresolved->getAs<DependentNameType>()) {
diff --git a/clang/test/CodeCompletion/member-access.c b/clang/test/CodeCompletion/member-access.c
index fc54993152815..f08d1957fb90a 100644
--- a/clang/test/CodeCompletion/member-access.c
+++ b/clang/test/CodeCompletion/member-access.c
@@ -36,3 +36,12 @@ void test4(struct Point *p) {
 }
 // RUN: %clang_cc1 -fsyntax-only -code-completion-with-fixits -code-completion-at=%s:%(line-3):13 %s -o - | FileCheck -check-prefix=CHECK-CC1 %s
 // RUN: %clang_cc1 -fsyntax-only -code-completion-with-fixits -code-completion-at=%s:%(line-3):23 %s -o - | FileCheck -check-prefix=CHECK-CC1 %s
+
+float test5(void) {
+  struct Point array[4];
+  return array->x;
+}
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-2):17 %s -o - | FileCheck -check-prefix=CHECK-CC4 %s
+// CHECK-CC4: COMPLETION: x : [#float#]x
+// CHECK-CC4: COMPLETION: y : [#float#]y
+// CHECK-CC4: COMPLETION: z : [#float#]z

From 1545f1139127b92be15fcd2964114028a2d07194 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Tue, 15 Apr 2025 10:39:08 -0700
Subject: [PATCH 013/710] [NFC][NVPTX] Use StringRef for Modifier arg in
 NVPTXInstPrinter (#135793)

- Use StringRef type for Modifier instead of const char *.
- Remove Modifier arg from functions that do not need them.
---
 .../NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp   | 28 +++++++-----------
 .../NVPTX/MCTargetDesc/NVPTXInstPrinter.h     | 29 ++++++++-----------
 2 files changed, 23 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index e42e738b9973f..4e2e4c99df803 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -95,10 +95,9 @@ void NVPTXInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
 }
 
 void NVPTXInstPrinter::printCvtMode(const MCInst *MI, int OpNum, raw_ostream &O,
-                                    const char *M) {
+                                    StringRef Modifier) {
   const MCOperand &MO = MI->getOperand(OpNum);
   int64_t Imm = MO.getImm();
-  llvm::StringRef Modifier(M);
 
   if (Modifier == "ftz") {
     // FTZ flag
@@ -155,10 +154,9 @@ void NVPTXInstPrinter::printCvtMode(const MCInst *MI, int OpNum, raw_ostream &O,
 }
 
 void NVPTXInstPrinter::printCmpMode(const MCInst *MI, int OpNum, raw_ostream &O,
-                                    const char *M) {
+                                    StringRef Modifier) {
   const MCOperand &MO = MI->getOperand(OpNum);
   int64_t Imm = MO.getImm();
-  llvm::StringRef Modifier(M);
 
   if (Modifier == "ftz") {
     // FTZ flag
@@ -229,8 +227,7 @@ void NVPTXInstPrinter::printCmpMode(const MCInst *MI, int OpNum, raw_ostream &O,
 }
 
 void NVPTXInstPrinter::printLdStCode(const MCInst *MI, int OpNum,
-                                     raw_ostream &O, const char *M) {
-  llvm::StringRef Modifier(M);
+                                     raw_ostream &O, StringRef Modifier) {
   const MCOperand &MO = MI->getOperand(OpNum);
   int Imm = (int)MO.getImm();
   if (Modifier == "sem") {
@@ -329,10 +326,9 @@ void NVPTXInstPrinter::printLdStCode(const MCInst *MI, int OpNum,
 }
 
 void NVPTXInstPrinter::printMmaCode(const MCInst *MI, int OpNum, raw_ostream &O,
-                                    const char *M) {
+                                    StringRef Modifier) {
   const MCOperand &MO = MI->getOperand(OpNum);
   int Imm = (int)MO.getImm();
-  llvm::StringRef Modifier(M);
   if (Modifier.empty() || Modifier == "version") {
     O << Imm; // Just print out PTX version
     return;
@@ -346,9 +342,8 @@ void NVPTXInstPrinter::printMmaCode(const MCInst *MI, int OpNum, raw_ostream &O,
 }
 
 void NVPTXInstPrinter::printMemOperand(const MCInst *MI, int OpNum,
-                                       raw_ostream &O, const char *M) {
+                                       raw_ostream &O, StringRef Modifier) {
   printOperand(MI, OpNum, O);
-  llvm::StringRef Modifier(M);
 
   if (Modifier == "add") {
     O << ", ";
@@ -363,7 +358,7 @@ void NVPTXInstPrinter::printMemOperand(const MCInst *MI, int OpNum,
 }
 
 void NVPTXInstPrinter::printOffseti32imm(const MCInst *MI, int OpNum,
-                                         raw_ostream &O, const char *Modifier) {
+                                         raw_ostream &O) {
   auto &Op = MI->getOperand(OpNum);
   assert(Op.isImm() && "Invalid operand");
   if (Op.getImm() != 0) {
@@ -373,13 +368,13 @@ void NVPTXInstPrinter::printOffseti32imm(const MCInst *MI, int OpNum,
 }
 
 void NVPTXInstPrinter::printHexu32imm(const MCInst *MI, int OpNum,
-                                      raw_ostream &O, const char *Modifier) {
+                                      raw_ostream &O) {
   int64_t Imm = MI->getOperand(OpNum).getImm();
   O << formatHex(Imm) << "U";
 }
 
 void NVPTXInstPrinter::printProtoIdent(const MCInst *MI, int OpNum,
-                                       raw_ostream &O, const char *Modifier) {
+                                       raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNum);
   assert(Op.isExpr() && "Call prototype is not an MCExpr?");
   const MCExpr *Expr = Op.getExpr();
@@ -388,7 +383,7 @@ void NVPTXInstPrinter::printProtoIdent(const MCInst *MI, int OpNum,
 }
 
 void NVPTXInstPrinter::printPrmtMode(const MCInst *MI, int OpNum,
-                                     raw_ostream &O, const char *Modifier) {
+                                     raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
   int64_t Imm = MO.getImm();
 
@@ -419,10 +414,9 @@ void NVPTXInstPrinter::printPrmtMode(const MCInst *MI, int OpNum,
 }
 
 void NVPTXInstPrinter::printTmaReductionMode(const MCInst *MI, int OpNum,
-                                             raw_ostream &O,
-                                             const char *Modifier) {
+                                             raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
-  using RedTy = llvm::nvvm::TMAReductionOp;
+  using RedTy = nvvm::TMAReductionOp;
 
   switch (static_cast<RedTy>(MO.getImm())) {
   case RedTy::ADD:
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index 2b19386ef17fe..a2dd772cd86d0 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -37,25 +37,20 @@ class NVPTXInstPrinter : public MCInstPrinter {
 
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printCvtMode(const MCInst *MI, int OpNum, raw_ostream &O,
-                    const char *Modifier = nullptr);
+                    StringRef Modifier = {});
   void printCmpMode(const MCInst *MI, int OpNum, raw_ostream &O,
-                    const char *Modifier = nullptr);
-  void printLdStCode(const MCInst *MI, int OpNum,
-                     raw_ostream &O, const char *Modifier = nullptr);
+                    StringRef Modifier = {});
+  void printLdStCode(const MCInst *MI, int OpNum, raw_ostream &O,
+                     StringRef Modifier = {});
   void printMmaCode(const MCInst *MI, int OpNum, raw_ostream &O,
-                    const char *Modifier = nullptr);
-  void printMemOperand(const MCInst *MI, int OpNum,
-                       raw_ostream &O, const char *Modifier = nullptr);
-  void printOffseti32imm(const MCInst *MI, int OpNum, raw_ostream &O,
-                         const char *Modifier = nullptr);
-  void printHexu32imm(const MCInst *MI, int OpNum, raw_ostream &O,
-                      const char *Modifier = nullptr);
-  void printProtoIdent(const MCInst *MI, int OpNum,
-                       raw_ostream &O, const char *Modifier = nullptr);
-  void printPrmtMode(const MCInst *MI, int OpNum, raw_ostream &O,
-                     const char *Modifier = nullptr);
-  void printTmaReductionMode(const MCInst *MI, int OpNum, raw_ostream &O,
-                             const char *Modifier = nullptr);
+                    StringRef Modifier = {});
+  void printMemOperand(const MCInst *MI, int OpNum, raw_ostream &O,
+                       StringRef Modifier = {});
+  void printOffseti32imm(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printHexu32imm(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printProtoIdent(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printPrmtMode(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printTmaReductionMode(const MCInst *MI, int OpNum, raw_ostream &O);
 };
 
 }

From 68383fc20880f2d6ec6618d8870cd89e727bdc19 Mon Sep 17 00:00:00 2001
From: Nicolas van Kempen <nvankemp@gmail.com>
Date: Tue, 15 Apr 2025 13:42:54 -0400
Subject: [PATCH 014/710] [NFC][clang-tidy] Remove {{^}} clauses in some tests
 (1/N) (#134737)

`check_clang_tidy` now matches full lines only, so `{{^}}` clauses are
no longer necessary.

I am splitting those changes over multiple PRs to make review easier.
Numbering them but the actual order doesn't matter.
---
 .../checkers/performance/for-range-copy.cpp   |  16 +-
 .../performance/inefficient-algorithm.cpp     |  34 +-
 .../performance/trivially-destructible.cpp    |   6 +-
 .../performance/type-promotion-in-math-fn.cpp | 126 ++++----
 .../readability/delete-null-pointer.cpp       |  20 +-
 .../readability/else-after-return-no-warn.cpp |   6 +-
 ...g-hungarian-notation-lower-case-prefix.cpp | 302 +++++++++---------
 .../identifier-naming-outofline.cpp           |   6 +-
 .../readability/isolate-declaration-cxx17.cpp |  22 +-
 .../readability/isolate-declaration.c         |   2 +-
 .../redundant-declaration-ignore-macros.cpp   |  10 +-
 11 files changed, 274 insertions(+), 276 deletions(-)

diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance/for-range-copy.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance/for-range-copy.cpp
index 00e135bd2c920..0b5ef50fdbd7f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/performance/for-range-copy.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance/for-range-copy.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy --match-partial-fixes %s performance-for-range-copy %t -- -- -fno-delayed-template-parsing
+// RUN: %check_clang_tidy %s performance-for-range-copy %t -- -- -fno-delayed-template-parsing
 
 namespace std {
 
@@ -79,7 +79,7 @@ template <typename T>
 void uninstantiated() {
   for (const S S1 : View<Iterator<S>>()) {}
   // CHECK-MESSAGES: [[@LINE-1]]:16: warning: the loop variable's type is not a reference type; this creates a copy in each iteration; consider making this a reference [performance-for-range-copy]
-  // CHECK-FIXES: {{^}}  for (const S& S1 : View<Iterator<S>>()) {}
+  // CHECK-FIXES: for (const S& S1 : View<Iterator<S>>()) {}
 
   // Don't warn on dependent types.
   for (const T t1 : View<Iterator<T>>()) {
@@ -90,15 +90,15 @@ template <typename T>
 void instantiated() {
   for (const S S2 : View<Iterator<S>>()) {}
   // CHECK-MESSAGES: [[@LINE-1]]:16: warning: the loop variable's type is {{.*}}
-  // CHECK-FIXES: {{^}}  for (const S& S2 : View<Iterator<S>>()) {}
+  // CHECK-FIXES: for (const S& S2 : View<Iterator<S>>()) {}
 
   for (const auto [X, Y] : View<Iterator<Point>>()) {}
   // CHECK-MESSAGES: [[@LINE-1]]:19: warning: the loop variable's type is
-  // CHECK-FIXES: {{^}}  for (const auto& [X, Y] : View<Iterator<Point>>()) {}
+  // CHECK-FIXES: for (const auto& [X, Y] : View<Iterator<Point>>()) {}
 
   for (const T T2 : View<Iterator<T>>()) {}
   // CHECK-MESSAGES: [[@LINE-1]]:16: warning: the loop variable's type is {{.*}}
-  // CHECK-FIXES: {{^}}  for (const T& T2 : View<Iterator<T>>()) {}
+  // CHECK-FIXES: for (const T& T2 : View<Iterator<T>>()) {}
 }
 
 template <typename T>
@@ -311,10 +311,8 @@ View<Iterator<S>> createView(S) { return View<Iterator<S>>(); }
 
 void positiveValueIteratorUsedElseWhere() {
   for (const S SS : createView(*ValueReturningIterator<S>())) {
-    // CHECK-MESSAGES: [[@LINE-1]]:16: warning: the loop variable's type is not
-    // a reference type; this creates a copy in each iteration; consider making
-    // this a reference [performance-for-range-copy] CHECK-FIXES: for (const S&
-    // SS : createView(*ValueReturningIterator<S>())) {
+    // CHECK-MESSAGES: [[@LINE-1]]:16: warning: the loop variable's type is not a reference type; this creates a copy in each iteration; consider making this a reference [performance-for-range-copy]
+    // CHECK-FIXES: for (const S& SS : createView(*ValueReturningIterator<S>())) {
   }
 }
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance/inefficient-algorithm.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance/inefficient-algorithm.cpp
index 19a6701c5b6aa..dafff8c946bb0 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/performance/inefficient-algorithm.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance/inefficient-algorithm.cpp
@@ -69,83 +69,83 @@ template <typename T> void f(const T &t) {
   std::set<int> s;
   find(s.begin(), s.end(), 46);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}s.find(46);{{$}}
+  // CHECK-FIXES: s.find(46);
 
   find(t.begin(), t.end(), 46);
-  // CHECK-FIXES: {{^  }}find(t.begin(), t.end(), 46);{{$}}
+  // CHECK-FIXES: find(t.begin(), t.end(), 46);
 }
 
 int main() {
   std::set<int> s;
   auto it = std::find(s.begin(), s.end(), 43);
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: this STL algorithm call should be replaced with a container method [performance-inefficient-algorithm]
-  // CHECK-FIXES: {{^  }}auto it = s.find(43);{{$}}
+  // CHECK-FIXES: auto it = s.find(43);
   auto c = count(s.begin(), s.end(), 43);
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}auto c = s.count(43);{{$}}
+  // CHECK-FIXES: auto c = s.count(43);
 
 #define SECOND(x, y, z) y
   SECOND(q,std::count(s.begin(), s.end(), 22),w);
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}SECOND(q,s.count(22),w);{{$}}
+  // CHECK-FIXES: SECOND(q,s.count(22),w);
 
   it = find_if(s.begin(), s.end(), [](int) { return false; });
 
   std::multiset<int> ms;
   find(ms.begin(), ms.end(), 46);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}ms.find(46);{{$}}
+  // CHECK-FIXES: ms.find(46);
 
   const std::multiset<int> &msref = ms;
   find(msref.begin(), msref.end(), 46);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}msref.find(46);{{$}}
+  // CHECK-FIXES: msref.find(46);
 
   std::multiset<int> *msptr = &ms;
   find(msptr->begin(), msptr->end(), 46);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}msptr->find(46);{{$}}
+  // CHECK-FIXES: msptr->find(46);
 
   it = std::find(s.begin(), s.end(), 43, std::greater<int>());
   // CHECK-MESSAGES: :[[@LINE-1]]:42: warning: different comparers used in the algorithm and the container [performance-inefficient-algorithm]
 
   FIND_IN_SET(s);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}FIND_IN_SET(s);{{$}}
+  // CHECK-FIXES: FIND_IN_SET(s);
 
   f(s);
 
   std::unordered_set<int> us;
   lower_bound(us.begin(), us.end(), 10);
-  // CHECK-FIXES: {{^  }}lower_bound(us.begin(), us.end(), 10);{{$}}
+  // CHECK-FIXES: lower_bound(us.begin(), us.end(), 10);
   find(us.begin(), us.end(), 10);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}us.find(10);{{$}}
+  // CHECK-FIXES: us.find(10);
 
   std::unordered_multiset<int> ums;
   find(ums.begin(), ums.end(), 10);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}ums.find(10);{{$}}
+  // CHECK-FIXES: ums.find(10);
 
   std::map<int, int> intmap;
   find(intmap.begin(), intmap.end(), 46);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}find(intmap.begin(), intmap.end(), 46);{{$}}
+  // CHECK-FIXES: find(intmap.begin(), intmap.end(), 46);
 
   std::multimap<int, int> intmmap;
   find(intmmap.begin(), intmmap.end(), 46);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}find(intmmap.begin(), intmmap.end(), 46);{{$}}
+  // CHECK-FIXES: find(intmmap.begin(), intmmap.end(), 46);
 
   std::unordered_map<int, int> umap;
   find(umap.begin(), umap.end(), 46);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}find(umap.begin(), umap.end(), 46);{{$}}
+  // CHECK-FIXES: find(umap.begin(), umap.end(), 46);
 
   std::unordered_multimap<int, int> ummap;
   find(ummap.begin(), ummap.end(), 46);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}find(ummap.begin(), ummap.end(), 46);{{$}}
+  // CHECK-FIXES: find(ummap.begin(), ummap.end(), 46);
 }
 
 struct Value {
@@ -162,5 +162,5 @@ struct Ordering {
 void g(std::set<Value, Ordering> container, int value) {
   lower_bound(container.begin(), container.end(), value, Ordering());
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this STL algorithm call should be
-  // CHECK-FIXES: {{^  }}lower_bound(container.begin(), container.end(), value, Ordering());{{$}}
+  // CHECK-FIXES: lower_bound(container.begin(), container.end(), value, Ordering());
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance/trivially-destructible.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance/trivially-destructible.cpp
index 2ff3eda559a52..40cf90d21467a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/performance/trivially-destructible.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance/trivially-destructible.cpp
@@ -21,7 +21,7 @@ struct NotTriviallyDestructible1 : TriviallyDestructible2 {
 
 NotTriviallyDestructible1::~NotTriviallyDestructible1() = default; // to-be-removed
 // CHECK-MESSAGES: :[[@LINE-1]]:28: note: destructor definition is here
-// CHECK-FIXES: {{^}}// to-be-removed
+// CHECK-FIXES: // to-be-removed
 
 // Don't emit for class template with type-dependent fields.
 template <class T>
@@ -57,7 +57,7 @@ struct MaybeTriviallyDestructible1<T *> {
 template <class T>
 MaybeTriviallyDestructible1<T *>::~MaybeTriviallyDestructible1() noexcept = default; // to-be-removed
 // CHECK-MESSAGES: :[[@LINE-1]]:35: note: destructor definition is here
-// CHECK-FIXES: {{^}}// to-be-removed
+// CHECK-FIXES: // to-be-removed
 
 // Emit for explicit specializations.
 template <>
@@ -69,7 +69,7 @@ struct MaybeTriviallyDestructible1<double>: TriviallyDestructible1 {
 
 MaybeTriviallyDestructible1<double>::~MaybeTriviallyDestructible1() noexcept = default; // to-be-removed
 // CHECK-MESSAGES: :[[@LINE-1]]:38: note: destructor definition is here
-// CHECK-FIXES: {{^}}// to-be-removed
+// CHECK-FIXES: // to-be-removed
 
 struct NotTriviallyDestructible2 {
   virtual ~NotTriviallyDestructible2();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance/type-promotion-in-math-fn.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance/type-promotion-in-math-fn.cpp
index b2da7cc393a29..5309a1667d79a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/performance/type-promotion-in-math-fn.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance/type-promotion-in-math-fn.cpp
@@ -67,169 +67,169 @@ void check_all_fns() {
 
   acos(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'acos' promotes float to double [performance-type-promotion-in-math-fn]
-  // CHECK-FIXES: {{^}}  std::acos(a);{{$}}
+  // CHECK-FIXES: std::acos(a);
   acosh(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'acosh'
-  // CHECK-FIXES: {{^}}  std::acosh(a);{{$}}
+  // CHECK-FIXES: std::acosh(a);
   asin(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'asin'
-  // CHECK-FIXES: {{^}}  std::asin(a);{{$}}
+  // CHECK-FIXES: std::asin(a);
   asinh(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'asinh'
-  // CHECK-FIXES: {{^}}  std::asinh(a);{{$}}
+  // CHECK-FIXES: std::asinh(a);
   atan2(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'atan2'
-  // CHECK-FIXES: {{^}}  std::atan2(a, b);{{$}}
+  // CHECK-FIXES: std::atan2(a, b);
   atan(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'atan'
-  // CHECK-FIXES: {{^}}  std::atan(a);{{$}}
+  // CHECK-FIXES: std::atan(a);
   atanh(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'atanh'
-  // CHECK-FIXES: {{^}}  std::atanh(a);{{$}}
+  // CHECK-FIXES: std::atanh(a);
   cbrt(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'cbrt'
-  // CHECK-FIXES: {{^}}  std::cbrt(a);{{$}}
+  // CHECK-FIXES: std::cbrt(a);
   ceil(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'ceil'
-  // CHECK-FIXES: {{^}}  std::ceil(a);{{$}}
+  // CHECK-FIXES: std::ceil(a);
   copysign(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'copysign'
-  // CHECK-FIXES: {{^}}  std::copysign(a, b);{{$}}
+  // CHECK-FIXES: std::copysign(a, b);
   cos(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'cos'
-  // CHECK-FIXES: {{^}}  std::cos(a);{{$}}
+  // CHECK-FIXES: std::cos(a);
   cosh(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'cosh'
-  // CHECK-FIXES: {{^}}  std::cosh(a);{{$}}
+  // CHECK-FIXES: std::cosh(a);
   erf(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'erf'
-  // CHECK-FIXES: {{^}}  std::erf(a);{{$}}
+  // CHECK-FIXES: std::erf(a);
   erfc(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'erfc'
-  // CHECK-FIXES: {{^}}  std::erfc(a);{{$}}
+  // CHECK-FIXES: std::erfc(a);
   exp2(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'exp2'
-  // CHECK-FIXES: {{^}}  std::exp2(a);{{$}}
+  // CHECK-FIXES: std::exp2(a);
   exp(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'exp'
-  // CHECK-FIXES: {{^}}  std::exp(a);{{$}}
+  // CHECK-FIXES: std::exp(a);
   expm1(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'expm1'
-  // CHECK-FIXES: {{^}}  std::expm1(a);{{$}}
+  // CHECK-FIXES: std::expm1(a);
   fabs(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'fabs'
-  // CHECK-FIXES: {{^}}  std::fabs(a);{{$}}
+  // CHECK-FIXES: std::fabs(a);
   fdim(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'fdim'
-  // CHECK-FIXES: {{^}}  std::fdim(a, b);{{$}}
+  // CHECK-FIXES: std::fdim(a, b);
   floor(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'floor'
-  // CHECK-FIXES: {{^}}  std::floor(a);{{$}}
+  // CHECK-FIXES: std::floor(a);
   fma(a, b, c);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'fma'
-  // CHECK-FIXES: {{^}}  std::fma(a, b, c);{{$}}
+  // CHECK-FIXES: std::fma(a, b, c);
   fmax(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'fmax'
-  // CHECK-FIXES: {{^}}  std::fmax(a, b);{{$}}
+  // CHECK-FIXES: std::fmax(a, b);
   fmin(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'fmin'
-  // CHECK-FIXES: {{^}}  std::fmin(a, b);{{$}}
+  // CHECK-FIXES: std::fmin(a, b);
   fmod(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'fmod'
-  // CHECK-FIXES: {{^}}  std::fmod(a, b);{{$}}
+  // CHECK-FIXES: std::fmod(a, b);
   frexp(a, int_ptr);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'frexp'
-  // CHECK-FIXES: {{^}}  std::frexp(a, int_ptr);{{$}}
+  // CHECK-FIXES: std::frexp(a, int_ptr);
   hypot(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'hypot'
-  // CHECK-FIXES: {{^}}  std::hypot(a, b);{{$}}
+  // CHECK-FIXES: std::hypot(a, b);
   ilogb(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'ilogb'
-  // CHECK-FIXES: {{^}}  std::ilogb(a);{{$}}
+  // CHECK-FIXES: std::ilogb(a);
   ldexp(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'ldexp'
-  // CHECK-FIXES: {{^}}  std::ldexp(a, b);{{$}}
+  // CHECK-FIXES: std::ldexp(a, b);
   lgamma(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'lgamma'
-  // CHECK-FIXES: {{^}}  std::lgamma(a);{{$}}
+  // CHECK-FIXES: std::lgamma(a);
   llrint(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'llrint'
-  // CHECK-FIXES: {{^}}  std::llrint(a);{{$}}
+  // CHECK-FIXES: std::llrint(a);
   llround(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'llround'
-  // CHECK-FIXES: {{^}}  std::llround(a);{{$}}
+  // CHECK-FIXES: std::llround(a);
   log10(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'log10'
-  // CHECK-FIXES: {{^}}  std::log10(a);{{$}}
+  // CHECK-FIXES: std::log10(a);
   log1p(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'log1p'
-  // CHECK-FIXES: {{^}}  std::log1p(a);{{$}}
+  // CHECK-FIXES: std::log1p(a);
   log2(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'log2'
-  // CHECK-FIXES: {{^}}  std::log2(a);{{$}}
+  // CHECK-FIXES: std::log2(a);
   log(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'log'
-  // CHECK-FIXES: {{^}}  std::log(a);{{$}}
+  // CHECK-FIXES: std::log(a);
   logb(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'logb'
-  // CHECK-FIXES: {{^}}  std::logb(a);{{$}}
+  // CHECK-FIXES: std::logb(a);
   lrint(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'lrint'
-  // CHECK-FIXES: {{^}}  std::lrint(a);{{$}}
+  // CHECK-FIXES: std::lrint(a);
   lround(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'lround'
-  // CHECK-FIXES: {{^}}  std::lround(a);{{$}}
+  // CHECK-FIXES: std::lround(a);
   nearbyint(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'nearbyint'
-  // CHECK-FIXES: {{^}}  std::nearbyint(a);{{$}}
+  // CHECK-FIXES: std::nearbyint(a);
   nextafter(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'nextafter'
-  // CHECK-FIXES: {{^}}  std::nextafter(a, b);{{$}}
+  // CHECK-FIXES: std::nextafter(a, b);
   nexttoward(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'nexttoward'
-  // CHECK-FIXES: {{^}}  std::nexttoward(a, b);{{$}}
+  // CHECK-FIXES: std::nexttoward(a, b);
   pow(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'pow'
-  // CHECK-FIXES: {{^}}  std::pow(a, b);{{$}}
+  // CHECK-FIXES: std::pow(a, b);
   remainder(a, b);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'remainder'
-  // CHECK-FIXES: {{^}}  std::remainder(a, b);{{$}}
+  // CHECK-FIXES: std::remainder(a, b);
   remquo(a, b, int_ptr);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'remquo'
-  // CHECK-FIXES: {{^}}  std::remquo(a, b, int_ptr);{{$}}
+  // CHECK-FIXES: std::remquo(a, b, int_ptr);
   rint(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'rint'
-  // CHECK-FIXES: {{^}}  std::rint(a);{{$}}
+  // CHECK-FIXES: std::rint(a);
   round(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'round'
-  // CHECK-FIXES: {{^}}  std::round(a);{{$}}
+  // CHECK-FIXES: std::round(a);
   scalbln(a, l);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'scalbln'
-  // CHECK-FIXES: {{^}}  std::scalbln(a, l);{{$}}
+  // CHECK-FIXES: std::scalbln(a, l);
   scalbn(a, i);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'scalbn'
-  // CHECK-FIXES: {{^}}  std::scalbn(a, i);{{$}}
+  // CHECK-FIXES: std::scalbn(a, i);
   sin(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'sin'
-  // CHECK-FIXES: {{^}}  std::sin(a);{{$}}
+  // CHECK-FIXES: std::sin(a);
   sinh(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'sinh'
-  // CHECK-FIXES: {{^}}  std::sinh(a);{{$}}
+  // CHECK-FIXES: std::sinh(a);
   sqrt(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'sqrt'
-  // CHECK-FIXES: {{^}}  std::sqrt(a);{{$}}
+  // CHECK-FIXES: std::sqrt(a);
   tan(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'tan'
-  // CHECK-FIXES: {{^}}  std::tan(a);{{$}}
+  // CHECK-FIXES: std::tan(a);
   tanh(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'tanh'
-  // CHECK-FIXES: {{^}}  std::tanh(a);{{$}}
+  // CHECK-FIXES: std::tanh(a);
   tgamma(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'tgamma'
-  // CHECK-FIXES: {{^}}  std::tgamma(a);{{$}}
+  // CHECK-FIXES: std::tgamma(a);
   trunc(a);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'trunc'
-  // CHECK-FIXES: {{^}}  std::trunc(a);{{$}}
+  // CHECK-FIXES: std::trunc(a);
 }
 
 // nexttoward/nexttowardf are weird -- the second param is always long double.
@@ -237,16 +237,16 @@ void check_all_fns() {
 void check_nexttoward() {
   nexttoward(0.f, 0);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'nexttoward'
-  // CHECK-FIXES: {{^}}  std::nexttoward(0.f, 0);{{$}}
+  // CHECK-FIXES: std::nexttoward(0.f, 0);
   nexttoward(0.f, 0l);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'nexttoward'
-  // CHECK-FIXES: {{^}}  std::nexttoward(0.f, 0l);{{$}}
+  // CHECK-FIXES: std::nexttoward(0.f, 0l);
   nexttoward(0.f, 0.f);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'nexttoward'
-  // CHECK-FIXES: {{^}}  std::nexttoward(0.f, 0.f);{{$}}
+  // CHECK-FIXES: std::nexttoward(0.f, 0.f);
   nexttoward(0.f, 0.);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'nexttoward'
-  // CHECK-FIXES: {{^}}  std::nexttoward(0.f, 0.);{{$}}
+  // CHECK-FIXES: std::nexttoward(0.f, 0.);
 
   // No warnings for these.
   nexttoward(0., 0);
@@ -259,10 +259,10 @@ void check_nexttoward() {
 void check_scalbn() {
   scalbn(0.f, 0);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'scalbn'
-  // CHECK-FIXES: {{^}}  std::scalbn(0.f, 0);{{$}}
+  // CHECK-FIXES: std::scalbn(0.f, 0);
   scalbn(0.f, static_cast<char>(0));
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'scalbn'
-  // CHECK-FIXES: {{^}}  std::scalbn(0.f, static_cast<char>(0));{{$}}
+  // CHECK-FIXES: std::scalbn(0.f, static_cast<char>(0));
 
   // No warnings for these.
   scalbn(0., 0);
@@ -275,10 +275,10 @@ void check_scalbn() {
 void check_scalbln() {
   scalbln(0.f, 0);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'scalbln'
-  // CHECK-FIXES: {{^}}  std::scalbln(0.f, 0);{{$}}
+  // CHECK-FIXES: std::scalbln(0.f, 0);
   scalbln(0.f, 0l);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: call to 'scalbln'
-  // CHECK-FIXES: {{^}}  std::scalbln(0.f, 0l);{{$}}
+  // CHECK-FIXES: std::scalbln(0.f, 0l);
 
   // No warnings for these.
   scalbln(0., 0);
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/delete-null-pointer.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/delete-null-pointer.cpp
index 36e8f059c22b3..1a37c2bbf1133 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/delete-null-pointer.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/delete-null-pointer.cpp
@@ -10,7 +10,7 @@ struct Templ {
       delete mem;
     // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: 'if' statement is unnecessary;
     // CHECK-FIXES: // t1
-    // CHECK-FIXES-NEXT: {{^    }}// t2
+    // CHECK-FIXES-NEXT: // t2
     // CHECK-FIXES-NEXT: delete mem;
   }
   T mem;
@@ -24,7 +24,7 @@ struct TemplPtr {
       delete mem;
     // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: 'if' statement is unnecessary;
     // CHECK-FIXES: // t3
-    // CHECK-FIXES-NEXT: {{^    }}// t4
+    // CHECK-FIXES-NEXT: // t4
     // CHECK-FIXES-NEXT: delete mem;
   }
   T *mem;
@@ -45,7 +45,7 @@ struct NeverInstantiated {
       delete mem;
     // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: 'if' statement is unnecessary;
     // CHECK-FIXES: // t1
-    // CHECK-FIXES-NEXT: {{^    }}// t2
+    // CHECK-FIXES-NEXT: // t2
     // CHECK-FIXES-NEXT: delete mem;
   }
   T mem;
@@ -59,7 +59,7 @@ struct NeverInstantiatedPtr {
       delete mem;
     // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: 'if' statement is unnecessary;
     // CHECK-FIXES: // t3
-    // CHECK-FIXES-NEXT: {{^    }}// t4
+    // CHECK-FIXES-NEXT: // t4
     // CHECK-FIXES-NEXT: delete mem;
   }
   T *mem;
@@ -72,7 +72,7 @@ void f() {
   // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: 'if' statement is unnecessary; deleting null pointer has no effect [readability-delete-null-pointer]
 
   // CHECK-FIXES: int *ps = 0;
-  // CHECK-FIXES-NEXT: {{^  }}// #0
+  // CHECK-FIXES-NEXT: // #0
   // CHECK-FIXES-NEXT: delete ps;
 
   int *p = 0;
@@ -83,10 +83,10 @@ void f() {
   } // #3
   // CHECK-MESSAGES: :[[@LINE-3]]:3: warning: 'if' statement is unnecessary; deleting null pointer has no effect [readability-delete-null-pointer]
 
-  // CHECK-FIXES: {{^  }}// #1
-  // CHECK-FIXES-NEXT: {{^  }}// #2
+  // CHECK-FIXES: // #1
+  // CHECK-FIXES-NEXT: // #2
   // CHECK-FIXES-NEXT: delete p;
-  // CHECK-FIXES-NEXT: {{^  }}// #3
+  // CHECK-FIXES-NEXT: // #3
 
   int *p2 = new int[3];
   // #4
@@ -95,7 +95,7 @@ void f() {
   // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: 'if' statement is unnecessary;
 
   // CHECK-FIXES: // #4
-  // CHECK-FIXES-NEXT: {{^  }}// #5
+  // CHECK-FIXES-NEXT: // #5
   // CHECK-FIXES-NEXT: delete[] p2;
 
   int *p3 = 0;
@@ -136,7 +136,7 @@ void f() {
       if (mp) // #6
         delete mp;
       // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: 'if' statement is unnecessary; deleting null pointer has no effect [readability-delete-null-pointer]
-      // CHECK-FIXES: {{^      }}// #6
+      // CHECK-FIXES: // #6
       // CHECK-FIXES-NEXT: delete mp;
     }
     int *mp;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/else-after-return-no-warn.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/else-after-return-no-warn.cpp
index d17ed02148578..a67dc830b112e 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/else-after-return-no-warn.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/else-after-return-no-warn.cpp
@@ -20,12 +20,12 @@ int lifeTimeExtensionTests(int a) {
     b++;
   }
   if (int b = a) { // comment-0
-    // CHECK-FIXES:      {{^}}  int b = a;
-    // CHECK-FIXES-NEXT: {{^}}if (b) { // comment-0
+    // CHECK-FIXES:      int b = a;
+    // CHECK-FIXES-NEXT: if (b) { // comment-0
     return a;
   } else { // comment-0
            // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not use 'else' after 'return'
-           // CHECK-FIXES: {{^}}  } // comment-0
+           // CHECK-FIXES: } // comment-0
     return b;
   }
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp
index 65aee7e9f6ced..dd8a70299df42 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp
@@ -11,164 +11,164 @@ class C_MyClass1 {
 public:
   static int ClassMemberCase;
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for class member 'ClassMemberCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  static int i_ClassMemberCase;
+  // CHECK-FIXES: static int i_ClassMemberCase;
 
   char const ConstantMemberCase = 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for constant member 'ConstantMemberCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  char const c_ConstantMemberCase = 0;
+  // CHECK-FIXES: char const c_ConstantMemberCase = 0;
 
   void MyFunc1(const int ConstantParameterCase);
   // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: invalid case style for constant parameter 'ConstantParameterCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  void MyFunc1(const int i_ConstantParameterCase);
+  // CHECK-FIXES: void MyFunc1(const int i_ConstantParameterCase);
 
   void MyFunc2(const int* ConstantPointerParameterCase);
   // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: invalid case style for pointer parameter 'ConstantPointerParameterCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  void MyFunc2(const int* pi_ConstantPointerParameterCase);
+  // CHECK-FIXES: void MyFunc2(const int* pi_ConstantPointerParameterCase);
 
   static constexpr int ConstexprVariableCase = 123;
   // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for constexpr variable 'ConstexprVariableCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  static constexpr int i_ConstexprVariableCase = 123;
+  // CHECK-FIXES: static constexpr int i_ConstexprVariableCase = 123;
 };
 
 const int GlobalConstantCase = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global constant 'GlobalConstantCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const int i_GlobalConstantCase = 0;
+// CHECK-FIXES: const int i_GlobalConstantCase = 0;
 
 const int* GlobalConstantPointerCase = nullptr;
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global pointer 'GlobalConstantPointerCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const int* pi_GlobalConstantPointerCase = nullptr;
+// CHECK-FIXES: const int* pi_GlobalConstantPointerCase = nullptr;
 
 int* GlobalPointerCase = nullptr;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global pointer 'GlobalPointerCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int* pi_GlobalPointerCase = nullptr;
+// CHECK-FIXES: int* pi_GlobalPointerCase = nullptr;
 
 int GlobalVariableCase = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'GlobalVariableCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_GlobalVariableCase = 0;
+// CHECK-FIXES: int i_GlobalVariableCase = 0;
 
 void Func1(){
   int const LocalConstantCase = 3;
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for local constant 'LocalConstantCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  int const i_LocalConstantCase = 3;
+  // CHECK-FIXES: int const i_LocalConstantCase = 3;
 
   unsigned const ConstantCase = 1;
   // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for local constant 'ConstantCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  unsigned const u_ConstantCase = 1;
+  // CHECK-FIXES: unsigned const u_ConstantCase = 1;
 
   int* const LocalConstantPointerCase = nullptr;
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for local constant pointer 'LocalConstantPointerCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  int* const pi_LocalConstantPointerCase = nullptr;
+  // CHECK-FIXES: int* const pi_LocalConstantPointerCase = nullptr;
 
   int *LocalPointerCase = nullptr;
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for local pointer 'LocalPointerCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  int *pi_LocalPointerCase = nullptr;
+  // CHECK-FIXES: int *pi_LocalPointerCase = nullptr;
 
   int LocalVariableCase = 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for local variable 'LocalVariableCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  int i_LocalVariableCase = 0;
+  // CHECK-FIXES: int i_LocalVariableCase = 0;
 }
 
 class C_MyClass2 {
   char MemberCase;
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for private member 'MemberCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  char c_MemberCase;
+  // CHECK-FIXES: char c_MemberCase;
 
   void Func1(int ParameterCase);
   // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for parameter 'ParameterCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  void Func1(int i_ParameterCase);
+  // CHECK-FIXES: void Func1(int i_ParameterCase);
 
   void Func2(const int ParameterCase);
   // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for constant parameter 'ParameterCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  void Func2(const int i_ParameterCase);
+  // CHECK-FIXES: void Func2(const int i_ParameterCase);
 
   void Func3(const int *PointerParameterCase);
   // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: invalid case style for pointer parameter 'PointerParameterCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  void Func3(const int *pi_PointerParameterCase);
+  // CHECK-FIXES: void Func3(const int *pi_PointerParameterCase);
 };
 
 class C_MyClass3 {
 private:
   char PrivateMemberCase;
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for private member 'PrivateMemberCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  char c_PrivateMemberCase;
+  // CHECK-FIXES: char c_PrivateMemberCase;
 
 protected:
   char ProtectedMemberCase;
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for protected member 'ProtectedMemberCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  char c_ProtectedMemberCase;
+  // CHECK-FIXES: char c_ProtectedMemberCase;
 
 public:
   char PublicMemberCase;
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for public member 'PublicMemberCase' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  char c_PublicMemberCase;
+  // CHECK-FIXES: char c_PublicMemberCase;
 };
 
 static const int StaticConstantCase = 3;
 // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global constant 'StaticConstantCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}static const int i_StaticConstantCase = 3;
+// CHECK-FIXES: static const int i_StaticConstantCase = 3;
 
 static int StaticVariableCase = 3;
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'StaticVariableCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}static int i_StaticVariableCase = 3;
+// CHECK-FIXES: static int i_StaticVariableCase = 3;
 
 struct MyStruct { int StructCase; };
 // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: invalid case style for public member 'StructCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}struct MyStruct { int i_StructCase; };
+// CHECK-FIXES: struct MyStruct { int i_StructCase; };
 
 struct shouldBeCamelCaseStruct { int i_Field; };
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for struct 'shouldBeCamelCaseStruct' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}struct ShouldBeCamelCaseStruct { int i_Field; };
+// CHECK-FIXES: struct ShouldBeCamelCaseStruct { int i_Field; };
 
 union MyUnion { int UnionCase; long l_UnionCase; };
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for union 'MyUnion' [readability-identifier-naming]
 // CHECK-MESSAGES: :[[@LINE-2]]:21: warning: invalid case style for public member 'UnionCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}union myUnion { int i_UnionCase; long l_UnionCase; };
+// CHECK-FIXES: union myUnion { int i_UnionCase; long l_UnionCase; };
 
 //===----------------------------------------------------------------------===//
 // C string
 //===----------------------------------------------------------------------===//
 const char *NamePtr = "Name";
 // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global pointer 'NamePtr' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const char *sz_NamePtr = "Name";
+// CHECK-FIXES: const char *sz_NamePtr = "Name";
 
 const char NameArray[] = "Name";
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global constant 'NameArray' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const char sz_NameArray[] = "Name";
+// CHECK-FIXES: const char sz_NameArray[] = "Name";
 
 const char *NamePtrArray[] = {"AA", "BB"};
 // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'NamePtrArray' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const char *psz_NamePtrArray[] = {"AA", "BB"};
+// CHECK-FIXES: const char *psz_NamePtrArray[] = {"AA", "BB"};
 
 const wchar_t *WideNamePtr = L"Name";
 // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'WideNamePtr' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const wchar_t *wsz_WideNamePtr = L"Name";
+// CHECK-FIXES: const wchar_t *wsz_WideNamePtr = L"Name";
 
 const wchar_t WideNameArray[] = L"Name";
 // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global constant 'WideNameArray' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const wchar_t wsz_WideNameArray[] = L"Name";
+// CHECK-FIXES: const wchar_t wsz_WideNameArray[] = L"Name";
 
 const wchar_t *WideNamePtrArray[] = {L"AA", L"BB"};
 // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global variable 'WideNamePtrArray' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const wchar_t *pwsz_WideNamePtrArray[] = {L"AA", L"BB"};
+// CHECK-FIXES: const wchar_t *pwsz_WideNamePtrArray[] = {L"AA", L"BB"};
 
 class C_MyClass4 {
 private:
   char *Name = "Text";
   // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for private member 'Name' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  char *sz_Name = "Text";
+  // CHECK-FIXES: char *sz_Name = "Text";
 
   const char *ConstName = "Text";
   // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for private member 'ConstName' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  const char *sz_ConstName = "Text";
+  // CHECK-FIXES: const char *sz_ConstName = "Text";
 
 public:
   const char* DuplicateString(const char* Input, size_t n_RequiredSize);
   // CHECK-MESSAGES: :[[@LINE-1]]:43: warning: invalid case style for pointer parameter 'Input' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  const char* DuplicateString(const char* sz_Input, size_t n_RequiredSize);
+  // CHECK-FIXES: const char* DuplicateString(const char* sz_Input, size_t n_RequiredSize);
 
   size_t UpdateText(const char* Buffer, size_t n_BufferSize);
   // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: invalid case style for pointer parameter 'Buffer' [readability-identifier-naming]
-  // CHECK-FIXES: {{^}}  size_t UpdateText(const char* sz_Buffer, size_t n_BufferSize);
+  // CHECK-FIXES: size_t UpdateText(const char* sz_Buffer, size_t n_BufferSize);
 };
 
 
@@ -177,123 +177,123 @@ class C_MyClass4 {
 //===----------------------------------------------------------------------===//
 DWORD MsDword = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsDword' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}DWORD dw_MsDword = 0;
+// CHECK-FIXES: DWORD dw_MsDword = 0;
 
 BYTE MsByte = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsByte' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}BYTE by_MsByte = 0;
+// CHECK-FIXES: BYTE by_MsByte = 0;
 
 WORD MsWord = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsWord' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}WORD w_MsWord = 0;
+// CHECK-FIXES: WORD w_MsWord = 0;
 
 BOOL MsBool = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsBool' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}BOOL b_MsBool = 0;
+// CHECK-FIXES: BOOL b_MsBool = 0;
 
 BOOLEAN MsBoolean = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsBoolean' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}BOOLEAN b_MsBoolean = 0;
+// CHECK-FIXES: BOOLEAN b_MsBoolean = 0;
 
 CHAR MsValueChar = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueChar' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}CHAR c_MsValueChar = 0;
+// CHECK-FIXES: CHAR c_MsValueChar = 0;
 
 UCHAR MsValueUchar = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUchar' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}UCHAR uc_MsValueUchar = 0;
+// CHECK-FIXES: UCHAR uc_MsValueUchar = 0;
 
 SHORT MsValueShort = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueShort' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}SHORT s_MsValueShort = 0;
+// CHECK-FIXES: SHORT s_MsValueShort = 0;
 
 USHORT MsValueUshort = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUshort' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}USHORT us_MsValueUshort = 0;
+// CHECK-FIXES: USHORT us_MsValueUshort = 0;
 
 WORD MsValueWord = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueWord' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}WORD w_MsValueWord = 0;
+// CHECK-FIXES: WORD w_MsValueWord = 0;
 
 DWORD MsValueDword = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueDword' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}DWORD dw_MsValueDword = 0;
+// CHECK-FIXES: DWORD dw_MsValueDword = 0;
 
 DWORD32 MsValueDword32 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueDword32' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}DWORD32 dw32_MsValueDword32 = 0;
+// CHECK-FIXES: DWORD32 dw32_MsValueDword32 = 0;
 
 DWORD64 MsValueDword64 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueDword64' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}DWORD64 dw64_MsValueDword64 = 0;
+// CHECK-FIXES: DWORD64 dw64_MsValueDword64 = 0;
 
 LONG MsValueLong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}LONG l_MsValueLong = 0;
+// CHECK-FIXES: LONG l_MsValueLong = 0;
 
 ULONG MsValueUlong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUlong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}ULONG ul_MsValueUlong = 0;
+// CHECK-FIXES: ULONG ul_MsValueUlong = 0;
 
 ULONG32 MsValueUlong32 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueUlong32' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}ULONG32 ul32_MsValueUlong32 = 0;
+// CHECK-FIXES: ULONG32 ul32_MsValueUlong32 = 0;
 
 ULONG64 MsValueUlong64 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueUlong64' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}ULONG64 ul64_MsValueUlong64 = 0;
+// CHECK-FIXES: ULONG64 ul64_MsValueUlong64 = 0;
 
 ULONGLONG MsValueUlongLong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'MsValueUlongLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}ULONGLONG ull_MsValueUlongLong = 0;
+// CHECK-FIXES: ULONGLONG ull_MsValueUlongLong = 0;
 
 HANDLE MsValueHandle = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'MsValueHandle' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}HANDLE h_MsValueHandle = 0;
+// CHECK-FIXES: HANDLE h_MsValueHandle = 0;
 
 INT MsValueInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'MsValueInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}INT i_MsValueInt = 0;
+// CHECK-FIXES: INT i_MsValueInt = 0;
 
 INT8 MsValueInt8 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueInt8' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}INT8 i8_MsValueInt8 = 0;
+// CHECK-FIXES: INT8 i8_MsValueInt8 = 0;
 
 INT16 MsValueInt16 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueInt16' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}INT16 i16_MsValueInt16 = 0;
+// CHECK-FIXES: INT16 i16_MsValueInt16 = 0;
 
 INT32 MsValueInt32 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueInt32' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}INT32 i32_MsValueInt32 = 0;
+// CHECK-FIXES: INT32 i32_MsValueInt32 = 0;
 
 INT64 MsValueINt64 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueINt64' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}INT64 i64_MsValueINt64 = 0;
+// CHECK-FIXES: INT64 i64_MsValueINt64 = 0;
 
 UINT MsValueUint = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueUint' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}UINT ui_MsValueUint = 0;
+// CHECK-FIXES: UINT ui_MsValueUint = 0;
 
 UINT8 MsValueUint8 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUint8' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}UINT8 u8_MsValueUint8 = 0;
+// CHECK-FIXES: UINT8 u8_MsValueUint8 = 0;
 
 UINT16 MsValueUint16 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint16' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}UINT16 u16_MsValueUint16 = 0;
+// CHECK-FIXES: UINT16 u16_MsValueUint16 = 0;
 
 UINT32 MsValueUint32 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint32' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}UINT32 u32_MsValueUint32 = 0;
+// CHECK-FIXES: UINT32 u32_MsValueUint32 = 0;
 
 UINT64 MsValueUint64 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint64' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}UINT64 u64_MsValueUint64 = 0;
+// CHECK-FIXES: UINT64 u64_MsValueUint64 = 0;
 
 PVOID MsValuePvoid = NULL;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'MsValuePvoid' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}PVOID p_MsValuePvoid = NULL;
+// CHECK-FIXES: PVOID p_MsValuePvoid = NULL;
 
 
 //===----------------------------------------------------------------------===//
@@ -301,19 +301,19 @@ PVOID MsValuePvoid = NULL;
 //===----------------------------------------------------------------------===//
 unsigned GlobalUnsignedArray[] = {1, 2, 3};
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'GlobalUnsignedArray' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned a_GlobalUnsignedArray[] = {1, 2, 3};
+// CHECK-FIXES: unsigned a_GlobalUnsignedArray[] = {1, 2, 3};
 
 int GlobalIntArray[] = {1, 2, 3};
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'GlobalIntArray' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int a_GlobalIntArray[] = {1, 2, 3};
+// CHECK-FIXES: int a_GlobalIntArray[] = {1, 2, 3};
 
 int DataInt[1] = {0};
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'DataInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int a_DataInt[1] = {0};
+// CHECK-FIXES: int a_DataInt[1] = {0};
 
 int DataArray[2] = {0};
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'DataArray' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int a_DataArray[2] = {0};
+// CHECK-FIXES: int a_DataArray[2] = {0};
 
 
 //===----------------------------------------------------------------------===//
@@ -321,56 +321,56 @@ int DataArray[2] = {0};
 //===----------------------------------------------------------------------===//
 int *DataIntPtr[1] = {0};
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'DataIntPtr' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int *pa_DataIntPtr[1] = {0};
+// CHECK-FIXES: int *pa_DataIntPtr[1] = {0};
 
 void *BufferPtr1;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'BufferPtr1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}void *p_BufferPtr1;
+// CHECK-FIXES: void *p_BufferPtr1;
 
 void **BufferPtr2;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'BufferPtr2' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}void **pp_BufferPtr2;
+// CHECK-FIXES: void **pp_BufferPtr2;
 
 void **pBufferPtr3;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'pBufferPtr3' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}void **pp_BufferPtr3;
+// CHECK-FIXES: void **pp_BufferPtr3;
 
 int *pBufferPtr4;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global pointer 'pBufferPtr4' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int *pi_BufferPtr4;
+// CHECK-FIXES: int *pi_BufferPtr4;
 
 typedef void (*FUNC_PTR_HELLO)();
 FUNC_PTR_HELLO Hello = NULL;
 // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'Hello' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}FUNC_PTR_HELLO fn_Hello = NULL;
+// CHECK-FIXES: FUNC_PTR_HELLO fn_Hello = NULL;
 
 void *ValueVoidPtr = NULL;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'ValueVoidPtr' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}void *p_ValueVoidPtr = NULL;
+// CHECK-FIXES: void *p_ValueVoidPtr = NULL;
 
 ptrdiff_t PtrDiff = NULL;
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'PtrDiff' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}ptrdiff_t p_PtrDiff = NULL;
+// CHECK-FIXES: ptrdiff_t p_PtrDiff = NULL;
 
 int8_t *ValueI8Ptr;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global pointer 'ValueI8Ptr' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int8_t *pi8_ValueI8Ptr;
+// CHECK-FIXES: int8_t *pi8_ValueI8Ptr;
 
 uint8_t *ValueU8Ptr;
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global pointer 'ValueU8Ptr' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}uint8_t *pu8_ValueU8Ptr;
+// CHECK-FIXES: uint8_t *pu8_ValueU8Ptr;
 
 unsigned char *ValueUcPtr;
 // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'ValueUcPtr' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned char *puc_ValueUcPtr;
+// CHECK-FIXES: unsigned char *puc_ValueUcPtr;
 
 unsigned char **ValueUcPtr2;
 // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for global pointer 'ValueUcPtr2' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned char **ppuc_ValueUcPtr2;
+// CHECK-FIXES: unsigned char **ppuc_ValueUcPtr2;
 
 void MyFunc2(void* Val){}
 // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for pointer parameter 'Val' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}void MyFunc2(void* p_Val){}
+// CHECK-FIXES: void MyFunc2(void* p_Val){}
 
 
 //===----------------------------------------------------------------------===//
@@ -379,16 +379,16 @@ void MyFunc2(void* Val){}
 int i_ValueIndex = 1;
 int &RefValueIndex = i_ValueIndex;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'RefValueIndex' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int &i_RefValueIndex = i_ValueIndex;
+// CHECK-FIXES: int &i_RefValueIndex = i_ValueIndex;
 
 const int &ConstRefValue = i_ValueIndex;
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ConstRefValue' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}const int &i_ConstRefValue = i_ValueIndex;
+// CHECK-FIXES: const int &i_ConstRefValue = i_ValueIndex;
 
 long long ll_ValueLongLong = 2;
 long long &RefValueLongLong = ll_ValueLongLong;
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'RefValueLongLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}long long &ll_RefValueLongLong = ll_ValueLongLong;
+// CHECK-FIXES: long long &ll_RefValueLongLong = ll_ValueLongLong;
 
 
 //===----------------------------------------------------------------------===//
@@ -396,151 +396,151 @@ long long &RefValueLongLong = ll_ValueLongLong;
 //===----------------------------------------------------------------------===//
 int8_t ValueI8;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueI8' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int8_t i8_ValueI8;
+// CHECK-FIXES: int8_t i8_ValueI8;
 
 int16_t ValueI16 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI16' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int16_t i16_ValueI16 = 0;
+// CHECK-FIXES: int16_t i16_ValueI16 = 0;
 
 int32_t ValueI32 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI32' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int32_t i32_ValueI32 = 0;
+// CHECK-FIXES: int32_t i32_ValueI32 = 0;
 
 int64_t ValueI64 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI64' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int64_t i64_ValueI64 = 0;
+// CHECK-FIXES: int64_t i64_ValueI64 = 0;
 
 uint8_t ValueU8 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueU8' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}uint8_t u8_ValueU8 = 0;
+// CHECK-FIXES: uint8_t u8_ValueU8 = 0;
 
 uint16_t ValueU16 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU16' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}uint16_t u16_ValueU16 = 0;
+// CHECK-FIXES: uint16_t u16_ValueU16 = 0;
 
 uint32_t ValueU32 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU32' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}uint32_t u32_ValueU32 = 0;
+// CHECK-FIXES: uint32_t u32_ValueU32 = 0;
 
 uint64_t ValueU64 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU64' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}uint64_t u64_ValueU64 = 0;
+// CHECK-FIXES: uint64_t u64_ValueU64 = 0;
 
 float ValueFloat = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'ValueFloat' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}float f_ValueFloat = 0;
+// CHECK-FIXES: float f_ValueFloat = 0;
 
 double ValueDouble = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueDouble' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}double d_ValueDouble = 0;
+// CHECK-FIXES: double d_ValueDouble = 0;
 
 char ValueChar = 'c';
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueChar' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}char c_ValueChar = 'c';
+// CHECK-FIXES: char c_ValueChar = 'c';
 
 bool ValueBool = true;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueBool' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}bool b_ValueBool = true;
+// CHECK-FIXES: bool b_ValueBool = true;
 
 int ValueInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'ValueInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_ValueInt = 0;
+// CHECK-FIXES: int i_ValueInt = 0;
 
 size_t ValueSize = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueSize' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}size_t n_ValueSize = 0;
+// CHECK-FIXES: size_t n_ValueSize = 0;
 
 wchar_t ValueWchar = 'w';
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueWchar' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}wchar_t wc_ValueWchar = 'w';
+// CHECK-FIXES: wchar_t wc_ValueWchar = 'w';
 
 short ValueShort = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'ValueShort' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}short s_ValueShort = 0;
+// CHECK-FIXES: short s_ValueShort = 0;
 
 unsigned ValueUnsigned = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueUnsigned' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned u_ValueUnsigned = 0;
+// CHECK-FIXES: unsigned u_ValueUnsigned = 0;
 
 signed ValueSigned = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueSigned' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}signed s_ValueSigned = 0;
+// CHECK-FIXES: signed s_ValueSigned = 0;
 
 long ValueLong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}long l_ValueLong = 0;
+// CHECK-FIXES: long l_ValueLong = 0;
 
 long long ValueLongLong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'ValueLongLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}long long ll_ValueLongLong = 0;
+// CHECK-FIXES: long long ll_ValueLongLong = 0;
 
 long long int ValueLongLongInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueLongLongInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}long long int lli_ValueLongLongInt = 0;
+// CHECK-FIXES: long long int lli_ValueLongLongInt = 0;
 
 long double ValueLongDouble = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'ValueLongDouble' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}long double ld_ValueLongDouble = 0;
+// CHECK-FIXES: long double ld_ValueLongDouble = 0;
 
 signed int ValueSignedInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ValueSignedInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}signed int si_ValueSignedInt = 0;
+// CHECK-FIXES: signed int si_ValueSignedInt = 0;
 
 signed short ValueSignedShort = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'ValueSignedShort' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}signed short ss_ValueSignedShort = 0;
+// CHECK-FIXES: signed short ss_ValueSignedShort = 0;
 
 signed short int ValueSignedShortInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ValueSignedShortInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}signed short int ssi_ValueSignedShortInt = 0;
+// CHECK-FIXES: signed short int ssi_ValueSignedShortInt = 0;
 
 signed long long ValueSignedLongLong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ValueSignedLongLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}signed long long sll_ValueSignedLongLong = 0;
+// CHECK-FIXES: signed long long sll_ValueSignedLongLong = 0;
 
 signed long int ValueSignedLongInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for global variable 'ValueSignedLongInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}signed long int sli_ValueSignedLongInt = 0;
+// CHECK-FIXES: signed long int sli_ValueSignedLongInt = 0;
 
 signed long ValueSignedLong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'ValueSignedLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}signed long sl_ValueSignedLong = 0;
+// CHECK-FIXES: signed long sl_ValueSignedLong = 0;
 
 unsigned long long int ValueUnsignedLongLongInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for global variable 'ValueUnsignedLongLongInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned long long int ulli_ValueUnsignedLongLongInt = 0;
+// CHECK-FIXES: unsigned long long int ulli_ValueUnsignedLongLongInt = 0;
 
 unsigned long long ValueUnsignedLongLong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for global variable 'ValueUnsignedLongLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned long long ull_ValueUnsignedLongLong = 0;
+// CHECK-FIXES: unsigned long long ull_ValueUnsignedLongLong = 0;
 
 unsigned long int ValueUnsignedLongInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:19: warning: invalid case style for global variable 'ValueUnsignedLongInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned long int uli_ValueUnsignedLongInt = 0;
+// CHECK-FIXES: unsigned long int uli_ValueUnsignedLongInt = 0;
 
 unsigned long ValueUnsignedLong = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueUnsignedLong' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned long ul_ValueUnsignedLong = 0;
+// CHECK-FIXES: unsigned long ul_ValueUnsignedLong = 0;
 
 unsigned short int ValueUnsignedShortInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for global variable 'ValueUnsignedShortInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned short int usi_ValueUnsignedShortInt = 0;
+// CHECK-FIXES: unsigned short int usi_ValueUnsignedShortInt = 0;
 
 unsigned short ValueUnsignedShort = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global variable 'ValueUnsignedShort' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned short us_ValueUnsignedShort = 0;
+// CHECK-FIXES: unsigned short us_ValueUnsignedShort = 0;
 
 unsigned int ValueUnsignedInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'ValueUnsignedInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned int ui_ValueUnsignedInt = 0;
+// CHECK-FIXES: unsigned int ui_ValueUnsignedInt = 0;
 
 unsigned char ValueUnsignedChar = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueUnsignedChar' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}unsigned char uc_ValueUnsignedChar = 0;
+// CHECK-FIXES: unsigned char uc_ValueUnsignedChar = 0;
 
 long int ValueLongInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueLongInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}long int li_ValueLongInt = 0;
+// CHECK-FIXES: long int li_ValueLongInt = 0;
 
 
 //===----------------------------------------------------------------------===//
@@ -548,25 +548,25 @@ long int ValueLongInt = 0;
 //===----------------------------------------------------------------------===//
 volatile int VolatileInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'VolatileInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}volatile int i_VolatileInt = 0;
+// CHECK-FIXES: volatile int i_VolatileInt = 0;
 
 thread_local int ThreadLocalValueInt = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ThreadLocalValueInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}thread_local int i_ThreadLocalValueInt = 0;
+// CHECK-FIXES: thread_local int i_ThreadLocalValueInt = 0;
 
 extern int ExternValueInt;
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ExternValueInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}extern int i_ExternValueInt;
+// CHECK-FIXES: extern int i_ExternValueInt;
 
 struct DataBuffer {
     mutable size_t Size;
 };
 // CHECK-MESSAGES: :[[@LINE-2]]:20: warning: invalid case style for public member 'Size' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}    mutable size_t n_Size;
+// CHECK-FIXES: mutable size_t n_Size;
 
 static constexpr int const &ConstExprInt = 42;
 // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: invalid case style for constexpr variable 'ConstExprInt' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}static constexpr int const &i_ConstExprInt = 42;
+// CHECK-FIXES: static constexpr int const &i_ConstExprInt = 42;
 
 
 //===----------------------------------------------------------------------===//
@@ -575,7 +575,7 @@ static constexpr int const &ConstExprInt = 42;
 typedef int INDEX;
 INDEX iIndex = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'iIndex' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}INDEX Index = 0;
+// CHECK-FIXES: INDEX Index = 0;
 
 
 //===----------------------------------------------------------------------===//
@@ -583,96 +583,96 @@ INDEX iIndex = 0;
 //===----------------------------------------------------------------------===//
 class ClassCase { int Func(); };
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for class 'ClassCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}class C_ClassCase { int Func(); };
+// CHECK-FIXES: class C_ClassCase { int Func(); };
 
 class AbstractClassCase { virtual int Func() = 0; };
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for abstract class 'AbstractClassCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}class I_AbstractClassCase { virtual int Func() = 0; };
+// CHECK-FIXES: class I_AbstractClassCase { virtual int Func() = 0; };
 
 class AbstractClassCase1 { virtual int Func1() = 0; int Func2(); };
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for abstract class 'AbstractClassCase1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}class I_AbstractClassCase1 { virtual int Func1() = 0; int Func2(); };
+// CHECK-FIXES: class I_AbstractClassCase1 { virtual int Func1() = 0; int Func2(); };
 
 class ClassConstantCase { public: static const int i_ConstantCase; };
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for class 'ClassConstantCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}class C_ClassConstantCase { public: static const int i_ConstantCase; };
+// CHECK-FIXES: class C_ClassConstantCase { public: static const int i_ConstantCase; };
 
 //===----------------------------------------------------------------------===//
 // Other Cases
 //===----------------------------------------------------------------------===//
 int lower_case = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_LowerCase = 0;
+// CHECK-FIXES: int i_LowerCase = 0;
 
 int lower_case1 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_LowerCase1 = 0;
+// CHECK-FIXES: int i_LowerCase1 = 0;
 
 int lower_case_2 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case_2' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_LowerCase2 = 0;
+// CHECK-FIXES: int i_LowerCase2 = 0;
 
 int UPPER_CASE = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'UPPER_CASE' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_UpperCase = 0;
+// CHECK-FIXES: int i_UpperCase = 0;
 
 int UPPER_CASE_1 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'UPPER_CASE_1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_UpperCase1 = 0;
+// CHECK-FIXES: int i_UpperCase1 = 0;
 
 int camelBack = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelBack = 0;
+// CHECK-FIXES: int i_CamelBack = 0;
 
 int camelBack_1 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack_1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelBack1 = 0;
+// CHECK-FIXES: int i_CamelBack1 = 0;
 
 int camelBack2 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack2' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelBack2 = 0;
+// CHECK-FIXES: int i_CamelBack2 = 0;
 
 int CamelCase = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelCase = 0;
+// CHECK-FIXES: int i_CamelCase = 0;
 
 int CamelCase_1 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase_1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelCase1 = 0;
+// CHECK-FIXES: int i_CamelCase1 = 0;
 
 int CamelCase2 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase2' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelCase2 = 0;
+// CHECK-FIXES: int i_CamelCase2 = 0;
 
 int camel_Snake_Back = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camel_Snake_Back' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelSnakeBack = 0;
+// CHECK-FIXES: int i_CamelSnakeBack = 0;
 
 int camel_Snake_Back_1 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camel_Snake_Back_1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelSnakeBack1 = 0;
+// CHECK-FIXES: int i_CamelSnakeBack1 = 0;
 
 int Camel_Snake_Case = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'Camel_Snake_Case' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelSnakeCase = 0;
+// CHECK-FIXES: int i_CamelSnakeCase = 0;
 
 int Camel_Snake_Case_1 = 0;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'Camel_Snake_Case_1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}int i_CamelSnakeCase1 = 0;
+// CHECK-FIXES: int i_CamelSnakeCase1 = 0;
 
 //===----------------------------------------------------------------------===//
 // Enum
 //===----------------------------------------------------------------------===//
 enum REV_TYPE { RevValid };
 // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for enum constant 'RevValid' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}enum REV_TYPE { rt_RevValid };
+// CHECK-FIXES: enum REV_TYPE { rt_RevValid };
 
 enum EnumConstantCase { OneByte, TwoByte };
 // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: invalid case style for enum constant 'OneByte' [readability-identifier-naming]
 // CHECK-MESSAGES: :[[@LINE-2]]:34: warning: invalid case style for enum constant 'TwoByte' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}enum EnumConstantCase { ecc_OneByte, ecc_TwoByte };
+// CHECK-FIXES: enum EnumConstantCase { ecc_OneByte, ecc_TwoByte };
 
 enum class ScopedEnumConstantCase { Case1 };
 // CHECK-MESSAGES: :[[@LINE-1]]:37: warning: invalid case style for scoped enum constant 'Case1' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}enum class ScopedEnumConstantCase { secc_Case1 };
+// CHECK-FIXES: enum class ScopedEnumConstantCase { secc_Case1 };
 // clang-format on
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-outofline.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-outofline.cpp
index f807875e27698..2945047dee4ca 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-outofline.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-outofline.cpp
@@ -11,11 +11,11 @@ class SomeClass {
     template <typename T>
     int someMethod();
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for method 'someMethod' [readability-identifier-naming]
-// CHECK-FIXES: {{^}}    int SomeMethod();
+// CHECK-FIXES: int SomeMethod();
 };
 template <typename T>
 int SomeClass::someMethod() {
-// CHECK-FIXES: {{^}}int SomeClass::SomeMethod() {
+// CHECK-FIXES: int SomeClass::SomeMethod() {
     return 5;
 }
 
@@ -24,7 +24,7 @@ int SomeClass::someMethod() {
 void someFunc() {
     Inner::SomeClass S;
     S.someMethod<int>();
-// CHECK-FIXES: {{^}}    S.SomeMethod<int>();
+// CHECK-FIXES: S.SomeMethod<int>();
 }
 
 } // namespace SomeNamespace
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/isolate-declaration-cxx17.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/isolate-declaration-cxx17.cpp
index b50ad4ce25839..a811db1879ecb 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/isolate-declaration-cxx17.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/isolate-declaration-cxx17.cpp
@@ -63,31 +63,31 @@ void modern() {
   auto autoInt1 = 3, autoInt2 = 4;
   // CHECK-MESSAGES: [[@LINE-1]]:3: warning: multiple declarations in a single statement reduces readability
   // CHECK-FIXES: auto autoInt1 = 3;
-  // CHECK-FIXES: {{^  }}auto autoInt2 = 4;
+  // CHECK-FIXES: auto autoInt2 = 4;
 
   decltype(int()) declnottouch = 4;
   decltype(int()) declint1 = 5, declint2 = 3;
   // CHECK-MESSAGES: [[@LINE-1]]:3: warning: multiple declarations in a single statement reduces readability
   // CHECK-FIXES: decltype(int()) declint1 = 5;
-  // CHECK-FIXES: {{^  }}decltype(int()) declint2 = 3;
+  // CHECK-FIXES: decltype(int()) declint2 = 3;
 
   std::vector<int> vectorA = {1, 2}, vectorB = {1, 2, 3}, vectorC({1, 1, 1});
   // CHECK-MESSAGES: [[@LINE-1]]:3: warning: multiple declarations in a single statement reduces readability
   // CHECK-FIXES: std::vector<int> vectorA = {1, 2};
-  // CHECK-FIXES: {{^  }}std::vector<int> vectorB = {1, 2, 3};
-  // CHECK-FIXES: {{^  }}std::vector<int> vectorC({1, 1, 1});
+  // CHECK-FIXES: std::vector<int> vectorB = {1, 2, 3};
+  // CHECK-FIXES: std::vector<int> vectorC({1, 1, 1});
 
   using uType = int;
   uType utype1, utype2;
   // CHECK-MESSAGES: [[@LINE-1]]:3: warning: multiple declarations in a single statement reduces readability
   // CHECK-FIXES: uType utype1;
-  // CHECK-FIXES: {{^  }}uType utype2;
+  // CHECK-FIXES: uType utype2;
 
   Types::MyType mytype1, mytype2, mytype3 = 3;
   // CHECK-MESSAGES: [[@LINE-1]]:3: warning: multiple declarations in a single statement reduces readability
   // CHECK-FIXES: Types::MyType mytype1;
-  // CHECK-FIXES: {{^  }}Types::MyType mytype2;
-  // CHECK-FIXES: {{^  }}Types::MyType mytype3 = 3;
+  // CHECK-FIXES: Types::MyType mytype2;
+  // CHECK-FIXES: Types::MyType mytype3 = 3;
 
   {
     using namespace std::string_literals;
@@ -95,9 +95,9 @@ void modern() {
     std::vector<std::string> s{"foo"s, "bar"s}, t{"foo"s}, u, a({"hey", "you"}), bb = {"h", "a"};
     // CHECK-MESSAGES: [[@LINE-1]]:5: warning: multiple declarations in a single statement reduces readability
     // CHECK-FIXES: std::vector<std::string> s{"foo"s, "bar"s};
-    // CHECK-FIXES: {{^    }}std::vector<std::string> t{"foo"s};
-    // CHECK-FIXES: {{^    }}std::vector<std::string> u;
-    // CHECK-FIXES: {{^    }}std::vector<std::string> a({"hey", "you"});
-    // CHECK-FIXES: {{^    }}std::vector<std::string> bb = {"h", "a"};
+    // CHECK-FIXES: std::vector<std::string> t{"foo"s};
+    // CHECK-FIXES: std::vector<std::string> u;
+    // CHECK-FIXES: std::vector<std::string> a({"hey", "you"});
+    // CHECK-FIXES: std::vector<std::string> bb = {"h", "a"};
   }
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/isolate-declaration.c b/clang-tools-extra/test/clang-tidy/checkers/readability/isolate-declaration.c
index e1960a15b8d83..1354a29ffedaf 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/isolate-declaration.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/isolate-declaration.c
@@ -6,7 +6,7 @@ void c_specific(void) {
   int j = sizeof(struct T { int i; }), k;
   // CHECK-MESSAGES: [[@LINE-1]]:3: warning: multiple declarations in a single statement reduces readability
   // CHECK-FIXES: int j = sizeof(struct T { int i; });
-  // CHECK-FIXES: {{^  }}int k;
+  // CHECK-FIXES: int k;
 
   void g(struct U { int i; } s);                // One decl
   void h(struct V { int i; } s), m(int i, ...); // Two decls
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-declaration-ignore-macros.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-declaration-ignore-macros.cpp
index 1a124adc55f6a..1fe7d1d356db3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-declaration-ignore-macros.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-declaration-ignore-macros.cpp
@@ -6,16 +6,16 @@
 extern int Xyz;
 extern int Xyz; // Xyz
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: redundant 'Xyz' declaration [readability-redundant-declaration]
-// CHECK-FIXES: {{^}}// Xyz{{$}}
+// CHECK-FIXES: // Xyz
 
 namespace macros {
 #define DECLARE(x) extern int x
 #define DEFINE(x) extern int x; int x = 42
 DECLARE(test);
 DEFINE(test);
-// CHECK-FIXES: {{^}}#define DECLARE(x) extern int x{{$}}
-// CHECK-FIXES: {{^}}#define DEFINE(x) extern int x; int x = 42{{$}}
-// CHECK-FIXES: {{^}}DECLARE(test);{{$}}
-// CHECK-FIXES: {{^}}DEFINE(test);{{$}}
+// CHECK-FIXES: #define DECLARE(x) extern int x
+// CHECK-FIXES: #define DEFINE(x) extern int x; int x = 42
+// CHECK-FIXES: DECLARE(test);
+// CHECK-FIXES: DEFINE(test);
 
 } // namespace macros

From 7a41761407c485d18b7d48232b308556b3b43934 Mon Sep 17 00:00:00 2001
From: Wanyi <wanyi@meta.com>
Date: Tue, 15 Apr 2025 13:46:15 -0400
Subject: [PATCH 015/710] [lldb] Make SBProcess thread related actions listen
 to StopLocker (#134339)

# Summary

This PR updates `SBProcess::GetNumThreads()` and
`SBProcess::GetThreadAtIndex()` to listen to the stop locker.
`SBProcess::GetNumThreads()` will return 0 if the process is running.

## Problem Description

Recently upon debugging a program with thousands of threads in VS Code,
lldb-dap would hang at a `threads` request sent right after receiving
the `configurationDone` response. Soon after it will end the debug
session with the following error
```
Process <pid> exited with status = -1 (0xffffffff) lost connection
```

This is because LLDB is still in the middle of resuming all the threads.
And requesting threads will end up interrupt the process on Linux. From
the gdb-remote log it ended up getting `lldb::StateType::eStateInvalid`
and just exit with status -1.

I don't think it's reasonable to allow getting threads from a running
process. There are a few approaches to fix this:
1) Send the stopped event to IDE after `configurationDone`. This aligns
with the CLI behavior.
2) However, the above approach will break the existing user facing
behavior. The alternative will be reject the `threads` request if the
process is not stopped.
3) Improve the run lock. This is a synchronize issue where process was
in the middle of resuming while lldb-dap attempts to interrupt it.

**This PR implements the option 3**

## HOWEVER

This fixed the "lost connection" issue below but new issue has surfaced.
From testing, and also from checking the [VSCode source
code](https://github.com/microsoft/vscode/blob/174af221c9ea2ccdb64abe4aab8e1a805e77beae/src/vs/workbench/contrib/debug/browser/debugSession.ts#L791),
it expects having threadID to perform `pause`. So after attaching,
without any threads reported to the client, the user will not be able to
pause the attached process. `setBreakpoint` will still work and once we
make a stop at the bp (or any stop that will report threads, client can
perform pause again.

## NEXT
1) Made an attempt to return initial thread list so that VSCode can
pause (second commit in the PR)
2) Investigate why threads will trigger unwinding the second frame of a
thread, which leads to sending the interrupt
3) Decided if we want to support `stopOnEntry` for attaching, given
  i. This is not an official specification
ii. If enable stopOnEntry, we need to fix attaching on Linux, to send
only one stopped event. Currently, all threads upon attaching will have
stop reason `SIGSTOP` and lldb-dap will send `stopped` event for each
one of them. Every `stopped` will trigger the client request for
threads.
iii. Alternatively, we can support auto continue correspond to `(lldb)
process attach --continue`. This require the ii above.


### Additionally

lldb-dap will not send a `continued` event after `configurationDone`
because it checks `dap.focus_tid == LLDB_INVALID_THREAD_ID` (so that we
don't send it for `launch` request). Notice `dap.focus_tid` will only
get assigned when handling stop or stepping.

According to DAP

> Please note: a debug adapter is not expected to send this event in
response to a request that implies that execution continues, e.g. launch
or continue.
It is only necessary to send a continued event if there was no previous
request that implied this.

So I guess we are not violating DAP if we don't send `continued` event.
But I'd like to get some sense about this.


## Test Plan
Used following program for testing:
https://gist.github.com/kusmour/1729d2e07b7b1063897db77de194e47d
**NOTE: Utilize stdin to get pid and attach AFTER hitting enter. Attach
should happen when all the threads start running.**

DAP messages before the change
<img width="1165" alt="image"
src="https://github.com/user-attachments/assets/a9ad85fb-81ce-419c-95e5-612639905c66"
/>

DAP message after the change - report zero threads after attaching
<img width="1165" alt="image"
src="https://github.com/user-attachments/assets/a1179e18-6844-437a-938c-0383702294cd"
/>

---------

Co-authored-by: Jonas Devlieghere <jonas@devlieghere.com>
---
 .../test/tools/lldb-dap/dap_server.py         |  4 ++++
 lldb/source/API/SBProcess.cpp                 | 20 ++++++++++---------
 .../tools/lldb-dap/attach/TestDAP_attach.py   |  2 +-
 lldb/tools/lldb-dap/DAP.h                     |  3 +++
 .../ConfigurationDoneRequestHandler.cpp       | 10 +++++++++-
 .../Handler/ThreadsRequestHandler.cpp         | 17 +++++++++++-----
 lldb/tools/lldb-dap/JSONUtils.cpp             | 13 ++++++++++++
 lldb/tools/lldb-dap/JSONUtils.h               |  2 ++
 8 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index 45403e9df8525..61d7fa94479b8 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -649,6 +649,10 @@ def request_configurationDone(self):
         response = self.send_recv(command_dict)
         if response:
             self.configuration_done_sent = True
+            # Client requests the baseline of currently existing threads after
+            # a successful launch or attach.
+            # Kick off the threads request that follows
+            self.request_threads()
         return response
 
     def _process_stopped(self):
diff --git a/lldb/source/API/SBProcess.cpp b/lldb/source/API/SBProcess.cpp
index 23ea449b30cca..ba77b2beed5ea 100644
--- a/lldb/source/API/SBProcess.cpp
+++ b/lldb/source/API/SBProcess.cpp
@@ -193,10 +193,11 @@ uint32_t SBProcess::GetNumThreads() {
   if (process_sp) {
     Process::StopLocker stop_locker;
 
-    const bool can_update = stop_locker.TryLock(&process_sp->GetRunLock());
-    std::lock_guard<std::recursive_mutex> guard(
-        process_sp->GetTarget().GetAPIMutex());
-    num_threads = process_sp->GetThreadList().GetSize(can_update);
+    if (stop_locker.TryLock(&process_sp->GetRunLock())) {
+      std::lock_guard<std::recursive_mutex> guard(
+          process_sp->GetTarget().GetAPIMutex());
+      num_threads = process_sp->GetThreadList().GetSize();
+    }
   }
 
   return num_threads;
@@ -393,11 +394,12 @@ SBThread SBProcess::GetThreadAtIndex(size_t index) {
   ProcessSP process_sp(GetSP());
   if (process_sp) {
     Process::StopLocker stop_locker;
-    const bool can_update = stop_locker.TryLock(&process_sp->GetRunLock());
-    std::lock_guard<std::recursive_mutex> guard(
-        process_sp->GetTarget().GetAPIMutex());
-    thread_sp = process_sp->GetThreadList().GetThreadAtIndex(index, can_update);
-    sb_thread.SetThread(thread_sp);
+    if (stop_locker.TryLock(&process_sp->GetRunLock())) {
+      std::lock_guard<std::recursive_mutex> guard(
+          process_sp->GetTarget().GetAPIMutex());
+      thread_sp = process_sp->GetThreadList().GetThreadAtIndex(index, false);
+      sb_thread.SetThread(thread_sp);
+    }
   }
 
   return sb_thread;
diff --git a/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py b/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
index 9df44cc454d5d..b9fbf2c8d14f9 100644
--- a/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
+++ b/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
@@ -1,5 +1,5 @@
 """
-Test lldb-dap setBreakpoints request
+Test lldb-dap attach request
 """
 
 
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index 8d32a18fb711e..b79a0d9d0f25c 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -211,6 +211,9 @@ struct DAP {
   /// The set of features supported by the connected client.
   llvm::DenseSet<ClientFeature> clientFeatures;
 
+  /// The initial thread list upon attaching.
+  std::optional<llvm::json::Array> initial_thread_list;
+
   /// Creates a new DAP sessions.
   ///
   /// \param[in] log
diff --git a/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
index cd120e1fdfaba..f39bbdefdbb95 100644
--- a/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
@@ -44,6 +44,7 @@ namespace lldb_dap {
 //             just an acknowledgement, so no body field is required."
 //             }]
 // },
+
 void ConfigurationDoneRequestHandler::operator()(
     const llvm::json::Object &request) const {
   llvm::json::Object response;
@@ -52,8 +53,15 @@ void ConfigurationDoneRequestHandler::operator()(
   dap.configuration_done_sent = true;
   if (dap.stop_at_entry)
     SendThreadStoppedEvent(dap);
-  else
+  else {
+    // Client requests the baseline of currently existing threads after
+    // a successful launch or attach by sending a 'threads' request
+    // right after receiving the configurationDone response.
+    // Obtain the list of threads before we resume the process
+    dap.initial_thread_list =
+        GetThreads(dap.target.GetProcess(), dap.thread_format);
     dap.target.GetProcess().Continue();
+  }
 }
 
 } // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/Handler/ThreadsRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/ThreadsRequestHandler.cpp
index 2b857f7f6a02b..16d797c2ab327 100644
--- a/lldb/tools/lldb-dap/Handler/ThreadsRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/ThreadsRequestHandler.cpp
@@ -50,16 +50,23 @@ namespace lldb_dap {
 // }
 void ThreadsRequestHandler::operator()(
     const llvm::json::Object &request) const {
-  lldb::SBProcess process = dap.target.GetProcess();
   llvm::json::Object response;
   FillResponse(request, response);
 
-  const uint32_t num_threads = process.GetNumThreads();
   llvm::json::Array threads;
-  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-    lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
-    threads.emplace_back(CreateThread(thread, dap.thread_format));
+  // Client requests the baseline of currently existing threads after
+  // a successful launch or attach by sending a 'threads' request
+  // right after receiving the configurationDone response.
+  // If no thread has reported to the client, it prevents something
+  // like the pause request from working in the running state.
+  // Return the cache of initial threads as the process might have resumed
+  if (dap.initial_thread_list) {
+    threads = dap.initial_thread_list.value();
+    dap.initial_thread_list.reset();
+  } else {
+    threads = GetThreads(dap.target.GetProcess(), dap.thread_format);
   }
+
   if (threads.size() == 0) {
     response["success"] = llvm::json::Value(false);
   }
diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp
index 7660403666150..33f10c93d2ada 100644
--- a/lldb/tools/lldb-dap/JSONUtils.cpp
+++ b/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -870,6 +870,19 @@ llvm::json::Value CreateThread(lldb::SBThread &thread, lldb::SBFormat &format) {
   return llvm::json::Value(std::move(object));
 }
 
+llvm::json::Array GetThreads(lldb::SBProcess process, lldb::SBFormat &format) {
+  lldb::SBMutex lock = process.GetTarget().GetAPIMutex();
+  std::lock_guard<lldb::SBMutex> guard(lock);
+
+  llvm::json::Array threads;
+  const uint32_t num_threads = process.GetNumThreads();
+  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
+    lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
+    threads.emplace_back(CreateThread(thread, format));
+  }
+  return threads;
+}
+
 // "StoppedEvent": {
 //   "allOf": [ { "$ref": "#/definitions/Event" }, {
 //     "type": "object",
diff --git a/lldb/tools/lldb-dap/JSONUtils.h b/lldb/tools/lldb-dap/JSONUtils.h
index da91797290ff0..b8c53353bf42d 100644
--- a/lldb/tools/lldb-dap/JSONUtils.h
+++ b/lldb/tools/lldb-dap/JSONUtils.h
@@ -414,6 +414,8 @@ llvm::json::Value CreateExtendedStackFrameLabel(lldb::SBThread &thread,
 ///     definition outlined by Microsoft.
 llvm::json::Value CreateThread(lldb::SBThread &thread, lldb::SBFormat &format);
 
+llvm::json::Array GetThreads(lldb::SBProcess process, lldb::SBFormat &format);
+
 /// Create a "StoppedEvent" object for a LLDB thread object.
 ///
 /// This function will fill in the following keys in the returned

From 9b13d345303d819bb83de7ebbeb826d704add0bc Mon Sep 17 00:00:00 2001
From: Alexey Samsonov <vonosmas@gmail.com>
Date: Tue, 15 Apr 2025 10:49:52 -0700
Subject: [PATCH 016/710] [libc][bazel] Remove a no-op libc_internal_target
 macro. (#135818)

This macro is a no-op after 90c001ac9e1d92a1a95d191d1640ab5337a937e5:
libc_function macro now produce a "regular" cc_library target, without
modifying its name, and this target is intended to only be used in
tests.

Thus, libc_internal_target macro is no longer needed, and we can safely
treat libc_function rules and libc_support_library rules identically for
test purposes.

`libc_function_deps` attribute of a `libc_test` macro can also be
cleaned up, but I plan to do this in a subsequent change.
---
 .../llvm-project-overlay/libc/libc_build_rules.bzl   |  9 +--------
 .../libc/test/libc_test_rules.bzl                    | 12 +++++-------
 2 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
index 86dfb53a86014..60add23a46c48 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
@@ -10,10 +10,6 @@ load(":libc_configure_options.bzl", "LIBC_CONFIGURE_OPTIONS")
 load(":libc_namespace.bzl", "LIBC_NAMESPACE")
 load(":platforms.bzl", "PLATFORM_CPU_X86_64")
 
-# TODO: Remove this helper function once all donwstream users are migrated.
-def libc_internal_target(name):
-    return name
-
 def libc_common_copts():
     root_label = Label(":libc")
     libc_include_path = paths.join(root_label.workspace_root, root_label.package)
@@ -84,10 +80,7 @@ def libc_function(name, **kwargs):
     # Builds "internal" library with a function, exposed as a C++ function in
     # the "LIBC_NAMESPACE" namespace. This allows us to test the function in the
     # presence of another libc.
-    _libc_library(
-        name = libc_internal_target(name),
-        **kwargs
-    )
+    _libc_library(name = name, **kwargs)
 
 LibcLibraryInfo = provider(
     "All source files and textual headers for building a particular library.",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
index 8c20a9172989c..7e798429ef19b 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
@@ -12,26 +12,24 @@ They come in two flavors:
 When performing tests we make sure to always use the internal version.
 """
 
-load("//libc:libc_build_rules.bzl", "libc_common_copts", "libc_internal_target")
+load("//libc:libc_build_rules.bzl", "libc_common_copts")
 load("//libc:libc_configure_options.bzl", "LIBC_CONFIGURE_OPTIONS")
 
-def libc_test(name, srcs, libc_function_deps = [], copts = [], deps = [], local_defines = [], **kwargs):
+def libc_test(name, libc_function_deps = [], copts = [], deps = [], local_defines = [], **kwargs):
     """Add target for a libc test.
 
     Args:
       name: Test target name
-      srcs: List of sources for the test.
       libc_function_deps: List of libc_function targets used by this test.
       copts: The list of options to add to the C++ compilation command.
       deps: The list of other libraries to be linked in to the test target.
       local_defines: The list of target local_defines if any.
-      **kwargs: Attributes relevant for a libc_test. For example, name, srcs.
+      **kwargs: Attributes relevant for a cc_test.
     """
     native.cc_test(
         name = name,
-        srcs = srcs,
         local_defines = local_defines + LIBC_CONFIGURE_OPTIONS,
-        deps = [libc_internal_target(d) for d in libc_function_deps] + [
+        deps = [
             "//libc/test/UnitTest:LibcUnitTest",
             "//libc:__support_macros_config",
             "//libc:errno",
@@ -39,7 +37,7 @@ def libc_test(name, srcs, libc_function_deps = [], copts = [], deps = [], local_
             "//libc:func_free",
             "//libc:func_malloc",
             "//libc:func_realloc",
-        ] + deps,
+        ] + libc_function_deps + deps,
         copts = copts + libc_common_copts(),
         linkstatic = 1,
         **kwargs

From 3b9103044361094a8fde16a877f2e8cb0f96ce24 Mon Sep 17 00:00:00 2001
From: Andrew Rogers <andrurogerz@gmail.com>
Date: Tue, 15 Apr 2025 10:51:02 -0700
Subject: [PATCH 017/710] [llvm] add documentation for public interface
 annotations (LLVM_ABI, etc)

## Purpose
Add documentation for the existing family of `LLVM_ABI` annotation
macros defined in llvm/Support/Compiler.h. These annotations are used to
describe LLVM's public interface.

## Background
This documentation is in support of the annotation effort described
[here](https://discourse.llvm.org/t/psa-annotating-llvm-public-interface/85307/).

## Validation
Manually inspected rendered ReST document on GitHub.

Co-authored-by: Saleem Abdulrasool <compnerd@compnerd.org>
---
 llvm/docs/InterfaceExportAnnotations.rst | 376 +++++++++++++++++++++++
 llvm/docs/Reference.rst                  |   1 +
 2 files changed, 377 insertions(+)
 create mode 100644 llvm/docs/InterfaceExportAnnotations.rst

diff --git a/llvm/docs/InterfaceExportAnnotations.rst b/llvm/docs/InterfaceExportAnnotations.rst
new file mode 100644
index 0000000000000..eecf6ffe6eaca
--- /dev/null
+++ b/llvm/docs/InterfaceExportAnnotations.rst
@@ -0,0 +1,376 @@
+LLVM Interface Export Annotations
+=================================
+Symbols that are part of LLVM's public interface must be explicitly annotated
+to support shared library builds with hidden default symbol visibility. This
+document provides background and guidelines for annotating the codebase.
+
+LLVM Shared Library
+-------------------
+LLVM builds as a static library by default, but it can also be built as a shared
+library with the following configuration:
+
+::
+
+   LLVM_BUILD_LLVM_DYLIB=On
+   LLVM_LINK_LLVM_DYLIB=On
+
+There are three shared library executable formats we're interested in: PE
+Dynamic Link Library (.dll) on Windows, Mach-O Shared Object (.dylib) on Apple
+systems, and ELF Shared Object (.so) on Linux, BSD and other Unix-like systems.
+
+ELF and Mach-O Shared Object files can be built with no additional setup or
+configuration. This is because all global symbols in the library are exported by
+default -- the same as when building a static library. However, when building a
+DLL for Windows, the situation is more complex:
+
+- Symbols are not exported from a DLL by default. Symbols must be annotated with
+  ``__declspec(dllexport)`` when building the library to be externally visible.
+
+- Symbols imported from a Windows DLL should generally be annotated with
+  ``__declspec(dllimport)`` when compiling clients.
+
+- A single Windows DLL can export a maximum of 65,535 symbols.
+
+Because of the requirements for Windows DLLs, additional work must be done to
+ensure the proper set of public symbols is exported and visible to clients.
+
+Annotation Macros
+-----------------
+The distinct DLL import and export annotations required for Windows DLLs
+typically lead developers to define a preprocessor macro for annotating exported
+symbols in header public files. The custom macro resolves to the _export_
+annotation when building the library and the _import_ annotation when building
+the client.
+
+We have defined the `LLVM_ABI` macro in `llvm/Support/Compiler.h
+<https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/Support/Compiler.h#L152>`__
+for this purpose:
+
+.. code:: cpp
+
+   #if defined(LLVM_EXPORTS)
+   #define LLVM_ABI __declspec(dllexport)
+   #else
+   #define LLVM_ABI __declspec(dllimport)
+   #endif
+
+Windows DLL symbol visibility requirements are approximated on ELF and Mach-O
+shared library builds by setting default symbol visibility to hidden
+(``-fvisibility-default=hidden``) when building with the following
+configuration:
+
+::
+
+   LLVM_BUILD_LLVM_DYLIB_VIS=On
+
+For an ELF or Mach-O platform with this setting, the ``LLVM_ABI`` macro is
+defined to override the default hidden symbol visibility:
+
+.. code:: cpp
+
+   #define LLVM_ABI __attribute__((visibility("default")))
+
+In addition to ``LLVM_ABI``, there are a few other macros for use in less
+common cases described below.
+
+Export macros are used to annotate symbols only within their intended shared
+library. This is necessary because of the way Windows handles import/export
+annotations.
+
+For example, ``LLVM_ABI`` resolves to ``__declspec(dllexport)`` only when
+building source that is part of the LLVM shared library (e.g. source under
+``llvm-project/llvm``). If ``LLVM_ABI`` were incorrectly used to annotate a
+symbol from a different LLVM project (such as Clang) it would always resolve to
+``__declspec(dllimport)`` and the symbol would not be properly exported.
+
+Annotating Symbols
+------------------
+Functions
+~~~~~~~~~
+Exported function declarations in header files must be annotated with
+``LLVM_ABI``.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   LLVM_ABI void exported_function(int a, int b);
+
+Global Variables
+~~~~~~~~~~~~~~~~
+Exported global variables must be annotated with ``LLVM_ABI`` at their
+``extern`` declarations.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   LLVM_ABI extern int exported_global_variable;
+
+Classes, Structs, and Unions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Classes, structs, and unions can be annotated with ``LLVM_ABI`` at their
+declaration, but this option is generally discouraged because it will
+export every class member, vtable, and type information. Instead, ``LLVM_ABI``
+should be applied to individual class members that require export.
+
+In the most common case, public and protected methods without a body in the
+class declaration must be annotated with ``LLVM_ABI``.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   class ExampleClass {
+   public:
+     // Public methods defined externally must be annotatated.
+     LLVM_ABI int sourceDefinedPublicMethod(int a, int b);
+
+     // Methods defined in the class definition do not need annotation.
+     int headerDefinedPublicMethod(int a, int b) {
+       return a + b;
+     }
+
+     // Constructors and destructors must be annotated if defined externally.
+     ExampleClass() {}
+     LLVM_ABI ~ExampleClass();
+
+     // Public static methods defined externally must be annotatated.
+     LLVM_ABI static int sourceDefinedPublicStaticMethod(int a, int b);
+   };
+
+Additionally, public and protected static fields that are not initialized at
+declaration must be annotated with ``LLVM_ABI``.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   class ExampleClass {
+   public:
+     // Public static fields defined externally must be annotated.
+     LLVM_ABI static int mutableStaticField;
+     LLVM_ABI static const int constStaticField;
+
+     // Static members initialized at declaration do not need to be annotated.
+     static const int initializedConstStaticField = 0;
+     static constexpr int initializedConstexprStaticField = 0;
+   };
+
+Private methods may also require ``LLVM_ABI`` annotation in certain cases. This
+situation occurs when a method defined in a header calls the private method. The
+private method call may be from within the class, a parent class, or a friend
+class.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   class ExampleClass {
+   private:
+     // Private methods must be annotated if referenced by a public method defined a
+     // header file.
+     LLVM_ABI int privateMethod(int a, int b);
+
+   public:
+     // Inlineable method defined in the class definition calls a private method
+     // defined externally. If the private method is not annotated for export, this
+     // method will fail to link.
+     int publicMethod(int a, int b) {
+       return privateMethod(a, b);
+     }
+   };
+
+There are less common cases where you may also need to annotate an inline
+function even though it is fully defined in a header. Annotating an inline
+function for export does not prevent it being inlined into client code. However,
+it does ensure there is a single, stable address for the function exported from
+the shared library.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   // Annotate the function so it is exported from the library at a fixed
+   // address.
+   LLVM_ABI inline int inlineFunction(int a, int b) {
+     return a + b;
+   }
+
+Similarly, if a stable pointer-to-member function address is required for a
+method in a C++ class, it may be annotated for export.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   class ExampleClass {
+   public:
+     // Annotate the method so it is exported from the library at a fixed
+     // address.
+     LLVM_ABI inline int inlineMethod(int a, int b) {
+       return a + b;
+     }
+   };
+
+.. note::
+
+   When an inline function is annotated for export, the header containing the
+   function definition **must** be included by at least one of the library's
+   source files or the function will never be compiled with the export
+   annotation.
+
+Friend Functions
+~~~~~~~~~~~~~~~~
+Friend functions declared in a class, struct or union must be annotated with
+``LLVM_ABI`` if the corresponding function declaration is also annotated. This
+requirement applies even when the class itself is annotated with ``LLVM_ABI``.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   // An exported function that has friend access to ExampleClass internals.
+   LLVM_ABI int friend_function(ExampleClass &obj);
+
+   class ExampleClass {
+     // Friend declaration of a function must be annotated the same as the actual
+     // function declaration.
+     LLVM_ABI friend int friend_function(ExampleClass &obj);
+   };
+
+.. note::
+
+   Annotating the friend declaration avoids an “inconsistent dll linkage”
+   compiler error when building for Windows. This annotation is harmless but not
+   required when building ELF or Mach-O shared libraries.
+
+Virtual Table and Type Info
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Classes and structs with exported virtual methods, or child classes that export
+overridden virtual methods, must also export their vtable for ELF and Mach-O
+builds. This can be achieved by annotating the class rather than individual
+class members.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   class ParentClass {
+   public:
+     virtual int virtualMethod(int a, int b);
+     virtual int anotherVirtualMethod(int a, int b);
+     virtual ~ParentClass();
+   };
+
+   // Annotating the class exports vtable and type information as well as all
+   // class members.
+   class LLVM_ABI ChildClass : public ParentClass {
+   public:
+     // Inline method override does not require the class be annotated.
+     int virtualMethod(int a, int b) override {
+       return ParentClass::virtualMethod(a, b);
+     }
+
+     // Overriding a virtual method from the parent requires the class be
+     // annotated. The parent class may require annotation as well.
+     int pureVirtualMethod(int a, int b) override;
+     ~ChildClass();
+   };
+
+If annotating a type with ``LLVM_ABI`` causes compilation issues such as those
+described
+`here <https://devblogs.microsoft.com/oldnewthing/20190927-00/?p=102932>`__,
+the class may require modification. Often, explicitly deleting the copy
+constructor and copy assignment operator will resolve the issue.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   #include <vector>
+
+   class LLVM_ABI ExportedClass {
+   public:
+     // Explicitly delete the copy constructor and assignment operator.
+     ExportedClass(ExportedClass const&) = delete;
+     ExportedClass& operator=(ExportedClass const&) = delete;
+   };
+
+Templates
+~~~~~~~~~
+Most template classes are entirely header-defined and do not need to be exported
+because they will be instantiated and compiled into the client as needed. Such
+template classes require no export annotations. However, there are some less
+common cases where annotations are required for templates.
+
+Specialized Template Functions
+++++++++++++++++++++++++++++++
+As with any other exported function, an exported specialization of a template
+function not defined in a header file must have its declaration annotated with
+``LLVM_ABI``.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   template <typename T> T templateMethod(T a, T b) {
+     return a + b;
+   }
+
+   // The explicitly specialized definition of templateMethod for int is located in
+   // a source file. This declaration must be annotated with LLVM_ABI to export it.
+   template <> LLVM_ABI int templateMethod(int a, int b);
+
+Similarly, an exported specialization of a method in a template class must have
+its declaration annotated with ``LLVM_ABI``.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   template <typename T> class TemplateClass {
+   public:
+     int method(int a, int b) {
+       return a + b;
+     }
+   };
+
+   // The explicitly specialized definition of method for int is defined in a
+   // source file. The declaration must be annotated with LLVM_ABI to export it.
+   template <> LLVM_ABI int TemplateStruct<int>::method(int a, int b);
+
+Explicitly Instantiated Template Classes
+++++++++++++++++++++++++++++++++++++++++
+Explicitly instantiated template classes must be annotated with
+template-specific annotations at both declaration and definition.
+
+An extern template instantiation in a header file must be annotated with
+``LLVM_TEMPLATE_ABI``. This will typically be located in a header file.
+
+.. code:: cpp
+
+   #include "llvm/Support/Compiler.h"
+
+   template <typename T> class TemplateClass {
+   public:
+     TemplateClass(T val) : val_(val) {}
+
+     T get() const { return val_;  }
+
+   private:
+     const T val_;
+   };
+
+   // Explicitly instantiate and export TempalateClass for int type.
+   extern template class LLVM_TEMPLATE_ABI TemplateClass<int>;
+
+The corresponding definition of the template instantiation must be annotated
+with ``LLVM_EXPORT_TEMPLATE``. This will typically be located in a source file.
+
+.. code:: cpp
+
+   #include "TemplateClass.h"
+
+   // Explicitly instantiate and export TempalateClass for int type.
+   template class LLVM_EXPORT_TEMPLATE TemplateClass<int>;
diff --git a/llvm/docs/Reference.rst b/llvm/docs/Reference.rst
index 470b6bd024b89..e1f46b00f2b30 100644
--- a/llvm/docs/Reference.rst
+++ b/llvm/docs/Reference.rst
@@ -31,6 +31,7 @@ LLVM and API reference documentation.
    HowToSetUpLLVMStyleRTTI
    HowToUseAttributes
    InAlloca
+   InterfaceExportAnnotations
    LangRef
    LibFuzzer
    MarkedUpDisassembly

From 30d13e359190f7a0e2122292ec4a4fc1a6c71acc Mon Sep 17 00:00:00 2001
From: Amr Hesham <amr96@programmer.net>
Date: Tue, 15 Apr 2025 19:52:26 +0200
Subject: [PATCH 018/710] [CIR] Upstream ArraySubscriptExpr from function
 parameter with pointer base (#135493)

This change adds an ArraySubscriptExpr from the function parameter with
base type as Pointer

Issue https://github.com/llvm/llvm-project/issues/130197
---
 clang/lib/CIR/CodeGen/CIRGenExpr.cpp |  43 ++++++++--
 clang/test/CIR/CodeGen/array.cpp     | 122 ++++++++++++++++++++++++---
 2 files changed, 148 insertions(+), 17 deletions(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index f0732a8ea60af..cffe5c5cd1ec3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -552,7 +552,19 @@ CIRGenFunction::emitArraySubscriptExpr(const clang::ArraySubscriptExpr *e) {
   // in lexical order (this complexity is, sadly, required by C++17).
   assert((e->getIdx() == e->getLHS() || e->getIdx() == e->getRHS()) &&
          "index was neither LHS nor RHS");
-  const mlir::Value idx = emitScalarExpr(e->getIdx());
+
+  auto emitIdxAfterBase = [&]() -> mlir::Value {
+    const mlir::Value idx = emitScalarExpr(e->getIdx());
+
+    // Extend or truncate the index type to 32 or 64-bits.
+    auto ptrTy = mlir::dyn_cast<cir::PointerType>(idx.getType());
+    if (ptrTy && mlir::isa<cir::IntType>(ptrTy.getPointee()))
+      cgm.errorNYI(e->getSourceRange(),
+                   "emitArraySubscriptExpr: index type cast");
+    return idx;
+  };
+
+  const mlir::Value idx = emitIdxAfterBase();
   if (const Expr *array = getSimpleArrayDecayOperand(e->getBase())) {
     LValue arrayLV;
     if (const auto *ase = dyn_cast<ArraySubscriptExpr>(array))
@@ -566,13 +578,34 @@ CIRGenFunction::emitArraySubscriptExpr(const clang::ArraySubscriptExpr *e) {
         arrayLV.getAddress(), e->getType(), idx, cgm.getLoc(e->getExprLoc()),
         /*shouldDecay=*/true);
 
-    return LValue::makeAddr(addr, e->getType(), LValueBaseInfo());
+    const LValue lv = LValue::makeAddr(addr, e->getType(), LValueBaseInfo());
+
+    if (getLangOpts().ObjC && getLangOpts().getGC() != LangOptions::NonGC) {
+      cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: ObjC with GC");
+    }
+
+    return lv;
   }
 
   // The base must be a pointer; emit it with an estimate of its alignment.
-  cgm.errorNYI(e->getSourceRange(),
-               "emitArraySubscriptExpr: The base must be a pointer");
-  return {};
+  assert(e->getBase()->getType()->isPointerType() &&
+         "The base must be a pointer");
+
+  LValueBaseInfo eltBaseInfo;
+  const Address ptrAddr = emitPointerWithAlignment(e->getBase(), &eltBaseInfo);
+  // Propagate the alignment from the array itself to the result.
+  const Address addxr = emitArraySubscriptPtr(
+      *this, cgm.getLoc(e->getBeginLoc()), cgm.getLoc(e->getEndLoc()), ptrAddr,
+      e->getType(), idx, cgm.getLoc(e->getExprLoc()),
+      /*shouldDecay=*/false);
+
+  const LValue lv = LValue::makeAddr(addxr, e->getType(), eltBaseInfo);
+
+  if (getLangOpts().ObjC && getLangOpts().getGC() != LangOptions::NonGC) {
+    cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: ObjC with GC");
+  }
+
+  return lv;
 }
 
 LValue CIRGenFunction::emitBinaryOperatorLValue(const BinaryOperator *e) {
diff --git a/clang/test/CIR/CodeGen/array.cpp b/clang/test/CIR/CodeGen/array.cpp
index 5cda061cdbf12..08f6d730f161a 100644
--- a/clang/test/CIR/CodeGen/array.cpp
+++ b/clang/test/CIR/CodeGen/array.cpp
@@ -350,20 +350,118 @@ void func7() {
 // OGCG: %[[ARR:.*]] = alloca [1 x ptr], align 8
 // OGCG: call void @llvm.memset.p0.i64(ptr align 8 %[[ARR]], i8 0, i64 8, i1 false)
 
-void func8(int p[10]) {}
-// CIR: cir.func @func8(%arg0: !cir.ptr<!s32i>
-// CIR: cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["p", init]
+void func8(int arr[10]) {
+  int e = arr[0];
+  int e2 = arr[1];
+}
 
-// LLVM: define void @func8(ptr {{%.*}})
-// LLVM-NEXT: alloca ptr, i64 1, align 8
+// CIR: cir.func @func8(%[[ARG:.*]]: !cir.ptr<!s32i>
+// CIR:  %[[ARR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arr", init]
+// CIR:  %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
+// CIR:  %[[INIT_2:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e2", init]
+// CIR:  cir.store %[[ARG]], %[[ARR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+// CIR:  %[[IDX:.*]] = cir.const #cir.int<0> : !s32i
+// CIR:  %[[TMP_1:.*]] = cir.load %[[ARR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR:  %[[ELE_0:.*]] = cir.ptr_stride(%[[TMP_1]] : !cir.ptr<!s32i>, %[[IDX]] : !s32i), !cir.ptr<!s32i>
+// CIR:  %[[TMP_2:.*]] = cir.load %[[ELE_0]] : !cir.ptr<!s32i>, !s32i
+// CIR:  cir.store %[[TMP_2]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
+// CIR:  %[[IDX_1:.*]] = cir.const #cir.int<1> : !s32i
+// CIR:  %[[TMP_3:.*]] = cir.load %[[ARR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR:  %[[ELE_1:.*]] = cir.ptr_stride(%[[TMP_3]] : !cir.ptr<!s32i>, %[[IDX_1]] : !s32i), !cir.ptr<!s32i>
+// CIR:  %[[TMP_4:.*]] = cir.load %[[ELE_1]] : !cir.ptr<!s32i>, !s32i
+// CIR:  cir.store %[[TMP_4]], %[[INIT_2]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: define void @func8(ptr %[[ARG:.*]])
+// LLVM:  %[[ARR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:  %[[INIT:.*]] = alloca i32, i64 1, align 4
+// LLVM:  %[[INIT_2:.*]] = alloca i32, i64 1, align 4
+// LLVM:  store ptr %[[ARG]], ptr %[[ARR]], align 8
+// LLVM:  %[[TMP_1:.*]] = load ptr, ptr %[[ARR]], align 8
+// LLVM:  %[[ELE_0:.*]] = getelementptr i32, ptr %[[TMP_1]], i64 0
+// LLVM:  %[[TMP_2:.*]] = load i32, ptr %[[ELE_0]], align 4
+// LLVM:  store i32 %[[TMP_2]], ptr %[[INIT]], align 4
+// LLVM:  %[[TMP_3:.*]] = load ptr, ptr %[[ARR]], align 8
+// LLVM:  %[[ELE_1:.*]] = getelementptr i32, ptr %[[TMP_3]], i64 1
+// LLVM:  %[[TMP_4:.*]] = load i32, ptr %[[ELE_1]], align 4
+// LLVM:  store i32 %[[TMP_4]], ptr %[[INIT_2]], align 4
+
+// OGCG: %[[ARR:.*]] = alloca ptr, align 8
+// OGCG: %[[INIT:.*]] = alloca i32, align 4
+// OGCG: %[[INIT_2:.*]] = alloca i32, align 4
+// OGCG: store ptr {{%.*}}, ptr %[[ARR]], align 8
+// OGCG: %[[TMP_1:.*]] = load ptr, ptr %[[ARR]], align 8
+// OGCG: %[[ELE_0:.*]] = getelementptr inbounds i32, ptr %[[TMP_1]], i64 0
+// OGCG: %[[TMP_2:.*]] = load i32, ptr %[[ELE_0]], align 4
+// OGCG: store i32 %[[TMP_2]], ptr %[[INIT]], align 4
+// OGCG: %[[TMP_3:.*]] = load ptr, ptr %[[ARR]], align 8
+// OGCG: %[[ELE_1:.*]] = getelementptr inbounds i32, ptr %[[TMP_3]], i64 1
+// OGCG: %[[TMP_2:.*]] = load i32, ptr %[[ELE_1]], align 4
+// OGCG: store i32 %[[TMP_2]], ptr %[[INIT_2]], align 4
 
-// OGCG: alloca ptr, align 8
+void func9(int arr[10][5]) {
+  int e = arr[1][2];
+}
 
-void func9(int pp[10][5]) {}
-// CIR: cir.func @func9(%arg0: !cir.ptr<!cir.array<!s32i x 5>>
-// CIR: cir.alloca !cir.ptr<!cir.array<!s32i x 5>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 5>>>
+// CIR: cir.func @func9(%[[ARG:.*]]: !cir.ptr<!cir.array<!s32i x 5>>
+// CIR:  %[[ARR:.*]] = cir.alloca !cir.ptr<!cir.array<!s32i x 5>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 5>>>, ["arr", init]
+// CIR:  %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
+// CIR:  cir.store %[[ARG]], %[[ARR]] : !cir.ptr<!cir.array<!s32i x 5>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 5>>>
+// CIR:  %[[IDX:.*]] = cir.const #cir.int<2> : !s32i
+// CIR:  %[[IDX_1:.*]] = cir.const #cir.int<1> : !s32i
+// CIR:  %[[TMP_1:.*]] = cir.load %[[ARR]] : !cir.ptr<!cir.ptr<!cir.array<!s32i x 5>>>, !cir.ptr<!cir.array<!s32i x 5>>
+// CIR:  %[[ARR_1:.*]] = cir.ptr_stride(%[[TMP_1]] : !cir.ptr<!cir.array<!s32i x 5>>, %[[IDX_1]] : !s32i), !cir.ptr<!cir.array<!s32i x 5>>
+// CIR:  %[[ARR_1_PTR:.*]] = cir.cast(array_to_ptrdecay, %[[ARR_1]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CIR:  %[[ARR_1_2:.*]] = cir.ptr_stride(%[[ARR_1_PTR]] : !cir.ptr<!s32i>, %[[IDX]] : !s32i), !cir.ptr<!s32i>
+// CIR:  %[[TMP_2:.*]] = cir.load %[[ARR_1_2]] : !cir.ptr<!s32i>, !s32i
+// CIR:  cir.store %[[TMP_2]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: define void @func9(ptr %[[ARG:.*]])
+// LLVM:  %[[ARR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:  %[[INIT:.*]] = alloca i32, i64 1, align 4
+// LLVM:  store ptr %[[ARG]], ptr %[[ARR]], align 8
+// LLVM:  %[[TMP_1:.*]] = load ptr, ptr %[[ARR]], align 8
+// LLVM:  %[[ARR_1:.*]] = getelementptr [5 x i32], ptr %[[TMP_1]], i64 1
+// LLVM:  %[[ARR_1_PTR:.*]] = getelementptr i32, ptr %[[ARR_1]], i32 0
+// LLVM:  %[[ARR_1_2:.*]] = getelementptr i32, ptr %[[ARR_1_PTR]], i64 2
+// LLVM:  %[[TMP_2:.*]] = load i32, ptr %[[ARR_1_2]], align 4
+// LLVM:  store i32 %[[TMP_2]], ptr %[[INIT]], align 4
+
+// OGCG: %[[ARR:.*]] = alloca ptr, align 8
+// OGCG: %[[INIT:.*]] = alloca i32, align 4
+// OGCG: store ptr {{%.*}}, ptr %[[ARR]], align 8
+// OGCG: %[[TMP_1:.*]] = load ptr, ptr %[[ARR]], align 8
+// OGCG: %[[ARR_1:.*]] = getelementptr inbounds [5 x i32], ptr %[[TMP_1]], i64 1
+// OGCG: %[[ARR_1_2:.*]] = getelementptr inbounds [5 x i32], ptr %[[ARR_1]], i64 0, i64 2
+// OGCG: %[[TMP_2:.*]] = load i32, ptr %[[ARR_1_2]], align 4
+// OGCG: store i32 %[[TMP_2]], ptr %[[INIT]], align 4
+
+void func10(int *a) {
+  int e = a[5];
+}
 
-// LLVM: define void @func9(ptr {{%.*}})
-// LLVM-NEXT: alloca ptr, i64 1, align 8
+// CIR: cir.func @func10(%[[ARG:.*]]: !cir.ptr<!s32i>
+// CIR: %[[ARR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["a", init]
+// CIR: %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
+// CIR: cir.store %[[ARG]], %[[ARR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+// CIR: %[[IDX:.*]] = cir.const #cir.int<5> : !s32i
+// CIR: %[[TMP_1:.*]] = cir.load %[[ARR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR: %[[ELE:.*]] = cir.ptr_stride(%[[TMP_1]] : !cir.ptr<!s32i>, %[[IDX]] : !s32i), !cir.ptr<!s32i>
+// CIR: %[[TMP_2:.*]] = cir.load %[[ELE]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.store %[[TMP_2]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: define void @func10(ptr %[[ARG:.*]]) {
+// LLVM:  %[[ARR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:  %[[INIT:.*]] = alloca i32, i64 1, align 4
+// LLVM:  store ptr %[[ARG]], ptr %[[ARR]], align 8
+// LLVM:  %[[TMP_1:.*]] = load ptr, ptr %[[ARR]], align 8
+// LLVM:  %[[ELE:.*]] = getelementptr i32, ptr %[[TMP_1]], i64 5
+// LLVM:  %[[TMP_2:.*]] = load i32, ptr %[[ELE]], align 4
+// LLVM:  store i32 %[[TMP_2]], ptr %[[INIT]], align 4
 
-// OGCG: alloca ptr, align 8
+// OGCG:  %[[ARR:.*]] = alloca ptr, align 8
+// OGCG:  %[[INIT:.*]] = alloca i32, align 4
+// OGCG:  store ptr {{%.*}}, ptr %[[ARR]], align 8
+// OGCG:  %[[TMP_1:.*]] = load ptr, ptr %[[ARR]], align 8
+// OGCG:  %[[ELE:.*]] = getelementptr inbounds i32, ptr %[[TMP_1]], i64 5
+// OGCG:  %[[TMP_2:.*]] = load i32, ptr %[[ELE]], align 4
+// OGCG:  store i32 %[[TMP_2]], ptr %[[INIT]], align 4

From 3f58ff20fe540fbbc2e5bfea1606f8cdc00d4157 Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <pcc@google.com>
Date: Tue, 15 Apr 2025 11:08:48 -0700
Subject: [PATCH 019/710] AArch64: Remove the PAUTH_BLEND pseudo-instruction.

It can be represented using a regular MOVK instruction
which also has the advantage of sometimes being selectable
without a preceding MOV.

Reviewers: ahmedbougacha, asl, atrosinenko

Reviewed By: atrosinenko

Pull Request: https://github.com/llvm/llvm-project/pull/134765
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  5 +--
 .../lib/Target/AArch64/AArch64PointerAuth.cpp | 37 -------------------
 .../AArch64/ptrauth-pseudo-instructions.mir   | 27 --------------
 3 files changed, 1 insertion(+), 68 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/ptrauth-pseudo-instructions.mir

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index b90792d60d102..99f2b79d31bb7 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1798,9 +1798,6 @@ def PAUTH_PROLOGUE : Pseudo<(outs), (ins), []>, Sched<[]> {
 def PAUTH_EPILOGUE : Pseudo<(outs), (ins), []>, Sched<[]>;
 }
 
-def PAUTH_BLEND : Pseudo<(outs GPR64:$disc),
-                         (ins GPR64:$addr_disc, i32imm:$int_disc), []>, Sched<[]>;
-
 // These pointer authentication instructions require armv8.3a
 let Predicates = [HasPAuth] in {
 
@@ -10130,7 +10127,7 @@ let Predicates = [HasMOPS, HasMTE], Defs = [NZCV], Size = 12, mayLoad = 0, maySt
 // v8.3 Pointer Authentication late patterns
 
 def : Pat<(int_ptrauth_blend GPR64:$Rd, imm64_0_65535:$imm),
-          (PAUTH_BLEND GPR64:$Rd, (trunc_imm imm64_0_65535:$imm))>;
+          (MOVKXi GPR64:$Rd, (trunc_imm imm64_0_65535:$imm), 48)>;
 def : Pat<(int_ptrauth_blend GPR64:$Rd, GPR64:$Rn),
           (BFMXri GPR64:$Rd, GPR64:$Rn, 16, 15)>;
 
diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
index c3bc70ad6f427..ba03f4e257b69 100644
--- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
@@ -42,13 +42,6 @@ class AArch64PointerAuth : public MachineFunctionPass {
   void authenticateLR(MachineFunction &MF,
                       MachineBasicBlock::iterator MBBI) const;
 
-  /// Stores blend(AddrDisc, IntDisc) to the Result register.
-  void emitBlend(MachineBasicBlock::iterator MBBI, Register Result,
-                 Register AddrDisc, unsigned IntDisc) const;
-
-  /// Expands PAUTH_BLEND pseudo instruction.
-  void expandPAuthBlend(MachineBasicBlock::iterator MBBI) const;
-
   bool checkAuthenticatedLR(MachineBasicBlock::iterator TI) const;
 };
 
@@ -249,32 +242,6 @@ unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) {
   llvm_unreachable("Unknown AuthCheckMethod enum");
 }
 
-void AArch64PointerAuth::emitBlend(MachineBasicBlock::iterator MBBI,
-                                   Register Result, Register AddrDisc,
-                                   unsigned IntDisc) const {
-  MachineBasicBlock &MBB = *MBBI->getParent();
-  DebugLoc DL = MBBI->getDebugLoc();
-
-  if (Result != AddrDisc)
-    BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), Result)
-        .addReg(AArch64::XZR)
-        .addReg(AddrDisc)
-        .addImm(0);
-
-  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), Result)
-      .addReg(Result)
-      .addImm(IntDisc)
-      .addImm(48);
-}
-
-void AArch64PointerAuth::expandPAuthBlend(
-    MachineBasicBlock::iterator MBBI) const {
-  Register ResultReg = MBBI->getOperand(0).getReg();
-  Register AddrDisc = MBBI->getOperand(1).getReg();
-  unsigned IntDisc = MBBI->getOperand(2).getImm();
-  emitBlend(MBBI, ResultReg, AddrDisc, IntDisc);
-}
-
 bool AArch64PointerAuth::runOnMachineFunction(MachineFunction &MF) {
   Subtarget = &MF.getSubtarget<AArch64Subtarget>();
   TII = Subtarget->getInstrInfo();
@@ -290,7 +257,6 @@ bool AArch64PointerAuth::runOnMachineFunction(MachineFunction &MF) {
         break;
       case AArch64::PAUTH_PROLOGUE:
       case AArch64::PAUTH_EPILOGUE:
-      case AArch64::PAUTH_BLEND:
         PAuthPseudoInstrs.push_back(MI.getIterator());
         break;
       }
@@ -305,9 +271,6 @@ bool AArch64PointerAuth::runOnMachineFunction(MachineFunction &MF) {
     case AArch64::PAUTH_EPILOGUE:
       authenticateLR(MF, It);
       break;
-    case AArch64::PAUTH_BLEND:
-      expandPAuthBlend(It);
-      break;
     default:
       llvm_unreachable("Unhandled opcode");
     }
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-pseudo-instructions.mir b/llvm/test/CodeGen/AArch64/ptrauth-pseudo-instructions.mir
deleted file mode 100644
index d7fe1953deb47..0000000000000
--- a/llvm/test/CodeGen/AArch64/ptrauth-pseudo-instructions.mir
+++ /dev/null
@@ -1,27 +0,0 @@
-# RUN: llc -mtriple=aarch64--- -run-pass=aarch64-ptrauth -verify-machineinstrs %s -o - | FileCheck %s
-
-# Test the corner cases that cannot be reliably tested using LLVM IR as input.
-
---- |
-  define i64 @blend_untied(i64 %unused, i64 %ptr_arg) {
-    ret i64 0
-  }
-...
----
-# Check that the input register is copied to the output one, if not tied.
-
-name:            blend_untied
-tracksRegLiveness: true
-body:             |
-  bb.0:
-  liveins: $lr, $x0, $x1
-    $x0 = PAUTH_BLEND $x1, 42
-    RET undef $lr
-
-# CHECK:       liveins: $lr, $x0, $x1
-# CHECK-NEXT:    {{^ +$}}
-# CHECK-NEXT:    $x0 = ORRXrs $xzr, $x1, 0
-# CHECK-NEXT:    $x0 = MOVKXi $x0, 42, 48
-# CHECK-NEXT:    RET undef $lr
-
-...

From a5aa0c46c3274eaf25dde4d792a1abd6191cccf9 Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <pcc@google.com>
Date: Tue, 15 Apr 2025 11:12:05 -0700
Subject: [PATCH 020/710] Introduce -funique-source-file-names flag.

The purpose of this flag is to allow the compiler to assume that each
object file passed to the linker has been compiled using a unique
source file name. This is useful for reducing link times when doing
ThinLTO in combination with whole-program devirtualization or CFI,
as it allows modules without exported symbols to be built with ThinLTO.

Reviewers: vitalybuka, teresajohnson

Reviewed By: teresajohnson

Pull Request: https://github.com/llvm/llvm-project/pull/135728
---
 clang/docs/ControlFlowIntegrity.rst           |  5 +++
 clang/docs/UsersManual.rst                    | 10 +++++
 clang/include/clang/Basic/CodeGenOptions.def  |  2 +
 clang/include/clang/Driver/Options.td         |  7 ++++
 clang/lib/CodeGen/CodeGenModule.cpp           |  4 ++
 clang/lib/Driver/ToolChains/Clang.cpp         |  3 ++
 clang/test/CodeGen/unique-source-file-names.c |  2 +
 clang/test/Driver/unique-source-file-names.c  |  5 +++
 llvm/lib/Transforms/Utils/ModuleUtils.cpp     | 40 +++++++++----------
 .../unique-source-file-names.ll               | 22 ++++++++++
 10 files changed, 79 insertions(+), 21 deletions(-)
 create mode 100644 clang/test/CodeGen/unique-source-file-names.c
 create mode 100644 clang/test/Driver/unique-source-file-names.c
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/unique-source-file-names.ll

diff --git a/clang/docs/ControlFlowIntegrity.rst b/clang/docs/ControlFlowIntegrity.rst
index 2f2f8ccf4481b..baff9ab54ff26 100644
--- a/clang/docs/ControlFlowIntegrity.rst
+++ b/clang/docs/ControlFlowIntegrity.rst
@@ -42,6 +42,11 @@ default visibility setting is ``-fvisibility=default``, which would disable
 CFI checks for classes without visibility attributes. Most users will want
 to specify ``-fvisibility=hidden``, which enables CFI checks for such classes.
 
+When using ``-fsanitize=cfi*`` with ``-flto=thin``, it is recommended
+to reduce link times by passing `-funique-source-file-names
+<UsersManual.html#cmdoption-f-no-unique-source-file-names>`_, provided
+that your program is compatible with it.
+
 Experimental support for :ref:`cross-DSO control flow integrity
 <cfi-cross-dso>` exists that does not require classes to have hidden LTO
 visibility. This cross-DSO support has unstable ABI at this time.
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 2a93c2552d7dc..d4656a7e63c99 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -2297,6 +2297,16 @@ are listed below.
    pure ThinLTO, as all split regular LTO modules are merged and LTO linked
    with regular LTO.
 
+.. option:: -f[no-]unique-source-file-names
+
+   When enabled, allows the compiler to assume that each object file
+   passed to the linker has been compiled using a unique source file
+   name. This is useful for reducing link times when doing ThinLTO
+   in combination with whole-program devirtualization or CFI.
+
+   A misuse of this flag will generally result in a duplicate symbol
+   error at link time.
+
 .. option:: -fforce-emit-vtables
 
    In order to improve devirtualization, forces emitting of vtables even in
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index a436c0ec98d5b..c5990fb248689 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -278,6 +278,8 @@ CODEGENOPT(SanitizeCfiICallNormalizeIntegers, 1, 0) ///< Normalize integer types
                                                     ///< CFI icall function signatures
 CODEGENOPT(SanitizeCfiCanonicalJumpTables, 1, 0) ///< Make jump table symbols canonical
                                                  ///< instead of creating a local jump table.
+CODEGENOPT(UniqueSourceFileNames, 1, 0) ///< Allow the compiler to assume that TUs
+                                        ///< have unique source file names at link time
 CODEGENOPT(SanitizeKcfiArity, 1, 0) ///< Embed arity in KCFI patchable function prefix
 CODEGENOPT(SanitizeCoverageType, 2, 0) ///< Type of sanitizer coverage
                                        ///< instrumentation.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index c9d2bc5e81976..e9acb20348654 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -4140,6 +4140,13 @@ def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>,
 def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>,
   HelpText<"Do not process trigraph sequences">,
   Visibility<[ClangOption, CC1Option]>;
+defm unique_source_file_names: BoolOption<"f", "unique-source-file-names",
+  CodeGenOpts<"UniqueSourceFileNames">, DefaultFalse,
+  PosFlag<SetTrue, [], [CC1Option], "Allow">,
+  NegFlag<SetFalse, [], [], "Do not allow">,
+  BothFlags<[], [ClangOption], " the compiler to assume that each translation unit has a unique "
+                               "source file name at link time">>,
+  Group<f_clang_Group>;
 def funsigned_bitfields : Flag<["-"], "funsigned-bitfields">, Group<f_Group>;
 def funsigned_char : Flag<["-"], "funsigned-char">, Group<f_Group>;
 def fno_unsigned_char : Flag<["-"], "fno-unsigned-char">;
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 4a48c2f35ff23..26e09fe239242 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1144,6 +1144,10 @@ void CodeGenModule::Release() {
                               1);
   }
 
+  if (CodeGenOpts.UniqueSourceFileNames) {
+    getModule().addModuleFlag(llvm::Module::Max, "Unique Source File Names", 1);
+  }
+
   if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) {
     getModule().addModuleFlag(llvm::Module::Override, "kcfi", 1);
     // KCFI assumes patchable-function-prefix is the same for all indirectly
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 65910e7fdaaa6..8506a5c00e7bc 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7744,6 +7744,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.addOptInFlag(CmdArgs, options::OPT_fexperimental_late_parse_attributes,
                     options::OPT_fno_experimental_late_parse_attributes);
 
+  Args.addOptInFlag(CmdArgs, options::OPT_funique_source_file_names,
+                    options::OPT_fno_unique_source_file_names);
+
   // Setup statistics file output.
   SmallString<128> StatsFile = getStatsFileName(Args, Output, Input, D);
   if (!StatsFile.empty()) {
diff --git a/clang/test/CodeGen/unique-source-file-names.c b/clang/test/CodeGen/unique-source-file-names.c
new file mode 100644
index 0000000000000..1d5a4a5e8e4c5
--- /dev/null
+++ b/clang/test/CodeGen/unique-source-file-names.c
@@ -0,0 +1,2 @@
+// RUN: %clang_cc1 -funique-source-file-names -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// CHECK:  !{i32 7, !"Unique Source File Names", i32 1}
diff --git a/clang/test/Driver/unique-source-file-names.c b/clang/test/Driver/unique-source-file-names.c
new file mode 100644
index 0000000000000..8322f0e37b0c7
--- /dev/null
+++ b/clang/test/Driver/unique-source-file-names.c
@@ -0,0 +1,5 @@
+// RUN: %clang -funique-source-file-names -### %s 2> %t
+// RUN: FileCheck < %t %s
+
+// CHECK: "-cc1"
+// CHECK: "-funique-source-file-names"
diff --git a/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
index 1c31e851ef4b2..10efdd61d4553 100644
--- a/llvm/lib/Transforms/Utils/ModuleUtils.cpp
+++ b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
@@ -345,27 +345,25 @@ void llvm::filterDeadComdatFunctions(
 
 std::string llvm::getUniqueModuleId(Module *M) {
   MD5 Md5;
-  bool ExportsSymbols = false;
-  auto AddGlobal = [&](GlobalValue &GV) {
-    if (GV.isDeclaration() || GV.getName().starts_with("llvm.") ||
-        !GV.hasExternalLinkage() || GV.hasComdat())
-      return;
-    ExportsSymbols = true;
-    Md5.update(GV.getName());
-    Md5.update(ArrayRef<uint8_t>{0});
-  };
-
-  for (auto &F : *M)
-    AddGlobal(F);
-  for (auto &GV : M->globals())
-    AddGlobal(GV);
-  for (auto &GA : M->aliases())
-    AddGlobal(GA);
-  for (auto &IF : M->ifuncs())
-    AddGlobal(IF);
-
-  if (!ExportsSymbols)
-    return "";
+
+  auto *UniqueSourceFileNames = mdconst::extract_or_null<ConstantInt>(
+      M->getModuleFlag("Unique Source File Names"));
+  if (UniqueSourceFileNames && UniqueSourceFileNames->getZExtValue()) {
+    Md5.update(M->getSourceFileName());
+  } else {
+    bool ExportsSymbols = false;
+    for (auto &GV : M->global_values()) {
+      if (GV.isDeclaration() || GV.getName().starts_with("llvm.") ||
+          !GV.hasExternalLinkage() || GV.hasComdat())
+        continue;
+      ExportsSymbols = true;
+      Md5.update(GV.getName());
+      Md5.update(ArrayRef<uint8_t>{0});
+    }
+
+    if (!ExportsSymbols)
+      return "";
+  }
 
   MD5::MD5Result R;
   Md5.final(R);
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/unique-source-file-names.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/unique-source-file-names.ll
new file mode 100644
index 0000000000000..0f3fd566f9b1c
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/unique-source-file-names.ll
@@ -0,0 +1,22 @@
+; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 1 -o %t1 %t
+; RUN: llvm-dis -o - %t1 | FileCheck %s
+
+source_filename = "unique-source-file-names.c"
+
+@llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @f, ptr null }]
+
+; CHECK: @g.45934e8a5251fb7adbecfff71a4e70ed =
+@g = internal global i8 42, !type !0
+
+declare void @sink(ptr)
+
+define internal void @f() {
+  call void @sink(ptr @g)
+  ret void
+}
+
+!0 = !{i32 0, !"typeid"}
+
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"Unique Source File Names", i32 1}

From d0c973a7a0149db3b71767d4c5a20a31e6a8ed5b Mon Sep 17 00:00:00 2001
From: Michael Spencer <bigcheesegs@gmail.com>
Date: Tue, 15 Apr 2025 11:19:07 -0700
Subject: [PATCH 021/710] [llvm][clang] Allocate a new stack instead of
 spawning a new thread to get more stack space (#133173)

Clang spawns a new thread to avoid running out of stack space. This can
make debugging and performance analysis more difficult as how the
threads are connected is difficult to recover.

This patch introduces `runOnNewStack` and applies it in Clang. On
platforms that have good support for it this allocates a new stack and
moves to it using assembly. Doing split stacks like this actually runs
on most platforms, but many debuggers and unwinders reject the large or
backwards stack offsets that occur. Apple platforms and tools are known
to support this, so this only enables it there for now.
---
 clang/docs/ReleaseNotes.rst                   |   4 +
 clang/include/clang/Basic/Stack.h             |   5 +-
 clang/lib/Basic/Stack.cpp                     |  40 ++----
 clang/lib/Frontend/CompilerInstance.cpp       |   2 +-
 .../llvm/Support/CrashRecoveryContext.h       |   3 +
 llvm/include/llvm/Support/ProgramStack.h      |  62 ++++++++++
 llvm/lib/Support/CMakeLists.txt               |   1 +
 llvm/lib/Support/CrashRecoveryContext.cpp     |  11 ++
 llvm/lib/Support/ProgramStack.cpp             | 114 ++++++++++++++++++
 llvm/unittests/Support/CMakeLists.txt         |   1 +
 llvm/unittests/Support/ProgramStackTest.cpp   |  35 ++++++
 11 files changed, 248 insertions(+), 30 deletions(-)
 create mode 100644 llvm/include/llvm/Support/ProgramStack.h
 create mode 100644 llvm/lib/Support/ProgramStack.cpp
 create mode 100644 llvm/unittests/Support/ProgramStackTest.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index e63de32a0b2aa..22543821ee4b3 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -195,6 +195,10 @@ Non-comprehensive list of changes in this release
 - Added `__builtin_elementwise_exp10`.
 - For AMDPGU targets, added `__builtin_v_cvt_off_f32_i4` that maps to the `v_cvt_off_f32_i4` instruction.
 - Added `__builtin_elementwise_minnum` and `__builtin_elementwise_maxnum`.
+- Clang itself now uses split stacks instead of threads for allocating more
+  stack space when running on Apple AArch64 based platforms. This means that
+  stack traces of Clang from debuggers, crashes, and profilers may look
+  different than before.
 
 New Compiler Flags
 ------------------
diff --git a/clang/include/clang/Basic/Stack.h b/clang/include/clang/Basic/Stack.h
index 30ebd94aedd1f..9674b9d9b62c3 100644
--- a/clang/include/clang/Basic/Stack.h
+++ b/clang/include/clang/Basic/Stack.h
@@ -27,7 +27,10 @@ namespace clang {
 
   /// Call this once on each thread, as soon after starting the thread as
   /// feasible, to note the approximate address of the bottom of the stack.
-  void noteBottomOfStack();
+  ///
+  /// \param ForceSet set to true if you know the call is near the bottom of a
+  ///                 new stack. Used for split stacks.
+  void noteBottomOfStack(bool ForceSet = false);
 
   /// Determine whether the stack is nearly exhausted.
   bool isStackNearlyExhausted();
diff --git a/clang/lib/Basic/Stack.cpp b/clang/lib/Basic/Stack.cpp
index aa15d8e66950f..8cbb84943f8d3 100644
--- a/clang/lib/Basic/Stack.cpp
+++ b/clang/lib/Basic/Stack.cpp
@@ -13,33 +13,13 @@
 
 #include "clang/Basic/Stack.h"
 #include "llvm/Support/CrashRecoveryContext.h"
+#include "llvm/Support/ProgramStack.h"
 
-#ifdef _MSC_VER
-#include <intrin.h>  // for _AddressOfReturnAddress
-#endif
+static LLVM_THREAD_LOCAL uintptr_t BottomOfStack = 0;
 
-static LLVM_THREAD_LOCAL void *BottomOfStack = nullptr;
-
-static void *getStackPointer() {
-#if __GNUC__ || __has_builtin(__builtin_frame_address)
-  return __builtin_frame_address(0);
-#elif defined(_MSC_VER)
-  return _AddressOfReturnAddress();
-#else
-  char CharOnStack = 0;
-  // The volatile store here is intended to escape the local variable, to
-  // prevent the compiler from optimizing CharOnStack into anything other
-  // than a char on the stack.
-  //
-  // Tested on: MSVC 2015 - 2019, GCC 4.9 - 9, Clang 3.2 - 9, ICC 13 - 19.
-  char *volatile Ptr = &CharOnStack;
-  return Ptr;
-#endif
-}
-
-void clang::noteBottomOfStack() {
-  if (!BottomOfStack)
-    BottomOfStack = getStackPointer();
+void clang::noteBottomOfStack(bool ForceSet) {
+  if (!BottomOfStack || ForceSet)
+    BottomOfStack = llvm::getStackPointer();
 }
 
 bool clang::isStackNearlyExhausted() {
@@ -51,7 +31,8 @@ bool clang::isStackNearlyExhausted() {
   if (!BottomOfStack)
     return false;
 
-  intptr_t StackDiff = (intptr_t)getStackPointer() - (intptr_t)BottomOfStack;
+  intptr_t StackDiff =
+      (intptr_t)llvm::getStackPointer() - (intptr_t)BottomOfStack;
   size_t StackUsage = (size_t)std::abs(StackDiff);
 
   // If the stack pointer has a surprising value, we do not understand this
@@ -66,9 +47,12 @@ bool clang::isStackNearlyExhausted() {
 void clang::runWithSufficientStackSpaceSlow(llvm::function_ref<void()> Diag,
                                             llvm::function_ref<void()> Fn) {
   llvm::CrashRecoveryContext CRC;
-  CRC.RunSafelyOnThread([&] {
-    noteBottomOfStack();
+  // Preserve the BottomOfStack in case RunSafelyOnNewStack uses split stacks.
+  uintptr_t PrevBottom = BottomOfStack;
+  CRC.RunSafelyOnNewStack([&] {
+    noteBottomOfStack(true);
     Diag();
     Fn();
   }, DesiredStackSize);
+  BottomOfStack = PrevBottom;
 }
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 243e0a3c15b05..5fe80fc16482e 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1265,7 +1265,7 @@ bool CompilerInstance::compileModule(SourceLocation ImportLoc,
 
   // Execute the action to actually build the module in-place. Use a separate
   // thread so that we get a stack large enough.
-  bool Crashed = !llvm::CrashRecoveryContext().RunSafelyOnThread(
+  bool Crashed = !llvm::CrashRecoveryContext().RunSafelyOnNewStack(
       [&]() {
         GenerateModuleFromModuleMapAction Action;
         Instance.ExecuteAction(Action);
diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h
index 26ddf97b3ef02..31293d6715757 100644
--- a/llvm/include/llvm/Support/CrashRecoveryContext.h
+++ b/llvm/include/llvm/Support/CrashRecoveryContext.h
@@ -97,6 +97,9 @@ class CrashRecoveryContext {
     return RunSafelyOnThread([&]() { Fn(UserData); }, RequestedStackSize);
   }
 
+  bool RunSafelyOnNewStack(function_ref<void()>,
+                           unsigned RequestedStackSize = 0);
+
   /// Explicitly trigger a crash recovery in the current process, and
   /// return failure from RunSafely(). This function does not return.
   [[noreturn]] void HandleExit(int RetCode);
diff --git a/llvm/include/llvm/Support/ProgramStack.h b/llvm/include/llvm/Support/ProgramStack.h
new file mode 100644
index 0000000000000..3ce5de1c0e0d6
--- /dev/null
+++ b/llvm/include/llvm/Support/ProgramStack.h
@@ -0,0 +1,62 @@
+//===--- ProgramStack.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_PROGRAMSTACK_H
+#define LLVM_SUPPORT_PROGRAMSTACK_H
+
+#include "llvm/ADT/STLFunctionalExtras.h"
+
+// LLVM_HAS_SPLIT_STACKS is exposed in the header because CrashRecoveryContext
+// needs to know if it's running on another thread or not.
+//
+// Currently only Apple AArch64 is known to support split stacks in the debugger
+// and other tooling.
+#if defined(__APPLE__) && defined(__aarch64__) &&                              \
+    LLVM_HAS_CPP_ATTRIBUTE(gnu::naked) && __has_extension(gnu_asm)
+# define LLVM_HAS_SPLIT_STACKS
+# define LLVM_HAS_SPLIT_STACKS_AARCH64
+#endif
+
+namespace llvm {
+
+/// \returns an address close to the current value of the stack pointer.
+///
+/// The value is not guaranteed to point to anything specific. It can be used to
+/// estimate how much stack space has been used since the previous call.
+uintptr_t getStackPointer();
+
+/// \returns the default stack size for this platform.
+///
+/// Based on \p RLIMIT_STACK or the equivalent.
+unsigned getDefaultStackSize();
+
+/// Runs Fn on a new stack of at least the given size.
+///
+/// \param StackSize requested stack size. A size of 0 uses the default stack
+///                  size of the platform.
+///
+/// The preferred implementation is split stacks on platforms that have a good
+/// debugging experience for them. On other platforms a new thread is used.
+void runOnNewStack(unsigned StackSize, function_ref<void()> Fn);
+
+template <typename R, typename... Ts>
+R runOnNewStack(unsigned StackSize, function_ref<R(Ts...)> Fn, Ts &&...Args) {
+  std::optional<R> Ret;
+  runOnNewStack(StackSize, [&]() { Ret = Fn(std::forward<Ts>(Args)...); });
+  return std::move(*Ret);
+}
+
+template <typename... Ts>
+void runOnNewStack(unsigned StackSize, function_ref<void(Ts...)> Fn,
+                   Ts &&...Args) {
+  runOnNewStack(StackSize, [&]() { Fn(std::forward<Ts>(Args)...); });
+}
+
+} // namespace llvm
+
+#endif // LLVM_SUPPORT_PROGRAMSTACK_H
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 98ffd829d80b8..def37f3f278d0 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -295,6 +295,7 @@ add_llvm_component_library(LLVMSupport
   Path.cpp
   Process.cpp
   Program.cpp
+  ProgramStack.cpp
   RWMutex.cpp
   Signals.cpp
   Threading.cpp
diff --git a/llvm/lib/Support/CrashRecoveryContext.cpp b/llvm/lib/Support/CrashRecoveryContext.cpp
index f53aea177d612..88c38d7526e71 100644
--- a/llvm/lib/Support/CrashRecoveryContext.cpp
+++ b/llvm/lib/Support/CrashRecoveryContext.cpp
@@ -10,6 +10,7 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ExitCodes.h"
+#include "llvm/Support/ProgramStack.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/thread.h"
 #include <cassert>
@@ -523,3 +524,13 @@ bool CrashRecoveryContext::RunSafelyOnThread(function_ref<void()> Fn,
     CRC->setSwitchedThread();
   return Info.Result;
 }
+
+bool CrashRecoveryContext::RunSafelyOnNewStack(function_ref<void()> Fn,
+                                               unsigned RequestedStackSize) {
+#ifdef LLVM_HAS_SPLIT_STACKS
+  return runOnNewStack(RequestedStackSize,
+                       function_ref<bool()>([&]() { return RunSafely(Fn); }));
+#else
+  return RunSafelyOnThread(Fn, RequestedStackSize);
+#endif
+}
diff --git a/llvm/lib/Support/ProgramStack.cpp b/llvm/lib/Support/ProgramStack.cpp
new file mode 100644
index 0000000000000..9e5a546b34974
--- /dev/null
+++ b/llvm/lib/Support/ProgramStack.cpp
@@ -0,0 +1,114 @@
+//===--- RunOnNewStack.cpp - Crash Recovery -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ProgramStack.h"
+#include "llvm/Config/config.h"
+#include "llvm/Support/Compiler.h"
+
+#ifdef LLVM_ON_UNIX
+# include <sys/resource.h> // for getrlimit
+#endif
+
+#ifdef _MSC_VER
+# include <intrin.h>  // for _AddressOfReturnAddress
+#endif
+
+#ifndef LLVM_HAS_SPLIT_STACKS
+# include "llvm/Support/thread.h"
+#endif
+
+using namespace llvm;
+
+uintptr_t llvm::getStackPointer() {
+#if __GNUC__ || __has_builtin(__builtin_frame_address)
+  return (uintptr_t)__builtin_frame_address(0);
+#elif defined(_MSC_VER)
+  return (uintptr_t)_AddressOfReturnAddress();
+#else
+  volatile char CharOnStack = 0;
+  // The volatile store here is intended to escape the local variable, to
+  // prevent the compiler from optimizing CharOnStack into anything other
+  // than a char on the stack.
+  //
+  // Tested on: MSVC 2015 - 2019, GCC 4.9 - 9, Clang 3.2 - 9, ICC 13 - 19.
+  char *volatile Ptr = &CharOnStack;
+  return (uintptr_t)Ptr;
+#endif
+}
+
+unsigned llvm::getDefaultStackSize() {
+#ifdef LLVM_ON_UNIX
+  rlimit RL;
+  getrlimit(RLIMIT_STACK, &RL);
+  return RL.rlim_cur;
+#else
+  // Clang recursively parses, instantiates templates, and evaluates constant
+  // expressions. We've found 8MiB to be a reasonable stack size given the way
+  // Clang works and the way C++ is commonly written.
+  return 8 << 20;
+#endif
+}
+
+namespace {
+#ifdef LLVM_HAS_SPLIT_STACKS_AARCH64
+[[gnu::naked]] void runOnNewStackImpl(void *Stack, void (*Fn)(void *),
+                                      void *Ctx) {
+  __asm__ volatile(
+      "mov       x16, sp\n\t"
+      "sub       x0, x0, #0x20\n\t"            // subtract space from stack
+      "stp       xzr, x16, [x0, #0x00]\n\t"    // save old sp
+      "stp       x29, x30, [x0, #0x10]\n\t"    // save fp, lr
+      "mov       sp, x0\n\t"                   // switch to new stack
+      "add       x29, x0, #0x10\n\t"           // switch to new frame
+      ".cfi_def_cfa w29, 16\n\t"
+      ".cfi_offset w30, -8\n\t"                // lr
+      ".cfi_offset w29, -16\n\t"               // fp
+
+      "mov       x0, x2\n\t"                   // Ctx is the only argument
+      "blr       x1\n\t"                       // call Fn
+
+      "ldp       x29, x30, [sp, #0x10]\n\t"    // restore fp, lr
+      "ldp       xzr, x16, [sp, #0x00]\n\t"    // load old sp
+      "mov       sp, x16\n\t"
+      "ret"
+  );
+}
+#endif
+
+#ifdef LLVM_HAS_SPLIT_STACKS
+void callback(void *Ctx) {
+  (*reinterpret_cast<function_ref<void()> *>(Ctx))();
+}
+#endif
+} // namespace
+
+#ifdef LLVM_HAS_SPLIT_STACKS
+void llvm::runOnNewStack(unsigned StackSize, function_ref<void()> Fn) {
+  if (StackSize == 0)
+    StackSize = getDefaultStackSize();
+
+  // We use malloc here instead of mmap because:
+  //   - it's simpler,
+  //   - many malloc implementations will reuse the allocation in cases where
+  //     we're bouncing accross the edge of a stack boundry, and
+  //   - many malloc implemenations will already provide guard pages for
+  //     allocations this large.
+  void *Stack = malloc(StackSize);
+  void *BottomOfStack = (char *)Stack + StackSize;
+
+  runOnNewStackImpl(BottomOfStack, callback, &Fn);
+
+  free(Stack);
+}
+#else
+void llvm::runOnNewStack(unsigned StackSize, function_ref<void()> Fn) {
+  llvm::thread Thread(
+      StackSize == 0 ? std::nullopt : std::optional<unsigned>(StackSize), Fn);
+  Thread.join();
+}
+#endif
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 6c4e7cb689b20..e5bf820fb4d1c 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -70,6 +70,7 @@ add_llvm_unittest(SupportTests
   PerThreadBumpPtrAllocatorTest.cpp
   ProcessTest.cpp
   ProgramTest.cpp
+  ProgramStackTest.cpp
   RecyclerTest.cpp
   RegexTest.cpp
   ReverseIterationTest.cpp
diff --git a/llvm/unittests/Support/ProgramStackTest.cpp b/llvm/unittests/Support/ProgramStackTest.cpp
new file mode 100644
index 0000000000000..31dfb3b88ade6
--- /dev/null
+++ b/llvm/unittests/Support/ProgramStackTest.cpp
@@ -0,0 +1,35 @@
+//===- unittest/Support/ProgramStackTest.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ProgramStack.h"
+#include "llvm/Support/Process.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+static uintptr_t func(int &A) {
+  A = 7;
+  return getStackPointer();
+}
+
+static void func2(int &A) {
+  A = 5;
+}
+
+TEST(ProgramStackTest, runOnNewStack) {
+  int A = 0;
+  uintptr_t Stack = runOnNewStack(0, function_ref<uintptr_t(int &)>(func), A);
+  EXPECT_EQ(A, 7);
+  intptr_t StackDiff = (intptr_t)llvm::getStackPointer() - (intptr_t)Stack;
+  size_t StackDistance = (size_t)std::abs(StackDiff);
+  // Page size is used as it's large enough to guarantee were not on the same
+  // stack but not too large to cause spurious failures.
+  EXPECT_GT(StackDistance, llvm::sys::Process::getPageSizeEstimate());
+  runOnNewStack(0, function_ref<void(int &)>(func2), A);
+  EXPECT_EQ(A, 5);
+}

From 429a84f8a4bf559f43f50072747ef49d3e3b2cf1 Mon Sep 17 00:00:00 2001
From: Michael Spencer <bigcheesegs@gmail.com>
Date: Tue, 15 Apr 2025 11:34:26 -0700
Subject: [PATCH 022/710] [clang] Fix ambiguity in `runOnNewStack`

---
 llvm/include/llvm/Support/ProgramStack.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Support/ProgramStack.h b/llvm/include/llvm/Support/ProgramStack.h
index 3ce5de1c0e0d6..232a7b5670b44 100644
--- a/llvm/include/llvm/Support/ProgramStack.h
+++ b/llvm/include/llvm/Support/ProgramStack.h
@@ -45,7 +45,8 @@ unsigned getDefaultStackSize();
 void runOnNewStack(unsigned StackSize, function_ref<void()> Fn);
 
 template <typename R, typename... Ts>
-R runOnNewStack(unsigned StackSize, function_ref<R(Ts...)> Fn, Ts &&...Args) {
+std::enable_if_t<!std::is_same_v<R, void>, R>
+runOnNewStack(unsigned StackSize, function_ref<R(Ts...)> Fn, Ts &&...Args) {
   std::optional<R> Ret;
   runOnNewStack(StackSize, [&]() { Ret = Fn(std::forward<Ts>(Args)...); });
   return std::move(*Ret);

From 227f4066befadf0d10eddb39947e35dbf820b1bb Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Tue, 15 Apr 2025 18:36:11 +0000
Subject: [PATCH 023/710] [gn build] Port d0c973a7a014

---
 llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn       | 1 +
 llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn | 1 +
 2 files changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 3a9f43b1070a7..0d2330cba6a7a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -124,6 +124,7 @@ static_library("Support") {
     "Parallel.cpp",
     "PluginLoader.cpp",
     "PrettyStackTrace.cpp",
+    "ProgramStack.cpp",
     "RISCVAttributeParser.cpp",
     "RISCVAttributes.cpp",
     "RISCVISAUtils.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index bf6a0b7523279..19418ad52147b 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -71,6 +71,7 @@ unittest("SupportTests") {
     "Path.cpp",
     "PerThreadBumpPtrAllocatorTest.cpp",
     "ProcessTest.cpp",
+    "ProgramStackTest.cpp",
     "ProgramTest.cpp",
     "RISCVAttributeParserTest.cpp",
     "RecyclerTest.cpp",

From 13615f7b506a693783764da87dc80e97cf59b95c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Tue, 15 Apr 2025 11:56:35 -0700
Subject: [PATCH 024/710] [flang][openacc] Allow if clause on atomic directives
 (#135451)

The new version of the OpenACC specification will allow the if clause on
the atomic directives. Allow it in `ACC.td` and update the parse node
and parser in flang to support it.

OpenACC dialect will need to be updated to support it as well.
---
 flang/include/flang/Parser/parse-tree.h       | 10 +++---
 flang/lib/Parser/openacc-parsers.cpp          | 25 ++++++++-------
 .../Semantics/OpenACC/acc-atomic-validity.f90 | 32 +++++++++++++++++++
 llvm/include/llvm/Frontend/OpenACC/ACC.td     |  1 +
 4 files changed, 53 insertions(+), 15 deletions(-)

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index eeb438991feee..0c2a5de3b71d2 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -5244,21 +5244,23 @@ EMPTY_CLASS(AccEndAtomic);
 // ACC ATOMIC READ
 struct AccAtomicRead {
   TUPLE_CLASS_BOILERPLATE(AccAtomicRead);
-  std::tuple<Verbatim, Statement<AssignmentStmt>, std::optional<AccEndAtomic>>
+  std::tuple<Verbatim, AccClauseList, Statement<AssignmentStmt>,
+      std::optional<AccEndAtomic>>
       t;
 };
 
 // ACC ATOMIC WRITE
 struct AccAtomicWrite {
   TUPLE_CLASS_BOILERPLATE(AccAtomicWrite);
-  std::tuple<Verbatim, Statement<AssignmentStmt>, std::optional<AccEndAtomic>>
+  std::tuple<Verbatim, AccClauseList, Statement<AssignmentStmt>,
+      std::optional<AccEndAtomic>>
       t;
 };
 
 // ACC ATOMIC UPDATE
 struct AccAtomicUpdate {
   TUPLE_CLASS_BOILERPLATE(AccAtomicUpdate);
-  std::tuple<std::optional<Verbatim>, Statement<AssignmentStmt>,
+  std::tuple<std::optional<Verbatim>, AccClauseList, Statement<AssignmentStmt>,
       std::optional<AccEndAtomic>>
       t;
 };
@@ -5268,7 +5270,7 @@ struct AccAtomicCapture {
   TUPLE_CLASS_BOILERPLATE(AccAtomicCapture);
   WRAPPER_CLASS(Stmt1, Statement<AssignmentStmt>);
   WRAPPER_CLASS(Stmt2, Statement<AssignmentStmt>);
-  std::tuple<Verbatim, Stmt1, Stmt2, AccEndAtomic> t;
+  std::tuple<Verbatim, AccClauseList, Stmt1, Stmt2, AccEndAtomic> t;
 };
 
 struct OpenACCAtomicConstruct {
diff --git a/flang/lib/Parser/openacc-parsers.cpp b/flang/lib/Parser/openacc-parsers.cpp
index fb731ee52cbba..072eba99826a1 100644
--- a/flang/lib/Parser/openacc-parsers.cpp
+++ b/flang/lib/Parser/openacc-parsers.cpp
@@ -187,22 +187,25 @@ TYPE_PARSER(construct<AccBeginCombinedDirective>(
 // 2.12 Atomic constructs
 TYPE_PARSER(construct<AccEndAtomic>(startAccLine >> "END ATOMIC"_tok))
 
-TYPE_PARSER("ATOMIC" >>
-    construct<AccAtomicRead>(verbatim("READ"_tok) / endAccLine,
-        statement(assignmentStmt), maybe(Parser<AccEndAtomic>{} / endAccLine)))
+TYPE_PARSER("ATOMIC" >> construct<AccAtomicRead>(verbatim("READ"_tok),
+                            Parser<AccClauseList>{} / endAccLine,
+                            statement(assignmentStmt),
+                            maybe(Parser<AccEndAtomic>{} / endAccLine)))
 
-TYPE_PARSER("ATOMIC" >>
-    construct<AccAtomicWrite>(verbatim("WRITE"_tok) / endAccLine,
-        statement(assignmentStmt), maybe(Parser<AccEndAtomic>{} / endAccLine)))
+TYPE_PARSER("ATOMIC" >> construct<AccAtomicWrite>(verbatim("WRITE"_tok),
+                            Parser<AccClauseList>{} / endAccLine,
+                            statement(assignmentStmt),
+                            maybe(Parser<AccEndAtomic>{} / endAccLine)))
 
 TYPE_PARSER("ATOMIC" >>
-    construct<AccAtomicUpdate>(maybe(verbatim("UPDATE"_tok)) / endAccLine,
-        statement(assignmentStmt), maybe(Parser<AccEndAtomic>{} / endAccLine)))
+    construct<AccAtomicUpdate>(maybe(verbatim("UPDATE"_tok)),
+        Parser<AccClauseList>{} / endAccLine, statement(assignmentStmt),
+        maybe(Parser<AccEndAtomic>{} / endAccLine)))
 
 TYPE_PARSER("ATOMIC" >>
-    construct<AccAtomicCapture>(verbatim("CAPTURE"_tok) / endAccLine,
-        statement(assignmentStmt), statement(assignmentStmt),
-        Parser<AccEndAtomic>{} / endAccLine))
+    construct<AccAtomicCapture>(verbatim("CAPTURE"_tok),
+        Parser<AccClauseList>{} / endAccLine, statement(assignmentStmt),
+        statement(assignmentStmt), Parser<AccEndAtomic>{} / endAccLine))
 
 TYPE_PARSER(
     sourced(construct<OpenACCAtomicConstruct>(Parser<AccAtomicRead>{})) ||
diff --git a/flang/test/Semantics/OpenACC/acc-atomic-validity.f90 b/flang/test/Semantics/OpenACC/acc-atomic-validity.f90
index ba68031b0f18b..07fb864695737 100644
--- a/flang/test/Semantics/OpenACC/acc-atomic-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-atomic-validity.f90
@@ -10,6 +10,7 @@ program openacc_atomic_validity
   integer :: i
   integer, parameter :: N = 256
   integer, dimension(N) :: c
+  logical :: l
 
 
   !$acc parallel
@@ -23,27 +24,58 @@ program openacc_atomic_validity
   !$acc atomic write
   c(i) = 10
 
+  !$acc atomic write if(l)
+  c(i) = 10
+
   !$acc atomic write
   c(i) = 10
   !$acc end atomic
 
+  !$acc atomic write if(.true.)
+  c(i) = 10
+  !$acc end atomic
+
   !$acc atomic read
   i = c(i)
+  
+  !$acc atomic read if(.true.)
+  i = c(i)
 
   !$acc atomic read
   i = c(i)
   !$acc end atomic
 
+  !$acc atomic read if(l) 
+  i = c(i)
+  !$acc end atomic
+
+  !ERROR: FINALIZE clause is not allowed on the ATOMIC READ FINALIZE IF(L)
+  !$acc atomic read finalize if(l) 
+  i = c(i)
+  !$acc end atomic
+
   !$acc atomic capture
   c(i) = i
   i = i + 1
   !$acc end atomic
 
+  !$acc atomic capture if(l .EQV. .false.)
+  c(i) = i
+  i = i + 1
+  !$acc end atomic
+
   !$acc atomic update
   !ERROR: RHS of atomic update statement must be scalar
   !ERROR: LHS of atomic update statement must be scalar
   c = c + 1
 
+  !$acc atomic update if(i == 0)
+  c(i) = c(i) + 1
+
+  !ERROR: At most one IF clause can appear on the ATOMIC UPDATE IF(I == 0) IF(.TRUE.)
+  !$acc atomic update if(i == 0) if(.true.)
+  c(i) = c(i) + 1
+
   !$acc end parallel
 
 end program openacc_atomic_validity
diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index 2acee9bcc7195..e1a4183785d1f 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -270,6 +270,7 @@ def ACCC_Unknown : Clause<"unknown"> {
 
 // 2.12
 def ACC_Atomic : Directive<"atomic"> {
+  let allowedOnceClauses = [VersionedClause<ACCC_If, 34>];
   let association = AS_Block;
   let category = CA_Executable;
 }

From 8f25e43055058a6a16bf44573feb37a9ce51dc1a Mon Sep 17 00:00:00 2001
From: AdityaK <hiraditya@msn.com>
Date: Tue, 15 Apr 2025 12:11:24 -0700
Subject: [PATCH 025/710] [NFC] Rename hasSameElementsOrSplat to
 hasSameNumElementsOrSplat (#133183)

Makes it less confusing as this function only matches the number of
elements
---
 mlir/lib/IR/BuiltinAttributes.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mlir/lib/IR/BuiltinAttributes.cpp b/mlir/lib/IR/BuiltinAttributes.cpp
index daf79dc5de981..67d1ad927cacc 100644
--- a/mlir/lib/IR/BuiltinAttributes.cpp
+++ b/mlir/lib/IR/BuiltinAttributes.cpp
@@ -589,7 +589,7 @@ static APInt readBits(const char *rawData, size_t bitPos, size_t bitWidth) {
 /// Returns true if 'values' corresponds to a splat, i.e. one element, or has
 /// the same element count as 'type'.
 template <typename Values>
-static bool hasSameElementsOrSplat(ShapedType type, const Values &values) {
+static bool hasSameNumElementsOrSplat(ShapedType type, const Values &values) {
   return (values.size() == 1) ||
          (type.getNumElements() == static_cast<int64_t>(values.size()));
 }
@@ -901,7 +901,7 @@ bool DenseElementsAttr::classof(Attribute attr) {
 
 DenseElementsAttr DenseElementsAttr::get(ShapedType type,
                                          ArrayRef<Attribute> values) {
-  assert(hasSameElementsOrSplat(type, values));
+  assert(hasSameNumElementsOrSplat(type, values));
 
   Type eltType = type.getElementType();
 
@@ -985,7 +985,7 @@ DenseElementsAttr DenseElementsAttr::get(ShapedType type,
 
 DenseElementsAttr DenseElementsAttr::get(ShapedType type,
                                          ArrayRef<bool> values) {
-  assert(hasSameElementsOrSplat(type, values));
+  assert(hasSameNumElementsOrSplat(type, values));
   assert(type.getElementType().isInteger(1));
 
   std::vector<char> buff(llvm::divideCeil(values.size(), CHAR_BIT));
@@ -1020,7 +1020,7 @@ DenseElementsAttr DenseElementsAttr::get(ShapedType type,
 DenseElementsAttr DenseElementsAttr::get(ShapedType type,
                                          ArrayRef<APInt> values) {
   assert(type.getElementType().isIntOrIndex());
-  assert(hasSameElementsOrSplat(type, values));
+  assert(hasSameNumElementsOrSplat(type, values));
   size_t storageBitWidth = getDenseElementStorageWidth(type.getElementType());
   return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, values);
 }
@@ -1028,7 +1028,7 @@ DenseElementsAttr DenseElementsAttr::get(ShapedType type,
                                          ArrayRef<std::complex<APInt>> values) {
   ComplexType complex = llvm::cast<ComplexType>(type.getElementType());
   assert(llvm::isa<IntegerType>(complex.getElementType()));
-  assert(hasSameElementsOrSplat(type, values));
+  assert(hasSameNumElementsOrSplat(type, values));
   size_t storageBitWidth = getDenseElementStorageWidth(complex) / 2;
   ArrayRef<APInt> intVals(reinterpret_cast<const APInt *>(values.data()),
                           values.size() * 2);
@@ -1041,7 +1041,7 @@ DenseElementsAttr DenseElementsAttr::get(ShapedType type,
 DenseElementsAttr DenseElementsAttr::get(ShapedType type,
                                          ArrayRef<APFloat> values) {
   assert(llvm::isa<FloatType>(type.getElementType()));
-  assert(hasSameElementsOrSplat(type, values));
+  assert(hasSameNumElementsOrSplat(type, values));
   size_t storageBitWidth = getDenseElementStorageWidth(type.getElementType());
   return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, values);
 }
@@ -1050,7 +1050,7 @@ DenseElementsAttr::get(ShapedType type,
                        ArrayRef<std::complex<APFloat>> values) {
   ComplexType complex = llvm::cast<ComplexType>(type.getElementType());
   assert(llvm::isa<FloatType>(complex.getElementType()));
-  assert(hasSameElementsOrSplat(type, values));
+  assert(hasSameNumElementsOrSplat(type, values));
   ArrayRef<APFloat> apVals(reinterpret_cast<const APFloat *>(values.data()),
                            values.size() * 2);
   size_t storageBitWidth = getDenseElementStorageWidth(complex) / 2;

From 9a1ece26126363c64c67d9a6e357076e814acf9e Mon Sep 17 00:00:00 2001
From: marius doerner <marius.doerner1@icloud.com>
Date: Tue, 15 Apr 2025 21:13:56 +0200
Subject: [PATCH 026/710] [clang] Clear `NeedsCleaning` flag after
 `ExpandBuiltinMacro` (#133574)

After builtin macro expansion in `Preprocessor::ExpandBuiltinMacro` the
result token may have the `Token::NeedsCleaning` flag set which causes
an assertion failure later on when the lexer retrieves the spelling of
the token in `getSpellingSlow`.

This commit adds an `Tok.clearFlag(Token::NeedsCleaning)` call to the
end of `ExpandBuiltinMacro`.

Closes #128384
---
 clang/docs/ReleaseNotes.rst                 |   2 +
 clang/lib/Lex/PPMacroExpansion.cpp          |   1 +
 clang/test/Preprocessor/embed___has_embed.c |  19 +++
 clang/test/Preprocessor/has_attribute.c     |  20 ++++
 clang/test/Preprocessor/has_attribute.cpp   |  20 ++++
 clang/test/Preprocessor/has_c_attribute.c   |  20 ++++
 clang/test/Preprocessor/has_include.c       |  49 ++++++++
 clang/test/Preprocessor/pr133574.c          | 121 ++++++++++++++++++++
 8 files changed, 252 insertions(+)
 create mode 100644 clang/test/Preprocessor/pr133574.c

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 22543821ee4b3..6025e76029d19 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -404,6 +404,8 @@ Bug Fixes in This Version
 - Defining an integer literal suffix (e.g., ``LL``) before including
   ``<stdint.h>`` in a freestanding build no longer causes invalid token pasting
   when using the ``INTn_C`` macros. (#GH85995)
+- Fixed an assertion failure in the expansion of builtin macros like ``__has_embed()`` with line breaks before the
+  closing paren. (#GH133574)
 - Clang no longer accepts invalid integer constants which are too large to fit
   into any (standard or extended) integer type when the constant is unevaluated.
   Merely forming the token is sufficient to render the program invalid. Code
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index 8e35d56d3f1a6..37ac1bf07e9c0 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -2089,6 +2089,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
   CreateString(OS.str(), Tok, Tok.getLocation(), Tok.getLocation());
   Tok.setFlagValue(Token::StartOfLine, IsAtStartOfLine);
   Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+  Tok.clearFlag(Token::NeedsCleaning);
 }
 
 void Preprocessor::markMacroAsUsed(MacroInfo *MI) {
diff --git a/clang/test/Preprocessor/embed___has_embed.c b/clang/test/Preprocessor/embed___has_embed.c
index 43a3068b5f53a..2705b5ef6fd5b 100644
--- a/clang/test/Preprocessor/embed___has_embed.c
+++ b/clang/test/Preprocessor/embed___has_embed.c
@@ -58,3 +58,22 @@ unsigned char buffer[] = {
 #else
 #error 17
 #endif
+
+#if __has_embed(__FILE__\
+)
+#else
+#error 18
+#endif
+
+#define F __FI\
+LE__
+#if __has_embed(F)
+#else
+#error 19
+#endif
+
+#if __has_embed(F\
+)
+#else 
+#error 20
+#endif
diff --git a/clang/test/Preprocessor/has_attribute.c b/clang/test/Preprocessor/has_attribute.c
index 0ba664a53e649..6f6f519bd299d 100644
--- a/clang/test/Preprocessor/has_attribute.c
+++ b/clang/test/Preprocessor/has_attribute.c
@@ -68,3 +68,23 @@ int has_no_volatile_attribute();
 int has_fallthrough;
 #endif
 // CHECK: int has_fallthrough;
+
+#if __has_attribute(F\
+)
+int has_fallthrough_2;
+#endif
+// CHECK: int has_fallthrough_2;
+
+#define F_2 fall\
+through
+
+#if __has_attribute(F_2)
+int has_fallthrough_3;
+#endif
+// CHECK: int has_fallthrough_3;
+
+#if __has_attribute(F_2\
+)
+int has_fallthrough_4;
+#endif
+// CHECK: int has_fallthrough_4;
diff --git a/clang/test/Preprocessor/has_attribute.cpp b/clang/test/Preprocessor/has_attribute.cpp
index 00ec57615c84b..72af6de27e8bb 100644
--- a/clang/test/Preprocessor/has_attribute.cpp
+++ b/clang/test/Preprocessor/has_attribute.cpp
@@ -116,6 +116,26 @@ int funclike_1;
 int funclike_2;
 #endif
 // CHECK: int funclike_2;
+
+#if __has_cpp_attribute(CF\
+)
+int has_clang_falthrough_5;
+#endif
+// CHECK: int has_clang_falthrough_5;
+
+#define CF_2 clang::\
+fallthrough
+
+#if __has_cpp_attribute(CF_2)
+int has_clang_falthrough_6;
+#endif
+// CHECK: int has_clang_falthrough_6;
+
+#if __has_cpp_attribute(CF_2\
+)
+int has_clang_falthrough_7;
+#endif
+// CHECK: int has_clang_falthrough_7;
 }
 
 // Test for Microsoft __declspec attributes
diff --git a/clang/test/Preprocessor/has_c_attribute.c b/clang/test/Preprocessor/has_c_attribute.c
index 3332571d758c8..d8be13f5898a9 100644
--- a/clang/test/Preprocessor/has_c_attribute.c
+++ b/clang/test/Preprocessor/has_c_attribute.c
@@ -84,3 +84,23 @@ int funclike_1;
 int funclike_2;
 #endif
 // CHECK: int funclike_2;
+
+#if __has_c_attribute(CL\
+)
+int has_clang_likely_5;
+#endif
+// CHECK: int has_clang_likely_5;
+
+#define CL_2 clang::\
+likely
+
+#if __has_c_attribute(CL_2)
+int has_clang_likely_6;
+#endif
+// CHECK: int has_clang_likely_6;
+
+#if __has_c_attribute(CL_2\
+)
+int has_clang_likely_7;
+#endif
+// CHECK: int has_clang_likely_7;
diff --git a/clang/test/Preprocessor/has_include.c b/clang/test/Preprocessor/has_include.c
index c95025d83860a..ff199bf23063f 100644
--- a/clang/test/Preprocessor/has_include.c
+++ b/clang/test/Preprocessor/has_include.c
@@ -197,3 +197,52 @@ __has_include
 #ifdef FOO
 #elif __has_include(<foo>)
 #endif
+
+#if __has_include(<stdint.h>\
+)
+#else
+  #error "__has_include failed (10)."
+#endif
+
+#define MACRO6 <stdint.h>
+#if __has_include(MACRO6\
+)
+#else
+  #error "__has_include failed (11)."
+#endif
+
+#if __has_include_next(<stdint.h>/*expected-warning {{#include_next in primary source file}}*/\
+)
+#else
+  #error "__has_include_next failed (9)."
+#endif
+
+#if __has_include_next(MACRO6/*expected-warning {{#include_next in primary source file}}*/\
+) 
+#else
+  #error "__has_include_next failed (10)."
+#endif
+
+#define MACRO7 <std\
+int.h>
+#if __has_include(MACRO7)
+#else
+  #error "__has_include failed (12)."
+#endif
+
+#if __has_include(MACRO7\
+)
+#else
+  #error "__has_include failed (13)."
+#endif
+
+#if __has_include_next(MACRO7) //expected-warning {{#include_next in primary source file}}
+#else
+  #error "__has_include_next failed (11)."
+#endif
+
+#if __has_include_next(MACRO7/*expected-warning {{#include_next in primary source file}}*/\
+) 
+#else
+  #error "__has_include_next failed (12)."
+#endif
diff --git a/clang/test/Preprocessor/pr133574.c b/clang/test/Preprocessor/pr133574.c
new file mode 100644
index 0000000000000..a34073e63b760
--- /dev/null
+++ b/clang/test/Preprocessor/pr133574.c
@@ -0,0 +1,121 @@
+// RUN: %clang_cc1 -E -verify %s
+// expected-no-diagnostics
+
+#define DATE_LBR __D\
+ATE__
+
+const char* test1(void) {
+    return __DATE\
+__;
+}
+const char* test2(void) {
+    return DATE_LBR;
+}
+
+#define TIME_LBR __TIME_\
+_
+
+const char* test3(void) {
+    return __TIM\
+E__;
+}
+
+const char* test4(void) {
+    return TIME_LBR;
+}
+
+#define LINE_LBR __LI\
+NE__
+
+int test5(void) {
+    return _\
+_LINE__;
+}
+
+int test6(void) {
+    return LINE_LBR;
+}
+
+#define FILE_LBR __FI\
+LE__
+
+const char* test7(void) {
+    return __\
+FILE__;
+}
+
+const char* test8(void) {
+    return FILE_LBR;
+}
+
+#define FILE_NAME_LBR __FILE_NA\
+ME__
+
+const char* test9(void) {
+    return __FILE_NAM\
+E__;
+}
+
+const char* test10(void) {
+    return FILE_NAME_LBR;
+}
+
+#define BASE_FILE_LBR __BASE_FIL\
+E__
+
+const char* test11(void) {
+    return __BASE_\
+FILE__;
+}
+
+const char* test12(void) {
+    return BASE_FILE_LBR;
+}
+
+#define INCLUDE_LEVEL_LBR __INCLUDE\
+_LEVEL__
+
+int test13(void) {
+    return __IN\
+CLUDE_LEVEL__;
+}
+
+int test14(void) {
+    return INCLUDE_LEVEL_LBR;
+}
+
+#define TIMESTAMP_LBR __TIMESTA\
+MP__
+
+const char* test15(void) {
+    return __TIMESTA\
+MP__;
+}
+
+const char* test16(void) {
+    return TIMESTAMP_LBR;
+}
+
+#define FLT_EVAL_METHOD_LBR __FLT_EVAL_METH\
+OD__
+
+int test17(void) {
+    return __FL\
+T_EVAL_METHOD__;
+}
+
+int test18(void) {
+    return FLT_EVAL_METHOD_LBR;
+}
+
+#define COUNTER_LBR __COUNTE\
+R__
+
+int test19(void) {
+    return _\
+_COUNTER__;
+}
+
+int test20(void) {
+    return COUNTER_LBR;
+}

From b581bd3429b28420ff473f700fe96c18127a475d Mon Sep 17 00:00:00 2001
From: Scott Manley <rscottmanley@gmail.com>
Date: Tue, 15 Apr 2025 14:21:19 -0500
Subject: [PATCH 027/710] [flang][OpenACC] use correct type when create private
 box init recipe (#135698)

The recipe for initializing private box types was incorrect because
hlfir::createTempFromMold() is not a suitable utility function when the
box element type is a trivial type.
---
 flang/lib/Lower/OpenACC.cpp              | 24 ++++++++++++-------
 flang/test/Lower/OpenACC/acc-private.f90 | 30 ++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 3dd35ed9ae481..c83e277b996f3 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -522,13 +522,17 @@ static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe,
                                      mlir::Type ty, mlir::Location loc) {
   mlir::Value retVal = recipe.getInitRegion().front().getArgument(0);
   ty = fir::unwrapRefType(ty);
-  if (fir::isa_trivial(ty)) {
+
+  auto getDeclareOpForType = [&](mlir::Type ty) -> hlfir::DeclareOp {
     auto alloca = builder.create<fir::AllocaOp>(loc, ty);
-    auto declareOp = builder.create<hlfir::DeclareOp>(
+    return builder.create<hlfir::DeclareOp>(
         loc, alloca, accPrivateInitName, /*shape=*/nullptr,
         llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
         fir::FortranVariableFlagsAttr{});
-    retVal = declareOp.getBase();
+  };
+
+  if (fir::isa_trivial(ty)) {
+    retVal = getDeclareOpForType(ty).getBase();
   } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
     if (fir::isa_trivial(seqTy.getEleTy())) {
       mlir::Value shape;
@@ -552,12 +556,16 @@ static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe,
     }
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
     mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
-    if (!fir::isa_trivial(innerTy) && !mlir::isa<fir::SequenceType>(innerTy))
+    if (fir::isa_trivial(innerTy)) {
+      retVal = getDeclareOpForType(ty).getBase();
+    } else if (mlir::isa<fir::SequenceType>(innerTy)) {
+      fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
+      hlfir::Entity source = hlfir::Entity{retVal};
+      auto [temp, cleanup] = hlfir::createTempFromMold(loc, firBuilder, source);
+      retVal = temp;
+    } else {
       TODO(loc, "Unsupported boxed type in OpenACC privatization");
-    fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
-    hlfir::Entity source = hlfir::Entity{retVal};
-    auto [temp, cleanup] = hlfir::createTempFromMold(loc, firBuilder, source);
-    retVal = temp;
+    }
   }
   builder.create<mlir::acc::YieldOp>(loc, retVal);
 }
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index 50c7a258bb567..356bb9d825d8e 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -87,6 +87,13 @@
 ! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xi32>>
 ! CHECK: }
 
+! CHECK-LABEL: @privatization_ref_box_heap_i32 : !fir.ref<!fir.box<!fir.heap<i32>>> init {
+! CHECK: ^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<i32>>>):
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.heap<i32>>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.private.init"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK: }
+
 ! CHECK-LABEL: acc.private.recipe @privatization_ref_box_heap_Uxi32 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> init {
 ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>):
 ! CHECK:   %[[LOADBOX:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
@@ -292,6 +299,29 @@ subroutine acc_private_allocatable_array(a, n)
 ! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK: acc.serial private(@privatization_ref_box_heap_Uxi32 -> %{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 
+subroutine acc_private_allocatable_scalar(b, a, n)
+  integer :: a(n)
+  integer, allocatable :: b
+  integer :: i, n
+
+  !$acc parallel loop private(b)
+  do i = 1, n
+    a(i) = b
+  end do
+
+  !$acc serial private(b)
+  a(i) = b
+  !$acc end serial
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_private_allocatable_scalar(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<i32>>> {fir.bindc_name = "b"}
+! CHECK: %[[DECLA_B:.*]]:2 = hlfir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_scalarEb"} : (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK: acc.parallel {{.*}} {
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_B]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = "b"}
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_i32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK: acc.serial private(@privatization_ref_box_heap_i32 -> %{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>) {
+
 subroutine acc_private_pointer_array(a, n)
   integer, pointer :: a(:)
   integer :: i, n

From bd9c5112c750f07df2886562b227d9e2aeb338c8 Mon Sep 17 00:00:00 2001
From: Tai Ly <tai.ly@arm.com>
Date: Tue, 15 Apr 2025 14:36:42 -0500
Subject: [PATCH 028/710] [mlir][tosa] Add error_if checks for Transpose
 (#135219)

This adds missing error_if checking for Transpose Op
also moved all transpose op's verifier tests from
invalid.mlir to verifier.mlir

Signed-off-by: Tai Ly <tai.ly@arm.com>
---
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp |  48 ++++++----
 mlir/test/Dialect/Tosa/invalid.mlir  | 112 ------------------------
 mlir/test/Dialect/Tosa/verifier.mlir | 126 +++++++++++++++++++++++++++
 3 files changed, 155 insertions(+), 131 deletions(-)
 create mode 100644 mlir/test/Dialect/Tosa/verifier.mlir

diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 9579d71a2afe9..bce5b226635f3 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -1981,23 +1981,28 @@ LogicalResult tosa::TransposeOp::verify() {
           .failed()) {
     return failure();
   }
-  TensorType inputType = getInput1().getType();
-  TensorType outputType = getOutput().getType();
+
+  const ShapeAdaptor inputShape(getInput1().getType());
+  const ShapeAdaptor outputShape(getOutput().getType());
+
   const llvm::ArrayRef<int32_t> constantPerms = getPerms();
 
-  if (inputType.hasRank() &&
-      constantPerms.size() != static_cast<size_t>(inputType.getRank()))
+  if (inputShape.hasRank() &&
+      constantPerms.size() != static_cast<size_t>(inputShape.getRank()))
     return emitOpError() << "expected perms attribute to have size "
-                         << inputType.getRank() << " (input rank) but got size "
+                         << inputShape.getRank()
+                         << " (input rank) but got size "
                          << constantPerms.size();
-  if (inputType.hasRank() && outputType.hasRank() &&
-      inputType.getRank() != outputType.getRank())
+
+  if (inputShape.hasRank() && outputShape.hasRank() &&
+      inputShape.getRank() != outputShape.getRank())
     return emitOpError()
            << "expected input tensor rank to equal result tensor rank";
-  if (outputType.hasRank() &&
-      constantPerms.size() != static_cast<size_t>(outputType.getRank()))
+
+  if (outputShape.hasRank() &&
+      constantPerms.size() != static_cast<size_t>(outputShape.getRank()))
     return emitOpError() << "expected perms attribute to have size "
-                         << outputType.getRank()
+                         << outputShape.getRank()
                          << " (output rank) but got size "
                          << constantPerms.size();
 
@@ -2010,22 +2015,27 @@ LogicalResult tosa::TransposeOp::verify() {
           constantPerms, [](int32_t v) -> int64_t { return v; }))))
     return emitOpError() << "expected valid permutation indices";
 
+  // ERROR_IF(tensor_size(shape1) != tensor_size(shape))
+  if (inputShape.hasStaticShape() && outputShape.hasStaticShape() &&
+      inputShape.getNumElements() != outputShape.getNumElements())
+    return emitOpError() << "expected input1 and output to have same numbers "
+                            "of elements, got "
+                         << inputShape.getNumElements() << " and "
+                         << outputShape.getNumElements();
+
   // Verify that the types of the input and output tensors are properly
   // permuted.
-  if (inputType.hasRank() && outputType.hasRank()) {
-    assert(constantPerms.size() == static_cast<size_t>(inputType.getRank()) &&
-           inputType.getRank() == outputType.getRank());
-
-    for (auto i = 0; i < outputType.getRank(); i++) {
-      if (inputType.isDynamicDim(constantPerms[i]) ||
-          outputType.isDynamicDim(i))
+  if (inputShape.hasRank() && outputShape.hasRank()) {
+    for (auto i = 0; i < outputShape.getRank(); i++) {
+      if (inputShape.isDynamicDim(constantPerms[i]) ||
+          outputShape.isDynamicDim(i))
         continue;
 
-      if (inputType.getDimSize(constantPerms[i]) != outputType.getDimSize(i))
+      if (inputShape.getDimSize(constantPerms[i]) != outputShape.getDimSize(i))
         return emitOpError()
                << "expected output tensor dim " << i << " to match "
                << "input dim " << constantPerms[i] << " with value of "
-               << inputType.getDimSize(constantPerms[i]);
+               << inputShape.getDimSize(constantPerms[i]);
     }
   }
 
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index 8d8f4f562c4b6..c0b251885de5c 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -368,79 +368,6 @@ func.func @test_pad_padding_shape_mismatch(%arg0: tensor<13x21x3xf32>) -> tensor
 
 // -----
 
-func.func @test_transpose_io_rank_mismatch(%arg0: tensor<13x21x3xf32>, %arg1: tensor<3xi32>) -> tensor<3x13x21x1xf32> {
-  // expected-error@+1 {{'tosa.transpose' op expected input tensor rank to equal result tensor rank}}
-  %0 = tosa.transpose %arg0 {perms = array<i32: 2, 1, 0>}: (tensor<13x21x3xf32>) -> tensor<3x13x21x1xf32>
-  return %0 : tensor<3x13x21x1xf32>
-}
-
-// -----
-
-func.func @test_transpose_rank0_perms() {
-  %14 = tensor.empty() : tensor<5x27xi64>
-  // expected-error@+1 {{'tosa.transpose' op expected perms attribute to have size 2 (input rank) but got size 0}}
-  %72 = tosa.transpose %14 {perms = array<i32> }: (tensor<5x27xi64>) -> tensor<?x?xi64>
-  return
-}
-
-// -----
-
-func.func @test_transpose_invalid_perms_size(%arg0: tensor<13x21x3xf32>) -> tensor<3x13x21xf32> {
-  // expected-error@+1 {{'tosa.transpose' op expected perms attribute to have size 3 (input rank) but got size 7}}
-  %0 = tosa.transpose %arg0 {perms = array<i32: 6, 5, 4, 3, 2, 1, 0> }: (tensor<13x21x3xf32>) -> tensor<3x13x21xf32>
-  return %0 : tensor<3x13x21xf32>
-}
-
-// -----
-
-func.func @test_transpose_invalid_permutation_tensor(%arg0: tensor<13x21x3xf32>) -> tensor<?x?x?xf32> {
-  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
-  %0 = tosa.transpose %arg0 {perms = array<i32: 2, 0, 0> }: (tensor<13x21x3xf32>) -> tensor<?x?x?xf32>
-  return %0 : tensor<?x?x?xf32>
-}
-
-// -----
-
-func.func @test_transpose_invalid_permutation_negative(%arg0: tensor<3x2xi32>) -> tensor<*xi32> {
-  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
-  %1 = tosa.transpose %arg0 {perms = array<i32: -1, 0> }: (tensor<3x2xi32>) -> tensor<*xi32>
-  return %1 : tensor<*xi32>
-}
-
-// -----
-
-func.func @test_transpose_invalid_permutation_tensor_above_range(%arg0: tensor<3x2xi32>) -> tensor<*xi32> {
-  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
-  %1 = tosa.transpose %arg0 {perms = array<i32: 2, 0> }: (tensor<3x2xi32>) -> tensor<*xi32>
-  return %1 : tensor<*xi32>
-}
-
-// -----
-
-func.func @test_transpose_invalid_permutation_types(%arg0: tensor<3x2xi32>) -> tensor<3x4xi32> {
-  // expected-error@+1 {{'tosa.transpose' op expected output tensor dim 0 to match input dim 1 with value of 2}}
-  %1 = tosa.transpose %arg0 {perms = array<i32: 1, 0> }: (tensor<3x2xi32>) -> tensor<3x4xi32>
-  return %1 : tensor<3x4xi32>
-}
-
-// -----
-
-func.func @test_transpose_invalid_permutation_types_dynamic_dim_ok(%arg0: tensor<2x?xi32>) -> tensor<3x4xi32> {
-  // expected-error@+1 {{'tosa.transpose' op expected output tensor dim 1 to match input dim 0 with value of 2}}
-  %1 = tosa.transpose %arg0 {perms = array<i32: 1, 0> }: (tensor<2x?xi32>) -> tensor<3x4xi32>
-  return %1 : tensor<3x4xi32>
-}
-
-// -----
-
-func.func @test_transpose_element_type_mismatch(%arg0: tensor<2x3xi32>) -> tensor<3x2xf32> {
-  // expected-error@+1 {{'tosa.transpose' op failed to verify that all of {input1, output} have same element type}}
-  %1 = tosa.transpose %arg0 {perms = array<i32: 1, 0>} : (tensor<2x3xi32>) -> tensor<3x2xf32>
-  return %1 : tensor<3x2xf32>
-}
-
-// -----
-
 func.func @test_reduce_sum_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
   // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tosa.reduce_sum' op inferred type(s) 'tensor<1x3x4x5xf32>' are incompatible with return type(s) of operation 'tensor<1x3x4x5xi32>'}}
@@ -783,37 +710,6 @@ func.func @test_tile_io_rank_mismatch() {
   return
 }
 
-// -----
-
-// CHECK-LABEL: @test_invalid_constant_permutation
-func.func @test_invalid_constant_permutation() {
-  %0 = tensor.empty() : tensor<3x4x5xi32>
-  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
-  %2 = tosa.transpose %0 {perms = array<i32: 3, 0, 1>}: (tensor<3x4x5xi32>) -> tensor<3x4x5xi32>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: test_rank_size_constant_permutation
-func.func @test_rank_size_constant_permutation() {
-  %0 = arith.constant 6 : index
-  %2 = tensor.empty(%0) : tensor<?x27xi64>
-  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
-  %3 = tosa.transpose %2 {perms = array<i32: 0, 2>}: (tensor<?x27xi64>) -> tensor<?x27xi64>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: test_large_constant_permutation
-func.func @test_large_constant_permutation() {
-  %0 = arith.constant 6 : index
-  %2 = tensor.empty(%0) : tensor<?x27xi64>
-  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
-  %3 = tosa.transpose %2 {perms = array<i32: 1185677355, 332462212>}: (tensor<?x27xi64>) -> tensor<?x27xi64>
-  return
-}
 
 // -----
 
@@ -2061,14 +1957,6 @@ func.func @test_scalar_tile(%arg0: tensor<f32>) -> tensor<*xf32> {
 
 // -----
 
-func.func @test_scalar_output_transpose(%arg0: tensor<*xf32>) -> tensor<f32> {
-  // expected-error@+1 {{'tosa.transpose' op result #0 must be tosa-conformant tensor of at least rank 1, but got 'tensor<f32>'}}
-  %1 = tosa.transpose %arg0 {perms = array<i32: 2, 0, 1>} : (tensor<*xf32>) -> tensor<f32>
-  return %1 : tensor<f32>
-}
-
-// -----
-
 // CHECK-LABEL: test_add_i1
 func.func @test_add_i1(%arg0: tensor<13x21x1xi1>, %arg1: tensor<13x21x3xi1>) -> tensor<13x21x3xi1> {
   // expected-error@+1 {{'tosa.add' op illegal: operand/result data types not supported}}
diff --git a/mlir/test/Dialect/Tosa/verifier.mlir b/mlir/test/Dialect/Tosa/verifier.mlir
new file mode 100644
index 0000000000000..c49cbecd25c78
--- /dev/null
+++ b/mlir/test/Dialect/Tosa/verifier.mlir
@@ -0,0 +1,126 @@
+//--------------------------------------------------------------------------------------------------
+// Test expected errors generated by verifier checks.
+//--------------------------------------------------------------------------------------------------
+
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+
+// -----
+
+func.func @test_transpose_io_rank_mismatch(%arg0: tensor<13x21x3xf32>, %arg1: tensor<3xi32>) -> tensor<3x13x21x1xf32> {
+  // expected-error@+1 {{'tosa.transpose' op expected input tensor rank to equal result tensor rank}}
+  %0 = tosa.transpose %arg0 {perms = array<i32: 2, 1, 0>}: (tensor<13x21x3xf32>) -> tensor<3x13x21x1xf32>
+  return %0 : tensor<3x13x21x1xf32>
+}
+
+// -----
+
+func.func @test_transpose_rank0_perms() {
+  %14 = tensor.empty() : tensor<5x27xi64>
+  // expected-error@+1 {{'tosa.transpose' op expected perms attribute to have size 2 (input rank) but got size 0}}
+  %72 = tosa.transpose %14 {perms = array<i32> }: (tensor<5x27xi64>) -> tensor<?x?xi64>
+  return
+}
+
+// -----
+
+func.func @test_transpose_invalid_perms_size(%arg0: tensor<13x21x3xf32>) -> tensor<3x13x21xf32> {
+  // expected-error@+1 {{'tosa.transpose' op expected perms attribute to have size 3 (input rank) but got size 7}}
+  %0 = tosa.transpose %arg0 {perms = array<i32: 6, 5, 4, 3, 2, 1, 0> }: (tensor<13x21x3xf32>) -> tensor<3x13x21xf32>
+  return %0 : tensor<3x13x21xf32>
+}
+
+// -----
+
+func.func @test_transpose_invalid_permutation_tensor(%arg0: tensor<13x21x3xf32>) -> tensor<?x?x?xf32> {
+  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
+  %0 = tosa.transpose %arg0 {perms = array<i32: 2, 0, 0> }: (tensor<13x21x3xf32>) -> tensor<?x?x?xf32>
+  return %0 : tensor<?x?x?xf32>
+}
+
+// -----
+
+func.func @test_transpose_invalid_permutation_negative(%arg0: tensor<3x2xi32>) -> tensor<*xi32> {
+  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
+  %1 = tosa.transpose %arg0 {perms = array<i32: -1, 0> }: (tensor<3x2xi32>) -> tensor<*xi32>
+  return %1 : tensor<*xi32>
+}
+
+// -----
+
+func.func @test_transpose_invalid_permutation_tensor_above_range(%arg0: tensor<3x2xi32>) -> tensor<*xi32> {
+  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
+  %1 = tosa.transpose %arg0 {perms = array<i32: 2, 0> }: (tensor<3x2xi32>) -> tensor<*xi32>
+  return %1 : tensor<*xi32>
+}
+
+// -----
+
+func.func @test_transpose_invalid_num_elements(%arg0: tensor<3x2xi32>) -> tensor<3x4xi32> {
+  // expected-error@+1 {{'tosa.transpose' op expected input1 and output to have same numbers of elements, got 6 and 12}}
+  %1 = tosa.transpose %arg0 {perms = array<i32: 1, 0> }: (tensor<3x2xi32>) -> tensor<3x4xi32>
+  return %1 : tensor<3x4xi32>
+}
+
+// -----
+
+func.func @test_transpose_invalid_permutation_types(%arg0: tensor<3x2xi32>) -> tensor<3x2xi32> {
+  // expected-error@+1 {{'tosa.transpose' op expected output tensor dim 0 to match input dim 1 with value of 2}}
+  %1 = tosa.transpose %arg0 {perms = array<i32: 1, 0> }: (tensor<3x2xi32>) -> tensor<3x2xi32>
+  return %1 : tensor<3x2xi32>
+}
+
+// -----
+
+func.func @test_transpose_invalid_permutation_types_dynamic_dim_ok(%arg0: tensor<2x?xi32>) -> tensor<3x4xi32> {
+  // expected-error@+1 {{'tosa.transpose' op expected output tensor dim 1 to match input dim 0 with value of 2}}
+  %1 = tosa.transpose %arg0 {perms = array<i32: 1, 0> }: (tensor<2x?xi32>) -> tensor<3x4xi32>
+  return %1 : tensor<3x4xi32>
+}
+
+// -----
+
+func.func @test_transpose_element_type_mismatch(%arg0: tensor<2x3xi32>) -> tensor<3x2xf32> {
+  // expected-error@+1 {{'tosa.transpose' op failed to verify that all of {input1, output} have same element type}}
+  %1 = tosa.transpose %arg0 {perms = array<i32: 1, 0>} : (tensor<2x3xi32>) -> tensor<3x2xf32>
+  return %1 : tensor<3x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @test_invalid_constant_permutation
+func.func @test_invalid_constant_permutation() {
+  %0 = tensor.empty() : tensor<3x4x5xi32>
+  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
+  %2 = tosa.transpose %0 {perms = array<i32: 3, 0, 1>}: (tensor<3x4x5xi32>) -> tensor<3x4x5xi32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: test_rank_size_constant_permutation
+func.func @test_rank_size_constant_permutation() {
+  %0 = arith.constant 6 : index
+  %2 = tensor.empty(%0) : tensor<?x27xi64>
+  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
+  %3 = tosa.transpose %2 {perms = array<i32: 0, 2>}: (tensor<?x27xi64>) -> tensor<?x27xi64>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: test_large_constant_permutation
+func.func @test_large_constant_permutation() {
+  %0 = arith.constant 6 : index
+  %2 = tensor.empty(%0) : tensor<?x27xi64>
+  // expected-error@+1 {{'tosa.transpose' op expected valid permutation indices}}
+  %3 = tosa.transpose %2 {perms = array<i32: 1185677355, 332462212>}: (tensor<?x27xi64>) -> tensor<?x27xi64>
+  return
+}
+
+// -----
+
+func.func @test_scalar_output_transpose(%arg0: tensor<*xf32>) -> tensor<f32> {
+  // expected-error@+1 {{'tosa.transpose' op result #0 must be tosa-conformant tensor of at least rank 1, but got 'tensor<f32>'}}
+  %1 = tosa.transpose %arg0 {perms = array<i32: 2, 0, 1>} : (tensor<*xf32>) -> tensor<f32>
+  return %1 : tensor<f32>
+}

From 96064e1b516aba4d7cbea2ab183b20b19b7eea86 Mon Sep 17 00:00:00 2001
From: Tai Ly <tai.ly@arm.com>
Date: Tue, 15 Apr 2025 14:36:53 -0500
Subject: [PATCH 029/710] [mlir][tosa] Add table size check for Table Op
 (#135262)

Add table size check for Table Op
and add lit tests to error_if_check.mlir
also corrected some existing tests that violated the table size checks

Signed-off-by: Tai Ly <tai.ly@arm.com>
---
 .../Tosa/Transforms/TosaValidation.cpp        | 24 ++++++++++++++++++-
 mlir/test/Dialect/Tosa/dynamic_extension.mlir |  4 ++--
 mlir/test/Dialect/Tosa/error_if_check.mlir    | 16 +++++++++++++
 mlir/test/Dialect/Tosa/invalid_extension.mlir |  4 ++--
 4 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index 11eb0d969d78b..ef9d27f8df0ad 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -1012,8 +1012,30 @@ bool checkErrorIfMul(Operation *op) {
   return true;
 }
 
+bool checkErrorIfTable(Operation *op) {
+  auto table = dyn_cast<tosa::TableOp>(op);
+  if (!table)
+    return true;
+
+  // REQUIRE(length(table) == TABLE_SIZE) where TABLE_SIZE is 256 or 513
+  const auto inputElemType = getElementTypeOrSelf(table.getInput1().getType());
+  const int tableSize = inputElemType.isInteger(8) ? 256 : 513;
+
+  const ShapeAdaptor tableShape(table.getTable().getType());
+  if (tableShape.hasStaticShape()) {
+    const auto numElements = tableShape.getNumElements();
+    if (numElements != tableSize) {
+      op->emitOpError() << "requires table size of " << tableSize << ", got "
+                        << numElements;
+      return false;
+    }
+  }
+
+  return true;
+}
+
 LogicalResult TosaValidation::applyErrorIfCheck(Operation *op) {
-  if (!checkErrorIfResize(op) || !checkErrorIfMul(op))
+  if (!checkErrorIfResize(op) || !checkErrorIfMul(op) || !checkErrorIfTable(op))
     return failure();
   return success();
 }
diff --git a/mlir/test/Dialect/Tosa/dynamic_extension.mlir b/mlir/test/Dialect/Tosa/dynamic_extension.mlir
index 0ec46022157d7..25e1aa195c3a0 100644
--- a/mlir/test/Dialect/Tosa/dynamic_extension.mlir
+++ b/mlir/test/Dialect/Tosa/dynamic_extension.mlir
@@ -13,8 +13,8 @@ func.func @test_mul_non_const(%arg0: tensor<13x21x3xi8>, %arg1: tensor<13x1x3xi8
 
 // -----
 
-func.func @test_table_non_const(%arg0 : tensor<4x5xi8>, %arg1 : tensor<513xi8>) -> () {
-  %0 = tosa.table %arg0, %arg1 : (tensor<4x5xi8>, tensor<513xi8>) -> tensor<4x5xi8>
+func.func @test_table_non_const(%arg0 : tensor<4x5xi8>, %arg1 : tensor<256xi8>) -> () {
+  %0 = tosa.table %arg0, %arg1 : (tensor<4x5xi8>, tensor<256xi8>) -> tensor<4x5xi8>
   return
 }
 
diff --git a/mlir/test/Dialect/Tosa/error_if_check.mlir b/mlir/test/Dialect/Tosa/error_if_check.mlir
index f7ca0faa8bc9e..65a69be91e0c8 100644
--- a/mlir/test/Dialect/Tosa/error_if_check.mlir
+++ b/mlir/test/Dialect/Tosa/error_if_check.mlir
@@ -113,3 +113,19 @@ func.func @test_mul_non_zero_shift(%arg0: tensor<1x8x8x8xi16>, %arg1: tensor<1x8
   %mul = tosa.mul %arg0, %arg1, %shift : (tensor<1x8x8x8xi16>, tensor<1x8x8x8xi16>, tensor<1xi8>) -> tensor<1x8x8x8xi32>
   return %mul : tensor<1x8x8x8xi32>
 }
+
+// -----
+// CHECK-LABEL: test_i16_table_size
+func.func @test_i16_table_size(%arg0: tensor<2x64xi16>, %arg1: tensor<256xi16>) -> tensor<2x64xi32> {
+  // expected-error@+1 {{'tosa.table' op requires table size of 513, got 256}}
+    %0 = tosa.table %arg0, %arg1 : (tensor<2x64xi16>, tensor<256xi16>) -> tensor<2x64xi32>
+    return %0 : tensor<2x64xi32>
+}
+
+// -----
+// CHECK-LABEL: test_i8_table_size
+func.func @test_i8_table_size(%arg0: tensor<2x64xi8>, %arg1: tensor<513xi8>) -> tensor<2x64xi8> {
+  // expected-error@+1 {{'tosa.table' op requires table size of 256, got 513}}
+    %0 = tosa.table %arg0, %arg1 : (tensor<2x64xi8>, tensor<513xi8>) -> tensor<2x64xi8>
+    return %0 : tensor<2x64xi8>
+}
diff --git a/mlir/test/Dialect/Tosa/invalid_extension.mlir b/mlir/test/Dialect/Tosa/invalid_extension.mlir
index 241e603e91c61..7386b1ba9df99 100644
--- a/mlir/test/Dialect/Tosa/invalid_extension.mlir
+++ b/mlir/test/Dialect/Tosa/invalid_extension.mlir
@@ -497,9 +497,9 @@ func.func @test_mul_non_const(%arg0: tensor<13x21x3xi8>, %arg1: tensor<13x1x3xi8
 
 // -----
 
-func.func @test_table_non_const(%arg0 : tensor<4x5xi8>, %arg1 : tensor<513xi8>) -> () {
+func.func @test_table_non_const(%arg0 : tensor<4x5xi8>, %arg1 : tensor<256xi8>) -> () {
   // expected-error@+1 {{'tosa.table' op expected compile time resolvable constant, but got variable value for operand #1}}
-  %0 = tosa.table %arg0, %arg1 : (tensor<4x5xi8>, tensor<513xi8>) -> tensor<4x5xi8>
+  %0 = tosa.table %arg0, %arg1 : (tensor<4x5xi8>, tensor<256xi8>) -> tensor<4x5xi8>
   return
 }
 

From 4f64c80d5a23c244f942193e58ecac666c173308 Mon Sep 17 00:00:00 2001
From: Michael Spencer <bigcheesegs@gmail.com>
Date: Tue, 15 Apr 2025 12:39:00 -0700
Subject: [PATCH 030/710] [llvm] Add missing include for !LLVM_ENABLE_THREADS

thread.h used report_fatal_error without including ErrorHandling.h
---
 llvm/include/llvm/Support/thread.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/include/llvm/Support/thread.h b/llvm/include/llvm/Support/thread.h
index e3005fdb63175..ef2fba822cb1c 100644
--- a/llvm/include/llvm/Support/thread.h
+++ b/llvm/include/llvm/Support/thread.h
@@ -213,6 +213,7 @@ inline thread::id get_id() { return std::this_thread::get_id(); }
 
 #else // !LLVM_ENABLE_THREADS
 
+#include "llvm/Support/ErrorHandling.h"
 #include <utility>
 
 namespace llvm {

From e6e56f5b6a80c6ce55630d6075475cb363afb149 Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Tue, 15 Apr 2025 12:45:18 -0700
Subject: [PATCH 031/710] [MemProf] Handle recursion during stack node update
 (#135837)

If we are replacing a sequence of stack nodes with a single node
representing inlined IR, and the stack id sequence contains recursion,
we may have already removed some edges. Handle this case correctly by
skipping the now removed edge.
---
 .../Transforms/IPO/MemProfContextDisambiguation.cpp   |  7 ++++++-
 .../MemProfContextDisambiguation/inlined2.ll          | 11 +++++++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index a5e0251277d8f..561c01cb01f82 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -1711,7 +1711,12 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
       // edge from the prior node.
       if (PrevNode) {
         auto *PrevEdge = CurNode->findEdgeFromCallee(PrevNode);
-        assert(PrevEdge);
+        // If the sequence contained recursion, we might have already removed
+        // some edges during the connectNewNode calls above.
+        if (!PrevEdge) {
+          PrevNode = CurNode;
+          continue;
+        }
         set_subtract(PrevEdge->getContextIds(), SavedContextIds);
         if (PrevEdge->getContextIds().empty())
           removeEdgeFromGraph(PrevEdge);
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/inlined2.ll b/llvm/test/Transforms/MemProfContextDisambiguation/inlined2.ll
index cc97b5290e25a..2cc655e927d12 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/inlined2.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/inlined2.ll
@@ -40,6 +40,9 @@
 ;; in the input IR to ensure that the MIB call chain is matched to the longer
 ;; inline sequences from main.
 ;;
+;; Update: the inlined sequence of callsite ids was manually modified to
+;; include some recursion, which reproduced an error before it was fixed.
+;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
 ; RUN: opt -passes=memprof-context-disambiguation -supports-hot-cold-new \
@@ -96,13 +99,13 @@ attributes #7 = { builtin }
 !6 = !{i32 7, !"frame-pointer", i32 2}
 !7 = !{!8, !10}
 !8 = !{!9, !"notcold"}
-!9 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 8632435727821051414}
+!9 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 -5964873800580613432, i64 8632435727821051414}
 !10 = !{!11, !"cold"}
-!11 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 -3421689549917153178}
+!11 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 -5964873800580613432, i64 -3421689549917153178}
 !12 = !{i64 9086428284934609951}
 !13 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
-!14 = !{i64 -5964873800580613432, i64 2732490490862098848, i64 8632435727821051414}
-!15 = !{i64 -5964873800580613432, i64 2732490490862098848, i64 -3421689549917153178}
+!14 = !{i64 -5964873800580613432, i64 2732490490862098848, i64 -5964873800580613432, i64 8632435727821051414}
+!15 = !{i64 -5964873800580613432, i64 2732490490862098848, i64 -5964873800580613432, i64 -3421689549917153178}
 
 
 ; DUMP: CCG before cloning:

From a3283a92aea147e89d9d404fa7c8500223c7c22a Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Tue, 15 Apr 2025 12:54:25 -0700
Subject: [PATCH 032/710] [PAC] Add support for __ptrauth type qualifier
 (#100830)

The qualifier allows programmer to directly control how pointers are
signed when they are stored in a particular variable.

The qualifier takes three arguments: the signing key, a flag specifying
whether address discrimination should be used, and a non-negative
integer that is used for additional discrimination.

```
typedef void (*my_callback)(const void*);
my_callback __ptrauth(ptrauth_key_process_dependent_code, 1, 0xe27a) callback;
```

Co-Authored-By: John McCall rjmccall@apple.com
---
 clang/docs/PointerAuthentication.rst          |  46 ++
 clang/docs/ReleaseNotes.rst                   |   2 +
 clang/include/clang/AST/Type.h                |  21 +-
 clang/include/clang/Basic/Attr.td             |   8 +
 clang/include/clang/Basic/AttrDocs.td         |  28 +
 .../clang/Basic/DiagnosticParseKinds.td       |   3 +
 .../clang/Basic/DiagnosticSemaKinds.td        |  43 +-
 clang/include/clang/Basic/Features.def        |   1 +
 clang/include/clang/Basic/TokenKinds.def      |   1 +
 clang/include/clang/Parse/Parser.h            |   2 +
 clang/include/clang/Sema/Sema.h               |  11 +
 clang/lib/AST/ASTContext.cpp                  |   1 +
 clang/lib/AST/DeclCXX.cpp                     |  27 +
 clang/lib/AST/ItaniumMangle.cpp               |  20 +
 clang/lib/AST/MicrosoftMangle.cpp             |  13 +
 clang/lib/AST/TypePrinter.cpp                 |  40 +
 clang/lib/CodeGen/CGClass.cpp                 |   3 +
 clang/lib/CodeGen/CGDebugInfo.cpp             |  19 +-
 clang/lib/CodeGen/CGDecl.cpp                  |  12 +-
 clang/lib/CodeGen/CGExpr.cpp                  |  94 ++-
 clang/lib/CodeGen/CGExprConstant.cpp          |  23 +-
 clang/lib/CodeGen/CGExprScalar.cpp            |  62 ++
 clang/lib/CodeGen/CGPointerAuth.cpp           | 129 +++
 clang/lib/CodeGen/CodeGenFunction.h           |  26 +-
 clang/lib/Parse/ParseDecl.cpp                 |  52 ++
 clang/lib/Sema/SemaCast.cpp                   |  19 +-
 clang/lib/Sema/SemaChecking.cpp               |  57 ++
 clang/lib/Sema/SemaDecl.cpp                   |  13 +-
 clang/lib/Sema/SemaDeclCXX.cpp                |  38 +
 clang/lib/Sema/SemaExpr.cpp                   |  14 +
 clang/lib/Sema/SemaExprCXX.cpp                |   5 +
 clang/lib/Sema/SemaObjCProperty.cpp           |   3 +
 clang/lib/Sema/SemaOverload.cpp               |  15 +
 clang/lib/Sema/SemaType.cpp                   |  79 ++
 clang/lib/Sema/TreeTransform.h                |  11 +
 clang/test/AST/ast-dump-ptrauth-json.cpp      |   3 +
 clang/test/CodeGen/ptrauth-debuginfo.c        |  35 +
 .../CodeGen/ptrauth-qualifier-const-init.c    |  86 ++
 .../test/CodeGen/ptrauth-qualifier-function.c | 145 ++++
 .../CodeGen/ptrauth-qualifier-loadstore.c     | 745 ++++++++++++++++++
 .../CodeGenCXX/mangle-itanium-ptrauth.cpp     |  12 +
 clang/test/CodeGenCXX/mangle-ms-ptrauth.cpp   |  17 +
 .../CodeGenCXX/ptrauth-qualifier-struct.cpp   | 168 ++++
 .../CodeGenObjCXX/ptrauth-struct-cxx-abi.mm   |  35 +
 clang/test/Parser/ptrauth-qualifier.c         |  18 +
 clang/test/Preprocessor/ptrauth_extension.c   |  13 +
 clang/test/Sema/ptrauth-atomic-ops.c          | 118 +++
 clang/test/Sema/ptrauth-qualifier.c           | 103 +++
 clang/test/SemaCXX/ptrauth-qualifier.cpp      | 213 +++++
 .../SemaCXX/ptrauth-template-parameters.cpp   |  29 +
 clang/test/SemaObjC/ptrauth-qualifier.m       |  56 ++
 libcxxabi/test/test_demangle.pass.cpp         |   3 +
 .../include/llvm/Demangle/MicrosoftDemangle.h |   8 +
 .../llvm/Demangle/MicrosoftDemangleNodes.h    |  22 +-
 llvm/lib/Demangle/MicrosoftDemangle.cpp       |  48 +-
 llvm/lib/Demangle/MicrosoftDemangleNodes.cpp  |  10 +
 llvm/test/Demangle/ms-ptrauth.test            |  12 +
 57 files changed, 2801 insertions(+), 39 deletions(-)
 create mode 100644 clang/test/CodeGen/ptrauth-debuginfo.c
 create mode 100644 clang/test/CodeGen/ptrauth-qualifier-const-init.c
 create mode 100644 clang/test/CodeGen/ptrauth-qualifier-function.c
 create mode 100644 clang/test/CodeGen/ptrauth-qualifier-loadstore.c
 create mode 100644 clang/test/CodeGenCXX/mangle-itanium-ptrauth.cpp
 create mode 100644 clang/test/CodeGenCXX/mangle-ms-ptrauth.cpp
 create mode 100644 clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
 create mode 100644 clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
 create mode 100644 clang/test/Parser/ptrauth-qualifier.c
 create mode 100644 clang/test/Preprocessor/ptrauth_extension.c
 create mode 100644 clang/test/Sema/ptrauth-atomic-ops.c
 create mode 100644 clang/test/Sema/ptrauth-qualifier.c
 create mode 100644 clang/test/SemaCXX/ptrauth-qualifier.cpp
 create mode 100644 clang/test/SemaCXX/ptrauth-template-parameters.cpp
 create mode 100644 clang/test/SemaObjC/ptrauth-qualifier.m
 create mode 100644 llvm/test/Demangle/ms-ptrauth.test

diff --git a/clang/docs/PointerAuthentication.rst b/clang/docs/PointerAuthentication.rst
index 68674f318c84f..41818d43ac687 100644
--- a/clang/docs/PointerAuthentication.rst
+++ b/clang/docs/PointerAuthentication.rst
@@ -280,6 +280,52 @@ a number of different tests.
   normal interface.  This may be true even on targets where pointer
   authentication is not enabled by default.
 
+__ptrauth Qualifier
+^^^^^^^^^^^^^^^^^^^
+
+``__ptrauth(key, address, discriminator)`` is an extended type
+qualifier which causes so-qualified objects to hold pointers signed using the
+specified schema rather than the default schema for such types.
+
+In the current implementation in Clang, the qualified type must be a C pointer
+type, either to a function or to an object.  It currently cannot be an
+Objective-C pointer type, a C++ reference type, or a block pointer type; these
+restrictions may be lifted in the future.
+
+The qualifier's operands are as follows:
+
+- ``key`` - an expression evaluating to a key value from ``<ptrauth.h>``; must
+  be a constant expression
+
+- ``address`` - whether to use address diversity (1) or not (0); must be
+  a constant expression with one of these two values
+
+- ``discriminator`` - a constant discriminator; must be a constant expression
+
+See `Discriminators`_ for more information about discriminators.
+
+Currently the operands must be constant-evaluable even within templates. In the
+future this restriction may be lifted to allow value-dependent expressions as
+long as they instantiate to a constant expression.
+
+Consistent with the ordinary C/C++ rule for parameters, top-level ``__ptrauth``
+qualifiers on a parameter (after parameter type adjustment) are ignored when
+deriving the type of the function.  The parameter will be passed using the
+default ABI for the unqualified pointer type.
+
+If ``x`` is an object of type ``__ptrauth(key, address, discriminator) T``,
+then the signing schema of the value stored in ``x`` is a key of ``key`` and
+a discriminator determined as follows:
+
+- if ``address`` is 0, then the discriminator is ``discriminator``;
+
+- if ``address`` is 1 and ``discriminator`` is 0, then the discriminator is
+  ``&x``; otherwise
+
+- if ``address`` is 1 and ``discriminator`` is non-zero, then the discriminator
+  is ``ptrauth_blend_discriminator(&x, discriminator)``; see
+  `ptrauth_blend_discriminator`_.
+
 ``<ptrauth.h>``
 ~~~~~~~~~~~~~~~
 
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 6025e76029d19..38142ad32bea0 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -552,6 +552,8 @@ Arm and AArch64 Support
   ARM targets, however this will now disable NEON instructions being generated. The ``simd`` option is 
   also now printed when the ``--print-supported-extensions`` option is used.
 
+-  Support for __ptrauth type qualifier has been added.
+
 Android Support
 ^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 74886ef0cd824..5bf036e3347eb 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -312,6 +312,12 @@ class PointerAuthQualifier {
     return Result;
   }
 
+  std::string getAsString() const;
+  std::string getAsString(const PrintingPolicy &Policy) const;
+
+  bool isEmptyWhenPrinted(const PrintingPolicy &Policy) const;
+  void print(raw_ostream &OS, const PrintingPolicy &Policy) const;
+
   void Profile(llvm::FoldingSetNodeID &ID) const { ID.AddInteger(Data); }
 };
 
@@ -562,7 +568,7 @@ class Qualifiers {
 
   bool hasAddressSpace() const { return Mask & AddressSpaceMask; }
   LangAS getAddressSpace() const {
-    return static_cast<LangAS>(Mask >> AddressSpaceShift);
+    return static_cast<LangAS>((Mask & AddressSpaceMask) >> AddressSpaceShift);
   }
   bool hasTargetSpecificAddressSpace() const {
     return isTargetAddressSpace(getAddressSpace());
@@ -803,6 +809,9 @@ class Qualifiers {
   static_assert(sizeof(PointerAuthQualifier) == sizeof(uint32_t),
                 "PointerAuthQualifier must be 32 bits");
 
+  static constexpr uint64_t PtrAuthShift = 32;
+  static constexpr uint64_t PtrAuthMask = UINT64_C(0xffffffff) << PtrAuthShift;
+
   static constexpr uint64_t UMask = 0x8;
   static constexpr uint64_t UShift = 3;
   static constexpr uint64_t GCAttrMask = 0x30;
@@ -810,10 +819,8 @@ class Qualifiers {
   static constexpr uint64_t LifetimeMask = 0x1C0;
   static constexpr uint64_t LifetimeShift = 6;
   static constexpr uint64_t AddressSpaceMask =
-      ~(CVRMask | UMask | GCAttrMask | LifetimeMask);
+      ~(CVRMask | UMask | GCAttrMask | LifetimeMask | PtrAuthMask);
   static constexpr uint64_t AddressSpaceShift = 9;
-  static constexpr uint64_t PtrAuthShift = 32;
-  static constexpr uint64_t PtrAuthMask = uint64_t(0xffffffff) << PtrAuthShift;
 };
 
 class QualifiersAndAtomic {
@@ -1449,6 +1456,12 @@ class QualType {
     return getQualifiers().getPointerAuth();
   }
 
+  bool hasAddressDiscriminatedPointerAuth() const {
+    if (PointerAuthQualifier PtrAuth = getPointerAuth())
+      return PtrAuth.isAddressDiscriminated();
+    return false;
+  }
+
   enum PrimitiveDefaultInitializeKind {
     /// The type does not fall into any of the following categories. Note that
     /// this case is zero-valued so that values of this enum can be used as a
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index b7ad432738b29..9d4900f3029c8 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3548,6 +3548,14 @@ def ObjCRequiresPropertyDefs : InheritableAttr {
   let SimpleHandler = 1;
 }
 
+def PointerAuth : TypeAttr {
+  let Spellings = [CustomKeyword<"__ptrauth">];
+  let Args = [IntArgument<"Key">,
+              BoolArgument<"AddressDiscriminated", 1>,
+              IntArgument<"ExtraDiscriminator", 1>];
+  let Documentation = [PtrAuthDocs];
+}
+
 def Unused : InheritableAttr {
   let Spellings = [CXX11<"", "maybe_unused", 201603>, GCC<"unused">,
                    C23<"", "maybe_unused", 202106>];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 97a5f24d35d7d..76f805ef373dd 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -2179,6 +2179,34 @@ Also see the documentation for `@available
   }];
 }
 
+def PtrAuthDocs : Documentation {
+  let Category = DocCatVariable;
+  let Heading = "__ptrauth";
+  let Content = [{
+The ``__ptrauth`` qualifier allows the programmer to directly control
+how pointers are signed when they are stored in a particular variable.
+This can be used to strengthen the default protections of pointer
+authentication and make it more difficult for an attacker to escalate
+an ability to alter memory into full control of a process.
+
+.. code-block:: c
+
+  #include <ptrauth.h>
+
+  typedef void (*my_callback)(const void*);
+  my_callback __ptrauth(ptrauth_key_process_dependent_code, 1, 0xe27a) callback;
+
+The first argument to ``__ptrauth`` is the name of the signing key.
+Valid key names for the target are defined in ``<ptrauth.h>``.
+
+The second argument to ``__ptrauth`` is a flag (0 or 1) specifying whether
+the object should use address discrimination.
+
+The third argument to ``__ptrauth`` is a 16-bit non-negative integer which
+allows additional discrimination between objects.
+  }];
+}
+
 def ExternalSourceSymbolDocs : Documentation {
   let Category = DocCatDecl;
   let Content = [{
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 7a3cac528a363..9975520f4f9ff 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1721,6 +1721,9 @@ def warn_pragma_unroll_cuda_value_in_parens : Warning<
   "argument to '#pragma unroll' should not be in parentheses in CUDA C/C++">,
   InGroup<CudaCompat>;
 
+def err_ptrauth_qualifier_bad_arg_count : Error<
+  "'__ptrauth' qualifier must take between 1 and 3 arguments">;
+
 def warn_cuda_attr_lambda_position : Warning<
   "nvcc does not allow '__%0__' to appear after the parameter list in lambdas">,
   InGroup<CudaCompat>;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index f4ab620ae61d2..3f7499d8656bd 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -1014,6 +1014,22 @@ def err_ptrauth_indirect_goto_addrlabel_arithmetic : Error<
   "%select{subtraction|addition}0 of address-of-label expressions is not "
   "supported with ptrauth indirect gotos">;
 
+// __ptrauth qualifier
+def err_ptrauth_qualifier_invalid : Error<
+  "%select{return type|parameter type|property}1 may not be qualified with '__ptrauth'; type is %0">;
+def err_ptrauth_qualifier_cast : Error<
+  "cannot cast to '__ptrauth'-qualified type %0">;
+def err_ptrauth_qualifier_nonpointer : Error<
+  "'__ptrauth' qualifier only applies to pointer types; %0 is invalid">;
+def err_ptrauth_qualifier_redundant : Error<
+  "type %0 is already %1-qualified">;
+def err_ptrauth_arg_not_ice : Error<
+  "argument to '__ptrauth' must be an integer constant expression">;
+def err_ptrauth_address_discrimination_invalid : Error<
+  "invalid address discrimination flag '%0'; '__ptrauth' requires '0' or '1'">;
+def err_ptrauth_extra_discriminator_invalid : Error<
+  "invalid extra discriminator flag '%0'; '__ptrauth' requires a value between '0' and '%1'">;
+
 /// main()
 // static main() is not an error in C, just in C++.
 def warn_static_main : Warning<"'main' should not be declared static">,
@@ -3923,7 +3939,8 @@ def note_cannot_use_trivial_abi_reason : Note<
   "its copy constructors and move constructors are all deleted|"
   "it is polymorphic|"
   "it has a base of a non-trivial class type|it has a virtual base|"
-  "it has a __weak field|it has a field of a non-trivial class type}1">;
+  "it has a __weak field|it has a field of a non-trivial class type|"
+  "it has an address-discriminated '__ptrauth' field}1">;
 
 // Availability attribute
 def warn_availability_unknown_platform : Warning<
@@ -5021,6 +5038,10 @@ def note_ovl_candidate_bad_ownership : Note<
     "%select{no|__unsafe_unretained|__strong|__weak|__autoreleasing}4 ownership,"
     " but parameter has %select{no|__unsafe_unretained|__strong|__weak|"
     "__autoreleasing}5 ownership">;
+def note_ovl_candidate_bad_ptrauth : Note<
+    "candidate %sub{select_ovl_candidate_kind}0,1,2 not viable: "
+    "%ordinal8 argument (%3) has %select{no '__ptrauth'|%5}4 qualifier,"
+    " but parameter has %select{no '__ptrauth'|%7}6 qualifier">;
 def note_ovl_candidate_bad_cvr_this : Note<
     "candidate %sub{select_ovl_candidate_kind}0,1,2 not viable: "
     "'this' argument has type %3, but method is not marked "
@@ -6092,7 +6113,7 @@ def note_deleted_special_member_class_subobject : Note<
   "%select{default|corresponding|default|default|default}4 constructor}0|"
   "destructor}5"
   "%select{||s||}4"
-  "|is an ObjC pointer}6">;
+  "|is an ObjC pointer|has an address-discriminated '__ptrauth' qualifier}6">;
 def note_default_constructed_field
     : Note<"default constructed field %0 declared here">;
 def note_deleted_default_ctor_uninit_field : Note<
@@ -8938,6 +8959,19 @@ def err_typecheck_incompatible_ownership : Error<
   "sending to parameter of different type}0,1"
   "|%diff{casting $ to type $|casting between types}0,1}2"
   " changes retain/release properties of pointer">;
+def err_typecheck_incompatible_ptrauth : Error<
+  "%enum_select<AssignmentAction>{%Assigning{%diff{assigning $ to $|assigning to different types}1,0}"
+  "|%Passing{%diff{passing $ to parameter of type $|"
+  "passing to parameter of different type}0,1}"
+  "|%Returning{%diff{returning $ from a function with result type $|"
+  "returning from function with different return type}0,1}"
+  "|%Converting{%diff{converting $ to type $|converting between types}0,1}"
+  "|%Initializing{%diff{initializing $ with an expression of type $|"
+  "initializing with expression of different type}0,1}"
+  "|%Sending{%diff{sending $ to parameter of type $|"
+  "sending to parameter of different type}0,1}"
+  "|%Casting{%diff{casting $ to type $|casting between types}0,1}}2"
+  " changes pointer authentication of pointee type">;
 def err_typecheck_comparison_of_distinct_blocks : Error<
   "comparison of distinct block types%diff{ ($ and $)|}0,1">;
 
@@ -9066,6 +9100,9 @@ def err_atomic_op_needs_non_const_atomic : Error<
 def err_atomic_op_needs_non_const_pointer : Error<
   "address argument to atomic operation must be a pointer to non-const "
   "type (%0 invalid)">;
+def err_atomic_op_needs_non_address_discriminated_pointer : Error<
+  "address argument to %select{atomic|__sync}0 operation must be a pointer to a non address discriminated "
+  "type (%1 invalid)">;
 def err_atomic_op_needs_trivial_copy : Error<
   "address argument to atomic operation must be a pointer to a "
   "trivially-copyable type (%0 invalid)">;
@@ -9343,6 +9380,8 @@ def ext_typecheck_cond_pointer_integer_mismatch : ExtWarn<
   "pointer/integer type mismatch in conditional expression"
   "%diff{ ($ and $)|}0,1">,
   InGroup<DiagGroup<"conditional-type-mismatch">>;
+def err_typecheck_cond_incompatible_ptrauth : Error<
+  "'__ptrauth' qualification mismatch%diff{ ($ and $)|}0,1">;
 def err_typecheck_choose_expr_requires_constant : Error<
   "'__builtin_choose_expr' requires a constant expression">;
 def warn_unused_expr : Warning<"expression result unused">,
diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index b4409efaa9c04..14bff8a68846d 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -107,6 +107,7 @@ FEATURE(thread_sanitizer, LangOpts.Sanitize.has(SanitizerKind::Thread))
 FEATURE(dataflow_sanitizer, LangOpts.Sanitize.has(SanitizerKind::DataFlow))
 FEATURE(scudo, LangOpts.Sanitize.hasOneOf(SanitizerKind::Scudo))
 FEATURE(ptrauth_intrinsics, LangOpts.PointerAuthIntrinsics)
+EXTENSION(ptrauth_qualifier, LangOpts.PointerAuthIntrinsics)
 FEATURE(ptrauth_calls, LangOpts.PointerAuthCalls)
 FEATURE(ptrauth_returns, LangOpts.PointerAuthReturns)
 FEATURE(ptrauth_vtable_pointer_address_discrimination, LangOpts.PointerAuthVTPtrAddressDiscrimination)
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 880928ae0447d..868e851342eb8 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -348,6 +348,7 @@ KEYWORD(_Thread_local               , KEYALL)
 KEYWORD(__func__                    , KEYALL)
 KEYWORD(__objc_yes                  , KEYALL)
 KEYWORD(__objc_no                   , KEYALL)
+KEYWORD(__ptrauth                   , KEYALL)
 
 // C2y
 UNARY_EXPR_OR_TYPE_TRAIT(_Countof, CountOf, KEYNOCXX)
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 53da6269a3b11..9ebcf144ba59e 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3169,6 +3169,8 @@ class Parser : public CodeCompletionHandler {
                                SourceLocation *endLoc = nullptr);
   ExprResult ParseExtIntegerArgument();
 
+  void ParsePtrauthQualifier(ParsedAttributes &Attrs);
+
   VirtSpecifiers::Specifier isCXX11VirtSpecifier(const Token &Tok) const;
   VirtSpecifiers::Specifier isCXX11VirtSpecifier() const {
     return isCXX11VirtSpecifier(Tok);
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 5ab0af8234e26..fe37fd7701ce3 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3547,6 +3547,17 @@ class Sema final : public SemaBase {
 
   bool checkConstantPointerAuthKey(Expr *keyExpr, unsigned &key);
 
+  enum PointerAuthDiscArgKind {
+    // Address discrimination argument of __ptrauth.
+    PADAK_AddrDiscPtrAuth,
+
+    // Extra discriminator argument of __ptrauth.
+    PADAK_ExtraDiscPtrAuth,
+  };
+
+  bool checkPointerAuthDiscriminatorArg(Expr *Arg, PointerAuthDiscArgKind Kind,
+                                        unsigned &IntVal);
+
   /// Diagnose function specifiers on a declaration of an identifier that
   /// does not identify a function.
   void DiagnoseFunctionSpecifiers(const DeclSpec &DS);
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index dfae2f5511b43..c6ffe7bbf5257 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -11454,6 +11454,7 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
     if (LQuals.getCVRQualifiers() != RQuals.getCVRQualifiers() ||
         LQuals.getAddressSpace() != RQuals.getAddressSpace() ||
         LQuals.getObjCLifetime() != RQuals.getObjCLifetime() ||
+        !LQuals.getPointerAuth().isEquivalent(RQuals.getPointerAuth()) ||
         LQuals.hasUnaligned() != RQuals.hasUnaligned())
       return {};
 
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 36131d19cbcdf..4d07efd58f518 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -1118,6 +1118,33 @@ void CXXRecordDecl::addedMember(Decl *D) {
     } else if (!T.isCXX98PODType(Context))
       data().PlainOldData = false;
 
+    // If a class has an address-discriminated signed pointer member, it is a
+    // non-POD type and its copy constructor, move constructor, copy assignment
+    // operator, move assignment operator are non-trivial.
+    if (PointerAuthQualifier Q = T.getPointerAuth()) {
+      if (Q.isAddressDiscriminated()) {
+        struct DefinitionData &Data = data();
+        Data.PlainOldData = false;
+        Data.HasTrivialSpecialMembers &=
+            ~(SMF_CopyConstructor | SMF_MoveConstructor | SMF_CopyAssignment |
+              SMF_MoveAssignment);
+        setArgPassingRestrictions(RecordArgPassingKind::CanNeverPassInRegs);
+
+        // Copy/move constructors/assignment operators of a union are deleted by
+        // default if it has an address-discriminated ptrauth field.
+        if (isUnion()) {
+          data().DefaultedCopyConstructorIsDeleted = true;
+          data().DefaultedMoveConstructorIsDeleted = true;
+          data().DefaultedCopyAssignmentIsDeleted = true;
+          data().DefaultedMoveAssignmentIsDeleted = true;
+          data().NeedOverloadResolutionForCopyConstructor = true;
+          data().NeedOverloadResolutionForMoveConstructor = true;
+          data().NeedOverloadResolutionForCopyAssignment = true;
+          data().NeedOverloadResolutionForMoveAssignment = true;
+        }
+      }
+    }
+
     if (Field->hasAttr<ExplicitInitAttr>())
       setHasUninitializedExplicitInitFields(true);
 
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 140f29b431fc3..d0ab60700cb15 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2895,6 +2895,26 @@ void CXXNameMangler::mangleQualifiers(Qualifiers Quals, const DependentAddressSp
   if (Quals.hasUnaligned())
     mangleVendorQualifier("__unaligned");
 
+  // __ptrauth.  Note that this is parameterized.
+  if (PointerAuthQualifier PtrAuth = Quals.getPointerAuth()) {
+    mangleVendorQualifier("__ptrauth");
+    // For now, since we only allow non-dependent arguments, we can just
+    // inline the mangling of those arguments as literals.  We treat the
+    // key and extra-discriminator arguments as 'unsigned int' and the
+    // address-discriminated argument as 'bool'.
+    Out << "I"
+           "Lj"
+        << PtrAuth.getKey()
+        << "E"
+           "Lb"
+        << unsigned(PtrAuth.isAddressDiscriminated())
+        << "E"
+           "Lj"
+        << PtrAuth.getExtraDiscriminator()
+        << "E"
+           "E";
+  }
+
   // Remaining ARC ownership qualifiers.
   switch (Quals.getObjCLifetime()) {
   case Qualifiers::OCL_None:
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index a6efd887d4e13..20bfb7f89625b 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -430,6 +430,7 @@ class MicrosoftCXXNameMangler {
   void mangleRefQualifier(RefQualifierKind RefQualifier);
   void manglePointerCVQualifiers(Qualifiers Quals);
   void manglePointerExtQualifiers(Qualifiers Quals, QualType PointeeType);
+  void manglePointerAuthQualifier(Qualifiers Quals);
 
   void mangleUnscopedTemplateName(GlobalDecl GD);
   void
@@ -2340,6 +2341,17 @@ void MicrosoftCXXNameMangler::manglePointerExtQualifiers(Qualifiers Quals,
     Out << 'F';
 }
 
+void MicrosoftCXXNameMangler::manglePointerAuthQualifier(Qualifiers Quals) {
+  PointerAuthQualifier PointerAuth = Quals.getPointerAuth();
+  if (!PointerAuth)
+    return;
+
+  Out << "__ptrauth";
+  mangleNumber(PointerAuth.getKey());
+  mangleNumber(PointerAuth.isAddressDiscriminated());
+  mangleNumber(PointerAuth.getExtraDiscriminator());
+}
+
 void MicrosoftCXXNameMangler::manglePointerCVQualifiers(Qualifiers Quals) {
   // <pointer-cv-qualifiers> ::= P  # no qualifiers
   //                         ::= Q  # const
@@ -3372,6 +3384,7 @@ void MicrosoftCXXNameMangler::mangleType(const PointerType *T, Qualifiers Quals,
   QualType PointeeType = T->getPointeeType();
   manglePointerCVQualifiers(Quals);
   manglePointerExtQualifiers(Quals, PointeeType);
+  manglePointerAuthQualifier(Quals);
 
   // For pointer size address spaces, go down the same type mangling path as
   // non address space types.
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index cc8c874140167..7b1d1c7ae2131 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2016,6 +2016,7 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   case attr::Ptr64:
   case attr::SPtr:
   case attr::UPtr:
+  case attr::PointerAuth:
   case attr::AddressSpace:
   case attr::CmseNSCall:
   case attr::AnnotateType:
@@ -2512,6 +2513,33 @@ void clang::printTemplateArgumentList(raw_ostream &OS,
   printTo(OS, Args, InnerPolicy, TPL, /*isPack*/ false, /*parmIndex*/ 0);
 }
 
+std::string PointerAuthQualifier::getAsString() const {
+  LangOptions LO;
+  return getAsString(PrintingPolicy(LO));
+}
+
+std::string PointerAuthQualifier::getAsString(const PrintingPolicy &P) const {
+  SmallString<64> Buf;
+  llvm::raw_svector_ostream StrOS(Buf);
+  print(StrOS, P);
+  return StrOS.str().str();
+}
+
+bool PointerAuthQualifier::isEmptyWhenPrinted(const PrintingPolicy &P) const {
+  return !isPresent();
+}
+
+void PointerAuthQualifier::print(raw_ostream &OS,
+                                 const PrintingPolicy &P) const {
+  if (!isPresent())
+    return;
+
+  OS << "__ptrauth(";
+  OS << getKey();
+  OS << "," << unsigned(isAddressDiscriminated()) << ","
+     << getExtraDiscriminator() << ")";
+}
+
 std::string Qualifiers::getAsString() const {
   LangOptions LO;
   return getAsString(PrintingPolicy(LO));
@@ -2541,6 +2569,10 @@ bool Qualifiers::isEmptyWhenPrinted(const PrintingPolicy &Policy) const {
     if (!(lifetime == Qualifiers::OCL_Strong && Policy.SuppressStrongLifetime))
       return false;
 
+  if (PointerAuthQualifier PointerAuth = getPointerAuth();
+      PointerAuth && !PointerAuth.isEmptyWhenPrinted(Policy))
+    return false;
+
   return true;
 }
 
@@ -2651,6 +2683,14 @@ void Qualifiers::print(raw_ostream &OS, const PrintingPolicy& Policy,
     }
   }
 
+  if (PointerAuthQualifier PointerAuth = getPointerAuth()) {
+    if (addSpace)
+      OS << ' ';
+    addSpace = true;
+
+    PointerAuth.print(OS, Policy);
+  }
+
   if (appendSpaceIfNonEmpty && addSpace)
     OS << ' ';
 }
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 1c73a5bf75f37..7176fe025b386 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -925,6 +925,9 @@ namespace {
       Qualifiers Qual = F->getType().getQualifiers();
       if (Qual.hasVolatile() || Qual.hasObjCLifetime())
         return false;
+      if (PointerAuthQualifier Q = F->getType().getPointerAuth();
+          Q && Q.isAddressDiscriminated())
+        return false;
       return true;
     }
 
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index e63341c180420..f3ec498d4064b 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -1059,8 +1059,23 @@ llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty,
   // additional ones.
   llvm::dwarf::Tag Tag = getNextQualifier(Qc);
   if (!Tag) {
-    assert(Qc.empty() && "Unknown type qualifier for debug info");
-    return getOrCreateType(QualType(T, 0), Unit);
+    if (Qc.getPointerAuth()) {
+      unsigned Key = Qc.getPointerAuth().getKey();
+      bool IsDiscr = Qc.getPointerAuth().isAddressDiscriminated();
+      unsigned ExtraDiscr = Qc.getPointerAuth().getExtraDiscriminator();
+      bool IsaPointer = Qc.getPointerAuth().isIsaPointer();
+      bool AuthenticatesNullValues =
+          Qc.getPointerAuth().authenticatesNullValues();
+      Qc.removePointerAuth();
+      assert(Qc.empty() && "Unknown type qualifier for debug info");
+      llvm::DIType *FromTy = getOrCreateType(QualType(T, 0), Unit);
+      return DBuilder.createPtrAuthQualifiedType(FromTy, Key, IsDiscr,
+                                                 ExtraDiscr, IsaPointer,
+                                                 AuthenticatesNullValues);
+    } else {
+      assert(Qc.empty() && "Unknown type qualifier for debug info");
+      return getOrCreateType(QualType(T, 0), Unit);
+    }
   }
 
   auto *FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit);
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index db8dbf86eca4f..db34e2738b4cf 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -776,11 +776,17 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D,
                                      LValue lvalue, bool capturedByInit) {
   Qualifiers::ObjCLifetime lifetime = lvalue.getObjCLifetime();
   if (!lifetime) {
-    llvm::Value *value = EmitScalarExpr(init);
+    llvm::Value *Value;
+    if (PointerAuthQualifier PtrAuth = lvalue.getQuals().getPointerAuth()) {
+      Value = EmitPointerAuthQualify(PtrAuth, init, lvalue.getAddress());
+      lvalue.getQuals().removePointerAuth();
+    } else {
+      Value = EmitScalarExpr(init);
+    }
     if (capturedByInit)
       drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
-    EmitNullabilityCheck(lvalue, value, init->getExprLoc());
-    EmitStoreThroughLValue(RValue::get(value), lvalue, true);
+    EmitNullabilityCheck(lvalue, Value, init->getExprLoc());
+    EmitStoreThroughLValue(RValue::get(Value), lvalue, true);
     return;
   }
 
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 3da21cebd9d68..abb88477062fc 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -569,7 +569,15 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
     // initialized it.
     if (!Var->hasInitializer()) {
       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
-      EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
+      QualType RefType = M->getType().withoutLocalFastQualifiers();
+      if (RefType.getPointerAuth()) {
+        // Use the qualifier of the reference temporary to sign the pointer.
+        LValue LV = MakeRawAddrLValue(Object.getPointer(), RefType,
+                                      Object.getAlignment());
+        EmitScalarInit(E, M->getExtendingDecl(), LV, false);
+      } else {
+        EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/ true);
+      }
     }
   } else {
     switch (M->getStorageDuration()) {
@@ -1770,16 +1778,16 @@ static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
 /// for instance if a block or lambda or a member of a local class uses a
 /// const int variable or constexpr variable from an enclosing function.
 CodeGenFunction::ConstantEmission
-CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
-  ValueDecl *value = refExpr->getDecl();
+CodeGenFunction::tryEmitAsConstant(const DeclRefExpr *RefExpr) {
+  const ValueDecl *Value = RefExpr->getDecl();
 
   // The value needs to be an enum constant or a constant variable.
   ConstantEmissionKind CEK;
-  if (isa<ParmVarDecl>(value)) {
+  if (isa<ParmVarDecl>(Value)) {
     CEK = CEK_None;
-  } else if (auto *var = dyn_cast<VarDecl>(value)) {
+  } else if (const auto *var = dyn_cast<VarDecl>(Value)) {
     CEK = checkVarTypeForConstantEmission(var->getType());
-  } else if (isa<EnumConstantDecl>(value)) {
+  } else if (isa<EnumConstantDecl>(Value)) {
     CEK = CEK_AsValueOnly;
   } else {
     CEK = CEK_None;
@@ -1792,15 +1800,15 @@ CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
 
   // It's best to evaluate all the way as an r-value if that's permitted.
   if (CEK != CEK_AsReferenceOnly &&
-      refExpr->EvaluateAsRValue(result, getContext())) {
+      RefExpr->EvaluateAsRValue(result, getContext())) {
     resultIsReference = false;
-    resultType = refExpr->getType();
+    resultType = RefExpr->getType().getUnqualifiedType();
 
   // Otherwise, try to evaluate as an l-value.
   } else if (CEK != CEK_AsValueOnly &&
-             refExpr->EvaluateAsLValue(result, getContext())) {
+             RefExpr->EvaluateAsLValue(result, getContext())) {
     resultIsReference = true;
-    resultType = value->getType();
+    resultType = Value->getType();
 
   // Failure.
   } else {
@@ -1819,7 +1827,7 @@ CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
   // accessible on device. The DRE of the captured reference variable has to be
   // loaded from captures.
   if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
-      refExpr->refersToEnclosingVariableOrCapture()) {
+      RefExpr->refersToEnclosingVariableOrCapture()) {
     auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
     if (isLambdaMethod(MD) && MD->getOverloadedOperator() == OO_Call) {
       const APValue::LValueBase &base = result.Val.getLValueBase();
@@ -1834,17 +1842,17 @@ CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
   }
 
   // Emit as a constant.
-  auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(),
-                                               result.Val, resultType);
+  llvm::Constant *C = ConstantEmitter(*this).emitAbstract(
+      RefExpr->getLocation(), result.Val, resultType);
 
   // Make sure we emit a debug reference to the global variable.
   // This should probably fire even for
-  if (isa<VarDecl>(value)) {
-    if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
-      EmitDeclRefExprDbgValue(refExpr, result.Val);
+  if (isa<VarDecl>(Value)) {
+    if (!getContext().DeclMustBeEmitted(cast<VarDecl>(Value)))
+      EmitDeclRefExprDbgValue(RefExpr, result.Val);
   } else {
-    assert(isa<EnumConstantDecl>(value));
-    EmitDeclRefExprDbgValue(refExpr, result.Val);
+    assert(isa<EnumConstantDecl>(Value));
+    EmitDeclRefExprDbgValue(RefExpr, result.Val);
   }
 
   // If we emitted a reference constant, we need to dereference that.
@@ -2235,6 +2243,15 @@ RValue CodeGenFunction::EmitLoadOfAnyValue(LValue LV, AggValueSlot Slot,
 /// method emits the address of the lvalue, then loads the result as an rvalue,
 /// returning the rvalue.
 RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
+  // Load from __ptrauth.
+  if (PointerAuthQualifier PtrAuth = LV.getQuals().getPointerAuth()) {
+    LV.getQuals().removePointerAuth();
+    llvm::Value *Value = EmitLoadOfLValue(LV, Loc).getScalarVal();
+    return RValue::get(EmitPointerAuthUnqualify(PtrAuth, Value, LV.getType(),
+                                                LV.getAddress(),
+                                                /*known nonnull*/ false));
+  }
+
   if (LV.isObjCWeak()) {
     // load of a __weak object.
     Address AddrWeakObj = LV.getAddress();
@@ -2490,6 +2507,13 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
     return EmitStoreThroughBitfieldLValue(Src, Dst);
   }
 
+  // Handle __ptrauth qualification by re-signing the value.
+  if (PointerAuthQualifier PointerAuth = Dst.getQuals().getPointerAuth()) {
+    Src = RValue::get(EmitPointerAuthQualify(PointerAuth, Src.getScalarVal(),
+                                             Dst.getType(), Dst.getAddress(),
+                                             /*known nonnull*/ false));
+  }
+
   // There's special magic for assigning into an ARC-qualified l-value.
   if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
     switch (Lifetime) {
@@ -5792,6 +5816,28 @@ CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
       return EmitCallee(ICE->getSubExpr());
     }
 
+    // Try to remember the original __ptrauth qualifier for loads of
+    // function pointers.
+    if (ICE->getCastKind() == CK_LValueToRValue) {
+      const Expr *SubExpr = ICE->getSubExpr();
+      if (const auto *PtrType = SubExpr->getType()->getAs<PointerType>()) {
+        std::pair<llvm::Value *, CGPointerAuthInfo> Result =
+            EmitOrigPointerRValue(E);
+
+        QualType FunctionType = PtrType->getPointeeType();
+        assert(FunctionType->isFunctionType());
+
+        GlobalDecl GD;
+        if (const auto *VD =
+                dyn_cast_or_null<VarDecl>(E->getReferencedDeclOfCallee())) {
+          GD = GlobalDecl(VD);
+        }
+        CGCalleeInfo CalleeInfo(FunctionType->getAs<FunctionProtoType>(), GD);
+        CGCallee Callee(CalleeInfo, Result.first, Result.second);
+        return Callee;
+      }
+    }
+
   // Resolve direct calls.
   } else if (auto DRE = dyn_cast<DeclRefExpr>(E)) {
     if (auto FD = dyn_cast<FunctionDecl>(DRE->getDecl())) {
@@ -5854,6 +5900,18 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
 
   switch (getEvaluationKind(E->getType())) {
   case TEK_Scalar: {
+    if (PointerAuthQualifier PtrAuth =
+            E->getLHS()->getType().getPointerAuth()) {
+      LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);
+      LValue CopiedLV = LV;
+      CopiedLV.getQuals().removePointerAuth();
+      llvm::Value *RV =
+          EmitPointerAuthQualify(PtrAuth, E->getRHS(), CopiedLV.getAddress());
+      EmitNullabilityCheck(CopiedLV, RV, E->getExprLoc());
+      EmitStoreThroughLValue(RValue::get(RV), CopiedLV);
+      return LV;
+    }
+
     switch (E->getLHS()->getType().getObjCLifetime()) {
     case Qualifiers::OCL_Strong:
       return EmitARCStoreStrong(E, /*ignored*/ false).first;
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index b016c6e36d1a8..b21ebeee4bed1 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -2049,10 +2049,13 @@ namespace {
 struct ConstantLValue {
   llvm::Constant *Value;
   bool HasOffsetApplied;
+  bool HasDestPointerAuth;
 
   /*implicit*/ ConstantLValue(llvm::Constant *value,
-                              bool hasOffsetApplied = false)
-    : Value(value), HasOffsetApplied(hasOffsetApplied) {}
+                              bool hasOffsetApplied = false,
+                              bool hasDestPointerAuth = false)
+      : Value(value), HasOffsetApplied(hasOffsetApplied),
+        HasDestPointerAuth(hasDestPointerAuth) {}
 
   /*implicit*/ ConstantLValue(ConstantAddress address)
     : ConstantLValue(address.getPointer()) {}
@@ -2157,6 +2160,14 @@ llvm::Constant *ConstantLValueEmitter::tryEmit() {
     value = applyOffset(value);
   }
 
+  // Apply pointer-auth signing from the destination type.
+  if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth();
+      PointerAuth && !result.HasDestPointerAuth) {
+    value = Emitter.tryEmitConstantSignedPointer(value, PointerAuth);
+    if (!value)
+      return nullptr;
+  }
+
   // Convert to the appropriate type; this could be an lvalue for
   // an integer.  FIXME: performAddrSpaceCast
   if (isa<llvm::PointerType>(destTy))
@@ -2200,6 +2211,12 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
       return CGM.GetWeakRefReference(D).getPointer();
 
     auto PtrAuthSign = [&](llvm::Constant *C) {
+      if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth()) {
+        C = applyOffset(C);
+        C = Emitter.tryEmitConstantSignedPointer(C, PointerAuth);
+        return ConstantLValue(C, /*applied offset*/ true, /*signed*/ true);
+      }
+
       CGPointerAuthInfo AuthInfo;
 
       if (EnablePtrAuthFunctionTypeDiscrimination)
@@ -2213,7 +2230,7 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
         C = CGM.getConstantSignedPointer(
             C, AuthInfo.getKey(), nullptr,
             cast_or_null<llvm::ConstantInt>(AuthInfo.getDiscriminator()));
-        return ConstantLValue(C, /*applied offset*/ true);
+        return ConstantLValue(C, /*applied offset*/ true, /*signed*/ true);
       }
 
       return ConstantLValue(C);
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index e9a7ba509350c..8dbbcdaef25d8 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2261,6 +2261,53 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
   return V;
 }
 
+static bool isDeclRefKnownNonNull(CodeGenFunction &CGF, const ValueDecl *D) {
+  return !D->isWeak();
+}
+
+static bool isLValueKnownNonNull(CodeGenFunction &CGF, const Expr *E) {
+  E = E->IgnoreParens();
+
+  if (const auto *UO = dyn_cast<UnaryOperator>(E))
+    if (UO->getOpcode() == UO_Deref)
+      return CGF.isPointerKnownNonNull(UO->getSubExpr());
+
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+    return isDeclRefKnownNonNull(CGF, DRE->getDecl());
+
+  if (const auto *ME = dyn_cast<MemberExpr>(E)) {
+    if (isa<FieldDecl>(ME->getMemberDecl()))
+      return true;
+    return isDeclRefKnownNonNull(CGF, ME->getMemberDecl());
+  }
+
+  // Array subscripts?  Anything else?
+
+  return false;
+}
+
+bool CodeGenFunction::isPointerKnownNonNull(const Expr *E) {
+  assert(E->getType()->isSignableType());
+
+  E = E->IgnoreParens();
+
+  if (isa<CXXThisExpr>(E))
+    return true;
+
+  if (const auto *UO = dyn_cast<UnaryOperator>(E))
+    if (UO->getOpcode() == UO_AddrOf)
+      return isLValueKnownNonNull(*this, UO->getSubExpr());
+
+  if (const auto *CE = dyn_cast<CastExpr>(E))
+    if (CE->getCastKind() == CK_FunctionToPointerDecay ||
+        CE->getCastKind() == CK_ArrayToPointerDecay)
+      return isLValueKnownNonNull(*this, CE->getSubExpr());
+
+  // Maybe honor __nonnull?
+
+  return false;
+}
+
 bool CodeGenFunction::ShouldNullCheckClassCastValue(const CastExpr *CE) {
   const Expr *E = CE->getSubExpr();
 
@@ -4985,6 +5032,21 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
   Value *RHS;
   LValue LHS;
 
+  if (PointerAuthQualifier PtrAuth = E->getLHS()->getType().getPointerAuth()) {
+    LValue LV = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
+    LV.getQuals().removePointerAuth();
+    llvm::Value *RV =
+        CGF.EmitPointerAuthQualify(PtrAuth, E->getRHS(), LV.getAddress());
+    CGF.EmitNullabilityCheck(LV, RV, E->getExprLoc());
+    CGF.EmitStoreThroughLValue(RValue::get(RV), LV);
+
+    if (Ignore)
+      return nullptr;
+    RV = CGF.EmitPointerAuthUnqualify(PtrAuth, RV, LV.getType(),
+                                      LV.getAddress(), /*nonnull*/ false);
+    return RV;
+  }
+
   switch (E->getLHS()->getType().getObjCLifetime()) {
   case Qualifiers::OCL_Strong:
     std::tie(LHS, RHS) = CGF.EmitARCStoreStrong(E, Ignore);
diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp
index 4b032306ead72..0a183a8524c17 100644
--- a/clang/lib/CodeGen/CGPointerAuth.cpp
+++ b/clang/lib/CodeGen/CGPointerAuth.cpp
@@ -125,6 +125,33 @@ CGPointerAuthInfo CodeGenFunction::EmitPointerAuthInfo(
                            Schema.authenticatesNullValues(), Discriminator);
 }
 
+CGPointerAuthInfo
+CodeGenFunction::EmitPointerAuthInfo(PointerAuthQualifier Qual,
+                                     Address StorageAddress) {
+  assert(Qual && "don't call this if you don't know that the Qual is present");
+  if (Qual.hasKeyNone())
+    return CGPointerAuthInfo();
+
+  llvm::Value *Discriminator = nullptr;
+  if (unsigned Extra = Qual.getExtraDiscriminator())
+    Discriminator = llvm::ConstantInt::get(IntPtrTy, Extra);
+
+  if (Qual.isAddressDiscriminated()) {
+    assert(StorageAddress.isValid() &&
+           "address discrimination without address");
+    llvm::Value *StoragePtr = StorageAddress.emitRawPointer(*this);
+    if (Discriminator)
+      Discriminator =
+          EmitPointerAuthBlendDiscriminator(StoragePtr, Discriminator);
+    else
+      Discriminator = Builder.CreatePtrToInt(StoragePtr, IntPtrTy);
+  }
+
+  return CGPointerAuthInfo(Qual.getKey(), Qual.getAuthenticationMode(),
+                           Qual.isIsaPointer(), Qual.authenticatesNullValues(),
+                           Discriminator);
+}
+
 /// Return the natural pointer authentication for values of the given
 /// pointee type.
 static CGPointerAuthInfo
@@ -166,6 +193,91 @@ CGPointerAuthInfo CodeGenModule::getPointerAuthInfoForType(QualType T) {
   return ::getPointerAuthInfoForType(*this, T);
 }
 
+static std::pair<llvm::Value *, CGPointerAuthInfo>
+emitLoadOfOrigPointerRValue(CodeGenFunction &CGF, const LValue &LV,
+                            SourceLocation Loc) {
+  llvm::Value *Value = CGF.EmitLoadOfScalar(LV, Loc);
+  CGPointerAuthInfo AuthInfo;
+  if (PointerAuthQualifier PtrAuth = LV.getQuals().getPointerAuth())
+    AuthInfo = CGF.EmitPointerAuthInfo(PtrAuth, LV.getAddress());
+  else
+    AuthInfo = getPointerAuthInfoForType(CGF.CGM, LV.getType());
+  return {Value, AuthInfo};
+}
+
+/// Retrieve a pointer rvalue and its ptrauth info. When possible, avoid
+/// needlessly resigning the pointer.
+std::pair<llvm::Value *, CGPointerAuthInfo>
+CodeGenFunction::EmitOrigPointerRValue(const Expr *E) {
+  assert(E->getType()->isSignableType());
+
+  E = E->IgnoreParens();
+  if (const auto *Load = dyn_cast<ImplicitCastExpr>(E)) {
+    if (Load->getCastKind() == CK_LValueToRValue) {
+      E = Load->getSubExpr()->IgnoreParens();
+
+      // We're semantically required to not emit loads of certain DREs naively.
+      if (const auto *RefExpr = dyn_cast<DeclRefExpr>(E)) {
+        if (ConstantEmission Result = tryEmitAsConstant(RefExpr)) {
+          // Fold away a use of an intermediate variable.
+          if (!Result.isReference())
+            return {Result.getValue(),
+                    getPointerAuthInfoForType(CGM, RefExpr->getType())};
+
+          // Fold away a use of an intermediate reference.
+          LValue LV = Result.getReferenceLValue(*this, RefExpr);
+          return emitLoadOfOrigPointerRValue(*this, LV, RefExpr->getLocation());
+        }
+      }
+
+      // Otherwise, load and use the pointer
+      LValue LV = EmitCheckedLValue(E, CodeGenFunction::TCK_Load);
+      return emitLoadOfOrigPointerRValue(*this, LV, E->getExprLoc());
+    }
+  }
+
+  // Fallback: just use the normal rules for the type.
+  llvm::Value *Value = EmitScalarExpr(E);
+  return {Value, getPointerAuthInfoForType(CGM, E->getType())};
+}
+
+llvm::Value *
+CodeGenFunction::EmitPointerAuthQualify(PointerAuthQualifier DestQualifier,
+                                        const Expr *E,
+                                        Address DestStorageAddress) {
+  assert(DestQualifier);
+  auto [Value, CurAuthInfo] = EmitOrigPointerRValue(E);
+
+  CGPointerAuthInfo DestAuthInfo =
+      EmitPointerAuthInfo(DestQualifier, DestStorageAddress);
+  return emitPointerAuthResign(Value, E->getType(), CurAuthInfo, DestAuthInfo,
+                               isPointerKnownNonNull(E));
+}
+
+llvm::Value *CodeGenFunction::EmitPointerAuthQualify(
+    PointerAuthQualifier DestQualifier, llvm::Value *Value,
+    QualType PointerType, Address DestStorageAddress, bool IsKnownNonNull) {
+  assert(DestQualifier);
+
+  CGPointerAuthInfo CurAuthInfo = getPointerAuthInfoForType(CGM, PointerType);
+  CGPointerAuthInfo DestAuthInfo =
+      EmitPointerAuthInfo(DestQualifier, DestStorageAddress);
+  return emitPointerAuthResign(Value, PointerType, CurAuthInfo, DestAuthInfo,
+                               IsKnownNonNull);
+}
+
+llvm::Value *CodeGenFunction::EmitPointerAuthUnqualify(
+    PointerAuthQualifier CurQualifier, llvm::Value *Value, QualType PointerType,
+    Address CurStorageAddress, bool IsKnownNonNull) {
+  assert(CurQualifier);
+
+  CGPointerAuthInfo CurAuthInfo =
+      EmitPointerAuthInfo(CurQualifier, CurStorageAddress);
+  CGPointerAuthInfo DestAuthInfo = getPointerAuthInfoForType(CGM, PointerType);
+  return emitPointerAuthResign(Value, PointerType, CurAuthInfo, DestAuthInfo,
+                               IsKnownNonNull);
+}
+
 static bool isZeroConstant(const llvm::Value *Value) {
   if (const auto *CI = dyn_cast<llvm::ConstantInt>(Value))
     return CI->isZero();
@@ -288,6 +400,23 @@ llvm::Value *CodeGenFunction::emitPointerAuthResign(
   return Value;
 }
 
+void CodeGenFunction::EmitPointerAuthCopy(PointerAuthQualifier Qual, QualType T,
+                                          Address DestAddress,
+                                          Address SrcAddress) {
+  assert(Qual);
+  llvm::Value *Value = Builder.CreateLoad(SrcAddress);
+
+  // If we're using address-discrimination, we have to re-sign the value.
+  if (Qual.isAddressDiscriminated()) {
+    CGPointerAuthInfo SrcPtrAuth = EmitPointerAuthInfo(Qual, SrcAddress);
+    CGPointerAuthInfo DestPtrAuth = EmitPointerAuthInfo(Qual, DestAddress);
+    Value = emitPointerAuthResign(Value, T, SrcPtrAuth, DestPtrAuth,
+                                  /*IsKnownNonNull=*/false);
+  }
+
+  Builder.CreateStore(Value, DestAddress);
+}
+
 llvm::Constant *
 CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key,
                                         llvm::Constant *StorageAddress,
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index aa07e5d6c8099..4c5e8a8a44926 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4432,10 +4432,10 @@ class CodeGenFunction : public CodeGenTypeCache {
     }
 
     bool isReference() const { return ValueAndIsReference.getInt(); }
-    LValue getReferenceLValue(CodeGenFunction &CGF, Expr *refExpr) const {
+    LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
       assert(isReference());
       return CGF.MakeNaturalAlignAddrLValue(ValueAndIsReference.getPointer(),
-                                            refExpr->getType());
+                                            RefExpr->getType());
     }
 
     llvm::Constant *getValue() const {
@@ -4444,7 +4444,7 @@ class CodeGenFunction : public CodeGenTypeCache {
     }
   };
 
-  ConstantEmission tryEmitAsConstant(DeclRefExpr *refExpr);
+  ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
   ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
   llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
 
@@ -4588,6 +4588,26 @@ class CodeGenFunction : public CodeGenTypeCache {
       const CGPointerAuthInfo &Info,
       SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
 
+  CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
+                                        Address StorageAddress);
+  llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
+                                      llvm::Value *Pointer, QualType ValueType,
+                                      Address StorageAddress,
+                                      bool IsKnownNonNull);
+  llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
+                                      const Expr *PointerExpr,
+                                      Address StorageAddress);
+  llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
+                                        llvm::Value *Pointer,
+                                        QualType PointerType,
+                                        Address StorageAddress,
+                                        bool IsKnownNonNull);
+  void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
+                           Address DestField, Address SrcField);
+
+  std::pair<llvm::Value *, CGPointerAuthInfo>
+  EmitOrigPointerRValue(const Expr *E);
+
   llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
                                         QualType SourceType, QualType DestType);
   Address authPointerToPointerCast(Address Ptr, QualType SourceType,
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index d77400e0f8272..8fa74ecff19aa 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -3400,6 +3400,45 @@ void Parser::DistributeCLateParsedAttrs(Decl *Dcl,
   }
 }
 
+/// type-qualifier:
+///    ('__ptrauth') '(' constant-expression
+///                   (',' constant-expression)[opt]
+///                   (',' constant-expression)[opt] ')'
+void Parser::ParsePtrauthQualifier(ParsedAttributes &Attrs) {
+  assert(Tok.is(tok::kw___ptrauth));
+
+  IdentifierInfo *KwName = Tok.getIdentifierInfo();
+  SourceLocation KwLoc = ConsumeToken();
+
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  if (T.expectAndConsume())
+    return;
+
+  ArgsVector ArgExprs;
+  do {
+    ExprResult ER = ParseAssignmentExpression();
+    if (ER.isInvalid()) {
+      T.skipToEnd();
+      return;
+    }
+    ArgExprs.push_back(ER.get());
+  } while (TryConsumeToken(tok::comma));
+
+  T.consumeClose();
+  SourceLocation EndLoc = T.getCloseLocation();
+
+  if (ArgExprs.empty() || ArgExprs.size() > 3) {
+    Diag(KwLoc, diag::err_ptrauth_qualifier_bad_arg_count);
+    return;
+  }
+
+  Attrs.addNew(KwName, SourceRange(KwLoc, EndLoc),
+               /*scope*/ nullptr, SourceLocation(), ArgExprs.data(),
+               ArgExprs.size(),
+               ParsedAttr::Form::Keyword(/*IsAlignAs=*/false,
+                                         /*IsRegularKeywordAttribute=*/false));
+}
+
 /// Bounds attributes (e.g., counted_by):
 ///   AttrName '(' expression ')'
 void Parser::ParseBoundsAttribute(IdentifierInfo &AttrName,
@@ -4267,6 +4306,11 @@ void Parser::ParseDeclarationSpecifiers(
                                  getLangOpts());
       break;
 
+    // __ptrauth qualifier.
+    case tok::kw___ptrauth:
+      ParsePtrauthQualifier(DS.getAttributes());
+      continue;
+
     case tok::kw___sptr:
     case tok::kw___uptr:
     case tok::kw___ptr64:
@@ -5980,6 +6024,7 @@ bool Parser::isTypeSpecifierQualifier() {
   case tok::kw___ptr32:
   case tok::kw___pascal:
   case tok::kw___unaligned:
+  case tok::kw___ptrauth:
 
   case tok::kw__Nonnull:
   case tok::kw__Nullable:
@@ -6269,6 +6314,7 @@ bool Parser::isDeclarationSpecifier(
   case tok::kw___forceinline:
   case tok::kw___pascal:
   case tok::kw___unaligned:
+  case tok::kw___ptrauth:
 
   case tok::kw__Nonnull:
   case tok::kw__Nullable:
@@ -6533,6 +6579,12 @@ void Parser::ParseTypeQualifierListOpt(
       ParseHLSLQualifiers(DS.getAttributes());
       continue;
 
+    // __ptrauth qualifier.
+    case tok::kw___ptrauth:
+      ParsePtrauthQualifier(DS.getAttributes());
+      EndLoc = PrevTokLocation;
+      continue;
+
     case tok::kw___unaligned:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_unaligned, Loc, PrevSpec, DiagID,
                                  getLangOpts());
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 1591075ff05d8..14e16bc39eb3a 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -72,8 +72,11 @@ namespace {
       //   Preceding an expression by a parenthesized type name converts the
       //   value of the expression to the unqualified, non-atomic version of
       //   the named type.
+      // Don't drop __ptrauth qualifiers. We want to treat casting to a
+      // __ptrauth-qualified type as an error instead of implicitly ignoring
+      // the qualifier.
       if (!S.Context.getLangOpts().ObjC && !DestType->isRecordType() &&
-          !DestType->isArrayType()) {
+          !DestType->isArrayType() && !DestType.getPointerAuth()) {
         DestType = DestType.getAtomicUnqualifiedType();
       }
 
@@ -168,6 +171,14 @@ namespace {
       SrcExpr = src;
     }
 
+    void checkQualifiedDestType() {
+      // Destination type may not be qualified with __ptrauth.
+      if (DestType.getPointerAuth()) {
+        Self.Diag(DestRange.getBegin(), diag::err_ptrauth_qualifier_cast)
+            << DestType << DestRange;
+      }
+    }
+
     /// Check for and handle non-overload placeholder expressions.
     void checkNonOverloadPlaceholders() {
       if (!isPlaceholder() || isPlaceholder(BuiltinType::Overload))
@@ -309,6 +320,8 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
   Op.OpRange = SourceRange(OpLoc, Parens.getEnd());
   Op.DestRange = AngleBrackets;
 
+  Op.checkQualifiedDestType();
+
   switch (Kind) {
   default: llvm_unreachable("Unknown C++ cast!");
 
@@ -3412,6 +3425,8 @@ ExprResult Sema::BuildCStyleCastExpr(SourceLocation LPLoc,
   // -Wcast-qual
   DiagnoseCastQual(Op.Self, Op.SrcExpr, Op.DestType);
 
+  Op.checkQualifiedDestType();
+
   return Op.complete(CStyleCastExpr::Create(
       Context, Op.ResultType, Op.ValueKind, Op.Kind, Op.SrcExpr.get(),
       &Op.BasePath, CurFPFeatureOverrides(), CastTypeInfo, LPLoc, RPLoc));
@@ -3431,6 +3446,8 @@ ExprResult Sema::BuildCXXFunctionalCastExpr(TypeSourceInfo *CastTypeInfo,
   if (Op.SrcExpr.isInvalid())
     return ExprError();
 
+  Op.checkQualifiedDestType();
+
   auto *SubExpr = Op.SrcExpr.get();
   if (auto *BindExpr = dyn_cast<CXXBindTemporaryExpr>(SubExpr))
     SubExpr = BindExpr->getSubExpr();
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index d0143d29a4bcc..42da9ba97e0d3 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1550,6 +1550,48 @@ bool Sema::checkConstantPointerAuthKey(Expr *Arg, unsigned &Result) {
   return false;
 }
 
+bool Sema::checkPointerAuthDiscriminatorArg(Expr *Arg,
+                                            PointerAuthDiscArgKind Kind,
+                                            unsigned &IntVal) {
+  if (!Arg) {
+    IntVal = 0;
+    return true;
+  }
+
+  std::optional<llvm::APSInt> Result = Arg->getIntegerConstantExpr(Context);
+  if (!Result) {
+    Diag(Arg->getExprLoc(), diag::err_ptrauth_arg_not_ice);
+    return false;
+  }
+
+  unsigned Max;
+  bool IsAddrDiscArg = false;
+
+  switch (Kind) {
+  case PADAK_AddrDiscPtrAuth:
+    Max = 1;
+    IsAddrDiscArg = true;
+    break;
+  case PADAK_ExtraDiscPtrAuth:
+    Max = PointerAuthQualifier::MaxDiscriminator;
+    break;
+  };
+
+  if (*Result < 0 || *Result > Max) {
+    if (IsAddrDiscArg)
+      Diag(Arg->getExprLoc(), diag::err_ptrauth_address_discrimination_invalid)
+          << Result->getExtValue();
+    else
+      Diag(Arg->getExprLoc(), diag::err_ptrauth_extra_discriminator_invalid)
+          << Result->getExtValue() << Max;
+
+    return false;
+  };
+
+  IntVal = Result->getZExtValue();
+  return true;
+}
+
 static std::pair<const ValueDecl *, CharUnits>
 findConstantBaseAndOffset(Sema &S, Expr *E) {
   // Must evaluate as a pointer.
@@ -3957,6 +3999,14 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange,
     ValType = AtomTy;
   }
 
+  PointerAuthQualifier PointerAuth = AtomTy.getPointerAuth();
+  if (PointerAuth && PointerAuth.isAddressDiscriminated()) {
+    Diag(ExprRange.getBegin(),
+         diag::err_atomic_op_needs_non_address_discriminated_pointer)
+        << 0 << Ptr->getType() << Ptr->getSourceRange();
+    return ExprError();
+  }
+
   // For an arithmetic operation, the implied arithmetic must be well-formed.
   if (Form == Arithmetic) {
     // GCC does not enforce these rules for GNU atomics, but we do to help catch
@@ -4329,6 +4379,13 @@ ExprResult Sema::BuiltinAtomicOverloaded(ExprResult TheCallResult) {
         << FirstArg->getType() << 0 << FirstArg->getSourceRange();
     return ExprError();
   }
+  PointerAuthQualifier PointerAuth = ValType.getPointerAuth();
+  if (PointerAuth && PointerAuth.isAddressDiscriminated()) {
+    Diag(FirstArg->getBeginLoc(),
+         diag::err_atomic_op_needs_non_address_discriminated_pointer)
+        << 1 << ValType << FirstArg->getSourceRange();
+    return ExprError();
+  }
 
   if (ValType.isConstQualified()) {
     Diag(DRE->getBeginLoc(), diag::err_atomic_builtin_cannot_be_const)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 240ce5391af81..5f811c824e11d 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -15454,6 +15454,12 @@ ParmVarDecl *Sema::CheckParameter(DeclContext *DC, SourceLocation StartLoc,
     New->setType(T);
   }
 
+  // __ptrauth is forbidden on parameters.
+  if (T.getPointerAuth()) {
+    Diag(NameLoc, diag::err_ptrauth_qualifier_invalid) << T << 1;
+    New->setInvalidDecl();
+  }
+
   // ISO/IEC TR 18037 S6.7.3: "The type of an object with automatic storage
   // duration shall not be qualified by an address-space qualifier."
   // Since all parameters have automatic store duration, they can not have
@@ -19456,9 +19462,14 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
             RecordArgPassingKind::CanNeverPassInRegs)
           Record->setArgPassingRestrictions(
               RecordArgPassingKind::CanNeverPassInRegs);
-      } else if (FT.getQualifiers().getObjCLifetime() == Qualifiers::OCL_Weak)
+      } else if (FT.getQualifiers().getObjCLifetime() == Qualifiers::OCL_Weak) {
         Record->setArgPassingRestrictions(
             RecordArgPassingKind::CanNeverPassInRegs);
+      } else if (PointerAuthQualifier Q = FT.getPointerAuth();
+                 Q && Q.isAddressDiscriminated()) {
+        Record->setArgPassingRestrictions(
+            RecordArgPassingKind::CanNeverPassInRegs);
+      }
     }
 
     if (Record && FD->getType().isVolatileQualified())
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 74f925f18560a..2247aded9384a 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -9471,6 +9471,8 @@ struct SpecialMemberDeletionInfo
 
   bool shouldDeleteForVariantObjCPtrMember(FieldDecl *FD, QualType FieldType);
 
+  bool shouldDeleteForVariantPtrAuthMember(const FieldDecl *FD);
+
   bool visitBase(CXXBaseSpecifier *Base) { return shouldDeleteForBase(Base); }
   bool visitField(FieldDecl *Field) { return shouldDeleteForField(Field); }
 
@@ -9639,6 +9641,30 @@ bool SpecialMemberDeletionInfo::shouldDeleteForVariantObjCPtrMember(
   return true;
 }
 
+bool SpecialMemberDeletionInfo::shouldDeleteForVariantPtrAuthMember(
+    const FieldDecl *FD) {
+  QualType FieldType = S.Context.getBaseElementType(FD->getType());
+  // Copy/move constructors/assignment operators are deleted if the field has an
+  // address-discriminated ptrauth qualifier.
+  PointerAuthQualifier Q = FieldType.getPointerAuth();
+
+  if (!Q || !Q.isAddressDiscriminated())
+    return false;
+
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor ||
+      CSM == CXXSpecialMemberKind::Destructor)
+    return false;
+
+  if (Diagnose) {
+    auto *ParentClass = cast<CXXRecordDecl>(FD->getParent());
+    S.Diag(FD->getLocation(), diag::note_deleted_special_member_class_subobject)
+        << llvm::to_underlying(getEffectiveCSM()) << ParentClass
+        << /*IsField*/ true << FD << 4 << /*IsDtorCallInCtor*/ false << 2;
+  }
+
+  return true;
+}
+
 /// Check whether we should delete a special member function due to the class
 /// having a particular direct or virtual base class.
 bool SpecialMemberDeletionInfo::shouldDeleteForBase(CXXBaseSpecifier *Base) {
@@ -9677,6 +9703,9 @@ bool SpecialMemberDeletionInfo::shouldDeleteForField(FieldDecl *FD) {
   if (inUnion() && shouldDeleteForVariantObjCPtrMember(FD, FieldType))
     return true;
 
+  if (inUnion() && shouldDeleteForVariantPtrAuthMember(FD))
+    return true;
+
   if (CSM == CXXSpecialMemberKind::DefaultConstructor) {
     // For a default constructor, all references must be initialized in-class
     // and, if a union, it must have a non-const member.
@@ -9740,6 +9769,9 @@ bool SpecialMemberDeletionInfo::shouldDeleteForField(FieldDecl *FD) {
         if (shouldDeleteForVariantObjCPtrMember(&*UI, UnionFieldType))
           return true;
 
+        if (shouldDeleteForVariantPtrAuthMember(&*UI))
+          return true;
+
         if (!UnionFieldType.isConstQualified())
           AllVariantFieldsAreConst = false;
 
@@ -10589,6 +10621,12 @@ void Sema::checkIllFormedTrivialABIStruct(CXXRecordDecl &RD) {
       return;
     }
 
+    // Ill-formed if the field is an address-discriminated pointer.
+    if (FT.hasAddressDiscriminatedPointerAuth()) {
+      PrintDiagAndRemoveAttr(6);
+      return;
+    }
+
     if (const auto *RT = FT->getBaseElementTypeUnsafe()->getAs<RecordType>())
       if (!RT->isDependentType() &&
           !cast<CXXRecordDecl>(RT->getDecl())->canPassInRegisters()) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index c25daaa022f49..3ac7d61546ceb 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -8107,6 +8107,13 @@ static QualType checkConditionalPointerCompatibility(Sema &S, ExprResult &LHS,
   lhQual.removeCVRQualifiers();
   rhQual.removeCVRQualifiers();
 
+  if (!lhQual.getPointerAuth().isEquivalent(rhQual.getPointerAuth())) {
+    S.Diag(Loc, diag::err_typecheck_cond_incompatible_ptrauth)
+        << LHSTy << RHSTy << LHS.get()->getSourceRange()
+        << RHS.get()->getSourceRange();
+    return QualType();
+  }
+
   // OpenCL v2.0 specification doesn't extend compatibility of type qualifiers
   // (C99 6.7.3) for address spaces. We assume that the check should behave in
   // the same manner as it's defined for CVR qualifiers, so for OpenCL two
@@ -9027,6 +9034,10 @@ checkPointerTypesForAssignment(Sema &S, QualType LHSType, QualType RHSType,
     else if (lhq.getObjCLifetime() != rhq.getObjCLifetime())
       ConvTy = Sema::IncompatiblePointerDiscardsQualifiers;
 
+    // Treat pointer-auth mismatches as fatal.
+    else if (!lhq.getPointerAuth().isEquivalent(rhq.getPointerAuth()))
+      ConvTy = Sema::IncompatiblePointerDiscardsQualifiers;
+
     // For GCC/MS compatibility, other qualifier mismatches are treated
     // as still compatible in C.
     else ConvTy = Sema::CompatiblePointerDiscardsQualifiers;
@@ -17025,6 +17036,9 @@ bool Sema::DiagnoseAssignmentResult(AssignConvertType ConvTy,
     } else if (lhq.getObjCLifetime() != rhq.getObjCLifetime()) {
       DiagKind = diag::err_typecheck_incompatible_ownership;
       break;
+    } else if (!lhq.getPointerAuth().isEquivalent(rhq.getPointerAuth())) {
+      DiagKind = diag::err_typecheck_incompatible_ptrauth;
+      break;
     }
 
     llvm_unreachable("unknown error case for discarding qualifiers!");
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index dfb5824a1c3d7..8df590fa624cf 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -7770,6 +7770,11 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
       else
         return QualType();
 
+      if (Q1.getPointerAuth().isEquivalent(Q2.getPointerAuth()))
+        Quals.setPointerAuth(Q1.getPointerAuth());
+      else
+        return QualType();
+
       Steps.back().Quals = Quals;
       if (Q1 != Quals || Q2 != Quals)
         NeedConstBefore = Steps.size() - 1;
diff --git a/clang/lib/Sema/SemaObjCProperty.cpp b/clang/lib/Sema/SemaObjCProperty.cpp
index 6db2c246de791..f37982eddace9 100644
--- a/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/clang/lib/Sema/SemaObjCProperty.cpp
@@ -180,6 +180,9 @@ Decl *SemaObjC::ActOnProperty(Scope *S, SourceLocation AtLoc,
                            0);
   TypeSourceInfo *TSI = SemaRef.GetTypeForDeclarator(FD.D);
   QualType T = TSI->getType();
+  if (T.getPointerAuth().isPresent()) {
+    Diag(AtLoc, diag::err_ptrauth_qualifier_invalid) << T << 2;
+  }
   if (!getOwnershipRule(Attributes)) {
     Attributes |= deducePropertyOwnershipFromType(SemaRef, T);
   }
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 9c2df0b21d278..55634aa75ae25 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -3709,6 +3709,10 @@ static bool isQualificationConversionStep(QualType FromType, QualType ToType,
     ToQuals.removeObjCGCAttr();
   }
 
+  // __ptrauth qualifiers must match exactly.
+  if (FromQuals.getPointerAuth() != ToQuals.getPointerAuth())
+    return false;
+
   //   -- for every j > 0, if const is in cv 1,j then const is in cv
   //      2,j, and similarly for volatile.
   if (!CStyle && !ToQuals.compatiblyIncludes(FromQuals, Ctx))
@@ -11467,6 +11471,17 @@ static void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand,
       return;
     }
 
+    if (!FromQs.getPointerAuth().isEquivalent(ToQs.getPointerAuth())) {
+      S.Diag(Fn->getLocation(), diag::note_ovl_candidate_bad_ptrauth)
+          << (unsigned)FnKindPair.first << (unsigned)FnKindPair.second << FnDesc
+          << FromTy << !!FromQs.getPointerAuth()
+          << FromQs.getPointerAuth().getAsString() << !!ToQs.getPointerAuth()
+          << ToQs.getPointerAuth().getAsString() << I + 1
+          << (FromExpr ? FromExpr->getSourceRange() : SourceRange());
+      MaybeEmitInheritedConstructorNote(S, Cand->FoundDecl);
+      return;
+    }
+
     unsigned CVR = FromQs.getCVRQualifiers() & ~ToQs.getCVRQualifiers();
     assert(CVR && "expected qualifiers mismatch");
 
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 33b1d8ca4dfa0..eba7267904fb2 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2552,6 +2552,12 @@ bool Sema::CheckFunctionReturnType(QualType T, SourceLocation Loc) {
     return true;
   }
 
+  // __ptrauth is illegal on a function return type.
+  if (T.getPointerAuth()) {
+    Diag(Loc, diag::err_ptrauth_qualifier_invalid) << T << 0;
+    return true;
+  }
+
   if (T.hasNonTrivialToPrimitiveDestructCUnion() ||
       T.hasNonTrivialToPrimitiveCopyCUnion())
     checkNonTrivialCUnion(T, Loc, NTCUC_FunctionReturn,
@@ -2657,6 +2663,10 @@ QualType Sema::BuildFunctionType(QualType T,
     } else if (ParamType->isWebAssemblyTableType()) {
       Diag(Loc, diag::err_wasm_table_as_function_parameter);
       Invalid = true;
+    } else if (ParamType.getPointerAuth()) {
+      // __ptrauth is illegal on a function return type.
+      Diag(Loc, diag::err_ptrauth_qualifier_invalid) << T << 1;
+      Invalid = true;
     }
 
     // C++2a [dcl.fct]p4:
@@ -4974,6 +4984,11 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
         }
       }
 
+      // __ptrauth is illegal on a function return type.
+      if (T.getPointerAuth()) {
+        S.Diag(DeclType.Loc, diag::err_ptrauth_qualifier_invalid) << T << 0;
+      }
+
       if (LangOpts.OpenCL) {
         // OpenCL v2.0 s6.12.5 - A block cannot be the return value of a
         // function.
@@ -8333,6 +8348,65 @@ static void HandleNeonVectorTypeAttr(QualType &CurType, const ParsedAttr &Attr,
   CurType = S.Context.getVectorType(CurType, numElts, VecKind);
 }
 
+/// Handle the __ptrauth qualifier.
+static void HandlePtrAuthQualifier(ASTContext &Ctx, QualType &T,
+                                   const ParsedAttr &Attr, Sema &S) {
+
+  assert((Attr.getNumArgs() > 0 && Attr.getNumArgs() <= 3) &&
+         "__ptrauth qualifier takes between 1 and 3 arguments");
+  Expr *KeyArg = Attr.getArgAsExpr(0);
+  Expr *IsAddressDiscriminatedArg =
+      Attr.getNumArgs() >= 2 ? Attr.getArgAsExpr(1) : nullptr;
+  Expr *ExtraDiscriminatorArg =
+      Attr.getNumArgs() >= 3 ? Attr.getArgAsExpr(2) : nullptr;
+
+  unsigned Key;
+  if (S.checkConstantPointerAuthKey(KeyArg, Key)) {
+    Attr.setInvalid();
+    return;
+  }
+  assert(Key <= PointerAuthQualifier::MaxKey && "ptrauth key is out of range");
+
+  bool IsInvalid = false;
+  unsigned IsAddressDiscriminated, ExtraDiscriminator;
+  IsInvalid |= !S.checkPointerAuthDiscriminatorArg(IsAddressDiscriminatedArg,
+                                                   Sema::PADAK_AddrDiscPtrAuth,
+                                                   IsAddressDiscriminated);
+  IsInvalid |= !S.checkPointerAuthDiscriminatorArg(
+      ExtraDiscriminatorArg, Sema::PADAK_ExtraDiscPtrAuth, ExtraDiscriminator);
+
+  if (IsInvalid) {
+    Attr.setInvalid();
+    return;
+  }
+
+  if (!T->isSignableType() && !T->isDependentType()) {
+    S.Diag(Attr.getLoc(), diag::err_ptrauth_qualifier_nonpointer) << T;
+    Attr.setInvalid();
+    return;
+  }
+
+  if (T.getPointerAuth()) {
+    S.Diag(Attr.getLoc(), diag::err_ptrauth_qualifier_redundant)
+        << T << Attr.getAttrName()->getName();
+    Attr.setInvalid();
+    return;
+  }
+
+  if (!S.getLangOpts().PointerAuthIntrinsics) {
+    S.Diag(Attr.getLoc(), diag::err_ptrauth_disabled) << Attr.getRange();
+    Attr.setInvalid();
+    return;
+  }
+
+  assert((!IsAddressDiscriminatedArg || IsAddressDiscriminated <= 1) &&
+         "address discriminator arg should be either 0 or 1");
+  PointerAuthQualifier Qual = PointerAuthQualifier::Create(
+      Key, IsAddressDiscriminated, ExtraDiscriminator,
+      PointerAuthenticationMode::SignAndAuth, false, false);
+  T = S.Context.getPointerAuthType(T, Qual);
+}
+
 /// HandleArmSveVectorBitsTypeAttr - The "arm_sve_vector_bits" attribute is
 /// used to create fixed-length versions of sizeless SVE types defined by
 /// the ACLE, such as svint32_t and svbool_t.
@@ -8788,6 +8862,11 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
       HandleOpenCLAccessAttr(type, attr, state.getSema());
       attr.setUsedAsTypeAttr();
       break;
+    case ParsedAttr::AT_PointerAuth:
+      HandlePtrAuthQualifier(state.getSema().Context, type, attr,
+                             state.getSema());
+      attr.setUsedAsTypeAttr();
+      break;
     case ParsedAttr::AT_LifetimeBound:
       if (TAL == TAL_DeclChunk)
         HandleLifetimeBoundAttr(state, type, attr);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index bb58ec49612c8..2469991bf2ce8 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -5287,6 +5287,17 @@ QualType TreeTransform<Derived>::RebuildQualifiedType(QualType T,
     return QualType();
   }
 
+  PointerAuthQualifier LocalPointerAuth = Quals.getPointerAuth();
+  if (LocalPointerAuth.isPresent()) {
+    if (T.getPointerAuth().isPresent()) {
+      SemaRef.Diag(Loc, diag::err_ptrauth_qualifier_redundant)
+          << TL.getType() << "__ptrauth";
+      return QualType();
+    } else if (!T->isSignableType() && !T->isDependentType()) {
+      SemaRef.Diag(Loc, diag::err_ptrauth_qualifier_nonpointer) << T;
+      return QualType();
+    }
+  }
   // C++ [dcl.fct]p7:
   //   [When] adding cv-qualifications on top of the function type [...] the
   //   cv-qualifiers are ignored.
diff --git a/clang/test/AST/ast-dump-ptrauth-json.cpp b/clang/test/AST/ast-dump-ptrauth-json.cpp
index 125cda0cff53a..8526598c491c1 100644
--- a/clang/test/AST/ast-dump-ptrauth-json.cpp
+++ b/clang/test/AST/ast-dump-ptrauth-json.cpp
@@ -1,5 +1,8 @@
 // RUN: %clang_cc1 -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics -std=c++11 -ast-dump=json %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics -std=c++11 -ast-dump=json %s | FileCheck %s
 
 // CHECK: "name": "__builtin_ptrauth_type_discriminator",
+// CHECK: "qualType": "int *__ptrauth(1,1,123)"
 
 int d = __builtin_ptrauth_type_discriminator(int());
+int * __ptrauth(1,1,123) p;
diff --git a/clang/test/CodeGen/ptrauth-debuginfo.c b/clang/test/CodeGen/ptrauth-debuginfo.c
new file mode 100644
index 0000000000000..b76baffadd9a1
--- /dev/null
+++ b/clang/test/CodeGen/ptrauth-debuginfo.c
@@ -0,0 +1,35 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios \
+// RUN:   -fptrauth-calls -fptrauth-intrinsics -emit-llvm -fblocks \
+// RUN:   %s -debug-info-kind=limited -o - | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu \
+// RUN:   -fptrauth-calls -fptrauth-intrinsics -emit-llvm -fblocks \
+// RUN:   %s -debug-info-kind=limited -o - | FileCheck %s
+
+// Constant initializers for data pointers.
+extern int external_int;
+
+int *__ptrauth(1, 0, 1234) g1 = &external_int;
+// CHECK: !DIDerivedType(tag: DW_TAG_LLVM_ptrauth_type,
+// CHECK-SAME:           ptrAuthKey: 1,
+// CHECK-SAME:           ptrAuthIsAddressDiscriminated: false,
+// CHECK-SAME:           ptrAuthExtraDiscriminator: 1234,
+// CHECK-SAME:           ptrAuthIsaPointer: false,
+// CHECK-SAME:           ptrAuthAuthenticatesNullValues: false)
+
+struct A {
+  int value;
+};
+struct A *createA(void);
+
+void f() {
+  __block struct A *__ptrauth(0, 1, 1236) ptr = createA();
+  ^{
+    (void)ptr->value;
+  }();
+}
+// CHECK: !DIDerivedType(tag: DW_TAG_LLVM_ptrauth_type,
+// CHECK-NOT:            ptrAuthKey
+// CHECK-SAME:           ptrAuthIsAddressDiscriminated: true,
+// CHECK-SAME:           ptrAuthExtraDiscriminator: 1236,
+// CHECK-SAME:           ptrAuthIsaPointer: false,
+// CHECK-SAME:           ptrAuthAuthenticatesNullValues: false)
diff --git a/clang/test/CodeGen/ptrauth-qualifier-const-init.c b/clang/test/CodeGen/ptrauth-qualifier-const-init.c
new file mode 100644
index 0000000000000..174f328628f19
--- /dev/null
+++ b/clang/test/CodeGen/ptrauth-qualifier-const-init.c
@@ -0,0 +1,86 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics -emit-llvm %s  -o - | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics -emit-llvm %s  -o - | FileCheck %s
+
+// Constant initializers for data pointers.
+extern int external_int;
+
+// CHECK: @g1 = global ptr ptrauth (ptr @external_int, i32 1, i64 56)
+int * __ptrauth(1,0,56) g1 = &external_int;
+
+// CHECK: @g2 = global ptr ptrauth (ptr @external_int, i32 1, i64 1272, ptr @g2)
+int * __ptrauth(1,1,1272) g2 = &external_int;
+
+// CHECK: @g3 = global ptr null
+int * __ptrauth(1,1,871) g3 = 0;
+
+// FIXME: should we make a ptrauth constant for this absolute symbol?
+// CHECK: @g4 = global ptr inttoptr (i64 1230 to ptr)
+int * __ptrauth(1,1,1902) g4 = (int*) 1230;
+
+// CHECK: @ga = global [3 x ptr] [
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 712, ptr @ga),
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 712, ptr getelementptr inbounds ([3 x ptr], ptr @ga, i32 0, i32 1)),
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 712, ptr getelementptr inbounds ([3 x ptr], ptr @ga, i32 0, i32 2))]
+int * __ptrauth(1,1,712) ga[3] = { &external_int, &external_int, &external_int };
+
+struct A {
+  int * __ptrauth(1,0,431) f0;
+  int * __ptrauth(1,0,9182) f1;
+  int * __ptrauth(1,0,783) f2;
+};
+
+// CHECK: @gs1 = global %struct.A {
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 431),
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 9182),
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 783) }
+struct A gs1 = { &external_int, &external_int, &external_int };
+
+struct B {
+  int * __ptrauth(1,1,1276) f0;
+  int * __ptrauth(1,1,23674) f1;
+  int * __ptrauth(1,1,163) f2;
+};
+
+// CHECK: @gs2 = global %struct.B {
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 1276, ptr @gs2),
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 23674, ptr getelementptr inbounds (%struct.B, ptr @gs2, i32 0, i32 1)),
+// CHECK-SAME: ptr ptrauth (ptr @external_int, i32 1, i64 163, ptr getelementptr inbounds (%struct.B, ptr @gs2, i32 0, i32 2)) }
+struct B gs2 = { &external_int, &external_int, &external_int };
+
+// Constant initializers for function pointers.
+extern void external_function(void);
+typedef void (*fpt)(void);
+
+// CHECK: @f1 = global ptr ptrauth (ptr @external_function, i32 1, i64 56)
+fpt __ptrauth(1,0,56) f1 = &external_function;
+
+// CHECK: @f2 = global ptr ptrauth (ptr @external_function, i32 1, i64 1272, ptr @f2)
+fpt __ptrauth(1,1,1272) f2 = &external_function;
+
+// CHECK: @fa = global [3 x ptr] [
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 712, ptr @fa),
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 712, ptr getelementptr inbounds ([3 x ptr], ptr @fa, i32 0, i32 1)),
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 712, ptr getelementptr inbounds ([3 x ptr], ptr @fa, i32 0, i32 2))]
+fpt __ptrauth(1,1,712) fa[3] = { &external_function, &external_function, &external_function };
+
+struct C {
+  fpt __ptrauth(1,0,431) f0;
+  fpt __ptrauth(1,0,9182) f1;
+  fpt __ptrauth(1,0,783) f2;
+};
+// CHECK: @fs1 = global %struct.C {
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 431),
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 9182),
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 783) }
+struct C fs1 = { &external_function, &external_function, &external_function };
+
+struct D {
+  fpt __ptrauth(1,1,1276) f0;
+  fpt __ptrauth(1,1,23674) f1;
+  fpt __ptrauth(1,1,163) f2;
+};
+// CHECK: @fs2 = global %struct.D {
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 1276, ptr @fs2),
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 23674, ptr getelementptr inbounds (%struct.D, ptr @fs2, i32 0, i32 1)),
+// CHECK-SAME: ptr ptrauth (ptr @external_function, i32 1, i64 163, ptr getelementptr inbounds (%struct.D, ptr @fs2, i32 0, i32 2)) }
+struct D fs2 = { &external_function, &external_function, &external_function };
diff --git a/clang/test/CodeGen/ptrauth-qualifier-function.c b/clang/test/CodeGen/ptrauth-qualifier-function.c
new file mode 100644
index 0000000000000..cd25b77a01548
--- /dev/null
+++ b/clang/test/CodeGen/ptrauth-qualifier-function.c
@@ -0,0 +1,145 @@
+// RUN: %clang_cc1 %s       -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck --check-prefixes=CHECK,TYPE %s
+// RUN: %clang_cc1 %s       -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck --check-prefixes=CHECK,TYPE %s
+// RUN: %clang_cc1 %s       -triple arm64-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck --check-prefixes=CHECK,ZERO %s
+// RUN: %clang_cc1 %s       -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck --check-prefixes=CHECK,ZERO %s
+// RUN: %clang_cc1 -xc++ %s -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios13 -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck --check-prefixes=CHECK,TYPE,CHECK-CXX %s
+// RUN: %clang_cc1 -xc++ %s -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics -disable-llvm-passes -emit-llvm -o- | FileCheck --check-prefixes=CHECK,TYPE,CHECK-CXX %s
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void (*fptr)(void);
+void (* __ptrauth(0, 0, 42) f2ptr_42_discm)(int);
+void f(int);
+void (* const __ptrauth(0, 0, 42) f_const_ptr)(int) = &f;
+
+// CHECK-LABEL: define {{.*}}void @test_assign_to_qualified
+void test_assign_to_qualified() {
+  f2ptr_42_discm = (void (*)(int))fptr;
+
+  // CHECK: [[ENTRY:.*]]:{{$}}
+  // CHECK: [[FPTR:%.*]] = load ptr, ptr @fptr
+  // CHECK-NEXT: [[CMP:%.*]] = icmp ne ptr [[FPTR]], null
+  // TYPE-NEXT: br i1 [[CMP]], label %[[RESIGN1:.*]], label %[[JOIN1:.*]]
+  // ZERO-NEXT: br i1 [[CMP]], label %[[RESIGN2:.*]], label %[[JOIN2:.*]]
+
+  // TYPE: [[RESIGN1]]:
+  // TYPE-NEXT: [[FPTR2:%.*]] = ptrtoint ptr [[FPTR]] to i64
+  // TYPE-NEXT: [[FPTR4:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[FPTR2]], i32 0, i64 18983, i32 0, i64 2712)
+  // TYPE-NEXT: [[FPTR5:%.*]] = inttoptr i64 [[FPTR4]] to ptr
+  // TYPE-NEXT: br label %[[JOIN1]]
+
+  // TYPE: [[JOIN1]]:
+  // TYPE-NEXT: [[FPTR6:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[FPTR5]], %[[RESIGN1]] ]
+  // TYPE-NEXT: [[CMP:%.*]] = icmp ne ptr [[FPTR6]], null
+  // TYPE-NEXT: br i1 [[CMP]], label %[[RESIGN2:.*]], label %[[JOIN2:.*]]
+
+  // CHECK: [[RESIGN2]]:
+  // TYPE-NEXT: [[FPTR7:%.*]] = ptrtoint ptr [[FPTR6]] to i64
+  // TYPE-NEXT: [[FPTR8:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[FPTR7]], i32 0, i64 2712, i32 0, i64 42)
+  // ZERO-NEXT: [[FPTR7:%.*]] = ptrtoint ptr [[FPTR]] to i64
+  // ZERO-NEXT: [[FPTR8:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[FPTR7]], i32 0, i64 0, i32 0, i64 42)
+  // CHECK-NEXT: [[FPTR9:%.*]] = inttoptr i64 [[FPTR8]] to ptr
+  // CHECK-NEXT: br label %[[JOIN2]]
+
+  // CHECK: [[JOIN2]]
+  // TYPE-NEXT: [[FPTR10:%.*]] = phi ptr [ null, %[[JOIN1]] ], [ [[FPTR9]], %[[RESIGN2]] ]
+  // ZERO-NEXT: [[FPTR10:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[FPTR9]], %[[RESIGN2]] ]
+  // CHECK-NEXT store void (i32)* [[FPTR10]], void (i32)** @f2ptr_42_discm
+}
+
+// CHECK-LABEL: define {{.*}}void @test_assign_from_qualified
+void test_assign_from_qualified() {
+  fptr = (void (*)(void))f2ptr_42_discm;
+
+  // CHECK: [[ENTRY:.*]]:{{$}}
+  // CHECK: [[FPTR:%.*]] = load ptr, ptr @f2ptr_42_discm
+  // CHECK-NEXT: [[CMP:%.*]] = icmp ne ptr [[FPTR]], null
+  // TYPE-NEXT: br i1 [[CMP]], label %[[RESIGN1:.*]], label %[[JOIN1:.*]]
+  // ZERO-NEXT: br i1 [[CMP]], label %[[RESIGN2:.*]], label %[[JOIN2:.*]]
+
+  // TYPE: [[RESIGN1]]:
+  // TYPE-NEXT: [[FPTR1:%.*]] = ptrtoint ptr [[FPTR]] to i64
+  // TYPE-NEXT: [[FPTR2:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[FPTR1]], i32 0, i64 42, i32 0, i64 2712)
+  // TYPE-NEXT: [[FPTR3:%.*]] = inttoptr i64 [[FPTR2]] to ptr
+  // TYPE-NEXT: br label %[[JOIN1]]
+
+  // TYPE: [[JOIN1]]:
+  // TYPE-NEXT: [[FPTR4:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[FPTR3]], %[[RESIGN1]] ]
+  // TYPE-NEXT: [[CMP:%.*]] = icmp ne ptr [[FPTR4]], null
+  // TYPE-NEXT: br i1 [[CMP]], label %[[RESIGN2:.*]], label %[[JOIN2:.*]]
+
+  // CHECK: [[RESIGN2]]:
+  // TYPE-NEXT: [[FPTR6:%.*]] = ptrtoint ptr [[FPTR4]] to i64
+  // TYPE-NEXT: [[FPTR7:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[FPTR6]], i32 0, i64 2712, i32 0, i64 18983)
+  // ZERO-NEXT: [[FPTR6:%.*]] = ptrtoint ptr [[FPTR]] to i64
+  // ZERO-NEXT: [[FPTR7:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[FPTR6]], i32 0, i64 42, i32 0, i64 0)
+  // CHECK-NEXT: [[FPTR8:%.*]] = inttoptr i64 [[FPTR7]] to ptr
+  // CHECK-NEXT: br label %[[JOIN2]]
+
+  // CHECK: [[JOIN2]]
+  // TYPE-NEXT: [[FPTR9:%.*]] = phi ptr [ null, %[[JOIN1]] ], [ [[FPTR8]], %[[RESIGN2]] ]
+  // ZERO-NEXT: [[FPTR9:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[FPTR8]], %[[RESIGN2]] ]
+  // CHECK-NEXT store void ()* [[FPTR10]], void ()** @f2ptr_42_discm
+}
+
+// CHECK-LABEL: define {{.*}}void @test_const_ptr_function_call()
+void test_const_ptr_function_call(void) {
+  f_const_ptr(1);
+
+  // TYPE: call void ptrauth (ptr @f, i32 0, i64 2712)(i32 noundef 1) [ "ptrauth"(i32 0, i64 2712) ]
+  // ZERO: call void ptrauth (ptr @f, i32 0)(i32 noundef 1) [ "ptrauth"(i32 0, i64 0) ]
+}
+
+#ifdef __cplusplus
+void (* get_fptr(void))(int);
+void (* __ptrauth(0, 0, 42) f_const_ptr2)(int) = get_fptr();
+void (* const __ptrauth(0, 1, 43) &f_ref)(int) = f_const_ptr2;
+
+// CHECK-CXX-LABEL: define {{.*}}internal void @__cxx_global_var_init()
+// CHECK-CXX: [[ENTRY:.*]]:
+// CHECK-CXX: %[[CALL:.*]] = call ptr @get_fptr()
+// CHECK-CXX: %[[V0:.*]] = icmp ne ptr %[[CALL]], null
+// CHECK-CXX: br i1 %[[V0]], label %[[RESIGN_NONNULL:.*]], label %[[RESIGN_CONT:.*]]
+
+// CHECK-CXX: [[RESIGN_NONNULL]]:
+// CHECK-CXX: %[[V1:.*]] = ptrtoint ptr %[[CALL]] to i64
+// CHECK-CXX: %[[V2:.*]] = call i64 @llvm.ptrauth.resign(i64 %[[V1]], i32 0, i64 2712, i32 0, i64 42)
+// CHECK-CXX: %[[V3:.*]] = inttoptr i64 %[[V2]] to ptr
+// CHECK-CXX: br label %[[RESIGN_CONT]]
+
+// CHECK-CXX: [[RESIGN_CONT]]:
+// CHECK-CXX: %[[V4:.*]] = phi ptr [ null, %[[ENTRY]] ], [ %[[V3]], %[[RESIGN_NONNULL]] ]
+// CHECK-CXX: store ptr %[[V4]], ptr @f_const_ptr2, align 8
+
+// CHECK-CXX-LABEL: define {{.*}}internal void @__cxx_global_var_init.1()
+// CHECK-CXX: [[ENTRY:.*]]:
+// CHECK-CXX: %[[V0:.*]] = load ptr, ptr @f_const_ptr2, align 8
+// CHECK-CXX: %[[V1:.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @_ZGR5f_ref_ to i64), i64 43)
+// CHECK-CXX: %[[V2:.*]] = icmp ne ptr %[[V0]], null
+// CHECK-CXX: br i1 %[[V2]], label %[[RESIGN_NONNULL:.*]], label %[[RESIGN_CONT:.*]]
+
+// CHECK-CXX: [[RESIGN_NONNULL]]:
+// CHECK-CXX: %[[V3:.*]] = ptrtoint ptr %[[V0]] to i64
+// CHECK-CXX: %[[V4:.*]] = call i64 @llvm.ptrauth.resign(i64 %[[V3]], i32 0, i64 42, i32 0, i64 %[[V1]])
+// CHECK-CXX: %[[V5:.*]] = inttoptr i64 %[[V4]] to ptr
+// CHECK-CXX: br label %[[RESIGN_CONT]]
+
+// CHECK-CXX: [[RESIGN_CONT]]:
+// CHECK-CXX: %[[V6:.*]] = phi ptr [ null, %[[ENTRY]] ], [ %[[V5]], %[[RESIGN_NONNULL]] ]
+// CHECK-CXX: store ptr %[[V6]], ptr @_ZGR5f_ref_, align 8
+// CHECK-CXX: store ptr @_ZGR5f_ref_, ptr @f_ref, align 8
+
+// CHECK-CXX-LABEL: define {{.*}}void @test_const_ptr_ref_function_call()
+void test_const_ptr_ref_function_call(void) {
+  f_ref(1);
+
+  // CHECK-CXX: %[[V0:.*]] = load ptr, ptr @f_ref, align 8
+  // CHECK-CXX: %[[V1:.*]] = load ptr, ptr %[[V0]], align 8
+  // CHECK-CXX: %[[V2:.*]] = ptrtoint ptr %[[V0]] to i64
+  // CHECK-CXX: %[[V3:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V2]], i64 43)
+  // CHECK-CXX: call void %[[V1]](i32 noundef 1) [ "ptrauth"(i32 0, i64 %[[V3]]) ]
+}
+}
+#endif
diff --git a/clang/test/CodeGen/ptrauth-qualifier-loadstore.c b/clang/test/CodeGen/ptrauth-qualifier-loadstore.c
new file mode 100644
index 0000000000000..db259ed950fec
--- /dev/null
+++ b/clang/test/CodeGen/ptrauth-qualifier-loadstore.c
@@ -0,0 +1,745 @@
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics -emit-llvm %s  -o - | FileCheck %s
+// RUN: %clang_cc1 -fptrauth-function-pointer-type-discrimination -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics -emit-llvm %s  -o - | FileCheck %s
+
+#define IQ __ptrauth(1,0,50)
+#define AQ __ptrauth(1,1,50)
+#define DIFF_IQ __ptrauth(1,0,100)
+#define DIFF_AQ __ptrauth(1,1,100)
+#define ZERO_IQ __ptrauth(1,0,0)
+#define ZERO_AQ __ptrauth(1,1,0)
+
+extern int external_int;
+extern int * global_upi;
+extern int * IQ global_iqpi;
+extern int * AQ global_aqpi;
+extern void use_upi(int *ptr);
+
+typedef void func_t(void);
+extern void external_func(void);
+extern func_t *global_upf;
+extern func_t * IQ global_iqpf;
+extern func_t * AQ global_aqpf;
+extern void use_upf(func_t *ptr);
+
+// Data with address-independent qualifiers.
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_i_constant()
+void test_store_data_i_constant() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[SIGN:%.*]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr @external_int to i64), i32 1, i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = inttoptr i64 [[SIGN]] to ptr
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * IQ iqpi = &external_int;
+// CHECK-NEXT:    [[T0:%.*]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr @external_int to i64), i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T0]] to ptr
+// CHECK-NEXT:    store ptr [[SIGNED]], ptr [[V]],
+// CHECK-NEXT:    ret void
+  iqpi = &external_int;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_iu()
+void test_store_data_iu() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_upi,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.sign(i64 [[T0]], i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * IQ iqpi = global_upi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_upi,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.sign(i64 [[T0]], i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  iqpi = global_upi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_ia()
+void test_store_data_ia() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * IQ iqpi = global_aqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  iqpi = global_aqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[RESULT:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[RESULT]], ptr [[V]],
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[RESULT]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[RESULT]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T0]], i32 1, i64 50)
+// CHECK-NEXT:    [[AUTHED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[RESULT:%.*]] = phi ptr [ null, {{.*}} ], [ [[AUTHED]], {{.*}} ]
+// CHECK-NEXT:    call void @use_upi(ptr noundef [[RESULT]])
+  use_upi(iqpi = global_aqpi);
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_ii_same()
+void test_store_data_ii_same() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    store ptr [[LOAD]], ptr [[V]],
+  int * IQ iqpi = global_iqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    store ptr [[LOAD]], ptr [[V]],
+  iqpi = global_iqpi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_ii_different()
+void test_store_data_ii_different() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 100)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * DIFF_IQ iqpi = global_iqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 100)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  iqpi = global_iqpi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_ii_zero()
+void test_store_data_ii_zero() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 0)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * ZERO_IQ iqpi = global_iqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr [[V]]
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 0, i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr @global_iqpi,
+  global_iqpi = iqpi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_load_data_i()
+void test_load_data_i() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T0]], i32 1, i64 50)
+// CHECK-NEXT:    [[AUTHED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[AUTHED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int *upi = global_iqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T0]], i32 1, i64 50)
+// CHECK-NEXT:    [[AUTHED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[AUTHED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  upi = global_iqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T0]], i32 1, i64 50)
+// CHECK-NEXT:    [[AUTHED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[AUTHED]], {{.*}} ]
+// CHECK-NEXT:    call void @use_upi(ptr noundef [[T0]])
+  use_upi(global_iqpi);
+}
+
+// Data with address-discriminated qualifiers.
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_a_constant()
+void test_store_data_a_constant() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[SIGN:%.*]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr @external_int to i64), i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[T0:%.*]] = inttoptr i64 [[SIGN]] to ptr
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * AQ aqpi = &external_int;
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[SIGN:%.*]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr @external_int to i64), i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[T0:%.*]] = inttoptr i64 [[SIGN]] to ptr
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpi = &external_int;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_au()
+void test_store_data_au() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_upi,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.sign(i64 [[T0]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * AQ aqpi = global_upi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_upi,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.sign(i64 [[T0]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpi = global_upi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_ai()
+void test_store_data_ai() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * AQ aqpi = global_iqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpi,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpi = global_iqpi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_aa_same()
+void test_store_data_aa_same() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * AQ aqpi = global_aqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpi = global_aqpi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_aa_different()
+void test_store_data_aa_different() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 100)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * DIFF_AQ aqpi = global_aqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 100)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpi = global_aqpi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_data_aa_zero()
+void test_store_data_aa_zero() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[NEWDISC:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int * ZERO_AQ aqpi = global_aqpi;
+// CHECK:         [[LOAD:%.*]] = load ptr, ptr [[V]],
+// CHECK-NEXT:    [[OLDDISC:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr @global_aqpi,
+  global_aqpi = aqpi;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_load_data_a()
+void test_load_data_a() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T0]], i32 1, i64 [[OLDDISC]])
+// CHECK-NEXT:    [[AUTHED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[AUTHED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  int *upi = global_aqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T0]], i32 1, i64 [[OLDDISC]])
+// CHECK-NEXT:    [[AUTHED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[AUTHED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  upi = global_aqpi;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpi,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpi to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T0]], i32 1, i64 [[OLDDISC]])
+// CHECK-NEXT:    [[AUTHED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[AUTHED]], {{.*}} ]
+// CHECK-NEXT:    call void @use_upi(ptr noundef [[T0]])
+  use_upi(global_aqpi);
+}
+
+// Function with address-independent qualifiers.
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_i_constant()
+void test_store_function_i_constant() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[SIGN:%.*]] = call i64 @llvm.ptrauth.resign(i64 ptrtoint (ptr ptrauth (ptr @external_func, i32 0, i64 18983) to i64), i32 0, i64 18983, i32 1, i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = inttoptr i64 [[SIGN]] to ptr
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * IQ iqpf = &external_func;
+// CHECK-NEXT:    [[SIGN:%.*]] = call i64 @llvm.ptrauth.resign(i64 ptrtoint (ptr ptrauth (ptr @external_func, i32 0, i64 18983) to i64), i32 0, i64 18983, i32 1, i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = inttoptr i64 [[SIGN]] to ptr
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  iqpf = &external_func;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_iu()
+void test_store_function_iu() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_upf,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 0, i64 18983, i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * IQ iqpf = global_upf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_upf,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 0, i64 18983, i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  iqpf = global_upf;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_ia()
+void test_store_function_ia() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * IQ iqpf = global_aqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  iqpf = global_aqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 50)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[RESULT:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[RESULT]], ptr [[V]],
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[RESULT]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[RESULT]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 0, i64 18983)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    call void @use_upf(ptr noundef [[T0]])
+  use_upf(iqpf = global_aqpf);
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_ii_same()
+void test_store_function_ii_same() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    store ptr [[LOAD]], ptr [[V]],
+  func_t * IQ iqpf = global_iqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    store ptr [[LOAD]], ptr [[V]],
+  iqpf = global_iqpf;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_ii_different()
+void test_store_function_ii_different() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 100)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * DIFF_IQ iqpf = global_iqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 100)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  iqpf = global_iqpf;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_load_function_i()
+void test_load_function_i() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 0, i64 18983)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t *upf = global_iqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 0, i64 18983)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  upf = global_iqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 0, i64 18983)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    call void @use_upf(ptr noundef [[T0]])
+  use_upf(global_iqpf);
+}
+
+// Function with address-discriminated qualifiers.
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_a_constant()
+void test_store_function_a_constant() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[SIGN:%.*]] = call i64 @llvm.ptrauth.resign(i64 ptrtoint (ptr ptrauth (ptr @external_func, i32 0, i64 18983) to i64), i32 0, i64 18983, i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[T0:%.*]] = inttoptr i64 [[SIGN]] to ptr
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * AQ aqpf = &external_func;
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[SIGN:%.*]] = call i64 @llvm.ptrauth.resign(i64 ptrtoint (ptr ptrauth (ptr @external_func, i32 0, i64 18983) to i64), i32 0, i64 18983, i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[T0:%.*]] = inttoptr i64 [[SIGN]] to ptr
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpf = &external_func;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_au()
+void test_store_function_au() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_upf,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 0, i64 18983, i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * AQ aqpf = global_upf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_upf,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 0, i64 18983, i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpf = global_upf;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_ai()
+void test_store_function_ai() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * AQ aqpf = global_iqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_iqpf,
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 50, i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpf = global_iqpf;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_aa_same()
+void test_store_function_aa_same() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * AQ aqpf = global_aqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpf = global_aqpf;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_store_function_aa_different()
+void test_store_function_aa_different() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 100)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t * DIFF_AQ aqpf = global_aqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = ptrtoint ptr [[V]] to i64
+// CHECK-NEXT:    [[NEWDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T0]], i64 100)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 1, i64 [[NEWDISC]])
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  aqpf = global_aqpf;
+}
+
+// CHECK-LABEL: define {{.*}}void @test_load_function_a()
+void test_load_function_a() {
+// CHECK:         [[V:%.*]] = alloca ptr,
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 0, i64 18983)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  func_t *upf = global_aqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 0, i64 18983)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    store ptr [[T0]], ptr [[V]],
+  upf = global_aqpf;
+// CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr @global_aqpf,
+// CHECK-NEXT:    [[OLDDISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 ptrtoint (ptr @global_aqpf to i64), i64 50)
+// CHECK-NEXT:    [[T0:%.*]] = icmp ne ptr [[LOAD]], null
+// CHECK-NEXT:    br i1 [[T0]],
+// CHECK:         [[T0:%.*]] = ptrtoint ptr [[LOAD]] to i64
+// CHECK-NEXT:    [[T1:%.*]] = call i64 @llvm.ptrauth.resign(i64 [[T0]], i32 1, i64 [[OLDDISC]], i32 0, i64 18983)
+// CHECK-NEXT:    [[SIGNED:%.*]] = inttoptr i64 [[T1]] to ptr
+// CHECK-NEXT:    br label
+// CHECK:         [[T0:%.*]] = phi ptr [ null, {{.*}} ], [ [[SIGNED]], {{.*}} ]
+// CHECK-NEXT:    call void @use_upf(ptr noundef [[T0]])
+  use_upf(global_aqpf);
+}
diff --git a/clang/test/CodeGenCXX/mangle-itanium-ptrauth.cpp b/clang/test/CodeGenCXX/mangle-itanium-ptrauth.cpp
new file mode 100644
index 0000000000000..88d80423c3764
--- /dev/null
+++ b/clang/test/CodeGenCXX/mangle-itanium-ptrauth.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -std=c++11 -fptrauth-intrinsics -fptrauth-calls -emit-llvm -o - -triple=arm64-apple-ios %s | FileCheck %s
+// RUN: %clang_cc1 -std=c++11 -fptrauth-intrinsics -fptrauth-calls -emit-llvm -o - -triple=aarch64-linux-gnu %s | FileCheck %s
+
+// CHECK: define {{.*}}void @_Z3fooPU9__ptrauthILj3ELb1ELj234EEPi(
+void foo(int * __ptrauth(3, 1, 234) *) {}
+
+template <class T>
+void foo(T t) {}
+
+// CHECK: define weak_odr void @_Z3fooIPU9__ptrauthILj1ELb0ELj64EEPiEvT_(
+template void foo<int * __ptrauth(1, 0, 64) *>(int * __ptrauth(1, 0, 64) *);
+
diff --git a/clang/test/CodeGenCXX/mangle-ms-ptrauth.cpp b/clang/test/CodeGenCXX/mangle-ms-ptrauth.cpp
new file mode 100644
index 0000000000000..95e5efa472dfd
--- /dev/null
+++ b/clang/test/CodeGenCXX/mangle-ms-ptrauth.cpp
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -std=c++11 -fptrauth-intrinsics -fptrauth-calls -emit-llvm -o - -triple=aarch64-windows-msvc %s | FileCheck %s
+
+template <class T>
+struct S {};
+
+// CHECK: @"?s@@3U?$S@PE__ptrauth1A@ENC@AH@@A" =
+S<int * __ptrauth(2, 0, 1234)> s;
+
+// CHECK: define dso_local void @"?foo@@YAXPEAPE__ptrauth20OK@AH@Z"(
+void foo(int * __ptrauth(3, 1, 234) *) {}
+
+template <class T>
+void foo(T t) {}
+
+// CHECK: define weak_odr dso_local void @"??$foo@PEAPE__ptrauth0A@EA@AH@@YAXPEAPE__ptrauth0A@EA@AH@Z"(
+template void foo<int * __ptrauth(1, 0, 64) *>(int * __ptrauth(1, 0, 64) *);
+
diff --git a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
new file mode 100644
index 0000000000000..7d6de50d926b5
--- /dev/null
+++ b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
@@ -0,0 +1,168 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -fptrauth-calls -fptrauth-intrinsics -std=c++11 -emit-llvm %s  -o - | FileCheck -check-prefixes=CHECK,IOS %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-calls -fptrauth-intrinsics -std=c++11 -emit-llvm %s  -o - | FileCheck %s
+
+#define AQ __ptrauth(1,1,50)
+#define IQ __ptrauth(1,0,50)
+
+// CHECK: %[[STRUCT_SA:.*]] = type { ptr, ptr }
+// CHECK: %[[STRUCT_SI:.*]] = type { ptr }
+
+struct SA {
+  int * AQ m0; // Signed using address discrimination.
+  int * AQ m1; // Signed using address discrimination.
+};
+
+struct SI {
+  int * IQ m; // No address discrimination.
+};
+
+struct __attribute__((trivial_abi)) TrivialSA {
+  int * AQ m0; // Signed using address discrimination.
+  int * AQ m1; // Signed using address discrimination.
+};
+
+// Check that TrivialSA is passed indirectly despite being annotated with
+// 'trivial_abi'.
+
+// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr noundef %{{.*}})
+
+void testParamTrivialSA(TrivialSA a) {
+}
+
+// CHECK: define {{.*}}void @_Z19testCopyConstructor2SA(ptr
+// CHECK: call {{.*}}@_ZN2SAC1ERKS_(
+
+// CHECK: define linkonce_odr {{.*}}@_ZN2SAC1ERKS_(
+// CHECK: call {{.*}}@_ZN2SAC2ERKS_(
+
+void testCopyConstructor(SA a) {
+  SA t = a;
+}
+
+// CHECK: define {{.*}}void @_Z19testMoveConstructor2SA(ptr
+// CHECK: call {{.*}}@_ZN2SAC1EOS_(
+
+// CHECK: define linkonce_odr {{.*}}@_ZN2SAC1EOS_(
+// CHECK: call {{.*}}@_ZN2SAC2EOS_(
+
+void testMoveConstructor(SA a) {
+  SA t = static_cast<SA &&>(a);
+}
+
+// CHECK: define {{.*}}void @_Z18testCopyAssignment2SA(ptr
+// CHECK: call noundef nonnull align 8 dereferenceable(16) ptr @_ZN2SAaSERKS_(
+
+// CHECK: define {{.*}}linkonce_odr noundef nonnull align 8 dereferenceable(16) ptr @_ZN2SAaSERKS_(ptr noundef nonnull align 8 dereferenceable(16) %[[THIS:.*]], ptr noundef nonnull align 8 dereferenceable(16) %0)
+// CHECK: %[[THIS_ADDR:.*]] = alloca ptr, align 8
+// CHECK: %[[_ADDR:.*]] = alloca ptr, align 8
+// CHECK: store ptr %[[THIS]], ptr %[[THIS_ADDR]], align 8
+// CHECK: store ptr %[[V0:.*]], ptr %[[_ADDR]], align 8
+// CHECK: %[[THISI:.*]] = load ptr, ptr %[[THIS_ADDR]], align 8
+// CHECK: %[[M0:.*]] = getelementptr inbounds nuw %[[STRUCT_SA]], ptr %[[THISI]], i32 0, i32 0
+// CHECK: %[[V1:.*]] = load ptr, ptr %[[_ADDR]], align 8
+// CHECK: %[[M02:.*]] = getelementptr inbounds nuw %[[STRUCT_SA]], ptr %[[V1]], i32 0, i32 0
+// CHECK: %[[V2:.*]] = load ptr, ptr %[[M02]], align 8
+// CHECK: %[[V3:.*]] = ptrtoint ptr %[[M02]] to i64
+// CHECK: %[[V4:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V3]], i64 50)
+// CHECK: %[[V5:.*]] = ptrtoint ptr %[[M0]] to i64
+// CHECK: %[[V6:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V5]], i64 50)
+// CHECK: %[[V8:.*]] = ptrtoint ptr %[[V2]] to i64
+// CHECK: %[[V9:.*]] = call i64 @llvm.ptrauth.resign(i64 %[[V8]], i32 1, i64 %[[V4]], i32 1, i64 %[[V6]])
+
+void testCopyAssignment(SA a) {
+  SA t;
+  t = a;
+}
+
+// CHECK: define {{.*}}void @_Z18testMoveAssignment2SA(ptr
+// CHECK: call noundef nonnull align 8 dereferenceable(16) ptr @_ZN2SAaSEOS_(
+
+// CHECK: define {{.*}}linkonce_odr noundef nonnull align 8 dereferenceable(16) ptr @_ZN2SAaSEOS_(ptr noundef nonnull align 8 dereferenceable(16) %[[THIS:.*]], ptr noundef nonnull align 8 dereferenceable(16) %0)
+// CHECK: %[[THIS_ADDR:.*]] = alloca ptr, align 8
+// CHECK: %[[_ADDR:.*]] = alloca ptr, align 8
+// CHECK: store ptr %[[THIS]], ptr %[[THIS_ADDR]], align 8
+// CHECK: store ptr %[[V0:.*]], ptr %[[_ADDR]], align 8
+// CHECK: %[[THISI:.*]] = load ptr, ptr %[[THIS_ADDR]], align 8
+// CHECK: %[[M0:.*]] = getelementptr inbounds nuw %[[STRUCT_SA]], ptr %[[THISI]], i32 0, i32 0
+// CHECK: %[[V1:.*]] = load ptr, ptr %[[_ADDR]], align 8
+// CHECK: %[[M02:.*]] = getelementptr inbounds nuw %[[STRUCT_SA]], ptr %[[V1]], i32 0, i32 0
+// CHECK: %[[V2:.*]] = load ptr, ptr %[[M02]], align 8
+// CHECK: %[[V3:.*]] = ptrtoint ptr %[[M02]] to i64
+// CHECK: %[[V4:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V3]], i64 50)
+// CHECK: %[[V5:.*]] = ptrtoint ptr %[[M0]] to i64
+// CHECK: %[[V6:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V5]], i64 50)
+// CHECK: %[[V8:.*]] = ptrtoint ptr %[[V2]] to i64
+// CHECK: %[[V9:.*]] = call i64 @llvm.ptrauth.resign(i64 %[[V8]], i32 1, i64 %[[V4]], i32 1, i64 %[[V6]])
+
+void testMoveAssignment(SA a) {
+  SA t;
+  t = static_cast<SA &&>(a);
+}
+
+// CHECK: define {{.*}}void @_Z19testCopyConstructor2SI(i
+// CHECK: call void @llvm.memcpy.p0.p0.i64(
+
+void testCopyConstructor(SI a) {
+  SI t = a;
+}
+
+// CHECK: define {{.*}}void @_Z19testMoveConstructor2SI(
+// CHECK: call void @llvm.memcpy.p0.p0.i64(
+
+void testMoveConstructor(SI a) {
+  SI t = static_cast<SI &&>(a);
+}
+
+// CHECK: define {{.*}}void @_Z18testCopyAssignment2SI(
+// CHECK: call void @llvm.memcpy.p0.p0.i64(
+
+void testCopyAssignment(SI a) {
+  SI t;
+  t = a;
+}
+
+// CHECK: define {{.*}}void @_Z18testMoveAssignment2SI(
+// CHECK: call void @llvm.memcpy.p0.p0.i64(
+
+void testMoveAssignment(SI a) {
+  SI t;
+  t = static_cast<SI &&>(a);
+}
+
+// CHECK: define linkonce_odr {{.*}}@_ZN2SAC2ERKS_(ptr noundef nonnull align 8 dereferenceable(16) %[[THIS:.*]], ptr noundef nonnull align 8 dereferenceable(16) %0)
+// IOS: %[[RETVAL:.*]] = alloca ptr, align 8
+// CHECK: %[[THIS_ADDR:.*]] = alloca ptr, align 8
+// CHECK: %[[_ADDR:.*]] = alloca ptr, align 8
+// CHECK: store ptr %[[THIS]], ptr %[[THIS_ADDR]], align 8
+// CHECK: store ptr %[[V0:.*]], ptr %[[_ADDR]], align 8
+// CHECK: %[[THIS1:.*]] = load ptr, ptr %[[THIS_ADDR]], align 8
+// IOS: store ptr %[[THIS1]], ptr %[[RETVAL]], align 8
+// CHECK: %[[M0:.*]] = getelementptr inbounds nuw %[[STRUCT_SA]], ptr %[[THIS1]], i32 0, i32 0
+// CHECK: %[[V1:.*]] = load ptr, ptr %[[_ADDR]], align 8
+// CHECK: %[[M02:.*]] = getelementptr inbounds nuw %[[STRUCT_SA]], ptr %[[V1]], i32 0, i32 0
+// CHECK: %[[V2:.*]] = load ptr, ptr %[[M02]], align 8
+// CHECK: %[[V3:.*]] = ptrtoint ptr %[[M02]] to i64
+// CHECK: %[[V4:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V3]], i64 50)
+// CHECK: %[[V5:.*]] = ptrtoint ptr %[[M0]] to i64
+// CHECK: %[[V6:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V5]], i64 50)
+// CHECK: %[[V8:.*]] = ptrtoint ptr %[[V2]] to i64
+// CHECK: %[[V9:.*]] = call i64 @llvm.ptrauth.resign(i64 %[[V8]], i32 1, i64 %[[V4]], i32 1, i64 %[[V6]])
+
+// CHECK: define linkonce_odr {{.*}}@_ZN2SAC2EOS_(ptr noundef nonnull align 8 dereferenceable(16) %[[THIS:.*]], ptr noundef nonnull align 8 dereferenceable(16) %0)
+// IOS: %[[RETVAL:.*]] = alloca ptr, align 8
+// CHECK: %[[THIS_ADDR:.*]] = alloca ptr, align 8
+// CHECK: %[[_ADDR:.*]] = alloca ptr, align 8
+// CHECK: store ptr %[[THIS]], ptr %[[THIS_ADDR]], align 8
+// CHECK: store ptr %[[V0:.*]], ptr %[[_ADDR]], align 8
+// CHECK: %[[THIS1:.*]] = load ptr, ptr %[[THIS_ADDR]], align 8
+// IOS: store ptr %[[THIS1]], ptr %[[RETVAL]], align 8
+// CHECK: %[[M0:.*]] = getelementptr inbounds nuw %[[STRUCT_SA]], ptr %[[THIS1]], i32 0, i32 0
+// CHECK: %[[V1:.*]] = load ptr, ptr %[[_ADDR]], align 8
+// CHECK: %[[M02:.*]] = getelementptr inbounds nuw %[[STRUCT_SA]], ptr %[[V1]], i32 0, i32 0
+// CHECK: %[[V2:.*]] = load ptr, ptr %[[M02]], align 8
+// CHECK: %[[V3:.*]] = ptrtoint ptr %[[M02]] to i64
+// CHECK: %[[V4:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V3]], i64 50)
+// CHECK: %[[V5:.*]] = ptrtoint ptr %[[M0]] to i64
+// CHECK: %[[V6:.*]] = call i64 @llvm.ptrauth.blend(i64 %[[V5]], i64 50)
+// CHECK: %[[V8:.*]] = ptrtoint ptr %[[V2]] to i64
+// CHECK: %[[V9:.*]] = call i64 @llvm.ptrauth.resign(i64 %[[V8]], i32 1, i64 %[[V4]], i32 1, i64 %[[V6]])
diff --git a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
new file mode 100644
index 0000000000000..e5cb71bad47c0
--- /dev/null
+++ b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
@@ -0,0 +1,35 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios11 -fptrauth-calls -fptrauth-intrinsics -std=c++11 -fobjc-arc -emit-llvm -o - %s | FileCheck %s
+
+#define AQ __ptrauth(1,1,50)
+
+struct AddrDiscStrong0 {
+  int * AQ f0; // Signed using address discrimination.
+  __strong id f1;
+};
+
+struct AddrDiscStrong1 {
+  AddrDiscStrong1(const AddrDiscStrong1 &);
+  int * AQ f0; // Signed using address discrimination.
+  __strong id f1;
+};
+
+// Check that AddrDiscStrong0 is destructed in the callee.
+
+// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr noundef %[[A:.*]])
+// CHECK: call noundef ptr @_ZN15AddrDiscStrong0D1Ev(ptr noundef nonnull align {{[0-9]+}} dereferenceable(16) %[[A]])
+// CHECK: ret void
+
+// CHECK: define linkonce_odr noundef ptr @_ZN15AddrDiscStrong0D1Ev(
+
+void testParamAddrDiscStrong0(AddrDiscStrong0 a) {
+}
+
+// Check that AddrDiscStrong1 is not destructed in the callee because it has a
+// non-trivial copy constructor.
+
+// CHECK: define void @_Z24testParamAddrDiscStrong115AddrDiscStrong1(ptr noundef %{{.*}})
+// CHECK-NOT: call
+// CHECK: ret void
+
+void testParamAddrDiscStrong1(AddrDiscStrong1 a) {
+}
diff --git a/clang/test/Parser/ptrauth-qualifier.c b/clang/test/Parser/ptrauth-qualifier.c
new file mode 100644
index 0000000000000..2071ac6c2d661
--- /dev/null
+++ b/clang/test/Parser/ptrauth-qualifier.c
@@ -0,0 +1,18 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify -fptrauth-intrinsics %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fsyntax-only -verify -fptrauth-intrinsics %s
+
+#if __aarch64__
+#define VALID_DATA_KEY 2
+#else
+#error Provide these constants if you port this test
+#endif
+
+int * __ptrauth(VALID_DATA_KEY) valid0;
+
+typedef int *intp;
+
+int nonConstantGlobal = 5;
+
+__ptrauth int invalid0; // expected-error{{expected '('}}
+__ptrauth() int invalid1; // expected-error{{expected expression}}
+int * __ptrauth(VALID_DATA_KEY, 1, 1000, 12) invalid12; // expected-error{{qualifier must take between 1 and 3 arguments}}
diff --git a/clang/test/Preprocessor/ptrauth_extension.c b/clang/test/Preprocessor/ptrauth_extension.c
new file mode 100644
index 0000000000000..d6b79187ba62d
--- /dev/null
+++ b/clang/test/Preprocessor/ptrauth_extension.c
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -E %s -triple=aarch64 -fptrauth-intrinsics | \
+// RUN:   FileCheck %s --check-prefixes=INTRIN
+
+// RUN: %clang_cc1 -E %s -triple=aarch64 -fptrauth-calls | \
+// RUN:   FileCheck %s --check-prefixes=NOINTRIN
+
+#if __has_extension(ptrauth_qualifier)
+// INTRIN: has_ptrauth_qualifier
+void has_ptrauth_qualifier() {}
+#else
+// NOINTRIN: no_ptrauth_qualifier
+void no_ptrauth_qualifier() {}
+#endif
diff --git a/clang/test/Sema/ptrauth-atomic-ops.c b/clang/test/Sema/ptrauth-atomic-ops.c
new file mode 100644
index 0000000000000..ccb9a1abcc14d
--- /dev/null
+++ b/clang/test/Sema/ptrauth-atomic-ops.c
@@ -0,0 +1,118 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify -fptrauth-intrinsics %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fsyntax-only -verify -fptrauth-intrinsics %s
+
+#include <stdatomic.h>
+
+int i;
+int *__ptrauth(2, 1, 100) authenticated_ptr = &i;
+int *__ptrauth(2, 0, 200) non_addr_discriminatedauthenticated_ptr = &i;
+int * wat = &i;
+#define ATOMIZE(p) (__typeof__(p) volatile _Atomic *)(long)(&p)
+
+void f() {
+  static int j = 1;
+  __c11_atomic_init(ATOMIZE(authenticated_ptr), 5);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __c11_atomic_store(ATOMIZE(authenticated_ptr), 0, memory_order_relaxed);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __c11_atomic_load(ATOMIZE(authenticated_ptr), memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __c11_atomic_store(ATOMIZE(authenticated_ptr), 1, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_store_n(ATOMIZE(authenticated_ptr), 4, memory_order_release);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_store(ATOMIZE(authenticated_ptr), j, memory_order_release);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __c11_atomic_exchange(ATOMIZE(authenticated_ptr), 1, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_exchange(ATOMIZE(authenticated_ptr), &j, &j, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __c11_atomic_fetch_add(ATOMIZE(authenticated_ptr), 1, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_fetch_add(ATOMIZE(authenticated_ptr), 3, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_fetch_sub(ATOMIZE(authenticated_ptr), 3, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_fetch_min(ATOMIZE(authenticated_ptr), 3, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_fetch_max(ATOMIZE(authenticated_ptr), 3, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __c11_atomic_fetch_and(ATOMIZE(authenticated_ptr), 1, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_fetch_and(ATOMIZE(authenticated_ptr), 3, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_fetch_or(ATOMIZE(authenticated_ptr), 3, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+  __atomic_fetch_xor(ATOMIZE(authenticated_ptr), 3, memory_order_seq_cst);
+  // expected-error@-1 {{address argument to atomic operation must be a pointer to a non address discriminated type ('volatile __ptrauth(2,1,100) _Atomic(int *) *' invalid)}}
+
+  __c11_atomic_init(ATOMIZE(non_addr_discriminatedauthenticated_ptr), &j);
+  __c11_atomic_store(ATOMIZE(non_addr_discriminatedauthenticated_ptr), 0, memory_order_relaxed);
+  __c11_atomic_load(ATOMIZE(non_addr_discriminatedauthenticated_ptr), memory_order_seq_cst);
+  __atomic_store(&j, ATOMIZE(non_addr_discriminatedauthenticated_ptr), memory_order_release);
+  // expected-warning@-1 {{incompatible pointer types passing 'volatile __ptrauth(2,0,200) _Atomic(int *) *' to parameter of type 'int *'}}
+  __c11_atomic_exchange(ATOMIZE(j), ATOMIZE(non_addr_discriminatedauthenticated_ptr), memory_order_seq_cst);
+  // expected-error@-1 {{incompatible pointer to integer conversion passing 'volatile __ptrauth(2,0,200) _Atomic(int *) *' to parameter of type 'typeof (j)' (aka 'int')}}
+  __c11_atomic_fetch_add(ATOMIZE(non_addr_discriminatedauthenticated_ptr), ATOMIZE(j), memory_order_seq_cst);
+  // expected-error@-1 {{incompatible pointer to integer conversion passing 'volatile _Atomic(typeof (j)) *' to parameter of type 'long'}}
+  __c11_atomic_fetch_and(ATOMIZE(j), ATOMIZE(non_addr_discriminatedauthenticated_ptr), memory_order_seq_cst);
+  // expected-error@-1 {{incompatible pointer to integer conversion passing 'volatile __ptrauth(2,0,200) _Atomic(int *) *' to parameter of type 'typeof (j)' (aka 'int')}}
+
+
+  __sync_fetch_and_add(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_fetch_and_sub(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_fetch_and_or(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_fetch_and_and(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_fetch_and_xor(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_fetch_and_nand(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+
+  __sync_add_and_fetch(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_sub_and_fetch(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_or_and_fetch(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_and_and_fetch(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_xor_and_fetch(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_nand_and_fetch(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+
+  __sync_bool_compare_and_swap(&authenticated_ptr, 1, 0);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_val_compare_and_swap(&authenticated_ptr, 1, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+
+  __sync_lock_test_and_set(&authenticated_ptr, 1);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+  __sync_lock_release(&authenticated_ptr);
+  // expected-error@-1 {{address argument to __sync operation must be a pointer to a non address discriminated type ('int *__ptrauth(2,1,100)' invalid)}}
+
+
+int i = 0;
+
+  __sync_fetch_and_add(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_fetch_and_sub(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_fetch_and_or(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_fetch_and_and(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_fetch_and_xor(&non_addr_discriminatedauthenticated_ptr, &i);
+
+  __sync_add_and_fetch(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_sub_and_fetch(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_or_and_fetch(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_and_and_fetch(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_xor_and_fetch(&non_addr_discriminatedauthenticated_ptr, &i);
+
+  __sync_bool_compare_and_swap(&non_addr_discriminatedauthenticated_ptr, &i, &i);
+  __sync_val_compare_and_swap(&non_addr_discriminatedauthenticated_ptr, &i, &i);
+
+  __sync_lock_test_and_set(&non_addr_discriminatedauthenticated_ptr, &i);
+  __sync_lock_release(&non_addr_discriminatedauthenticated_ptr);
+}
diff --git a/clang/test/Sema/ptrauth-qualifier.c b/clang/test/Sema/ptrauth-qualifier.c
new file mode 100644
index 0000000000000..99d16b062ca6f
--- /dev/null
+++ b/clang/test/Sema/ptrauth-qualifier.c
@@ -0,0 +1,103 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -std=c23 -fsyntax-only -verify -fptrauth-intrinsics %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -std=c23 -fsyntax-only -verify -fptrauth-intrinsics %s
+
+#if __has_feature(ptrauth_qualifier)
+#warning __ptrauth qualifier enabled!
+// expected-warning@-1 {{__ptrauth qualifier enabled!}}
+#endif
+
+#if __aarch64__
+#define VALID_CODE_KEY 0
+#define VALID_DATA_KEY 2
+#define INVALID_KEY 200
+#else
+#error Provide these constants if you port this test
+#endif
+
+int * __ptrauth(VALID_DATA_KEY) valid0;
+int *ptr0;
+
+typedef int *intp;
+
+int nonConstantGlobal = 5;
+
+__ptrauth(INVALID_KEY) int invalid2; // expected-error{{200 does not identify a valid pointer authentication key for the current target}}
+__ptrauth(VALID_DATA_KEY) int invalid3; // expected-error {{'__ptrauth' qualifier only applies to pointer types; 'int' is invalid}}
+__ptrauth(VALID_DATA_KEY) int *invalid4; // expected-error {{'__ptrauth' qualifier only applies to pointer types; 'int' is invalid}}
+int * (__ptrauth(VALID_DATA_KEY) invalid5); // expected-error{{expected identifier or '('}} expected-error{{expected ')'}} expected-note {{to match this '('}}
+int *__ptrauth(VALID_DATA_KEY) __ptrauth(VALID_DATA_KEY) invalid6; // expected-error{{type 'int *__ptrauth(2,0,0)' is already __ptrauth-qualified}}
+int * __ptrauth(VALID_DATA_KEY, 2) invalid7; // expected-error {{invalid address discrimination flag '2'; '__ptrauth' requires '0' or '1'}}
+int * __ptrauth(VALID_DATA_KEY, -1) invalid8; // expected-error {{invalid address discrimination flag '-1'; '__ptrauth' requires '0' or '1'}}
+int * __ptrauth(VALID_DATA_KEY, 1, -1) invalid9; // expected-error {{invalid extra discriminator flag '-1'; '__ptrauth' requires a value between '0' and '65535'}}
+int * __ptrauth(VALID_DATA_KEY, 1, 100000) invalid10; // expected-error {{invalid extra discriminator flag '100000'; '__ptrauth' requires a value between '0' and '65535'}}
+int * __ptrauth(VALID_DATA_KEY, 1, nonConstantGlobal) invalid12; // expected-error {{argument to '__ptrauth' must be an integer constant expression}}
+int * __ptrauth(VALID_DATA_KEY, nonConstantGlobal, 1000) invalid13; // expected-error {{argument to '__ptrauth' must be an integer constant expression}}
+int * __ptrauth(nonConstantGlobal, 1, 1000) invalid14; // expected-error{{expression is not an integer constant expression}}
+
+int * __ptrauth(VALID_DATA_KEY) valid0;
+int * __ptrauth(VALID_DATA_KEY) *valid1;
+__ptrauth(VALID_DATA_KEY) intp valid2;
+__ptrauth(VALID_DATA_KEY) intp *valid3;
+intp __ptrauth(VALID_DATA_KEY) valid4;
+intp __ptrauth(VALID_DATA_KEY) *valid5;
+int * __ptrauth(VALID_DATA_KEY, 0) valid6;
+int * __ptrauth(VALID_DATA_KEY, 1) valid7;
+int * __ptrauth(VALID_DATA_KEY, (_Bool) 1) valid8;
+int * __ptrauth(VALID_DATA_KEY, 1, 0) valid9;
+int * __ptrauth(VALID_DATA_KEY, 1, 65535) valid10;
+
+int * __ptrauth(VALID_DATA_KEY) array0[10];
+int (* __ptrauth(VALID_DATA_KEY) array1)[10];
+
+extern intp redeclaration0; // expected-note {{previous declaration}}
+extern intp __ptrauth(VALID_DATA_KEY) redeclaration0; // expected-error{{redeclaration of 'redeclaration0' with a different type: '__ptrauth(2,0,0) intp' (aka 'int *__ptrauth(2,0,0)') vs 'intp' (aka 'int *')}}
+
+extern intp redeclaration1; // expected-note {{previous declaration}}
+extern intp __ptrauth(VALID_DATA_KEY) redeclaration1; // expected-error{{redeclaration of 'redeclaration1' with a different type: '__ptrauth(2,0,0) intp' (aka 'int *__ptrauth(2,0,0)') vs 'intp' (aka 'int *')}}
+
+intp __ptrauth(VALID_DATA_KEY) redeclaration2; // expected-note {{previous definition}}
+intp redeclaration2 = 0;                       // expected-error{{redefinition of 'redeclaration2' with a different type: 'intp' (aka 'int *') vs '__ptrauth(2,0,0) intp' (aka 'int *__ptrauth(2,0,0)')}}
+
+intp __ptrauth(VALID_DATA_KEY) redeclaration3; // expected-note {{previous definition}}
+intp redeclaration3 = 0;                       // expected-error{{redefinition of 'redeclaration3' with a different type: 'intp' (aka 'int *') vs '__ptrauth(2,0,0) intp' (aka 'int *__ptrauth(2,0,0)')}}
+
+void illegal0(intp __ptrauth(VALID_DATA_KEY)); // expected-error {{parameter type may not be qualified with '__ptrauth'; type is '__ptrauth(2,0,0) intp' (aka 'int *__ptrauth(2,0,0)')}}
+intp __ptrauth(VALID_DATA_KEY) illegal1(void); // expected-error {{return type may not be qualified with '__ptrauth'; type is '__ptrauth(2,0,0) intp' (aka 'int *__ptrauth(2,0,0)')}}
+
+static_assert(_Generic(typeof(valid0), int * __ptrauth(VALID_DATA_KEY) : 1, int * : 0, default : 0));
+static_assert(_Generic(typeof(valid0), int * __ptrauth(VALID_CODE_KEY) : 0, default : 1));
+static_assert(_Generic(typeof_unqual(valid0), int * __ptrauth(VALID_DATA_KEY) : 0, int * : 1, default : 0));
+static_assert(_Generic(valid0, int * __ptrauth(VALID_DATA_KEY) : 0, int * : 1, default : 0)); // expected-warning {{association of type 'int *__ptrauth(2,0,0)' will never be selected}}
+
+static_assert(_Generic(array0, int * __ptrauth(VALID_DATA_KEY) * : 1, default : 0));
+static_assert(_Generic(*array1, int * : 1, default : 0));
+
+void test_code(intp p) {
+  p = (intp __ptrauth(VALID_DATA_KEY)) 0; // expected-error {{cannot cast to '__ptrauth'-qualified type '__ptrauth(2,0,0) intp' (aka 'int *__ptrauth(2,0,0)')}}
+
+  __ptrauth(VALID_DATA_KEY) intp pSpecial = p;
+  pSpecial = p;
+  intp pNormal = pSpecial;
+  pNormal = pSpecial;
+
+  intp __ptrauth(VALID_DATA_KEY) *ppSpecial0 = &pSpecial;
+  intp __ptrauth(VALID_DATA_KEY) *ppSpecial1 = &pNormal; // expected-error {{initializing '__ptrauth(2,0,0) intp *' (aka 'int *__ptrauth(2,0,0) *') with an expression of type 'intp *' (aka 'int **') changes pointer authentication of pointee type}}
+  intp *ppNormal0 = &pSpecial; // expected-error {{initializing 'intp *' (aka 'int **') with an expression of type '__ptrauth(2,0,0) intp *' (aka 'int *__ptrauth(2,0,0) *') changes pointer authentication of pointee type}}
+  intp *ppNormal1 = &pNormal;
+
+  intp *pp5 = (p ? &pSpecial : &pNormal); // expected-error {{'__ptrauth' qualification mismatch ('__ptrauth(2,0,0) intp *' (aka 'int *__ptrauth(2,0,0) *') and 'intp *' (aka 'int **'))}}
+}
+
+void test_array(void) {
+  intp __ptrauth(VALID_DATA_KEY) pSpecialArray[10];
+  intp __ptrauth(VALID_DATA_KEY) *ppSpecial0 = pSpecialArray;
+  intp __ptrauth(VALID_DATA_KEY) *ppSpecial1 = &pSpecialArray[0];
+}
+
+__attribute__((overloadable)) int overload_func(int **);
+__attribute__((overloadable)) float overload_func(int * __ptrauth(VALID_DATA_KEY) *);
+
+static_assert(_Generic(typeof(overload_func(&ptr0)), int : 1, default : 0));
+static_assert(_Generic(typeof(overload_func(&valid0)), float : 1, default : 0));
+
+void func(int array[__ptrauth(VALID_DATA_KEY) 10]); // expected-error {{'__ptrauth' qualifier only applies to pointer types; 'int[10]' is invalid}}
diff --git a/clang/test/SemaCXX/ptrauth-qualifier.cpp b/clang/test/SemaCXX/ptrauth-qualifier.cpp
new file mode 100644
index 0000000000000..a7dc6ae2ffe86
--- /dev/null
+++ b/clang/test/SemaCXX/ptrauth-qualifier.cpp
@@ -0,0 +1,213 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -std=c++20  -fptrauth-calls -fptrauth-intrinsics -verify -fsyntax-only %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -std=c++20  -fptrauth-calls -fptrauth-intrinsics -verify -fsyntax-only %s
+
+#define AQ __ptrauth(1,1,50)
+#define AQ2 __ptrauth(1,1,51)
+#define IQ __ptrauth(1,0,50)
+
+struct __attribute__((trivial_abi)) AddrDisc { // expected-warning {{'trivial_abi' cannot be applied to 'AddrDisc'}} expected-note {{'trivial_abi' is disallowed on 'AddrDisc' because it has an address-discriminated '__ptrauth' field}}
+  int * AQ m0;
+};
+
+struct __attribute__((trivial_abi)) NoAddrDisc {
+  int * IQ m0;
+};
+
+namespace test_union {
+
+  union U0 {
+    int * AQ f0; // expected-note 4 {{'U0' is implicitly deleted because variant field 'f0' has an address-discriminated '__ptrauth' qualifier}}
+
+    // ptrauth fields that don't have an address-discriminated qualifier don't
+    // delete the special functions.
+    int * IQ f1;
+  };
+
+  union U1 {
+    int * AQ f0; // expected-note 8 {{'U1' is implicitly deleted because variant field 'f0' has an address-discriminated '__ptrauth' qualifier}}
+    U1() = default;
+    ~U1() = default;
+    U1(const U1 &) = default; // expected-warning {{explicitly defaulted copy constructor is implicitly deleted}} expected-note 2 {{explicitly defaulted function was implicitly deleted here}} expected-note{{replace 'default'}}
+    U1(U1 &&) = default; // expected-warning {{explicitly defaulted move constructor is implicitly deleted}} expected-note{{replace 'default'}}
+    U1 & operator=(const U1 &) = default; // expected-warning {{explicitly defaulted copy assignment operator is implicitly deleted}} expected-note 2 {{explicitly defaulted function was implicitly deleted here}} expected-note{{replace 'default'}}
+    U1 & operator=(U1 &&) = default; // expected-warning {{explicitly defaulted move assignment operator is implicitly deleted}} expected-note{{replace 'default'}}
+  };
+
+  // It's fine if the user has explicitly defined the special functions.
+  union U2 {
+    int * AQ f0;
+    U2() = default;
+    ~U2() = default;
+    U2(const U2 &);
+    U2(U2 &&);
+    U2 & operator=(const U2 &);
+    U2 & operator=(U2 &&);
+  };
+
+  // Address-discriminated ptrauth fields in anonymous union fields delete the
+  // defaulted copy/move constructors/assignment operators of the containing
+  // class.
+  struct S0 {
+    union {
+      int * AQ f0; // expected-note 4 {{' is implicitly deleted because variant field 'f0' has an address-discriminated '__ptrauth' qualifier}}
+      char f1;
+    };
+  };
+
+  struct S1 {
+    union {
+      union {
+        int * AQ f0; // expected-note 4 {{implicitly deleted because variant field 'f0' has an address-discriminated '__ptrauth' qualifier}}
+        char f1;
+      } u; // expected-note 4 {{'S1' is implicitly deleted because field 'u' has a deleted}}
+      int f2;
+    };
+  };
+
+  U0 *x0;
+  U1 *x1;
+  U2 *x2;
+  S0 *x3;
+  S1 *x4;
+
+  // No diagnostics since constructors/destructors of the unions aren't deleted by default.
+  void testDefaultConstructor() {
+    U0 u0;
+    U1 u1;
+    U2 u2;
+    S0 s0;
+    S1 s1;
+  }
+
+  // No diagnostics since destructors of the unions aren't deleted by default.
+  void testDestructor(U0 *u0, U1 *u1, U2 *u2, S0 *s0, S1 *s1) {
+    delete u0;
+    delete u1;
+    delete u2;
+    delete s0;
+    delete s1;
+  }
+
+  void testCopyConstructor(U0 *u0, U1 *u1, U2 *u2, S0 *s0, S1 *s1) {
+    U0 t0(*u0); // expected-error {{call to implicitly-deleted copy constructor}}
+    U1 t1(*u1); // expected-error {{call to implicitly-deleted copy constructor}}
+    U2 t2(*u2);
+    S0 t3(*s0); // expected-error {{call to implicitly-deleted copy constructor}}
+    S1 t4(*s1); // expected-error {{call to implicitly-deleted copy constructor}}
+  }
+
+  void testCopyAssignment(U0 *u0, U1 *u1, U2 *u2, S0 *s0, S1 *s1) {
+    *x0 = *u0; // expected-error {{cannot be assigned because its copy assignment operator is implicitly deleted}}
+    *x1 = *u1; // expected-error {{cannot be assigned because its copy assignment operator is implicitly deleted}}
+    *x2 = *u2;
+    *x3 = *s0; // expected-error {{cannot be assigned because its copy assignment operator is implicitly deleted}}
+    *x4 = *s1; // expected-error {{cannot be assigned because its copy assignment operator is implicitly deleted}}
+  }
+
+  void testMoveConstructor(U0 *u0, U1 *u1, U2 *u2, S0 *s0, S1 *s1) {
+    U0 t0(static_cast<U0 &&>(*u0)); // expected-error {{call to implicitly-deleted copy constructor}}
+    U1 t1(static_cast<U1 &&>(*u1)); // expected-error {{call to implicitly-deleted copy constructor}}
+    U2 t2(static_cast<U2 &&>(*u2));
+    S0 t3(static_cast<S0 &&>(*s0)); // expected-error {{call to implicitly-deleted copy constructor}}
+    S1 t4(static_cast<S1 &&>(*s1)); // expected-error {{call to implicitly-deleted copy constructor}}
+  }
+
+  void testMoveAssignment(U0 *u0, U1 *u1, U2 *u2, S0 *s0, S1 *s1) {
+    *x0 = static_cast<U0 &&>(*u0); // expected-error {{cannot be assigned because its copy assignment operator is implicitly deleted}}
+    *x1 = static_cast<U1 &&>(*u1); // expected-error {{cannot be assigned because its copy assignment operator is implicitly deleted}}
+    *x2 = static_cast<U2 &&>(*u2);
+    *x3 = static_cast<S0 &&>(*s0); // expected-error {{cannot be assigned because its copy assignment operator is implicitly deleted}}
+    *x4 = static_cast<S1 &&>(*s1); // expected-error {{cannot be assigned because its copy assignment operator is implicitly deleted}}
+  }
+}
+
+bool test_composite_type0(bool c, int * AQ * a0, int * AQ * a1) {
+  auto t = c ? a0 : a1;
+  return a0 == a1;
+}
+
+bool test_composite_type1(bool c, int * AQ * a0, int * AQ2 * a1) {
+  auto t = c ? a0 : a1; // expected-error {{incompatible operand types ('int *__ptrauth(1,1,50) *' and 'int *__ptrauth(1,1,51) *')}}
+  return a0 == a1;      // expected-error {{comparison of distinct pointer types ('int *__ptrauth(1,1,50) *' and 'int *__ptrauth(1,1,51) *')}}
+}
+
+void test_bad_call_diag(void *AQ *ptr); // expected-note{{candidate function not viable: 1st argument ('void *__ptrauth(1,1,51) *') has __ptrauth(1,1,51) qualifier, but parameter has __ptrauth(1,1,50) qualifier}} expected-note {{candidate function not viable: 1st argument ('void **') has no '__ptrauth' qualifier, but parameter has __ptrauth(1,1,50) qualifier}}
+void test_bad_call_diag2(void **ptr); // expected-note {{candidate function not viable: 1st argument ('void *__ptrauth(1,1,50) *') has __ptrauth(1,1,50) qualifier, but parameter has no '__ptrauth' qualifier}}
+
+int test_call_diag() {
+  void *AQ ptr1, *AQ2 ptr2, *ptr3;
+  test_bad_call_diag(&ptr2); // expected-error {{no matching function for call to 'test_bad_call_diag'}}
+  test_bad_call_diag(&ptr3); // expected-error {{no matching function for call to 'test_bad_call_diag'}}
+  test_bad_call_diag2(&ptr1); // expected-error {{no matching function for call to 'test_bad_call_diag2'}}
+}
+
+namespace test_constexpr {
+  constexpr int i = 100;
+  constexpr const int * AQ p = &i;
+  constexpr const int * const AQ *pp = &p;
+  constexpr int i1 = **((const int * const AQ *)pp);
+  constexpr int i2 = **((const int * const AQ2 *)pp);
+  // expected-error@-1 {{constexpr variable 'i2' must be initialized by a constant expression}}
+  // expected-note@-2 {{cast that performs the conversions of a reinterpret_cast is not allowed in a constant expression}}
+}
+
+namespace test_lambda {
+  void test() {
+    int * AQ v0;
+    int * AQ *v1;
+
+    [v0, v1]() {
+      static_assert(__is_same(decltype(v0), int * AQ));
+      static_assert(__is_same(decltype(v1), int * AQ *));
+    }();
+
+    [v2 = v0, v3 = v1]() {
+      static_assert(__is_same(decltype(v2), int *));
+      static_assert(__is_same(decltype(v3), int * AQ *));
+    }();
+  }
+}
+
+namespace test_concept {
+  template <typename T> struct is_qualified {
+    static constexpr bool value = false;
+  };
+
+  template <typename T> struct is_qualified<T * AQ> {
+    static constexpr bool value = true;
+  };
+
+  template <typename T>
+  concept Ptrauthable = is_qualified<T>::value;
+  // expected-note@-1 2 {{because 'is_qualified<int *>::value' evaluated to false}}
+  // expected-note@-2 2 {{because 'is_qualified<int *__ptrauth(1,1,51)>::value' evaluated to false}}
+
+  template <typename T>
+    requires(Ptrauthable<T>)
+  struct S {};
+  // expected-note@-2 {{because 'int *' does not satisfy 'Ptrauthable'}}
+  // expected-note@-3 {{because 'int *__ptrauth(1,1,51)' does not satisfy 'Ptrauthable'}}
+
+  S<int * AQ> s0;
+  S<int *> s1;
+  // expected-error@-1 {{constraints not satisfied for class template 'S' [with T = int *]}}
+  S<int * AQ2> s1;
+  // expected-error@-1 {{constraints not satisfied for class template 'S' [with T = int *__ptrauth(1,1,51)]}}
+
+  template <typename T>
+    requires(Ptrauthable<T>)
+  void func(T *);
+  // expected-note@-1 {{candidate template ignored: constraints not satisfied [with T = int *]}}
+  // expected-note@-3 {{because 'int *' does not satisfy 'Ptrauthable'}}
+  // expected-note@-3 {{candidate template ignored: constraints not satisfied [with T = int *__ptrauth(1,1,51)]}}
+  // expected-note@-5 {{because 'int *__ptrauth(1,1,51)' does not satisfy 'Ptrauthable'}}
+
+  void test() {
+    int * AQ p0;
+    int *p1;
+    int * AQ2 p2;
+    func(&p0);
+    func(&p1); // expected-error {{no matching function for call to 'func'}}
+    func(&p2); // expected-error {{no matching function for call to 'func'}}
+  }
+}
diff --git a/clang/test/SemaCXX/ptrauth-template-parameters.cpp b/clang/test/SemaCXX/ptrauth-template-parameters.cpp
new file mode 100644
index 0000000000000..ee23d3f2ec456
--- /dev/null
+++ b/clang/test/SemaCXX/ptrauth-template-parameters.cpp
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify -fptrauth-intrinsics -std=c++11 %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fsyntax-only -verify -fptrauth-intrinsics -std=c++11 %s
+
+template <typename T> struct G {
+  T __ptrauth(0,0,1234) test;
+  // expected-error@-1 2 {{type '__ptrauth(0,0,1234) T' is already __ptrauth-qualified}}
+};
+
+template <typename T> struct Indirect {
+  G<T> layers;
+  // expected-note@-1{{in instantiation of template class 'G<void *__ptrauth(0,0,1235)>' requested here}}
+  // expected-note@-2{{in instantiation of template class 'G<void *__ptrauth(0,0,1234)>' requested here}}
+};
+
+template <int K, int A, int D>
+struct TemplateParameters {
+  void * __ptrauth(K, 0, 100) m1; // expected-error {{expression is not an integer constant expression}}
+  void * __ptrauth(0, A, 100) m2; // expected-error {{argument to '__ptrauth' must be an integer constant expression}}
+  void * __ptrauth(0, 0, D) m3; // expected-error {{argument to '__ptrauth' must be an integer constant expression}}
+};
+
+void f3() {
+  // FIXME: consider loosening the restrictions so that the first two cases are accepted.
+  Indirect<void* __ptrauth(0,0,1234)> one;
+  // expected-note@-1{{in instantiation of template class 'Indirect<void *__ptrauth(0,0,1234)>' requested here}}
+  Indirect<void* __ptrauth(0,0,1235)> two;
+  // expected-note@-1{{in instantiation of template class 'Indirect<void *__ptrauth(0,0,1235)>' requested here}}
+  Indirect<void*> three;
+}
diff --git a/clang/test/SemaObjC/ptrauth-qualifier.m b/clang/test/SemaObjC/ptrauth-qualifier.m
new file mode 100644
index 0000000000000..4836a653dd02f
--- /dev/null
+++ b/clang/test/SemaObjC/ptrauth-qualifier.m
@@ -0,0 +1,56 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify -fptrauth-intrinsics %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fsyntax-only -verify -fptrauth-intrinsics %s
+
+#if __has_feature(ptrauth_qualifier)
+#warning __ptrauth qualifier enabled!
+// expected-warning@-1 {{__ptrauth qualifier enabled!}}
+#endif
+
+@interface Foo
+// expected-warning@-1 {{class 'Foo' defined without specifying a base class}}
+// expected-note@-2 {{add a super class to fix this problem}}
+
+@property void *__ptrauth(1, 1, 1) invalid1;
+// expected-error@-1 {{property may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,1,1)'}}
+
+@property void *__ptrauth(1, 0, 1) invalid2;
+// expected-error@-1 {{property may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,0,1)'}}
+
+- (void *__ptrauth(1, 1, 1))invalid5;
+// expected-error@-1 {{return type may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,1,1)'}}
+
+- (void *__ptrauth(1, 0, 1))invalid6;
+// expected-error@-1 {{return type may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,0,1)'}}
+
+- (void)invalid9:(void *__ptrauth(1, 1, 1))a;
+// expected-error@-1 {{parameter type may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,1,1)'}}
+// expected-note@-2 {{method 'invalid9:' declared here}}
+
+- (void)invalid10:(void *__ptrauth(1, 0, 1))a;
+// expected-error@-1 {{parameter type may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,0,1)'}}
+// expected-note@-2 {{method 'invalid10:' declared here}}
+
+@end
+
+@implementation Foo
+// expected-warning@-1 2{{method definition for}}
+
+- (void *__ptrauth(1, 1, 1))invalid13 {
+// expected-error@-1 {{return type may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,1,1)'}}
+  return 0;
+}
+
+- (void *__ptrauth(1, 0, 1))invalid14 {
+// expected-error@-1 {{return type may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,0,1)'}}
+  return 0;
+}
+
+- (void)invalid17:(void *__ptrauth(1, 1, 1))a {
+// expected-error@-1 {{parameter type may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,1,1)'}}
+}
+
+- (void)invalid18:(void *__ptrauth(1, 0, 1))a {
+// expected-error@-1 {{parameter type may not be qualified with '__ptrauth'; type is 'void *__ptrauth(1,0,1)'}}
+}
+
+@end
diff --git a/libcxxabi/test/test_demangle.pass.cpp b/libcxxabi/test/test_demangle.pass.cpp
index abaa787f5432b..53da1bf6765e7 100644
--- a/libcxxabi/test/test_demangle.pass.cpp
+++ b/libcxxabi/test/test_demangle.pass.cpp
@@ -30243,6 +30243,9 @@ const char* cases[][2] = {
     {"_Z1fDSDRm", "f(_Sat unsigned long _Fract)"},
 
     {"_Z11bfloat16addDF16bDF16b", "bfloat16add(std::bfloat16_t, std::bfloat16_t)"},
+
+    {"_Z3fooPU9__ptrauthILj3ELb1ELj234EEPi", "foo(int* __ptrauth<3u, true, 234u>*)"},
+    {"_Z3fooIPU9__ptrauthILj1ELb0ELj64EEPiEvT_", "void foo<int* __ptrauth<1u, false, 64u>*>(int* __ptrauth<1u, false, 64u>*)"},
     // clang-format on
 };
 
diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangle.h b/llvm/include/llvm/Demangle/MicrosoftDemangle.h
index 276efa7603690..b9a25e361eec0 100644
--- a/llvm/include/llvm/Demangle/MicrosoftDemangle.h
+++ b/llvm/include/llvm/Demangle/MicrosoftDemangle.h
@@ -173,6 +173,14 @@ class Demangler {
 
   Qualifiers demanglePointerExtQualifiers(std::string_view &MangledName);
 
+  bool isMemberPointer(std::string_view MangledName, bool &Error);
+
+  std::optional<PointerAuthQualifierNode::ArgArray>
+  demanglePointerAuthQualifier(std::string_view &MangledName);
+
+  PointerAuthQualifierNode *
+  createPointerAuthQualifier(std::string_view &MangledName);
+
   // Parser functions. This is a recursive-descent parser.
   TypeNode *demangleType(std::string_view &MangledName,
                          QualifierMangleMode QMM);
diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
index 09b9d947464ae..d72fb47cd9b04 100644
--- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
+++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
@@ -253,7 +253,8 @@ enum class NodeKind {
   LocalStaticGuardVariable,
   FunctionSymbol,
   VariableSymbol,
-  SpecialTableSymbol
+  SpecialTableSymbol,
+  PointerAuthQualifier,
 };
 
 struct Node {
@@ -295,6 +296,7 @@ struct SymbolNode;
 struct FunctionSymbolNode;
 struct VariableSymbolNode;
 struct SpecialTableSymbolNode;
+struct PointerAuthQualifierNode;
 
 struct TypeNode : public Node {
   explicit TypeNode(NodeKind K) : Node(K) {}
@@ -467,6 +469,8 @@ struct PointerTypeNode : public TypeNode {
   // If this is a member pointer, this is the class that the member is in.
   QualifiedNameNode *ClassParent = nullptr;
 
+  PointerAuthQualifierNode *PointerAuthQualifier = nullptr;
+
   // Represents a type X in "a pointer to X", "a reference to X", or
   // "rvalue-reference to X"
   TypeNode *Pointee = nullptr;
@@ -625,6 +629,22 @@ struct FunctionSymbolNode : public SymbolNode {
   FunctionSignatureNode *Signature = nullptr;
 };
 
+struct PointerAuthQualifierNode : public Node {
+  PointerAuthQualifierNode() : Node(NodeKind::PointerAuthQualifier) {}
+
+  // __ptrauth takes three arguments:
+  //  - key
+  //  - isAddressDiscriminated
+  //  - extra discriminator
+  static constexpr unsigned NumArgs = 3;
+  typedef std::array<uint64_t, NumArgs> ArgArray;
+
+  void output(OutputBuffer &OB, OutputFlags Flags) const override;
+
+  // List of arguments.
+  NodeArrayNode *Components = nullptr;
+};
+
 } // namespace ms_demangle
 } // namespace llvm
 
diff --git a/llvm/lib/Demangle/MicrosoftDemangle.cpp b/llvm/lib/Demangle/MicrosoftDemangle.cpp
index 8d5f6b21e2e76..b22928be3be50 100644
--- a/llvm/lib/Demangle/MicrosoftDemangle.cpp
+++ b/llvm/lib/Demangle/MicrosoftDemangle.cpp
@@ -66,7 +66,7 @@ static bool startsWith(std::string_view S, std::string_view PrefixA,
   return llvm::itanium_demangle::starts_with(S, Prefix);
 }
 
-static bool isMemberPointer(std::string_view MangledName, bool &Error) {
+bool Demangler::isMemberPointer(std::string_view MangledName, bool &Error) {
   Error = false;
   const char F = MangledName.front();
   MangledName.remove_prefix(1);
@@ -107,6 +107,7 @@ static bool isMemberPointer(std::string_view MangledName, bool &Error) {
   consumeFront(MangledName, 'E'); // 64-bit
   consumeFront(MangledName, 'I'); // restrict
   consumeFront(MangledName, 'F'); // unaligned
+  demanglePointerAuthQualifier(MangledName);
 
   if (MangledName.empty()) {
     Error = true;
@@ -2099,6 +2100,8 @@ PointerTypeNode *Demangler::demanglePointerType(std::string_view &MangledName) {
   Qualifiers ExtQuals = demanglePointerExtQualifiers(MangledName);
   Pointer->Quals = Qualifiers(Pointer->Quals | ExtQuals);
 
+  Pointer->PointerAuthQualifier = createPointerAuthQualifier(MangledName);
+
   Pointer->Pointee = demangleType(MangledName, QualifierMangleMode::Mangle);
   return Pointer;
 }
@@ -2147,6 +2150,49 @@ Demangler::demanglePointerExtQualifiers(std::string_view &MangledName) {
   return Quals;
 }
 
+std::optional<PointerAuthQualifierNode::ArgArray>
+Demangler::demanglePointerAuthQualifier(std::string_view &MangledName) {
+  if (!consumeFront(MangledName, "__ptrauth"))
+    return std::nullopt;
+
+  constexpr unsigned NumArgs = PointerAuthQualifierNode::NumArgs;
+  PointerAuthQualifierNode::ArgArray Array;
+
+  for (unsigned I = 0; I < NumArgs; ++I) {
+    bool IsNegative = false;
+    uint64_t Value = 0;
+    std::tie(Value, IsNegative) = demangleNumber(MangledName);
+    if (IsNegative)
+      return std::nullopt;
+
+    Array[I] = Value;
+  }
+
+  return Array;
+}
+
+PointerAuthQualifierNode *
+Demangler::createPointerAuthQualifier(std::string_view &MangledName) {
+  constexpr unsigned NumArgs = PointerAuthQualifierNode::NumArgs;
+  std::optional<PointerAuthQualifierNode::ArgArray> Vals =
+      demanglePointerAuthQualifier(MangledName);
+
+  if (!Vals)
+    return nullptr;
+
+  PointerAuthQualifierNode *PtrAuthQual =
+      Arena.alloc<PointerAuthQualifierNode>();
+  NodeArrayNode *Array = Arena.alloc<NodeArrayNode>();
+  PtrAuthQual->Components = Array;
+  Array->Count = NumArgs;
+  Array->Nodes = Arena.allocArray<Node *>(NumArgs);
+
+  for (unsigned I = 0; I < NumArgs; ++I)
+    Array->Nodes[I] = Arena.alloc<IntegerLiteralNode>((*Vals)[I], false);
+
+  return PtrAuthQual;
+}
+
 ArrayTypeNode *Demangler::demangleArrayType(std::string_view &MangledName) {
   assert(MangledName.front() == 'Y');
   MangledName.remove_prefix(1);
diff --git a/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp b/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
index ec6e67058c683..61e4961c714bc 100644
--- a/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
+++ b/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
@@ -521,6 +521,9 @@ void PointerTypeNode::outputPre(OutputBuffer &OB, OutputFlags Flags) const {
     assert(false);
   }
   outputQualifiers(OB, Quals, false, false);
+
+  if (PointerAuthQualifier)
+    PointerAuthQualifier->output(OB, Flags);
 }
 
 void PointerTypeNode::outputPost(OutputBuffer &OB, OutputFlags Flags) const {
@@ -591,6 +594,13 @@ void FunctionSymbolNode::output(OutputBuffer &OB, OutputFlags Flags) const {
   Signature->outputPost(OB, Flags);
 }
 
+void PointerAuthQualifierNode::output(OutputBuffer &OB,
+                                      OutputFlags Flags) const {
+  OB << "__ptrauth(";
+  Components->output(OB, Flags);
+  OB << ")";
+}
+
 void VariableSymbolNode::output(OutputBuffer &OB, OutputFlags Flags) const {
   const char *AccessSpec = nullptr;
   bool IsStatic = true;
diff --git a/llvm/test/Demangle/ms-ptrauth.test b/llvm/test/Demangle/ms-ptrauth.test
new file mode 100644
index 0000000000000..18a9f37bec67a
--- /dev/null
+++ b/llvm/test/Demangle/ms-ptrauth.test
@@ -0,0 +1,12 @@
+; RUN: llvm-undname < %s | FileCheck %s
+
+; CHECK-NOT: Invalid mangled name
+
+?s@@3U?$S@PE__ptrauth1A@ENC@AH@@A
+; CHECK: struct S<int *__ptrauth(2, 0, 1234)> s
+
+?foo@@YAXPEAPE__ptrauth20OK@AH@Z
+; CHECK: void __cdecl foo(int *__ptrauth(3, 1, 234)*)
+
+??$foo@PEAPE__ptrauth0A@EA@AH@@YAXPEAPE__ptrauth0A@EA@AH@Z
+; CHECK: void __cdecl foo<int *__ptrauth(1, 0, 64)*>(int *__ptrauth(1, 0, 64)*)

From a1d52fcdca8bda4fe8c6652b2de83f408f4ad4f2 Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <peter@pcc.me.uk>
Date: Tue, 15 Apr 2025 13:03:58 -0700
Subject: [PATCH 033/710] Clarify documentation for -funique-source-file-names.

Reviewers: efriedma-quic, teresajohnson

Reviewed By: teresajohnson, efriedma-quic

Pull Request: https://github.com/llvm/llvm-project/pull/135832
---
 clang/docs/UsersManual.rst | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index d4656a7e63c99..69256527f40c9 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -2301,11 +2301,31 @@ are listed below.
 
    When enabled, allows the compiler to assume that each object file
    passed to the linker has been compiled using a unique source file
-   name. This is useful for reducing link times when doing ThinLTO
+   path. This is useful for reducing link times when doing ThinLTO
    in combination with whole-program devirtualization or CFI.
 
-   A misuse of this flag will generally result in a duplicate symbol
-   error at link time.
+   The full source path passed to the compiler must be unique. This
+   means that, for example, the following is a usage error:
+
+   .. code-block:: console
+
+     $ cd foo
+     $ clang -funique-source-file-names -c foo.c
+     $ cd ../bar
+     $ clang -funique-source-file-names -c foo.c
+     $ cd ..
+     $ clang foo/foo.o bar/foo.o
+    
+   but this is not:
+
+   .. code-block:: console
+
+     $ clang -funique-source-file-names -c foo/foo.c
+     $ clang -funique-source-file-names -c bar/foo.c
+     $ clang foo/foo.o bar/foo.o
+
+   A misuse of this flag may result in a duplicate symbol error at
+   link time.
 
 .. option:: -fforce-emit-vtables
 

From 2271f0bebd48c9ed8b16b500886a819c4f269a6a Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 15 Apr 2025 13:02:32 -0700
Subject: [PATCH 034/710] [SLP]Check for perfect/shuffled match for the split
 node

If the potential split node is a perfect/shuffled match of another split
node, need to skip creation of the another split node with the same
scalars, it should be a buildvector.

Fixes #135800
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 18 +++++
 .../X86/split-node-full-match.ll              | 74 +++++++++++++++++++
 .../X86/split-node-no-reorder-copy.ll         |  3 +-
 3 files changed, 94 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/split-node-full-match.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 253933a2438cd..234cd340ebc13 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9575,6 +9575,24 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
       !SplitAlternateInstructions)
     return false;
 
+  // Check if this is a duplicate of another split entry.
+  LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *LocalState.getMainOp()
+                    << ".\n");
+  for (TreeEntry *E : getSplitTreeEntries(LocalState.getMainOp())) {
+    if (E->isSame(VL)) {
+      LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at "
+                        << *LocalState.getMainOp() << ".\n");
+      return false;
+    }
+    SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
+    if (all_of(VL, [&](Value *V) {
+          return isa<PoisonValue>(V) || Values.contains(V);
+        })) {
+      LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
+      return false;
+    }
+  }
+
   ReorderIndices.assign(VL.size(), VL.size());
   SmallBitVector Op1Indices(VL.size());
   for (auto [Idx, V] : enumerate(VL)) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-full-match.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-full-match.ll
new file mode 100644
index 0000000000000..10e73b042f19b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-full-match.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(double %0) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: double [[TMP0:%.*]]) {
+; CHECK-NEXT:  [[_THREAD:.*:]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call double null(ptr null, ptr null, ptr null)
+; CHECK-NEXT:    [[TMP2:%.*]] = call double null(ptr null, ptr null, ptr null)
+; CHECK-NEXT:    br i1 false, label %[[BB3:.*]], label %[[BB7:.*]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = call double null(ptr null, ptr null, ptr null)
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double poison, double 0.000000e+00>, double [[TMP0]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP4]], i32 0
+; CHECK-NEXT:    br label %[[BB7]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <4 x double> [ [[TMP6]], %[[BB3]] ], [ zeroinitializer, [[DOTTHREAD:%.*]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x double> [[TMP9]], double [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = call <6 x double> @llvm.vector.insert.v6f64.v2f64(<6 x double> [[TMP11]], <2 x double> [[TMP10]], i64 4)
+; CHECK-NEXT:    br i1 false, label %[[DOTLR_PH272_PREHEADER:.*]], [[DOT_CRIT_EDGE:label %.*]]
+; CHECK:       [[_LR_PH272_PREHEADER:.*:]]
+; CHECK-NEXT:    br i1 false, [[DOT_CRIT_EDGE]], label %[[BB13:.*]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <6 x double> [[TMP12]], <6 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP16:%.*]] = call <6 x double> @llvm.vector.insert.v6f64.v2f64(<6 x double> [[TMP15]], <2 x double> splat (double 0x7FF8000000000000), i64 4)
+; CHECK-NEXT:    br i1 false, label %[[BB17:.*]], [[DOT_CRIT_EDGE]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <6 x double> <double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double poison, double 0.000000e+00, double 0.000000e+00>, double [[TMP0]], i32 3
+; CHECK-NEXT:    br [[DOT_CRIT_EDGE]]
+; CHECK:       [[__CRIT_EDGE:.*:]]
+; CHECK-NEXT:    [[TMP19:%.*]] = phi <6 x double> [ [[TMP12]], %[[BB7]] ], [ [[TMP18]], %[[BB17]] ], [ [[TMP16]], %[[BB13]] ], [ [[TMP12]], %[[DOTLR_PH272_PREHEADER]] ]
+; CHECK-NEXT:    ret void
+;
+.thread:
+  %1 = call double null(ptr null, ptr null, ptr null)
+  %2 = call double null(ptr null, ptr null, ptr null)
+  br i1 false, label %3, label %5
+
+3:
+  %4 = call double null(ptr null, ptr null, ptr null)
+  br label %5
+
+5:
+  %.1226 = phi double [ %4, %3 ], [ 0.000000e+00, %.thread ]
+  %.1222 = phi double [ 0.000000e+00, %3 ], [ 0.000000e+00, %.thread ]
+  %.1218 = phi double [ %0, %3 ], [ 0.000000e+00, %.thread ]
+  %.1216 = phi double [ 0.000000e+00, %3 ], [ 0.000000e+00, %.thread ]
+  br i1 false, label %.lr.ph272.preheader, label %._crit_edge
+
+.lr.ph272.preheader:
+  br i1 false, label %._crit_edge, label %6
+
+6:
+  %7 = fdiv double 0.000000e+00, 0.000000e+00
+  %8 = fsub double 0.000000e+00, %7
+  %9 = fdiv double 0.000000e+00, 0.000000e+00
+  %10 = fsub double 0.000000e+00, %9
+  br i1 false, label %11, label %._crit_edge
+
+11:
+  br label %._crit_edge
+
+._crit_edge:
+  %.2227.lcssa = phi double [ %.1226, %5 ], [ 0.000000e+00, %11 ], [ %.1226, %6 ], [ %.1226, %.lr.ph272.preheader ]
+  %.2223.lcssa = phi double [ %.1222, %5 ], [ 0.000000e+00, %11 ], [ %.1222, %6 ], [ %.1222, %.lr.ph272.preheader ]
+  %.2219.lcssa = phi double [ %.1218, %5 ], [ 0.000000e+00, %11 ], [ %.1218, %6 ], [ %.1218, %.lr.ph272.preheader ]
+  %.2.lcssa = phi double [ %.1216, %5 ], [ %0, %11 ], [ %.1216, %6 ], [ %.1216, %.lr.ph272.preheader ]
+  %.0213.lcssa = phi double [ %2, %5 ], [ 0.000000e+00, %11 ], [ %10, %6 ], [ %2, %.lr.ph272.preheader ]
+  %.0211.lcssa = phi double [ %1, %5 ], [ 0.000000e+00, %11 ], [ %8, %6 ], [ %1, %.lr.ph272.preheader ]
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-no-reorder-copy.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-no-reorder-copy.ll
index b7b6c10137b64..9abb994db1e73 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-no-reorder-copy.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-no-reorder-copy.ll
@@ -15,8 +15,9 @@ define i1 @test(ptr %0, ptr %1, <2 x float> %2, <2 x float> %3, <2 x float> %4)
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> [[TMP3]], <8 x i32> <i32 0, i32 0, i32 2, i32 0, i32 0, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x float> [[TMP14]], float [[TMP9]], i32 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <8 x float> [[TMP13]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP17:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP16]], <8 x float> [[TMP15]], i64 8)
 ; CHECK-NEXT:    [[TMP18:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP16]], <8 x float> [[TMP15]], i64 8)
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <8 x float> [[TMP14]], <8 x float> [[TMP12]], <16 x i32> <i32 8, i32 9, i32 9, i32 9, i32 9, i32 9, i32 14, i32 14, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <16 x float> [[TMP19]], float [[TMP9]], i32 15
 ; CHECK-NEXT:    [[TMP20:%.*]] = fmul <16 x float> [[TMP18]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call reassoc nsz float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[TMP20]])
 ; CHECK-NEXT:    [[TMP22:%.*]] = call float @foo(float [[TMP21]])

From 823adc7a2dc90cdd0f953f3dc9684481368f2b62 Mon Sep 17 00:00:00 2001
From: YongKang Zhu <yongzhu@fb.com>
Date: Tue, 15 Apr 2025 13:19:15 -0700
Subject: [PATCH 035/710] [BOLT] Validate secondary entry point (#135731)

Some functions have their sizes as zero in input binary's symbol
table, like those compiled by assembler. When figuring out function
sizes, we may create label symbol if it doesn't point to any constant
island. However, before function size is known, marker symbol can
not be correctly associated to a function and therefore all such
checks would fail and we could end up adding a code label pointing
to constant island as secondary entry point and later mistakenly
marking the function as not simple.

Querying the global marker symbol array has big throughput overhead.
Instead we can run an extra check when post processing entry points
to identify such label symbols that actually point to constant islands.
---
 bolt/include/bolt/Core/BinaryFunction.h       |  9 +++++
 bolt/lib/Core/BinaryFunction.cpp              |  9 +++++
 .../AArch64/validate-secondary-entry-point.s  | 34 +++++++++++++++++++
 .../RISCV/validate-secondary-entry-point.s    | 34 +++++++++++++++++++
 4 files changed, 86 insertions(+)
 create mode 100644 bolt/test/AArch64/validate-secondary-entry-point.s
 create mode 100644 bolt/test/RISCV/validate-secondary-entry-point.s

diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index d3d11f8c5fb73..a52998564ee1b 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -1174,6 +1174,11 @@ class BinaryFunction {
     return getSecondaryEntryPointSymbol(BB.getLabel());
   }
 
+  /// Remove a label from the secondary entry point map.
+  void removeSymbolFromSecondaryEntryPointMap(const MCSymbol *Label) {
+    SecondaryEntryPoints.erase(Label);
+  }
+
   /// Return true if the basic block is an entry point into the function
   /// (either primary or secondary).
   bool isEntryPoint(const BinaryBasicBlock &BB) const {
@@ -2126,6 +2131,10 @@ class BinaryFunction {
     return Islands && !Islands->DataOffsets.empty();
   }
 
+  bool isStartOfConstantIsland(uint64_t Offset) const {
+    return hasConstantIsland() && Islands->DataOffsets.count(Offset);
+  }
+
   /// Return true iff the symbol could be seen inside this function otherwise
   /// it is probably another function.
   bool isSymbolValidInScope(const SymbolRef &Symbol, uint64_t SymbolSize) const;
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index c4f4d234b30c0..184a4462b356a 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -1896,6 +1896,15 @@ void BinaryFunction::postProcessEntryPoints() {
     if (BC.isAArch64() && Offset == getSize())
       continue;
 
+    // If we have grabbed a wrong code label which actually points to some
+    // constant island inside the function, ignore this label and remove it
+    // from the secondary entry point map.
+    if (isStartOfConstantIsland(Offset)) {
+      BC.SymbolToFunctionMap.erase(Label);
+      removeSymbolFromSecondaryEntryPointMap(Label);
+      continue;
+    }
+
     BC.errs() << "BOLT-WARNING: reference in the middle of instruction "
                  "detected in function "
               << *this << " at offset 0x" << Twine::utohexstr(Offset) << '\n';
diff --git a/bolt/test/AArch64/validate-secondary-entry-point.s b/bolt/test/AArch64/validate-secondary-entry-point.s
new file mode 100644
index 0000000000000..0099a0ee4fe99
--- /dev/null
+++ b/bolt/test/AArch64/validate-secondary-entry-point.s
@@ -0,0 +1,34 @@
+# This test is to verify that BOLT won't take a label pointing to constant
+# island as a secondary entry point (function `_start` doesn't have ELF size
+# set originally) and the function won't otherwise be mistaken as non-simple.
+
+# RUN: %clang %cflags -pie %s -o %t.so -Wl,-q -Wl,--init=_foo -Wl,--fini=_foo
+# RUN: llvm-bolt %t.so -o %t.bolt.so --print-cfg 2>&1 | FileCheck %s
+# CHECK-NOT: BOLT-WARNING: reference in the middle of instruction detected \
+# CHECK-NOT:   function _start at offset 0x{{[0-9a-f]+}}
+# CHECK: Binary Function "_start" after building cfg
+
+  .text
+
+  .global _foo
+  .type _foo, %function
+_foo:
+  ret
+
+  .global _start
+  .type _start, %function
+_start:
+  b _foo
+
+  .balign 16
+_random_consts:
+  .long 0x12345678
+  .long 0x90abcdef
+
+  .global _bar
+  .type _bar, %function
+_bar:
+  ret
+
+  # Dummy relocation to force relocation mode
+  .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/RISCV/validate-secondary-entry-point.s b/bolt/test/RISCV/validate-secondary-entry-point.s
new file mode 100644
index 0000000000000..0c29f5c97c689
--- /dev/null
+++ b/bolt/test/RISCV/validate-secondary-entry-point.s
@@ -0,0 +1,34 @@
+# This test is to verify that BOLT won't take a label pointing to constant
+# island as a secondary entry point (function `_start` doesn't have ELF size
+# set originally) and the function won't otherwise be mistaken as non-simple.
+
+# RUN: %clang %cflags -pie %s -o %t.so -Wl,-q -Wl,--init=_foo -Wl,--fini=_foo
+# RUN: llvm-bolt %t.so -o %t.bolt.so --print-cfg 2>&1 | FileCheck %s
+# CHECK-NOT: BOLT-WARNING: reference in the middle of instruction detected \
+# CHECK-NOT:   function _start at offset 0x{{[0-9a-f]+}}
+# CHECK: Binary Function "_start" after building cfg
+
+  .text
+
+  .global _foo
+  .type _foo, %function
+_foo:
+  ret
+
+  .global _start
+  .type _start, %function
+_start:
+  j _foo
+
+  .balign 16
+_random_consts:
+  .long 0x12345678
+  .long 0x90abcdef
+
+  .global _bar
+  .type _bar, %function
+_bar:
+  ret
+
+  # Dummy relocation to force relocation mode
+  .reloc 0, R_RISCV_NONE

From 14cb6566d6701feaef2ffd686af5de4ff9e3eb29 Mon Sep 17 00:00:00 2001
From: John Harrison <harjohn@google.com>
Date: Tue, 15 Apr 2025 13:30:33 -0700
Subject: [PATCH 036/710] [lldb-dap] Improve error reporting for dap command
 arguments. (#135684)

Previously the error only contained the failed to parse JSON message,
which has no additional context.

This improves the error messages and improves the consistency of
handling properties in protocol structures. Updating the fields to use
'ObjectMapper.map' instead of 'ObjectMapper.mapOptional' caught that
adapterID was misspelled as well.

For example, previously:

```
$ echo 'Content-Length: 81\r\n\r\n{"type":"request","command":"initialize","seq":1,"arguments":{"adapterID":12345}} | lldb-dap
```

Worked without an error but now it reports:

```
invalid arguments for request 'initialize': expected string at arguments.adapterID
{
  "adapterID": /* error: expected string */ 12345
}
```
---
 lldb/tools/lldb-dap/Handler/RequestHandler.h  |  8 ++--
 lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp |  2 +-
 .../lldb-dap/Protocol/ProtocolRequests.cpp    | 42 ++++++++++++-------
 .../lldb-dap/Protocol/ProtocolRequests.h      |  4 +-
 .../tools/lldb-dap/Protocol/ProtocolTypes.cpp |  6 +--
 5 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/lldb/tools/lldb-dap/Handler/RequestHandler.h b/lldb/tools/lldb-dap/Handler/RequestHandler.h
index 50795f8252de3..7e56c258ad78a 100644
--- a/lldb/tools/lldb-dap/Handler/RequestHandler.h
+++ b/lldb/tools/lldb-dap/Handler/RequestHandler.h
@@ -120,11 +120,13 @@ class RequestHandler : public BaseRequestHandler {
     }
 
     Args arguments;
-    llvm::json::Path::Root root;
-    if (request.arguments && !fromJSON(request.arguments, arguments, root)) {
+    llvm::json::Path::Root root("arguments");
+    if (request.arguments && !fromJSON(*request.arguments, arguments, root)) {
       std::string parse_failure;
       llvm::raw_string_ostream OS(parse_failure);
-      root.printErrorContext(request.arguments, OS);
+      OS << "invalid arguments for request '" << request.command
+         << "': " << llvm::toString(root.getError()) << "\n";
+      root.printErrorContext(*request.arguments, OS);
 
       protocol::ErrorMessage error_message;
       error_message.format = parse_failure;
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
index af63cc803e545..bfd68448fb483 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
@@ -178,7 +178,7 @@ bool fromJSON(json::Value const &Params, Response &R, json::Path P) {
     return false;
   }
 
-  return O.map("success", R.success) && O.mapOptional("message", R.message) &&
+  return O.map("success", R.success) && O.map("message", R.message) &&
          mapRaw(Params, "body", R.body, P);
 }
 
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
index 7163399899f7e..3523f8ac87ec9 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
@@ -20,16 +20,16 @@ namespace lldb_dap::protocol {
 bool fromJSON(const llvm::json::Value &Params, CancelArguments &CA,
               llvm::json::Path P) {
   llvm::json::ObjectMapper O(Params, P);
-  return O && O.mapOptional("requestId", CA.requestId) &&
-         O.mapOptional("progressId", CA.progressId);
+  return O && O.map("requestId", CA.requestId) &&
+         O.map("progressId", CA.progressId);
 }
 
 bool fromJSON(const json::Value &Params, DisconnectArguments &DA,
               json::Path P) {
   json::ObjectMapper O(Params, P);
-  return O && O.mapOptional("restart", DA.restart) &&
-         O.mapOptional("terminateDebuggee", DA.terminateDebuggee) &&
-         O.mapOptional("suspendDebuggee", DA.suspendDebuggee);
+  return O && O.map("restart", DA.restart) &&
+         O.map("terminateDebuggee", DA.terminateDebuggee) &&
+         O.map("suspendDebuggee", DA.suspendDebuggee);
 }
 
 bool fromJSON(const llvm::json::Value &Params, PathFormat &PF,
@@ -75,23 +75,33 @@ bool fromJSON(const llvm::json::Value &Params, InitializeRequestArguments &IRA,
 
   const json::Object *O = Params.getAsObject();
 
-  for (auto &kv : ClientFeatureByKey)
-    if (std::optional<bool> v = O->getBoolean(kv.first()); v && *v)
+  for (auto &kv : ClientFeatureByKey) {
+    const json::Value *value_ref = O->get(kv.first());
+    if (!value_ref)
+      continue;
+
+    const std::optional<bool> value = value_ref->getAsBoolean();
+    if (!value) {
+      P.field(kv.first()).report("expected bool");
+      return false;
+    }
+
+    if (*value)
       IRA.supportedFeatures.insert(kv.second);
+  }
 
-  return OM.mapOptional("adatperID", IRA.adatperID) &&
-         OM.mapOptional("clientID", IRA.clientID) &&
-         OM.mapOptional("clientName", IRA.clientName) &&
-         OM.mapOptional("locale", IRA.locale) &&
-         OM.mapOptional("linesStartAt1", IRA.linesStartAt1) &&
-         OM.mapOptional("columnsStartAt1", IRA.columnsStartAt1) &&
-         OM.mapOptional("pathFormat", IRA.pathFormat) &&
-         OM.mapOptional("$__lldb_sourceInitFile", IRA.lldbExtSourceInitFile);
+  return OM.map("adapterID", IRA.adapterID) &&
+         OM.map("clientID", IRA.clientID) &&
+         OM.map("clientName", IRA.clientName) && OM.map("locale", IRA.locale) &&
+         OM.map("linesStartAt1", IRA.linesStartAt1) &&
+         OM.map("columnsStartAt1", IRA.columnsStartAt1) &&
+         OM.map("pathFormat", IRA.pathFormat) &&
+         OM.map("$__lldb_sourceInitFile", IRA.lldbExtSourceInitFile);
 }
 
 bool fromJSON(const json::Value &Params, SourceArguments &SA, json::Path P) {
   json::ObjectMapper O(Params, P);
-  return O && O.mapOptional("source", SA.source) &&
+  return O && O.map("source", SA.source) &&
          O.map("sourceReference", SA.sourceReference);
 }
 
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
index 22d400fd494a5..6623dfa0db05c 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
@@ -100,7 +100,7 @@ enum PathFormat : unsigned { ePatFormatPath, ePathFormatURI };
 /// Arguments for `initialize` request.
 struct InitializeRequestArguments {
   /// The ID of the debug adapter.
-  std::string adatperID;
+  std::string adapterID;
 
   /// The ID of the client using this adapter.
   std::optional<std::string> clientID;
@@ -113,7 +113,7 @@ struct InitializeRequestArguments {
 
   /// Determines in what format paths are specified. The default is `path`,
   /// which is the native format.
-  std::optional<PathFormat> pathFormat = ePatFormatPath;
+  PathFormat pathFormat = ePatFormatPath;
 
   /// If true all line numbers are 1-based (default).
   std::optional<bool> linesStartAt1;
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
index f4f0bf8dcea84..4d1e90215bbb4 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
@@ -38,9 +38,9 @@ bool fromJSON(const json::Value &Params, PresentationHint &PH, json::Path P) {
 
 bool fromJSON(const json::Value &Params, Source &S, json::Path P) {
   json::ObjectMapper O(Params, P);
-  return O && O.mapOptional("name", S.name) && O.mapOptional("path", S.path) &&
-         O.mapOptional("presentationHint", S.presentationHint) &&
-         O.mapOptional("sourceReference", S.sourceReference);
+  return O && O.map("name", S.name) && O.map("path", S.path) &&
+         O.map("presentationHint", S.presentationHint) &&
+         O.map("sourceReference", S.sourceReference);
 }
 
 json::Value toJSON(const ExceptionBreakpointsFilter &EBF) {

From 85eb44e304e0a0a7da78448ceee60fdfec235edb Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 15 Apr 2025 13:29:11 -0700
Subject: [PATCH 037/710] [SLP]Fix number of operands for the split node

FOr the split node number of operands should be requested via
getNumOperands() function, even if the main op is CallInst.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |   2 +
 .../X86/split-node-num-operands.ll            | 121 ++++++++++++++++++
 2 files changed, 123 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/split-node-num-operands.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 234cd340ebc13..b174f0f03fca6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7577,6 +7577,8 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
             return Res.takeVector();
           };
           auto GetNumOperands = [](const TreeEntry *TE) {
+            if (TE->State == TreeEntry::SplitVectorize)
+              return TE->getNumOperands();
             if (auto *CI = dyn_cast<CallInst>(TE->getMainOp()); CI)
               return CI->arg_size();
             return TE->getNumOperands();
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-num-operands.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-num-operands.ll
new file mode 100644
index 0000000000000..5aa4dba2b8a1b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-num-operands.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mattr=+avx -slp-threshold=-1000 < %s | FileCheck %s
+
+define i64 @Foo(ptr align 8 dereferenceable(344) %0, i64 %1) {
+; CHECK-LABEL: define i64 @Foo(
+; CHECK-SAME: ptr align 8 dereferenceable(344) [[TMP0:%.*]], i64 [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP0]], i64 104
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP0]], i64 112
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP0]], i64 24
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i64> <i64 0, i64 poison>, i64 [[TMP1]], i32 1
+; CHECK-NEXT:    br label %[[BB16:.*]]
+; CHECK:       [[BB16]]:
+; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i64> [ [[TMP11]], [[TMP2:%.*]] ], [ zeroinitializer, %[[TMP25:.*]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = phi <2 x i64> [ [[TMP13]], [[TMP2]] ], [ [[TMP29:%.*]], %[[TMP25]] ]
+; CHECK-NEXT:    switch i32 0, label %[[BB19:.*]] [
+; CHECK-NEXT:      i32 0, label %[[TMP25]]
+; CHECK-NEXT:    ]
+; CHECK:       [[BB19]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x i64> [[TMP18]], <2 x i64> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i64> [[TMP20]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <4 x i64> [[TMP21]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i64> [[TMP22]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP14]], <2 x i64> [[TMP18]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    br label %[[TMP25]]
+; CHECK:       [[TMP25]]:
+; CHECK-NEXT:    [[TMP26:%.*]] = phi <2 x i64> [ [[TMP17]], %[[BB19]] ], [ zeroinitializer, %[[BB16]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = phi <4 x i64> [ [[TMP23]], %[[BB19]] ], [ zeroinitializer, %[[BB16]] ]
+; CHECK-NEXT:    [[TMP28:%.*]] = phi <2 x i64> [ [[TMP24]], %[[BB19]] ], [ [[TMP15]], %[[BB16]] ]
+; CHECK-NEXT:    [[TMP29]] = shufflevector <2 x i64> [[TMP18]], <2 x i64> <i64 0, i64 poison>, <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    br i1 false, label %[[DOTLOOPEXIT206:.*]], label %[[BB16]]
+; CHECK:       [[_LOOPEXIT206:.*:]]
+; CHECK-NEXT:    switch i32 0, label %[[BB32:.*]] [
+; CHECK-NEXT:      i32 0, [[DOTCONT174:label %.*]]
+; CHECK-NEXT:      i32 1, label %[[BB30:.*]]
+; CHECK-NEXT:    ]
+; CHECK:       [[BB30]]:
+; CHECK-NEXT:    [[TMP31:%.*]] = shufflevector <4 x i64> [[TMP27]], <4 x i64> <i64 0, i64 0, i64 poison, i64 0>, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    br [[DOTCONT174]]
+; CHECK:       [[BB32]]:
+; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <4 x i64> [[TMP27]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <4 x i64> [[TMP33]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i64> [[TMP34]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <2 x i64> [[TMP28]], i64 0, i32 0
+; CHECK-NEXT:    br [[DOTCONT174]]
+; CHECK:       [[_CONT174:.*:]]
+; CHECK-NEXT:    [[TMP37:%.*]] = phi <2 x i64> [ [[TMP26]], %[[BB32]] ], [ zeroinitializer, %[[BB30]] ], [ [[TMP26]], %[[DOTLOOPEXIT206]] ]
+; CHECK-NEXT:    [[TMP38:%.*]] = phi <4 x i64> [ [[TMP35]], %[[BB32]] ], [ [[TMP31]], %[[BB30]] ], [ [[TMP27]], %[[DOTLOOPEXIT206]] ]
+; CHECK-NEXT:    [[TMP39:%.*]] = phi <2 x i64> [ [[TMP36]], %[[BB32]] ], [ zeroinitializer, %[[BB30]] ], [ [[TMP28]], %[[DOTLOOPEXIT206]] ]
+; CHECK-NEXT:    ret i64 0
+;
+  %3 = getelementptr i8, ptr %0, i64 104
+  %4 = getelementptr i8, ptr %0, i64 112
+  %5 = getelementptr i8, ptr %0, i64 24
+  %6 = load i64, ptr %3, align 8
+  %7 = load i64, ptr %4, align 8
+  %8 = load i64, ptr %5, align 8
+  %9 = load i64, ptr %0, align 8
+  br label %10
+
+10:
+  %11 = phi i64 [ %9, %2 ], [ 0, %18 ]
+  %12 = phi i64 [ %8, %2 ], [ %12, %18 ]
+  %13 = phi i64 [ %7, %2 ], [ 0, %18 ]
+  %14 = phi i64 [ %6, %2 ], [ 0, %18 ]
+  switch i32 0, label %15 [
+  i32 0, label %18
+  ]
+
+15:
+  %16 = tail call i64 @llvm.umin.i64(i64 0, i64 0)
+  %17 = tail call i64 @llvm.umax.i64(i64 0, i64 0)
+  br label %18
+
+18:
+  %19 = phi i64 [ %17, %15 ], [ 0, %10 ]
+  %20 = phi i64 [ %16, %15 ], [ 0, %10 ]
+  %21 = phi i64 [ %11, %15 ], [ 0, %10 ]
+  %22 = phi i64 [ %12, %15 ], [ 0, %10 ]
+  %23 = phi i64 [ %13, %15 ], [ %1, %10 ]
+  %24 = phi i64 [ %14, %15 ], [ 0, %10 ]
+  br i1 false, label %.loopexit206, label %10
+
+.loopexit206:
+  switch i32 0, label %26 [
+  i32 0, label %.cont174
+  i32 1, label %25
+  ]
+
+25:
+  br label %.cont174
+
+26:
+  %27 = tail call i64 @llvm.umin.i64(i64 0, i64 0)
+  %28 = tail call i64 @llvm.umax.i64(i64 0, i64 0)
+  br label %.cont174
+
+.cont174:
+  %.sroa.139.1 = phi i64 [ %28, %26 ], [ %19, %25 ], [ %19, %.loopexit206 ]
+  %.sroa.133.1 = phi i64 [ %27, %26 ], [ 0, %25 ], [ %20, %.loopexit206 ]
+  %.sroa.81.1 = phi i64 [ %23, %26 ], [ 0, %25 ], [ %23, %.loopexit206 ]
+  %.sroa.75.1 = phi i64 [ %24, %26 ], [ 0, %25 ], [ %24, %.loopexit206 ]
+  %.sroa.21.1 = phi i64 [ %21, %26 ], [ 0, %25 ], [ %21, %.loopexit206 ]
+  %.sroa.15.1 = phi i64 [ %22, %26 ], [ 0, %25 ], [ %22, %.loopexit206 ]
+  %29 = phi i64 [ %28, %26 ], [ 0, %25 ], [ %19, %.loopexit206 ]
+  %30 = phi i64 [ %27, %26 ], [ 0, %25 ], [ %20, %.loopexit206 ]
+  ret i64 0
+}
+
+declare i64 @llvm.umax.i64(i64, i64)
+
+declare i64 @llvm.umin.i64(i64, i64)
+

From 2b983a24583dd4e131d727717872a56712b5dd52 Mon Sep 17 00:00:00 2001
From: Zhuoran Yin <zhuoryin@amd.com>
Date: Tue, 15 Apr 2025 16:36:25 -0400
Subject: [PATCH 038/710] [MLIR][AMDGPU] Adding dynamic size check to avoid
 subword buffer load (#135014)

Motivation: amdgpu buffer load instruction will return all zeros when
loading sub-word values. For example, assuming the buffer size is
exactly one word and we attempt to invoke
`llvm.amdgcn.raw.ptr.buffer.load.v2i32` starting from byte 2 of the
word, we will not receive the actual value of the buffer but all zeros
for the first word. This is because the boundary has been crossed for
the first word.

This PR come up with a fix to this problem, such that, it creates a
bounds check against the buffer load instruction. It will compare the
offset + vector size to see if the upper bound of the address will
exceed the buffer size. If it does, masked transfer read will be
optimized to `vector.load` + `arith.select`, else, it will continue to
fall back to default lowering of the masked vector load.
---
 .../mlir/Dialect/AMDGPU/Transforms/Passes.td  |  13 +-
 .../Dialect/AMDGPU/Transforms/CMakeLists.txt  |   1 +
 .../AMDGPU/Transforms/TransferReadToLoad.cpp  | 159 ++++++++++++++++--
 .../Dialect/AMDGPU/transfer-read-to-load.mlir |  94 +++++++++--
 .../llvm-project-overlay/mlir/BUILD.bazel     |   1 +
 5 files changed, 233 insertions(+), 35 deletions(-)

diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
index 761caa448a57c..0e858108acf35 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
@@ -54,15 +54,20 @@ def AmdgpuResolveStridedMetadataPass : Pass<"amdgpu-resolve-strided-metadata"> {
 def AmdgpuTransferReadToLoadPass : Pass<"amdgpu-transfer-read-to-load"> {
   let summary = "Lower the operations from the vector transfer_read to vector load";
   let description = [{
-    This pass creates a transfer read op lowering. A vector trasfer read op
-    will be lowered to a combination of vector.load, arith.select and
-    vector.broadcast.
+    This pass creates a transfer read op lowering optimization. The lowering
+    will produce a conditional check at runtime. If within bounds, a vector
+    trasfer read op will be lowered to a combination of vector.load, arith.select
+    and vector.broadcast. If not, it will fallback to the default lowering
+    of the transfer_read op.
 
     This pattern will make it possible for masked transfer_read to be lowered
     towards buffer load with bounds check, allowing a more optimized global
     load accessing pattern compared with existing implementation of
     llvm.intr.masked.load on vectors.
   }];
-  let dependentDialects = [];
+  let dependentDialects = [
+    "scf::SCFDialect",
+    "memref::MemRefDialect"
+  ];
 }
 #endif // MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_TD_
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
index bc5b6e9186449..8709a27e0168e 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
@@ -14,6 +14,7 @@ add_mlir_dialect_library(MLIRAMDGPUTransforms
   MLIRAMDGPUUtils
   MLIRArithDialect
   MLIRMemRefDialect
+  MLIRSCFDialect
   MLIRVectorDialect
   MLIRControlFlowDialect
   MLIRFuncDialect
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
index 3c1a2eb962037..f665c1794cdd4 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
@@ -9,13 +9,22 @@
 #include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
 
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Arith/Utils/Utils.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/LogicalResult.h"
-#include "mlir/Transforms/WalkPatternRewriteDriver.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/Support/MathExtras.h"
 
 namespace mlir::amdgpu {
 #define GEN_PASS_DEF_AMDGPUTRANSFERREADTOLOADPASS
@@ -67,6 +76,9 @@ static LogicalResult transferPreconditions(
   if (!memRefType.isLastDimUnitStride())
     return rewriter.notifyMatchFailure(xferOp, "!= 1 stride needs VectorToSCF");
 
+  if (memRefType.getElementTypeBitWidth() < 8)
+    return rewriter.notifyMatchFailure(xferOp, "unsupported sub-byte type");
+
   // If there is broadcasting involved then we first load the unbroadcasted
   // vector, and then broadcast it with `vector.broadcast`.
   ArrayRef<int64_t> vectorShape = xferOp.getVectorType().getShape();
@@ -101,6 +113,26 @@ static LogicalResult transferPreconditions(
   return success();
 }
 
+static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
+                                           vector::TransferReadOp readOp,
+                                           bool requiresBroadcasting,
+                                           VectorType unbroadcastedVectorType) {
+  Value fill = builder.create<vector::SplatOp>(loc, unbroadcastedVectorType,
+                                               readOp.getPadding());
+  Value load = builder.create<vector::LoadOp>(
+      loc, unbroadcastedVectorType, readOp.getSource(), readOp.getIndices());
+  Value res = builder.create<arith::SelectOp>(loc, unbroadcastedVectorType,
+                                              readOp.getMask(), load, fill);
+  // Insert a broadcasting op if required.
+  if (requiresBroadcasting) {
+    res = builder.create<vector::BroadcastOp>(loc, readOp.getVectorType(), res);
+  }
+  return res;
+}
+
+static constexpr char kTransferReadNeedsMask[] =
+    "amdgpu.buffer_transfer_read_needs_mask";
+
 namespace {
 
 struct TransferReadLowering final : OpRewritePattern<vector::TransferReadOp> {
@@ -108,6 +140,8 @@ struct TransferReadLowering final : OpRewritePattern<vector::TransferReadOp> {
 
   LogicalResult matchAndRewrite(vector::TransferReadOp readOp,
                                 PatternRewriter &rewriter) const override {
+    if (readOp->hasAttr(kTransferReadNeedsMask))
+      return failure();
 
     bool requiresBroadcasting = false;
     VectorType unbroadcastedVectorType;
@@ -117,20 +151,115 @@ struct TransferReadLowering final : OpRewritePattern<vector::TransferReadOp> {
     }
 
     Location loc = readOp.getLoc();
-    Value fill = rewriter.create<vector::SplatOp>(loc, unbroadcastedVectorType,
-                                                  readOp.getPadding());
-    Value load = rewriter.create<vector::LoadOp>(
-        loc, unbroadcastedVectorType, readOp.getSource(), readOp.getIndices());
-    Value res = rewriter.create<arith::SelectOp>(loc, unbroadcastedVectorType,
-                                                 readOp.getMask(), load, fill);
-
-    // Insert a broadcasting op if required.
-    if (requiresBroadcasting) {
-      res = rewriter.create<vector::BroadcastOp>(loc, readOp.getVectorType(),
-                                                 res);
+    Value src = readOp.getSource();
+
+    VectorType vectorType = readOp.getVectorType();
+    int64_t vectorSize = vectorType.getNumElements();
+    int64_t elementBitWidth = vectorType.getElementTypeBitWidth();
+    SmallVector<OpFoldResult> indices = readOp.getIndices();
+
+    auto stridedMetadata =
+        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
+    SmallVector<OpFoldResult> strides =
+        stridedMetadata.getConstifiedMixedStrides();
+    SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
+    OpFoldResult offset = stridedMetadata.getConstifiedMixedOffset();
+    OpFoldResult linearizedIndices;
+    std::tie(std::ignore, linearizedIndices) =
+        memref::getLinearizedMemRefOffsetAndSize(rewriter, loc, elementBitWidth,
+                                                 elementBitWidth, offset, sizes,
+                                                 strides, indices);
+
+    // TODO(jerryyin): Fix the getLinearizedMemRefOffsetAndSize() function
+    // Note below doesn't give the correct result for the linearized size.
+    // Value totalSize = getValueOrCreateConstantIndexOp(
+    //    rewriter, loc, linearizedInfo.linearizedSize);
+    // It computes the multiplied sizes of all dimensions instead of taking
+    // the maximum of each dimension size * stride.
+    SmallVector<AffineExpr> productExpressions;
+    SmallVector<Value> productResults;
+    unsigned sourceRank = cast<ShapedType>(src.getType()).getRank();
+
+    SmallVector<AffineExpr> symbols(2 * sourceRank);
+    SmallVector<Value> offsetValues;
+    bindSymbolsList(rewriter.getContext(), MutableArrayRef{symbols});
+
+    size_t symbolIndex = 0;
+    for (size_t i = 0; i < sourceRank; ++i) {
+      AffineExpr strideExpr, sizeExpr;
+      OpFoldResult stride = strides[i];
+      OpFoldResult size = sizes[i];
+      if (auto constantStride = getConstantIntValue(stride)) {
+        strideExpr = rewriter.getAffineConstantExpr(*constantStride);
+      } else {
+        strideExpr = symbols[symbolIndex++];
+        offsetValues.push_back(
+            getValueOrCreateConstantIndexOp(rewriter, loc, stride));
+      }
+
+      if (auto constantSize = getConstantIntValue(size)) {
+        sizeExpr = rewriter.getAffineConstantExpr(*constantSize);
+      } else {
+        sizeExpr = symbols[symbolIndex++];
+        offsetValues.push_back(
+            getValueOrCreateConstantIndexOp(rewriter, loc, size));
+      }
+
+      productExpressions.push_back(strideExpr * sizeExpr);
     }
 
-    rewriter.replaceOp(readOp, res);
+    AffineMap maxMap = AffineMap::get(
+        /*dimCount=*/0, /*symbolCount=*/symbolIndex, productExpressions,
+        rewriter.getContext());
+    Value totalSize =
+        rewriter.create<affine::AffineMaxOp>(loc, maxMap, offsetValues);
+
+    // delta = bufferSize - linearizedOffset
+    Value vectorSizeOffset =
+        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
+    Value linearIndex =
+        getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
+    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
+
+    // 1) check if delta < vectorSize
+    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
+
+    // 2) check if (detla_bytes % (32 / elementBitwidth) != 0)
+    Value deltaBytes = rewriter.create<arith::MulIOp>(
+        loc, delta,
+        rewriter.create<arith::ConstantIndexOp>(loc, elementBitWidth / 8));
+    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
+        loc, llvm::divideCeil(32, elementBitWidth));
+    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::ne,
+        rewriter.create<arith::RemUIOp>(loc, deltaBytes, elementsPerWord),
+        rewriter.create<arith::ConstantIndexOp>(loc, 0));
+
+    // We take the fallback of transfer_read default lowering only it is both
+    // out-of-bounds and not word aligned. The fallback ensures correct results
+    // when loading at the boundary of the buffer since buffer load returns
+    // inconsistent zeros for the whole word when boundary is crossed.
+    Value ifCondition =
+        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
+
+    auto thenBuilder = [&](OpBuilder &builder, Location loc) {
+      Operation *read = builder.clone(*readOp.getOperation());
+      read->setAttr(kTransferReadNeedsMask, builder.getUnitAttr());
+      Value readResult = read->getResult(0);
+      builder.create<scf::YieldOp>(loc, readResult);
+    };
+
+    auto elseBuilder = [&](OpBuilder &builder, Location loc) {
+      Value res = createVectorLoadForMaskedLoad(
+          builder, loc, readOp, requiresBroadcasting, unbroadcastedVectorType);
+      rewriter.create<scf::YieldOp>(loc, res);
+    };
+
+    auto ifOp =
+        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
+
+    rewriter.replaceOp(readOp, ifOp);
 
     return success();
   }
@@ -149,6 +278,8 @@ struct AmdgpuTransferReadToLoadPass final
   void runOnOperation() override {
     RewritePatternSet patterns(&getContext());
     populateAmdgpuTransferReadToLoadPatterns(patterns);
-    walkAndApplyPatterns(getOperation(), std::move(patterns));
+    if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) {
+      return signalPassFailure();
+    }
   }
 };
diff --git a/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir b/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
index 3e1283579f2b1..d0805b6b8a973 100644
--- a/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
+++ b/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
@@ -9,11 +9,71 @@ func.func @transfer_to_maskedload_fatrawbuffer(%mem : memref<8x8xf32, #amdgpu.ad
   %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf32>
   return %res : vector<4xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.0
-// CHECK: %[[SPLAT:.*]] = vector.splat %[[CST]]
+
+// CHECK: %[[FALSE:.*]] = arith.constant false
+// CHECK: %[[IF:.*]] = scf.if %[[FALSE]] -> (vector<4xf32>) {
+// CHECK: vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]]
+
+// CHECK: } else {
 // CHECK: %[[LOAD:.*]] = vector.load %arg0[%arg1, %arg1]
-// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[SPLAT]]
-// CHECK: return %[[SELECT]] : vector<4xf32>
+// CHECK: %[[SELECT:.*]] = arith.select %[[ARG2]], %[[LOAD]]
+
+// CHECK: return %[[IF]] : vector<4xf32>
+
+// -----
+
+// CHECK: #map = affine_map<()[s0, s1] -> (s0 * 8 + s1)>
+// CHECK-LABEL: func @transfer_to_maskedload_fatrawbuffer_f16(
+// CHECK-SAME: %[[ARG0:.+]]: memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>,
+// CHECK-SAME: %[[ARG1:.+]]: index, %[[ARG2:.+]]: index,
+// CHECK-SAME: %[[ARG3:.+]]: vector<4xi1>)
+func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>) -> vector<4xf16> {
+  %cf0 = arith.constant 0.0 : f16
+  %res = vector.transfer_read %mem[%idx0, %idx1], %cf0, %mask {in_bounds = [true]} : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf16>
+  return %res : vector<4xf16>
+}
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0
+// CHECK-DAG: %[[SIZE:.*]] = arith.constant 64
+// CHECK-DAG: %[[BYTES:.*]] = arith.constant 2
+// CHECK-DAG: %[[VECTORSIZE:.*]] = arith.constant 4
+
+// CHECK: %[[LINEAR:.*]] = affine.apply #map()[%[[ARG1]], %[[ARG2]]]
+// CHECK: %[[DELTA:.*]] = arith.subi %[[SIZE]], %[[LINEAR]]
+// CHECK: %[[COND1:.*]] = arith.cmpi ult, %[[DELTA]], %[[VECTORSIZE]]
+
+// CHECK: %[[DELTABYTES:.*]] = arith.muli %[[DELTA]], %[[BYTES]]
+// CHECK: %[[REM:.*]] = arith.remui %[[DELTABYTES]], %[[BYTES]]
+// CHECK: %[[COND2:.*]] = arith.cmpi ne, %[[REM]], %[[C0]]
+
+// CHECK: %[[COND:.*]] = arith.andi %[[COND1]], %[[COND2]]
+// CHECK: %[[IF:.*]] = scf.if %[[COND]] -> (vector<4xf16>) {
+// CHECK: vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG2]]]
+// CHECK: } else {
+// CHECK: %[[LOAD:.*]] = vector.load %[[ARG0]][%[[ARG1]], %[[ARG2]]]
+// CHECK: return %[[IF]] : vector<4xf16>
+
+// -----
+
+// CHECK: #map = affine_map<()[s0, s1, s2] -> (s0 * s1 + s2)>
+// CHECK: #map1 = affine_map<()[s0, s1, s2] -> (s0 * s1, s2)>
+// CHECK-LABEL: func @transfer_to_maskedload_fatrawbuffer_dynamic_i8(
+// CHECK-SAME: %[[ARG0:.*]]: memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>
+// CHECK-SAME: %[[ARG1:.*]]: index, %[[ARG2:.*]]: index
+// CHECK-SAME: %[[ARG3:.*]]: vector<4xi1>
+func.func @transfer_to_maskedload_fatrawbuffer_dynamic_i8(%mem : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>) -> vector<4xi8> {
+  %cf0 = arith.constant 0 : i8
+  %res = vector.transfer_read %mem[%idx0, %idx1], %cf0, %mask {in_bounds = [true]} : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi8>
+  return %res : vector<4xi8>
+}
+
+// CHECK: %[[CST:.*]] = arith.constant dense<0> : vector<4xi8>
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[C4:.*]] = arith.constant 4 : index
+// CHECK: %[[BASE:.*]], %[[OFFSET:.*]], %[[SIZES:.*]]:2, %[[STRIDES:.*]]:2 = memref.extract_strided_metadata %[[ARG0]]
+// CHECK: %[[LINEAR:.*]] = affine.apply #map()[%[[ARG1]], %[[STRIDES]]#0, %[[ARG2]]]
+// CHECK: %[[SIZE:.*]] = affine.max #map1()[%[[STRIDES]]#0, %[[SIZES]]#0, %[[SIZES]]#1]
+// CHECK: %[[IF:.*]] = scf.if
+// CHECK: return
 
 // -----
 
@@ -26,8 +86,8 @@ func.func @transfer_to_maskedload_regular(%mem : memref<8x8xf32>, %idx : index,
   %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32>, vector<4xf32>
   return %res : vector<4xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.0
-// CHECK: %[[RES:.*]] = vector.transfer_read %arg0[%arg1, %arg1], %[[CST]], %arg2 {in_bounds = [true]} : memref<8x8xf32>, vector<4xf32>
+// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
+// CHECK: %[[RES:.*]] = vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[CST]], %[[ARG2]]
 // CHECK: return %[[RES]] : vector<4xf32>
 
 // -----
@@ -41,8 +101,8 @@ func.func @transfer_to_maskedload_addrspace(%mem : memref<8x8xf32, #gpu.address_
   %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32, #gpu.address_space<workgroup>>, vector<4xf32>
   return %res : vector<4xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.0
-// CHECK: %[[RES:.*]] = vector.transfer_read %arg0[%arg1, %arg1], %[[CST]], %arg2 {in_bounds = [true]} : memref<8x8xf32, #gpu.address_space<workgroup>>, vector<4xf32>
+// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
+// CHECK: %[[RES:.*]] = vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[CST]], %[[ARG2]]
 // CHECK: return %[[RES]] : vector<4xf32>
 
 // -----
@@ -59,12 +119,12 @@ func.func @transfer_broadcasting(%mem : memref<8x8xf32, #amdgpu.address_space<fa
       : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf32>
   return %res : vector<4xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.0
-// CHECK: %[[SPLAT:.*]] = vector.splat %[[CST]]
+// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
+// CHECK: %[[FALSE:.*]] = arith.constant false
+// CHECK: %[[IF:.*]] = scf.if %[[FALSE]] -> (vector<4xf32>) {
 // CHECK: %[[LOAD:.*]] = vector.load %arg0[%arg1, %arg1]
-// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[SPLAT]]
+// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[CST]]
 // CHECK: %[[BROADCAST:.*]] = vector.broadcast %[[SELECT]] : vector<1xf32> to vector<4xf32>
-// CHECK: return %[[BROADCAST]] : vector<4xf32>
 
 // -----
 
@@ -79,8 +139,8 @@ func.func @transfer_scalar(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_
       : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<1xf32>
   return %res : vector<1xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.0
-// CHECK: %[[SPLAT:.*]] = vector.splat %[[CST]]
-// CHECK: %[[LOAD:.*]] = vector.load %arg0[%arg1, %arg1]
-// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[SPLAT]]
-// CHECK: return %[[SELECT]] : vector<1xf32>
+// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
+// CHECK: %[[FALSE:.*]] = arith.constant false
+// CHECK: %[[IF:.*]] = scf.if %[[FALSE]] -> (vector<1xf32>) {
+// CHECK: %[[LOAD:.*]] = vector.load %[[ARG0]][%[[ARG1]], %[[ARG1]]]
+// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[CST]]
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 07c522a237a8a..10503fe1d123b 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -1569,6 +1569,7 @@ cc_library(
         ":IR",
         ":MemRefDialect",
         ":Pass",
+        ":SCFDialect",
         ":SideEffectInterfaces",
         ":Support",
         ":TransformUtils",

From f83c5fe01fbee0f53ecf69d887e7a7b054f2a9ae Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 15 Apr 2025 13:55:40 -0700
Subject: [PATCH 039/710] [nfc] Expose `canReturn` from FunctionAttrs (#135650)

This is a fairly light-weight traversal and is needed in instrumentation. No need to run the whole `FunctionAttrs` pass at this stage. To avoid layering issues, this patch factors `canRun` and related under Analysis/CFG.
---
 llvm/include/llvm/Analysis/CFG.h          |  1 +
 llvm/lib/Analysis/CFG.cpp                 | 34 ++++++++++++++++++++++
 llvm/lib/Transforms/IPO/FunctionAttrs.cpp | 35 -----------------------
 3 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/llvm/include/llvm/Analysis/CFG.h b/llvm/include/llvm/Analysis/CFG.h
index 23bc10a4a9d1b..8451e88146d7c 100644
--- a/llvm/include/llvm/Analysis/CFG.h
+++ b/llvm/include/llvm/Analysis/CFG.h
@@ -174,6 +174,7 @@ bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI) {
 
   return false;
 }
+bool canReturn(const Function &F);
 } // End llvm namespace
 
 #endif
diff --git a/llvm/lib/Analysis/CFG.cpp b/llvm/lib/Analysis/CFG.cpp
index 841b835052380..8ced4a901557d 100644
--- a/llvm/lib/Analysis/CFG.cpp
+++ b/llvm/lib/Analysis/CFG.cpp
@@ -322,3 +322,37 @@ bool llvm::isPotentiallyReachable(
   return isPotentiallyReachable(
       A->getParent(), B->getParent(), ExclusionSet, DT, LI);
 }
+
+static bool instructionDoesNotReturn(const Instruction &I) {
+  if (auto *CB = dyn_cast<CallBase>(&I))
+    return CB->hasFnAttr(Attribute::NoReturn);
+  return false;
+}
+
+// A basic block can only return if it terminates with a ReturnInst and does not
+// contain calls to noreturn functions.
+static bool basicBlockCanReturn(const BasicBlock &BB) {
+  if (!isa<ReturnInst>(BB.getTerminator()))
+    return false;
+  return none_of(BB, instructionDoesNotReturn);
+}
+
+// FIXME: this doesn't handle recursion.
+bool llvm::canReturn(const Function &F) {
+  SmallVector<const BasicBlock *, 16> Worklist;
+  SmallPtrSet<const BasicBlock *, 16> Visited;
+
+  Visited.insert(&F.front());
+  Worklist.push_back(&F.front());
+
+  do {
+    const BasicBlock *BB = Worklist.pop_back_val();
+    if (basicBlockCanReturn(*BB))
+      return true;
+    for (const BasicBlock *Succ : successors(BB))
+      if (Visited.insert(Succ).second)
+        Worklist.push_back(Succ);
+  } while (!Worklist.empty());
+
+  return false;
+}
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index ef7989507c89f..bbfed2ac2c090 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -2090,41 +2090,6 @@ static void addNoRecurseAttrs(const SCCNodeSet &SCCNodes,
   Changed.insert(F);
 }
 
-static bool instructionDoesNotReturn(Instruction &I) {
-  if (auto *CB = dyn_cast<CallBase>(&I))
-    return CB->hasFnAttr(Attribute::NoReturn);
-  return false;
-}
-
-// A basic block can only return if it terminates with a ReturnInst and does not
-// contain calls to noreturn functions.
-static bool basicBlockCanReturn(BasicBlock &BB) {
-  if (!isa<ReturnInst>(BB.getTerminator()))
-    return false;
-  return none_of(BB, instructionDoesNotReturn);
-}
-
-// FIXME: this doesn't handle recursion.
-static bool canReturn(Function &F) {
-  SmallVector<BasicBlock *, 16> Worklist;
-  SmallPtrSet<BasicBlock *, 16> Visited;
-
-  Visited.insert(&F.front());
-  Worklist.push_back(&F.front());
-
-  do {
-    BasicBlock *BB = Worklist.pop_back_val();
-    if (basicBlockCanReturn(*BB))
-      return true;
-    for (BasicBlock *Succ : successors(BB))
-      if (Visited.insert(Succ).second)
-        Worklist.push_back(Succ);
-  } while (!Worklist.empty());
-
-  return false;
-}
-
-
 // Set the noreturn function attribute if possible.
 static void addNoReturnAttrs(const SCCNodeSet &SCCNodes,
                              SmallSet<Function *, 8> &Changed) {

From 12697c5516f8a9e4407e01d99324ce6958910184 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 15 Apr 2025 13:56:07 -0700
Subject: [PATCH 040/710] [LegalizeTypes] Check getTypeAction before calling
 GetScalarizedVector. (#135838)

Use getTypeAction instead of trying to guess how a type will be
legalized. On AArch64, v1f16 is scalarized but v1f16 is widened.

Fixes #135776
---
 .../SelectionDAG/LegalizeVectorTypes.cpp        |  4 +---
 llvm/test/CodeGen/AArch64/pr135776.ll           | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/pr135776.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index f934d8b37561e..a01e1cff74564 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -400,9 +400,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
   SDValue Op = N->getOperand(0);
-  if (Op.getValueType().isVector()
-      && Op.getValueType().getVectorNumElements() == 1
-      && !isSimpleLegalType(Op.getValueType()))
+  if (getTypeAction(Op.getValueType()) == TargetLowering::TypeScalarizeVector)
     Op = GetScalarizedVector(Op);
   EVT NewVT = N->getValueType(0).getVectorElementType();
   return DAG.getNode(ISD::BITCAST, SDLoc(N),
diff --git a/llvm/test/CodeGen/AArch64/pr135776.ll b/llvm/test/CodeGen/AArch64/pr135776.ll
new file mode 100644
index 0000000000000..6f026234664fe
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/pr135776.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+
+define i32 @bitcast_failure(ptr %0, <1 x i16> %1) {
+; CHECK-LABEL: bitcast_failure:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, x0
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str h0, [x8]
+; CHECK-NEXT:    ret
+  %3 = bitcast <1 x i16> %1 to <1 x half>
+  %4 = extractelement <1 x half> %3, i64 0
+  store half %4, ptr %0, align 2
+  ret i32 0
+}

From ddb12674300eb1af5e6945b5447e7bff7cff4cd8 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 15 Apr 2025 13:53:16 -0700
Subject: [PATCH 041/710] [SLP]Insert vector instruction after landingpad

If the node must be emitted in the landingpad block, need to insert the
instructions after the landingpad instruction to avoid a crash.

Fixes #135781
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  6 +-
 .../X86/landing-pad-for-split-node.ll         | 77 +++++++++++++++++++
 2 files changed, 82 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/landing-pad-for-split-node.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b174f0f03fca6..b48674f6993e3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -16010,8 +16010,12 @@ void BoUpSLP::setInsertPointAfterBundle(const TreeEntry *E) {
   BasicBlock::iterator LastInstIt = LastInst->getIterator();
   // If the instruction is PHI, set the insert point after all the PHIs.
   bool IsPHI = isa<PHINode>(LastInst);
-  if (IsPHI)
+  if (IsPHI) {
     LastInstIt = LastInst->getParent()->getFirstNonPHIIt();
+    if (LastInstIt != LastInst->getParent()->end() &&
+        LastInstIt->getParent()->isLandingPad())
+      LastInstIt = std::next(LastInstIt);
+  }
   if (IsPHI ||
       (!E->isGather() && E->State != TreeEntry::SplitVectorize &&
        doesNotNeedToSchedule(E->Scalars)) ||
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/landing-pad-for-split-node.ll b/llvm/test/Transforms/SLPVectorizer/X86/landing-pad-for-split-node.ll
new file mode 100644
index 0000000000000..d6552adbd4abf
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/landing-pad-for-split-node.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-9999 < %s | FileCheck %s
+
+define void @test(i32 %arg) personality ptr null {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i32 [[ARG:%.*]]) personality ptr null {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(3) null, align 4
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(3) null, align 4
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i32, ptr addrspace(3) null, align 4
+; CHECK-NEXT:    [[LOAD3:%.*]] = load i32, ptr addrspace(3) null, align 4
+; CHECK-NEXT:    [[INVOKE:%.*]] = invoke i32 null(ptr addrspace(1) null, i32 0)
+; CHECK-NEXT:            to label %[[BB4:.*]] unwind label %[[BB5:.*]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, %[[BB]] ]
+; CHECK-NEXT:    [[PHI6:%.*]] = phi i32 [ 0, %[[BB]] ]
+; CHECK-NEXT:    [[PHI7:%.*]] = phi i32 [ 0, %[[BB]] ]
+; CHECK-NEXT:    [[PHI8:%.*]] = phi i32 [ 0, %[[BB]] ]
+; CHECK-NEXT:    [[LANDINGPAD:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:            cleanup
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[LOAD]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[LOAD1]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[LOAD3]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[LOAD2]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> poison, i32 [[PHI]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[PHI8]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[PHI6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[PHI7]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP7]], i64 4)
+; CHECK-NEXT:    br label %[[BB11:.*]]
+; CHECK:       [[BB9:.*]]:
+; CHECK-NEXT:    [[LANDINGPAD10:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:            cleanup
+; CHECK-NEXT:    br label %[[BB11]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi <8 x i32> [ poison, %[[BB9]] ], [ [[TMP9]], %[[BB5]] ]
+; CHECK-NEXT:    ret void
+;
+bb:
+  %load = load i32, ptr addrspace(3) null, align 4
+  %load1 = load i32, ptr addrspace(3) null, align 4
+  %load2 = load i32, ptr addrspace(3) null, align 4
+  %load3 = load i32, ptr addrspace(3) null, align 4
+  %invoke = invoke i32 null(ptr addrspace(1) null, i32 0)
+  to label %bb4 unwind label %bb5
+
+bb4:                                              ; preds = %bb
+  ret void
+
+bb5:                                              ; preds = %bb
+  %phi = phi i32 [ 0, %bb ]
+  %phi6 = phi i32 [ 0, %bb ]
+  %phi7 = phi i32 [ 0, %bb ]
+  %phi8 = phi i32 [ 0, %bb ]
+  %landingpad = landingpad { ptr, i32 }
+  cleanup
+  br label %bb11
+
+bb9:                                              ; No predecessors!
+  %landingpad10 = landingpad { ptr, i32 }
+  cleanup
+  br label %bb11
+
+bb11:                                             ; preds = %bb9, %bb5
+  %phi12 = phi i32 [ 0, %bb9 ], [ %phi, %bb5 ]
+  %phi13 = phi i32 [ 0, %bb9 ], [ %phi8, %bb5 ]
+  %phi14 = phi i32 [ 0, %bb9 ], [ %phi6, %bb5 ]
+  %phi15 = phi i32 [ %arg, %bb9 ], [ %phi7, %bb5 ]
+  %phi16 = phi i32 [ 0, %bb9 ], [ %load, %bb5 ]
+  %phi17 = phi i32 [ 0, %bb9 ], [ %load1, %bb5 ]
+  %phi18 = phi i32 [ %arg, %bb9 ], [ %load2, %bb5 ]
+  %phi19 = phi i32 [ 0, %bb9 ], [ %load3, %bb5 ]
+  ret void
+}

From 0f3e460e06e03ce37445546457a16d6f1eee1e21 Mon Sep 17 00:00:00 2001
From: MaheshRavishankar <1663364+MaheshRavishankar@users.noreply.github.com>
Date: Tue, 15 Apr 2025 14:10:18 -0700
Subject: [PATCH 042/710] [mlir][Tensor] Generalize the pattern to swap
 `tensor.collapse_shape` -> `tensor.expand_shape`. (#133819)

The current patterns compared the reassocation indices for the two ops
and failed if neither of them were of size 1. This patch relaxes this
restriction by handling a new case where the reassociation indices might
be of the same size.

Also generalizes to cases where when generating the swapped
`tensor.expand_shape` -> `tensor.collapse_shape` if one of them is
degenerate, those are not generated.

Signed-off-by: MaheshRavishankar <mahesh.ravishankar@gmail.com>
---
 .../Tensor/Transforms/ReshapePatterns.cpp     | 112 ++++++++++++++----
 mlir/test/Dialect/Tensor/bubble-reshapes.mlir |  63 +++++++++-
 2 files changed, 149 insertions(+), 26 deletions(-)

diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index eed44e60d6591..a3de7f9b44ae6 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -167,10 +167,39 @@ struct BubbleUpExpandThroughParallelCollapse
       return failure();
     }
 
-    // Reshapes are parallel to each other if none of the reassociation indices
-    // have greater than 1 index for both reshapes.
+    // Reshapes are parallel to each other (by construction the number of
+    // reassociations specified in the collapse and expand are the same), if at
+    // any position
+    // 1. either the reassociation indices are of the same size, or
+    // 2. either the reassociation in the collapse or the expand is of size 1.
+    ArrayRef<int64_t> staticSourceSize = collapseOp.getSrcType().getShape();
+    ArrayRef<int64_t> staticResultSize = expandOp.getStaticOutputShape();
     for (auto [expandReassociation, collapseReassociation] :
          llvm::zip_equal(expandReInds, collapseReInds)) {
+      if (collapseReassociation.size() == expandReassociation.size()) {
+        // Even if the reassociations are the same, the collapse/expand should
+        // result in the same dimensions. i.e  4x8x2 into 64 should be expanded
+        // into 4x8x2 again. In presense of dynamic dimensions one can only
+        // verify "equality" when there is only one dynamic dimension present,
+        // and all other static dimensions are equal.
+        ArrayRef<int64_t> collapsedStaticShapes = staticSourceSize.slice(
+            collapseReassociation.front(), collapseReassociation.size());
+        int64_t numCollapsedDynamic =
+            llvm::count_if(collapsedStaticShapes,
+                           [](int64_t d) { return ShapedType::isDynamic(d); });
+        ArrayRef<int64_t> expandedStaticShapes = staticResultSize.slice(
+            expandReassociation.front(), expandReassociation.size());
+        int64_t numExpandedDynamic =
+            llvm::count_if(expandedStaticShapes,
+                           [](int64_t d) { return ShapedType::isDynamic(d); });
+        if (numCollapsedDynamic > 1 || numExpandedDynamic > 1 ||
+            collapsedStaticShapes != expandedStaticShapes) {
+          return failure();
+        }
+        continue;
+      }
+      // If the reassociations are not same, one or the other needs to be of
+      // size one.
       if (collapseReassociation.size() != 1 && expandReassociation.size() != 1)
         return failure();
     }
@@ -178,33 +207,60 @@ struct BubbleUpExpandThroughParallelCollapse
     // Compute new reassociation indices and expanded/collaped shapes.
     SmallVector<ReassociationIndices> newExpandReInds, newCollapseReInds;
     Location loc = expandOp->getLoc();
-    SmallVector<OpFoldResult> collapseSizes =
+    SmallVector<OpFoldResult> sourceSizes =
         tensor::getMixedSizes(rewriter, loc, collapseOp.getSrc());
-    SmallVector<OpFoldResult> expandSizes(getMixedValues(
-        expandOp.getStaticOutputShape(), expandOp.getOutputShape(), rewriter));
+    SmallVector<OpFoldResult> resultSizes = expandOp.getMixedOutputShape();
     SmallVector<OpFoldResult> newExpandSizes;
-    int64_t index = 0, expandIndex = 0, collapseIndex = 0;
-    for (auto [idx, collapseReassociation] : llvm::enumerate(collapseReInds)) {
+
+    int64_t newExpandIndex = 0, newCollapseIndex = 0, sourceSizeIndex = 0,
+            resultSizeIndex = 0;
+
+    for (size_t idx = 0, idxEnd = collapseReInds.size(); idx < idxEnd; idx++) {
+      auto &collapseReassociation = collapseReInds[idx];
+      auto &expandReassociation = expandReInds[idx];
+
+      // Case 1. The reassociations are same in the collapse producer
+      // and expand consumer. In the swapped expand, each of the final
+      // dimensions are kept as is in the expand and the collapse. So,
+      // for every element in the `ReassocationIndices` vector add a new
+      // `ReassociationIndices` vector for the swapped expand and collapse
+      // (of size 1).
+      if (collapseReassociation.size() == expandReassociation.size()) {
+        for (size_t i = 0; i < collapseReassociation.size(); ++i) {
+          newCollapseReInds.push_back({newCollapseIndex++});
+          newExpandReInds.push_back({newExpandIndex++});
+          newExpandSizes.push_back(resultSizes[resultSizeIndex++]);
+          sourceSizeIndex++;
+        }
+        continue;
+      }
+
+      // Case 2. The `ReassociationIndices` in the collapse is of size > 1 (and
+      // in the expand is of size == 1). In this case, the original dimensions
+      // are preserved on expansion and collapsed subsequently.
       if (collapseReassociation.size() != 1) {
         ReassociationIndices newCollapseReassociation;
         for (size_t i = 0; i < collapseReassociation.size(); ++i) {
-          newCollapseReassociation.push_back(index);
-          newExpandReInds.push_back({index++});
-          newExpandSizes.push_back(collapseSizes[collapseIndex++]);
+          newCollapseReassociation.push_back(newCollapseIndex++);
+          newExpandReInds.push_back({newExpandIndex++});
+          newExpandSizes.push_back(sourceSizes[sourceSizeIndex++]);
         }
+        resultSizeIndex++;
         newCollapseReInds.push_back(newCollapseReassociation);
-        expandIndex++;
         continue;
       }
+
+      // Case 3. The `ReassociationIndices` in the expand is of size > 1 (and
+      // in the collapse is of size == 1). In this case, the expansion happens
+      // first and the expanded dimensions are preserved on collapse.
       ReassociationIndices newExpandReassociation;
-      auto expandReassociation = expandReInds[idx];
       for (size_t i = 0; i < expandReassociation.size(); ++i) {
-        newExpandReassociation.push_back(index);
-        newCollapseReInds.push_back({index++});
-        newExpandSizes.push_back(expandSizes[expandIndex++]);
+        newExpandReassociation.push_back(newExpandIndex++);
+        newCollapseReInds.push_back({newCollapseIndex++});
+        newExpandSizes.push_back(resultSizes[resultSizeIndex++]);
       }
       newExpandReInds.push_back(newExpandReassociation);
-      collapseIndex++;
+      sourceSizeIndex++;
     }
 
     // Swap reshape order.
@@ -212,11 +268,25 @@ struct BubbleUpExpandThroughParallelCollapse
     SmallVector<int64_t> staticSizes;
     dispatchIndexOpFoldResults(newExpandSizes, dynamicSizes, staticSizes);
     auto expandResultType = expandOp.getResultType().clone(staticSizes);
-    auto newExpand = rewriter.create<tensor::ExpandShapeOp>(
-        loc, expandResultType, collapseOp.getSrc(), newExpandReInds,
-        newExpandSizes);
-    rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
-        expandOp, newExpand.getResult(), newCollapseReInds);
+    Value newCollapseSrc = collapseOp.getSrc();
+    // If the number of reassociation indices in the new `expand_shape` op
+    // matches the number of dimensions of the result, then the expand_shape
+    // is a no-op.
+    if (newExpandReInds.size() != newExpandSizes.size()) {
+      newCollapseSrc = rewriter.create<tensor::ExpandShapeOp>(
+          loc, expandResultType, newCollapseSrc, newExpandReInds,
+          newExpandSizes);
+    }
+
+    // If the number of reassociation indices in the new `collapse_shape` op
+    // matches the number of dimensions of the source, then the collapse_shape
+    // is a no-op.
+    Value replacement = newCollapseSrc;
+    if (newCollapseReInds.size() != newExpandSizes.size()) {
+      replacement = rewriter.create<tensor::CollapseShapeOp>(
+          loc, newCollapseSrc, newCollapseReInds);
+    }
+    rewriter.replaceOp(expandOp, replacement);
     return success();
   }
 };
diff --git a/mlir/test/Dialect/Tensor/bubble-reshapes.mlir b/mlir/test/Dialect/Tensor/bubble-reshapes.mlir
index eeed794884942..81bf8e3f60e2c 100644
--- a/mlir/test/Dialect/Tensor/bubble-reshapes.mlir
+++ b/mlir/test/Dialect/Tensor/bubble-reshapes.mlir
@@ -48,14 +48,67 @@ func.func @no_bubble_partial_intersecting_reshapes(%arg0: tensor<?x?x?x?xf32>, %
 
 // -----
 
-func.func @no_bubble_0d_tensor_reshapes(%arg0: tensor<?xf32>, %s0: index, %s1: index, %s2: index, %s3: index) -> tensor<?x?x?x?xf32> {
-  %collapse = tensor.collapse_shape %arg0 [] : tensor<?xf32> into tensor<f32>
+func.func @no_bubble_0d_tensor_reshapes(%arg0: tensor<1x1xf32>) -> tensor<1x1x1xf32> {
+  %collapse = tensor.collapse_shape %arg0 [] : tensor<1x1xf32> into tensor<f32>
   %expand = tensor.expand_shape %collapse []
-              output_shape [%s0, %s1, %s2, %s3] : tensor<f32> into tensor<?x?x?x?xf32>
-  return %expand : tensor<?x?x?x?xf32>
+              output_shape [1, 1, 1] : tensor<f32> into tensor<1x1x1xf32>
+  return %expand : tensor<1x1x1xf32>
 }
 //      CHECK: func @no_bubble_0d_tensor_reshapes
-// CHECK-SAME:   %[[ARG0:.+]]: tensor<?xf32>
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<1x1xf32>
 //      CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %[[ARG0]] {{\[}}]
 //      CHECK:   %[[EXPAND:.+]] = tensor.expand_shape %[[COLLAPSE]] {{\[}}]
 //      CHECK:   return %[[EXPAND]]
+
+// -----
+
+// Test the case where the reassocation indices in the collapse and expand
+// are of same size.
+func.func @bubble_expand_match_non_unit_size_reassocation(
+      %arg0 : tensor<4x?x4x32x4x?xf16>, %arg1 : index, %arg2 : index) -> tensor<4x?x4x128x?x32xf16> {
+  %collapsed = tensor.collapse_shape %arg0 [[0, 1, 2], [3, 4], [5]]
+      : tensor<4x?x4x32x4x?xf16> into tensor<?x128x?xf16>
+  %expanded = tensor.expand_shape %collapsed [[0, 1, 2], [3], [4, 5]] output_shape [4, %arg1, 4, 128, %arg2, 32]
+      : tensor<?x128x?xf16> into tensor<4x?x4x128x?x32xf16>
+  return %expanded : tensor<4x?x4x128x?x32xf16>
+}
+//      CHECK: func @bubble_expand_match_non_unit_size_reassocation
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<4x?x4x32x4x?xf16>
+// CHECK-SAME:     %[[ARG1:[a-zA-z0-9]+]]: index
+// CHECK-SAME:     %[[ARG2:[a-zA-z0-9]+]]: index
+//      CHECK:   %[[EXPANDED:.+]] = tensor.expand_shape %[[ARG0]]
+// CHECK-SAME:       {{\[}}[0], [1], [2], [3], [4], [5, 6]{{\]}}
+// CHECK-SAME:       [4, %[[ARG1]], 4, 32, 4, %[[ARG2]], 32]
+//      CHECK:   %[[COLLAPSED:.+]] = tensor.collapse_shape %[[EXPANDED]]
+// CHECK-SAME:       {{\[}}[0], [1], [2], [3, 4], [5], [6]{{\]}}
+//      CHECK:   return %[[COLLAPSED]]
+
+// -----
+
+// Test the case where the trailing collapse isnt needed.
+func.func @no_collapse_generated(
+      %arg0 : tensor<4x?x4x128x?xf16>, %arg1 : index, %arg2 : index) -> tensor<4x?x4x128x?x32xf16> {
+  %collapsed = tensor.collapse_shape %arg0 [[0, 1, 2], [3], [4]]
+      : tensor<4x?x4x128x?xf16> into tensor<?x128x?xf16>
+  %expanded = tensor.expand_shape %collapsed [[0, 1, 2], [3], [4, 5]] output_shape [4, %arg1, 4, 128, %arg2, 32]
+      : tensor<?x128x?xf16> into tensor<4x?x4x128x?x32xf16>
+  return %expanded : tensor<4x?x4x128x?x32xf16>
+}
+//      CHECK: func @no_collapse_generated
+//      CHECK:   %[[EXPANDED:.+]] = tensor.expand_shape 
+//      CHECK:   return %[[EXPANDED]]
+
+// -----
+
+// Test the case where the leading expand isnt needed.
+func.func @no_expand_generated(
+      %arg0 : tensor<4x?x4x128x?x?x?xf16>, %arg1 : index, %arg2 : index, %arg3 : index) -> tensor<4x?x4x128x?x?xf16> {
+  %collapsed = tensor.collapse_shape %arg0 [[0, 1, 2], [3], [4], [5, 6]]
+      : tensor<4x?x4x128x?x?x?xf16> into tensor<?x128x?x?xf16>
+  %expanded = tensor.expand_shape %collapsed [[0, 1, 2], [3], [4], [5]] output_shape [4, %arg1, 4, 128, %arg2, %arg3]
+      : tensor<?x128x?x?xf16> into tensor<4x?x4x128x?x?xf16>
+  return %expanded : tensor<4x?x4x128x?x?xf16>
+}
+//      CHECK: func @no_expand_generated
+//      CHECK:   %[[EXPANDED:.+]] = tensor.collapse_shape
+//      CHECK:   return %[[EXPANDED]]

From d30a5b41fe72a1dd83714d3e21fd539b91e63c8c Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic@htecgroup.com>
Date: Tue, 15 Apr 2025 23:17:03 +0200
Subject: [PATCH 043/710] [RISCV] Fix xmipscmov extension name (#135647)

The right name was used in riscv-toolchain-conventions docs.
---
 clang/test/Driver/print-supported-extensions-riscv.c |  2 +-
 llvm/docs/RISCVUsage.rst                             |  2 +-
 .../Target/RISCV/Disassembler/RISCVDisassembler.cpp  |  4 ++--
 llvm/lib/Target/RISCV/RISCVFeatures.td               | 12 ++++++------
 llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td         |  4 ++--
 llvm/lib/Target/RISCV/RISCVProcessors.td             |  2 +-
 llvm/lib/Target/RISCV/RISCVSubtarget.cpp             |  2 +-
 llvm/test/CodeGen/RISCV/features-info.ll             |  2 +-
 llvm/test/CodeGen/RISCV/select-and.ll                |  2 +-
 llvm/test/CodeGen/RISCV/select-bare.ll               |  2 +-
 llvm/test/CodeGen/RISCV/select-cc.ll                 |  2 +-
 llvm/test/CodeGen/RISCV/select-or.ll                 |  2 +-
 llvm/test/MC/RISCV/xmips-invalid.s                   |  4 ++--
 llvm/test/MC/RISCV/xmips-valid.s                     |  6 +++---
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp     |  2 +-
 15 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index d06cedac5b1eb..39002d7b4780a 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -161,7 +161,7 @@
 // CHECK-NEXT:     xcvmac               1.0       'XCVmac' (CORE-V Multiply-Accumulate)
 // CHECK-NEXT:     xcvmem               1.0       'XCVmem' (CORE-V Post-incrementing Load & Store)
 // CHECK-NEXT:     xcvsimd              1.0       'XCVsimd' (CORE-V SIMD ALU)
-// CHECK-NEXT:     xmipscmove           1.0       'XMIPSCMove' (MIPS conditional move instruction(s) (ccmov))
+// CHECK-NEXT:     xmipscmov            1.0       'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov))
 // CHECK-NEXT:     xmipslsp             1.0       'XMIPSLSP' (MIPS optimization for hardware load-store bonding)
 // CHECK-NEXT:     xsfcease             1.0       'XSfcease' (SiFive sf.cease Instruction)
 // CHECK-NEXT:     xsfvcp               1.0       'XSfvcp' (SiFive Custom Vector Coprocessor Interface Instructions)
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index cda7e5fec8488..137b537f00ea0 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -491,7 +491,7 @@ The current vendor extensions supported are:
 ``experimental-Xqcisync``
   LLVM implements `version 0.2 of the Qualcomm uC Sync Delay extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
 
-``Xmipscmove``
+``Xmipscmov``
   LLVM implements conditional move for the `p8700 processor <https://mips.com/products/hardware/p8700/>` by MIPS.
 
 ``Xmipslsp``
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 366291b53bebb..27809d96b647c 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -737,8 +737,8 @@ static constexpr DecoderListEntry DecoderList32[]{
     {DecoderTableXSfsystem32, XSfSystemGroup, "SiFive system extensions"},
     {DecoderTableXSfcease32, {RISCV::FeatureVendorXSfcease}, "SiFive sf.cease"},
     {DecoderTableXmipslsp32, {RISCV::FeatureVendorXMIPSLSP}, "MIPS mips.lsp"},
-    {DecoderTableXmipscmove32,
-     {RISCV::FeatureVendorXMIPSCMove},
+    {DecoderTableXmipscmov32,
+     {RISCV::FeatureVendorXMIPSCMov},
      "MIPS mips.ccmov"},
     // Standard Extensions
     {DecoderTableXCV32, XCVFeatureGroup, "CORE-V extensions"},
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 21bcf343139c2..f51fcf82077f4 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1300,12 +1300,12 @@ def HasVendorXCVbi
                          "'XCVbi' (CORE-V Immediate Branching)">;
 // MIPS Extensions
 
-def FeatureVendorXMIPSCMove
-    : RISCVExtension<1, 0, "MIPS conditional move instruction(s) (ccmov)">;
-def HasVendorXMIPSCMove
-    : Predicate<"Subtarget->hasVendorXMIPSCMove()">,
-      AssemblerPredicate<(all_of FeatureVendorXMIPSCMove),
-                         "'Xmipscmove' ('mips.ccmov' instruction)">;
+def FeatureVendorXMIPSCMov
+    : RISCVExtension<1, 0, "MIPS conditional move instruction (mips.ccmov)">;
+def HasVendorXMIPSCMov
+    : Predicate<"Subtarget->hasVendorXMIPSCMov()">,
+      AssemblerPredicate<(all_of FeatureVendorXMIPSCMov),
+                         "'Xmipscmov' ('mips.ccmov' instruction)">;
 def UseCCMovInsn : Predicate<"Subtarget->useCCMovInsn()">;
 def FeatureVendorXMIPSLSP
     : RISCVExtension<1, 0, "MIPS optimization for hardware load-store bonding">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
index 9be424310d660..ff751994b89b9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
@@ -107,8 +107,8 @@ class SWPFormat<dag outs, dag ins, string opcodestr, string argstr>
 // MIPS extensions
 //===----------------------------------------------------------------------===//
 
-let Predicates = [HasVendorXMIPSCMove], hasSideEffects = 0, mayLoad = 0, mayStore = 0,
-                 DecoderNamespace = "Xmipscmove" in {
+let Predicates = [HasVendorXMIPSCMov], hasSideEffects = 0, mayLoad = 0, mayStore = 0,
+                 DecoderNamespace = "Xmipscmov" in {
 def MIPS_CCMOV : RVInstR4<0b11, 0b011, OPC_CUSTOM_0, (outs GPR:$rd),
                           (ins GPR:$rs1, GPR:$rs2, GPR:$rs3),
                           "mips.ccmov", "$rd, $rs2, $rs1, $rs3">,
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index 9d48adeec5e86..ece12c48b1cd9 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -119,7 +119,7 @@ def MIPS_P8700 : RISCVProcessorModel<"mips-p8700",
                                       FeatureStdExtZbb,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtZicsr,
-                                      FeatureVendorXMIPSCMove,
+                                      FeatureVendorXMIPSCMov,
                                       FeatureVendorXMIPSLSP],
                                      [TuneMIPSP8700]>;
 
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
index 3c996c82fcec4..b3c313f2ed394 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
@@ -253,5 +253,5 @@ bool RISCVSubtarget::useLoadStorePairs() const {
 }
 
 bool RISCVSubtarget::useCCMovInsn() const {
-  return UseCCMovInsn && HasVendorXMIPSCMove;
+  return UseCCMovInsn && HasVendorXMIPSCMov;
 }
diff --git a/llvm/test/CodeGen/RISCV/features-info.ll b/llvm/test/CodeGen/RISCV/features-info.ll
index 5d2f0881048bd..d377bda059d33 100644
--- a/llvm/test/CodeGen/RISCV/features-info.ll
+++ b/llvm/test/CodeGen/RISCV/features-info.ll
@@ -174,7 +174,7 @@
 ; CHECK-NEXT:   xcvmac                           - 'XCVmac' (CORE-V Multiply-Accumulate).
 ; CHECK-NEXT:   xcvmem                           - 'XCVmem' (CORE-V Post-incrementing Load & Store).
 ; CHECK-NEXT:   xcvsimd                          - 'XCVsimd' (CORE-V SIMD ALU).
-; CHECK-NEXT:   xmipscmove                       - 'XMIPSCMove' (MIPS conditional move instruction(s) (ccmov)).
+; CHECK-NEXT:   xmipscmov                        - 'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov)).
 ; CHECK-NEXT:   xmipslsp                         - 'XMIPSLSP' (MIPS optimization for hardware load-store bonding).
 ; CHECK-NEXT:   xsfcease                         - 'XSfcease' (SiFive sf.cease Instruction).
 ; CHECK-NEXT:   xsfvcp                           - 'XSfvcp' (SiFive Custom Vector Coprocessor Interface Instructions).
diff --git a/llvm/test/CodeGen/RISCV/select-and.ll b/llvm/test/CodeGen/RISCV/select-and.ll
index f827e840f4a36..2c9d0a8b56425 100644
--- a/llvm/test/CodeGen/RISCV/select-and.ll
+++ b/llvm/test/CodeGen/RISCV/select-and.ll
@@ -3,7 +3,7 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
-; RUN: llc -mtriple=riscv64 -mattr=+xmipscmove -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmov -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I-CCMOV %s
 
 ;; There are a few different ways to lower (select (and A, B), X, Y). This test
diff --git a/llvm/test/CodeGen/RISCV/select-bare.ll b/llvm/test/CodeGen/RISCV/select-bare.ll
index c9e108a1ca9d0..fc8eaa480b116 100644
--- a/llvm/test/CodeGen/RISCV/select-bare.ll
+++ b/llvm/test/CodeGen/RISCV/select-bare.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV32I
-; RUN: llc -mtriple=riscv64 -mattr=+xmipscmove -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmov -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I-CCMOV %s
 
 define i32 @bare_select(i1 %a, i32 %b, i32 %c) nounwind {
diff --git a/llvm/test/CodeGen/RISCV/select-cc.ll b/llvm/test/CodeGen/RISCV/select-cc.ll
index 1c2a0cf007d11..e69dc303d85dc 100644
--- a/llvm/test/CodeGen/RISCV/select-cc.ll
+++ b/llvm/test/CodeGen/RISCV/select-cc.ll
@@ -3,7 +3,7 @@
 ; RUN:   | FileCheck -check-prefixes=RV32I %s
 ; RUN: llc -mtriple=riscv64 -disable-block-placement -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV64I %s
-; RUN: llc -mtriple=riscv64 -mattr=+xmipscmove -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmov -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I-CCMOV %s
 
 define signext i32 @foo(i32 signext %a, ptr %b) nounwind {
diff --git a/llvm/test/CodeGen/RISCV/select-or.ll b/llvm/test/CodeGen/RISCV/select-or.ll
index 338c7c06c3ab8..091c8b1a11e71 100644
--- a/llvm/test/CodeGen/RISCV/select-or.ll
+++ b/llvm/test/CodeGen/RISCV/select-or.ll
@@ -3,7 +3,7 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
-; RUN: llc -mtriple=riscv64 -mattr=+xmipscmove -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmov -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I-CCMOV %s
 
 ;; There are a few different ways to lower (select (or A, B), X, Y). This test
diff --git a/llvm/test/MC/RISCV/xmips-invalid.s b/llvm/test/MC/RISCV/xmips-invalid.s
index a1c1fd0666e0a..b3834e7b3407f 100644
--- a/llvm/test/MC/RISCV/xmips-invalid.s
+++ b/llvm/test/MC/RISCV/xmips-invalid.s
@@ -1,5 +1,5 @@
 # RUN: not llvm-mc -triple=riscv64 < %s 2>&1 | FileCheck %s -check-prefixes=CHECK-FEATURE
-# RUN: not llvm-mc -triple=riscv64 -mattr=+xmipslsp,+xmipscmove < %s 2>&1 | FileCheck %s
+# RUN: not llvm-mc -triple=riscv64 -mattr=+xmipslsp,+xmipscmov < %s 2>&1 | FileCheck %s
 
 mips.ccmov x0, x1, 0x10
 # CHECK: error: invalid operand for instruction
@@ -8,7 +8,7 @@ mips.ccmov x10
 # CHECK: error: too few operands for instruction
 
 mips.ccmov	s0, s1, s2, s3
-# CHECK-FEATURE: error: instruction requires the following: 'Xmipscmove' ('mips.ccmov' instruction)
+# CHECK-FEATURE: error: instruction requires the following: 'Xmipscmov' ('mips.ccmov' instruction)
 
 mips.lwp x10, x11
 # CHECK: error: too few operands for instruction
diff --git a/llvm/test/MC/RISCV/xmips-valid.s b/llvm/test/MC/RISCV/xmips-valid.s
index ba256a823f511..9f31e4fa2038c 100644
--- a/llvm/test/MC/RISCV/xmips-valid.s
+++ b/llvm/test/MC/RISCV/xmips-valid.s
@@ -1,7 +1,7 @@
-# RUN: llvm-mc %s -triple=riscv64 -mattr=+xmipslsp,+xmipscmove -M no-aliases -show-encoding \
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+xmipslsp,+xmipscmov -M no-aliases -show-encoding \
 # RUN:   | FileCheck -check-prefixes=CHECK-INST,CHECK-ENC %s
-# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+xmipslsp,+xmipscmove < %s \
-# RUN:   | llvm-objdump --mattr=+xmipslsp,+xmipscmove -M no-aliases -d - \
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+xmipslsp,+xmipscmov < %s \
+# RUN:   | llvm-objdump --mattr=+xmipslsp,+xmipscmov -M no-aliases -d - \
 # RUN:   | FileCheck -check-prefix=CHECK-DIS %s
 
 # CHECK-INST: mips.ccmov	s0, s1, s2, s3
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 2ec27ba9d91b7..ff0a5e64ab3e1 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1114,7 +1114,7 @@ R"(All available -march extensions for RISC-V
     xcvmac               1.0
     xcvmem               1.0
     xcvsimd              1.0
-    xmipscmove           1.0
+    xmipscmov            1.0
     xmipslsp             1.0
     xsfcease             1.0
     xsfvcp               1.0

From 9c73eba8aa17cb7ca4248ab1c7f67ea7ec9b50b1 Mon Sep 17 00:00:00 2001
From: Aaron Puchert <aaronpuchert@alice-dsl.net>
Date: Tue, 15 Apr 2025 23:21:34 +0200
Subject: [PATCH 044/710] Merge similar Clang Thread Safety attributes
 (#135561)

Some of the old lock-based and new capability-based spellings behave
basically in the same way, so merging them simplifies the code
significantly.

There are two minor functional changes: we only warn (instead of an
error) when the try_acquire_capability attribute is used on something
else than a function. The alternative would have been to produce an
error for the old spelling, but we seem to only warn for all function
attributes, so this is arguably more consistent.

The second change is that we also check the first argument (which is the
value returned for a successful try-acquire) for `this`. But from what I
can tell, this code is defunct anyway at the moment (see #31414).
---
 clang/include/clang/Basic/Attr.td       | 65 +++++--------------------
 clang/lib/AST/ASTImporter.cpp           | 28 -----------
 clang/lib/Analysis/ThreadSafety.cpp     | 60 ++---------------------
 clang/lib/Sema/SemaDeclAttr.cpp         | 55 ---------------------
 clang/lib/Sema/SemaDeclCXX.cpp          | 13 ++---
 clang/test/Sema/attr-capabilities.c     |  2 +-
 clang/unittests/AST/ASTImporterTest.cpp | 36 --------------
 7 files changed, 21 insertions(+), 238 deletions(-)

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 9d4900f3029c8..9465451cbfe1f 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3818,7 +3818,9 @@ def Capability : InheritableAttr {
 
 def AssertCapability : InheritableAttr {
   let Spellings = [Clang<"assert_capability", 0>,
-                   Clang<"assert_shared_capability", 0>];
+                   Clang<"assert_shared_capability", 0>,
+                   GNU<"assert_exclusive_lock">,
+                   GNU<"assert_shared_lock">];
   let Subjects = SubjectList<[Function]>;
   let LateParsed = LateAttrParseStandard;
   let TemplateDependent = 1;
@@ -3826,7 +3828,8 @@ def AssertCapability : InheritableAttr {
   let InheritEvenIfAlreadyPresent = 1;
   let Args = [VariadicExprArgument<"Args">];
   let Accessors = [Accessor<"isShared",
-                    [Clang<"assert_shared_capability", 0>]>];
+                    [Clang<"assert_shared_capability", 0>,
+                     GNU<"assert_shared_lock">]>];
   let Documentation = [AssertCapabilityDocs];
 }
 
@@ -3849,16 +3852,18 @@ def AcquireCapability : InheritableAttr {
 
 def TryAcquireCapability : InheritableAttr {
   let Spellings = [Clang<"try_acquire_capability", 0>,
-                   Clang<"try_acquire_shared_capability", 0>];
-  let Subjects = SubjectList<[Function],
-                             ErrorDiag>;
+                   Clang<"try_acquire_shared_capability", 0>,
+                   GNU<"exclusive_trylock_function">,
+                   GNU<"shared_trylock_function">];
+  let Subjects = SubjectList<[Function]>;
   let LateParsed = LateAttrParseStandard;
   let TemplateDependent = 1;
   let ParseArgumentsAsUnevaluated = 1;
   let InheritEvenIfAlreadyPresent = 1;
   let Args = [ExprArgument<"SuccessValue">, VariadicExprArgument<"Args">];
   let Accessors = [Accessor<"isShared",
-                    [Clang<"try_acquire_shared_capability", 0>]>];
+                    [Clang<"try_acquire_shared_capability", 0>,
+                     GNU<"shared_trylock_function">]>];
   let Documentation = [TryAcquireCapabilityDocs];
 }
 
@@ -3948,54 +3953,6 @@ def AcquiredBefore : InheritableAttr {
   let Documentation = [Undocumented];
 }
 
-def AssertExclusiveLock : InheritableAttr {
-  let Spellings = [GNU<"assert_exclusive_lock">];
-  let Args = [VariadicExprArgument<"Args">];
-  let LateParsed = LateAttrParseStandard;
-  let TemplateDependent = 1;
-  let ParseArgumentsAsUnevaluated = 1;
-  let InheritEvenIfAlreadyPresent = 1;
-  let Subjects = SubjectList<[Function]>;
-  let Documentation = [Undocumented];
-}
-
-def AssertSharedLock : InheritableAttr {
-  let Spellings = [GNU<"assert_shared_lock">];
-  let Args = [VariadicExprArgument<"Args">];
-  let LateParsed = LateAttrParseStandard;
-  let TemplateDependent = 1;
-  let ParseArgumentsAsUnevaluated = 1;
-  let InheritEvenIfAlreadyPresent = 1;
-  let Subjects = SubjectList<[Function]>;
-  let Documentation = [Undocumented];
-}
-
-// The first argument is an integer or boolean value specifying the return value
-// of a successful lock acquisition.
-def ExclusiveTrylockFunction : InheritableAttr {
-  let Spellings = [GNU<"exclusive_trylock_function">];
-  let Args = [ExprArgument<"SuccessValue">, VariadicExprArgument<"Args">];
-  let LateParsed = LateAttrParseStandard;
-  let TemplateDependent = 1;
-  let ParseArgumentsAsUnevaluated = 1;
-  let InheritEvenIfAlreadyPresent = 1;
-  let Subjects = SubjectList<[Function]>;
-  let Documentation = [Undocumented];
-}
-
-// The first argument is an integer or boolean value specifying the return value
-// of a successful lock acquisition.
-def SharedTrylockFunction : InheritableAttr {
-  let Spellings = [GNU<"shared_trylock_function">];
-  let Args = [ExprArgument<"SuccessValue">, VariadicExprArgument<"Args">];
-  let LateParsed = LateAttrParseStandard;
-  let TemplateDependent = 1;
-  let ParseArgumentsAsUnevaluated = 1;
-  let InheritEvenIfAlreadyPresent = 1;
-  let Subjects = SubjectList<[Function]>;
-  let Documentation = [Undocumented];
-}
-
 def LockReturned : InheritableAttr {
   let Spellings = [GNU<"lock_returned">];
   let Args = [ExprArgument<"Arg">];
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index b55b8f2c14147..00628602e61fa 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -9425,34 +9425,6 @@ Expected<Attr *> ASTImporter::Import(const Attr *FromAttr) {
                   From->args_size());
     break;
   }
-  case attr::AssertExclusiveLock: {
-    const auto *From = cast<AssertExclusiveLockAttr>(FromAttr);
-    AI.importAttr(From,
-                  AI.importArrayArg(From->args(), From->args_size()).value(),
-                  From->args_size());
-    break;
-  }
-  case attr::AssertSharedLock: {
-    const auto *From = cast<AssertSharedLockAttr>(FromAttr);
-    AI.importAttr(From,
-                  AI.importArrayArg(From->args(), From->args_size()).value(),
-                  From->args_size());
-    break;
-  }
-  case attr::ExclusiveTrylockFunction: {
-    const auto *From = cast<ExclusiveTrylockFunctionAttr>(FromAttr);
-    AI.importAttr(From, AI.importArg(From->getSuccessValue()).value(),
-                  AI.importArrayArg(From->args(), From->args_size()).value(),
-                  From->args_size());
-    break;
-  }
-  case attr::SharedTrylockFunction: {
-    const auto *From = cast<SharedTrylockFunctionAttr>(FromAttr);
-    AI.importAttr(From, AI.importArg(From->getSuccessValue()).value(),
-                  AI.importArrayArg(From->args(), From->args_size()).value(),
-                  From->args_size());
-    break;
-  }
   case attr::LockReturned: {
     const auto *From = cast<LockReturnedAttr>(FromAttr);
     AI.importAttr(From, AI.importArg(From->getArg()).value());
diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp
index 6b5b49377fa08..42fb0fe7dcdaa 100644
--- a/clang/lib/Analysis/ThreadSafety.cpp
+++ b/clang/lib/Analysis/ThreadSafety.cpp
@@ -1511,38 +1511,17 @@ void ThreadSafetyAnalyzer::getEdgeLockset(FactSet& Result,
     return;
 
   auto *FunDecl = dyn_cast_or_null<NamedDecl>(Exp->getCalleeDecl());
-  if(!FunDecl || !FunDecl->hasAttrs())
+  if (!FunDecl || !FunDecl->hasAttr<TryAcquireCapabilityAttr>())
     return;
 
   CapExprSet ExclusiveLocksToAdd;
   CapExprSet SharedLocksToAdd;
 
   // If the condition is a call to a Trylock function, then grab the attributes
-  for (const auto *Attr : FunDecl->attrs()) {
-    switch (Attr->getKind()) {
-      case attr::TryAcquireCapability: {
-        auto *A = cast<TryAcquireCapabilityAttr>(Attr);
-        getMutexIDs(A->isShared() ? SharedLocksToAdd : ExclusiveLocksToAdd, A,
-                    Exp, FunDecl, PredBlock, CurrBlock, A->getSuccessValue(),
-                    Negate);
-        break;
-      };
-      case attr::ExclusiveTrylockFunction: {
-        const auto *A = cast<ExclusiveTrylockFunctionAttr>(Attr);
-        getMutexIDs(ExclusiveLocksToAdd, A, Exp, FunDecl, PredBlock, CurrBlock,
-                    A->getSuccessValue(), Negate);
-        break;
-      }
-      case attr::SharedTrylockFunction: {
-        const auto *A = cast<SharedTrylockFunctionAttr>(Attr);
-        getMutexIDs(SharedLocksToAdd, A, Exp, FunDecl, PredBlock, CurrBlock,
-                    A->getSuccessValue(), Negate);
-        break;
-      }
-      default:
-        break;
-    }
-  }
+  for (const auto *Attr : FunDecl->specific_attrs<TryAcquireCapabilityAttr>())
+    getMutexIDs(Attr->isShared() ? SharedLocksToAdd : ExclusiveLocksToAdd, Attr,
+                Exp, FunDecl, PredBlock, CurrBlock, Attr->getSuccessValue(),
+                Negate);
 
   // Add and remove locks.
   SourceLocation Loc = Exp->getExprLoc();
@@ -1882,29 +1861,6 @@ void BuildLockset::handleCall(const Expr *Exp, const NamedDecl *D,
       // An assert will add a lock to the lockset, but will not generate
       // a warning if it is already there, and will not generate a warning
       // if it is not removed.
-      case attr::AssertExclusiveLock: {
-        const auto *A = cast<AssertExclusiveLockAttr>(At);
-
-        CapExprSet AssertLocks;
-        Analyzer->getMutexIDs(AssertLocks, A, Exp, D, Self);
-        for (const auto &AssertLock : AssertLocks)
-          Analyzer->addLock(
-              FSet, std::make_unique<LockableFactEntry>(
-                        AssertLock, LK_Exclusive, Loc, FactEntry::Asserted));
-        break;
-      }
-      case attr::AssertSharedLock: {
-        const auto *A = cast<AssertSharedLockAttr>(At);
-
-        CapExprSet AssertLocks;
-        Analyzer->getMutexIDs(AssertLocks, A, Exp, D, Self);
-        for (const auto &AssertLock : AssertLocks)
-          Analyzer->addLock(
-              FSet, std::make_unique<LockableFactEntry>(
-                        AssertLock, LK_Shared, Loc, FactEntry::Asserted));
-        break;
-      }
-
       case attr::AssertCapability: {
         const auto *A = cast<AssertCapabilityAttr>(At);
         CapExprSet AssertLocks;
@@ -2499,12 +2455,6 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
         getMutexIDs(A->isShared() ? SharedLocksAcquired
                                   : ExclusiveLocksAcquired,
                     A, nullptr, D);
-      } else if (isa<ExclusiveTrylockFunctionAttr>(Attr)) {
-        // Don't try to check trylock functions for now.
-        return;
-      } else if (isa<SharedTrylockFunctionAttr>(Attr)) {
-        // Don't try to check trylock functions for now.
-        return;
       } else if (isa<TryAcquireCapabilityAttr>(Attr)) {
         // Don't try to check trylock functions for now.
         return;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 20ea38b7e05db..bc891fb009410 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -538,29 +538,6 @@ static bool checkLockFunAttrCommon(Sema &S, Decl *D, const ParsedAttr &AL,
   return true;
 }
 
-static void handleAssertSharedLockAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
-  SmallVector<Expr *, 1> Args;
-  if (!checkLockFunAttrCommon(S, D, AL, Args))
-    return;
-
-  unsigned Size = Args.size();
-  Expr **StartArg = Size == 0 ? nullptr : &Args[0];
-  D->addAttr(::new (S.Context)
-                 AssertSharedLockAttr(S.Context, AL, StartArg, Size));
-}
-
-static void handleAssertExclusiveLockAttr(Sema &S, Decl *D,
-                                          const ParsedAttr &AL) {
-  SmallVector<Expr *, 1> Args;
-  if (!checkLockFunAttrCommon(S, D, AL, Args))
-    return;
-
-  unsigned Size = Args.size();
-  Expr **StartArg = Size == 0 ? nullptr : &Args[0];
-  D->addAttr(::new (S.Context)
-                 AssertExclusiveLockAttr(S.Context, AL, StartArg, Size));
-}
-
 /// Checks to be sure that the given parameter number is in bounds, and
 /// is an integral type. Will emit appropriate diagnostics if this returns
 /// false.
@@ -640,26 +617,6 @@ static bool checkTryLockFunAttrCommon(Sema &S, Decl *D, const ParsedAttr &AL,
   return true;
 }
 
-static void handleSharedTrylockFunctionAttr(Sema &S, Decl *D,
-                                            const ParsedAttr &AL) {
-  SmallVector<Expr*, 2> Args;
-  if (!checkTryLockFunAttrCommon(S, D, AL, Args))
-    return;
-
-  D->addAttr(::new (S.Context) SharedTrylockFunctionAttr(
-      S.Context, AL, AL.getArgAsExpr(0), Args.data(), Args.size()));
-}
-
-static void handleExclusiveTrylockFunctionAttr(Sema &S, Decl *D,
-                                               const ParsedAttr &AL) {
-  SmallVector<Expr*, 2> Args;
-  if (!checkTryLockFunAttrCommon(S, D, AL, Args))
-    return;
-
-  D->addAttr(::new (S.Context) ExclusiveTrylockFunctionAttr(
-      S.Context, AL, AL.getArgAsExpr(0), Args.data(), Args.size()));
-}
-
 static void handleLockReturnedAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   // check that the argument is lockable object
   SmallVector<Expr*, 1> Args;
@@ -7528,12 +7485,6 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
     break;
 
   // Thread safety attributes:
-  case ParsedAttr::AT_AssertExclusiveLock:
-    handleAssertExclusiveLockAttr(S, D, AL);
-    break;
-  case ParsedAttr::AT_AssertSharedLock:
-    handleAssertSharedLockAttr(S, D, AL);
-    break;
   case ParsedAttr::AT_PtGuardedVar:
     handlePtGuardedVarAttr(S, D, AL);
     break;
@@ -7549,18 +7500,12 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_PtGuardedBy:
     handlePtGuardedByAttr(S, D, AL);
     break;
-  case ParsedAttr::AT_ExclusiveTrylockFunction:
-    handleExclusiveTrylockFunctionAttr(S, D, AL);
-    break;
   case ParsedAttr::AT_LockReturned:
     handleLockReturnedAttr(S, D, AL);
     break;
   case ParsedAttr::AT_LocksExcluded:
     handleLocksExcludedAttr(S, D, AL);
     break;
-  case ParsedAttr::AT_SharedTrylockFunction:
-    handleSharedTrylockFunctionAttr(S, D, AL);
-    break;
   case ParsedAttr::AT_AcquiredBefore:
     handleAcquiredBeforeAttr(S, D, AL);
     break;
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 2247aded9384a..05991228dbfc2 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -19401,13 +19401,7 @@ bool Sema::checkThisInStaticMemberFunctionAttributes(CXXMethodDecl *Method) {
       Args = llvm::ArrayRef(AA->args_begin(), AA->args_size());
     else if (const auto *AB = dyn_cast<AcquiredBeforeAttr>(A))
       Args = llvm::ArrayRef(AB->args_begin(), AB->args_size());
-    else if (const auto *ETLF = dyn_cast<ExclusiveTrylockFunctionAttr>(A)) {
-      Arg = ETLF->getSuccessValue();
-      Args = llvm::ArrayRef(ETLF->args_begin(), ETLF->args_size());
-    } else if (const auto *STLF = dyn_cast<SharedTrylockFunctionAttr>(A)) {
-      Arg = STLF->getSuccessValue();
-      Args = llvm::ArrayRef(STLF->args_begin(), STLF->args_size());
-    } else if (const auto *LR = dyn_cast<LockReturnedAttr>(A))
+    else if (const auto *LR = dyn_cast<LockReturnedAttr>(A))
       Arg = LR->getArg();
     else if (const auto *LE = dyn_cast<LocksExcludedAttr>(A))
       Args = llvm::ArrayRef(LE->args_begin(), LE->args_size());
@@ -19415,9 +19409,10 @@ bool Sema::checkThisInStaticMemberFunctionAttributes(CXXMethodDecl *Method) {
       Args = llvm::ArrayRef(RC->args_begin(), RC->args_size());
     else if (const auto *AC = dyn_cast<AcquireCapabilityAttr>(A))
       Args = llvm::ArrayRef(AC->args_begin(), AC->args_size());
-    else if (const auto *AC = dyn_cast<TryAcquireCapabilityAttr>(A))
+    else if (const auto *AC = dyn_cast<TryAcquireCapabilityAttr>(A)) {
+      Arg = AC->getSuccessValue();
       Args = llvm::ArrayRef(AC->args_begin(), AC->args_size());
-    else if (const auto *RC = dyn_cast<ReleaseCapabilityAttr>(A))
+    } else if (const auto *RC = dyn_cast<ReleaseCapabilityAttr>(A))
       Args = llvm::ArrayRef(RC->args_begin(), RC->args_size());
 
     if (Arg && !Finder.TraverseStmt(Arg))
diff --git a/clang/test/Sema/attr-capabilities.c b/clang/test/Sema/attr-capabilities.c
index 5138803bd5eb7..12b18b687803e 100644
--- a/clang/test/Sema/attr-capabilities.c
+++ b/clang/test/Sema/attr-capabilities.c
@@ -14,7 +14,7 @@ struct __attribute__((capability("custom"))) CustomName {};
 int Test1 __attribute__((capability("test1")));  // expected-error {{'capability' attribute only applies to structs, unions, classes, and typedefs}}
 int Test2 __attribute__((shared_capability("test2"))); // expected-error {{'shared_capability' attribute only applies to structs, unions, classes, and typedefs}}
 int Test3 __attribute__((acquire_capability("test3")));  // expected-warning {{'acquire_capability' attribute only applies to functions}}
-int Test4 __attribute__((try_acquire_capability("test4"))); // expected-error {{'try_acquire_capability' attribute only applies to functions}}
+int Test4 __attribute__((try_acquire_capability("test4"))); // expected-warning {{'try_acquire_capability' attribute only applies to functions}}
 int Test5 __attribute__((release_capability("test5"))); // expected-warning {{'release_capability' attribute only applies to functions}}
 
 struct __attribute__((capability(12))) Test3 {}; // expected-error {{expected string literal as argument of 'capability' attribute}}
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index 40e1197bc21f1..4192faee1af80 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -7977,42 +7977,6 @@ TEST_P(ImportAttributes, ImportAcquiredBefore) {
   checkImportVariadicArg(FromAttr->args(), ToAttr->args());
 }
 
-TEST_P(ImportAttributes, ImportAssertExclusiveLock) {
-  AssertExclusiveLockAttr *FromAttr, *ToAttr;
-  importAttr<FunctionDecl>("void test(int A1, int A2) "
-                           "__attribute__((assert_exclusive_lock(A1, A2)));",
-                           FromAttr, ToAttr);
-  checkImportVariadicArg(FromAttr->args(), ToAttr->args());
-}
-
-TEST_P(ImportAttributes, ImportAssertSharedLock) {
-  AssertSharedLockAttr *FromAttr, *ToAttr;
-  importAttr<FunctionDecl>(
-      "void test(int A1, int A2) __attribute__((assert_shared_lock(A1, A2)));",
-      FromAttr, ToAttr);
-  checkImportVariadicArg(FromAttr->args(), ToAttr->args());
-}
-
-TEST_P(ImportAttributes, ImportExclusiveTrylockFunction) {
-  ExclusiveTrylockFunctionAttr *FromAttr, *ToAttr;
-  importAttr<FunctionDecl>(
-      "void test(int A1, int A2) __attribute__((exclusive_trylock_function(1, "
-      "A1, A2)));",
-      FromAttr, ToAttr);
-  checkImported(FromAttr->getSuccessValue(), ToAttr->getSuccessValue());
-  checkImportVariadicArg(FromAttr->args(), ToAttr->args());
-}
-
-TEST_P(ImportAttributes, ImportSharedTrylockFunction) {
-  SharedTrylockFunctionAttr *FromAttr, *ToAttr;
-  importAttr<FunctionDecl>(
-      "void test(int A1, int A2) __attribute__((shared_trylock_function(1, A1, "
-      "A2)));",
-      FromAttr, ToAttr);
-  checkImported(FromAttr->getSuccessValue(), ToAttr->getSuccessValue());
-  checkImportVariadicArg(FromAttr->args(), ToAttr->args());
-}
-
 TEST_P(ImportAttributes, ImportLockReturned) {
   LockReturnedAttr *FromAttr, *ToAttr;
   importAttr<FunctionDecl>(

From 7cb7b2d39d3b4aa984bfaeaf5e69fbfb074edd41 Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <alex_toresh@yahoo.fr>
Date: Tue, 15 Apr 2025 17:37:01 -0400
Subject: [PATCH 045/710] [llvm] Build Windows release package with clang-cl if
 possible (#135446)

If `clang-cl.exe` and `lld-link.exe` are installed in `%PATH%`, the
Windows release build script will now use these by default, in place of
MSVC. The reason for doing this is that MSVC still has, for the past
year(s), a O(N^2) behavior when building certain LLVM source files,
which leads to long build times (minutes per file). A report was filled
here:
https://developercommunity.visualstudio.com/t/ON2-in-SparseBitVectorBase-when-com/10657991

Also added a `--force-msvc` option to the script, to use MSVC even if
clang-cl is installed.
---
 llvm/utils/release/build_llvm_release.bat | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/release/build_llvm_release.bat b/llvm/utils/release/build_llvm_release.bat
index 588d7201fcb92..3042fc2d77dd1 100755
--- a/llvm/utils/release/build_llvm_release.bat
+++ b/llvm/utils/release/build_llvm_release.bat
@@ -7,7 +7,7 @@ goto begin
 echo Script for building the LLVM installer on Windows,
 echo used for the releases at https://github.com/llvm/llvm-project/releases
 echo.
-echo Usage: build_llvm_release.bat --version ^<version^> [--x86,--x64, --arm64] [--skip-checkout] [--local-python]
+echo Usage: build_llvm_release.bat --version ^<version^> [--x86,--x64, --arm64] [--skip-checkout] [--local-python] [--force-msvc]
 echo.
 echo Options:
 echo --version: [required] version to build
@@ -17,6 +17,7 @@ echo --x64: build and test x64 variant
 echo --arm64: build and test arm64 variant
 echo --skip-checkout: use local git checkout instead of downloading src.zip
 echo --local-python: use installed Python and does not try to use a specific version (3.10)
+echo --force-msvc: use MSVC compiler for stage0, even if clang-cl is present
 echo.
 echo Note: At least one variant to build is required.
 echo.
@@ -34,6 +35,7 @@ set x64=
 set arm64=
 set skip-checkout=
 set local-python=
+set force-msvc=
 call :parse_args %*
 
 if "%help%" NEQ "" goto usage
@@ -165,6 +167,24 @@ set common_cmake_flags=^
   -DLLVM_ENABLE_RPMALLOC=ON ^
   -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;lld;compiler-rt;lldb;openmp"
 
+if "%force-msvc%" == "" (
+  where /q clang-cl
+  if errorlevel 0 (
+    where /q lld-link
+    if errorlevel 0 (
+      set common_compiler_flags=%common_compiler_flags% -fuse-ld=lld
+      
+      set common_cmake_flags=%common_cmake_flags%^
+        -DCMAKE_C_COMPILER=clang-cl.exe ^
+        -DCMAKE_CXX_COMPILER=clang-cl.exe ^
+        -DCMAKE_LINKER=lld-link.exe ^
+        -DLLVM_ENABLE_LLD=ON ^
+        -DCMAKE_C_FLAGS="%common_compiler_flags%" ^
+        -DCMAKE_CXX_FLAGS="%common_compiler_flags%"
+    )
+  )
+)
+
 set cmake_profile_flags=""
 
 REM Preserve original path

From d0372179fbbcb7b3fa680a78919a980fa4384c46 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 15 Apr 2025 14:43:28 -0700
Subject: [PATCH 046/710] [nfc] Add doc comment for `canReturn` in
 Analysis/CFG.h (#135862)

---
 llvm/include/llvm/Analysis/CFG.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Analysis/CFG.h b/llvm/include/llvm/Analysis/CFG.h
index 8451e88146d7c..052ffb2872af7 100644
--- a/llvm/include/llvm/Analysis/CFG.h
+++ b/llvm/include/llvm/Analysis/CFG.h
@@ -174,7 +174,10 @@ bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI) {
 
   return false;
 }
+
+/// Return true if there is at least a path through which F can return, false if
+/// there is no such path.
 bool canReturn(const Function &F);
-} // End llvm namespace
+} // namespace llvm
 
 #endif

From 2d98bdc12c291523c3543ceaf1c526e25dcaedc6 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn@google.com>
Date: Tue, 15 Apr 2025 14:46:55 -0700
Subject: [PATCH 047/710] =?UTF-8?q?Revert=20"[llvm][clang]=20Allocate=20a?=
 =?UTF-8?q?=20new=20stack=20instead=20of=20spawning=20a=20new=20=E2=80=A6?=
 =?UTF-8?q?=20(#135865)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…thread to get more stack space (#133173)"

This change breaks the Clang build on Mac AArch64.

This reverts commit d0c973a7a0149db3b71767d4c5a20a31e6a8ed5b. This
reverts commit 429a84f8a4bf559f43f50072747ef49d3e3b2cf1. This reverts
commit 4f64c80d5a23c244f942193e58ecac666c173308.
---
 clang/docs/ReleaseNotes.rst                   |   4 -
 clang/include/clang/Basic/Stack.h             |   5 +-
 clang/lib/Basic/Stack.cpp                     |  40 ++++--
 clang/lib/Frontend/CompilerInstance.cpp       |   2 +-
 .../llvm/Support/CrashRecoveryContext.h       |   3 -
 llvm/include/llvm/Support/ProgramStack.h      |  63 ----------
 llvm/include/llvm/Support/thread.h            |   1 -
 llvm/lib/Support/CMakeLists.txt               |   1 -
 llvm/lib/Support/CrashRecoveryContext.cpp     |  11 --
 llvm/lib/Support/ProgramStack.cpp             | 114 ------------------
 llvm/unittests/Support/CMakeLists.txt         |   1 -
 llvm/unittests/Support/ProgramStackTest.cpp   |  35 ------
 12 files changed, 30 insertions(+), 250 deletions(-)
 delete mode 100644 llvm/include/llvm/Support/ProgramStack.h
 delete mode 100644 llvm/lib/Support/ProgramStack.cpp
 delete mode 100644 llvm/unittests/Support/ProgramStackTest.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 38142ad32bea0..166f26921cb71 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -195,10 +195,6 @@ Non-comprehensive list of changes in this release
 - Added `__builtin_elementwise_exp10`.
 - For AMDPGU targets, added `__builtin_v_cvt_off_f32_i4` that maps to the `v_cvt_off_f32_i4` instruction.
 - Added `__builtin_elementwise_minnum` and `__builtin_elementwise_maxnum`.
-- Clang itself now uses split stacks instead of threads for allocating more
-  stack space when running on Apple AArch64 based platforms. This means that
-  stack traces of Clang from debuggers, crashes, and profilers may look
-  different than before.
 
 New Compiler Flags
 ------------------
diff --git a/clang/include/clang/Basic/Stack.h b/clang/include/clang/Basic/Stack.h
index 9674b9d9b62c3..30ebd94aedd1f 100644
--- a/clang/include/clang/Basic/Stack.h
+++ b/clang/include/clang/Basic/Stack.h
@@ -27,10 +27,7 @@ namespace clang {
 
   /// Call this once on each thread, as soon after starting the thread as
   /// feasible, to note the approximate address of the bottom of the stack.
-  ///
-  /// \param ForceSet set to true if you know the call is near the bottom of a
-  ///                 new stack. Used for split stacks.
-  void noteBottomOfStack(bool ForceSet = false);
+  void noteBottomOfStack();
 
   /// Determine whether the stack is nearly exhausted.
   bool isStackNearlyExhausted();
diff --git a/clang/lib/Basic/Stack.cpp b/clang/lib/Basic/Stack.cpp
index 8cbb84943f8d3..aa15d8e66950f 100644
--- a/clang/lib/Basic/Stack.cpp
+++ b/clang/lib/Basic/Stack.cpp
@@ -13,13 +13,33 @@
 
 #include "clang/Basic/Stack.h"
 #include "llvm/Support/CrashRecoveryContext.h"
-#include "llvm/Support/ProgramStack.h"
 
-static LLVM_THREAD_LOCAL uintptr_t BottomOfStack = 0;
+#ifdef _MSC_VER
+#include <intrin.h>  // for _AddressOfReturnAddress
+#endif
 
-void clang::noteBottomOfStack(bool ForceSet) {
-  if (!BottomOfStack || ForceSet)
-    BottomOfStack = llvm::getStackPointer();
+static LLVM_THREAD_LOCAL void *BottomOfStack = nullptr;
+
+static void *getStackPointer() {
+#if __GNUC__ || __has_builtin(__builtin_frame_address)
+  return __builtin_frame_address(0);
+#elif defined(_MSC_VER)
+  return _AddressOfReturnAddress();
+#else
+  char CharOnStack = 0;
+  // The volatile store here is intended to escape the local variable, to
+  // prevent the compiler from optimizing CharOnStack into anything other
+  // than a char on the stack.
+  //
+  // Tested on: MSVC 2015 - 2019, GCC 4.9 - 9, Clang 3.2 - 9, ICC 13 - 19.
+  char *volatile Ptr = &CharOnStack;
+  return Ptr;
+#endif
+}
+
+void clang::noteBottomOfStack() {
+  if (!BottomOfStack)
+    BottomOfStack = getStackPointer();
 }
 
 bool clang::isStackNearlyExhausted() {
@@ -31,8 +51,7 @@ bool clang::isStackNearlyExhausted() {
   if (!BottomOfStack)
     return false;
 
-  intptr_t StackDiff =
-      (intptr_t)llvm::getStackPointer() - (intptr_t)BottomOfStack;
+  intptr_t StackDiff = (intptr_t)getStackPointer() - (intptr_t)BottomOfStack;
   size_t StackUsage = (size_t)std::abs(StackDiff);
 
   // If the stack pointer has a surprising value, we do not understand this
@@ -47,12 +66,9 @@ bool clang::isStackNearlyExhausted() {
 void clang::runWithSufficientStackSpaceSlow(llvm::function_ref<void()> Diag,
                                             llvm::function_ref<void()> Fn) {
   llvm::CrashRecoveryContext CRC;
-  // Preserve the BottomOfStack in case RunSafelyOnNewStack uses split stacks.
-  uintptr_t PrevBottom = BottomOfStack;
-  CRC.RunSafelyOnNewStack([&] {
-    noteBottomOfStack(true);
+  CRC.RunSafelyOnThread([&] {
+    noteBottomOfStack();
     Diag();
     Fn();
   }, DesiredStackSize);
-  BottomOfStack = PrevBottom;
 }
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 5fe80fc16482e..243e0a3c15b05 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1265,7 +1265,7 @@ bool CompilerInstance::compileModule(SourceLocation ImportLoc,
 
   // Execute the action to actually build the module in-place. Use a separate
   // thread so that we get a stack large enough.
-  bool Crashed = !llvm::CrashRecoveryContext().RunSafelyOnNewStack(
+  bool Crashed = !llvm::CrashRecoveryContext().RunSafelyOnThread(
       [&]() {
         GenerateModuleFromModuleMapAction Action;
         Instance.ExecuteAction(Action);
diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h
index 31293d6715757..26ddf97b3ef02 100644
--- a/llvm/include/llvm/Support/CrashRecoveryContext.h
+++ b/llvm/include/llvm/Support/CrashRecoveryContext.h
@@ -97,9 +97,6 @@ class CrashRecoveryContext {
     return RunSafelyOnThread([&]() { Fn(UserData); }, RequestedStackSize);
   }
 
-  bool RunSafelyOnNewStack(function_ref<void()>,
-                           unsigned RequestedStackSize = 0);
-
   /// Explicitly trigger a crash recovery in the current process, and
   /// return failure from RunSafely(). This function does not return.
   [[noreturn]] void HandleExit(int RetCode);
diff --git a/llvm/include/llvm/Support/ProgramStack.h b/llvm/include/llvm/Support/ProgramStack.h
deleted file mode 100644
index 232a7b5670b44..0000000000000
--- a/llvm/include/llvm/Support/ProgramStack.h
+++ /dev/null
@@ -1,63 +0,0 @@
-//===--- ProgramStack.h -----------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_PROGRAMSTACK_H
-#define LLVM_SUPPORT_PROGRAMSTACK_H
-
-#include "llvm/ADT/STLFunctionalExtras.h"
-
-// LLVM_HAS_SPLIT_STACKS is exposed in the header because CrashRecoveryContext
-// needs to know if it's running on another thread or not.
-//
-// Currently only Apple AArch64 is known to support split stacks in the debugger
-// and other tooling.
-#if defined(__APPLE__) && defined(__aarch64__) &&                              \
-    LLVM_HAS_CPP_ATTRIBUTE(gnu::naked) && __has_extension(gnu_asm)
-# define LLVM_HAS_SPLIT_STACKS
-# define LLVM_HAS_SPLIT_STACKS_AARCH64
-#endif
-
-namespace llvm {
-
-/// \returns an address close to the current value of the stack pointer.
-///
-/// The value is not guaranteed to point to anything specific. It can be used to
-/// estimate how much stack space has been used since the previous call.
-uintptr_t getStackPointer();
-
-/// \returns the default stack size for this platform.
-///
-/// Based on \p RLIMIT_STACK or the equivalent.
-unsigned getDefaultStackSize();
-
-/// Runs Fn on a new stack of at least the given size.
-///
-/// \param StackSize requested stack size. A size of 0 uses the default stack
-///                  size of the platform.
-///
-/// The preferred implementation is split stacks on platforms that have a good
-/// debugging experience for them. On other platforms a new thread is used.
-void runOnNewStack(unsigned StackSize, function_ref<void()> Fn);
-
-template <typename R, typename... Ts>
-std::enable_if_t<!std::is_same_v<R, void>, R>
-runOnNewStack(unsigned StackSize, function_ref<R(Ts...)> Fn, Ts &&...Args) {
-  std::optional<R> Ret;
-  runOnNewStack(StackSize, [&]() { Ret = Fn(std::forward<Ts>(Args)...); });
-  return std::move(*Ret);
-}
-
-template <typename... Ts>
-void runOnNewStack(unsigned StackSize, function_ref<void(Ts...)> Fn,
-                   Ts &&...Args) {
-  runOnNewStack(StackSize, [&]() { Fn(std::forward<Ts>(Args)...); });
-}
-
-} // namespace llvm
-
-#endif // LLVM_SUPPORT_PROGRAMSTACK_H
diff --git a/llvm/include/llvm/Support/thread.h b/llvm/include/llvm/Support/thread.h
index ef2fba822cb1c..e3005fdb63175 100644
--- a/llvm/include/llvm/Support/thread.h
+++ b/llvm/include/llvm/Support/thread.h
@@ -213,7 +213,6 @@ inline thread::id get_id() { return std::this_thread::get_id(); }
 
 #else // !LLVM_ENABLE_THREADS
 
-#include "llvm/Support/ErrorHandling.h"
 #include <utility>
 
 namespace llvm {
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index def37f3f278d0..98ffd829d80b8 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -295,7 +295,6 @@ add_llvm_component_library(LLVMSupport
   Path.cpp
   Process.cpp
   Program.cpp
-  ProgramStack.cpp
   RWMutex.cpp
   Signals.cpp
   Threading.cpp
diff --git a/llvm/lib/Support/CrashRecoveryContext.cpp b/llvm/lib/Support/CrashRecoveryContext.cpp
index 88c38d7526e71..f53aea177d612 100644
--- a/llvm/lib/Support/CrashRecoveryContext.cpp
+++ b/llvm/lib/Support/CrashRecoveryContext.cpp
@@ -10,7 +10,6 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ExitCodes.h"
-#include "llvm/Support/ProgramStack.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/thread.h"
 #include <cassert>
@@ -524,13 +523,3 @@ bool CrashRecoveryContext::RunSafelyOnThread(function_ref<void()> Fn,
     CRC->setSwitchedThread();
   return Info.Result;
 }
-
-bool CrashRecoveryContext::RunSafelyOnNewStack(function_ref<void()> Fn,
-                                               unsigned RequestedStackSize) {
-#ifdef LLVM_HAS_SPLIT_STACKS
-  return runOnNewStack(RequestedStackSize,
-                       function_ref<bool()>([&]() { return RunSafely(Fn); }));
-#else
-  return RunSafelyOnThread(Fn, RequestedStackSize);
-#endif
-}
diff --git a/llvm/lib/Support/ProgramStack.cpp b/llvm/lib/Support/ProgramStack.cpp
deleted file mode 100644
index 9e5a546b34974..0000000000000
--- a/llvm/lib/Support/ProgramStack.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-//===--- RunOnNewStack.cpp - Crash Recovery -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/ProgramStack.h"
-#include "llvm/Config/config.h"
-#include "llvm/Support/Compiler.h"
-
-#ifdef LLVM_ON_UNIX
-# include <sys/resource.h> // for getrlimit
-#endif
-
-#ifdef _MSC_VER
-# include <intrin.h>  // for _AddressOfReturnAddress
-#endif
-
-#ifndef LLVM_HAS_SPLIT_STACKS
-# include "llvm/Support/thread.h"
-#endif
-
-using namespace llvm;
-
-uintptr_t llvm::getStackPointer() {
-#if __GNUC__ || __has_builtin(__builtin_frame_address)
-  return (uintptr_t)__builtin_frame_address(0);
-#elif defined(_MSC_VER)
-  return (uintptr_t)_AddressOfReturnAddress();
-#else
-  volatile char CharOnStack = 0;
-  // The volatile store here is intended to escape the local variable, to
-  // prevent the compiler from optimizing CharOnStack into anything other
-  // than a char on the stack.
-  //
-  // Tested on: MSVC 2015 - 2019, GCC 4.9 - 9, Clang 3.2 - 9, ICC 13 - 19.
-  char *volatile Ptr = &CharOnStack;
-  return (uintptr_t)Ptr;
-#endif
-}
-
-unsigned llvm::getDefaultStackSize() {
-#ifdef LLVM_ON_UNIX
-  rlimit RL;
-  getrlimit(RLIMIT_STACK, &RL);
-  return RL.rlim_cur;
-#else
-  // Clang recursively parses, instantiates templates, and evaluates constant
-  // expressions. We've found 8MiB to be a reasonable stack size given the way
-  // Clang works and the way C++ is commonly written.
-  return 8 << 20;
-#endif
-}
-
-namespace {
-#ifdef LLVM_HAS_SPLIT_STACKS_AARCH64
-[[gnu::naked]] void runOnNewStackImpl(void *Stack, void (*Fn)(void *),
-                                      void *Ctx) {
-  __asm__ volatile(
-      "mov       x16, sp\n\t"
-      "sub       x0, x0, #0x20\n\t"            // subtract space from stack
-      "stp       xzr, x16, [x0, #0x00]\n\t"    // save old sp
-      "stp       x29, x30, [x0, #0x10]\n\t"    // save fp, lr
-      "mov       sp, x0\n\t"                   // switch to new stack
-      "add       x29, x0, #0x10\n\t"           // switch to new frame
-      ".cfi_def_cfa w29, 16\n\t"
-      ".cfi_offset w30, -8\n\t"                // lr
-      ".cfi_offset w29, -16\n\t"               // fp
-
-      "mov       x0, x2\n\t"                   // Ctx is the only argument
-      "blr       x1\n\t"                       // call Fn
-
-      "ldp       x29, x30, [sp, #0x10]\n\t"    // restore fp, lr
-      "ldp       xzr, x16, [sp, #0x00]\n\t"    // load old sp
-      "mov       sp, x16\n\t"
-      "ret"
-  );
-}
-#endif
-
-#ifdef LLVM_HAS_SPLIT_STACKS
-void callback(void *Ctx) {
-  (*reinterpret_cast<function_ref<void()> *>(Ctx))();
-}
-#endif
-} // namespace
-
-#ifdef LLVM_HAS_SPLIT_STACKS
-void llvm::runOnNewStack(unsigned StackSize, function_ref<void()> Fn) {
-  if (StackSize == 0)
-    StackSize = getDefaultStackSize();
-
-  // We use malloc here instead of mmap because:
-  //   - it's simpler,
-  //   - many malloc implementations will reuse the allocation in cases where
-  //     we're bouncing accross the edge of a stack boundry, and
-  //   - many malloc implemenations will already provide guard pages for
-  //     allocations this large.
-  void *Stack = malloc(StackSize);
-  void *BottomOfStack = (char *)Stack + StackSize;
-
-  runOnNewStackImpl(BottomOfStack, callback, &Fn);
-
-  free(Stack);
-}
-#else
-void llvm::runOnNewStack(unsigned StackSize, function_ref<void()> Fn) {
-  llvm::thread Thread(
-      StackSize == 0 ? std::nullopt : std::optional<unsigned>(StackSize), Fn);
-  Thread.join();
-}
-#endif
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index e5bf820fb4d1c..6c4e7cb689b20 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -70,7 +70,6 @@ add_llvm_unittest(SupportTests
   PerThreadBumpPtrAllocatorTest.cpp
   ProcessTest.cpp
   ProgramTest.cpp
-  ProgramStackTest.cpp
   RecyclerTest.cpp
   RegexTest.cpp
   ReverseIterationTest.cpp
diff --git a/llvm/unittests/Support/ProgramStackTest.cpp b/llvm/unittests/Support/ProgramStackTest.cpp
deleted file mode 100644
index 31dfb3b88ade6..0000000000000
--- a/llvm/unittests/Support/ProgramStackTest.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-//===- unittest/Support/ProgramStackTest.cpp ------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/ProgramStack.h"
-#include "llvm/Support/Process.h"
-#include "gtest/gtest.h"
-
-using namespace llvm;
-
-static uintptr_t func(int &A) {
-  A = 7;
-  return getStackPointer();
-}
-
-static void func2(int &A) {
-  A = 5;
-}
-
-TEST(ProgramStackTest, runOnNewStack) {
-  int A = 0;
-  uintptr_t Stack = runOnNewStack(0, function_ref<uintptr_t(int &)>(func), A);
-  EXPECT_EQ(A, 7);
-  intptr_t StackDiff = (intptr_t)llvm::getStackPointer() - (intptr_t)Stack;
-  size_t StackDistance = (size_t)std::abs(StackDiff);
-  // Page size is used as it's large enough to guarantee were not on the same
-  // stack but not too large to cause spurious failures.
-  EXPECT_GT(StackDistance, llvm::sys::Process::getPageSizeEstimate());
-  runOnNewStack(0, function_ref<void(int &)>(func2), A);
-  EXPECT_EQ(A, 5);
-}

From 8ed397d8e4d014ecc5df89a9d908c5808f201b65 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 15 Apr 2025 14:58:15 -0700
Subject: [PATCH 048/710] [DAGCombiner] Disable narrowExtractedVectorLoad for
 indexed loads. (#135847)

The later code does not expect or preserve the index output.

Fixes #135821
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  2 +-
 llvm/test/CodeGen/AArch64/pr135821.ll         | 27 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AArch64/pr135821.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index b322fe670d4a7..d72be359867ca 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -25183,7 +25183,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
     return SDValue();
 
   auto *Ld = dyn_cast<LoadSDNode>(Extract->getOperand(0));
-  if (!Ld || Ld->getExtensionType() || !Ld->isSimple())
+  if (!Ld || !ISD::isNormalLoad(Ld) || !Ld->isSimple())
     return SDValue();
 
   // Allow targets to opt-out.
diff --git a/llvm/test/CodeGen/AArch64/pr135821.ll b/llvm/test/CodeGen/AArch64/pr135821.ll
new file mode 100644
index 0000000000000..cfd6cd086e130
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/pr135821.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu  | FileCheck %s
+
+define <4 x float> @f(ptr  %0) {
+; CHECK-LABEL: f:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    ldr q1, [x0, #56]!
+; CHECK-NEXT:    ldr d0, [x0, #16]
+; CHECK-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl use
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+  %2 = getelementptr inbounds nuw i8, ptr %0, i64 56
+  %3 = load <6 x float>, ptr %2, align 4
+  %4 = shufflevector <6 x float> %3, <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  tail call void @use(ptr %2)
+  ret <4 x float> %4
+}
+
+declare void @use(ptr)

From a6208ce4c15142c26c6b73651bf466ae6b470cb0 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 15 Apr 2025 15:07:03 -0700
Subject: [PATCH 049/710] [nfc] move `isPresplitCoroSuspendExitEdge` to
 Analysis/CFG (#135849)

---
 llvm/include/llvm/Analysis/CFG.h              | 19 +++++++++++++++++++
 .../llvm/Transforms/Utils/BasicBlockUtils.h   | 18 ------------------
 llvm/lib/Analysis/CFG.cpp                     | 13 +++++++++++++
 .../Instrumentation/InstrProfiling.cpp        |  1 +
 .../Instrumentation/PGOCtxProfFlattening.cpp  |  1 +
 llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 12 ------------
 6 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/llvm/include/llvm/Analysis/CFG.h b/llvm/include/llvm/Analysis/CFG.h
index 052ffb2872af7..64e2079df9db2 100644
--- a/llvm/include/llvm/Analysis/CFG.h
+++ b/llvm/include/llvm/Analysis/CFG.h
@@ -175,6 +175,25 @@ bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI) {
   return false;
 }
 
+// Returns true if these basic blocks belong to a presplit coroutine and the
+// edge corresponds to the 'default' case in the switch statement in the
+// pattern:
+//
+// %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+// switch i8 %0, label %suspend [i8 0, label %resume
+//                               i8 1, label %cleanup]
+//
+// i.e. the edge to the `%suspend` BB. This edge is special in that it will
+// be elided by coroutine lowering (coro-split), and the `%suspend` BB needs
+// to be kept as-is. It's not a real CFG edge - post-lowering, it will end
+// up being a `ret`, and it must be thus lowerable to support symmetric
+// transfer. For example:
+//  - this edge is not a loop exit edge if encountered in a loop (and should
+//    be ignored)
+//  - must not be split for PGO instrumentation, for example.
+bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src,
+                                   const BasicBlock &Dest);
+
 /// Return true if there is at least a path through which F can return, false if
 /// there is no such path.
 bool canReturn(const Function &F);
diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index 6faff3d1fd8e3..adc1851c2ec2f 100644
--- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -610,24 +610,6 @@ void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder);
 // br/brcond/unreachable/ret
 bool hasOnlySimpleTerminator(const Function &F);
 
-// Returns true if these basic blocks belong to a presplit coroutine and the
-// edge corresponds to the 'default' case in the switch statement in the
-// pattern:
-//
-// %0 = call i8 @llvm.coro.suspend(token none, i1 false)
-// switch i8 %0, label %suspend [i8 0, label %resume
-//                               i8 1, label %cleanup]
-//
-// i.e. the edge to the `%suspend` BB. This edge is special in that it will
-// be elided by coroutine lowering (coro-split), and the `%suspend` BB needs
-// to be kept as-is. It's not a real CFG edge - post-lowering, it will end
-// up being a `ret`, and it must be thus lowerable to support symmetric
-// transfer. For example:
-//  - this edge is not a loop exit edge if encountered in a loop (and should
-//    be ignored)
-//  - must not be split for PGO instrumentation, for example.
-bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src,
-                                   const BasicBlock &Dest);
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_UTILS_BASICBLOCKUTILS_H
diff --git a/llvm/lib/Analysis/CFG.cpp b/llvm/lib/Analysis/CFG.cpp
index 8ced4a901557d..0d32e101ee0b4 100644
--- a/llvm/lib/Analysis/CFG.cpp
+++ b/llvm/lib/Analysis/CFG.cpp
@@ -14,6 +14,7 @@
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
@@ -356,3 +357,15 @@ bool llvm::canReturn(const Function &F) {
 
   return false;
 }
+
+bool llvm::isPresplitCoroSuspendExitEdge(const BasicBlock &Src,
+                                         const BasicBlock &Dest) {
+  assert(Src.getParent() == Dest.getParent());
+  if (!Src.getParent()->isPresplitCoroutine())
+    return false;
+  if (auto *SW = dyn_cast<SwitchInst>(Src.getTerminator()))
+    if (auto *Intr = dyn_cast<IntrinsicInst>(SW->getCondition()))
+      return Intr->getIntrinsicID() == Intrinsic::coro_suspend &&
+             SW->getDefaultDest() == &Dest;
+  return false;
+}
\ No newline at end of file
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 008c1faf0a0c3..84bf4c62c7aad 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/Attributes.h"
diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
index 05f364a974c6c..508a41684ed20 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
@@ -21,6 +21,7 @@
 #include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/CtxProfAnalysis.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/IR/Analysis.h"
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 6f36e24000aa5..b78270f6309ff 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1916,15 +1916,3 @@ bool llvm::hasOnlySimpleTerminator(const Function &F) {
   }
   return true;
 }
-
-bool llvm::isPresplitCoroSuspendExitEdge(const BasicBlock &Src,
-                                         const BasicBlock &Dest) {
-  assert(Src.getParent() == Dest.getParent());
-  if (!Src.getParent()->isPresplitCoroutine())
-    return false;
-  if (auto *SW = dyn_cast<SwitchInst>(Src.getTerminator()))
-    if (auto *Intr = dyn_cast<IntrinsicInst>(SW->getCondition()))
-      return Intr->getIntrinsicID() == Intrinsic::coro_suspend &&
-             SW->getDefaultDest() == &Dest;
-  return false;
-}

From 6e2bca840df9dfcffc5068c1ad0c9575f0c57e76 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Tue, 15 Apr 2025 22:07:57 +0000
Subject: [PATCH 050/710] [gn build] Port 2d98bdc12c29

---
 llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn       | 1 -
 llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn | 1 -
 2 files changed, 2 deletions(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 0d2330cba6a7a..3a9f43b1070a7 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -124,7 +124,6 @@ static_library("Support") {
     "Parallel.cpp",
     "PluginLoader.cpp",
     "PrettyStackTrace.cpp",
-    "ProgramStack.cpp",
     "RISCVAttributeParser.cpp",
     "RISCVAttributes.cpp",
     "RISCVISAUtils.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index 19418ad52147b..bf6a0b7523279 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -71,7 +71,6 @@ unittest("SupportTests") {
     "Path.cpp",
     "PerThreadBumpPtrAllocatorTest.cpp",
     "ProcessTest.cpp",
-    "ProgramStackTest.cpp",
     "ProgramTest.cpp",
     "RISCVAttributeParserTest.cpp",
     "RecyclerTest.cpp",

From 31f39c83259401a26b3660dd75f645002258571d Mon Sep 17 00:00:00 2001
From: Jun Wang <jwang86@yahoo.com>
Date: Tue, 15 Apr 2025 15:17:33 -0700
Subject: [PATCH 051/710] [AMDGPU] Remove the AnnotateKernelFeatures pass
 (#130198)

Previously the AnnotateKernelFeatures pass infers two attributes:
amdgpu-calls and amdgpu-stack-objects, which are used to help determine
if flat scratch init is allowed. PR #118907 created the
amdgpu-no-flat-scratch-init attribute. Continuing with that work, this
patch makes use of this attribute to determine flat scratch init,
replacing amdgpu-calls and amdgpu-stack-objects. This also leads to the
removal of the AnnotateKernelFeatures pass.
---
 llvm/lib/Target/AMDGPU/AMDGPU.h               |    3 -
 .../AMDGPU/AMDGPUAnnotateKernelFeatures.cpp   |    9 -
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |    6 -
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |    7 -
 llvm/lib/Target/AMDGPU/GCNSubtarget.cpp       |   15 +-
 .../AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll  |  368 ++++-
 .../AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll  |  390 ++++-
 .../AMDGPU/GlobalISel/extractelement.ll       |   71 +-
 .../AMDGPU/GlobalISel/flat-scratch-init.ll    |    4 +-
 ...licit-kernarg-backend-usage-global-isel.ll |   30 +-
 .../GlobalISel/insertelement-stack-lower.ll   |    2 +-
 .../AMDGPU/GlobalISel/lds-global-value.ll     |    5 +-
 .../GlobalISel/llvm.amdgcn.if.break.i64.ll    |    3 +
 .../GlobalISel/llvm.amdgcn.trig.preop.ll      |   24 +
 .../test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll |   33 +
 .../test/CodeGen/AMDGPU/GlobalISel/udivrem.ll |   33 +
 .../abi-attribute-hints-undefined-behavior.ll |   18 +-
 .../AMDGPU/addrspacecast-constantexpr.ll      |   62 -
 llvm/test/CodeGen/AMDGPU/always-uniform.ll    |    3 +
 ...amdgpu-codegenprepare-fold-binop-select.ll |    3 +
 .../CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll   |    4 +-
 .../annotate-kernel-features-hsa-call.ll      |  331 ----
 .../AMDGPU/annotate-kernel-features-hsa.ll    |  165 --
 .../AMDGPU/annotate-kernel-features.ll        |  103 --
 .../attr-amdgpu-flat-work-group-size.ll       |    4 +-
 .../CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll    |    4 +-
 .../AMDGPU/attr-amdgpu-waves-per-eu.ll        |    4 +-
 .../attributor-flatscratchinit-globalisel.ll  |   21 +-
 ...utor-flatscratchinit-undefined-behavior.ll |   63 +
 ...tor-flatscratchinit-undefined-behavior2.ll |  870 +++++++++++
 llvm/test/CodeGen/AMDGPU/attributor-noopt.ll  |    4 +-
 .../callee-special-input-sgprs-fixed-abi.ll   |   40 +-
 llvm/test/CodeGen/AMDGPU/code-object-v3.ll    |    8 +-
 .../CodeGen/AMDGPU/combine-reg-or-const.ll    |    3 +
 ...dagcomb-extract-vec-elt-different-sizes.ll |    2 +
 .../AMDGPU/duplicate-attribute-indirect.ll    |   13 -
 ...cannot-create-empty-or-backward-segment.ll |    2 +-
 .../expand-scalar-carry-out-select-user.ll    |    3 +
 .../CodeGen/AMDGPU/extract_vector_elt-i8.ll   |  100 +-
 llvm/test/CodeGen/AMDGPU/fabs.f16.ll          |   66 +
 .../fast-unaligned-load-store.global.ll       |   20 +-
 llvm/test/CodeGen/AMDGPU/fcanonicalize.ll     |  236 ++-
 .../flat-for-global-subtarget-feature.ll      |    8 +-
 llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll  |   80 +-
 .../AMDGPU/fmul-2-combine-multi-use.ll        |   48 +
 llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll     |   60 +
 .../CodeGen/AMDGPU/fneg-modifier-casting.ll   |    3 +
 llvm/test/CodeGen/AMDGPU/fneg.f16.ll          |   62 +-
 llvm/test/CodeGen/AMDGPU/half.ll              |  231 +++
 .../AMDGPU/hsa-metadata-kernel-code-props.ll  |    4 +-
 llvm/test/CodeGen/AMDGPU/hsa.ll               |    4 +-
 .../AMDGPU/implicit-kernarg-backend-usage.ll  |   10 +-
 llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll   |   37 +-
 .../AMDGPU/insert_vector_elt.v2bf16.ll        |   58 +-
 .../CodeGen/AMDGPU/insert_vector_elt.v2i16.ll |  214 ++-
 .../CodeGen/AMDGPU/invalid-addrspacecast.ll   |    3 +
 .../CodeGen/AMDGPU/invalid-cast-load-i1.ll    |    2 +
 llvm/test/CodeGen/AMDGPU/kernarg-size.ll      |    2 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |   30 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.is.private.ll  |   12 +
 .../CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll   |   12 +
 .../AMDGPU/llvm.amdgcn.lds.kernel.id.ll       |    8 +-
 .../AMDGPU/llvm.amdgcn.readfirstlane.ll       |   70 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.readlane.ll    |  114 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.writelane.ll   |  126 +-
 llvm/test/CodeGen/AMDGPU/load-constant-f64.ll |    6 +
 llvm/test/CodeGen/AMDGPU/load-constant-i16.ll |  125 +-
 llvm/test/CodeGen/AMDGPU/load-constant-i32.ll |   83 +-
 llvm/test/CodeGen/AMDGPU/load-constant-i64.ll |   18 +
 llvm/test/CodeGen/AMDGPU/load-constant-i8.ll  |  164 +-
 llvm/test/CodeGen/AMDGPU/load-global-i16.ll   |  129 +-
 llvm/test/CodeGen/AMDGPU/load-global-i32.ll   |  105 +-
 llvm/test/CodeGen/AMDGPU/load-select-ptr.ll   |    3 +-
 .../CodeGen/AMDGPU/mad24-get-global-id.ll     |    2 +-
 .../match-perm-extract-vector-elt-bug.ll      |    8 +-
 llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll    |  300 ++--
 .../AMDGPU/memory-legalizer-flat-agent.ll     | 1380 +++++++++++++++++
 .../memory-legalizer-flat-nontemporal.ll      |   75 +
 .../memory-legalizer-flat-singlethread.ll     | 1380 +++++++++++++++++
 .../AMDGPU/memory-legalizer-flat-system.ll    | 1380 +++++++++++++++++
 .../AMDGPU/memory-legalizer-flat-volatile.ll  |   66 +
 .../AMDGPU/memory-legalizer-flat-wavefront.ll | 1365 ++++++++++++++++
 .../AMDGPU/memory-legalizer-flat-workgroup.ll | 1320 ++++++++++++++++
 .../AMDGPU/memory-legalizer-global-agent.ll   |  273 ++++
 .../memory-legalizer-global-nontemporal.ll    |   15 +
 .../memory-legalizer-global-singlethread.ll   |  276 ++++
 .../AMDGPU/memory-legalizer-global-system.ll  |  261 ++++
 .../memory-legalizer-global-volatile.ll       |   18 +
 .../memory-legalizer-global-wavefront.ll      |  276 ++++
 .../memory-legalizer-global-workgroup.ll      |  276 ++++
 .../memory-legalizer-local-nontemporal.ll     |    9 +
 .../AMDGPU/memory-legalizer-local-volatile.ll |    6 +
 .../memory-legalizer-private-nontemporal.ll   |   59 +-
 .../memory-legalizer-private-volatile.ll      |   30 +-
 llvm/test/CodeGen/AMDGPU/min.ll               |  210 +++
 llvm/test/CodeGen/AMDGPU/pack.v2f16.ll        |   21 +
 llvm/test/CodeGen/AMDGPU/pack.v2i16.ll        |   18 +
 .../AMDGPU/pal-simple-indirect-call.ll        |    8 -
 ...al-regcopy-and-spill-missed-at-regalloc.ll |   51 +-
 .../AMDGPU/preload-implicit-kernargs.ll       |  178 +--
 llvm/test/CodeGen/AMDGPU/preload-kernargs.ll  |  379 +++--
 llvm/test/CodeGen/AMDGPU/sad.ll               |  114 +-
 .../CodeGen/AMDGPU/scalar_to_vector.v8i16.ll  |   16 +
 .../scc-clobbered-sgpr-to-vmem-spill.ll       |  464 +++---
 .../CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll     |    2 +-
 llvm/test/CodeGen/AMDGPU/shift-i128.ll        |   24 +-
 .../CodeGen/AMDGPU/simple-indirect-call.ll    |   15 -
 llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll    |   70 +-
 .../CodeGen/AMDGPU/spill-vector-superclass.ll |    6 +-
 llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll |    6 +
 llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll  |    2 +-
 llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll  |    2 +-
 llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll   |    2 +-
 llvm/test/CodeGen/AMDGPU/trap-abis.ll         |   16 +-
 llvm/test/CodeGen/AMDGPU/udiv.ll              |   45 +
 llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll    |   91 +-
 .../AMDGPU/vgpr-spill-placement-issue61083.ll |    2 +-
 ...ine-function-info-long-branch-reg-debug.ll |    2 +-
 .../machine-function-info-long-branch-reg.ll  |    2 +-
 119 files changed, 13654 insertions(+), 1853 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior2.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 03cd45d7de6f2..4ff761ec19b3c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -95,11 +95,8 @@ void initializeAMDGPUDAGToDAGISelLegacyPass(PassRegistry &);
 
 void initializeAMDGPUAlwaysInlinePass(PassRegistry&);
 
-Pass *createAMDGPUAnnotateKernelFeaturesPass();
 Pass *createAMDGPUAttributorLegacyPass();
 void initializeAMDGPUAttributorLegacyPass(PassRegistry &);
-void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &);
-extern char &AMDGPUAnnotateKernelFeaturesID;
 
 // DPP/Iterative option enables the atomic optimizer with given strategy
 // whereas None disables the atomic optimizer.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
index a9bd41382c255..9c9fa5c6e2f0f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
@@ -52,11 +52,6 @@ class AMDGPUAnnotateKernelFeatures : public CallGraphSCCPass {
 
 char AMDGPUAnnotateKernelFeatures::ID = 0;
 
-char &llvm::AMDGPUAnnotateKernelFeaturesID = AMDGPUAnnotateKernelFeatures::ID;
-
-INITIALIZE_PASS(AMDGPUAnnotateKernelFeatures, DEBUG_TYPE,
-                "Add AMDGPU function attributes", false, false)
-
 bool AMDGPUAnnotateKernelFeatures::addFeatureAttributes(Function &F) {
   bool HaveStackObjects = false;
   bool Changed = false;
@@ -131,7 +126,3 @@ bool AMDGPUAnnotateKernelFeatures::doInitialization(CallGraph &CG) {
   TM = &TPC->getTM<TargetMachine>();
   return false;
 }
-
-Pass *llvm::createAMDGPUAnnotateKernelFeaturesPass() {
-  return new AMDGPUAnnotateKernelFeatures();
-}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 538b1b181f643..60d27a7fbef29 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -149,9 +149,3 @@ DUMMY_MACHINE_FUNCTION_PASS("amdgpu-regbanklegalize", AMDGPURegBankLegalizePass(
 DUMMY_MACHINE_FUNCTION_PASS("amdgpu-regbank-combiner", AMDGPURegBankCombinerPass())
 
 #undef DUMMY_MACHINE_FUNCTION_PASS
-
-
-#define DUMMY_CGSCC_PASS(NAME, CREATE_PASS)
-DUMMY_CGSCC_PASS("amdgpu-annotate-kernel-features", AMDGPUAnnotateKernelFeaturesPass())
-
-#undef DUMMY_CGSCC_PASS
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 5b2e0558d5664..34dacd5f9209d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -515,7 +515,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUAlwaysInlinePass(*PR);
   initializeAMDGPUSwLowerLDSLegacyPass(*PR);
   initializeAMDGPUAttributorLegacyPass(*PR);
-  initializeAMDGPUAnnotateKernelFeaturesPass(*PR);
   initializeAMDGPUAnnotateUniformValuesLegacyPass(*PR);
   initializeAMDGPUArgumentUsageInfoPass(*PR);
   initializeAMDGPUAtomicOptimizerPass(*PR);
@@ -1317,12 +1316,6 @@ void AMDGPUPassConfig::addIRPasses() {
 }
 
 void AMDGPUPassConfig::addCodeGenPrepare() {
-  if (TM->getTargetTriple().isAMDGCN()) {
-    // FIXME: This pass adds 2 hacky attributes that can be replaced with an
-    // analysis, and should be removed.
-    addPass(createAMDGPUAnnotateKernelFeaturesPass());
-  }
-
   if (TM->getTargetTriple().isAMDGCN() && EnableLowerKernelArguments)
     addPass(createAMDGPULowerKernelArgumentsPass());
 
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
index 53f5c1efd14eb..d6153ce93b451 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
@@ -601,12 +601,6 @@ GCNUserSGPRUsageInfo::GCNUserSGPRUsageInfo(const Function &F,
   const CallingConv::ID CC = F.getCallingConv();
   const bool IsKernel =
       CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL;
-  // FIXME: Should have analysis or something rather than attribute to detect
-  // calls.
-  const bool HasCalls = F.hasFnAttribute("amdgpu-calls");
-  // FIXME: This attribute is a hack, we just need an analysis on the function
-  // to look for allocas.
-  const bool HasStackObjects = F.hasFnAttribute("amdgpu-stack-objects");
 
   if (IsKernel && (!F.arg_empty() || ST.getImplicitArgNumBytes(F) != 0))
     KernargSegmentPtr = true;
@@ -629,12 +623,13 @@ GCNUserSGPRUsageInfo::GCNUserSGPRUsageInfo(const Function &F,
       DispatchID = true;
   }
 
-  // TODO: This could be refined a lot. The attribute is a poor way of
-  // detecting calls or stack objects that may require it before argument
-  // lowering.
   if (ST.hasFlatAddressSpace() && AMDGPU::isEntryFunctionCC(CC) &&
       (IsAmdHsaOrMesa || ST.enableFlatScratch()) &&
-      (HasCalls || HasStackObjects || ST.enableFlatScratch()) &&
+      // FlatScratchInit cannot be true for graphics CC if enableFlatScratch()
+      // is false.
+      (ST.enableFlatScratch() ||
+       (!AMDGPU::isGraphics(CC) &&
+        !F.hasFnAttribute("amdgpu-no-flat-scratch-init"))) &&
       !ST.flatScratchIsArchitected()) {
     FlatScratchInit = true;
   }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
index d9be677a0e58d..aeb301939e986 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
@@ -20,11 +20,14 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
 ; CI-NEXT:    s_mov_b32 m0, -1
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s2
 ; CI-NEXT:    ds_dec_rtn_u32 v2, v1, v0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
@@ -35,11 +38,14 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
 ; VI-NEXT:    s_mov_b32 m0, -1
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    ds_dec_rtn_u32 v2, v1, v0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -97,11 +103,14 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
 ; CI-NEXT:    s_mov_b32 m0, -1
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s2
 ; CI-NEXT:    ds_dec_rtn_u32 v2, v1, v0 offset:16
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
@@ -112,11 +121,14 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
 ; VI-NEXT:    s_mov_b32 m0, -1
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    ds_dec_rtn_u32 v2, v1, v0 offset:16
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -287,6 +299,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr
 ; CI-LABEL: global_atomic_dec_ret_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
@@ -302,6 +317,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr
 ; VI-LABEL: global_atomic_dec_ret_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -359,6 +377,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(ptr addrspace(1) %ou
 ; CI-LABEL: global_atomic_dec_ret_i32_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 16
@@ -376,6 +397,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(ptr addrspace(1) %ou
 ; VI-LABEL: global_atomic_dec_ret_i32_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 16
@@ -436,6 +460,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_system(ptr addrspace
 ; CI-LABEL: global_atomic_dec_ret_i32_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 16
@@ -453,6 +480,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_system(ptr addrspace
 ; VI-LABEL: global_atomic_dec_ret_i32_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 16
@@ -513,6 +543,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32(ptr addrspace(1) %ptr) #1
 ; CI-LABEL: global_atomic_dec_noret_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -525,6 +558,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32(ptr addrspace(1) %ptr) #1
 ; VI-LABEL: global_atomic_dec_noret_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -575,6 +611,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(ptr addrspace(1) %
 ; CI-LABEL: global_atomic_dec_noret_i32_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 16
@@ -589,6 +628,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(ptr addrspace(1) %
 ; VI-LABEL: global_atomic_dec_noret_i32_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 16
@@ -642,6 +684,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_system(ptr addrspa
 ; CI-LABEL: global_atomic_dec_noret_i32_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 16
@@ -656,6 +701,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_system(ptr addrspa
 ; VI-LABEL: global_atomic_dec_noret_i32_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 16
@@ -710,7 +758,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; CI-NEXT:    v_mov_b32_e32 v3, 42
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -718,6 +768,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    v_mov_b32_e32 v3, 42
 ; CI-NEXT:    flat_atomic_dec v3, v[0:1], v3 glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_wbinvl1_vol
@@ -732,7 +783,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT:    v_mov_b32_e32 v3, 42
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -740,6 +793,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, 42
 ; VI-NEXT:    flat_atomic_dec v3, v[0:1], v3 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
@@ -802,6 +856,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspa
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -819,6 +876,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspa
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -878,6 +938,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_dec_ret_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
@@ -893,6 +956,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_dec_ret_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -908,6 +974,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_dec_ret_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
@@ -922,6 +990,10 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_dec_ret_i32:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -958,6 +1030,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #1
 ; CI-LABEL: flat_atomic_dec_ret_i32_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 16
@@ -975,6 +1050,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #1
 ; VI-LABEL: flat_atomic_dec_ret_i32_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 16
@@ -992,6 +1070,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #1
 ; GFX9-LABEL: flat_atomic_dec_ret_i32_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
@@ -1006,6 +1086,10 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_dec_ret_i32_offset:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1045,6 +1129,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_system(ptr %out, ptr %
 ; CI-LABEL: flat_atomic_dec_ret_i32_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 16
@@ -1062,6 +1149,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_system(ptr %out, ptr %
 ; VI-LABEL: flat_atomic_dec_ret_i32_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 16
@@ -1079,6 +1169,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_system(ptr %out, ptr %
 ; GFX9-LABEL: flat_atomic_dec_ret_i32_offset_system:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
@@ -1093,6 +1185,10 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_system(ptr %out, ptr %
 ;
 ; GFX10-LABEL: flat_atomic_dec_ret_i32_offset_system:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1132,6 +1228,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_dec_noret_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1144,6 +1243,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_dec_noret_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1156,6 +1258,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_dec_noret_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
@@ -1167,6 +1271,10 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_dec_noret_i32:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1199,6 +1307,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_dec_noret_i32_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 16
@@ -1213,6 +1324,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_dec_noret_i32_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 16
@@ -1227,6 +1341,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_dec_noret_i32_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
@@ -1238,6 +1354,10 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_dec_noret_i32_offset:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1273,6 +1393,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_system(ptr %ptr) #1
 ; CI-LABEL: flat_atomic_dec_noret_i32_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 16
@@ -1287,6 +1410,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_system(ptr %ptr) #1
 ; VI-LABEL: flat_atomic_dec_noret_i32_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 16
@@ -1301,6 +1427,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_system(ptr %ptr) #1
 ; GFX9-LABEL: flat_atomic_dec_noret_i32_offset_system:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
@@ -1312,6 +1440,10 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_system(ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_dec_noret_i32_offset_system:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1348,7 +1480,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; CI-NEXT:    v_mov_b32_e32 v3, 42
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1356,6 +1490,7 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    v_mov_b32_e32 v3, 42
 ; CI-NEXT:    flat_atomic_dec v3, v[0:1], v3 glc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CI-NEXT:    buffer_wbinvl1_vol
@@ -1370,7 +1505,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT:    v_mov_b32_e32 v3, 42
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1378,6 +1515,7 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, 42
 ; VI-NEXT:    flat_atomic_dec v3, v[0:1], v3 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
@@ -1392,6 +1530,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
@@ -1410,6 +1550,10 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %
 ;
 ; GFX10-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 42
@@ -1466,6 +1610,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -1483,6 +1630,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -1500,6 +1650,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
@@ -1513,6 +1665,10 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1559,10 +1715,13 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_dec_ret_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1580,10 +1739,13 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_dec_ret_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1601,7 +1763,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_dec_ret_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
@@ -1616,6 +1780,10 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_dec_ret_i64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -1654,12 +1822,15 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #1
 ; CI-LABEL: flat_atomic_dec_ret_i64_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 32
 ; CI-NEXT:    s_addc_u32 s3, s3, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1677,12 +1848,15 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #1
 ; VI-LABEL: flat_atomic_dec_ret_i64_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 32
 ; VI-NEXT:    s_addc_u32 s3, s3, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1700,7 +1874,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #1
 ; GFX9-LABEL: flat_atomic_dec_ret_i64_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
@@ -1715,6 +1891,10 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_dec_ret_i64_offset:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -1756,10 +1936,13 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_dec_noret_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1769,10 +1952,13 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_dec_noret_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1782,7 +1968,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_dec_noret_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
@@ -1794,6 +1982,10 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_dec_noret_i64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -1828,12 +2020,15 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_dec_noret_i64_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 32
 ; CI-NEXT:    s_addc_u32 s1, s1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1843,12 +2038,15 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_dec_noret_i64_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 32
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1858,7 +2056,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_dec_noret_i64_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
@@ -1870,6 +2070,10 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_dec_noret_i64_offset:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -1907,12 +2111,15 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_system(ptr %ptr) #1
 ; CI-LABEL: flat_atomic_dec_noret_i64_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 32
 ; CI-NEXT:    s_addc_u32 s1, s1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1922,12 +2129,15 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_system(ptr %ptr) #1
 ; VI-LABEL: flat_atomic_dec_noret_i64_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 32
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1937,7 +2147,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_system(ptr %ptr) #1
 ; GFX9-LABEL: flat_atomic_dec_noret_i64_offset_system:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
@@ -1949,6 +2161,10 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_system(ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_dec_noret_i64_offset_system:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -1987,6 +2203,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr %
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2013,6 +2232,9 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2039,12 +2261,14 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr %
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v5, 3, v0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 42
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s3
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v5
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
 ; GFX9-NEXT:    flat_atomic_dec_x2 v[0:1], v[3:4], v[1:2] offset:40 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2058,6 +2282,10 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr %
 ;
 ; GFX10-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2116,6 +2344,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -2134,6 +2365,9 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -2152,12 +2386,14 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 42
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
 ; GFX9-NEXT:    flat_atomic_dec_x2 v[3:4], v[1:2] offset:40
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2166,6 +2402,10 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2219,8 +2459,11 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0(ptr addrspace(1) %out, ptr
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    ds_dec_rtn_u32 v2, v1, v2 offset:8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_add_i32_e32 v3, vcc, 2, v0
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    flat_store_dword v[0:1], v3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2237,8 +2480,11 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0(ptr addrspace(1) %out, ptr
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    ds_dec_rtn_u32 v2, v1, v2 offset:8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_add_u32_e32 v3, vcc, 2, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    flat_store_dword v[0:1], v3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2312,7 +2558,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1]
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
@@ -2328,7 +2577,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
@@ -2394,7 +2646,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out,
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
@@ -2410,7 +2665,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out,
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
@@ -2594,10 +2852,13 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr
 ; CI-LABEL: global_atomic_dec_ret_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -2610,10 +2871,13 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr
 ; VI-LABEL: global_atomic_dec_ret_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -2671,12 +2935,15 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(ptr addrspace(1) %ou
 ; CI-LABEL: global_atomic_dec_ret_i64_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 32
 ; CI-NEXT:    s_addc_u32 s3, s3, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -2689,12 +2956,15 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(ptr addrspace(1) %ou
 ; VI-LABEL: global_atomic_dec_ret_i64_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 32
 ; VI-NEXT:    s_addc_u32 s3, s3, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -2753,12 +3023,15 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_system(ptr addrspace
 ; CI-LABEL: global_atomic_dec_ret_i64_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 32
 ; CI-NEXT:    s_addc_u32 s3, s3, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -2771,12 +3044,15 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_system(ptr addrspace
 ; VI-LABEL: global_atomic_dec_ret_i64_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 32
 ; VI-NEXT:    s_addc_u32 s3, s3, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -2835,10 +3111,13 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64(ptr addrspace(1) %ptr) #1
 ; CI-LABEL: global_atomic_dec_noret_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -2848,10 +3127,13 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64(ptr addrspace(1) %ptr) #1
 ; VI-LABEL: global_atomic_dec_noret_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -2902,12 +3184,15 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(ptr addrspace(1) %
 ; CI-LABEL: global_atomic_dec_noret_i64_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 32
 ; CI-NEXT:    s_addc_u32 s1, s1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -2917,12 +3202,15 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(ptr addrspace(1) %
 ; VI-LABEL: global_atomic_dec_noret_i64_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 32
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -2974,12 +3262,15 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_system(ptr addrspa
 ; CI-LABEL: global_atomic_dec_noret_i64_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 32
 ; CI-NEXT:    s_addc_u32 s1, s1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -2989,12 +3280,15 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_system(ptr addrspa
 ; VI-LABEL: global_atomic_dec_noret_i64_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 32
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -3047,6 +3341,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(ptr addrspace
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -3070,6 +3367,9 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(ptr addrspace
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -3144,6 +3444,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspa
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -3162,6 +3465,9 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspa
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -3232,7 +3538,10 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(ptr addrspace(1) %out,
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:16
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v4, s3
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 2, v0
 ; CI-NEXT:    v_mov_b32_e32 v3, s2
 ; CI-NEXT:    flat_store_dword v[3:4], v0
@@ -3251,7 +3560,10 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(ptr addrspace(1) %out,
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:16
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v4, s3
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 2, v0
 ; VI-NEXT:    v_mov_b32_e32 v3, s2
 ; VI-NEXT:    flat_store_dword v[3:4], v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
index 92a7de9aaefd2..1d401a4ee33d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
@@ -21,11 +21,14 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
 ; CI-NEXT:    s_mov_b32 m0, -1
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s2
 ; CI-NEXT:    ds_inc_rtn_u32 v2, v1, v0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
@@ -36,11 +39,14 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
 ; VI-NEXT:    s_mov_b32 m0, -1
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    ds_inc_rtn_u32 v2, v1, v0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -110,11 +116,14 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out,
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
 ; CI-NEXT:    s_mov_b32 m0, -1
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s2
 ; CI-NEXT:    ds_inc_rtn_u32 v2, v1, v0 offset:16
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
@@ -125,11 +134,14 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out,
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
 ; VI-NEXT:    s_mov_b32 m0, -1
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    ds_inc_rtn_u32 v2, v1, v0 offset:16
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -332,6 +344,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr
 ; CI-LABEL: global_atomic_inc_ret_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
@@ -347,6 +362,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr
 ; VI-LABEL: global_atomic_inc_ret_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -415,6 +433,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %ou
 ; CI-LABEL: global_atomic_inc_ret_i32_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 16
@@ -432,6 +453,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %ou
 ; VI-LABEL: global_atomic_inc_ret_i32_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 16
@@ -503,6 +527,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_sistem(ptr addrspace
 ; CI-LABEL: global_atomic_inc_ret_i32_offset_sistem:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 16
@@ -520,6 +547,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_sistem(ptr addrspace
 ; VI-LABEL: global_atomic_inc_ret_i32_offset_sistem:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 16
@@ -592,6 +622,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) #1
 ; CI-LABEL: global_atomic_inc_noret_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -604,6 +637,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) #1
 ; VI-LABEL: global_atomic_inc_noret_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -664,6 +700,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) %
 ; CI-LABEL: global_atomic_inc_noret_i32_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 16
@@ -678,6 +717,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) %
 ; VI-LABEL: global_atomic_inc_noret_i32_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 16
@@ -741,6 +783,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_system(ptr addrspa
 ; CI-LABEL: global_atomic_inc_noret_i32_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 16
@@ -755,6 +800,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_system(ptr addrspa
 ; VI-LABEL: global_atomic_inc_noret_i32_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 16
@@ -820,7 +868,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; CI-NEXT:    v_mov_b32_e32 v3, 42
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -828,6 +878,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    v_mov_b32_e32 v3, 42
 ; CI-NEXT:    flat_atomic_inc v3, v[0:1], v3 glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_wbinvl1_vol
@@ -842,7 +893,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT:    v_mov_b32_e32 v3, 42
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -850,6 +903,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, 42
 ; VI-NEXT:    flat_atomic_inc v3, v[0:1], v3 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
@@ -925,6 +979,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspa
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -942,6 +999,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspa
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -1019,8 +1079,11 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(ptr addrspace(1) %out,
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    ds_inc_rtn_u32 v2, v1, v2 offset:8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_add_i32_e32 v3, vcc, 2, v0
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    flat_store_dword v[0:1], v3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1037,8 +1100,11 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(ptr addrspace(1) %out,
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    ds_inc_rtn_u32 v2, v1, v2 offset:8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_add_u32_e32 v3, vcc, 2, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    flat_store_dword v[0:1], v3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1129,7 +1195,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1]
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
@@ -1145,7 +1214,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
@@ -1224,7 +1296,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out,
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
@@ -1240,7 +1315,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out,
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
@@ -1459,10 +1537,13 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr
 ; CI-LABEL: global_atomic_inc_ret_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -1475,10 +1556,13 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr
 ; VI-LABEL: global_atomic_inc_ret_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -1548,12 +1632,15 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %ou
 ; CI-LABEL: global_atomic_inc_ret_i64_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 32
 ; CI-NEXT:    s_addc_u32 s3, s3, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -1566,12 +1653,15 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %ou
 ; VI-LABEL: global_atomic_inc_ret_i64_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 32
 ; VI-NEXT:    s_addc_u32 s3, s3, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -1642,12 +1732,15 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_system(ptr addrspace
 ; CI-LABEL: global_atomic_inc_ret_i64_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 32
 ; CI-NEXT:    s_addc_u32 s3, s3, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -1660,12 +1753,15 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_system(ptr addrspace
 ; VI-LABEL: global_atomic_inc_ret_i64_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 32
 ; VI-NEXT:    s_addc_u32 s3, s3, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -1737,10 +1833,13 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) #1
 ; CI-LABEL: global_atomic_inc_noret_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -1750,10 +1849,13 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) #1
 ; VI-LABEL: global_atomic_inc_noret_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -1815,12 +1917,15 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) %
 ; CI-LABEL: global_atomic_inc_noret_i64_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 32
 ; CI-NEXT:    s_addc_u32 s1, s1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -1830,12 +1935,15 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) %
 ; VI-LABEL: global_atomic_inc_noret_i64_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 32
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -1898,12 +2006,15 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_system(ptr addrspa
 ; CI-LABEL: global_atomic_inc_noret_i64_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 32
 ; CI-NEXT:    s_addc_u32 s1, s1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -1913,12 +2024,15 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_system(ptr addrspa
 ; VI-LABEL: global_atomic_inc_noret_i64_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 32
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -1983,6 +2097,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2006,6 +2123,9 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2094,6 +2214,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspa
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -2112,6 +2235,9 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspa
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -2188,6 +2314,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_inc_ret_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
@@ -2203,6 +2332,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_inc_ret_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -2218,6 +2350,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_inc_ret_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
@@ -2232,6 +2366,10 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_inc_ret_i32:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2281,6 +2419,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1
 ; CI-LABEL: flat_atomic_inc_ret_i32_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 16
@@ -2298,6 +2439,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1
 ; VI-LABEL: flat_atomic_inc_ret_i32_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 16
@@ -2315,6 +2459,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1
 ; GFX9-LABEL: flat_atomic_inc_ret_i32_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
@@ -2329,6 +2475,10 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_inc_ret_i32_offset:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2381,6 +2531,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_system(ptr %out, ptr %
 ; CI-LABEL: flat_atomic_inc_ret_i32_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 16
@@ -2398,6 +2551,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_system(ptr %out, ptr %
 ; VI-LABEL: flat_atomic_inc_ret_i32_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 16
@@ -2415,6 +2571,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_system(ptr %out, ptr %
 ; GFX9-LABEL: flat_atomic_inc_ret_i32_offset_system:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
@@ -2429,6 +2587,10 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_system(ptr %out, ptr %
 ;
 ; GFX10-LABEL: flat_atomic_inc_ret_i32_offset_system:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2482,6 +2644,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_inc_noret_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2494,6 +2659,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_inc_noret_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2506,6 +2674,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_inc_noret_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
@@ -2517,6 +2687,10 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_inc_noret_i32:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2560,6 +2734,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_inc_noret_i32_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 16
@@ -2574,6 +2751,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_inc_noret_i32_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 16
@@ -2588,6 +2768,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_inc_noret_i32_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
@@ -2599,6 +2781,10 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_inc_noret_i32_offset:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2645,6 +2831,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_system(ptr %ptr) #1
 ; CI-LABEL: flat_atomic_inc_noret_i32_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    v_mov_b32_e32 v2, 42
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 16
@@ -2659,6 +2848,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_system(ptr %ptr) #1
 ; VI-LABEL: flat_atomic_inc_noret_i32_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    v_mov_b32_e32 v2, 42
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 16
@@ -2673,6 +2865,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_system(ptr %ptr) #1
 ; GFX9-LABEL: flat_atomic_inc_noret_i32_offset_system:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
@@ -2684,6 +2878,10 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_system(ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_inc_noret_i32_offset_system:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2732,7 +2930,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; CI-NEXT:    v_mov_b32_e32 v3, 42
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2740,6 +2940,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 20, v0
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    v_mov_b32_e32 v3, 42
 ; CI-NEXT:    flat_atomic_inc v3, v[0:1], v3 glc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CI-NEXT:    buffer_wbinvl1_vol
@@ -2754,7 +2955,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT:    v_mov_b32_e32 v3, 42
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2762,6 +2965,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 20, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, 42
 ; VI-NEXT:    flat_atomic_inc v3, v[0:1], v3 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
@@ -2776,6 +2980,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
@@ -2794,6 +3000,10 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ;
 ; GFX10-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 42
@@ -2871,6 +3081,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -2888,6 +3101,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -2905,6 +3121,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
@@ -2918,6 +3136,10 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2988,7 +3210,10 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(ptr addrspace(1) %out,
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v4, s3
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 2, v0
 ; CI-NEXT:    v_mov_b32_e32 v3, s2
 ; CI-NEXT:    flat_store_dword v[3:4], v0
@@ -3007,7 +3232,10 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(ptr addrspace(1) %out,
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v4, s3
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 2, v0
 ; VI-NEXT:    v_mov_b32_e32 v3, s2
 ; VI-NEXT:    flat_store_dword v[3:4], v0
@@ -3097,10 +3325,13 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_inc_ret_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3118,10 +3349,13 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_inc_ret_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3139,7 +3373,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_inc_ret_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
@@ -3154,6 +3390,10 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_inc_ret_i64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -3206,12 +3446,15 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1
 ; CI-LABEL: flat_atomic_inc_ret_i64_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 32
 ; CI-NEXT:    s_addc_u32 s3, s3, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3229,12 +3472,15 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1
 ; VI-LABEL: flat_atomic_inc_ret_i64_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 32
 ; VI-NEXT:    s_addc_u32 s3, s3, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3252,7 +3498,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1
 ; GFX9-LABEL: flat_atomic_inc_ret_i64_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
@@ -3267,6 +3515,10 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_inc_ret_i64_offset:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -3322,12 +3574,15 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_system(ptr %out, ptr %
 ; CI-LABEL: flat_atomic_inc_ret_i64_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s2, s2, 32
 ; CI-NEXT:    s_addc_u32 s3, s3, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3345,12 +3600,15 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_system(ptr %out, ptr %
 ; VI-LABEL: flat_atomic_inc_ret_i64_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s2, s2, 32
 ; VI-NEXT:    s_addc_u32 s3, s3, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3368,7 +3626,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_system(ptr %out, ptr %
 ; GFX9-LABEL: flat_atomic_inc_ret_i64_offset_system:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
@@ -3383,6 +3643,10 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_system(ptr %out, ptr %
 ;
 ; GFX10-LABEL: flat_atomic_inc_ret_i64_offset_system:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -3439,10 +3703,13 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_inc_noret_i64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3452,10 +3719,13 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_inc_noret_i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3465,7 +3735,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_inc_noret_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
@@ -3477,6 +3749,10 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_inc_noret_i64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -3523,12 +3799,15 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 {
 ; CI-LABEL: flat_atomic_inc_noret_i64_offset:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 32
 ; CI-NEXT:    s_addc_u32 s1, s1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3538,12 +3817,15 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 {
 ; VI-LABEL: flat_atomic_inc_noret_i64_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 32
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3553,7 +3835,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 {
 ; GFX9-LABEL: flat_atomic_inc_noret_i64_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
@@ -3565,6 +3849,10 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 {
 ;
 ; GFX10-LABEL: flat_atomic_inc_noret_i64_offset:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -3614,12 +3902,15 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_system(ptr %ptr) #1
 ; CI-LABEL: flat_atomic_inc_noret_i64_offset_system:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
-; CI-NEXT:    v_mov_b32_e32 v1, 0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s0, s0, 32
 ; CI-NEXT:    s_addc_u32 s1, s1, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3629,12 +3920,15 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_system(ptr %ptr) #1
 ; VI-LABEL: flat_atomic_inc_noret_i64_offset_system:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s0, s0, 32
 ; VI-NEXT:    s_addc_u32 s1, s1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3644,7 +3938,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_system(ptr %ptr) #1
 ; GFX9-LABEL: flat_atomic_inc_noret_i64_offset_system:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
@@ -3656,6 +3952,10 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_system(ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_inc_noret_i64_offset_system:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -3707,6 +4007,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -3733,6 +4036,9 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -3759,12 +4065,14 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v5, 3, v0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 42
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s3
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v5
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
 ; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[3:4], v[1:2] offset:40 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3778,6 +4086,10 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
 ;
 ; GFX10-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3858,6 +4170,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -3876,6 +4191,9 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -3894,12 +4212,14 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 42
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
 ; GFX9-NEXT:    flat_atomic_inc_x2 v[3:4], v[1:2] offset:40
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3908,6 +4228,10 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
 ;
 ; GFX10-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3975,6 +4299,7 @@ define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(ptr addrspace(1) %out0,
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
 ; CI-NEXT:    s_mov_b32 m0, -1
+; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s4
 ; CI-NEXT:    ds_inc_rtn_u32 v2, v1, v0
@@ -3982,6 +4307,8 @@ define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(ptr addrspace(1) %out0,
 ; CI-NEXT:    ds_inc_rtn_u32 v3, v1, v0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
@@ -3995,6 +4322,7 @@ define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(ptr addrspace(1) %out0,
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
 ; VI-NEXT:    s_mov_b32 m0, -1
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-NEXT:    ds_inc_rtn_u32 v2, v1, v0
@@ -4002,6 +4330,8 @@ define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(ptr addrspace(1) %out0,
 ; VI-NEXT:    ds_inc_rtn_u32 v3, v1, v0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
index 31a229a908142..9ef16aef0dd16 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
@@ -3016,7 +3016,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; GPRIDX-NEXT:     kernel_code_entry_byte_offset = 256
 ; GPRIDX-NEXT:     kernel_code_prefetch_byte_size = 0
 ; GPRIDX-NEXT:     granulated_workitem_vgpr_count = 0
-; GPRIDX-NEXT:     granulated_wavefront_sgpr_count = 1
+; GPRIDX-NEXT:     granulated_wavefront_sgpr_count = 2
 ; GPRIDX-NEXT:     priority = 0
 ; GPRIDX-NEXT:     float_mode = 240
 ; GPRIDX-NEXT:     priv = 0
@@ -3027,7 +3027,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; GPRIDX-NEXT:     enable_mem_ordered = 0
 ; GPRIDX-NEXT:     enable_fwd_progress = 0
 ; GPRIDX-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; GPRIDX-NEXT:     user_sgpr_count = 12
+; GPRIDX-NEXT:     user_sgpr_count = 14
 ; GPRIDX-NEXT:     enable_trap_handler = 0
 ; GPRIDX-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; GPRIDX-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -3042,7 +3042,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; GPRIDX-NEXT:     enable_sgpr_queue_ptr = 1
 ; GPRIDX-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; GPRIDX-NEXT:     enable_sgpr_dispatch_id = 1
-; GPRIDX-NEXT:     enable_sgpr_flat_scratch_init = 0
+; GPRIDX-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; GPRIDX-NEXT:     enable_sgpr_private_segment_size = 0
 ; GPRIDX-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; GPRIDX-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
@@ -3059,7 +3059,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; GPRIDX-NEXT:     gds_segment_byte_size = 0
 ; GPRIDX-NEXT:     kernarg_segment_byte_size = 28
 ; GPRIDX-NEXT:     workgroup_fbarrier_count = 0
-; GPRIDX-NEXT:     wavefront_sgpr_count = 15
+; GPRIDX-NEXT:     wavefront_sgpr_count = 17
 ; GPRIDX-NEXT:     workitem_vgpr_count = 3
 ; GPRIDX-NEXT:     reserved_vgpr_first = 0
 ; GPRIDX-NEXT:     reserved_vgpr_count = 0
@@ -3107,7 +3107,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; MOVREL-NEXT:     kernel_code_entry_byte_offset = 256
 ; MOVREL-NEXT:     kernel_code_prefetch_byte_size = 0
 ; MOVREL-NEXT:     granulated_workitem_vgpr_count = 0
-; MOVREL-NEXT:     granulated_wavefront_sgpr_count = 1
+; MOVREL-NEXT:     granulated_wavefront_sgpr_count = 2
 ; MOVREL-NEXT:     priority = 0
 ; MOVREL-NEXT:     float_mode = 240
 ; MOVREL-NEXT:     priv = 0
@@ -3118,7 +3118,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; MOVREL-NEXT:     enable_mem_ordered = 0
 ; MOVREL-NEXT:     enable_fwd_progress = 0
 ; MOVREL-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; MOVREL-NEXT:     user_sgpr_count = 12
+; MOVREL-NEXT:     user_sgpr_count = 14
 ; MOVREL-NEXT:     enable_trap_handler = 0
 ; MOVREL-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; MOVREL-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -3133,7 +3133,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; MOVREL-NEXT:     enable_sgpr_queue_ptr = 1
 ; MOVREL-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; MOVREL-NEXT:     enable_sgpr_dispatch_id = 1
-; MOVREL-NEXT:     enable_sgpr_flat_scratch_init = 0
+; MOVREL-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; MOVREL-NEXT:     enable_sgpr_private_segment_size = 0
 ; MOVREL-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; MOVREL-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
@@ -3150,7 +3150,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; MOVREL-NEXT:     gds_segment_byte_size = 0
 ; MOVREL-NEXT:     kernarg_segment_byte_size = 28
 ; MOVREL-NEXT:     workgroup_fbarrier_count = 0
-; MOVREL-NEXT:     wavefront_sgpr_count = 10
+; MOVREL-NEXT:     wavefront_sgpr_count = 24
 ; MOVREL-NEXT:     workitem_vgpr_count = 4
 ; MOVREL-NEXT:     reserved_vgpr_first = 0
 ; MOVREL-NEXT:     reserved_vgpr_count = 0
@@ -3168,21 +3168,24 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; MOVREL-NEXT:  ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; MOVREL-NEXT:    s_load_dword s8, s[8:9], 0x8
+; MOVREL-NEXT:    s_add_i32 s12, s12, s17
+; MOVREL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; MOVREL-NEXT:    s_mov_b32 s4, 0
 ; MOVREL-NEXT:    s_mov_b32 s5, 0x40080000
-; MOVREL-NEXT:    s_mov_b32 s2, 0
-; MOVREL-NEXT:    s_mov_b32 s3, 0x40140000
 ; MOVREL-NEXT:    s_waitcnt lgkmcnt(0)
 ; MOVREL-NEXT:    s_cmp_eq_u32 s8, 1
 ; MOVREL-NEXT:    s_cselect_b64 s[6:7], 2.0, 1.0
 ; MOVREL-NEXT:    s_cmp_eq_u32 s8, 2
+; MOVREL-NEXT:    s_mov_b32 s2, 0
 ; MOVREL-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
 ; MOVREL-NEXT:    s_cmp_eq_u32 s8, 3
+; MOVREL-NEXT:    s_mov_b32 s3, 0x40140000
 ; MOVREL-NEXT:    s_cselect_b64 s[4:5], 4.0, s[4:5]
 ; MOVREL-NEXT:    s_cmp_eq_u32 s8, 4
 ; MOVREL-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[4:5]
 ; MOVREL-NEXT:    v_mov_b32_e32 v0, s2
 ; MOVREL-NEXT:    v_mov_b32_e32 v3, s1
+; MOVREL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; MOVREL-NEXT:    v_mov_b32_e32 v1, s3
 ; MOVREL-NEXT:    v_mov_b32_e32 v2, s0
 ; MOVREL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -3210,7 +3213,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; GFX10-NEXT:     enable_mem_ordered = 1
 ; GFX10-NEXT:     enable_fwd_progress = 0
 ; GFX10-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; GFX10-NEXT:     user_sgpr_count = 12
+; GFX10-NEXT:     user_sgpr_count = 14
 ; GFX10-NEXT:     enable_trap_handler = 0
 ; GFX10-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; GFX10-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -3225,7 +3228,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
 ; GFX10-NEXT:     enable_sgpr_queue_ptr = 1
 ; GFX10-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; GFX10-NEXT:     enable_sgpr_dispatch_id = 1
-; GFX10-NEXT:     enable_sgpr_flat_scratch_init = 0
+; GFX10-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; GFX10-NEXT:     enable_sgpr_private_segment_size = 0
 ; GFX10-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; GFX10-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
@@ -4042,7 +4045,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GPRIDX-NEXT:     enable_mem_ordered = 0
 ; GPRIDX-NEXT:     enable_fwd_progress = 0
 ; GPRIDX-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; GPRIDX-NEXT:     user_sgpr_count = 12
+; GPRIDX-NEXT:     user_sgpr_count = 14
 ; GPRIDX-NEXT:     enable_trap_handler = 0
 ; GPRIDX-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; GPRIDX-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -4057,7 +4060,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GPRIDX-NEXT:     enable_sgpr_queue_ptr = 1
 ; GPRIDX-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; GPRIDX-NEXT:     enable_sgpr_dispatch_id = 1
-; GPRIDX-NEXT:     enable_sgpr_flat_scratch_init = 0
+; GPRIDX-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; GPRIDX-NEXT:     enable_sgpr_private_segment_size = 0
 ; GPRIDX-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; GPRIDX-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
@@ -4074,7 +4077,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GPRIDX-NEXT:     gds_segment_byte_size = 0
 ; GPRIDX-NEXT:     kernarg_segment_byte_size = 28
 ; GPRIDX-NEXT:     workgroup_fbarrier_count = 0
-; GPRIDX-NEXT:     wavefront_sgpr_count = 14
+; GPRIDX-NEXT:     wavefront_sgpr_count = 16
 ; GPRIDX-NEXT:     workitem_vgpr_count = 2
 ; GPRIDX-NEXT:     reserved_vgpr_first = 0
 ; GPRIDX-NEXT:     reserved_vgpr_count = 0
@@ -4115,7 +4118,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:     kernel_code_entry_byte_offset = 256
 ; MOVREL-NEXT:     kernel_code_prefetch_byte_size = 0
 ; MOVREL-NEXT:     granulated_workitem_vgpr_count = 0
-; MOVREL-NEXT:     granulated_wavefront_sgpr_count = 1
+; MOVREL-NEXT:     granulated_wavefront_sgpr_count = 2
 ; MOVREL-NEXT:     priority = 0
 ; MOVREL-NEXT:     float_mode = 240
 ; MOVREL-NEXT:     priv = 0
@@ -4126,7 +4129,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:     enable_mem_ordered = 0
 ; MOVREL-NEXT:     enable_fwd_progress = 0
 ; MOVREL-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; MOVREL-NEXT:     user_sgpr_count = 12
+; MOVREL-NEXT:     user_sgpr_count = 14
 ; MOVREL-NEXT:     enable_trap_handler = 0
 ; MOVREL-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; MOVREL-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -4141,7 +4144,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:     enable_sgpr_queue_ptr = 1
 ; MOVREL-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; MOVREL-NEXT:     enable_sgpr_dispatch_id = 1
-; MOVREL-NEXT:     enable_sgpr_flat_scratch_init = 0
+; MOVREL-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; MOVREL-NEXT:     enable_sgpr_private_segment_size = 0
 ; MOVREL-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; MOVREL-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
@@ -4158,7 +4161,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:     gds_segment_byte_size = 0
 ; MOVREL-NEXT:     kernarg_segment_byte_size = 28
 ; MOVREL-NEXT:     workgroup_fbarrier_count = 0
-; MOVREL-NEXT:     wavefront_sgpr_count = 10
+; MOVREL-NEXT:     wavefront_sgpr_count = 24
 ; MOVREL-NEXT:     workitem_vgpr_count = 3
 ; MOVREL-NEXT:     reserved_vgpr_first = 0
 ; MOVREL-NEXT:     reserved_vgpr_count = 0
@@ -4176,6 +4179,9 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:  ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; MOVREL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; MOVREL-NEXT:    s_add_i32 s12, s12, s17
+; MOVREL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; MOVREL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; MOVREL-NEXT:    s_waitcnt lgkmcnt(0)
 ; MOVREL-NEXT:    s_cmp_eq_u32 s2, 1
 ; MOVREL-NEXT:    s_cselect_b32 s3, 2.0, 1.0
@@ -4211,7 +4217,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GFX10-NEXT:     enable_mem_ordered = 1
 ; GFX10-NEXT:     enable_fwd_progress = 0
 ; GFX10-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; GFX10-NEXT:     user_sgpr_count = 12
+; GFX10-NEXT:     user_sgpr_count = 14
 ; GFX10-NEXT:     enable_trap_handler = 0
 ; GFX10-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; GFX10-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -4226,7 +4232,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GFX10-NEXT:     enable_sgpr_queue_ptr = 1
 ; GFX10-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; GFX10-NEXT:     enable_sgpr_dispatch_id = 1
-; GFX10-NEXT:     enable_sgpr_flat_scratch_init = 0
+; GFX10-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; GFX10-NEXT:     enable_sgpr_private_segment_size = 0
 ; GFX10-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; GFX10-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
@@ -4387,7 +4393,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GPRIDX-NEXT:     enable_mem_ordered = 0
 ; GPRIDX-NEXT:     enable_fwd_progress = 0
 ; GPRIDX-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; GPRIDX-NEXT:     user_sgpr_count = 12
+; GPRIDX-NEXT:     user_sgpr_count = 14
 ; GPRIDX-NEXT:     enable_trap_handler = 0
 ; GPRIDX-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; GPRIDX-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -4402,7 +4408,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GPRIDX-NEXT:     enable_sgpr_queue_ptr = 1
 ; GPRIDX-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; GPRIDX-NEXT:     enable_sgpr_dispatch_id = 1
-; GPRIDX-NEXT:     enable_sgpr_flat_scratch_init = 0
+; GPRIDX-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; GPRIDX-NEXT:     enable_sgpr_private_segment_size = 0
 ; GPRIDX-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; GPRIDX-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
@@ -4419,7 +4425,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GPRIDX-NEXT:     gds_segment_byte_size = 0
 ; GPRIDX-NEXT:     kernarg_segment_byte_size = 28
 ; GPRIDX-NEXT:     workgroup_fbarrier_count = 0
-; GPRIDX-NEXT:     wavefront_sgpr_count = 14
+; GPRIDX-NEXT:     wavefront_sgpr_count = 16
 ; GPRIDX-NEXT:     workitem_vgpr_count = 3
 ; GPRIDX-NEXT:     reserved_vgpr_first = 0
 ; GPRIDX-NEXT:     reserved_vgpr_count = 0
@@ -4463,7 +4469,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:     kernel_code_entry_byte_offset = 256
 ; MOVREL-NEXT:     kernel_code_prefetch_byte_size = 0
 ; MOVREL-NEXT:     granulated_workitem_vgpr_count = 0
-; MOVREL-NEXT:     granulated_wavefront_sgpr_count = 1
+; MOVREL-NEXT:     granulated_wavefront_sgpr_count = 2
 ; MOVREL-NEXT:     priority = 0
 ; MOVREL-NEXT:     float_mode = 240
 ; MOVREL-NEXT:     priv = 0
@@ -4474,7 +4480,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:     enable_mem_ordered = 0
 ; MOVREL-NEXT:     enable_fwd_progress = 0
 ; MOVREL-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; MOVREL-NEXT:     user_sgpr_count = 12
+; MOVREL-NEXT:     user_sgpr_count = 14
 ; MOVREL-NEXT:     enable_trap_handler = 0
 ; MOVREL-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; MOVREL-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -4489,7 +4495,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:     enable_sgpr_queue_ptr = 1
 ; MOVREL-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; MOVREL-NEXT:     enable_sgpr_dispatch_id = 1
-; MOVREL-NEXT:     enable_sgpr_flat_scratch_init = 0
+; MOVREL-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; MOVREL-NEXT:     enable_sgpr_private_segment_size = 0
 ; MOVREL-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; MOVREL-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
@@ -4506,7 +4512,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:     gds_segment_byte_size = 0
 ; MOVREL-NEXT:     kernarg_segment_byte_size = 28
 ; MOVREL-NEXT:     workgroup_fbarrier_count = 0
-; MOVREL-NEXT:     wavefront_sgpr_count = 10
+; MOVREL-NEXT:     wavefront_sgpr_count = 24
 ; MOVREL-NEXT:     workitem_vgpr_count = 4
 ; MOVREL-NEXT:     reserved_vgpr_first = 0
 ; MOVREL-NEXT:     reserved_vgpr_count = 0
@@ -4524,10 +4530,12 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:  ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; MOVREL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; MOVREL-NEXT:    s_add_i32 s12, s12, s17
+; MOVREL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; MOVREL-NEXT:    s_mov_b32 s2, 0
-; MOVREL-NEXT:    s_mov_b32 s3, 0x40080000
 ; MOVREL-NEXT:    s_waitcnt lgkmcnt(0)
 ; MOVREL-NEXT:    s_cmp_eq_u32 s6, 1
+; MOVREL-NEXT:    s_mov_b32 s3, 0x40080000
 ; MOVREL-NEXT:    s_cselect_b64 s[4:5], 2.0, 1.0
 ; MOVREL-NEXT:    s_cmp_eq_u32 s6, 2
 ; MOVREL-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[4:5]
@@ -4535,6 +4543,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; MOVREL-NEXT:    s_cselect_b64 s[2:3], 4.0, s[2:3]
 ; MOVREL-NEXT:    v_mov_b32_e32 v0, s2
 ; MOVREL-NEXT:    v_mov_b32_e32 v3, s1
+; MOVREL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; MOVREL-NEXT:    v_mov_b32_e32 v1, s3
 ; MOVREL-NEXT:    v_mov_b32_e32 v2, s0
 ; MOVREL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -4562,7 +4571,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GFX10-NEXT:     enable_mem_ordered = 1
 ; GFX10-NEXT:     enable_fwd_progress = 0
 ; GFX10-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
-; GFX10-NEXT:     user_sgpr_count = 12
+; GFX10-NEXT:     user_sgpr_count = 14
 ; GFX10-NEXT:     enable_trap_handler = 0
 ; GFX10-NEXT:     enable_sgpr_workgroup_id_x = 1
 ; GFX10-NEXT:     enable_sgpr_workgroup_id_y = 1
@@ -4577,7 +4586,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
 ; GFX10-NEXT:     enable_sgpr_queue_ptr = 1
 ; GFX10-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
 ; GFX10-NEXT:     enable_sgpr_dispatch_id = 1
-; GFX10-NEXT:     enable_sgpr_flat_scratch_init = 0
+; GFX10-NEXT:     enable_sgpr_flat_scratch_init = 1
 ; GFX10-NEXT:     enable_sgpr_private_segment_size = 0
 ; GFX10-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
 ; GFX10-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.ll
index 00c44c27257bb..e207d95287783 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.ll
@@ -35,7 +35,7 @@ define amdgpu_kernel void @stack_object_addrspacecast_in_kernel_no_calls() {
 ; RO-FLAT:     scratch_store_dword
 ; RW-FLAT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; RO-FLAT-NOT: .amdhsa_user_sgpr_private_segment_buffer
-; RW-FLAT:     .amdhsa_user_sgpr_flat_scratch_init 1
+; RW-FLAT:     .amdhsa_user_sgpr_flat_scratch_init 0
 ; RO-FLAT-NOT: .amdhsa_user_sgpr_flat_scratch_init
 ; RW-FLAT:     .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; RW-FLAT-NOT: .amdhsa_enable_private_segment
@@ -43,7 +43,7 @@ define amdgpu_kernel void @stack_object_addrspacecast_in_kernel_no_calls() {
 ; RO-FLAT:     .amdhsa_enable_private_segment 1
 ; RW-FLAT:     .amdhsa_reserve_flat_scratch 0
 ; GCN:         COMPUTE_PGM_RSRC2:SCRATCH_EN: 1
-; RW-FLAT:     COMPUTE_PGM_RSRC2:USER_SGPR: 6
+; RW-FLAT:     COMPUTE_PGM_RSRC2:USER_SGPR: 4
 ; RO-FLAT:     COMPUTE_PGM_RSRC2:USER_SGPR: 0
 define amdgpu_kernel void @stack_object_in_kernel_no_calls() {
   %alloca = alloca i32, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
index 676035735d0af..86766e2904619 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
@@ -12,7 +12,9 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX8V4-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x40
-; GFX8V4-NEXT:    v_mov_b32_e32 v2, 1
+; GFX8V4-NEXT:    s_add_i32 s12, s12, s17
+; GFX8V4-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8V4-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX8V4-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8V4-NEXT:    s_mov_b32 s4, s0
 ; GFX8V4-NEXT:    s_mov_b32 s5, s3
@@ -23,6 +25,7 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX8V4-NEXT:    s_cmp_lg_u32 s1, -1
 ; GFX8V4-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX8V4-NEXT:    s_cselect_b64 s[0:1], s[6:7], 0
+; GFX8V4-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX8V4-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8V4-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8V4-NEXT:    s_waitcnt vmcnt(0)
@@ -37,7 +40,9 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX8V5:       ; %bb.0:
 ; GFX8V5-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX8V5-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0xc8
-; GFX8V5-NEXT:    v_mov_b32_e32 v2, 1
+; GFX8V5-NEXT:    s_add_i32 s12, s12, s17
+; GFX8V5-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8V5-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX8V5-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8V5-NEXT:    s_mov_b32 s4, s0
 ; GFX8V5-NEXT:    s_mov_b32 s5, s2
@@ -47,6 +52,7 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX8V5-NEXT:    s_cmp_lg_u32 s1, -1
 ; GFX8V5-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX8V5-NEXT:    s_cselect_b64 s[0:1], s[2:3], 0
+; GFX8V5-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX8V5-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8V5-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8V5-NEXT:    s_waitcnt vmcnt(0)
@@ -60,9 +66,10 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX9V4-LABEL: addrspacecast:
 ; GFX9V4:       ; %bb.0:
 ; GFX9V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9V4-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9V4-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9V4-NEXT:    s_mov_b64 s[2:3], src_private_base
 ; GFX9V4-NEXT:    s_mov_b64 s[4:5], src_shared_base
-; GFX9V4-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX9V4-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9V4-NEXT:    s_mov_b32 s2, s0
 ; GFX9V4-NEXT:    s_cmp_lg_u32 s0, -1
@@ -71,6 +78,7 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX9V4-NEXT:    s_cmp_lg_u32 s1, -1
 ; GFX9V4-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9V4-NEXT:    s_cselect_b64 s[0:1], s[4:5], 0
+; GFX9V4-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX9V4-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9V4-NEXT:    flat_store_dword v[0:1], v2
 ; GFX9V4-NEXT:    s_waitcnt vmcnt(0)
@@ -84,9 +92,10 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX9V5-LABEL: addrspacecast:
 ; GFX9V5:       ; %bb.0:
 ; GFX9V5-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9V5-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9V5-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9V5-NEXT:    s_mov_b64 s[2:3], src_private_base
 ; GFX9V5-NEXT:    s_mov_b64 s[4:5], src_shared_base
-; GFX9V5-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX9V5-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9V5-NEXT:    s_mov_b32 s2, s0
 ; GFX9V5-NEXT:    s_cmp_lg_u32 s0, -1
@@ -95,6 +104,7 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX9V5-NEXT:    s_cmp_lg_u32 s1, -1
 ; GFX9V5-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9V5-NEXT:    s_cselect_b64 s[0:1], s[4:5], 0
+; GFX9V5-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX9V5-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9V5-NEXT:    flat_store_dword v[0:1], v2
 ; GFX9V5-NEXT:    s_waitcnt vmcnt(0)
@@ -111,7 +121,7 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
   ret void
 }
 
-define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) {
+define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) #0 {
 ; GFX8V4-LABEL: llvm_amdgcn_is_shared:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -167,7 +177,7 @@ define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) {
   ret void
 }
 
-define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) {
+define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) #0 {
 ; GFX8V4-LABEL: llvm_amdgcn_is_private:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -223,7 +233,7 @@ define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) {
   ret void
 }
 
-define amdgpu_kernel void @llvm_trap() {
+define amdgpu_kernel void @llvm_trap() #0 {
 ; GFX8V4-LABEL: llvm_trap:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_mov_b64 s[0:1], s[6:7]
@@ -246,7 +256,7 @@ define amdgpu_kernel void @llvm_trap() {
   unreachable
 }
 
-define amdgpu_kernel void @llvm_debugtrap() {
+define amdgpu_kernel void @llvm_debugtrap() #0 {
 ; GFX8V4-LABEL: llvm_debugtrap:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_trap 3
@@ -266,7 +276,7 @@ define amdgpu_kernel void @llvm_debugtrap() {
   unreachable
 }
 
-define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr)  {
+define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr)  #0 {
 ; GFX8V4-LABEL: llvm_amdgcn_queue_ptr:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    v_mov_b32_e32 v0, s6
@@ -374,3 +384,5 @@ declare void @llvm.debugtrap()
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION}
+
+attributes #0 = { "amdgpu-no-flat-scratch-init" }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement-stack-lower.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement-stack-lower.ll
index 378c6312c52be..94853767ccfac 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement-stack-lower.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement-stack-lower.ll
@@ -9,7 +9,7 @@ define amdgpu_kernel void @v_insert_v64i32_varidx(ptr addrspace(1) %out.ptr, ptr
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[20:23], s[8:9], 0x0
 ; GCN-NEXT:    s_load_dwordx2 s[24:25], s[8:9], 0x10
-; GCN-NEXT:    s_add_u32 s0, s0, s15
+; GCN-NEXT:    s_add_u32 s0, s0, s17
 ; GCN-NEXT:    s_addc_u32 s1, s1, 0
 ; GCN-NEXT:    v_mov_b32_e32 v64, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll
index a6a7f35a774db..859f7ef16e395 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll
@@ -11,13 +11,16 @@ define amdgpu_kernel void @use_lds_globals(ptr addrspace(1) %out, ptr addrspace(
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 4
 ; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-NEXT:    ds_read_b32 v2, v0
-; CHECK-NEXT:    v_mov_b32_e32 v3, 9
+; CHECK-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_add_u32 s0, s0, 4
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-NEXT:    v_mov_b32_e32 v3, 9
 ; CHECK-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x200
 ; CHECK-NEXT:    ds_write_b32 v0, v3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
index dcc2c23cae046..a5a75f74833f1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i64.ll
@@ -6,6 +6,9 @@ define amdgpu_kernel void @test_wave64(i32 %arg0, [8 x i32], i64 %saved) {
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dword s2, s[8:9], 0x0
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0xa
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s2, 0
 ; GCN-NEXT:    s_cselect_b32 s2, 1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
index ad588ebee2f9e..1deee215e522b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
@@ -42,6 +42,9 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_trig_preop_f64 v[0:1], s[0:1], v0
@@ -59,6 +62,9 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_trig_preop_f64 v[0:1], s[0:1], v0
@@ -76,6 +82,8 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_trig_preop_f64 v[0:1], s[0:1], v0
@@ -85,6 +93,10 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
 ;
 ; GFX10-LABEL: s_trig_preop_f64:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    s_load_dword s2, s[8:9], 0x8
@@ -113,6 +125,9 @@ define amdgpu_kernel void @s_trig_preop_f64_imm(double %a, i32 %b) {
 ; CI-LABEL: s_trig_preop_f64_imm:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_trig_preop_f64 v[0:1], s[0:1], 7
 ; CI-NEXT:    s_add_u32 s0, s0, 4
@@ -128,6 +143,9 @@ define amdgpu_kernel void @s_trig_preop_f64_imm(double %a, i32 %b) {
 ; VI-LABEL: s_trig_preop_f64_imm:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_trig_preop_f64 v[0:1], s[0:1], 7
 ; VI-NEXT:    s_add_u32 s0, s0, 4
@@ -143,6 +161,8 @@ define amdgpu_kernel void @s_trig_preop_f64_imm(double %a, i32 %b) {
 ; GFX9-LABEL: s_trig_preop_f64_imm:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_trig_preop_f64 v[0:1], s[0:1], 7
 ; GFX9-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
@@ -151,6 +171,10 @@ define amdgpu_kernel void @s_trig_preop_f64_imm(double %a, i32 %b) {
 ;
 ; GFX10-LABEL: s_trig_preop_f64_imm:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_trig_preop_f64 v[0:1], s[0:1], 7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
index 40f29c56c8f12..b59f85b2dfa38 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
@@ -7,6 +7,9 @@ define amdgpu_kernel void @sdivrem_i32(ptr addrspace(1) %out0, ptr addrspace(1)
 ; GFX8-LABEL: sdivrem_i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_ashr_i32 s6, s5, 31
 ; GFX8-NEXT:    s_add_i32 s0, s5, s6
@@ -146,6 +149,9 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
 ; GFX8-LABEL: sdivrem_i64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_ashr_i32 s2, s9, 31
 ; GFX8-NEXT:    s_ashr_i32 s12, s11, 31
@@ -617,6 +623,9 @@ define amdgpu_kernel void @sdivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX8-LABEL: sdivrem_v2i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_ashr_i32 s2, s10, 31
 ; GFX8-NEXT:    s_add_i32 s0, s10, s2
@@ -845,6 +854,9 @@ define amdgpu_kernel void @sdivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
 define amdgpu_kernel void @sdivrem_v4i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <4 x i32> %x, <4 x i32> %y) {
 ; GFX8-LABEL: sdivrem_v4i32:
 ; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_load_dwordx8 s[12:19], s[8:9], 0x10
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1271,6 +1283,9 @@ define amdgpu_kernel void @sdivrem_v4i32(ptr addrspace(1) %out0, ptr addrspace(1
 define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <2 x i64> %x, <2 x i64> %y) {
 ; GFX8-LABEL: sdivrem_v2i64:
 ; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_load_dwordx8 s[12:19], s[8:9], 0x0
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x20
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2187,6 +2202,9 @@ define amdgpu_kernel void @sdiv_i8(ptr addrspace(1) %out0, ptr addrspace(1) %out
 ; GFX8-LABEL: sdiv_i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_bfe_i32 s0, s4, 0x80008
 ; GFX8-NEXT:    s_ashr_i32 s5, s0, 31
@@ -2332,6 +2350,9 @@ define amdgpu_kernel void @sdivrem_v2i8(ptr addrspace(1) %out0, ptr addrspace(1)
 ; GFX8-LABEL: sdivrem_v2i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_bfe_i32 s0, s2, 0x80010
 ; GFX8-NEXT:    s_ashr_i32 s3, s0, 31
@@ -2596,6 +2617,9 @@ define amdgpu_kernel void @sdiv_i16(ptr addrspace(1) %out0, ptr addrspace(1) %ou
 ; GFX8-LABEL: sdiv_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_bfe_i32 s0, s4, 0x100010
 ; GFX8-NEXT:    s_ashr_i32 s5, s0, 31
@@ -2741,6 +2765,9 @@ define amdgpu_kernel void @sdivrem_v2i16(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX8-LABEL: sdivrem_v2i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_sext_i32_i16 s0, s3
 ; GFX8-NEXT:    s_ashr_i32 s10, s0, 31
@@ -3002,6 +3029,9 @@ define amdgpu_kernel void @sdivrem_i3(ptr addrspace(1) %out0, ptr addrspace(1) %
 ; GFX8-LABEL: sdivrem_i3:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_bfe_i32 s0, s4, 0x30008
 ; GFX8-NEXT:    s_ashr_i32 s5, s0, 31
@@ -3153,6 +3183,9 @@ define amdgpu_kernel void @sdivrem_i27(ptr addrspace(1) %out0, ptr addrspace(1)
 ; GFX8-LABEL: sdivrem_i27:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_bfe_i32 s0, s5, 0x1b0000
 ; GFX8-NEXT:    s_ashr_i32 s5, s0, 31
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
index e3c1a52696b47..ff0114cfc3ddb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
@@ -7,6 +7,9 @@ define amdgpu_kernel void @udivrem_i32(ptr addrspace(1) %out0, ptr addrspace(1)
 ; GFX8-LABEL: udivrem_i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s5
 ; GFX8-NEXT:    s_sub_i32 s0, 0, s5
@@ -113,6 +116,9 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
 ; GFX8-LABEL: udivrem_i64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s11
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v1, s10
@@ -523,6 +529,9 @@ define amdgpu_kernel void @udivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX8-LABEL: udivrem_v2i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s10
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v1, s11
@@ -685,6 +694,9 @@ define amdgpu_kernel void @udivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
 define amdgpu_kernel void @udivrem_v4i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <4 x i32> %x, <4 x i32> %y) {
 ; GFX8-LABEL: udivrem_v4i32:
 ; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_load_dwordx8 s[12:19], s[8:9], 0x10
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
@@ -980,6 +992,9 @@ define amdgpu_kernel void @udivrem_v4i32(ptr addrspace(1) %out0, ptr addrspace(1
 define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <2 x i64> %x, <2 x i64> %y) {
 ; GFX8-LABEL: udivrem_v2i64:
 ; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_load_dwordx4 s[12:15], s[8:9], 0x20
 ; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[8:9], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1772,6 +1787,9 @@ define amdgpu_kernel void @udiv_i8(ptr addrspace(1) %out0, ptr addrspace(1) %out
 ; GFX8-LABEL: udiv_i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x80008
 ; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, s5
@@ -1885,6 +1903,9 @@ define amdgpu_kernel void @udivrem_v2i8(ptr addrspace(1) %out0, ptr addrspace(1)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s0, s[8:9], 0x10
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_bfe_u32 s2, s0, 0x80010
 ; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, s2
@@ -2081,6 +2102,9 @@ define amdgpu_kernel void @udiv_i16(ptr addrspace(1) %out0, ptr addrspace(1) %ou
 ; GFX8-LABEL: udiv_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_lshr_b32 s5, s4, 16
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s5
@@ -2194,6 +2218,9 @@ define amdgpu_kernel void @udivrem_v2i16(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x10
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_and_b32 s2, s1, 0xffff
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s2
@@ -2387,6 +2414,9 @@ define amdgpu_kernel void @udivrem_i3(ptr addrspace(1) %out0, ptr addrspace(1) %
 ; GFX8-LABEL: udivrem_i3:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_bfe_u32 s5, s4, 0x30008
 ; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, s5
@@ -2505,6 +2535,9 @@ define amdgpu_kernel void @udivrem_i27(ptr addrspace(1) %out0, ptr addrspace(1)
 ; GFX8-LABEL: udivrem_i27:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_and_b32 s5, s5, 0x7ffffff
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s5
diff --git a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
index 29fb320bf1283..c78f0a4eb61e9 100644
--- a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
+++ b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
@@ -135,6 +135,9 @@ define amdgpu_kernel void @marked_kernel_use_workitem_id(ptr addrspace(1) %ptr)
 ; FIXEDABI-LABEL: marked_kernel_use_workitem_id:
 ; FIXEDABI:       ; %bb.0:
 ; FIXEDABI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; FIXEDABI-NEXT:    s_add_i32 s6, s6, s11
+; FIXEDABI-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; FIXEDABI-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; FIXEDABI-NEXT:    s_waitcnt lgkmcnt(0)
 ; FIXEDABI-NEXT:    v_mov_b32_e32 v4, s1
 ; FIXEDABI-NEXT:    v_mov_b32_e32 v3, s0
@@ -181,16 +184,19 @@ define amdgpu_kernel void @marked_kernel_use_workgroup_id(ptr addrspace(1) %ptr)
 ; FIXEDABI-LABEL: marked_kernel_use_workgroup_id:
 ; FIXEDABI:       ; %bb.0:
 ; FIXEDABI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; FIXEDABI-NEXT:    v_mov_b32_e32 v2, s6
+; FIXEDABI-NEXT:    s_add_i32 s6, s6, s11
+; FIXEDABI-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; FIXEDABI-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
+; FIXEDABI-NEXT:    v_mov_b32_e32 v2, s8
 ; FIXEDABI-NEXT:    s_waitcnt lgkmcnt(0)
 ; FIXEDABI-NEXT:    v_mov_b32_e32 v0, s0
 ; FIXEDABI-NEXT:    v_mov_b32_e32 v1, s1
 ; FIXEDABI-NEXT:    flat_store_dword v[0:1], v2
 ; FIXEDABI-NEXT:    s_waitcnt vmcnt(0)
-; FIXEDABI-NEXT:    v_mov_b32_e32 v2, s7
+; FIXEDABI-NEXT:    v_mov_b32_e32 v2, s9
 ; FIXEDABI-NEXT:    flat_store_dword v[0:1], v2
 ; FIXEDABI-NEXT:    s_waitcnt vmcnt(0)
-; FIXEDABI-NEXT:    v_mov_b32_e32 v2, s8
+; FIXEDABI-NEXT:    v_mov_b32_e32 v2, s10
 ; FIXEDABI-NEXT:    flat_store_dword v[0:1], v2
 ; FIXEDABI-NEXT:    s_waitcnt vmcnt(0)
 ; FIXEDABI-NEXT:    s_endpgm
@@ -238,6 +244,9 @@ define void @marked_func_use_other_sgpr(ptr addrspace(1) %ptr) #0 {
 define amdgpu_kernel void @marked_kernel_use_other_sgpr(ptr addrspace(1) %ptr) #0 {
 ; FIXEDABI-LABEL: marked_kernel_use_other_sgpr:
 ; FIXEDABI:       ; %bb.0:
+; FIXEDABI-NEXT:    s_add_i32 s6, s6, s11
+; FIXEDABI-NEXT:    s_mov_b32 flat_scratch_lo, s7
+; FIXEDABI-NEXT:    s_lshr_b32 flat_scratch_hi, s6, 8
 ; FIXEDABI-NEXT:    s_add_u32 s0, s4, 8
 ; FIXEDABI-NEXT:    flat_load_ubyte v0, v[0:1] glc
 ; FIXEDABI-NEXT:    s_addc_u32 s1, s5, 0
@@ -261,7 +270,10 @@ define amdgpu_kernel void @marked_kernel_use_other_sgpr(ptr addrspace(1) %ptr) #
 define amdgpu_kernel void @marked_kernel_nokernargs_implicitarg_ptr() #0 {
 ; FIXEDABI-LABEL: marked_kernel_nokernargs_implicitarg_ptr:
 ; FIXEDABI:       ; %bb.0:
+; FIXEDABI-NEXT:    s_add_i32 s4, s4, s9
 ; FIXEDABI-NEXT:    v_mov_b32_e32 v0, 0
+; FIXEDABI-NEXT:    s_mov_b32 flat_scratch_lo, s5
+; FIXEDABI-NEXT:    s_lshr_b32 flat_scratch_hi, s4, 8
 ; FIXEDABI-NEXT:    v_mov_b32_e32 v1, 0
 ; FIXEDABI-NEXT:    flat_load_ubyte v0, v[0:1] glc
 ; FIXEDABI-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
index 59bd4e9ac8ce6..3eba47d7d7852 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -amdgpu-annotate-kernel-features < %s | FileCheck -check-prefixes=HSA,AKF_HSA %s
 ; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=HSA,ATTRIBUTOR_HSA %s
 
 declare void @llvm.memcpy.p1.p4.i32(ptr addrspace(1) nocapture, ptr addrspace(4) nocapture, i32, i1) #0
@@ -27,11 +26,6 @@ define amdgpu_kernel void @store_cast_0_flat_to_group_addrspacecast() #1 {
 }
 
 define amdgpu_kernel void @store_cast_0_group_to_flat_addrspacecast() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@store_cast_0_group_to_flat_addrspacecast
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) null to ptr addrspace(4)), align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_cast_0_group_to_flat_addrspacecast
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) null to ptr addrspace(4)), align 4
@@ -42,11 +36,6 @@ define amdgpu_kernel void @store_cast_0_group_to_flat_addrspacecast() #1 {
 }
 
 define amdgpu_kernel void @store_constant_cast_group_gv_to_flat() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_to_flat
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.i32 to ptr addrspace(4)), align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_to_flat
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.i32 to ptr addrspace(4)), align 4
@@ -57,11 +46,6 @@ define amdgpu_kernel void @store_constant_cast_group_gv_to_flat() #1 {
 }
 
 define amdgpu_kernel void @store_constant_cast_group_gv_gep_to_flat() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_gep_to_flat
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    store i32 7, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_gep_to_flat
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    store i32 7, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), align 4
@@ -92,12 +76,6 @@ define amdgpu_kernel void @store_constant_cast_global_gv_gep_to_flat() #1 {
 }
 
 define amdgpu_kernel void @load_constant_cast_group_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@load_constant_cast_group_gv_gep_to_flat
-; AKF_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), align 4
-; AKF_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[OUT]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@load_constant_cast_group_gv_gep_to_flat
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), align 4
@@ -110,12 +88,6 @@ define amdgpu_kernel void @load_constant_cast_group_gv_gep_to_flat(ptr addrspace
 }
 
 define amdgpu_kernel void @atomicrmw_constant_cast_group_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@atomicrmw_constant_cast_group_gv_gep_to_flat
-; AKF_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = atomicrmw add ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 1 seq_cst, align 4
-; AKF_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[OUT]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@atomicrmw_constant_cast_group_gv_gep_to_flat
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = atomicrmw add ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 1 seq_cst, align 4
@@ -128,13 +100,6 @@ define amdgpu_kernel void @atomicrmw_constant_cast_group_gv_gep_to_flat(ptr addr
 }
 
 define amdgpu_kernel void @cmpxchg_constant_cast_group_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@cmpxchg_constant_cast_group_gv_gep_to_flat
-; AKF_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = cmpxchg ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 0, i32 1 seq_cst seq_cst, align 4
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = extractvalue { i32, i1 } [[VAL]], 0
-; AKF_HSA-NEXT:    store i32 [[VAL0]], ptr addrspace(1) [[OUT]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@cmpxchg_constant_cast_group_gv_gep_to_flat
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = cmpxchg ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 0, i32 1 seq_cst seq_cst, align 4
@@ -149,11 +114,6 @@ define amdgpu_kernel void @cmpxchg_constant_cast_group_gv_gep_to_flat(ptr addrsp
 }
 
 define amdgpu_kernel void @memcpy_constant_cast_group_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@memcpy_constant_cast_group_gv_gep_to_flat
-; AKF_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @llvm.memcpy.p1.p4.i32(ptr addrspace(1) align 4 [[OUT]], ptr addrspace(4) align 4 getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 32, i1 false)
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@memcpy_constant_cast_group_gv_gep_to_flat
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @llvm.memcpy.p1.p4.i32(ptr addrspace(1) align 4 [[OUT]], ptr addrspace(4) align 4 getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 32, i1 false)
@@ -165,11 +125,6 @@ define amdgpu_kernel void @memcpy_constant_cast_group_gv_gep_to_flat(ptr addrspa
 
 ; Can't just search the pointer value
 define amdgpu_kernel void @store_value_constant_cast_lds_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@store_value_constant_cast_lds_gv_gep_to_flat
-; AKF_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    store ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), ptr addrspace(1) [[OUT]], align 8
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_value_constant_cast_lds_gv_gep_to_flat
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    store ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), ptr addrspace(1) [[OUT]], align 8
@@ -181,11 +136,6 @@ define amdgpu_kernel void @store_value_constant_cast_lds_gv_gep_to_flat(ptr addr
 
 ; Can't just search pointer types
 define amdgpu_kernel void @store_ptrtoint_value_constant_cast_lds_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@store_ptrtoint_value_constant_cast_lds_gv_gep_to_flat
-; AKF_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    store i64 ptrtoint (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to i64), ptr addrspace(1) [[OUT]], align 8
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_ptrtoint_value_constant_cast_lds_gv_gep_to_flat
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    store i64 ptrtoint (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to i64), ptr addrspace(1) [[OUT]], align 8
@@ -197,11 +147,6 @@ define amdgpu_kernel void @store_ptrtoint_value_constant_cast_lds_gv_gep_to_flat
 
 ; Cast group to flat, do GEP, cast back to group
 define amdgpu_kernel void @store_constant_cast_group_gv_gep_to_flat_to_group() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_gep_to_flat_to_group
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    store i32 7, ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3)), align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_gep_to_flat_to_group
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    store i32 7, ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3)), align 4
@@ -212,10 +157,6 @@ define amdgpu_kernel void @store_constant_cast_group_gv_gep_to_flat_to_group() #
 }
 
 define ptr addrspace(3) @ret_constant_cast_group_gv_gep_to_flat_to_group() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@ret_constant_cast_group_gv_gep_to_flat_to_group
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    ret ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3))
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@ret_constant_cast_group_gv_gep_to_flat_to_group
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    ret ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3))
@@ -229,14 +170,11 @@ attributes #1 = { nounwind }
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
-; AKF_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
-; AKF_HSA: attributes #[[ATTR1]] = { nounwind }
 ;.
 ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
-; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
 ; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/always-uniform.ll b/llvm/test/CodeGen/AMDGPU/always-uniform.ll
index b6c0271e5f56f..4e7022710c671 100644
--- a/llvm/test/CodeGen/AMDGPU/always-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/always-uniform.ll
@@ -8,8 +8,10 @@ define amdgpu_kernel void @readfirstlane_uniform(ptr addrspace(1) noalias nocapt
 ; GCN-LABEL: readfirstlane_uniform:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
 ; GCN-NEXT:    v_readfirstlane_b32 s4, v0
 ; GCN-NEXT:    s_mov_b32 s5, 0
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_add_u32 s0, s0, s4
@@ -18,6 +20,7 @@ define amdgpu_kernel void @readfirstlane_uniform(ptr addrspace(1) noalias nocapt
 ; GCN-NEXT:    s_add_u32 s0, s2, 40
 ; GCN-NEXT:    s_addc_u32 s1, s3, 0
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v2, s4
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
index 7fdc012d4f1b5..e71bf15384727 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
@@ -393,6 +393,9 @@ define amdgpu_kernel void @select_add_lhs_const_i16(i1 %cond) {
 ; GCN-LABEL: select_add_lhs_const_i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_bitcmp1_b32 s0, 0
 ; GCN-NEXT:    s_movk_i32 s0, 0x80
diff --git a/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll b/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
index 3e19ee5567929..85b5c7c870b23 100644
--- a/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
@@ -34,7 +34,7 @@ define amdgpu_kernel void @amdhsa_trap_num_sgprs(
     ptr addrspace(1) %out26, i32 %in26,
     ptr addrspace(1) %out27, i32 %in27,
     ptr addrspace(1) %out28, i32 %in28,
-    ptr addrspace(1) %out29, i32 %in29) {
+    ptr addrspace(1) %out29, i32 %in29) #0 {
 entry:
   store i32 %in0, ptr addrspace(1) %out0
   store i32 %in1, ptr addrspace(1) %out1
@@ -68,3 +68,5 @@ entry:
   store i32 %in29, ptr addrspace(1) %out29
   ret void
 }
+
+attributes #0 = { "amdgpu-no-flat-scratch-init" }
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
index 6d205921923d3..8389a8e86cb44 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -amdgpu-annotate-kernel-features < %s | FileCheck -check-prefixes=AKF_HSA %s
 ; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=ATTRIBUTOR_HSA %s
 
 ; TODO: The test contains UB which is refined by the Attributor and should be removed.
@@ -19,12 +18,6 @@ declare ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() #0
 declare i64 @llvm.amdgcn.dispatch.id() #0
 
 define void @use_workitem_id_x() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_workitem_id_x
-; AKF_HSA-SAME: () #[[ATTR1:[0-9]+]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_workitem_id_x
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR1:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
@@ -37,12 +30,6 @@ define void @use_workitem_id_x() #1 {
 }
 
 define void @use_workitem_id_y() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_workitem_id_y
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_workitem_id_y
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
@@ -55,12 +42,6 @@ define void @use_workitem_id_y() #1 {
 }
 
 define void @use_workitem_id_z() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_workitem_id_z
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_workitem_id_z
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR3:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
@@ -73,12 +54,6 @@ define void @use_workitem_id_z() #1 {
 }
 
 define void @use_workgroup_id_x() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_workgroup_id_x
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_workgroup_id_x
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR4:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
@@ -91,12 +66,6 @@ define void @use_workgroup_id_x() #1 {
 }
 
 define void @use_workgroup_id_y() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_workgroup_id_y
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_workgroup_id_y
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR5:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
@@ -109,12 +78,6 @@ define void @use_workgroup_id_y() #1 {
 }
 
 define void @use_workgroup_id_z() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_workgroup_id_z
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_workgroup_id_z
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR6:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
@@ -127,12 +90,6 @@ define void @use_workgroup_id_z() #1 {
 }
 
 define void @use_dispatch_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_dispatch_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; AKF_HSA-NEXT:    store volatile ptr addrspace(4) [[DISPATCH_PTR]], ptr addrspace(1) poison, align 8
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_dispatch_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR7:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
@@ -145,12 +102,6 @@ define void @use_dispatch_ptr() #1 {
 }
 
 define void @use_queue_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_queue_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[QUEUE_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
-; AKF_HSA-NEXT:    store volatile ptr addrspace(4) [[QUEUE_PTR]], ptr addrspace(1) poison, align 8
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_queue_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR8:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[QUEUE_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
@@ -163,12 +114,6 @@ define void @use_queue_ptr() #1 {
 }
 
 define void @use_dispatch_id() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_dispatch_id
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i64 @llvm.amdgcn.dispatch.id()
-; AKF_HSA-NEXT:    store volatile i64 [[VAL]], ptr addrspace(1) poison, align 8
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_dispatch_id
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR9:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i64 @llvm.amdgcn.dispatch.id()
@@ -181,14 +126,6 @@ define void @use_dispatch_id() #1 {
 }
 
 define void @use_workgroup_id_y_workgroup_id_z() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_workgroup_id_y_workgroup_id_z
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_workgroup_id_y_workgroup_id_z
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR10:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
@@ -205,11 +142,6 @@ define void @use_workgroup_id_y_workgroup_id_z() #1 {
 }
 
 define void @func_indirect_use_workitem_id_x() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workitem_id_x
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_workitem_id_x()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workitem_id_x
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR1]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_workitem_id_x()
@@ -220,11 +152,6 @@ define void @func_indirect_use_workitem_id_x() #1 {
 }
 
 define void @kernel_indirect_use_workitem_id_x() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@kernel_indirect_use_workitem_id_x
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_workitem_id_x()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kernel_indirect_use_workitem_id_x
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR1]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_workitem_id_x()
@@ -235,11 +162,6 @@ define void @kernel_indirect_use_workitem_id_x() #1 {
 }
 
 define void @func_indirect_use_workitem_id_y() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workitem_id_y
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_workitem_id_y()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workitem_id_y
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_workitem_id_y()
@@ -250,11 +172,6 @@ define void @func_indirect_use_workitem_id_y() #1 {
 }
 
 define void @func_indirect_use_workitem_id_z() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workitem_id_z
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_workitem_id_z()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workitem_id_z
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR3]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_workitem_id_z()
@@ -265,11 +182,6 @@ define void @func_indirect_use_workitem_id_z() #1 {
 }
 
 define void @func_indirect_use_workgroup_id_x() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workgroup_id_x
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_workgroup_id_x()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workgroup_id_x
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR4]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_workgroup_id_x()
@@ -280,11 +192,6 @@ define void @func_indirect_use_workgroup_id_x() #1 {
 }
 
 define void @kernel_indirect_use_workgroup_id_x() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@kernel_indirect_use_workgroup_id_x
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_workgroup_id_x()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kernel_indirect_use_workgroup_id_x
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR4]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_workgroup_id_x()
@@ -295,11 +202,6 @@ define void @kernel_indirect_use_workgroup_id_x() #1 {
 }
 
 define void @func_indirect_use_workgroup_id_y() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workgroup_id_y
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_workgroup_id_y()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workgroup_id_y
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR5]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_workgroup_id_y()
@@ -310,11 +212,6 @@ define void @func_indirect_use_workgroup_id_y() #1 {
 }
 
 define void @func_indirect_use_workgroup_id_z() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workgroup_id_z
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_workgroup_id_z()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workgroup_id_z
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR6]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_workgroup_id_z()
@@ -325,11 +222,6 @@ define void @func_indirect_use_workgroup_id_z() #1 {
 }
 
 define void @func_indirect_indirect_use_workgroup_id_y() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_indirect_use_workgroup_id_y
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @func_indirect_use_workgroup_id_y()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_indirect_use_workgroup_id_y
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR5]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @func_indirect_use_workgroup_id_y()
@@ -340,11 +232,6 @@ define void @func_indirect_indirect_use_workgroup_id_y() #1 {
 }
 
 define void @indirect_x2_use_workgroup_id_y() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@indirect_x2_use_workgroup_id_y
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @func_indirect_indirect_use_workgroup_id_y()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@indirect_x2_use_workgroup_id_y
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR5]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @func_indirect_indirect_use_workgroup_id_y()
@@ -355,11 +242,6 @@ define void @indirect_x2_use_workgroup_id_y() #1 {
 }
 
 define void @func_indirect_use_dispatch_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_dispatch_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_dispatch_ptr()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_dispatch_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR7]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_dispatch_ptr()
@@ -370,11 +252,6 @@ define void @func_indirect_use_dispatch_ptr() #1 {
 }
 
 define void @func_indirect_use_queue_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_queue_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_queue_ptr()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_queue_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR8]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_queue_ptr()
@@ -385,11 +262,6 @@ define void @func_indirect_use_queue_ptr() #1 {
 }
 
 define void @func_indirect_use_dispatch_id() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_dispatch_id
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_dispatch_id()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_dispatch_id
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR9]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_dispatch_id()
@@ -400,11 +272,6 @@ define void @func_indirect_use_dispatch_id() #1 {
 }
 
 define void @func_indirect_use_workgroup_id_y_workgroup_id_z() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workgroup_id_y_workgroup_id_z
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @func_indirect_use_workgroup_id_y_workgroup_id_z()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_workgroup_id_y_workgroup_id_z
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR11:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @func_indirect_use_workgroup_id_y_workgroup_id_z()
@@ -415,13 +282,6 @@ define void @func_indirect_use_workgroup_id_y_workgroup_id_z() #1 {
 }
 
 define void @recursive_use_workitem_id_y() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@recursive_use_workitem_id_y
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    call void @recursive_use_workitem_id_y()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@recursive_use_workitem_id_y
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
@@ -436,11 +296,6 @@ define void @recursive_use_workitem_id_y() #1 {
 }
 
 define void @call_recursive_use_workitem_id_y() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@call_recursive_use_workitem_id_y
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @recursive_use_workitem_id_y()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@call_recursive_use_workitem_id_y
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @recursive_use_workitem_id_y()
@@ -451,12 +306,6 @@ define void @call_recursive_use_workitem_id_y() #1 {
 }
 
 define void @use_group_to_flat_addrspacecast(ptr addrspace(3) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast
-; AKF_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr addrspace(4)
-; AKF_HSA-NEXT:    store volatile i32 0, ptr addrspace(4) [[STOF]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR12:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr addrspace(4)
@@ -470,12 +319,6 @@ define void @use_group_to_flat_addrspacecast(ptr addrspace(3) %ptr) #1 {
 
 
 define void @use_group_to_flat_addrspacecast_gfx9(ptr addrspace(3) %ptr) #2 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast_gfx9
-; AKF_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
-; AKF_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr addrspace(4)
-; AKF_HSA-NEXT:    store volatile i32 0, ptr addrspace(4) [[STOF]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast_gfx9
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR13:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr addrspace(4)
@@ -488,13 +331,6 @@ define void @use_group_to_flat_addrspacecast_gfx9(ptr addrspace(3) %ptr) #2 {
 }
 
 define void @use_group_to_flat_addrspacecast_queue_ptr_gfx9(ptr addrspace(3) %ptr) #2 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast_queue_ptr_gfx9
-; AKF_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR2]] {
-; AKF_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr addrspace(4)
-; AKF_HSA-NEXT:    store volatile i32 0, ptr addrspace(4) [[STOF]], align 4
-; AKF_HSA-NEXT:    call void @func_indirect_use_queue_ptr()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast_queue_ptr_gfx9
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR14:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr addrspace(4)
@@ -509,11 +345,6 @@ define void @use_group_to_flat_addrspacecast_queue_ptr_gfx9(ptr addrspace(3) %pt
 }
 
 define void @indirect_use_group_to_flat_addrspacecast() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@indirect_use_group_to_flat_addrspacecast
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_group_to_flat_addrspacecast(ptr addrspace(3) null)
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@indirect_use_group_to_flat_addrspacecast
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR12]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_group_to_flat_addrspacecast(ptr addrspace(3) null)
@@ -524,11 +355,6 @@ define void @indirect_use_group_to_flat_addrspacecast() #1 {
 }
 
 define void @indirect_use_group_to_flat_addrspacecast_gfx9() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@indirect_use_group_to_flat_addrspacecast_gfx9
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_group_to_flat_addrspacecast_gfx9(ptr addrspace(3) null)
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@indirect_use_group_to_flat_addrspacecast_gfx9
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR11]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_group_to_flat_addrspacecast_gfx9(ptr addrspace(3) null)
@@ -539,11 +365,6 @@ define void @indirect_use_group_to_flat_addrspacecast_gfx9() #1 {
 }
 
 define void @indirect_use_group_to_flat_addrspacecast_queue_ptr_gfx9() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@indirect_use_group_to_flat_addrspacecast_queue_ptr_gfx9
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_group_to_flat_addrspacecast_queue_ptr_gfx9(ptr addrspace(3) null)
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@indirect_use_group_to_flat_addrspacecast_queue_ptr_gfx9
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR8]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_group_to_flat_addrspacecast_queue_ptr_gfx9(ptr addrspace(3) null)
@@ -554,12 +375,6 @@ define void @indirect_use_group_to_flat_addrspacecast_queue_ptr_gfx9() #1 {
 }
 
 define void @use_kernarg_segment_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_kernarg_segment_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[KERNARG_SEGMENT_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; AKF_HSA-NEXT:    store volatile ptr addrspace(4) [[KERNARG_SEGMENT_PTR]], ptr addrspace(1) poison, align 8
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_kernarg_segment_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR11]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[KERNARG_SEGMENT_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
@@ -571,11 +386,6 @@ define void @use_kernarg_segment_ptr() #1 {
   ret void
 }
 define void @func_indirect_use_kernarg_segment_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_kernarg_segment_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_kernarg_segment_ptr()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_kernarg_segment_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR11]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_kernarg_segment_ptr()
@@ -586,12 +396,6 @@ define void @func_indirect_use_kernarg_segment_ptr() #1 {
 }
 
 define amdgpu_kernel void @kern_use_implicitarg_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@kern_use_implicitarg_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; AKF_HSA-NEXT:    store volatile ptr addrspace(4) [[IMPLICITARG_PTR]], ptr addrspace(1) poison, align 8
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_use_implicitarg_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR12]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
@@ -604,12 +408,6 @@ define amdgpu_kernel void @kern_use_implicitarg_ptr() #1 {
 }
 
 define void @use_implicitarg_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_implicitarg_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; AKF_HSA-NEXT:    store volatile ptr addrspace(4) [[IMPLICITARG_PTR]], ptr addrspace(1) poison, align 8
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_implicitarg_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR12]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
@@ -622,11 +420,6 @@ define void @use_implicitarg_ptr() #1 {
 }
 
 define void @func_indirect_use_implicitarg_ptr() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_implicitarg_ptr
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    call void @use_implicitarg_ptr()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_implicitarg_ptr
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR12]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @use_implicitarg_ptr()
@@ -640,10 +433,6 @@ declare void @external.func() #3
 
 ; This function gets deleted.
 define internal void @defined.func() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@defined.func
-; AKF_HSA-SAME: () #[[ATTR3:[0-9]+]] {
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@defined.func
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR16:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    ret void
@@ -652,11 +441,6 @@ define internal void @defined.func() #3 {
 }
 
 define void @func_call_external() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_call_external
-; AKF_HSA-SAME: () #[[ATTR3]] {
-; AKF_HSA-NEXT:    call void @external.func()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_call_external
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR15:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @external.func()
@@ -667,11 +451,6 @@ define void @func_call_external() #3 {
 }
 
 define void @func_call_defined() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_call_defined
-; AKF_HSA-SAME: () #[[ATTR3]] {
-; AKF_HSA-NEXT:    call void @defined.func()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_call_defined
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR16]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @defined.func()
@@ -681,11 +460,6 @@ define void @func_call_defined() #3 {
   ret void
 }
 define void @func_call_asm() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_call_asm
-; AKF_HSA-SAME: () #[[ATTR3]] {
-; AKF_HSA-NEXT:    call void asm sideeffect "", ""() #[[ATTR3]]
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_call_asm
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR16]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void asm sideeffect "", ""() #[[ATTR26:[0-9]+]]
@@ -696,11 +470,6 @@ define void @func_call_asm() #3 {
 }
 
 define amdgpu_kernel void @kern_call_external() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@kern_call_external
-; AKF_HSA-SAME: () #[[ATTR4:[0-9]+]] {
-; AKF_HSA-NEXT:    call void @external.func()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_call_external
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR15]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @external.func()
@@ -711,11 +480,6 @@ define amdgpu_kernel void @kern_call_external() #3 {
 }
 
 define amdgpu_kernel void @func_kern_defined() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_kern_defined
-; AKF_HSA-SAME: () #[[ATTR4]] {
-; AKF_HSA-NEXT:    call void @defined.func()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_kern_defined
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR17:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @defined.func()
@@ -726,12 +490,6 @@ define amdgpu_kernel void @func_kern_defined() #3 {
 }
 
 define i32 @use_dispatch_ptr_ret_type() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_dispatch_ptr_ret_type
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; AKF_HSA-NEXT:    store volatile ptr addrspace(4) [[DISPATCH_PTR]], ptr addrspace(1) poison, align 8
-; AKF_HSA-NEXT:    ret i32 0
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_dispatch_ptr_ret_type
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR7]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
@@ -744,12 +502,6 @@ define i32 @use_dispatch_ptr_ret_type() #1 {
 }
 
 define float @func_indirect_use_dispatch_ptr_constexpr_cast_func() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_use_dispatch_ptr_constexpr_cast_func
-; AKF_HSA-SAME: () #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[F:%.*]] = call float @use_dispatch_ptr_ret_type()
-; AKF_HSA-NEXT:    [[FADD:%.*]] = fadd float [[F]], 1.000000e+00
-; AKF_HSA-NEXT:    ret float [[FADD]]
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_use_dispatch_ptr_constexpr_cast_func
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR7]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[F:%.*]] = call float @use_dispatch_ptr_ret_type()
@@ -762,12 +514,6 @@ define float @func_indirect_use_dispatch_ptr_constexpr_cast_func() #1 {
 }
 
 define float @func_indirect_call(ptr %fptr) #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_call
-; AKF_HSA-SAME: (ptr [[FPTR:%.*]]) #[[ATTR3]] {
-; AKF_HSA-NEXT:    [[F:%.*]] = call float [[FPTR]]()
-; AKF_HSA-NEXT:    [[FADD:%.*]] = fadd float [[F]], 1.000000e+00
-; AKF_HSA-NEXT:    ret float [[FADD]]
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_call
 ; ATTRIBUTOR_HSA-SAME: (ptr [[FPTR:%.*]]) #[[ATTR15]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[F:%.*]] = call float [[FPTR]]()
@@ -781,12 +527,6 @@ define float @func_indirect_call(ptr %fptr) #3 {
 
 declare float @extern() #3
 define float @func_extern_call() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_extern_call
-; AKF_HSA-SAME: () #[[ATTR3]] {
-; AKF_HSA-NEXT:    [[F:%.*]] = call float @extern()
-; AKF_HSA-NEXT:    [[FADD:%.*]] = fadd float [[F]], 1.000000e+00
-; AKF_HSA-NEXT:    ret float [[FADD]]
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_extern_call
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR15]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[F:%.*]] = call float @extern()
@@ -799,12 +539,6 @@ define float @func_extern_call() #3 {
 }
 
 define float @func_null_call(ptr %fptr) #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_null_call
-; AKF_HSA-SAME: (ptr [[FPTR:%.*]]) #[[ATTR3]] {
-; AKF_HSA-NEXT:    [[F:%.*]] = call float null()
-; AKF_HSA-NEXT:    [[FADD:%.*]] = fadd float [[F]], 1.000000e+00
-; AKF_HSA-NEXT:    ret float [[FADD]]
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_null_call
 ; ATTRIBUTOR_HSA-SAME: (ptr [[FPTR:%.*]]) #[[ATTR15]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[F:%.*]] = call float null()
@@ -820,12 +554,6 @@ declare float @llvm.amdgcn.rcp.f32(float) #0
 
 ; Calls some other recognized intrinsic
 define float @func_other_intrinsic_call(float %arg) #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_other_intrinsic_call
-; AKF_HSA-SAME: (float [[ARG:%.*]]) #[[ATTR3]] {
-; AKF_HSA-NEXT:    [[F:%.*]] = call float @llvm.amdgcn.rcp.f32(float [[ARG]])
-; AKF_HSA-NEXT:    [[FADD:%.*]] = fadd float [[F]], 1.000000e+00
-; AKF_HSA-NEXT:    ret float [[FADD]]
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_other_intrinsic_call
 ; ATTRIBUTOR_HSA-SAME: (float [[ARG:%.*]]) #[[ATTR16]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[F:%.*]] = call float @llvm.amdgcn.rcp.f32(float [[ARG]])
@@ -839,11 +567,6 @@ define float @func_other_intrinsic_call(float %arg) #3 {
 
 ; Hostcall needs to be enabled for sanitizers
 define amdgpu_kernel void @kern_sanitize_address() #4 {
-; AKF_HSA-LABEL: define {{[^@]+}}@kern_sanitize_address
-; AKF_HSA-SAME: () #[[ATTR5:[0-9]+]] {
-; AKF_HSA-NEXT:    store volatile i32 0, ptr addrspace(1) null, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_sanitize_address
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR18:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    store volatile i32 0, ptr addrspace(1) null, align 4
@@ -855,11 +578,6 @@ define amdgpu_kernel void @kern_sanitize_address() #4 {
 
 ; Hostcall needs to be enabled for sanitizers
 define void @func_sanitize_address() #4 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_sanitize_address
-; AKF_HSA-SAME: () #[[ATTR5]] {
-; AKF_HSA-NEXT:    store volatile i32 0, ptr addrspace(1) null, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_sanitize_address
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR18]] {
 ; ATTRIBUTOR_HSA-NEXT:    store volatile i32 0, ptr addrspace(1) null, align 4
@@ -871,11 +589,6 @@ define void @func_sanitize_address() #4 {
 
 ; Hostcall needs to be enabled for sanitizers
 define void @func_indirect_sanitize_address() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@func_indirect_sanitize_address
-; AKF_HSA-SAME: () #[[ATTR3]] {
-; AKF_HSA-NEXT:    call void @func_sanitize_address()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@func_indirect_sanitize_address
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR19:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @func_sanitize_address()
@@ -887,11 +600,6 @@ define void @func_indirect_sanitize_address() #3 {
 
 ; Hostcall needs to be enabled for sanitizers
 define amdgpu_kernel void @kern_indirect_sanitize_address() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@kern_indirect_sanitize_address
-; AKF_HSA-SAME: () #[[ATTR4]] {
-; AKF_HSA-NEXT:    call void @func_sanitize_address()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_indirect_sanitize_address
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR19]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @func_sanitize_address()
@@ -906,11 +614,6 @@ define amdgpu_kernel void @kern_indirect_sanitize_address() #3 {
 declare void @extern_func_sanitize_address() #5
 
 define amdgpu_kernel void @kern_decl_sanitize_address() #3 {
-; AKF_HSA-LABEL: define {{[^@]+}}@kern_decl_sanitize_address
-; AKF_HSA-SAME: () #[[ATTR4]] {
-; AKF_HSA-NEXT:    call void @extern_func_sanitize_address()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_decl_sanitize_address
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR15]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @extern_func_sanitize_address()
@@ -923,10 +626,6 @@ define amdgpu_kernel void @kern_decl_sanitize_address() #3 {
 declare void @enqueue_block_decl() #6
 
 define internal void @enqueue_block_def() #6 {
-; AKF_HSA-LABEL: define {{[^@]+}}@enqueue_block_def
-; AKF_HSA-SAME: () #[[ATTR7:[0-9]+]] {
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@enqueue_block_def
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR22:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    ret void
@@ -935,11 +634,6 @@ define internal void @enqueue_block_def() #6 {
 }
 
 define amdgpu_kernel void @kern_call_enqueued_block_decl() {
-; AKF_HSA-LABEL: define {{[^@]+}}@kern_call_enqueued_block_decl
-; AKF_HSA-SAME: () #[[ATTR8:[0-9]+]] {
-; AKF_HSA-NEXT:    call void @enqueue_block_decl()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_call_enqueued_block_decl
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR23:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @enqueue_block_decl()
@@ -950,11 +644,6 @@ define amdgpu_kernel void @kern_call_enqueued_block_decl() {
 }
 
 define amdgpu_kernel void @kern_call_enqueued_block_def() {
-; AKF_HSA-LABEL: define {{[^@]+}}@kern_call_enqueued_block_def
-; AKF_HSA-SAME: () #[[ATTR8]] {
-; AKF_HSA-NEXT:    call void @enqueue_block_def()
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_call_enqueued_block_def
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR24:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @enqueue_block_def()
@@ -965,9 +654,6 @@ define amdgpu_kernel void @kern_call_enqueued_block_def() {
 }
 
 define void @unused_enqueue_block() {
-; AKF_HSA-LABEL: define {{[^@]+}}@unused_enqueue_block() {
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@unused_enqueue_block
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR25:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    ret void
@@ -976,9 +662,6 @@ define void @unused_enqueue_block() {
 }
 
 define internal void @known_func() {
-; AKF_HSA-LABEL: define {{[^@]+}}@known_func() {
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@known_func
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR25]] {
 ; ATTRIBUTOR_HSA-NEXT:    ret void
@@ -988,11 +671,6 @@ define internal void @known_func() {
 
 ; Should never happen
 define amdgpu_kernel void @kern_callsite_enqueue_block() {
-; AKF_HSA-LABEL: define {{[^@]+}}@kern_callsite_enqueue_block
-; AKF_HSA-SAME: () #[[ATTR8]] {
-; AKF_HSA-NEXT:    call void @known_func() #[[ATTR7]]
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@kern_callsite_enqueue_block
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR24]] {
 ; ATTRIBUTOR_HSA-NEXT:    call void @known_func() #[[ATTR27:[0-9]+]]
@@ -1014,15 +692,6 @@ attributes #6 = { "enqueued-block" }
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
-; AKF_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; AKF_HSA: attributes #[[ATTR1]] = { nounwind "target-cpu"="fiji" }
-; AKF_HSA: attributes #[[ATTR2]] = { nounwind "target-cpu"="gfx900" }
-; AKF_HSA: attributes #[[ATTR3]] = { nounwind }
-; AKF_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-calls" }
-; AKF_HSA: attributes #[[ATTR5]] = { nounwind sanitize_address }
-; AKF_HSA: attributes #[[ATTR6:[0-9]+]] = { nounwind sanitize_address "amdgpu-no-implicitarg-ptr" }
-; AKF_HSA: attributes #[[ATTR7]] = { "enqueued-block" }
-; AKF_HSA: attributes #[[ATTR8]] = { "amdgpu-calls" }
 ;.
 ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="fiji" "uniform-work-group-size"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
index 2809f0957462a..32bb22b699b61 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -amdgpu-annotate-kernel-features < %s | FileCheck -check-prefixes=HSA,AKF_HSA %s
 ; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=HSA,ATTRIBUTOR_HSA %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
@@ -33,12 +32,6 @@ define amdgpu_kernel void @use_tgid_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_y(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tgid_y
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_y
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
@@ -51,14 +44,6 @@ define amdgpu_kernel void @use_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @multi_use_tgid_y(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@multi_use_tgid_y
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@multi_use_tgid_y
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
@@ -75,14 +60,6 @@ define amdgpu_kernel void @multi_use_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_y(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tgid_x_y
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_x_y
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
@@ -99,12 +76,6 @@ define amdgpu_kernel void @use_tgid_x_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_z(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tgid_z
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; AKF_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_z
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
@@ -117,14 +88,6 @@ define amdgpu_kernel void @use_tgid_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_z(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tgid_x_z
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_x_z
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
@@ -141,14 +104,6 @@ define amdgpu_kernel void @use_tgid_x_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_y_z(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tgid_y_z
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_y_z
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
@@ -165,16 +120,6 @@ define amdgpu_kernel void @use_tgid_y_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_y_z(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tgid_x_y_z
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_x_y_z
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
@@ -207,12 +152,6 @@ define amdgpu_kernel void @use_tidig_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_y(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tidig_y
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; AKF_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tidig_y
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
@@ -225,12 +164,6 @@ define amdgpu_kernel void @use_tidig_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_z(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tidig_z
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
-; AKF_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tidig_z
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
@@ -259,14 +192,6 @@ define amdgpu_kernel void @use_tidig_x_tgid_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_y_tgid_y(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tidig_y_tgid_y
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tidig_y_tgid_y
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR7:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
@@ -283,16 +208,6 @@ define amdgpu_kernel void @use_tidig_y_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_x_y_z(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_tidig_x_y_z
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; AKF_HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tidig_x_y_z
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR8:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
@@ -313,22 +228,6 @@ define amdgpu_kernel void @use_tidig_x_y_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_all_workitems(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_all_workitems
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; AKF_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; AKF_HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
-; AKF_HSA-NEXT:    [[VAL3:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; AKF_HSA-NEXT:    [[VAL4:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; AKF_HSA-NEXT:    [[VAL5:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; AKF_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL3]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL4]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    store volatile i32 [[VAL5]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_all_workitems
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR9:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
@@ -361,13 +260,6 @@ define amdgpu_kernel void @use_all_workitems(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_dispatch_ptr(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_dispatch_ptr
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; AKF_HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) [[DISPATCH_PTR]], align 4
-; AKF_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_dispatch_ptr
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR10:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
@@ -382,13 +274,6 @@ define amdgpu_kernel void @use_dispatch_ptr(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_queue_ptr(ptr addrspace(1) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_queue_ptr
-; AKF_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
-; AKF_HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) [[DISPATCH_PTR]], align 4
-; AKF_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_queue_ptr
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR11:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
@@ -417,12 +302,6 @@ define amdgpu_kernel void @use_kernarg_segment_ptr(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_group_to_flat_addrspacecast(ptr addrspace(3) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast
-; AKF_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
-; AKF_HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR12:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
@@ -435,12 +314,6 @@ define amdgpu_kernel void @use_group_to_flat_addrspacecast(ptr addrspace(3) %ptr
 }
 
 define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_private_to_flat_addrspacecast
-; AKF_HSA-SAME: (ptr addrspace(5) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
-; AKF_HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_private_to_flat_addrspacecast
 ; ATTRIBUTOR_HSA-SAME: (ptr addrspace(5) [[PTR:%.*]]) #[[ATTR13:[0-9]+]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
@@ -526,13 +399,6 @@ define amdgpu_kernel void @use_flat_to_constant_addrspacecast(ptr %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_is_shared(ptr %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_is_shared
-; AKF_HSA-SAME: (ptr [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
-; AKF_HSA-NEXT:    [[EXT:%.*]] = zext i1 [[IS_SHARED]] to i32
-; AKF_HSA-NEXT:    store i32 [[EXT]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_is_shared
 ; ATTRIBUTOR_HSA-SAME: (ptr [[PTR:%.*]]) #[[ATTR12]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
@@ -547,13 +413,6 @@ define amdgpu_kernel void @use_is_shared(ptr %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_is_private(ptr %ptr) #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_is_private
-; AKF_HSA-SAME: (ptr [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_HSA-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
-; AKF_HSA-NEXT:    [[EXT:%.*]] = zext i1 [[IS_PRIVATE]] to i32
-; AKF_HSA-NEXT:    store i32 [[EXT]], ptr addrspace(1) poison, align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_is_private
 ; ATTRIBUTOR_HSA-SAME: (ptr [[PTR:%.*]]) #[[ATTR12]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
@@ -568,12 +427,6 @@ define amdgpu_kernel void @use_is_private(ptr %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_alloca() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_alloca
-; AKF_HSA-SAME: () #[[ATTR2:[0-9]+]] {
-; AKF_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
-; AKF_HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_alloca
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR1]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
@@ -586,15 +439,6 @@ define amdgpu_kernel void @use_alloca() #1 {
 }
 
 define amdgpu_kernel void @use_alloca_non_entry_block() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_alloca_non_entry_block
-; AKF_HSA-SAME: () #[[ATTR2]] {
-; AKF_HSA-NEXT:  entry:
-; AKF_HSA-NEXT:    br label [[BB:%.*]]
-; AKF_HSA:       bb:
-; AKF_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
-; AKF_HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_alloca_non_entry_block
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR1]] {
 ; ATTRIBUTOR_HSA-NEXT:  entry:
@@ -614,12 +458,6 @@ bb:
 }
 
 define void @use_alloca_func() #1 {
-; AKF_HSA-LABEL: define {{[^@]+}}@use_alloca_func
-; AKF_HSA-SAME: () #[[ATTR2]] {
-; AKF_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
-; AKF_HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
-; AKF_HSA-NEXT:    ret void
-;
 ; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_alloca_func
 ; ATTRIBUTOR_HSA-SAME: () #[[ATTR1]] {
 ; ATTRIBUTOR_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
@@ -638,9 +476,6 @@ attributes #1 = { nounwind }
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
 
 ;.
-; AKF_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; AKF_HSA: attributes #[[ATTR1]] = { nounwind }
-; AKF_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-stack-objects" }
 ;.
 ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
index 20ce05278d213..15dc1a0529254 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -S -mtriple=amdgcn-unknown-unknown -amdgpu-annotate-kernel-features < %s | FileCheck -check-prefixes=CHECK,AKF_CHECK %s
 ; RUN: opt -S -mtriple=amdgcn-unknown-unknown -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=CHECK,ATTRIBUTOR_CHECK %s
 
 declare i32 @llvm.r600.read.tgid.x() #0
@@ -27,12 +26,6 @@ define amdgpu_kernel void @use_tgid_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_y(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tgid_y
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; AKF_CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_y
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tgid.y()
@@ -45,14 +38,6 @@ define amdgpu_kernel void @use_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @multi_use_tgid_y(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@multi_use_tgid_y
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@multi_use_tgid_y
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.y()
@@ -69,14 +54,6 @@ define amdgpu_kernel void @multi_use_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_y(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_y
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
-; AKF_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_y
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
@@ -93,12 +70,6 @@ define amdgpu_kernel void @use_tgid_x_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_z(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tgid_z
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; AKF_CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_z
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tgid.z()
@@ -111,14 +82,6 @@ define amdgpu_kernel void @use_tgid_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_z(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_z
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
-; AKF_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_z
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
@@ -135,14 +98,6 @@ define amdgpu_kernel void @use_tgid_x_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_y_z(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tgid_y_z
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; AKF_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_y_z
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.y()
@@ -159,16 +114,6 @@ define amdgpu_kernel void @use_tgid_y_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_y_z(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_y_z
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
-; AKF_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; AKF_CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_y_z
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
@@ -201,12 +146,6 @@ define amdgpu_kernel void @use_tidig_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_y(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tidig_y
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tidig.y()
-; AKF_CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tidig_y
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tidig.y()
@@ -219,12 +158,6 @@ define amdgpu_kernel void @use_tidig_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_z(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tidig_z
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tidig.z()
-; AKF_CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tidig_z
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tidig.z()
@@ -253,14 +186,6 @@ define amdgpu_kernel void @use_tidig_x_tgid_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_y_tgid_y(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tidig_y_tgid_y
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.y()
-; AKF_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tidig_y_tgid_y
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR7:[0-9]+]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.y()
@@ -277,16 +202,6 @@ define amdgpu_kernel void @use_tidig_y_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_x_y_z(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_tidig_x_y_z
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.x()
-; AKF_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tidig.y()
-; AKF_CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tidig.z()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tidig_x_y_z
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR8:[0-9]+]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.x()
@@ -307,22 +222,6 @@ define amdgpu_kernel void @use_tidig_x_y_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_all_workitems(ptr addrspace(1) %ptr) #1 {
-; AKF_CHECK-LABEL: define {{[^@]+}}@use_all_workitems
-; AKF_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; AKF_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.x()
-; AKF_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tidig.y()
-; AKF_CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tidig.z()
-; AKF_CHECK-NEXT:    [[VAL3:%.*]] = call i32 @llvm.r600.read.tgid.x()
-; AKF_CHECK-NEXT:    [[VAL4:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; AKF_CHECK-NEXT:    [[VAL5:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL3]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL4]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    store volatile i32 [[VAL5]], ptr addrspace(1) [[PTR]], align 4
-; AKF_CHECK-NEXT:    ret void
-;
 ; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_all_workitems
 ; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR9:[0-9]+]] {
 ; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.x()
@@ -394,8 +293,6 @@ attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
 
 ;.
-; AKF_CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; AKF_CHECK: attributes #[[ATTR1]] = { nounwind }
 ;.
 ; ATTRIBUTOR_CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ; ATTRIBUTOR_CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
index fc13b86566f76..22cc5af30da66 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
@@ -35,9 +35,9 @@ entry:
 attributes #2 = {"amdgpu-flat-work-group-size"="128,128"}
 
 ; CHECK-LABEL: {{^}}min_1024_max_1024
-; CHECK: SGPRBlocks: 0
+; CHECK: SGPRBlocks: 2
 ; CHECK: VGPRBlocks: 10
-; CHECK: NumSGPRsForWavesPerEU: 2{{$}}
+; CHECK: NumSGPRsForWavesPerEU: 24{{$}}
 ; CHECK: NumVGPRsForWavesPerEU: 43
 @var = addrspace(1) global float 0.0
 define amdgpu_kernel void @min_1024_max_1024() #3 {
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll
index 46edf06c3b62c..d0107eb3ade27 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll
@@ -4,8 +4,8 @@
 
 ; ALL-LABEL: {{^}}max_10_sgprs:
 
-; ALL: SGPRBlocks: 1
-; ALL: NumSGPRsForWavesPerEU: 10
+; ALL: SGPRBlocks: 2
+; ALL: NumSGPRsForWavesPerEU: 24
 define amdgpu_kernel void @max_10_sgprs() #0 {
   %one = load volatile i32, ptr addrspace(4) poison
   %two = load volatile i32, ptr addrspace(4) poison
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-waves-per-eu.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-waves-per-eu.ll
index 14519f5a5e77c..4507fd5865989 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-waves-per-eu.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-waves-per-eu.ll
@@ -116,9 +116,9 @@ attributes #8 = {"amdgpu-waves-per-eu"="5,10"}
 
 ; Exactly 10 waves per execution unit.
 ; CHECK-LABEL: {{^}}exactly_10:
-; CHECK: SGPRBlocks: 2
+; CHECK: SGPRBlocks: 3
 ; CHECK: VGPRBlocks: 5
-; CHECK: NumSGPRsForWavesPerEU: 20
+; CHECK: NumSGPRsForWavesPerEU: 30
 ; CHECK: NumVGPRsForWavesPerEU: 24
 define amdgpu_kernel void @exactly_10() #9 {
   %val0 = load volatile float, ptr addrspace(1) @var
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll
index 682a57571d11e..35f0ccf5ba62f 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll
@@ -392,7 +392,8 @@ define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
 ; GFX10:       argumentInfo:
 ; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
 ; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
-; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr8' }
 ;
 ; GFX10: name:            call_without_private_to_flat_addrspacecast
 ; GFX10:       argumentInfo:
@@ -420,7 +421,8 @@ define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
 ; GFX10:       argumentInfo:
 ; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
 ; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
-; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr8' }
 ;
 ; GFX10: name:            call_both_with_and_without_private_to_flat_addrspacecast
 ; GFX10:       argumentInfo:
@@ -434,7 +436,8 @@ define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
 ; GFX10:       argumentInfo:
 ; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
 ; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
-; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr8' }
 ;
 ; GFX10: name:            call_call_without_private_to_flat_addrspacecast
 ; GFX10:       argumentInfo:
@@ -462,7 +465,8 @@ define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
 ; GFX10:       argumentInfo:
 ; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
 ; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
-; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr8' }
 ;
 ; GFX10: name:            call_call_both_with_and_without_private_to_flat_addrspacecast
 ; GFX10:       argumentInfo:
@@ -476,7 +480,8 @@ define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
 ; GFX10:       argumentInfo:
 ; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
 ; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
-; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr8' }
 ;
 ; GFX10: name:            with_cast_call_without_private_to_flat_addrspacecast
 ; GFX10:       argumentInfo:
@@ -490,7 +495,8 @@ define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
 ; GFX10:       argumentInfo:
 ; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
 ; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
-; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr8' }
 ;
 ; GFX10: name:            with_cast_call_with_private_to_flat_addrspacecast
 ; GFX10:       argumentInfo:
@@ -504,7 +510,8 @@ define amdgpu_kernel void @call_use_intrinsic_workitem_id_x_cc_kernel() {
 ; GFX10:       argumentInfo:
 ; GFX10-NEXT:    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
 ; GFX10-NEXT:    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
-; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr6' }
+; GFX10-NEXT:    flatScratchInit: { reg: '$sgpr6_sgpr7' }
+; GFX10-NEXT:    workGroupIDX:    { reg: '$sgpr8' }
 ;
 ; GFX10: name:            with_indirect_call
 ; GFX10:       argumentInfo:
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior.ll b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior.ll
new file mode 100644
index 0000000000000..1b422252573db
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=GFX10 %s
+
+;
+; None of these functions should have the attribute amdgpu-no-flat-scratch-init. In these tests
+; we manually set the attribute for the functions. The purpose is to test how the amdgpu-attributor pass
+; handles this situation.
+;
+;; tests of addrspacecast
+
+define void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) #0 {
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) #0 {
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) #0 {
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) #0 {
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+;; tests of addrspacecast in a constant
+
+define amdgpu_kernel void @private_constant_expression_use(ptr addrspace(1) nocapture %out) #0 {
+  store volatile ptr addrspacecast (ptr addrspace(5) inttoptr (i32 123 to ptr addrspace(5)) to ptr), ptr addrspace(1) %out, align 8
+  ret void
+}
+
+;; tests of intrinsics
+
+define amdgpu_kernel void @calls_intrin_ascast_cc_kernel(ptr addrspace(3) %ptr) #0 {
+  %1 = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p3(ptr addrspace(3) %ptr)
+  store volatile i32 7, ptr %1, align 4
+  ret void
+}
+
+define void @calls_intrin_ascast(ptr addrspace(3) %ptr) #0 {
+  %1 = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p3(ptr addrspace(3) %ptr)
+  store volatile i32 7, ptr %1, align 4
+  ret void
+}
+
+define amdgpu_kernel void @call_calls_intrin_ascast_cc_kernel(ptr addrspace(3) %ptr) #0 {
+  call void @calls_intrin_ascast(ptr addrspace(3) %ptr)
+  ret void
+}
+
+attributes #0 = { "amdgpu-no-flat-scratch-init" }
+
+; GFX9: attributes #0 = { "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx900" "uniform-work-group-size"="false" }
+; GFX10: attributes #0 = { "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx1010" "uniform-work-group-size"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior2.ll b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior2.ll
new file mode 100644
index 0000000000000..51caa84450ff3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior2.ll
@@ -0,0 +1,870 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=+architected-flat-scratch < %s | FileCheck -check-prefixes=GFX8-ARCH-FLAT %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch < %s | FileCheck -check-prefixes=GFX9-ARCH-FLAT %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -mattr=+architected-flat-scratch < %s | FileCheck -check-prefixes=GFX942-ARCH-FLAT %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10 %s
+
+;
+; None of these functions should have the attribute amdgpu-no-flat-scratch-init. In these tests
+; we manually set the attribute for the functions. The purpose is to test how llc handles this.
+;
+
+;; tests of addrspacecast
+
+define void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) #0 {
+; GFX8-LABEL: with_private_to_flat_addrspacecast:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0xc0
+; GFX8-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-ARCH-FLAT-LABEL: with_private_to_flat_addrspacecast:
+; GFX8-ARCH-FLAT:       ; %bb.0:
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], 0xc0
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX8-ARCH-FLAT-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-ARCH-FLAT-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-ARCH-FLAT-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: with_private_to_flat_addrspacecast:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    flat_store_dword v[0:1], v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-ARCH-FLAT-LABEL: with_private_to_flat_addrspacecast:
+; GFX9-ARCH-FLAT:       ; %bb.0:
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-ARCH-FLAT-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-ARCH-FLAT-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-ARCH-FLAT-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-ARCH-FLAT-LABEL: with_private_to_flat_addrspacecast:
+; GFX942-ARCH-FLAT:       ; %bb.0:
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s1
+; GFX942-ARCH-FLAT-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-ARCH-FLAT-NEXT:    s_nop 0
+; GFX942-ARCH-FLAT-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX942-ARCH-FLAT-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX942-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: with_private_to_flat_addrspacecast:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, -1, v0
+; GFX10-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, s5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX10-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) #0 {
+; GFX8-LABEL: with_private_to_flat_addrspacecast_cc_kernel:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX8-NEXT:    s_load_dword s1, s[8:9], 0xc8
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_cmp_lg_u32 s0, -1
+; GFX8-NEXT:    s_cselect_b32 s1, s1, 0
+; GFX8-NEXT:    s_cselect_b32 s0, s0, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_endpgm
+;
+; GFX8-ARCH-FLAT-LABEL: with_private_to_flat_addrspacecast_cc_kernel:
+; GFX8-ARCH-FLAT:       ; %bb.0:
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s1, s[4:5], 0xc8
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_cmp_lg_u32 s0, -1
+; GFX8-ARCH-FLAT-NEXT:    s_cselect_b32 s1, s1, 0
+; GFX8-ARCH-FLAT-NEXT:    s_cselect_b32 s0, s0, 0
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX9-LABEL: with_private_to_flat_addrspacecast_cc_kernel:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x0
+; GFX9-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_cmp_lg_u32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s0, s1, 0
+; GFX9-NEXT:    s_cselect_b32 s1, s2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    flat_store_dword v[0:1], v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-ARCH-FLAT-LABEL: with_private_to_flat_addrspacecast_cc_kernel:
+; GFX9-ARCH-FLAT:       ; %bb.0:
+; GFX9-ARCH-FLAT-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_cmp_lg_u32 s2, -1
+; GFX9-ARCH-FLAT-NEXT:    s_cselect_b32 s0, s1, 0
+; GFX9-ARCH-FLAT-NEXT:    s_cselect_b32 s1, s2, 0
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX942-ARCH-FLAT-LABEL: with_private_to_flat_addrspacecast_cc_kernel:
+; GFX942-ARCH-FLAT:       ; %bb.0:
+; GFX942-ARCH-FLAT-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_cmp_lg_u32 s2, -1
+; GFX942-ARCH-FLAT-NEXT:    s_cselect_b32 s0, s1, 0
+; GFX942-ARCH-FLAT-NEXT:    s_cselect_b32 s1, s2, 0
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s1
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s0
+; GFX942-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX10-LABEL: with_private_to_flat_addrspacecast_cc_kernel:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dword s2, s[8:9], 0x0
+; GFX10-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_cmp_lg_u32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s0, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_endpgm
+  %stof = addrspacecast ptr addrspace(5) %ptr to ptr
+  store volatile i32 0, ptr %stof
+  ret void
+}
+
+define void @call_with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) #0 {
+; GFX8-LABEL: call_with_private_to_flat_addrspacecast:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s18, s33
+; GFX8-NEXT:    s_mov_b32 s33, s32
+; GFX8-NEXT:    s_xor_saveexec_b64 s[16:17], -1
+; GFX8-NEXT:    buffer_store_dword v3, off, s[0:3], s33 ; 4-byte Folded Spill
+; GFX8-NEXT:    s_mov_b64 exec, s[16:17]
+; GFX8-NEXT:    s_addk_i32 s32, 0x400
+; GFX8-NEXT:    s_getpc_b64 s[16:17]
+; GFX8-NEXT:    s_add_u32 s16, s16, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX8-NEXT:    s_addc_u32 s17, s17, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX8-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX8-NEXT:    v_writelane_b32 v3, s30, 0
+; GFX8-NEXT:    v_writelane_b32 v3, s31, 1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX8-NEXT:    v_readlane_b32 s31, v3, 1
+; GFX8-NEXT:    v_readlane_b32 s30, v3, 0
+; GFX8-NEXT:    s_mov_b32 s32, s33
+; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    buffer_load_dword v3, off, s[0:3], s33 ; 4-byte Folded Reload
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    s_mov_b32 s33, s18
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-ARCH-FLAT-LABEL: call_with_private_to_flat_addrspacecast:
+; GFX8-ARCH-FLAT:       ; %bb.0:
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s2, s33
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s33, s32
+; GFX8-ARCH-FLAT-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; GFX8-ARCH-FLAT-NEXT:    s_add_i32 s3, s33, 8
+; GFX8-ARCH-FLAT-NEXT:    scratch_store_dword off, v3, s3 ; 4-byte Folded Spill
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX8-ARCH-FLAT-NEXT:    s_add_i32 s32, s32, 16
+; GFX8-ARCH-FLAT-NEXT:    s_getpc_b64 s[0:1]
+; GFX8-ARCH-FLAT-NEXT:    s_add_u32 s0, s0, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX8-ARCH-FLAT-NEXT:    s_addc_u32 s1, s1, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX8-ARCH-FLAT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX8-ARCH-FLAT-NEXT:    v_writelane_b32 v3, s30, 0
+; GFX8-ARCH-FLAT-NEXT:    v_writelane_b32 v3, s31, 1
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX8-ARCH-FLAT-NEXT:    v_readlane_b32 s31, v3, 1
+; GFX8-ARCH-FLAT-NEXT:    v_readlane_b32 s30, v3, 0
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s32, s33
+; GFX8-ARCH-FLAT-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; GFX8-ARCH-FLAT-NEXT:    s_add_i32 s3, s33, 8
+; GFX8-ARCH-FLAT-NEXT:    scratch_load_dword v3, off, s3 ; 4-byte Folded Reload
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s33, s2
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: call_with_private_to_flat_addrspacecast:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s18, s33
+; GFX9-NEXT:    s_mov_b32 s33, s32
+; GFX9-NEXT:    s_xor_saveexec_b64 s[16:17], -1
+; GFX9-NEXT:    buffer_store_dword v3, off, s[0:3], s33 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_mov_b64 exec, s[16:17]
+; GFX9-NEXT:    s_addk_i32 s32, 0x400
+; GFX9-NEXT:    s_getpc_b64 s[16:17]
+; GFX9-NEXT:    s_add_u32 s16, s16, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s17, s17, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX9-NEXT:    v_writelane_b32 v3, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v3, s31, 1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT:    v_readlane_b32 s31, v3, 1
+; GFX9-NEXT:    v_readlane_b32 s30, v3, 0
+; GFX9-NEXT:    s_mov_b32 s32, s33
+; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s33 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s33, s18
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-ARCH-FLAT-LABEL: call_with_private_to_flat_addrspacecast:
+; GFX9-ARCH-FLAT:       ; %bb.0:
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s2, s33
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s33, s32
+; GFX9-ARCH-FLAT-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; GFX9-ARCH-FLAT-NEXT:    scratch_store_dword off, v3, s33 ; 4-byte Folded Spill
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-ARCH-FLAT-NEXT:    s_add_i32 s32, s32, 16
+; GFX9-ARCH-FLAT-NEXT:    s_getpc_b64 s[0:1]
+; GFX9-ARCH-FLAT-NEXT:    s_add_u32 s0, s0, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX9-ARCH-FLAT-NEXT:    s_addc_u32 s1, s1, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX9-ARCH-FLAT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX9-ARCH-FLAT-NEXT:    v_writelane_b32 v3, s30, 0
+; GFX9-ARCH-FLAT-NEXT:    v_writelane_b32 v3, s31, 1
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX9-ARCH-FLAT-NEXT:    v_readlane_b32 s31, v3, 1
+; GFX9-ARCH-FLAT-NEXT:    v_readlane_b32 s30, v3, 0
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s32, s33
+; GFX9-ARCH-FLAT-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; GFX9-ARCH-FLAT-NEXT:    scratch_load_dword v3, off, s33 ; 4-byte Folded Reload
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s33, s2
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-ARCH-FLAT-LABEL: call_with_private_to_flat_addrspacecast:
+; GFX942-ARCH-FLAT:       ; %bb.0:
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s2, s33
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s33, s32
+; GFX942-ARCH-FLAT-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; GFX942-ARCH-FLAT-NEXT:    scratch_store_dword off, v3, s33 ; 4-byte Folded Spill
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX942-ARCH-FLAT-NEXT:    s_add_i32 s32, s32, 16
+; GFX942-ARCH-FLAT-NEXT:    s_getpc_b64 s[0:1]
+; GFX942-ARCH-FLAT-NEXT:    s_add_u32 s0, s0, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX942-ARCH-FLAT-NEXT:    s_addc_u32 s1, s1, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX942-ARCH-FLAT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX942-ARCH-FLAT-NEXT:    v_writelane_b32 v3, s30, 0
+; GFX942-ARCH-FLAT-NEXT:    v_writelane_b32 v3, s31, 1
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX942-ARCH-FLAT-NEXT:    v_readlane_b32 s31, v3, 1
+; GFX942-ARCH-FLAT-NEXT:    v_readlane_b32 s30, v3, 0
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s32, s33
+; GFX942-ARCH-FLAT-NEXT:    s_xor_saveexec_b64 s[0:1], -1
+; GFX942-ARCH-FLAT-NEXT:    scratch_load_dword v3, off, s33 ; 4-byte Folded Reload
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s33, s2
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: call_with_private_to_flat_addrspacecast:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s18, s33
+; GFX10-NEXT:    s_mov_b32 s33, s32
+; GFX10-NEXT:    s_xor_saveexec_b32 s16, -1
+; GFX10-NEXT:    buffer_store_dword v3, off, s[0:3], s33 ; 4-byte Folded Spill
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_mov_b32 exec_lo, s16
+; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    s_getpc_b64 s[16:17]
+; GFX10-NEXT:    s_add_u32 s16, s16, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX10-NEXT:    s_addc_u32 s17, s17, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX10-NEXT:    v_writelane_b32 v3, s30, 0
+; GFX10-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX10-NEXT:    v_writelane_b32 v3, s31, 1
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX10-NEXT:    v_readlane_b32 s31, v3, 1
+; GFX10-NEXT:    v_readlane_b32 s30, v3, 0
+; GFX10-NEXT:    s_mov_b32 s32, s33
+; GFX10-NEXT:    s_xor_saveexec_b32 s4, -1
+; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s33 ; 4-byte Folded Reload
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_mov_b32 exec_lo, s4
+; GFX10-NEXT:    s_mov_b32 s33, s18
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+define amdgpu_kernel void @call_with_private_to_flat_addrspacecast_cc_kernel(ptr addrspace(5) %ptr) #0 {
+; GFX8-LABEL: call_with_private_to_flat_addrspacecast_cc_kernel:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s0, s0, s15
+; GFX8-NEXT:    s_addc_u32 s1, s1, 0
+; GFX8-NEXT:    s_load_dword s15, s[8:9], 0x0
+; GFX8-NEXT:    s_add_u32 s8, s8, 8
+; GFX8-NEXT:    s_addc_u32 s9, s9, 0
+; GFX8-NEXT:    s_getpc_b64 s[16:17]
+; GFX8-NEXT:    s_add_u32 s16, s16, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX8-NEXT:    s_addc_u32 s17, s17, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX8-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s15
+; GFX8-NEXT:    s_mov_b32 s32, 0
+; GFX8-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX8-ARCH-FLAT-LABEL: call_with_private_to_flat_addrspacecast_cc_kernel:
+; GFX8-ARCH-FLAT:       ; %bb.0:
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s12, s8
+; GFX8-ARCH-FLAT-NEXT:    s_add_u32 s8, s4, 8
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s13, s9
+; GFX8-ARCH-FLAT-NEXT:    s_addc_u32 s9, s5, 0
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s15, s[4:5], 0x0
+; GFX8-ARCH-FLAT-NEXT:    s_getpc_b64 s[4:5]
+; GFX8-ARCH-FLAT-NEXT:    s_add_u32 s4, s4, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX8-ARCH-FLAT-NEXT:    s_addc_u32 s5, s5, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX8-ARCH-FLAT-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX8-ARCH-FLAT-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX8-ARCH-FLAT-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX8-ARCH-FLAT-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s14, s10
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX8-ARCH-FLAT-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s15
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s32, 0
+; GFX8-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX8-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX9-LABEL: call_with_private_to_flat_addrspacecast_cc_kernel:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s0, s0, s15
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
+; GFX9-NEXT:    s_load_dword s15, s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 s8, s8, 8
+; GFX9-NEXT:    s_addc_u32 s9, s9, 0
+; GFX9-NEXT:    s_getpc_b64 s[16:17]
+; GFX9-NEXT:    s_add_u32 s16, s16, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s17, s17, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s15
+; GFX9-NEXT:    s_mov_b32 s32, 0
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-ARCH-FLAT-LABEL: call_with_private_to_flat_addrspacecast_cc_kernel:
+; GFX9-ARCH-FLAT:       ; %bb.0:
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s12, s8
+; GFX9-ARCH-FLAT-NEXT:    s_add_u32 s8, s4, 8
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s13, s9
+; GFX9-ARCH-FLAT-NEXT:    s_addc_u32 s9, s5, 0
+; GFX9-ARCH-FLAT-NEXT:    s_load_dword s15, s[4:5], 0x0
+; GFX9-ARCH-FLAT-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-ARCH-FLAT-NEXT:    s_add_u32 s4, s4, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX9-ARCH-FLAT-NEXT:    s_addc_u32 s5, s5, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX9-ARCH-FLAT-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-ARCH-FLAT-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX9-ARCH-FLAT-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s14, s10
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX9-ARCH-FLAT-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s15
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s32, 0
+; GFX9-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX9-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX942-ARCH-FLAT-LABEL: call_with_private_to_flat_addrspacecast_cc_kernel:
+; GFX942-ARCH-FLAT:       ; %bb.0:
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s12, s8
+; GFX942-ARCH-FLAT-NEXT:    s_add_u32 s8, s4, 8
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s13, s9
+; GFX942-ARCH-FLAT-NEXT:    s_addc_u32 s9, s5, 0
+; GFX942-ARCH-FLAT-NEXT:    s_load_dword s15, s[4:5], 0x0
+; GFX942-ARCH-FLAT-NEXT:    s_getpc_b64 s[4:5]
+; GFX942-ARCH-FLAT-NEXT:    s_add_u32 s4, s4, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX942-ARCH-FLAT-NEXT:    s_addc_u32 s5, s5, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX942-ARCH-FLAT-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s14, s10
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v31, v0
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s15
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s32, 0
+; GFX942-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX942-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX10-LABEL: call_with_private_to_flat_addrspacecast_cc_kernel:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    s_load_dword s15, s[8:9], 0x0
+; GFX10-NEXT:    s_add_u32 s8, s8, 8
+; GFX10-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-NEXT:    s_getpc_b64 s[16:17]
+; GFX10-NEXT:    s_add_u32 s16, s16, with_private_to_flat_addrspacecast@gotpcrel32@lo+4
+; GFX10-NEXT:    s_addc_u32 s17, s17, with_private_to_flat_addrspacecast@gotpcrel32@hi+12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX10-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX10-NEXT:    s_mov_b32 s32, 0
+; GFX10-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s15
+; GFX10-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX10-NEXT:    s_endpgm
+  call void @with_private_to_flat_addrspacecast(ptr addrspace(5) %ptr)
+  ret void
+}
+
+;; tests of addrspacecast in a constant
+
+define amdgpu_kernel void @private_constant_expression_use(ptr addrspace(1) nocapture %out) #0 {
+; GFX8-LABEL: private_constant_expression_use:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s2, s[8:9], 0xc8
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_endpgm
+;
+; GFX8-ARCH-FLAT-LABEL: private_constant_expression_use:
+; GFX8-ARCH-FLAT:       ; %bb.0:
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s2, s[4:5], 0xc8
+; GFX8-ARCH-FLAT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-ARCH-FLAT-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX9-LABEL: private_constant_expression_use:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-ARCH-FLAT-LABEL: private_constant_expression_use:
+; GFX9-ARCH-FLAT:       ; %bb.0:
+; GFX9-ARCH-FLAT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX942-ARCH-FLAT-LABEL: private_constant_expression_use:
+; GFX942-ARCH-FLAT:       ; %bb.0:
+; GFX942-ARCH-FLAT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s3
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX10-LABEL: private_constant_expression_use:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX10-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_endpgm
+  store volatile ptr addrspacecast (ptr addrspace(5) inttoptr (i32 123 to ptr addrspace(5)) to ptr), ptr addrspace(1) %out, align 8
+  ret void
+}
+
+;; tests of intrinsics
+
+define amdgpu_kernel void @calls_intrin_ascast_cc_kernel(ptr addrspace(3) %ptr) #0 {
+; GFX8-LABEL: calls_intrin_ascast_cc_kernel:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX8-NEXT:    s_load_dword s1, s[8:9], 0xcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, 7
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_endpgm
+;
+; GFX8-ARCH-FLAT-LABEL: calls_intrin_ascast_cc_kernel:
+; GFX8-ARCH-FLAT:       ; %bb.0:
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s1, s[4:5], 0xcc
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 7
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX9-LABEL: calls_intrin_ascast_cc_kernel:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x0
+; GFX9-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 7
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    flat_store_dword v[0:1], v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-ARCH-FLAT-LABEL: calls_intrin_ascast_cc_kernel:
+; GFX9-ARCH-FLAT:       ; %bb.0:
+; GFX9-ARCH-FLAT-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 7
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX942-ARCH-FLAT-LABEL: calls_intrin_ascast_cc_kernel:
+; GFX942-ARCH-FLAT:       ; %bb.0:
+; GFX942-ARCH-FLAT-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s1
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 7
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s2
+; GFX942-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX10-LABEL: calls_intrin_ascast_cc_kernel:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dword s2, s[8:9], 0x0
+; GFX10-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX10-NEXT:    v_mov_b32_e32 v2, 7
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_endpgm
+  %1 = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p3(ptr addrspace(3) %ptr)
+  store volatile i32 7, ptr %1, align 4
+  ret void
+}
+
+define void @calls_intrin_ascast(ptr addrspace(3) %ptr) #0 {
+; GFX8-LABEL: calls_intrin_ascast:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0xc4
+; GFX8-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 7
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-ARCH-FLAT-LABEL: calls_intrin_ascast:
+; GFX8-ARCH-FLAT:       ; %bb.0:
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], 0xc4
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 7
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: calls_intrin_ascast:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], src_shared_base
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, 7
+; GFX9-NEXT:    flat_store_dword v[0:1], v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-ARCH-FLAT-LABEL: calls_intrin_ascast:
+; GFX9-ARCH-FLAT:       ; %bb.0:
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 7
+; GFX9-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-ARCH-FLAT-LABEL: calls_intrin_ascast:
+; GFX942-ARCH-FLAT:       ; %bb.0:
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v1, s1
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v2, 7
+; GFX942-ARCH-FLAT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: calls_intrin_ascast:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b64 s[4:5], src_shared_base
+; GFX10-NEXT:    v_mov_b32_e32 v2, 7
+; GFX10-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %1 = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p3(ptr addrspace(3) %ptr)
+  store volatile i32 7, ptr %1, align 4
+  ret void
+}
+
+define amdgpu_kernel void @call_calls_intrin_ascast_cc_kernel(ptr addrspace(3) %ptr) #0 {
+; GFX8-LABEL: call_calls_intrin_ascast_cc_kernel:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s0, s0, s15
+; GFX8-NEXT:    s_addc_u32 s1, s1, 0
+; GFX8-NEXT:    s_load_dword s15, s[8:9], 0x0
+; GFX8-NEXT:    s_add_u32 s8, s8, 8
+; GFX8-NEXT:    s_addc_u32 s9, s9, 0
+; GFX8-NEXT:    s_getpc_b64 s[16:17]
+; GFX8-NEXT:    s_add_u32 s16, s16, calls_intrin_ascast@gotpcrel32@lo+4
+; GFX8-NEXT:    s_addc_u32 s17, s17, calls_intrin_ascast@gotpcrel32@hi+12
+; GFX8-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s15
+; GFX8-NEXT:    s_mov_b32 s32, 0
+; GFX8-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX8-ARCH-FLAT-LABEL: call_calls_intrin_ascast_cc_kernel:
+; GFX8-ARCH-FLAT:       ; %bb.0:
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s12, s8
+; GFX8-ARCH-FLAT-NEXT:    s_add_u32 s8, s4, 8
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s13, s9
+; GFX8-ARCH-FLAT-NEXT:    s_addc_u32 s9, s5, 0
+; GFX8-ARCH-FLAT-NEXT:    s_load_dword s15, s[4:5], 0x0
+; GFX8-ARCH-FLAT-NEXT:    s_getpc_b64 s[4:5]
+; GFX8-ARCH-FLAT-NEXT:    s_add_u32 s4, s4, calls_intrin_ascast@gotpcrel32@lo+4
+; GFX8-ARCH-FLAT-NEXT:    s_addc_u32 s5, s5, calls_intrin_ascast@gotpcrel32@hi+12
+; GFX8-ARCH-FLAT-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX8-ARCH-FLAT-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX8-ARCH-FLAT-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX8-ARCH-FLAT-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s14, s10
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX8-ARCH-FLAT-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX8-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s15
+; GFX8-ARCH-FLAT-NEXT:    s_mov_b32 s32, 0
+; GFX8-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX8-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX9-LABEL: call_calls_intrin_ascast_cc_kernel:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s0, s0, s15
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
+; GFX9-NEXT:    s_load_dword s15, s[8:9], 0x0
+; GFX9-NEXT:    s_add_u32 s8, s8, 8
+; GFX9-NEXT:    s_addc_u32 s9, s9, 0
+; GFX9-NEXT:    s_getpc_b64 s[16:17]
+; GFX9-NEXT:    s_add_u32 s16, s16, calls_intrin_ascast@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s17, s17, calls_intrin_ascast@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s15
+; GFX9-NEXT:    s_mov_b32 s32, 0
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-ARCH-FLAT-LABEL: call_calls_intrin_ascast_cc_kernel:
+; GFX9-ARCH-FLAT:       ; %bb.0:
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s12, s8
+; GFX9-ARCH-FLAT-NEXT:    s_add_u32 s8, s4, 8
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s13, s9
+; GFX9-ARCH-FLAT-NEXT:    s_addc_u32 s9, s5, 0
+; GFX9-ARCH-FLAT-NEXT:    s_load_dword s15, s[4:5], 0x0
+; GFX9-ARCH-FLAT-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-ARCH-FLAT-NEXT:    s_add_u32 s4, s4, calls_intrin_ascast@gotpcrel32@lo+4
+; GFX9-ARCH-FLAT-NEXT:    s_addc_u32 s5, s5, calls_intrin_ascast@gotpcrel32@hi+12
+; GFX9-ARCH-FLAT-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-ARCH-FLAT-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX9-ARCH-FLAT-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s14, s10
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX9-ARCH-FLAT-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX9-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s15
+; GFX9-ARCH-FLAT-NEXT:    s_mov_b32 s32, 0
+; GFX9-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX9-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX942-ARCH-FLAT-LABEL: call_calls_intrin_ascast_cc_kernel:
+; GFX942-ARCH-FLAT:       ; %bb.0:
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s12, s8
+; GFX942-ARCH-FLAT-NEXT:    s_add_u32 s8, s4, 8
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s13, s9
+; GFX942-ARCH-FLAT-NEXT:    s_addc_u32 s9, s5, 0
+; GFX942-ARCH-FLAT-NEXT:    s_load_dword s15, s[4:5], 0x0
+; GFX942-ARCH-FLAT-NEXT:    s_getpc_b64 s[4:5]
+; GFX942-ARCH-FLAT-NEXT:    s_add_u32 s4, s4, calls_intrin_ascast@gotpcrel32@lo+4
+; GFX942-ARCH-FLAT-NEXT:    s_addc_u32 s5, s5, calls_intrin_ascast@gotpcrel32@hi+12
+; GFX942-ARCH-FLAT-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s14, s10
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v31, v0
+; GFX942-ARCH-FLAT-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-ARCH-FLAT-NEXT:    v_mov_b32_e32 v0, s15
+; GFX942-ARCH-FLAT-NEXT:    s_mov_b32 s32, 0
+; GFX942-ARCH-FLAT-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX942-ARCH-FLAT-NEXT:    s_endpgm
+;
+; GFX10-LABEL: call_calls_intrin_ascast_cc_kernel:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    s_load_dword s15, s[8:9], 0x0
+; GFX10-NEXT:    s_add_u32 s8, s8, 8
+; GFX10-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-NEXT:    s_getpc_b64 s[16:17]
+; GFX10-NEXT:    s_add_u32 s16, s16, calls_intrin_ascast@gotpcrel32@lo+4
+; GFX10-NEXT:    s_addc_u32 s17, s17, calls_intrin_ascast@gotpcrel32@hi+12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX10-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX10-NEXT:    s_mov_b32 s32, 0
+; GFX10-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s15
+; GFX10-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX10-NEXT:    s_endpgm
+  call void @calls_intrin_ascast(ptr addrspace(3) %ptr)
+  ret void
+}
+
+attributes #0 = { "amdgpu-no-flat-scratch-init" }
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noopt.ll b/llvm/test/CodeGen/AMDGPU/attributor-noopt.ll
index 55ed11ac62972..4f341fa71cf68 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-noopt.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-noopt.ll
@@ -30,9 +30,11 @@
 ; NOOPT: .amdhsa_system_sgpr_workgroup_id_z 1
 ; NOOPT: .amdhsa_system_sgpr_workgroup_info 0
 ; NOOPT: .amdhsa_system_vgpr_workitem_id 2
-define amdgpu_kernel void @foo() {
+define amdgpu_kernel void @foo() #0 {
   ret void
 }
 
+attributes #0 = { "amdgpu-no-flat-scratch-init" }
+
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION}
diff --git a/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll b/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll
index d5da3e00df1a6..10ca3c9d5f2c8 100644
--- a/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll
+++ b/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll
@@ -198,11 +198,11 @@ define hidden void @use_workgroup_id_yz() #1 {
 
 ; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_x:
 ; GCN-NOT: s6
-; GCN: s_getpc_b64 s[4:5]
-; GCN-NEXT: s_add_u32 s4, s4, use_workgroup_id_x@rel32@lo+4
-; GCN-NEXT: s_addc_u32 s5, s5, use_workgroup_id_x@rel32@hi+12
+; GCN: s_getpc_b64 s[6:7]
+; GCN-NEXT: s_add_u32 s6, s6, use_workgroup_id_x@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s7, s7, use_workgroup_id_x@rel32@hi+12
 ; GCN-NOT: s6
-; GCN: s_mov_b32 s12, s6
+; GCN: s_mov_b32 s12, s4
 ; GCN: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
 ; GCN-NEXT: s_endpgm
@@ -217,7 +217,7 @@ define amdgpu_kernel void @kern_indirect_use_workgroup_id_x() #1 {
 
 ; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_y:
 ; GCN-NOT: s12
-; GCN: s_mov_b32 s13, s7
+; GCN: s_mov_b32 s13, s5
 ; GCN-NOT: s12
 ; GCN: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
@@ -233,7 +233,7 @@ define amdgpu_kernel void @kern_indirect_use_workgroup_id_y() #1 {
 ; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_z:
 ; GCN-NOT: s12
 ; GCN-NOT: s13
-; GCN: s_mov_b32 s14, s7
+; GCN: s_mov_b32 s14, s5
 ; GCN-NOT: s12
 ; GCN-NOT: s13
 
@@ -250,8 +250,8 @@ define amdgpu_kernel void @kern_indirect_use_workgroup_id_z() #1 {
 
 ; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_xy:
 ; GCN-NOT: s14
-; GCN: s_mov_b32 s12, s6
-; GCN-NEXT: s_mov_b32 s13, s7
+; GCN: s_mov_b32 s12, s4
+; GCN-NEXT: s_mov_b32 s13, s5
 ; GCN-NOT: s14
 
 ; GCN: s_mov_b32 s32, 0
@@ -266,9 +266,9 @@ define amdgpu_kernel void @kern_indirect_use_workgroup_id_xy() #1 {
 }
 
 ; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_xyz:
-; GCN: s_mov_b32 s12, s6
-; GCN: s_mov_b32 s13, s7
-; GCN: s_mov_b32 s14, s8
+; GCN: s_mov_b32 s12, s4
+; GCN: s_mov_b32 s13, s5
+; GCN: s_mov_b32 s14, s6
 ; GCN: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
 
@@ -283,8 +283,8 @@ define amdgpu_kernel void @kern_indirect_use_workgroup_id_xyz() #1 {
 ; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_xz:
 
 ; GCN-NOT: s13
-; GCN: s_mov_b32 s12, s6
-; GCN-NEXT: s_mov_b32 s14, s7
+; GCN: s_mov_b32 s12, s4
+; GCN-NEXT: s_mov_b32 s14, s5
 ; GCN-NOT: s13
 
 ; GCN: s_mov_b32 s32, 0
@@ -300,8 +300,8 @@ define amdgpu_kernel void @kern_indirect_use_workgroup_id_xz() #1 {
 
 ; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_yz:
 
-; GCN: s_mov_b32 s13, s7
-; GCN: s_mov_b32 s14, s8
+; GCN: s_mov_b32 s13, s5
+; GCN: s_mov_b32 s14, s6
 
 ; GCN: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
@@ -382,7 +382,7 @@ define hidden void @other_arg_use_workgroup_id_z(i32 %arg0) #1 {
 
 ; GCN-NOT: s13
 ; GCN-NOT: s14
-; GCN-DAG: s_mov_b32 s12, s6
+; GCN-DAG: s_mov_b32 s12, s4
 ; GCN-DAG: v_mov_b32_e32 v0, 0x22b
 ; GCN-NOT: s13
 ; GCN-NOT: s14
@@ -400,7 +400,7 @@ define amdgpu_kernel void @kern_indirect_other_arg_use_workgroup_id_x() #1 {
 
 ; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workgroup_id_y:
 ; GCN-DAG: v_mov_b32_e32 v0, 0x22b
-; GCN-DAG: s_mov_b32 s13, s7
+; GCN-DAG: s_mov_b32 s13, s5
 
 ; GCN-DAG: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
@@ -415,7 +415,7 @@ define amdgpu_kernel void @kern_indirect_other_arg_use_workgroup_id_y() #1 {
 
 ; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workgroup_id_z:
 ; GCN-DAG: v_mov_b32_e32 v0, 0x22b
-; GCN-DAG: s_mov_b32 s14, s7
+; GCN-DAG: s_mov_b32 s14, s5
 
 ; GCN: s_mov_b32 s32, 0
 ; GCN: s_swappc_b64
@@ -474,7 +474,7 @@ define hidden void @use_every_sgpr_input() #1 {
 ; GCN: .amdhsa_user_sgpr_queue_ptr 1
 ; GCN: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 ; GCN: .amdhsa_user_sgpr_dispatch_id 1
-; GCN: .amdhsa_user_sgpr_flat_scratch_init 1
+; GCN: .amdhsa_user_sgpr_flat_scratch_init 0
 ; GCN: .amdhsa_user_sgpr_private_segment_size 0
 ; GCN: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; GCN: .amdhsa_system_sgpr_workgroup_id_x 1
@@ -499,7 +499,7 @@ define amdgpu_kernel void @kern_indirect_use_every_sgpr_input(i8) #1 {
 ; GCN: .amdhsa_user_sgpr_queue_ptr 1
 ; GCN: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; GCN: .amdhsa_user_sgpr_dispatch_id 1
-; GCN: .amdhsa_user_sgpr_flat_scratch_init 1
+; GCN: .amdhsa_user_sgpr_flat_scratch_init 0
 ; GCN: .amdhsa_user_sgpr_private_segment_size 0
 ; GCN: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; GCN: .amdhsa_system_sgpr_workgroup_id_x 1
diff --git a/llvm/test/CodeGen/AMDGPU/code-object-v3.ll b/llvm/test/CodeGen/AMDGPU/code-object-v3.ll
index ee4a2ed883b63..3fe3cafd729a7 100644
--- a/llvm/test/CodeGen/AMDGPU/code-object-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/code-object-v3.ll
@@ -68,7 +68,7 @@
 define amdgpu_kernel void @fadd(
     ptr addrspace(1) %r,
     ptr addrspace(1) %a,
-    ptr addrspace(1) %b) {
+    ptr addrspace(1) %b) #0 {
 entry:
   %a.val = load float, ptr addrspace(1) %a
   %b.val = load float, ptr addrspace(1) %b
@@ -80,7 +80,7 @@ entry:
 define amdgpu_kernel void @fsub(
     ptr addrspace(1) %r,
     ptr addrspace(1) %a,
-    ptr addrspace(1) %b) {
+    ptr addrspace(1) %b) #0 {
 entry:
   %a.val = load float, ptr addrspace(1) %a
   %b.val = load float, ptr addrspace(1) %b
@@ -99,7 +99,9 @@ define amdgpu_kernel void @empty(
     i32 %i,
     ptr addrspace(1) %r,
     ptr addrspace(1) %a,
-    ptr addrspace(1) %b) {
+    ptr addrspace(1) %b) #0 {
 entry:
   ret void
 }
+
+attributes #0 = { "amdgpu-no-flat-scratch-init" }
diff --git a/llvm/test/CodeGen/AMDGPU/combine-reg-or-const.ll b/llvm/test/CodeGen/AMDGPU/combine-reg-or-const.ll
index c17cf1cd6bca4..c167834470e3b 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-reg-or-const.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-reg-or-const.ll
@@ -5,6 +5,9 @@
 define protected amdgpu_kernel void @_Z11test_kernelPii(ptr addrspace(1) nocapture %Ad.coerce, i32 %s) local_unnamed_addr #5 {
 ; CHECK-LABEL: _Z11test_kernelPii:
 ; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-NEXT:    s_load_dword s0, s[8:9], 0x2
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_cmp_lg_u32 s0, 3
diff --git a/llvm/test/CodeGen/AMDGPU/dagcomb-extract-vec-elt-different-sizes.ll b/llvm/test/CodeGen/AMDGPU/dagcomb-extract-vec-elt-different-sizes.ll
index fcb8fa5997b7e..fc17d9288bf40 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcomb-extract-vec-elt-different-sizes.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcomb-extract-vec-elt-different-sizes.ll
@@ -6,6 +6,8 @@
 define amdgpu_kernel void @eggs(i1 %arg, ptr addrspace(1) %arg1, ptr %arg2, ptr %arg3, ptr %arg4, ptr %arg5, ptr %arg6, ptr %arg7, ptr %arg8, ptr %arg9) {
 ; CHECK-LABEL: eggs:
 ; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_load_dword s0, s[8:9], 0x0
 ; CHECK-NEXT:    s_load_dwordx16 s[12:27], s[8:9], 0x8
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll b/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll
index 39554e05c96b4..f964170ccdda5 100644
--- a/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll
+++ b/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll
@@ -1,11 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-annotate-kernel-features  %s | FileCheck -check-prefix=AKF_GCN %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-attributor %s | FileCheck -check-prefix=ATTRIBUTOR_GCN %s
 
 define internal void @indirect() {
-; AKF_GCN-LABEL: define {{[^@]+}}@indirect() {
-; AKF_GCN-NEXT:    ret void
-;
 ; ATTRIBUTOR_GCN-LABEL: define {{[^@]+}}@indirect
 ; ATTRIBUTOR_GCN-SAME: () #[[ATTR0:[0-9]+]] {
 ; ATTRIBUTOR_GCN-NEXT:    ret void
@@ -14,14 +10,6 @@ define internal void @indirect() {
 }
 
 define amdgpu_kernel void @test_simple_indirect_call() #0 {
-; AKF_GCN-LABEL: define {{[^@]+}}@test_simple_indirect_call
-; AKF_GCN-SAME: () #[[ATTR0:[0-9]+]] {
-; AKF_GCN-NEXT:    [[FPTR:%.*]] = alloca ptr, align 8, addrspace(5)
-; AKF_GCN-NEXT:    store ptr @indirect, ptr addrspace(5) [[FPTR]], align 8
-; AKF_GCN-NEXT:    [[FP:%.*]] = load ptr, ptr addrspace(5) [[FPTR]], align 8
-; AKF_GCN-NEXT:    call void [[FP]]()
-; AKF_GCN-NEXT:    ret void
-;
 ; ATTRIBUTOR_GCN-LABEL: define {{[^@]+}}@test_simple_indirect_call
 ; ATTRIBUTOR_GCN-SAME: () #[[ATTR1:[0-9]+]] {
 ; ATTRIBUTOR_GCN-NEXT:    [[FPTR:%.*]] = alloca ptr, align 8, addrspace(5)
@@ -40,7 +28,6 @@ define amdgpu_kernel void @test_simple_indirect_call() #0 {
 attributes #0 = { "amdgpu-no-dispatch-id" }
 
 ;.
-; AKF_GCN: attributes #[[ATTR0]] = { "amdgpu-calls" "amdgpu-no-dispatch-id" "amdgpu-stack-objects" }
 ;.
 ; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/exec-mask-opt-cannot-create-empty-or-backward-segment.ll b/llvm/test/CodeGen/AMDGPU/exec-mask-opt-cannot-create-empty-or-backward-segment.ll
index 9104dc68eb9b4..72913d2596ebf 100644
--- a/llvm/test/CodeGen/AMDGPU/exec-mask-opt-cannot-create-empty-or-backward-segment.ll
+++ b/llvm/test/CodeGen/AMDGPU/exec-mask-opt-cannot-create-empty-or-backward-segment.ll
@@ -9,7 +9,7 @@ define amdgpu_kernel void @cannot_create_empty_or_backwards_segment(i1 %arg, i1
 ; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x0
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CHECK-NEXT:    s_load_dword s6, s[8:9], 0x4
-; CHECK-NEXT:    s_add_u32 s24, s24, s15
+; CHECK-NEXT:    s_add_u32 s24, s24, s17
 ; CHECK-NEXT:    s_addc_u32 s25, s25, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll b/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
index f3aec696abdee..e6f02295e67d5 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
+++ b/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
@@ -94,6 +94,9 @@ define amdgpu_kernel void @s_add_co_br_user(i32 %i) {
 ; GFX7-LABEL: s_add_co_br_user:
 ; GFX7:       ; %bb.0: ; %bb
 ; GFX7-NEXT:    s_load_dword s2, s[8:9], 0x0
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_i32 s0, s2, s2
 ; GFX7-NEXT:    s_cmp_lt_u32 s0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
index 0c25ca5076790..fac9f5bf826a6 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
@@ -5,6 +5,9 @@
 define amdgpu_kernel void @extract_vector_elt_v1i8(ptr addrspace(1) %out, <1 x i8> %foo) #0 {
 ; SI-LABEL: extract_vector_elt_v1i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -18,6 +21,9 @@ define amdgpu_kernel void @extract_vector_elt_v1i8(ptr addrspace(1) %out, <1 x i
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -32,6 +38,9 @@ define amdgpu_kernel void @extract_vector_elt_v1i8(ptr addrspace(1) %out, <1 x i
 define amdgpu_kernel void @extract_vector_elt_v2i8(ptr addrspace(1) %out, <2 x i8> %foo) #0 {
 ; SI-LABEL: extract_vector_elt_v2i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -54,6 +63,9 @@ define amdgpu_kernel void @extract_vector_elt_v2i8(ptr addrspace(1) %out, <2 x i
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -80,6 +92,9 @@ define amdgpu_kernel void @extract_vector_elt_v2i8(ptr addrspace(1) %out, <2 x i
 define amdgpu_kernel void @extract_vector_elt_v3i8(ptr addrspace(1) %out, <3 x i8> %foo) #0 {
 ; SI-LABEL: extract_vector_elt_v3i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -102,6 +117,9 @@ define amdgpu_kernel void @extract_vector_elt_v3i8(ptr addrspace(1) %out, <3 x i
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -128,6 +146,9 @@ define amdgpu_kernel void @extract_vector_elt_v3i8(ptr addrspace(1) %out, <3 x i
 define amdgpu_kernel void @extract_vector_elt_v4i8(ptr addrspace(1) %out, <4 x i8> %foo) #0 {
 ; SI-LABEL: extract_vector_elt_v4i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -150,6 +171,9 @@ define amdgpu_kernel void @extract_vector_elt_v4i8(ptr addrspace(1) %out, <4 x i
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -176,6 +200,9 @@ define amdgpu_kernel void @extract_vector_elt_v4i8(ptr addrspace(1) %out, <4 x i
 define amdgpu_kernel void @extract_vector_elt_v8i8(<8 x i8> %foo) #0 {
 ; SI-LABEL: extract_vector_elt_v8i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dword s0, s[8:9], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_lshr_b32 s1, s0, 16
@@ -192,10 +219,13 @@ define amdgpu_kernel void @extract_vector_elt_v8i8(<8 x i8> %foo) #0 {
 ; VI-LABEL: extract_vector_elt_v8i8:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s0, s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s1, s0, 16
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_byte v[0:1], v3
@@ -213,6 +243,9 @@ define amdgpu_kernel void @extract_vector_elt_v8i8(<8 x i8> %foo) #0 {
 define amdgpu_kernel void @extract_vector_elt_v16i8(ptr addrspace(1) %out, <16 x i8> %foo) #0 {
 ; SI-LABEL: extract_vector_elt_v16i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -235,6 +268,9 @@ define amdgpu_kernel void @extract_vector_elt_v16i8(ptr addrspace(1) %out, <16 x
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -261,6 +297,9 @@ define amdgpu_kernel void @extract_vector_elt_v16i8(ptr addrspace(1) %out, <16 x
 define amdgpu_kernel void @extract_vector_elt_v32i8(<32 x i8> %foo) #0 {
 ; SI-LABEL: extract_vector_elt_v32i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dword s0, s[8:9], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_lshr_b32 s1, s0, 16
@@ -277,10 +316,13 @@ define amdgpu_kernel void @extract_vector_elt_v32i8(<32 x i8> %foo) #0 {
 ; VI-LABEL: extract_vector_elt_v32i8:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s0, s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
-; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s1, s0, 16
+; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_byte v[0:1], v3
@@ -298,6 +340,9 @@ define amdgpu_kernel void @extract_vector_elt_v32i8(<32 x i8> %foo) #0 {
 define amdgpu_kernel void @extract_vector_elt_v64i8(ptr addrspace(1) %out, <64 x i8> %foo) #0 {
 ; SI-LABEL: extract_vector_elt_v64i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x10
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -320,6 +365,9 @@ define amdgpu_kernel void @extract_vector_elt_v64i8(ptr addrspace(1) %out, <64 x
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x40
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -351,6 +399,9 @@ define amdgpu_kernel void @extract_vector_elt_v64i8(ptr addrspace(1) %out, <64 x
 define amdgpu_kernel void @dynamic_extract_vector_elt_v2i8(ptr addrspace(1) %out, [8 x i32], <2 x i8> %foo, [8 x i32], i32 %idx) #0 {
 ; SI-LABEL: dynamic_extract_vector_elt_v2i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0xa
 ; SI-NEXT:    s_load_dword s3, s[8:9], 0x13
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -370,11 +421,14 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v2i8(ptr addrspace(1) %out
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x4c
 ; VI-NEXT:    s_load_dword s3, s[8:9], 0x28
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshl_b32 s2, s2, 3
 ; VI-NEXT:    s_and_b32 s3, s3, 0xffff
 ; VI-NEXT:    s_lshr_b32 s2, s3, s2
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    flat_store_byte v[0:1], v2
@@ -388,6 +442,9 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v2i8(ptr addrspace(1) %out
 define amdgpu_kernel void @dynamic_extract_vector_elt_v3i8(ptr addrspace(1) %out, [8 x i32], <3 x i8> %foo, [8 x i32], i32 %idx) #0 {
 ; SI-LABEL: dynamic_extract_vector_elt_v3i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x13
 ; SI-NEXT:    s_load_dword s3, s[8:9], 0xa
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -406,10 +463,13 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v3i8(ptr addrspace(1) %out
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x4c
 ; VI-NEXT:    s_load_dword s3, s[8:9], 0x28
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshl_b32 s2, s2, 3
 ; VI-NEXT:    s_lshr_b32 s2, s3, s2
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    flat_store_byte v[0:1], v2
@@ -424,6 +484,9 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v3i8(ptr addrspace(1) %out
 define amdgpu_kernel void @dynamic_extract_vector_elt_v4i8(ptr addrspace(1) %out, ptr addrspace(4) %vec.ptr, [8 x i32], i32 %idx) #0 {
 ; SI-LABEL: dynamic_extract_vector_elt_v4i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; SI-NEXT:    s_load_dword s4, s[8:9], 0xc
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -442,6 +505,9 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v4i8(ptr addrspace(1) %out
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -463,6 +529,9 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v4i8(ptr addrspace(1) %out
 define amdgpu_kernel void @dynamic_extract_vector_elt_v8i8(ptr addrspace(1) %out, ptr addrspace(4) %vec.ptr, i32 %idx) #0 {
 ; SI-LABEL: dynamic_extract_vector_elt_v8i8:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; SI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -481,6 +550,9 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v8i8(ptr addrspace(1) %out
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -502,6 +574,9 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v8i8(ptr addrspace(1) %out
 define amdgpu_kernel void @reduce_load_vector_v8i8_extract_0123() #0 {
 ; SI-LABEL: reduce_load_vector_v8i8_extract_0123:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_mov_b64 s[0:1], 0
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -526,6 +601,9 @@ define amdgpu_kernel void @reduce_load_vector_v8i8_extract_0123() #0 {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_mov_b64 s[0:1], 0
 ; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s1, s0, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -558,6 +636,9 @@ define amdgpu_kernel void @reduce_load_vector_v8i8_extract_0123() #0 {
 define amdgpu_kernel void @reduce_load_vector_v8i8_extract_0145() #0 {
 ; SI-LABEL: reduce_load_vector_v8i8_extract_0145:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_mov_b64 s[0:1], 0
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -581,6 +662,9 @@ define amdgpu_kernel void @reduce_load_vector_v8i8_extract_0145() #0 {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_mov_b64 s[0:1], 0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s2, s0, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -612,6 +696,9 @@ define amdgpu_kernel void @reduce_load_vector_v8i8_extract_0145() #0 {
 define amdgpu_kernel void @reduce_load_vector_v8i8_extract_45() #0 {
 ; SI-LABEL: reduce_load_vector_v8i8_extract_45:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_mov_b64 s[0:1], 4
 ; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -628,6 +715,9 @@ define amdgpu_kernel void @reduce_load_vector_v8i8_extract_45() #0 {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_mov_b64 s[0:1], 4
 ; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s1, s0, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -649,6 +739,9 @@ define amdgpu_kernel void @reduce_load_vector_v8i8_extract_45() #0 {
 define amdgpu_kernel void @reduce_load_vector_v16i8_extract_0145() #0 {
 ; SI-LABEL: reduce_load_vector_v16i8_extract_0145:
 ; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_mov_b64 s[0:1], 0
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -672,6 +765,9 @@ define amdgpu_kernel void @reduce_load_vector_v16i8_extract_0145() #0 {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_mov_b64 s[0:1], 0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s2, s0, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.f16.ll b/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
index 32f75f3835226..7b6a363c42708 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
@@ -14,6 +14,9 @@ define amdgpu_kernel void @s_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_and_b32 s2, s2, 0x7fff
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -26,6 +29,9 @@ define amdgpu_kernel void @s_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_and_b32 s2, s2, 0x7fff
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -80,6 +86,9 @@ define amdgpu_kernel void @s_fabs_f16(ptr addrspace(1) %out, half %in) {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_and_b32 s2, s2, 0x7fff
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -92,6 +101,9 @@ define amdgpu_kernel void @s_fabs_f16(ptr addrspace(1) %out, half %in) {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_and_b32 s2, s2, 0x7fff
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -145,6 +157,9 @@ define amdgpu_kernel void @s_fabs_v2f16(ptr addrspace(1) %out, <2 x half> %in) {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -157,6 +172,9 @@ define amdgpu_kernel void @s_fabs_v2f16(ptr addrspace(1) %out, <2 x half> %in) {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -196,6 +214,9 @@ define amdgpu_kernel void @s_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in) {
 ; CI-LABEL: s_fabs_v4f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_and_b32 s3, s3, 0x7fff7fff
 ; CI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
@@ -209,6 +230,9 @@ define amdgpu_kernel void @s_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in) {
 ; VI-LABEL: s_fabs_v4f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_and_b32 s3, s3, 0x7fff7fff
 ; VI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
@@ -251,6 +275,9 @@ define amdgpu_kernel void @fabs_fold_f16(ptr addrspace(1) %out, half %in0, half
 ; CI-LABEL: fabs_fold_f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s0, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_cvt_f32_f16_e64 v0, |s0|
 ; CI-NEXT:    s_lshr_b32 s0, s0, 16
@@ -268,6 +295,9 @@ define amdgpu_kernel void @fabs_fold_f16(ptr addrspace(1) %out, half %in0, half
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s3
@@ -325,6 +355,9 @@ define amdgpu_kernel void @v_fabs_v2f16(ptr addrspace(1) %out, ptr addrspace(1)
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
@@ -339,6 +372,9 @@ define amdgpu_kernel void @v_fabs_v2f16(ptr addrspace(1) %out, ptr addrspace(1)
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -386,6 +422,9 @@ define amdgpu_kernel void @fabs_free_v2f16(ptr addrspace(1) %out, i32 %in) #0 {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -398,6 +437,9 @@ define amdgpu_kernel void @fabs_free_v2f16(ptr addrspace(1) %out, i32 %in) #0 {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -441,6 +483,9 @@ define amdgpu_kernel void @v_fabs_fold_self_v2f16(ptr addrspace(1) %out, ptr add
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
@@ -467,6 +512,9 @@ define amdgpu_kernel void @v_fabs_fold_self_v2f16(ptr addrspace(1) %out, ptr add
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -523,9 +571,12 @@ define amdgpu_kernel void @v_fabs_fold_v2f16(ptr addrspace(1) %out, ptr addrspac
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dword v0, v[0:1]
 ; CI-NEXT:    s_lshr_b32 s2, s4, 16
@@ -551,9 +602,12 @@ define amdgpu_kernel void @v_fabs_fold_v2f16(ptr addrspace(1) %out, ptr addrspac
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v2, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -610,6 +664,9 @@ define amdgpu_kernel void @v_extract_fabs_fold_v2f16(ptr addrspace(1) %in) #0 {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
@@ -633,6 +690,9 @@ define amdgpu_kernel void @v_extract_fabs_fold_v2f16(ptr addrspace(1) %in) #0 {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -718,6 +778,9 @@ define amdgpu_kernel void @v_extract_fabs_no_fold_v2f16(ptr addrspace(1) %in) #0
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
@@ -736,6 +799,9 @@ define amdgpu_kernel void @v_extract_fabs_no_fold_v2f16(ptr addrspace(1) %in) #0
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
index 6496b70b4d697..60334e46a4454 100644
--- a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
+++ b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
@@ -74,6 +74,9 @@ define amdgpu_kernel void @global_store_2xi16_align2(ptr addrspace(1) %p, ptr ad
 ; GFX7-ALIGNED-LABEL: global_store_2xi16_align2:
 ; GFX7-ALIGNED:       ; %bb.0:
 ; GFX7-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; GFX7-ALIGNED-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-ALIGNED-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-ALIGNED-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX7-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
@@ -90,6 +93,9 @@ define amdgpu_kernel void @global_store_2xi16_align2(ptr addrspace(1) %p, ptr ad
 ; GFX7-UNALIGNED-LABEL: global_store_2xi16_align2:
 ; GFX7-UNALIGNED:       ; %bb.0:
 ; GFX7-UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; GFX7-UNALIGNED-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-UNALIGNED-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-UNALIGNED-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 0x20001
 ; GFX7-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
@@ -216,8 +222,10 @@ define amdgpu_kernel void @global_store_2xi16_align1(ptr addrspace(1) %p, ptr ad
 ; GFX7-ALIGNED-LABEL: global_store_2xi16_align1:
 ; GFX7-ALIGNED:       ; %bb.0:
 ; GFX7-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; GFX7-ALIGNED-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-ALIGNED-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-ALIGNED-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, 1
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX7-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-ALIGNED-NEXT:    s_add_u32 s2, s0, 2
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
@@ -227,6 +235,7 @@ define amdgpu_kernel void @global_store_2xi16_align1(ptr addrspace(1) %p, ptr ad
 ; GFX7-ALIGNED-NEXT:    flat_store_byte v[0:1], v2
 ; GFX7-ALIGNED-NEXT:    s_addc_u32 s5, s1, 0
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-ALIGNED-NEXT:    s_add_u32 s0, s0, 3
 ; GFX7-ALIGNED-NEXT:    flat_store_byte v[0:1], v3
@@ -243,6 +252,9 @@ define amdgpu_kernel void @global_store_2xi16_align1(ptr addrspace(1) %p, ptr ad
 ; GFX7-UNALIGNED-LABEL: global_store_2xi16_align1:
 ; GFX7-UNALIGNED:       ; %bb.0:
 ; GFX7-UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; GFX7-UNALIGNED-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-UNALIGNED-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-UNALIGNED-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 0x20001
 ; GFX7-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
@@ -351,6 +363,9 @@ define amdgpu_kernel void @global_store_2xi16_align4(ptr addrspace(1) %p, ptr ad
 ; GFX7-ALIGNED-LABEL: global_store_2xi16_align4:
 ; GFX7-ALIGNED:       ; %bb.0:
 ; GFX7-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; GFX7-ALIGNED-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-ALIGNED-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-ALIGNED-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, 0x20001
 ; GFX7-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
@@ -361,6 +376,9 @@ define amdgpu_kernel void @global_store_2xi16_align4(ptr addrspace(1) %p, ptr ad
 ; GFX7-UNALIGNED-LABEL: global_store_2xi16_align4:
 ; GFX7-UNALIGNED:       ; %bb.0:
 ; GFX7-UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; GFX7-UNALIGNED-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-UNALIGNED-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-UNALIGNED-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 0x20001
 ; GFX7-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index 4e12a30c6f6f4..9919497acea73 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -24,6 +24,9 @@ define amdgpu_kernel void @v_test_canonicalize_var_f32(ptr addrspace(1) %out) #1
 ; GFX678-LABEL: v_test_canonicalize_var_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
@@ -76,6 +79,9 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mul_f32_e64 v2, 1.0, s2
 ; GFX6-NEXT:    v_mov_b32_e32 v0, s0
@@ -87,6 +93,9 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mul_f32_e64 v2, 1.0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
@@ -132,6 +141,9 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f32(ptr addrspace(1) %ou
 ; GFX678-LABEL: v_test_canonicalize_fabs_var_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
@@ -184,6 +196,9 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f32(ptr addrspace(1
 ; GFX678-LABEL: v_test_canonicalize_fneg_fabs_var_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
@@ -237,6 +252,9 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f32(ptr addrspace(1) %ou
 ; GFX678-LABEL: v_test_canonicalize_fneg_var_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
@@ -289,6 +307,9 @@ define amdgpu_kernel void @test_fold_canonicalize_undef_f32(ptr addrspace(1) %ou
 ; GFX678-LABEL: test_fold_canonicalize_undef_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -328,6 +349,9 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f32(ptr addrspace(1) %out)
 ; GFX678-LABEL: test_fold_canonicalize_p0_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -367,6 +391,9 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f32(ptr addrspace(1) %out)
 ; GFX678-LABEL: test_fold_canonicalize_n0_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_bfrev_b32_e32 v2, 1
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -409,6 +436,9 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f32(ptr addrspace(1) %out)
 ; GFX678-LABEL: test_fold_canonicalize_p1_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -449,6 +479,9 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f32(ptr addrspace(1) %out)
 ; GFX678-LABEL: test_fold_canonicalize_n1_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, -1.0
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -489,6 +522,9 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f32(ptr addrspace(1) %
 ; GFX678-LABEL: test_fold_canonicalize_literal_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x41800000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -529,6 +565,9 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32(ptr
 ; GFX678-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -568,10 +607,13 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
 ; GFX678-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
 ; GFX678-NEXT:    s_mov_b32 s2, 0x7fffff
-; GFX678-NEXT:    v_mul_f32_e64 v2, 1.0, s2
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
+; GFX678-NEXT:    v_mul_f32_e64 v2, 1.0, s2
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX678-NEXT:    flat_store_dword v[0:1], v2
 ; GFX678-NEXT:    s_endpgm
@@ -612,10 +654,13 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
 ; GFX678-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_out:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
 ; GFX678-NEXT:    s_mov_b32 s2, 0x7fffff
-; GFX678-NEXT:    v_mul_f32_e64 v2, 1.0, s2
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
+; GFX678-NEXT:    v_mul_f32_e64 v2, 1.0, s2
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX678-NEXT:    flat_store_dword v[0:1], v2
 ; GFX678-NEXT:    s_endpgm
@@ -656,10 +701,13 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
 ; GFX678-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_in:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
 ; GFX678-NEXT:    s_mov_b32 s2, 0x7fffff
-; GFX678-NEXT:    v_mul_f32_e64 v2, 1.0, s2
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
+; GFX678-NEXT:    v_mul_f32_e64 v2, 1.0, s2
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX678-NEXT:    flat_store_dword v[0:1], v2
 ; GFX678-NEXT:    s_endpgm
@@ -700,6 +748,9 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f32(ptr ad
 ; GFX678-LABEL: test_denormals_fold_canonicalize_denormal0_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x7fffff
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -740,6 +791,9 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f32(ptr
 ; GFX678-LABEL: test_no_denormals_fold_canonicalize_denormal1_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_bfrev_b32_e32 v2, 1
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -782,6 +836,9 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f32(ptr ad
 ; GFX678-LABEL: test_denormals_fold_canonicalize_denormal1_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x807fffff
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -822,6 +879,9 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f32(ptr addrspace(1) %out
 ; GFX678-LABEL: test_fold_canonicalize_qnan_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -862,6 +922,9 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f32(ptr addrsp
 ; GFX678-LABEL: test_fold_canonicalize_qnan_value_neg1_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -902,6 +965,9 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f32(ptr addrsp
 ; GFX678-LABEL: test_fold_canonicalize_qnan_value_neg2_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -942,6 +1008,9 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f32(ptr addrspace(
 ; GFX678-LABEL: test_fold_canonicalize_snan0_value_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -982,6 +1051,9 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f32(ptr addrspace(
 ; GFX678-LABEL: test_fold_canonicalize_snan1_value_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -1022,6 +1094,9 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f32(ptr addrspace(
 ; GFX678-LABEL: test_fold_canonicalize_snan2_value_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -1062,6 +1137,9 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f32(ptr addrspace(
 ; GFX678-LABEL: test_fold_canonicalize_snan3_value_f32:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
@@ -1102,6 +1180,9 @@ define amdgpu_kernel void @v_test_canonicalize_var_f64(ptr addrspace(1) %out) #1
 ; GFX678-LABEL: v_test_canonicalize_var_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
@@ -1153,6 +1234,9 @@ define amdgpu_kernel void @s_test_canonicalize_var_f64(ptr addrspace(1) %out, do
 ; GFX6-LABEL: s_test_canonicalize_var_f64:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_max_f64 v[2:3], s[2:3], s[2:3]
 ; GFX6-NEXT:    v_mov_b32_e32 v0, s0
@@ -1163,6 +1247,9 @@ define amdgpu_kernel void @s_test_canonicalize_var_f64(ptr addrspace(1) %out, do
 ; GFX8-LABEL: s_test_canonicalize_var_f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_max_f64 v[0:1], s[2:3], s[2:3]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s0
@@ -1205,6 +1292,9 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f64(ptr addrspace(1) %ou
 ; GFX678-LABEL: v_test_canonicalize_fabs_var_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
@@ -1257,6 +1347,9 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(ptr addrspace(1
 ; GFX678-LABEL: v_test_canonicalize_fneg_fabs_var_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
@@ -1310,6 +1403,9 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f64(ptr addrspace(1) %ou
 ; GFX678-LABEL: v_test_canonicalize_fneg_var_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX678-NEXT:    v_mov_b32_e32 v1, s1
@@ -1362,10 +1458,13 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f64(ptr addrspace(1) %out)
 ; GFX678-LABEL: test_fold_canonicalize_p0_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, v0
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1407,10 +1506,13 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f64(ptr addrspace(1) %out)
 ; GFX678-LABEL: test_fold_canonicalize_n0_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_bfrev_b32_e32 v1, 1
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_bfrev_b32_e32 v1, 1
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1450,10 +1552,13 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f64(ptr addrspace(1) %out)
 ; GFX678-LABEL: test_fold_canonicalize_p1_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x3ff00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x3ff00000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1491,10 +1596,13 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f64(ptr addrspace(1) %out)
 ; GFX678-LABEL: test_fold_canonicalize_n1_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0xbff00000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0xbff00000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1532,10 +1640,13 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f64(ptr addrspace(1) %
 ; GFX678-LABEL: test_fold_canonicalize_literal_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x40300000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x40300000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1573,10 +1684,13 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f64(ptr
 ; GFX678-LABEL: test_no_denormals_fold_canonicalize_denormal0_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, v0
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1618,10 +1732,13 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f64(ptr ad
 ; GFX678-LABEL: test_denormals_fold_canonicalize_denormal0_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, -1
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0xfffff
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0xfffff
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1662,10 +1779,13 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f64(ptr
 ; GFX678-LABEL: test_no_denormals_fold_canonicalize_denormal1_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_bfrev_b32_e32 v1, 1
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_bfrev_b32_e32 v1, 1
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1705,10 +1825,13 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f64(ptr ad
 ; GFX678-LABEL: test_denormals_fold_canonicalize_denormal1_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, -1
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x800fffff
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x800fffff
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1749,10 +1872,13 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f64(ptr addrspace(1) %out
 ; GFX678-LABEL: test_fold_canonicalize_qnan_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1790,10 +1916,13 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f64(ptr addrsp
 ; GFX678-LABEL: test_fold_canonicalize_qnan_value_neg1_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1831,10 +1960,13 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f64(ptr addrsp
 ; GFX678-LABEL: test_fold_canonicalize_qnan_value_neg2_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1872,10 +2004,13 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f64(ptr addrspace(
 ; GFX678-LABEL: test_fold_canonicalize_snan0_value_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1913,10 +2048,13 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f64(ptr addrspace(
 ; GFX678-LABEL: test_fold_canonicalize_snan1_value_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1954,10 +2092,13 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f64(ptr addrspace(
 ; GFX678-LABEL: test_fold_canonicalize_snan2_value_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -1995,10 +2136,13 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f64(ptr addrspace(
 ; GFX678-LABEL: test_fold_canonicalize_snan3_value_f64:
 ; GFX678:       ; %bb.0:
 ; GFX678-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX678-NEXT:    s_add_i32 s12, s12, s17
+; GFX678-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX678-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX678-NEXT:    v_mov_b32_e32 v0, 0
-; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX678-NEXT:    v_mov_b32_e32 v3, s1
+; GFX678-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX678-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX678-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX678-NEXT:    s_endpgm
@@ -2037,6 +2181,9 @@ define amdgpu_kernel void @test_canonicalize_value_f64_flush(ptr addrspace(1) %a
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -2054,6 +2201,9 @@ define amdgpu_kernel void @test_canonicalize_value_f64_flush(ptr addrspace(1) %a
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -2117,6 +2267,9 @@ define amdgpu_kernel void @test_canonicalize_value_f32_flush(ptr addrspace(1) %a
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -2134,6 +2287,9 @@ define amdgpu_kernel void @test_canonicalize_value_f32_flush(ptr addrspace(1) %a
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -2197,6 +2353,9 @@ define amdgpu_kernel void @test_canonicalize_value_f16_flush(ptr addrspace(1) %a
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -2215,6 +2374,9 @@ define amdgpu_kernel void @test_canonicalize_value_f16_flush(ptr addrspace(1) %a
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -2279,6 +2441,9 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_flush(ptr addrspace(1)
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -2302,6 +2467,9 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_flush(ptr addrspace(1)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -2368,6 +2536,9 @@ define amdgpu_kernel void @test_canonicalize_value_f64_denorm(ptr addrspace(1) %
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -2385,6 +2556,9 @@ define amdgpu_kernel void @test_canonicalize_value_f64_denorm(ptr addrspace(1) %
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -2448,6 +2622,9 @@ define amdgpu_kernel void @test_canonicalize_value_f32_denorm(ptr addrspace(1) %
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -2465,6 +2642,9 @@ define amdgpu_kernel void @test_canonicalize_value_f32_denorm(ptr addrspace(1) %
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -2529,6 +2709,9 @@ define amdgpu_kernel void @test_canonicalize_value_f16_denorm(ptr addrspace(1) %
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -2547,6 +2730,9 @@ define amdgpu_kernel void @test_canonicalize_value_f16_denorm(ptr addrspace(1) %
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -2612,6 +2798,9 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_denorm(ptr addrspace(1)
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -2635,6 +2824,9 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_denorm(ptr addrspace(1)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -2700,6 +2892,9 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2f64(ptr addrspace(1) %out)
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GFX6-NEXT:    s_add_i32 s12, s12, s17
+; GFX6-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX6-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
@@ -2717,6 +2912,9 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2f64(ptr addrspace(1) %out)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll b/llvm/test/CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll
index fee6540f43c64..fc316b736d5f1 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll
@@ -22,12 +22,14 @@
 ; NOHSA-DEFAULT: buffer_store_dword
 ; NOHSA-NODEFAULT: flat_store_dword
 ; NOHSA-NOADDR64: flat_store_dword
-define amdgpu_kernel void @test(ptr addrspace(1) %out) {
+define amdgpu_kernel void @test(ptr addrspace(1) %out) #0 {
 entry:
   store i32 0, ptr addrspace(1) %out
   ret void
 }
 
+; ALL-LABEL: {{^}}test_addr64:
+
 ; HSA-DEFAULT: flat_store_dword
 ; HSA-NODEFAULT: buffer_store_dword
 ; HSA-NOADDR64: flat_store_dword
@@ -35,7 +37,7 @@ entry:
 ; NOHSA-DEFAULT: buffer_store_dword
 ; NOHSA-NODEFAULT: flat_store_dword
 ; NOHSA-NOADDR64: flat_store_dword
-define amdgpu_kernel void @test_addr64(ptr addrspace(1) %out) {
+define amdgpu_kernel void @test_addr64(ptr addrspace(1) %out) #0 {
 entry:
   %out.addr = alloca ptr addrspace(1), align 4, addrspace(5)
 
@@ -51,5 +53,7 @@ entry:
   ret void
 }
 
+attributes #0 = { "amdgpu-no-flat-scratch-init" }
+
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 400}
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
index 45223a24e021a..a59382ba20dc5 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
@@ -8,28 +8,34 @@
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=stoney -mattr=+xnack | FileCheck -check-prefix=VI-XNACK  -check-prefix=GCN %s
 
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | FileCheck -check-prefixes=GCN %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-xnack | FileCheck -check-prefixes=VI-NOXNACK,HSA-VI-NOXNACK,GCN %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=+xnack | FileCheck -check-prefixes=VI-XNACK,HSA-VI-XNACK,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-xnack | FileCheck -check-prefixes=HSA-VI-NOXNACK,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=+xnack | FileCheck -check-prefixes=HSA-VI-XNACK,GCN %s
 
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch | FileCheck -check-prefixes=GCN %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch,-xnack | FileCheck -check-prefixes=HSA-VI-NOXNACK,GFX9-ARCH-FLAT,GCN %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch,+xnack | FileCheck -check-prefixes=HSA-VI-XNACK,GFX9-ARCH-FLAT,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch,-xnack | FileCheck -check-prefixes=GFX9-ARCH-FLAT-NOXNACK,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch,+xnack | FileCheck -check-prefixes=GFX9-ARCH-FLAT-XNACK,GCN %s
 
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch | FileCheck -check-prefixes=GCN %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch,-xnack | FileCheck -check-prefixes=HSA-VI-NOXNACK,GFX10-ARCH-FLAT,GCN %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch,+xnack | FileCheck -check-prefixes=HSA-VI-XNACK,GFX10-ARCH-FLAT,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch,-xnack | FileCheck -check-prefixes=GFX10-ARCH-FLAT-NOXNACK,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch,+xnack | FileCheck -check-prefixes=GFX10-ARCH-FLAT-XNACK,GCN %s
 
 ; GCN-LABEL: {{^}}no_vcc_no_flat:
 
 ; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX9-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX10-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: ; TotalNumSgprs: 8
 ; VI-NOXNACK: ; TotalNumSgprs: 8
+; HSA-VI-NOXNACK: ; TotalNumSgprs: 8
 ; VI-XNACK: ; TotalNumSgprs: 12
-; GFX9-ARCH-FLAT: ; TotalNumSgprs: 14
-; GFX10-ARCH-FLAT: ; TotalNumSgprs: 8
+; HSA-VI-XNACK: ; TotalNumSgprs: 12
+; GFX9-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 14
+; GFX9-ARCH-FLAT-XNACK: ; TotalNumSgprs: 14
+; GFX10-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 8
+; GFX10-ARCH-FLAT-XNACK: ; TotalNumSgprs: 8
 define amdgpu_kernel void @no_vcc_no_flat() {
 entry:
   call void asm sideeffect "", "~{s7}"()
@@ -41,12 +47,18 @@ entry:
 ; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX9-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX10-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: ; TotalNumSgprs: 10
 ; VI-NOXNACK: ; TotalNumSgprs: 10
+; HSA-VI-NOXNACK: ; TotalNumSgprs: 10
 ; VI-XNACK: ; TotalNumSgprs: 12
-; GFX9-ARCH-FLAT: ; TotalNumSgprs: 14
-; GFX10-ARCH-FLAT: ; TotalNumSgprs: 10
+; HSA-VI-XNACK: ; TotalNumSgprs: 12
+; GFX9-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 14
+; GFX9-ARCH-FLAT-XNACK: ; TotalNumSgprs: 14
+; GFX10-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 10
+; GFX10-ARCH-FLAT-XNACK: ; TotalNumSgprs: 10
 define amdgpu_kernel void @vcc_no_flat() {
 entry:
   call void asm sideeffect "", "~{s7},~{vcc}"()
@@ -58,12 +70,18 @@ entry:
 ; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX9-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX10-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: ; TotalNumSgprs: 12
 ; VI-NOXNACK: ; TotalNumSgprs: 14
+; HSA-VI-NOXNACK: ; TotalNumSgprs: 24
 ; VI-XNACK: ; TotalNumSgprs: 14
-; GFX9-ARCH-FLAT: ; TotalNumSgprs: 14
-; GFX10-ARCH-FLAT: ; TotalNumSgprs: 8
+; HSA-VI-XNACK: ; TotalNumSgprs: 24
+; GFX9-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 14
+; GFX9-ARCH-FLAT-XNACK: ; TotalNumSgprs: 14
+; GFX10-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 8
+; GFX10-ARCH-FLAT-XNACK: ; TotalNumSgprs: 8
 define amdgpu_kernel void @no_vcc_flat() {
 entry:
   call void asm sideeffect "", "~{s7},~{flat_scratch}"()
@@ -75,12 +93,18 @@ entry:
 ; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX9-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX10-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: ; TotalNumSgprs: 12
 ; VI-NOXNACK: ; TotalNumSgprs: 14
+; HSA-VI-NOXNACK: ; TotalNumSgprs: 24
 ; VI-XNACK: ; TotalNumSgprs: 14
-; GFX9-ARCH-FLAT: ; TotalNumSgprs: 14
-; GFX10-ARCH-FLAT: ; TotalNumSgprs: 10
+; HSA-VI-XNACK: ; TotalNumSgprs: 24
+; GFX9-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 14
+; GFX9-ARCH-FLAT-XNACK: ; TotalNumSgprs: 14
+; GFX10-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 10
+; GFX10-ARCH-FLAT-XNACK: ; TotalNumSgprs: 10
 define amdgpu_kernel void @vcc_flat() {
 entry:
   call void asm sideeffect "", "~{s7},~{vcc},~{flat_scratch}"()
@@ -95,12 +119,18 @@ entry:
 ; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX9-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX10-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: NumSgprs: 4
 ; VI-NOXNACK: NumSgprs: 6
+; HSA-VI-NOXNACK: NumSgprs: 24
 ; VI-XNACK: NumSgprs: 6
-; GFX9-ARCH-FLAT: ; TotalNumSgprs: 6
-; GFX10-ARCH-FLAT: ; TotalNumSgprs: 0
+; HSA-VI-XNACK: NumSgprs: 24
+; GFX9-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 6
+; GFX9-ARCH-FLAT-XNACK: ; TotalNumSgprs: 6
+; GFX10-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 0
+; GFX10-ARCH-FLAT-XNACK: ; TotalNumSgprs: 0
 define amdgpu_kernel void @use_flat_scr() #0 {
 entry:
   call void asm sideeffect "; clobber ", "~{flat_scratch}"()
@@ -115,9 +145,13 @@ entry:
 
 ; CI: NumSgprs: 4
 ; VI-NOXNACK: NumSgprs: 6
+; HSA-VI-NOXNACK: NumSgprs: 24
 ; VI-XNACK: NumSgprs: 6
-; GFX9-ARCH-FLAT: ; TotalNumSgprs: 6
-; GFX10-ARCH-FLAT: ; TotalNumSgprs: 0
+; HSA-VI-XNACK: NumSgprs: 24
+; GFX9-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 6
+; GFX9-ARCH-FLAT-XNACK: ; TotalNumSgprs: 6
+; GFX10-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 0
+; GFX10-ARCH-FLAT-XNACK: ; TotalNumSgprs: 0
 define amdgpu_kernel void @use_flat_scr_lo() #0 {
 entry:
   call void asm sideeffect "; clobber ", "~{flat_scratch_lo}"()
@@ -129,12 +163,18 @@ entry:
 ; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
 ; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
 ; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX9-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
+; GFX10-ARCH-FLAT-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: NumSgprs: 4
 ; VI-NOXNACK: NumSgprs: 6
+; HSA-VI-NOXNACK: NumSgprs: 24
 ; VI-XNACK: NumSgprs: 6
-; GFX9-ARCH-FLAT: ; TotalNumSgprs: 6
-; GFX10-ARCH-FLAT: ; TotalNumSgprs: 0
+; HSA-VI-XNACK: NumSgprs: 24
+; GFX9-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 6
+; GFX9-ARCH-FLAT-XNACK: ; TotalNumSgprs: 6
+; GFX10-ARCH-FLAT-NOXNACK: ; TotalNumSgprs: 0
+; GFX10-ARCH-FLAT-XNACK: ; TotalNumSgprs: 0
 define amdgpu_kernel void @use_flat_scr_hi() #0 {
 entry:
   call void asm sideeffect "; clobber ", "~{flat_scratch_hi}"()
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index 64be9cb72a6ee..fb2448fb80744 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -16,6 +16,9 @@ define amdgpu_kernel void @multiple_fadd_use_test_f32(ptr addrspace(1) %out, flo
 ; VI-LABEL: multiple_fadd_use_test_f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_add_f32_e64 v0, s3, -1.0
 ; VI-NEXT:    v_add_f32_e64 v1, s2, -1.0
@@ -80,8 +83,11 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f32(ptr addrspace(1) %out, flo
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; VI-NEXT:    s_load_dword s3, s[8:9], 0x2c
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_add_u32 s2, s0, 4
 ; VI-NEXT:    v_add_f32_e64 v2, s4, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -139,6 +145,9 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f32(ptr addrspace(1) %out, flo
 ; VI-LABEL: multiple_use_fadd_fmad_f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    s_add_u32 s4, s0, 4
@@ -194,6 +203,9 @@ define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f32(ptr addrspace(1) %ou
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s6, s4, 4
 ; VI-NEXT:    v_mov_b32_e32 v0, s1
@@ -255,6 +267,9 @@ define amdgpu_kernel void @fmul_x2_xn2_f32(ptr addrspace(1) %out, float %x, floa
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mul_f32_e64 v0, s2, -4.0
 ; VI-NEXT:    v_mul_f32_e32 v2, s2, v0
@@ -303,10 +318,13 @@ define amdgpu_kernel void @fmul_x2_xn3_f32(ptr addrspace(1) %out, float %x, floa
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, 0xc0c00000
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mul_f32_e32 v0, s2, v0
 ; VI-NEXT:    v_mul_f32_e32 v2, s2, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -350,6 +368,9 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
 ; VI-DENORM:       ; %bb.0:
 ; VI-DENORM-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-DENORM-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-DENORM-NEXT:    s_add_i32 s12, s12, s17
+; VI-DENORM-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-DENORM-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-DENORM-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-DENORM-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-DENORM-NEXT:    v_add_f16_e64 v0, s2, -1.0
@@ -368,6 +389,9 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
 ; VI-FLUSH:       ; %bb.0:
 ; VI-FLUSH-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-FLUSH-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-FLUSH-NEXT:    s_add_i32 s12, s12, s17
+; VI-FLUSH-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-FLUSH-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-FLUSH-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-FLUSH-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-FLUSH-NEXT:    v_add_f16_e64 v0, s2, -1.0
@@ -482,6 +506,9 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
 ; VI-DENORM:       ; %bb.0:
 ; VI-DENORM-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; VI-DENORM-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-DENORM-NEXT:    s_add_i32 s12, s12, s17
+; VI-DENORM-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-DENORM-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-DENORM-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-DENORM-NEXT:    s_lshr_b32 s3, s4, 16
 ; VI-DENORM-NEXT:    v_mov_b32_e32 v0, s3
@@ -503,6 +530,9 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
 ; VI-FLUSH:       ; %bb.0:
 ; VI-FLUSH-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; VI-FLUSH-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-FLUSH-NEXT:    s_add_i32 s12, s12, s17
+; VI-FLUSH-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-FLUSH-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-FLUSH-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-FLUSH-NEXT:    s_lshr_b32 s3, s4, 16
 ; VI-FLUSH-NEXT:    v_mov_b32_e32 v0, s0
@@ -599,6 +629,9 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(ptr addrspace(1) %out, i16
 ; VI-DENORM:       ; %bb.0:
 ; VI-DENORM-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; VI-DENORM-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-DENORM-NEXT:    s_add_i32 s12, s12, s17
+; VI-DENORM-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-DENORM-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-DENORM-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-DENORM-NEXT:    s_lshr_b32 s3, s4, 16
 ; VI-DENORM-NEXT:    v_mov_b32_e32 v0, s3
@@ -620,6 +653,9 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(ptr addrspace(1) %out, i16
 ; VI-FLUSH:       ; %bb.0:
 ; VI-FLUSH-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; VI-FLUSH-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-FLUSH-NEXT:    s_add_i32 s12, s12, s17
+; VI-FLUSH-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-FLUSH-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-FLUSH-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-FLUSH-NEXT:    s_lshr_b32 s3, s4, 16
 ; VI-FLUSH-NEXT:    v_mov_b32_e32 v0, s3
@@ -718,6 +754,8 @@ define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f16(ptr addrspace(1) %ou
 ; VI-DENORM-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
 ; VI-DENORM-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
 ; VI-DENORM-NEXT:    s_load_dword s6, s[8:9], 0x8
+; VI-DENORM-NEXT:    s_add_i32 s12, s12, s17
+; VI-DENORM-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-DENORM-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-DENORM-NEXT:    s_lshr_b32 s0, s0, 16
 ; VI-DENORM-NEXT:    v_mov_b32_e32 v0, s0
@@ -725,6 +763,7 @@ define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f16(ptr addrspace(1) %ou
 ; VI-DENORM-NEXT:    v_mov_b32_e32 v0, s1
 ; VI-DENORM-NEXT:    v_fma_f16 v3, |s6|, 2.0, v0
 ; VI-DENORM-NEXT:    v_mov_b32_e32 v0, s2
+; VI-DENORM-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-DENORM-NEXT:    s_add_u32 s4, s2, 2
 ; VI-DENORM-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-DENORM-NEXT:    s_addc_u32 s5, s3, 0
@@ -741,6 +780,8 @@ define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f16(ptr addrspace(1) %ou
 ; VI-FLUSH-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
 ; VI-FLUSH-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
 ; VI-FLUSH-NEXT:    s_load_dword s6, s[8:9], 0x8
+; VI-FLUSH-NEXT:    s_add_i32 s12, s12, s17
+; VI-FLUSH-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-FLUSH-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-FLUSH-NEXT:    s_lshr_b32 s0, s0, 16
 ; VI-FLUSH-NEXT:    v_mov_b32_e32 v0, s0
@@ -748,6 +789,7 @@ define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f16(ptr addrspace(1) %ou
 ; VI-FLUSH-NEXT:    v_mov_b32_e32 v0, s1
 ; VI-FLUSH-NEXT:    v_mad_f16 v3, |s6|, 2.0, v0
 ; VI-FLUSH-NEXT:    v_mov_b32_e32 v0, s2
+; VI-FLUSH-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-FLUSH-NEXT:    s_add_u32 s4, s2, 2
 ; VI-FLUSH-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-FLUSH-NEXT:    s_addc_u32 s5, s3, 0
@@ -847,6 +889,9 @@ define amdgpu_kernel void @fmul_x2_xn2_f16(ptr addrspace(1) %out, i16 zeroext %x
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mul_f16_e64 v0, s2, -4.0
 ; VI-NEXT:    v_mul_f16_e32 v2, s2, v0
@@ -898,10 +943,13 @@ define amdgpu_kernel void @fmul_x2_xn3_f16(ptr addrspace(1) %out, i16 zeroext %x
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, 0xc600
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mul_f16_e32 v0, s2, v0
 ; VI-NEXT:    v_mul_f16_e32 v2, s2, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_short v[0:1], v2
 ; VI-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
index 67bec43078803..eca8c2837b0fc 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
@@ -9,6 +9,9 @@ define amdgpu_kernel void @fneg_fabs_fadd_f16(ptr addrspace(1) %out, half %x, ha
 ; CI-LABEL: fneg_fabs_fadd_f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s0, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_cvt_f32_f16_e64 v0, |s0|
 ; CI-NEXT:    s_lshr_b32 s0, s0, 16
@@ -26,6 +29,9 @@ define amdgpu_kernel void @fneg_fabs_fadd_f16(ptr addrspace(1) %out, half %x, ha
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -83,6 +89,9 @@ define amdgpu_kernel void @fneg_fabs_fmul_f16(ptr addrspace(1) %out, half %x, ha
 ; CI-LABEL: fneg_fabs_fmul_f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s0, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_and_b32 s1, s0, 0x7fff
 ; CI-NEXT:    s_lshr_b32 s0, s0, 16
@@ -101,6 +110,9 @@ define amdgpu_kernel void @fneg_fabs_fmul_f16(ptr addrspace(1) %out, half %x, ha
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -162,6 +174,9 @@ define amdgpu_kernel void @fneg_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_bitset1_b32 s2, 15
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -174,6 +189,9 @@ define amdgpu_kernel void @fneg_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_bitset1_b32 s2, 15
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -229,6 +247,9 @@ define amdgpu_kernel void @fneg_fabs_f16(ptr addrspace(1) %out, half %in) {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_bitset1_b32 s2, 15
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -241,6 +262,9 @@ define amdgpu_kernel void @fneg_fabs_f16(ptr addrspace(1) %out, half %in) {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_bitset1_b32 s2, 15
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -294,6 +318,9 @@ define amdgpu_kernel void @v_fneg_fabs_f16(ptr addrspace(1) %out, ptr addrspace(
 ; CIVI-LABEL: v_fneg_fabs_f16:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s2
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s3
@@ -348,6 +375,9 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16_non_bc_src(ptr addrspace(1) %out, <
 ; CI-LABEL: s_fneg_fabs_v2f16_non_bc_src:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s0, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s1, s0, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v1, s1
@@ -370,7 +400,9 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16_non_bc_src(ptr addrspace(1) %out, <
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 0x4000
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v2, s3
@@ -379,6 +411,7 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16_non_bc_src(ptr addrspace(1) %out, <
 ; VI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; VI-NEXT:    v_or_b32_e32 v2, 0x80008000, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -421,6 +454,9 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16_bc_src(ptr addrspace(1) %out, <2 x
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_or_b32 s2, s2, 0x80008000
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -433,6 +469,9 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16_bc_src(ptr addrspace(1) %out, <2 x
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_or_b32 s2, s2, 0x80008000
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -473,6 +512,9 @@ define amdgpu_kernel void @fneg_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in
 ; CIVI-LABEL: fneg_fabs_v4f16:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    s_or_b32 s3, s3, 0x80008000
 ; CIVI-NEXT:    s_or_b32 s2, s2, 0x80008000
@@ -516,6 +558,9 @@ define amdgpu_kernel void @fold_user_fneg_fabs_v2f16(ptr addrspace(1) %out, <2 x
 ; CI-LABEL: fold_user_fneg_fabs_v2f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s0, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s1, s0, 16
 ; CI-NEXT:    v_cvt_f32_f16_e64 v1, |s1|
@@ -537,7 +582,9 @@ define amdgpu_kernel void @fold_user_fneg_fabs_v2f16(ptr addrspace(1) %out, <2 x
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v0, 0xc400
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v2, s3
@@ -545,6 +592,7 @@ define amdgpu_kernel void @fold_user_fneg_fabs_v2f16(ptr addrspace(1) %out, <2 x
 ; VI-NEXT:    v_mul_f16_sdwa v0, |v2|, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v2, v1, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -584,6 +632,9 @@ define amdgpu_kernel void @s_fneg_multi_use_fabs_v2f16(ptr addrspace(1) %out0, p
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    s_and_b32 s0, s4, 0x7fff7fff
@@ -601,6 +652,9 @@ define amdgpu_kernel void @s_fneg_multi_use_fabs_v2f16(ptr addrspace(1) %out0, p
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    s_and_b32 s0, s4, 0x7fff7fff
@@ -655,6 +709,9 @@ define amdgpu_kernel void @s_fneg_multi_use_fabs_foldable_neg_v2f16(ptr addrspac
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    s_bfe_u32 s0, s4, 0xf0010
@@ -679,7 +736,9 @@ define amdgpu_kernel void @s_fneg_multi_use_fabs_foldable_neg_v2f16(ptr addrspac
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    v_mov_b32_e32 v5, 0xc400
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    s_lshr_b32 s1, s4, 16
@@ -688,6 +747,7 @@ define amdgpu_kernel void @s_fneg_multi_use_fabs_foldable_neg_v2f16(ptr addrspac
 ; VI-NEXT:    s_and_b32 s0, s4, 0x7fff7fff
 ; VI-NEXT:    v_mul_f16_sdwa v4, |v4|, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_mul_f16_e64 v5, |s4|, -4.0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; VI-NEXT:    v_mov_b32_e32 v5, s0
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index 781a2ca3146f5..058c273a65d99 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -1477,6 +1477,8 @@ define amdgpu_kernel void @multiple_uses_fneg_select_f64(double %x, double %y, i
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x4
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x6
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_bitcmp1_b32 s6, 0
 ; GFX7-NEXT:    s_cselect_b64 vcc, -1, 0
@@ -1488,6 +1490,7 @@ define amdgpu_kernel void @multiple_uses_fneg_select_f64(double %x, double %y, i
 ; GFX7-NEXT:    s_cselect_b32 s0, s0, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, -v0, vcc
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg.f16.ll
index 23e4ba9fd4ed7..98e0b27cd955d 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.f16.ll
@@ -11,6 +11,9 @@ define amdgpu_kernel void @s_fneg_f16(ptr addrspace(1) %out, half %in) #0 {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_xor_b32 s2, s2, 0x8000
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -23,6 +26,9 @@ define amdgpu_kernel void @s_fneg_f16(ptr addrspace(1) %out, half %in) #0 {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_xor_b32 s2, s2, 0x8000
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
@@ -78,6 +84,9 @@ define amdgpu_kernel void @v_fneg_f16(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
@@ -92,6 +101,9 @@ define amdgpu_kernel void @v_fneg_f16(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -152,6 +164,9 @@ define amdgpu_kernel void @s_fneg_free_f16(ptr addrspace(1) %out, i16 %in) #0 {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_xor_b32 s2, s2, 0x8000
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -164,6 +179,9 @@ define amdgpu_kernel void @s_fneg_free_f16(ptr addrspace(1) %out, i16 %in) #0 {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_xor_b32 s2, s2, 0x8000
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
@@ -217,6 +235,9 @@ define amdgpu_kernel void @v_fneg_fold_f16(ptr addrspace(1) %out, ptr addrspace(
 ; CI-LABEL: v_fneg_fold_f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -234,6 +255,9 @@ define amdgpu_kernel void @v_fneg_fold_f16(ptr addrspace(1) %out, ptr addrspace(
 ; GFX8-LABEL: v_fneg_fold_f16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
@@ -289,6 +313,9 @@ define amdgpu_kernel void @s_fneg_v2f16(ptr addrspace(1) %out, <2 x half> %in) #
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_xor_b32 s2, s2, 0x80008000
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -301,6 +328,9 @@ define amdgpu_kernel void @s_fneg_v2f16(ptr addrspace(1) %out, <2 x half> %in) #
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_xor_b32 s2, s2, 0x80008000
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
@@ -340,14 +370,17 @@ define amdgpu_kernel void @s_fneg_v2f16_nonload(ptr addrspace(1) %out) #0 {
 ; CIVI-LABEL: s_fneg_v2f16_nonload:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
 ; CIVI-NEXT:    ;;#ASMSTART
 ; CIVI-NEXT:    ; def s2
 ; CIVI-NEXT:    ;;#ASMEND
 ; CIVI-NEXT:    s_xor_b32 s2, s2, 0x80008000
-; CIVI-NEXT:    v_mov_b32_e32 v2, s2
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s1
+; CIVI-NEXT:    v_mov_b32_e32 v2, s2
 ; CIVI-NEXT:    flat_store_dword v[0:1], v2
 ; CIVI-NEXT:    s_endpgm
 ;
@@ -388,6 +421,9 @@ define amdgpu_kernel void @v_fneg_v2f16(ptr addrspace(1) %out, ptr addrspace(1)
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
@@ -402,6 +438,9 @@ define amdgpu_kernel void @v_fneg_v2f16(ptr addrspace(1) %out, ptr addrspace(1)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -449,6 +488,9 @@ define amdgpu_kernel void @fneg_free_v2f16(ptr addrspace(1) %out, i32 %in) #0 {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_xor_b32 s2, s2, 0x80008000
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -461,6 +503,9 @@ define amdgpu_kernel void @fneg_free_v2f16(ptr addrspace(1) %out, i32 %in) #0 {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_xor_b32 s2, s2, 0x80008000
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
@@ -501,6 +546,9 @@ define amdgpu_kernel void @v_fneg_fold_v2f16(ptr addrspace(1) %out, ptr addrspac
 ; CI-LABEL: v_fneg_fold_v2f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -527,6 +575,9 @@ define amdgpu_kernel void @v_fneg_fold_v2f16(ptr addrspace(1) %out, ptr addrspac
 ; GFX8-LABEL: v_fneg_fold_v2f16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
@@ -572,6 +623,9 @@ define amdgpu_kernel void @v_extract_fneg_fold_v2f16(ptr addrspace(1) %in) #0 {
 ; CI-LABEL: v_extract_fneg_fold_v2f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -593,6 +647,9 @@ define amdgpu_kernel void @v_extract_fneg_fold_v2f16(ptr addrspace(1) %in) #0 {
 ; GFX8-LABEL: v_extract_fneg_fold_v2f16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
@@ -672,6 +729,9 @@ define amdgpu_kernel void @v_extract_fneg_no_fold_v2f16(ptr addrspace(1) %in) #0
 ; CIVI-LABEL: v_extract_fneg_no_fold_v2f16:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s1
diff --git a/llvm/test/CodeGen/AMDGPU/half.ll b/llvm/test/CodeGen/AMDGPU/half.ll
index a2fca33af1046..10573aad38a51 100644
--- a/llvm/test/CodeGen/AMDGPU/half.ll
+++ b/llvm/test/CodeGen/AMDGPU/half.ll
@@ -10,6 +10,9 @@ define amdgpu_kernel void @load_f16_arg(ptr addrspace(1) %out, half %arg) #0 {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -21,6 +24,9 @@ define amdgpu_kernel void @load_f16_arg(ptr addrspace(1) %out, half %arg) #0 {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -46,6 +52,9 @@ define amdgpu_kernel void @load_v2f16_arg(ptr addrspace(1) %out, <2 x half> %arg
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -57,6 +66,9 @@ define amdgpu_kernel void @load_v2f16_arg(ptr addrspace(1) %out, <2 x half> %arg
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -81,6 +93,9 @@ define amdgpu_kernel void @load_v3f16_arg(ptr addrspace(1) %out, <3 x half> %arg
 ; CIVI-LABEL: load_v3f16_arg:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    s_add_u32 s4, s0, 4
 ; CIVI-NEXT:    s_addc_u32 s5, s1, 0
@@ -114,6 +129,9 @@ define amdgpu_kernel void @load_v4f16_arg(ptr addrspace(1) %out, <4 x half> %arg
 ; CIVI-LABEL: load_v4f16_arg:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
 ; CIVI-NEXT:    v_mov_b32_e32 v2, s2
@@ -139,6 +157,9 @@ define amdgpu_kernel void @load_v8f16_arg(ptr addrspace(1) %out, <8 x half> %arg
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v4, s4
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -153,6 +174,9 @@ define amdgpu_kernel void @load_v8f16_arg(ptr addrspace(1) %out, <8 x half> %arg
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, s4
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -183,6 +207,9 @@ define amdgpu_kernel void @extload_v2f16_arg(ptr addrspace(1) %out, <2 x half> %
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s3, s2, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v1, s3
@@ -196,6 +223,9 @@ define amdgpu_kernel void @extload_v2f16_arg(ptr addrspace(1) %out, <2 x half> %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_cvt_f32_f16_e32 v1, s3
@@ -227,6 +257,9 @@ define amdgpu_kernel void @extload_f16_to_f32_arg(ptr addrspace(1) %out, half %a
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_cvt_f32_f16_e32 v2, s2
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -238,6 +271,9 @@ define amdgpu_kernel void @extload_f16_to_f32_arg(ptr addrspace(1) %out, half %a
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f32_f16_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -265,6 +301,9 @@ define amdgpu_kernel void @extload_v2f16_to_v2f32_arg(ptr addrspace(1) %out, <2
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s3, s2, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v1, s3
@@ -278,6 +317,9 @@ define amdgpu_kernel void @extload_v2f16_to_v2f32_arg(ptr addrspace(1) %out, <2
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_cvt_f32_f16_e32 v1, s3
@@ -308,6 +350,9 @@ define amdgpu_kernel void @extload_v3f16_to_v3f32_arg(ptr addrspace(1) %out, <3
 ; CI-LABEL: extload_v3f16_to_v3f32_arg:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s4, s2, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v2, s3
@@ -321,6 +366,9 @@ define amdgpu_kernel void @extload_v3f16_to_v3f32_arg(ptr addrspace(1) %out, <3
 ; VI-LABEL: extload_v3f16_to_v3f32_arg:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s4, s2, 16
 ; VI-NEXT:    v_cvt_f32_f16_e32 v0, s2
@@ -351,6 +399,9 @@ define amdgpu_kernel void @extload_v4f16_to_v4f32_arg(ptr addrspace(1) %out, <4
 ; CI-LABEL: extload_v4f16_to_v4f32_arg:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s4, s3, 16
 ; CI-NEXT:    s_lshr_b32 s5, s2, 16
@@ -366,6 +417,9 @@ define amdgpu_kernel void @extload_v4f16_to_v4f32_arg(ptr addrspace(1) %out, <4
 ; VI-LABEL: extload_v4f16_to_v4f32_arg:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s4, s3, 16
 ; VI-NEXT:    s_lshr_b32 s5, s2, 16
@@ -401,6 +455,9 @@ define amdgpu_kernel void @extload_v8f16_to_v8f32_arg(ptr addrspace(1) %out, <8
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s6, s1, 16
 ; CI-NEXT:    s_lshr_b32 s7, s0, 16
@@ -429,6 +486,9 @@ define amdgpu_kernel void @extload_v8f16_to_v8f32_arg(ptr addrspace(1) %out, <8
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s6, s1, 16
 ; VI-NEXT:    s_lshr_b32 s7, s0, 16
@@ -485,6 +545,9 @@ define amdgpu_kernel void @extload_f16_to_f64_arg(ptr addrspace(1) %out, half %a
 ; CI-LABEL: extload_f16_to_f64_arg:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s0, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, s0
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -498,6 +561,9 @@ define amdgpu_kernel void @extload_f16_to_f64_arg(ptr addrspace(1) %out, half %a
 ; VI-LABEL: extload_f16_to_f64_arg:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s0, s[8:9], 0x8
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f32_f16_e32 v0, s0
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -529,6 +595,9 @@ define amdgpu_kernel void @extload_v2f16_to_v2f64_arg(ptr addrspace(1) %out, <2
 ; CI-LABEL: extload_v2f16_to_v2f64_arg:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s0, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s1, s0, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, s1
@@ -545,6 +614,9 @@ define amdgpu_kernel void @extload_v2f16_to_v2f64_arg(ptr addrspace(1) %out, <2
 ; VI-LABEL: extload_v2f16_to_v2f64_arg:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s0, s[8:9], 0x8
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s1, s0, 16
 ; VI-NEXT:    v_cvt_f32_f16_e32 v0, s1
@@ -582,6 +654,9 @@ define amdgpu_kernel void @extload_v3f16_to_v3f64_arg(ptr addrspace(1) %out, <3
 ; CI-LABEL: extload_v3f16_to_v3f64_arg:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, s3
 ; CI-NEXT:    s_lshr_b32 s4, s2, 16
@@ -603,6 +678,9 @@ define amdgpu_kernel void @extload_v3f16_to_v3f64_arg(ptr addrspace(1) %out, <3
 ; VI-LABEL: extload_v3f16_to_v3f64_arg:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f32_f16_e32 v1, s3
 ; VI-NEXT:    s_lshr_b32 s4, s2, 16
@@ -648,6 +726,9 @@ define amdgpu_kernel void @extload_v4f16_to_v4f64_arg(ptr addrspace(1) %out, <4
 ; CI-LABEL: extload_v4f16_to_v4f64_arg:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s4, s3, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, s3
@@ -673,6 +754,9 @@ define amdgpu_kernel void @extload_v4f16_to_v4f64_arg(ptr addrspace(1) %out, <4
 ; VI-LABEL: extload_v4f16_to_v4f64_arg:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s5, s3, 16
 ; VI-NEXT:    v_cvt_f32_f16_e32 v0, s3
@@ -726,6 +810,9 @@ define amdgpu_kernel void @extload_v8f16_to_v8f64_arg(ptr addrspace(1) %out, <8
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s6, s3, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, s6
@@ -773,6 +860,9 @@ define amdgpu_kernel void @extload_v8f16_to_v8f64_arg(ptr addrspace(1) %out, <8
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s6, s0, 16
 ; VI-NEXT:    s_lshr_b32 s8, s2, 16
@@ -858,6 +948,9 @@ define amdgpu_kernel void @global_load_store_f16(ptr addrspace(1) %out, ptr addr
 ; CIVI-LABEL: global_load_store_f16:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s2
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s3
@@ -886,6 +979,9 @@ define amdgpu_kernel void @global_load_store_v2f16(ptr addrspace(1) %out, ptr ad
 ; CIVI-LABEL: global_load_store_v2f16:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s2
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s3
@@ -914,6 +1010,9 @@ define amdgpu_kernel void @global_load_store_v4f16(ptr addrspace(1) %in, ptr add
 ; CIVI-LABEL: global_load_store_v4f16:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s1
@@ -942,6 +1041,9 @@ define amdgpu_kernel void @global_load_store_v8f16(ptr addrspace(1) %out, ptr ad
 ; CIVI-LABEL: global_load_store_v8f16:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s2
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s3
@@ -970,6 +1072,9 @@ define amdgpu_kernel void @global_extload_f16_to_f32(ptr addrspace(1) %out, ptr
 ; CIVI-LABEL: global_extload_f16_to_f32:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s2
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1001,6 +1106,9 @@ define amdgpu_kernel void @global_extload_v2f16_to_v2f32(ptr addrspace(1) %out,
 ; CI-LABEL: global_extload_v2f16_to_v2f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1017,6 +1125,9 @@ define amdgpu_kernel void @global_extload_v2f16_to_v2f32(ptr addrspace(1) %out,
 ; VI-LABEL: global_extload_v2f16_to_v2f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1052,6 +1163,9 @@ define amdgpu_kernel void @global_extload_v3f16_to_v3f32(ptr addrspace(1) %out,
 ; CI-LABEL: global_extload_v3f16_to_v3f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1069,6 +1183,9 @@ define amdgpu_kernel void @global_extload_v3f16_to_v3f32(ptr addrspace(1) %out,
 ; VI-LABEL: global_extload_v3f16_to_v3f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1106,6 +1223,9 @@ define amdgpu_kernel void @global_extload_v4f16_to_v4f32(ptr addrspace(1) %out,
 ; CI-LABEL: global_extload_v4f16_to_v4f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1125,6 +1245,9 @@ define amdgpu_kernel void @global_extload_v4f16_to_v4f32(ptr addrspace(1) %out,
 ; VI-LABEL: global_extload_v4f16_to_v4f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1165,6 +1288,9 @@ define amdgpu_kernel void @global_extload_v8f16_to_v8f32(ptr addrspace(1) %out,
 ; CI-LABEL: global_extload_v8f16_to_v8f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1195,6 +1321,9 @@ define amdgpu_kernel void @global_extload_v8f16_to_v8f32(ptr addrspace(1) %out,
 ; VI-LABEL: global_extload_v8f16_to_v8f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1251,6 +1380,9 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f32(ptr addrspace(1) %out
 ; CI-LABEL: global_extload_v16f16_to_v16f32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s4, s2, 16
 ; CI-NEXT:    v_mov_b32_e32 v5, s3
@@ -1309,6 +1441,9 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f32(ptr addrspace(1) %out
 ; VI-LABEL: global_extload_v16f16_to_v16f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1406,6 +1541,9 @@ define amdgpu_kernel void @global_extload_f16_to_f64(ptr addrspace(1) %out, ptr
 ; CIVI-LABEL: global_extload_f16_to_f64:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s2
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1440,6 +1578,9 @@ define amdgpu_kernel void @global_extload_v2f16_to_v2f64(ptr addrspace(1) %out,
 ; CI-LABEL: global_extload_v2f16_to_v2f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1458,6 +1599,9 @@ define amdgpu_kernel void @global_extload_v2f16_to_v2f64(ptr addrspace(1) %out,
 ; VI-LABEL: global_extload_v2f16_to_v2f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1498,6 +1642,9 @@ define amdgpu_kernel void @global_extload_v3f16_to_v3f64(ptr addrspace(1) %out,
 ; CI-LABEL: global_extload_v3f16_to_v3f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1523,6 +1670,9 @@ define amdgpu_kernel void @global_extload_v3f16_to_v3f64(ptr addrspace(1) %out,
 ; VI-LABEL: global_extload_v3f16_to_v3f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1574,6 +1724,9 @@ define amdgpu_kernel void @global_extload_v4f16_to_v4f64(ptr addrspace(1) %out,
 ; CI-LABEL: global_extload_v4f16_to_v4f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1602,6 +1755,9 @@ define amdgpu_kernel void @global_extload_v4f16_to_v4f64(ptr addrspace(1) %out,
 ; VI-LABEL: global_extload_v4f16_to_v4f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1659,6 +1815,9 @@ define amdgpu_kernel void @global_extload_v8f16_to_v8f64(ptr addrspace(1) %out,
 ; CI-LABEL: global_extload_v8f16_to_v8f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1707,6 +1866,9 @@ define amdgpu_kernel void @global_extload_v8f16_to_v8f64(ptr addrspace(1) %out,
 ; VI-LABEL: global_extload_v8f16_to_v8f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1791,6 +1953,9 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
 ; CI-LABEL: global_extload_v16f16_to_v16f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1885,6 +2050,9 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
 ; VI-LABEL: global_extload_v16f16_to_v16f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2039,6 +2207,9 @@ define amdgpu_kernel void @global_truncstore_f32_to_f16(ptr addrspace(1) %out, p
 ; CIVI-LABEL: global_truncstore_f32_to_f16:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s2
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2070,6 +2241,9 @@ define amdgpu_kernel void @global_truncstore_v2f32_to_v2f16(ptr addrspace(1) %ou
 ; CI-LABEL: global_truncstore_v2f32_to_v2f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2087,6 +2261,9 @@ define amdgpu_kernel void @global_truncstore_v2f32_to_v2f16(ptr addrspace(1) %ou
 ; VI-LABEL: global_truncstore_v2f32_to_v2f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2123,6 +2300,9 @@ define amdgpu_kernel void @global_truncstore_v3f32_to_v3f16(ptr addrspace(1) %ou
 ; CI-LABEL: global_truncstore_v3f32_to_v3f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2146,6 +2326,9 @@ define amdgpu_kernel void @global_truncstore_v3f32_to_v3f16(ptr addrspace(1) %ou
 ; VI-LABEL: global_truncstore_v3f32_to_v3f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2191,6 +2374,9 @@ define amdgpu_kernel void @global_truncstore_v4f32_to_v4f16(ptr addrspace(1) %ou
 ; CI-LABEL: global_truncstore_v4f32_to_v4f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2212,6 +2398,9 @@ define amdgpu_kernel void @global_truncstore_v4f32_to_v4f16(ptr addrspace(1) %ou
 ; VI-LABEL: global_truncstore_v4f32_to_v4f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2254,6 +2443,9 @@ define amdgpu_kernel void @global_truncstore_v8f32_to_v8f16(ptr addrspace(1) %ou
 ; CI-LABEL: global_truncstore_v8f32_to_v8f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2289,6 +2481,9 @@ define amdgpu_kernel void @global_truncstore_v8f32_to_v8f16(ptr addrspace(1) %ou
 ; VI-LABEL: global_truncstore_v8f32_to_v8f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2352,6 +2547,9 @@ define amdgpu_kernel void @global_truncstore_v16f32_to_v16f16(ptr addrspace(1) %
 ; CI-LABEL: global_truncstore_v16f32_to_v16f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_add_u32 s4, s2, 32
 ; CI-NEXT:    s_addc_u32 s5, s3, 0
@@ -2420,6 +2618,9 @@ define amdgpu_kernel void @global_truncstore_v16f32_to_v16f16(ptr addrspace(1) %
 ; VI-LABEL: global_truncstore_v16f32_to_v16f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s4, s2, 32
 ; VI-NEXT:    s_addc_u32 s5, s3, 0
@@ -2530,6 +2731,9 @@ define amdgpu_kernel void @fadd_f16(ptr addrspace(1) %out, half %a, half %b) #0
 ; CI-LABEL: fadd_f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s0, s[8:9], 0x2
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, s0
 ; CI-NEXT:    s_lshr_b32 s0, s0, 16
@@ -2547,6 +2751,9 @@ define amdgpu_kernel void @fadd_f16(ptr addrspace(1) %out, half %a, half %b) #0
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s3, s2, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s3
@@ -2577,6 +2784,9 @@ define amdgpu_kernel void @fadd_v2f16(ptr addrspace(1) %out, <2 x half> %a, <2 x
 ; CI-LABEL: fadd_v2f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s4, s2, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, s2
@@ -2598,6 +2808,9 @@ define amdgpu_kernel void @fadd_v2f16(ptr addrspace(1) %out, <2 x half> %a, <2 x
 ; VI-LABEL: fadd_v2f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s4, s3, 16
 ; VI-NEXT:    s_lshr_b32 s5, s2, 16
@@ -2629,6 +2842,9 @@ define amdgpu_kernel void @fadd_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; CI-LABEL: fadd_v4f16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2666,6 +2882,9 @@ define amdgpu_kernel void @fadd_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; VI-LABEL: fadd_v4f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -2706,6 +2925,9 @@ define amdgpu_kernel void @fadd_v8f16(ptr addrspace(1) %out, <8 x half> %a, <8 x
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x4
 ; CI-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s10, s0, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v4, s0
@@ -2764,6 +2986,9 @@ define amdgpu_kernel void @fadd_v8f16(ptr addrspace(1) %out, <8 x half> %a, <8 x
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s10, s7, 16
 ; VI-NEXT:    s_lshr_b32 s11, s3, 16
@@ -2824,6 +3049,9 @@ define amdgpu_kernel void @test_bitcast_from_half(ptr addrspace(1) %in, ptr addr
 ; CIVI-LABEL: test_bitcast_from_half:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s1
@@ -2853,6 +3081,9 @@ define amdgpu_kernel void @test_bitcast_to_half(ptr addrspace(1) %out, ptr addrs
 ; CIVI-LABEL: test_bitcast_to_half:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s2
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s3
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
index 8c017fa5ec263..cd89a36fe538b 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
@@ -23,7 +23,7 @@
 define amdgpu_kernel void @test(
     ptr addrspace(1) %r,
     ptr addrspace(1) %a,
-    ptr addrspace(1) %b) "amdgpu-no-implicitarg-ptr" {
+    ptr addrspace(1) %b) "amdgpu-no-implicitarg-ptr" "amdgpu-no-flat-scratch-init" {
 entry:
   %a.val = load half, ptr addrspace(1) %a
   %b.val = load half, ptr addrspace(1) %b
@@ -170,7 +170,7 @@ define amdgpu_kernel void @num_spilled_vgprs() #1 {
 ; CHECK-NEXT: - 1
 ; CHECK-NEXT: - 1
 
-attributes #0 = { "amdgpu-num-sgpr"="20" }
+attributes #0 = { "amdgpu-num-sgpr"="20" "amdgpu-no-flat-scratch-init" }
 attributes #1 = { "amdgpu-num-vgpr"="20" }
 attributes #2 = { "amdgpu-flat-work-group-size"="1,256" }
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa.ll b/llvm/test/CodeGen/AMDGPU/hsa.ll
index 5a2a976e23846..024593c49dba1 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa.ll
@@ -43,7 +43,7 @@
 ; ELF:   00E0: 6E616D65 A673696D 706C65BB 2E707269
 ; ELF:   00F0: 76617465 5F736567 6D656E74 5F666978
 ; ELF:   0100: 65645F73 697A6500 AB2E7367 70725F63
-; ELF:   0110: 6F756E74 06B12E73 6770725F 7370696C
+; ELF:   0110: 6F756E74 0EB12E73 6770725F 7370696C
 ; ELF:   0120: 6C5F636F 756E7400 A72E7379 6D626F6C
 ; ELF:   0130: A973696D 706C652E 6B64AB2E 76677072
 ; ELF:   0140: 5F636F75 6E7403B1 2E766770 725F7370
@@ -59,7 +59,7 @@
 ; ELF:   01E0: 73696D70 6C655F6E 6F5F6B65 726E6172
 ; ELF:   01F0: 6773BB2E 70726976 6174655F 7365676D
 ; ELF:   0200: 656E745F 66697865 645F7369 7A6500AB
-; ELF:   0210: 2E736770 725F636F 756E7400 B12E7367
+; ELF:   0210: 2E736770 725F636F 756E740C B12E7367
 ; ELF:   0220: 70725F73 70696C6C 5F636F75 6E7400A7
 ; ELF:   0230: 2E73796D 626F6CB5 73696D70 6C655F6E
 ; ELF:   0240: 6F5F6B65 726E6172 67732E6B 64AB2E76
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
index b4b6bef7a7ed3..ec80efc5f0362 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
@@ -7,7 +7,7 @@
 ; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 | FileCheck --check-prefixes=GFX9V5 %s
 ; RUN: sed 's/CODE_OBJECT_VERSION/600/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 | FileCheck --check-prefixes=GFX9V5 %s
 
-define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addrspace(3) %ptr.local) {
+define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addrspace(3) %ptr.local) #0 {
 ; GFX8V4-LABEL: addrspacecast:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -109,7 +109,7 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
   ret void
 }
 
-define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) {
+define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) #0 {
 ; GFX8V4-LABEL: llvm_amdgcn_is_shared:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dword s0, s[6:7], 0x40
@@ -163,7 +163,7 @@ define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) {
   ret void
 }
 
-define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) {
+define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) #0 {
 ; GFX8V4-LABEL: llvm_amdgcn_is_private:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dword s0, s[6:7], 0x44
@@ -283,7 +283,7 @@ define amdgpu_kernel void @llvm_ubsantrap() {
   unreachable
 }
 
-define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr)  {
+define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr)  #0 {
 ; GFX8V4-LABEL: llvm_amdgcn_queue_ptr:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    v_mov_b32_e32 v0, s6
@@ -389,3 +389,5 @@ declare void @llvm.debugtrap()
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION}
+
+attributes #0 = { "amdgpu-no-flat-scratch-init" }
diff --git a/llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll b/llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll
index 9eb966b4e2a94..0ca180ed6e105 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll
@@ -8,16 +8,16 @@
 define amdgpu_kernel void @s_input_output_i128() {
   ; GFX908-LABEL: name: s_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 7208970 /* regdef:SGPR_128 */, def %12
-  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_128 = COPY %12
-  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 7208969 /* reguse:SGPR_128 */, [[COPY]]
+  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 7208970 /* regdef:SGPR_128 */, def %13
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_128 = COPY %13
+  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 7208969 /* reguse:SGPR_128 */, %14
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: s_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 7208970 /* regdef:SGPR_128 */, def %10
-  ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:sgpr_128 = COPY %10
-  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 7208969 /* reguse:SGPR_128 */, [[COPY]]
+  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 7208970 /* regdef:SGPR_128 */, def %11
+  ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:sgpr_128 = COPY %11
+  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 7208969 /* reguse:SGPR_128 */, %12
   ; GFX90A-NEXT:   S_ENDPGM 0
   %val = tail call i128 asm sideeffect "; def $0", "=s"()
   call void asm sideeffect "; use $0", "s"(i128 %val)
@@ -27,16 +27,16 @@ define amdgpu_kernel void @s_input_output_i128() {
 define amdgpu_kernel void @v_input_output_i128() {
   ; GFX908-LABEL: name: v_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %12
-  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vreg_128 = COPY %12
-  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6094857 /* reguse:VReg_128 */, [[COPY]]
+  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %13
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vreg_128 = COPY %13
+  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6094857 /* reguse:VReg_128 */, %14
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: v_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def %10
-  ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:vreg_128_align2 = COPY %10
-  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6422537 /* reguse:VReg_128_Align2 */, [[COPY]]
+  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def %11
+  ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:vreg_128_align2 = COPY %11
+  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6422537 /* reguse:VReg_128_Align2 */, %12
   ; GFX90A-NEXT:   S_ENDPGM 0
   %val = tail call i128 asm sideeffect "; def $0", "=v"()
   call void asm sideeffect "; use $0", "v"(i128 %val)
@@ -46,16 +46,17 @@ define amdgpu_kernel void @v_input_output_i128() {
 define amdgpu_kernel void @a_input_output_i128() {
   ; GFX908-LABEL: name: a_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6029322 /* regdef:AReg_128 */, def %12
-  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY %12
-  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6029321 /* reguse:AReg_128 */, [[COPY]]
+  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6029322 /* regdef:AReg_128 */, def %13
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY %13
+  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6029321 /* reguse:AReg_128 */, %14
+
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: a_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6291466 /* regdef:AReg_128_Align2 */, def %10
-  ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY %10
-  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY]]
+  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6291466 /* regdef:AReg_128_Align2 */, def %11
+  ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY %11
+  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, %12
   ; GFX90A-NEXT:   S_ENDPGM 0
   %val = call i128 asm sideeffect "; def $0", "=a"()
   call void asm sideeffect "; use $0", "a"(i128 %val)
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
index 75db7571444bc..b51cb9df8d784 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
@@ -22,6 +22,9 @@ define amdgpu_kernel void @s_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr a
 ; VI-LABEL: s_insertelement_v2bf16_0:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -82,6 +85,9 @@ define amdgpu_kernel void @s_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr a
 ; VI-LABEL: s_insertelement_v2bf16_1:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -144,6 +150,9 @@ define amdgpu_kernel void @v_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr a
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -216,6 +225,9 @@ define amdgpu_kernel void @v_insertelement_v2bf16_0_inlineimm(ptr addrspace(1) %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -286,6 +298,9 @@ define amdgpu_kernel void @v_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr a
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -358,6 +373,9 @@ define amdgpu_kernel void @v_insertelement_v2bf16_1_inlineimm(ptr addrspace(1) %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -435,11 +453,14 @@ define amdgpu_kernel void @v_insertelement_v2bf16_dynamic_vgpr(ptr addrspace(1)
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
@@ -531,14 +552,17 @@ define amdgpu_kernel void @v_insertelement_v4bf16_0(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v0, s4, v0, v4
@@ -611,14 +635,17 @@ define amdgpu_kernel void @v_insertelement_v4bf16_1(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v0, v0, s4, v4
@@ -689,14 +716,17 @@ define amdgpu_kernel void @v_insertelement_v4bf16_2(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v1, s4, v1, v4
@@ -769,14 +799,17 @@ define amdgpu_kernel void @v_insertelement_v4bf16_3(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v1, v1, s4, v4
@@ -853,9 +886,12 @@ define amdgpu_kernel void @v_insertelement_v4bf16_dynamic_sgpr(ptr addrspace(1)
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
@@ -948,9 +984,12 @@ define amdgpu_kernel void @v_insertelement_v8bf16_3(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
@@ -1065,9 +1104,12 @@ define amdgpu_kernel void @v_insertelement_v8bf16_dynamic(ptr addrspace(1) %out,
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
@@ -1245,11 +1287,14 @@ define amdgpu_kernel void @v_insertelement_v16bf16_3(ptr addrspace(1) %out, ptr
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 16, v0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
@@ -1417,11 +1462,14 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s3
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, s2, v8
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v0, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v4
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
index 97c97ac8a7ad3..e11900ac0ca68 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
@@ -21,6 +21,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0(ptr addrspace(1) %out, ptr ad
 ; CIVI-LABEL: s_insertelement_v2i16_0:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
@@ -68,6 +71,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reg(ptr addrspace(1) %out, pt
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -84,6 +90,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reg(ptr addrspace(1) %out, pt
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0xc
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -152,6 +161,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_multi_use_hi_reg(ptr addrspac
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -172,6 +184,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_multi_use_hi_reg(ptr addrspac
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0xc
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -251,6 +266,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi(ptr addrspace(1) %out,
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -266,6 +284,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi(ptr addrspace(1) %out,
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0xc
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -320,6 +341,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi_multi_use_1(ptr addrspa
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -339,6 +363,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi_multi_use_1(ptr addrspa
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -407,6 +434,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi_both_multi_use_1(ptr ad
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -429,6 +459,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi_both_multi_use_1(ptr ad
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -498,6 +531,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_1(ptr addrspace(1) %out, ptr ad
 ; CIVI-LABEL: s_insertelement_v2i16_1:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
@@ -544,6 +580,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_1_reg(ptr addrspace(1) %out, pt
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -560,6 +599,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_1_reg(ptr addrspace(1) %out, pt
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0xc
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -623,6 +665,9 @@ define amdgpu_kernel void @s_insertelement_v2f16_0(ptr addrspace(1) %out, ptr ad
 ; CIVI-LABEL: s_insertelement_v2f16_0:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
@@ -668,6 +713,9 @@ define amdgpu_kernel void @s_insertelement_v2f16_1(ptr addrspace(1) %out, ptr ad
 ; CIVI-LABEL: s_insertelement_v2f16_1:
 ; CIVI:       ; %bb.0:
 ; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT:    s_add_i32 s12, s12, s17
+; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
@@ -714,6 +762,9 @@ define amdgpu_kernel void @v_insertelement_v2i16_0(ptr addrspace(1) %out, ptr ad
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -732,6 +783,9 @@ define amdgpu_kernel void @v_insertelement_v2i16_0(ptr addrspace(1) %out, ptr ad
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
@@ -788,9 +842,12 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_reghi(ptr addrspace(1) %out,
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
@@ -807,9 +864,12 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_reghi(ptr addrspace(1) %out,
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dword v3, v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
@@ -880,6 +940,9 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_inlineimm(ptr addrspace(1) %o
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -898,6 +961,9 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_inlineimm(ptr addrspace(1) %o
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
@@ -953,6 +1019,9 @@ define amdgpu_kernel void @v_insertelement_v2i16_1(ptr addrspace(1) %out, ptr ad
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -971,6 +1040,9 @@ define amdgpu_kernel void @v_insertelement_v2i16_1(ptr addrspace(1) %out, ptr ad
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
@@ -1038,6 +1110,9 @@ define amdgpu_kernel void @v_insertelement_v2i16_1_inlineimm(ptr addrspace(1) %o
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -1056,6 +1131,9 @@ define amdgpu_kernel void @v_insertelement_v2i16_1_inlineimm(ptr addrspace(1) %o
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
@@ -1123,6 +1201,9 @@ define amdgpu_kernel void @v_insertelement_v2f16_0(ptr addrspace(1) %out, ptr ad
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -1141,6 +1222,9 @@ define amdgpu_kernel void @v_insertelement_v2f16_0(ptr addrspace(1) %out, ptr ad
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
@@ -1195,6 +1279,9 @@ define amdgpu_kernel void @v_insertelement_v2f16_0_inlineimm(ptr addrspace(1) %o
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -1213,6 +1300,9 @@ define amdgpu_kernel void @v_insertelement_v2f16_0_inlineimm(ptr addrspace(1) %o
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
@@ -1267,6 +1357,9 @@ define amdgpu_kernel void @v_insertelement_v2f16_1(ptr addrspace(1) %out, ptr ad
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -1285,6 +1378,9 @@ define amdgpu_kernel void @v_insertelement_v2f16_1(ptr addrspace(1) %out, ptr ad
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
@@ -1353,6 +1449,9 @@ define amdgpu_kernel void @v_insertelement_v2f16_1_inlineimm(ptr addrspace(1) %o
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -1371,6 +1470,9 @@ define amdgpu_kernel void @v_insertelement_v2f16_1_inlineimm(ptr addrspace(1) %o
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
@@ -1445,6 +1547,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_dynamic(ptr addrspace(1) %out,
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
@@ -1464,6 +1569,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_dynamic(ptr addrspace(1) %out,
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
@@ -1526,9 +1634,12 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(ptr addrspace(1) %
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -1547,9 +1658,12 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(ptr addrspace(1) %
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dword v3, v[0:1]
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
@@ -1612,11 +1726,14 @@ define amdgpu_kernel void @v_insertelement_v2f16_dynamic_vgpr(ptr addrspace(1) %
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
@@ -1639,11 +1756,14 @@ define amdgpu_kernel void @v_insertelement_v2f16_dynamic_vgpr(ptr addrspace(1) %
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    v_mov_b32_e32 v1, s5
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    v_mov_b32_e32 v3, s3
 ; CI-NEXT:    flat_load_dword v4, v[0:1]
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
@@ -1712,14 +1832,17 @@ define amdgpu_kernel void @v_insertelement_v4f16_0(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v0, s4, v0, v4
@@ -1731,9 +1854,12 @@ define amdgpu_kernel void @v_insertelement_v4f16_0(ptr addrspace(1) %out, ptr ad
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0xc
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
@@ -1790,14 +1916,17 @@ define amdgpu_kernel void @v_insertelement_v4f16_1(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v0, v0, s4, v4
@@ -1809,9 +1938,12 @@ define amdgpu_kernel void @v_insertelement_v4f16_1(ptr addrspace(1) %out, ptr ad
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
@@ -1883,14 +2015,17 @@ define amdgpu_kernel void @v_insertelement_v4f16_2(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x30
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v1, s4, v1, v4
@@ -1902,9 +2037,12 @@ define amdgpu_kernel void @v_insertelement_v4f16_2(ptr addrspace(1) %out, ptr ad
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0xc
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
@@ -1961,14 +2099,17 @@ define amdgpu_kernel void @v_insertelement_v4f16_3(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v1, v1, s4, v4
@@ -1980,9 +2121,12 @@ define amdgpu_kernel void @v_insertelement_v4f16_3(ptr addrspace(1) %out, ptr ad
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
@@ -2054,14 +2198,17 @@ define amdgpu_kernel void @v_insertelement_v4i16_2(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
-; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_perm_b32 v1, s4, v1, v4
@@ -2073,9 +2220,12 @@ define amdgpu_kernel void @v_insertelement_v4i16_2(ptr addrspace(1) %out, ptr ad
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
@@ -2138,6 +2288,9 @@ define amdgpu_kernel void @v_insertelement_v4i16_dynamic_vgpr(ptr addrspace(1) %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    flat_load_dword v4, v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
@@ -2165,6 +2318,9 @@ define amdgpu_kernel void @v_insertelement_v4i16_dynamic_vgpr(ptr addrspace(1) %
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    flat_load_dword v4, v[0:1] glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
@@ -2268,9 +2424,12 @@ define amdgpu_kernel void @v_insertelement_v4f16_dynamic_sgpr(ptr addrspace(1) %
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
@@ -2294,9 +2453,12 @@ define amdgpu_kernel void @v_insertelement_v4f16_dynamic_sgpr(ptr addrspace(1) %
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
@@ -2363,9 +2525,12 @@ define amdgpu_kernel void @v_insertelement_v8f16_3(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
@@ -2383,9 +2548,12 @@ define amdgpu_kernel void @v_insertelement_v8f16_3(ptr addrspace(1) %out, ptr ad
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v5, s1
@@ -2457,9 +2625,12 @@ define amdgpu_kernel void @v_insertelement_v8i16_6(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
@@ -2477,9 +2648,12 @@ define amdgpu_kernel void @v_insertelement_v8i16_6(ptr addrspace(1) %out, ptr ad
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v5, s1
@@ -2568,9 +2742,12 @@ define amdgpu_kernel void @v_insertelement_v8f16_dynamic(ptr addrspace(1) %out,
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
@@ -2622,9 +2799,12 @@ define amdgpu_kernel void @v_insertelement_v8f16_dynamic(ptr addrspace(1) %out,
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; CI-NEXT:    v_mov_b32_e32 v5, s1
@@ -2803,11 +2983,14 @@ define amdgpu_kernel void @v_insertelement_v16f16_3(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 16, v0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
@@ -2830,9 +3013,12 @@ define amdgpu_kernel void @v_insertelement_v16f16_3(ptr addrspace(1) %out, ptr a
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s3
 ; CI-NEXT:    v_add_i32_e32 v4, vcc, s2, v8
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v0, vcc
 ; CI-NEXT:    flat_load_dwordx4 v[0:3], v[4:5]
 ; CI-NEXT:    v_add_i32_e32 v4, vcc, 16, v4
@@ -2923,12 +3109,14 @@ define amdgpu_kernel void @v_insertelement_v16i16_6(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
-; VI-NEXT:    v_mov_b32_e32 v12, 0x3020504
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 16, v0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
@@ -2936,6 +3124,7 @@ define amdgpu_kernel void @v_insertelement_v16i16_6(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, s0, v8
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 16, v8
+; VI-NEXT:    v_mov_b32_e32 v12, 0x3020504
 ; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v9, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_perm_b32 v3, s4, v3, v12
@@ -2949,11 +3138,14 @@ define amdgpu_kernel void @v_insertelement_v16i16_6(ptr addrspace(1) %out, ptr a
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; CI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
@@ -3087,11 +3279,14 @@ define amdgpu_kernel void @v_insertelement_v16f16_dynamic(ptr addrspace(1) %out,
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s3
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, s2, v8
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v0, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v4
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
 ; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
@@ -3184,11 +3379,14 @@ define amdgpu_kernel void @v_insertelement_v16f16_dynamic(ptr addrspace(1) %out,
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 5, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, 16, v0
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dwordx4 v[7:10], v[2:3]
 ; CI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/invalid-addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/invalid-addrspacecast.ll
index f0609f62a9024..5dff7372ab561 100644
--- a/llvm/test/CodeGen/AMDGPU/invalid-addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/invalid-addrspacecast.ll
@@ -7,6 +7,9 @@
 define amdgpu_kernel void @use_group_to_global_addrspacecast(ptr addrspace(3) %ptr) {
 ; CHECK-LABEL: use_group_to_global_addrspacecast:
 ; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    flat_store_dword v[0:1], v0
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll b/llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll
index 621187100f323..55a5d50f06bbd 100644
--- a/llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll
@@ -6,6 +6,8 @@ define amdgpu_kernel void @load_idx_idy(ptr addrspace(4) %disp, ptr %g) {
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_load_dword s6, s[4:5], 0x4
 ; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_lshr_b32 s4, s6, 16
diff --git a/llvm/test/CodeGen/AMDGPU/kernarg-size.ll b/llvm/test/CodeGen/AMDGPU/kernarg-size.ll
index 496a1c652da25..1a32953305bbc 100644
--- a/llvm/test/CodeGen/AMDGPU/kernarg-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernarg-size.ll
@@ -7,7 +7,7 @@ declare void @llvm.trap() #0
 ; DOORBELL-NEXT:     .amdhsa_group_segment_fixed_size 0
 ; DOORBELL-NEXT:     .amdhsa_private_segment_fixed_size 0
 ; DOORBELL-NEXT:     .amdhsa_kernarg_size 8
-; DOORBELL-NEXT:     .amdhsa_user_sgpr_count 12
+; DOORBELL-NEXT:     .amdhsa_user_sgpr_count 14
 ; DOORBELL-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; DOORBELL:      .end_amdhsa_kernel
 
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 4b6cc32522f5b..7179f687c70f2 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -47,11 +47,7 @@
 ; GCN-O0-NEXT:      Instrument function entry/exit with calls to e.g. mcount() (post inlining)
 ; GCN-O0-NEXT:      Scalarize Masked Memory Intrinsics
 ; GCN-O0-NEXT:      Expand reduction intrinsics
-; GCN-O0-NEXT:    CallGraph Construction
-; GCN-O0-NEXT:    Call Graph SCC Pass Manager
-; GCN-O0-NEXT:      AMDGPU Annotate Kernel Features
-; GCN-O0-NEXT:      FunctionPass Manager
-; GCN-O0-NEXT:        AMDGPU Lower Kernel Arguments
+; GCN-O0-NEXT:      AMDGPU Lower Kernel Arguments
 ; GCN-O0-NEXT:    Lower buffer fat pointer operations to buffer resources
 ; GCN-O0-NEXT:    CallGraph Construction
 ; GCN-O0-NEXT:    Call Graph SCC Pass Manager
@@ -232,11 +228,7 @@
 ; GCN-O1-NEXT:      Instrument function entry/exit with calls to e.g. mcount() (post inlining)
 ; GCN-O1-NEXT:      Scalarize Masked Memory Intrinsics
 ; GCN-O1-NEXT:      Expand reduction intrinsics
-; GCN-O1-NEXT:    CallGraph Construction
-; GCN-O1-NEXT:    Call Graph SCC Pass Manager
-; GCN-O1-NEXT:      AMDGPU Annotate Kernel Features
-; GCN-O1-NEXT:      FunctionPass Manager
-; GCN-O1-NEXT:        AMDGPU Lower Kernel Arguments
+; GCN-O1-NEXT:      AMDGPU Lower Kernel Arguments
 ; GCN-O1-NEXT:    Lower buffer fat pointer operations to buffer resources
 ; GCN-O1-NEXT:    CallGraph Construction
 ; GCN-O1-NEXT:    Call Graph SCC Pass Manager
@@ -531,11 +523,7 @@
 ; GCN-O1-OPTS-NEXT:      Scalarize Masked Memory Intrinsics
 ; GCN-O1-OPTS-NEXT:      Expand reduction intrinsics
 ; GCN-O1-OPTS-NEXT:      Early CSE
-; GCN-O1-OPTS-NEXT:    CallGraph Construction
-; GCN-O1-OPTS-NEXT:    Call Graph SCC Pass Manager
-; GCN-O1-OPTS-NEXT:      AMDGPU Annotate Kernel Features
-; GCN-O1-OPTS-NEXT:      FunctionPass Manager
-; GCN-O1-OPTS-NEXT:        AMDGPU Lower Kernel Arguments
+; GCN-O1-OPTS-NEXT:      AMDGPU Lower Kernel Arguments
 ; GCN-O1-OPTS-NEXT:    Lower buffer fat pointer operations to buffer resources
 ; GCN-O1-OPTS-NEXT:    CallGraph Construction
 ; GCN-O1-OPTS-NEXT:    Call Graph SCC Pass Manager
@@ -848,11 +836,7 @@
 ; GCN-O2-NEXT:      Scalarize Masked Memory Intrinsics
 ; GCN-O2-NEXT:      Expand reduction intrinsics
 ; GCN-O2-NEXT:      Early CSE
-; GCN-O2-NEXT:    CallGraph Construction
-; GCN-O2-NEXT:    Call Graph SCC Pass Manager
-; GCN-O2-NEXT:      AMDGPU Annotate Kernel Features
-; GCN-O2-NEXT:      FunctionPass Manager
-; GCN-O2-NEXT:        AMDGPU Lower Kernel Arguments
+; GCN-O2-NEXT:      AMDGPU Lower Kernel Arguments
 ; GCN-O2-NEXT:    Lower buffer fat pointer operations to buffer resources
 ; GCN-O2-NEXT:    CallGraph Construction
 ; GCN-O2-NEXT:    Call Graph SCC Pass Manager
@@ -1180,11 +1164,7 @@
 ; GCN-O3-NEXT:      Lazy Block Frequency Analysis
 ; GCN-O3-NEXT:      Optimization Remark Emitter
 ; GCN-O3-NEXT:      Global Value Numbering
-; GCN-O3-NEXT:    CallGraph Construction
-; GCN-O3-NEXT:    Call Graph SCC Pass Manager
-; GCN-O3-NEXT:      AMDGPU Annotate Kernel Features
-; GCN-O3-NEXT:      FunctionPass Manager
-; GCN-O3-NEXT:        AMDGPU Lower Kernel Arguments
+; GCN-O3-NEXT:      AMDGPU Lower Kernel Arguments
 ; GCN-O3-NEXT:    Lower buffer fat pointer operations to buffer resources
 ; GCN-O3-NEXT:    CallGraph Construction
 ; GCN-O3-NEXT:    Call Graph SCC Pass Manager
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
index f93d80cc7adf8..4edd0357c6e7a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
@@ -30,9 +30,12 @@ define amdgpu_kernel void @is_private_vgpr(ptr addrspace(1) %ptr.ptr) {
 ; CI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-SDAG-NEXT:    s_load_dword s2, s[8:9], 0x32
 ; CI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CI-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CI-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-SDAG-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-SDAG-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-SDAG-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
 ; CI-SDAG-NEXT:    s_waitcnt vmcnt(0)
@@ -59,10 +62,13 @@ define amdgpu_kernel void @is_private_vgpr(ptr addrspace(1) %ptr.ptr) {
 ; CI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-GISEL-NEXT:    s_load_dword s2, s[8:9], 0x32
 ; CI-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CI-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CI-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-GISEL-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-GISEL-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
 ; CI-GISEL-NEXT:    s_waitcnt vmcnt(0)
@@ -133,6 +139,9 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
 ; CI-SDAG:       ; %bb.0:
 ; CI-SDAG-NEXT:    s_load_dword s0, s[8:9], 0x1
 ; CI-SDAG-NEXT:    s_load_dword s1, s[8:9], 0x32
+; CI-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CI-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-SDAG-NEXT:    s_cmp_eq_u32 s0, s1
 ; CI-SDAG-NEXT:    s_cselect_b64 s[0:1], -1, 0
@@ -166,6 +175,9 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
 ; CI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-GISEL-NEXT:    s_load_dword s0, s[8:9], 0x32
+; CI-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CI-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-GISEL-NEXT:    s_cmp_lg_u32 s1, s0
 ; CI-GISEL-NEXT:    s_cbranch_scc1 .LBB1_2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
index 637d8388cddf1..9d078f7906b4d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
@@ -63,9 +63,12 @@ define amdgpu_kernel void @is_local_vgpr(ptr addrspace(1) %ptr.ptr) {
 ; CI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-SDAG-NEXT:    s_load_dword s2, s[8:9], 0x33
 ; CI-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CI-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CI-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-SDAG-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-SDAG-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-SDAG-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
 ; CI-SDAG-NEXT:    s_waitcnt vmcnt(0)
@@ -92,10 +95,13 @@ define amdgpu_kernel void @is_local_vgpr(ptr addrspace(1) %ptr.ptr) {
 ; CI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-GISEL-NEXT:    s_load_dword s2, s[8:9], 0x33
 ; CI-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CI-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CI-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CI-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-GISEL-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-GISEL-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
 ; CI-GISEL-NEXT:    s_waitcnt vmcnt(0)
@@ -200,6 +206,9 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
 ; CI-SDAG:       ; %bb.0:
 ; CI-SDAG-NEXT:    s_load_dword s0, s[8:9], 0x1
 ; CI-SDAG-NEXT:    s_load_dword s1, s[8:9], 0x33
+; CI-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CI-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-SDAG-NEXT:    s_cmp_eq_u32 s0, s1
 ; CI-SDAG-NEXT:    s_cselect_b64 s[0:1], -1, 0
@@ -233,6 +242,9 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
 ; CI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-GISEL-NEXT:    s_load_dword s0, s[8:9], 0x33
+; CI-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CI-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-GISEL-NEXT:    s_cmp_lg_u32 s1, s0
 ; CI-GISEL-NEXT:    s_cbranch_scc1 .LBB1_2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.lds.kernel.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.lds.kernel.id.ll
index 97219a8f143ce..0fe371c1b51fe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.lds.kernel.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.lds.kernel.id.ll
@@ -23,8 +23,11 @@ define void @function_lds_id(ptr addrspace(1) %out) {
 define amdgpu_kernel void @kernel_lds_id(ptr addrspace(1) %out) !llvm.amdgcn.lds.kernel.id !0 {
 ; GCN-LABEL: kernel_lds_id:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT:    s_add_i32 s2, s12, 42
+; GCN-NEXT:    s_add_i32 s2, s14, 42
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
@@ -74,6 +77,9 @@ define amdgpu_kernel void @indirect_lds_id(ptr addrspace(1) %out) !llvm.amdgcn.l
 define amdgpu_kernel void @doesnt_use_it(ptr addrspace(1) %out) !llvm.amdgcn.lds.kernel.id !0 {
 ; GCN-LABEL: doesnt_use_it:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GCN-NEXT:    v_mov_b32_e32 v2, 0x64
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index 55fa02a0c582c..cc9e34be209b4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -284,6 +284,9 @@ define amdgpu_kernel void @test_readfirstlane_imm_fold_i32(ptr addrspace(1) %out
 ; CHECK-SDAG-LABEL: test_readfirstlane_imm_fold_i32:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, 32
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s0
@@ -294,6 +297,9 @@ define amdgpu_kernel void @test_readfirstlane_imm_fold_i32(ptr addrspace(1) %out
 ; CHECK-GISEL-LABEL: test_readfirstlane_imm_fold_i32:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, 32
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s0
@@ -309,10 +315,13 @@ define amdgpu_kernel void @test_readfirstlane_imm_fold_i64(ptr addrspace(1) %out
 ; CHECK-SDAG-LABEL: test_readfirstlane_imm_fold_i64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, 32
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -321,10 +330,13 @@ define amdgpu_kernel void @test_readfirstlane_imm_fold_i64(ptr addrspace(1) %out
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CHECK-GISEL-NEXT:    s_mov_b64 s[2:3], 32
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -337,10 +349,13 @@ define amdgpu_kernel void @test_readfirstlane_imm_fold_f64(ptr addrspace(1) %out
 ; CHECK-SDAG-LABEL: test_readfirstlane_imm_fold_f64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40400000
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40400000
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -349,11 +364,14 @@ define amdgpu_kernel void @test_readfirstlane_imm_fold_f64(ptr addrspace(1) %out
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CHECK-GISEL-NEXT:    s_mov_b32 s2, 0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    s_mov_b32 s3, 0x40400000
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -366,12 +384,15 @@ define amdgpu_kernel void @test_readfirstlane_m0(ptr addrspace(1) %out) {
 ; CHECK-SDAG-LABEL: test_readfirstlane_m0:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    s_mov_b32 m0, -1
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, m0
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, m0
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s1
 ; CHECK-SDAG-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -379,12 +400,15 @@ define amdgpu_kernel void @test_readfirstlane_m0(ptr addrspace(1) %out) {
 ; CHECK-GISEL-LABEL: test_readfirstlane_m0:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    s_mov_b32 m0, -1
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, m0
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, m0
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; CHECK-GISEL-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -398,25 +422,31 @@ define amdgpu_kernel void @test_readfirstlane_copy_from_sgpr_i32(ptr addrspace(1
 ; CHECK-SDAG-LABEL: test_readfirstlane_copy_from_sgpr_i32:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    s_mov_b32 s2, 0
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s0
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-SDAG-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-SDAG-NEXT:    s_endpgm
 ;
 ; CHECK-GISEL-LABEL: test_readfirstlane_copy_from_sgpr_i32:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    s_mov_b32 s2, 0
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; CHECK-GISEL-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -430,13 +460,16 @@ define amdgpu_kernel void @test_readfirstlane_copy_from_sgpr_i64(ptr addrspace(1
 ; CHECK-SDAG-LABEL: test_readfirstlane_copy_from_sgpr_i64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -444,13 +477,16 @@ define amdgpu_kernel void @test_readfirstlane_copy_from_sgpr_i64(ptr addrspace(1
 ; CHECK-GISEL-LABEL: test_readfirstlane_copy_from_sgpr_i64:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -464,13 +500,16 @@ define amdgpu_kernel void @test_readfirstlane_copy_from_sgpr_f64(ptr addrspace(1
 ; CHECK-SDAG-LABEL: test_readfirstlane_copy_from_sgpr_f64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -478,13 +517,16 @@ define amdgpu_kernel void @test_readfirstlane_copy_from_sgpr_f64(ptr addrspace(1
 ; CHECK-GISEL-LABEL: test_readfirstlane_copy_from_sgpr_f64:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-GISEL-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
index edb6ebcee1325..f2b0959cc706e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
@@ -179,6 +179,9 @@ define amdgpu_kernel void @test_readlane_imm_sreg_i32(ptr addrspace(1) %out, i32
 ; CHECK-SDAG-LABEL: test_readlane_imm_sreg_i32:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, 32
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s0
@@ -189,6 +192,9 @@ define amdgpu_kernel void @test_readlane_imm_sreg_i32(ptr addrspace(1) %out, i32
 ; CHECK-GISEL-LABEL: test_readlane_imm_sreg_i32:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, 32
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s0
@@ -204,10 +210,13 @@ define amdgpu_kernel void @test_readlane_imm_sreg_i64(ptr addrspace(1) %out, i32
 ; CHECK-SDAG-LABEL: test_readlane_imm_sreg_i64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, 32
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -216,10 +225,13 @@ define amdgpu_kernel void @test_readlane_imm_sreg_i64(ptr addrspace(1) %out, i32
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CHECK-GISEL-NEXT:    s_mov_b64 s[2:3], 32
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -232,10 +244,13 @@ define amdgpu_kernel void @test_readlane_imm_sreg_f64(ptr addrspace(1) %out, i32
 ; CHECK-SDAG-LABEL: test_readlane_imm_sreg_f64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40400000
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40400000
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -244,11 +259,14 @@ define amdgpu_kernel void @test_readlane_imm_sreg_f64(ptr addrspace(1) %out, i32
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CHECK-GISEL-NEXT:    s_mov_b32 s2, 0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    s_mov_b32 s3, 0x40400000
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -262,6 +280,9 @@ define amdgpu_kernel void @test_readlane_vregs_i32(ptr addrspace(1) %out, ptr ad
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CHECK-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -281,6 +302,9 @@ define amdgpu_kernel void @test_readlane_vregs_i32(ptr addrspace(1) %out, ptr ad
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CHECK-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
@@ -311,6 +335,9 @@ define amdgpu_kernel void @test_readlane_vregs_i64(ptr addrspace(1) %out, ptr ad
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CHECK-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -332,6 +359,9 @@ define amdgpu_kernel void @test_readlane_vregs_i64(ptr addrspace(1) %out, ptr ad
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CHECK-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 4, v0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
@@ -365,6 +395,9 @@ define amdgpu_kernel void @test_readlane_vregs_f64(ptr addrspace(1) %out, ptr ad
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CHECK-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -386,6 +419,9 @@ define amdgpu_kernel void @test_readlane_vregs_f64(ptr addrspace(1) %out, ptr ad
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CHECK-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 4, v0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
@@ -419,12 +455,15 @@ define amdgpu_kernel void @test_readlane_m0_sreg(ptr addrspace(1) %out, i32 %src
 ; CHECK-SDAG-LABEL: test_readlane_m0_sreg:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    s_mov_b32 m0, -1
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, m0
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, m0
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s1
 ; CHECK-SDAG-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -432,12 +471,15 @@ define amdgpu_kernel void @test_readlane_m0_sreg(ptr addrspace(1) %out, i32 %src
 ; CHECK-GISEL-LABEL: test_readlane_m0_sreg:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    s_mov_b32 m0, -1
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, m0
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, m0
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; CHECK-GISEL-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -454,11 +496,14 @@ define amdgpu_kernel void @test_readlane_vgpr_imm_i32(ptr addrspace(1) %out) #1
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    ; def v0
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s2, v0, 32
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-SDAG-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-SDAG-NEXT:    s_endpgm
 ;
@@ -468,10 +513,13 @@ define amdgpu_kernel void @test_readlane_vgpr_imm_i32(ptr addrspace(1) %out) #1
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    ; def v0
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s2, v0, 32
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s2
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; CHECK-GISEL-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -485,14 +533,17 @@ define amdgpu_kernel void @test_readlane_vgpr_imm_i64(ptr addrspace(1) %out) #1
 ; CHECK-SDAG-LABEL: test_readlane_vgpr_imm_i64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    ; def v[0:1]
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s2, v1, 32
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s3, v0, 32
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s3
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s3
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -505,10 +556,13 @@ define amdgpu_kernel void @test_readlane_vgpr_imm_i64(ptr addrspace(1) %out) #1
 ; CHECK-GISEL-NEXT:    ; def v[0:1]
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s2, v0, 32
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s3, v1, 32
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -523,14 +577,17 @@ define amdgpu_kernel void @test_readlane_vgpr_imm_f64(ptr addrspace(1) %out) #1
 ; CHECK-SDAG-LABEL: test_readlane_vgpr_imm_f64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    ; def v[0:1]
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s2, v1, 32
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s3, v0, 32
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s3
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s3
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -543,10 +600,13 @@ define amdgpu_kernel void @test_readlane_vgpr_imm_f64(ptr addrspace(1) %out) #1
 ; CHECK-GISEL-NEXT:    ; def v[0:1]
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s2, v0, 32
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s3, v1, 32
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -561,25 +621,31 @@ define amdgpu_kernel void @test_readlane_copy_from_sgpr_i32(ptr addrspace(1) %ou
 ; CHECK-SDAG-LABEL: test_readlane_copy_from_sgpr_i32:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    s_mov_b32 s2, 0
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s0
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-SDAG-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-SDAG-NEXT:    s_endpgm
 ;
 ; CHECK-GISEL-LABEL: test_readlane_copy_from_sgpr_i32:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    s_mov_b32 s2, 0
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s2
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; CHECK-GISEL-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -593,13 +659,16 @@ define amdgpu_kernel void @test_readlane_copy_from_sgpr_i64(ptr addrspace(1) %ou
 ; CHECK-SDAG-LABEL: test_readlane_copy_from_sgpr_i64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -607,13 +676,16 @@ define amdgpu_kernel void @test_readlane_copy_from_sgpr_i64(ptr addrspace(1) %ou
 ; CHECK-GISEL-LABEL: test_readlane_copy_from_sgpr_i64:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-GISEL-NEXT:    s_endpgm
@@ -627,13 +699,16 @@ define amdgpu_kernel void @test_readlane_copy_from_sgpr_f64(ptr addrspace(1) %ou
 ; CHECK-SDAG-LABEL: test_readlane_copy_from_sgpr_f64:
 ; CHECK-SDAG:       ; %bb.0:
 ; CHECK-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; CHECK-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-SDAG-NEXT:    s_endpgm
@@ -641,13 +716,16 @@ define amdgpu_kernel void @test_readlane_copy_from_sgpr_f64(ptr addrspace(1) %ou
 ; CHECK-GISEL-LABEL: test_readlane_copy_from_sgpr_f64:
 ; CHECK-GISEL:       ; %bb.0:
 ; CHECK-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    s_mov_b64 s[2:3], 0
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s3
 ; CHECK-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; CHECK-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CHECK-GISEL-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
index 04d179478590b..4ac2cc98970b5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
@@ -15,6 +15,9 @@ define amdgpu_kernel void @test_writelane_sreg_i32(ptr addrspace(1) %out, i32 %s
 ; GFX802-SDAG-LABEL: test_writelane_sreg_i32:
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s3
 ; GFX802-SDAG-NEXT:    s_load_dword s3, s[0:1], 0x0
@@ -53,6 +56,9 @@ define amdgpu_kernel void @test_writelane_sreg_i32(ptr addrspace(1) %out, i32 %s
 ; GFX802-GISEL-LABEL: test_writelane_sreg_i32:
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s3
 ; GFX802-GISEL-NEXT:    s_load_dword s3, s[0:1], 0x0
@@ -98,6 +104,9 @@ define amdgpu_kernel void @test_writelane_sreg_i64(ptr addrspace(1) %out, i64 %s
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s6, s[8:9], 0x10
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s6
@@ -147,6 +156,9 @@ define amdgpu_kernel void @test_writelane_sreg_i64(ptr addrspace(1) %out, i64 %s
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dword s6, s[8:9], 0x10
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s6
@@ -202,6 +214,9 @@ define amdgpu_kernel void @test_writelane_sreg_f64(ptr addrspace(1) %out, double
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s6, s[8:9], 0x10
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s6
@@ -251,6 +266,9 @@ define amdgpu_kernel void @test_writelane_sreg_f64(ptr addrspace(1) %out, double
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dword s6, s[8:9], 0x10
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s6
@@ -306,6 +324,9 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i3
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dword s3, s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, s0
@@ -348,6 +369,9 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i3
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dword s3, s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s0
@@ -396,6 +420,9 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i64(ptr addrspace(1) %out, i3
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s4, s[8:9], 0x8
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v3, s1
@@ -444,6 +471,9 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i64(ptr addrspace(1) %out, i3
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dword s4, s[8:9], 0x8
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s1
@@ -498,11 +528,14 @@ define amdgpu_kernel void @test_writelane_imm_sreg_f64(ptr addrspace(1) %out, i3
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s4, s[8:9], 0x8
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-SDAG-NEXT:    s_mov_b32 s5, 0x40400000
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s4
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s3
@@ -551,11 +584,14 @@ define amdgpu_kernel void @test_writelane_imm_sreg_f64(ptr addrspace(1) %out, i3
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dword s4, s[8:9], 0x8
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-GISEL-NEXT:    s_mov_b32 s5, 0x40400000
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s4
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s2
@@ -609,6 +645,9 @@ define amdgpu_kernel void @test_writelane_vreg_lane_i32(ptr addrspace(1) %out, p
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX802-SDAG-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -668,6 +707,9 @@ define amdgpu_kernel void @test_writelane_vreg_lane_i32(ptr addrspace(1) %out, p
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, s3
@@ -738,6 +780,9 @@ define amdgpu_kernel void @test_writelane_vreg_lane_i64(ptr addrspace(1) %out, p
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX802-SDAG-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -803,6 +848,9 @@ define amdgpu_kernel void @test_writelane_vreg_lane_i64(ptr addrspace(1) %out, p
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 4, v0
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, s3
@@ -877,7 +925,9 @@ define amdgpu_kernel void @test_writelane_vreg_lane_f64(ptr addrspace(1) %out, p
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
-; GFX802-SDAG-NEXT:    s_mov_b32 s4, 0x40280000
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX802-SDAG-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -886,6 +936,7 @@ define amdgpu_kernel void @test_writelane_vreg_lane_f64(ptr addrspace(1) %out, p
 ; GFX802-SDAG-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX802-SDAG-NEXT:    flat_load_dword v2, v[0:1]
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX802-SDAG-NEXT:    s_mov_b32 s4, 0x40280000
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s3
@@ -946,7 +997,9 @@ define amdgpu_kernel void @test_writelane_vreg_lane_f64(ptr addrspace(1) %out, p
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 4, v0
-; GFX802-GISEL-NEXT:    s_mov_b32 s4, 0x40280000
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, s3
@@ -956,6 +1009,7 @@ define amdgpu_kernel void @test_writelane_vreg_lane_f64(ptr addrspace(1) %out, p
 ; GFX802-GISEL-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX802-GISEL-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX802-GISEL-NEXT:    s_mov_b32 s4, 0x40280000
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1028,15 +1082,18 @@ define amdgpu_kernel void @test_writelane_m0_sreg_i32(ptr addrspace(1) %out, i32
 ; GFX802-SDAG-NEXT:    ;;#ASMSTART
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, -1
 ; GFX802-SDAG-NEXT:    ;;#ASMEND
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-SDAG-NEXT:    s_mov_b32 s4, m0
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dword s3, s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s2
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s3
 ; GFX802-SDAG-NEXT:    v_writelane_b32 v2, s4, m0
+; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX802-SDAG-NEXT:    flat_store_dword v[0:1], v2
 ; GFX802-SDAG-NEXT:    s_endpgm
 ;
@@ -1081,15 +1138,18 @@ define amdgpu_kernel void @test_writelane_m0_sreg_i32(ptr addrspace(1) %out, i32
 ; GFX802-GISEL-NEXT:    ;;#ASMSTART
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, -1
 ; GFX802-GISEL-NEXT:    ;;#ASMEND
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-GISEL-NEXT:    s_mov_b32 s4, m0
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dword s3, s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s2
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, s3
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v2, s4, m0
+; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX802-GISEL-NEXT:    flat_store_dword v[0:1], v2
 ; GFX802-GISEL-NEXT:    s_endpgm
 ;
@@ -1138,6 +1198,9 @@ define amdgpu_kernel void @test_writelane_imm_i32(ptr addrspace(1) %out, i32 %sr
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dword s3, s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, s0
@@ -1180,6 +1243,9 @@ define amdgpu_kernel void @test_writelane_imm_i32(ptr addrspace(1) %out, i32 %sr
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dword s3, s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s0
@@ -1227,6 +1293,9 @@ define amdgpu_kernel void @test_writelane_imm_i64(ptr addrspace(1) %out, i64 %sr
 ; GFX802-SDAG-LABEL: test_writelane_imm_i64:
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v3, s1
@@ -1270,6 +1339,9 @@ define amdgpu_kernel void @test_writelane_imm_i64(ptr addrspace(1) %out, i64 %sr
 ; GFX802-GISEL-LABEL: test_writelane_imm_i64:
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s1
@@ -1319,6 +1391,9 @@ define amdgpu_kernel void @test_writelane_imm_f64(ptr addrspace(1) %out, double
 ; GFX802-SDAG-LABEL: test_writelane_imm_f64:
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v3, s1
@@ -1362,6 +1437,9 @@ define amdgpu_kernel void @test_writelane_imm_f64(ptr addrspace(1) %out, double
 ; GFX802-GISEL-LABEL: test_writelane_imm_f64:
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s1
@@ -1412,6 +1490,9 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i32(i32 inreg %oldval, ptr
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s3
@@ -1449,6 +1530,9 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i32(i32 inreg %oldval, ptr
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s3
@@ -1492,10 +1576,13 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i64(i64 inreg %oldval, ptr
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s6, s[8:9], 0x18
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s6
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s0
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX802-SDAG-NEXT:    v_writelane_b32 v3, s5, m0
@@ -1538,11 +1625,14 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i64(i64 inreg %oldval, ptr
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dword s6, s[8:9], 0x18
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s6
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v0, s4, m0
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v1, s5, m0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s3
@@ -1589,10 +1679,13 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_f64(double inreg %oldval,
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s6, s[8:9], 0x18
 ; GFX802-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s6
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s0
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX802-SDAG-NEXT:    v_writelane_b32 v3, s5, m0
@@ -1635,11 +1728,14 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_f64(double inreg %oldval,
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-GISEL-NEXT:    s_load_dword s6, s[8:9], 0x18
 ; GFX802-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s6
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v0, s4, m0
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v1, s5, m0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s3
@@ -1684,7 +1780,10 @@ define amdgpu_kernel void @test_writelane_imm_oldval_i32(ptr addrspace(1) %out,
 ; GFX802-SDAG-LABEL: test_writelane_imm_oldval_i32:
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, 42
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s3
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, s0
@@ -1716,7 +1815,10 @@ define amdgpu_kernel void @test_writelane_imm_oldval_i32(ptr addrspace(1) %out,
 ; GFX802-GISEL-LABEL: test_writelane_imm_oldval_i32:
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, 42
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s3
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, s0
@@ -1754,11 +1856,14 @@ define amdgpu_kernel void @test_writelane_imm_oldval_i64(ptr addrspace(1) %out,
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, 42
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s4
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX802-SDAG-NEXT:    v_writelane_b32 v1, s3, m0
 ; GFX802-SDAG-NEXT:    v_writelane_b32 v0, s2, m0
@@ -1797,11 +1902,14 @@ define amdgpu_kernel void @test_writelane_imm_oldval_i64(ptr addrspace(1) %out,
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s4
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v0, s2, m0
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v1, s3, m0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, s0
@@ -1845,11 +1953,14 @@ define amdgpu_kernel void @test_writelane_imm_oldval_f64(ptr addrspace(1) %out,
 ; GFX802-SDAG:       ; %bb.0:
 ; GFX802-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX802-SDAG-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX802-SDAG-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40450000
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX802-SDAG-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX802-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX802-SDAG-NEXT:    s_mov_b32 m0, s4
+; GFX802-SDAG-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX802-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX802-SDAG-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX802-SDAG-NEXT:    v_writelane_b32 v1, s3, m0
 ; GFX802-SDAG-NEXT:    v_writelane_b32 v0, s2, m0
@@ -1888,11 +1999,14 @@ define amdgpu_kernel void @test_writelane_imm_oldval_f64(ptr addrspace(1) %out,
 ; GFX802-GISEL:       ; %bb.0:
 ; GFX802-GISEL-NEXT:    s_load_dword s4, s[8:9], 0x10
 ; GFX802-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX802-GISEL-NEXT:    s_add_i32 s12, s12, s17
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v1, 0x40450000
 ; GFX802-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX802-GISEL-NEXT:    s_mov_b32 m0, s4
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX802-GISEL-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v0, s2, m0
 ; GFX802-GISEL-NEXT:    v_writelane_b32 v1, s3, m0
 ; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, s0
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-f64.ll b/llvm/test/CodeGen/AMDGPU/load-constant-f64.ll
index 6f95364ac3644..919c1dfd4694e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-f64.ll
@@ -22,6 +22,9 @@ define amdgpu_kernel void @constant_load_f64(ptr addrspace(1) %out, ptr addrspac
 ; GFX7-HSA-LABEL: constant_load_f64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -90,7 +93,10 @@ define amdgpu_kernel void @constant_load_2v4f64(ptr addrspace(4) noalias nocaptu
 ;
 ; GFX7-HSA-LABEL: constant_load_2v4f64:
 ; GFX7-HSA:       ; %bb.0: ; %entry
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[20:21], s[18:19], 0x0
 ; GFX7-HSA-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 51dfbda53ad4c..817c5def5614f 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -27,6 +27,9 @@ define amdgpu_kernel void @constant_load_i16(ptr addrspace(1) %out, ptr addrspac
 ; GCN-HSA-LABEL: constant_load_i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -117,6 +120,9 @@ define amdgpu_kernel void @constant_load_v2i16(ptr addrspace(1) %out, ptr addrsp
 ; GCN-HSA-LABEL: constant_load_v2i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -188,6 +194,9 @@ define amdgpu_kernel void @constant_load_v3i16(ptr addrspace(1) %out, ptr addrsp
 ; GCN-HSA-LABEL: constant_load_v3i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GCN-HSA-NEXT:    s_add_u32 s4, s0, 4
@@ -286,6 +295,9 @@ define amdgpu_kernel void @constant_load_v4i16(ptr addrspace(1) %out, ptr addrsp
 ; GCN-HSA-LABEL: constant_load_v4i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -360,6 +372,9 @@ define amdgpu_kernel void @constant_load_v8i16(ptr addrspace(1) %out, ptr addrsp
 ; GCN-HSA-LABEL: constant_load_v8i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -445,6 +460,9 @@ define amdgpu_kernel void @constant_load_v16i16(ptr addrspace(1) %out, ptr addrs
 ; GCN-HSA-LABEL: constant_load_v16i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
 ; GCN-HSA-NEXT:    s_add_u32 s10, s8, 16
@@ -584,6 +602,9 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
 ; GCN-HSA-LABEL: constant_load_v16i16_align2:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
@@ -831,6 +852,9 @@ define amdgpu_kernel void @constant_zextload_i16_to_i32(ptr addrspace(1) %out, p
 ; GCN-HSA-LABEL: constant_zextload_i16_to_i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -906,6 +930,9 @@ define amdgpu_kernel void @constant_sextload_i16_to_i32(ptr addrspace(1) %out, p
 ; GCN-HSA-LABEL: constant_sextload_i16_to_i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -982,6 +1009,9 @@ define amdgpu_kernel void @constant_zextload_v1i16_to_v1i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v1i16_to_v1i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1057,6 +1087,9 @@ define amdgpu_kernel void @constant_sextload_v1i16_to_v1i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v1i16_to_v1i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1131,6 +1164,9 @@ define amdgpu_kernel void @constant_zextload_v2i16_to_v2i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v2i16_to_v2i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -1216,6 +1252,9 @@ define amdgpu_kernel void @constant_sextload_v2i16_to_v2i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v2i16_to_v2i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -1305,6 +1344,9 @@ define amdgpu_kernel void @constant_zextload_v3i16_to_v3i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v3i16_to_v3i32:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s0
@@ -1402,6 +1444,9 @@ define amdgpu_kernel void @constant_sextload_v3i16_to_v3i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v3i16_to_v3i32:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s0
@@ -1504,6 +1549,9 @@ define amdgpu_kernel void @constant_zextload_v4i16_to_v4i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v4i16_to_v4i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -1610,6 +1658,9 @@ define amdgpu_kernel void @constant_sextload_v4i16_to_v4i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v4i16_to_v4i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -1727,6 +1778,9 @@ define amdgpu_kernel void @constant_zextload_v8i16_to_v8i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v8i16_to_v8i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1885,6 +1939,9 @@ define amdgpu_kernel void @constant_sextload_v8i16_to_v8i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v8i16_to_v8i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2062,6 +2119,9 @@ define amdgpu_kernel void @constant_zextload_v16i16_to_v16i32(ptr addrspace(1) %
 ; GCN-HSA-LABEL: constant_zextload_v16i16_to_v16i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2324,6 +2384,9 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i32(ptr addrspace(1) %
 ; GCN-HSA-LABEL: constant_sextload_v16i16_to_v16i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2631,7 +2694,10 @@ define amdgpu_kernel void @constant_zextload_v32i16_to_v32i32(ptr addrspace(1) %
 ;
 ; GCN-HSA-LABEL: constant_zextload_v32i16_to_v32i32:
 ; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3112,7 +3178,10 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i32(ptr addrspace(1) %
 ;
 ; GCN-HSA-LABEL: constant_sextload_v32i16_to_v32i32:
 ; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3680,7 +3749,10 @@ define amdgpu_kernel void @constant_zextload_v64i16_to_v64i32(ptr addrspace(1) %
 ;
 ; GCN-HSA-LABEL: constant_zextload_v64i16_to_v64i32:
 ; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -4596,7 +4668,10 @@ define amdgpu_kernel void @constant_sextload_v64i16_to_v64i32(ptr addrspace(1) %
 ;
 ; GCN-HSA-LABEL: constant_sextload_v64i16_to_v64i32:
 ; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -5383,6 +5458,9 @@ define amdgpu_kernel void @constant_zextload_i16_to_i64(ptr addrspace(1) %out, p
 ; GCN-HSA-LABEL: constant_zextload_i16_to_i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5480,6 +5558,9 @@ define amdgpu_kernel void @constant_sextload_i16_to_i64(ptr addrspace(1) %out, p
 ; GCN-HSA-LABEL: constant_sextload_i16_to_i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5578,6 +5659,9 @@ define amdgpu_kernel void @constant_zextload_v1i16_to_v1i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v1i16_to_v1i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5670,6 +5754,9 @@ define amdgpu_kernel void @constant_sextload_v1i16_to_v1i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v1i16_to_v1i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5767,12 +5854,15 @@ define amdgpu_kernel void @constant_zextload_v2i16_to_v2i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v2i16_to_v2i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_lshr_b32 s0, s2, 16
 ; GCN-HSA-NEXT:    s_and_b32 s1, s2, 0xffff
@@ -5877,6 +5967,9 @@ define amdgpu_kernel void @constant_sextload_v2i16_to_v2i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v2i16_to_v2i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -5980,10 +6073,13 @@ define amdgpu_kernel void @constant_zextload_v4i16_to_v4i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v4i16_to_v4i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_lshr_b32 s4, s3, 16
 ; GCN-HSA-NEXT:    s_lshr_b32 s5, s2, 16
@@ -6136,6 +6232,9 @@ define amdgpu_kernel void @constant_sextload_v4i16_to_v4i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v4i16_to_v4i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6292,10 +6391,13 @@ define amdgpu_kernel void @constant_zextload_v8i16_to_v8i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_zextload_v8i16_to_v8i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_lshr_b32 s8, s5, 16
 ; GCN-HSA-NEXT:    s_lshr_b32 s2, s7, 16
@@ -6510,6 +6612,9 @@ define amdgpu_kernel void @constant_sextload_v8i16_to_v8i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: constant_sextload_v8i16_to_v8i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6771,10 +6876,13 @@ define amdgpu_kernel void @constant_zextload_v16i16_to_v16i64(ptr addrspace(1) %
 ; GCN-HSA-LABEL: constant_zextload_v16i16_to_v16i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_lshr_b32 s12, s5, 16
 ; GCN-HSA-NEXT:    s_lshr_b32 s13, s7, 16
@@ -7156,6 +7264,9 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) %
 ; GCN-HSA-LABEL: constant_sextload_v16i16_to_v16i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx8 s[12:19], s[2:3], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -7631,7 +7742,10 @@ define amdgpu_kernel void @constant_zextload_v32i16_to_v32i64(ptr addrspace(1) %
 ;
 ; GCN-HSA-LABEL: constant_zextload_v32i16_to_v32i64:
 ; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -8354,7 +8468,10 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
 ;
 ; GCN-HSA-LABEL: constant_sextload_v32i16_to_v32i64:
 ; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
index 120f47a277ee6..68a6a148819e8 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
@@ -23,6 +23,9 @@ define amdgpu_kernel void @constant_load_i32(ptr addrspace(1) %out, ptr addrspac
 ; GFX7-HSA-LABEL: constant_load_i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -103,6 +106,9 @@ define amdgpu_kernel void @constant_load_v2i32(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-HSA-LABEL: constant_load_v2i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -190,6 +196,9 @@ define amdgpu_kernel void @constant_load_v3i32(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-HSA-LABEL: constant_load_v3i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s0
@@ -284,6 +293,9 @@ define amdgpu_kernel void @constant_load_v4i32(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-HSA-LABEL: constant_load_v4i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -383,6 +395,9 @@ define amdgpu_kernel void @constant_load_v8i32(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-HSA-LABEL: constant_load_v8i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
 ; GFX7-HSA-NEXT:    s_add_u32 s10, s8, 16
@@ -517,6 +532,9 @@ define amdgpu_kernel void @constant_load_v9i32(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-HSA-LABEL: constant_load_v9i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s12, s[10:11], 0x8
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
@@ -678,6 +696,9 @@ define amdgpu_kernel void @constant_load_v10i32(ptr addrspace(1) %out, ptr addrs
 ; GFX7-HSA-LABEL: constant_load_v10i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[12:13], s[10:11], 0x8
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
@@ -847,6 +868,9 @@ define amdgpu_kernel void @constant_load_v11i32(ptr addrspace(1) %out, ptr addrs
 ; GFX7-HSA-LABEL: constant_load_v11i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[12:15], s[10:11], 0x8
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
@@ -1023,6 +1047,9 @@ define amdgpu_kernel void @constant_load_v12i32(ptr addrspace(1) %out, ptr addrs
 ; GFX7-HSA-LABEL: constant_load_v12i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[12:15], s[10:11], 0x8
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
@@ -1202,7 +1229,10 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX7-HSA-LABEL: constant_load_v16i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GFX7-HSA-NEXT:    s_add_u32 s18, s16, 48
@@ -1389,6 +1419,9 @@ define amdgpu_kernel void @constant_zextload_i32_to_i64(ptr addrspace(1) %out, p
 ; GFX7-HSA-LABEL: constant_zextload_i32_to_i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
@@ -1473,6 +1506,9 @@ define amdgpu_kernel void @constant_sextload_i32_to_i64(ptr addrspace(1) %out, p
 ; GFX7-HSA-LABEL: constant_sextload_i32_to_i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -1563,6 +1599,9 @@ define amdgpu_kernel void @constant_zextload_v1i32_to_v1i64(ptr addrspace(1) %ou
 ; GFX7-HSA-LABEL: constant_zextload_v1i32_to_v1i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
@@ -1647,6 +1686,9 @@ define amdgpu_kernel void @constant_sextload_v1i32_to_v1i64(ptr addrspace(1) %ou
 ; GFX7-HSA-LABEL: constant_sextload_v1i32_to_v1i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -1739,12 +1781,15 @@ define amdgpu_kernel void @constant_zextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX7-HSA-LABEL: constant_zextload_v2i32_to_v2i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s3
@@ -1837,6 +1882,9 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX7-HSA-LABEL: constant_sextload_v2i32_to_v2i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -1949,13 +1997,16 @@ define amdgpu_kernel void @constant_zextload_v4i32_to_v4i64(ptr addrspace(1) %ou
 ; GFX7-HSA-LABEL: constant_zextload_v4i32_to_v4i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_add_u32 s2, s0, 16
 ; GFX7-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s2
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s6
@@ -2082,6 +2133,9 @@ define amdgpu_kernel void @constant_sextload_v4i32_to_v4i64(ptr addrspace(1) %ou
 ; GFX7-HSA-LABEL: constant_sextload_v4i32_to_v4i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2244,8 +2298,10 @@ define amdgpu_kernel void @constant_zextload_v8i32_to_v8i64(ptr addrspace(1) %ou
 ; GFX7-HSA-LABEL: constant_zextload_v8i32_to_v8i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_add_u32 s2, s0, 48
@@ -2253,6 +2309,7 @@ define amdgpu_kernel void @constant_zextload_v8i32_to_v8i64(ptr addrspace(1) %ou
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s2
 ; GFX7-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s10
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s11
@@ -2452,6 +2509,9 @@ define amdgpu_kernel void @constant_sextload_v8i32_to_v8i64(ptr addrspace(1) %ou
 ; GFX7-HSA-LABEL: constant_sextload_v8i32_to_v8i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2748,7 +2808,10 @@ define amdgpu_kernel void @constant_sextload_v16i32_to_v16i64(ptr addrspace(1) %
 ;
 ; GFX7-HSA-LABEL: constant_sextload_v16i32_to_v16i64:
 ; GFX7-HSA:       ; %bb.0:
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3196,7 +3259,10 @@ define amdgpu_kernel void @constant_zextload_v16i32_to_v16i64(ptr addrspace(1) %
 ;
 ; GFX7-HSA-LABEL: constant_zextload_v16i32_to_v16i64:
 ; GFX7-HSA:       ; %bb.0:
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3628,7 +3694,10 @@ define amdgpu_kernel void @constant_sextload_v32i32_to_v32i64(ptr addrspace(1) %
 ;
 ; GFX7-HSA-LABEL: constant_sextload_v32i32_to_v32i64:
 ; GFX7-HSA:       ; %bb.0:
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x10
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -4479,8 +4548,10 @@ define amdgpu_kernel void @constant_zextload_v32i32_to_v32i64(ptr addrspace(1) %
 ; GFX7-HSA-LABEL: constant_zextload_v32i32_to_v32i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[36:39], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_add_u32 s0, s36, 0xf0
 ; GFX7-HSA-NEXT:    s_addc_u32 s1, s37, 0
@@ -4509,6 +4580,7 @@ define amdgpu_kernel void @constant_zextload_v32i32_to_v32i64(ptr addrspace(1) %
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v14, s0
 ; GFX7-HSA-NEXT:    s_add_u32 s0, s36, 0x90
 ; GFX7-HSA-NEXT:    s_addc_u32 s1, s37, 0
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s30
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s31
@@ -5097,7 +5169,10 @@ define amdgpu_kernel void @constant_load_v32i32(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX7-HSA-LABEL: constant_load_v32i32:
 ; GFX7-HSA:       ; %bb.0:
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x10
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i64.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i64.ll
index b3e75e767ae64..2219ceea7ec9b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i64.ll
@@ -22,6 +22,9 @@ define amdgpu_kernel void @constant_load_i64(ptr addrspace(1) %out, ptr addrspac
 ; GFX7-LABEL: constant_load_i64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
@@ -95,6 +98,9 @@ define amdgpu_kernel void @constant_load_v2i64(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-LABEL: constant_load_v2i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-NEXT:    v_mov_b32_e32 v4, s0
@@ -179,6 +185,9 @@ define amdgpu_kernel void @constant_load_v3i64(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-LABEL: constant_load_v3i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[2:3], 0x4
 ; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
@@ -294,6 +303,9 @@ define amdgpu_kernel void @constant_load_v4i64(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-LABEL: constant_load_v4i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
 ; GFX7-NEXT:    s_add_u32 s10, s8, 16
@@ -421,7 +433,10 @@ define amdgpu_kernel void @constant_load_v8i64(ptr addrspace(1) %out, ptr addrsp
 ;
 ; GFX7-LABEL: constant_load_v8i64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GFX7-NEXT:    s_add_u32 s18, s16, 48
@@ -638,7 +653,10 @@ define amdgpu_kernel void @constant_load_v16i64(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX7-LABEL: constant_load_v16i64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x10
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index c608bef3f726e..4031be65fab61 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -27,6 +27,9 @@ define amdgpu_kernel void @constant_load_i8(ptr addrspace(1) %out, ptr addrspace
 ; GFX7-HSA-LABEL: constant_load_i8:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -112,6 +115,9 @@ define amdgpu_kernel void @constant_load_v2i8(ptr addrspace(1) %out, ptr addrspa
 ; GFX7-HSA-LABEL: constant_load_v2i8:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -195,6 +201,9 @@ define amdgpu_kernel void @constant_load_v3i8(ptr addrspace(1) %out, ptr addrspa
 ; GFX7-HSA-LABEL: constant_load_v3i8:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -305,6 +314,9 @@ define amdgpu_kernel void @constant_load_v4i8(ptr addrspace(1) %out, ptr addrspa
 ; GFX7-HSA-LABEL: constant_load_v4i8:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -374,6 +386,9 @@ define amdgpu_kernel void @constant_load_v8i8(ptr addrspace(1) %out, ptr addrspa
 ; GFX7-HSA-LABEL: constant_load_v8i8:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -448,6 +463,9 @@ define amdgpu_kernel void @constant_load_v16i8(ptr addrspace(1) %out, ptr addrsp
 ; GFX7-HSA-LABEL: constant_load_v16i8:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -529,6 +547,9 @@ define amdgpu_kernel void @constant_zextload_i8_to_i32(ptr addrspace(1) %out, pt
 ; GFX7-HSA-LABEL: constant_zextload_i8_to_i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -604,6 +625,9 @@ define amdgpu_kernel void @constant_sextload_i8_to_i32(ptr addrspace(1) %out, pt
 ; GFX7-HSA-LABEL: constant_sextload_i8_to_i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -680,6 +704,9 @@ define amdgpu_kernel void @constant_zextload_v1i8_to_v1i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v1i8_to_v1i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -755,6 +782,9 @@ define amdgpu_kernel void @constant_sextload_v1i8_to_v1i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v1i8_to_v1i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -834,6 +864,9 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v2i8_to_v2i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -933,6 +966,9 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v2i8_to_v2i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1030,6 +1066,9 @@ define amdgpu_kernel void @constant_zextload_v3i8_to_v3i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v3i8_to_v3i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s0
@@ -1131,6 +1170,9 @@ define amdgpu_kernel void @constant_sextload_v3i8_to_v3i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v3i8_to_v3i32:
 ; GFX7-HSA:       ; %bb.0: ; %entry
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s0
@@ -1232,6 +1274,9 @@ define amdgpu_kernel void @constant_zextload_v4i8_to_v4i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v4i8_to_v4i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -1336,6 +1381,9 @@ define amdgpu_kernel void @constant_sextload_v4i8_to_v4i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v4i8_to_v4i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -1453,6 +1501,9 @@ define amdgpu_kernel void @constant_zextload_v8i8_to_v8i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v8i8_to_v8i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1612,6 +1663,9 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i32(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v8i8_to_v8i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1794,6 +1848,9 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i32(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_zextload_v16i8_to_v16i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2060,6 +2117,9 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i32(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_sextload_v16i8_to_v16i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2374,6 +2434,9 @@ define amdgpu_kernel void @constant_zextload_v32i8_to_v32i32(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_zextload_v32i8_to_v32i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2856,6 +2919,9 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i32(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_sextload_v32i8_to_v32i32:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3437,7 +3503,10 @@ define amdgpu_kernel void @constant_zextload_v64i8_to_v64i32(ptr addrspace(1) %o
 ;
 ; GFX7-HSA-LABEL: constant_zextload_v64i8_to_v64i32:
 ; GFX7-HSA:       ; %bb.0:
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -4353,7 +4422,10 @@ define amdgpu_kernel void @constant_sextload_v64i8_to_v64i32(ptr addrspace(1) %o
 ;
 ; GFX7-HSA-LABEL: constant_sextload_v64i8_to_v64i32:
 ; GFX7-HSA:       ; %bb.0:
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[16:19], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx16 s[0:15], s[18:19], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -5161,6 +5233,9 @@ define amdgpu_kernel void @constant_zextload_i8_to_i64(ptr addrspace(1) %out, pt
 ; GFX7-HSA-LABEL: constant_zextload_i8_to_i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5243,6 +5318,9 @@ define amdgpu_kernel void @constant_sextload_i8_to_i64(ptr addrspace(1) %out, pt
 ; GFX7-HSA-LABEL: constant_sextload_i8_to_i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5328,6 +5406,9 @@ define amdgpu_kernel void @constant_zextload_v1i8_to_v1i64(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v1i8_to_v1i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5408,6 +5489,9 @@ define amdgpu_kernel void @constant_sextload_v1i8_to_v1i64(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v1i8_to_v1i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5496,6 +5580,9 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i64(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v2i8_to_v2i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5603,6 +5690,9 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i64(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v2i8_to_v2i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5716,10 +5806,13 @@ define amdgpu_kernel void @constant_zextload_v4i8_to_v4i64(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v4i8_to_v4i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_bfe_u32 s4, s2, 0x80008
 ; GFX7-HSA-NEXT:    s_lshr_b32 s3, s2, 24
@@ -5854,6 +5947,9 @@ define amdgpu_kernel void @constant_sextload_v4i8_to_v4i64(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v4i8_to_v4i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6013,10 +6109,13 @@ define amdgpu_kernel void @constant_zextload_v8i8_to_v8i64(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v8i8_to_v8i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_lshr_b32 s4, s2, 24
 ; GFX7-HSA-NEXT:    s_lshr_b32 s5, s3, 24
@@ -6235,6 +6334,9 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v8i8_to_v8i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6504,10 +6606,13 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i64(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_zextload_v16i8_to_v16i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_lshr_b32 s8, s5, 24
 ; GFX7-HSA-NEXT:    s_lshr_b32 s9, s4, 24
@@ -6898,6 +7003,9 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_sextload_v16i8_to_v16i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -7387,10 +7495,13 @@ define amdgpu_kernel void @constant_zextload_v32i8_to_v32i64(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_zextload_v32i8_to_v32i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[8:15], s[2:3], 0x0
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_lshr_b32 s16, s8, 24
 ; GFX7-HSA-NEXT:    s_lshr_b32 s17, s9, 24
@@ -8128,6 +8239,9 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_sextload_v32i8_to_v32i64:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -8898,6 +9012,9 @@ define amdgpu_kernel void @constant_zextload_i8_to_i16(ptr addrspace(1) %out, pt
 ; GFX7-HSA-LABEL: constant_zextload_i8_to_i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -8982,6 +9099,9 @@ define amdgpu_kernel void @constant_sextload_i8_to_i16(ptr addrspace(1) %out, pt
 ; GFX7-HSA-LABEL: constant_sextload_i8_to_i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -9068,6 +9188,9 @@ define amdgpu_kernel void @constant_zextload_v1i8_to_v1i16(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v1i8_to_v1i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -9152,6 +9275,9 @@ define amdgpu_kernel void @constant_sextload_v1i8_to_v1i16(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v1i8_to_v1i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -9241,6 +9367,9 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i16(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v2i8_to_v2i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -9340,6 +9469,9 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i16(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v2i8_to_v2i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -9452,6 +9584,9 @@ define amdgpu_kernel void @constant_zextload_v4i8_to_v4i16(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v4i8_to_v4i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -9560,6 +9695,9 @@ define amdgpu_kernel void @constant_sextload_v4i8_to_v4i16(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v4i8_to_v4i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
@@ -9683,6 +9821,9 @@ define amdgpu_kernel void @constant_zextload_v8i8_to_v8i16(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_zextload_v8i8_to_v8i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -9832,6 +9973,9 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i16(ptr addrspace(1) %out
 ; GFX7-HSA-LABEL: constant_sextload_v8i8_to_v8i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
@@ -10014,6 +10158,9 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i16(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_zextload_v16i8_to_v16i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10261,6 +10408,9 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i16(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_sextload_v16i8_to_v16i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[4:7], s[2:3], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10574,6 +10724,9 @@ define amdgpu_kernel void @constant_zextload_v32i8_to_v32i16(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_zextload_v32i8_to_v32i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11018,6 +11171,9 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i16(ptr addrspace(1) %o
 ; GFX7-HSA-LABEL: constant_sextload_v32i8_to_v32i16:
 ; GFX7-HSA:       ; %bb.0:
 ; GFX7-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
index c5771bc73b945..9054e509cde8e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -28,6 +28,9 @@ define amdgpu_kernel void @global_load_i16(ptr addrspace(1) %out, ptr addrspace(
 ; GCN-HSA-LABEL: global_load_i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -133,6 +136,9 @@ define amdgpu_kernel void @global_load_v2i16(ptr addrspace(1) %out, ptr addrspac
 ; GCN-HSA-LABEL: global_load_v2i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -219,6 +225,9 @@ define amdgpu_kernel void @global_load_v3i16(ptr addrspace(1) %out, ptr addrspac
 ; GCN-HSA-LABEL: global_load_v3i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -339,6 +348,9 @@ define amdgpu_kernel void @global_load_v4i16(ptr addrspace(1) %out, ptr addrspac
 ; GCN-HSA-LABEL: global_load_v4i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -424,6 +436,9 @@ define amdgpu_kernel void @global_load_v8i16(ptr addrspace(1) %out, ptr addrspac
 ; GCN-HSA-LABEL: global_load_v8i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -512,6 +527,9 @@ define amdgpu_kernel void @global_load_v16i16(ptr addrspace(1) %out, ptr addrspa
 ; GCN-HSA-LABEL: global_load_v16i16:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_add_u32 s4, s0, 16
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
@@ -662,6 +680,9 @@ define amdgpu_kernel void @global_load_v16i16_align2(ptr addrspace(1) %in, ptr a
 ; GCN-HSA-LABEL: global_load_v16i16_align2:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_add_u32 s4, s2, 16
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
@@ -811,6 +832,9 @@ define amdgpu_kernel void @global_zextload_i16_to_i32(ptr addrspace(1) %out, ptr
 ; GCN-HSA-LABEL: global_zextload_i16_to_i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -896,6 +920,9 @@ define amdgpu_kernel void @global_sextload_i16_to_i32(ptr addrspace(1) %out, ptr
 ; GCN-HSA-LABEL: global_sextload_i16_to_i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -984,6 +1011,9 @@ define amdgpu_kernel void @global_zextload_v1i16_to_v1i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v1i16_to_v1i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1069,6 +1099,9 @@ define amdgpu_kernel void @global_sextload_v1i16_to_v1i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v1i16_to_v1i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1159,6 +1192,9 @@ define amdgpu_kernel void @global_zextload_v2i16_to_v2i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v2i16_to_v2i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1258,6 +1294,9 @@ define amdgpu_kernel void @global_sextload_v2i16_to_v2i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v2i16_to_v2i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1359,6 +1398,9 @@ define amdgpu_kernel void @global_zextload_v3i16_to_v3i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v3i16_to_v3i32:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1469,6 +1511,9 @@ define amdgpu_kernel void @global_sextload_v3i16_to_v3i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v3i16_to_v3i32:
 ; GCN-HSA:       ; %bb.0: ; %entry
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1586,6 +1631,9 @@ define amdgpu_kernel void @global_zextload_v4i16_to_v4i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v4i16_to_v4i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1701,6 +1749,9 @@ define amdgpu_kernel void @global_sextload_v4i16_to_v4i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v4i16_to_v4i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1823,6 +1874,9 @@ define amdgpu_kernel void @global_zextload_v8i16_to_v8i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v8i16_to_v8i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1972,6 +2026,9 @@ define amdgpu_kernel void @global_sextload_v8i16_to_v8i32(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v8i16_to_v8i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -2136,6 +2193,9 @@ define amdgpu_kernel void @global_zextload_v16i16_to_v16i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_zextload_v16i16_to_v16i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_add_u32 s4, s2, 16
 ; GCN-HSA-NEXT:    s_addc_u32 s5, s3, 0
@@ -2372,6 +2432,9 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_sextload_v16i16_to_v16i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -2643,6 +2706,9 @@ define amdgpu_kernel void @global_zextload_v32i16_to_v32i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_zextload_v32i16_to_v32i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_add_u32 s4, s2, 16
 ; GCN-HSA-NEXT:    s_addc_u32 s5, s3, 0
@@ -3054,6 +3120,9 @@ define amdgpu_kernel void @global_sextload_v32i16_to_v32i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_sextload_v32i16_to_v32i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -3573,6 +3642,9 @@ define amdgpu_kernel void @global_zextload_v64i16_to_v64i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_zextload_v64i16_to_v64i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -4377,6 +4449,9 @@ define amdgpu_kernel void @global_sextload_v64i16_to_v64i32(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_sextload_v64i16_to_v64i32:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5142,6 +5217,9 @@ define amdgpu_kernel void @global_zextload_i16_to_i64(ptr addrspace(1) %out, ptr
 ; GCN-HSA-LABEL: global_zextload_i16_to_i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5239,6 +5317,9 @@ define amdgpu_kernel void @global_sextload_i16_to_i64(ptr addrspace(1) %out, ptr
 ; GCN-HSA-LABEL: global_sextload_i16_to_i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5334,6 +5415,9 @@ define amdgpu_kernel void @global_zextload_v1i16_to_v1i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v1i16_to_v1i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5426,6 +5510,9 @@ define amdgpu_kernel void @global_sextload_v1i16_to_v1i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v1i16_to_v1i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5524,6 +5611,9 @@ define amdgpu_kernel void @global_zextload_v2i16_to_v2i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v2i16_to_v2i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5633,6 +5723,9 @@ define amdgpu_kernel void @global_sextload_v2i16_to_v2i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v2i16_to_v2i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5751,6 +5844,9 @@ define amdgpu_kernel void @global_zextload_v4i16_to_v4i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v4i16_to_v4i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -5896,6 +5992,9 @@ define amdgpu_kernel void @global_sextload_v4i16_to_v4i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v4i16_to_v4i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -6056,10 +6155,10 @@ define amdgpu_kernel void @global_zextload_v8i16_to_v8i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_zextload_v8i16_to_v8i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v6, v4
-; GCN-HSA-NEXT:    v_mov_b32_e32 v8, v4
-; GCN-HSA-NEXT:    v_mov_b32_e32 v10, v4
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -6074,8 +6173,11 @@ define amdgpu_kernel void @global_zextload_v8i16_to_v8i64(ptr addrspace(1) %out,
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s0
 ; GCN-HSA-NEXT:    s_add_u32 s0, s0, 32
 ; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v6, v4
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v24, s3
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v26, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, v4
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v12, v4
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v14, v4
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v16, v4
@@ -6275,6 +6377,9 @@ define amdgpu_kernel void @global_sextload_v8i16_to_v8i64(ptr addrspace(1) %out,
 ; GCN-HSA-LABEL: global_sextload_v8i16_to_v8i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -6525,10 +6630,10 @@ define amdgpu_kernel void @global_zextload_v16i16_to_v16i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_zextload_v16i16_to_v16i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v12, v8
-; GCN-HSA-NEXT:    v_mov_b32_e32 v14, v8
-; GCN-HSA-NEXT:    v_mov_b32_e32 v15, v8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -6545,7 +6650,10 @@ define amdgpu_kernel void @global_zextload_v16i16_to_v16i64(ptr addrspace(1) %ou
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v22, s5
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s4
 ; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0x50
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, v8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, v8
 ; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, v8
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v17, v8
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v18, v8
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v20, v8
@@ -6905,6 +7013,9 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_sextload_v16i16_to_v16i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -7376,6 +7487,9 @@ define amdgpu_kernel void @global_zextload_v32i16_to_v32i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_zextload_v32i16_to_v32i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    s_add_u32 s4, s2, 16
 ; GCN-HSA-NEXT:    s_addc_u32 s5, s3, 0
@@ -8078,6 +8192,9 @@ define amdgpu_kernel void @global_sextload_v32i16_to_v32i64(ptr addrspace(1) %ou
 ; GCN-HSA-LABEL: global_sextload_v32i16_to_v32i64:
 ; GCN-HSA:       ; %bb.0:
 ; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
index 033a66abcedb9..e8c862a3cb93c 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
@@ -27,6 +27,9 @@ define amdgpu_kernel void @global_load_i32(ptr addrspace(1) %out, ptr addrspace(
 ; GCNX3-HSA-LABEL: global_load_i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -106,6 +109,9 @@ define amdgpu_kernel void @global_load_v2i32(ptr addrspace(1) %out, ptr addrspac
 ; GCNX3-HSA-LABEL: global_load_v2i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -186,6 +192,9 @@ define amdgpu_kernel void @global_load_v3i32(ptr addrspace(1) %out, ptr addrspac
 ; GCNX3-HSA-LABEL: global_load_v3i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -270,6 +279,9 @@ define amdgpu_kernel void @global_load_v4i32(ptr addrspace(1) %out, ptr addrspac
 ; GCNX3-HSA-LABEL: global_load_v4i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -352,6 +364,9 @@ define amdgpu_kernel void @global_load_v8i32(ptr addrspace(1) %out, ptr addrspac
 ; GCNX3-HSA-LABEL: global_load_v8i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v5, s3
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, s2
@@ -458,6 +473,9 @@ define amdgpu_kernel void @global_load_v9i32(ptr addrspace(1) %out, ptr addrspac
 ; GCNX3-HSA-LABEL: global_load_v9i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 32
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
@@ -589,6 +607,9 @@ define amdgpu_kernel void @global_load_v10i32(ptr addrspace(1) %out, ptr addrspa
 ; GCNX3-HSA-LABEL: global_load_v10i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 32
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
@@ -719,6 +740,9 @@ define amdgpu_kernel void @global_load_v11i32(ptr addrspace(1) %out, ptr addrspa
 ; GCNX3-HSA-LABEL: global_load_v11i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 32
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
@@ -854,6 +878,9 @@ define amdgpu_kernel void @global_load_v12i32(ptr addrspace(1) %out, ptr addrspa
 ; GCNX3-HSA-LABEL: global_load_v12i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 32
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
@@ -987,6 +1014,9 @@ define amdgpu_kernel void @global_load_v16i32(ptr addrspace(1) %out, ptr addrspa
 ; GCNX3-HSA-LABEL: global_load_v16i32:
 ; GCNX3-HSA:       ; %bb.0: ; %entry
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 16
 ; GCNX3-HSA-NEXT:    s_addc_u32 s5, s3, 0
@@ -1134,6 +1164,9 @@ define amdgpu_kernel void @global_zextload_i32_to_i64(ptr addrspace(1) %out, ptr
 ; GCNX3-HSA-LABEL: global_zextload_i32_to_i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1217,6 +1250,9 @@ define amdgpu_kernel void @global_sextload_i32_to_i64(ptr addrspace(1) %out, ptr
 ; GCNX3-HSA-LABEL: global_sextload_i32_to_i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1301,6 +1337,9 @@ define amdgpu_kernel void @global_zextload_v1i32_to_v1i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-LABEL: global_zextload_v1i32_to_v1i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1384,6 +1423,9 @@ define amdgpu_kernel void @global_sextload_v1i32_to_v1i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-LABEL: global_sextload_v1i32_to_v1i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1471,6 +1513,9 @@ define amdgpu_kernel void @global_zextload_v2i32_to_v2i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-LABEL: global_zextload_v2i32_to_v2i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1569,6 +1614,9 @@ define amdgpu_kernel void @global_sextload_v2i32_to_v2i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-LABEL: global_sextload_v2i32_to_v2i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1674,8 +1722,10 @@ define amdgpu_kernel void @global_zextload_v4i32_to_v4i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-LABEL: global_zextload_v4i32_to_v4i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v5, 0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v7, v5
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1683,6 +1733,7 @@ define amdgpu_kernel void @global_zextload_v4i32_to_v4i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 16
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, s3
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v7, v5
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v8, s2
 ; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v4, v2
@@ -1800,6 +1851,9 @@ define amdgpu_kernel void @global_sextload_v4i32_to_v4i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-LABEL: global_sextload_v4i32_to_v4i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1941,8 +1995,10 @@ define amdgpu_kernel void @global_zextload_v8i32_to_v8i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-LABEL: global_zextload_v8i32_to_v8i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v9, 0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v11, v9
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -1957,6 +2013,7 @@ define amdgpu_kernel void @global_zextload_v8i32_to_v8i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v15, s3
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v14, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v11, v9
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v13, s1
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v12, s0
@@ -2134,6 +2191,9 @@ define amdgpu_kernel void @global_sextload_v8i32_to_v8i64(ptr addrspace(1) %out,
 ; GCNX3-HSA-LABEL: global_sextload_v8i32_to_v8i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -2370,6 +2430,9 @@ define amdgpu_kernel void @global_sextload_v16i32_to_v16i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-LABEL: global_sextload_v16i32_to_v16i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -2731,8 +2794,10 @@ define amdgpu_kernel void @global_zextload_v16i32_to_v16i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-LABEL: global_zextload_v16i32_to_v16i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v17, 0
-; GCNX3-HSA-NEXT:    v_mov_b32_e32 v19, v17
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 16
 ; GCNX3-HSA-NEXT:    s_addc_u32 s5, s3, 0
@@ -2766,6 +2831,7 @@ define amdgpu_kernel void @global_zextload_v16i32_to_v16i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v21, s1
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v26, s2
 ; GCNX3-HSA-NEXT:    s_add_u32 s2, s0, 64
+; GCNX3-HSA-NEXT:    v_mov_b32_e32 v19, v17
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v20, s0
 ; GCNX3-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCNX3-HSA-NEXT:    s_waitcnt vmcnt(3)
@@ -3122,6 +3188,9 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-LABEL: global_sextload_v32i32_to_v32i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCNX3-HSA-NEXT:    v_mov_b32_e32 v1, s3
@@ -3589,12 +3658,12 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ;
 ; GCN-GFX900-HSA-LABEL: global_sextload_v32i32_to_v32i64:
 ; GCN-GFX900-HSA:       ; %bb.0:
-; GCN-GFX900-HSA-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; GCN-GFX900-HSA-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; GCN-GFX900-HSA-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; GCN-GFX900-HSA-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; GCN-GFX900-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-GFX900-HSA-NEXT:    s_add_u32 s16, s16, s15
-; GCN-GFX900-HSA-NEXT:    s_addc_u32 s17, s17, 0
+; GCN-GFX900-HSA-NEXT:    s_add_u32 s20, s20, s17
+; GCN-GFX900-HSA-NEXT:    s_addc_u32 s21, s21, 0
 ; GCN-GFX900-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-GFX900-HSA-NEXT:    global_load_dwordx4 v[0:3], v8, s[2:3] offset:96
 ; GCN-GFX900-HSA-NEXT:    global_load_dwordx4 v[4:7], v8, s[2:3] offset:112
@@ -3620,11 +3689,11 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
 ; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v4, v0
 ; GCN-GFX900-HSA-NEXT:    v_mov_b32_e32 v6, v1
-; GCN-GFX900-HSA-NEXT:    buffer_store_dword v25, off, s[16:19], 0 ; 4-byte Folded Spill
+; GCN-GFX900-HSA-NEXT:    buffer_store_dword v25, off, s[20:23], 0 ; 4-byte Folded Spill
 ; GCN-GFX900-HSA-NEXT:    s_nop 0
-; GCN-GFX900-HSA-NEXT:    buffer_store_dword v26, off, s[16:19], 0 offset:4 ; 4-byte Folded Spill
-; GCN-GFX900-HSA-NEXT:    buffer_store_dword v27, off, s[16:19], 0 offset:8 ; 4-byte Folded Spill
-; GCN-GFX900-HSA-NEXT:    buffer_store_dword v28, off, s[16:19], 0 offset:12 ; 4-byte Folded Spill
+; GCN-GFX900-HSA-NEXT:    buffer_store_dword v26, off, s[20:23], 0 offset:4 ; 4-byte Folded Spill
+; GCN-GFX900-HSA-NEXT:    buffer_store_dword v27, off, s[20:23], 0 offset:8 ; 4-byte Folded Spill
+; GCN-GFX900-HSA-NEXT:    buffer_store_dword v28, off, s[20:23], 0 offset:12 ; 4-byte Folded Spill
 ; GCN-GFX900-HSA-NEXT:    s_waitcnt vmcnt(7)
 ; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v28, 31, v12
 ; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v26, 31, v11
@@ -3667,11 +3736,11 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v8, v[33:36], s[0:1] offset:224
 ; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v8, v[29:32], s[0:1] offset:240
 ; GCN-GFX900-HSA-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:192
-; GCN-GFX900-HSA-NEXT:    buffer_load_dword v32, off, s[16:19], 0 ; 4-byte Folded Reload
+; GCN-GFX900-HSA-NEXT:    buffer_load_dword v32, off, s[20:23], 0 ; 4-byte Folded Reload
 ; GCN-GFX900-HSA-NEXT:    s_nop 0
-; GCN-GFX900-HSA-NEXT:    buffer_load_dword v33, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
-; GCN-GFX900-HSA-NEXT:    buffer_load_dword v34, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
-; GCN-GFX900-HSA-NEXT:    buffer_load_dword v35, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
+; GCN-GFX900-HSA-NEXT:    buffer_load_dword v33, off, s[20:23], 0 offset:4 ; 4-byte Folded Reload
+; GCN-GFX900-HSA-NEXT:    buffer_load_dword v34, off, s[20:23], 0 offset:8 ; 4-byte Folded Reload
+; GCN-GFX900-HSA-NEXT:    buffer_load_dword v35, off, s[20:23], 0 offset:12 ; 4-byte Folded Reload
 ; GCN-GFX900-HSA-NEXT:    s_waitcnt vmcnt(8)
 ; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v60, 31, v52
 ; GCN-GFX900-HSA-NEXT:    v_ashrrev_i32_e32 v58, 31, v51
@@ -3913,6 +3982,9 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
 ; GCNX3-HSA-LABEL: global_zextload_v32i32_to_v32i64:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 16
 ; GCNX3-HSA-NEXT:    s_addc_u32 s5, s3, 0
@@ -4437,6 +4509,9 @@ define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspa
 ; GCNX3-HSA-LABEL: global_load_v32i32:
 ; GCNX3-HSA:       ; %bb.0:
 ; GCNX3-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCNX3-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCNX3-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCNX3-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCNX3-HSA-NEXT:    s_add_u32 s4, s2, 16
 ; GCNX3-HSA-NEXT:    s_addc_u32 s5, s3, 0
diff --git a/llvm/test/CodeGen/AMDGPU/load-select-ptr.ll b/llvm/test/CodeGen/AMDGPU/load-select-ptr.ll
index 4dfc773d615e4..1a6fa3c518ca7 100644
--- a/llvm/test/CodeGen/AMDGPU/load-select-ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-select-ptr.ll
@@ -13,7 +13,8 @@
 ; GCN: s_cselect_b32
 
 ; GCN-NOT: load_dword
-; GCN: flat_load_dwordx2
+; GCN: flat_load_dword
+; GCN: flat_load_dword
 ; GCN-NOT: load_dword
 
 ; GCN: flat_store_dwordx2
diff --git a/llvm/test/CodeGen/AMDGPU/mad24-get-global-id.ll b/llvm/test/CodeGen/AMDGPU/mad24-get-global-id.ll
index 245a2775d9f2f..07b5e1610cfc0 100644
--- a/llvm/test/CodeGen/AMDGPU/mad24-get-global-id.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad24-get-global-id.ll
@@ -9,7 +9,7 @@ declare ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() #0
 
 ; GCN-LABEL: {{^}}get_global_id_0:
 ; GCN: s_and_b32 [[WGSIZEX:s[0-9]+]], {{s[0-9]+}}, 0xffff
-; GCN: s_mul_i32 [[MUL:s[0-9]+]], s12, [[WGSIZEX]]
+; GCN: s_mul_i32 [[MUL:s[0-9]+]], s14, [[WGSIZEX]]
 ; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, [[MUL]], v0
 define amdgpu_kernel void @get_global_id_0(ptr addrspace(1) %out) #1 {
   %dispatch.ptr = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
diff --git a/llvm/test/CodeGen/AMDGPU/match-perm-extract-vector-elt-bug.ll b/llvm/test/CodeGen/AMDGPU/match-perm-extract-vector-elt-bug.ll
index b3b529d4e5e5b..4896e504cfdf4 100644
--- a/llvm/test/CodeGen/AMDGPU/match-perm-extract-vector-elt-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/match-perm-extract-vector-elt-bug.ll
@@ -11,8 +11,8 @@ define amdgpu_kernel void @test(ptr addrspace(1) %src, ptr addrspace(1) %dst) {
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX9-NEXT:    s_mul_i32 s12, s12, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s12
+; GFX9-NEXT:    s_mul_i32 s14, s14, s4
+; GFX9-NEXT:    s_add_i32 s5, s5, s14
 ; GFX9-NEXT:    v_add_u32_e32 v0, s5, v0
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX9-NEXT:    v_lshlrev_b64 v[4:5], 4, v[0:1]
@@ -39,8 +39,8 @@ define amdgpu_kernel void @test(ptr addrspace(1) %src, ptr addrspace(1) %dst) {
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX10-NEXT:    s_mul_i32 s12, s12, s4
-; GFX10-NEXT:    v_add3_u32 v0, s5, s12, v0
+; GFX10-NEXT:    s_mul_i32 s14, s14, s4
+; GFX10-NEXT:    v_add3_u32 v0, s5, s14, v0
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX10-NEXT:    v_lshlrev_b64 v[4:5], 4, v[0:1]
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, s0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
index 88ee2a34dd49f..8d020b9e1a603 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
@@ -9,6 +9,8 @@ define amdgpu_kernel void @memcpy_p0_p0_minsize(ptr %dest, ptr readonly %src) #0
 ; CHECK-LABEL: memcpy_p0_p0_minsize:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v12, s3
 ; CHECK-NEXT:    v_mov_b32_e32 v11, s2
@@ -94,12 +96,12 @@ entry:
 define amdgpu_kernel void @memcpy_p5_p4_minsize(ptr addrspace(5) %local, ptr addrspace(4) %0) #0 {
 ; CHECK-LABEL: memcpy_p5_p4_minsize:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; CHECK-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; CHECK-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
 ; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x0
 ; CHECK-NEXT:    v_mov_b32_e32 v24, 0
-; CHECK-NEXT:    s_add_u32 s16, s16, s15
+; CHECK-NEXT:    s_add_u32 s20, s20, s17
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:112
 ; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1] offset:96
@@ -107,50 +109,50 @@ define amdgpu_kernel void @memcpy_p5_p4_minsize(ptr addrspace(5) %local, ptr add
 ; CHECK-NEXT:    global_load_dwordx4 v[12:15], v24, s[0:1] offset:64
 ; CHECK-NEXT:    global_load_dwordx4 v[16:19], v24, s[0:1] offset:48
 ; CHECK-NEXT:    global_load_dwordx4 v[20:23], v24, s[0:1] offset:32
-; CHECK-NEXT:    s_addc_u32 s17, s17, 0
+; CHECK-NEXT:    s_addc_u32 s21, s21, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v25, s2
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_dword v3, v25, s[16:19], 0 offen offset:124
-; CHECK-NEXT:    buffer_store_dword v2, v25, s[16:19], 0 offen offset:120
-; CHECK-NEXT:    buffer_store_dword v1, v25, s[16:19], 0 offen offset:116
-; CHECK-NEXT:    buffer_store_dword v0, v25, s[16:19], 0 offen offset:112
+; CHECK-NEXT:    buffer_store_dword v3, v25, s[20:23], 0 offen offset:124
+; CHECK-NEXT:    buffer_store_dword v2, v25, s[20:23], 0 offen offset:120
+; CHECK-NEXT:    buffer_store_dword v1, v25, s[20:23], 0 offen offset:116
+; CHECK-NEXT:    buffer_store_dword v0, v25, s[20:23], 0 offen offset:112
 ; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_dword v7, v25, s[16:19], 0 offen offset:108
-; CHECK-NEXT:    buffer_store_dword v6, v25, s[16:19], 0 offen offset:104
-; CHECK-NEXT:    buffer_store_dword v5, v25, s[16:19], 0 offen offset:100
-; CHECK-NEXT:    buffer_store_dword v4, v25, s[16:19], 0 offen offset:96
+; CHECK-NEXT:    buffer_store_dword v7, v25, s[20:23], 0 offen offset:108
+; CHECK-NEXT:    buffer_store_dword v6, v25, s[20:23], 0 offen offset:104
+; CHECK-NEXT:    buffer_store_dword v5, v25, s[20:23], 0 offen offset:100
+; CHECK-NEXT:    buffer_store_dword v4, v25, s[20:23], 0 offen offset:96
 ; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1]
 ; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_dword v11, v25, s[16:19], 0 offen offset:92
-; CHECK-NEXT:    buffer_store_dword v10, v25, s[16:19], 0 offen offset:88
-; CHECK-NEXT:    buffer_store_dword v9, v25, s[16:19], 0 offen offset:84
-; CHECK-NEXT:    buffer_store_dword v8, v25, s[16:19], 0 offen offset:80
+; CHECK-NEXT:    buffer_store_dword v11, v25, s[20:23], 0 offen offset:92
+; CHECK-NEXT:    buffer_store_dword v10, v25, s[20:23], 0 offen offset:88
+; CHECK-NEXT:    buffer_store_dword v9, v25, s[20:23], 0 offen offset:84
+; CHECK-NEXT:    buffer_store_dword v8, v25, s[20:23], 0 offen offset:80
 ; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_dword v15, v25, s[16:19], 0 offen offset:76
-; CHECK-NEXT:    buffer_store_dword v14, v25, s[16:19], 0 offen offset:72
-; CHECK-NEXT:    buffer_store_dword v13, v25, s[16:19], 0 offen offset:68
-; CHECK-NEXT:    buffer_store_dword v12, v25, s[16:19], 0 offen offset:64
+; CHECK-NEXT:    buffer_store_dword v15, v25, s[20:23], 0 offen offset:76
+; CHECK-NEXT:    buffer_store_dword v14, v25, s[20:23], 0 offen offset:72
+; CHECK-NEXT:    buffer_store_dword v13, v25, s[20:23], 0 offen offset:68
+; CHECK-NEXT:    buffer_store_dword v12, v25, s[20:23], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_dword v19, v25, s[16:19], 0 offen offset:60
-; CHECK-NEXT:    buffer_store_dword v18, v25, s[16:19], 0 offen offset:56
-; CHECK-NEXT:    buffer_store_dword v17, v25, s[16:19], 0 offen offset:52
-; CHECK-NEXT:    buffer_store_dword v16, v25, s[16:19], 0 offen offset:48
+; CHECK-NEXT:    buffer_store_dword v19, v25, s[20:23], 0 offen offset:60
+; CHECK-NEXT:    buffer_store_dword v18, v25, s[20:23], 0 offen offset:56
+; CHECK-NEXT:    buffer_store_dword v17, v25, s[20:23], 0 offen offset:52
+; CHECK-NEXT:    buffer_store_dword v16, v25, s[20:23], 0 offen offset:48
 ; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_dword v23, v25, s[16:19], 0 offen offset:44
-; CHECK-NEXT:    buffer_store_dword v22, v25, s[16:19], 0 offen offset:40
-; CHECK-NEXT:    buffer_store_dword v21, v25, s[16:19], 0 offen offset:36
-; CHECK-NEXT:    buffer_store_dword v20, v25, s[16:19], 0 offen offset:32
+; CHECK-NEXT:    buffer_store_dword v23, v25, s[20:23], 0 offen offset:44
+; CHECK-NEXT:    buffer_store_dword v22, v25, s[20:23], 0 offen offset:40
+; CHECK-NEXT:    buffer_store_dword v21, v25, s[20:23], 0 offen offset:36
+; CHECK-NEXT:    buffer_store_dword v20, v25, s[20:23], 0 offen offset:32
 ; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_dword v3, v25, s[16:19], 0 offen offset:28
-; CHECK-NEXT:    buffer_store_dword v2, v25, s[16:19], 0 offen offset:24
-; CHECK-NEXT:    buffer_store_dword v1, v25, s[16:19], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_dword v0, v25, s[16:19], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v3, v25, s[20:23], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v2, v25, s[20:23], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v1, v25, s[20:23], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v0, v25, s[20:23], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_dword v7, v25, s[16:19], 0 offen offset:12
-; CHECK-NEXT:    buffer_store_dword v6, v25, s[16:19], 0 offen offset:8
-; CHECK-NEXT:    buffer_store_dword v5, v25, s[16:19], 0 offen offset:4
-; CHECK-NEXT:    buffer_store_dword v4, v25, s[16:19], 0 offen
+; CHECK-NEXT:    buffer_store_dword v7, v25, s[20:23], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v6, v25, s[20:23], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v5, v25, s[20:23], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v4, v25, s[20:23], 0 offen
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %local, ptr addrspace(4) %0, i64 128, i1 false)
@@ -160,55 +162,57 @@ entry:
 define amdgpu_kernel void @memcpy_p0_p5_minsize(ptr %generic, ptr addrspace(5) %src) #0 {
 ; CHECK-LABEL: memcpy_p0_p5_minsize:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; CHECK-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; CHECK-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; CHECK-NEXT:    s_load_dword s0, s[8:9], 0x8
-; CHECK-NEXT:    s_add_u32 s16, s16, s15
-; CHECK-NEXT:    s_addc_u32 s17, s17, 0
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; CHECK-NEXT:    s_add_u32 s20, s20, s17
+; CHECK-NEXT:    s_addc_u32 s21, s21, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v26, s0
-; CHECK-NEXT:    buffer_load_dword v3, v26, s[16:19], 0 offen offset:124
-; CHECK-NEXT:    buffer_load_dword v2, v26, s[16:19], 0 offen offset:120
-; CHECK-NEXT:    buffer_load_dword v1, v26, s[16:19], 0 offen offset:116
-; CHECK-NEXT:    buffer_load_dword v0, v26, s[16:19], 0 offen offset:112
-; CHECK-NEXT:    buffer_load_dword v7, v26, s[16:19], 0 offen offset:108
-; CHECK-NEXT:    buffer_load_dword v6, v26, s[16:19], 0 offen offset:104
-; CHECK-NEXT:    buffer_load_dword v5, v26, s[16:19], 0 offen offset:100
-; CHECK-NEXT:    buffer_load_dword v4, v26, s[16:19], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen offset:96
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CHECK-NEXT:    buffer_load_dword v8, v26, s[16:19], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v9, v26, s[16:19], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v10, v26, s[16:19], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v11, v26, s[16:19], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v12, v26, s[16:19], 0 offen offset:32
-; CHECK-NEXT:    buffer_load_dword v13, v26, s[16:19], 0 offen offset:36
-; CHECK-NEXT:    buffer_load_dword v14, v26, s[16:19], 0 offen offset:40
-; CHECK-NEXT:    buffer_load_dword v15, v26, s[16:19], 0 offen offset:44
-; CHECK-NEXT:    buffer_load_dword v16, v26, s[16:19], 0 offen offset:48
-; CHECK-NEXT:    buffer_load_dword v17, v26, s[16:19], 0 offen offset:52
-; CHECK-NEXT:    buffer_load_dword v18, v26, s[16:19], 0 offen offset:56
-; CHECK-NEXT:    buffer_load_dword v19, v26, s[16:19], 0 offen offset:60
-; CHECK-NEXT:    buffer_load_dword v23, v26, s[16:19], 0 offen offset:92
-; CHECK-NEXT:    buffer_load_dword v22, v26, s[16:19], 0 offen offset:88
-; CHECK-NEXT:    buffer_load_dword v21, v26, s[16:19], 0 offen offset:84
-; CHECK-NEXT:    buffer_load_dword v20, v26, s[16:19], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v8, v26, s[20:23], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v9, v26, s[20:23], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v10, v26, s[20:23], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v11, v26, s[20:23], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v12, v26, s[20:23], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v13, v26, s[20:23], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v14, v26, s[20:23], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v15, v26, s[20:23], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v16, v26, s[20:23], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v17, v26, s[20:23], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v18, v26, s[20:23], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v19, v26, s[20:23], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v23, v26, s[20:23], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v22, v26, s[20:23], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v21, v26, s[20:23], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v20, v26, s[20:23], 0 offen offset:80
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v25, s1
 ; CHECK-NEXT:    v_mov_b32_e32 v24, s0
 ; CHECK-NEXT:    s_waitcnt vmcnt(20)
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:112
-; CHECK-NEXT:    buffer_load_dword v3, v26, s[16:19], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:76
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_dword v2, v26, s[16:19], 0 offen offset:72
-; CHECK-NEXT:    buffer_load_dword v1, v26, s[16:19], 0 offen offset:68
-; CHECK-NEXT:    buffer_load_dword v0, v26, s[16:19], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7] offset:96
-; CHECK-NEXT:    buffer_load_dword v4, v26, s[16:19], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_dword v5, v26, s[16:19], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v6, v26, s[16:19], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v7, v26, s[16:19], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:12
 ; CHECK-NEXT:    s_nop 0
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[20:23] offset:80
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:64
@@ -268,6 +272,8 @@ define amdgpu_kernel void @memcpy_p0_p3_minsize(ptr %generic) #0 {
 ; CHECK-NEXT:    ds_read2_b64 v[4:7], v16 offset0:2 offset1:3
 ; CHECK-NEXT:    ds_read2_b64 v[8:11], v16 offset0:4 offset1:5
 ; CHECK-NEXT:    ds_read2_b64 v[12:15], v16 offset0:6 offset1:7
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v21, s1
 ; CHECK-NEXT:    v_mov_b32_e32 v20, s0
@@ -294,6 +300,8 @@ define amdgpu_kernel void @memcpy_p0_p0_optsize(ptr %dest, ptr %src) #1 {
 ; CHECK-LABEL: memcpy_p0_p0_optsize:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v12, s3
 ; CHECK-NEXT:    v_mov_b32_e32 v11, s2
@@ -379,12 +387,12 @@ entry:
 define amdgpu_kernel void @memcpy_p5_p4_optsize(ptr addrspace(5) %local, ptr addrspace(4) %0) #1 {
 ; CHECK-LABEL: memcpy_p5_p4_optsize:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; CHECK-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; CHECK-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
 ; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x0
 ; CHECK-NEXT:    v_mov_b32_e32 v24, 0
-; CHECK-NEXT:    s_add_u32 s16, s16, s15
+; CHECK-NEXT:    s_add_u32 s20, s20, s17
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:112
 ; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1] offset:96
@@ -392,50 +400,50 @@ define amdgpu_kernel void @memcpy_p5_p4_optsize(ptr addrspace(5) %local, ptr add
 ; CHECK-NEXT:    global_load_dwordx4 v[12:15], v24, s[0:1] offset:64
 ; CHECK-NEXT:    global_load_dwordx4 v[16:19], v24, s[0:1] offset:48
 ; CHECK-NEXT:    global_load_dwordx4 v[20:23], v24, s[0:1] offset:32
-; CHECK-NEXT:    s_addc_u32 s17, s17, 0
+; CHECK-NEXT:    s_addc_u32 s21, s21, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v25, s2
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_dword v3, v25, s[16:19], 0 offen offset:124
-; CHECK-NEXT:    buffer_store_dword v2, v25, s[16:19], 0 offen offset:120
-; CHECK-NEXT:    buffer_store_dword v1, v25, s[16:19], 0 offen offset:116
-; CHECK-NEXT:    buffer_store_dword v0, v25, s[16:19], 0 offen offset:112
+; CHECK-NEXT:    buffer_store_dword v3, v25, s[20:23], 0 offen offset:124
+; CHECK-NEXT:    buffer_store_dword v2, v25, s[20:23], 0 offen offset:120
+; CHECK-NEXT:    buffer_store_dword v1, v25, s[20:23], 0 offen offset:116
+; CHECK-NEXT:    buffer_store_dword v0, v25, s[20:23], 0 offen offset:112
 ; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_dword v7, v25, s[16:19], 0 offen offset:108
-; CHECK-NEXT:    buffer_store_dword v6, v25, s[16:19], 0 offen offset:104
-; CHECK-NEXT:    buffer_store_dword v5, v25, s[16:19], 0 offen offset:100
-; CHECK-NEXT:    buffer_store_dword v4, v25, s[16:19], 0 offen offset:96
+; CHECK-NEXT:    buffer_store_dword v7, v25, s[20:23], 0 offen offset:108
+; CHECK-NEXT:    buffer_store_dword v6, v25, s[20:23], 0 offen offset:104
+; CHECK-NEXT:    buffer_store_dword v5, v25, s[20:23], 0 offen offset:100
+; CHECK-NEXT:    buffer_store_dword v4, v25, s[20:23], 0 offen offset:96
 ; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1]
 ; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_dword v11, v25, s[16:19], 0 offen offset:92
-; CHECK-NEXT:    buffer_store_dword v10, v25, s[16:19], 0 offen offset:88
-; CHECK-NEXT:    buffer_store_dword v9, v25, s[16:19], 0 offen offset:84
-; CHECK-NEXT:    buffer_store_dword v8, v25, s[16:19], 0 offen offset:80
+; CHECK-NEXT:    buffer_store_dword v11, v25, s[20:23], 0 offen offset:92
+; CHECK-NEXT:    buffer_store_dword v10, v25, s[20:23], 0 offen offset:88
+; CHECK-NEXT:    buffer_store_dword v9, v25, s[20:23], 0 offen offset:84
+; CHECK-NEXT:    buffer_store_dword v8, v25, s[20:23], 0 offen offset:80
 ; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_dword v15, v25, s[16:19], 0 offen offset:76
-; CHECK-NEXT:    buffer_store_dword v14, v25, s[16:19], 0 offen offset:72
-; CHECK-NEXT:    buffer_store_dword v13, v25, s[16:19], 0 offen offset:68
-; CHECK-NEXT:    buffer_store_dword v12, v25, s[16:19], 0 offen offset:64
+; CHECK-NEXT:    buffer_store_dword v15, v25, s[20:23], 0 offen offset:76
+; CHECK-NEXT:    buffer_store_dword v14, v25, s[20:23], 0 offen offset:72
+; CHECK-NEXT:    buffer_store_dword v13, v25, s[20:23], 0 offen offset:68
+; CHECK-NEXT:    buffer_store_dword v12, v25, s[20:23], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_dword v19, v25, s[16:19], 0 offen offset:60
-; CHECK-NEXT:    buffer_store_dword v18, v25, s[16:19], 0 offen offset:56
-; CHECK-NEXT:    buffer_store_dword v17, v25, s[16:19], 0 offen offset:52
-; CHECK-NEXT:    buffer_store_dword v16, v25, s[16:19], 0 offen offset:48
+; CHECK-NEXT:    buffer_store_dword v19, v25, s[20:23], 0 offen offset:60
+; CHECK-NEXT:    buffer_store_dword v18, v25, s[20:23], 0 offen offset:56
+; CHECK-NEXT:    buffer_store_dword v17, v25, s[20:23], 0 offen offset:52
+; CHECK-NEXT:    buffer_store_dword v16, v25, s[20:23], 0 offen offset:48
 ; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_dword v23, v25, s[16:19], 0 offen offset:44
-; CHECK-NEXT:    buffer_store_dword v22, v25, s[16:19], 0 offen offset:40
-; CHECK-NEXT:    buffer_store_dword v21, v25, s[16:19], 0 offen offset:36
-; CHECK-NEXT:    buffer_store_dword v20, v25, s[16:19], 0 offen offset:32
+; CHECK-NEXT:    buffer_store_dword v23, v25, s[20:23], 0 offen offset:44
+; CHECK-NEXT:    buffer_store_dword v22, v25, s[20:23], 0 offen offset:40
+; CHECK-NEXT:    buffer_store_dword v21, v25, s[20:23], 0 offen offset:36
+; CHECK-NEXT:    buffer_store_dword v20, v25, s[20:23], 0 offen offset:32
 ; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_dword v3, v25, s[16:19], 0 offen offset:28
-; CHECK-NEXT:    buffer_store_dword v2, v25, s[16:19], 0 offen offset:24
-; CHECK-NEXT:    buffer_store_dword v1, v25, s[16:19], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_dword v0, v25, s[16:19], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v3, v25, s[20:23], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v2, v25, s[20:23], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v1, v25, s[20:23], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v0, v25, s[20:23], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_dword v7, v25, s[16:19], 0 offen offset:12
-; CHECK-NEXT:    buffer_store_dword v6, v25, s[16:19], 0 offen offset:8
-; CHECK-NEXT:    buffer_store_dword v5, v25, s[16:19], 0 offen offset:4
-; CHECK-NEXT:    buffer_store_dword v4, v25, s[16:19], 0 offen
+; CHECK-NEXT:    buffer_store_dword v7, v25, s[20:23], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v6, v25, s[20:23], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v5, v25, s[20:23], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v4, v25, s[20:23], 0 offen
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %local, ptr addrspace(4) %0, i64 128, i1 false)
@@ -445,55 +453,57 @@ entry:
 define amdgpu_kernel void @memcpy_p0_p5_optsize(ptr %generic, ptr addrspace(5) %src) #1 {
 ; CHECK-LABEL: memcpy_p0_p5_optsize:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; CHECK-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; CHECK-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; CHECK-NEXT:    s_load_dword s0, s[8:9], 0x8
-; CHECK-NEXT:    s_add_u32 s16, s16, s15
-; CHECK-NEXT:    s_addc_u32 s17, s17, 0
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; CHECK-NEXT:    s_add_u32 s20, s20, s17
+; CHECK-NEXT:    s_addc_u32 s21, s21, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v26, s0
-; CHECK-NEXT:    buffer_load_dword v3, v26, s[16:19], 0 offen offset:124
-; CHECK-NEXT:    buffer_load_dword v2, v26, s[16:19], 0 offen offset:120
-; CHECK-NEXT:    buffer_load_dword v1, v26, s[16:19], 0 offen offset:116
-; CHECK-NEXT:    buffer_load_dword v0, v26, s[16:19], 0 offen offset:112
-; CHECK-NEXT:    buffer_load_dword v7, v26, s[16:19], 0 offen offset:108
-; CHECK-NEXT:    buffer_load_dword v6, v26, s[16:19], 0 offen offset:104
-; CHECK-NEXT:    buffer_load_dword v5, v26, s[16:19], 0 offen offset:100
-; CHECK-NEXT:    buffer_load_dword v4, v26, s[16:19], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen offset:96
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CHECK-NEXT:    buffer_load_dword v8, v26, s[16:19], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v9, v26, s[16:19], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v10, v26, s[16:19], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v11, v26, s[16:19], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v12, v26, s[16:19], 0 offen offset:32
-; CHECK-NEXT:    buffer_load_dword v13, v26, s[16:19], 0 offen offset:36
-; CHECK-NEXT:    buffer_load_dword v14, v26, s[16:19], 0 offen offset:40
-; CHECK-NEXT:    buffer_load_dword v15, v26, s[16:19], 0 offen offset:44
-; CHECK-NEXT:    buffer_load_dword v16, v26, s[16:19], 0 offen offset:48
-; CHECK-NEXT:    buffer_load_dword v17, v26, s[16:19], 0 offen offset:52
-; CHECK-NEXT:    buffer_load_dword v18, v26, s[16:19], 0 offen offset:56
-; CHECK-NEXT:    buffer_load_dword v19, v26, s[16:19], 0 offen offset:60
-; CHECK-NEXT:    buffer_load_dword v23, v26, s[16:19], 0 offen offset:92
-; CHECK-NEXT:    buffer_load_dword v22, v26, s[16:19], 0 offen offset:88
-; CHECK-NEXT:    buffer_load_dword v21, v26, s[16:19], 0 offen offset:84
-; CHECK-NEXT:    buffer_load_dword v20, v26, s[16:19], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v8, v26, s[20:23], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v9, v26, s[20:23], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v10, v26, s[20:23], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v11, v26, s[20:23], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v12, v26, s[20:23], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v13, v26, s[20:23], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v14, v26, s[20:23], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v15, v26, s[20:23], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v16, v26, s[20:23], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v17, v26, s[20:23], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v18, v26, s[20:23], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v19, v26, s[20:23], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v23, v26, s[20:23], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v22, v26, s[20:23], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v21, v26, s[20:23], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v20, v26, s[20:23], 0 offen offset:80
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v25, s1
 ; CHECK-NEXT:    v_mov_b32_e32 v24, s0
 ; CHECK-NEXT:    s_waitcnt vmcnt(20)
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:112
-; CHECK-NEXT:    buffer_load_dword v3, v26, s[16:19], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:76
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_dword v2, v26, s[16:19], 0 offen offset:72
-; CHECK-NEXT:    buffer_load_dword v1, v26, s[16:19], 0 offen offset:68
-; CHECK-NEXT:    buffer_load_dword v0, v26, s[16:19], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7] offset:96
-; CHECK-NEXT:    buffer_load_dword v4, v26, s[16:19], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_dword v5, v26, s[16:19], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v6, v26, s[16:19], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v7, v26, s[16:19], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:12
 ; CHECK-NEXT:    s_nop 0
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[20:23] offset:80
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:64
@@ -553,6 +563,8 @@ define amdgpu_kernel void @memcpy_p0_p3_optsize(ptr %generic) #1 {
 ; CHECK-NEXT:    ds_read2_b64 v[4:7], v16 offset0:2 offset1:3
 ; CHECK-NEXT:    ds_read2_b64 v[8:11], v16 offset0:4 offset1:5
 ; CHECK-NEXT:    ds_read2_b64 v[12:15], v16 offset0:6 offset1:7
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v21, s1
 ; CHECK-NEXT:    v_mov_b32_e32 v20, s0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
index 5af37809443e0..07ad8cb0c4a3d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
@@ -15,6 +15,9 @@
 define amdgpu_kernel void @flat_agent_unordered_load(
 ; GFX7-LABEL: flat_agent_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -29,6 +32,10 @@ define amdgpu_kernel void @flat_agent_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_agent_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -43,6 +50,10 @@ define amdgpu_kernel void @flat_agent_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_agent_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -71,6 +82,8 @@ define amdgpu_kernel void @flat_agent_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -83,6 +96,8 @@ define amdgpu_kernel void @flat_agent_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -182,6 +197,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_load(
 ; GFX7-LABEL: flat_agent_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -196,6 +214,10 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -210,6 +232,10 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -238,6 +264,8 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -250,6 +278,8 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -349,6 +379,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_load(
 ; GFX7-LABEL: flat_agent_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -364,6 +397,10 @@ define amdgpu_kernel void @flat_agent_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -380,6 +417,10 @@ define amdgpu_kernel void @flat_agent_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -410,6 +451,8 @@ define amdgpu_kernel void @flat_agent_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -423,6 +466,8 @@ define amdgpu_kernel void @flat_agent_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -531,6 +576,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_load(
 ; GFX7-LABEL: flat_agent_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -547,6 +595,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -565,6 +617,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -598,6 +654,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -612,6 +670,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -739,6 +799,9 @@ entry:
 define amdgpu_kernel void @flat_agent_unordered_store(
 ; GFX7-LABEL: flat_agent_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -750,6 +813,10 @@ define amdgpu_kernel void @flat_agent_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_agent_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -761,6 +828,10 @@ define amdgpu_kernel void @flat_agent_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_agent_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -783,6 +854,8 @@ define amdgpu_kernel void @flat_agent_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -793,6 +866,8 @@ define amdgpu_kernel void @flat_agent_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -873,6 +948,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_store(
 ; GFX7-LABEL: flat_agent_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -884,6 +962,10 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -895,6 +977,10 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -917,6 +1003,8 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -927,6 +1015,8 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1007,6 +1097,9 @@ entry:
 define amdgpu_kernel void @flat_agent_release_store(
 ; GFX7-LABEL: flat_agent_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1019,6 +1112,10 @@ define amdgpu_kernel void @flat_agent_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_agent_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1032,6 +1129,10 @@ define amdgpu_kernel void @flat_agent_release_store(
 ;
 ; GFX10-CU-LABEL: flat_agent_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1057,6 +1158,8 @@ define amdgpu_kernel void @flat_agent_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1068,6 +1171,8 @@ define amdgpu_kernel void @flat_agent_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1165,6 +1270,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_store(
 ; GFX7-LABEL: flat_agent_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1177,6 +1285,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1190,6 +1302,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1215,6 +1331,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1226,6 +1344,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1323,6 +1443,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_agent_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1334,6 +1457,10 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1345,6 +1472,10 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1367,6 +1498,8 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1377,6 +1510,8 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1457,6 +1592,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
 ; GFX7-LABEL: flat_agent_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1470,6 +1608,10 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1485,6 +1627,10 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1512,6 +1658,8 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1524,6 +1672,8 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1622,6 +1772,9 @@ entry:
 define amdgpu_kernel void @flat_agent_release_atomicrmw(
 ; GFX7-LABEL: flat_agent_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1634,6 +1787,10 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1647,6 +1804,10 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1672,6 +1833,8 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1683,6 +1846,8 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1780,6 +1945,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_agent_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1794,6 +1962,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1811,6 +1983,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1841,6 +2017,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1854,6 +2032,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1969,6 +2149,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_agent_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1983,6 +2166,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2000,6 +2187,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2030,6 +2221,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2043,6 +2236,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2158,6 +2353,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_agent_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2174,6 +2372,10 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2191,6 +2393,10 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2223,6 +2429,8 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2237,6 +2445,8 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2352,6 +2562,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_agent_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2369,6 +2582,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2388,6 +2605,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2423,6 +2644,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2438,6 +2661,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2574,6 +2799,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_agent_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2591,6 +2819,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2610,6 +2842,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2645,6 +2881,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2660,6 +2898,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2796,6 +3036,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2821,6 +3064,10 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2846,6 +3093,10 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2896,6 +3147,8 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2910,6 +3163,8 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3019,6 +3274,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3046,6 +3304,10 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3075,6 +3337,10 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3130,6 +3396,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3146,6 +3414,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3273,6 +3543,9 @@ entry:
 define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3299,6 +3572,10 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3326,6 +3603,10 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3379,6 +3660,8 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3394,6 +3677,8 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3520,6 +3805,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3548,6 +3836,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3579,6 +3871,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3637,6 +3933,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3654,6 +3952,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3798,6 +4098,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3826,6 +4129,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3857,6 +4164,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3915,6 +4226,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3932,6 +4245,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4076,6 +4391,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4103,6 +4421,10 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4132,6 +4454,10 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4187,6 +4513,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4203,6 +4531,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4330,6 +4660,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4357,6 +4690,10 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4386,6 +4723,10 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4441,6 +4782,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4457,6 +4800,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4584,6 +4929,9 @@ entry:
 define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4612,6 +4960,10 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4643,6 +4995,10 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4701,6 +5057,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4718,6 +5076,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4862,6 +5222,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4890,6 +5253,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4921,6 +5288,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4979,6 +5350,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4996,6 +5369,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5140,6 +5515,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5168,6 +5546,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5199,6 +5581,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5257,6 +5643,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5274,6 +5662,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5418,6 +5808,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5446,6 +5839,10 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5477,6 +5874,10 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5535,6 +5936,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5552,6 +5955,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5696,6 +6101,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5724,6 +6132,10 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5755,6 +6167,10 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5813,6 +6229,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5830,6 +6248,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5974,6 +6394,9 @@ entry:
 define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6002,6 +6425,10 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6033,6 +6460,10 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6091,6 +6522,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6108,6 +6541,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6252,6 +6687,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6280,6 +6718,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6311,6 +6753,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6369,6 +6815,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6386,6 +6834,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6530,6 +6980,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6558,6 +7011,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6589,6 +7046,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6647,6 +7108,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6664,6 +7127,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6808,6 +7273,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6837,6 +7305,10 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6866,6 +7338,10 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6924,6 +7400,8 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6941,6 +7419,8 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7077,6 +7557,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7107,6 +7590,10 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7138,6 +7625,10 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7198,6 +7689,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7216,6 +7709,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7361,6 +7856,9 @@ entry:
 define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7391,6 +7889,10 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7422,6 +7924,10 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7483,6 +7989,8 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7501,6 +8009,8 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7654,6 +8164,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7685,6 +8198,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7718,6 +8235,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7781,6 +8302,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7800,6 +8323,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7966,6 +8491,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7997,6 +8525,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8030,6 +8562,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8093,6 +8629,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8112,6 +8650,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8278,6 +8818,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8308,6 +8851,10 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8339,6 +8886,10 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8399,6 +8950,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8417,6 +8970,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8566,6 +9121,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8596,6 +9154,10 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8627,6 +9189,10 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8687,6 +9253,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8705,6 +9273,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8850,6 +9420,9 @@ entry:
 define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8881,6 +9454,10 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8914,6 +9491,10 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8977,6 +9558,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8996,6 +9579,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9162,6 +9747,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9193,6 +9781,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9226,6 +9818,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9289,6 +9885,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9308,6 +9906,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9474,6 +10074,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9505,6 +10108,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9538,6 +10145,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9601,6 +10212,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9620,6 +10233,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9786,6 +10401,9 @@ entry:
 define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9817,6 +10435,10 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9850,6 +10472,10 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9913,6 +10539,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9932,6 +10560,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10098,6 +10728,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10129,6 +10762,10 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10162,6 +10799,10 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10225,6 +10866,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10244,6 +10887,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10406,6 +11051,9 @@ entry:
 define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10437,6 +11085,10 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10470,6 +11122,10 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10533,6 +11189,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10552,6 +11210,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10718,6 +11378,9 @@ entry:
 define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10749,6 +11412,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10782,6 +11449,10 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10845,6 +11516,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10864,6 +11537,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -11030,6 +11705,9 @@ entry:
 define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -11061,6 +11739,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -11094,6 +11776,10 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -11157,6 +11843,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -11176,6 +11864,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -11342,6 +12032,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_unordered_load(
 ; GFX7-LABEL: flat_agent_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11356,6 +12049,10 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11370,6 +12067,10 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11398,6 +12099,8 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11410,6 +12113,8 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11509,6 +12214,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11523,6 +12231,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11537,6 +12249,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11565,6 +12281,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11577,6 +12295,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11676,6 +12396,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_load(
 ; GFX7-LABEL: flat_agent_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11692,6 +12415,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11709,6 +12436,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11741,6 +12472,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11755,6 +12488,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11868,6 +12603,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11885,6 +12623,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11904,6 +12646,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11939,6 +12685,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11954,6 +12702,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12086,6 +12836,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_unordered_store(
 ; GFX7-LABEL: flat_agent_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12097,6 +12850,10 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12108,6 +12865,10 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12130,6 +12891,8 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12140,6 +12903,8 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12220,6 +12985,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12231,6 +12999,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12242,6 +13014,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12264,6 +13040,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12274,6 +13052,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12354,6 +13134,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_release_store(
 ; GFX7-LABEL: flat_agent_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12366,6 +13149,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12379,6 +13166,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12404,6 +13195,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12415,6 +13208,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12512,6 +13307,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12524,6 +13322,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12537,6 +13339,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12562,6 +13368,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12573,6 +13381,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12670,6 +13480,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12681,6 +13494,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12692,6 +13509,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12714,6 +13535,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12724,6 +13547,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12804,6 +13629,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
 ; GFX7-LABEL: flat_agent_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12817,6 +13645,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12831,6 +13663,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12857,6 +13693,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12869,6 +13707,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12965,6 +13805,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
 ; GFX7-LABEL: flat_agent_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12977,6 +13820,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12990,6 +13837,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13015,6 +13866,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13026,6 +13879,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13123,6 +13978,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13137,6 +13995,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13153,6 +14015,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13182,6 +14048,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13195,6 +14063,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13308,6 +14178,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13322,6 +14195,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13338,6 +14215,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13367,6 +14248,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13380,6 +14263,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13493,6 +14378,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13510,6 +14398,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13528,6 +14420,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13562,6 +14458,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13577,6 +14475,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13697,6 +14597,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13715,6 +14618,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13735,6 +14642,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13772,6 +14683,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13788,6 +14701,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13929,6 +14844,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13947,6 +14865,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13967,6 +14889,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14004,6 +14930,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14020,6 +14948,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14161,6 +15091,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14186,6 +15119,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14211,6 +15148,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14261,6 +15202,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14275,6 +15218,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14384,6 +15329,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14411,6 +15359,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14439,6 +15391,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14493,6 +15449,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14509,6 +15467,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14634,6 +15594,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14660,6 +15623,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14687,6 +15654,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14740,6 +15711,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14755,6 +15728,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14881,6 +15856,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14909,6 +15887,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14939,6 +15921,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14996,6 +15982,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15013,6 +16001,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15155,6 +16145,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15183,6 +16176,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15213,6 +16210,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15270,6 +16271,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15287,6 +16290,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15429,6 +16434,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15456,6 +16464,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15484,6 +16496,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15538,6 +16554,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15554,6 +16572,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15679,6 +16699,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15706,6 +16729,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15734,6 +16761,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15788,6 +16819,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15804,6 +16837,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15929,6 +16964,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15957,6 +16995,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15987,6 +17029,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16044,6 +17090,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16061,6 +17109,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16203,6 +17253,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16231,6 +17284,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16261,6 +17318,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16318,6 +17379,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16335,6 +17398,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16477,6 +17542,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16505,6 +17573,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16535,6 +17607,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16592,6 +17668,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16609,6 +17687,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16751,6 +17831,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16779,6 +17862,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16809,6 +17896,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16866,6 +17957,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16883,6 +17976,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17025,6 +18120,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17053,6 +18151,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17083,6 +18185,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17140,6 +18246,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17157,6 +18265,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17299,6 +18409,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17327,6 +18440,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17357,6 +18474,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17414,6 +18535,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17431,6 +18554,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17573,6 +18698,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17601,6 +18729,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17631,6 +18763,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17688,6 +18824,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17705,6 +18843,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17847,6 +18987,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17875,6 +19018,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17905,6 +19052,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17962,6 +19113,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17979,6 +19132,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18121,6 +19276,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18150,6 +19308,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18179,6 +19341,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18237,6 +19403,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18254,6 +19422,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18390,6 +19560,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18421,6 +19594,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18453,6 +19630,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18515,6 +19696,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18534,6 +19717,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18684,6 +19869,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18714,6 +19902,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18745,6 +19937,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18806,6 +20002,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18824,6 +20022,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18977,6 +20177,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19009,6 +20212,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19043,6 +20250,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19108,6 +20319,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19128,6 +20341,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19299,6 +20514,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19331,6 +20549,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19365,6 +20587,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19430,6 +20656,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19450,6 +20678,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19621,6 +20851,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19652,6 +20885,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19684,6 +20921,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19746,6 +20987,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19765,6 +21008,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19919,6 +21164,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19950,6 +21198,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19982,6 +21234,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20044,6 +21300,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20063,6 +21321,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20213,6 +21473,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20245,6 +21508,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20279,6 +21546,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20344,6 +21615,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20364,6 +21637,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20535,6 +21810,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20567,6 +21845,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20601,6 +21883,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20666,6 +21952,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20686,6 +21974,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20857,6 +22147,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20889,6 +22182,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20923,6 +22220,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20988,6 +22289,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21008,6 +22311,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21179,6 +22484,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21211,6 +22519,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21245,6 +22557,10 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21310,6 +22626,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21330,6 +22648,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21501,6 +22821,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21533,6 +22856,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21567,6 +22894,10 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21632,6 +22963,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21652,6 +22985,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21819,6 +23154,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21851,6 +23189,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21885,6 +23227,10 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21950,6 +23296,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21970,6 +23318,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22141,6 +23491,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -22173,6 +23526,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22207,6 +23564,10 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22272,6 +23633,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22292,6 +23655,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22463,6 +23828,9 @@ entry:
 define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -22495,6 +23863,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22529,6 +23901,10 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22594,6 +23970,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22614,6 +23992,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index 30c0a322d7ddc..3c24c36ec547d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -15,6 +15,9 @@
 define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX7-LABEL: flat_nontemporal_load_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -29,6 +32,10 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_load_0:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -43,6 +50,10 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_load_0:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -71,6 +82,8 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_nontemporal_load_0:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -83,6 +96,8 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_nontemporal_load_0:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -182,6 +197,9 @@ entry:
 define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX7-LABEL: flat_nontemporal_load_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
@@ -211,6 +229,10 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_load_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_nop 0
@@ -240,6 +262,10 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_load_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_nop 0
@@ -298,6 +324,8 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_nontemporal_load_1:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_nop 0
@@ -329,6 +357,8 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_nontemporal_load_1:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_nop 0
@@ -537,6 +567,9 @@ entry:
 define amdgpu_kernel void @flat_nontemporal_store_0(
 ; GFX7-LABEL: flat_nontemporal_store_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -551,6 +584,10 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_store_0:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -565,6 +602,10 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_store_0:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -593,6 +634,8 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_nontemporal_store_0:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -605,6 +648,8 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_nontemporal_store_0:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -704,6 +749,9 @@ entry:
 define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX7-LABEL: flat_nontemporal_store_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -732,6 +780,10 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_store_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -759,6 +811,10 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_store_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -814,6 +870,8 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_nontemporal_store_1:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -843,6 +901,8 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_nontemporal_store_1:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1047,6 +1107,9 @@ entry:
 define amdgpu_kernel void @flat_nontemporal_volatile_load(
 ; GFX7-LABEL: flat_nontemporal_volatile_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1062,6 +1125,10 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_volatile_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1077,6 +1144,10 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_volatile_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1107,6 +1178,8 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_nontemporal_volatile_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1120,6 +1193,8 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_nontemporal_volatile_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
index b80dfaea01653..b88a10ab24a98 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
@@ -15,6 +15,9 @@
 define amdgpu_kernel void @flat_singlethread_unordered_load(
 ; GFX7-LABEL: flat_singlethread_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -29,6 +32,10 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -43,6 +50,10 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -71,6 +82,8 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -83,6 +96,8 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -182,6 +197,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_load(
 ; GFX7-LABEL: flat_singlethread_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -196,6 +214,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -210,6 +232,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -238,6 +264,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -250,6 +278,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -349,6 +379,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_load(
 ; GFX7-LABEL: flat_singlethread_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -363,6 +396,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -377,6 +414,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -405,6 +446,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -417,6 +460,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -516,6 +561,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_load(
 ; GFX7-LABEL: flat_singlethread_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -530,6 +578,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -544,6 +596,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -572,6 +628,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -584,6 +642,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -683,6 +743,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_unordered_store(
 ; GFX7-LABEL: flat_singlethread_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -694,6 +757,10 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -705,6 +772,10 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -727,6 +798,8 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -737,6 +810,8 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -817,6 +892,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_store(
 ; GFX7-LABEL: flat_singlethread_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -828,6 +906,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -839,6 +921,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -861,6 +947,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -871,6 +959,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -951,6 +1041,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_release_store(
 ; GFX7-LABEL: flat_singlethread_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -962,6 +1055,10 @@ define amdgpu_kernel void @flat_singlethread_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -973,6 +1070,10 @@ define amdgpu_kernel void @flat_singlethread_release_store(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -995,6 +1096,8 @@ define amdgpu_kernel void @flat_singlethread_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1005,6 +1108,8 @@ define amdgpu_kernel void @flat_singlethread_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1085,6 +1190,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_store(
 ; GFX7-LABEL: flat_singlethread_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1096,6 +1204,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1107,6 +1219,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1129,6 +1245,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1139,6 +1257,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1219,6 +1339,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1230,6 +1353,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1241,6 +1368,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1263,6 +1394,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1273,6 +1406,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1353,6 +1488,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1364,6 +1502,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1375,6 +1517,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1397,6 +1543,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1407,6 +1555,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1487,6 +1637,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1498,6 +1651,10 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1509,6 +1666,10 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1531,6 +1692,8 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1541,6 +1704,8 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1621,6 +1786,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1632,6 +1800,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1643,6 +1815,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1665,6 +1841,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1675,6 +1853,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1755,6 +1935,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1766,6 +1949,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1777,6 +1964,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1799,6 +1990,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1809,6 +2002,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1889,6 +2084,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1904,6 +2102,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1919,6 +2121,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1949,6 +2155,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1962,6 +2170,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2068,6 +2278,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2083,6 +2296,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2098,6 +2315,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2128,6 +2349,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2141,6 +2364,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2247,6 +2472,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2262,6 +2490,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2277,6 +2509,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2307,6 +2543,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2320,6 +2558,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2426,6 +2666,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2451,6 +2694,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2476,6 +2723,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2526,6 +2777,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2540,6 +2793,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2649,6 +2904,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2674,6 +2932,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2699,6 +2961,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2749,6 +3015,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2763,6 +3031,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2872,6 +3142,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2897,6 +3170,10 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2922,6 +3199,10 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2972,6 +3253,8 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2986,6 +3269,8 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3095,6 +3380,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3120,6 +3408,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3145,6 +3437,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3195,6 +3491,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3209,6 +3507,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3318,6 +3618,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3343,6 +3646,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3368,6 +3675,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3418,6 +3729,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3432,6 +3745,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3541,6 +3856,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3566,6 +3884,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3591,6 +3913,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3641,6 +3967,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3655,6 +3983,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3764,6 +4094,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3789,6 +4122,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3814,6 +4151,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3864,6 +4205,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3878,6 +4221,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3987,6 +4332,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4012,6 +4360,10 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4037,6 +4389,10 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4087,6 +4443,8 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4101,6 +4459,8 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4210,6 +4570,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4235,6 +4598,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4260,6 +4627,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4310,6 +4681,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4324,6 +4697,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4433,6 +4808,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4458,6 +4836,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4483,6 +4865,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4533,6 +4919,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4547,6 +4935,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4656,6 +5046,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4681,6 +5074,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4706,6 +5103,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4756,6 +5157,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4770,6 +5173,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4879,6 +5284,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4904,6 +5312,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4929,6 +5341,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4979,6 +5395,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4993,6 +5411,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5102,6 +5522,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5127,6 +5550,10 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5152,6 +5579,10 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5202,6 +5633,8 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5216,6 +5649,8 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5325,6 +5760,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5350,6 +5788,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5375,6 +5817,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5425,6 +5871,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5439,6 +5887,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5548,6 +5998,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5573,6 +6026,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5598,6 +6055,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5648,6 +6109,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5662,6 +6125,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5771,6 +6236,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -5800,6 +6268,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -5829,6 +6301,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -5887,6 +6363,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5904,6 +6382,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6040,6 +6520,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6069,6 +6552,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6098,6 +6585,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6156,6 +6647,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6173,6 +6666,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6309,6 +6804,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6338,6 +6836,10 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6367,6 +6869,10 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6425,6 +6931,8 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6442,6 +6950,8 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6578,6 +7088,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6607,6 +7120,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6636,6 +7153,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6694,6 +7215,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6711,6 +7234,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6847,6 +7372,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6876,6 +7404,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6905,6 +7437,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6963,6 +7499,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6980,6 +7518,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7116,6 +7656,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7145,6 +7688,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7174,6 +7721,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7232,6 +7783,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7249,6 +7802,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7385,6 +7940,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7414,6 +7972,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7443,6 +8005,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7501,6 +8067,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7518,6 +8086,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7654,6 +8224,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7683,6 +8256,10 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7712,6 +8289,10 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7770,6 +8351,8 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7787,6 +8370,8 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7923,6 +8508,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7952,6 +8540,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7981,6 +8573,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8039,6 +8635,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8056,6 +8654,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8192,6 +8792,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8221,6 +8824,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8250,6 +8857,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8308,6 +8919,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8325,6 +8938,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8461,6 +9076,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8490,6 +9108,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8519,6 +9141,10 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8577,6 +9203,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8594,6 +9222,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8730,6 +9360,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8759,6 +9392,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8788,6 +9425,10 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8846,6 +9487,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8863,6 +9506,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8999,6 +9644,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9028,6 +9676,10 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9057,6 +9709,10 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9115,6 +9771,8 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9132,6 +9790,8 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9268,6 +9928,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9297,6 +9960,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9326,6 +9993,10 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9384,6 +10055,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9401,6 +10074,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9537,6 +10212,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9566,6 +10244,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9595,6 +10277,10 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9653,6 +10339,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9670,6 +10358,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9806,6 +10496,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
 ; GFX7-LABEL: flat_singlethread_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9820,6 +10513,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9834,6 +10531,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9862,6 +10563,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9874,6 +10577,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9973,6 +10678,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9987,6 +10695,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10001,6 +10713,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10029,6 +10745,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10041,6 +10759,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10140,6 +10860,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10154,6 +10877,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10168,6 +10895,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10196,6 +10927,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10208,6 +10941,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10307,6 +11042,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10321,6 +11059,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10335,6 +11077,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10363,6 +11109,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10375,6 +11123,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10474,6 +11224,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
 ; GFX7-LABEL: flat_singlethread_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10485,6 +11238,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10496,6 +11253,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10518,6 +11279,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10528,6 +11291,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10608,6 +11373,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10619,6 +11387,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10630,6 +11402,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10652,6 +11428,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10662,6 +11440,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10742,6 +11522,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_release_store(
 ; GFX7-LABEL: flat_singlethread_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10753,6 +11536,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10764,6 +11551,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10786,6 +11577,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10796,6 +11589,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10876,6 +11671,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10887,6 +11685,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10898,6 +11700,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10920,6 +11726,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10930,6 +11738,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11010,6 +11820,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11021,6 +11834,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11032,6 +11849,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11054,6 +11875,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11064,6 +11887,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11144,6 +11969,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11155,6 +11983,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11166,6 +11998,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11188,6 +12024,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11198,6 +12036,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11278,6 +12118,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11289,6 +12132,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11300,6 +12147,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11322,6 +12173,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11332,6 +12185,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11412,6 +12267,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11423,6 +12281,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11434,6 +12296,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11456,6 +12322,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11466,6 +12334,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11546,6 +12416,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11557,6 +12430,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11568,6 +12445,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11590,6 +12471,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11600,6 +12483,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11680,6 +12565,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11695,6 +12583,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11710,6 +12602,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11740,6 +12636,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11753,6 +12651,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11859,6 +12759,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11874,6 +12777,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11889,6 +12796,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11919,6 +12830,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11932,6 +12845,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12038,6 +12953,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12053,6 +12971,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12068,6 +12990,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12098,6 +13024,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12111,6 +13039,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12217,6 +13147,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12242,6 +13175,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12267,6 +13204,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12317,6 +13258,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12331,6 +13274,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12440,6 +13385,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12465,6 +13413,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12490,6 +13442,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12540,6 +13496,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12554,6 +13512,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12663,6 +13623,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12688,6 +13651,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12713,6 +13680,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12763,6 +13734,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12777,6 +13750,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12886,6 +13861,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12911,6 +13889,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12936,6 +13918,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12986,6 +13972,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13000,6 +13988,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13109,6 +14099,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13134,6 +14127,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13159,6 +14156,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13209,6 +14210,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13223,6 +14226,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13332,6 +14337,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13357,6 +14365,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13382,6 +14394,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13432,6 +14448,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13446,6 +14464,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13555,6 +14575,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13580,6 +14603,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13605,6 +14632,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13655,6 +14686,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13669,6 +14702,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13778,6 +14813,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13803,6 +14841,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13828,6 +14870,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13878,6 +14924,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13892,6 +14940,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14001,6 +15051,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14026,6 +15079,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14051,6 +15108,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14101,6 +15162,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14115,6 +15178,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14224,6 +15289,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14249,6 +15317,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14274,6 +15346,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14324,6 +15400,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14338,6 +15416,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14447,6 +15527,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14472,6 +15555,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14497,6 +15584,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14547,6 +15638,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14561,6 +15654,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14670,6 +15765,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14695,6 +15793,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14720,6 +15822,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14770,6 +15876,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14784,6 +15892,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14893,6 +16003,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14918,6 +16031,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14943,6 +16060,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14993,6 +16114,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15007,6 +16130,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15116,6 +16241,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15141,6 +16269,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15166,6 +16298,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15216,6 +16352,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15230,6 +16368,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15339,6 +16479,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15364,6 +16507,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15389,6 +16536,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15439,6 +16590,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15453,6 +16606,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15562,6 +16717,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -15591,6 +16749,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -15620,6 +16782,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -15678,6 +16844,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15695,6 +16863,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15831,6 +17001,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -15860,6 +17033,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -15889,6 +17066,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -15947,6 +17128,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15964,6 +17147,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16100,6 +17285,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16129,6 +17317,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16158,6 +17350,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16216,6 +17412,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16233,6 +17431,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16369,6 +17569,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16398,6 +17601,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16427,6 +17634,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16485,6 +17696,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16502,6 +17715,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16638,6 +17853,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16667,6 +17885,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16696,6 +17918,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16754,6 +17980,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16771,6 +17999,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16907,6 +18137,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16936,6 +18169,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16965,6 +18202,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17023,6 +18264,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17040,6 +18283,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17176,6 +18421,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17205,6 +18453,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17234,6 +18486,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17292,6 +18548,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17309,6 +18567,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17445,6 +18705,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17474,6 +18737,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17503,6 +18770,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17561,6 +18832,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17578,6 +18851,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17714,6 +18989,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17743,6 +19021,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17772,6 +19054,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17830,6 +19116,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17847,6 +19135,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17983,6 +19273,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18012,6 +19305,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18041,6 +19338,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18099,6 +19400,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18116,6 +19419,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18252,6 +19557,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18281,6 +19589,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18310,6 +19622,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18368,6 +19684,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18385,6 +19703,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18521,6 +19841,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18550,6 +19873,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18579,6 +19906,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18637,6 +19968,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18654,6 +19987,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18790,6 +20125,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18819,6 +20157,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18848,6 +20190,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18906,6 +20252,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18923,6 +20271,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19059,6 +20409,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19088,6 +20441,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19117,6 +20474,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19175,6 +20536,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19192,6 +20555,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19328,6 +20693,9 @@ entry:
 define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19357,6 +20725,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19386,6 +20758,10 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19444,6 +20820,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19461,6 +20839,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
index 1ec942ea5f47b..919fc3e8f4e4f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
@@ -15,6 +15,9 @@
 define amdgpu_kernel void @flat_system_unordered_load(
 ; GFX7-LABEL: flat_system_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -29,6 +32,10 @@ define amdgpu_kernel void @flat_system_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_system_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -43,6 +50,10 @@ define amdgpu_kernel void @flat_system_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_system_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -71,6 +82,8 @@ define amdgpu_kernel void @flat_system_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -83,6 +96,8 @@ define amdgpu_kernel void @flat_system_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -182,6 +197,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_load(
 ; GFX7-LABEL: flat_system_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -196,6 +214,10 @@ define amdgpu_kernel void @flat_system_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -210,6 +232,10 @@ define amdgpu_kernel void @flat_system_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -238,6 +264,8 @@ define amdgpu_kernel void @flat_system_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -250,6 +278,8 @@ define amdgpu_kernel void @flat_system_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -349,6 +379,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_load(
 ; GFX7-LABEL: flat_system_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -364,6 +397,10 @@ define amdgpu_kernel void @flat_system_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -380,6 +417,10 @@ define amdgpu_kernel void @flat_system_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -410,6 +451,8 @@ define amdgpu_kernel void @flat_system_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -424,6 +467,8 @@ define amdgpu_kernel void @flat_system_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -533,6 +578,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_load(
 ; GFX7-LABEL: flat_system_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -549,6 +597,10 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -567,6 +619,10 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -600,6 +656,8 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -615,6 +673,8 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -743,6 +803,9 @@ entry:
 define amdgpu_kernel void @flat_system_unordered_store(
 ; GFX7-LABEL: flat_system_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -754,6 +817,10 @@ define amdgpu_kernel void @flat_system_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_system_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -765,6 +832,10 @@ define amdgpu_kernel void @flat_system_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_system_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -787,6 +858,8 @@ define amdgpu_kernel void @flat_system_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -797,6 +870,8 @@ define amdgpu_kernel void @flat_system_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -877,6 +952,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_store(
 ; GFX7-LABEL: flat_system_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -888,6 +966,10 @@ define amdgpu_kernel void @flat_system_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -899,6 +981,10 @@ define amdgpu_kernel void @flat_system_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -921,6 +1007,8 @@ define amdgpu_kernel void @flat_system_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -931,6 +1019,8 @@ define amdgpu_kernel void @flat_system_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1011,6 +1101,9 @@ entry:
 define amdgpu_kernel void @flat_system_release_store(
 ; GFX7-LABEL: flat_system_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1023,6 +1116,10 @@ define amdgpu_kernel void @flat_system_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_system_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1036,6 +1133,10 @@ define amdgpu_kernel void @flat_system_release_store(
 ;
 ; GFX10-CU-LABEL: flat_system_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1061,6 +1162,8 @@ define amdgpu_kernel void @flat_system_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1073,6 +1176,8 @@ define amdgpu_kernel void @flat_system_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1173,6 +1278,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_store(
 ; GFX7-LABEL: flat_system_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1185,6 +1293,10 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1198,6 +1310,10 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1223,6 +1339,8 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1235,6 +1353,8 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1335,6 +1455,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_system_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1346,6 +1469,10 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1357,6 +1484,10 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1379,6 +1510,8 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1389,6 +1522,8 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1469,6 +1604,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_atomicrmw(
 ; GFX7-LABEL: flat_system_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1482,6 +1620,10 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1497,6 +1639,10 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1524,6 +1670,8 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1537,6 +1685,8 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1636,6 +1786,9 @@ entry:
 define amdgpu_kernel void @flat_system_release_atomicrmw(
 ; GFX7-LABEL: flat_system_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1648,6 +1801,10 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1661,6 +1818,10 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1686,6 +1847,8 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1698,6 +1861,8 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1798,6 +1963,9 @@ entry:
 define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_system_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1812,6 +1980,10 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1829,6 +2001,10 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1859,6 +2035,8 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1874,6 +2052,8 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1993,6 +2173,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_system_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2007,6 +2190,10 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2024,6 +2211,10 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2054,6 +2245,8 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2069,6 +2262,8 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2188,6 +2383,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_system_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2204,6 +2402,10 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2221,6 +2423,10 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2253,6 +2459,8 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2268,6 +2476,8 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2384,6 +2594,9 @@ entry:
 define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_system_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2401,6 +2614,10 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2420,6 +2637,10 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2455,6 +2676,8 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2472,6 +2695,8 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2612,6 +2837,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_system_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2629,6 +2857,10 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2648,6 +2880,10 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2683,6 +2919,8 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2700,6 +2938,8 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2840,6 +3080,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2865,6 +3108,10 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2890,6 +3137,10 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2940,6 +3191,8 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2954,6 +3207,8 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3063,6 +3318,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3090,6 +3348,10 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3119,6 +3381,10 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3174,6 +3440,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3191,6 +3459,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3319,6 +3589,9 @@ entry:
 define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3345,6 +3618,10 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3372,6 +3649,10 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3425,6 +3706,8 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3441,6 +3724,8 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3570,6 +3855,9 @@ entry:
 define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3598,6 +3886,10 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3629,6 +3921,10 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3687,6 +3983,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3706,6 +4004,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3854,6 +4154,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3882,6 +4185,10 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3913,6 +4220,10 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3971,6 +4282,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3990,6 +4303,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4138,6 +4453,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4165,6 +4483,10 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4194,6 +4516,10 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4249,6 +4575,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4266,6 +4594,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4394,6 +4724,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4421,6 +4754,10 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4450,6 +4787,10 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4505,6 +4846,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4522,6 +4865,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4650,6 +4995,9 @@ entry:
 define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4678,6 +5026,10 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4709,6 +5061,10 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4767,6 +5123,8 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4786,6 +5144,8 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4934,6 +5294,9 @@ entry:
 define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4962,6 +5325,10 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4993,6 +5360,10 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5051,6 +5422,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5070,6 +5443,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5218,6 +5593,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5246,6 +5624,10 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5277,6 +5659,10 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5335,6 +5721,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5354,6 +5742,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5502,6 +5892,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5530,6 +5923,10 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5561,6 +5958,10 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5619,6 +6020,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5638,6 +6041,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5786,6 +6191,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5814,6 +6222,10 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5845,6 +6257,10 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5903,6 +6319,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5922,6 +6340,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6070,6 +6490,9 @@ entry:
 define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6098,6 +6521,10 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6129,6 +6556,10 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6187,6 +6618,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6206,6 +6639,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6354,6 +6789,9 @@ entry:
 define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6382,6 +6820,10 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6413,6 +6855,10 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6471,6 +6917,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6490,6 +6938,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6638,6 +7088,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6666,6 +7119,10 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6697,6 +7154,10 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -6755,6 +7216,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6774,6 +7237,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6922,6 +7387,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6951,6 +7419,10 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6980,6 +7452,10 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7038,6 +7514,8 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7055,6 +7533,8 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7191,6 +7671,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7221,6 +7704,10 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7252,6 +7739,10 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7312,6 +7803,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7331,6 +7824,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7477,6 +7972,9 @@ entry:
 define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7507,6 +8005,10 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7538,6 +8040,10 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7599,6 +8105,8 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7618,6 +8126,8 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7774,6 +8284,9 @@ entry:
 define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7805,6 +8318,10 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7838,6 +8355,10 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7901,6 +8422,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7922,6 +8445,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8092,6 +8617,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8123,6 +8651,10 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8156,6 +8688,10 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8219,6 +8755,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8240,6 +8778,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8410,6 +8950,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8440,6 +8983,10 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8471,6 +9018,10 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8531,6 +9082,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8550,6 +9103,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8700,6 +9255,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8730,6 +9288,10 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8761,6 +9323,10 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8821,6 +9387,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8840,6 +9408,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8986,6 +9556,9 @@ entry:
 define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9017,6 +9590,10 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9050,6 +9627,10 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9113,6 +9694,8 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9134,6 +9717,8 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9304,6 +9889,9 @@ entry:
 define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9335,6 +9923,10 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9368,6 +9960,10 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9431,6 +10027,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9452,6 +10050,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9622,6 +10222,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9653,6 +10256,10 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9686,6 +10293,10 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9749,6 +10360,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9770,6 +10383,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9940,6 +10555,9 @@ entry:
 define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9971,6 +10589,10 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10004,6 +10626,10 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10067,6 +10693,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10088,6 +10716,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10258,6 +10888,9 @@ entry:
 define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10289,6 +10922,10 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10322,6 +10959,10 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10385,6 +11026,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10406,6 +11049,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10572,6 +11217,9 @@ entry:
 define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10603,6 +11251,10 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10636,6 +11288,10 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10699,6 +11355,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10720,6 +11378,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -10890,6 +11550,9 @@ entry:
 define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10921,6 +11584,10 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -10954,6 +11621,10 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -11017,6 +11688,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -11038,6 +11711,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -11208,6 +11883,9 @@ entry:
 define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -11239,6 +11917,10 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -11272,6 +11954,10 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -11335,6 +12021,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -11356,6 +12044,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -11526,6 +12216,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_unordered_load(
 ; GFX7-LABEL: flat_system_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11540,6 +12233,10 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11554,6 +12251,10 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11582,6 +12283,8 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11594,6 +12297,8 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11693,6 +12398,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_load(
 ; GFX7-LABEL: flat_system_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11707,6 +12415,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11721,6 +12433,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11749,6 +12465,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11761,6 +12479,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11860,6 +12580,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_load(
 ; GFX7-LABEL: flat_system_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11876,6 +12599,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11893,6 +12620,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11925,6 +12656,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11940,6 +12673,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12054,6 +12789,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12071,6 +12809,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12090,6 +12832,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12125,6 +12871,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12141,6 +12889,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12274,6 +13024,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_unordered_store(
 ; GFX7-LABEL: flat_system_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12285,6 +13038,10 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12296,6 +13053,10 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12318,6 +13079,8 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12328,6 +13091,8 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12408,6 +13173,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_store(
 ; GFX7-LABEL: flat_system_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12419,6 +13187,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12430,6 +13202,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12452,6 +13228,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12462,6 +13240,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12542,6 +13322,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_release_store(
 ; GFX7-LABEL: flat_system_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12554,6 +13337,10 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12567,6 +13354,10 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12592,6 +13383,8 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12604,6 +13397,8 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12704,6 +13499,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12716,6 +13514,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12729,6 +13531,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12754,6 +13560,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12766,6 +13574,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12866,6 +13676,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_system_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12877,6 +13690,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12888,6 +13705,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12910,6 +13731,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12920,6 +13743,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13000,6 +13825,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
 ; GFX7-LABEL: flat_system_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13013,6 +13841,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13027,6 +13859,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13053,6 +13889,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13066,6 +13904,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13163,6 +14003,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
 ; GFX7-LABEL: flat_system_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13175,6 +14018,10 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13188,6 +14035,10 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13213,6 +14064,8 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13225,6 +14078,8 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13325,6 +14180,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_system_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13339,6 +14197,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13355,6 +14217,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13384,6 +14250,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13399,6 +14267,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13516,6 +14386,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13530,6 +14403,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13546,6 +14423,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13575,6 +14456,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13590,6 +14473,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13707,6 +14592,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13724,6 +14612,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13742,6 +14634,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13776,6 +14672,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13792,6 +14690,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13913,6 +14813,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13931,6 +14834,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13951,6 +14858,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13988,6 +14899,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14006,6 +14919,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14151,6 +15066,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14169,6 +15087,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14189,6 +15111,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14226,6 +15152,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14244,6 +15172,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14389,6 +15319,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14414,6 +15347,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14439,6 +15376,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14489,6 +15430,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14503,6 +15446,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14612,6 +15557,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14639,6 +15587,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14667,6 +15619,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14721,6 +15677,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14738,6 +15696,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14864,6 +15824,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14890,6 +15853,10 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14917,6 +15884,10 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14970,6 +15941,8 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14986,6 +15959,8 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15115,6 +16090,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15143,6 +16121,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15173,6 +16155,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15230,6 +16216,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15249,6 +16237,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15395,6 +16385,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15423,6 +16416,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15453,6 +16450,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15510,6 +16511,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15529,6 +16532,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15675,6 +16680,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15702,6 +16710,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15730,6 +16742,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15784,6 +16800,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15801,6 +16819,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15927,6 +16947,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15954,6 +16977,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15982,6 +17009,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16036,6 +17067,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16053,6 +17086,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16179,6 +17214,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16207,6 +17245,10 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16237,6 +17279,10 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16294,6 +17340,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16313,6 +17361,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16459,6 +17509,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16487,6 +17540,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16517,6 +17574,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16574,6 +17635,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16593,6 +17656,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16739,6 +17804,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16767,6 +17835,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16797,6 +17869,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -16854,6 +17930,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16873,6 +17951,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17019,6 +18099,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17047,6 +18130,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17077,6 +18164,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17134,6 +18225,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17153,6 +18246,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17299,6 +18394,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17327,6 +18425,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17357,6 +18459,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17414,6 +18520,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17433,6 +18541,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17579,6 +18689,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17607,6 +18720,10 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17637,6 +18754,10 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17694,6 +18815,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17713,6 +18836,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17859,6 +18984,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17887,6 +19015,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17917,6 +19049,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -17974,6 +19110,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17993,6 +19131,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18139,6 +19279,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -18167,6 +19310,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -18197,6 +19344,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -18254,6 +19405,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18273,6 +19426,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18419,6 +19574,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18448,6 +19606,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18477,6 +19639,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18535,6 +19701,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18552,6 +19720,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18688,6 +19858,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18719,6 +19892,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18751,6 +19928,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18813,6 +19994,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18833,6 +20016,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18984,6 +20169,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19014,6 +20202,10 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19045,6 +20237,10 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19106,6 +20302,8 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19125,6 +20323,8 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19281,6 +20481,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19313,6 +20516,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19347,6 +20554,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19412,6 +20623,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19434,6 +20647,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19609,6 +20824,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19641,6 +20859,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19675,6 +20897,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19740,6 +20966,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19762,6 +20990,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19937,6 +21167,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19968,6 +21201,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20000,6 +21237,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20062,6 +21303,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20082,6 +21325,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20237,6 +21482,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20268,6 +21516,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20300,6 +21552,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20362,6 +21618,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20382,6 +21640,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20533,6 +21793,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20565,6 +21828,10 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20599,6 +21866,10 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20664,6 +21935,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20686,6 +21959,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20861,6 +22136,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20893,6 +22171,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20927,6 +22209,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20992,6 +22278,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21014,6 +22302,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21189,6 +22479,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21221,6 +22514,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21255,6 +22552,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21320,6 +22621,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21342,6 +22645,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21517,6 +22822,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21549,6 +22857,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21583,6 +22895,10 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21648,6 +22964,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21670,6 +22988,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21845,6 +23165,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21877,6 +23200,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21911,6 +23238,10 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -21976,6 +23307,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -21998,6 +23331,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22169,6 +23504,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -22201,6 +23539,10 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22235,6 +23577,10 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22300,6 +23646,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22322,6 +23670,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22497,6 +23847,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -22529,6 +23882,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22563,6 +23920,10 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22628,6 +23989,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22650,6 +24013,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22825,6 +24190,9 @@ entry:
 define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -22857,6 +24225,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22891,6 +24263,10 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -22956,6 +24332,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -22978,6 +24356,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
index e1f82a70b4c0a..a88e0e217fdb4 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
@@ -11,6 +11,9 @@
 define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX7-LABEL: flat_nontemporal_load_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -26,6 +29,10 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_load_0:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -41,6 +48,10 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_load_0:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -142,6 +153,9 @@ entry:
 define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX7-LABEL: flat_nontemporal_load_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
@@ -172,6 +186,10 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_load_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_nop 0
@@ -202,6 +220,10 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_load_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_nop 0
@@ -405,6 +427,9 @@ entry:
 define amdgpu_kernel void @flat_nontemporal_store_0(
 ; GFX7-LABEL: flat_nontemporal_store_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -420,6 +445,10 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_store_0:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -435,6 +464,10 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_store_0:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -540,6 +573,9 @@ entry:
 define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX7-LABEL: flat_nontemporal_store_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -569,6 +605,10 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ;
 ; GFX10-WGP-LABEL: flat_nontemporal_store_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -597,6 +637,10 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ;
 ; GFX10-CU-LABEL: flat_nontemporal_store_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -799,6 +843,9 @@ entry:
 define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
 ; GFX7-LABEL: flat_volatile_workgroup_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -814,6 +861,10 @@ define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_volatile_workgroup_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -829,6 +880,10 @@ define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_volatile_workgroup_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -926,6 +981,9 @@ entry:
 define amdgpu_kernel void @flat_volatile_workgroup_release_store(
 ; GFX7-LABEL: flat_volatile_workgroup_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -938,6 +996,10 @@ define amdgpu_kernel void @flat_volatile_workgroup_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_volatile_workgroup_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -951,6 +1013,10 @@ define amdgpu_kernel void @flat_volatile_workgroup_release_store(
 ;
 ; GFX10-CU-LABEL: flat_volatile_workgroup_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
index 588f06f1be054..7c637a20ab47b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
@@ -15,6 +15,9 @@
 define amdgpu_kernel void @flat_wavefront_unordered_load(
 ; GFX7-LABEL: flat_wavefront_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -29,6 +32,10 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -43,6 +50,10 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -71,6 +82,8 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -83,6 +96,8 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -182,6 +197,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_load(
 ; GFX7-LABEL: flat_wavefront_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -196,6 +214,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -210,6 +232,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -238,6 +264,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -250,6 +278,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -349,6 +379,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_load(
 ; GFX7-LABEL: flat_wavefront_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -363,6 +396,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -377,6 +414,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -405,6 +446,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -417,6 +460,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -516,6 +561,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_load(
 ; GFX7-LABEL: flat_wavefront_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -530,6 +578,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -544,6 +596,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -572,6 +628,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -584,6 +642,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -683,6 +743,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_unordered_store(
 ; GFX7-LABEL: flat_wavefront_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -694,6 +757,10 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -705,6 +772,10 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -727,6 +798,8 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -737,6 +810,8 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -817,6 +892,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_store(
 ; GFX7-LABEL: flat_wavefront_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -828,6 +906,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -839,6 +921,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -861,6 +947,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -871,6 +959,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -951,6 +1041,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_release_store(
 ; GFX7-LABEL: flat_wavefront_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -962,6 +1055,10 @@ define amdgpu_kernel void @flat_wavefront_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -973,6 +1070,10 @@ define amdgpu_kernel void @flat_wavefront_release_store(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -995,6 +1096,8 @@ define amdgpu_kernel void @flat_wavefront_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1005,6 +1108,8 @@ define amdgpu_kernel void @flat_wavefront_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1085,6 +1190,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_store(
 ; GFX7-LABEL: flat_wavefront_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1096,6 +1204,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1107,6 +1219,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1129,6 +1245,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1139,6 +1257,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1219,6 +1339,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1230,6 +1353,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1241,6 +1368,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1263,6 +1394,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1273,6 +1406,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1353,6 +1488,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1364,6 +1502,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1375,6 +1517,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1397,6 +1543,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1407,6 +1555,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1487,6 +1637,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1498,6 +1651,10 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1509,6 +1666,10 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1531,6 +1692,8 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1541,6 +1704,8 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1621,6 +1786,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1632,6 +1800,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1643,6 +1815,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1665,6 +1841,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1675,6 +1853,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1755,6 +1935,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1766,6 +1949,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1777,6 +1964,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1799,6 +1990,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1809,6 +2002,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1889,6 +2084,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1904,6 +2102,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1919,6 +2121,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1949,6 +2155,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1962,6 +2170,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2068,6 +2278,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2083,6 +2296,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2098,6 +2315,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2128,6 +2349,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2141,6 +2364,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2247,6 +2472,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2262,6 +2490,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2277,6 +2509,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2307,6 +2543,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2320,6 +2558,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2426,6 +2666,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2451,6 +2694,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2476,6 +2723,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2526,6 +2777,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2540,6 +2793,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2649,6 +2904,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2674,6 +2932,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2699,6 +2961,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2749,6 +3015,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2763,6 +3031,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2872,6 +3142,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2897,6 +3170,10 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2922,6 +3199,10 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2972,6 +3253,8 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2986,6 +3269,8 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3095,6 +3380,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3120,6 +3408,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3145,6 +3437,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3195,6 +3491,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3209,6 +3507,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3318,6 +3618,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3343,6 +3646,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3368,6 +3675,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3418,6 +3729,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3432,6 +3745,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3541,6 +3856,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3566,6 +3884,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3591,6 +3913,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3641,6 +3967,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3655,6 +3983,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3764,6 +4094,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3789,6 +4122,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3814,6 +4151,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3864,6 +4205,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3878,6 +4221,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3987,6 +4332,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4012,6 +4360,10 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4037,6 +4389,10 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4087,6 +4443,8 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4101,6 +4459,8 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4210,6 +4570,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4235,6 +4598,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4260,6 +4627,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4310,6 +4681,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4324,6 +4697,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4433,6 +4808,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4458,6 +4836,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4483,6 +4865,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4533,6 +4919,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4547,6 +4935,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4656,6 +5046,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4681,6 +5074,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4706,6 +5103,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4756,6 +5157,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4770,6 +5173,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4879,6 +5284,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4904,6 +5312,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4929,6 +5341,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4979,6 +5395,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4993,6 +5411,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5102,6 +5522,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5127,6 +5550,10 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5152,6 +5579,10 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5202,6 +5633,8 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5216,6 +5649,8 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5325,6 +5760,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5350,6 +5788,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5375,6 +5817,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5425,6 +5871,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5439,6 +5887,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5548,6 +5998,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5573,6 +6026,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5598,6 +6055,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5648,6 +6109,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5662,6 +6125,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5771,6 +6236,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -5800,6 +6268,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -5829,6 +6301,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -5887,6 +6363,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5904,6 +6382,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6040,6 +6520,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6069,6 +6552,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6098,6 +6585,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6156,6 +6647,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6173,6 +6666,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6309,6 +6804,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6338,6 +6836,10 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6367,6 +6869,10 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6425,6 +6931,8 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6442,6 +6950,8 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6578,6 +7088,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6607,6 +7120,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6636,6 +7153,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6694,6 +7215,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6711,6 +7234,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6847,6 +7372,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6876,6 +7404,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6905,6 +7437,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6963,6 +7499,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6980,6 +7518,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7116,6 +7656,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7145,6 +7688,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7174,6 +7721,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7232,6 +7783,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7249,6 +7802,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7385,6 +7940,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7414,6 +7972,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7443,6 +8005,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7501,6 +8067,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7518,6 +8086,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7654,6 +8224,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7683,6 +8256,10 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7712,6 +8289,10 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7770,6 +8351,8 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7787,6 +8370,8 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7923,6 +8508,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7952,6 +8540,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7981,6 +8573,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8039,6 +8635,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8056,6 +8654,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8192,6 +8792,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8221,6 +8824,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8250,6 +8857,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8308,6 +8919,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8325,6 +8938,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8461,6 +9076,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8490,6 +9108,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8519,6 +9141,10 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8577,6 +9203,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8594,6 +9222,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8730,6 +9360,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8759,6 +9392,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8788,6 +9425,10 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8846,6 +9487,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8863,6 +9506,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8999,6 +9644,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9028,6 +9676,10 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9057,6 +9709,10 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9115,6 +9771,8 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9132,6 +9790,8 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9268,6 +9928,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9297,6 +9960,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9326,6 +9993,10 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9384,6 +10055,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9401,6 +10074,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9537,6 +10212,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9566,6 +10244,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9595,6 +10277,10 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9653,6 +10339,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9670,6 +10358,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9806,6 +10496,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
 ; GFX7-LABEL: flat_wavefront_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9820,6 +10513,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9834,6 +10531,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9862,6 +10563,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9874,6 +10577,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9973,6 +10678,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9987,6 +10695,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10001,6 +10713,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10029,6 +10745,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10041,6 +10759,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10140,6 +10860,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10154,6 +10877,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10168,6 +10895,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10196,6 +10927,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10208,6 +10941,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10307,6 +11042,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10321,6 +11059,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10335,6 +11077,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10363,6 +11109,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10375,6 +11123,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10474,6 +11224,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
 ; GFX7-LABEL: flat_wavefront_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10485,6 +11238,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10496,6 +11253,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10518,6 +11279,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10528,6 +11291,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10608,6 +11373,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10619,6 +11387,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10630,6 +11402,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10652,6 +11428,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10662,6 +11440,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10742,6 +11522,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_release_store(
 ; GFX7-LABEL: flat_wavefront_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10753,6 +11536,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10764,6 +11551,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10786,6 +11577,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10796,6 +11589,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10876,6 +11671,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10887,6 +11685,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10898,6 +11700,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10920,6 +11726,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10930,6 +11738,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11010,6 +11820,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11021,6 +11834,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11032,6 +11849,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11054,6 +11875,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11064,6 +11887,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11144,6 +11969,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11155,6 +11983,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11166,6 +11998,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11188,6 +12024,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11198,6 +12036,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11278,6 +12118,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11289,6 +12132,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11300,6 +12147,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11322,6 +12173,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11332,6 +12185,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11412,6 +12267,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11423,6 +12281,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11434,6 +12296,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11456,6 +12322,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11466,6 +12334,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11546,6 +12416,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11557,6 +12430,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11568,6 +12445,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11590,6 +12471,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11600,6 +12483,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11680,6 +12565,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11695,6 +12583,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11710,6 +12602,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11740,6 +12636,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11753,6 +12651,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11859,6 +12759,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11874,6 +12777,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11889,6 +12796,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11919,6 +12830,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11932,6 +12845,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12038,6 +12953,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12053,6 +12971,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12068,6 +12990,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12098,6 +13024,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12111,6 +13039,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12217,6 +13147,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12242,6 +13175,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12267,6 +13204,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12317,6 +13258,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12331,6 +13274,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12440,6 +13385,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12465,6 +13413,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12490,6 +13442,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12540,6 +13496,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12554,6 +13512,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12663,6 +13623,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12688,6 +13651,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12713,6 +13680,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12763,6 +13734,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12777,6 +13750,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12886,6 +13861,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12911,6 +13889,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12936,6 +13918,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12986,6 +13972,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13000,6 +13988,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13109,6 +14099,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13134,6 +14127,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13159,6 +14156,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13209,6 +14210,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13223,6 +14226,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13332,6 +14337,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13357,6 +14365,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13382,6 +14394,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13432,6 +14448,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13446,6 +14464,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13555,6 +14575,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13580,6 +14603,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13605,6 +14632,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13655,6 +14686,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13669,6 +14702,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13778,6 +14813,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13803,6 +14841,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13828,6 +14870,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13878,6 +14924,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13892,6 +14940,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14001,6 +15051,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14026,6 +15079,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14051,6 +15108,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14101,6 +15162,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14115,6 +15178,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14224,6 +15289,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14249,6 +15317,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14274,6 +15346,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14324,6 +15400,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14338,6 +15416,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14447,6 +15527,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14472,6 +15555,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14497,6 +15584,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14547,6 +15638,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14561,6 +15654,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14670,6 +15765,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14695,6 +15793,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14720,6 +15822,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14770,6 +15876,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14784,6 +15892,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14893,6 +16003,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14918,6 +16031,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14943,6 +16060,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14993,6 +16114,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15007,6 +16130,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15116,6 +16241,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15141,6 +16269,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15166,6 +16298,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15216,6 +16352,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15230,6 +16368,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15339,6 +16479,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15364,6 +16507,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15389,6 +16536,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15439,6 +16590,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15453,6 +16606,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15562,6 +16717,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -15591,6 +16749,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -15620,6 +16782,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -15678,6 +16844,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15695,6 +16863,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15831,6 +17001,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -15860,6 +17033,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -15889,6 +17066,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -15947,6 +17128,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15964,6 +17147,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16100,6 +17285,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16129,6 +17317,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16158,6 +17350,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16216,6 +17412,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16233,6 +17431,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16369,6 +17569,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16398,6 +17601,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16427,6 +17634,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16485,6 +17696,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16502,6 +17715,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16638,6 +17853,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16667,6 +17885,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16696,6 +17918,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16754,6 +17980,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16771,6 +17999,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16907,6 +18137,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16936,6 +18169,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16965,6 +18202,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17023,6 +18264,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17040,6 +18283,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17176,6 +18421,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17205,6 +18453,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17234,6 +18486,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17292,6 +18548,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17309,6 +18567,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17445,6 +18705,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17474,6 +18737,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17503,6 +18770,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17561,6 +18832,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17578,6 +18851,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17714,6 +18989,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17743,6 +19021,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17772,6 +19054,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17830,6 +19116,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17847,6 +19135,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17983,6 +19273,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18012,6 +19305,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18041,6 +19338,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18099,6 +19400,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18116,6 +19419,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18252,6 +19557,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18281,6 +19589,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18310,6 +19622,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18368,6 +19684,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18385,6 +19703,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18521,6 +19841,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18550,6 +19873,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18579,6 +19906,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18637,6 +19968,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18654,6 +19987,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18790,6 +20125,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18819,6 +20157,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18848,6 +20190,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18906,6 +20252,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18923,6 +20271,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19059,6 +20409,9 @@ entry:
 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19088,6 +20441,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19117,6 +20474,10 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19175,6 +20536,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19192,6 +20555,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
index ee7d79a8a8cbb..0fd4aa4a7a93f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
@@ -15,6 +15,9 @@
 define amdgpu_kernel void @flat_workgroup_unordered_load(
 ; GFX7-LABEL: flat_workgroup_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -29,6 +32,10 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -43,6 +50,10 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -71,6 +82,8 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -83,6 +96,8 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -182,6 +197,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_monotonic_load(
 ; GFX7-LABEL: flat_workgroup_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -196,6 +214,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -210,6 +232,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -238,6 +264,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -250,6 +278,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -349,6 +379,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acquire_load(
 ; GFX7-LABEL: flat_workgroup_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -364,6 +397,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -379,6 +416,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -409,6 +450,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -422,6 +465,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -528,6 +573,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_load(
 ; GFX7-LABEL: flat_workgroup_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -544,6 +592,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -561,6 +613,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -593,6 +649,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -607,6 +665,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -726,6 +786,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_unordered_store(
 ; GFX7-LABEL: flat_workgroup_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -737,6 +800,10 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -748,6 +815,10 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -770,6 +841,8 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -780,6 +853,8 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -860,6 +935,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_monotonic_store(
 ; GFX7-LABEL: flat_workgroup_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -871,6 +949,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -882,6 +964,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -904,6 +990,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -914,6 +1002,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -994,6 +1084,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_release_store(
 ; GFX7-LABEL: flat_workgroup_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1006,6 +1099,10 @@ define amdgpu_kernel void @flat_workgroup_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1019,6 +1116,10 @@ define amdgpu_kernel void @flat_workgroup_release_store(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1043,6 +1144,8 @@ define amdgpu_kernel void @flat_workgroup_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1054,6 +1157,8 @@ define amdgpu_kernel void @flat_workgroup_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1145,6 +1250,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_store(
 ; GFX7-LABEL: flat_workgroup_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1157,6 +1265,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1170,6 +1282,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1194,6 +1310,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1205,6 +1323,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1296,6 +1416,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1307,6 +1430,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1318,6 +1445,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1340,6 +1471,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1350,6 +1483,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1430,6 +1565,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1442,6 +1580,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1456,6 +1598,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1480,6 +1626,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1491,6 +1639,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1583,6 +1733,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1595,6 +1748,10 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1608,6 +1765,10 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1632,6 +1793,8 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1643,6 +1806,8 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1734,6 +1899,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1747,6 +1915,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1763,6 +1935,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1789,6 +1965,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1801,6 +1979,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1904,6 +2084,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1917,6 +2100,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1933,6 +2120,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1959,6 +2150,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1971,6 +2164,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2074,6 +2269,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2090,6 +2288,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2106,6 +2308,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2138,6 +2344,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2152,6 +2360,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2265,6 +2475,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2282,6 +2495,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2300,6 +2517,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2334,6 +2555,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2349,6 +2572,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2475,6 +2700,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2492,6 +2720,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2510,6 +2742,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2544,6 +2780,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2559,6 +2797,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2685,6 +2925,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2710,6 +2953,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2735,6 +2982,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2785,6 +3036,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2799,6 +3052,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -2908,6 +3163,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2934,6 +3192,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -2962,6 +3224,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3014,6 +3280,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3029,6 +3297,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3150,6 +3420,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3176,6 +3449,10 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3203,6 +3480,10 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3255,6 +3536,8 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3270,6 +3553,8 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3390,6 +3675,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3417,6 +3705,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3447,6 +3739,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3501,6 +3797,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3517,6 +3815,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3649,6 +3949,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3676,6 +3979,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3706,6 +4013,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3760,6 +4071,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3776,6 +4089,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -3908,6 +4223,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3934,6 +4252,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -3962,6 +4284,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4014,6 +4340,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4029,6 +4357,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4150,6 +4480,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4176,6 +4509,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4204,6 +4541,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4256,6 +4597,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4271,6 +4614,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4392,6 +4737,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4419,6 +4767,10 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4449,6 +4801,10 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4503,6 +4859,8 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4519,6 +4877,8 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4651,6 +5011,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4678,6 +5041,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4708,6 +5075,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4762,6 +5133,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4778,6 +5151,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -4910,6 +5285,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4937,6 +5315,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -4967,6 +5349,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5021,6 +5407,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5037,6 +5425,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5169,6 +5559,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5196,6 +5589,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5226,6 +5623,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -5280,6 +5681,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5296,6 +5699,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5428,6 +5833,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -5457,6 +5865,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -5486,6 +5898,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -5544,6 +5960,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5561,6 +5979,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5697,6 +6117,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -5727,6 +6150,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -5757,6 +6184,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -5817,6 +6248,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5835,6 +6268,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -5978,6 +6413,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6008,6 +6446,10 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6039,6 +6481,10 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6099,6 +6545,8 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6117,6 +6565,8 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6264,6 +6714,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6295,6 +6748,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6327,6 +6784,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6389,6 +6850,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6408,6 +6871,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6564,6 +7029,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6595,6 +7063,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6627,6 +7099,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6689,6 +7165,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6708,6 +7186,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -6864,6 +7344,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6894,6 +7377,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6924,6 +7411,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -6984,6 +7475,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7002,6 +7495,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7147,6 +7642,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7177,6 +7675,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7207,6 +7709,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7267,6 +7773,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7285,6 +7793,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7428,6 +7938,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7459,6 +7972,10 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7491,6 +8008,10 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7553,6 +8074,8 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7572,6 +8095,8 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7728,6 +8253,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7759,6 +8287,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7791,6 +8323,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -7853,6 +8389,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -7872,6 +8410,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8028,6 +8568,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8059,6 +8602,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8091,6 +8638,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8153,6 +8704,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8172,6 +8725,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8328,6 +8883,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8359,6 +8917,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8391,6 +8953,10 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8453,6 +9019,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8472,6 +9040,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8628,6 +9198,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8659,6 +9232,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8691,6 +9268,10 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8753,6 +9334,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8772,6 +9355,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -8926,6 +9511,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8957,6 +9545,10 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -8989,6 +9581,10 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9051,6 +9647,8 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9070,6 +9668,8 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9226,6 +9826,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9257,6 +9860,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9289,6 +9896,10 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9351,6 +9962,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9370,6 +9983,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9526,6 +10141,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9557,6 +10175,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9589,6 +10211,10 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -9651,6 +10277,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9670,6 +10298,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -9826,6 +10456,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
 ; GFX7-LABEL: flat_workgroup_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9840,6 +10473,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9854,6 +10491,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9882,6 +10523,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9894,6 +10537,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9993,6 +10638,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10007,6 +10655,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10021,6 +10673,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10049,6 +10705,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10061,6 +10719,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10160,6 +10820,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10174,6 +10837,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10190,6 +10857,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10218,6 +10889,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10230,6 +10903,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10335,6 +11010,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10349,6 +11027,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10367,6 +11049,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10395,6 +11081,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10407,6 +11095,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10522,6 +11212,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
 ; GFX7-LABEL: flat_workgroup_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10533,6 +11226,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10544,6 +11241,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10566,6 +11267,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10576,6 +11279,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10656,6 +11361,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10667,6 +11375,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10678,6 +11390,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10700,6 +11416,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10710,6 +11428,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10790,6 +11510,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_release_store(
 ; GFX7-LABEL: flat_workgroup_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10801,6 +11524,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10814,6 +11541,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10836,6 +11567,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10846,6 +11579,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10934,6 +11669,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10945,6 +11683,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10958,6 +11700,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10980,6 +11726,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10990,6 +11738,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11078,6 +11828,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11089,6 +11842,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11100,6 +11857,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11122,6 +11883,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11132,6 +11895,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11212,6 +11977,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11223,6 +11991,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11236,6 +12008,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11258,6 +12034,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11268,6 +12046,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11356,6 +12136,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11367,6 +12150,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11380,6 +12167,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11402,6 +12193,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11412,6 +12205,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11500,6 +12295,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11511,6 +12309,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11526,6 +12328,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11548,6 +12354,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11558,6 +12366,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11654,6 +12464,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11665,6 +12478,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11680,6 +12497,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11702,6 +12523,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11712,6 +12535,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11808,6 +12633,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11823,6 +12651,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11840,6 +12672,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11870,6 +12706,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11883,6 +12721,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11995,6 +12835,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12010,6 +12853,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12029,6 +12876,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12059,6 +12910,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12072,6 +12925,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12194,6 +13049,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12209,6 +13067,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12228,6 +13090,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12258,6 +13124,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12271,6 +13139,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12393,6 +13263,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12418,6 +13291,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12443,6 +13320,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12493,6 +13374,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12507,6 +13390,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12616,6 +13501,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12641,6 +13529,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12668,6 +13560,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12718,6 +13614,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12732,6 +13630,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12849,6 +13749,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12874,6 +13777,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12901,6 +13808,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -12951,6 +13862,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -12965,6 +13878,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13082,6 +13997,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13107,6 +14025,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13136,6 +14058,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13186,6 +14112,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13200,6 +14128,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13325,6 +14255,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13350,6 +14283,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13379,6 +14316,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13429,6 +14370,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13443,6 +14386,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13568,6 +14513,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13593,6 +14541,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13620,6 +14572,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13670,6 +14626,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13684,6 +14642,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13801,6 +14761,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13826,6 +14789,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13853,6 +14820,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -13903,6 +14874,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -13917,6 +14890,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14034,6 +15009,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14059,6 +15037,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14088,6 +15070,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14138,6 +15124,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14152,6 +15140,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14277,6 +15267,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14302,6 +15295,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14331,6 +15328,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14381,6 +15382,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14395,6 +15398,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14520,6 +15525,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14545,6 +15553,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14574,6 +15586,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14624,6 +15640,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14638,6 +15656,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14763,6 +15783,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14788,6 +15811,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14817,6 +15844,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -14867,6 +15898,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -14881,6 +15914,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15006,6 +16041,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15031,6 +16069,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15060,6 +16102,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15110,6 +16156,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15124,6 +16172,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15249,6 +16299,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15274,6 +16327,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15303,6 +16360,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15353,6 +16414,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15367,6 +16430,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15492,6 +16557,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15517,6 +16585,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15546,6 +16618,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15596,6 +16672,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15610,6 +16688,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15735,6 +16815,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15760,6 +16843,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15789,6 +16876,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[4:5], 0x8
@@ -15839,6 +16930,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15853,6 +16946,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -15978,6 +17073,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16007,6 +17105,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16036,6 +17138,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16094,6 +17200,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16111,6 +17219,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16247,6 +17357,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16276,6 +17389,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16307,6 +17424,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16365,6 +17486,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16382,6 +17505,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16524,6 +17649,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16553,6 +17681,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16584,6 +17716,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16642,6 +17778,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16659,6 +17797,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16803,6 +17943,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16832,6 +17975,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16865,6 +18012,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -16923,6 +18074,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -16940,6 +18093,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17092,6 +18247,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17121,6 +18279,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17154,6 +18316,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17212,6 +18378,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17229,6 +18397,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17381,6 +18551,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17410,6 +18583,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17441,6 +18618,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17499,6 +18680,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17516,6 +18699,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17660,6 +18845,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17689,6 +18877,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17720,6 +18912,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17778,6 +18974,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17795,6 +18993,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -17937,6 +19137,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17966,6 +19169,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -17999,6 +19206,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18057,6 +19268,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18074,6 +19287,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18226,6 +19441,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18255,6 +19473,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18288,6 +19510,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18346,6 +19572,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18363,6 +19591,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18515,6 +19745,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18544,6 +19777,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18577,6 +19814,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18635,6 +19876,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18652,6 +19895,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18804,6 +20049,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18833,6 +20081,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18866,6 +20118,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -18924,6 +20180,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -18941,6 +20199,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19093,6 +20353,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19122,6 +20385,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19155,6 +20422,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19213,6 +20484,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19230,6 +20503,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19380,6 +20655,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19409,6 +20687,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19442,6 +20724,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19500,6 +20786,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19517,6 +20805,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19669,6 +20959,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19698,6 +20991,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19731,6 +21028,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -19789,6 +21090,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19806,6 +21109,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -19958,6 +21263,9 @@ entry:
 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19987,6 +21295,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-WGP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-WGP-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-WGP-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20020,6 +21332,10 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-CU-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-CU-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX10-CU-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s9, s[6:7], 0x8
@@ -20078,6 +21394,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
@@ -20095,6 +21413,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-TGSPLIT-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x8
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0xc
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
index b9487f8e14c2b..8b600c835a160 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
@@ -41,6 +41,9 @@ define amdgpu_kernel void @global_agent_unordered_load(
 ;
 ; GFX7-LABEL: global_agent_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -222,6 +225,9 @@ define amdgpu_kernel void @global_agent_monotonic_load(
 ;
 ; GFX7-LABEL: global_agent_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -404,6 +410,9 @@ define amdgpu_kernel void @global_agent_acquire_load(
 ;
 ; GFX7-LABEL: global_agent_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -602,6 +611,9 @@ define amdgpu_kernel void @global_agent_seq_cst_load(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -809,6 +821,9 @@ define amdgpu_kernel void @global_agent_unordered_store(
 ;
 ; GFX7-LABEL: global_agent_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -962,6 +977,9 @@ define amdgpu_kernel void @global_agent_monotonic_store(
 ;
 ; GFX7-LABEL: global_agent_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1116,6 +1134,9 @@ define amdgpu_kernel void @global_agent_release_store(
 ;
 ; GFX7-LABEL: global_agent_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1294,6 +1315,9 @@ define amdgpu_kernel void @global_agent_seq_cst_store(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1470,6 +1494,9 @@ define amdgpu_kernel void @global_agent_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1623,6 +1650,9 @@ define amdgpu_kernel void @global_agent_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1802,6 +1832,9 @@ define amdgpu_kernel void @global_agent_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1980,6 +2013,9 @@ define amdgpu_kernel void @global_agent_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2185,6 +2221,9 @@ define amdgpu_kernel void @global_agent_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2390,6 +2429,9 @@ define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2587,6 +2629,9 @@ define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2812,6 +2857,9 @@ define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3038,6 +3086,9 @@ define amdgpu_kernel void @global_agent_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3256,6 +3307,9 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3500,6 +3554,9 @@ define amdgpu_kernel void @global_agent_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3743,6 +3800,9 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4013,6 +4073,9 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4282,6 +4345,9 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4527,6 +4593,9 @@ define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4773,6 +4842,9 @@ define amdgpu_kernel void @global_agent_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5043,6 +5115,9 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5313,6 +5388,9 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5583,6 +5661,9 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5853,6 +5934,9 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6123,6 +6207,9 @@ define amdgpu_kernel void @global_agent_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6393,6 +6480,9 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6663,6 +6753,9 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6933,6 +7026,9 @@ define amdgpu_kernel void @global_agent_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7182,6 +7278,9 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7447,6 +7546,9 @@ define amdgpu_kernel void @global_agent_release_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7721,6 +7823,9 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8015,6 +8120,9 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8308,6 +8416,9 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8577,6 +8688,9 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8843,6 +8957,9 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9137,6 +9254,9 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9431,6 +9551,9 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9725,6 +9848,9 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10019,6 +10145,9 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10309,6 +10438,9 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10603,6 +10735,9 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10897,6 +11032,9 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -11189,6 +11327,9 @@ define amdgpu_kernel void @global_agent_one_as_unordered_load(
 ;
 ; GFX7-LABEL: global_agent_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11370,6 +11511,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_load(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11552,6 +11696,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_load(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11750,6 +11897,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_load(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11957,6 +12107,9 @@ define amdgpu_kernel void @global_agent_one_as_unordered_store(
 ;
 ; GFX7-LABEL: global_agent_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12110,6 +12263,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_store(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12264,6 +12420,9 @@ define amdgpu_kernel void @global_agent_one_as_release_store(
 ;
 ; GFX7-LABEL: global_agent_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12442,6 +12601,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_store(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12618,6 +12780,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12771,6 +12936,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12950,6 +13118,9 @@ define amdgpu_kernel void @global_agent_one_as_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13128,6 +13299,9 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13333,6 +13507,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13538,6 +13715,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13735,6 +13915,9 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13960,6 +14143,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -14186,6 +14372,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14404,6 +14593,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14648,6 +14840,9 @@ define amdgpu_kernel void @global_agent_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14891,6 +15086,9 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15161,6 +15359,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15430,6 +15631,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15675,6 +15879,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15921,6 +16128,9 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16191,6 +16401,9 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16461,6 +16674,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16731,6 +16947,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17001,6 +17220,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17271,6 +17493,9 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17541,6 +17766,9 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -17811,6 +18039,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -18081,6 +18312,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18330,6 +18564,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18596,6 +18833,9 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18890,6 +19130,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19183,6 +19426,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19452,6 +19698,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19718,6 +19967,9 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20012,6 +20264,9 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20306,6 +20561,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20600,6 +20858,9 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20894,6 +21155,9 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21184,6 +21448,9 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21478,6 +21745,9 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21772,6 +22042,9 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index a6bd1b678f95e..16e55058e4fc8 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -36,6 +36,9 @@ define amdgpu_kernel void @global_nontemporal_load_0(
 ;
 ; GFX7-LABEL: global_nontemporal_load_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -227,6 +230,9 @@ define amdgpu_kernel void @global_nontemporal_load_1(
 ;
 ; GFX7-LABEL: global_nontemporal_load_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
@@ -474,6 +480,9 @@ define amdgpu_kernel void @global_nontemporal_store_0(
 ;
 ; GFX7-LABEL: global_nontemporal_store_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -658,6 +667,9 @@ define amdgpu_kernel void @global_nontemporal_store_1(
 ;
 ; GFX7-LABEL: global_nontemporal_store_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -891,6 +903,9 @@ define amdgpu_kernel void @global_nontemporal_volatile_load(
 ;
 ; GFX7-LABEL: global_nontemporal_volatile_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
index a5de6a92db1af..8042d38716107 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
@@ -41,6 +41,9 @@ define amdgpu_kernel void @global_singlethread_unordered_load(
 ;
 ; GFX7-LABEL: global_singlethread_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -222,6 +225,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_load(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -403,6 +409,9 @@ define amdgpu_kernel void @global_singlethread_acquire_load(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -584,6 +593,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_load(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -758,6 +770,9 @@ define amdgpu_kernel void @global_singlethread_unordered_store(
 ;
 ; GFX7-LABEL: global_singlethread_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -911,6 +926,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_store(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1064,6 +1082,9 @@ define amdgpu_kernel void @global_singlethread_release_store(
 ;
 ; GFX7-LABEL: global_singlethread_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1217,6 +1238,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_store(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1369,6 +1393,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1520,6 +1547,9 @@ define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1671,6 +1701,9 @@ define amdgpu_kernel void @global_singlethread_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1822,6 +1855,9 @@ define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1973,6 +2009,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2126,6 +2165,9 @@ define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2306,6 +2348,9 @@ define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2486,6 +2531,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2669,6 +2717,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2885,6 +2936,9 @@ define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3101,6 +3155,9 @@ define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3317,6 +3374,9 @@ define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3533,6 +3593,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3749,6 +3812,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3965,6 +4031,9 @@ define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4181,6 +4250,9 @@ define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4397,6 +4469,9 @@ define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4613,6 +4688,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4829,6 +4907,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5045,6 +5126,9 @@ define amdgpu_kernel void @global_singlethread_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5261,6 +5345,9 @@ define amdgpu_kernel void @global_singlethread_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5477,6 +5564,9 @@ define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5693,6 +5783,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5912,6 +6005,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6160,6 +6256,9 @@ define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6408,6 +6507,9 @@ define amdgpu_kernel void @global_singlethread_release_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6656,6 +6758,9 @@ define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6904,6 +7009,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7152,6 +7260,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7400,6 +7511,9 @@ define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7648,6 +7762,9 @@ define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7896,6 +8013,9 @@ define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8144,6 +8264,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8392,6 +8515,9 @@ define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8640,6 +8766,9 @@ define amdgpu_kernel void @global_singlethread_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8888,6 +9017,9 @@ define amdgpu_kernel void @global_singlethread_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9136,6 +9268,9 @@ define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9384,6 +9519,9 @@ define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9632,6 +9770,9 @@ define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9813,6 +9954,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9994,6 +10138,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10175,6 +10322,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10349,6 +10499,9 @@ define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10502,6 +10655,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10655,6 +10811,9 @@ define amdgpu_kernel void @global_singlethread_one_as_release_store(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10808,6 +10967,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10960,6 +11122,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11111,6 +11276,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11262,6 +11430,9 @@ define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11413,6 +11584,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11564,6 +11738,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11717,6 +11894,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11897,6 +12077,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12077,6 +12260,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12260,6 +12446,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12476,6 +12665,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12692,6 +12884,9 @@ define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12908,6 +13103,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13124,6 +13322,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13340,6 +13541,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13556,6 +13760,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13772,6 +13979,9 @@ define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13988,6 +14198,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14204,6 +14417,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14420,6 +14636,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14636,6 +14855,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14852,6 +15074,9 @@ define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15068,6 +15293,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15284,6 +15512,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15503,6 +15734,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_ret_cm
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -15751,6 +15985,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpx
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -15999,6 +16236,9 @@ define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_ret_cmpx
 ;
 ; GFX7-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16247,6 +16487,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpx
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16495,6 +16738,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpx
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16743,6 +16989,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_ret_cmpx
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16991,6 +17240,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17239,6 +17491,9 @@ define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17487,6 +17742,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17735,6 +17993,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17983,6 +18244,9 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_ret_cmpx
 ;
 ; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18231,6 +18495,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_ret_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18479,6 +18746,9 @@ define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_ret_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18727,6 +18997,9 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18975,6 +19248,9 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxch
 ;
 ; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
index 69404247ccd6e..9c11781da56f2 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
@@ -41,6 +41,9 @@ define amdgpu_kernel void @global_system_unordered_load(
 ;
 ; GFX7-LABEL: global_system_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -222,6 +225,9 @@ define amdgpu_kernel void @global_system_monotonic_load(
 ;
 ; GFX7-LABEL: global_system_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -404,6 +410,9 @@ define amdgpu_kernel void @global_system_acquire_load(
 ;
 ; GFX7-LABEL: global_system_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -604,6 +613,9 @@ define amdgpu_kernel void @global_system_seq_cst_load(
 ;
 ; GFX7-LABEL: global_system_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -813,6 +825,9 @@ define amdgpu_kernel void @global_system_unordered_store(
 ;
 ; GFX7-LABEL: global_system_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -966,6 +981,9 @@ define amdgpu_kernel void @global_system_monotonic_store(
 ;
 ; GFX7-LABEL: global_system_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1120,6 +1138,9 @@ define amdgpu_kernel void @global_system_release_store(
 ;
 ; GFX7-LABEL: global_system_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1302,6 +1323,9 @@ define amdgpu_kernel void @global_system_seq_cst_store(
 ;
 ; GFX7-LABEL: global_system_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1482,6 +1506,9 @@ define amdgpu_kernel void @global_system_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1635,6 +1662,9 @@ define amdgpu_kernel void @global_system_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1816,6 +1846,9 @@ define amdgpu_kernel void @global_system_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1998,6 +2031,9 @@ define amdgpu_kernel void @global_system_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2209,6 +2245,9 @@ define amdgpu_kernel void @global_system_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2420,6 +2459,9 @@ define amdgpu_kernel void @global_system_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2619,6 +2661,9 @@ define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2850,6 +2895,9 @@ define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3082,6 +3130,9 @@ define amdgpu_kernel void @global_system_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3300,6 +3351,9 @@ define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3546,6 +3600,9 @@ define amdgpu_kernel void @global_system_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3793,6 +3850,9 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4069,6 +4129,9 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4344,6 +4407,9 @@ define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4591,6 +4657,9 @@ define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4839,6 +4908,9 @@ define amdgpu_kernel void @global_system_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5115,6 +5187,9 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5391,6 +5466,9 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5667,6 +5745,9 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5943,6 +6024,9 @@ define amdgpu_kernel void @global_system_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6192,6 +6276,9 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6460,6 +6547,9 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6760,6 +6850,9 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7059,6 +7152,9 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7330,6 +7426,9 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7598,6 +7697,9 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7898,6 +8000,9 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8198,6 +8303,9 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8498,6 +8606,9 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8798,6 +8909,9 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9094,6 +9208,9 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9394,6 +9511,9 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9694,6 +9814,9 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9992,6 +10115,9 @@ define amdgpu_kernel void @global_system_one_as_unordered_load(
 ;
 ; GFX7-LABEL: global_system_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10173,6 +10299,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_load(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10355,6 +10484,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_load(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10555,6 +10687,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_load(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10764,6 +10899,9 @@ define amdgpu_kernel void @global_system_one_as_unordered_store(
 ;
 ; GFX7-LABEL: global_system_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10917,6 +11055,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_store(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11071,6 +11212,9 @@ define amdgpu_kernel void @global_system_one_as_release_store(
 ;
 ; GFX7-LABEL: global_system_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11253,6 +11397,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_store(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11433,6 +11580,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11586,6 +11736,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11767,6 +11920,9 @@ define amdgpu_kernel void @global_system_one_as_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11949,6 +12105,9 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12160,6 +12319,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12371,6 +12533,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12570,6 +12735,9 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12801,6 +12969,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13033,6 +13204,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13251,6 +13425,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13497,6 +13674,9 @@ define amdgpu_kernel void @global_system_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13744,6 +13924,9 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14020,6 +14203,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14295,6 +14481,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14542,6 +14731,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14790,6 +14982,9 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15066,6 +15261,9 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15342,6 +15540,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15618,6 +15819,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15894,6 +16098,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16170,6 +16377,9 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16446,6 +16656,9 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16722,6 +16935,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16998,6 +17214,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17247,6 +17466,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17514,6 +17736,9 @@ define amdgpu_kernel void @global_system_one_as_release_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17792,6 +18017,9 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18092,6 +18320,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18391,6 +18622,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18662,6 +18896,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18930,6 +19167,9 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19230,6 +19470,9 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19530,6 +19773,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19830,6 +20076,9 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20130,6 +20379,9 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20426,6 +20678,9 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20726,6 +20981,9 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -21026,6 +21284,9 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
index 7dfd5e60c24f8..8a5c5dda9f79c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
@@ -37,6 +37,9 @@ define amdgpu_kernel void @global_volatile_load_0(
 ;
 ; GFX7-LABEL: global_volatile_load_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -184,6 +187,9 @@ define amdgpu_kernel void @global_volatile_load_1(
 ;
 ; GFX7-LABEL: global_volatile_load_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
@@ -372,6 +378,9 @@ define amdgpu_kernel void @global_volatile_store_0(
 ;
 ; GFX7-LABEL: global_volatile_store_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -527,6 +536,9 @@ define amdgpu_kernel void @global_volatile_store_1(
 ;
 ; GFX7-LABEL: global_volatile_store_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -718,6 +730,9 @@ define amdgpu_kernel void @global_volatile_workgroup_acquire_load(
 ;
 ; GFX7-LABEL: global_volatile_workgroup_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -852,6 +867,9 @@ define amdgpu_kernel void @global_volatile_workgroup_release_store(
 ;
 ; GFX7-LABEL: global_volatile_workgroup_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
index 4b6c99282dc13..151ba07a0b531 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
@@ -41,6 +41,9 @@ define amdgpu_kernel void @global_wavefront_unordered_load(
 ;
 ; GFX7-LABEL: global_wavefront_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -222,6 +225,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_load(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -403,6 +409,9 @@ define amdgpu_kernel void @global_wavefront_acquire_load(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -584,6 +593,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_load(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -758,6 +770,9 @@ define amdgpu_kernel void @global_wavefront_unordered_store(
 ;
 ; GFX7-LABEL: global_wavefront_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -911,6 +926,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_store(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1064,6 +1082,9 @@ define amdgpu_kernel void @global_wavefront_release_store(
 ;
 ; GFX7-LABEL: global_wavefront_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1217,6 +1238,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_store(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1369,6 +1393,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1520,6 +1547,9 @@ define amdgpu_kernel void @global_wavefront_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1671,6 +1701,9 @@ define amdgpu_kernel void @global_wavefront_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1822,6 +1855,9 @@ define amdgpu_kernel void @global_wavefront_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1973,6 +2009,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2126,6 +2165,9 @@ define amdgpu_kernel void @global_wavefront_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2306,6 +2348,9 @@ define amdgpu_kernel void @global_wavefront_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2486,6 +2531,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2669,6 +2717,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -2885,6 +2936,9 @@ define amdgpu_kernel void @global_wavefront_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3101,6 +3155,9 @@ define amdgpu_kernel void @global_wavefront_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3317,6 +3374,9 @@ define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3533,6 +3593,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3749,6 +3812,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3965,6 +4031,9 @@ define amdgpu_kernel void @global_wavefront_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4181,6 +4250,9 @@ define amdgpu_kernel void @global_wavefront_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4397,6 +4469,9 @@ define amdgpu_kernel void @global_wavefront_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4613,6 +4688,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4829,6 +4907,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5045,6 +5126,9 @@ define amdgpu_kernel void @global_wavefront_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5261,6 +5345,9 @@ define amdgpu_kernel void @global_wavefront_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5477,6 +5564,9 @@ define amdgpu_kernel void @global_wavefront_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5693,6 +5783,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5912,6 +6005,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6160,6 +6256,9 @@ define amdgpu_kernel void @global_wavefront_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6408,6 +6507,9 @@ define amdgpu_kernel void @global_wavefront_release_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6656,6 +6758,9 @@ define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6904,6 +7009,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7152,6 +7260,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7400,6 +7511,9 @@ define amdgpu_kernel void @global_wavefront_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7648,6 +7762,9 @@ define amdgpu_kernel void @global_wavefront_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7896,6 +8013,9 @@ define amdgpu_kernel void @global_wavefront_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8144,6 +8264,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8392,6 +8515,9 @@ define amdgpu_kernel void @global_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8640,6 +8766,9 @@ define amdgpu_kernel void @global_wavefront_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8888,6 +9017,9 @@ define amdgpu_kernel void @global_wavefront_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9136,6 +9268,9 @@ define amdgpu_kernel void @global_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9384,6 +9519,9 @@ define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9632,6 +9770,9 @@ define amdgpu_kernel void @global_wavefront_one_as_unordered_load(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9813,6 +9954,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_load(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -9994,6 +10138,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_load(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10175,6 +10322,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_load(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10349,6 +10499,9 @@ define amdgpu_kernel void @global_wavefront_one_as_unordered_store(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10502,6 +10655,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_store(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10655,6 +10811,9 @@ define amdgpu_kernel void @global_wavefront_one_as_release_store(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10808,6 +10967,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_store(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10960,6 +11122,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11111,6 +11276,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11262,6 +11430,9 @@ define amdgpu_kernel void @global_wavefront_one_as_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11413,6 +11584,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11564,6 +11738,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11717,6 +11894,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11897,6 +12077,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12077,6 +12260,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12260,6 +12446,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12476,6 +12665,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12692,6 +12884,9 @@ define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -12908,6 +13103,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13124,6 +13322,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13340,6 +13541,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13556,6 +13760,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13772,6 +13979,9 @@ define amdgpu_kernel void @global_wavefront_one_as_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13988,6 +14198,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14204,6 +14417,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14420,6 +14636,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14636,6 +14855,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14852,6 +15074,9 @@ define amdgpu_kernel void @global_wavefront_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15068,6 +15293,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15284,6 +15512,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15503,6 +15734,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_ret_cmpxc
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -15751,6 +15985,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -15999,6 +16236,9 @@ define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_wavefront_one_as_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16247,6 +16487,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16495,6 +16738,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16743,6 +16989,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_acquire_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16991,6 +17240,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17239,6 +17491,9 @@ define amdgpu_kernel void @global_wavefront_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17487,6 +17742,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17735,6 +17993,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17983,6 +18244,9 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18231,6 +18495,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18479,6 +18746,9 @@ define amdgpu_kernel void @global_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18727,6 +18997,9 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18975,6 +19248,9 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
index 46d65187cb1b2..69b0c7f93ab0e 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
@@ -41,6 +41,9 @@ define amdgpu_kernel void @global_workgroup_unordered_load(
 ;
 ; GFX7-LABEL: global_workgroup_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -222,6 +225,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_load(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -403,6 +409,9 @@ define amdgpu_kernel void @global_workgroup_acquire_load(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -590,6 +599,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_load(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -780,6 +792,9 @@ define amdgpu_kernel void @global_workgroup_unordered_store(
 ;
 ; GFX7-LABEL: global_workgroup_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -933,6 +948,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_store(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1087,6 +1105,9 @@ define amdgpu_kernel void @global_workgroup_release_store(
 ;
 ; GFX7-LABEL: global_workgroup_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1258,6 +1279,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_store(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1427,6 +1451,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1578,6 +1605,9 @@ define amdgpu_kernel void @global_workgroup_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1740,6 +1770,9 @@ define amdgpu_kernel void @global_workgroup_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1909,6 +1942,9 @@ define amdgpu_kernel void @global_workgroup_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2088,6 +2124,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2268,6 +2307,9 @@ define amdgpu_kernel void @global_workgroup_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2454,6 +2496,9 @@ define amdgpu_kernel void @global_workgroup_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2659,6 +2704,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2866,6 +2914,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3082,6 +3133,9 @@ define amdgpu_kernel void @global_workgroup_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3309,6 +3363,9 @@ define amdgpu_kernel void @global_workgroup_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3543,6 +3600,9 @@ define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -3787,6 +3847,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4030,6 +4093,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4256,6 +4322,9 @@ define amdgpu_kernel void @global_workgroup_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4483,6 +4552,9 @@ define amdgpu_kernel void @global_workgroup_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4727,6 +4799,9 @@ define amdgpu_kernel void @global_workgroup_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -4971,6 +5046,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5215,6 +5293,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5459,6 +5540,9 @@ define amdgpu_kernel void @global_workgroup_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5703,6 +5787,9 @@ define amdgpu_kernel void @global_workgroup_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -5947,6 +6034,9 @@ define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6191,6 +6281,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -6437,6 +6530,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6685,6 +6781,9 @@ define amdgpu_kernel void @global_workgroup_acquire_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -6939,6 +7038,9 @@ define amdgpu_kernel void @global_workgroup_release_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7205,6 +7307,9 @@ define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7478,6 +7583,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -7750,6 +7858,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8005,6 +8116,9 @@ define amdgpu_kernel void @global_workgroup_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8259,6 +8373,9 @@ define amdgpu_kernel void @global_workgroup_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8532,6 +8649,9 @@ define amdgpu_kernel void @global_workgroup_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -8805,6 +8925,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9078,6 +9201,9 @@ define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9351,6 +9477,9 @@ define amdgpu_kernel void @global_workgroup_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9622,6 +9751,9 @@ define amdgpu_kernel void @global_workgroup_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -9895,6 +10027,9 @@ define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10168,6 +10303,9 @@ define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -10440,6 +10578,9 @@ define amdgpu_kernel void @global_workgroup_one_as_unordered_load(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_unordered_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10621,6 +10762,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_load(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10802,6 +10946,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_load(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -10988,6 +11135,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_load(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11175,6 +11325,9 @@ define amdgpu_kernel void @global_workgroup_one_as_unordered_store(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_unordered_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11328,6 +11481,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_store(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11481,6 +11637,9 @@ define amdgpu_kernel void @global_workgroup_one_as_release_store(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_release_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11644,6 +11803,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_store(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_store:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11806,6 +11968,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -11957,6 +12122,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12118,6 +12286,9 @@ define amdgpu_kernel void @global_workgroup_one_as_release_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_release_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12279,6 +12450,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12450,6 +12624,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12623,6 +12800,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -12808,6 +12988,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13005,6 +13188,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_ret_atomicrmw(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x2
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
@@ -13205,6 +13391,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13421,6 +13610,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13647,6 +13839,9 @@ define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -13873,6 +14068,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14109,6 +14307,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14345,6 +14546,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14571,6 +14775,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -14797,6 +15004,9 @@ define amdgpu_kernel void @global_workgroup_one_as_release_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15033,6 +15243,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15269,6 +15482,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15505,6 +15721,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15741,6 +15960,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -15977,6 +16199,9 @@ define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16213,6 +16438,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16449,6 +16677,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[4:5], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s7, s[4:5], 0x2
@@ -16688,6 +16919,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_ret_cmpxc
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -16936,6 +17170,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17189,6 +17426,9 @@ define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17447,6 +17687,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17712,6 +17955,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -17977,6 +18223,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18232,6 +18481,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18485,6 +18737,9 @@ define amdgpu_kernel void @global_workgroup_one_as_release_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -18750,6 +19005,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19015,6 +19273,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19280,6 +19541,9 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg
 ;
 ; GFX7-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19545,6 +19809,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -19808,6 +20075,9 @@ define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20073,6 +20343,9 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
@@ -20338,6 +20611,9 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ;
 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_mov_b64 s[6:7], s[8:9]
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x0
 ; GFX7-NEXT:    s_load_dword s9, s[6:7], 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
index 04b0f00fe77b5..78209ee34cad4 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -38,6 +38,9 @@ define amdgpu_kernel void @local_nontemporal_load_0(
 ;
 ; GFX7-LABEL: local_nontemporal_load_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_mov_b32 m0, -1
@@ -224,6 +227,9 @@ define amdgpu_kernel void @local_nontemporal_load_1(
 ;
 ; GFX7-LABEL: local_nontemporal_load_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_mov_b32 s7, 2
@@ -830,6 +836,9 @@ define amdgpu_kernel void @local_nontemporal_volatile_load(
 ;
 ; GFX7-LABEL: local_nontemporal_volatile_load:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_mov_b32 m0, -1
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
index 9e5f5fcffca9f..bc2508411ed6b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
@@ -34,6 +34,9 @@ define amdgpu_kernel void @local_volatile_load_0(
 ;
 ; GFX7-LABEL: local_volatile_load_0:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_mov_b32 m0, -1
@@ -172,6 +175,9 @@ define amdgpu_kernel void @local_volatile_load_1(
 ;
 ; GFX7-LABEL: local_volatile_load_1:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
 ; GFX7-NEXT:    s_mov_b32 s7, 2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index fceee413f3f97..2aa4f021c259c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -38,7 +38,10 @@ define amdgpu_kernel void @private_nontemporal_load_0(
 ;
 ; GFX7-LABEL: private_nontemporal_load_0:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
@@ -53,7 +56,7 @@ define amdgpu_kernel void @private_nontemporal_load_0(
 ;
 ; GFX10-WGP-LABEL: private_nontemporal_load_0:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -67,7 +70,7 @@ define amdgpu_kernel void @private_nontemporal_load_0(
 ;
 ; GFX10-CU-LABEL: private_nontemporal_load_0:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -107,7 +110,7 @@ define amdgpu_kernel void @private_nontemporal_load_0(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_load_0:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -121,7 +124,7 @@ define amdgpu_kernel void @private_nontemporal_load_0(
 ;
 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_load_0:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-TGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -232,7 +235,10 @@ define amdgpu_kernel void @private_nontemporal_load_1(
 ;
 ; GFX7-LABEL: private_nontemporal_load_1:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
@@ -249,7 +255,7 @@ define amdgpu_kernel void @private_nontemporal_load_1(
 ;
 ; GFX10-WGP-LABEL: private_nontemporal_load_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[8:9], 0x0
@@ -265,7 +271,7 @@ define amdgpu_kernel void @private_nontemporal_load_1(
 ;
 ; GFX10-CU-LABEL: private_nontemporal_load_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[8:9], 0x0
@@ -309,7 +315,7 @@ define amdgpu_kernel void @private_nontemporal_load_1(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_load_1:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x0
@@ -328,7 +334,7 @@ define amdgpu_kernel void @private_nontemporal_load_1(
 ;
 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_load_1:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-TGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s7, s[8:9], 0x0
@@ -470,7 +476,7 @@ define amdgpu_kernel void @private_nontemporal_store_0(
 ;
 ; GFX7-LABEL: private_nontemporal_store_0:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
@@ -484,7 +490,7 @@ define amdgpu_kernel void @private_nontemporal_store_0(
 ;
 ; GFX10-WGP-LABEL: private_nontemporal_store_0:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
@@ -498,7 +504,7 @@ define amdgpu_kernel void @private_nontemporal_store_0(
 ;
 ; GFX10-CU-LABEL: private_nontemporal_store_0:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
@@ -530,7 +536,7 @@ define amdgpu_kernel void @private_nontemporal_store_0(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_store_0:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
@@ -544,7 +550,7 @@ define amdgpu_kernel void @private_nontemporal_store_0(
 ;
 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_store_0:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-TGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s4, s[8:9], 0x8
@@ -647,7 +653,7 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ;
 ; GFX7-LABEL: private_nontemporal_store_1:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[8:9], 0x2
@@ -663,7 +669,7 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ;
 ; GFX10-WGP-LABEL: private_nontemporal_store_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
@@ -678,7 +684,7 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ;
 ; GFX10-CU-LABEL: private_nontemporal_store_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
@@ -713,7 +719,7 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_store_1:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
@@ -731,7 +737,7 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ;
 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_store_1:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-TGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x8
@@ -874,7 +880,10 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
 ;
 ; GFX7-LABEL: private_nontemporal_volatile_load:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
@@ -889,7 +898,7 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
 ;
 ; GFX10-WGP-LABEL: private_nontemporal_volatile_load:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -903,7 +912,7 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
 ;
 ; GFX10-CU-LABEL: private_nontemporal_volatile_load:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -943,7 +952,7 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
 ;
 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_volatile_load:
 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-NOTTGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-NOTTGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -957,7 +966,7 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
 ;
 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_volatile_load:
 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
-; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s15
+; GFX90A-TGSPLIT-NEXT:    s_add_u32 s0, s0, s17
 ; GFX90A-TGSPLIT-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
index f8fb7986938f2..df4193969f8a0 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
@@ -38,7 +38,10 @@ define amdgpu_kernel void @private_volatile_load_0(
 ;
 ; GFX7-LABEL: private_volatile_load_0:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
@@ -53,7 +56,7 @@ define amdgpu_kernel void @private_volatile_load_0(
 ;
 ; GFX10-WGP-LABEL: private_volatile_load_0:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -67,7 +70,7 @@ define amdgpu_kernel void @private_volatile_load_0(
 ;
 ; GFX10-CU-LABEL: private_volatile_load_0:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x8
@@ -190,7 +193,10 @@ define amdgpu_kernel void @private_volatile_load_1(
 ;
 ; GFX7-LABEL: private_volatile_load_1:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX7-NEXT:    s_add_i32 s12, s12, s17
+; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x2
@@ -207,7 +213,7 @@ define amdgpu_kernel void @private_volatile_load_1(
 ;
 ; GFX10-WGP-LABEL: private_volatile_load_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX10-WGP-NEXT:    s_load_dword s7, s[8:9], 0x0
@@ -223,7 +229,7 @@ define amdgpu_kernel void @private_volatile_load_1(
 ;
 ; GFX10-CU-LABEL: private_volatile_load_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX10-CU-NEXT:    s_load_dword s7, s[8:9], 0x0
@@ -365,7 +371,7 @@ define amdgpu_kernel void @private_volatile_store_0(
 ;
 ; GFX7-LABEL: private_volatile_store_0:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s4, s[8:9], 0x2
@@ -380,7 +386,7 @@ define amdgpu_kernel void @private_volatile_store_0(
 ;
 ; GFX10-WGP-LABEL: private_volatile_store_0:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s4, s[8:9], 0x8
@@ -395,7 +401,7 @@ define amdgpu_kernel void @private_volatile_store_0(
 ;
 ; GFX10-CU-LABEL: private_volatile_store_0:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s4, s[8:9], 0x8
@@ -515,7 +521,7 @@ define amdgpu_kernel void @private_volatile_store_1(
 ;
 ; GFX7-LABEL: private_volatile_store_1:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_add_u32 s0, s0, s15
+; GFX7-NEXT:    s_add_u32 s0, s0, s17
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[8:9], 0x2
@@ -532,7 +538,7 @@ define amdgpu_kernel void @private_volatile_store_1(
 ;
 ; GFX10-WGP-LABEL: private_volatile_store_1:
 ; GFX10-WGP:       ; %bb.0: ; %entry
-; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-WGP-NEXT:    s_load_dword s6, s[8:9], 0x8
@@ -548,7 +554,7 @@ define amdgpu_kernel void @private_volatile_store_1(
 ;
 ; GFX10-CU-LABEL: private_volatile_store_1:
 ; GFX10-CU:       ; %bb.0: ; %entry
-; GFX10-CU-NEXT:    s_add_u32 s0, s0, s15
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, s17
 ; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-CU-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-CU-NEXT:    s_load_dword s6, s[8:9], 0x8
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index aaf81e2fa4000..07072f6a36296 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -34,10 +34,13 @@ define amdgpu_kernel void @v_test_imin_sle_i32(ptr addrspace(1) %out, ptr addrsp
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, s4, v4
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -56,10 +59,13 @@ define amdgpu_kernel void @v_test_imin_sle_i32(ptr addrspace(1) %out, ptr addrsp
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -144,6 +150,9 @@ define amdgpu_kernel void @s_test_imin_sle_i32(ptr addrspace(1) %out, i32 %a, i3
 ; CI-LABEL: s_test_imin_sle_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_i32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -155,6 +164,9 @@ define amdgpu_kernel void @s_test_imin_sle_i32(ptr addrspace(1) %out, i32 %a, i3
 ; VI-LABEL: s_test_imin_sle_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_i32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -214,6 +226,9 @@ define amdgpu_kernel void @s_test_imin_sle_v1i32(ptr addrspace(1) %out, <1 x i32
 ; CI-LABEL: s_test_imin_sle_v1i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_i32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -225,6 +240,9 @@ define amdgpu_kernel void @s_test_imin_sle_v1i32(ptr addrspace(1) %out, <1 x i32
 ; VI-LABEL: s_test_imin_sle_v1i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_i32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -288,6 +306,9 @@ define amdgpu_kernel void @s_test_imin_sle_v4i32(ptr addrspace(1) %out, <4 x i32
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x4
 ; CI-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_i32 s3, s3, s7
 ; CI-NEXT:    s_min_i32 s2, s2, s6
@@ -306,6 +327,9 @@ define amdgpu_kernel void @s_test_imin_sle_v4i32(ptr addrspace(1) %out, <4 x i32
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_i32 s3, s3, s7
 ; VI-NEXT:    s_min_i32 s2, s2, s6
@@ -414,11 +438,14 @@ define amdgpu_kernel void @s_test_imin_sle_i8(ptr addrspace(1) %out, [8 x i32],
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0xa
 ; CI-NEXT:    s_load_dword s3, s[8:9], 0x13
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_sext_i32_i8 s2, s2
 ; CI-NEXT:    s_sext_i32_i8 s3, s3
 ; CI-NEXT:    s_min_i32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    flat_store_byte v[0:1], v2
@@ -429,11 +456,14 @@ define amdgpu_kernel void @s_test_imin_sle_i8(ptr addrspace(1) %out, [8 x i32],
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x28
 ; VI-NEXT:    s_load_dword s3, s[8:9], 0x4c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_sext_i32_i8 s2, s2
 ; VI-NEXT:    s_sext_i32_i8 s3, s3
 ; VI-NEXT:    s_min_i32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    flat_store_byte v[0:1], v2
@@ -549,6 +579,8 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0xa
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    s_load_dword s3, s[8:9], 0x13
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_ashr_i32 s4, s2, 24
 ; CI-NEXT:    s_sext_i32_i8 s5, s2
@@ -572,6 +604,7 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
 ; CI-NEXT:    s_and_b32 s3, s3, 0xffff
 ; CI-NEXT:    s_or_b32 s2, s3, s2
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    flat_store_dword v[0:1], v2
@@ -582,6 +615,8 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x28
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    s_load_dword s3, s[8:9], 0x4c
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_ashr_i32 s4, s2, 24
 ; VI-NEXT:    s_bfe_i32 s5, s2, 0x80010
@@ -605,6 +640,7 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
 ; VI-NEXT:    s_and_b32 s2, s2, 0xffff
 ; VI-NEXT:    s_or_b32 s2, s2, s4
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
@@ -757,6 +793,9 @@ define amdgpu_kernel void @s_test_imin_sle_v2i16(ptr addrspace(1) %out, <2 x i16
 ; CI-LABEL: s_test_imin_sle_v2i16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_ashr_i32 s4, s2, 16
 ; CI-NEXT:    s_sext_i32_i16 s2, s2
@@ -776,6 +815,9 @@ define amdgpu_kernel void @s_test_imin_sle_v2i16(ptr addrspace(1) %out, <2 x i16
 ; VI-LABEL: s_test_imin_sle_v2i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_ashr_i32 s4, s2, 16
 ; VI-NEXT:    s_sext_i32_i16 s2, s2
@@ -857,6 +899,9 @@ define amdgpu_kernel void @s_test_imin_sle_v4i16(ptr addrspace(1) %out, <4 x i16
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_ashr_i32 s6, s0, 16
 ; CI-NEXT:    s_ashr_i32 s7, s1, 16
@@ -887,6 +932,9 @@ define amdgpu_kernel void @s_test_imin_sle_v4i16(ptr addrspace(1) %out, <4 x i16
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_ashr_i32 s6, s1, 16
 ; VI-NEXT:    s_sext_i32_i16 s1, s1
@@ -983,10 +1031,13 @@ define amdgpu_kernel void @v_test_imin_slt_i32(ptr addrspace(1) %out, ptr addrsp
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, s4, v4
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -1005,10 +1056,13 @@ define amdgpu_kernel void @v_test_imin_slt_i32(ptr addrspace(1) %out, ptr addrsp
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -1122,10 +1176,13 @@ define amdgpu_kernel void @v_test_imin_slt_i16(ptr addrspace(1) %out, ptr addrsp
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 1, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, s4, v4
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -1144,10 +1201,13 @@ define amdgpu_kernel void @v_test_imin_slt_i16(ptr addrspace(1) %out, ptr addrsp
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 1, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -1233,6 +1293,9 @@ define amdgpu_kernel void @s_test_imin_slt_i32(ptr addrspace(1) %out, i32 %a, i3
 ; CI-LABEL: s_test_imin_slt_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_i32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1244,6 +1307,9 @@ define amdgpu_kernel void @s_test_imin_slt_i32(ptr addrspace(1) %out, i32 %a, i3
 ; VI-LABEL: s_test_imin_slt_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_i32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1305,6 +1371,9 @@ define amdgpu_kernel void @s_test_imin_slt_v2i32(ptr addrspace(1) %out, <2 x i32
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_i32 s1, s1, s3
 ; CI-NEXT:    s_min_i32 s0, s0, s2
@@ -1319,6 +1388,9 @@ define amdgpu_kernel void @s_test_imin_slt_v2i32(ptr addrspace(1) %out, <2 x i32
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_i32 s1, s1, s3
 ; VI-NEXT:    s_min_i32 s0, s0, s2
@@ -1391,6 +1463,9 @@ define amdgpu_kernel void @s_test_imin_slt_imm_i32(ptr addrspace(1) %out, i32 %a
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_i32 s2, s2, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1403,6 +1478,9 @@ define amdgpu_kernel void @s_test_imin_slt_imm_i32(ptr addrspace(1) %out, i32 %a
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_i32 s2, s2, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1468,6 +1546,9 @@ define amdgpu_kernel void @s_test_imin_sle_imm_i32(ptr addrspace(1) %out, i32 %a
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_i32 s2, s2, 8
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1480,6 +1561,9 @@ define amdgpu_kernel void @s_test_imin_sle_imm_i32(ptr addrspace(1) %out, i32 %a
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_i32 s2, s2, 8
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1557,10 +1641,13 @@ define amdgpu_kernel void @v_test_umin_ule_i32(ptr addrspace(1) %out, ptr addrsp
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, s4, v4
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -1579,10 +1666,13 @@ define amdgpu_kernel void @v_test_umin_ule_i32(ptr addrspace(1) %out, ptr addrsp
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -1686,12 +1776,15 @@ define amdgpu_kernel void @v_test_umin_ule_v3i32(ptr addrspace(1) %out, ptr addr
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v6, 4, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v6
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_mov_b32_e32 v2, s5
 ; CI-NEXT:    v_add_i32_e32 v3, vcc, s4, v6
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v2, vcc
 ; CI-NEXT:    flat_load_dwordx3 v[0:2], v[0:1]
 ; CI-NEXT:    flat_load_dwordx3 v[3:5], v[3:4]
@@ -1710,12 +1803,15 @@ define amdgpu_kernel void @v_test_umin_ule_v3i32(ptr addrspace(1) %out, ptr addr
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v6, 4, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v6
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
 ; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v6
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v2, vcc
 ; VI-NEXT:    flat_load_dwordx3 v[0:2], v[0:1]
 ; VI-NEXT:    flat_load_dwordx3 v[3:5], v[3:4]
@@ -1838,12 +1934,15 @@ define amdgpu_kernel void @v_test_umin_ule_v3i16(ptr addrspace(1) %out, ptr addr
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, s4, v4
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; CI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
@@ -1874,12 +1973,15 @@ define amdgpu_kernel void @v_test_umin_ule_v3i16(ptr addrspace(1) %out, ptr addr
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v4
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
@@ -1976,6 +2078,9 @@ define amdgpu_kernel void @s_test_umin_ule_i32(ptr addrspace(1) %out, i32 %a, i3
 ; CI-LABEL: s_test_umin_ule_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_u32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1987,6 +2092,9 @@ define amdgpu_kernel void @s_test_umin_ule_i32(ptr addrspace(1) %out, i32 %a, i3
 ; VI-LABEL: s_test_umin_ule_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_u32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2059,10 +2167,13 @@ define amdgpu_kernel void @v_test_umin_ult_i32(ptr addrspace(1) %out, ptr addrsp
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v4
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, s4, v4
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -2081,10 +2192,13 @@ define amdgpu_kernel void @v_test_umin_ult_i32(ptr addrspace(1) %out, ptr addrsp
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -2188,6 +2302,9 @@ define amdgpu_kernel void @v_test_umin_ult_i8(ptr addrspace(1) %out, ptr addrspa
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s3
 ; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v0
@@ -2209,6 +2326,9 @@ define amdgpu_kernel void @v_test_umin_ult_i8(ptr addrspace(1) %out, ptr addrspa
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s3
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v0
@@ -2294,6 +2414,9 @@ define amdgpu_kernel void @s_test_umin_ult_i32(ptr addrspace(1) %out, i32 %a, i3
 ; CI-LABEL: s_test_umin_ult_i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_u32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2305,6 +2428,9 @@ define amdgpu_kernel void @s_test_umin_ult_i32(ptr addrspace(1) %out, i32 %a, i3
 ; VI-LABEL: s_test_umin_ult_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_u32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2386,6 +2512,9 @@ define amdgpu_kernel void @v_test_umin_ult_i32_multi_use(ptr addrspace(1) %out0,
 ; CI-LABEL: v_test_umin_ult_i32_multi_use:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; CI-NEXT:    s_load_dword s5, s[6:7], 0x0
@@ -2407,6 +2536,9 @@ define amdgpu_kernel void @v_test_umin_ult_i32_multi_use(ptr addrspace(1) %out0,
 ; VI-LABEL: v_test_umin_ult_i32_multi_use:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; VI-NEXT:    s_load_dword s5, s[6:7], 0x0
@@ -2534,6 +2666,9 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
 ; CI-LABEL: v_test_umin_ult_i16_multi_use:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s4
 ; CI-NEXT:    v_mov_b32_e32 v1, s5
@@ -2556,6 +2691,9 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
 ; VI-LABEL: v_test_umin_ult_i16_multi_use:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
@@ -2646,6 +2784,9 @@ define amdgpu_kernel void @s_test_umin_ult_v1i32(ptr addrspace(1) %out, <1 x i32
 ; CI-LABEL: s_test_umin_ult_v1i32:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_min_u32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2657,6 +2798,9 @@ define amdgpu_kernel void @s_test_umin_ult_v1i32(ptr addrspace(1) %out, <1 x i32
 ; VI-LABEL: s_test_umin_ult_v1i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_min_u32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -2726,6 +2870,9 @@ define amdgpu_kernel void @s_test_umin_ult_v8i32(ptr addrspace(1) %out, <8 x i32
 ;
 ; CI-LABEL: s_test_umin_ult_v8i32:
 ; CI:       ; %bb.0:
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_load_dwordx16 s[12:27], s[8:9], 0x8
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2757,6 +2904,9 @@ define amdgpu_kernel void @s_test_umin_ult_v8i32(ptr addrspace(1) %out, <8 x i32
 ;
 ; VI-LABEL: s_test_umin_ult_v8i32:
 ; VI:       ; %bb.0:
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_load_dwordx16 s[12:27], s[8:9], 0x20
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2921,6 +3071,9 @@ define amdgpu_kernel void @s_test_umin_ult_v8i16(ptr addrspace(1) %out, <8 x i16
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x4
 ; CI-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshr_b32 s10, s0, 16
 ; CI-NEXT:    s_and_b32 s0, s0, 0xffff
@@ -2967,6 +3120,9 @@ define amdgpu_kernel void @s_test_umin_ult_v8i16(ptr addrspace(1) %out, <8 x i16
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s10, s3, 16
 ; VI-NEXT:    s_and_b32 s3, s3, 0xffff
@@ -3088,11 +3244,14 @@ define amdgpu_kernel void @simplify_demanded_bits_test_umin_ult_i16(ptr addrspac
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0xa
 ; CI-NEXT:    s_load_dword s3, s[8:9], 0x13
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_and_b32 s2, s2, 0xffff
 ; CI-NEXT:    s_and_b32 s3, s3, 0xffff
 ; CI-NEXT:    s_min_u32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    flat_store_dword v[0:1], v2
@@ -3103,11 +3262,14 @@ define amdgpu_kernel void @simplify_demanded_bits_test_umin_ult_i16(ptr addrspac
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x28
 ; VI-NEXT:    s_load_dword s3, s[8:9], 0x4c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_and_b32 s2, s2, 0xffff
 ; VI-NEXT:    s_and_b32 s3, s3, 0xffff
 ; VI-NEXT:    s_min_u32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
@@ -3195,11 +3357,14 @@ define amdgpu_kernel void @simplify_demanded_bits_test_min_slt_i16(ptr addrspace
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0xa
 ; CI-NEXT:    s_load_dword s3, s[8:9], 0x13
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_sext_i32_i16 s2, s2
 ; CI-NEXT:    s_sext_i32_i16 s3, s3
 ; CI-NEXT:    s_min_i32 s2, s2, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    flat_store_dword v[0:1], v2
@@ -3210,11 +3375,14 @@ define amdgpu_kernel void @simplify_demanded_bits_test_min_slt_i16(ptr addrspace
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x28
 ; VI-NEXT:    s_load_dword s3, s[8:9], 0x4c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_sext_i32_i16 s2, s2
 ; VI-NEXT:    s_sext_i32_i16 s3, s3
 ; VI-NEXT:    s_min_i32 s2, s2, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
@@ -3309,6 +3477,9 @@ define amdgpu_kernel void @s_test_imin_sle_i16(ptr addrspace(1) %out, i16 %a, i1
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_sext_i32_i16 s3, s2
 ; CI-NEXT:    s_ashr_i32 s2, s2, 16
@@ -3323,6 +3494,9 @@ define amdgpu_kernel void @s_test_imin_sle_i16(ptr addrspace(1) %out, i16 %a, i1
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_sext_i32_i16 s3, s2
 ; VI-NEXT:    s_ashr_i32 s2, s2, 16
@@ -3403,6 +3577,9 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s4
@@ -3421,6 +3598,9 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
@@ -3510,6 +3690,9 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s4
@@ -3528,6 +3711,9 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
@@ -3617,6 +3803,9 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s4
@@ -3635,6 +3824,9 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
@@ -3724,6 +3916,9 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s4
@@ -3742,6 +3937,9 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
@@ -3855,9 +4053,12 @@ define amdgpu_kernel void @v_test_imin_sle_v2i16(ptr addrspace(1) %out, ptr addr
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    flat_load_dword v4, v[0:1]
@@ -3886,10 +4087,13 @@ define amdgpu_kernel void @v_test_imin_sle_v2i16(ptr addrspace(1) %out, ptr addr
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
@@ -4005,9 +4209,12 @@ define amdgpu_kernel void @v_test_imin_ule_v2i16(ptr addrspace(1) %out, ptr addr
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    v_mov_b32_e32 v3, s5
 ; CI-NEXT:    flat_load_dword v4, v[0:1]
@@ -4035,10 +4242,13 @@ define amdgpu_kernel void @v_test_imin_ule_v2i16(ptr addrspace(1) %out, ptr addr
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
diff --git a/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll b/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
index 337320b9eeea1..b1ce5a3423f20 100644
--- a/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
@@ -180,6 +180,9 @@ define amdgpu_kernel void @v_pack_v2f16(ptr addrspace(1) %in0, ptr addrspace(1)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -260,6 +263,9 @@ define amdgpu_kernel void @v_pack_v2f16_user(ptr addrspace(1) %in0, ptr addrspac
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -341,6 +347,9 @@ define amdgpu_kernel void @v_pack_v2f16_imm_lo(ptr addrspace(1) %in1) #0 {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -403,6 +412,9 @@ define amdgpu_kernel void @v_pack_v2f16_inline_imm_lo(ptr addrspace(1) %in1) #0
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -465,6 +477,9 @@ define amdgpu_kernel void @v_pack_v2f16_imm_hi(ptr addrspace(1) %in0) #0 {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -527,6 +542,9 @@ define amdgpu_kernel void @v_pack_v2f16_inline_f16imm_hi(ptr addrspace(1) %in0)
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -588,6 +606,9 @@ define amdgpu_kernel void @v_pack_v2f16_inline_imm_hi(ptr addrspace(1) %in0) #0
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll b/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
index bc1710686a087..5803821a1d2c0 100644
--- a/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
@@ -176,6 +176,9 @@ define amdgpu_kernel void @v_pack_v2i16(ptr addrspace(1) %in0, ptr addrspace(1)
 ; GFX803:       ; %bb.0:
 ; GFX803-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX803-NEXT:    s_add_i32 s12, s12, s17
+; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX803-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -254,6 +257,9 @@ define amdgpu_kernel void @v_pack_v2i16_user(ptr addrspace(1) %in0, ptr addrspac
 ; GFX803:       ; %bb.0:
 ; GFX803-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX803-NEXT:    s_add_i32 s12, s12, s17
+; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX803-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
@@ -333,6 +339,9 @@ define amdgpu_kernel void @v_pack_v2i16_imm_lo(ptr addrspace(1) %in1) #0 {
 ; GFX803:       ; %bb.0:
 ; GFX803-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX803-NEXT:    s_add_i32 s12, s12, s17
+; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX803-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -393,6 +402,9 @@ define amdgpu_kernel void @v_pack_v2i16_inline_imm_lo(ptr addrspace(1) %in1) #0
 ; GFX803:       ; %bb.0:
 ; GFX803-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX803-NEXT:    s_add_i32 s12, s12, s17
+; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX803-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -454,6 +466,9 @@ define amdgpu_kernel void @v_pack_v2i16_imm_hi(ptr addrspace(1) %in0) #0 {
 ; GFX803:       ; %bb.0:
 ; GFX803-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX803-NEXT:    s_add_i32 s12, s12, s17
+; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX803-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
@@ -514,6 +529,9 @@ define amdgpu_kernel void @v_pack_v2i16_inline_imm_hi(ptr addrspace(1) %in0) #0
 ; GFX803:       ; %bb.0:
 ; GFX803-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GFX803-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX803-NEXT:    s_add_i32 s12, s12, s17
+; GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX803-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/pal-simple-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/pal-simple-indirect-call.ll
index 2e9f09ad41813..7c9ecc892478c 100644
--- a/llvm/test/CodeGen/AMDGPU/pal-simple-indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/pal-simple-indirect-call.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
 ; Check that no attributes are added to graphics functions
-; RUN: opt -S -mtriple=amdgcn-amd-amdpal -amdgpu-annotate-kernel-features  %s | FileCheck -check-prefixes=AKF_GCN %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdpal -passes=amdgpu-attributor %s | FileCheck -check-prefixes=ATTRIBUTOR_GCN %s
 
 ; Check that it doesn't crash
@@ -12,12 +11,6 @@ target datalayout = "A5"
 
 
 define amdgpu_cs void @test_simple_indirect_call() {
-; AKF_GCN-LABEL: define {{[^@]+}}@test_simple_indirect_call() {
-; AKF_GCN-NEXT:    [[PC:%.*]] = call i64 @llvm.amdgcn.s.getpc()
-; AKF_GCN-NEXT:    [[FUN:%.*]] = inttoptr i64 [[PC]] to ptr
-; AKF_GCN-NEXT:    call amdgpu_gfx void [[FUN]]()
-; AKF_GCN-NEXT:    ret void
-;
 ; ATTRIBUTOR_GCN-LABEL: define {{[^@]+}}@test_simple_indirect_call
 ; ATTRIBUTOR_GCN-SAME: () #[[ATTR0:[0-9]+]] {
 ; ATTRIBUTOR_GCN-NEXT:    [[PC:%.*]] = call i64 @llvm.amdgcn.s.getpc()
@@ -68,7 +61,6 @@ declare i64 @llvm.amdgcn.s.getpc() #0
 
 attributes #0 = { nounwind readnone speculatable willreturn }
 ;.
-; AKF_GCN: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ;.
 ; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_GCN: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 1a34fa3bbbf4d..24677b60be6c2 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -10,32 +10,33 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX908: bb.0 (%ir-block.0):
   ; REGALLOC-GFX908-NEXT:   liveins: $sgpr4_sgpr5
   ; REGALLOC-GFX908-NEXT: {{  $}}
-  ; REGALLOC-GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef %5:agpr_32
-  ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %6
-  ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3538954 /* regdef:VReg_64 */, def %7
-  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %14:vreg_64, %6, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+  ; REGALLOC-GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef %6:agpr_32
+  ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %7
+  ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3538954 /* regdef:VReg_64 */, def %8
+
+  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
   ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
   ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
   ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
-  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX2 undef %16:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
-  ; REGALLOC-GFX908-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
-  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %18:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+  ; REGALLOC-GFX908-NEXT:   [[COPY3:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
+  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY3]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   S_ENDPGM 0
   ;
   ; PEI-GFX908-LABEL: name: partial_copy
   ; PEI-GFX908: bb.0 (%ir-block.0):
-  ; PEI-GFX908-NEXT:   liveins: $agpr4, $sgpr4_sgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr7
+  ; PEI-GFX908-NEXT:   liveins: $agpr4, $sgpr4_sgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr9
   ; PEI-GFX908-NEXT: {{  $}}
-  ; PEI-GFX908-NEXT:   $sgpr8_sgpr9_sgpr10_sgpr11 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3
-  ; PEI-GFX908-NEXT:   $sgpr8 = S_ADD_U32 $sgpr8, $sgpr7, implicit-def $scc, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
-  ; PEI-GFX908-NEXT:   $sgpr9 = S_ADDC_U32 $sgpr9, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
+  ; PEI-GFX908-NEXT:   $sgpr12_sgpr13_sgpr14_sgpr15 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3
+  ; PEI-GFX908-NEXT:   $sgpr12 = S_ADD_U32 $sgpr12, $sgpr9, implicit-def $scc, implicit-def $sgpr12_sgpr13_sgpr14_sgpr15
+  ; PEI-GFX908-NEXT:   $sgpr13 = S_ADDC_U32 $sgpr13, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr12_sgpr13_sgpr14_sgpr15
   ; PEI-GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef renamable $agpr0
   ; PEI-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def renamable $vgpr0_vgpr1_vgpr2_vgpr3
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
   ; PEI-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3538954 /* regdef:VReg_64 */, def renamable $vgpr0_vgpr1
-  ; PEI-GFX908-NEXT:   BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store (s32) into %stack.0, addrspace 5)
+  ; PEI-GFX908-NEXT:   BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store (s32) into %stack.0, addrspace 5)
   ; PEI-GFX908-NEXT:   $agpr4 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr1, implicit $exec, implicit killed $vgpr0_vgpr1
   ; PEI-GFX908-NEXT:   renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3, implicit $exec
   ; PEI-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
@@ -44,7 +45,7 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX908-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
   ; PEI-GFX908-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
-  ; PEI-GFX908-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
+  ; PEI-GFX908-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
   ; PEI-GFX908-NEXT:   $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
   ; PEI-GFX908-NEXT:   GLOBAL_STORE_DWORDX2 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX908-NEXT:   renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3, implicit $exec
@@ -55,31 +56,31 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX90A: bb.0 (%ir-block.0):
   ; REGALLOC-GFX90A-NEXT:   liveins: $sgpr4_sgpr5
   ; REGALLOC-GFX90A-NEXT: {{  $}}
-  ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef %5:agpr_32
-  ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def %6
-  ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3866634 /* regdef:VReg_64_Align2 */, def %7
-  ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %14:vreg_64_align2, %6, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+  ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef %6:agpr_32
+  ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def %7
+  ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3866634 /* regdef:VReg_64_Align2 */, def %8
+  ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX90A-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
   ; REGALLOC-GFX90A-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
   ; REGALLOC-GFX90A-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
   ; REGALLOC-GFX90A-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
-  ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX2 undef %16:vreg_64_align2, %7, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
-  ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %18:vreg_64_align2, [[V_MFMA_I32_4X4X4I8_e64_]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+  ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64_align2, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+  ; REGALLOC-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64_align2, [[V_MFMA_I32_4X4X4I8_e64_]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX90A-NEXT:   S_ENDPGM 0
   ;
   ; PEI-GFX90A-LABEL: name: partial_copy
   ; PEI-GFX90A: bb.0 (%ir-block.0):
-  ; PEI-GFX90A-NEXT:   liveins: $agpr4, $sgpr4_sgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr7
+  ; PEI-GFX90A-NEXT:   liveins: $agpr4, $sgpr4_sgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr9
   ; PEI-GFX90A-NEXT: {{  $}}
-  ; PEI-GFX90A-NEXT:   $sgpr8_sgpr9_sgpr10_sgpr11 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3
-  ; PEI-GFX90A-NEXT:   $sgpr8 = S_ADD_U32 $sgpr8, $sgpr7, implicit-def $scc, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
-  ; PEI-GFX90A-NEXT:   $sgpr9 = S_ADDC_U32 $sgpr9, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
+  ; PEI-GFX90A-NEXT:   $sgpr12_sgpr13_sgpr14_sgpr15 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3
+  ; PEI-GFX90A-NEXT:   $sgpr12 = S_ADD_U32 $sgpr12, $sgpr9, implicit-def $scc, implicit-def $sgpr12_sgpr13_sgpr14_sgpr15
+  ; PEI-GFX90A-NEXT:   $sgpr13 = S_ADDC_U32 $sgpr13, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr12_sgpr13_sgpr14_sgpr15
   ; PEI-GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef renamable $agpr0
   ; PEI-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def renamable $vgpr0_vgpr1_vgpr2_vgpr3
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
   ; PEI-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3866634 /* regdef:VReg_64_Align2 */, def renamable $vgpr0_vgpr1
-  ; PEI-GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store (s32) into %stack.0, addrspace 5)
+  ; PEI-GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store (s32) into %stack.0, addrspace 5)
   ; PEI-GFX90A-NEXT:   $agpr4 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr1, implicit $exec, implicit killed $vgpr0_vgpr1
   ; PEI-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX90A-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
@@ -87,7 +88,7 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
   ; PEI-GFX90A-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
-  ; PEI-GFX90A-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
+  ; PEI-GFX90A-NEXT:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
   ; PEI-GFX90A-NEXT:   $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
   ; PEI-GFX90A-NEXT:   GLOBAL_STORE_DWORDX2 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
   ; PEI-GFX90A-NEXT:   GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index 00507c1eafd6e..c26f0926d86b2 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -19,16 +19,16 @@ define amdgpu_kernel void @preload_block_count_x(ptr addrspace(1) inreg %out) #0
 ;
 ; GFX90a-LABEL: preload_block_count_x:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB0_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB0_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
@@ -54,17 +54,16 @@ define amdgpu_kernel void @preload_unused_arg_block_count_x(ptr addrspace(1) inr
 ;
 ; GFX90a-LABEL: preload_unused_arg_block_count_x:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s12, s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB1_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB1_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s12
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
@@ -90,7 +89,7 @@ define amdgpu_kernel void @no_free_sgprs_block_count_x(ptr addrspace(1) inreg %o
 ;
 ; GFX90a-LABEL: no_free_sgprs_block_count_x:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx4 s[12:15], s[8:9], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[14:15], s[8:9], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB2_0
 ; GFX90a-NEXT:    .p2align 8
@@ -100,7 +99,7 @@ define amdgpu_kernel void @no_free_sgprs_block_count_x(ptr addrspace(1) inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[12:13]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[14:15]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
@@ -181,7 +180,7 @@ define amdgpu_kernel void @incorrect_type_i64_block_count_x(ptr addrspace(1) inr
 ;
 ; GFX90a-LABEL: incorrect_type_i64_block_count_x:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB5_0
 ; GFX90a-NEXT:    .p2align 8
@@ -191,7 +190,7 @@ define amdgpu_kernel void @incorrect_type_i64_block_count_x(ptr addrspace(1) inr
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i64, ptr addrspace(4) %imp_arg_ptr
@@ -217,7 +216,7 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
 ;
 ; GFX90a-LABEL: incorrect_type_i16_block_count_x:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB6_0
 ; GFX90a-NEXT:    .p2align 8
@@ -227,7 +226,7 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i16, ptr addrspace(4) %imp_arg_ptr
@@ -252,16 +251,15 @@ define amdgpu_kernel void @preload_block_count_y(ptr addrspace(1) inreg %out) #0
 ;
 ; GFX90a-LABEL: preload_block_count_y:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB7_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB7_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 4
@@ -289,7 +287,7 @@ define amdgpu_kernel void @random_incorrect_offset(ptr addrspace(1) inreg %out)
 ;
 ; GFX90a-LABEL: random_incorrect_offset:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB8_0
 ; GFX90a-NEXT:    .p2align 8
@@ -300,7 +298,7 @@ define amdgpu_kernel void @random_incorrect_offset(ptr addrspace(1) inreg %out)
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 2
@@ -327,17 +325,16 @@ define amdgpu_kernel void @preload_block_count_z(ptr addrspace(1) inreg %out) #0
 ;
 ; GFX90a-LABEL: preload_block_count_z:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s12, s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB9_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB9_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s12
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 8
@@ -366,19 +363,18 @@ define amdgpu_kernel void @preload_block_count_x_imparg_align_ptr_i8(ptr addrspa
 ;
 ; GFX90a-LABEL: preload_block_count_x_imparg_align_ptr_i8:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s12, s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB10_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB10_0:
-; GFX90a-NEXT:    s_and_b32 s0, s8, 0xff
-; GFX90a-NEXT:    s_add_i32 s0, s10, s0
+; GFX90a-NEXT:    s_and_b32 s0, s10, 0xff
+; GFX90a-NEXT:    s_add_i32 s0, s12, s0
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
@@ -408,19 +404,18 @@ define amdgpu_kernel void @preload_block_count_xyz(ptr addrspace(1) inreg %out)
 ;
 ; GFX90a-LABEL: preload_block_count_xyz:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s12, s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB11_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB11_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep_x = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 0
@@ -454,17 +449,17 @@ define amdgpu_kernel void @preload_workgroup_size_x(ptr addrspace(1) inreg %out)
 ;
 ; GFX90a-LABEL: preload_workgroup_size_x:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB12_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB12_0:
-; GFX90a-NEXT:    s_and_b32 s0, s11, 0xffff
+; GFX90a-NEXT:    s_and_b32 s0, s13, 0xffff
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 12
@@ -492,17 +487,17 @@ define amdgpu_kernel void @preload_workgroup_size_y(ptr addrspace(1) inreg %out)
 ;
 ; GFX90a-LABEL: preload_workgroup_size_y:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB13_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB13_0:
-; GFX90a-NEXT:    s_lshr_b32 s0, s11, 16
+; GFX90a-NEXT:    s_lshr_b32 s0, s13, 16
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 14
@@ -531,18 +526,18 @@ define amdgpu_kernel void @preload_workgroup_size_z(ptr addrspace(1) inreg %out)
 ;
 ; GFX90a-LABEL: preload_workgroup_size_z:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dword s12, s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dword s14, s[4:5], 0x18
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB14_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB14_0:
-; GFX90a-NEXT:    s_and_b32 s0, s12, 0xffff
+; GFX90a-NEXT:    s_and_b32 s0, s14, 0xffff
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 16
@@ -575,22 +570,22 @@ define amdgpu_kernel void @preload_workgroup_size_xyz(ptr addrspace(1) inreg %ou
 ;
 ; GFX90a-LABEL: preload_workgroup_size_xyz:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dword s12, s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dword s14, s[4:5], 0x18
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB15_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB15_0:
-; GFX90a-NEXT:    s_lshr_b32 s0, s11, 16
-; GFX90a-NEXT:    s_and_b32 s1, s11, 0xffff
-; GFX90a-NEXT:    s_and_b32 s2, s12, 0xffff
+; GFX90a-NEXT:    s_lshr_b32 s0, s13, 16
+; GFX90a-NEXT:    s_and_b32 s1, s13, 0xffff
+; GFX90a-NEXT:    s_and_b32 s2, s14, 0xffff
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep_x = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 12
@@ -628,18 +623,18 @@ define amdgpu_kernel void @preload_remainder_x(ptr addrspace(1) inreg %out) #0 {
 ;
 ; GFX90a-LABEL: preload_remainder_x:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dword s12, s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dword s14, s[4:5], 0x18
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB16_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB16_0:
-; GFX90a-NEXT:    s_lshr_b32 s0, s12, 16
+; GFX90a-NEXT:    s_lshr_b32 s0, s14, 16
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 18
@@ -668,18 +663,16 @@ define amdgpu_kernel void @preloadremainder_y(ptr addrspace(1) inreg %out) #0 {
 ;
 ; GFX90a-LABEL: preloadremainder_y:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB17_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB17_0:
-; GFX90a-NEXT:    s_and_b32 s0, s13, 0xffff
+; GFX90a-NEXT:    s_and_b32 s0, s15, 0xffff
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 20
@@ -708,18 +701,16 @@ define amdgpu_kernel void @preloadremainder_z(ptr addrspace(1) inreg %out) #0 {
 ;
 ; GFX90a-LABEL: preloadremainder_z:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB18_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB18_0:
-; GFX90a-NEXT:    s_lshr_b32 s0, s13, 16
+; GFX90a-NEXT:    s_lshr_b32 s0, s15, 16
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 22
@@ -752,22 +743,20 @@ define amdgpu_kernel void @preloadremainder_xyz(ptr addrspace(1) inreg %out) #0
 ;
 ; GFX90a-LABEL: preloadremainder_xyz:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB19_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB19_0:
-; GFX90a-NEXT:    s_lshr_b32 s0, s13, 16
-; GFX90a-NEXT:    s_lshr_b32 s1, s12, 16
-; GFX90a-NEXT:    s_and_b32 s2, s13, 0xffff
+; GFX90a-NEXT:    s_lshr_b32 s0, s15, 16
+; GFX90a-NEXT:    s_lshr_b32 s1, s14, 16
+; GFX90a-NEXT:    s_and_b32 s2, s15, 0xffff
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s0
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep_x = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 18
@@ -803,7 +792,7 @@ define amdgpu_kernel void @no_free_sgprs_preloadremainder_z(ptr addrspace(1) inr
 ;
 ; GFX90a-LABEL: no_free_sgprs_preloadremainder_z:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[8:9], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[14:15], s[8:9], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB20_0
 ; GFX90a-NEXT:    .p2align 8
@@ -814,7 +803,7 @@ define amdgpu_kernel void @no_free_sgprs_preloadremainder_z(ptr addrspace(1) inr
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_lshr_b32 s0, s0, 16
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[12:13]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[14:15]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 22
@@ -844,10 +833,7 @@ define amdgpu_kernel void @preload_block_max_user_sgprs(ptr addrspace(1) inreg %
 ;
 ; GFX90a-LABEL: preload_block_max_user_sgprs:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
-; GFX90a-NEXT:    s_load_dword s14, s[4:5], 0x20
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB21_0
 ; GFX90a-NEXT:    .p2align 8
@@ -857,7 +843,7 @@ define amdgpu_kernel void @preload_block_max_user_sgprs(ptr addrspace(1) inreg %
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
@@ -887,21 +873,23 @@ define amdgpu_kernel void @preload_block_count_z_workgroup_size_z_remainder_z(pt
 ;
 ; GFX90a-LABEL: preload_block_count_z_workgroup_size_z_remainder_z:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dword s14, s[4:5], 0x18
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB22_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB22_0:
-; GFX90a-NEXT:    s_lshr_b32 s0, s13, 16
-; GFX90a-NEXT:    s_and_b32 s1, s12, 0xffff
+; GFX90a-NEXT:    s_load_dword s0, s[4:5], 0x1c
+; GFX90a-NEXT:    s_and_b32 s1, s14, 0xffff
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s1
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    s_lshr_b32 s0, s0, 16
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s0
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep0 = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 8
diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
index fe6378435a42e..7ae0c11dca279 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
@@ -21,17 +21,17 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) inreg %out, i8 inreg %arg0)
 ;
 ; GFX90a-LABEL: ptr1_i8:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB0_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB0_0:
-; GFX90a-NEXT:    s_and_b32 s0, s8, 0xff
+; GFX90a-NEXT:    s_and_b32 s0, s10, 0xff
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out
@@ -56,17 +56,17 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) inreg %out, i8 zero
 ;
 ; GFX90a-LABEL: ptr1_i8_zext_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB1_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB1_0:
-; GFX90a-NEXT:    s_and_b32 s0, s8, 0xff
+; GFX90a-NEXT:    s_and_b32 s0, s10, 0xff
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out, align 4
@@ -91,17 +91,17 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) inreg %out, i16
 ;
 ; GFX90a-LABEL: ptr1_i16_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB2_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB2_0:
-; GFX90a-NEXT:    s_and_b32 s0, s8, 0xffff
+; GFX90a-NEXT:    s_and_b32 s0, s10, 0xffff
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %ext = zext i16 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out, align 4
@@ -125,16 +125,16 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) inreg %out, i32
 ;
 ; GFX90a-LABEL: ptr1_i32_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB3_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB3_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store i32 %arg0, ptr addrspace(1) %out
   ret void
@@ -160,18 +160,17 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 inreg %arg0, ptr addrspa
 ;
 ; GFX90a-LABEL: i32_ptr1_i32_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s12, s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB4_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB4_0:
-; GFX90a-NEXT:    s_add_i32 s0, s6, s10
+; GFX90a-NEXT:    s_add_i32 s0, s8, s12
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[10:11]
 ; GFX90a-NEXT:    s_endpgm
   %add = add i32 %arg0, %arg1
   store i32 %add, ptr addrspace(1) %out
@@ -198,19 +197,19 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) inreg %out,
 ;
 ; GFX90a-LABEL: ptr1_i16_i16_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB5_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB5_0:
-; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
-; GFX90a-NEXT:    s_and_b32 s1, s8, 0xffff
+; GFX90a-NEXT:    s_lshr_b32 s0, s10, 16
+; GFX90a-NEXT:    s_and_b32 s1, s10, 0xffff
 ; GFX90a-NEXT:    s_add_i32 s0, s1, s0
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %ext = zext i16 %arg0 to i32
   %ext1 = zext i16 %arg1 to i32
@@ -236,16 +235,16 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) inreg %out, <2
 ;
 ; GFX90a-LABEL: ptr1_v2i8_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB6_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB6_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <2 x i8> %in, ptr addrspace(1) %out
   ret void
@@ -274,7 +273,7 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) inreg %out, ptr ad
 ;
 ; GFX90a-LABEL: byref_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB7_0
 ; GFX90a-NEXT:    .p2align 8
@@ -285,9 +284,9 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) inreg %out, ptr ad
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-NEXT:    global_store_dword v0, v2, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v2, s[8:9]
 ; GFX90a-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90a-NEXT:    s_endpgm
   %in = load i32, ptr addrspace(4) %in.byref
@@ -320,7 +319,7 @@ define amdgpu_kernel void @byref_staggered_preload_arg(ptr addrspace(1) inreg %o
 ;
 ; GFX90a-LABEL: byref_staggered_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB8_0
 ; GFX90a-NEXT:    .p2align 8
@@ -331,9 +330,9 @@ define amdgpu_kernel void @byref_staggered_preload_arg(ptr addrspace(1) inreg %o
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-NEXT:    global_store_dword v0, v2, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v2, s[8:9]
 ; GFX90a-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90a-NEXT:    s_endpgm
   %in = load i32, ptr addrspace(4) %in.byref
@@ -370,26 +369,26 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture inreg %out, <8 x
 ;
 ; GFX90a-LABEL: v8i32_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB9_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB9_0:
-; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; GFX90a-NEXT:    s_load_dwordx8 s[12:19], s[4:5], 0x20
 ; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s16
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s17
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s18
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s19
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9] offset:16
+; GFX90a-NEXT:    s_nop 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
-; GFX90a-NEXT:    s_nop 0
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <8 x i32> %in, ptr addrspace(1) %out, align 4
   ret void
@@ -414,18 +413,17 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture inreg %o
 ;
 ; GFX90a-LABEL: v3i16_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB10_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB10_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7] offset:4
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9] offset:4
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <3 x i16> %in, ptr addrspace(1) %out, align 4
   ret void
@@ -451,19 +449,17 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture inreg %o
 ;
 ; GFX90a-LABEL: v3i32_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB11_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB11_0:
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <3 x i32> %in, ptr addrspace(1) %out, align 4
   ret void
@@ -489,19 +485,17 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture inreg %o
 ;
 ; GFX90a-LABEL: v3f32_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB12_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB12_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <3 x float> %in, ptr addrspace(1) %out, align 4
   ret void
@@ -533,25 +527,24 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture inreg %ou
 ;
 ; GFX90a-LABEL: v5i8_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB13_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB13_0:
-; GFX90a-NEXT:    s_lshr_b32 s1, s8, 24
+; GFX90a-NEXT:    s_lshr_b32 s1, s10, 24
 ; GFX90a-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX90a-NEXT:    s_bfe_u32 s2, s8, 0x80010
+; GFX90a-NEXT:    s_bfe_u32 s2, s10, 0x80010
 ; GFX90a-NEXT:    s_or_b32 s1, s2, s1
-; GFX90a-NEXT:    s_and_b32 s0, s8, 0xffff
+; GFX90a-NEXT:    s_and_b32 s0, s10, 0xffff
 ; GFX90a-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX90a-NEXT:    s_or_b32 s0, s0, s1
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    global_store_byte v0, v1, s[6:7] offset:4
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_byte v0, v1, s[8:9] offset:4
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <5 x i8> %in, ptr addrspace(1) %out, align 4
   ret void
@@ -587,29 +580,29 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture inreg %out, <5 x
 ;
 ; GFX90a-LABEL: v5f64_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB14_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB14_0:
 ; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
+; GFX90a-NEXT:    s_load_dwordx8 s[12:19], s[4:5], 0x40
 ; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s16
+; GFX90a-NEXT:    global_store_dwordx2 v4, v[2:3], s[8:9] offset:32
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s17
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s18
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s19
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9] offset:16
+; GFX90a-NEXT:    s_nop 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-NEXT:    global_store_dwordx2 v4, v[2:3], s[6:7] offset:32
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
-; GFX90a-NEXT:    s_nop 0
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <5 x double> %in, ptr addrspace(1) %out, align 8
   ret void
@@ -647,31 +640,30 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) inreg %out, <8 x i8
 ;
 ; GFX90a-LABEL: v8i8_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB15_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB15_0:
-; GFX90a-NEXT:    s_lshr_b32 s1, s9, 24
+; GFX90a-NEXT:    s_lshr_b32 s1, s11, 24
 ; GFX90a-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX90a-NEXT:    s_bfe_u32 s2, s9, 0x80010
+; GFX90a-NEXT:    s_bfe_u32 s2, s11, 0x80010
 ; GFX90a-NEXT:    s_or_b32 s1, s2, s1
-; GFX90a-NEXT:    s_lshr_b32 s2, s8, 24
+; GFX90a-NEXT:    s_lshr_b32 s2, s10, 24
 ; GFX90a-NEXT:    s_lshl_b32 s2, s2, 8
-; GFX90a-NEXT:    s_bfe_u32 s3, s8, 0x80010
-; GFX90a-NEXT:    s_and_b32 s0, s9, 0xffff
+; GFX90a-NEXT:    s_bfe_u32 s3, s10, 0x80010
+; GFX90a-NEXT:    s_and_b32 s0, s11, 0xffff
 ; GFX90a-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX90a-NEXT:    s_or_b32 s2, s3, s2
 ; GFX90a-NEXT:    s_or_b32 s0, s0, s1
-; GFX90a-NEXT:    s_and_b32 s1, s8, 0xffff
+; GFX90a-NEXT:    s_and_b32 s1, s10, 0xffff
 ; GFX90a-NEXT:    s_lshl_b32 s2, s2, 16
 ; GFX90a-NEXT:    s_or_b32 s1, s1, s2
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <8 x i8> %in, ptr addrspace(1) %out
   ret void
@@ -694,16 +686,15 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) inreg %out, i
 ;
 ; GFX90a-LABEL: i64_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB16_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB16_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store i64 %a, ptr addrspace(1) %out, align 8
   ret void
@@ -726,16 +717,15 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) inreg %out, d
 ;
 ; GFX90a-LABEL: f64_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB17_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB17_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store double %in, ptr addrspace(1) %out
   ret void
@@ -758,16 +748,16 @@ define amdgpu_kernel void @half_kernel_preload_arg(ptr addrspace(1) inreg %out,
 ;
 ; GFX90a-LABEL: half_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB18_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB18_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store half %in, ptr addrspace(1) %out
   ret void
@@ -790,16 +780,16 @@ define amdgpu_kernel void @bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out
 ;
 ; GFX90a-LABEL: bfloat_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB19_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB19_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store bfloat %in, ptr addrspace(1) %out
   ret void
@@ -822,16 +812,16 @@ define amdgpu_kernel void @v2bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
 ;
 ; GFX90a-LABEL: v2bfloat_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB20_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB20_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <2 x bfloat> %in, ptr addrspace(1) %out
   ret void
@@ -856,18 +846,17 @@ define amdgpu_kernel void @v3bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
 ;
 ; GFX90a-LABEL: v3bfloat_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB21_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB21_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7] offset:4
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9] offset:4
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <3 x bfloat> %in, ptr addrspace(1) %out
   ret void
@@ -893,19 +882,17 @@ define amdgpu_kernel void @v6bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
 ;
 ; GFX90a-LABEL: v6bfloat_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB22_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB22_0:
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <6 x bfloat> %in, ptr addrspace(1) %out
   ret void
@@ -934,24 +921,24 @@ define amdgpu_kernel void @half_v7bfloat_kernel_preload_arg(ptr addrspace(1) inr
 ;
 ; GFX90a-LABEL: half_v7bfloat_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB23_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB23_0:
-; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x20
+; GFX90a-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x20
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-NEXT:    global_store_short v3, v0, s[6:7]
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s13
-; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NEXT:    global_store_short v3, v0, s[0:1] offset:12
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[0:1]
+; GFX90a-NEXT:    global_store_short v3, v0, s[8:9]
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s3
+; GFX90a-NEXT:    global_store_short v3, v0, s[6:7] offset:12
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s2
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s1
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
 ; GFX90a-NEXT:    s_endpgm
   store half %in, ptr addrspace(1) %out
   store <7 x bfloat> %in2, ptr addrspace(1) %out2
@@ -976,17 +963,17 @@ define amdgpu_kernel void @i1_kernel_preload_arg(ptr addrspace(1) inreg %out, i1
 ;
 ; GFX90a-LABEL: i1_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s10, s[4:5], 0x8
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB24_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB24_0:
-; GFX90a-NEXT:    s_and_b32 s0, s8, 1
+; GFX90a-NEXT:    s_and_b32 s0, s10, 1
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_byte v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_byte v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store i1 %in, ptr addrspace(1) %out
   ret void
@@ -1013,20 +1000,18 @@ define amdgpu_kernel void @fp128_kernel_preload_arg(ptr addrspace(1) inreg %out,
 ;
 ; GFX90a-LABEL: fp128_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB25_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB25_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
-; GFX90a-NEXT:    v_mov_b32_e32 v3, s13
-; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store fp128 %in, ptr addrspace(1) %out
   ret void
@@ -1059,26 +1044,25 @@ define amdgpu_kernel void @v7i8_kernel_preload_arg(ptr addrspace(1) inreg %out,
 ;
 ; GFX90a-LABEL: v7i8_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB26_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB26_0:
-; GFX90a-NEXT:    s_lshr_b32 s1, s8, 24
+; GFX90a-NEXT:    s_lshr_b32 s1, s10, 24
 ; GFX90a-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX90a-NEXT:    s_bfe_u32 s2, s8, 0x80010
+; GFX90a-NEXT:    s_bfe_u32 s2, s10, 0x80010
 ; GFX90a-NEXT:    s_or_b32 s1, s2, s1
-; GFX90a-NEXT:    s_and_b32 s0, s8, 0xffff
+; GFX90a-NEXT:    s_and_b32 s0, s10, 0xffff
 ; GFX90a-NEXT:    s_lshl_b32 s1, s1, 16
 ; GFX90a-NEXT:    s_or_b32 s0, s0, s1
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    global_store_byte_d16_hi v0, v1, s[6:7] offset:6
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_byte_d16_hi v0, v1, s[8:9] offset:6
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9] offset:4
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <7 x i8> %in, ptr addrspace(1) %out
   ret void
@@ -1106,21 +1090,19 @@ define amdgpu_kernel void @v7half_kernel_preload_arg(ptr addrspace(1) inreg %out
 ;
 ; GFX90a-LABEL: v7half_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x18
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB27_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB27_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s13
-; GFX90a-NEXT:    global_store_short v3, v0, s[6:7] offset:12
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s15
+; GFX90a-NEXT:    global_store_short v3, v0, s[8:9] offset:12
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   store <7 x half> %in, ptr addrspace(1) %out
   ret void
@@ -1145,18 +1127,18 @@ define amdgpu_kernel void @i16_i32_kernel_preload_arg(ptr addrspace(1) inreg %ou
 ;
 ; GFX90a-LABEL: i16_i32_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB28_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB28_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NEXT:    global_store_dword v0, v1, s[10:11]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_dword v0, v1, s[12:13]
 ; GFX90a-NEXT:    s_endpgm
   store i16 %in, ptr addrspace(1) %out
   store i32 %in2, ptr addrspace(1) %out2
@@ -1184,22 +1166,22 @@ define amdgpu_kernel void @i16_v3i32_kernel_preload_arg(ptr addrspace(1) inreg %
 ;
 ; GFX90a-LABEL: i16_v3i32_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB29_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB29_0:
-; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x20
+; GFX90a-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v4, s8
-; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
-; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
-; GFX90a-NEXT:    global_store_short v3, v4, s[6:7]
+; GFX90a-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x20
+; GFX90a-NEXT:    v_mov_b32_e32 v4, s10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[0:1]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s1
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s2
+; GFX90a-NEXT:    global_store_short v3, v4, s[8:9]
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[4:5]
 ; GFX90a-NEXT:    s_endpgm
   store i16 %in, ptr addrspace(1) %out
   store <3 x i32> %in2, ptr addrspace(1) %out2
@@ -1224,17 +1206,17 @@ define amdgpu_kernel void @i16_i16_kernel_preload_arg(ptr addrspace(1) inreg %ou
 ;
 ; GFX90a-LABEL: i16_i16_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB30_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB30_0:
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
-; GFX90a-NEXT:    global_store_short_d16_hi v0, v1, s[10:11]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
+; GFX90a-NEXT:    global_store_short_d16_hi v0, v1, s[12:13]
 ; GFX90a-NEXT:    s_endpgm
   store i16 %in, ptr addrspace(1) %out
   store i16 %in2, ptr addrspace(1) %out2
@@ -1264,22 +1246,22 @@ define amdgpu_kernel void @i16_v2i8_kernel_preload_arg(ptr addrspace(1) inreg %o
 ;
 ; GFX90a-LABEL: i16_v2i8_kernel_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x10
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB31_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB31_0:
-; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    s_lshr_b32 s0, s10, 24
 ; GFX90a-NEXT:    s_lshl_b32 s0, s0, 8
-; GFX90a-NEXT:    s_bfe_u32 s1, s8, 0x80010
+; GFX90a-NEXT:    s_bfe_u32 s1, s10, 0x80010
 ; GFX90a-NEXT:    s_or_b32 s0, s1, s0
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
-; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
+; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_short v0, v1, s[10:11]
+; GFX90a-NEXT:    global_store_short v0, v1, s[12:13]
 ; GFX90a-NEXT:    s_endpgm
   store i16 %in, ptr addrspace(1) %out
   store <2 x i8> %in2, ptr addrspace(1) %out2
@@ -1308,7 +1290,7 @@ define amdgpu_kernel void @i32_ptr1_i32_staggered_preload_arg(i32 inreg %arg0, p
 ;
 ; GFX90a-LABEL: i32_ptr1_i32_staggered_preload_arg:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dword s6, s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dword s8, s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB32_0
 ; GFX90a-NEXT:    .p2align 8
@@ -1318,7 +1300,7 @@ define amdgpu_kernel void @i32_ptr1_i32_staggered_preload_arg(i32 inreg %arg0, p
 ; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NEXT:    s_add_i32 s2, s6, s2
+; GFX90a-NEXT:    s_add_i32 s2, s8, s2
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX90a-NEXT:    s_endpgm
@@ -1345,17 +1327,16 @@ define amdgpu_kernel void @ptr1_i8_trailing_unused(ptr addrspace(1) inreg %out,
 ;
 ; GFX90a-LABEL: ptr1_i8_trailing_unused:
 ; GFX90a:       ; %bb.1:
-; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x8
+; GFX90a-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x0
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    s_branch .LBB33_0
 ; GFX90a-NEXT:    .p2align 8
 ; GFX90a-NEXT:  ; %bb.2:
 ; GFX90a-NEXT:  .LBB33_0:
-; GFX90a-NEXT:    s_and_b32 s0, s8, 0xff
+; GFX90a-NEXT:    s_and_b32 s0, s10, 0xff
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index 5474338514522..8f25e6519588b 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -6,6 +6,9 @@ define amdgpu_kernel void @v_sad_u32_pat1(ptr addrspace(1) %out, i32 %a, i32 %b,
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v1, s2
@@ -33,9 +36,12 @@ define amdgpu_kernel void @v_sad_u32_constant_pat1(ptr addrspace(1) %out, i32 %a
 ; GCN-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0x5a
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_sad_u32 v2, s2, v0, 20
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    flat_store_dword v[0:1], v2
 ; GCN-NEXT:    s_endpgm
@@ -57,6 +63,9 @@ define amdgpu_kernel void @v_sad_u32_pat2(ptr addrspace(1) %out, i32 %a, i32 %b,
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v1, s2
@@ -79,12 +88,14 @@ define amdgpu_kernel void @v_sad_u32_pat2(ptr addrspace(1) %out, i32 %a, i32 %b,
 define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat1(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
 ; GCN-LABEL: v_sad_u32_multi_use_sub_pat1:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; GCN-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; GCN-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; GCN-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GCN-NEXT:    s_add_u32 s16, s16, s15
-; GCN-NEXT:    s_addc_u32 s17, s17, 0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_add_u32 s20, s20, s17
+; GCN-NEXT:    s_addc_u32 s21, s21, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_min_u32 s3, s0, s1
 ; GCN-NEXT:    s_max_u32 s0, s0, s1
@@ -92,8 +103,9 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat1(ptr addrspace(1) %out, i
 ; GCN-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    s_add_i32 s0, s0, s2
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    v_mov_b32_e32 v1, s5
-; GCN-NEXT:    buffer_store_dword v2, v0, s[16:19], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v0, s[20:23], 0 offen
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    flat_store_dword v[0:1], v2
@@ -115,19 +127,22 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat1(ptr addrspace(1) %out, i
 define amdgpu_kernel void @v_sad_u32_multi_use_add_pat1(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
 ; GCN-LABEL: v_sad_u32_multi_use_add_pat1:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; GCN-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; GCN-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; GCN-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GCN-NEXT:    s_add_u32 s16, s16, s15
-; GCN-NEXT:    s_addc_u32 s17, s17, 0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_add_u32 s20, s20, s17
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v2, s1
 ; GCN-NEXT:    v_mov_b32_e32 v3, s2
 ; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_addc_u32 s21, s21, 0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-NEXT:    v_sad_u32 v2, s0, v2, v3
-; GCN-NEXT:    buffer_store_dword v2, v0, s[16:19], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v0, s[20:23], 0 offen
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    flat_store_dword v[0:1], v2
 ; GCN-NEXT:    s_endpgm
@@ -147,21 +162,24 @@ define amdgpu_kernel void @v_sad_u32_multi_use_add_pat1(ptr addrspace(1) %out, i
 define amdgpu_kernel void @v_sad_u32_multi_use_max_pat1(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
 ; GCN-LABEL: v_sad_u32_multi_use_max_pat1:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; GCN-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; GCN-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; GCN-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GCN-NEXT:    s_add_u32 s16, s16, s15
-; GCN-NEXT:    s_addc_u32 s17, s17, 0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_add_u32 s20, s20, s17
+; GCN-NEXT:    s_addc_u32 s21, s21, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_max_u32 s3, s0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v1, s2
 ; GCN-NEXT:    v_mov_b32_e32 v2, s3
 ; GCN-NEXT:    v_sad_u32 v3, s0, v0, v1
-; GCN-NEXT:    buffer_store_dword v2, v0, s[16:19], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v0, s[20:23], 0 offen
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-NEXT:    flat_store_dword v[0:1], v3
 ; GCN-NEXT:    s_endpgm
@@ -182,21 +200,24 @@ define amdgpu_kernel void @v_sad_u32_multi_use_max_pat1(ptr addrspace(1) %out, i
 define amdgpu_kernel void @v_sad_u32_multi_use_min_pat1(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
 ; GCN-LABEL: v_sad_u32_multi_use_min_pat1:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; GCN-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; GCN-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; GCN-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GCN-NEXT:    s_add_u32 s16, s16, s15
-; GCN-NEXT:    s_addc_u32 s17, s17, 0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_add_u32 s20, s20, s17
+; GCN-NEXT:    s_addc_u32 s21, s21, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_min_u32 s3, s0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v1, s2
 ; GCN-NEXT:    v_mov_b32_e32 v2, s3
 ; GCN-NEXT:    v_sad_u32 v3, s0, v0, v1
-; GCN-NEXT:    buffer_store_dword v2, v0, s[16:19], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v0, s[20:23], 0 offen
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-NEXT:    flat_store_dword v[0:1], v3
 ; GCN-NEXT:    s_endpgm
@@ -218,21 +239,24 @@ define amdgpu_kernel void @v_sad_u32_multi_use_min_pat1(ptr addrspace(1) %out, i
 define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat2(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
 ; GCN-LABEL: v_sad_u32_multi_use_sub_pat2:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; GCN-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; GCN-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; GCN-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GCN-NEXT:    s_add_u32 s16, s16, s15
-; GCN-NEXT:    s_addc_u32 s17, s17, 0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_add_u32 s20, s20, s17
+; GCN-NEXT:    s_addc_u32 s21, s21, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s3, s0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v0, s1
 ; GCN-NEXT:    v_mov_b32_e32 v1, s2
 ; GCN-NEXT:    v_mov_b32_e32 v2, s3
 ; GCN-NEXT:    v_sad_u32 v3, s0, v0, v1
-; GCN-NEXT:    buffer_store_dword v2, v0, s[16:19], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v0, s[20:23], 0 offen
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-NEXT:    flat_store_dword v[0:1], v3
 ; GCN-NEXT:    s_endpgm
@@ -251,12 +275,14 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat2(ptr addrspace(1) %out, i
 define amdgpu_kernel void @v_sad_u32_multi_use_select_pat2(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
 ; GCN-LABEL: v_sad_u32_multi_use_select_pat2:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b64 s[18:19], s[2:3]
-; GCN-NEXT:    s_mov_b64 s[16:17], s[0:1]
+; GCN-NEXT:    s_mov_b64 s[22:23], s[2:3]
+; GCN-NEXT:    s_mov_b64 s[20:21], s[0:1]
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GCN-NEXT:    s_add_u32 s16, s16, s15
-; GCN-NEXT:    s_addc_u32 s17, s17, 0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_add_u32 s20, s20, s17
+; GCN-NEXT:    s_addc_u32 s21, s21, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_min_u32 s3, s0, s1
 ; GCN-NEXT:    s_max_u32 s0, s0, s1
@@ -264,8 +290,9 @@ define amdgpu_kernel void @v_sad_u32_multi_use_select_pat2(ptr addrspace(1) %out
 ; GCN-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    s_add_i32 s0, s0, s2
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    v_mov_b32_e32 v1, s5
-; GCN-NEXT:    buffer_store_dword v2, v0, s[16:19], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v0, s[20:23], 0 offen
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    flat_store_dword v[0:1], v2
@@ -285,6 +312,9 @@ define amdgpu_kernel void @v_sad_u32_multi_use_select_pat2(ptr addrspace(1) %out
 define amdgpu_kernel void @v_sad_u32_vector_pat1(ptr addrspace(1) %out, <4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; GCN-LABEL: v_sad_u32_vector_pat1:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x4
 ; GCN-NEXT:    s_load_dwordx4 s[12:15], s[8:9], 0xc
 ; GCN-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
@@ -321,6 +351,9 @@ define amdgpu_kernel void @v_sad_u32_vector_pat1(ptr addrspace(1) %out, <4 x i32
 define amdgpu_kernel void @v_sad_u32_vector_pat2(ptr addrspace(1) %out, <4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; GCN-LABEL: v_sad_u32_vector_pat2:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x4
 ; GCN-NEXT:    s_load_dwordx4 s[12:15], s[8:9], 0xc
 ; GCN-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
@@ -358,6 +391,8 @@ define amdgpu_kernel void @v_sad_u32_i16_pat1(ptr addrspace(1) %out, i16 %a, i16
 ; GCN-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_and_b32 s4, s4, 0xffff
 ; GCN-NEXT:    s_lshr_b32 s0, s0, 16
@@ -365,6 +400,7 @@ define amdgpu_kernel void @v_sad_u32_i16_pat1(ptr addrspace(1) %out, i16 %a, i16
 ; GCN-NEXT:    v_mov_b32_e32 v1, s0
 ; GCN-NEXT:    v_sad_u32 v2, s4, v1, v0
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-NEXT:    flat_store_short v[0:1], v2
 ; GCN-NEXT:    s_endpgm
@@ -384,6 +420,9 @@ define amdgpu_kernel void @v_sad_u32_i16_pat1(ptr addrspace(1) %out, i16 %a, i16
 define amdgpu_kernel void @v_sad_u32_i16_pat2(ptr addrspace(1) %out) {
 ; GCN-LABEL: v_sad_u32_i16_pat2:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    flat_load_ushort v0, v[0:1] glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -416,6 +455,9 @@ define amdgpu_kernel void @v_sad_u32_i8_pat1(ptr addrspace(1) %out, i8 %a, i8 %b
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_and_b32 s3, s2, 0xff
 ; GCN-NEXT:    s_bfe_u32 s4, s2, 0x80008
@@ -443,6 +485,9 @@ define amdgpu_kernel void @v_sad_u32_i8_pat1(ptr addrspace(1) %out, i8 %a, i8 %b
 define amdgpu_kernel void @v_sad_u32_i8_pat2(ptr addrspace(1) %out) {
 ; GCN-LABEL: v_sad_u32_i8_pat2:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    flat_load_ubyte v0, v[0:1] glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
@@ -475,6 +520,9 @@ define amdgpu_kernel void @s_sad_u32_i8_pat2(ptr addrspace(1) %out, i8 zeroext %
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_and_b32 s3, s2, 0xff
 ; GCN-NEXT:    s_bfe_u32 s4, s2, 0x80008
@@ -502,6 +550,9 @@ define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat1(ptr addrspace(1) %
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_max_u32 s6, s0, s1
 ; GCN-NEXT:    s_cmp_le_u32 s0, s1
@@ -531,6 +582,9 @@ define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat2(ptr addrspace(1) %
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
 ; GCN-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s3, s0, s3
 ; GCN-NEXT:    s_sub_i32 s6, s1, s0
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll
index 884ba3fc34dff..29448ab2d822e 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll
@@ -9,6 +9,8 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 {
 ; GFX900:       ; %bb.0: ; %entry
 ; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX900-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX900-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX900-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
@@ -24,6 +26,8 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 {
 ; GFX906:       ; %bb.0: ; %entry
 ; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX906-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX906-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX906-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX906-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX906-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
@@ -39,6 +43,8 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 {
 ; GFX908:       ; %bb.0: ; %entry
 ; GFX908-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX908-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX908-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX908-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX908-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
@@ -55,6 +61,8 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 {
 ; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX90A-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v4, 4, v4
+; GFX90A-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
@@ -88,6 +96,8 @@ define amdgpu_kernel void @scalar_to_vector_v8f16(<2 x float> %in, ptr %out) #0
 ; GFX900:       ; %bb.0: ; %entry
 ; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX900-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX900-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX900-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
@@ -103,6 +113,8 @@ define amdgpu_kernel void @scalar_to_vector_v8f16(<2 x float> %in, ptr %out) #0
 ; GFX906:       ; %bb.0: ; %entry
 ; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX906-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX906-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX906-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX906-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX906-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
@@ -118,6 +130,8 @@ define amdgpu_kernel void @scalar_to_vector_v8f16(<2 x float> %in, ptr %out) #0
 ; GFX908:       ; %bb.0: ; %entry
 ; GFX908-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX908-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX908-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX908-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX908-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
@@ -134,6 +148,8 @@ define amdgpu_kernel void @scalar_to_vector_v8f16(<2 x float> %in, ptr %out) #0
 ; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX90A-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v4, 4, v4
+; GFX90A-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; GFX90A-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
diff --git a/llvm/test/CodeGen/AMDGPU/scc-clobbered-sgpr-to-vmem-spill.ll b/llvm/test/CodeGen/AMDGPU/scc-clobbered-sgpr-to-vmem-spill.ll
index 0ad10437299f4..90dfd5a21d107 100644
--- a/llvm/test/CodeGen/AMDGPU/scc-clobbered-sgpr-to-vmem-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/scc-clobbered-sgpr-to-vmem-spill.ll
@@ -20,179 +20,183 @@ define amdgpu_kernel void @kernel0(ptr addrspace(1) %out, i32 %in) #1 {
 ; CHECK-NEXT:    ; def s[2:3]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[4:7]
-; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_load_dword s0, s[8:9], 0x8
 ; CHECK-NEXT:    v_writelane_b32 v22, s2, 0
 ; CHECK-NEXT:    v_writelane_b32 v22, s3, 1
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def s[48:51]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def s[4:11]
+; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    v_writelane_b32 v22, s4, 2
 ; CHECK-NEXT:    v_writelane_b32 v22, s5, 3
 ; CHECK-NEXT:    v_writelane_b32 v22, s6, 4
-; CHECK-NEXT:    s_load_dword s0, s[8:9], 0x8
 ; CHECK-NEXT:    v_writelane_b32 v22, s7, 5
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[4:11]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v22, s4, 6
-; CHECK-NEXT:    v_writelane_b32 v22, s5, 7
-; CHECK-NEXT:    v_writelane_b32 v22, s6, 8
-; CHECK-NEXT:    v_writelane_b32 v22, s7, 9
-; CHECK-NEXT:    v_writelane_b32 v22, s8, 10
-; CHECK-NEXT:    v_writelane_b32 v22, s9, 11
-; CHECK-NEXT:    v_writelane_b32 v22, s10, 12
-; CHECK-NEXT:    v_writelane_b32 v22, s11, 13
+; CHECK-NEXT:    v_writelane_b32 v22, s8, 6
+; CHECK-NEXT:    v_writelane_b32 v22, s9, 7
+; CHECK-NEXT:    v_writelane_b32 v22, s10, 8
+; CHECK-NEXT:    v_writelane_b32 v22, s11, 9
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[4:19]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v22, s4, 14
-; CHECK-NEXT:    v_writelane_b32 v22, s5, 15
-; CHECK-NEXT:    v_writelane_b32 v22, s6, 16
-; CHECK-NEXT:    v_writelane_b32 v22, s7, 17
-; CHECK-NEXT:    v_writelane_b32 v22, s8, 18
-; CHECK-NEXT:    v_writelane_b32 v22, s9, 19
-; CHECK-NEXT:    v_writelane_b32 v22, s10, 20
-; CHECK-NEXT:    v_writelane_b32 v22, s11, 21
-; CHECK-NEXT:    v_writelane_b32 v22, s12, 22
-; CHECK-NEXT:    v_writelane_b32 v22, s13, 23
-; CHECK-NEXT:    v_writelane_b32 v22, s14, 24
-; CHECK-NEXT:    v_writelane_b32 v22, s15, 25
-; CHECK-NEXT:    v_writelane_b32 v22, s16, 26
-; CHECK-NEXT:    v_writelane_b32 v22, s17, 27
-; CHECK-NEXT:    v_writelane_b32 v22, s18, 28
-; CHECK-NEXT:    v_writelane_b32 v22, s19, 29
+; CHECK-NEXT:    v_writelane_b32 v22, s4, 10
+; CHECK-NEXT:    v_writelane_b32 v22, s5, 11
+; CHECK-NEXT:    v_writelane_b32 v22, s6, 12
+; CHECK-NEXT:    v_writelane_b32 v22, s7, 13
+; CHECK-NEXT:    v_writelane_b32 v22, s8, 14
+; CHECK-NEXT:    v_writelane_b32 v22, s9, 15
+; CHECK-NEXT:    v_writelane_b32 v22, s10, 16
+; CHECK-NEXT:    v_writelane_b32 v22, s11, 17
+; CHECK-NEXT:    v_writelane_b32 v22, s12, 18
+; CHECK-NEXT:    v_writelane_b32 v22, s13, 19
+; CHECK-NEXT:    v_writelane_b32 v22, s14, 20
+; CHECK-NEXT:    v_writelane_b32 v22, s15, 21
+; CHECK-NEXT:    v_writelane_b32 v22, s16, 22
+; CHECK-NEXT:    v_writelane_b32 v22, s17, 23
+; CHECK-NEXT:    v_writelane_b32 v22, s18, 24
+; CHECK-NEXT:    v_writelane_b32 v22, s19, 25
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[42:43]
+; CHECK-NEXT:    ; def s[38:39]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[52:55]
+; CHECK-NEXT:    ; def s[44:47]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[4:11]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v22, s4, 30
-; CHECK-NEXT:    v_writelane_b32 v22, s5, 31
-; CHECK-NEXT:    v_writelane_b32 v22, s6, 32
-; CHECK-NEXT:    v_writelane_b32 v22, s7, 33
-; CHECK-NEXT:    v_writelane_b32 v22, s8, 34
-; CHECK-NEXT:    v_writelane_b32 v22, s9, 35
-; CHECK-NEXT:    v_writelane_b32 v22, s10, 36
-; CHECK-NEXT:    v_writelane_b32 v22, s11, 37
+; CHECK-NEXT:    v_writelane_b32 v22, s4, 26
+; CHECK-NEXT:    v_writelane_b32 v22, s5, 27
+; CHECK-NEXT:    v_writelane_b32 v22, s6, 28
+; CHECK-NEXT:    v_writelane_b32 v22, s7, 29
+; CHECK-NEXT:    v_writelane_b32 v22, s8, 30
+; CHECK-NEXT:    v_writelane_b32 v22, s9, 31
+; CHECK-NEXT:    v_writelane_b32 v22, s10, 32
+; CHECK-NEXT:    v_writelane_b32 v22, s11, 33
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[16:31]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[40:41]
+; CHECK-NEXT:    ; def s[36:37]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[36:39]
+; CHECK-NEXT:    ; def s[40:43]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[44:51]
+; CHECK-NEXT:    ; def s[0:7]
 ; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_writelane_b32 v22, s0, 34
+; CHECK-NEXT:    v_writelane_b32 v22, s1, 35
+; CHECK-NEXT:    v_writelane_b32 v22, s2, 36
+; CHECK-NEXT:    v_writelane_b32 v22, s3, 37
+; CHECK-NEXT:    v_writelane_b32 v22, s4, 38
+; CHECK-NEXT:    v_writelane_b32 v22, s5, 39
+; CHECK-NEXT:    v_writelane_b32 v22, s6, 40
+; CHECK-NEXT:    v_writelane_b32 v22, s7, 41
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[0:15]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v22, s0, 38
-; CHECK-NEXT:    v_writelane_b32 v22, s1, 39
-; CHECK-NEXT:    v_writelane_b32 v22, s2, 40
-; CHECK-NEXT:    v_writelane_b32 v22, s3, 41
-; CHECK-NEXT:    v_writelane_b32 v22, s4, 42
-; CHECK-NEXT:    v_writelane_b32 v22, s5, 43
-; CHECK-NEXT:    v_writelane_b32 v22, s6, 44
-; CHECK-NEXT:    v_writelane_b32 v22, s7, 45
-; CHECK-NEXT:    v_writelane_b32 v22, s8, 46
-; CHECK-NEXT:    v_writelane_b32 v22, s9, 47
-; CHECK-NEXT:    v_writelane_b32 v22, s10, 48
-; CHECK-NEXT:    v_writelane_b32 v22, s11, 49
-; CHECK-NEXT:    v_writelane_b32 v22, s12, 50
-; CHECK-NEXT:    v_writelane_b32 v22, s13, 51
-; CHECK-NEXT:    v_writelane_b32 v22, s14, 52
-; CHECK-NEXT:    v_writelane_b32 v22, s15, 53
+; CHECK-NEXT:    v_writelane_b32 v22, s0, 42
+; CHECK-NEXT:    v_writelane_b32 v22, s1, 43
+; CHECK-NEXT:    v_writelane_b32 v22, s2, 44
+; CHECK-NEXT:    v_writelane_b32 v22, s3, 45
+; CHECK-NEXT:    v_writelane_b32 v22, s4, 46
+; CHECK-NEXT:    v_writelane_b32 v22, s5, 47
+; CHECK-NEXT:    v_writelane_b32 v22, s6, 48
+; CHECK-NEXT:    v_writelane_b32 v22, s7, 49
+; CHECK-NEXT:    v_writelane_b32 v22, s8, 50
+; CHECK-NEXT:    v_writelane_b32 v22, s9, 51
+; CHECK-NEXT:    v_writelane_b32 v22, s10, 52
+; CHECK-NEXT:    v_writelane_b32 v22, s11, 53
+; CHECK-NEXT:    v_writelane_b32 v22, s12, 54
+; CHECK-NEXT:    v_writelane_b32 v22, s13, 55
+; CHECK-NEXT:    v_writelane_b32 v22, s14, 56
+; CHECK-NEXT:    v_writelane_b32 v22, s15, 57
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[34:35]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[0:3]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v22, s0, 54
-; CHECK-NEXT:    v_writelane_b32 v22, s1, 55
-; CHECK-NEXT:    v_writelane_b32 v22, s2, 56
-; CHECK-NEXT:    v_writelane_b32 v22, s3, 57
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[0:7]
-; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    v_writelane_b32 v22, s0, 58
 ; CHECK-NEXT:    v_writelane_b32 v22, s1, 59
 ; CHECK-NEXT:    v_writelane_b32 v22, s2, 60
-; CHECK-NEXT:    ; implicit-def: $vgpr23 : SGPR spill to VGPR lane
 ; CHECK-NEXT:    v_writelane_b32 v22, s3, 61
-; CHECK-NEXT:    v_writelane_b32 v22, s4, 62
-; CHECK-NEXT:    v_writelane_b32 v23, s6, 0
-; CHECK-NEXT:    v_writelane_b32 v22, s5, 63
-; CHECK-NEXT:    v_writelane_b32 v23, s7, 1
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def s[0:7]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ; implicit-def: $vgpr23 : SGPR spill to VGPR lane
+; CHECK-NEXT:    v_writelane_b32 v22, s0, 62
+; CHECK-NEXT:    v_writelane_b32 v23, s2, 0
+; CHECK-NEXT:    v_writelane_b32 v23, s3, 1
+; CHECK-NEXT:    v_writelane_b32 v23, s4, 2
+; CHECK-NEXT:    v_writelane_b32 v23, s5, 3
+; CHECK-NEXT:    v_writelane_b32 v23, s6, 4
+; CHECK-NEXT:    v_writelane_b32 v22, s1, 63
+; CHECK-NEXT:    v_writelane_b32 v23, s7, 5
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[0:15]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v23, s0, 2
-; CHECK-NEXT:    v_writelane_b32 v23, s1, 3
-; CHECK-NEXT:    v_writelane_b32 v23, s2, 4
-; CHECK-NEXT:    v_writelane_b32 v23, s3, 5
-; CHECK-NEXT:    v_writelane_b32 v23, s4, 6
-; CHECK-NEXT:    v_writelane_b32 v23, s5, 7
-; CHECK-NEXT:    v_writelane_b32 v23, s6, 8
-; CHECK-NEXT:    v_writelane_b32 v23, s7, 9
-; CHECK-NEXT:    v_writelane_b32 v23, s8, 10
-; CHECK-NEXT:    v_writelane_b32 v23, s9, 11
-; CHECK-NEXT:    v_writelane_b32 v23, s10, 12
-; CHECK-NEXT:    v_writelane_b32 v23, s11, 13
-; CHECK-NEXT:    v_writelane_b32 v23, s12, 14
-; CHECK-NEXT:    v_writelane_b32 v23, s13, 15
-; CHECK-NEXT:    v_writelane_b32 v23, s14, 16
-; CHECK-NEXT:    v_writelane_b32 v23, s15, 17
+; CHECK-NEXT:    v_writelane_b32 v23, s0, 6
+; CHECK-NEXT:    v_writelane_b32 v23, s1, 7
+; CHECK-NEXT:    v_writelane_b32 v23, s2, 8
+; CHECK-NEXT:    v_writelane_b32 v23, s3, 9
+; CHECK-NEXT:    v_writelane_b32 v23, s4, 10
+; CHECK-NEXT:    v_writelane_b32 v23, s5, 11
+; CHECK-NEXT:    v_writelane_b32 v23, s6, 12
+; CHECK-NEXT:    v_writelane_b32 v23, s7, 13
+; CHECK-NEXT:    v_writelane_b32 v23, s8, 14
+; CHECK-NEXT:    v_writelane_b32 v23, s9, 15
+; CHECK-NEXT:    v_writelane_b32 v23, s10, 16
+; CHECK-NEXT:    v_writelane_b32 v23, s11, 17
+; CHECK-NEXT:    v_writelane_b32 v23, s12, 18
+; CHECK-NEXT:    v_writelane_b32 v23, s13, 19
+; CHECK-NEXT:    v_writelane_b32 v23, s14, 20
+; CHECK-NEXT:    v_writelane_b32 v23, s15, 21
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[0:1]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v23, s0, 18
-; CHECK-NEXT:    v_writelane_b32 v23, s1, 19
+; CHECK-NEXT:    v_writelane_b32 v23, s0, 22
+; CHECK-NEXT:    v_writelane_b32 v23, s1, 23
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[0:3]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v23, s0, 20
-; CHECK-NEXT:    v_writelane_b32 v23, s1, 21
-; CHECK-NEXT:    v_writelane_b32 v23, s2, 22
-; CHECK-NEXT:    v_writelane_b32 v23, s3, 23
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; def s[0:7]
-; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    v_writelane_b32 v23, s0, 24
 ; CHECK-NEXT:    v_writelane_b32 v23, s1, 25
 ; CHECK-NEXT:    v_writelane_b32 v23, s2, 26
 ; CHECK-NEXT:    v_writelane_b32 v23, s3, 27
-; CHECK-NEXT:    v_writelane_b32 v23, s4, 28
-; CHECK-NEXT:    v_writelane_b32 v23, s5, 29
-; CHECK-NEXT:    v_writelane_b32 v23, s6, 30
-; CHECK-NEXT:    v_writelane_b32 v23, s7, 31
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def s[0:7]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_writelane_b32 v23, s0, 28
+; CHECK-NEXT:    v_writelane_b32 v23, s1, 29
+; CHECK-NEXT:    v_writelane_b32 v23, s2, 30
+; CHECK-NEXT:    v_writelane_b32 v23, s3, 31
+; CHECK-NEXT:    v_writelane_b32 v23, s4, 32
+; CHECK-NEXT:    v_writelane_b32 v23, s5, 33
+; CHECK-NEXT:    v_writelane_b32 v23, s6, 34
+; CHECK-NEXT:    v_writelane_b32 v23, s7, 35
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; def s[0:15]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_writelane_b32 v23, s0, 32
-; CHECK-NEXT:    v_writelane_b32 v23, s1, 33
-; CHECK-NEXT:    v_writelane_b32 v23, s2, 34
-; CHECK-NEXT:    v_writelane_b32 v23, s3, 35
-; CHECK-NEXT:    v_writelane_b32 v23, s4, 36
-; CHECK-NEXT:    v_writelane_b32 v23, s5, 37
-; CHECK-NEXT:    v_writelane_b32 v23, s6, 38
-; CHECK-NEXT:    v_writelane_b32 v23, s7, 39
-; CHECK-NEXT:    v_writelane_b32 v23, s8, 40
-; CHECK-NEXT:    v_writelane_b32 v23, s9, 41
-; CHECK-NEXT:    v_writelane_b32 v23, s10, 42
-; CHECK-NEXT:    v_writelane_b32 v23, s11, 43
-; CHECK-NEXT:    v_writelane_b32 v23, s12, 44
-; CHECK-NEXT:    v_writelane_b32 v23, s13, 45
-; CHECK-NEXT:    v_writelane_b32 v23, s14, 46
-; CHECK-NEXT:    v_writelane_b32 v23, s15, 47
+; CHECK-NEXT:    v_writelane_b32 v23, s0, 36
+; CHECK-NEXT:    v_writelane_b32 v23, s1, 37
+; CHECK-NEXT:    v_writelane_b32 v23, s2, 38
+; CHECK-NEXT:    v_writelane_b32 v23, s3, 39
+; CHECK-NEXT:    v_writelane_b32 v23, s4, 40
+; CHECK-NEXT:    v_writelane_b32 v23, s5, 41
+; CHECK-NEXT:    v_writelane_b32 v23, s6, 42
+; CHECK-NEXT:    v_writelane_b32 v23, s7, 43
+; CHECK-NEXT:    v_writelane_b32 v23, s8, 44
+; CHECK-NEXT:    v_writelane_b32 v23, s9, 45
+; CHECK-NEXT:    v_writelane_b32 v23, s10, 46
+; CHECK-NEXT:    v_writelane_b32 v23, s11, 47
+; CHECK-NEXT:    v_writelane_b32 v23, s12, 48
+; CHECK-NEXT:    v_writelane_b32 v23, s13, 49
+; CHECK-NEXT:    v_writelane_b32 v23, s14, 50
+; CHECK-NEXT:    v_writelane_b32 v23, s15, 51
 ; CHECK-NEXT:    s_cbranch_scc0 .LBB0_2
 ; CHECK-NEXT:  ; %bb.1: ; %ret
 ; CHECK-NEXT:    s_endpgm
@@ -206,166 +210,170 @@ define amdgpu_kernel void @kernel0(ptr addrspace(1) %out, i32 %in) #1 {
 ; CHECK-NEXT:    v_readlane_b32 s1, v22, 3
 ; CHECK-NEXT:    v_readlane_b32 s2, v22, 4
 ; CHECK-NEXT:    v_readlane_b32 s3, v22, 5
+; CHECK-NEXT:    v_readlane_b32 s4, v22, 6
+; CHECK-NEXT:    v_readlane_b32 s5, v22, 7
+; CHECK-NEXT:    v_readlane_b32 s6, v22, 8
+; CHECK-NEXT:    v_readlane_b32 s7, v22, 9
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use s[0:3]
+; CHECK-NEXT:    ; use s[48:51]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v22, 6
-; CHECK-NEXT:    v_readlane_b32 s1, v22, 7
-; CHECK-NEXT:    v_readlane_b32 s2, v22, 8
-; CHECK-NEXT:    v_readlane_b32 s3, v22, 9
-; CHECK-NEXT:    v_readlane_b32 s4, v22, 10
-; CHECK-NEXT:    v_readlane_b32 s5, v22, 11
-; CHECK-NEXT:    v_readlane_b32 s6, v22, 12
-; CHECK-NEXT:    v_readlane_b32 s7, v22, 13
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:7]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v22, 14
-; CHECK-NEXT:    v_readlane_b32 s1, v22, 15
-; CHECK-NEXT:    v_readlane_b32 s2, v22, 16
-; CHECK-NEXT:    v_readlane_b32 s3, v22, 17
-; CHECK-NEXT:    v_readlane_b32 s4, v22, 18
-; CHECK-NEXT:    v_readlane_b32 s5, v22, 19
-; CHECK-NEXT:    v_readlane_b32 s6, v22, 20
-; CHECK-NEXT:    v_readlane_b32 s7, v22, 21
-; CHECK-NEXT:    v_readlane_b32 s8, v22, 22
-; CHECK-NEXT:    v_readlane_b32 s9, v22, 23
-; CHECK-NEXT:    v_readlane_b32 s10, v22, 24
-; CHECK-NEXT:    v_readlane_b32 s11, v22, 25
-; CHECK-NEXT:    v_readlane_b32 s12, v22, 26
-; CHECK-NEXT:    v_readlane_b32 s13, v22, 27
-; CHECK-NEXT:    v_readlane_b32 s14, v22, 28
-; CHECK-NEXT:    v_readlane_b32 s15, v22, 29
+; CHECK-NEXT:    v_readlane_b32 s0, v22, 10
+; CHECK-NEXT:    v_readlane_b32 s1, v22, 11
+; CHECK-NEXT:    v_readlane_b32 s2, v22, 12
+; CHECK-NEXT:    v_readlane_b32 s3, v22, 13
+; CHECK-NEXT:    v_readlane_b32 s4, v22, 14
+; CHECK-NEXT:    v_readlane_b32 s5, v22, 15
+; CHECK-NEXT:    v_readlane_b32 s6, v22, 16
+; CHECK-NEXT:    v_readlane_b32 s7, v22, 17
+; CHECK-NEXT:    v_readlane_b32 s8, v22, 18
+; CHECK-NEXT:    v_readlane_b32 s9, v22, 19
+; CHECK-NEXT:    v_readlane_b32 s10, v22, 20
+; CHECK-NEXT:    v_readlane_b32 s11, v22, 21
+; CHECK-NEXT:    v_readlane_b32 s12, v22, 22
+; CHECK-NEXT:    v_readlane_b32 s13, v22, 23
+; CHECK-NEXT:    v_readlane_b32 s14, v22, 24
+; CHECK-NEXT:    v_readlane_b32 s15, v22, 25
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:15]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v22, 30
-; CHECK-NEXT:    v_readlane_b32 s1, v22, 31
-; CHECK-NEXT:    v_readlane_b32 s2, v22, 32
-; CHECK-NEXT:    v_readlane_b32 s3, v22, 33
-; CHECK-NEXT:    v_readlane_b32 s4, v22, 34
-; CHECK-NEXT:    v_readlane_b32 s5, v22, 35
-; CHECK-NEXT:    v_readlane_b32 s6, v22, 36
-; CHECK-NEXT:    v_readlane_b32 s7, v22, 37
+; CHECK-NEXT:    v_readlane_b32 s0, v22, 26
+; CHECK-NEXT:    v_readlane_b32 s1, v22, 27
+; CHECK-NEXT:    v_readlane_b32 s2, v22, 28
+; CHECK-NEXT:    v_readlane_b32 s3, v22, 29
+; CHECK-NEXT:    v_readlane_b32 s4, v22, 30
+; CHECK-NEXT:    v_readlane_b32 s5, v22, 31
+; CHECK-NEXT:    v_readlane_b32 s6, v22, 32
+; CHECK-NEXT:    v_readlane_b32 s7, v22, 33
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use s[42:43]
+; CHECK-NEXT:    ; use s[38:39]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use s[52:55]
+; CHECK-NEXT:    ; use s[44:47]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:7]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v22, 38
-; CHECK-NEXT:    v_readlane_b32 s1, v22, 39
-; CHECK-NEXT:    v_readlane_b32 s2, v22, 40
-; CHECK-NEXT:    v_readlane_b32 s3, v22, 41
+; CHECK-NEXT:    v_readlane_b32 s0, v22, 34
+; CHECK-NEXT:    v_readlane_b32 s1, v22, 35
+; CHECK-NEXT:    v_readlane_b32 s2, v22, 36
+; CHECK-NEXT:    v_readlane_b32 s3, v22, 37
+; CHECK-NEXT:    v_readlane_b32 s4, v22, 38
+; CHECK-NEXT:    v_readlane_b32 s5, v22, 39
+; CHECK-NEXT:    v_readlane_b32 s6, v22, 40
+; CHECK-NEXT:    v_readlane_b32 s7, v22, 41
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[16:31]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use s[40:41]
+; CHECK-NEXT:    ; use s[36:37]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use s[36:39]
+; CHECK-NEXT:    ; use s[40:43]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use s[44:51]
+; CHECK-NEXT:    ; use s[0:7]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s4, v22, 42
-; CHECK-NEXT:    v_readlane_b32 s5, v22, 43
-; CHECK-NEXT:    v_readlane_b32 s6, v22, 44
-; CHECK-NEXT:    v_readlane_b32 s7, v22, 45
-; CHECK-NEXT:    v_readlane_b32 s8, v22, 46
-; CHECK-NEXT:    v_readlane_b32 s9, v22, 47
-; CHECK-NEXT:    v_readlane_b32 s10, v22, 48
-; CHECK-NEXT:    v_readlane_b32 s11, v22, 49
-; CHECK-NEXT:    v_readlane_b32 s12, v22, 50
-; CHECK-NEXT:    v_readlane_b32 s13, v22, 51
-; CHECK-NEXT:    v_readlane_b32 s14, v22, 52
-; CHECK-NEXT:    v_readlane_b32 s15, v22, 53
+; CHECK-NEXT:    v_readlane_b32 s0, v22, 42
+; CHECK-NEXT:    v_readlane_b32 s1, v22, 43
+; CHECK-NEXT:    v_readlane_b32 s2, v22, 44
+; CHECK-NEXT:    v_readlane_b32 s3, v22, 45
+; CHECK-NEXT:    v_readlane_b32 s4, v22, 46
+; CHECK-NEXT:    v_readlane_b32 s5, v22, 47
+; CHECK-NEXT:    v_readlane_b32 s6, v22, 48
+; CHECK-NEXT:    v_readlane_b32 s7, v22, 49
+; CHECK-NEXT:    v_readlane_b32 s8, v22, 50
+; CHECK-NEXT:    v_readlane_b32 s9, v22, 51
+; CHECK-NEXT:    v_readlane_b32 s10, v22, 52
+; CHECK-NEXT:    v_readlane_b32 s11, v22, 53
+; CHECK-NEXT:    v_readlane_b32 s12, v22, 54
+; CHECK-NEXT:    v_readlane_b32 s13, v22, 55
+; CHECK-NEXT:    v_readlane_b32 s14, v22, 56
+; CHECK-NEXT:    v_readlane_b32 s15, v22, 57
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:15]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v22, 54
-; CHECK-NEXT:    v_readlane_b32 s1, v22, 55
-; CHECK-NEXT:    v_readlane_b32 s2, v22, 56
-; CHECK-NEXT:    v_readlane_b32 s3, v22, 57
+; CHECK-NEXT:    v_readlane_b32 s0, v22, 58
+; CHECK-NEXT:    v_readlane_b32 s1, v22, 59
+; CHECK-NEXT:    v_readlane_b32 s2, v22, 60
+; CHECK-NEXT:    v_readlane_b32 s3, v22, 61
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[34:35]
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:3]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v22, 58
-; CHECK-NEXT:    v_readlane_b32 s1, v22, 59
-; CHECK-NEXT:    v_readlane_b32 s2, v22, 60
-; CHECK-NEXT:    v_readlane_b32 s3, v22, 61
-; CHECK-NEXT:    v_readlane_b32 s4, v22, 62
-; CHECK-NEXT:    v_readlane_b32 s5, v22, 63
-; CHECK-NEXT:    v_readlane_b32 s6, v23, 0
-; CHECK-NEXT:    v_readlane_b32 s7, v23, 1
+; CHECK-NEXT:    v_readlane_b32 s0, v22, 62
+; CHECK-NEXT:    v_readlane_b32 s1, v22, 63
+; CHECK-NEXT:    v_readlane_b32 s2, v23, 0
+; CHECK-NEXT:    v_readlane_b32 s3, v23, 1
+; CHECK-NEXT:    v_readlane_b32 s4, v23, 2
+; CHECK-NEXT:    v_readlane_b32 s5, v23, 3
+; CHECK-NEXT:    v_readlane_b32 s6, v23, 4
+; CHECK-NEXT:    v_readlane_b32 s7, v23, 5
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:7]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v23, 2
-; CHECK-NEXT:    v_readlane_b32 s1, v23, 3
-; CHECK-NEXT:    v_readlane_b32 s2, v23, 4
-; CHECK-NEXT:    v_readlane_b32 s3, v23, 5
-; CHECK-NEXT:    v_readlane_b32 s4, v23, 6
-; CHECK-NEXT:    v_readlane_b32 s5, v23, 7
-; CHECK-NEXT:    v_readlane_b32 s6, v23, 8
-; CHECK-NEXT:    v_readlane_b32 s7, v23, 9
-; CHECK-NEXT:    v_readlane_b32 s8, v23, 10
-; CHECK-NEXT:    v_readlane_b32 s9, v23, 11
-; CHECK-NEXT:    v_readlane_b32 s10, v23, 12
-; CHECK-NEXT:    v_readlane_b32 s11, v23, 13
-; CHECK-NEXT:    v_readlane_b32 s12, v23, 14
-; CHECK-NEXT:    v_readlane_b32 s13, v23, 15
-; CHECK-NEXT:    v_readlane_b32 s14, v23, 16
-; CHECK-NEXT:    v_readlane_b32 s15, v23, 17
+; CHECK-NEXT:    v_readlane_b32 s0, v23, 6
+; CHECK-NEXT:    v_readlane_b32 s1, v23, 7
+; CHECK-NEXT:    v_readlane_b32 s2, v23, 8
+; CHECK-NEXT:    v_readlane_b32 s3, v23, 9
+; CHECK-NEXT:    v_readlane_b32 s4, v23, 10
+; CHECK-NEXT:    v_readlane_b32 s5, v23, 11
+; CHECK-NEXT:    v_readlane_b32 s6, v23, 12
+; CHECK-NEXT:    v_readlane_b32 s7, v23, 13
+; CHECK-NEXT:    v_readlane_b32 s8, v23, 14
+; CHECK-NEXT:    v_readlane_b32 s9, v23, 15
+; CHECK-NEXT:    v_readlane_b32 s10, v23, 16
+; CHECK-NEXT:    v_readlane_b32 s11, v23, 17
+; CHECK-NEXT:    v_readlane_b32 s12, v23, 18
+; CHECK-NEXT:    v_readlane_b32 s13, v23, 19
+; CHECK-NEXT:    v_readlane_b32 s14, v23, 20
+; CHECK-NEXT:    v_readlane_b32 s15, v23, 21
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:15]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v23, 18
-; CHECK-NEXT:    v_readlane_b32 s1, v23, 19
+; CHECK-NEXT:    v_readlane_b32 s0, v23, 22
+; CHECK-NEXT:    v_readlane_b32 s1, v23, 23
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:1]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v23, 20
-; CHECK-NEXT:    v_readlane_b32 s1, v23, 21
-; CHECK-NEXT:    v_readlane_b32 s2, v23, 22
-; CHECK-NEXT:    v_readlane_b32 s3, v23, 23
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use s[0:3]
-; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    v_readlane_b32 s0, v23, 24
 ; CHECK-NEXT:    v_readlane_b32 s1, v23, 25
 ; CHECK-NEXT:    v_readlane_b32 s2, v23, 26
 ; CHECK-NEXT:    v_readlane_b32 s3, v23, 27
-; CHECK-NEXT:    v_readlane_b32 s4, v23, 28
-; CHECK-NEXT:    v_readlane_b32 s5, v23, 29
-; CHECK-NEXT:    v_readlane_b32 s6, v23, 30
-; CHECK-NEXT:    v_readlane_b32 s7, v23, 31
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use s[0:3]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_readlane_b32 s0, v23, 28
+; CHECK-NEXT:    v_readlane_b32 s1, v23, 29
+; CHECK-NEXT:    v_readlane_b32 s2, v23, 30
+; CHECK-NEXT:    v_readlane_b32 s3, v23, 31
+; CHECK-NEXT:    v_readlane_b32 s4, v23, 32
+; CHECK-NEXT:    v_readlane_b32 s5, v23, 33
+; CHECK-NEXT:    v_readlane_b32 s6, v23, 34
+; CHECK-NEXT:    v_readlane_b32 s7, v23, 35
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:7]
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s0, v23, 32
-; CHECK-NEXT:    v_readlane_b32 s1, v23, 33
-; CHECK-NEXT:    v_readlane_b32 s2, v23, 34
-; CHECK-NEXT:    v_readlane_b32 s3, v23, 35
-; CHECK-NEXT:    v_readlane_b32 s4, v23, 36
-; CHECK-NEXT:    v_readlane_b32 s5, v23, 37
-; CHECK-NEXT:    v_readlane_b32 s6, v23, 38
-; CHECK-NEXT:    v_readlane_b32 s7, v23, 39
-; CHECK-NEXT:    v_readlane_b32 s8, v23, 40
-; CHECK-NEXT:    v_readlane_b32 s9, v23, 41
-; CHECK-NEXT:    v_readlane_b32 s10, v23, 42
-; CHECK-NEXT:    v_readlane_b32 s11, v23, 43
-; CHECK-NEXT:    v_readlane_b32 s12, v23, 44
-; CHECK-NEXT:    v_readlane_b32 s13, v23, 45
-; CHECK-NEXT:    v_readlane_b32 s14, v23, 46
-; CHECK-NEXT:    v_readlane_b32 s15, v23, 47
+; CHECK-NEXT:    v_readlane_b32 s0, v23, 36
+; CHECK-NEXT:    v_readlane_b32 s1, v23, 37
+; CHECK-NEXT:    v_readlane_b32 s2, v23, 38
+; CHECK-NEXT:    v_readlane_b32 s3, v23, 39
+; CHECK-NEXT:    v_readlane_b32 s4, v23, 40
+; CHECK-NEXT:    v_readlane_b32 s5, v23, 41
+; CHECK-NEXT:    v_readlane_b32 s6, v23, 42
+; CHECK-NEXT:    v_readlane_b32 s7, v23, 43
+; CHECK-NEXT:    v_readlane_b32 s8, v23, 44
+; CHECK-NEXT:    v_readlane_b32 s9, v23, 45
+; CHECK-NEXT:    v_readlane_b32 s10, v23, 46
+; CHECK-NEXT:    v_readlane_b32 s11, v23, 47
+; CHECK-NEXT:    v_readlane_b32 s12, v23, 48
+; CHECK-NEXT:    v_readlane_b32 s13, v23, 49
+; CHECK-NEXT:    v_readlane_b32 s14, v23, 50
+; CHECK-NEXT:    v_readlane_b32 s15, v23, 51
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; use s[0:15]
 ; CHECK-NEXT:    ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll
index 455d22f2aa29c..cdfba3cf0db7f 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll
@@ -7,7 +7,7 @@
 define amdgpu_kernel void @partial_no_vgprs_last_sgpr_spill(ptr addrspace(1) %out, i32 %in) #1 {
 ; GCN-LABEL: partial_no_vgprs_last_sgpr_spill:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_add_u32 s0, s0, s15
+; GCN-NEXT:    s_add_u32 s0, s0, s17
 ; GCN-NEXT:    s_addc_u32 s1, s1, 0
 ; GCN-NEXT:    s_load_dword s4, s[8:9], 0x2
 ; GCN-NEXT:    ;;#ASMSTART
diff --git a/llvm/test/CodeGen/AMDGPU/shift-i128.ll b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
index a423b6f831a9d..65a17ed67481c 100644
--- a/llvm/test/CodeGen/AMDGPU/shift-i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
@@ -182,8 +182,10 @@ define amdgpu_kernel void @s_shl_i128_ss(i128 %lhs, i128 %rhs) {
 ; GCN-LABEL: s_shl_i128_ss:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s5, s4, 64
 ; GCN-NEXT:    s_sub_i32 s12, 64, s4
@@ -203,6 +205,7 @@ define amdgpu_kernel void @s_shl_i128_ss(i128 %lhs, i128 %rhs) {
 ; GCN-NEXT:    v_mov_b32_e32 v0, s6
 ; GCN-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-NEXT:    v_mov_b32_e32 v3, s1
 ; GCN-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GCN-NEXT:    s_endpgm
@@ -215,8 +218,10 @@ define amdgpu_kernel void @s_lshr_i128_ss(i128 %lhs, i128 %rhs) {
 ; GCN-LABEL: s_lshr_i128_ss:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s5, s4, 64
 ; GCN-NEXT:    s_sub_i32 s12, 64, s4
@@ -236,6 +241,7 @@ define amdgpu_kernel void @s_lshr_i128_ss(i128 %lhs, i128 %rhs) {
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-NEXT:    v_mov_b32_e32 v3, s5
 ; GCN-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GCN-NEXT:    s_endpgm
@@ -248,8 +254,10 @@ define amdgpu_kernel void @s_ashr_i128_ss(i128 %lhs, i128 %rhs) {
 ; GCN-LABEL: s_ashr_i128_ss:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_sub_i32 s5, 64, s4
 ; GCN-NEXT:    s_lshr_b64 s[6:7], s[0:1], s4
@@ -270,6 +278,7 @@ define amdgpu_kernel void @s_ashr_i128_ss(i128 %lhs, i128 %rhs) {
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-NEXT:    v_mov_b32_e32 v3, s3
 ; GCN-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GCN-NEXT:    s_endpgm
@@ -430,6 +439,9 @@ define <2 x i128> @v_ashr_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
 define amdgpu_kernel void @s_shl_v2i128ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-LABEL: s_shl_v2i128ss:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_load_dwordx16 s[0:15], s[8:9], 0x0
 ; GCN-NEXT:    v_mov_b32_e32 v6, 16
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
@@ -502,6 +514,9 @@ define amdgpu_kernel void @s_shl_v2i128ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 define amdgpu_kernel void @s_lshr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-LABEL: s_lshr_v2i128_ss:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_load_dwordx16 s[0:15], s[8:9], 0x0
 ; GCN-NEXT:    v_mov_b32_e32 v6, 16
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
@@ -574,6 +589,9 @@ define amdgpu_kernel void @s_lshr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 define amdgpu_kernel void @s_ashr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GCN-LABEL: s_ashr_v2i128_ss:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_load_dwordx16 s[0:15], s[8:9], 0x0
 ; GCN-NEXT:    v_mov_b32_e32 v6, 16
 ; GCN-NEXT:    v_mov_b32_e32 v4, 0
diff --git a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll
index 8531b2ad4e405..3c47e2504747d 100644
--- a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-annotate-kernel-features  %s | FileCheck -check-prefix=AKF_GCN %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-attributor %s | FileCheck -check-prefix=ATTRIBUTOR_GCN %s
 
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
@@ -7,9 +6,6 @@
 target datalayout = "A5"
 
 define internal void @indirect() {
-; AKF_GCN-LABEL: define {{[^@]+}}@indirect() {
-; AKF_GCN-NEXT:    ret void
-;
 ; ATTRIBUTOR_GCN-LABEL: define {{[^@]+}}@indirect
 ; ATTRIBUTOR_GCN-SAME: () #[[ATTR0:[0-9]+]] {
 ; ATTRIBUTOR_GCN-NEXT:    ret void
@@ -22,15 +18,6 @@ define internal void @indirect() {
 }
 
 define amdgpu_kernel void @test_simple_indirect_call() {
-; AKF_GCN-LABEL: define {{[^@]+}}@test_simple_indirect_call
-; AKF_GCN-SAME: () #[[ATTR0:[0-9]+]] {
-; AKF_GCN-NEXT:    [[FPTR:%.*]] = alloca ptr, align 8, addrspace(5)
-; AKF_GCN-NEXT:    [[FPTR_CAST:%.*]] = addrspacecast ptr addrspace(5) [[FPTR]] to ptr
-; AKF_GCN-NEXT:    store ptr @indirect, ptr [[FPTR_CAST]], align 8
-; AKF_GCN-NEXT:    [[FP:%.*]] = load ptr, ptr [[FPTR_CAST]], align 8
-; AKF_GCN-NEXT:    call void [[FP]]()
-; AKF_GCN-NEXT:    ret void
-;
 ; ATTRIBUTOR_GCN-LABEL: define {{[^@]+}}@test_simple_indirect_call
 ; ATTRIBUTOR_GCN-SAME: () #[[ATTR1:[0-9]+]] {
 ; ATTRIBUTOR_GCN-NEXT:    [[FPTR:%.*]] = alloca ptr, align 8, addrspace(5)
@@ -79,12 +66,10 @@ define amdgpu_kernel void @test_simple_indirect_call() {
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
-; AKF_GCN: attributes #[[ATTR0]] = { "amdgpu-calls" "amdgpu-stack-objects" }
 ;.
 ; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
-; AKF_GCN: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
 ; ATTRIBUTOR_GCN: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
index 8129a7ac51df9..d71d0f78fe1c3 100644
--- a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
@@ -9,6 +9,9 @@ define amdgpu_kernel void @sint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %in)
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_cvt_f64_i32_e32 v[0:1], s2
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
@@ -20,6 +23,9 @@ define amdgpu_kernel void @sint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %in)
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f64_i32_e32 v[0:1], s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
@@ -38,11 +44,14 @@ define amdgpu_kernel void @sint_to_fp_i1_f64(ptr addrspace(1) %out, i32 %in) {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_cmp_eq_u32 s2, 0
 ; CI-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, s2
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -52,11 +61,14 @@ define amdgpu_kernel void @sint_to_fp_i1_f64(ptr addrspace(1) %out, i32 %in) {
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -72,6 +84,9 @@ define amdgpu_kernel void @sint_to_fp_i1_f64_load(ptr addrspace(1) %out, i1 %in)
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_bitcmp1_b32 s2, 0
 ; CI-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -86,6 +101,9 @@ define amdgpu_kernel void @sint_to_fp_i1_f64_load(ptr addrspace(1) %out, i1 %in)
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_bitcmp1_b32 s2, 0
 ; VI-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -104,6 +122,9 @@ define amdgpu_kernel void @s_sint_to_fp_i64_to_f64(ptr addrspace(1) %out, i64 %i
 ; CI-LABEL: s_sint_to_fp_i64_to_f64:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_cvt_f64_i32_e32 v[0:1], s3
 ; CI-NEXT:    v_cvt_f64_u32_e32 v[2:3], s2
@@ -117,6 +138,9 @@ define amdgpu_kernel void @s_sint_to_fp_i64_to_f64(ptr addrspace(1) %out, i64 %i
 ; VI-LABEL: s_sint_to_fp_i64_to_f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f64_i32_e32 v[0:1], s3
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[2:3], s2
@@ -136,6 +160,9 @@ define amdgpu_kernel void @v_sint_to_fp_i64_to_f64(ptr addrspace(1) %out, ptr ad
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
@@ -155,6 +182,9 @@ define amdgpu_kernel void @v_sint_to_fp_i64_to_f64(ptr addrspace(1) %out, ptr ad
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -183,6 +213,9 @@ define amdgpu_kernel void @s_sint_to_fp_i8_to_f64(ptr addrspace(1) %out, i8 %in)
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_sext_i32_i8 s2, s2
 ; CI-NEXT:    v_cvt_f64_i32_e32 v[0:1], s2
@@ -195,6 +228,9 @@ define amdgpu_kernel void @s_sint_to_fp_i8_to_f64(ptr addrspace(1) %out, i8 %in)
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_sext_i32_i8 s2, s2
 ; VI-NEXT:    v_cvt_f64_i32_e32 v[0:1], s2
@@ -231,11 +267,14 @@ define amdgpu_kernel void @s_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_cmp_eq_u32 s2, 0
 ; CI-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, s2
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -245,11 +284,14 @@ define amdgpu_kernel void @s_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -282,11 +324,14 @@ define amdgpu_kernel void @s_select_sint_to_fp_i1_vals_i64(ptr addrspace(1) %out
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_cmp_eq_u32 s2, 0
 ; CI-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, s2
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -296,11 +341,14 @@ define amdgpu_kernel void @s_select_sint_to_fp_i1_vals_i64(ptr addrspace(1) %out
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -352,11 +400,14 @@ define amdgpu_kernel void @s_swap_select_sint_to_fp_i1_vals_f64(ptr addrspace(1)
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT:    v_mov_b32_e32 v0, 0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_cmp_eq_u32 s2, 0
 ; CI-NEXT:    s_cselect_b32 s2, 0, 0xbff00000
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v0, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, s2
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -366,11 +417,14 @@ define amdgpu_kernel void @s_swap_select_sint_to_fp_i1_vals_f64(ptr addrspace(1)
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    s_cselect_b32 s2, 0, 0xbff00000
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index 5ae339454a0ba..bd255e88b9512 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -12,10 +12,10 @@ define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
   ; GCN-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
   ; GCN-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
   ; GCN-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
-  ; GCN-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 2228234 /* regdef:VGPR_32 */, def undef %13.sub0
+  ; GCN-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 2228234 /* regdef:VGPR_32 */, def undef %14.sub0
   ; GCN-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
-  ; GCN-NEXT:   GLOBAL_STORE_DWORDX4 undef %23:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
-  ; GCN-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3538953 /* reguse:VReg_64 */, %13
+  ; GCN-NEXT:   GLOBAL_STORE_DWORDX4 undef %24:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+  ; GCN-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3538953 /* reguse:VReg_64 */, %14
   ; GCN-NEXT:   S_ENDPGM 0
   %v0 = call i32 asm sideeffect "; def $0", "=v"()
   %tmp = insertelement <2 x i32> poison, i32 %v0, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
index f791135d45e9a..ef92cf3214e7f 100644
--- a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
@@ -50,7 +50,10 @@ define void @local_store_i56(ptr addrspace(3) %ptr, i56 %arg) #0 {
 define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
 ; HAWAII-LABEL: local_store_i55:
 ; HAWAII:       ; %bb.0:
+; HAWAII-NEXT:    s_add_i32 s12, s12, s17
 ; HAWAII-NEXT:    s_or_b32 s0, s8, 14
+; HAWAII-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; HAWAII-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; HAWAII-NEXT:    v_mov_b32_e32 v0, s0
 ; HAWAII-NEXT:    v_mov_b32_e32 v1, s9
 ; HAWAII-NEXT:    flat_load_ubyte v0, v[0:1]
@@ -70,7 +73,10 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
 ;
 ; FIJI-LABEL: local_store_i55:
 ; FIJI:       ; %bb.0:
+; FIJI-NEXT:    s_add_i32 s12, s12, s17
 ; FIJI-NEXT:    s_or_b32 s0, s8, 14
+; FIJI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; FIJI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; FIJI-NEXT:    v_mov_b32_e32 v0, s0
 ; FIJI-NEXT:    v_mov_b32_e32 v1, s9
 ; FIJI-NEXT:    flat_load_ubyte v0, v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll
index 19d633651fdd0..30accc846d2b6 100644
--- a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll
+++ b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll
@@ -14,7 +14,7 @@ define amdgpu_kernel void @kern() #0 {
 ; OBJ-NEXT: 0000 00000000 00000000 00000000 00000000  ................
 ; OBJ-NEXT: 0010 00000000 00000000 00000000 00000000  ................
 ; OBJ-NEXT: 0020 00000000 00000000 00000000 00000000  ................
-; OBJ-NEXT: 0030 4000af00 88000000 01000000 00000000  @...............
+; OBJ-NEXT: 0030 4000af00 8c000000 21000000 00000000 @.......!.......
 
 ; ELF: AMDGPU Metadata
 ; ELF: .sgpr_count:     9
diff --git a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll
index 2097579e0c995..4f84b31f1877b 100644
--- a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll
+++ b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll
@@ -14,7 +14,7 @@ define amdgpu_kernel void @kern() #0 {
 ; OBJ-NEXT: 0000 00000000 00000000 00000000 00000000  ................
 ; OBJ-NEXT: 0010 00000000 00000000 00000000 00000000  ................
 ; OBJ-NEXT: 0020 00000000 00000000 00000000 00000000  ................
-; OBJ-NEXT: 0030 0000af00 88000000 01000000 00000000  ................
+; OBJ-NEXT: 0030 0000af00 8c000000 21000000 00000000 ........!.......
 
 ; ELF: AMDGPU Metadata
 ; ELF: .sgpr_count:     5
diff --git a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll
index 775c62e73261a..644f434923368 100644
--- a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll
+++ b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll
@@ -14,7 +14,7 @@ define amdgpu_kernel void @kern() #0 {
 ; OBJ-NEXT: 0000 00000000 00000000 00000000 00000000  ................
 ; OBJ-NEXT: 0010 00000000 00000000 00000000 00000000  ................
 ; OBJ-NEXT: 0020 00000000 00000000 00000000 00000000  ................
-; OBJ-NEXT: 0030 4000af00 88000000 01000000 00000000  @...............
+; OBJ-NEXT: 0030 4000af00 8c000000 21000000 00000000 @.......!.......
 
 ; ELF: AMDGPU Metadata
 ; ELF: .sgpr_count:     9
diff --git a/llvm/test/CodeGen/AMDGPU/trap-abis.ll b/llvm/test/CodeGen/AMDGPU/trap-abis.ll
index b8f0d7617167e..69cc63eba6243 100644
--- a/llvm/test/CodeGen/AMDGPU/trap-abis.ll
+++ b/llvm/test/CodeGen/AMDGPU/trap-abis.ll
@@ -23,11 +23,14 @@ define amdgpu_kernel void @trap(ptr addrspace(1) nocapture readonly %arg0) {
 ; HSA-TRAP-GFX803-LABEL: trap:
 ; HSA-TRAP-GFX803:       ; %bb.0:
 ; HSA-TRAP-GFX803-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
+; HSA-TRAP-GFX803-NEXT:    s_add_i32 s12, s12, s17
+; HSA-TRAP-GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; HSA-TRAP-GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v2, 1
-; HSA-TRAP-GFX803-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; HSA-TRAP-GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v0, s2
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v1, s3
+; HSA-TRAP-GFX803-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; HSA-TRAP-GFX803-NEXT:    flat_store_dword v[0:1], v2
 ; HSA-TRAP-GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; HSA-TRAP-GFX803-NEXT:    s_trap 2
@@ -121,6 +124,9 @@ define amdgpu_kernel void @non_entry_trap(ptr addrspace(1) nocapture readonly %a
 ; HSA-TRAP-GFX803-LABEL: non_entry_trap:
 ; HSA-TRAP-GFX803:       ; %bb.0: ; %entry
 ; HSA-TRAP-GFX803-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; HSA-TRAP-GFX803-NEXT:    s_add_i32 s12, s12, s17
+; HSA-TRAP-GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; HSA-TRAP-GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; HSA-TRAP-GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v0, s0
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v1, s1
@@ -280,6 +286,9 @@ define amdgpu_kernel void @trap_with_use_after(ptr addrspace(1) %arg0, ptr addrs
 ; HSA-TRAP-GFX803:       ; %bb.0:
 ; HSA-TRAP-GFX803-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; HSA-TRAP-GFX803-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; HSA-TRAP-GFX803-NEXT:    s_add_i32 s12, s12, s17
+; HSA-TRAP-GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; HSA-TRAP-GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; HSA-TRAP-GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v0, s4
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v1, s5
@@ -411,10 +420,13 @@ define amdgpu_kernel void @debugtrap(ptr addrspace(1) nocapture readonly %arg0)
 ; HSA-TRAP-GFX803-LABEL: debugtrap:
 ; HSA-TRAP-GFX803:       ; %bb.0:
 ; HSA-TRAP-GFX803-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; HSA-TRAP-GFX803-NEXT:    s_add_i32 s12, s12, s17
+; HSA-TRAP-GFX803-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; HSA-TRAP-GFX803-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v2, 1
-; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v3, 2
 ; HSA-TRAP-GFX803-NEXT:    s_waitcnt lgkmcnt(0)
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v3, 2
 ; HSA-TRAP-GFX803-NEXT:    v_mov_b32_e32 v1, s1
 ; HSA-TRAP-GFX803-NEXT:    flat_store_dword v[0:1], v2
 ; HSA-TRAP-GFX803-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index 6e29536feb51b..660ff4677547a 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -81,6 +81,9 @@ define amdgpu_kernel void @udiv_i32(ptr addrspace(1) %out, ptr addrspace(1) %in)
 ; GCN-LABEL: udiv_i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
@@ -252,6 +255,9 @@ define amdgpu_kernel void @s_udiv_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
 ; GCN-LABEL: s_udiv_i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s3
 ; GCN-NEXT:    s_sub_i32 s4, 0, s3
@@ -457,6 +463,9 @@ define amdgpu_kernel void @udiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-LABEL: udiv_v2i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
@@ -810,6 +819,9 @@ define amdgpu_kernel void @udiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-LABEL: udiv_v4i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_add_u32 s4, s2, 16
 ; GCN-NEXT:    s_addc_u32 s5, s3, 0
@@ -1135,6 +1147,9 @@ define amdgpu_kernel void @udiv_i32_div_pow2(ptr addrspace(1) %out, ptr addrspac
 ; GCN-LABEL: udiv_i32_div_pow2:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
@@ -1224,6 +1239,9 @@ define amdgpu_kernel void @udiv_i32_div_k_even(ptr addrspace(1) %out, ptr addrsp
 ; GCN-LABEL: udiv_i32_div_k_even:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
@@ -1318,6 +1336,9 @@ define amdgpu_kernel void @udiv_i32_div_k_odd(ptr addrspace(1) %out, ptr addrspa
 ; GCN-LABEL: udiv_i32_div_k_odd:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
@@ -1430,6 +1451,9 @@ define amdgpu_kernel void @v_udiv_i8(ptr addrspace(1) %out, ptr addrspace(1) %in
 ; GCN-LABEL: v_udiv_i8:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
@@ -1570,6 +1594,9 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-LABEL: v_udiv_i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
@@ -1726,6 +1753,9 @@ define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-LABEL: v_udiv_i23:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_add_u32 s4, s2, 4
 ; GCN-NEXT:    s_addc_u32 s5, s3, 0
@@ -1923,6 +1953,9 @@ define amdgpu_kernel void @v_udiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-LABEL: v_udiv_i24:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_add_u32 s4, s2, 4
 ; GCN-NEXT:    s_addc_u32 s5, s3, 0
@@ -2105,6 +2138,9 @@ define amdgpu_kernel void @scalarize_mulhu_4xi32(ptr addrspace(1) nocapture read
 ; GCN-LABEL: scalarize_mulhu_4xi32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -2218,6 +2254,9 @@ define amdgpu_kernel void @test_udiv2(i32 %p) {
 ; GCN-LABEL: test_udiv2:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_lshr_b32 s0, s0, 1
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
@@ -2281,6 +2320,9 @@ define amdgpu_kernel void @test_udiv_3_mulhu(i32 %p) {
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dword s0, s[8:9], 0x0
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0xaaaaaaab
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
 ; GCN-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
@@ -2378,6 +2420,9 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
 ; GCN-LABEL: fdiv_test_denormals:
 ; GCN:       ; %bb.0: ; %bb
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
diff --git a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
index 55cbc14a46706..97738a7944741 100644
--- a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
@@ -9,6 +9,9 @@ define amdgpu_kernel void @v_uint_to_fp_i64_to_f64(ptr addrspace(1) %out, ptr ad
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
@@ -28,6 +31,9 @@ define amdgpu_kernel void @v_uint_to_fp_i64_to_f64(ptr addrspace(1) %out, ptr ad
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
@@ -54,6 +60,9 @@ define amdgpu_kernel void @s_uint_to_fp_i64_to_f64(ptr addrspace(1) %out, i64 %i
 ; SI-LABEL: s_uint_to_fp_i64_to_f64:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[2:3], s2
@@ -67,6 +76,9 @@ define amdgpu_kernel void @s_uint_to_fp_i64_to_f64(ptr addrspace(1) %out, i64 %i
 ; VI-LABEL: s_uint_to_fp_i64_to_f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[2:3], s2
@@ -86,6 +98,9 @@ define amdgpu_kernel void @s_uint_to_fp_v2i64_to_v2f64(ptr addrspace(1) %out, <2
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[2:3], s1
@@ -103,6 +118,9 @@ define amdgpu_kernel void @s_uint_to_fp_v2i64_to_v2f64(ptr addrspace(1) %out, <2
 ; VI-LABEL: s_uint_to_fp_v2i64_to_v2f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[2:3], s1
@@ -128,6 +146,9 @@ define amdgpu_kernel void @s_uint_to_fp_v4i64_to_v4f64(ptr addrspace(1) %out, <4
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x8
 ; SI-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[4:5], s1
@@ -160,6 +181,9 @@ define amdgpu_kernel void @s_uint_to_fp_v4i64_to_v4f64(ptr addrspace(1) %out, <4
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x20
 ; VI-NEXT:    s_load_dwordx2 s[8:9], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[2:3], s7
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[4:5], s5
@@ -196,6 +220,9 @@ define amdgpu_kernel void @s_uint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %i
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; SI-NEXT:    v_mov_b32_e32 v3, s1
@@ -207,6 +234,9 @@ define amdgpu_kernel void @s_uint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %i
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
@@ -222,6 +252,9 @@ define amdgpu_kernel void @s_uint_to_fp_v2i32_to_v2f64(ptr addrspace(1) %out, <2
 ; GCN-LABEL: s_uint_to_fp_v2i32_to_v2f64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT:    s_add_i32 s12, s12, s17
+; GCN-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_f64_u32_e32 v[2:3], s3
 ; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
@@ -239,6 +272,9 @@ define amdgpu_kernel void @s_uint_to_fp_v4i32_to_v4f64(ptr addrspace(1) %out, <4
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[6:7], s3
@@ -259,6 +295,9 @@ define amdgpu_kernel void @s_uint_to_fp_v4i32_to_v4f64(ptr addrspace(1) %out, <4
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[6:7], s3
@@ -286,11 +325,14 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64(ptr addrspace(1) %out, i32 %in)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_eq_u32 s2, 0
 ; SI-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
 ; SI-NEXT:    v_mov_b32_e32 v3, s1
+; SI-NEXT:    v_mov_b32_e32 v0, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s2
 ; SI-NEXT:    v_mov_b32_e32 v2, s0
 ; SI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -300,11 +342,14 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64(ptr addrspace(1) %out, i32 %in)
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -320,6 +365,9 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64_load(ptr addrspace(1) %out, i1 %
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_bitcmp1_b32 s2, 0
 ; SI-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -334,6 +382,9 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64_load(ptr addrspace(1) %out, i1 %
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_bitcmp1_b32 s2, 0
 ; VI-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -353,6 +404,9 @@ define amdgpu_kernel void @s_uint_to_fp_i8_to_f64(ptr addrspace(1) %out, i8 %in)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_and_b32 s2, s2, 0xff
 ; SI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
@@ -365,6 +419,9 @@ define amdgpu_kernel void @s_uint_to_fp_i8_to_f64(ptr addrspace(1) %out, i8 %in)
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_and_b32 s2, s2, 0xff
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
@@ -402,11 +459,14 @@ define amdgpu_kernel void @s_select_uint_to_fp_i1_vals_f64(ptr addrspace(1) %out
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_eq_u32 s2, 0
 ; SI-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
 ; SI-NEXT:    v_mov_b32_e32 v3, s1
+; SI-NEXT:    v_mov_b32_e32 v0, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s2
 ; SI-NEXT:    v_mov_b32_e32 v2, s0
 ; SI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -416,11 +476,14 @@ define amdgpu_kernel void @s_select_uint_to_fp_i1_vals_f64(ptr addrspace(1) %out
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -453,11 +516,14 @@ define amdgpu_kernel void @s_select_uint_to_fp_i1_vals_i64(ptr addrspace(1) %out
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_eq_u32 s2, 0
 ; SI-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
 ; SI-NEXT:    v_mov_b32_e32 v3, s1
+; SI-NEXT:    v_mov_b32_e32 v0, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s2
 ; SI-NEXT:    v_mov_b32_e32 v2, s0
 ; SI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -467,11 +533,14 @@ define amdgpu_kernel void @s_select_uint_to_fp_i1_vals_i64(ptr addrspace(1) %out
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -505,11 +574,14 @@ define amdgpu_kernel void @s_swap_select_uint_to_fp_i1_vals_f64(ptr addrspace(1)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dword s2, s[8:9], 0x2
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    s_add_i32 s12, s12, s17
+; SI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; SI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_eq_u32 s2, 0
 ; SI-NEXT:    s_cselect_b32 s2, 0, 0x3ff00000
 ; SI-NEXT:    v_mov_b32_e32 v3, s1
+; SI-NEXT:    v_mov_b32_e32 v0, 0
 ; SI-NEXT:    v_mov_b32_e32 v1, s2
 ; SI-NEXT:    v_mov_b32_e32 v2, s0
 ; SI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -519,11 +591,14 @@ define amdgpu_kernel void @s_swap_select_uint_to_fp_i1_vals_f64(ptr addrspace(1)
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s2, 0
 ; VI-NEXT:    s_cselect_b32 s2, 0, 0x3ff00000
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll b/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll
index 45ea6b62761cc..ab7e85fdff516 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll
@@ -11,7 +11,7 @@
 define amdgpu_kernel void @__omp_offloading_16_dd2df_main_l9()  {
 ; CHECK-LABEL: __omp_offloading_16_dd2df_main_l9:
 ; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_add_u32 s0, s0, s15
+; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v1, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
index e99a06f497016..1bc25a1386074 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
@@ -95,7 +95,7 @@
   ; Function Attrs: convergent nocallback nofree nounwind willreturn
   declare void @llvm.amdgcn.end.cf.i64(i64) #2
 
-  attributes #0 = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+  attributes #0 = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
   attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
   attributes #2 = { convergent nocallback nofree nounwind willreturn }
   attributes #3 = { convergent nocallback nofree nounwind willreturn memory(none) }
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
index 076d7c9cd8842..0515ffa094329 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
@@ -69,5 +69,5 @@ bb4:
   ret void
 }
 
-attributes #0 = { nounwind "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
+attributes #0 = { nounwind "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
 attributes #1 = { nounwind readnone }

From 16980d5463c787a48ffb78fd9bbe3d9d32757f34 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Tue, 15 Apr 2025 19:34:55 -0300
Subject: [PATCH 052/710] Revert "[Clang] Fix dependent local class
 instantiation bugs" (#135870)

Reverts llvm/llvm-project#134038

This crashes clang as reported here:
https://github.com/llvm/llvm-project/pull/134038#issuecomment-2807092646
---
 clang/docs/ReleaseNotes.rst                   |  1 -
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  3 +
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  | 56 +---------------
 .../CodeGenCXX/local-class-instantiation.cpp  | 64 -------------------
 4 files changed, 4 insertions(+), 120 deletions(-)
 delete mode 100644 clang/test/CodeGenCXX/local-class-instantiation.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 166f26921cb71..ee69af5632f6e 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -457,7 +457,6 @@ Bug Fixes to C++ Support
   by template argument deduction.
 - Clang is now better at instantiating the function definition after its use inside
   of a constexpr lambda. (#GH125747)
-- Fixed a local class member function instantiation bug inside dependent lambdas. (#GH59734), (#GH132208)
 - Clang no longer crashes when trying to unify the types of arrays with
   certain differences in qualifiers (this could happen during template argument
   deduction or when building a ternary operator). (#GH97005)
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 0e81804f8c1e7..d2408a94ad0ab 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -4126,6 +4126,9 @@ Sema::InstantiateClassMembers(SourceLocation PointOfInstantiation,
       if (FunctionDecl *Pattern =
               Function->getInstantiatedFromMemberFunction()) {
 
+        if (Function->isIneligibleOrNotSelected())
+          continue;
+
         if (Function->getTrailingRequiresClause()) {
           ConstraintSatisfaction Satisfaction;
           if (CheckFunctionConstraints(Function, Satisfaction) ||
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index bf5a882ba4f12..5c80077f294c6 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -5597,61 +5597,7 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   Function->setLocation(PatternDecl->getLocation());
   Function->setInnerLocStart(PatternDecl->getInnerLocStart());
   Function->setRangeEnd(PatternDecl->getEndLoc());
-  // Let the instantiation use the Pattern's DeclarationNameLoc, due to the
-  // following awkwardness:
-  //
-  //   1. There are out-of-tree users of getNameInfo().getSourceRange(), who
-  //   expect the source range of the instantiated declaration to be set to
-  //   point to the definition.
-  //
-  //   2. That getNameInfo().getSourceRange() might return the TypeLocInfo's
-  //   location it tracked.
-  //
-  //   3. Function might come from an (implicit) declaration, while the pattern
-  //   comes from a definition. In these cases, we need the PatternDecl's source
-  //   location.
-  //
-  // To that end, we need to more or less tweak the DeclarationNameLoc. However,
-  // we can't blindly copy the DeclarationNameLoc from the PatternDecl to the
-  // function, since it contains associated TypeLocs that should have already
-  // been transformed. So, we rebuild the TypeLoc for that purpose. Technically,
-  // we should create a new function declaration and assign everything we need,
-  // but InstantiateFunctionDefinition updates the declaration in place.
-  auto NameLocPointsToPattern = [&] {
-    DeclarationNameInfo PatternName = PatternDecl->getNameInfo();
-    DeclarationNameLoc PatternNameLoc = PatternName.getInfo();
-    switch (PatternName.getName().getNameKind()) {
-    case DeclarationName::CXXConstructorName:
-    case DeclarationName::CXXDestructorName:
-    case DeclarationName::CXXConversionFunctionName:
-      break;
-    default:
-      // Cases where DeclarationNameLoc doesn't matter, as it merely contains a
-      // source range.
-      return PatternNameLoc;
-    }
-
-    TypeSourceInfo *TSI = Function->getNameInfo().getNamedTypeInfo();
-    // TSI might be null if the function is named by a constructor template id.
-    // E.g. S<T>() {} for class template S with a template parameter T.
-    if (!TSI) {
-      // We don't care about the DeclarationName of the instantiated function,
-      // but only the DeclarationNameLoc. So if the TypeLoc is absent, we do
-      // nothing.
-      return PatternNameLoc;
-    }
-
-    QualType InstT = TSI->getType();
-    // We want to use a TypeLoc that reflects the transformed type while
-    // preserving the source location from the pattern.
-    TypeLocBuilder TLB;
-    TLB.pushTrivial(
-        Context, InstT,
-        PatternNameLoc.getNamedTypeInfo()->getTypeLoc().getBeginLoc());
-    return DeclarationNameLoc::makeNamedTypeLoc(
-        TLB.getTypeSourceInfo(Context, InstT));
-  };
-  Function->setDeclarationNameLoc(NameLocPointsToPattern());
+  Function->setDeclarationNameLoc(PatternDecl->getNameInfo().getInfo());
 
   EnterExpressionEvaluationContext EvalContext(
       *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
diff --git a/clang/test/CodeGenCXX/local-class-instantiation.cpp b/clang/test/CodeGenCXX/local-class-instantiation.cpp
deleted file mode 100644
index 34103a1ee55ef..0000000000000
--- a/clang/test/CodeGenCXX/local-class-instantiation.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-// RUN: %clang_cc1 -std=c++17 %s -emit-llvm -triple %itanium_abi_triple -o - | FileCheck %s
-
-namespace LambdaContainingLocalClasses {
-
-template <typename F>
-void GH59734() {
-  [&](auto param) {
-    struct Guard {
-      Guard() {
-        // Check that we're able to create DeclRefExpr to param at this point.
-        static_assert(__is_same(decltype(param), int), "");
-      }
-      ~Guard() {
-        static_assert(__is_same(decltype(param), int), "");
-      }
-      operator decltype(param)() {
-        return decltype(param)();
-      }
-    };
-    Guard guard;
-    param = guard;
-  }(42);
-}
-
-// Guard::Guard():
-// CHECK-DAG: define {{.*}} @_ZZZN28LambdaContainingLocalClasses7GH59734IiEEvvENKUlT_E_clIiEEDaS1_EN5GuardC2Ev
-// Guard::operator int():
-// CHECK-DAG: define {{.*}} @_ZZZN28LambdaContainingLocalClasses7GH59734IiEEvvENKUlT_E_clIiEEDaS1_EN5GuardcviEv
-// Guard::~Guard():
-// CHECK-DAG: define {{.*}} @_ZZZN28LambdaContainingLocalClasses7GH59734IiEEvvENKUlT_E_clIiEEDaS1_EN5GuardD2Ev
-
-struct S {};
-
-template <class T = void>
-auto GH132208 = [](auto param) {
-  struct OnScopeExit {
-    OnScopeExit() {
-      static_assert(__is_same(decltype(param), S), "");
-    }
-    ~OnScopeExit() {
-      static_assert(__is_same(decltype(param), S), "");
-    }
-    operator decltype(param)() {
-      return decltype(param)();
-    }
-  } pending;
-
-  param = pending;
-};
-
-void bar() {
-  GH59734<int>();
-
-  GH132208<void>(S{});
-}
-
-// OnScopeExit::OnScopeExit():
-// CHECK-DAG: define {{.*}} @_ZZNK28LambdaContainingLocalClasses8GH132208IvEMUlT_E_clINS_1SEEEDaS2_EN11OnScopeExitC2Ev
-// OnScopeExit::operator S():
-// CHECK-DAG: define {{.*}} @_ZZNK28LambdaContainingLocalClasses8GH132208IvEMUlT_E_clINS_1SEEEDaS2_EN11OnScopeExitcvS5_Ev
-// OnScopeExit::~OnScopeExit():
-// CHECK-DAG: define {{.*}} @_ZZNK28LambdaContainingLocalClasses8GH132208IvEMUlT_E_clINS_1SEEEDaS2_EN11OnScopeExitD2Ev
-
-} // namespace LambdaContainingLocalClasses

From af63e1b505453de3e6a281d1b72e62fa8d396b23 Mon Sep 17 00:00:00 2001
From: Erich Keane <ekeane@nvidia.com>
Date: Tue, 15 Apr 2025 15:52:04 -0700
Subject: [PATCH 053/710] [OpenACC][CIR] Implement 'self' lowering on compute
 constructs (#135851)

This is our first attempt at lowering a clause that is an 'operand' in
the OpenACC operand, so it does quite a bit of refactoring. My previous
plans on how to emit the clauses was not viable, so we instead do
'create the op, then use the visitor to fill in the operands'. This
resulted in the 'applyAttributes' function getting removed and a few
other functions simplified.

Additionally, it requires setting the insertion point a little to make
sure we're inserting 'around' the operation correctly.

Finally, since the OpenACC dialect only understands the MLIR types, we
had to introduce a use of the unrealized-conversion-cast, which we'll
probably getting good use out of in the future.
---
 clang/include/clang/AST/OpenACCClause.h     |   5 +
 clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp | 216 +++++++++++---------
 clang/test/CIR/CodeGenOpenACC/kernels.c     |  30 ++-
 clang/test/CIR/CodeGenOpenACC/parallel.c    |  30 ++-
 clang/test/CIR/CodeGenOpenACC/serial.c      |  30 ++-
 5 files changed, 205 insertions(+), 106 deletions(-)

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 3687af76a559f..681567228cbb0 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -430,6 +430,11 @@ class OpenACCSelfClause final
   }
 
   bool isConditionExprClause() const { return HasConditionExpr.has_value(); }
+  bool isVarListClause() const { return !isConditionExprClause(); }
+  bool isEmptySelfClause() const {
+    return (isConditionExprClause() && !hasConditionExpr()) ||
+           (!isConditionExprClause() && getVarList().empty());
+  }
 
   bool hasConditionExpr() const {
     assert(HasConditionExpr.has_value() &&
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index 152f996ed0fed..3bcc6f908a841 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -32,46 +32,51 @@ constexpr bool isOneOfTypes =
 template <typename ToTest, typename T>
 constexpr bool isOneOfTypes<ToTest, T> = std::is_same_v<ToTest, T>;
 
+template <typename OpTy>
 class OpenACCClauseCIREmitter final
-    : public OpenACCClauseVisitor<OpenACCClauseCIREmitter> {
-  CIRGenModule &cgm;
+    : public OpenACCClauseVisitor<OpenACCClauseCIREmitter<OpTy>> {
+  OpTy &operation;
+  CIRGenFunction &cgf;
+  CIRGenBuilderTy &builder;
+
   // This is necessary since a few of the clauses emit differently based on the
   // directive kind they are attached to.
   OpenACCDirectiveKind dirKind;
+  // TODO(cir): This source location should be able to go away once the NYI
+  // diagnostics are gone.
   SourceLocation dirLoc;
 
-  struct AttributeData {
-    // Value of the 'default' attribute, added on 'data' and 'compute'/etc
-    // constructs as a 'default-attr'.
-    std::optional<ClauseDefaultValue> defaultVal = std::nullopt;
-    // For directives that have their device type architectures listed in
-    // attributes (init/shutdown/etc), the list of architectures to be emitted.
-    llvm::SmallVector<mlir::acc::DeviceType> deviceTypeArchs{};
-  } attrData;
-
   void clauseNotImplemented(const OpenACCClause &c) {
-    cgm.errorNYI(c.getSourceRange(), "OpenACC Clause", c.getClauseKind());
+    cgf.cgm.errorNYI(c.getSourceRange(), "OpenACC Clause", c.getClauseKind());
   }
 
 public:
-  OpenACCClauseCIREmitter(CIRGenModule &cgm, OpenACCDirectiveKind dirKind,
-                          SourceLocation dirLoc)
-      : cgm(cgm), dirKind(dirKind), dirLoc(dirLoc) {}
+  OpenACCClauseCIREmitter(OpTy &operation, CIRGenFunction &cgf,
+                          CIRGenBuilderTy &builder,
+                          OpenACCDirectiveKind dirKind, SourceLocation dirLoc)
+      : operation(operation), cgf(cgf), builder(builder), dirKind(dirKind),
+        dirLoc(dirLoc) {}
 
   void VisitClause(const OpenACCClause &clause) {
     clauseNotImplemented(clause);
   }
 
   void VisitDefaultClause(const OpenACCDefaultClause &clause) {
-    switch (clause.getDefaultClauseKind()) {
-    case OpenACCDefaultClauseKind::None:
-      attrData.defaultVal = ClauseDefaultValue::None;
-      break;
-    case OpenACCDefaultClauseKind::Present:
-      attrData.defaultVal = ClauseDefaultValue::Present;
-      break;
-    case OpenACCDefaultClauseKind::Invalid:
-      break;
+    // This type-trait checks if 'op'(the first arg) is one of the mlir::acc
+    // operations listed in the rest of the arguments.
+    if constexpr (isOneOfTypes<OpTy, ParallelOp, SerialOp, KernelsOp, DataOp>) {
+      switch (clause.getDefaultClauseKind()) {
+      case OpenACCDefaultClauseKind::None:
+        operation.setDefaultAttr(ClauseDefaultValue::None);
+        break;
+      case OpenACCDefaultClauseKind::Present:
+        operation.setDefaultAttr(ClauseDefaultValue::Present);
+        break;
+      case OpenACCDefaultClauseKind::Invalid:
+        break;
+      }
+    } else {
+      return clauseNotImplemented(clause);
     }
   }
 
@@ -89,64 +94,70 @@ class OpenACCClauseCIREmitter final
   }
 
   void VisitDeviceTypeClause(const OpenACCDeviceTypeClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, InitOp, ShutdownOp>) {
+      llvm::SmallVector<mlir::Attribute> deviceTypes;
+      std::optional<mlir::ArrayAttr> existingDeviceTypes =
+          operation.getDeviceTypes();
+
+      // Ensure we keep the existing ones, and in the correct 'new' order.
+      if (existingDeviceTypes) {
+        for (const mlir::Attribute &Attr : *existingDeviceTypes)
+          deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+              builder.getContext(),
+              cast<mlir::acc::DeviceTypeAttr>(Attr).getValue()));
+      }
 
-    switch (dirKind) {
-    case OpenACCDirectiveKind::Init:
-    case OpenACCDirectiveKind::Set:
-    case OpenACCDirectiveKind::Shutdown: {
-      // Device type has a list that is either a 'star' (emitted as 'star'),
-      // or an identifer list, all of which get added for attributes.
-
-      for (const DeviceTypeArgument &arg : clause.getArchitectures())
-        attrData.deviceTypeArchs.push_back(decodeDeviceType(arg.first));
-      break;
-    }
-    default:
+      for (const DeviceTypeArgument &arg : clause.getArchitectures()) {
+        deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+            builder.getContext(), decodeDeviceType(arg.first)));
+      }
+      operation.removeDeviceTypesAttr();
+      operation.setDeviceTypesAttr(
+          mlir::ArrayAttr::get(builder.getContext(), deviceTypes));
+    } else if constexpr (isOneOfTypes<OpTy, SetOp>) {
+      assert(!operation.getDeviceTypeAttr() && "already have device-type?");
+      assert(clause.getArchitectures().size() <= 1);
+
+      if (!clause.getArchitectures().empty())
+        operation.setDeviceType(
+            decodeDeviceType(clause.getArchitectures()[0].first));
+    } else {
       return clauseNotImplemented(clause);
     }
   }
 
-  // Apply any of the clauses that resulted in an 'attribute'.
-  template <typename Op>
-  void applyAttributes(CIRGenBuilderTy &builder, Op &op) {
-
-    if (attrData.defaultVal.has_value()) {
-      // FIXME: OpenACC: as we implement this for other directive kinds, we have
-      // to expand this list.
-      // This type-trait checks if 'op'(the first arg) is one of the mlir::acc
-      // operations listed in the rest of the arguments.
-      if constexpr (isOneOfTypes<Op, ParallelOp, SerialOp, KernelsOp, DataOp>)
-        op.setDefaultAttr(*attrData.defaultVal);
-      else
-        cgm.errorNYI(dirLoc, "OpenACC 'default' clause lowering for ", dirKind);
-    }
-
-    if (!attrData.deviceTypeArchs.empty()) {
-      // FIXME: OpenACC: as we implement this for other directive kinds, we have
-      // to expand this list, or more likely, have a 'noop' branch as most other
-      // uses of this apply to the operands instead.
-      // This type-trait checks if 'op'(the first arg) is one of the mlir::acc
-      if constexpr (isOneOfTypes<Op, InitOp, ShutdownOp>) {
-        llvm::SmallVector<mlir::Attribute> deviceTypes;
-        for (mlir::acc::DeviceType DT : attrData.deviceTypeArchs)
-          deviceTypes.push_back(
-              mlir::acc::DeviceTypeAttr::get(builder.getContext(), DT));
-
-        op.setDeviceTypesAttr(
-            mlir::ArrayAttr::get(builder.getContext(), deviceTypes));
-      } else if constexpr (isOneOfTypes<Op, SetOp>) {
-        assert(attrData.deviceTypeArchs.size() <= 1 &&
-               "Set can only have a single architecture");
-        if (!attrData.deviceTypeArchs.empty())
-          op.setDeviceType(attrData.deviceTypeArchs[0]);
+  void VisitSelfClause(const OpenACCSelfClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, ParallelOp, SerialOp, KernelsOp>) {
+      if (clause.isEmptySelfClause()) {
+        operation.setSelfAttr(true);
+      } else if (clause.isConditionExprClause()) {
+        assert(clause.hasConditionExpr());
+        mlir::Value condition =
+            cgf.evaluateExprAsBool(clause.getConditionExpr());
+
+        mlir::Location exprLoc =
+            cgf.cgm.getLoc(clause.getConditionExpr()->getBeginLoc());
+        mlir::IntegerType targetType = mlir::IntegerType::get(
+            &cgf.getMLIRContext(), /*width=*/1,
+            mlir::IntegerType::SignednessSemantics::Signless);
+        auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>(
+            exprLoc, targetType, condition);
+        operation.getSelfCondMutable().append(conversionOp.getResult(0));
       } else {
-        cgm.errorNYI(dirLoc, "OpenACC 'device_type' clause lowering for ",
-                     dirKind);
+        llvm_unreachable("var-list version of self shouldn't get here");
       }
+    } else {
+      return clauseNotImplemented(clause);
     }
   }
 };
 
+template <typename OpTy>
+auto makeClauseEmitter(OpTy &op, CIRGenFunction &cgf, CIRGenBuilderTy &builder,
+                       OpenACCDirectiveKind dirKind, SourceLocation dirLoc) {
+  return OpenACCClauseCIREmitter<OpTy>(op, cgf, builder, dirKind, dirLoc);
+}
+
 } // namespace
 
 template <typename Op, typename TermOp>
@@ -158,24 +169,27 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpAssociatedStmt(
 
   llvm::SmallVector<mlir::Type> retTy;
   llvm::SmallVector<mlir::Value> operands;
-
-  // Clause-emitter must be here because it might modify operands.
-  OpenACCClauseCIREmitter clauseEmitter(getCIRGenModule(), dirKind, dirLoc);
-  clauseEmitter.VisitClauseList(clauses);
-
   auto op = builder.create<Op>(start, retTy, operands);
 
-  // Apply the attributes derived from the clauses.
-  clauseEmitter.applyAttributes(builder, op);
+  {
+    mlir::OpBuilder::InsertionGuard guardCase(builder);
+    // Sets insertion point before the 'op', since every new expression needs to
+    // be before the operation.
+    builder.setInsertionPoint(op);
+    makeClauseEmitter(op, *this, builder, dirKind, dirLoc)
+        .VisitClauseList(clauses);
+  }
 
-  mlir::Block &block = op.getRegion().emplaceBlock();
-  mlir::OpBuilder::InsertionGuard guardCase(builder);
-  builder.setInsertionPointToEnd(&block);
+  {
+    mlir::Block &block = op.getRegion().emplaceBlock();
+    mlir::OpBuilder::InsertionGuard guardCase(builder);
+    builder.setInsertionPointToEnd(&block);
 
-  LexicalScope ls{*this, start, builder.getInsertionBlock()};
-  res = emitStmt(associatedStmt, /*useCurrentScope=*/true);
+    LexicalScope ls{*this, start, builder.getInsertionBlock()};
+    res = emitStmt(associatedStmt, /*useCurrentScope=*/true);
 
-  builder.create<TermOp>(end);
+    builder.create<TermOp>(end);
+  }
   return res;
 }
 
@@ -187,14 +201,16 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOp(
 
   llvm::SmallVector<mlir::Type> retTy;
   llvm::SmallVector<mlir::Value> operands;
-
-  // Clause-emitter must be here because it might modify operands.
-  OpenACCClauseCIREmitter clauseEmitter(getCIRGenModule(), dirKind, dirLoc);
-  clauseEmitter.VisitClauseList(clauses);
-
   auto op = builder.create<Op>(start, retTy, operands);
-  // Apply the attributes derived from the clauses.
-  clauseEmitter.applyAttributes(builder, op);
+
+  {
+    mlir::OpBuilder::InsertionGuard guardCase(builder);
+    // Sets insertion point before the 'op', since every new expression needs to
+    // be before the operation.
+    builder.setInsertionPoint(op);
+    makeClauseEmitter(op, *this, builder, dirKind, dirLoc)
+        .VisitClauseList(clauses);
+  }
   return res;
 }
 
@@ -254,46 +270,46 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCShutdownConstruct(
 
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC Loop Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC Loop Construct");
   return mlir::failure();
 }
 mlir::LogicalResult CIRGenFunction::emitOpenACCCombinedConstruct(
     const OpenACCCombinedConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC Combined Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC Combined Construct");
   return mlir::failure();
 }
 mlir::LogicalResult CIRGenFunction::emitOpenACCEnterDataConstruct(
     const OpenACCEnterDataConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC EnterData Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC EnterData Construct");
   return mlir::failure();
 }
 mlir::LogicalResult CIRGenFunction::emitOpenACCExitDataConstruct(
     const OpenACCExitDataConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC ExitData Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC ExitData Construct");
   return mlir::failure();
 }
 mlir::LogicalResult CIRGenFunction::emitOpenACCHostDataConstruct(
     const OpenACCHostDataConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC HostData Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC HostData Construct");
   return mlir::failure();
 }
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCWaitConstruct(const OpenACCWaitConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC Wait Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC Wait Construct");
   return mlir::failure();
 }
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC Update Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC Update Construct");
   return mlir::failure();
 }
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC Atomic Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC Atomic Construct");
   return mlir::failure();
 }
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s) {
-  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenACC Cache Construct");
+  cgm.errorNYI(s.getSourceRange(), "OpenACC Cache Construct");
   return mlir::failure();
 }
diff --git a/clang/test/CIR/CodeGenOpenACC/kernels.c b/clang/test/CIR/CodeGenOpenACC/kernels.c
index 0c950fe3d0f9c..934daf9e8ecc0 100644
--- a/clang/test/CIR/CodeGenOpenACC/kernels.c
+++ b/clang/test/CIR/CodeGenOpenACC/kernels.c
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
 
-void acc_kernels(void) {
-  // CHECK: cir.func @acc_kernels() {
+void acc_kernels(int cond) {
+  // CHECK: cir.func @acc_kernels(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 #pragma acc kernels
   {}
 
@@ -38,5 +40,29 @@ void acc_kernels(void) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT:}
 
+#pragma acc kernels self
+  {}
+  // CHECK-NEXT: acc.kernels {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } attributes {selfAttr}
+
+#pragma acc kernels self(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[COND_LOAD]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.kernels self(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels self(0)
+  {}
+  // CHECK-NEXT: %[[ZERO_LITERAL:.*]] = cir.const #cir.int<0> : !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[ZERO_LITERAL]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.kernels self(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+ 
   // CHECK-NEXT: cir.return
 }
diff --git a/clang/test/CIR/CodeGenOpenACC/parallel.c b/clang/test/CIR/CodeGenOpenACC/parallel.c
index e18270435460c..c7a4bda6faa74 100644
--- a/clang/test/CIR/CodeGenOpenACC/parallel.c
+++ b/clang/test/CIR/CodeGenOpenACC/parallel.c
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
 
-void acc_parallel(void) {
-  // CHECK: cir.func @acc_parallel() {
+void acc_parallel(int cond) {
+  // CHECK: cir.func @acc_parallel(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 #pragma acc parallel
   {}
   // CHECK-NEXT: acc.parallel {
@@ -37,5 +39,29 @@ void acc_parallel(void) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT:}
 
+#pragma acc parallel self
+  {}
+  // CHECK-NEXT: acc.parallel {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } attributes {selfAttr}
+
+#pragma acc parallel self(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[COND_LOAD]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.parallel self(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel self(0)
+  {}
+  // CHECK-NEXT: %[[ZERO_LITERAL:.*]] = cir.const #cir.int<0> : !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[ZERO_LITERAL]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.parallel self(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }
diff --git a/clang/test/CIR/CodeGenOpenACC/serial.c b/clang/test/CIR/CodeGenOpenACC/serial.c
index 72a0995549da3..38a38ad6c9514 100644
--- a/clang/test/CIR/CodeGenOpenACC/serial.c
+++ b/clang/test/CIR/CodeGenOpenACC/serial.c
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
 
-void acc_serial(void) {
-  // CHECK: cir.func @acc_serial() {
+void acc_serial(int cond) {
+  // CHECK: cir.func @acc_serial(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 #pragma acc serial
   {}
 
@@ -38,5 +40,29 @@ void acc_serial(void) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT:}
 
+#pragma acc serial self
+  {}
+  // CHECK-NEXT: acc.serial {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } attributes {selfAttr}
+
+#pragma acc serial self(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[COND_LOAD]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.serial self(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial self(0)
+  {}
+  // CHECK-NEXT: %[[ZERO_LITERAL:.*]] = cir.const #cir.int<0> : !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[ZERO_LITERAL]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.serial self(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }

From e4d951d2e42a9124bd87275a864804c4b84b62e3 Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <peter@pcc.me.uk>
Date: Tue, 15 Apr 2025 16:03:03 -0700
Subject: [PATCH 054/710] LowerTypeTests: Fix quadratic complexity.

Currently we have quadratic complexity in LowerTypeTests because
ScopedSaveAliaseesAndUsed loops over all aliases for each disjoint
set, and the number of aliases and number of disjoint sets is
roughly proportional to the program size. Fix that by moving
ScopedSaveAliaseesAndUsed to LowerTypeTestsModule::lower() so that
we do this only once.

Reviewers: fmayer, vitalybuka

Reviewed By: vitalybuka

Pull Request: https://github.com/llvm/llvm-project/pull/135875
---
 llvm/lib/Transforms/IPO/LowerTypeTests.cpp | 164 ++++++++++-----------
 1 file changed, 81 insertions(+), 83 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 7cf7d74acfcfa..8e9f24dfc31fa 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -1669,61 +1669,55 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
 
   lowerTypeTestCalls(TypeIds, JumpTable, GlobalLayout);
 
-  {
-    ScopedSaveAliaseesAndUsed S(M);
+  // Build aliases pointing to offsets into the jump table, and replace
+  // references to the original functions with references to the aliases.
+  for (unsigned I = 0; I != Functions.size(); ++I) {
+    Function *F = cast<Function>(Functions[I]->getGlobal());
+    bool IsJumpTableCanonical = Functions[I]->isJumpTableCanonical();
 
-    // Build aliases pointing to offsets into the jump table, and replace
-    // references to the original functions with references to the aliases.
-    for (unsigned I = 0; I != Functions.size(); ++I) {
-      Function *F = cast<Function>(Functions[I]->getGlobal());
-      bool IsJumpTableCanonical = Functions[I]->isJumpTableCanonical();
-
-      Constant *CombinedGlobalElemPtr = ConstantExpr::getInBoundsGetElementPtr(
-          JumpTableType, JumpTable,
-          ArrayRef<Constant *>{ConstantInt::get(IntPtrTy, 0),
-                               ConstantInt::get(IntPtrTy, I)});
-
-      const bool IsExported = Functions[I]->isExported();
-      if (!IsJumpTableCanonical) {
-        GlobalValue::LinkageTypes LT = IsExported
-                                           ? GlobalValue::ExternalLinkage
-                                           : GlobalValue::InternalLinkage;
-        GlobalAlias *JtAlias = GlobalAlias::create(F->getValueType(), 0, LT,
-                                                   F->getName() + ".cfi_jt",
-                                                   CombinedGlobalElemPtr, &M);
-        if (IsExported)
-          JtAlias->setVisibility(GlobalValue::HiddenVisibility);
-        else
-          appendToUsed(M, {JtAlias});
-      }
+    Constant *CombinedGlobalElemPtr = ConstantExpr::getInBoundsGetElementPtr(
+        JumpTableType, JumpTable,
+        ArrayRef<Constant *>{ConstantInt::get(IntPtrTy, 0),
+                             ConstantInt::get(IntPtrTy, I)});
+
+    const bool IsExported = Functions[I]->isExported();
+    if (!IsJumpTableCanonical) {
+      GlobalValue::LinkageTypes LT = IsExported ? GlobalValue::ExternalLinkage
+                                                : GlobalValue::InternalLinkage;
+      GlobalAlias *JtAlias = GlobalAlias::create(F->getValueType(), 0, LT,
+                                                 F->getName() + ".cfi_jt",
+                                                 CombinedGlobalElemPtr, &M);
+      if (IsExported)
+        JtAlias->setVisibility(GlobalValue::HiddenVisibility);
+      else
+        appendToUsed(M, {JtAlias});
+    }
 
-      if (IsExported) {
-        if (IsJumpTableCanonical)
-          ExportSummary->cfiFunctionDefs().emplace(F->getName());
-        else
-          ExportSummary->cfiFunctionDecls().emplace(F->getName());
-      }
+    if (IsExported) {
+      if (IsJumpTableCanonical)
+        ExportSummary->cfiFunctionDefs().emplace(F->getName());
+      else
+        ExportSummary->cfiFunctionDecls().emplace(F->getName());
+    }
 
-      if (!IsJumpTableCanonical) {
-        if (F->hasExternalWeakLinkage())
-          replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr,
-                                                 IsJumpTableCanonical);
-        else
-          replaceCfiUses(F, CombinedGlobalElemPtr, IsJumpTableCanonical);
-      } else {
-        assert(F->getType()->getAddressSpace() == 0);
-
-        GlobalAlias *FAlias =
-            GlobalAlias::create(F->getValueType(), 0, F->getLinkage(), "",
-                                CombinedGlobalElemPtr, &M);
-        FAlias->setVisibility(F->getVisibility());
-        FAlias->takeName(F);
-        if (FAlias->hasName())
-          F->setName(FAlias->getName() + ".cfi");
-        replaceCfiUses(F, FAlias, IsJumpTableCanonical);
-        if (!F->hasLocalLinkage())
-          F->setVisibility(GlobalVariable::HiddenVisibility);
-      }
+    if (!IsJumpTableCanonical) {
+      if (F->hasExternalWeakLinkage())
+        replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr,
+                                               IsJumpTableCanonical);
+      else
+        replaceCfiUses(F, CombinedGlobalElemPtr, IsJumpTableCanonical);
+    } else {
+      assert(F->getType()->getAddressSpace() == 0);
+
+      GlobalAlias *FAlias = GlobalAlias::create(
+          F->getValueType(), 0, F->getLinkage(), "", CombinedGlobalElemPtr, &M);
+      FAlias->setVisibility(F->getVisibility());
+      FAlias->takeName(F);
+      if (FAlias->hasName())
+        F->setName(FAlias->getName() + ".cfi");
+      replaceCfiUses(F, FAlias, IsJumpTableCanonical);
+      if (!F->hasLocalLinkage())
+        F->setVisibility(GlobalVariable::HiddenVisibility);
     }
   }
 
@@ -2339,39 +2333,43 @@ bool LowerTypeTestsModule::lower() {
   if (GlobalClasses.empty())
     return false;
 
-  // For each disjoint set we found...
-  for (const auto &C : GlobalClasses) {
-    if (!C->isLeader())
-      continue;
-
-    ++NumTypeIdDisjointSets;
-    // Build the list of type identifiers in this disjoint set.
-    std::vector<Metadata *> TypeIds;
-    std::vector<GlobalTypeMember *> Globals;
-    std::vector<ICallBranchFunnel *> ICallBranchFunnels;
-    for (auto M : GlobalClasses.members(*C)) {
-      if (isa<Metadata *>(M))
-        TypeIds.push_back(cast<Metadata *>(M));
-      else if (isa<GlobalTypeMember *>(M))
-        Globals.push_back(cast<GlobalTypeMember *>(M));
-      else
-        ICallBranchFunnels.push_back(cast<ICallBranchFunnel *>(M));
-    }
-
-    // Order type identifiers by unique ID for determinism. This ordering is
-    // stable as there is a one-to-one mapping between metadata and unique IDs.
-    llvm::sort(TypeIds, [&](Metadata *M1, Metadata *M2) {
-      return TypeIdInfo[M1].UniqueId < TypeIdInfo[M2].UniqueId;
-    });
+  {
+    ScopedSaveAliaseesAndUsed S(M);
+    // For each disjoint set we found...
+    for (const auto &C : GlobalClasses) {
+      if (!C->isLeader())
+        continue;
 
-    // Same for the branch funnels.
-    llvm::sort(ICallBranchFunnels,
-               [&](ICallBranchFunnel *F1, ICallBranchFunnel *F2) {
-                 return F1->UniqueId < F2->UniqueId;
-               });
+      ++NumTypeIdDisjointSets;
+      // Build the list of type identifiers in this disjoint set.
+      std::vector<Metadata *> TypeIds;
+      std::vector<GlobalTypeMember *> Globals;
+      std::vector<ICallBranchFunnel *> ICallBranchFunnels;
+      for (auto M : GlobalClasses.members(*C)) {
+        if (isa<Metadata *>(M))
+          TypeIds.push_back(cast<Metadata *>(M));
+        else if (isa<GlobalTypeMember *>(M))
+          Globals.push_back(cast<GlobalTypeMember *>(M));
+        else
+          ICallBranchFunnels.push_back(cast<ICallBranchFunnel *>(M));
+      }
 
-    // Build bitsets for this disjoint set.
-    buildBitSetsFromDisjointSet(TypeIds, Globals, ICallBranchFunnels);
+      // Order type identifiers by unique ID for determinism. This ordering is
+      // stable as there is a one-to-one mapping between metadata and unique
+      // IDs.
+      llvm::sort(TypeIds, [&](Metadata *M1, Metadata *M2) {
+        return TypeIdInfo[M1].UniqueId < TypeIdInfo[M2].UniqueId;
+      });
+
+      // Same for the branch funnels.
+      llvm::sort(ICallBranchFunnels,
+                 [&](ICallBranchFunnel *F1, ICallBranchFunnel *F2) {
+                   return F1->UniqueId < F2->UniqueId;
+                 });
+
+      // Build bitsets for this disjoint set.
+      buildBitSetsFromDisjointSet(TypeIds, Globals, ICallBranchFunnels);
+    }
   }
 
   allocateByteArrays();

From 3428cc94c893f9a09728c707baf018b9cdfaf243 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn@nvidia.com>
Date: Tue, 15 Apr 2025 19:04:59 -0400
Subject: [PATCH 055/710] [flang] Implement external routine usage of hostnm()
 (#134900)

Previously, `hostnm` extended intrinsic was implemented as proper
intrinsic. Since then we found out that some applications use `hostnm`
as external routine via `external hostnm`. This prevents `hostnm` from
being recognized as an intrinsic. This PR implements `hostnm` as
external routine.
---
 flang-rt/lib/runtime/extensions.cpp      | 32 ++++++++++++++++++++++++
 flang/include/flang/Runtime/extensions.h |  2 +-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/flang-rt/lib/runtime/extensions.cpp b/flang-rt/lib/runtime/extensions.cpp
index 6b553ff97e5ab..e70dff3997233 100644
--- a/flang-rt/lib/runtime/extensions.cpp
+++ b/flang-rt/lib/runtime/extensions.cpp
@@ -264,6 +264,38 @@ int RTNAME(Chdir)(const char *name) {
 #endif
 }
 
+int FORTRAN_PROCEDURE_NAME(hostnm)(char *hn, int length) {
+  std::int32_t status{0};
+
+  if (!hn || length < 0) {
+    return EINVAL;
+  }
+
+#ifdef _WIN32
+  DWORD dwSize{static_cast<DWORD>(length)};
+
+  // Note: Winsock has gethostname(), but use Win32 API GetComputerNameEx(),
+  // in order to avoid adding dependency on Winsock.
+  if (!GetComputerNameExA(ComputerNameDnsHostname, hn, &dwSize)) {
+    status = GetLastError();
+  }
+#else
+  if (gethostname(hn, length) < 0) {
+    status = errno;
+  }
+#endif
+
+  if (status == 0) {
+    // Find zero terminator and fill the string from the
+    // zero terminator to the end with spaces
+    char *str_end{hn + length};
+    char *str_zero{std::find(hn, str_end, '\0')};
+    std::fill(str_zero, str_end, ' ');
+  }
+
+  return status;
+}
+
 int FORTRAN_PROCEDURE_NAME(ierrno)() { return errno; }
 
 void FORTRAN_PROCEDURE_NAME(qsort)(int *array, int *len, int *isize,
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index db2245875e85a..06ae7f35d9b5b 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -60,7 +60,7 @@ uid_t RTNAME(GetUID)();
 void FORTRAN_PROCEDURE_NAME(getlog)(char *name, std::int64_t length);
 
 // GNU extension subroutine HOSTNM(C)
-void FORTRAN_PROCEDURE_NAME(hostnm)(char *name, std::int64_t length);
+int FORTRAN_PROCEDURE_NAME(hostnm)(char *hn, int length);
 
 std::intptr_t RTNAME(Malloc)(std::size_t size);
 

From 58c3fba7063eaca926931a412c329e9ac4deefd6 Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <peter@pcc.me.uk>
Date: Tue, 15 Apr 2025 16:20:45 -0700
Subject: [PATCH 056/710] Revert "LowerTypeTests: Fix quadratic complexity."

This reverts commit e4d951d2e42a9124bd87275a864804c4b84b62e3.

Need to investigate some test failures.
---
 llvm/lib/Transforms/IPO/LowerTypeTests.cpp | 164 +++++++++++----------
 1 file changed, 83 insertions(+), 81 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 8e9f24dfc31fa..7cf7d74acfcfa 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -1669,55 +1669,61 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
 
   lowerTypeTestCalls(TypeIds, JumpTable, GlobalLayout);
 
-  // Build aliases pointing to offsets into the jump table, and replace
-  // references to the original functions with references to the aliases.
-  for (unsigned I = 0; I != Functions.size(); ++I) {
-    Function *F = cast<Function>(Functions[I]->getGlobal());
-    bool IsJumpTableCanonical = Functions[I]->isJumpTableCanonical();
+  {
+    ScopedSaveAliaseesAndUsed S(M);
 
-    Constant *CombinedGlobalElemPtr = ConstantExpr::getInBoundsGetElementPtr(
-        JumpTableType, JumpTable,
-        ArrayRef<Constant *>{ConstantInt::get(IntPtrTy, 0),
-                             ConstantInt::get(IntPtrTy, I)});
-
-    const bool IsExported = Functions[I]->isExported();
-    if (!IsJumpTableCanonical) {
-      GlobalValue::LinkageTypes LT = IsExported ? GlobalValue::ExternalLinkage
-                                                : GlobalValue::InternalLinkage;
-      GlobalAlias *JtAlias = GlobalAlias::create(F->getValueType(), 0, LT,
-                                                 F->getName() + ".cfi_jt",
-                                                 CombinedGlobalElemPtr, &M);
-      if (IsExported)
-        JtAlias->setVisibility(GlobalValue::HiddenVisibility);
-      else
-        appendToUsed(M, {JtAlias});
-    }
+    // Build aliases pointing to offsets into the jump table, and replace
+    // references to the original functions with references to the aliases.
+    for (unsigned I = 0; I != Functions.size(); ++I) {
+      Function *F = cast<Function>(Functions[I]->getGlobal());
+      bool IsJumpTableCanonical = Functions[I]->isJumpTableCanonical();
+
+      Constant *CombinedGlobalElemPtr = ConstantExpr::getInBoundsGetElementPtr(
+          JumpTableType, JumpTable,
+          ArrayRef<Constant *>{ConstantInt::get(IntPtrTy, 0),
+                               ConstantInt::get(IntPtrTy, I)});
+
+      const bool IsExported = Functions[I]->isExported();
+      if (!IsJumpTableCanonical) {
+        GlobalValue::LinkageTypes LT = IsExported
+                                           ? GlobalValue::ExternalLinkage
+                                           : GlobalValue::InternalLinkage;
+        GlobalAlias *JtAlias = GlobalAlias::create(F->getValueType(), 0, LT,
+                                                   F->getName() + ".cfi_jt",
+                                                   CombinedGlobalElemPtr, &M);
+        if (IsExported)
+          JtAlias->setVisibility(GlobalValue::HiddenVisibility);
+        else
+          appendToUsed(M, {JtAlias});
+      }
 
-    if (IsExported) {
-      if (IsJumpTableCanonical)
-        ExportSummary->cfiFunctionDefs().emplace(F->getName());
-      else
-        ExportSummary->cfiFunctionDecls().emplace(F->getName());
-    }
+      if (IsExported) {
+        if (IsJumpTableCanonical)
+          ExportSummary->cfiFunctionDefs().emplace(F->getName());
+        else
+          ExportSummary->cfiFunctionDecls().emplace(F->getName());
+      }
 
-    if (!IsJumpTableCanonical) {
-      if (F->hasExternalWeakLinkage())
-        replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr,
-                                               IsJumpTableCanonical);
-      else
-        replaceCfiUses(F, CombinedGlobalElemPtr, IsJumpTableCanonical);
-    } else {
-      assert(F->getType()->getAddressSpace() == 0);
-
-      GlobalAlias *FAlias = GlobalAlias::create(
-          F->getValueType(), 0, F->getLinkage(), "", CombinedGlobalElemPtr, &M);
-      FAlias->setVisibility(F->getVisibility());
-      FAlias->takeName(F);
-      if (FAlias->hasName())
-        F->setName(FAlias->getName() + ".cfi");
-      replaceCfiUses(F, FAlias, IsJumpTableCanonical);
-      if (!F->hasLocalLinkage())
-        F->setVisibility(GlobalVariable::HiddenVisibility);
+      if (!IsJumpTableCanonical) {
+        if (F->hasExternalWeakLinkage())
+          replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr,
+                                                 IsJumpTableCanonical);
+        else
+          replaceCfiUses(F, CombinedGlobalElemPtr, IsJumpTableCanonical);
+      } else {
+        assert(F->getType()->getAddressSpace() == 0);
+
+        GlobalAlias *FAlias =
+            GlobalAlias::create(F->getValueType(), 0, F->getLinkage(), "",
+                                CombinedGlobalElemPtr, &M);
+        FAlias->setVisibility(F->getVisibility());
+        FAlias->takeName(F);
+        if (FAlias->hasName())
+          F->setName(FAlias->getName() + ".cfi");
+        replaceCfiUses(F, FAlias, IsJumpTableCanonical);
+        if (!F->hasLocalLinkage())
+          F->setVisibility(GlobalVariable::HiddenVisibility);
+      }
     }
   }
 
@@ -2333,43 +2339,39 @@ bool LowerTypeTestsModule::lower() {
   if (GlobalClasses.empty())
     return false;
 
-  {
-    ScopedSaveAliaseesAndUsed S(M);
-    // For each disjoint set we found...
-    for (const auto &C : GlobalClasses) {
-      if (!C->isLeader())
-        continue;
-
-      ++NumTypeIdDisjointSets;
-      // Build the list of type identifiers in this disjoint set.
-      std::vector<Metadata *> TypeIds;
-      std::vector<GlobalTypeMember *> Globals;
-      std::vector<ICallBranchFunnel *> ICallBranchFunnels;
-      for (auto M : GlobalClasses.members(*C)) {
-        if (isa<Metadata *>(M))
-          TypeIds.push_back(cast<Metadata *>(M));
-        else if (isa<GlobalTypeMember *>(M))
-          Globals.push_back(cast<GlobalTypeMember *>(M));
-        else
-          ICallBranchFunnels.push_back(cast<ICallBranchFunnel *>(M));
-      }
+  // For each disjoint set we found...
+  for (const auto &C : GlobalClasses) {
+    if (!C->isLeader())
+      continue;
 
-      // Order type identifiers by unique ID for determinism. This ordering is
-      // stable as there is a one-to-one mapping between metadata and unique
-      // IDs.
-      llvm::sort(TypeIds, [&](Metadata *M1, Metadata *M2) {
-        return TypeIdInfo[M1].UniqueId < TypeIdInfo[M2].UniqueId;
-      });
-
-      // Same for the branch funnels.
-      llvm::sort(ICallBranchFunnels,
-                 [&](ICallBranchFunnel *F1, ICallBranchFunnel *F2) {
-                   return F1->UniqueId < F2->UniqueId;
-                 });
-
-      // Build bitsets for this disjoint set.
-      buildBitSetsFromDisjointSet(TypeIds, Globals, ICallBranchFunnels);
+    ++NumTypeIdDisjointSets;
+    // Build the list of type identifiers in this disjoint set.
+    std::vector<Metadata *> TypeIds;
+    std::vector<GlobalTypeMember *> Globals;
+    std::vector<ICallBranchFunnel *> ICallBranchFunnels;
+    for (auto M : GlobalClasses.members(*C)) {
+      if (isa<Metadata *>(M))
+        TypeIds.push_back(cast<Metadata *>(M));
+      else if (isa<GlobalTypeMember *>(M))
+        Globals.push_back(cast<GlobalTypeMember *>(M));
+      else
+        ICallBranchFunnels.push_back(cast<ICallBranchFunnel *>(M));
     }
+
+    // Order type identifiers by unique ID for determinism. This ordering is
+    // stable as there is a one-to-one mapping between metadata and unique IDs.
+    llvm::sort(TypeIds, [&](Metadata *M1, Metadata *M2) {
+      return TypeIdInfo[M1].UniqueId < TypeIdInfo[M2].UniqueId;
+    });
+
+    // Same for the branch funnels.
+    llvm::sort(ICallBranchFunnels,
+               [&](ICallBranchFunnel *F1, ICallBranchFunnel *F2) {
+                 return F1->UniqueId < F2->UniqueId;
+               });
+
+    // Build bitsets for this disjoint set.
+    buildBitSetsFromDisjointSet(TypeIds, Globals, ICallBranchFunnels);
   }
 
   allocateByteArrays();

From 860d0383db80ea881e957a5628d04e9d725b919d Mon Sep 17 00:00:00 2001
From: Haowei <haowei@google.com>
Date: Tue, 15 Apr 2025 16:23:53 -0700
Subject: [PATCH 057/710] [Fuchsia] Not building llvm-mt when LIBXML2 is not
 enabled. (#135877)

This patch prevents including the llvm-mt to `LLVM_TOOLCHAIN_TOOLS` in the
Fuchsia toolchain when LIBXML2 is not explicitly enabled.
---
 clang/cmake/caches/Fuchsia-stage2.cmake | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 99890b8246ad7..e10855f5ef31b 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -459,7 +459,6 @@ set(LLVM_TOOLCHAIN_TOOLS
   llvm-libtool-darwin
   llvm-lipo
   llvm-ml
-  llvm-mt
   llvm-nm
   llvm-objcopy
   llvm-objdump
@@ -481,6 +480,10 @@ set(LLVM_TOOLCHAIN_TOOLS
   scan-build-py
   CACHE STRING "")
 
+if (LLVM_ENABLE_LIBXML2)
+  list(APPEND LLVM_TOOLCHAIN_TOOLS llvm-mt)
+endif()
+
 set(LLVM_Toolchain_DISTRIBUTION_COMPONENTS
   bolt
   clang

From 6ad922b75a41911e0e394d5d367bee1240ad509f Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Tue, 15 Apr 2025 16:14:49 -0700
Subject: [PATCH 058/710] [OpenACC][CIR] Implement lowering for 'if' on compute
 constructs

This is the same for these as the 'self' was, except it doesn't support
the 'empty' variant, so we have to just generate the condition.  This
patch does that, and extracts the 'condition' emission to a separate
function since the two share it.
---
 clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp | 40 +++++++++++++-----
 clang/test/CIR/CodeGenOpenACC/kernels.c     | 46 ++++++++++++++++++++-
 clang/test/CIR/CodeGenOpenACC/parallel.c    | 44 +++++++++++++++++++-
 clang/test/CIR/CodeGenOpenACC/serial.c      | 44 +++++++++++++++++++-
 4 files changed, 159 insertions(+), 15 deletions(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index 3bcc6f908a841..c14ff9a16841d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -50,6 +50,21 @@ class OpenACCClauseCIREmitter final
     cgf.cgm.errorNYI(c.getSourceRange(), "OpenACC Clause", c.getClauseKind());
   }
 
+  // 'condition' as an OpenACC grammar production is used for 'if' and (some
+  // variants of) 'self'.  It needs to be emitted as a signless-1-bit value, so
+  // this function emits the expression, then sets the unrealized conversion
+  // cast correctly, and returns the completed value.
+  mlir::Value createCondition(const Expr *condExpr) {
+    mlir::Value condition = cgf.evaluateExprAsBool(condExpr);
+    mlir::Location exprLoc = cgf.cgm.getLoc(condExpr->getBeginLoc());
+    mlir::IntegerType targetType = mlir::IntegerType::get(
+        &cgf.getMLIRContext(), /*width=*/1,
+        mlir::IntegerType::SignednessSemantics::Signless);
+    auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>(
+        exprLoc, targetType, condition);
+    return conversionOp.getResult(0);
+  }
+
 public:
   OpenACCClauseCIREmitter(OpTy &operation, CIRGenFunction &cgf,
                           CIRGenBuilderTy &builder,
@@ -132,17 +147,8 @@ class OpenACCClauseCIREmitter final
         operation.setSelfAttr(true);
       } else if (clause.isConditionExprClause()) {
         assert(clause.hasConditionExpr());
-        mlir::Value condition =
-            cgf.evaluateExprAsBool(clause.getConditionExpr());
-
-        mlir::Location exprLoc =
-            cgf.cgm.getLoc(clause.getConditionExpr()->getBeginLoc());
-        mlir::IntegerType targetType = mlir::IntegerType::get(
-            &cgf.getMLIRContext(), /*width=*/1,
-            mlir::IntegerType::SignednessSemantics::Signless);
-        auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>(
-            exprLoc, targetType, condition);
-        operation.getSelfCondMutable().append(conversionOp.getResult(0));
+        operation.getSelfCondMutable().append(
+            createCondition(clause.getConditionExpr()));
       } else {
         llvm_unreachable("var-list version of self shouldn't get here");
       }
@@ -150,6 +156,18 @@ class OpenACCClauseCIREmitter final
       return clauseNotImplemented(clause);
     }
   }
+
+  void VisitIfClause(const OpenACCIfClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, ParallelOp, SerialOp, KernelsOp>) {
+      operation.getIfCondMutable().append(
+          createCondition(clause.getConditionExpr()));
+    } else {
+      // 'if' applies to most of the constructs, but hold off on lowering them
+      // until we can write tests/know what we're doing with codegen to make
+      // sure we get it right.
+      return clauseNotImplemented(clause);
+    }
+  }
 };
 
 template <typename OpTy>
diff --git a/clang/test/CIR/CodeGenOpenACC/kernels.c b/clang/test/CIR/CodeGenOpenACC/kernels.c
index 934daf9e8ecc0..ca5bfebcb4ff3 100644
--- a/clang/test/CIR/CodeGenOpenACC/kernels.c
+++ b/clang/test/CIR/CodeGenOpenACC/kernels.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_kernels(int cond) {
   // CHECK: cir.func @acc_kernels(%[[ARG:.*]]: !s32i{{.*}}) {
@@ -63,6 +63,48 @@ void acc_kernels(int cond) {
   // CHECK-NEXT: acc.kernels self(%[[CONV_CAST]]) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
- 
+
+#pragma acc kernels if(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[COND_LOAD]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.kernels if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels if(1)
+  {}
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[ONE_LITERAL]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.kernels if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels if(cond == 1)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[EQ_RES:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[ONE_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES]] : !cir.bool to i1
+  // CHECK-NEXT: acc.kernels if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels if(cond == 1) self(cond == 2)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[EQ_RES_IF:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[ONE_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST_IF:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES_IF]] : !cir.bool to i1
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[EQ_RES_SELF:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[TWO_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST_SELF:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES_SELF]] : !cir.bool to i1
+  // CHECK-NEXT: acc.kernels self(%[[CONV_CAST_SELF]]) if(%[[CONV_CAST_IF]]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }
diff --git a/clang/test/CIR/CodeGenOpenACC/parallel.c b/clang/test/CIR/CodeGenOpenACC/parallel.c
index c7a4bda6faa74..3fb0b987409db 100644
--- a/clang/test/CIR/CodeGenOpenACC/parallel.c
+++ b/clang/test/CIR/CodeGenOpenACC/parallel.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_parallel(int cond) {
   // CHECK: cir.func @acc_parallel(%[[ARG:.*]]: !s32i{{.*}}) {
@@ -63,5 +63,47 @@ void acc_parallel(int cond) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
 
+#pragma acc parallel if(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[COND_LOAD]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.parallel if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel if(1)
+  {}
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[ONE_LITERAL]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.parallel if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel if(cond == 1)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[EQ_RES:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[ONE_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES]] : !cir.bool to i1
+  // CHECK-NEXT: acc.parallel if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel if(cond == 1) self(cond == 2)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[EQ_RES_IF:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[ONE_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST_IF:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES_IF]] : !cir.bool to i1
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[EQ_RES_SELF:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[TWO_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST_SELF:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES_SELF]] : !cir.bool to i1
+  // CHECK-NEXT: acc.parallel self(%[[CONV_CAST_SELF]]) if(%[[CONV_CAST_IF]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }
diff --git a/clang/test/CIR/CodeGenOpenACC/serial.c b/clang/test/CIR/CodeGenOpenACC/serial.c
index 38a38ad6c9514..b72f44a2ea473 100644
--- a/clang/test/CIR/CodeGenOpenACC/serial.c
+++ b/clang/test/CIR/CodeGenOpenACC/serial.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_serial(int cond) {
   // CHECK: cir.func @acc_serial(%[[ARG:.*]]: !s32i{{.*}}) {
@@ -64,5 +64,47 @@ void acc_serial(int cond) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
 
+#pragma acc serial if(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[COND_LOAD]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.serial if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial if(1)
+  {}
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[BOOL_CAST:.*]] = cir.cast(int_to_bool, %[[ONE_LITERAL]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[BOOL_CAST]] : !cir.bool to i1
+  // CHECK-NEXT: acc.serial if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial if(cond == 1)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[EQ_RES:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[ONE_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES]] : !cir.bool to i1
+  // CHECK-NEXT: acc.serial if(%[[CONV_CAST]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial if(cond == 1) self(cond == 2)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[ONE_LITERAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[EQ_RES_IF:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[ONE_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST_IF:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES_IF]] : !cir.bool to i1
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[EQ_RES_SELF:.*]] = cir.cmp(eq, %[[COND_LOAD]], %[[TWO_LITERAL]]) : !s32i, !cir.bool
+  // CHECK-NEXT: %[[CONV_CAST_SELF:.*]] = builtin.unrealized_conversion_cast %[[EQ_RES_SELF]] : !cir.bool to i1
+  // CHECK-NEXT: acc.serial self(%[[CONV_CAST_SELF]]) if(%[[CONV_CAST_IF]]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }

From 0b8f817aab7a242e0bfb519cb07c8979ffadef36 Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Tue, 15 Apr 2025 16:59:05 -0700
Subject: [PATCH 059/710] [BOLT] Fix conditional compilation of hugify.cpp
 (#135880)

Fix builds after #117158: do not build hugify.cpp on Apple platforms.
---
 bolt/runtime/hugify.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index 67d5fa26007d2..672b04247dfa4 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -6,8 +6,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#if defined(__x86_64__) ||                                                     \
-    (defined(__aarch64__) || defined(__arm64__)) && !defined(__APPLE__)
+#if (defined(__x86_64__) || defined(__aarch64__) || defined(__arm64__)) &&     \
+    !defined(__APPLE__)
 
 #include "common.h"
 

From bd4d3519c708d70ed8c827a27b63f13b0229ef00 Mon Sep 17 00:00:00 2001
From: Camsyn <camsyn@foxmail.com>
Date: Wed, 16 Apr 2025 08:15:57 +0800
Subject: [PATCH 060/710] [ASan] Prevent ASan/LSan deadlock by preloading
 modules before error reporting (#131756)

### Description
This PR resolves a deadlock between AddressSanitizer (ASan) and
LeakSanitizer (LSan)
that occurs when both sanitizers attempt to acquire locks in conflicting
orders across
threads. The fix ensures safe lock acquisition ordering by preloading
module information
before error reporting.

---

### Issue Details
**Reproducer**
```cpp
// Thread 1: ASan error path
int arr[1] = {0};
std::thread t([&]() {
  arr[1] = 1; // Triggers ASan OOB error
});

// Thread 2: LSan check path
__lsan_do_leak_check();
```

**Lock Order Conflict**:

- Thread 1 (ASan error reporting):
  1. Acquires ASan thread registry lock (B)
  1. Attempts to acquire libdl lock (A) via `dl_iterate_phdr`

- Thread 2 (LSan leak check):
  1. Acquires libdl lock (A) via `dl_iterate_phdr`
  1. Attempts to acquire ASan thread registry lock (B)


This creates a circular wait condition (A -> B -> A) meeting all four
Coffman deadlock criteria.

---

### Fix Strategy
The root cause lies in ASan's error reporting path needing
`dl_iterate_phdr` (requiring lock A)
while already holding its thread registry lock (B). The solution:

1. **Preload Modules Early**: Force module list initialization _before_
acquiring ASan's thread lock
2. **Avoid Nested Locking**: Ensure symbolization (via dl_iterate_phdr)
completes before error reporting locks

Key code change:
```cpp
// Before acquiring ASan's thread registry lock:
Symbolizer::GetOrInit()->GetRefreshedListOfModules();
```

This guarantees module information is cached before lock acquisition,
eliminating
the need for `dl_iterate_phdr` calls during error reporting.

---

### Testing
Added **asan_lsan_deadlock.cpp** test case:
- Reproduces deadlock reliably without fix **under idle system
conditions**
   - Uses watchdog thread to detect hangs
   - Verifies ASan error reports correctly without deadlock

**Note**: Due to the inherent non-determinism of thread scheduling and
lock acquisition timing,
this test may not reliably reproduce the deadlock on busy systems (e.g.,
during parallel
   `ninja check-asan` runs).



---

### Impact
- Fixes rare but severe deadlocks in mixed ASan+LSan environments
- Maintains thread safety guarantees for both sanitizers
- No user-visible behavior changes except deadlock elimination

---

### Relevant Buggy Code
- Code in ASan's asan_report.cpp
```cpp
  explicit ScopedInErrorReport(bool fatal = false)
      : halt_on_error_(fatal || flags()->halt_on_error) {
    // Acquire lock B
    asanThreadRegistry().Lock();
  }
  ~ScopedInErrorReport() {
    ...
    // Try to acquire lock A under holding lock B via the following path
    // #4  0x000071a353d83e93 in __GI___dl_iterate_phdr (
    //     callback=0x5d1a07a39580 <__sanitizer::dl_iterate_phdr_cb(dl_phdr_info*, unsigned long, void*)>,
    //     data=0x6da3510fd3f0) at ./elf/dl-iteratephdr.c:39
    // #5  0x00005d1a07a39574 in __sanitizer::ListOfModules::init (this=0x71a353ebc080)
    //     at llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp:784
    // #6  0x00005d1a07a429e3 in __sanitizer::Symbolizer::RefreshModules (this=0x71a353ebc058)
    //     at llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp:188
    // #7  __sanitizer::Symbolizer::FindModuleForAddress (this=this@entry=0x71a353ebc058,
    //     address=address@entry=102366378805727)
    //     at llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp:214
    // #8  0x00005d1a07a4291b in __sanitizer::Symbolizer::SymbolizePC (this=0x71a353ebc058, addr=102366378805727)
    //     at llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp:88
    // #9  0x00005d1a07a40df7 in __sanitizer::(anonymous namespace)::StackTraceTextPrinter::ProcessAddressFrames (
    //     this=this@entry=0x6da3510fd520, pc=102366378805727)
    //     at llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp:37
    // #10 0x00005d1a07a40d27 in __sanitizer::StackTrace::PrintTo (this=this@entry=0x6da3510fd5e8,
    //     output=output@entry=0x6da3510fd588)
    //     at llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp:110
    // #11 0x00005d1a07a410a1 in __sanitizer::StackTrace::Print (this=0x6da3510fd5e8)
    //     at llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp:133
    // #12 0x00005d1a0798758d in __asan::ErrorGeneric::Print (
    //     this=0x5d1a07aa4e08 <__asan::ScopedInErrorReport::current_error_+8>)
    //     at llvm-project/compiler-rt/lib/asan/asan_errors.cpp:617
    current_error_.Print();
    ...
  }
```

- Code in LSan's lsan_common_linux.cpp
```cpp
void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
                              CheckForLeaksParam *argument) {
  // Acquire lock A
  dl_iterate_phdr(LockStuffAndStopTheWorldCallback, &param);
}

static int LockStuffAndStopTheWorldCallback(struct dl_phdr_info *info,
                                            size_t size, void *data) {
  // Try to acquire lock B under holding lock A via the following path
  // #3  0x000055555562b34a in __sanitizer::ThreadRegistry::Lock (this=<optimized out>)
  //     at llvm-project/compiler-rt/lib/asan/../sanitizer_common/sanitizer_thread_registry.h:99
  // #4  __lsan::LockThreads () at llvm-project/compiler-rt/lib/asan/asan_thread.cpp:484
  // #5  0x0000555555652629 in __lsan::ScopedStopTheWorldLock::ScopedStopTheWorldLock (this=<optimized out>)
  //     at llvm-project/compiler-rt/lib/lsan/lsan_common.h:164
  // #6  __lsan::LockStuffAndStopTheWorldCallback (info=<optimized out>, size=<optimized out>, data=0x0,
  //     data@entry=0x7fffffffd158) at llvm-project/compiler-rt/lib/lsan/lsan_common_linux.cpp:120
  ScopedStopTheWorldLock lock;
  DoStopTheWorldParam *param = reinterpret_cast<DoStopTheWorldParam *>(data);
  StopTheWorld(param->callback, param->argument);
  return 1;
}
```
---
 compiler-rt/lib/asan/asan_report.cpp          | 25 +++++++
 .../asan/TestCases/asan_lsan_deadlock.cpp     | 72 +++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100644 compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp

diff --git a/compiler-rt/lib/asan/asan_report.cpp b/compiler-rt/lib/asan/asan_report.cpp
index 6302866805f37..e515f20548c00 100644
--- a/compiler-rt/lib/asan/asan_report.cpp
+++ b/compiler-rt/lib/asan/asan_report.cpp
@@ -126,6 +126,31 @@ class ScopedInErrorReport {
  public:
   explicit ScopedInErrorReport(bool fatal = false)
       : halt_on_error_(fatal || flags()->halt_on_error) {
+    // Deadlock Prevention Between ASan and LSan
+    //
+    // Background:
+    // - The `dl_iterate_phdr` function requires holding libdl's internal lock
+    //   (Lock A).
+    // - LSan acquires the ASan thread registry lock (Lock B) *after* calling
+    //   `dl_iterate_phdr`.
+    //
+    // Problem Scenario:
+    // When ASan attempts to call `dl_iterate_phdr` while holding Lock B (e.g.,
+    // during error reporting via `ErrorDescription::Print`), a circular lock
+    // dependency may occur:
+    //   1. Thread 1: Holds Lock B → Requests Lock A (via dl_iterate_phdr)
+    //   2. Thread 2: Holds Lock A → Requests Lock B (via LSan operations)
+    //
+    // Solution:
+    // Proactively load all required modules before acquiring Lock B.
+    // This ensures:
+    // 1. Any `dl_iterate_phdr` calls during module loading complete before
+    //    locking.
+    // 2. Subsequent error reporting avoids nested lock acquisition patterns.
+    // 3. Eliminates the lock order inversion risk between libdl and ASan's
+    //    thread registry.
+    Symbolizer::GetOrInit()->GetRefreshedListOfModules();
+
     // Make sure the registry and sanitizer report mutexes are locked while
     // we're printing an error report.
     // We can lock them only here to avoid self-deadlock in case of
diff --git a/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp b/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
new file mode 100644
index 0000000000000..4e1a2415ad013
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
@@ -0,0 +1,72 @@
+// Test for potential deadlock in LeakSanitizer+AddressSanitizer.
+// REQUIRES: leak-detection
+//
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=detect_leaks=1 not %run %t 2>&1 | FileCheck %s
+
+/*
+ * Purpose: Verify deadlock prevention between ASan error reporting and LSan leak checking.
+ * 
+ * Test Design:
+ * 1. Creates contention scenario between:
+ *    - ASan's error reporting (requires lock B -> lock A ordering)
+ *    - LSan's leak check (requires lock A -> lock B ordering)
+ * 2. Thread timing:
+ *    - Main thread: Holds 'in' mutex -> Triggers LSan check (lock A then B) 
+ *    - Worker thread: Triggers ASan OOB error (lock B then A via symbolization)
+ * 
+ * Deadlock Condition (if unfixed):
+ * Circular lock dependency forms when:
+ * [Main Thread] LSan: lock A -> requests lock B
+ * [Worker Thread] ASan: lock B -> requests lock A
+ * 
+ * Success Criteria: 
+ * With proper lock ordering enforcement, watchdog should NOT trigger - test exits normally.
+ * If deadlock occurs, watchdog terminates via _exit(1) after 10s timeout.
+ */
+
+#include <mutex>
+#include <sanitizer/lsan_interface.h>
+#include <stdio.h>
+#include <thread>
+#include <unistd.h>
+
+void Watchdog() {
+  // Safety mechanism: Turn infinite deadlock into finite test failure
+  usleep(10000000);
+  // CHECK-NOT: Timeout! Deadlock detected.
+  puts("Timeout! Deadlock detected.");
+  fflush(stdout);
+  _exit(1);
+}
+
+int main(int argc, char **argv) {
+  int arr[1] = {0};
+  std::mutex in;
+  in.lock();
+
+  std::thread w(Watchdog);
+  w.detach();
+
+  std::thread t([&]() {
+    in.unlock();
+    /* 
+     * Provoke ASan error: ASan's error reporting acquires: 
+     * 1. ASan's thread registry lock (B) during the reporting 
+     * 2. dl_iterate_phdr lock (A) during symbolization
+     */
+    // CHECK: SUMMARY: AddressSanitizer: stack-buffer-overflow
+    arr[argc] = 1; // Deliberate OOB access
+  });
+
+  in.lock();
+  /* 
+   * Critical section: LSan's check acquires: 
+   * 1. dl_iterate_phdr lock (A)
+   * 2. ASan's thread registry lock (B)
+   * before Stop The World.
+   */
+  __lsan_do_leak_check();
+  t.join();
+  return 0;
+}

From 77f0708b9d4feee8b8a67a5f571be741be4e26af Mon Sep 17 00:00:00 2001
From: Vinay Deshmukh <32487576+vinay-deshmukh@users.noreply.github.com>
Date: Tue, 15 Apr 2025 20:24:07 -0400
Subject: [PATCH 061/710] [libc]: Remove `-Wglobal-constructors` for libc tests
 (#131485)

* Relates to: https://github.com/llvm/llvm-project/issues/119281
* Removes `-Wglobal-constructors` as per:
https://github.com/llvm/llvm-project/pull/131485#pullrequestreview-2728020622
---
 libc/cmake/modules/LLVMLibCTestRules.cmake | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index 63a8e9ecda002..a28e15fc5e394 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -57,7 +57,6 @@ function(_get_common_test_compile_options output_var c_test flags)
       list(APPEND compile_options "-Wnewline-eof")
       list(APPEND compile_options "-Wnonportable-system-include-path")
       list(APPEND compile_options "-Wthread-safety")
-      # list(APPEND compile_options "-Wglobal-constructors")
     endif()
   endif()
   set(${output_var} ${compile_options} PARENT_SCOPE)

From 6c6ab2a270b799f1397926c9064fa30fe2be1d96 Mon Sep 17 00:00:00 2001
From: YunQiang Su <syq@debian.org>
Date: Wed, 16 Apr 2025 09:09:13 +0800
Subject: [PATCH 062/710] AArch64: Set FMAXIMUMNUM and FMINIMUMNUM as Promote
 if not fullfp16 (#135708)

Since Promote will emit FP_EXTEND, the result of it will never be sNaN,
so we don't need worry about duplicated of FCANONICALIZE in
expandFMINIMUMNUM_FMAXIMUMNUM.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |    2 +
 .../AArch64/fp-maximumnum-minimumnum.ll       | 1751 ++++++++++++++---
 2 files changed, 1480 insertions(+), 273 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 830ec6886e6bc..bea8087750d6e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -795,6 +795,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
              ISD::FMAXNUM,
              ISD::FMINIMUM,
              ISD::FMAXIMUM,
+             ISD::FMINIMUMNUM,
+             ISD::FMAXIMUMNUM,
              ISD::FCANONICALIZE,
              ISD::STRICT_FADD,
              ISD::STRICT_FSUB,
diff --git a/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll b/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
index bb3f9a3e52a16..c6b8e41f9bdfd 100644
--- a/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
+++ b/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=aarch64 --mattr=+fullfp16 < %s | FileCheck %s --check-prefix=AARCH64
+; RUN: llc --mtriple=aarch64 --mattr=+fullfp16 < %s | FileCheck %s --check-prefixes=AARCH64,FULLFP16
+; RUN: llc --mtriple=aarch64 < %s | FileCheck %s --check-prefixes=AARCH64,NOFULLFP16
 
 ;;;;;;;;;;;;;;;;  max_f64
 define double @max_nnan_f64(double %a, double %b) {
@@ -142,96 +143,397 @@ entry:
 
 ;;;;;;;;;;;;;;;;;; max_f16
 define half @max_nnan_f16(half %a, half %b) {
-; AARCH64-LABEL: max_nnan_f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fmaxnm h0, h0, h1
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_nnan_f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fmaxnm h0, h0, h1
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_nnan_f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    fmaxnm s0, s0, s1
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan half @llvm.maximumnum.f16(half %a, half %b)
   ret half %c
 }
 
 define <2 x half> @max_nnan_v2f16(<2 x half> %a, <2 x half> %b) {
-; AARCH64-LABEL: max_nnan_v2f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_nnan_v2f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_nnan_v2f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    mov h4, v1.h[2]
+; NOFULLFP16-NEXT:    mov h5, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h1
+; NOFULLFP16-NEXT:    fcvt s7, h0
+; NOFULLFP16-NEXT:    mov h1, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fmaxnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h4
+; NOFULLFP16-NEXT:    fcvt s4, h5
+; NOFULLFP16-NEXT:    fmaxnm s5, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v0.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s3, s4, s3
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fcvt h0, s5
+; NOFULLFP16-NEXT:    fcvt s4, h6
+; NOFULLFP16-NEXT:    mov v0.h[1], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    fmaxnm s1, s4, s1
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v0.h[3], v1.h[0]
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <2 x half> @llvm.maximumnum.v2f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %c
 }
 
 define <4 x half> @max_nnan_v4f16(<4 x half> %a, <4 x half> %b) {
-; AARCH64-LABEL: max_nnan_v4f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_nnan_v4f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_nnan_v4f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    mov h4, v1.h[2]
+; NOFULLFP16-NEXT:    mov h5, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h1
+; NOFULLFP16-NEXT:    fcvt s7, h0
+; NOFULLFP16-NEXT:    mov h1, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fmaxnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h4
+; NOFULLFP16-NEXT:    fcvt s4, h5
+; NOFULLFP16-NEXT:    fmaxnm s5, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v0.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s3, s4, s3
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fcvt h0, s5
+; NOFULLFP16-NEXT:    fcvt s4, h6
+; NOFULLFP16-NEXT:    mov v0.h[1], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    fmaxnm s1, s4, s1
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v0.h[3], v1.h[0]
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <4 x half> @llvm.maximumnum.v4f16(<4 x half> %a, <4 x half> %b)
   ret <4 x half> %c
 }
 
 define <8 x half> @max_nnan_v8f16(<8 x half> %a, <8 x half> %b) {
-; AARCH64-LABEL: max_nnan_v8f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_nnan_v8f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_nnan_v8f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    fcvt s4, h1
+; NOFULLFP16-NEXT:    fcvt s5, h0
+; NOFULLFP16-NEXT:    mov h6, v1.h[2]
+; NOFULLFP16-NEXT:    mov h7, v0.h[2]
+; NOFULLFP16-NEXT:    mov h16, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fmaxnm s4, s5, s4
+; NOFULLFP16-NEXT:    mov h5, v0.h[3]
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fmaxnm s3, s3, s2
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    fcvt h2, s4
+; NOFULLFP16-NEXT:    fmaxnm s4, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v1.h[4]
+; NOFULLFP16-NEXT:    mov h7, v0.h[4]
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fmaxnm s5, s5, s16
+; NOFULLFP16-NEXT:    mov h16, v0.h[5]
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    mov v2.h[1], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt s3, h6
+; NOFULLFP16-NEXT:    fcvt s6, h7
+; NOFULLFP16-NEXT:    mov h7, v1.h[5]
+; NOFULLFP16-NEXT:    fcvt h5, s5
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    mov v2.h[2], v4.h[0]
+; NOFULLFP16-NEXT:    mov h4, v1.h[6]
+; NOFULLFP16-NEXT:    fmaxnm s3, s6, s3
+; NOFULLFP16-NEXT:    mov h6, v0.h[6]
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    mov h1, v1.h[7]
+; NOFULLFP16-NEXT:    mov h0, v0.h[7]
+; NOFULLFP16-NEXT:    mov v2.h[3], v5.h[0]
+; NOFULLFP16-NEXT:    fcvt s4, h4
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fcvt s5, h6
+; NOFULLFP16-NEXT:    fmaxnm s6, s16, s7
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    mov v2.h[4], v3.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s4, s5, s4
+; NOFULLFP16-NEXT:    fcvt h3, s6
+; NOFULLFP16-NEXT:    fmaxnm s0, s0, s1
+; NOFULLFP16-NEXT:    mov v2.h[5], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt h3, s4
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    mov v2.h[6], v3.h[0]
+; NOFULLFP16-NEXT:    mov v2.h[7], v0.h[0]
+; NOFULLFP16-NEXT:    mov v0.16b, v2.16b
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <8 x half> @llvm.maximumnum.v8f16(<8 x half> %a, <8 x half> %b)
   ret <8 x half> %c
 }
 
 define <9 x half> @max_nnan_v9f16(<9 x half> %a, <9 x half> %b) {
-; AARCH64-LABEL: max_nnan_v9f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    // kill: def $h0 killed $h0 def $q0
-; AARCH64-NEXT:    // kill: def $h1 killed $h1 def $q1
-; AARCH64-NEXT:    // kill: def $h2 killed $h2 def $q2
-; AARCH64-NEXT:    add x9, sp, #16
-; AARCH64-NEXT:    // kill: def $h3 killed $h3 def $q3
-; AARCH64-NEXT:    // kill: def $h4 killed $h4 def $q4
-; AARCH64-NEXT:    // kill: def $h5 killed $h5 def $q5
-; AARCH64-NEXT:    // kill: def $h6 killed $h6 def $q6
-; AARCH64-NEXT:    // kill: def $h7 killed $h7 def $q7
-; AARCH64-NEXT:    mov v0.h[1], v1.h[0]
-; AARCH64-NEXT:    ldr h1, [sp, #8]
-; AARCH64-NEXT:    ld1 { v1.h }[1], [x9]
-; AARCH64-NEXT:    add x9, sp, #24
-; AARCH64-NEXT:    mov v0.h[2], v2.h[0]
-; AARCH64-NEXT:    ldr h2, [sp, #72]
-; AARCH64-NEXT:    ld1 { v1.h }[2], [x9]
-; AARCH64-NEXT:    add x9, sp, #32
-; AARCH64-NEXT:    mov v0.h[3], v3.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[3], [x9]
-; AARCH64-NEXT:    add x9, sp, #40
-; AARCH64-NEXT:    ldr h3, [sp]
-; AARCH64-NEXT:    ld1 { v1.h }[4], [x9]
-; AARCH64-NEXT:    add x9, sp, #48
-; AARCH64-NEXT:    fmaxnm v2.8h, v3.8h, v2.8h
-; AARCH64-NEXT:    mov v0.h[4], v4.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[5], [x9]
-; AARCH64-NEXT:    add x9, sp, #56
-; AARCH64-NEXT:    str h2, [x8, #16]
-; AARCH64-NEXT:    mov v0.h[5], v5.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[6], [x9]
-; AARCH64-NEXT:    add x9, sp, #64
-; AARCH64-NEXT:    mov v0.h[6], v6.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[7], [x9]
-; AARCH64-NEXT:    mov v0.h[7], v7.h[0]
-; AARCH64-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
-; AARCH64-NEXT:    str q0, [x8]
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_nnan_v9f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    // kill: def $h0 killed $h0 def $q0
+; FULLFP16-NEXT:    // kill: def $h1 killed $h1 def $q1
+; FULLFP16-NEXT:    // kill: def $h2 killed $h2 def $q2
+; FULLFP16-NEXT:    add x9, sp, #16
+; FULLFP16-NEXT:    // kill: def $h3 killed $h3 def $q3
+; FULLFP16-NEXT:    // kill: def $h4 killed $h4 def $q4
+; FULLFP16-NEXT:    // kill: def $h5 killed $h5 def $q5
+; FULLFP16-NEXT:    // kill: def $h6 killed $h6 def $q6
+; FULLFP16-NEXT:    // kill: def $h7 killed $h7 def $q7
+; FULLFP16-NEXT:    mov v0.h[1], v1.h[0]
+; FULLFP16-NEXT:    ldr h1, [sp, #8]
+; FULLFP16-NEXT:    ld1 { v1.h }[1], [x9]
+; FULLFP16-NEXT:    add x9, sp, #24
+; FULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; FULLFP16-NEXT:    ldr h2, [sp, #72]
+; FULLFP16-NEXT:    ld1 { v1.h }[2], [x9]
+; FULLFP16-NEXT:    add x9, sp, #32
+; FULLFP16-NEXT:    mov v0.h[3], v3.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[3], [x9]
+; FULLFP16-NEXT:    add x9, sp, #40
+; FULLFP16-NEXT:    ldr h3, [sp]
+; FULLFP16-NEXT:    ld1 { v1.h }[4], [x9]
+; FULLFP16-NEXT:    add x9, sp, #48
+; FULLFP16-NEXT:    fmaxnm v2.8h, v3.8h, v2.8h
+; FULLFP16-NEXT:    mov v0.h[4], v4.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[5], [x9]
+; FULLFP16-NEXT:    add x9, sp, #56
+; FULLFP16-NEXT:    str h2, [x8, #16]
+; FULLFP16-NEXT:    mov v0.h[5], v5.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[6], [x9]
+; FULLFP16-NEXT:    add x9, sp, #64
+; FULLFP16-NEXT:    mov v0.h[6], v6.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[7], [x9]
+; FULLFP16-NEXT:    mov v0.h[7], v7.h[0]
+; FULLFP16-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
+; FULLFP16-NEXT:    str q0, [x8]
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_nnan_v9f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    ldr h16, [sp, #16]
+; NOFULLFP16-NEXT:    ldr h17, [sp, #8]
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    ldr h18, [sp, #24]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fcvt s17, h17
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s18, h18
+; NOFULLFP16-NEXT:    fcvt s4, h4
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    fmaxnm s1, s1, s16
+; NOFULLFP16-NEXT:    fmaxnm s0, s0, s17
+; NOFULLFP16-NEXT:    ldr h16, [sp, #32]
+; NOFULLFP16-NEXT:    fmaxnm s2, s2, s18
+; NOFULLFP16-NEXT:    ldr h17, [sp, #40]
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fcvt s17, h17
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fmaxnm s3, s3, s16
+; NOFULLFP16-NEXT:    fmaxnm s4, s4, s17
+; NOFULLFP16-NEXT:    mov v0.h[1], v1.h[0]
+; NOFULLFP16-NEXT:    ldr h1, [sp, #48]
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    ldr h2, [sp, #56]
+; NOFULLFP16-NEXT:    fmaxnm s1, s5, s1
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    ldr h5, [sp, #64]
+; NOFULLFP16-NEXT:    mov v0.h[3], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt s3, h6
+; NOFULLFP16-NEXT:    ldr h6, [sp, #72]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    mov v0.h[4], v4.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h5
+; NOFULLFP16-NEXT:    fcvt s4, h7
+; NOFULLFP16-NEXT:    ldr h5, [sp]
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    mov v0.h[5], v1.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s2
+; NOFULLFP16-NEXT:    fmaxnm s2, s4, s3
+; NOFULLFP16-NEXT:    fmaxnm s3, s5, s6
+; NOFULLFP16-NEXT:    mov v0.h[6], v1.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s2
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    mov v0.h[7], v1.h[0]
+; NOFULLFP16-NEXT:    str h2, [x8, #16]
+; NOFULLFP16-NEXT:    str q0, [x8]
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <9 x half> @llvm.maximumnum.v9f16(<9 x half> %a, <9 x half> %b)
   ret <9 x half> %c
 }
 
 define <16 x half> @max_nnan_v16f16(<16 x half> %a, <16 x half> %b) {
-; AARCH64-LABEL: max_nnan_v16f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fmaxnm v1.8h, v1.8h, v3.8h
-; AARCH64-NEXT:    fmaxnm v0.8h, v0.8h, v2.8h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_nnan_v16f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fmaxnm v1.8h, v1.8h, v3.8h
+; FULLFP16-NEXT:    fmaxnm v0.8h, v0.8h, v2.8h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_nnan_v16f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    mov h6, v2.h[1]
+; NOFULLFP16-NEXT:    mov h7, v0.h[1]
+; NOFULLFP16-NEXT:    fcvt s4, h2
+; NOFULLFP16-NEXT:    fcvt s5, h0
+; NOFULLFP16-NEXT:    mov h16, v3.h[1]
+; NOFULLFP16-NEXT:    mov h17, v1.h[1]
+; NOFULLFP16-NEXT:    mov h18, v2.h[2]
+; NOFULLFP16-NEXT:    mov h19, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s20, h3
+; NOFULLFP16-NEXT:    fcvt s21, h1
+; NOFULLFP16-NEXT:    mov h22, v3.h[2]
+; NOFULLFP16-NEXT:    mov h23, v1.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    mov h24, v0.h[6]
+; NOFULLFP16-NEXT:    fmaxnm s4, s5, s4
+; NOFULLFP16-NEXT:    fcvt s5, h16
+; NOFULLFP16-NEXT:    fcvt s16, h17
+; NOFULLFP16-NEXT:    fcvt s17, h18
+; NOFULLFP16-NEXT:    fcvt s18, h19
+; NOFULLFP16-NEXT:    mov h19, v0.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s20, s21, s20
+; NOFULLFP16-NEXT:    fcvt s21, h22
+; NOFULLFP16-NEXT:    mov h22, v3.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s6, s7, s6
+; NOFULLFP16-NEXT:    mov h7, v2.h[3]
+; NOFULLFP16-NEXT:    mov h25, v1.h[6]
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    fmaxnm s5, s16, s5
+; NOFULLFP16-NEXT:    fcvt s16, h23
+; NOFULLFP16-NEXT:    mov h23, v1.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s17, s18, s17
+; NOFULLFP16-NEXT:    fcvt s18, h19
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    fcvt h19, s5
+; NOFULLFP16-NEXT:    fcvt h5, s20
+; NOFULLFP16-NEXT:    fmaxnm s16, s16, s21
+; NOFULLFP16-NEXT:    fcvt s20, h23
+; NOFULLFP16-NEXT:    fcvt h17, s17
+; NOFULLFP16-NEXT:    mov h21, v2.h[4]
+; NOFULLFP16-NEXT:    mov h23, v1.h[4]
+; NOFULLFP16-NEXT:    mov v4.h[1], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt s6, h22
+; NOFULLFP16-NEXT:    mov h22, v0.h[4]
+; NOFULLFP16-NEXT:    fmaxnm s7, s18, s7
+; NOFULLFP16-NEXT:    mov h18, v3.h[4]
+; NOFULLFP16-NEXT:    mov v5.h[1], v19.h[0]
+; NOFULLFP16-NEXT:    fcvt h16, s16
+; NOFULLFP16-NEXT:    fmaxnm s6, s20, s6
+; NOFULLFP16-NEXT:    mov v4.h[2], v17.h[0]
+; NOFULLFP16-NEXT:    fcvt s17, h21
+; NOFULLFP16-NEXT:    fcvt s19, h22
+; NOFULLFP16-NEXT:    fcvt h7, s7
+; NOFULLFP16-NEXT:    fcvt s18, h18
+; NOFULLFP16-NEXT:    fcvt s20, h23
+; NOFULLFP16-NEXT:    mov h21, v2.h[5]
+; NOFULLFP16-NEXT:    mov h22, v0.h[5]
+; NOFULLFP16-NEXT:    mov v5.h[2], v16.h[0]
+; NOFULLFP16-NEXT:    mov h16, v3.h[5]
+; NOFULLFP16-NEXT:    mov h23, v1.h[5]
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    mov h0, v0.h[7]
+; NOFULLFP16-NEXT:    mov h1, v1.h[7]
+; NOFULLFP16-NEXT:    fmaxnm s17, s19, s17
+; NOFULLFP16-NEXT:    mov h19, v2.h[6]
+; NOFULLFP16-NEXT:    mov v4.h[3], v7.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s18, s20, s18
+; NOFULLFP16-NEXT:    mov h20, v3.h[6]
+; NOFULLFP16-NEXT:    fcvt s7, h21
+; NOFULLFP16-NEXT:    fcvt s21, h22
+; NOFULLFP16-NEXT:    fcvt s22, h24
+; NOFULLFP16-NEXT:    mov h2, v2.h[7]
+; NOFULLFP16-NEXT:    mov v5.h[3], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt s6, h16
+; NOFULLFP16-NEXT:    fcvt s16, h23
+; NOFULLFP16-NEXT:    fcvt h17, s17
+; NOFULLFP16-NEXT:    fcvt s19, h19
+; NOFULLFP16-NEXT:    fcvt s23, h25
+; NOFULLFP16-NEXT:    fcvt h18, s18
+; NOFULLFP16-NEXT:    fcvt s20, h20
+; NOFULLFP16-NEXT:    mov h3, v3.h[7]
+; NOFULLFP16-NEXT:    fmaxnm s7, s21, s7
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    fmaxnm s6, s16, s6
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    mov v4.h[4], v17.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s16, s22, s19
+; NOFULLFP16-NEXT:    mov v5.h[4], v18.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s17, s23, s20
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt h7, s7
+; NOFULLFP16-NEXT:    fmaxnm s0, s0, s2
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    fcvt h2, s16
+; NOFULLFP16-NEXT:    fmaxnm s1, s1, s3
+; NOFULLFP16-NEXT:    mov v4.h[5], v7.h[0]
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    mov v5.h[5], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt h6, s17
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v4.h[6], v2.h[0]
+; NOFULLFP16-NEXT:    mov v5.h[6], v6.h[0]
+; NOFULLFP16-NEXT:    mov v4.h[7], v0.h[0]
+; NOFULLFP16-NEXT:    mov v5.h[7], v1.h[0]
+; NOFULLFP16-NEXT:    mov v0.16b, v4.16b
+; NOFULLFP16-NEXT:    mov v1.16b, v5.16b
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <16 x half> @llvm.maximumnum.v16f16(<16 x half> %a, <16 x half> %b)
   ret <16 x half> %c
@@ -378,96 +680,397 @@ entry:
 
 ;;;;;;;;;;;;;;;;;; min_f16
 define half @min_nnan_f16(half %a, half %b) {
-; AARCH64-LABEL: min_nnan_f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm h0, h0, h1
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_nnan_f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm h0, h0, h1
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_nnan_f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    fminnm s0, s0, s1
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan half @llvm.minimumnum.f16(half %a, half %b)
   ret half %c
 }
 
 define <2 x half> @min_nnan_v2f16(<2 x half> %a, <2 x half> %b) {
-; AARCH64-LABEL: min_nnan_v2f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v0.4h, v0.4h, v1.4h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_nnan_v2f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v0.4h, v0.4h, v1.4h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_nnan_v2f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    mov h4, v1.h[2]
+; NOFULLFP16-NEXT:    mov h5, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h1
+; NOFULLFP16-NEXT:    fcvt s7, h0
+; NOFULLFP16-NEXT:    mov h1, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fminnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h4
+; NOFULLFP16-NEXT:    fcvt s4, h5
+; NOFULLFP16-NEXT:    fminnm s5, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v0.h[3]
+; NOFULLFP16-NEXT:    fminnm s3, s4, s3
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fcvt h0, s5
+; NOFULLFP16-NEXT:    fcvt s4, h6
+; NOFULLFP16-NEXT:    mov v0.h[1], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    fminnm s1, s4, s1
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v0.h[3], v1.h[0]
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <2 x half> @llvm.minimumnum.v2f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %c
 }
 
 define <4 x half> @min_nnan_v4f16(<4 x half> %a, <4 x half> %b) {
-; AARCH64-LABEL: min_nnan_v4f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v0.4h, v0.4h, v1.4h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_nnan_v4f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v0.4h, v0.4h, v1.4h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_nnan_v4f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    mov h4, v1.h[2]
+; NOFULLFP16-NEXT:    mov h5, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h1
+; NOFULLFP16-NEXT:    fcvt s7, h0
+; NOFULLFP16-NEXT:    mov h1, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fminnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h4
+; NOFULLFP16-NEXT:    fcvt s4, h5
+; NOFULLFP16-NEXT:    fminnm s5, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v0.h[3]
+; NOFULLFP16-NEXT:    fminnm s3, s4, s3
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fcvt h0, s5
+; NOFULLFP16-NEXT:    fcvt s4, h6
+; NOFULLFP16-NEXT:    mov v0.h[1], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    fminnm s1, s4, s1
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v0.h[3], v1.h[0]
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <4 x half> @llvm.minimumnum.v4f16(<4 x half> %a, <4 x half> %b)
   ret <4 x half> %c
 }
 
 define <8 x half> @min_nnan_v8f16(<8 x half> %a, <8 x half> %b) {
-; AARCH64-LABEL: min_nnan_v8f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v1.8h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_nnan_v8f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v1.8h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_nnan_v8f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    fcvt s4, h1
+; NOFULLFP16-NEXT:    fcvt s5, h0
+; NOFULLFP16-NEXT:    mov h6, v1.h[2]
+; NOFULLFP16-NEXT:    mov h7, v0.h[2]
+; NOFULLFP16-NEXT:    mov h16, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fminnm s4, s5, s4
+; NOFULLFP16-NEXT:    mov h5, v0.h[3]
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fminnm s3, s3, s2
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    fcvt h2, s4
+; NOFULLFP16-NEXT:    fminnm s4, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v1.h[4]
+; NOFULLFP16-NEXT:    mov h7, v0.h[4]
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fminnm s5, s5, s16
+; NOFULLFP16-NEXT:    mov h16, v0.h[5]
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    mov v2.h[1], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt s3, h6
+; NOFULLFP16-NEXT:    fcvt s6, h7
+; NOFULLFP16-NEXT:    mov h7, v1.h[5]
+; NOFULLFP16-NEXT:    fcvt h5, s5
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    mov v2.h[2], v4.h[0]
+; NOFULLFP16-NEXT:    mov h4, v1.h[6]
+; NOFULLFP16-NEXT:    fminnm s3, s6, s3
+; NOFULLFP16-NEXT:    mov h6, v0.h[6]
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    mov h1, v1.h[7]
+; NOFULLFP16-NEXT:    mov h0, v0.h[7]
+; NOFULLFP16-NEXT:    mov v2.h[3], v5.h[0]
+; NOFULLFP16-NEXT:    fcvt s4, h4
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fcvt s5, h6
+; NOFULLFP16-NEXT:    fminnm s6, s16, s7
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    mov v2.h[4], v3.h[0]
+; NOFULLFP16-NEXT:    fminnm s4, s5, s4
+; NOFULLFP16-NEXT:    fcvt h3, s6
+; NOFULLFP16-NEXT:    fminnm s0, s0, s1
+; NOFULLFP16-NEXT:    mov v2.h[5], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt h3, s4
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    mov v2.h[6], v3.h[0]
+; NOFULLFP16-NEXT:    mov v2.h[7], v0.h[0]
+; NOFULLFP16-NEXT:    mov v0.16b, v2.16b
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <8 x half> @llvm.minimumnum.v8f16(<8 x half> %a, <8 x half> %b)
   ret <8 x half> %c
 }
 
 define <9 x half> @min_nnan_v9f16(<9 x half> %a, <9 x half> %b) {
-; AARCH64-LABEL: min_nnan_v9f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    // kill: def $h0 killed $h0 def $q0
-; AARCH64-NEXT:    // kill: def $h1 killed $h1 def $q1
-; AARCH64-NEXT:    // kill: def $h2 killed $h2 def $q2
-; AARCH64-NEXT:    add x9, sp, #16
-; AARCH64-NEXT:    // kill: def $h3 killed $h3 def $q3
-; AARCH64-NEXT:    // kill: def $h4 killed $h4 def $q4
-; AARCH64-NEXT:    // kill: def $h5 killed $h5 def $q5
-; AARCH64-NEXT:    // kill: def $h6 killed $h6 def $q6
-; AARCH64-NEXT:    // kill: def $h7 killed $h7 def $q7
-; AARCH64-NEXT:    mov v0.h[1], v1.h[0]
-; AARCH64-NEXT:    ldr h1, [sp, #8]
-; AARCH64-NEXT:    ld1 { v1.h }[1], [x9]
-; AARCH64-NEXT:    add x9, sp, #24
-; AARCH64-NEXT:    mov v0.h[2], v2.h[0]
-; AARCH64-NEXT:    ldr h2, [sp, #72]
-; AARCH64-NEXT:    ld1 { v1.h }[2], [x9]
-; AARCH64-NEXT:    add x9, sp, #32
-; AARCH64-NEXT:    mov v0.h[3], v3.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[3], [x9]
-; AARCH64-NEXT:    add x9, sp, #40
-; AARCH64-NEXT:    ldr h3, [sp]
-; AARCH64-NEXT:    ld1 { v1.h }[4], [x9]
-; AARCH64-NEXT:    add x9, sp, #48
-; AARCH64-NEXT:    fminnm v2.8h, v3.8h, v2.8h
-; AARCH64-NEXT:    mov v0.h[4], v4.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[5], [x9]
-; AARCH64-NEXT:    add x9, sp, #56
-; AARCH64-NEXT:    str h2, [x8, #16]
-; AARCH64-NEXT:    mov v0.h[5], v5.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[6], [x9]
-; AARCH64-NEXT:    add x9, sp, #64
-; AARCH64-NEXT:    mov v0.h[6], v6.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[7], [x9]
-; AARCH64-NEXT:    mov v0.h[7], v7.h[0]
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v1.8h
-; AARCH64-NEXT:    str q0, [x8]
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_nnan_v9f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    // kill: def $h0 killed $h0 def $q0
+; FULLFP16-NEXT:    // kill: def $h1 killed $h1 def $q1
+; FULLFP16-NEXT:    // kill: def $h2 killed $h2 def $q2
+; FULLFP16-NEXT:    add x9, sp, #16
+; FULLFP16-NEXT:    // kill: def $h3 killed $h3 def $q3
+; FULLFP16-NEXT:    // kill: def $h4 killed $h4 def $q4
+; FULLFP16-NEXT:    // kill: def $h5 killed $h5 def $q5
+; FULLFP16-NEXT:    // kill: def $h6 killed $h6 def $q6
+; FULLFP16-NEXT:    // kill: def $h7 killed $h7 def $q7
+; FULLFP16-NEXT:    mov v0.h[1], v1.h[0]
+; FULLFP16-NEXT:    ldr h1, [sp, #8]
+; FULLFP16-NEXT:    ld1 { v1.h }[1], [x9]
+; FULLFP16-NEXT:    add x9, sp, #24
+; FULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; FULLFP16-NEXT:    ldr h2, [sp, #72]
+; FULLFP16-NEXT:    ld1 { v1.h }[2], [x9]
+; FULLFP16-NEXT:    add x9, sp, #32
+; FULLFP16-NEXT:    mov v0.h[3], v3.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[3], [x9]
+; FULLFP16-NEXT:    add x9, sp, #40
+; FULLFP16-NEXT:    ldr h3, [sp]
+; FULLFP16-NEXT:    ld1 { v1.h }[4], [x9]
+; FULLFP16-NEXT:    add x9, sp, #48
+; FULLFP16-NEXT:    fminnm v2.8h, v3.8h, v2.8h
+; FULLFP16-NEXT:    mov v0.h[4], v4.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[5], [x9]
+; FULLFP16-NEXT:    add x9, sp, #56
+; FULLFP16-NEXT:    str h2, [x8, #16]
+; FULLFP16-NEXT:    mov v0.h[5], v5.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[6], [x9]
+; FULLFP16-NEXT:    add x9, sp, #64
+; FULLFP16-NEXT:    mov v0.h[6], v6.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[7], [x9]
+; FULLFP16-NEXT:    mov v0.h[7], v7.h[0]
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v1.8h
+; FULLFP16-NEXT:    str q0, [x8]
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_nnan_v9f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    ldr h16, [sp, #16]
+; NOFULLFP16-NEXT:    ldr h17, [sp, #8]
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    ldr h18, [sp, #24]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fcvt s17, h17
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s18, h18
+; NOFULLFP16-NEXT:    fcvt s4, h4
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    fminnm s1, s1, s16
+; NOFULLFP16-NEXT:    fminnm s0, s0, s17
+; NOFULLFP16-NEXT:    ldr h16, [sp, #32]
+; NOFULLFP16-NEXT:    fminnm s2, s2, s18
+; NOFULLFP16-NEXT:    ldr h17, [sp, #40]
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fcvt s17, h17
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fminnm s3, s3, s16
+; NOFULLFP16-NEXT:    fminnm s4, s4, s17
+; NOFULLFP16-NEXT:    mov v0.h[1], v1.h[0]
+; NOFULLFP16-NEXT:    ldr h1, [sp, #48]
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    ldr h2, [sp, #56]
+; NOFULLFP16-NEXT:    fminnm s1, s5, s1
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    ldr h5, [sp, #64]
+; NOFULLFP16-NEXT:    mov v0.h[3], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt s3, h6
+; NOFULLFP16-NEXT:    ldr h6, [sp, #72]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    mov v0.h[4], v4.h[0]
+; NOFULLFP16-NEXT:    fminnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h5
+; NOFULLFP16-NEXT:    fcvt s4, h7
+; NOFULLFP16-NEXT:    ldr h5, [sp]
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    mov v0.h[5], v1.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s2
+; NOFULLFP16-NEXT:    fminnm s2, s4, s3
+; NOFULLFP16-NEXT:    fminnm s3, s5, s6
+; NOFULLFP16-NEXT:    mov v0.h[6], v1.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s2
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    mov v0.h[7], v1.h[0]
+; NOFULLFP16-NEXT:    str h2, [x8, #16]
+; NOFULLFP16-NEXT:    str q0, [x8]
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <9 x half> @llvm.minimumnum.v9f16(<9 x half> %a, <9 x half> %b)
   ret <9 x half> %c
 }
 
 define <16 x half> @min_nnan_v16f16(<16 x half> %a, <16 x half> %b) {
-; AARCH64-LABEL: min_nnan_v16f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v1.8h, v1.8h, v3.8h
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v2.8h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_nnan_v16f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v3.8h
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v2.8h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_nnan_v16f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    mov h6, v2.h[1]
+; NOFULLFP16-NEXT:    mov h7, v0.h[1]
+; NOFULLFP16-NEXT:    fcvt s4, h2
+; NOFULLFP16-NEXT:    fcvt s5, h0
+; NOFULLFP16-NEXT:    mov h16, v3.h[1]
+; NOFULLFP16-NEXT:    mov h17, v1.h[1]
+; NOFULLFP16-NEXT:    mov h18, v2.h[2]
+; NOFULLFP16-NEXT:    mov h19, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s20, h3
+; NOFULLFP16-NEXT:    fcvt s21, h1
+; NOFULLFP16-NEXT:    mov h22, v3.h[2]
+; NOFULLFP16-NEXT:    mov h23, v1.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    mov h24, v0.h[6]
+; NOFULLFP16-NEXT:    fminnm s4, s5, s4
+; NOFULLFP16-NEXT:    fcvt s5, h16
+; NOFULLFP16-NEXT:    fcvt s16, h17
+; NOFULLFP16-NEXT:    fcvt s17, h18
+; NOFULLFP16-NEXT:    fcvt s18, h19
+; NOFULLFP16-NEXT:    mov h19, v0.h[3]
+; NOFULLFP16-NEXT:    fminnm s20, s21, s20
+; NOFULLFP16-NEXT:    fcvt s21, h22
+; NOFULLFP16-NEXT:    mov h22, v3.h[3]
+; NOFULLFP16-NEXT:    fminnm s6, s7, s6
+; NOFULLFP16-NEXT:    mov h7, v2.h[3]
+; NOFULLFP16-NEXT:    mov h25, v1.h[6]
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    fminnm s5, s16, s5
+; NOFULLFP16-NEXT:    fcvt s16, h23
+; NOFULLFP16-NEXT:    mov h23, v1.h[3]
+; NOFULLFP16-NEXT:    fminnm s17, s18, s17
+; NOFULLFP16-NEXT:    fcvt s18, h19
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    fcvt h19, s5
+; NOFULLFP16-NEXT:    fcvt h5, s20
+; NOFULLFP16-NEXT:    fminnm s16, s16, s21
+; NOFULLFP16-NEXT:    fcvt s20, h23
+; NOFULLFP16-NEXT:    fcvt h17, s17
+; NOFULLFP16-NEXT:    mov h21, v2.h[4]
+; NOFULLFP16-NEXT:    mov h23, v1.h[4]
+; NOFULLFP16-NEXT:    mov v4.h[1], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt s6, h22
+; NOFULLFP16-NEXT:    mov h22, v0.h[4]
+; NOFULLFP16-NEXT:    fminnm s7, s18, s7
+; NOFULLFP16-NEXT:    mov h18, v3.h[4]
+; NOFULLFP16-NEXT:    mov v5.h[1], v19.h[0]
+; NOFULLFP16-NEXT:    fcvt h16, s16
+; NOFULLFP16-NEXT:    fminnm s6, s20, s6
+; NOFULLFP16-NEXT:    mov v4.h[2], v17.h[0]
+; NOFULLFP16-NEXT:    fcvt s17, h21
+; NOFULLFP16-NEXT:    fcvt s19, h22
+; NOFULLFP16-NEXT:    fcvt h7, s7
+; NOFULLFP16-NEXT:    fcvt s18, h18
+; NOFULLFP16-NEXT:    fcvt s20, h23
+; NOFULLFP16-NEXT:    mov h21, v2.h[5]
+; NOFULLFP16-NEXT:    mov h22, v0.h[5]
+; NOFULLFP16-NEXT:    mov v5.h[2], v16.h[0]
+; NOFULLFP16-NEXT:    mov h16, v3.h[5]
+; NOFULLFP16-NEXT:    mov h23, v1.h[5]
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    mov h0, v0.h[7]
+; NOFULLFP16-NEXT:    mov h1, v1.h[7]
+; NOFULLFP16-NEXT:    fminnm s17, s19, s17
+; NOFULLFP16-NEXT:    mov h19, v2.h[6]
+; NOFULLFP16-NEXT:    mov v4.h[3], v7.h[0]
+; NOFULLFP16-NEXT:    fminnm s18, s20, s18
+; NOFULLFP16-NEXT:    mov h20, v3.h[6]
+; NOFULLFP16-NEXT:    fcvt s7, h21
+; NOFULLFP16-NEXT:    fcvt s21, h22
+; NOFULLFP16-NEXT:    fcvt s22, h24
+; NOFULLFP16-NEXT:    mov h2, v2.h[7]
+; NOFULLFP16-NEXT:    mov v5.h[3], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt s6, h16
+; NOFULLFP16-NEXT:    fcvt s16, h23
+; NOFULLFP16-NEXT:    fcvt h17, s17
+; NOFULLFP16-NEXT:    fcvt s19, h19
+; NOFULLFP16-NEXT:    fcvt s23, h25
+; NOFULLFP16-NEXT:    fcvt h18, s18
+; NOFULLFP16-NEXT:    fcvt s20, h20
+; NOFULLFP16-NEXT:    mov h3, v3.h[7]
+; NOFULLFP16-NEXT:    fminnm s7, s21, s7
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    fminnm s6, s16, s6
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    mov v4.h[4], v17.h[0]
+; NOFULLFP16-NEXT:    fminnm s16, s22, s19
+; NOFULLFP16-NEXT:    mov v5.h[4], v18.h[0]
+; NOFULLFP16-NEXT:    fminnm s17, s23, s20
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt h7, s7
+; NOFULLFP16-NEXT:    fminnm s0, s0, s2
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    fcvt h2, s16
+; NOFULLFP16-NEXT:    fminnm s1, s1, s3
+; NOFULLFP16-NEXT:    mov v4.h[5], v7.h[0]
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    mov v5.h[5], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt h6, s17
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v4.h[6], v2.h[0]
+; NOFULLFP16-NEXT:    mov v5.h[6], v6.h[0]
+; NOFULLFP16-NEXT:    mov v4.h[7], v0.h[0]
+; NOFULLFP16-NEXT:    mov v5.h[7], v1.h[0]
+; NOFULLFP16-NEXT:    mov v0.16b, v4.16b
+; NOFULLFP16-NEXT:    mov v1.16b, v5.16b
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call nnan <16 x half> @llvm.minimumnum.v16f16(<16 x half> %a, <16 x half> %b)
   ret <16 x half> %c
@@ -642,112 +1245,413 @@ entry:
 
 ;;;;;;;;;;;;;;;;;; max_f16
 define half @max_f16(half %a, half %b) {
-; AARCH64-LABEL: max_f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm h1, h1, h1
-; AARCH64-NEXT:    fminnm h0, h0, h0
-; AARCH64-NEXT:    fmaxnm h0, h0, h1
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm h1, h1, h1
+; FULLFP16-NEXT:    fminnm h0, h0, h0
+; FULLFP16-NEXT:    fmaxnm h0, h0, h1
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    fmaxnm s0, s0, s1
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call half @llvm.maximumnum.f16(half %a, half %b)
   ret half %c
 }
 
 define <2 x half> @max_v2f16(<2 x half> %a, <2 x half> %b) {
-; AARCH64-LABEL: max_v2f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v1.4h, v1.4h, v1.4h
-; AARCH64-NEXT:    fminnm v0.4h, v0.4h, v0.4h
-; AARCH64-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_v2f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v1.4h, v1.4h, v1.4h
+; FULLFP16-NEXT:    fminnm v0.4h, v0.4h, v0.4h
+; FULLFP16-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_v2f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    mov h4, v1.h[2]
+; NOFULLFP16-NEXT:    mov h5, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h1
+; NOFULLFP16-NEXT:    fcvt s7, h0
+; NOFULLFP16-NEXT:    mov h1, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fmaxnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h4
+; NOFULLFP16-NEXT:    fcvt s4, h5
+; NOFULLFP16-NEXT:    fmaxnm s5, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v0.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s3, s4, s3
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fcvt h0, s5
+; NOFULLFP16-NEXT:    fcvt s4, h6
+; NOFULLFP16-NEXT:    mov v0.h[1], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    fmaxnm s1, s4, s1
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v0.h[3], v1.h[0]
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <2 x half> @llvm.maximumnum.v2f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %c
 }
 
 define <4 x half> @max_v4f16(<4 x half> %a, <4 x half> %b) {
-; AARCH64-LABEL: max_v4f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v1.4h, v1.4h, v1.4h
-; AARCH64-NEXT:    fminnm v0.4h, v0.4h, v0.4h
-; AARCH64-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_v4f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v1.4h, v1.4h, v1.4h
+; FULLFP16-NEXT:    fminnm v0.4h, v0.4h, v0.4h
+; FULLFP16-NEXT:    fmaxnm v0.4h, v0.4h, v1.4h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_v4f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    mov h4, v1.h[2]
+; NOFULLFP16-NEXT:    mov h5, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h1
+; NOFULLFP16-NEXT:    fcvt s7, h0
+; NOFULLFP16-NEXT:    mov h1, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fmaxnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h4
+; NOFULLFP16-NEXT:    fcvt s4, h5
+; NOFULLFP16-NEXT:    fmaxnm s5, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v0.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s3, s4, s3
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fcvt h0, s5
+; NOFULLFP16-NEXT:    fcvt s4, h6
+; NOFULLFP16-NEXT:    mov v0.h[1], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    fmaxnm s1, s4, s1
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v0.h[3], v1.h[0]
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <4 x half> @llvm.maximumnum.v4f16(<4 x half> %a, <4 x half> %b)
   ret <4 x half> %c
 }
 
 define <8 x half> @max_v8f16(<8 x half> %a, <8 x half> %b) {
-; AARCH64-LABEL: max_v8f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v1.8h, v1.8h, v1.8h
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v0.8h
-; AARCH64-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_v8f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v1.8h
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v0.8h
+; FULLFP16-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_v8f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    fcvt s4, h1
+; NOFULLFP16-NEXT:    fcvt s5, h0
+; NOFULLFP16-NEXT:    mov h6, v1.h[2]
+; NOFULLFP16-NEXT:    mov h7, v0.h[2]
+; NOFULLFP16-NEXT:    mov h16, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fmaxnm s4, s5, s4
+; NOFULLFP16-NEXT:    mov h5, v0.h[3]
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fmaxnm s3, s3, s2
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    fcvt h2, s4
+; NOFULLFP16-NEXT:    fmaxnm s4, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v1.h[4]
+; NOFULLFP16-NEXT:    mov h7, v0.h[4]
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fmaxnm s5, s5, s16
+; NOFULLFP16-NEXT:    mov h16, v0.h[5]
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    mov v2.h[1], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt s3, h6
+; NOFULLFP16-NEXT:    fcvt s6, h7
+; NOFULLFP16-NEXT:    mov h7, v1.h[5]
+; NOFULLFP16-NEXT:    fcvt h5, s5
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    mov v2.h[2], v4.h[0]
+; NOFULLFP16-NEXT:    mov h4, v1.h[6]
+; NOFULLFP16-NEXT:    fmaxnm s3, s6, s3
+; NOFULLFP16-NEXT:    mov h6, v0.h[6]
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    mov h1, v1.h[7]
+; NOFULLFP16-NEXT:    mov h0, v0.h[7]
+; NOFULLFP16-NEXT:    mov v2.h[3], v5.h[0]
+; NOFULLFP16-NEXT:    fcvt s4, h4
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fcvt s5, h6
+; NOFULLFP16-NEXT:    fmaxnm s6, s16, s7
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    mov v2.h[4], v3.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s4, s5, s4
+; NOFULLFP16-NEXT:    fcvt h3, s6
+; NOFULLFP16-NEXT:    fmaxnm s0, s0, s1
+; NOFULLFP16-NEXT:    mov v2.h[5], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt h3, s4
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    mov v2.h[6], v3.h[0]
+; NOFULLFP16-NEXT:    mov v2.h[7], v0.h[0]
+; NOFULLFP16-NEXT:    mov v0.16b, v2.16b
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <8 x half> @llvm.maximumnum.v8f16(<8 x half> %a, <8 x half> %b)
   ret <8 x half> %c
 }
 
 define <9 x half> @max_v9f16(<9 x half> %a, <9 x half> %b) {
-; AARCH64-LABEL: max_v9f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    // kill: def $h0 killed $h0 def $q0
-; AARCH64-NEXT:    // kill: def $h1 killed $h1 def $q1
-; AARCH64-NEXT:    // kill: def $h2 killed $h2 def $q2
-; AARCH64-NEXT:    add x9, sp, #16
-; AARCH64-NEXT:    // kill: def $h3 killed $h3 def $q3
-; AARCH64-NEXT:    // kill: def $h4 killed $h4 def $q4
-; AARCH64-NEXT:    // kill: def $h5 killed $h5 def $q5
-; AARCH64-NEXT:    // kill: def $h6 killed $h6 def $q6
-; AARCH64-NEXT:    // kill: def $h7 killed $h7 def $q7
-; AARCH64-NEXT:    mov v0.h[1], v1.h[0]
-; AARCH64-NEXT:    ldr h1, [sp, #8]
-; AARCH64-NEXT:    ld1 { v1.h }[1], [x9]
-; AARCH64-NEXT:    add x9, sp, #24
-; AARCH64-NEXT:    mov v0.h[2], v2.h[0]
-; AARCH64-NEXT:    ldr h2, [sp]
-; AARCH64-NEXT:    ld1 { v1.h }[2], [x9]
-; AARCH64-NEXT:    add x9, sp, #32
-; AARCH64-NEXT:    fminnm v2.8h, v2.8h, v2.8h
-; AARCH64-NEXT:    mov v0.h[3], v3.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[3], [x9]
-; AARCH64-NEXT:    add x9, sp, #40
-; AARCH64-NEXT:    ldr h3, [sp, #72]
-; AARCH64-NEXT:    ld1 { v1.h }[4], [x9]
-; AARCH64-NEXT:    add x9, sp, #48
-; AARCH64-NEXT:    fminnm v3.8h, v3.8h, v3.8h
-; AARCH64-NEXT:    mov v0.h[4], v4.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[5], [x9]
-; AARCH64-NEXT:    add x9, sp, #56
-; AARCH64-NEXT:    fmaxnm v2.8h, v2.8h, v3.8h
-; AARCH64-NEXT:    mov v0.h[5], v5.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[6], [x9]
-; AARCH64-NEXT:    add x9, sp, #64
-; AARCH64-NEXT:    str h2, [x8, #16]
-; AARCH64-NEXT:    mov v0.h[6], v6.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[7], [x9]
-; AARCH64-NEXT:    fminnm v1.8h, v1.8h, v1.8h
-; AARCH64-NEXT:    mov v0.h[7], v7.h[0]
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v0.8h
-; AARCH64-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
-; AARCH64-NEXT:    str q0, [x8]
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_v9f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    // kill: def $h0 killed $h0 def $q0
+; FULLFP16-NEXT:    // kill: def $h1 killed $h1 def $q1
+; FULLFP16-NEXT:    // kill: def $h2 killed $h2 def $q2
+; FULLFP16-NEXT:    add x9, sp, #16
+; FULLFP16-NEXT:    // kill: def $h3 killed $h3 def $q3
+; FULLFP16-NEXT:    // kill: def $h4 killed $h4 def $q4
+; FULLFP16-NEXT:    // kill: def $h5 killed $h5 def $q5
+; FULLFP16-NEXT:    // kill: def $h6 killed $h6 def $q6
+; FULLFP16-NEXT:    // kill: def $h7 killed $h7 def $q7
+; FULLFP16-NEXT:    mov v0.h[1], v1.h[0]
+; FULLFP16-NEXT:    ldr h1, [sp, #8]
+; FULLFP16-NEXT:    ld1 { v1.h }[1], [x9]
+; FULLFP16-NEXT:    add x9, sp, #24
+; FULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; FULLFP16-NEXT:    ldr h2, [sp]
+; FULLFP16-NEXT:    ld1 { v1.h }[2], [x9]
+; FULLFP16-NEXT:    add x9, sp, #32
+; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v2.8h
+; FULLFP16-NEXT:    mov v0.h[3], v3.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[3], [x9]
+; FULLFP16-NEXT:    add x9, sp, #40
+; FULLFP16-NEXT:    ldr h3, [sp, #72]
+; FULLFP16-NEXT:    ld1 { v1.h }[4], [x9]
+; FULLFP16-NEXT:    add x9, sp, #48
+; FULLFP16-NEXT:    fminnm v3.8h, v3.8h, v3.8h
+; FULLFP16-NEXT:    mov v0.h[4], v4.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[5], [x9]
+; FULLFP16-NEXT:    add x9, sp, #56
+; FULLFP16-NEXT:    fmaxnm v2.8h, v2.8h, v3.8h
+; FULLFP16-NEXT:    mov v0.h[5], v5.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[6], [x9]
+; FULLFP16-NEXT:    add x9, sp, #64
+; FULLFP16-NEXT:    str h2, [x8, #16]
+; FULLFP16-NEXT:    mov v0.h[6], v6.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[7], [x9]
+; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v1.8h
+; FULLFP16-NEXT:    mov v0.h[7], v7.h[0]
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v0.8h
+; FULLFP16-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
+; FULLFP16-NEXT:    str q0, [x8]
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_v9f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    ldr h16, [sp, #16]
+; NOFULLFP16-NEXT:    ldr h17, [sp, #8]
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    ldr h18, [sp, #24]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fcvt s17, h17
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s18, h18
+; NOFULLFP16-NEXT:    fcvt s4, h4
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    fmaxnm s1, s1, s16
+; NOFULLFP16-NEXT:    fmaxnm s0, s0, s17
+; NOFULLFP16-NEXT:    ldr h16, [sp, #32]
+; NOFULLFP16-NEXT:    fmaxnm s2, s2, s18
+; NOFULLFP16-NEXT:    ldr h17, [sp, #40]
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fcvt s17, h17
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fmaxnm s3, s3, s16
+; NOFULLFP16-NEXT:    fmaxnm s4, s4, s17
+; NOFULLFP16-NEXT:    mov v0.h[1], v1.h[0]
+; NOFULLFP16-NEXT:    ldr h1, [sp, #48]
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    ldr h2, [sp, #56]
+; NOFULLFP16-NEXT:    fmaxnm s1, s5, s1
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    ldr h5, [sp, #64]
+; NOFULLFP16-NEXT:    mov v0.h[3], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt s3, h6
+; NOFULLFP16-NEXT:    ldr h6, [sp, #72]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    mov v0.h[4], v4.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h5
+; NOFULLFP16-NEXT:    fcvt s4, h7
+; NOFULLFP16-NEXT:    ldr h5, [sp]
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    mov v0.h[5], v1.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s2
+; NOFULLFP16-NEXT:    fmaxnm s2, s4, s3
+; NOFULLFP16-NEXT:    fmaxnm s3, s5, s6
+; NOFULLFP16-NEXT:    mov v0.h[6], v1.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s2
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    mov v0.h[7], v1.h[0]
+; NOFULLFP16-NEXT:    str h2, [x8, #16]
+; NOFULLFP16-NEXT:    str q0, [x8]
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <9 x half> @llvm.maximumnum.v9f16(<9 x half> %a, <9 x half> %b)
   ret <9 x half> %c
 }
 
 define <16 x half> @max_v16f16(<16 x half> %a, <16 x half> %b) {
-; AARCH64-LABEL: max_v16f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v2.8h, v2.8h, v2.8h
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v0.8h
-; AARCH64-NEXT:    fminnm v3.8h, v3.8h, v3.8h
-; AARCH64-NEXT:    fminnm v1.8h, v1.8h, v1.8h
-; AARCH64-NEXT:    fmaxnm v0.8h, v0.8h, v2.8h
-; AARCH64-NEXT:    fmaxnm v1.8h, v1.8h, v3.8h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: max_v16f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v2.8h
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v0.8h
+; FULLFP16-NEXT:    fminnm v3.8h, v3.8h, v3.8h
+; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v1.8h
+; FULLFP16-NEXT:    fmaxnm v0.8h, v0.8h, v2.8h
+; FULLFP16-NEXT:    fmaxnm v1.8h, v1.8h, v3.8h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: max_v16f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    mov h6, v2.h[1]
+; NOFULLFP16-NEXT:    mov h7, v0.h[1]
+; NOFULLFP16-NEXT:    fcvt s4, h2
+; NOFULLFP16-NEXT:    fcvt s5, h0
+; NOFULLFP16-NEXT:    mov h16, v3.h[1]
+; NOFULLFP16-NEXT:    mov h17, v1.h[1]
+; NOFULLFP16-NEXT:    mov h18, v2.h[2]
+; NOFULLFP16-NEXT:    mov h19, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s20, h3
+; NOFULLFP16-NEXT:    fcvt s21, h1
+; NOFULLFP16-NEXT:    mov h22, v3.h[2]
+; NOFULLFP16-NEXT:    mov h23, v1.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    mov h24, v0.h[6]
+; NOFULLFP16-NEXT:    fmaxnm s4, s5, s4
+; NOFULLFP16-NEXT:    fcvt s5, h16
+; NOFULLFP16-NEXT:    fcvt s16, h17
+; NOFULLFP16-NEXT:    fcvt s17, h18
+; NOFULLFP16-NEXT:    fcvt s18, h19
+; NOFULLFP16-NEXT:    mov h19, v0.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s20, s21, s20
+; NOFULLFP16-NEXT:    fcvt s21, h22
+; NOFULLFP16-NEXT:    mov h22, v3.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s6, s7, s6
+; NOFULLFP16-NEXT:    mov h7, v2.h[3]
+; NOFULLFP16-NEXT:    mov h25, v1.h[6]
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    fmaxnm s5, s16, s5
+; NOFULLFP16-NEXT:    fcvt s16, h23
+; NOFULLFP16-NEXT:    mov h23, v1.h[3]
+; NOFULLFP16-NEXT:    fmaxnm s17, s18, s17
+; NOFULLFP16-NEXT:    fcvt s18, h19
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    fcvt h19, s5
+; NOFULLFP16-NEXT:    fcvt h5, s20
+; NOFULLFP16-NEXT:    fmaxnm s16, s16, s21
+; NOFULLFP16-NEXT:    fcvt s20, h23
+; NOFULLFP16-NEXT:    fcvt h17, s17
+; NOFULLFP16-NEXT:    mov h21, v2.h[4]
+; NOFULLFP16-NEXT:    mov h23, v1.h[4]
+; NOFULLFP16-NEXT:    mov v4.h[1], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt s6, h22
+; NOFULLFP16-NEXT:    mov h22, v0.h[4]
+; NOFULLFP16-NEXT:    fmaxnm s7, s18, s7
+; NOFULLFP16-NEXT:    mov h18, v3.h[4]
+; NOFULLFP16-NEXT:    mov v5.h[1], v19.h[0]
+; NOFULLFP16-NEXT:    fcvt h16, s16
+; NOFULLFP16-NEXT:    fmaxnm s6, s20, s6
+; NOFULLFP16-NEXT:    mov v4.h[2], v17.h[0]
+; NOFULLFP16-NEXT:    fcvt s17, h21
+; NOFULLFP16-NEXT:    fcvt s19, h22
+; NOFULLFP16-NEXT:    fcvt h7, s7
+; NOFULLFP16-NEXT:    fcvt s18, h18
+; NOFULLFP16-NEXT:    fcvt s20, h23
+; NOFULLFP16-NEXT:    mov h21, v2.h[5]
+; NOFULLFP16-NEXT:    mov h22, v0.h[5]
+; NOFULLFP16-NEXT:    mov v5.h[2], v16.h[0]
+; NOFULLFP16-NEXT:    mov h16, v3.h[5]
+; NOFULLFP16-NEXT:    mov h23, v1.h[5]
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    mov h0, v0.h[7]
+; NOFULLFP16-NEXT:    mov h1, v1.h[7]
+; NOFULLFP16-NEXT:    fmaxnm s17, s19, s17
+; NOFULLFP16-NEXT:    mov h19, v2.h[6]
+; NOFULLFP16-NEXT:    mov v4.h[3], v7.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s18, s20, s18
+; NOFULLFP16-NEXT:    mov h20, v3.h[6]
+; NOFULLFP16-NEXT:    fcvt s7, h21
+; NOFULLFP16-NEXT:    fcvt s21, h22
+; NOFULLFP16-NEXT:    fcvt s22, h24
+; NOFULLFP16-NEXT:    mov h2, v2.h[7]
+; NOFULLFP16-NEXT:    mov v5.h[3], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt s6, h16
+; NOFULLFP16-NEXT:    fcvt s16, h23
+; NOFULLFP16-NEXT:    fcvt h17, s17
+; NOFULLFP16-NEXT:    fcvt s19, h19
+; NOFULLFP16-NEXT:    fcvt s23, h25
+; NOFULLFP16-NEXT:    fcvt h18, s18
+; NOFULLFP16-NEXT:    fcvt s20, h20
+; NOFULLFP16-NEXT:    mov h3, v3.h[7]
+; NOFULLFP16-NEXT:    fmaxnm s7, s21, s7
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    fmaxnm s6, s16, s6
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    mov v4.h[4], v17.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s16, s22, s19
+; NOFULLFP16-NEXT:    mov v5.h[4], v18.h[0]
+; NOFULLFP16-NEXT:    fmaxnm s17, s23, s20
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt h7, s7
+; NOFULLFP16-NEXT:    fmaxnm s0, s0, s2
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    fcvt h2, s16
+; NOFULLFP16-NEXT:    fmaxnm s1, s1, s3
+; NOFULLFP16-NEXT:    mov v4.h[5], v7.h[0]
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    mov v5.h[5], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt h6, s17
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v4.h[6], v2.h[0]
+; NOFULLFP16-NEXT:    mov v5.h[6], v6.h[0]
+; NOFULLFP16-NEXT:    mov v4.h[7], v0.h[0]
+; NOFULLFP16-NEXT:    mov v5.h[7], v1.h[0]
+; NOFULLFP16-NEXT:    mov v0.16b, v4.16b
+; NOFULLFP16-NEXT:    mov v1.16b, v5.16b
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <16 x half> @llvm.maximumnum.v16f16(<16 x half> %a, <16 x half> %b)
   ret <16 x half> %c
@@ -922,112 +1826,413 @@ entry:
 
 ;;;;;;;;;;;;;;;;;; min_f16
 define half @min_f16(half %a, half %b) {
-; AARCH64-LABEL: min_f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm h1, h1, h1
-; AARCH64-NEXT:    fminnm h0, h0, h0
-; AARCH64-NEXT:    fminnm h0, h0, h1
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm h1, h1, h1
+; FULLFP16-NEXT:    fminnm h0, h0, h0
+; FULLFP16-NEXT:    fminnm h0, h0, h1
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    fminnm s0, s0, s1
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call half @llvm.minimumnum.f16(half %a, half %b)
   ret half %c
 }
 
 define <2 x half> @min_v2f16(<2 x half> %a, <2 x half> %b) {
-; AARCH64-LABEL: min_v2f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v1.4h, v1.4h, v1.4h
-; AARCH64-NEXT:    fminnm v0.4h, v0.4h, v0.4h
-; AARCH64-NEXT:    fminnm v0.4h, v0.4h, v1.4h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_v2f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v1.4h, v1.4h, v1.4h
+; FULLFP16-NEXT:    fminnm v0.4h, v0.4h, v0.4h
+; FULLFP16-NEXT:    fminnm v0.4h, v0.4h, v1.4h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_v2f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    mov h4, v1.h[2]
+; NOFULLFP16-NEXT:    mov h5, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h1
+; NOFULLFP16-NEXT:    fcvt s7, h0
+; NOFULLFP16-NEXT:    mov h1, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fminnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h4
+; NOFULLFP16-NEXT:    fcvt s4, h5
+; NOFULLFP16-NEXT:    fminnm s5, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v0.h[3]
+; NOFULLFP16-NEXT:    fminnm s3, s4, s3
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fcvt h0, s5
+; NOFULLFP16-NEXT:    fcvt s4, h6
+; NOFULLFP16-NEXT:    mov v0.h[1], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    fminnm s1, s4, s1
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v0.h[3], v1.h[0]
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <2 x half> @llvm.minimumnum.v2f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %c
 }
 
 define <4 x half> @min_v4f16(<4 x half> %a, <4 x half> %b) {
-; AARCH64-LABEL: min_v4f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v1.4h, v1.4h, v1.4h
-; AARCH64-NEXT:    fminnm v0.4h, v0.4h, v0.4h
-; AARCH64-NEXT:    fminnm v0.4h, v0.4h, v1.4h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_v4f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v1.4h, v1.4h, v1.4h
+; FULLFP16-NEXT:    fminnm v0.4h, v0.4h, v0.4h
+; FULLFP16-NEXT:    fminnm v0.4h, v0.4h, v1.4h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_v4f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    mov h4, v1.h[2]
+; NOFULLFP16-NEXT:    mov h5, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h1
+; NOFULLFP16-NEXT:    fcvt s7, h0
+; NOFULLFP16-NEXT:    mov h1, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fminnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h4
+; NOFULLFP16-NEXT:    fcvt s4, h5
+; NOFULLFP16-NEXT:    fminnm s5, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v0.h[3]
+; NOFULLFP16-NEXT:    fminnm s3, s4, s3
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fcvt h0, s5
+; NOFULLFP16-NEXT:    fcvt s4, h6
+; NOFULLFP16-NEXT:    mov v0.h[1], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    fminnm s1, s4, s1
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v0.h[3], v1.h[0]
+; NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <4 x half> @llvm.minimumnum.v4f16(<4 x half> %a, <4 x half> %b)
   ret <4 x half> %c
 }
 
 define <8 x half> @min_v8f16(<8 x half> %a, <8 x half> %b) {
-; AARCH64-LABEL: min_v8f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v1.8h, v1.8h, v1.8h
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v0.8h
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v1.8h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_v8f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v1.8h
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v0.8h
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v1.8h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_v8f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    mov h2, v1.h[1]
+; NOFULLFP16-NEXT:    mov h3, v0.h[1]
+; NOFULLFP16-NEXT:    fcvt s4, h1
+; NOFULLFP16-NEXT:    fcvt s5, h0
+; NOFULLFP16-NEXT:    mov h6, v1.h[2]
+; NOFULLFP16-NEXT:    mov h7, v0.h[2]
+; NOFULLFP16-NEXT:    mov h16, v1.h[3]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fminnm s4, s5, s4
+; NOFULLFP16-NEXT:    mov h5, v0.h[3]
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fminnm s3, s3, s2
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    fcvt h2, s4
+; NOFULLFP16-NEXT:    fminnm s4, s7, s6
+; NOFULLFP16-NEXT:    mov h6, v1.h[4]
+; NOFULLFP16-NEXT:    mov h7, v0.h[4]
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fminnm s5, s5, s16
+; NOFULLFP16-NEXT:    mov h16, v0.h[5]
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    mov v2.h[1], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt s3, h6
+; NOFULLFP16-NEXT:    fcvt s6, h7
+; NOFULLFP16-NEXT:    mov h7, v1.h[5]
+; NOFULLFP16-NEXT:    fcvt h5, s5
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    mov v2.h[2], v4.h[0]
+; NOFULLFP16-NEXT:    mov h4, v1.h[6]
+; NOFULLFP16-NEXT:    fminnm s3, s6, s3
+; NOFULLFP16-NEXT:    mov h6, v0.h[6]
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    mov h1, v1.h[7]
+; NOFULLFP16-NEXT:    mov h0, v0.h[7]
+; NOFULLFP16-NEXT:    mov v2.h[3], v5.h[0]
+; NOFULLFP16-NEXT:    fcvt s4, h4
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fcvt s5, h6
+; NOFULLFP16-NEXT:    fminnm s6, s16, s7
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    mov v2.h[4], v3.h[0]
+; NOFULLFP16-NEXT:    fminnm s4, s5, s4
+; NOFULLFP16-NEXT:    fcvt h3, s6
+; NOFULLFP16-NEXT:    fminnm s0, s0, s1
+; NOFULLFP16-NEXT:    mov v2.h[5], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt h3, s4
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    mov v2.h[6], v3.h[0]
+; NOFULLFP16-NEXT:    mov v2.h[7], v0.h[0]
+; NOFULLFP16-NEXT:    mov v0.16b, v2.16b
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <8 x half> @llvm.minimumnum.v8f16(<8 x half> %a, <8 x half> %b)
   ret <8 x half> %c
 }
 
 define <9 x half> @min_v9f16(<9 x half> %a, <9 x half> %b) {
-; AARCH64-LABEL: min_v9f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    // kill: def $h0 killed $h0 def $q0
-; AARCH64-NEXT:    // kill: def $h1 killed $h1 def $q1
-; AARCH64-NEXT:    // kill: def $h2 killed $h2 def $q2
-; AARCH64-NEXT:    add x9, sp, #16
-; AARCH64-NEXT:    // kill: def $h3 killed $h3 def $q3
-; AARCH64-NEXT:    // kill: def $h4 killed $h4 def $q4
-; AARCH64-NEXT:    // kill: def $h5 killed $h5 def $q5
-; AARCH64-NEXT:    // kill: def $h6 killed $h6 def $q6
-; AARCH64-NEXT:    // kill: def $h7 killed $h7 def $q7
-; AARCH64-NEXT:    mov v0.h[1], v1.h[0]
-; AARCH64-NEXT:    ldr h1, [sp, #8]
-; AARCH64-NEXT:    ld1 { v1.h }[1], [x9]
-; AARCH64-NEXT:    add x9, sp, #24
-; AARCH64-NEXT:    mov v0.h[2], v2.h[0]
-; AARCH64-NEXT:    ldr h2, [sp]
-; AARCH64-NEXT:    ld1 { v1.h }[2], [x9]
-; AARCH64-NEXT:    add x9, sp, #32
-; AARCH64-NEXT:    fminnm v2.8h, v2.8h, v2.8h
-; AARCH64-NEXT:    mov v0.h[3], v3.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[3], [x9]
-; AARCH64-NEXT:    add x9, sp, #40
-; AARCH64-NEXT:    ldr h3, [sp, #72]
-; AARCH64-NEXT:    ld1 { v1.h }[4], [x9]
-; AARCH64-NEXT:    add x9, sp, #48
-; AARCH64-NEXT:    fminnm v3.8h, v3.8h, v3.8h
-; AARCH64-NEXT:    mov v0.h[4], v4.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[5], [x9]
-; AARCH64-NEXT:    add x9, sp, #56
-; AARCH64-NEXT:    fminnm v2.8h, v2.8h, v3.8h
-; AARCH64-NEXT:    mov v0.h[5], v5.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[6], [x9]
-; AARCH64-NEXT:    add x9, sp, #64
-; AARCH64-NEXT:    str h2, [x8, #16]
-; AARCH64-NEXT:    mov v0.h[6], v6.h[0]
-; AARCH64-NEXT:    ld1 { v1.h }[7], [x9]
-; AARCH64-NEXT:    fminnm v1.8h, v1.8h, v1.8h
-; AARCH64-NEXT:    mov v0.h[7], v7.h[0]
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v0.8h
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v1.8h
-; AARCH64-NEXT:    str q0, [x8]
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_v9f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    // kill: def $h0 killed $h0 def $q0
+; FULLFP16-NEXT:    // kill: def $h1 killed $h1 def $q1
+; FULLFP16-NEXT:    // kill: def $h2 killed $h2 def $q2
+; FULLFP16-NEXT:    add x9, sp, #16
+; FULLFP16-NEXT:    // kill: def $h3 killed $h3 def $q3
+; FULLFP16-NEXT:    // kill: def $h4 killed $h4 def $q4
+; FULLFP16-NEXT:    // kill: def $h5 killed $h5 def $q5
+; FULLFP16-NEXT:    // kill: def $h6 killed $h6 def $q6
+; FULLFP16-NEXT:    // kill: def $h7 killed $h7 def $q7
+; FULLFP16-NEXT:    mov v0.h[1], v1.h[0]
+; FULLFP16-NEXT:    ldr h1, [sp, #8]
+; FULLFP16-NEXT:    ld1 { v1.h }[1], [x9]
+; FULLFP16-NEXT:    add x9, sp, #24
+; FULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; FULLFP16-NEXT:    ldr h2, [sp]
+; FULLFP16-NEXT:    ld1 { v1.h }[2], [x9]
+; FULLFP16-NEXT:    add x9, sp, #32
+; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v2.8h
+; FULLFP16-NEXT:    mov v0.h[3], v3.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[3], [x9]
+; FULLFP16-NEXT:    add x9, sp, #40
+; FULLFP16-NEXT:    ldr h3, [sp, #72]
+; FULLFP16-NEXT:    ld1 { v1.h }[4], [x9]
+; FULLFP16-NEXT:    add x9, sp, #48
+; FULLFP16-NEXT:    fminnm v3.8h, v3.8h, v3.8h
+; FULLFP16-NEXT:    mov v0.h[4], v4.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[5], [x9]
+; FULLFP16-NEXT:    add x9, sp, #56
+; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v3.8h
+; FULLFP16-NEXT:    mov v0.h[5], v5.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[6], [x9]
+; FULLFP16-NEXT:    add x9, sp, #64
+; FULLFP16-NEXT:    str h2, [x8, #16]
+; FULLFP16-NEXT:    mov v0.h[6], v6.h[0]
+; FULLFP16-NEXT:    ld1 { v1.h }[7], [x9]
+; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v1.8h
+; FULLFP16-NEXT:    mov v0.h[7], v7.h[0]
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v0.8h
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v1.8h
+; FULLFP16-NEXT:    str q0, [x8]
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_v9f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    ldr h16, [sp, #16]
+; NOFULLFP16-NEXT:    ldr h17, [sp, #8]
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    ldr h18, [sp, #24]
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fcvt s17, h17
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt s18, h18
+; NOFULLFP16-NEXT:    fcvt s4, h4
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    fminnm s1, s1, s16
+; NOFULLFP16-NEXT:    fminnm s0, s0, s17
+; NOFULLFP16-NEXT:    ldr h16, [sp, #32]
+; NOFULLFP16-NEXT:    fminnm s2, s2, s18
+; NOFULLFP16-NEXT:    ldr h17, [sp, #40]
+; NOFULLFP16-NEXT:    fcvt s16, h16
+; NOFULLFP16-NEXT:    fcvt s17, h17
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    fcvt h2, s2
+; NOFULLFP16-NEXT:    fminnm s3, s3, s16
+; NOFULLFP16-NEXT:    fminnm s4, s4, s17
+; NOFULLFP16-NEXT:    mov v0.h[1], v1.h[0]
+; NOFULLFP16-NEXT:    ldr h1, [sp, #48]
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    fcvt h3, s3
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    mov v0.h[2], v2.h[0]
+; NOFULLFP16-NEXT:    ldr h2, [sp, #56]
+; NOFULLFP16-NEXT:    fminnm s1, s5, s1
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    ldr h5, [sp, #64]
+; NOFULLFP16-NEXT:    mov v0.h[3], v3.h[0]
+; NOFULLFP16-NEXT:    fcvt s3, h6
+; NOFULLFP16-NEXT:    ldr h6, [sp, #72]
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    mov v0.h[4], v4.h[0]
+; NOFULLFP16-NEXT:    fminnm s2, s3, s2
+; NOFULLFP16-NEXT:    fcvt s3, h5
+; NOFULLFP16-NEXT:    fcvt s4, h7
+; NOFULLFP16-NEXT:    ldr h5, [sp]
+; NOFULLFP16-NEXT:    fcvt s5, h5
+; NOFULLFP16-NEXT:    mov v0.h[5], v1.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s2
+; NOFULLFP16-NEXT:    fminnm s2, s4, s3
+; NOFULLFP16-NEXT:    fminnm s3, s5, s6
+; NOFULLFP16-NEXT:    mov v0.h[6], v1.h[0]
+; NOFULLFP16-NEXT:    fcvt h1, s2
+; NOFULLFP16-NEXT:    fcvt h2, s3
+; NOFULLFP16-NEXT:    mov v0.h[7], v1.h[0]
+; NOFULLFP16-NEXT:    str h2, [x8, #16]
+; NOFULLFP16-NEXT:    str q0, [x8]
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <9 x half> @llvm.minimumnum.v9f16(<9 x half> %a, <9 x half> %b)
   ret <9 x half> %c
 }
 
 define <16 x half> @min_v16f16(<16 x half> %a, <16 x half> %b) {
-; AARCH64-LABEL: min_v16f16:
-; AARCH64:       // %bb.0: // %entry
-; AARCH64-NEXT:    fminnm v2.8h, v2.8h, v2.8h
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v0.8h
-; AARCH64-NEXT:    fminnm v3.8h, v3.8h, v3.8h
-; AARCH64-NEXT:    fminnm v1.8h, v1.8h, v1.8h
-; AARCH64-NEXT:    fminnm v0.8h, v0.8h, v2.8h
-; AARCH64-NEXT:    fminnm v1.8h, v1.8h, v3.8h
-; AARCH64-NEXT:    ret
+; FULLFP16-LABEL: min_v16f16:
+; FULLFP16:       // %bb.0: // %entry
+; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v2.8h
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v0.8h
+; FULLFP16-NEXT:    fminnm v3.8h, v3.8h, v3.8h
+; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v1.8h
+; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v2.8h
+; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v3.8h
+; FULLFP16-NEXT:    ret
+;
+; NOFULLFP16-LABEL: min_v16f16:
+; NOFULLFP16:       // %bb.0: // %entry
+; NOFULLFP16-NEXT:    mov h6, v2.h[1]
+; NOFULLFP16-NEXT:    mov h7, v0.h[1]
+; NOFULLFP16-NEXT:    fcvt s4, h2
+; NOFULLFP16-NEXT:    fcvt s5, h0
+; NOFULLFP16-NEXT:    mov h16, v3.h[1]
+; NOFULLFP16-NEXT:    mov h17, v1.h[1]
+; NOFULLFP16-NEXT:    mov h18, v2.h[2]
+; NOFULLFP16-NEXT:    mov h19, v0.h[2]
+; NOFULLFP16-NEXT:    fcvt s20, h3
+; NOFULLFP16-NEXT:    fcvt s21, h1
+; NOFULLFP16-NEXT:    mov h22, v3.h[2]
+; NOFULLFP16-NEXT:    mov h23, v1.h[2]
+; NOFULLFP16-NEXT:    fcvt s6, h6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    mov h24, v0.h[6]
+; NOFULLFP16-NEXT:    fminnm s4, s5, s4
+; NOFULLFP16-NEXT:    fcvt s5, h16
+; NOFULLFP16-NEXT:    fcvt s16, h17
+; NOFULLFP16-NEXT:    fcvt s17, h18
+; NOFULLFP16-NEXT:    fcvt s18, h19
+; NOFULLFP16-NEXT:    mov h19, v0.h[3]
+; NOFULLFP16-NEXT:    fminnm s20, s21, s20
+; NOFULLFP16-NEXT:    fcvt s21, h22
+; NOFULLFP16-NEXT:    mov h22, v3.h[3]
+; NOFULLFP16-NEXT:    fminnm s6, s7, s6
+; NOFULLFP16-NEXT:    mov h7, v2.h[3]
+; NOFULLFP16-NEXT:    mov h25, v1.h[6]
+; NOFULLFP16-NEXT:    fcvt h4, s4
+; NOFULLFP16-NEXT:    fminnm s5, s16, s5
+; NOFULLFP16-NEXT:    fcvt s16, h23
+; NOFULLFP16-NEXT:    mov h23, v1.h[3]
+; NOFULLFP16-NEXT:    fminnm s17, s18, s17
+; NOFULLFP16-NEXT:    fcvt s18, h19
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    fcvt s7, h7
+; NOFULLFP16-NEXT:    fcvt h19, s5
+; NOFULLFP16-NEXT:    fcvt h5, s20
+; NOFULLFP16-NEXT:    fminnm s16, s16, s21
+; NOFULLFP16-NEXT:    fcvt s20, h23
+; NOFULLFP16-NEXT:    fcvt h17, s17
+; NOFULLFP16-NEXT:    mov h21, v2.h[4]
+; NOFULLFP16-NEXT:    mov h23, v1.h[4]
+; NOFULLFP16-NEXT:    mov v4.h[1], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt s6, h22
+; NOFULLFP16-NEXT:    mov h22, v0.h[4]
+; NOFULLFP16-NEXT:    fminnm s7, s18, s7
+; NOFULLFP16-NEXT:    mov h18, v3.h[4]
+; NOFULLFP16-NEXT:    mov v5.h[1], v19.h[0]
+; NOFULLFP16-NEXT:    fcvt h16, s16
+; NOFULLFP16-NEXT:    fminnm s6, s20, s6
+; NOFULLFP16-NEXT:    mov v4.h[2], v17.h[0]
+; NOFULLFP16-NEXT:    fcvt s17, h21
+; NOFULLFP16-NEXT:    fcvt s19, h22
+; NOFULLFP16-NEXT:    fcvt h7, s7
+; NOFULLFP16-NEXT:    fcvt s18, h18
+; NOFULLFP16-NEXT:    fcvt s20, h23
+; NOFULLFP16-NEXT:    mov h21, v2.h[5]
+; NOFULLFP16-NEXT:    mov h22, v0.h[5]
+; NOFULLFP16-NEXT:    mov v5.h[2], v16.h[0]
+; NOFULLFP16-NEXT:    mov h16, v3.h[5]
+; NOFULLFP16-NEXT:    mov h23, v1.h[5]
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    mov h0, v0.h[7]
+; NOFULLFP16-NEXT:    mov h1, v1.h[7]
+; NOFULLFP16-NEXT:    fminnm s17, s19, s17
+; NOFULLFP16-NEXT:    mov h19, v2.h[6]
+; NOFULLFP16-NEXT:    mov v4.h[3], v7.h[0]
+; NOFULLFP16-NEXT:    fminnm s18, s20, s18
+; NOFULLFP16-NEXT:    mov h20, v3.h[6]
+; NOFULLFP16-NEXT:    fcvt s7, h21
+; NOFULLFP16-NEXT:    fcvt s21, h22
+; NOFULLFP16-NEXT:    fcvt s22, h24
+; NOFULLFP16-NEXT:    mov h2, v2.h[7]
+; NOFULLFP16-NEXT:    mov v5.h[3], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt s6, h16
+; NOFULLFP16-NEXT:    fcvt s16, h23
+; NOFULLFP16-NEXT:    fcvt h17, s17
+; NOFULLFP16-NEXT:    fcvt s19, h19
+; NOFULLFP16-NEXT:    fcvt s23, h25
+; NOFULLFP16-NEXT:    fcvt h18, s18
+; NOFULLFP16-NEXT:    fcvt s20, h20
+; NOFULLFP16-NEXT:    mov h3, v3.h[7]
+; NOFULLFP16-NEXT:    fminnm s7, s21, s7
+; NOFULLFP16-NEXT:    fcvt s2, h2
+; NOFULLFP16-NEXT:    fcvt s0, h0
+; NOFULLFP16-NEXT:    fminnm s6, s16, s6
+; NOFULLFP16-NEXT:    fcvt s1, h1
+; NOFULLFP16-NEXT:    mov v4.h[4], v17.h[0]
+; NOFULLFP16-NEXT:    fminnm s16, s22, s19
+; NOFULLFP16-NEXT:    mov v5.h[4], v18.h[0]
+; NOFULLFP16-NEXT:    fminnm s17, s23, s20
+; NOFULLFP16-NEXT:    fcvt s3, h3
+; NOFULLFP16-NEXT:    fcvt h7, s7
+; NOFULLFP16-NEXT:    fminnm s0, s0, s2
+; NOFULLFP16-NEXT:    fcvt h6, s6
+; NOFULLFP16-NEXT:    fcvt h2, s16
+; NOFULLFP16-NEXT:    fminnm s1, s1, s3
+; NOFULLFP16-NEXT:    mov v4.h[5], v7.h[0]
+; NOFULLFP16-NEXT:    fcvt h0, s0
+; NOFULLFP16-NEXT:    mov v5.h[5], v6.h[0]
+; NOFULLFP16-NEXT:    fcvt h6, s17
+; NOFULLFP16-NEXT:    fcvt h1, s1
+; NOFULLFP16-NEXT:    mov v4.h[6], v2.h[0]
+; NOFULLFP16-NEXT:    mov v5.h[6], v6.h[0]
+; NOFULLFP16-NEXT:    mov v4.h[7], v0.h[0]
+; NOFULLFP16-NEXT:    mov v5.h[7], v1.h[0]
+; NOFULLFP16-NEXT:    mov v0.16b, v4.16b
+; NOFULLFP16-NEXT:    mov v1.16b, v5.16b
+; NOFULLFP16-NEXT:    ret
 entry:
   %c = call <16 x half> @llvm.minimumnum.v16f16(<16 x half> %a, <16 x half> %b)
   ret <16 x half> %c

From 4c97c5131f9ca32ce644a0be6e3586077ee03aa6 Mon Sep 17 00:00:00 2001
From: Sudharsan Veeravalli <quic_svs@quicinc.com>
Date: Wed, 16 Apr 2025 06:48:56 +0530
Subject: [PATCH 063/710] [RISCV] Add ISel patterns for Xqcilia instructions
 (#135724)

This patch adds instruction selection patterns for generating the long
immediate arithmetic instructions.

We prefer generating instructions that have a 26 bit immediate to a 32
bit immediate given that both are of the same size but the former might
be easier to register allocate for. Base RISC-V arithmetic instructions
will be preferred, when applicable.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td |  30 ++++++
 llvm/test/CodeGen/RISCV/xqcilia.ll          | 108 ++++++++++++++++++++
 2 files changed, 138 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/xqcilia.ll

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index 8eaa5e394a91c..2458bda80b1d6 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -118,6 +118,12 @@ def simm20_li : RISCVOp<XLenVT> {
 
 def simm26 : RISCVSImmLeafOp<26>;
 
+def simm26_nosimm12 : ImmLeaf<XLenVT, [{
+  return isInt<26>(Imm) && !isInt<12>(Imm);}]>;
+
+def simm32_nosimm26 : ImmLeaf<XLenVT, [{
+  return isInt<32>(Imm) && !isInt<26>(Imm);}]>;
+
 class BareSImmNAsmOperand<int width>
     : ImmAsmOperand<"BareS", width, ""> {
   let PredicateMethod = "isBareSimmN<" # width # ">";
@@ -1223,5 +1229,29 @@ def PseudoQC_E_SW : PseudoStore<"qc.e.sw">;
 // Code Gen Patterns
 //===----------------------------------------------------------------------===//
 
+/// Generic pattern classes
+
+class PatGprNoX0Simm26NoSimm12<SDPatternOperator OpNode, RVInst48 Inst>
+    : Pat<(i32 (OpNode (i32 GPRNoX0:$rs1), simm26_nosimm12:$imm)),
+          (Inst GPRNoX0:$rs1, simm26_nosimm12:$imm)>;
+
+class PatGprNoX0Simm32NoSimm26<SDPatternOperator OpNode, RVInst48 Inst>
+    : Pat<(i32 (OpNode (i32 GPRNoX0:$rs1), simm32_nosimm26:$imm)),
+          (Inst GPRNoX0:$rs1, simm32_nosimm26:$imm)>;
+
+/// Simple arithmetic operations
+
+let Predicates = [HasVendorXqcilia, IsRV32] in {
+def : PatGprNoX0Simm32NoSimm26<add, QC_E_ADDAI>;
+def : PatGprNoX0Simm32NoSimm26<and, QC_E_ANDAI>;
+def : PatGprNoX0Simm32NoSimm26<or, QC_E_ORAI>;
+def : PatGprNoX0Simm32NoSimm26<xor, QC_E_XORAI>;
+
+def : PatGprNoX0Simm26NoSimm12<add, QC_E_ADDI>;
+def : PatGprNoX0Simm26NoSimm12<and, QC_E_ANDI>;
+def : PatGprNoX0Simm26NoSimm12<or, QC_E_ORI>;
+def : PatGprNoX0Simm26NoSimm12<xor, QC_E_XORI>;
+} // Predicates = [HasVendorXqcilia, IsRV32]
+
 let Predicates = [HasVendorXqciint, IsRV32] in
 def : Pat<(riscv_mileaveret_glue), (QC_C_MILEAVERET)>;
diff --git a/llvm/test/CodeGen/RISCV/xqcilia.ll b/llvm/test/CodeGen/RISCV/xqcilia.ll
new file mode 100644
index 0000000000000..0f14044d62dc8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xqcilia.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test that we are able to generate the Xqcilia instructions
+; RUN: llc < %s -mtriple=riscv32 | FileCheck %s -check-prefix=RV32I
+; RUN: llc < %s -mtriple=riscv32 -mattr=+experimental-xqcilia | FileCheck %s -check-prefix=RV32XQCILIA
+
+define i32 @add(i32 %a, i32 %b) {
+; RV32I-LABEL: add:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 65536
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    lui a2, 573
+; RV32I-NEXT:    addi a2, a2, -1330
+; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    addi a0, a0, 13
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: add:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    qc.e.addi a1, a1, 2345678
+; RV32XQCILIA-NEXT:    qc.e.addai a0, 268435456
+; RV32XQCILIA-NEXT:    and a0, a0, a1
+; RV32XQCILIA-NEXT:    addi a0, a0, 13
+; RV32XQCILIA-NEXT:    ret
+ %addai = add i32 %a, 268435456
+ %add = add i32 %b, 2345678
+ %and = and i32 %add, %addai
+ %res = add i32 %and, 13
+ ret i32 %res
+}
+
+define i32 @and(i32 %a, i32 %b) {
+; RV32I-LABEL: and:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 65536
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    lui a2, 573
+; RV32I-NEXT:    addi a2, a2, -1330
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srl a0, a1, a0
+; RV32I-NEXT:    andi a0, a0, 10
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: and:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    qc.e.andi a1, a1, 2345678
+; RV32XQCILIA-NEXT:    qc.e.andai a0, 268435456
+; RV32XQCILIA-NEXT:    srl a0, a1, a0
+; RV32XQCILIA-NEXT:    andi a0, a0, 10
+; RV32XQCILIA-NEXT:    ret
+ %andai = and i32 %a, 268435456
+ %and = and i32 %b, 2345678
+ %srl = lshr i32 %and, %andai
+ %res = and i32 %srl, 10
+ ret i32 %res
+}
+
+define i32 @or(i32 %a, i32 %b) {
+; RV32I-LABEL: or:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 65536
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    lui a2, 573
+; RV32I-NEXT:    addi a2, a2, -1330
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    ori a0, a0, 13
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: or:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    qc.e.ori a1, a1, 2345678
+; RV32XQCILIA-NEXT:    qc.e.orai a0, 268435456
+; RV32XQCILIA-NEXT:    add a0, a0, a1
+; RV32XQCILIA-NEXT:    ori a0, a0, 13
+; RV32XQCILIA-NEXT:    ret
+ %orai = or i32 %a, 268435456
+ %or = or i32 %b, 2345678
+ %add = add i32 %or, %orai
+ %res = or i32 %add, 13
+ ret i32 %res
+}
+
+define i32 @xor(i32 %a, i32 %b) {
+; RV32I-LABEL: xor:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 65536
+; RV32I-NEXT:    xor a0, a0, a2
+; RV32I-NEXT:    lui a2, 573
+; RV32I-NEXT:    addi a2, a2, -1330
+; RV32I-NEXT:    xor a1, a1, a2
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    xori a0, a0, 13
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: xor:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    qc.e.xori a1, a1, 2345678
+; RV32XQCILIA-NEXT:    qc.e.xorai a0, 268435456
+; RV32XQCILIA-NEXT:    add a0, a0, a1
+; RV32XQCILIA-NEXT:    xori a0, a0, 13
+; RV32XQCILIA-NEXT:    ret
+ %xorai = xor i32 %a, 268435456
+ %xor = xor i32 %b, 2345678
+ %add = add i32 %xor, %xorai
+ %res = xor i32 %add, 13
+ ret i32 %res
+}

From 0ce8ad68e44aaf50d1e2aa304fa8a1127e311e1d Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Wed, 16 Apr 2025 08:33:42 +0700
Subject: [PATCH 064/710] [SPARC] Use fzero/fzeros to materialize FP zeros when
 we have VIS

Reviewers: rorth, brad0, s-barannikov

Reviewed By: s-barannikov

Pull Request: https://github.com/llvm/llvm-project/pull/135712
---
 llvm/lib/Target/Sparc/SparcISelLowering.cpp |   6 +
 llvm/lib/Target/Sparc/SparcISelLowering.h   |   3 +
 llvm/lib/Target/Sparc/SparcInstrVIS.td      |  17 ++-
 llvm/test/CodeGen/SPARC/float-constants.ll  | 115 ++++++++++++++++++++
 4 files changed, 139 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 85b8750d40f46..bce8ddbd47586 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -3560,6 +3560,12 @@ bool SparcTargetLowering::useLoadStackGuardNode(const Module &M) const {
   return true;
 }
 
+bool SparcTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
+                                       bool ForCodeSize) const {
+  return Subtarget->isVIS() && (VT == MVT::f32 || VT == MVT::f64) &&
+         Imm.isZero();
+}
+
 // Override to disable global variable loading on Linux.
 void SparcTargetLowering::insertSSPDeclarations(Module &M) const {
   if (!Subtarget->isTargetLinux())
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.h b/llvm/lib/Target/Sparc/SparcISelLowering.h
index 1bee5f4cfe84d..c09e465f5d05e 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.h
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.h
@@ -207,6 +207,9 @@ namespace llvm {
       return VT != MVT::f128;
     }
 
+    bool isFPImmLegal(const APFloat &Imm, EVT VT,
+                      bool ForCodeSize) const override;
+
     bool shouldInsertFencesForAtomic(const Instruction *I) const override {
       // FIXME: We insert fences for each atomics and generate
       // sub-optimal code for PSO/TSO. (Approximately nobody uses any
diff --git a/llvm/lib/Target/Sparc/SparcInstrVIS.td b/llvm/lib/Target/Sparc/SparcInstrVIS.td
index 8ce8f37f34040..7be45fe9faf3f 100644
--- a/llvm/lib/Target/Sparc/SparcInstrVIS.td
+++ b/llvm/lib/Target/Sparc/SparcInstrVIS.td
@@ -45,10 +45,10 @@ class VISInst2<bits<9> opfval, string OpcStr, RegisterClass RC = DFPRegs>
         !strconcat(OpcStr, " $rs2, $rd")>;
 
 // For VIS Instructions with only rd operand.
-let Constraints = "$rd = $f", rs1 = 0, rs2 = 0 in
+let rs1 = 0, rs2 = 0 in
 class VISInstD<bits<9> opfval, string OpcStr, RegisterClass RC = DFPRegs>
        : VISInstFormat<opfval,
-        (outs RC:$rd), (ins RC:$f),
+        (outs RC:$rd), (ins),
         !strconcat(OpcStr, " $rd")>;
 
 // VIS 1 Instructions
@@ -277,3 +277,16 @@ def UMULXHI  : VISInst<0b000010110, "umulxhi", I64Regs>;
 def XMULX    : VISInst<0b100010101, "xmulx",   I64Regs>;
 def XMULXHI  : VISInst<0b100010110, "xmulxhi", I64Regs>;
 } // Predicates = [IsVIS3]
+
+// FP immediate patterns.
+def fpimm0 : FPImmLeaf<fAny, [{return Imm.isExactlyValue(+0.0);}]>;
+def fpnegimm0 : FPImmLeaf<fAny, [{return Imm.isExactlyValue(-0.0);}]>;
+
+// VIS instruction patterns.
+let Predicates = [HasVIS] in {
+// Zero immediate.
+def : Pat<(f64 fpimm0), (FZERO)>;
+def : Pat<(f32 fpimm0), (FZEROS)>;
+def : Pat<(f64 fpnegimm0), (FNEGD (FZERO))>;
+def : Pat<(f32 fpnegimm0), (FNEGS (FZEROS))>;
+} // Predicates = [HasVIS]
diff --git a/llvm/test/CodeGen/SPARC/float-constants.ll b/llvm/test/CodeGen/SPARC/float-constants.ll
index b04ec68ed3d7e..440c75bfca9f9 100644
--- a/llvm/test/CodeGen/SPARC/float-constants.ll
+++ b/llvm/test/CodeGen/SPARC/float-constants.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc < %s -mtriple=sparc | FileCheck %s
 ; RUN: llc < %s -mtriple=sparcel | FileCheck %s --check-prefix=CHECK-LE
+; RUN: llc < %s -mtriple=sparcv9 -mattr=+vis | FileCheck %s --check-prefix=CHECK-VIS
 
 ;; Bitcast should not do a runtime conversion, but rather emit a
 ;; constant into integer registers directly.
@@ -17,6 +18,12 @@ define <2 x i32> @bitcast() nounwind {
 ; CHECK-LE-NEXT:    sethi 1049856, %o1
 ; CHECK-LE-NEXT:    retl
 ; CHECK-LE-NEXT:    mov %g0, %o0
+;
+; CHECK-VIS-LABEL: bitcast:
+; CHECK-VIS:       ! %bb.0:
+; CHECK-VIS-NEXT:    sethi 1049856, %o0
+; CHECK-VIS-NEXT:    retl
+; CHECK-VIS-NEXT:    mov %g0, %o1
   %1 = bitcast double 5.0 to <2 x i32>
   ret <2 x i32> %1
 }
@@ -43,6 +50,17 @@ define void @test_call() nounwind {
 ; CHECK-LE-NEXT:    mov %g0, %o0
 ; CHECK-LE-NEXT:    ret
 ; CHECK-LE-NEXT:    restore
+;
+; CHECK-VIS-LABEL: test_call:
+; CHECK-VIS:       ! %bb.0:
+; CHECK-VIS-NEXT:    save %sp, -176, %sp
+; CHECK-VIS-NEXT:    sethi %h44(.LCPI1_0), %i0
+; CHECK-VIS-NEXT:    add %i0, %m44(.LCPI1_0), %i0
+; CHECK-VIS-NEXT:    sllx %i0, 12, %i0
+; CHECK-VIS-NEXT:    call a
+; CHECK-VIS-NEXT:    ldd [%i0+%l44(.LCPI1_0)], %f0
+; CHECK-VIS-NEXT:    ret
+; CHECK-VIS-NEXT:    restore
   call void @a(double 5.0)
   ret void
 }
@@ -75,6 +93,103 @@ define double @test_intrins_call() nounwind {
 ; CHECK-LE-NEXT:    mov %o1, %o3
 ; CHECK-LE-NEXT:    ret
 ; CHECK-LE-NEXT:    restore
+;
+; CHECK-VIS-LABEL: test_intrins_call:
+; CHECK-VIS:       ! %bb.0:
+; CHECK-VIS-NEXT:    save %sp, -176, %sp
+; CHECK-VIS-NEXT:    sethi %h44(.LCPI2_0), %i0
+; CHECK-VIS-NEXT:    add %i0, %m44(.LCPI2_0), %i0
+; CHECK-VIS-NEXT:    sllx %i0, 12, %i0
+; CHECK-VIS-NEXT:    ldd [%i0+%l44(.LCPI2_0)], %f0
+; CHECK-VIS-NEXT:    fmovd %f0, %f2
+; CHECK-VIS-NEXT:    call pow
+; CHECK-VIS-NEXT:    nop
+; CHECK-VIS-NEXT:    ret
+; CHECK-VIS-NEXT:    restore
   %1 = call double @llvm.pow.f64(double 2.0, double 2.0)
   ret double %1
 }
+
+;; When we have VIS, f32/f64 zero constant should be materialized from fzero/fzeros.
+
+define double @pos_zero_double() nounwind {
+; CHECK-LABEL: pos_zero_double:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    sethi %hi(.LCPI3_0), %o0
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    ldd [%o0+%lo(.LCPI3_0)], %f0
+;
+; CHECK-LE-LABEL: pos_zero_double:
+; CHECK-LE:       ! %bb.0:
+; CHECK-LE-NEXT:    sethi %hi(.LCPI3_0), %o0
+; CHECK-LE-NEXT:    retl
+; CHECK-LE-NEXT:    ldd [%o0+%lo(.LCPI3_0)], %f0
+;
+; CHECK-VIS-LABEL: pos_zero_double:
+; CHECK-VIS:       ! %bb.0:
+; CHECK-VIS-NEXT:    retl
+; CHECK-VIS-NEXT:    fzero %f0
+  ret double +0.0
+}
+
+define double @neg_zero_double() nounwind {
+; CHECK-LABEL: neg_zero_double:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    sethi %hi(.LCPI4_0), %o0
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    ldd [%o0+%lo(.LCPI4_0)], %f0
+;
+; CHECK-LE-LABEL: neg_zero_double:
+; CHECK-LE:       ! %bb.0:
+; CHECK-LE-NEXT:    sethi %hi(.LCPI4_0), %o0
+; CHECK-LE-NEXT:    retl
+; CHECK-LE-NEXT:    ldd [%o0+%lo(.LCPI4_0)], %f0
+;
+; CHECK-VIS-LABEL: neg_zero_double:
+; CHECK-VIS:       ! %bb.0:
+; CHECK-VIS-NEXT:    fzero %f0
+; CHECK-VIS-NEXT:    retl
+; CHECK-VIS-NEXT:    fnegd %f0, %f0
+  ret double -0.0
+}
+
+define float @pos_zero_float() nounwind {
+; CHECK-LABEL: pos_zero_float:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    sethi %hi(.LCPI5_0), %o0
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    ld [%o0+%lo(.LCPI5_0)], %f0
+;
+; CHECK-LE-LABEL: pos_zero_float:
+; CHECK-LE:       ! %bb.0:
+; CHECK-LE-NEXT:    sethi %hi(.LCPI5_0), %o0
+; CHECK-LE-NEXT:    retl
+; CHECK-LE-NEXT:    ld [%o0+%lo(.LCPI5_0)], %f0
+;
+; CHECK-VIS-LABEL: pos_zero_float:
+; CHECK-VIS:       ! %bb.0:
+; CHECK-VIS-NEXT:    retl
+; CHECK-VIS-NEXT:    fzeros %f0
+  ret float +0.0
+}
+
+define float @neg_zero_float() nounwind {
+; CHECK-LABEL: neg_zero_float:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    sethi %hi(.LCPI6_0), %o0
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    ld [%o0+%lo(.LCPI6_0)], %f0
+;
+; CHECK-LE-LABEL: neg_zero_float:
+; CHECK-LE:       ! %bb.0:
+; CHECK-LE-NEXT:    sethi %hi(.LCPI6_0), %o0
+; CHECK-LE-NEXT:    retl
+; CHECK-LE-NEXT:    ld [%o0+%lo(.LCPI6_0)], %f0
+;
+; CHECK-VIS-LABEL: neg_zero_float:
+; CHECK-VIS:       ! %bb.0:
+; CHECK-VIS-NEXT:    fzeros %f0
+; CHECK-VIS-NEXT:    retl
+; CHECK-VIS-NEXT:    fnegs %f0, %f0
+  ret float -0.0
+}

From f3de63c64998eb46db9cf26aca9ebcc5453f6f44 Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Wed, 16 Apr 2025 08:36:47 +0700
Subject: [PATCH 065/710] [SPARC] Use addxccc to do multiword addition when we
 have VIS3

Reviewers: brad0, s-barannikov, rorth

Reviewed By: s-barannikov

Pull Request: https://github.com/llvm/llvm-project/pull/135713
---
 llvm/lib/Target/Sparc/SparcISelLowering.cpp |   5 +
 llvm/lib/Target/Sparc/SparcInstr64Bit.td    |   2 +
 llvm/lib/Target/Sparc/SparcInstrVIS.td      |   5 +
 llvm/test/CodeGen/SPARC/2011-01-11-CC.ll    | 118 ++++++++++++++++++++
 4 files changed, 130 insertions(+)

diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index bce8ddbd47586..098e5f22834f4 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1737,6 +1737,11 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::SUBC, MVT::i32, Legal);
   setOperationAction(ISD::SUBE, MVT::i32, Legal);
 
+  if (Subtarget->isVIS3()) {
+    setOperationAction(ISD::ADDC, MVT::i64, Legal);
+    setOperationAction(ISD::ADDE, MVT::i64, Legal);
+  }
+
   if (Subtarget->is64Bit()) {
     setOperationAction(ISD::BITCAST, MVT::f64, Expand);
     setOperationAction(ISD::BITCAST, MVT::i64, Expand);
diff --git a/llvm/lib/Target/Sparc/SparcInstr64Bit.td b/llvm/lib/Target/Sparc/SparcInstr64Bit.td
index 56fab2f26a19e..000612534e89d 100644
--- a/llvm/lib/Target/Sparc/SparcInstr64Bit.td
+++ b/llvm/lib/Target/Sparc/SparcInstr64Bit.td
@@ -157,9 +157,11 @@ def : Pat<(and i64:$lhs, (not i64:$rhs)), (ANDNrr $lhs, $rhs)>;
 def : Pat<(or  i64:$lhs, (not i64:$rhs)), (ORNrr  $lhs, $rhs)>;
 def : Pat<(not (xor i64:$lhs, i64:$rhs)), (XNORrr $lhs, $rhs)>;
 
+def : Pat<(addc i64:$lhs, i64:$rhs), (ADDCCrr $lhs, $rhs)>, Requires<[HasVIS3]>;
 def : Pat<(add i64:$lhs, i64:$rhs), (ADDrr $lhs, $rhs)>;
 def : Pat<(sub i64:$lhs, i64:$rhs), (SUBrr $lhs, $rhs)>;
 
+def : Pat<(addc i64:$lhs, (i64 simm13:$rhs)), (ADDCCri $lhs, imm:$rhs)>, Requires<[HasVIS3]>;
 def : Pat<(add i64:$lhs, (i64 simm13:$rhs)), (ADDri $lhs, imm:$rhs)>;
 def : Pat<(sub i64:$lhs, (i64 simm13:$rhs)), (SUBri $lhs, imm:$rhs)>;
 
diff --git a/llvm/lib/Target/Sparc/SparcInstrVIS.td b/llvm/lib/Target/Sparc/SparcInstrVIS.td
index 7be45fe9faf3f..ee24d8a54fe8e 100644
--- a/llvm/lib/Target/Sparc/SparcInstrVIS.td
+++ b/llvm/lib/Target/Sparc/SparcInstrVIS.td
@@ -290,3 +290,8 @@ def : Pat<(f32 fpimm0), (FZEROS)>;
 def : Pat<(f64 fpnegimm0), (FNEGD (FZERO))>;
 def : Pat<(f32 fpnegimm0), (FNEGS (FZEROS))>;
 } // Predicates = [HasVIS]
+
+// VIS3 instruction patterns.
+let Predicates = [HasVIS3] in {
+def : Pat<(i64 (adde i64:$lhs, i64:$rhs)), (ADDXCCC $lhs, $rhs)>;
+} // Predicates = [HasVIS3]
diff --git a/llvm/test/CodeGen/SPARC/2011-01-11-CC.ll b/llvm/test/CodeGen/SPARC/2011-01-11-CC.ll
index 1560bc687b7dd..e05c47bfee766 100644
--- a/llvm/test/CodeGen/SPARC/2011-01-11-CC.ll
+++ b/llvm/test/CodeGen/SPARC/2011-01-11-CC.ll
@@ -2,6 +2,7 @@
 ; RUN: llc -mtriple=sparc %s -o - | FileCheck %s -check-prefix=V8
 ; RUN: llc -mtriple=sparc -mattr=v9 %s -o - | FileCheck %s -check-prefix=V9
 ; RUN: llc -mtriple=sparc64-unknown-linux %s -o - | FileCheck %s -check-prefix=SPARC64
+; RUN: llc -mtriple=sparc64-unknown-linux -mattr=vis3 %s -o - | FileCheck %s -check-prefix=SPARC64-VIS3
 
 define i32 @test_addx(i64 %a, i64 %b, i64 %c) nounwind {
 ; V8-LABEL: test_addx:
@@ -60,6 +61,15 @@ define i32 @test_addx(i64 %a, i64 %b, i64 %c) nounwind {
 ; SPARC64-NEXT:    movgu %xcc, 1, %o3
 ; SPARC64-NEXT:    retl
 ; SPARC64-NEXT:    srl %o3, 0, %o0
+;
+; SPARC64-VIS3-LABEL: test_addx:
+; SPARC64-VIS3:       ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    mov %g0, %o3
+; SPARC64-VIS3-NEXT:    add %o0, %o1, %o0
+; SPARC64-VIS3-NEXT:    cmp %o0, %o2
+; SPARC64-VIS3-NEXT:    movgu %xcc, 1, %o3
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    srl %o3, 0, %o0
 entry:
   %0 = add i64 %a, %b
   %1 = icmp ugt i64 %0, %c
@@ -92,6 +102,13 @@ define i32 @test_select_int_icc(i32 %a, i32 %b, i32 %c) nounwind {
 ; SPARC64-NEXT:    move %icc, %o1, %o2
 ; SPARC64-NEXT:    retl
 ; SPARC64-NEXT:    mov %o2, %o0
+;
+; SPARC64-VIS3-LABEL: test_select_int_icc:
+; SPARC64-VIS3:       ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    cmp %o0, 0
+; SPARC64-VIS3-NEXT:    move %icc, %o1, %o2
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    mov %o2, %o0
 entry:
   %0 = icmp eq i32 %a, 0
   %1 = select i1 %0, i32 %b, i32 %c
@@ -133,6 +150,13 @@ define float @test_select_fp_icc(i32 %a, float %f1, float %f2) nounwind {
 ; SPARC64-NEXT:    cmp %o0, 0
 ; SPARC64-NEXT:    retl
 ; SPARC64-NEXT:    fmovse %icc, %f3, %f0
+;
+; SPARC64-VIS3-LABEL: test_select_fp_icc:
+; SPARC64-VIS3:       ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    fmovs %f5, %f0
+; SPARC64-VIS3-NEXT:    cmp %o0, 0
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    fmovse %icc, %f3, %f0
 entry:
   %0 = icmp eq i32 %a, 0
   %1 = select i1 %0, float %f1, float %f2
@@ -182,6 +206,13 @@ define double @test_select_dfp_icc(i32 %a, double %f1, double %f2) nounwind {
 ; SPARC64-NEXT:    cmp %o0, 0
 ; SPARC64-NEXT:    retl
 ; SPARC64-NEXT:    fmovde %icc, %f2, %f0
+;
+; SPARC64-VIS3-LABEL: test_select_dfp_icc:
+; SPARC64-VIS3:       ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    fmovd %f4, %f0
+; SPARC64-VIS3-NEXT:    cmp %o0, 0
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    fmovde %icc, %f2, %f0
 entry:
   %0 = icmp eq i32 %a, 0
   %1 = select i1 %0, double %f1, double %f2
@@ -229,6 +260,17 @@ define i32 @test_select_int_fcc(float %f, i32 %a, i32 %b) nounwind {
 ; SPARC64-NEXT:    fcmps %fcc0, %f1, %f0
 ; SPARC64-NEXT:    retl
 ; SPARC64-NEXT:    movne %fcc0, %o1, %o0
+;
+; SPARC64-VIS3-LABEL: test_select_int_fcc:
+; SPARC64-VIS3:       ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    sethi %h44(.LCPI4_0), %o0
+; SPARC64-VIS3-NEXT:    add %o0, %m44(.LCPI4_0), %o0
+; SPARC64-VIS3-NEXT:    sllx %o0, 12, %o0
+; SPARC64-VIS3-NEXT:    ld [%o0+%l44(.LCPI4_0)], %f0
+; SPARC64-VIS3-NEXT:    mov %o2, %o0
+; SPARC64-VIS3-NEXT:    fcmps %fcc0, %f1, %f0
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    movne %fcc0, %o1, %o0
 entry:
   %0 = fcmp une float %f, 0.000000e+00
   %a.b = select i1 %0, i32 %a, i32 %b
@@ -284,6 +326,17 @@ define float @test_select_fp_fcc(float %f, float %f1, float %f2) nounwind {
 ; SPARC64-NEXT:    fcmps %fcc0, %f1, %f2
 ; SPARC64-NEXT:    retl
 ; SPARC64-NEXT:    fmovsne %fcc0, %f3, %f0
+;
+; SPARC64-VIS3-LABEL: test_select_fp_fcc:
+; SPARC64-VIS3:       ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    sethi %h44(.LCPI5_0), %o0
+; SPARC64-VIS3-NEXT:    add %o0, %m44(.LCPI5_0), %o0
+; SPARC64-VIS3-NEXT:    sllx %o0, 12, %o0
+; SPARC64-VIS3-NEXT:    ld [%o0+%l44(.LCPI5_0)], %f2
+; SPARC64-VIS3-NEXT:    fmovs %f5, %f0
+; SPARC64-VIS3-NEXT:    fcmps %fcc0, %f1, %f2
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    fmovsne %fcc0, %f3, %f0
 entry:
   %0 = fcmp une float %f, 0.000000e+00
   %1 = select i1 %0, float %f1, float %f2
@@ -352,6 +405,18 @@ define double @test_select_dfp_fcc(double %f, double %f1, double %f2) nounwind {
 ; SPARC64-NEXT:    fmovd %f4, %f0
 ; SPARC64-NEXT:    retl
 ; SPARC64-NEXT:    nop
+;
+; SPARC64-VIS3-LABEL: test_select_dfp_fcc:
+; SPARC64-VIS3:       ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    sethi %h44(.LCPI6_0), %o0
+; SPARC64-VIS3-NEXT:    add %o0, %m44(.LCPI6_0), %o0
+; SPARC64-VIS3-NEXT:    sllx %o0, 12, %o0
+; SPARC64-VIS3-NEXT:    ldd [%o0+%l44(.LCPI6_0)], %f6
+; SPARC64-VIS3-NEXT:    fcmpd %fcc0, %f0, %f6
+; SPARC64-VIS3-NEXT:    fmovdne %fcc0, %f2, %f4
+; SPARC64-VIS3-NEXT:    fmovd %f4, %f0
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    nop
 entry:
   %0 = fcmp une double %f, 0.000000e+00
   %1 = select i1 %0, double %f1, double %f2
@@ -453,6 +518,31 @@ define i32 @test_float_cc(double %a, double %b, i32 %c, i32 %d) nounwind {
 ; SPARC64-NEXT:  ! %bb.4: ! %exit.0
 ; SPARC64-NEXT:    retl
 ; SPARC64-NEXT:    mov %g0, %o0
+;
+; SPARC64-VIS3-LABEL: test_float_cc:
+; SPARC64-VIS3:       ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    sethi %h44(.LCPI7_0), %o0
+; SPARC64-VIS3-NEXT:    add %o0, %m44(.LCPI7_0), %o0
+; SPARC64-VIS3-NEXT:    sllx %o0, 12, %o0
+; SPARC64-VIS3-NEXT:    ldd [%o0+%l44(.LCPI7_0)], %f4
+; SPARC64-VIS3-NEXT:    fcmpd %fcc0, %f0, %f4
+; SPARC64-VIS3-NEXT:    fbuge %fcc0, .LBB7_3
+; SPARC64-VIS3-NEXT:    nop
+; SPARC64-VIS3-NEXT:  ! %bb.1: ! %loop.2
+; SPARC64-VIS3-NEXT:    fcmpd %fcc0, %f2, %f4
+; SPARC64-VIS3-NEXT:    fbule %fcc0, .LBB7_3
+; SPARC64-VIS3-NEXT:    nop
+; SPARC64-VIS3-NEXT:  ! %bb.2: ! %exit.1
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    mov 1, %o0
+; SPARC64-VIS3-NEXT:  .LBB7_3: ! %loop
+; SPARC64-VIS3-NEXT:    ! =>This Inner Loop Header: Depth=1
+; SPARC64-VIS3-NEXT:    cmp %o2, 10
+; SPARC64-VIS3-NEXT:    be %icc, .LBB7_3
+; SPARC64-VIS3-NEXT:    nop
+; SPARC64-VIS3-NEXT:  ! %bb.4: ! %exit.0
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    mov %g0, %o0
 entry:
    %0 = fcmp uge double %a, 0.000000e+00
    br i1 %0, label %loop, label %loop.2
@@ -558,6 +648,34 @@ define void @test_adde_sube(ptr %a, ptr %b, ptr %sum, ptr %diff) nounwind {
 ; SPARC64-NEXT:    stx %i0, [%i3]
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore
+;
+; SPARC64-VIS3-LABEL: test_adde_sube:
+; SPARC64-VIS3:         .register %g2, #scratch
+; SPARC64-VIS3-NEXT:  ! %bb.0: ! %entry
+; SPARC64-VIS3-NEXT:    save %sp, -128, %sp
+; SPARC64-VIS3-NEXT:    ldx [%i0+8], %i4
+; SPARC64-VIS3-NEXT:    ldx [%i0], %i5
+; SPARC64-VIS3-NEXT:    ldx [%i1+8], %g2
+; SPARC64-VIS3-NEXT:    ldx [%i1], %i1
+; SPARC64-VIS3-NEXT:    addcc %i4, %g2, %g2
+; SPARC64-VIS3-NEXT:    addxccc %i5, %i1, %i1
+; SPARC64-VIS3-NEXT:    stx %i1, [%i2]
+; SPARC64-VIS3-NEXT:    stx %g2, [%i2+8]
+; SPARC64-VIS3-NEXT:    !APP
+; SPARC64-VIS3-NEXT:    !NO_APP
+; SPARC64-VIS3-NEXT:    ldx [%i0+8], %i1
+; SPARC64-VIS3-NEXT:    mov %g0, %i2
+; SPARC64-VIS3-NEXT:    ldx [%i0], %i0
+; SPARC64-VIS3-NEXT:    cmp %i4, %i1
+; SPARC64-VIS3-NEXT:    movcs %xcc, 1, %i2
+; SPARC64-VIS3-NEXT:    srl %i2, 0, %i2
+; SPARC64-VIS3-NEXT:    sub %i5, %i0, %i0
+; SPARC64-VIS3-NEXT:    sub %i0, %i2, %i0
+; SPARC64-VIS3-NEXT:    sub %i4, %i1, %i1
+; SPARC64-VIS3-NEXT:    stx %i1, [%i3+8]
+; SPARC64-VIS3-NEXT:    stx %i0, [%i3]
+; SPARC64-VIS3-NEXT:    ret
+; SPARC64-VIS3-NEXT:    restore
 entry:
    %0 = bitcast ptr %a to ptr
    %1 = bitcast ptr %b to ptr

From e4f2191f568db718ed67defa664f83f763e7e74a Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Wed, 16 Apr 2025 08:38:29 +0700
Subject: [PATCH 066/710] [SPARC] Use umulxhi to do extending 64x64->128
 multiply when we have VIS3

Reviewers: s-barannikov, rorth, brad0

Reviewed By: s-barannikov

Pull Request: https://github.com/llvm/llvm-project/pull/135714
---
 llvm/lib/Target/Sparc/SparcISelLowering.cpp   |  6 +-
 llvm/lib/Target/Sparc/SparcInstrVIS.td        | 10 ++++
 llvm/test/CodeGen/SPARC/multiply-extension.ll | 59 +++++++++++++++++++
 .../SPARC/smulo-128-legalisation-lowering.ll  | 44 ++++++++++++++
 .../SPARC/umulo-128-legalisation-lowering.ll  | 33 +++++++++++
 5 files changed, 150 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPARC/multiply-extension.ll

diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 098e5f22834f4..0ad261135651f 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1854,8 +1854,10 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
   if (Subtarget->is64Bit()) {
     setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
     setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
-    setOperationAction(ISD::MULHU,     MVT::i64, Expand);
-    setOperationAction(ISD::MULHS,     MVT::i64, Expand);
+    setOperationAction(ISD::MULHU, MVT::i64,
+                       Subtarget->isVIS3() ? Legal : Expand);
+    setOperationAction(ISD::MULHS, MVT::i64,
+                       Subtarget->isVIS3() ? Legal : Expand);
 
     setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
     setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
diff --git a/llvm/lib/Target/Sparc/SparcInstrVIS.td b/llvm/lib/Target/Sparc/SparcInstrVIS.td
index ee24d8a54fe8e..d9fe3b49821e5 100644
--- a/llvm/lib/Target/Sparc/SparcInstrVIS.td
+++ b/llvm/lib/Target/Sparc/SparcInstrVIS.td
@@ -294,4 +294,14 @@ def : Pat<(f32 fpnegimm0), (FNEGS (FZEROS))>;
 // VIS3 instruction patterns.
 let Predicates = [HasVIS3] in {
 def : Pat<(i64 (adde i64:$lhs, i64:$rhs)), (ADDXCCC $lhs, $rhs)>;
+
+def : Pat<(i64 (mulhu i64:$lhs, i64:$rhs)), (UMULXHI $lhs, $rhs)>;
+// Signed "MULXHI".
+// Based on the formula presented in OSA2011 §7.140, but with bitops to select
+// the values to be added.
+// TODO: This expansion should probably be moved to DAG legalization phase.
+def : Pat<(i64 (mulhs i64:$lhs, i64:$rhs)),
+      (SUBrr (UMULXHI $lhs, $rhs),
+             (ADDrr (ANDrr (SRAXri $lhs, 63), $rhs),
+                    (ANDrr (SRAXri $rhs, 63), $lhs)))>;
 } // Predicates = [HasVIS3]
diff --git a/llvm/test/CodeGen/SPARC/multiply-extension.ll b/llvm/test/CodeGen/SPARC/multiply-extension.ll
new file mode 100644
index 0000000000000..4d752ff101ca2
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/multiply-extension.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=sparcv9 | FileCheck %s -check-prefix=V9
+; RUN: llc < %s -mtriple=sparcv9 -mattr=+vis3 | FileCheck %s -check-prefix=VIS3
+
+define i128 @signed_multiply_extend(i64 %0, i64 %1) nounwind {
+; V9-LABEL: signed_multiply_extend:
+; V9:       ! %bb.0:
+; V9-NEXT:    save %sp, -176, %sp
+; V9-NEXT:    srax %i0, 63, %o2
+; V9-NEXT:    srax %i1, 63, %o0
+; V9-NEXT:    mov %i1, %o1
+; V9-NEXT:    call __multi3
+; V9-NEXT:    mov %i0, %o3
+; V9-NEXT:    mov %o0, %i0
+; V9-NEXT:    ret
+; V9-NEXT:    restore %g0, %o1, %o1
+;
+; VIS3-LABEL: signed_multiply_extend:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    srax %o0, 63, %o2
+; VIS3-NEXT:    and %o2, %o1, %o2
+; VIS3-NEXT:    srax %o1, 63, %o3
+; VIS3-NEXT:    and %o3, %o0, %o3
+; VIS3-NEXT:    add %o3, %o2, %o2
+; VIS3-NEXT:    umulxhi %o1, %o0, %o3
+; VIS3-NEXT:    sub %o3, %o2, %o2
+; VIS3-NEXT:    mulx %o1, %o0, %o1
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    mov %o2, %o0
+  %3 = sext i64 %0 to i128
+  %4 = sext i64 %1 to i128
+  %5 = mul nsw i128 %4, %3
+  ret i128 %5
+}
+
+define i128 @unsigned_multiply_extend(i64 %0, i64 %1) nounwind {
+; V9-LABEL: unsigned_multiply_extend:
+; V9:       ! %bb.0:
+; V9-NEXT:    save %sp, -176, %sp
+; V9-NEXT:    mov %g0, %o0
+; V9-NEXT:    mov %i1, %o1
+; V9-NEXT:    mov %g0, %o2
+; V9-NEXT:    call __multi3
+; V9-NEXT:    mov %i0, %o3
+; V9-NEXT:    mov %o0, %i0
+; V9-NEXT:    ret
+; V9-NEXT:    restore %g0, %o1, %o1
+;
+; VIS3-LABEL: unsigned_multiply_extend:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    umulxhi %o1, %o0, %o2
+; VIS3-NEXT:    mulx %o1, %o0, %o1
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    mov %o2, %o0
+  %3 = zext i64 %0 to i128
+  %4 = zext i64 %1 to i128
+  %5 = mul nuw i128 %4, %3
+  ret i128 %5
+}
diff --git a/llvm/test/CodeGen/SPARC/smulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/SPARC/smulo-128-legalisation-lowering.ll
index 07e4c408a3ff0..1e5ab7922de08 100644
--- a/llvm/test/CodeGen/SPARC/smulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/SPARC/smulo-128-legalisation-lowering.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=sparc-unknown-linux-gnu | FileCheck %s --check-prefixes=SPARC
 ; RUN: llc < %s -mtriple=sparc64-unknown-linux-gnu | FileCheck %s --check-prefixes=SPARC64
+; RUN: llc < %s -mtriple=sparc64-unknown-linux-gnu -mattr=vis3 | FileCheck %s --check-prefixes=SPARC64-VIS3
 
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) nounwind {
 ; SPARC-LABEL: muloti_test:
@@ -213,6 +214,49 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) nounwind {
 ; SPARC64-NEXT:    srl %i3, 0, %i2
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore
+;
+; SPARC64-VIS3-LABEL: muloti_test:
+; SPARC64-VIS3:         .register %g2, #scratch
+; SPARC64-VIS3-NEXT:    .register %g3, #scratch
+; SPARC64-VIS3-NEXT:  ! %bb.0: ! %start
+; SPARC64-VIS3-NEXT:    save %sp, -128, %sp
+; SPARC64-VIS3-NEXT:    mov %g0, %i5
+; SPARC64-VIS3-NEXT:    umulxhi %i0, %i3, %i4
+; SPARC64-VIS3-NEXT:    srax %i0, 63, %g2
+; SPARC64-VIS3-NEXT:    mulx %g2, %i3, %g3
+; SPARC64-VIS3-NEXT:    add %i4, %g3, %i4
+; SPARC64-VIS3-NEXT:    umulxhi %i1, %i3, %g3
+; SPARC64-VIS3-NEXT:    mulx %i0, %i3, %g4
+; SPARC64-VIS3-NEXT:    addcc %g4, %g3, %g3
+; SPARC64-VIS3-NEXT:    addxccc %i4, %g0, %g4
+; SPARC64-VIS3-NEXT:    umulxhi %i1, %i2, %i4
+; SPARC64-VIS3-NEXT:    srax %i2, 63, %g5
+; SPARC64-VIS3-NEXT:    mulx %i1, %g5, %l0
+; SPARC64-VIS3-NEXT:    add %i4, %l0, %l0
+; SPARC64-VIS3-NEXT:    mulx %i1, %i2, %i4
+; SPARC64-VIS3-NEXT:    addcc %i4, %g3, %i4
+; SPARC64-VIS3-NEXT:    addxccc %l0, %g0, %g3
+; SPARC64-VIS3-NEXT:    srax %g3, 63, %l0
+; SPARC64-VIS3-NEXT:    addcc %g4, %g3, %g3
+; SPARC64-VIS3-NEXT:    srax %g4, 63, %g4
+; SPARC64-VIS3-NEXT:    addxccc %g4, %l0, %g4
+; SPARC64-VIS3-NEXT:    and %g5, %i0, %g5
+; SPARC64-VIS3-NEXT:    and %g2, %i2, %g2
+; SPARC64-VIS3-NEXT:    add %g2, %g5, %g2
+; SPARC64-VIS3-NEXT:    umulxhi %i0, %i2, %g5
+; SPARC64-VIS3-NEXT:    sub %g5, %g2, %g2
+; SPARC64-VIS3-NEXT:    mulx %i0, %i2, %i0
+; SPARC64-VIS3-NEXT:    addcc %i0, %g3, %i0
+; SPARC64-VIS3-NEXT:    addxccc %g2, %g4, %i2
+; SPARC64-VIS3-NEXT:    srax %i4, 63, %g2
+; SPARC64-VIS3-NEXT:    xor %i2, %g2, %i2
+; SPARC64-VIS3-NEXT:    xor %i0, %g2, %i0
+; SPARC64-VIS3-NEXT:    or %i0, %i2, %i0
+; SPARC64-VIS3-NEXT:    movrnz %i0, 1, %i5
+; SPARC64-VIS3-NEXT:    mulx %i1, %i3, %i1
+; SPARC64-VIS3-NEXT:    srl %i5, 0, %i2
+; SPARC64-VIS3-NEXT:    ret
+; SPARC64-VIS3-NEXT:    restore %g0, %i4, %o0
 start:
   %0 = tail call { i128, i1 } @llvm.smul.with.overflow.i128(i128 %l, i128 %r)
   %1 = extractvalue { i128, i1 } %0, 0
diff --git a/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll
index f3835790210a0..6d197c88bfecd 100644
--- a/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=sparc-unknown-linux-gnu | FileCheck %s --check-prefixes=SPARC
 ; RUN: llc < %s -mtriple=sparc64-unknown-linux-gnu | FileCheck %s --check-prefixes=SPARC64
+; RUN: llc < %s -mtriple=sparc64-unknown-linux-gnu -mattr=vis3 | FileCheck %s --check-prefixes=SPARC64-VIS3
 
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) nounwind {
 ; SPARC-LABEL: muloti_test:
@@ -199,6 +200,38 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) nounwind {
 ; SPARC64-NEXT:    srl %i1, 0, %i2
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore %g0, %o1, %o1
+;
+; SPARC64-VIS3-LABEL: muloti_test:
+; SPARC64-VIS3:         .register %g2, #scratch
+; SPARC64-VIS3-NEXT:    .register %g3, #scratch
+; SPARC64-VIS3-NEXT:  ! %bb.0: ! %start
+; SPARC64-VIS3-NEXT:    save %sp, -128, %sp
+; SPARC64-VIS3-NEXT:    mov %g0, %i5
+; SPARC64-VIS3-NEXT:    mov %g0, %g2
+; SPARC64-VIS3-NEXT:    mov %g0, %g3
+; SPARC64-VIS3-NEXT:    mov %g0, %g4
+; SPARC64-VIS3-NEXT:    mov %g0, %g5
+; SPARC64-VIS3-NEXT:    mulx %i2, %i1, %i4
+; SPARC64-VIS3-NEXT:    mulx %i0, %i3, %l0
+; SPARC64-VIS3-NEXT:    add %l0, %i4, %i4
+; SPARC64-VIS3-NEXT:    umulxhi %i1, %i3, %l0
+; SPARC64-VIS3-NEXT:    add %l0, %i4, %i4
+; SPARC64-VIS3-NEXT:    cmp %i4, %l0
+; SPARC64-VIS3-NEXT:    movrnz %i2, 1, %g2
+; SPARC64-VIS3-NEXT:    movrnz %i0, 1, %g3
+; SPARC64-VIS3-NEXT:    and %g3, %g2, %g2
+; SPARC64-VIS3-NEXT:    umulxhi %i0, %i3, %i0
+; SPARC64-VIS3-NEXT:    movrnz %i0, 1, %g4
+; SPARC64-VIS3-NEXT:    movcs %xcc, 1, %i5
+; SPARC64-VIS3-NEXT:    or %g2, %g4, %i0
+; SPARC64-VIS3-NEXT:    umulxhi %i2, %i1, %i2
+; SPARC64-VIS3-NEXT:    movrnz %i2, 1, %g5
+; SPARC64-VIS3-NEXT:    or %i0, %g5, %i0
+; SPARC64-VIS3-NEXT:    or %i0, %i5, %i0
+; SPARC64-VIS3-NEXT:    mulx %i1, %i3, %i1
+; SPARC64-VIS3-NEXT:    srl %i0, 0, %i2
+; SPARC64-VIS3-NEXT:    ret
+; SPARC64-VIS3-NEXT:    restore %g0, %i4, %o0
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r)
   %1 = extractvalue { i128, i1 } %0, 0

From 0439a4eca78fa1e3aa45b49ff349c3da4fb02f48 Mon Sep 17 00:00:00 2001
From: Jim Lin <jim@andestech.com>
Date: Wed, 16 Apr 2025 10:16:31 +0800
Subject: [PATCH 067/710] [RISCV] Add new CondCode
 COND_CV_BEQIMM/COND_CV_BNEIMM for CV immediate branch (#135771)

If there is another branch instruction also with immediate operand, but
it is used to specify which bit to be tested is set or clear. We only
check whether operand2 is immediate or not here. There are no way to
distinguish between them.

So add new CondCode COND_CV_BEQIMM/COND_CV_BNEIMM that we can know what
kinds of immediate branch instruction are matched in Select_* Pseudo.
---
 .../RISCV/GISel/RISCVInstructionSelector.cpp  |  2 +-
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  2 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      | 42 +++++++++----------
 llvm/lib/Target/RISCV/RISCVInstrInfo.h        |  6 ++-
 llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td    |  9 +++-
 5 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 18ce5407f816c..f83c2b6da8923 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -789,7 +789,7 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     RISCVCC::CondCode CC;
     getOperandsForBranch(MI.getOperand(0).getReg(), CC, LHS, RHS, *MRI);
 
-    auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(STI, CC), {}, {LHS, RHS})
+    auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
                    .addMBB(MI.getOperand(1).getMBB());
     MI.eraseFromParent();
     return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index f24752b8721f5..beea13d5f8f3e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -20645,7 +20645,7 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
 
   // Insert appropriate branch.
   if (MI.getOperand(2).isImm())
-    BuildMI(HeadMBB, DL, TII.getBrCond(CC, MI.getOperand(2).isImm()))
+    BuildMI(HeadMBB, DL, TII.getBrCond(CC))
         .addReg(LHS)
         .addImm(MI.getOperand(2).getImm())
         .addMBB(TailMBB);
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index f8a35533ba952..5d661a3438b1c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -974,10 +974,6 @@ static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) {
   switch (Opc) {
   default:
     return RISCVCC::COND_INVALID;
-  case RISCV::CV_BEQIMM:
-    return RISCVCC::COND_EQ;
-  case RISCV::CV_BNEIMM:
-    return RISCVCC::COND_NE;
   case RISCV::BEQ:
     return RISCVCC::COND_EQ;
   case RISCV::BNE:
@@ -990,6 +986,10 @@ static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) {
     return RISCVCC::COND_LTU;
   case RISCV::BGEU:
     return RISCVCC::COND_GEU;
+  case RISCV::CV_BEQIMM:
+    return RISCVCC::COND_CV_BEQIMM;
+  case RISCV::CV_BNEIMM:
+    return RISCVCC::COND_CV_BNEIMM;
   }
 }
 
@@ -1027,23 +1027,14 @@ static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
   Cond.push_back(LastInst.getOperand(1));
 }
 
-unsigned RISCVCC::getBrCond(const RISCVSubtarget &STI, RISCVCC::CondCode CC,
-                            bool Imm) {
+unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC) {
   switch (CC) {
   default:
     llvm_unreachable("Unknown condition code!");
   case RISCVCC::COND_EQ:
-    if (!Imm)
-      return RISCV::BEQ;
-    if (STI.hasVendorXCVbi())
-      return RISCV::CV_BEQIMM;
-    llvm_unreachable("Unknown branch immediate!");
+    return RISCV::BEQ;
   case RISCVCC::COND_NE:
-    if (!Imm)
-      return RISCV::BNE;
-    if (STI.hasVendorXCVbi())
-      return RISCV::CV_BNEIMM;
-    llvm_unreachable("Unknown branch immediate!");
+    return RISCV::BNE;
   case RISCVCC::COND_LT:
     return RISCV::BLT;
   case RISCVCC::COND_GE:
@@ -1052,12 +1043,15 @@ unsigned RISCVCC::getBrCond(const RISCVSubtarget &STI, RISCVCC::CondCode CC,
     return RISCV::BLTU;
   case RISCVCC::COND_GEU:
     return RISCV::BGEU;
+  case RISCVCC::COND_CV_BEQIMM:
+    return RISCV::CV_BEQIMM;
+  case RISCVCC::COND_CV_BNEIMM:
+    return RISCV::CV_BNEIMM;
   }
 }
 
-const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC,
-                                             bool Imm) const {
-  return get(RISCVCC::getBrCond(STI, CC, Imm));
+const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC) const {
+  return get(RISCVCC::getBrCond(CC));
 }
 
 RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) {
@@ -1076,6 +1070,10 @@ RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) {
     return RISCVCC::COND_GEU;
   case RISCVCC::COND_GEU:
     return RISCVCC::COND_LTU;
+  case RISCVCC::COND_CV_BEQIMM:
+    return RISCVCC::COND_CV_BNEIMM;
+  case RISCVCC::COND_CV_BNEIMM:
+    return RISCVCC::COND_CV_BEQIMM;
   }
 }
 
@@ -1206,10 +1204,8 @@ unsigned RISCVInstrInfo::insertBranch(
 
   // Either a one or two-way conditional branch.
   auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
-  MachineInstr &CondMI = *BuildMI(&MBB, DL, getBrCond(CC, Cond[2].isImm()))
-                              .add(Cond[1])
-                              .add(Cond[2])
-                              .addMBB(TBB);
+  MachineInstr &CondMI =
+      *BuildMI(&MBB, DL, getBrCond(CC)).add(Cond[1]).add(Cond[2]).addMBB(TBB);
   if (BytesAdded)
     *BytesAdded += getInstSizeInBytes(CondMI);
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index bf0d6b2c59a45..67e457d64f6e3 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -41,11 +41,13 @@ enum CondCode {
   COND_GE,
   COND_LTU,
   COND_GEU,
+  COND_CV_BEQIMM,
+  COND_CV_BNEIMM,
   COND_INVALID
 };
 
 CondCode getOppositeBranchCondition(CondCode);
-unsigned getBrCond(const RISCVSubtarget &STI, CondCode CC, bool Imm = false);
+unsigned getBrCond(CondCode CC);
 
 } // end of namespace RISCVCC
 
@@ -65,7 +67,7 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
   explicit RISCVInstrInfo(RISCVSubtarget &STI);
 
   MCInst getNop() const override;
-  const MCInstrDesc &getBrCond(RISCVCC::CondCode CC, bool Imm = false) const;
+  const MCInstrDesc &getBrCond(RISCVCC::CondCode CC) const;
 
   Register isLoadFromStackSlot(const MachineInstr &MI,
                                int &FrameIndex) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index b5df7b54fc9f1..5ce08d64b141d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -791,6 +791,13 @@ let Predicates = [HasVendorXCValu, IsRV32], AddedComplexity = 1 in {
 // Patterns for immediate branching operations
 //===----------------------------------------------------------------------===//
 
+def IntCCtoRISCVCCCV : SDNodeXForm<riscv_selectcc, [{
+  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
+  assert(CC == ISD::SETEQ || CC == ISD::SETNE);
+  RISCVCC::CondCode BrCC = CC == ISD::SETEQ ? RISCVCC::COND_CV_BEQIMM : RISCVCC::COND_CV_BNEIMM;
+  return CurDAG->getTargetConstant(BrCC, SDLoc(N), Subtarget->getXLenVT());
+}]>;
+
 let Predicates = [HasVendorXCVbi, IsRV32], AddedComplexity = 2 in {
   def : Pat<(riscv_brcc GPR:$rs1, simm5:$imm5, SETEQ, bb:$imm12),
             (CV_BEQIMM GPR:$rs1, simm5:$imm5, bare_simm13_lsb0:$imm12)>;
@@ -807,7 +814,7 @@ let Predicates = [HasVendorXCVbi, IsRV32], AddedComplexity = 2 in {
       : Pat<(riscv_selectcc_frag:$cc (i32 GPR:$lhs), simm5:$Constant, Cond,
                                      (i32 GPR:$truev), GPR:$falsev),
             (Select_GPR_Using_CC_Imm GPR:$lhs, simm5:$Constant,
-             (IntCCtoRISCVCC $cc), GPR:$truev, GPR:$falsev)>;
+             (IntCCtoRISCVCCCV $cc), GPR:$truev, GPR:$falsev)>;
 
   def : Selectbi<SETEQ>;
   def : Selectbi<SETNE>;

From 5e9650ec2deb2f2bb6d5ad28e83bb6cd3c4189e4 Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Wed, 16 Apr 2025 09:27:17 +0700
Subject: [PATCH 068/710] Revert "[SPARC] Use umulxhi to do extending
 64x64->128 multiply when we have VIS3" (#135897)

This change breaks multiply tests on SPARC.

https://lab.llvm.org/buildbot/#/builders/108/builds/11691/steps/6/logs/FAIL__LLVM__multiply-extension_ll

Reverts llvm/llvm-project#135714
---
 llvm/lib/Target/Sparc/SparcISelLowering.cpp   |  6 +-
 llvm/lib/Target/Sparc/SparcInstrVIS.td        | 10 ----
 llvm/test/CodeGen/SPARC/multiply-extension.ll | 59 -------------------
 .../SPARC/smulo-128-legalisation-lowering.ll  | 44 --------------
 .../SPARC/umulo-128-legalisation-lowering.ll  | 33 -----------
 5 files changed, 2 insertions(+), 150 deletions(-)
 delete mode 100644 llvm/test/CodeGen/SPARC/multiply-extension.ll

diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 0ad261135651f..098e5f22834f4 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1854,10 +1854,8 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
   if (Subtarget->is64Bit()) {
     setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
     setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
-    setOperationAction(ISD::MULHU, MVT::i64,
-                       Subtarget->isVIS3() ? Legal : Expand);
-    setOperationAction(ISD::MULHS, MVT::i64,
-                       Subtarget->isVIS3() ? Legal : Expand);
+    setOperationAction(ISD::MULHU,     MVT::i64, Expand);
+    setOperationAction(ISD::MULHS,     MVT::i64, Expand);
 
     setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
     setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
diff --git a/llvm/lib/Target/Sparc/SparcInstrVIS.td b/llvm/lib/Target/Sparc/SparcInstrVIS.td
index d9fe3b49821e5..ee24d8a54fe8e 100644
--- a/llvm/lib/Target/Sparc/SparcInstrVIS.td
+++ b/llvm/lib/Target/Sparc/SparcInstrVIS.td
@@ -294,14 +294,4 @@ def : Pat<(f32 fpnegimm0), (FNEGS (FZEROS))>;
 // VIS3 instruction patterns.
 let Predicates = [HasVIS3] in {
 def : Pat<(i64 (adde i64:$lhs, i64:$rhs)), (ADDXCCC $lhs, $rhs)>;
-
-def : Pat<(i64 (mulhu i64:$lhs, i64:$rhs)), (UMULXHI $lhs, $rhs)>;
-// Signed "MULXHI".
-// Based on the formula presented in OSA2011 §7.140, but with bitops to select
-// the values to be added.
-// TODO: This expansion should probably be moved to DAG legalization phase.
-def : Pat<(i64 (mulhs i64:$lhs, i64:$rhs)),
-      (SUBrr (UMULXHI $lhs, $rhs),
-             (ADDrr (ANDrr (SRAXri $lhs, 63), $rhs),
-                    (ANDrr (SRAXri $rhs, 63), $lhs)))>;
 } // Predicates = [HasVIS3]
diff --git a/llvm/test/CodeGen/SPARC/multiply-extension.ll b/llvm/test/CodeGen/SPARC/multiply-extension.ll
deleted file mode 100644
index 4d752ff101ca2..0000000000000
--- a/llvm/test/CodeGen/SPARC/multiply-extension.ll
+++ /dev/null
@@ -1,59 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=sparcv9 | FileCheck %s -check-prefix=V9
-; RUN: llc < %s -mtriple=sparcv9 -mattr=+vis3 | FileCheck %s -check-prefix=VIS3
-
-define i128 @signed_multiply_extend(i64 %0, i64 %1) nounwind {
-; V9-LABEL: signed_multiply_extend:
-; V9:       ! %bb.0:
-; V9-NEXT:    save %sp, -176, %sp
-; V9-NEXT:    srax %i0, 63, %o2
-; V9-NEXT:    srax %i1, 63, %o0
-; V9-NEXT:    mov %i1, %o1
-; V9-NEXT:    call __multi3
-; V9-NEXT:    mov %i0, %o3
-; V9-NEXT:    mov %o0, %i0
-; V9-NEXT:    ret
-; V9-NEXT:    restore %g0, %o1, %o1
-;
-; VIS3-LABEL: signed_multiply_extend:
-; VIS3:       ! %bb.0:
-; VIS3-NEXT:    srax %o0, 63, %o2
-; VIS3-NEXT:    and %o2, %o1, %o2
-; VIS3-NEXT:    srax %o1, 63, %o3
-; VIS3-NEXT:    and %o3, %o0, %o3
-; VIS3-NEXT:    add %o3, %o2, %o2
-; VIS3-NEXT:    umulxhi %o1, %o0, %o3
-; VIS3-NEXT:    sub %o3, %o2, %o2
-; VIS3-NEXT:    mulx %o1, %o0, %o1
-; VIS3-NEXT:    retl
-; VIS3-NEXT:    mov %o2, %o0
-  %3 = sext i64 %0 to i128
-  %4 = sext i64 %1 to i128
-  %5 = mul nsw i128 %4, %3
-  ret i128 %5
-}
-
-define i128 @unsigned_multiply_extend(i64 %0, i64 %1) nounwind {
-; V9-LABEL: unsigned_multiply_extend:
-; V9:       ! %bb.0:
-; V9-NEXT:    save %sp, -176, %sp
-; V9-NEXT:    mov %g0, %o0
-; V9-NEXT:    mov %i1, %o1
-; V9-NEXT:    mov %g0, %o2
-; V9-NEXT:    call __multi3
-; V9-NEXT:    mov %i0, %o3
-; V9-NEXT:    mov %o0, %i0
-; V9-NEXT:    ret
-; V9-NEXT:    restore %g0, %o1, %o1
-;
-; VIS3-LABEL: unsigned_multiply_extend:
-; VIS3:       ! %bb.0:
-; VIS3-NEXT:    umulxhi %o1, %o0, %o2
-; VIS3-NEXT:    mulx %o1, %o0, %o1
-; VIS3-NEXT:    retl
-; VIS3-NEXT:    mov %o2, %o0
-  %3 = zext i64 %0 to i128
-  %4 = zext i64 %1 to i128
-  %5 = mul nuw i128 %4, %3
-  ret i128 %5
-}
diff --git a/llvm/test/CodeGen/SPARC/smulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/SPARC/smulo-128-legalisation-lowering.ll
index 1e5ab7922de08..07e4c408a3ff0 100644
--- a/llvm/test/CodeGen/SPARC/smulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/SPARC/smulo-128-legalisation-lowering.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=sparc-unknown-linux-gnu | FileCheck %s --check-prefixes=SPARC
 ; RUN: llc < %s -mtriple=sparc64-unknown-linux-gnu | FileCheck %s --check-prefixes=SPARC64
-; RUN: llc < %s -mtriple=sparc64-unknown-linux-gnu -mattr=vis3 | FileCheck %s --check-prefixes=SPARC64-VIS3
 
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) nounwind {
 ; SPARC-LABEL: muloti_test:
@@ -214,49 +213,6 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) nounwind {
 ; SPARC64-NEXT:    srl %i3, 0, %i2
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore
-;
-; SPARC64-VIS3-LABEL: muloti_test:
-; SPARC64-VIS3:         .register %g2, #scratch
-; SPARC64-VIS3-NEXT:    .register %g3, #scratch
-; SPARC64-VIS3-NEXT:  ! %bb.0: ! %start
-; SPARC64-VIS3-NEXT:    save %sp, -128, %sp
-; SPARC64-VIS3-NEXT:    mov %g0, %i5
-; SPARC64-VIS3-NEXT:    umulxhi %i0, %i3, %i4
-; SPARC64-VIS3-NEXT:    srax %i0, 63, %g2
-; SPARC64-VIS3-NEXT:    mulx %g2, %i3, %g3
-; SPARC64-VIS3-NEXT:    add %i4, %g3, %i4
-; SPARC64-VIS3-NEXT:    umulxhi %i1, %i3, %g3
-; SPARC64-VIS3-NEXT:    mulx %i0, %i3, %g4
-; SPARC64-VIS3-NEXT:    addcc %g4, %g3, %g3
-; SPARC64-VIS3-NEXT:    addxccc %i4, %g0, %g4
-; SPARC64-VIS3-NEXT:    umulxhi %i1, %i2, %i4
-; SPARC64-VIS3-NEXT:    srax %i2, 63, %g5
-; SPARC64-VIS3-NEXT:    mulx %i1, %g5, %l0
-; SPARC64-VIS3-NEXT:    add %i4, %l0, %l0
-; SPARC64-VIS3-NEXT:    mulx %i1, %i2, %i4
-; SPARC64-VIS3-NEXT:    addcc %i4, %g3, %i4
-; SPARC64-VIS3-NEXT:    addxccc %l0, %g0, %g3
-; SPARC64-VIS3-NEXT:    srax %g3, 63, %l0
-; SPARC64-VIS3-NEXT:    addcc %g4, %g3, %g3
-; SPARC64-VIS3-NEXT:    srax %g4, 63, %g4
-; SPARC64-VIS3-NEXT:    addxccc %g4, %l0, %g4
-; SPARC64-VIS3-NEXT:    and %g5, %i0, %g5
-; SPARC64-VIS3-NEXT:    and %g2, %i2, %g2
-; SPARC64-VIS3-NEXT:    add %g2, %g5, %g2
-; SPARC64-VIS3-NEXT:    umulxhi %i0, %i2, %g5
-; SPARC64-VIS3-NEXT:    sub %g5, %g2, %g2
-; SPARC64-VIS3-NEXT:    mulx %i0, %i2, %i0
-; SPARC64-VIS3-NEXT:    addcc %i0, %g3, %i0
-; SPARC64-VIS3-NEXT:    addxccc %g2, %g4, %i2
-; SPARC64-VIS3-NEXT:    srax %i4, 63, %g2
-; SPARC64-VIS3-NEXT:    xor %i2, %g2, %i2
-; SPARC64-VIS3-NEXT:    xor %i0, %g2, %i0
-; SPARC64-VIS3-NEXT:    or %i0, %i2, %i0
-; SPARC64-VIS3-NEXT:    movrnz %i0, 1, %i5
-; SPARC64-VIS3-NEXT:    mulx %i1, %i3, %i1
-; SPARC64-VIS3-NEXT:    srl %i5, 0, %i2
-; SPARC64-VIS3-NEXT:    ret
-; SPARC64-VIS3-NEXT:    restore %g0, %i4, %o0
 start:
   %0 = tail call { i128, i1 } @llvm.smul.with.overflow.i128(i128 %l, i128 %r)
   %1 = extractvalue { i128, i1 } %0, 0
diff --git a/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll
index 6d197c88bfecd..f3835790210a0 100644
--- a/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=sparc-unknown-linux-gnu | FileCheck %s --check-prefixes=SPARC
 ; RUN: llc < %s -mtriple=sparc64-unknown-linux-gnu | FileCheck %s --check-prefixes=SPARC64
-; RUN: llc < %s -mtriple=sparc64-unknown-linux-gnu -mattr=vis3 | FileCheck %s --check-prefixes=SPARC64-VIS3
 
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) nounwind {
 ; SPARC-LABEL: muloti_test:
@@ -200,38 +199,6 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) nounwind {
 ; SPARC64-NEXT:    srl %i1, 0, %i2
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore %g0, %o1, %o1
-;
-; SPARC64-VIS3-LABEL: muloti_test:
-; SPARC64-VIS3:         .register %g2, #scratch
-; SPARC64-VIS3-NEXT:    .register %g3, #scratch
-; SPARC64-VIS3-NEXT:  ! %bb.0: ! %start
-; SPARC64-VIS3-NEXT:    save %sp, -128, %sp
-; SPARC64-VIS3-NEXT:    mov %g0, %i5
-; SPARC64-VIS3-NEXT:    mov %g0, %g2
-; SPARC64-VIS3-NEXT:    mov %g0, %g3
-; SPARC64-VIS3-NEXT:    mov %g0, %g4
-; SPARC64-VIS3-NEXT:    mov %g0, %g5
-; SPARC64-VIS3-NEXT:    mulx %i2, %i1, %i4
-; SPARC64-VIS3-NEXT:    mulx %i0, %i3, %l0
-; SPARC64-VIS3-NEXT:    add %l0, %i4, %i4
-; SPARC64-VIS3-NEXT:    umulxhi %i1, %i3, %l0
-; SPARC64-VIS3-NEXT:    add %l0, %i4, %i4
-; SPARC64-VIS3-NEXT:    cmp %i4, %l0
-; SPARC64-VIS3-NEXT:    movrnz %i2, 1, %g2
-; SPARC64-VIS3-NEXT:    movrnz %i0, 1, %g3
-; SPARC64-VIS3-NEXT:    and %g3, %g2, %g2
-; SPARC64-VIS3-NEXT:    umulxhi %i0, %i3, %i0
-; SPARC64-VIS3-NEXT:    movrnz %i0, 1, %g4
-; SPARC64-VIS3-NEXT:    movcs %xcc, 1, %i5
-; SPARC64-VIS3-NEXT:    or %g2, %g4, %i0
-; SPARC64-VIS3-NEXT:    umulxhi %i2, %i1, %i2
-; SPARC64-VIS3-NEXT:    movrnz %i2, 1, %g5
-; SPARC64-VIS3-NEXT:    or %i0, %g5, %i0
-; SPARC64-VIS3-NEXT:    or %i0, %i5, %i0
-; SPARC64-VIS3-NEXT:    mulx %i1, %i3, %i1
-; SPARC64-VIS3-NEXT:    srl %i0, 0, %i2
-; SPARC64-VIS3-NEXT:    ret
-; SPARC64-VIS3-NEXT:    restore %g0, %i4, %o0
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r)
   %1 = extractvalue { i128, i1 } %0, 0

From bed03ae36600f83f214c41af333f47fe8ead9ede Mon Sep 17 00:00:00 2001
From: Feng Zou <feng.zou@intel.com>
Date: Wed, 16 Apr 2025 10:47:05 +0800
Subject: [PATCH 069/710] [X86] Fix the issue of creating index reg negations
 (#135632)

The 8 and 16 bit LEA instruction support was added by PR #122102, and we
have to update creating index register negations accordingly. The issue
is exposed with APX NDD instructions.
---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp       | 19 ++++-
 .../CodeGen/X86/apx/ndd-neg-addr-index.ll     | 71 +++++++++++++++++++
 2 files changed, 88 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index d322e70fc0c20..01118beb9cf5e 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -275,8 +275,23 @@ namespace {
 #define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
       // Negate the index if needed.
       if (AM.NegateIndex) {
-        unsigned NegOpc = VT == MVT::i64 ? GET_ND_IF_ENABLED(X86::NEG64r)
-                                         : GET_ND_IF_ENABLED(X86::NEG32r);
+        unsigned NegOpc;
+        switch (VT.SimpleTy) {
+        default:
+          llvm_unreachable("Unsupported VT!");
+        case MVT::i64:
+          NegOpc = GET_ND_IF_ENABLED(X86::NEG64r);
+          break;
+        case MVT::i32:
+          NegOpc = GET_ND_IF_ENABLED(X86::NEG32r);
+          break;
+        case MVT::i16:
+          NegOpc = GET_ND_IF_ENABLED(X86::NEG16r);
+          break;
+        case MVT::i8:
+          NegOpc = GET_ND_IF_ENABLED(X86::NEG8r);
+          break;
+        }
         SDValue Neg = SDValue(CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32,
                                                      AM.IndexReg), 0);
         AM.IndexReg = Neg;
diff --git a/llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll b/llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll
new file mode 100644
index 0000000000000..6679b5f58e8c1
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/ndd-neg-addr-index.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc %s -mtriple=x86_64-unknown -mattr=+ndd -verify-machineinstrs --show-mc-encoding -o - | FileCheck %s --check-prefix=NDD
+
+
+define void @neg_8bit_1(i1 %cmp) {
+; NDD-LABEL: neg_8bit_1:
+; NDD:       # %bb.0: # %entry
+; NDD-NEXT:    andb $1, %dil, %al # encoding: [0x62,0xf4,0x7c,0x18,0x80,0xe7,0x01]
+; NDD-NEXT:    movzbl 0, %ecx # encoding: [0x0f,0xb6,0x0c,0x25,0x00,0x00,0x00,0x00]
+; NDD-NEXT:    negb %al, %al # encoding: [0x62,0xf4,0x7c,0x18,0xf6,0xd8]
+; NDD-NEXT:    leab 2(%rcx,%rax), %al # encoding: [0x66,0x8d,0x44,0x01,0x02]
+; NDD-NEXT:    movb %al, 0 # encoding: [0x88,0x04,0x25,0x00,0x00,0x00,0x00]
+; NDD-NEXT:    retq # encoding: [0xc3]
+entry:
+  %cond = select i1 %cmp, i8 1, i8 2
+  %0 = load i8, ptr null, align 4
+  %add = add i8 %cond, %0
+  store i8 %add, ptr null, align 4
+  ret void
+}
+
+define void @neg_8bit_2(i8 %int8) {
+; NDD-LABEL: neg_8bit_2:
+; NDD:       # %bb.0: # %entry
+; NDD-NEXT:    # kill: def $edi killed $edi def $rdi
+; NDD-NEXT:    addb %dil, %dil, %al # encoding: [0x62,0xf4,0x7c,0x18,0x00,0xff]
+; NDD-NEXT:    negb %al, %al # encoding: [0x62,0xf4,0x7c,0x18,0xf6,0xd8]
+; NDD-NEXT:    leab 1(%rdi,%rax), %al # encoding: [0x66,0x8d,0x44,0x07,0x01]
+; NDD-NEXT:    mulb %dil # encoding: [0x40,0xf6,0xe7]
+; NDD-NEXT:    testb %al, %al # encoding: [0x84,0xc0]
+; NDD-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = shl i8 %int8, 1
+  %sub = sub i8 %int8, %0
+  %add = add i8 %sub, 1
+  %div = mul i8 %add, %int8
+  %cmp = icmp slt i8 %div, 0
+  br i1 %cmp, label %label2, label %label1
+
+label1:                             ; preds = %entry
+  ret void
+
+label2:                             ; preds = %entry
+  ret void
+}
+
+define i32 @neg_16bit(i16 %0) {
+; NDD-LABEL: neg_16bit:
+; NDD:       # %bb.0: # %entry
+; NDD-NEXT:    # kill: def $edi killed $edi def $rdi
+; NDD-NEXT:    incw %di, %ax # encoding: [0x62,0xf4,0x7d,0x18,0xff,0xc7]
+; NDD-NEXT:    addw $256, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x81,0xc7,0x00,0x01]
+; NDD-NEXT:    # imm = 0x100
+; NDD-NEXT:    testw %ax, %ax # encoding: [0x66,0x85,0xc0]
+; NDD-NEXT:    cmovsl %ecx, %eax # EVEX TO LEGACY Compression encoding: [0x0f,0x48,0xc1]
+; NDD-NEXT:    andw $-256, %ax # EVEX TO LEGACY Compression encoding: [0x66,0x25,0x00,0xff]
+; NDD-NEXT:    negw %ax, %ax # encoding: [0x62,0xf4,0x7d,0x18,0xf7,0xd8]
+; NDD-NEXT:    leaw 1(%rdi,%rax), %ax # encoding: [0x66,0x8d,0x44,0x07,0x01]
+; NDD-NEXT:    movzwl %ax, %eax # encoding: [0x0f,0xb7,0xc0]
+; NDD-NEXT:    movq %rax, 0 # encoding: [0x48,0x89,0x04,0x25,0x00,0x00,0x00,0x00]
+; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
+; NDD-NEXT:    retq # encoding: [0xc3]
+entry:
+  %add = add i16 %0, 1
+  %rem = srem i16 %add, 256
+  %1 = zext i16 %rem to i19
+  %2 = sext i19 %1 to i64
+  %3 = getelementptr i8, ptr null, i64 %2
+  store ptr %3, ptr null, align 4
+  ret i32 0
+}

From e676866368a84c88aad90e138268e00a1c56a230 Mon Sep 17 00:00:00 2001
From: yingopq <115543042+yingopq@users.noreply.github.com>
Date: Wed, 16 Apr 2025 10:56:06 +0800
Subject: [PATCH 070/710] [Mips] Fix clang crashes when compiling a variadic
 function while targeting mips3 (#130558)

issue reason:
Because mips3 has the feature 'FeatureGP64Bit', when target mips3
process function `writeVarArgRegs`, the result of `getGPRSizeInBytes` is
8 and the result of `GetVarArgRegs` is `Mips::A0, Mips::A1, Mips::A2,
Mips::A3`. This would generate `gpr64 = COPY $a1` which should be `gpr64
= COPY $a1_64`.

Also when process `CC_Mips_FixedArg`, mips would CCDelegateTo
    `CC_MipsO32_FP`. In fact, it should CCDelegateTo `CC_MipsN`.

Fix #98716.
---
 .../Target/Mips/MCTargetDesc/MipsABIInfo.cpp  | 10 ++--
 .../Target/Mips/MCTargetDesc/MipsABIInfo.h    |  2 +-
 llvm/lib/Target/Mips/MipsCallLowering.cpp     |  3 +-
 llvm/lib/Target/Mips/MipsCallingConv.td       |  2 +-
 llvm/lib/Target/Mips/MipsISelLowering.cpp     |  3 +-
 llvm/test/CodeGen/Mips/vararg.ll              | 54 +++++++++++++++++++
 6 files changed, 66 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/CodeGen/Mips/vararg.ll

diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
index 08cbba952ccc8..1be29cf3c94b9 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
@@ -37,9 +37,13 @@ ArrayRef<MCPhysReg> MipsABIInfo::GetByValArgRegs() const {
   llvm_unreachable("Unhandled ABI");
 }
 
-ArrayRef<MCPhysReg> MipsABIInfo::GetVarArgRegs() const {
-  if (IsO32())
-    return ArrayRef(O32IntRegs);
+ArrayRef<MCPhysReg> MipsABIInfo::getVarArgRegs(bool isGP64bit) const {
+  if (IsO32()) {
+    if (isGP64bit)
+      return ArrayRef(Mips64IntRegs);
+    else
+      return ArrayRef(O32IntRegs);
+  }
   if (IsN32() || IsN64())
     return ArrayRef(Mips64IntRegs);
   llvm_unreachable("Unhandled ABI");
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
index 41f80771142de..44b023c7c3ef6 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
@@ -46,7 +46,7 @@ class MipsABIInfo {
   ArrayRef<MCPhysReg> GetByValArgRegs() const;
 
   /// The registers to use for the variable argument list.
-  ArrayRef<MCPhysReg> GetVarArgRegs() const;
+  ArrayRef<MCPhysReg> getVarArgRegs(bool isGP64bit) const;
 
   /// Obtain the size of the area allocated by the callee for arguments.
   /// CallingConv::FastCall affects the value for O32.
diff --git a/llvm/lib/Target/Mips/MipsCallLowering.cpp b/llvm/lib/Target/Mips/MipsCallLowering.cpp
index b856290211277..01c9d0b38323e 100644
--- a/llvm/lib/Target/Mips/MipsCallLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsCallLowering.cpp
@@ -406,7 +406,8 @@ bool MipsCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
     return false;
 
   if (F.isVarArg()) {
-    ArrayRef<MCPhysReg> ArgRegs = ABI.GetVarArgRegs();
+    ArrayRef<MCPhysReg> ArgRegs =
+        ABI.getVarArgRegs(MF.getSubtarget<MipsSubtarget>().isGP64bit());
     unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
 
     int VaArgOffset;
diff --git a/llvm/lib/Target/Mips/MipsCallingConv.td b/llvm/lib/Target/Mips/MipsCallingConv.td
index 25384a3fe8de3..3c60114f507b9 100644
--- a/llvm/lib/Target/Mips/MipsCallingConv.td
+++ b/llvm/lib/Target/Mips/MipsCallingConv.td
@@ -339,7 +339,7 @@ def CC_Mips_FixedArg : CallingConv<[
 
   CCIfCC<"CallingConv::Fast", CCDelegateTo<CC_Mips_FastCC>>,
 
-  CCIfSubtarget<"isABI_O32()", CCDelegateTo<CC_MipsO32_FP>>,
+  CCIfSubtarget<"isABI_O32()", CCIfSubtargetNot<"isGP64bit()", CCDelegateTo<CC_MipsO32_FP>>>,
   CCDelegateTo<CC_MipsN>
 ]>;
 
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index fa8a4704730cf..55fc636d3c781 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -3400,7 +3400,6 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   SDValue StackPtr =
       DAG.getCopyFromReg(Chain, DL, ABI.IsN64() ? Mips::SP_64 : Mips::SP,
                          getPointerTy(DAG.getDataLayout()));
-
   std::deque<std::pair<unsigned, SDValue>> RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
 
@@ -4654,7 +4653,7 @@ void MipsTargetLowering::writeVarArgRegs(std::vector<SDValue> &OutChains,
                                          SDValue Chain, const SDLoc &DL,
                                          SelectionDAG &DAG,
                                          CCState &State) const {
-  ArrayRef<MCPhysReg> ArgRegs = ABI.GetVarArgRegs();
+  ArrayRef<MCPhysReg> ArgRegs = ABI.getVarArgRegs(Subtarget.isGP64bit());
   unsigned Idx = State.getFirstUnallocated(ArgRegs);
   unsigned RegSizeInBytes = Subtarget.getGPRSizeInBytes();
   MVT RegTy = MVT::getIntegerVT(RegSizeInBytes * 8);
diff --git a/llvm/test/CodeGen/Mips/vararg.ll b/llvm/test/CodeGen/Mips/vararg.ll
new file mode 100644
index 0000000000000..ed4a805af0c0c
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/vararg.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=mipsel-linux-gnu -mcpu=mips3 -target-abi o32 < %s | FileCheck %s -check-prefixes=MIPS3-O32
+; RUN: llc -mtriple=mipsel-linux-gnu -mcpu=mips3 -target-abi n32 < %s | FileCheck %s -check-prefixes=MIPS3-N32
+; RUN: llc -mtriple=mipsel-linux-gnu -mcpu=mips3 -target-abi n64 < %s | FileCheck %s -check-prefixes=MIPS3-N64
+
+define void @func(ptr nocapture %x, ...) nounwind {
+; MIPS3-O32-LABEL: func:
+; MIPS32-O32:       # %bb.0: # %entry
+; MIPS32-O32-NEXT:    addiu $sp, $sp, -48
+; MIPS32-O32-NEXT:    sd $11, 56($sp)
+; MIPS32-O32-NEXT:    sd $10, 48($sp)
+; MIPS32-O32-NEXT:    sd $9, 40($sp)
+; MIPS32-O32-NEXT:    sd $8, 32($sp)
+; MIPS32-O32-NEXT:    sd $7, 24($sp)
+; MIPS32-O32-NEXT:    sd $6, 16($sp)
+; MIPS32-O32-NEXT:    sd $5, 8($sp)
+; MIPS32-O32-NEXT:    sw $4, 4($sp)
+; MIPS32-O32-NEXT:    jr $ra
+; MIPS32-O32-NEXT:    addiu $sp, $sp, 48
+;
+; MIPS3-N32-LABEL: func:
+; MIPS32-N32:       # %bb.0: # %entry
+; MIPS32-N32-NEXT:    addiu $sp, $sp, -64
+; MIPS32-N32-NEXT:    sd $11, 56($sp)
+; MIPS32-N32-NEXT:    sd $10, 48($sp)
+; MIPS32-N32-NEXT:    sd $9, 40($sp)
+; MIPS32-N32-NEXT:    sd $8, 32($sp)
+; MIPS32-N32-NEXT:    sd $7, 24($sp)
+; MIPS32-N32-NEXT:    sd $6, 16($sp)
+; MIPS32-N32-NEXT:    sd $5, 8($sp)
+; MIPS32-N32-NEXT:    sw $4, 4($sp)
+; MIPS32-N32-NEXT:    jr $ra
+; MIPS32-N32-NEXT:    addiu $sp, $sp, 64
+;
+; MIPS3-N64-LABEL: func:
+; MIPS32-N64:       # %bb.0: # %entry
+; MIPS32-N64-NEXT:    addiu $sp, $sp, -64
+; MIPS32-N64-NEXT:    sdl $4, 7($sp)
+; MIPS32-N64-NEXT:    sd $11, 56($sp)
+; MIPS32-N64-NEXT:    sd $10, 48($sp)
+; MIPS32-N64-NEXT:    sd $9, 40($sp)
+; MIPS32-N64-NEXT:    sd $8, 32($sp)
+; MIPS32-N64-NEXT:    sd $7, 24($sp)
+; MIPS32-N64-NEXT:    sd $6, 16($sp)
+; MIPS32-N64-NEXT:    sd $5, 8($sp)
+; MIPS32-N64-NEXT:    sw $4, 4($sp)
+; MIPS32-N64-NEXT:    jr $ra
+; MIPS32-N64-NEXT:    addiu $sp, $sp, 64
+
+entry:
+  %x.addr = alloca ptr, align 4
+  store ptr %x, ptr %x.addr, align 4
+  ret void
+}

From 517605c20e6014543e91d45524a17c443aa11bd4 Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Tue, 15 Apr 2025 20:00:51 -0700
Subject: [PATCH 071/710] [alpha.webkit.UnretainedCallArgsChecker] Add the
 support for RetainPtrArc (#135532)

WebKit uses #define to rename RetainPtr to RetainPtrArc so add the
support for it.
---
 .../Checkers/WebKit/PtrTypesSemantics.cpp         | 15 +++++++++------
 .../Checkers/WebKit/RetainPtrCtorAdoptChecker.cpp |  8 ++++----
 .../Analysis/Checkers/WebKit/objc-mock-types.h    |  5 +++++
 .../Checkers/WebKit/unretained-call-args-arc.mm   | 11 +++++++++++
 .../Checkers/WebKit/unretained-call-args.mm       | 11 +++++++++++
 5 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 134afcd124526..811888e119449 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -119,7 +119,9 @@ bool isRefType(const std::string &Name) {
          Name == "RefPtr" || Name == "RefPtrAllowingPartiallyDestroyed";
 }
 
-bool isRetainPtr(const std::string &Name) { return Name == "RetainPtr"; }
+bool isRetainPtr(const std::string &Name) {
+  return Name == "RetainPtr" || Name == "RetainPtrArc";
+}
 
 bool isCheckedPtr(const std::string &Name) {
   return Name == "CheckedPtr" || Name == "CheckedRef";
@@ -157,7 +159,8 @@ bool isCtorOfCheckedPtr(const clang::FunctionDecl *F) {
 bool isCtorOfRetainPtr(const clang::FunctionDecl *F) {
   const std::string &FunctionName = safeGetName(F);
   return FunctionName == "RetainPtr" || FunctionName == "adoptNS" ||
-         FunctionName == "adoptCF" || FunctionName == "retainPtr";
+         FunctionName == "adoptCF" || FunctionName == "retainPtr" ||
+         FunctionName == "RetainPtrArc" || FunctionName == "adoptNSArc";
 }
 
 bool isCtorOfSafePtr(const clang::FunctionDecl *F) {
@@ -190,7 +193,7 @@ bool isRefOrCheckedPtrType(const clang::QualType T) {
 }
 
 bool isRetainPtrType(const clang::QualType T) {
-  return isPtrOfType(T, [](auto Name) { return Name == "RetainPtr"; });
+  return isPtrOfType(T, [](auto Name) { return isRetainPtr(Name); });
 }
 
 bool isOwnerPtrType(const clang::QualType T) {
@@ -374,7 +377,7 @@ std::optional<bool> isGetterOfSafePtr(const CXXMethodDecl *M) {
          method == "impl"))
       return true;
 
-    if (className == "RetainPtr" && method == "get")
+    if (isRetainPtr(className) && method == "get")
       return true;
 
     // Ref<T> -> T conversion
@@ -395,7 +398,7 @@ std::optional<bool> isGetterOfSafePtr(const CXXMethodDecl *M) {
       }
     }
 
-    if (className == "RetainPtr") {
+    if (isRetainPtr(className)) {
       if (auto *maybeRefToRawOperator = dyn_cast<CXXConversionDecl>(M)) {
         auto QT = maybeRefToRawOperator->getConversionType();
         auto *T = QT.getTypePtrOrNull();
@@ -429,7 +432,7 @@ bool isCheckedPtr(const CXXRecordDecl *R) {
 bool isRetainPtr(const CXXRecordDecl *R) {
   assert(R);
   if (auto *TmplR = R->getTemplateInstantiationPattern())
-    return safeGetName(TmplR) == "RetainPtr";
+    return isRetainPtr(safeGetName(TmplR));
   return false;
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/RetainPtrCtorAdoptChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RetainPtrCtorAdoptChecker.cpp
index d372c5d1ba626..d3eee11311d91 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/RetainPtrCtorAdoptChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RetainPtrCtorAdoptChecker.cpp
@@ -71,7 +71,7 @@ class RetainPtrCtorAdoptChecker
       }
 
       bool TraverseClassTemplateDecl(ClassTemplateDecl *CTD) {
-        if (safeGetName(CTD) == "RetainPtr")
+        if (isRetainPtr(safeGetName(CTD)))
           return true; // Skip the contents of RetainPtr.
         return Base::TraverseClassTemplateDecl(CTD);
       }
@@ -193,7 +193,7 @@ class RetainPtrCtorAdoptChecker
     if (!Cls)
       return;
 
-    if (safeGetName(Cls) != "RetainPtr" || !CE->getNumArgs())
+    if (!isRetainPtr(safeGetName(Cls)) || !CE->getNumArgs())
       return;
 
     // Ignore RetainPtr construction inside adoptNS, adoptCF, and retainPtr.
@@ -322,12 +322,12 @@ class RetainPtrCtorAdoptChecker
           if (auto *CD = dyn_cast<CXXConversionDecl>(MD)) {
             auto QT = CD->getConversionType().getCanonicalType();
             auto *ResultType = QT.getTypePtrOrNull();
-            if (safeGetName(Cls) == "RetainPtr" && ResultType &&
+            if (isRetainPtr(safeGetName(Cls)) && ResultType &&
                 (ResultType->isPointerType() || ResultType->isReferenceType() ||
                  ResultType->isObjCObjectPointerType()))
               return IsOwnedResult::NotOwned;
           }
-          if (safeGetName(MD) == "leakRef" && safeGetName(Cls) == "RetainPtr")
+          if (safeGetName(MD) == "leakRef" && isRetainPtr(safeGetName(Cls)))
             return IsOwnedResult::Owned;
         }
       }
diff --git a/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h b/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
index 51de81ac0f033..a4332df682060 100644
--- a/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
+++ b/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
@@ -17,6 +17,7 @@ typedef const struct CF_BRIDGED_TYPE(NSString) __CFString * CFStringRef;
 typedef const struct CF_BRIDGED_TYPE(NSArray) __CFArray * CFArrayRef;
 typedef struct CF_BRIDGED_MUTABLE_TYPE(NSMutableArray) __CFArray * CFMutableArrayRef;
 typedef struct CF_BRIDGED_MUTABLE_TYPE(CFRunLoopRef) __CFRunLoop * CFRunLoopRef;
+typedef struct CF_BRIDGED_TYPE(id) CGImage *CGImageRef;
 
 #define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
 #define CF_CONSUMED __attribute__((cf_consumed))
@@ -150,6 +151,10 @@ namespace WTF {
 
 void WTFCrash(void);
 
+#if __has_feature(objc_arc)
+#define RetainPtr RetainPtrArc
+#endif
+
 template<typename T> class RetainPtr;
 template<typename T> RetainPtr<T> adoptNS(T*);
 template<typename T> RetainPtr<T> adoptCF(T);
diff --git a/clang/test/Analysis/Checkers/WebKit/unretained-call-args-arc.mm b/clang/test/Analysis/Checkers/WebKit/unretained-call-args-arc.mm
index f1f4d912663aa..4207c1836079f 100644
--- a/clang/test/Analysis/Checkers/WebKit/unretained-call-args-arc.mm
+++ b/clang/test/Analysis/Checkers/WebKit/unretained-call-args-arc.mm
@@ -5,6 +5,8 @@
 SomeObj *provide();
 CFMutableArrayRef provide_cf();
 void someFunction();
+CGImageRef provideImage();
+NSString *stringForImage(CGImageRef);
 
 namespace raw_ptr {
 
@@ -36,4 +38,13 @@ - (SomeObj *)getSomeObj {
 - (void)doWorkOnSomeObj {
     [[self getSomeObj] doWork];
 }
+
+- (CGImageRef)createImage {
+  return provideImage();
+}
+
+- (NSString *)convertImage {
+  RetainPtr<CGImageRef> image = [self createImage];
+  return stringForImage(image.get());
+}
 @end
diff --git a/clang/test/Analysis/Checkers/WebKit/unretained-call-args.mm b/clang/test/Analysis/Checkers/WebKit/unretained-call-args.mm
index 0667e4964f1a8..eb36b49313d42 100644
--- a/clang/test/Analysis/Checkers/WebKit/unretained-call-args.mm
+++ b/clang/test/Analysis/Checkers/WebKit/unretained-call-args.mm
@@ -9,6 +9,9 @@
 CFMutableArrayRef provide_cf();
 void consume_cf(CFMutableArrayRef);
 
+CGImageRef provideImage();
+NSString *stringForImage(CGImageRef);
+
 void some_function();
 
 namespace simple {
@@ -440,4 +443,12 @@ - (void)doWorkOnSomeObj {
     [[self getSomeObj] doWork];
 }
 
+- (CGImageRef)createImage {
+  return provideImage();
+}
+
+- (NSString *)convertImage {
+  RetainPtr<CGImageRef> image = [self createImage];
+  return stringForImage(image.get());
+}
 @end

From 71d091699f956c89135bc165165e815ab7876359 Mon Sep 17 00:00:00 2001
From: Alexey Samsonov <vonosmas@gmail.com>
Date: Tue, 15 Apr 2025 20:12:17 -0700
Subject: [PATCH 072/710] [libc][bazel] Fold "libc_function_deps" into "deps"
 for libc_tests. (#135835)

libc_function_deps and deps are now identical, as we no longer need or
have special treatment for libc_function targets. Merge these attributes
passed to the libc_test macro, and fix all relevant libc_test macro
invocations. This change is a no-op.

This concludes cleanup started in
9b13d345303d819bb83de7ebbeb826d704add0bc.
---
 .../libc/test/libc_test_rules.bzl             |   7 +-
 .../libc/test/src/complex/BUILD.bazel         |  12 +--
 .../libc/test/src/fenv/BUILD.bazel            |  52 +++------
 .../libc/test/src/inttypes/BUILD.bazel        |   6 +-
 .../test/src/math/libc_math_test_rules.bzl    |   3 +-
 .../libc/test/src/stdbit/BUILD.bazel          |  11 +-
 .../libc/test/src/stdio/BUILD.bazel           |  46 ++++----
 .../libc/test/src/stdlib/BUILD.bazel          | 100 +++++++++++-------
 .../libc/test/src/string/BUILD.bazel          |  48 ++++-----
 .../libc/test/src/strings/BUILD.bazel         |  12 +--
 .../libc/test/src/sys/epoll/BUILD.bazel       |  50 ++++-----
 .../libc/test/src/sys/socket/BUILD.bazel      |  42 +++-----
 .../libc/test/src/unistd/BUILD.bazel          | 100 +++++++-----------
 13 files changed, 208 insertions(+), 281 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
index 7e798429ef19b..123e05727aeff 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
@@ -15,14 +15,13 @@ When performing tests we make sure to always use the internal version.
 load("//libc:libc_build_rules.bzl", "libc_common_copts")
 load("//libc:libc_configure_options.bzl", "LIBC_CONFIGURE_OPTIONS")
 
-def libc_test(name, libc_function_deps = [], copts = [], deps = [], local_defines = [], **kwargs):
+def libc_test(name, copts = [], deps = [], local_defines = [], **kwargs):
     """Add target for a libc test.
 
     Args:
       name: Test target name
-      libc_function_deps: List of libc_function targets used by this test.
       copts: The list of options to add to the C++ compilation command.
-      deps: The list of other libraries to be linked in to the test target.
+      deps: The list of libc functions and libraries to be linked in.
       local_defines: The list of target local_defines if any.
       **kwargs: Attributes relevant for a cc_test.
     """
@@ -37,7 +36,7 @@ def libc_test(name, libc_function_deps = [], copts = [], deps = [], local_define
             "//libc:func_free",
             "//libc:func_malloc",
             "//libc:func_realloc",
-        ] + libc_function_deps + deps,
+        ] + deps,
         copts = copts + libc_common_copts(),
         linkstatic = 1,
         **kwargs
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/complex/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/complex/BUILD.bazel
index d0965bb2ee147..cc3a8d8b4b96a 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/complex/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/complex/BUILD.bazel
@@ -7,8 +7,7 @@ load("//libc/test:libc_test_rules.bzl", "libc_test")
             "CImagTest.h",
             func_name + "_test.cpp",
         ],
-        libc_function_deps = ["//libc:func_name".replace("func_name", func_name)],
-        deps = [
+        deps = ["//libc:func_name".replace("func_name", func_name)] + [
             "//libc:hdr_math_macros",
             "//libc/test/UnitTest:fp_test_helpers",
         ],
@@ -29,8 +28,7 @@ load("//libc/test:libc_test_rules.bzl", "libc_test")
             "ConjTest.h",
             func_name + "_test.cpp",
         ],
-        libc_function_deps = ["//libc:func_name".replace("func_name", func_name)],
-        deps = [
+        deps = ["//libc:func_name".replace("func_name", func_name)] + [
             "//libc:hdr_math_macros",
             "//libc/test/UnitTest:fp_test_helpers",
         ],
@@ -51,8 +49,7 @@ load("//libc/test:libc_test_rules.bzl", "libc_test")
             "CprojTest.h",
             func_name + "_test.cpp",
         ],
-        libc_function_deps = ["//libc:func_name".replace("func_name", func_name)],
-        deps = [
+        deps = ["//libc:func_name".replace("func_name", func_name)] + [
             "//libc:hdr_math_macros",
             "//libc/test/UnitTest:fp_test_helpers",
         ] + (["//libc/utils/MPCWrapper:mpc_wrapper"] if func_name == "cprojf" else []),
@@ -73,8 +70,7 @@ load("//libc/test:libc_test_rules.bzl", "libc_test")
             "CRealTest.h",
             func_name + "_test.cpp",
         ],
-        libc_function_deps = ["//libc:func_name".replace("func_name", func_name)],
-        deps = [
+        deps = ["//libc:func_name".replace("func_name", func_name)] + [
             "//libc:hdr_math_macros",
             "//libc/test/UnitTest:fp_test_helpers",
         ],
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel
index c6ae534b0f640..1af1a984db92a 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel
@@ -16,14 +16,12 @@ libc_test(
         "exception_status_test.cpp",
         "excepts.h",
     ],
-    libc_function_deps = [
+    deps = [
+        "//libc:__support_fputil_fenv_impl",
         "//libc:feclearexcept",
         "//libc:feraiseexcept",
         "//libc:fesetexcept",
         "//libc:fetestexcept",
-    ],
-    deps = [
-        "//libc:__support_fputil_fenv_impl",
         "//libc:hdr_fenv_macros",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
@@ -35,11 +33,9 @@ libc_test(
         "excepts.h",
         "rounding_mode_test.cpp",
     ],
-    libc_function_deps = [
+    deps = [
         "//libc:fegetround",
         "//libc:fesetround",
-    ],
-    deps = [
         "//libc:hdr_fenv_macros",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
@@ -51,16 +47,14 @@ libc_test(
         "enabled_exceptions_test.cpp",
         "excepts.h",
     ],
-    libc_function_deps = [
-        "//libc:feclearexcept",
-        "//libc:feraiseexcept",
-        "//libc:fetestexcept",
-    ],
     tags = ["nosan"],
     deps = [
         "//libc:__support_common",
         "//libc:__support_fputil_fenv_impl",
         "//libc:__support_macros_properties_architectures",
+        "//libc:feclearexcept",
+        "//libc:feraiseexcept",
+        "//libc:fetestexcept",
         "//libc:hdr_fenv_macros",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
@@ -72,14 +66,12 @@ libc_test(
         "excepts.h",
         "feholdexcept_test.cpp",
     ],
-    libc_function_deps = [
-        "//libc:feholdexcept",
-    ],
     tags = ["nosan"],
     deps = [
         "//libc:__support_common",
         "//libc:__support_fputil_fenv_impl",
         "//libc:__support_macros_properties_architectures",
+        "//libc:feholdexcept",
         "//libc:hdr_fenv_macros",
         "//libc:types_fenv_t",
         "//libc/test/UnitTest:fp_test_helpers",
@@ -92,13 +84,11 @@ libc_test(
         "exception_flags_test.cpp",
         "excepts.h",
     ],
-    libc_function_deps = [
+    deps = [
+        "//libc:__support_fputil_fenv_impl",
         "//libc:fegetexceptflag",
         "//libc:fesetexceptflag",
         "//libc:fetestexceptflag",
-    ],
-    deps = [
-        "//libc:__support_fputil_fenv_impl",
         "//libc:hdr_fenv_macros",
         "//libc:types_fexcept_t",
         "//libc/test/UnitTest:fp_test_helpers",
@@ -111,11 +101,9 @@ libc_test(
         "excepts.h",
         "feclearexcept_test.cpp",
     ],
-    libc_function_deps = [
-        "//libc:feclearexcept",
-    ],
     deps = [
         "//libc:__support_fputil_fenv_impl",
+        "//libc:feclearexcept",
         "//libc:hdr_fenv_macros",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
@@ -127,14 +115,12 @@ libc_test(
         "excepts.h",
         "feenableexcept_test.cpp",
     ],
-    libc_function_deps = [
-        "//libc:fedisableexcept",
-        "//libc:feenableexcept",
-        "//libc:fegetexcept",
-    ],
     deps = [
         "//libc:__support_common",
         "//libc:__support_macros_properties_architectures",
+        "//libc:fedisableexcept",
+        "//libc:feenableexcept",
+        "//libc:fegetexcept",
         "//libc:hdr_fenv_macros",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
@@ -146,11 +132,9 @@ libc_test(
         "excepts.h",
         "feupdateenv_test.cpp",
     ],
-    libc_function_deps = [
-        "//libc:feupdateenv",
-    ],
     deps = [
         "//libc:__support_fputil_fenv_impl",
+        "//libc:feupdateenv",
         "//libc:hdr_fenv_macros",
         "//libc:types_fenv_t",
         "//libc/test/UnitTest:fp_test_helpers",
@@ -163,15 +147,13 @@ libc_test(
         "excepts.h",
         "getenv_and_setenv_test.cpp",
     ],
-    libc_function_deps = [
+    deps = [
+        "//libc:__support_fputil_fenv_impl",
+        "//libc:__support_macros_properties_os",
         "//libc:fegetenv",
         "//libc:fegetround",
         "//libc:fesetenv",
         "//libc:fesetround",
-    ],
-    deps = [
-        "//libc:__support_fputil_fenv_impl",
-        "//libc:__support_macros_properties_os",
         "//libc:hdr_fenv_macros",
         "//libc:types_fenv_t",
         "//libc/test/UnitTest:fp_test_helpers",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/inttypes/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/inttypes/BUILD.bazel
index bda7245d1f677..3dd4ab379efe0 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/inttypes/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/inttypes/BUILD.bazel
@@ -13,7 +13,7 @@ licenses(["notice"])
 libc_test(
     name = "imaxabs_test",
     srcs = ["imaxabs_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:imaxabs",
     ],
 )
@@ -21,10 +21,8 @@ libc_test(
 libc_test(
     name = "imaxdiv_test",
     srcs = ["imaxdiv_test.cpp"],
-    libc_function_deps = [
-        "//libc:imaxdiv",
-    ],
     deps = [
+        "//libc:imaxdiv",
         "//libc/test/src/stdlib:div_test_helper",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl
index 16845ab66dfd4..d2297d51383cc 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/libc_math_test_rules.bzl
@@ -23,8 +23,7 @@ def math_test(name, hdrs = [], deps = [], **kwargs):
     libc_test(
         name = test_name,
         srcs = [test_name + ".cpp"] + hdrs,
-        libc_function_deps = ["//libc:func_name".replace("func_name", name)],
-        deps = [
+        deps = ["//libc:func_name".replace("func_name", name)] + [
             "//libc:__support_cpp_algorithm",
             "//libc:__support_cpp_bit",
             "//libc:__support_cpp_limits",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/stdbit/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/stdbit/BUILD.bazel
index b9d153947dc7d..119d7d5337750 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/stdbit/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/stdbit/BUILD.bazel
@@ -39,11 +39,12 @@ bit_prefix_list = [
     libc_test(
         name = bit_prefix + bit_suffix + "_test",
         srcs = [bit_prefix + bit_suffix + "_test.cpp"],
-        libc_function_deps = ["//libc:func_name".replace(
-            "func_name",
-            bit_prefix + bit_suffix,
-        )],
-        deps = ["//libc:__support_cpp_limits"],
+        deps = [
+            "//libc:func_name".replace(
+                "func_name",
+                bit_prefix + bit_suffix,
+            ),
+        ] + ["//libc:__support_cpp_limits"],
     )
     for bit_prefix in bit_prefix_list
     for bit_suffix in bit_suffix_list
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
index c3865ea07ea91..484d3e5e0a24e 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
@@ -13,8 +13,6 @@ licenses(["notice"])
 libc_test(
     name = "printf_parser_test",
     srcs = ["printf_core/parser_test.cpp"],
-    libc_function_deps = [
-    ],
     deps = [
         "//libc:__support_arg_list",
         "//libc:__support_cpp_bit",
@@ -28,8 +26,6 @@ libc_test(
 libc_test(
     name = "printf_writer_test",
     srcs = ["printf_core/writer_test.cpp"],
-    libc_function_deps = [
-    ],
     deps = [
         "//libc:__support_arg_list",
         "//libc:__support_cpp_string_view",
@@ -42,8 +38,6 @@ libc_test(
 libc_test(
     name = "printf_converter_test",
     srcs = ["printf_core/converter_test.cpp"],
-    libc_function_deps = [
-    ],
     deps = [
         "//libc:__support_arg_list",
         "//libc:__support_cpp_string_view",
@@ -56,11 +50,9 @@ libc_test(
 libc_test(
     name = "sprintf_test",
     srcs = ["sprintf_test.cpp"],
-    libc_function_deps = [
-        "//libc:sprintf",
-    ],
     deps = [
         "//libc:__support_fputil_fp_bits",
+        "//libc:sprintf",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
 )
@@ -68,7 +60,7 @@ libc_test(
 libc_test(
     name = "snprintf_test",
     srcs = ["snprintf_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:snprintf",
     ],
 )
@@ -76,7 +68,7 @@ libc_test(
 libc_test(
     name = "printf_test",
     srcs = ["printf_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:printf",
     ],
 )
@@ -84,7 +76,7 @@ libc_test(
 libc_test(
     name = "fprintf_test",
     srcs = ["fprintf_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:fprintf",
     ],
 )
@@ -92,7 +84,7 @@ libc_test(
 libc_test(
     name = "vsprintf_test",
     srcs = ["vsprintf_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:vsprintf",
     ],
 )
@@ -100,7 +92,7 @@ libc_test(
 libc_test(
     name = "vsnprintf_test",
     srcs = ["vsnprintf_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:vsnprintf",
     ],
 )
@@ -108,7 +100,7 @@ libc_test(
 libc_test(
     name = "vprintf_test",
     srcs = ["vprintf_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:vprintf",
     ],
 )
@@ -116,7 +108,7 @@ libc_test(
 libc_test(
     name = "vfprintf_test",
     srcs = ["vfprintf_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:vfprintf",
     ],
 )
@@ -124,25 +116,23 @@ libc_test(
 libc_test(
     name = "remove_test",
     srcs = ["remove_test.cpp"],
-    libc_function_deps = [
-        "//libc:remove",
-        "//libc:open",
-        "//libc:mkdirat",
+    deps = [
         "//libc:access",
         "//libc:close",
+        "//libc:mkdirat",
+        "//libc:open",
+        "//libc:remove",
     ],
 )
 
 libc_test(
     name = "sscanf_test",
     srcs = ["sscanf_test.cpp"],
-    libc_function_deps = [
-        "//libc:sscanf",
-    ],
     deps = [
         "//libc:__support_cpp_limits",
         "//libc:__support_fputil_fp_bits",
         "//libc:hdr_stdio_macros",
+        "//libc:sscanf",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
 )
@@ -150,16 +140,16 @@ libc_test(
 libc_test(
     name = "fscanf_test",
     srcs = ["fscanf_test.cpp"],
-    libc_function_deps = [
+    deps = [
+        "//libc:__support_cpp_string_view",
         "//libc:fscanf",
     ],
-    deps = ["//libc:__support_cpp_string_view"],
 )
 
 libc_test(
     name = "vsscanf_test",
     srcs = ["vsscanf_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:vsscanf",
     ],
 )
@@ -167,8 +157,8 @@ libc_test(
 libc_test(
     name = "vfscanf_test",
     srcs = ["vfscanf_test.cpp"],
-    libc_function_deps = [
+    deps = [
+        "//libc:__support_cpp_string_view",
         "//libc:vfscanf",
     ],
-    deps = ["//libc:__support_cpp_string_view"],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
index 5f43ec7c7a109..40f672d8099f1 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
@@ -13,19 +13,19 @@ licenses(["notice"])
 libc_test(
     name = "abs_test",
     srcs = ["abs_test.cpp"],
-    libc_function_deps = ["//libc:abs"],
+    deps = ["//libc:abs"],
 )
 
 libc_test(
     name = "labs_test",
     srcs = ["labs_test.cpp"],
-    libc_function_deps = ["//libc:labs"],
+    deps = ["//libc:labs"],
 )
 
 libc_test(
     name = "llabs_test",
     srcs = ["llabs_test.cpp"],
-    libc_function_deps = ["//libc:llabs"],
+    deps = ["//libc:llabs"],
 )
 
 libc_test_library(
@@ -37,9 +37,9 @@ libc_test_library(
 libc_test(
     name = "div_test",
     srcs = ["div_test.cpp"],
-    libc_function_deps = ["//libc:div"],
     deps = [
         ":div_test_helper",
+        "//libc:div",
         "//libc:types_div_t",
     ],
 )
@@ -47,9 +47,9 @@ libc_test(
 libc_test(
     name = "ldiv_test",
     srcs = ["ldiv_test.cpp"],
-    libc_function_deps = ["//libc:ldiv"],
     deps = [
         ":div_test_helper",
+        "//libc:ldiv",
         "//libc:types_ldiv_t",
     ],
 )
@@ -57,9 +57,9 @@ libc_test(
 libc_test(
     name = "lldiv_test",
     srcs = ["lldiv_test.cpp"],
-    libc_function_deps = ["//libc:lldiv"],
     deps = [
         ":div_test_helper",
+        "//libc:lldiv",
         "//libc:types_lldiv_t",
     ],
 )
@@ -77,36 +77,46 @@ libc_test_library(
 libc_test(
     name = "atoi_test",
     srcs = ["atoi_test.cpp"],
-    libc_function_deps = ["//libc:atoi"],
-    deps = [":atoi_test_helper"],
+    deps = [
+        ":atoi_test_helper",
+        "//libc:atoi",
+    ],
 )
 
 libc_test(
     name = "atol_test",
     srcs = ["atol_test.cpp"],
-    libc_function_deps = ["//libc:atol"],
-    deps = [":atoi_test_helper"],
+    deps = [
+        ":atoi_test_helper",
+        "//libc:atol",
+    ],
 )
 
 libc_test(
     name = "atoll_test",
     srcs = ["atoll_test.cpp"],
-    libc_function_deps = ["//libc:atoll"],
-    deps = [":atoi_test_helper"],
+    deps = [
+        ":atoi_test_helper",
+        "//libc:atoll",
+    ],
 )
 
 libc_test(
     name = "atof_test",
     srcs = ["atof_test.cpp"],
-    libc_function_deps = ["//libc:atof"],
-    deps = ["//libc:__support_fputil_fp_bits"],
+    deps = [
+        "//libc:__support_fputil_fp_bits",
+        "//libc:atof",
+    ],
 )
 
 libc_test(
     name = "bsearch_test",
     srcs = ["bsearch_test.cpp"],
-    libc_function_deps = ["//libc:bsearch"],
-    deps = ["//libc:types_size_t"],
+    deps = [
+        "//libc:bsearch",
+        "//libc:types_size_t",
+    ],
 )
 
 libc_test_library(
@@ -121,9 +131,9 @@ libc_test_library(
 libc_test(
     name = "quick_sort_test",
     srcs = ["quick_sort_test.cpp"],
-    libc_function_deps = ["//libc:qsort"],
     deps = [
         ":qsort_test_helper",
+        "//libc:qsort",
         "//libc:qsort_util",
         "//libc:types_size_t",
     ],
@@ -132,9 +142,9 @@ libc_test(
 libc_test(
     name = "heap_sort_test",
     srcs = ["heap_sort_test.cpp"],
-    libc_function_deps = ["//libc:qsort"],
     deps = [
         ":qsort_test_helper",
+        "//libc:qsort",
         "//libc:qsort_util",
         "//libc:types_size_t",
     ],
@@ -143,8 +153,10 @@ libc_test(
 libc_test(
     name = "qsort_r_test",
     srcs = ["qsort_r_test.cpp"],
-    libc_function_deps = ["//libc:qsort_r"],
-    deps = ["//libc:types_size_t"],
+    deps = [
+        "//libc:qsort_r",
+        "//libc:types_size_t",
+    ],
 )
 
 libc_test_library(
@@ -160,22 +172,28 @@ libc_test_library(
 libc_test(
     name = "strfromf_test",
     srcs = ["strfromf_test.cpp"],
-    libc_function_deps = ["//libc:strfromf"],
-    deps = [":strfrom_test_helper"],
+    deps = [
+        ":strfrom_test_helper",
+        "//libc:strfromf",
+    ],
 )
 
 libc_test(
     name = "strfromd_test",
     srcs = ["strfromd_test.cpp"],
-    libc_function_deps = ["//libc:strfromd"],
-    deps = [":strfrom_test_helper"],
+    deps = [
+        ":strfrom_test_helper",
+        "//libc:strfromd",
+    ],
 )
 
 libc_test(
     name = "strfroml_test",
     srcs = ["strfroml_test.cpp"],
-    libc_function_deps = ["//libc:strfroml"],
-    deps = [":strfrom_test_helper"],
+    deps = [
+        ":strfrom_test_helper",
+        "//libc:strfroml",
+    ],
 )
 
 libc_test_library(
@@ -194,37 +212,45 @@ libc_test_library(
 libc_test(
     name = "strtol_test",
     srcs = ["strtol_test.cpp"],
-    libc_function_deps = ["//libc:strtol"],
-    deps = [":strtol_test_helper"],
+    deps = [
+        ":strtol_test_helper",
+        "//libc:strtol",
+    ],
 )
 
 libc_test(
     name = "strtoll_test",
     srcs = ["strtoll_test.cpp"],
-    libc_function_deps = ["//libc:strtoll"],
-    deps = [":strtol_test_helper"],
+    deps = [
+        ":strtol_test_helper",
+        "//libc:strtoll",
+    ],
 )
 
 libc_test(
     name = "strtoul_test",
     srcs = ["strtoul_test.cpp"],
-    libc_function_deps = ["//libc:strtoul"],
-    deps = [":strtol_test_helper"],
+    deps = [
+        ":strtol_test_helper",
+        "//libc:strtoul",
+    ],
 )
 
 libc_test(
     name = "strtoull_test",
     srcs = ["strtoull_test.cpp"],
-    libc_function_deps = ["//libc:strtoull"],
-    deps = [":strtol_test_helper"],
+    deps = [
+        ":strtol_test_helper",
+        "//libc:strtoull",
+    ],
 )
 
 libc_test(
     name = "strtof_test",
     srcs = ["strtof_test.cpp"],
-    libc_function_deps = ["//libc:strtof"],
     deps = [
         "//libc:__support_fputil_fp_bits",
+        "//libc:strtof",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
 )
@@ -232,9 +258,9 @@ libc_test(
 libc_test(
     name = "strtod_test",
     srcs = ["strtod_test.cpp"],
-    libc_function_deps = ["//libc:strtod"],
     deps = [
         "//libc:__support_fputil_fp_bits",
+        "//libc:strtod",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
 )
@@ -242,9 +268,9 @@ libc_test(
 libc_test(
     name = "strtold_test",
     srcs = ["strtold_test.cpp"],
-    libc_function_deps = ["//libc:strtold"],
     deps = [
         "//libc:__support_fputil_fp_bits",
         "//libc:__support_uint128",
+        "//libc:strtold",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
index 7274819e6758c..7555c9f35e3a0 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
@@ -13,7 +13,7 @@ licenses(["notice"])
 libc_test(
     name = "strlen_test",
     srcs = ["strlen_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:strlen",
     ],
 )
@@ -21,7 +21,7 @@ libc_test(
 libc_test(
     name = "strcpy_test",
     srcs = ["strcpy_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:strcpy",
     ],
 )
@@ -29,7 +29,7 @@ libc_test(
 libc_test(
     name = "strcmp_test",
     srcs = ["strcmp_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:strcmp",
     ],
 )
@@ -37,7 +37,7 @@ libc_test(
 libc_test(
     name = "memchr_test",
     srcs = ["memchr_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:memchr",
     ],
 )
@@ -51,16 +51,16 @@ libc_test_library(
 libc_test(
     name = "strchr_test",
     srcs = ["strchr_test.cpp"],
-    libc_function_deps = [
+    deps = [
+        ":strchr_test_helper",
         "//libc:strchr",
     ],
-    deps = [":strchr_test_helper"],
 )
 
 libc_test(
     name = "strstr_test",
     srcs = ["strstr_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:strstr",
     ],
 )
@@ -68,7 +68,7 @@ libc_test(
 libc_test(
     name = "strnlen_test",
     srcs = ["strnlen_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:strnlen",
     ],
 )
@@ -76,7 +76,7 @@ libc_test(
 libc_test(
     name = "memrchr_test",
     srcs = ["memrchr_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:memrchr",
     ],
 )
@@ -84,16 +84,16 @@ libc_test(
 libc_test(
     name = "strrchr_test",
     srcs = ["strrchr_test.cpp"],
-    libc_function_deps = [
+    deps = [
+        ":strchr_test_helper",
         "//libc:strrchr",
     ],
-    deps = [":strchr_test_helper"],
 )
 
 libc_test(
     name = "strcspn_test",
     srcs = ["strcspn_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:strcspn",
     ],
 )
@@ -101,7 +101,7 @@ libc_test(
 libc_test(
     name = "strspn_test",
     srcs = ["strspn_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:strspn",
     ],
 )
@@ -109,7 +109,7 @@ libc_test(
 libc_test(
     name = "strtok_test",
     srcs = ["strtok_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:strtok",
     ],
 )
@@ -138,20 +138,18 @@ libc_test_library(
 libc_test(
     name = "memcpy_test",
     srcs = ["memcpy_test.cpp"],
-    libc_function_deps = [
-        "//libc:memcpy",
-    ],
     deps = [
         ":memory_check_utils",
         ":protected_pages",
         "//libc:__support_macros_properties_os",
+        "//libc:memcpy",
     ],
 )
 
 libc_test(
     name = "mempcpy_test",
     srcs = ["mempcpy_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:mempcpy",
     ],
 )
@@ -159,26 +157,22 @@ libc_test(
 libc_test(
     name = "memset_test",
     srcs = ["memset_test.cpp"],
-    libc_function_deps = [
-        "//libc:memset",
-    ],
     deps = [
         ":memory_check_utils",
         ":protected_pages",
         "//libc:__support_macros_properties_os",
+        "//libc:memset",
     ],
 )
 
 libc_test(
     name = "memmove_test",
     srcs = ["memmove_test.cpp"],
-    libc_function_deps = [
-        "//libc:memcmp",
-        "//libc:memmove",
-    ],
     deps = [
         ":memory_check_utils",
         "//libc:__support_cpp_span",
+        "//libc:memcmp",
+        "//libc:memmove",
         "//libc/test/UnitTest:memory_matcher",
     ],
 )
@@ -186,11 +180,9 @@ libc_test(
 libc_test(
     name = "memcmp_test",
     srcs = ["memcmp_test.cpp"],
-    libc_function_deps = [
-        "//libc:memcmp",
-    ],
     deps = [
         ":memory_check_utils",
+        "//libc:memcmp",
         "//libc/test/UnitTest:test_logger",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel
index bcf4b4201e043..2e6f5644eec71 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel
@@ -13,11 +13,9 @@ licenses(["notice"])
 libc_test(
     name = "bcopy_test",
     srcs = ["bcopy_test.cpp"],
-    libc_function_deps = [
-        "//libc:bcopy",
-    ],
     deps = [
         "//libc:__support_cpp_span",
+        "//libc:bcopy",
         "//libc/test/UnitTest:memory_matcher",
         "//libc/test/src/string:memory_check_utils",
     ],
@@ -26,10 +24,8 @@ libc_test(
 libc_test(
     name = "bcmp_test",
     srcs = ["bcmp_test.cpp"],
-    libc_function_deps = [
-        "//libc:bcmp",
-    ],
     deps = [
+        "//libc:bcmp",
         "//libc/test/UnitTest:test_logger",
         "//libc/test/src/string:memory_check_utils",
     ],
@@ -38,10 +34,8 @@ libc_test(
 libc_test(
     name = "bzero_test",
     srcs = ["bzero_test.cpp"],
-    libc_function_deps = [
-        "//libc:bzero",
-    ],
     deps = [
+        "//libc:bzero",
         "//libc/test/src/string:memory_check_utils",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
index 63ddebdadbdc9..e391703075aa7 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
@@ -13,11 +13,9 @@ licenses(["notice"])
 libc_test(
     name = "epoll_create_test",
     srcs = ["linux/epoll_create_test.cpp"],
-    libc_function_deps = [
-        "//libc:epoll_create",
-        "//libc:close",
-    ],
     deps = [
+        "//libc:close",
+        "//libc:epoll_create",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -25,11 +23,9 @@ libc_test(
 libc_test(
     name = "epoll_create1_test",
     srcs = ["linux/epoll_create1_test.cpp"],
-    libc_function_deps = [
-        "//libc:epoll_create1",
-        "//libc:close",
-    ],
     deps = [
+        "//libc:close",
+        "//libc:epoll_create1",
         "//libc:hdr_sys_epoll_macros",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
@@ -38,14 +34,12 @@ libc_test(
 libc_test(
     name = "epoll_ctl_test",
     srcs = ["linux/epoll_ctl_test.cpp"],
-    libc_function_deps = [
+    deps = [
+        "//libc:close",
         "//libc:epoll_create1",
         "//libc:epoll_ctl",
-        "//libc:pipe",
-        "//libc:close",
-    ],
-    deps = [
         "//libc:hdr_sys_epoll_macros",
+        "//libc:pipe",
         "//libc:types_struct_epoll_event",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
@@ -54,15 +48,13 @@ libc_test(
 libc_test(
     name = "epoll_wait_test",
     srcs = ["linux/epoll_wait_test.cpp"],
-    libc_function_deps = [
-        "//libc:epoll_wait",
+    deps = [
+        "//libc:close",
         "//libc:epoll_create1",
         "//libc:epoll_ctl",
-        "//libc:pipe",
-        "//libc:close",
-    ],
-    deps = [
+        "//libc:epoll_wait",
         "//libc:hdr_sys_epoll_macros",
+        "//libc:pipe",
         "//libc:types_struct_epoll_event",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
@@ -71,15 +63,13 @@ libc_test(
 libc_test(
     name = "epoll_pwait_test",
     srcs = ["linux/epoll_pwait_test.cpp"],
-    libc_function_deps = [
-        "//libc:epoll_pwait",
+    deps = [
+        "//libc:close",
         "//libc:epoll_create1",
         "//libc:epoll_ctl",
-        "//libc:pipe",
-        "//libc:close",
-    ],
-    deps = [
+        "//libc:epoll_pwait",
         "//libc:hdr_sys_epoll_macros",
+        "//libc:pipe",
         "//libc:types_struct_epoll_event",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
@@ -88,15 +78,13 @@ libc_test(
 libc_test(
     name = "epoll_pwait2_test",
     srcs = ["linux/epoll_pwait2_test.cpp"],
-    libc_function_deps = [
-        "//libc:epoll_pwait2",
+    deps = [
+        "//libc:close",
         "//libc:epoll_create1",
         "//libc:epoll_ctl",
-        "//libc:pipe",
-        "//libc:close",
-    ],
-    deps = [
+        "//libc:epoll_pwait2",
         "//libc:hdr_sys_epoll_macros",
+        "//libc:pipe",
         "//libc:types_struct_epoll_event",
         "//libc:types_struct_timespec",
         "//libc/test/UnitTest:errno_test_helpers",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/sys/socket/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/sys/socket/BUILD.bazel
index ac7f48d0aeebb..743bf2a4743b7 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/sys/socket/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/sys/socket/BUILD.bazel
@@ -13,11 +13,9 @@ licenses(["notice"])
 libc_test(
     name = "socket_test",
     srcs = ["linux/socket_test.cpp"],
-    libc_function_deps = [
-        "//libc:socket",
-        "//libc:close",
-    ],
     deps = [
+        "//libc:close",
+        "//libc:socket",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -25,11 +23,9 @@ libc_test(
 libc_test(
     name = "socketpair_test",
     srcs = ["linux/socketpair_test.cpp"],
-    libc_function_deps = [
-        "//libc:socketpair",
-        "//libc:close",
-    ],
     deps = [
+        "//libc:close",
+        "//libc:socketpair",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -37,13 +33,11 @@ libc_test(
 libc_test(
     name = "send_recv_test",
     srcs = ["linux/send_recv_test.cpp"],
-    libc_function_deps = [
-        "//libc:socketpair",
-        "//libc:send",
-        "//libc:recv",
-        "//libc:close",
-    ],
     deps = [
+        "//libc:close",
+        "//libc:recv",
+        "//libc:send",
+        "//libc:socketpair",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -51,13 +45,11 @@ libc_test(
 libc_test(
     name = "sendto_recvfrom_test",
     srcs = ["linux/sendto_recvfrom_test.cpp"],
-    libc_function_deps = [
-        "//libc:socketpair",
-        "//libc:sendto",
-        "//libc:recvfrom",
-        "//libc:close",
-    ],
     deps = [
+        "//libc:close",
+        "//libc:recvfrom",
+        "//libc:sendto",
+        "//libc:socketpair",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -65,13 +57,11 @@ libc_test(
 libc_test(
     name = "sendmsg_recvmsg_test",
     srcs = ["linux/sendmsg_recvmsg_test.cpp"],
-    libc_function_deps = [
-        "//libc:socketpair",
-        "//libc:sendmsg",
-        "//libc:recvmsg",
-        "//libc:close",
-    ],
     deps = [
+        "//libc:close",
+        "//libc:recvmsg",
+        "//libc:sendmsg",
+        "//libc:socketpair",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/unistd/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/unistd/BUILD.bazel
index 62641889f9a72..661e0a6ff5dfe 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/unistd/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/unistd/BUILD.bazel
@@ -13,13 +13,11 @@ licenses(["notice"])
 libc_test(
     name = "access_test",
     srcs = ["access_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:access",
         "//libc:close",
+        "//libc:open",
         "//libc:unlink",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -27,15 +25,13 @@ libc_test(
 libc_test(
     name = "dup_test",
     srcs = ["dup_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:close",
         "//libc:dup",
+        "//libc:open",
         "//libc:read",
         "//libc:unlink",
         "//libc:write",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -43,15 +39,13 @@ libc_test(
 libc_test(
     name = "dup2_test",
     srcs = ["dup2_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:close",
         "//libc:dup2",
+        "//libc:open",
         "//libc:read",
         "//libc:unlink",
         "//libc:write",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -59,15 +53,13 @@ libc_test(
 libc_test(
     name = "dup3_test",
     srcs = ["dup3_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:close",
         "//libc:dup3",
+        "//libc:open",
         "//libc:read",
         "//libc:unlink",
         "//libc:write",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -75,16 +67,14 @@ libc_test(
 libc_test(
     name = "ftruncate_test",
     srcs = ["ftruncate_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
+        "//libc:__support_cpp_string_view",
         "//libc:close",
-        "//libc:read",
         "//libc:ftruncate",
+        "//libc:open",
+        "//libc:read",
         "//libc:unlink",
         "//libc:write",
-    ],
-    deps = [
-        "//libc:__support_cpp_string_view",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -92,16 +82,14 @@ libc_test(
 libc_test(
     name = "pread_pwrite_test",
     srcs = ["pread_pwrite_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:close",
         "//libc:fsync",
+        "//libc:open",
         "//libc:pread",
         "//libc:pwrite",
         "//libc:unlink",
         "//libc:write",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -109,15 +97,13 @@ libc_test(
 libc_test(
     name = "read_write_test",
     srcs = ["read_write_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:close",
         "//libc:fsync",
+        "//libc:open",
         "//libc:read",
-        "//libc:write",
         "//libc:remove",
-    ],
-    deps = [
+        "//libc:write",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -125,13 +111,11 @@ libc_test(
 libc_test(
     name = "link_test",
     srcs = ["link_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:close",
         "//libc:link",
+        "//libc:open",
         "//libc:unlink",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -139,24 +123,20 @@ libc_test(
 libc_test(
     name = "swab_test",
     srcs = ["swab_test.cpp"],
-    libc_function_deps = [
-        "//libc:swab",
-    ],
     deps = [
         "//libc:string_utils",
+        "//libc:swab",
     ],
 )
 
 libc_test(
     name = "symlink_test",
     srcs = ["symlink_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:close",
+        "//libc:open",
         "//libc:symlink",
         "//libc:unlink",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -164,16 +144,14 @@ libc_test(
 libc_test(
     name = "truncate_test",
     srcs = ["truncate_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
+        "//libc:__support_cpp_string_view",
         "//libc:close",
+        "//libc:open",
         "//libc:read",
         "//libc:truncate",
         "//libc:unlink",
         "//libc:write",
-    ],
-    deps = [
-        "//libc:__support_cpp_string_view",
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -181,12 +159,10 @@ libc_test(
 libc_test(
     name = "unlink_test",
     srcs = ["unlink_test.cpp"],
-    libc_function_deps = [
-        "//libc:open",
+    deps = [
         "//libc:close",
+        "//libc:open",
         "//libc:unlink",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -194,7 +170,7 @@ libc_test(
 libc_test(
     name = "getppid_test",
     srcs = ["getppid_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:getppid",
     ],
 )
@@ -202,7 +178,7 @@ libc_test(
 libc_test(
     name = "getuid_test",
     srcs = ["getuid_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:getuid",
     ],
 )
@@ -210,12 +186,10 @@ libc_test(
 libc_test(
     name = "isatty_test",
     srcs = ["isatty_test.cpp"],
-    libc_function_deps = [
+    deps = [
+        "//libc:close",
         "//libc:isatty",
         "//libc:open",
-        "//libc:close",
-    ],
-    deps = [
         "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
@@ -223,7 +197,7 @@ libc_test(
 libc_test(
     name = "geteuid_test",
     srcs = ["geteuid_test.cpp"],
-    libc_function_deps = [
+    deps = [
         "//libc:geteuid",
     ],
 )
@@ -236,7 +210,7 @@ libc_test(
 # libc_test(
 #     name = "syscall_test",
 #     srcs = ["syscall_test.cpp"],
-#     libc_function_deps = [
+#     deps = [
 #         "//libc:syscall",
 #     ],
 # )
@@ -246,7 +220,7 @@ libc_test(
 # libc_test(
 #     name = "sysconf_test",
 #     srcs = ["sysconf_test.cpp"],
-#     libc_function_deps = [
+#     deps = [
 #         "//libc:sysconf",
 #     ],
 # )
@@ -256,12 +230,10 @@ libc_test(
 # libc_test(
 #     name = "getopt_test",
 #     srcs = ["getopt_test.cpp"],
-#     libc_function_deps = [
+#     deps = [
+#         "//libc:__support_cpp_array",
 #         "//libc:getopt",
 #         "//libc:fopencookie",
 #         "//libc:fflush",
 #     ],
-#     deps = [
-#         "//libc:__support_cpp_array",
-#     ],
 # )

From a024d13f84dbe7b3d1eee555ddc82cdd1af814e0 Mon Sep 17 00:00:00 2001
From: tangaac <tangyan01@loongson.cn>
Date: Wed, 16 Apr 2025 11:33:29 +0800
Subject: [PATCH 073/710] [LoongArch] make ABDS/ABDU legal for lsx/lasx
 (#134190)

---
 .../LoongArch/LoongArchISelLowering.cpp       |   4 +
 .../LoongArch/LoongArchLASXInstrInfo.td       |   4 +
 .../Target/LoongArch/LoongArchLSXInstrInfo.td |   4 +
 .../LoongArch/lasx/ir-instruction/absd.ll     | 151 +++++-------------
 .../LoongArch/lsx/ir-instruction/absd.ll      | 151 +++++-------------
 5 files changed, 86 insertions(+), 228 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 002d88cbeeba3..e5ccbe897d19c 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -272,6 +272,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
           {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
           Expand);
       setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+      setOperationAction(ISD::ABDS, VT, Legal);
+      setOperationAction(ISD::ABDU, VT, Legal);
     }
     for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
       setOperationAction(ISD::BITREVERSE, VT, Custom);
@@ -336,6 +338,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
           {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
           Expand);
       setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+      setOperationAction(ISD::ABDS, VT, Legal);
+      setOperationAction(ISD::ABDU, VT, Legal);
     }
     for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
       setOperationAction(ISD::BITREVERSE, VT, Custom);
diff --git a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
index d6d532cddb594..aaa4b94b6e994 100644
--- a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
@@ -1840,6 +1840,10 @@ def : Pat<(vt (concat_vectors LSX128:$vd, LSX128:$vj)),
           (XVPERMI_Q (SUBREG_TO_REG (i64 0), LSX128:$vd, sub_128),
                      (SUBREG_TO_REG (i64 0), LSX128:$vj, sub_128), 2)>;
 
+// XVABSD_{B/H/W/D}[U]
+defm : PatXrXr<abds, "XVABSD">;
+defm : PatXrXrU<abdu, "XVABSD">;
+
 } // Predicates = [HasExtLASX]
 
 /// Intrinsic pattern
diff --git a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
index b0d880749bf92..e7327ce7461f7 100644
--- a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
@@ -2002,6 +2002,10 @@ def : Pat<(f32 f32imm_vldi:$in),
 def : Pat<(f64 f64imm_vldi:$in),
           (f64 (EXTRACT_SUBREG (VLDI (to_f64imm_vldi f64imm_vldi:$in)), sub_64))>;
 
+// VABSD_{B/H/W/D}[U]
+defm : PatVrVr<abds, "VABSD">;
+defm : PatVrVrU<abdu, "VABSD">;
+
 } // Predicates = [HasExtLSX]
 
 /// Intrinsic pattern
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/absd.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/absd.ll
index bd5b16f5147a2..c5df9f8420837 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/absd.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/absd.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=loongarch64 -mattr=+lasx < %s | FileCheck %s
 
-;; TODO: Currently LoongArch generates sub-optimal code for these cases
 ;; 1. trunc(abs(sub(sext(a),sext(b)))) -> abds(a,b) or abdu(a,b)
 ;; 2. abs(sub_nsw(x, y)) -> abds(a,b)
 ;; 3. sub(smax(a,b),smin(a,b)) -> abds(a,b) or abdu(a,b)
@@ -9,16 +8,12 @@
 ;; 5. sub(select(icmp(a,b),a,b),select(icmp(a,b),b,a)) -> abds(a,b) or abdu(a,b)
 ;;
 ;; abds / abdu can be lowered to xvabsd.{b/h/w/d} / xvabsd.{b/h/w/d}u  instruction.
-;;
-;; Later patch will address it.
 
 ;; trunc(abs(sub(sext(a),sext(b)))) -> abds(a,b)
 define <32 x i8> @xvabsd_b(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK-LABEL: xvabsd_b:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.b $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.b $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a.sext = sext <32 x i8> %a to <32 x i16>
   %b.sext = sext <32 x i8> %b to <32 x i16>
@@ -31,9 +26,7 @@ define <32 x i8> @xvabsd_b(<32 x i8> %a, <32 x i8> %b) {
 define <16 x i16> @xvabsd_h(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: xvabsd_h:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.h $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.h $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a.sext = sext <16 x i16> %a to <16 x i32>
   %b.sext = sext <16 x i16> %b to <16 x i32>
@@ -46,9 +39,7 @@ define <16 x i16> @xvabsd_h(<16 x i16> %a, <16 x i16> %b) {
 define <8 x i32> @xvabsd_w(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: xvabsd_w:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.w $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.w $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.w $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a.sext = sext <8 x i32> %a to <8 x i64>
   %b.sext = sext <8 x i32> %b to <8 x i64>
@@ -61,9 +52,7 @@ define <8 x i32> @xvabsd_w(<8 x i32> %a, <8 x i32> %b) {
 define <4 x i64> @xvabsd_d(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-LABEL: xvabsd_d:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.d $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.d $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.d $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a.sext = sext <4 x i64> %a to <4 x i128>
   %b.sext = sext <4 x i64> %b to <4 x i128>
@@ -76,9 +65,7 @@ define <4 x i64> @xvabsd_d(<4 x i64> %a, <4 x i64> %b) {
 define <32 x i8> @xvabsd_bu(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK-LABEL: xvabsd_bu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.bu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.bu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a.zext = zext <32 x i8> %a to <32 x i16>
   %b.zext = zext <32 x i8> %b to <32 x i16>
@@ -91,9 +78,7 @@ define <32 x i8> @xvabsd_bu(<32 x i8> %a, <32 x i8> %b) {
 define <16 x i16> @xvabsd_hu(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: xvabsd_hu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.hu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.hu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.hu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a.zext = zext <16 x i16> %a to <16 x i32>
   %b.zext = zext <16 x i16> %b to <16 x i32>
@@ -106,9 +91,7 @@ define <16 x i16> @xvabsd_hu(<16 x i16> %a, <16 x i16> %b) {
 define <8 x i32> @xvabsd_wu(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: xvabsd_wu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.wu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.wu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.wu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a.zext = zext <8 x i32> %a to <8 x i64>
   %b.zext = zext <8 x i32> %b to <8 x i64>
@@ -121,9 +104,7 @@ define <8 x i32> @xvabsd_wu(<8 x i32> %a, <8 x i32> %b) {
 define <4 x i64> @xvabsd_du(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-LABEL: xvabsd_du:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.du $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.du $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.du $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a.zext = zext <4 x i64> %a to <4 x i128>
   %b.zext = zext <4 x i64> %b to <4 x i128>
@@ -137,9 +118,7 @@ define <4 x i64> @xvabsd_du(<4 x i64> %a, <4 x i64> %b) {
 define <32 x i8> @xvabsd_b_nsw(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK-LABEL: xvabsd_b_nsw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvneg.b $xr1, $xr0
-; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvabsd.b $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %sub = sub nsw <32 x i8> %a, %b
   %abs = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %sub, i1 true)
@@ -149,9 +128,7 @@ define <32 x i8> @xvabsd_b_nsw(<32 x i8> %a, <32 x i8> %b) {
 define <16 x i16> @xvabsd_h_nsw(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: xvabsd_h_nsw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvneg.h $xr1, $xr0
-; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvabsd.h $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %sub = sub nsw <16 x i16> %a, %b
   %abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %sub, i1 true)
@@ -161,9 +138,7 @@ define <16 x i16> @xvabsd_h_nsw(<16 x i16> %a, <16 x i16> %b) {
 define <8 x i32> @xvabsd_w_nsw(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: xvabsd_w_nsw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvneg.w $xr1, $xr0
-; CHECK-NEXT:    xvmax.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvabsd.w $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %sub = sub nsw <8 x i32> %a, %b
   %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 true)
@@ -173,9 +148,7 @@ define <8 x i32> @xvabsd_w_nsw(<8 x i32> %a, <8 x i32> %b) {
 define <4 x i64> @xvabsd_d_nsw(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-LABEL: xvabsd_d_nsw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvneg.d $xr1, $xr0
-; CHECK-NEXT:    xvmax.d $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvabsd.d $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %sub = sub nsw <4 x i64> %a, %b
   %abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %sub, i1 true)
@@ -186,9 +159,7 @@ define <4 x i64> @xvabsd_d_nsw(<4 x i64> %a, <4 x i64> %b) {
 define <32 x i8> @maxmin_b(<32 x i8> %0, <32 x i8> %1) {
 ; CHECK-LABEL: maxmin_b:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.b $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.b $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <32 x i8> @llvm.smax.v32i8(<32 x i8> %0, <32 x i8> %1)
   %b = tail call <32 x i8> @llvm.smin.v32i8(<32 x i8> %0, <32 x i8> %1)
@@ -199,9 +170,7 @@ define <32 x i8> @maxmin_b(<32 x i8> %0, <32 x i8> %1) {
 define <16 x i16> @maxmin_h(<16 x i16> %0, <16 x i16> %1) {
 ; CHECK-LABEL: maxmin_h:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.h $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.h $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <16 x i16> @llvm.smax.v16i16(<16 x i16> %0, <16 x i16> %1)
   %b = tail call <16 x i16> @llvm.smin.v16i16(<16 x i16> %0, <16 x i16> %1)
@@ -212,9 +181,7 @@ define <16 x i16> @maxmin_h(<16 x i16> %0, <16 x i16> %1) {
 define <8 x i32> @maxmin_w(<8 x i32> %0, <8 x i32> %1) {
 ; CHECK-LABEL: maxmin_w:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.w $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.w $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.w $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <8 x i32> @llvm.smax.v8i32(<8 x i32> %0, <8 x i32> %1)
   %b = tail call <8 x i32> @llvm.smin.v8i32(<8 x i32> %0, <8 x i32> %1)
@@ -225,9 +192,7 @@ define <8 x i32> @maxmin_w(<8 x i32> %0, <8 x i32> %1) {
 define <4 x i64> @maxmin_d(<4 x i64> %0, <4 x i64> %1) {
 ; CHECK-LABEL: maxmin_d:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.d $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.d $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.d $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <4 x i64> @llvm.smax.v4i64(<4 x i64> %0, <4 x i64> %1)
   %b = tail call <4 x i64> @llvm.smin.v4i64(<4 x i64> %0, <4 x i64> %1)
@@ -238,9 +203,7 @@ define <4 x i64> @maxmin_d(<4 x i64> %0, <4 x i64> %1) {
 define <32 x i8> @maxmin_bu(<32 x i8> %0, <32 x i8> %1) {
 ; CHECK-LABEL: maxmin_bu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.bu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.bu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <32 x i8> @llvm.umax.v32i8(<32 x i8> %0, <32 x i8> %1)
   %b = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> %0, <32 x i8> %1)
@@ -251,9 +214,7 @@ define <32 x i8> @maxmin_bu(<32 x i8> %0, <32 x i8> %1) {
 define <16 x i16> @maxmin_hu(<16 x i16> %0, <16 x i16> %1) {
 ; CHECK-LABEL: maxmin_hu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.hu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.hu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.hu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <16 x i16> @llvm.umax.v16i16(<16 x i16> %0, <16 x i16> %1)
   %b = tail call <16 x i16> @llvm.umin.v16i16(<16 x i16> %0, <16 x i16> %1)
@@ -264,9 +225,7 @@ define <16 x i16> @maxmin_hu(<16 x i16> %0, <16 x i16> %1) {
 define <8 x i32> @maxmin_wu(<8 x i32> %0, <8 x i32> %1) {
 ; CHECK-LABEL: maxmin_wu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.wu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.wu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.wu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <8 x i32> @llvm.umax.v8i32(<8 x i32> %0, <8 x i32> %1)
   %b = tail call <8 x i32> @llvm.umin.v8i32(<8 x i32> %0, <8 x i32> %1)
@@ -277,9 +236,7 @@ define <8 x i32> @maxmin_wu(<8 x i32> %0, <8 x i32> %1) {
 define <4 x i64> @maxmin_du(<4 x i64> %0, <4 x i64> %1) {
 ; CHECK-LABEL: maxmin_du:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.du $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.du $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.du $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <4 x i64> @llvm.umax.v4i64(<4 x i64> %0, <4 x i64> %1)
   %b = tail call <4 x i64> @llvm.umin.v4i64(<4 x i64> %0, <4 x i64> %1)
@@ -290,9 +247,7 @@ define <4 x i64> @maxmin_du(<4 x i64> %0, <4 x i64> %1) {
 define <32 x i8> @maxmin_bu_com1(<32 x i8> %0, <32 x i8> %1) {
 ; CHECK-LABEL: maxmin_bu_com1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.bu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.bu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %a = tail call <32 x i8> @llvm.umax.v32i8(<32 x i8> %0, <32 x i8> %1)
   %b = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> %1, <32 x i8> %0)
@@ -304,9 +259,7 @@ define <32 x i8> @maxmin_bu_com1(<32 x i8> %0, <32 x i8> %1) {
 define <32 x i8> @xvabsd_b_cmp(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; CHECK-LABEL: xvabsd_b_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.b $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.b $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <32 x i8> %a, %b
   %ab = sub <32 x i8> %a, %b
@@ -318,9 +271,7 @@ define <32 x i8> @xvabsd_b_cmp(<32 x i8> %a, <32 x i8> %b) nounwind {
 define <16 x i16> @xvabsd_h_cmp(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; CHECK-LABEL: xvabsd_h_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.h $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.h $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <16 x i16> %a, %b
   %ab = sub <16 x i16> %a, %b
@@ -332,9 +283,7 @@ define <16 x i16> @xvabsd_h_cmp(<16 x i16> %a, <16 x i16> %b) nounwind {
 define <8 x i32> @xvabsd_w_cmp(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; CHECK-LABEL: xvabsd_w_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.w $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.w $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.w $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <8 x i32> %a, %b
   %ab = sub <8 x i32> %a, %b
@@ -346,9 +295,7 @@ define <8 x i32> @xvabsd_w_cmp(<8 x i32> %a, <8 x i32> %b) nounwind {
 define <4 x i64> @xvabsd_d_cmp(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; CHECK-LABEL: xvabsd_d_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.d $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.d $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.d $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <4 x i64> %a, %b
   %ab = sub <4 x i64> %a, %b
@@ -360,9 +307,7 @@ define <4 x i64> @xvabsd_d_cmp(<4 x i64> %a, <4 x i64> %b) nounwind {
 define <32 x i8> @xvabsd_bu_cmp(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; CHECK-LABEL: xvabsd_bu_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.bu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.bu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <32 x i8> %a, %b
   %ab = sub <32 x i8> %a, %b
@@ -374,9 +319,7 @@ define <32 x i8> @xvabsd_bu_cmp(<32 x i8> %a, <32 x i8> %b) nounwind {
 define <16 x i16> @xvabsd_hu_cmp(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; CHECK-LABEL: xvabsd_hu_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.hu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.hu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.hu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <16 x i16> %a, %b
   %ab = sub <16 x i16> %a, %b
@@ -388,9 +331,7 @@ define <16 x i16> @xvabsd_hu_cmp(<16 x i16> %a, <16 x i16> %b) nounwind {
 define <8 x i32> @xvabsd_wu_cmp(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; CHECK-LABEL: xvabsd_wu_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.wu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.wu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.wu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <8 x i32> %a, %b
   %ab = sub <8 x i32> %a, %b
@@ -402,9 +343,7 @@ define <8 x i32> @xvabsd_wu_cmp(<8 x i32> %a, <8 x i32> %b) nounwind {
 define <4 x i64> @xvabsd_du_cmp(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; CHECK-LABEL: xvabsd_du_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.du $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.du $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.du $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <4 x i64> %a, %b
   %ab = sub <4 x i64> %a, %b
@@ -417,9 +356,7 @@ define <4 x i64> @xvabsd_du_cmp(<4 x i64> %a, <4 x i64> %b) nounwind {
 define <32 x i8> @xvabsd_b_select(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; CHECK-LABEL: xvabsd_b_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.b $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.b $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <32 x i8> %a, %b
   %ab = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b
@@ -431,9 +368,7 @@ define <32 x i8> @xvabsd_b_select(<32 x i8> %a, <32 x i8> %b) nounwind {
 define <16 x i16> @xvabsd_h_select(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; CHECK-LABEL: xvabsd_h_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.h $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.h $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp sle <16 x i16> %a, %b
   %ab = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b
@@ -445,9 +380,7 @@ define <16 x i16> @xvabsd_h_select(<16 x i16> %a, <16 x i16> %b) nounwind {
 define <8 x i32> @xvabsd_w_select(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; CHECK-LABEL: xvabsd_w_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.w $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.w $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.w $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp sgt <8 x i32> %a, %b
   %ab = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b
@@ -459,9 +392,7 @@ define <8 x i32> @xvabsd_w_select(<8 x i32> %a, <8 x i32> %b) nounwind {
 define <4 x i64> @xvabsd_d_select(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; CHECK-LABEL: xvabsd_d_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.d $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.d $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.d $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp sge <4 x i64> %a, %b
   %ab = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b
@@ -473,9 +404,7 @@ define <4 x i64> @xvabsd_d_select(<4 x i64> %a, <4 x i64> %b) nounwind {
 define <32 x i8> @xvabsd_bu_select(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; CHECK-LABEL: xvabsd_bu_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.bu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.b $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.bu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <32 x i8> %a, %b
   %ab = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b
@@ -487,9 +416,7 @@ define <32 x i8> @xvabsd_bu_select(<32 x i8> %a, <32 x i8> %b) nounwind {
 define <16 x i16> @xvabsd_hu_select(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; CHECK-LABEL: xvabsd_hu_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.hu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.hu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.h $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.hu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ule <16 x i16> %a, %b
   %ab = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b
@@ -501,9 +428,7 @@ define <16 x i16> @xvabsd_hu_select(<16 x i16> %a, <16 x i16> %b) nounwind {
 define <8 x i32> @xvabsd_wu_select(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; CHECK-LABEL: xvabsd_wu_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.wu $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.wu $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.w $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.wu $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ugt <8 x i32> %a, %b
   %ab = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b
@@ -515,9 +440,7 @@ define <8 x i32> @xvabsd_wu_select(<8 x i32> %a, <8 x i32> %b) nounwind {
 define <4 x i64> @xvabsd_du_select(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; CHECK-LABEL: xvabsd_du_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvmin.du $xr2, $xr0, $xr1
-; CHECK-NEXT:    xvmax.du $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvsub.d $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvabsd.du $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %cmp = icmp uge <4 x i64> %a, %b
   %ab = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/absd.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/absd.ll
index 2cbd74204d5d6..f77a31b600761 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/absd.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/absd.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=loongarch64 -mattr=+lsx < %s | FileCheck %s
 
-;; TODO: Currently LoongArch generates sub-optimal code for five cases
 ;; 1. trunc(abs(sub(sext(a),sext(b)))) -> abds(a,b) or abdu(a,b)
 ;; 2. abs(sub_nsw(x, y)) -> abds(a,b)
 ;; 3. sub(smax(a,b),smin(a,b)) -> abds(a,b) or abdu(a,b)
@@ -9,16 +8,12 @@
 ;; 5. sub(select(icmp(a,b),a,b),select(icmp(a,b),b,a)) -> abds(a,b) or abdu(a,b)
 ;;
 ;; abds / abdu can be lowered to vabsd.{b/h/w/d} / vabsd.{b/h/w/d}u  instruction
-;;
-;; Later patch will address it.
 
 ;; trunc(abs(sub(sext(a),sext(b)))) -> abds(a,b)
 define <16 x i8> @vabsd_b(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: vabsd_b:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.b $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.b $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a.sext = sext <16 x i8> %a to <16 x i16>
   %b.sext = sext <16 x i8> %b to <16 x i16>
@@ -31,9 +26,7 @@ define <16 x i8> @vabsd_b(<16 x i8> %a, <16 x i8> %b) {
 define <8 x i16> @vabsd_h(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: vabsd_h:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.h $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.h $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a.sext = sext <8 x i16> %a to <8 x i32>
   %b.sext = sext <8 x i16> %b to <8 x i32>
@@ -46,9 +39,7 @@ define <8 x i16> @vabsd_h(<8 x i16> %a, <8 x i16> %b) {
 define <4 x i32> @vabsd_w(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: vabsd_w:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.w $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.w $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.w $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a.sext = sext <4 x i32> %a to <4 x i64>
   %b.sext = sext <4 x i32> %b to <4 x i64>
@@ -61,9 +52,7 @@ define <4 x i32> @vabsd_w(<4 x i32> %a, <4 x i32> %b) {
 define <2 x i64> @vabsd_d(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: vabsd_d:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.d $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.d $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.d $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a.sext = sext <2 x i64> %a to <2 x i128>
   %b.sext = sext <2 x i64> %b to <2 x i128>
@@ -76,9 +65,7 @@ define <2 x i64> @vabsd_d(<2 x i64> %a, <2 x i64> %b) {
 define <16 x i8> @vabsd_bu(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: vabsd_bu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.bu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.bu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a.zext = zext <16 x i8> %a to <16 x i16>
   %b.zext = zext <16 x i8> %b to <16 x i16>
@@ -91,9 +78,7 @@ define <16 x i8> @vabsd_bu(<16 x i8> %a, <16 x i8> %b) {
 define <8 x i16> @vabsd_hu(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: vabsd_hu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.hu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.hu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a.zext = zext <8 x i16> %a to <8 x i32>
   %b.zext = zext <8 x i16> %b to <8 x i32>
@@ -106,9 +91,7 @@ define <8 x i16> @vabsd_hu(<8 x i16> %a, <8 x i16> %b) {
 define <4 x i32> @vabsd_wu(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: vabsd_wu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.wu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.wu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.wu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a.zext = zext <4 x i32> %a to <4 x i64>
   %b.zext = zext <4 x i32> %b to <4 x i64>
@@ -121,9 +104,7 @@ define <4 x i32> @vabsd_wu(<4 x i32> %a, <4 x i32> %b) {
 define <2 x i64> @vabsd_du(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: vabsd_du:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.du $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.du $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.du $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a.zext = zext <2 x i64> %a to <2 x i128>
   %b.zext = zext <2 x i64> %b to <2 x i128>
@@ -137,9 +118,7 @@ define <2 x i64> @vabsd_du(<2 x i64> %a, <2 x i64> %b) {
 define <16 x i8> @vabsd_b_nsw(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: vabsd_b_nsw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr1
-; CHECK-NEXT:    vneg.b $vr1, $vr0
-; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vabsd.b $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %sub = sub nsw <16 x i8> %a, %b
   %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true)
@@ -149,9 +128,7 @@ define <16 x i8> @vabsd_b_nsw(<16 x i8> %a, <16 x i8> %b) {
 define <8 x i16> @vabsd_h_nsw(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: vabsd_h_nsw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr1
-; CHECK-NEXT:    vneg.h $vr1, $vr0
-; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vabsd.h $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %sub = sub nsw <8 x i16> %a, %b
   %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
@@ -161,9 +138,7 @@ define <8 x i16> @vabsd_h_nsw(<8 x i16> %a, <8 x i16> %b) {
 define <4 x i32> @vabsd_w_nsw(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: vabsd_w_nsw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr1
-; CHECK-NEXT:    vneg.w $vr1, $vr0
-; CHECK-NEXT:    vmax.w $vr0, $vr0, $vr1
+; CHECK-NEXT:    vabsd.w $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %sub = sub nsw <4 x i32> %a, %b
   %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
@@ -173,9 +148,7 @@ define <4 x i32> @vabsd_w_nsw(<4 x i32> %a, <4 x i32> %b) {
 define <2 x i64> @vabsd_d_nsw(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: vabsd_d_nsw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr1
-; CHECK-NEXT:    vneg.d $vr1, $vr0
-; CHECK-NEXT:    vmax.d $vr0, $vr0, $vr1
+; CHECK-NEXT:    vabsd.d $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %sub = sub nsw <2 x i64> %a, %b
   %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
@@ -186,9 +159,7 @@ define <2 x i64> @vabsd_d_nsw(<2 x i64> %a, <2 x i64> %b) {
 define <16 x i8> @maxmin_b(<16 x i8> %0, <16 x i8> %1) {
 ; CHECK-LABEL: maxmin_b:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.b $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.b $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <16 x i8> @llvm.smax.v16i8(<16 x i8> %0, <16 x i8> %1)
   %b = tail call <16 x i8> @llvm.smin.v16i8(<16 x i8> %0, <16 x i8> %1)
@@ -199,9 +170,7 @@ define <16 x i8> @maxmin_b(<16 x i8> %0, <16 x i8> %1) {
 define <8 x i16> @maxmin_h(<8 x i16> %0, <8 x i16> %1) {
 ; CHECK-LABEL: maxmin_h:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.h $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.h $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <8 x i16> @llvm.smax.v8i16(<8 x i16> %0, <8 x i16> %1)
   %b = tail call <8 x i16> @llvm.smin.v8i16(<8 x i16> %0, <8 x i16> %1)
@@ -212,9 +181,7 @@ define <8 x i16> @maxmin_h(<8 x i16> %0, <8 x i16> %1) {
 define <4 x i32> @maxmin_w(<4 x i32> %0, <4 x i32> %1) {
 ; CHECK-LABEL: maxmin_w:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.w $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.w $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.w $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> %0, <4 x i32> %1)
   %b = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %0, <4 x i32> %1)
@@ -225,9 +192,7 @@ define <4 x i32> @maxmin_w(<4 x i32> %0, <4 x i32> %1) {
 define <2 x i64> @maxmin_d(<2 x i64> %0, <2 x i64> %1) {
 ; CHECK-LABEL: maxmin_d:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.d $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.d $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.d $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <2 x i64> @llvm.smax.v2i64(<2 x i64> %0, <2 x i64> %1)
   %b = tail call <2 x i64> @llvm.smin.v2i64(<2 x i64> %0, <2 x i64> %1)
@@ -238,9 +203,7 @@ define <2 x i64> @maxmin_d(<2 x i64> %0, <2 x i64> %1) {
 define <16 x i8> @maxmin_bu(<16 x i8> %0, <16 x i8> %1) {
 ; CHECK-LABEL: maxmin_bu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.bu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.bu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1)
   %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %0, <16 x i8> %1)
@@ -251,9 +214,7 @@ define <16 x i8> @maxmin_bu(<16 x i8> %0, <16 x i8> %1) {
 define <8 x i16> @maxmin_hu(<8 x i16> %0, <8 x i16> %1) {
 ; CHECK-LABEL: maxmin_hu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.hu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.hu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> %0, <8 x i16> %1)
   %b = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %0, <8 x i16> %1)
@@ -264,9 +225,7 @@ define <8 x i16> @maxmin_hu(<8 x i16> %0, <8 x i16> %1) {
 define <4 x i32> @maxmin_wu(<4 x i32> %0, <4 x i32> %1) {
 ; CHECK-LABEL: maxmin_wu:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.wu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.wu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.wu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <4 x i32> @llvm.umax.v4i32(<4 x i32> %0, <4 x i32> %1)
   %b = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> %0, <4 x i32> %1)
@@ -277,9 +236,7 @@ define <4 x i32> @maxmin_wu(<4 x i32> %0, <4 x i32> %1) {
 define <2 x i64> @maxmin_du(<2 x i64> %0, <2 x i64> %1) {
 ; CHECK-LABEL: maxmin_du:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.du $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.du $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.du $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <2 x i64> @llvm.umax.v2i64(<2 x i64> %0, <2 x i64> %1)
   %b = tail call <2 x i64> @llvm.umin.v2i64(<2 x i64> %0, <2 x i64> %1)
@@ -290,9 +247,7 @@ define <2 x i64> @maxmin_du(<2 x i64> %0, <2 x i64> %1) {
 define <16 x i8> @maxmin_bu_com1(<16 x i8> %0, <16 x i8> %1) {
 ; CHECK-LABEL: maxmin_bu_com1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.bu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.bu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1)
   %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %1, <16 x i8> %0)
@@ -304,9 +259,7 @@ define <16 x i8> @maxmin_bu_com1(<16 x i8> %0, <16 x i8> %1) {
 define <16 x i8> @vabsd_b_cmp(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: vabsd_b_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.b $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.b $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <16 x i8> %a, %b
   %ab = sub <16 x i8> %a, %b
@@ -318,9 +271,7 @@ define <16 x i8> @vabsd_b_cmp(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @vabsd_h_cmp(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; CHECK-LABEL: vabsd_h_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.h $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.h $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <8 x i16> %a, %b
   %ab = sub <8 x i16> %a, %b
@@ -332,9 +283,7 @@ define <8 x i16> @vabsd_h_cmp(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <4 x i32> @vabsd_w_cmp(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: vabsd_w_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.w $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.w $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.w $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <4 x i32> %a, %b
   %ab = sub <4 x i32> %a, %b
@@ -346,9 +295,7 @@ define <4 x i32> @vabsd_w_cmp(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <2 x i64> @vabsd_d_cmp(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; CHECK-LABEL: vabsd_d_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.d $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.d $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.d $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <2 x i64> %a, %b
   %ab = sub <2 x i64> %a, %b
@@ -360,9 +307,7 @@ define <2 x i64> @vabsd_d_cmp(<2 x i64> %a, <2 x i64> %b) nounwind {
 define <16 x i8> @vabsd_bu_cmp(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: vabsd_bu_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.bu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.bu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <16 x i8> %a, %b
   %ab = sub <16 x i8> %a, %b
@@ -374,9 +319,7 @@ define <16 x i8> @vabsd_bu_cmp(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @vabsd_hu_cmp(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; CHECK-LABEL: vabsd_hu_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.hu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.hu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <8 x i16> %a, %b
   %ab = sub <8 x i16> %a, %b
@@ -388,9 +331,7 @@ define <8 x i16> @vabsd_hu_cmp(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <4 x i32> @vabsd_wu_cmp(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: vabsd_wu_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.wu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.wu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.wu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <4 x i32> %a, %b
   %ab = sub <4 x i32> %a, %b
@@ -402,9 +343,7 @@ define <4 x i32> @vabsd_wu_cmp(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <2 x i64> @vabsd_du_cmp(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; CHECK-LABEL: vabsd_du_cmp:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.du $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.du $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.du $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <2 x i64> %a, %b
   %ab = sub <2 x i64> %a, %b
@@ -417,9 +356,7 @@ define <2 x i64> @vabsd_du_cmp(<2 x i64> %a, <2 x i64> %b) nounwind {
 define <16 x i8> @vabsd_b_select(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: vabsd_b_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.b $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.b $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp slt <16 x i8> %a, %b
   %ab = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b
@@ -431,9 +368,7 @@ define <16 x i8> @vabsd_b_select(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @vabsd_h_select(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; CHECK-LABEL: vabsd_h_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.h $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.h $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp sle <8 x i16> %a, %b
   %ab = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
@@ -445,9 +380,7 @@ define <8 x i16> @vabsd_h_select(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <4 x i32> @vabsd_w_select(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: vabsd_w_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.w $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.w $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.w $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp sgt <4 x i32> %a, %b
   %ab = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
@@ -459,9 +392,7 @@ define <4 x i32> @vabsd_w_select(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <2 x i64> @vabsd_d_select(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; CHECK-LABEL: vabsd_d_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.d $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.d $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.d $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp sge <2 x i64> %a, %b
   %ab = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b
@@ -473,9 +404,7 @@ define <2 x i64> @vabsd_d_select(<2 x i64> %a, <2 x i64> %b) nounwind {
 define <16 x i8> @vabsd_bu_select(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: vabsd_bu_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.bu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.b $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.bu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ult <16 x i8> %a, %b
   %ab = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b
@@ -487,9 +416,7 @@ define <16 x i8> @vabsd_bu_select(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @vabsd_hu_select(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; CHECK-LABEL: vabsd_hu_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.hu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.h $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.hu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ule <8 x i16> %a, %b
   %ab = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
@@ -501,9 +428,7 @@ define <8 x i16> @vabsd_hu_select(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <4 x i32> @vabsd_wu_select(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: vabsd_wu_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.wu $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.wu $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.w $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.wu $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp ugt <4 x i32> %a, %b
   %ab = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
@@ -515,9 +440,7 @@ define <4 x i32> @vabsd_wu_select(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <2 x i64> @vabsd_du_select(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; CHECK-LABEL: vabsd_du_select:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmin.du $vr2, $vr0, $vr1
-; CHECK-NEXT:    vmax.du $vr0, $vr0, $vr1
-; CHECK-NEXT:    vsub.d $vr0, $vr0, $vr2
+; CHECK-NEXT:    vabsd.du $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %cmp = icmp uge <2 x i64> %a, %b
   %ab = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b

From b07c88563febdb62b82daad0480d7b6131bc54d4 Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Tue, 15 Apr 2025 23:44:33 -0400
Subject: [PATCH 074/710] [Support] Add format object for interleaved ranges
 (#135517)

Add two new format functions for printing ranges: `interleaved` and
`interleaved_array`.

This is meant to improve the ergonomics of printing ranges. Before this
patch, we have to either use `llvm::interleave` or write a for loop by
hand. For example:

Before:
```c++
ArrayRef<Type> types = ...;
ArrayRef<Values> values = ...;
LLVM_DEBUG({
  llvm::dbgs() << "Types: ";
  llvm::interleave_comma(llvm::dbgs(), types);
  llvm::dbgs() << "\n";
  llvm::dbgs() << "Values: [";
  llvm::interleave_comma(llvm::dbgs(), values);
  llvm::dbgs() << "]\n";
}):
```

After:
```c++
ArrayRef<Type> types = ...;
ArrayRef<Values> values = ...;
LLVM_DEBUG(llvm::dbgs() << "Types: " << interleaved(types) << "\n");
LLVM_DEBUG(llvm::dbgs() << "Values: " << interleaved_array(values) << "\n");
```

The separator and the prefix/suffix strings are customizable.
---
 llvm/include/llvm/Support/InterleavedRange.h  | 99 +++++++++++++++++++
 llvm/unittests/Support/CMakeLists.txt         |  3 +-
 .../Support/InterleavedRangeTest.cpp          | 70 +++++++++++++
 3 files changed, 171 insertions(+), 1 deletion(-)
 create mode 100644 llvm/include/llvm/Support/InterleavedRange.h
 create mode 100644 llvm/unittests/Support/InterleavedRangeTest.cpp

diff --git a/llvm/include/llvm/Support/InterleavedRange.h b/llvm/include/llvm/Support/InterleavedRange.h
new file mode 100644
index 0000000000000..4e70028504806
--- /dev/null
+++ b/llvm/include/llvm/Support/InterleavedRange.h
@@ -0,0 +1,99 @@
+//===- InterleavedRange.h - Output stream formatting for ranges -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements format objects for printing ranges to output streams.
+// For example:
+// ```c++
+//    ArrayRef<Type> Types = ...;
+//    OS << "Types: " << interleaved(Types); // ==> "Types: i32, f16, i8"
+//    ArrayRef<int> Values = ...;
+//    OS << "Values: " << interleaved_array(Values); // ==> "Values: [1, 2, 3]"
+// ```
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_INTERLEAVED_RANGE_H
+#define LLVM_SUPPORT_INTERLEAVED_RANGE_H
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+
+/// Format object class for interleaved ranges. Supports specifying the
+/// separator and, optionally, the prefix and suffix to be printed surrounding
+/// the range.
+/// Uses the operator '<<' of the range element type for printing. The range
+/// type itself does not have to have an '<<' operator defined.
+template <typename Range> class InterleavedRange {
+  const Range &TheRange;
+  StringRef Separator;
+  StringRef Prefix;
+  StringRef Suffix;
+
+public:
+  InterleavedRange(const Range &R, StringRef Separator, StringRef Prefix,
+                   StringRef Suffix)
+      : TheRange(R), Separator(Separator), Prefix(Prefix), Suffix(Suffix) {}
+
+  friend raw_ostream &operator<<(raw_ostream &OS,
+                                 const InterleavedRange &Interleaved) {
+    if (!Interleaved.Prefix.empty())
+      OS << Interleaved.Prefix;
+    llvm::interleave(Interleaved.TheRange, OS, Interleaved.Separator);
+    if (!Interleaved.Suffix.empty())
+      OS << Interleaved.Suffix;
+    return OS;
+  }
+
+  std::string str() const {
+    std::string Result;
+    raw_string_ostream Stream(Result);
+    Stream << *this;
+    Stream.flush();
+    return Result;
+  }
+
+  operator std::string() const { return str(); }
+};
+
+/// Output range `R` as a sequence of interleaved elements. Requires the range
+/// element type to be printable using `raw_ostream& operator<<`. The
+/// `Separator` and `Prefix` / `Suffix` can be customized. Examples:
+/// ```c++
+///   SmallVector<int> Vals = {1, 2, 3};
+///   OS << interleaved(Vals);                 // ==> "1, 2, 3"
+///   OS << interleaved(Vals, ";");            // ==> "1;2;3"
+///   OS << interleaved(Vals, " ", "{", "}");  // ==> "{1 2 3}"
+/// ```
+template <typename Range>
+InterleavedRange<Range> interleaved(const Range &R, StringRef Separator = ", ",
+                                    StringRef Prefix = "",
+                                    StringRef Suffix = "") {
+  return {R, Separator, Prefix, Suffix};
+}
+
+/// Output range `R` as an array of interleaved elements. Requires the range
+/// element type to be printable using `raw_ostream& operator<<`. The
+/// `Separator` can be customized. Examples:
+/// ```c++
+///   SmallVector<int> Vals = {1, 2, 3};
+///   OS << interleaved_array(Vals);       // ==> "[1, 2, 3]"
+///   OS << interleaved_array(Vals, ";");  // ==> "[1;2;3]"
+///   OS << interleaved_array(Vals, " ");  // ==> "[1 2 3]"
+/// ```
+template <typename Range>
+InterleavedRange<Range> interleaved_array(const Range &R,
+                                          StringRef Separator = ", ") {
+  return {R, Separator, "[", "]"};
+}
+
+} // end namespace llvm
+
+#endif // LLVM_SUPPORT_INTERLEAVED_RANGE_H
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 6c4e7cb689b20..4a12a928af119 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -49,6 +49,7 @@ add_llvm_unittest(SupportTests
   HashBuilderTest.cpp
   IndexedAccessorTest.cpp
   InstructionCostTest.cpp
+  InterleavedRangeTest.cpp
   JSONTest.cpp
   KnownBitsTest.cpp
   LEB128Test.cpp
@@ -61,7 +62,7 @@ add_llvm_unittest(SupportTests
   MemoryBufferRefTest.cpp
   MemoryBufferTest.cpp
   MemoryTest.cpp
-  MustacheTest.cpp      
+  MustacheTest.cpp
   ModRefTest.cpp
   NativeFormatTests.cpp
   OptimizedStructLayoutTest.cpp
diff --git a/llvm/unittests/Support/InterleavedRangeTest.cpp b/llvm/unittests/Support/InterleavedRangeTest.cpp
new file mode 100644
index 0000000000000..8640b81fe8ad8
--- /dev/null
+++ b/llvm/unittests/Support/InterleavedRangeTest.cpp
@@ -0,0 +1,70 @@
+//===- InterleavedRangeTest.cpp - Unit tests for interleaved format -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/InterleavedRange.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+TEST(InterleavedRangeTest, VectorInt) {
+  SmallVector<int> V = {0, 1, 2, 3};
+
+  // First, make sure that the raw print API works as expected.
+  std::string Buff;
+  raw_string_ostream OS(Buff);
+  OS << interleaved(V);
+  EXPECT_EQ("0, 1, 2, 3", Buff);
+  Buff.clear();
+  OS << interleaved_array(V);
+  EXPECT_EQ("[0, 1, 2, 3]", Buff);
+
+  // In the rest of the tests, use `.str()` for convenience.
+  EXPECT_EQ("0, 1, 2, 3", interleaved(V).str());
+  EXPECT_EQ("{{0,1,2,3}}", interleaved(V, ",", "{{", "}}").str());
+  EXPECT_EQ("[0, 1, 2, 3]", interleaved_array(V).str());
+  EXPECT_EQ("[0;1;2;3]", interleaved_array(V, ";").str());
+  EXPECT_EQ("0;1;2;3", interleaved(V, ";").str());
+}
+
+TEST(InterleavedRangeTest, VectorIntEmpty) {
+  SmallVector<int> V = {};
+  EXPECT_EQ("", interleaved(V).str());
+  EXPECT_EQ("{{}}", interleaved(V, ",", "{{", "}}").str());
+  EXPECT_EQ("[]", interleaved_array(V).str());
+  EXPECT_EQ("", interleaved(V, ";").str());
+}
+
+TEST(InterleavedRangeTest, VectorIntOneElem) {
+  SmallVector<int> V = {42};
+  EXPECT_EQ("42", interleaved(V).str());
+  EXPECT_EQ("{{42}}", interleaved(V, ",", "{{", "}}").str());
+  EXPECT_EQ("[42]", interleaved_array(V).str());
+  EXPECT_EQ("42", interleaved(V, ";").str());
+}
+
+struct CustomPrint {
+  int N;
+  friend raw_ostream &operator<<(raw_ostream &OS, const CustomPrint &CP) {
+    OS << "$$" << CP.N << "##";
+    return OS;
+  }
+};
+
+TEST(InterleavedRangeTest, CustomPrint) {
+  CustomPrint V[] = {{3}, {4}, {5}};
+  EXPECT_EQ("$$3##, $$4##, $$5##", interleaved(V).str());
+  EXPECT_EQ("{{$$3##;$$4##;$$5##}}", interleaved(V, ";", "{{", "}}").str());
+  EXPECT_EQ("[$$3##, $$4##, $$5##]", interleaved_array(V).str());
+}
+
+} // namespace

From 04b87e15e40f8857e29ade8321b8b67691545a50 Mon Sep 17 00:00:00 2001
From: Kareem Ergawy <kareem.ergawy@amd.com>
Date: Wed, 16 Apr 2025 06:14:38 +0200
Subject: [PATCH 075/710] [flang][fir] Lower `do concurrent` loop nests to
 `fir.do_concurrent` (#132904)

Adds support for lowering `do concurrent` nests from PFT to the new
`fir.do_concurrent` MLIR op as well as its special terminator
`fir.do_concurrent.loop` which models the actual loop nest.

To that end, this PR emits the allocations for the iteration variables
within the block of the `fir.do_concurrent` op and creates a region for
the `fir.do_concurrent.loop` op that accepts arguments equal in number
to the number of the input `do concurrent` iteration ranges.

For example, given the following input:
```fortran
   do concurrent(i=1:10, j=11:20)
   end do
```
the changes in this PR emit the following MLIR:
```mlir
    fir.do_concurrent {
      %22 = fir.alloca i32 {bindc_name = "i"}
      %23:2 = hlfir.declare %22 {uniq_name = "_QFsub1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
      %24 = fir.alloca i32 {bindc_name = "j"}
      %25:2 = hlfir.declare %24 {uniq_name = "_QFsub1Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
      fir.do_concurrent.loop (%arg1, %arg2) = (%18, %20) to (%19, %21) step (%c1, %c1_0) {
        %26 = fir.convert %arg1 : (index) -> i32
        fir.store %26 to %23#0 : !fir.ref<i32>
        %27 = fir.convert %arg2 : (index) -> i32
        fir.store %27 to %25#0 : !fir.ref<i32>
      }
    }
```
---
 flang/lib/Lower/Bridge.cpp                    | 228 +++++++++++-------
 flang/lib/Optimizer/Builder/FIRBuilder.cpp    |   3 +
 flang/test/Lower/do_concurrent.f90            |  39 ++-
 .../do_concurrent_local_default_init.f90      |   4 +-
 flang/test/Lower/loops.f90                    |  37 +--
 flang/test/Lower/loops3.f90                   |   4 +-
 flang/test/Lower/nsw.f90                      |   5 +-
 .../Transforms/DoConcurrent/basic_host.f90    |   3 +
 .../DoConcurrent/locally_destroyed_temp.f90   |   3 +
 .../DoConcurrent/loop_nest_test.f90           |   3 +
 .../multiple_iteration_ranges.f90             |   3 +
 .../DoConcurrent/non_const_bounds.f90         |   3 +
 .../DoConcurrent/not_perfectly_nested.f90     |   3 +
 13 files changed, 208 insertions(+), 130 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index b4d1197822a43..625dd116fe726 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -94,10 +94,11 @@ struct IncrementLoopInfo {
   template <typename T>
   explicit IncrementLoopInfo(Fortran::semantics::Symbol &sym, const T &lower,
                              const T &upper, const std::optional<T> &step,
-                             bool isUnordered = false)
+                             bool isConcurrent = false)
       : loopVariableSym{&sym}, lowerExpr{Fortran::semantics::GetExpr(lower)},
         upperExpr{Fortran::semantics::GetExpr(upper)},
-        stepExpr{Fortran::semantics::GetExpr(step)}, isUnordered{isUnordered} {}
+        stepExpr{Fortran::semantics::GetExpr(step)},
+        isConcurrent{isConcurrent} {}
 
   IncrementLoopInfo(IncrementLoopInfo &&) = default;
   IncrementLoopInfo &operator=(IncrementLoopInfo &&x) = default;
@@ -120,7 +121,7 @@ struct IncrementLoopInfo {
   const Fortran::lower::SomeExpr *upperExpr;
   const Fortran::lower::SomeExpr *stepExpr;
   const Fortran::lower::SomeExpr *maskExpr = nullptr;
-  bool isUnordered; // do concurrent, forall
+  bool isConcurrent;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localSymList;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localInitSymList;
   llvm::SmallVector<
@@ -130,7 +131,7 @@ struct IncrementLoopInfo {
   mlir::Value loopVariable = nullptr;
 
   // Data members for structured loops.
-  fir::DoLoopOp doLoop = nullptr;
+  mlir::Operation *loopOp = nullptr;
 
   // Data members for unstructured loops.
   bool hasRealControl = false;
@@ -1980,7 +1981,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     llvm_unreachable("illegal reduction operator");
   }
 
-  /// Collect DO CONCURRENT or FORALL loop control information.
+  /// Collect DO CONCURRENT loop control information.
   IncrementLoopNestInfo getConcurrentControl(
       const Fortran::parser::ConcurrentHeader &header,
       const std::list<Fortran::parser::LocalitySpec> &localityList = {}) {
@@ -2291,8 +2292,14 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get(
         builder->getContext(), {}, /*vectorize=*/va, {}, /*unroll*/ ua,
         /*unroll_and_jam*/ uja, {}, {}, {}, {}, {}, {}, {}, {}, {}, {});
-    if (has_attrs)
-      info.doLoop.setLoopAnnotationAttr(la);
+    if (has_attrs) {
+      if (auto loopOp = mlir::dyn_cast<fir::DoLoopOp>(info.loopOp))
+        loopOp.setLoopAnnotationAttr(la);
+
+      if (auto doConcurrentOp =
+              mlir::dyn_cast<fir::DoConcurrentLoopOp>(info.loopOp))
+        doConcurrentOp.setLoopAnnotationAttr(la);
+    }
   }
 
   /// Generate FIR to begin a structured or unstructured increment loop nest.
@@ -2301,96 +2308,77 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       llvm::SmallVectorImpl<const Fortran::parser::CompilerDirective *> &dirs) {
     assert(!incrementLoopNestInfo.empty() && "empty loop nest");
     mlir::Location loc = toLocation();
-    mlir::Operation *boundsAndStepIP = nullptr;
     mlir::arith::IntegerOverflowFlags iofBackup{};
 
+    llvm::SmallVector<mlir::Value> nestLBs;
+    llvm::SmallVector<mlir::Value> nestUBs;
+    llvm::SmallVector<mlir::Value> nestSts;
+    llvm::SmallVector<mlir::Value> nestReduceOperands;
+    llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
+    bool genDoConcurrent = false;
+
     for (IncrementLoopInfo &info : incrementLoopNestInfo) {
-      mlir::Value lowerValue;
-      mlir::Value upperValue;
-      mlir::Value stepValue;
+      genDoConcurrent = info.isStructured() && info.isConcurrent;
 
-      {
-        mlir::OpBuilder::InsertionGuard guard(*builder);
+      if (!genDoConcurrent)
+        info.loopVariable = genLoopVariableAddress(loc, *info.loopVariableSym,
+                                                   info.isConcurrent);
 
-        // Set the IP before the first loop in the nest so that all nest bounds
-        // and step values are created outside the nest.
-        if (boundsAndStepIP)
-          builder->setInsertionPointAfter(boundsAndStepIP);
+      if (!getLoweringOptions().getIntegerWrapAround()) {
+        iofBackup = builder->getIntegerOverflowFlags();
+        builder->setIntegerOverflowFlags(
+            mlir::arith::IntegerOverflowFlags::nsw);
+      }
 
-        info.loopVariable = genLoopVariableAddress(loc, *info.loopVariableSym,
-                                                   info.isUnordered);
-        if (!getLoweringOptions().getIntegerWrapAround()) {
-          iofBackup = builder->getIntegerOverflowFlags();
-          builder->setIntegerOverflowFlags(
-              mlir::arith::IntegerOverflowFlags::nsw);
-        }
-        lowerValue = genControlValue(info.lowerExpr, info);
-        upperValue = genControlValue(info.upperExpr, info);
-        bool isConst = true;
-        stepValue = genControlValue(info.stepExpr, info,
-                                    info.isStructured() ? nullptr : &isConst);
-        if (!getLoweringOptions().getIntegerWrapAround())
-          builder->setIntegerOverflowFlags(iofBackup);
-        boundsAndStepIP = stepValue.getDefiningOp();
-
-        // Use a temp variable for unstructured loops with non-const step.
-        if (!isConst) {
-          info.stepVariable =
-              builder->createTemporary(loc, stepValue.getType());
-          boundsAndStepIP =
-              builder->create<fir::StoreOp>(loc, stepValue, info.stepVariable);
+      nestLBs.push_back(genControlValue(info.lowerExpr, info));
+      nestUBs.push_back(genControlValue(info.upperExpr, info));
+      bool isConst = true;
+      nestSts.push_back(genControlValue(
+          info.stepExpr, info, info.isStructured() ? nullptr : &isConst));
+
+      if (!getLoweringOptions().getIntegerWrapAround())
+        builder->setIntegerOverflowFlags(iofBackup);
+
+      // Use a temp variable for unstructured loops with non-const step.
+      if (!isConst) {
+        mlir::Value stepValue = nestSts.back();
+        info.stepVariable = builder->createTemporary(loc, stepValue.getType());
+        builder->create<fir::StoreOp>(loc, stepValue, info.stepVariable);
+      }
+
+      if (genDoConcurrent && nestReduceOperands.empty()) {
+        // Create DO CONCURRENT reduce operands and attributes
+        for (const auto &reduceSym : info.reduceSymList) {
+          const fir::ReduceOperationEnum reduceOperation = reduceSym.first;
+          const Fortran::semantics::Symbol *sym = reduceSym.second;
+          fir::ExtendedValue exv = getSymbolExtendedValue(*sym, nullptr);
+          nestReduceOperands.push_back(fir::getBase(exv));
+          auto reduceAttr =
+              fir::ReduceAttr::get(builder->getContext(), reduceOperation);
+          nestReduceAttrs.push_back(reduceAttr);
         }
       }
+    }
 
+    for (auto [info, lowerValue, upperValue, stepValue] :
+         llvm::zip_equal(incrementLoopNestInfo, nestLBs, nestUBs, nestSts)) {
       // Structured loop - generate fir.do_loop.
       if (info.isStructured()) {
+        if (genDoConcurrent)
+          continue;
+
+        // The loop variable is a doLoop op argument.
         mlir::Type loopVarType = info.getLoopVariableType();
-        mlir::Value loopValue;
-        if (info.isUnordered) {
-          llvm::SmallVector<mlir::Value> reduceOperands;
-          llvm::SmallVector<mlir::Attribute> reduceAttrs;
-          // Create DO CONCURRENT reduce operands and attributes
-          for (const auto &reduceSym : info.reduceSymList) {
-            const fir::ReduceOperationEnum reduce_operation = reduceSym.first;
-            const Fortran::semantics::Symbol *sym = reduceSym.second;
-            fir::ExtendedValue exv = getSymbolExtendedValue(*sym, nullptr);
-            reduceOperands.push_back(fir::getBase(exv));
-            auto reduce_attr =
-                fir::ReduceAttr::get(builder->getContext(), reduce_operation);
-            reduceAttrs.push_back(reduce_attr);
-          }
-          // The loop variable value is explicitly updated.
-          info.doLoop = builder->create<fir::DoLoopOp>(
-              loc, lowerValue, upperValue, stepValue, /*unordered=*/true,
-              /*finalCountValue=*/false, /*iterArgs=*/std::nullopt,
-              llvm::ArrayRef<mlir::Value>(reduceOperands), reduceAttrs);
-          builder->setInsertionPointToStart(info.doLoop.getBody());
-          loopValue = builder->createConvert(loc, loopVarType,
-                                             info.doLoop.getInductionVar());
-        } else {
-          // The loop variable is a doLoop op argument.
-          info.doLoop = builder->create<fir::DoLoopOp>(
-              loc, lowerValue, upperValue, stepValue, /*unordered=*/false,
-              /*finalCountValue=*/true,
-              builder->createConvert(loc, loopVarType, lowerValue));
-          builder->setInsertionPointToStart(info.doLoop.getBody());
-          loopValue = info.doLoop.getRegionIterArgs()[0];
-        }
+        auto loopOp = builder->create<fir::DoLoopOp>(
+            loc, lowerValue, upperValue, stepValue, /*unordered=*/false,
+            /*finalCountValue=*/true,
+            builder->createConvert(loc, loopVarType, lowerValue));
+        info.loopOp = loopOp;
+        builder->setInsertionPointToStart(loopOp.getBody());
+        mlir::Value loopValue = loopOp.getRegionIterArgs()[0];
+
         // Update the loop variable value in case it has non-index references.
         builder->create<fir::StoreOp>(loc, loopValue, info.loopVariable);
-        if (info.maskExpr) {
-          Fortran::lower::StatementContext stmtCtx;
-          mlir::Value maskCond = createFIRExpr(loc, info.maskExpr, stmtCtx);
-          stmtCtx.finalizeAndReset();
-          mlir::Value maskCondCast =
-              builder->createConvert(loc, builder->getI1Type(), maskCond);
-          auto ifOp = builder->create<fir::IfOp>(loc, maskCondCast,
-                                                 /*withElseRegion=*/false);
-          builder->setInsertionPointToStart(&ifOp.getThenRegion().front());
-        }
-        if (info.hasLocalitySpecs())
-          handleLocalitySpecs(info);
-
         addLoopAnnotationAttr(info, dirs);
         continue;
       }
@@ -2454,6 +2442,60 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         builder->restoreInsertionPoint(insertPt);
       }
     }
+
+    if (genDoConcurrent) {
+      auto loopWrapperOp = builder->create<fir::DoConcurrentOp>(loc);
+      builder->setInsertionPointToStart(
+          builder->createBlock(&loopWrapperOp.getRegion()));
+
+      for (IncrementLoopInfo &info : llvm::reverse(incrementLoopNestInfo)) {
+        info.loopVariable = genLoopVariableAddress(loc, *info.loopVariableSym,
+                                                   info.isConcurrent);
+      }
+
+      builder->setInsertionPointToEnd(loopWrapperOp.getBody());
+      auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
+          loc, nestLBs, nestUBs, nestSts, nestReduceOperands,
+          nestReduceAttrs.empty()
+              ? nullptr
+              : mlir::ArrayAttr::get(builder->getContext(), nestReduceAttrs),
+          nullptr);
+
+      llvm::SmallVector<mlir::Type> loopBlockArgTypes(
+          incrementLoopNestInfo.size(), builder->getIndexType());
+      llvm::SmallVector<mlir::Location> loopBlockArgLocs(
+          incrementLoopNestInfo.size(), loc);
+      mlir::Region &loopRegion = loopOp.getRegion();
+      mlir::Block *loopBlock = builder->createBlock(
+          &loopRegion, loopRegion.begin(), loopBlockArgTypes, loopBlockArgLocs);
+      builder->setInsertionPointToStart(loopBlock);
+
+      for (auto [info, blockArg] :
+           llvm::zip_equal(incrementLoopNestInfo, loopBlock->getArguments())) {
+        info.loopOp = loopOp;
+        mlir::Value loopValue =
+            builder->createConvert(loc, info.getLoopVariableType(), blockArg);
+        builder->create<fir::StoreOp>(loc, loopValue, info.loopVariable);
+
+        if (info.maskExpr) {
+          Fortran::lower::StatementContext stmtCtx;
+          mlir::Value maskCond = createFIRExpr(loc, info.maskExpr, stmtCtx);
+          stmtCtx.finalizeAndReset();
+          mlir::Value maskCondCast =
+              builder->createConvert(loc, builder->getI1Type(), maskCond);
+          auto ifOp = builder->create<fir::IfOp>(loc, maskCondCast,
+                                                 /*withElseRegion=*/false);
+          builder->setInsertionPointToStart(&ifOp.getThenRegion().front());
+        }
+      }
+
+      IncrementLoopInfo &innermostInfo = incrementLoopNestInfo.back();
+
+      if (innermostInfo.hasLocalitySpecs())
+        handleLocalitySpecs(innermostInfo);
+
+      addLoopAnnotationAttr(innermostInfo, dirs);
+    }
   }
 
   /// Generate FIR to end a structured or unstructured increment loop nest.
@@ -2470,29 +2512,31 @@ class FirConverter : public Fortran::lower::AbstractConverter {
          it != rend; ++it) {
       IncrementLoopInfo &info = *it;
       if (info.isStructured()) {
-        // End fir.do_loop.
-        if (info.isUnordered) {
-          builder->setInsertionPointAfter(info.doLoop);
+        // End fir.do_concurent.loop.
+        if (info.isConcurrent) {
+          builder->setInsertionPointAfter(info.loopOp->getParentOp());
           continue;
         }
+
+        // End fir.do_loop.
         // Decrement tripVariable.
-        builder->setInsertionPointToEnd(info.doLoop.getBody());
+        auto doLoopOp = mlir::cast<fir::DoLoopOp>(info.loopOp);
+        builder->setInsertionPointToEnd(doLoopOp.getBody());
         llvm::SmallVector<mlir::Value, 2> results;
         results.push_back(builder->create<mlir::arith::AddIOp>(
-            loc, info.doLoop.getInductionVar(), info.doLoop.getStep(),
-            iofAttr));
+            loc, doLoopOp.getInductionVar(), doLoopOp.getStep(), iofAttr));
         // Step loopVariable to help optimizations such as vectorization.
         // Induction variable elimination will clean up as necessary.
         mlir::Value step = builder->createConvert(
-            loc, info.getLoopVariableType(), info.doLoop.getStep());
+            loc, info.getLoopVariableType(), doLoopOp.getStep());
         mlir::Value loopVar =
             builder->create<fir::LoadOp>(loc, info.loopVariable);
         results.push_back(
             builder->create<mlir::arith::AddIOp>(loc, loopVar, step, iofAttr));
         builder->create<fir::ResultOp>(loc, results);
-        builder->setInsertionPointAfter(info.doLoop);
+        builder->setInsertionPointAfter(doLoopOp);
         // The loop control variable may be used after the loop.
-        builder->create<fir::StoreOp>(loc, info.doLoop.getResult(1),
+        builder->create<fir::StoreOp>(loc, doLoopOp.getResult(1),
                                       info.loopVariable);
         continue;
       }
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 3cf9b5ae72d9e..d35367d7657cf 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -280,6 +280,9 @@ mlir::Block *fir::FirOpBuilder::getAllocaBlock() {
   if (auto cufKernelOp = getRegion().getParentOfType<cuf::KernelOp>())
     return &cufKernelOp.getRegion().front();
 
+  if (auto doConcurentOp = getRegion().getParentOfType<fir::DoConcurrentOp>())
+    return doConcurentOp.getBody();
+
   return getEntryBlock();
 }
 
diff --git a/flang/test/Lower/do_concurrent.f90 b/flang/test/Lower/do_concurrent.f90
index ef93d2d6b035b..cc113f59c35e3 100644
--- a/flang/test/Lower/do_concurrent.f90
+++ b/flang/test/Lower/do_concurrent.f90
@@ -14,6 +14,9 @@ subroutine sub1(n)
    implicit none
    integer :: n, m, i, j, k
    integer, dimension(n) :: a
+!CHECK: %[[N_DECL:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{.*}} {uniq_name = "_QFsub1En"}
+!CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub1Ea"}
+
 !CHECK: %[[LB1:.*]] = arith.constant 1 : i32
 !CHECK: %[[LB1_CVT:.*]] = fir.convert %[[LB1]] : (i32) -> index
 !CHECK: %[[UB1:.*]] = fir.load %{{.*}}#0 : !fir.ref<i32>
@@ -29,10 +32,30 @@ subroutine sub1(n)
 !CHECK: %[[UB3:.*]] = arith.constant 10 : i32
 !CHECK: %[[UB3_CVT:.*]] = fir.convert %[[UB3]] : (i32) -> index
 
-!CHECK: fir.do_loop %{{.*}} = %[[LB1_CVT]] to %[[UB1_CVT]] step %{{.*}} unordered
-!CHECK: fir.do_loop %{{.*}} = %[[LB2_CVT]] to %[[UB2_CVT]] step %{{.*}} unordered
-!CHECK: fir.do_loop %{{.*}} = %[[LB3_CVT]] to %[[UB3_CVT]] step %{{.*}} unordered
+!CHECK: fir.do_concurrent
+!CHECK:   %[[I:.*]] = fir.alloca i32 {bindc_name = "i"}
+!CHECK:   %[[I_DECL:.*]]:2 = hlfir.declare %[[I]]
+!CHECK:   %[[J:.*]] = fir.alloca i32 {bindc_name = "j"}
+!CHECK:   %[[J_DECL:.*]]:2 = hlfir.declare %[[J]]
+!CHECK:   %[[K:.*]] = fir.alloca i32 {bindc_name = "k"}
+!CHECK:   %[[K_DECL:.*]]:2 = hlfir.declare %[[K]]
+
+!CHECK:   fir.do_concurrent.loop (%[[I_IV:.*]], %[[J_IV:.*]], %[[K_IV:.*]]) =
+!CHECK-SAME:                     (%[[LB1_CVT]], %[[LB2_CVT]], %[[LB3_CVT]]) to
+!CHECK-SAME:                     (%[[UB1_CVT]], %[[UB2_CVT]], %[[UB3_CVT]]) step
+!CHECK-SAME:                     (%{{.*}}, %{{.*}}, %{{.*}}) {
+!CHECK:       %[[I_IV_CVT:.*]] = fir.convert %[[I_IV]] : (index) -> i32
+!CHECK:       fir.store %[[I_IV_CVT]] to %[[I_DECL]]#0 : !fir.ref<i32>
+!CHECK:       %[[J_IV_CVT:.*]] = fir.convert %[[J_IV]] : (index) -> i32
+!CHECK:       fir.store %[[J_IV_CVT]] to %[[J_DECL]]#0 : !fir.ref<i32>
+!CHECK:       %[[K_IV_CVT:.*]] = fir.convert %[[K_IV]] : (index) -> i32
+!CHECK:       fir.store %[[K_IV_CVT]] to %[[K_DECL]]#0 : !fir.ref<i32>
 
+!CHECK:       %[[N_VAL:.*]] = fir.load %[[N_DECL]]#0 : !fir.ref<i32>
+!CHECK:       %[[I_VAL:.*]] = fir.load %[[I_DECL]]#0 : !fir.ref<i32>
+!CHECK:       %[[I_VAL_CVT:.*]] = fir.convert %[[I_VAL]] : (i32) -> i64
+!CHECK:       %[[A_ELEM:.*]] = hlfir.designate %[[A_DECL]]#0 (%[[I_VAL_CVT]])
+!CHECK:       hlfir.assign %[[N_VAL]] to %[[A_ELEM]] : i32, !fir.ref<i32>
    do concurrent(i=1:n, j=1:bar(n*m, n/m), k=5:10)
       a(i) = n
    end do
@@ -45,14 +68,17 @@ subroutine sub2(n)
    integer, dimension(n) :: a
 !CHECK: %[[LB1:.*]] = arith.constant 1 : i32
 !CHECK: %[[LB1_CVT:.*]] = fir.convert %[[LB1]] : (i32) -> index
-!CHECK: %[[UB1:.*]] = fir.load %5#0 : !fir.ref<i32>
+!CHECK: %[[UB1:.*]] = fir.load %{{.*}}#0 : !fir.ref<i32>
 !CHECK: %[[UB1_CVT:.*]] = fir.convert %[[UB1]] : (i32) -> index
-!CHECK: fir.do_loop %{{.*}} = %[[LB1_CVT]] to %[[UB1_CVT]] step %{{.*}} unordered
+!CHECK: fir.do_concurrent
+!CHECK:   fir.do_concurrent.loop (%{{.*}}) = (%[[LB1_CVT]]) to (%[[UB1_CVT]]) step (%{{.*}})
+
 !CHECK: %[[LB2:.*]] = arith.constant 1 : i32
 !CHECK: %[[LB2_CVT:.*]] = fir.convert %[[LB2]] : (i32) -> index
 !CHECK: %[[UB2:.*]] = fir.call @_QPbar(%{{.*}}, %{{.*}}) proc_attrs<pure> fastmath<contract> : (!fir.ref<i32>, !fir.ref<i32>) -> i32
 !CHECK: %[[UB2_CVT:.*]] = fir.convert %[[UB2]] : (i32) -> index
-!CHECK: fir.do_loop %{{.*}} = %[[LB2_CVT]] to %[[UB2_CVT]] step %{{.*}} unordered
+!CHECK: fir.do_concurrent
+!CHECK:   fir.do_concurrent.loop (%{{.*}}) = (%[[LB2_CVT]]) to (%[[UB2_CVT]]) step (%{{.*}})
    do concurrent(i=1:n)
       do concurrent(j=1:bar(n*m, n/m))
          a(i) = n
@@ -60,7 +86,6 @@ subroutine sub2(n)
    end do
 end subroutine
 
-
 !CHECK-LABEL: unstructured
 subroutine unstructured(inner_step)
   integer(4) :: i, j, inner_step
diff --git a/flang/test/Lower/do_concurrent_local_default_init.f90 b/flang/test/Lower/do_concurrent_local_default_init.f90
index 7652e4fcd0402..207704ac1a990 100644
--- a/flang/test/Lower/do_concurrent_local_default_init.f90
+++ b/flang/test/Lower/do_concurrent_local_default_init.f90
@@ -29,7 +29,7 @@ subroutine test_default_init()
 ! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>> {fir.bindc_name = "p"}) {
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>
 ! CHECK:           %[[VAL_7:.*]] = fir.box_elesize %[[VAL_6]] : (!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>) -> index
-! CHECK:           fir.do_loop
+! CHECK:           fir.do_concurrent.loop
 ! CHECK:             %[[VAL_16:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>> {bindc_name = "p", pinned, uniq_name = "_QFtest_ptrEp"}
 ! CHECK:             %[[VAL_17:.*]] = fir.zero_bits !fir.ptr<!fir.array<?x!fir.char<1,?>>>
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 0 : index
@@ -43,7 +43,7 @@ subroutine test_default_init()
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QPtest_default_init(
-! CHECK:           fir.do_loop
+! CHECK:           fir.do_concurrent.loop
 ! CHECK:             %[[VAL_26:.*]] = fir.alloca !fir.type<_QFtest_default_initTt{i:i32}> {bindc_name = "a", pinned, uniq_name = "_QFtest_default_initEa"}
 ! CHECK:             %[[VAL_27:.*]] = fir.embox %[[VAL_26]] : (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>) -> !fir.box<!fir.type<_QFtest_default_initTt{i:i32}>>
 ! CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_27]] : (!fir.box<!fir.type<_QFtest_default_initTt{i:i32}>>) -> !fir.box<none>
diff --git a/flang/test/Lower/loops.f90 b/flang/test/Lower/loops.f90
index ea65ba3e4d66d..60df27a591dc3 100644
--- a/flang/test/Lower/loops.f90
+++ b/flang/test/Lower/loops.f90
@@ -2,15 +2,6 @@
 
 ! CHECK-LABEL: loop_test
 subroutine loop_test
-  ! CHECK: %[[VAL_2:.*]] = fir.alloca i16 {bindc_name = "i"}
-  ! CHECK: %[[VAL_3:.*]] = fir.alloca i16 {bindc_name = "i"}
-  ! CHECK: %[[VAL_4:.*]] = fir.alloca i16 {bindc_name = "i"}
-  ! CHECK: %[[VAL_5:.*]] = fir.alloca i8 {bindc_name = "k"}
-  ! CHECK: %[[VAL_6:.*]] = fir.alloca i8 {bindc_name = "j"}
-  ! CHECK: %[[VAL_7:.*]] = fir.alloca i8 {bindc_name = "i"}
-  ! CHECK: %[[VAL_8:.*]] = fir.alloca i32 {bindc_name = "k"}
-  ! CHECK: %[[VAL_9:.*]] = fir.alloca i32 {bindc_name = "j"}
-  ! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i"}
   ! CHECK: %[[VAL_11:.*]] = fir.alloca !fir.array<5x5x5xi32> {bindc_name = "a", uniq_name = "_QFloop_testEa"}
   ! CHECK: %[[VAL_12:.*]] = fir.alloca i32 {bindc_name = "asum", uniq_name = "_QFloop_testEasum"}
   ! CHECK: %[[VAL_13:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFloop_testEi"}
@@ -25,7 +16,7 @@ subroutine loop_test
   j = 200
   k = 300
 
-  ! CHECK-COUNT-3: fir.do_loop {{.*}} unordered
+  ! CHECK: fir.do_concurrent.loop (%{{.*}}, %{{.*}}, %{{.*}}) = {{.*}}
   do concurrent (i=1:5, j=1:5, k=1:5) ! shared(a)
     ! CHECK: fir.coordinate_of
     a(i,j,k) = 0
@@ -33,7 +24,7 @@ subroutine loop_test
   ! CHECK: fir.call @_FortranAioBeginExternalListOutput
   print*, 'A:', i, j, k
 
-  ! CHECK-COUNT-3: fir.do_loop {{.*}} unordered
+  ! CHECK: fir.do_concurrent.loop (%{{.*}}, %{{.*}}, %{{.*}}) = {{.*}}
   ! CHECK: fir.if
   do concurrent (integer(1)::i=1:5, j=1:5, k=1:5, i.ne.j .and. k.ne.3) shared(a)
     ! CHECK-COUNT-2: fir.coordinate_of
@@ -53,7 +44,7 @@ subroutine loop_test
   ! CHECK: fir.call @_FortranAioBeginExternalListOutput
   print*, 'B:', i, j, k, '-', asum
 
-  ! CHECK: fir.do_loop {{.*}} unordered
+  ! CHECK: fir.do_concurrent.loop (%{{.*}}) = {{.*}}
   ! CHECK-COUNT-2: fir.if
   do concurrent (integer(2)::i=1:5, i.ne.3)
     if (i.eq.2 .or. i.eq.4) goto 5 ! fir.if
@@ -62,7 +53,7 @@ subroutine loop_test
   5 continue
   enddo
 
-  ! CHECK: fir.do_loop {{.*}} unordered
+  ! CHECK: fir.do_concurrent.loop (%{{.*}}) = {{.*}}
   ! CHECK-COUNT-2: fir.if
   do concurrent (integer(2)::i=1:5, i.ne.3)
     if (i.eq.2 .or. i.eq.4) then ! fir.if
@@ -93,10 +84,6 @@ end subroutine loop_test
 
 ! CHECK-LABEL: c.func @_QPlis
 subroutine lis(n)
-  ! CHECK-DAG: fir.alloca i32 {bindc_name = "m"}
-  ! CHECK-DAG: fir.alloca i32 {bindc_name = "j"}
-  ! CHECK-DAG: fir.alloca i32 {bindc_name = "i"}
-  ! CHECK-DAG: fir.alloca i8 {bindc_name = "i"}
   ! CHECK-DAG: fir.alloca i32 {bindc_name = "j", uniq_name = "_QFlisEj"}
   ! CHECK-DAG: fir.alloca i32 {bindc_name = "k", uniq_name = "_QFlisEk"}
   ! CHECK-DAG: fir.alloca !fir.box<!fir.ptr<!fir.array<?x?x?xi32>>> {bindc_name = "p", uniq_name = "_QFlisEp"}
@@ -117,8 +104,8 @@ subroutine lis(n)
   ! CHECK:     }
   r = 0
 
-  ! CHECK:     fir.do_loop %arg1 = %{{.*}} to %{{.*}} step %{{.*}} unordered {
-  ! CHECK:       fir.do_loop %arg2 = %{{.*}} to %{{.*}} step %c1{{.*}} iter_args(%arg3 = %{{.*}}) -> (index, i32) {
+  ! CHECK:     fir.do_concurrent {
+  ! CHECK:       fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
   ! CHECK:       }
   ! CHECK:     }
   do concurrent (integer(kind=1)::i=n:1:-1)
@@ -128,16 +115,18 @@ subroutine lis(n)
     enddo
   enddo
 
-  ! CHECK:     fir.do_loop %arg1 = %{{.*}} to %{{.*}} step %c1{{.*}} unordered {
-  ! CHECK:       fir.do_loop %arg2 = %{{.*}} to %{{.*}} step %c1{{.*}} unordered {
+  ! CHECK:       fir.do_concurrent.loop (%{{.*}}, %{{.*}}) = (%{{.*}}, %{{.*}}) to (%{{.*}}, %{{.*}}) step (%{{.*}}, %{{.*}}) {
   ! CHECK:         fir.if %{{.*}} {
   ! CHECK:           %[[V_95:[0-9]+]] = fir.alloca !fir.array<?x?xi32>, %{{.*}}, %{{.*}} {bindc_name = "t", pinned, uniq_name = "_QFlisEt"}
   ! CHECK:           %[[V_96:[0-9]+]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x?x?xi32>>> {bindc_name = "p", pinned, uniq_name = "_QFlisEp"}
   ! CHECK:           fir.store %{{.*}} to %[[V_96]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>
   ! CHECK:           fir.do_loop %arg3 = %{{.*}} to %{{.*}} step %c1{{.*}} iter_args(%arg4 = %{{.*}}) -> (index, i32) {
-  ! CHECK:             fir.do_loop %arg5 = %{{.*}} to %{{.*}} step %c1{{.*}} unordered {
-  ! CHECK:               fir.load %[[V_96]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>
-  ! CHECK:               fir.convert %[[V_95]] : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+  ! CHECK:             fir.do_concurrent {
+  ! CHECK:               fir.alloca i32 {bindc_name = "m"}
+  ! CHECK:               fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
+  ! CHECK:                 fir.load %[[V_96]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>
+  ! CHECK:                 fir.convert %[[V_95]] : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+  ! CHECK:               }
   ! CHECK:             }
   ! CHECK:           }
   ! CHECK:           fir.convert %[[V_95]] : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?xi32>>
diff --git a/flang/test/Lower/loops3.f90 b/flang/test/Lower/loops3.f90
index 78f39e1013082..84db1972cca16 100644
--- a/flang/test/Lower/loops3.f90
+++ b/flang/test/Lower/loops3.f90
@@ -12,9 +12,7 @@ subroutine loop_test
 
   ! CHECK: %[[VAL_0:.*]] = fir.alloca f32 {bindc_name = "m", uniq_name = "_QFloop_testEm"}
   ! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QFloop_testEsum) : !fir.ref<i32>
-  ! CHECK: fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} unordered reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
-  ! CHECK: fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} unordered reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
-  ! CHECK: fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} unordered reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
+  ! CHECK: fir.do_concurrent.loop ({{.*}}) = ({{.*}}) to ({{.*}}) step ({{.*}}) reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
   do concurrent (i=1:5, j=1:5, k=1:5) local(tmp) reduce(+:sum) reduce(max:m)
     tmp = i + j + k
     sum = tmp + sum
diff --git a/flang/test/Lower/nsw.f90 b/flang/test/Lower/nsw.f90
index 4ee9e5da829e6..2ec1efb2af42a 100644
--- a/flang/test/Lower/nsw.f90
+++ b/flang/test/Lower/nsw.f90
@@ -139,7 +139,6 @@ subroutine loop_params3(a,lb,ub,st)
 ! CHECK-LABEL:   func.func @_QPloop_params3(
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
-! CHECK:           %[[VAL_9:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
 ! CHECK:           %[[VAL_11:.*]] = fir.declare %{{.*}}lb"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
 ! CHECK:           %[[VAL_12:.*]] = fir.declare %{{.*}}ub"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
 ! CHECK:           %[[VAL_14:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
@@ -153,4 +152,6 @@ subroutine loop_params3(a,lb,ub,st)
 ! CHECK:           %[[VAL_31:.*]] = fir.load %[[VAL_15]] : !fir.ref<i32>
 ! CHECK:           %[[VAL_32:.*]] = arith.muli %[[VAL_31]], %[[VAL_4]] overflow<nsw> : i32
 ! CHECK:           %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (i32) -> index
-! CHECK:           fir.do_loop %[[VAL_34:.*]] = %[[VAL_28]] to %[[VAL_30]] step %[[VAL_33]] unordered {
+! CHECK:           fir.do_concurrent {
+! CHECK:             %[[VAL_9:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK:             fir.do_concurrent.loop (%[[VAL_34:.*]]) = (%[[VAL_28]]) to (%[[VAL_30]]) step (%[[VAL_33]]) {
diff --git a/flang/test/Transforms/DoConcurrent/basic_host.f90 b/flang/test/Transforms/DoConcurrent/basic_host.f90
index 12f63031cbaee..b84d4481ac766 100644
--- a/flang/test/Transforms/DoConcurrent/basic_host.f90
+++ b/flang/test/Transforms/DoConcurrent/basic_host.f90
@@ -1,3 +1,6 @@
+! Fails until we update the pass to use the `fir.do_concurrent` op.
+! XFAIL: *
+
 ! Tests mapping of a basic `do concurrent` loop to `!$omp parallel do`.
 
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -fdo-concurrent-to-openmp=host %s -o - \
diff --git a/flang/test/Transforms/DoConcurrent/locally_destroyed_temp.f90 b/flang/test/Transforms/DoConcurrent/locally_destroyed_temp.f90
index f82696669eca6..4e13c0919589a 100644
--- a/flang/test/Transforms/DoConcurrent/locally_destroyed_temp.f90
+++ b/flang/test/Transforms/DoConcurrent/locally_destroyed_temp.f90
@@ -1,3 +1,6 @@
+! Fails until we update the pass to use the `fir.do_concurrent` op.
+! XFAIL: *
+
 ! Tests that "loop-local values" are properly handled by localizing them to the
 ! body of the loop nest. See `collectLoopLocalValues` and `localizeLoopLocalValue`
 ! for a definition of "loop-local values" and how they are handled.
diff --git a/flang/test/Transforms/DoConcurrent/loop_nest_test.f90 b/flang/test/Transforms/DoConcurrent/loop_nest_test.f90
index 32bed61fe69e4..adc4a488d1ec9 100644
--- a/flang/test/Transforms/DoConcurrent/loop_nest_test.f90
+++ b/flang/test/Transforms/DoConcurrent/loop_nest_test.f90
@@ -1,3 +1,6 @@
+! Fails until we update the pass to use the `fir.do_concurrent` op.
+! XFAIL: *
+
 ! Tests loop-nest detection algorithm for do-concurrent mapping.
 
 ! REQUIRES: asserts
diff --git a/flang/test/Transforms/DoConcurrent/multiple_iteration_ranges.f90 b/flang/test/Transforms/DoConcurrent/multiple_iteration_ranges.f90
index d0210726de83e..26800678d381c 100644
--- a/flang/test/Transforms/DoConcurrent/multiple_iteration_ranges.f90
+++ b/flang/test/Transforms/DoConcurrent/multiple_iteration_ranges.f90
@@ -1,3 +1,6 @@
+! Fails until we update the pass to use the `fir.do_concurrent` op.
+! XFAIL: *
+
 ! Tests mapping of a `do concurrent` loop with multiple iteration ranges.
 
 ! RUN: split-file %s %t
diff --git a/flang/test/Transforms/DoConcurrent/non_const_bounds.f90 b/flang/test/Transforms/DoConcurrent/non_const_bounds.f90
index cd1bd4f98a3f5..23a3aae976c07 100644
--- a/flang/test/Transforms/DoConcurrent/non_const_bounds.f90
+++ b/flang/test/Transforms/DoConcurrent/non_const_bounds.f90
@@ -1,3 +1,6 @@
+! Fails until we update the pass to use the `fir.do_concurrent` op.
+! XFAIL: *
+
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -fdo-concurrent-to-openmp=host %s -o - \
 ! RUN:   | FileCheck %s
 
diff --git a/flang/test/Transforms/DoConcurrent/not_perfectly_nested.f90 b/flang/test/Transforms/DoConcurrent/not_perfectly_nested.f90
index 184fdfe00d397..d1c02101318ab 100644
--- a/flang/test/Transforms/DoConcurrent/not_perfectly_nested.f90
+++ b/flang/test/Transforms/DoConcurrent/not_perfectly_nested.f90
@@ -1,3 +1,6 @@
+! Fails until we update the pass to use the `fir.do_concurrent` op.
+! XFAIL: *
+
 ! Tests that if `do concurrent` is not perfectly nested in its parent loop, that
 ! we skip converting the not-perfectly nested `do concurrent` loop.
 

From 3264a50fe2b61e79572d1623d0cceb2fe88da533 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar@nvidia.com>
Date: Wed, 16 Apr 2025 10:03:21 +0530
Subject: [PATCH 076/710] [clang][NVPTX] Add builtins and intrinsics for
 conversions of new FP types (#134345)

This change:
- Adds NVVM intrinsics and clang builtins for the cvt instruction
variants of types (FP6) `.e2m3x2`, `.e3m2x2`, and (FP8) `.ue8m0x2`
introduced in PTX 8.6 for `sm_100a`, `sm_101a`, and `sm_120a`.
- Adds clang builtins for cvt instruction variant of type tf32.

Tests are added in `NVPTX/convert-sm100a.ll` and
`clang/test/CodeGen/builtins-nvptx.c` and verified through ptxas 12.8.0.

PTX Spec Reference:
https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt
---
 clang/include/clang/Basic/BuiltinsNVPTX.td |  31 +++
 clang/test/CodeGen/builtins-nvptx.c        | 150 ++++++++++-
 llvm/include/llvm/IR/IntrinsicsNVVM.td     |  31 +++
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td    |  35 +++
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td   |  56 ++++
 llvm/test/CodeGen/NVPTX/convert-sm100a.ll  | 290 +++++++++++++++++++++
 6 files changed, 589 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/convert-sm100a.ll

diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.td b/clang/include/clang/Basic/BuiltinsNVPTX.td
index 61e48b31c244b..bdbdfa2cea6c6 100644
--- a/clang/include/clang/Basic/BuiltinsNVPTX.td
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.td
@@ -580,6 +580,15 @@ def __nvvm_f2bf16_rz : NVPTXBuiltinSMAndPTX<"__bf16(float)", SM_80, PTX70>;
 def __nvvm_f2bf16_rz_relu : NVPTXBuiltinSMAndPTX<"__bf16(float)", SM_80, PTX70>;
 
 def __nvvm_f2tf32_rna : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_80, PTX70>;
+def __nvvm_f2tf32_rna_satfinite : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_89, PTX81>;
+def __nvvm_f2tf32_rn : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_90, PTX78>;
+def __nvvm_f2tf32_rn_relu : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_90, PTX78>;
+def __nvvm_f2tf32_rn_satfinite : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_100, PTX86>;
+def __nvvm_f2tf32_rn_relu_satfinite : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_100, PTX86>;
+def __nvvm_f2tf32_rz : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_90, PTX78>;
+def __nvvm_f2tf32_rz_relu : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_90, PTX78>;
+def __nvvm_f2tf32_rz_satfinite : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_100, PTX86>;
+def __nvvm_f2tf32_rz_relu_satfinite : NVPTXBuiltinSMAndPTX<"int32_t(float)", SM_100, PTX86>;
 
 def __nvvm_ff_to_e4m3x2_rn : NVPTXBuiltinSMAndPTX<"short(float, float)", SM_89, PTX81>;
 def __nvvm_ff_to_e4m3x2_rn_relu : NVPTXBuiltinSMAndPTX<"short(float, float)", SM_89, PTX81>;
@@ -596,6 +605,28 @@ def __nvvm_e4m3x2_to_f16x2_rn_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(sh
 def __nvvm_e5m2x2_to_f16x2_rn : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM_89, PTX81>;
 def __nvvm_e5m2x2_to_f16x2_rn_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM_89, PTX81>;
 
+def __nvvm_ff_to_e2m3x2_rn_satfinite : NVPTXBuiltinSMAndPTX<"short(float, float)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_ff_to_e2m3x2_rn_relu_satfinite : NVPTXBuiltinSMAndPTX<"short(float, float)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_ff_to_e3m2x2_rn_satfinite : NVPTXBuiltinSMAndPTX<"short(float, float)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_ff_to_e3m2x2_rn_relu_satfinite : NVPTXBuiltinSMAndPTX<"short(float, float)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+
+def __nvvm_e2m3x2_to_f16x2_rn : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_e2m3x2_to_f16x2_rn_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_e3m2x2_to_f16x2_rn : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_e3m2x2_to_f16x2_rn_relu : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(short)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+
+def __nvvm_ff_to_ue8m0x2_rz : NVPTXBuiltinSMAndPTX<"short(float, float)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_ff_to_ue8m0x2_rz_satfinite : NVPTXBuiltinSMAndPTX<"short(float, float)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_ff_to_ue8m0x2_rp : NVPTXBuiltinSMAndPTX<"short(float, float)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_ff_to_ue8m0x2_rp_satfinite : NVPTXBuiltinSMAndPTX<"short(float, float)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+
+def __nvvm_bf16x2_to_ue8m0x2_rz : NVPTXBuiltinSMAndPTX<"short(_Vector<2, __bf16>)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_bf16x2_to_ue8m0x2_rz_satfinite : NVPTXBuiltinSMAndPTX<"short(_Vector<2, __bf16>)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_bf16x2_to_ue8m0x2_rp : NVPTXBuiltinSMAndPTX<"short(_Vector<2, __bf16>)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+def __nvvm_bf16x2_to_ue8m0x2_rp_satfinite : NVPTXBuiltinSMAndPTX<"short(_Vector<2, __bf16>)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+
+def __nvvm_ue8m0x2_to_bf16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __bf16>(short)", SM<"100a", [SM_101a, SM_120a]>, PTX86>;
+
 // FNS
 let Attributes = [NoThrow] in {
   def __nvvm_fns : NVPTXBuiltinPTX<"unsigned int(unsigned int, unsigned int, int)", PTX60>;
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index e2c159aac903f..7404ce01c535c 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -25,14 +25,29 @@
 // RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_86 -target-feature +ptx72 \
 // RUN:            -disable-llvm-optzns -fcuda-is-device -emit-llvm -o - -x cuda %s \
 // RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX72_SM86 -check-prefix=LP64 %s
-// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_89 -target-feature +ptx81 \
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_89 -target-feature +ptx81 -DPTX=81\
 // RUN:            -disable-llvm-optzns -fcuda-is-device -emit-llvm -o - -x cuda %s \
 // RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX81_SM89 %s
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_90 -target-feature +ptx78 -DPTX=78 \
+// RUN:            -disable-llvm-optzns -fcuda-is-device -emit-llvm -o - -x cuda %s \
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX78_SM90 %s
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_100 -target-feature +ptx86 -DPTX=86 \
+// RUN:            -disable-llvm-optzns -fcuda-is-device -emit-llvm -o - -x cuda %s \
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX86_SM100 %s
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_100a -target-feature +ptx86 -DPTX=86 \
+// RUN:            -disable-llvm-optzns -fcuda-is-device -emit-llvm -o - -x cuda %s \
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX86_SM100a %s
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_101a -target-feature +ptx86 -DPTX=86 \
+// RUN:            -disable-llvm-optzns -fcuda-is-device -emit-llvm -o - -x cuda %s \
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX86_SM101a %s
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_120a -target-feature +ptx86 -DPTX=86 \
+// RUN:            -disable-llvm-optzns -fcuda-is-device -emit-llvm -o - -x cuda %s \
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX86_SM120a %s
 // ###  The last run to check with the highest SM and PTX version available
 // ###  to make sure target builtins are still accepted.
-// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_100a -target-feature +ptx87 \
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_120a -target-feature +ptx87 -DPTX=87 \
 // RUN:            -disable-llvm-optzns -fcuda-is-device -emit-llvm -o - -x cuda %s \
-// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX81_SM89 %s
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX86_SM120a %s
 
 #define __device__ __attribute__((device))
 #define __global__ __attribute__((global))
@@ -995,7 +1010,7 @@ __device__ void nvvm_cvt_sm80() {
 
 // CHECK-LABEL: nvvm_cvt_sm89
 __device__ void nvvm_cvt_sm89() {
-#if __CUDA_ARCH__ >= 890
+#if (PTX >= 81) && (__CUDA_ARCH__ >= 890)
   // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e4m3x2.rn(float 1.000000e+00, float 1.000000e+00)
   __nvvm_ff_to_e4m3x2_rn(1.0f, 1.0f);
   // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e4m3x2.rn.relu(float 1.000000e+00, float 1.000000e+00)
@@ -1022,6 +1037,133 @@ __device__ void nvvm_cvt_sm89() {
   __nvvm_e5m2x2_to_f16x2_rn(0x4c4c);
   // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn.relu(i16 19532)
   __nvvm_e5m2x2_to_f16x2_rn_relu(0x4c4c);
+
+  // CHECK_PTX81_SM89: call i32 @llvm.nvvm.f2tf32.rna.satfinite(float 1.000000e+00)
+  __nvvm_f2tf32_rna_satfinite(1.0f);
+#endif
+  // CHECK: ret void
+}
+
+// CHECK-LABEL: nvvm_cvt_sm90
+__device__ void nvvm_cvt_sm90() {
+#if (PTX >= 78) && (__CUDA_ARCH__ >= 900)
+  // CHECK_PTX78_SM90: call i32 @llvm.nvvm.f2tf32.rn(float 1.000000e+00)
+  __nvvm_f2tf32_rn(1.0f); 
+  // CHECK_PTX78_SM90: call i32 @llvm.nvvm.f2tf32.rn.relu(float 1.000000e+00)
+  __nvvm_f2tf32_rn_relu(1.0f); 
+  // CHECK_PTX78_SM90: call i32 @llvm.nvvm.f2tf32.rz(float 1.000000e+00)
+  __nvvm_f2tf32_rz(1.0f); 
+  // CHECK_PTX78_SM90: call i32 @llvm.nvvm.f2tf32.rz.relu(float 1.000000e+00)
+  __nvvm_f2tf32_rz_relu(1.0f); 
+#endif
+  // CHECK: ret void
+}
+
+// CHECK-LABEL: nvvm_cvt_sm100
+__device__ void nvvm_cvt_sm100() {
+#if (PTX >= 86) && (__CUDA_ARCH__ >= 1000)
+  // CHECK_PTX86_SM100: call i32 @llvm.nvvm.f2tf32.rn.satfinite(float 1.000000e+00)
+  __nvvm_f2tf32_rn_satfinite(1.0f); 
+  // CHECK_PTX86_SM100: call i32 @llvm.nvvm.f2tf32.rn.relu.satfinite(float 1.000000e+00)
+  __nvvm_f2tf32_rn_relu_satfinite(1.0f); 
+  // CHECK_PTX86_SM100: call i32 @llvm.nvvm.f2tf32.rz.satfinite(float 1.000000e+00)
+  __nvvm_f2tf32_rz_satfinite(1.0f); 
+  // CHECK_PTX86_SM100: call i32 @llvm.nvvm.f2tf32.rz.relu.satfinite(float 1.000000e+00)
+  __nvvm_f2tf32_rz_relu_satfinite(1.0f); 
+#endif
+  // CHECK: ret void
+}
+
+// CHECK-LABEL: nvvm_cvt_sm100a_sm101a_sm120a
+__device__ void nvvm_cvt_sm100a_sm101a_sm120a() {
+#if (PTX >= 86) && \
+    (__CUDA_ARCH_FEAT_SM100_ALL || __CUDA_ARCH_FEAT_SM101_ALL || \
+     __CUDA_ARCH_FEAT_SM120_ALL)
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.ff.to.e2m3x2.rn.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.ff.to.e2m3x2.rn.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.ff.to.e2m3x2.rn.satfinite(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e2m3x2_rn_satfinite(1.0f, 1.0f);
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.ff.to.e2m3x2.rn.relu.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.ff.to.e2m3x2.rn.relu.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.ff.to.e2m3x2.rn.relu.satfinite(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e2m3x2_rn_relu_satfinite(1.0f, 1.0f);
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.ff.to.e3m2x2.rn.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.ff.to.e3m2x2.rn.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.ff.to.e3m2x2.rn.satfinite(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e3m2x2_rn_satfinite(1.0f, 1.0f);
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.ff.to.e3m2x2.rn.relu.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.ff.to.e3m2x2.rn.relu.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.ff.to.e3m2x2.rn.relu.satfinite(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e3m2x2_rn_relu_satfinite(1.0f, 1.0f);
+
+  // CHECK_PTX86_SM100a: call <2 x half> @llvm.nvvm.e2m3x2.to.f16x2.rn(i16 19532)
+  // CHECK_PTX86_SM101a: call <2 x half> @llvm.nvvm.e2m3x2.to.f16x2.rn(i16 19532)
+  // CHECK_PTX86_SM120a: call <2 x half> @llvm.nvvm.e2m3x2.to.f16x2.rn(i16 19532)
+  __nvvm_e2m3x2_to_f16x2_rn(0x4C4C);
+
+  // CHECK_PTX86_SM100a: call <2 x half> @llvm.nvvm.e2m3x2.to.f16x2.rn.relu(i16 18504)
+  // CHECK_PTX86_SM101a: call <2 x half> @llvm.nvvm.e2m3x2.to.f16x2.rn.relu(i16 18504)
+  // CHECK_PTX86_SM120a: call <2 x half> @llvm.nvvm.e2m3x2.to.f16x2.rn.relu(i16 18504)
+  __nvvm_e2m3x2_to_f16x2_rn_relu(0x4848);
+
+  // CHECK_PTX86_SM100a: call <2 x half> @llvm.nvvm.e3m2x2.to.f16x2.rn(i16 18504)
+  // CHECK_PTX86_SM101a: call <2 x half> @llvm.nvvm.e3m2x2.to.f16x2.rn(i16 18504)
+  // CHECK_PTX86_SM120a: call <2 x half> @llvm.nvvm.e3m2x2.to.f16x2.rn(i16 18504)
+  __nvvm_e3m2x2_to_f16x2_rn(0x4848);
+
+  // CHECK_PTX86_SM100a: call <2 x half> @llvm.nvvm.e3m2x2.to.f16x2.rn.relu(i16 19532)
+  // CHECK_PTX86_SM101a: call <2 x half> @llvm.nvvm.e3m2x2.to.f16x2.rn.relu(i16 19532)
+  // CHECK_PTX86_SM120a: call <2 x half> @llvm.nvvm.e3m2x2.to.f16x2.rn.relu(i16 19532)
+  __nvvm_e3m2x2_to_f16x2_rn_relu(0x4C4C);
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rz(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rz(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rz(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_ue8m0x2_rz(1.0f, 1.0f);
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rz.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rz.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rz.satfinite(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_ue8m0x2_rz_satfinite(1.0f, 1.0f);
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rp(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rp(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rp(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_ue8m0x2_rp(1.0f, 1.0f);
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rp.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rp.satfinite(float 1.000000e+00, float 1.000000e+00)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.ff.to.ue8m0x2.rp.satfinite(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_ue8m0x2_rp_satfinite(1.0f, 1.0f);
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rz(<2 x bfloat> splat (bfloat 0xR3DCD)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rz(<2 x bfloat> splat (bfloat 0xR3DCD)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rz(<2 x bfloat> splat (bfloat 0xR3DCD)
+  __nvvm_bf16x2_to_ue8m0x2_rz({(__bf16)0.1f, (__bf16)0.1f});
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rz.satfinite(<2 x bfloat> splat (bfloat 0xR3DCD)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rz.satfinite(<2 x bfloat> splat (bfloat 0xR3DCD)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rz.satfinite(<2 x bfloat> splat (bfloat 0xR3DCD)
+  __nvvm_bf16x2_to_ue8m0x2_rz_satfinite({(__bf16)0.1f, (__bf16)0.1f});
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rp(<2 x bfloat> splat (bfloat 0xR3DCD)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rp(<2 x bfloat> splat (bfloat 0xR3DCD)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rp(<2 x bfloat> splat (bfloat 0xR3DCD)
+  __nvvm_bf16x2_to_ue8m0x2_rp({(__bf16)0.1f, (__bf16)0.1f});
+
+  // CHECK_PTX86_SM100a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rp.satfinite(<2 x bfloat> splat (bfloat 0xR3DCD)
+  // CHECK_PTX86_SM101a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rp.satfinite(<2 x bfloat> splat (bfloat 0xR3DCD)
+  // CHECK_PTX86_SM120a: call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rp.satfinite(<2 x bfloat> splat (bfloat 0xR3DCD)
+  __nvvm_bf16x2_to_ue8m0x2_rp_satfinite({(__bf16)0.1f, (__bf16)0.1f});
+
+  // CHECK_PTX86_SM100a: call <2 x bfloat> @llvm.nvvm.ue8m0x2.to.bf16x2(i16 19532)
+  // CHECK_PTX86_SM101a: call <2 x bfloat> @llvm.nvvm.ue8m0x2.to.bf16x2(i16 19532)
+  // CHECK_PTX86_SM120a: call <2 x bfloat> @llvm.nvvm.ue8m0x2.to.bf16x2(i16 19532)
+  __nvvm_ue8m0x2_to_bf16x2(0x4C4C);
+  
 #endif
   // CHECK: ret void
 }
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 4aeb1d8a2779e..7af051d22f9b7 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1628,6 +1628,37 @@ let TargetPrefix = "nvvm" in {
       Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
   def int_nvvm_e5m2x2_to_f16x2_rn_relu : ClangBuiltin<"__nvvm_e5m2x2_to_f16x2_rn_relu">,
       Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+  
+  class CVT_FF_TO_I16<string type, string suffix> : ClangBuiltin<!strconcat("__nvvm_ff_to_", type, suffix)>,
+      DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+      
+  class CVT_I16_TO_F16X2<string type, string suffix>
+    : ClangBuiltin<!strconcat("__nvvm_", type, "_to_f16x2", suffix)>,
+      DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+
+  class CVT_BF16X2_TO_I16<string type, string suffix> : ClangBuiltin<!strconcat("__nvvm_bf16x2_to_", type, suffix)>,
+      DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_v2bf16_ty], [IntrNoMem, IntrNoCallback]>;
+
+  // FP6 conversions.
+  foreach type = ["e2m3x2", "e3m2x2"] in {
+    foreach relu = ["", "_relu"] in {
+      defvar suffix = !strconcat("_rn", relu);
+      def int_nvvm_ff_to_ # type # suffix # _satfinite : CVT_FF_TO_I16<type, !strconcat(suffix, "_satfinite")>;
+      def int_nvvm_ # type # _to_f16x2 # suffix : CVT_I16_TO_F16X2<type, suffix>;
+    }
+  }
+  
+  // UE8M0x2 conversions.
+  foreach rmode = ["_rz", "_rp"] in {
+    foreach satmode = ["", "_satfinite"] in {
+      defvar suffix = !strconcat(rmode, satmode);
+      def int_nvvm_ff_to_ue8m0x2 # suffix : CVT_FF_TO_I16<"ue8m0x2", suffix>;
+      def int_nvvm_bf16x2_to_ue8m0x2 # suffix : CVT_BF16X2_TO_I16<"ue8m0x2", suffix>;
+    }
+  }
+      
+  def int_nvvm_ue8m0x2_to_bf16x2 : ClangBuiltin<"__nvvm_ue8m0x2_to_bf16x2">,
+      Intrinsic<[llvm_v2bf16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
 
 // FNS
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index ca15783654381..46549d850cf0a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -703,6 +703,41 @@ let hasSideEffects = false in {
   defm CVT_to_tf32_rz_satf : CVT_TO_TF32<"rz.satfinite", [hasPTX<86>, hasSM<100>]>;
   defm CVT_to_tf32_rn_relu_satf  : CVT_TO_TF32<"rn.relu.satfinite", [hasPTX<86>, hasSM<100>]>;
   defm CVT_to_tf32_rz_relu_satf  : CVT_TO_TF32<"rz.relu.satfinite", [hasPTX<86>, hasSM<100>]>;
+
+  // FP6 conversions.
+  foreach type = ["e2m3x2", "e3m2x2"] in {
+    def CVT_ # type # _f32_sf : NVPTXInst<(outs Int16Regs:$dst),
+                                          (ins Float32Regs:$src1, 
+                                            Float32Regs:$src2, CvtMode:$mode),
+                                          "cvt${mode:base}.satfinite${mode:relu}." 
+                                            # type # ".f32 \t$dst, $src1, $src2;", []>;
+    def CVT_f16x2_ # type : NVPTXInst<(outs Int32Regs:$dst),
+                                      (ins Int16Regs:$src, CvtMode:$mode),
+                                      "cvt${mode:base}${mode:relu}.f16x2." 
+                                        # type # " \t$dst, $src;", []>;
+  }
+  
+  // UE8M0x2 conversions.
+  class CVT_f32_to_ue8m0x2<string sat = ""> :
+    NVPTXInst<(outs Int16Regs:$dst),
+              (ins Float32Regs:$src1, Float32Regs:$src2, CvtMode:$mode),
+              "cvt${mode:base}" # sat # ".ue8m0x2.f32 \t$dst, $src1, $src2;", []>;
+  
+  class CVT_bf16x2_to_ue8m0x2<string sat = ""> :
+    NVPTXInst<(outs Int16Regs:$dst),
+              (ins Int32Regs:$src, CvtMode:$mode),
+              "cvt${mode:base}" # sat # ".ue8m0x2.bf16x2 \t$dst, $src;", []>;
+              
+  def CVT_ue8m0x2_f32 : CVT_f32_to_ue8m0x2;
+  def CVT_ue8m0x2_f32_sf : CVT_f32_to_ue8m0x2<".satfinite">;
+  def CVT_ue8m0x2_bf16x2 : CVT_bf16x2_to_ue8m0x2;
+  def CVT_ue8m0x2_bf16x2_sf : CVT_bf16x2_to_ue8m0x2<".satfinite">;
+
+  def CVT_bf16x2_ue8m0x2 :
+      NVPTXInst<(outs Int32Regs:$dst),
+                (ins Int16Regs:$src),
+                "cvt.rn.bf16x2.ue8m0x2 \t$dst, $src;", []>;
+
 }
 
 def fpround_oneuse : OneUse1<fpround>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 8528ff702f236..b5d3c9a05ec34 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1944,6 +1944,62 @@ def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn Int16Regs:$a),
 def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn_relu Int16Regs:$a),
           (CVT_f16x2_e5m2x2 $a, CvtRN_RELU)>;
 
+def : Pat<(int_nvvm_ff_to_e2m3x2_rn_satfinite f32:$a, f32:$b),
+          (CVT_e2m3x2_f32_sf $a, $b, CvtRN)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_ff_to_e2m3x2_rn_relu_satfinite f32:$a, f32:$b),
+          (CVT_e2m3x2_f32_sf $a, $b, CvtRN_RELU)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_ff_to_e3m2x2_rn_satfinite f32:$a, f32:$b),
+          (CVT_e3m2x2_f32_sf $a, $b, CvtRN)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_ff_to_e3m2x2_rn_relu_satfinite f32:$a, f32:$b),
+          (CVT_e3m2x2_f32_sf $a, $b, CvtRN_RELU)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+
+def : Pat<(int_nvvm_e2m3x2_to_f16x2_rn i16:$a),
+          (CVT_f16x2_e2m3x2 $a, CvtRN)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_e2m3x2_to_f16x2_rn_relu i16:$a),
+          (CVT_f16x2_e2m3x2 $a, CvtRN_RELU)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_e3m2x2_to_f16x2_rn i16:$a),
+          (CVT_f16x2_e3m2x2 $a, CvtRN)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_e3m2x2_to_f16x2_rn_relu i16:$a),
+          (CVT_f16x2_e3m2x2 $a, CvtRN_RELU)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+
+def : Pat<(int_nvvm_ff_to_ue8m0x2_rz f32:$a, f32:$b),
+          (CVT_ue8m0x2_f32 $a, $b, CvtRZ)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_ff_to_ue8m0x2_rz_satfinite f32:$a, f32:$b),
+          (CVT_ue8m0x2_f32_sf $a, $b, CvtRZ)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_ff_to_ue8m0x2_rp f32:$a, f32:$b),
+          (CVT_ue8m0x2_f32 $a, $b, CvtRP)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_ff_to_ue8m0x2_rp_satfinite f32:$a, f32:$b),
+          (CVT_ue8m0x2_f32_sf $a, $b, CvtRP)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+          
+def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rz Int32Regs:$a),
+          (CVT_ue8m0x2_bf16x2 $a, CvtRZ)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rz_satfinite Int32Regs:$a),
+          (CVT_ue8m0x2_bf16x2_sf $a, CvtRZ)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rp Int32Regs:$a),
+          (CVT_ue8m0x2_bf16x2 $a, CvtRP)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rp_satfinite Int32Regs:$a),
+          (CVT_ue8m0x2_bf16x2_sf $a, CvtRP)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+          
+def : Pat<(int_nvvm_ue8m0x2_to_bf16x2 i16:$a),
+          (CVT_bf16x2_ue8m0x2 $a)>,
+      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+
 //
 // FNS
 //
diff --git a/llvm/test/CodeGen/NVPTX/convert-sm100a.ll b/llvm/test/CodeGen/NVPTX/convert-sm100a.ll
new file mode 100644
index 0000000000000..f0dd5f084026b
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/convert-sm100a.ll
@@ -0,0 +1,290 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_101a -mattr=+ptx86 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_120a -mattr=+ptx86 | FileCheck %s
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86 | %ptxas-verify -arch=sm_100a %}
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_101a -mattr=+ptx86 | %ptxas-verify -arch=sm_101a %}
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_120a -mattr=+ptx86 | %ptxas-verify -arch=sm_120a %}
+
+define i16 @cvt_rn_sf_e2m3x2_f32(float %f1, float %f2) {
+; CHECK-LABEL: cvt_rn_sf_e2m3x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_sf_e2m3x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rn_sf_e2m3x2_f32_param_1];
+; CHECK-NEXT:    cvt.rn.satfinite.e2m3x2.f32 %rs1, %f1, %f2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.ff.to.e2m3x2.rn.satfinite(float %f1, float %f2)
+    ret i16 %val
+}
+
+define i16 @cvt_rn_relu_sf_e2m3x2_f32(float %f1, float %f2) {
+; CHECK-LABEL: cvt_rn_relu_sf_e2m3x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_relu_sf_e2m3x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rn_relu_sf_e2m3x2_f32_param_1];
+; CHECK-NEXT:    cvt.rn.satfinite.relu.e2m3x2.f32 %rs1, %f1, %f2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.ff.to.e2m3x2.rn.relu.satfinite(float %f1, float %f2)
+    ret i16 %val
+}
+
+define i16 @cvt_rn_sf_e3m2x2_f32(float %f1, float %f2) {
+; CHECK-LABEL: cvt_rn_sf_e3m2x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_sf_e3m2x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rn_sf_e3m2x2_f32_param_1];
+; CHECK-NEXT:    cvt.rn.satfinite.e3m2x2.f32 %rs1, %f1, %f2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.ff.to.e3m2x2.rn.satfinite(float %f1, float %f2)
+    ret i16 %val
+}
+
+define i16 @cvt_rn_relu_sf_e3m2x2_f32(float %f1, float %f2) {
+; CHECK-LABEL: cvt_rn_relu_sf_e3m2x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rn_relu_sf_e3m2x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rn_relu_sf_e3m2x2_f32_param_1];
+; CHECK-NEXT:    cvt.rn.satfinite.relu.e3m2x2.f32 %rs1, %f1, %f2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.ff.to.e3m2x2.rn.relu.satfinite(float %f1, float %f2)
+    ret i16 %val
+}
+
+define <2 x half> @cvt_rn_f16x2_e2m3x2(i16 %in) {
+; CHECK-LABEL: cvt_rn_f16x2_e2m3x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [cvt_rn_f16x2_e2m3x2_param_0];
+; CHECK-NEXT:    cvt.rn.f16x2.e2m3x2 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call <2 x half> @llvm.nvvm.e2m3x2.to.f16x2.rn(i16 %in)
+    ret <2 x half> %val
+}
+
+define <2 x half> @cvt_rn_relu_f16x2_e2m3x2_relu(i16 %in) {
+; CHECK-LABEL: cvt_rn_relu_f16x2_e2m3x2_relu(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [cvt_rn_relu_f16x2_e2m3x2_relu_param_0];
+; CHECK-NEXT:    cvt.rn.relu.f16x2.e2m3x2 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call <2 x half> @llvm.nvvm.e2m3x2.to.f16x2.rn.relu(i16 %in)
+    ret <2 x half> %val
+}
+
+define <2 x half> @cvt_rn_f16x2_e3m2x2(i16 %in) {
+; CHECK-LABEL: cvt_rn_f16x2_e3m2x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [cvt_rn_f16x2_e3m2x2_param_0];
+; CHECK-NEXT:    cvt.rn.f16x2.e3m2x2 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call <2 x half> @llvm.nvvm.e3m2x2.to.f16x2.rn(i16 %in)
+    ret <2 x half> %val
+}
+
+define <2 x half> @cvt_rn_relu_f16x2_e3m2x2(i16 %in) {
+; CHECK-LABEL: cvt_rn_relu_f16x2_e3m2x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [cvt_rn_relu_f16x2_e3m2x2_param_0];
+; CHECK-NEXT:    cvt.rn.relu.f16x2.e3m2x2 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call <2 x half> @llvm.nvvm.e3m2x2.to.f16x2.rn.relu(i16 %in)
+    ret <2 x half> %val
+}
+
+define i16 @cvt_rz_ue8m0x2_f32(float %f1, float %f2) {
+; CHECK-LABEL: cvt_rz_ue8m0x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_ue8m0x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rz_ue8m0x2_f32_param_1];
+; CHECK-NEXT:    cvt.rz.ue8m0x2.f32 %rs1, %f1, %f2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.ff.to.ue8m0x2.rz(float %f1, float %f2)
+    ret i16 %val
+}
+
+define i16 @cvt_rz_sf_ue8m0x2_f32(float %f1, float %f2) {
+; CHECK-LABEL: cvt_rz_sf_ue8m0x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rz_sf_ue8m0x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rz_sf_ue8m0x2_f32_param_1];
+; CHECK-NEXT:    cvt.rz.satfinite.ue8m0x2.f32 %rs1, %f1, %f2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.ff.to.ue8m0x2.rz.satfinite(float %f1, float %f2)
+    ret i16 %val
+}
+
+define i16 @cvt_rp_ue8m0x2_f32(float %f1, float %f2) {
+; CHECK-LABEL: cvt_rp_ue8m0x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rp_ue8m0x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rp_ue8m0x2_f32_param_1];
+; CHECK-NEXT:    cvt.rp.ue8m0x2.f32 %rs1, %f1, %f2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.ff.to.ue8m0x2.rp(float %f1, float %f2)
+    ret i16 %val
+}
+
+define i16 @cvt_rp_sf_ue8m0x2_f32(float %f1, float %f2) {
+; CHECK-LABEL: cvt_rp_sf_ue8m0x2_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [cvt_rp_sf_ue8m0x2_f32_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [cvt_rp_sf_ue8m0x2_f32_param_1];
+; CHECK-NEXT:    cvt.rp.satfinite.ue8m0x2.f32 %rs1, %f1, %f2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.ff.to.ue8m0x2.rp.satfinite(float %f1, float %f2)
+    ret i16 %val
+}
+
+define i16 @cvt_rz_ue8m0x2_bf16x2(<2 x bfloat> %in) {
+; CHECK-LABEL: cvt_rz_ue8m0x2_bf16x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [cvt_rz_ue8m0x2_bf16x2_param_0];
+; CHECK-NEXT:    cvt.rz.ue8m0x2.bf16x2 %rs1, %r1;
+; CHECK-NEXT:    cvt.u32.u16 %r2, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rz(<2 x bfloat> %in)
+    ret i16 %val
+}
+
+define i16 @cvt_rz_sf_ue8m0x2_bf16x2(<2 x bfloat> %in) {
+; CHECK-LABEL: cvt_rz_sf_ue8m0x2_bf16x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [cvt_rz_sf_ue8m0x2_bf16x2_param_0];
+; CHECK-NEXT:    cvt.rz.satfinite.ue8m0x2.bf16x2 %rs1, %r1;
+; CHECK-NEXT:    cvt.u32.u16 %r2, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rz.satfinite(<2 x bfloat> %in)
+    ret i16 %val
+}
+
+define i16 @cvt_rp_ue8m0x2_bf16x2(<2 x bfloat> %in) {
+; CHECK-LABEL: cvt_rp_ue8m0x2_bf16x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [cvt_rp_ue8m0x2_bf16x2_param_0];
+; CHECK-NEXT:    cvt.rp.ue8m0x2.bf16x2 %rs1, %r1;
+; CHECK-NEXT:    cvt.u32.u16 %r2, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rp(<2 x bfloat> %in)
+    ret i16 %val
+}
+
+define i16 @cvt_rp_sf_ue8m0x2_bf16x2(<2 x bfloat> %in) {
+; CHECK-LABEL: cvt_rp_sf_ue8m0x2_bf16x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [cvt_rp_sf_ue8m0x2_bf16x2_param_0];
+; CHECK-NEXT:    cvt.rp.satfinite.ue8m0x2.bf16x2 %rs1, %r1;
+; CHECK-NEXT:    cvt.u32.u16 %r2, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
+    %val = call i16 @llvm.nvvm.bf16x2.to.ue8m0x2.rp.satfinite(<2 x bfloat> %in)
+    ret i16 %val
+}
+
+define <2 x bfloat> @cvt_bf16x2_ue8m0x2(i16 %in) {
+; CHECK-LABEL: cvt_bf16x2_ue8m0x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [cvt_bf16x2_ue8m0x2_param_0];
+; CHECK-NEXT:    cvt.rn.bf16x2.ue8m0x2 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+    %val = call <2 x bfloat> @llvm.nvvm.ue8m0x2.to.bf16x2(i16 %in)
+    ret <2 x bfloat> %val
+}

From 2a024046217a1acae4806328ac77bd88648c2bab Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Wed, 16 Apr 2025 02:09:32 -0300
Subject: [PATCH 077/710] [clang] fix a crash in error recovery in expressions
 resolving to templates (#135893)

We were using AssumedTemplate incorrectly for error recovery.

Fixes #135621
---
 clang/docs/ReleaseNotes.rst                |  3 ++-
 clang/lib/AST/Type.cpp                     |  3 ++-
 clang/lib/Sema/SemaExpr.cpp                | 12 ++++++++----
 clang/test/SemaTemplate/recovery-crash.cpp | 11 +++++++++++
 4 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index ee69af5632f6e..84ad253c1ec4f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -402,6 +402,7 @@ Bug Fixes in This Version
   when using the ``INTn_C`` macros. (#GH85995)
 - Fixed an assertion failure in the expansion of builtin macros like ``__has_embed()`` with line breaks before the
   closing paren. (#GH133574)
+- Fixed a crash in error recovery for expressions resolving to templates. (#GH135621)
 - Clang no longer accepts invalid integer constants which are too large to fit
   into any (standard or extended) integer type when the constant is unevaluated.
   Merely forming the token is sufficient to render the program invalid. Code
@@ -544,7 +545,7 @@ Arm and AArch64 Support
 ^^^^^^^^^^^^^^^^^^^^^^^
 - For ARM targets, cc1as now considers the FPU's features for the selected CPU or Architecture.
 - The ``+nosimd`` attribute is now fully supported for ARM. Previously, this had no effect when being used with
-  ARM targets, however this will now disable NEON instructions being generated. The ``simd`` option is 
+  ARM targets, however this will now disable NEON instructions being generated. The ``simd`` option is
   also now printed when the ``--print-supported-extensions`` option is used.
 
 -  Support for __ptrauth type qualifier has been added.
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 62e48062cf241..53620003c9655 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -4401,7 +4401,8 @@ TemplateSpecializationType::TemplateSpecializationType(
           T.getKind() == TemplateName::SubstTemplateTemplateParmPack ||
           T.getKind() == TemplateName::UsingTemplate ||
           T.getKind() == TemplateName::QualifiedTemplate ||
-          T.getKind() == TemplateName::DeducedTemplate) &&
+          T.getKind() == TemplateName::DeducedTemplate ||
+          T.getKind() == TemplateName::AssumedTemplate) &&
          "Unexpected template name for TemplateSpecializationType");
 
   auto *TemplateArgs =
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 3ac7d61546ceb..c65b4eadf9c67 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -21111,11 +21111,15 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
     const bool IsTypeAliasTemplateDecl = isa<TypeAliasTemplateDecl>(Temp);
 
     NestedNameSpecifier *NNS = ULE->getQualifierLoc().getNestedNameSpecifier();
-    TemplateName TN(dyn_cast<TemplateDecl>(Temp));
-    if (TN.isNull())
+    // FIXME: AssumedTemplate is not very appropriate for error recovery here,
+    // as it models only the unqualified-id case, where this case can clearly be
+    // qualified. Thus we can't just qualify an assumed template.
+    TemplateName TN;
+    if (auto *TD = dyn_cast<TemplateDecl>(Temp))
+      TN = Context.getQualifiedTemplateName(NNS, ULE->hasTemplateKeyword(),
+                                            TemplateName(TD));
+    else
       TN = Context.getAssumedTemplateName(NameInfo.getName());
-    TN = Context.getQualifiedTemplateName(NNS,
-                                          /*TemplateKeyword=*/true, TN);
 
     Diag(NameInfo.getLoc(), diag::err_template_kw_refers_to_type_template)
         << TN << ULE->getSourceRange() << IsTypeAliasTemplateDecl;
diff --git a/clang/test/SemaTemplate/recovery-crash.cpp b/clang/test/SemaTemplate/recovery-crash.cpp
index ac8053da101ab..9b106f1f21fc5 100644
--- a/clang/test/SemaTemplate/recovery-crash.cpp
+++ b/clang/test/SemaTemplate/recovery-crash.cpp
@@ -67,3 +67,14 @@ namespace test1 {
   // expected-note@#defined-here {{defined here}}
   void NonTemplateClass::UndeclaredMethod() {}
 }
+
+namespace GH135621 {
+  template <class T> struct S {};
+  // expected-note@-1 {{class template declared here}}
+  template <class T2> void f() {
+    S<T2>::template S<int>;
+    // expected-error@-1 {{'S' is expected to be a non-type template, but instantiated to a class template}}
+  }
+  template void f<int>();
+  // expected-note@-1 {{requested here}}
+} // namespace GH135621

From 3de88fe40fd0dc5f52ba0bc7ebbaf16e938d6670 Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Tue, 15 Apr 2025 22:33:27 -0700
Subject: [PATCH 078/710] [DirectX] Implement the DXILCBufferAccess pass
 (#134571)

This introduces a pass that walks accesses to globals in cbuffers and
replaces them with accesses via the cbuffer handle itself. The logic to
interpret the cbuffer metadata is kept in `lib/Frontend/HLSL` so that it
can be reused by other consumers of that metadata.

Fixes #124630.
---
 llvm/include/llvm/Frontend/HLSL/CBuffer.h     |  64 ++++++
 llvm/lib/Frontend/HLSL/CBuffer.cpp            |  71 ++++++
 llvm/lib/Frontend/HLSL/CMakeLists.txt         |   1 +
 llvm/lib/Target/DirectX/CMakeLists.txt        |   1 +
 llvm/lib/Target/DirectX/DXILCBufferAccess.cpp | 210 ++++++++++++++++++
 llvm/lib/Target/DirectX/DXILCBufferAccess.h   |  28 +++
 llvm/lib/Target/DirectX/DirectX.h             |   6 +
 .../Target/DirectX/DirectXPassRegistry.def    |   1 +
 .../Target/DirectX/DirectXTargetMachine.cpp   |   3 +
 .../DirectX/CBufferAccess/array-typedgep.ll   |  32 +++
 .../CodeGen/DirectX/CBufferAccess/arrays.ll   | 124 +++++++++++
 .../CodeGen/DirectX/CBufferAccess/float.ll    |  25 +++
 .../DirectX/CBufferAccess/gep-ce-two-uses.ll  |  36 +++
 .../CodeGen/DirectX/CBufferAccess/scalars.ll  | 100 +++++++++
 .../CodeGen/DirectX/CBufferAccess/vectors.ll  | 119 ++++++++++
 llvm/test/CodeGen/DirectX/llc-pipeline.ll     |   1 +
 16 files changed, 822 insertions(+)
 create mode 100644 llvm/include/llvm/Frontend/HLSL/CBuffer.h
 create mode 100644 llvm/lib/Frontend/HLSL/CBuffer.cpp
 create mode 100644 llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
 create mode 100644 llvm/lib/Target/DirectX/DXILCBufferAccess.h
 create mode 100644 llvm/test/CodeGen/DirectX/CBufferAccess/array-typedgep.ll
 create mode 100644 llvm/test/CodeGen/DirectX/CBufferAccess/arrays.ll
 create mode 100644 llvm/test/CodeGen/DirectX/CBufferAccess/float.ll
 create mode 100644 llvm/test/CodeGen/DirectX/CBufferAccess/gep-ce-two-uses.ll
 create mode 100644 llvm/test/CodeGen/DirectX/CBufferAccess/scalars.ll
 create mode 100644 llvm/test/CodeGen/DirectX/CBufferAccess/vectors.ll

diff --git a/llvm/include/llvm/Frontend/HLSL/CBuffer.h b/llvm/include/llvm/Frontend/HLSL/CBuffer.h
new file mode 100644
index 0000000000000..694a7fa854576
--- /dev/null
+++ b/llvm/include/llvm/Frontend/HLSL/CBuffer.h
@@ -0,0 +1,64 @@
+//===- CBuffer.h - HLSL constant buffer handling ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains utilities to work with constant buffers in HLSL.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_HLSL_CBUFFER_H
+#define LLVM_FRONTEND_HLSL_CBUFFER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+#include <optional>
+
+namespace llvm {
+class Module;
+class GlobalVariable;
+class NamedMDNode;
+
+namespace hlsl {
+
+struct CBufferMember {
+  GlobalVariable *GV;
+  size_t Offset;
+
+  CBufferMember(GlobalVariable *GV, size_t Offset) : GV(GV), Offset(Offset) {}
+};
+
+struct CBufferMapping {
+  GlobalVariable *Handle;
+  SmallVector<CBufferMember> Members;
+
+  CBufferMapping(GlobalVariable *Handle) : Handle(Handle) {}
+};
+
+class CBufferMetadata {
+  NamedMDNode *MD;
+  SmallVector<CBufferMapping> Mappings;
+
+  CBufferMetadata(NamedMDNode *MD) : MD(MD) {}
+
+public:
+  static std::optional<CBufferMetadata> get(Module &M);
+
+  using iterator = SmallVector<CBufferMapping>::iterator;
+  iterator begin() { return Mappings.begin(); }
+  iterator end() { return Mappings.end(); }
+
+  void eraseFromModule();
+};
+
+APInt translateCBufArrayOffset(const DataLayout &DL, APInt Offset,
+                               ArrayType *Ty);
+
+} // namespace hlsl
+} // namespace llvm
+
+#endif // LLVM_FRONTEND_HLSL_CBUFFER_H
diff --git a/llvm/lib/Frontend/HLSL/CBuffer.cpp b/llvm/lib/Frontend/HLSL/CBuffer.cpp
new file mode 100644
index 0000000000000..37c0d912e09ee
--- /dev/null
+++ b/llvm/lib/Frontend/HLSL/CBuffer.cpp
@@ -0,0 +1,71 @@
+//===- CBuffer.cpp - HLSL constant buffer handling ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/HLSL/CBuffer.h"
+#include "llvm/Frontend/HLSL/HLSLResource.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+
+using namespace llvm;
+using namespace llvm::hlsl;
+
+static size_t getMemberOffset(GlobalVariable *Handle, size_t Index) {
+  auto *HandleTy = cast<TargetExtType>(Handle->getValueType());
+  assert(HandleTy->getName().ends_with(".CBuffer") && "Not a cbuffer type");
+  assert(HandleTy->getNumTypeParameters() == 1 && "Expected layout type");
+
+  auto *LayoutTy = cast<TargetExtType>(HandleTy->getTypeParameter(0));
+  assert(LayoutTy->getName().ends_with(".Layout") && "Not a layout type");
+
+  // Skip the "size" parameter.
+  size_t ParamIndex = Index + 1;
+  assert(LayoutTy->getNumIntParameters() > ParamIndex &&
+         "Not enough parameters");
+
+  return LayoutTy->getIntParameter(ParamIndex);
+}
+
+std::optional<CBufferMetadata> CBufferMetadata::get(Module &M) {
+  NamedMDNode *CBufMD = M.getNamedMetadata("hlsl.cbs");
+  if (!CBufMD)
+    return std::nullopt;
+
+  std::optional<CBufferMetadata> Result({CBufMD});
+
+  for (const MDNode *MD : CBufMD->operands()) {
+    assert(MD->getNumOperands() && "Invalid cbuffer metadata");
+
+    auto *Handle = cast<GlobalVariable>(
+        cast<ValueAsMetadata>(MD->getOperand(0))->getValue());
+    CBufferMapping &Mapping = Result->Mappings.emplace_back(Handle);
+
+    for (int I = 1, E = MD->getNumOperands(); I < E; ++I) {
+      Metadata *OpMD = MD->getOperand(I);
+      // Some members may be null if they've been optimized out.
+      if (!OpMD)
+        continue;
+      auto *V = cast<GlobalVariable>(cast<ValueAsMetadata>(OpMD)->getValue());
+      Mapping.Members.emplace_back(V, getMemberOffset(Handle, I - 1));
+    }
+  }
+
+  return Result;
+}
+
+void CBufferMetadata::eraseFromModule() {
+  // Remove the cbs named metadata
+  MD->eraseFromParent();
+}
+
+APInt hlsl::translateCBufArrayOffset(const DataLayout &DL, APInt Offset,
+                                     ArrayType *Ty) {
+  int64_t TypeSize = DL.getTypeSizeInBits(Ty->getElementType()) / 8;
+  int64_t RoundUp = alignTo(TypeSize, Align(CBufferRowSizeInBytes));
+  return Offset.udiv(TypeSize) * RoundUp;
+}
diff --git a/llvm/lib/Frontend/HLSL/CMakeLists.txt b/llvm/lib/Frontend/HLSL/CMakeLists.txt
index eda6cb8e69a49..07a0c845ceef6 100644
--- a/llvm/lib/Frontend/HLSL/CMakeLists.txt
+++ b/llvm/lib/Frontend/HLSL/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_llvm_component_library(LLVMFrontendHLSL
+  CBuffer.cpp
   HLSLResource.cpp
 
   ADDITIONAL_HEADER_DIRS
diff --git a/llvm/lib/Target/DirectX/CMakeLists.txt b/llvm/lib/Target/DirectX/CMakeLists.txt
index 13f8adbe4f132..c55028bc75dd6 100644
--- a/llvm/lib/Target/DirectX/CMakeLists.txt
+++ b/llvm/lib/Target/DirectX/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_target(DirectXCodeGen
   DirectXTargetMachine.cpp
   DirectXTargetTransformInfo.cpp
   DXContainerGlobals.cpp
+  DXILCBufferAccess.cpp
   DXILDataScalarization.cpp
   DXILFinalizeLinkage.cpp
   DXILFlattenArrays.cpp
diff --git a/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp b/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
new file mode 100644
index 0000000000000..7559f61b4cfb9
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
@@ -0,0 +1,210 @@
+//===- DXILCBufferAccess.cpp - Translate CBuffer Loads --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DXILCBufferAccess.h"
+#include "DirectX.h"
+#include "llvm/Frontend/HLSL/CBuffer.h"
+#include "llvm/Frontend/HLSL/HLSLResource.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicsDirectX.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils/Local.h"
+
+#define DEBUG_TYPE "dxil-cbuffer-access"
+using namespace llvm;
+
+namespace {
+/// Helper for building a `load.cbufferrow` intrinsic given a simple type.
+struct CBufferRowIntrin {
+  Intrinsic::ID IID;
+  Type *RetTy;
+  unsigned int EltSize;
+  unsigned int NumElts;
+
+  CBufferRowIntrin(const DataLayout &DL, Type *Ty) {
+    assert(Ty == Ty->getScalarType() && "Expected scalar type");
+
+    switch (DL.getTypeSizeInBits(Ty)) {
+    case 16:
+      IID = Intrinsic::dx_resource_load_cbufferrow_8;
+      RetTy = StructType::get(Ty, Ty, Ty, Ty, Ty, Ty, Ty, Ty);
+      EltSize = 2;
+      NumElts = 8;
+      break;
+    case 32:
+      IID = Intrinsic::dx_resource_load_cbufferrow_4;
+      RetTy = StructType::get(Ty, Ty, Ty, Ty);
+      EltSize = 4;
+      NumElts = 4;
+      break;
+    case 64:
+      IID = Intrinsic::dx_resource_load_cbufferrow_2;
+      RetTy = StructType::get(Ty, Ty);
+      EltSize = 8;
+      NumElts = 2;
+      break;
+    default:
+      llvm_unreachable("Only 16, 32, and 64 bit types supported");
+    }
+  }
+};
+} // namespace
+
+static size_t getOffsetForCBufferGEP(GEPOperator *GEP, GlobalVariable *Global,
+                                     const DataLayout &DL) {
+  // Since we should always have a constant offset, we should only ever have a
+  // single GEP of indirection from the Global.
+  assert(GEP->getPointerOperand() == Global &&
+         "Indirect access to resource handle");
+
+  APInt ConstantOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
+  bool Success = GEP->accumulateConstantOffset(DL, ConstantOffset);
+  (void)Success;
+  assert(Success && "Offsets into cbuffer globals must be constant");
+
+  if (auto *ATy = dyn_cast<ArrayType>(Global->getValueType()))
+    ConstantOffset = hlsl::translateCBufArrayOffset(DL, ConstantOffset, ATy);
+
+  return ConstantOffset.getZExtValue();
+}
+
+/// Replace access via cbuffer global with a load from the cbuffer handle
+/// itself.
+static void replaceAccess(LoadInst *LI, GlobalVariable *Global,
+                          GlobalVariable *HandleGV, size_t BaseOffset,
+                          SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
+  const DataLayout &DL = HandleGV->getDataLayout();
+
+  size_t Offset = BaseOffset;
+  if (auto *GEP = dyn_cast<GEPOperator>(LI->getPointerOperand()))
+    Offset += getOffsetForCBufferGEP(GEP, Global, DL);
+  else if (LI->getPointerOperand() != Global)
+    llvm_unreachable("Load instruction doesn't reference cbuffer global");
+
+  IRBuilder<> Builder(LI);
+  auto *Handle = Builder.CreateLoad(HandleGV->getValueType(), HandleGV,
+                                    HandleGV->getName());
+
+  Type *Ty = LI->getType();
+  CBufferRowIntrin Intrin(DL, Ty->getScalarType());
+  // The cbuffer consists of some number of 16-byte rows.
+  unsigned int CurrentRow = Offset / hlsl::CBufferRowSizeInBytes;
+  unsigned int CurrentIndex =
+      (Offset % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
+
+  auto *CBufLoad = Builder.CreateIntrinsic(
+      Intrin.RetTy, Intrin.IID,
+      {Handle, ConstantInt::get(Builder.getInt32Ty(), CurrentRow)}, nullptr,
+      LI->getName());
+  auto *Elt =
+      Builder.CreateExtractValue(CBufLoad, {CurrentIndex++}, LI->getName());
+
+  Value *Result = nullptr;
+  unsigned int Remaining =
+      ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
+  if (Remaining == 0) {
+    // We only have a single element, so we're done.
+    Result = Elt;
+
+    // However, if we loaded a <1 x T>, then we need to adjust the type here.
+    if (auto *VT = dyn_cast<FixedVectorType>(LI->getType())) {
+      assert(VT->getNumElements() == 1 && "Can't have multiple elements here");
+      Result = Builder.CreateInsertElement(PoisonValue::get(VT), Result,
+                                           Builder.getInt32(0));
+    }
+  } else {
+    // Walk each element and extract it, wrapping to new rows as needed.
+    SmallVector<Value *> Extracts{Elt};
+    while (Remaining--) {
+      CurrentIndex %= Intrin.NumElts;
+
+      if (CurrentIndex == 0)
+        CBufLoad = Builder.CreateIntrinsic(
+            Intrin.RetTy, Intrin.IID,
+            {Handle, ConstantInt::get(Builder.getInt32Ty(), ++CurrentRow)},
+            nullptr, LI->getName());
+
+      Extracts.push_back(Builder.CreateExtractValue(CBufLoad, {CurrentIndex++},
+                                                    LI->getName()));
+    }
+
+    // Finally, we build up the original loaded value.
+    Result = PoisonValue::get(Ty);
+    for (int I = 0, E = Extracts.size(); I < E; ++I)
+      Result =
+          Builder.CreateInsertElement(Result, Extracts[I], Builder.getInt32(I));
+  }
+
+  LI->replaceAllUsesWith(Result);
+  DeadInsts.push_back(LI);
+}
+
+static void replaceAccessesWithHandle(GlobalVariable *Global,
+                                      GlobalVariable *HandleGV,
+                                      size_t BaseOffset) {
+  SmallVector<WeakTrackingVH> DeadInsts;
+
+  SmallVector<User *> ToProcess{Global->users()};
+  while (!ToProcess.empty()) {
+    User *Cur = ToProcess.pop_back_val();
+
+    // If we have a load instruction, replace the access.
+    if (auto *LI = dyn_cast<LoadInst>(Cur)) {
+      replaceAccess(LI, Global, HandleGV, BaseOffset, DeadInsts);
+      continue;
+    }
+
+    // Otherwise, walk users looking for a load...
+    ToProcess.append(Cur->user_begin(), Cur->user_end());
+  }
+  RecursivelyDeleteTriviallyDeadInstructions(DeadInsts);
+}
+
+static bool replaceCBufferAccesses(Module &M) {
+  std::optional<hlsl::CBufferMetadata> CBufMD = hlsl::CBufferMetadata::get(M);
+  if (!CBufMD)
+    return false;
+
+  for (const hlsl::CBufferMapping &Mapping : *CBufMD)
+    for (const hlsl::CBufferMember &Member : Mapping.Members) {
+      replaceAccessesWithHandle(Member.GV, Mapping.Handle, Member.Offset);
+      Member.GV->removeFromParent();
+    }
+
+  CBufMD->eraseFromModule();
+  return true;
+}
+
+PreservedAnalyses DXILCBufferAccess::run(Module &M, ModuleAnalysisManager &AM) {
+  PreservedAnalyses PA;
+  bool Changed = replaceCBufferAccesses(M);
+
+  if (!Changed)
+    return PreservedAnalyses::all();
+  return PA;
+}
+
+namespace {
+class DXILCBufferAccessLegacy : public ModulePass {
+public:
+  bool runOnModule(Module &M) override { return replaceCBufferAccesses(M); }
+  StringRef getPassName() const override { return "DXIL CBuffer Access"; }
+  DXILCBufferAccessLegacy() : ModulePass(ID) {}
+
+  static char ID; // Pass identification.
+};
+char DXILCBufferAccessLegacy::ID = 0;
+} // end anonymous namespace
+
+INITIALIZE_PASS(DXILCBufferAccessLegacy, DEBUG_TYPE, "DXIL CBuffer Access",
+                false, false)
+
+ModulePass *llvm::createDXILCBufferAccessLegacyPass() {
+  return new DXILCBufferAccessLegacy();
+}
diff --git a/llvm/lib/Target/DirectX/DXILCBufferAccess.h b/llvm/lib/Target/DirectX/DXILCBufferAccess.h
new file mode 100644
index 0000000000000..6c1cde164004e
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DXILCBufferAccess.h
@@ -0,0 +1,28 @@
+//===- DXILCBufferAccess.h - Translate CBuffer Loads ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// \file Pass for replacing loads from cbuffers in the cbuffer address space to
+// cbuffer load intrinsics.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_DIRECTX_DXILCBUFFERACCESS_H
+#define LLVM_LIB_TARGET_DIRECTX_DXILCBUFFERACCESS_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class DXILCBufferAccess : public PassInfoMixin<DXILCBufferAccess> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_DIRECTX_DXILCBUFFERACCESS_H
diff --git a/llvm/lib/Target/DirectX/DirectX.h b/llvm/lib/Target/DirectX/DirectX.h
index 96a8a08c875f8..c0eb221d12203 100644
--- a/llvm/lib/Target/DirectX/DirectX.h
+++ b/llvm/lib/Target/DirectX/DirectX.h
@@ -35,6 +35,12 @@ void initializeDXILIntrinsicExpansionLegacyPass(PassRegistry &);
 /// Pass to expand intrinsic operations that lack DXIL opCodes
 ModulePass *createDXILIntrinsicExpansionLegacyPass();
 
+/// Initializer for DXIL CBuffer Access Pass
+void initializeDXILCBufferAccessLegacyPass(PassRegistry &);
+
+/// Pass to translate loads in the cbuffer address space to intrinsics
+ModulePass *createDXILCBufferAccessLegacyPass();
+
 /// Initializer for DXIL Data Scalarization Pass
 void initializeDXILDataScalarizationLegacyPass(PassRegistry &);
 
diff --git a/llvm/lib/Target/DirectX/DirectXPassRegistry.def b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
index 87d91ead1896f..37093f16680a9 100644
--- a/llvm/lib/Target/DirectX/DirectXPassRegistry.def
+++ b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
@@ -23,6 +23,7 @@ MODULE_ANALYSIS("dxil-root-signature-analysis", dxil::RootSignatureAnalysis())
 #ifndef MODULE_PASS
 #define MODULE_PASS(NAME, CREATE_PASS)
 #endif
+MODULE_PASS("dxil-cbuffer-access", DXILCBufferAccess())
 MODULE_PASS("dxil-data-scalarization", DXILDataScalarization())
 MODULE_PASS("dxil-flatten-arrays", DXILFlattenArrays())
 MODULE_PASS("dxil-intrinsic-expansion", DXILIntrinsicExpansion())
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index 747e4b3eb9411..41f6f37a41f9d 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DirectXTargetMachine.h"
+#include "DXILCBufferAccess.h"
 #include "DXILDataScalarization.h"
 #include "DXILFlattenArrays.h"
 #include "DXILIntrinsicExpansion.h"
@@ -65,6 +66,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeDirectXTarget() {
   initializeRootSignatureAnalysisWrapperPass(*PR);
   initializeDXILFinalizeLinkageLegacyPass(*PR);
   initializeDXILPrettyPrinterLegacyPass(*PR);
+  initializeDXILCBufferAccessLegacyPass(*PR);
 }
 
 class DXILTargetObjectFile : public TargetLoweringObjectFile {
@@ -96,6 +98,7 @@ class DirectXPassConfig : public TargetPassConfig {
   void addCodeGenPrepare() override {
     addPass(createDXILFinalizeLinkageLegacyPass());
     addPass(createDXILIntrinsicExpansionLegacyPass());
+    addPass(createDXILCBufferAccessLegacyPass());
     addPass(createDXILDataScalarizationLegacyPass());
     addPass(createDXILFlattenArraysLegacyPass());
     addPass(createDXILResourceAccessLegacyPass());
diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/array-typedgep.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/array-typedgep.ll
new file mode 100644
index 0000000000000..dbd01b323aa2a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/CBufferAccess/array-typedgep.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s
+
+; cbuffer CB : register(b0) {
+;   float a1[3];
+; }
+%__cblayout_CB = type <{ [3 x float] }>
+
+@CB.cb = global target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 36, 0)) poison
+; CHECK: @CB.cb =
+; CHECK-NOT: external {{.*}} addrspace(2) global
+@a1 = external addrspace(2) global [3 x float], align 4
+
+; CHECK: define void @f
+define void @f(ptr %dst) {
+entry:
+  %CB.cb_h = call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 36, 0)) @llvm.dx.resource.handlefrombinding.tdx.CBuffer_tdx.Layout_s___cblayout_CBs_36_0tt(i32 0, i32 0, i32 1, i32 0, i1 false)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 36, 0)) %CB.cb_h, ptr @CB.cb, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+  ; CHECK: [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+  ; CHECK: store float [[X]], ptr %dst
+  %a1 = load float, ptr addrspace(2) getelementptr inbounds ([3 x float], ptr addrspace(2) @a1, i32 0, i32 1), align 4
+  store float %a1, ptr %dst, align 32
+
+  ret void
+}
+
+; CHECK-NOT: !hlsl.cbs =
+!hlsl.cbs = !{!0}
+
+!0 = !{ptr @CB.cb, ptr addrspace(2) @a1}
diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/arrays.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/arrays.ll
new file mode 100644
index 0000000000000..42d7943953b84
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/CBufferAccess/arrays.ll
@@ -0,0 +1,124 @@
+; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s
+
+; cbuffer CB : register(b0) {
+;   float a1[3];
+;   double3 a2[2];
+;   float16_t a3[2][2];
+;   uint64_t a4[3];
+;   int4 a5[2][3][4];
+;   uint16_t a6[1];
+;   int64_t a7[2];
+;   bool a8[4];
+; }
+%__cblayout_CB = type <{ [3 x float], [2 x <3 x double>], [2 x [2 x half]], [3 x i64], [2 x [3 x [4 x <4 x i32>]]], [1 x i16], [2 x i64], [4 x i32] }>
+
+@CB.cb = local_unnamed_addr global target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 608, 624, 656)) poison
+; CHECK: @CB.cb =
+; CHECK-NOT: external {{.*}} addrspace(2) global
+@a1 = external local_unnamed_addr addrspace(2) global [3 x float], align 4
+@a2 = external local_unnamed_addr addrspace(2) global [2 x <3 x double>], align 32
+@a3 = external local_unnamed_addr addrspace(2) global [2 x [2 x half]], align 2
+@a4 = external local_unnamed_addr addrspace(2) global [3 x i64], align 8
+@a5 = external local_unnamed_addr addrspace(2) global [2 x [3 x [4 x <4 x i32>]]], align 16
+@a6 = external local_unnamed_addr addrspace(2) global [1 x i16], align 2
+@a7 = external local_unnamed_addr addrspace(2) global [2 x i64], align 8
+@a8 = external local_unnamed_addr addrspace(2) global [4 x i32], align 4
+
+; CHECK: define void @f
+define void @f(ptr %dst) {
+entry:
+  %CB.cb_h.i.i = tail call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 608, 624, 656)) @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 608, 624, 656)) %CB.cb_h.i.i, ptr @CB.cb, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+  ; CHECK: [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+  ; CHECK: store float [[X]], ptr %dst
+  %a1 = load float, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a1, i32 4), align 4
+  store float %a1, ptr %dst, align 32
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 5)
+  ; CHECK: [[X:%.*]] = extractvalue { double, double } [[LOAD]], 0
+  ; CHECK: [[Y:%.*]] = extractvalue { double, double } [[LOAD]], 1
+  ; CHECK: [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 6)
+  ; CHECK: [[Z:%.*]] = extractvalue { double, double } [[LOAD]], 0
+  ; CHECK: [[VEC0:%.*]] = insertelement <3 x double> poison, double [[X]], i32 0
+  ; CHECK: [[VEC1:%.*]] = insertelement <3 x double> [[VEC0]], double [[Y]], i32 1
+  ; CHECK: [[VEC2:%.*]] = insertelement <3 x double> [[VEC1]], double [[Z]], i32 2
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 8
+  ; CHECK: store <3 x double> [[VEC2]], ptr [[PTR]]
+  %a2 = load <3 x double>, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a2, i32 32), align 8
+  %a2.i = getelementptr inbounds nuw i8, ptr %dst, i32 8
+  store <3 x double> %a2, ptr %a2.i, align 32
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 8)
+  ; CHECK: [[X:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 32
+  ; CHECK: store half [[X]], ptr [[PTR]]
+  %a3 = load half, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a3, i32 6), align 2
+  %a3.i = getelementptr inbounds nuw i8, ptr %dst, i32 32
+  store half %a3, ptr %a3.i, align 2
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 12)
+  ; CHECK: [[X:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 40
+  ; CHECK: store i64 [[X]], ptr [[PTR]]
+  %a4 = load i64, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a4, i32 8), align 8
+  %a4.i = getelementptr inbounds nuw i8, ptr %dst, i32 40
+  store i64 %a4, ptr %a4.i, align 8
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 26)
+  ; CHECK: [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+  ; CHECK: [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+  ; CHECK: [[Z:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 2
+  ; CHECK: [[A:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 3
+  ; CHECK: [[VEC0:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i32 0
+  ; CHECK: [[VEC1:%.*]] = insertelement <4 x i32> [[VEC0]], i32 [[Y]], i32 1
+  ; CHECK: [[VEC2:%.*]] = insertelement <4 x i32> [[VEC1]], i32 [[Z]], i32 2
+  ; CHECK: [[VEC3:%.*]] = insertelement <4 x i32> [[VEC2]], i32 [[A]], i32 3
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 48
+  ; CHECK: store <4 x i32> [[VEC3]], ptr [[PTR]]
+  %a5 = load <4 x i32>, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a5, i32 272), align 4
+  %a5.i = getelementptr inbounds nuw i8, ptr %dst, i32 48
+  store <4 x i32> %a5, ptr %a5.i, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i16, i16, i16, i16, i16, i16, i16, i16 } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 38)
+  ; CHECK: [[X:%.*]] = extractvalue { i16, i16, i16, i16, i16, i16, i16, i16 } [[LOAD]], 0
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 64
+  ; CHECK: store i16 [[X]], ptr [[PTR]]
+  %a6 = load i16, ptr addrspace(2) @a6, align 2
+  %a6.i = getelementptr inbounds nuw i8, ptr %dst, i32 64
+  store i16 %a6, ptr %a6.i, align 2
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 40)
+  ; CHECK: [[X:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 72
+  ; CHECK: store i64 [[X]], ptr [[PTR]]
+  %a7 = load i64, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a7, i32 8), align 8
+  %a7.i = getelementptr inbounds nuw i8, ptr %dst, i32 72
+  store i64 %a7, ptr %a7.i, align 8
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 42)
+  ; CHECK: [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 80
+  ; CHECK: store i32 [[X]], ptr [[PTR]]
+  %a8 = load i32, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a8, i32 4), align 4, !range !1, !noundef !2
+  %a8.i = getelementptr inbounds nuw i8, ptr %dst, i32 80
+  store i32 %a8, ptr %a8.i, align 4
+
+  ret void
+}
+
+; CHECK-NOT: !hlsl.cbs =
+!hlsl.cbs = !{!0}
+
+!0 = !{ptr @CB.cb, ptr addrspace(2) @a1, ptr addrspace(2) @a2, ptr addrspace(2) @a3, ptr addrspace(2) @a4, ptr addrspace(2) @a5, ptr addrspace(2) @a6, ptr addrspace(2) @a7, ptr addrspace(2) @a8}
+!1 = !{i32 0, i32 2}
+!2 = !{}
diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/float.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/float.ll
new file mode 100644
index 0000000000000..d7272b449166d
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/CBufferAccess/float.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s
+
+%__cblayout_CB = type <{ float }>
+
+@CB.cb = local_unnamed_addr global target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 4, 0)) poison
+; CHECK: @CB.cb =
+; CHECK-NOT: external {{.*}} addrspace(2) global
+@x = external local_unnamed_addr addrspace(2) global float, align 4
+
+; CHECK: define void @f
+define void @f(ptr %dst) {
+entry:
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+  ; CHECK: [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+  ; CHECK: store float [[X]], ptr %dst
+  %x = load float, ptr addrspace(2) @x, align 4
+  store float %x, ptr %dst, align 4
+  ret void
+}
+
+; CHECK-NOT: !hlsl.cbs =
+!hlsl.cbs = !{!0}
+
+!0 = !{ptr @CB.cb, ptr addrspace(2) @x}
diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/gep-ce-two-uses.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/gep-ce-two-uses.ll
new file mode 100644
index 0000000000000..abe087dbe6100
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/CBufferAccess/gep-ce-two-uses.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s
+
+; cbuffer CB : register(b0) {
+;   float a1[3];
+; }
+%__cblayout_CB = type <{ [3 x float] }>
+
+@CB.cb = local_unnamed_addr global target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 36, 0)) poison
+; CHECK: @CB.cb =
+; CHECK-NOT: external {{.*}} addrspace(2) global
+@a1 = external local_unnamed_addr addrspace(2) global [3 x float], align 4
+
+; CHECK: define void @f
+define void @f(ptr %dst) {
+entry:
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+  ; CHECK: [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+  ; CHECK: store float [[X]], ptr %dst
+  %a1 = load float, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a1, i32 4), align 4
+  store float %a1, ptr %dst, align 32
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+  ; CHECK: [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+  ; CHECK: store float [[X]], ptr %dst
+  %a2 = load float, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @a1, i32 4), align 4
+  store float %a2, ptr %dst, align 32
+
+  ret void
+}
+
+; CHECK-NOT: !hlsl.cbs =
+!hlsl.cbs = !{!0}
+
+!0 = !{ptr @CB.cb, ptr addrspace(2) @a1}
diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/scalars.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/scalars.ll
new file mode 100644
index 0000000000000..125d6b66c0107
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/CBufferAccess/scalars.ll
@@ -0,0 +1,100 @@
+; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s
+
+; cbuffer CB {
+;   float a1;     // offset  0, size  4
+;   int a2;       // offset  4, size  4
+;   bool a3;      // offset  8, size  4
+;   float16_t a4; // offset 12, size  2
+;   uint16_t a5;  // offset 14, size  2
+;   double a6;    // offset 16, size  8
+;   int64_t a7;   // offset 24, size  8
+; }
+%__cblayout_CB = type <{ float, i32, i32, half, i16, double, i64 }>
+
+@CB.cb = local_unnamed_addr global target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 32, 0, 4, 8, 12, 14, 16, 24)) poison
+; CHECK: @CB.cb =
+; CHECK-NOT: external {{.*}} addrspace(2) global
+@a1 = external local_unnamed_addr addrspace(2) global float, align 4
+@a2 = external local_unnamed_addr addrspace(2) global i32, align 4
+@a3 = external local_unnamed_addr addrspace(2) global i32, align 4
+@a4 = external local_unnamed_addr addrspace(2) global half, align 2
+@a5 = external local_unnamed_addr addrspace(2) global i16, align 2
+@a6 = external local_unnamed_addr addrspace(2) global double, align 8
+@a7 = external local_unnamed_addr addrspace(2) global i64, align 8
+
+; CHECK: define void @f
+define void @f(ptr %dst) {
+entry:
+  %CB.cb_h.i.i = tail call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 32, 0, 4, 8, 12, 14, 16, 24)) @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 32, 0, 4, 8, 12, 14, 16, 24)) %CB.cb_h.i.i, ptr @CB.cb, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+  ; CHECK: [[A1:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+  ; CHECK: store float [[A1]], ptr %dst
+  %a1 = load float, ptr addrspace(2) @a1, align 4
+  store float %a1, ptr %dst, align 8
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+  ; CHECK: [[A2:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 4
+  ; CHECK: store i32 [[A2]], ptr [[PTR]]
+  %a2 = load i32, ptr addrspace(2) @a2, align 4
+  %a2.i = getelementptr inbounds nuw i8, ptr %dst, i32 4
+  store i32 %a2, ptr %a2.i, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+  ; CHECK: [[A3:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 2
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 8
+  ; CHECK: store i32 [[A3]], ptr [[PTR]]
+  %a3 = load i32, ptr addrspace(2) @a3, align 4, !range !1, !noundef !2
+  %a3.i = getelementptr inbounds nuw i8, ptr %dst, i32 8
+  store i32 %a3, ptr %a3.i, align 8
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+  ; CHECK: [[A4:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 6
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 12
+  ; CHECK: store half [[A4]], ptr [[PTR]]
+  %a4 = load half, ptr addrspace(2) @a4, align 2
+  %a4.i = getelementptr inbounds nuw i8, ptr %dst, i32 12
+  store half %a4, ptr %a4.i, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i16, i16, i16, i16, i16, i16, i16, i16 } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+  ; CHECK: [[A5:%.*]] = extractvalue { i16, i16, i16, i16, i16, i16, i16, i16 } [[LOAD]], 7
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 14
+  ; CHECK: store i16 [[A5]], ptr [[PTR]]
+  %a5 = load i16, ptr addrspace(2) @a5, align 2
+  %a5.i = getelementptr inbounds nuw i8, ptr %dst, i32 14
+  store i16 %a5, ptr %a5.i, align 2
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+  ; CHECK: [[A6:%.*]] = extractvalue { double, double } [[LOAD]], 0
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 16
+  ; CHECK: store double [[A6]], ptr [[PTR]]
+  %a6 = load double, ptr addrspace(2) @a6, align 8
+  %a6.i = getelementptr inbounds nuw i8, ptr %dst, i32 16
+  store double %a6, ptr %a6.i, align 8
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+  ; CHECK: [[A7:%.*]] = extractvalue { i64, i64 } [[LOAD]], 1
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 24
+  ; CHECK: store i64 [[A7]], ptr [[PTR]]
+  %a7 = load i64, ptr addrspace(2) @a7, align 8
+  %a7.i = getelementptr inbounds nuw i8, ptr %dst, i32 24
+  store i64 %a7, ptr %a7.i, align 8
+
+  ret void
+}
+
+; CHECK-NOT: !hlsl.cbs =
+!hlsl.cbs = !{!0}
+
+!0 = !{ptr @CB.cb, ptr addrspace(2) @a1, ptr addrspace(2) @a2, ptr addrspace(2) @a3, ptr addrspace(2) @a4, ptr addrspace(2) @a5, ptr addrspace(2) @a6, ptr addrspace(2) @a7}
+!1 = !{i32 0, i32 2}
+!2 = !{}
diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/vectors.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/vectors.ll
new file mode 100644
index 0000000000000..6addf7482ac37
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/CBufferAccess/vectors.ll
@@ -0,0 +1,119 @@
+; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s
+
+; cbuffer CB {
+;   float3 a1;     // offset   0, size 12 (+4)
+;   double3 a2;    // offset  16, size 24
+;   float16_t2 a3; // offset  40, size  4 (+4)
+;   uint64_t3 a4;  // offset  48, size 24 (+8)
+;   int4 a5;       // offset  80, size 16
+;   uint16_t3 a6;  // offset  96, size  6 (+10)
+; };
+%__cblayout_CB = type <{ <3 x float>, <3 x double>, <2 x half>, <3 x i64>, <4 x i32>, <3 x i16> }>
+
+@CB.cb = local_unnamed_addr global target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 102, 0, 16, 40, 48, 80, 96)) poison
+; CHECK: @CB.cb =
+; CHECK-NOT: external {{.*}} addrspace(2) global
+@a1 = external local_unnamed_addr addrspace(2) global <3 x float>, align 16
+@a2 = external local_unnamed_addr addrspace(2) global <3 x double>, align 32
+@a3 = external local_unnamed_addr addrspace(2) global <2 x half>, align 4
+@a4 = external local_unnamed_addr addrspace(2) global <3 x i64>, align 32
+@a5 = external local_unnamed_addr addrspace(2) global <4 x i32>, align 16
+@a6 = external local_unnamed_addr addrspace(2) global <3 x i16>, align 8
+
+; CHECK: define void @f
+define void @f(ptr %dst) {
+entry:
+  %CB.cb_h.i.i = tail call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 102, 0, 16, 40, 48, 80, 96)) @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 102, 0, 16, 40, 48, 80, 96)) %CB.cb_h.i.i, ptr @CB.cb, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+  ; CHECK: [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+  ; CHECK: [[Y:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 1
+  ; CHECK: [[Z:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 2
+  ; CHECK: [[VEC0:%.*]] = insertelement <3 x float> poison, float [[X]], i32 0
+  ; CHECK: [[VEC1:%.*]] = insertelement <3 x float> [[VEC0]], float [[Y]], i32 1
+  ; CHECK: [[VEC2:%.*]] = insertelement <3 x float> [[VEC1]], float [[Z]], i32 2
+  ; CHECK: store <3 x float> [[VEC2]], ptr %dst
+  %a1 = load <3 x float>, ptr addrspace(2) @a1, align 16
+  store <3 x float> %a1, ptr %dst, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+  ; CHECK: [[X:%.*]] = extractvalue { double, double } [[LOAD]], 0
+  ; CHECK: [[Y:%.*]] = extractvalue { double, double } [[LOAD]], 1
+  ; CHECK: [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 2)
+  ; CHECK: [[Z:%.*]] = extractvalue { double, double } [[LOAD]], 0
+  ; CHECK: [[VEC0:%.*]] = insertelement <3 x double> poison, double [[X]], i32 0
+  ; CHECK: [[VEC1:%.*]] = insertelement <3 x double> [[VEC0]], double [[Y]], i32 1
+  ; CHECK: [[VEC2:%.*]] = insertelement <3 x double> [[VEC1]], double [[Z]], i32 2
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 16
+  ; CHECK: store <3 x double> [[VEC2]], ptr [[PTR]]
+  %a2 = load <3 x double>, ptr addrspace(2) @a2, align 32
+  %a2.i = getelementptr inbounds nuw i8, ptr %dst, i32 16
+  store <3 x double> %a2, ptr %a2.i, align 8
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 2)
+  ; CHECK: [[X:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 4
+  ; CHECK: [[Y:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 5
+  ; CHECK: [[VEC0:%.*]] = insertelement <2 x half> poison, half [[X]], i32 0
+  ; CHECK: [[VEC1:%.*]] = insertelement <2 x half> [[VEC0]], half [[Y]], i32 1
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 40
+  ; CHECK: store <2 x half> [[VEC1]], ptr [[PTR]]
+  %a3 = load <2 x half>, ptr addrspace(2) @a3, align 4
+  %a3.i = getelementptr inbounds nuw i8, ptr %dst, i32 40
+  store <2 x half> %a3, ptr %a3.i, align 2
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 3)
+  ; CHECK: [[X:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+  ; CHECK: [[Y:%.*]] = extractvalue { i64, i64 } [[LOAD]], 1
+  ; CHECK: [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 4)
+  ; CHECK: [[Z:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+  ; CHECK: [[VEC0:%.*]] = insertelement <3 x i64> poison, i64 [[X]], i32 0
+  ; CHECK: [[VEC1:%.*]] = insertelement <3 x i64> [[VEC0]], i64 [[Y]], i32 1
+  ; CHECK: [[VEC2:%.*]] = insertelement <3 x i64> [[VEC1]], i64 [[Z]], i32 2
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 48
+  ; CHECK: store <3 x i64> [[VEC2]], ptr [[PTR]]
+  %a4 = load <3 x i64>, ptr addrspace(2) @a4, align 32
+  %a4.i = getelementptr inbounds nuw i8, ptr %dst, i32 48
+  store <3 x i64> %a4, ptr %a4.i, align 8
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 5)
+  ; CHECK: [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+  ; CHECK: [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+  ; CHECK: [[Z:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 2
+  ; CHECK: [[A:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 3
+  ; CHECK: [[VEC0:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i32 0
+  ; CHECK: [[VEC1:%.*]] = insertelement <4 x i32> [[VEC0]], i32 [[Y]], i32 1
+  ; CHECK: [[VEC2:%.*]] = insertelement <4 x i32> [[VEC1]], i32 [[Z]], i32 2
+  ; CHECK: [[VEC3:%.*]] = insertelement <4 x i32> [[VEC2]], i32 [[A]], i32 3
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 72
+  ; CHECK: store <4 x i32> [[VEC3]], ptr [[PTR]]
+  %a5 = load <4 x i32>, ptr addrspace(2) @a5, align 16
+  %a5.i = getelementptr inbounds nuw i8, ptr %dst, i32 72
+  store <4 x i32> %a5, ptr %a5.i, align 4
+
+  ; CHECK: [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb
+  ; CHECK: [[LOAD:%.*]] = call { i16, i16, i16, i16, i16, i16, i16, i16 } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 6)
+  ; CHECK: [[X:%.*]] = extractvalue { i16, i16, i16, i16, i16, i16, i16, i16 } [[LOAD]], 0
+  ; CHECK: [[Y:%.*]] = extractvalue { i16, i16, i16, i16, i16, i16, i16, i16 } [[LOAD]], 1
+  ; CHECK: [[Z:%.*]] = extractvalue { i16, i16, i16, i16, i16, i16, i16, i16 } [[LOAD]], 2
+  ; CHECK: [[VEC0:%.*]] = insertelement <3 x i16> poison, i16 [[X]], i32 0
+  ; CHECK: [[VEC1:%.*]] = insertelement <3 x i16> [[VEC0]], i16 [[Y]], i32 1
+  ; CHECK: [[VEC2:%.*]] = insertelement <3 x i16> [[VEC1]], i16 [[Z]], i32 2
+  ; CHECK: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr %dst, i32 88
+  ; CHECK: store <3 x i16> [[VEC2]], ptr [[PTR]]
+  %a6 = load <3 x i16>, ptr addrspace(2) @a6, align 8
+  %a6.i = getelementptr inbounds nuw i8, ptr %dst, i32 88
+  store <3 x i16> %a6, ptr %a6.i, align 2
+
+  ret void
+}
+
+; CHECK-NOT: !hlsl.cbs =
+!hlsl.cbs = !{!0}
+
+!0 = !{ptr @CB.cb, ptr addrspace(2) @a1, ptr addrspace(2) @a2, ptr addrspace(2) @a3, ptr addrspace(2) @a4, ptr addrspace(2) @a5, ptr addrspace(2) @a6}
diff --git a/llvm/test/CodeGen/DirectX/llc-pipeline.ll b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
index ee70cec534bc5..b1bd9f16f4efa 100644
--- a/llvm/test/CodeGen/DirectX/llc-pipeline.ll
+++ b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
@@ -15,6 +15,7 @@
 ; CHECK-NEXT: ModulePass Manager
 ; CHECK-NEXT:   DXIL Finalize Linkage
 ; CHECK-NEXT:   DXIL Intrinsic Expansion
+; CHECK-NEXT:   DXIL CBuffer Access
 ; CHECK-NEXT:   DXIL Data Scalarization
 ; CHECK-NEXT:   DXIL Array Flattener
 ; CHECK-NEXT:   FunctionPass Manager

From 40460a5cf76c973a783fb2f5229e1076398df96e Mon Sep 17 00:00:00 2001
From: Sudharsan Veeravalli <quic_svs@quicinc.com>
Date: Wed, 16 Apr 2025 11:19:13 +0530
Subject: [PATCH 079/710] [RISCV] Add basic ISel patterns for Xqcilo
 instructions (#135901)

This patch adds basic instruction selection patterns for generating the
48 bit load/store instructions that are a part of the Qualcomm uC Xqcilo
vendor extension.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td |  27 ++++
 llvm/test/CodeGen/RISCV/xqcilo.ll           | 143 ++++++++++++++++++++
 2 files changed, 170 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/xqcilo.ll

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index 2458bda80b1d6..6736b0f1d0328 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -159,6 +159,11 @@ def bare_simm32_lsb0 : Operand<OtherVT> {
   let OperandType = "OPERAND_PCREL";
 }
 
+def AddLike: PatFrags<(ops node:$A, node:$B),
+                      [(add node:$A, node:$B), (or node:$A, node:$B)], [{
+    return CurDAG->isBaseWithConstantOffset(SDValue(N, 0));
+}]>;
+
 //===----------------------------------------------------------------------===//
 // Instruction Formats
 //===----------------------------------------------------------------------===//
@@ -1239,6 +1244,14 @@ class PatGprNoX0Simm32NoSimm26<SDPatternOperator OpNode, RVInst48 Inst>
     : Pat<(i32 (OpNode (i32 GPRNoX0:$rs1), simm32_nosimm26:$imm)),
           (Inst GPRNoX0:$rs1, simm32_nosimm26:$imm)>;
 
+class QC48LdPat<PatFrag LoadOp, RVInst48 Inst>
+    : Pat<(i32 (LoadOp (AddLike (i32 GPR:$rs1), simm26_nosimm12:$imm26))),
+          (Inst GPR:$rs1, simm26_nosimm12:$imm26)>;
+
+class QC48StPat<PatFrag StoreOp, RVInst48 Inst>
+    : Pat<(StoreOp (i32 GPR:$rs2), (AddLike (i32 GPR:$rs1), simm26_nosimm12:$imm26)),
+          (Inst GPR:$rs2, GPR:$rs1, simm26_nosimm12:$imm26)>;
+
 /// Simple arithmetic operations
 
 let Predicates = [HasVendorXqcilia, IsRV32] in {
@@ -1253,5 +1266,19 @@ def : PatGprNoX0Simm26NoSimm12<or, QC_E_ORI>;
 def : PatGprNoX0Simm26NoSimm12<xor, QC_E_XORI>;
 } // Predicates = [HasVendorXqcilia, IsRV32]
 
+let Predicates = [HasVendorXqcilo, IsRV32], AddedComplexity = 2 in {
+  def : QC48LdPat<sextloadi8, QC_E_LB>;
+  def : QC48LdPat<extloadi8, QC_E_LBU>; // Prefer unsigned due to no c.lb in Zcb.
+  def : QC48LdPat<sextloadi16, QC_E_LH>;
+  def : QC48LdPat<extloadi16, QC_E_LH>;
+  def : QC48LdPat<load, QC_E_LW>;
+  def : QC48LdPat<zextloadi8, QC_E_LBU>;
+  def : QC48LdPat<zextloadi16, QC_E_LHU>;
+
+  def : QC48StPat<truncstorei8, QC_E_SB>;
+  def : QC48StPat<truncstorei16, QC_E_SH>;
+  def : QC48StPat<store, QC_E_SW>;
+} // Predicates = [HasVendorXqcilo, IsRV32], AddedComplexity = 2
+
 let Predicates = [HasVendorXqciint, IsRV32] in
 def : Pat<(riscv_mileaveret_glue), (QC_C_MILEAVERET)>;
diff --git a/llvm/test/CodeGen/RISCV/xqcilo.ll b/llvm/test/CodeGen/RISCV/xqcilo.ll
new file mode 100644
index 0000000000000..fb06f21b3ab98
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xqcilo.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32I
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcilo -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32IXQCILO
+
+define i32 @lb_ri(i8* %a) {
+; RV32I-LABEL: lb_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 2
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lb a0, 1808(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILO-LABEL: lb_ri:
+; RV32IXQCILO:       # %bb.0:
+; RV32IXQCILO-NEXT:    qc.e.lb a0, 10000(a0)
+; RV32IXQCILO-NEXT:    ret
+  %1 = getelementptr i8, i8* %a, i32 10000
+  %2 = load i8, i8* %1
+  %3 = sext i8 %2 to i32
+  ret i32 %3
+}
+
+define i32 @lbu_ri(i8* %a) {
+; RV32I-LABEL: lbu_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1048574
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lbu a0, 192(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILO-LABEL: lbu_ri:
+; RV32IXQCILO:       # %bb.0:
+; RV32IXQCILO-NEXT:    qc.e.lbu a0, -8000(a0)
+; RV32IXQCILO-NEXT:    ret
+  %1 = getelementptr i8, i8* %a, i32 -8000
+  %2 = load i8, i8* %1
+  %3 = zext i8 %2 to i32
+  ret i32 %3
+}
+
+define i32 @lh_ri(i16* %a) {
+; RV32I-LABEL: lh_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 11
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lhu a0, -612(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILO-LABEL: lh_ri:
+; RV32IXQCILO:       # %bb.0:
+; RV32IXQCILO-NEXT:    qc.e.lhu a0, 44444(a0)
+; RV32IXQCILO-NEXT:    ret
+  %1 = getelementptr i16, i16* %a, i32 22222
+  %2 = load i16, i16* %1
+  %3 = zext i16 %2 to i32
+  ret i32 %3
+}
+
+define i32 @lhu_ri(i16* %a) {
+; RV32I-LABEL: lhu_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1048570
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lhu a0, 120(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILO-LABEL: lhu_ri:
+; RV32IXQCILO:       # %bb.0:
+; RV32IXQCILO-NEXT:    qc.e.lhu a0, -24456(a0)
+; RV32IXQCILO-NEXT:    ret
+  %1 = getelementptr i16, i16* %a, i32 -12228
+  %2 = load i16, i16* %1
+  %3 = zext i16 %2 to i32
+  ret i32 %3
+}
+
+define i32 @lw_ri(i32* %a) {
+; RV32I-LABEL: lw_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, 2047
+; RV32I-NEXT:    lw a0, 1953(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILO-LABEL: lw_ri:
+; RV32IXQCILO:       # %bb.0:
+; RV32IXQCILO-NEXT:    qc.e.lw a0, 4000(a0)
+; RV32IXQCILO-NEXT:    ret
+  %1 = getelementptr i32, i32* %a, i32 1000
+  %2 = load i32, i32* %1
+  ret i32 %2
+}
+
+define void @sb_ri(i8* %a, i8 %b) {
+; RV32I-LABEL: sb_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 2
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    sb a1, 1808(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILO-LABEL: sb_ri:
+; RV32IXQCILO:       # %bb.0:
+; RV32IXQCILO-NEXT:    qc.e.sb a1, 10000(a0)
+; RV32IXQCILO-NEXT:    ret
+  %1 = getelementptr i8, i8* %a, i32 10000
+  store i8 %b, i8* %1
+  ret void
+}
+
+define void @sh_ri(i16* %a, i16 %b) {
+; RV32I-LABEL: sh_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 11
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    sh a1, -612(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILO-LABEL: sh_ri:
+; RV32IXQCILO:       # %bb.0:
+; RV32IXQCILO-NEXT:    qc.e.sh a1, 44444(a0)
+; RV32IXQCILO-NEXT:    ret
+  %1 = getelementptr i16, i16* %a, i32 22222
+  store i16 %b, i16* %1
+  ret void
+}
+
+define void @sw_ri(i32* %a, i32 %b) {
+; RV32I-LABEL: sw_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, 2047
+; RV32I-NEXT:    sw a1, 1953(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILO-LABEL: sw_ri:
+; RV32IXQCILO:       # %bb.0:
+; RV32IXQCILO-NEXT:    qc.e.sw a1, 4000(a0)
+; RV32IXQCILO-NEXT:    ret
+  %1 = getelementptr i32, i32* %a, i32 1000
+  store i32 %b, i32* %1
+  ret void
+}

From 123b0e2a1e9de7465be8fb337a80d5d8984f93ae Mon Sep 17 00:00:00 2001
From: Vikram Hegde <115221833+vikramRH@users.noreply.github.com>
Date: Wed, 16 Apr 2025 11:28:28 +0530
Subject: [PATCH 080/710] Reapply "[AMDGPU][GlobalISel] Properly handle lane op
 lowering for larger vector types (#132358)" (#135758)

reapply https://github.com/llvm/llvm-project/pull/132358, tests updated.
---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   12 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.permlane.ll    | 1012 +++++++++++++++++
 .../CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll  |  625 +++++++++-
 .../AMDGPU/llvm.amdgcn.readfirstlane.ll       |  169 ++-
 .../CodeGen/AMDGPU/llvm.amdgcn.readlane.ll    |  168 +++
 .../CodeGen/AMDGPU/llvm.amdgcn.writelane.ll   |  798 +++++++++++++
 6 files changed, 2750 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 275d0193452a5..5fcbf810abcbd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -5580,6 +5580,7 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
     return false;
 
   LLT PartialResTy = LLT::scalar(SplitSize);
+  bool NeedsBitcast = false;
   if (Ty.isVector()) {
     LLT EltTy = Ty.getElementType();
     unsigned EltSize = EltTy.getSizeInBits();
@@ -5588,8 +5589,10 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
     } else if (EltSize == 16 || EltSize == 32) {
       unsigned NElem = SplitSize / EltSize;
       PartialResTy = Ty.changeElementCount(ElementCount::getFixed(NElem));
+    } else {
+      // Handle all other cases via S32/S64 pieces
+      NeedsBitcast = true;
     }
-    // Handle all other cases via S32/S64 pieces;
   }
 
   SmallVector<Register, 4> PartialRes;
@@ -5615,7 +5618,12 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
     PartialRes.push_back(createLaneOp(Src0, Src1, Src2, PartialResTy));
   }
 
-  B.buildMergeLikeInstr(DstReg, PartialRes);
+  if (NeedsBitcast)
+    B.buildBitcast(DstReg, B.buildMergeLikeInstr(
+                               LLT::scalar(Ty.getSizeInBits()), PartialRes));
+  else
+    B.buildMergeLikeInstr(DstReg, PartialRes);
+
   MI.eraseFromParent();
   return true;
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
index db557ff23c085..693e0ebd0280c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
@@ -9398,3 +9398,1015 @@ define void @v_permlanex16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %sr
   store <8 x i16> %v, ptr addrspace(1) %out
   ret void
 }
+
+define void @v_permlane16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_v2i64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s5, v7
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v5, v5, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v4, v4, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v3, v3, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v2, s4, s5
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_permlane16_v2i64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s5, v7
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_permlane16_v2i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_permlane16_v2i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: v_permlane16_v2i64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: v_permlane16_v2i64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <2 x i64> @llvm.amdgcn.permlane16.v2i64(<2 x i64> %src0, <2 x i64> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+  store <2 x i64> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @v_permlane16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_v3i64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v7, v7, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v6, v6, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v5, v5, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v4, v4, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v3, v3, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v2, s4, s5
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:16
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_permlane16_v3i64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:16
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_permlane16_v3i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_permlane16_v3i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s0, v8
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: v_permlane16_v3i64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: v_permlane16_v3i64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v8
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-GISEL-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <3 x i64> @llvm.amdgcn.permlane16.v3i64(<3 x i64> %src0, <3 x i64> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+  store <3 x i64> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @v_permlane16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_v4f64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s4, v10
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s5, v11
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v9, v9, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v8, v8, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v7, v7, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v6, v6, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v5, v5, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v4, v4, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v3, v3, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v2, s4, s5
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_permlane16_v4f64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s4, v10
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s5, v11
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v9, v9, s4, s5
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_permlane16_v4f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v10
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s1, v11
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v9, v9, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v8, v8, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_permlane16_v4f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s0, v10
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s1, v11
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v8, v8, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v9, v9, s0, s1
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: v_permlane16_v4f64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v10
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v11
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v9, v9, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v8, v8, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: v_permlane16_v4f64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v10
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v11
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v8, v8, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v9, v9, s0, s1
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <4 x double> @llvm.amdgcn.permlane16.v4f64(<4 x double> %src0, <4 x double> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+  store <4 x double> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @v_permlane16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_v8f64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s4, v18
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s5, v19
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v17, v17, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v16, v16, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v15, v15, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v14, v14, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v13, v13, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v12, v12, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v11, v11, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v10, v10, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v9, v9, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v8, v8, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v7, v7, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v6, v6, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v5, v5, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v4, v4, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v3, v3, s4, s5
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v2, s4, s5
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[14:17], off offset:48
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[10:13], off offset:32
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_permlane16_v8f64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s4, v18
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s5, v19
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v9, v9, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v10, v10, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v11, v11, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v12, v12, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v13, v13, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v14, v14, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v15, v15, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v16, v16, s4, s5
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v17, v17, s4, s5
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[10:13], off offset:32
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[14:17], off offset:48
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_permlane16_v8f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s1, v19
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v17, v17, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v16, v16, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v15, v15, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v14, v14, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v13, v13, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v12, v12, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v11, v11, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v10, v10, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v9, v9, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v8, v8, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX11-SDAG-NEXT:    s_clause 0x3
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_permlane16_v8f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s1, v19
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v8, v8, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v9, v9, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v10, v10, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v11, v11, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v12, v12, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v13, v13, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v14, v14, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v15, v15, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v16, v16, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v17, v17, s0, s1
+; GFX11-GISEL-NEXT:    s_clause 0x3
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: v_permlane16_v8f64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v19
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v17, v17, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v16, v16, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v15, v15, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v14, v14, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v13, v13, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v12, v12, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v11, v11, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v10, v10, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v9, v9, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v8, v8, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX12-SDAG-NEXT:    s_clause 0x3
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: v_permlane16_v8f64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v19
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v5, v5, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v6, v6, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v7, v7, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v8, v8, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v9, v9, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v10, v10, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v11, v11, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v12, v12, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v13, v13, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v14, v14, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v15, v15, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v16, v16, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v17, v17, s0, s1
+; GFX12-GISEL-NEXT:    s_clause 0x3
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <8 x double> @llvm.amdgcn.permlane16.v8f64(<8 x double> %src0, <8 x double> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+  store <8 x double> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @v_permlanex16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_v2i64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s5, v7
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_permlanex16_v2i64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s5, v7
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_permlanex16_v2i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_permlanex16_v2i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: v_permlanex16_v2i64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: v_permlanex16_v2i64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <2 x i64> @llvm.amdgcn.permlanex16.v2i64(<2 x i64> %src0, <2 x i64> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+  store <2 x i64> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @v_permlanex16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_v3i64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:16
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_permlanex16_v3i64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:16
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_permlanex16_v3i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_permlanex16_v3i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s0, v8
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: v_permlanex16_v3i64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: v_permlanex16_v3i64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v8
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v9
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-GISEL-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <3 x i64> @llvm.amdgcn.permlanex16.v3i64(<3 x i64> %src0, <3 x i64> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+  store <3 x i64> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @v_permlanex16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_v4f64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s4, v10
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s5, v11
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v9, v9, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_permlanex16_v4f64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s4, v10
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s5, v11
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v9, v9, s4, s5
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_permlanex16_v4f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v10
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s1, v11
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_permlanex16_v4f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s0, v10
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s1, v11
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: v_permlanex16_v4f64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v10
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v11
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: v_permlanex16_v4f64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v10
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v11
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <4 x double> @llvm.amdgcn.permlanex16.v4f64(<4 x double> %src0, <4 x double> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+  store <4 x double> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @v_permlanex16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_v8f64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s4, v18
+; GFX10-SDAG-NEXT:    v_readfirstlane_b32 s5, v19
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v17, v17, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v16, v16, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v15, v15, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v14, v14, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v13, v13, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v12, v12, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v11, v11, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v10, v10, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v9, v9, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[14:17], off offset:48
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[10:13], off offset:32
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: v_permlanex16_v8f64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s4, v18
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s5, v19
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v8, v8, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v9, v9, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v10, v10, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v11, v11, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v12, v12, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v13, v13, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v14, v14, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v15, v15, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v16, v16, s4, s5
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v17, v17, s4, s5
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[10:13], off offset:32
+; GFX10-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[14:17], off offset:48
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: v_permlanex16_v8f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s1, v19
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v17, v17, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v16, v16, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v15, v15, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v14, v14, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v13, v13, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v12, v12, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v11, v11, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v10, v10, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX11-SDAG-NEXT:    s_clause 0x3
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: v_permlanex16_v8f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s1, v19
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v10, v10, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v11, v11, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v12, v12, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v13, v13, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v14, v14, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v15, v15, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v16, v16, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v17, v17, s0, s1
+; GFX11-GISEL-NEXT:    s_clause 0x3
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: v_permlanex16_v8f64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v19
+; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v17, v17, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v16, v16, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v15, v15, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v14, v14, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v13, v13, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v12, v12, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v11, v11, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v10, v10, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX12-SDAG-NEXT:    s_clause 0x3
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX12-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: v_permlanex16_v8f64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v19
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v10, v10, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v11, v11, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v12, v12, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v13, v13, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v14, v14, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v15, v15, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v16, v16, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v17, v17, s0, s1
+; GFX12-GISEL-NEXT:    s_clause 0x3
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX12-GISEL-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <8 x double> @llvm.amdgcn.permlanex16.v8f64(<8 x double> %src0, <8 x double> %src0, i32 %src1, i32 %src2, i1 false, i1 false)
+  store <8 x double> %v, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll
index f23f9595446eb..6698d360aff4c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11-SDAG %s
+; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11-GISEL %s
 
 declare i32 @llvm.amdgcn.permlane64(i32)
 declare i32 @llvm.amdgcn.workitem.id.x()
 
-define amdgpu_kernel void @test_s(ptr addrspace(1) %out, i32 %src0) {
+define amdgpu_kernel void @test_s_i32(ptr addrspace(1) %out, i32 %src0) {
 ; GFX11-LABEL: test_s:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_clause 0x1
@@ -17,12 +17,93 @@ define amdgpu_kernel void @test_s(ptr addrspace(1) %out, i32 %src0) {
 ; GFX11-NEXT:    v_permlane64_b32 v0, v0
 ; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %v = call i32 @llvm.amdgcn.permlane64(i32 %src0)
+; GFX11-SDAG-LABEL: test_s_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-SDAG-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_s_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %v = call i32 @llvm.amdgcn.permlane64.i32(i32 %src0)
   store i32 %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @test_i(ptr addrspace(1) %out) {
+define amdgpu_kernel void @test_s_i64(ptr addrspace(1) %out, i64 %src0) {
+; GFX11-SDAG-LABEL: test_s_i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v0
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v2
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_s_i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v1, v1
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %v = call i64 @llvm.amdgcn.permlane64.i64(i64 %src0)
+  store i64 %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @test_s_f64(ptr addrspace(1) %out, double %src0) {
+; GFX11-SDAG-LABEL: test_s_f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v0
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v2
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_s_f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v1, v1
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %v = call double @llvm.amdgcn.permlane64.f64(double %src0)
+  store double %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @test_i_i32(ptr addrspace(1) %out) {
 ; GFX11-LABEL: test_i:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
@@ -32,12 +113,115 @@ define amdgpu_kernel void @test_i(ptr addrspace(1) %out) {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %v = call i32 @llvm.amdgcn.permlane64(i32 99)
+; GFX11-SDAG-LABEL: test_i_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, 0x63 :: v_dual_mov_b32 v1, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_i_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v0, 0x63 :: v_dual_mov_b32 v1, 0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %v = call i32 @llvm.amdgcn.permlane64.i32(i32 99)
   store i32 %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @test_v(ptr addrspace(1) %out, i32 %src0) #1 {
+define amdgpu_kernel void @test_i_f32(ptr addrspace(1) %out) {
+; GFX11-SDAG-LABEL: test_i_f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, 0x449a5000 :: v_dual_mov_b32 v1, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_i_f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v0, 0x449a5000 :: v_dual_mov_b32 v1, 0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %v = call float @llvm.amdgcn.permlane64.f32(float 1234.5)
+  store float %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @test_i_i64(ptr addrspace(1) %out) {
+; GFX11-SDAG-LABEL: test_i_i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, 0x63
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v2
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_i_i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v0, 0x63
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v1, v2
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %v = call i64 @llvm.amdgcn.permlane64.i64(i64 99)
+  store i64 %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @test_i_f64(ptr addrspace(1) %out) {
+; GFX11-SDAG-LABEL: test_i_f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, 0x40934a00
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v0
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v2
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_i_f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, 0x40934a00
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v2
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v1, v1
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %v = call double @llvm.amdgcn.permlane64.f64(double 1234.5)
+  store double %v, ptr addrspace(1) %out
+  ret void
+}
+
+
+
+define amdgpu_kernel void @test_v_i32(ptr addrspace(1) %out, i32 %src0) #1 {
 ; GFX11-LABEL: test_v:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
@@ -47,11 +231,430 @@ define amdgpu_kernel void @test_v(ptr addrspace(1) %out, i32 %src0) #1 {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: test_v_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_v_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %v = call i32 @llvm.amdgcn.permlane64(i32 %tidx)
+  %v = call i32 @llvm.amdgcn.permlane64.i32(i32 %tidx)
   store i32 %v, ptr addrspace(1) %out
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-GISEL: {{.*}}
-; GFX11-SDAG: {{.*}}
+
+define amdgpu_kernel void @test_v_f32(ptr addrspace(1) %out, float %src0) #1 {
+; GFX11-SDAG-LABEL: test_v_f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_v_f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_f32 = bitcast i32 %tidx to float
+  %v = call float @llvm.amdgcn.permlane64.f32(float %tidx_f32)
+  store float %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @test_v_i64(ptr addrspace(1) %out, i64 %src0) #1 {
+; GFX11-SDAG-LABEL: test_v_i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v2
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_v_i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v1, v2
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_i64 = zext i32 %tidx to i64
+  %v = call i64 @llvm.amdgcn.permlane64.i64(i64 %tidx_i64)
+  store i64 %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @test_v_f64(ptr addrspace(1) %out, double %src0) #1 {
+; GFX11-SDAG-LABEL: test_v_f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v1
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: test_v_f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v0, v0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v1, v1
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_f32 = bitcast i32 %tidx to float
+  %tidx_f64 = fpext float %tidx_f32 to double
+  %v = call double @llvm.amdgcn.permlane64.f64(double %tidx_f64)
+  store double %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_half(ptr addrspace(1) %out, half %src0) {
+; GFX11-SDAG-LABEL: test_half:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    global_store_b16 v[0:1], v2, off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_half:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    global_store_b16 v[0:1], v2, off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call half @llvm.amdgcn.permlane64.f16(half %src0)
+  store half %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_bfloat(ptr addrspace(1) %out, bfloat %src0) {
+; GFX11-SDAG-LABEL: test_bfloat:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    global_store_b16 v[0:1], v2, off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_bfloat:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    global_store_b16 v[0:1], v2, off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call bfloat @llvm.amdgcn.permlane64.bf16(bfloat %src0)
+  store bfloat %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_i16(ptr addrspace(1) %out, i16 %src0) {
+; GFX11-SDAG-LABEL: test_i16:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    global_store_b16 v[0:1], v2, off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_i16:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    global_store_b16 v[0:1], v2, off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call i16 @llvm.amdgcn.permlane64.i16(i16 %src0)
+  store i16 %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_v2f16(ptr addrspace(1) %out, <2 x half> %src0) {
+; GFX11-SDAG-LABEL: test_v2f16:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_v2f16:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    global_store_b32 v[0:1], v2, off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <2 x half> @llvm.amdgcn.permlane64.v2f16(<2 x half> %src0)
+  store <2 x half> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_v2f32(ptr addrspace(1) %out, <2 x float> %src0) {
+; GFX11-SDAG-LABEL: test_v2f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_v2f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-GISEL-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <2 x float> @llvm.amdgcn.permlane64.v2f32(<2 x float> %src0)
+  store <2 x float> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_v7i32(ptr addrspace(1) %out, <7 x i32> %src0) {
+; GFX11-SDAG-LABEL: test_v7i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v8, v8
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v7, v7
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v6, v6
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_v7i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v6, v6
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v7, v7
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v8, v8
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <7 x i32> @llvm.amdgcn.permlane64.v7i32(<7 x i32> %src0)
+  store <7 x i32> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_v8i16(ptr addrspace(1) %out, <8 x i16> %src0) {
+; GFX11-SDAG-LABEL: test_v8i16:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_v8i16:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <8 x i16> @llvm.amdgcn.permlane64.v8i16(<8 x i16> %src0)
+  store <8 x i16> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_v2i64(ptr addrspace(1) %out, <2 x i64> %src0) {
+; GFX11-SDAG-LABEL: test_v2i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_v2i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <2 x i64> @llvm.amdgcn.permlane64.v2i64(<2 x i64> %src0)
+  store <2 x i64> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_v3i64(ptr addrspace(1) %out, <3 x i64> %src0) {
+; GFX11-SDAG-LABEL: test_v3i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v7, v7
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v6, v6
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_v3i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v6, v6
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v7, v7
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <3 x i64> @llvm.amdgcn.permlane64.v3i64(<3 x i64> %src0)
+  store <3 x i64> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_v4f64(ptr addrspace(1) %out, <4 x double> %src0) {
+; GFX11-SDAG-LABEL: test_v4f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v9, v9
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v8, v8
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v7, v7
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v6, v6
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_v4f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v6, v6
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v7, v7
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v8, v8
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v9, v9
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <4 x double> @llvm.amdgcn.permlane64.v4f64(<4 x double> %src0)
+  store <4 x double> %v, ptr addrspace(1) %out
+  ret void
+}
+
+define void @test_v8f64(ptr addrspace(1) %out, <8 x double> %src0) {
+; GFX11-SDAG-LABEL: test_v8f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v17, v17
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v16, v16
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v15, v15
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v14, v14
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v13, v13
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v12, v12
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v11, v11
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v10, v10
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v9, v9
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v8, v8
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v7, v7
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v6, v6
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-SDAG-NEXT:    s_clause 0x3
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-SDAG-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_v8f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v2, v2
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v3, v3
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v4, v4
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v5, v5
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v6, v6
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v7, v7
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v8, v8
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v9, v9
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v10, v10
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v11, v11
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v12, v12
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v13, v13
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v14, v14
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v15, v15
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v16, v16
+; GFX11-GISEL-NEXT:    v_permlane64_b32 v17, v17
+; GFX11-GISEL-NEXT:    s_clause 0x3
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX11-GISEL-NEXT:    global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %v = call <8 x double> @llvm.amdgcn.permlane64.v8f64(<8 x double> %src0)
+  store <8 x double> %v, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index cc9e34be209b4..c0afc0a443955 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -163,30 +163,157 @@ define void @test_readfirstlane_i64(ptr addrspace(1) %out, i64 %src) {
   ret void
 }
 
-; FIXME: Broken
-; define void @test_readfirstlane_v2i64(ptr addrspace(1) %out, <2 x i64> %src) {
-;   %x = call <2 x i64> @llvm.amdgcn.readfirstlane.v2i64(<2 x i64> %src)
-;   call void asm sideeffect "; use $0", "s"(<2 x i64> %x)
-;   ret void
-; }
+define void @test_readfirstlane_v2i64(ptr addrspace(1) %out, <2 x i64> %src) {
+; CHECK-SDAG-LABEL: test_readfirstlane_v2i64:
+; CHECK-SDAG:       ; %bb.0:
+; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s7, v5
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s5, v3
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s4, v2
+; CHECK-SDAG-NEXT:    ;;#ASMSTART
+; CHECK-SDAG-NEXT:    ; use s[4:7]
+; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; CHECK-GISEL-LABEL: test_readfirstlane_v2i64:
+; CHECK-GISEL:       ; %bb.0:
+; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s5, v3
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s6, v4
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s7, v5
+; CHECK-GISEL-NEXT:    ;;#ASMSTART
+; CHECK-GISEL-NEXT:    ; use s[4:7]
+; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %x = call <2 x i64> @llvm.amdgcn.readfirstlane.v2i64(<2 x i64> %src)
+  call void asm sideeffect "; use $0", "s"(<2 x i64> %x)
+  ret void
+}
 
-; define void @test_readfirstlane_v3i64(ptr addrspace(1) %out, <3 x i64> %src) {
-;   %x = call <3 x i64> @llvm.amdgcn.readfirstlane.v3i64(<3 x i64> %src)
-;   call void asm sideeffect "; use $0", "s"(<3 x i64> %x)
-;   ret void
-; }
+define void @test_readfirstlane_v3i64(ptr addrspace(1) %out, <3 x i64> %src) {
+; CHECK-SDAG-LABEL: test_readfirstlane_v3i64:
+; CHECK-SDAG:       ; %bb.0:
+; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s9, v7
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s8, v6
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s7, v5
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s5, v3
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s4, v2
+; CHECK-SDAG-NEXT:    ;;#ASMSTART
+; CHECK-SDAG-NEXT:    ; use s[4:9]
+; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; CHECK-GISEL-LABEL: test_readfirstlane_v3i64:
+; CHECK-GISEL:       ; %bb.0:
+; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s5, v3
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s6, v4
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s7, v5
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s8, v6
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s9, v7
+; CHECK-GISEL-NEXT:    ;;#ASMSTART
+; CHECK-GISEL-NEXT:    ; use s[4:9]
+; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %x = call <3 x i64> @llvm.amdgcn.readfirstlane.v3i64(<3 x i64> %src)
+  call void asm sideeffect "; use $0", "s"(<3 x i64> %x)
+  ret void
+}
 
-; define void @test_readfirstlane_v4i64(ptr addrspace(1) %out, <4 x i64> %src) {
-;   %x = call <4 x i64> @llvm.amdgcn.readfirstlane.v4i64(<4 x i64> %src)
-;   call void asm sideeffect "; use $0", "s"(<4 x i64> %x)
-;   ret void
-; }
+define void @test_readfirstlane_v4i64(ptr addrspace(1) %out, <4 x i64> %src) {
+; CHECK-SDAG-LABEL: test_readfirstlane_v4i64:
+; CHECK-SDAG:       ; %bb.0:
+; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s11, v9
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s10, v8
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s9, v7
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s8, v6
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s7, v5
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s5, v3
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s4, v2
+; CHECK-SDAG-NEXT:    ;;#ASMSTART
+; CHECK-SDAG-NEXT:    ; use s[4:11]
+; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; CHECK-GISEL-LABEL: test_readfirstlane_v4i64:
+; CHECK-GISEL:       ; %bb.0:
+; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s5, v3
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s6, v4
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s7, v5
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s8, v6
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s9, v7
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s10, v8
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s11, v9
+; CHECK-GISEL-NEXT:    ;;#ASMSTART
+; CHECK-GISEL-NEXT:    ; use s[4:11]
+; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %x = call <4 x i64> @llvm.amdgcn.readfirstlane.v4i64(<4 x i64> %src)
+  call void asm sideeffect "; use $0", "s"(<4 x i64> %x)
+  ret void
+}
 
-; define void @test_readfirstlane_v8i64(ptr addrspace(1) %out, <8 x i64> %src) {
-;   %x = call <8 x i64> @llvm.amdgcn.readfirstlane.v8i64(<8 x i64> %src)
-;   call void asm sideeffect "; use $0", "s"(<8 x i64> %x)
-;   ret void
-; }
+define void @test_readfirstlane_v8i64(ptr addrspace(1) %out, <8 x i64> %src) {
+; CHECK-SDAG-LABEL: test_readfirstlane_v8i64:
+; CHECK-SDAG:       ; %bb.0:
+; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s19, v17
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s18, v16
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s17, v15
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s16, v14
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s15, v13
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s14, v12
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s13, v11
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s12, v10
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s11, v9
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s10, v8
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s9, v7
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s8, v6
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s7, v5
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s5, v3
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s4, v2
+; CHECK-SDAG-NEXT:    ;;#ASMSTART
+; CHECK-SDAG-NEXT:    ; use s[4:19]
+; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; CHECK-GISEL-LABEL: test_readfirstlane_v8i64:
+; CHECK-GISEL:       ; %bb.0:
+; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s5, v3
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s6, v4
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s7, v5
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s8, v6
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s9, v7
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s10, v8
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s11, v9
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s12, v10
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s13, v11
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s14, v12
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s15, v13
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s16, v14
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s17, v15
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s18, v16
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s19, v17
+; CHECK-GISEL-NEXT:    ;;#ASMSTART
+; CHECK-GISEL-NEXT:    ; use s[4:19]
+; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %x = call <8 x i64> @llvm.amdgcn.readfirstlane.v8i64(<8 x i64> %src)
+  call void asm sideeffect "; use $0", "s"(<8 x i64> %x)
+  ret void
+}
 
 define void @test_readfirstlane_f64(ptr addrspace(1) %out, double %src) {
 ; CHECK-SDAG-LABEL: test_readfirstlane_f64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
index f2b0959cc706e..42aab1878efa6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
@@ -972,6 +972,174 @@ define void @test_readlane_v8i16(ptr addrspace(1) %out, <8 x i16> %src, i32 %src
   ret void
 }
 
+define void @test_readlane_v2i64(ptr addrspace(1) %out, <2 x i64> %src, i32 %src1) {
+; CHECK-SDAG-LABEL: test_readlane_v2i64:
+; CHECK-SDAG:       ; %bb.0:
+; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s4, v6
+; CHECK-SDAG-NEXT:    s_nop 3
+; CHECK-SDAG-NEXT:    v_readlane_b32 s7, v5, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s6, v4, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s5, v3, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s4, v2, s4
+; CHECK-SDAG-NEXT:    ;;#ASMSTART
+; CHECK-SDAG-NEXT:    ; use s[4:7]
+; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; CHECK-GISEL-LABEL: test_readlane_v2i64:
+; CHECK-GISEL:       ; %bb.0:
+; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s7, v6
+; CHECK-GISEL-NEXT:    s_nop 3
+; CHECK-GISEL-NEXT:    v_readlane_b32 s4, v2, s7
+; CHECK-GISEL-NEXT:    v_readlane_b32 s5, v3, s7
+; CHECK-GISEL-NEXT:    v_readlane_b32 s6, v4, s7
+; CHECK-GISEL-NEXT:    v_readlane_b32 s7, v5, s7
+; CHECK-GISEL-NEXT:    ;;#ASMSTART
+; CHECK-GISEL-NEXT:    ; use s[4:7]
+; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %x = call <2 x i64> @llvm.amdgcn.readlane.v2i64(<2 x i64> %src, i32 %src1)
+  call void asm sideeffect "; use $0", "s"(<2 x i64> %x)
+  ret void
+}
+
+define void @test_readlane_v3i64(ptr addrspace(1) %out, <3 x i64> %src, i32 %src1) {
+; CHECK-SDAG-LABEL: test_readlane_v3i64:
+; CHECK-SDAG:       ; %bb.0:
+; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; CHECK-SDAG-NEXT:    s_nop 3
+; CHECK-SDAG-NEXT:    v_readlane_b32 s9, v7, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s8, v6, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s7, v5, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s6, v4, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s5, v3, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s4, v2, s4
+; CHECK-SDAG-NEXT:    ;;#ASMSTART
+; CHECK-SDAG-NEXT:    ; use s[4:9]
+; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; CHECK-GISEL-LABEL: test_readlane_v3i64:
+; CHECK-GISEL:       ; %bb.0:
+; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s9, v8
+; CHECK-GISEL-NEXT:    s_nop 3
+; CHECK-GISEL-NEXT:    v_readlane_b32 s4, v2, s9
+; CHECK-GISEL-NEXT:    v_readlane_b32 s5, v3, s9
+; CHECK-GISEL-NEXT:    v_readlane_b32 s6, v4, s9
+; CHECK-GISEL-NEXT:    v_readlane_b32 s7, v5, s9
+; CHECK-GISEL-NEXT:    v_readlane_b32 s8, v6, s9
+; CHECK-GISEL-NEXT:    v_readlane_b32 s9, v7, s9
+; CHECK-GISEL-NEXT:    ;;#ASMSTART
+; CHECK-GISEL-NEXT:    ; use s[4:9]
+; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %x = call <3 x i64> @llvm.amdgcn.readlane.v3i64(<3 x i64> %src, i32 %src1)
+  call void asm sideeffect "; use $0", "s"(<3 x i64> %x)
+  ret void
+}
+
+define void @test_readlane_v4f64(ptr addrspace(1) %out, <4 x double> %src, i32 %src1) {
+; CHECK-SDAG-LABEL: test_readlane_v4f64:
+; CHECK-SDAG:       ; %bb.0:
+; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s4, v10
+; CHECK-SDAG-NEXT:    s_nop 3
+; CHECK-SDAG-NEXT:    v_readlane_b32 s11, v9, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s10, v8, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s9, v7, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s8, v6, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s7, v5, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s6, v4, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s5, v3, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s4, v2, s4
+; CHECK-SDAG-NEXT:    ;;#ASMSTART
+; CHECK-SDAG-NEXT:    ; use s[4:11]
+; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; CHECK-GISEL-LABEL: test_readlane_v4f64:
+; CHECK-GISEL:       ; %bb.0:
+; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s11, v10
+; CHECK-GISEL-NEXT:    s_nop 3
+; CHECK-GISEL-NEXT:    v_readlane_b32 s4, v2, s11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s5, v3, s11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s6, v4, s11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s7, v5, s11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s8, v6, s11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s9, v7, s11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s10, v8, s11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s11, v9, s11
+; CHECK-GISEL-NEXT:    ;;#ASMSTART
+; CHECK-GISEL-NEXT:    ; use s[4:11]
+; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %x = call <4 x double> @llvm.amdgcn.readlane.v4f64(<4 x double> %src, i32 %src1)
+  call void asm sideeffect "; use $0", "s"(<4 x double> %x)
+  ret void
+}
+
+define void @test_readlane_v8f64(ptr addrspace(1) %out, <8 x double> %src, i32 %src1) {
+; CHECK-SDAG-LABEL: test_readlane_v8f64:
+; CHECK-SDAG:       ; %bb.0:
+; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s4, v18
+; CHECK-SDAG-NEXT:    s_nop 3
+; CHECK-SDAG-NEXT:    v_readlane_b32 s19, v17, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s18, v16, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s17, v15, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s16, v14, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s15, v13, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s14, v12, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s13, v11, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s12, v10, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s11, v9, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s10, v8, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s9, v7, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s8, v6, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s7, v5, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s6, v4, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s5, v3, s4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s4, v2, s4
+; CHECK-SDAG-NEXT:    ;;#ASMSTART
+; CHECK-SDAG-NEXT:    ; use s[4:19]
+; CHECK-SDAG-NEXT:    ;;#ASMEND
+; CHECK-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; CHECK-GISEL-LABEL: test_readlane_v8f64:
+; CHECK-GISEL:       ; %bb.0:
+; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s19, v18
+; CHECK-GISEL-NEXT:    s_nop 3
+; CHECK-GISEL-NEXT:    v_readlane_b32 s4, v2, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s5, v3, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s6, v4, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s7, v5, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s8, v6, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s9, v7, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s10, v8, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s11, v9, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s12, v10, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s13, v11, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s14, v12, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s15, v13, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s16, v14, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s17, v15, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s18, v16, s19
+; CHECK-GISEL-NEXT:    v_readlane_b32 s19, v17, s19
+; CHECK-GISEL-NEXT:    ;;#ASMSTART
+; CHECK-GISEL-NEXT:    ; use s[4:19]
+; CHECK-GISEL-NEXT:    ;;#ASMEND
+; CHECK-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %x = call <8 x double> @llvm.amdgcn.readlane.v4f64(<8 x double> %src, i32 %src1)
+  call void asm sideeffect "; use $0", "s"(<8 x double> %x)
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #2
 
 attributes #0 = { nounwind readnone convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
index 4ac2cc98970b5..8cf7497fca640 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
@@ -2801,6 +2801,804 @@ define void @test_writelane_v8i16(ptr addrspace(1) %out, <8 x i16> %src, i32 %sr
   ret void
 }
 
+define void @test_writelane_v2i64(ptr addrspace(1) %out, <2 x i64> %src, i32 %src1) {
+; GFX802-SDAG-LABEL: test_writelane_v2i64:
+; GFX802-SDAG:       ; %bb.0:
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX802-SDAG-NEXT:    flat_load_dwordx4 v[7:10], v[0:1]
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s8, v2
+; GFX802-SDAG-NEXT:    s_mov_b32 m0, s4
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v10, s5, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v9, s6, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v8, s7, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v7, s8, m0
+; GFX802-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-SDAG-LABEL: test_writelane_v2i64:
+; GFX1010-SDAG:       ; %bb.0:
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-SDAG-NEXT:    global_load_dwordx4 v[7:10], v[0:1], off
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s4, v5
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s5, v6
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s8, v2
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v10, s4, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v9, s6, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v8, s7, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v7, s8, s5
+; GFX1010-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off
+; GFX1010-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: test_writelane_v2i64:
+; GFX1100-SDAG:       ; %bb.0:
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT:    global_load_b128 v[7:10], v[0:1], off
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v4
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v10, s0, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v9, s2, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v8, s3, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v7, s4, s1
+; GFX1100-SDAG-NEXT:    global_store_b128 v[0:1], v[7:10], off
+; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX802-GISEL-LABEL: test_writelane_v2i64:
+; GFX802-GISEL:       ; %bb.0:
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[7:10], v[0:1]
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s5, v6
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s8, v5
+; GFX802-GISEL-NEXT:    s_mov_b32 m0, s5
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v7, s4, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v8, s6, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v9, s7, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v10, s8, m0
+; GFX802-GISEL-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-GISEL-LABEL: test_writelane_v2i64:
+; GFX1010-GISEL:       ; %bb.0:
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[7:10], v[0:1], off
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s5, v6
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s8, v5
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v7, s4, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v8, s6, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v9, s7, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v10, s8, s5
+; GFX1010-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off
+; GFX1010-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1100-GISEL-LABEL: test_writelane_v2i64:
+; GFX1100-GISEL:       ; %bb.0:
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-GISEL-NEXT:    global_load_b128 v[7:10], v[0:1], off
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s3, v4
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s4, v5
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v7, s0, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v8, s2, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v9, s3, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v10, s4, s1
+; GFX1100-GISEL-NEXT:    global_store_b128 v[0:1], v[7:10], off
+; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %oldval = load <2 x i64>, ptr addrspace(1) %out
+  %writelane = call <2 x i64> @llvm.amdgcn.writelane.v2i64(<2 x i64> %src, i32 %src1, <2 x i64> %oldval)
+  store <2 x i64> %writelane, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define void @test_writelane_v3i64(ptr addrspace(1) %out, <3 x i64> %src, i32 %src1) {
+; GFX802-SDAG-LABEL: test_writelane_v3i64:
+; GFX802-SDAG:       ; %bb.0:
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX802-SDAG-NEXT:    v_add_u32_e32 v13, vcc, 16, v0
+; GFX802-SDAG-NEXT:    flat_load_dwordx4 v[9:12], v[0:1]
+; GFX802-SDAG-NEXT:    v_addc_u32_e32 v14, vcc, 0, v1, vcc
+; GFX802-SDAG-NEXT:    flat_load_dwordx2 v[15:16], v[13:14]
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s8, v4
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s9, v3
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX802-SDAG-NEXT:    s_mov_b32 m0, s4
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s5, v7
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v12, s7, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v11, s8, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v10, s9, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v9, s10, m0
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v16, s5, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v15, s6, m0
+; GFX802-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[9:12]
+; GFX802-SDAG-NEXT:    flat_store_dwordx2 v[13:14], v[15:16]
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-SDAG-LABEL: test_writelane_v3i64:
+; GFX1010-SDAG:       ; %bb.0:
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-SDAG-NEXT:    s_clause 0x1
+; GFX1010-SDAG-NEXT:    global_load_dwordx2 v[13:14], v[0:1], off offset:16
+; GFX1010-SDAG-NEXT:    global_load_dwordx4 v[9:12], v[0:1], off
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s5, v8
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s8, v4
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s9, v3
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s4, v7
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v14, s4, s5
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v12, s7, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v11, s8, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v10, s9, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v9, s10, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v13, s6, s5
+; GFX1010-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[9:12], off
+; GFX1010-SDAG-NEXT:    global_store_dwordx2 v[0:1], v[13:14], off offset:16
+; GFX1010-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: test_writelane_v3i64:
+; GFX1100-SDAG:       ; %bb.0:
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT:    s_clause 0x1
+; GFX1100-SDAG-NEXT:    global_load_b64 v[13:14], v[0:1], off offset:16
+; GFX1100-SDAG-NEXT:    global_load_b128 v[9:12], v[0:1], off
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v8
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s3, v5
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v7
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v14, s0, s1
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v12, s3, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v11, s4, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v10, s5, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v9, s6, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v13, s2, s1
+; GFX1100-SDAG-NEXT:    s_clause 0x1
+; GFX1100-SDAG-NEXT:    global_store_b128 v[0:1], v[9:12], off
+; GFX1100-SDAG-NEXT:    global_store_b64 v[0:1], v[13:14], off offset:16
+; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX802-GISEL-LABEL: test_writelane_v3i64:
+; GFX802-GISEL:       ; %bb.0:
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX802-GISEL-NEXT:    v_add_u32_e32 v17, vcc, 16, v0
+; GFX802-GISEL-NEXT:    v_addc_u32_e32 v18, vcc, 0, v1, vcc
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[9:12], v[0:1]
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[13:16], v[17:18]
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s5, v8
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s8, v5
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s9, v6
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s10, v7
+; GFX802-GISEL-NEXT:    s_mov_b32 m0, s5
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v9, s4, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v10, s6, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v11, s7, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v12, s8, m0
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v13, s9, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v14, s10, m0
+; GFX802-GISEL-NEXT:    v_mov_b32_e32 v2, v13
+; GFX802-GISEL-NEXT:    v_mov_b32_e32 v3, v14
+; GFX802-GISEL-NEXT:    flat_store_dwordx4 v[0:1], v[9:12]
+; GFX802-GISEL-NEXT:    flat_store_dwordx2 v[17:18], v[2:3]
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-GISEL-LABEL: test_writelane_v3i64:
+; GFX1010-GISEL:       ; %bb.0:
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-GISEL-NEXT:    s_clause 0x1
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[9:12], v[0:1], off
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[13:16], v[0:1], off offset:16
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s5, v8
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s7, v6
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s8, v7
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s9, v4
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s10, v5
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v9, s4, s5
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v13, s7, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v14, s8, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v10, s6, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v11, s9, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v12, s10, s5
+; GFX1010-GISEL-NEXT:    v_mov_b32_e32 v2, v13
+; GFX1010-GISEL-NEXT:    v_mov_b32_e32 v3, v14
+; GFX1010-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[9:12], off
+; GFX1010-GISEL-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off offset:16
+; GFX1010-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1100-GISEL-LABEL: test_writelane_v3i64:
+; GFX1100-GISEL:       ; %bb.0:
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-GISEL-NEXT:    s_clause 0x1
+; GFX1100-GISEL-NEXT:    global_load_b128 v[9:12], v[0:1], off
+; GFX1100-GISEL-NEXT:    global_load_b128 v[13:16], v[0:1], off offset:16
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v8
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s3, v6
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s4, v7
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s5, v4
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s6, v5
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v9, s0, s1
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v13, s3, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v14, s4, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v10, s2, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v11, s5, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v12, s6, s1
+; GFX1100-GISEL-NEXT:    v_mov_b32_e32 v2, v13
+; GFX1100-GISEL-NEXT:    v_mov_b32_e32 v3, v14
+; GFX1100-GISEL-NEXT:    s_clause 0x1
+; GFX1100-GISEL-NEXT:    global_store_b128 v[0:1], v[9:12], off
+; GFX1100-GISEL-NEXT:    global_store_b64 v[0:1], v[2:3], off offset:16
+; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %oldval = load <3 x i64>, ptr addrspace(1) %out
+  %writelane = call <3 x i64> @llvm.amdgcn.writelane.v2i64(<3 x i64> %src, i32 %src1, <3 x i64> %oldval)
+  store <3 x i64> %writelane, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define void @test_writelane_v4f64(ptr addrspace(1) %out, <4 x double> %src, i32 %src1) {
+; GFX802-SDAG-LABEL: test_writelane_v4f64:
+; GFX802-SDAG:       ; %bb.0:
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX802-SDAG-NEXT:    v_add_u32_e32 v19, vcc, 16, v0
+; GFX802-SDAG-NEXT:    flat_load_dwordx4 v[11:14], v[0:1]
+; GFX802-SDAG-NEXT:    v_addc_u32_e32 v20, vcc, 0, v1, vcc
+; GFX802-SDAG-NEXT:    flat_load_dwordx4 v[15:18], v[19:20]
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s4, v10
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s9, v5
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s10, v4
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s12, v2
+; GFX802-SDAG-NEXT:    s_mov_b32 m0, s4
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s6, v8
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s8, v6
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v14, s9, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v13, s10, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v12, s11, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v11, s12, m0
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v18, s5, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v17, s6, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v16, s7, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v15, s8, m0
+; GFX802-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[11:14]
+; GFX802-SDAG-NEXT:    flat_store_dwordx4 v[19:20], v[15:18]
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-SDAG-LABEL: test_writelane_v4f64:
+; GFX1010-SDAG:       ; %bb.0:
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-SDAG-NEXT:    s_clause 0x1
+; GFX1010-SDAG-NEXT:    global_load_dwordx4 v[11:14], v[0:1], off offset:16
+; GFX1010-SDAG-NEXT:    global_load_dwordx4 v[15:18], v[0:1], off
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s5, v10
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s9, v5
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s10, v4
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s12, v2
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s4, v9
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s6, v8
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s8, v6
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v14, s4, s5
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v18, s9, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v17, s10, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v16, s11, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v15, s12, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v13, s6, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v12, s7, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v11, s8, s5
+; GFX1010-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[15:18], off
+; GFX1010-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[11:14], off offset:16
+; GFX1010-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: test_writelane_v4f64:
+; GFX1100-SDAG:       ; %bb.0:
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT:    s_clause 0x1
+; GFX1100-SDAG-NEXT:    global_load_b128 v[11:14], v[0:1], off offset:16
+; GFX1100-SDAG-NEXT:    global_load_b128 v[15:18], v[0:1], off
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v10
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s8, v2
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v9
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v8
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v14, s0, s1
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v18, s5, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v17, s6, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v16, s7, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v15, s8, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v13, s2, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v12, s3, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v11, s4, s1
+; GFX1100-SDAG-NEXT:    s_clause 0x1
+; GFX1100-SDAG-NEXT:    global_store_b128 v[0:1], v[15:18], off
+; GFX1100-SDAG-NEXT:    global_store_b128 v[0:1], v[11:14], off offset:16
+; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX802-GISEL-LABEL: test_writelane_v4f64:
+; GFX802-GISEL:       ; %bb.0:
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX802-GISEL-NEXT:    v_add_u32_e32 v19, vcc, 16, v0
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[11:14], v[0:1]
+; GFX802-GISEL-NEXT:    v_addc_u32_e32 v20, vcc, 0, v1, vcc
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[15:18], v[19:20]
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s5, v10
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s8, v5
+; GFX802-GISEL-NEXT:    s_mov_b32 m0, s5
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s9, v6
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s10, v7
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s11, v8
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s12, v9
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v11, s4, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v12, s6, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v13, s7, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v14, s8, m0
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v15, s9, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v16, s10, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v17, s11, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v18, s12, m0
+; GFX802-GISEL-NEXT:    flat_store_dwordx4 v[0:1], v[11:14]
+; GFX802-GISEL-NEXT:    flat_store_dwordx4 v[19:20], v[15:18]
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-GISEL-LABEL: test_writelane_v4f64:
+; GFX1010-GISEL:       ; %bb.0:
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-GISEL-NEXT:    s_clause 0x1
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[11:14], v[0:1], off
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[15:18], v[0:1], off offset:16
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s5, v10
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s8, v5
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s9, v6
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s10, v7
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s11, v8
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s12, v9
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v11, s4, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v12, s6, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v13, s7, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v14, s8, s5
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v15, s9, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v16, s10, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v17, s11, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v18, s12, s5
+; GFX1010-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[11:14], off
+; GFX1010-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[15:18], off offset:16
+; GFX1010-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1100-GISEL-LABEL: test_writelane_v4f64:
+; GFX1100-GISEL:       ; %bb.0:
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-GISEL-NEXT:    s_clause 0x1
+; GFX1100-GISEL-NEXT:    global_load_b128 v[11:14], v[0:1], off
+; GFX1100-GISEL-NEXT:    global_load_b128 v[15:18], v[0:1], off offset:16
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v10
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s3, v4
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s4, v5
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s5, v6
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s6, v7
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s7, v8
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s8, v9
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v11, s0, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v12, s2, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v13, s3, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v14, s4, s1
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v15, s5, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v16, s6, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v17, s7, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v18, s8, s1
+; GFX1100-GISEL-NEXT:    s_clause 0x1
+; GFX1100-GISEL-NEXT:    global_store_b128 v[0:1], v[11:14], off
+; GFX1100-GISEL-NEXT:    global_store_b128 v[0:1], v[15:18], off offset:16
+; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %oldval = load <4 x double>, ptr addrspace(1) %out
+  %writelane = call <4 x double> @llvm.amdgcn.writelane.v4f64(<4 x double> %src, i32 %src1, <4 x double> %oldval)
+  store <4 x double> %writelane, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define void @test_writelane_v8f64(ptr addrspace(1) %out, <8 x double> %src, i32 %src1) {
+; GFX802-SDAG-LABEL: test_writelane_v8f64:
+; GFX802-SDAG:       ; %bb.0:
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s4, v18
+; GFX802-SDAG-NEXT:    flat_load_dwordx4 v[18:21], v[0:1]
+; GFX802-SDAG-NEXT:    v_add_u32_e32 v22, vcc, 16, v0
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s8, v2
+; GFX802-SDAG-NEXT:    v_addc_u32_e32 v23, vcc, 0, v1, vcc
+; GFX802-SDAG-NEXT:    flat_load_dwordx4 v[2:5], v[22:23]
+; GFX802-SDAG-NEXT:    s_mov_b32 m0, s4
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s4, v9
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s10, v15
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s11, v14
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s12, v13
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s13, v12
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s14, v11
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s15, v10
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s9, v16
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v21, s5, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v20, s6, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v19, s7, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v18, s8, m0
+; GFX802-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[18:21]
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s5, v8
+; GFX802-SDAG-NEXT:    v_add_u32_e32 v18, vcc, 32, v0
+; GFX802-SDAG-NEXT:    v_addc_u32_e32 v19, vcc, 0, v1, vcc
+; GFX802-SDAG-NEXT:    v_add_u32_e32 v0, vcc, 48, v0
+; GFX802-SDAG-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s6, v7
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s7, v6
+; GFX802-SDAG-NEXT:    flat_load_dwordx4 v[6:9], v[0:1]
+; GFX802-SDAG-NEXT:    flat_load_dwordx4 v[12:15], v[18:19]
+; GFX802-SDAG-NEXT:    v_readfirstlane_b32 s8, v17
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(3)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v5, s4, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v4, s5, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v3, s6, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v2, s7, m0
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v9, s8, m0
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-SDAG-NEXT:    v_writelane_b32 v15, s12, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v14, s13, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v13, s14, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v12, s15, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v8, s9, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v7, s10, m0
+; GFX802-SDAG-NEXT:    v_writelane_b32 v6, s11, m0
+; GFX802-SDAG-NEXT:    flat_store_dwordx4 v[18:19], v[12:15]
+; GFX802-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[6:9]
+; GFX802-SDAG-NEXT:    flat_store_dwordx4 v[22:23], v[2:5]
+; GFX802-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-SDAG-LABEL: test_writelane_v8f64:
+; GFX1010-SDAG:       ; %bb.0:
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-SDAG-NEXT:    s_clause 0x3
+; GFX1010-SDAG-NEXT:    global_load_dwordx4 v[19:22], v[0:1], off offset:16
+; GFX1010-SDAG-NEXT:    global_load_dwordx4 v[23:26], v[0:1], off
+; GFX1010-SDAG-NEXT:    global_load_dwordx4 v[27:30], v[0:1], off offset:48
+; GFX1010-SDAG-NEXT:    global_load_dwordx4 v[31:34], v[0:1], off offset:32
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s5, v18
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s17, v13
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s18, v12
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s19, v11
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s20, v10
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s13, v17
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s14, v16
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s16, v14
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s9, v5
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s10, v4
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s12, v2
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s4, v9
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s6, v8
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX1010-SDAG-NEXT:    v_readfirstlane_b32 s8, v6
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(3)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v22, s4, s5
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(2)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v26, s9, s5
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v30, s13, s5
+; GFX1010-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v34, s17, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v33, s18, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v32, s19, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v31, s20, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v29, s14, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v28, s15, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v27, s16, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v25, s10, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v24, s11, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v23, s12, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v21, s6, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v20, s7, s5
+; GFX1010-SDAG-NEXT:    v_writelane_b32 v19, s8, s5
+; GFX1010-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[31:34], off offset:32
+; GFX1010-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[27:30], off offset:48
+; GFX1010-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[23:26], off
+; GFX1010-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[19:22], off offset:16
+; GFX1010-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-LABEL: test_writelane_v8f64:
+; GFX1100-SDAG:       ; %bb.0:
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-NEXT:    s_clause 0x3
+; GFX1100-SDAG-NEXT:    global_load_b128 v[19:22], v[0:1], off offset:16
+; GFX1100-SDAG-NEXT:    global_load_b128 v[23:26], v[0:1], off
+; GFX1100-SDAG-NEXT:    global_load_b128 v[27:30], v[0:1], off offset:48
+; GFX1100-SDAG-NEXT:    global_load_b128 v[31:34], v[0:1], off offset:32
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v18
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s14, v12
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s15, v11
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s16, v10
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s9, v17
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s10, v16
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s11, v15
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s12, v14
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s8, v2
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v9
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v8
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(3)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v22, s0, s1
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(2)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v26, s5, s1
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v30, s9, s1
+; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v34, s13, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v33, s14, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v32, s15, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v31, s16, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v29, s10, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v28, s11, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v27, s12, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v25, s6, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v24, s7, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v23, s8, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v21, s2, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v20, s3, s1
+; GFX1100-SDAG-NEXT:    v_writelane_b32 v19, s4, s1
+; GFX1100-SDAG-NEXT:    s_clause 0x3
+; GFX1100-SDAG-NEXT:    global_store_b128 v[0:1], v[31:34], off offset:32
+; GFX1100-SDAG-NEXT:    global_store_b128 v[0:1], v[27:30], off offset:48
+; GFX1100-SDAG-NEXT:    global_store_b128 v[0:1], v[23:26], off
+; GFX1100-SDAG-NEXT:    global_store_b128 v[0:1], v[19:22], off offset:16
+; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX802-GISEL-LABEL: test_writelane_v8f64:
+; GFX802-GISEL:       ; %bb.0:
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s5, v18
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[18:21], v[0:1]
+; GFX802-GISEL-NEXT:    v_add_u32_e32 v22, vcc, 16, v0
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s8, v5
+; GFX802-GISEL-NEXT:    v_addc_u32_e32 v23, vcc, 0, v1, vcc
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[2:5], v[22:23]
+; GFX802-GISEL-NEXT:    s_mov_b32 m0, s5
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s5, v7
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s9, v11
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s10, v12
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s11, v13
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s12, v14
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s13, v15
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s14, v16
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s15, v17
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v18, s4, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v19, s6, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v20, s7, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v21, s8, m0
+; GFX802-GISEL-NEXT:    flat_store_dwordx4 v[0:1], v[18:21]
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX802-GISEL-NEXT:    v_add_u32_e32 v18, vcc, 32, v0
+; GFX802-GISEL-NEXT:    v_addc_u32_e32 v19, vcc, 0, v1, vcc
+; GFX802-GISEL-NEXT:    v_add_u32_e32 v0, vcc, 48, v0
+; GFX802-GISEL-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s6, v8
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s7, v9
+; GFX802-GISEL-NEXT:    v_readfirstlane_b32 s8, v10
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[6:9], v[18:19]
+; GFX802-GISEL-NEXT:    flat_load_dwordx4 v[10:13], v[0:1]
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(3)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v2, s4, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v3, s5, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v4, s6, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v5, s7, m0
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v6, s8, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v7, s9, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v8, s10, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v9, s11, m0
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    v_writelane_b32 v10, s12, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v11, s13, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v12, s14, m0
+; GFX802-GISEL-NEXT:    v_writelane_b32 v13, s15, m0
+; GFX802-GISEL-NEXT:    flat_store_dwordx4 v[22:23], v[2:5]
+; GFX802-GISEL-NEXT:    flat_store_dwordx4 v[18:19], v[6:9]
+; GFX802-GISEL-NEXT:    flat_store_dwordx4 v[0:1], v[10:13]
+; GFX802-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX802-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1010-GISEL-LABEL: test_writelane_v8f64:
+; GFX1010-GISEL:       ; %bb.0:
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-GISEL-NEXT:    s_clause 0x3
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[19:22], v[0:1], off
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[23:26], v[0:1], off offset:16
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[27:30], v[0:1], off offset:32
+; GFX1010-GISEL-NEXT:    global_load_dwordx4 v[31:34], v[0:1], off offset:48
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s5, v18
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s8, v5
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s9, v6
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s10, v7
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s11, v8
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s12, v9
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s13, v10
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s14, v11
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s15, v12
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s16, v13
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s17, v14
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s18, v15
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s19, v16
+; GFX1010-GISEL-NEXT:    v_readfirstlane_b32 s20, v17
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(3)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v19, s4, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v20, s6, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v21, s7, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v22, s8, s5
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(2)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v23, s9, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v24, s10, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v25, s11, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v26, s12, s5
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v27, s13, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v28, s14, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v29, s15, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v30, s16, s5
+; GFX1010-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v31, s17, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v32, s18, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v33, s19, s5
+; GFX1010-GISEL-NEXT:    v_writelane_b32 v34, s20, s5
+; GFX1010-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[19:22], off
+; GFX1010-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[23:26], off offset:16
+; GFX1010-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[27:30], off offset:32
+; GFX1010-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[31:34], off offset:48
+; GFX1010-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1100-GISEL-LABEL: test_writelane_v8f64:
+; GFX1100-GISEL:       ; %bb.0:
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-GISEL-NEXT:    s_clause 0x3
+; GFX1100-GISEL-NEXT:    global_load_b128 v[19:22], v[0:1], off
+; GFX1100-GISEL-NEXT:    global_load_b128 v[23:26], v[0:1], off offset:16
+; GFX1100-GISEL-NEXT:    global_load_b128 v[27:30], v[0:1], off offset:32
+; GFX1100-GISEL-NEXT:    global_load_b128 v[31:34], v[0:1], off offset:48
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v18
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s3, v4
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s4, v5
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s5, v6
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s6, v7
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s7, v8
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s8, v9
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s9, v10
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s10, v11
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s11, v12
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s12, v13
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s13, v14
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s14, v15
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s15, v16
+; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s16, v17
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(3)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v19, s0, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v20, s2, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v21, s3, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v22, s4, s1
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(2)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v23, s5, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v24, s6, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v25, s7, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v26, s8, s1
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v27, s9, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v28, s10, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v29, s11, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v30, s12, s1
+; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v31, s13, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v32, s14, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v33, s15, s1
+; GFX1100-GISEL-NEXT:    v_writelane_b32 v34, s16, s1
+; GFX1100-GISEL-NEXT:    s_clause 0x3
+; GFX1100-GISEL-NEXT:    global_store_b128 v[0:1], v[19:22], off
+; GFX1100-GISEL-NEXT:    global_store_b128 v[0:1], v[23:26], off offset:16
+; GFX1100-GISEL-NEXT:    global_store_b128 v[0:1], v[27:30], off offset:32
+; GFX1100-GISEL-NEXT:    global_store_b128 v[0:1], v[31:34], off offset:48
+; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %oldval = load <8 x double>, ptr addrspace(1) %out
+  %writelane = call <8 x double> @llvm.amdgcn.writelane.v8f64(<8 x double> %src, i32 %src1, <8 x double> %oldval)
+  store <8 x double> %writelane, ptr addrspace(1) %out, align 4
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #2
 
 attributes #0 = { nounwind readnone convergent }

From f3c77445791b510858561cb424ffa1cd7513250b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=A9sz=C3=A1ros=20Gergely?= <gergely.meszaros@intel.com>
Date: Wed, 16 Apr 2025 08:02:42 +0200
Subject: [PATCH 081/710] [Clang][Sema] Fix
 -Whigher-precision-for-complex-division (#131477)

- Fix false positive when divisor is a real number.
- Fix false negative when divident is real, but divisor is complex.
- Fix false negative when due to promotion the division is performed in
higher precision than the divident.
- Fix false negative in divide and assign (`a /= b`).

Fixes: #131127

---------

Co-authored-by: Zahira Ammarguellat <zahira.ammarguellat@intel.com>
---
 clang/docs/ReleaseNotes.rst                   |  8 ++
 clang/lib/Sema/SemaExpr.cpp                   | 78 ++++++++--------
 .../complex-div-warn-higher-precision.cpp     | 93 +++++++++++++++++++
 3 files changed, 141 insertions(+), 38 deletions(-)
 create mode 100644 clang/test/Sema/complex-div-warn-higher-precision.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 84ad253c1ec4f..5af4c08f64cd8 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -371,6 +371,14 @@ Improvements to Clang's diagnostics
 
 - An error is now emitted when a ``musttail`` call is made to a function marked with the ``not_tail_called`` attribute. (#GH133509).
 
+- ``-Whigher-precisision-for-complex-divison`` warns when:
+
+  -	The divisor is complex.
+  -	When the complex division happens in a higher precision type due to arithmetic promotion.
+  -	When using the divide and assign operator (``/=``).
+
+  Fixes #GH131127
+
 Improvements to Clang's time-trace
 ----------------------------------
 
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index c65b4eadf9c67..6830bb5c01c7d 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -10602,6 +10602,45 @@ static void checkArithmeticNull(Sema &S, ExprResult &LHS, ExprResult &RHS,
       << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
 }
 
+static void DetectPrecisionLossInComplexDivision(Sema &S, QualType DivisorTy,
+                                                 SourceLocation OpLoc) {
+  // If the divisor is real, then this is real/real or complex/real division.
+  // Either way there can be no precision loss.
+  auto *CT = DivisorTy->getAs<ComplexType>();
+  if (!CT)
+    return;
+
+  QualType ElementType = CT->getElementType();
+  bool IsComplexRangePromoted = S.getLangOpts().getComplexRange() ==
+                                LangOptions::ComplexRangeKind::CX_Promoted;
+  if (!ElementType->isFloatingType() || !IsComplexRangePromoted)
+    return;
+
+  ASTContext &Ctx = S.getASTContext();
+  QualType HigherElementType = Ctx.GetHigherPrecisionFPType(ElementType);
+  const llvm::fltSemantics &ElementTypeSemantics =
+      Ctx.getFloatTypeSemantics(ElementType);
+  const llvm::fltSemantics &HigherElementTypeSemantics =
+      Ctx.getFloatTypeSemantics(HigherElementType);
+
+  if ((llvm::APFloat::semanticsMaxExponent(ElementTypeSemantics) * 2 + 1 >
+       llvm::APFloat::semanticsMaxExponent(HigherElementTypeSemantics)) ||
+      (HigherElementType == Ctx.LongDoubleTy &&
+       !Ctx.getTargetInfo().hasLongDoubleType())) {
+    // Retain the location of the first use of higher precision type.
+    if (!S.LocationOfExcessPrecisionNotSatisfied.isValid())
+      S.LocationOfExcessPrecisionNotSatisfied = OpLoc;
+    for (auto &[Type, Num] : S.ExcessPrecisionNotSatisfied) {
+      if (Type == HigherElementType) {
+        Num++;
+        return;
+      }
+    }
+    S.ExcessPrecisionNotSatisfied.push_back(std::make_pair(
+        HigherElementType, S.ExcessPrecisionNotSatisfied.size()));
+  }
+}
+
 static void DiagnoseDivisionSizeofPointerOrArray(Sema &S, Expr *LHS, Expr *RHS,
                                           SourceLocation Loc) {
   const auto *LUE = dyn_cast<UnaryExprOrTypeTraitExpr>(LHS);
@@ -10696,6 +10735,7 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
   if (compType.isNull() || !compType->isArithmeticType())
     return InvalidOperands(Loc, LHS, RHS);
   if (IsDiv) {
+    DetectPrecisionLossInComplexDivision(*this, RHS.get()->getType(), Loc);
     DiagnoseBadDivideOrRemainderValues(*this, LHS, RHS, Loc, IsDiv);
     DiagnoseDivisionSizeofPointerOrArray(*this, LHS.get(), RHS.get(), Loc);
   }
@@ -15347,39 +15387,6 @@ static void DiagnoseBinOpPrecedence(Sema &Self, BinaryOperatorKind Opc,
     DiagnoseShiftCompare(Self, OpLoc, LHSExpr, RHSExpr);
 }
 
-static void DetectPrecisionLossInComplexDivision(Sema &S, SourceLocation OpLoc,
-                                                 Expr *Operand) {
-  if (auto *CT = Operand->getType()->getAs<ComplexType>()) {
-    QualType ElementType = CT->getElementType();
-    bool IsComplexRangePromoted = S.getLangOpts().getComplexRange() ==
-                                  LangOptions::ComplexRangeKind::CX_Promoted;
-    if (ElementType->isFloatingType() && IsComplexRangePromoted) {
-      ASTContext &Ctx = S.getASTContext();
-      QualType HigherElementType = Ctx.GetHigherPrecisionFPType(ElementType);
-      const llvm::fltSemantics &ElementTypeSemantics =
-          Ctx.getFloatTypeSemantics(ElementType);
-      const llvm::fltSemantics &HigherElementTypeSemantics =
-          Ctx.getFloatTypeSemantics(HigherElementType);
-      if ((llvm::APFloat::semanticsMaxExponent(ElementTypeSemantics) * 2 + 1 >
-           llvm::APFloat::semanticsMaxExponent(HigherElementTypeSemantics)) ||
-          (HigherElementType == Ctx.LongDoubleTy &&
-           !Ctx.getTargetInfo().hasLongDoubleType())) {
-        // Retain the location of the first use of higher precision type.
-        if (!S.LocationOfExcessPrecisionNotSatisfied.isValid())
-          S.LocationOfExcessPrecisionNotSatisfied = OpLoc;
-        for (auto &[Type, Num] : S.ExcessPrecisionNotSatisfied) {
-          if (Type == HigherElementType) {
-            Num++;
-            return;
-          }
-        }
-        S.ExcessPrecisionNotSatisfied.push_back(std::make_pair(
-            HigherElementType, S.ExcessPrecisionNotSatisfied.size()));
-      }
-    }
-  }
-}
-
 ExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
                             tok::TokenKind Kind,
                             Expr *LHSExpr, Expr *RHSExpr) {
@@ -15390,11 +15397,6 @@ ExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
   // Emit warnings for tricky precedence issues, e.g. "bitfield & 0x4 == 0"
   DiagnoseBinOpPrecedence(*this, Opc, TokLoc, LHSExpr, RHSExpr);
 
-  // Emit warnings if the requested higher precision type equal to the current
-  // type precision.
-  if (Kind == tok::TokenKind::slash)
-    DetectPrecisionLossInComplexDivision(*this, TokLoc, LHSExpr);
-
   BuiltinCountedByRefKind K =
       BinaryOperator::isAssignmentOp(Opc) ? AssignmentKind : BinaryExprKind;
 
diff --git a/clang/test/Sema/complex-div-warn-higher-precision.cpp b/clang/test/Sema/complex-div-warn-higher-precision.cpp
new file mode 100644
index 0000000000000..a7cc2fbc7ed21
--- /dev/null
+++ b/clang/test/Sema/complex-div-warn-higher-precision.cpp
@@ -0,0 +1,93 @@
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-linux -verify=no-diag \
+// RUN: -DDIV_CC -DDIV_RC -DDIVASSIGN -DDIVMIXEDFD -DDIVMIXEDFD2 -DDIVMIXEDID -DDIVASSIGN_MIXEDFD
+
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-windows -verify=no-diag
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-windows -verify -DDIV_CC
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-windows -verify -DDIV_RC
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-windows -verify -DDIVASSIGN
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-windows -verify -DDIVMIXEDFD
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-windows -verify -DDIVMIXEDFD2
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-windows -verify -DDIVMIXEDID
+// RUN: %clang_cc1 %s -complex-range=promoted -fsyntax-only -triple x86_64-unknown-windows -verify -DDIVASSIGN_MIXEDFD
+
+_Complex double div_ccf(_Complex float a, _Complex float b) {
+    return a / b;
+}
+
+_Complex double div_cr(_Complex double a, double b) {
+    return a / b;
+}
+
+_Complex double div_cr_mixed1(_Complex double a, float b) {
+    return a / b;
+}
+
+_Complex double div_cr_mixed2(_Complex float a, double b) {
+    return a / b;
+}
+
+_Complex double div_rr(double a, double b) {
+    return a / b;
+}
+
+_Complex int div_ii(_Complex int a, _Complex int b) {
+    return a / b;
+}
+
+struct UserT {
+    friend UserT operator/(UserT, _Complex double);
+    friend UserT operator/(_Complex double, UserT);
+};
+
+UserT div_uc(UserT a, _Complex double b) {
+    return a / b;
+}
+
+UserT div_cu(_Complex double a, UserT b) {
+    return a / b;
+}
+
+#ifdef DIV_CC
+_Complex double div_cc(_Complex double a, const _Complex double b) {
+    return a / b; // #1
+}
+#endif // DIV_CC
+
+#ifdef DIV_RC
+_Complex double div_rc(double a, _Complex float b) {
+    return a / b; // #1
+}
+#endif // DIV_RC
+
+#ifdef DIVASSIGN
+_Complex double divassign(_Complex double a, _Complex double b) {
+    return a /= b; // #1
+}
+#endif // DIVASSIGN
+
+#ifdef DIVMIXEDFD
+_Complex double divmixedfd(_Complex float a, _Complex double b) {
+    return a / b; // #1
+}
+#endif // DIVMIXEDFD
+
+#ifdef DIVMIXEDFD2
+_Complex double divmixedfd2(_Complex double a, _Complex float b) {
+    return a / b; // #1
+}
+#endif // DIVMIXEDFD2
+
+#ifdef DIVMIXEDID
+_Complex double divmixedid(_Complex int a, _Complex double b) {
+    return a / b; // #1
+}
+#endif // DIVMIXEDID
+
+#ifdef DIVASSIGN_MIXEDFD
+_Complex double divassign_mixedfd(_Complex float a, _Complex double b) {
+    return a /= b; // #1
+}
+#endif // DIVMIXEDFD
+
+// no-diag-no-diagnostics
+// expected-warning@#1 {{excess precision is requested but the target does not support excess precision which may result in observable differences in complex division behavior}}

From dfb5b6e27ca3f8b79ebd3346d11b3088c1600b81 Mon Sep 17 00:00:00 2001
From: leecheechen <chenli@loongson.cn>
Date: Wed, 16 Apr 2025 14:12:00 +0800
Subject: [PATCH 082/710] [LoongArch] Don't crash on instruction prefetch
 intrinsics (#135760)

Instead of failing to select during isel, drop the intrinsic in
lowering.

Similar as the X86's PR. Seeing: https://reviews.llvm.org/D151050.

Fixes #134624
---
 .../LoongArch/LoongArchISelLowering.cpp       | 16 ++++++++-
 .../Target/LoongArch/LoongArchISelLowering.h  |  1 +
 llvm/test/CodeGen/LoongArch/prefetchi.ll      | 33 +++++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/LoongArch/prefetchi.ll

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index e5ccbe897d19c..f72b55e1d175c 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -99,7 +99,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 
-  setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
+  setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
   // Expand bitreverse.i16 with native-width bitrev and shift for now, before
   // we get to know which of sll and revb.2h is faster.
@@ -469,10 +469,24 @@ SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
     return lowerBITREVERSE(Op, DAG);
   case ISD::SCALAR_TO_VECTOR:
     return lowerSCALAR_TO_VECTOR(Op, DAG);
+  case ISD::PREFETCH:
+    return lowerPREFETCH(Op, DAG);
   }
   return SDValue();
 }
 
+SDValue LoongArchTargetLowering::lowerPREFETCH(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  unsigned IsData = Op.getConstantOperandVal(4);
+
+  // We don't support non-data prefetch.
+  // Just preserve the chain.
+  if (!IsData)
+    return Op.getOperand(0);
+
+  return Op;
+}
+
 SDValue
 LoongArchTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
                                                SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 52d88b9b24a6b..8c2d2597a26ec 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -353,6 +353,7 @@ class LoongArchTargetLowering : public TargetLowering {
   SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBITREVERSE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
 
   bool isFPImmLegal(const APFloat &Imm, EVT VT,
                     bool ForCodeSize) const override;
diff --git a/llvm/test/CodeGen/LoongArch/prefetchi.ll b/llvm/test/CodeGen/LoongArch/prefetchi.ll
new file mode 100644
index 0000000000000..a00f6f8161862
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/prefetchi.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+
+declare void @llvm.prefetch(ptr, i32, i32, i32) nounwind
+
+define dso_local void @prefetch_no_offset(ptr %ptr) nounwind {
+; LA32-LABEL: prefetch_no_offset:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: prefetch_no_offset:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    ret
+entry:
+  tail call void @llvm.prefetch(ptr %ptr, i32 0, i32 3, i32 0)
+  ret void
+}
+
+
+define dso_local void @prefetch_with_offset(ptr %ptr) nounwind {
+; LA32-LABEL: prefetch_with_offset:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: prefetch_with_offset:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    ret
+entry:
+  %addr = getelementptr i8, ptr %ptr, i64 200
+  tail call void @llvm.prefetch(ptr %addr, i32 0, i32 3, i32 0)
+  ret void
+}

From 9e650349bfdcd34998c461e1441f02a79c664d38 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 15 Apr 2025 23:16:52 -0700
Subject: [PATCH 083/710] [mlir] Construct SmallVector with ArrayRef (NFC)
 (#135899)

---
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index bce5b226635f3..9a1dfd8e17b85 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -483,8 +483,7 @@ LogicalResult tosa::ArgMaxOp::verify() {
 
   const ArrayRef<int64_t> inputShape = inputType.getShape();
   const ArrayRef<int64_t> outputShape = resultType.getShape();
-  llvm::SmallVector<int64_t> expectedOutputShape(inputShape.begin(),
-                                                 inputShape.end());
+  llvm::SmallVector<int64_t> expectedOutputShape(inputShape);
   expectedOutputShape.erase(expectedOutputShape.begin() + axis);
   if (failed(verifyCompatibleShape(expectedOutputShape, outputShape)))
     return emitOpError("expected output shape '")

From 52e3f3d68cbabf81c4c118cfb823828f03b712c4 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 15 Apr 2025 23:17:33 -0700
Subject: [PATCH 084/710] [mlir] Use llvm::make_first_range (NFC) (#135900)

---
 mlir/lib/Bytecode/Reader/BytecodeReader.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mlir/lib/Bytecode/Reader/BytecodeReader.cpp b/mlir/lib/Bytecode/Reader/BytecodeReader.cpp
index 0f2057cb32ff1..1052946d4550b 100644
--- a/mlir/lib/Bytecode/Reader/BytecodeReader.cpp
+++ b/mlir/lib/Bytecode/Reader/BytecodeReader.cpp
@@ -1984,8 +1984,7 @@ LogicalResult BytecodeReader::Impl::sortUseListOrder(Value value) {
     // If the bytecode file did not contain any custom use-list order, it means
     // that the order was descending useID. Hence, shuffle by the first index
     // of the `currentOrder` pair.
-    SmallVector<unsigned> shuffle = SmallVector<unsigned>(
-        llvm::map_range(currentOrder, [&](auto item) { return item.first; }));
+    SmallVector<unsigned> shuffle(llvm::make_first_range(currentOrder));
     value.shuffleUseList(shuffle);
     return success();
   }

From ac4712482e3ff886eee7c044dd33dd4b5d648036 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 15 Apr 2025 23:55:49 -0700
Subject: [PATCH 085/710] [LoongArch] Use FirstRelocationKind to remove
 ELFObjectWriter::recordRelocation special case

The current implementation of R_LARCH_SUB{8,16,32,64} and TLS relocation types relies on fixup kinds FirstLiteralRelocationKind + offset (originally intended for .reloc directives). While this is clever and prevents switch cases like

```
case fixup_...sub8:
  return ELF::R_LARCH_SUB8;
```

it needs revision.

GNU Assembler treats .reloc directives differently from standard relocations, notably by skipping

* Skipping STT_SECTION adjustments (when a referenced symbol is local and satisfies certain conditions, it can be redirected to a section symbol).
* Skipping STT_TLS symbol type setting for TLS relocations.

Encode relocatin type t with FirstRelocationKind+t instead of
FirstLiteralRelocationKind+t. The new value is less than
FirstLiteralRelocationKind and will not be treated as a .reloc
directive.

Close #135521
---
 llvm/include/llvm/MC/MCFixup.h                |  5 ++-
 llvm/lib/MC/ELFObjectWriter.cpp               |  3 +-
 .../MCTargetDesc/LoongArchAsmBackend.cpp      | 34 +++++++++----------
 .../MCTargetDesc/LoongArchELFObjectWriter.cpp | 14 ++++----
 .../MCTargetDesc/LoongArchFixupKinds.h        | 15 ++++----
 5 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/llvm/include/llvm/MC/MCFixup.h b/llvm/include/llvm/MC/MCFixup.h
index 7cf8ac2e39092..f27ddeae8b173 100644
--- a/llvm/include/llvm/MC/MCFixup.h
+++ b/llvm/include/llvm/MC/MCFixup.h
@@ -36,10 +36,13 @@ enum MCFixupKind {
 
   FirstTargetFixupKind = 128,
 
+  /// Targets can use FirstRelocationKind+t to encode relocation type t.
+  FirstRelocationKind = 256,
+
   /// The range [FirstLiteralRelocationKind, MaxTargetFixupKind) is used for
   /// relocations coming from .reloc directive. Fixup kind
   /// FirstLiteralRelocationKind+V represents the relocation type with number V.
-  FirstLiteralRelocationKind = 256,
+  FirstLiteralRelocationKind = 256 + 1032 + 32,
 
   /// Set limit to accommodate the highest reloc type in use for all Targets,
   /// currently R_AARCH64_IRELATIVE at 1032, including room for expansion.
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index db4b41e754581..46c71e28ebc13 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1385,8 +1385,7 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
 
   auto EMachine = TargetObjectWriter->getEMachine();
   unsigned Type;
-  if (Fixup.getKind() >= FirstLiteralRelocationKind &&
-      EMachine != ELF::EM_LOONGARCH)
+  if (Fixup.getKind() >= FirstLiteralRelocationKind)
     Type = Fixup.getKind() - FirstLiteralRelocationKind;
   else
     Type = TargetObjectWriter->getRelocType(Ctx, Target, Fixup, IsPCRel);
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index b6a98b3ff9aeb..78a54a9385b08 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -70,7 +70,7 @@ LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 
   // Fixup kinds from .reloc directive are like R_LARCH_NONE. They
   // do not require any extra processing.
-  if (Kind >= FirstLiteralRelocationKind)
+  if (unsigned(Kind) >= FirstRelocationKind)
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
@@ -152,10 +152,10 @@ void LoongArchAsmBackend::applyFixup(const MCAssembler &Asm,
   if (!Value)
     return; // Doesn't change encoding.
 
-  MCFixupKind Kind = Fixup.getKind();
-  if (Kind >= FirstLiteralRelocationKind)
+  auto Kind = Fixup.getTargetKind();
+  if (Kind >= FirstRelocationKind)
     return;
-  MCFixupKindInfo Info = getFixupKindInfo(Kind);
+  MCFixupKindInfo Info = getFixupKindInfo(MCFixupKind(Kind));
   MCContext &Ctx = Asm.getContext();
 
   // Fixup leb128 separately.
@@ -271,29 +271,27 @@ getRelocPairForSize(unsigned Size) {
   default:
     llvm_unreachable("unsupported fixup size");
   case 6:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_ADD6),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_SUB6));
+    return std::make_pair(MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD6),
+                          MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB6));
   case 8:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_ADD8),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_SUB8));
+    return std::make_pair(MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD8),
+                          MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB8));
   case 16:
     return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_ADD16),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_SUB16));
+        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD16),
+        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB16));
   case 32:
     return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_ADD32),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_SUB32));
+        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD32),
+        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB32));
   case 64:
     return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_ADD64),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_SUB64));
+        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD64),
+        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB64));
   case 128:
     return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_ADD_ULEB128),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_LARCH_SUB_ULEB128));
+        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD_ULEB128),
+        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB_ULEB128));
   }
 }
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
index 2e2a503d5304f..c117e9a60939f 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
@@ -50,6 +50,12 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx,
                                                 const MCValue &Target,
                                                 const MCFixup &Fixup,
                                                 bool IsPCRel) const {
+  // Determine the type of the relocation
+  unsigned Kind = Fixup.getTargetKind();
+
+  if (Kind >= FirstLiteralRelocationKind)
+    return Kind - FirstLiteralRelocationKind;
+
   switch (Target.getSpecifier()) {
   case LoongArchMCExpr::VK_TLS_LE_HI20:
   case LoongArchMCExpr::VK_TLS_IE_PC_HI20:
@@ -71,12 +77,8 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx,
     break;
   }
 
-  // Determine the type of the relocation
-  unsigned Kind = Fixup.getTargetKind();
-
-  if (Kind >= FirstLiteralRelocationKind)
-    return Kind - FirstLiteralRelocationKind;
-
+  if (Kind >= FirstRelocationKind)
+    return Kind - FirstRelocationKind;
   switch (Kind) {
   default:
     Ctx.reportError(Fixup.getLoc(), "Unsupported relocation type");
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
index 370f5b0189b51..aac295c35db69 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
@@ -52,11 +52,10 @@ enum Fixups {
   fixup_loongarch_invalid,
   NumTargetFixupKinds = fixup_loongarch_invalid - FirstTargetFixupKind,
 
-  // Define fixups for force relocation as FirstLiteralRelocationKind+V
+  // Define fixups for force relocation as FirstRelocationKind+V
   // represents the relocation type with number V.
   // 20-bit fixup corresponding to %pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_pcala_hi20 =
-      FirstLiteralRelocationKind + ELF::R_LARCH_PCALA_HI20,
+  fixup_loongarch_pcala_hi20 = FirstRelocationKind + ELF::R_LARCH_PCALA_HI20,
   // 12-bit fixup corresponding to %pc_lo12(foo) for instructions like addi.w/d.
   fixup_loongarch_pcala_lo12,
   // 20-bit fixup corresponding to %pc64_lo20(foo) for instruction lu32i.d.
@@ -83,7 +82,7 @@ enum Fixups {
   // Skip R_LARCH_TLS_LE_*.
   // 20-bit fixup corresponding to %ie_pc_hi20(foo) for instruction pcalau12i.
   fixup_loongarch_tls_ie_pc_hi20 =
-      FirstLiteralRelocationKind + ELF::R_LARCH_TLS_IE_PC_HI20,
+      FirstRelocationKind + ELF::R_LARCH_TLS_IE_PC_HI20,
   // 12-bit fixup corresponding to %ie_pc_lo12(foo) for instructions
   // ld.w/ld.d/add.d.
   fixup_loongarch_tls_ie_pc_lo12,
@@ -108,17 +107,17 @@ enum Fixups {
   // 20-bit fixup corresponding to %gd_hi20(foo) for instruction lu12i.w.
   fixup_loongarch_tls_gd_hi20,
   // Generate an R_LARCH_RELAX which indicates the linker may relax here.
-  fixup_loongarch_relax = FirstLiteralRelocationKind + ELF::R_LARCH_RELAX,
+  fixup_loongarch_relax = FirstRelocationKind + ELF::R_LARCH_RELAX,
   // Generate an R_LARCH_ALIGN which indicates the linker may fixup align here.
-  fixup_loongarch_align = FirstLiteralRelocationKind + ELF::R_LARCH_ALIGN,
+  fixup_loongarch_align = FirstRelocationKind + ELF::R_LARCH_ALIGN,
   // 20-bit fixup corresponding to %pcrel_20(foo) for instruction pcaddi.
   fixup_loongarch_pcrel20_s2,
   // 36-bit fixup corresponding to %call36(foo) for a pair instructions:
   // pcaddu18i+jirl.
-  fixup_loongarch_call36 = FirstLiteralRelocationKind + ELF::R_LARCH_CALL36,
+  fixup_loongarch_call36 = FirstRelocationKind + ELF::R_LARCH_CALL36,
   // 20-bit fixup corresponding to %desc_pc_hi20(foo) for instruction pcalau12i.
   fixup_loongarch_tls_desc_pc_hi20 =
-      FirstLiteralRelocationKind + ELF::R_LARCH_TLS_DESC_PC_HI20,
+      FirstRelocationKind + ELF::R_LARCH_TLS_DESC_PC_HI20,
   // 12-bit fixup corresponding to %desc_pc_lo12(foo) for instructions like
   // addi.w/d.
   fixup_loongarch_tls_desc_pc_lo12,

From a56f966417bc53051fa39e3db6fcc95f9abf0b5c Mon Sep 17 00:00:00 2001
From: Mythreya <git@mythreya.dev>
Date: Wed, 16 Apr 2025 00:00:14 -0700
Subject: [PATCH 086/710] [clangd][docs] Fix incorrect docstring for
 header-insertion "Never" (#135921)

Docstring fix for changes introduced in PR #128503
---
 clang-tools-extra/clangd/ConfigFragment.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h
index f05ed4d1acdfc..2363b483ab96d 100644
--- a/clang-tools-extra/clangd/ConfigFragment.h
+++ b/clang-tools-extra/clangd/ConfigFragment.h
@@ -347,7 +347,7 @@ struct Fragment {
     ///   "IWYU": Include what you use. Insert the owning header for top-level
     ///     symbols, unless the header is already directly included or the
     ///     symbol is forward-declared
-    ///   "NeverInsert": Never insert headers
+    ///   "Never": Never insert headers
     std::optional<Located<std::string>> HeaderInsertion;
   };
   CompletionBlock Completion;

From 05eafd9f2b14f2e8d2d95f46465c5cc53aafbc56 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Wed, 16 Apr 2025 09:00:52 +0200
Subject: [PATCH 087/710] [clang][bytecode] Explicitly mark constexpr-unknown
 variables as such (#135806)

Instead of trying to figure out what's constexpr-unknown later on.
---
 clang/lib/AST/ByteCode/Compiler.cpp | 24 +++++++++++++-------
 clang/lib/AST/ByteCode/Compiler.h   | 12 ++++++----
 clang/lib/AST/ByteCode/Descriptor.h |  1 +
 clang/lib/AST/ByteCode/Disasm.cpp   |  2 ++
 clang/lib/AST/ByteCode/Interp.cpp   |  9 ++++----
 clang/lib/AST/ByteCode/Program.cpp  |  2 +-
 clang/test/AST/ByteCode/codegen.cpp | 35 +++++++++++++++++++++++++++--
 7 files changed, 65 insertions(+), 20 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 2e22c85ed5f6d..afd8d09a088cd 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -4293,7 +4293,8 @@ bool Compiler<Emitter>::emitConst(const APSInt &Value, const Expr *E) {
 
 template <class Emitter>
 unsigned Compiler<Emitter>::allocateLocalPrimitive(
-    DeclTy &&Src, PrimType Ty, bool IsConst, const ValueDecl *ExtendingDecl) {
+    DeclTy &&Src, PrimType Ty, bool IsConst, const ValueDecl *ExtendingDecl,
+    bool IsConstexprUnknown) {
   // Make sure we don't accidentally register the same decl twice.
   if (const auto *VD =
           dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
@@ -4307,6 +4308,7 @@ unsigned Compiler<Emitter>::allocateLocalPrimitive(
   //   or isa<MaterializeTemporaryExpr>().
   Descriptor *D = P.createDescriptor(Src, Ty, nullptr, Descriptor::InlineDescMD,
                                      IsConst, isa<const Expr *>(Src));
+  D->IsConstexprUnknown = IsConstexprUnknown;
   Scope::Local Local = this->createLocal(D);
   if (auto *VD = dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>()))
     Locals.insert({VD, Local});
@@ -4320,7 +4322,8 @@ unsigned Compiler<Emitter>::allocateLocalPrimitive(
 template <class Emitter>
 std::optional<unsigned>
 Compiler<Emitter>::allocateLocal(DeclTy &&Src, QualType Ty,
-                                 const ValueDecl *ExtendingDecl) {
+                                 const ValueDecl *ExtendingDecl,
+                                 bool IsConstexprUnknown) {
   // Make sure we don't accidentally register the same decl twice.
   if ([[maybe_unused]] const auto *VD =
           dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
@@ -4349,6 +4352,7 @@ Compiler<Emitter>::allocateLocal(DeclTy &&Src, QualType Ty,
       IsTemporary, /*IsMutable=*/false, Init);
   if (!D)
     return std::nullopt;
+  D->IsConstexprUnknown = IsConstexprUnknown;
 
   Scope::Local Local = this->createLocal(D);
   if (Key)
@@ -4460,9 +4464,10 @@ bool Compiler<Emitter>::visitExpr(const Expr *E, bool DestroyToplevelScope) {
 }
 
 template <class Emitter>
-VarCreationState Compiler<Emitter>::visitDecl(const VarDecl *VD) {
+VarCreationState Compiler<Emitter>::visitDecl(const VarDecl *VD,
+                                              bool IsConstexprUnknown) {
 
-  auto R = this->visitVarDecl(VD, /*Toplevel=*/true);
+  auto R = this->visitVarDecl(VD, /*Toplevel=*/true, IsConstexprUnknown);
 
   if (R.notCreated())
     return R;
@@ -4550,7 +4555,8 @@ bool Compiler<Emitter>::visitDeclAndReturn(const VarDecl *VD,
 
 template <class Emitter>
 VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
-                                                 bool Toplevel) {
+                                                 bool Toplevel,
+                                                 bool IsConstexprUnknown) {
   // We don't know what to do with these, so just return false.
   if (VD->getType().isNull())
     return false;
@@ -4620,7 +4626,8 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
 
     if (VarT) {
       unsigned Offset = this->allocateLocalPrimitive(
-          VD, *VarT, VD->getType().isConstQualified());
+          VD, *VarT, VD->getType().isConstQualified(), nullptr,
+          IsConstexprUnknown);
       if (Init) {
         // If this is a toplevel declaration, create a scope for the
         // initializer.
@@ -4636,7 +4643,8 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
         }
       }
     } else {
-      if (std::optional<unsigned> Offset = this->allocateLocal(VD)) {
+      if (std::optional<unsigned> Offset = this->allocateLocal(
+              VD, VD->getType(), nullptr, IsConstexprUnknown)) {
         if (!Init)
           return true;
 
@@ -6461,7 +6469,7 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
 
   // In case we need to re-visit a declaration.
   auto revisit = [&](const VarDecl *VD) -> bool {
-    auto VarState = this->visitDecl(VD);
+    auto VarState = this->visitDecl(VD, /*IsConstexprUnknown=*/true);
 
     if (VarState.notCreated())
       return true;
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index 858957367d85d..a3090a8a31189 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -286,8 +286,10 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   /// intact.
   bool delegate(const Expr *E);
   /// Creates and initializes a variable from the given decl.
-  VarCreationState visitVarDecl(const VarDecl *VD, bool Toplevel = false);
-  VarCreationState visitDecl(const VarDecl *VD);
+  VarCreationState visitVarDecl(const VarDecl *VD, bool Toplevel = false,
+                                bool IsConstexprUnknown = false);
+  VarCreationState visitDecl(const VarDecl *VD,
+                             bool IsConstexprUnknown = false);
   /// Visit an APValue.
   bool visitAPValue(const APValue &Val, PrimType ValType, const Expr *E);
   bool visitAPValueInitializer(const APValue &Val, const Expr *E, QualType T);
@@ -303,12 +305,14 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
 
   /// Creates a local primitive value.
   unsigned allocateLocalPrimitive(DeclTy &&Decl, PrimType Ty, bool IsConst,
-                                  const ValueDecl *ExtendingDecl = nullptr);
+                                  const ValueDecl *ExtendingDecl = nullptr,
+                                  bool IsConstexprUnknown = false);
 
   /// Allocates a space storing a local given its type.
   std::optional<unsigned>
   allocateLocal(DeclTy &&Decl, QualType Ty = QualType(),
-                const ValueDecl *ExtendingDecl = nullptr);
+                const ValueDecl *ExtendingDecl = nullptr,
+                bool IsConstexprUnknown = false);
   std::optional<unsigned> allocateTemporary(const Expr *E);
 
 private:
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 9acabfd31d80b..a0705cc8c377f 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -168,6 +168,7 @@ struct Descriptor final {
   const bool IsArray = false;
   /// Flag indicating if this is a dummy descriptor.
   bool IsDummy = false;
+  bool IsConstexprUnknown = false;
 
   /// Storage management methods.
   const BlockCtorFn CtorFn = nullptr;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index d4c9ce6050b85..4bdf0f0afb1b0 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -376,6 +376,8 @@ LLVM_DUMP_METHOD void Descriptor::dump(llvm::raw_ostream &OS) const {
 
   if (isDummy())
     OS << " dummy";
+  if (IsConstexprUnknown)
+    OS << " constexpr-unknown";
 }
 
 /// Dump descriptor, including all valid offsets.
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 3e1f36da8925f..4625154ac353d 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -299,15 +299,14 @@ void cleanupAfterFunctionCall(InterpState &S, CodePtr OpPC,
     TYPE_SWITCH(Ty, S.Stk.discard<T>());
 }
 
-// FIXME: Instead of using this fairly expensive test, we should
-// just mark constexpr-unknown values when creating them.
 bool isConstexprUnknown(const Pointer &P) {
   if (!P.isBlockPointer())
     return false;
+
   if (P.isDummy())
-    return false;
-  const VarDecl *VD = P.block()->getDescriptor()->asVarDecl();
-  return VD && VD->hasLocalStorage() && !isa<ParmVarDecl>(VD);
+    return isa_and_nonnull<ParmVarDecl>(P.getDeclDesc()->asValueDecl());
+
+  return P.getDeclDesc()->IsConstexprUnknown;
 }
 
 bool CheckBCPResult(InterpState &S, const Pointer &Ptr) {
diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
index c83f5579fd55f..fdb7b960ff06d 100644
--- a/clang/lib/AST/ByteCode/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -156,7 +156,7 @@ unsigned Program::getOrCreateDummy(const DeclTy &D) {
   if (const auto *E = dyn_cast<const Expr *>(D)) {
     QT = E->getType();
   } else {
-    const ValueDecl *VD = cast<ValueDecl>(cast<const Decl *>(D));
+    const auto *VD = cast<ValueDecl>(cast<const Decl *>(D));
     IsWeak = VD->isWeak();
     QT = VD->getType();
     if (const auto *RT = QT->getAs<ReferenceType>())
diff --git a/clang/test/AST/ByteCode/codegen.cpp b/clang/test/AST/ByteCode/codegen.cpp
index 7c853a20362b8..6f9e75eac6026 100644
--- a/clang/test/AST/ByteCode/codegen.cpp
+++ b/clang/test/AST/ByteCode/codegen.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s                                         | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s -fexperimental-new-constant-interpreter | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s -fcxx-exceptions                                         | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s -fcxx-exceptions -fexperimental-new-constant-interpreter | FileCheck %s
 
 #ifdef __SIZEOF_INT128__
 // CHECK: @PR11705 = global i128 0
@@ -104,3 +104,34 @@ int notdead() {
 }
 // CHECK: _ZZ7notdeadvEN3$_0clEv
 // CHECK: ret i32 %cond
+
+/// The conmparison of those two parameters should NOT work.
+bool paramcmp(const int& lhs, const int& rhs) {
+  if (&lhs == &rhs)
+    return true;
+  return false;
+}
+// CHECK: _Z8paramcmpRKiS0_
+// CHECK: if.then
+// CHECK: if.end
+
+/// &x == &OuterX should work and return 0.
+class X {
+public:
+  X();
+  X(const X&);
+  X(const volatile X &);
+  ~X();
+};
+
+extern X OuterX;
+
+X test24() {
+  X x;
+  if (&x == &OuterX)
+    throw 0;
+  return x;
+}
+// CHECK: _Z6test24v
+// CHECK-NOT: eh.resume
+// CHECK-NOT: unreachable

From c1fc4c6a69c6ca0ae6aa060e4377ab5040505a66 Mon Sep 17 00:00:00 2001
From: Clo91eaf <Clo91eaf@qq.com>
Date: Wed, 16 Apr 2025 15:05:15 +0800
Subject: [PATCH 088/710] [mlir][SMT] fix the operation name in
 ArrayBroadcastOp description (#135746)

---
 mlir/include/mlir/Dialect/SMT/IR/SMTArrayOps.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/SMT/IR/SMTArrayOps.td b/mlir/include/mlir/Dialect/SMT/IR/SMTArrayOps.td
index 1869f4ae81595..ac1c2f3ed409a 100644
--- a/mlir/include/mlir/Dialect/SMT/IR/SMTArrayOps.td
+++ b/mlir/include/mlir/Dialect/SMT/IR/SMTArrayOps.td
@@ -75,7 +75,7 @@ def ArrayBroadcastOp : SMTArrayOp<"broadcast", [
     This operation represents a broadcast of the 'value' operand to all indices
     of the array. It is equivalent to
     ```
-    %0 = smt.declare "array" : !smt.array<[!smt.int -> !smt.bool]>
+    %0 = smt.declare_fun "array" : !smt.array<[!smt.int -> !smt.bool]>
     %1 = smt.forall ["idx"] {
     ^bb0(%idx: !smt.int):
       %2 = smt.array.select %0[%idx] : !smt.array<[!smt.int -> !smt.bool]>

From a630ef71e84a6bf09a99053ea42d37632ca0d18a Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 16 Apr 2025 00:07:16 -0700
Subject: [PATCH 089/710] ELFObjectWriter: Disable STT_SECTION adjustment for
 .reloc

... to match GNU Assembler. This generalizes the
SHT_LLVM_CALL_GRAPH_PROFILE special case (which uses .reloc with
BFD_RELOC_NONE https://reviews.llvm.org/D104080).

Targets that want STT_SECTION adjustment cannot use
FirstLiteralRelocationKind derived fixup kinds.

Depends on the fix of #135521

Pull Request: https://github.com/llvm/llvm-project/pull/135519
---
 llvm/lib/MC/ELFObjectWriter.cpp    | 5 ++---
 llvm/test/MC/ELF/reloc-directive.s | 7 ++++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 46c71e28ebc13..2ae9c21271624 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1395,9 +1395,8 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
   if (UseSectionSym) {
     UseSectionSym = useSectionSymbol(Asm, Target, SymA, C, Type);
 
-    // Disable STT_SECTION adjustment for CG Profile to help with --cg-profile.
-    const auto *Parent = cast<MCSectionELF>(Fragment->getParent());
-    UseSectionSym &= Parent->getType() != ELF::SHT_LLVM_CALL_GRAPH_PROFILE;
+    // Disable STT_SECTION adjustment for .reloc directives.
+    UseSectionSym &= Fixup.getKind() < FirstLiteralRelocationKind;
   }
 
   uint64_t Addend = UseSectionSym ? C + Asm.getSymbolOffset(*SymA) : C;
diff --git a/llvm/test/MC/ELF/reloc-directive.s b/llvm/test/MC/ELF/reloc-directive.s
index f4121ef071810..42995aa9e7d81 100644
--- a/llvm/test/MC/ELF/reloc-directive.s
+++ b/llvm/test/MC/ELF/reloc-directive.s
@@ -8,24 +8,25 @@
 # ASM-NEXT: .Ltmp1:
 # ASM-NEXT:  .reloc .Ltmp1-1, R_X86_64_NONE, foo
 # ASM-NEXT: .Ltmp2:
-# ASM-NEXT:  .reloc 2+.Ltmp2, R_X86_64_NONE, foo
+# ASM-NEXT:  .reloc 2+.Ltmp2, R_X86_64_NONE, local
 # ASM-NEXT:  .reloc 1+foo+3, R_X86_64_NONE, data+1
 # ASM-NEXT: .Ltmp3:
 # ASM-NEXT:  .reloc .Ltmp3, BFD_RELOC_NONE, unused
 
 # CHECK:      0x2 R_X86_64_NONE foo 0x0
 # CHECK-NEXT: 0x0 R_X86_64_NONE foo 0x0
-# CHECK-NEXT: 0x3 R_X86_64_NONE foo 0x0
+# CHECK-NEXT: 0x3 R_X86_64_NONE local 0x0
 # CHECK-NEXT: 0x4 R_X86_64_NONE data 0x1
 # CHECK-NEXT: 0x1 R_X86_64_NONE unused 0x0
 
 .text
 .globl foo
 foo:
+local:
   ret
   .reloc .+3-2, R_X86_64_NONE, foo
   .reloc .-1, R_X86_64_NONE, foo
-  .reloc 2+., R_X86_64_NONE, foo
+  .reloc 2+., R_X86_64_NONE, local
   .reloc 1+foo+3, R_X86_64_NONE, data+1
   .reloc ., BFD_RELOC_NONE, unused
 

From 507d7dc651b28d8a975ba8ca6e8f5906b07e37e7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 16 Apr 2025 00:09:07 -0700
Subject: [PATCH 090/710] [LoongArch] Simplify getRelocType

---
 .../LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp    | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
index c117e9a60939f..d129e0e2497e4 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
@@ -50,12 +50,6 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx,
                                                 const MCValue &Target,
                                                 const MCFixup &Fixup,
                                                 bool IsPCRel) const {
-  // Determine the type of the relocation
-  unsigned Kind = Fixup.getTargetKind();
-
-  if (Kind >= FirstLiteralRelocationKind)
-    return Kind - FirstLiteralRelocationKind;
-
   switch (Target.getSpecifier()) {
   case LoongArchMCExpr::VK_TLS_LE_HI20:
   case LoongArchMCExpr::VK_TLS_IE_PC_HI20:
@@ -77,6 +71,7 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx,
     break;
   }
 
+  unsigned Kind = Fixup.getTargetKind();
   if (Kind >= FirstRelocationKind)
     return Kind - FirstRelocationKind;
   switch (Kind) {

From 3d97d71e66036f51cf0b45cc7d5f3a0a14192eb4 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 00:33:29 -0700
Subject: [PATCH 091/710] [Lex] Use llvm::make_second_range (NFC) (#135902)

---
 clang/lib/Lex/HeaderSearch.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
index 9283a0f4fce55..2665580e5afce 100644
--- a/clang/lib/Lex/HeaderSearch.cpp
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -1932,8 +1932,7 @@ void HeaderSearch::collectAllModules(SmallVectorImpl<Module *> &Modules) {
   }
 
   // Populate the list of modules.
-  llvm::transform(ModMap.modules(), std::back_inserter(Modules),
-                  [](const auto &NameAndMod) { return NameAndMod.second; });
+  llvm::append_range(Modules, llvm::make_second_range(ModMap.modules()));
 }
 
 void HeaderSearch::loadTopLevelSystemModules() {

From 1256ca04c2064f2ef05625ff93a7954642af84a1 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 00:36:36 -0700
Subject: [PATCH 092/710] [CodeGen] Call DenseMap::erase directly (NFC)
 (#135898)

---
 llvm/lib/CodeGen/MachineFunction.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index eb0a16732b058..833b765b14d35 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -946,9 +946,7 @@ void MachineFunction::eraseAdditionalCallInfo(const MachineInstr *MI) {
   if (CSIt != CallSitesInfo.end())
     CallSitesInfo.erase(CSIt);
 
-  CalledGlobalsMap::iterator CGIt = CalledGlobalsInfo.find(CallMI);
-  if (CGIt != CalledGlobalsInfo.end())
-    CalledGlobalsInfo.erase(CGIt);
+  CalledGlobalsInfo.erase(CallMI);
 }
 
 void MachineFunction::copyAdditionalCallInfo(const MachineInstr *Old,

From bf0de88696095342aaa58e5e0a0105403d5ebd5e Mon Sep 17 00:00:00 2001
From: Nuno Lopes <nuno.lopes@tecnico.ulisboa.pt>
Date: Wed, 16 Apr 2025 08:38:37 +0100
Subject: [PATCH 093/710] code format checker: fix python error when the diff
 becomes empty

---
 llvm/utils/git/code-format-helper.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py
index 69d654b87e856..ed102b54f9b52 100755
--- a/llvm/utils/git/code-format-helper.py
+++ b/llvm/utils/git/code-format-helper.py
@@ -385,7 +385,9 @@ def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str
         # Each file is prefixed like:
         # diff --git a/file b/file
         for file in re.split("^diff --git ", stdout, 0, re.MULTILINE):
-            filename = re.match("a/([^ ]+)", file.splitlines()[0])[1]
+            lines = file.splitlines()
+            match = re.match("a/([^ ]+)", lines[0] if lines else "")
+            filename = match[1] if match else ""
             if filename.endswith(".ll"):
                 undef_regex = r"(?<!%)\bundef\b"
             else:

From 11857bef8a5fdfb8ab65971c3da6593c6076ff62 Mon Sep 17 00:00:00 2001
From: Sudharsan Veeravalli <quic_svs@quicinc.com>
Date: Wed, 16 Apr 2025 13:10:56 +0530
Subject: [PATCH 094/710] [RISCV] Add basic ISel patterns for Xqcisls
 instructions (#135918)

This patch adds basic instruction selection patterns for generating the
scaled load/store instructions that are a part of the Qualcomm uC
Xqcisls vendor extension.
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.td     |   2 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td |  28 +++
 llvm/test/CodeGen/RISCV/xqcisls.ll          | 207 ++++++++++++++++++++
 3 files changed, 236 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/RISCV/xqcisls.ll

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 1104d9089536f..baf2bae367df1 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -248,7 +248,7 @@ def InsnDirectiveOpcode : AsmOperandClass {
 
 def uimm1 : RISCVUImmLeafOp<1>;
 def uimm2 : RISCVUImmLeafOp<2>;
-def uimm3 : RISCVUImmOp<3>;
+def uimm3 : RISCVUImmLeafOp<3>;
 def uimm4 : RISCVUImmLeafOp<4>;
 def uimm5 : RISCVUImmLeafOp<5>;
 def uimm6 : RISCVUImmLeafOp<6>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index 6736b0f1d0328..bb0818b001d38 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -164,6 +164,9 @@ def AddLike: PatFrags<(ops node:$A, node:$B),
     return CurDAG->isBaseWithConstantOffset(SDValue(N, 0));
 }]>;
 
+def AddShl : PatFrag<(ops node:$Ra, node:$Rb, node:$SH3),
+                     (add node:$Ra, (shl node:$Rb, node:$SH3))>;
+
 //===----------------------------------------------------------------------===//
 // Instruction Formats
 //===----------------------------------------------------------------------===//
@@ -1252,6 +1255,14 @@ class QC48StPat<PatFrag StoreOp, RVInst48 Inst>
     : Pat<(StoreOp (i32 GPR:$rs2), (AddLike (i32 GPR:$rs1), simm26_nosimm12:$imm26)),
           (Inst GPR:$rs2, GPR:$rs1, simm26_nosimm12:$imm26)>;
 
+class QCScaledLdPat<PatFrag LoadOp, RVInst Inst>
+    : Pat<(i32 (LoadOp (AddShl (i32 GPRMem:$rs1), (i32 GPRNoX0:$rs2), uimm3:$shamt))),
+          (Inst GPRMem:$rs1, GPRNoX0:$rs2, uimm3:$shamt)>;
+
+class QCScaledStPat<PatFrag StoreOp, RVInst Inst>
+    : Pat<(StoreOp (i32 GPR:$rd), (AddShl (i32 GPRMem:$rs1), (i32 GPRNoX0:$rs2), uimm3:$shamt)),
+          (Inst GPR:$rd, GPRMem:$rs1, GPRNoX0:$rs2, uimm3:$shamt)>;
+
 /// Simple arithmetic operations
 
 let Predicates = [HasVendorXqcilia, IsRV32] in {
@@ -1266,6 +1277,8 @@ def : PatGprNoX0Simm26NoSimm12<or, QC_E_ORI>;
 def : PatGprNoX0Simm26NoSimm12<xor, QC_E_XORI>;
 } // Predicates = [HasVendorXqcilia, IsRV32]
 
+/// Load/Store operations
+
 let Predicates = [HasVendorXqcilo, IsRV32], AddedComplexity = 2 in {
   def : QC48LdPat<sextloadi8, QC_E_LB>;
   def : QC48LdPat<extloadi8, QC_E_LBU>; // Prefer unsigned due to no c.lb in Zcb.
@@ -1280,5 +1293,20 @@ let Predicates = [HasVendorXqcilo, IsRV32], AddedComplexity = 2 in {
   def : QC48StPat<store, QC_E_SW>;
 } // Predicates = [HasVendorXqcilo, IsRV32], AddedComplexity = 2
 
+
+let Predicates = [HasVendorXqcisls, IsRV32], AddedComplexity = 1 in {
+  def : QCScaledLdPat<sextloadi8, QC_LRB>;
+  def : QCScaledLdPat<extloadi8, QC_LRBU>;
+  def : QCScaledLdPat<sextloadi16, QC_LRH>;
+  def : QCScaledLdPat<extloadi16, QC_LRH>;
+  def : QCScaledLdPat<load, QC_LRW>;
+  def : QCScaledLdPat<zextloadi8, QC_LRBU>;
+  def : QCScaledLdPat<zextloadi16, QC_LRHU>;
+
+  def : QCScaledStPat<truncstorei8, QC_SRB>;
+  def : QCScaledStPat<truncstorei16, QC_SRH>;
+  def : QCScaledStPat<store, QC_SRW>;
+} // Predicates = [HasVendorXqcisls, IsRV32], AddedComplexity = 1
+
 let Predicates = [HasVendorXqciint, IsRV32] in
 def : Pat<(riscv_mileaveret_glue), (QC_C_MILEAVERET)>;
diff --git a/llvm/test/CodeGen/RISCV/xqcisls.ll b/llvm/test/CodeGen/RISCV/xqcisls.ll
new file mode 100644
index 0000000000000..b9263d487b60f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xqcisls.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32I
+; RUN: llc -mtriple=riscv32 --mattr=+zba -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32IZBA
+; RUN: llc -mtriple=riscv32 -mattr=+zba,+experimental-xqcisls -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32IZBAXQCISLS
+
+define i32 @lb_ri(i8* %a, i32 %b) {
+; RV32I-LABEL: lb_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 3
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lb_ri:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    sh3add a0, a1, a0
+; RV32IZBA-NEXT:    lb a0, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lb_ri:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrb a0, a0, a1, 3
+; RV32IZBAXQCISLS-NEXT:    ret
+  %shl = shl i32 %b, 3
+  %1 = getelementptr i8, i8* %a, i32 %shl
+  %2 = load i8, i8* %1
+  %3 = sext i8 %2 to i32
+  ret i32 %3
+}
+
+define i32 @lbu_ri(i8* %a, i32 %b) {
+; RV32I-LABEL: lbu_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 2
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lbu a0, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lbu_ri:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    sh2add a0, a1, a0
+; RV32IZBA-NEXT:    lbu a0, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lbu_ri:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrbu a0, a0, a1, 2
+; RV32IZBAXQCISLS-NEXT:    ret
+  %shl = shl i32 %b, 2
+  %1 = getelementptr i8, i8* %a, i32 %shl
+  %2 = load i8, i8* %1
+  %3 = zext i8 %2 to i32
+  ret i32 %3
+}
+
+define i32 @lh_ri(i16* %a, i32 %b) {
+; RV32I-LABEL: lh_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 5
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lh a0, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lh_ri:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    slli a1, a1, 5
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lh a0, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lh_ri:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrh a0, a0, a1, 5
+; RV32IZBAXQCISLS-NEXT:    ret
+  %shl = shl i32 %b, 4
+  %1 = getelementptr i16, i16* %a, i32 %shl
+  %2 = load i16, i16* %1
+  %3 = sext i16 %2 to i32
+  ret i32 %3
+}
+
+define i32 @lhu_ri(i16* %a, i32 %b) {
+; RV32I-LABEL: lhu_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 6
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lhu a0, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lhu_ri:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    slli a1, a1, 6
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lhu a0, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lhu_ri:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrhu a0, a0, a1, 6
+; RV32IZBAXQCISLS-NEXT:    ret
+  %shl = shl i32 %b, 5
+  %1 = getelementptr i16, i16* %a, i32 %shl
+  %2 = load i16, i16* %1
+  %3 = zext i16 %2 to i32
+  ret i32 %3
+}
+
+define i32 @lw_ri(i32* %a, i32 %b) {
+; RV32I-LABEL: lw_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 6
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lw_ri:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    slli a1, a1, 6
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lw a0, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lw_ri:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrw a0, a0, a1, 6
+; RV32IZBAXQCISLS-NEXT:    ret
+  %shl = shl i32 %b, 4
+  %1 = getelementptr i32, i32* %a, i32 %shl
+  %2 = load i32, i32* %1
+  ret i32 %2
+}
+
+define void @sb_ri(i8* %a, i8 %b, i32 %c) {
+; RV32I-LABEL: sb_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a2, 7
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    sb a1, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: sb_ri:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    slli a2, a2, 7
+; RV32IZBA-NEXT:    add a0, a0, a2
+; RV32IZBA-NEXT:    sb a1, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: sb_ri:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.srb a1, a0, a2, 7
+; RV32IZBAXQCISLS-NEXT:    ret
+  %shl = shl i32 %c, 7
+  %1 = getelementptr i8, i8* %a, i32 %shl
+  store i8 %b, i8* %1
+  ret void
+}
+
+define void @sh_ri(i16* %a, i16 %b, i32 %c) {
+; RV32I-LABEL: sh_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a2, 3
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    sh a1, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: sh_ri:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    sh3add a0, a2, a0
+; RV32IZBA-NEXT:    sh a1, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: sh_ri:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.srh a1, a0, a2, 3
+; RV32IZBAXQCISLS-NEXT:    ret
+  %shl = shl i32 %c, 2
+  %1 = getelementptr i16, i16* %a, i32 %shl
+  store i16 %b, i16* %1
+  ret void
+}
+
+define void @sw_ri(i32* %a, i32 %b, i32 %c) {
+; RV32I-LABEL: sw_ri:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a2, 3
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: sw_ri:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    sh3add a0, a2, a0
+; RV32IZBA-NEXT:    sw a1, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: sw_ri:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.srw a1, a0, a2, 3
+; RV32IZBAXQCISLS-NEXT:    ret
+  %shl = shl i32 %c, 1
+  %1 = getelementptr i32, i32* %a, i32 %shl
+  store i32 %b, i32* %1
+  ret void
+}

From 559df834df4c5653227cb85129904004164e4c4f Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Wed, 16 Apr 2025 10:48:42 +0200
Subject: [PATCH 095/710] [clang][bytecode] Fix subtracting zero-sized pointers
 (#135929)

Add the appropriate diagnostic and fix the d-d case.
---
 clang/lib/AST/ByteCode/Interp.h        | 30 ++++++++++++--------------
 clang/lib/AST/ByteCode/Pointer.h       |  9 ++++++--
 clang/test/AST/ByteCode/arrays.cpp     |  6 +++++-
 clang/test/AST/ByteCode/new-delete.cpp | 25 +++++++++++++++++++++
 4 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index b4e15b3ffbe68..88a011efe708e 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -2141,12 +2141,25 @@ static inline bool DecPtr(InterpState &S, CodePtr OpPC) {
 
 /// 1) Pops a Pointer from the stack.
 /// 2) Pops another Pointer from the stack.
-/// 3) Pushes the different of the indices of the two pointers on the stack.
+/// 3) Pushes the difference of the indices of the two pointers on the stack.
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 inline bool SubPtr(InterpState &S, CodePtr OpPC) {
   const Pointer &LHS = S.Stk.pop<Pointer>();
   const Pointer &RHS = S.Stk.pop<Pointer>();
 
+  if (!Pointer::hasSameBase(LHS, RHS) && S.getLangOpts().CPlusPlus) {
+    S.FFDiag(S.Current->getSource(OpPC),
+             diag::note_constexpr_pointer_arith_unspecified)
+        << LHS.toDiagnosticString(S.getASTContext())
+        << RHS.toDiagnosticString(S.getASTContext());
+    return false;
+  }
+
+  if (LHS == RHS) {
+    S.Stk.push<T>();
+    return true;
+  }
+
   for (const Pointer &P : {LHS, RHS}) {
     if (P.isZeroSizeArray()) {
       QualType PtrT = P.getType();
@@ -2163,21 +2176,6 @@ inline bool SubPtr(InterpState &S, CodePtr OpPC) {
     }
   }
 
-  if (RHS.isZero()) {
-    S.Stk.push<T>(T::from(LHS.getIndex()));
-    return true;
-  }
-
-  if (!Pointer::hasSameBase(LHS, RHS) && S.getLangOpts().CPlusPlus) {
-    // TODO: Diagnose.
-    return false;
-  }
-
-  if (LHS.isZero() && RHS.isZero()) {
-    S.Stk.push<T>();
-    return true;
-  }
-
   T A = LHS.isBlockPointer()
             ? (LHS.isElementPastEnd() ? T::from(LHS.getNumElems())
                                       : T::from(LHS.getIndex()))
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index 5eef9d2e1885e..8ede706f2736f 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -129,12 +129,17 @@ class Pointer {
       return false;
     if (isIntegralPointer())
       return P.asIntPointer().Value == asIntPointer().Value &&
-             Offset == P.Offset;
+             P.asIntPointer().Desc == asIntPointer().Desc && P.Offset == Offset;
+
+    if (isFunctionPointer())
+      return P.asFunctionPointer().getFunction() ==
+                 asFunctionPointer().getFunction() &&
+             P.Offset == Offset;
 
     assert(isBlockPointer());
     return P.asBlockPointer().Pointee == asBlockPointer().Pointee &&
            P.asBlockPointer().Base == asBlockPointer().Base &&
-           Offset == P.Offset;
+           P.Offset == Offset;
   }
 
   bool operator!=(const Pointer &P) const { return !(P == *this); }
diff --git a/clang/test/AST/ByteCode/arrays.cpp b/clang/test/AST/ByteCode/arrays.cpp
index 8af82163fd815..f60cc19b09bd2 100644
--- a/clang/test/AST/ByteCode/arrays.cpp
+++ b/clang/test/AST/ByteCode/arrays.cpp
@@ -106,7 +106,8 @@ constexpr int k1 = &arr[1] - &arr[0];
 static_assert(k1 == 1, "");
 static_assert((&arr[0] - &arr[1]) == -1, "");
 
-constexpr int k2 = &arr2[1] - &arr[0]; // both-error {{must be initialized by a constant expression}}
+constexpr int k2 = &arr2[1] - &arr[0]; // both-error {{must be initialized by a constant expression}} \
+                                       // expected-note {{arithmetic involving unrelated objects}}
 
 static_assert((arr + 0) == arr, "");
 static_assert(&arr[0] == arr, "");
@@ -735,6 +736,9 @@ namespace ZeroSizeTypes {
     return &arr[3] - &arr[0]; // both-note {{subtraction of pointers to type 'int[0]' of zero size}} \
                               // both-warning {{subtraction of pointers to type 'int[0]' of zero size has undefined behavior}}
   }
+
+  constexpr int z[0]{};
+  static_assert((z - z) == 0);
 }
 
 namespace InvalidIndex {
diff --git a/clang/test/AST/ByteCode/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
index bd7351cbc3d4c..5ddd7070f6710 100644
--- a/clang/test/AST/ByteCode/new-delete.cpp
+++ b/clang/test/AST/ByteCode/new-delete.cpp
@@ -967,6 +967,31 @@ namespace PR45350 {
   static_assert(f(6) == 543210);
 }
 
+namespace ZeroSizeSub {
+  consteval unsigned ptr_diff1() {
+    int *b = new int[0];
+    unsigned d = 0;
+    d = b - b;
+    delete[] b;
+
+    return d;
+  }
+  static_assert(ptr_diff1() == 0);
+
+
+  consteval unsigned ptr_diff2() { // both-error {{never produces a constant expression}}
+    int *a = new int[0];
+    int *b = new int[0];
+
+    unsigned d = a - b; // both-note 2{{arithmetic involving unrelated objects}}
+    delete[] b;
+    delete[] a;
+    return d;
+  }
+  static_assert(ptr_diff2() == 0); // both-error {{not an integral constant expression}} \
+                                   // both-note {{in call to}}
+}
+
 #else
 /// Make sure we reject this prior to C++20
 constexpr int a() { // both-error {{never produces a constant expression}}

From 70d34e4bfd32d8518a70226d3d68398d94c1d68f Mon Sep 17 00:00:00 2001
From: Hendrik_Klug <43926224+Jimmy2027@users.noreply.github.com>
Date: Wed, 16 Apr 2025 11:00:48 +0200
Subject: [PATCH 096/710] [MLIR][Linalg] Remove debug print from
 FoldIntoElementwise pass (#135928)

---
 mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp b/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
index bdd4f6025b051..fcf049e9ce722 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
@@ -72,7 +72,6 @@ struct LinalgFoldIntoElementwisePass
       LinalgFoldIntoElementwisePass>::LinalgFoldIntoElementwisePassBase;
 
   void runOnOperation() override {
-    llvm::outs() << "Hellow from fold into elemenwise \n";
     Operation *op = getOperation();
     RewritePatternSet patterns(op->getContext());
     populateLinalgFoldIntoElementwisePatterns(patterns);
@@ -86,4 +85,4 @@ struct LinalgFoldIntoElementwisePass
 void mlir::linalg::populateLinalgFoldIntoElementwisePatterns(
     RewritePatternSet &patterns) {
   patterns.add<FoldTransposePattern>(patterns.getContext());
-}
\ No newline at end of file
+}

From 51b8c66b0867154730d07e1ee4016b5116440293 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9=20=D0=98=D0=B7?=
 =?UTF-8?q?=D0=B2=D0=BE=D0=BB=D0=BE=D0=B2?= <dmitriy@izvolov.ru>
Date: Wed, 16 Apr 2025 12:34:11 +0300
Subject: [PATCH 097/710] [libc++] Extend the scope of radix sorting inside
 std::stable_sort to floating-point types (#129452)

These changes speed up `std::stable_sort` in the case of sorting
floating-point types.
This applies only to IEEE 754 floats.
The speedup is similar to that achieved for integers in PR #104683 (see
benchmarks below).

Why does this worth doing?
Previously, `std::stable_sort` had almost no chance of beating
`std::sort`.
Now there are cases when `std::stable_sort` is preferrable, and the
difference is significant.
```
---------------------------------------------------------------------------
Benchmark             |  std::stable_sort  |   std::sort | std::stable_sort
                      | without radix_sort |             | with radix_sort
---------------------------------------------------------------------------
float_Random_1        |       1.62 ns      |     2.15 ns |          1.61 ns
float_Random_4        |       18.0 ns      |     2.71 ns |          16.3 ns
float_Random_16       |        118 ns      |      113 ns |           112 ns
float_Random_64       |        751 ns      |      647 ns |           730 ns
float_Random_256      |       4715 ns      |     2937 ns |          4669 ns
float_Random_1024     |      25713 ns      |    13172 ns |          5959 ns <--
float_Random_4096     |     131307 ns      |    56870 ns |         19294 ns <--
float_Random_16384    |     624996 ns      |   242953 ns |         64264 ns <--
float_Random_65536    |    2895661 ns      |  1027279 ns |        288553 ns <--
float_Random_262144   |   13285372 ns      |  4342593 ns |       3022377 ns <--
float_Random_1048576  |   60595871 ns      | 19087591 ns |      18690457 ns <--
float_Random_2097152  |  131336117 ns      | 38800396 ns |      52325016 ns
float_Random_4194304  |  270043042 ns      | 79978019 ns |     102907726 ns
double_Random_1       |       1.60 ns      |     2.15 ns |          1.61 ns
double_Random_4       |       15.2 ns      |     2.70 ns |          16.9 ns
double_Random_16      |        104 ns      |      112 ns |           119 ns
double_Random_64      |        712 ns      |      614 ns |           755 ns
double_Random_256     |       4496 ns      |     2966 ns |          4820 ns
double_Random_1024    |      24722 ns      |    12679 ns |          6189 ns <--
double_Random_4096    |     126075 ns      |    54484 ns |         20999 ns <--
double_Random_16384   |     613782 ns      |   232557 ns |        110276 ns <--
double_Random_65536   |    2894972 ns      |   988531 ns |        774302 ns <--
double_Random_262144  |   13460273 ns      |  4278059 ns |       5115123 ns
double_Random_1048576 |   61119996 ns      | 18408462 ns |      27166574 ns
double_Random_2097152 |  132511525 ns      | 37986158 ns |      54423869 ns
double_Random_4194304 |  272949862 ns      | 77912616 ns |     147670834 ns
```

Comparison for only `std::stable_sort`:
```
Benchmark                                                         Time      Time Old      Time New
--------------------------------------------------------------------------------------------------
BM_StableSort_float_Random_1024                                -0.7997         25438          5096
BM_StableSort_float_Random_4096                                -0.8731        128157         16260
BM_StableSort_float_Random_16384                               -0.9024        621271         60623
BM_StableSort_float_Random_65536                               -0.9081       2922413        268619
BM_StableSort_float_Random_262144                              -0.7766      13386345       2990408
BM_StableSort_float_Random_1048576                             -0.6954      60673010      18481751
BM_StableSort_float_Random_2097152                             -0.6026     130977358      52052182
BM_StableSort_float_Random_4194304                             -0.6252     271556583     101770500
BM_StableSort_float_Ascending_1024                             -0.6430          6711          2396
BM_StableSort_float_Ascending_4096                             -0.7979         38460          7773
BM_StableSort_float_Ascending_16384                            -0.8471        191069         29222
BM_StableSort_float_Ascending_65536                            -0.8683        882321        116194
BM_StableSort_float_Ascending_262144                           -0.8346       3868552        639937
BM_StableSort_float_Ascending_1048576                          -0.7460      16521233       4195953
BM_StableSort_float_Ascending_2097152                          -0.5439      21757532       9922776
BM_StableSort_float_Ascending_4194304                          -0.7525      67847496      16791582
BM_StableSort_float_Descending_1024                            -0.6359         15038          5475
BM_StableSort_float_Descending_4096                            -0.7090         62810         18278
BM_StableSort_float_Descending_16384                           -0.7763        311844         69750
BM_StableSort_float_Descending_65536                           -0.7228       1270513        352202
BM_StableSort_float_Descending_262144                          -0.6785       5484173       1763045
BM_StableSort_float_Descending_1048576                         -0.5084      20223149       9941852
BM_StableSort_float_Descending_2097152                         -0.7646      60523254      14247014
BM_StableSort_float_Descending_4194304                         -0.5638      95706839      41748858
BM_StableSort_float_SingleElement_1024                         +0.3715          1732          2375
BM_StableSort_float_SingleElement_4096                         -0.1685          9357          7781
BM_StableSort_float_SingleElement_16384                        -0.3793         47307         29362
BM_StableSort_float_SingleElement_65536                        -0.4925        227666        115536
BM_StableSort_float_SingleElement_262144                       -0.4271       1075853        616387
BM_StableSort_float_SingleElement_1048576                      -0.3736       5097599       3193279
BM_StableSort_float_SingleElement_2097152                      -0.2470       9854161       7420158
BM_StableSort_float_SingleElement_4194304                      -0.3384      22175964      14670720
BM_StableSort_float_PipeOrgan_1024                             -0.4885         10664          5455
BM_StableSort_float_PipeOrgan_4096                             -0.6340         50095         18337
BM_StableSort_float_PipeOrgan_16384                            -0.7078        238700         69739
BM_StableSort_float_PipeOrgan_65536                            -0.6740       1102419        359378
BM_StableSort_float_PipeOrgan_262144                           -0.7460       4698739       1193511
BM_StableSort_float_PipeOrgan_1048576                          -0.5657      18493972       8032392
BM_StableSort_float_PipeOrgan_2097152                          -0.7116      41089206      11850349
BM_StableSort_float_PipeOrgan_4194304                          -0.6650      83445011      27955737
BM_StableSort_float_QuickSortAdversary_1024                    -0.6863         17402          5460
BM_StableSort_float_QuickSortAdversary_4096                    -0.7715         79864         18247
BM_StableSort_float_QuickSortAdversary_16384                   -0.7800        317480         69839
BM_StableSort_float_QuickSortAdversary_65536                   -0.7400       1357601        352967
BM_StableSort_float_QuickSortAdversary_262144                  -0.6450       5662094       2009769
BM_StableSort_float_QuickSortAdversary_1048576                 -0.5092      21173627      10392107
BM_StableSort_float_QuickSortAdversary_2097152                 -0.7333      61748178      16469993
BM_StableSort_float_QuickSortAdversary_4194304                 -0.5607      98459863      43250182
BM_StableSort_double_Random_1024                               -0.7657         24769          5802
BM_StableSort_double_Random_4096                               -0.8441        126449         19717
BM_StableSort_double_Random_16384                              -0.8269        614910        106447
BM_StableSort_double_Random_65536                              -0.7413       2905000        751427
BM_StableSort_double_Random_262144                             -0.6287      13449514       4994348
BM_StableSort_double_Random_1048576                            -0.5635      60863246      26568349
BM_StableSort_double_Random_2097152                            -0.5959     130293892      52654532
BM_StableSort_double_Random_4194304                            -0.4772     272616445     142526267
BM_StableSort_double_Ascending_1024                            -0.4870          6757          3466
BM_StableSort_double_Ascending_4096                            -0.7360         37592          9923
BM_StableSort_double_Ascending_16384                           -0.7971        183967         37324
BM_StableSort_double_Ascending_65536                           -0.7465        897116        227398
BM_StableSort_double_Ascending_262144                          -0.6764       4020980       1301033
BM_StableSort_double_Ascending_1048576                         -0.6407      16421799       5900751
BM_StableSort_double_Ascending_2097152                         -0.6380      29347139      10622419
BM_StableSort_double_Ascending_4194304                         -0.5934      70439925      28644185
BM_StableSort_double_Descending_1024                           -0.5988         15216          6105
BM_StableSort_double_Descending_4096                           -0.6857         65069         20449
BM_StableSort_double_Descending_16384                          -0.6922        329321        101381
BM_StableSort_double_Descending_65536                          -0.7038       1367970        405242
BM_StableSort_double_Descending_262144                         -0.6472       5361644       1891429
BM_StableSort_double_Descending_1048576                        -0.6656      22031404       7366459
BM_StableSort_double_Descending_2097152                        -0.7593      68922467      16591242
BM_StableSort_double_Descending_4194304                        -0.6392      96283643      34743223
BM_StableSort_double_SingleElement_1024                        +0.9128          1895          3625
BM_StableSort_double_SingleElement_4096                        +0.1475         10013         11490
BM_StableSort_double_SingleElement_16384                       -0.1901         52382         42424
BM_StableSort_double_SingleElement_65536                       -0.2096        254698        201313
BM_StableSort_double_SingleElement_262144                      -0.1833       1248478       1019648
BM_StableSort_double_SingleElement_1048576                     -0.1741       5703397       4710603
BM_StableSort_double_SingleElement_2097152                     -0.1751      10922197       9009835
BM_StableSort_double_SingleElement_4194304                     -0.1538      26571923      22485137
BM_StableSort_double_PipeOrgan_1024                            -0.4406         10752          6014
BM_StableSort_double_PipeOrgan_4096                            -0.5917         49456         20195
BM_StableSort_double_PipeOrgan_16384                           -0.6258        270515        101221
BM_StableSort_double_PipeOrgan_65536                           -0.7098       1159462        336457
BM_StableSort_double_PipeOrgan_262144                          -0.6591       4735711       1614433
BM_StableSort_double_PipeOrgan_1048576                         -0.6620      19353110       6541172
BM_StableSort_double_PipeOrgan_2097152                         -0.7288      49131812      13323391
BM_StableSort_double_PipeOrgan_4194304                         -0.5988      81958974      32878171
BM_StableSort_double_QuickSortAdversary_1024                   -0.6516         17948          6254
BM_StableSort_double_QuickSortAdversary_4096                   -0.7527         82359         20363
BM_StableSort_double_QuickSortAdversary_16384                  -0.7009        340410        101811
BM_StableSort_double_QuickSortAdversary_65536                  -0.6854       1487480        467928
BM_StableSort_double_QuickSortAdversary_262144                 -0.6386       5648460       2041377
BM_StableSort_double_QuickSortAdversary_1048576                -0.6127      22859142       8852587
BM_StableSort_double_QuickSortAdversary_2097152                -0.7161      68693975      19499381
BM_StableSort_double_QuickSortAdversary_4194304                -0.5909      95532179      39077491
OVERALL_GEOMEAN                                                -0.6472             0             0
```
---
 libcxx/docs/ReleaseNotes/21.rst               |   3 +
 libcxx/include/__algorithm/radix_sort.h       | 100 +++++++++++++++++-
 libcxx/include/__algorithm/stable_sort.h      |  15 ++-
 .../alg.sort/stable.sort/stable_sort.pass.cpp |  59 ++++++++++-
 4 files changed, 166 insertions(+), 11 deletions(-)

diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst
index ac54922eb4c59..2091a713ea200 100644
--- a/libcxx/docs/ReleaseNotes/21.rst
+++ b/libcxx/docs/ReleaseNotes/21.rst
@@ -64,6 +64,9 @@ Improvements and New Features
 
 - The ``num_put::do_put`` integral overloads have been optimized, resulting in a performance improvement of up to 2.4x.
 
+- The ``std::stable_sort`` algorithm uses radix sort for floating-point types now, which can improve the performance
+  up to 10x, depending on type of sorted elements and the initial state of the sorted array.
+
 Deprecations and Removals
 -------------------------
 
diff --git a/libcxx/include/__algorithm/radix_sort.h b/libcxx/include/__algorithm/radix_sort.h
index f6d9fb1ad7ca9..055d8a0765d7c 100644
--- a/libcxx/include/__algorithm/radix_sort.h
+++ b/libcxx/include/__algorithm/radix_sort.h
@@ -29,9 +29,10 @@
 
 #include <__algorithm/for_each.h>
 #include <__algorithm/move.h>
+#include <__bit/bit_cast.h>
 #include <__bit/bit_log2.h>
-#include <__bit/countl.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__functional/identity.h>
 #include <__iterator/access.h>
 #include <__iterator/distance.h>
@@ -44,9 +45,12 @@
 #include <__type_traits/enable_if.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_assignable.h>
+#include <__type_traits/is_enum.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/is_unsigned.h>
 #include <__type_traits/make_unsigned.h>
+#include <__type_traits/void_t.h>
+#include <__utility/declval.h>
 #include <__utility/forward.h>
 #include <__utility/integer_sequence.h>
 #include <__utility/move.h>
@@ -298,6 +302,96 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto __shift_to_unsigned(_Ip __n) {
   return static_cast<make_unsigned_t<_Ip> >(__n ^ __min_value);
 }
 
+template <size_t _Size>
+struct __unsigned_integer_of_size;
+
+template <>
+struct __unsigned_integer_of_size<1> {
+  using type _LIBCPP_NODEBUG = uint8_t;
+};
+
+template <>
+struct __unsigned_integer_of_size<2> {
+  using type _LIBCPP_NODEBUG = uint16_t;
+};
+
+template <>
+struct __unsigned_integer_of_size<4> {
+  using type _LIBCPP_NODEBUG = uint32_t;
+};
+
+template <>
+struct __unsigned_integer_of_size<8> {
+  using type _LIBCPP_NODEBUG = uint64_t;
+};
+
+#  if _LIBCPP_HAS_INT128
+template <>
+struct __unsigned_integer_of_size<16> {
+  using type _LIBCPP_NODEBUG = unsigned __int128;
+};
+#  endif
+
+template <size_t _Size>
+using __unsigned_integer_of_size_t _LIBCPP_NODEBUG = typename __unsigned_integer_of_size<_Size>::type;
+
+template <class _Sc>
+using __unsigned_representation_for_t _LIBCPP_NODEBUG = __unsigned_integer_of_size_t<sizeof(_Sc)>;
+
+// The function `__to_ordered_integral` is defined for integers and IEEE 754 floating-point numbers.
+// Returns an integer representation such that for any `x` and `y` such that `x < y`, the expression
+// `__to_ordered_integral(x) < __to_ordered_integral(y)` is true, where `x`, `y` are integers or IEEE 754 floats.
+template <class _Integral, enable_if_t< is_integral<_Integral>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __to_ordered_integral(_Integral __n) {
+  return __n;
+}
+
+// An overload for IEEE 754 floating-point numbers
+
+// For the floats conforming to IEEE 754 (IEC 559) standard, we know that:
+// 1. The bit representation of positive floats directly reflects their order:
+//    When comparing floats by magnitude, the number with the larger exponent is greater, and if the exponents are
+//    equal, the one with the larger mantissa is greater.
+// 2. The bit representation of negative floats reflects their reverse order (for the same reasons).
+// 3. The most significant bit (sign bit) is zero for positive floats and one for negative floats. Therefore, in the raw
+//    bit representation, any negative number will be greater than any positive number.
+
+// The only exception from this rule is `NaN`, which is unordered by definition.
+
+// Based on the above, to obtain correctly ordered integral representation of floating-point numbers, we need to:
+// 1. Invert the bit representation (including the sign bit) of negative floats to switch from reverse order to direct
+//    order;
+// 2. Invert the sign bit for positive floats.
+
+// Thus, in final integral representation, we have reversed the order for negative floats and made all negative floats
+// smaller than all positive numbers (by inverting the sign bit).
+template <class _Floating, enable_if_t< numeric_limits<_Floating>::is_iec559, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __to_ordered_integral(_Floating __f) {
+  using __integral_type          = __unsigned_representation_for_t<_Floating>;
+  constexpr auto __bit_count     = std::numeric_limits<__integral_type>::digits;
+  constexpr auto __sign_bit_mask = static_cast<__integral_type>(__integral_type{1} << (__bit_count - 1));
+
+  const auto __u = std::__bit_cast<__integral_type>(__f);
+
+  return static_cast<__integral_type>(__u & __sign_bit_mask ? ~__u : __u ^ __sign_bit_mask);
+}
+
+// There may exist user-defined comparison for enum, so we cannot compare enums just like integers.
+template <class _Enum, enable_if_t< is_enum<_Enum>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI constexpr auto __to_ordered_integral(_Enum __e) = delete;
+
+// `long double` varies significantly across platforms and compilers, making it practically
+// impossible to determine its actual bit width for conversion to an ordered integer.
+inline _LIBCPP_HIDE_FROM_ABI constexpr auto __to_ordered_integral(long double) = delete;
+
+template <class _Tp, class = void>
+inline const bool __is_ordered_integer_representable_v = false;
+
+template <class _Tp>
+inline const bool
+    __is_ordered_integer_representable_v<_Tp, __void_t<decltype(std::__to_ordered_integral(std::declval<_Tp>()))>> =
+        true;
+
 struct __low_byte_fn {
   template <class _Ip>
   _LIBCPP_HIDE_FROM_ABI constexpr uint8_t operator()(_Ip __integer) const {
@@ -314,7 +408,9 @@ __radix_sort(_RandomAccessIterator1 __first,
              _RandomAccessIterator2 __buffer,
              _Map __map,
              _Radix __radix) {
-  auto __map_to_unsigned = [__map = std::move(__map)](const auto& __x) { return std::__shift_to_unsigned(__map(__x)); };
+  auto __map_to_unsigned = [__map = std::move(__map)](const auto& __x) {
+    return std::__shift_to_unsigned(__map(std::__to_ordered_integral(__x)));
+  };
   std::__radix_sort_impl(__first, __last, __buffer, __map_to_unsigned, __radix);
 }
 
diff --git a/libcxx/include/__algorithm/stable_sort.h b/libcxx/include/__algorithm/stable_sort.h
index c7f9780e3f627..1ca66f6a51687 100644
--- a/libcxx/include/__algorithm/stable_sort.h
+++ b/libcxx/include/__algorithm/stable_sort.h
@@ -26,7 +26,6 @@
 #include <__type_traits/desugars_to.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_constant_evaluated.h>
-#include <__type_traits/is_integral.h>
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_trivially_assignable.h>
 #include <__utility/move.h>
@@ -201,7 +200,7 @@ struct __stable_sort_switch {
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr unsigned __radix_sort_min_bound() {
-  static_assert(is_integral<_Tp>::value);
+  static_assert(__is_ordered_integer_representable_v<_Tp>);
   if constexpr (sizeof(_Tp) == 1) {
     return 1 << 8;
   }
@@ -211,7 +210,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr unsigned __radix_sort_min_bound() {
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr unsigned __radix_sort_max_bound() {
-  static_assert(is_integral<_Tp>::value);
+  static_assert(__is_ordered_integer_representable_v<_Tp>);
   if constexpr (sizeof(_Tp) >= 8) {
     return 1 << 15;
   }
@@ -245,11 +244,11 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void __stable_sort(
   }
 
 #if _LIBCPP_STD_VER >= 17
-  constexpr auto __default_comp = __desugars_to_v<__totally_ordered_less_tag, _Compare, value_type, value_type >;
-  constexpr auto __integral_value =
-      is_integral_v<value_type > && is_same_v< value_type&, __iter_reference<_RandomAccessIterator>>;
-  constexpr auto __allowed_radix_sort = __default_comp && __integral_value;
-  if constexpr (__allowed_radix_sort) {
+  constexpr auto __default_comp = __desugars_to_v<__less_tag, _Compare, value_type, value_type >;
+  constexpr auto __radix_sortable =
+      __is_ordered_integer_representable_v<value_type> &&
+      is_same_v< value_type&, __iter_reference<_RandomAccessIterator>>;
+  if constexpr (__default_comp && __radix_sortable) {
     if (__len <= __buff_size && __len >= static_cast<difference_type>(std::__radix_sort_min_bound<value_type>()) &&
         __len <= static_cast<difference_type>(std::__radix_sort_max_bound<value_type>())) {
       if (__libcpp_is_constant_evaluated()) {
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.pass.cpp
index 4ee1d795a23b2..e05457492db32 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.sort/stable.sort/stable_sort.pass.cpp
@@ -18,7 +18,9 @@
 #include <algorithm>
 #include <cassert>
 #include <iterator>
+#include <limits>
 #include <random>
+#include <type_traits>
 #include <vector>
 
 #include "count_new.h"
@@ -68,6 +70,13 @@ TEST_CONSTEXPR_CXX26 std::vector<T> generate_sawtooth(int N, int M) {
     if (++x == M)
       x = 0;
   }
+
+  if (std::is_signed<T>::value) {
+    for (auto& a : v) {
+      a -= (M / 2);
+    }
+  }
+
   return v;
 }
 
@@ -193,12 +202,60 @@ TEST_CONSTEXPR_CXX26 bool test() {
   return true;
 }
 
+template <class T>
+bool test_floating_special_values() {
+  static_assert(std::is_floating_point<T>::value, "");
+
+  auto v = generate_sawtooth<T>(1024, 512);
+  v.insert(v.end(), 256, static_cast<T>(0.0));
+  v.insert(v.end(), 256, static_cast<T>(-0.0));
+  v.insert(v.end(), 256, std::numeric_limits<T>::infinity());
+  v.insert(v.end(), 256, -std::numeric_limits<T>::infinity());
+
+  std::mt19937 randomness;
+  std::shuffle(v.begin(), v.end(), randomness);
+
+  std::stable_sort(v.begin(), v.end());
+  assert(std::is_sorted(v.begin(), v.end()));
+
+  return true;
+}
+
+template <class T>
+bool test_floating() {
+  return test<T>() && test_floating_special_values<T>();
+}
+
+enum struct Enum : int { a, b, c, d, e, f, g, h };
+TEST_CONSTEXPR_CXX26 bool operator<(Enum x, Enum y) { return static_cast<int>(x) > static_cast<int>(y); }
+
+TEST_CONSTEXPR_CXX26 bool test_enum() {
+  auto v = std::vector<Enum>(128, Enum::a);
+  v.resize(v.size() + 128, Enum::b);
+  v.resize(v.size() + 128, Enum::c);
+  v.resize(v.size() + 128, Enum::d);
+  v.resize(v.size() + 128, Enum::e);
+  v.resize(v.size() + 128, Enum::f);
+  v.resize(v.size() + 128, Enum::g);
+  v.resize(v.size() + 128, Enum::h);
+
+  // Order is reversed by definition
+  std::stable_sort(v.begin(), v.end());
+  assert(std::is_sorted(v.begin(), v.end()));
+
+  return true;
+}
+
 int main(int, char**) {
   test<int>();
-  test<float>();
+  test_floating<float>();
+  test_floating<double>();
+  test_floating<long double>();
+  test_enum();
 #if TEST_STD_VER >= 26
   static_assert(test<int>());
   static_assert(test<float>());
+  static_assert(test<double>());
   // test constexprness of radix sort branch
   static_assert(test<char>());
 #endif

From 1f96aea037e612d87b4e2e20825973e45680921c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 16 Apr 2025 10:44:40 +0100
Subject: [PATCH 098/710] [X86] Add test coverage for #135917

---
 llvm/test/CodeGen/X86/pr135917.ll | 58 +++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 llvm/test/CodeGen/X86/pr135917.ll

diff --git a/llvm/test/CodeGen/X86/pr135917.ll b/llvm/test/CodeGen/X86/pr135917.ll
new file mode 100644
index 0000000000000..9eed955128b74
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr135917.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64    | FileCheck %s --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefix=SSE4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefix=AVX512
+
+define i32 @PR135917(i1 %a0) {
+; SSE2-LABEL: PR135917:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movd %edi, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    movd %xmm0, %ecx
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    addl %ecx, %eax
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: PR135917:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movd %edi, %xmm0
+; SSE4-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE4-NEXT:    movd %xmm0, %ecx
+; SSE4-NEXT:    pextrd $1, %xmm0, %eax
+; SSE4-NEXT:    addl %ecx, %eax
+; SSE4-NEXT:    retq
+;
+; AVX2-LABEL: PR135917:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovd %edi, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
+; AVX2-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %ecx
+; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX2-NEXT:    addl %ecx, %eax
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: PR135917:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    andb $1, %dil
+; AVX512-NEXT:    negb %dil
+; AVX512-NEXT:    kmovd %edi, %k0
+; AVX512-NEXT:    knotw %k0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %xmm0
+; AVX512-NEXT:    vpsrld $31, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %ecx
+; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX512-NEXT:    addl %ecx, %eax
+; AVX512-NEXT:    retq
+  %splat = insertelement <4 x i1> poison, i1 %a0, i64 0
+  %xor = xor <4 x i1> %splat, splat (i1 true)
+  %not = shufflevector <4 x i1> %xor, <4 x i1> poison, <4 x i32> zeroinitializer
+  %zext = zext <4 x i1> %not to <4 x i32>
+  %elt0 = extractelement <4 x i32> %zext, i64 0
+  %elt1 = extractelement <4 x i32> %zext, i64 1
+  %res = add i32 %elt0, %elt1
+  ret i32 %res
+}
+

From b9ce185d4e542dde5e8d152f30314b6637a0d87b Mon Sep 17 00:00:00 2001
From: Robert Konicar <rkonicar@mail.muni.cz>
Date: Wed, 16 Apr 2025 11:49:19 +0200
Subject: [PATCH 099/710] [MLIR][LLVM] Fix #llvm.constant_range crashing in
 storage uniquer (#135772)

Add APIntParameter with custom implementation for comparison and use it
in llvm.constant_range attribute. This is necessary because the default
equality operator of APInt asserts when the bit widths of the compared
APInts differ. The comparison is used by StorageUniquer when hashes of
two ranges with different bit widths collide.
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td |  4 ++--
 mlir/include/mlir/IR/AttrTypeBase.td             |  8 ++++++++
 mlir/test/Dialect/LLVMIR/range-attr.mlir         | 10 ++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)
 create mode 100644 mlir/test/Dialect/LLVMIR/range-attr.mlir

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 690243525ede4..bb528fec8c684 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -1095,8 +1095,8 @@ def LLVM_TBAATagArrayAttr
 //===----------------------------------------------------------------------===//
 def LLVM_ConstantRangeAttr : LLVM_Attr<"ConstantRange", "constant_range"> {
   let parameters = (ins
-    "::llvm::APInt":$lower,
-    "::llvm::APInt":$upper
+    APIntParameter<"">:$lower,
+    APIntParameter<"">:$upper
   );
   let summary = "A range of two integers, corresponding to LLVM's ConstantRange";
   let description = [{
diff --git a/mlir/include/mlir/IR/AttrTypeBase.td b/mlir/include/mlir/IR/AttrTypeBase.td
index 38d38cf098df3..f6ec4989c9787 100644
--- a/mlir/include/mlir/IR/AttrTypeBase.td
+++ b/mlir/include/mlir/IR/AttrTypeBase.td
@@ -383,6 +383,14 @@ class StringRefParameter<string desc = "", string value = ""> :
   let defaultValue = value;
 }
 
+// For APInts, which require comparison supporting different bitwidths. The
+// default APInt comparison operator asserts when the bitwidths differ, so
+// a custom implementation is necessary.
+class APIntParameter<string desc> :
+    AttrOrTypeParameter<"::llvm::APInt", desc> {
+  let comparator = "$_lhs.getBitWidth() == $_rhs.getBitWidth() && $_lhs == $_rhs";
+}
+
 // For APFloats, which require comparison.
 class APFloatParameter<string desc> :
     AttrOrTypeParameter<"::llvm::APFloat", desc> {
diff --git a/mlir/test/Dialect/LLVMIR/range-attr.mlir b/mlir/test/Dialect/LLVMIR/range-attr.mlir
new file mode 100644
index 0000000000000..5f2b67609743b
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/range-attr.mlir
@@ -0,0 +1,10 @@
+// RUN: mlir-opt %s -o - | FileCheck %s
+
+// CHECK: #llvm.constant_range<i32, 0, 12>
+llvm.func external @foo1(!llvm.ptr, i64) -> (i32 {llvm.range = #llvm.constant_range<i32, 0, 12>})
+// CHECK: #llvm.constant_range<i8, 1, 10>
+llvm.func external @foo2(!llvm.ptr, i64) -> (i8 {llvm.range = #llvm.constant_range<i8, 1, 10>})
+// CHECK: #llvm.constant_range<i64, 0, 2147483648>
+llvm.func external @foo3(!llvm.ptr, i64) -> (i64 {llvm.range = #llvm.constant_range<i64, 0, 2147483648>})
+// CHECK: #llvm.constant_range<i32, 1, -2147483648>
+llvm.func external @foo4(!llvm.ptr, i64) -> (i32 {llvm.range = #llvm.constant_range<i32, 1, -2147483648>})

From 3d7e56fd28cd2195e7f330f933d491530e274401 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray@arm.com>
Date: Wed, 16 Apr 2025 10:59:07 +0100
Subject: [PATCH 100/710] [AArch64][clang][llvm] Add structured sparsity outer
 product (TMOP) intrinsics (#135145)

Implement all {BF/F/S/U/SU/US}TMOP intrinsics in clang and llvm
following the ACLE in https://github.com/ARM-software/acle/pull/380/files
---
 clang/include/clang/Basic/arm_sme.td          |  27 +++
 .../AArch64/sme2-intrinsics/acle_sme2_tmop.c  | 202 ++++++++++++++++++
 .../acle_sme2_tmop.cpp                        | 120 +++++++++++
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |  18 ++
 .../lib/Target/AArch64/AArch64RegisterInfo.td |   6 +-
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |  53 +++--
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  68 +++---
 .../AArch64/GlobalISel/regbank-inlineasm.mir  |   8 +-
 .../emit_fneg_with_non_register_operand.mir   |   8 +-
 .../CodeGen/AArch64/peephole-insvigpr.mir     |   4 +-
 .../CodeGen/AArch64/sme2-intrinsics-tmop.ll   | 162 ++++++++++++++
 11 files changed, 605 insertions(+), 71 deletions(-)
 create mode 100644 clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
 create mode 100644 clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
 create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-tmop.ll

diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 1bfcf4c31d552..3647fc7a27d83 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -907,6 +907,33 @@ let SMETargetGuard = "sme-f16f16" in {
 }
 
 
+////////////////////////////////////////////////////////////////////////////////
+// SME2 - TMOP, SUTMOP, USTMOP
+
+let SMETargetGuard = "sme2,sme-tmop" in {
+  def SVTMOPA_ZA32   : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "hbf",  MergeNone, "aarch64_sme_ftmopa_za32",  [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+  def SVSTMOPA_ZA32  : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "cs",   MergeNone, "aarch64_sme_stmopa_za32",  [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+  def SVUTMOPA_ZA32  : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "UcUs", MergeNone, "aarch64_sme_utmopa_za32",  [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+  def SVSUTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{3}]", "vi2u[i", "c",    MergeNone, "aarch64_sme_sutmopa_za32", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+  def SVUSTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{3}]", "vi2x[i", "Uc",   MergeNone, "aarch64_sme_ustmopa_za32", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop,sme-f16f16" in {
+  def SVTMOPA_F16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "h", MergeNone, "aarch64_sme_ftmopa_za16", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop,sme-b16b16" in {
+  def SVTMOPA_BF16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "b", MergeNone, "aarch64_sme_ftmopa_za16", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop,sme-f8f16" in {
+  def SVTMOPA_ZA16_FPM : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2.dd[i>", "m", MergeNone, "aarch64_sme_ftmopa_za16", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop,sme-f8f32" in {
+  def SVTMOPA_ZA32_FPM : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2.dd[i>", "m", MergeNone, "aarch64_sme_ftmopa_za32", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
 multiclass ZAReadz<string n_suffix, string vg_num, string t, string i_prefix, list<ImmCheck> ch> {
   let SMETargetGuard = "sme2p1" in {
     def NAME # _H : SInst<"svreadz_hor_" # n_suffix # "_{d}_vg" # vg_num, vg_num # "im", t,
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
new file mode 100644
index 0000000000000..55d0074663bc9
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
@@ -0,0 +1,202 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+#include <arm_sme.h>
+
+#ifdef SME_OVERLOADED_FORMS
+#define SME_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_s8_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.stmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z28test_svtmopa_lane_za32_s8_s810svint8x2_tu10__SVInt8_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.stmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_s8_s8(svint8x2_t zn, svint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_s8_s8,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_u8_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.utmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z28test_svtmopa_lane_za32_u8_u811svuint8x2_tu11__SVUint8_tS0_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.utmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_u8_u8(svuint8x2_t zn, svuint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_u8_u8,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_s8_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sutmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z28test_svtmopa_lane_za32_s8_u810svint8x2_tu11__SVUint8_tS0_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.sutmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_s8_u8(svint8x2_t zn, svuint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_s8_u8,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_u8_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ustmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z28test_svtmopa_lane_za32_u8_s811svuint8x2_tu10__SVInt8_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.ustmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_u8_s8(svuint8x2_t zn, svint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_u8_s8,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_s16_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.stmopa.za32.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za32_s16_s1611svint16x2_tu11__SVInt16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.stmopa.za32.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_s16_s16(svint16x2_t zn, svint16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_s16_s16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_u16_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.utmopa.za32.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za32_u16_u1612svuint16x2_tu12__SVUint16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.utmopa.za32.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_u16_u16(svuint16x2_t zn, svuint16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_u16_u16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_f16_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za32.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za32_f16_f1613svfloat16x2_tu13__SVFloat16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za32.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_f16_f16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_f32_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za32.nxv4f32(i32 1, <vscale x 4 x float> [[ZN_COERCE0:%.*]], <vscale x 4 x float> [[ZN_COERCE1:%.*]], <vscale x 4 x float> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za32_f32_f3213svfloat32x2_tu13__SVFloat32_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za32.nxv4f32(i32 1, <vscale x 4 x float> [[ZN_COERCE0:%.*]], <vscale x 4 x float> [[ZN_COERCE1:%.*]], <vscale x 4 x float> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_f32_f32(svfloat32x2_t zn, svfloat32_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_f32_f32,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_bf16_bf16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za32.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z32test_svtmopa_lane_za32_bf16_bf1614svbfloat16x2_tu14__SVBfloat16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za32.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_bf16_bf16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za16_f16_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za16.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za16_f16_f1613svfloat16x2_tu13__SVFloat16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za16.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za16_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za16,_f16_f16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za16_bf16_bf16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za16.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z32test_svtmopa_lane_za16_bf16_bf1614svbfloat16x2_tu14__SVBfloat16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za16.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za16_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za16,_bf16_bf16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za16_mf8_mf8_fpm(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za16.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z34test_svtmopa_lane_za16_mf8_mf8_fpm13svmfloat8x2_tu13__SVMfloat8_tu11__SVUint8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za16.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za16_mf8_mf8_fpm(svmfloat8x2_t zn, svmfloat8_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za16,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_mf8_mf8_fpm(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z34test_svtmopa_lane_za32_mf8_mf8_fpm13svmfloat8x2_tu13__SVMfloat8_tu11__SVUint8_tm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.ftmopa.za32.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svtmopa_lane_za32_mf8_mf8_fpm(svmfloat8x2_t zn, svmfloat8_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svtmopa_lane_za32,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
+}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
new file mode 100644
index 0000000000000..5c229d0825366
--- /dev/null
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
@@ -0,0 +1,120 @@
+// RUN: %clang_cc1 -triple aarch64 \
+// RUN:   -target-feature +sme -target-feature +sme2 -verify -emit-llvm -o - %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+void test_features(svuint8x2_t zn_u8,      svuint8_t zm_u8,
+                   svint8x2_t zn_s8,       svint8_t zm_s8,
+                   svint16x2_t zn_s16,     svint16_t zm_s16,
+                   svuint16x2_t zn_u16,    svuint16_t zm_u16,
+                   svfloat16x2_t zn_f16,   svfloat16_t zm_f16,
+                   svbfloat16x2_t zn_bf16, svbfloat16_t zm_bf16,
+                   svfloat32x2_t zn_f32,   svfloat32_t zm_f32,
+                   svmfloat8x2_t zn_f8,    svmfloat8_t zm_f8,
+                   svuint8_t zk,           fpm_t fpm) __arm_streaming __arm_inout("za") {
+
+// expected-error@+1 {{'svtmopa_lane_za32_s8_s8' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za32_u8_u8' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_u8_u8(0, zn_u8, zm_u8, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za32_s8_u8' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_s8_u8(0, zn_s8, zm_u8, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za32_u8_s8' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_u8_s8(0, zn_u8, zm_s8, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za32_s16_s16' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_s16_s16(0, zn_s16, zm_s16, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za32_u16_u16' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za32_f16_f16' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za32_f32_f32' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_f32_f32(0, zn_f32, zm_f32, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za32_bf16_bf16' needs target feature sme,sme2,sme-tmop}}
+    svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za16_f16_f16' needs target feature sme,sme2,sme-tmop,sme-f16f16}}
+    svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za16_bf16_bf16' needs target feature sme,sme2,sme-tmop,sme-b16b16}}
+    svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, 0);
+// expected-error@+1 {{'svtmopa_lane_za16_mf8_mf8_fpm' needs target feature sme,sme2,sme-tmop,sme-f8f16}}
+    svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 0, fpm);
+// expected-error@+1 {{'svtmopa_lane_za32_mf8_mf8_fpm' needs target feature sme,sme2,sme-tmop,sme-f8f32}}
+    svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 0, fpm);
+}
+
+void test_imm(svuint8x2_t zn_u8,      svuint8_t zm_u8,
+              svint8x2_t zn_s8,       svint8_t zm_s8,
+              svint16x2_t zn_s16,     svint16_t zm_s16,
+              svuint16x2_t zn_u16,    svuint16_t zm_u16,
+              svfloat16x2_t zn_f16,   svfloat16_t zm_f16,
+              svbfloat16x2_t zn_bf16, svbfloat16_t zm_bf16,
+              svfloat32x2_t zn_f32,   svfloat32_t zm_f32,
+              svmfloat8x2_t zn_f8,    svmfloat8_t zm_f8,
+              svuint8_t zk,           fpm_t fpm) __arm_streaming __arm_inout("za") {
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_s8_s8(4, zn_s8, zm_s8, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_u8_u8(0, zn_u8, zm_u8, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_u8_u8(4, zn_u8, zm_u8, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_s8_u8(0, zn_s8, zm_u8, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_s8_u8(4, zn_s8, zm_u8, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_u8_s8(0, zn_u8, zm_s8, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_u8_s8(4, zn_u8, zm_s8, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_s16_s16(0, zn_s16, zm_s16, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_s16_s16(4, zn_s16, zm_s16, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_u16_u16(4, zn_u16, zm_u16, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_f16_f16(4, zn_f16, zm_f16, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_f32_f32(0, zn_f32, zm_f32, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_f32_f32(4, zn_f32, zm_f32, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, 4);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_bf16_bf16(4, zn_bf16, zm_bf16, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, 4);
+// expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+    svtmopa_lane_za16_f16_f16(2, zn_f16, zm_f16, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, 4);
+// expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+    svtmopa_lane_za16_bf16_bf16(2, zn_bf16, zm_bf16, zk, 0);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 4, fpm);
+// expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+    svtmopa_lane_za16_mf8_mf8_fpm(2, zn_f8, zm_f8, zk, 0, fpm);
+
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 4, fpm);
+// expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+    svtmopa_lane_za32_mf8_mf8_fpm(4, zn_f8, zm_f8, zk, 0, fpm);
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 2c6129cedebbf..c493198e8ef8f 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3107,6 +3107,24 @@ let TargetPrefix = "aarch64" in {
     }
   }
 
+  class SME_OuterProduct_TMOP_Intrinsic
+     : DefaultAttrsIntrinsic<[],
+         [llvm_i32_ty,
+          llvm_anyvector_ty,
+          LLVMMatchType<0>,
+          LLVMMatchType<0>,
+          llvm_nxv16i8_ty,
+          llvm_i32_ty],
+         [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<5>>,
+          IntrInaccessibleMemOnly]>;
+
+  def int_aarch64_sme_ftmopa_za16 : SME_OuterProduct_TMOP_Intrinsic;
+  def int_aarch64_sme_ftmopa_za32 : SME_OuterProduct_TMOP_Intrinsic;
+  def int_aarch64_sme_stmopa_za32 : SME_OuterProduct_TMOP_Intrinsic;
+  def int_aarch64_sme_utmopa_za32 : SME_OuterProduct_TMOP_Intrinsic;
+  def int_aarch64_sme_sutmopa_za32 : SME_OuterProduct_TMOP_Intrinsic;
+  def int_aarch64_sme_ustmopa_za32 : SME_OuterProduct_TMOP_Intrinsic;
+
   class SME_AddVectorToTile_Intrinsic
       : DefaultAttrsIntrinsic<[],
           [llvm_i32_ty,
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index fed9b7b173e9c..40553aff04919 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -1269,8 +1269,10 @@ def ZPRMul2AsmOp32_Hi : ZPRAsmOperand<"VectorS_Hi", 32, "Mul2_Hi">;
 def ZPRMul2AsmOp64_Lo : ZPRAsmOperand<"VectorD_Lo", 64, "Mul2_Lo">;
 def ZPRMul2AsmOp64_Hi : ZPRAsmOperand<"VectorD_Hi", 64, "Mul2_Hi">;
 
-def ZPR_K : RegisterClass<"AArch64", [untyped], 128,
-                          (add Z20, Z21, Z22, Z23, Z28, Z29, Z30, Z31)>;
+def ZPR_K : RegisterClass<"AArch64", [nxv16i8], 128,
+                          (add Z20, Z21, Z22, Z23, Z28, Z29, Z30, Z31)>{
+                             let Size = 128;
+}
 
 def ZK : RegisterOperand<ZPR_K, "printSVERegOp<>">{
   let EncoderMethod = "EncodeZK";
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index f992f73171e0e..a83b17f7c9000 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -175,12 +175,31 @@ let Predicates = [HasSME_MOP4, HasSMEI16I64] in {
 }
 
 let Predicates = [HasSME_TMOP] in {
-def STMOPA_M2ZZZI_BtoS  : sme_int_sparse_outer_product_i32<0b00100, ZZ_b_mul_r, ZPR8,  "stmopa">;
-def STMOPA_M2ZZZI_HtoS  : sme_int_sparse_outer_product_i32<0b00101, ZZ_h_mul_r, ZPR16, "stmopa">;
-def UTMOPA_M2ZZZI_BtoS  : sme_int_sparse_outer_product_i32<0b11100, ZZ_b_mul_r, ZPR8,  "utmopa">;
-def UTMOPA_M2ZZZI_HtoS  : sme_int_sparse_outer_product_i32<0b10101, ZZ_h_mul_r, ZPR16, "utmopa">;
-def SUTMOPA_M2ZZZI_BtoS : sme_int_sparse_outer_product_i32<0b01100, ZZ_b_mul_r, ZPR8, "sutmopa">;
-def USTMOPA_M2ZZZI_BtoS : sme_int_sparse_outer_product_i32<0b10100, ZZ_b_mul_r, ZPR8, "ustmopa">;
+  defm STMOPA_M2ZZZI_BtoS  : sme_tmopa_32b<0b00100, ZZ_b_mul_r, ZPR8,  nxv16i8, "stmopa", int_aarch64_sme_stmopa_za32>;
+  defm STMOPA_M2ZZZI_HtoS  : sme_tmopa_32b<0b00101, ZZ_h_mul_r, ZPR16, nxv8i16, "stmopa", int_aarch64_sme_stmopa_za32>;
+  defm UTMOPA_M2ZZZI_BtoS  : sme_tmopa_32b<0b11100, ZZ_b_mul_r, ZPR8,  nxv16i8, "utmopa", int_aarch64_sme_utmopa_za32>;
+  defm UTMOPA_M2ZZZI_HtoS  : sme_tmopa_32b<0b10101, ZZ_h_mul_r, ZPR16, nxv8i16, "utmopa", int_aarch64_sme_utmopa_za32>;
+  defm SUTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b01100, ZZ_b_mul_r, ZPR8,  nxv16i8, "sutmopa", int_aarch64_sme_sutmopa_za32>;
+  defm USTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b10100, ZZ_b_mul_r, ZPR8,  nxv16i8, "ustmopa", int_aarch64_sme_ustmopa_za32>;
+  defm FTMOPA_M2ZZZI_HtoS  : sme_tmopa_32b<0b11000, ZZ_h_mul_r, ZPR16, nxv8f16,  "ftmopa", int_aarch64_sme_ftmopa_za32, [FPCR]>;
+  defm FTMOPA_M2ZZZI_StoS  : sme_tmopa_32b<0b00000, ZZ_s_mul_r, ZPR32, nxv4f32,  "ftmopa", int_aarch64_sme_ftmopa_za32, [FPCR]>;
+  defm BFTMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b10000, ZZ_h_mul_r, ZPR16, nxv8bf16, "bftmopa", int_aarch64_sme_ftmopa_za32, [FPCR]>;
+}
+
+let Predicates = [HasSME_TMOP, HasSMEF16F16] in {
+  defm FTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b10001, ZZ_h_mul_r, ZPR16, nxv8f16, "ftmopa", int_aarch64_sme_ftmopa_za16, [FPCR]>;
+}
+
+let Predicates = [HasSME_TMOP, HasSMEB16B16] in {
+  defm BFTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b11001, ZZ_h_mul_r, ZPR16, nxv8bf16, "bftmopa", int_aarch64_sme_ftmopa_za16, [FPCR]>;
+}
+
+let Predicates = [HasSME_TMOP, HasSMEF8F16] in {
+  defm FTMOPA_M2ZZZI_BtoH : sme_tmopa_16b<0b01001, ZZ_b_mul_r, ZPR8, nxv16i8, "ftmopa", int_aarch64_sme_ftmopa_za16, [FPMR, FPCR]>;
+}
+
+let Predicates = [HasSME_TMOP, HasSMEF8F32] in {
+  defm FTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b01000, ZZ_b_mul_r, ZPR8, nxv16i8, "ftmopa", int_aarch64_sme_ftmopa_za32, [FPMR, FPCR]>;
 }
 
 let Predicates = [HasSME] in {
@@ -1064,12 +1083,6 @@ let Predicates = [HasSME_MOP4] in {
   defm FMOP4S : sme2_fmop4as_fp32_non_widening<1, "fmop4s", "int_aarch64_sme_mop4s">;
 }
 
-let Predicates = [HasSME_TMOP] in {
-  def FTMOPA_M2ZZZI_HtoS  : sme_tmopa_32b<0b11000, ZZ_h_mul_r, ZPR16, "ftmopa">;
-  def FTMOPA_M2ZZZI_StoS  : sme_tmopa_32b<0b00000, ZZ_s_mul_r, ZPR32, "ftmopa">;
-  def BFTMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b10000, ZZ_h_mul_r, ZPR16, "bftmopa">;
-}
-
 let Predicates = [HasSME2p2] in {
   defm FMUL_2ZZ  : sme2_multi2_fmul_sm<"fmul">;
   defm FMUL_2Z2Z : sme2_multi2_fmul_mm< "fmul">;
@@ -1078,26 +1091,10 @@ let Predicates = [HasSME2p2] in {
 
 } // [HasSME2p2]
 
-let Predicates = [HasSME_TMOP, HasSMEB16B16] in {
-  def BFTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b11001, ZZ_h_mul_r, ZPR16, "bftmopa">;
-}
-
-let Predicates = [HasSME_TMOP, HasSMEF8F32], Uses = [FPMR, FPCR] in {
-  def FTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b01000, ZZ_b_mul_r, ZPR8, "ftmopa">;
-}
-
-let Predicates = [HasSME_TMOP, HasSMEF8F16], Uses = [FPMR, FPCR] in {
-  def FTMOPA_M2ZZZI_BtoH : sme_tmopa_16b<0b01001, ZZ_b_mul_r, ZPR8, "ftmopa">;
-}
-
 let Predicates = [HasSME_MOP4, HasSMEF8F16], Uses = [FPMR, FPCR] in {
   defm FMOP4A : sme2_fmop4a_fp8_fp16_2way<"fmop4a">;
 }
 
-let Predicates = [HasSME_TMOP, HasSMEF16F16] in {
-  def FTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b10001, ZZ_h_mul_r, ZPR16, "ftmopa">;
-}
-
 let Predicates = [HasSME_MOP4, HasSMEF16F16] in {
   defm FMOP4A : sme2_fmop4as_fp16_non_widening<0, "fmop4a", "int_aarch64_sme_mop4a">;
   defm FMOP4S : sme2_fmop4as_fp16_non_widening<1, "fmop4s", "int_aarch64_sme_mop4s">;
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index b611dddb0b045..b174a2a733851 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -106,6 +106,16 @@ class sme_outer_product_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag>
   let mayStore = 1;
 }
 
+class sme_sparse_outer_product_pseudo<RegisterOperand zn_ty, RegisterOperand zm_ty, SMEMatrixTypeEnum za_flag>
+    : Pseudo<(outs), (ins i32imm:$tile, zn_ty:$zn, zm_ty:$zm, ZK:$zk, i32imm:$idx), []>,
+      Sched<[]> {
+  // Translated to the actual instructions in AArch64ISelLowering.cpp
+  let SMEMatrixType = za_flag;
+  let usesCustomInserter = 1;
+  let mayLoad = 1;
+  let mayStore = 1;
+}
+
 class sme2_quarter_tile_outer_product_pseudo<RegisterOperand zn_ty, RegisterOperand zm_ty, SMEMatrixTypeEnum za_flag>
     : Pseudo<(outs), (ins i32imm:$tile,
                           zn_ty:$zn, zm_ty:$zm), []>,
@@ -296,6 +306,12 @@ class SME2_ZA_Tile_Vec_Multi_Single_Pat<string name, SDPatternOperator intrinsic
 class SME2_ZA_Tile_Vec_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType vt>
     : Pat<(intrinsic imm_ty:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2),
           (!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;
+
+class SME2_ZA_TMOP_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt>
+    : Pat<(intrinsic tile_imm:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, nxv16i8:$Zk, timm32_0_3:$idx),
+          (!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), $Zm, $Zk, $idx)>;
+
+
 //===----------------------------------------------------------------------===//
 // SME pattern match helpers.
 //===----------------------------------------------------------------------===//
@@ -489,35 +505,6 @@ multiclass sme_int_outer_product_i64<bits<3> opc, string mnemonic,
   def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv8i1, nxv8i16>;
 }
 
-class sme_int_sparse_outer_product_i32<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
-    : I<(outs TileOp32:$ZAda),
-        (ins  TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm, ZK:$Zk, VectorIndexS32b:$imm),
-        mnemonic, "\t$ZAda, $Zn, $Zm, $Zk$imm",
-        "", []>,
-      Sched<[]> {
-  bits<2> ZAda;
-  bits<4> Zn;
-  bits<5> Zm;
-  bits<3> Zk;
-  bits<2> imm;
-  let Inst{31-25} = 0b1000000;
-  let Inst{24}    = opc{4};
-  let Inst{23-22} = 0b01;
-  let Inst{21}    = opc{3};
-  let Inst{20-16} = Zm;
-  let Inst{15}    = opc{2};
-  let Inst{14}    = 0b0;
-  let Inst{13}    = opc{1};
-  let Inst{12-10} = Zk;
-  let Inst{9-6}   = Zn;
-  let Inst{5-4}   = imm;
-  let Inst{3}     = opc{0};
-  let Inst{2}     = 0b0;
-  let Inst{1-0}   = ZAda;
-
-  let Constraints = "$ZAda = $_ZAda";
-}
-
 class sme_outer_product_widening_inst<bits<3> opc, ZPRRegOp zpr_ty, string mnemonic>
     : I<(outs TileOp32:$ZAda),
         (ins  TileOp32:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),
@@ -3562,7 +3549,7 @@ multiclass  sme2_int_bmopx_tile<string mnemonic, bits<3> op, SDPatternOperator i
 //===----------------------------------------------------------------------===//
 // SME2 Sparse Outer Product and Accumulate
 
-class sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
+class sme_int_sparse_outer_product_i16<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
     : I<(outs TileOp16:$ZAda),
         (ins  TileOp16:$_ZAda, zn_ty:$Zn, zm_ty:$Zm, ZK:$Zk, VectorIndexS32b:$imm),
         mnemonic, "\t$ZAda, $Zn, $Zm, $Zk$imm",
@@ -3591,7 +3578,7 @@ class sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, s
   let Constraints = "$ZAda = $_ZAda";
 }
 
-class sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
+class sme_int_sparse_outer_product_i32<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
     : I<(outs TileOp32:$ZAda),
         (ins  TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm, ZK:$Zk, VectorIndexS32b:$imm),
         mnemonic, "\t$ZAda, $Zn, $Zm, $Zk$imm",
@@ -3620,6 +3607,25 @@ class sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, s
   let Constraints = "$ZAda = $_ZAda";
 }
 
+multiclass sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, SDPatternOperator intrinsic, list<Register> uses=[]> {
+  def NAME : sme_int_sparse_outer_product_i16<opc, zn_ty, zm_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+    let Uses = uses;
+  }
+
+  def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;
+
+  def : SME2_ZA_TMOP_Pat<NAME, intrinsic, timm32_0_1, vt>;
+}
+
+multiclass sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, SDPatternOperator intrinsic, list<Register> uses=[]> {
+  def NAME : sme_int_sparse_outer_product_i32<opc, zn_ty, zm_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+    let Uses = uses;
+  }
+
+  def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
+
+  def : SME2_ZA_TMOP_Pat<NAME, intrinsic, timm32_0_3, vt>;
+}
 
 //===----------------------------------------------------------------------===///
 // SME2 Zero Lookup Table.
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir b/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
index 216f94b2b51e3..37d6e852429fe 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
@@ -57,11 +57,11 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: inlineasm_virt_reg_output
-    ; CHECK: INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0
+    ; CHECK: INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2818058 /* regdef:GPR32common */, def %0
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr(s32) = COPY %0
     ; CHECK-NEXT: $w0 = COPY [[COPY]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
-    INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0:gpr32common
+    INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2818058 /* regdef:GPR32common */, def %0:gpr32common
     %1:_(s32) = COPY %0
     $w0 = COPY %1(s32)
     RET_ReallyLR implicit $w0
@@ -75,12 +75,12 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: inlineasm_virt_mixed_types
-    ; CHECK: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0, 3735562 /* regdef:FPR64 */, def %1
+    ; CHECK: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2818058 /* regdef:GPR32common */, def %0, 3670026 /* regdef:FPR64 */, def %1
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr(s32) = COPY %0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr(s64) = COPY %1
     ; CHECK-NEXT: $d0 = COPY [[COPY1]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0:gpr32common, 3735562 /* regdef:FPR64 */, def %1:fpr64
+    INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2818058 /* regdef:GPR32common */, def %0:gpr32common, 3670026 /* regdef:FPR64 */, def %1:fpr64
     %3:_(s32) = COPY %0
     %4:_(s64) = COPY %1
     $d0 = COPY %4(s64)
diff --git a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
index 253e6a9c076c6..615accc7f75e5 100644
--- a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
+++ b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
@@ -91,10 +91,10 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[LOADgot:%[0-9]+]]:gpr64common = LOADgot target-flags(aarch64-got) @c
   ; CHECK-NEXT:   [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[LOADgot]], 0 :: (dereferenceable load (s64) from @c)
-  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, [[LDRDui]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 3670026 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, [[LDRDui]](tied-def 3)
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:fpr64 = COPY %2
   ; CHECK-NEXT:   [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[LOADgot]], 0 :: (dereferenceable load (s64) from @c)
-  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[LDRDui1]](tied-def 3)
+  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 3670026 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[LDRDui1]](tied-def 3)
   ; CHECK-NEXT:   [[FNEGDr:%[0-9]+]]:fpr64 = FNEGDr %2
   ; CHECK-NEXT:   nofpexcept FCMPDrr %4, killed [[FNEGDr]], implicit-def $nzcv, implicit $fpcr
   ; CHECK-NEXT:   Bcc 1, %bb.2, implicit $nzcv
@@ -111,10 +111,10 @@ body:             |
 
     %6:gpr64common = LOADgot target-flags(aarch64-got) @c
     %3:fpr64 = LDRDui %6, 0 :: (dereferenceable load (s64) from @c)
-    INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, %3(tied-def 3)
+    INLINEASM &"", 1 /* sideeffect attdialect */, 3670026 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, %3(tied-def 3)
     %0:fpr64 = COPY %2
     %5:fpr64 = LDRDui %6, 0 :: (dereferenceable load (s64) from @c)
-    INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, %5(tied-def 3)
+    INLINEASM &"", 1 /* sideeffect attdialect */, 3670026 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, %5(tied-def 3)
     %7:fpr64 = FNEGDr %2
     nofpexcept FCMPDrr %4, killed %7, implicit-def $nzcv, implicit $fpcr
     Bcc 1, %bb.2, implicit $nzcv
diff --git a/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir b/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
index 3174d3c8c1a73..aef01e42ed7cc 100644
--- a/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
+++ b/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
@@ -487,7 +487,7 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
     ; CHECK-NEXT: [[DEF:%[0-9]+]]:gpr64all = IMPLICIT_DEF
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY [[DEF]]
-    ; CHECK-NEXT: INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3735562 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3670026 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed [[COPY1]]
     ; CHECK-NEXT: [[MOVIv2d_ns:%[0-9]+]]:fpr128 = MOVIv2d_ns 0
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:fpr64 = COPY [[MOVIv2d_ns]].dsub
     ; CHECK-NEXT: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
@@ -505,7 +505,7 @@ body:             |
     %0:gpr64common = COPY $x0
     %2:gpr64all = IMPLICIT_DEF
     %3:gpr64sp = COPY %2
-    INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3735562 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed %3
+    INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3670026 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed %3
     %4:fpr128 = MOVIv2d_ns 0
     %5:fpr64 = COPY %4.dsub
     %7:fpr128 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-tmop.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-tmop.ll
new file mode 100644
index 0000000000000..efd5f951eced3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-tmop.ll
@@ -0,0 +1,162 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -force-streaming -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+define void @stmopa_za32_s8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: stmopa_za32_s8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    stmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.stmopa.za32.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @utmopa_za32_u8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: utmopa_za32_u8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    utmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.utmopa.za32.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @ustmopa_za32_u8_s8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: ustmopa_za32_u8_s8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    ustmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.ustmopa.za32.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @sutmopa_za32_s8_u8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: sutmopa_za32_s8_u8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    sutmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.sutmopa.za32.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @stmopa_za32_s16(<vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: stmopa_za32_s16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    stmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.stmopa.za32.nxv8i16(i32 0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @utmopa_za32_u16(<vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: utmopa_za32_u16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    utmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.utmopa.za32.nxv8i16(i32 0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @ftmopa_za32_f16(<vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: ftmopa_za32_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    ftmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.ftmopa.za32.nxv8f16(i32 0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @bftmopa_za32_bf16(<vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: bftmopa_za32_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bftmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.ftmopa.za32.nxv8bf16(i32 0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @ftmopa_za32_f32(<vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: ftmopa_za32_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    ftmopa za0.s, { z0.s, z1.s }, z2.s, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.ftmopa.za32.nxv4f32(i32 0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @ftmopa_za16_f16(<vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: ftmopa_za16_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    ftmopa za0.h, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.ftmopa.za16.nxv8f16(i32 0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @bftmopa_za16_bf16(<vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: bftmopa_za16_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bftmopa za0.h, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.ftmopa.za16.nxv8bf16(i32 0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @ftmopa_za16_f8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: ftmopa_za16_f8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    ftmopa za0.h, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.ftmopa.za16.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+define void @ftmopa_za32_f8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: ftmopa_za32_f8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z28.d, z3.d
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    ftmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.ftmopa.za32.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+  ret void
+}
+
+attributes #0 = {nounwind "target-features" = "+sme2,+sme-tmop,+sme-f16f16,+sme-b16b16,+sme-f8f16,+sme-f8f32,+sme2p1,+bf16" }

From d508f0cb009a0be98ca97f4dc0498294e0681a66 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Wed, 16 Apr 2025 11:41:07 +0100
Subject: [PATCH 101/710] [AArch64] Fix FPMR handling when switching streaming
 mode (#135827)

According to the
[documentation](https://developer.arm.com/documentation/ddi0601/latest/AArch64-Registers/FPMR--Floating-point-Mode-Register),
the FPMR register is set to 0 when entering or exiting streaming mode.
This patch models that behavior by adding FPMR as an implicit def to the
instructions used for entering and exiting streaming mode.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  3 +++
 llvm/test/CodeGen/AArch64/sme-write-fpmr.ll   | 23 +++++++++++++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/sme-write-fpmr.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index bea8087750d6e..a95d8d343adf2 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -8882,12 +8882,15 @@ void AArch64TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
         MI.removeOperand(I);
 
     // The SVE vector length can change when entering/leaving streaming mode.
+    // FPMR is set to 0 when entering/leaving streaming mode.
     if (MI.getOperand(0).getImm() == AArch64SVCR::SVCRSM ||
         MI.getOperand(0).getImm() == AArch64SVCR::SVCRSMZA) {
       MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/false,
                                               /*IsImplicit=*/true));
       MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/true,
                                               /*IsImplicit=*/true));
+      MI.addOperand(MachineOperand::CreateReg(AArch64::FPMR, /*IsDef=*/true,
+                                              /*IsImplicit=*/true));
     }
   }
 
diff --git a/llvm/test/CodeGen/AArch64/sme-write-fpmr.ll b/llvm/test/CodeGen/AArch64/sme-write-fpmr.ll
new file mode 100644
index 0000000000000..074d65831a584
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-write-fpmr.ll
@@ -0,0 +1,23 @@
+; RUN: llc -mattr=+sme -stop-after=finalize-isel < %s | FileCheck %s
+
+target triple = "aarch64"
+
+; Check that we don't define FPMR for 'smstart za' and 'smstop za'
+define void @smstart_za() "aarch64_new_za" nounwind {
+  ; CHECK-LABEL:    name: smstart_za
+  ; CHECK-NOT:        implicit-def {{[^,]*}}$fpmr
+  ret void
+}
+
+; Check that we do define FPMR for 'smstart sm' and 'smstop sm'
+define void @smstart_sm() nounwind {
+  ; CHECK-LABEL: name: smstart_sm
+  ; CHECK:          MSRpstatesvcrImm1 1, 1,
+  ; CHECK-SAME:       implicit-def {{[^,]*}}$fpmr
+  ; CHECK:          MSRpstatesvcrImm1 1, 0,
+  ; CHECK-SAME:       implicit-def {{[^,]*}}$fpmr
+  call void @require_sm()
+  ret void
+}
+
+declare void @require_sm() "aarch64_pstate_sm_enabled"

From 41c97afea055a5b7264167ec47b8c14c0f471f2f Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 16 Apr 2025 06:53:10 -0400
Subject: [PATCH 102/710] [SLP][NFC]Remove handling of duplicates from
 getGatherCost

Duplicates are handled in BoUpSLP::processBuildVector (see TryPackScalars), support for duplicates in getGatherCost is not needed anymore.

Reviewers: hiraditya, RKSimon

Reviewed By: hiraditya, RKSimon

Pull Request: https://github.com/llvm/llvm-project/pull/135834
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 34 ++-----------------
 1 file changed, 2 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b48674f6993e3..f9acc276f37f9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15727,13 +15727,10 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
                                        Type *ScalarTy) const {
   const unsigned VF = VL.size();
   auto *VecTy = getWidenedType(ScalarTy, VF);
-  bool DuplicateNonConst = false;
   // Find the cost of inserting/extracting values from the vector.
   // Check if the same elements are inserted several times and count them as
   // shuffle candidates.
-  APInt ShuffledElements = APInt::getZero(VF);
   APInt DemandedElements = APInt::getZero(VF);
-  DenseMap<Value *, unsigned> UniqueElements;
   constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   InstructionCost Cost;
   auto EstimateInsertCost = [&](unsigned I, Value *V) {
@@ -15742,32 +15739,18 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
       Cost += TTI->getCastInstrCost(Instruction::Trunc, ScalarTy, V->getType(),
                                     TTI::CastContextHint::None, CostKind);
   };
-  SmallVector<int> ShuffleMask(VF, PoisonMaskElem);
   SmallVector<int> ConstantShuffleMask(VF, PoisonMaskElem);
   std::iota(ConstantShuffleMask.begin(), ConstantShuffleMask.end(), 0);
   for (auto [I, V] : enumerate(VL)) {
     // No need to shuffle duplicates for constants.
-    if ((ForPoisonSrc && isConstant(V)) || isa<UndefValue>(V)) {
-      ShuffledElements.setBit(I);
-      ShuffleMask[I] = isa<PoisonValue>(V) ? PoisonMaskElem : I;
+    if ((ForPoisonSrc && isConstant(V)) || isa<UndefValue>(V))
       continue;
-    }
 
     if (isConstant(V)) {
       ConstantShuffleMask[I] = I + VF;
-      ShuffleMask[I] = I;
-      continue;
-    }
-    auto Res = UniqueElements.try_emplace(V, I);
-    if (Res.second) {
-      EstimateInsertCost(I, V);
-      ShuffleMask[I] = I;
       continue;
     }
-
-    DuplicateNonConst = true;
-    ShuffledElements.setBit(I);
-    ShuffleMask[I] = Res.first->second;
+    EstimateInsertCost(I, V);
   }
   // FIXME: add a cost for constant vector materialization.
   bool IsAnyNonUndefConst =
@@ -15776,15 +15759,6 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
   if (!ForPoisonSrc && IsAnyNonUndefConst) {
     Cost += ::getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteTwoSrc, VecTy,
                              ConstantShuffleMask);
-    // Update the shuffle mask for shuffling with incoming source (all elements
-    // are used!) or with constant subvector.
-    for_each(enumerate(ShuffleMask), [&](auto P) {
-      if ((!ForPoisonSrc && P.value() == PoisonMaskElem) ||
-          ConstantShuffleMask[P.index()] != PoisonMaskElem)
-        P.value() = P.index();
-      else if (P.value() != PoisonMaskElem)
-        P.value() += VF;
-    });
   }
 
   // 2. Insert unique non-constants.
@@ -15793,10 +15767,6 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
                                      /*Insert=*/true,
                                      /*Extract=*/false, CostKind,
                                      ForPoisonSrc && !IsAnyNonUndefConst, VL);
-  // 3. Shuffle duplicates.
-  if (DuplicateNonConst)
-    Cost += ::getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteSingleSrc,
-                             VecTy, ShuffleMask, CostKind);
   return Cost;
 }
 

From 1e61b374ba3ba2891dc1abda732b0b9263216785 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Wed, 16 Apr 2025 13:16:04 +0200
Subject: [PATCH 103/710] [mlir][vector] Tighten the semantics of vector.gather
 (#135749)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch restricts `vector.gather` to only accept tensors and memrefs
as valid sources. Currently, the source is typed as `AnyShaped`, which
also includes vectors—allowing the following (invalid) construct to pass
verification:

```mlir
  %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru
       : vector<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
```
(Note: the source %base here is a vector, which is incorrect.)

In contrast, `vector.scatter` currently only accepts memrefs, so some
asymmetry remains between the two ops. This PR is a step toward aligning
their semantics.
---
 .../mlir/Dialect/Vector/IR/VectorOps.td       |  2 +-
 mlir/include/mlir/IR/CommonTypeConstraints.td | 15 ++++++++++++-
 mlir/test/Dialect/Vector/invalid.mlir         | 21 +++++++++++++++++++
 3 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 7fc56b1aa4e7e..d7518943229ea 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1972,7 +1972,7 @@ def Vector_GatherOp :
     DeclareOpInterfaceMethods<MaskableOpInterface>,
     DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
   ]>,
-    Arguments<(ins Arg<AnyShaped, "", [MemRead]>:$base,
+    Arguments<(ins Arg<TensorOrMemRef<[AnyType]>, "", [MemRead]>:$base,
                Variadic<Index>:$indices,
                VectorOfNonZeroRankOf<[AnyInteger, Index]>:$index_vec,
                VectorOfNonZeroRankOf<[I1]>:$mask,
diff --git a/mlir/include/mlir/IR/CommonTypeConstraints.td b/mlir/include/mlir/IR/CommonTypeConstraints.td
index e6f17ded4628b..45ec1846580f2 100644
--- a/mlir/include/mlir/IR/CommonTypeConstraints.td
+++ b/mlir/include/mlir/IR/CommonTypeConstraints.td
@@ -63,6 +63,9 @@ def IsTensorTypePred : CPred<"::llvm::isa<::mlir::TensorType>($_self)">;
 // Whether a type is a MemRefType.
 def IsMemRefTypePred : CPred<"::llvm::isa<::mlir::MemRefType>($_self)">;
 
+// Whether a type is a TensorType or a MemRefType.
+def IsTensorOrMemRefTypePred : Or<[IsTensorTypePred, IsMemRefTypePred]>;
+
 // Whether a type is an UnrankedMemRefType
 def IsUnrankedMemRefTypePred
         : CPred<"::llvm::isa<::mlir::UnrankedMemRefType>($_self)">;
@@ -426,7 +429,9 @@ class ValueSemanticsContainerOf<list<Type> allowedTypes> :
   ShapedContainerType<allowedTypes, HasValueSemanticsPred,
   "container with value semantics">;
 
+//===----------------------------------------------------------------------===//
 // Vector types.
+//===----------------------------------------------------------------------===//
 
 class VectorOfNonZeroRankOf<list<Type> allowedTypes> :
   ShapedContainerType<allowedTypes, IsVectorOfNonZeroRankTypePred, "vector",
@@ -755,7 +760,7 @@ class StaticShapeTensorOf<list<Type> allowedTypes>
 def AnyStaticShapeTensor : StaticShapeTensorOf<[AnyType]>;
 
 //===----------------------------------------------------------------------===//
-// Memref type.
+// Memref types.
 //===----------------------------------------------------------------------===//
 
 // Any unranked memref whose element type is from the given `allowedTypes` list.
@@ -878,6 +883,14 @@ class NestedTupleOf<list<Type> allowedTypes> :
                        "getFlattenedTypes(::llvm::cast<::mlir::TupleType>($_self))",
                        "nested tuple">;
 
+//===----------------------------------------------------------------------===//
+// Mixed types
+//===----------------------------------------------------------------------===//
+
+class TensorOrMemRef<list<Type> allowedTypes> :
+  ShapedContainerType<allowedTypes, IsTensorOrMemRefTypePred, "Tensor or MemRef",
+                      "::mlir::ShapedType">;
+
 //===----------------------------------------------------------------------===//
 // Common type constraints
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index dbf829e014b8d..3a8320971bac4 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -1409,6 +1409,16 @@ func.func @maskedstore_memref_mismatch(%base: memref<?xf32>, %mask: vector<16xi1
 
 // -----
 
+func.func @gather_from_vector(%base: vector<16xf32>, %indices: vector<16xi32>,
+                                %mask: vector<16xi1>, %pass_thru: vector<16xf32>) {
+  %c0 = arith.constant 0 : index
+  // expected-error@+1 {{'vector.gather' op operand #0 must be Tensor or MemRef of any type values, but got 'vector<16xf32>'}}
+  %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru
+    : vector<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
+}
+
+// -----
+
 func.func @gather_base_type_mismatch(%base: memref<?xf64>, %indices: vector<16xi32>,
                                 %mask: vector<16xi1>, %pass_thru: vector<16xf32>) {
   %c0 = arith.constant 0 : index
@@ -1469,6 +1479,17 @@ func.func @gather_pass_thru_type_mismatch(%base: memref<?xf32>, %indices: vector
 
 // -----
 
+func.func @scatter_to_vector(%base: vector<16xf32>, %indices: vector<16xi32>,
+                             %mask: vector<16xi1>, %pass_thru: vector<16xf32>) {
+  %c0 = arith.constant 0 : index
+  // expected-error@+2 {{custom op 'vector.scatter' invalid kind of type specified}}
+  vector.scatter %base[%c0][%indices], %mask, %pass_thru
+    : vector<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
+}
+
+// -----
+
+
 func.func @scatter_base_type_mismatch(%base: memref<?xf64>, %indices: vector<16xi32>,
                                  %mask: vector<16xi1>, %value: vector<16xf32>) {
   %c0 = arith.constant 0 : index

From 2d63faead4e6339e679ab62113f47112d67a5b06 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Wed, 16 Apr 2025 13:21:25 +0200
Subject: [PATCH 104/710] [clang][bytecode][NFC] Remove PT_FnPtr (#135947)

We don't need this anymore since we don't return it from classify()
anymore.
---
 clang/lib/AST/ByteCode/Compiler.cpp        | 15 ++++++-------
 clang/lib/AST/ByteCode/Disasm.cpp          |  2 --
 clang/lib/AST/ByteCode/EvalEmitter.cpp     | 12 -----------
 clang/lib/AST/ByteCode/FunctionPointer.cpp | 13 ++++-------
 clang/lib/AST/ByteCode/FunctionPointer.h   | 25 ++--------------------
 clang/lib/AST/ByteCode/Interp.h            | 21 ------------------
 clang/lib/AST/ByteCode/InterpBuiltin.cpp   |  6 +-----
 clang/lib/AST/ByteCode/InterpStack.h       |  2 --
 clang/lib/AST/ByteCode/Opcodes.td          |  7 ++----
 clang/lib/AST/ByteCode/Pointer.cpp         |  6 +++---
 clang/lib/AST/ByteCode/PrimType.h          |  9 ++------
 11 files changed, 20 insertions(+), 98 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index afd8d09a088cd..157e306e5cdb3 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -4057,7 +4057,7 @@ template <class Emitter> bool Compiler<Emitter>::visitBool(const Expr *E) {
     return true;
 
   // Convert pointers to bool.
-  if (T == PT_Ptr || T == PT_FnPtr) {
+  if (T == PT_Ptr) {
     if (!this->emitNull(*T, 0, nullptr, E))
       return false;
     return this->emitNE(*T, E);
@@ -4103,8 +4103,6 @@ bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
   case PT_Ptr:
     return this->emitNullPtr(Ctx.getASTContext().getTargetNullPointerValue(QT),
                              nullptr, E);
-  case PT_FnPtr:
-    return this->emitNullFnPtr(0, nullptr, E);
   case PT_MemberPtr:
     return this->emitNullMemberPtr(0, nullptr, E);
   case PT_Float:
@@ -4255,7 +4253,6 @@ bool Compiler<Emitter>::emitConst(T Value, PrimType Ty, const Expr *E) {
   case PT_Bool:
     return this->emitConstBool(Value, E);
   case PT_Ptr:
-  case PT_FnPtr:
   case PT_MemberPtr:
   case PT_Float:
   case PT_IntAP:
@@ -4956,7 +4953,7 @@ bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
     // If we know the callee already, check the known parametrs for nullability.
     if (FuncDecl && NonNullArgs[ArgIndex]) {
       PrimType ArgT = classify(Arg).value_or(PT_Ptr);
-      if (ArgT == PT_Ptr || ArgT == PT_FnPtr) {
+      if (ArgT == PT_Ptr) {
         if (!this->emitCheckNonNullArg(ArgT, Arg))
           return false;
       }
@@ -5997,7 +5994,7 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visit(SubExpr))
       return false;
 
-    if (T == PT_Ptr || T == PT_FnPtr) {
+    if (T == PT_Ptr) {
       if (!this->emitIncPtr(E))
         return false;
 
@@ -6021,7 +6018,7 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visit(SubExpr))
       return false;
 
-    if (T == PT_Ptr || T == PT_FnPtr) {
+    if (T == PT_Ptr) {
       if (!this->emitDecPtr(E))
         return false;
 
@@ -6045,7 +6042,7 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visit(SubExpr))
       return false;
 
-    if (T == PT_Ptr || T == PT_FnPtr) {
+    if (T == PT_Ptr) {
       if (!this->emitLoadPtr(E))
         return false;
       if (!this->emitConstUint8(1, E))
@@ -6088,7 +6085,7 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visit(SubExpr))
       return false;
 
-    if (T == PT_Ptr || T == PT_FnPtr) {
+    if (T == PT_Ptr) {
       if (!this->emitLoadPtr(E))
         return false;
       if (!this->emitConstUint8(1, E))
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index 4bdf0f0afb1b0..83b9af9de0796 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -256,8 +256,6 @@ static const char *primTypeToString(PrimType T) {
     return "Float";
   case PT_Ptr:
     return "Ptr";
-  case PT_FnPtr:
-    return "FnPtr";
   case PT_MemberPtr:
     return "MemberPtr";
   case PT_FixedPoint:
diff --git a/clang/lib/AST/ByteCode/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp
index a37be38c7a839..71d688498ffa5 100644
--- a/clang/lib/AST/ByteCode/EvalEmitter.cpp
+++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp
@@ -142,10 +142,6 @@ bool EvalEmitter::speculate(const CallExpr *E, const LabelTy &EndLabel) {
   if (T == PT_Ptr) {
     const auto &Ptr = S.Stk.pop<Pointer>();
     return this->emitBool(CheckBCPResult(S, Ptr), E);
-  } else if (T == PT_FnPtr) {
-    S.Stk.discard<FunctionPointer>();
-    // Never accepted
-    return this->emitBool(false, E);
   }
 
   // Otherwise, this is fine!
@@ -210,14 +206,6 @@ template <> bool EvalEmitter::emitRet<PT_Ptr>(const SourceInfo &Info) {
 
   return true;
 }
-template <> bool EvalEmitter::emitRet<PT_FnPtr>(const SourceInfo &Info) {
-  if (!isActive())
-    return true;
-
-  // Function pointers cannot be converted to rvalues.
-  EvalResult.setFunctionPointer(S.Stk.pop<FunctionPointer>());
-  return true;
-}
 
 bool EvalEmitter::emitRetVoid(const SourceInfo &Info) {
   EvalResult.setValid();
diff --git a/clang/lib/AST/ByteCode/FunctionPointer.cpp b/clang/lib/AST/ByteCode/FunctionPointer.cpp
index 6b0b559a63386..4ab7af170efe4 100644
--- a/clang/lib/AST/ByteCode/FunctionPointer.cpp
+++ b/clang/lib/AST/ByteCode/FunctionPointer.cpp
@@ -16,27 +16,22 @@ APValue FunctionPointer::toAPValue(const ASTContext &) const {
     return APValue(static_cast<Expr *>(nullptr), CharUnits::Zero(), {},
                    /*OnePastTheEnd=*/false, /*IsNull=*/true);
 
-  if (!Valid)
-    return APValue(static_cast<Expr *>(nullptr),
-                   CharUnits::fromQuantity(getIntegerRepresentation()), {},
-                   /*OnePastTheEnd=*/false, /*IsNull=*/false);
-
   if (Func->getDecl())
-    return APValue(Func->getDecl(), CharUnits::fromQuantity(Offset), {},
+    return APValue(Func->getDecl(), CharUnits::fromQuantity(0), {},
                    /*OnePastTheEnd=*/false, /*IsNull=*/false);
-  return APValue(Func->getExpr(), CharUnits::fromQuantity(Offset), {},
+  return APValue(Func->getExpr(), CharUnits::fromQuantity(0), {},
                  /*OnePastTheEnd=*/false, /*IsNull=*/false);
 }
 
 void FunctionPointer::print(llvm::raw_ostream &OS) const {
   OS << "FnPtr(";
-  if (Func && Valid)
+  if (Func)
     OS << Func->getName();
   else if (Func)
     OS << reinterpret_cast<uintptr_t>(Func);
   else
     OS << "nullptr";
-  OS << ") + " << Offset;
+  OS << ")";
 }
 
 } // namespace interp
diff --git a/clang/lib/AST/ByteCode/FunctionPointer.h b/clang/lib/AST/ByteCode/FunctionPointer.h
index e2b45b2344fdc..9e8ea2f1af5f8 100644
--- a/clang/lib/AST/ByteCode/FunctionPointer.h
+++ b/clang/lib/AST/ByteCode/FunctionPointer.h
@@ -20,24 +20,15 @@ namespace interp {
 class FunctionPointer final {
 private:
   const Function *Func;
-  uint64_t Offset;
-  bool Valid;
 
 public:
   FunctionPointer() = default;
-  FunctionPointer(const Function *Func, uint64_t Offset = 0)
-      : Func(Func), Offset(Offset), Valid(true) {}
-
-  FunctionPointer(uintptr_t IntVal, const Descriptor *Desc = nullptr)
-      : Func(reinterpret_cast<const Function *>(IntVal)), Offset(0),
-        Valid(false) {}
+  FunctionPointer(const Function *Func) : Func(Func) {}
 
   const Function *getFunction() const { return Func; }
-  uint64_t getOffset() const { return Offset; }
   bool isZero() const { return !Func; }
-  bool isValid() const { return Valid; }
   bool isWeak() const {
-    if (!Func || !Valid || !Func->getDecl())
+    if (!Func || !Func->getDecl())
       return false;
 
     return Func->getDecl()->isWeak();
@@ -56,20 +47,8 @@ class FunctionPointer final {
   uint64_t getIntegerRepresentation() const {
     return static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Func));
   }
-
-  ComparisonCategoryResult compare(const FunctionPointer &RHS) const {
-    if (Func == RHS.Func && Offset == RHS.Offset)
-      return ComparisonCategoryResult::Equal;
-    return ComparisonCategoryResult::Unordered;
-  }
 };
 
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
-                                     FunctionPointer FP) {
-  FP.print(OS);
-  return OS;
-}
-
 } // namespace interp
 } // namespace clang
 
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 88a011efe708e..bd58c2a88e9d9 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -20,7 +20,6 @@
 #include "FixedPoint.h"
 #include "Floating.h"
 #include "Function.h"
-#include "FunctionPointer.h"
 #include "InterpBuiltinBitCast.h"
 #include "InterpFrame.h"
 #include "InterpStack.h"
@@ -984,26 +983,6 @@ bool CmpHelperEQ(InterpState &S, CodePtr OpPC, CompareFn Fn) {
   return CmpHelper<T>(S, OpPC, Fn);
 }
 
-template <>
-inline bool CmpHelperEQ<FunctionPointer>(InterpState &S, CodePtr OpPC,
-                                         CompareFn Fn) {
-  const auto &RHS = S.Stk.pop<FunctionPointer>();
-  const auto &LHS = S.Stk.pop<FunctionPointer>();
-
-  // We cannot compare against weak declarations at compile time.
-  for (const auto &FP : {LHS, RHS}) {
-    if (FP.isWeak()) {
-      const SourceInfo &Loc = S.Current->getSource(OpPC);
-      S.FFDiag(Loc, diag::note_constexpr_pointer_weak_comparison)
-          << FP.toDiagnosticString(S.getASTContext());
-      return false;
-    }
-  }
-
-  S.Stk.push<Boolean>(Boolean::from(Fn(LHS.compare(RHS))));
-  return true;
-}
-
 template <>
 inline bool CmpHelper<Pointer>(InterpState &S, CodePtr OpPC, CompareFn Fn) {
   using BoolT = PrimConv<PT_Bool>::T;
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index bde416d98edd3..d06941bf10fe0 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -130,7 +130,6 @@ static bool retPrimValue(InterpState &S, CodePtr OpPC,
     return Ret<X>(S, OpPC);
   switch (*T) {
     RET_CASE(PT_Ptr);
-    RET_CASE(PT_FnPtr);
     RET_CASE(PT_Float);
     RET_CASE(PT_Bool);
     RET_CASE(PT_Sint8);
@@ -766,10 +765,7 @@ static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
   assert(Call->getArg(0)->isLValue());
   PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
 
-  if (PtrT == PT_FnPtr) {
-    const FunctionPointer &Arg = S.Stk.peek<FunctionPointer>();
-    S.Stk.push<FunctionPointer>(Arg);
-  } else if (PtrT == PT_Ptr) {
+  if (PtrT == PT_Ptr) {
     const Pointer &Arg = S.Stk.peek<Pointer>();
     S.Stk.push<Pointer>(Arg);
   } else {
diff --git a/clang/lib/AST/ByteCode/InterpStack.h b/clang/lib/AST/ByteCode/InterpStack.h
index f7b8c386bcc13..0b76f1d650580 100644
--- a/clang/lib/AST/ByteCode/InterpStack.h
+++ b/clang/lib/AST/ByteCode/InterpStack.h
@@ -183,8 +183,6 @@ class InterpStack final {
       return PT_Uint64;
     else if constexpr (std::is_same_v<T, Floating>)
       return PT_Float;
-    else if constexpr (std::is_same_v<T, FunctionPointer>)
-      return PT_FnPtr;
     else if constexpr (std::is_same_v<T, IntegralAP<true>>)
       return PT_IntAP;
     else if constexpr (std::is_same_v<T, IntegralAP<false>>)
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 798771bf91f05..5a9079fea0846 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -29,7 +29,6 @@ def IntAP : Type;
 def IntAPS : Type;
 def Float : Type;
 def Ptr : Type;
-def FnPtr : Type;
 def MemberPtr : Type;
 def FixedPoint : Type;
 
@@ -106,9 +105,7 @@ def AluTypeClass : TypeClass {
   let Types = !listconcat(IntegerTypeClass.Types, [Bool], [FixedPoint]);
 }
 
-def PtrTypeClass : TypeClass {
-  let Types = [Ptr, FnPtr, MemberPtr];
-}
+def PtrTypeClass : TypeClass { let Types = [Ptr, MemberPtr]; }
 
 def NonPtrTypeClass : TypeClass {
   let Types = !listconcat(IntegerTypeClass.Types, [Bool], [Float], [FixedPoint]);
@@ -119,7 +116,7 @@ def AllTypeClass : TypeClass {
 }
 
 def ComparableTypeClass : TypeClass {
-  let Types = !listconcat(AluTypeClass.Types, [Ptr], [Float], [FnPtr]);
+  let Types = !listconcat(AluTypeClass.Types, [Ptr], [Float]);
 }
 
 class SingletonTypeClass<Type Ty> : TypeClass {
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index c09d3224b1f36..c43c0a063bd9e 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -155,10 +155,10 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   if (isFunctionPointer()) {
     const FunctionPointer &FP = asFunctionPointer();
     if (const FunctionDecl *FD = FP.getFunction()->getDecl())
-      return APValue(FD, CharUnits::fromQuantity(FP.getOffset() + Offset), {},
+      return APValue(FD, CharUnits::fromQuantity(Offset), {},
                      /*OnePastTheEnd=*/false, /*IsNull=*/false);
-    return APValue(FP.getFunction()->getExpr(),
-                   CharUnits::fromQuantity(FP.getOffset() + Offset), {},
+    return APValue(FP.getFunction()->getExpr(), CharUnits::fromQuantity(Offset),
+                   {},
                    /*OnePastTheEnd=*/false, /*IsNull=*/false);
   }
 
diff --git a/clang/lib/AST/ByteCode/PrimType.h b/clang/lib/AST/ByteCode/PrimType.h
index 59c04c4673d93..a3c0b0f3ceca8 100644
--- a/clang/lib/AST/ByteCode/PrimType.h
+++ b/clang/lib/AST/ByteCode/PrimType.h
@@ -46,12 +46,11 @@ enum PrimType : unsigned {
   PT_FixedPoint = 11,
   PT_Float = 12,
   PT_Ptr = 13,
-  PT_FnPtr = 14,
-  PT_MemberPtr = 15,
+  PT_MemberPtr = 14,
 };
 
 inline constexpr bool isPtrType(PrimType T) {
-  return T == PT_Ptr || T == PT_FnPtr || T == PT_MemberPtr;
+  return T == PT_Ptr || T == PT_MemberPtr;
 }
 
 enum class CastKind : uint8_t {
@@ -114,9 +113,6 @@ template <> struct PrimConv<PT_Bool> {
 template <> struct PrimConv<PT_Ptr> {
   using T = Pointer;
 };
-template <> struct PrimConv<PT_FnPtr> {
-  using T = FunctionPointer;
-};
 template <> struct PrimConv<PT_MemberPtr> {
   using T = MemberPointer;
 };
@@ -166,7 +162,6 @@ static inline bool aligned(const void *P) {
       TYPE_SWITCH_CASE(PT_Float, B)                                            \
       TYPE_SWITCH_CASE(PT_Bool, B)                                             \
       TYPE_SWITCH_CASE(PT_Ptr, B)                                              \
-      TYPE_SWITCH_CASE(PT_FnPtr, B)                                            \
       TYPE_SWITCH_CASE(PT_MemberPtr, B)                                        \
       TYPE_SWITCH_CASE(PT_FixedPoint, B)                                       \
     }                                                                          \

From bc03d6cce25712601423398350f56114e64e4e29 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 16 Apr 2025 13:30:45 +0200
Subject: [PATCH 105/710] [VPlan] Introduce all loop regions as VPlan
 transform. (NFC) (#129402)

Further simplify VPlan CFG builder by moving introduction of inner
regions to a VPlan transform, building on
https://github.com/llvm/llvm-project/pull/128419.

The HCFG builder now only constructs plain CFGs. I will move it to
VPlanConstruction as follow-up.

Depends on https://github.com/llvm/llvm-project/pull/128419.

PR: https://github.com/llvm/llvm-project/pull/129402
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  16 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |   8 +
 .../Vectorize/VPlanConstruction.cpp           | 103 +++++++++---
 .../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 150 +++---------------
 .../Transforms/Vectorize/VPlanHCFGBuilder.h   |   8 +-
 .../Transforms/Vectorize/VPlanTransforms.h    |  19 ++-
 llvm/lib/Transforms/Vectorize/VPlanValue.h    |   6 +
 .../vplan-printing-outer-loop.ll              |  29 ++--
 .../LoopVectorize/vplan_hcfg_stress_test.ll   |   2 +-
 .../Transforms/Vectorize/VPlanTestBase.h      |   6 +-
 10 files changed, 149 insertions(+), 198 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index af94dc01c8c5c..dd7f05465a50b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9544,14 +9544,14 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
           Range);
   auto Plan = std::make_unique<VPlan>(OrigLoop);
   // Build hierarchical CFG.
-  // Convert to VPlan-transform and consoliate all transforms for VPlan
+  // TODO: Convert to VPlan-transform and consolidate all transforms for VPlan
   // creation.
   VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
-  HCFGBuilder.buildHierarchicalCFG();
+  HCFGBuilder.buildPlainCFG();
 
-  VPlanTransforms::introduceTopLevelVectorLoopRegion(
-      *Plan, Legal->getWidestInductionType(), PSE, RequiresScalarEpilogueCheck,
-      CM.foldTailByMasking(), OrigLoop);
+  VPlanTransforms::createLoopRegions(*Plan, Legal->getWidestInductionType(),
+                                     PSE, RequiresScalarEpilogueCheck,
+                                     CM.foldTailByMasking(), OrigLoop);
 
   // Don't use getDecisionAndClampRange here, because we don't know the UF
   // so this function is better to be conservative, rather than to split
@@ -9851,10 +9851,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
   auto Plan = std::make_unique<VPlan>(OrigLoop);
   // Build hierarchical CFG
   VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
-  HCFGBuilder.buildHierarchicalCFG();
+  HCFGBuilder.buildPlainCFG();
 
-  VPlanTransforms::introduceTopLevelVectorLoopRegion(
-      *Plan, Legal->getWidestInductionType(), PSE, true, false, OrigLoop);
+  VPlanTransforms::createLoopRegions(*Plan, Legal->getWidestInductionType(),
+                                     PSE, true, false, OrigLoop);
 
   for (ElementCount VF : Range)
     Plan->addVF(VF);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 94b5167c60089..7084676af6d5b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -57,6 +57,7 @@ class SCEV;
 class Type;
 class VPBasicBlock;
 class VPBuilder;
+class VPDominatorTree;
 class VPRegionBlock;
 class VPlan;
 class VPLane;
@@ -303,6 +304,13 @@ class VPBlockBase {
   /// Remove all the successors of this block.
   void clearSuccessors() { Successors.clear(); }
 
+  /// Swap predecessors of the block. The block must have exactly 2
+  /// predecessors.
+  void swapPredecessors() {
+    assert(Predecessors.size() == 2 && "must have 2 predecessors to swap");
+    std::swap(Predecessors[0], Predecessors[1]);
+  }
+
   /// Swap successors of the block. The block must have exactly 2 successors.
   // TODO: This should be part of introducing conditional branch recipes rather
   // than being independent.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index f58f0290b5fa9..1e687d0879f18 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -14,26 +14,88 @@
 #include "LoopVectorizationPlanner.h"
 #include "VPlan.h"
 #include "VPlanCFG.h"
+#include "VPlanDominatorTree.h"
 #include "VPlanTransforms.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 
 using namespace llvm;
 
-void VPlanTransforms::introduceTopLevelVectorLoopRegion(
-    VPlan &Plan, Type *InductionTy, PredicatedScalarEvolution &PSE,
-    bool RequiresScalarEpilogueCheck, bool TailFolded, Loop *TheLoop) {
-  // TODO: Generalize to introduce all loop regions.
-  auto *HeaderVPBB = cast<VPBasicBlock>(Plan.getEntry()->getSingleSuccessor());
-  VPBlockUtils::disconnectBlocks(Plan.getEntry(), HeaderVPBB);
+/// Checks if \p HeaderVPB is a loop header block in the plain CFG; that is, it
+/// has exactly 2 predecessors (preheader and latch), where the block
+/// dominates the latch and the preheader dominates the block. If it is a
+/// header block return true, making sure the preheader appears first and
+/// the latch second. Otherwise return false.
+static bool canonicalHeader(VPBlockBase *HeaderVPB,
+                            const VPDominatorTree &VPDT) {
+  ArrayRef<VPBlockBase *> Preds = HeaderVPB->getPredecessors();
+  if (Preds.size() != 2)
+    return false;
 
-  VPBasicBlock *OriginalLatch =
-      cast<VPBasicBlock>(HeaderVPBB->getSinglePredecessor());
-  VPBlockUtils::disconnectBlocks(OriginalLatch, HeaderVPBB);
-  VPBasicBlock *VecPreheader = Plan.createVPBasicBlock("vector.ph");
-  VPBlockUtils::connectBlocks(Plan.getEntry(), VecPreheader);
-  assert(OriginalLatch->getNumSuccessors() == 0 &&
-         "Plan should end at top level latch");
+  auto *PreheaderVPBB = Preds[0];
+  auto *LatchVPBB = Preds[1];
+  if (VPDT.dominates(PreheaderVPBB, HeaderVPB) &&
+      VPDT.dominates(HeaderVPB, LatchVPBB))
+    return true;
+
+  std::swap(PreheaderVPBB, LatchVPBB);
+
+  if (VPDT.dominates(PreheaderVPBB, HeaderVPB) &&
+      VPDT.dominates(HeaderVPB, LatchVPBB)) {
+    // Canonicalize predecessors of header so that preheader is first and latch
+    // second.
+    HeaderVPB->swapPredecessors();
+    for (VPRecipeBase &R : cast<VPBasicBlock>(HeaderVPB)->phis())
+      R.swapOperands();
+    return true;
+  }
+
+  return false;
+}
+
+/// Create a new VPRegionBlock for the loop starting at \p HeaderVPB.
+static void createLoopRegion(VPlan &Plan, VPBlockBase *HeaderVPB) {
+  auto *PreheaderVPBB = HeaderVPB->getPredecessors()[0];
+  auto *LatchVPBB = HeaderVPB->getPredecessors()[1];
+
+  VPBlockUtils::disconnectBlocks(PreheaderVPBB, HeaderVPB);
+  VPBlockUtils::disconnectBlocks(LatchVPBB, HeaderVPB);
+  VPBlockBase *Succ = LatchVPBB->getSingleSuccessor();
+  assert(LatchVPBB->getNumSuccessors() <= 1 &&
+         "Latch has more than one successor");
+  if (Succ)
+    VPBlockUtils::disconnectBlocks(LatchVPBB, Succ);
+
+  auto *R = Plan.createVPRegionBlock(HeaderVPB, LatchVPBB, "",
+                                     false /*isReplicator*/);
+  R->setParent(HeaderVPB->getParent());
+  // All VPBB's reachable shallowly from HeaderVPB belong to top level loop,
+  // because VPlan is expected to end at top level latch disconnected above.
+  for (VPBlockBase *VPBB : vp_depth_first_shallow(HeaderVPB))
+    VPBB->setParent(R);
+
+  VPBlockUtils::insertBlockAfter(R, PreheaderVPBB);
+  if (Succ)
+    VPBlockUtils::connectBlocks(R, Succ);
+}
+
+void VPlanTransforms::createLoopRegions(VPlan &Plan, Type *InductionTy,
+                                        PredicatedScalarEvolution &PSE,
+                                        bool RequiresScalarEpilogueCheck,
+                                        bool TailFolded, Loop *TheLoop) {
+  VPDominatorTree VPDT;
+  VPDT.recalculate(Plan);
+  for (VPBlockBase *HeaderVPB : vp_depth_first_shallow(Plan.getEntry()))
+    if (canonicalHeader(HeaderVPB, VPDT))
+      createLoopRegion(Plan, HeaderVPB);
+
+  VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
+  auto *OrigExiting = TopRegion->getExiting();
+  VPBasicBlock *LatchVPBB = Plan.createVPBasicBlock("vector.latch");
+  VPBlockUtils::insertBlockAfter(LatchVPBB, OrigExiting);
+  TopRegion->setExiting(LatchVPBB);
+  TopRegion->setName("vector loop");
+  TopRegion->getEntryBasicBlock()->setName("vector.body");
 
   // Create SCEV and VPValue for the trip count.
   // We use the symbolic max backedge-taken-count, which works also when
@@ -47,18 +109,9 @@ void VPlanTransforms::introduceTopLevelVectorLoopRegion(
   Plan.setTripCount(
       vputils::getOrCreateVPValueForSCEVExpr(Plan, TripCount, SE));
 
-  // Create VPRegionBlock, with existing header and new empty latch block, to be
-  // filled.
-  VPBasicBlock *LatchVPBB = Plan.createVPBasicBlock("vector.latch");
-  VPBlockUtils::insertBlockAfter(LatchVPBB, OriginalLatch);
-  auto *TopRegion = Plan.createVPRegionBlock(
-      HeaderVPBB, LatchVPBB, "vector loop", false /*isReplicator*/);
-  // All VPBB's reachable shallowly from HeaderVPBB belong to top level loop,
-  // because VPlan is expected to end at top level latch.
-  for (VPBlockBase *VPBB : vp_depth_first_shallow(HeaderVPBB))
-    VPBB->setParent(TopRegion);
-
-  VPBlockUtils::insertBlockAfter(TopRegion, VecPreheader);
+  VPBasicBlock *VecPreheader = Plan.createVPBasicBlock("vector.ph");
+  VPBlockUtils::insertBlockAfter(VecPreheader, Plan.getEntry());
+
   VPBasicBlock *MiddleVPBB = Plan.createVPBasicBlock("middle.block");
   VPBlockUtils::insertBlockAfter(MiddleVPBB, TopRegion);
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 4b8a2420b3037..5bacd2d4e6d88 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -12,9 +12,7 @@
 /// components and steps:
 //
 /// 1. PlainCFGBuilder class: builds a plain VPBasicBlock-based CFG that
-/// faithfully represents the CFG in the incoming IR. A VPRegionBlock (Top
-/// Region) is created to enclose and serve as parent of all the VPBasicBlocks
-/// in the plain CFG.
+/// faithfully represents the CFG in the incoming IR.
 /// NOTE: At this point, there is a direct correspondence between all the
 /// VPBasicBlocks created for the initial plain CFG and the incoming
 /// BasicBlocks. However, this might change in the future.
@@ -57,12 +55,8 @@ class PlainCFGBuilder {
   // Hold phi node's that need to be fixed once the plain CFG has been built.
   SmallVector<PHINode *, 8> PhisToFix;
 
-  /// Maps loops in the original IR to their corresponding region.
-  DenseMap<Loop *, VPRegionBlock *> Loop2Region;
-
   // Utility functions.
   void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
-  void setRegionPredsFromBB(VPRegionBlock *VPBB, BasicBlock *BB);
   void fixHeaderPhis();
   VPBasicBlock *getOrCreateVPBB(BasicBlock *BB);
 #ifndef NDEBUG
@@ -83,25 +77,6 @@ class PlainCFGBuilder {
 // Set predecessors of \p VPBB in the same order as they are in \p BB. \p VPBB
 // must have no predecessors.
 void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) {
-  auto GetLatchOfExit = [this](BasicBlock *BB) -> BasicBlock * {
-    auto *SinglePred = BB->getSinglePredecessor();
-    Loop *LoopForBB = LI->getLoopFor(BB);
-    if (!SinglePred || LI->getLoopFor(SinglePred) == LoopForBB)
-      return nullptr;
-    // The input IR must be in loop-simplify form, ensuring a single predecessor
-    // for exit blocks.
-    assert(SinglePred == LI->getLoopFor(SinglePred)->getLoopLatch() &&
-           "SinglePred must be the only loop latch");
-    return SinglePred;
-  };
-  if (auto *LatchBB = GetLatchOfExit(BB)) {
-    auto *PredRegion = getOrCreateVPBB(LatchBB)->getParent();
-    assert(VPBB == cast<VPBasicBlock>(PredRegion->getSingleSuccessor()) &&
-           "successor must already be set for PredRegion; it must have VPBB "
-           "as single successor");
-    VPBB->setPredecessors({PredRegion});
-    return;
-  }
   // Collect VPBB predecessors.
   SmallVector<VPBlockBase *, 2> VPBBPreds;
   for (BasicBlock *Pred : predecessors(BB))
@@ -113,13 +88,6 @@ static bool isHeaderBB(BasicBlock *BB, Loop *L) {
   return L && BB == L->getHeader();
 }
 
-void PlainCFGBuilder::setRegionPredsFromBB(VPRegionBlock *Region,
-                                           BasicBlock *BB) {
-  // BB is a loop header block. Connect the region to the loop preheader.
-  Loop *LoopOfBB = LI->getLoopFor(BB);
-  Region->setPredecessors({getOrCreateVPBB(LoopOfBB->getLoopPredecessor())});
-}
-
 // Add operands to VPInstructions representing phi nodes from the input IR.
 void PlainCFGBuilder::fixHeaderPhis() {
   for (auto *Phi : PhisToFix) {
@@ -130,43 +98,18 @@ void PlainCFGBuilder::fixHeaderPhis() {
     auto *VPPhi = cast<VPWidenPHIRecipe>(VPVal);
     assert(VPPhi->getNumOperands() == 0 &&
            "Expected VPInstruction with no operands.");
-
-    Loop *L = LI->getLoopFor(Phi->getParent());
-    assert(isHeaderBB(Phi->getParent(), L));
-    // For header phis, make sure the incoming value from the loop
-    // predecessor is the first operand of the recipe.
+    assert(isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent())) &&
+           "Expected Phi in header block.");
     assert(Phi->getNumOperands() == 2 &&
            "header phi must have exactly 2 operands");
-    BasicBlock *LoopPred = L->getLoopPredecessor();
-    VPPhi->addOperand(
-        getOrCreateVPOperand(Phi->getIncomingValueForBlock(LoopPred)));
-    BasicBlock *LoopLatch = L->getLoopLatch();
-    VPPhi->addOperand(
-        getOrCreateVPOperand(Phi->getIncomingValueForBlock(LoopLatch)));
-  }
-}
-
-static bool isHeaderVPBB(VPBasicBlock *VPBB) {
-  return VPBB->getParent() && VPBB->getParent()->getEntry() == VPBB;
-}
-
-/// Return true of \p L loop is contained within \p OuterLoop.
-static bool doesContainLoop(const Loop *L, const Loop *OuterLoop) {
-  if (L->getLoopDepth() < OuterLoop->getLoopDepth())
-    return false;
-  const Loop *P = L;
-  while (P) {
-    if (P == OuterLoop)
-      return true;
-    P = P->getParentLoop();
+    for (BasicBlock *Pred : predecessors(Phi->getParent()))
+      VPPhi->addOperand(
+          getOrCreateVPOperand(Phi->getIncomingValueForBlock(Pred)));
   }
-  return false;
 }
 
-// Create a new empty VPBasicBlock for an incoming BasicBlock in the region
-// corresponding to the containing loop  or retrieve an existing one if it was
-// already created. If no region exists yet for the loop containing \p BB, a new
-// one is created.
+// Create a new empty VPBasicBlock for an incoming BasicBlock or retrieve an
+// existing one if it was already created.
 VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
   if (auto *VPBB = BB2VPBB.lookup(BB)) {
     // Retrieve existing VPBB.
@@ -174,32 +117,10 @@ VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
   }
 
   // Create new VPBB.
-  StringRef Name = isHeaderBB(BB, TheLoop) ? "vector.body" : BB->getName();
+  StringRef Name = BB->getName();
   LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << Name << "\n");
   VPBasicBlock *VPBB = Plan.createVPBasicBlock(Name);
   BB2VPBB[BB] = VPBB;
-
-  // Get or create a region for the loop containing BB, except for the top
-  // region of TheLoop which is created later.
-  Loop *LoopOfBB = LI->getLoopFor(BB);
-  if (!LoopOfBB || LoopOfBB == TheLoop || !doesContainLoop(LoopOfBB, TheLoop))
-    return VPBB;
-
-  auto *RegionOfVPBB = Loop2Region.lookup(LoopOfBB);
-  if (!isHeaderBB(BB, LoopOfBB)) {
-    assert(RegionOfVPBB &&
-           "Region should have been created by visiting header earlier");
-    VPBB->setParent(RegionOfVPBB);
-    return VPBB;
-  }
-
-  assert(!RegionOfVPBB &&
-         "First visit of a header basic block expects to register its region.");
-  // Handle a header - take care of its Region.
-  RegionOfVPBB = Plan.createVPRegionBlock(Name.str(), false /*isReplicator*/);
-  RegionOfVPBB->setParent(Loop2Region[LoopOfBB->getParentLoop()]);
-  RegionOfVPBB->setEntry(VPBB);
-  Loop2Region[LoopOfBB] = RegionOfVPBB;
   return VPBB;
 }
 
@@ -351,6 +272,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
 // Main interface to build the plain CFG.
 void PlainCFGBuilder::buildPlainCFG(
     DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB) {
+  VPIRBasicBlock *Entry = cast<VPIRBasicBlock>(Plan.getEntry());
+  BB2VPBB[Entry->getIRBasicBlock()] = Entry;
 
   // 1. Scan the body of the loop in a topological order to visit each basic
   // block after having visited its predecessor basic blocks. Create a VPBB for
@@ -376,26 +299,13 @@ void PlainCFGBuilder::buildPlainCFG(
   for (BasicBlock *BB : RPO) {
     // Create or retrieve the VPBasicBlock for this BB.
     VPBasicBlock *VPBB = getOrCreateVPBB(BB);
-    VPRegionBlock *Region = VPBB->getParent();
     Loop *LoopForBB = LI->getLoopFor(BB);
     // Set VPBB predecessors in the same order as they are in the incoming BB.
-    if (!isHeaderBB(BB, LoopForBB)) {
-      setVPBBPredsFromBB(VPBB, BB);
-    } else if (Region) {
-      // BB is a loop header and there's a corresponding region, set the
-      // predecessor for it.
-      setRegionPredsFromBB(Region, BB);
-    }
+    setVPBBPredsFromBB(VPBB, BB);
 
     // Create VPInstructions for BB.
     createVPInstructionsForVPBB(VPBB, BB);
 
-    if (BB == TheLoop->getLoopLatch()) {
-      VPBasicBlock *HeaderVPBB = getOrCreateVPBB(LoopForBB->getHeader());
-      VPBlockUtils::connectBlocks(VPBB, HeaderVPBB);
-      continue;
-    }
-
     // Set VPBB successors. We create empty VPBBs for successors if they don't
     // exist already. Recipes will be created when the successor is visited
     // during the RPO traversal.
@@ -410,10 +320,7 @@ void PlainCFGBuilder::buildPlainCFG(
     auto *BI = cast<BranchInst>(BB->getTerminator());
     unsigned NumSuccs = succ_size(BB);
     if (NumSuccs == 1) {
-      auto *Successor = getOrCreateVPBB(BB->getSingleSuccessor());
-      VPBB->setOneSuccessor(isHeaderVPBB(Successor)
-                                ? Successor->getParent()
-                                : static_cast<VPBlockBase *>(Successor));
+      VPBB->setOneSuccessor(getOrCreateVPBB(BB->getSingleSuccessor()));
       continue;
     }
     assert(BI->isConditional() && NumSuccs == 2 && BI->isConditional() &&
@@ -423,21 +330,11 @@ void PlainCFGBuilder::buildPlainCFG(
     BasicBlock *IRSucc1 = BI->getSuccessor(1);
     VPBasicBlock *Successor0 = getOrCreateVPBB(IRSucc0);
     VPBasicBlock *Successor1 = getOrCreateVPBB(IRSucc1);
-    if (BB == LoopForBB->getLoopLatch()) {
-      // For a latch we need to set the successor of the region rather than that
-      // of VPBB and it should be set to the exit, i.e., non-header successor,
-      // except for the top region, which is handled elsewhere.
-      assert(LoopForBB != TheLoop &&
-             "Latch of the top region should have been handled earlier");
-      Region->setOneSuccessor(isHeaderVPBB(Successor0) ? Successor1
-                                                       : Successor0);
-      Region->setExiting(VPBB);
-      continue;
-    }
 
-    // Don't connect any blocks outside the current loop except the latch for
-    // now. The latch is handled above.
-    if (LoopForBB) {
+    // Don't connect any blocks outside the current loop except the latches for
+    // inner loops.
+    // TODO: Also connect exit blocks during initial VPlan construction.
+    if (LoopForBB == TheLoop || BB != LoopForBB->getLoopLatch()) {
       if (!LoopForBB->contains(IRSucc0)) {
         VPBB->setOneSuccessor(Successor1);
         continue;
@@ -456,21 +353,16 @@ void PlainCFGBuilder::buildPlainCFG(
   // corresponding VPlan operands.
   fixHeaderPhis();
 
-  VPBlockUtils::connectBlocks(Plan.getEntry(),
-                              getOrCreateVPBB(TheLoop->getHeader()));
+  Plan.getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader()));
+  Plan.getEntry()->setPlan(&Plan);
 
   for (const auto &[IRBB, VPB] : BB2VPBB)
     VPB2IRBB[VPB] = IRBB;
+
+  LLVM_DEBUG(Plan.setName("Plain CFG\n"); dbgs() << Plan);
 }
 
 void VPlanHCFGBuilder::buildPlainCFG() {
   PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan);
   PCFGBuilder.buildPlainCFG(VPB2IRBB);
 }
-
-// Public interface to build a H-CFG.
-void VPlanHCFGBuilder::buildHierarchicalCFG() {
-  // Build Top Region enclosing the plain CFG.
-  buildPlainCFG();
-  LLVM_DEBUG(Plan.setName("HCFGBuilder: Plain CFG\n"); dbgs() << Plan);
-}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
index f7f98ed7b1755..f2e90d3f4d9b3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
@@ -30,7 +30,6 @@ namespace llvm {
 
 class Loop;
 class LoopInfo;
-class VPRegionBlock;
 class VPlan;
 class VPlanTestIRBase;
 class VPBlockBase;
@@ -54,15 +53,12 @@ class VPlanHCFGBuilder {
   /// created for a input IR basic block.
   DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
 
-  /// Build plain CFG for TheLoop and connects it to Plan's entry.
-  void buildPlainCFG();
-
 public:
   VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
       : TheLoop(Lp), LI(LI), Plan(P) {}
 
-  /// Build H-CFG for TheLoop and update Plan accordingly.
-  void buildHierarchicalCFG();
+  /// Build plain CFG for TheLoop and connects it to Plan's entry.
+  void buildPlainCFG();
 
   /// Return the input IR BasicBlock corresponding to \p VPB. Returns nullptr if
   /// there is no such corresponding block.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index ee3642a8aff73..a9461b261ddb6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -52,20 +52,19 @@ struct VPlanTransforms {
       verifyVPlanIsValid(Plan);
   }
 
-  /// Introduce the top-level VPRegionBlock for the main loop in \p Plan. Coming
-  /// into this function, \p Plan's top-level loop is modeled using a plain CFG.
-  /// This transform wraps the plain CFG of the top-level loop within a
-  /// VPRegionBlock and creates a VPValue expression for the original trip
-  /// count. It will also introduce a dedicated VPBasicBlock for the vector
-  /// pre-header as well a VPBasicBlock as exit block of the region
-  /// (middle.block). If a check is needed to guard executing the scalar
+  /// Replace loops in \p Plan's flat CFG with VPRegionBlocks, turing \p Plan's
+  /// flat CFG into a hierarchical CFG. It also creates a VPValue expression for
+  /// the original trip count. It will also introduce a dedicated VPBasicBlock
+  /// for the vector pre-header as well a VPBasicBlock as exit block of the
+  /// region (middle.block). If a check is needed to guard executing the scalar
   /// epilogue loop, it will be added to the middle block, together with
   /// VPBasicBlocks for the scalar preheader and exit blocks. \p InductionTy is
   /// the type of the canonical induction and used for related values, like the
   /// trip count expression.
-  static void introduceTopLevelVectorLoopRegion(
-      VPlan &Plan, Type *InductionTy, PredicatedScalarEvolution &PSE,
-      bool RequiresScalarEpilogueCheck, bool TailFolded, Loop *TheLoop);
+  static void createLoopRegions(VPlan &Plan, Type *InductionTy,
+                                PredicatedScalarEvolution &PSE,
+                                bool RequiresScalarEpilogueCheck,
+                                bool TailFolded, Loop *TheLoop);
 
   /// Replaces the VPInstructions in \p Plan with corresponding
   /// widen recipes. Returns false if any VPInstructions could not be converted
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index ced60a30ad56e..638156eab7a84 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -246,6 +246,12 @@ class VPUser {
     New->addUser(*this);
   }
 
+  /// Swap operands of the VPUser. It must have exactly 2 operands.
+  void swapOperands() {
+    assert(Operands.size() == 2 && "must have 2 operands to swap");
+    std::swap(Operands[0], Operands[1]);
+  }
+
   /// Replaces all uses of \p From in the VPUser with \p To.
   void replaceUsesOfWith(VPValue *From, VPValue *To);
 
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 625a32c098f94..91a5ea6b7fe36 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -6,35 +6,32 @@
 @arr = external global [8 x [8 x i64]], align 16
 
 define void @foo(i64 %n) {
-; CHECK:      VPlan 'HCFGBuilder: Plain CFG
+; CHECK:      VPlan 'Plain CFG
 ; CHECK-NEXT: {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: Successor(s): vector.body
+; CHECK-NEXT: Successor(s): outer.header
 ; CHECK-EMPTY:
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
+; CHECK-NEXT: outer.header:
+; CHECK-NEXT:   WIDEN-PHI ir<%outer.iv> = phi ir<%outer.iv.next>, ir<0>
 ; CHECK-NEXT:   EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
 ; CHECK-NEXT:   EMIT store ir<%outer.iv>, ir<%gep.1>
 ; CHECK-NEXT:   EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
 ; CHECK-NEXT: Successor(s): inner
 ; CHECK-EMPTY:
-; CHECK-NEXT: <x1> inner: {
-; CHECK-NEXT:   inner:
-; CHECK-NEXT:     WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
-; CHECK-NEXT:     EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
-; CHECK-NEXT:     EMIT store ir<%add>, ir<%gep.2>
-; CHECK-NEXT:     EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
-; CHECK-NEXT:     EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
-; CHECK-NEXT:     EMIT branch-on-cond ir<%inner.ec>
-; CHECK-NEXT:   No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): outer.latch
+; CHECK-NEXT: inner:
+; CHECK-NEXT:   WIDEN-PHI ir<%inner.iv> = phi ir<%inner.iv.next>, ir<0>
+; CHECK-NEXT:   EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
+; CHECK-NEXT:   EMIT store ir<%add>, ir<%gep.2>
+; CHECK-NEXT:   EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
+; CHECK-NEXT:   EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
+; CHECK-NEXT:   EMIT branch-on-cond ir<%inner.ec>
+; CHECK-NEXT: Successor(s): outer.latch, inner
 ; CHECK-EMPTY:
 ; CHECK-NEXT: outer.latch:
 ; CHECK-NEXT:   EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
 ; CHECK-NEXT:   EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
-; CHECK-NEXT: Successor(s): vector.body
+; CHECK-NEXT: Successor(s): outer.header
 ; CHECK-NEXT: }
 entry:
   br label %outer.header
diff --git a/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll b/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
index 89eaca0cfa8c8..29aeb7c4e97f9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
@@ -4,7 +4,7 @@
 ; Verify that the stress testing flag for the VPlan H-CFG builder works as
 ; expected with and without enabling the VPlan H-CFG Verifier.
 
-; CHECK: VPlan 'HCFGBuilder: Plain CFG
+; CHECK: VPlan 'Plain CFG
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index caf5d2357411d..486296535996b 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -73,9 +73,9 @@ class VPlanTestIRBase : public testing::Test {
     PredicatedScalarEvolution PSE(*SE, *L);
     auto Plan = std::make_unique<VPlan>(L);
     VPlanHCFGBuilder HCFGBuilder(L, LI.get(), *Plan);
-    HCFGBuilder.buildHierarchicalCFG();
-    VPlanTransforms::introduceTopLevelVectorLoopRegion(
-        *Plan, IntegerType::get(*Ctx, 64), PSE, true, false, L);
+    HCFGBuilder.buildPlainCFG();
+    VPlanTransforms::createLoopRegions(*Plan, IntegerType::get(*Ctx, 64), PSE,
+                                       true, false, L);
     return Plan;
   }
 };

From bb5f53c727419c90e3ad6ca7db49330c64a8f54c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 16 Apr 2025 12:34:11 +0100
Subject: [PATCH 106/710] [DAG] isSplatValue - only treat binop splats with
 repeated undef elements as undef (#135945)

#135597 didn't correctly fix the issue of binops with an undef element
from only one operand - only reporting the common undef elements could
incorrectly recognise splats where the (binop X, undef) fold might
actually be different - we need to ensure both operands have the same
demanded undefs for certainty.

Fixes #135917
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  10 +-
 .../rvv/fixed-vectors-buildvec-of-binop.ll    |   3 +-
 llvm/test/CodeGen/X86/pr134602.ll             |   2 +-
 llvm/test/CodeGen/X86/pr135917.ll             |  46 +---
 .../CodeGen/X86/vector-fshl-rot-sub128.ll     |  92 +++++--
 llvm/test/CodeGen/X86/vector-fshl-sub128.ll   | 250 ++++++++++++++---
 .../CodeGen/X86/vector-fshr-rot-sub128.ll     | 106 ++++++--
 llvm/test/CodeGen/X86/vector-fshr-sub128.ll   | 256 +++++++++++++-----
 8 files changed, 568 insertions(+), 197 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 46fc8856640de..8682c40898046 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -3002,12 +3002,14 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
     APInt UndefLHS, UndefRHS;
     SDValue LHS = V.getOperand(0);
     SDValue RHS = V.getOperand(1);
-    // Only propagate common undef elts for both operands, otherwise we might
-    // fail to handle binop-specific undef handling.
+    // Only recognize splats with the same demanded undef elements for both
+    // operands, otherwise we might fail to handle binop-specific undef
+    // handling.
     // e.g. (and undef, 0) -> 0 etc.
     if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
-        isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
-      UndefElts = UndefLHS & UndefRHS;
+        isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1) &&
+        (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
+      UndefElts = UndefLHS | UndefRHS;
       return true;
     }
     return false;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
index 3df63b4de82e3..dbbb8362144ca 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
@@ -452,7 +452,8 @@ define void @buggy(i32 %0) #0 {
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
 ; RV64-NEXT:    vor.vi v8, v8, 1
-; RV64-NEXT:    vse32.v v8, (zero)
+; RV64-NEXT:    vrgather.vi v9, v8, 0
+; RV64-NEXT:    vse32.v v9, (zero)
 ; RV64-NEXT:    ret
 entry:
   %mul.us.us.i.3 = shl i32 %0, 1
diff --git a/llvm/test/CodeGen/X86/pr134602.ll b/llvm/test/CodeGen/X86/pr134602.ll
index 50efcde462532..063b6f31fe630 100644
--- a/llvm/test/CodeGen/X86/pr134602.ll
+++ b/llvm/test/CodeGen/X86/pr134602.ll
@@ -17,7 +17,7 @@ define i32 @PR134602(i16 %a0) {
 ; X64-NEXT:    movzwl %di, %eax
 ; X64-NEXT:    movd %eax, %xmm0
 ; X64-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[2,2,2,2,4,5,6,7]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-NEXT:    paddw %xmm0, %xmm1
 ; X64-NEXT:    movdqa %xmm1, %xmm0
 ; X64-NEXT:    psrld $16, %xmm0
diff --git a/llvm/test/CodeGen/X86/pr135917.ll b/llvm/test/CodeGen/X86/pr135917.ll
index 9eed955128b74..2061e3e7cc395 100644
--- a/llvm/test/CodeGen/X86/pr135917.ll
+++ b/llvm/test/CodeGen/X86/pr135917.ll
@@ -1,46 +1,26 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64    | FileCheck %s --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefix=SSE4
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64    | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefix=AVX512
 
 define i32 @PR135917(i1 %a0) {
-; SSE2-LABEL: PR135917:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd %edi, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    addl %ecx, %eax
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: PR135917:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    movd %edi, %xmm0
-; SSE4-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE4-NEXT:    movd %xmm0, %ecx
-; SSE4-NEXT:    pextrd $1, %xmm0, %eax
-; SSE4-NEXT:    addl %ecx, %eax
-; SSE4-NEXT:    retq
-;
-; AVX2-LABEL: PR135917:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovd %edi, %xmm0
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1]
-; AVX2-NEXT:    vpandn %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %ecx
-; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX2-NEXT:    addl %ecx, %eax
-; AVX2-NEXT:    retq
+; CHECK-LABEL: PR135917:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    notl %edi
+; CHECK-NEXT:    andl $1, %edi
+; CHECK-NEXT:    leal (%rdi,%rdi), %eax
+; CHECK-NEXT:    retq
 ;
 ; AVX512-LABEL: PR135917:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    andb $1, %dil
-; AVX512-NEXT:    negb %dil
 ; AVX512-NEXT:    kmovd %edi, %k0
 ; AVX512-NEXT:    knotw %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm0
+; AVX512-NEXT:    vpbroadcastd %xmm0, %xmm0
+; AVX512-NEXT:    vpmovd2m %xmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %xmm0
 ; AVX512-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %ecx
 ; AVX512-NEXT:    vpextrd $1, %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
index 8523cb4973827..9ecc6296a844a 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
@@ -162,42 +162,72 @@ define <2 x i32> @var_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind {
 define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind {
 ; SSE2-LABEL: splatvar_funnnel_v2i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE2-NEXT:    pslld $23, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; SSE2-NEXT:    psllq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; SSE2-NEXT:    psllq %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3]
+; SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_funnnel_v2i32:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pslld $23, %xmm1
 ; SSE41-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; SSE41-NEXT:    psllq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; SSE41-NEXT:    psllq %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3]
+; SSE41-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pmuludq %xmm2, %xmm3
+; SSE41-NEXT:    pmuludq %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_funnnel_v2i32:
 ; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3]
+; AVX1-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; AVX2-NEXT:    vpsllq %xmm1, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; AVX2-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3]
+; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpslld %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v2i32:
@@ -259,12 +289,22 @@ define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind
 ;
 ; X86-SSE2-LABEL: splatvar_funnnel_v2i32:
 ; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; X86-SSE2-NEXT:    pslld $23, %xmm1
 ; X86-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; X86-SSE2-NEXT:    psllq %xmm1, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; X86-SSE2-NEXT:    psllq %xmm1, %xmm0
-; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3]
+; X86-SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    por %xmm3, %xmm0
 ; X86-SSE2-NEXT:    retl
   %splat = shufflevector <2 x i32> %amt, <2 x i32> undef, <2 x i32> zeroinitializer
   %res = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %x, <2 x i32> %x, <2 x i32> %splat)
diff --git a/llvm/test/CodeGen/X86/vector-fshl-sub128.ll b/llvm/test/CodeGen/X86/vector-fshl-sub128.ll
index eb4d84b8d7dd6..322ebe22671e6 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-sub128.ll
@@ -248,27 +248,162 @@ define <2 x i32> @var_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %amt)
 ;
 
 define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %amt) nounwind {
-; SSE-LABEL: splatvar_funnnel_v2i32:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm1, %xmm3
-; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT:    psllq %xmm2, %xmm3
-; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE-NEXT:    psllq %xmm2, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm3[1,3]
-; SSE-NEXT:    movaps %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: splatvar_funnnel_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [31,31,31,31]
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm4, %xmm5
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[2,3,3,3,4,5,6,7]
+; SSE2-NEXT:    psrld $1, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    psrld %xmm2, %xmm6
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm5[0,1,1,1,4,5,6,7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrld %xmm7, %xmm2
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm5[2,3,3,3,4,5,6,7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm7
+; SSE2-NEXT:    psrld %xmm6, %xmm7
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,1,1,1,4,5,6,7]
+; SSE2-NEXT:    psrld %xmm5, %xmm1
+; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm7[1]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[0,3]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pslld $23, %xmm3
+; SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
 ;
-; AVX-LABEL: splatvar_funnnel_v2i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX-NEXT:    vpsllq %xmm2, %xmm3, %xmm3
-; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX-NEXT:    vpsllq %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
-; AVX-NEXT:    retq
+; SSE41-LABEL: splatvar_funnnel_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE41-NEXT:    pmovsxbd {{.*#+}} xmm3 = [31,31,31,31]
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pandn %xmm3, %xmm4
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
+; SSE41-NEXT:    psrld $1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    psrld %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm7 = xmm5[2,3,3,3,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm8
+; SSE41-NEXT:    psrld %xmm7, %xmm8
+; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm6[0,1,2,3],xmm8[4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    psrld %xmm4, %xmm6
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,1,1,1,4,5,6,7]
+; SSE41-NEXT:    psrld %xmm4, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3],xmm6[4,5],xmm8[6,7]
+; SSE41-NEXT:    pand %xmm3, %xmm2
+; SSE41-NEXT:    pslld $23, %xmm2
+; SSE41-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm1
+; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: splatvar_funnnel_v2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [31,31,31,31]
+; AVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrld $1, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrld %xmm5, %xmm1, %xmm5
+; AVX1-NEXT:    vpsrlq $32, %xmm4, %xmm6
+; AVX1-NEXT:    vpsrld %xmm6, %xmm1, %xmm6
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; AVX1-NEXT:    vpsrld %xmm6, %xmm1, %xmm6
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX1-NEXT:    vpsrld %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm6[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,3],xmm1[4,5],xmm5[6,7]
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: splatvar_funnnel_v2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX2-NEXT:    vpandn %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpsrld $1, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrlvd %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX2-NEXT:    vpslld %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: splatvar_funnnel_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX512F-NEXT:    vpandn %xmm4, %xmm3, %xmm3
+; AVX512F-NEXT:    vpsrld $1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpsrlvd %xmm3, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX512F-NEXT:    vpslld %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX512VL-NEXT:    vpandn %xmm4, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpsrld $1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsrlvd %xmm3, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX512VL-NEXT:    vpslld %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v2i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX512BW-NEXT:    vpandn %xmm4, %xmm3, %xmm3
+; AVX512BW-NEXT:    vpsrld $1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsrlvd %xmm3, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX512BW-NEXT:    vpslld %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v2i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX512VLBW-NEXT:    vpandn %xmm4, %xmm3, %xmm3
+; AVX512VLBW-NEXT:    vpsrld $1, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsrlvd %xmm3, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX512VLBW-NEXT:    vpslld %xmm2, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v2i32:
 ; AVX512VBMI2:       # %bb.0:
@@ -286,26 +421,67 @@ define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %
 ; AVX512VLVBMI2-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0
 ; AVX512VLVBMI2-NEXT:    retq
 ;
-; XOP-LABEL: splatvar_funnnel_v2i32:
-; XOP:       # %bb.0:
-; XOP-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; XOP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; XOP-NEXT:    vpsllq %xmm2, %xmm3, %xmm3
-; XOP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; XOP-NEXT:    vpsllq %xmm2, %xmm0, %xmm0
-; XOP-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
-; XOP-NEXT:    retq
+; XOPAVX1-LABEL: splatvar_funnnel_v2i32:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [31,31,31,31]
+; XOPAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; XOPAVX1-NEXT:    vpshld %xmm4, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vpsrld $1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpshld %xmm2, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: splatvar_funnnel_v2i32:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastd %xmm2, %xmm3
+; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; XOPAVX2-NEXT:    vpandn %xmm4, %xmm3, %xmm3
+; XOPAVX2-NEXT:    vpsrld $1, %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpsrlvd %xmm3, %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; XOPAVX2-NEXT:    vpslld %xmm2, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    retq
 ;
 ; X86-SSE2-LABEL: splatvar_funnnel_v2i32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    movdqa %xmm1, %xmm3
-; X86-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; X86-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
-; X86-SSE2-NEXT:    psllq %xmm2, %xmm3
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-SSE2-NEXT:    psllq %xmm2, %xmm1
-; X86-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm3[1,3]
-; X86-SSE2-NEXT:    movaps %xmm1, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [31,31,31,31]
+; X86-SSE2-NEXT:    movdqa %xmm3, %xmm5
+; X86-SSE2-NEXT:    pandn %xmm4, %xmm5
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[2,3,3,3,4,5,6,7]
+; X86-SSE2-NEXT:    psrld $1, %xmm1
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm6
+; X86-SSE2-NEXT:    psrld %xmm2, %xmm6
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm5[0,1,1,1,4,5,6,7]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT:    psrld %xmm7, %xmm2
+; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm5[2,3,3,3,4,5,6,7]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm7
+; X86-SSE2-NEXT:    psrld %xmm6, %xmm7
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,1,1,1,4,5,6,7]
+; X86-SSE2-NEXT:    psrld %xmm5, %xmm1
+; X86-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm7[1]
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[0,3]
+; X86-SSE2-NEXT:    pand %xmm4, %xmm3
+; X86-SSE2-NEXT:    pslld $23, %xmm3
+; X86-SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm3
+; X86-SSE2-NEXT:    cvttps2dq %xmm3, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
 ; X86-SSE2-NEXT:    retl
   %splat = shufflevector <2 x i32> %amt, <2 x i32> undef, <2 x i32> zeroinitializer
   %res = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %splat)
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
index d4874ad2cbd78..178c02f384f9b 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
@@ -172,42 +172,78 @@ define <2 x i32> @var_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind {
 define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind {
 ; SSE2-LABEL: splatvar_funnnel_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; SSE2-NEXT:    psrlq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; SSE2-NEXT:    psrlq %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    psubd %xmm1, %xmm2
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_funnnel_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; SSE41-NEXT:    psrlq %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    pslld $23, %xmm2
+; SSE41-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE41-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pmuludq %xmm2, %xmm3
+; SSE41-NEXT:    pmuludq %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_funnnel_v2i32:
 ; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; AVX1-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; AVX2-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; AVX2-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpsrld %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpslld %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v2i32:
@@ -273,12 +309,24 @@ define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind
 ;
 ; X86-SSE2-LABEL: splatvar_funnnel_v2i32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]
-; X86-SSE2-NEXT:    psrlq %xmm1, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; X86-SSE2-NEXT:    psrlq %xmm1, %xmm0
-; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE2-NEXT:    psubd %xmm1, %xmm2
+; X86-SSE2-NEXT:    pslld $23, %xmm2
+; X86-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
+; X86-SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
+; X86-SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    por %xmm3, %xmm0
 ; X86-SSE2-NEXT:    retl
   %splat = shufflevector <2 x i32> %amt, <2 x i32> undef, <2 x i32> zeroinitializer
   %res = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %x, <2 x i32> %splat)
diff --git a/llvm/test/CodeGen/X86/vector-fshr-sub128.ll b/llvm/test/CodeGen/X86/vector-fshr-sub128.ll
index 58dc17988b646..372deb05e550c 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-sub128.ll
@@ -249,78 +249,161 @@ define <2 x i32> @var_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %amt)
 ;
 
 define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %amt) nounwind {
-; SSE-LABEL: splatvar_funnnel_v2i32:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm1, %xmm3
-; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT:    psrlq %xmm2, %xmm3
-; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE-NEXT:    psrlq %xmm2, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
-; SSE-NEXT:    movaps %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: splatvar_funnnel_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [31,31,31,31]
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[2,3,3,3,4,5,6,7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    psrld %xmm2, %xmm6
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm5[0,1,1,1,4,5,6,7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrld %xmm7, %xmm2
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm5[2,3,3,3,4,5,6,7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm7
+; SSE2-NEXT:    psrld %xmm6, %xmm7
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,1,1,1,4,5,6,7]
+; SSE2-NEXT:    psrld %xmm5, %xmm1
+; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm7[1]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[0,3]
+; SSE2-NEXT:    pandn %xmm4, %xmm3
+; SSE2-NEXT:    pslld $23, %xmm3
+; SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm1
+; SSE2-NEXT:    paddd %xmm0, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: splatvar_funnnel_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; SSE41-NEXT:    pmovsxbd {{.*#+}} xmm3 = [31,31,31,31]
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pand %xmm3, %xmm4
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    psrld %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm7 = xmm5[2,3,3,3,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm8
+; SSE41-NEXT:    psrld %xmm7, %xmm8
+; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm6[0,1,2,3],xmm8[4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm1, %xmm6
+; SSE41-NEXT:    psrld %xmm4, %xmm6
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,1,1,1,4,5,6,7]
+; SSE41-NEXT:    psrld %xmm4, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1,2,3],xmm1[4,5,6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3],xmm6[4,5],xmm8[6,7]
+; SSE41-NEXT:    pandn %xmm3, %xmm2
+; SSE41-NEXT:    pslld $23, %xmm2
+; SSE41-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm1
+; SSE41-NEXT:    paddd %xmm0, %xmm0
+; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_funnnel_v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [31,31,31,31]
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrld %xmm5, %xmm1, %xmm5
+; AVX1-NEXT:    vpsrlq $32, %xmm4, %xmm6
+; AVX1-NEXT:    vpsrld %xmm6, %xmm1, %xmm6
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]
+; AVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; AVX1-NEXT:    vpsrld %xmm6, %xmm1, %xmm6
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX1-NEXT:    vpsrld %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm6[4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,3],xmm1[4,5],xmm5[6,7]
+; AVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX2-NEXT:    vpsrlq %xmm2, %xmm3, %xmm3
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX2-NEXT:    vpsrlq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; AVX2-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX2-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX2-NEXT:    vpsrld %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpandn %xmm4, %xmm3, %xmm2
+; AVX2-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v2i32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpsrlq %xmm2, %xmm3, %xmm3
-; AVX512F-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX512F-NEXT:    vpsrlq %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; AVX512F-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX512F-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX512F-NEXT:    vpsrld %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpandn %xmm4, %xmm3, %xmm2
+; AVX512F-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v2i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX512VL-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
-; AVX512VL-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovqd %ymm0, %xmm0
-; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX512VL-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX512VL-NEXT:    vpsrld %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpandn %xmm4, %xmm3, %xmm2
+; AVX512VL-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v2i32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsrlq %xmm2, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX512BW-NEXT:    vpsrlq %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; AVX512BW-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX512BW-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX512BW-NEXT:    vpsrld %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpandn %xmm4, %xmm3, %xmm2
+; AVX512BW-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v2i32:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512VLBW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX512VLBW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
-; AVX512VLBW-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpmovqd %ymm0, %xmm0
-; AVX512VLBW-NEXT:    vzeroupper
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %xmm3
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; AVX512VLBW-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; AVX512VLBW-NEXT:    vpsrld %xmm2, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpandn %xmm4, %xmm3, %xmm2
+; AVX512VLBW-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v2i32:
@@ -340,26 +423,67 @@ define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %
 ; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VLVBMI2-NEXT:    retq
 ;
-; XOP-LABEL: splatvar_funnnel_v2i32:
-; XOP:       # %bb.0:
-; XOP-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; XOP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; XOP-NEXT:    vpsrlq %xmm2, %xmm3, %xmm3
-; XOP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; XOP-NEXT:    vpsrlq %xmm2, %xmm0, %xmm0
-; XOP-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
-; XOP-NEXT:    retq
+; XOPAVX1-LABEL: splatvar_funnnel_v2i32:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
+; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [31,31,31,31]
+; XOPAVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm4
+; XOPAVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpshld %xmm4, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vpshld %xmm2, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: splatvar_funnnel_v2i32:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastd %xmm2, %xmm3
+; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
+; XOPAVX2-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; XOPAVX2-NEXT:    vpsrld %xmm2, %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpandn %xmm4, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    retq
 ;
 ; X86-SSE2-LABEL: splatvar_funnnel_v2i32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    movdqa %xmm1, %xmm3
-; X86-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; X86-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
-; X86-SSE2-NEXT:    psrlq %xmm2, %xmm3
-; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-SSE2-NEXT:    psrlq %xmm2, %xmm1
-; X86-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
-; X86-SSE2-NEXT:    movaps %xmm1, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [31,31,31,31]
+; X86-SSE2-NEXT:    movdqa %xmm3, %xmm5
+; X86-SSE2-NEXT:    pand %xmm4, %xmm5
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[2,3,3,3,4,5,6,7]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm6
+; X86-SSE2-NEXT:    psrld %xmm2, %xmm6
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm5[0,1,1,1,4,5,6,7]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT:    psrld %xmm7, %xmm2
+; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm5[2,3,3,3,4,5,6,7]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm7
+; X86-SSE2-NEXT:    psrld %xmm6, %xmm7
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,1,1,1,4,5,6,7]
+; X86-SSE2-NEXT:    psrld %xmm5, %xmm1
+; X86-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm7[1]
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[0,3]
+; X86-SSE2-NEXT:    pandn %xmm4, %xmm3
+; X86-SSE2-NEXT:    pslld $23, %xmm3
+; X86-SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm3
+; X86-SSE2-NEXT:    cvttps2dq %xmm3, %xmm1
+; X86-SSE2-NEXT:    paddd %xmm0, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
 ; X86-SSE2-NEXT:    retl
   %splat = shufflevector <2 x i32> %amt, <2 x i32> undef, <2 x i32> zeroinitializer
   %res = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %splat)

From cf2399a2ee5b8a721eff385acbaa68fb9b00127c Mon Sep 17 00:00:00 2001
From: Wenju He <wenju.he@intel.com>
Date: Wed, 16 Apr 2025 11:34:59 +0000
Subject: [PATCH 107/710] [CI] enable code-format-helper for .cl files
 (#135748)

In clang-format, OpenCL .cl file uses default C++ formatting. There are
many pull-requests in libclc project that change OpenCL files. It is
beneficial to enable clang-format for them in CI.
---
 llvm/utils/git/code-format-helper.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py
index ed102b54f9b52..4e4145dcbb8c2 100755
--- a/llvm/utils/git/code-format-helper.py
+++ b/llvm/utils/git/code-format-helper.py
@@ -186,7 +186,17 @@ def filter_changed_files(self, changed_files: List[str]) -> List[str]:
         filtered_files = []
         for path in changed_files:
             _, ext = os.path.splitext(path)
-            if ext in (".cpp", ".c", ".h", ".hpp", ".hxx", ".cxx", ".inc", ".cppm"):
+            if ext in (
+                ".cpp",
+                ".c",
+                ".h",
+                ".hpp",
+                ".hxx",
+                ".cxx",
+                ".inc",
+                ".cppm",
+                ".cl",
+            ):
                 filtered_files.append(path)
             elif ext == "" and self.should_include_extensionless_file(path):
                 filtered_files.append(path)

From 38ca73db223031b1831cd24ef66ddb6a8546a16c Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Wed, 16 Apr 2025 13:37:03 +0200
Subject: [PATCH 108/710] [clang][bytecode] Give typeinfo APValues an
 LValuePath (#135948)

That's what the current interpreter does as well.
---
 clang/lib/AST/ByteCode/Pointer.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index c43c0a063bd9e..686ec381d232a 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -167,7 +167,8 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
     return APValue(
         APValue::LValueBase::getTypeInfo(
             TypeInfo, QualType(PointeeStorage.Typeid.TypeInfoType, 0)),
-        CharUnits::Zero(), APValue::NoLValuePath{});
+        CharUnits::Zero(), {},
+        /*OnePastTheEnd=*/false, /*IsNull=*/false);
   }
 
   // Build the lvalue base from the block.

From 27c1aa9b9cf9e0b14211758ff8f7d3aaba24ffcf Mon Sep 17 00:00:00 2001
From: mgschossmann <109181247+mgschossmann@users.noreply.github.com>
Date: Wed, 16 Apr 2025 13:42:37 +0200
Subject: [PATCH 109/710] [Clang,debuginfo] added vtt parameter in destructor
 DISubroutineType (#130674)

Fixes issue #104765: When creating a virtual destructor with an
artificial "vtt" argument, the type of "vtt" was previously missing in
the `DISubroutineType` `types` array.

This commit fixes this behavior and adds a regression test.
---
 clang/lib/CodeGen/CGDebugInfo.cpp             | 21 +++++++++++++---
 clang/lib/CodeGen/CGDebugInfo.h               |  7 +++++-
 .../debug-info-dtor-implicit-args.cpp         | 24 +++++++++++++++++++
 3 files changed, 48 insertions(+), 4 deletions(-)
 create mode 100644 clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp

diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index f3ec498d4064b..1582d911ba052 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -2018,8 +2018,17 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
   return getOrCreateInstanceMethodType(ThisType, Func, Unit);
 }
 
-llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType(
-    QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile *Unit) {
+llvm::DISubroutineType *CGDebugInfo::getOrCreateMethodTypeForDestructor(
+    const CXXMethodDecl *Method, llvm::DIFile *Unit, QualType FNType) {
+  const FunctionProtoType *Func = FNType->getAs<FunctionProtoType>();
+  // skip the first param since it is also this
+  return getOrCreateInstanceMethodType(Method->getThisType(), Func, Unit, true);
+}
+
+llvm::DISubroutineType *
+CGDebugInfo::getOrCreateInstanceMethodType(QualType ThisPtr,
+                                           const FunctionProtoType *Func,
+                                           llvm::DIFile *Unit, bool SkipFirst) {
   FunctionProtoType::ExtProtoInfo EPI = Func->getExtProtoInfo();
   Qualifiers &Qc = EPI.TypeQuals;
   Qc.removeConst();
@@ -2059,7 +2068,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType(
   }
 
   // Copy rest of the arguments.
-  for (unsigned i = 1, e = Args.size(); i != e; ++i)
+  for (unsigned i = (SkipFirst ? 2 : 1), e = Args.size(); i != e; ++i)
     Elts.push_back(Args[i]);
 
   // Attach FlagObjectPointer to the explicit "this" parameter.
@@ -4372,6 +4381,12 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D,
     // subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields.
     return DBuilder.createSubroutineType(DBuilder.getOrCreateTypeArray({}));
 
+  if (const auto *Method = dyn_cast<CXXDestructorDecl>(D)) {
+    // Read method type from 'FnType' because 'D.getType()' does not cover
+    // implicit arguments for destructors.
+    return getOrCreateMethodTypeForDestructor(Method, F, FnType);
+  }
+
   if (const auto *Method = dyn_cast<CXXMethodDecl>(D))
     return getOrCreateMethodType(Method, F);
 
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index b287ce7b92eee..771c129230eea 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -249,9 +249,14 @@ class CGDebugInfo {
   /// to get a method type which includes \c this pointer.
   llvm::DISubroutineType *getOrCreateMethodType(const CXXMethodDecl *Method,
                                                 llvm::DIFile *F);
+
+  llvm::DISubroutineType *
+  getOrCreateMethodTypeForDestructor(const CXXMethodDecl *Method,
+                                     llvm::DIFile *F, QualType FNType);
+
   llvm::DISubroutineType *
   getOrCreateInstanceMethodType(QualType ThisPtr, const FunctionProtoType *Func,
-                                llvm::DIFile *Unit);
+                                llvm::DIFile *Unit, bool SkipFirst = false);
   llvm::DISubroutineType *
   getOrCreateFunctionType(const Decl *D, QualType FnType, llvm::DIFile *F);
   /// \return debug info descriptor for vtable.
diff --git a/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp b/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp
new file mode 100644
index 0000000000000..4bb51dcc4da51
--- /dev/null
+++ b/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple x86_64-none-linux-gnu -emit-llvm -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -emit-llvm -debug-info-kind=limited %s -o - | FileCheck --check-prefix MSVC %s
+
+struct B {
+    virtual ~B() {}
+};
+
+struct A : virtual B {
+};
+
+A a;
+
+
+// CHECK-DAG: !{{[0-9]+}} = !DILocalVariable(name: "vtt", arg: 2, scope: ![[destructor:[0-9]+]], type: ![[vtttype:[0-9]+]], flags: DIFlagArtificial)
+// CHECK-DAG: ![[destructor]] = distinct !DISubprogram(name: "~A", {{.*}}, type: ![[subroutinetype:[0-9]+]]
+// CHECK-DAG: ![[subroutinetype]] = !DISubroutineType(types: ![[types:[0-9]+]])
+// CHECK-DAG: [[types]] = !{null, !{{[0-9]+}}, ![[vtttype]]}
+
+// MSVC-DAG: ![[inttype:[0-9]+]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+// MSVC-DAG: ![[voidpointertype:[0-9]+]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+// MSVC-DAG: ![[destructor:[0-9]+]] = distinct !DISubprogram(name: "~A", linkageName: "??_GA@@UEAAPEAXI@Z", {{.*}}, type: ![[subroutinetype:[0-9]+]]
+// MSVC-DAG: !{{[0-9]+}} = !DILocalVariable(name: "should_call_delete", arg: 2, scope: ![[destructor]], type: ![[inttype]], flags: DIFlagArtificial)
+// MSVC-DAG: ![[subroutinetype]] = !DISubroutineType(types: ![[types:[0-9]+]])
+// MSVC-DAG: [[types]] = !{![[voidpointertype]], !{{[0-9]+}}, ![[inttype]]}

From 616613c80b75614736d0781d12c0e1237d79738f Mon Sep 17 00:00:00 2001
From: Sirraide <aeternalmail@gmail.com>
Date: Wed, 16 Apr 2025 14:08:59 +0200
Subject: [PATCH 110/710] [Clang] [Sema] Fix a crash when a `friend` function
 is redefined as deleted (#135679)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

NB: This only fixes the crash introduced in Clang 19; we still accept
this code even though we shouldn’t:
```c++
struct S {
    friend int f() { return 3; }
    friend int f() = delete;
};
```
I tried figuring out a way to diagnose this redeclaration, but it seems
tricky because I kept running into issues around defaulted comparison
operators. From my testing, however, this fix here would still be
required even once we do start diagnosing this.

Fixes #135506.
---
 clang/docs/ReleaseNotes.rst                   |  2 ++
 clang/lib/Sema/SemaDecl.cpp                   | 15 ++++------
 .../SemaCXX/cxx2c-delete-with-message.cpp     | 30 +++++++++++++++++++
 3 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 5af4c08f64cd8..0891fd058bb57 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -425,6 +425,8 @@ Bug Fixes in This Version
 - Fixed a clang 20 regression where diagnostics attached to some calls to member functions
   using C++23 "deducing this" did not have a diagnostic location (#GH135522)
 
+- Fixed a crash when a ``friend`` function is redefined as deleted. (#GH135506)
+
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 5f811c824e11d..127c0a4500a43 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -16199,16 +16199,11 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
     // This is meant to pop the context added in ActOnStartOfFunctionDef().
     ExitFunctionBodyRAII ExitRAII(*this, isLambdaCallOperator(FD));
     if (FD) {
-      // If this is called by Parser::ParseFunctionDefinition() after marking
-      // the declaration as deleted, and if the deleted-function-body contains
-      // a message (C++26), then a DefaultedOrDeletedInfo will have already been
-      // added to store that message; do not overwrite it in that case.
-      //
-      // Since this would always set the body to 'nullptr' in that case anyway,
-      // which is already done when the function decl is initially created,
-      // always skipping this irrespective of whether there is a delete message
-      // should not be a problem.
-      if (!FD->isDeletedAsWritten())
+      // The function body and the DefaultedOrDeletedInfo, if present, use
+      // the same storage; don't overwrite the latter if the former is null
+      // (the body is initialised to null anyway, so even if the latter isn't
+      // present, this would still be a no-op).
+      if (Body)
         FD->setBody(Body);
       FD->setWillHaveBody(false);
 
diff --git a/clang/test/SemaCXX/cxx2c-delete-with-message.cpp b/clang/test/SemaCXX/cxx2c-delete-with-message.cpp
index 22e65d902ecd4..5609da18c05aa 100644
--- a/clang/test/SemaCXX/cxx2c-delete-with-message.cpp
+++ b/clang/test/SemaCXX/cxx2c-delete-with-message.cpp
@@ -271,3 +271,33 @@ void operators() {
   if (to_int_int) {} // expected-error {{attempt to use a deleted function: deleted (TO<int, int>, operator bool)}}
   static_cast<bool>(to_int_int); // expected-error {{static_cast from 'TO<int, int>' to 'bool' uses deleted function: deleted (TO<int, int>, operator bool)}}
 };
+
+namespace gh135506 {
+struct a {
+  // FIXME: We currently don't diagnose these invalid redeclarations if the
+  // second declaration is defaulted or deleted. This probably needs to be
+  // handled in ParseCXXInlineMethodDef() after parsing the defaulted/deleted
+  // body.
+  friend consteval int f() { return 3; }
+  friend consteval int f() = delete("foo");
+
+  friend consteval int g() { return 3; }
+  friend consteval int g() = delete;
+
+  friend int h() { return 3; }
+  friend int h() = delete;
+
+  friend consteval int i() = delete; // expected-note {{previous definition is here}}
+  friend consteval int i() { return 3; } // expected-error {{redefinition of 'i'}}
+};
+
+struct b {
+  friend consteval bool operator==(b, b) { return true; } // expected-note {{previous declaration is here}}
+  friend consteval bool operator==(b, b) = default; // expected-error {{defaulting this equality comparison operator is not allowed because it was already declared outside the class}}
+};
+
+struct c {
+  friend consteval bool operator==(c, c) = default; // expected-note {{previous definition is here}}
+  friend consteval bool operator==(c, c) { return true; } // expected-error {{redefinition of 'operator=='}}
+};
+}

From 30990c09c99bdcbfa7084d32b2b9851e19b6fb2a Mon Sep 17 00:00:00 2001
From: Kareem Ergawy <kareem.ergawy@amd.com>
Date: Wed, 16 Apr 2025 14:20:27 +0200
Subject: [PATCH 111/710] Revert "[flang][fir] Lower `do concurrent` loop nests
 to `fir.do_concurrent` (#132904)" (#135904)

This reverts commit 04b87e15e40f8857e29ade8321b8b67691545a50.

The reasons for reverting is that the following:
1. I still need need to upstream some part of the do concurrent to
OpenMP pass from our downstream implementation and taking this in
downstream will make things more difficult.
2. I still need to work on a solution for modeling locality specifiers
on `hlfir.do_concurrent` ops. I would prefer to do that and merge the
entire stack together instead of having a partial solution.

After merging the revert I will reopen the origianl PR and keep it
updated against main until I finish the above.
---
 flang/lib/Lower/Bridge.cpp                    | 228 +++++++-----------
 flang/lib/Optimizer/Builder/FIRBuilder.cpp    |   3 -
 flang/test/Lower/do_concurrent.f90            |  39 +--
 .../do_concurrent_local_default_init.f90      |   4 +-
 flang/test/Lower/loops.f90                    |  37 ++-
 flang/test/Lower/loops3.f90                   |   4 +-
 flang/test/Lower/nsw.f90                      |   5 +-
 .../Transforms/DoConcurrent/basic_host.f90    |   3 -
 .../DoConcurrent/locally_destroyed_temp.f90   |   3 -
 .../DoConcurrent/loop_nest_test.f90           |   3 -
 .../multiple_iteration_ranges.f90             |   3 -
 .../DoConcurrent/non_const_bounds.f90         |   3 -
 .../DoConcurrent/not_perfectly_nested.f90     |   3 -
 13 files changed, 130 insertions(+), 208 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 625dd116fe726..b4d1197822a43 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -94,11 +94,10 @@ struct IncrementLoopInfo {
   template <typename T>
   explicit IncrementLoopInfo(Fortran::semantics::Symbol &sym, const T &lower,
                              const T &upper, const std::optional<T> &step,
-                             bool isConcurrent = false)
+                             bool isUnordered = false)
       : loopVariableSym{&sym}, lowerExpr{Fortran::semantics::GetExpr(lower)},
         upperExpr{Fortran::semantics::GetExpr(upper)},
-        stepExpr{Fortran::semantics::GetExpr(step)},
-        isConcurrent{isConcurrent} {}
+        stepExpr{Fortran::semantics::GetExpr(step)}, isUnordered{isUnordered} {}
 
   IncrementLoopInfo(IncrementLoopInfo &&) = default;
   IncrementLoopInfo &operator=(IncrementLoopInfo &&x) = default;
@@ -121,7 +120,7 @@ struct IncrementLoopInfo {
   const Fortran::lower::SomeExpr *upperExpr;
   const Fortran::lower::SomeExpr *stepExpr;
   const Fortran::lower::SomeExpr *maskExpr = nullptr;
-  bool isConcurrent;
+  bool isUnordered; // do concurrent, forall
   llvm::SmallVector<const Fortran::semantics::Symbol *> localSymList;
   llvm::SmallVector<const Fortran::semantics::Symbol *> localInitSymList;
   llvm::SmallVector<
@@ -131,7 +130,7 @@ struct IncrementLoopInfo {
   mlir::Value loopVariable = nullptr;
 
   // Data members for structured loops.
-  mlir::Operation *loopOp = nullptr;
+  fir::DoLoopOp doLoop = nullptr;
 
   // Data members for unstructured loops.
   bool hasRealControl = false;
@@ -1981,7 +1980,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     llvm_unreachable("illegal reduction operator");
   }
 
-  /// Collect DO CONCURRENT loop control information.
+  /// Collect DO CONCURRENT or FORALL loop control information.
   IncrementLoopNestInfo getConcurrentControl(
       const Fortran::parser::ConcurrentHeader &header,
       const std::list<Fortran::parser::LocalitySpec> &localityList = {}) {
@@ -2292,14 +2291,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get(
         builder->getContext(), {}, /*vectorize=*/va, {}, /*unroll*/ ua,
         /*unroll_and_jam*/ uja, {}, {}, {}, {}, {}, {}, {}, {}, {}, {});
-    if (has_attrs) {
-      if (auto loopOp = mlir::dyn_cast<fir::DoLoopOp>(info.loopOp))
-        loopOp.setLoopAnnotationAttr(la);
-
-      if (auto doConcurrentOp =
-              mlir::dyn_cast<fir::DoConcurrentLoopOp>(info.loopOp))
-        doConcurrentOp.setLoopAnnotationAttr(la);
-    }
+    if (has_attrs)
+      info.doLoop.setLoopAnnotationAttr(la);
   }
 
   /// Generate FIR to begin a structured or unstructured increment loop nest.
@@ -2308,77 +2301,96 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       llvm::SmallVectorImpl<const Fortran::parser::CompilerDirective *> &dirs) {
     assert(!incrementLoopNestInfo.empty() && "empty loop nest");
     mlir::Location loc = toLocation();
+    mlir::Operation *boundsAndStepIP = nullptr;
     mlir::arith::IntegerOverflowFlags iofBackup{};
 
-    llvm::SmallVector<mlir::Value> nestLBs;
-    llvm::SmallVector<mlir::Value> nestUBs;
-    llvm::SmallVector<mlir::Value> nestSts;
-    llvm::SmallVector<mlir::Value> nestReduceOperands;
-    llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
-    bool genDoConcurrent = false;
-
     for (IncrementLoopInfo &info : incrementLoopNestInfo) {
-      genDoConcurrent = info.isStructured() && info.isConcurrent;
-
-      if (!genDoConcurrent)
-        info.loopVariable = genLoopVariableAddress(loc, *info.loopVariableSym,
-                                                   info.isConcurrent);
-
-      if (!getLoweringOptions().getIntegerWrapAround()) {
-        iofBackup = builder->getIntegerOverflowFlags();
-        builder->setIntegerOverflowFlags(
-            mlir::arith::IntegerOverflowFlags::nsw);
-      }
+      mlir::Value lowerValue;
+      mlir::Value upperValue;
+      mlir::Value stepValue;
 
-      nestLBs.push_back(genControlValue(info.lowerExpr, info));
-      nestUBs.push_back(genControlValue(info.upperExpr, info));
-      bool isConst = true;
-      nestSts.push_back(genControlValue(
-          info.stepExpr, info, info.isStructured() ? nullptr : &isConst));
+      {
+        mlir::OpBuilder::InsertionGuard guard(*builder);
 
-      if (!getLoweringOptions().getIntegerWrapAround())
-        builder->setIntegerOverflowFlags(iofBackup);
+        // Set the IP before the first loop in the nest so that all nest bounds
+        // and step values are created outside the nest.
+        if (boundsAndStepIP)
+          builder->setInsertionPointAfter(boundsAndStepIP);
 
-      // Use a temp variable for unstructured loops with non-const step.
-      if (!isConst) {
-        mlir::Value stepValue = nestSts.back();
-        info.stepVariable = builder->createTemporary(loc, stepValue.getType());
-        builder->create<fir::StoreOp>(loc, stepValue, info.stepVariable);
-      }
-
-      if (genDoConcurrent && nestReduceOperands.empty()) {
-        // Create DO CONCURRENT reduce operands and attributes
-        for (const auto &reduceSym : info.reduceSymList) {
-          const fir::ReduceOperationEnum reduceOperation = reduceSym.first;
-          const Fortran::semantics::Symbol *sym = reduceSym.second;
-          fir::ExtendedValue exv = getSymbolExtendedValue(*sym, nullptr);
-          nestReduceOperands.push_back(fir::getBase(exv));
-          auto reduceAttr =
-              fir::ReduceAttr::get(builder->getContext(), reduceOperation);
-          nestReduceAttrs.push_back(reduceAttr);
+        info.loopVariable = genLoopVariableAddress(loc, *info.loopVariableSym,
+                                                   info.isUnordered);
+        if (!getLoweringOptions().getIntegerWrapAround()) {
+          iofBackup = builder->getIntegerOverflowFlags();
+          builder->setIntegerOverflowFlags(
+              mlir::arith::IntegerOverflowFlags::nsw);
+        }
+        lowerValue = genControlValue(info.lowerExpr, info);
+        upperValue = genControlValue(info.upperExpr, info);
+        bool isConst = true;
+        stepValue = genControlValue(info.stepExpr, info,
+                                    info.isStructured() ? nullptr : &isConst);
+        if (!getLoweringOptions().getIntegerWrapAround())
+          builder->setIntegerOverflowFlags(iofBackup);
+        boundsAndStepIP = stepValue.getDefiningOp();
+
+        // Use a temp variable for unstructured loops with non-const step.
+        if (!isConst) {
+          info.stepVariable =
+              builder->createTemporary(loc, stepValue.getType());
+          boundsAndStepIP =
+              builder->create<fir::StoreOp>(loc, stepValue, info.stepVariable);
         }
       }
-    }
 
-    for (auto [info, lowerValue, upperValue, stepValue] :
-         llvm::zip_equal(incrementLoopNestInfo, nestLBs, nestUBs, nestSts)) {
       // Structured loop - generate fir.do_loop.
       if (info.isStructured()) {
-        if (genDoConcurrent)
-          continue;
-
-        // The loop variable is a doLoop op argument.
         mlir::Type loopVarType = info.getLoopVariableType();
-        auto loopOp = builder->create<fir::DoLoopOp>(
-            loc, lowerValue, upperValue, stepValue, /*unordered=*/false,
-            /*finalCountValue=*/true,
-            builder->createConvert(loc, loopVarType, lowerValue));
-        info.loopOp = loopOp;
-        builder->setInsertionPointToStart(loopOp.getBody());
-        mlir::Value loopValue = loopOp.getRegionIterArgs()[0];
-
+        mlir::Value loopValue;
+        if (info.isUnordered) {
+          llvm::SmallVector<mlir::Value> reduceOperands;
+          llvm::SmallVector<mlir::Attribute> reduceAttrs;
+          // Create DO CONCURRENT reduce operands and attributes
+          for (const auto &reduceSym : info.reduceSymList) {
+            const fir::ReduceOperationEnum reduce_operation = reduceSym.first;
+            const Fortran::semantics::Symbol *sym = reduceSym.second;
+            fir::ExtendedValue exv = getSymbolExtendedValue(*sym, nullptr);
+            reduceOperands.push_back(fir::getBase(exv));
+            auto reduce_attr =
+                fir::ReduceAttr::get(builder->getContext(), reduce_operation);
+            reduceAttrs.push_back(reduce_attr);
+          }
+          // The loop variable value is explicitly updated.
+          info.doLoop = builder->create<fir::DoLoopOp>(
+              loc, lowerValue, upperValue, stepValue, /*unordered=*/true,
+              /*finalCountValue=*/false, /*iterArgs=*/std::nullopt,
+              llvm::ArrayRef<mlir::Value>(reduceOperands), reduceAttrs);
+          builder->setInsertionPointToStart(info.doLoop.getBody());
+          loopValue = builder->createConvert(loc, loopVarType,
+                                             info.doLoop.getInductionVar());
+        } else {
+          // The loop variable is a doLoop op argument.
+          info.doLoop = builder->create<fir::DoLoopOp>(
+              loc, lowerValue, upperValue, stepValue, /*unordered=*/false,
+              /*finalCountValue=*/true,
+              builder->createConvert(loc, loopVarType, lowerValue));
+          builder->setInsertionPointToStart(info.doLoop.getBody());
+          loopValue = info.doLoop.getRegionIterArgs()[0];
+        }
         // Update the loop variable value in case it has non-index references.
         builder->create<fir::StoreOp>(loc, loopValue, info.loopVariable);
+        if (info.maskExpr) {
+          Fortran::lower::StatementContext stmtCtx;
+          mlir::Value maskCond = createFIRExpr(loc, info.maskExpr, stmtCtx);
+          stmtCtx.finalizeAndReset();
+          mlir::Value maskCondCast =
+              builder->createConvert(loc, builder->getI1Type(), maskCond);
+          auto ifOp = builder->create<fir::IfOp>(loc, maskCondCast,
+                                                 /*withElseRegion=*/false);
+          builder->setInsertionPointToStart(&ifOp.getThenRegion().front());
+        }
+        if (info.hasLocalitySpecs())
+          handleLocalitySpecs(info);
+
         addLoopAnnotationAttr(info, dirs);
         continue;
       }
@@ -2442,60 +2454,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         builder->restoreInsertionPoint(insertPt);
       }
     }
-
-    if (genDoConcurrent) {
-      auto loopWrapperOp = builder->create<fir::DoConcurrentOp>(loc);
-      builder->setInsertionPointToStart(
-          builder->createBlock(&loopWrapperOp.getRegion()));
-
-      for (IncrementLoopInfo &info : llvm::reverse(incrementLoopNestInfo)) {
-        info.loopVariable = genLoopVariableAddress(loc, *info.loopVariableSym,
-                                                   info.isConcurrent);
-      }
-
-      builder->setInsertionPointToEnd(loopWrapperOp.getBody());
-      auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
-          loc, nestLBs, nestUBs, nestSts, nestReduceOperands,
-          nestReduceAttrs.empty()
-              ? nullptr
-              : mlir::ArrayAttr::get(builder->getContext(), nestReduceAttrs),
-          nullptr);
-
-      llvm::SmallVector<mlir::Type> loopBlockArgTypes(
-          incrementLoopNestInfo.size(), builder->getIndexType());
-      llvm::SmallVector<mlir::Location> loopBlockArgLocs(
-          incrementLoopNestInfo.size(), loc);
-      mlir::Region &loopRegion = loopOp.getRegion();
-      mlir::Block *loopBlock = builder->createBlock(
-          &loopRegion, loopRegion.begin(), loopBlockArgTypes, loopBlockArgLocs);
-      builder->setInsertionPointToStart(loopBlock);
-
-      for (auto [info, blockArg] :
-           llvm::zip_equal(incrementLoopNestInfo, loopBlock->getArguments())) {
-        info.loopOp = loopOp;
-        mlir::Value loopValue =
-            builder->createConvert(loc, info.getLoopVariableType(), blockArg);
-        builder->create<fir::StoreOp>(loc, loopValue, info.loopVariable);
-
-        if (info.maskExpr) {
-          Fortran::lower::StatementContext stmtCtx;
-          mlir::Value maskCond = createFIRExpr(loc, info.maskExpr, stmtCtx);
-          stmtCtx.finalizeAndReset();
-          mlir::Value maskCondCast =
-              builder->createConvert(loc, builder->getI1Type(), maskCond);
-          auto ifOp = builder->create<fir::IfOp>(loc, maskCondCast,
-                                                 /*withElseRegion=*/false);
-          builder->setInsertionPointToStart(&ifOp.getThenRegion().front());
-        }
-      }
-
-      IncrementLoopInfo &innermostInfo = incrementLoopNestInfo.back();
-
-      if (innermostInfo.hasLocalitySpecs())
-        handleLocalitySpecs(innermostInfo);
-
-      addLoopAnnotationAttr(innermostInfo, dirs);
-    }
   }
 
   /// Generate FIR to end a structured or unstructured increment loop nest.
@@ -2512,31 +2470,29 @@ class FirConverter : public Fortran::lower::AbstractConverter {
          it != rend; ++it) {
       IncrementLoopInfo &info = *it;
       if (info.isStructured()) {
-        // End fir.do_concurent.loop.
-        if (info.isConcurrent) {
-          builder->setInsertionPointAfter(info.loopOp->getParentOp());
+        // End fir.do_loop.
+        if (info.isUnordered) {
+          builder->setInsertionPointAfter(info.doLoop);
           continue;
         }
-
-        // End fir.do_loop.
         // Decrement tripVariable.
-        auto doLoopOp = mlir::cast<fir::DoLoopOp>(info.loopOp);
-        builder->setInsertionPointToEnd(doLoopOp.getBody());
+        builder->setInsertionPointToEnd(info.doLoop.getBody());
         llvm::SmallVector<mlir::Value, 2> results;
         results.push_back(builder->create<mlir::arith::AddIOp>(
-            loc, doLoopOp.getInductionVar(), doLoopOp.getStep(), iofAttr));
+            loc, info.doLoop.getInductionVar(), info.doLoop.getStep(),
+            iofAttr));
         // Step loopVariable to help optimizations such as vectorization.
         // Induction variable elimination will clean up as necessary.
         mlir::Value step = builder->createConvert(
-            loc, info.getLoopVariableType(), doLoopOp.getStep());
+            loc, info.getLoopVariableType(), info.doLoop.getStep());
         mlir::Value loopVar =
             builder->create<fir::LoadOp>(loc, info.loopVariable);
         results.push_back(
             builder->create<mlir::arith::AddIOp>(loc, loopVar, step, iofAttr));
         builder->create<fir::ResultOp>(loc, results);
-        builder->setInsertionPointAfter(doLoopOp);
+        builder->setInsertionPointAfter(info.doLoop);
         // The loop control variable may be used after the loop.
-        builder->create<fir::StoreOp>(loc, doLoopOp.getResult(1),
+        builder->create<fir::StoreOp>(loc, info.doLoop.getResult(1),
                                       info.loopVariable);
         continue;
       }
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index d35367d7657cf..3cf9b5ae72d9e 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -280,9 +280,6 @@ mlir::Block *fir::FirOpBuilder::getAllocaBlock() {
   if (auto cufKernelOp = getRegion().getParentOfType<cuf::KernelOp>())
     return &cufKernelOp.getRegion().front();
 
-  if (auto doConcurentOp = getRegion().getParentOfType<fir::DoConcurrentOp>())
-    return doConcurentOp.getBody();
-
   return getEntryBlock();
 }
 
diff --git a/flang/test/Lower/do_concurrent.f90 b/flang/test/Lower/do_concurrent.f90
index cc113f59c35e3..ef93d2d6b035b 100644
--- a/flang/test/Lower/do_concurrent.f90
+++ b/flang/test/Lower/do_concurrent.f90
@@ -14,9 +14,6 @@ subroutine sub1(n)
    implicit none
    integer :: n, m, i, j, k
    integer, dimension(n) :: a
-!CHECK: %[[N_DECL:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{.*}} {uniq_name = "_QFsub1En"}
-!CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub1Ea"}
-
 !CHECK: %[[LB1:.*]] = arith.constant 1 : i32
 !CHECK: %[[LB1_CVT:.*]] = fir.convert %[[LB1]] : (i32) -> index
 !CHECK: %[[UB1:.*]] = fir.load %{{.*}}#0 : !fir.ref<i32>
@@ -32,30 +29,10 @@ subroutine sub1(n)
 !CHECK: %[[UB3:.*]] = arith.constant 10 : i32
 !CHECK: %[[UB3_CVT:.*]] = fir.convert %[[UB3]] : (i32) -> index
 
-!CHECK: fir.do_concurrent
-!CHECK:   %[[I:.*]] = fir.alloca i32 {bindc_name = "i"}
-!CHECK:   %[[I_DECL:.*]]:2 = hlfir.declare %[[I]]
-!CHECK:   %[[J:.*]] = fir.alloca i32 {bindc_name = "j"}
-!CHECK:   %[[J_DECL:.*]]:2 = hlfir.declare %[[J]]
-!CHECK:   %[[K:.*]] = fir.alloca i32 {bindc_name = "k"}
-!CHECK:   %[[K_DECL:.*]]:2 = hlfir.declare %[[K]]
-
-!CHECK:   fir.do_concurrent.loop (%[[I_IV:.*]], %[[J_IV:.*]], %[[K_IV:.*]]) =
-!CHECK-SAME:                     (%[[LB1_CVT]], %[[LB2_CVT]], %[[LB3_CVT]]) to
-!CHECK-SAME:                     (%[[UB1_CVT]], %[[UB2_CVT]], %[[UB3_CVT]]) step
-!CHECK-SAME:                     (%{{.*}}, %{{.*}}, %{{.*}}) {
-!CHECK:       %[[I_IV_CVT:.*]] = fir.convert %[[I_IV]] : (index) -> i32
-!CHECK:       fir.store %[[I_IV_CVT]] to %[[I_DECL]]#0 : !fir.ref<i32>
-!CHECK:       %[[J_IV_CVT:.*]] = fir.convert %[[J_IV]] : (index) -> i32
-!CHECK:       fir.store %[[J_IV_CVT]] to %[[J_DECL]]#0 : !fir.ref<i32>
-!CHECK:       %[[K_IV_CVT:.*]] = fir.convert %[[K_IV]] : (index) -> i32
-!CHECK:       fir.store %[[K_IV_CVT]] to %[[K_DECL]]#0 : !fir.ref<i32>
+!CHECK: fir.do_loop %{{.*}} = %[[LB1_CVT]] to %[[UB1_CVT]] step %{{.*}} unordered
+!CHECK: fir.do_loop %{{.*}} = %[[LB2_CVT]] to %[[UB2_CVT]] step %{{.*}} unordered
+!CHECK: fir.do_loop %{{.*}} = %[[LB3_CVT]] to %[[UB3_CVT]] step %{{.*}} unordered
 
-!CHECK:       %[[N_VAL:.*]] = fir.load %[[N_DECL]]#0 : !fir.ref<i32>
-!CHECK:       %[[I_VAL:.*]] = fir.load %[[I_DECL]]#0 : !fir.ref<i32>
-!CHECK:       %[[I_VAL_CVT:.*]] = fir.convert %[[I_VAL]] : (i32) -> i64
-!CHECK:       %[[A_ELEM:.*]] = hlfir.designate %[[A_DECL]]#0 (%[[I_VAL_CVT]])
-!CHECK:       hlfir.assign %[[N_VAL]] to %[[A_ELEM]] : i32, !fir.ref<i32>
    do concurrent(i=1:n, j=1:bar(n*m, n/m), k=5:10)
       a(i) = n
    end do
@@ -68,17 +45,14 @@ subroutine sub2(n)
    integer, dimension(n) :: a
 !CHECK: %[[LB1:.*]] = arith.constant 1 : i32
 !CHECK: %[[LB1_CVT:.*]] = fir.convert %[[LB1]] : (i32) -> index
-!CHECK: %[[UB1:.*]] = fir.load %{{.*}}#0 : !fir.ref<i32>
+!CHECK: %[[UB1:.*]] = fir.load %5#0 : !fir.ref<i32>
 !CHECK: %[[UB1_CVT:.*]] = fir.convert %[[UB1]] : (i32) -> index
-!CHECK: fir.do_concurrent
-!CHECK:   fir.do_concurrent.loop (%{{.*}}) = (%[[LB1_CVT]]) to (%[[UB1_CVT]]) step (%{{.*}})
-
+!CHECK: fir.do_loop %{{.*}} = %[[LB1_CVT]] to %[[UB1_CVT]] step %{{.*}} unordered
 !CHECK: %[[LB2:.*]] = arith.constant 1 : i32
 !CHECK: %[[LB2_CVT:.*]] = fir.convert %[[LB2]] : (i32) -> index
 !CHECK: %[[UB2:.*]] = fir.call @_QPbar(%{{.*}}, %{{.*}}) proc_attrs<pure> fastmath<contract> : (!fir.ref<i32>, !fir.ref<i32>) -> i32
 !CHECK: %[[UB2_CVT:.*]] = fir.convert %[[UB2]] : (i32) -> index
-!CHECK: fir.do_concurrent
-!CHECK:   fir.do_concurrent.loop (%{{.*}}) = (%[[LB2_CVT]]) to (%[[UB2_CVT]]) step (%{{.*}})
+!CHECK: fir.do_loop %{{.*}} = %[[LB2_CVT]] to %[[UB2_CVT]] step %{{.*}} unordered
    do concurrent(i=1:n)
       do concurrent(j=1:bar(n*m, n/m))
          a(i) = n
@@ -86,6 +60,7 @@ subroutine sub2(n)
    end do
 end subroutine
 
+
 !CHECK-LABEL: unstructured
 subroutine unstructured(inner_step)
   integer(4) :: i, j, inner_step
diff --git a/flang/test/Lower/do_concurrent_local_default_init.f90 b/flang/test/Lower/do_concurrent_local_default_init.f90
index 207704ac1a990..7652e4fcd0402 100644
--- a/flang/test/Lower/do_concurrent_local_default_init.f90
+++ b/flang/test/Lower/do_concurrent_local_default_init.f90
@@ -29,7 +29,7 @@ subroutine test_default_init()
 ! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>> {fir.bindc_name = "p"}) {
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>
 ! CHECK:           %[[VAL_7:.*]] = fir.box_elesize %[[VAL_6]] : (!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>) -> index
-! CHECK:           fir.do_concurrent.loop
+! CHECK:           fir.do_loop
 ! CHECK:             %[[VAL_16:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>> {bindc_name = "p", pinned, uniq_name = "_QFtest_ptrEp"}
 ! CHECK:             %[[VAL_17:.*]] = fir.zero_bits !fir.ptr<!fir.array<?x!fir.char<1,?>>>
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 0 : index
@@ -43,7 +43,7 @@ subroutine test_default_init()
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QPtest_default_init(
-! CHECK:           fir.do_concurrent.loop
+! CHECK:           fir.do_loop
 ! CHECK:             %[[VAL_26:.*]] = fir.alloca !fir.type<_QFtest_default_initTt{i:i32}> {bindc_name = "a", pinned, uniq_name = "_QFtest_default_initEa"}
 ! CHECK:             %[[VAL_27:.*]] = fir.embox %[[VAL_26]] : (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>) -> !fir.box<!fir.type<_QFtest_default_initTt{i:i32}>>
 ! CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_27]] : (!fir.box<!fir.type<_QFtest_default_initTt{i:i32}>>) -> !fir.box<none>
diff --git a/flang/test/Lower/loops.f90 b/flang/test/Lower/loops.f90
index 60df27a591dc3..ea65ba3e4d66d 100644
--- a/flang/test/Lower/loops.f90
+++ b/flang/test/Lower/loops.f90
@@ -2,6 +2,15 @@
 
 ! CHECK-LABEL: loop_test
 subroutine loop_test
+  ! CHECK: %[[VAL_2:.*]] = fir.alloca i16 {bindc_name = "i"}
+  ! CHECK: %[[VAL_3:.*]] = fir.alloca i16 {bindc_name = "i"}
+  ! CHECK: %[[VAL_4:.*]] = fir.alloca i16 {bindc_name = "i"}
+  ! CHECK: %[[VAL_5:.*]] = fir.alloca i8 {bindc_name = "k"}
+  ! CHECK: %[[VAL_6:.*]] = fir.alloca i8 {bindc_name = "j"}
+  ! CHECK: %[[VAL_7:.*]] = fir.alloca i8 {bindc_name = "i"}
+  ! CHECK: %[[VAL_8:.*]] = fir.alloca i32 {bindc_name = "k"}
+  ! CHECK: %[[VAL_9:.*]] = fir.alloca i32 {bindc_name = "j"}
+  ! CHECK: %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i"}
   ! CHECK: %[[VAL_11:.*]] = fir.alloca !fir.array<5x5x5xi32> {bindc_name = "a", uniq_name = "_QFloop_testEa"}
   ! CHECK: %[[VAL_12:.*]] = fir.alloca i32 {bindc_name = "asum", uniq_name = "_QFloop_testEasum"}
   ! CHECK: %[[VAL_13:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFloop_testEi"}
@@ -16,7 +25,7 @@ subroutine loop_test
   j = 200
   k = 300
 
-  ! CHECK: fir.do_concurrent.loop (%{{.*}}, %{{.*}}, %{{.*}}) = {{.*}}
+  ! CHECK-COUNT-3: fir.do_loop {{.*}} unordered
   do concurrent (i=1:5, j=1:5, k=1:5) ! shared(a)
     ! CHECK: fir.coordinate_of
     a(i,j,k) = 0
@@ -24,7 +33,7 @@ subroutine loop_test
   ! CHECK: fir.call @_FortranAioBeginExternalListOutput
   print*, 'A:', i, j, k
 
-  ! CHECK: fir.do_concurrent.loop (%{{.*}}, %{{.*}}, %{{.*}}) = {{.*}}
+  ! CHECK-COUNT-3: fir.do_loop {{.*}} unordered
   ! CHECK: fir.if
   do concurrent (integer(1)::i=1:5, j=1:5, k=1:5, i.ne.j .and. k.ne.3) shared(a)
     ! CHECK-COUNT-2: fir.coordinate_of
@@ -44,7 +53,7 @@ subroutine loop_test
   ! CHECK: fir.call @_FortranAioBeginExternalListOutput
   print*, 'B:', i, j, k, '-', asum
 
-  ! CHECK: fir.do_concurrent.loop (%{{.*}}) = {{.*}}
+  ! CHECK: fir.do_loop {{.*}} unordered
   ! CHECK-COUNT-2: fir.if
   do concurrent (integer(2)::i=1:5, i.ne.3)
     if (i.eq.2 .or. i.eq.4) goto 5 ! fir.if
@@ -53,7 +62,7 @@ subroutine loop_test
   5 continue
   enddo
 
-  ! CHECK: fir.do_concurrent.loop (%{{.*}}) = {{.*}}
+  ! CHECK: fir.do_loop {{.*}} unordered
   ! CHECK-COUNT-2: fir.if
   do concurrent (integer(2)::i=1:5, i.ne.3)
     if (i.eq.2 .or. i.eq.4) then ! fir.if
@@ -84,6 +93,10 @@ end subroutine loop_test
 
 ! CHECK-LABEL: c.func @_QPlis
 subroutine lis(n)
+  ! CHECK-DAG: fir.alloca i32 {bindc_name = "m"}
+  ! CHECK-DAG: fir.alloca i32 {bindc_name = "j"}
+  ! CHECK-DAG: fir.alloca i32 {bindc_name = "i"}
+  ! CHECK-DAG: fir.alloca i8 {bindc_name = "i"}
   ! CHECK-DAG: fir.alloca i32 {bindc_name = "j", uniq_name = "_QFlisEj"}
   ! CHECK-DAG: fir.alloca i32 {bindc_name = "k", uniq_name = "_QFlisEk"}
   ! CHECK-DAG: fir.alloca !fir.box<!fir.ptr<!fir.array<?x?x?xi32>>> {bindc_name = "p", uniq_name = "_QFlisEp"}
@@ -104,8 +117,8 @@ subroutine lis(n)
   ! CHECK:     }
   r = 0
 
-  ! CHECK:     fir.do_concurrent {
-  ! CHECK:       fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
+  ! CHECK:     fir.do_loop %arg1 = %{{.*}} to %{{.*}} step %{{.*}} unordered {
+  ! CHECK:       fir.do_loop %arg2 = %{{.*}} to %{{.*}} step %c1{{.*}} iter_args(%arg3 = %{{.*}}) -> (index, i32) {
   ! CHECK:       }
   ! CHECK:     }
   do concurrent (integer(kind=1)::i=n:1:-1)
@@ -115,18 +128,16 @@ subroutine lis(n)
     enddo
   enddo
 
-  ! CHECK:       fir.do_concurrent.loop (%{{.*}}, %{{.*}}) = (%{{.*}}, %{{.*}}) to (%{{.*}}, %{{.*}}) step (%{{.*}}, %{{.*}}) {
+  ! CHECK:     fir.do_loop %arg1 = %{{.*}} to %{{.*}} step %c1{{.*}} unordered {
+  ! CHECK:       fir.do_loop %arg2 = %{{.*}} to %{{.*}} step %c1{{.*}} unordered {
   ! CHECK:         fir.if %{{.*}} {
   ! CHECK:           %[[V_95:[0-9]+]] = fir.alloca !fir.array<?x?xi32>, %{{.*}}, %{{.*}} {bindc_name = "t", pinned, uniq_name = "_QFlisEt"}
   ! CHECK:           %[[V_96:[0-9]+]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x?x?xi32>>> {bindc_name = "p", pinned, uniq_name = "_QFlisEp"}
   ! CHECK:           fir.store %{{.*}} to %[[V_96]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>
   ! CHECK:           fir.do_loop %arg3 = %{{.*}} to %{{.*}} step %c1{{.*}} iter_args(%arg4 = %{{.*}}) -> (index, i32) {
-  ! CHECK:             fir.do_concurrent {
-  ! CHECK:               fir.alloca i32 {bindc_name = "m"}
-  ! CHECK:               fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
-  ! CHECK:                 fir.load %[[V_96]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>
-  ! CHECK:                 fir.convert %[[V_95]] : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?xi32>>
-  ! CHECK:               }
+  ! CHECK:             fir.do_loop %arg5 = %{{.*}} to %{{.*}} step %c1{{.*}} unordered {
+  ! CHECK:               fir.load %[[V_96]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>
+  ! CHECK:               fir.convert %[[V_95]] : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?xi32>>
   ! CHECK:             }
   ! CHECK:           }
   ! CHECK:           fir.convert %[[V_95]] : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?xi32>>
diff --git a/flang/test/Lower/loops3.f90 b/flang/test/Lower/loops3.f90
index 84db1972cca16..78f39e1013082 100644
--- a/flang/test/Lower/loops3.f90
+++ b/flang/test/Lower/loops3.f90
@@ -12,7 +12,9 @@ subroutine loop_test
 
   ! CHECK: %[[VAL_0:.*]] = fir.alloca f32 {bindc_name = "m", uniq_name = "_QFloop_testEm"}
   ! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QFloop_testEsum) : !fir.ref<i32>
-  ! CHECK: fir.do_concurrent.loop ({{.*}}) = ({{.*}}) to ({{.*}}) step ({{.*}}) reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
+  ! CHECK: fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} unordered reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
+  ! CHECK: fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} unordered reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
+  ! CHECK: fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} unordered reduce(#fir.reduce_attr<add> -> %[[VAL_1:.*]] : !fir.ref<i32>, #fir.reduce_attr<max> -> %[[VAL_0:.*]] : !fir.ref<f32>) {
   do concurrent (i=1:5, j=1:5, k=1:5) local(tmp) reduce(+:sum) reduce(max:m)
     tmp = i + j + k
     sum = tmp + sum
diff --git a/flang/test/Lower/nsw.f90 b/flang/test/Lower/nsw.f90
index 2ec1efb2af42a..4ee9e5da829e6 100644
--- a/flang/test/Lower/nsw.f90
+++ b/flang/test/Lower/nsw.f90
@@ -139,6 +139,7 @@ subroutine loop_params3(a,lb,ub,st)
 ! CHECK-LABEL:   func.func @_QPloop_params3(
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_9:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
 ! CHECK:           %[[VAL_11:.*]] = fir.declare %{{.*}}lb"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
 ! CHECK:           %[[VAL_12:.*]] = fir.declare %{{.*}}ub"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
 ! CHECK:           %[[VAL_14:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
@@ -152,6 +153,4 @@ subroutine loop_params3(a,lb,ub,st)
 ! CHECK:           %[[VAL_31:.*]] = fir.load %[[VAL_15]] : !fir.ref<i32>
 ! CHECK:           %[[VAL_32:.*]] = arith.muli %[[VAL_31]], %[[VAL_4]] overflow<nsw> : i32
 ! CHECK:           %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (i32) -> index
-! CHECK:           fir.do_concurrent {
-! CHECK:             %[[VAL_9:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>) -> !fir.ref<i32>
-! CHECK:             fir.do_concurrent.loop (%[[VAL_34:.*]]) = (%[[VAL_28]]) to (%[[VAL_30]]) step (%[[VAL_33]]) {
+! CHECK:           fir.do_loop %[[VAL_34:.*]] = %[[VAL_28]] to %[[VAL_30]] step %[[VAL_33]] unordered {
diff --git a/flang/test/Transforms/DoConcurrent/basic_host.f90 b/flang/test/Transforms/DoConcurrent/basic_host.f90
index b84d4481ac766..12f63031cbaee 100644
--- a/flang/test/Transforms/DoConcurrent/basic_host.f90
+++ b/flang/test/Transforms/DoConcurrent/basic_host.f90
@@ -1,6 +1,3 @@
-! Fails until we update the pass to use the `fir.do_concurrent` op.
-! XFAIL: *
-
 ! Tests mapping of a basic `do concurrent` loop to `!$omp parallel do`.
 
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -fdo-concurrent-to-openmp=host %s -o - \
diff --git a/flang/test/Transforms/DoConcurrent/locally_destroyed_temp.f90 b/flang/test/Transforms/DoConcurrent/locally_destroyed_temp.f90
index 4e13c0919589a..f82696669eca6 100644
--- a/flang/test/Transforms/DoConcurrent/locally_destroyed_temp.f90
+++ b/flang/test/Transforms/DoConcurrent/locally_destroyed_temp.f90
@@ -1,6 +1,3 @@
-! Fails until we update the pass to use the `fir.do_concurrent` op.
-! XFAIL: *
-
 ! Tests that "loop-local values" are properly handled by localizing them to the
 ! body of the loop nest. See `collectLoopLocalValues` and `localizeLoopLocalValue`
 ! for a definition of "loop-local values" and how they are handled.
diff --git a/flang/test/Transforms/DoConcurrent/loop_nest_test.f90 b/flang/test/Transforms/DoConcurrent/loop_nest_test.f90
index adc4a488d1ec9..32bed61fe69e4 100644
--- a/flang/test/Transforms/DoConcurrent/loop_nest_test.f90
+++ b/flang/test/Transforms/DoConcurrent/loop_nest_test.f90
@@ -1,6 +1,3 @@
-! Fails until we update the pass to use the `fir.do_concurrent` op.
-! XFAIL: *
-
 ! Tests loop-nest detection algorithm for do-concurrent mapping.
 
 ! REQUIRES: asserts
diff --git a/flang/test/Transforms/DoConcurrent/multiple_iteration_ranges.f90 b/flang/test/Transforms/DoConcurrent/multiple_iteration_ranges.f90
index 26800678d381c..d0210726de83e 100644
--- a/flang/test/Transforms/DoConcurrent/multiple_iteration_ranges.f90
+++ b/flang/test/Transforms/DoConcurrent/multiple_iteration_ranges.f90
@@ -1,6 +1,3 @@
-! Fails until we update the pass to use the `fir.do_concurrent` op.
-! XFAIL: *
-
 ! Tests mapping of a `do concurrent` loop with multiple iteration ranges.
 
 ! RUN: split-file %s %t
diff --git a/flang/test/Transforms/DoConcurrent/non_const_bounds.f90 b/flang/test/Transforms/DoConcurrent/non_const_bounds.f90
index 23a3aae976c07..cd1bd4f98a3f5 100644
--- a/flang/test/Transforms/DoConcurrent/non_const_bounds.f90
+++ b/flang/test/Transforms/DoConcurrent/non_const_bounds.f90
@@ -1,6 +1,3 @@
-! Fails until we update the pass to use the `fir.do_concurrent` op.
-! XFAIL: *
-
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -fdo-concurrent-to-openmp=host %s -o - \
 ! RUN:   | FileCheck %s
 
diff --git a/flang/test/Transforms/DoConcurrent/not_perfectly_nested.f90 b/flang/test/Transforms/DoConcurrent/not_perfectly_nested.f90
index d1c02101318ab..184fdfe00d397 100644
--- a/flang/test/Transforms/DoConcurrent/not_perfectly_nested.f90
+++ b/flang/test/Transforms/DoConcurrent/not_perfectly_nested.f90
@@ -1,6 +1,3 @@
-! Fails until we update the pass to use the `fir.do_concurrent` op.
-! XFAIL: *
-
 ! Tests that if `do concurrent` is not perfectly nested in its parent loop, that
 ! we skip converting the not-perfectly nested `do concurrent` loop.
 

From fe4a31d59db7b18dc45c3593bf100c101e725b79 Mon Sep 17 00:00:00 2001
From: Benjamin Chetioui <3920784+bchetioui@users.noreply.github.com>
Date: Wed, 16 Apr 2025 14:29:44 +0200
Subject: [PATCH 112/710] [bazel] Fix bazel build after
 00eaff3e9c897c263a879416d0f151d7ca7eeaff. (#135949)

---
 .../IR/BufferizationTypeInterfaces.h          |  1 +
 .../llvm-project-overlay/mlir/BUILD.bazel     | 24 +++++++++++++++++++
 .../mlir/test/BUILD.bazel                     |  7 ++++++
 3 files changed, 32 insertions(+)

diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
index f6b296eccd748..8672aa60a43c5 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
@@ -14,5 +14,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h.inc"
+#include "mlir/IR/Types.h"
 
 #endif // MLIR_DIALECT_BUFFERIZATION_IR_BUFFERIZATIONTYPEINTERFACES_H_
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 10503fe1d123b..85049ff8339c1 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -12520,6 +12520,28 @@ gentbl_cc_library(
     deps = [":BufferizationEnumsTdFiles"],
 )
 
+td_library(
+    name = "BufferizationTypeInterfacesTdFiles",
+    srcs = [
+        "include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.td",
+    ],
+    includes = ["include"],
+    deps = [
+        ":OpBaseTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "BufferizationTypeInterfacesIncGen",
+    tbl_outs = {
+        "include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h.inc": ["-gen-type-interface-decls"],
+        "include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.cpp.inc": ["-gen-type-interface-defs"],
+    },
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.td",
+    deps = [":BufferizationTypeInterfacesTdFiles"],
+)
+
 td_library(
     name = "BufferizationTransformOpsTdFiles",
     srcs = [
@@ -12593,6 +12615,7 @@ cc_library(
         "include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h",
         "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h",
         "include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h",
+        "include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h",
     ],
     includes = ["include"],
     deps = [
@@ -12600,6 +12623,7 @@ cc_library(
         ":BufferViewFlowOpInterfaceIncGen",
         ":BufferizableOpInterfaceIncGen",
         ":BufferizationEnumsIncGen",
+        ":BufferizationTypeInterfacesIncGen",
         ":FunctionInterfaces",
         ":IR",
         ":Support",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 611fbecd215cd..59410286f22d8 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -110,6 +110,7 @@ td_library(
     name = "TestOpTdFiles",
     srcs = glob(["lib/Dialect/Test/*.td"]),
     deps = [
+        "//mlir:BufferizationTypeInterfacesTdFiles",
         "//mlir:BuiltinDialectTdFiles",
         "//mlir:CallInterfacesTdFiles",
         "//mlir:ControlFlowInterfacesTdFiles",
@@ -240,6 +241,7 @@ gentbl_cc_library(
     test = True,
     deps = [
         ":TestOpTdFiles",
+        "//mlir:BufferizationTypeInterfacesTdFiles",
         "//mlir:BuiltinDialectTdFiles",
     ],
 )
@@ -342,6 +344,7 @@ cc_library(
         "//llvm:IRReader",
         "//llvm:Support",
         "//mlir:ArithDialect",
+        "//mlir:BufferizationInterfaces",
         "//mlir:BytecodeOpInterface",
         "//mlir:CallOpInterfaces",
         "//mlir:ControlFlowInterfaces",
@@ -404,6 +407,7 @@ cc_library(
         "//llvm:Support",
         "//mlir:Analysis",
         "//mlir:ArithDialect",
+        "//mlir:BufferizationInterfaces",
         "//mlir:BytecodeOpInterface",
         "//mlir:BytecodeReader",
         "//mlir:BytecodeWriter",
@@ -986,8 +990,11 @@ cc_library(
     srcs = glob(["lib/Dialect/Bufferization/*.cpp"]),
     includes = ["lib/Dialect/Test"],
     deps = [
+        ":TestDialect",
         "//mlir:BufferizationDialect",
+        "//mlir:BufferizationInterfaces",
         "//mlir:BufferizationTransforms",
+        "//mlir:FuncDialect",
         "//mlir:IR",
         "//mlir:Pass",
     ],

From d3153ad66c539ad146062b6e65741901e5b5e1cc Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777@gmail.com>
Date: Wed, 16 Apr 2025 20:53:25 +0800
Subject: [PATCH 113/710] [clang] Unify `SourceLocation` and `IdentifierInfo*`
 pair-like data structures to `IdentifierLoc` (#135808)

I found this issue when I working on
https://github.com/llvm/llvm-project/pull/107168.

Currently we have many similiar data structures like:
 - `std::pair<IdentifierInfo *, SourceLocation>`.
 - Element type of `ModuleIdPath`.
 - `IdentifierLocPair`.
 - `IdentifierLoc`.

This PR unify these data structures to `IdentifierLoc`, moved
`IdentifierLoc` definition to SourceLocation.h, and deleted other
similer data structures.

---------

Signed-off-by: yronglin <yronglin777@gmail.com>
---
 .../pp-trace/PPCallbacksTracker.cpp           |   4 +-
 clang/include/clang/AST/OpenACCClause.h       |  20 +--
 clang/include/clang/Basic/IdentifierTable.h   |  26 +++-
 clang/include/clang/Lex/ModuleLoader.h        |   3 +-
 clang/include/clang/Lex/PPCallbacks.h         |   1 +
 clang/include/clang/Lex/Preprocessor.h        |   9 +-
 clang/include/clang/Parse/LoopHint.h          |   2 +-
 clang/include/clang/Parse/Parser.h            |  13 +-
 clang/include/clang/Sema/ParsedAttr.h         |  10 --
 clang/include/clang/Sema/Sema.h               |   2 +-
 clang/include/clang/Sema/SemaCodeCompletion.h |   3 +-
 clang/include/clang/Sema/SemaObjC.h           |   4 +-
 clang/include/clang/Sema/SemaOpenACC.h        |   2 +-
 clang/lib/AST/OpenACCClause.cpp               |   4 +-
 clang/lib/AST/TextNodeDumper.cpp              |   4 +-
 clang/lib/Frontend/CompilerInstance.cpp       |  53 ++++----
 clang/lib/Frontend/FrontendActions.cpp        |   4 +-
 clang/lib/Lex/PPDirectives.cpp                |  22 ++--
 clang/lib/Lex/PPLexerChange.cpp               |   6 +-
 clang/lib/Lex/Pragma.cpp                      |  73 +++++------
 clang/lib/Lex/Preprocessor.cpp                |  16 +--
 clang/lib/Parse/ParseDecl.cpp                 |  28 ++--
 clang/lib/Parse/ParseExpr.cpp                 |   7 +-
 clang/lib/Parse/ParseHLSL.cpp                 |   2 +-
 clang/lib/Parse/ParseObjc.cpp                 |  38 +++---
 clang/lib/Parse/ParseOpenACC.cpp              |  12 +-
 clang/lib/Parse/ParsePragma.cpp               |  15 +--
 clang/lib/Parse/ParseStmt.cpp                 |   6 +-
 clang/lib/Parse/Parser.cpp                    |  19 ++-
 clang/lib/Sema/ParsedAttr.cpp                 |   8 --
 clang/lib/Sema/SemaARM.cpp                    |   2 +-
 clang/lib/Sema/SemaCodeComplete.cpp           |   8 +-
 clang/lib/Sema/SemaDeclAttr.cpp               | 124 +++++++++---------
 clang/lib/Sema/SemaDeclObjC.cpp               |  35 ++---
 clang/lib/Sema/SemaHLSL.cpp                   |  12 +-
 clang/lib/Sema/SemaModule.cpp                 |  42 +++---
 clang/lib/Sema/SemaObjC.cpp                   |  45 +++----
 clang/lib/Sema/SemaOpenACCClause.cpp          |  11 +-
 clang/lib/Sema/SemaStmtAttr.cpp               |  29 ++--
 clang/lib/Sema/SemaSwift.cpp                  |  24 ++--
 clang/lib/Sema/SemaTemplateVariadic.cpp       |  10 +-
 clang/lib/Sema/SemaType.cpp                   |  13 +-
 clang/lib/Serialization/ASTReader.cpp         |   2 +-
 clang/lib/Serialization/ASTWriter.cpp         |   8 +-
 .../DependencyScanning/ModuleDepCollector.cpp |   2 +-
 45 files changed, 399 insertions(+), 384 deletions(-)

diff --git a/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp b/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
index 3bb30fd15b2e1..4c916fa30685b 100644
--- a/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
+++ b/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
@@ -547,8 +547,8 @@ void PPCallbacksTracker::appendArgument(const char *Name, ModuleIdPath Value) {
     if (I)
       SS << ", ";
     SS << "{"
-       << "Name: " << Value[I].first->getName() << ", "
-       << "Loc: " << getSourceLocationString(PP, Value[I].second) << "}";
+       << "Name: " << Value[I].getIdentifierInfo()->getName() << ", "
+       << "Loc: " << getSourceLocationString(PP, Value[I].getLoc()) << "}";
   }
   SS << "]";
   appendArgument(Name, SS.str());
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 681567228cbb0..f18a6cf62f2c5 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -258,7 +258,7 @@ inline bool operator!=(const OpenACCBindClause &LHS,
   return !(LHS == RHS);
 }
 
-using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
+using DeviceTypeArgument = IdentifierLoc;
 /// A 'device_type' or 'dtype' clause, takes a list of either an 'asterisk' or
 /// an identifier. The 'asterisk' means 'the rest'.
 class OpenACCDeviceTypeClause final
@@ -280,16 +280,16 @@ class OpenACCDeviceTypeClause final
         "Invalid clause kind for device-type");
 
     assert(!llvm::any_of(Archs, [](const DeviceTypeArgument &Arg) {
-      return Arg.second.isInvalid();
+      return Arg.getLoc().isInvalid();
     }) && "Invalid SourceLocation for an argument");
 
-    assert(
-        (Archs.size() == 1 || !llvm::any_of(Archs,
-                                            [](const DeviceTypeArgument &Arg) {
-                                              return Arg.first == nullptr;
-                                            })) &&
-        "Only a single asterisk version is permitted, and must be the "
-        "only one");
+    assert((Archs.size() == 1 ||
+            !llvm::any_of(Archs,
+                          [](const DeviceTypeArgument &Arg) {
+                            return Arg.getIdentifierInfo() == nullptr;
+                          })) &&
+           "Only a single asterisk version is permitted, and must be the "
+           "only one");
 
     std::uninitialized_copy(Archs.begin(), Archs.end(),
                             getTrailingObjects<DeviceTypeArgument>());
@@ -302,7 +302,7 @@ class OpenACCDeviceTypeClause final
   }
   bool hasAsterisk() const {
     return getArchitectures().size() > 0 &&
-           getArchitectures()[0].first == nullptr;
+           getArchitectures()[0].getIdentifierInfo() == nullptr;
   }
 
   ArrayRef<DeviceTypeArgument> getArchitectures() const {
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 0347880244a40..1275b056227b5 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -18,6 +18,7 @@
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/DiagnosticIDs.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/FoldingSet.h"
@@ -76,9 +77,6 @@ inline bool isReservedInAllContexts(ReservedIdentifierStatus Status) {
          Status != ReservedIdentifierStatus::StartsWithUnderscoreAndIsExternC;
 }
 
-/// A simple pair of identifier info and location.
-using IdentifierLocPair = std::pair<IdentifierInfo *, SourceLocation>;
-
 /// IdentifierInfo and other related classes are aligned to
 /// 8 bytes so that DeclarationName can use the lower 3 bits
 /// of a pointer to one of these classes.
@@ -1165,6 +1163,28 @@ class SelectorTable {
   static std::string getPropertyNameFromSetterSelector(Selector Sel);
 };
 
+/// A simple pair of identifier info and location.
+class IdentifierLoc {
+  SourceLocation Loc;
+  IdentifierInfo *II = nullptr;
+
+public:
+  IdentifierLoc() = default;
+  IdentifierLoc(SourceLocation L, IdentifierInfo *Ident) : Loc(L), II(Ident) {}
+
+  void setLoc(SourceLocation L) { Loc = L; }
+  void setIdentifierInfo(IdentifierInfo *Ident) { II = Ident; }
+  SourceLocation getLoc() const { return Loc; }
+  IdentifierInfo *getIdentifierInfo() const { return II; }
+
+  bool operator==(const IdentifierLoc &X) const {
+    return Loc == X.Loc && II == X.II;
+  }
+
+  bool operator!=(const IdentifierLoc &X) const {
+    return Loc != X.Loc || II != X.II;
+  }
+};
 }  // namespace clang
 
 namespace llvm {
diff --git a/clang/include/clang/Lex/ModuleLoader.h b/clang/include/clang/Lex/ModuleLoader.h
index f880a9091a2ed..a58407200c41c 100644
--- a/clang/include/clang/Lex/ModuleLoader.h
+++ b/clang/include/clang/Lex/ModuleLoader.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_LEX_MODULELOADER_H
 #define LLVM_CLANG_LEX_MODULELOADER_H
 
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceLocation.h"
@@ -29,7 +30,7 @@ class IdentifierInfo;
 
 /// A sequence of identifier/location pairs used to describe a particular
 /// module or submodule, e.g., std.vector.
-using ModuleIdPath = ArrayRef<std::pair<IdentifierInfo *, SourceLocation>>;
+using ModuleIdPath = ArrayRef<IdentifierLoc>;
 
 /// Describes the result of attempting to load a module.
 class ModuleLoadResult {
diff --git a/clang/include/clang/Lex/PPCallbacks.h b/clang/include/clang/Lex/PPCallbacks.h
index 46cc564086f1c..313b730afbab8 100644
--- a/clang/include/clang/Lex/PPCallbacks.h
+++ b/clang/include/clang/Lex/PPCallbacks.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_LEX_PPCALLBACKS_H
 
 #include "clang/Basic/DiagnosticIDs.h"
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/ModuleLoader.h"
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 24bb524783e93..f8f2f567f9171 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -327,7 +327,7 @@ class Preprocessor {
   SourceLocation ModuleImportLoc;
 
   /// The import path for named module that we're currently processing.
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> NamedModuleImportPath;
+  SmallVector<IdentifierLoc, 2> NamedModuleImportPath;
 
   llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
   unsigned CheckPointCounter = 0;
@@ -622,7 +622,7 @@ class Preprocessor {
 
   /// The identifier and source location of the currently-active
   /// \#pragma clang arc_cf_code_audited begin.
-  std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
+  IdentifierLoc PragmaARCCFCodeAuditedInfo;
 
   /// The source location of the currently-active
   /// \#pragma clang assume_nonnull begin.
@@ -1998,8 +1998,7 @@ class Preprocessor {
   /// arc_cf_code_audited begin.
   ///
   /// Returns an invalid location if there is no such pragma active.
-  std::pair<IdentifierInfo *, SourceLocation>
-  getPragmaARCCFCodeAuditedInfo() const {
+  IdentifierLoc getPragmaARCCFCodeAuditedInfo() const {
     return PragmaARCCFCodeAuditedInfo;
   }
 
@@ -2007,7 +2006,7 @@ class Preprocessor {
   /// arc_cf_code_audited begin.  An invalid location ends the pragma.
   void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident,
                                      SourceLocation Loc) {
-    PragmaARCCFCodeAuditedInfo = {Ident, Loc};
+    PragmaARCCFCodeAuditedInfo = IdentifierLoc(Loc, Ident);
   }
 
   /// The location of the currently-active \#pragma clang
diff --git a/clang/include/clang/Parse/LoopHint.h b/clang/include/clang/Parse/LoopHint.h
index cec5605ea3615..72be043d3c5a4 100644
--- a/clang/include/clang/Parse/LoopHint.h
+++ b/clang/include/clang/Parse/LoopHint.h
@@ -9,12 +9,12 @@
 #ifndef LLVM_CLANG_PARSE_LOOPHINT_H
 #define LLVM_CLANG_PARSE_LOOPHINT_H
 
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 
 namespace clang {
 
 class Expr;
-struct IdentifierLoc;
 
 /// Loop optimization hint for loop and unroll pragmas.
 struct LoopHint {
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 9ebcf144ba59e..662f54d0e8d8a 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -1725,8 +1725,8 @@ class Parser : public CodeCompletionHandler {
   ObjCTypeParamList *parseObjCTypeParamList();
   ObjCTypeParamList *parseObjCTypeParamListOrProtocolRefs(
       ObjCTypeParamListScope &Scope, SourceLocation &lAngleLoc,
-      SmallVectorImpl<IdentifierLocPair> &protocolIdents,
-      SourceLocation &rAngleLoc, bool mayBeProtocolList = true);
+      SmallVectorImpl<IdentifierLoc> &protocolIdents, SourceLocation &rAngleLoc,
+      bool mayBeProtocolList = true);
 
   void HelperActionsForIvarDeclarations(ObjCContainerDecl *interfaceDecl,
                                         SourceLocation atLoc,
@@ -3818,8 +3818,7 @@ class Parser : public CodeCompletionHandler {
                                SourceLocation Loc,
                                llvm::SmallVectorImpl<Expr *> &IntExprs);
   /// Parses the 'device-type-list', which is a list of identifiers.
-  bool ParseOpenACCDeviceTypeList(
-      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs);
+  bool ParseOpenACCDeviceTypeList(llvm::SmallVector<IdentifierLoc> &Archs);
   /// Parses the 'async-argument', which is an integral value with two
   /// 'special' values that are likely negative (but come from Macros).
   OpenACCIntExprParseResult ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK,
@@ -3951,10 +3950,8 @@ class Parser : public CodeCompletionHandler {
     return false;
   }
 
-  bool ParseModuleName(
-      SourceLocation UseLoc,
-      SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
-      bool IsImport);
+  bool ParseModuleName(SourceLocation UseLoc,
+                       SmallVectorImpl<IdentifierLoc> &Path, bool IsImport);
 
   //===--------------------------------------------------------------------===//
   // C++11/G++: Type Traits [Type-Traits.html in the GCC manual]
diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h
index b88b871dc8821..428d3111de80d 100644
--- a/clang/include/clang/Sema/ParsedAttr.h
+++ b/clang/include/clang/Sema/ParsedAttr.h
@@ -40,7 +40,6 @@ class LangOptions;
 class Sema;
 class Stmt;
 class TargetInfo;
-struct IdentifierLoc;
 
 /// Represents information about a change in availability for
 /// an entity, which is part of the encoding of the 'availability'
@@ -99,15 +98,6 @@ struct PropertyData {
 
 } // namespace detail
 
-/// Wraps an identifier and optional source location for the identifier.
-struct IdentifierLoc {
-  SourceLocation Loc;
-  IdentifierInfo *Ident;
-
-  static IdentifierLoc *create(ASTContext &Ctx, SourceLocation Loc,
-                               IdentifierInfo *Ident);
-};
-
 /// A union of the various pointer types that can be passed to an
 /// ParsedAttr as an argument.
 using ArgsUnion = llvm::PointerUnion<Expr *, IdentifierLoc *>;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index fe37fd7701ce3..1f23b754a69cb 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -143,7 +143,7 @@ enum class LangAS : unsigned int;
 class LocalInstantiationScope;
 class LookupResult;
 class MangleNumberingContext;
-typedef ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> ModuleIdPath;
+typedef ArrayRef<IdentifierLoc> ModuleIdPath;
 class ModuleLoader;
 class MultiLevelTemplateArgumentList;
 struct NormalizedConstraint;
diff --git a/clang/include/clang/Sema/SemaCodeCompletion.h b/clang/include/clang/Sema/SemaCodeCompletion.h
index 72159de3a6e72..3029e56e5cfe2 100644
--- a/clang/include/clang/Sema/SemaCodeCompletion.h
+++ b/clang/include/clang/Sema/SemaCodeCompletion.h
@@ -193,8 +193,7 @@ class SemaCodeCompletion : public SemaBase {
   void CodeCompleteObjCForCollection(Scope *S, DeclGroupPtrTy IterationVar);
   void CodeCompleteObjCSelector(Scope *S,
                                 ArrayRef<const IdentifierInfo *> SelIdents);
-  void
-  CodeCompleteObjCProtocolReferences(ArrayRef<IdentifierLocPair> Protocols);
+  void CodeCompleteObjCProtocolReferences(ArrayRef<IdentifierLoc> Protocols);
   void CodeCompleteObjCProtocolDecl(Scope *S);
   void CodeCompleteObjCInterfaceDecl(Scope *S);
   void CodeCompleteObjCClassForwardDecl(Scope *S);
diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
index 791a7f45b832f..4cda41a82b61f 100644
--- a/clang/include/clang/Sema/SemaObjC.h
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -307,11 +307,11 @@ class SemaObjC : public SemaBase {
 
   DeclGroupPtrTy
   ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc,
-                                  ArrayRef<IdentifierLocPair> IdentList,
+                                  ArrayRef<IdentifierLoc> IdentList,
                                   const ParsedAttributesView &attrList);
 
   void FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer,
-                               ArrayRef<IdentifierLocPair> ProtocolId,
+                               ArrayRef<IdentifierLoc> ProtocolId,
                                SmallVectorImpl<Decl *> &Protocols);
 
   void DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId,
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 4c3a13a3b044f..8d31d46444c7e 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -212,7 +212,7 @@ class SemaOpenACC : public SemaBase {
   } LoopWithoutSeqInfo;
 
   // Redeclaration of the version in OpenACCClause.h.
-  using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
+  using DeviceTypeArgument = IdentifierLoc;
 
   /// A type to represent all the data for an OpenACC Clause that has been
   /// parsed, but not yet created/semantically analyzed. This is effectively a
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index d7cbb51335359..2820d7b288658 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -891,10 +891,10 @@ void OpenACCClausePrinter::VisitDeviceTypeClause(
   OS << "(";
   llvm::interleaveComma(C.getArchitectures(), OS,
                         [&](const DeviceTypeArgument &Arch) {
-                          if (Arch.first == nullptr)
+                          if (Arch.getIdentifierInfo() == nullptr)
                             OS << "*";
                           else
-                            OS << Arch.first->getName();
+                            OS << Arch.getIdentifierInfo()->getName();
                         });
   OS << ")";
 }
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index c8b459ee78e6b..1bd94a3ac6431 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -500,10 +500,10 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       llvm::interleaveComma(
           cast<OpenACCDeviceTypeClause>(C)->getArchitectures(), OS,
           [&](const DeviceTypeArgument &Arch) {
-            if (Arch.first == nullptr)
+            if (Arch.getIdentifierInfo() == nullptr)
               OS << "*";
             else
-              OS << Arch.first->getName();
+              OS << Arch.getIdentifierInfo()->getName();
           });
       OS << ")";
       break;
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 243e0a3c15b05..93e4e31c2891d 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -35,6 +35,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/GlobalModuleIndex.h"
@@ -2009,8 +2010,8 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
                              Module::NameVisibilityKind Visibility,
                              bool IsInclusionDirective) {
   // Determine what file we're searching from.
-  StringRef ModuleName = Path[0].first->getName();
-  SourceLocation ModuleNameLoc = Path[0].second;
+  StringRef ModuleName = Path[0].getIdentifierInfo()->getName();
+  SourceLocation ModuleNameLoc = Path[0].getLoc();
 
   // If we've already handled this import, just return the cached result.
   // This one-element cache is important to eliminate redundant diagnostics
@@ -2026,7 +2027,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
   // If we don't already have information on this module, load the module now.
   Module *Module = nullptr;
   ModuleMap &MM = getPreprocessor().getHeaderSearchInfo().getModuleMap();
-  if (auto MaybeModule = MM.getCachedModuleLoad(*Path[0].first)) {
+  if (auto MaybeModule = MM.getCachedModuleLoad(*Path[0].getIdentifierInfo())) {
     // Use the cached result, which may be nullptr.
     Module = *MaybeModule;
     // Config macros are already checked before building a module, but they need
@@ -2046,7 +2047,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     // * `Preprocessor::HandleHeaderIncludeOrImport` will never call this
     //   function as the `#include` or `#import` is textual.
 
-    MM.cacheModuleLoad(*Path[0].first, Module);
+    MM.cacheModuleLoad(*Path[0].getIdentifierInfo(), Module);
   } else {
     ModuleLoadResult Result = findOrCompileModuleAndReadAST(
         ModuleName, ImportLoc, ModuleNameLoc, IsInclusionDirective);
@@ -2055,7 +2056,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (!Result)
       DisableGeneratingGlobalModuleIndex = true;
     Module = Result;
-    MM.cacheModuleLoad(*Path[0].first, Module);
+    MM.cacheModuleLoad(*Path[0].getIdentifierInfo(), Module);
   }
 
   // If we never found the module, fail.  Otherwise, verify the module and link
@@ -2067,7 +2068,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
   // a submodule.
   bool MapPrivateSubModToTopLevel = false;
   for (unsigned I = 1, N = Path.size(); I != N; ++I) {
-    StringRef Name = Path[I].first->getName();
+    StringRef Name = Path[I].getIdentifierInfo()->getName();
     clang::Module *Sub = Module->findSubmodule(Name);
 
     // If the user is requesting Foo.Private and it doesn't exist, try to
@@ -2078,10 +2079,10 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
       SmallString<128> PrivateModule(Module->Name);
       PrivateModule.append("_Private");
 
-      SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> PrivPath;
+      SmallVector<IdentifierLoc, 2> PrivPath;
       auto &II = PP->getIdentifierTable().get(
           PrivateModule, PP->getIdentifierInfo(Module->Name)->getTokenID());
-      PrivPath.push_back(std::make_pair(&II, Path[0].second));
+      PrivPath.emplace_back(Path[0].getLoc(), &II);
 
       std::string FileName;
       // If there is a modulemap module or prebuilt module, load it.
@@ -2095,11 +2096,12 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
         PP->markClangModuleAsAffecting(Module);
         if (!getDiagnostics().isIgnored(
                 diag::warn_no_priv_submodule_use_toplevel, ImportLoc)) {
-          getDiagnostics().Report(Path[I].second,
+          getDiagnostics().Report(Path[I].getLoc(),
                                   diag::warn_no_priv_submodule_use_toplevel)
-              << Path[I].first << Module->getFullModuleName() << PrivateModule
-              << SourceRange(Path[0].second, Path[I].second)
-              << FixItHint::CreateReplacement(SourceRange(Path[0].second),
+              << Path[I].getIdentifierInfo() << Module->getFullModuleName()
+              << PrivateModule
+              << SourceRange(Path[0].getLoc(), Path[I].getLoc())
+              << FixItHint::CreateReplacement(SourceRange(Path[0].getLoc()),
                                               PrivateModule);
           getDiagnostics().Report(Sub->DefinitionLoc,
                                   diag::note_private_top_level_defined);
@@ -2128,10 +2130,11 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
 
       // If there was a clear winner, user it.
       if (Best.size() == 1) {
-        getDiagnostics().Report(Path[I].second, diag::err_no_submodule_suggest)
-            << Path[I].first << Module->getFullModuleName() << Best[0]
-            << SourceRange(Path[0].second, Path[I - 1].second)
-            << FixItHint::CreateReplacement(SourceRange(Path[I].second),
+        getDiagnostics().Report(Path[I].getLoc(),
+                                diag::err_no_submodule_suggest)
+            << Path[I].getIdentifierInfo() << Module->getFullModuleName()
+            << Best[0] << SourceRange(Path[0].getLoc(), Path[I - 1].getLoc())
+            << FixItHint::CreateReplacement(SourceRange(Path[I].getLoc()),
                                             Best[0]);
 
         Sub = Module->findSubmodule(Best[0]);
@@ -2141,9 +2144,9 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (!Sub) {
       // No submodule by this name. Complain, and don't look for further
       // submodules.
-      getDiagnostics().Report(Path[I].second, diag::err_no_submodule)
-          << Path[I].first << Module->getFullModuleName()
-          << SourceRange(Path[0].second, Path[I - 1].second);
+      getDiagnostics().Report(Path[I].getLoc(), diag::err_no_submodule)
+          << Path[I].getIdentifierInfo() << Module->getFullModuleName()
+          << SourceRange(Path[0].getLoc(), Path[I - 1].getLoc());
       break;
     }
 
@@ -2161,8 +2164,8 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
       // FIXME: Should we detect this at module load time? It seems fairly
       // expensive (and rare).
       getDiagnostics().Report(ImportLoc, diag::warn_missing_submodule)
-        << Module->getFullModuleName()
-        << SourceRange(Path.front().second, Path.back().second);
+          << Module->getFullModuleName()
+          << SourceRange(Path.front().getLoc(), Path.back().getLoc());
 
       return ModuleLoadResult(Module, ModuleLoadResult::MissingExpected);
     }
@@ -2171,7 +2174,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (Preprocessor::checkModuleIsAvailable(getLangOpts(), getTarget(),
                                              *Module, getDiagnostics())) {
       getDiagnostics().Report(ImportLoc, diag::note_module_import_here)
-        << SourceRange(Path.front().second, Path.back().second);
+          << SourceRange(Path.front().getLoc(), Path.back().getLoc());
       LastModuleImportLoc = ImportLoc;
       LastModuleImportResult = ModuleLoadResult();
       return ModuleLoadResult();
@@ -2296,9 +2299,9 @@ GlobalModuleIndex *CompilerInstance::loadGlobalModuleIndex(
       Module *TheModule = I->second;
       OptionalFileEntryRef Entry = TheModule->getASTFile();
       if (!Entry) {
-        SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
-        Path.push_back(std::make_pair(
-            getPreprocessor().getIdentifierInfo(TheModule->Name), TriggerLoc));
+        SmallVector<IdentifierLoc, 2> Path;
+        Path.emplace_back(TriggerLoc,
+                          getPreprocessor().getIdentifierInfo(TheModule->Name));
         std::reverse(Path.begin(), Path.end());
         // Load a module as hidden.  This also adds it to the global index.
         loadModule(TheModule->DefinitionLoc, Path, Module::Hidden, false);
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index c5aeb92c7af73..e6c7b9f32c29b 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -1216,9 +1216,9 @@ void GetDependenciesByModuleNameAction::ExecuteAction() {
   SourceManager &SM = PP.getSourceManager();
   FileID MainFileID = SM.getMainFileID();
   SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+  SmallVector<IdentifierLoc, 2> Path;
   IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
-  Path.push_back(std::make_pair(ModuleID, FileStart));
+  Path.emplace_back(FileStart, ModuleID);
   auto ModResult = CI.loadModule(FileStart, Path, Module::Hidden, false);
   PPCallbacks *CB = PP.getPPCallbacks();
   CB->moduleImport(SourceLocation(), Path, ModResult);
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 8411526019f3e..21ec83b437ef4 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -1916,15 +1916,15 @@ void Preprocessor::EnterAnnotationToken(SourceRange Range,
 
 /// Produce a diagnostic informing the user that a #include or similar
 /// was implicitly treated as a module import.
-static void diagnoseAutoModuleImport(
-    Preprocessor &PP, SourceLocation HashLoc, Token &IncludeTok,
-    ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> Path,
-    SourceLocation PathEnd) {
+static void diagnoseAutoModuleImport(Preprocessor &PP, SourceLocation HashLoc,
+                                     Token &IncludeTok,
+                                     ArrayRef<IdentifierLoc> Path,
+                                     SourceLocation PathEnd) {
   SmallString<128> PathString;
   for (size_t I = 0, N = Path.size(); I != N; ++I) {
     if (I)
       PathString += '.';
-    PathString += Path[I].first->getName();
+    PathString += Path[I].getIdentifierInfo()->getName();
   }
 
   int IncludeKind = 0;
@@ -2273,12 +2273,12 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
   SourceLocation StartLoc = IsImportDecl ? IncludeTok.getLocation() : HashLoc;
 
   // Complain about attempts to #include files in an audit pragma.
-  if (PragmaARCCFCodeAuditedInfo.second.isValid()) {
+  if (PragmaARCCFCodeAuditedInfo.getLoc().isValid()) {
     Diag(StartLoc, diag::err_pp_include_in_arc_cf_code_audited) << IsImportDecl;
-    Diag(PragmaARCCFCodeAuditedInfo.second, diag::note_pragma_entered_here);
+    Diag(PragmaARCCFCodeAuditedInfo.getLoc(), diag::note_pragma_entered_here);
 
     // Immediately leave the pragma.
-    PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
+    PragmaARCCFCodeAuditedInfo = IdentifierLoc();
   }
 
   // Complain about attempts to #include files in an assume-nonnull pragma.
@@ -2403,10 +2403,10 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
     // Compute the module access path corresponding to this module.
     // FIXME: Should we have a second loadModule() overload to avoid this
     // extra lookup step?
-    SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+    SmallVector<IdentifierLoc, 2> Path;
     for (Module *Mod = ModuleToImport; Mod; Mod = Mod->Parent)
-      Path.push_back(std::make_pair(getIdentifierInfo(Mod->Name),
-                                    FilenameTok.getLocation()));
+      Path.emplace_back(FilenameTok.getLocation(),
+                        getIdentifierInfo(Mod->Name));
     std::reverse(Path.begin(), Path.end());
 
     // Warn that we're replacing the include/import with a module import.
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index a373a52506a24..db6069e31fa46 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -409,13 +409,13 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
   // Complain about reaching a true EOF within arc_cf_code_audited.
   // We don't want to complain about reaching the end of a macro
   // instantiation or a _Pragma.
-  if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro &&
+  if (PragmaARCCFCodeAuditedInfo.getLoc().isValid() && !isEndOfMacro &&
       !(CurLexer && CurLexer->Is_PragmaLexer)) {
-    Diag(PragmaARCCFCodeAuditedInfo.second,
+    Diag(PragmaARCCFCodeAuditedInfo.getLoc(),
          diag::err_pp_eof_in_arc_cf_code_audited);
 
     // Recover by leaving immediately.
-    PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
+    PragmaARCCFCodeAuditedInfo = IdentifierLoc();
   }
 
   // Complain about reaching a true EOF within assume_nonnull.
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
index 91c1619e35623..5b6a29bdad910 100644
--- a/clang/lib/Lex/Pragma.cpp
+++ b/clang/lib/Lex/Pragma.cpp
@@ -763,20 +763,19 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) {
 
 // Lex a component of a module name: either an identifier or a string literal;
 // for components that can be expressed both ways, the two forms are equivalent.
-static bool LexModuleNameComponent(
-    Preprocessor &PP, Token &Tok,
-    std::pair<IdentifierInfo *, SourceLocation> &ModuleNameComponent,
-    bool First) {
+static bool LexModuleNameComponent(Preprocessor &PP, Token &Tok,
+                                   IdentifierLoc &ModuleNameComponent,
+                                   bool First) {
   PP.LexUnexpandedToken(Tok);
   if (Tok.is(tok::string_literal) && !Tok.hasUDSuffix()) {
     StringLiteralParser Literal(Tok, PP);
     if (Literal.hadError)
       return true;
-    ModuleNameComponent = std::make_pair(
-        PP.getIdentifierInfo(Literal.GetString()), Tok.getLocation());
+    ModuleNameComponent = IdentifierLoc(
+        Tok.getLocation(), PP.getIdentifierInfo(Literal.GetString()));
   } else if (!Tok.isAnnotation() && Tok.getIdentifierInfo()) {
     ModuleNameComponent =
-        std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation());
+        IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
   } else {
     PP.Diag(Tok.getLocation(), diag::err_pp_expected_module_name) << First;
     return true;
@@ -784,12 +783,10 @@ static bool LexModuleNameComponent(
   return false;
 }
 
-static bool LexModuleName(
-    Preprocessor &PP, Token &Tok,
-    llvm::SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>>
-        &ModuleName) {
+static bool LexModuleName(Preprocessor &PP, Token &Tok,
+                          llvm::SmallVectorImpl<IdentifierLoc> &ModuleName) {
   while (true) {
-    std::pair<IdentifierInfo*, SourceLocation> NameComponent;
+    IdentifierLoc NameComponent;
     if (LexModuleNameComponent(PP, Tok, NameComponent, ModuleName.empty()))
       return true;
     ModuleName.push_back(NameComponent);
@@ -803,10 +800,10 @@ static bool LexModuleName(
 void Preprocessor::HandlePragmaModuleBuild(Token &Tok) {
   SourceLocation Loc = Tok.getLocation();
 
-  std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc;
+  IdentifierLoc ModuleNameLoc;
   if (LexModuleNameComponent(*this, Tok, ModuleNameLoc, true))
     return;
-  IdentifierInfo *ModuleName = ModuleNameLoc.first;
+  IdentifierInfo *ModuleName = ModuleNameLoc.getIdentifierInfo();
 
   LexUnexpandedToken(Tok);
   if (Tok.isNot(tok::eod)) {
@@ -1109,17 +1106,17 @@ struct PragmaDebugHandler : public PragmaHandler {
         PP.Diag(MacroName, diag::warn_pragma_debug_missing_argument)
             << II->getName();
     } else if (II->isStr("module_map")) {
-      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
-          ModuleName;
+      llvm::SmallVector<IdentifierLoc, 8> ModuleName;
       if (LexModuleName(PP, Tok, ModuleName))
         return;
       ModuleMap &MM = PP.getHeaderSearchInfo().getModuleMap();
       Module *M = nullptr;
       for (auto IIAndLoc : ModuleName) {
-        M = MM.lookupModuleQualified(IIAndLoc.first->getName(), M);
+        M = MM.lookupModuleQualified(IIAndLoc.getIdentifierInfo()->getName(),
+                                     M);
         if (!M) {
-          PP.Diag(IIAndLoc.second, diag::warn_pragma_debug_unknown_module)
-              << IIAndLoc.first->getName();
+          PP.Diag(IIAndLoc.getLoc(), diag::warn_pragma_debug_unknown_module)
+              << IIAndLoc.getIdentifierInfo()->getName();
           return;
         }
       }
@@ -1707,8 +1704,7 @@ struct PragmaModuleImportHandler : public PragmaHandler {
     SourceLocation ImportLoc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
-        ModuleName;
+    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1723,7 +1719,7 @@ struct PragmaModuleImportHandler : public PragmaHandler {
       return;
 
     PP.makeModuleVisible(Imported, ImportLoc);
-    PP.EnterAnnotationToken(SourceRange(ImportLoc, ModuleName.back().second),
+    PP.EnterAnnotationToken(SourceRange(ImportLoc, ModuleName.back().getLoc()),
                             tok::annot_module_include, Imported);
     if (auto *CB = PP.getPPCallbacks())
       CB->moduleImport(ImportLoc, ModuleName, Imported);
@@ -1744,8 +1740,7 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
     SourceLocation BeginLoc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
-        ModuleName;
+    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1754,10 +1749,11 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
 
     // We can only enter submodules of the current module.
     StringRef Current = PP.getLangOpts().CurrentModule;
-    if (ModuleName.front().first->getName() != Current) {
-      PP.Diag(ModuleName.front().second, diag::err_pp_module_begin_wrong_module)
-        << ModuleName.front().first << (ModuleName.size() > 1)
-        << Current.empty() << Current;
+    if (ModuleName.front().getIdentifierInfo()->getName() != Current) {
+      PP.Diag(ModuleName.front().getLoc(),
+              diag::err_pp_module_begin_wrong_module)
+          << ModuleName.front().getIdentifierInfo() << (ModuleName.size() > 1)
+          << Current.empty() << Current;
       return;
     }
 
@@ -1765,17 +1761,19 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
     // be loaded or implicitly loadable.
     auto &HSI = PP.getHeaderSearchInfo();
     auto &MM = HSI.getModuleMap();
-    Module *M = HSI.lookupModule(Current, ModuleName.front().second);
+    Module *M = HSI.lookupModule(Current, ModuleName.front().getLoc());
     if (!M) {
-      PP.Diag(ModuleName.front().second,
-              diag::err_pp_module_begin_no_module_map) << Current;
+      PP.Diag(ModuleName.front().getLoc(),
+              diag::err_pp_module_begin_no_module_map)
+          << Current;
       return;
     }
     for (unsigned I = 1; I != ModuleName.size(); ++I) {
-      auto *NewM = MM.findOrInferSubmodule(M, ModuleName[I].first->getName());
+      auto *NewM = MM.findOrInferSubmodule(
+          M, ModuleName[I].getIdentifierInfo()->getName());
       if (!NewM) {
-        PP.Diag(ModuleName[I].second, diag::err_pp_module_begin_no_submodule)
-          << M->getFullModuleName() << ModuleName[I].first;
+        PP.Diag(ModuleName[I].getLoc(), diag::err_pp_module_begin_no_submodule)
+            << M->getFullModuleName() << ModuleName[I].getIdentifierInfo();
         return;
       }
       M = NewM;
@@ -1791,7 +1789,7 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
 
     // Enter the scope of the submodule.
     PP.EnterSubmodule(M, BeginLoc, /*ForPragma*/true);
-    PP.EnterAnnotationToken(SourceRange(BeginLoc, ModuleName.back().second),
+    PP.EnterAnnotationToken(SourceRange(BeginLoc, ModuleName.back().getLoc()),
                             tok::annot_module_begin, M);
   }
 };
@@ -1835,8 +1833,7 @@ struct PragmaModuleLoadHandler : public PragmaHandler {
     SourceLocation Loc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
-        ModuleName;
+    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1901,7 +1898,7 @@ struct PragmaARCCFCodeAuditedHandler : public PragmaHandler {
       PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma";
 
     // The start location of the active audit.
-    SourceLocation BeginLoc = PP.getPragmaARCCFCodeAuditedInfo().second;
+    SourceLocation BeginLoc = PP.getPragmaARCCFCodeAuditedInfo().getLoc();
 
     // The start location we want after processing this.
     SourceLocation NewLoc;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index c25a3efd899e0..4c050bf1f5bb2 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -1159,8 +1159,8 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
     if (Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
       std::string Name = ModuleDeclState.getPrimaryName().str();
       Name += ":";
-      NamedModuleImportPath.push_back(
-          {getIdentifierInfo(Name), Result.getLocation()});
+      NamedModuleImportPath.emplace_back(Result.getLocation(),
+                                         getIdentifierInfo(Name));
       CurLexerCallback = CLK_LexAfterModuleImport;
       return true;
     }
@@ -1258,8 +1258,8 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
   if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
     // We expected to see an identifier here, and we did; continue handling
     // identifiers.
-    NamedModuleImportPath.push_back(
-        std::make_pair(Result.getIdentifierInfo(), Result.getLocation()));
+    NamedModuleImportPath.emplace_back(Result.getLocation(),
+                                       Result.getIdentifierInfo());
     ModuleImportExpectsIdentifier = false;
     CurLexerCallback = CLK_LexAfterModuleImport;
     return true;
@@ -1302,12 +1302,12 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
       // If the FlatModuleName ends with colon, it implies it is a partition.
       if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
         FlatModuleName += ".";
-      FlatModuleName += Piece.first->getName();
+      FlatModuleName += Piece.getIdentifierInfo()->getName();
     }
-    SourceLocation FirstPathLoc = NamedModuleImportPath[0].second;
+    SourceLocation FirstPathLoc = NamedModuleImportPath[0].getLoc();
     NamedModuleImportPath.clear();
-    NamedModuleImportPath.push_back(
-        std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
+    NamedModuleImportPath.emplace_back(FirstPathLoc,
+                                       getIdentifierInfo(FlatModuleName));
   }
 
   Module *Imported = nullptr;
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 8fa74ecff19aa..8444ff3332e08 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -432,9 +432,8 @@ static bool attributeParsedArgsUnevaluated(const IdentifierInfo &II,
 
 IdentifierLoc *Parser::ParseIdentifierLoc() {
   assert(Tok.is(tok::identifier) && "expected an identifier");
-  IdentifierLoc *IL = IdentifierLoc::create(Actions.Context,
-                                            Tok.getLocation(),
-                                            Tok.getIdentifierInfo());
+  IdentifierLoc *IL = new (Actions.Context)
+      IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
   ConsumeToken();
   return IL;
 }
@@ -1353,20 +1352,21 @@ void Parser::ParseAvailabilityAttribute(
     return;
   }
   IdentifierLoc *Platform = ParseIdentifierLoc();
-  if (const IdentifierInfo *const Ident = Platform->Ident) {
+  if (const IdentifierInfo *const Ident = Platform->getIdentifierInfo()) {
     // Disallow xrOS for availability attributes.
     if (Ident->getName().contains("xrOS") || Ident->getName().contains("xros"))
-      Diag(Platform->Loc, diag::warn_availability_unknown_platform) << Ident;
+      Diag(Platform->getLoc(), diag::warn_availability_unknown_platform)
+          << Ident;
     // Canonicalize platform name from "macosx" to "macos".
     else if (Ident->getName() == "macosx")
-      Platform->Ident = PP.getIdentifierInfo("macos");
+      Platform->setIdentifierInfo(PP.getIdentifierInfo("macos"));
     // Canonicalize platform name from "macosx_app_extension" to
     // "macos_app_extension".
     else if (Ident->getName() == "macosx_app_extension")
-      Platform->Ident = PP.getIdentifierInfo("macos_app_extension");
+      Platform->setIdentifierInfo(PP.getIdentifierInfo("macos_app_extension"));
     else
-      Platform->Ident = PP.getIdentifierInfo(
-          AvailabilityAttr::canonicalizePlatformName(Ident->getName()));
+      Platform->setIdentifierInfo(PP.getIdentifierInfo(
+          AvailabilityAttr::canonicalizePlatformName(Ident->getName())));
   }
 
   // Parse the ',' following the platform name.
@@ -1418,8 +1418,8 @@ void Parser::ParseAvailabilityAttribute(
       continue;
     }
 
-    if (Keyword == Ident_deprecated && Platform->Ident &&
-        Platform->Ident->isStr("swift")) {
+    if (Keyword == Ident_deprecated && Platform->getIdentifierInfo() &&
+        Platform->getIdentifierInfo()->isStr("swift")) {
       // For swift, we deprecate for all versions.
       if (Changes[Deprecated].KeywordLoc.isValid()) {
         Diag(KeywordLoc, diag::err_availability_redundant)
@@ -1436,7 +1436,7 @@ void Parser::ParseAvailabilityAttribute(
     if (Keyword == Ident_environment) {
       if (EnvironmentLoc != nullptr) {
         Diag(KeywordLoc, diag::err_availability_redundant)
-            << Keyword << SourceRange(EnvironmentLoc->Loc);
+            << Keyword << SourceRange(EnvironmentLoc->getLoc());
       }
     }
 
@@ -1792,8 +1792,8 @@ void Parser::ParseSwiftNewTypeAttribute(
     return;
   }
 
-  auto *SwiftType = IdentifierLoc::create(Actions.Context, Tok.getLocation(),
-                                          Tok.getIdentifierInfo());
+  auto *SwiftType = new (Actions.Context)
+      IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
   ConsumeToken();
 
   // Closing ')'
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 0a22f7372a9f9..1416d52157dca 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -4006,19 +4006,20 @@ std::optional<AvailabilitySpec> Parser::ParseAvailabilitySpec() {
     if (Version.empty())
       return std::nullopt;
 
-    StringRef GivenPlatform = PlatformIdentifier->Ident->getName();
+    StringRef GivenPlatform =
+        PlatformIdentifier->getIdentifierInfo()->getName();
     StringRef Platform =
         AvailabilityAttr::canonicalizePlatformName(GivenPlatform);
 
     if (AvailabilityAttr::getPrettyPlatformName(Platform).empty() ||
         (GivenPlatform.contains("xros") || GivenPlatform.contains("xrOS"))) {
-      Diag(PlatformIdentifier->Loc,
+      Diag(PlatformIdentifier->getLoc(),
            diag::err_avail_query_unrecognized_platform_name)
           << GivenPlatform;
       return std::nullopt;
     }
 
-    return AvailabilitySpec(Version, Platform, PlatformIdentifier->Loc,
+    return AvailabilitySpec(Version, Platform, PlatformIdentifier->getLoc(),
                             VersionRange.getEnd());
   }
 }
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index f4c109f9a81a2..7ffacc4b79f79 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -115,7 +115,7 @@ static void fixSeparateAttrArgAndNumber(StringRef ArgStr, SourceLocation ArgLoc,
       << FixedArg
       << FixItHint::CreateReplacement(SourceRange(ArgLoc, EndNumLoc), FixedArg);
   ArgsUnion &Slot = ArgExprs.back();
-  Slot = IdentifierLoc::create(Ctx, ArgLoc, PP.getIdentifierInfo(FixedArg));
+  Slot = new (Ctx) IdentifierLoc(ArgLoc, PP.getIdentifierInfo(FixedArg));
 }
 
 void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index bcbf4dfbabafa..d872177b3d7aa 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -261,7 +261,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
   // case, LAngleLoc will be valid and ProtocolIdents will capture the
   // protocol references (that have not yet been resolved).
   SourceLocation LAngleLoc, EndProtoLoc;
-  SmallVector<IdentifierLocPair, 8> ProtocolIdents;
+  SmallVector<IdentifierLoc, 8> ProtocolIdents;
   ObjCTypeParamList *typeParameterList = nullptr;
   ObjCTypeParamListScope typeParamScope(Actions, getCurScope());
   if (Tok.is(tok::less))
@@ -361,8 +361,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
     if (!ProtocolIdents.empty()) {
       // We already parsed the protocols named when we thought we had a
       // type parameter list. Translate them into actual protocol references.
-      for (const auto &pair : ProtocolIdents) {
-        protocolLocs.push_back(pair.second);
+      for (const auto &Loc : ProtocolIdents) {
+        protocolLocs.push_back(Loc.getLoc());
       }
       Actions.ObjC().FindProtocolDeclaration(/*WarnOnDeclarations=*/true,
                                              /*ForObjCContainer=*/true,
@@ -459,8 +459,8 @@ static void addContextSensitiveTypeNullability(Parser &P,
 /// \param rAngleLoc The location of the ending '>'.
 ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     ObjCTypeParamListScope &Scope, SourceLocation &lAngleLoc,
-    SmallVectorImpl<IdentifierLocPair> &protocolIdents,
-    SourceLocation &rAngleLoc, bool mayBeProtocolList) {
+    SmallVectorImpl<IdentifierLoc> &protocolIdents, SourceLocation &rAngleLoc,
+    bool mayBeProtocolList) {
   assert(Tok.is(tok::less) && "Not at the beginning of a type parameter list");
 
   // Within the type parameter list, don't treat '>' as an operator.
@@ -474,7 +474,8 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     for (const auto &pair : protocolIdents) {
       DeclResult typeParam = Actions.ObjC().actOnObjCTypeParam(
           getCurScope(), ObjCTypeParamVariance::Invariant, SourceLocation(),
-          index++, pair.first, pair.second, SourceLocation(), nullptr);
+          index++, pair.getIdentifierInfo(), pair.getLoc(), SourceLocation(),
+          nullptr);
       if (typeParam.isUsable())
         typeParams.push_back(typeParam.get());
     }
@@ -546,7 +547,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     } else if (mayBeProtocolList) {
       // If this could still be a protocol list, just capture the identifier.
       // We don't want to turn it into a parameter.
-      protocolIdents.push_back(std::make_pair(paramName, paramLoc));
+      protocolIdents.emplace_back(paramLoc, paramName);
       continue;
     }
 
@@ -606,7 +607,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
 /// Parse an objc-type-parameter-list.
 ObjCTypeParamList *Parser::parseObjCTypeParamList() {
   SourceLocation lAngleLoc;
-  SmallVector<IdentifierLocPair, 1> protocolIdents;
+  SmallVector<IdentifierLoc, 1> protocolIdents;
   SourceLocation rAngleLoc;
 
   ObjCTypeParamListScope Scope(Actions, getCurScope());
@@ -1598,7 +1599,7 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
 
   LAngleLoc = ConsumeToken(); // the "<"
 
-  SmallVector<IdentifierLocPair, 8> ProtocolIdents;
+  SmallVector<IdentifierLoc, 8> ProtocolIdents;
 
   while (true) {
     if (Tok.is(tok::code_completion)) {
@@ -1612,8 +1613,7 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
       SkipUntil(tok::greater, StopAtSemi);
       return true;
     }
-    ProtocolIdents.push_back(std::make_pair(Tok.getIdentifierInfo(),
-                                       Tok.getLocation()));
+    ProtocolIdents.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
     ProtocolLocs.push_back(Tok.getLocation());
     ConsumeToken();
 
@@ -1693,10 +1693,9 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
 
     if (Tok.is(tok::code_completion)) {
       // FIXME: Also include types here.
-      SmallVector<IdentifierLocPair, 4> identifierLocPairs;
+      SmallVector<IdentifierLoc, 4> identifierLocPairs;
       for (unsigned i = 0, n = identifiers.size(); i != n; ++i) {
-        identifierLocPairs.push_back(IdentifierLocPair(identifiers[i],
-                                                       identifierLocs[i]));
+        identifierLocPairs.emplace_back(identifierLocs[i], identifiers[i]);
       }
 
       QualType BaseT = Actions.GetTypeFromParser(baseType);
@@ -2094,7 +2093,7 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   SourceLocation nameLoc = ConsumeToken();
 
   if (TryConsumeToken(tok::semi)) { // forward declaration of one protocol.
-    IdentifierLocPair ProtoInfo(protocolName, nameLoc);
+    IdentifierLoc ProtoInfo(nameLoc, protocolName);
     return Actions.ObjC().ActOnForwardProtocolDeclaration(AtLoc, ProtoInfo,
                                                           attrs);
   }
@@ -2102,8 +2101,8 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   CheckNestedObjCContexts(AtLoc);
 
   if (Tok.is(tok::comma)) { // list of forward declarations.
-    SmallVector<IdentifierLocPair, 8> ProtocolRefs;
-    ProtocolRefs.push_back(std::make_pair(protocolName, nameLoc));
+    SmallVector<IdentifierLoc, 8> ProtocolRefs;
+    ProtocolRefs.emplace_back(nameLoc, protocolName);
 
     // Parse the list of forward declarations.
     while (true) {
@@ -2112,8 +2111,7 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
         SkipUntil(tok::semi);
         return nullptr;
       }
-      ProtocolRefs.push_back(IdentifierLocPair(Tok.getIdentifierInfo(),
-                                               Tok.getLocation()));
+      ProtocolRefs.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
       ConsumeToken(); // the identifier
 
       if (Tok.isNot(tok::comma))
@@ -2196,7 +2194,7 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
   // permitted here. Parse and diagnose them.
   if (Tok.is(tok::less)) {
     SourceLocation lAngleLoc, rAngleLoc;
-    SmallVector<IdentifierLocPair, 8> protocolIdents;
+    SmallVector<IdentifierLoc, 8> protocolIdents;
     SourceLocation diagLoc = Tok.getLocation();
     ObjCTypeParamListScope typeParamScope(Actions, getCurScope());
     if (parseObjCTypeParamListOrProtocolRefs(typeParamScope, lAngleLoc,
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 64916995907c5..337b3eca49764 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -15,6 +15,7 @@
 #include "clang/Basic/OpenACCKinds.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/SemaOpenACC.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -814,7 +815,7 @@ bool Parser::ParseOpenACCIntExprList(OpenACCDirectiveKind DK,
 ///
 /// The device_type clause may be abbreviated to dtype.
 bool Parser::ParseOpenACCDeviceTypeList(
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs) {
+    llvm::SmallVector<IdentifierLoc> &Archs) {
 
   if (expectIdentifierOrKeyword(*this)) {
     SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
@@ -822,7 +823,7 @@ bool Parser::ParseOpenACCDeviceTypeList(
     return true;
   }
   IdentifierInfo *Ident = getCurToken().getIdentifierInfo();
-  Archs.emplace_back(Ident, ConsumeToken());
+  Archs.emplace_back(ConsumeToken(), Ident);
 
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     ExpectAndConsume(tok::comma);
@@ -833,7 +834,7 @@ bool Parser::ParseOpenACCDeviceTypeList(
       return true;
     }
     Ident = getCurToken().getIdentifierInfo();
-    Archs.emplace_back(Ident, ConsumeToken());
+    Archs.emplace_back(ConsumeToken(), Ident);
   }
   return false;
 }
@@ -1154,11 +1155,12 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
     }
     case OpenACCClauseKind::DType:
     case OpenACCClauseKind::DeviceType: {
-      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> Archs;
+      llvm::SmallVector<IdentifierLoc> Archs;
       if (getCurToken().is(tok::star)) {
         // FIXME: We want to mark that this is an 'everything else' type of
         // device_type in Sema.
-        ParsedClause.setDeviceTypeDetails({{nullptr, ConsumeToken()}});
+        ParsedClause.setDeviceTypeDetails(
+            {IdentifierLoc(ConsumeToken(), nullptr)});
       } else if (!ParseOpenACCDeviceTypeList(Archs)) {
         ParsedClause.setDeviceTypeDetails(std::move(Archs));
       } else {
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index 21ebff1e50559..17b2b30942582 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -1419,16 +1419,16 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
       static_cast<PragmaLoopHintInfo *>(Tok.getAnnotationValue());
 
   IdentifierInfo *PragmaNameInfo = Info->PragmaName.getIdentifierInfo();
-  Hint.PragmaNameLoc = IdentifierLoc::create(
-      Actions.Context, Info->PragmaName.getLocation(), PragmaNameInfo);
+  Hint.PragmaNameLoc = new (Actions.Context)
+      IdentifierLoc(Info->PragmaName.getLocation(), PragmaNameInfo);
 
   // It is possible that the loop hint has no option identifier, such as
   // #pragma unroll(4).
   IdentifierInfo *OptionInfo = Info->Option.is(tok::identifier)
                                    ? Info->Option.getIdentifierInfo()
                                    : nullptr;
-  Hint.OptionLoc = IdentifierLoc::create(
-      Actions.Context, Info->Option.getLocation(), OptionInfo);
+  Hint.OptionLoc = new (Actions.Context)
+      IdentifierLoc(Info->Option.getLocation(), OptionInfo);
 
   llvm::ArrayRef<Token> Toks = Info->Toks;
 
@@ -1508,7 +1508,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
     if (Toks.size() > 2)
       Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
           << PragmaLoopHintString(Info->PragmaName, Info->Option);
-    Hint.StateLoc = IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
+    Hint.StateLoc = new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
   } else if (OptionInfo && OptionInfo->getName() == "vectorize_width") {
     PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false,
                         /*IsReinject=*/false);
@@ -1529,8 +1529,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
           ConsumeAnyToken();
       }
 
-      Hint.StateLoc =
-          IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
+      Hint.StateLoc = new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
 
       ConsumeToken(); // Consume the constant expression eof terminator.
     } else {
@@ -1554,7 +1553,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
           Arg2Error = true;
         } else
           Hint.StateLoc =
-              IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
+              new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
 
         PP.Lex(Tok); // Identifier
       }
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index e8ec140fbe3e5..4a82d57fe566b 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -2545,9 +2545,9 @@ StmtResult Parser::ParsePragmaLoopHint(StmtVector &Stmts,
 
     ArgsUnion ArgHints[] = {Hint.PragmaNameLoc, Hint.OptionLoc, Hint.StateLoc,
                             ArgsUnion(Hint.ValueExpr)};
-    TempAttrs.addNew(Hint.PragmaNameLoc->Ident, Hint.Range, nullptr,
-                     Hint.PragmaNameLoc->Loc, ArgHints, 4,
-                     ParsedAttr::Form::Pragma());
+    TempAttrs.addNew(Hint.PragmaNameLoc->getIdentifierInfo(), Hint.Range,
+                     /*scopeName=*/nullptr, Hint.PragmaNameLoc->getLoc(),
+                     ArgHints, /*numArgs=*/4, ParsedAttr::Form::Pragma());
   }
 
   // Get the next statement.
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index f3191762b1244..d528664bca352 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -2541,17 +2541,17 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
     return Actions.ActOnPrivateModuleFragmentDecl(ModuleLoc, PrivateLoc);
   }
 
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+  SmallVector<IdentifierLoc, 2> Path;
   if (ParseModuleName(ModuleLoc, Path, /*IsImport*/ false))
     return nullptr;
 
   // Parse the optional module-partition.
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Partition;
+  SmallVector<IdentifierLoc, 2> Partition;
   if (Tok.is(tok::colon)) {
     SourceLocation ColonLoc = ConsumeToken();
     if (!getLangOpts().CPlusPlusModules)
       Diag(ColonLoc, diag::err_unsupported_module_partition)
-          << SourceRange(ColonLoc, Partition.back().second);
+          << SourceRange(ColonLoc, Partition.back().getLoc());
     // Recover by ignoring the partition name.
     else if (ParseModuleName(ModuleLoc, Partition, /*IsImport*/ false))
       return nullptr;
@@ -2600,7 +2600,7 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
   SourceLocation ImportLoc = ConsumeToken();
 
   // For C++20 modules, we can have "name" or ":Partition name" as valid input.
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+  SmallVector<IdentifierLoc, 2> Path;
   bool IsPartition = false;
   Module *HeaderUnit = nullptr;
   if (Tok.is(tok::header_name)) {
@@ -2616,7 +2616,7 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
     SourceLocation ColonLoc = ConsumeToken();
     if (!getLangOpts().CPlusPlusModules)
       Diag(ColonLoc, diag::err_unsupported_module_partition)
-          << SourceRange(ColonLoc, Path.back().second);
+          << SourceRange(ColonLoc, Path.back().getLoc());
     // Recover by leaving partition empty.
     else if (ParseModuleName(ColonLoc, Path, /*IsImport*/ true))
       return nullptr;
@@ -2718,10 +2718,9 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
 ///           module-name-qualifier[opt] identifier
 ///         module-name-qualifier:
 ///           module-name-qualifier[opt] identifier '.'
-bool Parser::ParseModuleName(
-    SourceLocation UseLoc,
-    SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
-    bool IsImport) {
+bool Parser::ParseModuleName(SourceLocation UseLoc,
+                             SmallVectorImpl<IdentifierLoc> &Path,
+                             bool IsImport) {
   // Parse the module path.
   while (true) {
     if (!Tok.is(tok::identifier)) {
@@ -2737,7 +2736,7 @@ bool Parser::ParseModuleName(
     }
 
     // Record this part of the module path.
-    Path.push_back(std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation()));
+    Path.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
     ConsumeToken();
 
     if (Tok.isNot(tok::period))
diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp
index b19a02b8c1a09..c149cef478539 100644
--- a/clang/lib/Sema/ParsedAttr.cpp
+++ b/clang/lib/Sema/ParsedAttr.cpp
@@ -23,14 +23,6 @@
 
 using namespace clang;
 
-IdentifierLoc *IdentifierLoc::create(ASTContext &Ctx, SourceLocation Loc,
-                                     IdentifierInfo *Ident) {
-  IdentifierLoc *Result = new (Ctx) IdentifierLoc;
-  Result->Loc = Loc;
-  Result->Ident = Ident;
-  return Result;
-}
-
 size_t ParsedAttr::allocated_size() const {
   if (IsAvailability) return AttributeFactory::AvailabilityAllocSize;
   else if (IsTypeTagForDatatype)
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 3f53fb200a93d..5bcbe78e9d633 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -1178,7 +1178,7 @@ void SemaARM::handleBuiltinAliasAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Ident = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *Ident = AL.getArgAsIdent(0)->getIdentifierInfo();
   unsigned BuiltinID = Ident->getBuiltinID();
   StringRef AliasName = cast<FunctionDecl>(D)->getIdentifier()->getName();
 
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index f6ec4cb0f069e..1e4e6fdc78351 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -8718,7 +8718,7 @@ static void AddProtocolResults(DeclContext *Ctx, DeclContext *CurContext,
 }
 
 void SemaCodeCompletion::CodeCompleteObjCProtocolReferences(
-    ArrayRef<IdentifierLocPair> Protocols) {
+    ArrayRef<IdentifierLoc> Protocols) {
   ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCProtocolName);
@@ -8729,9 +8729,9 @@ void SemaCodeCompletion::CodeCompleteObjCProtocolReferences(
     // Tell the result set to ignore all of the protocols we have
     // already seen.
     // FIXME: This doesn't work when caching code-completion results.
-    for (const IdentifierLocPair &Pair : Protocols)
-      if (ObjCProtocolDecl *Protocol =
-              SemaRef.ObjC().LookupProtocol(Pair.first, Pair.second))
+    for (const IdentifierLoc &Pair : Protocols)
+      if (ObjCProtocolDecl *Protocol = SemaRef.ObjC().LookupProtocol(
+              Pair.getIdentifierInfo(), Pair.getLoc()))
         Results.Ignore(Protocol);
 
     // Add all protocols.
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index bc891fb009410..7dd20a8795fc9 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -135,13 +135,13 @@ bool Sema::checkStringLiteralArgumentAttr(const ParsedAttr &AL, unsigned ArgNum,
   // Look for identifiers. If we have one emit a hint to fix it to a literal.
   if (AL.isArgIdent(ArgNum)) {
     IdentifierLoc *Loc = AL.getArgAsIdent(ArgNum);
-    Diag(Loc->Loc, diag::err_attribute_argument_type)
+    Diag(Loc->getLoc(), diag::err_attribute_argument_type)
         << AL << AANT_ArgumentString
-        << FixItHint::CreateInsertion(Loc->Loc, "\"")
-        << FixItHint::CreateInsertion(getLocForEndOfToken(Loc->Loc), "\"");
-    Str = Loc->Ident->getName();
+        << FixItHint::CreateInsertion(Loc->getLoc(), "\"")
+        << FixItHint::CreateInsertion(getLocForEndOfToken(Loc->getLoc()), "\"");
+    Str = Loc->getIdentifierInfo()->getName();
     if (ArgLocation)
-      *ArgLocation = Loc->Loc;
+      *ArgLocation = Loc->getLoc();
     return true;
   }
 
@@ -768,7 +768,7 @@ static void handleDiagnoseAsBuiltinAttr(Sema &S, Decl *D,
       auto Union = AL.getArg(Index - 1);
       if (auto *E = dyn_cast<Expr *>(Union))
         return E->getBeginLoc();
-      return cast<IdentifierLoc *>(Union)->Loc;
+      return cast<IdentifierLoc *>(Union)->getLoc();
     }();
 
     S.Diag(Loc, diag::err_attribute_argument_n_type) << AL << Index << T;
@@ -960,10 +960,10 @@ static void handleConsumableAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
-    if (!ConsumableAttr::ConvertStrToConsumedState(IL->Ident->getName(),
-                                                   DefaultState)) {
-      S.Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL
-                                                               << IL->Ident;
+    if (!ConsumableAttr::ConvertStrToConsumedState(
+            IL->getIdentifierInfo()->getName(), DefaultState)) {
+      S.Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
+          << AL << IL->getIdentifierInfo();
       return;
     }
   } else {
@@ -1005,8 +1005,8 @@ static void handleCallableWhenAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     SourceLocation Loc;
     if (AL.isArgIdent(ArgIndex)) {
       IdentifierLoc *Ident = AL.getArgAsIdent(ArgIndex);
-      StateString = Ident->Ident->getName();
-      Loc = Ident->Loc;
+      StateString = Ident->getIdentifierInfo()->getName();
+      Loc = Ident->getLoc();
     } else {
       if (!S.checkStringLiteralArgumentAttr(AL, ArgIndex, StateString, &Loc))
         return;
@@ -1030,11 +1030,11 @@ static void handleParamTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef StateString = Ident->Ident->getName();
+    StringRef StateString = Ident->getIdentifierInfo()->getName();
 
     if (!ParamTypestateAttr::ConvertStrToConsumedState(StateString,
                                                        ParamState)) {
-      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported)
+      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
           << AL << StateString;
       return;
     }
@@ -1064,10 +1064,10 @@ static void handleReturnTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
-    if (!ReturnTypestateAttr::ConvertStrToConsumedState(IL->Ident->getName(),
-                                                        ReturnState)) {
-      S.Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL
-                                                               << IL->Ident;
+    if (!ReturnTypestateAttr::ConvertStrToConsumedState(
+            IL->getIdentifierInfo()->getName(), ReturnState)) {
+      S.Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
+          << AL << IL->getIdentifierInfo();
       return;
     }
   } else {
@@ -1111,10 +1111,10 @@ static void handleSetTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   SetTypestateAttr::ConsumedState NewState;
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef Param = Ident->Ident->getName();
+    StringRef Param = Ident->getIdentifierInfo()->getName();
     if (!SetTypestateAttr::ConvertStrToConsumedState(Param, NewState)) {
-      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported) << AL
-                                                                  << Param;
+      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
+          << AL << Param;
       return;
     }
   } else {
@@ -1133,10 +1133,10 @@ static void handleTestTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   TestTypestateAttr::ConsumedState TestState;
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef Param = Ident->Ident->getName();
+    StringRef Param = Ident->getIdentifierInfo()->getName();
     if (!TestTypestateAttr::ConvertStrToConsumedState(Param, TestState)) {
-      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported) << AL
-                                                                  << Param;
+      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
+          << AL << Param;
       return;
     }
   } else {
@@ -1497,7 +1497,7 @@ static void handleOwnershipAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Module = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *Module = AL.getArgAsIdent(0)->getIdentifierInfo();
 
   StringRef ModuleName = Module->getName();
   if (normalizeName(ModuleName)) {
@@ -1864,10 +1864,10 @@ static void handleCPUSpecificAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *CPUArg = AL.getArgAsIdent(ArgNo);
-    StringRef CPUName = CPUArg->Ident->getName().trim();
+    StringRef CPUName = CPUArg->getIdentifierInfo()->getName().trim();
 
     if (!S.Context.getTargetInfo().validateCPUSpecificCPUDispatch(CPUName)) {
-      S.Diag(CPUArg->Loc, diag::err_invalid_cpu_specific_dispatch_value)
+      S.Diag(CPUArg->getLoc(), diag::err_invalid_cpu_specific_dispatch_value)
           << CPUName << (AL.getKind() == ParsedAttr::AT_CPUDispatch);
       return;
     }
@@ -1880,7 +1880,7 @@ static void handleCPUSpecificAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
       S.Diag(AL.getLoc(), diag::warn_multiversion_duplicate_entries);
       return;
     }
-    CPUs.push_back(CPUArg->Ident);
+    CPUs.push_back(CPUArg->getIdentifierInfo());
   }
 
   FD->setIsMultiVersion(true);
@@ -2358,10 +2358,10 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   IdentifierLoc *Platform = AL.getArgAsIdent(0);
 
-  IdentifierInfo *II = Platform->Ident;
+  IdentifierInfo *II = Platform->getIdentifierInfo();
   if (AvailabilityAttr::getPrettyPlatformName(II->getName()).empty())
-    S.Diag(Platform->Loc, diag::warn_availability_unknown_platform)
-      << Platform->Ident;
+    S.Diag(Platform->getLoc(), diag::warn_availability_unknown_platform)
+        << Platform->getIdentifierInfo();
 
   auto *ND = dyn_cast<NamedDecl>(D);
   if (!ND) // We warned about this already, so just return.
@@ -2410,14 +2410,16 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   IdentifierInfo *IIEnvironment = nullptr;
   if (EnvironmentLoc) {
     if (S.getLangOpts().HLSL) {
-      IIEnvironment = EnvironmentLoc->Ident;
+      IIEnvironment = EnvironmentLoc->getIdentifierInfo();
       if (AvailabilityAttr::getEnvironmentType(
-              EnvironmentLoc->Ident->getName()) ==
+              EnvironmentLoc->getIdentifierInfo()->getName()) ==
           llvm::Triple::EnvironmentType::UnknownEnvironment)
-        S.Diag(EnvironmentLoc->Loc, diag::warn_availability_unknown_environment)
-            << EnvironmentLoc->Ident;
+        S.Diag(EnvironmentLoc->getLoc(),
+               diag::warn_availability_unknown_environment)
+            << EnvironmentLoc->getIdentifierInfo();
     } else {
-      S.Diag(EnvironmentLoc->Loc, diag::err_availability_unexpected_parameter)
+      S.Diag(EnvironmentLoc->getLoc(),
+             diag::err_availability_unexpected_parameter)
           << "environment" << /* C/C++ */ 1;
     }
   }
@@ -3630,7 +3632,7 @@ static void handleEnumExtensibilityAttr(Sema &S, Decl *D,
   }
 
   EnumExtensibilityAttr::Kind ExtensibilityKind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   if (!EnumExtensibilityAttr::ConvertStrToKind(II->getName(),
                                                ExtensibilityKind)) {
     S.Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
@@ -3853,7 +3855,7 @@ static bool handleFormatAttrCommon(Sema &S, Decl *D, const ParsedAttr &AL,
   bool HasImplicitThisParam = isInstanceMethod(D);
   Info->NumArgs = getFunctionOrMethodNumParams(D) + HasImplicitThisParam;
 
-  Info->Identifier = AL.getArgAsIdent(0)->Ident;
+  Info->Identifier = AL.getArgAsIdent(0)->getIdentifierInfo();
   StringRef Format = Info->Identifier->getName();
 
   if (normalizeName(Format)) {
@@ -4017,14 +4019,14 @@ static void handleCallbackAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
     if (AL.isArgIdent(I)) {
       IdentifierLoc *IdLoc = AL.getArgAsIdent(I);
-      auto It = NameIdxMapping.find(IdLoc->Ident->getName());
+      auto It = NameIdxMapping.find(IdLoc->getIdentifierInfo()->getName());
       if (It == UnknownName) {
         S.Diag(AL.getLoc(), diag::err_callback_attribute_argument_unknown)
-            << IdLoc->Ident << IdLoc->Loc;
+            << IdLoc->getIdentifierInfo() << IdLoc->getLoc();
         return;
       }
 
-      SR = SourceRange(IdLoc->Loc);
+      SR = SourceRange(IdLoc->getLoc());
       ArgIdx = It->second;
     } else if (AL.isArgExpr(I)) {
       Expr *IdxExpr = AL.getArgAsExpr(I);
@@ -4142,13 +4144,14 @@ LifetimeCaptureByAttr *Sema::ParseLifetimeCaptureByAttr(const ParsedAttr &AL,
     }
     assert(AL.isArgIdent(I));
     IdentifierLoc *IdLoc = AL.getArgAsIdent(I);
-    if (IdLoc->Ident->getName() == ParamName) {
-      Diag(IdLoc->Loc, diag::err_capture_by_references_itself) << IdLoc->Loc;
+    if (IdLoc->getIdentifierInfo()->getName() == ParamName) {
+      Diag(IdLoc->getLoc(), diag::err_capture_by_references_itself)
+          << IdLoc->getLoc();
       IsValid = false;
       continue;
     }
-    ParamIdents[I] = IdLoc->Ident;
-    ParamLocs[I] = IdLoc->Loc;
+    ParamIdents[I] = IdLoc->getIdentifierInfo();
+    ParamLocs[I] = IdLoc->getLoc();
   }
   if (!IsValid)
     return nullptr;
@@ -4754,7 +4757,7 @@ static void handleModeAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Name = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *Name = AL.getArgAsIdent(0)->getIdentifierInfo();
 
   S.AddModeAttr(D, AL, Name);
 }
@@ -5727,8 +5730,8 @@ static void handleArgumentWithTypeTagAttr(Sema &S, Decl *D,
   }
 
   D->addAttr(::new (S.Context) ArgumentWithTypeTagAttr(
-      S.Context, AL, AL.getArgAsIdent(0)->Ident, ArgumentIdx, TypeTagIdx,
-      IsPointer));
+      S.Context, AL, AL.getArgAsIdent(0)->getIdentifierInfo(), ArgumentIdx,
+      TypeTagIdx, IsPointer));
 }
 
 static void handleTypeTagForDatatypeAttr(Sema &S, Decl *D,
@@ -5748,7 +5751,7 @@ static void handleTypeTagForDatatypeAttr(Sema &S, Decl *D,
     return;
   }
 
-  IdentifierInfo *PointerKind = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *PointerKind = AL.getArgAsIdent(0)->getIdentifierInfo();
   TypeSourceInfo *MatchingCTypeLoc = nullptr;
   S.GetTypeFromParser(AL.getMatchingCType(), &MatchingCTypeLoc);
   assert(MatchingCTypeLoc && "no type source info for attribute argument");
@@ -5819,7 +5822,7 @@ static void handleBuiltinAliasAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Ident = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *Ident = AL.getArgAsIdent(0)->getIdentifierInfo();
   unsigned BuiltinID = Ident->getBuiltinID();
   StringRef AliasName = cast<FunctionDecl>(D)->getIdentifier()->getName();
 
@@ -6585,7 +6588,7 @@ static void handleCFGuardAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 
   CFGuardAttr::GuardArg Arg;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   if (!CFGuardAttr::ConvertStrToGuardArg(II->getName(), Arg)) {
     S.Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
     return;
@@ -6687,8 +6690,9 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
     if (!VTablePointerAuthenticationAttr::ConvertStrToVPtrAuthKeyType(
-            IL->Ident->getName(), KeyType)) {
-      S.Diag(IL->Loc, diag::err_invalid_authentication_key) << IL->Ident;
+            IL->getIdentifierInfo()->getName(), KeyType)) {
+      S.Diag(IL->getLoc(), diag::err_invalid_authentication_key)
+          << IL->getIdentifierInfo();
       AL.setInvalid();
     }
     if (KeyType == VTablePointerAuthenticationAttr::DefaultKey &&
@@ -6708,15 +6712,16 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
     if (AL.isArgIdent(1)) {
       IdentifierLoc *IL = AL.getArgAsIdent(1);
       if (!VTablePointerAuthenticationAttr::
-              ConvertStrToAddressDiscriminationMode(IL->Ident->getName(),
-                                                    AddressDiversityMode)) {
-        S.Diag(IL->Loc, diag::err_invalid_address_discrimination) << IL->Ident;
+              ConvertStrToAddressDiscriminationMode(
+                  IL->getIdentifierInfo()->getName(), AddressDiversityMode)) {
+        S.Diag(IL->getLoc(), diag::err_invalid_address_discrimination)
+            << IL->getIdentifierInfo();
         AL.setInvalid();
       }
       if (AddressDiversityMode ==
               VTablePointerAuthenticationAttr::DefaultAddressDiscrimination &&
           !S.getLangOpts().PointerAuthCalls) {
-        S.Diag(IL->Loc, diag::err_no_default_vtable_pointer_auth) << 1;
+        S.Diag(IL->getLoc(), diag::err_no_default_vtable_pointer_auth) << 1;
         AL.setInvalid();
       }
     } else {
@@ -6731,8 +6736,9 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
     if (AL.isArgIdent(2)) {
       IdentifierLoc *IL = AL.getArgAsIdent(2);
       if (!VTablePointerAuthenticationAttr::ConvertStrToExtraDiscrimination(
-              IL->Ident->getName(), ED)) {
-        S.Diag(IL->Loc, diag::err_invalid_extra_discrimination) << IL->Ident;
+              IL->getIdentifierInfo()->getName(), ED)) {
+        S.Diag(IL->getLoc(), diag::err_invalid_extra_discrimination)
+            << IL->getIdentifierInfo();
         AL.setInvalid();
       }
       if (ED == VTablePointerAuthenticationAttr::DefaultExtraDiscrimination &&
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index ba9d3dcf19617..0a14ce23a396e 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -1310,24 +1310,26 @@ static bool NestedProtocolHasNoDefinition(ObjCProtocolDecl *PDecl,
 /// protocol declarations in its 'Protocols' argument.
 void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
                                        bool ForObjCContainer,
-                                       ArrayRef<IdentifierLocPair> ProtocolId,
+                                       ArrayRef<IdentifierLoc> ProtocolId,
                                        SmallVectorImpl<Decl *> &Protocols) {
-  for (const IdentifierLocPair &Pair : ProtocolId) {
-    ObjCProtocolDecl *PDecl = LookupProtocol(Pair.first, Pair.second);
+  for (const IdentifierLoc &Pair : ProtocolId) {
+    ObjCProtocolDecl *PDecl =
+        LookupProtocol(Pair.getIdentifierInfo(), Pair.getLoc());
     if (!PDecl) {
       DeclFilterCCC<ObjCProtocolDecl> CCC{};
-      TypoCorrection Corrected =
-          SemaRef.CorrectTypo(DeclarationNameInfo(Pair.first, Pair.second),
-                              Sema::LookupObjCProtocolName, SemaRef.TUScope,
-                              nullptr, CCC, Sema::CTK_ErrorRecovery);
+      TypoCorrection Corrected = SemaRef.CorrectTypo(
+          DeclarationNameInfo(Pair.getIdentifierInfo(), Pair.getLoc()),
+          Sema::LookupObjCProtocolName, SemaRef.TUScope, nullptr, CCC,
+          Sema::CTK_ErrorRecovery);
       if ((PDecl = Corrected.getCorrectionDeclAs<ObjCProtocolDecl>()))
         SemaRef.diagnoseTypo(Corrected,
                              PDiag(diag::err_undeclared_protocol_suggest)
-                                 << Pair.first);
+                                 << Pair.getIdentifierInfo());
     }
 
     if (!PDecl) {
-      Diag(Pair.second, diag::err_undeclared_protocol) << Pair.first;
+      Diag(Pair.getLoc(), diag::err_undeclared_protocol)
+          << Pair.getIdentifierInfo();
       continue;
     }
     // If this is a forward protocol declaration, get its definition.
@@ -1337,7 +1339,7 @@ void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
     // For an objc container, delay protocol reference checking until after we
     // can set the objc decl as the availability context, otherwise check now.
     if (!ForObjCContainer) {
-      (void)SemaRef.DiagnoseUseOfDecl(PDecl, Pair.second);
+      (void)SemaRef.DiagnoseUseOfDecl(PDecl, Pair.getLoc());
     }
 
     // If this is a forward declaration and we are supposed to warn in this
@@ -1347,7 +1349,8 @@ void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
 
     if (WarnOnDeclarations &&
         NestedProtocolHasNoDefinition(PDecl, UndefinedProtocol)) {
-      Diag(Pair.second, diag::warn_undef_protocolref) << Pair.first;
+      Diag(Pair.getLoc(), diag::warn_undef_protocolref)
+          << Pair.getIdentifierInfo();
       Diag(UndefinedProtocol->getLocation(), diag::note_protocol_decl_undefined)
         << UndefinedProtocol;
     }
@@ -1784,17 +1787,17 @@ void SemaObjC::DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
 
 /// ActOnForwardProtocolDeclaration - Handle \@protocol foo;
 SemaObjC::DeclGroupPtrTy SemaObjC::ActOnForwardProtocolDeclaration(
-    SourceLocation AtProtocolLoc, ArrayRef<IdentifierLocPair> IdentList,
+    SourceLocation AtProtocolLoc, ArrayRef<IdentifierLoc> IdentList,
     const ParsedAttributesView &attrList) {
   ASTContext &Context = getASTContext();
   SmallVector<Decl *, 8> DeclsInGroup;
-  for (const IdentifierLocPair &IdentPair : IdentList) {
-    IdentifierInfo *Ident = IdentPair.first;
+  for (const IdentifierLoc &IdentPair : IdentList) {
+    IdentifierInfo *Ident = IdentPair.getIdentifierInfo();
     ObjCProtocolDecl *PrevDecl = LookupProtocol(
-        Ident, IdentPair.second, SemaRef.forRedeclarationInCurContext());
+        Ident, IdentPair.getLoc(), SemaRef.forRedeclarationInCurContext());
     ObjCProtocolDecl *PDecl =
         ObjCProtocolDecl::Create(Context, SemaRef.CurContext, Ident,
-                                 IdentPair.second, AtProtocolLoc, PrevDecl);
+                                 IdentPair.getLoc(), AtProtocolLoc, PrevDecl);
 
     SemaRef.PushOnScopeChains(PDecl, SemaRef.TUScope);
     CheckObjCDeclScope(PDecl);
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 0b442b75d174d..11f156ae09216 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -1274,8 +1274,8 @@ bool SemaHLSL::handleResourceTypeAttr(QualType T, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *Loc = AL.getArgAsIdent(0);
-    StringRef Identifier = Loc->Ident->getName();
-    SourceLocation ArgLoc = Loc->Loc;
+    StringRef Identifier = Loc->getIdentifierInfo()->getName();
+    SourceLocation ArgLoc = Loc->getLoc();
 
     // Validate resource class value
     ResourceClass RC;
@@ -1534,8 +1534,8 @@ void SemaHLSL::handleResourceBindingAttr(Decl *TheDecl, const ParsedAttr &AL) {
   }
 
   IdentifierLoc *Loc = AL.getArgAsIdent(0);
-  StringRef Str = Loc->Ident->getName();
-  SourceLocation ArgLoc = Loc->Loc;
+  StringRef Str = Loc->getIdentifierInfo()->getName();
+  SourceLocation ArgLoc = Loc->getLoc();
 
   SourceLocation SpaceArgLoc;
   bool SpecifiedSpace = false;
@@ -1549,8 +1549,8 @@ void SemaHLSL::handleResourceBindingAttr(Decl *TheDecl, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *Loc = AL.getArgAsIdent(1);
-    Space = Loc->Ident->getName();
-    SpaceArgLoc = Loc->Loc;
+    Space = Loc->getIdentifierInfo()->getName();
+    SpaceArgLoc = Loc->getLoc();
   } else {
     Slot = Str;
   }
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index 76589bff40be9..4bba57193ded6 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/StringExtras.h"
 
@@ -68,7 +69,7 @@ static std::string stringFromPath(ModuleIdPath Path) {
   for (auto &Piece : Path) {
     if (!Name.empty())
       Name += ".";
-    Name += Piece.first->getName();
+    Name += Piece.getIdentifierInfo()->getName();
   }
   return Name;
 }
@@ -350,17 +351,18 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
 
   // Test the first part of the path to see if it's std[0-9]+ but allow the
   // name in a system header.
-  StringRef FirstComponentName = Path[0].first->getName();
-  if (!getSourceManager().isInSystemHeader(Path[0].second) &&
+  StringRef FirstComponentName = Path[0].getIdentifierInfo()->getName();
+  if (!getSourceManager().isInSystemHeader(Path[0].getLoc()) &&
       (FirstComponentName == "std" ||
        (FirstComponentName.starts_with("std") &&
         llvm::all_of(FirstComponentName.drop_front(3), &llvm::isDigit))))
-    Diag(Path[0].second, diag::warn_reserved_module_name) << Path[0].first;
+    Diag(Path[0].getLoc(), diag::warn_reserved_module_name)
+        << Path[0].getIdentifierInfo();
 
   // Then test all of the components in the path to see if any of them are
   // using another kind of reserved or invalid identifier.
   for (auto Part : Path) {
-    if (DiagReservedModuleName(*this, Part.first, Part.second))
+    if (DiagReservedModuleName(*this, Part.getIdentifierInfo(), Part.getLoc()))
       return nullptr;
   }
 
@@ -376,10 +378,10 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
   // correct.
   if (!getLangOpts().CurrentModule.empty() &&
       getLangOpts().CurrentModule != ModuleName) {
-    Diag(Path.front().second, diag::err_current_module_name_mismatch)
-        << SourceRange(Path.front().second, IsPartition
-                                                ? Partition.back().second
-                                                : Path.back().second)
+    Diag(Path.front().getLoc(), diag::err_current_module_name_mismatch)
+        << SourceRange(Path.front().getLoc(), IsPartition
+                                                  ? Partition.back().getLoc()
+                                                  : Path.back().getLoc())
         << getLangOpts().CurrentModule;
     return nullptr;
   }
@@ -394,7 +396,7 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
     // We can't have parsed or imported a definition of this module or parsed a
     // module map defining it already.
     if (auto *M = Map.findModule(ModuleName)) {
-      Diag(Path[0].second, diag::err_module_redefinition) << ModuleName;
+      Diag(Path[0].getLoc(), diag::err_module_redefinition) << ModuleName;
       if (M->DefinitionLoc.isValid())
         Diag(M->DefinitionLoc, diag::note_prev_module_definition);
       else if (OptionalFileEntryRef FE = M->getASTFile())
@@ -417,8 +419,8 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
     // keyword nor a module-partition implicitly imports the primary
     // module interface unit of the module as if by a module-import-
     // declaration.
-    std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc(
-        PP.getIdentifierInfo(ModuleName), Path[0].second);
+    IdentifierLoc ModuleNameLoc(Path[0].getLoc(),
+                                PP.getIdentifierInfo(ModuleName));
 
     // The module loader will assume we're trying to import the module that
     // we're building if `LangOpts.CurrentModule` equals to 'ModuleName'.
@@ -490,7 +492,7 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
 
     // Make the import decl for the interface in the impl module.
     ImportDecl *Import = ImportDecl::Create(Context, CurContext, ModuleLoc,
-                                            Interface, Path[0].second);
+                                            Interface, Path[0].getLoc());
     CurContext->addDecl(Import);
 
     // Sequence initialization of the imported module before that of the current
@@ -579,7 +581,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
 
   // For a C++20 module name, flatten into a single identifier with the source
   // location of the first component.
-  std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc;
+  IdentifierLoc ModuleNameLoc;
 
   std::string ModuleName;
   if (IsPartition) {
@@ -591,11 +593,13 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
     ModuleName = NamedMod->getPrimaryModuleInterfaceName().str();
     ModuleName += ":";
     ModuleName += stringFromPath(Path);
-    ModuleNameLoc = {PP.getIdentifierInfo(ModuleName), Path[0].second};
+    ModuleNameLoc =
+        IdentifierLoc(Path[0].getLoc(), PP.getIdentifierInfo(ModuleName));
     Path = ModuleIdPath(ModuleNameLoc);
   } else if (getLangOpts().CPlusPlusModules) {
     ModuleName = stringFromPath(Path);
-    ModuleNameLoc = {PP.getIdentifierInfo(ModuleName), Path[0].second};
+    ModuleNameLoc =
+        IdentifierLoc(Path[0].getLoc(), PP.getIdentifierInfo(ModuleName));
     Path = ModuleIdPath(ModuleNameLoc);
   }
 
@@ -680,7 +684,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
       IdentifierLocs.push_back(SourceLocation());
   } else if (getLangOpts().CPlusPlusModules && !Mod->Parent) {
     // A single identifier for the whole name.
-    IdentifierLocs.push_back(Path[0].second);
+    IdentifierLocs.push_back(Path[0].getLoc());
   } else {
     Module *ModCheck = Mod;
     for (unsigned I = 0, N = Path.size(); I != N; ++I) {
@@ -690,7 +694,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
         break;
       ModCheck = ModCheck->Parent;
 
-      IdentifierLocs.push_back(Path[I].second);
+      IdentifierLocs.push_back(Path[I].getLoc());
     }
   }
 
@@ -707,7 +711,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
   if (getLangOpts().CPlusPlusModules && ExportLoc.isValid() &&
       Mod->Kind == Module::ModuleKind::ModulePartitionImplementation) {
     Diag(ExportLoc, diag::err_export_partition_impl)
-        << SourceRange(ExportLoc, Path.back().second);
+        << SourceRange(ExportLoc, Path.back().getLoc());
   } else if (!ModuleScopes.empty() && !currentModuleIsImplementation()) {
     // Re-export the module if the imported module is exported.
     // Note that we don't need to add re-exported module to Imports field
diff --git a/clang/lib/Sema/SemaObjC.cpp b/clang/lib/Sema/SemaObjC.cpp
index 073d9791d037b..9b24b5f052119 100644
--- a/clang/lib/Sema/SemaObjC.cpp
+++ b/clang/lib/Sema/SemaObjC.cpp
@@ -1446,10 +1446,8 @@ SemaObjC::ObjCSubscriptKind SemaObjC::CheckSubscriptingKind(Expr *FromE) {
 
 void SemaObjC::AddCFAuditedAttribute(Decl *D) {
   ASTContext &Context = getASTContext();
-  IdentifierInfo *Ident;
-  SourceLocation Loc;
-  std::tie(Ident, Loc) = SemaRef.PP.getPragmaARCCFCodeAuditedInfo();
-  if (!Loc.isValid())
+  auto IdLoc = SemaRef.PP.getPragmaARCCFCodeAuditedInfo();
+  if (!IdLoc.getLoc().isValid())
     return;
 
   // Don't add a redundant or conflicting attribute.
@@ -1457,7 +1455,8 @@ void SemaObjC::AddCFAuditedAttribute(Decl *D) {
       D->hasAttr<CFUnknownTransferAttr>())
     return;
 
-  AttributeCommonInfo Info(Ident, SourceRange(Loc),
+  AttributeCommonInfo Info(IdLoc.getIdentifierInfo(),
+                           SourceRange(IdLoc.getLoc()),
                            AttributeCommonInfo::Form::Pragma());
   D->addAttr(CFAuditedTransferAttr::CreateImplicit(Context, Info));
 }
@@ -1642,8 +1641,10 @@ void SemaObjC::handleMethodFamilyAttr(Decl *D, const ParsedAttr &AL) {
 
   IdentifierLoc *IL = AL.getArgAsIdent(0);
   ObjCMethodFamilyAttr::FamilyKind F;
-  if (!ObjCMethodFamilyAttr::ConvertStrToFamilyKind(IL->Ident->getName(), F)) {
-    Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL << IL->Ident;
+  if (!ObjCMethodFamilyAttr::ConvertStrToFamilyKind(
+          IL->getIdentifierInfo()->getName(), F)) {
+    Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
+        << AL << IL->getIdentifierInfo();
     return;
   }
 
@@ -1706,7 +1707,7 @@ void SemaObjC::handleBlocksAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   BlocksAttr::BlockType type;
   if (!BlocksAttr::ConvertStrToBlockType(II->getName(), type)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
@@ -1998,7 +1999,7 @@ void SemaObjC::handleNSErrorDomain(Decl *D, const ParsedAttr &Attr) {
 
   IdentifierLoc *IdentLoc =
       Attr.isArgIdent(0) ? Attr.getArgAsIdent(0) : nullptr;
-  if (!IdentLoc || !IdentLoc->Ident) {
+  if (!IdentLoc || !IdentLoc->getIdentifierInfo()) {
     // Try to locate the argument directly.
     SourceLocation Loc = Attr.getLoc();
     if (Attr.isArgExpr(0) && Attr.getArgAsExpr(0))
@@ -2009,18 +2010,18 @@ void SemaObjC::handleNSErrorDomain(Decl *D, const ParsedAttr &Attr) {
   }
 
   // Verify that the identifier is a valid decl in the C decl namespace.
-  LookupResult Result(SemaRef, DeclarationName(IdentLoc->Ident),
+  LookupResult Result(SemaRef, DeclarationName(IdentLoc->getIdentifierInfo()),
                       SourceLocation(),
                       Sema::LookupNameKind::LookupOrdinaryName);
   if (!SemaRef.LookupName(Result, SemaRef.TUScope) ||
       !Result.getAsSingle<VarDecl>()) {
-    Diag(IdentLoc->Loc, diag::err_nserrordomain_invalid_decl)
-        << 1 << IdentLoc->Ident;
+    Diag(IdentLoc->getLoc(), diag::err_nserrordomain_invalid_decl)
+        << 1 << IdentLoc->getIdentifierInfo();
     return;
   }
 
-  D->addAttr(::new (getASTContext())
-                 NSErrorDomainAttr(getASTContext(), Attr, IdentLoc->Ident));
+  D->addAttr(::new (getASTContext()) NSErrorDomainAttr(
+      getASTContext(), Attr, IdentLoc->getIdentifierInfo()));
 }
 
 void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
@@ -2033,7 +2034,7 @@ void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
 
   // Typedefs only allow objc_bridge(id) and have some additional checking.
   if (const auto *TD = dyn_cast<TypedefNameDecl>(D)) {
-    if (!Parm->Ident->isStr("id")) {
+    if (!Parm->getIdentifierInfo()->isStr("id")) {
       Diag(AL.getLoc(), diag::err_objc_attr_typedef_not_id) << AL;
       return;
     }
@@ -2046,8 +2047,8 @@ void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
     }
   }
 
-  D->addAttr(::new (getASTContext())
-                 ObjCBridgeAttr(getASTContext(), AL, Parm->Ident));
+  D->addAttr(::new (getASTContext()) ObjCBridgeAttr(getASTContext(), AL,
+                                                    Parm->getIdentifierInfo()));
 }
 
 void SemaObjC::handleBridgeMutableAttr(Decl *D, const ParsedAttr &AL) {
@@ -2058,21 +2059,21 @@ void SemaObjC::handleBridgeMutableAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  D->addAttr(::new (getASTContext())
-                 ObjCBridgeMutableAttr(getASTContext(), AL, Parm->Ident));
+  D->addAttr(::new (getASTContext()) ObjCBridgeMutableAttr(
+      getASTContext(), AL, Parm->getIdentifierInfo()));
 }
 
 void SemaObjC::handleBridgeRelatedAttr(Decl *D, const ParsedAttr &AL) {
   IdentifierInfo *RelatedClass =
-      AL.isArgIdent(0) ? AL.getArgAsIdent(0)->Ident : nullptr;
+      AL.isArgIdent(0) ? AL.getArgAsIdent(0)->getIdentifierInfo() : nullptr;
   if (!RelatedClass) {
     Diag(D->getBeginLoc(), diag::err_objc_attr_not_id) << AL << 0;
     return;
   }
   IdentifierInfo *ClassMethod =
-      AL.getArgAsIdent(1) ? AL.getArgAsIdent(1)->Ident : nullptr;
+      AL.getArgAsIdent(1) ? AL.getArgAsIdent(1)->getIdentifierInfo() : nullptr;
   IdentifierInfo *InstanceMethod =
-      AL.getArgAsIdent(2) ? AL.getArgAsIdent(2)->Ident : nullptr;
+      AL.getArgAsIdent(2) ? AL.getArgAsIdent(2)->getIdentifierInfo() : nullptr;
   D->addAttr(::new (getASTContext()) ObjCBridgeRelatedAttr(
       getASTContext(), AL, RelatedClass, ClassMethod, InstanceMethod));
 }
diff --git a/clang/lib/Sema/SemaOpenACCClause.cpp b/clang/lib/Sema/SemaOpenACCClause.cpp
index ab25dcfd1a081..049baead031a1 100644
--- a/clang/lib/Sema/SemaOpenACCClause.cpp
+++ b/clang/lib/Sema/SemaOpenACCClause.cpp
@@ -1343,7 +1343,7 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
   // the limitation, since the Dialect requires this.
   if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Set &&
       Clause.getDeviceTypeArchitectures().size() > 1) {
-    SemaRef.Diag(Clause.getDeviceTypeArchitectures()[1].second,
+    SemaRef.Diag(Clause.getDeviceTypeArchitectures()[1].getLoc(),
                  diag::err_acc_device_type_multiple_archs);
     return nullptr;
   }
@@ -1369,16 +1369,17 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
   bool Diagnosed = false;
   auto FilterPred = [&](const DeviceTypeArgument &Arch) {
     // The '*' case.
-    if (!Arch.first)
+    if (!Arch.getIdentifierInfo())
       return false;
     return llvm::find_if(ValidValues, [&](StringRef RHS) {
-             return Arch.first->getName().equals_insensitive(RHS);
+             return Arch.getIdentifierInfo()->getName().equals_insensitive(RHS);
            }) == ValidValues.end();
   };
 
   auto Diagnose = [&](const DeviceTypeArgument &Arch) {
-    Diagnosed = SemaRef.Diag(Arch.second, diag::err_acc_invalid_default_type)
-                << Arch.first << Clause.getClauseKind() << ValidValuesString;
+    Diagnosed = SemaRef.Diag(Arch.getLoc(), diag::err_acc_invalid_default_type)
+                << Arch.getIdentifierInfo() << Clause.getClauseKind()
+                << ValidValuesString;
   };
 
   // There aren't stable enumertor versions of 'for-each-then-erase', so do it
diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index 2f719c6d7a21e..a09626c3a9a8c 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -79,9 +79,10 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
   Expr *ValueExpr = A.getArgAsExpr(3);
 
   StringRef PragmaName =
-      llvm::StringSwitch<StringRef>(PragmaNameLoc->Ident->getName())
+      llvm::StringSwitch<StringRef>(
+          PragmaNameLoc->getIdentifierInfo()->getName())
           .Cases("unroll", "nounroll", "unroll_and_jam", "nounroll_and_jam",
-                 PragmaNameLoc->Ident->getName())
+                 PragmaNameLoc->getIdentifierInfo()->getName())
           .Default("clang loop");
 
   // This could be handled automatically by adding a Subjects definition in
@@ -127,10 +128,10 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
       SetHints(LoopHintAttr::UnrollAndJam, LoopHintAttr::Enable);
   } else {
     // #pragma clang loop ...
-    assert(OptionLoc && OptionLoc->Ident &&
+    assert(OptionLoc && OptionLoc->getIdentifierInfo() &&
            "Attribute must have valid option info.");
     Option = llvm::StringSwitch<LoopHintAttr::OptionType>(
-                 OptionLoc->Ident->getName())
+                 OptionLoc->getIdentifierInfo()->getName())
                  .Case("vectorize", LoopHintAttr::Vectorize)
                  .Case("vectorize_width", LoopHintAttr::VectorizeWidth)
                  .Case("interleave", LoopHintAttr::Interleave)
@@ -144,12 +145,13 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                  .Case("distribute", LoopHintAttr::Distribute)
                  .Default(LoopHintAttr::Vectorize);
     if (Option == LoopHintAttr::VectorizeWidth) {
-      assert((ValueExpr || (StateLoc && StateLoc->Ident)) &&
+      assert((ValueExpr || (StateLoc && StateLoc->getIdentifierInfo())) &&
              "Attribute must have a valid value expression or argument.");
       if (ValueExpr && S.CheckLoopHintExpr(ValueExpr, St->getBeginLoc(),
                                            /*AllowZero=*/false))
         return nullptr;
-      if (StateLoc && StateLoc->Ident && StateLoc->Ident->isStr("scalable"))
+      if (StateLoc && StateLoc->getIdentifierInfo() &&
+          StateLoc->getIdentifierInfo()->isStr("scalable"))
         State = LoopHintAttr::ScalableWidth;
       else
         State = LoopHintAttr::FixedWidth;
@@ -167,14 +169,15 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                Option == LoopHintAttr::Unroll ||
                Option == LoopHintAttr::Distribute ||
                Option == LoopHintAttr::PipelineDisabled) {
-      assert(StateLoc && StateLoc->Ident && "Loop hint must have an argument");
-      if (StateLoc->Ident->isStr("disable"))
+      assert(StateLoc && StateLoc->getIdentifierInfo() &&
+             "Loop hint must have an argument");
+      if (StateLoc->getIdentifierInfo()->isStr("disable"))
         State = LoopHintAttr::Disable;
-      else if (StateLoc->Ident->isStr("assume_safety"))
+      else if (StateLoc->getIdentifierInfo()->isStr("assume_safety"))
         State = LoopHintAttr::AssumeSafety;
-      else if (StateLoc->Ident->isStr("full"))
+      else if (StateLoc->getIdentifierInfo()->isStr("full"))
         State = LoopHintAttr::Full;
-      else if (StateLoc->Ident->isStr("enable"))
+      else if (StateLoc->getIdentifierInfo()->isStr("enable"))
         State = LoopHintAttr::Enable;
       else
         llvm_unreachable("bad loop hint argument");
@@ -644,8 +647,8 @@ static Attr *handleAtomicAttr(Sema &S, Stmt *St, const ParsedAttr &AL,
     }
 
     IdentifierLoc *Ident = AL.getArgAsIdent(ArgIndex);
-    OptionString = Ident->Ident->getName();
-    Loc = Ident->Loc;
+    OptionString = Ident->getIdentifierInfo()->getName();
+    Loc = Ident->getLoc();
     if (!AtomicAttr::ConvertStrToConsumedOption(OptionString, Option)) {
       S.Diag(Loc, diag::err_attribute_invalid_atomic_argument) << OptionString;
       return nullptr;
diff --git a/clang/lib/Sema/SemaSwift.cpp b/clang/lib/Sema/SemaSwift.cpp
index fe72d6c85c37a..4aae855a24b8f 100644
--- a/clang/lib/Sema/SemaSwift.cpp
+++ b/clang/lib/Sema/SemaSwift.cpp
@@ -148,8 +148,8 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
       return true;
 
     S.Diag(AL.getLoc(), diag::err_attr_swift_error_return_type)
-        << AL << AL.getArgAsIdent(0)->Ident->getName() << isa<ObjCMethodDecl>(D)
-        << /*pointer*/ 1;
+        << AL << AL.getArgAsIdent(0)->getIdentifierInfo()->getName()
+        << isa<ObjCMethodDecl>(D) << /*pointer*/ 1;
     return false;
   };
 
@@ -159,8 +159,8 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
       return true;
 
     S.Diag(AL.getLoc(), diag::err_attr_swift_error_return_type)
-        << AL << AL.getArgAsIdent(0)->Ident->getName() << isa<ObjCMethodDecl>(D)
-        << /*integral*/ 0;
+        << AL << AL.getArgAsIdent(0)->getIdentifierInfo()->getName()
+        << isa<ObjCMethodDecl>(D) << /*integral*/ 0;
     return false;
   };
 
@@ -169,10 +169,10 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
 
   IdentifierLoc *Loc = AL.getArgAsIdent(0);
   SwiftErrorAttr::ConventionKind Convention;
-  if (!SwiftErrorAttr::ConvertStrToConventionKind(Loc->Ident->getName(),
-                                                  Convention)) {
+  if (!SwiftErrorAttr::ConvertStrToConventionKind(
+          Loc->getIdentifierInfo()->getName(), Convention)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported)
-        << AL << Loc->Ident;
+        << AL << Loc->getIdentifierInfo();
     return;
   }
 
@@ -287,10 +287,10 @@ static void checkSwiftAsyncErrorBlock(Sema &S, Decl *D,
 void SemaSwift::handleAsyncError(Decl *D, const ParsedAttr &AL) {
   IdentifierLoc *IDLoc = AL.getArgAsIdent(0);
   SwiftAsyncErrorAttr::ConventionKind ConvKind;
-  if (!SwiftAsyncErrorAttr::ConvertStrToConventionKind(IDLoc->Ident->getName(),
-                                                       ConvKind)) {
+  if (!SwiftAsyncErrorAttr::ConvertStrToConventionKind(
+          IDLoc->getIdentifierInfo()->getName(), ConvKind)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported)
-        << AL << IDLoc->Ident;
+        << AL << IDLoc->getIdentifierInfo();
     return;
   }
 
@@ -643,7 +643,7 @@ void SemaSwift::handleNewType(Decl *D, const ParsedAttr &AL) {
   }
 
   SwiftNewTypeAttr::NewtypeKind Kind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   if (!SwiftNewTypeAttr::ConvertStrToNewtypeKind(II->getName(), Kind)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
     return;
@@ -667,7 +667,7 @@ void SemaSwift::handleAsyncAttr(Decl *D, const ParsedAttr &AL) {
   }
 
   SwiftAsyncAttr::Kind Kind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   if (!SwiftAsyncAttr::ConvertStrToKind(II->getName(), Kind)) {
     Diag(AL.getLoc(), diag::err_swift_async_no_access) << AL << II;
     return;
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index dc7e3a0bf8875..87682233c5246 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -14,6 +14,7 @@
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Sema/Lookup.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
@@ -755,7 +756,7 @@ bool Sema::CheckParameterPacksForExpansion(
     bool &RetainExpansion, UnsignedOrNone &NumExpansions) {
   ShouldExpand = true;
   RetainExpansion = false;
-  std::pair<IdentifierInfo *, SourceLocation> FirstPack;
+  IdentifierLoc FirstPack;
   bool HaveFirstPack = false;
   UnsignedOrNone NumPartialExpansions = std::nullopt;
   SourceLocation PartiallySubstitutedPackLoc;
@@ -867,8 +868,7 @@ bool Sema::CheckParameterPacksForExpansion(
       // This is the first pack we've seen for which we have an argument.
       // Record it.
       NumExpansions = NewPackSize;
-      FirstPack.first = Name;
-      FirstPack.second = ParmPack.second;
+      FirstPack = IdentifierLoc(ParmPack.second, Name);
       HaveFirstPack = true;
       continue;
     }
@@ -905,9 +905,9 @@ bool Sema::CheckParameterPacksForExpansion(
       //   the same number of arguments specified.
       if (HaveFirstPack)
         Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict)
-            << FirstPack.first << Name << *NumExpansions
+            << FirstPack.getIdentifierInfo() << Name << *NumExpansions
             << (LeastNewPackSize != NewPackSize) << LeastNewPackSize
-            << SourceRange(FirstPack.second) << SourceRange(ParmPack.second);
+            << SourceRange(FirstPack.getLoc()) << SourceRange(ParmPack.second);
       else
         Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict_multilevel)
             << Name << *NumExpansions << (LeastNewPackSize != NewPackSize)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index eba7267904fb2..6e7ee8b5506ff 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -99,8 +99,8 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
   StringRef name = attr.getAttrName()->getName();
 
   // The GC attributes are usually written with macros;  special-case them.
-  IdentifierInfo *II = attr.isArgIdent(0) ? attr.getArgAsIdent(0)->Ident
-                                          : nullptr;
+  IdentifierInfo *II =
+      attr.isArgIdent(0) ? attr.getArgAsIdent(0)->getIdentifierInfo() : nullptr;
   if (useExpansionLoc && loc.isMacroID() && II) {
     if (II->isStr("strong")) {
       if (S.findMacroSpelling(loc, "__strong")) name = "__strong";
@@ -5732,8 +5732,7 @@ static void transferARCOwnershipToDeclaratorChunk(TypeProcessingState &state,
   }
 
   IdentifierLoc *Arg = new (S.Context) IdentifierLoc;
-  Arg->Ident = &S.Context.Idents.get(attrStr);
-  Arg->Loc = SourceLocation();
+  Arg->setIdentifierInfo(&S.Context.Idents.get(attrStr));
 
   ArgsUnion Args(Arg);
 
@@ -6633,7 +6632,7 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
     return true;
   }
 
-  IdentifierInfo *II = attr.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = attr.getArgAsIdent(0)->getIdentifierInfo();
   Qualifiers::ObjCLifetime lifetime;
   if (II->isStr("none"))
     lifetime = Qualifiers::OCL_ExplicitNone;
@@ -6811,7 +6810,7 @@ static bool handleObjCGCTypeAttr(TypeProcessingState &state, ParsedAttr &attr,
     return true;
   }
 
-  IdentifierInfo *II = attr.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = attr.getArgAsIdent(0)->getIdentifierInfo();
   if (II->isStr("weak"))
     GCAttr = Qualifiers::Weak;
   else if (II->isStr("strong"))
@@ -7541,7 +7540,7 @@ static Attr *getCCTypeAttr(ASTContext &Ctx, ParsedAttr &Attr) {
     if (Attr.isArgExpr(0))
       Str = cast<StringLiteral>(Attr.getArgAsExpr(0))->getString();
     else
-      Str = Attr.getArgAsIdent(0)->Ident->getName();
+      Str = Attr.getArgAsIdent(0)->getIdentifierInfo()->getName();
     PcsAttr::PCSType Type;
     if (!PcsAttr::ConvertStrToPCSType(Str, Type))
       llvm_unreachable("already validated the attribute");
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 02c31dff620ec..b404015867087 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12811,7 +12811,7 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     for (unsigned I = 0; I < NumArchs; ++I) {
       IdentifierInfo *Ident = readBool() ? readIdentifier() : nullptr;
       SourceLocation Loc = readSourceLocation();
-      Archs.emplace_back(Ident, Loc);
+      Archs.emplace_back(Loc, Ident);
     }
 
     return OpenACCDeviceTypeClause::Create(getContext(), ClauseKind, BeginLoc,
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 95b5718f1d140..4dca0613cb9ae 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8774,10 +8774,10 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeSourceLocation(DTC->getLParenLoc());
     writeUInt32(DTC->getArchitectures().size());
     for (const DeviceTypeArgument &Arg : DTC->getArchitectures()) {
-      writeBool(Arg.first);
-      if (Arg.first)
-        AddIdentifierRef(Arg.first);
-      writeSourceLocation(Arg.second);
+      writeBool(Arg.getIdentifierInfo());
+      if (Arg.getIdentifierInfo())
+        AddIdentifierRef(Arg.getIdentifierInfo());
+      writeSourceLocation(Arg.getLoc());
     }
     return;
   }
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 429bf823616da..07856dbdba4b4 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -664,7 +664,7 @@ void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
                                         const Module *Imported) {
   if (MDC.ScanInstance.getPreprocessor().isInImportingCXXNamedModules()) {
     P1689ModuleInfo RequiredModule;
-    RequiredModule.ModuleName = Path[0].first->getName().str();
+    RequiredModule.ModuleName = Path[0].getIdentifierInfo()->getName().str();
     RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
     MDC.RequiredStdCXXModules.push_back(RequiredModule);
     return;

From af28c9c65a23806a09d7929792df5ed2e9bdf946 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 16 Apr 2025 06:20:34 -0700
Subject: [PATCH 114/710] [SLP]Do not reorder split node operand with reuses,
 if not possible

Need to check if the operand node of the split vectorize node has reuses
and check if it is possible to build the order for this node to reorder
it correctly.

Fixes #135912
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |   4 +-
 .../X86/split-vector-operand-with-reuses.ll   | 151 ++++++++++++++++++
 2 files changed, 153 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/split-vector-operand-with-reuses.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f9acc276f37f9..f97386159d029 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7479,8 +7479,8 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
         for (const auto &P : Data.first->CombinedEntriesWithIndices) {
           TreeEntry &OpTE = *VectorizableTree[P.first].get();
           OrdersType Order = OpTE.ReorderIndices;
-          if (Order.empty()) {
-            if (!OpTE.isGather())
+          if (Order.empty() || !OpTE.ReuseShuffleIndices.empty()) {
+            if (!OpTE.isGather() && OpTE.ReuseShuffleIndices.empty())
               continue;
             const auto BestOrder =
                 getReorderingData(OpTE, /*TopToBottom=*/false, IgnoreReorder);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-vector-operand-with-reuses.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-vector-operand-with-reuses.ll
new file mode 100644
index 0000000000000..dd804663ff121
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-vector-operand-with-reuses.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(ptr %p) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[ARRAYIDX7_US_I_841:%.*]] = getelementptr i8, ptr [[P]], i64 36
+; CHECK-NEXT:    [[ARRAYIDX7_US_I_1261:%.*]] = getelementptr i8, ptr [[P]], i64 52
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[ARRAYIDX7_US_I_1261]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, <4 x i32> [[TMP2]], i64 4)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr [[ARRAYIDX7_US_I_841]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <12 x i32> @llvm.vector.insert.v12i32.v4i32(<12 x i32> [[TMP6]], <4 x i32> [[TMP5]], i64 8)
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, <16 x i32> [[TMP9]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 18, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[TMP0]], i32 6
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 6, i32 6, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP13:%.*]] = add <16 x i32> [[TMP3]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = srem <16 x i32> [[TMP13]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP15:%.*]] = or <12 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = srem <12 x i32> [[TMP15]], <i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    br label %[[FOR_COND1_PREHEADER_US_I:.*]]
+; CHECK:       [[FOR_COND1_PREHEADER_US_I]]:
+; CHECK-NEXT:    [[A_PROMOTED253537_US_I:%.*]] = phi i32 [ [[OP_RDX8:%.*]], %[[FOR_COND1_PREHEADER_US_I]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP14]])
+; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v12i32(<12 x i32> [[TMP16]])
+; CHECK-NEXT:    [[OP_RDX:%.*]] = add i32 [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[OP_RDX8]] = add i32 [[OP_RDX]], 0
+; CHECK-NEXT:    br label %[[FOR_COND1_PREHEADER_US_I]]
+;
+entry:
+  %arrayidx7.us.i.841 = getelementptr i8, ptr %p, i64 36
+  %arrayidx7.us.i.946 = getelementptr i8, ptr %p, i64 40
+  %arrayidx7.us.i.1051 = getelementptr i8, ptr %p, i64 44
+  %arrayidx7.us.i.1156 = getelementptr i8, ptr %p, i64 48
+  %arrayidx7.us.i.1261 = getelementptr i8, ptr %p, i64 52
+  %arrayidx7.us.i.1366 = getelementptr i8, ptr %p, i64 56
+  %arrayidx7.us.i.1471 = getelementptr i8, ptr %p, i64 60
+  %arrayidx7.us.i.1576 = getelementptr i8, ptr %p, i64 64
+  %add8.us.i.1.4 = add i32 0, 0
+  %rem.us.i.1.4 = srem i32 %add8.us.i.1.4, 1
+  %add8.us.i.1.5 = add i32 0, 0
+  %rem.us.i.1.5 = srem i32 %add8.us.i.1.5, 1
+  %invariant.op91 = add i32 %rem.us.i.1.4, %rem.us.i.1.5
+  %add8.us.i.1.6 = add i32 0, 0
+  %rem.us.i.1.6 = srem i32 %add8.us.i.1.6, 1
+  %invariant.op92 = add i32 %invariant.op91, %rem.us.i.1.6
+  %0 = load i32, ptr %arrayidx7.us.i.841, align 4
+  %1 = load i32, ptr %arrayidx7.us.i.946, align 4
+  %2 = load i32, ptr %arrayidx7.us.i.1051, align 4
+  %3 = load i32, ptr %arrayidx7.us.i.1156, align 4
+  %4 = load i32, ptr %arrayidx7.us.i.1261, align 4
+  %5 = load i32, ptr %arrayidx7.us.i.1366, align 4
+  %add8.us.i.7.6 = or i32 %5, 0
+  %rem.us.i.7.6 = srem i32 %add8.us.i.7.6, 1
+  %6 = load i32, ptr %arrayidx7.us.i.1471, align 4
+  %add8.us.i.7.7 = or i32 %6, 0
+  %rem.us.i.7.7 = srem i32 %add8.us.i.7.7, 1
+  %invariant.op165 = add i32 %rem.us.i.7.6, %rem.us.i.7.7
+  %7 = load i32, ptr %arrayidx7.us.i.1576, align 4
+  %add8.us.i.7.8 = or i32 %7, 0
+  %rem.us.i.7.8 = srem i32 %add8.us.i.7.8, 1
+  %invariant.op166 = add i32 %invariant.op165, %rem.us.i.7.8
+  %add8.us.i.8 = or i32 %0, 0
+  %rem.us.i.8 = srem i32 %add8.us.i.8, 1
+  %invariant.op167 = add i32 %invariant.op166, %rem.us.i.8
+  %add8.us.i.8.1 = or i32 %1, 0
+  %rem.us.i.8.1 = srem i32 %add8.us.i.8.1, 1
+  %invariant.op168 = add i32 %invariant.op167, %rem.us.i.8.1
+  %add8.us.i.8.2 = or i32 %2, 0
+  %rem.us.i.8.2 = srem i32 %add8.us.i.8.2, 1
+  %invariant.op169 = add i32 %invariant.op168, %rem.us.i.8.2
+  %add8.us.i.8.3 = or i32 %3, 0
+  %rem.us.i.8.3 = srem i32 %add8.us.i.8.3, 1
+  %invariant.op170 = add i32 %invariant.op169, %rem.us.i.8.3
+  %add8.us.i.8.4 = or i32 %4, 0
+  %rem.us.i.8.4 = srem i32 %add8.us.i.8.4, 1
+  %invariant.op171 = add i32 %invariant.op170, %rem.us.i.8.4
+  %add8.us.i.8.5 = or i32 %5, 0
+  %rem.us.i.8.5 = srem i32 %add8.us.i.8.5, 1
+  %invariant.op172 = add i32 %invariant.op171, %rem.us.i.8.5
+  %add8.us.i.8.6 = or i32 %6, 0
+  %rem.us.i.8.6 = srem i32 %add8.us.i.8.6, 0
+  %invariant.op173 = add i32 %invariant.op172, %rem.us.i.8.6
+  %add8.us.i.8.7 = or i32 %7, 0
+  %rem.us.i.8.7 = srem i32 %add8.us.i.8.7, 0
+  %invariant.op174 = add i32 %invariant.op173, %rem.us.i.8.7
+  %invariant.op181 = add i32 %invariant.op174, 0
+  %invariant.op182 = add i32 %invariant.op181, 0
+  %invariant.op183 = add i32 %invariant.op182, 0
+  %invariant.op184 = add i32 %invariant.op183, 0
+  %invariant.op185 = add i32 %invariant.op184, 0
+  %invariant.op186 = add i32 %invariant.op185, 0
+  %invariant.op187 = add i32 %invariant.op186, 0
+  %invariant.op188 = add i32 %invariant.op187, 0
+  %add8.us.i.11.1 = or i32 %4, 0
+  %rem.us.i.11.1 = srem i32 %add8.us.i.11.1, 1
+  %invariant.op189 = add i32 %invariant.op188, %rem.us.i.11.1
+  %add8.us.i.11.2 = add i32 0, 0
+  %rem.us.i.11.2 = srem i32 %add8.us.i.11.2, 1
+  %invariant.op190 = add i32 %invariant.op189, %rem.us.i.11.2
+  %add8.us.i.11.3 = add i32 %6, %2
+  %rem.us.i.11.3 = srem i32 %add8.us.i.11.3, 1
+  %invariant.op191 = add i32 %invariant.op190, %rem.us.i.11.3
+  %add8.us.i.11.4 = add i32 %7, %2
+  %rem.us.i.11.4 = srem i32 %add8.us.i.11.4, 1
+  %invariant.op192 = add i32 %invariant.op191, %rem.us.i.11.4
+  %8 = load i32, ptr %p, align 4
+  %add8.us.i.12 = add i32 %4, %8
+  %rem.us.i.12 = srem i32 %add8.us.i.12, 1
+  %invariant.op193 = add i32 %invariant.op192, %rem.us.i.12
+  %add8.us.i.12.1 = add i32 %5, %8
+  %rem.us.i.12.1 = srem i32 %add8.us.i.12.1, 1
+  %invariant.op194 = add i32 %invariant.op193, %rem.us.i.12.1
+  %add8.us.i.12.2 = add i32 0, 0
+  %rem.us.i.12.2 = srem i32 %add8.us.i.12.2, 1
+  %invariant.op195 = add i32 %invariant.op194, %rem.us.i.12.2
+  %add8.us.i.12.3 = add i32 0, 0
+  %rem.us.i.12.3 = srem i32 %add8.us.i.12.3, 1
+  %invariant.op196 = add i32 %invariant.op195, %rem.us.i.12.3
+  %add8.us.i.13 = add i32 0, 0
+  %rem.us.i.13 = srem i32 %add8.us.i.13, 0
+  %invariant.op197 = add i32 %invariant.op196, %rem.us.i.13
+  %add8.us.i.13.1 = add i32 0, 0
+  %rem.us.i.13.1 = srem i32 %add8.us.i.13.1, 0
+  %invariant.op198 = add i32 %invariant.op197, %rem.us.i.13.1
+  %add8.us.i.13.2 = add i32 0, 0
+  %rem.us.i.13.2 = srem i32 %add8.us.i.13.2, 1
+  %invariant.op199 = add i32 %invariant.op198, %rem.us.i.13.2
+  %add8.us.i.14 = add i32 0, 0
+  %rem.us.i.14 = srem i32 %add8.us.i.14, 1
+  %invariant.op200 = add i32 %invariant.op199, %rem.us.i.14
+  %add8.us.i.14.1 = add i32 0, 0
+  %rem.us.i.14.1 = srem i32 %add8.us.i.14.1, 1
+  %invariant.op201 = add i32 %invariant.op200, %rem.us.i.14.1
+  %add8.us.i.15 = add i32 0, 0
+  %rem.us.i.15 = srem i32 %add8.us.i.15, 1
+  %invariant.op202 = add i32 %invariant.op201, %rem.us.i.15
+  br label %for.cond1.preheader.us.i
+
+for.cond1.preheader.us.i:
+  %a.promoted253537.us.i = phi i32 [ %add9.us.i.15.reass, %for.cond1.preheader.us.i ], [ 0, %entry ]
+  %add9.us.i.15.reass = add i32 %invariant.op92, %invariant.op202
+  br label %for.cond1.preheader.us.i
+}
+

From 1bfd44462886b167f0d82e44e6a9856a830c1f8b Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Wed, 16 Apr 2025 06:48:42 -0700
Subject: [PATCH 115/710] [DAGCombiner] Fold and/or of NaN SETCC (#135645)

Fold an AND or OR of two NaN SETCC nodes into a single SETCC where
possible. This optimization already exists in InstCombine but adding in
here as well can allow for additional folding if more logical operations
are exposed.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  6 ++
 llvm/test/CodeGen/NVPTX/and-or-setcc.ll       | 45 +++++++++++++++
 llvm/test/CodeGen/X86/and-or-setcc.ll         | 57 +++++++++++++++++++
 3 files changed, 108 insertions(+)
 create mode 100644 llvm/test/CodeGen/NVPTX/and-or-setcc.ll
 create mode 100644 llvm/test/CodeGen/X86/and-or-setcc.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d72be359867ca..ab8e18267f3f5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -6448,6 +6448,12 @@ static SDValue foldAndOrOfSETCC(SDNode *LogicOp, SelectionDAG &DAG) {
     }
   }
 
+  if (LHS0 == LHS1 && RHS0 == RHS1 && CCL == CCR &&
+      LHS0.getValueType() == RHS0.getValueType() &&
+      ((LogicOp->getOpcode() == ISD::AND && CCL == ISD::SETO) ||
+       (LogicOp->getOpcode() == ISD::OR && CCL == ISD::SETUO)))
+    return DAG.getSetCC(DL, VT, LHS0, RHS0, CCL);
+
   if (TargetPreference == AndOrSETCCFoldKind::None)
     return SDValue();
 
diff --git a/llvm/test/CodeGen/NVPTX/and-or-setcc.ll b/llvm/test/CodeGen/NVPTX/and-or-setcc.ll
new file mode 100644
index 0000000000000..21be9df94d553
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/and-or-setcc.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 | %ptxas-verify %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define i1 @and_ord(float %a, float %b) {
+; CHECK-LABEL: and_ord(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [and_ord_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [and_ord_param_1];
+; CHECK-NEXT:    setp.num.f32 %p1, %f1, %f2;
+; CHECK-NEXT:    selp.b32 %r1, 1, 0, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %c = fcmp ord float %a, 0.0
+  %d = fcmp ord float %b, 0.0
+  %e = and i1 %c, %d
+  ret i1 %e
+}
+
+define i1 @or_uno(float %a, float %b) {
+; CHECK-LABEL: or_uno(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [or_uno_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [or_uno_param_1];
+; CHECK-NEXT:    setp.nan.f32 %p1, %f1, %f2;
+; CHECK-NEXT:    selp.b32 %r1, 1, 0, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %c = fcmp uno float %a, 0.0
+  %d = fcmp uno float %b, 0.0
+  %e = or i1 %c, %d
+  ret i1 %e
+}
diff --git a/llvm/test/CodeGen/X86/and-or-setcc.ll b/llvm/test/CodeGen/X86/and-or-setcc.ll
new file mode 100644
index 0000000000000..cb8ecca9348e6
--- /dev/null
+++ b/llvm/test/CodeGen/X86/and-or-setcc.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64
+
+define i1 @and_ord(float %a, float %b) {
+; X86-LABEL: and_ord:
+; X86:       # %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_ord:
+; X64:       # %bb.0:
+; X64-NEXT:    xorps %xmm2, %xmm2
+; X64-NEXT:    cmpordps %xmm2, %xmm1
+; X64-NEXT:    cmpordps %xmm2, %xmm0
+; X64-NEXT:    andps %xmm1, %xmm0
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %c = fcmp ord float %a, 0.0
+  %d = fcmp ord float %b, 0.0
+  %e = and i1 %c, %d
+  ret i1 %e
+}
+
+define i1 @or_uno(float %a, float %b) {
+; X86-LABEL: or_uno:
+; X86:       # %bb.0:
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: or_uno:
+; X64:       # %bb.0:
+; X64-NEXT:    xorps %xmm2, %xmm2
+; X64-NEXT:    cmpunordps %xmm2, %xmm1
+; X64-NEXT:    cmpunordps %xmm2, %xmm0
+; X64-NEXT:    orps %xmm1, %xmm0
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %c = fcmp uno float %a, 0.0
+  %d = fcmp uno float %b, 0.0
+  %e = or i1 %c, %d
+  ret i1 %e
+}

From 181872ffcc7dc7f20ed2b84e8fa39beba41cb6d3 Mon Sep 17 00:00:00 2001
From: Brox Chen <guochen2@amd.com>
Date: Wed, 16 Apr 2025 10:03:08 -0400
Subject: [PATCH 116/710] [AMDGPU][True16][MC] update a few mc test for true16
 (#135816)

This is another NFC patch.

Update mc test for a few true16 instructions by duplicating the file to
fake16 versions and udpate `mattr` flag with +/-real-true16. Also added
some fake16 file that are not properly created before
---
 llvm/test/MC/AMDGPU/bf16_imm-fake16.s         | 114 ++++++
 llvm/test/MC/AMDGPU/bf16_imm.s                |  64 ++--
 llvm/test/MC/AMDGPU/gfx11-promotions-fake16.s | 353 ++++++++++++++++++
 llvm/test/MC/AMDGPU/gfx11-promotions.s        |  66 ++--
 .../MC/AMDGPU/gfx1150_asm_features-fake16.s   |  48 +++
 llvm/test/MC/AMDGPU/gfx1150_asm_features.s    |  20 +-
 .../MC/AMDGPU/gfx11_asm_vinterp_err-fake16.s  |  43 +++
 llvm/test/MC/AMDGPU/gfx11_asm_vop1.s          |  30 ++
 .../MC/AMDGPU/gfx11_asm_vop3_alias-fake16.s   |  15 +
 llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s    |  12 +-
 llvm/test/MC/AMDGPU/gfx11_asm_vop3_features.s |  77 ++++
 llvm/test/MC/AMDGPU/gfx12_asm_vop1.s          |  31 ++
 12 files changed, 792 insertions(+), 81 deletions(-)
 create mode 100644 llvm/test/MC/AMDGPU/bf16_imm-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11-promotions-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1150_asm_features-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vop3_features.s

diff --git a/llvm/test/MC/AMDGPU/bf16_imm-fake16.s b/llvm/test/MC/AMDGPU/bf16_imm-fake16.s
new file mode 100644
index 0000000000000..ee697bee6ab2d
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/bf16_imm-fake16.s
@@ -0,0 +1,114 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -show-encoding %s | FileCheck %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -show-encoding %s | FileCheck %s
+
+v_dot2_bf16_bf16 v5, v1, v2, 100.0
+// CHECK: v_dot2_bf16_bf16 v5, v1, v2, 0x42c8     ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0xfe,0x03,0xc8,0x42,0x00,0x00]
+
+v_dot2_bf16_bf16 v2, v0, 1.0, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, 1.0, v2        ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe5,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, 1.0, v0, v2
+// CHECK: v_dot2_bf16_bf16 v2, 1.0, v0, v2        ; encoding: [0x02,0x00,0x67,0xd6,0xf2,0x00,0x0a,0x04]
+
+v_dot2_bf16_bf16 v5, v1, v2, 1.0
+// CHECK: v_dot2_bf16_bf16 v5, v1, v2, 1.0        ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0xca,0x03]
+
+v_dot2_bf16_bf16 v2, v0, -1.0, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, -1.0, v2       ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe7,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, 0.5, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, 0.5, v2        ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe1,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, -0.5, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, -0.5, v2       ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe3,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, 2.0, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, 2.0, v2        ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe9,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, -2.0, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, -2.0, v2       ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xeb,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, 4.0, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, 4.0, v2        ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xed,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, -4.0, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, -4.0, v2       ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xef,0x09,0x04]
+
+// Check 1/(2*pi) rounded value and ideomatic fp32 0.15915494 value
+// which cannot be accurately represented in bf16.
+
+v_dot2_bf16_bf16 v2, v0, 0.158203125, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, 0.15915494, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, 0.15915494, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, 0.15915494, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, 0x3e22, v2
+// CHECK: v_dot2_bf16_bf16 v2, v0, 0.15915494, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
+
+v_dot2_bf16_bf16 v2, v0, v2, 0.15915494
+// CHECK: v_dot2_bf16_bf16 v2, v0, v2, 0.15915494 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0x05,0xe2,0x03]
+
+v_dot2_f32_bf16 v2, v1, 0, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, 0, v2           ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0x01,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, 0.5, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, 0.5, v2         ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xe1,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, -0.5, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, -0.5, v2        ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xe3,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, 1.0, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, 1.0, v2         ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xe5,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, -1.0, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, -1.0, v2        ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xe7,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, 2.0, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, 2.0, v2         ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xe9,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, -2.0, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, -2.0, v2        ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xeb,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, 4.0, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, 4.0, v2         ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xed,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, -4.0, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, -4.0, v2        ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xef,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, 0.15915494, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, 0.15915494, v2  ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xf1,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, v1, 0x3e22, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, 0.15915494, v2  ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xf1,0x09,0x1c]
+
+v_dot2_f32_bf16 v2, 0.5, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, 0.5, v1, v2         ; encoding: [0x02,0x40,0x1a,0xcc,0xf0,0x02,0x0a,0x1c]
+
+v_dot2_f32_bf16 v2, -0.5, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, -0.5, v1, v2        ; encoding: [0x02,0x40,0x1a,0xcc,0xf1,0x02,0x0a,0x1c]
+
+v_dot2_f32_bf16 v2, 1.0, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, 1.0, v1, v2         ; encoding: [0x02,0x40,0x1a,0xcc,0xf2,0x02,0x0a,0x1c]
+
+v_dot2_f32_bf16 v2, -1.0, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, -1.0, v1, v2        ; encoding: [0x02,0x40,0x1a,0xcc,0xf3,0x02,0x0a,0x1c]
+
+v_dot2_f32_bf16 v2, 2.0, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, 2.0, v1, v2         ; encoding: [0x02,0x40,0x1a,0xcc,0xf4,0x02,0x0a,0x1c]
+
+v_dot2_f32_bf16 v2, -2.0, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, -2.0, v1, v2        ; encoding: [0x02,0x40,0x1a,0xcc,0xf5,0x02,0x0a,0x1c]
+
+v_dot2_f32_bf16 v2, 4.0, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, 4.0, v1, v2         ; encoding: [0x02,0x40,0x1a,0xcc,0xf6,0x02,0x0a,0x1c]
+
+v_dot2_f32_bf16 v2, -4.0, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, -4.0, v1, v2        ; encoding: [0x02,0x40,0x1a,0xcc,0xf7,0x02,0x0a,0x1c]
+
+v_dot2_f32_bf16 v2, 100.0, v1, v2
+// CHECK: v_dot2_f32_bf16 v2, 0x42c8, v1, v2      ; encoding: [0x02,0x40,0x1a,0xcc,0xff,0x02,0x0a,0x1c,0xc8,0x42,0x00,0x00]
+
+v_dot2_f32_bf16 v2, v1, 100.0, v2
+// CHECK: v_dot2_f32_bf16 v2, v1, 0x42c8, v2      ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0xff,0x09,0x1c,0xc8,0x42,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/bf16_imm.s b/llvm/test/MC/AMDGPU/bf16_imm.s
index 7cf18103adfe5..d79649073aa89 100644
--- a/llvm/test/MC/AMDGPU/bf16_imm.s
+++ b/llvm/test/MC/AMDGPU/bf16_imm.s
@@ -1,54 +1,54 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -show-encoding %s | FileCheck %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s | FileCheck %s
 
-v_dot2_bf16_bf16 v5, v1, v2, 100.0
-// CHECK: v_dot2_bf16_bf16 v5, v1, v2, 0x42c8     ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0xfe,0x03,0xc8,0x42,0x00,0x00]
+v_dot2_bf16_bf16 v5.l, v1, v2, 100.0
+// CHECK: v_dot2_bf16_bf16 v5.l, v1, v2, 0x42c8   ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0xfe,0x03,0xc8,0x42,0x00,0x00]
 
-v_dot2_bf16_bf16 v2, v0, 1.0, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, 1.0, v2        ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe5,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, 1.0, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, 1.0, v2.l    ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe5,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, 1.0, v0, v2
-// CHECK: v_dot2_bf16_bf16 v2, 1.0, v0, v2        ; encoding: [0x02,0x00,0x67,0xd6,0xf2,0x00,0x0a,0x04]
+v_dot2_bf16_bf16 v2.l, 1.0, v0, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, 1.0, v0, v2.l    ; encoding: [0x02,0x00,0x67,0xd6,0xf2,0x00,0x0a,0x04]
 
-v_dot2_bf16_bf16 v5, v1, v2, 1.0
-// CHECK: v_dot2_bf16_bf16 v5, v1, v2, 1.0        ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0xca,0x03]
+v_dot2_bf16_bf16 v5.l, v1, v2, 1.0
+// CHECK: v_dot2_bf16_bf16 v5.l, v1, v2, 1.0      ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0xca,0x03]
 
-v_dot2_bf16_bf16 v2, v0, -1.0, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, -1.0, v2       ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe7,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, -1.0, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, -1.0, v2.l   ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe7,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, 0.5, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, 0.5, v2        ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe1,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, 0.5, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, 0.5, v2.l    ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe1,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, -0.5, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, -0.5, v2       ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe3,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, -0.5, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, -0.5, v2.l   ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe3,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, 2.0, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, 2.0, v2        ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe9,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, 2.0, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, 2.0, v2.l    ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xe9,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, -2.0, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, -2.0, v2       ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xeb,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, -2.0, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, -2.0, v2.l   ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xeb,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, 4.0, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, 4.0, v2        ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xed,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, 4.0, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, 4.0, v2.l    ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xed,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, -4.0, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, -4.0, v2       ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xef,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, -4.0, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, -4.0, v2.l   ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xef,0x09,0x04]
 
 // Check 1/(2*pi) rounded value and ideomatic fp32 0.15915494 value
 // which cannot be accurately represented in bf16.
 
-v_dot2_bf16_bf16 v2, v0, 0.158203125, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, 0.15915494, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, 0.158203125, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, 0.15915494, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, 0.15915494, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, 0.15915494, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, 0.15915494, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, 0.15915494, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, 0x3e22, v2
-// CHECK: v_dot2_bf16_bf16 v2, v0, 0.15915494, v2 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
+v_dot2_bf16_bf16 v2.l, v0, 0x3e22, v2.l
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, 0.15915494, v2.l ; encoding: [0x02,0x00,0x67,0xd6,0x00,0xf1,0x09,0x04]
 
-v_dot2_bf16_bf16 v2, v0, v2, 0.15915494
-// CHECK: v_dot2_bf16_bf16 v2, v0, v2, 0.15915494 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0x05,0xe2,0x03]
+v_dot2_bf16_bf16 v2.l, v0, v2, 0.15915494
+// CHECK: v_dot2_bf16_bf16 v2.l, v0, v2, 0.15915494 ; encoding: [0x02,0x00,0x67,0xd6,0x00,0x05,0xe2,0x03]
 
 v_dot2_f32_bf16 v2, v1, 0, v2
 // CHECK: v_dot2_f32_bf16 v2, v1, 0, v2           ; encoding: [0x02,0x40,0x1a,0xcc,0x01,0x01,0x09,0x1c]
diff --git a/llvm/test/MC/AMDGPU/gfx11-promotions-fake16.s b/llvm/test/MC/AMDGPU/gfx11-promotions-fake16.s
new file mode 100644
index 0000000000000..95a52ffe103fa
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11-promotions-fake16.s
@@ -0,0 +1,353 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 %s | FileCheck --check-prefix=GFX11 %s
+
+// Check opcode promotions and forced suffices.
+// 1. When a suffix is optional, check that it may be omitted.
+// 2. When a suffix is optional, check that it may be specified w/o any effect.
+// 3. When a suffix is required, check that specifying it enforces opcode promotion.
+// 4. When a suffix is required, check that omitting the suffix results in a different encoding.
+
+//===----------------------------------------------------------------------===//
+// VOP1.
+//===----------------------------------------------------------------------===//
+
+v_mov_b32 v0, v1
+// GFX11: v_mov_b32_e32 v0, v1                    ; encoding: [0x01,0x03,0x00,0x7e]
+
+v_mov_b32_e32 v0, v1
+// GFX11: v_mov_b32_e32 v0, v1                    ; encoding: [0x01,0x03,0x00,0x7e]
+
+//===----------------------------------------------------------------------===//
+// VOP2.
+//===----------------------------------------------------------------------===//
+
+v_add_f16 v5, v1, v2
+// GFX11: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
+
+v_add_f16_e32 v5, v1, v2
+// GFX11: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
+
+//===----------------------------------------------------------------------===//
+// VOPC.
+//===----------------------------------------------------------------------===//
+
+v_cmp_lt_f32 vcc_lo, v1, v2
+// GFX11: v_cmp_lt_f32_e32 vcc_lo, v1, v2         ; encoding: [0x01,0x05,0x22,0x7c]
+
+v_cmp_lt_f32_e32 vcc_lo, v1, v2
+// GFX11: v_cmp_lt_f32_e32 vcc_lo, v1, v2         ; encoding: [0x01,0x05,0x22,0x7c]
+
+//===----------------------------------------------------------------------===//
+// VOPCX.
+//===----------------------------------------------------------------------===//
+
+v_cmpx_class_f16 v1, v2
+// GFX11: v_cmpx_class_f16_e32 v1, v2             ; encoding: [0x01,0x05,0xfa,0x7d]
+
+v_cmpx_class_f16_e32 v1, v2
+// GFX11: v_cmpx_class_f16_e32 v1, v2             ; encoding: [0x01,0x05,0xfa,0x7d]
+
+//===----------------------------------------------------------------------===//
+// VOP1.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_bfrev_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_bfrev_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x70,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+v_bfrev_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_bfrev_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x70,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+//===----------------------------------------------------------------------===//
+// VOP1.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_bfrev_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX11: v_bfrev_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x70,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+
+v_bfrev_b32_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX11: v_bfrev_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x70,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+
+//===----------------------------------------------------------------------===//
+// VOP2.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+
+v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+
+//===----------------------------------------------------------------------===//
+// VOP2.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_add_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+
+v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+
+//===----------------------------------------------------------------------===//
+// VOPC.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_cmp_le_u16 v1, v2 dpp8:[7,7,7,3,4,4,6,7] fi:1
+// GFX11: v_cmp_le_u16 vcc_lo, v1, v2 dpp8:[7,7,7,3,4,4,6,7] fi:1 ; encoding: [0xea,0x04,0x76,0x7c,0x01,0xff,0x47,0xfa]
+
+v_cmp_le_u16_dpp v1, v2 dpp8:[7,7,7,3,4,4,6,7] fi:1
+// GFX11: v_cmp_le_u16 vcc_lo, v1, v2 dpp8:[7,7,7,3,4,4,6,7] fi:1 ; encoding: [0xea,0x04,0x76,0x7c,0x01,0xff,0x47,0xfa]
+
+//===----------------------------------------------------------------------===//
+// VOPC.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_cmp_gt_u16 v1, v2 row_shl:0x7 row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_cmp_gt_u16 vcc_lo, v1, v2 row_shl:7 row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x04,0x78,0x7c,0x01,0x07,0x05,0x00]
+
+v_cmp_gt_u16_dpp v1, v2 row_shl:0x7 row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_cmp_gt_u16 vcc_lo, v1, v2 row_shl:7 row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x04,0x78,0x7c,0x01,0x07,0x05,0x00]
+
+//===----------------------------------------------------------------------===//
+// VOPCX.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f16_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
+
+//===----------------------------------------------------------------------===//
+// VOPCX.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
+
+v_cmpx_class_f16_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
+
+//===----------------------------------------------------------------------===//
+// VOP1 -> VOP3.
+//===----------------------------------------------------------------------===//
+
+v_sin_f32 v5, 0.5 mul:2
+// GFX11: v_sin_f32_e64 v5, 0.5 mul:2             ; encoding: [0x05,0x00,0xb5,0xd5,0xf0,0x00,0x00,0x08]
+
+v_sin_f32_e64 v5, 0.5 mul:2
+// GFX11: v_sin_f32_e64 v5, 0.5 mul:2             ; encoding: [0x05,0x00,0xb5,0xd5,0xf0,0x00,0x00,0x08]
+
+v_sin_f32_e64 v5, v1
+// GFX11: v_sin_f32_e64 v5, v1                    ; encoding: [0x05,0x00,0xb5,0xd5,0x01,0x01,0x00,0x00]
+
+v_sin_f32 v5, v1
+// GFX11: v_sin_f32_e32 v5, v1                    ; encoding: [0x01,0x6b,0x0a,0x7e]
+
+//===----------------------------------------------------------------------===//
+// VOP2 -> VOP3.
+//===----------------------------------------------------------------------===//
+
+v_add_f32 v5, v1, -v2
+// GFX11: v_add_f32_e64 v5, v1, -v2               ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x05,0x02,0x40]
+
+v_add_f32_e64 v5, v1, -v2
+// GFX11: v_add_f32_e64 v5, v1, -v2               ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x05,0x02,0x40]
+
+v_add_f32_e64 v5, v1, v2
+// GFX11: v_add_f32_e64 v5, v1, v2                ; encoding: [0x05,0x00,0x03,0xd5,0x01,0x05,0x02,0x00]
+
+v_add_f32 v5, v1, v2
+// GFX11: v_add_f32_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x06]
+
+//===----------------------------------------------------------------------===//
+// VOPC -> VOP3.
+//===----------------------------------------------------------------------===//
+
+v_cmp_f_f32 s10, -v1, v2
+// GFX11: v_cmp_f_f32_e64 s10, -v1, v2            ; encoding: [0x0a,0x00,0x10,0xd4,0x01,0x05,0x02,0x20]
+
+v_cmp_f_f32_e64 s10, -v1, v2
+// GFX11: v_cmp_f_f32_e64 s10, -v1, v2            ; encoding: [0x0a,0x00,0x10,0xd4,0x01,0x05,0x02,0x20]
+
+v_cmp_f_f32_e64 vcc_lo, v1, v2
+// GFX11: v_cmp_f_f32_e64 vcc_lo, v1, v2          ; encoding: [0x6a,0x00,0x10,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmp_f_f32 vcc_lo, v1, v2
+// GFX11: v_cmp_f_f32_e32 vcc_lo, v1, v2          ; encoding: [0x01,0x05,0x20,0x7c]
+
+//===----------------------------------------------------------------------===//
+// VOPCX -> VOP3.
+//===----------------------------------------------------------------------===//
+
+v_cmpx_f_f32 -v1, v2
+// GFX11: v_cmpx_f_f32_e64 -v1, v2                ; encoding: [0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x20]
+
+v_cmpx_f_f32_e64 -v1, v2
+// GFX11: v_cmpx_f_f32_e64 -v1, v2                ; encoding: [0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x20]
+
+v_cmpx_f_f32_e64 v1, v2
+// GFX11: v_cmpx_f_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_f_f32 v1, v2
+// GFX11: v_cmpx_f_f32_e32 v1, v2                 ; encoding: [0x01,0x05,0x20,0x7d]
+
+//===----------------------------------------------------------------------===//
+// VOP3.
+//===----------------------------------------------------------------------===//
+
+v_add3_u32 v5, v1, v2, s3
+// GFX11: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
+
+v_add3_u32_e64 v5, v1, v2, s3
+// GFX11: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
+
+//===----------------------------------------------------------------------===//
+// VOP1 -> VOP3.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_sin_f32 v5, v1 div:2 dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_sin_f32_e64_dpp v5, v1 div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x05,0x00,0xb5,0xd5,0xe9,0x00,0x00,0x18,0x01,0x00,0x00,0x00]
+
+v_sin_f32_e64_dpp v5, v1 div:2 dpp8:[0,0,0,0,0,0,0,0]
+// GFX11: v_sin_f32_e64_dpp v5, v1 div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x05,0x00,0xb5,0xd5,0xe9,0x00,0x00,0x18,0x01,0x00,0x00,0x00]
+
+v_sin_f32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sin_f32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xb5,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sin_f32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sin_f32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x6a,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+//===----------------------------------------------------------------------===//
+// VOP2 -> VOP3.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_add_f32 v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0x04,0x02,0x10,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 mul:4 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0x04,0x02,0x10,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x03,0xd5,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f32_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x06,0x01,0x77,0x39,0x05]
+
+//===----------------------------------------------------------------------===//
+// VOPC -> VOP3.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_cmp_class_f32 s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f32_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmp_class_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f32_e64_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x7e,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmp_class_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmp_class_f32 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfc,0x7c,0x01,0x77,0x39,0x05]
+
+//===----------------------------------------------------------------------===//
+// VOPCX -> VOP3.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_cmpx_class_f32 -v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f32_e64_dpp -v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfe,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32_e64_dpp -v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f32_e64_dpp -v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfe,0xd4,0xe9,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfe,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f32 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfc,0x7d,0x01,0x77,0x39,0x05]
+
+//===----------------------------------------------------------------------===//
+// VOP1 -> VOP3.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_sin_f32 v5, v1 div:2 row_xmask:15
+// GFX11: v_sin_f32_e64_dpp v5, v1 div:2 row_xmask:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xb5,0xd5,0xfa,0x00,0x00,0x18,0x01,0x6f,0x01,0xff]
+
+v_sin_f32_e64_dpp v5, v1 div:2 row_xmask:15
+// GFX11: v_sin_f32_e64_dpp v5, v1 div:2 row_xmask:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xb5,0xd5,0xfa,0x00,0x00,0x18,0x01,0x6f,0x01,0xff]
+
+v_sin_f32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX11: v_sin_f32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xb5,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_sin_f32 v5, v1 quad_perm:[3,2,1,0]
+// GFX11: v_sin_f32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x6a,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+
+//===----------------------------------------------------------------------===//
+// VOP2 -> VOP3.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_add_f32 v5, v1, v2 div:2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 div:2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x18,0x01,0x1b,0x00,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 div:2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 div:2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x18,0x01,0x1b,0x00,0xff]
+
+v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_add_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_add_f32_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x06,0x01,0x1b,0x00,0xff]
+
+//===----------------------------------------------------------------------===//
+// VOPC -> VOP3.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_cmp_class_f32 s5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_class_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmp_class_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmp_class_f32_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmp_class_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmp_class_f32_e64_dpp vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x7e,0xd4,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_cmp_class_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX11: v_cmp_class_f32 vcc_lo, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfc,0x7c,0x01,0x50,0x01,0xff]
+
+//===----------------------------------------------------------------------===//
+// VOPCX -> VOP3.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_cmpx_class_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f32_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfe,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_cmpx_class_f32 v1, v2 quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f32 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfc,0x7d,0x01,0x1b,0x00,0xff]
+
+//===----------------------------------------------------------------------===//
+// VOP3P.
+//===----------------------------------------------------------------------===//
+
+v_dot2_f32_f16 v0, v1, v2, v3
+// GFX11: v_dot2_f32_f16 v0, v1, v2, v3           ; encoding: [0x00,0x40,0x13,0xcc,0x01,0x05,0x0e,0x1c]
+
+v_dot2_f32_f16_e64 v0, v1, v2, v3
+// GFX11: v_dot2_f32_f16 v0, v1, v2, v3           ; encoding: [0x00,0x40,0x13,0xcc,0x01,0x05,0x0e,0x1c]
+
+//===----------------------------------------------------------------------===//
+// VOP3P.DPP8.
+//===----------------------------------------------------------------------===//
+
+v_dot2_f32_f16 v0, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x40,0x13,0xcc,0xe9,0x04,0x0e,0x1c,0x01,0x77,0x39,0x05]
+
+v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x40,0x13,0xcc,0xe9,0x04,0x0e,0x1c,0x01,0x77,0x39,0x05]
+
+//===----------------------------------------------------------------------===//
+// VOP3P.DPP16.
+//===----------------------------------------------------------------------===//
+
+v_dot2_f32_f16 v0, v1, v2, v3 quad_perm:[1,2,3,0]
+// GFX11: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x13,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
+
+v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0]
+// GFX11: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x13,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11-promotions.s b/llvm/test/MC/AMDGPU/gfx11-promotions.s
index 0d1568e7c2765..d9499b37b6994 100644
--- a/llvm/test/MC/AMDGPU/gfx11-promotions.s
+++ b/llvm/test/MC/AMDGPU/gfx11-promotions.s
@@ -1,5 +1,5 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1100 -mattr=+wavefrontsize32 %s | FileCheck --check-prefix=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1100 -mattr=+real-true16 %s | FileCheck --check-prefix=GFX11 %s
 
 // Check opcode promotions and forced suffices.
 // 1. When a suffix is optional, check that it may be omitted.
@@ -21,11 +21,11 @@ v_mov_b32_e32 v0, v1
 // VOP2.
 //===----------------------------------------------------------------------===//
 
-v_add_f16 v5, v1, v2
-// GFX11: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
+v_add_f16 v5.l, v1.l, v2.l
+// GFX11: v_add_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x64]
 
-v_add_f16_e32 v5, v1, v2
-// GFX11: v_add_f16_e32 v5, v1, v2                ; encoding: [0x01,0x05,0x0a,0x64]
+v_add_f16_e32 v5.l, v1.l, v2.l
+// GFX11: v_add_f16_e32 v5.l, v1.l, v2.l          ; encoding: [0x01,0x05,0x0a,0x64]
 
 //===----------------------------------------------------------------------===//
 // VOPC.
@@ -41,11 +41,11 @@ v_cmp_lt_f32_e32 vcc_lo, v1, v2
 // VOPCX.
 //===----------------------------------------------------------------------===//
 
-v_cmpx_class_f16 v1, v2
-// GFX11: v_cmpx_class_f16_e32 v1, v2             ; encoding: [0x01,0x05,0xfa,0x7d]
+v_cmpx_class_f16 v1.l, v2.l
+// GFX11: v_cmpx_class_f16_e32 v1.l, v2.l         ; encoding: [0x01,0x05,0xfa,0x7d]
 
-v_cmpx_class_f16_e32 v1, v2
-// GFX11: v_cmpx_class_f16_e32 v1, v2             ; encoding: [0x01,0x05,0xfa,0x7d]
+v_cmpx_class_f16_e32 v1.l, v2.l
+// GFX11: v_cmpx_class_f16_e32 v1.l, v2.l         ; encoding: [0x01,0x05,0xfa,0x7d]
 
 //===----------------------------------------------------------------------===//
 // VOP1.DPP8.
@@ -71,61 +71,61 @@ v_bfrev_b32_dpp v5, v1 quad_perm:[3,2,1,0]
 // VOP2.DPP8.
 //===----------------------------------------------------------------------===//
 
-v_add_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+v_add_f16 v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
-v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_add_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
+v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x64,0x01,0x77,0x39,0x05]
 
 //===----------------------------------------------------------------------===//
 // VOP2.DPP16.
 //===----------------------------------------------------------------------===//
 
-v_add_f16 v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+v_add_f16 v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
 
-v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_add_f16_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
+v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_add_f16_dpp v5.l, v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x64,0x01,0x1b,0x00,0xff]
 
 //===----------------------------------------------------------------------===//
 // VOPC.DPP8.
 //===----------------------------------------------------------------------===//
 
-v_cmp_le_u16 v1, v2 dpp8:[7,7,7,3,4,4,6,7] fi:1
-// GFX11: v_cmp_le_u16 vcc_lo, v1, v2 dpp8:[7,7,7,3,4,4,6,7] fi:1 ; encoding: [0xea,0x04,0x76,0x7c,0x01,0xff,0x47,0xfa]
+v_cmp_le_u16 v1.l, v2.l dpp8:[7,7,7,3,4,4,6,7] fi:1
+// GFX11: v_cmp_le_u16 vcc_lo, v1.l, v2.l dpp8:[7,7,7,3,4,4,6,7] fi:1 ; encoding: [0xea,0x04,0x76,0x7c,0x01,0xff,0x47,0xfa]
 
-v_cmp_le_u16_dpp v1, v2 dpp8:[7,7,7,3,4,4,6,7] fi:1
-// GFX11: v_cmp_le_u16 vcc_lo, v1, v2 dpp8:[7,7,7,3,4,4,6,7] fi:1 ; encoding: [0xea,0x04,0x76,0x7c,0x01,0xff,0x47,0xfa]
+v_cmp_le_u16_dpp v1.l, v2.l dpp8:[7,7,7,3,4,4,6,7] fi:1
+// GFX11: v_cmp_le_u16 vcc_lo, v1.l, v2.l dpp8:[7,7,7,3,4,4,6,7] fi:1 ; encoding: [0xea,0x04,0x76,0x7c,0x01,0xff,0x47,0xfa]
 
 //===----------------------------------------------------------------------===//
 // VOPC.DPP16.
 //===----------------------------------------------------------------------===//
 
-v_cmp_gt_u16 v1, v2 row_shl:0x7 row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11: v_cmp_gt_u16 vcc_lo, v1, v2 row_shl:7 row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x04,0x78,0x7c,0x01,0x07,0x05,0x00]
+v_cmp_gt_u16 v1.l, v2.l row_shl:0x7 row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_cmp_gt_u16 vcc_lo, v1.l, v2.l row_shl:7 row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x04,0x78,0x7c,0x01,0x07,0x05,0x00]
 
-v_cmp_gt_u16_dpp v1, v2 row_shl:0x7 row_mask:0x0 bank_mask:0x0 fi:1
-// GFX11: v_cmp_gt_u16 vcc_lo, v1, v2 row_shl:7 row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x04,0x78,0x7c,0x01,0x07,0x05,0x00]
+v_cmp_gt_u16_dpp v1.l, v2.l row_shl:0x7 row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11: v_cmp_gt_u16 vcc_lo, v1.l, v2.l row_shl:7 row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x04,0x78,0x7c,0x01,0x07,0x05,0x00]
 
 //===----------------------------------------------------------------------===//
 // VOPCX.DPP8.
 //===----------------------------------------------------------------------===//
 
-v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_class_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
 
-v_cmpx_class_f16_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
+v_cmpx_class_f16_dpp v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cmpx_class_f16 v1.l, v2.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05]
 
 //===----------------------------------------------------------------------===//
 // VOPCX.DPP16.
 //===----------------------------------------------------------------------===//
 
-v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_class_f16 v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
 
-v_cmpx_class_f16_dpp v1, v2 quad_perm:[3,2,1,0]
-// GFX11: v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
+v_cmpx_class_f16_dpp v1.l, v2.l quad_perm:[3,2,1,0]
+// GFX11: v_cmpx_class_f16 v1.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff]
 
 //===----------------------------------------------------------------------===//
 // VOP1 -> VOP3.
diff --git a/llvm/test/MC/AMDGPU/gfx1150_asm_features-fake16.s b/llvm/test/MC/AMDGPU/gfx1150_asm_features-fake16.s
new file mode 100644
index 0000000000000..b3297377ed584
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1150_asm_features-fake16.s
@@ -0,0 +1,48 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1150 -mattr=-real-true16 %s | FileCheck --check-prefix=GFX1150 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1151 -mattr=-real-true16 %s | FileCheck --check-prefix=GFX1150 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1152 -mattr=-real-true16 %s | FileCheck --check-prefix=GFX1150 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1153 -mattr=-real-true16 %s | FileCheck --check-prefix=GFX1150 %s
+
+//
+// Subtargets allow src1 of VOP3 DPP instructions to be SGPR or inlinable
+// constant.
+//
+
+v_add3_u32_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
+// GFX1150: encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff]
+
+v_add3_u32_e64_dpp v5, v1, 42, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
+// GFX1150: encoding: [0x05,0x00,0x55,0xd6,0xfa,0x54,0x0d,0x04,0x01,0x1b,0x00,0xff]
+
+v_add3_u32_e64_dpp v5, v1, s2, v0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1150: encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x00,0x04,0x01,0x77,0x39,0x05]
+
+v_add3_u32_e64_dpp v5, v1, 42, v0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1150: encoding: [0x05,0x00,0x55,0xd6,0xe9,0x54,0x01,0x04,0x01,0x77,0x39,0x05]
+
+v_add3_u32_e64_dpp v5, v1, s2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1150: encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0c,0x00,0x01,0x77,0x39,0x05]
+
+v_cmp_ne_i32_e64_dpp vcc_lo, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1150: encoding: [0x6a,0x00,0x45,0xd4,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_add_f32_e64_dpp v5, v1, s2 row_mirror
+// GFX1150: encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x00,0x00,0x01,0x40,0x01,0xff]
+
+v_min3_f16 v5, v1, s2, 2.0 op_sel:[1,1,0,1] quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf
+// GFX1150: encoding: [0x05,0x58,0x49,0xd6,0xfa,0x04,0xd0,0x03,0x01,0x55,0x00,0xff]
+
+v_cmp_le_f32 vcc_lo, v1, v2 row_mirror
+// GFX1150: encoding: [0xfa,0x04,0x26,0x7c,0x01,0x40,0x01,0xff]
+
+v_cmp_le_f32 vcc_lo, v1, s2 row_mirror
+// GFX1150: encoding: [0x6a,0x00,0x13,0xd4,0xfa,0x04,0x00,0x00,0x01,0x40,0x01,0xff]
+
+v_cmp_le_f32 vcc_lo, v1, s2 quad_perm:[1,1,1,1]
+// GFX1150: encoding: [0x6a,0x00,0x13,0xd4,0xfa,0x04,0x00,0x00,0x01,0x55,0x00,0xff]
+
+v_cmpx_neq_f16 v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1150: encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+
+v_cmpx_class_f16 v1, 2.0 quad_perm:[1,1,1,1]
+// GFX1150: encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x55,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx1150_asm_features.s b/llvm/test/MC/AMDGPU/gfx1150_asm_features.s
index 5c498a28fd8c7..6e5bda3c3d0c9 100644
--- a/llvm/test/MC/AMDGPU/gfx1150_asm_features.s
+++ b/llvm/test/MC/AMDGPU/gfx1150_asm_features.s
@@ -1,8 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1150 %s | FileCheck --check-prefix=GFX1150 %s
-// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1151 %s | FileCheck --check-prefix=GFX1150 %s
-// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1152 %s | FileCheck --check-prefix=GFX1150 %s
-// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1153 %s | FileCheck --check-prefix=GFX1150 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1150 -mattr=+real-true16 %s | FileCheck --check-prefix=GFX1150 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1151 -mattr=+real-true16 %s | FileCheck --check-prefix=GFX1150 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1152 -mattr=+real-true16 %s | FileCheck --check-prefix=GFX1150 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1153 -mattr=+real-true16 %s | FileCheck --check-prefix=GFX1150 %s
 
 // Subtargets allow src1 of VOP3 DPP instructions to be SGPR or inlinable
 // constant.
@@ -28,8 +28,8 @@ v_cmp_ne_i32_e64_dpp vcc_lo, v1, s2 dpp8:[7,6,5,4,3,2,1,0]
 v_add_f32_e64_dpp v5, v1, s2 row_mirror
 // GFX1150: v_add_f32_e64_dpp v5, v1, s2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x03,0xd5,0xfa,0x04,0x00,0x00,0x01,0x40,0x01,0xff]
 
-v_min3_f16 v5, v1, s2, 2.0 op_sel:[1,1,0,1] quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf
-// GFX1150: v_min3_f16_e64_dpp v5, v1, s2, 2.0 op_sel:[1,1,0,1] quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x49,0xd6,0xfa,0x04,0xd0,0x03,0x01,0x55,0x00,0xff]
+v_min3_f16 v5.h, v1.h, s2, 2.0 op_sel:[1,1,0,1] quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf
+// GFX1150: v_min3_f16_e64_dpp v5.h, v1.h, s2, 2.0 op_sel:[1,1,0,1] quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x58,0x49,0xd6,0xfa,0x04,0xd0,0x03,0x01,0x55,0x00,0xff]
 
 v_cmp_le_f32 vcc_lo, v1, v2 row_mirror
 // GFX1150: v_cmp_le_f32 vcc_lo, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x26,0x7c,0x01,0x40,0x01,0xff]
@@ -40,8 +40,8 @@ v_cmp_le_f32 vcc_lo, v1, s2 row_mirror
 v_cmp_le_f32 vcc_lo, v1, s2 quad_perm:[1,1,1,1]
 // GFX1150: v_cmp_le_f32_e64_dpp vcc_lo, v1, s2 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf ; encoding: [0x6a,0x00,0x13,0xd4,0xfa,0x04,0x00,0x00,0x01,0x55,0x00,0xff]
 
-v_cmpx_neq_f16 v1, 2.0 dpp8:[7,6,5,4,3,2,1,0]
-// GFX1150: v_cmpx_neq_f16_e64_dpp v1, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
+v_cmpx_neq_f16 v1.l, 2.0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1150: v_cmpx_neq_f16_e64_dpp v1.l, 2.0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0x8d,0xd4,0xe9,0xe8,0x01,0x00,0x01,0x77,0x39,0x05]
 
-v_cmpx_class_f16 v1, 2.0 quad_perm:[1,1,1,1]
-// GFX1150: v_cmpx_class_f16_e64_dpp v1, 2.0 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x55,0x00,0xff]
+v_cmpx_class_f16 v1.l, 2.0 quad_perm:[1,1,1,1]
+// GFX1150: v_cmpx_class_f16_e64_dpp v1.l, 2.0 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xe8,0x01,0x00,0x01,0x55,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err-fake16.s
new file mode 100644
index 0000000000000..7450d4b86cabc
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err-fake16.s
@@ -0,0 +1,43 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 %s 2>&1 | FileCheck %s -check-prefix=GCN-ERR --implicit-check-not=error: --strict-whitespace
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 %s 2>&1 | FileCheck %s -check-prefix=GCN-ERR --implicit-check-not=error: --strict-whitespace
+
+//===----------------------------------------------------------------------===//
+// VINTERP src operands must be VGPRs.
+// Check that other operand kinds are rejected by assembler.
+//===----------------------------------------------------------------------===//
+
+v_interp_p10_f32 v0, s1, v2, v3
+// GCN-ERR: :[[@LINE-1]]:22: error: invalid operand for instruction
+
+v_interp_p10_f32 v0, v1, s2, v3
+// GCN-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_interp_p10_f32 v0, v1, v2, s3
+// GCN-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+
+v_interp_p2_f32 v0, 1, v2, v3
+// GCN-ERR: :[[@LINE-1]]:21: error: invalid operand for instruction
+
+v_interp_p2_f32 v0, v1, 2, v3
+// GCN-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_interp_p2_f32 v0, v1, v2, 3
+// GCN-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction
+
+v_interp_p10_f16_f32 v0, s1, v2, v3
+// GCN-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_interp_p10_f16_f32 v0, v1, s2, v3
+// GCN-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+
+v_interp_p10_f16_f32 v0, v1, v2, s3
+// GCN-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_interp_p2_f16_f32 v0, 1, v2, v3
+// GCN-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_interp_p2_f16_f32 v0, v1, 2, v3
+// GCN-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction
+
+v_interp_p2_f16_f32 v0, v1, v2, 3
+// GCN-ERR: :[[@LINE-1]]:33: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
index 1aefd1f0a7d19..58641332a08da 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
@@ -2630,6 +2630,36 @@ v_log_f32 v5, src_scc
 v_log_f32 v255, 0xaf123456
 // GFX11: v_log_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
+v_mov_b16_e32 v0.l, v1.l
+// GFX11: v_mov_b16_e32 v0.l, v1.l                ; encoding: [0x01,0x39,0x00,0x7e]
+
+v_mov_b16_e32 v0.l, s1
+// GFX11: v_mov_b16_e32 v0.l, s1                  ; encoding: [0x01,0x38,0x00,0x7e]
+
+v_mov_b16_e32 v0.h, 0
+// GFX11: v_mov_b16_e32 v0.h, 0                   ; encoding: [0x80,0x38,0x00,0x7f]
+
+v_mov_b16_e32 v0.h, 1.0
+// GFX11: v_mov_b16_e32 v0.h, 1.0                 ; encoding: [0xf2,0x38,0x00,0x7f]
+
+v_mov_b16_e32 v0.l, 0x1234
+// GFX11: v_mov_b16_e32 v0.l, 0x1234              ; encoding: [0xff,0x38,0x00,0x7e,0x34,0x12,0x00,0x00]
+
+v_mov_b16_e64 v0.l, v1.l
+// GFX11: v_mov_b16_e64 v0.l, v1.l                ; encoding: [0x00,0x00,0x9c,0xd5,0x01,0x01,0x00,0x00]
+
+v_mov_b16_e64 v200.l, v1.h
+// GFX11: v_mov_b16_e64 v200.l, v1.h op_sel:[1,0] ; encoding: [0xc8,0x08,0x9c,0xd5,0x01,0x01,0x00,0x00]
+
+v_mov_b16_e64 v0.l, s1
+// GFX11: v_mov_b16_e64 v0.l, s1                  ; encoding: [0x00,0x00,0x9c,0xd5,0x01,0x00,0x00,0x00]
+
+v_mov_b16_e64 v200.h, 1
+// GFX11: v_mov_b16_e64 v200.h, 1 op_sel:[0,1]    ; encoding: [0xc8,0x40,0x9c,0xd5,0x81,0x00,0x00,0x00]
+
+v_mov_b16_e64 v0.l, 0x1234
+// GFX11: v_mov_b16_e64 v0.l, 0x1234              ; encoding: [0x00,0x00,0x9c,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
 v_mov_b32 v5, v1
 // GFX11: v_mov_b32_e32 v5, v1                    ; encoding: [0x01,0x03,0x0a,0x7e]
 
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias-fake16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias-fake16.s
new file mode 100644
index 0000000000000..34f519e1e0e47
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias-fake16.s
@@ -0,0 +1,15 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-real-true16 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,-real-true16 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
+
+v_cvt_pknorm_i16_f16 v5, v1, v2
+// GFX11: v_cvt_pk_norm_i16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00]
+
+v_cvt_pknorm_u16_f16 v5, v1, v2
+// GFX11: v_cvt_pk_norm_u16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
+
+v_add3_nc_u32 v5, v1, v2, s3
+// GFX11: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
+
+v_xor_add_u32 v5, v1, v2, s3
+// GFX11: v_xad_u32 v5, v1, v2, s3                ; encoding: [0x05,0x00,0x45,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s
index f6ac190e16285..b34c94de7160a 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_alias.s
@@ -1,12 +1,12 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
 
-v_cvt_pknorm_i16_f16 v5, v1, v2
-// GFX11: v_cvt_pk_norm_i16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00]
+v_cvt_pknorm_i16_f16 v5, v1.l, v2.l
+// GFX11: v_cvt_pk_norm_i16_f16 v5, v1.l, v2.l    ; encoding: [0x05,0x00,0x12,0xd7,0x01,0x05,0x02,0x00]
 
-v_cvt_pknorm_u16_f16 v5, v1, v2
-// GFX11: v_cvt_pk_norm_u16_f16 v5, v1, v2        ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
+v_cvt_pknorm_u16_f16 v5, v1.l, v2.l
+// GFX11: v_cvt_pk_norm_u16_f16 v5, v1.l, v2.l    ; encoding: [0x05,0x00,0x13,0xd7,0x01,0x05,0x02,0x00]
 
 v_add3_nc_u32 v5, v1, v2, s3
 // GFX11: v_add3_u32 v5, v1, v2, s3               ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_features.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_features.s
new file mode 100644
index 0000000000000..e15a48a469a2a
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_features.s
@@ -0,0 +1,77 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX11,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W32-ERR,GFX11-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=W64-ERR,GFX11-ERR --implicit-check-not=error: %s
+
+//===----------------------------------------------------------------------===//
+// HW correctly handles fp inline constants for src2 (they have f16 type).
+// Check that inline constants are not converted to literals.
+//===----------------------------------------------------------------------===//
+
+v_cmp_class_f16_e64 s[10:11], v1.l, 0.5
+// W64: v_cmp_class_f16_e64 s[10:11], v1.l, 0.5 ; encoding: [0x0a,0x00,0x7d,0xd4,0x01,0xe1,0x01,0x00]
+// W32-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmp_class_f16_e64 s10, v1.l, 0.5
+// W32: v_cmp_class_f16_e64 s10, v1.l, 0.5      ; encoding: [0x0a,0x00,0x7d,0xd4,0x01,0xe1,0x01,0x00]
+// W64-ERR: :[[@LINE-2]]:21: error: invalid operand for instruction
+
+v_cmpx_class_f16_e64 v1.l, 0.5
+// GFX11: v_cmpx_class_f16_e64 v1.l, 0.5          ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0xe1,0x01,0x00]
+
+//===----------------------------------------------------------------------===//
+// src0 and src2 are packed operands.
+// Check that op_sel is not allowed with these operands.
+//===----------------------------------------------------------------------===//
+
+v_dot2_f16_f16_e64 v0.l, v1.h, v2, v3.l
+// GFX11-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.l, v1, v2.h, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.l, v1.h, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64 v0.l, v1, v2.h, v3.l
+// GFX11-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1.h, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2.h, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11-ERR: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+//===----------------------------------------------------------------------===//
+// src0 and src1 are vector operands.
+// Check that SGPRs are not allowed for these operands.
+//===----------------------------------------------------------------------===//
+
+v_dot2_f16_f16_e64_dpp v0.l, s1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+// GFX11-ERR: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// GFX11-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// GFX11-ERR: :[[@LINE-1]]:36: error: invalid operand for instruction
+
+// Ensure bits 8-15 are not zeroed out and .h which should be present on src0 and dst are present.
+v_mul_f16_e64 v5.h, v1.h, v2.l
+// GFX11: v_mul_f16_e64 v5.h, v1.h, v2.l op_sel:[1,0,1] ; encoding: [0x05,0x48,0x35,0xd5,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
index e21e5bf827ed1..fe8858dabfbc2 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
@@ -2696,6 +2696,37 @@ v_log_f32 v5, src_scc
 v_log_f32 v255, 0xaf123456
 // GFX12: v_log_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
+v_mov_b16_e32 v0.l, v1.l
+// GFX12: v_mov_b16_e32 v0.l, v1.l ; encoding: [0x01,0x39,0x00,0x7e]
+
+v_mov_b16_e32 v0.l, s1
+// GFX12: v_mov_b16_e32 v0.l, s1 ; encoding: [0x01,0x38,0x00,0x7e]
+
+v_mov_b16_e32 v0.h, 0
+// GFX12: v_mov_b16_e32 v0.h, 0 ; encoding: [0x80,0x38,0x00,0x7f]
+
+v_mov_b16_e32 v0.h, 1.0
+// GFX12-ASM: v_mov_b16_e32 v0.h, 1.0 ; encoding: [0xf2,0x38,0x00,0x7f]
+// GFX12-DIS: v_mov_b16_e32 v0.h, 0x3c00 ; encoding: [0xff,0x38,0x00,0x7f,0x00,0x3c,0x00,0x00]
+
+v_mov_b16_e32 v0.l, 0x1234
+// GFX12: v_mov_b16_e32 v0.l, 0x1234 ; encoding: [0xff,0x38,0x00,0x7e,0x34,0x12,0x00,0x00]
+
+v_mov_b16_e64 v0.l, v1.l
+// GFX12: v_mov_b16_e64 v0.l, v1.l ; encoding: [0x00,0x00,0x9c,0xd5,0x01,0x01,0x00,0x00]
+
+v_mov_b16_e64 v200.l, v1.h
+// GFX12: v_mov_b16_e64 v200.l, v1.h op_sel:[1,0] ; encoding: [0xc8,0x08,0x9c,0xd5,0x01,0x01,0x00,0x00]
+
+v_mov_b16_e64 v0.l, s1
+// GFX12: v_mov_b16_e64 v0.l, s1 ; encoding: [0x00,0x00,0x9c,0xd5,0x01,0x00,0x00,0x00]
+
+v_mov_b16_e64 v200.h, 1
+// GFX12: v_mov_b16_e64 v200.h, 1 op_sel:[0,1] ; encoding: [0xc8,0x40,0x9c,0xd5,0x81,0x00,0x00,0x00]
+
+v_mov_b16_e64 v0.l, 0x1234
+// GFX12: v_mov_b16_e64 v0.l, 0x1234 ; encoding: [0x00,0x00,0x9c,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
 v_mov_b32 v5, v1
 // GFX12: v_mov_b32_e32 v5, v1 ; encoding: [0x01,0x03,0x0a,0x7e]
 

From 6cfec29cb9bc44ec907eeda99df508985ecbd49b Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <arvind.sudarsanam@intel.com>
Date: Wed, 16 Apr 2025 10:13:30 -0400
Subject: [PATCH 117/710] [Offload][SYCL] Refactor OffloadKind implementation
 (#135809)

Following are the changes:

1. Make OffloadKind enum values to be powers of two so we can use them
like a bitfield
2. Include OFK_SYCL enum value
3. Modify ActiveOffloadKinds support in clang-linker-wrapper to use
bitfields instead of a vector.

Thanks

---------

Signed-off-by: Arvind Sudarsanam <arvind.sudarsanam@intel.com>
---
 clang/test/CodeGenCUDA/offloading-entries.cu  | 24 +++++++++----------
 clang/test/Driver/linker-wrapper-image.c      |  2 +-
 .../ClangLinkerWrapper.cpp                    | 10 ++++----
 llvm/include/llvm/Object/OffloadBinary.h      |  9 +++----
 4 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/clang/test/CodeGenCUDA/offloading-entries.cu b/clang/test/CodeGenCUDA/offloading-entries.cu
index c053cf586f8f5..ac0680ff08ea2 100644
--- a/clang/test/CodeGenCUDA/offloading-entries.cu
+++ b/clang/test/CodeGenCUDA/offloading-entries.cu
@@ -34,17 +34,17 @@
 // HIP: @managed.managed = global i32 0, align 4
 // HIP: @managed = externally_initialized global ptr null
 // HIP: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading"
-// HIP: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 0, ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "llvm_offload_entries"
 // HIP: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading"
-// HIP: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 0, ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries"
 // HIP: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading"
-// HIP: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @var, ptr @.offloading.entry_name.2, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 0, ptr @var, ptr @.offloading.entry_name.2, i64 4, i64 0, ptr null }, section "llvm_offload_entries"
 // HIP: @.offloading.entry_name.3 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading"
-// HIP: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 1, ptr @managed.managed, ptr @.offloading.entry_name.3, i64 4, i64 4, ptr @managed }, section "llvm_offload_entries"
+// HIP: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 1, ptr @managed.managed, ptr @.offloading.entry_name.3, i64 4, i64 4, ptr @managed }, section "llvm_offload_entries"
 // HIP: @.offloading.entry_name.4 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading"
-// HIP: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 2, ptr @surf, ptr @.offloading.entry_name.4, i64 4, i64 1, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 2, ptr @surf, ptr @.offloading.entry_name.4, i64 4, i64 1, ptr null }, section "llvm_offload_entries"
 // HIP: @.offloading.entry_name.5 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading"
-// HIP: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 3, ptr @tex, ptr @.offloading.entry_name.5, i64 4, i64 1, ptr null }, section "llvm_offload_entries"
+// HIP: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 3, ptr @tex, ptr @.offloading.entry_name.5, i64 4, i64 1, ptr null }, section "llvm_offload_entries"
 //.
 // CUDA-COFF: @managed = dso_local global i32 undef, align 4
 // CUDA-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading"
@@ -63,17 +63,17 @@
 // HIP-COFF: @managed.managed = dso_local global i32 0, align 4
 // HIP-COFF: @managed = dso_local externally_initialized global ptr null
 // HIP-COFF: @.offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00", section ".llvm.rodata.offloading"
-// HIP-COFF: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 0, ptr @_Z3foov, ptr @.offloading.entry_name, i64 0, i64 0, ptr null }, section "llvm_offload_entries$OE"
 // HIP-COFF: @.offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00", section ".llvm.rodata.offloading"
-// HIP-COFF: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 0, ptr @_Z6kernelv, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries$OE"
 // HIP-COFF: @.offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00", section ".llvm.rodata.offloading"
-// HIP-COFF: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 0, ptr @var, ptr @.offloading.entry_name.2, i64 4, i64 0, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry.var = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 0, ptr @var, ptr @.offloading.entry_name.2, i64 4, i64 0, ptr null }, section "llvm_offload_entries$OE"
 // HIP-COFF: @.offloading.entry_name.3 = internal unnamed_addr constant [8 x i8] c"managed\00", section ".llvm.rodata.offloading"
-// HIP-COFF: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 1, ptr @managed.managed, ptr @.offloading.entry_name.3, i64 4, i64 4, ptr @managed }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry.managed = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 1, ptr @managed.managed, ptr @.offloading.entry_name.3, i64 4, i64 4, ptr @managed }, section "llvm_offload_entries$OE"
 // HIP-COFF: @.offloading.entry_name.4 = internal unnamed_addr constant [5 x i8] c"surf\00", section ".llvm.rodata.offloading"
-// HIP-COFF: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 2, ptr @surf, ptr @.offloading.entry_name.4, i64 4, i64 1, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry.surf = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 2, ptr @surf, ptr @.offloading.entry_name.4, i64 4, i64 1, ptr null }, section "llvm_offload_entries$OE"
 // HIP-COFF: @.offloading.entry_name.5 = internal unnamed_addr constant [4 x i8] c"tex\00", section ".llvm.rodata.offloading"
-// HIP-COFF: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 3, i32 3, ptr @tex, ptr @.offloading.entry_name.5, i64 4, i64 1, ptr null }, section "llvm_offload_entries$OE"
+// HIP-COFF: @.offloading.entry.tex = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 4, i32 3, ptr @tex, ptr @.offloading.entry_name.5, i64 4, i64 1, ptr null }, section "llvm_offload_entries$OE"
 //.
 // CUDA-LABEL: @_Z18__device_stub__foov(
 // CUDA-NEXT:  entry:
diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c
index 775385137c75f..c0de56d58196a 100644
--- a/clang/test/Driver/linker-wrapper-image.c
+++ b/clang/test/Driver/linker-wrapper-image.c
@@ -220,7 +220,7 @@
 // HIP-NEXT:   %constant = lshr i32 %11, 4
 // HIP-NEXT:   %12 = and i32 %flags, 32
 // HIP-NEXT:   %normalized = lshr i32 %12, 5
-// HIP-NEXT:   %13 = icmp eq i16 %kind, 3
+// HIP-NEXT:   %13 = icmp eq i16 %kind, 4
 // HIP-NEXT:   br i1 %13, label %if.kind, label %if.end
 //
 //      HIP: if.kind:
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 52d922abbcaec..082355e6c716f 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -923,10 +923,9 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
         });
     auto LinkerArgs = getLinkerArgs(Input, BaseArgs);
 
-    DenseSet<OffloadKind> ActiveOffloadKinds;
+    uint16_t ActiveOffloadKindMask = 0u;
     for (const auto &File : Input)
-      if (File.getBinary()->getOffloadKind() != OFK_None)
-        ActiveOffloadKinds.insert(File.getBinary()->getOffloadKind());
+      ActiveOffloadKindMask |= File.getBinary()->getOffloadKind();
 
     // Write any remaining device inputs to an output file.
     SmallVector<StringRef> InputFiles;
@@ -943,7 +942,10 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
       return OutputOrErr.takeError();
 
     // Store the offloading image for each linked output file.
-    for (OffloadKind Kind : ActiveOffloadKinds) {
+    for (OffloadKind Kind = OFK_OpenMP; Kind != OFK_LAST;
+         Kind = static_cast<OffloadKind>((uint16_t)(Kind) << 1)) {
+      if ((ActiveOffloadKindMask & Kind) == 0)
+        continue;
       llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileOrErr =
           llvm::MemoryBuffer::getFileOrSTDIN(*OutputOrErr);
       if (std::error_code EC = FileOrErr.getError()) {
diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h
index c02aec8d956ed..a3b78b8ec6261 100644
--- a/llvm/include/llvm/Object/OffloadBinary.h
+++ b/llvm/include/llvm/Object/OffloadBinary.h
@@ -32,10 +32,11 @@ namespace object {
 /// The producer of the associated offloading image.
 enum OffloadKind : uint16_t {
   OFK_None = 0,
-  OFK_OpenMP,
-  OFK_Cuda,
-  OFK_HIP,
-  OFK_LAST,
+  OFK_OpenMP = (1 << 0),
+  OFK_Cuda = (1 << 1),
+  OFK_HIP = (1 << 2),
+  OFK_SYCL = (1 << 3),
+  OFK_LAST = (1 << 4),
 };
 
 /// The type of contents the offloading image contains.

From de90487fc17fb928de7d0cd75d47a44db5181c14 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Wed, 16 Apr 2025 15:20:05 +0100
Subject: [PATCH 118/710] [AARCH64] Add FEAT_SSVE_FEXPA and fix unsupported
 features list (#134368)

This patch adds new feature introduced in [2025-03
release](https://developer.arm.com/documentation/ddi0602/2025-03/SVE-Instructions/FEXPA--Floating-point-exponential-accelerator-)
and changes feature requirements for fexpa instructions and intrinsics.

Additionally it fixes unsupported features list by moving fearures
dependent on sme2p1 to correct location.
---
 clang/include/clang/Basic/arm_sve.td               |  6 ++++--
 .../CodeGen/AArch64/sve-intrinsics/acle_sve_expa.c | 14 +++++++++++---
 .../Driver/print-supported-extensions-aarch64.c    |  1 +
 llvm/lib/Target/AArch64/AArch64.td                 |  9 +++++----
 llvm/lib/Target/AArch64/AArch64Features.td         |  4 +++-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td        |  4 ++++
 llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td     |  2 +-
 llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll  |  2 +-
 llvm/test/MC/AArch64/SVE/fexpa.s                   |  8 ++++----
 9 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 35263541b67ae..f09f40ce9202e 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -836,9 +836,11 @@ defm SVRINTP : SInstZPZ<"svrintp", "hfd", "aarch64_sve_frintp">;
 defm SVRINTX : SInstZPZ<"svrintx", "hfd", "aarch64_sve_frintx">;
 defm SVRINTZ : SInstZPZ<"svrintz", "hfd", "aarch64_sve_frintz">;
 defm SVSQRT  : SInstZPZ<"svsqrt",  "hfd", "aarch64_sve_fsqrt">;
-
+def SVEXPA  : SInst<"svexpa[_{d}]",  "du",   "hfd", MergeNone, "aarch64_sve_fexpa_x", [VerifyRuntimeMode]>{
+   let SVETargetGuard = "sve";
+   let SMETargetGuard = "sme2,ssve-fexpa";
+}
 let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
-def SVEXPA  : SInst<"svexpa[_{d}]",  "du",   "hfd", MergeNone, "aarch64_sve_fexpa_x">;
 def SVTMAD  : SInst<"svtmad[_{d}]",  "dddi", "hfd", MergeNone, "aarch64_sve_ftmad_x", [], [ImmCheck<2, ImmCheck0_7>]>;
 def SVTSMUL : SInst<"svtsmul[_{d}]", "ddu",  "hfd", MergeNone, "aarch64_sve_ftsmul_x">;
 def SVTSSEL : SInst<"svtssel[_{d}]", "ddu",  "hfd", MergeNone, "aarch64_sve_ftssel_x">;
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_expa.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_expa.c
index 52b6822a833f7..8c34017b7750b 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_expa.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_expa.c
@@ -1,10 +1,12 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sve -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +ssve-fexpa -target-feature +sme2 -target-feature +sme -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sve -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sve -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +ssve-fexpa -target-feature +sme2 -target-feature +sme -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
 #ifdef SVE_OVERLOADED_FORMS
@@ -14,6 +16,12 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
+#ifdef __ARM_FEATURE_SME
+#define STREAMING __arm_streaming
+#else
+#define STREAMING
+#endif
+
 // CHECK-LABEL: @test_svexpa_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fexpa.x.nxv8f16(<vscale x 8 x i16> [[OP:%.*]])
@@ -24,7 +32,7 @@
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fexpa.x.nxv8f16(<vscale x 8 x i16> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x half> [[TMP0]]
 //
-svfloat16_t test_svexpa_f16(svuint16_t op)
+svfloat16_t test_svexpa_f16(svuint16_t op) STREAMING
 {
   return SVE_ACLE_FUNC(svexpa,_f16,,)(op);
 }
@@ -39,7 +47,7 @@ svfloat16_t test_svexpa_f16(svuint16_t op)
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.fexpa.x.nxv4f32(<vscale x 4 x i32> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
 //
-svfloat32_t test_svexpa_f32(svuint32_t op)
+svfloat32_t test_svexpa_f32(svuint32_t op) STREAMING
 {
   return SVE_ACLE_FUNC(svexpa,_f32,,)(op);
 }
@@ -54,7 +62,7 @@ svfloat32_t test_svexpa_f32(svuint32_t op)
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.fexpa.x.nxv2f64(<vscale x 2 x i64> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x double> [[TMP0]]
 //
-svfloat64_t test_svexpa_f64(svuint64_t op)
+svfloat64_t test_svexpa_f64(svuint64_t op) STREAMING
 {
   return SVE_ACLE_FUNC(svexpa,_f64,,)(op);
 }
diff --git a/clang/test/Driver/print-supported-extensions-aarch64.c b/clang/test/Driver/print-supported-extensions-aarch64.c
index 38a3f54eb4794..539c1937a9712 100644
--- a/clang/test/Driver/print-supported-extensions-aarch64.c
+++ b/clang/test/Driver/print-supported-extensions-aarch64.c
@@ -81,6 +81,7 @@
 // CHECK-NEXT:     ssbs                FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
 // CHECK-NEXT:     ssve-aes            FEAT_SSVE_AES                                          Enable Armv9.6-A SVE AES support in streaming SVE mode
 // CHECK-NEXT:     ssve-bitperm        FEAT_SSVE_BitPerm                                      Enable Armv9.6-A SVE BitPerm support in streaming SVE mode
+// CHECK-NEXT:     ssve-fexpa          FEAT_SSVE_FEXPA                                        Enable SVE FEXPA instruction in Streaming SVE mode
 // CHECK-NEXT:     ssve-fp8dot2        FEAT_SSVE_FP8DOT2                                      Enable SVE2 FP8 2-way dot product instructions
 // CHECK-NEXT:     ssve-fp8dot4        FEAT_SSVE_FP8DOT4                                      Enable SVE2 FP8 4-way dot product instructions
 // CHECK-NEXT:     ssve-fp8fma         FEAT_SSVE_FP8FMA                                       Enable SVE2 FP8 multiply-add instructions
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index 3677f669c3481..29dd6227ba021 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -74,13 +74,14 @@ def SVEUnsupported : AArch64Unsupported {
 }
 
 let F = [HasSME2p2, HasSVE2p2_or_SME2p2, HasNonStreamingSVE_or_SME2p2,
-         HasNonStreamingSVE2p2_or_SME2p2, HasNonStreamingSVE2_or_SSVE_BitPerm,
-         HasSME_MOP4, HasSME_TMOP] in
+         HasNonStreamingSVE2p2_or_SME2p2] in
 def SME2p2Unsupported : AArch64Unsupported;
 
 def SME2p1Unsupported : AArch64Unsupported {
-  let F = !listconcat([HasSME2p1, HasSVE2p1_or_SME2p1, HasNonStreamingSVE2p1_or_SSVE_AES],
-                      SME2p2Unsupported.F);
+  let F = !listconcat([HasSME2p1, HasSVE2p1_or_SME2p1, HasNonStreamingSVE2p1_or_SSVE_AES, 
+                       HasSME_MOP4, HasSME_TMOP, HasNonStreamingSVE_or_SSVE_FEXPA, 
+                       HasNonStreamingSVE2_or_SSVE_BitPerm],
+                       SME2p2Unsupported.F);
 }
 
 def SME2Unsupported : AArch64Unsupported {
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 357f526d5e308..f4f931a5cdab1 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -576,7 +576,9 @@ def FeatureSME_MOP4: ExtensionWithMArch<"sme-mop4", "SME_MOP4", "FEAT_SME_MOP4",
 def FeatureSME_TMOP: ExtensionWithMArch<"sme-tmop", "SME_TMOP", "FEAT_SME_TMOP",
   "Enable SME Structured sparsity outer product instructions.", [FeatureSME2]>;
 
-//===----------------------------------------------------------------------===//
+def FeatureSSVE_FEXPA : ExtensionWithMArch<"ssve-fexpa", "SSVE_FEXPA", "FEAT_SSVE_FEXPA",
+  "Enable SVE FEXPA instruction in Streaming SVE mode", [FeatureSME2]>;
+
 //  Other Features
 //===----------------------------------------------------------------------===//
 
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 99f2b79d31bb7..a7a01ed785afa 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -294,6 +294,10 @@ def HasNonStreamingSVE2_or_SSVE_BitPerm
     : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2()) ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_BitPerm())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSSVE_BitPerm), "sve2 or ssve-bitperm">;
+def HasNonStreamingSVE_or_SSVE_FEXPA
+    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
+                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_FEXPA())">,
+                AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_FEXPA), "sve or ssve-fexpa">;
 
 // A subset of NEON instructions are legal in Streaming SVE execution mode,
 // so don't need the additional check for 'isNeonAvailable'.
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index a2f326c994c2f..b40c82a25e7ba 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -980,7 +980,7 @@ let Predicates = [HasSVE_or_SME] in {
   def MOVPRFX_ZZ : sve_int_bin_cons_misc_0_c<0b00000001, "movprfx", ZPRAny>;
 } // End HasSVE_or_SME
 
-let Predicates = [HasNonStreamingSVE_or_SME2p2] in {
+let Predicates = [HasNonStreamingSVE_or_SSVE_FEXPA] in {
   defm FEXPA_ZZ : sve_int_bin_cons_misc_0_c_fexpa<"fexpa", int_aarch64_sve_fexpa_x>;
 } // End HasSVE
 
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
index 00e000f642377..021d4855905e7 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -force-streaming -mattr=+sme2p2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -force-streaming -mattr=+ssve-fexpa < %s | FileCheck %s
 
 define <vscale x 8 x half> @fexpa_h(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: fexpa_h:
diff --git a/llvm/test/MC/AArch64/SVE/fexpa.s b/llvm/test/MC/AArch64/SVE/fexpa.s
index c51b1e2b1d3e5..c6386255b274e 100644
--- a/llvm/test/MC/AArch64/SVE/fexpa.s
+++ b/llvm/test/MC/AArch64/SVE/fexpa.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p2 < %s \
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+ssve-fexpa < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 // RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
@@ -14,17 +14,17 @@
 fexpa z0.h, z31.h
 // CHECK-INST: fexpa	z0.h, z31.h
 // CHECK-ENCODING: [0xe0,0xbb,0x60,0x04]
-// CHECK-ERROR: instruction requires: sve or sme2p2
+// CHECK-ERROR: instruction requires: sve or ssve-fexpa
 // CHECK-UNKNOWN: 0460bbe0 <unknown>
 
 fexpa z0.s, z31.s
 // CHECK-INST: fexpa	z0.s, z31.s
 // CHECK-ENCODING: [0xe0,0xbb,0xa0,0x04]
-// CHECK-ERROR: instruction requires: sve or sme2p2
+// CHECK-ERROR: instruction requires: sve or ssve-fexpa
 // CHECK-UNKNOWN: 04a0bbe0 <unknown>
 
 fexpa z0.d, z31.d
 // CHECK-INST: fexpa	z0.d, z31.d
 // CHECK-ENCODING: [0xe0,0xbb,0xe0,0x04]
-// CHECK-ERROR: instruction requires: sve or sme2p2
+// CHECK-ERROR: instruction requires: sve or ssve-fexpa
 // CHECK-UNKNOWN: 04e0bbe0 <unknown>

From 9483aaaaaa427b5dcb9a7af8f232a4696eef94bf Mon Sep 17 00:00:00 2001
From: Benjamin Chetioui <3920784+bchetioui@users.noreply.github.com>
Date: Wed, 16 Apr 2025 16:32:14 +0200
Subject: [PATCH 119/710] [bazel] Fix bazel build after 00eaff3 #2. (#135962)

The linter messed up the order of includes, which is necessary as is.
---
 .../Dialect/Bufferization/IR/BufferizationTypeInterfaces.h     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
index 8672aa60a43c5..5faa1479ee542 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
@@ -13,7 +13,8 @@
 // Bufferization Type Interfaces
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h.inc"
 #include "mlir/IR/Types.h"
 
+#include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h.inc"
+
 #endif // MLIR_DIALECT_BUFFERIZATION_IR_BUFFERIZATIONTYPEINTERFACES_H_

From 183cb45c1280b80a0022649d1db8a93544bb97b0 Mon Sep 17 00:00:00 2001
From: John Harrison <harjohn@google.com>
Date: Wed, 16 Apr 2025 07:52:16 -0700
Subject: [PATCH 120/710] [lldb-dap] Fixing a race during disconnect. (#135872)

While attempting to disconnect the DAP transport reader thread is
setting
`disconnecting` as soon as it sees a [disconnect
request](https://microsoft.github.io/debug-adapter-protocol/specification#Requests_Disconnect).

However, if it is processing another request when this disconnect
arrives the `DAP::Loop` handler may exit the loop without replying to
the disconnect request.

There has been some instability on the CI jobs due to this race, for
example https://lab.llvm.org/buildbot/#/builders/59/builds/16076

To address this, ensure we only return from `DAP::Loop` once we've
emptied the queue.
---
 lldb/tools/lldb-dap/DAP.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index b752e9cfaeb85..597fe3a1e323b 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -921,7 +921,7 @@ llvm::Error DAP::Loop() {
     StopEventHandlers();
   });
 
-  while (!disconnecting) {
+  while (!disconnecting || !m_queue.empty()) {
     std::unique_lock<std::mutex> lock(m_queue_mutex);
     m_queue_cv.wait(lock, [&] { return disconnecting || !m_queue.empty(); });
 

From ef1abbe32e66c16118ded6dd9f7b1a55dea8c2b6 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg@nvidia.com>
Date: Wed, 16 Apr 2025 08:02:34 -0700
Subject: [PATCH 121/710] [NVPTX] Remove extraneous
 initializeNVVMReflectLegacyPassPass declaration (#135825)

This was already declared in NVPTX.h and I accidentally added it back in
#134416.
---
 llvm/lib/Target/NVPTX/NVVMReflect.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVVMReflect.cpp b/llvm/lib/Target/NVPTX/NVVMReflect.cpp
index 7273b30e4ae2e..30b522efa4361 100644
--- a/llvm/lib/Target/NVPTX/NVVMReflect.cpp
+++ b/llvm/lib/Target/NVPTX/NVVMReflect.cpp
@@ -52,10 +52,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "nvvm-reflect"
 
-namespace llvm {
-void initializeNVVMReflectLegacyPassPass(PassRegistry &);
-} // namespace llvm
-
 namespace {
 class NVVMReflect {
   // Map from reflect function call arguments to the value to replace the call

From 99c08ff1cb96fc4f471aca0dd253060b3f32e8bc Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Wed, 16 Apr 2025 17:05:53 +0200
Subject: [PATCH 122/710] Revert "[clang] Unify `SourceLocation` and
 `IdentifierInfo*` pair-like data structures to `IdentifierLoc`" (#135974)

Reverts llvm/llvm-project#135808

Example from the LLDB macOS CI:
https://green.lab.llvm.org/job/llvm.org/view/LLDB/job/as-lldb-cmake/24084/execution/node/54/log/?consoleFull
```
/Users/ec2-user/jenkins/workspace/llvm.org/as-lldb-cmake/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp:360:49: error: no viable conversion from 'std::pair<clang::IdentifierInfo *, clang::SourceLocation>' to 'clang::ModuleIdPath' (aka 'ArrayRef<IdentifierLoc>')
  clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
                                                ^~~~~~~~~~~~~~~~~~
/Users/ec2-user/jenkins/workspace/llvm.org/as-lldb-cmake/llvm-project/llvm/include/llvm/ADT/ArrayRef.h:41:40: note: candidate constructor (the implicit copy constructor) not viable: no known conversion from 'std::pair<clang::IdentifierInfo *, clang::SourceLocation>' to 'const llvm::ArrayRef<clang::IdentifierLoc> &' for 1st argument
  class LLVM_GSL_POINTER [[nodiscard]] ArrayRef {
                                       ^
/Users/ec2-user/jenkins/workspace/llvm.org/as-lldb-cmake/llvm-project/llvm/include/llvm/ADT/ArrayRef.h:41:40: note: candidate constructor (the implicit move constructor) not viable: no known conversion from 'std::pair<clang::IdentifierInfo *, clang::SourceLocation>' to 'llvm::ArrayRef<clang::IdentifierLoc> &&' for 1st argument
/Users/ec2-user/jenkins/workspace/llvm.org/as-lldb-cmake/llvm-project/llvm/include/llvm/ADT/ArrayRef.h:70:18: note: candidate constructor not viable: no known conversion from 'std::pair<clang::IdentifierInfo *, clang::SourceLocation>' to 'std::nullopt_t' for 1st argument
    /*implicit*/ ArrayRef(std::nullopt_t) {}
```
---
 .../pp-trace/PPCallbacksTracker.cpp           |   4 +-
 clang/include/clang/AST/OpenACCClause.h       |  20 +--
 clang/include/clang/Basic/IdentifierTable.h   |  26 +---
 clang/include/clang/Lex/ModuleLoader.h        |   3 +-
 clang/include/clang/Lex/PPCallbacks.h         |   1 -
 clang/include/clang/Lex/Preprocessor.h        |   9 +-
 clang/include/clang/Parse/LoopHint.h          |   2 +-
 clang/include/clang/Parse/Parser.h            |  13 +-
 clang/include/clang/Sema/ParsedAttr.h         |  10 ++
 clang/include/clang/Sema/Sema.h               |   2 +-
 clang/include/clang/Sema/SemaCodeCompletion.h |   3 +-
 clang/include/clang/Sema/SemaObjC.h           |   4 +-
 clang/include/clang/Sema/SemaOpenACC.h        |   2 +-
 clang/lib/AST/OpenACCClause.cpp               |   4 +-
 clang/lib/AST/TextNodeDumper.cpp              |   4 +-
 clang/lib/Frontend/CompilerInstance.cpp       |  53 ++++----
 clang/lib/Frontend/FrontendActions.cpp        |   4 +-
 clang/lib/Lex/PPDirectives.cpp                |  22 ++--
 clang/lib/Lex/PPLexerChange.cpp               |   6 +-
 clang/lib/Lex/Pragma.cpp                      |  73 ++++++-----
 clang/lib/Lex/Preprocessor.cpp                |  16 +--
 clang/lib/Parse/ParseDecl.cpp                 |  28 ++--
 clang/lib/Parse/ParseExpr.cpp                 |   7 +-
 clang/lib/Parse/ParseHLSL.cpp                 |   2 +-
 clang/lib/Parse/ParseObjc.cpp                 |  38 +++---
 clang/lib/Parse/ParseOpenACC.cpp              |  12 +-
 clang/lib/Parse/ParsePragma.cpp               |  15 ++-
 clang/lib/Parse/ParseStmt.cpp                 |   6 +-
 clang/lib/Parse/Parser.cpp                    |  19 +--
 clang/lib/Sema/ParsedAttr.cpp                 |   8 ++
 clang/lib/Sema/SemaARM.cpp                    |   2 +-
 clang/lib/Sema/SemaCodeComplete.cpp           |   8 +-
 clang/lib/Sema/SemaDeclAttr.cpp               | 124 +++++++++---------
 clang/lib/Sema/SemaDeclObjC.cpp               |  35 +++--
 clang/lib/Sema/SemaHLSL.cpp                   |  12 +-
 clang/lib/Sema/SemaModule.cpp                 |  42 +++---
 clang/lib/Sema/SemaObjC.cpp                   |  45 ++++---
 clang/lib/Sema/SemaOpenACCClause.cpp          |  11 +-
 clang/lib/Sema/SemaStmtAttr.cpp               |  29 ++--
 clang/lib/Sema/SemaSwift.cpp                  |  24 ++--
 clang/lib/Sema/SemaTemplateVariadic.cpp       |  10 +-
 clang/lib/Sema/SemaType.cpp                   |  13 +-
 clang/lib/Serialization/ASTReader.cpp         |   2 +-
 clang/lib/Serialization/ASTWriter.cpp         |   8 +-
 .../DependencyScanning/ModuleDepCollector.cpp |   2 +-
 45 files changed, 384 insertions(+), 399 deletions(-)

diff --git a/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp b/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
index 4c916fa30685b..3bb30fd15b2e1 100644
--- a/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
+++ b/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
@@ -547,8 +547,8 @@ void PPCallbacksTracker::appendArgument(const char *Name, ModuleIdPath Value) {
     if (I)
       SS << ", ";
     SS << "{"
-       << "Name: " << Value[I].getIdentifierInfo()->getName() << ", "
-       << "Loc: " << getSourceLocationString(PP, Value[I].getLoc()) << "}";
+       << "Name: " << Value[I].first->getName() << ", "
+       << "Loc: " << getSourceLocationString(PP, Value[I].second) << "}";
   }
   SS << "]";
   appendArgument(Name, SS.str());
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index f18a6cf62f2c5..681567228cbb0 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -258,7 +258,7 @@ inline bool operator!=(const OpenACCBindClause &LHS,
   return !(LHS == RHS);
 }
 
-using DeviceTypeArgument = IdentifierLoc;
+using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
 /// A 'device_type' or 'dtype' clause, takes a list of either an 'asterisk' or
 /// an identifier. The 'asterisk' means 'the rest'.
 class OpenACCDeviceTypeClause final
@@ -280,16 +280,16 @@ class OpenACCDeviceTypeClause final
         "Invalid clause kind for device-type");
 
     assert(!llvm::any_of(Archs, [](const DeviceTypeArgument &Arg) {
-      return Arg.getLoc().isInvalid();
+      return Arg.second.isInvalid();
     }) && "Invalid SourceLocation for an argument");
 
-    assert((Archs.size() == 1 ||
-            !llvm::any_of(Archs,
-                          [](const DeviceTypeArgument &Arg) {
-                            return Arg.getIdentifierInfo() == nullptr;
-                          })) &&
-           "Only a single asterisk version is permitted, and must be the "
-           "only one");
+    assert(
+        (Archs.size() == 1 || !llvm::any_of(Archs,
+                                            [](const DeviceTypeArgument &Arg) {
+                                              return Arg.first == nullptr;
+                                            })) &&
+        "Only a single asterisk version is permitted, and must be the "
+        "only one");
 
     std::uninitialized_copy(Archs.begin(), Archs.end(),
                             getTrailingObjects<DeviceTypeArgument>());
@@ -302,7 +302,7 @@ class OpenACCDeviceTypeClause final
   }
   bool hasAsterisk() const {
     return getArchitectures().size() > 0 &&
-           getArchitectures()[0].getIdentifierInfo() == nullptr;
+           getArchitectures()[0].first == nullptr;
   }
 
   ArrayRef<DeviceTypeArgument> getArchitectures() const {
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 1275b056227b5..0347880244a40 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -18,7 +18,6 @@
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/DiagnosticIDs.h"
 #include "clang/Basic/LLVM.h"
-#include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/FoldingSet.h"
@@ -77,6 +76,9 @@ inline bool isReservedInAllContexts(ReservedIdentifierStatus Status) {
          Status != ReservedIdentifierStatus::StartsWithUnderscoreAndIsExternC;
 }
 
+/// A simple pair of identifier info and location.
+using IdentifierLocPair = std::pair<IdentifierInfo *, SourceLocation>;
+
 /// IdentifierInfo and other related classes are aligned to
 /// 8 bytes so that DeclarationName can use the lower 3 bits
 /// of a pointer to one of these classes.
@@ -1163,28 +1165,6 @@ class SelectorTable {
   static std::string getPropertyNameFromSetterSelector(Selector Sel);
 };
 
-/// A simple pair of identifier info and location.
-class IdentifierLoc {
-  SourceLocation Loc;
-  IdentifierInfo *II = nullptr;
-
-public:
-  IdentifierLoc() = default;
-  IdentifierLoc(SourceLocation L, IdentifierInfo *Ident) : Loc(L), II(Ident) {}
-
-  void setLoc(SourceLocation L) { Loc = L; }
-  void setIdentifierInfo(IdentifierInfo *Ident) { II = Ident; }
-  SourceLocation getLoc() const { return Loc; }
-  IdentifierInfo *getIdentifierInfo() const { return II; }
-
-  bool operator==(const IdentifierLoc &X) const {
-    return Loc == X.Loc && II == X.II;
-  }
-
-  bool operator!=(const IdentifierLoc &X) const {
-    return Loc != X.Loc || II != X.II;
-  }
-};
 }  // namespace clang
 
 namespace llvm {
diff --git a/clang/include/clang/Lex/ModuleLoader.h b/clang/include/clang/Lex/ModuleLoader.h
index a58407200c41c..f880a9091a2ed 100644
--- a/clang/include/clang/Lex/ModuleLoader.h
+++ b/clang/include/clang/Lex/ModuleLoader.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_CLANG_LEX_MODULELOADER_H
 #define LLVM_CLANG_LEX_MODULELOADER_H
 
-#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceLocation.h"
@@ -30,7 +29,7 @@ class IdentifierInfo;
 
 /// A sequence of identifier/location pairs used to describe a particular
 /// module or submodule, e.g., std.vector.
-using ModuleIdPath = ArrayRef<IdentifierLoc>;
+using ModuleIdPath = ArrayRef<std::pair<IdentifierInfo *, SourceLocation>>;
 
 /// Describes the result of attempting to load a module.
 class ModuleLoadResult {
diff --git a/clang/include/clang/Lex/PPCallbacks.h b/clang/include/clang/Lex/PPCallbacks.h
index 313b730afbab8..46cc564086f1c 100644
--- a/clang/include/clang/Lex/PPCallbacks.h
+++ b/clang/include/clang/Lex/PPCallbacks.h
@@ -15,7 +15,6 @@
 #define LLVM_CLANG_LEX_PPCALLBACKS_H
 
 #include "clang/Basic/DiagnosticIDs.h"
-#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/ModuleLoader.h"
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index f8f2f567f9171..24bb524783e93 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -327,7 +327,7 @@ class Preprocessor {
   SourceLocation ModuleImportLoc;
 
   /// The import path for named module that we're currently processing.
-  SmallVector<IdentifierLoc, 2> NamedModuleImportPath;
+  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> NamedModuleImportPath;
 
   llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
   unsigned CheckPointCounter = 0;
@@ -622,7 +622,7 @@ class Preprocessor {
 
   /// The identifier and source location of the currently-active
   /// \#pragma clang arc_cf_code_audited begin.
-  IdentifierLoc PragmaARCCFCodeAuditedInfo;
+  std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
 
   /// The source location of the currently-active
   /// \#pragma clang assume_nonnull begin.
@@ -1998,7 +1998,8 @@ class Preprocessor {
   /// arc_cf_code_audited begin.
   ///
   /// Returns an invalid location if there is no such pragma active.
-  IdentifierLoc getPragmaARCCFCodeAuditedInfo() const {
+  std::pair<IdentifierInfo *, SourceLocation>
+  getPragmaARCCFCodeAuditedInfo() const {
     return PragmaARCCFCodeAuditedInfo;
   }
 
@@ -2006,7 +2007,7 @@ class Preprocessor {
   /// arc_cf_code_audited begin.  An invalid location ends the pragma.
   void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident,
                                      SourceLocation Loc) {
-    PragmaARCCFCodeAuditedInfo = IdentifierLoc(Loc, Ident);
+    PragmaARCCFCodeAuditedInfo = {Ident, Loc};
   }
 
   /// The location of the currently-active \#pragma clang
diff --git a/clang/include/clang/Parse/LoopHint.h b/clang/include/clang/Parse/LoopHint.h
index 72be043d3c5a4..cec5605ea3615 100644
--- a/clang/include/clang/Parse/LoopHint.h
+++ b/clang/include/clang/Parse/LoopHint.h
@@ -9,12 +9,12 @@
 #ifndef LLVM_CLANG_PARSE_LOOPHINT_H
 #define LLVM_CLANG_PARSE_LOOPHINT_H
 
-#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 
 namespace clang {
 
 class Expr;
+struct IdentifierLoc;
 
 /// Loop optimization hint for loop and unroll pragmas.
 struct LoopHint {
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 662f54d0e8d8a..9ebcf144ba59e 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -1725,8 +1725,8 @@ class Parser : public CodeCompletionHandler {
   ObjCTypeParamList *parseObjCTypeParamList();
   ObjCTypeParamList *parseObjCTypeParamListOrProtocolRefs(
       ObjCTypeParamListScope &Scope, SourceLocation &lAngleLoc,
-      SmallVectorImpl<IdentifierLoc> &protocolIdents, SourceLocation &rAngleLoc,
-      bool mayBeProtocolList = true);
+      SmallVectorImpl<IdentifierLocPair> &protocolIdents,
+      SourceLocation &rAngleLoc, bool mayBeProtocolList = true);
 
   void HelperActionsForIvarDeclarations(ObjCContainerDecl *interfaceDecl,
                                         SourceLocation atLoc,
@@ -3818,7 +3818,8 @@ class Parser : public CodeCompletionHandler {
                                SourceLocation Loc,
                                llvm::SmallVectorImpl<Expr *> &IntExprs);
   /// Parses the 'device-type-list', which is a list of identifiers.
-  bool ParseOpenACCDeviceTypeList(llvm::SmallVector<IdentifierLoc> &Archs);
+  bool ParseOpenACCDeviceTypeList(
+      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs);
   /// Parses the 'async-argument', which is an integral value with two
   /// 'special' values that are likely negative (but come from Macros).
   OpenACCIntExprParseResult ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK,
@@ -3950,8 +3951,10 @@ class Parser : public CodeCompletionHandler {
     return false;
   }
 
-  bool ParseModuleName(SourceLocation UseLoc,
-                       SmallVectorImpl<IdentifierLoc> &Path, bool IsImport);
+  bool ParseModuleName(
+      SourceLocation UseLoc,
+      SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
+      bool IsImport);
 
   //===--------------------------------------------------------------------===//
   // C++11/G++: Type Traits [Type-Traits.html in the GCC manual]
diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h
index 428d3111de80d..b88b871dc8821 100644
--- a/clang/include/clang/Sema/ParsedAttr.h
+++ b/clang/include/clang/Sema/ParsedAttr.h
@@ -40,6 +40,7 @@ class LangOptions;
 class Sema;
 class Stmt;
 class TargetInfo;
+struct IdentifierLoc;
 
 /// Represents information about a change in availability for
 /// an entity, which is part of the encoding of the 'availability'
@@ -98,6 +99,15 @@ struct PropertyData {
 
 } // namespace detail
 
+/// Wraps an identifier and optional source location for the identifier.
+struct IdentifierLoc {
+  SourceLocation Loc;
+  IdentifierInfo *Ident;
+
+  static IdentifierLoc *create(ASTContext &Ctx, SourceLocation Loc,
+                               IdentifierInfo *Ident);
+};
+
 /// A union of the various pointer types that can be passed to an
 /// ParsedAttr as an argument.
 using ArgsUnion = llvm::PointerUnion<Expr *, IdentifierLoc *>;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 1f23b754a69cb..fe37fd7701ce3 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -143,7 +143,7 @@ enum class LangAS : unsigned int;
 class LocalInstantiationScope;
 class LookupResult;
 class MangleNumberingContext;
-typedef ArrayRef<IdentifierLoc> ModuleIdPath;
+typedef ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> ModuleIdPath;
 class ModuleLoader;
 class MultiLevelTemplateArgumentList;
 struct NormalizedConstraint;
diff --git a/clang/include/clang/Sema/SemaCodeCompletion.h b/clang/include/clang/Sema/SemaCodeCompletion.h
index 3029e56e5cfe2..72159de3a6e72 100644
--- a/clang/include/clang/Sema/SemaCodeCompletion.h
+++ b/clang/include/clang/Sema/SemaCodeCompletion.h
@@ -193,7 +193,8 @@ class SemaCodeCompletion : public SemaBase {
   void CodeCompleteObjCForCollection(Scope *S, DeclGroupPtrTy IterationVar);
   void CodeCompleteObjCSelector(Scope *S,
                                 ArrayRef<const IdentifierInfo *> SelIdents);
-  void CodeCompleteObjCProtocolReferences(ArrayRef<IdentifierLoc> Protocols);
+  void
+  CodeCompleteObjCProtocolReferences(ArrayRef<IdentifierLocPair> Protocols);
   void CodeCompleteObjCProtocolDecl(Scope *S);
   void CodeCompleteObjCInterfaceDecl(Scope *S);
   void CodeCompleteObjCClassForwardDecl(Scope *S);
diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
index 4cda41a82b61f..791a7f45b832f 100644
--- a/clang/include/clang/Sema/SemaObjC.h
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -307,11 +307,11 @@ class SemaObjC : public SemaBase {
 
   DeclGroupPtrTy
   ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc,
-                                  ArrayRef<IdentifierLoc> IdentList,
+                                  ArrayRef<IdentifierLocPair> IdentList,
                                   const ParsedAttributesView &attrList);
 
   void FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer,
-                               ArrayRef<IdentifierLoc> ProtocolId,
+                               ArrayRef<IdentifierLocPair> ProtocolId,
                                SmallVectorImpl<Decl *> &Protocols);
 
   void DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId,
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 8d31d46444c7e..4c3a13a3b044f 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -212,7 +212,7 @@ class SemaOpenACC : public SemaBase {
   } LoopWithoutSeqInfo;
 
   // Redeclaration of the version in OpenACCClause.h.
-  using DeviceTypeArgument = IdentifierLoc;
+  using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
 
   /// A type to represent all the data for an OpenACC Clause that has been
   /// parsed, but not yet created/semantically analyzed. This is effectively a
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index 2820d7b288658..d7cbb51335359 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -891,10 +891,10 @@ void OpenACCClausePrinter::VisitDeviceTypeClause(
   OS << "(";
   llvm::interleaveComma(C.getArchitectures(), OS,
                         [&](const DeviceTypeArgument &Arch) {
-                          if (Arch.getIdentifierInfo() == nullptr)
+                          if (Arch.first == nullptr)
                             OS << "*";
                           else
-                            OS << Arch.getIdentifierInfo()->getName();
+                            OS << Arch.first->getName();
                         });
   OS << ")";
 }
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 1bd94a3ac6431..c8b459ee78e6b 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -500,10 +500,10 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       llvm::interleaveComma(
           cast<OpenACCDeviceTypeClause>(C)->getArchitectures(), OS,
           [&](const DeviceTypeArgument &Arch) {
-            if (Arch.getIdentifierInfo() == nullptr)
+            if (Arch.first == nullptr)
               OS << "*";
             else
-              OS << Arch.getIdentifierInfo()->getName();
+              OS << Arch.first->getName();
           });
       OS << ")";
       break;
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 93e4e31c2891d..243e0a3c15b05 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -35,7 +35,6 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
-#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/GlobalModuleIndex.h"
@@ -2010,8 +2009,8 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
                              Module::NameVisibilityKind Visibility,
                              bool IsInclusionDirective) {
   // Determine what file we're searching from.
-  StringRef ModuleName = Path[0].getIdentifierInfo()->getName();
-  SourceLocation ModuleNameLoc = Path[0].getLoc();
+  StringRef ModuleName = Path[0].first->getName();
+  SourceLocation ModuleNameLoc = Path[0].second;
 
   // If we've already handled this import, just return the cached result.
   // This one-element cache is important to eliminate redundant diagnostics
@@ -2027,7 +2026,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
   // If we don't already have information on this module, load the module now.
   Module *Module = nullptr;
   ModuleMap &MM = getPreprocessor().getHeaderSearchInfo().getModuleMap();
-  if (auto MaybeModule = MM.getCachedModuleLoad(*Path[0].getIdentifierInfo())) {
+  if (auto MaybeModule = MM.getCachedModuleLoad(*Path[0].first)) {
     // Use the cached result, which may be nullptr.
     Module = *MaybeModule;
     // Config macros are already checked before building a module, but they need
@@ -2047,7 +2046,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     // * `Preprocessor::HandleHeaderIncludeOrImport` will never call this
     //   function as the `#include` or `#import` is textual.
 
-    MM.cacheModuleLoad(*Path[0].getIdentifierInfo(), Module);
+    MM.cacheModuleLoad(*Path[0].first, Module);
   } else {
     ModuleLoadResult Result = findOrCompileModuleAndReadAST(
         ModuleName, ImportLoc, ModuleNameLoc, IsInclusionDirective);
@@ -2056,7 +2055,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (!Result)
       DisableGeneratingGlobalModuleIndex = true;
     Module = Result;
-    MM.cacheModuleLoad(*Path[0].getIdentifierInfo(), Module);
+    MM.cacheModuleLoad(*Path[0].first, Module);
   }
 
   // If we never found the module, fail.  Otherwise, verify the module and link
@@ -2068,7 +2067,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
   // a submodule.
   bool MapPrivateSubModToTopLevel = false;
   for (unsigned I = 1, N = Path.size(); I != N; ++I) {
-    StringRef Name = Path[I].getIdentifierInfo()->getName();
+    StringRef Name = Path[I].first->getName();
     clang::Module *Sub = Module->findSubmodule(Name);
 
     // If the user is requesting Foo.Private and it doesn't exist, try to
@@ -2079,10 +2078,10 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
       SmallString<128> PrivateModule(Module->Name);
       PrivateModule.append("_Private");
 
-      SmallVector<IdentifierLoc, 2> PrivPath;
+      SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> PrivPath;
       auto &II = PP->getIdentifierTable().get(
           PrivateModule, PP->getIdentifierInfo(Module->Name)->getTokenID());
-      PrivPath.emplace_back(Path[0].getLoc(), &II);
+      PrivPath.push_back(std::make_pair(&II, Path[0].second));
 
       std::string FileName;
       // If there is a modulemap module or prebuilt module, load it.
@@ -2096,12 +2095,11 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
         PP->markClangModuleAsAffecting(Module);
         if (!getDiagnostics().isIgnored(
                 diag::warn_no_priv_submodule_use_toplevel, ImportLoc)) {
-          getDiagnostics().Report(Path[I].getLoc(),
+          getDiagnostics().Report(Path[I].second,
                                   diag::warn_no_priv_submodule_use_toplevel)
-              << Path[I].getIdentifierInfo() << Module->getFullModuleName()
-              << PrivateModule
-              << SourceRange(Path[0].getLoc(), Path[I].getLoc())
-              << FixItHint::CreateReplacement(SourceRange(Path[0].getLoc()),
+              << Path[I].first << Module->getFullModuleName() << PrivateModule
+              << SourceRange(Path[0].second, Path[I].second)
+              << FixItHint::CreateReplacement(SourceRange(Path[0].second),
                                               PrivateModule);
           getDiagnostics().Report(Sub->DefinitionLoc,
                                   diag::note_private_top_level_defined);
@@ -2130,11 +2128,10 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
 
       // If there was a clear winner, user it.
       if (Best.size() == 1) {
-        getDiagnostics().Report(Path[I].getLoc(),
-                                diag::err_no_submodule_suggest)
-            << Path[I].getIdentifierInfo() << Module->getFullModuleName()
-            << Best[0] << SourceRange(Path[0].getLoc(), Path[I - 1].getLoc())
-            << FixItHint::CreateReplacement(SourceRange(Path[I].getLoc()),
+        getDiagnostics().Report(Path[I].second, diag::err_no_submodule_suggest)
+            << Path[I].first << Module->getFullModuleName() << Best[0]
+            << SourceRange(Path[0].second, Path[I - 1].second)
+            << FixItHint::CreateReplacement(SourceRange(Path[I].second),
                                             Best[0]);
 
         Sub = Module->findSubmodule(Best[0]);
@@ -2144,9 +2141,9 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (!Sub) {
       // No submodule by this name. Complain, and don't look for further
       // submodules.
-      getDiagnostics().Report(Path[I].getLoc(), diag::err_no_submodule)
-          << Path[I].getIdentifierInfo() << Module->getFullModuleName()
-          << SourceRange(Path[0].getLoc(), Path[I - 1].getLoc());
+      getDiagnostics().Report(Path[I].second, diag::err_no_submodule)
+          << Path[I].first << Module->getFullModuleName()
+          << SourceRange(Path[0].second, Path[I - 1].second);
       break;
     }
 
@@ -2164,8 +2161,8 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
       // FIXME: Should we detect this at module load time? It seems fairly
       // expensive (and rare).
       getDiagnostics().Report(ImportLoc, diag::warn_missing_submodule)
-          << Module->getFullModuleName()
-          << SourceRange(Path.front().getLoc(), Path.back().getLoc());
+        << Module->getFullModuleName()
+        << SourceRange(Path.front().second, Path.back().second);
 
       return ModuleLoadResult(Module, ModuleLoadResult::MissingExpected);
     }
@@ -2174,7 +2171,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (Preprocessor::checkModuleIsAvailable(getLangOpts(), getTarget(),
                                              *Module, getDiagnostics())) {
       getDiagnostics().Report(ImportLoc, diag::note_module_import_here)
-          << SourceRange(Path.front().getLoc(), Path.back().getLoc());
+        << SourceRange(Path.front().second, Path.back().second);
       LastModuleImportLoc = ImportLoc;
       LastModuleImportResult = ModuleLoadResult();
       return ModuleLoadResult();
@@ -2299,9 +2296,9 @@ GlobalModuleIndex *CompilerInstance::loadGlobalModuleIndex(
       Module *TheModule = I->second;
       OptionalFileEntryRef Entry = TheModule->getASTFile();
       if (!Entry) {
-        SmallVector<IdentifierLoc, 2> Path;
-        Path.emplace_back(TriggerLoc,
-                          getPreprocessor().getIdentifierInfo(TheModule->Name));
+        SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+        Path.push_back(std::make_pair(
+            getPreprocessor().getIdentifierInfo(TheModule->Name), TriggerLoc));
         std::reverse(Path.begin(), Path.end());
         // Load a module as hidden.  This also adds it to the global index.
         loadModule(TheModule->DefinitionLoc, Path, Module::Hidden, false);
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index e6c7b9f32c29b..c5aeb92c7af73 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -1216,9 +1216,9 @@ void GetDependenciesByModuleNameAction::ExecuteAction() {
   SourceManager &SM = PP.getSourceManager();
   FileID MainFileID = SM.getMainFileID();
   SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
-  SmallVector<IdentifierLoc, 2> Path;
+  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
   IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
-  Path.emplace_back(FileStart, ModuleID);
+  Path.push_back(std::make_pair(ModuleID, FileStart));
   auto ModResult = CI.loadModule(FileStart, Path, Module::Hidden, false);
   PPCallbacks *CB = PP.getPPCallbacks();
   CB->moduleImport(SourceLocation(), Path, ModResult);
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 21ec83b437ef4..8411526019f3e 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -1916,15 +1916,15 @@ void Preprocessor::EnterAnnotationToken(SourceRange Range,
 
 /// Produce a diagnostic informing the user that a #include or similar
 /// was implicitly treated as a module import.
-static void diagnoseAutoModuleImport(Preprocessor &PP, SourceLocation HashLoc,
-                                     Token &IncludeTok,
-                                     ArrayRef<IdentifierLoc> Path,
-                                     SourceLocation PathEnd) {
+static void diagnoseAutoModuleImport(
+    Preprocessor &PP, SourceLocation HashLoc, Token &IncludeTok,
+    ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> Path,
+    SourceLocation PathEnd) {
   SmallString<128> PathString;
   for (size_t I = 0, N = Path.size(); I != N; ++I) {
     if (I)
       PathString += '.';
-    PathString += Path[I].getIdentifierInfo()->getName();
+    PathString += Path[I].first->getName();
   }
 
   int IncludeKind = 0;
@@ -2273,12 +2273,12 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
   SourceLocation StartLoc = IsImportDecl ? IncludeTok.getLocation() : HashLoc;
 
   // Complain about attempts to #include files in an audit pragma.
-  if (PragmaARCCFCodeAuditedInfo.getLoc().isValid()) {
+  if (PragmaARCCFCodeAuditedInfo.second.isValid()) {
     Diag(StartLoc, diag::err_pp_include_in_arc_cf_code_audited) << IsImportDecl;
-    Diag(PragmaARCCFCodeAuditedInfo.getLoc(), diag::note_pragma_entered_here);
+    Diag(PragmaARCCFCodeAuditedInfo.second, diag::note_pragma_entered_here);
 
     // Immediately leave the pragma.
-    PragmaARCCFCodeAuditedInfo = IdentifierLoc();
+    PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
   }
 
   // Complain about attempts to #include files in an assume-nonnull pragma.
@@ -2403,10 +2403,10 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
     // Compute the module access path corresponding to this module.
     // FIXME: Should we have a second loadModule() overload to avoid this
     // extra lookup step?
-    SmallVector<IdentifierLoc, 2> Path;
+    SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
     for (Module *Mod = ModuleToImport; Mod; Mod = Mod->Parent)
-      Path.emplace_back(FilenameTok.getLocation(),
-                        getIdentifierInfo(Mod->Name));
+      Path.push_back(std::make_pair(getIdentifierInfo(Mod->Name),
+                                    FilenameTok.getLocation()));
     std::reverse(Path.begin(), Path.end());
 
     // Warn that we're replacing the include/import with a module import.
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index db6069e31fa46..a373a52506a24 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -409,13 +409,13 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
   // Complain about reaching a true EOF within arc_cf_code_audited.
   // We don't want to complain about reaching the end of a macro
   // instantiation or a _Pragma.
-  if (PragmaARCCFCodeAuditedInfo.getLoc().isValid() && !isEndOfMacro &&
+  if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro &&
       !(CurLexer && CurLexer->Is_PragmaLexer)) {
-    Diag(PragmaARCCFCodeAuditedInfo.getLoc(),
+    Diag(PragmaARCCFCodeAuditedInfo.second,
          diag::err_pp_eof_in_arc_cf_code_audited);
 
     // Recover by leaving immediately.
-    PragmaARCCFCodeAuditedInfo = IdentifierLoc();
+    PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
   }
 
   // Complain about reaching a true EOF within assume_nonnull.
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
index 5b6a29bdad910..91c1619e35623 100644
--- a/clang/lib/Lex/Pragma.cpp
+++ b/clang/lib/Lex/Pragma.cpp
@@ -763,19 +763,20 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) {
 
 // Lex a component of a module name: either an identifier or a string literal;
 // for components that can be expressed both ways, the two forms are equivalent.
-static bool LexModuleNameComponent(Preprocessor &PP, Token &Tok,
-                                   IdentifierLoc &ModuleNameComponent,
-                                   bool First) {
+static bool LexModuleNameComponent(
+    Preprocessor &PP, Token &Tok,
+    std::pair<IdentifierInfo *, SourceLocation> &ModuleNameComponent,
+    bool First) {
   PP.LexUnexpandedToken(Tok);
   if (Tok.is(tok::string_literal) && !Tok.hasUDSuffix()) {
     StringLiteralParser Literal(Tok, PP);
     if (Literal.hadError)
       return true;
-    ModuleNameComponent = IdentifierLoc(
-        Tok.getLocation(), PP.getIdentifierInfo(Literal.GetString()));
+    ModuleNameComponent = std::make_pair(
+        PP.getIdentifierInfo(Literal.GetString()), Tok.getLocation());
   } else if (!Tok.isAnnotation() && Tok.getIdentifierInfo()) {
     ModuleNameComponent =
-        IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
+        std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation());
   } else {
     PP.Diag(Tok.getLocation(), diag::err_pp_expected_module_name) << First;
     return true;
@@ -783,10 +784,12 @@ static bool LexModuleNameComponent(Preprocessor &PP, Token &Tok,
   return false;
 }
 
-static bool LexModuleName(Preprocessor &PP, Token &Tok,
-                          llvm::SmallVectorImpl<IdentifierLoc> &ModuleName) {
+static bool LexModuleName(
+    Preprocessor &PP, Token &Tok,
+    llvm::SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>>
+        &ModuleName) {
   while (true) {
-    IdentifierLoc NameComponent;
+    std::pair<IdentifierInfo*, SourceLocation> NameComponent;
     if (LexModuleNameComponent(PP, Tok, NameComponent, ModuleName.empty()))
       return true;
     ModuleName.push_back(NameComponent);
@@ -800,10 +803,10 @@ static bool LexModuleName(Preprocessor &PP, Token &Tok,
 void Preprocessor::HandlePragmaModuleBuild(Token &Tok) {
   SourceLocation Loc = Tok.getLocation();
 
-  IdentifierLoc ModuleNameLoc;
+  std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc;
   if (LexModuleNameComponent(*this, Tok, ModuleNameLoc, true))
     return;
-  IdentifierInfo *ModuleName = ModuleNameLoc.getIdentifierInfo();
+  IdentifierInfo *ModuleName = ModuleNameLoc.first;
 
   LexUnexpandedToken(Tok);
   if (Tok.isNot(tok::eod)) {
@@ -1106,17 +1109,17 @@ struct PragmaDebugHandler : public PragmaHandler {
         PP.Diag(MacroName, diag::warn_pragma_debug_missing_argument)
             << II->getName();
     } else if (II->isStr("module_map")) {
-      llvm::SmallVector<IdentifierLoc, 8> ModuleName;
+      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
+          ModuleName;
       if (LexModuleName(PP, Tok, ModuleName))
         return;
       ModuleMap &MM = PP.getHeaderSearchInfo().getModuleMap();
       Module *M = nullptr;
       for (auto IIAndLoc : ModuleName) {
-        M = MM.lookupModuleQualified(IIAndLoc.getIdentifierInfo()->getName(),
-                                     M);
+        M = MM.lookupModuleQualified(IIAndLoc.first->getName(), M);
         if (!M) {
-          PP.Diag(IIAndLoc.getLoc(), diag::warn_pragma_debug_unknown_module)
-              << IIAndLoc.getIdentifierInfo()->getName();
+          PP.Diag(IIAndLoc.second, diag::warn_pragma_debug_unknown_module)
+              << IIAndLoc.first->getName();
           return;
         }
       }
@@ -1704,7 +1707,8 @@ struct PragmaModuleImportHandler : public PragmaHandler {
     SourceLocation ImportLoc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
+    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
+        ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1719,7 +1723,7 @@ struct PragmaModuleImportHandler : public PragmaHandler {
       return;
 
     PP.makeModuleVisible(Imported, ImportLoc);
-    PP.EnterAnnotationToken(SourceRange(ImportLoc, ModuleName.back().getLoc()),
+    PP.EnterAnnotationToken(SourceRange(ImportLoc, ModuleName.back().second),
                             tok::annot_module_include, Imported);
     if (auto *CB = PP.getPPCallbacks())
       CB->moduleImport(ImportLoc, ModuleName, Imported);
@@ -1740,7 +1744,8 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
     SourceLocation BeginLoc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
+    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
+        ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1749,11 +1754,10 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
 
     // We can only enter submodules of the current module.
     StringRef Current = PP.getLangOpts().CurrentModule;
-    if (ModuleName.front().getIdentifierInfo()->getName() != Current) {
-      PP.Diag(ModuleName.front().getLoc(),
-              diag::err_pp_module_begin_wrong_module)
-          << ModuleName.front().getIdentifierInfo() << (ModuleName.size() > 1)
-          << Current.empty() << Current;
+    if (ModuleName.front().first->getName() != Current) {
+      PP.Diag(ModuleName.front().second, diag::err_pp_module_begin_wrong_module)
+        << ModuleName.front().first << (ModuleName.size() > 1)
+        << Current.empty() << Current;
       return;
     }
 
@@ -1761,19 +1765,17 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
     // be loaded or implicitly loadable.
     auto &HSI = PP.getHeaderSearchInfo();
     auto &MM = HSI.getModuleMap();
-    Module *M = HSI.lookupModule(Current, ModuleName.front().getLoc());
+    Module *M = HSI.lookupModule(Current, ModuleName.front().second);
     if (!M) {
-      PP.Diag(ModuleName.front().getLoc(),
-              diag::err_pp_module_begin_no_module_map)
-          << Current;
+      PP.Diag(ModuleName.front().second,
+              diag::err_pp_module_begin_no_module_map) << Current;
       return;
     }
     for (unsigned I = 1; I != ModuleName.size(); ++I) {
-      auto *NewM = MM.findOrInferSubmodule(
-          M, ModuleName[I].getIdentifierInfo()->getName());
+      auto *NewM = MM.findOrInferSubmodule(M, ModuleName[I].first->getName());
       if (!NewM) {
-        PP.Diag(ModuleName[I].getLoc(), diag::err_pp_module_begin_no_submodule)
-            << M->getFullModuleName() << ModuleName[I].getIdentifierInfo();
+        PP.Diag(ModuleName[I].second, diag::err_pp_module_begin_no_submodule)
+          << M->getFullModuleName() << ModuleName[I].first;
         return;
       }
       M = NewM;
@@ -1789,7 +1791,7 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
 
     // Enter the scope of the submodule.
     PP.EnterSubmodule(M, BeginLoc, /*ForPragma*/true);
-    PP.EnterAnnotationToken(SourceRange(BeginLoc, ModuleName.back().getLoc()),
+    PP.EnterAnnotationToken(SourceRange(BeginLoc, ModuleName.back().second),
                             tok::annot_module_begin, M);
   }
 };
@@ -1833,7 +1835,8 @@ struct PragmaModuleLoadHandler : public PragmaHandler {
     SourceLocation Loc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
+    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
+        ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1898,7 +1901,7 @@ struct PragmaARCCFCodeAuditedHandler : public PragmaHandler {
       PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma";
 
     // The start location of the active audit.
-    SourceLocation BeginLoc = PP.getPragmaARCCFCodeAuditedInfo().getLoc();
+    SourceLocation BeginLoc = PP.getPragmaARCCFCodeAuditedInfo().second;
 
     // The start location we want after processing this.
     SourceLocation NewLoc;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 4c050bf1f5bb2..c25a3efd899e0 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -1159,8 +1159,8 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
     if (Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
       std::string Name = ModuleDeclState.getPrimaryName().str();
       Name += ":";
-      NamedModuleImportPath.emplace_back(Result.getLocation(),
-                                         getIdentifierInfo(Name));
+      NamedModuleImportPath.push_back(
+          {getIdentifierInfo(Name), Result.getLocation()});
       CurLexerCallback = CLK_LexAfterModuleImport;
       return true;
     }
@@ -1258,8 +1258,8 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
   if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
     // We expected to see an identifier here, and we did; continue handling
     // identifiers.
-    NamedModuleImportPath.emplace_back(Result.getLocation(),
-                                       Result.getIdentifierInfo());
+    NamedModuleImportPath.push_back(
+        std::make_pair(Result.getIdentifierInfo(), Result.getLocation()));
     ModuleImportExpectsIdentifier = false;
     CurLexerCallback = CLK_LexAfterModuleImport;
     return true;
@@ -1302,12 +1302,12 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
       // If the FlatModuleName ends with colon, it implies it is a partition.
       if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
         FlatModuleName += ".";
-      FlatModuleName += Piece.getIdentifierInfo()->getName();
+      FlatModuleName += Piece.first->getName();
     }
-    SourceLocation FirstPathLoc = NamedModuleImportPath[0].getLoc();
+    SourceLocation FirstPathLoc = NamedModuleImportPath[0].second;
     NamedModuleImportPath.clear();
-    NamedModuleImportPath.emplace_back(FirstPathLoc,
-                                       getIdentifierInfo(FlatModuleName));
+    NamedModuleImportPath.push_back(
+        std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
   }
 
   Module *Imported = nullptr;
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 8444ff3332e08..8fa74ecff19aa 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -432,8 +432,9 @@ static bool attributeParsedArgsUnevaluated(const IdentifierInfo &II,
 
 IdentifierLoc *Parser::ParseIdentifierLoc() {
   assert(Tok.is(tok::identifier) && "expected an identifier");
-  IdentifierLoc *IL = new (Actions.Context)
-      IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
+  IdentifierLoc *IL = IdentifierLoc::create(Actions.Context,
+                                            Tok.getLocation(),
+                                            Tok.getIdentifierInfo());
   ConsumeToken();
   return IL;
 }
@@ -1352,21 +1353,20 @@ void Parser::ParseAvailabilityAttribute(
     return;
   }
   IdentifierLoc *Platform = ParseIdentifierLoc();
-  if (const IdentifierInfo *const Ident = Platform->getIdentifierInfo()) {
+  if (const IdentifierInfo *const Ident = Platform->Ident) {
     // Disallow xrOS for availability attributes.
     if (Ident->getName().contains("xrOS") || Ident->getName().contains("xros"))
-      Diag(Platform->getLoc(), diag::warn_availability_unknown_platform)
-          << Ident;
+      Diag(Platform->Loc, diag::warn_availability_unknown_platform) << Ident;
     // Canonicalize platform name from "macosx" to "macos".
     else if (Ident->getName() == "macosx")
-      Platform->setIdentifierInfo(PP.getIdentifierInfo("macos"));
+      Platform->Ident = PP.getIdentifierInfo("macos");
     // Canonicalize platform name from "macosx_app_extension" to
     // "macos_app_extension".
     else if (Ident->getName() == "macosx_app_extension")
-      Platform->setIdentifierInfo(PP.getIdentifierInfo("macos_app_extension"));
+      Platform->Ident = PP.getIdentifierInfo("macos_app_extension");
     else
-      Platform->setIdentifierInfo(PP.getIdentifierInfo(
-          AvailabilityAttr::canonicalizePlatformName(Ident->getName())));
+      Platform->Ident = PP.getIdentifierInfo(
+          AvailabilityAttr::canonicalizePlatformName(Ident->getName()));
   }
 
   // Parse the ',' following the platform name.
@@ -1418,8 +1418,8 @@ void Parser::ParseAvailabilityAttribute(
       continue;
     }
 
-    if (Keyword == Ident_deprecated && Platform->getIdentifierInfo() &&
-        Platform->getIdentifierInfo()->isStr("swift")) {
+    if (Keyword == Ident_deprecated && Platform->Ident &&
+        Platform->Ident->isStr("swift")) {
       // For swift, we deprecate for all versions.
       if (Changes[Deprecated].KeywordLoc.isValid()) {
         Diag(KeywordLoc, diag::err_availability_redundant)
@@ -1436,7 +1436,7 @@ void Parser::ParseAvailabilityAttribute(
     if (Keyword == Ident_environment) {
       if (EnvironmentLoc != nullptr) {
         Diag(KeywordLoc, diag::err_availability_redundant)
-            << Keyword << SourceRange(EnvironmentLoc->getLoc());
+            << Keyword << SourceRange(EnvironmentLoc->Loc);
       }
     }
 
@@ -1792,8 +1792,8 @@ void Parser::ParseSwiftNewTypeAttribute(
     return;
   }
 
-  auto *SwiftType = new (Actions.Context)
-      IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
+  auto *SwiftType = IdentifierLoc::create(Actions.Context, Tok.getLocation(),
+                                          Tok.getIdentifierInfo());
   ConsumeToken();
 
   // Closing ')'
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 1416d52157dca..0a22f7372a9f9 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -4006,20 +4006,19 @@ std::optional<AvailabilitySpec> Parser::ParseAvailabilitySpec() {
     if (Version.empty())
       return std::nullopt;
 
-    StringRef GivenPlatform =
-        PlatformIdentifier->getIdentifierInfo()->getName();
+    StringRef GivenPlatform = PlatformIdentifier->Ident->getName();
     StringRef Platform =
         AvailabilityAttr::canonicalizePlatformName(GivenPlatform);
 
     if (AvailabilityAttr::getPrettyPlatformName(Platform).empty() ||
         (GivenPlatform.contains("xros") || GivenPlatform.contains("xrOS"))) {
-      Diag(PlatformIdentifier->getLoc(),
+      Diag(PlatformIdentifier->Loc,
            diag::err_avail_query_unrecognized_platform_name)
           << GivenPlatform;
       return std::nullopt;
     }
 
-    return AvailabilitySpec(Version, Platform, PlatformIdentifier->getLoc(),
+    return AvailabilitySpec(Version, Platform, PlatformIdentifier->Loc,
                             VersionRange.getEnd());
   }
 }
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index 7ffacc4b79f79..f4c109f9a81a2 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -115,7 +115,7 @@ static void fixSeparateAttrArgAndNumber(StringRef ArgStr, SourceLocation ArgLoc,
       << FixedArg
       << FixItHint::CreateReplacement(SourceRange(ArgLoc, EndNumLoc), FixedArg);
   ArgsUnion &Slot = ArgExprs.back();
-  Slot = new (Ctx) IdentifierLoc(ArgLoc, PP.getIdentifierInfo(FixedArg));
+  Slot = IdentifierLoc::create(Ctx, ArgLoc, PP.getIdentifierInfo(FixedArg));
 }
 
 void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index d872177b3d7aa..bcbf4dfbabafa 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -261,7 +261,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
   // case, LAngleLoc will be valid and ProtocolIdents will capture the
   // protocol references (that have not yet been resolved).
   SourceLocation LAngleLoc, EndProtoLoc;
-  SmallVector<IdentifierLoc, 8> ProtocolIdents;
+  SmallVector<IdentifierLocPair, 8> ProtocolIdents;
   ObjCTypeParamList *typeParameterList = nullptr;
   ObjCTypeParamListScope typeParamScope(Actions, getCurScope());
   if (Tok.is(tok::less))
@@ -361,8 +361,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
     if (!ProtocolIdents.empty()) {
       // We already parsed the protocols named when we thought we had a
       // type parameter list. Translate them into actual protocol references.
-      for (const auto &Loc : ProtocolIdents) {
-        protocolLocs.push_back(Loc.getLoc());
+      for (const auto &pair : ProtocolIdents) {
+        protocolLocs.push_back(pair.second);
       }
       Actions.ObjC().FindProtocolDeclaration(/*WarnOnDeclarations=*/true,
                                              /*ForObjCContainer=*/true,
@@ -459,8 +459,8 @@ static void addContextSensitiveTypeNullability(Parser &P,
 /// \param rAngleLoc The location of the ending '>'.
 ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     ObjCTypeParamListScope &Scope, SourceLocation &lAngleLoc,
-    SmallVectorImpl<IdentifierLoc> &protocolIdents, SourceLocation &rAngleLoc,
-    bool mayBeProtocolList) {
+    SmallVectorImpl<IdentifierLocPair> &protocolIdents,
+    SourceLocation &rAngleLoc, bool mayBeProtocolList) {
   assert(Tok.is(tok::less) && "Not at the beginning of a type parameter list");
 
   // Within the type parameter list, don't treat '>' as an operator.
@@ -474,8 +474,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     for (const auto &pair : protocolIdents) {
       DeclResult typeParam = Actions.ObjC().actOnObjCTypeParam(
           getCurScope(), ObjCTypeParamVariance::Invariant, SourceLocation(),
-          index++, pair.getIdentifierInfo(), pair.getLoc(), SourceLocation(),
-          nullptr);
+          index++, pair.first, pair.second, SourceLocation(), nullptr);
       if (typeParam.isUsable())
         typeParams.push_back(typeParam.get());
     }
@@ -547,7 +546,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     } else if (mayBeProtocolList) {
       // If this could still be a protocol list, just capture the identifier.
       // We don't want to turn it into a parameter.
-      protocolIdents.emplace_back(paramLoc, paramName);
+      protocolIdents.push_back(std::make_pair(paramName, paramLoc));
       continue;
     }
 
@@ -607,7 +606,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
 /// Parse an objc-type-parameter-list.
 ObjCTypeParamList *Parser::parseObjCTypeParamList() {
   SourceLocation lAngleLoc;
-  SmallVector<IdentifierLoc, 1> protocolIdents;
+  SmallVector<IdentifierLocPair, 1> protocolIdents;
   SourceLocation rAngleLoc;
 
   ObjCTypeParamListScope Scope(Actions, getCurScope());
@@ -1599,7 +1598,7 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
 
   LAngleLoc = ConsumeToken(); // the "<"
 
-  SmallVector<IdentifierLoc, 8> ProtocolIdents;
+  SmallVector<IdentifierLocPair, 8> ProtocolIdents;
 
   while (true) {
     if (Tok.is(tok::code_completion)) {
@@ -1613,7 +1612,8 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
       SkipUntil(tok::greater, StopAtSemi);
       return true;
     }
-    ProtocolIdents.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
+    ProtocolIdents.push_back(std::make_pair(Tok.getIdentifierInfo(),
+                                       Tok.getLocation()));
     ProtocolLocs.push_back(Tok.getLocation());
     ConsumeToken();
 
@@ -1693,9 +1693,10 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
 
     if (Tok.is(tok::code_completion)) {
       // FIXME: Also include types here.
-      SmallVector<IdentifierLoc, 4> identifierLocPairs;
+      SmallVector<IdentifierLocPair, 4> identifierLocPairs;
       for (unsigned i = 0, n = identifiers.size(); i != n; ++i) {
-        identifierLocPairs.emplace_back(identifierLocs[i], identifiers[i]);
+        identifierLocPairs.push_back(IdentifierLocPair(identifiers[i],
+                                                       identifierLocs[i]));
       }
 
       QualType BaseT = Actions.GetTypeFromParser(baseType);
@@ -2093,7 +2094,7 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   SourceLocation nameLoc = ConsumeToken();
 
   if (TryConsumeToken(tok::semi)) { // forward declaration of one protocol.
-    IdentifierLoc ProtoInfo(nameLoc, protocolName);
+    IdentifierLocPair ProtoInfo(protocolName, nameLoc);
     return Actions.ObjC().ActOnForwardProtocolDeclaration(AtLoc, ProtoInfo,
                                                           attrs);
   }
@@ -2101,8 +2102,8 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   CheckNestedObjCContexts(AtLoc);
 
   if (Tok.is(tok::comma)) { // list of forward declarations.
-    SmallVector<IdentifierLoc, 8> ProtocolRefs;
-    ProtocolRefs.emplace_back(nameLoc, protocolName);
+    SmallVector<IdentifierLocPair, 8> ProtocolRefs;
+    ProtocolRefs.push_back(std::make_pair(protocolName, nameLoc));
 
     // Parse the list of forward declarations.
     while (true) {
@@ -2111,7 +2112,8 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
         SkipUntil(tok::semi);
         return nullptr;
       }
-      ProtocolRefs.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
+      ProtocolRefs.push_back(IdentifierLocPair(Tok.getIdentifierInfo(),
+                                               Tok.getLocation()));
       ConsumeToken(); // the identifier
 
       if (Tok.isNot(tok::comma))
@@ -2194,7 +2196,7 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
   // permitted here. Parse and diagnose them.
   if (Tok.is(tok::less)) {
     SourceLocation lAngleLoc, rAngleLoc;
-    SmallVector<IdentifierLoc, 8> protocolIdents;
+    SmallVector<IdentifierLocPair, 8> protocolIdents;
     SourceLocation diagLoc = Tok.getLocation();
     ObjCTypeParamListScope typeParamScope(Actions, getCurScope());
     if (parseObjCTypeParamListOrProtocolRefs(typeParamScope, lAngleLoc,
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 337b3eca49764..64916995907c5 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -15,7 +15,6 @@
 #include "clang/Basic/OpenACCKinds.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
-#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/SemaOpenACC.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -815,7 +814,7 @@ bool Parser::ParseOpenACCIntExprList(OpenACCDirectiveKind DK,
 ///
 /// The device_type clause may be abbreviated to dtype.
 bool Parser::ParseOpenACCDeviceTypeList(
-    llvm::SmallVector<IdentifierLoc> &Archs) {
+    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs) {
 
   if (expectIdentifierOrKeyword(*this)) {
     SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
@@ -823,7 +822,7 @@ bool Parser::ParseOpenACCDeviceTypeList(
     return true;
   }
   IdentifierInfo *Ident = getCurToken().getIdentifierInfo();
-  Archs.emplace_back(ConsumeToken(), Ident);
+  Archs.emplace_back(Ident, ConsumeToken());
 
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     ExpectAndConsume(tok::comma);
@@ -834,7 +833,7 @@ bool Parser::ParseOpenACCDeviceTypeList(
       return true;
     }
     Ident = getCurToken().getIdentifierInfo();
-    Archs.emplace_back(ConsumeToken(), Ident);
+    Archs.emplace_back(Ident, ConsumeToken());
   }
   return false;
 }
@@ -1155,12 +1154,11 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
     }
     case OpenACCClauseKind::DType:
     case OpenACCClauseKind::DeviceType: {
-      llvm::SmallVector<IdentifierLoc> Archs;
+      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> Archs;
       if (getCurToken().is(tok::star)) {
         // FIXME: We want to mark that this is an 'everything else' type of
         // device_type in Sema.
-        ParsedClause.setDeviceTypeDetails(
-            {IdentifierLoc(ConsumeToken(), nullptr)});
+        ParsedClause.setDeviceTypeDetails({{nullptr, ConsumeToken()}});
       } else if (!ParseOpenACCDeviceTypeList(Archs)) {
         ParsedClause.setDeviceTypeDetails(std::move(Archs));
       } else {
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index 17b2b30942582..21ebff1e50559 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -1419,16 +1419,16 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
       static_cast<PragmaLoopHintInfo *>(Tok.getAnnotationValue());
 
   IdentifierInfo *PragmaNameInfo = Info->PragmaName.getIdentifierInfo();
-  Hint.PragmaNameLoc = new (Actions.Context)
-      IdentifierLoc(Info->PragmaName.getLocation(), PragmaNameInfo);
+  Hint.PragmaNameLoc = IdentifierLoc::create(
+      Actions.Context, Info->PragmaName.getLocation(), PragmaNameInfo);
 
   // It is possible that the loop hint has no option identifier, such as
   // #pragma unroll(4).
   IdentifierInfo *OptionInfo = Info->Option.is(tok::identifier)
                                    ? Info->Option.getIdentifierInfo()
                                    : nullptr;
-  Hint.OptionLoc = new (Actions.Context)
-      IdentifierLoc(Info->Option.getLocation(), OptionInfo);
+  Hint.OptionLoc = IdentifierLoc::create(
+      Actions.Context, Info->Option.getLocation(), OptionInfo);
 
   llvm::ArrayRef<Token> Toks = Info->Toks;
 
@@ -1508,7 +1508,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
     if (Toks.size() > 2)
       Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
           << PragmaLoopHintString(Info->PragmaName, Info->Option);
-    Hint.StateLoc = new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
+    Hint.StateLoc = IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
   } else if (OptionInfo && OptionInfo->getName() == "vectorize_width") {
     PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false,
                         /*IsReinject=*/false);
@@ -1529,7 +1529,8 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
           ConsumeAnyToken();
       }
 
-      Hint.StateLoc = new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
+      Hint.StateLoc =
+          IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
 
       ConsumeToken(); // Consume the constant expression eof terminator.
     } else {
@@ -1553,7 +1554,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
           Arg2Error = true;
         } else
           Hint.StateLoc =
-              new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
+              IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
 
         PP.Lex(Tok); // Identifier
       }
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 4a82d57fe566b..e8ec140fbe3e5 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -2545,9 +2545,9 @@ StmtResult Parser::ParsePragmaLoopHint(StmtVector &Stmts,
 
     ArgsUnion ArgHints[] = {Hint.PragmaNameLoc, Hint.OptionLoc, Hint.StateLoc,
                             ArgsUnion(Hint.ValueExpr)};
-    TempAttrs.addNew(Hint.PragmaNameLoc->getIdentifierInfo(), Hint.Range,
-                     /*scopeName=*/nullptr, Hint.PragmaNameLoc->getLoc(),
-                     ArgHints, /*numArgs=*/4, ParsedAttr::Form::Pragma());
+    TempAttrs.addNew(Hint.PragmaNameLoc->Ident, Hint.Range, nullptr,
+                     Hint.PragmaNameLoc->Loc, ArgHints, 4,
+                     ParsedAttr::Form::Pragma());
   }
 
   // Get the next statement.
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index d528664bca352..f3191762b1244 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -2541,17 +2541,17 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
     return Actions.ActOnPrivateModuleFragmentDecl(ModuleLoc, PrivateLoc);
   }
 
-  SmallVector<IdentifierLoc, 2> Path;
+  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
   if (ParseModuleName(ModuleLoc, Path, /*IsImport*/ false))
     return nullptr;
 
   // Parse the optional module-partition.
-  SmallVector<IdentifierLoc, 2> Partition;
+  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Partition;
   if (Tok.is(tok::colon)) {
     SourceLocation ColonLoc = ConsumeToken();
     if (!getLangOpts().CPlusPlusModules)
       Diag(ColonLoc, diag::err_unsupported_module_partition)
-          << SourceRange(ColonLoc, Partition.back().getLoc());
+          << SourceRange(ColonLoc, Partition.back().second);
     // Recover by ignoring the partition name.
     else if (ParseModuleName(ModuleLoc, Partition, /*IsImport*/ false))
       return nullptr;
@@ -2600,7 +2600,7 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
   SourceLocation ImportLoc = ConsumeToken();
 
   // For C++20 modules, we can have "name" or ":Partition name" as valid input.
-  SmallVector<IdentifierLoc, 2> Path;
+  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
   bool IsPartition = false;
   Module *HeaderUnit = nullptr;
   if (Tok.is(tok::header_name)) {
@@ -2616,7 +2616,7 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
     SourceLocation ColonLoc = ConsumeToken();
     if (!getLangOpts().CPlusPlusModules)
       Diag(ColonLoc, diag::err_unsupported_module_partition)
-          << SourceRange(ColonLoc, Path.back().getLoc());
+          << SourceRange(ColonLoc, Path.back().second);
     // Recover by leaving partition empty.
     else if (ParseModuleName(ColonLoc, Path, /*IsImport*/ true))
       return nullptr;
@@ -2718,9 +2718,10 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
 ///           module-name-qualifier[opt] identifier
 ///         module-name-qualifier:
 ///           module-name-qualifier[opt] identifier '.'
-bool Parser::ParseModuleName(SourceLocation UseLoc,
-                             SmallVectorImpl<IdentifierLoc> &Path,
-                             bool IsImport) {
+bool Parser::ParseModuleName(
+    SourceLocation UseLoc,
+    SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
+    bool IsImport) {
   // Parse the module path.
   while (true) {
     if (!Tok.is(tok::identifier)) {
@@ -2736,7 +2737,7 @@ bool Parser::ParseModuleName(SourceLocation UseLoc,
     }
 
     // Record this part of the module path.
-    Path.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
+    Path.push_back(std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation()));
     ConsumeToken();
 
     if (Tok.isNot(tok::period))
diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp
index c149cef478539..b19a02b8c1a09 100644
--- a/clang/lib/Sema/ParsedAttr.cpp
+++ b/clang/lib/Sema/ParsedAttr.cpp
@@ -23,6 +23,14 @@
 
 using namespace clang;
 
+IdentifierLoc *IdentifierLoc::create(ASTContext &Ctx, SourceLocation Loc,
+                                     IdentifierInfo *Ident) {
+  IdentifierLoc *Result = new (Ctx) IdentifierLoc;
+  Result->Loc = Loc;
+  Result->Ident = Ident;
+  return Result;
+}
+
 size_t ParsedAttr::allocated_size() const {
   if (IsAvailability) return AttributeFactory::AvailabilityAllocSize;
   else if (IsTypeTagForDatatype)
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 5bcbe78e9d633..3f53fb200a93d 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -1178,7 +1178,7 @@ void SemaARM::handleBuiltinAliasAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Ident = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *Ident = AL.getArgAsIdent(0)->Ident;
   unsigned BuiltinID = Ident->getBuiltinID();
   StringRef AliasName = cast<FunctionDecl>(D)->getIdentifier()->getName();
 
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 1e4e6fdc78351..f6ec4cb0f069e 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -8718,7 +8718,7 @@ static void AddProtocolResults(DeclContext *Ctx, DeclContext *CurContext,
 }
 
 void SemaCodeCompletion::CodeCompleteObjCProtocolReferences(
-    ArrayRef<IdentifierLoc> Protocols) {
+    ArrayRef<IdentifierLocPair> Protocols) {
   ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCProtocolName);
@@ -8729,9 +8729,9 @@ void SemaCodeCompletion::CodeCompleteObjCProtocolReferences(
     // Tell the result set to ignore all of the protocols we have
     // already seen.
     // FIXME: This doesn't work when caching code-completion results.
-    for (const IdentifierLoc &Pair : Protocols)
-      if (ObjCProtocolDecl *Protocol = SemaRef.ObjC().LookupProtocol(
-              Pair.getIdentifierInfo(), Pair.getLoc()))
+    for (const IdentifierLocPair &Pair : Protocols)
+      if (ObjCProtocolDecl *Protocol =
+              SemaRef.ObjC().LookupProtocol(Pair.first, Pair.second))
         Results.Ignore(Protocol);
 
     // Add all protocols.
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 7dd20a8795fc9..bc891fb009410 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -135,13 +135,13 @@ bool Sema::checkStringLiteralArgumentAttr(const ParsedAttr &AL, unsigned ArgNum,
   // Look for identifiers. If we have one emit a hint to fix it to a literal.
   if (AL.isArgIdent(ArgNum)) {
     IdentifierLoc *Loc = AL.getArgAsIdent(ArgNum);
-    Diag(Loc->getLoc(), diag::err_attribute_argument_type)
+    Diag(Loc->Loc, diag::err_attribute_argument_type)
         << AL << AANT_ArgumentString
-        << FixItHint::CreateInsertion(Loc->getLoc(), "\"")
-        << FixItHint::CreateInsertion(getLocForEndOfToken(Loc->getLoc()), "\"");
-    Str = Loc->getIdentifierInfo()->getName();
+        << FixItHint::CreateInsertion(Loc->Loc, "\"")
+        << FixItHint::CreateInsertion(getLocForEndOfToken(Loc->Loc), "\"");
+    Str = Loc->Ident->getName();
     if (ArgLocation)
-      *ArgLocation = Loc->getLoc();
+      *ArgLocation = Loc->Loc;
     return true;
   }
 
@@ -768,7 +768,7 @@ static void handleDiagnoseAsBuiltinAttr(Sema &S, Decl *D,
       auto Union = AL.getArg(Index - 1);
       if (auto *E = dyn_cast<Expr *>(Union))
         return E->getBeginLoc();
-      return cast<IdentifierLoc *>(Union)->getLoc();
+      return cast<IdentifierLoc *>(Union)->Loc;
     }();
 
     S.Diag(Loc, diag::err_attribute_argument_n_type) << AL << Index << T;
@@ -960,10 +960,10 @@ static void handleConsumableAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
-    if (!ConsumableAttr::ConvertStrToConsumedState(
-            IL->getIdentifierInfo()->getName(), DefaultState)) {
-      S.Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
-          << AL << IL->getIdentifierInfo();
+    if (!ConsumableAttr::ConvertStrToConsumedState(IL->Ident->getName(),
+                                                   DefaultState)) {
+      S.Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL
+                                                               << IL->Ident;
       return;
     }
   } else {
@@ -1005,8 +1005,8 @@ static void handleCallableWhenAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     SourceLocation Loc;
     if (AL.isArgIdent(ArgIndex)) {
       IdentifierLoc *Ident = AL.getArgAsIdent(ArgIndex);
-      StateString = Ident->getIdentifierInfo()->getName();
-      Loc = Ident->getLoc();
+      StateString = Ident->Ident->getName();
+      Loc = Ident->Loc;
     } else {
       if (!S.checkStringLiteralArgumentAttr(AL, ArgIndex, StateString, &Loc))
         return;
@@ -1030,11 +1030,11 @@ static void handleParamTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef StateString = Ident->getIdentifierInfo()->getName();
+    StringRef StateString = Ident->Ident->getName();
 
     if (!ParamTypestateAttr::ConvertStrToConsumedState(StateString,
                                                        ParamState)) {
-      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
+      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported)
           << AL << StateString;
       return;
     }
@@ -1064,10 +1064,10 @@ static void handleReturnTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
-    if (!ReturnTypestateAttr::ConvertStrToConsumedState(
-            IL->getIdentifierInfo()->getName(), ReturnState)) {
-      S.Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
-          << AL << IL->getIdentifierInfo();
+    if (!ReturnTypestateAttr::ConvertStrToConsumedState(IL->Ident->getName(),
+                                                        ReturnState)) {
+      S.Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL
+                                                               << IL->Ident;
       return;
     }
   } else {
@@ -1111,10 +1111,10 @@ static void handleSetTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   SetTypestateAttr::ConsumedState NewState;
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef Param = Ident->getIdentifierInfo()->getName();
+    StringRef Param = Ident->Ident->getName();
     if (!SetTypestateAttr::ConvertStrToConsumedState(Param, NewState)) {
-      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
-          << AL << Param;
+      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported) << AL
+                                                                  << Param;
       return;
     }
   } else {
@@ -1133,10 +1133,10 @@ static void handleTestTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   TestTypestateAttr::ConsumedState TestState;
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef Param = Ident->getIdentifierInfo()->getName();
+    StringRef Param = Ident->Ident->getName();
     if (!TestTypestateAttr::ConvertStrToConsumedState(Param, TestState)) {
-      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
-          << AL << Param;
+      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported) << AL
+                                                                  << Param;
       return;
     }
   } else {
@@ -1497,7 +1497,7 @@ static void handleOwnershipAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Module = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *Module = AL.getArgAsIdent(0)->Ident;
 
   StringRef ModuleName = Module->getName();
   if (normalizeName(ModuleName)) {
@@ -1864,10 +1864,10 @@ static void handleCPUSpecificAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *CPUArg = AL.getArgAsIdent(ArgNo);
-    StringRef CPUName = CPUArg->getIdentifierInfo()->getName().trim();
+    StringRef CPUName = CPUArg->Ident->getName().trim();
 
     if (!S.Context.getTargetInfo().validateCPUSpecificCPUDispatch(CPUName)) {
-      S.Diag(CPUArg->getLoc(), diag::err_invalid_cpu_specific_dispatch_value)
+      S.Diag(CPUArg->Loc, diag::err_invalid_cpu_specific_dispatch_value)
           << CPUName << (AL.getKind() == ParsedAttr::AT_CPUDispatch);
       return;
     }
@@ -1880,7 +1880,7 @@ static void handleCPUSpecificAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
       S.Diag(AL.getLoc(), diag::warn_multiversion_duplicate_entries);
       return;
     }
-    CPUs.push_back(CPUArg->getIdentifierInfo());
+    CPUs.push_back(CPUArg->Ident);
   }
 
   FD->setIsMultiVersion(true);
@@ -2358,10 +2358,10 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   IdentifierLoc *Platform = AL.getArgAsIdent(0);
 
-  IdentifierInfo *II = Platform->getIdentifierInfo();
+  IdentifierInfo *II = Platform->Ident;
   if (AvailabilityAttr::getPrettyPlatformName(II->getName()).empty())
-    S.Diag(Platform->getLoc(), diag::warn_availability_unknown_platform)
-        << Platform->getIdentifierInfo();
+    S.Diag(Platform->Loc, diag::warn_availability_unknown_platform)
+      << Platform->Ident;
 
   auto *ND = dyn_cast<NamedDecl>(D);
   if (!ND) // We warned about this already, so just return.
@@ -2410,16 +2410,14 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   IdentifierInfo *IIEnvironment = nullptr;
   if (EnvironmentLoc) {
     if (S.getLangOpts().HLSL) {
-      IIEnvironment = EnvironmentLoc->getIdentifierInfo();
+      IIEnvironment = EnvironmentLoc->Ident;
       if (AvailabilityAttr::getEnvironmentType(
-              EnvironmentLoc->getIdentifierInfo()->getName()) ==
+              EnvironmentLoc->Ident->getName()) ==
           llvm::Triple::EnvironmentType::UnknownEnvironment)
-        S.Diag(EnvironmentLoc->getLoc(),
-               diag::warn_availability_unknown_environment)
-            << EnvironmentLoc->getIdentifierInfo();
+        S.Diag(EnvironmentLoc->Loc, diag::warn_availability_unknown_environment)
+            << EnvironmentLoc->Ident;
     } else {
-      S.Diag(EnvironmentLoc->getLoc(),
-             diag::err_availability_unexpected_parameter)
+      S.Diag(EnvironmentLoc->Loc, diag::err_availability_unexpected_parameter)
           << "environment" << /* C/C++ */ 1;
     }
   }
@@ -3632,7 +3630,7 @@ static void handleEnumExtensibilityAttr(Sema &S, Decl *D,
   }
 
   EnumExtensibilityAttr::Kind ExtensibilityKind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
   if (!EnumExtensibilityAttr::ConvertStrToKind(II->getName(),
                                                ExtensibilityKind)) {
     S.Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
@@ -3855,7 +3853,7 @@ static bool handleFormatAttrCommon(Sema &S, Decl *D, const ParsedAttr &AL,
   bool HasImplicitThisParam = isInstanceMethod(D);
   Info->NumArgs = getFunctionOrMethodNumParams(D) + HasImplicitThisParam;
 
-  Info->Identifier = AL.getArgAsIdent(0)->getIdentifierInfo();
+  Info->Identifier = AL.getArgAsIdent(0)->Ident;
   StringRef Format = Info->Identifier->getName();
 
   if (normalizeName(Format)) {
@@ -4019,14 +4017,14 @@ static void handleCallbackAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
     if (AL.isArgIdent(I)) {
       IdentifierLoc *IdLoc = AL.getArgAsIdent(I);
-      auto It = NameIdxMapping.find(IdLoc->getIdentifierInfo()->getName());
+      auto It = NameIdxMapping.find(IdLoc->Ident->getName());
       if (It == UnknownName) {
         S.Diag(AL.getLoc(), diag::err_callback_attribute_argument_unknown)
-            << IdLoc->getIdentifierInfo() << IdLoc->getLoc();
+            << IdLoc->Ident << IdLoc->Loc;
         return;
       }
 
-      SR = SourceRange(IdLoc->getLoc());
+      SR = SourceRange(IdLoc->Loc);
       ArgIdx = It->second;
     } else if (AL.isArgExpr(I)) {
       Expr *IdxExpr = AL.getArgAsExpr(I);
@@ -4144,14 +4142,13 @@ LifetimeCaptureByAttr *Sema::ParseLifetimeCaptureByAttr(const ParsedAttr &AL,
     }
     assert(AL.isArgIdent(I));
     IdentifierLoc *IdLoc = AL.getArgAsIdent(I);
-    if (IdLoc->getIdentifierInfo()->getName() == ParamName) {
-      Diag(IdLoc->getLoc(), diag::err_capture_by_references_itself)
-          << IdLoc->getLoc();
+    if (IdLoc->Ident->getName() == ParamName) {
+      Diag(IdLoc->Loc, diag::err_capture_by_references_itself) << IdLoc->Loc;
       IsValid = false;
       continue;
     }
-    ParamIdents[I] = IdLoc->getIdentifierInfo();
-    ParamLocs[I] = IdLoc->getLoc();
+    ParamIdents[I] = IdLoc->Ident;
+    ParamLocs[I] = IdLoc->Loc;
   }
   if (!IsValid)
     return nullptr;
@@ -4757,7 +4754,7 @@ static void handleModeAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Name = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *Name = AL.getArgAsIdent(0)->Ident;
 
   S.AddModeAttr(D, AL, Name);
 }
@@ -5730,8 +5727,8 @@ static void handleArgumentWithTypeTagAttr(Sema &S, Decl *D,
   }
 
   D->addAttr(::new (S.Context) ArgumentWithTypeTagAttr(
-      S.Context, AL, AL.getArgAsIdent(0)->getIdentifierInfo(), ArgumentIdx,
-      TypeTagIdx, IsPointer));
+      S.Context, AL, AL.getArgAsIdent(0)->Ident, ArgumentIdx, TypeTagIdx,
+      IsPointer));
 }
 
 static void handleTypeTagForDatatypeAttr(Sema &S, Decl *D,
@@ -5751,7 +5748,7 @@ static void handleTypeTagForDatatypeAttr(Sema &S, Decl *D,
     return;
   }
 
-  IdentifierInfo *PointerKind = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *PointerKind = AL.getArgAsIdent(0)->Ident;
   TypeSourceInfo *MatchingCTypeLoc = nullptr;
   S.GetTypeFromParser(AL.getMatchingCType(), &MatchingCTypeLoc);
   assert(MatchingCTypeLoc && "no type source info for attribute argument");
@@ -5822,7 +5819,7 @@ static void handleBuiltinAliasAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Ident = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *Ident = AL.getArgAsIdent(0)->Ident;
   unsigned BuiltinID = Ident->getBuiltinID();
   StringRef AliasName = cast<FunctionDecl>(D)->getIdentifier()->getName();
 
@@ -6588,7 +6585,7 @@ static void handleCFGuardAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 
   CFGuardAttr::GuardArg Arg;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
   if (!CFGuardAttr::ConvertStrToGuardArg(II->getName(), Arg)) {
     S.Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
     return;
@@ -6690,9 +6687,8 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
     if (!VTablePointerAuthenticationAttr::ConvertStrToVPtrAuthKeyType(
-            IL->getIdentifierInfo()->getName(), KeyType)) {
-      S.Diag(IL->getLoc(), diag::err_invalid_authentication_key)
-          << IL->getIdentifierInfo();
+            IL->Ident->getName(), KeyType)) {
+      S.Diag(IL->Loc, diag::err_invalid_authentication_key) << IL->Ident;
       AL.setInvalid();
     }
     if (KeyType == VTablePointerAuthenticationAttr::DefaultKey &&
@@ -6712,16 +6708,15 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
     if (AL.isArgIdent(1)) {
       IdentifierLoc *IL = AL.getArgAsIdent(1);
       if (!VTablePointerAuthenticationAttr::
-              ConvertStrToAddressDiscriminationMode(
-                  IL->getIdentifierInfo()->getName(), AddressDiversityMode)) {
-        S.Diag(IL->getLoc(), diag::err_invalid_address_discrimination)
-            << IL->getIdentifierInfo();
+              ConvertStrToAddressDiscriminationMode(IL->Ident->getName(),
+                                                    AddressDiversityMode)) {
+        S.Diag(IL->Loc, diag::err_invalid_address_discrimination) << IL->Ident;
         AL.setInvalid();
       }
       if (AddressDiversityMode ==
               VTablePointerAuthenticationAttr::DefaultAddressDiscrimination &&
           !S.getLangOpts().PointerAuthCalls) {
-        S.Diag(IL->getLoc(), diag::err_no_default_vtable_pointer_auth) << 1;
+        S.Diag(IL->Loc, diag::err_no_default_vtable_pointer_auth) << 1;
         AL.setInvalid();
       }
     } else {
@@ -6736,9 +6731,8 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
     if (AL.isArgIdent(2)) {
       IdentifierLoc *IL = AL.getArgAsIdent(2);
       if (!VTablePointerAuthenticationAttr::ConvertStrToExtraDiscrimination(
-              IL->getIdentifierInfo()->getName(), ED)) {
-        S.Diag(IL->getLoc(), diag::err_invalid_extra_discrimination)
-            << IL->getIdentifierInfo();
+              IL->Ident->getName(), ED)) {
+        S.Diag(IL->Loc, diag::err_invalid_extra_discrimination) << IL->Ident;
         AL.setInvalid();
       }
       if (ED == VTablePointerAuthenticationAttr::DefaultExtraDiscrimination &&
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index 0a14ce23a396e..ba9d3dcf19617 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -1310,26 +1310,24 @@ static bool NestedProtocolHasNoDefinition(ObjCProtocolDecl *PDecl,
 /// protocol declarations in its 'Protocols' argument.
 void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
                                        bool ForObjCContainer,
-                                       ArrayRef<IdentifierLoc> ProtocolId,
+                                       ArrayRef<IdentifierLocPair> ProtocolId,
                                        SmallVectorImpl<Decl *> &Protocols) {
-  for (const IdentifierLoc &Pair : ProtocolId) {
-    ObjCProtocolDecl *PDecl =
-        LookupProtocol(Pair.getIdentifierInfo(), Pair.getLoc());
+  for (const IdentifierLocPair &Pair : ProtocolId) {
+    ObjCProtocolDecl *PDecl = LookupProtocol(Pair.first, Pair.second);
     if (!PDecl) {
       DeclFilterCCC<ObjCProtocolDecl> CCC{};
-      TypoCorrection Corrected = SemaRef.CorrectTypo(
-          DeclarationNameInfo(Pair.getIdentifierInfo(), Pair.getLoc()),
-          Sema::LookupObjCProtocolName, SemaRef.TUScope, nullptr, CCC,
-          Sema::CTK_ErrorRecovery);
+      TypoCorrection Corrected =
+          SemaRef.CorrectTypo(DeclarationNameInfo(Pair.first, Pair.second),
+                              Sema::LookupObjCProtocolName, SemaRef.TUScope,
+                              nullptr, CCC, Sema::CTK_ErrorRecovery);
       if ((PDecl = Corrected.getCorrectionDeclAs<ObjCProtocolDecl>()))
         SemaRef.diagnoseTypo(Corrected,
                              PDiag(diag::err_undeclared_protocol_suggest)
-                                 << Pair.getIdentifierInfo());
+                                 << Pair.first);
     }
 
     if (!PDecl) {
-      Diag(Pair.getLoc(), diag::err_undeclared_protocol)
-          << Pair.getIdentifierInfo();
+      Diag(Pair.second, diag::err_undeclared_protocol) << Pair.first;
       continue;
     }
     // If this is a forward protocol declaration, get its definition.
@@ -1339,7 +1337,7 @@ void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
     // For an objc container, delay protocol reference checking until after we
     // can set the objc decl as the availability context, otherwise check now.
     if (!ForObjCContainer) {
-      (void)SemaRef.DiagnoseUseOfDecl(PDecl, Pair.getLoc());
+      (void)SemaRef.DiagnoseUseOfDecl(PDecl, Pair.second);
     }
 
     // If this is a forward declaration and we are supposed to warn in this
@@ -1349,8 +1347,7 @@ void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
 
     if (WarnOnDeclarations &&
         NestedProtocolHasNoDefinition(PDecl, UndefinedProtocol)) {
-      Diag(Pair.getLoc(), diag::warn_undef_protocolref)
-          << Pair.getIdentifierInfo();
+      Diag(Pair.second, diag::warn_undef_protocolref) << Pair.first;
       Diag(UndefinedProtocol->getLocation(), diag::note_protocol_decl_undefined)
         << UndefinedProtocol;
     }
@@ -1787,17 +1784,17 @@ void SemaObjC::DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
 
 /// ActOnForwardProtocolDeclaration - Handle \@protocol foo;
 SemaObjC::DeclGroupPtrTy SemaObjC::ActOnForwardProtocolDeclaration(
-    SourceLocation AtProtocolLoc, ArrayRef<IdentifierLoc> IdentList,
+    SourceLocation AtProtocolLoc, ArrayRef<IdentifierLocPair> IdentList,
     const ParsedAttributesView &attrList) {
   ASTContext &Context = getASTContext();
   SmallVector<Decl *, 8> DeclsInGroup;
-  for (const IdentifierLoc &IdentPair : IdentList) {
-    IdentifierInfo *Ident = IdentPair.getIdentifierInfo();
+  for (const IdentifierLocPair &IdentPair : IdentList) {
+    IdentifierInfo *Ident = IdentPair.first;
     ObjCProtocolDecl *PrevDecl = LookupProtocol(
-        Ident, IdentPair.getLoc(), SemaRef.forRedeclarationInCurContext());
+        Ident, IdentPair.second, SemaRef.forRedeclarationInCurContext());
     ObjCProtocolDecl *PDecl =
         ObjCProtocolDecl::Create(Context, SemaRef.CurContext, Ident,
-                                 IdentPair.getLoc(), AtProtocolLoc, PrevDecl);
+                                 IdentPair.second, AtProtocolLoc, PrevDecl);
 
     SemaRef.PushOnScopeChains(PDecl, SemaRef.TUScope);
     CheckObjCDeclScope(PDecl);
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 11f156ae09216..0b442b75d174d 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -1274,8 +1274,8 @@ bool SemaHLSL::handleResourceTypeAttr(QualType T, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *Loc = AL.getArgAsIdent(0);
-    StringRef Identifier = Loc->getIdentifierInfo()->getName();
-    SourceLocation ArgLoc = Loc->getLoc();
+    StringRef Identifier = Loc->Ident->getName();
+    SourceLocation ArgLoc = Loc->Loc;
 
     // Validate resource class value
     ResourceClass RC;
@@ -1534,8 +1534,8 @@ void SemaHLSL::handleResourceBindingAttr(Decl *TheDecl, const ParsedAttr &AL) {
   }
 
   IdentifierLoc *Loc = AL.getArgAsIdent(0);
-  StringRef Str = Loc->getIdentifierInfo()->getName();
-  SourceLocation ArgLoc = Loc->getLoc();
+  StringRef Str = Loc->Ident->getName();
+  SourceLocation ArgLoc = Loc->Loc;
 
   SourceLocation SpaceArgLoc;
   bool SpecifiedSpace = false;
@@ -1549,8 +1549,8 @@ void SemaHLSL::handleResourceBindingAttr(Decl *TheDecl, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *Loc = AL.getArgAsIdent(1);
-    Space = Loc->getIdentifierInfo()->getName();
-    SpaceArgLoc = Loc->getLoc();
+    Space = Loc->Ident->getName();
+    SpaceArgLoc = Loc->Loc;
   } else {
     Slot = Str;
   }
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index 4bba57193ded6..76589bff40be9 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -15,7 +15,6 @@
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
-#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/StringExtras.h"
 
@@ -69,7 +68,7 @@ static std::string stringFromPath(ModuleIdPath Path) {
   for (auto &Piece : Path) {
     if (!Name.empty())
       Name += ".";
-    Name += Piece.getIdentifierInfo()->getName();
+    Name += Piece.first->getName();
   }
   return Name;
 }
@@ -351,18 +350,17 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
 
   // Test the first part of the path to see if it's std[0-9]+ but allow the
   // name in a system header.
-  StringRef FirstComponentName = Path[0].getIdentifierInfo()->getName();
-  if (!getSourceManager().isInSystemHeader(Path[0].getLoc()) &&
+  StringRef FirstComponentName = Path[0].first->getName();
+  if (!getSourceManager().isInSystemHeader(Path[0].second) &&
       (FirstComponentName == "std" ||
        (FirstComponentName.starts_with("std") &&
         llvm::all_of(FirstComponentName.drop_front(3), &llvm::isDigit))))
-    Diag(Path[0].getLoc(), diag::warn_reserved_module_name)
-        << Path[0].getIdentifierInfo();
+    Diag(Path[0].second, diag::warn_reserved_module_name) << Path[0].first;
 
   // Then test all of the components in the path to see if any of them are
   // using another kind of reserved or invalid identifier.
   for (auto Part : Path) {
-    if (DiagReservedModuleName(*this, Part.getIdentifierInfo(), Part.getLoc()))
+    if (DiagReservedModuleName(*this, Part.first, Part.second))
       return nullptr;
   }
 
@@ -378,10 +376,10 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
   // correct.
   if (!getLangOpts().CurrentModule.empty() &&
       getLangOpts().CurrentModule != ModuleName) {
-    Diag(Path.front().getLoc(), diag::err_current_module_name_mismatch)
-        << SourceRange(Path.front().getLoc(), IsPartition
-                                                  ? Partition.back().getLoc()
-                                                  : Path.back().getLoc())
+    Diag(Path.front().second, diag::err_current_module_name_mismatch)
+        << SourceRange(Path.front().second, IsPartition
+                                                ? Partition.back().second
+                                                : Path.back().second)
         << getLangOpts().CurrentModule;
     return nullptr;
   }
@@ -396,7 +394,7 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
     // We can't have parsed or imported a definition of this module or parsed a
     // module map defining it already.
     if (auto *M = Map.findModule(ModuleName)) {
-      Diag(Path[0].getLoc(), diag::err_module_redefinition) << ModuleName;
+      Diag(Path[0].second, diag::err_module_redefinition) << ModuleName;
       if (M->DefinitionLoc.isValid())
         Diag(M->DefinitionLoc, diag::note_prev_module_definition);
       else if (OptionalFileEntryRef FE = M->getASTFile())
@@ -419,8 +417,8 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
     // keyword nor a module-partition implicitly imports the primary
     // module interface unit of the module as if by a module-import-
     // declaration.
-    IdentifierLoc ModuleNameLoc(Path[0].getLoc(),
-                                PP.getIdentifierInfo(ModuleName));
+    std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc(
+        PP.getIdentifierInfo(ModuleName), Path[0].second);
 
     // The module loader will assume we're trying to import the module that
     // we're building if `LangOpts.CurrentModule` equals to 'ModuleName'.
@@ -492,7 +490,7 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
 
     // Make the import decl for the interface in the impl module.
     ImportDecl *Import = ImportDecl::Create(Context, CurContext, ModuleLoc,
-                                            Interface, Path[0].getLoc());
+                                            Interface, Path[0].second);
     CurContext->addDecl(Import);
 
     // Sequence initialization of the imported module before that of the current
@@ -581,7 +579,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
 
   // For a C++20 module name, flatten into a single identifier with the source
   // location of the first component.
-  IdentifierLoc ModuleNameLoc;
+  std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc;
 
   std::string ModuleName;
   if (IsPartition) {
@@ -593,13 +591,11 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
     ModuleName = NamedMod->getPrimaryModuleInterfaceName().str();
     ModuleName += ":";
     ModuleName += stringFromPath(Path);
-    ModuleNameLoc =
-        IdentifierLoc(Path[0].getLoc(), PP.getIdentifierInfo(ModuleName));
+    ModuleNameLoc = {PP.getIdentifierInfo(ModuleName), Path[0].second};
     Path = ModuleIdPath(ModuleNameLoc);
   } else if (getLangOpts().CPlusPlusModules) {
     ModuleName = stringFromPath(Path);
-    ModuleNameLoc =
-        IdentifierLoc(Path[0].getLoc(), PP.getIdentifierInfo(ModuleName));
+    ModuleNameLoc = {PP.getIdentifierInfo(ModuleName), Path[0].second};
     Path = ModuleIdPath(ModuleNameLoc);
   }
 
@@ -684,7 +680,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
       IdentifierLocs.push_back(SourceLocation());
   } else if (getLangOpts().CPlusPlusModules && !Mod->Parent) {
     // A single identifier for the whole name.
-    IdentifierLocs.push_back(Path[0].getLoc());
+    IdentifierLocs.push_back(Path[0].second);
   } else {
     Module *ModCheck = Mod;
     for (unsigned I = 0, N = Path.size(); I != N; ++I) {
@@ -694,7 +690,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
         break;
       ModCheck = ModCheck->Parent;
 
-      IdentifierLocs.push_back(Path[I].getLoc());
+      IdentifierLocs.push_back(Path[I].second);
     }
   }
 
@@ -711,7 +707,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
   if (getLangOpts().CPlusPlusModules && ExportLoc.isValid() &&
       Mod->Kind == Module::ModuleKind::ModulePartitionImplementation) {
     Diag(ExportLoc, diag::err_export_partition_impl)
-        << SourceRange(ExportLoc, Path.back().getLoc());
+        << SourceRange(ExportLoc, Path.back().second);
   } else if (!ModuleScopes.empty() && !currentModuleIsImplementation()) {
     // Re-export the module if the imported module is exported.
     // Note that we don't need to add re-exported module to Imports field
diff --git a/clang/lib/Sema/SemaObjC.cpp b/clang/lib/Sema/SemaObjC.cpp
index 9b24b5f052119..073d9791d037b 100644
--- a/clang/lib/Sema/SemaObjC.cpp
+++ b/clang/lib/Sema/SemaObjC.cpp
@@ -1446,8 +1446,10 @@ SemaObjC::ObjCSubscriptKind SemaObjC::CheckSubscriptingKind(Expr *FromE) {
 
 void SemaObjC::AddCFAuditedAttribute(Decl *D) {
   ASTContext &Context = getASTContext();
-  auto IdLoc = SemaRef.PP.getPragmaARCCFCodeAuditedInfo();
-  if (!IdLoc.getLoc().isValid())
+  IdentifierInfo *Ident;
+  SourceLocation Loc;
+  std::tie(Ident, Loc) = SemaRef.PP.getPragmaARCCFCodeAuditedInfo();
+  if (!Loc.isValid())
     return;
 
   // Don't add a redundant or conflicting attribute.
@@ -1455,8 +1457,7 @@ void SemaObjC::AddCFAuditedAttribute(Decl *D) {
       D->hasAttr<CFUnknownTransferAttr>())
     return;
 
-  AttributeCommonInfo Info(IdLoc.getIdentifierInfo(),
-                           SourceRange(IdLoc.getLoc()),
+  AttributeCommonInfo Info(Ident, SourceRange(Loc),
                            AttributeCommonInfo::Form::Pragma());
   D->addAttr(CFAuditedTransferAttr::CreateImplicit(Context, Info));
 }
@@ -1641,10 +1642,8 @@ void SemaObjC::handleMethodFamilyAttr(Decl *D, const ParsedAttr &AL) {
 
   IdentifierLoc *IL = AL.getArgAsIdent(0);
   ObjCMethodFamilyAttr::FamilyKind F;
-  if (!ObjCMethodFamilyAttr::ConvertStrToFamilyKind(
-          IL->getIdentifierInfo()->getName(), F)) {
-    Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
-        << AL << IL->getIdentifierInfo();
+  if (!ObjCMethodFamilyAttr::ConvertStrToFamilyKind(IL->Ident->getName(), F)) {
+    Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL << IL->Ident;
     return;
   }
 
@@ -1707,7 +1706,7 @@ void SemaObjC::handleBlocksAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
   BlocksAttr::BlockType type;
   if (!BlocksAttr::ConvertStrToBlockType(II->getName(), type)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
@@ -1999,7 +1998,7 @@ void SemaObjC::handleNSErrorDomain(Decl *D, const ParsedAttr &Attr) {
 
   IdentifierLoc *IdentLoc =
       Attr.isArgIdent(0) ? Attr.getArgAsIdent(0) : nullptr;
-  if (!IdentLoc || !IdentLoc->getIdentifierInfo()) {
+  if (!IdentLoc || !IdentLoc->Ident) {
     // Try to locate the argument directly.
     SourceLocation Loc = Attr.getLoc();
     if (Attr.isArgExpr(0) && Attr.getArgAsExpr(0))
@@ -2010,18 +2009,18 @@ void SemaObjC::handleNSErrorDomain(Decl *D, const ParsedAttr &Attr) {
   }
 
   // Verify that the identifier is a valid decl in the C decl namespace.
-  LookupResult Result(SemaRef, DeclarationName(IdentLoc->getIdentifierInfo()),
+  LookupResult Result(SemaRef, DeclarationName(IdentLoc->Ident),
                       SourceLocation(),
                       Sema::LookupNameKind::LookupOrdinaryName);
   if (!SemaRef.LookupName(Result, SemaRef.TUScope) ||
       !Result.getAsSingle<VarDecl>()) {
-    Diag(IdentLoc->getLoc(), diag::err_nserrordomain_invalid_decl)
-        << 1 << IdentLoc->getIdentifierInfo();
+    Diag(IdentLoc->Loc, diag::err_nserrordomain_invalid_decl)
+        << 1 << IdentLoc->Ident;
     return;
   }
 
-  D->addAttr(::new (getASTContext()) NSErrorDomainAttr(
-      getASTContext(), Attr, IdentLoc->getIdentifierInfo()));
+  D->addAttr(::new (getASTContext())
+                 NSErrorDomainAttr(getASTContext(), Attr, IdentLoc->Ident));
 }
 
 void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
@@ -2034,7 +2033,7 @@ void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
 
   // Typedefs only allow objc_bridge(id) and have some additional checking.
   if (const auto *TD = dyn_cast<TypedefNameDecl>(D)) {
-    if (!Parm->getIdentifierInfo()->isStr("id")) {
+    if (!Parm->Ident->isStr("id")) {
       Diag(AL.getLoc(), diag::err_objc_attr_typedef_not_id) << AL;
       return;
     }
@@ -2047,8 +2046,8 @@ void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
     }
   }
 
-  D->addAttr(::new (getASTContext()) ObjCBridgeAttr(getASTContext(), AL,
-                                                    Parm->getIdentifierInfo()));
+  D->addAttr(::new (getASTContext())
+                 ObjCBridgeAttr(getASTContext(), AL, Parm->Ident));
 }
 
 void SemaObjC::handleBridgeMutableAttr(Decl *D, const ParsedAttr &AL) {
@@ -2059,21 +2058,21 @@ void SemaObjC::handleBridgeMutableAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  D->addAttr(::new (getASTContext()) ObjCBridgeMutableAttr(
-      getASTContext(), AL, Parm->getIdentifierInfo()));
+  D->addAttr(::new (getASTContext())
+                 ObjCBridgeMutableAttr(getASTContext(), AL, Parm->Ident));
 }
 
 void SemaObjC::handleBridgeRelatedAttr(Decl *D, const ParsedAttr &AL) {
   IdentifierInfo *RelatedClass =
-      AL.isArgIdent(0) ? AL.getArgAsIdent(0)->getIdentifierInfo() : nullptr;
+      AL.isArgIdent(0) ? AL.getArgAsIdent(0)->Ident : nullptr;
   if (!RelatedClass) {
     Diag(D->getBeginLoc(), diag::err_objc_attr_not_id) << AL << 0;
     return;
   }
   IdentifierInfo *ClassMethod =
-      AL.getArgAsIdent(1) ? AL.getArgAsIdent(1)->getIdentifierInfo() : nullptr;
+      AL.getArgAsIdent(1) ? AL.getArgAsIdent(1)->Ident : nullptr;
   IdentifierInfo *InstanceMethod =
-      AL.getArgAsIdent(2) ? AL.getArgAsIdent(2)->getIdentifierInfo() : nullptr;
+      AL.getArgAsIdent(2) ? AL.getArgAsIdent(2)->Ident : nullptr;
   D->addAttr(::new (getASTContext()) ObjCBridgeRelatedAttr(
       getASTContext(), AL, RelatedClass, ClassMethod, InstanceMethod));
 }
diff --git a/clang/lib/Sema/SemaOpenACCClause.cpp b/clang/lib/Sema/SemaOpenACCClause.cpp
index 049baead031a1..ab25dcfd1a081 100644
--- a/clang/lib/Sema/SemaOpenACCClause.cpp
+++ b/clang/lib/Sema/SemaOpenACCClause.cpp
@@ -1343,7 +1343,7 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
   // the limitation, since the Dialect requires this.
   if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Set &&
       Clause.getDeviceTypeArchitectures().size() > 1) {
-    SemaRef.Diag(Clause.getDeviceTypeArchitectures()[1].getLoc(),
+    SemaRef.Diag(Clause.getDeviceTypeArchitectures()[1].second,
                  diag::err_acc_device_type_multiple_archs);
     return nullptr;
   }
@@ -1369,17 +1369,16 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
   bool Diagnosed = false;
   auto FilterPred = [&](const DeviceTypeArgument &Arch) {
     // The '*' case.
-    if (!Arch.getIdentifierInfo())
+    if (!Arch.first)
       return false;
     return llvm::find_if(ValidValues, [&](StringRef RHS) {
-             return Arch.getIdentifierInfo()->getName().equals_insensitive(RHS);
+             return Arch.first->getName().equals_insensitive(RHS);
            }) == ValidValues.end();
   };
 
   auto Diagnose = [&](const DeviceTypeArgument &Arch) {
-    Diagnosed = SemaRef.Diag(Arch.getLoc(), diag::err_acc_invalid_default_type)
-                << Arch.getIdentifierInfo() << Clause.getClauseKind()
-                << ValidValuesString;
+    Diagnosed = SemaRef.Diag(Arch.second, diag::err_acc_invalid_default_type)
+                << Arch.first << Clause.getClauseKind() << ValidValuesString;
   };
 
   // There aren't stable enumertor versions of 'for-each-then-erase', so do it
diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index a09626c3a9a8c..2f719c6d7a21e 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -79,10 +79,9 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
   Expr *ValueExpr = A.getArgAsExpr(3);
 
   StringRef PragmaName =
-      llvm::StringSwitch<StringRef>(
-          PragmaNameLoc->getIdentifierInfo()->getName())
+      llvm::StringSwitch<StringRef>(PragmaNameLoc->Ident->getName())
           .Cases("unroll", "nounroll", "unroll_and_jam", "nounroll_and_jam",
-                 PragmaNameLoc->getIdentifierInfo()->getName())
+                 PragmaNameLoc->Ident->getName())
           .Default("clang loop");
 
   // This could be handled automatically by adding a Subjects definition in
@@ -128,10 +127,10 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
       SetHints(LoopHintAttr::UnrollAndJam, LoopHintAttr::Enable);
   } else {
     // #pragma clang loop ...
-    assert(OptionLoc && OptionLoc->getIdentifierInfo() &&
+    assert(OptionLoc && OptionLoc->Ident &&
            "Attribute must have valid option info.");
     Option = llvm::StringSwitch<LoopHintAttr::OptionType>(
-                 OptionLoc->getIdentifierInfo()->getName())
+                 OptionLoc->Ident->getName())
                  .Case("vectorize", LoopHintAttr::Vectorize)
                  .Case("vectorize_width", LoopHintAttr::VectorizeWidth)
                  .Case("interleave", LoopHintAttr::Interleave)
@@ -145,13 +144,12 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                  .Case("distribute", LoopHintAttr::Distribute)
                  .Default(LoopHintAttr::Vectorize);
     if (Option == LoopHintAttr::VectorizeWidth) {
-      assert((ValueExpr || (StateLoc && StateLoc->getIdentifierInfo())) &&
+      assert((ValueExpr || (StateLoc && StateLoc->Ident)) &&
              "Attribute must have a valid value expression or argument.");
       if (ValueExpr && S.CheckLoopHintExpr(ValueExpr, St->getBeginLoc(),
                                            /*AllowZero=*/false))
         return nullptr;
-      if (StateLoc && StateLoc->getIdentifierInfo() &&
-          StateLoc->getIdentifierInfo()->isStr("scalable"))
+      if (StateLoc && StateLoc->Ident && StateLoc->Ident->isStr("scalable"))
         State = LoopHintAttr::ScalableWidth;
       else
         State = LoopHintAttr::FixedWidth;
@@ -169,15 +167,14 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                Option == LoopHintAttr::Unroll ||
                Option == LoopHintAttr::Distribute ||
                Option == LoopHintAttr::PipelineDisabled) {
-      assert(StateLoc && StateLoc->getIdentifierInfo() &&
-             "Loop hint must have an argument");
-      if (StateLoc->getIdentifierInfo()->isStr("disable"))
+      assert(StateLoc && StateLoc->Ident && "Loop hint must have an argument");
+      if (StateLoc->Ident->isStr("disable"))
         State = LoopHintAttr::Disable;
-      else if (StateLoc->getIdentifierInfo()->isStr("assume_safety"))
+      else if (StateLoc->Ident->isStr("assume_safety"))
         State = LoopHintAttr::AssumeSafety;
-      else if (StateLoc->getIdentifierInfo()->isStr("full"))
+      else if (StateLoc->Ident->isStr("full"))
         State = LoopHintAttr::Full;
-      else if (StateLoc->getIdentifierInfo()->isStr("enable"))
+      else if (StateLoc->Ident->isStr("enable"))
         State = LoopHintAttr::Enable;
       else
         llvm_unreachable("bad loop hint argument");
@@ -647,8 +644,8 @@ static Attr *handleAtomicAttr(Sema &S, Stmt *St, const ParsedAttr &AL,
     }
 
     IdentifierLoc *Ident = AL.getArgAsIdent(ArgIndex);
-    OptionString = Ident->getIdentifierInfo()->getName();
-    Loc = Ident->getLoc();
+    OptionString = Ident->Ident->getName();
+    Loc = Ident->Loc;
     if (!AtomicAttr::ConvertStrToConsumedOption(OptionString, Option)) {
       S.Diag(Loc, diag::err_attribute_invalid_atomic_argument) << OptionString;
       return nullptr;
diff --git a/clang/lib/Sema/SemaSwift.cpp b/clang/lib/Sema/SemaSwift.cpp
index 4aae855a24b8f..fe72d6c85c37a 100644
--- a/clang/lib/Sema/SemaSwift.cpp
+++ b/clang/lib/Sema/SemaSwift.cpp
@@ -148,8 +148,8 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
       return true;
 
     S.Diag(AL.getLoc(), diag::err_attr_swift_error_return_type)
-        << AL << AL.getArgAsIdent(0)->getIdentifierInfo()->getName()
-        << isa<ObjCMethodDecl>(D) << /*pointer*/ 1;
+        << AL << AL.getArgAsIdent(0)->Ident->getName() << isa<ObjCMethodDecl>(D)
+        << /*pointer*/ 1;
     return false;
   };
 
@@ -159,8 +159,8 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
       return true;
 
     S.Diag(AL.getLoc(), diag::err_attr_swift_error_return_type)
-        << AL << AL.getArgAsIdent(0)->getIdentifierInfo()->getName()
-        << isa<ObjCMethodDecl>(D) << /*integral*/ 0;
+        << AL << AL.getArgAsIdent(0)->Ident->getName() << isa<ObjCMethodDecl>(D)
+        << /*integral*/ 0;
     return false;
   };
 
@@ -169,10 +169,10 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
 
   IdentifierLoc *Loc = AL.getArgAsIdent(0);
   SwiftErrorAttr::ConventionKind Convention;
-  if (!SwiftErrorAttr::ConvertStrToConventionKind(
-          Loc->getIdentifierInfo()->getName(), Convention)) {
+  if (!SwiftErrorAttr::ConvertStrToConventionKind(Loc->Ident->getName(),
+                                                  Convention)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported)
-        << AL << Loc->getIdentifierInfo();
+        << AL << Loc->Ident;
     return;
   }
 
@@ -287,10 +287,10 @@ static void checkSwiftAsyncErrorBlock(Sema &S, Decl *D,
 void SemaSwift::handleAsyncError(Decl *D, const ParsedAttr &AL) {
   IdentifierLoc *IDLoc = AL.getArgAsIdent(0);
   SwiftAsyncErrorAttr::ConventionKind ConvKind;
-  if (!SwiftAsyncErrorAttr::ConvertStrToConventionKind(
-          IDLoc->getIdentifierInfo()->getName(), ConvKind)) {
+  if (!SwiftAsyncErrorAttr::ConvertStrToConventionKind(IDLoc->Ident->getName(),
+                                                       ConvKind)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported)
-        << AL << IDLoc->getIdentifierInfo();
+        << AL << IDLoc->Ident;
     return;
   }
 
@@ -643,7 +643,7 @@ void SemaSwift::handleNewType(Decl *D, const ParsedAttr &AL) {
   }
 
   SwiftNewTypeAttr::NewtypeKind Kind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
   if (!SwiftNewTypeAttr::ConvertStrToNewtypeKind(II->getName(), Kind)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
     return;
@@ -667,7 +667,7 @@ void SemaSwift::handleAsyncAttr(Decl *D, const ParsedAttr &AL) {
   }
 
   SwiftAsyncAttr::Kind Kind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
   if (!SwiftAsyncAttr::ConvertStrToKind(II->getName(), Kind)) {
     Diag(AL.getLoc(), diag::err_swift_async_no_access) << AL << II;
     return;
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index 87682233c5246..dc7e3a0bf8875 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -14,7 +14,6 @@
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Sema/Lookup.h"
-#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
@@ -756,7 +755,7 @@ bool Sema::CheckParameterPacksForExpansion(
     bool &RetainExpansion, UnsignedOrNone &NumExpansions) {
   ShouldExpand = true;
   RetainExpansion = false;
-  IdentifierLoc FirstPack;
+  std::pair<IdentifierInfo *, SourceLocation> FirstPack;
   bool HaveFirstPack = false;
   UnsignedOrNone NumPartialExpansions = std::nullopt;
   SourceLocation PartiallySubstitutedPackLoc;
@@ -868,7 +867,8 @@ bool Sema::CheckParameterPacksForExpansion(
       // This is the first pack we've seen for which we have an argument.
       // Record it.
       NumExpansions = NewPackSize;
-      FirstPack = IdentifierLoc(ParmPack.second, Name);
+      FirstPack.first = Name;
+      FirstPack.second = ParmPack.second;
       HaveFirstPack = true;
       continue;
     }
@@ -905,9 +905,9 @@ bool Sema::CheckParameterPacksForExpansion(
       //   the same number of arguments specified.
       if (HaveFirstPack)
         Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict)
-            << FirstPack.getIdentifierInfo() << Name << *NumExpansions
+            << FirstPack.first << Name << *NumExpansions
             << (LeastNewPackSize != NewPackSize) << LeastNewPackSize
-            << SourceRange(FirstPack.getLoc()) << SourceRange(ParmPack.second);
+            << SourceRange(FirstPack.second) << SourceRange(ParmPack.second);
       else
         Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict_multilevel)
             << Name << *NumExpansions << (LeastNewPackSize != NewPackSize)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 6e7ee8b5506ff..eba7267904fb2 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -99,8 +99,8 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
   StringRef name = attr.getAttrName()->getName();
 
   // The GC attributes are usually written with macros;  special-case them.
-  IdentifierInfo *II =
-      attr.isArgIdent(0) ? attr.getArgAsIdent(0)->getIdentifierInfo() : nullptr;
+  IdentifierInfo *II = attr.isArgIdent(0) ? attr.getArgAsIdent(0)->Ident
+                                          : nullptr;
   if (useExpansionLoc && loc.isMacroID() && II) {
     if (II->isStr("strong")) {
       if (S.findMacroSpelling(loc, "__strong")) name = "__strong";
@@ -5732,7 +5732,8 @@ static void transferARCOwnershipToDeclaratorChunk(TypeProcessingState &state,
   }
 
   IdentifierLoc *Arg = new (S.Context) IdentifierLoc;
-  Arg->setIdentifierInfo(&S.Context.Idents.get(attrStr));
+  Arg->Ident = &S.Context.Idents.get(attrStr);
+  Arg->Loc = SourceLocation();
 
   ArgsUnion Args(Arg);
 
@@ -6632,7 +6633,7 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
     return true;
   }
 
-  IdentifierInfo *II = attr.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *II = attr.getArgAsIdent(0)->Ident;
   Qualifiers::ObjCLifetime lifetime;
   if (II->isStr("none"))
     lifetime = Qualifiers::OCL_ExplicitNone;
@@ -6810,7 +6811,7 @@ static bool handleObjCGCTypeAttr(TypeProcessingState &state, ParsedAttr &attr,
     return true;
   }
 
-  IdentifierInfo *II = attr.getArgAsIdent(0)->getIdentifierInfo();
+  IdentifierInfo *II = attr.getArgAsIdent(0)->Ident;
   if (II->isStr("weak"))
     GCAttr = Qualifiers::Weak;
   else if (II->isStr("strong"))
@@ -7540,7 +7541,7 @@ static Attr *getCCTypeAttr(ASTContext &Ctx, ParsedAttr &Attr) {
     if (Attr.isArgExpr(0))
       Str = cast<StringLiteral>(Attr.getArgAsExpr(0))->getString();
     else
-      Str = Attr.getArgAsIdent(0)->getIdentifierInfo()->getName();
+      Str = Attr.getArgAsIdent(0)->Ident->getName();
     PcsAttr::PCSType Type;
     if (!PcsAttr::ConvertStrToPCSType(Str, Type))
       llvm_unreachable("already validated the attribute");
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index b404015867087..02c31dff620ec 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12811,7 +12811,7 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     for (unsigned I = 0; I < NumArchs; ++I) {
       IdentifierInfo *Ident = readBool() ? readIdentifier() : nullptr;
       SourceLocation Loc = readSourceLocation();
-      Archs.emplace_back(Loc, Ident);
+      Archs.emplace_back(Ident, Loc);
     }
 
     return OpenACCDeviceTypeClause::Create(getContext(), ClauseKind, BeginLoc,
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 4dca0613cb9ae..95b5718f1d140 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8774,10 +8774,10 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeSourceLocation(DTC->getLParenLoc());
     writeUInt32(DTC->getArchitectures().size());
     for (const DeviceTypeArgument &Arg : DTC->getArchitectures()) {
-      writeBool(Arg.getIdentifierInfo());
-      if (Arg.getIdentifierInfo())
-        AddIdentifierRef(Arg.getIdentifierInfo());
-      writeSourceLocation(Arg.getLoc());
+      writeBool(Arg.first);
+      if (Arg.first)
+        AddIdentifierRef(Arg.first);
+      writeSourceLocation(Arg.second);
     }
     return;
   }
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 07856dbdba4b4..429bf823616da 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -664,7 +664,7 @@ void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
                                         const Module *Imported) {
   if (MDC.ScanInstance.getPreprocessor().isInImportingCXXNamedModules()) {
     P1689ModuleInfo RequiredModule;
-    RequiredModule.ModuleName = Path[0].getIdentifierInfo()->getName().str();
+    RequiredModule.ModuleName = Path[0].first->getName().str();
     RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
     MDC.RequiredStdCXXModules.push_back(RequiredModule);
     return;

From ab7e0c0fc00b2c0ccae735cb0def103831d15b3b Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Wed, 16 Apr 2025 17:08:10 +0200
Subject: [PATCH 123/710] [clang][bytecode] Implement __builtin_wmem{cpy,move}
 (#135969)

---
 clang/lib/AST/ByteCode/InterpBuiltin.cpp      | 30 ++++++---
 clang/test/AST/ByteCode/builtin-functions.cpp | 62 +++++++++++++++++++
 2 files changed, 84 insertions(+), 8 deletions(-)

diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index d06941bf10fe0..b694a34e47ee0 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -1788,14 +1788,18 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
   Pointer DestPtr = getParam<Pointer>(Frame, 0);
   const ASTContext &ASTCtx = S.getASTContext();
   const Pointer &SrcPtr = getParam<Pointer>(Frame, 1);
-  const APSInt &Size =
-      peekToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)));
+  APSInt Size = peekToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)));
   assert(!Size.isSigned() && "memcpy and friends take an unsigned size");
 
   if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
     diagnoseNonConstexprBuiltin(S, OpPC, ID);
 
-  bool Move = (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove);
+  bool Move =
+      (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
+       ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
+  bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
+               ID == Builtin::BI__builtin_wmemcpy ||
+               ID == Builtin::BI__builtin_wmemmove;
 
   // If the size is zero, we treat this as always being a valid no-op.
   if (Size.isZero()) {
@@ -1806,7 +1810,7 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
   if (SrcPtr.isZero() || DestPtr.isZero()) {
     Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
     S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
-        << /*IsMove=*/Move << /*IsWchar=*/false << !SrcPtr.isZero()
+        << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
         << DiagPtr.toDiagnosticString(ASTCtx);
     return false;
   }
@@ -1818,7 +1822,7 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
                    ? std::to_string(SrcPtr.getIntegerRepresentation())
                    : std::to_string(DestPtr.getIntegerRepresentation());
     S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
-        << Move << false << DestPtr.isIntegralPointer() << DiagVal;
+        << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
     return false;
   }
 
@@ -1837,11 +1841,17 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
   }
   unsigned DestElemSize = ASTCtx.getTypeSizeInChars(DestElemType).getQuantity();
 
+  if (WChar) {
+    uint64_t WCharSize =
+        ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();
+    Size *= APSInt(APInt(Size.getBitWidth(), WCharSize, /*IsSigned=*/false),
+                   /*IsUnsigend=*/true);
+  }
+
   if (Size.urem(DestElemSize) != 0) {
     S.FFDiag(S.Current->getSource(OpPC),
              diag::note_constexpr_memcpy_unsupported)
-        << Move << /*IsWchar=*/false << 0 << DestElemType << Size
-        << DestElemSize;
+        << Move << WChar << 0 << DestElemType << Size << DestElemSize;
     return false;
   }
 
@@ -1869,7 +1879,7 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
     APInt N = Size.udiv(DestElemSize);
     S.FFDiag(S.Current->getSource(OpPC),
              diag::note_constexpr_memcpy_unsupported)
-        << Move << /*IsWChar*/ false << (Size.ugt(RemainingSrcBytes) ? 1 : 2)
+        << Move << WChar << (Size.ugt(RemainingSrcBytes) ? 1 : 2)
         << DestElemType << toString(N, 10, /*Signed=*/false);
     return false;
   }
@@ -2587,8 +2597,12 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
 
   case Builtin::BI__builtin_memcpy:
   case Builtin::BImemcpy:
+  case Builtin::BI__builtin_wmemcpy:
+  case Builtin::BIwmemcpy:
   case Builtin::BI__builtin_memmove:
   case Builtin::BImemmove:
+  case Builtin::BI__builtin_wmemmove:
+  case Builtin::BIwmemmove:
     if (!interp__builtin_memcpy(S, OpPC, Frame, F, Call))
       return false;
     break;
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 40f7a18119751..a57b4530d2264 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -30,6 +30,7 @@ extern "C" {
   extern wchar_t *wcschr(const wchar_t *s, wchar_t c);
   extern int wcscmp(const wchar_t *s1, const wchar_t *s2);
   extern int wcsncmp(const wchar_t *s1, const wchar_t *s2, size_t n);
+  extern wchar_t *wmemcpy(wchar_t *d, const wchar_t *s, size_t n);
 }
 
 namespace strcmp {
@@ -1592,6 +1593,67 @@ namespace WMemChr {
                                               // both-note {{non-constexpr function 'wcschr' cannot be used in a constant expression}}
 }
 
+namespace WMemCpy {
+  template<typename T>
+  constexpr T result(T (&arr)[4]) {
+    return arr[0] * 1000 + arr[1] * 100 + arr[2] * 10 + arr[3];
+  }
+  constexpr int test_wmemcpy(int a, int b, int n) {
+    wchar_t arr[4] = {1, 2, 3, 4};
+    __builtin_wmemcpy(arr + a, arr + b, n);
+    // both-note@-1 2{{overlapping memory regions}}
+    // both-note@-2 {{source is not a contiguous array of at least 2 elements of type 'wchar_t'}}
+    // both-note@-3 {{destination is not a contiguous array of at least 3 elements of type 'wchar_t'}}
+    return result(arr);
+  }
+  static_assert(test_wmemcpy(1, 2, 1) == 1334);
+  static_assert(test_wmemcpy(2, 1, 1) == 1224);
+  static_assert(test_wmemcpy(0, 1, 2) == 2334); // both-error {{constant}} both-note {{in call}}
+  static_assert(test_wmemcpy(1, 0, 2) == 1124); // both-error {{constant}} both-note {{in call}}
+  static_assert(test_wmemcpy(1, 2, 1) == 1334);
+  static_assert(test_wmemcpy(0, 3, 1) == 4234);
+  static_assert(test_wmemcpy(0, 3, 2) == 4234); // both-error {{constant}} both-note {{in call}}
+  static_assert(test_wmemcpy(2, 0, 3) == 4234); // both-error {{constant}} both-note {{in call}}
+
+  wchar_t global;
+  constexpr wchar_t *null = 0;
+  static_assert(__builtin_wmemcpy(&global, null, sizeof(wchar_t))); // both-error {{}} \
+                                                                    // both-note {{source of 'wmemcpy' is nullptr}}
+  static_assert(__builtin_wmemcpy(null, &global, sizeof(wchar_t))); // both-error {{}} \
+                                                                    // both-note {{destination of 'wmemcpy' is nullptr}}
+}
+
+namespace WMemMove {
+  template<typename T>
+  constexpr T result(T (&arr)[4]) {
+    return arr[0] * 1000 + arr[1] * 100 + arr[2] * 10 + arr[3];
+  }
+
+  constexpr int test_wmemmove(int a, int b, int n) {
+    wchar_t arr[4] = {1, 2, 3, 4};
+    __builtin_wmemmove(arr + a, arr + b, n);
+    // both-note@-1 {{source is not a contiguous array of at least 2 elements of type 'wchar_t'}}
+    // both-note@-2 {{destination is not a contiguous array of at least 3 elements of type 'wchar_t'}}
+    return result(arr);
+  }
+
+  static_assert(test_wmemmove(1, 2, 1) == 1334);
+  static_assert(test_wmemmove(2, 1, 1) == 1224);
+  static_assert(test_wmemmove(0, 1, 2) == 2334);
+  static_assert(test_wmemmove(1, 0, 2) == 1124);
+  static_assert(test_wmemmove(1, 2, 1) == 1334);
+  static_assert(test_wmemmove(0, 3, 1) == 4234);
+  static_assert(test_wmemmove(0, 3, 2) == 4234); // both-error {{constant}} both-note {{in call}}
+  static_assert(test_wmemmove(2, 0, 3) == 4234); // both-error {{constant}} both-note {{in call}}
+
+  wchar_t global;
+  constexpr wchar_t *null = 0;
+  static_assert(__builtin_wmemmove(&global, null, sizeof(wchar_t))); // both-error {{}} \
+                                                                     // both-note {{source of 'wmemmove' is nullptr}}
+  static_assert(__builtin_wmemmove(null, &global, sizeof(wchar_t))); // both-error {{}} \
+                                                                     // both-note {{destination of 'wmemmove' is nullptr}}
+}
+
 namespace Invalid {
   constexpr int test() { // both-error {{never produces a constant expression}}
     __builtin_abort(); // both-note 2{{subexpression not valid in a constant expression}}

From 2e9ab7cf96d802a906de342f32bc844036152ada Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 08:09:34 -0700
Subject: [PATCH 124/710] [NFC][Driver][CFI] Update boolean expression
 (#135881)

Show why we don't need regular CFI runtime, when CFI diag runtime is
linked.
---
 clang/lib/Driver/SanitizerArgs.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index 3c7cd562a14e3..1db9da1b60939 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -371,8 +371,7 @@ bool SanitizerArgs::needsUbsanCXXRt() const {
 }
 
 bool SanitizerArgs::needsCfiRt() const {
-  return !(Sanitizers.Mask & SanitizerKind::CFI & ~TrapSanitizers.Mask) &&
-         CfiCrossDso && !ImplicitCfiRuntime;
+  return !needsCfiDiagRt() && CfiCrossDso && !ImplicitCfiRuntime;
 }
 
 bool SanitizerArgs::needsCfiDiagRt() const {

From f875dd10162dcfb8f4625cef2bfc8e6b9f73f8fc Mon Sep 17 00:00:00 2001
From: Felipe de Azevedo Piovezan <fpiovezan@apple.com>
Date: Wed, 16 Apr 2025 08:10:29 -0700
Subject: [PATCH 125/710] [lldb][nfc] Remove redundant check in if statement
 (#135869)

We already check this boolean in the `if` statement two lines above.
---
 lldb/source/Target/ThreadPlanStepInRange.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Target/ThreadPlanStepInRange.cpp b/lldb/source/Target/ThreadPlanStepInRange.cpp
index 8a2417e9da326..0e93691de68af 100644
--- a/lldb/source/Target/ThreadPlanStepInRange.cpp
+++ b/lldb/source/Target/ThreadPlanStepInRange.cpp
@@ -370,7 +370,7 @@ bool ThreadPlanStepInRange::DefaultShouldStopHereCallback(
   if (!should_stop_here)
     return false;
 
-  if (should_stop_here && current_plan->GetKind() == eKindStepInRange &&
+  if (current_plan->GetKind() == eKindStepInRange &&
       operation == eFrameCompareYounger) {
     ThreadPlanStepInRange *step_in_range_plan =
         static_cast<ThreadPlanStepInRange *>(current_plan);

From d13135134c43af674584780b1494316f6fdaf027 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 08:12:27 -0700
Subject: [PATCH 126/710] [NFC][Driver][CFI] Rename to clarify purpose of CFI
 runtime (#135885)

CFI runtime (diagnostics, or not) is only needed
for cross-dso support.
---
 clang/include/clang/Driver/SanitizerArgs.h |  4 ++--
 clang/lib/Driver/SanitizerArgs.cpp         | 12 +++++++-----
 clang/lib/Driver/ToolChains/CommonArgs.cpp |  6 +++---
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/clang/include/clang/Driver/SanitizerArgs.h b/clang/include/clang/Driver/SanitizerArgs.h
index 6a866ded0e75c..528e3b400f3dc 100644
--- a/clang/include/clang/Driver/SanitizerArgs.h
+++ b/clang/include/clang/Driver/SanitizerArgs.h
@@ -105,8 +105,8 @@ class SanitizerArgs {
   bool requiresMinimalRuntime() const { return MinimalRuntime; }
   bool needsDfsanRt() const { return Sanitizers.has(SanitizerKind::DataFlow); }
   bool needsSafeStackRt() const { return SafeStackRuntime; }
-  bool needsCfiRt() const;
-  bool needsCfiDiagRt() const;
+  bool needsCfiCrossDsoRt() const;
+  bool needsCfiCrossDsoDiagRt() const;
   bool needsStatsRt() const { return Stats; }
   bool needsScudoRt() const { return Sanitizers.has(SanitizerKind::Scudo); }
   bool needsNsanRt() const {
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index 1db9da1b60939..ee151d5c68b85 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -354,8 +354,8 @@ bool SanitizerArgs::needsFuzzerInterceptors() const {
 bool SanitizerArgs::needsUbsanRt() const {
   // All of these include ubsan.
   if (needsAsanRt() || needsMsanRt() || needsNsanRt() || needsHwasanRt() ||
-      needsTsanRt() || needsDfsanRt() || needsLsanRt() || needsCfiDiagRt() ||
-      (needsScudoRt() && !requiresMinimalRuntime()))
+      needsTsanRt() || needsDfsanRt() || needsLsanRt() ||
+      needsCfiCrossDsoDiagRt() || (needsScudoRt() && !requiresMinimalRuntime()))
     return false;
 
   return (Sanitizers.Mask & NeedsUbsanRt & ~TrapSanitizers.Mask) ||
@@ -370,11 +370,13 @@ bool SanitizerArgs::needsUbsanCXXRt() const {
                            ~TrapSanitizers.Mask);
 }
 
-bool SanitizerArgs::needsCfiRt() const {
-  return !needsCfiDiagRt() && CfiCrossDso && !ImplicitCfiRuntime;
+bool SanitizerArgs::needsCfiCrossDsoRt() const {
+  // Diag runtime includes cross dso runtime.
+  return !needsCfiCrossDsoDiagRt() && CfiCrossDso && !ImplicitCfiRuntime;
 }
 
-bool SanitizerArgs::needsCfiDiagRt() const {
+bool SanitizerArgs::needsCfiCrossDsoDiagRt() const {
+  // UBSsan handles CFI diagnostics without cross-DSO suppport.
   return (Sanitizers.Mask & SanitizerKind::CFI & ~TrapSanitizers.Mask) &&
          CfiCrossDso && !ImplicitCfiRuntime;
 }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index ddeadff8f6dfb..7aab849abe0c1 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1550,14 +1550,14 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
     RequiredSymbols.push_back("__safestack_init");
   }
   if (!(SanArgs.needsSharedRt() && SanArgs.needsUbsanRt())) {
-    if (SanArgs.needsCfiRt())
+    if (SanArgs.needsCfiCrossDsoRt())
       StaticRuntimes.push_back("cfi");
-    if (SanArgs.needsCfiDiagRt())
+    if (SanArgs.needsCfiCrossDsoDiagRt())
       StaticRuntimes.push_back("cfi_diag");
   }
   if (SanArgs.linkCXXRuntimes() && !SanArgs.requiresMinimalRuntime() &&
       ((!SanArgs.needsSharedRt() && SanArgs.needsUbsanCXXRt()) ||
-       SanArgs.needsCfiDiagRt())) {
+       SanArgs.needsCfiCrossDsoDiagRt())) {
     StaticRuntimes.push_back("ubsan_standalone_cxx");
   }
   if (SanArgs.needsStatsRt()) {

From b73e5419f66a2501b4491514a72c7e361c25d57e Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 08:13:47 -0700
Subject: [PATCH 127/710] [NFC][CFI] Don't mix CFI and non-CFI flags on the
 same line (#135890)

---
 clang/test/Driver/sanitizer-ld.c | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index f9c3506e43208..52c1f6bf96242 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -830,58 +830,63 @@
 // CHECK-NSAN-UBSAN: "--whole-archive" "{{[^"]*}}libclang_rt.nsan.a" "--no-whole-archive"
 
 // CFI by itself does not link runtime libraries.
-// RUN: not %clang -fsanitize=cfi -### %s 2>&1 \
+// RUN: not %clang -fsanitize=cfi \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
+// RUN:     -### %s 2>&1 \
 // RUN:   | %{filecheck} --check-prefix=CHECK-CFI-LINUX
 // CHECK-CFI-LINUX: "{{.*}}ld{{(.exe)?}}"
 
 // CFI with diagnostics links the UBSan runtime.
 // RUN: not %clang -fsanitize=cfi -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
-// RUN:     -### %s 2>&1\
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
+// RUN:     -### %s 2>&1 \
 // RUN:   | %{filecheck} --check-prefix=CHECK-CFI-DIAG-LINUX
 // CHECK-CFI-DIAG-LINUX: "{{.*}}ld{{(.exe)?}}"
 // CHECK-CFI-DIAG-LINUX: "--whole-archive" "{{[^"]*}}libclang_rt.ubsan_standalone.a" "--no-whole-archive"
 
 // Cross-DSO CFI links the CFI runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso -### %s 2>&1 \
+// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
+// RUN:     -### %s 2>&1 \
 // RUN:   | %{filecheck} --check-prefix=CHECK-CFI-CROSS-DSO-LINUX
 // CHECK-CFI-CROSS-DSO-LINUX: "{{.*}}ld{{(.exe)?}}"
 // CHECK-CFI-CROSS-DSO-LINUX: "--whole-archive" "{{[^"]*}}libclang_rt.cfi.a" "--no-whole-archive"
 // CHECK-CFI-CROSS-DSO-LINUX: -export-dynamic
 
 // Cross-DSO CFI with diagnostics links just the CFI runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso -### %s 2>&1 \
+// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
 // RUN:     -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
+// RUN:     -### %s 2>&1 \
 // RUN:   | %{filecheck} --check-prefix=CHECK-CFI-CROSS-DSO-DIAG-LINUX
 // CHECK-CFI-CROSS-DSO-DIAG-LINUX: "{{.*}}ld{{(.exe)?}}"
 // CHECK-CFI-CROSS-DSO-DIAG-LINUX: "--whole-archive" "{{[^"]*}}libclang_rt.cfi_diag.a" "--no-whole-archive"
 // CHECK-CFI-CROSS-DSO-DIAG-LINUX: -export-dynamic
 
 // Cross-DSO CFI on Android does not link runtime libraries.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso -### %s 2>&1 \
+// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
 // RUN:     --target=aarch64-linux-android -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree \
+// RUN:     -### %s 2>&1 \
 // RUN:   | %{filecheck} --check-prefix=CHECK-CFI-CROSS-DSO-ANDROID
 // CHECK-CFI-CROSS-DSO-ANDROID: "{{.*}}ld{{(.exe)?}}"
 
 // Cross-DSO CFI with diagnostics on Android links just the UBSAN runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso -### %s 2>&1 \
+// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
 // RUN:     -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=aarch64-linux-android -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree \
+// RUN:     -### %s 2>&1 \
 // RUN:   | %{filecheck} --check-prefix=CHECK-CFI-CROSS-DSO-DIAG-ANDROID
 // CHECK-CFI-CROSS-DSO-DIAG-ANDROID: "{{.*}}ld{{(.exe)?}}"
 // CHECK-CFI-CROSS-DSO-DIAG-ANDROID: "{{[^"]*}}libclang_rt.ubsan_standalone.so"

From 8c04656c457e28680c60e8edc15a4b170b684ca2 Mon Sep 17 00:00:00 2001
From: Benjamin Chetioui <3920784+bchetioui@users.noreply.github.com>
Date: Wed, 16 Apr 2025 17:19:47 +0200
Subject: [PATCH 128/710] [bazel] Fix bazel build after
 2b983a24583dd4e131d727717872a56712b5dd52. (#135976)

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 85049ff8339c1..f7b379c2cd6d8 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -1562,18 +1562,23 @@ cc_library(
         ":AMDGPUDialect",
         ":AMDGPUPassIncGen",
         ":AMDGPUUtils",
+        ":AffineDialect",
         ":ArithDialect",
+        ":ArithUtils",
         ":ControlFlowDialect",
         ":FuncDialect",
         ":GPUDialect",
         ":IR",
         ":MemRefDialect",
+        ":MemRefUtils",
         ":Pass",
         ":SCFDialect",
         ":SideEffectInterfaces",
         ":Support",
         ":TransformUtils",
         ":VectorDialect",
+        ":VectorTransforms",
+        "//llvm:Support",
     ],
 )
 

From 30259076fec0af97e604ca943b61fb686b9b21ef Mon Sep 17 00:00:00 2001
From: Brox Chen <guochen2@amd.com>
Date: Wed, 16 Apr 2025 11:26:53 -0400
Subject: [PATCH 129/710] [AMDGPU][True16][MC] fix opsel for v_cmpx 16bit inst
 (#135441)

Fixed inst printer so that no opsel is generated for dst reg of cmpx
16bit insts
---
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp |   9 +-
 .../MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s     | 118 ++++++++---------
 .../MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s   | 120 +++++++++---------
 llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s        | 110 ++++++++--------
 .../MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s   | 108 ++++++++--------
 .../AMDGPU/gfx11_dasm_vop3_from_vopcx.txt     | 118 ++++++++---------
 .../Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt | 110 ++++++++--------
 .../AMDGPU/gfx12_dasm_vop3cx_dpp16.txt        | 110 ++++++++--------
 8 files changed, 403 insertions(+), 400 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index caff8be3d7348..677df64555623 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -1220,6 +1220,10 @@ void AMDGPUInstPrinter::printPackedModifier(const MCInst *MI,
         (ModIdx != -1) ? MI->getOperand(ModIdx).getImm() : DefaultValue;
   }
 
+  const bool HasDst =
+      (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst) != -1) ||
+      (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst) != -1);
+
   // Print three values of neg/opsel for wmma instructions (prints 0 when there
   // is no src_modifier operand instead of not printing anything).
   if (MII.get(MI->getOpcode()).TSFlags & SIInstrFlags::IsSWMMAC ||
@@ -1238,9 +1242,8 @@ void AMDGPUInstPrinter::printPackedModifier(const MCInst *MI,
   }
 
   const bool HasDstSel =
-    NumOps > 0 &&
-    Mod == SISrcMods::OP_SEL_0 &&
-    MII.get(MI->getOpcode()).TSFlags & SIInstrFlags::VOP3_OPSEL;
+      HasDst && NumOps > 0 && Mod == SISrcMods::OP_SEL_0 &&
+      MII.get(MI->getOpcode()).TSFlags & SIInstrFlags::VOP3_OPSEL;
 
   const bool IsPacked =
     MII.get(MI->getOpcode()).TSFlags & SIInstrFlags::IsPacked;
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
index b7345ee6208ed..1e517754b0ecd 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
@@ -51,13 +51,13 @@ v_cmpx_class_f16_e64 v1.l, 0.5
 // GFX11: v_cmpx_class_f16_e64 v1.l, 0.5          ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_class_f16_e64 v1.h, v2.h
-// GFX11: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_class_f16_e64 v255.h, v2.l
-// GFX11: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_class_f16_e64 s105, v255.h
-// GFX11: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+// GFX11: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 
 v_cmpx_class_f32_e64 v1, v2
 // GFX11: v_cmpx_class_f32_e64 v1, v2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00]
@@ -195,10 +195,10 @@ v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_f32_e64 v1, v2
 // GFX11: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
@@ -327,10 +327,10 @@ v_cmpx_eq_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_i32_e64 v1, v2
 // GFX11: v_cmpx_eq_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00]
@@ -459,10 +459,10 @@ v_cmpx_eq_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_u32_e64 v1, v2
 // GFX11: v_cmpx_eq_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00]
@@ -591,10 +591,10 @@ v_cmpx_f_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_f_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x80,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_f_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_f_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x80,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_f_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x80,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_f_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_f_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x80,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_f_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x80,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_f_f32_e64 v1, v2
 // GFX11: v_cmpx_f_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x00]
@@ -885,10 +885,10 @@ v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_f32_e64 v1, v2
 // GFX11: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
@@ -1017,10 +1017,10 @@ v_cmpx_ge_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_i32_e64 v1, v2
 // GFX11: v_cmpx_ge_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00]
@@ -1149,10 +1149,10 @@ v_cmpx_ge_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_u32_e64 v1, v2
 // GFX11: v_cmpx_ge_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00]
@@ -1281,10 +1281,10 @@ v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_f32_e64 v1, v2
 // GFX11: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
@@ -1413,10 +1413,10 @@ v_cmpx_gt_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_i32_e64 v1, v2
 // GFX11: v_cmpx_gt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00]
@@ -1545,10 +1545,10 @@ v_cmpx_gt_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_u32_e64 v1, v2
 // GFX11: v_cmpx_gt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00]
@@ -1677,10 +1677,10 @@ v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_f32_e64 v1, v2
 // GFX11: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
@@ -1809,10 +1809,10 @@ v_cmpx_le_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_i32_e64 v1, v2
 // GFX11: v_cmpx_le_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00]
@@ -1941,10 +1941,10 @@ v_cmpx_le_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_u32_e64 v1, v2
 // GFX11: v_cmpx_le_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00]
@@ -2073,10 +2073,10 @@ v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lg_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lg_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lg_f32_e64 v1, v2
 // GFX11: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
@@ -2205,10 +2205,10 @@ v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_f32_e64 v1, v2
 // GFX11: v_cmpx_lt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00]
@@ -2337,10 +2337,10 @@ v_cmpx_lt_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_i32_e64 v1, v2
 // GFX11: v_cmpx_lt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00]
@@ -2469,10 +2469,10 @@ v_cmpx_lt_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_u32_e64 v1, v2
 // GFX11: v_cmpx_lt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00]
@@ -2601,10 +2601,10 @@ v_cmpx_ne_i16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ne_i16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ne_i16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ne_i32_e64 v1, v2
 // GFX11: v_cmpx_ne_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00]
@@ -2733,10 +2733,10 @@ v_cmpx_ne_u16_e64 0xfe0b, vcc_hi
 // GFX11: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ne_u16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ne_u16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ne_u32_e64 v1, v2
 // GFX11: v_cmpx_ne_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00]
@@ -2865,10 +2865,10 @@ v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_neq_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_neq_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_neq_f32_e64 v1, v2
 // GFX11: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
@@ -2997,10 +2997,10 @@ v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nge_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_nge_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nge_f32_e64 v1, v2
 // GFX11: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
@@ -3129,10 +3129,10 @@ v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ngt_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ngt_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ngt_f32_e64 v1, v2
 // GFX11: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
@@ -3261,10 +3261,10 @@ v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nle_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_nle_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nle_f32_e64 v1, v2
 // GFX11: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
@@ -3393,10 +3393,10 @@ v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nlg_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_nlg_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nlg_f32_e64 v1, v2
 // GFX11: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
@@ -3525,10 +3525,10 @@ v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nlt_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_nlt_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nlt_f32_e64 v1, v2
 // GFX11: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
@@ -3657,10 +3657,10 @@ v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_o_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_o_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_o_f32_e64 v1, v2
 // GFX11: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
@@ -4077,10 +4077,10 @@ v_cmpx_tru_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_t_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_t_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_t_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8f,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_t_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8f,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_t_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_t_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8f,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_t_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8f,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_tru_f32_e64 v1, v2
 // GFX11: v_cmpx_t_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x9f,0xd4,0x01,0x05,0x02,0x00]
@@ -4209,10 +4209,10 @@ v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX11: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_u_f16_e64 v1.h, v2.l
-// GFX11: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+// GFX11: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_u_f16_e64 v255.l, v255.h
-// GFX11: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+// GFX11: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_u_f32_e64 v1, v2
 // GFX11: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
index 20864f291f71c..50feb32387eda 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx_t16_promote.s
@@ -2,7 +2,7 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 %s
 
 v_cmpx_class_f16 v1.h, v255.h
-// GFX11: v_cmpx_class_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_class_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_class_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -20,7 +20,7 @@ v_cmpx_class_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_class_f16 v255.h, v2.h
-// GFX11: v_cmpx_class_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_class_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_class_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -38,7 +38,7 @@ v_cmpx_class_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_class_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_f16 v1.h, v255.h
-// GFX11: v_cmpx_eq_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x82,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -56,7 +56,7 @@ v_cmpx_eq_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_f16 v255.h, v2.h
-// GFX11: v_cmpx_eq_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x82,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -74,7 +74,7 @@ v_cmpx_eq_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_i16 v1.h, v255.h
-// GFX11: v_cmpx_eq_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb2,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -92,7 +92,7 @@ v_cmpx_eq_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_i16 v255.h, v2.h
-// GFX11: v_cmpx_eq_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb2,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -110,7 +110,7 @@ v_cmpx_eq_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_u16 v1.h, v255.h
-// GFX11: v_cmpx_eq_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_eq_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xba,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -128,7 +128,7 @@ v_cmpx_eq_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_u16 v255.h, v2.h
-// GFX11: v_cmpx_eq_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_eq_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xba,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_eq_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -146,7 +146,7 @@ v_cmpx_eq_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_eq_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_f_f16 v1.h, v255.h
-// GFX11: v_cmpx_f_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x80,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_f_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x80,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_f_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_f_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x80,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -164,7 +164,7 @@ v_cmpx_f_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_f_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_f_f16 v255.h, v2.h
-// GFX11: v_cmpx_f_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x80,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_f_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x80,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_f_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_f_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x80,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -182,7 +182,7 @@ v_cmpx_f_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_f_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x80,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_f16 v1.h, v255.h
-// GFX11: v_cmpx_ge_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x86,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -200,7 +200,7 @@ v_cmpx_ge_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_f16 v255.h, v2.h
-// GFX11: v_cmpx_ge_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x86,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -218,7 +218,7 @@ v_cmpx_ge_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_i16 v1.h, v255.h
-// GFX11: v_cmpx_ge_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb6,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -236,7 +236,7 @@ v_cmpx_ge_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_i16 v255.h, v2.h
-// GFX11: v_cmpx_ge_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb6,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -254,7 +254,7 @@ v_cmpx_ge_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_u16 v1.h, v255.h
-// GFX11: v_cmpx_ge_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ge_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbe,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -272,7 +272,7 @@ v_cmpx_ge_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_u16 v255.h, v2.h
-// GFX11: v_cmpx_ge_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ge_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbe,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ge_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -290,7 +290,7 @@ v_cmpx_ge_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ge_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_f16 v1.h, v255.h
-// GFX11: v_cmpx_gt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x84,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -308,7 +308,7 @@ v_cmpx_gt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_f16 v255.h, v2.h
-// GFX11: v_cmpx_gt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x84,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -326,7 +326,7 @@ v_cmpx_gt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_i16 v1.h, v255.h
-// GFX11: v_cmpx_gt_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb4,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -344,7 +344,7 @@ v_cmpx_gt_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_i16 v255.h, v2.h
-// GFX11: v_cmpx_gt_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb4,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -362,7 +362,7 @@ v_cmpx_gt_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_u16 v1.h, v255.h
-// GFX11: v_cmpx_gt_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_gt_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbc,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -380,7 +380,7 @@ v_cmpx_gt_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_u16 v255.h, v2.h
-// GFX11: v_cmpx_gt_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_gt_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbc,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_gt_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -398,7 +398,7 @@ v_cmpx_gt_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_gt_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_f16 v1.h, v255.h
-// GFX11: v_cmpx_le_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x83,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -416,7 +416,7 @@ v_cmpx_le_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_f16 v255.h, v2.h
-// GFX11: v_cmpx_le_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x83,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -434,7 +434,7 @@ v_cmpx_le_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_i16 v1.h, v255.h
-// GFX11: v_cmpx_le_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb3,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -452,7 +452,7 @@ v_cmpx_le_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_i16 v255.h, v2.h
-// GFX11: v_cmpx_le_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb3,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -470,7 +470,7 @@ v_cmpx_le_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_u16 v1.h, v255.h
-// GFX11: v_cmpx_le_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_le_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbb,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -488,7 +488,7 @@ v_cmpx_le_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_u16 v255.h, v2.h
-// GFX11: v_cmpx_le_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_le_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbb,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_le_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -506,7 +506,7 @@ v_cmpx_le_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_le_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lg_f16 v1.h, v255.h
-// GFX11: v_cmpx_lg_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_lg_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x85,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lg_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -524,7 +524,7 @@ v_cmpx_lg_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lg_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lg_f16 v255.h, v2.h
-// GFX11: v_cmpx_lg_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_lg_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x85,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lg_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -542,7 +542,7 @@ v_cmpx_lg_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lg_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_f16 v1.h, v255.h
-// GFX11: v_cmpx_lt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x81,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -560,7 +560,7 @@ v_cmpx_lt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_f16 v255.h, v2.h
-// GFX11: v_cmpx_lt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x81,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -578,7 +578,7 @@ v_cmpx_lt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_i16 v1.h, v255.h
-// GFX11: v_cmpx_lt_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb1,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -596,7 +596,7 @@ v_cmpx_lt_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_i16 v255.h, v2.h
-// GFX11: v_cmpx_lt_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb1,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -614,7 +614,7 @@ v_cmpx_lt_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_u16 v1.h, v255.h
-// GFX11: v_cmpx_lt_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_lt_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb9,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -632,7 +632,7 @@ v_cmpx_lt_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_u16 v255.h, v2.h
-// GFX11: v_cmpx_lt_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_lt_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb9,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_lt_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -650,7 +650,7 @@ v_cmpx_lt_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_lt_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ne_i16 v1.h, v255.h
-// GFX11: v_cmpx_ne_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ne_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb5,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ne_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ne_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -668,7 +668,7 @@ v_cmpx_ne_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ne_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ne_i16 v255.h, v2.h
-// GFX11: v_cmpx_ne_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ne_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb5,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ne_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ne_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -686,7 +686,7 @@ v_cmpx_ne_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ne_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ne_u16 v1.h, v255.h
-// GFX11: v_cmpx_ne_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ne_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbd,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ne_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ne_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -704,7 +704,7 @@ v_cmpx_ne_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ne_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ne_u16 v255.h, v2.h
-// GFX11: v_cmpx_ne_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ne_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ne_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ne_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -722,7 +722,7 @@ v_cmpx_ne_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ne_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_neq_f16 v1.h, v255.h
-// GFX11: v_cmpx_neq_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_neq_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8d,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_neq_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_neq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -740,7 +740,7 @@ v_cmpx_neq_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_neq_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_neq_f16 v255.h, v2.h
-// GFX11: v_cmpx_neq_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_neq_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8d,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_neq_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_neq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -758,7 +758,7 @@ v_cmpx_neq_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_neq_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_nge_f16 v1.h, v255.h
-// GFX11: v_cmpx_nge_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_nge_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x89,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_nge_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -776,7 +776,7 @@ v_cmpx_nge_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nge_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_nge_f16 v255.h, v2.h
-// GFX11: v_cmpx_nge_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_nge_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x89,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_nge_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -794,7 +794,7 @@ v_cmpx_nge_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nge_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ngt_f16 v1.h, v255.h
-// GFX11: v_cmpx_ngt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_ngt_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8b,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ngt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ngt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -812,7 +812,7 @@ v_cmpx_ngt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ngt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ngt_f16 v255.h, v2.h
-// GFX11: v_cmpx_ngt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_ngt_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8b,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ngt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_ngt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -830,7 +830,7 @@ v_cmpx_ngt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_ngt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_nle_f16 v1.h, v255.h
-// GFX11: v_cmpx_nle_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_nle_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8c,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_nle_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nle_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -848,7 +848,7 @@ v_cmpx_nle_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nle_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_nle_f16 v255.h, v2.h
-// GFX11: v_cmpx_nle_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_nle_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8c,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_nle_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nle_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -866,7 +866,7 @@ v_cmpx_nle_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nle_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_nlg_f16 v1.h, v255.h
-// GFX11: v_cmpx_nlg_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_nlg_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8a,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_nlg_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nlg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -884,7 +884,7 @@ v_cmpx_nlg_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nlg_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_nlg_f16 v255.h, v2.h
-// GFX11: v_cmpx_nlg_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_nlg_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8a,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_nlg_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nlg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -902,7 +902,7 @@ v_cmpx_nlg_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nlg_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_nlt_f16 v1.h, v255.h
-// GFX11: v_cmpx_nlt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_nlt_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8e,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_nlt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nlt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -920,7 +920,7 @@ v_cmpx_nlt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nlt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_nlt_f16 v255.h, v2.h
-// GFX11: v_cmpx_nlt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_nlt_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8e,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_nlt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_nlt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -938,7 +938,7 @@ v_cmpx_nlt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_nlt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_o_f16 v1.h, v255.h
-// GFX11: v_cmpx_o_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_o_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x87,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_o_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_o_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -956,7 +956,7 @@ v_cmpx_o_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_o_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_o_f16 v255.h, v2.h
-// GFX11: v_cmpx_o_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_o_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x87,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_o_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_o_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -974,7 +974,7 @@ v_cmpx_o_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_o_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_t_f16 v1.h, v255.h
-// GFX11: v_cmpx_t_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_t_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8f,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_t_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_t_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -992,7 +992,7 @@ v_cmpx_t_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_t_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_t_f16 v255.h, v2.h
-// GFX11: v_cmpx_t_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_t_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8f,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_t_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_t_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -1010,7 +1010,7 @@ v_cmpx_t_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_t_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_tru_f16 v1.h, v255.h
-// GFX11: v_cmpx_t_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_t_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8f,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_tru_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_t_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -1028,7 +1028,7 @@ v_cmpx_tru_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_t_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_tru_f16 v255.h, v2.h
-// GFX11: v_cmpx_t_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_t_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8f,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_tru_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_t_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8f,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -1046,7 +1046,7 @@ v_cmpx_tru_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_t_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8f,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_u_f16 v1.h, v255.h
-// GFX11: v_cmpx_u_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0x01,0xff,0x03,0x00]
+// GFX11: v_cmpx_u_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x88,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_u_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_u_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -1064,7 +1064,7 @@ v_cmpx_u_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX11: v_cmpx_u_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_u_f16 v255.h, v2.h
-// GFX11: v_cmpx_u_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xff,0x05,0x02,0x00]
+// GFX11: v_cmpx_u_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x88,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_u_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX11: v_cmpx_u_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
index c654afa57a67c..71245d9dc3286 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
@@ -48,13 +48,13 @@ v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi
 // GFX12: v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi  ; encoding: [0x7e,0x01,0xfd,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_class_f16_e64 v1.h, v2.h
-// GFX12: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_class_f16_e64 v255.h, v2.l
-// GFX12: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_class_f16_e64 s105, v255.h
-// GFX12: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+// GFX12: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 
 v_cmpx_class_f32_e64 v1, v2
 // GFX12: v_cmpx_class_f32_e64 v1, v2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00]
@@ -192,10 +192,10 @@ v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_f32_e64 v1, v2
 // GFX12: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
@@ -324,10 +324,10 @@ v_cmpx_eq_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_i32_e64 v1, v2
 // GFX12: v_cmpx_eq_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00]
@@ -456,10 +456,10 @@ v_cmpx_eq_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_eq_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_eq_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_eq_u32_e64 v1, v2
 // GFX12: v_cmpx_eq_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00]
@@ -588,10 +588,10 @@ v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_f32_e64 v1, v2
 // GFX12: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
@@ -720,10 +720,10 @@ v_cmpx_ge_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_i32_e64 v1, v2
 // GFX12: v_cmpx_ge_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00]
@@ -852,10 +852,10 @@ v_cmpx_ge_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ge_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ge_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ge_u32_e64 v1, v2
 // GFX12: v_cmpx_ge_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00]
@@ -984,10 +984,10 @@ v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_f32_e64 v1, v2
 // GFX12: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
@@ -1116,10 +1116,10 @@ v_cmpx_gt_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_i32_e64 v1, v2
 // GFX12: v_cmpx_gt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00]
@@ -1248,10 +1248,10 @@ v_cmpx_gt_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_gt_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_gt_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_gt_u32_e64 v1, v2
 // GFX12: v_cmpx_gt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00]
@@ -1380,10 +1380,10 @@ v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_f32_e64 v1, v2
 // GFX12: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
@@ -1512,10 +1512,10 @@ v_cmpx_le_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_i32_e64 v1, v2
 // GFX12: v_cmpx_le_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00]
@@ -1644,10 +1644,10 @@ v_cmpx_le_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_le_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_le_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_le_u32_e64 v1, v2
 // GFX12: v_cmpx_le_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00]
@@ -1776,10 +1776,10 @@ v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lg_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lg_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lg_f32_e64 v1, v2
 // GFX12: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
@@ -1908,10 +1908,10 @@ v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_f32_e64 v1, v2
 // GFX12: v_cmpx_lt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00]
@@ -2040,10 +2040,10 @@ v_cmpx_lt_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_i32_e64 v1, v2
 // GFX12: v_cmpx_lt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00]
@@ -2172,10 +2172,10 @@ v_cmpx_lt_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_lt_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_lt_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_lt_u32_e64 v1, v2
 // GFX12: v_cmpx_lt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00]
@@ -2304,10 +2304,10 @@ v_cmpx_ne_i16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ne_i16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ne_i16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ne_i32_e64 v1, v2
 // GFX12: v_cmpx_ne_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00]
@@ -2436,10 +2436,10 @@ v_cmpx_ne_u16_e64 0xfe0b, vcc_hi
 // GFX12: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ne_u16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ne_u16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ne_u32_e64 v1, v2
 // GFX12: v_cmpx_ne_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00]
@@ -2568,10 +2568,10 @@ v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_neq_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_neq_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_neq_f32_e64 v1, v2
 // GFX12: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
@@ -2700,10 +2700,10 @@ v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nge_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_nge_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nge_f32_e64 v1, v2
 // GFX12: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
@@ -2832,10 +2832,10 @@ v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_ngt_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_ngt_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_ngt_f32_e64 v1, v2
 // GFX12: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
@@ -2964,10 +2964,10 @@ v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nle_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_nle_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nle_f32_e64 v1, v2
 // GFX12: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
@@ -3096,10 +3096,10 @@ v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nlg_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_nlg_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nlg_f32_e64 v1, v2
 // GFX12: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
@@ -3228,10 +3228,10 @@ v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_nlt_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_nlt_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_nlt_f32_e64 v1, v2
 // GFX12: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
@@ -3360,10 +3360,10 @@ v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_o_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_o_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_o_f32_e64 v1, v2
 // GFX12: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
@@ -3492,10 +3492,10 @@ v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
 // GFX12: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 v_cmpx_u_f16_e64 v1.h, v2.l
-// GFX12: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+// GFX12: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
 
 v_cmpx_u_f16_e64 v255.l, v255.h
-// GFX12: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+// GFX12: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 v_cmpx_u_f32_e64 v1, v2
 // GFX12: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
index 3c14bec145232..7767e89c5b7c2 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s
@@ -2,7 +2,7 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 %s
 
 v_cmpx_class_f16 v1.h, v255.h
-// GFX12: v_cmpx_class_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_class_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_class_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_class_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -20,7 +20,7 @@ v_cmpx_class_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_class_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_class_f16 v255.h, v2.h
-// GFX12: v_cmpx_class_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_class_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_class_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_class_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -38,7 +38,7 @@ v_cmpx_class_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_class_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_f16 v1.h, v255.h
-// GFX12: v_cmpx_eq_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x82,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -56,7 +56,7 @@ v_cmpx_eq_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_f16 v255.h, v2.h
-// GFX12: v_cmpx_eq_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x82,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x82,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -74,7 +74,7 @@ v_cmpx_eq_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x82,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_i16 v1.h, v255.h
-// GFX12: v_cmpx_eq_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb2,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -92,7 +92,7 @@ v_cmpx_eq_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_i16 v255.h, v2.h
-// GFX12: v_cmpx_eq_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb2,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb2,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -110,7 +110,7 @@ v_cmpx_eq_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_eq_u16 v1.h, v255.h
-// GFX12: v_cmpx_eq_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_eq_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xba,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_eq_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -128,7 +128,7 @@ v_cmpx_eq_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_eq_u16 v255.h, v2.h
-// GFX12: v_cmpx_eq_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_eq_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xba,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_eq_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_eq_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xba,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -146,7 +146,7 @@ v_cmpx_eq_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_eq_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_f16 v1.h, v255.h
-// GFX12: v_cmpx_ge_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x86,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -164,7 +164,7 @@ v_cmpx_ge_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_f16 v255.h, v2.h
-// GFX12: v_cmpx_ge_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x86,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x86,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -182,7 +182,7 @@ v_cmpx_ge_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x86,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_i16 v1.h, v255.h
-// GFX12: v_cmpx_ge_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb6,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -200,7 +200,7 @@ v_cmpx_ge_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_i16 v255.h, v2.h
-// GFX12: v_cmpx_ge_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb6,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb6,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -218,7 +218,7 @@ v_cmpx_ge_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ge_u16 v1.h, v255.h
-// GFX12: v_cmpx_ge_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ge_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbe,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ge_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -236,7 +236,7 @@ v_cmpx_ge_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ge_u16 v255.h, v2.h
-// GFX12: v_cmpx_ge_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ge_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbe,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ge_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ge_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbe,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -254,7 +254,7 @@ v_cmpx_ge_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ge_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_f16 v1.h, v255.h
-// GFX12: v_cmpx_gt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x84,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -272,7 +272,7 @@ v_cmpx_gt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_f16 v255.h, v2.h
-// GFX12: v_cmpx_gt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x84,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x84,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -290,7 +290,7 @@ v_cmpx_gt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x84,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_i16 v1.h, v255.h
-// GFX12: v_cmpx_gt_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb4,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -308,7 +308,7 @@ v_cmpx_gt_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_i16 v255.h, v2.h
-// GFX12: v_cmpx_gt_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb4,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb4,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -326,7 +326,7 @@ v_cmpx_gt_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_gt_u16 v1.h, v255.h
-// GFX12: v_cmpx_gt_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_gt_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbc,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_gt_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -344,7 +344,7 @@ v_cmpx_gt_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_gt_u16 v255.h, v2.h
-// GFX12: v_cmpx_gt_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_gt_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbc,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_gt_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_gt_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbc,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -362,7 +362,7 @@ v_cmpx_gt_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_gt_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_f16 v1.h, v255.h
-// GFX12: v_cmpx_le_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x83,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -380,7 +380,7 @@ v_cmpx_le_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_f16 v255.h, v2.h
-// GFX12: v_cmpx_le_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x83,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x83,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -398,7 +398,7 @@ v_cmpx_le_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x83,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_i16 v1.h, v255.h
-// GFX12: v_cmpx_le_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb3,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -416,7 +416,7 @@ v_cmpx_le_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_i16 v255.h, v2.h
-// GFX12: v_cmpx_le_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb3,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb3,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -434,7 +434,7 @@ v_cmpx_le_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_le_u16 v1.h, v255.h
-// GFX12: v_cmpx_le_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_le_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbb,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_le_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -452,7 +452,7 @@ v_cmpx_le_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_le_u16 v255.h, v2.h
-// GFX12: v_cmpx_le_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_le_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbb,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_le_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_le_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbb,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -470,7 +470,7 @@ v_cmpx_le_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_le_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lg_f16 v1.h, v255.h
-// GFX12: v_cmpx_lg_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_lg_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x85,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lg_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -488,7 +488,7 @@ v_cmpx_lg_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lg_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lg_f16 v255.h, v2.h
-// GFX12: v_cmpx_lg_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_lg_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x85,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lg_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x85,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -506,7 +506,7 @@ v_cmpx_lg_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lg_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x85,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_f16 v1.h, v255.h
-// GFX12: v_cmpx_lt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x81,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -524,7 +524,7 @@ v_cmpx_lt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_f16 v255.h, v2.h
-// GFX12: v_cmpx_lt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x81,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x81,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -542,7 +542,7 @@ v_cmpx_lt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x81,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_i16 v1.h, v255.h
-// GFX12: v_cmpx_lt_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb1,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -560,7 +560,7 @@ v_cmpx_lt_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_i16 v255.h, v2.h
-// GFX12: v_cmpx_lt_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb1,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb1,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -578,7 +578,7 @@ v_cmpx_lt_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_lt_u16 v1.h, v255.h
-// GFX12: v_cmpx_lt_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_lt_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb9,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_lt_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -596,7 +596,7 @@ v_cmpx_lt_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_lt_u16 v255.h, v2.h
-// GFX12: v_cmpx_lt_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_lt_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb9,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_lt_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_lt_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb9,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -614,7 +614,7 @@ v_cmpx_lt_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_lt_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ne_i16 v1.h, v255.h
-// GFX12: v_cmpx_ne_i16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ne_i16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb5,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ne_i16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ne_i16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -632,7 +632,7 @@ v_cmpx_ne_i16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ne_i16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ne_i16 v255.h, v2.h
-// GFX12: v_cmpx_ne_i16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ne_i16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xb5,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ne_i16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ne_i16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xb5,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -650,7 +650,7 @@ v_cmpx_ne_i16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ne_i16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ne_u16 v1.h, v255.h
-// GFX12: v_cmpx_ne_u16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ne_u16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbd,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ne_u16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ne_u16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -668,7 +668,7 @@ v_cmpx_ne_u16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ne_u16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ne_u16 v255.h, v2.h
-// GFX12: v_cmpx_ne_u16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ne_u16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xbd,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ne_u16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ne_u16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0xbd,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -686,7 +686,7 @@ v_cmpx_ne_u16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ne_u16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_neq_f16 v1.h, v255.h
-// GFX12: v_cmpx_neq_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_neq_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8d,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_neq_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_neq_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -704,7 +704,7 @@ v_cmpx_neq_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_neq_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_neq_f16 v255.h, v2.h
-// GFX12: v_cmpx_neq_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_neq_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8d,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_neq_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_neq_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8d,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -722,7 +722,7 @@ v_cmpx_neq_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_neq_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8d,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_nge_f16 v1.h, v255.h
-// GFX12: v_cmpx_nge_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_nge_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x89,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_nge_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nge_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -740,7 +740,7 @@ v_cmpx_nge_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nge_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_nge_f16 v255.h, v2.h
-// GFX12: v_cmpx_nge_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_nge_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x89,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_nge_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nge_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x89,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -758,7 +758,7 @@ v_cmpx_nge_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nge_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x89,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_ngt_f16 v1.h, v255.h
-// GFX12: v_cmpx_ngt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_ngt_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8b,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_ngt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ngt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -776,7 +776,7 @@ v_cmpx_ngt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ngt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_ngt_f16 v255.h, v2.h
-// GFX12: v_cmpx_ngt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_ngt_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8b,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_ngt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_ngt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8b,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -794,7 +794,7 @@ v_cmpx_ngt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_ngt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8b,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_nle_f16 v1.h, v255.h
-// GFX12: v_cmpx_nle_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_nle_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8c,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_nle_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nle_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -812,7 +812,7 @@ v_cmpx_nle_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nle_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_nle_f16 v255.h, v2.h
-// GFX12: v_cmpx_nle_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_nle_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8c,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_nle_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nle_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8c,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -830,7 +830,7 @@ v_cmpx_nle_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nle_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8c,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_nlg_f16 v1.h, v255.h
-// GFX12: v_cmpx_nlg_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_nlg_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8a,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_nlg_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nlg_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -848,7 +848,7 @@ v_cmpx_nlg_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nlg_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_nlg_f16 v255.h, v2.h
-// GFX12: v_cmpx_nlg_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_nlg_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8a,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_nlg_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nlg_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8a,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -866,7 +866,7 @@ v_cmpx_nlg_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nlg_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8a,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_nlt_f16 v1.h, v255.h
-// GFX12: v_cmpx_nlt_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_nlt_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8e,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_nlt_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nlt_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -884,7 +884,7 @@ v_cmpx_nlt_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nlt_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_nlt_f16 v255.h, v2.h
-// GFX12: v_cmpx_nlt_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_nlt_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x8e,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_nlt_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_nlt_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x8e,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -902,7 +902,7 @@ v_cmpx_nlt_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_nlt_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x8e,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_o_f16 v1.h, v255.h
-// GFX12: v_cmpx_o_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_o_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x87,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_o_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_o_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -920,7 +920,7 @@ v_cmpx_o_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_o_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_o_f16 v255.h, v2.h
-// GFX12: v_cmpx_o_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_o_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x87,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_o_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_o_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x87,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
@@ -938,7 +938,7 @@ v_cmpx_o_f16 v255.l, v2.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_o_f16_e64_dpp v255.l, v2.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x87,0xd4,0xfa,0x04,0x02,0x00,0xff,0x1b,0x00,0xff]
 
 v_cmpx_u_f16 v1.h, v255.h
-// GFX12: v_cmpx_u_f16_e64 v1.h, v255.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0x01,0xff,0x03,0x00]
+// GFX12: v_cmpx_u_f16_e64 v1.h, v255.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x88,0xd4,0x01,0xff,0x03,0x00]
 
 v_cmpx_u_f16 v1.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_u_f16_e64_dpp v1.h, v255.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xe9,0xfe,0x03,0x00,0x01,0x77,0x39,0x05]
@@ -956,7 +956,7 @@ v_cmpx_u_f16 v1.l, v255.l quad_perm:[3,2,1,0]
 // GFX12: v_cmpx_u_f16_e64_dpp v1.l, v255.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0x88,0xd4,0xfa,0xfe,0x03,0x00,0x01,0x1b,0x00,0xff]
 
 v_cmpx_u_f16 v255.h, v2.h
-// GFX12: v_cmpx_u_f16_e64 v255.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xff,0x05,0x02,0x00]
+// GFX12: v_cmpx_u_f16_e64 v255.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0x88,0xd4,0xff,0x05,0x02,0x00]
 
 v_cmpx_u_f16 v255.h, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cmpx_u_f16_e64_dpp v255.h, v2.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x18,0x88,0xd4,0xe9,0x04,0x02,0x00,0xff,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vopcx.txt
index 40c34708d863e..73da324785749 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vopcx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vopcx.txt
@@ -58,15 +58,15 @@
 # GFX11: v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi  ; encoding: [0x7e,0x01,0xfd,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_class_f16_e64 v1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_class_f16_e64 v255, v2           ; encoding: [0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00
-# GFX11-REAL16: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+# GFX11-REAL16: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_class_f16_e64 s105, v255         ; encoding: [0x7e,0x00,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 
 0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00
@@ -207,11 +207,11 @@
 # GFX11: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00
@@ -343,11 +343,11 @@
 # GFX11: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_eq_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_eq_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00
@@ -479,11 +479,11 @@
 # GFX11: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_eq_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_eq_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00
@@ -615,11 +615,11 @@
 # GFX11: v_cmpx_f_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x80,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x80,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_f_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x80,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_f_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x80,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_f_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x80,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x80,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_f_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x80,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_f_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x80,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_f_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x80,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x90,0xd4,0x01,0x05,0x02,0x00
@@ -913,11 +913,11 @@
 # GFX11: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00
@@ -1049,11 +1049,11 @@
 # GFX11: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_ge_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_ge_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00
@@ -1185,11 +1185,11 @@
 # GFX11: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_ge_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_ge_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00
@@ -1321,11 +1321,11 @@
 # GFX11: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00
@@ -1457,11 +1457,11 @@
 # GFX11: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_gt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_gt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00
@@ -1593,11 +1593,11 @@
 # GFX11: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_gt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_gt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00
@@ -1729,11 +1729,11 @@
 # GFX11: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00
@@ -1865,11 +1865,11 @@
 # GFX11: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_le_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_le_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00
@@ -2001,11 +2001,11 @@
 # GFX11: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_le_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_le_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00
@@ -2137,11 +2137,11 @@
 # GFX11: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00
@@ -2273,11 +2273,11 @@
 # GFX11: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_lt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_lt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x81,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00
@@ -2409,11 +2409,11 @@
 # GFX11: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_lt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_lt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00
@@ -2545,11 +2545,11 @@
 # GFX11: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_lt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_lt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00
@@ -2681,11 +2681,11 @@
 # GFX11: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_ne_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_ne_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00
@@ -2817,11 +2817,11 @@
 # GFX11: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_ne_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_ne_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00
@@ -2953,11 +2953,11 @@
 # GFX11: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00
@@ -3089,11 +3089,11 @@
 # GFX11: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00
@@ -3225,11 +3225,11 @@
 # GFX11: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00
@@ -3361,11 +3361,11 @@
 # GFX11: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00
@@ -3497,11 +3497,11 @@
 # GFX11: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00
@@ -3633,11 +3633,11 @@
 # GFX11: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00
@@ -3769,11 +3769,11 @@
 # GFX11: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00
@@ -3905,11 +3905,11 @@
 # GFX11: v_cmpx_t_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8f,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8f,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_t_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8f,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_t_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8f,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_t_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x8f,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8f,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_t_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8f,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_t_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8f,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_t_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x8f,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x9f,0xd4,0x01,0x05,0x02,0x00
@@ -4203,11 +4203,11 @@
 # GFX11: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00
-# GFX11-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX11-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
 # GFX11-FAKE16: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00
-# GFX11-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX11-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
 # GFX11-FAKE16: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt
index ab2d154e9ef9f..8001267eb6a86 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt
@@ -54,15 +54,15 @@
 # GFX12: v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi  ; encoding: [0x7e,0x01,0xfd,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_class_f16_e64 v1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_class_f16_e64 v255, v2           ; encoding: [0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00
-# GFX12-REAL16: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+# GFX12-REAL16: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_class_f16_e64 s105, v255         ; encoding: [0x7e,0x00,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 
 0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00
@@ -203,11 +203,11 @@
 # GFX12: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -340,11 +340,11 @@
 # GFX12: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -477,11 +477,11 @@
 # GFX12: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -614,11 +614,11 @@
 # GFX12: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -751,11 +751,11 @@
 # GFX12: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -888,11 +888,11 @@
 # GFX12: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -1025,11 +1025,11 @@
 # GFX12: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -1162,11 +1162,11 @@
 # GFX12: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -1299,11 +1299,11 @@
 # GFX12: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -1436,11 +1436,11 @@
 # GFX12: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -1573,11 +1573,11 @@
 # GFX12: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_le_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_le_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -1710,11 +1710,11 @@
 # GFX12: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_le_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_le_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -1847,11 +1847,11 @@
 # GFX12: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -1984,11 +1984,11 @@
 # GFX12: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_lt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x81,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00
@@ -2120,11 +2120,11 @@
 # GFX12: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -2257,11 +2257,11 @@
 # GFX12: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -2394,11 +2394,11 @@
 # GFX12: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -2531,11 +2531,11 @@
 # GFX12: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00
@@ -2667,11 +2667,11 @@
 # GFX12: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -2804,11 +2804,11 @@
 # GFX12: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -2941,11 +2941,11 @@
 # GFX12: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -3078,11 +3078,11 @@
 # GFX12: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -3215,11 +3215,11 @@
 # GFX12: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -3352,11 +3352,11 @@
 # GFX12: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -3489,11 +3489,11 @@
 # GFX12: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 
@@ -3626,11 +3626,11 @@
 # GFX12: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
 
 0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt
index f8ce4fafc0252..4a7a64d12fc72 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt
@@ -65,11 +65,11 @@
 # GFX12-FAKE16: v_cmpx_class_f16_e64_dpp -|v255|, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x01,0xfd,0xd4,0xfa,0xfe,0x03,0x20,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1,0] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_class_f16_e64 v1.h, v2.h op_sel:[1,1] ; encoding: [0x7e,0x18,0xfd,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_class_f16_e64 v1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_class_f16_e64 v255.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xfd,0xd4,0xff,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_class_f16_e64 v255, v2           ; encoding: [0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00]
 
 0x7e,0x00,0xfd,0xd4,0x01,0x04,0x02,0x00
@@ -77,7 +77,7 @@
 # GFX12-FAKE16: v_cmpx_class_f16_e64 s1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x04,0x02,0x00]
 
 0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00
-# GFX12-REAL16: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+# GFX12-REAL16: v_cmpx_class_f16_e64 s105, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_class_f16_e64 s105, v255         ; encoding: [0x7e,0x00,0xfd,0xd4,0x69,0xfe,0x03,0x00]
 
 0x7e,0x00,0xfd,0xd4,0x6a,0x04,0x00,0x00
@@ -231,11 +231,11 @@
 # GFX12-FAKE16: v_cmpx_eq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x82,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x82,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_eq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x82,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00
@@ -395,11 +395,11 @@
 # GFX12-FAKE16: v_cmpx_eq_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb2,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_eq_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb2,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_eq_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb2,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_eq_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xb2,0xd4,0x01,0x04,0x00,0x00
@@ -562,11 +562,11 @@
 # GFX12-FAKE16: v_cmpx_eq_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xba,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_eq_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xba,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_eq_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xba,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_eq_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xba,0xd4,0x01,0x04,0x00,0x00
@@ -726,11 +726,11 @@
 # GFX12-FAKE16: v_cmpx_ge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x86,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x86,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x86,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00
@@ -890,11 +890,11 @@
 # GFX12-FAKE16: v_cmpx_ge_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb6,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ge_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb6,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ge_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb6,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ge_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xb6,0xd4,0x01,0x04,0x00,0x00
@@ -1054,11 +1054,11 @@
 # GFX12-FAKE16: v_cmpx_ge_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xbe,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ge_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbe,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ge_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbe,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ge_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xbe,0xd4,0x01,0x04,0x00,0x00
@@ -1218,11 +1218,11 @@
 # GFX12-FAKE16: v_cmpx_gt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x84,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x84,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_gt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x84,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00
@@ -1382,11 +1382,11 @@
 # GFX12-FAKE16: v_cmpx_gt_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb4,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_gt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb4,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_gt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb4,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_gt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xb4,0xd4,0x01,0x04,0x00,0x00
@@ -1546,11 +1546,11 @@
 # GFX12-FAKE16: v_cmpx_gt_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xbc,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_gt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbc,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_gt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbc,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_gt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xbc,0xd4,0x01,0x04,0x00,0x00
@@ -1710,11 +1710,11 @@
 # GFX12-FAKE16: v_cmpx_le_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x83,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_le_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x83,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_le_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x83,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00
@@ -1874,11 +1874,11 @@
 # GFX12-FAKE16: v_cmpx_le_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb3,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_le_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb3,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_le_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_le_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb3,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_le_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xb3,0xd4,0x01,0x04,0x00,0x00
@@ -2038,11 +2038,11 @@
 # GFX12-FAKE16: v_cmpx_le_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xbb,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_le_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbb,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_le_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_le_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbb,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_le_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xbb,0xd4,0x01,0x04,0x00,0x00
@@ -2202,11 +2202,11 @@
 # GFX12-FAKE16: v_cmpx_lg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x85,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x85,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x85,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00
@@ -2366,11 +2366,11 @@
 # GFX12-FAKE16: v_cmpx_lt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x81,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x81,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_lt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x81,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x81,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x81,0xd4,0x01,0x04,0x00,0x00
@@ -2530,11 +2530,11 @@
 # GFX12-FAKE16: v_cmpx_lt_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb1,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lt_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb1,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lt_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb1,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xb1,0xd4,0x01,0x04,0x00,0x00
@@ -2694,11 +2694,11 @@
 # GFX12-FAKE16: v_cmpx_lt_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb9,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_lt_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb9,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_lt_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb9,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_lt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xb9,0xd4,0x01,0x04,0x00,0x00
@@ -2858,11 +2858,11 @@
 # GFX12-FAKE16: v_cmpx_ne_i16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xb5,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ne_i16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xb5,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ne_i16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xb5,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ne_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xb5,0xd4,0x01,0x04,0x00,0x00
@@ -3022,11 +3022,11 @@
 # GFX12-FAKE16: v_cmpx_ne_u16_e64_dpp v255, v255 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x00,0xbd,0xd4,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ne_u16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0xbd,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ne_u16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0xbd,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ne_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0xbd,0xd4,0x01,0x04,0x00,0x00
@@ -3186,11 +3186,11 @@
 # GFX12-FAKE16: v_cmpx_neq_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8d,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8d,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_neq_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8d,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00
@@ -3350,11 +3350,11 @@
 # GFX12-FAKE16: v_cmpx_nge_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x89,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x89,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nge_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x89,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00
@@ -3514,11 +3514,11 @@
 # GFX12-FAKE16: v_cmpx_ngt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8b,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8b,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_ngt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8b,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00
@@ -3678,11 +3678,11 @@
 # GFX12-FAKE16: v_cmpx_nle_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8c,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8c,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nle_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8c,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00
@@ -3842,11 +3842,11 @@
 # GFX12-FAKE16: v_cmpx_nlg_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8a,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8a,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nlg_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8a,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00
@@ -4006,11 +4006,11 @@
 # GFX12-FAKE16: v_cmpx_nlt_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x8e,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x8e,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_nlt_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x8e,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00
@@ -4170,11 +4170,11 @@
 # GFX12-FAKE16: v_cmpx_o_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x87,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_o_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x87,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_o_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x87,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00
@@ -4334,11 +4334,11 @@
 # GFX12-FAKE16: v_cmpx_u_f16_e64_dpp -|v255|, -|v255| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0x7e,0x83,0x88,0xd4,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00
-# GFX12-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
+# GFX12-REAL16: v_cmpx_u_f16_e64 v1.h, v2.l op_sel:[1,0] ; encoding: [0x7e,0x08,0x88,0xd4,0x01,0x05,0x02,0x00]
 # GFX12-FAKE16: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
 
 0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00
-# GFX12-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1,0] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
+# GFX12-REAL16: v_cmpx_u_f16_e64 v255.l, v255.h op_sel:[0,1] ; encoding: [0x7e,0x10,0x88,0xd4,0xff,0xff,0x03,0x00]
 # GFX12-FAKE16: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
 
 0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00

From aca710ac3655fcd3f057edc1382ceec7fc58ef19 Mon Sep 17 00:00:00 2001
From: Tom Honermann <tom.honermann@intel.com>
Date: Wed, 16 Apr 2025 08:28:09 -0700
Subject: [PATCH 130/710] [NFC][Clang] Introduce type aliases to replace use of
 auto in clang/lib/CodeGen/CGCall.cpp. (#135861)

CGCall.cpp declares several functions with a return type that is an
explicitly spelled out specialization of `SmallVector`. Previously,
`auto` was used in several places to avoid repeating the long type name;
a use that Clang maintainers find unjustified. This change introduces
type aliases and replaces the existing uses of `auto` with the
corresponding alias name.
---
 clang/lib/CodeGen/CGCall.cpp | 75 ++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 38 deletions(-)

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index b25cdf9523ae1..bc1035163a8eb 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -199,15 +199,17 @@ static void appendParameterTypes(const CodeGenTypes &CGT,
                               prefix.size());
 }
 
+using ExtParameterInfoList =
+    SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
+
 /// Arrange the LLVM function layout for a value of the given function
 /// type, on top of any implicit parameters already stored.
 static const CGFunctionInfo &
 arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
                         SmallVectorImpl<CanQualType> &prefix,
                         CanQual<FunctionProtoType> FTP) {
-  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
+  ExtParameterInfoList paramInfos;
   RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
-  // FIXME: Kill copy.
   appendParameterTypes(CGT, prefix, paramInfos, FTP);
   CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
 
@@ -217,11 +219,13 @@ arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
                                      FTP->getExtInfo(), paramInfos, Required);
 }
 
+using CanQualTypeList = SmallVector<CanQualType, 16>;
+
 /// Arrange the argument and result information for a value of the
 /// given freestanding function type.
 const CGFunctionInfo &
 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
-  SmallVector<CanQualType, 16> argTypes;
+  CanQualTypeList argTypes;
   return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
                                    FTP);
 }
@@ -319,7 +323,7 @@ const CGFunctionInfo &
 CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
                                    const FunctionProtoType *FTP,
                                    const CXXMethodDecl *MD) {
-  SmallVector<CanQualType, 16> argTypes;
+  CanQualTypeList argTypes;
 
   // Add the 'this' pointer.
   argTypes.push_back(DeriveThisType(RD, MD));
@@ -375,8 +379,8 @@ const CGFunctionInfo &
 CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
   auto *MD = cast<CXXMethodDecl>(GD.getDecl());
 
-  SmallVector<CanQualType, 16> argTypes;
-  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
+  CanQualTypeList argTypes;
+  ExtParameterInfoList paramInfos;
 
   const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
   argTypes.push_back(DeriveThisType(ThisType, MD));
@@ -421,26 +425,26 @@ CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
                                  argTypes, extInfo, paramInfos, required);
 }
 
-static SmallVector<CanQualType, 16>
-getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
-  SmallVector<CanQualType, 16> argTypes;
+static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
+                                          const CallArgList &args) {
+  CanQualTypeList argTypes;
   for (auto &arg : args)
     argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
   return argTypes;
 }
 
-static SmallVector<CanQualType, 16>
-getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
-  SmallVector<CanQualType, 16> argTypes;
+static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
+                                                 const FunctionArgList &args) {
+  CanQualTypeList argTypes;
   for (auto &arg : args)
     argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
   return argTypes;
 }
 
-static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
-getExtParameterInfosForCall(const FunctionProtoType *proto,
-                            unsigned prefixArgs, unsigned totalArgs) {
-  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
+static ExtParameterInfoList
+getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
+                            unsigned totalArgs) {
+  ExtParameterInfoList result;
   if (proto->hasExtParameterInfos()) {
     addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
   }
@@ -462,8 +466,7 @@ CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
                                         unsigned ExtraPrefixArgs,
                                         unsigned ExtraSuffixArgs,
                                         bool PassProtoArgs) {
-  // FIXME: Kill copy.
-  SmallVector<CanQualType, 16> ArgTypes;
+  CanQualTypeList ArgTypes;
   for (const auto &Arg : args)
     ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
 
@@ -483,7 +486,7 @@ CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
                                : Context.VoidTy;
 
   FunctionType::ExtInfo Info = FPT->getExtInfo();
-  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
+  ExtParameterInfoList ParamInfos;
   // If the prototype args are elided, we should only have ABI-specific args,
   // which never have param info.
   if (PassProtoArgs && FPT->hasExtParameterInfos()) {
@@ -546,13 +549,11 @@ CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
 const CGFunctionInfo &
 CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
                                               QualType receiverType) {
-  SmallVector<CanQualType, 16> argTys;
-  SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(
-      MD->isDirectMethod() ? 1 : 2);
+  CanQualTypeList argTys;
+  ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
   argTys.push_back(Context.getCanonicalParamType(receiverType));
   if (!MD->isDirectMethod())
     argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
-  // FIXME: Kill copy?
   for (const auto *I : MD->parameters()) {
     argTys.push_back(Context.getCanonicalParamType(I->getType()));
     auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
@@ -579,7 +580,7 @@ CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
 const CGFunctionInfo &
 CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
                                                  const CallArgList &args) {
-  auto argTypes = getArgTypesForCall(Context, args);
+  CanQualTypeList argTypes = getArgTypesForCall(Context, args);
   FunctionType::ExtInfo einfo;
 
   return arrangeLLVMFunctionInfo(GetReturnType(returnType), FnInfoOpts::None,
@@ -641,7 +642,7 @@ arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
                             bool chainCall) {
   assert(args.size() >= numExtraRequiredArgs);
 
-  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
+  ExtParameterInfoList paramInfos;
 
   // In most cases, there are no optional arguments.
   RequiredArgs required = RequiredArgs::All;
@@ -666,8 +667,7 @@ arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
     required = RequiredArgs(args.size());
   }
 
-  // FIXME: Kill copy.
-  SmallVector<CanQualType, 16> argTypes;
+  CanQualTypeList argTypes;
   for (const auto &arg : args)
     argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
   FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
@@ -700,8 +700,9 @@ CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
 const CGFunctionInfo &
 CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
                                               const FunctionArgList &params) {
-  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
-  auto argTypes = getArgTypesForDeclaration(Context, params);
+  ExtParameterInfoList paramInfos =
+      getExtParameterInfosForCall(proto, 1, params.size());
+  CanQualTypeList argTypes = getArgTypesForDeclaration(Context, params);
 
   return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
                                  FnInfoOpts::None, argTypes,
@@ -712,8 +713,7 @@ CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
 const CGFunctionInfo &
 CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
                                          const CallArgList &args) {
-  // FIXME: Kill copy.
-  SmallVector<CanQualType, 16> argTypes;
+  CanQualTypeList argTypes;
   for (const auto &Arg : args)
     argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
   return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,
@@ -724,7 +724,7 @@ CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
 const CGFunctionInfo &
 CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
                                                 const FunctionArgList &args) {
-  auto argTypes = getArgTypesForDeclaration(Context, args);
+  CanQualTypeList argTypes = getArgTypesForDeclaration(Context, args);
 
   return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,
                                  argTypes, FunctionType::ExtInfo(), {},
@@ -752,11 +752,10 @@ CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
          "Emitting a call with less args than the required prefix?");
   // Add one to account for `this`. It's a bit awkward here, but we don't count
   // `this` in similar places elsewhere.
-  auto paramInfos =
-    getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
+  ExtParameterInfoList paramInfos =
+      getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
 
-  // FIXME: Kill copy.
-  auto argTypes = getArgTypesForCall(Context, args);
+  CanQualTypeList argTypes = getArgTypesForCall(Context, args);
 
   FunctionType::ExtInfo info = proto->getExtInfo();
   return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
@@ -777,14 +776,14 @@ CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
   if (signature.arg_size() == args.size())
     return signature;
 
-  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
+  ExtParameterInfoList paramInfos;
   auto sigParamInfos = signature.getExtParameterInfos();
   if (!sigParamInfos.empty()) {
     paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
     paramInfos.resize(args.size());
   }
 
-  auto argTypes = getArgTypesForCall(Context, args);
+  CanQualTypeList argTypes = getArgTypesForCall(Context, args);
 
   assert(signature.getRequiredArgs().allowsOptionalArgs());
   FnInfoOpts opts = FnInfoOpts::None;

From 81b4fc2bedc411c257fdf24540318e24fe669b8b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 08:37:56 -0700
Subject: [PATCH 131/710] [CodeGen] Construct SmallVector with ArrayRef (NFC)
 (#135930)

Note that we can drop the call to reserve because the constructor that
takes ArrayRef calls append, which in turn calls reserve.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ab8e18267f3f5..b175e35385ec6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24747,10 +24747,8 @@ static SDValue combineConcatVectorOfShuffleAndItsOperands(
 
   // We are going to pad the shuffle operands, so any indice, that was picking
   // from the second operand, must be adjusted.
-  SmallVector<int, 16> AdjustedMask;
-  AdjustedMask.reserve(SVN->getMask().size());
+  SmallVector<int, 16> AdjustedMask(SVN->getMask());
   assert(SVN->getOperand(1).isUndef() && "Expected unary shuffle!");
-  append_range(AdjustedMask, SVN->getMask());
 
   // Identity masks for the operands of the (padded) shuffle.
   SmallVector<int, 32> IdentityMask(2 * OpVT.getVectorNumElements());

From 842bc07946e7ac71692ae235e784b9bc6c3d6535 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 08:38:16 -0700
Subject: [PATCH 132/710] [DebugInfo] Use StringRef::starts_with (NFC)
 (#135933)

---
 llvm/lib/DebugInfo/Symbolize/Symbolize.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
index d154f16f272bd..1d8217ad587ec 100644
--- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
+++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
@@ -753,7 +753,7 @@ LLVMSymbolizer::DemangleName(StringRef Name,
   if (nonMicrosoftDemangle(Name, Result))
     return Result;
 
-  if (!Name.empty() && Name.front() == '?') {
+  if (Name.starts_with('?')) {
     // Only do MSVC C++ demangling on symbols starting with '?'.
     int status = 0;
     char *DemangledName = microsoftDemangle(

From 05772406153c390e61809757643ad49bff7dc71d Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 08:39:07 -0700
Subject: [PATCH 133/710] [Utils] Use StringRef::ends_with (NFC) (#135934)

---
 llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 4e37c587dc975..941e787f91eff 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1936,7 +1936,7 @@ static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
   bool IsIntrinsic = CalleeFn->isIntrinsic();
   if (!IsIntrinsic) {
     StringRef CallerName = CI->getFunction()->getName();
-    if (!CallerName.empty() && CallerName.back() == 'f' &&
+    if (CallerName.ends_with('f') &&
         CallerName.size() == (CalleeName.size() + 1) &&
         CallerName.starts_with(CalleeName))
       return nullptr;

From 0045b82a42bd36306a14f8b40bd9b54470c299ea Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 08:39:55 -0700
Subject: [PATCH 134/710] [Vectorize] Construct SmallVector with an iterator
 range (NFC) (#135936)

---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f97386159d029..764f3e2dc64d9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -8402,11 +8402,7 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
             continue;
           }
           SmallVector<std::pair<LoadInst *, int>> LocalLoadsDists(LoadsDists);
-          SmallVector<LoadInst *> OriginalLoads(LocalLoadsDists.size());
-          transform(LoadsDists, OriginalLoads.begin(),
-                    [](const std::pair<LoadInst *, int> &L) -> LoadInst * {
-                      return L.first;
-                    });
+          SmallVector<LoadInst *> OriginalLoads(make_first_range(LoadsDists));
           stable_sort(LocalLoadsDists, LoadSorter);
           SmallVector<LoadInst *> Loads;
           unsigned MaxConsecutiveDistance = 0;

From 419fa1b06a36336ad85f1c71fc72ffa719ceb659 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Wed, 16 Apr 2025 17:57:51 +0200
Subject: [PATCH 135/710] [lldb][DataFormatter] Surface CalculateNumChildren
 errors in std::vector summary (#135944)

When the data-formatters happen to break (e.g., due to layout changes in
libc++), there's no clear indicator of them failing from a user's
perspective. E.g., for `std::vector`s we would just show:
```
(std::vector<int>) v = size=0 {}
```
which is highly misleading, especially if `v.size()` returns a non-zero
size.

This patch surfaces the various errors that could occur when calculating
the number of children of a vector.

rdar://146964266
---
 .../Language/CPlusPlus/LibCxxVector.cpp       | 21 +++++++---
 lldb/source/ValueObject/ValueObject.cpp       | 12 ++++--
 .../libcxx-simulators/invalid-vector/Makefile |  3 ++
 ...taFormatterLibcxxInvalidVectorSimulator.py | 39 +++++++++++++++++++
 .../libcxx-simulators/invalid-vector/main.cpp | 37 ++++++++++++++++++
 5 files changed, 104 insertions(+), 8 deletions(-)
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/Makefile
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp

diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
index d538cac9f9134..ce2261b6f03c3 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
@@ -83,19 +83,30 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
 llvm::Expected<uint32_t> lldb_private::formatters::
     LibcxxStdVectorSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
-    return 0;
+    return llvm::createStringError(
+        "Failed to determine start/end of vector data.");
+
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
   uint64_t finish_val = m_finish->GetValueAsUnsigned(0);
 
-  if (start_val == 0 || finish_val == 0)
+  // A default-initialized empty vector.
+  if (start_val == 0 && finish_val == 0)
     return 0;
 
-  if (start_val >= finish_val)
-    return 0;
+  if (start_val == 0)
+    return llvm::createStringError("Invalid value for start of vector.");
+
+  if (finish_val == 0)
+    return llvm::createStringError("Invalid value for end of vector.");
+
+  if (start_val > finish_val)
+    return llvm::createStringError(
+        "Start of vector data begins after end pointer.");
 
   size_t num_children = (finish_val - start_val);
   if (num_children % m_element_size)
-    return 0;
+    return llvm::createStringError("Size not multiple of element size.");
+
   return num_children / m_element_size;
 }
 
diff --git a/lldb/source/ValueObject/ValueObject.cpp b/lldb/source/ValueObject/ValueObject.cpp
index eac24353de90b..8741cb7343166 100644
--- a/lldb/source/ValueObject/ValueObject.cpp
+++ b/lldb/source/ValueObject/ValueObject.cpp
@@ -1521,10 +1521,16 @@ bool ValueObject::DumpPrintableRepresentation(
       str = GetLocationAsCString();
       break;
 
-    case eValueObjectRepresentationStyleChildrenCount:
-      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildrenIgnoringErrors());
-      str = strm.GetString();
+    case eValueObjectRepresentationStyleChildrenCount: {
+      if (auto err = GetNumChildren()) {
+        strm.Printf("%" PRIu32, *err);
+        str = strm.GetString();
+      } else {
+        strm << "error: " << toString(err.takeError());
+        str = strm.GetString();
+      }
       break;
+    }
 
     case eValueObjectRepresentationStyleType:
       str = GetTypeName().GetStringRef();
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/Makefile
new file mode 100644
index 0000000000000..38cfa81053488
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+override CXXFLAGS_EXTRAS += -std=c++14
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
new file mode 100644
index 0000000000000..8788ea7be882d
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
@@ -0,0 +1,39 @@
+"""
+Test we can understand various layouts of the libc++'s std::string
+"""
+
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+import functools
+
+
+class LibcxxInvalidVectorDataFormatterSimulatorTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "return 0", lldb.SBFileSpec("main.cpp"))
+
+        self.expect(
+            "frame variable v1",
+            substrs=["size=error: Invalid value for end of vector."],
+        )
+        self.expect(
+            "frame variable v2",
+            substrs=["size=error: Invalid value for start of vector."],
+        )
+        self.expect(
+            "frame variable v3",
+            substrs=["size=error: Start of vector data begins after end pointer."],
+        )
+        self.expect(
+            "frame variable v4",
+            substrs=["size=error: Failed to determine start/end of vector data."],
+        )
+        self.expect(
+            "frame variable v5",
+            substrs=["size=error: Size not multiple of element size."],
+        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
new file mode 100644
index 0000000000000..c9f04f60ec24d
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
@@ -0,0 +1,37 @@
+#define COMPRESSED_PAIR_REV 2
+#include <libcxx-simulators-common/compressed_pair.h>
+
+namespace std {
+namespace __1 {
+template <typename T> struct vector {
+  T *__begin_;
+  T *__end_;
+  _LLDB_COMPRESSED_PAIR(T *, __cap_ = nullptr, void *, __alloc_);
+};
+} // namespace __1
+
+namespace __2 {
+template <typename T> struct vector {};
+} // namespace __2
+
+namespace __3 {
+template <typename T> struct vector {
+  T *__begin_;
+  T *__end_;
+  _LLDB_COMPRESSED_PAIR(short *, __cap_ = nullptr, void *, __alloc_);
+};
+} // namespace __3
+} // namespace std
+
+int main() {
+  int arr[] = {1, 2, 3};
+  std::__1::vector<int> v1{.__begin_ = arr, .__end_ = nullptr};
+  std::__1::vector<int> v2{.__begin_ = nullptr, .__end_ = arr};
+  std::__1::vector<int> v3{.__begin_ = &arr[2], .__end_ = arr};
+  std::__2::vector<int> v4;
+
+  char carr[] = {'a'};
+  std::__3::vector<char> v5{.__begin_ = carr, .__end_ = carr + 1};
+
+  return 0;
+}

From 34598fdadc06bd3b21aa97342dda05ecd9233912 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Wed, 16 Apr 2025 18:05:27 +0200
Subject: [PATCH 136/710] [llvm][ItaniumDemangle] Use __LDBL_MANT_DIG__ for
 configuring demangling of long doubles (#135968)

Syncing in the changes from
https://github.com/llvm/llvm-project/pull/134976 using the
`cp-to-llvm.sh` script.
---
 llvm/include/llvm/Demangle/ItaniumDemangle.h | 22 ++++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index b0363c1a7a786..28fdfc5eff1b6 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -38,8 +38,10 @@
 DEMANGLE_NAMESPACE_BEGIN
 
 template <class T, size_t N> class PODSmallVector {
-  static_assert(std::is_trivial<T>::value,
-                "T is required to be a trivial type");
+  static_assert(std::is_trivially_copyable<T>::value,
+                "T is required to be a trivially copyable type");
+  static_assert(std::is_trivially_default_constructible<T>::value,
+                "T is required to be trivially default constructible");
   T *First = nullptr;
   T *Last = nullptr;
   T *Cap = nullptr;
@@ -5739,14 +5741,16 @@ struct FloatData<double>
 template <>
 struct FloatData<long double>
 {
-#if defined(__mips__) && defined(__mips_n64) || defined(__aarch64__) || \
-    defined(__wasm__) || defined(__riscv) || defined(__loongarch__) || \
-    defined(__ve__)
-    static const size_t mangled_size = 32;
-#elif defined(__arm__) || defined(__mips__) || defined(__hexagon__)
-    static const size_t mangled_size = 16;
+#if __LDBL_MANT_DIG__ == 113 || __LDBL_MANT_DIG__ == 106
+  static const size_t mangled_size = 32;
+#elif __LDBL_MANT_DIG__ == 53 || defined(_MSC_VER)
+  // MSVC doesn't define __LDBL_MANT_DIG__, but it has long double equal to
+  // regular double on all current architectures.
+  static const size_t mangled_size = 16;
+#elif __LDBL_MANT_DIG__ == 64
+  static const size_t mangled_size = 20;
 #else
-    static const size_t mangled_size = 20;  // May need to be adjusted to 16 or 24 on other platforms
+#error Unknown size for __LDBL_MANT_DIG__
 #endif
     // `-0x1.ffffffffffffffffffffffffffffp+16383` + 'L' + '\0' == 42 bytes.
     // 28 'f's * 4 bits == 112 bits, which is the number of mantissa bits.

From 7f4422d99115efbb770e13ccb60cf6bfc190c245 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Wed, 16 Apr 2025 17:14:14 +0100
Subject: [PATCH 137/710] [AArch64] Add testing for shuffles that extend into
 new types. NFC

---
 llvm/test/CodeGen/AArch64/shuffle-extend.ll | 264 ++++++++++++++++++++
 1 file changed, 264 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/shuffle-extend.ll

diff --git a/llvm/test/CodeGen/AArch64/shuffle-extend.ll b/llvm/test/CodeGen/AArch64/shuffle-extend.ll
new file mode 100644
index 0000000000000..bb31380cc3ade
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shuffle-extend.ll
@@ -0,0 +1,264 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+
+define <2 x i8> @test_v16i8_v2i32_824(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_v16i8_v2i32_824:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.b[8]
+; CHECK-NEXT:    umov w9, v1.b[8]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    add v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    ret
+  %c = shufflevector <16 x i8> %a, <16 x i8> %b, <2 x i32> <i32 8, i32 24>
+  %d = add <2 x i8> %c, %c
+  ret <2 x i8> %d
+}
+
+define <2 x i8> @test_v16i8_v2i32_016(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_v16i8_v2i32_016:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.b[0]
+; CHECK-NEXT:    umov w9, v1.b[0]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    add v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    ret
+  %c = shufflevector <16 x i8> %a, <16 x i8> %b, <2 x i32> <i32 0, i32 16>
+  %d = add <2 x i8> %c, %c
+  ret <2 x i8> %d
+}
+
+define <2 x i8> @test_v8i8_v2i32_08(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: test_v8i8_v2i32_08:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umov w8, v0.b[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    umov w9, v1.b[0]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    add v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    ret
+  %c = shufflevector <8 x i8> %a, <8 x i8> %b, <2 x i32> <i32 0, i32 8>
+  %d = add <2 x i8> %c, %c
+  ret <2 x i8> %d
+}
+
+define <2 x i16> @test_v8i16_v2i32_08(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_v8i16_v2i32_08:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[0]
+; CHECK-NEXT:    umov w9, v1.h[0]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    add v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    ret
+  %c = shufflevector <8 x i16> %a, <8 x i16> %b, <2 x i32> <i32 0, i32 8>
+  %d = add <2 x i16> %c, %c
+  ret <2 x i16> %d
+}
+
+define <2 x i16> @test_v4i16_v2i32_04(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: test_v4i16_v2i32_04:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umov w8, v0.h[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    umov w9, v1.h[0]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    add v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    ret
+  %c = shufflevector <4 x i16> %a, <4 x i16> %b, <2 x i32> <i32 0, i32 4>
+  %d = add <2 x i16> %c, %c
+  ret <2 x i16> %d
+}
+
+
+define <4 x i8> @test_v16i8_v4i16_824(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_v16i8_v4i16_824:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.b[8]
+; CHECK-NEXT:    umov w9, v1.b[8]
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    umov w8, v0.b[0]
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    umov w8, v1.b[0]
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    add v0.4h, v2.4h, v2.4h
+; CHECK-NEXT:    ret
+  %c = shufflevector <16 x i8> %a, <16 x i8> %b, <4 x i32> <i32 8, i32 24, i32 0, i32 16>
+  %d = add <4 x i8> %c, %c
+  ret <4 x i8> %d
+}
+
+define <4 x i8> @test_v16i8_v4i16_016(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_v16i8_v4i16_016:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.b[0]
+; CHECK-NEXT:    umov w9, v1.b[0]
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    umov w8, v0.b[4]
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    umov w8, v1.b[4]
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    add v0.4h, v2.4h, v2.4h
+; CHECK-NEXT:    ret
+  %c = shufflevector <16 x i8> %a, <16 x i8> %b, <4 x i32> <i32 0, i32 16, i32 4, i32 20>
+  %d = add <4 x i8> %c, %c
+  ret <4 x i8> %d
+}
+
+define <4 x i8> @test_v8i8_v4i16_08(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: test_v8i8_v4i16_08:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umov w8, v0.b[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    umov w9, v1.b[0]
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    umov w8, v0.b[4]
+; CHECK-NEXT:    mov v2.h[1], w9
+; CHECK-NEXT:    mov v2.h[2], w8
+; CHECK-NEXT:    umov w8, v1.b[4]
+; CHECK-NEXT:    mov v2.h[3], w8
+; CHECK-NEXT:    add v0.4h, v2.4h, v2.4h
+; CHECK-NEXT:    ret
+  %c = shufflevector <8 x i8> %a, <8 x i8> %b, <4 x i32> <i32 0, i32 8, i32 4, i32 12>
+  %d = add <4 x i8> %c, %c
+  ret <4 x i8> %d
+}
+
+define <4 x i16> @test_v8i16_v4i16_08(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_v8i16_v4i16_08:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI8_0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI8_0]
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-NEXT:    add v0.4h, v0.4h, v0.4h
+; CHECK-NEXT:    ret
+  %c = shufflevector <8 x i16> %a, <8 x i16> %b, <4 x i32> <i32 0, i32 7, i32 6, i32 12>
+  %d = add <4 x i16> %c, %c
+  ret <4 x i16> %d
+}
+
+define <4 x i16> @test_v4i16_v4i16_04(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: test_v4i16_v4i16_04:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.8b, v1.8b, v0.8b, #2
+; CHECK-NEXT:    trn2 v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ext v0.8b, v0.8b, v1.8b, #4
+; CHECK-NEXT:    add v0.4h, v0.4h, v0.4h
+; CHECK-NEXT:    ret
+  %c = shufflevector <4 x i16> %a, <4 x i16> %b, <4 x i32> <i32 0, i32 3, i32 5, i32 6>
+  %d = add <4 x i16> %c, %c
+  ret <4 x i16> %d
+}
+
+
+define i1 @test1(ptr %add.ptr, ptr %result, <2 x i64> %hi, <2 x i64> %lo) {
+; CHECK-LABEL: test1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q2, [x0]
+; CHECK-NEXT:    movi v3.16b, #1
+; CHECK-NEXT:    mov w12, #1 // =0x1
+; CHECK-NEXT:    cmgt v0.2d, v2.2d, v0.2d
+; CHECK-NEXT:    cmgt v4.2d, v1.2d, v2.2d
+; CHECK-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    umov w8, v0.b[8]
+; CHECK-NEXT:    umov w9, v3.b[8]
+; CHECK-NEXT:    umov w10, v0.b[0]
+; CHECK-NEXT:    umov w11, v3.b[0]
+; CHECK-NEXT:    sub v0.2d, v2.2d, v1.2d
+; CHECK-NEXT:    dup v1.2d, x12
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    orr w8, w10, w8, lsl #1
+; CHECK-NEXT:    orr w8, w8, w11
+; CHECK-NEXT:    str q0, [x1]
+; CHECK-NEXT:    tst w8, #0x3
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %19 = load <2 x i64>, ptr %add.ptr, align 8
+  %cmp = icmp sgt <2 x i64> %19, %hi
+  %sext = sext <2 x i1> %cmp to <2 x i64>
+  %20 = bitcast <2 x i64> %sext to <16 x i8>
+  %21 = and <16 x i8> %20, <i8 1, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 1, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>
+  %storedv = extractelement <16 x i8> %21, i64 0
+  %storedv.1 = extractelement <16 x i8> %21, i64 8
+  %22 = shl nuw nsw i8 %storedv.1, 1
+  %or.111 = or disjoint i8 %22, %storedv
+  %cmp101 = icmp slt <2 x i64> %19, %lo
+  %sext102 = sext <2 x i1> %cmp101 to <2 x i64>
+  %23 = bitcast <2 x i64> %sext102 to <16 x i8>
+  %24 = and <16 x i8> %23, <i8 1, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 1, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>
+  %storedv104 = extractelement <16 x i8> %24, i64 0
+  %storedv.1105 = extractelement <16 x i8> %24, i64 8
+  %25 = shl nuw nsw i8 %storedv.1105, 1
+  %or.111106 = or disjoint i8 %25, %storedv104
+  %reass.sub = sub <2 x i64> %19, %lo
+  %add = add <2 x i64> %reass.sub, splat (i64 1)
+  store <2 x i64> %add, ptr %result, align 8
+  %or118 = or i8 %or.111, %or.111106
+  %cmp24.not = icmp eq i8 %or118, 0
+  ret i1 %cmp24.not
+}
+
+define i1 @test2(ptr %add.ptr, ptr %result, <2 x i64> %hi, <2 x i64> %lo) {
+; CHECK-LABEL: test2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q2, [x0]
+; CHECK-NEXT:    movi v3.16b, #1
+; CHECK-NEXT:    cmgt v0.2d, v2.2d, v0.2d
+; CHECK-NEXT:    cmgt v4.2d, v1.2d, v2.2d
+; CHECK-NEXT:    sub v1.2d, v2.2d, v1.2d
+; CHECK-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    umov w8, v0.b[8]
+; CHECK-NEXT:    umov w9, v3.b[8]
+; CHECK-NEXT:    umov w10, v0.b[0]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    umov w8, v3.b[0]
+; CHECK-NEXT:    fmov s3, w10
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    mov w9, #1 // =0x1
+; CHECK-NEXT:    mov v3.s[1], w8
+; CHECK-NEXT:    dup v2.2d, x9
+; CHECK-NEXT:    add v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    orr v0.8b, v0.8b, v3.8b
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    add v0.2d, v1.2d, v2.2d
+; CHECK-NEXT:    str q0, [x1]
+; CHECK-NEXT:    orr w8, w9, w8
+; CHECK-NEXT:    tst w8, #0xff
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %1 = load <2 x i64>, ptr %add.ptr, align 8
+  %cmp = icmp sgt <2 x i64> %1, %hi
+  %sext = sext <2 x i1> %cmp to <2 x i64>
+  %2 = bitcast <2 x i64> %sext to <16 x i8>
+  %3 = and <16 x i8> %2, <i8 1, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 1, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>
+  %cmp101 = icmp slt <2 x i64> %1, %lo
+  %sext102 = sext <2 x i1> %cmp101 to <2 x i64>
+  %4 = bitcast <2 x i64> %sext102 to <16 x i8>
+  %5 = and <16 x i8> %4, <i8 1, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 1, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>
+  %6 = shufflevector <16 x i8> %3, <16 x i8> %5, <2 x i32> <i32 8, i32 24>
+  %7 = shl nuw nsw <2 x i8> %6, splat (i8 1)
+  %8 = shufflevector <16 x i8> %3, <16 x i8> %5, <2 x i32> <i32 0, i32 16>
+  %9 = or disjoint <2 x i8> %7, %8
+  %reass.sub = sub <2 x i64> %1, %lo
+  %add = add <2 x i64> %reass.sub, splat (i64 1)
+  store <2 x i64> %add, ptr %result, align 8
+  %10 = extractelement <2 x i8> %9, i32 0
+  %11 = extractelement <2 x i8> %9, i32 1
+  %or118 = or i8 %10, %11
+  %cmp24.not = icmp eq i8 %or118, 0
+  ret i1 %cmp24.not
+}

From 24171f4d12a02b49de1cc7a1beb2dc19c740a9f1 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 09:29:28 -0700
Subject: [PATCH 138/710] [NFC][CFI] Add test to check for '-flto' and
 '-fvisibility=' flags (#135892)

---
 clang/test/Driver/sanitizer-ld.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index 52c1f6bf96242..67ca33d676d20 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -829,6 +829,16 @@
 
 // CHECK-NSAN-UBSAN: "--whole-archive" "{{[^"]*}}libclang_rt.nsan.a" "--no-whole-archive"
 
+// CFI requirements.
+// RUN: not %clang -fsanitize=cfi \
+// RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
+// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
+// RUN:     -### %s 2>&1 \
+// RUN:   | %{filecheck} --check-prefix=CHECK-CFI-PREREQ-LINUX
+// CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-flto'
+// CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-fvisibility='
+
 // CFI by itself does not link runtime libraries.
 // RUN: not %clang -fsanitize=cfi \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \

From 76b5fcbf975547251faaeed8b567ea09d139a607 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 16 Apr 2025 09:39:52 -0700
Subject: [PATCH 139/710] [TableGen] Store flat source operand number in
 OperandMap in PseudoLoweringEmitter. NFC (#135886)

Previously we stored the index into the source CodeGenInstruction's
operand list. Any operand with sub operands stored the same index into
all of the OperandMap entries for that operand. The emitting loop would
look up the MIOperandNo for the source and add the sub index.

This patch moves the logic into the loop that updates the OperandMap.
Now the emitting loop only needs to print the value.

While there, I've added a check that MIOperandNo is the same for source
and destination.
---
 llvm/utils/TableGen/PseudoLoweringEmitter.cpp | 29 +++++++++++--------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index 7f67c13c0bbbd..96325eac95004 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -181,24 +181,32 @@ void PseudoLoweringEmitter::evaluateExpansion(const Record *Rec) {
     SourceOperands[SrcOp.Name] = Idx;
 
   LLVM_DEBUG(dbgs() << "  Operand mapping:\n");
-  for (unsigned i = 0, e = Insn.Operands.size(); i != e; ++i) {
+  for (const auto &[Idx, Opnd] : enumerate(Insn.Operands)) {
     // We've already handled constant values. Just map instruction operands
     // here.
-    if (OperandMap[Insn.Operands[i].MIOperandNo].Kind != OpData::Operand)
+    if (OperandMap[Opnd.MIOperandNo].Kind != OpData::Operand)
       continue;
     StringMap<unsigned>::iterator SourceOp =
-        SourceOperands.find(Dag->getArgNameStr(i));
+        SourceOperands.find(Dag->getArgNameStr(Idx));
     if (SourceOp == SourceOperands.end())
       PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
-                               "', output operand '" + Dag->getArgNameStr(i) +
+                               "', output operand '" + Dag->getArgNameStr(Idx) +
                                "' has no matching source operand");
+    const auto &SrcOpnd = SourceInsn.Operands[SourceOp->getValue()];
+    if (Opnd.MINumOperands != SrcOpnd.MINumOperands)
+      PrintFatalError(
+          Rec,
+          "In pseudo instruction '" + Rec->getName() + "', output operand '" +
+              Opnd.Rec->getName() +
+              "' has a different number of sub operands than source operand '" +
+              SrcOpnd.Rec->getName() + "'");
+
     // Map the source operand to the destination operand index for each
     // MachineInstr operand.
-    for (unsigned I = 0, E = Insn.Operands[i].MINumOperands; I != E; ++I)
-      OperandMap[Insn.Operands[i].MIOperandNo + I].Data.Operand =
-          SourceOp->getValue();
+    for (unsigned I = 0, E = Opnd.MINumOperands; I != E; ++I)
+      OperandMap[Opnd.MIOperandNo + I].Data.Operand = SrcOpnd.MIOperandNo + I;
 
-    LLVM_DEBUG(dbgs() << "    " << SourceOp->getValue() << " ==> " << i
+    LLVM_DEBUG(dbgs() << "    " << SourceOp->getValue() << " ==> " << Idx
                       << "\n");
   }
 
@@ -236,10 +244,7 @@ void PseudoLoweringEmitter::emitLoweringEmitter(raw_ostream &o) {
           switch (Expansion.OperandMap[MIOpNo + i].Kind) {
           case OpData::Operand:
             o << "    lowerOperand(MI->getOperand("
-              << Source.Operands[Expansion.OperandMap[MIOpNo].Data.Operand]
-                         .MIOperandNo +
-                     i
-              << "), MCOp);\n"
+              << Expansion.OperandMap[MIOpNo + i].Data.Operand << "), MCOp);\n"
               << "    Inst.addOperand(MCOp);\n";
             break;
           case OpData::Imm:

From 76b7ae7e454a1f0d814406d16926aa6722afcda4 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 16 Apr 2025 09:30:23 -0700
Subject: [PATCH 140/710] [SLP][NFC]Remove std::placeholders:: qualifiers, NFC

---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 24 ++++++++-----------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 764f3e2dc64d9..810d44343c4a9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -21008,14 +21008,13 @@ bool SLPVectorizerPass::vectorizeStores(
           AnyProfitableGraph = false;
           unsigned StartIdx = std::distance(
               RangeSizes.begin(),
-              find_if(RangeSizes, std::bind(IsNotVectorized, Size >= MaxRegVF,
-                                            std::placeholders::_1)));
+              find_if(RangeSizes,
+                      std::bind(IsNotVectorized, Size >= MaxRegVF, _1)));
           while (StartIdx < End) {
-            unsigned EndIdx =
-                std::distance(RangeSizes.begin(),
-                              find_if(RangeSizes.drop_front(StartIdx),
-                                      std::bind(IsVectorized, Size >= MaxRegVF,
-                                                std::placeholders::_1)));
+            unsigned EndIdx = std::distance(
+                RangeSizes.begin(),
+                find_if(RangeSizes.drop_front(StartIdx),
+                        std::bind(IsVectorized, Size >= MaxRegVF, _1)));
             unsigned Sz = EndIdx >= End ? End : EndIdx;
             for (unsigned Cnt = StartIdx; Cnt + Size <= Sz;) {
               if (!checkTreeSizes(RangeSizes.slice(Cnt, Size),
@@ -21085,7 +21084,7 @@ bool SLPVectorizerPass::vectorizeStores(
               if (Size > 2 && Res &&
                   !all_of(RangeSizes.slice(Cnt, Size),
                           std::bind(VFIsProfitable, Size >= MaxRegVF, TreeSize,
-                                    std::placeholders::_1))) {
+                                    _1))) {
                 Cnt += Size;
                 continue;
               }
@@ -21093,8 +21092,7 @@ bool SLPVectorizerPass::vectorizeStores(
               // trees, just with larger number of elements.
               if (Size > MaxRegVF && TreeSize > 1 &&
                   all_of(RangeSizes.slice(Cnt, Size),
-                         std::bind(FirstSizeSame, TreeSize,
-                                   std::placeholders::_1))) {
+                         std::bind(FirstSizeSame, TreeSize, _1))) {
                 Cnt += Size;
                 while (Cnt != Sz && RangeSizes[Cnt].first == TreeSize)
                   ++Cnt;
@@ -21118,8 +21116,7 @@ bool SLPVectorizerPass::vectorizeStores(
             StartIdx = std::distance(
                 RangeSizes.begin(),
                 find_if(RangeSizes.drop_front(Sz),
-                        std::bind(IsNotVectorized, Size >= MaxRegVF,
-                                  std::placeholders::_1)));
+                        std::bind(IsNotVectorized, Size >= MaxRegVF, _1)));
           }
           if (!AnyProfitableGraph && Size >= MaxRegVF && has_single_bit(Size))
             break;
@@ -21140,8 +21137,7 @@ bool SLPVectorizerPass::vectorizeStores(
                 End -
                 std::distance(
                     RangeSizes.begin(),
-                    find_if(RangeSizes, std::bind(IsNotVectorized, true,
-                                                  std::placeholders::_1))) +
+                    find_if(RangeSizes, std::bind(IsNotVectorized, true, _1))) +
                 1));
         unsigned VF = bit_ceil(CandidateVFs.front()) * 2;
         unsigned Limit =

From 726a5c2c57c486e69df2dfc296482e1d8014ab62 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 09:54:27 -0700
Subject: [PATCH 141/710] [NFC][CFI] Avoid clang error in CFI tests (#135981)

In these tests we test correct linking flags set,
and it's confusing that command fails because of
other missing required flags.

Clang diagnostics opportunistically proceed after
error report on required flags, but there is no
guaranty that processing of tested flags are the same
in supported and erroneous flag sets.
---
 clang/test/Driver/sanitizer-ld.c | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index 67ca33d676d20..a00ec029d3d46 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -840,7 +840,8 @@
 // CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-fvisibility='
 
 // CFI by itself does not link runtime libraries.
-// RUN: not %clang -fsanitize=cfi \
+// RUN: %clang -fsanitize=cfi \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -849,7 +850,8 @@
 // CHECK-CFI-LINUX: "{{.*}}ld{{(.exe)?}}"
 
 // CFI with diagnostics links the UBSan runtime.
-// RUN: not %clang -fsanitize=cfi -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
+// RUN: %clang -fsanitize=cfi -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -859,7 +861,8 @@
 // CHECK-CFI-DIAG-LINUX: "--whole-archive" "{{[^"]*}}libclang_rt.ubsan_standalone.a" "--no-whole-archive"
 
 // Cross-DSO CFI links the CFI runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -870,7 +873,8 @@
 // CHECK-CFI-CROSS-DSO-LINUX: -export-dynamic
 
 // Cross-DSO CFI with diagnostics links just the CFI runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
@@ -882,7 +886,8 @@
 // CHECK-CFI-CROSS-DSO-DIAG-LINUX: -export-dynamic
 
 // Cross-DSO CFI on Android does not link runtime libraries.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     --target=aarch64-linux-android -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree \
@@ -891,7 +896,8 @@
 // CHECK-CFI-CROSS-DSO-ANDROID: "{{.*}}ld{{(.exe)?}}"
 
 // Cross-DSO CFI with diagnostics on Android links just the UBSAN runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=aarch64-linux-android -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \

From 72506eb37d9440d32e6dada187785b06aecb415c Mon Sep 17 00:00:00 2001
From: Kostiantyn Lazukin <konstantin.lazukin@gmail.com>
Date: Wed, 16 Apr 2025 17:54:42 +0100
Subject: [PATCH 142/710] [compiler-rt] Fix `addtf3_test.c` being skipped due
 to misplaced include (#134106)

[compiler-rt] The test `addtf3_test.c` is currently guarded by `#if
defined(CRT_HAS_IEEE_TF)`, a macro that is declared in `int_lib.h`.
However, `int_lib.h` is included *after* the preprocessor check, which
results in the macro not being defined in time and causes the test to
always be skipped.

This patch moves the includes of `fp_test.h` and `int_lib.h` to the top
of the file so that `CRT_HAS_IEEE_TF` is defined before it is checked.

Co-authored-by: Kostiantyn Lazukin <koslaz01@ip-10-252-21-142.eu-west-1.compute.internal>
---
 compiler-rt/test/builtins/Unit/addtf3_test.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/test/builtins/Unit/addtf3_test.c b/compiler-rt/test/builtins/Unit/addtf3_test.c
index cd5872e7dedf4..0ab73172c193a 100644
--- a/compiler-rt/test/builtins/Unit/addtf3_test.c
+++ b/compiler-rt/test/builtins/Unit/addtf3_test.c
@@ -4,14 +4,14 @@
 #include <fenv.h>
 #include <stdio.h>
 
+#include "fp_test.h"
+#include "int_lib.h"
+
 // The testcase currently assumes IEEE TF format, once that has been
 // fixed the defined(CRT_HAS_IEEE_TF) guard can be removed to enable it for
 // IBM 128 floats as well.
 #if defined(CRT_HAS_IEEE_TF)
 
-#  include "fp_test.h"
-#  include "int_lib.h"
-
 // Returns: a + b
 COMPILER_RT_ABI tf_float __addtf3(tf_float a, tf_float b);
 
@@ -62,7 +62,13 @@ int main() {
       defined(i386) || defined(__x86_64__) ||                                  \
       (defined(__loongarch__) && __loongarch_frlen != 0)
   // Rounding mode tests on supported architectures
-  const tf_float m = 1234.0L, n = 0.01L;
+  // Use explicit values because the binary representation of long double
+  // is platform dependent. Intended values:
+  // m = 1234.0L, n = 0.01L (where L is a literal for 128 bit long double)
+  const tf_float m =
+      fromRep128(UINT64_C(0x4009348000000000), UINT64_C(0x0000000000000000));
+  const tf_float n =
+      fromRep128(UINT64_C(0x3FF847AE147AE147), UINT64_C(0xAE147AE147AE147B));
 
   fesetround(FE_UPWARD);
   if (test__addtf3(m, n, UINT64_C(0x40093480a3d70a3d),

From 51fa6cde7d773aa7f41b410c8263884ad32eca86 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 16 Apr 2025 09:52:27 -0700
Subject: [PATCH 143/710] [SLP][NFC]Add a test with missing unsigned promotion
 for smax reduction, NFC

---
 .../smax-reduction-unsigned-missing-sign.ll   | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/smax-reduction-unsigned-missing-sign.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/smax-reduction-unsigned-missing-sign.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/smax-reduction-unsigned-missing-sign.ll
new file mode 100644
index 0000000000000..e6408572acf8f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/smax-reduction-unsigned-missing-sign.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=riscv64-unknown-linux-gnu -mattr=+v < %s | FileCheck %s
+
+define i32 @test(i8 %0) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i8 [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i8> <i8 poison, i8 0, i8 0, i8 0>, i8 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <4 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.smax.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+entry:
+  %1 = icmp ne i8 0, 0
+  %2 = zext i1 %1 to i32
+  %3 = icmp ne i8 %0, 0
+  %4 = zext i1 %3 to i32
+  %5 = icmp ne i8 0, 0
+  %6 = zext i1 %5 to i32
+  %7 = icmp ne i8 0, 0
+  %8 = zext i1 %7 to i32
+  %cond27.2 = tail call i32 @llvm.smax.i32(i32 %4, i32 %2)
+  %cond27.3 = tail call i32 @llvm.smax.i32(i32 %6, i32 %cond27.2)
+  %cond27.4 = tail call i32 @llvm.smax.i32(i32 %8, i32 %cond27.3)
+  ret i32 %cond27.4
+}
+
+declare i32 @llvm.smax.i32(i32, i32)

From ed9bcb52954f8e6171563d2b8310b0ca6d03d655 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Wed, 16 Apr 2025 20:05:13 +0300
Subject: [PATCH 144/710] [CodeGen][RISCV] Add helper class for emitting CFI
 instructions into MIR (#135845)

PR: https://github.com/llvm/llvm-project/pull/135845
---
 llvm/include/llvm/CodeGen/CFIInstBuilder.h   |  88 ++++++
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 272 ++++---------------
 2 files changed, 146 insertions(+), 214 deletions(-)
 create mode 100644 llvm/include/llvm/CodeGen/CFIInstBuilder.h

diff --git a/llvm/include/llvm/CodeGen/CFIInstBuilder.h b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
new file mode 100644
index 0000000000000..e799b47a0c974
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_CFIINSTBUILDER_H
+#define LLVM_CODEGEN_CFIINSTBUILDER_H
+
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/MC/MCDwarf.h"
+
+namespace llvm {
+
+/// Helper class for creating CFI instructions and inserting them into MIR.
+class CFIInstBuilder {
+  MachineFunction &MF;
+  MachineBasicBlock &MBB;
+  MachineBasicBlock::iterator InsertPt;
+
+  /// MIFlag to set on a MachineInstr. Typically, FrameSetup or FrameDestroy.
+  MachineInstr::MIFlag MIFlag;
+
+  /// Selects DWARF register numbering: debug or exception handling. Should be
+  /// consistent with the choice of the ELF section (.debug_frame or .eh_frame)
+  /// where CFI will be encoded.
+  bool IsEH;
+
+  // Cache frequently used variables.
+  const TargetRegisterInfo &TRI;
+  const MCInstrDesc &CFIID;
+  const MIMetadata MIMD; // Default-initialized, no debug location desired.
+
+public:
+  CFIInstBuilder(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
+                 MachineInstr::MIFlag MIFlag, bool IsEH = true)
+      : MF(*MBB.getParent()), MBB(MBB), MIFlag(MIFlag), IsEH(IsEH),
+        TRI(*MF.getSubtarget().getRegisterInfo()),
+        CFIID(MF.getSubtarget().getInstrInfo()->get(
+            TargetOpcode::CFI_INSTRUCTION)) {
+    setInsertPoint(InsertPt);
+  }
+
+  void setInsertPoint(MachineBasicBlock::iterator IP) { InsertPt = IP; }
+
+  void insertCFIInst(const MCCFIInstruction &CFIInst) const {
+    BuildMI(MBB, InsertPt, MIMD, CFIID)
+        .addCFIIndex(MF.addFrameInst(CFIInst))
+        .setMIFlag(MIFlag);
+  }
+
+  void buildDefCFA(MCRegister Reg, int64_t Offset) const {
+    insertCFIInst(MCCFIInstruction::cfiDefCfa(
+        nullptr, TRI.getDwarfRegNum(Reg, IsEH), Offset));
+  }
+
+  void buildDefCFARegister(MCRegister Reg) const {
+    insertCFIInst(MCCFIInstruction::createDefCfaRegister(
+        nullptr, TRI.getDwarfRegNum(Reg, IsEH)));
+  }
+
+  void buildDefCFAOffset(int64_t Offset) const {
+    insertCFIInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
+  }
+
+  void buildOffset(MCRegister Reg, int64_t Offset) const {
+    insertCFIInst(MCCFIInstruction::createOffset(
+        nullptr, TRI.getDwarfRegNum(Reg, IsEH), Offset));
+  }
+
+  void buildRestore(MCRegister Reg) const {
+    insertCFIInst(MCCFIInstruction::createRestore(
+        nullptr, TRI.getDwarfRegNum(Reg, IsEH)));
+  }
+
+  void buildEscape(StringRef Bytes, StringRef Comment = "") const {
+    insertCFIInst(
+        MCCFIInstruction::createEscape(nullptr, Bytes, SMLoc(), Comment));
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_CFIINSTBUILDER_H
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index c7b2b781422d1..cefe7a732519d 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -14,6 +14,7 @@
 #include "RISCVMachineFunctionInfo.h"
 #include "RISCVSubtarget.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -28,64 +29,6 @@
 
 using namespace llvm;
 
-namespace {
-
-class CFISaveRegisterEmitter {
-  MachineFunction &MF;
-  MachineFrameInfo &MFI;
-
-public:
-  CFISaveRegisterEmitter(MachineFunction &MF)
-      : MF{MF}, MFI{MF.getFrameInfo()} {};
-
-  void emit(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-            const RISCVRegisterInfo &RI, const RISCVInstrInfo &TII,
-            const DebugLoc &DL, const CalleeSavedInfo &CS) const {
-    int FrameIdx = CS.getFrameIdx();
-    int64_t Offset = MFI.getObjectOffset(FrameIdx);
-    MCRegister Reg = CS.getReg();
-    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
-        nullptr, RI.getDwarfRegNum(Reg, true), Offset));
-    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameSetup);
-  }
-};
-
-class CFIRestoreRegisterEmitter {
-  MachineFunction &MF;
-
-public:
-  CFIRestoreRegisterEmitter(MachineFunction &MF) : MF{MF} {};
-
-  void emit(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-            const RISCVRegisterInfo &RI, const RISCVInstrInfo &TII,
-            const DebugLoc &DL, const CalleeSavedInfo &CS) const {
-    MCRegister Reg = CS.getReg();
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::createRestore(nullptr, RI.getDwarfRegNum(Reg, true)));
-    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameDestroy);
-  }
-};
-
-} // namespace
-
-template <typename Emitter>
-void RISCVFrameLowering::emitCFIForCSI(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-    const SmallVector<CalleeSavedInfo, 8> &CSI) const {
-  MachineFunction *MF = MBB.getParent();
-  const RISCVRegisterInfo *RI = STI.getRegisterInfo();
-  const RISCVInstrInfo *TII = STI.getInstrInfo();
-  DebugLoc DL = MBB.findDebugLoc(MBBI);
-
-  Emitter E{*MF};
-  for (const auto &CS : CSI)
-    E.emit(MBB, MBBI, *RI, *TII, DL, CS);
-}
-
 static Align getABIStackAlignment(RISCVABI::ABI ABI) {
   if (ABI == RISCVABI::ABI_ILP32E)
     return Align(4);
@@ -209,11 +152,8 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
       Offset, // addend (sleb128)
   };
 
-  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
-      nullptr, StringRef(CFIInst, sizeof(CFIInst))));
-  BuildMI(MBB, MI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex)
-      .setMIFlag(MachineInstr::FrameSetup);
+  CFIInstBuilder(MBB, MI, MachineInstr::FrameSetup)
+      .buildEscape(StringRef(CFIInst, sizeof(CFIInst)));
 }
 
 static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -257,11 +197,7 @@ static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
       .addImm(-SlotSize)
       .setMIFlag(MachineInstr::FrameDestroy);
   // Restore the SCS pointer
-  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
-      nullptr, STI.getRegisterInfo()->getDwarfRegNum(SCSPReg, /*IsEH*/ true)));
-  BuildMI(MBB, MI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex)
-      .setMIFlags(MachineInstr::FrameDestroy);
+  CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
 }
 
 // Get the ID of the libcall used for spilling and restoring callee saved
@@ -531,14 +467,10 @@ void RISCVFrameLowering::allocateAndProbeStackForRVV(
       .setMIFlag(Flag);
   TII->mulImm(MF, MBB, MBBI, DL, TargetReg, NumOfVReg, Flag);
 
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
   if (EmitCFI) {
     // Set the CFA register to TargetReg.
-    unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(TargetReg, true);
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, -Amount));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIBuilder.buildDefCFA(TargetReg, -Amount);
   }
 
   // It will be expanded to a probe loop in `inlineStackProbe`.
@@ -548,12 +480,7 @@ void RISCVFrameLowering::allocateAndProbeStackForRVV(
 
   if (EmitCFI) {
     // Set the CFA register back to SP.
-    unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(SPReg, true);
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIBuilder.buildDefCFARegister(SPReg);
   }
 
   // SUB SP, SP, T1
@@ -665,20 +592,15 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
   const RISCVInstrInfo *TII = STI.getInstrInfo();
   bool IsRV64 = STI.is64Bit();
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
 
   // Simply allocate the stack if it's not big enough to require a probe.
   if (!NeedProbe || Offset <= ProbeSize) {
     RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-Offset),
                   MachineInstr::FrameSetup, getStackAlign());
 
-    if (EmitCFI) {
-      // Emit ".cfi_def_cfa_offset RealStackSize"
-      unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize));
-      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlag(MachineInstr::FrameSetup);
-    }
+    if (EmitCFI)
+      CFIBuilder.buildDefCFAOffset(RealStackSize);
 
     if (NeedProbe && DynAllocation) {
       // s[d|w] zero, 0(sp)
@@ -707,14 +629,8 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
           .setMIFlags(MachineInstr::FrameSetup);
 
       CurrentOffset += ProbeSize;
-      if (EmitCFI) {
-        // Emit ".cfi_def_cfa_offset CurrentOffset"
-        unsigned CFIIndex = MF.addFrameInst(
-            MCCFIInstruction::cfiDefCfaOffset(nullptr, CurrentOffset));
-        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex)
-            .setMIFlag(MachineInstr::FrameSetup);
-      }
+      if (EmitCFI)
+        CFIBuilder.buildDefCFAOffset(CurrentOffset);
     }
 
     uint64_t Residual = Offset - CurrentOffset;
@@ -722,14 +638,8 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
       RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
                     StackOffset::getFixed(-Residual), MachineInstr::FrameSetup,
                     getStackAlign());
-      if (EmitCFI) {
-        // Emit ".cfi_def_cfa_offset Offset"
-        unsigned CFIIndex =
-            MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
-        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex)
-            .setMIFlag(MachineInstr::FrameSetup);
-      }
+      if (EmitCFI)
+        CFIBuilder.buildDefCFAOffset(Offset);
 
       if (DynAllocation) {
         // s[d|w] zero, 0(sp)
@@ -756,12 +666,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
 
   if (EmitCFI) {
     // Set the CFA register to TargetReg.
-    unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(TargetReg, true);
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, RoundedSize));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIBuilder.buildDefCFA(TargetReg, RoundedSize);
   }
 
   // It will be expanded to a probe loop in `inlineStackProbe`.
@@ -771,12 +676,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
 
   if (EmitCFI) {
     // Set the CFA register back to SP.
-    unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(SPReg, true);
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIBuilder.buildDefCFARegister(SPReg);
   }
 
   if (Residual) {
@@ -792,14 +692,8 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
     }
   }
 
-  if (EmitCFI) {
-    // Emit ".cfi_def_cfa_offset Offset"
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
-  }
+  if (EmitCFI)
+    CFIBuilder.buildDefCFAOffset(Offset);
 }
 
 static bool isPush(unsigned Opcode) {
@@ -855,7 +749,6 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
-  const RISCVInstrInfo *TII = STI.getInstrInfo();
   MachineBasicBlock::iterator MBBI = MBB.begin();
 
   Register BPReg = RISCVABI::getBPReg();
@@ -888,6 +781,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   // callee-saved register.
   MBBI = std::prev(MBBI, getRVVCalleeSavedInfo(MF, CSI).size() +
                              getUnmanagedCSI(MF, CSI).size());
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
 
   // If libcalls are used to spill and restore callee-saved registers, the frame
   // has two sections; the opaque section managed by the libcalls, and the
@@ -915,14 +809,10 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
         alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
     RVFI->setLibCallStackSize(LibCallFrameSize);
 
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::cfiDefCfaOffset(nullptr, LibCallFrameSize));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameSetup);
-
-    emitCFIForCSI<CFISaveRegisterEmitter>(MBB, MBBI,
-                                          getPushOrLibCallsSavedInfo(MF, CSI));
+    CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
+    for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+      CFIBuilder.buildOffset(CS.getReg(),
+                             MFI.getObjectOffset(CS.getFrameIdx()));
   }
 
   // FIXME (note copied from Lanai): This appears to be overallocating.  Needs
@@ -949,13 +839,10 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   }
 
   if (RVFI->useQCIInterrupt(MF)) {
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::cfiDefCfaOffset(nullptr, QCIInterruptPushAmount));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameSetup);
-
-    emitCFIForCSI<CFISaveRegisterEmitter>(MBB, MBBI, getQCISavedInfo(MF, CSI));
+    CFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
+    for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
+      CFIBuilder.buildOffset(CS.getReg(),
+                             MFI.getObjectOffset(CS.getFrameIdx()));
   } else if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
              isPush(FirstFrameSetup->getOpcode())) {
     // Use available stack adjustment in push instruction to allocate additional
@@ -967,14 +854,10 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     FirstFrameSetup->getOperand(1).setImm(StackAdj);
     StackSize -= StackAdj;
 
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize - StackSize));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameSetup);
-
-    emitCFIForCSI<CFISaveRegisterEmitter>(MBB, MBBI,
-                                          getPushOrLibCallsSavedInfo(MF, CSI));
+    CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
+    for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+      CFIBuilder.buildOffset(CS.getReg(),
+                             MFI.getObjectOffset(CS.getFrameIdx()));
   }
 
   // Allocate space on the stack if necessary.
@@ -995,10 +878,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   // FIXME: assumes exactly one instruction is used to save each callee-saved
   // register.
   std::advance(MBBI, getUnmanagedCSI(MF, CSI).size());
+  CFIBuilder.setInsertPoint(MBBI);
 
   // Iterate over list of callee-saved registers and emit .cfi_offset
   // directives.
-  emitCFIForCSI<CFISaveRegisterEmitter>(MBB, MBBI, getUnmanagedCSI(MF, CSI));
+  for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+    CFIBuilder.buildOffset(CS.getReg(), MFI.getObjectOffset(CS.getFrameIdx()));
 
   // Generate new FP.
   if (hasFP(MF)) {
@@ -1017,12 +902,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
           MachineInstr::FrameSetup, getStackAlign());
     }
 
-    // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
-    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
-        nullptr, RI->getDwarfRegNum(FPReg, true), RVFI->getVarArgsSaveSize()));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameSetup);
+    CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
   }
 
   uint64_t SecondSPAdjustAmount = 0;
@@ -1052,11 +932,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
 
     if (!hasFP(MF)) {
       // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
-      unsigned CFIIndex = MF.addFrameInst(createDefCFAExpression(
+      CFIBuilder.insertCFIInst(createDefCFAExpression(
           *RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8));
-      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlag(MachineInstr::FrameSetup);
     }
 
     std::advance(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
@@ -1122,17 +999,13 @@ void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
                                          uint64_t &StackSize,
                                          int64_t CFAOffset) const {
   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
-  const RISCVInstrInfo *TII = STI.getInstrInfo();
 
   RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize),
                 MachineInstr::FrameDestroy, getStackAlign());
   StackSize = 0;
 
-  unsigned CFIIndex =
-      MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset));
-  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex)
-      .setMIFlag(MachineInstr::FrameDestroy);
+  CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
+      .buildDefCFAOffset(CFAOffset);
 }
 
 void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
@@ -1140,7 +1013,6 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
-  const RISCVInstrInfo *TII = STI.getInstrInfo();
 
   // All calls are tail calls in GHC calling conv, and functions have no
   // prologue/epilogue.
@@ -1171,6 +1043,8 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   // callee-saved register.
   auto FirstScalarCSRRestoreInsn =
       std::next(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
+  CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
+                            MachineInstr::FrameDestroy);
 
   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
   uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
@@ -1191,14 +1065,8 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                     StackOffset::getScalable(RVVStackSize),
                     MachineInstr::FrameDestroy, getStackAlign());
 
-    if (!hasFP(MF)) {
-      unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
-          nullptr, RI->getDwarfRegNum(SPReg, true), RealStackSize));
-      BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
-              TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlag(MachineInstr::FrameDestroy);
-    }
+    if (!hasFP(MF))
+      CFIBuilder.buildDefCFA(SPReg, RealStackSize);
 
     emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
   }
@@ -1216,14 +1084,8 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                     StackOffset::getFixed(SecondSPAdjustAmount),
                     MachineInstr::FrameDestroy, getStackAlign());
 
-    if (!hasFP(MF)) {
-      unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::cfiDefCfaOffset(nullptr, FirstSPAdjustAmount));
-      BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
-              TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlag(MachineInstr::FrameDestroy);
-    }
+    if (!hasFP(MF))
+      CFIBuilder.buildDefCFAOffset(FirstSPAdjustAmount);
   }
 
   // Restore the stack pointer using the value of the frame pointer. Only
@@ -1243,19 +1105,14 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                   getStackAlign());
   }
 
-  if (hasFP(MF)) {
-    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
-        nullptr, RI->getDwarfRegNum(SPReg, true), RealStackSize));
-    BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
-            TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameDestroy);
-  }
+  if (hasFP(MF))
+    CFIBuilder.buildDefCFA(SPReg, RealStackSize);
 
   // Skip to after the restores of scalar callee-saved registers
   // FIXME: assumes exactly one instruction is used to restore each
   // callee-saved register.
   MBBI = std::next(FirstScalarCSRRestoreInsn, getUnmanagedCSI(MF, CSI).size());
+  CFIBuilder.setInsertPoint(MBBI);
 
   if (getLibCallID(MF, CSI) != -1) {
     // tail __riscv_restore_[0-12] instruction is considered as a terminator,
@@ -1271,7 +1128,8 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 
   // Recover callee-saved registers.
-  emitCFIForCSI<CFIRestoreRegisterEmitter>(MBB, MBBI, getUnmanagedCSI(MF, CSI));
+  for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+    CFIBuilder.buildRestore(CS.getReg());
 
   if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(MBBI->getOpcode())) {
     // Use available stack adjustment in pop instruction to deallocate stack
@@ -1290,17 +1148,14 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
     auto NextI = next_nodbg(MBBI, MBB.end());
     if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
       ++MBBI;
+      CFIBuilder.setInsertPoint(MBBI);
 
-      emitCFIForCSI<CFIRestoreRegisterEmitter>(
-          MBB, MBBI, getPushOrLibCallsSavedInfo(MF, CSI));
+      for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+        CFIBuilder.buildRestore(CS.getReg());
 
       // Update CFA offset. After CM_POP SP should be equal to CFA, so CFA
       // offset should be a zero.
-      unsigned CFIIndex =
-          MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
-      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlag(MachineInstr::FrameDestroy);
+      CFIBuilder.buildDefCFAOffset(0);
     }
   }
 
@@ -2052,9 +1907,7 @@ void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
   MachineFunction *MF = MBB.getParent();
   const MachineFrameInfo &MFI = MF->getFrameInfo();
   RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
   const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
-  DebugLoc DL = MBB.findDebugLoc(MI);
 
   const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, MFI.getCalleeSavedInfo());
   if (RVVCSI.empty())
@@ -2068,17 +1921,15 @@ void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
     FixedSize -= ScalarLocalVarSize;
   }
 
+  CFIInstBuilder CFIBuilder(MBB, MI, MachineInstr::FrameSetup);
   for (auto &CS : RVVCSI) {
     // Insert the spill to the stack frame.
     int FI = CS.getFrameIdx();
     MCRegister BaseReg = getRVVBaseRegister(TRI, CS.getReg());
     unsigned NumRegs = getCalleeSavedRVVNumRegs(CS.getReg());
     for (unsigned i = 0; i < NumRegs; ++i) {
-      unsigned CFIIndex = MF->addFrameInst(createDefCFAOffset(
+      CFIBuilder.insertCFIInst(createDefCFAOffset(
           TRI, BaseReg + i, -FixedSize, MFI.getObjectOffset(FI) / 8 + i));
-      BuildMI(MBB, MI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlag(MachineInstr::FrameSetup);
     }
   }
 }
@@ -2087,22 +1938,15 @@ void RISCVFrameLowering::emitCalleeSavedRVVEpilogCFI(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const {
   MachineFunction *MF = MBB.getParent();
   const MachineFrameInfo &MFI = MF->getFrameInfo();
-  const RISCVRegisterInfo *RI = STI.getRegisterInfo();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
   const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
-  DebugLoc DL = MBB.findDebugLoc(MI);
 
+  CFIInstBuilder CFIHelper(MBB, MI, MachineInstr::FrameDestroy);
   const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, MFI.getCalleeSavedInfo());
   for (auto &CS : RVVCSI) {
     MCRegister BaseReg = getRVVBaseRegister(TRI, CS.getReg());
     unsigned NumRegs = getCalleeSavedRVVNumRegs(CS.getReg());
-    for (unsigned i = 0; i < NumRegs; ++i) {
-      unsigned CFIIndex = MF->addFrameInst(MCCFIInstruction::createRestore(
-          nullptr, RI->getDwarfRegNum(BaseReg + i, true)));
-      BuildMI(MBB, MI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlag(MachineInstr::FrameDestroy);
-    }
+    for (unsigned i = 0; i < NumRegs; ++i)
+      CFIHelper.buildRestore(BaseReg + i);
   }
 }
 

From 0daf20b3605f19271af7afa4175e7d62194e5578 Mon Sep 17 00:00:00 2001
From: James Newling <james.newling@gmail.com>
Date: Wed, 16 Apr 2025 10:08:36 -0700
Subject: [PATCH 145/710] [mlir][vector] transpose(broadcast) -> broadcast
 canonicalization (#135096)

Example seen in the 'real world':

 ```
 %0 = vector.broadcast %arg0 : vector<1xi8> to vector<1x8xi8>
 %1 = vector.transpose %0, [1, 0] : vector<1x8xi8> to vector<8x1xi8>
 ```

 This PR adds a canonicalizer that rewrites the above as

```
  %1 = vector.broadcast %arg0 : vector<1xi8> to vector<8x1xi8>
```

It works by determining if a transpose is only shuffling contiguous
broadcast dimensions.
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      | 120 ++++++++++++---
 mlir/test/Dialect/Vector/canonicalize.mlir    |  24 ---
 .../Vector/canonicalize/vector-transpose.mlir | 139 ++++++++++++++++++
 3 files changed, 235 insertions(+), 48 deletions(-)
 create mode 100644 mlir/test/Dialect/Vector/canonicalize/vector-transpose.mlir

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index bee5c1fd6ed58..504032a398fbe 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -6085,28 +6085,6 @@ class TransposeFolder final : public OpRewritePattern<vector::TransposeOp> {
   }
 };
 
-// Folds transpose(broadcast(<scalar>)) into broadcast(<scalar>).
-struct FoldTransposedScalarBroadcast final
-    : public OpRewritePattern<vector::TransposeOp> {
-  using OpRewritePattern::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(vector::TransposeOp transposeOp,
-                                PatternRewriter &rewriter) const override {
-    auto bcastOp = transposeOp.getVector().getDefiningOp<vector::BroadcastOp>();
-    if (!bcastOp)
-      return failure();
-
-    auto srcVectorType = llvm::dyn_cast<VectorType>(bcastOp.getSourceType());
-    if (!srcVectorType || srcVectorType.getNumElements() == 1) {
-      rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
-          transposeOp, transposeOp.getResultVectorType(), bcastOp.getSource());
-      return success();
-    }
-
-    return failure();
-  }
-};
-
 // Folds transpose(splat x : src_type) : res_type into splat x : res_type.
 class FoldTransposeSplat final : public OpRewritePattern<TransposeOp> {
 public:
@@ -6161,12 +6139,106 @@ class FoldTransposeCreateMask final : public OpRewritePattern<TransposeOp> {
   }
 };
 
+/// Folds transpose(broadcast(x)) to broadcast(x) if the transpose is
+/// 'order preserving', where 'order preserving' means the flattened
+/// inputs and outputs of the transpose have identical (numerical) values.
+///
+/// Example:
+/// ```
+///  %0 = vector.broadcast %input : vector<1x1xi32> to vector<1x8xi32>
+///  %1 = vector.transpose %0, [1, 0] : vector<1x8xi32>
+///                                                 to vector<8x1xi32>
+/// ```
+/// can be rewritten as the equivalent
+/// ```
+///  %0 = vector.broadcast %input : vector<1x1xi32> to vector<8x1xi32>.
+/// ```
+/// The algorithm works by partitioning dimensions into groups that can be
+/// locally permuted while preserving order, and checks that the transpose
+/// only permutes within these groups.
+///
+/// Groups are either contiguous sequences of 1s, or non-1s (1-element groups).
+/// Consider broadcasting 4x1x1x7 to 2x3x4x5x6x7. This is equivalent to
+/// broadcasting from 1x1x4x1x1x7.
+///                   ^^^ ^ ^^^ ^
+///          groups:   0  1  2  3
+/// Order preserving permutations for this example are ones that only permute
+/// within the groups [0,1] and [3,4], like (1 0 2 4 3 5 6).
+class FoldTransposeBroadcast : public OpRewritePattern<vector::TransposeOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  FoldTransposeBroadcast(MLIRContext *context, PatternBenefit benefit = 1)
+      : OpRewritePattern<vector::TransposeOp>(context, benefit) {}
+
+  LogicalResult matchAndRewrite(vector::TransposeOp transpose,
+                                PatternRewriter &rewriter) const override {
+
+    vector::BroadcastOp broadcast =
+        transpose.getVector().getDefiningOp<vector::BroadcastOp>();
+    if (!broadcast) {
+      return rewriter.notifyMatchFailure(transpose,
+                                         "not preceded by a broadcast");
+    }
+
+    auto inputType = dyn_cast<VectorType>(broadcast.getSourceType());
+    VectorType outputType = transpose.getResultVectorType();
+
+    // transpose(broadcast(scalar)) -> broadcast(scalar) is always valid
+    bool inputIsScalar = !inputType;
+    if (inputIsScalar) {
+      rewriter.replaceOpWithNewOp<vector::BroadcastOp>(transpose, outputType,
+                                                       transpose.getVector());
+      return success();
+    }
+
+    ArrayRef<int64_t> permutation = transpose.getPermutation();
+    ArrayRef<int64_t> inputShape = inputType.getShape();
+    int64_t inputRank = inputType.getRank();
+    int64_t outputRank = transpose.getType().getRank();
+    int64_t deltaRank = outputRank - inputRank;
+
+    int low = 0;
+    for (int inputIndex = 0; inputIndex < inputRank; ++inputIndex) {
+      bool notOne = inputShape[inputIndex] != 1;
+      bool prevNotOne = (inputIndex != 0 && inputShape[inputIndex - 1] != 1);
+      bool groupEndFound = notOne || prevNotOne;
+      if (groupEndFound) {
+        int high = inputIndex + deltaRank;
+        // Return failure if not all permutation destinations for indices in
+        // [low, high) are in [low, high), i.e. the permutation is not local to
+        // the group.
+        for (int i = low; i < high; ++i) {
+          if (permutation[i] < low || permutation[i] >= high) {
+            return rewriter.notifyMatchFailure(
+                transpose, "permutation not local to group");
+          }
+        }
+      }
+    }
+
+    // We don't need to check the final group [low, outputRank) because if it is
+    // not locally bound, there must be a preceding group that already failed
+    // the check (impossible to have just 1 non-locally bound group).
+
+    // The preceding logic also ensures that at this point, the output of the
+    // transpose is definitely broadcastable from the input shape, assert so:
+    assert(vector::isBroadcastableTo(inputType, outputType) ==
+               vector::BroadcastableToResult::Success &&
+           "not broadcastable directly to transpose output");
+
+    rewriter.replaceOpWithNewOp<vector::BroadcastOp>(transpose, outputType,
+                                                     transpose.getVector());
+
+    return success();
+  }
+};
+
 } // namespace
 
 void vector::TransposeOp::getCanonicalizationPatterns(
     RewritePatternSet &results, MLIRContext *context) {
-  results.add<FoldTransposeCreateMask, FoldTransposedScalarBroadcast,
-              TransposeFolder, FoldTransposeSplat>(context);
+  results.add<FoldTransposeCreateMask, TransposeFolder, FoldTransposeSplat,
+              FoldTransposeBroadcast>(context);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 78b0ea78849e8..733a2c67d2c0c 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -2218,30 +2218,6 @@ func.func @shuffle_nofold1(%v0 : vector<4xi32>, %v1 : vector<2xi32>) -> vector<5
 
 // -----
 
-// CHECK-LABEL: func @transpose_scalar_broadcast1
-//  CHECK-SAME: (%[[ARG:.+]]: vector<1xf32>)
-//       CHECK:   %[[V:.+]] = vector.broadcast %[[ARG]] : vector<1xf32> to vector<1x8xf32>
-//       CHECK:   return %[[V]] : vector<1x8xf32>
-func.func @transpose_scalar_broadcast1(%value: vector<1xf32>) -> vector<1x8xf32> {
-  %bcast = vector.broadcast %value : vector<1xf32> to vector<8x1xf32>
-  %t = vector.transpose %bcast, [1, 0] : vector<8x1xf32> to vector<1x8xf32>
-  return %t : vector<1x8xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_scalar_broadcast2
-//  CHECK-SAME: (%[[ARG:.+]]: f32)
-//       CHECK:   %[[V:.+]] = vector.broadcast %[[ARG]] : f32 to vector<1x8xf32>
-//       CHECK:   return %[[V]] : vector<1x8xf32>
-func.func @transpose_scalar_broadcast2(%value: f32) -> vector<1x8xf32> {
-  %bcast = vector.broadcast %value : f32 to vector<8x1xf32>
-  %t = vector.transpose %bcast, [1, 0] : vector<8x1xf32> to vector<1x8xf32>
-  return %t : vector<1x8xf32>
-}
-
-// -----
-
 // CHECK-LABEL: func @transpose_splat_constant
 //       CHECK:   %[[CST:.+]] = arith.constant dense<5.000000e+00> : vector<8x4xf32>
 //       CHECK:   return %[[CST]]
diff --git a/mlir/test/Dialect/Vector/canonicalize/vector-transpose.mlir b/mlir/test/Dialect/Vector/canonicalize/vector-transpose.mlir
new file mode 100644
index 0000000000000..e97e147459de2
--- /dev/null
+++ b/mlir/test/Dialect/Vector/canonicalize/vector-transpose.mlir
@@ -0,0 +1,139 @@
+// RUN: mlir-opt %s -canonicalize="test-convergence" -split-input-file -allow-unregistered-dialect | FileCheck %s
+
+// This file contains some canonicalizations tests involving vector.transpose.
+
+// CHECK-LABEL: func @transpose_scalar_broadcast1
+//  CHECK-SAME: (%[[ARG:.+]]: vector<1xf32>)
+//       CHECK:   %[[V:.+]] = vector.broadcast %[[ARG]] : vector<1xf32> to vector<1x8xf32>
+//       CHECK:   return %[[V]] : vector<1x8xf32>
+func.func @transpose_scalar_broadcast1(%value: vector<1xf32>) -> vector<1x8xf32> {
+  %bcast = vector.broadcast %value : vector<1xf32> to vector<8x1xf32>
+  %t = vector.transpose %bcast, [1, 0] : vector<8x1xf32> to vector<1x8xf32>
+  return %t : vector<1x8xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_scalar_broadcast2
+//  CHECK-SAME: (%[[ARG:.+]]: f32)
+//       CHECK:   %[[V:.+]] = vector.broadcast %[[ARG]] : f32 to vector<1x8xf32>
+//       CHECK:   return %[[V]] : vector<1x8xf32>
+func.func @transpose_scalar_broadcast2(%value: f32) -> vector<1x8xf32> {
+  %bcast = vector.broadcast %value : f32 to vector<8x1xf32>
+  %t = vector.transpose %bcast, [1, 0] : vector<8x1xf32> to vector<1x8xf32>
+  return %t : vector<1x8xf32>
+}
+
+// -----
+
+
+// CHECK-LABEL: broadcast_transpose_scalar_to_broadcast
+//  CHECK-SAME:  %[[ARG:.*]]: i8) -> vector<2x3x4xi8> {
+func.func @broadcast_transpose_scalar_to_broadcast(%arg0 : i8) -> vector<2x3x4xi8> {
+//       CHECK:  %[[BC:.*]] = vector.broadcast %[[ARG]] : i8 to vector<2x3x4xi8>
+  %0 = vector.broadcast %arg0 : i8 to vector<3x4x2xi8>
+  %1 = vector.transpose %0, [2, 0, 1] : vector<3x4x2xi8> to vector<2x3x4xi8>
+//       CHECK:  return %[[BC]] : vector<2x3x4xi8>
+  return %1 : vector<2x3x4xi8>
+}
+
+// -----
+
+// CHECK-LABEL: broadcast_transpose_ones_to_broadcast
+//  CHECK-SAME:  %[[ARG:.*]]: vector<1x1x1xi8>) -> vector<2x3x4xi8> {
+//       CHECK:  %[[RES:.*]] = vector.broadcast %[[ARG]] : vector<1x1x1xi8> to vector<2x3x4xi8>
+//       CHECK:  return %[[RES]] : vector<2x3x4xi8>
+func.func @broadcast_transpose_ones_to_broadcast(%arg0 : vector<1x1x1xi8>) -> vector<2x3x4xi8> {
+  %0 = vector.broadcast %arg0 : vector<1x1x1xi8> to vector<3x4x2xi8>
+  %1 = vector.transpose %0, [2, 0, 1] : vector<3x4x2xi8> to vector<2x3x4xi8>
+  return %1 : vector<2x3x4xi8>
+}
+
+// -----
+
+// CHECK-LABEL: broadcast_transpose_partial_ones_to_broadcast
+//  CHECK-SAME:  %[[ARG:.*]]: vector<1xi8>) -> vector<8x1xi8> {
+//       CHECK:  %[[RES:.*]] = vector.broadcast %[[ARG]] : vector<1xi8> to vector<8x1xi8>
+//       CHECK:  return %[[RES]] : vector<8x1xi8>
+func.func @broadcast_transpose_partial_ones_to_broadcast(%arg0 : vector<1xi8>) -> vector<8x1xi8> {
+  %0 = vector.broadcast %arg0 : vector<1xi8> to vector<1x8xi8>
+  %1 = vector.transpose %0, [1, 0] : vector<1x8xi8> to vector<8x1xi8>
+  return %1 : vector<8x1xi8>
+}
+
+// -----
+
+// CHECK-LABEL: broadcast_transpose_mixed_example
+//  CHECK-SAME:  %[[ARG:.*]]: vector<4x1x1x7xi8>) -> vector<3x2x4x5x6x7xi8> {
+//       CHECK:  %[[RES:.*]] = vector.broadcast %[[ARG]] : vector<4x1x1x7xi8> to vector<3x2x4x5x6x7xi8>
+//       CHECK:  return %[[RES]] : vector<3x2x4x5x6x7xi8>
+func.func @broadcast_transpose_mixed_example(%arg0 : vector<4x1x1x7xi8>) -> vector<3x2x4x5x6x7xi8> {
+  %0 = vector.broadcast %arg0 : vector<4x1x1x7xi8> to vector<2x3x4x5x6x7xi8>
+  %1 = vector.transpose %0, [1, 0, 2, 3, 4, 5] : vector<2x3x4x5x6x7xi8> to vector<3x2x4x5x6x7xi8>
+  return %1 : vector<3x2x4x5x6x7xi8>
+}
+
+// -----
+
+// CHECK-LABEL: broadcast_transpose_final_group
+//  CHECK-SAME:  %[[ARG:.*]]: vector<4x7x1x1xi8>) -> vector<4x7x2x3xi8> {
+//       CHECK:  %[[RES:.*]] = vector.broadcast %[[ARG]] : vector<4x7x1x1xi8> to vector<4x7x2x3xi8>
+//       CHECK:  return %[[RES]] : vector<4x7x2x3xi8>
+func.func @broadcast_transpose_final_group(%arg0 : vector<4x7x1x1xi8>) -> vector<4x7x2x3xi8> {
+  %0 = vector.broadcast %arg0 : vector<4x7x1x1xi8> to vector<4x7x3x2xi8>
+  %1 = vector.transpose %0, [0, 1, 3, 2] : vector<4x7x3x2xi8> to vector<4x7x2x3xi8>
+  return %1 : vector<4x7x2x3xi8>
+}
+
+// -----
+
+// CHECK-LABEL: negative_broadcast_transpose_square
+//  CHECK-SAME:  %[[ARG:.*]]:
+//       CHECK:  %[[BCT:.*]] = vector.broadcast %[[ARG]]
+//       CHECK:  %[[TRP:.*]] = vector.transpose %[[BCT]], [1, 0]
+//       CHECK:  return %[[TRP]] : vector<4x4xi8>
+func.func @negative_broadcast_transpose_square(%arg0 : vector<4x1xi8>) -> vector<4x4xi8> {
+  %0 = vector.broadcast %arg0 : vector<4x1xi8> to vector<4x4xi8>
+  %1 = vector.transpose %0, [1, 0] : vector<4x4xi8> to vector<4x4xi8>
+  return %1 : vector<4x4xi8>
+}
+
+// -----
+
+// CHECK-LABEL: negative_broadcast_transpose_hypercube
+//  CHECK-SAME:  %[[ARG:.*]]:
+//       CHECK:  %[[BCT:.*]] = vector.broadcast %[[ARG]]
+//       CHECK:  %[[TRP:.*]] = vector.transpose %[[BCT]], [1, 0, 3, 2]
+//       CHECK:  return %[[TRP]] : vector<4x4x4x4xi8>
+func.func @negative_broadcast_transpose_hypercube(%arg0 : vector<1x1x4xi8>) -> vector<4x4x4x4xi8> {
+  %0 = vector.broadcast %arg0 : vector<1x1x4xi8> to vector<4x4x4x4xi8>
+  %1 = vector.transpose %0, [1, 0, 3, 2] : vector<4x4x4x4xi8> to vector<4x4x4x4xi8>
+  return %1 : vector<4x4x4x4xi8>
+}
+
+// -----
+
+// CHECK-LABEL: negative_broadcast_transpose_102
+//  CHECK-SAME:  %[[ARG:.*]]:
+//       CHECK:  %[[BCT:.*]] = vector.broadcast %[[ARG]]
+//       CHECK:  %[[TRP:.*]] = vector.transpose %[[BCT]], [1, 0, 2]
+//       CHECK:  return %[[TRP]] : vector<3x3x3xi8>
+func.func @negative_broadcast_transpose_102(%arg0 : vector<3x1x3xi8>) -> vector<3x3x3xi8> {
+  %0 = vector.broadcast %arg0 : vector<3x1x3xi8> to vector<3x3x3xi8>
+  %1 = vector.transpose %0, [1, 0, 2] : vector<3x3x3xi8> to vector<3x3x3xi8>
+  return %1 : vector<3x3x3xi8>
+}
+
+// -----
+
+// CHECK-LABEL: negative_broadcast_transpose_021
+//  CHECK-SAME:  %[[ARG:.*]]:
+//       CHECK:  %[[BCT:.*]] = vector.broadcast %[[ARG]]
+//       CHECK:  %[[TRP:.*]] = vector.transpose %[[BCT]], [0, 2, 1]
+//       CHECK:  return %[[TRP]] : vector<3x3x3xi8>
+func.func @negative_broadcast_transpose_021(%arg0 : vector<3x1x3xi8>) -> vector<3x3x3xi8> {
+  %0 = vector.broadcast %arg0 : vector<3x1x3xi8> to vector<3x3x3xi8>
+  %1 = vector.transpose %0, [0, 2, 1] : vector<3x3x3xi8> to vector<3x3x3xi8>
+  return %1 : vector<3x3x3xi8>
+}
+

From d88a3a36ad26e68281873fab9a35389f6eb5c919 Mon Sep 17 00:00:00 2001
From: James Newling <james.newling@gmail.com>
Date: Wed, 16 Apr 2025 10:08:57 -0700
Subject: [PATCH 146/710] [mlir][vector] Remove redundant
 shape_cast(shape_cast(x)) pattern (#135447)

This PR removes one OpRewritePattern `shape_cast(shape_cast(x)) -> x`
that is already handled by `ShapeCastOp::fold`.

Note that this might affect downstream users who indirectly call
`populateShapeCastFoldingPatterns(RewritePatternSet &patterns,
PatternBenefit)` and then use `patterns` with a `GreedyRewriteConfig
config` that has `config.fold = false`. (only user I've checked is IREE,
that never uses config.fold = false).
---
 .../Vector/Transforms/VectorRewritePatterns.h |  4 --
 .../Transforms/VectorDropLeadUnitDim.cpp      |  2 -
 .../Transforms/VectorTransferOpTransforms.cpp |  2 -
 .../Vector/Transforms/VectorTransforms.cpp    | 68 +------------------
 4 files changed, 1 insertion(+), 75 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
index 7de4a6a315750..ce97847172197 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
@@ -306,10 +306,6 @@ void populateVectorUnrollPatterns(RewritePatternSet &patterns,
                                   const UnrollVectorOptions &options,
                                   PatternBenefit benefit = 1);
 
-/// Collect a set of vector.shape_cast folding patterns.
-void populateShapeCastFoldingPatterns(RewritePatternSet &patterns,
-                                      PatternBenefit benefit = 1);
-
 /// Collect a set of leading one dimension removal patterns.
 ///
 /// These patterns insert vector.shape_cast to remove leading one dimensions
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
index fda3baf3aa390..68a44ea889470 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp
@@ -8,7 +8,6 @@
 
 #include <numeric>
 
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
@@ -577,5 +576,4 @@ void mlir::vector::populateCastAwayVectorLeadingOneDimPatterns(
            CastAwayConstantMaskLeadingOneDim, CastAwayTransferReadLeadingOneDim,
            CastAwayTransferWriteLeadingOneDim, CastAwayElementwiseLeadingOneDim,
            CastAwayContractionLeadingOneDim>(patterns.getContext(), benefit);
-  populateShapeCastFoldingPatterns(patterns, benefit);
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 62dfd439b0ad1..999fb9c415886 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -976,7 +976,6 @@ void mlir::vector::populateVectorTransferDropUnitDimsPatterns(
   patterns
       .add<TransferReadDropUnitDimsPattern, TransferWriteDropUnitDimsPattern>(
           patterns.getContext(), benefit);
-  populateShapeCastFoldingPatterns(patterns);
 }
 
 void mlir::vector::populateFlattenVectorTransferPatterns(
@@ -985,6 +984,5 @@ void mlir::vector::populateFlattenVectorTransferPatterns(
   patterns.add<FlattenContiguousRowMajorTransferReadPattern,
                FlattenContiguousRowMajorTransferWritePattern>(
       patterns.getContext(), targetVectorBitwidth, benefit);
-  populateShapeCastFoldingPatterns(patterns, benefit);
   populateDropUnitDimWithShapeCastPatterns(patterns, benefit);
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index d50d5fe96f49a..89839d0440d3c 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -16,36 +16,24 @@
 #include <cstdint>
 #include <functional>
 #include <optional>
-#include <type_traits>
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
-#include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
-#include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "mlir/Interfaces/VectorInterfaces.h"
 
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
-#include "llvm/Support/raw_ostream.h"
 
 #define DEBUG_TYPE "vector-to-vector"
 
@@ -71,54 +59,6 @@ static std::optional<int64_t> getResultIndex(AffineMap map, int64_t index) {
 
 namespace {
 
-/// ShapeCastOpFolder folds cancelling ShapeCastOps away.
-//
-// Example:
-//
-//  The following MLIR with cancelling ShapeCastOps:
-//
-//   %0 = source : vector<5x4x2xf32>
-//   %1 = shape_cast %0 : vector<5x4x2xf32> to vector<20x2xf32>
-//   %2 = shape_cast %1 : vector<20x2xf32> to vector<5x4x2xf32>
-//   %3 = user %2 : vector<5x4x2xf32>
-//
-//  Should canonicalize to the following:
-//
-//   %0 = source : vector<5x4x2xf32>
-//   %1 = user %0 : vector<5x4x2xf32>
-//
-struct ShapeCastOpFolder : public OpRewritePattern<vector::ShapeCastOp> {
-  using OpRewritePattern::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(vector::ShapeCastOp shapeCastOp,
-                                PatternRewriter &rewriter) const override {
-    // Check if 'shapeCastOp' has vector source/result type.
-    auto sourceVectorType =
-        dyn_cast_or_null<VectorType>(shapeCastOp.getSource().getType());
-    auto resultVectorType =
-        dyn_cast_or_null<VectorType>(shapeCastOp.getResult().getType());
-    if (!sourceVectorType || !resultVectorType)
-      return failure();
-
-    // Check if shape cast op source operand is also a shape cast op.
-    auto sourceShapeCastOp = dyn_cast_or_null<vector::ShapeCastOp>(
-        shapeCastOp.getSource().getDefiningOp());
-    if (!sourceShapeCastOp)
-      return failure();
-    auto operandSourceVectorType =
-        cast<VectorType>(sourceShapeCastOp.getSource().getType());
-    auto operandResultVectorType = sourceShapeCastOp.getType();
-
-    // Check if shape cast operations invert each other.
-    if (operandSourceVectorType != resultVectorType ||
-        operandResultVectorType != sourceVectorType)
-      return failure();
-
-    rewriter.replaceOp(shapeCastOp, sourceShapeCastOp.getSource());
-    return success();
-  }
-};
-
 /// Convert MulIOp/MulFOp + MultiDimReductionOp<add> into ContractionOp.
 /// Ex:
 /// ```
@@ -2113,11 +2053,6 @@ void mlir::vector::populateVectorMaskMaterializationPatterns(
   patterns.add<FoldI1Select>(patterns.getContext(), benefit);
 }
 
-void mlir::vector::populateShapeCastFoldingPatterns(RewritePatternSet &patterns,
-                                                    PatternBenefit benefit) {
-  patterns.add<ShapeCastOpFolder>(patterns.getContext(), benefit);
-}
-
 void mlir::vector::populateDropUnitDimWithShapeCastPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
   // TODO: Consider either:
@@ -2126,8 +2061,7 @@ void mlir::vector::populateDropUnitDimWithShapeCastPatterns(
   //  * better naming to distinguish this and
   //    populateVectorTransferCollapseInnerMostContiguousDimsPatterns.
   patterns.add<DropUnitDimFromElementwiseOps, DropUnitDimsFromScfForOp,
-               DropUnitDimsFromTransposeOp, ShapeCastOpFolder>(
-      patterns.getContext(), benefit);
+               DropUnitDimsFromTransposeOp>(patterns.getContext(), benefit);
 }
 
 void mlir::vector::populateBubbleVectorBitCastOpPatterns(

From facc57fc25d0f05f5834fed421662dbad3ec5b50 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Wed, 16 Apr 2025 19:09:45 +0200
Subject: [PATCH 147/710] [Clang][RFC] Bypass TAD during overload resolution if
 a perfect match exists (#133426)

This implements the same overload resolution behavior as GCC,
as described in https://wg21.link/p3606 (sections 1-2, not 3)

If, during overload resolution, a non-template candidate is always
picked because each argument is a perfect match (i.e., the source and
target types are the same), we do not perform deduction for any template
candidate that might exist.

The goal is to be able to merge #122423 without being too disruptive.

This change means that the selection of the best viable candidate and
template argument deduction become interleaved.

To avoid rewriting half of Clang, we store in `OverloadCandidateSet`
enough information to deduce template candidates from
`OverloadCandidateSet::BestViableFunction`. This means the lifetime of
any object used by the template argument must outlive a call to
`Add*Template*Candidate`.

This two-phase resolution is not performed for some initialization as
there are cases where template candidates are a better match per the
standard. It's also bypassed for code completion.

The change has a nice impact on compile times

https://llvm-compile-time-tracker.com/compare.php?from=edc22c64e527171041876f26a491bb1d03d905d5&to=8170b860bd4b70917005796c05a9be013a95abb2&stat=instructions%3Au

Fixes #62096
Fixes #74581
Fixes #53454
---
 clang/docs/ReleaseNotes.rst                   |   6 +
 clang/include/clang/Sema/Overload.h           | 224 +++++++-
 clang/lib/Sema/SemaCodeComplete.cpp           |   6 +-
 clang/lib/Sema/SemaInit.cpp                   |  15 +-
 clang/lib/Sema/SemaOverload.cpp               | 522 ++++++++++++++----
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   4 +-
 .../constrant-satisfaction-conversions.cpp    |   8 +-
 clang/test/SemaCUDA/function-overload.cu      |   3 -
 .../SemaCXX/implicit-member-functions.cpp     |  21 +-
 ...overload-resolution-deferred-templates.cpp | 185 +++++++
 .../instantiate-function-params.cpp           |   7 +-
 .../Templight/templight-empty-entries-fix.cpp | 126 ++---
 12 files changed, 910 insertions(+), 217 deletions(-)
 create mode 100644 clang/test/SemaCXX/overload-resolution-deferred-templates.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0891fd058bb57..acbc9c5a6fac9 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -96,6 +96,12 @@ C++ Language Changes
       asm((std::string_view("nop")) ::: (std::string_view("memory")));
     }
 
+- Clang now implements the changes to overload resolution proposed by section 1 and 2 of
+  `P3606 <https://wg21.link/P3606R0>`_. If a non-template candidate exists in an overload set that is
+  a perfect match (all conversion sequences are identity conversions) template candiates are not instantiated.
+  Diagnostics that would have resulted from the instantiation of these template candidates are no longer
+  produced. This aligns Clang closer to the behavior of GCC, and fixes (#GH62096), (#GH74581), and (#GH74581).
+
 C++2c Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 6e08762dcc6d7..813811af06e89 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -407,6 +407,24 @@ class Sema;
              Third == ICK_Identity;
     }
 
+    /// A conversion sequence is perfect if it is an identity conversion and
+    /// the type of the source is the same as the type of the target.
+    bool isPerfect(const ASTContext &C) const {
+      if (!isIdentityConversion())
+        return false;
+      // If we are not performing a reference binding, we can skip comparing
+      // the types, which has a noticeable performance impact.
+      if (!ReferenceBinding) {
+        assert(First || C.hasSameUnqualifiedType(getFromType(), getToType(2)));
+        return true;
+      }
+      if (!C.hasSameType(getFromType(), getToType(2)))
+        return false;
+      if (BindsToRvalue && IsLvalueReference)
+        return false;
+      return true;
+    }
+
     ImplicitConversionRank getRank() const;
     NarrowingKind
     getNarrowingKind(ASTContext &Context, const Expr *Converted,
@@ -743,6 +761,12 @@ class Sema;
       Standard.setAllToTypes(T);
     }
 
+    /// A conversion sequence is perfect if it is an identity conversion and
+    /// the type of the source is the same as the type of the target.
+    bool isPerfect(const ASTContext &C) const {
+      return isStandard() && Standard.isPerfect(C);
+    }
+
     // True iff this is a conversion sequence from an initializer list to an
     // array or std::initializer.
     bool hasInitializerListContainerType() const {
@@ -979,6 +1003,20 @@ class Sema;
       return false;
     }
 
+    // An overload is a perfect match if the conversion
+    // sequences for each argument are perfect.
+    bool isPerfectMatch(const ASTContext &Ctx) const {
+      if (!Viable)
+        return false;
+      for (const auto &C : Conversions) {
+        if (!C.isInitialized() || !C.isPerfect(Ctx))
+          return false;
+      }
+      if (isa_and_nonnull<CXXConversionDecl>(Function))
+        return FinalConversion.isPerfect(Ctx);
+      return true;
+    }
+
     bool TryToFixBadConversion(unsigned Idx, Sema &S) {
       bool CanFix = Fix.tryToFixConversion(
                       Conversions[Idx].Bad.FromExpr,
@@ -1015,6 +1053,65 @@ class Sema;
           RewriteKind(CRK_None) {}
   };
 
+  struct DeferredTemplateOverloadCandidate {
+
+    // intrusive linked list support for allocateDeferredCandidate
+    DeferredTemplateOverloadCandidate *Next = nullptr;
+
+    enum Kind { Function, Method, Conversion };
+
+    LLVM_PREFERRED_TYPE(Kind)
+    unsigned Kind : 2;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowObjCConversionOnExplicit : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowResultConversion : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowExplicit : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned SuppressUserConversions : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned PartialOverloading : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AggregateCandidateDeduction : 1;
+  };
+
+  struct DeferredFunctionTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    ArrayRef<Expr *> Args;
+    CallExpr::ADLCallKind IsADLCandidate;
+    OverloadCandidateParamOrder PO;
+  };
+  static_assert(std::is_trivially_destructible_v<
+                DeferredFunctionTemplateOverloadCandidate>);
+
+  struct DeferredMethodTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    ArrayRef<Expr *> Args;
+    CXXRecordDecl *ActingContext;
+    Expr::Classification ObjectClassification;
+    QualType ObjectType;
+    OverloadCandidateParamOrder PO;
+  };
+  static_assert(std::is_trivially_destructible_v<
+                DeferredMethodTemplateOverloadCandidate>);
+
+  struct DeferredConversionTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    CXXRecordDecl *ActingContext;
+    Expr *From;
+    QualType ToType;
+  };
+
+  static_assert(std::is_trivially_destructible_v<
+                DeferredConversionTemplateOverloadCandidate>);
+
   /// OverloadCandidateSet - A set of overload candidates, used in C++
   /// overload resolution (C++ 13.3).
   class OverloadCandidateSet {
@@ -1043,6 +1140,11 @@ class Sema;
       /// C++ [over.match.call.general]
       /// Resolve a call through the address of an overload set.
       CSK_AddressOfOverloadSet,
+
+      /// When doing overload resolution during code completion,
+      /// we want to show all viable candidates, including otherwise
+      /// deferred template candidates.
+      CSK_CodeCompletion,
     };
 
     /// Information about operator rewrites to consider when adding operator
@@ -1117,7 +1219,15 @@ class Sema;
     SmallVector<OverloadCandidate, 16> Candidates;
     llvm::SmallPtrSet<uintptr_t, 16> Functions;
 
-    // Allocator for ConversionSequenceLists. We store the first few of these
+    DeferredTemplateOverloadCandidate *FirstDeferredCandidate = nullptr;
+    unsigned DeferredCandidatesCount : 8 * sizeof(unsigned) - 2;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasDeferredTemplateConstructors : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned ResolutionByPerfectCandidateIsDisabled : 1;
+
+    // Allocator for ConversionSequenceLists and deferred candidate args.
+    // We store the first few of these
     // inline to avoid allocation for small sets.
     llvm::BumpPtrAllocator SlabAllocator;
 
@@ -1125,8 +1235,11 @@ class Sema;
     CandidateSetKind Kind;
     OperatorRewriteInfo RewriteInfo;
 
+    /// Small storage size for ImplicitConversionSequences
+    /// and the persisted arguments of deferred candidates.
     constexpr static unsigned NumInlineBytes =
-        24 * sizeof(ImplicitConversionSequence);
+        32 * sizeof(ImplicitConversionSequence);
+
     unsigned NumInlineBytesUsed = 0;
     alignas(void *) char InlineSpace[NumInlineBytes];
 
@@ -1137,15 +1250,13 @@ class Sema;
     /// from the slab allocator.
     /// FIXME: It would probably be nice to have a SmallBumpPtrAllocator
     /// instead.
-    /// FIXME: Now that this only allocates ImplicitConversionSequences, do we
-    /// want to un-generalize this?
     template <typename T>
     T *slabAllocate(unsigned N) {
       // It's simpler if this doesn't need to consider alignment.
       static_assert(alignof(T) == alignof(void *),
                     "Only works for pointer-aligned types.");
-      static_assert(std::is_trivial<T>::value ||
-                        std::is_same<ImplicitConversionSequence, T>::value,
+      static_assert(std::is_trivially_destructible_v<T> ||
+                        (std::is_same_v<ImplicitConversionSequence, T>),
                     "Add destruction logic to OverloadCandidateSet::clear().");
 
       unsigned NBytes = sizeof(T) * N;
@@ -1159,12 +1270,34 @@ class Sema;
       return reinterpret_cast<T *>(FreeSpaceStart);
     }
 
+    // Because the size of OverloadCandidateSet has a noticeable impact on
+    // performance, we store each deferred template candidate in the slab
+    // allocator such that deferred candidates are ultimately a singly-linked
+    // intrusive linked list. This ends up being much more efficient than a
+    // SmallVector that is empty in the common case.
+    template <typename T> T *allocateDeferredCandidate() {
+      T *C = slabAllocate<T>(1);
+      if (!FirstDeferredCandidate)
+        FirstDeferredCandidate = C;
+      else {
+        auto *F = FirstDeferredCandidate;
+        while (F->Next)
+          F = F->Next;
+        F->Next = C;
+      }
+      DeferredCandidatesCount++;
+      return C;
+    }
+
     void destroyCandidates();
 
   public:
     OverloadCandidateSet(SourceLocation Loc, CandidateSetKind CSK,
                          OperatorRewriteInfo RewriteInfo = {})
-        : Loc(Loc), Kind(CSK), RewriteInfo(RewriteInfo) {}
+        : FirstDeferredCandidate(nullptr), DeferredCandidatesCount(0),
+          HasDeferredTemplateConstructors(false),
+          ResolutionByPerfectCandidateIsDisabled(false), Loc(Loc), Kind(CSK),
+          RewriteInfo(RewriteInfo) {}
     OverloadCandidateSet(const OverloadCandidateSet &) = delete;
     OverloadCandidateSet &operator=(const OverloadCandidateSet &) = delete;
     ~OverloadCandidateSet() { destroyCandidates(); }
@@ -1176,6 +1309,9 @@ class Sema;
     /// Whether diagnostics should be deferred.
     bool shouldDeferDiags(Sema &S, ArrayRef<Expr *> Args, SourceLocation OpLoc);
 
+    // Whether the resolution of template candidates should be deferred
+    bool shouldDeferTemplateArgumentDeduction(const LangOptions &Opts) const;
+
     /// Determine when this overload candidate will be new to the
     /// overload set.
     bool isNewCandidate(Decl *F, OverloadCandidateParamOrder PO =
@@ -1199,8 +1335,10 @@ class Sema;
     iterator begin() { return Candidates.begin(); }
     iterator end() { return Candidates.end(); }
 
-    size_t size() const { return Candidates.size(); }
-    bool empty() const { return Candidates.empty(); }
+    size_t size() const { return Candidates.size() + DeferredCandidatesCount; }
+    bool empty() const {
+      return Candidates.empty() && DeferredCandidatesCount == 0;
+    }
 
     /// Allocate storage for conversion sequences for NumConversions
     /// conversions.
@@ -1216,6 +1354,24 @@ class Sema;
       return ConversionSequenceList(Conversions, NumConversions);
     }
 
+    /// Provide storage for any Expr* arg that must be preserved
+    /// until deferred template candidates are deduced.
+    /// Typically this should be used for reversed operator arguments
+    /// and any time the argument array is transformed while adding
+    /// a template candidate.
+    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(unsigned N) {
+      Expr **Exprs = slabAllocate<Expr *>(N);
+      return llvm::MutableArrayRef<Expr *>(Exprs, N);
+    }
+
+    template <typename... T>
+    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(T *...Exprs) {
+      llvm::MutableArrayRef<Expr *> Arr =
+          getPersistentArgsArray(sizeof...(Exprs));
+      llvm::copy(std::initializer_list<Expr *>{Exprs...}, Arr.data());
+      return Arr;
+    }
+
     /// Add a new candidate with NumConversions conversion sequence slots
     /// to the overload set.
     OverloadCandidate &addCandidate(unsigned NumConversions = 0,
@@ -1231,6 +1387,32 @@ class Sema;
       return C;
     }
 
+    void AddDeferredTemplateCandidate(
+        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+        ArrayRef<Expr *> Args, bool SuppressUserConversions,
+        bool PartialOverloading, bool AllowExplicit,
+        CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+        bool AggregateCandidateDeduction);
+
+    void AddDeferredMethodTemplateCandidate(
+        FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+        CXXRecordDecl *ActingContext, QualType ObjectType,
+        Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+        bool SuppressUserConversions, bool PartialOverloading,
+        OverloadCandidateParamOrder PO);
+
+    void AddDeferredConversionTemplateCandidate(
+        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+        CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+        bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+        bool AllowResultConversion);
+
+    void InjectNonDeducedTemplateCandidates(Sema &S);
+
+    void DisableResolutionByPerfectCandidate() {
+      ResolutionByPerfectCandidateIsDisabled = true;
+    }
+
     /// Find the best viable function on this overload set, if it exists.
     OverloadingResult BestViableFunction(Sema &S, SourceLocation Loc,
                                          OverloadCandidateSet::iterator& Best);
@@ -1263,6 +1445,15 @@ class Sema;
       DestAS = AS;
     }
 
+  private:
+    OverloadingResult ResultForBestCandidate(const iterator &Best);
+    void CudaExcludeWrongSideCandidates(
+        Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates);
+    OverloadingResult
+    BestViableFunctionImpl(Sema &S, SourceLocation Loc,
+                           OverloadCandidateSet::iterator &Best);
+    void PerfectViableFunction(Sema &S, SourceLocation Loc,
+                               OverloadCandidateSet::iterator &Best);
   };
 
   bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
@@ -1311,6 +1502,21 @@ class Sema;
   // parameter.
   bool shouldEnforceArgLimit(bool PartialOverloading, FunctionDecl *Function);
 
+  inline bool OverloadCandidateSet::shouldDeferTemplateArgumentDeduction(
+      const LangOptions &Opts) const {
+    return
+        // For user defined conversion we need to check against different
+        // combination of CV qualifiers and look at any explicit specifier, so
+        // always deduce template candidates.
+        Kind != CSK_InitByUserDefinedConversion
+        // When doing code completion, we want to see all the
+        // viable candidates.
+        && Kind != CSK_CodeCompletion
+        // CUDA may prefer template candidates even when a non-candidate
+        // is a perfect match
+        && !Opts.CUDA;
+  }
+
 } // namespace clang
 
 #endif // LLVM_CLANG_SEMA_OVERLOAD_H
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index f6ec4cb0f069e..45405d4709e14 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -6354,7 +6354,8 @@ SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
   Expr *NakedFn = Fn->IgnoreParenCasts();
   // Build an overload candidate set based on the functions we find.
   SourceLocation Loc = Fn->getExprLoc();
-  OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
+  OverloadCandidateSet CandidateSet(Loc,
+                                    OverloadCandidateSet::CSK_CodeCompletion);
 
   if (auto ULE = dyn_cast<UnresolvedLookupExpr>(NakedFn)) {
     SemaRef.AddOverloadedCallCandidates(ULE, ArgsWithoutDependentTypes,
@@ -6557,7 +6558,8 @@ QualType SemaCodeCompletion::ProduceConstructorSignatureHelp(
   // FIXME: Provide support for variadic template constructors.
 
   if (CRD) {
-    OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
+    OverloadCandidateSet CandidateSet(Loc,
+                                      OverloadCandidateSet::CSK_CodeCompletion);
     for (NamedDecl *C : SemaRef.LookupConstructors(CRD)) {
       if (auto *FD = dyn_cast<FunctionDecl>(C)) {
         // FIXME: we can't yet provide correct signature help for initializer
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index a1e4bb4321d53..82489847b589b 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -10029,12 +10029,19 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     //   When [...] the constructor [...] is a candidate by
     //    - [over.match.copy] (in all cases)
     if (TD) {
-      SmallVector<Expr *, 8> TmpInits;
-      for (Expr *E : Inits)
+
+      // As template candidates are not deduced immediately,
+      // persist the array in the overload set.
+      MutableArrayRef<Expr *> TmpInits =
+          Candidates.getPersistentArgsArray(Inits.size());
+
+      for (auto [I, E] : llvm::enumerate(Inits)) {
         if (auto *DI = dyn_cast<DesignatedInitExpr>(E))
-          TmpInits.push_back(DI->getInit());
+          TmpInits[I] = DI->getInit();
         else
-          TmpInits.push_back(E);
+          TmpInits[I] = E;
+      }
+
       AddTemplateOverloadCandidate(
           TD, FoundDecl, /*ExplicitArgs=*/nullptr, TmpInits, Candidates,
           /*SuppressUserConversions=*/false,
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 55634aa75ae25..deef01c946feb 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1123,6 +1123,10 @@ void OverloadCandidateSet::clear(CandidateSetKind CSK) {
   Candidates.clear();
   Functions.clear();
   Kind = CSK;
+  FirstDeferredCandidate = nullptr;
+  DeferredCandidatesCount = 0;
+  HasDeferredTemplateConstructors = false;
+  ResolutionByPerfectCandidateIsDisabled = false;
 }
 
 namespace {
@@ -7795,15 +7799,14 @@ void Sema::AddMethodCandidate(
   }
 }
 
-void Sema::AddMethodTemplateCandidate(
+static void AddMethodTemplateCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
     CXXRecordDecl *ActingContext,
     TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
     Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, OverloadCandidateParamOrder PO) {
-  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
-    return;
+    bool SuppressUserConversions, bool PartialOverloading,
+    OverloadCandidateParamOrder PO) {
 
   // C++ [over.match.funcs]p7:
   //   In each case where a candidate is a function template, candidate
@@ -7817,12 +7820,12 @@ void Sema::AddMethodTemplateCandidate(
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           MethodTmpl, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, /*AggregateDeductionCandidate=*/false,
           /*PartialOrdering=*/false, ObjectType, ObjectClassification,
           [&](ArrayRef<QualType> ParamTypes) {
-            return CheckNonDependentConversions(
+            return S.CheckNonDependentConversions(
                 MethodTmpl, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, ActingContext, ObjectType,
                 ObjectClassification, PO);
@@ -7844,8 +7847,8 @@ void Sema::AddMethodTemplateCandidate(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                            Info);
+      Candidate.DeductionFailure =
+          MakeDeductionFailureInfo(S.Context, Result, Info);
     }
     return;
   }
@@ -7855,10 +7858,34 @@ void Sema::AddMethodTemplateCandidate(
   assert(Specialization && "Missing member function template specialization?");
   assert(isa<CXXMethodDecl>(Specialization) &&
          "Specialization is not a member function?");
-  AddMethodCandidate(cast<CXXMethodDecl>(Specialization), FoundDecl,
-                     ActingContext, ObjectType, ObjectClassification, Args,
-                     CandidateSet, SuppressUserConversions, PartialOverloading,
-                     Conversions, PO, Info.hasStrictPackMatch());
+  S.AddMethodCandidate(
+      cast<CXXMethodDecl>(Specialization), FoundDecl, ActingContext, ObjectType,
+      ObjectClassification, Args, CandidateSet, SuppressUserConversions,
+      PartialOverloading, Conversions, PO, Info.hasStrictPackMatch());
+}
+
+void Sema::AddMethodTemplateCandidate(
+    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext,
+    TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
+    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, OverloadCandidateParamOrder PO) {
+  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
+    return;
+
+  if (ExplicitTemplateArgs ||
+      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts())) {
+    AddMethodTemplateCandidateImmediately(
+        *this, CandidateSet, MethodTmpl, FoundDecl, ActingContext,
+        ExplicitTemplateArgs, ObjectType, ObjectClassification, Args,
+        SuppressUserConversions, PartialOverloading, PO);
+    return;
+  }
+
+  CandidateSet.AddDeferredMethodTemplateCandidate(
+      MethodTmpl, FoundDecl, ActingContext, ObjectType, ObjectClassification,
+      Args, SuppressUserConversions, PartialOverloading, PO);
 }
 
 /// Determine whether a given function template has a simple explicit specifier
@@ -7867,14 +7894,18 @@ static bool isNonDependentlyExplicit(FunctionTemplateDecl *FTD) {
   return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).isExplicit();
 }
 
-void Sema::AddTemplateOverloadCandidate(
+static bool hasDependentExplicit(FunctionTemplateDecl *FTD) {
+  return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).getKind() ==
+         ExplicitSpecKind::Unresolved;
+}
+
+static void AddTemplateOverloadCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
     TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
-    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
-  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
-    return;
+    bool SuppressUserConversions, bool PartialOverloading, bool AllowExplicit,
+    Sema::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+    bool AggregateCandidateDeduction) {
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -7901,14 +7932,14 @@ void Sema::AddTemplateOverloadCandidate(
                              FunctionTemplate->getTemplateDepth());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           FunctionTemplate, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, AggregateCandidateDeduction,
           /*PartialOrdering=*/false,
           /*ObjectType=*/QualType(),
           /*ObjectClassification=*/Expr::Classification(),
           [&](ArrayRef<QualType> ParamTypes) {
-            return CheckNonDependentConversions(
+            return S.CheckNonDependentConversions(
                 FunctionTemplate, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, nullptr, QualType(), {}, PO);
           });
@@ -7932,8 +7963,8 @@ void Sema::AddTemplateOverloadCandidate(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                            Info);
+      Candidate.DeductionFailure =
+          MakeDeductionFailureInfo(S.Context, Result, Info);
     }
     return;
   }
@@ -7941,7 +7972,7 @@ void Sema::AddTemplateOverloadCandidate(
   // Add the function template specialization produced by template argument
   // deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  AddOverloadCandidate(
+  S.AddOverloadCandidate(
       Specialization, FoundDecl, Args, CandidateSet, SuppressUserConversions,
       PartialOverloading, AllowExplicit,
       /*AllowExplicitConversions=*/false, IsADLCandidate, Conversions, PO,
@@ -7949,6 +7980,38 @@ void Sema::AddTemplateOverloadCandidate(
       Info.hasStrictPackMatch());
 }
 
+void Sema::AddTemplateOverloadCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
+    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
+  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
+    return;
+
+  bool DependentExplicitSpecifier = hasDependentExplicit(FunctionTemplate);
+
+  if (ExplicitTemplateArgs ||
+      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
+      (isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl()) &&
+       DependentExplicitSpecifier)) {
+
+    AddTemplateOverloadCandidateImmediately(
+        *this, CandidateSet, FunctionTemplate, FoundDecl, ExplicitTemplateArgs,
+        Args, SuppressUserConversions, PartialOverloading, AllowExplicit,
+        IsADLCandidate, PO, AggregateCandidateDeduction);
+
+    if (DependentExplicitSpecifier)
+      CandidateSet.DisableResolutionByPerfectCandidate();
+    return;
+  }
+
+  CandidateSet.AddDeferredTemplateCandidate(
+      FunctionTemplate, FoundDecl, Args, SuppressUserConversions,
+      PartialOverloading, AllowExplicit, IsADLCandidate, PO,
+      AggregateCandidateDeduction);
+}
+
 bool Sema::CheckNonDependentConversions(
     FunctionTemplateDecl *FunctionTemplate, ArrayRef<QualType> ParamTypes,
     ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet,
@@ -8264,16 +8327,12 @@ void Sema::AddConversionCandidate(
   }
 }
 
-void Sema::AddTemplateConversionCandidate(
+static void AddTemplateConversionCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
-    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
-    bool AllowExplicit, bool AllowResultConversion) {
-  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
-         "Only conversion function templates permitted here");
-
-  if (!CandidateSet.isNewCandidate(FunctionTemplate))
-    return;
+    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+    bool AllowResultConversion) {
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -8288,11 +8347,11 @@ void Sema::AddTemplateConversionCandidate(
   }
 
   QualType ObjectType = From->getType();
-  Expr::Classification ObjectClassification = From->Classify(getASTContext());
+  Expr::Classification ObjectClassification = From->Classify(S.Context);
 
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   CXXConversionDecl *Specialization = nullptr;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           FunctionTemplate, ObjectType, ObjectClassification, ToType,
           Specialization, Info);
       Result != TemplateDeductionResult::Success) {
@@ -8302,18 +8361,47 @@ void Sema::AddTemplateConversionCandidate(
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_deduction;
     Candidate.ExplicitCallArguments = 1;
-    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                          Info);
+    Candidate.DeductionFailure =
+        MakeDeductionFailureInfo(S.Context, Result, Info);
     return;
   }
 
   // Add the conversion function template specialization produced by
   // template argument deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  AddConversionCandidate(Specialization, FoundDecl, ActingDC, From, ToType,
-                         CandidateSet, AllowObjCConversionOnExplicit,
-                         AllowExplicit, AllowResultConversion,
-                         Info.hasStrictPackMatch());
+  S.AddConversionCandidate(Specialization, FoundDecl, ActingContext, From,
+                           ToType, CandidateSet, AllowObjCConversionOnExplicit,
+                           AllowExplicit, AllowResultConversion,
+                           Info.hasStrictPackMatch());
+}
+
+void Sema::AddTemplateConversionCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
+    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
+    bool AllowExplicit, bool AllowResultConversion) {
+  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
+         "Only conversion function templates permitted here");
+
+  if (!CandidateSet.isNewCandidate(FunctionTemplate))
+    return;
+
+  if (!CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
+      CandidateSet.getKind() ==
+          OverloadCandidateSet::CSK_InitByUserDefinedConversion ||
+      CandidateSet.getKind() == OverloadCandidateSet::CSK_InitByConstructor) {
+    AddTemplateConversionCandidateImmediately(
+        *this, CandidateSet, FunctionTemplate, FoundDecl, ActingDC, From,
+        ToType, AllowObjCConversionOnExplicit, AllowExplicit,
+        AllowResultConversion);
+
+    CandidateSet.DisableResolutionByPerfectCandidate();
+    return;
+  }
+
+  CandidateSet.AddDeferredConversionTemplateCandidate(
+      FunctionTemplate, FoundDecl, ActingDC, From, ToType,
+      AllowObjCConversionOnExplicit, AllowExplicit, AllowResultConversion);
 }
 
 void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
@@ -8463,11 +8551,17 @@ void Sema::AddNonMemberOperatorCandidates(
     if (FunTmpl) {
       AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
                                    FunctionArgs, CandidateSet);
-      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD))
-        AddTemplateOverloadCandidate(
-            FunTmpl, F.getPair(), ExplicitTemplateArgs,
-            {FunctionArgs[1], FunctionArgs[0]}, CandidateSet, false, false,
-            true, ADLCallKind::NotADL, OverloadCandidateParamOrder::Reversed);
+      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD)) {
+
+        // As template candidates are not deduced immediately,
+        // persist the array in the overload set.
+        ArrayRef<Expr *> Reversed = CandidateSet.getPersistentArgsArray(
+            FunctionArgs[1], FunctionArgs[0]);
+        AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
+                                     Reversed, CandidateSet, false, false, true,
+                                     ADLCallKind::NotADL,
+                                     OverloadCandidateParamOrder::Reversed);
+      }
     } else {
       if (ExplicitTemplateArgs)
         continue;
@@ -10199,6 +10293,8 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
   // FIXME: Pass in the explicit template arguments?
   ArgumentDependentLookup(Name, Loc, Args, Fns);
 
+  ArrayRef<Expr *> ReversedArgs;
+
   // Erase all of the candidates we already knew about.
   for (OverloadCandidateSet::iterator Cand = CandidateSet.begin(),
                                    CandEnd = CandidateSet.end();
@@ -10238,9 +10334,15 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
           /*AllowExplicit=*/true, ADLCallKind::UsesADL);
       if (CandidateSet.getRewriteInfo().shouldAddReversed(
               *this, Args, FTD->getTemplatedDecl())) {
+
+        // As template candidates are not deduced immediately,
+        // persist the array in the overload set.
+        if (ReversedArgs.empty())
+          ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
+
         AddTemplateOverloadCandidate(
-            FTD, FoundDecl, ExplicitTemplateArgs, {Args[1], Args[0]},
-            CandidateSet, /*SuppressUserConversions=*/false, PartialOverloading,
+            FTD, FoundDecl, ExplicitTemplateArgs, ReversedArgs, CandidateSet,
+            /*SuppressUserConversions=*/false, PartialOverloading,
             /*AllowExplicit=*/true, ADLCallKind::UsesADL,
             OverloadCandidateParamOrder::Reversed);
       }
@@ -10913,23 +11015,147 @@ bool OverloadCandidate::NotValidBecauseConstraintExprHasError() const {
              ->Satisfaction.ContainsErrors;
 }
 
-/// Computes the best viable function (C++ 13.3.3)
-/// within an overload candidate set.
-///
-/// \param Loc The location of the function name (or operator symbol) for
-/// which overload resolution occurs.
-///
-/// \param Best If overload resolution was successful or found a deleted
-/// function, \p Best points to the candidate function found.
-///
-/// \returns The result of overload resolution.
+void OverloadCandidateSet::AddDeferredTemplateCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    ArrayRef<Expr *> Args, bool SuppressUserConversions,
+    bool PartialOverloading, bool AllowExplicit,
+    CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+    bool AggregateCandidateDeduction) {
+
+  auto *C =
+      allocateDeferredCandidate<DeferredFunctionTemplateOverloadCandidate>();
+
+  C = new (C) DeferredFunctionTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Function,
+       /*AllowObjCConversionOnExplicit=*/false,
+       /*AllowResultConversion=*/false, AllowExplicit, SuppressUserConversions,
+       PartialOverloading, AggregateCandidateDeduction},
+      FunctionTemplate,
+      FoundDecl,
+      Args,
+      IsADLCandidate,
+      PO};
+
+  HasDeferredTemplateConstructors |=
+      isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl());
+}
+
+void OverloadCandidateSet::AddDeferredMethodTemplateCandidate(
+    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext, QualType ObjectType,
+    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+    bool SuppressUserConversions, bool PartialOverloading,
+    OverloadCandidateParamOrder PO) {
+
+  assert(!isa<CXXConstructorDecl>(MethodTmpl->getTemplatedDecl()));
+
+  auto *C =
+      allocateDeferredCandidate<DeferredMethodTemplateOverloadCandidate>();
+
+  C = new (C) DeferredMethodTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Method,
+       /*AllowObjCConversionOnExplicit=*/false,
+       /*AllowResultConversion=*/false,
+       /*AllowExplicit=*/false, SuppressUserConversions, PartialOverloading,
+       /*AggregateCandidateDeduction=*/false},
+      MethodTmpl,
+      FoundDecl,
+      Args,
+      ActingContext,
+      ObjectClassification,
+      ObjectType,
+      PO};
+}
+
+void OverloadCandidateSet::AddDeferredConversionTemplateCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+    bool AllowResultConversion) {
+
+  auto *C =
+      allocateDeferredCandidate<DeferredConversionTemplateOverloadCandidate>();
+
+  C = new (C) DeferredConversionTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Conversion,
+       AllowObjCConversionOnExplicit, AllowResultConversion,
+       /*AllowExplicit=*/false,
+       /*SuppressUserConversions=*/false,
+       /*PartialOverloading*/ false,
+       /*AggregateCandidateDeduction=*/false},
+      FunctionTemplate,
+      FoundDecl,
+      ActingContext,
+      From,
+      ToType};
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredMethodTemplateOverloadCandidate &C) {
+
+  AddMethodTemplateCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext,
+      /*ExplicitTemplateArgs=*/nullptr, C.ObjectType, C.ObjectClassification,
+      C.Args, C.SuppressUserConversions, C.PartialOverloading, C.PO);
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredFunctionTemplateOverloadCandidate &C) {
+  AddTemplateOverloadCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl,
+      /*ExplicitTemplateArgs=*/nullptr, C.Args, C.SuppressUserConversions,
+      C.PartialOverloading, C.AllowExplicit, C.IsADLCandidate, C.PO,
+      C.AggregateCandidateDeduction);
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredConversionTemplateOverloadCandidate &C) {
+  return AddTemplateConversionCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext, C.From,
+      C.ToType, C.AllowObjCConversionOnExplicit, C.AllowExplicit,
+      C.AllowResultConversion);
+}
+
+void OverloadCandidateSet::InjectNonDeducedTemplateCandidates(Sema &S) {
+  Candidates.reserve(Candidates.size() + DeferredCandidatesCount);
+  DeferredTemplateOverloadCandidate *Cand = FirstDeferredCandidate;
+  while (Cand) {
+    switch (Cand->Kind) {
+    case DeferredTemplateOverloadCandidate::Function:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredFunctionTemplateOverloadCandidate *>(Cand));
+      break;
+    case DeferredTemplateOverloadCandidate::Method:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredMethodTemplateOverloadCandidate *>(Cand));
+      break;
+    case DeferredTemplateOverloadCandidate::Conversion:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredConversionTemplateOverloadCandidate *>(Cand));
+      break;
+    }
+    Cand = Cand->Next;
+  }
+  FirstDeferredCandidate = nullptr;
+  DeferredCandidatesCount = 0;
+}
+
 OverloadingResult
-OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
-                                         iterator &Best) {
-  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
-  std::transform(begin(), end(), std::back_inserter(Candidates),
-                 [](OverloadCandidate &Cand) { return &Cand; });
+OverloadCandidateSet::ResultForBestCandidate(const iterator &Best) {
+  Best->Best = true;
+  if (Best->Function && Best->Function->isDeleted())
+    return OR_Deleted;
+  return OR_Success;
+}
 
+void OverloadCandidateSet::CudaExcludeWrongSideCandidates(
+    Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates) {
   // [CUDA] HD->H or HD->D calls are technically not allowed by CUDA but
   // are accepted by both clang and NVCC. However, during a particular
   // compilation mode only one call variant is viable. We need to
@@ -10941,27 +11167,112 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   // -fgpu-exclude-wrong-side-overloads is off. When
   // -fgpu-exclude-wrong-side-overloads is on, all candidates are compared
   // uniformly in isBetterOverloadCandidate.
-  if (S.getLangOpts().CUDA && !S.getLangOpts().GPUExcludeWrongSideOverloads) {
-    const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
-    bool ContainsSameSideCandidate =
-        llvm::any_of(Candidates, [&](OverloadCandidate *Cand) {
-          // Check viable function only.
-          return Cand->Viable && Cand->Function &&
-                 S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-                     SemaCUDA::CFP_SameSide;
-        });
-    if (ContainsSameSideCandidate) {
-      auto IsWrongSideCandidate = [&](OverloadCandidate *Cand) {
-        // Check viable function only to avoid unnecessary data copying/moving.
+  if (!S.getLangOpts().CUDA || S.getLangOpts().GPUExcludeWrongSideOverloads)
+    return;
+  const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
+
+  bool ContainsSameSideCandidate =
+      llvm::any_of(Candidates, [&](const OverloadCandidate *Cand) {
+        // Check viable function only.
         return Cand->Viable && Cand->Function &&
                S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-                   SemaCUDA::CFP_WrongSide;
-      };
-      llvm::erase_if(Candidates, IsWrongSideCandidate);
+                   SemaCUDA::CFP_SameSide;
+      });
+
+  if (!ContainsSameSideCandidate)
+    return;
+
+  auto IsWrongSideCandidate = [&](const OverloadCandidate *Cand) {
+    // Check viable function only to avoid unnecessary data copying/moving.
+    return Cand->Viable && Cand->Function &&
+           S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
+               SemaCUDA::CFP_WrongSide;
+  };
+  llvm::erase_if(Candidates, IsWrongSideCandidate);
+}
+
+/// Computes the best viable function (C++ 13.3.3)
+/// within an overload candidate set.
+///
+/// \param Loc The location of the function name (or operator symbol) for
+/// which overload resolution occurs.
+///
+/// \param Best If overload resolution was successful or found a deleted
+/// function, \p Best points to the candidate function found.
+///
+/// \returns The result of overload resolution.
+OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
+                                                           SourceLocation Loc,
+                                                           iterator &Best) {
+
+  assert(shouldDeferTemplateArgumentDeduction(S.getLangOpts()) ||
+         DeferredCandidatesCount == 0 &&
+             "Unexpected deferred template candidates");
+
+  bool TwoPhaseResolution =
+      DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
+
+  if (TwoPhaseResolution) {
+
+    PerfectViableFunction(S, Loc, Best);
+    if (Best != end())
+      return ResultForBestCandidate(Best);
+  }
+
+  InjectNonDeducedTemplateCandidates(S);
+  return BestViableFunctionImpl(S, Loc, Best);
+}
+
+void OverloadCandidateSet::PerfectViableFunction(
+    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
+
+  Best = end();
+  for (auto It = begin(); It != end(); ++It) {
+
+    if (!It->isPerfectMatch(S.getASTContext()))
+      continue;
+
+    // We found a suitable conversion function
+    // but if there is a template constructor in the target class
+    // we might prefer that instead.
+    if (HasDeferredTemplateConstructors &&
+        isa_and_nonnull<CXXConversionDecl>(It->Function)) {
+      Best = end();
+      break;
+    }
+
+    if (Best == end()) {
+      Best = It;
+      continue;
+    }
+    if (Best->Function && It->Function) {
+      FunctionDecl *D =
+          S.getMoreConstrainedFunction(Best->Function, It->Function);
+      if (D == nullptr) {
+        Best = end();
+        break;
+      }
+      if (D == It->Function)
+        Best = It;
+      continue;
     }
+    // ambiguous
+    Best = end();
+    break;
   }
+}
+
+OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
+    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
+
+  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
+  Candidates.reserve(this->Candidates.size());
+  std::transform(begin(), end(), std::back_inserter(Candidates),
+                 [](OverloadCandidate &Cand) { return &Cand; });
+
+  if (S.getLangOpts().CUDA)
+    CudaExcludeWrongSideCandidates(S, Candidates);
 
-  // Find the best viable function.
   Best = end();
   for (auto *Cand : Candidates) {
     Cand->Best = false;
@@ -10983,9 +11294,8 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   if (Best == end())
     return OR_No_Viable_Function;
 
+  llvm::SmallVector<OverloadCandidate *, 4> PendingBest;
   llvm::SmallVector<const NamedDecl *, 4> EquivalentCands;
-
-  llvm::SmallVector<OverloadCandidate*, 4> PendingBest;
   PendingBest.push_back(&*Best);
   Best->Best = true;
 
@@ -11008,25 +11318,15 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
     }
   }
 
-  // If we found more than one best candidate, this is ambiguous.
   if (Best == end())
     return OR_Ambiguous;
 
-  // Best is the best viable function.
-  if (Best->Function && Best->Function->isDeleted())
-    return OR_Deleted;
-
-  if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
-      Kind == CSK_AddressOfOverloadSet && M &&
-      M->isImplicitObjectMemberFunction()) {
-    return OR_No_Viable_Function;
-  }
+  OverloadingResult R = ResultForBestCandidate(Best);
 
   if (!EquivalentCands.empty())
     S.diagnoseEquivalentInternalLinkageDeclarations(Loc, Best->Function,
                                                     EquivalentCands);
-
-  return OR_Success;
+  return R;
 }
 
 namespace {
@@ -12733,6 +13033,9 @@ SmallVector<OverloadCandidate *, 32> OverloadCandidateSet::CompleteCandidates(
     Sema &S, OverloadCandidateDisplayKind OCD, ArrayRef<Expr *> Args,
     SourceLocation OpLoc,
     llvm::function_ref<bool(OverloadCandidate &)> Filter) {
+
+  InjectNonDeducedTemplateCandidates(S);
+
   // Sort the candidates by viability and position.  Sorting directly would
   // be prohibitive, so we make a set of pointers and sort those.
   SmallVector<OverloadCandidate*, 32> Cands;
@@ -14354,10 +14657,12 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
                                          Expr *ExecConfig,
                                          bool AllowTypoCorrection,
                                          bool CalleesAddressIsTaken) {
-  OverloadCandidateSet CandidateSet(
-      Fn->getExprLoc(), CalleesAddressIsTaken
-                            ? OverloadCandidateSet::CSK_AddressOfOverloadSet
-                            : OverloadCandidateSet::CSK_Normal);
+
+  OverloadCandidateSet::CandidateSetKind CSK =
+      CalleesAddressIsTaken ? OverloadCandidateSet::CSK_AddressOfOverloadSet
+                            : OverloadCandidateSet::CSK_Normal;
+
+  OverloadCandidateSet CandidateSet(Fn->getExprLoc(), CSK);
   ExprResult result;
 
   if (buildOverloadedCallSet(S, Fn, ULE, Args, LParenLoc, &CandidateSet,
@@ -14373,6 +14678,17 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
   OverloadingResult OverloadResult =
       CandidateSet.BestViableFunction(*this, Fn->getBeginLoc(), Best);
 
+  // [C++23][over.call.func]
+  // if overload resolution selects a non-static member function,
+  // the call is ill-formed;
+  if (CSK == OverloadCandidateSet::CSK_AddressOfOverloadSet &&
+      Best != CandidateSet.end()) {
+    if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
+        M && M->isImplicitObjectMemberFunction()) {
+      OverloadResult = OR_No_Viable_Function;
+    }
+  }
+
   // Model the case with a call to a templated function whose definition
   // encloses the call and whose return type contains a placeholder type as if
   // the UnresolvedLookupExpr was type-dependent.
@@ -14708,18 +15024,24 @@ void Sema::LookupOverloadedBinOp(OverloadCandidateSet &CandidateSet,
   // rewritten candidates using these functions if necessary.
   AddNonMemberOperatorCandidates(Fns, Args, CandidateSet);
 
+  // As template candidates are not deduced immediately,
+  // persist the array in the overload set.
+  ArrayRef<Expr *> ReversedArgs;
+  if (CandidateSet.getRewriteInfo().allowsReversed(Op) ||
+      CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
+    ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
+
   // Add operator candidates that are member functions.
   AddMemberOperatorCandidates(Op, OpLoc, Args, CandidateSet);
   if (CandidateSet.getRewriteInfo().allowsReversed(Op))
-    AddMemberOperatorCandidates(Op, OpLoc, {Args[1], Args[0]}, CandidateSet,
+    AddMemberOperatorCandidates(Op, OpLoc, ReversedArgs, CandidateSet,
                                 OverloadCandidateParamOrder::Reversed);
 
   // In C++20, also add any rewritten member candidates.
   if (ExtraOp) {
     AddMemberOperatorCandidates(ExtraOp, OpLoc, Args, CandidateSet);
     if (CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
-      AddMemberOperatorCandidates(ExtraOp, OpLoc, {Args[1], Args[0]},
-                                  CandidateSet,
+      AddMemberOperatorCandidates(ExtraOp, OpLoc, ReversedArgs, CandidateSet,
                                   OverloadCandidateParamOrder::Reversed);
   }
 
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 772962ac653f7..0ecdbb3ffb89f 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -6142,9 +6142,9 @@ FunctionDecl *Sema::getMoreConstrainedFunction(FunctionDecl *FD1,
   assert(!FD1->getDescribedTemplate() && !FD2->getDescribedTemplate() &&
          "not for function templates");
   assert(!FD1->isFunctionTemplateSpecialization() ||
-         isa<CXXConversionDecl>(FD1));
+         (isa<CXXConversionDecl, CXXConstructorDecl>(FD1)));
   assert(!FD2->isFunctionTemplateSpecialization() ||
-         isa<CXXConversionDecl>(FD2));
+         (isa<CXXConversionDecl, CXXConstructorDecl>(FD2)));
 
   FunctionDecl *F1 = FD1;
   if (FunctionDecl *P = FD1->getTemplateInstantiationPattern(false))
diff --git a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
index ba8e2dc372e98..083e743818121 100644
--- a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
+++ b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
@@ -14,7 +14,7 @@ template<typename T> struct S {
 // expected-note@#FINST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{})
 void f(T);
-void f(int);
+void f(long);
 
 // Ensure this applies to operator && as well.
 // expected-error@+3{{atomic constraint must be of type 'bool' (found 'S<int>')}}
@@ -22,7 +22,7 @@ void f(int);
 // expected-note@#F2INST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{} && true)
 void f2(T);
-void f2(int);
+void f2(long);
 
 template<typename T> requires requires {
   requires S<T>{};
@@ -36,12 +36,12 @@ template<typename T> requires requires {
   //
 }
 void f3(T);
-void f3(int);
+void f3(long);
 
 // Doesn't diagnose, since this is no longer a compound requirement.
 template<typename T> requires (bool(1 && 2))
 void f4(T);
-void f4(int);
+void f4(long);
 
 void g() {
   f(0); // #FINST
diff --git a/clang/test/SemaCUDA/function-overload.cu b/clang/test/SemaCUDA/function-overload.cu
index 4710c81763adf..3d05839af7528 100644
--- a/clang/test/SemaCUDA/function-overload.cu
+++ b/clang/test/SemaCUDA/function-overload.cu
@@ -1,6 +1,3 @@
-// REQUIRES: x86-registered-target
-// REQUIRES: nvptx-registered-target
-
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu -fsyntax-only \
 // RUN:   -verify=host,hostdefer,devdefer,expected %s
 // RUN: %clang_cc1 -std=c++14 -triple nvptx64-nvidia-cuda -fsyntax-only \
diff --git a/clang/test/SemaCXX/implicit-member-functions.cpp b/clang/test/SemaCXX/implicit-member-functions.cpp
index 1554b1af5d59a..8350eac5b88a0 100644
--- a/clang/test/SemaCXX/implicit-member-functions.cpp
+++ b/clang/test/SemaCXX/implicit-member-functions.cpp
@@ -54,31 +54,24 @@ namespace PR7594 {
 namespace Recursion {
   template<typename T> struct InvokeCopyConstructor {
     static const T &get();
-    typedef decltype(T(get())) type; // expected-error {{no matching conver}}
+    typedef decltype(T(get())) type;
   };
   struct B;
   struct A {
-    // expected-note@-1 {{while substituting deduced template arguments}}
     typedef B type;
     template<typename T,
              typename = typename InvokeCopyConstructor<typename T::type>::type>
-    // expected-note@-1 {{in instantiation of template class}}
     A(const T &);
-    // expected-note@-1 {{in instantiation of default argument}}
   };
-  struct B { // expected-note {{while declaring the implicit copy constructor for 'B'}}
-    // expected-note@-1 {{candidate constructor (the implicit move }}
-    B(); // expected-note {{candidate constructor not viable}}
+  struct B {
+    B();
     A a;
   };
   // Triggering the declaration of B's copy constructor causes overload
-  // resolution to occur for A's copying constructor, which instantiates
-  // InvokeCopyConstructor<B>, which triggers the declaration of B's copy
-  // constructor. Notionally, this happens when we get to the end of the
-  // definition of 'struct B', so there is no declared copy constructor yet.
-  //
-  // This behavior is g++-compatible, but isn't exactly right; the class is
-  // supposed to be incomplete when we implicitly declare its special members.
+  // resolution to occur for A's copying constructor, which picks
+  // the implicit copy constructor of A.
+  // Because that copy constructor is always a perfect match the template
+  // candidate is not instantiated.
   B b = B();
 
 
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
new file mode 100644
index 0000000000000..877816ca013ec
--- /dev/null
+++ b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -0,0 +1,185 @@
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++11 %s
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++20 %s
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++2c %s
+
+template <typename T>
+struct Invalid { static_assert(false, "instantiated Invalid"); }; // #err-invalid
+
+template <typename T>
+int f(T a, Invalid<T> = {}); // #note-f
+
+// sanity check
+int e1 = f(0);
+//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
+//expected-note@-2 {{in instantiation of default function argument expression for 'f<int>' required here}}
+//expected-note@#note-f {{in instantiation of template class 'Invalid<int>' requested here}}
+//expected-note@#note-f {{passing argument to parameter here}}
+
+int f(int);
+int ok1 = f(0);
+int e4 = f((const int&)(ok1));
+
+int f(int, int = 0);
+int ok2 = f(0, 0);
+
+int e2  = f(0L);
+//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
+//expected-note@-2 {{in instantiation of default function argument expression for 'f<long>' required here}}
+//expected-note@#note-f {{in instantiation of template class 'Invalid<long>' requested here}}
+//expected-note@#note-f {{passing argument to parameter here}}
+
+int f(long);
+int ok3 = f(0L);
+
+template <typename T>
+struct Invalid2 { static_assert(false, "instantiated Invalid2"); }; // #err-qualifiers
+
+template <typename T>
+int ref(T a, Invalid2<T> = {}); // expected-note 2{{here}}
+int ref(int&);
+int ref1 = ref(ok3);
+int ref2 = ref((const int&)ok3); // expected-note {{here}}
+//expected-error@#err-qualifiers {{static assertion failed: instantiated Invalid2}}
+
+
+template <typename T>
+int f_alias(T a, Invalid<T> = {});
+using Alias = int;
+int f_alias(Alias);
+int ok4 = f_alias(0);
+
+#if __cplusplus >= 202002
+
+struct Copyable {
+  template <typename T>
+  requires __is_constructible(Copyable, T)
+  explicit Copyable(T op) noexcept; // #1
+  Copyable(const Copyable&) noexcept = default; // #2
+};
+static_assert(__is_constructible(Copyable, const Copyable&));
+
+struct ImplicitlyCopyable {
+  template <typename T>
+  requires __is_constructible(ImplicitlyCopyable, T)
+  explicit ImplicitlyCopyable(T op) = delete; // #1
+};
+static_assert(__is_constructible(ImplicitlyCopyable, const ImplicitlyCopyable&));
+
+
+struct Movable {
+  template <typename T>
+  requires __is_constructible(Movable, T) // #err-self-constraint-1
+  explicit Movable(T op) noexcept; // #1
+  Movable(Movable&&) noexcept = default; // #2
+};
+static_assert(__is_constructible(Movable, Movable&&));
+static_assert(__is_constructible(Movable, const Movable&));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(Movable, const Movable &)'}}
+
+static_assert(__is_constructible(Movable, int));
+// expected-error@-1{{static assertion failed due to requirement '__is_constructible(Movable, int)'}} \
+// expected-note@-1 2{{}}
+// expected-error@#err-self-constraint-1{{satisfaction of constraint '__is_constructible(Movable, T)' depends on itself}}
+// expected-note@#err-self-constraint-1 4{{}}
+
+template <typename T>
+struct Members {
+    constexpr auto f(auto) {
+        static_assert(false, "");
+    }
+    constexpr auto f(int) { return 1; }
+    constexpr auto f(int) requires true { return 2; }
+
+    constexpr auto g(auto) {
+        static_assert(false, "instantiated member"); //#err-qualified-member
+        return 0;
+    }
+    constexpr auto g(int) & { return 1; }
+
+    static constexpr auto s(auto) {
+        static_assert(false, "");
+    }
+    static constexpr auto s(int) {
+        return 1;
+    }
+    static constexpr auto s(int) requires true {
+        return 2;
+    }
+};
+
+static_assert(Members<int[1]>{}.f(0) == 2);
+static_assert(Members<int[2]>{}.g(0) == 0);
+// expected-error@#err-qualified-member {{static assertion failed: instantiated member}} \
+// expected-note@-1{{in instantiation of function template specialization 'Members<int[2]>::g<int>' }}
+Members<int[3]> m1;
+static_assert(m1.g(0) == 1);
+static_assert(Members<int[3]>{}.s(0) == 2);
+
+
+namespace ConstructorInit{
+struct S {
+  template <typename T>
+  S(T&&) {}
+};
+struct Test {
+  operator S() = delete;
+};
+
+static_assert(__is_constructible(S, Test));
+}
+
+namespace RefBinding {
+
+template <typename> struct remove_reference;
+template <typename _Tp> struct remove_reference<_Tp &> {
+  using type = _Tp;
+};
+template <typename _Tp> remove_reference<_Tp>::type move(_Tp &&);
+template <typename _Head> struct _Head_base {
+  _Head_base(_Head &__h) : _M_head_impl(__h) {}
+  template <typename _UHead> _Head_base(_UHead &&);
+  _Head _M_head_impl;
+};
+
+template <typename _Elements> void forward_as_tuple(_Elements &&) {
+  _Head_base<_Elements &&>(_Elements{});
+}
+struct StringRef {
+  void operator[](const StringRef __k) { forward_as_tuple((move)(__k)); }
+};
+
+}
+
+template <class> struct tuple {};
+struct BonkersBananas {
+  template <class T> operator T();
+  template <class = void> explicit operator tuple<int>() = delete;
+};
+static_assert(!__is_constructible(tuple<int>, BonkersBananas));
+
+namespace GH62096 {
+template <typename T>
+struct Oops {
+  static_assert(sizeof(T) == 0); // #GH62096-err
+  static constexpr bool value = true;
+};
+
+template <class OPERATOR>
+concept Operator = Oops<OPERATOR>::value; // #GH62096-note1
+
+template <Operator OP> void f(OP op); // // #GH62096-note2
+void f(int);
+
+void g(int n) { f(n); } // OK
+void h(short n) { f(n); }
+// expected-error@#GH62096-err {{static assertion failed due to requirement 'sizeof(short) == 0'}} \
+// expected-note@-1{{in instantiation of function template specialization}} \
+// expected-note@-1{{while checking constraint satisfaction for template}}
+// expected-note@#GH62096-note1{{in instantiation}}
+// expected-note@#GH62096-note1{{while substituting template arguments into constraint expression here}}
+// expected-note@#GH62096-note2{{while substituting template arguments into constraint expression here}}
+// expected-note@#GH62096-note2{{while checking the satisfaction of concept}}
+// expected-note@#GH62096-err {{expression evaluates}}
+}
+
+#endif
diff --git a/clang/test/SemaTemplate/instantiate-function-params.cpp b/clang/test/SemaTemplate/instantiate-function-params.cpp
index 7dd5595de58a3..eb2a7c5d4e8d6 100644
--- a/clang/test/SemaTemplate/instantiate-function-params.cpp
+++ b/clang/test/SemaTemplate/instantiate-function-params.cpp
@@ -6,13 +6,12 @@ template<typename T1> struct if_ {
   typedef if_c< static_cast<bool>(T1::value)> almost_type_; // expected-note 7{{in instantiation}}
 };
 template <class Model, void (Model::*)()> struct wrap_constraints { };
-template <class Model> 
+template <class Model>
 inline char has_constraints_(Model* , // expected-note 3{{candidate template ignored}}
-                               wrap_constraints<Model,&Model::constraints>* = 0); // expected-note 4{{in instantiation}}
-
+                               wrap_constraints<Model,&Model::constraints>* = 0);
 template <class Model> struct not_satisfied {
   static const bool value = sizeof( has_constraints_((Model*)0)  == 1); // expected-error 3{{no matching function}} \
-  // expected-note 4{{while substituting deduced template arguments into function template 'has_constraints_' [with }}
+  // expected-note 4{{in instantiation}}
 };
 template <class ModelFn> struct requirement_;
 template <void(*)()> struct instantiate {
diff --git a/clang/test/Templight/templight-empty-entries-fix.cpp b/clang/test/Templight/templight-empty-entries-fix.cpp
index d13b748068efe..7f34b10134929 100644
--- a/clang/test/Templight/templight-empty-entries-fix.cpp
+++ b/clang/test/Templight/templight-empty-entries-fix.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -templight-dump -Wno-unused-value %s 2>&1 | FileCheck %s
 
-void a() {
+void a(long) {
   [] {};
 }
 
@@ -17,14 +17,14 @@ void a() {
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 // CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 
-template <int = 0> void a() { a(); }
+template <int = 0> void a(long) { a(0); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -42,29 +42,29 @@ template <int = 0> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 
 template <int> struct b { typedef int c; };
-template <bool d = true, class = typename b<d>::c> void a() { a(); }
+template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -130,25 +130,25 @@ template <bool d = true, class = typename b<d>::c> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -166,34 +166,10 @@ template <bool d = true, class = typename b<d>::c> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 
 template <bool = true> void d(int = 0) { d(); }
 
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
@@ -249,41 +225,41 @@ void e() {
 }
 
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
 
 
 template <template<typename> class>
@@ -299,71 +275,71 @@ void foo() {
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}

From 73b8750a970ddaec5da1540c100561bd5104bca6 Mon Sep 17 00:00:00 2001
From: James Newling <james.newling@gmail.com>
Date: Wed, 16 Apr 2025 10:10:23 -0700
Subject: [PATCH 148/710] [mlir][vector] fold transpose(poison) -> poison
 (#135675)

Following on from https://github.com/llvm/llvm-project/pull/133988

---------

Signed-off-by: James Newling <james.newling@gmail.com>
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp   |  4 ++++
 mlir/test/Dialect/Vector/canonicalize.mlir | 11 +++++++++++
 2 files changed, 15 insertions(+)

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 504032a398fbe..3ef947c7b14a4 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -6009,6 +6009,10 @@ OpFoldResult vector::TransposeOp::fold(FoldAdaptor adaptor) {
     if (attr.isSplat())
       return attr.reshape(getResultVectorType());
 
+  // Eliminate poison transpose ops.
+  if (llvm::dyn_cast_if_present<ub::PoisonAttr>(adaptor.getVector()))
+    return ub::PoisonAttr::get(getContext());
+
   // Eliminate identity transpose ops. This happens when the dimensions of the
   // input vector remain in their original order after the transpose operation.
   ArrayRef<int64_t> perm = getPermutation();
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 733a2c67d2c0c..c28afd0dee97e 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -2240,6 +2240,17 @@ func.func @transpose_splat2(%arg : f32) -> vector<3x4xf32> {
 
 // -----
 
+// CHECK-LABEL: transpose_poison
+//       CHECK:  %[[POISON:.*]] = ub.poison : vector<4x6xi8>
+//       CHECK:  return %[[POISON]] : vector<4x6xi8>
+func.func @transpose_poison() -> vector<4x6xi8> {
+  %poison = ub.poison : vector<6x4xi8>
+  %transpose = vector.transpose %poison, [1, 0] : vector<6x4xi8> to vector<4x6xi8>
+  return %transpose : vector<4x6xi8>
+}
+
+// -----
+
 // CHECK-LABEL: func.func @insert_1d_constant
 //   CHECK-DAG: %[[ACST:.*]] = arith.constant dense<[9, 1, 2]> : vector<3xi32>
 //   CHECK-DAG: %[[BCST:.*]] = arith.constant dense<[0, 9, 2]> : vector<3xi32>

From eea86489dd3df5b66d75ee2590f4824913c411d5 Mon Sep 17 00:00:00 2001
From: James Newling <james.newling@gmail.com>
Date: Wed, 16 Apr 2025 10:12:38 -0700
Subject: [PATCH 149/710] [mlir][vector] Fold broadcast(poison) -> poison
 (#135677)

In addition to the new folder, I've also a test for broadcast(splat) ->
splat which I think was missing

Signed-off-by: James Newling <james.newling@gmail.com>
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp   |  2 ++
 mlir/test/Dialect/Vector/canonicalize.mlir | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 3ef947c7b14a4..4b2fba03ce551 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -2590,6 +2590,8 @@ OpFoldResult BroadcastOp::fold(FoldAdaptor adaptor) {
   }
   if (auto attr = llvm::dyn_cast<SplatElementsAttr>(adaptor.getSource()))
     return DenseElementsAttr::get(vectorType, attr.getSplatValue<Attribute>());
+  if (llvm::dyn_cast<ub::PoisonAttr>(adaptor.getSource()))
+    return ub::PoisonAttr::get(getContext());
   return {};
 }
 
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index c28afd0dee97e..b24cf93707d8b 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -1151,6 +1151,28 @@ func.func @bitcast_i8_to_i32() -> (vector<4xi32>, vector<4xi32>) {
 
 // -----
 
+// CHECK-LABEL: broadcast_poison
+//       CHECK:  %[[POISON:.*]] = ub.poison : vector<4x6xi8>
+//       CHECK:  return %[[POISON]] : vector<4x6xi8>
+func.func @broadcast_poison() -> vector<4x6xi8> {
+  %poison = ub.poison : vector<6xi8>
+  %broadcast = vector.broadcast %poison : vector<6xi8> to vector<4x6xi8>
+  return %broadcast : vector<4x6xi8>
+}
+
+// ----- 
+
+// CHECK-LABEL:  broadcast_splat_constant
+//       CHECK:  %[[CONST:.*]] = arith.constant dense<1> : vector<4x6xi8>
+//       CHECK:  return %[[CONST]] : vector<4x6xi8>
+func.func @broadcast_splat_constant() -> vector<4x6xi8> {
+  %cst = arith.constant dense<1> : vector<6xi8>
+  %broadcast = vector.broadcast %cst : vector<6xi8> to vector<4x6xi8>
+  return %broadcast : vector<4x6xi8>
+}
+
+// -----
+
 // CHECK-LABEL: broadcast_folding1
 //       CHECK: %[[CST:.*]] = arith.constant dense<42> : vector<4xi32>
 //   CHECK-NOT: vector.broadcast

From 52e0337ea34142f55c427493e9ca2be5fce2dd38 Mon Sep 17 00:00:00 2001
From: Chris B <chris.bieneman@me.com>
Date: Wed, 16 Apr 2025 12:13:19 -0500
Subject: [PATCH 150/710] [HLSL][OpenCL] Strip addrspace from implicit cast
 diags (#135830)

The address space of a source value for an implicit cast isn't really
relevant when emitting conversion warnings. Since the lvalue->rvalue
cast effectively removes the address space they don't factor in, but
they do create visual noise in the diagnostics.

This is a small quality-of-life fixup to get in as HLSL adopts more
address space annotations.
---
 clang/lib/Sema/SemaChecking.cpp                    |  8 ++++++++
 clang/test/SemaHLSL/Language/ImpCastAddrSpace.hlsl | 12 ++++++++++++
 clang/test/SemaOpenCL/cl20-device-side-enqueue.cl  |  2 +-
 3 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/SemaHLSL/Language/ImpCastAddrSpace.hlsl

diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 42da9ba97e0d3..26c2dc655a931 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -11417,6 +11417,14 @@ static void AnalyzeAssignment(Sema &S, BinaryOperator *E) {
 static void DiagnoseImpCast(Sema &S, Expr *E, QualType SourceType, QualType T,
                             SourceLocation CContext, unsigned diag,
                             bool pruneControlFlow = false) {
+  // For languages like HLSL and OpenCL, implicit conversion diagnostics listing
+  // address space annotations isn't really useful. The warnings aren't because
+  // you're converting a `private int` to `unsigned int`, it is because you're
+  // conerting `int` to `unsigned int`.
+  if (SourceType.hasAddressSpace())
+    SourceType = S.getASTContext().removeAddrSpaceQualType(SourceType);
+  if (T.hasAddressSpace())
+    T = S.getASTContext().removeAddrSpaceQualType(T);
   if (pruneControlFlow) {
     S.DiagRuntimeBehavior(E->getExprLoc(), E,
                           S.PDiag(diag)
diff --git a/clang/test/SemaHLSL/Language/ImpCastAddrSpace.hlsl b/clang/test/SemaHLSL/Language/ImpCastAddrSpace.hlsl
new file mode 100644
index 0000000000000..61e71b219b721
--- /dev/null
+++ b/clang/test/SemaHLSL/Language/ImpCastAddrSpace.hlsl
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -finclude-default-header -Wconversion -fnative-half-type %s -verify
+
+static double D = 2.0;
+static int I = D; // expected-warning{{implicit conversion turns floating-point number into integer: 'double' to 'int'}}
+groupshared float F = I; // expected-warning{{implicit conversion from 'int' to 'float' may lose precision}}
+
+export void fn() {
+  half d = I; // expected-warning{{implicit conversion from 'int' to 'half' may lose precision}}
+  int i = D; // expected-warning{{implicit conversion turns floating-point number into integer: 'double' to 'int'}}
+  int j = F; // expected-warning{{implicit conversion turns floating-point number into integer: 'float' to 'int'}}
+  int k = d; // expected-warning{{implicit conversion turns floating-point number into integer: 'half' to 'int'}}
+}
diff --git a/clang/test/SemaOpenCL/cl20-device-side-enqueue.cl b/clang/test/SemaOpenCL/cl20-device-side-enqueue.cl
index 36b901fc5f29e..524de8ce2f7dc 100644
--- a/clang/test/SemaOpenCL/cl20-device-side-enqueue.cl
+++ b/clang/test/SemaOpenCL/cl20-device-side-enqueue.cl
@@ -97,7 +97,7 @@ kernel void enqueue_kernel_tests(void) {
                  },
                  c, 1024L);
 #ifdef WCONV
-// expected-warning-re@-2{{implicit conversion changes signedness: '__private char' to 'unsigned {{int|long}}'}}
+// expected-warning-re@-2{{implicit conversion changes signedness: 'char' to 'unsigned {{int|long}}'}}
 #endif
 #define UINT_MAX 4294967295
 

From 81739c39db11b7f9a4f3528c1c66b552e57b47e4 Mon Sep 17 00:00:00 2001
From: Volodymyr Sapsai <vsapsai@apple.com>
Date: Wed, 16 Apr 2025 10:14:05 -0700
Subject: [PATCH 151/710] [Modules] Fix an identifier hiding a function-like
 macro definition. (#135471)

We emit a macro definition only in a module defining it. But it means
that if another module has an identifier with the same name as the
macro, the users of such module won't be able to use the macro anymore.

Fix by storing that an identifier has a macro definition that's not in a
current module (`MacroDirectivesOffset == 0`). This way
`IdentifierLookupVisitor` knows not to stop at the first module with an
identifier but to keep checking included modules for the actual macro
definition.

Fixes issue #32040.

rdar://30258278
---
 clang/lib/Serialization/ASTReader.cpp        | 17 ++++++++----
 clang/lib/Serialization/ASTReaderInternals.h |  6 ++++
 clang/lib/Serialization/ASTWriter.cpp        | 16 +++++++----
 clang/test/Modules/macro-identifier-hiding.c | 29 ++++++++++++++++++++
 4 files changed, 58 insertions(+), 10 deletions(-)
 create mode 100644 clang/test/Modules/macro-identifier-hiding.c

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 02c31dff620ec..58d3d33b31644 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1131,7 +1131,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   bool HasRevertedTokenIDToIdentifier = readBit(Bits);
   bool Poisoned = readBit(Bits);
   bool ExtensionToken = readBit(Bits);
-  bool HadMacroDefinition = readBit(Bits);
+  bool HasMacroDefinition = readBit(Bits);
 
   assert(Bits == 0 && "Extra bits in the identifier?");
   DataLen -= sizeof(uint16_t) * 2;
@@ -1151,14 +1151,17 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
          "Incorrect C++ operator keyword flag");
   (void)CPlusPlusOperatorKeyword;
 
-  // If this identifier is a macro, deserialize the macro
-  // definition.
-  if (HadMacroDefinition) {
+  // If this identifier has a macro definition, deserialize it or notify the
+  // visitor the actual definition is in a different module.
+  if (HasMacroDefinition) {
     uint32_t MacroDirectivesOffset =
         endian::readNext<uint32_t, llvm::endianness::little>(d);
     DataLen -= 4;
 
-    Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
+    if (MacroDirectivesOffset)
+      Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
+    else
+      hasMacroDefinitionInDependencies = true;
   }
 
   Reader.SetIdentifierInfo(ID, II);
@@ -2419,6 +2422,10 @@ namespace {
       // declarations it needs.
       ++NumIdentifierLookupHits;
       Found = *Pos;
+      if (Trait.hasMoreInformationInDependencies()) {
+        // Look for the identifier in extra modules as they contain more info.
+        return false;
+      }
       return true;
     }
 
diff --git a/clang/lib/Serialization/ASTReaderInternals.h b/clang/lib/Serialization/ASTReaderInternals.h
index 353e0a53cad9b..4a7794889b039 100644
--- a/clang/lib/Serialization/ASTReaderInternals.h
+++ b/clang/lib/Serialization/ASTReaderInternals.h
@@ -286,6 +286,8 @@ class ASTIdentifierLookupTrait : public ASTIdentifierLookupTraitBase {
   // identifier that was constructed before the AST file was read.
   IdentifierInfo *KnownII;
 
+  bool hasMacroDefinitionInDependencies = false;
+
 public:
   using data_type = IdentifierInfo *;
 
@@ -300,6 +302,10 @@ class ASTIdentifierLookupTrait : public ASTIdentifierLookupTraitBase {
   IdentifierID ReadIdentifierID(const unsigned char *d);
 
   ASTReader &getReader() const { return Reader; }
+
+  bool hasMoreInformationInDependencies() const {
+    return hasMacroDefinitionInDependencies;
+  }
 };
 
 /// The on-disk hash table used to contain information about
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 95b5718f1d140..8c261e13d5ea4 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3795,7 +3795,10 @@ bool IsInterestingIdentifier(const IdentifierInfo *II, uint64_t MacroOffset,
       II->getNotableIdentifierID() != tok::NotableIdentifierKind::not_notable ||
       II->getBuiltinID() != Builtin::ID::NotBuiltin ||
       II->getObjCKeywordID() != tok::ObjCKeywordKind::objc_not_keyword;
-  if (MacroOffset || II->isPoisoned() || (!IsModule && IsInteresting) ||
+  if (MacroOffset ||
+      (II->hasMacroDefinition() &&
+       II->hasFETokenInfoChangedSinceDeserialization()) ||
+      II->isPoisoned() || (!IsModule && IsInteresting) ||
       II->hasRevertedTokenIDToIdentifier() ||
       (NeedDecls && II->getFETokenInfo()))
     return true;
@@ -3874,7 +3877,8 @@ class ASTIdentifierTableTrait {
     if (isInterestingIdentifier(II, MacroOffset)) {
       DataLen += 2; // 2 bytes for builtin ID
       DataLen += 2; // 2 bytes for flags
-      if (MacroOffset)
+      if (MacroOffset || (II->hasMacroDefinition() &&
+                          II->hasFETokenInfoChangedSinceDeserialization()))
         DataLen += 4; // MacroDirectives offset.
 
       if (NeedDecls && IdResolver)
@@ -3905,15 +3909,17 @@ class ASTIdentifierTableTrait {
     assert((Bits & 0xffff) == Bits && "ObjCOrBuiltinID too big for ASTReader.");
     LE.write<uint16_t>(Bits);
     Bits = 0;
-    bool HadMacroDefinition = MacroOffset != 0;
-    Bits = (Bits << 1) | unsigned(HadMacroDefinition);
+    bool HasMacroDefinition =
+        (MacroOffset != 0) || (II->hasMacroDefinition() &&
+                               II->hasFETokenInfoChangedSinceDeserialization());
+    Bits = (Bits << 1) | unsigned(HasMacroDefinition);
     Bits = (Bits << 1) | unsigned(II->isExtensionToken());
     Bits = (Bits << 1) | unsigned(II->isPoisoned());
     Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
     Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword());
     LE.write<uint16_t>(Bits);
 
-    if (HadMacroDefinition)
+    if (HasMacroDefinition)
       LE.write<uint32_t>(MacroOffset);
 
     if (NeedDecls && IdResolver) {
diff --git a/clang/test/Modules/macro-identifier-hiding.c b/clang/test/Modules/macro-identifier-hiding.c
new file mode 100644
index 0000000000000..4cd7cf0500322
--- /dev/null
+++ b/clang/test/Modules/macro-identifier-hiding.c
@@ -0,0 +1,29 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache \
+// RUN:            -fsyntax-only %t/test.c -verify
+// Test again with the populated module cache.
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache \
+// RUN:            -fsyntax-only %t/test.c -verify
+
+// Test that an identifier with the same name as a macro doesn't hide this
+// macro from the includers.
+
+//--- macro-definition.h
+#define __P(protos) ()
+#define __Q(protos) ()
+
+//--- macro-transitive.h
+#include "macro-definition.h"
+void test(int __P) {} // not "interesting" identifier
+struct __Q {};        // "interesting" identifier
+
+//--- module.modulemap
+module MacroDefinition { header "macro-definition.h" export * }
+module MacroTransitive { header "macro-transitive.h" export * }
+
+//--- test.c
+// expected-no-diagnostics
+#include "macro-transitive.h"
+void foo __P(());
+void bar __Q(());

From 913dcf1aa36f3ea2d67a0d2b05b9d1375987e553 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 16 Apr 2025 10:03:25 -0700
Subject: [PATCH 152/710] [SLP]Fix type promotion for smax reduction with
 unsigned reduced operands

Need to add an extra bit for sign info for unsigned reduced values to
generate correct code.
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp              | 3 ++-
 .../RISCV/smax-reduction-unsigned-missing-sign.ll            | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 810d44343c4a9..83252bdb51ea2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -20521,7 +20521,8 @@ void BoUpSLP::computeMinimumValueSizes() {
   }
   bool IsSignedCmp = false;
   if (UserIgnoreList && all_of(*UserIgnoreList, [](Value *V) {
-        return match(V, m_SMin(m_Value(), m_Value()));
+        return match(V, m_SMin(m_Value(), m_Value())) ||
+               match(V, m_SMax(m_Value(), m_Value()));
       }))
     IsSignedCmp = true;
   while (NodeIdx < VectorizableTree.size()) {
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/smax-reduction-unsigned-missing-sign.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/smax-reduction-unsigned-missing-sign.ll
index e6408572acf8f..a8efc2622aa80 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/smax-reduction-unsigned-missing-sign.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/smax-reduction-unsigned-missing-sign.ll
@@ -7,8 +7,9 @@ define i32 @test(i8 %0) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i8> <i8 poison, i8 0, i8 0, i8 0>, i8 [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <4 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.smax.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> [[TMP3]])
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP4]]
 ;
 entry:

From ce7466f66c8a279917cd1c9486846b6857a82fe8 Mon Sep 17 00:00:00 2001
From: AdityaK <hiraditya@msn.com>
Date: Wed, 16 Apr 2025 10:30:53 -0700
Subject: [PATCH 153/710] NFC: Rewrite auto castIter -> const auto *castIter
 (#133521)

---
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index d6b093c5fb86b..1c160911ce780 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -3598,7 +3598,7 @@ void MatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
   SmallVector<Value> yields;
 
   TypeFn castVal = TypeFn::cast_signed;
-  auto castIter = llvm::find_if(attrs, [&](const NamedAttribute &attr) {
+  const auto *castIter = llvm::find_if(attrs, [&](const NamedAttribute &attr) {
     return attr.getName() == "cast";
   });
   if (castIter != attrs.end()) {

From 80c19b3b1d59294be63d8b55fedc317305abbdbe Mon Sep 17 00:00:00 2001
From: Andy Kaylor <akaylor@nvidia.com>
Date: Wed, 16 Apr 2025 10:33:09 -0700
Subject: [PATCH 154/710] [CIR] Upstream initial support for complete record
 types (#135844)

This adds basic support for populating record types. In order to keep
the change small, everything non-essential was deferred to a later
change set. Only non-recursive structures are handled. Structures
padding is not yet implemented. Bitfields are not supported. No attempt
is made to handle ABI requirements for passing structure arguments.
---
 .../CIR/Dialect/Builder/CIRBaseBuilder.h      |   2 +
 .../include/clang/CIR/Dialect/IR/CIRTypes.td  |   3 +
 clang/include/clang/CIR/MissingFeatures.h     |   8 +
 clang/lib/CIR/CodeGen/CIRGenRecordLayout.h    |  54 +++++
 .../CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp | 223 ++++++++++++++++++
 clang/lib/CIR/CodeGen/CIRGenTypes.cpp         |  47 +++-
 clang/lib/CIR/CodeGen/CIRGenTypes.h           |  22 +-
 clang/lib/CIR/CodeGen/CMakeLists.txt          |   1 +
 clang/lib/CIR/Dialect/IR/CIRDialect.cpp       |   2 +-
 clang/lib/CIR/Dialect/IR/CIRTypes.cpp         |   6 +
 .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp |  29 +++
 clang/test/CIR/CodeGen/struct.c               |  27 +++
 12 files changed, 421 insertions(+), 3 deletions(-)
 create mode 100644 clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
 create mode 100644 clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp

diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index 68a4505ca7a5a..a787908fb9f8d 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -95,6 +95,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
       return getZeroAttr(arrTy);
     if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(ty))
       return getConstNullPtrAttr(ptrTy);
+    if (auto recordTy = mlir::dyn_cast<cir::RecordType>(ty))
+      return getZeroAttr(recordTy);
     if (mlir::isa<cir::BoolType>(ty)) {
       return getCIRBoolAttr(false);
     }
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
index c60af47f09def..23e20755dca95 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
@@ -499,6 +499,9 @@ def CIR_RecordType : CIR_Type<"Record", "record",
     std::string getPrefixedName() {
       return getKindAsStr() + "." + getName().getValue().str();
     }
+
+    void complete(llvm::ArrayRef<mlir::Type> members, bool packed,
+                  bool isPadded);
   }];
 
   let hasCustomAssemblyFormat = 1;
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 6f2fd2cb2b3ad..5ba55c53dfc4d 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -103,6 +103,14 @@ struct MissingFeatures {
 
   // RecordType
   static bool recordTypeLayoutInfo() { return false; }
+  static bool recursiveRecordLayout() { return false; }
+  static bool skippedLayout() { return false; }
+  static bool astRecordDeclAttr() { return false; }
+  static bool cxxSupport() { return false; }
+  static bool packedRecords() { return false; }
+  static bool recordPadding() { return false; }
+  static bool recordZeroInit() { return false; }
+  static bool zeroSizeRecordMembers() { return false; }
 
   // Misc
   static bool cxxABI() { return false; }
diff --git a/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h b/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
new file mode 100644
index 0000000000000..a51e0460d1074
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CIR_CIRGENRECORDLAYOUT_H
+#define LLVM_CLANG_LIB_CIR_CIRGENRECORDLAYOUT_H
+
+#include "clang/AST/Decl.h"
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
+
+namespace clang::CIRGen {
+
+/// This class handles record and union layout info while lowering AST types
+/// to CIR types.
+///
+/// These layout objects are only created on demand as CIR generation requires.
+class CIRGenRecordLayout {
+  friend class CIRGenTypes;
+
+  CIRGenRecordLayout(const CIRGenRecordLayout &) = delete;
+  void operator=(const CIRGenRecordLayout &) = delete;
+
+private:
+  /// The CIR type corresponding to this record layout; used when laying it out
+  /// as a complete object.
+  cir::RecordType completeObjectType;
+
+  /// Map from (non-bit-field) record field to the corresponding cir record type
+  /// field no. This info is populated by the record builder.
+  llvm::DenseMap<const clang::FieldDecl *, unsigned> fieldInfo;
+
+public:
+  CIRGenRecordLayout(cir::RecordType completeObjectType)
+      : completeObjectType(completeObjectType) {}
+
+  /// Return the "complete object" LLVM type associated with
+  /// this record.
+  cir::RecordType getCIRType() const { return completeObjectType; }
+
+  /// Return cir::RecordType element number that corresponds to the field FD.
+  unsigned getCIRFieldNo(const clang::FieldDecl *FD) const {
+    FD = FD->getCanonicalDecl();
+    assert(fieldInfo.count(FD) && "Invalid field for record!");
+    return fieldInfo.lookup(FD);
+  }
+};
+
+} // namespace clang::CIRGen
+
+#endif
diff --git a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
new file mode 100644
index 0000000000000..4e3adeaf50187
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
@@ -0,0 +1,223 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code to compute the layout of a record.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenBuilder.h"
+#include "CIRGenModule.h"
+#include "CIRGenTypes.h"
+
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/CIR/Dialect/IR/CIRAttrs.h"
+#include "llvm/Support/Casting.h"
+
+#include <memory>
+
+using namespace llvm;
+using namespace clang;
+using namespace clang::CIRGen;
+
+namespace {
+/// The CIRRecordLowering is responsible for lowering an ASTRecordLayout to an
+/// mlir::Type. Some of the lowering is straightforward, some is not.
+// TODO: Detail some of the complexities and weirdnesses?
+// (See CGRecordLayoutBuilder.cpp)
+struct CIRRecordLowering final {
+
+  // MemberInfo is a helper structure that contains information about a record
+  // member. In addition to the standard member types, there exists a sentinel
+  // member type that ensures correct rounding.
+  struct MemberInfo final {
+    CharUnits offset;
+    enum class InfoKind { Field } kind;
+    mlir::Type data;
+    union {
+      const FieldDecl *fieldDecl;
+      // CXXRecordDecl will be used here when base types are supported.
+    };
+    MemberInfo(CharUnits offset, InfoKind kind, mlir::Type data,
+               const FieldDecl *fieldDecl = nullptr)
+        : offset(offset), kind(kind), data(data), fieldDecl(fieldDecl) {};
+    // MemberInfos are sorted so we define a < operator.
+    bool operator<(const MemberInfo &other) const {
+      return offset < other.offset;
+    }
+  };
+  // The constructor.
+  CIRRecordLowering(CIRGenTypes &cirGenTypes, const RecordDecl *recordDecl,
+                    bool isPacked);
+
+  void lower();
+
+  void accumulateFields();
+
+  CharUnits bitsToCharUnits(uint64_t bitOffset) {
+    return astContext.toCharUnitsFromBits(bitOffset);
+  }
+
+  CharUnits getSize(mlir::Type Ty) {
+    assert(!cir::MissingFeatures::recordTypeLayoutInfo());
+    return CharUnits::One();
+  }
+  CharUnits getAlignment(mlir::Type Ty) {
+    assert(!cir::MissingFeatures::recordTypeLayoutInfo());
+    return CharUnits::One();
+  }
+
+  mlir::Type getStorageType(const FieldDecl *fieldDecl) {
+    mlir::Type type = cirGenTypes.convertTypeForMem(fieldDecl->getType());
+    if (fieldDecl->isBitField()) {
+      cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
+                                         "getStorageType for bitfields");
+    }
+    return type;
+  }
+
+  uint64_t getFieldBitOffset(const FieldDecl *fieldDecl) {
+    return astRecordLayout.getFieldOffset(fieldDecl->getFieldIndex());
+  }
+
+  /// Fills out the structures that are ultimately consumed.
+  void fillOutputFields();
+
+  CIRGenTypes &cirGenTypes;
+  CIRGenBuilderTy &builder;
+  const ASTContext &astContext;
+  const RecordDecl *recordDecl;
+  const ASTRecordLayout &astRecordLayout;
+  // Helpful intermediate data-structures
+  std::vector<MemberInfo> members;
+  // Output fields, consumed by CIRGenTypes::computeRecordLayout
+  llvm::SmallVector<mlir::Type, 16> fieldTypes;
+  llvm::DenseMap<const FieldDecl *, unsigned> fields;
+
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned zeroInitializable : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned packed : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned padded : 1;
+
+private:
+  CIRRecordLowering(const CIRRecordLowering &) = delete;
+  void operator=(const CIRRecordLowering &) = delete;
+}; // CIRRecordLowering
+} // namespace
+
+CIRRecordLowering::CIRRecordLowering(CIRGenTypes &cirGenTypes,
+                                     const RecordDecl *recordDecl,
+                                     bool isPacked)
+    : cirGenTypes(cirGenTypes), builder(cirGenTypes.getBuilder()),
+      astContext(cirGenTypes.getASTContext()), recordDecl(recordDecl),
+      astRecordLayout(
+          cirGenTypes.getASTContext().getASTRecordLayout(recordDecl)),
+      zeroInitializable(true), packed(isPacked), padded(false) {}
+
+void CIRRecordLowering::lower() {
+  if (recordDecl->isUnion()) {
+    cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
+                                       "lower: union");
+    return;
+  }
+
+  if (isa<CXXRecordDecl>(recordDecl)) {
+    cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
+                                       "lower: class");
+    return;
+  }
+
+  assert(!cir::MissingFeatures::cxxSupport());
+
+  accumulateFields();
+
+  llvm::stable_sort(members);
+  // TODO: implement clipTailPadding once bitfields are implemented
+  assert(!cir::MissingFeatures::bitfields());
+  // TODO: implemented packed records
+  assert(!cir::MissingFeatures::packedRecords());
+  // TODO: implement padding
+  assert(!cir::MissingFeatures::recordPadding());
+  // TODO: support zeroInit
+  assert(!cir::MissingFeatures::recordZeroInit());
+
+  fillOutputFields();
+}
+
+void CIRRecordLowering::fillOutputFields() {
+  for (const MemberInfo &member : members) {
+    if (member.data)
+      fieldTypes.push_back(member.data);
+    if (member.kind == MemberInfo::InfoKind::Field) {
+      if (member.fieldDecl)
+        fields[member.fieldDecl->getCanonicalDecl()] = fieldTypes.size() - 1;
+      // A field without storage must be a bitfield.
+      assert(!cir::MissingFeatures::bitfields());
+    }
+    assert(!cir::MissingFeatures::cxxSupport());
+  }
+}
+
+void CIRRecordLowering::accumulateFields() {
+  for (const FieldDecl *field : recordDecl->fields()) {
+    if (field->isBitField()) {
+      cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
+                                         "accumulate bitfields");
+      ++field;
+    } else if (!field->isZeroSize(astContext)) {
+      members.push_back(MemberInfo(bitsToCharUnits(getFieldBitOffset(field)),
+                                   MemberInfo::InfoKind::Field,
+                                   getStorageType(field), field));
+      ++field;
+    } else {
+      // TODO(cir): do we want to do anything special about zero size members?
+      assert(!cir::MissingFeatures::zeroSizeRecordMembers());
+      ++field;
+    }
+  }
+}
+
+std::unique_ptr<CIRGenRecordLayout>
+CIRGenTypes::computeRecordLayout(const RecordDecl *rd, cir::RecordType *ty) {
+  CIRRecordLowering lowering(*this, rd, /*packed=*/false);
+  assert(ty->isIncomplete() && "recomputing record layout?");
+  lowering.lower();
+
+  // If we're in C++, compute the base subobject type.
+  if (llvm::isa<CXXRecordDecl>(rd) && !rd->isUnion() &&
+      !rd->hasAttr<FinalAttr>()) {
+    cgm.errorNYI(rd->getSourceRange(), "computeRecordLayout: CXXRecordDecl");
+  }
+
+  // Fill in the record *after* computing the base type.  Filling in the body
+  // signifies that the type is no longer opaque and record layout is complete,
+  // but we may need to recursively layout rd while laying D out as a base type.
+  assert(!cir::MissingFeatures::astRecordDeclAttr());
+  ty->complete(lowering.fieldTypes, lowering.packed, lowering.padded);
+
+  auto rl = std::make_unique<CIRGenRecordLayout>(ty ? *ty : cir::RecordType());
+
+  assert(!cir::MissingFeatures::recordZeroInit());
+  assert(!cir::MissingFeatures::cxxSupport());
+  assert(!cir::MissingFeatures::bitfields());
+
+  // Add all the field numbers.
+  rl->fieldInfo.swap(lowering.fields);
+
+  // Dump the layout, if requested.
+  if (getASTContext().getLangOpts().DumpRecordLayouts) {
+    cgm.errorNYI(rd->getSourceRange(), "computeRecordLayout: dump layout");
+  }
+
+  // TODO: implement verification
+  return rl;
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index f625f83257859..ec77c4428d43b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -112,6 +112,18 @@ std::string CIRGenTypes::getRecordTypeName(const clang::RecordDecl *recordDecl,
   return builder.getUniqueRecordName(std::string(typeName));
 }
 
+// Return true if it is safe to convert the specified record decl to CIR and lay
+// it out, false if doing so would cause us to get into a recursive compilation
+// mess.
+static bool isSafeToConvert(const RecordDecl *RD, CIRGenTypes &CGT) {
+  // If no records are being laid out, we can certainly do this one.
+  if (CGT.noRecordsBeingLaidOut())
+    return true;
+
+  assert(!cir::MissingFeatures::recursiveRecordLayout());
+  return false;
+}
+
 /// Lay out a tagged decl type like struct or union.
 mlir::Type CIRGenTypes::convertRecordDeclType(const clang::RecordDecl *rd) {
   // TagDecl's are not necessarily unique, instead use the (clang) type
@@ -132,7 +144,40 @@ mlir::Type CIRGenTypes::convertRecordDeclType(const clang::RecordDecl *rd) {
   if (!rd || !rd->isCompleteDefinition() || entry.isComplete())
     return entry;
 
-  cgm.errorNYI(rd->getSourceRange(), "Complete record type");
+  // If converting this type would cause us to infinitely loop, don't do it!
+  if (!isSafeToConvert(rd, *this)) {
+    cgm.errorNYI(rd->getSourceRange(), "recursive record layout");
+    return entry;
+  }
+
+  // Okay, this is a definition of a type. Compile the implementation now.
+  bool insertResult = recordsBeingLaidOut.insert(key).second;
+  (void)insertResult;
+  assert(insertResult && "isSafeToCovert() should have caught this.");
+
+  // Force conversion of non-virtual base classes recursively.
+  if (const auto *cxxRecordDecl = dyn_cast<CXXRecordDecl>(rd)) {
+    cgm.errorNYI(rd->getSourceRange(), "CXXRecordDecl");
+  }
+
+  // Layout fields.
+  std::unique_ptr<CIRGenRecordLayout> layout = computeRecordLayout(rd, &entry);
+  recordDeclTypes[key] = entry;
+  cirGenRecordLayouts[key] = std::move(layout);
+
+  // We're done laying out this record.
+  bool eraseResult = recordsBeingLaidOut.erase(key);
+  (void)eraseResult;
+  assert(eraseResult && "record not in RecordsBeingLaidOut set?");
+
+  // If this record blocked a FunctionType conversion, then recompute whatever
+  // was derived from that.
+  assert(!cir::MissingFeatures::skippedLayout());
+
+  // If we're done converting the outer-most record, then convert any deferred
+  // records as well.
+  assert(!cir::MissingFeatures::recursiveRecordLayout());
+
   return entry;
 }
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.h b/clang/lib/CIR/CodeGen/CIRGenTypes.h
index fd855bf958ccb..38e4bb2f688ab 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.h
@@ -14,9 +14,10 @@
 #define LLVM_CLANG_LIB_CODEGEN_CODEGENTYPES_H
 
 #include "CIRGenFunctionInfo.h"
-#include "clang/CIR/Dialect/IR/CIRTypes.h"
+#include "CIRGenRecordLayout.h"
 
 #include "clang/AST/Type.h"
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
 
 #include "llvm/ADT/SmallPtrSet.h"
 
@@ -45,12 +46,22 @@ class CIRGenTypes {
   clang::ASTContext &astContext;
   CIRGenBuilderTy &builder;
 
+  /// Contains the CIR type for any converted RecordDecl.
+  llvm::DenseMap<const clang::Type *, std::unique_ptr<CIRGenRecordLayout>>
+      cirGenRecordLayouts;
+
   /// Contains the CIR type for any converted RecordDecl
   llvm::DenseMap<const clang::Type *, cir::RecordType> recordDeclTypes;
 
   /// Hold memoized CIRGenFunctionInfo results
   llvm::FoldingSet<CIRGenFunctionInfo> functionInfos;
 
+  /// This set keeps track of records that we're currently converting to a CIR
+  /// type. For example, when converting:
+  /// struct A { struct B { int x; } } when processing 'x', the 'A' and 'B'
+  /// types will be in this set.
+  llvm::SmallPtrSet<const clang::Type *, 4> recordsBeingLaidOut;
+
   llvm::SmallPtrSet<const CIRGenFunctionInfo *, 4> functionsBeingProcessed;
   /// Heper for convertType.
   mlir::Type convertFunctionTypeInternal(clang::QualType ft);
@@ -59,6 +70,9 @@ class CIRGenTypes {
   CIRGenTypes(CIRGenModule &cgm);
   ~CIRGenTypes();
 
+  CIRGenBuilderTy &getBuilder() const { return builder; }
+  CIRGenModule &getCGModule() const { return cgm; }
+
   /// Utility to check whether a function type can be converted to a CIR type
   /// (i.e. doesn't depend on an incomplete tag type).
   bool isFuncTypeConvertible(const clang::FunctionType *ft);
@@ -70,12 +84,18 @@ class CIRGenTypes {
   TypeCacheTy typeCache;
 
   mlir::MLIRContext &getMLIRContext() const;
+  clang::ASTContext &getASTContext() const { return astContext; }
+
+  bool noRecordsBeingLaidOut() const { return recordsBeingLaidOut.empty(); }
 
   /// Convert a Clang type into a mlir::Type.
   mlir::Type convertType(clang::QualType type);
 
   mlir::Type convertRecordDeclType(const clang::RecordDecl *recordDecl);
 
+  std::unique_ptr<CIRGenRecordLayout>
+  computeRecordLayout(const clang::RecordDecl *rd, cir::RecordType *ty);
+
   std::string getRecordTypeName(const clang::RecordDecl *,
                                 llvm::StringRef suffix);
 
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index dc18f7f2af160..418bc2db408cb 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -18,6 +18,7 @@ add_clang_library(clangCIR
   CIRGenExprScalar.cpp
   CIRGenFunction.cpp
   CIRGenModule.cpp
+  CIRGenRecordLayoutBuilder.cpp
   CIRGenStmt.cpp
   CIRGenStmtOpenACC.cpp
   CIRGenTypes.cpp
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index f3e5e572653da..1f4232b9e29ec 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -212,7 +212,7 @@ static LogicalResult checkConstantTypes(mlir::Operation *op, mlir::Type opType,
   }
 
   if (isa<cir::ZeroAttr>(attrType)) {
-    if (::mlir::isa<cir::ArrayType>(opType))
+    if (isa<cir::RecordType, cir::ArrayType>(opType))
       return success();
     return op->emitOpError("zero expects struct or array type");
   }
diff --git a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
index 9b0177f159084..160732d9c3610 100644
--- a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
@@ -214,6 +214,12 @@ cir::RecordType::RecordKind RecordType::getKind() const {
   return getImpl()->kind;
 }
 
+void RecordType::complete(ArrayRef<Type> members, bool packed, bool padded) {
+  assert(!cir::MissingFeatures::astRecordDeclAttr());
+  if (mutate(members, packed, padded).failed())
+    llvm_unreachable("failed to complete record");
+}
+
 //===----------------------------------------------------------------------===//
 // Data Layout information for types
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 7159f89c93a53..cb318c88c09c5 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -1337,6 +1337,35 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
   converter.addConversion([&](cir::BF16Type type) -> mlir::Type {
     return mlir::BFloat16Type::get(type.getContext());
   });
+  converter.addConversion([&](cir::RecordType type) -> mlir::Type {
+    // Convert struct members.
+    llvm::SmallVector<mlir::Type> llvmMembers;
+    switch (type.getKind()) {
+    case cir::RecordType::Struct:
+      for (mlir::Type ty : type.getMembers())
+        llvmMembers.push_back(convertTypeForMemory(converter, dataLayout, ty));
+      break;
+    // Unions are lowered as only the largest member.
+    case cir::RecordType::Union:
+      llvm_unreachable("Lowering of unions is NYI");
+      break;
+    }
+
+    // Record has a name: lower as an identified record.
+    mlir::LLVM::LLVMStructType llvmStruct;
+    if (type.getName()) {
+      llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(
+          type.getContext(), type.getPrefixedName());
+      assert(!cir::MissingFeatures::packedRecords());
+      if (llvmStruct.setBody(llvmMembers, /*isPacked=*/true).failed())
+        llvm_unreachable("Failed to set body of record");
+    } else { // Record has no name: lower as literal record.
+      llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(
+          type.getContext(), llvmMembers, /*isPacked=*/true);
+    }
+
+    return llvmStruct;
+  });
 }
 
 // The applyPartialConversion function traverses blocks in the dominance order,
diff --git a/clang/test/CIR/CodeGen/struct.c b/clang/test/CIR/CodeGen/struct.c
index 4edd591e609a2..e1b01e6a8e86c 100644
--- a/clang/test/CIR/CodeGen/struct.c
+++ b/clang/test/CIR/CodeGen/struct.c
@@ -11,6 +11,15 @@ struct IncompleteS *p;
 // LLVM: @p = dso_local global ptr null
 // OGCG: @p = global ptr null, align 8
 
+struct CompleteS {
+  int a;
+  char b;
+} cs;
+
+// CIR:      cir.global external @cs = #cir.zero : !cir.record<struct "CompleteS" {!s32i, !s8i}>
+// LLVM:      @cs = dso_local global %struct.CompleteS zeroinitializer
+// OGCG:      @cs = global %struct.CompleteS zeroinitializer, align 4
+
 void f(void) {
   struct IncompleteS *p;
 }
@@ -28,3 +37,21 @@ void f(void) {
 // OGCG-NEXT: entry:
 // OGCG-NEXT:   %[[P:.*]] = alloca ptr, align 8
 // OGCG-NEXT:   ret void
+
+void f2(void) {
+  struct CompleteS s;
+}
+
+// CIR:      cir.func @f2()
+// CIR-NEXT:   cir.alloca !cir.record<struct "CompleteS" {!s32i, !s8i}>,
+// CIR-SAME:       !cir.ptr<!cir.record<struct "CompleteS" {!s32i, !s8i}>>, ["s"]
+// CIR-NEXT:   cir.return
+
+// LLVM:      define void @f2()
+// LLVM-NEXT:   %[[S:.*]] = alloca %struct.CompleteS, i64 1, align 4
+// LLVM-NEXT:   ret void
+
+// OGCG:      define{{.*}} void @f2()
+// OGCG-NEXT: entry:
+// OGCG-NEXT:   %[[S:.*]] = alloca %struct.CompleteS, align 4
+// OGCG-NEXT:   ret void

From 1576fa10104b9a88bef88ae851c2df479502fea9 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Wed, 16 Apr 2025 10:39:34 -0700
Subject: [PATCH 155/710] [ctxprof] Extend the notion of "cannot return"
 (#135651)

At the time of instrumentation (and instrumentation lowering), `noreturn` is not applied uniformously. Rather than running `FunctionAttrs` pass, we just need to use `llvm::canReturn` exposed in PR #135650
---
 .../Instrumentation/PGOCtxProfLowering.cpp    | 19 ++++++++------
 .../ctx-instrumentation-invalid-roots.ll      | 25 +++++++++++--------
 .../PGOProfile/ctx-instrumentation.ll         | 13 ++++++++++
 3 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
index f99d7b9d03e02..d741695d4e53c 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
@@ -9,6 +9,7 @@
 
 #include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/CtxProfAnalysis.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/IR/Analysis.h"
@@ -105,6 +106,12 @@ std::pair<uint32_t, uint32_t> getNumCountersAndCallsites(const Function &F) {
   }
   return {NumCounters, NumCallsites};
 }
+
+void emitUnsupportedRootError(const Function &F, StringRef Reason) {
+  F.getContext().emitError("[ctxprof] The function " + F.getName() +
+                           " was indicated as context root but " + Reason +
+                           ", which is not supported.");
+}
 } // namespace
 
 // set up tie-in with compiler-rt.
@@ -164,12 +171,8 @@ CtxInstrumentationLowerer::CtxInstrumentationLowerer(Module &M,
       for (const auto &BB : *F)
         for (const auto &I : BB)
           if (const auto *CB = dyn_cast<CallBase>(&I))
-            if (CB->isMustTailCall()) {
-              M.getContext().emitError("The function " + Fname +
-                                       " was indicated as a context root, "
-                                       "but it features musttail "
-                                       "calls, which is not supported.");
-            }
+            if (CB->isMustTailCall())
+              emitUnsupportedRootError(*F, "it features musttail calls");
     }
   }
 
@@ -230,11 +233,13 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) {
 
   // Probably pointless to try to do anything here, unlikely to be
   // performance-affecting.
-  if (F.doesNotReturn()) {
+  if (!llvm::canReturn(F)) {
     for (auto &BB : F)
       for (auto &I : make_early_inc_range(BB))
         if (isa<InstrProfCntrInstBase>(&I))
           I.eraseFromParent();
+    if (ContextRootSet.contains(&F))
+      emitUnsupportedRootError(F, "it does not return");
     return true;
   }
 
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
index 454780153b823..b5ceb4602c60b 100644
--- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
@@ -1,17 +1,22 @@
-; RUN: not opt -passes=ctx-instr-gen,ctx-instr-lower -profile-context-root=good \
-; RUN:   -profile-context-root=bad \
-; RUN:   -S < %s 2>&1 | FileCheck %s
+; RUN: split-file %s %t
+; RUN: not opt -passes=ctx-instr-gen,ctx-instr-lower -profile-context-root=the_func -S %t/musttail.ll -o - 2>&1 | FileCheck %s
+; RUN: not opt -passes=ctx-instr-gen,ctx-instr-lower -profile-context-root=the_func -S %t/unreachable.ll -o - 2>&1 | FileCheck %s
+; RUN: not opt -passes=ctx-instr-gen,ctx-instr-lower -profile-context-root=the_func -S %t/noreturn.ll -o - 2>&1 | FileCheck %s
 
+;--- musttail.ll
 declare void @foo()
 
-define void @good() {
-  call void @foo()
-  ret void
-}
-
-define void @bad() {
+define void @the_func() {
   musttail call void @foo()
   ret void
 }
+;--- unreachable.ll
+define void @the_func() {
+  unreachable
+}
+;--- noreturn.ll
+define void @the_func() noreturn {
+  unreachable
+}
 
-; CHECK: error: The function bad was indicated as a context root, but it features musttail calls, which is not supported.
+; CHECK: error: [ctxprof] The function the_func was indicated as context root
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
index 6b2f25a585ec3..71d54f98d26e1 100644
--- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
@@ -323,6 +323,18 @@ define void @does_not_return() noreturn {
 ;
   unreachable
 }
+
+define void @unreachable() {
+; INSTRUMENT-LABEL: define void @unreachable() {
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @unreachable, i64 742261418966908927, i32 1, i32 0)
+; INSTRUMENT-NEXT:    unreachable
+;
+; LOWERING-LABEL: define void @unreachable(
+; LOWERING-SAME: ) !guid [[META9:![0-9]+]] {
+; LOWERING-NEXT:    unreachable
+;
+  unreachable
+}
 ;.
 ; LOWERING: attributes #[[ATTR0]] = { noreturn }
 ; LOWERING: attributes #[[ATTR1:[0-9]+]] = { nounwind }
@@ -340,4 +352,5 @@ define void @does_not_return() noreturn {
 ; LOWERING: [[META6]] = !{i64 -3771893999295659109}
 ; LOWERING: [[META7]] = !{i64 -4680624981836544329}
 ; LOWERING: [[META8]] = !{i64 5519225910966780583}
+; LOWERING: [[META9]] = !{i64 -565652589829076809}
 ;.

From 6ccc9280ba891bbea349c12a064bf23bdf9000e7 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Wed, 16 Apr 2025 19:40:28 +0200
Subject: [PATCH 156/710] Revert "[Clang][RFC] Bypass TAD during overload
 resolution if a perfect match exists" (#135993)

Reverts llvm/llvm-project#133426

This is failing on some bots
https://lab.llvm.org/buildbot/#/builders/163/builds/17265
---
 clang/docs/ReleaseNotes.rst                   |   6 -
 clang/include/clang/Sema/Overload.h           | 224 +-------
 clang/lib/Sema/SemaCodeComplete.cpp           |   6 +-
 clang/lib/Sema/SemaInit.cpp                   |  15 +-
 clang/lib/Sema/SemaOverload.cpp               | 522 ++++--------------
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   4 +-
 .../constrant-satisfaction-conversions.cpp    |   8 +-
 clang/test/SemaCUDA/function-overload.cu      |   3 +
 .../SemaCXX/implicit-member-functions.cpp     |  21 +-
 ...overload-resolution-deferred-templates.cpp | 185 -------
 .../instantiate-function-params.cpp           |   7 +-
 .../Templight/templight-empty-entries-fix.cpp | 126 +++--
 12 files changed, 217 insertions(+), 910 deletions(-)
 delete mode 100644 clang/test/SemaCXX/overload-resolution-deferred-templates.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index acbc9c5a6fac9..0891fd058bb57 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -96,12 +96,6 @@ C++ Language Changes
       asm((std::string_view("nop")) ::: (std::string_view("memory")));
     }
 
-- Clang now implements the changes to overload resolution proposed by section 1 and 2 of
-  `P3606 <https://wg21.link/P3606R0>`_. If a non-template candidate exists in an overload set that is
-  a perfect match (all conversion sequences are identity conversions) template candiates are not instantiated.
-  Diagnostics that would have resulted from the instantiation of these template candidates are no longer
-  produced. This aligns Clang closer to the behavior of GCC, and fixes (#GH62096), (#GH74581), and (#GH74581).
-
 C++2c Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 813811af06e89..6e08762dcc6d7 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -407,24 +407,6 @@ class Sema;
              Third == ICK_Identity;
     }
 
-    /// A conversion sequence is perfect if it is an identity conversion and
-    /// the type of the source is the same as the type of the target.
-    bool isPerfect(const ASTContext &C) const {
-      if (!isIdentityConversion())
-        return false;
-      // If we are not performing a reference binding, we can skip comparing
-      // the types, which has a noticeable performance impact.
-      if (!ReferenceBinding) {
-        assert(First || C.hasSameUnqualifiedType(getFromType(), getToType(2)));
-        return true;
-      }
-      if (!C.hasSameType(getFromType(), getToType(2)))
-        return false;
-      if (BindsToRvalue && IsLvalueReference)
-        return false;
-      return true;
-    }
-
     ImplicitConversionRank getRank() const;
     NarrowingKind
     getNarrowingKind(ASTContext &Context, const Expr *Converted,
@@ -761,12 +743,6 @@ class Sema;
       Standard.setAllToTypes(T);
     }
 
-    /// A conversion sequence is perfect if it is an identity conversion and
-    /// the type of the source is the same as the type of the target.
-    bool isPerfect(const ASTContext &C) const {
-      return isStandard() && Standard.isPerfect(C);
-    }
-
     // True iff this is a conversion sequence from an initializer list to an
     // array or std::initializer.
     bool hasInitializerListContainerType() const {
@@ -1003,20 +979,6 @@ class Sema;
       return false;
     }
 
-    // An overload is a perfect match if the conversion
-    // sequences for each argument are perfect.
-    bool isPerfectMatch(const ASTContext &Ctx) const {
-      if (!Viable)
-        return false;
-      for (const auto &C : Conversions) {
-        if (!C.isInitialized() || !C.isPerfect(Ctx))
-          return false;
-      }
-      if (isa_and_nonnull<CXXConversionDecl>(Function))
-        return FinalConversion.isPerfect(Ctx);
-      return true;
-    }
-
     bool TryToFixBadConversion(unsigned Idx, Sema &S) {
       bool CanFix = Fix.tryToFixConversion(
                       Conversions[Idx].Bad.FromExpr,
@@ -1053,65 +1015,6 @@ class Sema;
           RewriteKind(CRK_None) {}
   };
 
-  struct DeferredTemplateOverloadCandidate {
-
-    // intrusive linked list support for allocateDeferredCandidate
-    DeferredTemplateOverloadCandidate *Next = nullptr;
-
-    enum Kind { Function, Method, Conversion };
-
-    LLVM_PREFERRED_TYPE(Kind)
-    unsigned Kind : 2;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned AllowObjCConversionOnExplicit : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned AllowResultConversion : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned AllowExplicit : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned SuppressUserConversions : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned PartialOverloading : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned AggregateCandidateDeduction : 1;
-  };
-
-  struct DeferredFunctionTemplateOverloadCandidate
-      : public DeferredTemplateOverloadCandidate {
-    FunctionTemplateDecl *FunctionTemplate;
-    DeclAccessPair FoundDecl;
-    ArrayRef<Expr *> Args;
-    CallExpr::ADLCallKind IsADLCandidate;
-    OverloadCandidateParamOrder PO;
-  };
-  static_assert(std::is_trivially_destructible_v<
-                DeferredFunctionTemplateOverloadCandidate>);
-
-  struct DeferredMethodTemplateOverloadCandidate
-      : public DeferredTemplateOverloadCandidate {
-    FunctionTemplateDecl *FunctionTemplate;
-    DeclAccessPair FoundDecl;
-    ArrayRef<Expr *> Args;
-    CXXRecordDecl *ActingContext;
-    Expr::Classification ObjectClassification;
-    QualType ObjectType;
-    OverloadCandidateParamOrder PO;
-  };
-  static_assert(std::is_trivially_destructible_v<
-                DeferredMethodTemplateOverloadCandidate>);
-
-  struct DeferredConversionTemplateOverloadCandidate
-      : public DeferredTemplateOverloadCandidate {
-    FunctionTemplateDecl *FunctionTemplate;
-    DeclAccessPair FoundDecl;
-    CXXRecordDecl *ActingContext;
-    Expr *From;
-    QualType ToType;
-  };
-
-  static_assert(std::is_trivially_destructible_v<
-                DeferredConversionTemplateOverloadCandidate>);
-
   /// OverloadCandidateSet - A set of overload candidates, used in C++
   /// overload resolution (C++ 13.3).
   class OverloadCandidateSet {
@@ -1140,11 +1043,6 @@ class Sema;
       /// C++ [over.match.call.general]
       /// Resolve a call through the address of an overload set.
       CSK_AddressOfOverloadSet,
-
-      /// When doing overload resolution during code completion,
-      /// we want to show all viable candidates, including otherwise
-      /// deferred template candidates.
-      CSK_CodeCompletion,
     };
 
     /// Information about operator rewrites to consider when adding operator
@@ -1219,15 +1117,7 @@ class Sema;
     SmallVector<OverloadCandidate, 16> Candidates;
     llvm::SmallPtrSet<uintptr_t, 16> Functions;
 
-    DeferredTemplateOverloadCandidate *FirstDeferredCandidate = nullptr;
-    unsigned DeferredCandidatesCount : 8 * sizeof(unsigned) - 2;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned HasDeferredTemplateConstructors : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned ResolutionByPerfectCandidateIsDisabled : 1;
-
-    // Allocator for ConversionSequenceLists and deferred candidate args.
-    // We store the first few of these
+    // Allocator for ConversionSequenceLists. We store the first few of these
     // inline to avoid allocation for small sets.
     llvm::BumpPtrAllocator SlabAllocator;
 
@@ -1235,11 +1125,8 @@ class Sema;
     CandidateSetKind Kind;
     OperatorRewriteInfo RewriteInfo;
 
-    /// Small storage size for ImplicitConversionSequences
-    /// and the persisted arguments of deferred candidates.
     constexpr static unsigned NumInlineBytes =
-        32 * sizeof(ImplicitConversionSequence);
-
+        24 * sizeof(ImplicitConversionSequence);
     unsigned NumInlineBytesUsed = 0;
     alignas(void *) char InlineSpace[NumInlineBytes];
 
@@ -1250,13 +1137,15 @@ class Sema;
     /// from the slab allocator.
     /// FIXME: It would probably be nice to have a SmallBumpPtrAllocator
     /// instead.
+    /// FIXME: Now that this only allocates ImplicitConversionSequences, do we
+    /// want to un-generalize this?
     template <typename T>
     T *slabAllocate(unsigned N) {
       // It's simpler if this doesn't need to consider alignment.
       static_assert(alignof(T) == alignof(void *),
                     "Only works for pointer-aligned types.");
-      static_assert(std::is_trivially_destructible_v<T> ||
-                        (std::is_same_v<ImplicitConversionSequence, T>),
+      static_assert(std::is_trivial<T>::value ||
+                        std::is_same<ImplicitConversionSequence, T>::value,
                     "Add destruction logic to OverloadCandidateSet::clear().");
 
       unsigned NBytes = sizeof(T) * N;
@@ -1270,34 +1159,12 @@ class Sema;
       return reinterpret_cast<T *>(FreeSpaceStart);
     }
 
-    // Because the size of OverloadCandidateSet has a noticeable impact on
-    // performance, we store each deferred template candidate in the slab
-    // allocator such that deferred candidates are ultimately a singly-linked
-    // intrusive linked list. This ends up being much more efficient than a
-    // SmallVector that is empty in the common case.
-    template <typename T> T *allocateDeferredCandidate() {
-      T *C = slabAllocate<T>(1);
-      if (!FirstDeferredCandidate)
-        FirstDeferredCandidate = C;
-      else {
-        auto *F = FirstDeferredCandidate;
-        while (F->Next)
-          F = F->Next;
-        F->Next = C;
-      }
-      DeferredCandidatesCount++;
-      return C;
-    }
-
     void destroyCandidates();
 
   public:
     OverloadCandidateSet(SourceLocation Loc, CandidateSetKind CSK,
                          OperatorRewriteInfo RewriteInfo = {})
-        : FirstDeferredCandidate(nullptr), DeferredCandidatesCount(0),
-          HasDeferredTemplateConstructors(false),
-          ResolutionByPerfectCandidateIsDisabled(false), Loc(Loc), Kind(CSK),
-          RewriteInfo(RewriteInfo) {}
+        : Loc(Loc), Kind(CSK), RewriteInfo(RewriteInfo) {}
     OverloadCandidateSet(const OverloadCandidateSet &) = delete;
     OverloadCandidateSet &operator=(const OverloadCandidateSet &) = delete;
     ~OverloadCandidateSet() { destroyCandidates(); }
@@ -1309,9 +1176,6 @@ class Sema;
     /// Whether diagnostics should be deferred.
     bool shouldDeferDiags(Sema &S, ArrayRef<Expr *> Args, SourceLocation OpLoc);
 
-    // Whether the resolution of template candidates should be deferred
-    bool shouldDeferTemplateArgumentDeduction(const LangOptions &Opts) const;
-
     /// Determine when this overload candidate will be new to the
     /// overload set.
     bool isNewCandidate(Decl *F, OverloadCandidateParamOrder PO =
@@ -1335,10 +1199,8 @@ class Sema;
     iterator begin() { return Candidates.begin(); }
     iterator end() { return Candidates.end(); }
 
-    size_t size() const { return Candidates.size() + DeferredCandidatesCount; }
-    bool empty() const {
-      return Candidates.empty() && DeferredCandidatesCount == 0;
-    }
+    size_t size() const { return Candidates.size(); }
+    bool empty() const { return Candidates.empty(); }
 
     /// Allocate storage for conversion sequences for NumConversions
     /// conversions.
@@ -1354,24 +1216,6 @@ class Sema;
       return ConversionSequenceList(Conversions, NumConversions);
     }
 
-    /// Provide storage for any Expr* arg that must be preserved
-    /// until deferred template candidates are deduced.
-    /// Typically this should be used for reversed operator arguments
-    /// and any time the argument array is transformed while adding
-    /// a template candidate.
-    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(unsigned N) {
-      Expr **Exprs = slabAllocate<Expr *>(N);
-      return llvm::MutableArrayRef<Expr *>(Exprs, N);
-    }
-
-    template <typename... T>
-    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(T *...Exprs) {
-      llvm::MutableArrayRef<Expr *> Arr =
-          getPersistentArgsArray(sizeof...(Exprs));
-      llvm::copy(std::initializer_list<Expr *>{Exprs...}, Arr.data());
-      return Arr;
-    }
-
     /// Add a new candidate with NumConversions conversion sequence slots
     /// to the overload set.
     OverloadCandidate &addCandidate(unsigned NumConversions = 0,
@@ -1387,32 +1231,6 @@ class Sema;
       return C;
     }
 
-    void AddDeferredTemplateCandidate(
-        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-        ArrayRef<Expr *> Args, bool SuppressUserConversions,
-        bool PartialOverloading, bool AllowExplicit,
-        CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
-        bool AggregateCandidateDeduction);
-
-    void AddDeferredMethodTemplateCandidate(
-        FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
-        CXXRecordDecl *ActingContext, QualType ObjectType,
-        Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-        bool SuppressUserConversions, bool PartialOverloading,
-        OverloadCandidateParamOrder PO);
-
-    void AddDeferredConversionTemplateCandidate(
-        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-        CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
-        bool AllowObjCConversionOnExplicit, bool AllowExplicit,
-        bool AllowResultConversion);
-
-    void InjectNonDeducedTemplateCandidates(Sema &S);
-
-    void DisableResolutionByPerfectCandidate() {
-      ResolutionByPerfectCandidateIsDisabled = true;
-    }
-
     /// Find the best viable function on this overload set, if it exists.
     OverloadingResult BestViableFunction(Sema &S, SourceLocation Loc,
                                          OverloadCandidateSet::iterator& Best);
@@ -1445,15 +1263,6 @@ class Sema;
       DestAS = AS;
     }
 
-  private:
-    OverloadingResult ResultForBestCandidate(const iterator &Best);
-    void CudaExcludeWrongSideCandidates(
-        Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates);
-    OverloadingResult
-    BestViableFunctionImpl(Sema &S, SourceLocation Loc,
-                           OverloadCandidateSet::iterator &Best);
-    void PerfectViableFunction(Sema &S, SourceLocation Loc,
-                               OverloadCandidateSet::iterator &Best);
   };
 
   bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
@@ -1502,21 +1311,6 @@ class Sema;
   // parameter.
   bool shouldEnforceArgLimit(bool PartialOverloading, FunctionDecl *Function);
 
-  inline bool OverloadCandidateSet::shouldDeferTemplateArgumentDeduction(
-      const LangOptions &Opts) const {
-    return
-        // For user defined conversion we need to check against different
-        // combination of CV qualifiers and look at any explicit specifier, so
-        // always deduce template candidates.
-        Kind != CSK_InitByUserDefinedConversion
-        // When doing code completion, we want to see all the
-        // viable candidates.
-        && Kind != CSK_CodeCompletion
-        // CUDA may prefer template candidates even when a non-candidate
-        // is a perfect match
-        && !Opts.CUDA;
-  }
-
 } // namespace clang
 
 #endif // LLVM_CLANG_SEMA_OVERLOAD_H
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 45405d4709e14..f6ec4cb0f069e 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -6354,8 +6354,7 @@ SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
   Expr *NakedFn = Fn->IgnoreParenCasts();
   // Build an overload candidate set based on the functions we find.
   SourceLocation Loc = Fn->getExprLoc();
-  OverloadCandidateSet CandidateSet(Loc,
-                                    OverloadCandidateSet::CSK_CodeCompletion);
+  OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
 
   if (auto ULE = dyn_cast<UnresolvedLookupExpr>(NakedFn)) {
     SemaRef.AddOverloadedCallCandidates(ULE, ArgsWithoutDependentTypes,
@@ -6558,8 +6557,7 @@ QualType SemaCodeCompletion::ProduceConstructorSignatureHelp(
   // FIXME: Provide support for variadic template constructors.
 
   if (CRD) {
-    OverloadCandidateSet CandidateSet(Loc,
-                                      OverloadCandidateSet::CSK_CodeCompletion);
+    OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
     for (NamedDecl *C : SemaRef.LookupConstructors(CRD)) {
       if (auto *FD = dyn_cast<FunctionDecl>(C)) {
         // FIXME: we can't yet provide correct signature help for initializer
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 82489847b589b..a1e4bb4321d53 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -10029,19 +10029,12 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     //   When [...] the constructor [...] is a candidate by
     //    - [over.match.copy] (in all cases)
     if (TD) {
-
-      // As template candidates are not deduced immediately,
-      // persist the array in the overload set.
-      MutableArrayRef<Expr *> TmpInits =
-          Candidates.getPersistentArgsArray(Inits.size());
-
-      for (auto [I, E] : llvm::enumerate(Inits)) {
+      SmallVector<Expr *, 8> TmpInits;
+      for (Expr *E : Inits)
         if (auto *DI = dyn_cast<DesignatedInitExpr>(E))
-          TmpInits[I] = DI->getInit();
+          TmpInits.push_back(DI->getInit());
         else
-          TmpInits[I] = E;
-      }
-
+          TmpInits.push_back(E);
       AddTemplateOverloadCandidate(
           TD, FoundDecl, /*ExplicitArgs=*/nullptr, TmpInits, Candidates,
           /*SuppressUserConversions=*/false,
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index deef01c946feb..55634aa75ae25 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1123,10 +1123,6 @@ void OverloadCandidateSet::clear(CandidateSetKind CSK) {
   Candidates.clear();
   Functions.clear();
   Kind = CSK;
-  FirstDeferredCandidate = nullptr;
-  DeferredCandidatesCount = 0;
-  HasDeferredTemplateConstructors = false;
-  ResolutionByPerfectCandidateIsDisabled = false;
 }
 
 namespace {
@@ -7799,14 +7795,15 @@ void Sema::AddMethodCandidate(
   }
 }
 
-static void AddMethodTemplateCandidateImmediately(
-    Sema &S, OverloadCandidateSet &CandidateSet,
+void Sema::AddMethodTemplateCandidate(
     FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
     CXXRecordDecl *ActingContext,
     TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
     Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    bool SuppressUserConversions, bool PartialOverloading,
-    OverloadCandidateParamOrder PO) {
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, OverloadCandidateParamOrder PO) {
+  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
+    return;
 
   // C++ [over.match.funcs]p7:
   //   In each case where a candidate is a function template, candidate
@@ -7820,12 +7817,12 @@ static void AddMethodTemplateCandidateImmediately(
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = DeduceTemplateArguments(
           MethodTmpl, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, /*AggregateDeductionCandidate=*/false,
           /*PartialOrdering=*/false, ObjectType, ObjectClassification,
           [&](ArrayRef<QualType> ParamTypes) {
-            return S.CheckNonDependentConversions(
+            return CheckNonDependentConversions(
                 MethodTmpl, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, ActingContext, ObjectType,
                 ObjectClassification, PO);
@@ -7847,8 +7844,8 @@ static void AddMethodTemplateCandidateImmediately(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure =
-          MakeDeductionFailureInfo(S.Context, Result, Info);
+      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
+                                                            Info);
     }
     return;
   }
@@ -7858,34 +7855,10 @@ static void AddMethodTemplateCandidateImmediately(
   assert(Specialization && "Missing member function template specialization?");
   assert(isa<CXXMethodDecl>(Specialization) &&
          "Specialization is not a member function?");
-  S.AddMethodCandidate(
-      cast<CXXMethodDecl>(Specialization), FoundDecl, ActingContext, ObjectType,
-      ObjectClassification, Args, CandidateSet, SuppressUserConversions,
-      PartialOverloading, Conversions, PO, Info.hasStrictPackMatch());
-}
-
-void Sema::AddMethodTemplateCandidate(
-    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext,
-    TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
-    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, OverloadCandidateParamOrder PO) {
-  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
-    return;
-
-  if (ExplicitTemplateArgs ||
-      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts())) {
-    AddMethodTemplateCandidateImmediately(
-        *this, CandidateSet, MethodTmpl, FoundDecl, ActingContext,
-        ExplicitTemplateArgs, ObjectType, ObjectClassification, Args,
-        SuppressUserConversions, PartialOverloading, PO);
-    return;
-  }
-
-  CandidateSet.AddDeferredMethodTemplateCandidate(
-      MethodTmpl, FoundDecl, ActingContext, ObjectType, ObjectClassification,
-      Args, SuppressUserConversions, PartialOverloading, PO);
+  AddMethodCandidate(cast<CXXMethodDecl>(Specialization), FoundDecl,
+                     ActingContext, ObjectType, ObjectClassification, Args,
+                     CandidateSet, SuppressUserConversions, PartialOverloading,
+                     Conversions, PO, Info.hasStrictPackMatch());
 }
 
 /// Determine whether a given function template has a simple explicit specifier
@@ -7894,18 +7867,14 @@ static bool isNonDependentlyExplicit(FunctionTemplateDecl *FTD) {
   return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).isExplicit();
 }
 
-static bool hasDependentExplicit(FunctionTemplateDecl *FTD) {
-  return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).getKind() ==
-         ExplicitSpecKind::Unresolved;
-}
-
-static void AddTemplateOverloadCandidateImmediately(
-    Sema &S, OverloadCandidateSet &CandidateSet,
+void Sema::AddTemplateOverloadCandidate(
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
     TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
-    bool SuppressUserConversions, bool PartialOverloading, bool AllowExplicit,
-    Sema::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
-    bool AggregateCandidateDeduction) {
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
+    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
+  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
+    return;
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -7932,14 +7901,14 @@ static void AddTemplateOverloadCandidateImmediately(
                              FunctionTemplate->getTemplateDepth());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = DeduceTemplateArguments(
           FunctionTemplate, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, AggregateCandidateDeduction,
           /*PartialOrdering=*/false,
           /*ObjectType=*/QualType(),
           /*ObjectClassification=*/Expr::Classification(),
           [&](ArrayRef<QualType> ParamTypes) {
-            return S.CheckNonDependentConversions(
+            return CheckNonDependentConversions(
                 FunctionTemplate, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, nullptr, QualType(), {}, PO);
           });
@@ -7963,8 +7932,8 @@ static void AddTemplateOverloadCandidateImmediately(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure =
-          MakeDeductionFailureInfo(S.Context, Result, Info);
+      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
+                                                            Info);
     }
     return;
   }
@@ -7972,7 +7941,7 @@ static void AddTemplateOverloadCandidateImmediately(
   // Add the function template specialization produced by template argument
   // deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  S.AddOverloadCandidate(
+  AddOverloadCandidate(
       Specialization, FoundDecl, Args, CandidateSet, SuppressUserConversions,
       PartialOverloading, AllowExplicit,
       /*AllowExplicitConversions=*/false, IsADLCandidate, Conversions, PO,
@@ -7980,38 +7949,6 @@ static void AddTemplateOverloadCandidateImmediately(
       Info.hasStrictPackMatch());
 }
 
-void Sema::AddTemplateOverloadCandidate(
-    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
-    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
-  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
-    return;
-
-  bool DependentExplicitSpecifier = hasDependentExplicit(FunctionTemplate);
-
-  if (ExplicitTemplateArgs ||
-      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
-      (isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl()) &&
-       DependentExplicitSpecifier)) {
-
-    AddTemplateOverloadCandidateImmediately(
-        *this, CandidateSet, FunctionTemplate, FoundDecl, ExplicitTemplateArgs,
-        Args, SuppressUserConversions, PartialOverloading, AllowExplicit,
-        IsADLCandidate, PO, AggregateCandidateDeduction);
-
-    if (DependentExplicitSpecifier)
-      CandidateSet.DisableResolutionByPerfectCandidate();
-    return;
-  }
-
-  CandidateSet.AddDeferredTemplateCandidate(
-      FunctionTemplate, FoundDecl, Args, SuppressUserConversions,
-      PartialOverloading, AllowExplicit, IsADLCandidate, PO,
-      AggregateCandidateDeduction);
-}
-
 bool Sema::CheckNonDependentConversions(
     FunctionTemplateDecl *FunctionTemplate, ArrayRef<QualType> ParamTypes,
     ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet,
@@ -8327,12 +8264,16 @@ void Sema::AddConversionCandidate(
   }
 }
 
-static void AddTemplateConversionCandidateImmediately(
-    Sema &S, OverloadCandidateSet &CandidateSet,
+void Sema::AddTemplateConversionCandidate(
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
-    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
-    bool AllowResultConversion) {
+    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
+    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
+    bool AllowExplicit, bool AllowResultConversion) {
+  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
+         "Only conversion function templates permitted here");
+
+  if (!CandidateSet.isNewCandidate(FunctionTemplate))
+    return;
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -8347,11 +8288,11 @@ static void AddTemplateConversionCandidateImmediately(
   }
 
   QualType ObjectType = From->getType();
-  Expr::Classification ObjectClassification = From->Classify(S.Context);
+  Expr::Classification ObjectClassification = From->Classify(getASTContext());
 
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   CXXConversionDecl *Specialization = nullptr;
-  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = DeduceTemplateArguments(
           FunctionTemplate, ObjectType, ObjectClassification, ToType,
           Specialization, Info);
       Result != TemplateDeductionResult::Success) {
@@ -8361,47 +8302,18 @@ static void AddTemplateConversionCandidateImmediately(
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_deduction;
     Candidate.ExplicitCallArguments = 1;
-    Candidate.DeductionFailure =
-        MakeDeductionFailureInfo(S.Context, Result, Info);
+    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
+                                                          Info);
     return;
   }
 
   // Add the conversion function template specialization produced by
   // template argument deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  S.AddConversionCandidate(Specialization, FoundDecl, ActingContext, From,
-                           ToType, CandidateSet, AllowObjCConversionOnExplicit,
-                           AllowExplicit, AllowResultConversion,
-                           Info.hasStrictPackMatch());
-}
-
-void Sema::AddTemplateConversionCandidate(
-    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
-    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
-    bool AllowExplicit, bool AllowResultConversion) {
-  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
-         "Only conversion function templates permitted here");
-
-  if (!CandidateSet.isNewCandidate(FunctionTemplate))
-    return;
-
-  if (!CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
-      CandidateSet.getKind() ==
-          OverloadCandidateSet::CSK_InitByUserDefinedConversion ||
-      CandidateSet.getKind() == OverloadCandidateSet::CSK_InitByConstructor) {
-    AddTemplateConversionCandidateImmediately(
-        *this, CandidateSet, FunctionTemplate, FoundDecl, ActingDC, From,
-        ToType, AllowObjCConversionOnExplicit, AllowExplicit,
-        AllowResultConversion);
-
-    CandidateSet.DisableResolutionByPerfectCandidate();
-    return;
-  }
-
-  CandidateSet.AddDeferredConversionTemplateCandidate(
-      FunctionTemplate, FoundDecl, ActingDC, From, ToType,
-      AllowObjCConversionOnExplicit, AllowExplicit, AllowResultConversion);
+  AddConversionCandidate(Specialization, FoundDecl, ActingDC, From, ToType,
+                         CandidateSet, AllowObjCConversionOnExplicit,
+                         AllowExplicit, AllowResultConversion,
+                         Info.hasStrictPackMatch());
 }
 
 void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
@@ -8551,17 +8463,11 @@ void Sema::AddNonMemberOperatorCandidates(
     if (FunTmpl) {
       AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
                                    FunctionArgs, CandidateSet);
-      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD)) {
-
-        // As template candidates are not deduced immediately,
-        // persist the array in the overload set.
-        ArrayRef<Expr *> Reversed = CandidateSet.getPersistentArgsArray(
-            FunctionArgs[1], FunctionArgs[0]);
-        AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
-                                     Reversed, CandidateSet, false, false, true,
-                                     ADLCallKind::NotADL,
-                                     OverloadCandidateParamOrder::Reversed);
-      }
+      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD))
+        AddTemplateOverloadCandidate(
+            FunTmpl, F.getPair(), ExplicitTemplateArgs,
+            {FunctionArgs[1], FunctionArgs[0]}, CandidateSet, false, false,
+            true, ADLCallKind::NotADL, OverloadCandidateParamOrder::Reversed);
     } else {
       if (ExplicitTemplateArgs)
         continue;
@@ -10293,8 +10199,6 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
   // FIXME: Pass in the explicit template arguments?
   ArgumentDependentLookup(Name, Loc, Args, Fns);
 
-  ArrayRef<Expr *> ReversedArgs;
-
   // Erase all of the candidates we already knew about.
   for (OverloadCandidateSet::iterator Cand = CandidateSet.begin(),
                                    CandEnd = CandidateSet.end();
@@ -10334,15 +10238,9 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
           /*AllowExplicit=*/true, ADLCallKind::UsesADL);
       if (CandidateSet.getRewriteInfo().shouldAddReversed(
               *this, Args, FTD->getTemplatedDecl())) {
-
-        // As template candidates are not deduced immediately,
-        // persist the array in the overload set.
-        if (ReversedArgs.empty())
-          ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
-
         AddTemplateOverloadCandidate(
-            FTD, FoundDecl, ExplicitTemplateArgs, ReversedArgs, CandidateSet,
-            /*SuppressUserConversions=*/false, PartialOverloading,
+            FTD, FoundDecl, ExplicitTemplateArgs, {Args[1], Args[0]},
+            CandidateSet, /*SuppressUserConversions=*/false, PartialOverloading,
             /*AllowExplicit=*/true, ADLCallKind::UsesADL,
             OverloadCandidateParamOrder::Reversed);
       }
@@ -11015,147 +10913,23 @@ bool OverloadCandidate::NotValidBecauseConstraintExprHasError() const {
              ->Satisfaction.ContainsErrors;
 }
 
-void OverloadCandidateSet::AddDeferredTemplateCandidate(
-    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    ArrayRef<Expr *> Args, bool SuppressUserConversions,
-    bool PartialOverloading, bool AllowExplicit,
-    CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
-    bool AggregateCandidateDeduction) {
-
-  auto *C =
-      allocateDeferredCandidate<DeferredFunctionTemplateOverloadCandidate>();
-
-  C = new (C) DeferredFunctionTemplateOverloadCandidate{
-      {nullptr, DeferredFunctionTemplateOverloadCandidate::Function,
-       /*AllowObjCConversionOnExplicit=*/false,
-       /*AllowResultConversion=*/false, AllowExplicit, SuppressUserConversions,
-       PartialOverloading, AggregateCandidateDeduction},
-      FunctionTemplate,
-      FoundDecl,
-      Args,
-      IsADLCandidate,
-      PO};
-
-  HasDeferredTemplateConstructors |=
-      isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl());
-}
-
-void OverloadCandidateSet::AddDeferredMethodTemplateCandidate(
-    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext, QualType ObjectType,
-    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    bool SuppressUserConversions, bool PartialOverloading,
-    OverloadCandidateParamOrder PO) {
-
-  assert(!isa<CXXConstructorDecl>(MethodTmpl->getTemplatedDecl()));
-
-  auto *C =
-      allocateDeferredCandidate<DeferredMethodTemplateOverloadCandidate>();
-
-  C = new (C) DeferredMethodTemplateOverloadCandidate{
-      {nullptr, DeferredFunctionTemplateOverloadCandidate::Method,
-       /*AllowObjCConversionOnExplicit=*/false,
-       /*AllowResultConversion=*/false,
-       /*AllowExplicit=*/false, SuppressUserConversions, PartialOverloading,
-       /*AggregateCandidateDeduction=*/false},
-      MethodTmpl,
-      FoundDecl,
-      Args,
-      ActingContext,
-      ObjectClassification,
-      ObjectType,
-      PO};
-}
-
-void OverloadCandidateSet::AddDeferredConversionTemplateCandidate(
-    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
-    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
-    bool AllowResultConversion) {
-
-  auto *C =
-      allocateDeferredCandidate<DeferredConversionTemplateOverloadCandidate>();
-
-  C = new (C) DeferredConversionTemplateOverloadCandidate{
-      {nullptr, DeferredFunctionTemplateOverloadCandidate::Conversion,
-       AllowObjCConversionOnExplicit, AllowResultConversion,
-       /*AllowExplicit=*/false,
-       /*SuppressUserConversions=*/false,
-       /*PartialOverloading*/ false,
-       /*AggregateCandidateDeduction=*/false},
-      FunctionTemplate,
-      FoundDecl,
-      ActingContext,
-      From,
-      ToType};
-}
-
-static void
-AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
-                             DeferredMethodTemplateOverloadCandidate &C) {
-
-  AddMethodTemplateCandidateImmediately(
-      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext,
-      /*ExplicitTemplateArgs=*/nullptr, C.ObjectType, C.ObjectClassification,
-      C.Args, C.SuppressUserConversions, C.PartialOverloading, C.PO);
-}
-
-static void
-AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
-                             DeferredFunctionTemplateOverloadCandidate &C) {
-  AddTemplateOverloadCandidateImmediately(
-      S, CandidateSet, C.FunctionTemplate, C.FoundDecl,
-      /*ExplicitTemplateArgs=*/nullptr, C.Args, C.SuppressUserConversions,
-      C.PartialOverloading, C.AllowExplicit, C.IsADLCandidate, C.PO,
-      C.AggregateCandidateDeduction);
-}
-
-static void
-AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
-                             DeferredConversionTemplateOverloadCandidate &C) {
-  return AddTemplateConversionCandidateImmediately(
-      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext, C.From,
-      C.ToType, C.AllowObjCConversionOnExplicit, C.AllowExplicit,
-      C.AllowResultConversion);
-}
-
-void OverloadCandidateSet::InjectNonDeducedTemplateCandidates(Sema &S) {
-  Candidates.reserve(Candidates.size() + DeferredCandidatesCount);
-  DeferredTemplateOverloadCandidate *Cand = FirstDeferredCandidate;
-  while (Cand) {
-    switch (Cand->Kind) {
-    case DeferredTemplateOverloadCandidate::Function:
-      AddTemplateOverloadCandidate(
-          S, *this,
-          *static_cast<DeferredFunctionTemplateOverloadCandidate *>(Cand));
-      break;
-    case DeferredTemplateOverloadCandidate::Method:
-      AddTemplateOverloadCandidate(
-          S, *this,
-          *static_cast<DeferredMethodTemplateOverloadCandidate *>(Cand));
-      break;
-    case DeferredTemplateOverloadCandidate::Conversion:
-      AddTemplateOverloadCandidate(
-          S, *this,
-          *static_cast<DeferredConversionTemplateOverloadCandidate *>(Cand));
-      break;
-    }
-    Cand = Cand->Next;
-  }
-  FirstDeferredCandidate = nullptr;
-  DeferredCandidatesCount = 0;
-}
-
+/// Computes the best viable function (C++ 13.3.3)
+/// within an overload candidate set.
+///
+/// \param Loc The location of the function name (or operator symbol) for
+/// which overload resolution occurs.
+///
+/// \param Best If overload resolution was successful or found a deleted
+/// function, \p Best points to the candidate function found.
+///
+/// \returns The result of overload resolution.
 OverloadingResult
-OverloadCandidateSet::ResultForBestCandidate(const iterator &Best) {
-  Best->Best = true;
-  if (Best->Function && Best->Function->isDeleted())
-    return OR_Deleted;
-  return OR_Success;
-}
+OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
+                                         iterator &Best) {
+  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
+  std::transform(begin(), end(), std::back_inserter(Candidates),
+                 [](OverloadCandidate &Cand) { return &Cand; });
 
-void OverloadCandidateSet::CudaExcludeWrongSideCandidates(
-    Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates) {
   // [CUDA] HD->H or HD->D calls are technically not allowed by CUDA but
   // are accepted by both clang and NVCC. However, during a particular
   // compilation mode only one call variant is viable. We need to
@@ -11167,112 +10941,27 @@ void OverloadCandidateSet::CudaExcludeWrongSideCandidates(
   // -fgpu-exclude-wrong-side-overloads is off. When
   // -fgpu-exclude-wrong-side-overloads is on, all candidates are compared
   // uniformly in isBetterOverloadCandidate.
-  if (!S.getLangOpts().CUDA || S.getLangOpts().GPUExcludeWrongSideOverloads)
-    return;
-  const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
-
-  bool ContainsSameSideCandidate =
-      llvm::any_of(Candidates, [&](const OverloadCandidate *Cand) {
-        // Check viable function only.
+  if (S.getLangOpts().CUDA && !S.getLangOpts().GPUExcludeWrongSideOverloads) {
+    const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
+    bool ContainsSameSideCandidate =
+        llvm::any_of(Candidates, [&](OverloadCandidate *Cand) {
+          // Check viable function only.
+          return Cand->Viable && Cand->Function &&
+                 S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
+                     SemaCUDA::CFP_SameSide;
+        });
+    if (ContainsSameSideCandidate) {
+      auto IsWrongSideCandidate = [&](OverloadCandidate *Cand) {
+        // Check viable function only to avoid unnecessary data copying/moving.
         return Cand->Viable && Cand->Function &&
                S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-                   SemaCUDA::CFP_SameSide;
-      });
-
-  if (!ContainsSameSideCandidate)
-    return;
-
-  auto IsWrongSideCandidate = [&](const OverloadCandidate *Cand) {
-    // Check viable function only to avoid unnecessary data copying/moving.
-    return Cand->Viable && Cand->Function &&
-           S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-               SemaCUDA::CFP_WrongSide;
-  };
-  llvm::erase_if(Candidates, IsWrongSideCandidate);
-}
-
-/// Computes the best viable function (C++ 13.3.3)
-/// within an overload candidate set.
-///
-/// \param Loc The location of the function name (or operator symbol) for
-/// which overload resolution occurs.
-///
-/// \param Best If overload resolution was successful or found a deleted
-/// function, \p Best points to the candidate function found.
-///
-/// \returns The result of overload resolution.
-OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
-                                                           SourceLocation Loc,
-                                                           iterator &Best) {
-
-  assert(shouldDeferTemplateArgumentDeduction(S.getLangOpts()) ||
-         DeferredCandidatesCount == 0 &&
-             "Unexpected deferred template candidates");
-
-  bool TwoPhaseResolution =
-      DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
-
-  if (TwoPhaseResolution) {
-
-    PerfectViableFunction(S, Loc, Best);
-    if (Best != end())
-      return ResultForBestCandidate(Best);
-  }
-
-  InjectNonDeducedTemplateCandidates(S);
-  return BestViableFunctionImpl(S, Loc, Best);
-}
-
-void OverloadCandidateSet::PerfectViableFunction(
-    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
-
-  Best = end();
-  for (auto It = begin(); It != end(); ++It) {
-
-    if (!It->isPerfectMatch(S.getASTContext()))
-      continue;
-
-    // We found a suitable conversion function
-    // but if there is a template constructor in the target class
-    // we might prefer that instead.
-    if (HasDeferredTemplateConstructors &&
-        isa_and_nonnull<CXXConversionDecl>(It->Function)) {
-      Best = end();
-      break;
-    }
-
-    if (Best == end()) {
-      Best = It;
-      continue;
-    }
-    if (Best->Function && It->Function) {
-      FunctionDecl *D =
-          S.getMoreConstrainedFunction(Best->Function, It->Function);
-      if (D == nullptr) {
-        Best = end();
-        break;
-      }
-      if (D == It->Function)
-        Best = It;
-      continue;
+                   SemaCUDA::CFP_WrongSide;
+      };
+      llvm::erase_if(Candidates, IsWrongSideCandidate);
     }
-    // ambiguous
-    Best = end();
-    break;
   }
-}
-
-OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
-    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
-
-  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
-  Candidates.reserve(this->Candidates.size());
-  std::transform(begin(), end(), std::back_inserter(Candidates),
-                 [](OverloadCandidate &Cand) { return &Cand; });
-
-  if (S.getLangOpts().CUDA)
-    CudaExcludeWrongSideCandidates(S, Candidates);
 
+  // Find the best viable function.
   Best = end();
   for (auto *Cand : Candidates) {
     Cand->Best = false;
@@ -11294,8 +10983,9 @@ OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
   if (Best == end())
     return OR_No_Viable_Function;
 
-  llvm::SmallVector<OverloadCandidate *, 4> PendingBest;
   llvm::SmallVector<const NamedDecl *, 4> EquivalentCands;
+
+  llvm::SmallVector<OverloadCandidate*, 4> PendingBest;
   PendingBest.push_back(&*Best);
   Best->Best = true;
 
@@ -11318,15 +11008,25 @@ OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
     }
   }
 
+  // If we found more than one best candidate, this is ambiguous.
   if (Best == end())
     return OR_Ambiguous;
 
-  OverloadingResult R = ResultForBestCandidate(Best);
+  // Best is the best viable function.
+  if (Best->Function && Best->Function->isDeleted())
+    return OR_Deleted;
+
+  if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
+      Kind == CSK_AddressOfOverloadSet && M &&
+      M->isImplicitObjectMemberFunction()) {
+    return OR_No_Viable_Function;
+  }
 
   if (!EquivalentCands.empty())
     S.diagnoseEquivalentInternalLinkageDeclarations(Loc, Best->Function,
                                                     EquivalentCands);
-  return R;
+
+  return OR_Success;
 }
 
 namespace {
@@ -13033,9 +12733,6 @@ SmallVector<OverloadCandidate *, 32> OverloadCandidateSet::CompleteCandidates(
     Sema &S, OverloadCandidateDisplayKind OCD, ArrayRef<Expr *> Args,
     SourceLocation OpLoc,
     llvm::function_ref<bool(OverloadCandidate &)> Filter) {
-
-  InjectNonDeducedTemplateCandidates(S);
-
   // Sort the candidates by viability and position.  Sorting directly would
   // be prohibitive, so we make a set of pointers and sort those.
   SmallVector<OverloadCandidate*, 32> Cands;
@@ -14657,12 +14354,10 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
                                          Expr *ExecConfig,
                                          bool AllowTypoCorrection,
                                          bool CalleesAddressIsTaken) {
-
-  OverloadCandidateSet::CandidateSetKind CSK =
-      CalleesAddressIsTaken ? OverloadCandidateSet::CSK_AddressOfOverloadSet
-                            : OverloadCandidateSet::CSK_Normal;
-
-  OverloadCandidateSet CandidateSet(Fn->getExprLoc(), CSK);
+  OverloadCandidateSet CandidateSet(
+      Fn->getExprLoc(), CalleesAddressIsTaken
+                            ? OverloadCandidateSet::CSK_AddressOfOverloadSet
+                            : OverloadCandidateSet::CSK_Normal);
   ExprResult result;
 
   if (buildOverloadedCallSet(S, Fn, ULE, Args, LParenLoc, &CandidateSet,
@@ -14678,17 +14373,6 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
   OverloadingResult OverloadResult =
       CandidateSet.BestViableFunction(*this, Fn->getBeginLoc(), Best);
 
-  // [C++23][over.call.func]
-  // if overload resolution selects a non-static member function,
-  // the call is ill-formed;
-  if (CSK == OverloadCandidateSet::CSK_AddressOfOverloadSet &&
-      Best != CandidateSet.end()) {
-    if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
-        M && M->isImplicitObjectMemberFunction()) {
-      OverloadResult = OR_No_Viable_Function;
-    }
-  }
-
   // Model the case with a call to a templated function whose definition
   // encloses the call and whose return type contains a placeholder type as if
   // the UnresolvedLookupExpr was type-dependent.
@@ -15024,24 +14708,18 @@ void Sema::LookupOverloadedBinOp(OverloadCandidateSet &CandidateSet,
   // rewritten candidates using these functions if necessary.
   AddNonMemberOperatorCandidates(Fns, Args, CandidateSet);
 
-  // As template candidates are not deduced immediately,
-  // persist the array in the overload set.
-  ArrayRef<Expr *> ReversedArgs;
-  if (CandidateSet.getRewriteInfo().allowsReversed(Op) ||
-      CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
-    ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
-
   // Add operator candidates that are member functions.
   AddMemberOperatorCandidates(Op, OpLoc, Args, CandidateSet);
   if (CandidateSet.getRewriteInfo().allowsReversed(Op))
-    AddMemberOperatorCandidates(Op, OpLoc, ReversedArgs, CandidateSet,
+    AddMemberOperatorCandidates(Op, OpLoc, {Args[1], Args[0]}, CandidateSet,
                                 OverloadCandidateParamOrder::Reversed);
 
   // In C++20, also add any rewritten member candidates.
   if (ExtraOp) {
     AddMemberOperatorCandidates(ExtraOp, OpLoc, Args, CandidateSet);
     if (CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
-      AddMemberOperatorCandidates(ExtraOp, OpLoc, ReversedArgs, CandidateSet,
+      AddMemberOperatorCandidates(ExtraOp, OpLoc, {Args[1], Args[0]},
+                                  CandidateSet,
                                   OverloadCandidateParamOrder::Reversed);
   }
 
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 0ecdbb3ffb89f..772962ac653f7 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -6142,9 +6142,9 @@ FunctionDecl *Sema::getMoreConstrainedFunction(FunctionDecl *FD1,
   assert(!FD1->getDescribedTemplate() && !FD2->getDescribedTemplate() &&
          "not for function templates");
   assert(!FD1->isFunctionTemplateSpecialization() ||
-         (isa<CXXConversionDecl, CXXConstructorDecl>(FD1)));
+         isa<CXXConversionDecl>(FD1));
   assert(!FD2->isFunctionTemplateSpecialization() ||
-         (isa<CXXConversionDecl, CXXConstructorDecl>(FD2)));
+         isa<CXXConversionDecl>(FD2));
 
   FunctionDecl *F1 = FD1;
   if (FunctionDecl *P = FD1->getTemplateInstantiationPattern(false))
diff --git a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
index 083e743818121..ba8e2dc372e98 100644
--- a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
+++ b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
@@ -14,7 +14,7 @@ template<typename T> struct S {
 // expected-note@#FINST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{})
 void f(T);
-void f(long);
+void f(int);
 
 // Ensure this applies to operator && as well.
 // expected-error@+3{{atomic constraint must be of type 'bool' (found 'S<int>')}}
@@ -22,7 +22,7 @@ void f(long);
 // expected-note@#F2INST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{} && true)
 void f2(T);
-void f2(long);
+void f2(int);
 
 template<typename T> requires requires {
   requires S<T>{};
@@ -36,12 +36,12 @@ template<typename T> requires requires {
   //
 }
 void f3(T);
-void f3(long);
+void f3(int);
 
 // Doesn't diagnose, since this is no longer a compound requirement.
 template<typename T> requires (bool(1 && 2))
 void f4(T);
-void f4(long);
+void f4(int);
 
 void g() {
   f(0); // #FINST
diff --git a/clang/test/SemaCUDA/function-overload.cu b/clang/test/SemaCUDA/function-overload.cu
index 3d05839af7528..4710c81763adf 100644
--- a/clang/test/SemaCUDA/function-overload.cu
+++ b/clang/test/SemaCUDA/function-overload.cu
@@ -1,3 +1,6 @@
+// REQUIRES: x86-registered-target
+// REQUIRES: nvptx-registered-target
+
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu -fsyntax-only \
 // RUN:   -verify=host,hostdefer,devdefer,expected %s
 // RUN: %clang_cc1 -std=c++14 -triple nvptx64-nvidia-cuda -fsyntax-only \
diff --git a/clang/test/SemaCXX/implicit-member-functions.cpp b/clang/test/SemaCXX/implicit-member-functions.cpp
index 8350eac5b88a0..1554b1af5d59a 100644
--- a/clang/test/SemaCXX/implicit-member-functions.cpp
+++ b/clang/test/SemaCXX/implicit-member-functions.cpp
@@ -54,24 +54,31 @@ namespace PR7594 {
 namespace Recursion {
   template<typename T> struct InvokeCopyConstructor {
     static const T &get();
-    typedef decltype(T(get())) type;
+    typedef decltype(T(get())) type; // expected-error {{no matching conver}}
   };
   struct B;
   struct A {
+    // expected-note@-1 {{while substituting deduced template arguments}}
     typedef B type;
     template<typename T,
              typename = typename InvokeCopyConstructor<typename T::type>::type>
+    // expected-note@-1 {{in instantiation of template class}}
     A(const T &);
+    // expected-note@-1 {{in instantiation of default argument}}
   };
-  struct B {
-    B();
+  struct B { // expected-note {{while declaring the implicit copy constructor for 'B'}}
+    // expected-note@-1 {{candidate constructor (the implicit move }}
+    B(); // expected-note {{candidate constructor not viable}}
     A a;
   };
   // Triggering the declaration of B's copy constructor causes overload
-  // resolution to occur for A's copying constructor, which picks
-  // the implicit copy constructor of A.
-  // Because that copy constructor is always a perfect match the template
-  // candidate is not instantiated.
+  // resolution to occur for A's copying constructor, which instantiates
+  // InvokeCopyConstructor<B>, which triggers the declaration of B's copy
+  // constructor. Notionally, this happens when we get to the end of the
+  // definition of 'struct B', so there is no declared copy constructor yet.
+  //
+  // This behavior is g++-compatible, but isn't exactly right; the class is
+  // supposed to be incomplete when we implicitly declare its special members.
   B b = B();
 
 
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
deleted file mode 100644
index 877816ca013ec..0000000000000
--- a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
+++ /dev/null
@@ -1,185 +0,0 @@
-// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++11 %s
-// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++20 %s
-// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++2c %s
-
-template <typename T>
-struct Invalid { static_assert(false, "instantiated Invalid"); }; // #err-invalid
-
-template <typename T>
-int f(T a, Invalid<T> = {}); // #note-f
-
-// sanity check
-int e1 = f(0);
-//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
-//expected-note@-2 {{in instantiation of default function argument expression for 'f<int>' required here}}
-//expected-note@#note-f {{in instantiation of template class 'Invalid<int>' requested here}}
-//expected-note@#note-f {{passing argument to parameter here}}
-
-int f(int);
-int ok1 = f(0);
-int e4 = f((const int&)(ok1));
-
-int f(int, int = 0);
-int ok2 = f(0, 0);
-
-int e2  = f(0L);
-//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
-//expected-note@-2 {{in instantiation of default function argument expression for 'f<long>' required here}}
-//expected-note@#note-f {{in instantiation of template class 'Invalid<long>' requested here}}
-//expected-note@#note-f {{passing argument to parameter here}}
-
-int f(long);
-int ok3 = f(0L);
-
-template <typename T>
-struct Invalid2 { static_assert(false, "instantiated Invalid2"); }; // #err-qualifiers
-
-template <typename T>
-int ref(T a, Invalid2<T> = {}); // expected-note 2{{here}}
-int ref(int&);
-int ref1 = ref(ok3);
-int ref2 = ref((const int&)ok3); // expected-note {{here}}
-//expected-error@#err-qualifiers {{static assertion failed: instantiated Invalid2}}
-
-
-template <typename T>
-int f_alias(T a, Invalid<T> = {});
-using Alias = int;
-int f_alias(Alias);
-int ok4 = f_alias(0);
-
-#if __cplusplus >= 202002
-
-struct Copyable {
-  template <typename T>
-  requires __is_constructible(Copyable, T)
-  explicit Copyable(T op) noexcept; // #1
-  Copyable(const Copyable&) noexcept = default; // #2
-};
-static_assert(__is_constructible(Copyable, const Copyable&));
-
-struct ImplicitlyCopyable {
-  template <typename T>
-  requires __is_constructible(ImplicitlyCopyable, T)
-  explicit ImplicitlyCopyable(T op) = delete; // #1
-};
-static_assert(__is_constructible(ImplicitlyCopyable, const ImplicitlyCopyable&));
-
-
-struct Movable {
-  template <typename T>
-  requires __is_constructible(Movable, T) // #err-self-constraint-1
-  explicit Movable(T op) noexcept; // #1
-  Movable(Movable&&) noexcept = default; // #2
-};
-static_assert(__is_constructible(Movable, Movable&&));
-static_assert(__is_constructible(Movable, const Movable&));
-// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(Movable, const Movable &)'}}
-
-static_assert(__is_constructible(Movable, int));
-// expected-error@-1{{static assertion failed due to requirement '__is_constructible(Movable, int)'}} \
-// expected-note@-1 2{{}}
-// expected-error@#err-self-constraint-1{{satisfaction of constraint '__is_constructible(Movable, T)' depends on itself}}
-// expected-note@#err-self-constraint-1 4{{}}
-
-template <typename T>
-struct Members {
-    constexpr auto f(auto) {
-        static_assert(false, "");
-    }
-    constexpr auto f(int) { return 1; }
-    constexpr auto f(int) requires true { return 2; }
-
-    constexpr auto g(auto) {
-        static_assert(false, "instantiated member"); //#err-qualified-member
-        return 0;
-    }
-    constexpr auto g(int) & { return 1; }
-
-    static constexpr auto s(auto) {
-        static_assert(false, "");
-    }
-    static constexpr auto s(int) {
-        return 1;
-    }
-    static constexpr auto s(int) requires true {
-        return 2;
-    }
-};
-
-static_assert(Members<int[1]>{}.f(0) == 2);
-static_assert(Members<int[2]>{}.g(0) == 0);
-// expected-error@#err-qualified-member {{static assertion failed: instantiated member}} \
-// expected-note@-1{{in instantiation of function template specialization 'Members<int[2]>::g<int>' }}
-Members<int[3]> m1;
-static_assert(m1.g(0) == 1);
-static_assert(Members<int[3]>{}.s(0) == 2);
-
-
-namespace ConstructorInit{
-struct S {
-  template <typename T>
-  S(T&&) {}
-};
-struct Test {
-  operator S() = delete;
-};
-
-static_assert(__is_constructible(S, Test));
-}
-
-namespace RefBinding {
-
-template <typename> struct remove_reference;
-template <typename _Tp> struct remove_reference<_Tp &> {
-  using type = _Tp;
-};
-template <typename _Tp> remove_reference<_Tp>::type move(_Tp &&);
-template <typename _Head> struct _Head_base {
-  _Head_base(_Head &__h) : _M_head_impl(__h) {}
-  template <typename _UHead> _Head_base(_UHead &&);
-  _Head _M_head_impl;
-};
-
-template <typename _Elements> void forward_as_tuple(_Elements &&) {
-  _Head_base<_Elements &&>(_Elements{});
-}
-struct StringRef {
-  void operator[](const StringRef __k) { forward_as_tuple((move)(__k)); }
-};
-
-}
-
-template <class> struct tuple {};
-struct BonkersBananas {
-  template <class T> operator T();
-  template <class = void> explicit operator tuple<int>() = delete;
-};
-static_assert(!__is_constructible(tuple<int>, BonkersBananas));
-
-namespace GH62096 {
-template <typename T>
-struct Oops {
-  static_assert(sizeof(T) == 0); // #GH62096-err
-  static constexpr bool value = true;
-};
-
-template <class OPERATOR>
-concept Operator = Oops<OPERATOR>::value; // #GH62096-note1
-
-template <Operator OP> void f(OP op); // // #GH62096-note2
-void f(int);
-
-void g(int n) { f(n); } // OK
-void h(short n) { f(n); }
-// expected-error@#GH62096-err {{static assertion failed due to requirement 'sizeof(short) == 0'}} \
-// expected-note@-1{{in instantiation of function template specialization}} \
-// expected-note@-1{{while checking constraint satisfaction for template}}
-// expected-note@#GH62096-note1{{in instantiation}}
-// expected-note@#GH62096-note1{{while substituting template arguments into constraint expression here}}
-// expected-note@#GH62096-note2{{while substituting template arguments into constraint expression here}}
-// expected-note@#GH62096-note2{{while checking the satisfaction of concept}}
-// expected-note@#GH62096-err {{expression evaluates}}
-}
-
-#endif
diff --git a/clang/test/SemaTemplate/instantiate-function-params.cpp b/clang/test/SemaTemplate/instantiate-function-params.cpp
index eb2a7c5d4e8d6..7dd5595de58a3 100644
--- a/clang/test/SemaTemplate/instantiate-function-params.cpp
+++ b/clang/test/SemaTemplate/instantiate-function-params.cpp
@@ -6,12 +6,13 @@ template<typename T1> struct if_ {
   typedef if_c< static_cast<bool>(T1::value)> almost_type_; // expected-note 7{{in instantiation}}
 };
 template <class Model, void (Model::*)()> struct wrap_constraints { };
-template <class Model>
+template <class Model> 
 inline char has_constraints_(Model* , // expected-note 3{{candidate template ignored}}
-                               wrap_constraints<Model,&Model::constraints>* = 0);
+                               wrap_constraints<Model,&Model::constraints>* = 0); // expected-note 4{{in instantiation}}
+
 template <class Model> struct not_satisfied {
   static const bool value = sizeof( has_constraints_((Model*)0)  == 1); // expected-error 3{{no matching function}} \
-  // expected-note 4{{in instantiation}}
+  // expected-note 4{{while substituting deduced template arguments into function template 'has_constraints_' [with }}
 };
 template <class ModelFn> struct requirement_;
 template <void(*)()> struct instantiate {
diff --git a/clang/test/Templight/templight-empty-entries-fix.cpp b/clang/test/Templight/templight-empty-entries-fix.cpp
index 7f34b10134929..d13b748068efe 100644
--- a/clang/test/Templight/templight-empty-entries-fix.cpp
+++ b/clang/test/Templight/templight-empty-entries-fix.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -templight-dump -Wno-unused-value %s 2>&1 | FileCheck %s
 
-void a(long) {
+void a() {
   [] {};
 }
 
@@ -17,14 +17,14 @@ void a(long) {
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 // CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 
-template <int = 0> void a(long) { a(0); }
+template <int = 0> void a() { a(); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -42,29 +42,29 @@ template <int = 0> void a(long) { a(0); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
 
 template <int> struct b { typedef int c; };
-template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
+template <bool d = true, class = typename b<d>::c> void a() { a(); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -130,25 +130,25 @@ template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -166,10 +166,34 @@ template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 
 template <bool = true> void d(int = 0) { d(); }
 
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+a$}}
+// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
+// CHECK: {{^event:[ ]+Begin$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+a$}}
+// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
+// CHECK: {{^event:[ ]+End$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+a$}}
+// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
+// CHECK: {{^event:[ ]+Begin$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+a$}}
+// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
+// CHECK: {{^event:[ ]+End$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
@@ -225,41 +249,41 @@ void e() {
 }
 
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
 
 
 template <template<typename> class>
@@ -275,71 +299,71 @@ void foo() {
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}

From 6d03f51f0c59171f1ec3c5cc5c1fe71c30956273 Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1@linux.ibm.com>
Date: Wed, 16 Apr 2025 12:02:56 -0600
Subject: [PATCH 157/710] [SystemZ] Add support for 16-bit floating point.
 (#109164)

- _Float16 is now accepted by Clang.

- The half IR type is fully handled by the backend.

- These values are passed in FP registers and converted to/from float around
  each operation.

- Compiler-rt conversion functions are now built for s390x including the missing
  extendhfdf2 which was added.

Fixes #50374
---
 clang/docs/LanguageExtensions.rst             |   1 +
 clang/include/clang/Basic/TargetInfo.h        |   2 +-
 clang/lib/Basic/Targets/SystemZ.h             |  13 +
 clang/lib/CodeGen/Targets/SystemZ.cpp         |  40 +-
 clang/test/CodeGen/SystemZ/Float16.c          |  85 ++
 clang/test/CodeGen/SystemZ/fp16.c             |  39 +
 .../test/CodeGen/SystemZ/strictfp_builtins.c  |  14 +-
 clang/test/CodeGen/SystemZ/systemz-abi.c      |  53 ++
 .../test/CodeGen/SystemZ/systemz-inline-asm.c |   8 +
 compiler-rt/cmake/builtin-config-ix.cmake     |   3 +-
 compiler-rt/lib/builtins/CMakeLists.txt       |   6 +
 compiler-rt/lib/builtins/clear_cache.c        |   2 +
 compiler-rt/lib/builtins/extendhfdf2.c        |  15 +
 compiler-rt/test/builtins/CMakeLists.txt      |   2 +-
 .../test/builtins/Unit/extendhfdf2_test.c     |  87 +++
 llvm/docs/LangRef.rst                         |   2 +-
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |  19 +
 .../SystemZ/AsmParser/SystemZAsmParser.cpp    |  16 +
 .../MCTargetDesc/SystemZMCTargetDesc.cpp      |  16 +
 .../MCTargetDesc/SystemZMCTargetDesc.h        |   2 +
 llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp |  46 +-
 llvm/lib/Target/SystemZ/SystemZCallingConv.td |   4 +-
 llvm/lib/Target/SystemZ/SystemZFeatures.td    |   2 +
 .../Target/SystemZ/SystemZISelDAGToDAG.cpp    |   7 +-
 .../Target/SystemZ/SystemZISelLowering.cpp    | 412 ++++++++--
 llvm/lib/Target/SystemZ/SystemZISelLowering.h |  11 +
 llvm/lib/Target/SystemZ/SystemZInstrFP.td     |  43 +-
 llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp  |  19 +-
 llvm/lib/Target/SystemZ/SystemZInstrVector.td |   8 +-
 .../Target/SystemZ/SystemZRegisterInfo.cpp    |   5 +-
 llvm/lib/Target/SystemZ/SystemZRegisterInfo.h |   2 +-
 .../lib/Target/SystemZ/SystemZRegisterInfo.td |  34 +-
 llvm/lib/Target/SystemZ/SystemZScheduleZ13.td |  22 +-
 llvm/lib/Target/SystemZ/SystemZScheduleZ14.td |  22 +-
 llvm/lib/Target/SystemZ/SystemZScheduleZ15.td |  22 +-
 llvm/lib/Target/SystemZ/SystemZScheduleZ16.td |  22 +-
 llvm/lib/Target/SystemZ/SystemZScheduleZ17.td |  22 +-
 .../lib/Target/SystemZ/SystemZScheduleZ196.td |  14 +-
 .../Target/SystemZ/SystemZScheduleZEC12.td    |  14 +-
 llvm/test/CodeGen/SystemZ/asm-10.ll           |   9 +
 llvm/test/CodeGen/SystemZ/asm-17.ll           |  11 +
 llvm/test/CodeGen/SystemZ/asm-19.ll           |  19 +
 llvm/test/CodeGen/SystemZ/atomic-load-10.ll   |  22 +
 llvm/test/CodeGen/SystemZ/atomic-store-10.ll  |  24 +
 .../test/CodeGen/SystemZ/atomicrmw-fadd-04.ll |  76 ++
 .../CodeGen/SystemZ/fmuladd-soft-float.ll     |  37 +
 llvm/test/CodeGen/SystemZ/fp-abs-01.ll        |  12 +
 llvm/test/CodeGen/SystemZ/fp-abs-03.ll        |  12 +
 llvm/test/CodeGen/SystemZ/fp-abs-04.ll        |  16 +
 llvm/test/CodeGen/SystemZ/fp-add-01.ll        |  12 +
 llvm/test/CodeGen/SystemZ/fp-cmp-04.ll        |  68 +-
 llvm/test/CodeGen/SystemZ/fp-conv-05.ll       |  10 +
 llvm/test/CodeGen/SystemZ/fp-conv-06.ll       |  12 +
 llvm/test/CodeGen/SystemZ/fp-conv-07.ll       |  10 +
 llvm/test/CodeGen/SystemZ/fp-conv-08.ll       |  11 +
 llvm/test/CodeGen/SystemZ/fp-conv-09.ll       |  10 +
 llvm/test/CodeGen/SystemZ/fp-conv-10.ll       |  36 +-
 llvm/test/CodeGen/SystemZ/fp-conv-11.ll       |  10 +
 llvm/test/CodeGen/SystemZ/fp-conv-12.ll       |  36 +-
 llvm/test/CodeGen/SystemZ/fp-conv-13.ll       |  32 +-
 llvm/test/CodeGen/SystemZ/fp-conv-14.ll       |  32 +-
 llvm/test/CodeGen/SystemZ/fp-conv-20.ll       |  78 +-
 llvm/test/CodeGen/SystemZ/fp-copysign-03.ll   | 145 ++++
 llvm/test/CodeGen/SystemZ/fp-div-01.ll        |  12 +
 llvm/test/CodeGen/SystemZ/fp-half-cmp.ll      | 161 ++++
 llvm/test/CodeGen/SystemZ/fp-half-libcall.ll  | 312 ++++++++
 llvm/test/CodeGen/SystemZ/fp-half-mem.ll      |  63 ++
 llvm/test/CodeGen/SystemZ/fp-half-move.ll     |  83 ++
 llvm/test/CodeGen/SystemZ/fp-half-strict.ll   | 207 +++++
 llvm/test/CodeGen/SystemZ/fp-half-vector.ll   | 725 ++++++++++++++++++
 llvm/test/CodeGen/SystemZ/fp-half.ll          | 612 +++++++++++++++
 llvm/test/CodeGen/SystemZ/fp-libcall.ll       |  10 +
 llvm/test/CodeGen/SystemZ/fp-mul-01.ll        |  12 +
 llvm/test/CodeGen/SystemZ/fp-mul-06.ll        |  14 +
 llvm/test/CodeGen/SystemZ/fp-mul-08.ll        |  18 +
 llvm/test/CodeGen/SystemZ/fp-mul-10.ll        |  38 +-
 llvm/test/CodeGen/SystemZ/fp-mul-15.ll        |  20 +
 llvm/test/CodeGen/SystemZ/fp-neg-01.ll        |  11 +
 llvm/test/CodeGen/SystemZ/fp-neg-02.ll        |  11 +
 llvm/test/CodeGen/SystemZ/fp-round-01.ll      |  48 ++
 llvm/test/CodeGen/SystemZ/fp-round-02.ll      |  36 +
 llvm/test/CodeGen/SystemZ/fp-round-03.ll      |  15 +-
 llvm/test/CodeGen/SystemZ/fp-sqrt-01.ll       |  12 +
 llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll | 114 ++-
 .../test/CodeGen/SystemZ/fp-strict-cmps-01.ll |  20 +
 .../test/CodeGen/SystemZ/fp-strict-cmps-04.ll |  37 +
 .../test/CodeGen/SystemZ/fp-strict-conv-01.ll |  29 +
 .../test/CodeGen/SystemZ/fp-strict-conv-02.ll |  11 +
 .../test/CodeGen/SystemZ/fp-strict-conv-05.ll |  13 +
 .../test/CodeGen/SystemZ/fp-strict-conv-06.ll |  15 +
 .../test/CodeGen/SystemZ/fp-strict-conv-07.ll |  13 +
 .../test/CodeGen/SystemZ/fp-strict-conv-08.ll |  14 +
 .../test/CodeGen/SystemZ/fp-strict-conv-09.ll |  12 +
 .../test/CodeGen/SystemZ/fp-strict-conv-10.ll |  50 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-11.ll |  12 +
 .../test/CodeGen/SystemZ/fp-strict-conv-12.ll |  50 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-13.ll |  38 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-14.ll |  36 +-
 .../test/CodeGen/SystemZ/fp-strict-conv-15.ll |  28 +
 .../test/CodeGen/SystemZ/fp-strict-conv-17.ll |  88 ++-
 llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll |  16 +
 llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll |  18 +
 .../CodeGen/SystemZ/fp-strict-round-01.ll     |  71 ++
 .../CodeGen/SystemZ/fp-strict-round-02.ll     |  42 +
 .../CodeGen/SystemZ/fp-strict-round-03.ll     |  15 +
 .../test/CodeGen/SystemZ/fp-strict-sqrt-01.ll |  16 +
 llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll |  16 +
 llvm/test/CodeGen/SystemZ/fp-sub-01.ll        |  12 +
 ...-asm-fp-int-casting-explicit-regs-zEC12.ll |  34 +
 ...inline-asm-fp-int-casting-explicit-regs.ll |  40 +
 .../inline-asm-fp-int-casting-zEC12.ll        |  32 +
 .../SystemZ/inline-asm-fp-int-casting.ll      |  52 ++
 llvm/test/CodeGen/SystemZ/is_fpclass.ll       |  19 +
 llvm/test/CodeGen/SystemZ/spill-half-01.mir   |  63 ++
 llvm/test/CodeGen/SystemZ/spill-half-02.mir   |  27 +
 llvm/test/CodeGen/SystemZ/stackmap.ll         |  30 +-
 llvm/test/CodeGen/SystemZ/tdc-01.ll           |  12 +
 llvm/test/CodeGen/SystemZ/tdc-02.ll           |  17 +
 llvm/test/CodeGen/SystemZ/tdc-03.ll           |  26 +-
 llvm/test/CodeGen/SystemZ/tdc-04.ll           |  18 +-
 llvm/test/CodeGen/SystemZ/tdc-05.ll           |  25 +
 llvm/test/CodeGen/SystemZ/tdc-06.ll           |   2 -
 llvm/test/CodeGen/SystemZ/twoaddr-kill.mir    |   8 +-
 llvm/test/CodeGen/SystemZ/vec-max-05.ll       |  14 +
 llvm/test/CodeGen/SystemZ/vec-min-05.ll       |  14 +
 .../test/CodeGen/SystemZ/vec-strict-max-01.ll |  31 +
 .../test/CodeGen/SystemZ/vec-strict-min-01.ll |  31 +
 127 files changed, 5301 insertions(+), 296 deletions(-)
 create mode 100644 clang/test/CodeGen/SystemZ/Float16.c
 create mode 100644 clang/test/CodeGen/SystemZ/fp16.c
 create mode 100644 compiler-rt/lib/builtins/extendhfdf2.c
 create mode 100644 compiler-rt/test/builtins/Unit/extendhfdf2_test.c
 create mode 100644 llvm/test/CodeGen/SystemZ/atomic-load-10.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/atomic-store-10.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/atomicrmw-fadd-04.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-copysign-03.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-half-cmp.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-half-libcall.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-half-mem.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-half-move.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-half-strict.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-half-vector.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-half.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/fp-mul-15.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/spill-half-01.mir
 create mode 100644 llvm/test/CodeGen/SystemZ/spill-half-02.mir

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 971ab50cc9a69..7835eceadf660 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1000,6 +1000,7 @@ to ``float``; see below for more information on this emulation.
   * SPIR (natively)
   * X86 (if SSE2 is available; natively if AVX512-FP16 is also available)
   * RISC-V (natively if Zfh or Zhinx is available)
+  * SystemZ (emulated)
 
 * ``__bf16`` is supported on the following targets (currently never natively):
 
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 93cffe84e2f42..8c3dcda25bc8d 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -235,7 +235,7 @@ class TargetInfo : public TransferrableTargetInfo,
   bool NoAsmVariants;  // True if {|} are normal characters.
   bool HasLegalHalfType; // True if the backend supports operations on the half
                          // LLVM IR type.
-  bool HalfArgsAndReturns;
+  bool HalfArgsAndReturns; // OpenCL 6.1.1.1, NEON (IEEE 754-2008 half) type.
   bool HasFloat128;
   bool HasFloat16;
   bool HasBFloat16;
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index 4d1509b84e82b..cb71c5d7e75d8 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -93,11 +93,24 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
                       "-v128:64-a:8:16-n32:64");
     }
     MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 128;
+
+    // True if the backend supports operations on the half LLVM IR type.
+    // By setting this to false, conversions will happen for _Float16 around
+    // a statement by default, with operations done in float. However, if
+    // -ffloat16-excess-precision=none is given, no conversions will be made
+    // and instead the backend will promote each half operation to float
+    // individually.
+    HasLegalHalfType = false;
+    // Support _Float16.
+    HasFloat16 = true;
+
     HasStrictFP = true;
   }
 
   unsigned getMinGlobalAlign(uint64_t Size, bool HasNonWeakDef) const override;
 
+  bool useFP16ConversionIntrinsics() const override { return false; }
+
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp
index 8a9fddace76d9..6ea6c7a546436 100644
--- a/clang/lib/CodeGen/Targets/SystemZ.cpp
+++ b/clang/lib/CodeGen/Targets/SystemZ.cpp
@@ -31,7 +31,7 @@ class SystemZABIInfo : public ABIInfo {
   bool isPromotableIntegerTypeForABI(QualType Ty) const;
   bool isCompoundType(QualType Ty) const;
   bool isVectorArgumentType(QualType Ty) const;
-  bool isFPArgumentType(QualType Ty) const;
+  llvm::Type *getFPArgumentType(QualType Ty, uint64_t Size) const;
   QualType GetSingleElementType(QualType Ty) const;
 
   ABIArgInfo classifyReturnType(QualType RetTy) const;
@@ -107,7 +107,8 @@ class SystemZTargetCodeGenInfo : public TargetCodeGenInfo {
       return nullptr;
 
     llvm::Type *Ty = V->getType();
-    if (Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isFP128Ty()) {
+    if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy() ||
+        Ty->isFP128Ty()) {
       llvm::Module &M = CGM.getModule();
       auto &Ctx = M.getContext();
       llvm::Function *TDCFunc = llvm::Intrinsic::getOrInsertDeclaration(
@@ -179,20 +180,31 @@ bool SystemZABIInfo::isVectorArgumentType(QualType Ty) const {
           getContext().getTypeSize(Ty) <= 128);
 }
 
-bool SystemZABIInfo::isFPArgumentType(QualType Ty) const {
+// The Size argument will in case of af an overaligned single element struct
+// reflect the overalignment value. In such a case the argument will be
+// passed using the type matching Size.
+llvm::Type *SystemZABIInfo::getFPArgumentType(QualType Ty,
+                                              uint64_t Size) const {
   if (IsSoftFloatABI)
-    return false;
+    return nullptr;
 
   if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
     switch (BT->getKind()) {
+    case BuiltinType::Float16:
+      if (Size == 16)
+        return llvm::Type::getHalfTy(getVMContext());
+      LLVM_FALLTHROUGH;
     case BuiltinType::Float:
+      if (Size == 32)
+        return llvm::Type::getFloatTy(getVMContext());
+      LLVM_FALLTHROUGH;
     case BuiltinType::Double:
-      return true;
+      return llvm::Type::getDoubleTy(getVMContext());
     default:
-      return false;
+      return nullptr;
     }
 
-  return false;
+  return nullptr;
 }
 
 QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const {
@@ -277,7 +289,8 @@ RValue SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   } else {
     if (AI.getCoerceToType())
       ArgTy = AI.getCoerceToType();
-    InFPRs = (!IsSoftFloatABI && (ArgTy->isFloatTy() || ArgTy->isDoubleTy()));
+    InFPRs = (!IsSoftFloatABI &&
+              (ArgTy->isHalfTy() || ArgTy->isFloatTy() || ArgTy->isDoubleTy()));
     IsVector = ArgTy->isVectorTy();
     UnpaddedSize = TyInfo.Width;
     DirectAlign = TyInfo.Align;
@@ -447,12 +460,11 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
       return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),
                                      /*ByVal=*/false);
 
-    // The structure is passed as an unextended integer, a float, or a double.
-    if (isFPArgumentType(SingleElementTy)) {
-      assert(Size == 32 || Size == 64);
-      return ABIArgInfo::getDirect(
-          Size == 32 ? llvm::Type::getFloatTy(getVMContext())
-                     : llvm::Type::getDoubleTy(getVMContext()));
+    // The structure is passed as an unextended integer, a half, a float,
+    // or a double.
+    if (llvm::Type *FPArgTy = getFPArgumentType(SingleElementTy, Size)) {
+      assert(Size == 16 || Size == 32 || Size == 64);
+      return ABIArgInfo::getDirect(FPArgTy);
     } else {
       llvm::IntegerType *PassTy = llvm::IntegerType::get(getVMContext(), Size);
       return Size <= 32 ? ABIArgInfo::getNoExtend(PassTy)
diff --git a/clang/test/CodeGen/SystemZ/Float16.c b/clang/test/CodeGen/SystemZ/Float16.c
new file mode 100644
index 0000000000000..4444dbdcc23ca
--- /dev/null
+++ b/clang/test/CodeGen/SystemZ/Float16.c
@@ -0,0 +1,85 @@
+// RUN: %clang_cc1 -triple s390x-linux-gnu \
+// RUN: -ffloat16-excess-precision=standard -emit-llvm -o - %s \
+// RUN: | FileCheck %s -check-prefix=STANDARD
+
+// RUN: %clang_cc1 -triple s390x-linux-gnu \
+// RUN: -ffloat16-excess-precision=none -emit-llvm -o - %s \
+// RUN: | FileCheck %s -check-prefix=NONE
+
+// RUN: %clang_cc1 -triple s390x-linux-gnu \
+// RUN: -ffloat16-excess-precision=fast -emit-llvm -o - %s \
+// RUN: | FileCheck %s -check-prefix=FAST
+
+_Float16 f(_Float16 a, _Float16 b, _Float16 c, _Float16 d) {
+    return a * b + c * d;
+}
+
+// STANDARD-LABEL: define dso_local half @f(half noundef %a, half noundef %b, half noundef %c, half noundef %d) #0 {
+// STANDARD-NEXT:  entry:
+// STANDARD-NEXT:    %a.addr = alloca half, align 2
+// STANDARD-NEXT:    %b.addr = alloca half, align 2
+// STANDARD-NEXT:    %c.addr = alloca half, align 2
+// STANDARD-NEXT:    %d.addr = alloca half, align 2
+// STANDARD-NEXT:    store half %a, ptr %a.addr, align 2
+// STANDARD-NEXT:    store half %b, ptr %b.addr, align 2
+// STANDARD-NEXT:    store half %c, ptr %c.addr, align 2
+// STANDARD-NEXT:    store half %d, ptr %d.addr, align 2
+// STANDARD-NEXT:    %0 = load half, ptr %a.addr, align 2
+// STANDARD-NEXT:    %ext = fpext half %0 to float
+// STANDARD-NEXT:    %1 = load half, ptr %b.addr, align 2
+// STANDARD-NEXT:    %ext1 = fpext half %1 to float
+// STANDARD-NEXT:    %mul = fmul float %ext, %ext1
+// STANDARD-NEXT:    %2 = load half, ptr %c.addr, align 2
+// STANDARD-NEXT:    %ext2 = fpext half %2 to float
+// STANDARD-NEXT:    %3 = load half, ptr %d.addr, align 2
+// STANDARD-NEXT:    %ext3 = fpext half %3 to float
+// STANDARD-NEXT:    %mul4 = fmul float %ext2, %ext3
+// STANDARD-NEXT:    %add = fadd float %mul, %mul4
+// STANDARD-NEXT:    %unpromotion = fptrunc float %add to half
+// STANDARD-NEXT:    ret half %unpromotion
+// STANDARD-NEXT:  }
+
+// NONE-LABEL: define dso_local half @f(half noundef %a, half noundef %b, half noundef %c, half noundef %d) #0 {
+// NONE-NEXT:  entry:
+// NONE-NEXT:    %a.addr = alloca half, align 2
+// NONE-NEXT:    %b.addr = alloca half, align 2
+// NONE-NEXT:    %c.addr = alloca half, align 2
+// NONE-NEXT:    %d.addr = alloca half, align 2
+// NONE-NEXT:    store half %a, ptr %a.addr, align 2
+// NONE-NEXT:    store half %b, ptr %b.addr, align 2
+// NONE-NEXT:    store half %c, ptr %c.addr, align 2
+// NONE-NEXT:    store half %d, ptr %d.addr, align 2
+// NONE-NEXT:    %0 = load half, ptr %a.addr, align 2
+// NONE-NEXT:    %1 = load half, ptr %b.addr, align 2
+// NONE-NEXT:    %mul = fmul half %0, %1
+// NONE-NEXT:    %2 = load half, ptr %c.addr, align 2
+// NONE-NEXT:    %3 = load half, ptr %d.addr, align 2
+// NONE-NEXT:    %mul1 = fmul half %2, %3
+// NONE-NEXT:    %add = fadd half %mul, %mul1
+// NONE-NEXT:    ret half %add
+// NONE-NEXT:  }
+
+// FAST-LABEL: define dso_local half @f(half noundef %a, half noundef %b, half noundef %c, half noundef %d) #0 {
+// FAST-NEXT:  entry:
+// FAST-NEXT:    %a.addr = alloca half, align 2
+// FAST-NEXT:    %b.addr = alloca half, align 2
+// FAST-NEXT:    %c.addr = alloca half, align 2
+// FAST-NEXT:    %d.addr = alloca half, align 2
+// FAST-NEXT:    store half %a, ptr %a.addr, align 2
+// FAST-NEXT:    store half %b, ptr %b.addr, align 2
+// FAST-NEXT:    store half %c, ptr %c.addr, align 2
+// FAST-NEXT:    store half %d, ptr %d.addr, align 2
+// FAST-NEXT:    %0 = load half, ptr %a.addr, align 2
+// FAST-NEXT:    %ext = fpext half %0 to float
+// FAST-NEXT:    %1 = load half, ptr %b.addr, align 2
+// FAST-NEXT:    %ext1 = fpext half %1 to float
+// FAST-NEXT:    %mul = fmul float %ext, %ext1
+// FAST-NEXT:    %2 = load half, ptr %c.addr, align 2
+// FAST-NEXT:    %ext2 = fpext half %2 to float
+// FAST-NEXT:    %3 = load half, ptr %d.addr, align 2
+// FAST-NEXT:    %ext3 = fpext half %3 to float
+// FAST-NEXT:    %mul4 = fmul float %ext2, %ext3
+// FAST-NEXT:    %add = fadd float %mul, %mul4
+// FAST-NEXT:    %unpromotion = fptrunc float %add to half
+// FAST-NEXT:    ret half %unpromotion
+// FAST-NEXT:  }
diff --git a/clang/test/CodeGen/SystemZ/fp16.c b/clang/test/CodeGen/SystemZ/fp16.c
new file mode 100644
index 0000000000000..430958b69a177
--- /dev/null
+++ b/clang/test/CodeGen/SystemZ/fp16.c
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -triple s390x-linux-gnu -emit-llvm -o - %s \
+// RUN: | FileCheck %s
+
+void f(__fp16 *a, __fp16 *b, __fp16 *c, __fp16 *d, __fp16 *e) {
+  *e = (*a) * (*b) + (*c) * (*d);
+}
+
+// CHECK-LABEL: define dso_local void @f(ptr noundef %a, ptr noundef %b, ptr noundef %c, ptr noundef %d, ptr noundef %e) #0 {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    %a.addr = alloca ptr, align 8
+// CHECK-NEXT:    %b.addr = alloca ptr, align 8
+// CHECK-NEXT:    %c.addr = alloca ptr, align 8
+// CHECK-NEXT:    %d.addr = alloca ptr, align 8
+// CHECK-NEXT:    %e.addr = alloca ptr, align 8
+// CHECK-NEXT:    store ptr %a, ptr %a.addr, align 8
+// CHECK-NEXT:    store ptr %b, ptr %b.addr, align 8
+// CHECK-NEXT:    store ptr %c, ptr %c.addr, align 8
+// CHECK-NEXT:    store ptr %d, ptr %d.addr, align 8
+// CHECK-NEXT:    store ptr %e, ptr %e.addr, align 8
+// CHECK-NEXT:    %0 = load ptr, ptr %a.addr, align 8
+// CHECK-NEXT:    %1 = load half, ptr %0, align 2
+// CHECK-NEXT:    %conv = fpext half %1 to float
+// CHECK-NEXT:    %2 = load ptr, ptr %b.addr, align 8
+// CHECK-NEXT:    %3 = load half, ptr %2, align 2
+// CHECK-NEXT:    %conv1 = fpext half %3 to float
+// CHECK-NEXT:    %mul = fmul float %conv, %conv1
+// CHECK-NEXT:    %4 = load ptr, ptr %c.addr, align 8
+// CHECK-NEXT:    %5 = load half, ptr %4, align 2
+// CHECK-NEXT:    %conv2 = fpext half %5 to float
+// CHECK-NEXT:    %6 = load ptr, ptr %d.addr, align 8
+// CHECK-NEXT:    %7 = load half, ptr %6, align 2
+// CHECK-NEXT:    %conv3 = fpext half %7 to float
+// CHECK-NEXT:    %mul4 = fmul float %conv2, %conv3
+// CHECK-NEXT:    %add = fadd float %mul, %mul4
+// CHECK-NEXT:    %8 = fptrunc float %add to half
+// CHECK-NEXT:    %9 = load ptr, ptr %e.addr, align 8
+// CHECK-NEXT:    store half %8, ptr %9, align 2
+// CHECK-NEXT:    ret void
+// CHECK-NEXT:  }
diff --git a/clang/test/CodeGen/SystemZ/strictfp_builtins.c b/clang/test/CodeGen/SystemZ/strictfp_builtins.c
index 8c8f1f4cabd74..f871debde067e 100644
--- a/clang/test/CodeGen/SystemZ/strictfp_builtins.c
+++ b/clang/test/CodeGen/SystemZ/strictfp_builtins.c
@@ -4,12 +4,24 @@
 
 #pragma float_control(except, on)
 
+// CHECK-LABEL: @test_isnan__Float16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[F_ADDR:%.*]] = alloca half, align 2
+// CHECK-NEXT:    store half [[F:%.*]], ptr [[F_ADDR]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load half, ptr [[F_ADDR]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.s390.tdc.f16(half [[TMP0]], i64 15) #[[ATTR2:[0-9]+]]
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+int test_isnan__Float16(_Float16 f) {
+  return __builtin_isnan(f);
+}
+
 // CHECK-LABEL: @test_isnan_float(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[F_ADDR:%.*]] = alloca float, align 4
 // CHECK-NEXT:    store float [[F:%.*]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.s390.tdc.f32(float [[TMP0]], i64 15) #[[ATTR2:[0-9]+]]
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.s390.tdc.f32(float [[TMP0]], i64 15) #[[ATTR2]]
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int test_isnan_float(float f) {
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi.c b/clang/test/CodeGen/SystemZ/systemz-abi.c
index 7de425950e9fd..f26084ab44eae 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi.c
@@ -52,6 +52,9 @@ long long pass_longlong(long long arg) { return arg; }
 __int128 pass_int128(__int128 arg) { return arg; }
 // CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr %0)
 
+_Float16 pass__Float16(_Float16 arg) { return arg; }
+// CHECK-LABEL: define{{.*}} half @pass__Float16(half %{{.*}})
+
 float pass_float(float arg) { return arg; }
 // CHECK-LABEL: define{{.*}} float @pass_float(float %{{.*}})
 
@@ -79,6 +82,9 @@ _Complex long pass_complex_long(_Complex long arg) { return arg; }
 _Complex long long pass_complex_longlong(_Complex long long arg) { return arg; }
 // CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
 
+_Complex _Float16 pass_complex__Float16(_Complex _Float16 arg) { return arg; }
+// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr %{{.*}}arg)
+
 _Complex float pass_complex_float(_Complex float arg) { return arg; }
 // CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr %{{.*}}arg)
 
@@ -130,6 +136,11 @@ struct agg_16byte pass_agg_16byte(struct agg_16byte arg) { return arg; }
 
 // Float-like aggregate types
 
+struct agg__Float16 { _Float16 a; };
+struct agg__Float16 pass_agg__Float16(struct agg__Float16 arg) { return arg; }
+// HARD-FLOAT-LABEL: define{{.*}} void @pass_agg__Float16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16) align 2 %{{.*}}, half %{{.*}})
+// SOFT-FLOAT-LABEL: define{{.*}} void @pass_agg__Float16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16) align 2 %{{.*}}, i16 noext %{{.*}})
+
 struct agg_float { float a; };
 struct agg_float pass_agg_float(struct agg_float arg) { return arg; }
 // HARD-FLOAT-LABEL: define{{.*}} void @pass_agg_float(ptr dead_on_unwind noalias writable sret(%struct.agg_float) align 4 %{{.*}}, float %{{.*}})
@@ -144,6 +155,20 @@ struct agg_longdouble { long double a; };
 struct agg_longdouble pass_agg_longdouble(struct agg_longdouble arg) { return arg; }
 // CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr %{{.*}})
 
+struct agg__Float16_a4 { _Float16 a __attribute__((aligned (4))); };
+struct agg__Float16_a4 pass_agg__Float16_a4(struct agg__Float16_a4 arg) { return arg; }
+// HARD-FLOAT-LABEL: define{{.*}} void @pass_agg__Float16_a4(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a4) align 4 %{{.*}}, float %{{.*}})
+// SOFT-FLOAT-LABEL: define{{.*}} void @pass_agg__Float16_a4(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a4) align 4 %{{.*}}, i32 noext %{{.*}})
+
+struct agg__Float16_a8 { _Float16 a __attribute__((aligned (8))); };
+struct agg__Float16_a8 pass_agg__Float16_a8(struct agg__Float16_a8 arg) { return arg; }
+// HARD-FLOAT-LABEL: define{{.*}} void @pass_agg__Float16_a8(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a8) align 8 %{{.*}}, double %{{.*}})
+// SOFT-FLOAT-LABEL: define{{.*}} void @pass_agg__Float16_a8(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a8) align 8 %{{.*}}, i64 %{{.*}})
+
+struct agg__Float16_a16 { _Float16 a __attribute__((aligned (16))); };
+struct agg__Float16_a16 pass_agg__Float16_a16(struct agg__Float16_a16 arg) { return arg; }
+// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr %{{.*}})
+
 struct agg_float_a8 { float a __attribute__((aligned (8))); };
 struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
 // HARD-FLOAT-LABEL: define{{.*}} void @pass_agg_float_a8(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a8) align 8 %{{.*}}, double %{{.*}})
@@ -171,6 +196,10 @@ struct agg_nofloat3 pass_agg_nofloat3(struct agg_nofloat3 arg) { return arg; }
 
 // Union types likewise are *not* float-like aggregate types
 
+union union__Float16 { _Float16 a; };
+union union__Float16 pass_union__Float16(union union__Float16 arg) { return arg; }
+// CHECK-LABEL: define{{.*}} void @pass_union__Float16(ptr dead_on_unwind noalias writable sret(%union.union__Float16) align 2 %{{.*}}, i16 noext %{{.*}})
+
 union union_float { float a; };
 union union_float pass_union_float(union union_float arg) { return arg; }
 // CHECK-LABEL: define{{.*}} void @pass_union_float(ptr dead_on_unwind noalias writable sret(%union.union_float) align 4 %{{.*}}, i32 noext %{{.*}})
@@ -448,6 +477,30 @@ struct agg_8byte va_agg_8byte(__builtin_va_list l) { return __builtin_va_arg(l,
 // CHECK: [[VA_ARG_ADDR:%[^ ]+]] = phi ptr [ [[RAW_REG_ADDR]], %{{.*}} ], [ [[RAW_MEM_ADDR]], %{{.*}} ]
 // CHECK: ret void
 
+struct agg__Float16 va_agg__Float16(__builtin_va_list l) { return __builtin_va_arg(l, struct agg__Float16); }
+// CHECK-LABEL: define{{.*}} void @va_agg__Float16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16) align 2 %{{.*}}, ptr %{{.*}}
+// HARD-FLOAT: [[REG_COUNT_PTR:%[^ ]+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.*}}, i32 0, i32 1
+// SOFT-FLOAT: [[REG_COUNT_PTR:%[^ ]+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.*}}, i32 0, i32 0
+// CHECK: [[REG_COUNT:%[^ ]+]] = load i64, ptr [[REG_COUNT_PTR]]
+// HARD-FLOAT: [[FITS_IN_REGS:%[^ ]+]] = icmp ult i64 [[REG_COUNT]], 4
+// SOFT-FLOAT: [[FITS_IN_REGS:%[^ ]+]] = icmp ult i64 [[REG_COUNT]], 5
+// CHECK: br i1 [[FITS_IN_REGS]],
+// CHECK: [[SCALED_REG_COUNT:%[^ ]+]] = mul i64 [[REG_COUNT]], 8
+// HARD-FLOAT: [[REG_OFFSET:%[^ ]+]] = add i64 [[SCALED_REG_COUNT]], 128
+// SOFT-FLOAT: [[REG_OFFSET:%[^ ]+]] = add i64 [[SCALED_REG_COUNT]], 22
+// CHECK: [[REG_SAVE_AREA_PTR:%[^ ]+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.*}}, i32 0, i32 3
+// CHECK: [[REG_SAVE_AREA:%[^ ]+]] = load ptr, ptr [[REG_SAVE_AREA_PTR:[^ ]+]]
+// CHECK: [[RAW_REG_ADDR:%[^ ]+]] = getelementptr i8, ptr [[REG_SAVE_AREA]], i64 [[REG_OFFSET]]
+// CHECK: [[REG_COUNT1:%[^ ]+]] = add i64 [[REG_COUNT]], 1
+// CHECK: store i64 [[REG_COUNT1]], ptr [[REG_COUNT_PTR]]
+// CHECK: [[OVERFLOW_ARG_AREA_PTR:%[^ ]+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.*}}, i32 0, i32 2
+// CHECK: [[OVERFLOW_ARG_AREA:%[^ ]+]] = load ptr, ptr [[OVERFLOW_ARG_AREA_PTR]]
+// CHECK: [[RAW_MEM_ADDR:%[^ ]+]] = getelementptr i8, ptr [[OVERFLOW_ARG_AREA]], i64 6
+// CHECK: [[OVERFLOW_ARG_AREA2:%[^ ]+]] = getelementptr i8, ptr [[OVERFLOW_ARG_AREA]], i64 8
+// CHECK: store ptr [[OVERFLOW_ARG_AREA2]], ptr [[OVERFLOW_ARG_AREA_PTR]]
+// CHECK: [[VA_ARG_ADDR:%[^ ]+]] = phi ptr [ [[RAW_REG_ADDR]], %{{.*}} ], [ [[RAW_MEM_ADDR]], %{{.*}} ]
+// CHECK: ret void
+
 struct agg_float va_agg_float(__builtin_va_list l) { return __builtin_va_arg(l, struct agg_float); }
 // CHECK-LABEL: define{{.*}} void @va_agg_float(ptr dead_on_unwind noalias writable sret(%struct.agg_float) align 4 %{{.*}}, ptr %{{.*}}
 // HARD-FLOAT: [[REG_COUNT_PTR:%[^ ]+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.*}}, i32 0, i32 1
diff --git a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
index 9e62b8e107900..434937a66389c 100644
--- a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
+++ b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
@@ -106,6 +106,14 @@ void test_M(void) {
 // CHECK: call void asm sideeffect "#FOO $0", "M"(i32 2147483647)
 }
 
+_Float16 test_f16(_Float16 a) {
+  _Float16 f;
+  asm("ler %0, %1" : "=f" (f) : "f" (a));
+  return f;
+// CHECK-LABEL: define{{.*}} half @test_f16(half noundef %a)
+// CHECK: call half asm "ler $0, $1", "=f,f"(half %a)
+}
+
 float test_f32(float f, float g) {
   asm("aebr %0, %2" : "=f" (f) : "0" (f), "f" (g));
   return f;
diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake
index 7bd3269bd999d..cbb43a5958d2f 100644
--- a/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/compiler-rt/cmake/builtin-config-ix.cmake
@@ -73,6 +73,7 @@ set(PPC32 powerpc powerpcspe)
 set(PPC64 powerpc64 powerpc64le)
 set(RISCV32 riscv32)
 set(RISCV64 riscv64)
+set(S390X s390x)
 set(SPARC sparc)
 set(SPARCV9 sparcv9)
 set(WASM32 wasm32)
@@ -88,7 +89,7 @@ endif()
 set(ALL_BUILTIN_SUPPORTED_ARCH
   ${X86} ${X86_64} ${AMDGPU} ${ARM32} ${ARM64} ${AVR}
   ${HEXAGON} ${MIPS32} ${MIPS64} ${NVPTX} ${PPC32} ${PPC64}
-  ${RISCV32} ${RISCV64} ${SPARC} ${SPARCV9}
+  ${RISCV32} ${RISCV64} ${S390X} ${SPARC} ${SPARCV9}
   ${WASM32} ${WASM64} ${VE} ${LOONGARCH64})
 
 include(CompilerRTUtils)
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 3cdbf21ed403d..74d9627b9f102 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -104,6 +104,7 @@ set(GENERIC_SOURCES
   divti3.c
   extendsfdf2.c
   extendhfsf2.c
+  extendhfdf2.c
   ffsdi2.c
   ffssi2.c
   ffsti2.c
@@ -768,6 +769,11 @@ set(riscv64_SOURCES
 set(sparc_SOURCES ${GENERIC_SOURCES} ${GENERIC_TF_SOURCES})
 set(sparcv9_SOURCES ${GENERIC_SOURCES} ${GENERIC_TF_SOURCES})
 
+set(s390x_SOURCES
+  ${GENERIC_SOURCES}
+  ${GENERIC_TF_SOURCES}
+)
+
 set(wasm32_SOURCES
   ${GENERIC_TF_SOURCES}
   ${GENERIC_SOURCES}
diff --git a/compiler-rt/lib/builtins/clear_cache.c b/compiler-rt/lib/builtins/clear_cache.c
index 2ac99b25c243f..441eabd1fe922 100644
--- a/compiler-rt/lib/builtins/clear_cache.c
+++ b/compiler-rt/lib/builtins/clear_cache.c
@@ -62,6 +62,8 @@ void __clear_cache(void *start, void *end) {
 #if __i386__ || __x86_64__ || defined(_M_IX86) || defined(_M_X64)
 // Intel processors have a unified instruction and data cache
 // so there is nothing to do
+#elif defined(__s390__)
+// no-op
 #elif defined(_WIN32) && (defined(__arm__) || defined(__aarch64__))
   FlushInstructionCache(GetCurrentProcess(), start, end - start);
 #elif defined(__arm__) && !defined(__APPLE__)
diff --git a/compiler-rt/lib/builtins/extendhfdf2.c b/compiler-rt/lib/builtins/extendhfdf2.c
new file mode 100644
index 0000000000000..1cfbdb82730ad
--- /dev/null
+++ b/compiler-rt/lib/builtins/extendhfdf2.c
@@ -0,0 +1,15 @@
+//===-- lib/extendhfdf2.c - half -> single conversion -------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#define SRC_HALF
+#define DST_DOUBLE
+#include "fp_extend_impl.inc"
+
+COMPILER_RT_ABI NOINLINE dst_t __extendhfdf2(src_t a) {
+  return __extendXfYf2__(a);
+}
diff --git a/compiler-rt/test/builtins/CMakeLists.txt b/compiler-rt/test/builtins/CMakeLists.txt
index 8fdcec6029a2a..63f4c94605c90 100644
--- a/compiler-rt/test/builtins/CMakeLists.txt
+++ b/compiler-rt/test/builtins/CMakeLists.txt
@@ -56,7 +56,7 @@ foreach(arch ${BUILTIN_TEST_ARCH})
       string(REPLACE ";" " " BUILTINS_TEST_TARGET_CFLAGS "${BUILTINS_TEST_TARGET_CFLAGS}")
     endif()
   else()
-    if (${arch} MATCHES "arm|armhf|aarch64|arm64|i?86|x86_64|AMD64|riscv32|riscv64" AND COMPILER_RT_HAS_${arch}_FLOAT16)
+    if (${arch} MATCHES "arm|armhf|aarch64|arm64|i?86|x86_64|AMD64|riscv32|riscv64|s390x" AND COMPILER_RT_HAS_${arch}_FLOAT16)
       list(APPEND BUILTINS_TEST_TARGET_CFLAGS -DCOMPILER_RT_HAS_FLOAT16)
       string(REPLACE ";" " " BUILTINS_TEST_TARGET_CFLAGS "${BUILTINS_TEST_TARGET_CFLAGS}")
     endif()
diff --git a/compiler-rt/test/builtins/Unit/extendhfdf2_test.c b/compiler-rt/test/builtins/Unit/extendhfdf2_test.c
new file mode 100644
index 0000000000000..422e272c11f77
--- /dev/null
+++ b/compiler-rt/test/builtins/Unit/extendhfdf2_test.c
@@ -0,0 +1,87 @@
+// RUN: %clang_builtins %s %librt -o %t && %run %t
+// REQUIRES: librt_has_extendhfdf2
+
+#include <stdio.h>
+
+#include "fp_test.h"
+
+double __extendhfdf2(TYPE_FP16 a);
+
+int test__extendhfdf2(TYPE_FP16 a, uint64_t expected)
+{
+    double x = __extendhfdf2(a);
+    int ret = compareResultD(x, expected);
+
+    if (ret){
+        printf("error in test__extendhfdf2(%#.4x) = %f, "
+               "expected %f\n", toRep16(a), x, fromRep64(expected));
+    }
+    return ret;
+}
+
+char assumption_1[sizeof(TYPE_FP16) * CHAR_BIT == 16] = {0};
+
+int main()
+{
+    // qNaN
+    if (test__extendhfdf2(makeQNaN16(),
+                          UINT64_C(0x7ff8000000000000)))
+        return 1;
+    // NaN
+    if (test__extendhfdf2(fromRep16(0x7f80),
+                          UINT64_C(0x7ffe000000000000)))
+        return 1;
+    // inf
+    if (test__extendhfdf2(makeInf16(),
+                          UINT64_C(0x7ff0000000000000)))
+        return 1;
+    // -inf
+    if (test__extendhfdf2(makeNegativeInf16(),
+                          UINT64_C(0xfff0000000000000)))
+        return 1;
+    // zero
+    if (test__extendhfdf2(fromRep16(0x0),
+                          UINT64_C(0x0)))
+        return 1;
+    // -zero
+    if (test__extendhfdf2(fromRep16(0x8000),
+                          UINT64_C(0x8000000000000000)))
+        return 1;
+    if (test__extendhfdf2(fromRep16(0x4248),
+                          UINT64_C(0x4009200000000000)))
+        return 1;
+    if (test__extendhfdf2(fromRep16(0xc248),
+                          UINT64_C(0xc009200000000000)))
+        return 1;
+    if (test__extendhfdf2(fromRep16(0x6e62),
+                          UINT64_C(0x40b9880000000000)))
+        return 1;
+    if (test__extendhfdf2(fromRep16(0x3c00),
+                          UINT64_C(0x3ff0000000000000)))
+        return 1;
+    if (test__extendhfdf2(fromRep16(0x0400),
+                          UINT64_C(0x3f10000000000000)))
+        return 1;
+    // denormal
+    if (test__extendhfdf2(fromRep16(0x0010),
+                          UINT64_C(0x3eb0000000000000)))
+        return 1;
+    if (test__extendhfdf2(fromRep16(0x0001),
+                          UINT64_C(0x3e70000000000000)))
+        return 1;
+    if (test__extendhfdf2(fromRep16(0x8001),
+                          UINT64_C(0xbe70000000000000)))
+        return 1;
+    if (test__extendhfdf2(fromRep16(0x0001),
+                          UINT64_C(0x3e70000000000000)))
+        return 1;
+    // max (precise)
+    if (test__extendhfdf2(fromRep16(0x7bff),
+                          UINT64_C(0x40effc0000000000)))
+        return 1;
+    // max (rounded)
+    if (test__extendhfdf2(fromRep16(0x7bff),
+                          UINT64_C(0x40effc0000000000)))
+        return 1;
+    return 0;
+}
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 769003a90f959..110c30e19220f 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -5710,7 +5710,7 @@ SystemZ:
   address context evaluates as zero).
 - ``h``: A 32-bit value in the high part of a 64bit data register
   (LLVM-specific)
-- ``f``: A 32, 64, or 128-bit floating-point register.
+- ``f``: A 16, 32, 64, or 128-bit floating-point register.
 
 X86:
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index b8af281e1c24b..fddb99d2f0b22 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -5466,6 +5466,25 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
         DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
     break;
+
+  case ISD::STRICT_FMINIMUM: {
+  case ISD::STRICT_FMAXIMUM:
+    SDValue InChain = Node->getOperand(0);
+    SDVTList VTs = DAG.getVTList(NVT, MVT::Other);
+    Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, VTs, InChain,
+                       Node->getOperand(1));
+    Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, VTs, InChain,
+                       Node->getOperand(2));
+    SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
+    Tmp3 = DAG.getNode(Node->getOpcode(), dl, VTs, Ops, Node->getFlags());
+    Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, DAG.getVTList(OVT, MVT::Other),
+                       InChain, Tmp3,
+                       DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
+    Results.push_back(Tmp4);
+    Results.push_back(Tmp4.getValue(1));
+    break;
+  }
+
   case ISD::STRICT_FADD:
   case ISD::STRICT_FSUB:
   case ISD::STRICT_FMUL:
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 4fa5f026602ef..6d9a7a73f72db 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -61,9 +61,11 @@ enum RegisterKind {
   GRH32Reg,
   GR64Reg,
   GR128Reg,
+  FP16Reg,
   FP32Reg,
   FP64Reg,
   FP128Reg,
+  VR16Reg,
   VR32Reg,
   VR64Reg,
   VR128Reg,
@@ -365,9 +367,11 @@ class SystemZOperand : public MCParsedAsmOperand {
   bool isADDR32() const { return isReg(GR32Reg); }
   bool isADDR64() const { return isReg(GR64Reg); }
   bool isADDR128() const { return false; }
+  bool isFP16() const { return isReg(FP16Reg); }
   bool isFP32() const { return isReg(FP32Reg); }
   bool isFP64() const { return isReg(FP64Reg); }
   bool isFP128() const { return isReg(FP128Reg); }
+  bool isVR16() const { return isReg(VR16Reg); }
   bool isVR32() const { return isReg(VR32Reg); }
   bool isVR64() const { return isReg(VR64Reg); }
   bool isVF128() const { return false; }
@@ -544,6 +548,9 @@ class SystemZAsmParser : public MCTargetAsmParser {
   ParseStatus parseADDR128(OperandVector &Operands) {
     llvm_unreachable("Shouldn't be used as an operand");
   }
+  ParseStatus parseFP16(OperandVector &Operands) {
+    return parseRegister(Operands, FP16Reg);
+  }
   ParseStatus parseFP32(OperandVector &Operands) {
     return parseRegister(Operands, FP32Reg);
   }
@@ -553,6 +560,9 @@ class SystemZAsmParser : public MCTargetAsmParser {
   ParseStatus parseFP128(OperandVector &Operands) {
     return parseRegister(Operands, FP128Reg);
   }
+  ParseStatus parseVR16(OperandVector &Operands) {
+    return parseRegister(Operands, VR16Reg);
+  }
   ParseStatus parseVR32(OperandVector &Operands) {
     return parseRegister(Operands, VR32Reg);
   }
@@ -842,11 +852,13 @@ ParseStatus SystemZAsmParser::parseRegister(OperandVector &Operands,
   case GR128Reg:
     Group = RegGR;
     break;
+  case FP16Reg:
   case FP32Reg:
   case FP64Reg:
   case FP128Reg:
     Group = RegFP;
     break;
+  case VR16Reg:
   case VR32Reg:
   case VR64Reg:
   case VR128Reg:
@@ -889,21 +901,25 @@ ParseStatus SystemZAsmParser::parseRegister(OperandVector &Operands,
     return ParseStatus::NoMatch;
 
   // Determine the LLVM register number according to Kind.
+  // clang-format off
   const unsigned *Regs;
   switch (Kind) {
   case GR32Reg:  Regs = SystemZMC::GR32Regs;  break;
   case GRH32Reg: Regs = SystemZMC::GRH32Regs; break;
   case GR64Reg:  Regs = SystemZMC::GR64Regs;  break;
   case GR128Reg: Regs = SystemZMC::GR128Regs; break;
+  case FP16Reg:  Regs = SystemZMC::FP16Regs;  break;
   case FP32Reg:  Regs = SystemZMC::FP32Regs;  break;
   case FP64Reg:  Regs = SystemZMC::FP64Regs;  break;
   case FP128Reg: Regs = SystemZMC::FP128Regs; break;
+  case VR16Reg:  Regs = SystemZMC::VR16Regs;  break;
   case VR32Reg:  Regs = SystemZMC::VR32Regs;  break;
   case VR64Reg:  Regs = SystemZMC::VR64Regs;  break;
   case VR128Reg: Regs = SystemZMC::VR128Regs; break;
   case AR32Reg:  Regs = SystemZMC::AR32Regs;  break;
   case CR64Reg:  Regs = SystemZMC::CR64Regs;  break;
   }
+  // clang-format on
   if (Regs[Reg.Num] == 0)
     return Error(Reg.StartLoc, "invalid register pair");
 
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
index 2bef87696a913..d2ed5cac5c576 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
@@ -64,6 +64,12 @@ const unsigned SystemZMC::GR128Regs[16] = {
     SystemZ::R0Q, 0, SystemZ::R2Q,  0, SystemZ::R4Q,  0, SystemZ::R6Q,  0,
     SystemZ::R8Q, 0, SystemZ::R10Q, 0, SystemZ::R12Q, 0, SystemZ::R14Q, 0};
 
+const unsigned SystemZMC::FP16Regs[16] = {
+    SystemZ::F0H,  SystemZ::F1H,  SystemZ::F2H,  SystemZ::F3H,
+    SystemZ::F4H,  SystemZ::F5H,  SystemZ::F6H,  SystemZ::F7H,
+    SystemZ::F8H,  SystemZ::F9H,  SystemZ::F10H, SystemZ::F11H,
+    SystemZ::F12H, SystemZ::F13H, SystemZ::F14H, SystemZ::F15H};
+
 const unsigned SystemZMC::FP32Regs[16] = {
     SystemZ::F0S,  SystemZ::F1S,  SystemZ::F2S,  SystemZ::F3S,
     SystemZ::F4S,  SystemZ::F5S,  SystemZ::F6S,  SystemZ::F7S,
@@ -80,6 +86,15 @@ const unsigned SystemZMC::FP128Regs[16] = {
     SystemZ::F0Q, SystemZ::F1Q, 0, 0, SystemZ::F4Q,  SystemZ::F5Q,  0, 0,
     SystemZ::F8Q, SystemZ::F9Q, 0, 0, SystemZ::F12Q, SystemZ::F13Q, 0, 0};
 
+const unsigned SystemZMC::VR16Regs[32] = {
+    SystemZ::F0H,  SystemZ::F1H,  SystemZ::F2H,  SystemZ::F3H,  SystemZ::F4H,
+    SystemZ::F5H,  SystemZ::F6H,  SystemZ::F7H,  SystemZ::F8H,  SystemZ::F9H,
+    SystemZ::F10H, SystemZ::F11H, SystemZ::F12H, SystemZ::F13H, SystemZ::F14H,
+    SystemZ::F15H, SystemZ::F16H, SystemZ::F17H, SystemZ::F18H, SystemZ::F19H,
+    SystemZ::F20H, SystemZ::F21H, SystemZ::F22H, SystemZ::F23H, SystemZ::F24H,
+    SystemZ::F25H, SystemZ::F26H, SystemZ::F27H, SystemZ::F28H, SystemZ::F29H,
+    SystemZ::F30H, SystemZ::F31H};
+
 const unsigned SystemZMC::VR32Regs[32] = {
     SystemZ::F0S,  SystemZ::F1S,  SystemZ::F2S,  SystemZ::F3S,  SystemZ::F4S,
     SystemZ::F5S,  SystemZ::F6S,  SystemZ::F7S,  SystemZ::F8S,  SystemZ::F9S,
@@ -132,6 +147,7 @@ unsigned SystemZMC::getFirstReg(unsigned Reg) {
       Map[AR32Regs[I]] = I;
     }
     for (unsigned I = 0; I < 32; ++I) {
+      Map[VR16Regs[I]] = I;
       Map[VR32Regs[I]] = I;
       Map[VR64Regs[I]] = I;
       Map[VR128Regs[I]] = I;
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
index 39c1836a13700..1db1b4b9da002 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
@@ -43,9 +43,11 @@ extern const unsigned GR32Regs[16];
 extern const unsigned GRH32Regs[16];
 extern const unsigned GR64Regs[16];
 extern const unsigned GR128Regs[16];
+extern const unsigned FP16Regs[16];
 extern const unsigned FP32Regs[16];
 extern const unsigned FP64Regs[16];
 extern const unsigned FP128Regs[16];
+extern const unsigned VR16Regs[32];
 extern const unsigned VR32Regs[32];
 extern const unsigned VR64Regs[32];
 extern const unsigned VR128Regs[32];
diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index b3efa579dfe0c..c1ffc287235e5 100644
--- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -136,6 +136,25 @@ static MCInst lowerSubvectorStore(const MachineInstr *MI, unsigned Opcode) {
     .addImm(0);
 }
 
+// MI extracts the first element of the source vector.
+static MCInst lowerVecEltExtraction(const MachineInstr *MI, unsigned Opcode) {
+  return MCInstBuilder(Opcode)
+      .addReg(SystemZMC::getRegAsGR64(MI->getOperand(0).getReg()))
+      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(1).getReg()))
+      .addReg(0)
+      .addImm(0);
+}
+
+// MI inserts value into the first element of the destination vector.
+static MCInst lowerVecEltInsertion(const MachineInstr *MI, unsigned Opcode) {
+  return MCInstBuilder(Opcode)
+      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
+      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
+      .addReg(MI->getOperand(1).getReg())
+      .addReg(0)
+      .addImm(0);
+}
+
 // The XPLINK ABI requires that a no-op encoding the call type is emitted after
 // each call to a subroutine. This information can be used by the called
 // function to determine its entry point, e.g. for generating a backtrace. The
@@ -549,6 +568,10 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     lowerAlignmentHint(MI, LoweredMI, SystemZ::VSTMAlign);
     break;
 
+  case SystemZ::VL16:
+    LoweredMI = lowerSubvectorLoad(MI, SystemZ::VLREPH);
+    break;
+
   case SystemZ::VL32:
     LoweredMI = lowerSubvectorLoad(MI, SystemZ::VLREPF);
     break;
@@ -557,6 +580,10 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     LoweredMI = lowerSubvectorLoad(MI, SystemZ::VLREPG);
     break;
 
+  case SystemZ::VST16:
+    LoweredMI = lowerSubvectorStore(MI, SystemZ::VSTEH);
+    break;
+
   case SystemZ::VST32:
     LoweredMI = lowerSubvectorStore(MI, SystemZ::VSTEF);
     break;
@@ -566,18 +593,19 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     break;
 
   case SystemZ::LFER:
-    LoweredMI = MCInstBuilder(SystemZ::VLGVF)
-      .addReg(SystemZMC::getRegAsGR64(MI->getOperand(0).getReg()))
-      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(1).getReg()))
-      .addReg(0).addImm(0);
+    LoweredMI = lowerVecEltExtraction(MI, SystemZ::VLGVF);
+    break;
+
+  case SystemZ::LFER_16:
+    LoweredMI = lowerVecEltExtraction(MI, SystemZ::VLGVH);
     break;
 
   case SystemZ::LEFR:
-    LoweredMI = MCInstBuilder(SystemZ::VLVGF)
-      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
-      .addReg(SystemZMC::getRegAsVR128(MI->getOperand(0).getReg()))
-      .addReg(MI->getOperand(1).getReg())
-      .addReg(0).addImm(0);
+    LoweredMI = lowerVecEltInsertion(MI, SystemZ::VLVGF);
+    break;
+
+  case SystemZ::LEFR_16:
+    LoweredMI = lowerVecEltInsertion(MI, SystemZ::VLVGH);
     break;
 
 #define LOWER_LOW(NAME)                                                 \
diff --git a/llvm/lib/Target/SystemZ/SystemZCallingConv.td b/llvm/lib/Target/SystemZ/SystemZCallingConv.td
index 99bb697ce2014..0ad872bcb63a7 100644
--- a/llvm/lib/Target/SystemZ/SystemZCallingConv.td
+++ b/llvm/lib/Target/SystemZ/SystemZCallingConv.td
@@ -50,6 +50,7 @@ def RetCC_SystemZ_ELF : CallingConv<[
   // other floating-point argument registers available for code that
   // doesn't care about the ABI.  All floating-point argument registers
   // are call-clobbered, so we can use all of them here.
+  CCIfType<[f16], CCAssignToReg<[F0H, F2H, F4H, F6H]>>,
   CCIfType<[f32], CCAssignToReg<[F0S, F2S, F4S, F6S]>>,
   CCIfType<[f64], CCAssignToReg<[F0D, F2D, F4D, F6D]>>,
 
@@ -115,6 +116,7 @@ def CC_SystemZ_ELF : CallingConv<[
   CCIfType<[i64], CCAssignToReg<[R2D, R3D, R4D, R5D, R6D]>>,
 
   // The first 4 float and double arguments are passed in even registers F0-F6.
+  CCIfType<[f16], CCAssignToReg<[F0H, F2H, F4H, F6H]>>,
   CCIfType<[f32], CCAssignToReg<[F0S, F2S, F4S, F6S]>>,
   CCIfType<[f64], CCAssignToReg<[F0D, F2D, F4D, F6D]>>,
 
@@ -138,7 +140,7 @@ def CC_SystemZ_ELF : CallingConv<[
              CCAssignToStack<16, 8>>>,
 
   // Other arguments are passed in 8-byte-aligned 8-byte stack slots.
-  CCIfType<[i32, i64, f32, f64], CCAssignToStack<8, 8>>
+  CCIfType<[i32, i64, f16, f32, f64], CCAssignToStack<8, 8>>
 ]>;
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/SystemZ/SystemZFeatures.td b/llvm/lib/Target/SystemZ/SystemZFeatures.td
index ec1a7beeab213..2c48da8320fb9 100644
--- a/llvm/lib/Target/SystemZ/SystemZFeatures.td
+++ b/llvm/lib/Target/SystemZ/SystemZFeatures.td
@@ -196,6 +196,8 @@ def FeatureVector : SystemZFeature<
 >;
 def FeatureNoVector : SystemZMissingFeature<"Vector">;
 
+def NoVecHwMode : HwMode<"-vector", [FeatureNoVector]>;
+
 def Arch11NewFeatures : SystemZFeatureList<[
     FeatureLoadAndZeroRightmostByte,
     FeatureLoadStoreOnCond2,
diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
index caf01ccd1ef7c..6f146b67f8566 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
@@ -1204,9 +1204,10 @@ void SystemZDAGToDAGISel::loadVectorConstant(
     SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
     ReplaceNode(Node, BitCast.getNode());
     SelectCode(BitCast.getNode());
-  } else { // float or double
-    unsigned SubRegIdx =
-        (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
+  } else { // half, float or double
+    unsigned SubRegIdx = (VT.getSizeInBits() == 16   ? SystemZ::subreg_h16
+                          : VT.getSizeInBits() == 32 ? SystemZ::subreg_h32
+                                                     : SystemZ::subreg_h64);
     ReplaceNode(
         Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
   }
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 35cee7b39d143..fdbfc196e8fbc 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -103,9 +103,11 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass);
   if (!useSoftFloat()) {
     if (Subtarget.hasVector()) {
+      addRegisterClass(MVT::f16, &SystemZ::VR16BitRegClass);
       addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass);
       addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass);
     } else {
+      addRegisterClass(MVT::f16, &SystemZ::FP16BitRegClass);
       addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass);
       addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass);
     }
@@ -224,23 +226,20 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::SMUL_LOHI, VT, Custom);
       setOperationAction(ISD::UMUL_LOHI, VT, Custom);
 
-      // Only z196 and above have native support for conversions to unsigned.
-      // On z10, promoting to i64 doesn't generate an inexact condition for
-      // values that are outside the i32 range but in the i64 range, so use
-      // the default expansion.
-      if (!Subtarget.hasFPExtension())
-        setOperationAction(ISD::FP_TO_UINT, VT, Expand);
-
-      // Mirror those settings for STRICT_FP_TO_[SU]INT.  Note that these all
-      // default to Expand, so need to be modified to Legal where appropriate.
-      setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Legal);
-      if (Subtarget.hasFPExtension())
-        setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Legal);
-
-      // And similarly for STRICT_[SU]INT_TO_FP.
-      setOperationAction(ISD::STRICT_SINT_TO_FP, VT, Legal);
-      if (Subtarget.hasFPExtension())
-        setOperationAction(ISD::STRICT_UINT_TO_FP, VT, Legal);
+      // The fp<=>i32/i64 conversions are all Legal except for f16 and for
+      // unsigned on z10 (only z196 and above have native support for
+      // unsigned conversions).
+      for (auto Op : {ISD::FP_TO_SINT, ISD::STRICT_FP_TO_SINT,
+                      ISD::SINT_TO_FP, ISD::STRICT_SINT_TO_FP})
+        setOperationAction(Op, VT, Custom);
+      for (auto Op : {ISD::FP_TO_UINT, ISD::STRICT_FP_TO_UINT})
+        setOperationAction(Op, VT, Custom);
+      for (auto Op : {ISD::UINT_TO_FP, ISD::STRICT_UINT_TO_FP}) {
+        // Handle unsigned 32-bit input types as signed 64-bit types on z10.
+        auto OpAction =
+            (!Subtarget.hasFPExtension() && VT == MVT::i32) ? Promote : Custom;
+        setOperationAction(Op, VT, OpAction);
+      }
     }
   }
 
@@ -289,18 +288,18 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
     if (Subtarget.hasVectorEnhancements3()) {
       setOperationAction(ISD::ABS, MVT::i128, Legal);
     }
-
-    // We have to use libcalls for these.
-    setOperationAction(ISD::FP_TO_UINT, MVT::i128, LibCall);
-    setOperationAction(ISD::FP_TO_SINT, MVT::i128, LibCall);
-    setOperationAction(ISD::UINT_TO_FP, MVT::i128, LibCall);
-    setOperationAction(ISD::SINT_TO_FP, MVT::i128, LibCall);
-    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, LibCall);
-    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, LibCall);
-    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, LibCall);
-    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, LibCall);
   }
 
+  // These need custom handling in order to handle the f16 conversions.
+  setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
+  setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
+  setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
+  setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
+  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
+  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
+  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
+  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
+
   // Type legalization will convert 8- and 16-bit atomic operations into
   // forms that operate on i32s (but still keeping the original memory VT).
   // Lower them into full i32 operations.
@@ -343,15 +342,6 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   // Traps are legal, as we will convert them to "j .+2".
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
-  // z10 has instructions for signed but not unsigned FP conversion.
-  // Handle unsigned 32-bit types as signed 64-bit types.
-  if (!Subtarget.hasFPExtension()) {
-    setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote);
-    setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
-    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Promote);
-    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Expand);
-  }
-
   // We have native support for a 64-bit CTLZ, via FLOGR.
   setOperationAction(ISD::CTLZ, MVT::i32, Promote);
   setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Promote);
@@ -548,11 +538,29 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   }
 
   // Handle floating-point types.
+  if (!useSoftFloat()) {
+    // Promote all f16 operations to float, with some exceptions below.
+    for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
+      setOperationAction(Opc, MVT::f16, Promote);
+    setOperationAction(ISD::ConstantFP, MVT::f16, Expand);
+    for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
+      setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
+      setTruncStoreAction(VT, MVT::f16, Expand);
+    }
+    for (auto Op : {ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE})
+      setOperationAction(Op, MVT::f16, Subtarget.hasVector() ? Legal : Custom);
+    setOperationAction(ISD::FP_ROUND, MVT::f16, LibCall);
+    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, LibCall);
+    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
+    setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);
+    setOperationAction(ISD::FCOPYSIGN, MVT::f16, Legal);
+  }
+
   for (unsigned I = MVT::FIRST_FP_VALUETYPE;
        I <= MVT::LAST_FP_VALUETYPE;
        ++I) {
     MVT VT = MVT::SimpleValueType(I);
-    if (isTypeLegal(VT)) {
+    if (isTypeLegal(VT) && VT != MVT::f16) {
       // We can use FI for FRINT.
       setOperationAction(ISD::FRINT, VT, Legal);
 
@@ -585,7 +593,6 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
       setOperationAction(ISD::STRICT_FRINT, VT, Legal);
       setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal);
-      setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
       if (Subtarget.hasFPExtension()) {
         setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
         setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
@@ -594,6 +601,10 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::STRICT_FROUND, VT, Legal);
         setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
       }
+
+      // Extension from f16 needs libcall.
+      setOperationAction(ISD::FP_EXTEND, VT, Custom);
+      setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
     }
   }
 
@@ -1607,7 +1618,9 @@ SystemZTargetLowering::getRegForInlineAsmConstraint(
 
     case 'f': // Floating-point register
       if (!useSoftFloat()) {
-        if (VT.getSizeInBits() == 64)
+        if (VT.getSizeInBits() == 16)
+          return std::make_pair(0U, &SystemZ::FP16BitRegClass);
+        else if (VT.getSizeInBits() == 64)
           return std::make_pair(0U, &SystemZ::FP64BitRegClass);
         else if (VT.getSizeInBits() == 128)
           return std::make_pair(0U, &SystemZ::FP128BitRegClass);
@@ -1617,6 +1630,8 @@ SystemZTargetLowering::getRegForInlineAsmConstraint(
 
     case 'v': // Vector register
       if (Subtarget.hasVector()) {
+        if (VT.getSizeInBits() == 16)
+          return std::make_pair(0U, &SystemZ::VR16BitRegClass);
         if (VT.getSizeInBits() == 32)
           return std::make_pair(0U, &SystemZ::VR32BitRegClass);
         if (VT.getSizeInBits() == 64)
@@ -1652,6 +1667,9 @@ SystemZTargetLowering::getRegForInlineAsmConstraint(
       if (useSoftFloat())
         return std::make_pair(
             0u, static_cast<const TargetRegisterClass *>(nullptr));
+      if (getVTSizeInBits() == 16)
+        return parseRegisterNumber(Constraint, &SystemZ::FP16BitRegClass,
+                                   SystemZMC::FP16Regs, 16);
       if (getVTSizeInBits() == 32)
         return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass,
                                    SystemZMC::FP32Regs, 16);
@@ -1665,6 +1683,9 @@ SystemZTargetLowering::getRegForInlineAsmConstraint(
       if (!Subtarget.hasVector())
         return std::make_pair(
             0u, static_cast<const TargetRegisterClass *>(nullptr));
+      if (getVTSizeInBits() == 16)
+        return parseRegisterNumber(Constraint, &SystemZ::VR16BitRegClass,
+                                   SystemZMC::VR16Regs, 32);
       if (getVTSizeInBits() == 32)
         return parseRegisterNumber(Constraint, &SystemZ::VR32BitRegClass,
                                    SystemZMC::VR32Regs, 32);
@@ -1941,6 +1962,10 @@ SDValue SystemZTargetLowering::LowerFormalArguments(
         NumFixedGPRs += 1;
         RC = &SystemZ::GR64BitRegClass;
         break;
+      case MVT::f16:
+        NumFixedFPRs += 1;
+        RC = &SystemZ::FP16BitRegClass;
+        break;
       case MVT::f32:
         NumFixedFPRs += 1;
         RC = &SystemZ::FP32BitRegClass;
@@ -1985,9 +2010,12 @@ SDValue SystemZTargetLowering::LowerFormalArguments(
       // from this parameter.  Unpromoted ints and floats are
       // passed as right-justified 8-byte values.
       SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
-      if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
+      if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32 ||
+          VA.getLocVT() == MVT::f16) {
+        unsigned SlotOffs = VA.getLocVT() == MVT::f16 ? 6 : 4;
         FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
-                          DAG.getIntPtrConstant(4, DL));
+                          DAG.getIntPtrConstant(SlotOffs, DL));
+      }
       ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
                              MachinePointerInfo::getFixedStack(MF, FI));
     }
@@ -2300,6 +2328,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
                         VA.getLocMemOffset();
       if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
         Offset += 4;
+      else if (VA.getLocVT() == MVT::f16)
+        Offset += 6;
       SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
                                     DAG.getIntPtrConstant(Offset, DL));
 
@@ -2736,13 +2766,21 @@ static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
 static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
                                    unsigned Opcode) {
   // Copy all operands except the intrinsic ID.
+  SDLoc DL(Op);
   unsigned NumOps = Op.getNumOperands();
   SmallVector<SDValue, 6> Ops;
   Ops.reserve(NumOps - 1);
-  for (unsigned I = 1; I < NumOps; ++I)
-    Ops.push_back(Op.getOperand(I));
+  for (unsigned I = 1; I < NumOps; ++I) {
+    SDValue CurrOper = Op.getOperand(I);
+    if (CurrOper.getValueType() == MVT::f16) {
+      assert((Op.getConstantOperandVal(0) == Intrinsic::s390_tdc && I == 1) &&
+             "Unhandled intrinsic with f16 operand.");
+      CurrOper = DAG.getFPExtendOrRound(CurrOper, DL, MVT::f32);
+    }
+    Ops.push_back(CurrOper);
+  }
 
-  SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), Op->getVTList(), Ops);
+  SDValue Intr = DAG.getNode(Opcode, DL, Op->getVTList(), Ops);
   return Intr.getNode();
 }
 
@@ -3884,6 +3922,14 @@ SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
   SDLoc DL(Op);
 
+  // SELECT_CC involving f16 will not have the cmp-ops promoted by the
+  // legalizer, as it will be handled according to the type of the resulting
+  // value. Extend them here if needed.
+  if (CmpOp0.getSimpleValueType() == MVT::f16) {
+    CmpOp0 = DAG.getFPExtendOrRound(CmpOp0, SDLoc(CmpOp0), MVT::f32);
+    CmpOp1 = DAG.getFPExtendOrRound(CmpOp1, SDLoc(CmpOp1), MVT::f32);
+  }
+
   Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
 
   // Check for absolute and negative-absolute selections, including those
@@ -4973,6 +5019,22 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
   return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
 }
 
+SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
+                                                SelectionDAG &DAG) const {
+  EVT RegVT = Op.getValueType();
+  if (RegVT.getSizeInBits() == 128)
+    return lowerATOMIC_LDST_I128(Op, DAG);
+  return lowerLoadF16(Op, DAG);
+}
+
+SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  auto *Node = cast<AtomicSDNode>(Op.getNode());
+  if (Node->getMemoryVT().getSizeInBits() == 128)
+    return lowerATOMIC_LDST_I128(Op, DAG);
+  return lowerStoreF16(Op, DAG);
+}
+
 SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
                                                      SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
@@ -6736,6 +6798,161 @@ static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG) {
   return Op;
 }
 
+SDValue SystemZTargetLowering::lowerFP_EXTEND(SDValue Op,
+                                              SelectionDAG &DAG) const {
+  SDValue In = Op.getOperand(Op->isStrictFPOpcode() ? 1 : 0);
+  if (In.getSimpleValueType() != MVT::f16)
+    return Op;      // Legal
+  return SDValue(); // Let legalizer emit the libcall.
+}
+
+SDValue SystemZTargetLowering::useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
+                                          MVT VT, SDValue Arg, SDLoc DL,
+                                          SDValue Chain, bool IsStrict) const {
+  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
+  MakeLibCallOptions CallOptions;
+  SDValue Result;
+  std::tie(Result, Chain) =
+      makeLibCall(DAG, LC, VT, Arg, CallOptions, DL, Chain);
+  return IsStrict ? DAG.getMergeValues({Result, Chain}, DL) : Result;
+}
+
+SDValue SystemZTargetLowering::lower_FP_TO_INT(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  bool IsSigned = (Op->getOpcode() == ISD::FP_TO_SINT ||
+                   Op->getOpcode() == ISD::STRICT_FP_TO_SINT);
+  bool IsStrict = Op->isStrictFPOpcode();
+  SDLoc DL(Op);
+  MVT VT = Op.getSimpleValueType();
+  SDValue InOp = Op.getOperand(IsStrict ? 1 : 0);
+  SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
+  EVT InVT = InOp.getValueType();
+
+  // FP to unsigned is not directly supported on z10.  Promoting an i32
+  // result to (signed) i64 doesn't generate an inexact condition (fp
+  // exception) for values that are outside the i32 range but in the i64
+  // range, so use the default expansion.
+  if (!Subtarget.hasFPExtension() && !IsSigned)
+    // Expand i32/i64. F16 values will be recognized to fit and extended.
+    return SDValue();
+
+  // Conversion from f16 is done via f32.
+  if (InOp.getSimpleValueType() == MVT::f16) {
+    SmallVector<SDValue, 2> Results;
+    LowerOperationWrapper(Op.getNode(), Results, DAG);
+    return DAG.getMergeValues(Results, DL);
+  }
+
+  if (VT == MVT::i128) {
+    RTLIB::Libcall LC =
+        IsSigned ? RTLIB::getFPTOSINT(InVT, VT) : RTLIB::getFPTOUINT(InVT, VT);
+    return useLibCall(DAG, LC, VT, InOp, DL, Chain, IsStrict);
+  }
+
+  return Op; // Legal
+}
+
+SDValue SystemZTargetLowering::lower_INT_TO_FP(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  bool IsSigned = (Op->getOpcode() == ISD::SINT_TO_FP ||
+                   Op->getOpcode() == ISD::STRICT_SINT_TO_FP);
+  bool IsStrict = Op->isStrictFPOpcode();
+  SDLoc DL(Op);
+  MVT VT = Op.getSimpleValueType();
+  SDValue InOp = Op.getOperand(IsStrict ? 1 : 0);
+  SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
+  EVT InVT = InOp.getValueType();
+
+  // Conversion to f16 is done via f32.
+  if (VT == MVT::f16) {
+    SmallVector<SDValue, 2> Results;
+    LowerOperationWrapper(Op.getNode(), Results, DAG);
+    return DAG.getMergeValues(Results, DL);
+  }
+
+  // Unsigned to fp is not directly supported on z10.
+  if (!Subtarget.hasFPExtension() && !IsSigned)
+    return SDValue(); // Expand i64.
+
+  if (InVT == MVT::i128) {
+    RTLIB::Libcall LC =
+        IsSigned ? RTLIB::getSINTTOFP(InVT, VT) : RTLIB::getUINTTOFP(InVT, VT);
+    return useLibCall(DAG, LC, VT, InOp, DL, Chain, IsStrict);
+  }
+
+  return Op; // Legal
+}
+
+// Shift the lower 2 bytes of Op to the left in order to insert into the
+// upper 2 bytes of the FP register.
+static SDValue convertToF16(SDValue Op, SelectionDAG &DAG) {
+  assert(Op.getSimpleValueType() == MVT::i64 &&
+         "Expexted to convert i64 to f16.");
+  SDLoc DL(Op);
+  SDValue Shft = DAG.getNode(ISD::SHL, DL, MVT::i64, Op,
+                             DAG.getConstant(48, DL, MVT::i64));
+  SDValue BCast = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Shft);
+  SDValue F16Val =
+      DAG.getTargetExtractSubreg(SystemZ::subreg_h16, DL, MVT::f16, BCast);
+  return F16Val;
+}
+
+// Extract Op into GPR and shift the 2 f16 bytes to the right.
+static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
+  assert(Op.getSimpleValueType() == MVT::f16 &&
+         "Expected to convert f16 to i64.");
+  SDNode *U32 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
+  SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h16, DL, MVT::f64,
+                                           SDValue(U32, 0), Op);
+  SDValue BCast = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
+  SDValue Shft = DAG.getNode(ISD::SRL, DL, MVT::i64, BCast,
+                             DAG.getConstant(48, DL, MVT::i32));
+  return Shft;
+}
+
+// Lower an f16 LOAD in case of no vector support.
+SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
+                                            SelectionDAG &DAG) const {
+  EVT RegVT = Op.getValueType();
+  assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
+
+  // Load as integer.
+  SDLoc DL(Op);
+  SDValue NewLd;
+  if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Op.getNode())) {
+    assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
+    NewLd = DAG.getAtomic(ISD::ATOMIC_LOAD, DL, MVT::i16, MVT::i64,
+                          AtomicLd->getChain(), AtomicLd->getBasePtr(),
+                          AtomicLd->getMemOperand());
+    cast<AtomicSDNode>(NewLd)->setExtensionType(ISD::EXTLOAD);
+  } else {
+    LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
+    assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
+    NewLd =
+        DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i64, Ld->getChain(),
+                       Ld->getBasePtr(), Ld->getPointerInfo(), MVT::i16,
+                       Ld->getOriginalAlign(), Ld->getMemOperand()->getFlags());
+  }
+  SDValue F16Val = convertToF16(NewLd, DAG);
+  return DAG.getMergeValues({F16Val, NewLd.getValue(1)}, DL);
+}
+
+// Lower an f16 STORE in case of no vector support.
+SDValue SystemZTargetLowering::lowerStoreF16(SDValue Op,
+                                             SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  SDValue Shft = convertFromF16(Op->getOperand(1), DL, DAG);
+
+  if (auto *AtomicSt = dyn_cast<AtomicSDNode>(Op.getNode()))
+    return DAG.getAtomic(ISD::ATOMIC_STORE, DL, MVT::i16, AtomicSt->getChain(),
+                         Shft, AtomicSt->getBasePtr(),
+                         AtomicSt->getMemOperand());
+
+  StoreSDNode *St = cast<StoreSDNode>(Op.getNode());
+  return DAG.getTruncStore(St->getChain(), DL, Shft, St->getBasePtr(), MVT::i16,
+                           St->getMemOperand());
+}
+
 SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
                                                SelectionDAG &DAG) const {
   SDLoc DL(Op);
@@ -6766,6 +6983,8 @@ SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
     TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
   SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, MVT::i64);
 
+  if (Arg.getSimpleValueType() == MVT::f16)
+    Arg = DAG.getFPExtendOrRound(Arg, SDLoc(Arg), MVT::f32);
   SDValue Intr = DAG.getNode(SystemZISD::TDC, DL, ResultVT, Arg, TDCMaskV);
   return getCCResult(DAG, Intr);
 }
@@ -6859,8 +7078,9 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
   case ISD::ATOMIC_SWAP:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
   case ISD::ATOMIC_STORE:
+    return lowerATOMIC_STORE(Op, DAG);
   case ISD::ATOMIC_LOAD:
-    return lowerATOMIC_LDST_I128(Op, DAG);
+    return lowerATOMIC_LOAD(Op, DAG);
   case ISD::ATOMIC_LOAD_ADD:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
   case ISD::ATOMIC_LOAD_SUB:
@@ -6921,6 +7141,23 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
     return lowerFSHL(Op, DAG);
   case ISD::FSHR:
     return lowerFSHR(Op, DAG);
+  case ISD::FP_EXTEND:
+  case ISD::STRICT_FP_EXTEND:
+    return lowerFP_EXTEND(Op, DAG);
+  case ISD::FP_TO_UINT:
+  case ISD::FP_TO_SINT:
+  case ISD::STRICT_FP_TO_UINT:
+  case ISD::STRICT_FP_TO_SINT:
+    return lower_FP_TO_INT(Op, DAG);
+  case ISD::UINT_TO_FP:
+  case ISD::SINT_TO_FP:
+  case ISD::STRICT_UINT_TO_FP:
+  case ISD::STRICT_SINT_TO_FP:
+    return lower_INT_TO_FP(Op, DAG);
+  case ISD::LOAD:
+    return lowerLoadF16(Op, DAG);
+  case ISD::STORE:
+    return lowerStoreF16(Op, DAG);
   case ISD::IS_FPCLASS:
     return lowerIS_FPCLASS(Op, DAG);
   case ISD::GET_ROUNDING:
@@ -6984,8 +7221,7 @@ static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src,
   return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i128, Lo, Hi);
 }
 
-// Lower operations with invalid operand or result types (currently used
-// only for 128-bit integer types).
+// Lower operations with invalid operand or result types.
 void
 SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
                                              SmallVectorImpl<SDValue> &Results,
@@ -7045,11 +7281,87 @@ SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
     break;
   }
   case ISD::BITCAST: {
+    if (useSoftFloat())
+      return;
+    SDLoc DL(N);
     SDValue Src = N->getOperand(0);
-    if (N->getValueType(0) == MVT::i128 && Src.getValueType() == MVT::f128 &&
-        !useSoftFloat()) {
-      SDLoc DL(N);
+    EVT SrcVT = Src.getValueType();
+    EVT ResVT = N->getValueType(0);
+    if (ResVT == MVT::i128 && SrcVT == MVT::f128)
       Results.push_back(expandBitCastF128ToI128(DAG, Src, DL));
+    else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
+      if (Subtarget.hasVector()) {
+        SDValue In32 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src);
+        Results.push_back(SDValue(
+            DAG.getMachineNode(SystemZ::LEFR_16, DL, MVT::f16, In32), 0));
+      } else {
+        SDValue In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Src);
+        Results.push_back(convertToF16(In64, DAG));
+      }
+    } else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
+      SDValue ExtractedVal =
+          Subtarget.hasVector()
+              ? SDValue(DAG.getMachineNode(SystemZ::LFER_16, DL, MVT::i32, Src),
+                        0)
+              : convertFromF16(Src, DL, DAG);
+      Results.push_back(DAG.getZExtOrTrunc(ExtractedVal, DL, ResVT));
+    }
+    break;
+  }
+  case ISD::UINT_TO_FP:
+  case ISD::SINT_TO_FP:
+  case ISD::STRICT_UINT_TO_FP:
+  case ISD::STRICT_SINT_TO_FP: {
+    if (useSoftFloat())
+      return;
+    bool IsStrict = N->isStrictFPOpcode();
+    SDLoc DL(N);
+    SDValue InOp = N->getOperand(IsStrict ? 1 : 0);
+    EVT ResVT = N->getValueType(0);
+    SDValue Chain = IsStrict ? N->getOperand(0) : DAG.getEntryNode();
+    if (ResVT == MVT::f16) {
+      if (!IsStrict) {
+        SDValue OpF32 = DAG.getNode(N->getOpcode(), DL, MVT::f32, InOp);
+        Results.push_back(DAG.getFPExtendOrRound(OpF32, DL, MVT::f16));
+      } else {
+        SDValue OpF32 =
+            DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::f32, MVT::Other),
+                        {Chain, InOp});
+        SDValue F16Res;
+        std::tie(F16Res, Chain) = DAG.getStrictFPExtendOrRound(
+            OpF32, OpF32.getValue(1), DL, MVT::f16);
+        Results.push_back(F16Res);
+        Results.push_back(Chain);
+      }
+    }
+    break;
+  }
+  case ISD::FP_TO_UINT:
+  case ISD::FP_TO_SINT:
+  case ISD::STRICT_FP_TO_UINT:
+  case ISD::STRICT_FP_TO_SINT: {
+    if (useSoftFloat())
+      return;
+    bool IsStrict = N->isStrictFPOpcode();
+    SDLoc DL(N);
+    EVT ResVT = N->getValueType(0);
+    SDValue InOp = N->getOperand(IsStrict ? 1 : 0);
+    EVT InVT = InOp->getValueType(0);
+    SDValue Chain = IsStrict ? N->getOperand(0) : DAG.getEntryNode();
+    if (InVT == MVT::f16) {
+      if (!IsStrict) {
+        SDValue InF32 = DAG.getFPExtendOrRound(InOp, DL, MVT::f32);
+        Results.push_back(DAG.getNode(N->getOpcode(), DL, ResVT, InF32));
+      } else {
+        SDValue InF32;
+        std::tie(InF32, Chain) =
+            DAG.getStrictFPExtendOrRound(InOp, Chain, DL, MVT::f32);
+        SDValue OpF32 =
+            DAG.getNode(N->getOpcode(), DL, DAG.getVTList(ResVT, MVT::Other),
+                        {Chain, InF32});
+        Results.push_back(OpF32);
+        Results.push_back(OpF32.getValue(1));
+      }
     }
     break;
   }
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 4763af75552da..f438332c2dc4f 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -627,6 +627,9 @@ class SystemZTargetLowering : public TargetLowering {
                    bool IsSigned, SDLoc DL, bool DoesNotReturn,
                    bool IsReturnValueUsed) const;
 
+  SDValue useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, MVT VT, SDValue Arg,
+                     SDLoc DL, SDValue Chain, bool IsStrict) const;
+
   bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
                       bool isVarArg,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
@@ -720,6 +723,8 @@ class SystemZTargetLowering : public TargetLowering {
   SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerVECREDUCE_ADD(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LDST_I128(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
                               unsigned Opcode) const;
@@ -743,6 +748,12 @@ class SystemZTargetLowering : public TargetLowering {
   SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const;
   SDValue lowerFSHL(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFSHR(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lower_FP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lower_INT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerLoadF16(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerStoreF16(SDValue Op, SelectionDAG &DAG) const;
+
   SDValue lowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFP.td b/llvm/lib/Target/SystemZ/SystemZInstrFP.td
index bef38b9cb809b..7775f456bbdc1 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrFP.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrFP.td
@@ -36,6 +36,8 @@ defm CondStoreF64 : CondStores<FP64, simple_store,
 
 // Load zero.
 let isAsCheapAsAMove = 1, isMoveImm = 1 in {
+  let isCodeGenOnly = 1 in
+    def LZER_16 : InherentRRE<"lzer", 0xB374, FP16,  fpimm0>;
   def LZER : InherentRRE<"lzer", 0xB374, FP32,  fpimm0>;
   def LZDR : InherentRRE<"lzdr", 0xB375, FP64,  fpimm0>;
   def LZXR : InherentRRE<"lzxr", 0xB376, FP128, fpimm0>;
@@ -47,8 +49,11 @@ let isMoveReg = 1 in  {
   def LDR : UnaryRR <"ldr", 0x28,   null_frag, FP64,  FP64>;
   def LXR : UnaryRRE<"lxr", 0xB365, null_frag, FP128, FP128>;
   // For z13 we prefer LDR over LER to avoid partial register dependencies.
-  let isCodeGenOnly = 1 in
+  let isCodeGenOnly = 1 in {
+    def LER16 : UnaryRR<"ler", 0x38, null_frag, FP16, FP16>;
+    def LDR16 : UnaryRR<"ldr", 0x28, null_frag, FP16, FP16>;
     def LDR32 : UnaryRR<"ldr", 0x28, null_frag, FP32, FP32>;
+  }
 }
 
 
@@ -79,8 +84,25 @@ let Predicates = [FeatureNoVectorEnhancements1] in
 def LGDR : UnaryRRE<"lgdr", 0xB3CD, bitconvert, GR64, FP64>;
 def LDGR : UnaryRRE<"ldgr", 0xB3C1, bitconvert, FP64, GR64>;
 
+// fcopysign with an FP16 result.
+let isCodeGenOnly = 1 in {
+  def CPSDRhh : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP16, FP16, FP16>;
+  def CPSDRhs : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP16, FP16, FP32>;
+  def CPSDRhd : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP16, FP16, FP64>;
+}
+
+// The sign of an FP128 is in the high register.
+let Predicates = [FeatureNoVectorEnhancements1] in
+  def : Pat<(fcopysign FP16:$src1, (f128 FP128:$src2)),
+            (CPSDRhd FP16:$src1, (EXTRACT_SUBREG FP128:$src2, subreg_h64))>;
+let Predicates = [FeatureVectorEnhancements1] in
+  def : Pat<(fcopysign FP16:$src1, (f128 VR128:$src2)),
+            (CPSDRhd FP16:$src1, (EXTRACT_SUBREG VR128:$src2, subreg_h64))>;
+
+
 // fcopysign with an FP32 result.
 let isCodeGenOnly = 1 in {
+  def CPSDRsh : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP32, FP32, FP16>;
   def CPSDRss : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP32, FP32, FP32>;
   def CPSDRsd : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP32, FP32, FP64>;
 }
@@ -94,8 +116,10 @@ let Predicates = [FeatureVectorEnhancements1] in
             (CPSDRsd FP32:$src1, (EXTRACT_SUBREG VR128:$src2, subreg_h64))>;
 
 // fcopysign with an FP64 result.
-let isCodeGenOnly = 1 in
+let isCodeGenOnly = 1 in {
+  def CPSDRdh : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP64, FP64, FP16>;
   def CPSDRds : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP64, FP64, FP32>;
+}
 def CPSDRdd : BinaryRRFb<"cpsdr", 0xB372, fcopysign, FP64, FP64, FP64>;
 
 // The sign of an FP128 is in the high register.
@@ -113,6 +137,8 @@ class CopySign128<RegisterOperand cls, dag upper>
         (INSERT_SUBREG FP128:$src1, upper, subreg_h64)>;
 
 let Predicates = [FeatureNoVectorEnhancements1] in {
+  def : CopySign128<FP16,  (CPSDRdh (EXTRACT_SUBREG FP128:$src1, subreg_h64),
+                                    FP16:$src2)>;
   def : CopySign128<FP32,  (CPSDRds (EXTRACT_SUBREG FP128:$src1, subreg_h64),
                                     FP32:$src2)>;
   def : CopySign128<FP64,  (CPSDRdd (EXTRACT_SUBREG FP128:$src1, subreg_h64),
@@ -131,6 +157,9 @@ defm LoadStoreF128 : MVCLoadStore<load, f128, MVCImm, 15>;
 //===----------------------------------------------------------------------===//
 
 let canFoldAsLoad = 1, SimpleBDXLoad = 1, mayLoad = 1 in {
+  let isCodeGenOnly = 1 in
+    // Reload f16 from 4-byte spill slot.
+    defm LE16 : UnaryRXPair<"le", 0x78, 0xED64, z_load, FP16, 4>;
   defm LE : UnaryRXPair<"le", 0x78, 0xED64, z_load, FP32, 4>;
   defm LD : UnaryRXPair<"ld", 0x68, 0xED65, z_load, FP64, 8>;
 
@@ -151,6 +180,9 @@ let canFoldAsLoad = 1, SimpleBDXLoad = 1, mayLoad = 1 in {
 //===----------------------------------------------------------------------===//
 
 let SimpleBDXStore = 1, mayStore = 1 in {
+  let isCodeGenOnly = 1 in
+    // Spill f16 to 4-byte spill slot.
+    defm STE16 : StoreRXPair<"ste", 0x70, 0xED66, store, FP16, 4>;
   defm STE : StoreRXPair<"ste", 0x70, 0xED66, store, FP32, 4>;
   defm STD : StoreRXPair<"std", 0x60, 0xED67, store, FP64, 8>;
 
@@ -236,7 +268,7 @@ let Uses = [FPC], mayRaiseFPException = 1, Predicates = [FeatureFPExtension] in
   def CXGBRA : TernaryRRFe<"cxgbra", 0xB3A6, FP128, GR64>;
 }
 
-// Convert am unsigned integer register value to a floating-point one.
+// Convert an unsigned integer register value to a floating-point one.
 let Predicates = [FeatureFPExtension] in {
   let Uses = [FPC], mayRaiseFPException = 1 in {
     def CELFBR : TernaryRRFe<"celfbr", 0xB390, FP32,  GR32>;
@@ -333,8 +365,10 @@ let Defs = [CC], CCValues = 0xF, CompareZeroCCMask = 0xF in {
 }
 // Generic form, which does not set CC.
 def LCDFR : UnaryRRE<"lcdfr", 0xB373, fneg, FP64,  FP64>;
-let isCodeGenOnly = 1 in
+let isCodeGenOnly = 1 in {
+  def LCDFR_16 : UnaryRRE<"lcdfr", 0xB373, fneg, FP16,  FP16>;
   def LCDFR_32 : UnaryRRE<"lcdfr", 0xB373, fneg, FP32,  FP32>;
+}
 
 // Absolute value (Load Positive).
 let Defs = [CC], CCValues = 0xF, CompareZeroCCMask = 0xF in {
@@ -600,6 +634,7 @@ let hasSideEffects = 1 in {
 // Peepholes
 //===----------------------------------------------------------------------===//
 
+def : Pat<(f16  fpimmneg0), (LCDFR_16 (LZER_16))>;
 def : Pat<(f32  fpimmneg0), (LCDFR_32 (LZER))>;
 def : Pat<(f64  fpimmneg0), (LCDFR (LZDR))>;
 def : Pat<(f128 fpimmneg0), (LCXBR (LZXR))>;
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 91a4aa9c73010..1ae3994eb0e01 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -61,7 +61,8 @@ void SystemZInstrInfo::anchor() {}
 
 SystemZInstrInfo::SystemZInstrInfo(SystemZSubtarget &sti)
     : SystemZGenInstrInfo(-1, -1),
-      RI(sti.getSpecialRegisters()->getReturnFunctionAddressRegister()),
+      RI(sti.getSpecialRegisters()->getReturnFunctionAddressRegister(),
+         sti.getHwMode()),
       STI(sti) {}
 
 // MI is a 128-bit load or store.  Split it into two 64-bit loads or stores,
@@ -989,6 +990,8 @@ void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   unsigned Opcode;
   if (SystemZ::GR64BitRegClass.contains(DestReg, SrcReg))
     Opcode = SystemZ::LGR;
+  else if (SystemZ::FP16BitRegClass.contains(DestReg, SrcReg))
+    Opcode = STI.hasVector() ? SystemZ::LDR16 : SystemZ::LER16;
   else if (SystemZ::FP32BitRegClass.contains(DestReg, SrcReg))
     // For z13 we prefer LDR over LER to avoid partial register dependencies.
     Opcode = STI.hasVector() ? SystemZ::LDR32 : SystemZ::LER;
@@ -1259,9 +1262,10 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
     return nullptr;
 
   unsigned OpNum = Ops[0];
-  assert(Size * 8 ==
-           TRI->getRegSizeInBits(*MF.getRegInfo()
-                               .getRegClass(MI.getOperand(OpNum).getReg())) &&
+  const TargetRegisterClass *RC =
+      MF.getRegInfo().getRegClass(MI.getOperand(OpNum).getReg());
+  assert((Size * 8 == TRI->getRegSizeInBits(*RC) ||
+          (RC == &SystemZ::FP16BitRegClass && Size == 4 && !STI.hasVector())) &&
          "Invalid size combination");
 
   if ((Opcode == SystemZ::AHI || Opcode == SystemZ::AGHI) && OpNum == 0 &&
@@ -1900,6 +1904,9 @@ void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC,
              RC == &SystemZ::ADDR128BitRegClass) {
     LoadOpcode = SystemZ::L128;
     StoreOpcode = SystemZ::ST128;
+  } else if (RC == &SystemZ::FP16BitRegClass && !STI.hasVector()) {
+    LoadOpcode = SystemZ::LE16;
+    StoreOpcode = SystemZ::STE16;
   } else if (RC == &SystemZ::FP32BitRegClass) {
     LoadOpcode = SystemZ::LE;
     StoreOpcode = SystemZ::STE;
@@ -1909,6 +1916,10 @@ void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC,
   } else if (RC == &SystemZ::FP128BitRegClass) {
     LoadOpcode = SystemZ::LX;
     StoreOpcode = SystemZ::STX;
+  } else if (RC == &SystemZ::FP16BitRegClass ||
+             RC == &SystemZ::VR16BitRegClass) {
+    LoadOpcode = SystemZ::VL16;
+    StoreOpcode = SystemZ::VST16;
   } else if (RC == &SystemZ::VR32BitRegClass) {
     LoadOpcode = SystemZ::VL32;
     StoreOpcode = SystemZ::VST32;
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrVector.td b/llvm/lib/Target/SystemZ/SystemZInstrVector.td
index d8c48239ac633..10de8b05cf45f 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrVector.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrVector.td
@@ -142,7 +142,8 @@ let Predicates = [FeatureVector] in {
   // LEY and LDY offer full 20-bit displacement fields.  It's often better
   // to use those instructions rather than force a 20-bit displacement
   // into a GPR temporary.
-  let mayLoad = 1, canFoldAsLoad = 1 in {
+  let mayLoad = 1, SimpleBDXLoad = 1, canFoldAsLoad = 1 in {
+    def VL16 : UnaryAliasVRX<z_load, v16hb, bdxaddr12pair>;
     def VL32 : UnaryAliasVRX<z_load, v32sb, bdxaddr12pair>;
     def VL64 : UnaryAliasVRX<z_load, v64db, bdxaddr12pair>;
   }
@@ -239,7 +240,8 @@ let Predicates = [FeatureVector] in {
   // STEY and STDY offer full 20-bit displacement fields.  It's often better
   // to use those instructions rather than force a 20-bit displacement
   // into a GPR temporary.
-  let mayStore = 1 in {
+  let mayStore = 1, SimpleBDXStore = 1 in {
+    def VST16 : StoreAliasVRX<store, v16hb, bdxaddr12pair>;
     def VST32 : StoreAliasVRX<store, v32sb, bdxaddr12pair>;
     def VST64 : StoreAliasVRX<store, v64db, bdxaddr12pair>;
   }
@@ -2208,6 +2210,8 @@ let Predicates = [FeatureVector] in {
   def : Pat<(f32 (bitconvert (i32 GR32:$src))), (LEFR GR32:$src)>;
   def : Pat<(i32 (bitconvert (f32 VR32:$src))),
             (EXTRACT_SUBREG (LFER VR32:$src), subreg_l32)>;
+  def LEFR_16 : UnaryAliasVRS<VR16, GR32>;
+  def LFER_16 : UnaryAliasVRS<GR32, VR16>;
 }
 
 // Floating-point values are stored in element 0 of the corresponding
diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
index 1e0c043682157..177f2c50e4f01 100644
--- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -232,8 +232,9 @@ SystemZELFRegisters::getCallPreservedMask(const MachineFunction &MF,
   return CSR_SystemZ_ELF_RegMask;
 }
 
-SystemZRegisterInfo::SystemZRegisterInfo(unsigned int RA)
-    : SystemZGenRegisterInfo(RA) {}
+SystemZRegisterInfo::SystemZRegisterInfo(unsigned int RA, unsigned int HwMode)
+    : SystemZGenRegisterInfo(RA, /*DwarfFlavour=*/0, /*EHFlavour=*/0, /*PC=*/0,
+                             HwMode) {}
 
 const MCPhysReg *
 SystemZRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
index 4f497f8d23d29..460be432811a4 100644
--- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
@@ -129,7 +129,7 @@ class SystemZELFRegisters : public SystemZCallingConventionRegisters {
 
 struct SystemZRegisterInfo : public SystemZGenRegisterInfo {
 public:
-  SystemZRegisterInfo(unsigned int RA);
+  SystemZRegisterInfo(unsigned int RA, unsigned int HwMode);
 
   /// getPointerRegClass - Return the register class to use to hold pointers.
   /// This is currently only used by LOAD_STACK_GUARD, which requires a non-%r0
diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.td b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.td
index 8f9bb56f2eb3b..e79f12b449a88 100644
--- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.td
+++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.td
@@ -20,6 +20,7 @@ class SystemZRegWithSubregs<string n, list<Register> subregs>
 }
 
 let Namespace = "SystemZ" in {
+def subreg_h16   : SubRegIndex<16, 16>;
 def subreg_l32   : SubRegIndex<32, 0>;  // Also acts as subreg_hl32.
 def subreg_h32   : SubRegIndex<32, 32>; // Also acts as subreg_hh32.
 def subreg_l64   : SubRegIndex<64, 0>;
@@ -34,7 +35,9 @@ def subreg_ll32  : ComposedSubRegIndex<subreg_l64, subreg_l32>;
 // If the user provides an alternate order list of regs, it will be used for
 // XPLINK. Otherwise, by default, XPLINK will use the regList ordering as well
 multiclass SystemZRegClass<string name, list<ValueType> types, int size,
-                           dag regList, list<dag> altRegList = [regList], bit allocatable = 1> {
+                           dag regList, list<dag> altRegList = [regList],
+                           bit allocatable = 1,
+                           RegInfoByHwMode RI = RegInfoByHwMode<[], []>> {
   def AsmOperand : AsmOperandClass {
     let Name = name;
     let ParserMethod = "parse"#name;
@@ -48,6 +51,7 @@ multiclass SystemZRegClass<string name, list<ValueType> types, int size,
         const SystemZSubtarget &S = MF.getSubtarget<SystemZSubtarget>();
         return S.isTargetXPLINK64();
       }];
+      let RegInfos = RI;
     }
   def "" : RegisterOperand<!cast<RegisterClass>(name#"Bit")> {
     let ParserMatchClass = !cast<AsmOperandClass>(name#"AsmOperand");
@@ -201,9 +205,16 @@ def F27Dwarf : DwarfMapping<81>;
 def F29Dwarf : DwarfMapping<82>;
 def F31Dwarf : DwarfMapping<83>;
 
+// Upper 16 bits of one of the floating-point registers
+class FPR16<bits<16> num, string n> : SystemZReg<n> {
+  let HWEncoding = num;
+}
+
 // Upper 32 bits of one of the floating-point registers
-class FPR32<bits<16> num, string n> : SystemZReg<n> {
+class FPR32<bits<16> num, string n, FPR16 high>
+  : SystemZRegWithSubregs<n, [high]> {
   let HWEncoding = num;
+  let SubRegIndices = [subreg_h16];
 }
 
 // One of the floating-point registers.
@@ -223,12 +234,14 @@ class FPR128<bits<16> num, string n, FPR64 low, FPR64 high>
 
 // Floating-point registers.  Registers 16-31 require the vector facility.
 foreach I = 0-15 in {
-  def F#I#S : FPR32<I, "f"#I>;
+  def F#I#H : FPR16<I, "f"#I>;
+  def F#I#S : FPR32<I, "f"#I, !cast<FPR16>("F"#I#"H")>;
   def F#I#D : FPR64<I, "f"#I, !cast<FPR32>("F"#I#"S")>,
               DwarfRegNum<[!cast<DwarfMapping>("F"#I#"Dwarf").Id]>;
 }
 foreach I = 16-31 in {
-  def F#I#S : FPR32<I, "v"#I>;
+  def F#I#H : FPR16<I, "v"#I>;
+  def F#I#S : FPR32<I, "v"#I, !cast<FPR16>("F"#I#"H")>;
   def F#I#D : FPR64<I, "v"#I, !cast<FPR32>("F"#I#"S")>,
               DwarfRegNum<[!cast<DwarfMapping>("F"#I#"Dwarf").Id]>;
 }
@@ -240,6 +253,11 @@ foreach I = [0, 1, 4, 5, 8, 9, 12, 13] in {
 
 // There's no store-multiple instruction for FPRs, so we're not fussy
 // about the order in which call-saved registers are allocated.
+// Adjust the spill size of f16 to 32 bits in case of no vector support.
+def FP16RI : RegInfoByHwMode<[DefaultMode,       NoVecHwMode],
+                             [RegInfo<16,16,16>, RegInfo<16,32,32>]>;
+defm FP16  : SystemZRegClass<"FP16", [f16], 16, (sequence "F%uH", 0, 15),
+                             [(sequence "F%uH", 0, 15)], 1, FP16RI>;
 defm FP32  : SystemZRegClass<"FP32", [f32], 32, (sequence "F%uS", 0, 15)>;
 defm FP64  : SystemZRegClass<"FP64", [f64], 64, (sequence "F%uD", 0, 15)>;
 defm FP128 : SystemZRegClass<"FP128", [f128], 128,
@@ -262,6 +280,13 @@ foreach I = 0-31 in {
             DwarfRegNum<[!cast<DwarfMapping>("F"#I#"Dwarf").Id]>;
 }
 
+// Class used to store 16-bit fp values in the first element of a vector
+// register.
+defm VR16 : SystemZRegClass<"VR16", [f16], 16,
+                            (add (sequence "F%uH", 0, 7),
+                                 (sequence "F%uH", 16, 31),
+                                 (sequence "F%uH", 8, 15))>;
+
 // Class used to store 32-bit values in the first element of a vector
 // register.  f32 scalars are used for the WLEDB and WLDEB instructions.
 defm VR32 : SystemZRegClass<"VR32", [f32, v4i8, v2i16], 32,
@@ -298,6 +323,7 @@ class TypedReg<ValueType vtin, RegisterOperand opin> {
   RegisterOperand op = opin;
 }
 
+def v16hb   : TypedReg<f16,     VR16>;
 def v32f    : TypedReg<i32,     VR32>;
 def v32sb   : TypedReg<f32,     VR32>;
 def v64g    : TypedReg<i64,     VR64>;
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td
index 5f15348654c75..84c6ca21d7e93 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ13.td
@@ -773,12 +773,12 @@ def : InstRW<[], (instregex "Insn.*")>;
 //===----------------------------------------------------------------------===//
 
 // Load zero
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER)$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER|ER_16)$")>;
 def : InstRW<[WLat2, FXb2, Cracked], (instregex "LZXR$")>;
 
 // Load
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R32|GR)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER(16)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R16|R32|GR)$")>;
 def : InstRW<[WLat3, FXb, NormalGr], (instregex "LGDR$")>;
 def : InstRW<[WLat2, FXb2, GroupAlone], (instregex "LXR$")>;
 
@@ -787,13 +787,13 @@ def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "LT(E|D)BR$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "LTXBR$")>;
 
 // Copy sign
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s)(d|s)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s|h)(d|s|h)$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Load instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "LE(Y)?$")>;
+def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "L(E16|E)(Y)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LD(Y|E32)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 
@@ -801,7 +801,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 // FP: Store instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E|D)(Y)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E16|E|D)(Y)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "STX$")>;
 
 //===----------------------------------------------------------------------===//
@@ -840,7 +840,7 @@ def : InstRW<[WLat12, WLat12, FXb, VecDF2, Cracked], (instregex "CL(F|G)XBR$")>;
 
 // Load Complement / Negative / Positive
 def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "L(C|N|P)(E|D)BR$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32|_16)?$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "L(C|N|P)XBR$")>;
 
 // Square root
@@ -1191,7 +1191,7 @@ def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "VLEI(B|F|G|H)$")>;
 
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(Align)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(L|BB)$")>;
-def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(32|64)$")>;
+def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(16|32|64)$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLLEZ(B|F|G|H)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLREP(B|F|G|H)?$")>;
 def : InstRW<[WLat2LSU, RegReadAdv, VecXsPm, LSU, NormalGr],
@@ -1205,7 +1205,7 @@ def : InstRW<[WLat4LSU, WLat4LSU, LSU5, GroupAlone],
 // Vector: Stores
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|32|64)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|16|32|64)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VSTE(F|G)$")>;
 def : InstRW<[WLat1, FXb, LSU, VecXsPm, Cracked], (instregex "VSTE(B|H)$")>;
 def : InstRW<[WLat1, LSU2, FXb3, GroupAlone2], (instregex "VSTM(Align)?$")>;
@@ -1376,8 +1376,8 @@ def : InstRW<[WLat3, VecXsPm, NormalGr], (instregex "WF(C|K)DB$")>;
 // Vector: Floating-point insertion and extraction
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR$")>;
-def : InstRW<[WLat4, FXb, NormalGr], (instregex "LFER$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR(_16)?$")>;
+def : InstRW<[WLat4, FXb, NormalGr], (instregex "LFER(_16)?$")>;
 
 //===----------------------------------------------------------------------===//
 // Vector: String instructions
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ14.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ14.td
index 336bbe2483340..b48ed08c8a189 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ14.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ14.td
@@ -793,12 +793,12 @@ def : InstRW<[], (instregex "Insn.*")>;
 //===----------------------------------------------------------------------===//
 
 // Load zero
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER)$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER|ER_16)$")>;
 def : InstRW<[WLat2, FXb2, Cracked], (instregex "LZXR$")>;
 
 // Load
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R32|GR)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER(16)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R16|R32|GR)$")>;
 def : InstRW<[WLat3, FXb, NormalGr], (instregex "LGDR$")>;
 def : InstRW<[WLat2, FXb2, GroupAlone], (instregex "LXR$")>;
 
@@ -807,13 +807,13 @@ def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "LT(E|D)BR$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "LTXBR$")>;
 
 // Copy sign
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s)(d|s)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s|h)(d|s|h)$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Load instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "LE(Y)?$")>;
+def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "L(E16|E)(Y)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LD(Y|E32)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 
@@ -821,7 +821,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 // FP: Store instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E|D)(Y)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E16|E|D)(Y)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "STX$")>;
 
 //===----------------------------------------------------------------------===//
@@ -860,7 +860,7 @@ def : InstRW<[WLat12, WLat12, FXb, VecDF2, Cracked], (instregex "CL(F|G)XBR$")>;
 
 // Load Complement / Negative / Positive
 def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "L(C|N|P)(E|D)BR$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32|_16)?$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "L(C|N|P)XBR$")>;
 
 // Square root
@@ -1209,7 +1209,7 @@ def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "VLEI(B|F|G|H)$")>;
 
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(Align)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(L|BB)$")>;
-def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(32|64)$")>;
+def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(16|32|64)$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLLEZ(B|F|G|H|LF)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLREP(B|F|G|H)?$")>;
 def : InstRW<[WLat2LSU, RegReadAdv, VecXsPm, LSU, NormalGr],
@@ -1224,7 +1224,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLRL(R)?$")>;
 // Vector: Stores
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|32|64)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|16|32|64)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VSTE(F|G)$")>;
 def : InstRW<[WLat1, FXb, LSU, VecXsPm, Cracked], (instregex "VSTE(B|H)$")>;
 def : InstRW<[WLat1, LSU2, FXb3, GroupAlone2], (instregex "VSTM(Align)?$")>;
@@ -1448,8 +1448,8 @@ def : InstRW<[WLat3, VecDFX, NormalGr], (instregex "WF(C|K)XB$")>;
 // Vector: Floating-point insertion and extraction
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR$")>;
-def : InstRW<[WLat3, FXb, NormalGr], (instregex "LFER$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR(_16)?$")>;
+def : InstRW<[WLat3, FXb, NormalGr], (instregex "LFER(_16)?$")>;
 
 //===----------------------------------------------------------------------===//
 // Vector: String instructions
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
index 5f2a04c298a25..e3ec7a6994221 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
@@ -811,12 +811,12 @@ def : InstRW<[], (instregex "Insn.*")>;
 //===----------------------------------------------------------------------===//
 
 // Load zero
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER)$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER|ER_16)$")>;
 def : InstRW<[WLat2, FXb2, Cracked], (instregex "LZXR$")>;
 
 // Load
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R32|GR)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER(16)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R16|R32|GR)$")>;
 def : InstRW<[WLat3, FXb, NormalGr], (instregex "LGDR$")>;
 def : InstRW<[WLat2, FXb2, GroupAlone], (instregex "LXR$")>;
 
@@ -825,13 +825,13 @@ def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "LT(E|D)BR$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "LTXBR$")>;
 
 // Copy sign
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s)(d|s)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s|h)(d|s|h)$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Load instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "LE(Y)?$")>;
+def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "L(E16|E)(Y)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LD(Y|E32)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 
@@ -839,7 +839,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 // FP: Store instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E|D)(Y)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E16|E|D)(Y)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "STX$")>;
 
 //===----------------------------------------------------------------------===//
@@ -878,7 +878,7 @@ def : InstRW<[WLat12, WLat12, FXb, VecDF2, Cracked], (instregex "CL(F|G)XBR$")>;
 
 // Load Complement / Negative / Positive
 def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "L(C|N|P)(E|D)BR$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32|_16)?$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "L(C|N|P)XBR$")>;
 
 // Square root
@@ -1231,7 +1231,7 @@ def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "VLEI(B|F|G|H)$")>;
 
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(Align)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(L|BB)$")>;
-def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(32|64)$")>;
+def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(16|32|64)$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLLEZ(B|F|G|H|LF)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLREP(B|F|G|H)?$")>;
 def : InstRW<[WLat2LSU, RegReadAdv, VecXsPm, LSU, NormalGr],
@@ -1246,7 +1246,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLRL(R)?$")>;
 // Vector: Stores
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|32|64)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|16|32|64)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VSTE(F|G)$")>;
 def : InstRW<[WLat1, FXb, LSU, VecXsPm, Cracked], (instregex "VSTE(B|H)$")>;
 def : InstRW<[WLat1, LSU2, FXb3, GroupAlone2], (instregex "VSTM(Align)?$")>;
@@ -1491,8 +1491,8 @@ def : InstRW<[WLat3, VecDFX, NormalGr], (instregex "WF(C|K)XB$")>;
 // Vector: Floating-point insertion and extraction
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR$")>;
-def : InstRW<[WLat3, FXb, NormalGr], (instregex "LFER$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR(_16)?$")>;
+def : InstRW<[WLat3, FXb, NormalGr], (instregex "LFER(_16)?$")>;
 
 //===----------------------------------------------------------------------===//
 // Vector: String instructions
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ16.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ16.td
index 83e980940d758..4f904daec5052 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ16.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ16.td
@@ -812,12 +812,12 @@ def : InstRW<[], (instregex "Insn.*")>;
 //===----------------------------------------------------------------------===//
 
 // Load zero
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER)$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER|ER_16)$")>;
 def : InstRW<[WLat2, FXb2, Cracked], (instregex "LZXR$")>;
 
 // Load
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R32|GR)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER(16)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R16|R32|GR)$")>;
 def : InstRW<[WLat3, FXb, NormalGr], (instregex "LGDR$")>;
 def : InstRW<[WLat2, FXb2, GroupAlone], (instregex "LXR$")>;
 
@@ -826,13 +826,13 @@ def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "LT(E|D)BR$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "LTXBR$")>;
 
 // Copy sign
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s)(d|s)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s|h)(d|s|h)$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Load instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "LE(Y)?$")>;
+def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "L(E16|E)(Y)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LD(Y|E32)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 
@@ -840,7 +840,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 // FP: Store instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E|D)(Y)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E16|E|D)(Y)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "STX$")>;
 
 //===----------------------------------------------------------------------===//
@@ -879,7 +879,7 @@ def : InstRW<[WLat12, WLat12, FXb, VecDF2, Cracked], (instregex "CL(F|G)XBR$")>;
 
 // Load Complement / Negative / Positive
 def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "L(C|N|P)(E|D)BR$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32|_16)?$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "L(C|N|P)XBR$")>;
 
 // Square root
@@ -1237,7 +1237,7 @@ def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "VLEI(B|F|G|H)$")>;
 
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(Align)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(L|BB)$")>;
-def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(32|64)$")>;
+def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(16|32|64)$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLLEZ(B|F|G|H|LF)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLREP(B|F|G|H)?$")>;
 def : InstRW<[WLat2LSU, RegReadAdv, VecXsPm, LSU, NormalGr],
@@ -1252,7 +1252,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLRL(R)?$")>;
 // Vector: Stores
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|32|64)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|16|32|64)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VSTE(F|G)$")>;
 def : InstRW<[WLat1, FXb, LSU, VecXsPm, Cracked], (instregex "VSTE(B|H)$")>;
 def : InstRW<[WLat1, LSU2, FXb3, GroupAlone2], (instregex "VSTM(Align)?$")>;
@@ -1499,8 +1499,8 @@ def : InstRW<[WLat3, VecDFX, NormalGr], (instregex "WF(C|K)XB$")>;
 // Vector: Floating-point insertion and extraction
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR$")>;
-def : InstRW<[WLat3, FXb, NormalGr], (instregex "LFER$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR(_16)?$")>;
+def : InstRW<[WLat3, FXb, NormalGr], (instregex "LFER(_16)?$")>;
 
 //===----------------------------------------------------------------------===//
 // Vector: String instructions
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ17.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ17.td
index bd52627f636a7..3b5ce6c9b5a0e 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ17.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ17.td
@@ -827,12 +827,12 @@ def : InstRW<[], (instregex "Insn.*")>;
 //===----------------------------------------------------------------------===//
 
 // Load zero
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER)$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LZ(DR|ER|ER_16)$")>;
 def : InstRW<[WLat2, FXb2, Cracked], (instregex "LZXR$")>;
 
 // Load
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R32|GR)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "LER(16)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LD(R|R16|R32|GR)$")>;
 def : InstRW<[WLat3, FXb, NormalGr], (instregex "LGDR$")>;
 def : InstRW<[WLat2, FXb2, GroupAlone], (instregex "LXR$")>;
 
@@ -841,13 +841,13 @@ def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "LT(E|D)BR$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "LTXBR$")>;
 
 // Copy sign
-def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s)(d|s)$")>;
+def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "CPSDR(d|s|h)(d|s|h)$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Load instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "LE(Y)?$")>;
+def : InstRW<[WLat2LSU, VecXsPm, LSU, NormalGr], (instregex "L(E16|E)(Y)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LD(Y|E32)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 
@@ -855,7 +855,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 // FP: Store instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E|D)(Y)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "ST(E16|E|D)(Y)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "STX$")>;
 
 //===----------------------------------------------------------------------===//
@@ -894,7 +894,7 @@ def : InstRW<[WLat12, WLat12, FXb, VecDF2, Cracked], (instregex "CL(F|G)XBR$")>;
 
 // Load Complement / Negative / Positive
 def : InstRW<[WLat3, WLat3, VecXsPm, NormalGr], (instregex "L(C|N|P)(E|D)BR$")>;
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32)?$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "L(C|N|P)DFR(_32|_16)?$")>;
 def : InstRW<[WLat10, WLat10, VecDF4, GroupAlone], (instregex "L(C|N|P)XBR$")>;
 
 // Square root
@@ -1252,7 +1252,7 @@ def : InstRW<[WLat2, VecXsPm, NormalGr], (instregex "VLEI(B|F|G|H)$")>;
 
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(Align)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(L|BB)$")>;
-def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(32|64)$")>;
+def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VL(16|32|64)$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLLEZ(B|F|G|H|LF)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLREP(B|F|G|H)?$")>;
 def : InstRW<[WLat2LSU, RegReadAdv, VecXsPm, LSU, NormalGr],
@@ -1267,7 +1267,7 @@ def : InstRW<[LSULatency, LSU, NormalGr], (instregex "VLRL(R)?$")>;
 // Vector: Stores
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|32|64)?$")>;
+def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VST(Align|L|16|32|64)?$")>;
 def : InstRW<[WLat1, FXb, LSU, NormalGr], (instregex "VSTE(F|G)$")>;
 def : InstRW<[WLat1, FXb, LSU, VecXsPm, Cracked], (instregex "VSTE(B|H)$")>;
 def : InstRW<[WLat1, LSU2, FXb3, GroupAlone2], (instregex "VSTM(Align)?$")>;
@@ -1520,8 +1520,8 @@ def : InstRW<[WLat3, VecDFX, NormalGr], (instregex "WF(C|K)XB$")>;
 // Vector: Floating-point insertion and extraction
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR$")>;
-def : InstRW<[WLat3, FXb, NormalGr], (instregex "LFER$")>;
+def : InstRW<[WLat1, FXb, NormalGr], (instregex "LEFR(_16)?$")>;
+def : InstRW<[WLat3, FXb, NormalGr], (instregex "LFER(_16)?$")>;
 
 //===----------------------------------------------------------------------===//
 // Vector: String instructions
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td
index f8397921bf684..26433b97484da 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ196.td
@@ -705,12 +705,12 @@ def : InstRW<[], (instregex "Insn.*")>;
 //===----------------------------------------------------------------------===//
 
 // Load zero
-def : InstRW<[WLat1, FXU, NormalGr], (instregex "LZ(DR|ER)$")>;
+def : InstRW<[WLat1, FXU, NormalGr], (instregex "LZ(DR|ER|ER_16)$")>;
 def : InstRW<[WLat2, FXU2, GroupAlone2], (instregex "LZXR$")>;
 
 // Load
-def : InstRW<[WLat1, FXU, NormalGr], (instregex "LER$")>;
-def : InstRW<[WLat1, FXU, NormalGr], (instregex "LD(R|R32|GR)$")>;
+def : InstRW<[WLat1, FXU, NormalGr], (instregex "LER(16)?$")>;
+def : InstRW<[WLat1, FXU, NormalGr], (instregex "LD(R|R16|R32|GR)$")>;
 def : InstRW<[WLat3, FXU, NormalGr], (instregex "LGDR$")>;
 def : InstRW<[WLat2, FXU2, GroupAlone2], (instregex "LXR$")>;
 
@@ -719,20 +719,20 @@ def : InstRW<[WLat9, WLat9, FPU, NormalGr], (instregex "LT(E|D)BR$")>;
 def : InstRW<[WLat10, WLat10, FPU4, GroupAlone], (instregex "LTXBR$")>;
 
 // Copy sign
-def : InstRW<[WLat5, FXU2, GroupAlone], (instregex "CPSDR(d|s)(d|s)$")>;
+def : InstRW<[WLat5, FXU2, GroupAlone], (instregex "CPSDR(d|s|h)(d|s|h)$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Load instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[LSULatency, LSU, NormalGr], (instregex "L(E|D)(Y|E32)?$")>;
+def : InstRW<[LSULatency, LSU, NormalGr], (instregex "L(E16|E|D)(Y|E32)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Store instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXU, LSU, NormalGr], (instregex "ST(E|D)(Y)?$")>;
+def : InstRW<[WLat1, FXU, LSU, NormalGr], (instregex "ST(E16|E|D)(Y)?$")>;
 def : InstRW<[WLat1, FXU, LSU, NormalGr], (instregex "STX$")>;
 
 //===----------------------------------------------------------------------===//
@@ -771,7 +771,7 @@ def : InstRW<[WLat12, WLat12, FXU, FPU2, GroupAlone], (instregex "CL(F|G)XBR$")>
 
 // Load Complement / Negative / Positive
 def : InstRW<[WLat7, WLat7, FPU, NormalGr], (instregex "L(C|N|P)(E|D)BR$")>;
-def : InstRW<[WLat1, FXU, NormalGr], (instregex "L(C|N|P)DFR(_32)?$")>;
+def : InstRW<[WLat1, FXU, NormalGr], (instregex "L(C|N|P)DFR(_32|_16)?$")>;
 def : InstRW<[WLat10, WLat10, FPU4, GroupAlone], (instregex "L(C|N|P)XBR$")>;
 
 // Square root
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td b/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td
index 039c8146618fe..193a793f17367 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZEC12.td
@@ -743,12 +743,12 @@ def : InstRW<[], (instregex "Insn.*")>;
 //===----------------------------------------------------------------------===//
 
 // Load zero
-def : InstRW<[WLat1, FXU, NormalGr], (instregex "LZ(DR|ER)$")>;
+def : InstRW<[WLat1, FXU, NormalGr], (instregex "LZ(DR|ER|ER_16)$")>;
 def : InstRW<[WLat2, FXU2, GroupAlone], (instregex "LZXR$")>;
 
 // Load
-def : InstRW<[WLat1, FXU, NormalGr], (instregex "LER$")>;
-def : InstRW<[WLat1, FXU, NormalGr], (instregex "LD(R|R32|GR)$")>;
+def : InstRW<[WLat1, FXU, NormalGr], (instregex "LER(16)?$")>;
+def : InstRW<[WLat1, FXU, NormalGr], (instregex "LD(R|R16|R32|GR)$")>;
 def : InstRW<[WLat3, FXU, NormalGr], (instregex "LGDR$")>;
 def : InstRW<[WLat2, FXU2, GroupAlone], (instregex "LXR$")>;
 
@@ -757,20 +757,20 @@ def : InstRW<[WLat9, WLat9, FPU, NormalGr], (instregex "LT(E|D)BR$")>;
 def : InstRW<[WLat10, WLat10, FPU4, GroupAlone], (instregex "LTXBR$")>;
 
 // Copy sign
-def : InstRW<[WLat5, FXU2, GroupAlone], (instregex "CPSDR(d|s)(d|s)$")>;
+def : InstRW<[WLat5, FXU2, GroupAlone], (instregex "CPSDR(d|s|h)(d|s|h)$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Load instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[LSULatency, LSU, NormalGr], (instregex "L(E|D)(Y|E32)?$")>;
+def : InstRW<[LSULatency, LSU, NormalGr], (instregex "L(E16|E|D)(Y|E32)?$")>;
 def : InstRW<[LSULatency, LSU, NormalGr], (instregex "LX$")>;
 
 //===----------------------------------------------------------------------===//
 // FP: Store instructions
 //===----------------------------------------------------------------------===//
 
-def : InstRW<[WLat1, FXU, LSU, NormalGr], (instregex "ST(E|D)(Y)?$")>;
+def : InstRW<[WLat1, FXU, LSU, NormalGr], (instregex "ST(E16|E|D)(Y)?$")>;
 def : InstRW<[WLat1, FXU, LSU, NormalGr], (instregex "STX$")>;
 
 //===----------------------------------------------------------------------===//
@@ -809,7 +809,7 @@ def : InstRW<[WLat12, WLat12, FXU, FPU2, GroupAlone], (instregex "CL(F|G)XBR$")>
 
 // Load Complement / Negative / Positive
 def : InstRW<[WLat7, WLat7, FPU, NormalGr], (instregex "L(C|N|P)(E|D)BR$")>;
-def : InstRW<[WLat1, FXU, NormalGr], (instregex "L(C|N|P)DFR(_32)?$")>;
+def : InstRW<[WLat1, FXU, NormalGr], (instregex "L(C|N|P)DFR(_32|_16)?$")>;
 def : InstRW<[WLat10, WLat10, FPU4, GroupAlone], (instregex "L(C|N|P)XBR$")>;
 
 // Square root
diff --git a/llvm/test/CodeGen/SystemZ/asm-10.ll b/llvm/test/CodeGen/SystemZ/asm-10.ll
index b71db8350781d..8226b8a1a2d25 100644
--- a/llvm/test/CodeGen/SystemZ/asm-10.ll
+++ b/llvm/test/CodeGen/SystemZ/asm-10.ll
@@ -2,6 +2,15 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -no-integrated-as | FileCheck %s
 
+define half @f0() {
+; CHECK-LABEL: f0:
+; CHECK: lzer %f1
+; CHECK: blah %f0 %f1
+; CHECK: br %r14
+  %val = call half asm "blah $0 $1", "=&f,f" (half 0.0)
+  ret half %val
+}
+
 define float @f1() {
 ; CHECK-LABEL: f1:
 ; CHECK: lzer %f1
diff --git a/llvm/test/CodeGen/SystemZ/asm-17.ll b/llvm/test/CodeGen/SystemZ/asm-17.ll
index c9c4d73c66ebb..dad75d4d012d1 100644
--- a/llvm/test/CodeGen/SystemZ/asm-17.ll
+++ b/llvm/test/CodeGen/SystemZ/asm-17.ll
@@ -25,6 +25,17 @@ define i64 @f2() {
   ret i64 %ret
 }
 
+; Test 16-bit FPRs.
+define half @f3_half() {
+; CHECK-LABEL: f3_half:
+; CHECK: lzer %f4
+; CHECK: blah %f4
+; CHECK: ler %f0, %f4
+; CHECK: br %r14
+  %ret = call half asm "blah $0", "={f4},0" (half 0.0)
+  ret half %ret
+}
+
 ; Test i32 FPRs.
 define float @f3() {
 ; CHECK-LABEL: f3:
diff --git a/llvm/test/CodeGen/SystemZ/asm-19.ll b/llvm/test/CodeGen/SystemZ/asm-19.ll
index e16fdfa13fce6..6c77fb55071ca 100644
--- a/llvm/test/CodeGen/SystemZ/asm-19.ll
+++ b/llvm/test/CodeGen/SystemZ/asm-19.ll
@@ -3,6 +3,15 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -no-integrated-as | FileCheck %s
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -no-integrated-as | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-Z14
 
+define half @f0() {
+; CHECK-LABEL: f0:
+; CHECK: lzer %f1
+; CHECK: blah %f0 %f1
+; CHECK: br %r14
+  %val = call half asm "blah $0 $1", "=&v,v" (half 0.0)
+  ret half %val
+}
+
 define float @f1() {
 ; CHECK-LABEL: f1:
 ; CHECK: lzer %f1
@@ -86,6 +95,16 @@ define <4 x float> @f9() {
   ret <4 x float> %val
 }
 
+define half @f10_half() {
+; CHECK-LABEL: f10_half:
+; CHECK: lzer %f4
+; CHECK: blah %f4
+; CHECK: ldr %f0, %f4
+; CHECK: br %r14
+  %ret = call half asm "blah $0", "={v4},0" (half 0.0)
+  ret half %ret
+}
+
 define float @f10() {
 ; CHECK-LABEL: f10:
 ; CHECK: lzer %f4
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-10.ll b/llvm/test/CodeGen/SystemZ/atomic-load-10.ll
new file mode 100644
index 0000000000000..4135a55bb6fbc
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-10.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test fp16 atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -verify-machineinstrs | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -mtriple=s390x-linux-gnu -verify-machineinstrs -mcpu=z16 | FileCheck %s -check-prefix=VECTOR
+
+define half @f1(ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lgh %r0, 0(%r2)
+; CHECK-NEXT:    sllg %r0, %r0, 48
+; CHECK-NEXT:    ldgr %f0, %r0
+; CHECK-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; CHECK-NEXT:    br %r14
+;
+; VECTOR-LABEL: f1:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    vlreph %v0, 0(%r2)
+; VECTOR-NEXT:    br %r14
+  %val = load atomic half, ptr %src seq_cst, align 2
+  ret half %val
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-10.ll b/llvm/test/CodeGen/SystemZ/atomic-store-10.ll
new file mode 100644
index 0000000000000..3f228d58dcd8c
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-10.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test half atomic stores.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -verify-machineinstrs | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -mtriple=s390x-linux-gnu -verify-machineinstrs -mcpu=z16 | FileCheck %s -check-prefix=VECTOR
+
+define void @f1(ptr %src, half %val) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; CHECK-NEXT:    lgdr %r0, %f0
+; CHECK-NEXT:    srlg %r0, %r0, 48
+; CHECK-NEXT:    sth %r0, 0(%r2)
+; CHECK-NEXT:    bcr 15, %r0
+; CHECK-NEXT:    br %r14
+;
+; VECTOR-LABEL: f1:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    vsteh %v0, 0(%r2), 0
+; VECTOR-NEXT:    bcr 14, %r0
+; VECTOR-NEXT:    br %r14
+  store atomic half %val, ptr %src seq_cst, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-04.ll
new file mode 100644
index 0000000000000..a0869e13a013d
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-04.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test atomic half addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define half @f1(ptr %src, half %b) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r11, %r15, 88(%r15)
+; CHECK-NEXT:    .cfi_offset %r11, -72
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -184
+; CHECK-NEXT:    .cfi_def_cfa_offset 344
+; CHECK-NEXT:    std %f8, 176(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f9, 168(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f10, 160(%r15) # 8-byte Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    .cfi_offset %f10, -184
+; CHECK-NEXT:    lgr %r13, %r2
+; CHECK-NEXT:    lgh %r0, 0(%r2)
+; CHECK-NEXT:    ler %f8, %f0
+; CHECK-NEXT:    sllg %r0, %r0, 48
+; CHECK-NEXT:    ldgr %f9, %r0
+; CHECK-NEXT:    risbg %r12, %r2, 0, 189, 0
+; CHECK-NEXT:    sll %r13, 3
+; CHECK-NEXT:    lcr %r11, %r13
+; CHECK-NEXT:    j .LBB0_2
+; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
+; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    sllg %r0, %r3, 48
+; CHECK-NEXT:    ldgr %f9, %r0
+; CHECK-NEXT:    je .LBB0_5
+; CHECK-NEXT:  .LBB0_2: # %atomicrmw.start
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_3 Depth 2
+; CHECK-NEXT:    ler %f0, %f8
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f10, %f0
+; CHECK-NEXT:    ler %f0, %f9
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    aebr %f0, %f10
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; CHECK-NEXT:    lgdr %r1, %f0
+; CHECK-NEXT:    l %r0, 0(%r12)
+; CHECK-NEXT:    srlg %r1, %r1, 48
+; CHECK-NEXT:    lgdr %r2, %f9
+; CHECK-NEXT:    srlg %r2, %r2, 48
+; CHECK-NEXT:  .LBB0_3: # %atomicrmw.start
+; CHECK-NEXT:    # Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    rll %r3, %r0, 16(%r13)
+; CHECK-NEXT:    risbg %r1, %r3, 32, 47, 0
+; CHECK-NEXT:    llhr %r3, %r3
+; CHECK-NEXT:    cr %r3, %r2
+; CHECK-NEXT:    jlh .LBB0_1
+; CHECK-NEXT:  # %bb.4: # %atomicrmw.start
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=2
+; CHECK-NEXT:    rll %r4, %r1, -16(%r11)
+; CHECK-NEXT:    cs %r0, %r4, 0(%r12)
+; CHECK-NEXT:    jl .LBB0_3
+; CHECK-NEXT:    j .LBB0_1
+; CHECK-NEXT:  .LBB0_5: # %atomicrmw.end
+; CHECK-NEXT:    ler %f0, %f9
+; CHECK-NEXT:    ld %f8, 176(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f9, 168(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f10, 160(%r15) # 8-byte Reload
+; CHECK-NEXT:    lmg %r11, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %res = atomicrmw fadd ptr %src, half %b seq_cst
+  ret half %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/fmuladd-soft-float.ll b/llvm/test/CodeGen/SystemZ/fmuladd-soft-float.ll
index 1447c576f48ae..a982f9af52358 100644
--- a/llvm/test/CodeGen/SystemZ/fmuladd-soft-float.ll
+++ b/llvm/test/CodeGen/SystemZ/fmuladd-soft-float.ll
@@ -1,6 +1,43 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=s390x-linux < %s | FileCheck %s -check-prefix=SOFT-FLOAT
 
+define half @fmuladd_intrinsic_f16(half %a, half %b, half %c) #0 {
+; SOFT-FLOAT-LABEL: fmuladd_intrinsic_f16:
+; SOFT-FLOAT:       # %bb.0:
+; SOFT-FLOAT-NEXT:    stmg %r12, %r15, 96(%r15)
+; SOFT-FLOAT-NEXT:    .cfi_offset %r12, -64
+; SOFT-FLOAT-NEXT:    .cfi_offset %r13, -56
+; SOFT-FLOAT-NEXT:    .cfi_offset %r14, -48
+; SOFT-FLOAT-NEXT:    .cfi_offset %r15, -40
+; SOFT-FLOAT-NEXT:    aghi %r15, -160
+; SOFT-FLOAT-NEXT:    .cfi_def_cfa_offset 320
+; SOFT-FLOAT-NEXT:    # kill: def $r4l killed $r4l def $r4d
+; SOFT-FLOAT-NEXT:    llghr %r0, %r4
+; SOFT-FLOAT-NEXT:    lr %r13, %r3
+; SOFT-FLOAT-NEXT:    lr %r12, %r2
+; SOFT-FLOAT-NEXT:    lgr %r2, %r0
+; SOFT-FLOAT-NEXT:    brasl %r14, __extendhfsf2@PLT
+; SOFT-FLOAT-NEXT:    llghr %r0, %r12
+; SOFT-FLOAT-NEXT:    lgr %r12, %r2
+; SOFT-FLOAT-NEXT:    lgr %r2, %r0
+; SOFT-FLOAT-NEXT:    brasl %r14, __extendhfsf2@PLT
+; SOFT-FLOAT-NEXT:    llghr %r0, %r13
+; SOFT-FLOAT-NEXT:    lgr %r13, %r2
+; SOFT-FLOAT-NEXT:    lgr %r2, %r0
+; SOFT-FLOAT-NEXT:    brasl %r14, __extendhfsf2@PLT
+; SOFT-FLOAT-NEXT:    lgr %r3, %r2
+; SOFT-FLOAT-NEXT:    lgr %r2, %r13
+; SOFT-FLOAT-NEXT:    brasl %r14, __mulsf3@PLT
+; SOFT-FLOAT-NEXT:    lgr %r3, %r12
+; SOFT-FLOAT-NEXT:    brasl %r14, __addsf3@PLT
+; SOFT-FLOAT-NEXT:    brasl %r14, __truncsfhf2@PLT
+; SOFT-FLOAT-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; SOFT-FLOAT-NEXT:    lmg %r12, %r15, 256(%r15)
+; SOFT-FLOAT-NEXT:    br %r14
+  %result = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
+  ret half %result
+}
+
 define float @fmuladd_intrinsic_f32(float %a, float %b, float %c) #0 {
 ; SOFT-FLOAT-LABEL: fmuladd_intrinsic_f32:
 ; SOFT-FLOAT:       # %bb.0:
diff --git a/llvm/test/CodeGen/SystemZ/fp-abs-01.ll b/llvm/test/CodeGen/SystemZ/fp-abs-01.ll
index bf0870a86702c..0cfdefe3bd61b 100644
--- a/llvm/test/CodeGen/SystemZ/fp-abs-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-abs-01.ll
@@ -3,6 +3,18 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
+; Test f16.
+declare half @llvm.fabs.f16(half %f)
+define half @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK:      brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: lpdfr %f0, %f0
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.fabs.f16(half %f)
+  ret half %res
+}
+
 ; Test f32.
 declare float @llvm.fabs.f32(float %f)
 define float @f1(float %f) {
diff --git a/llvm/test/CodeGen/SystemZ/fp-abs-03.ll b/llvm/test/CodeGen/SystemZ/fp-abs-03.ll
index 72786ea203df4..29f2d06e75ff9 100644
--- a/llvm/test/CodeGen/SystemZ/fp-abs-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-abs-03.ll
@@ -2,6 +2,18 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
+; Test f16.
+declare half @llvm.fabs.f16(half %f)
+define half @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK:      brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: lpdfr %f0, %f0
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.fabs.f16(half %f)
+  ret half %res
+}
+
 ; Test f32.
 declare float @llvm.fabs.f32(float %f)
 define float @f1(float %f) {
diff --git a/llvm/test/CodeGen/SystemZ/fp-abs-04.ll b/llvm/test/CodeGen/SystemZ/fp-abs-04.ll
index b02abc8443491..afaf3f6d22ac8 100644
--- a/llvm/test/CodeGen/SystemZ/fp-abs-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-abs-04.ll
@@ -2,6 +2,22 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
+; Test f16.
+declare half @llvm.fabs.f16(half %f)
+define half @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK:      brasl   %r14, __extendhfsf2@PLT
+; CHECK-NEXT: lpdfr   %f0, %f0
+; CHECK-NEXT: brasl   %r14, __truncsfhf2@PLT
+; CHECK-NEXT: brasl   %r14, __extendhfsf2@PLT
+; CHECK-NEXT: lcdfr   %f0, %f0
+; CHECK-NEXT: brasl   %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %abs = call half @llvm.fabs.f16(half %f)
+  %res = fneg half %abs
+  ret half %res
+}
+
 ; Test f32.
 declare float @llvm.fabs.f32(float %f)
 define float @f1(float %f) {
diff --git a/llvm/test/CodeGen/SystemZ/fp-add-01.ll b/llvm/test/CodeGen/SystemZ/fp-add-01.ll
index eb845bae9b804..b65744c4aac0c 100644
--- a/llvm/test/CodeGen/SystemZ/fp-add-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-add-01.ll
@@ -6,6 +6,18 @@
 
 declare float @foo()
 
+; Check register addition.
+define half @f0(half %f1, half %f2) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: aebr %f0, %f9
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = fadd half %f1, %f2
+  ret half %res
+}
+
 ; Check register addition.
 define float @f1(float %f1, float %f2) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-cmp-04.ll b/llvm/test/CodeGen/SystemZ/fp-cmp-04.ll
index c1773abe92305..d3d641357ae58 100644
--- a/llvm/test/CodeGen/SystemZ/fp-cmp-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-cmp-04.ll
@@ -227,6 +227,38 @@ exit:
   ret float %add
 }
 
+define half @f12_half(half %dummy, half %val, ptr %dest) {
+; CHECK-LABEL: f12_half:
+; CHECK:      ler %f8, %f2
+; CHECK-NEXT: ler %f0, %f2
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah %f0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: ltebr %f0, %f0
+; CHECK-NEXT: jl .LBB11_2
+; CHECK-NEXT:# %bb.1:
+; CHECK-NEXT: lgdr %r0, %f8
+; CHECK-NEXT: srlg %r0, %r0, 48
+; CHECK-NEXT: sth  %r0, 0(%r13)
+; CHECK-NEXT:.LBB11_2:
+; CHECK-NEXT: ler %f0, %f8
+; CHECK-NEXT: ld %f8, 160(%r15)
+; CHECK-NEXT: lmg %r13, %r15, 272(%r15)
+; CHECK-NEXT: br %r14
+entry:
+  call void asm sideeffect "blah $0", "{f0}"(half %val)
+  %cmp = fcmp olt half %val, 0.0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store half %val, ptr %dest
+  br label %exit
+
+exit:
+  ret half %val
+}
+
 ; %val in %f2 must be preserved during comparison and also copied to %f0.
 define float @f12(float %dummy, float %val, ptr %dest) {
 ; CHECK-LABEL: f12:
@@ -304,6 +336,38 @@ exit:
   ret void
 }
 
+define half @f15_half(half %val, half %dummy, ptr %dest) {
+; CHECK-LABEL: f15_half:
+; CHECK:      ler %f8, %f0
+; CHECK-NEXT: ler %f2, %f0
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah %f2
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: ltebr %f0, %f0
+; CHECK-NEXT: jl .LBB15_2
+; CHECK-NEXT:# %bb.1:
+; CHECK-NEXT: lgdr %r0, %f8
+; CHECK-NEXT: srlg %r0, %r0, 48
+; CHECK-NEXT: sth %r0, 0(%r13)
+; CHECK-NEXT:.LBB15_2:
+; CHECK-NEXT: ler %f0, %f8
+; CHECK-NEXT: ld %f8, 160(%r15)
+; CHECK-NEXT: lmg %r13, %r15, 272(%r15)
+; CHECK-NEXT: br %r14
+entry:
+  call void asm sideeffect "blah $0", "{f2}"(half %val)
+  %cmp = fcmp olt half %val, 0.0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store half %val, ptr %dest
+  br label %exit
+
+exit:
+  ret half %val
+}
+
 define float @f15(float %val, float %dummy, ptr %dest) {
 ; CHECK-LABEL: f15:
 ; CHECK: ltebr %f1, %f0
@@ -374,7 +438,7 @@ define float @f18(float %dummy, float %a, ptr %dest) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lnebr %f0, %f2
 ; CHECK-NEXT:    blr %r14
-; CHECK-NEXT:  .LBB17_1: # %store
+; CHECK-NEXT:  .LBB19_1: # %store
 ; CHECK-NEXT:    ste %f0, 0(%r2)
 ; CHECK-NEXT:    br %r14
 entry:
@@ -397,7 +461,7 @@ define float @f19(float %dummy, float %a, ptr %dest) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lcebr %f0, %f2
 ; CHECK-NEXT:    bler %r14
-; CHECK-NEXT:  .LBB18_1: # %store
+; CHECK-NEXT:  .LBB20_1: # %store
 ; CHECK-NEXT:    ste %f0, 0(%r2)
 ; CHECK-NEXT:    br %r14
 entry:
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-05.ll b/llvm/test/CodeGen/SystemZ/fp-conv-05.ll
index 4596649d5659c..fef2b842c54aa 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-05.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-05.ll
@@ -2,6 +2,16 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+; Check i32->f16.
+define half @f0(i32 %i) {
+; CHECK-LABEL: f0:
+; CHECK: cefbr %f0, %r2
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = sitofp i32 %i to half
+  ret half %conv
+}
+
 ; Check i32->f32.
 define float @f1(i32 %i) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-06.ll b/llvm/test/CodeGen/SystemZ/fp-conv-06.ll
index e754a7e161f8f..deb22ee4d19b4 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-06.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-06.ll
@@ -2,6 +2,18 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
 
+; Check i32->f16.  There is no native instruction, so we must promote
+; to i64 first.
+define half @f0(i32 %i) {
+; CHECK-LABEL: f0:
+; CHECK: llgfr [[REGISTER:%r[0-5]]], %r2
+; CHECK: cegbr %f0, [[REGISTER]]
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = uitofp i32 %i to half
+  ret half %conv
+}
+
 ; Check i32->f32.  There is no native instruction, so we must promote
 ; to i64 first.
 define float @f1(i32 %i) {
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-07.ll b/llvm/test/CodeGen/SystemZ/fp-conv-07.ll
index 2941e77441461..02f47e481cc6a 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-07.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-07.ll
@@ -2,6 +2,16 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+; Test i64->f16.
+define half @f0(i64 %i) {
+; CHECK-LABEL: f0:
+; CHECK: cegbr %f0, %r2
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = sitofp i64 %i to half
+  ret half %conv
+}
+
 ; Test i64->f32.
 define float @f1(i64 %i) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-08.ll b/llvm/test/CodeGen/SystemZ/fp-conv-08.ll
index e2a5f74185216..f2590b6566a62 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-08.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-08.ll
@@ -2,6 +2,17 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
 
+; Test i64->f16. For z10, this results in just a single a libcall.
+define half @f0(i64 %i) {
+; CHECK-LABEL: f0:
+; CHECK: cegbr
+; CHECK: aebr
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = uitofp i64 %i to half
+  ret half %conv
+}
+
 ; Test i64->f32.  There's no native support for unsigned i64-to-fp conversions,
 ; but we should be able to implement them using signed i64-to-fp conversions.
 define float @f1(i64 %i) {
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-09.ll b/llvm/test/CodeGen/SystemZ/fp-conv-09.ll
index 0e730c3705030..c29a202807c69 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-09.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-09.ll
@@ -2,6 +2,16 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+; Test f16->i32.
+define i32 @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: cfebr %r2, 5, %f0
+; CHECK: br %r14
+  %conv = fptosi half %f to i32
+  ret i32 %conv
+}
+
 ; Test f32->i32.
 define i32 @f1(float %f) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-10.ll b/llvm/test/CodeGen/SystemZ/fp-conv-10.ll
index 82913265853a5..a0455a2cea2d6 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-10.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-10.ll
@@ -8,18 +8,36 @@
 ; Promoting to i64 doesn't generate an inexact condition for values that are
 ; outside the i32 range but in the i64 range, so use the default expansion.
 
+; Test f16->i32. Converted to signed as the max float value is smaller than
+; the signed integer range.
+define i32 @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    cfebr %r2, 5, %f0
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %conv = fptoui half %f to i32
+  ret i32 %conv
+}
+
 ; Test f32->i32.
 define i32 @f1(float %f) {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    larl %r1, .LCPI0_0
+; CHECK-NEXT:    larl %r1, .LCPI1_0
 ; CHECK-NEXT:    le %f1, 0(%r1)
 ; CHECK-NEXT:    cebr %f0, %f1
-; CHECK-NEXT:    jnl .LBB0_2
+; CHECK-NEXT:    jnl .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    cfebr %r2, 5, %f0
 ; CHECK-NEXT:    br %r14
-; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    sebr %f0, %f1
 ; CHECK-NEXT:    cfebr %r2, 5, %f0
 ; CHECK-NEXT:    xilf %r2, 2147483648
@@ -32,14 +50,14 @@ define i32 @f1(float %f) {
 define i32 @f2(double %f) {
 ; CHECK-LABEL: f2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    larl %r1, .LCPI1_0
+; CHECK-NEXT:    larl %r1, .LCPI2_0
 ; CHECK-NEXT:    ld %f1, 0(%r1)
 ; CHECK-NEXT:    cdbr %f0, %f1
-; CHECK-NEXT:    jnl .LBB1_2
+; CHECK-NEXT:    jnl .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    cfdbr %r2, 5, %f0
 ; CHECK-NEXT:    br %r14
-; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    sdbr %f0, %f1
 ; CHECK-NEXT:    cfdbr %r2, 5, %f0
 ; CHECK-NEXT:    xilf %r2, 2147483648
@@ -54,14 +72,14 @@ define i32 @f3(ptr %src) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ld %f0, 0(%r2)
 ; CHECK-NEXT:    ld %f2, 8(%r2)
-; CHECK-NEXT:    larl %r1, .LCPI2_0
+; CHECK-NEXT:    larl %r1, .LCPI3_0
 ; CHECK-NEXT:    lxeb %f1, 0(%r1)
 ; CHECK-NEXT:    cxbr %f0, %f1
-; CHECK-NEXT:    jnl .LBB2_2
+; CHECK-NEXT:    jnl .LBB3_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    cfxbr %r2, 5, %f0
 ; CHECK-NEXT:    br %r14
-; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    sxbr %f0, %f1
 ; CHECK-NEXT:    cfxbr %r2, 5, %f0
 ; CHECK-NEXT:    xilf %r2, 2147483648
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-11.ll b/llvm/test/CodeGen/SystemZ/fp-conv-11.ll
index 2dd543b5810bf..55a2f8a51a526 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-11.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-11.ll
@@ -2,6 +2,16 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+; Test f16->i64.
+define i64 @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: cgebr %r2, 5, %f0
+; CHECK: br %r14
+  %conv = fptosi half %f to i64
+  ret i64 %conv
+}
+
 ; Test f32->i64.
 define i64 @f1(float %f) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-12.ll b/llvm/test/CodeGen/SystemZ/fp-conv-12.ll
index 27afbf4d398a0..bb83a677210ac 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-12.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-12.ll
@@ -7,18 +7,36 @@
 ; they were added in z196 as the Convert to Logical family of instructions.
 ; Convert via signed i64s instead.
 
+; Test f16->i64. Converted to signed as the max float value is smaller than
+; the signed integer range.
+define i64 @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    cgebr %r2, 5, %f0
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %conv = fptoui half %f to i64
+  ret i64 %conv
+}
+
 ; Test f32->i64.
 define i64 @f1(float %f) {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    larl %r1, .LCPI0_0
+; CHECK-NEXT:    larl %r1, .LCPI1_0
 ; CHECK-NEXT:    le %f1, 0(%r1)
 ; CHECK-NEXT:    cebr %f0, %f1
-; CHECK-NEXT:    jnl .LBB0_2
+; CHECK-NEXT:    jnl .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    cgebr %r2, 5, %f0
 ; CHECK-NEXT:    br %r14
-; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    sebr %f0, %f1
 ; CHECK-NEXT:    cgebr %r2, 5, %f0
 ; CHECK-NEXT:    xihf %r2, 2147483648
@@ -31,14 +49,14 @@ define i64 @f1(float %f) {
 define i64 @f2(double %f) {
 ; CHECK-LABEL: f2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    larl %r1, .LCPI1_0
+; CHECK-NEXT:    larl %r1, .LCPI2_0
 ; CHECK-NEXT:    ld %f1, 0(%r1)
 ; CHECK-NEXT:    cdbr %f0, %f1
-; CHECK-NEXT:    jnl .LBB1_2
+; CHECK-NEXT:    jnl .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    cgdbr %r2, 5, %f0
 ; CHECK-NEXT:    br %r14
-; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    sdbr %f0, %f1
 ; CHECK-NEXT:    cgdbr %r2, 5, %f0
 ; CHECK-NEXT:    xihf %r2, 2147483648
@@ -53,14 +71,14 @@ define i64 @f3(ptr %src) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ld %f0, 0(%r2)
 ; CHECK-NEXT:    ld %f2, 8(%r2)
-; CHECK-NEXT:    larl %r1, .LCPI2_0
+; CHECK-NEXT:    larl %r1, .LCPI3_0
 ; CHECK-NEXT:    lxeb %f1, 0(%r1)
 ; CHECK-NEXT:    cxbr %f0, %f1
-; CHECK-NEXT:    jnl .LBB2_2
+; CHECK-NEXT:    jnl .LBB3_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    cgxbr %r2, 5, %f0
 ; CHECK-NEXT:    br %r14
-; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    sxbr %f0, %f1
 ; CHECK-NEXT:    cgxbr %r2, 5, %f0
 ; CHECK-NEXT:    xihf %r2, 2147483648
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-13.ll b/llvm/test/CodeGen/SystemZ/fp-conv-13.ll
index 6e6c96bea2b35..4869d070b6beb 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-13.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-13.ll
@@ -3,6 +3,16 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
 
+; Check i32->f16.
+define half @f0(i32 %i) {
+; CHECK-LABEL: f0:
+; CHECK: celfbr %f0, 0, %r2, 0
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = uitofp i32 %i to half
+  ret half %conv
+}
+
 ; Check i32->f32.
 define float @f1(i32 %i) {
 ; CHECK-LABEL: f1:
@@ -33,18 +43,28 @@ define void @f3(i32 %i, ptr %dst) {
   ret void
 }
 
-; Check i64->f32.
-define float @f4(i64 %i) {
+; Check i64->f16.
+define half @f4(i64 %i) {
 ; CHECK-LABEL: f4:
 ; CHECK: celgbr %f0, 0, %r2, 0
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = uitofp i64 %i to half
+  ret half %conv
+}
+
+; Check i64->f32.
+define float @f5(i64 %i) {
+; CHECK-LABEL: f5:
+; CHECK: celgbr %f0, 0, %r2, 0
 ; CHECK: br %r14
   %conv = uitofp i64 %i to float
   ret float %conv
 }
 
 ; Check i64->f64.
-define double @f5(i64 %i) {
-; CHECK-LABEL: f5:
+define double @f6(i64 %i) {
+; CHECK-LABEL: f6:
 ; CHECK: cdlgbr %f0, 0, %r2, 0
 ; CHECK: br %r14
   %conv = uitofp i64 %i to double
@@ -52,8 +72,8 @@ define double @f5(i64 %i) {
 }
 
 ; Check i64->f128.
-define void @f6(i64 %i, ptr %dst) {
-; CHECK-LABEL: f6:
+define void @f7(i64 %i, ptr %dst) {
+; CHECK-LABEL: f7:
 ; CHECK: cxlgbr %f0, 0, %r2, 0
 ; CHECK-DAG: std %f0, 0(%r3)
 ; CHECK-DAG: std %f2, 8(%r3)
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-14.ll b/llvm/test/CodeGen/SystemZ/fp-conv-14.ll
index 0d1f951994d27..c9448eac91fb1 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-14.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-14.ll
@@ -2,6 +2,16 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
 
+; Test f16->i32.
+define i32 @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: clfebr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %conv = fptoui half %f to i32
+  ret i32 %conv
+}
+
 ; Test f32->i32.
 define i32 @f1(float %f) {
 ; CHECK-LABEL: f1:
@@ -32,9 +42,19 @@ define i32 @f3(ptr %src) {
   ret i32 %conv
 }
 
-; Test f32->i64.
-define i64 @f4(float %f) {
+; Test f16->i64.
+define i64 @f4(half %f) {
 ; CHECK-LABEL: f4:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: clgebr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %conv = fptoui half %f to i64
+  ret i64 %conv
+}
+
+; Test f32->i64.
+define i64 @f5(float %f) {
+; CHECK-LABEL: f5:
 ; CHECK: clgebr %r2, 5, %f0, 0
 ; CHECK: br %r14
   %conv = fptoui float %f to i64
@@ -42,8 +62,8 @@ define i64 @f4(float %f) {
 }
 
 ; Test f64->i64.
-define i64 @f5(double %f) {
-; CHECK-LABEL: f5:
+define i64 @f6(double %f) {
+; CHECK-LABEL: f6:
 ; CHECK: clgdbr %r2, 5, %f0, 0
 ; CHECK: br %r14
   %conv = fptoui double %f to i64
@@ -51,8 +71,8 @@ define i64 @f5(double %f) {
 }
 
 ; Test f128->i64.
-define i64 @f6(ptr %src) {
-; CHECK-LABEL: f6:
+define i64 @f7(ptr %src) {
+; CHECK-LABEL: f7:
 ; CHECK-DAG: ld %f0, 0(%r2)
 ; CHECK-DAG: ld %f2, 8(%r2)
 ; CHECK: clgxbr %r2, 5, %f0, 0
diff --git a/llvm/test/CodeGen/SystemZ/fp-conv-20.ll b/llvm/test/CodeGen/SystemZ/fp-conv-20.ll
index 8006a8beb0789..58db2e10da8b2 100644
--- a/llvm/test/CodeGen/SystemZ/fp-conv-20.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-conv-20.ll
@@ -30,9 +30,19 @@ define float @f3(i128 %i) {
   ret float %conv
 }
 
-; Test unsigned i128->f128.
-define fp128 @f4(i128 %i) {
+; Test signed i128->f16.
+define half @f4(i128 %i) {
 ; CHECK-LABEL: f4:
+; CHECK: brasl   %r14, __floattisf@PLT
+; CHECK: brasl   %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = sitofp i128 %i to half
+  ret half %conv
+}
+
+; Test unsigned i128->f128.
+define fp128 @f5(i128 %i) {
+; CHECK-LABEL: f5:
 ; CHECK: brasl %r14, __floatuntitf@PLT
 ; CHECK: br %r14
   %conv = uitofp i128 %i to fp128
@@ -40,8 +50,8 @@ define fp128 @f4(i128 %i) {
 }
 
 ; Test unsigned i128->f64.
-define double @f5(i128 %i) {
-; CHECK-LABEL: f5:
+define double @f6(i128 %i) {
+; CHECK-LABEL: f6:
 ; CHECK: brasl %r14, __floatuntidf@PLT
 ; CHECK: br %r14
   %conv = uitofp i128 %i to double
@@ -49,17 +59,27 @@ define double @f5(i128 %i) {
 }
 
 ; Test unsigned i128->f32.
-define float @f6(i128 %i) {
-; CHECK-LABEL: f6:
+define float @f7(i128 %i) {
+; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, __floatuntisf@PLT
 ; CHECK: br %r14
   %conv = uitofp i128 %i to float
   ret float %conv
 }
 
+; Test unsigned i128->f16.
+define half @f8(i128 %i) {
+; CHECK-LABEL: f8:
+; CHECK: brasl   %r14, __floatuntisf@PLT
+; CHECK: brasl   %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = uitofp i128 %i to half
+  ret half %conv
+}
+
 ; Test signed f128->i128.
-define i128 @f7(fp128 %f) {
-; CHECK-LABEL: f7:
+define i128 @f9(fp128 %f) {
+; CHECK-LABEL: f9:
 ; CHECK: brasl %r14, __fixtfti@PLT
 ; CHECK: br %r14
   %conv = fptosi fp128 %f to i128
@@ -67,26 +87,36 @@ define i128 @f7(fp128 %f) {
 }
 
 ; Test signed f64->i128.
-define i128 @f8(double %f) {
-; CHECK-LABEL: f8:
+define i128 @f10(double %f) {
+; CHECK-LABEL: f10:
 ; CHECK: brasl %r14, __fixdfti@PLT
 ; CHECK: br %r14
   %conv = fptosi double %f to i128
   ret i128 %conv
 }
 
-; Test signed f9->i128.
-define i128 @f9(float %f) {
-; CHECK-LABEL: f9:
+; Test signed f32->i128.
+define i128 @f11(float %f) {
+; CHECK-LABEL: f11:
 ; CHECK: brasl %r14, __fixsfti@PLT
 ; CHECK: br %r14
   %conv = fptosi float %f to i128
   ret i128 %conv
 }
 
+; Test signed f16->i128.
+define i128 @f12(half %f) {
+; CHECK-LABEL: f12:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __fixsfti@PLT
+; CHECK: br %r14
+  %conv = fptosi half %f to i128
+  ret i128 %conv
+}
+
 ; Test unsigned f128->i128.
-define i128 @f10(fp128 %f) {
-; CHECK-LABEL: f10:
+define i128 @f13(fp128 %f) {
+; CHECK-LABEL: f13:
 ; CHECK: brasl %r14, __fixunstfti@PLT
 ; CHECK: br %r14
   %conv = fptoui fp128 %f to i128
@@ -94,8 +124,8 @@ define i128 @f10(fp128 %f) {
 }
 
 ; Test unsigned f64->i128.
-define i128 @f11(double %f) {
-; CHECK-LABEL: f11:
+define i128 @f14(double %f) {
+; CHECK-LABEL: f14:
 ; CHECK: brasl %r14, __fixunsdfti@PLT
 ; CHECK: br %r14
   %conv = fptoui double %f to i128
@@ -103,10 +133,20 @@ define i128 @f11(double %f) {
 }
 
 ; Test unsigned f32->i128.
-define i128 @f12(float %f) {
-; CHECK-LABEL: f12:
+define i128 @f15(float %f) {
+; CHECK-LABEL: f15:
 ; CHECK: brasl %r14, __fixunssfti@PLT
 ; CHECK: br %r14
   %conv = fptoui float %f to i128
   ret i128 %conv
 }
+
+; Test unsigned f16->i128.
+define i128 @f16(half %f) {
+; CHECK-LABEL: f16:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __fixunssfti@PLT
+; CHECK: br %r14
+  %conv = fptoui half %f to i128
+  ret i128 %conv
+}
diff --git a/llvm/test/CodeGen/SystemZ/fp-copysign-03.ll b/llvm/test/CodeGen/SystemZ/fp-copysign-03.ll
new file mode 100644
index 0000000000000..909519e8ace55
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-copysign-03.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s --check-prefixes=CHECK,Z10
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,Z16
+;
+; Test copysign intrinsics with half.
+
+declare half @llvm.copysign.f16(half, half)
+declare float @llvm.copysign.f32(float, float)
+declare double @llvm.copysign.f64(double, double)
+declare fp128 @llvm.copysign.f128(fp128, fp128)
+
+; Test copysign with an f16 result and f16 sign argument.
+define half @f0(half %a, half %b) {
+; CHECK-LABEL: f0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cpsdr %f0, %f2, %f0
+; CHECK-NEXT:    br %r14
+  %res = call half @llvm.copysign.f16(half %a, half %b)
+  ret half %res
+}
+
+; Test copysign with an f16 result and f32 sign argument.
+define half @f1(half %a, float %b) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cpsdr %f0, %f2, %f0
+; CHECK-NEXT:    br %r14
+  %bh = fptrunc float %b to half
+  %res = call half @llvm.copysign.f16(half %a, half %bh)
+  ret half %res
+}
+
+; Test copysign with an f16 result and f64 sign argument.
+define half @f2(half %a, double %b) {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cpsdr %f0, %f2, %f0
+; CHECK-NEXT:    br %r14
+  %bh = fptrunc double %b to half
+  %res = call half @llvm.copysign.f16(half %a, half %bh)
+  ret half %res
+}
+
+; Test copysign with an f16 result and f128 sign argument.
+; TODO: Let the DAGCombiner remove the fp_round.
+define half @f3(half %a, fp128 %b) {
+; Z10-LABEL: f3:
+; Z10:       # %bb.0:
+; Z10-NEXT:    stmg %r14, %r15, 112(%r15)
+; Z10-NEXT:    .cfi_offset %r14, -48
+; Z10-NEXT:    .cfi_offset %r15, -40
+; Z10-NEXT:    aghi %r15, -184
+; Z10-NEXT:    .cfi_def_cfa_offset 344
+; Z10-NEXT:    std %f8, 176(%r15) # 8-byte Spill
+; Z10-NEXT:    .cfi_offset %f8, -168
+; Z10-NEXT:    ld %f1, 0(%r2)
+; Z10-NEXT:    ld %f3, 8(%r2)
+; Z10-NEXT:    ler %f8, %f0
+; Z10-NEXT:    la %r2, 160(%r15)
+; Z10-NEXT:    std %f1, 160(%r15)
+; Z10-NEXT:    std %f3, 168(%r15)
+; Z10-NEXT:    brasl %r14, __trunctfhf2@PLT
+; Z10-NEXT:    cpsdr %f0, %f0, %f8
+; Z10-NEXT:    ld %f8, 176(%r15) # 8-byte Reload
+; Z10-NEXT:    lmg %r14, %r15, 296(%r15)
+; Z10-NEXT:    br %r14
+;
+; Z16-LABEL: f3:
+; Z16:       # %bb.0:
+; Z16-NEXT:    stmg %r14, %r15, 112(%r15)
+; Z16-NEXT:    .cfi_offset %r14, -48
+; Z16-NEXT:    .cfi_offset %r15, -40
+; Z16-NEXT:    aghi %r15, -184
+; Z16-NEXT:    .cfi_def_cfa_offset 344
+; Z16-NEXT:    std %f8, 176(%r15) # 8-byte Spill
+; Z16-NEXT:    .cfi_offset %f8, -168
+; Z16-NEXT:    ldr %f8, %f0
+; Z16-NEXT:    vl %v0, 0(%r2), 3
+; Z16-NEXT:    la %r2, 160(%r15)
+; Z16-NEXT:    vst %v0, 160(%r15), 3
+; Z16-NEXT:    brasl %r14, __trunctfhf2@PLT
+; Z16-NEXT:    cpsdr %f0, %f0, %f8
+; Z16-NEXT:    ld %f8, 176(%r15) # 8-byte Reload
+; Z16-NEXT:    lmg %r14, %r15, 296(%r15)
+; Z16-NEXT:    br %r14
+  %bh = fptrunc fp128 %b to half
+  %res = call half @llvm.copysign.f16(half %a, half %bh)
+  ret half %res
+}
+
+; Test copysign with an f32 result and half sign argument.
+define float @f4(float %a, half %b) {
+; CHECK-LABEL: f4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cpsdr %f0, %f2, %f0
+; CHECK-NEXT:    br %r14
+  %bf = fpext half %b to float
+  %res = call float @llvm.copysign.f32(float %a, float %bf)
+  ret float %res
+}
+
+; Test copysign with an f64 result and half sign argument.
+define double @f5(double %a, half %b) {
+; CHECK-LABEL: f5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cpsdr %f0, %f2, %f0
+; CHECK-NEXT:    br %r14
+  %bd = fpext half %b to double
+  %res = call double @llvm.copysign.f64(double %a, double %bd)
+  ret double %res
+}
+
+; Test copysign with an f128 result and half sign argument.
+define fp128 @f6(fp128 %a, half %b) {
+; Z10-LABEL: f6:
+; Z10:       # %bb.0:
+; Z10-NEXT:    ld %f1, 0(%r3)
+; Z10-NEXT:    ld %f3, 8(%r3)
+; Z10-NEXT:    cpsdr %f1, %f0, %f1
+; Z10-NEXT:    std %f1, 0(%r2)
+; Z10-NEXT:    std %f3, 8(%r2)
+; Z10-NEXT:    br %r14
+;
+; Z16-LABEL: f6:
+; Z16:       # %bb.0:
+; Z16-NEXT:    aghi %r15, -168
+; Z16-NEXT:    .cfi_def_cfa_offset 328
+; Z16-NEXT:    vl %v1, 0(%r3), 3
+; Z16-NEXT:    vsteh %v0, 164(%r15), 0
+; Z16-NEXT:    tm 164(%r15), 128
+; Z16-NEXT:    je .LBB6_2
+; Z16-NEXT:  # %bb.1:
+; Z16-NEXT:    wflnxb %v0, %v1
+; Z16-NEXT:    j .LBB6_3
+; Z16-NEXT:  .LBB6_2:
+; Z16-NEXT:    wflpxb %v0, %v1
+; Z16-NEXT:  .LBB6_3:
+; Z16-NEXT:    vst %v0, 0(%r2), 3
+; Z16-NEXT:    aghi %r15, 168
+; Z16-NEXT:    br %r14
+  %bd = fpext half %b to fp128
+  %res = call fp128 @llvm.copysign.f128(fp128 %a, fp128 %bd)
+  ret fp128 %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/fp-div-01.ll b/llvm/test/CodeGen/SystemZ/fp-div-01.ll
index d33e61bbd1eda..78df879613cb2 100644
--- a/llvm/test/CodeGen/SystemZ/fp-div-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-div-01.ll
@@ -6,6 +6,18 @@
 
 declare float @foo()
 
+; Check register division.
+define half @f0(half %f1, half %f2) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: debr %f0, %f9
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = fdiv half %f1, %f2
+  ret half %res
+}
+
 ; Check register division.
 define float @f1(float %f1, float %f2) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-half-cmp.ll b/llvm/test/CodeGen/SystemZ/fp-half-cmp.ll
new file mode 100644
index 0000000000000..2714d6ad9a92c
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-half-cmp.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 -verify-machineinstrs \
+; RUN:   | FileCheck %s
+;
+; Some tests with comparisons and their uses involving 16-bit floating point.
+
+; fcmp half; select half
+define half @fun0(half %Arg0, half %Arg1) {
+; CHECK-LABEL: fun0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    ldr %f8, %f2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ldr %f9, %f0
+; CHECK-NEXT:    ldr %f0, %f8
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ltebr %f1, %f9
+; CHECK-NEXT:    # kill: def $f0s killed $f0s def $v0
+; CHECK-NEXT:    jl .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    vgmf %v0, 2, 8
+; CHECK-NEXT:  .LBB0_2: # %entry
+; CHECK-NEXT:    # kill: def $f0s killed $f0s killed $v0
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; CHECK-NEXT:    lmg %r14, %r15, 288(%r15)
+; CHECK-NEXT:    br %r14
+entry:
+  %cmp = fcmp olt half %Arg0, 0xH0000
+  %cond = select i1 %cmp, half %Arg1, half 1.0
+  ret half %cond
+}
+
+; fcmp half; select i32
+define i32 @fun1(half %Arg0, i32 %Arg1) {
+; CHECK-LABEL: fun1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    lr %r13, %r2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ltebr %f0, %f0
+; CHECK-NEXT:    lochinl %r13, 0
+; CHECK-NEXT:    lr %r2, %r13
+; CHECK-NEXT:    lmg %r13, %r15, 264(%r15)
+; CHECK-NEXT:    br %r14
+entry:
+  %cmp = fcmp olt half %Arg0, 0xH0000
+  %cond = select i1 %cmp, i32 %Arg1, i32 0
+  ret i32 %cond
+}
+
+; icmp i32; select half
+define half @fun2(i32 %Arg0, half %Arg1) {
+; CHECK-LABEL: fun2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    lr %r13, %r2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    # kill: def $f0s killed $f0s def $v0
+; CHECK-NEXT:    cije %r13, 0, .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    vgmf %v0, 2, 8
+; CHECK-NEXT:  .LBB2_2: # %entry
+; CHECK-NEXT:    # kill: def $f0s killed $f0s killed $v0
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r13, %r15, 264(%r15)
+; CHECK-NEXT:    br %r14
+entry:
+  %cmp = icmp eq i32 %Arg0, 0
+  %cond = select i1 %cmp, half %Arg1, half 1.0
+  ret half %cond
+}
+
+define i64 @fun3(i64 %a, i64 %b, half %f1, half %f2) #0 {
+; CHECK-LABEL: fun3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r12, %r15, 96(%r15)
+; CHECK-NEXT:    .cfi_offset %r12, -64
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    ldr %f8, %f0
+; CHECK-NEXT:    ldr %f0, %f2
+; CHECK-NEXT:    lgr %r13, %r3
+; CHECK-NEXT:    lgr %r12, %r2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ldr %f9, %f0
+; CHECK-NEXT:    ldr %f0, %f8
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    cebr %f0, %f9
+; CHECK-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; CHECK-NEXT:    selgre %r2, %r12, %r13
+; CHECK-NEXT:    lmg %r12, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f32(
+                                               half %f1, half %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
+define half @fun4(half %Arg0, ptr %Dst) {
+; CHECK-LABEL: fun4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    lgr %r13, %r2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ltebr %f0, %f0
+; CHECK-NEXT:    je .LBB4_2
+; CHECK-NEXT:  # %bb.1: # %store
+; CHECK-NEXT:    lzer %f0
+; CHECK-NEXT:    vsteh %v0, 0(%r13), 0
+; CHECK-NEXT:  .LBB4_2: # %exit
+; CHECK-NEXT:    lzer %f0
+; CHECK-NEXT:    lmg %r13, %r15, 264(%r15)
+; CHECK-NEXT:    br %r14
+entry:
+  %cmp = fcmp oeq half %Arg0, 0.0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  store half 0.0, ptr %Dst
+  br label %exit
+
+exit:
+  ret half 0.0
+}
+
+declare i1 @llvm.experimental.constrained.fcmp.f32(half, half, metadata, metadata)
diff --git a/llvm/test/CodeGen/SystemZ/fp-half-libcall.ll b/llvm/test/CodeGen/SystemZ/fp-half-libcall.ll
new file mode 100644
index 0000000000000..d8db549388c46
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-half-libcall.ll
@@ -0,0 +1,312 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test that library calls are emitted for LLVM IR intrinsics
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define half @f1(half %x, i16 %y) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    lhr %r13, %r2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    llgfr %r2, %r13
+; CHECK-NEXT:    brasl %r14, __powisf2@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r13, %r15, 264(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.powi.f16.i16(half %x, i16 %y)
+  ret half %tmp
+}
+
+define half @f2(half %x, half %y) {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    ler %f8, %f2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f9, %f0
+; CHECK-NEXT:    ler %f0, %f8
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f2, %f0
+; CHECK-NEXT:    ler %f0, %f9
+; CHECK-NEXT:    brasl %r14, powf@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; CHECK-NEXT:    lmg %r14, %r15, 288(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.pow.f16(half %x, half %y)
+  ret half %tmp
+}
+
+define half @f3(half %x) {
+; CHECK-LABEL: f3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    brasl %r14, sinf@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.sin.f16(half %x)
+  ret half %tmp
+}
+
+define half @f4(half %x) {
+; CHECK-LABEL: f4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    brasl %r14, cosf@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.cos.f16(half %x)
+  ret half %tmp
+}
+
+define half @f5(half %x) {
+; CHECK-LABEL: f5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    brasl %r14, expf@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.exp.f16(half %x)
+  ret half %tmp
+}
+
+define half @f6(half %x) {
+; CHECK-LABEL: f6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    brasl %r14, exp2f@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.exp2.f16(half %x)
+  ret half %tmp
+}
+
+define half @f7(half %x) {
+; CHECK-LABEL: f7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    brasl %r14, logf@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.log.f16(half %x)
+  ret half %tmp
+}
+
+define half @f8(half %x) {
+; CHECK-LABEL: f8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    brasl %r14, log2f@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.log2.f16(half %x)
+  ret half %tmp
+}
+
+define half @f9(half %x) {
+; CHECK-LABEL: f9:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    brasl %r14, log10f@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.log10.f16(half %x)
+  ret half %tmp
+}
+
+define half @f10(half %x, half %y) {
+; CHECK-LABEL: f10:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    ler %f8, %f2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f9, %f0
+; CHECK-NEXT:    ler %f0, %f8
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f2, %f0
+; CHECK-NEXT:    ler %f0, %f9
+; CHECK-NEXT:    brasl %r14, fminf@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; CHECK-NEXT:    lmg %r14, %r15, 288(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.minnum.f16(half %x, half %y)
+  ret half %tmp
+}
+
+define half @f11(half %x, half %y) {
+; CHECK-LABEL: f11:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    ler %f8, %f2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f9, %f0
+; CHECK-NEXT:    ler %f0, %f8
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f2, %f0
+; CHECK-NEXT:    ler %f0, %f9
+; CHECK-NEXT:    brasl %r14, fmaxf@PLT
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; CHECK-NEXT:    lmg %r14, %r15, 288(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call half @llvm.maxnum.f16(half %x, half %y)
+  ret half %tmp
+}
+
+; Verify that "nnan" minnum/maxnum calls are transformed to
+; compare+select sequences instead of libcalls.
+define half @f12(half %x, half %y) {
+; CHECK-LABEL: f12:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    ler %f9, %f0
+; CHECK-NEXT:    ler %f0, %f2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f8, %f0
+; CHECK-NEXT:    ler %f0, %f9
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    cebr %f0, %f8
+; CHECK-NEXT:    jl .LBB11_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    ler %f0, %f8
+; CHECK-NEXT:  .LBB11_2:
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; CHECK-NEXT:    lmg %r14, %r15, 288(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call nnan half @llvm.minnum.f16(half %x, half %y)
+  ret half %tmp
+}
+
+define half @f13(half %x, half %y) {
+; CHECK-LABEL: f13:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; CHECK-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; CHECK-NEXT:    .cfi_offset %f8, -168
+; CHECK-NEXT:    .cfi_offset %f9, -176
+; CHECK-NEXT:    ler %f9, %f0
+; CHECK-NEXT:    ler %f0, %f2
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    ler %f8, %f0
+; CHECK-NEXT:    ler %f0, %f9
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    cebr %f0, %f8
+; CHECK-NEXT:    jh .LBB12_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    ler %f0, %f8
+; CHECK-NEXT:  .LBB12_2:
+; CHECK-NEXT:    brasl %r14, __truncsfhf2@PLT
+; CHECK-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; CHECK-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; CHECK-NEXT:    lmg %r14, %r15, 288(%r15)
+; CHECK-NEXT:    br %r14
+  %tmp = call nnan half @llvm.maxnum.f16(half %x, half %y)
+  ret half %tmp
+}
+
+declare half @llvm.powi.f16.i16(half, i16)
+declare half @llvm.pow.f16(half, half)
+
+declare half @llvm.sin.f16(half)
+declare half @llvm.cos.f16(half)
+
+declare half @llvm.exp.f16(half)
+declare half @llvm.exp2.f16(half)
+
+declare half @llvm.log.f16(half)
+declare half @llvm.log2.f16(half)
+declare half @llvm.log10.f16(half)
+
+declare half @llvm.minnum.f16(half, half)
+declare half @llvm.maxnum.f16(half, half)
diff --git a/llvm/test/CodeGen/SystemZ/fp-half-mem.ll b/llvm/test/CodeGen/SystemZ/fp-half-mem.ll
new file mode 100644
index 0000000000000..5988a379b3d9a
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-half-mem.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 -verify-machineinstrs \
+; RUN:   | FileCheck %s --check-prefix=NOVEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 -verify-machineinstrs \
+; RUN:   | FileCheck %s  --check-prefix=VECTOR
+
+declare void @foo(ptr)
+
+; Test an alloca.
+define half @f1() {
+; NOVEC-LABEL: f1:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -168
+; NOVEC-NEXT:    .cfi_def_cfa_offset 328
+; NOVEC-NEXT:    la %r2, 166(%r15)
+; NOVEC-NEXT:    brasl %r14, foo@PLT
+; NOVEC-NEXT:    lgh %r0, 166(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    lmg %r14, %r15, 280(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: f1:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -168
+; VECTOR-NEXT:    .cfi_def_cfa_offset 328
+; VECTOR-NEXT:    la %r2, 166(%r15)
+; VECTOR-NEXT:    brasl %r14, foo@PLT
+; VECTOR-NEXT:    vlreph %v0, 166(%r15)
+; VECTOR-NEXT:    lmg %r14, %r15, 280(%r15)
+; VECTOR-NEXT:    br %r14
+  %ptr = alloca half
+  call void @foo(ptr %ptr)
+  %orig = load half, ptr %ptr
+  ret half %orig
+}
+
+; Test accessing a half element of an aggregate type.
+%s.half = type { half, half, half, half, half }
+define half @f2(ptr %P) {
+; NOVEC-LABEL: f2:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    lgh %r0, 6(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: f2:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    vlreph %v0, 6(%r2)
+; VECTOR-NEXT:    br %r14
+  %gep = getelementptr inbounds %s.half, ptr %P, i64 0, i32 3
+  %res = load half, ptr %gep
+  ret half %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/fp-half-move.ll b/llvm/test/CodeGen/SystemZ/fp-half-move.ll
new file mode 100644
index 0000000000000..1e761d4c70b22
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-half-move.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 -verify-machineinstrs \
+; RUN:   | FileCheck %s --check-prefix=NOVEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 -verify-machineinstrs \
+; RUN:   | FileCheck %s  --check-prefix=VECTOR
+;
+; Test moves (bitcasts) between i16 and half.
+
+define half @f1(ptr %ptr) {
+; NOVEC-LABEL: f1:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    lh %r0, 0(%r2)
+; NOVEC-NEXT:    oill %r0, 255
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: f1:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    lh %r0, 0(%r2)
+; VECTOR-NEXT:    oill %r0, 255
+; VECTOR-NEXT:    vlvgh %v0, %r0, 0
+; VECTOR-NEXT:    br %r14
+  %L = load i16, ptr %ptr
+  %O = or i16 %L, 255
+  %res = bitcast i16 %O to half
+  ret half %res
+}
+
+define half @f2(i16 %Arg) {
+; NOVEC-LABEL: f2:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    # kill: def $r2l killed $r2l def $r2d
+; NOVEC-NEXT:    sllg %r0, %r2, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: f2:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    vlvgh %v0, %r2, 0
+; VECTOR-NEXT:    br %r14
+  %res = bitcast i16 %Arg to half
+  ret half %res
+}
+
+define void @f3(half %val, ptr %ptr) {
+; NOVEC-LABEL: f3:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    stc %r0, 0(%r2)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: f3:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    vlgvh %r0, %v0, 0
+; VECTOR-NEXT:    stc %r0, 0(%r2)
+; VECTOR-NEXT:    br %r14
+  %res = bitcast half %val to i16
+  %trunc = trunc i16 %res to i8
+  store i8 %trunc, ptr %ptr
+  ret void
+}
+
+define i16 @f4(half %Arg) {
+; NOVEC-LABEL: f4:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r2, %r0, 48
+; NOVEC-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: f4:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    vlgvh %r2, %v0, 0
+; VECTOR-NEXT:    br %r14
+  %res = bitcast half %Arg to i16
+  ret i16 %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/fp-half-strict.ll b/llvm/test/CodeGen/SystemZ/fp-half-strict.ll
new file mode 100644
index 0000000000000..4f58eb4c6cb20
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-half-strict.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 -verify-machineinstrs \
+; RUN:   | FileCheck %s --check-prefix=NOVEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 -verify-machineinstrs \
+; RUN:   | FileCheck %s  --check-prefix=VECTOR
+;
+; Tests for strict 16-bit floating point (half).
+
+declare half @llvm.experimental.constrained.fadd.f16(half, half, metadata, metadata)
+declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
+declare half @llvm.experimental.constrained.fmul.f16(half, half, metadata, metadata)
+
+; Test register addition.
+define half @fun0(half %f1, half %f2) #0 {
+; NOVEC-LABEL: fun0:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -176
+; NOVEC-NEXT:    .cfi_def_cfa_offset 336
+; NOVEC-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    ler %f8, %f0
+; NOVEC-NEXT:    ler %f0, %f2
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 288(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun0:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -176
+; VECTOR-NEXT:    .cfi_def_cfa_offset 336
+; VECTOR-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    .cfi_offset %f9, -176
+; VECTOR-NEXT:    ldr %f8, %f0
+; VECTOR-NEXT:    ldr %f0, %f2
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f9
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 288(%r15)
+; VECTOR-NEXT:    br %r14
+  %res = call half @llvm.experimental.constrained.fadd.f16(
+                       half %f1, half %f2,
+                       metadata !"round.dynamic",
+                       metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
+; Test atomic memory accesses and extension/truncation inside a strictfp
+; function.
+define void @fun1(ptr %Src, ptr %Dst) #0 {
+; NOVEC-LABEL: fun1:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r13, %r15, 104(%r15)
+; NOVEC-NEXT:    .cfi_offset %r13, -56
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -160
+; NOVEC-NEXT:    .cfi_def_cfa_offset 320
+; NOVEC-NEXT:    lgh %r0, 0(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    lgr %r13, %r3
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    brasl %r14, __extendhfdf2@PLT
+; NOVEC-NEXT:    adbr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncdfhf2@PLT
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r13)
+; NOVEC-NEXT:    bcr 14, %r0
+; NOVEC-NEXT:    lmg %r13, %r15, 264(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun1:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r13, %r15, 104(%r15)
+; VECTOR-NEXT:    .cfi_offset %r13, -56
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -160
+; VECTOR-NEXT:    .cfi_def_cfa_offset 320
+; VECTOR-NEXT:    vlreph %v0, 0(%r2)
+; VECTOR-NEXT:    lgr %r13, %r3
+; VECTOR-NEXT:    brasl %r14, __extendhfdf2@PLT
+; VECTOR-NEXT:    adbr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncdfhf2@PLT
+; VECTOR-NEXT:    vsteh %v0, 0(%r13), 0
+; VECTOR-NEXT:    bcr 14, %r0
+; VECTOR-NEXT:    lmg %r13, %r15, 264(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %Op0 = load atomic half, ptr %Src seq_cst, align 2
+  %E0 = fpext half %Op0 to double
+  %Add = call double @llvm.experimental.constrained.fadd.f64(
+                         double %E0, double %E0,
+                         metadata !"round.dynamic",
+                         metadata !"fpexcept.strict") #0
+  %Res = fptrunc double %Add to half
+  store atomic half %Res, ptr %Dst seq_cst, align 2
+  ret void
+}
+
+; Test a chain of half operations which should have each operation surrounded
+; by conversions to/from fp32 for proper emulation.
+define half @fun2(half %Op0, half %Op1, half %Op2)  #0 {
+; NOVEC-LABEL: fun2:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -184
+; NOVEC-NEXT:    .cfi_def_cfa_offset 344
+; NOVEC-NEXT:    std %f8, 176(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f10, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    .cfi_offset %f10, -184
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f2
+; NOVEC-NEXT:    ler %f8, %f4
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    ler %f10, %f0
+; NOVEC-NEXT:    ler %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    meebr %f0, %f10
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    meebr %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ld %f8, 176(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f10, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 296(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun2:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -184
+; VECTOR-NEXT:    .cfi_def_cfa_offset 344
+; VECTOR-NEXT:    std %f8, 176(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f9, 168(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f10, 160(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    .cfi_offset %f9, -176
+; VECTOR-NEXT:    .cfi_offset %f10, -184
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f2
+; VECTOR-NEXT:    ldr %f8, %f4
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    ldr %f10, %f0
+; VECTOR-NEXT:    ldr %f0, %f9
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    meebr %f0, %f10
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    wfmsb %f0, %f9, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ld %f8, 176(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f9, 168(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f10, 160(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 296(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %A0 = call half @llvm.experimental.constrained.fmul.f16(
+                       half %Op0, half %Op1,
+                       metadata !"round.dynamic",
+                       metadata !"fpexcept.strict") #0
+  %Res = call half @llvm.experimental.constrained.fmul.f16(
+                        half %A0, half %Op2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  ret half %Res
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-half-vector.ll b/llvm/test/CodeGen/SystemZ/fp-half-vector.ll
new file mode 100644
index 0000000000000..4997c5b0c617d
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-half-vector.ll
@@ -0,0 +1,725 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 -verify-machineinstrs \
+; RUN:   | FileCheck %s --check-prefix=NOVEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 -verify-machineinstrs \
+; RUN:   | FileCheck %s  --check-prefix=VECTOR
+
+; Add the <8 x half> argument with itself and return it.
+define <8 x half> @fun0(<8 x half> %Op) {
+; NOVEC-LABEL: fun0:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r13, %r15, 104(%r15)
+; NOVEC-NEXT:    .cfi_offset %r13, -56
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -224
+; NOVEC-NEXT:    .cfi_def_cfa_offset 384
+; NOVEC-NEXT:    std %f8, 216(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 208(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f10, 200(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f11, 192(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f12, 184(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f13, 176(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f14, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f15, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    .cfi_offset %f10, -184
+; NOVEC-NEXT:    .cfi_offset %f11, -192
+; NOVEC-NEXT:    .cfi_offset %f12, -200
+; NOVEC-NEXT:    .cfi_offset %f13, -208
+; NOVEC-NEXT:    .cfi_offset %f14, -216
+; NOVEC-NEXT:    .cfi_offset %f15, -224
+; NOVEC-NEXT:    lgh %r0, 414(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f14, %r0
+; NOVEC-NEXT:    lgh %r0, 406(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f12, %r0
+; NOVEC-NEXT:    lgh %r0, 398(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f9, %r0
+; NOVEC-NEXT:    lgh %r0, 390(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ler %f10, %f6
+; NOVEC-NEXT:    ler %f11, %f4
+; NOVEC-NEXT:    ler %f13, %f2
+; NOVEC-NEXT:    ler %f15, %f0
+; NOVEC-NEXT:    lgr %r13, %r2
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f8, %f0
+; NOVEC-NEXT:    ler %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f12
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f12, %f0
+; NOVEC-NEXT:    ler %f0, %f14
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f14, %f0
+; NOVEC-NEXT:    ler %f0, %f15
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f15, %f0
+; NOVEC-NEXT:    ler %f0, %f13
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f13, %f0
+; NOVEC-NEXT:    ler %f0, %f11
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f11, %f0
+; NOVEC-NEXT:    ler %f0, %f10
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 6(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f11
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 4(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f13
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 2(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f15
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f14
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 14(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f12
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 12(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f9
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 10(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f8
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 8(%r13)
+; NOVEC-NEXT:    ld %f8, 216(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 208(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f10, 200(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f11, 192(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f12, 184(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f13, 176(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f14, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f15, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r13, %r15, 328(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun0:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r13, %r15, 104(%r15)
+; VECTOR-NEXT:    .cfi_offset %r13, -56
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -224
+; VECTOR-NEXT:    .cfi_def_cfa_offset 384
+; VECTOR-NEXT:    std %f8, 216(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f9, 208(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f10, 200(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f11, 192(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f12, 184(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f13, 176(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f14, 168(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f15, 160(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    .cfi_offset %f9, -176
+; VECTOR-NEXT:    .cfi_offset %f10, -184
+; VECTOR-NEXT:    .cfi_offset %f11, -192
+; VECTOR-NEXT:    .cfi_offset %f12, -200
+; VECTOR-NEXT:    .cfi_offset %f13, -208
+; VECTOR-NEXT:    .cfi_offset %f14, -216
+; VECTOR-NEXT:    .cfi_offset %f15, -224
+; VECTOR-NEXT:    vlreph %v11, 414(%r15)
+; VECTOR-NEXT:    vlreph %v12, 406(%r15)
+; VECTOR-NEXT:    vlreph %v13, 398(%r15)
+; VECTOR-NEXT:    vlreph %v14, 390(%r15)
+; VECTOR-NEXT:    ldr %f8, %f6
+; VECTOR-NEXT:    ldr %f9, %f4
+; VECTOR-NEXT:    ldr %f10, %f2
+; VECTOR-NEXT:    lgr %r13, %r2
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f15, %f0
+; VECTOR-NEXT:    ldr %f0, %f10
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f10, %f0
+; VECTOR-NEXT:    ldr %f0, %f9
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f8, %f0
+; VECTOR-NEXT:    ldr %f0, %f14
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f14, %f0
+; VECTOR-NEXT:    ldr %f0, %f13
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f13, %f0
+; VECTOR-NEXT:    ldr %f0, %f12
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f12, %f0
+; VECTOR-NEXT:    ldr %f0, %f11
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    vsteh %v0, 14(%r13), 0
+; VECTOR-NEXT:    vsteh %v12, 12(%r13), 0
+; VECTOR-NEXT:    vsteh %v13, 10(%r13), 0
+; VECTOR-NEXT:    vsteh %v14, 8(%r13), 0
+; VECTOR-NEXT:    vsteh %v8, 6(%r13), 0
+; VECTOR-NEXT:    vsteh %v9, 4(%r13), 0
+; VECTOR-NEXT:    vsteh %v10, 2(%r13), 0
+; VECTOR-NEXT:    vsteh %v15, 0(%r13), 0
+; VECTOR-NEXT:    ld %f8, 216(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f9, 208(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f10, 200(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f11, 192(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f12, 184(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f13, 176(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f14, 168(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f15, 160(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r13, %r15, 328(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %Res = fadd <8 x half> %Op, %Op
+  ret <8 x half> %Res
+}
+
+; Same, but with partial vector values.
+define <4 x half> @fun1(<4 x half> %Op) {
+; NOVEC-LABEL: fun1:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -192
+; NOVEC-NEXT:    .cfi_def_cfa_offset 352
+; NOVEC-NEXT:    std %f8, 184(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 176(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f10, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f11, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    .cfi_offset %f10, -184
+; NOVEC-NEXT:    .cfi_offset %f11, -192
+; NOVEC-NEXT:    ler %f8, %f6
+; NOVEC-NEXT:    ler %f9, %f4
+; NOVEC-NEXT:    ler %f10, %f2
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f11, %f0
+; NOVEC-NEXT:    ler %f0, %f10
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f10, %f0
+; NOVEC-NEXT:    ler %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ler %f6, %f0
+; NOVEC-NEXT:    ler %f0, %f11
+; NOVEC-NEXT:    ler %f2, %f10
+; NOVEC-NEXT:    ler %f4, %f9
+; NOVEC-NEXT:    ld %f8, 184(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 176(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f10, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f11, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 304(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun1:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -192
+; VECTOR-NEXT:    .cfi_def_cfa_offset 352
+; VECTOR-NEXT:    std %f8, 184(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f9, 176(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f10, 168(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f11, 160(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    .cfi_offset %f9, -176
+; VECTOR-NEXT:    .cfi_offset %f10, -184
+; VECTOR-NEXT:    .cfi_offset %f11, -192
+; VECTOR-NEXT:    ldr %f8, %f6
+; VECTOR-NEXT:    ldr %f9, %f4
+; VECTOR-NEXT:    ldr %f10, %f2
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f11, %f0
+; VECTOR-NEXT:    ldr %f0, %f10
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f10, %f0
+; VECTOR-NEXT:    ldr %f0, %f9
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ldr %f6, %f0
+; VECTOR-NEXT:    ldr %f0, %f11
+; VECTOR-NEXT:    ldr %f2, %f10
+; VECTOR-NEXT:    ldr %f4, %f9
+; VECTOR-NEXT:    ld %f8, 184(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f9, 176(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f10, 168(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f11, 160(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 304(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %Res = fadd <4 x half> %Op, %Op
+  ret <4 x half> %Res
+}
+
+; Test a vector extension.
+define <2 x half> @fun2(<2 x half> %Op) {
+; NOVEC-LABEL: fun2:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -176
+; NOVEC-NEXT:    .cfi_def_cfa_offset 336
+; NOVEC-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    ler %f8, %f2
+; NOVEC-NEXT:    brasl %r14, __extendhfdf2@PLT
+; NOVEC-NEXT:    ldr %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhfdf2@PLT
+; NOVEC-NEXT:    adbr %f9, %f9
+; NOVEC-NEXT:    ldr %f8, %f0
+; NOVEC-NEXT:    adbr %f8, %f0
+; NOVEC-NEXT:    ldr %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __truncdfhf2@PLT
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ldr %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __truncdfhf2@PLT
+; NOVEC-NEXT:    ler %f2, %f0
+; NOVEC-NEXT:    ler %f0, %f9
+; NOVEC-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 288(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun2:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -184
+; VECTOR-NEXT:    .cfi_def_cfa_offset 344
+; VECTOR-NEXT:    std %f8, 176(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    ldr %f8, %f0
+; VECTOR-NEXT:    ldr %f0, %f2
+; VECTOR-NEXT:    brasl %r14, __extendhfdf2@PLT
+; VECTOR-NEXT:    # kill: def $f0d killed $f0d def $v0
+; VECTOR-NEXT:    vst %v0, 160(%r15), 3 # 16-byte Spill
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfdf2@PLT
+; VECTOR-NEXT:    vl %v1, 160(%r15), 3 # 16-byte Reload
+; VECTOR-NEXT:    # kill: def $f0d killed $f0d def $v0
+; VECTOR-NEXT:    vmrhg %v0, %v0, %v1
+; VECTOR-NEXT:    vfadb %v0, %v0, %v0
+; VECTOR-NEXT:    vst %v0, 160(%r15), 3 # 16-byte Spill
+; VECTOR-NEXT:    # kill: def $f0d killed $f0d killed $v0
+; VECTOR-NEXT:    brasl %r14, __truncdfhf2@PLT
+; VECTOR-NEXT:    ldr %f8, %f0
+; VECTOR-NEXT:    vl %v0, 160(%r15), 3 # 16-byte Reload
+; VECTOR-NEXT:    vrepg %v0, %v0, 1
+; VECTOR-NEXT:    # kill: def $f0d killed $f0d killed $v0
+; VECTOR-NEXT:    brasl %r14, __truncdfhf2@PLT
+; VECTOR-NEXT:    ldr %f2, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    ld %f8, 176(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 296(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %E = fpext <2 x half> %Op to <2 x double>
+  %Add = fadd <2 x double> %E, %E
+  %Res = fptrunc <2 x double> %Add to <2 x half>
+  ret <2 x half> %Res
+}
+
+; Load and store an <8 x half> vector.
+define void @fun3(ptr %Src, ptr %Dst) {
+; NOVEC-LABEL: fun3:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    lgh %r0, 0(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    lgh %r0, 2(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f1, %r0
+; NOVEC-NEXT:    lgh %r0, 4(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f2, %r0
+; NOVEC-NEXT:    lgh %r0, 6(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f3, %r0
+; NOVEC-NEXT:    lgh %r0, 8(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f4, %r0
+; NOVEC-NEXT:    lgh %r0, 10(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f5, %r0
+; NOVEC-NEXT:    lgh %r0, 12(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f6, %r0
+; NOVEC-NEXT:    lgh %r0, 14(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f7, %r0
+; NOVEC-NEXT:    lgdr %r0, %f7
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 14(%r3)
+; NOVEC-NEXT:    lgdr %r0, %f6
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 12(%r3)
+; NOVEC-NEXT:    lgdr %r0, %f5
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 10(%r3)
+; NOVEC-NEXT:    lgdr %r0, %f4
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 8(%r3)
+; NOVEC-NEXT:    lgdr %r0, %f3
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 6(%r3)
+; NOVEC-NEXT:    lgdr %r0, %f2
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 4(%r3)
+; NOVEC-NEXT:    lgdr %r0, %f1
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 2(%r3)
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r3)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun3:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    vlreph %v0, 0(%r2)
+; VECTOR-NEXT:    vlreph %v1, 2(%r2)
+; VECTOR-NEXT:    vlreph %v2, 4(%r2)
+; VECTOR-NEXT:    vlreph %v3, 6(%r2)
+; VECTOR-NEXT:    vlreph %v4, 8(%r2)
+; VECTOR-NEXT:    vlreph %v5, 10(%r2)
+; VECTOR-NEXT:    vlreph %v6, 12(%r2)
+; VECTOR-NEXT:    vlreph %v7, 14(%r2)
+; VECTOR-NEXT:    vsteh %v7, 14(%r3), 0
+; VECTOR-NEXT:    vsteh %v6, 12(%r3), 0
+; VECTOR-NEXT:    vsteh %v5, 10(%r3), 0
+; VECTOR-NEXT:    vsteh %v4, 8(%r3), 0
+; VECTOR-NEXT:    vsteh %v3, 6(%r3), 0
+; VECTOR-NEXT:    vsteh %v2, 4(%r3), 0
+; VECTOR-NEXT:    vsteh %v1, 2(%r3), 0
+; VECTOR-NEXT:    vsteh %v0, 0(%r3), 0
+; VECTOR-NEXT:    br %r14
+entry:
+  %L = load <8 x half>, ptr %Src
+  store <8 x half> %L, ptr %Dst
+  ret void
+}
+
+; Call a function with <8 x half> argument and return values.
+declare <8 x half> @foo(<8 x half>)
+define void @fun4(ptr %Src, ptr %Dst) {
+; NOVEC-LABEL: fun4:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r13, %r15, 104(%r15)
+; NOVEC-NEXT:    .cfi_offset %r13, -56
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -208
+; NOVEC-NEXT:    .cfi_def_cfa_offset 368
+; NOVEC-NEXT:    lgh %r0, 0(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    lgh %r0, 2(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f2, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    # kill: def $f2h killed $f2h killed $f2d
+; NOVEC-NEXT:    lgh %r0, 4(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f4, %r0
+; NOVEC-NEXT:    # kill: def $f4h killed $f4h killed $f4d
+; NOVEC-NEXT:    lgh %r0, 6(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f6, %r0
+; NOVEC-NEXT:    # kill: def $f6h killed $f6h killed $f6d
+; NOVEC-NEXT:    lgh %r0, 8(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f1, %r0
+; NOVEC-NEXT:    lgh %r0, 10(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f3, %r0
+; NOVEC-NEXT:    lgh %r0, 12(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f5, %r0
+; NOVEC-NEXT:    lgh %r0, 14(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f7, %r0
+; NOVEC-NEXT:    lgdr %r0, %f7
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 190(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f5
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 182(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f3
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 174(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f1
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    la %r2, 192(%r15)
+; NOVEC-NEXT:    lgr %r13, %r3
+; NOVEC-NEXT:    sth %r0, 166(%r15)
+; NOVEC-NEXT:    brasl %r14, foo@PLT
+; NOVEC-NEXT:    lgh %r0, 192(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    lgh %r0, 194(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f1, %r0
+; NOVEC-NEXT:    lgh %r0, 196(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f2, %r0
+; NOVEC-NEXT:    lgh %r0, 198(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f3, %r0
+; NOVEC-NEXT:    lgh %r0, 200(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f4, %r0
+; NOVEC-NEXT:    lgh %r0, 202(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f5, %r0
+; NOVEC-NEXT:    lgh %r0, 204(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f6, %r0
+; NOVEC-NEXT:    lgh %r0, 206(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f7, %r0
+; NOVEC-NEXT:    lgdr %r0, %f7
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 14(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f6
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 12(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f5
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 10(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f4
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 8(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f3
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 6(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f2
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 4(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f1
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 2(%r13)
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r13)
+; NOVEC-NEXT:    lmg %r13, %r15, 312(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun4:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r13, %r15, 104(%r15)
+; VECTOR-NEXT:    .cfi_offset %r13, -56
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -208
+; VECTOR-NEXT:    .cfi_def_cfa_offset 368
+; VECTOR-NEXT:    vlreph %v6, 6(%r2)
+; VECTOR-NEXT:    vlreph %v4, 4(%r2)
+; VECTOR-NEXT:    vlreph %v2, 2(%r2)
+; VECTOR-NEXT:    vlreph %v0, 0(%r2)
+; VECTOR-NEXT:    vlreph %v1, 8(%r2)
+; VECTOR-NEXT:    vlreph %v3, 10(%r2)
+; VECTOR-NEXT:    vlreph %v5, 12(%r2)
+; VECTOR-NEXT:    vlreph %v7, 14(%r2)
+; VECTOR-NEXT:    la %r2, 192(%r15)
+; VECTOR-NEXT:    lgr %r13, %r3
+; VECTOR-NEXT:    vsteh %v7, 190(%r15), 0
+; VECTOR-NEXT:    vsteh %v5, 182(%r15), 0
+; VECTOR-NEXT:    vsteh %v3, 174(%r15), 0
+; VECTOR-NEXT:    vsteh %v1, 166(%r15), 0
+; VECTOR-NEXT:    brasl %r14, foo@PLT
+; VECTOR-NEXT:    vlreph %v0, 192(%r15)
+; VECTOR-NEXT:    vlreph %v1, 194(%r15)
+; VECTOR-NEXT:    vlreph %v2, 196(%r15)
+; VECTOR-NEXT:    vlreph %v3, 198(%r15)
+; VECTOR-NEXT:    vlreph %v4, 200(%r15)
+; VECTOR-NEXT:    vlreph %v5, 202(%r15)
+; VECTOR-NEXT:    vlreph %v6, 204(%r15)
+; VECTOR-NEXT:    vlreph %v7, 206(%r15)
+; VECTOR-NEXT:    vsteh %v7, 14(%r13), 0
+; VECTOR-NEXT:    vsteh %v6, 12(%r13), 0
+; VECTOR-NEXT:    vsteh %v5, 10(%r13), 0
+; VECTOR-NEXT:    vsteh %v4, 8(%r13), 0
+; VECTOR-NEXT:    vsteh %v3, 6(%r13), 0
+; VECTOR-NEXT:    vsteh %v2, 4(%r13), 0
+; VECTOR-NEXT:    vsteh %v1, 2(%r13), 0
+; VECTOR-NEXT:    vsteh %v0, 0(%r13), 0
+; VECTOR-NEXT:    lmg %r13, %r15, 312(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %arg = load <8 x half>, ptr %Src
+  %Res = call <8 x half> @foo(<8 x half> %arg)
+  store <8 x half> %Res, ptr %Dst
+  ret void
+}
+
+; Receive and pass argument fully on stack.
+declare void @foo2(<4 x half> %dummy, <8 x half> %Arg5)
+define void @fun5(<4 x half> %dummy, <8 x half> %Arg5) {
+; NOVEC-LABEL: fun5:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -256
+; NOVEC-NEXT:    .cfi_def_cfa_offset 416
+; NOVEC-NEXT:    std %f8, 248(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 240(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f10, 232(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f11, 224(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    .cfi_offset %f10, -184
+; NOVEC-NEXT:    .cfi_offset %f11, -192
+; NOVEC-NEXT:    lgh %r0, 422(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f1, %r0
+; NOVEC-NEXT:    lgh %r0, 430(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f3, %r0
+; NOVEC-NEXT:    lgh %r0, 438(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f5, %r0
+; NOVEC-NEXT:    lgh %r0, 446(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f7, %r0
+; NOVEC-NEXT:    lgh %r0, 454(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f8, %r0
+; NOVEC-NEXT:    lgh %r0, 462(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f9, %r0
+; NOVEC-NEXT:    lgh %r0, 470(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f10, %r0
+; NOVEC-NEXT:    lgh %r0, 478(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f11, %r0
+; NOVEC-NEXT:    lgdr %r0, %f11
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 222(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f10
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 214(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f9
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 206(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f8
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 198(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f7
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 190(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f5
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 182(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f3
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 174(%r15)
+; NOVEC-NEXT:    lgdr %r0, %f1
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 166(%r15)
+; NOVEC-NEXT:    brasl %r14, foo2@PLT
+; NOVEC-NEXT:    ld %f8, 248(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 240(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f10, 232(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f11, 224(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 368(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun5:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -224
+; VECTOR-NEXT:    .cfi_def_cfa_offset 384
+; VECTOR-NEXT:    vlreph %v1, 390(%r15)
+; VECTOR-NEXT:    vlreph %v3, 398(%r15)
+; VECTOR-NEXT:    vlreph %v5, 406(%r15)
+; VECTOR-NEXT:    vlreph %v7, 414(%r15)
+; VECTOR-NEXT:    vlreph %v16, 422(%r15)
+; VECTOR-NEXT:    vlreph %v17, 430(%r15)
+; VECTOR-NEXT:    vlreph %v18, 438(%r15)
+; VECTOR-NEXT:    vlreph %v19, 446(%r15)
+; VECTOR-NEXT:    vsteh %v19, 222(%r15), 0
+; VECTOR-NEXT:    vsteh %v18, 214(%r15), 0
+; VECTOR-NEXT:    vsteh %v17, 206(%r15), 0
+; VECTOR-NEXT:    vsteh %v16, 198(%r15), 0
+; VECTOR-NEXT:    vsteh %v7, 190(%r15), 0
+; VECTOR-NEXT:    vsteh %v5, 182(%r15), 0
+; VECTOR-NEXT:    vsteh %v3, 174(%r15), 0
+; VECTOR-NEXT:    vsteh %v1, 166(%r15), 0
+; VECTOR-NEXT:    brasl %r14, foo2@PLT
+; VECTOR-NEXT:    lmg %r14, %r15, 336(%r15)
+; VECTOR-NEXT:    br %r14
+  call void @foo2(<4 x half> %dummy, <8 x half> %Arg5)
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/fp-half.ll b/llvm/test/CodeGen/SystemZ/fp-half.ll
new file mode 100644
index 0000000000000..f479e405b04e9
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-half.ll
@@ -0,0 +1,612 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 -verify-machineinstrs \
+; RUN:   | FileCheck %s --check-prefix=NOVEC
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 -verify-machineinstrs \
+; RUN:   | FileCheck %s  --check-prefix=VECTOR
+;
+; Tests for 16-bit floating point (half).
+
+; Incoming half arguments added together and returned.
+define half @fun0(half %Op0, half %Op1) {
+; NOVEC-LABEL: fun0:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -176
+; NOVEC-NEXT:    .cfi_def_cfa_offset 336
+; NOVEC-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    ler %f8, %f0
+; NOVEC-NEXT:    ler %f0, %f2
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 288(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun0:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -176
+; VECTOR-NEXT:    .cfi_def_cfa_offset 336
+; VECTOR-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    .cfi_offset %f9, -176
+; VECTOR-NEXT:    ldr %f8, %f0
+; VECTOR-NEXT:    ldr %f0, %f2
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f9
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 288(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %Res = fadd half %Op0, %Op1
+  ret half %Res
+}
+
+define half @fun1(half %Op0, half %Op1) {
+; NOVEC-LABEL: fun1:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -176
+; NOVEC-NEXT:    .cfi_def_cfa_offset 336
+; NOVEC-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    ler %f8, %f2
+; NOVEC-NEXT:    brasl %r14, __extendhfdf2@PLT
+; NOVEC-NEXT:    ldr %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhfdf2@PLT
+; NOVEC-NEXT:    adbr %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __truncdfhf2@PLT
+; NOVEC-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 288(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun1:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -176
+; VECTOR-NEXT:    .cfi_def_cfa_offset 336
+; VECTOR-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    .cfi_offset %f9, -176
+; VECTOR-NEXT:    ldr %f8, %f2
+; VECTOR-NEXT:    brasl %r14, __extendhfdf2@PLT
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfdf2@PLT
+; VECTOR-NEXT:    wfadb %f0, %f9, %f0
+; VECTOR-NEXT:    brasl %r14, __truncdfhf2@PLT
+; VECTOR-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 288(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %E0 = fpext half %Op0 to double
+  %E1 = fpext half %Op1 to double
+  %Add = fadd double %E0, %E1
+  %Res = fptrunc double %Add to half
+  ret half %Res
+}
+
+define half @fun2(half %Op0, half %Op1) {
+; NOVEC-LABEL: fun2:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -232
+; NOVEC-NEXT:    .cfi_def_cfa_offset 392
+; NOVEC-NEXT:    std %f8, 224(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 216(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f11, 208(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    .cfi_offset %f11, -184
+; NOVEC-NEXT:    la %r2, 160(%r15)
+; NOVEC-NEXT:    ler %f8, %f2
+; NOVEC-NEXT:    brasl %r14, __extendhftf2@PLT
+; NOVEC-NEXT:    ld %f9, 160(%r15)
+; NOVEC-NEXT:    ld %f11, 168(%r15)
+; NOVEC-NEXT:    la %r2, 176(%r15)
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhftf2@PLT
+; NOVEC-NEXT:    ld %f0, 176(%r15)
+; NOVEC-NEXT:    ld %f2, 184(%r15)
+; NOVEC-NEXT:    la %r2, 192(%r15)
+; NOVEC-NEXT:    axbr %f0, %f9
+; NOVEC-NEXT:    std %f0, 192(%r15)
+; NOVEC-NEXT:    std %f2, 200(%r15)
+; NOVEC-NEXT:    brasl %r14, __trunctfhf2@PLT
+; NOVEC-NEXT:    ld %f8, 224(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 216(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f11, 208(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 344(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun2:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -232
+; VECTOR-NEXT:    .cfi_def_cfa_offset 392
+; VECTOR-NEXT:    std %f8, 224(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    la %r2, 176(%r15)
+; VECTOR-NEXT:    ldr %f8, %f2
+; VECTOR-NEXT:    brasl %r14, __extendhftf2@PLT
+; VECTOR-NEXT:    mvc 160(16,%r15), 176(%r15)
+; VECTOR-NEXT:    la %r2, 192(%r15)
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhftf2@PLT
+; VECTOR-NEXT:    vl %v0, 192(%r15), 3
+; VECTOR-NEXT:    vl %v1, 160(%r15), 3 # 16-byte Reload
+; VECTOR-NEXT:    wfaxb %v0, %v1, %v0
+; VECTOR-NEXT:    la %r2, 208(%r15)
+; VECTOR-NEXT:    vst %v0, 208(%r15), 3
+; VECTOR-NEXT:    brasl %r14, __trunctfhf2@PLT
+; VECTOR-NEXT:    ld %f8, 224(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 344(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %E0 = fpext half %Op0 to fp128
+  %E1 = fpext half %Op1 to fp128
+  %Add = fadd fp128 %E0, %E1
+  %Res = fptrunc fp128 %Add to half
+  ret half %Res
+}
+
+; Test loading and storing a half value.
+define void @fun3(ptr %Src, ptr %Dst) {
+; NOVEC-LABEL: fun3:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    lgh %r0, 0(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r3)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun3:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    vlreph %v0, 0(%r2)
+; VECTOR-NEXT:    vsteh %v0, 0(%r3), 0
+; VECTOR-NEXT:    br %r14
+entry:
+  %L = load half, ptr %Src, align 2
+  store half %L, ptr %Dst, align 2
+  ret void
+}
+
+define void @fun4(ptr %Src, ptr %Dst) {
+; NOVEC-LABEL: fun4:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r13, %r15, 104(%r15)
+; NOVEC-NEXT:    .cfi_offset %r13, -56
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -160
+; NOVEC-NEXT:    .cfi_def_cfa_offset 320
+; NOVEC-NEXT:    lgh %r0, 0(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    lgr %r13, %r3
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    brasl %r14, __extendhfdf2@PLT
+; NOVEC-NEXT:    adbr %f0, %f0
+; NOVEC-NEXT:    brasl %r14, __truncdfhf2@PLT
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r13)
+; NOVEC-NEXT:    lmg %r13, %r15, 264(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun4:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r13, %r15, 104(%r15)
+; VECTOR-NEXT:    .cfi_offset %r13, -56
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -160
+; VECTOR-NEXT:    .cfi_def_cfa_offset 320
+; VECTOR-NEXT:    vlreph %v0, 0(%r2)
+; VECTOR-NEXT:    lgr %r13, %r3
+; VECTOR-NEXT:    brasl %r14, __extendhfdf2@PLT
+; VECTOR-NEXT:    adbr %f0, %f0
+; VECTOR-NEXT:    brasl %r14, __truncdfhf2@PLT
+; VECTOR-NEXT:    vsteh %v0, 0(%r13), 0
+; VECTOR-NEXT:    lmg %r13, %r15, 264(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %Op0 = load half, ptr %Src, align 2
+  %E0 = fpext half %Op0 to double
+  %Add = fadd double %E0, %E0
+  %Res = fptrunc double %Add to half
+  store half %Res, ptr %Dst, align 2
+  ret void
+}
+
+define void @fun5(ptr %Src, ptr %Dst) {
+; NOVEC-LABEL: fun5:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r13, %r15, 104(%r15)
+; NOVEC-NEXT:    .cfi_offset %r13, -56
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -192
+; NOVEC-NEXT:    .cfi_def_cfa_offset 352
+; NOVEC-NEXT:    lgh %r0, 0(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    la %r2, 160(%r15)
+; NOVEC-NEXT:    lgr %r13, %r3
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    brasl %r14, __extendhftf2@PLT
+; NOVEC-NEXT:    ld %f0, 160(%r15)
+; NOVEC-NEXT:    ld %f2, 168(%r15)
+; NOVEC-NEXT:    la %r2, 176(%r15)
+; NOVEC-NEXT:    axbr %f0, %f0
+; NOVEC-NEXT:    std %f0, 176(%r15)
+; NOVEC-NEXT:    std %f2, 184(%r15)
+; NOVEC-NEXT:    brasl %r14, __trunctfhf2@PLT
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r13)
+; NOVEC-NEXT:    lmg %r13, %r15, 296(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun5:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r13, %r15, 104(%r15)
+; VECTOR-NEXT:    .cfi_offset %r13, -56
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -192
+; VECTOR-NEXT:    .cfi_def_cfa_offset 352
+; VECTOR-NEXT:    vlreph %v0, 0(%r2)
+; VECTOR-NEXT:    la %r2, 160(%r15)
+; VECTOR-NEXT:    lgr %r13, %r3
+; VECTOR-NEXT:    brasl %r14, __extendhftf2@PLT
+; VECTOR-NEXT:    vl %v0, 160(%r15), 3
+; VECTOR-NEXT:    wfaxb %v0, %v0, %v0
+; VECTOR-NEXT:    la %r2, 176(%r15)
+; VECTOR-NEXT:    vst %v0, 176(%r15), 3
+; VECTOR-NEXT:    brasl %r14, __trunctfhf2@PLT
+; VECTOR-NEXT:    vsteh %v0, 0(%r13), 0
+; VECTOR-NEXT:    lmg %r13, %r15, 296(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %Op0 = load half, ptr %Src, align 2
+  %E0 = fpext half %Op0 to fp128
+  %Add = fadd fp128 %E0, %E0
+  %Res = fptrunc fp128 %Add to half
+  store half %Res, ptr %Dst, align 2
+  ret void
+}
+
+; Test a chain of half operations which should have each operation surrounded
+; by conversions to/from fp32 for proper emulation.
+define half @fun6(half %Op0, half %Op1, half %Op2) {
+; NOVEC-LABEL: fun6:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -184
+; NOVEC-NEXT:    .cfi_def_cfa_offset 344
+; NOVEC-NEXT:    std %f8, 176(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f10, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    .cfi_offset %f10, -184
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f2
+; NOVEC-NEXT:    ler %f8, %f4
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    ler %f10, %f0
+; NOVEC-NEXT:    ler %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f10
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ld %f8, 176(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f10, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 296(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun6:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -184
+; VECTOR-NEXT:    .cfi_def_cfa_offset 344
+; VECTOR-NEXT:    std %f8, 176(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f9, 168(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f10, 160(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    .cfi_offset %f9, -176
+; VECTOR-NEXT:    .cfi_offset %f10, -184
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f2
+; VECTOR-NEXT:    ldr %f8, %f4
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    ldr %f10, %f0
+; VECTOR-NEXT:    ldr %f0, %f9
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f10
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    wfasb %f0, %f9, %f0
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ld %f8, 176(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f9, 168(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f10, 160(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 296(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %A0 = fadd half %Op0, %Op1
+  %Res = fadd half %A0, %Op2
+  ret half %Res
+}
+
+; Store an incoming half argument and return a loaded one.
+define half @fun7(half %Op0, ptr %Dst, ptr %Src) {
+; NOVEC-LABEL: fun7:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r2)
+; NOVEC-NEXT:    lgh %r0, 0(%r3)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun7:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    vsteh %v0, 0(%r2), 0
+; VECTOR-NEXT:    vlreph %v0, 0(%r3)
+; VECTOR-NEXT:    br %r14
+entry:
+  store half %Op0, ptr %Dst
+  %Res = load half, ptr %Src
+  ret half %Res
+}
+
+; Call a function with half argument and return values.
+declare half @foo(half)
+define void @fun8(ptr %Src, ptr %Dst) {
+; NOVEC-LABEL: fun8:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r13, %r15, 104(%r15)
+; NOVEC-NEXT:    .cfi_offset %r13, -56
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -160
+; NOVEC-NEXT:    .cfi_def_cfa_offset 320
+; NOVEC-NEXT:    lgh %r0, 0(%r2)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    lgr %r13, %r3
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    brasl %r14, foo@PLT
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    sth %r0, 0(%r13)
+; NOVEC-NEXT:    lmg %r13, %r15, 264(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun8:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r13, %r15, 104(%r15)
+; VECTOR-NEXT:    .cfi_offset %r13, -56
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -160
+; VECTOR-NEXT:    .cfi_def_cfa_offset 320
+; VECTOR-NEXT:    vlreph %v0, 0(%r2)
+; VECTOR-NEXT:    lgr %r13, %r3
+; VECTOR-NEXT:    brasl %r14, foo@PLT
+; VECTOR-NEXT:    vsteh %v0, 0(%r13), 0
+; VECTOR-NEXT:    lmg %r13, %r15, 264(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  %arg = load half, ptr %Src
+  %Res = call half @foo(half %arg)
+  store half %Res, ptr %Dst
+  ret void
+}
+
+; Receive stack argument.
+define half @fun9(half %Arg0, half %Arg1, half %Arg2, half %Arg3, half %Arg4) {
+; NOVEC-LABEL: fun9:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -176
+; NOVEC-NEXT:    .cfi_def_cfa_offset 336
+; NOVEC-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; NOVEC-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; NOVEC-NEXT:    .cfi_offset %f8, -168
+; NOVEC-NEXT:    .cfi_offset %f9, -176
+; NOVEC-NEXT:    lgh %r0, 342(%r15)
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ler %f8, %f6
+; NOVEC-NEXT:    ldgr %f0, %r0
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    ler %f9, %f0
+; NOVEC-NEXT:    ler %f0, %f8
+; NOVEC-NEXT:    brasl %r14, __extendhfsf2@PLT
+; NOVEC-NEXT:    aebr %f0, %f9
+; NOVEC-NEXT:    brasl %r14, __truncsfhf2@PLT
+; NOVEC-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; NOVEC-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; NOVEC-NEXT:    lmg %r14, %r15, 288(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun9:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -176
+; VECTOR-NEXT:    .cfi_def_cfa_offset 336
+; VECTOR-NEXT:    std %f8, 168(%r15) # 8-byte Spill
+; VECTOR-NEXT:    std %f9, 160(%r15) # 8-byte Spill
+; VECTOR-NEXT:    .cfi_offset %f8, -168
+; VECTOR-NEXT:    .cfi_offset %f9, -176
+; VECTOR-NEXT:    vlreph %v0, 342(%r15)
+; VECTOR-NEXT:    ldr %f8, %f6
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    ldr %f9, %f0
+; VECTOR-NEXT:    ldr %f0, %f8
+; VECTOR-NEXT:    brasl %r14, __extendhfsf2@PLT
+; VECTOR-NEXT:    aebr %f0, %f9
+; VECTOR-NEXT:    brasl %r14, __truncsfhf2@PLT
+; VECTOR-NEXT:    ld %f8, 168(%r15) # 8-byte Reload
+; VECTOR-NEXT:    ld %f9, 160(%r15) # 8-byte Reload
+; VECTOR-NEXT:    lmg %r14, %r15, 288(%r15)
+; VECTOR-NEXT:    br %r14
+  %A0 = fadd half %Arg3, %Arg4
+  ret half %A0
+}
+
+; Pass stack argument.
+define void @fun10(half %Arg0) {
+; NOVEC-LABEL: fun10:
+; NOVEC:       # %bb.0:
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -168
+; NOVEC-NEXT:    .cfi_def_cfa_offset 328
+; NOVEC-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; NOVEC-NEXT:    lgdr %r0, %f0
+; NOVEC-NEXT:    srlg %r0, %r0, 48
+; NOVEC-NEXT:    ler %f2, %f0
+; NOVEC-NEXT:    ler %f4, %f0
+; NOVEC-NEXT:    ler %f6, %f0
+; NOVEC-NEXT:    sth %r0, 166(%r15)
+; NOVEC-NEXT:    brasl %r14, fun9@PLT
+; NOVEC-NEXT:    lmg %r14, %r15, 280(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun10:
+; VECTOR:       # %bb.0:
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -168
+; VECTOR-NEXT:    .cfi_def_cfa_offset 328
+; VECTOR-NEXT:    ldr %f2, %f0
+; VECTOR-NEXT:    ldr %f4, %f0
+; VECTOR-NEXT:    ldr %f6, %f0
+; VECTOR-NEXT:    vsteh %v0, 166(%r15), 0
+; VECTOR-NEXT:    brasl %r14, fun9@PLT
+; VECTOR-NEXT:    lmg %r14, %r15, 280(%r15)
+; VECTOR-NEXT:    br %r14
+  call void @fun9(half %Arg0, half %Arg0, half %Arg0, half %Arg0, half %Arg0)
+  ret void
+}
+
+; Test loading some immediates from the Constant Pool.
+declare void @foo2(half, half, half, half)
+define void @fun11() {
+; NOVEC-LABEL: fun11:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    stmg %r14, %r15, 112(%r15)
+; NOVEC-NEXT:    .cfi_offset %r14, -48
+; NOVEC-NEXT:    .cfi_offset %r15, -40
+; NOVEC-NEXT:    aghi %r15, -160
+; NOVEC-NEXT:    .cfi_def_cfa_offset 320
+; NOVEC-NEXT:    lghrl %r0, .LCPI11_0
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f4, %r0
+; NOVEC-NEXT:    lghrl %r0, .LCPI11_1
+; NOVEC-NEXT:    lzer %f2
+; NOVEC-NEXT:    lcdfr %f0, %f2
+; NOVEC-NEXT:    # kill: def $f4h killed $f4h killed $f4d
+; NOVEC-NEXT:    sllg %r0, %r0, 48
+; NOVEC-NEXT:    ldgr %f6, %r0
+; NOVEC-NEXT:    # kill: def $f6h killed $f6h killed $f6d
+; NOVEC-NEXT:    brasl %r14, foo2@PLT
+; NOVEC-NEXT:    lmg %r14, %r15, 272(%r15)
+; NOVEC-NEXT:    br %r14
+;
+; VECTOR-LABEL: fun11:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    stmg %r14, %r15, 112(%r15)
+; VECTOR-NEXT:    .cfi_offset %r14, -48
+; VECTOR-NEXT:    .cfi_offset %r15, -40
+; VECTOR-NEXT:    aghi %r15, -160
+; VECTOR-NEXT:    .cfi_def_cfa_offset 320
+; VECTOR-NEXT:    lzer %f2
+; VECTOR-NEXT:    vrepih %v4, 13824
+; VECTOR-NEXT:    vrepih %v6, 15360
+; VECTOR-NEXT:    lcdfr %f0, %f2
+; VECTOR-NEXT:    brasl %r14, foo2@PLT
+; VECTOR-NEXT:    lmg %r14, %r15, 272(%r15)
+; VECTOR-NEXT:    br %r14
+entry:
+  call void @foo2(half -0.0, half 0.0, half 0.375, half 1.0)
+  ret void
+}
+
+; Test a tail call.
+declare void @foo3(half)
+define void @fun12(half %Arg0) {
+; NOVEC-LABEL: fun12:
+; NOVEC:       # %bb.0: # %entry
+; NOVEC-NEXT:    jg foo3@PLT
+;
+; VECTOR-LABEL: fun12:
+; VECTOR:       # %bb.0: # %entry
+; VECTOR-NEXT:    jg foo3@PLT
+entry:
+  tail call void @foo3(half %Arg0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/fp-libcall.ll b/llvm/test/CodeGen/SystemZ/fp-libcall.ll
index 60b698e34fcfe..5069b9b257b80 100644
--- a/llvm/test/CodeGen/SystemZ/fp-libcall.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-libcall.ll
@@ -212,6 +212,16 @@ define fp128 @f30(fp128 %x, fp128 %y) {
   ret fp128 %tmp
 }
 
+define half @f31_half(half %x, half %y) {
+; CHECK-LABEL: f31_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, fmaxf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+  %tmp = call half @llvm.maxnum.f16(half %x, half %y)
+  ret half %tmp
+}
+
 define float @f31(float %x, float %y) {
 ; CHECK-LABEL: f31:
 ; CHECK: brasl %r14, fmaxf@PLT
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-01.ll b/llvm/test/CodeGen/SystemZ/fp-mul-01.ll
index c5e66ff72c2a4..323907f03b743 100644
--- a/llvm/test/CodeGen/SystemZ/fp-mul-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-01.ll
@@ -6,6 +6,18 @@
 
 declare float @foo()
 
+; Check register multiplication.
+define half @f0(half %f1, half %f2) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: meebr %f0, %f9
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = fmul half %f1, %f2
+  ret half %res
+}
+
 ; Check register multiplication.
 define float @f1(float %f1, float %f2) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-06.ll b/llvm/test/CodeGen/SystemZ/fp-mul-06.ll
index 8fd363bc397d0..6b285a49057dc 100644
--- a/llvm/test/CodeGen/SystemZ/fp-mul-06.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-06.ll
@@ -3,8 +3,22 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 \
 ; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
+declare half @llvm.fma.f16(half %f1, half %f2, half %f3)
 declare float @llvm.fma.f32(float %f1, float %f2, float %f3)
 
+define half @f0(half %f1, half %f2, half %acc) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-SCALAR: maebr %f0, %f9, %f10
+; CHECK-VECTOR: wfmasb %f0, %f0, %f8, %f10
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.fma.f16 (half %f1, half %f2, half %acc)
+  ret half %res
+}
+
 define float @f1(float %f1, float %f2, float %acc) {
 ; CHECK-LABEL: f1:
 ; CHECK-SCALAR: maebr %f4, %f0, %f2
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-08.ll b/llvm/test/CodeGen/SystemZ/fp-mul-08.ll
index 543ab95551690..2b18abec8d555 100644
--- a/llvm/test/CodeGen/SystemZ/fp-mul-08.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-08.ll
@@ -3,8 +3,26 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 \
 ; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
+declare half @llvm.fma.f16(half %f1, half %f2, half %f3)
 declare float @llvm.fma.f32(float %f1, float %f2, float %f3)
 
+define half @f0(half %f1, half %f2, half %acc) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: lcdfr %f0, %f0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-SCALAR: maebr %f0, %f9, %f8
+; CHECK-VECTOR: wfmasb %f0, %f0, %f8, %f10
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %negacc = fneg half %acc
+  %res = call half @llvm.fma.f16 (half %f1, half %f2, half %negacc)
+  ret half %res
+}
+
 define float @f1(float %f1, float %f2, float %acc) {
 ; CHECK-LABEL: f1:
 ; CHECK-SCALAR: msebr %f4, %f0, %f2
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-10.ll b/llvm/test/CodeGen/SystemZ/fp-mul-10.ll
index 669ccbacf7898..1ecf52fbde354 100644
--- a/llvm/test/CodeGen/SystemZ/fp-mul-10.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-10.ll
@@ -2,6 +2,7 @@
 
 declare double @llvm.fma.f64(double %f1, double %f2, double %f3)
 declare float @llvm.fma.f32(float %f1, float %f2, float %f3)
+declare half @llvm.fma.f16(half %f1, half %f2, half %f3)
 
 define double @f1(double %f1, double %f2, double %acc) {
 ; CHECK-LABEL: f1:
@@ -22,6 +23,22 @@ define double @f2(double %f1, double %f2, double %acc) {
   ret double %negres
 }
 
+define half @f3_half(half %f1, half %f2, half %acc) {
+; CHECK-LABEL: f3_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: wfmasb %f0, %f0, %f8, %f10
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: lcdfr %f0, %f0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.fma.f16 (half %f1, half %f2, half %acc)
+  %negres = fneg half %res
+  ret half %negres
+}
+
 define float @f3(float %f1, float %f2, float %acc) {
 ; CHECK-LABEL: f3:
 ; CHECK: wfnmasb %f0, %f0, %f2, %f4
@@ -31,6 +48,26 @@ define float @f3(float %f1, float %f2, float %acc) {
   ret float %negres
 }
 
+define half @f4_half(half %f1, half %f2, half %acc) {
+; CHECK-LABEL: f4_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: lcdfr %f0, %f0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: wfmasb %f0, %f0, %f8, %f10
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: lcdfr %f0, %f0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %negacc = fneg half %acc
+  %res = call half @llvm.fma.f16 (half %f1, half %f2, half %negacc)
+  %negres = fneg half %res
+  ret half %negres
+}
+
 define float @f4(float %f1, float %f2, float %acc) {
 ; CHECK-LABEL: f4:
 ; CHECK: wfnmssb %f0, %f0, %f2, %f4
@@ -40,4 +77,3 @@ define float @f4(float %f1, float %f2, float %acc) {
   %negres = fneg float %res
   ret float %negres
 }
-
diff --git a/llvm/test/CodeGen/SystemZ/fp-mul-15.ll b/llvm/test/CodeGen/SystemZ/fp-mul-15.ll
new file mode 100644
index 0000000000000..c897d05ab86df
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fp-mul-15.ll
@@ -0,0 +1,20 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 | FileCheck %s
+;
+; Check that a multiply-and-add  *not* result for half.
+
+define half @f1(half %arg, half %A2, half %A3) {
+; CHECK-LABEL: f1:
+; CHECK: brasl   %r14, __extendhfsf2@PLT
+; CHECK: brasl   %r14, __extendhfsf2@PLT
+; CHECK: meebr   %f0, %f10
+; CHECK: brasl   %r14, __truncsfhf2@PLT
+; CHECK: brasl   %r14, __extendhfsf2@PLT
+; CHECK: brasl   %r14, __extendhfsf2@PLT
+; CHECK: wfasb   %f0, %f9, %f0
+; CHECK: brasl   %r14, __truncsfhf2@PLT
+
+bb:
+  %i = fmul contract half %arg, %A2
+  %i4 = fadd contract half %i, %A3
+  ret half %i4
+}
diff --git a/llvm/test/CodeGen/SystemZ/fp-neg-01.ll b/llvm/test/CodeGen/SystemZ/fp-neg-01.ll
index 875905de4948d..a8fe8d5da7c8a 100644
--- a/llvm/test/CodeGen/SystemZ/fp-neg-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-neg-01.ll
@@ -3,6 +3,17 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
+; Test f16.
+define half @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: lcdfr %f0, %f0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = fneg half %f
+  ret half %res
+}
+
 ; Test f32.
 define float @f1(float %f) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-neg-02.ll b/llvm/test/CodeGen/SystemZ/fp-neg-02.ll
index 7cd66948e2fc7..848c4740d8540 100644
--- a/llvm/test/CodeGen/SystemZ/fp-neg-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-neg-02.ll
@@ -2,6 +2,17 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
+; Test f16.
+define half @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: lcdfr %f0, %f0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = fneg half %f
+  ret half %res
+}
+
 ; Test f32.
 define float @f1(float %f) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-round-01.ll b/llvm/test/CodeGen/SystemZ/fp-round-01.ll
index b1db2f547a832..21b354c7a83c4 100644
--- a/llvm/test/CodeGen/SystemZ/fp-round-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-round-01.ll
@@ -34,6 +34,18 @@ define void @f3(ptr %ptr) {
   ret void
 }
 
+; Test nearbyint for f16.
+declare half @llvm.nearbyint.f16(half %f)
+define half @f4_half(half %f) {
+; CHECK-LABEL: f4_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, nearbyintf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.nearbyint.f16(half %f)
+  ret half %res
+}
+
 ; Test nearbyint for f32.
 declare float @llvm.nearbyint.f32(float %f)
 define float @f4(float %f) {
@@ -66,6 +78,18 @@ define void @f6(ptr %ptr) {
   ret void
 }
 
+; Test floor for f16.
+declare half @llvm.floor.f16(half %f)
+define half @f7_half(half %f) {
+; CHECK-LABEL: f7_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, floorf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.floor.f16(half %f)
+  ret half %res
+}
+
 ; Test floor for f32.
 declare float @llvm.floor.f32(float %f)
 define float @f7(float %f) {
@@ -98,6 +122,18 @@ define void @f9(ptr %ptr) {
   ret void
 }
 
+; Test ceil for f16.
+declare half @llvm.ceil.f16(half %f)
+define half @f10_half(half %f) {
+; CHECK-LABEL: f10_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, ceilf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.ceil.f16(half %f)
+  ret half %res
+}
+
 ; Test ceil for f32.
 declare float @llvm.ceil.f32(float %f)
 define float @f10(float %f) {
@@ -162,6 +198,18 @@ define void @f15(ptr %ptr) {
   ret void
 }
 
+; Test round for f16.
+declare half @llvm.round.f16(half %f)
+define half @f16_half(half %f) {
+; CHECK-LABEL: f16_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, roundf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.round.f16(half %f)
+  ret half %res
+}
+
 ; Test round for f32.
 declare float @llvm.round.f32(float %f)
 define float @f16(float %f) {
diff --git a/llvm/test/CodeGen/SystemZ/fp-round-02.ll b/llvm/test/CodeGen/SystemZ/fp-round-02.ll
index 2cf009ad5b856..f1a0a2847a303 100644
--- a/llvm/test/CodeGen/SystemZ/fp-round-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-round-02.ll
@@ -134,6 +134,18 @@ define void @f12(ptr %ptr) {
   ret void
 }
 
+; Test trunc for f16.
+declare half @llvm.trunc.f16(half %f)
+define half @f13_half(half %f) {
+; CHECK-LABEL: f13_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: fiebra %f0, 5, %f0, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.trunc.f16(half %f)
+  ret half %res
+}
+
 ; Test trunc for f32.
 declare float @llvm.trunc.f32(float %f)
 define float @f13(float %f) {
@@ -166,6 +178,18 @@ define void @f15(ptr %ptr) {
   ret void
 }
 
+; Test round for f16.
+declare half @llvm.round.f16(half %f)
+define half @f16_half(half %f) {
+; CHECK-LABEL: f16_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: fiebra %f0, 1, %f0, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.round.f16(half %f)
+  ret half %res
+}
+
 ; Test round for f32.
 declare float @llvm.round.f32(float %f)
 define float @f16(float %f) {
@@ -198,6 +222,18 @@ define void @f18(ptr %ptr) {
   ret void
 }
 
+; Test roundeven for f16.
+declare half @llvm.roundeven.f16(half %f)
+define half @f19_half(half %f) {
+; CHECK-LABEL: f19_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: fiebra %f0, 4, %f0, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.roundeven.f16(half %f)
+  ret half %res
+}
+
 ; Test roundeven for f32.
 declare float @llvm.roundeven.f32(float %f)
 define float @f19(float %f) {
diff --git a/llvm/test/CodeGen/SystemZ/fp-round-03.ll b/llvm/test/CodeGen/SystemZ/fp-round-03.ll
index 3cae74749efbe..e0c059661137c 100644
--- a/llvm/test/CodeGen/SystemZ/fp-round-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-round-03.ll
@@ -1,6 +1,19 @@
 ; Test rounding functions for z14 and above.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -verify-machineinstrs \
+; RUN:   | FileCheck %s
+
+; Test that an f16 intrinsic can be lowered with promotion to float.
+declare half @llvm.rint.f16(half %f)
+define half @f0(half %f) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: fiebra %f0, 0, %f0, 0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.rint.f16(half %f)
+  ret half %res
+}
 
 ; Test rint for f32.
 declare float @llvm.rint.f32(float %f)
diff --git a/llvm/test/CodeGen/SystemZ/fp-sqrt-01.ll b/llvm/test/CodeGen/SystemZ/fp-sqrt-01.ll
index 996bdc458b9de..2f7d38339eacd 100644
--- a/llvm/test/CodeGen/SystemZ/fp-sqrt-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-sqrt-01.ll
@@ -4,9 +4,21 @@
 ; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-SCALAR %s
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
+declare half @llvm.sqrt.f16(half)
 declare float @llvm.sqrt.f32(float)
 declare float @sqrtf(float)
 
+; Check register square root.
+define half @f0(half %val) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: sqebr %f0, %f0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.sqrt.f16(half %val)
+  ret half %res
+}
+
 ; Check register square root.
 define float @f1(float %val) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll
index bf9ccbcd70550..dfefc43c02bed 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmp-04.ll
@@ -298,6 +298,43 @@ exit:
   ret float %add
 }
 
+define half @f12_half(half %dummy, half %val) #0 {
+; CHECK-LABEL: f12_half:
+; CHECK:      ler %f9, %f2
+; CHECK-NEXT: ler %f0, %f2
+; CHECK-NEXT: #APP
+; CHECK-NEXT: ler %f8, %f0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: lzer %f0
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: ler %f10, %f0
+; CHECK-NEXT: ler %f0, %f9
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: cebr %f0, %f10
+; CHECK-NEXT: jl .LBB11_2
+; CHECK-NEXT:# %bb.1:        # %store
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT:.LBB11_2:        # %exit
+; CHECK-NEXT: ler %f0, %f8
+; CHECK:      br  %r14
+entry:
+  %ret = call half asm "ler $0, $1", "=f,{f0}"(half %val) #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f16(
+                                               half %val, half 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""() #0
+  br label %exit
+
+exit:
+  ret half %ret
+}
+
 ; Test that LER does not get converted to LTEBR as %f0 is live after it.
 define float @f12(float %dummy, float %val) #0 {
 ; CHECK-LABEL: f12:
@@ -309,7 +346,7 @@ define float @f12(float %dummy, float %val) #0 {
 ; CHECK-NEXT: blr %r14
 ; CHECK: br %r14
 entry:
-  %ret = call float asm "blah $1", "=f,{f0}"(float %val) #0
+  %ret = call float asm "$0 = blah $1", "=f,{f0}"(float %val) #0
   %cmp = call i1 @llvm.experimental.constrained.fcmp.f32(
                                                float %val, float 0.0,
                                                metadata !"olt",
@@ -384,6 +421,43 @@ exit:
   ret void
 }
 
+define half @f15_half(half %val, half %dummy) #0 {
+; CHECK-LABEL: f15_half:
+; CHECK:      ler %f9, %f0
+; CHECK-NEXT: ler %f2, %f0
+; CHECK-NEXT: #APP
+; CHECK-NEXT: ler %f8, %f2
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: lzer %f0
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: ler %f10, %f0
+; CHECK-NEXT: ler %f0, %f9
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: cebr %f0, %f10
+; CHECK-NEXT: jl .LBB15_2
+; CHECK-NEXT:# %bb.1:          # %store
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT:.LBB15_2:         # %exit
+; CHECK-NEXT: ler %f0, %f8
+; CHECK: br %r14
+entry:
+  %ret = call half asm "ler $0, $1", "=f,{f2}"(half %val) #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f16(
+                                               half %val, half 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""() #0
+  br label %exit
+
+exit:
+  ret half %ret
+}
+
 ; Test a case where it is the source rather than destination of LER that
 ; we need, but cannot convert the LER.
 define float @f15(float %val, float %dummy) #0 {
@@ -491,6 +565,43 @@ exit:
   ret float %res
 }
 
+define half @f19_half(half %dummy, half %val) #0 {
+; CHECK-LABEL: f19_half:
+; CHECK:      ler %f9, %f2
+; CHECK-NEXT: ler %f0, %f2
+; CHECK-NEXT: #APP
+; CHECK-NEXT: ler %f8, %f0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: lzer %f0
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: ler %f10, %f0
+; CHECK-NEXT: ler %f0, %f9
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: cebr %f0, %f10
+; CHECK-NEXT: jl .LBB20_2
+; CHECK-NEXT:# %bb.1:           # %store
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT:.LBB20_2:          # %exit
+; CHECK-NEXT: ler %f0, %f8
+; CHECK: br %r14
+entry:
+  %ret = call half asm sideeffect "ler $0, $1", "=f,{f0}"(half %val) #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmp.f16(
+                                               half %val, half 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""() #0
+  br label %exit
+
+exit:
+  ret half %ret
+}
+
 ; Verify that we cannot convert LER to LTEBR and omit the compare if
 ; there may be an intervening change to the exception flags.
 define float @f19(float %dummy, float %val) #0 {
@@ -524,6 +635,7 @@ declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, me
 declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata)
 declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
 declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f16(half, half, metadata, metadata)
 declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
 declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
 declare i1 @llvm.experimental.constrained.fcmp.f128(fp128, fp128, metadata, metadata)
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-01.ll
index ac8894417921c..20efbf60fdbdc 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-01.ll
@@ -8,6 +8,26 @@
 
 declare float @foo()
 
+; Check comparison with registers.
+define i64 @f0(i64 %a, i64 %b, half %f1, half %f2) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: kebr %f0, %f9
+; CHECK-SCALAR-NEXT: je
+; CHECK-SCALAR: lgr %r13, %r12
+; CHECK-SCALAR: lgr %r2, %r13
+; CHECK-VECTOR: locgrne %r12, %r13
+; CHECK-VECTOR: lgr %r2, %r12
+; CHECK: br %r14
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f16(
+                                               half %f1, half %f2,
+                                               metadata !"oeq",
+                                               metadata !"fpexcept.strict") #0
+  %res = select i1 %cond, i64 %a, i64 %b
+  ret i64 %res
+}
+
 ; Check comparison with registers.
 define i64 @f1(i64 %a, i64 %b, float %f1, float %f2) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-04.ll b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-04.ll
index e178769f263e6..ad86df1753192 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-cmps-04.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-cmps-04.ll
@@ -110,6 +110,43 @@ exit:
   ret float %res
 }
 
+define half @f12_half(half %dummy, half %val) #0 {
+; CHECK-LABEL: f12_half:
+; CHECK:      ler %f9, %f2
+; CHECK-NEXT: ler %f0, %f2
+; CHECK-NEXT: #APP
+; CHECK-NEXT: ler %f8, %f0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: lzer %f0
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: ler %f10, %f0
+; CHECK-NEXT: ler %f0, %f9
+; CHECK-NEXT: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: kebr %f0, %f10
+; CHECK-NEXT: jl .LBB4_2
+; CHECK-NEXT:# %bb.1:      # %store
+; CHECK-NEXT: #APP
+; CHECK-NEXT: blah
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT:.LBB4_2:      # %exit
+; CHECK-NEXT: ler %f0, %f8
+; CHECK: br %r14
+entry:
+  %ret = call half asm "ler $0, $1", "=f,{f0}"(half %val) #0
+  %cmp = call i1 @llvm.experimental.constrained.fcmps.f16(
+                                               half %val, half 0.0,
+                                               metadata !"olt",
+                                               metadata !"fpexcept.strict") #0
+  br i1 %cmp, label %exit, label %store
+
+store:
+  call void asm sideeffect "blah", ""() #0
+  br label %exit
+
+exit:
+  ret half %ret
+}
+
 ; Test that LER does not get converted to LTEBR.
 define float @f12(float %dummy, float %val) #0 {
 ; CHECK-LABEL: f12:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-01.ll
index 6b9db1569cf8c..8df7ef5f3d7c8 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-01.ll
@@ -5,13 +5,29 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 \
 ; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
+
+declare half @llvm.experimental.constrained.fptrunc.f16.f64(double, metadata, metadata)
+
 declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+declare half @llvm.experimental.constrained.fptrunc.f16.f128(fp128, metadata, metadata)
 declare float @llvm.experimental.constrained.fptrunc.f32.f128(fp128, metadata, metadata)
 declare double @llvm.experimental.constrained.fptrunc.f64.f128(fp128, metadata, metadata)
 
 declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
 declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata)
 
+; Test f64->f16.
+define half @f0(double %d1, double %d2) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __truncdfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.fptrunc.f16.f64(
+                                               double %d2,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test f64->f32.
 define float @f1(double %d1, double %d2) #0 {
 ; CHECK-LABEL: f1:
@@ -25,6 +41,19 @@ define float @f1(double %d1, double %d2) #0 {
   ret float %res
 }
 
+; Test f128->f16.
+define half @f2_half(ptr %ptr) #0 {
+; CHECK-LABEL: f2_half:
+; CHECK: brasl %r14, __trunctfhf2@PLT
+; CHECK: br %r14
+  %val = load fp128, ptr %ptr
+  %res = call half @llvm.experimental.constrained.fptrunc.f16.f128(
+                                               fp128 %val,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test f128->f32.
 define float @f2(ptr %ptr) #0 {
 ; CHECK-LABEL: f2:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-02.ll
index c79f51dd1ae9e..725d53cabb937 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-02.ll
@@ -2,8 +2,19 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare double @llvm.experimental.constrained.fpext.f64.f16(half, metadata)
 declare double @llvm.experimental.constrained.fpext.f64.f32(float, metadata)
 
+; Check register extension.
+define double @f0(half %val) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfdf2@PLT
+; CHECK: br %r14
+  %res = call double @llvm.experimental.constrained.fpext.f64.f16(half %val,
+                                               metadata !"fpexcept.strict") #0
+  ret double %res
+}
+
 ; Check register extension.
 define double @f1(float %val) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-05.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-05.ll
index f2a66098d32e2..96f764fdab47a 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-05.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-05.ll
@@ -2,10 +2,23 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare half @llvm.experimental.constrained.sitofp.f16.i32(i32, metadata, metadata)
 declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
 declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32, metadata, metadata)
 
+; Check i32->f16.
+define half @f0(i32 %i) #0 {
+; CHECK-LABEL: f0:
+; CHECK: cefbr %f0, %r2
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %conv
+}
+
 ; Check i32->f32.
 define float @f1(i32 %i) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-06.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-06.ll
index e23eaf3ab359a..2bd8556edd664 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-06.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-06.ll
@@ -2,10 +2,25 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
 
+declare half @llvm.experimental.constrained.uitofp.f16.i32(i32, metadata, metadata)
 declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32, metadata, metadata)
 
+; Check i32->f16.  There is no native instruction, so we must promote
+; to i64 first.
+define half @f0(i32 %i) #0 {
+; CHECK-LABEL: f0:
+; CHECK: llgfr [[REGISTER:%r[0-5]]], %r2
+; CHECK: cegbr %f0, [[REGISTER]]
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = call half @llvm.experimental.constrained.uitofp.f16.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %conv
+}
+
 ; Check i32->f32.  There is no native instruction, so we must promote
 ; to i64 first.
 define float @f1(i32 %i) #0 {
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-07.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-07.ll
index d18aa38966009..d2a568ed19a4e 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-07.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-07.ll
@@ -2,10 +2,23 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare half @llvm.experimental.constrained.sitofp.f16.i64(i64, metadata, metadata)
 declare float @llvm.experimental.constrained.sitofp.f32.i64(i64, metadata, metadata)
 declare double @llvm.experimental.constrained.sitofp.f64.i64(i64, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64, metadata, metadata)
 
+; Test i64->f16.
+define half @f0(i64 %i) #0 {
+; CHECK-LABEL: f0:
+; CHECK: cegbr %f0, %r2
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = call half @llvm.experimental.constrained.sitofp.f16.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %conv
+}
+
 ; Test i64->f32.
 define float @f1(i64 %i) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-08.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-08.ll
index 2cbcf2a2ef0a3..c79f884ac4aeb 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-08.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-08.ll
@@ -2,10 +2,24 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s
 
+declare half @llvm.experimental.constrained.uitofp.f16.i64(i64, metadata, metadata)
 declare float @llvm.experimental.constrained.uitofp.f32.i64(i64, metadata, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64, metadata, metadata)
 
+; Test i64->f16. For z10, this results in just a single a libcall.
+define half @f0(i64 %i) #0 {
+; CHECK-LABEL: f0:
+; CHECK: cegbr
+; CHECK: aebr
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = call half @llvm.experimental.constrained.uitofp.f16.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %conv
+}
+
 ; Test i64->f32.  There's no native support for unsigned i64-to-fp conversions,
 ; but we should be able to implement them using signed i64-to-fp conversions.
 define float @f1(i64 %i) #0 {
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-09.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-09.ll
index a54055120f727..40da726b7a46a 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-09.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-09.ll
@@ -2,10 +2,22 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare i32 @llvm.experimental.constrained.fptosi.i32.f16(half, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.f32(float, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128, metadata)
 
+; Test f16->i32.
+define i32 @f0(half %f) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: cfebr %r2, 5, %f0
+; CHECK: br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
 ; Test f32->i32.
 define i32 @f1(float %f) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
index 7cbcfeea8cf60..d2206a40169e5 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
@@ -9,25 +9,45 @@
 ; outside the i32 range but in the i64 range, so use the default expansion.
 ; Note that the strict expansion sequence must be used.
 
+declare i32 @llvm.experimental.constrained.fptoui.i32.f16(half, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128, metadata)
 
+; Test f16->i32. Converted to signed as the max float value is smaller than
+; the signed integer range.
+define i32 @f0(half %f) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    cfebr %r2, 5, %f0
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f16(half %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
 ; Test f32->i32.
 define i32 @f1(float %f) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    larl %r1, .LCPI0_0
+; CHECK-NEXT:    larl %r1, .LCPI1_0
 ; CHECK-NEXT:    le %f1, 0(%r1)
 ; CHECK-NEXT:    kebr %f0, %f1
-; CHECK-NEXT:    jnl .LBB0_2
+; CHECK-NEXT:    jnl .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lhi %r0, 0
 ; CHECK-NEXT:    lzer %f1
-; CHECK-NEXT:    j .LBB0_3
-; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    j .LBB1_3
+; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    llilh %r0, 32768
-; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:  .LBB1_3:
 ; CHECK-NEXT:    sebr %f0, %f1
 ; CHECK-NEXT:    cfebr %r2, 5, %f0
 ; CHECK-NEXT:    xr %r2, %r0
@@ -41,17 +61,17 @@ define i32 @f1(float %f) #0 {
 define i32 @f2(double %f) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    larl %r1, .LCPI1_0
+; CHECK-NEXT:    larl %r1, .LCPI2_0
 ; CHECK-NEXT:    ld %f1, 0(%r1)
 ; CHECK-NEXT:    kdbr %f0, %f1
-; CHECK-NEXT:    jnl .LBB1_2
+; CHECK-NEXT:    jnl .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lhi %r0, 0
 ; CHECK-NEXT:    lzdr %f1
-; CHECK-NEXT:    j .LBB1_3
-; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    j .LBB2_3
+; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    llilh %r0, 32768
-; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:  .LBB2_3:
 ; CHECK-NEXT:    sdbr %f0, %f1
 ; CHECK-NEXT:    cfdbr %r2, 5, %f0
 ; CHECK-NEXT:    xr %r2, %r0
@@ -67,17 +87,17 @@ define i32 @f3(ptr %src) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ld %f0, 0(%r2)
 ; CHECK-NEXT:    ld %f2, 8(%r2)
-; CHECK-NEXT:    larl %r1, .LCPI2_0
+; CHECK-NEXT:    larl %r1, .LCPI3_0
 ; CHECK-NEXT:    lxeb %f1, 0(%r1)
 ; CHECK-NEXT:    kxbr %f0, %f1
-; CHECK-NEXT:    jnl .LBB2_2
+; CHECK-NEXT:    jnl .LBB3_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lhi %r0, 0
 ; CHECK-NEXT:    lzxr %f1
-; CHECK-NEXT:    j .LBB2_3
-; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    j .LBB3_3
+; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    llilh %r0, 32768
-; CHECK-NEXT:  .LBB2_3:
+; CHECK-NEXT:  .LBB3_3:
 ; CHECK-NEXT:    sxbr %f0, %f1
 ; CHECK-NEXT:    cfxbr %r2, 5, %f0
 ; CHECK-NEXT:    xr %r2, %r0
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-11.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-11.ll
index 27af314cff01b..dd8a708599629 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-11.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-11.ll
@@ -2,10 +2,22 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare i64 @llvm.experimental.constrained.fptosi.i64.f16(half, metadata)
 declare i64 @llvm.experimental.constrained.fptosi.i64.f32(float, metadata)
 declare i64 @llvm.experimental.constrained.fptosi.i64.f64(double, metadata)
 declare i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128, metadata)
 
+; Test f16->i64.
+define i64 @f0(half %f) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: cgebr %r2, 5, %f0
+; CHECK: br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptosi.i64.f16(half %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
 ; Test f32->i64.
 define i64 @f1(float %f) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
index 69bbd82e29898..76c7188641724 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
@@ -8,25 +8,45 @@
 ; Convert via signed i64s instead.
 ; Note that the strict expansion sequence must be used.
 
+declare i64 @llvm.experimental.constrained.fptoui.i64.f16(half, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.f32(float, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128, metadata)
 
+; Test f16->i64. Converted to signed as the max float value is smaller than
+; the signed integer range.
+define i64 @f0(half %f) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    cgebr %r2, 5, %f0
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f16(half %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
 ; Test f32->i64.
 define i64 @f1(float %f) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    larl %r1, .LCPI0_0
+; CHECK-NEXT:    larl %r1, .LCPI1_0
 ; CHECK-NEXT:    le %f1, 0(%r1)
 ; CHECK-NEXT:    kebr %f0, %f1
-; CHECK-NEXT:    jnl .LBB0_2
+; CHECK-NEXT:    jnl .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lghi %r0, 0
 ; CHECK-NEXT:    lzer %f1
-; CHECK-NEXT:    j .LBB0_3
-; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    j .LBB1_3
+; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    llihh %r0, 32768
-; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:  .LBB1_3:
 ; CHECK-NEXT:    sebr %f0, %f1
 ; CHECK-NEXT:    cgebr %r2, 5, %f0
 ; CHECK-NEXT:    xgr %r2, %r0
@@ -40,17 +60,17 @@ define i64 @f1(float %f) #0 {
 define i64 @f2(double %f) #0 {
 ; CHECK-LABEL: f2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    larl %r1, .LCPI1_0
+; CHECK-NEXT:    larl %r1, .LCPI2_0
 ; CHECK-NEXT:    ld %f1, 0(%r1)
 ; CHECK-NEXT:    kdbr %f0, %f1
-; CHECK-NEXT:    jnl .LBB1_2
+; CHECK-NEXT:    jnl .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lghi %r0, 0
 ; CHECK-NEXT:    lzdr %f1
-; CHECK-NEXT:    j .LBB1_3
-; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    j .LBB2_3
+; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    llihh %r0, 32768
-; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:  .LBB2_3:
 ; CHECK-NEXT:    sdbr %f0, %f1
 ; CHECK-NEXT:    cgdbr %r2, 5, %f0
 ; CHECK-NEXT:    xgr %r2, %r0
@@ -66,17 +86,17 @@ define i64 @f3(ptr %src) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ld %f0, 0(%r2)
 ; CHECK-NEXT:    ld %f2, 8(%r2)
-; CHECK-NEXT:    larl %r1, .LCPI2_0
+; CHECK-NEXT:    larl %r1, .LCPI3_0
 ; CHECK-NEXT:    lxeb %f1, 0(%r1)
 ; CHECK-NEXT:    kxbr %f0, %f1
-; CHECK-NEXT:    jnl .LBB2_2
+; CHECK-NEXT:    jnl .LBB3_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lghi %r0, 0
 ; CHECK-NEXT:    lzxr %f1
-; CHECK-NEXT:    j .LBB2_3
-; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    j .LBB3_3
+; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    llihh %r0, 32768
-; CHECK-NEXT:  .LBB2_3:
+; CHECK-NEXT:  .LBB3_3:
 ; CHECK-NEXT:    sxbr %f0, %f1
 ; CHECK-NEXT:    cgxbr %r2, 5, %f0
 ; CHECK-NEXT:    xgr %r2, %r0
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-13.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-13.ll
index 41913106f5340..2b1c47d0d91e4 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-13.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-13.ll
@@ -3,14 +3,28 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
 
+declare half @llvm.experimental.constrained.uitofp.f16.i32(i32, metadata, metadata)
 declare float @llvm.experimental.constrained.uitofp.f32.i32(i32, metadata, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i32(i32, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32, metadata, metadata)
 
+declare half @llvm.experimental.constrained.uitofp.f16.i64(i64, metadata, metadata)
 declare float @llvm.experimental.constrained.uitofp.f32.i64(i64, metadata, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i64(i64, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64, metadata, metadata)
 
+; Check i32->f16.
+define half @f0(i32 %i) #0 {
+; CHECK-LABEL: f0:
+; CHECK: celfbr %f0, 0, %r2, 0
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = call half @llvm.experimental.constrained.uitofp.f16.i32(i32 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %conv
+}
+
 ; Check i32->f32.
 define float @f1(i32 %i) #0 {
 ; CHECK-LABEL: f1:
@@ -47,10 +61,22 @@ define void @f3(i32 %i, ptr %dst) #0 {
   ret void
 }
 
-; Check i64->f32.
-define float @f4(i64 %i) #0 {
+; Check i64->f16.
+define half @f4(i64 %i) #0 {
 ; CHECK-LABEL: f4:
 ; CHECK: celgbr %f0, 0, %r2, 0
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = call half @llvm.experimental.constrained.uitofp.f16.i64(i64 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %conv
+}
+
+; Check i64->f32.
+define float @f5(i64 %i) #0 {
+; CHECK-LABEL: f5:
+; CHECK: celgbr %f0, 0, %r2, 0
 ; CHECK: br %r14
   %conv = call float @llvm.experimental.constrained.uitofp.f32.i64(i64 %i,
                                                metadata !"round.dynamic",
@@ -59,8 +85,8 @@ define float @f4(i64 %i) #0 {
 }
 
 ; Check i64->f64.
-define double @f5(i64 %i) #0 {
-; CHECK-LABEL: f5:
+define double @f6(i64 %i) #0 {
+; CHECK-LABEL: f6:
 ; CHECK: cdlgbr %f0, 0, %r2, 0
 ; CHECK: br %r14
   %conv = call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %i,
@@ -70,8 +96,8 @@ define double @f5(i64 %i) #0 {
 }
 
 ; Check i64->f128.
-define void @f6(i64 %i, ptr %dst) #0 {
-; CHECK-LABEL: f6:
+define void @f7(i64 %i, ptr %dst) #0 {
+; CHECK-LABEL: f7:
 ; CHECK: cxlgbr %f0, 0, %r2, 0
 ; CHECK-DAG: std %f0, 0(%r3)
 ; CHECK-DAG: std %f2, 8(%r3)
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-14.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-14.ll
index aa82a1d91c4a8..1d3387f610d72 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-14.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-14.ll
@@ -2,14 +2,27 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s
 
+declare i32 @llvm.experimental.constrained.fptoui.i32.f16(half, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.f64(double, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128, metadata)
 
+declare i64 @llvm.experimental.constrained.fptoui.i64.f16(half, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.f32(float, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128, metadata)
 
+; Test f16->i32.
+define i32 @f0(half %f) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: clfebr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %conv = call i32 @llvm.experimental.constrained.fptoui.i32.f16(half %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i32 %conv
+}
+
 ; Test f32->i32.
 define i32 @f1(float %f) #0 {
 ; CHECK-LABEL: f1:
@@ -43,9 +56,20 @@ define i32 @f3(ptr %src) #0 {
   ret i32 %conv
 }
 
-; Test f32->i64.
-define i64 @f4(float %f) #0 {
+; Test f16->i64.
+define i64 @f4(half %f) #0 {
 ; CHECK-LABEL: f4:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: clgebr %r2, 5, %f0, 0
+; CHECK: br %r14
+  %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f16(half %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i64 %conv
+}
+
+; Test f32->i64.
+define i64 @f5(float %f) #0 {
+; CHECK-LABEL: f5:
 ; CHECK: clgebr %r2, 5, %f0, 0
 ; CHECK: br %r14
   %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f32(float %f,
@@ -54,8 +78,8 @@ define i64 @f4(float %f) #0 {
 }
 
 ; Test f64->i64.
-define i64 @f5(double %f) #0 {
-; CHECK-LABEL: f5:
+define i64 @f6(double %f) #0 {
+; CHECK-LABEL: f6:
 ; CHECK: clgdbr %r2, 5, %f0, 0
 ; CHECK: br %r14
   %conv = call i64 @llvm.experimental.constrained.fptoui.i64.f64(double %f,
@@ -64,8 +88,8 @@ define i64 @f5(double %f) #0 {
 }
 
 ; Test f128->i64.
-define i64 @f6(ptr %src) #0 {
-; CHECK-LABEL: f6:
+define i64 @f7(ptr %src) #0 {
+; CHECK-LABEL: f7:
 ; CHECK-DAG: ld %f0, 0(%r2)
 ; CHECK-DAG: ld %f2, 8(%r2)
 ; CHECK: clgxbr %r2, 5, %f0, 0
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-15.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-15.ll
index de93192b5f305..a53a3537a7390 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-15.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-15.ll
@@ -2,9 +2,11 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
+declare half @llvm.experimental.constrained.fptrunc.f16.f128(fp128, metadata, metadata)
 declare float @llvm.experimental.constrained.fptrunc.f32.f128(fp128, metadata, metadata)
 declare double @llvm.experimental.constrained.fptrunc.f64.f128(fp128, metadata, metadata)
 
+declare fp128 @llvm.experimental.constrained.fpext.f128.f16(half, metadata)
 declare fp128 @llvm.experimental.constrained.fpext.f128.f32(float, metadata)
 declare fp128 @llvm.experimental.constrained.fpext.f128.f64(double, metadata)
 
@@ -22,6 +24,21 @@ define double @f1(ptr %ptr) #0 {
   ret double %res
 }
 
+; Test f128->f16.
+define half @f2_half(ptr %ptr) #0 {
+; CHECK-LABEL: f2_half:
+; CHECK: vl [[REG:%v[0-9]+]], 0(%r2)
+; CHECK: vst %v0, 160(%r15), 3
+; CHECK: brasl %r14, __trunctfhf2@PLT
+; CHECK: br %r14
+  %val = load fp128, ptr %ptr
+  %res = call half @llvm.experimental.constrained.fptrunc.f16.f128(
+                                               fp128 %val,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test f128->f32.
 define float @f2(ptr %ptr) #0 {
 ; CHECK-LABEL: f2:
@@ -62,4 +79,15 @@ define void @f4(ptr %dst, float %val) #0 {
   ret void
 }
 
+; Test f16->f128.
+define void @f5(ptr %dst, half %val) #0 {
+; CHECK-LABEL: f5:
+; CHECK: brasl %r14, __extendhftf2@PLT
+; CHECK: br %r14
+  %res = call fp128 @llvm.experimental.constrained.fpext.f128.f16(half %val,
+                                               metadata !"fpexcept.strict") #0
+  store fp128 %res, ptr %dst
+  ret void
+}
+
 attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-17.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-17.ll
index 3ff63242a6d82..2becd18277e2a 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-17.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-17.ll
@@ -6,18 +6,22 @@
 declare fp128 @llvm.experimental.constrained.sitofp.f128.i128(i128, metadata, metadata)
 declare double @llvm.experimental.constrained.sitofp.f64.i128(i128, metadata, metadata)
 declare float @llvm.experimental.constrained.sitofp.f32.i128(i128, metadata, metadata)
+declare half @llvm.experimental.constrained.sitofp.f16.i128(i128, metadata, metadata)
 
 declare fp128 @llvm.experimental.constrained.uitofp.f128.i128(i128, metadata, metadata)
 declare double @llvm.experimental.constrained.uitofp.f64.i128(i128, metadata, metadata)
 declare float @llvm.experimental.constrained.uitofp.f32.i128(i128, metadata, metadata)
+declare half @llvm.experimental.constrained.uitofp.f16.i128(i128, metadata, metadata)
 
 declare i128 @llvm.experimental.constrained.fptosi.i128.f128(fp128, metadata)
 declare i128 @llvm.experimental.constrained.fptosi.i128.f64(double, metadata)
 declare i128 @llvm.experimental.constrained.fptosi.i128.f32(float, metadata)
+declare i128 @llvm.experimental.constrained.fptosi.i128.f16(half, metadata)
 
 declare i128 @llvm.experimental.constrained.fptoui.i128.f128(fp128, metadata)
 declare i128 @llvm.experimental.constrained.fptoui.i128.f64(double, metadata)
 declare i128 @llvm.experimental.constrained.fptoui.i128.f32(float, metadata)
+declare i128 @llvm.experimental.constrained.fptoui.i128.f16(half, metadata)
 
 ; Test signed i128->f128.
 define fp128 @f1(i128 %i) #0 {
@@ -52,9 +56,21 @@ define float @f3(i128 %i) #0 {
   ret float %conv
 }
 
-; Test unsigned i128->f128.
-define fp128 @f4(i128 %i) #0 {
+; Test signed i128->f16.
+define half @f4(i128 %i) #0 {
 ; CHECK-LABEL: f4:
+; CHECK: %r14, __floattisf@PLT
+; CHECK: %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = call half @llvm.experimental.constrained.sitofp.f16.i128(i128 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %conv
+}
+
+; Test unsigned i128->f128.
+define fp128 @f5(i128 %i) #0 {
+; CHECK-LABEL: f5:
 ; CHECK: brasl %r14, __floatuntitf@PLT
 ; CHECK: br %r14
   %conv = call fp128 @llvm.experimental.constrained.uitofp.f128.i128(i128 %i,
@@ -64,8 +80,8 @@ define fp128 @f4(i128 %i) #0 {
 }
 
 ; Test unsigned i128->f64.
-define double @f5(i128 %i) #0 {
-; CHECK-LABEL: f5:
+define double @f6(i128 %i) #0 {
+; CHECK-LABEL: f6:
 ; CHECK: brasl %r14, __floatuntidf@PLT
 ; CHECK: br %r14
   %conv = call double @llvm.experimental.constrained.uitofp.f64.i128(i128 %i,
@@ -75,8 +91,8 @@ define double @f5(i128 %i) #0 {
 }
 
 ; Test unsigned i128->f32.
-define float @f6(i128 %i) #0 {
-; CHECK-LABEL: f6:
+define float @f7(i128 %i) #0 {
+; CHECK-LABEL: f7:
 ; CHECK: brasl %r14, __floatuntisf@PLT
 ; CHECK: br %r14
   %conv = call float @llvm.experimental.constrained.uitofp.f32.i128(i128 %i,
@@ -85,9 +101,21 @@ define float @f6(i128 %i) #0 {
   ret float %conv
 }
 
+; Test unsigned i128->f16.
+define half @f8(i128 %i) #0 {
+; CHECK-LABEL: f8:
+; CHECK: brasl %r14, __floatuntisf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %conv = call half @llvm.experimental.constrained.uitofp.f16.i128(i128 %i,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret half %conv
+}
+
 ; Test signed f128->i128.
-define i128 @f7(fp128 %f) #0 {
-; CHECK-LABEL: f7:
+define i128 @f9(fp128 %f) #0 {
+; CHECK-LABEL: f9:
 ; CHECK: brasl %r14, __fixtfti@PLT
 ; CHECK: br %r14
   %conv = call i128 @llvm.experimental.constrained.fptosi.i128.f128(fp128 %f,
@@ -96,8 +124,8 @@ define i128 @f7(fp128 %f) #0 {
 }
 
 ; Test signed f64->i128.
-define i128 @f8(double %f) #0 {
-; CHECK-LABEL: f8:
+define i128 @f10(double %f) #0 {
+; CHECK-LABEL: f10:
 ; CHECK: brasl %r14, __fixdfti@PLT
 ; CHECK: br %r14
   %conv = call i128 @llvm.experimental.constrained.fptosi.i128.f64(double %f,
@@ -105,9 +133,9 @@ define i128 @f8(double %f) #0 {
   ret i128 %conv
 }
 
-; Test signed f9->i128.
-define i128 @f9(float %f) #0 {
-; CHECK-LABEL: f9:
+; Test signed f32->i128.
+define i128 @f11(float %f) #0 {
+; CHECK-LABEL: f11:
 ; CHECK: brasl %r14, __fixsfti@PLT
 ; CHECK: br %r14
   %conv = call i128 @llvm.experimental.constrained.fptosi.i128.f32(float %f,
@@ -115,9 +143,20 @@ define i128 @f9(float %f) #0 {
   ret i128 %conv
 }
 
+; Test signed f16->i128.
+define i128 @f12(half %f) #0 {
+; CHECK-LABEL: f12:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __fixsfti@PLT
+; CHECK: br %r14
+  %conv = call i128 @llvm.experimental.constrained.fptosi.i128.f16(half %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i128 %conv
+}
+
 ; Test unsigned f128->i128.
-define i128 @f10(fp128 %f) #0 {
-; CHECK-LABEL: f10:
+define i128 @f13(fp128 %f) #0 {
+; CHECK-LABEL: f13:
 ; CHECK: brasl %r14, __fixunstfti@PLT
 ; CHECK: br %r14
   %conv = call i128 @llvm.experimental.constrained.fptoui.i128.f128(fp128 %f,
@@ -126,8 +165,8 @@ define i128 @f10(fp128 %f) #0 {
 }
 
 ; Test unsigned f64->i128.
-define i128 @f11(double %f) #0 {
-; CHECK-LABEL: f11:
+define i128 @f14(double %f) #0 {
+; CHECK-LABEL: f14:
 ; CHECK: brasl %r14, __fixunsdfti@PLT
 ; CHECK: br %r14
   %conv = call i128 @llvm.experimental.constrained.fptoui.i128.f64(double %f,
@@ -136,8 +175,8 @@ define i128 @f11(double %f) #0 {
 }
 
 ; Test unsigned f32->i128.
-define i128 @f12(float %f) #0 {
-; CHECK-LABEL: f12:
+define i128 @f15(float %f) #0 {
+; CHECK-LABEL: f15:
 ; CHECK: brasl %r14, __fixunssfti@PLT
 ; CHECK: br %r14
   %conv = call i128 @llvm.experimental.constrained.fptoui.i128.f32(float %f,
@@ -145,4 +184,15 @@ define i128 @f12(float %f) #0 {
   ret i128 %conv
 }
 
+; Test unsigned f16->i128.
+define i128 @f16(half %f) #0 {
+; CHECK-LABEL: f16:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __fixunssfti@PLT
+; CHECK: br %r14
+  %conv = call i128 @llvm.experimental.constrained.fptoui.i128.f16(half %f,
+                                               metadata !"fpexcept.strict") #0
+  ret i128 %conv
+}
+
 attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll
index 980df79481936..8b9dbbe9c9e6e 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-div-01.ll
@@ -5,8 +5,24 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
 declare float @foo()
+declare half @llvm.experimental.constrained.fdiv.f16(half, half, metadata, metadata)
 declare float @llvm.experimental.constrained.fdiv.f32(float, float, metadata, metadata)
 
+; Check register division.
+define half @f0(half %f1, half %f2) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: debr %f0, %f9
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.fdiv.f16(
+                        half %f1, half %f2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Check register division.
 define float @f1(float %f1, float %f2) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll b/llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll
index 4971375789407..c951c79aeb7c6 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-mul-06.ll
@@ -3,8 +3,26 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 \
 ; RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK-VECTOR %s
 
+declare half @llvm.experimental.constrained.fma.f16(half, half, half, metadata, metadata)
 declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
 
+define half @f0(half %f1, half %f2, half %acc) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK-SCALAR: maebr %f10, %f0, %f8
+; CHECK-SCALAR: ler %f0, %f10
+; CHECK-VECTOR: wfmasb %f0, %f0, %f8, %f10
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.fma.f16 (
+                        half %f1, half %f2, half %acc,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 define float @f1(float %f1, float %f2, float %acc) #0 {
 ; CHECK-LABEL: f1:
 ; CHECK-SCALAR: maebr %f4, %f0, %f2
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-round-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-round-01.ll
index 964f16d605db6..95a5fa1af832b 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-round-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-round-01.ll
@@ -43,6 +43,21 @@ define void @f3(ptr %ptr) #0 {
   ret void
 }
 
+; Test nearbyint for f16.
+declare half @llvm.experimental.constrained.nearbyint.f16(half, metadata, metadata)
+define half @f4_half(half %f) #0 {
+; CHECK-LABEL: f4_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, nearbyintf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.nearbyint.f16(
+                        half %f,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test nearbyint for f32.
 declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
 define float @f4(float %f) #0 {
@@ -84,6 +99,20 @@ define void @f6(ptr %ptr) #0 {
   ret void
 }
 
+; Test floor for f16.
+declare half @llvm.experimental.constrained.floor.f16(half, metadata)
+define half @f7_half(half %f) #0 {
+; CHECK-LABEL: f7_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, floorf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.floor.f16(
+                        half %f,
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test floor for f32.
 declare float @llvm.experimental.constrained.floor.f32(float, metadata)
 define float @f7(float %f) #0 {
@@ -122,6 +151,20 @@ define void @f9(ptr %ptr) #0 {
   ret void
 }
 
+; Test ceil for f16.
+declare half @llvm.experimental.constrained.ceil.f16(half, metadata)
+define half @f10_half(half %f) #0 {
+; CHECK-LABEL: f10_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, ceilf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.ceil.f16(
+                        half %f,
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test ceil for f32.
 declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
 define float @f10(float %f) #0 {
@@ -160,6 +203,20 @@ define void @f12(ptr %ptr) #0 {
   ret void
 }
 
+; Test trunc for f16.
+declare half @llvm.experimental.constrained.trunc.f16(half, metadata)
+define half @f13_half(half %f) #0 {
+; CHECK-LABEL: f13_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, truncf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.trunc.f16(
+                        half %f,
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test trunc for f32.
 declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
 define float @f13(float %f) #0 {
@@ -198,6 +255,20 @@ define void @f15(ptr %ptr) #0 {
   ret void
 }
 
+; Test round for f16.
+declare half @llvm.experimental.constrained.round.f16(half, metadata)
+define half @f16_half(half %f) #0 {
+; CHECK-LABEL: f16_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, roundf@PLT
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.round.f16(
+                        half %f,
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test round for f32.
 declare float @llvm.experimental.constrained.round.f32(float, metadata)
 define float @f16(float %f) #0 {
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-round-02.ll b/llvm/test/CodeGen/SystemZ/fp-strict-round-02.ll
index c7b721e3770e5..bdfd9adf2b400 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-round-02.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-round-02.ll
@@ -164,6 +164,20 @@ define void @f12(ptr %ptr) #0 {
   ret void
 }
 
+; Test trunc for f16.
+declare half @llvm.experimental.constrained.trunc.f16(half, metadata)
+define half @f13_half(half %f) #0 {
+; CHECK-LABEL: f13_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: fiebra %f0, 5, %f0, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.trunc.f16(
+                        half %f,
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test trunc for f32.
 declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
 define float @f13(float %f) #0 {
@@ -202,6 +216,20 @@ define void @f15(ptr %ptr) #0 {
   ret void
 }
 
+; Test round for f16.
+declare half @llvm.experimental.constrained.round.f16(half, metadata)
+define half @f16_half(half %f) #0 {
+; CHECK-LABEL: f16_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: fiebra %f0, 1, %f0, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.round.f16(
+                        half %f,
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test round for f32.
 declare float @llvm.experimental.constrained.round.f32(float, metadata)
 define float @f16(float %f) #0 {
@@ -240,6 +268,20 @@ define void @f18(ptr %ptr) #0 {
   ret void
 }
 
+; Test roundeven for f16.
+declare half @llvm.experimental.constrained.roundeven.f16(half, metadata)
+define half @f19_half(half %f) #0 {
+; CHECK-LABEL: f19_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: fiebra %f0, 4, %f0, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.roundeven.f16(
+                        half %f,
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test roundeven for f32.
 declare float @llvm.experimental.constrained.roundeven.f32(float, metadata)
 define float @f19(float %f) #0 {
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-round-03.ll b/llvm/test/CodeGen/SystemZ/fp-strict-round-03.ll
index e99d8b0f01650..5d9ee28ae8ea2 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-round-03.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-round-03.ll
@@ -2,6 +2,21 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
+; Test rint for f16.
+declare half @llvm.experimental.constrained.rint.f16(half, metadata, metadata)
+define half @f0(half %f) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: fiebra %f0, 0, %f0, 0
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.rint.f16(
+                        half %f,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Test rint for f32.
 declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
 define float @f1(float %f) #0 {
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-01.ll
index 88cdb71ff7d12..2db86d2de7f66 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sqrt-01.ll
@@ -6,8 +6,24 @@
 ; Test strict 32-bit square root.
 ;
 
+declare half @llvm.experimental.constrained.sqrt.f16(half, metadata, metadata)
 declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
 
+; Check register square root.
+define half @f0(half %val) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       # %bb.0:
+; CHECK:      brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT: sqebr %f0, %f0
+; CHECK-NEXT: brasl %r14, __truncsfhf2@PLT
+; CHECK:      br %r14
+  %res = call half @llvm.experimental.constrained.sqrt.f16(
+                        half %val,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Check register square root.
 define float @f1(float %val) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll b/llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll
index a677d471397f7..da91b6e69fd5f 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-sub-01.ll
@@ -5,8 +5,24 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
 declare float @foo()
+declare half @llvm.experimental.constrained.fsub.f16(half, half, metadata, metadata)
 declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata)
 
+; Check register subtraction.
+define half @f0(half %f1, half %f2) #0 {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: sebr %f0, %f9
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = call half @llvm.experimental.constrained.fsub.f16(
+                        half %f1, half %f2,
+                        metadata !"round.dynamic",
+                        metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 ; Check register subtraction.
 define float @f1(float %f1, float %f2) #0 {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/fp-sub-01.ll b/llvm/test/CodeGen/SystemZ/fp-sub-01.ll
index e875fa3be735b..7359f10f92852 100644
--- a/llvm/test/CodeGen/SystemZ/fp-sub-01.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-sub-01.ll
@@ -6,6 +6,18 @@
 
 declare float @foo()
 
+; Check register subtraction.
+define half @f0(half %f1, half %f2) {
+; CHECK-LABEL: f0:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: sebr %f0, %f9
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %res = fsub half %f1, %f2
+  ret half %res
+}
+
 ; Check register subtraction.
 define float @f1(float %f1, float %f2) {
 ; CHECK-LABEL: f1:
diff --git a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs-zEC12.ll b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs-zEC12.ll
index 44175f924f7fc..6228ffaa35fa2 100644
--- a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs-zEC12.ll
+++ b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs-zEC12.ll
@@ -3,6 +3,23 @@
 ;
 ; Test inline assembly where the operand is bitcasted.
 
+define signext i16 @short_and_f(i16 signext %cc_dep1) {
+; CHECK-LABEL: short_and_f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sllg %r0, %r2, 48
+; CHECK-NEXT:    ldgr %f1, %r0
+; CHECK-NEXT:    # kill: def $f1h killed $f1h killed $f1d
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    # kill: def $f1h killed $f1h def $f1d
+; CHECK-NEXT:    lgdr %r0, %f1
+; CHECK-NEXT:    srag %r2, %r0, 48
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call i16 asm sideeffect "", "={f1},0"(i16 %cc_dep1)
+  ret i16 %0
+}
+
 define signext i32 @int_and_f(i32 signext %cc_dep1) {
 ; CHECK-LABEL: int_and_f:
 ; CHECK:       # %bb.0: # %entry
@@ -51,6 +68,23 @@ entry:
   ret void
 }
 
+define half @half_and_r(half %cc_dep1) {
+; CHECK-LABEL: half_and_r:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; CHECK-NEXT:    lgdr %r0, %f0
+; CHECK-NEXT:    srlg %r2, %r0, 48
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    sllg %r0, %r2, 48
+; CHECK-NEXT:    ldgr %f0, %r0
+; CHECK-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call half asm sideeffect "", "={r2},0"(half %cc_dep1)
+  ret half %0
+}
+
 define float @float_and_r(float %cc_dep1) {
 ; CHECK-LABEL: float_and_r:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll
index 3cbf3d21dec5a..cf4dbbff8bec0 100644
--- a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll
+++ b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll
@@ -4,6 +4,20 @@
 ;
 ; Test inline assembly where the operand is bitcasted.
 
+define signext i16 @short_and_f(i16 signext %cc_dep1) {
+; CHECK-LABEL: short_and_f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vlvgh %v0, %r2, 0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vlgvh %r0, %v0, 0
+; CHECK-NEXT:    lghr %r2, %r0
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call i16 asm sideeffect "", "={f0},0"(i16 %cc_dep1)
+  ret i16 %0
+}
+
 define signext i32 @int_and_f(i32 signext %cc_dep1) {
 ; CHECK-LABEL: int_and_f:
 ; CHECK:       # %bb.0: # %entry
@@ -101,6 +115,19 @@ entry:
   ret void
 }
 
+define half @half_and_r(half %cc_dep1) {
+; CHECK-LABEL: half_and_r:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vlgvh %r0, %v0, 0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vlvgh %v0, %r0, 0
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call half asm sideeffect "", "={r0},0"(half %cc_dep1)
+  ret half %0
+}
+
 define float @float_and_r(float %cc_dep1) {
 ; CHECK-LABEL: float_and_r:
 ; CHECK:       # %bb.0: # %entry
@@ -145,6 +172,19 @@ entry:
   ret void
 }
 
+define half @half_and_v(half %cc_dep1) {
+; CHECK-LABEL: half_and_v:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ldr %f3, %f0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    ldr %f0, %f3
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call half asm sideeffect "", "={v3},0"(half %cc_dep1)
+  ret half %0
+}
+
 define float @float_and_v(float %cc_dep1) {
 ; CHECK-LABEL: float_and_v:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-zEC12.ll b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-zEC12.ll
index 1ef6eece80acb..19969ccf4e297 100644
--- a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-zEC12.ll
+++ b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-zEC12.ll
@@ -3,6 +3,21 @@
 ;
 ; Test inline assembly where the operand is bitcasted.
 
+define signext i16 @short_and_f(i16 signext %cc_dep1) {
+; CHECK-LABEL: short_and_f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sllg %r0, %r2, 48
+; CHECK-NEXT:    ldgr %f0, %r0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    lgdr %r0, %f0
+; CHECK-NEXT:    srag %r2, %r0, 48
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call i16 asm sideeffect "", "=f,0"(i16 %cc_dep1)
+  ret i16 %0
+}
+
 define signext i32 @int_and_f(i32 signext %cc_dep1) {
 ; CHECK-LABEL: int_and_f:
 ; CHECK:       # %bb.0: # %entry
@@ -49,6 +64,23 @@ entry:
   ret void
 }
 
+define half @half_and_r(half %cc_dep1) {
+; CHECK-LABEL: half_and_r:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    # kill: def $f0h killed $f0h def $f0d
+; CHECK-NEXT:    lgdr %r0, %f0
+; CHECK-NEXT:    srlg %r0, %r0, 48
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    sllg %r0, %r0, 48
+; CHECK-NEXT:    ldgr %f0, %r0
+; CHECK-NEXT:    # kill: def $f0h killed $f0h killed $f0d
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call half asm sideeffect "", "=r,0"(half %cc_dep1)
+  ret half %0
+}
+
 define float @float_and_r(float %cc_dep1) {
 ; CHECK-LABEL: float_and_r:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll
index 23d78a9315b40..b23b40e0f0e90 100644
--- a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll
+++ b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll
@@ -4,6 +4,20 @@
 ;
 ; Test inline assembly where the operand is bitcasted.
 
+define signext i16 @short_and_f(i16 signext %cc_dep1) {
+; CHECK-LABEL: short_and_f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vlvgh %v0, %r2, 0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vlgvh %r0, %v0, 0
+; CHECK-NEXT:    lghr %r2, %r0
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call i16 asm sideeffect "", "=f,0"(i16 %cc_dep1)
+  ret i16 %0
+}
+
 define signext i32 @int_and_f(i32 signext %cc_dep1) {
 ; CHECK-LABEL: int_and_f:
 ; CHECK:       # %bb.0: # %entry
@@ -58,6 +72,20 @@ entry:
   ret void
 }
 
+define signext i16 @short_and_v(i16 signext %cc_dep1) {
+; CHECK-LABEL: short_and_v:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vlvgh %v0, %r2, 0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vlgvh %r0, %v0, 0
+; CHECK-NEXT:    lghr %r2, %r0
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call i16 asm sideeffect "", "=v,0"(i16 %cc_dep1)
+  ret i16 %0
+}
+
 define signext i32 @int_and_v(i32 signext %cc_dep1) {
 ; CHECK-LABEL: int_and_v:
 ; CHECK:       # %bb.0: # %entry
@@ -100,6 +128,19 @@ entry:
   ret void
 }
 
+define half @half_and_r(half %cc_dep1) {
+; CHECK-LABEL: half_and_r:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vlgvh %r0, %v0, 0
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vlvgh %v0, %r0, 0
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call half asm sideeffect "", "=r,0"(half %cc_dep1)
+  ret half %0
+}
+
 define float @float_and_r(float %cc_dep1) {
 ; CHECK-LABEL: float_and_r:
 ; CHECK:       # %bb.0: # %entry
@@ -143,6 +184,17 @@ entry:
   ret void
 }
 
+define half @half_and_v(half %cc_dep1) {
+; CHECK-LABEL: half_and_v:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    br %r14
+entry:
+  %0 = tail call half asm sideeffect "", "=v,0"(half %cc_dep1)
+  ret half %0
+}
+
 define float @float_and_v(float %cc_dep1) {
 ; CHECK-LABEL: float_and_v:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/SystemZ/is_fpclass.ll b/llvm/test/CodeGen/SystemZ/is_fpclass.ll
index 7a02730047d20..98b856c5737ed 100644
--- a/llvm/test/CodeGen/SystemZ/is_fpclass.ll
+++ b/llvm/test/CodeGen/SystemZ/is_fpclass.ll
@@ -3,11 +3,30 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare i1 @llvm.is.fpclass.f16(half, i32)
 declare i1 @llvm.is.fpclass.f32(float, i32)
 declare i1 @llvm.is.fpclass.f64(double, i32)
 declare i1 @llvm.is.fpclass.f128(fp128, i32)
 
 
+define i1 @isnan_h(half %x) {
+; CHECK-LABEL: isnan_h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -160
+; CHECK-NEXT:    .cfi_def_cfa_offset 320
+; CHECK-NEXT:    brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:    tceb %f0, 15
+; CHECK-NEXT:    ipm %r2
+; CHECK-NEXT:    srl %r2, 28
+; CHECK-NEXT:    lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT:    br %r14
+  %1 = call i1 @llvm.is.fpclass.f16(half %x, i32 3)  ; nan
+  ret i1 %1
+}
+
 define i1 @isnan_f(float %x) {
 ; CHECK-LABEL: isnan_f:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/SystemZ/spill-half-01.mir b/llvm/test/CodeGen/SystemZ/spill-half-01.mir
new file mode 100644
index 0000000000000..2680d0225459b
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/spill-half-01.mir
@@ -0,0 +1,63 @@
+# RUN: llc -o - %s -mtriple=s390x-linux-gnu -mcpu=zEC12 -verify-machineinstrs \
+# RUN:   -start-before=greedy | FileCheck %s -check-prefix=CHECK
+# RUN: llc -o - %s -mtriple=s390x-linux-gnu -mcpu=z16 -verify-machineinstrs \
+# RUN:   -start-before=greedy | FileCheck %s -check-prefix=VECTOR
+
+# Test spilling / reloading fp16bit virtual registers.
+
+---
+name: fun0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $f0h, $f2h, $f4h
+
+    ; CHECK-LABEL: fun0:
+    ; CHECK:      aghi %r15, -240
+    ; CHECK:      ste %f4, 172(%r15)          # 4-byte Spill
+    ; CHECK-NEXT: ste %f2, 164(%r15)          # 4-byte Spill
+    ; CHECK-NEXT: ste %f0, 168(%r15)          # 4-byte Spill
+    ; CHECK-NEXT: #APP
+    ; CHECK-NEXT: #NO_APP
+    ; CHECK-NEXT: le %f0, 164(%r15)           # 4-byte Reload
+    ; CHECK:      le %f0, 168(%r15)           # 4-byte Reload
+    ; CHECK:      le %f0, 172(%r15)           # 4-byte Reload
+
+    ; VECTOR-LABEL: fun0:
+    ; VECTOR:      aghi %r15, -232
+    ; VECTOR:      vsteh %v4, 166(%r15), 0     # 2-byte Spill
+    ; VECTOR-NEXT: vsteh %v2, 162(%r15), 0     # 2-byte Spill
+    ; VECTOR-NEXT: vsteh %v0, 164(%r15), 0     # 2-byte Spill
+    ; VECTOR-NEXT: #APP
+    ; VECTOR-NEXT: #NO_APP
+    ; VECTOR-NEXT: vlreph %v0, 162(%r15)       # 2-byte Reload
+    ; VECTOR:      vlreph %v0, 164(%r15)       # 2-byte Reload
+    ; VECTOR:      vlreph %v0, 166(%r15)       # 2-byte Reload
+
+    %2:fp16bit = COPY $f4h
+    %1:fp16bit = COPY $f2h
+    %0:fp16bit = COPY $f0h
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d
+    $f0h = COPY %1
+    CallBRASL &__extendhfsf2, $f0h, csr_systemz_elf, implicit-def dead $r14d, implicit-def dead $cc, implicit $fpc, implicit-def $f0s
+    %3:fp32bit = COPY $f0s
+    $f0h = COPY %0
+    CallBRASL &__extendhfsf2, $f0h, csr_systemz_elf, implicit-def dead $r14d, implicit-def dead $cc, implicit $fpc, implicit-def $f0s
+    %5:fp32bit = COPY $f0s
+    %5:fp32bit = nofpexcept AEBR %5, %3, implicit-def dead $cc, implicit $fpc
+    $f0s = COPY %5
+    CallBRASL &__truncsfhf2, $f0s, csr_systemz_elf, implicit-def dead $r14d, implicit-def dead $cc, implicit $fpc, implicit-def $f0h
+    %6:fp16bit = COPY $f0h
+    $f0h = COPY %6
+    CallBRASL &__extendhfsf2, $f0h, csr_systemz_elf, implicit-def dead $r14d, implicit-def dead $cc, implicit $fpc, implicit-def $f0s
+    %7:fp32bit = COPY $f0s
+    $f0h = COPY %2
+    CallBRASL &__extendhfsf2, $f0h, csr_systemz_elf, implicit-def dead $r14d, implicit-def dead $cc, implicit $fpc, implicit-def $f0s
+    %9:fp32bit = COPY $f0s
+    %9:fp32bit = nofpexcept AEBR %9, %7, implicit-def dead $cc, implicit $fpc
+    $f0s = COPY %9
+    CallBRASL &__truncsfhf2, $f0s, csr_systemz_elf, implicit-def dead $r14d, implicit-def dead $cc, implicit $fpc, implicit-def $f0h
+    %10:fp16bit = COPY $f0h
+    $f0h = COPY %10
+    Return implicit $f0h
+...
diff --git a/llvm/test/CodeGen/SystemZ/spill-half-02.mir b/llvm/test/CodeGen/SystemZ/spill-half-02.mir
new file mode 100644
index 0000000000000..724b5d352b298
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/spill-half-02.mir
@@ -0,0 +1,27 @@
+# RUN: llc -o - %s -mtriple=s390x-linux-gnu -mcpu=z16 -verify-machineinstrs \
+# RUN:   -start-before=greedy | FileCheck %s
+
+# Test spilling / reloading of an vr16bit virtual register.
+
+---
+name: fun0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $r2d, $r3d
+
+    ; CHECK-LABEL: fun0:
+    ; CHECK:       vlreph %v0, 0(%r2)
+    ; CHECK-NEXT:  vsteh %v0, 166(%r15), 0        # 2-byte Spill
+    ; CHECK-NEXT:  #APP
+    ; CHECK-NEXT:  #NO_APP
+    ; CHECK-NEXT:  vlreph %v0, 166(%r15)          # 2-byte Reload
+    ; CHECK-NEXT:  vsteh %v0, 0(%r3), 0
+
+    %1:addr64bit = COPY $r3d
+    %0:addr64bit = COPY $r2d
+    %2:vr16bit = VL16 %0, 0, $noreg
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $f0d, 12, implicit-def dead early-clobber $f1d, 12, implicit-def dead early-clobber $f2d, 12, implicit-def dead early-clobber $f3d, 12, implicit-def dead early-clobber $f4d, 12, implicit-def dead early-clobber $f5d, 12, implicit-def dead early-clobber $f6d, 12, implicit-def dead early-clobber $f7d, 12, implicit-def dead early-clobber $f8d, 12, implicit-def dead early-clobber $f9d, 12, implicit-def dead early-clobber $f10d, 12, implicit-def dead early-clobber $f11d, 12, implicit-def dead early-clobber $f12d, 12, implicit-def dead early-clobber $f13d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f14d, 12, implicit-def dead early-clobber $f15d, 12, implicit-def dead early-clobber $f16d, 12, implicit-def dead early-clobber $f17d, 12, implicit-def dead early-clobber $f18d, 12, implicit-def dead early-clobber $f19d, 12, implicit-def dead early-clobber $f20d, 12, implicit-def dead early-clobber $f21d, 12, implicit-def dead early-clobber $f22d, 12, implicit-def dead early-clobber $f23d, 12, implicit-def dead early-clobber $f24d, 12, implicit-def dead early-clobber $f25d, 12, implicit-def dead early-clobber $f26d, 12, implicit-def dead early-clobber $f27d, 12, implicit-def dead early-clobber $f28d, 12, implicit-def dead early-clobber $f29d, 12, implicit-def dead early-clobber $f30d, 12, implicit-def dead early-clobber $f31d
+    VST16 %2, %1, 0, $noreg
+    Return
+...
diff --git a/llvm/test/CodeGen/SystemZ/stackmap.ll b/llvm/test/CodeGen/SystemZ/stackmap.ll
index 6156b7f2fc5a1..05b8de756c032 100644
--- a/llvm/test/CodeGen/SystemZ/stackmap.ll
+++ b/llvm/test/CodeGen/SystemZ/stackmap.ll
@@ -553,7 +553,14 @@ declare void @escape_values(...)
 ; CHECK-LABEL:  .long .L{{.*}}-floats
 ; CHECK-NEXT:   .short 0
 ; Num Locations
-; CHECK-NEXT:   .short 6
+; CHECK-NEXT:   .short 9
+; Loc 0: constant half stored to FP register
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .short  4
+; CHECK-NEXT:   .short  {{.*}}
+; CHECK-NEXT:   .short  0
+; CHECK-NEXT:   .long   16
 ; Loc 0: constant float stored to FP register
 ; CHECK-NEXT:   .byte   1
 ; CHECK-NEXT:   .byte   0
@@ -568,6 +575,13 @@ declare void @escape_values(...)
 ; CHECK-NEXT:   .short  {{.*}}
 ; CHECK-NEXT:   .short  0
 ; CHECK-NEXT:   .long   0
+; Loc 1: half value in FP register
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .short  4
+; CHECK-NEXT:   .short  {{.*}}
+; CHECK-NEXT:   .short  0
+; CHECK-NEXT:   .long   16
 ; Loc 1: float value in FP register
 ; CHECK-NEXT:   .byte   1
 ; CHECK-NEXT:   .byte   0
@@ -582,6 +596,13 @@ declare void @escape_values(...)
 ; CHECK-NEXT:   .short  {{.*}}
 ; CHECK-NEXT:   .short  0
 ; CHECK-NEXT:   .long   0
+; Loc 3: half on stack
+; CHECK-NEXT:   .byte   2
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .short  8
+; CHECK-NEXT:   .short  {{.*}}
+; CHECK-NEXT:   .short  0
+; CHECK-NEXT:   .long   {{.*}}
 ; Loc 3: float on stack
 ; CHECK-NEXT:   .byte   2
 ; CHECK-NEXT:   .byte   0
@@ -596,11 +617,12 @@ declare void @escape_values(...)
 ; CHECK-NEXT:   .short  {{.*}}
 ; CHECK-NEXT:   .short  0
 ; CHECK-NEXT:   .long   {{.*}}
-define void @floats(float %f, double %g) {
+define void @floats(half %e, float %f, double %g) {
+  %hh = alloca half
   %ff = alloca float
   %gg = alloca double
-  call void (i64, i32, ...) @llvm.experimental.stackmap(i64 888, i32 0, float 1.25,
-    double 1.5, float %f, double %g, ptr %ff, ptr %gg)
+  call void (i64, i32, ...) @llvm.experimental.stackmap(i64 888, i32 0, half 1.125,
+    float 1.25, double 1.5, half %e, float %f, double %g, ptr %hh, ptr %ff, ptr %gg)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/SystemZ/tdc-01.ll b/llvm/test/CodeGen/SystemZ/tdc-01.ll
index 052d895b798f6..a0c090f463a2c 100644
--- a/llvm/test/CodeGen/SystemZ/tdc-01.ll
+++ b/llvm/test/CodeGen/SystemZ/tdc-01.ll
@@ -2,10 +2,22 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare i32 @llvm.s390.tdc.f16(half, i64)
 declare i32 @llvm.s390.tdc.f32(float, i64)
 declare i32 @llvm.s390.tdc.f64(double, i64)
 declare i32 @llvm.s390.tdc.f128(fp128, i64)
 
+; Check using as i32 - f16
+define i32 @f0(half %x) {
+; CHECK-LABEL: f0
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: tceb %f0, 123
+; CHECK: ipm %r2
+; CHECK: srl %r2, 28
+  %res = call i32 @llvm.s390.tdc.f16(half %x, i64 123)
+  ret i32 %res
+}
+
 ; Check using as i32 - f32
 define i32 @f1(float %x) {
 ; CHECK-LABEL: f1
diff --git a/llvm/test/CodeGen/SystemZ/tdc-02.ll b/llvm/test/CodeGen/SystemZ/tdc-02.ll
index c0c4ac84349e3..ceb397c6cb9cb 100644
--- a/llvm/test/CodeGen/SystemZ/tdc-02.ll
+++ b/llvm/test/CodeGen/SystemZ/tdc-02.ll
@@ -2,10 +2,27 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare i32 @llvm.s390.tdc.f16(half, i64)
 declare i32 @llvm.s390.tdc.f32(float, i64)
 declare i32 @llvm.s390.tdc.f64(double, i64)
 declare i32 @llvm.s390.tdc.f128(fp128, i64)
 
+; Check using or i1
+define i32 @f0(half %x) {
+; CHECK-LABEL: f0
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: tceb %f0, 7
+; CHECK-NEXT: ipm [[REG1:%r[0-9]+]]
+; CHECK-NEXT: risbg %r2, [[REG1]], 63, 191, 36
+  %a = call i32 @llvm.s390.tdc.f16(half %x, i64 3)
+  %b = call i32 @llvm.s390.tdc.f16(half %x, i64 6)
+  %a1 = icmp ne i32 %a, 0
+  %b1 = icmp ne i32 %b, 0
+  %res = or i1 %a1, %b1
+  %xres = zext i1 %res to i32
+  ret i32 %xres
+}
+
 ; Check using or i1
 define i32 @f1(float %x) {
 ; CHECK-LABEL: f1
diff --git a/llvm/test/CodeGen/SystemZ/tdc-03.ll b/llvm/test/CodeGen/SystemZ/tdc-03.ll
index 95708f1effc6b..b6c12caef72fd 100644
--- a/llvm/test/CodeGen/SystemZ/tdc-03.ll
+++ b/llvm/test/CodeGen/SystemZ/tdc-03.ll
@@ -3,10 +3,23 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
+declare half @llvm.fabs.f16(half)
 declare float @llvm.fabs.f32(float)
 declare double @llvm.fabs.f64(double)
 declare fp128 @llvm.fabs.f128(fp128)
 
+; Compare with 0 (unworthy)
+define i32 @f0(half %x) {
+; CHECK-LABEL: f0
+; CHECK-NOT: tceb
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: ltebr {{%f[0-9]+}}, %f0
+; CHECK-NOT: tceb
+  %res = fcmp ugt half %x, 0.0
+  %xres = zext i1 %res to i32
+  ret i32 %xres
+}
+
 ; Compare with 0 (unworthy)
 define i32 @f1(float %x) {
 ; CHECK-LABEL: f1
@@ -41,9 +54,20 @@ define i32 @f3(float %x) {
   ret i32 %xres
 }
 
+; Compare fabs with inf
+define i32 @f4_half(half %x) {
+; CHECK-LABEL: f4_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: tceb %f0, 4047
+  %y = call half @llvm.fabs.f16(half %x)
+  %res = fcmp ult half %y, 0x7ff0000000000000
+  %xres = zext i1 %res to i32
+  ret i32 %xres
+}
+
 ; Compare fabs with inf
 define i32 @f4(float %x) {
-; CHECK-LABEL: f4
+; CHECK-LABEL: f4:
 ; CHECK: tceb %f0, 4047
   %y = call float @llvm.fabs.f32(float %x)
   %res = fcmp ult float %y, 0x7ff0000000000000
diff --git a/llvm/test/CodeGen/SystemZ/tdc-04.ll b/llvm/test/CodeGen/SystemZ/tdc-04.ll
index 8cc78f3de7522..bc719640e630d 100644
--- a/llvm/test/CodeGen/SystemZ/tdc-04.ll
+++ b/llvm/test/CodeGen/SystemZ/tdc-04.ll
@@ -1,10 +1,24 @@
 ; Test the Test Data Class instruction logic operation conversion from
 ; signbit extraction.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s --check-prefixes=CHECK,Z10
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s \
+; RUN:   --check-prefixes=CHECK,Z13
 ;
 
+; Extract sign bit.
+define i32 @f0(half %x) {
+; CHECK-LABEL: f0
+; Z10: lgdr %r0, %f0
+; Z10: srlg %r2, %r0, 63
+; Z13: vlgvh %r0, %v0, 0
+; Z13: risblg %r2, %r0, 31, 159, 49
+  %cast = bitcast half %x to i16
+  %res = icmp slt i16 %cast, 0
+  %xres = zext i1 %res to i32
+  ret i32 %xres
+}
+
 ; Extract sign bit.
 define i32 @f1(float %x) {
 ; CHECK-LABEL: f1
diff --git a/llvm/test/CodeGen/SystemZ/tdc-05.ll b/llvm/test/CodeGen/SystemZ/tdc-05.ll
index c639a9b7b4757..becf293c21f1f 100644
--- a/llvm/test/CodeGen/SystemZ/tdc-05.ll
+++ b/llvm/test/CodeGen/SystemZ/tdc-05.ll
@@ -8,6 +8,31 @@ declare float @llvm.fabs.f32(float)
 declare double @llvm.fabs.f64(double)
 declare fp128 @llvm.fabs.f128(fp128)
 
+; Compare with 0, extract sign bit
+define i32 @f0(half %x) {
+; CHECK-LABEL: f0
+; CHECK:       lgdr %r0, %f0
+; CHECK-NEXT:  srlg %r0, %r0, 48
+; CHECK-NEXT:  lhr %r0, %r0
+; CHECK-NEXT:  chi %r0, 0
+; CHECK-NEXT:  ipm %r0
+; CHECK-NEXT:  risbg %r13, %r0, 63, 191, 36
+; CHECK-NEXT:     # kill: def $f0h killed $f0h killed $f0d
+; CHECK-NEXT:  brasl %r14, __extendhfsf2@PLT
+; CHECK-NEXT:  ltebr %f0, %f0
+; CHECK-NEXT:  ipm %r0
+; CHECK-NEXT:  rosbg %r13, %r0, 63, 63, 35
+; CHECK-NEXT:  lr %r2, %r13
+; CHECK-NEXT:  lmg %r13, %r15, 264(%r15)
+; CHECK-NEXT:  br %r14
+  %cast = bitcast half %x to i16
+  %sign = icmp slt i16 %cast, 0
+  %fcmp = fcmp ugt half %x, 0.0
+  %res = or i1 %sign, %fcmp
+  %xres = zext i1 %res to i32
+  ret i32 %xres
+}
+
 ; Compare with 0, extract sign bit
 define i32 @f1(float %x) {
 ; CHECK-LABEL: f1
diff --git a/llvm/test/CodeGen/SystemZ/tdc-06.ll b/llvm/test/CodeGen/SystemZ/tdc-06.ll
index 4ebf020c973da..19536b26eb5ae 100644
--- a/llvm/test/CodeGen/SystemZ/tdc-06.ll
+++ b/llvm/test/CodeGen/SystemZ/tdc-06.ll
@@ -3,9 +3,7 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 ;
 
-declare float @llvm.fabs.f32(float)
 declare double @llvm.fabs.f64(double)
-declare fp128 @llvm.fabs.f128(fp128)
 
 define i32 @fpc(double %x) {
 entry:
diff --git a/llvm/test/CodeGen/SystemZ/twoaddr-kill.mir b/llvm/test/CodeGen/SystemZ/twoaddr-kill.mir
index 7fc7bd3e347bb..95ba0b4bf3466 100644
--- a/llvm/test/CodeGen/SystemZ/twoaddr-kill.mir
+++ b/llvm/test/CodeGen/SystemZ/twoaddr-kill.mir
@@ -18,19 +18,19 @@ body: |
     ; CHECK-NEXT: $r2l = COPY [[COPY]]
     ; CHECK-NEXT: $r3l = COPY killed [[COPY]]
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:grh32bit = COPY killed [[COPY1]]
-    ; CHECK-NEXT: INLINEASM &"stepa $1, $2, $3", 0 /* attdialect */, 393226 /* regdef:GRH32Bit */, def [[COPY2]], 2147483657 /* reguse tiedto:$0 */, [[COPY2]](tied-def 3), 9 /* reguse */, killed $r2l, 9 /* reguse */, killed $r3l
+    ; CHECK-NEXT: INLINEASM &"stepa $1, $2, $3", 0 /* attdialect */, 524298 /* regdef:GRH32Bit */, def [[COPY2]], 2147483657 /* reguse tiedto:$0 */, [[COPY2]](tied-def 3), 9 /* reguse */, killed $r2l, 9 /* reguse */, killed $r3l
     ; CHECK-NEXT: [[COPY3:%[0-9]+]]:grh32bit = COPY killed [[COPY2]]
     ; CHECK-NEXT: [[COPY4:%[0-9]+]]:grh32bit = COPY [[COPY3]]
-    ; CHECK-NEXT: INLINEASM &"stepb $1, $2", 0 /* attdialect */, 393227 /* regdef-ec:GRH32Bit */, def early-clobber [[COPY4]], 2147483657 /* reguse tiedto:$0 */, [[COPY4]](tied-def 3), 393225 /* reguse:GRH32Bit */, [[COPY3]]
+    ; CHECK-NEXT: INLINEASM &"stepb $1, $2", 0 /* attdialect */, 524299 /* regdef-ec:GRH32Bit */, def early-clobber [[COPY4]], 2147483657 /* reguse tiedto:$0 */, [[COPY4]](tied-def 3), 524297 /* reguse:GRH32Bit */, [[COPY3]]
     ; CHECK-NEXT: $r2l = COPY killed [[COPY4]]
     ; CHECK-NEXT: Return implicit killed $r2l
     %0:gr32bit = COPY killed $r2l
     %2:grh32bit = COPY %0
     $r2l = COPY %0
     $r3l = COPY killed %0
-    INLINEASM &"stepa $1, $2, $3", 0 /* attdialect */, 393226 /* regdef:GRH32Bit */, def %1:grh32bit, 2147483657 /* reguse tiedto:$0 */, killed %2(tied-def 3), 9 /* reguse */, killed $r2l, 9 /* reguse */, killed $r3l
+    INLINEASM &"stepa $1, $2, $3", 0 /* attdialect */, 524298 /* regdef:GRH32Bit */, def %1:grh32bit, 2147483657 /* reguse tiedto:$0 */, killed %2(tied-def 3), 9 /* reguse */, killed $r2l, 9 /* reguse */, killed $r3l
     %4:grh32bit = COPY killed %1
-    INLINEASM &"stepb $1, $2", 0 /* attdialect */, 393227 /* regdef-ec:GRH32Bit */, def early-clobber %3:grh32bit, 2147483657 /* reguse tiedto:$0 */, killed %4(tied-def 3), 393225 /* reguse:GRH32Bit */, %4
+    INLINEASM &"stepb $1, $2", 0 /* attdialect */, 524299 /* regdef-ec:GRH32Bit */, def early-clobber %3:grh32bit, 2147483657 /* reguse tiedto:$0 */, killed %4(tied-def 3), 524297 /* reguse:GRH32Bit */, %4
     $r2l = COPY killed %3
     Return implicit killed $r2l
 ...
diff --git a/llvm/test/CodeGen/SystemZ/vec-max-05.ll b/llvm/test/CodeGen/SystemZ/vec-max-05.ll
index 7bdf4e06029d2..09d40c77a1fb9 100644
--- a/llvm/test/CodeGen/SystemZ/vec-max-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-max-05.ll
@@ -14,6 +14,8 @@ declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>)
 declare float @llvm.maximum.f32(float, float)
 declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
 
+declare half @llvm.maxnum.f16(half, half)
+
 declare fp128 @fmaxl(fp128, fp128)
 declare fp128 @llvm.maxnum.f128(fp128, fp128)
 declare fp128 @llvm.maximum.f128(fp128, fp128)
@@ -96,6 +98,18 @@ define float @f11(float %dummy, float %val1, float %val2) {
   ret float %ret
 }
 
+; Test the f16 maxnum intrinsic.
+define half @f12_half(half %dummy, half %val1, half %val2) {
+; CHECK-LABEL: f12_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: wfmaxsb %f0, %f0, %f9, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %ret = call half @llvm.maxnum.f16(half %val1, half %val2)
+  ret half %ret
+}
+
 ; Test the f32 maxnum intrinsic.
 define float @f12(float %dummy, float %val1, float %val2) {
 ; CHECK-LABEL: f12:
diff --git a/llvm/test/CodeGen/SystemZ/vec-min-05.ll b/llvm/test/CodeGen/SystemZ/vec-min-05.ll
index bf27eb3e56036..b7b288f531041 100644
--- a/llvm/test/CodeGen/SystemZ/vec-min-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-min-05.ll
@@ -14,6 +14,8 @@ declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>)
 declare float @llvm.minimum.f32(float, float)
 declare <4 x float> @llvm.minimum.v4f32(<4 x float>, <4 x float>)
 
+declare half @llvm.minnum.f16(half, half)
+
 declare fp128 @fminl(fp128, fp128)
 declare fp128 @llvm.minnum.f128(fp128, fp128)
 declare fp128 @llvm.minimum.f128(fp128, fp128)
@@ -96,6 +98,18 @@ define float @f11(float %dummy, float %val1, float %val2) {
   ret float %ret
 }
 
+; Test the f16 minnum intrinsic.
+define half @f12_half(half %dummy, half %val1, half %val2) {
+; CHECK-LABEL: f12_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: wfminsb %f0, %f0, %f9, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %ret = call half @llvm.minnum.f16(half %val1, half %val2)
+  ret half %ret
+}
+
 ; Test the f32 minnum intrinsic.
 define float @f12(float %dummy, float %val1, float %val2) {
 ; CHECK-LABEL: f12:
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-max-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-max-01.ll
index 66870a797a7a5..ff1875a731fbf 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-max-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-max-01.ll
@@ -12,6 +12,9 @@ declare <4 x float> @llvm.experimental.constrained.maxnum.v4f32(<4 x float>, <4
 declare float @llvm.experimental.constrained.maximum.f32(float, float, metadata)
 declare <4 x float> @llvm.experimental.constrained.maximum.v4f32(<4 x float>, <4 x float>, metadata)
 
+declare half @llvm.experimental.constrained.maxnum.f16(half, half, metadata)
+declare half @llvm.experimental.constrained.maximum.f16(half, half, metadata)
+
 declare fp128 @llvm.experimental.constrained.maxnum.f128(fp128, fp128, metadata)
 declare fp128 @llvm.experimental.constrained.maximum.f128(fp128, fp128, metadata)
 
@@ -38,6 +41,20 @@ define <2 x double> @f2(<2 x double> %dummy, <2 x double> %val1,
   ret <2 x double> %ret
 }
 
+; Test the f16 maxnum intrinsic.
+define half @f3_half(half %dummy, half %val1, half %val2) #0 {
+; CHECK-LABEL: f3_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: wfmaxsb %f0, %f0, %f9, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %ret = call half @llvm.experimental.constrained.maxnum.f16(
+                        half %val1, half %val2,
+                        metadata !"fpexcept.strict") #0
+  ret half %ret
+}
+
 ; Test the f32 maxnum intrinsic.
 define float @f3(float %dummy, float %val1, float %val2) #0 {
 ; CHECK-LABEL: f3:
@@ -101,6 +118,20 @@ define <2 x double> @f12(<2 x double> %dummy, <2 x double> %val1,
   ret <2 x double> %ret
 }
 
+; Test the f16 maximum intrinsic.
+define half @f13_half(half %dummy, half %val1, half %val2) #0 {
+; CHECK-LABEL: f13_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: wfmaxsb %f0, %f0, %f9, 1
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %ret = call half @llvm.experimental.constrained.maximum.f16(
+                        half %val1, half %val2,
+                        metadata !"fpexcept.strict") #0
+  ret half %ret
+}
+
 ; Test the f32 maximum intrinsic.
 define float @f13(float %dummy, float %val1, float %val2) #0 {
 ; CHECK-LABEL: f13:
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-min-01.ll b/llvm/test/CodeGen/SystemZ/vec-strict-min-01.ll
index cf5332ff4f1d1..ddbffd735f699 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-min-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-min-01.ll
@@ -12,6 +12,9 @@ declare <4 x float> @llvm.experimental.constrained.minnum.v4f32(<4 x float>, <4
 declare float @llvm.experimental.constrained.minimum.f32(float, float, metadata)
 declare <4 x float> @llvm.experimental.constrained.minimum.v4f32(<4 x float>, <4 x float>, metadata)
 
+declare half @llvm.experimental.constrained.minnum.f16(half, half, metadata)
+declare half @llvm.experimental.constrained.minimum.f16(half, half, metadata)
+
 declare fp128 @llvm.experimental.constrained.minnum.f128(fp128, fp128, metadata)
 declare fp128 @llvm.experimental.constrained.minimum.f128(fp128, fp128, metadata)
 
@@ -38,6 +41,20 @@ define <2 x double> @f2(<2 x double> %dummy, <2 x double> %val1,
   ret <2 x double> %ret
 }
 
+; Test the f16 minnum intrinsic.
+define half @f3_half(half %dummy, half %val1, half %val2) #0 {
+; CHECK-LABEL: f3_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: wfminsb %f0, %f0, %f9, 4
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %ret = call half @llvm.experimental.constrained.minnum.f16(
+                        half %val1, half %val2,
+                        metadata !"fpexcept.strict") #0
+  ret half %ret
+}
+
 ; Test the f32 minnum intrinsic.
 define float @f3(float %dummy, float %val1, float %val2) #0 {
 ; CHECK-LABEL: f3:
@@ -101,6 +118,20 @@ define <2 x double> @f12(<2 x double> %dummy, <2 x double> %val1,
   ret <2 x double> %ret
 }
 
+; Test the f32 minimum intrinsic.
+define half @f13_half(half %dummy, half %val1, half %val2) #0 {
+; CHECK-LABEL: f13_half:
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: brasl %r14, __extendhfsf2@PLT
+; CHECK: wfminsb %f0, %f0, %f9, 1
+; CHECK: brasl %r14, __truncsfhf2@PLT
+; CHECK: br %r14
+  %ret = call half @llvm.experimental.constrained.minimum.f16(
+                        half %val1, half %val2,
+                        metadata !"fpexcept.strict") #0
+  ret half %ret
+}
+
 ; Test the f32 minimum intrinsic.
 define float @f13(float %dummy, float %val1, float %val2) #0 {
 ; CHECK-LABEL: f13:

From ad12323fbf8f34fcb3bd3a75ed410d3d5b0ca42c Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Wed, 16 Apr 2025 11:15:54 -0700
Subject: [PATCH 158/710] [HLSL] Don't invoke `dxv` from `clang-dxc` for text
 output (#135876)

Running `clang-dxc` with textual output was emitting various spurious
warnings (if `dxv` wasn't on your path) or errors (if it was). Avoid
these by not attempting to run this tool when it doesn't make sense to
do so.

Fixes #135874.
---
 clang/lib/Driver/ToolChains/HLSL.cpp        |  3 +++
 clang/test/Driver/HLSL/metal-converter.hlsl | 10 ++++++----
 clang/test/Driver/dxc_D.hlsl                |  3 ---
 clang/test/Driver/dxc_dxv_path.hlsl         |  6 +++---
 clang/test/Driver/dxc_options.hlsl          |  3 +--
 clang/test/Driver/hlsl-lang-targets.hlsl    |  8 ++++----
 6 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index 22498bff1f251..59e9050af8a76 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -309,6 +309,9 @@ HLSLToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
 }
 
 bool HLSLToolChain::requiresValidation(DerivedArgList &Args) const {
+  if (!Args.hasArg(options::OPT_dxc_Fo))
+    return false;
+
   if (Args.getLastArg(options::OPT_dxc_disable_validation))
     return false;
 
diff --git a/clang/test/Driver/HLSL/metal-converter.hlsl b/clang/test/Driver/HLSL/metal-converter.hlsl
index 536f24be6e73b..3c4257b3fbb28 100644
--- a/clang/test/Driver/HLSL/metal-converter.hlsl
+++ b/clang/test/Driver/HLSL/metal-converter.hlsl
@@ -1,11 +1,13 @@
-// RUN: %clang_dxc -T cs_6_0 %s -metal -Fo %t.mtl -### 2>&1 | FileCheck %s
-// RUN: %clang_dxc -T cs_6_0 %s -metal -Vd -Fo %t.mtl -### 2>&1 | FileCheck %s
-// CHECK: "{{.*}}metal-shaderconverter{{(.exe)?}}" "{{.*}}.obj" "-o" "{{.*}}.mtl"
+// RUN: echo "dxv" > %T/dxv && chmod 754 %T/dxv
+
+// RUN: env PATH="" %clang_dxc -T cs_6_0 %s -metal -Fo %t.mtl -### 2>&1 | FileCheck --check-prefix=NO_DXV %s
+// RUN: env PATH="" %clang_dxc -T cs_6_0 %s -metal -Vd -Fo %t.mtl -### 2>&1 | FileCheck --check-prefix=NO_DXV %s
+// RUN: env PATH="" %clang_dxc -T cs_6_0 %s --dxv-path=%T -metal -Vd -Fo %t.mtl -### 2>&1 | FileCheck --check-prefix=NO_DXV %s
+// NO_DXV: "{{.*}}metal-shaderconverter{{(.exe)?}}" "{{.*}}.obj" "-o" "{{.*}}.mtl"
 
 // RUN: %clang_dxc -T cs_6_0 %s -metal -### 2>&1 | FileCheck --check-prefix=NO_MTL %s
 // NO_MTL-NOT: metal-shaderconverter
 
-// RUN: echo "dxv" > %T/dxv && chmod 754 %T/dxv
 // RUN: %clang_dxc -T cs_6_0 %s --dxv-path=%T -metal -Fo %t.mtl -### 2>&1 | FileCheck --check-prefix=DXV %s
 // DXV: "{{.*}}dxv{{(.exe)?}}" "{{.*}}.obj" "-o" "{{.*}}.dxo"
 // DXV: "{{.*}}metal-shaderconverter{{(.exe)?}}" "{{.*}}.dxo" "-o" "{{.*}}.mtl"
diff --git a/clang/test/Driver/dxc_D.hlsl b/clang/test/Driver/dxc_D.hlsl
index f32a22c503327..f941974c61093 100644
--- a/clang/test/Driver/dxc_D.hlsl
+++ b/clang/test/Driver/dxc_D.hlsl
@@ -1,5 +1,4 @@
 // RUN: %clang_dxc -DTEST=2 -Tlib_6_7 -### %s 2>&1 | FileCheck %s
-// RUN: %clang_dxc -DTEST=2  -Tlib_6_7 %s -fcgl -Fo - | FileCheck %s --check-prefix=ERROR
 
 // Make sure -D send to cc1.
 // CHECK:"-D" "TEST=2"
@@ -9,5 +8,3 @@
 #elif TEST != 2
 #error "TEST defined to wrong value"
 #endif
-
-// ERROR-NOT: error:
diff --git a/clang/test/Driver/dxc_dxv_path.hlsl b/clang/test/Driver/dxc_dxv_path.hlsl
index 55a07f34a648e..65e386f2f35ab 100644
--- a/clang/test/Driver/dxc_dxv_path.hlsl
+++ b/clang/test/Driver/dxc_dxv_path.hlsl
@@ -1,10 +1,10 @@
-// RUN: %clang_dxc -I test -Tlib_6_3  -### %s 2>&1 | FileCheck %s
+// RUN: env PATH="" %clang_dxc -I test -Tlib_6_3 -Fo %T/a.dxo  -### %s 2>&1 | FileCheck %s
 
 // Make sure report warning.
 // CHECK:dxv not found
 
-// RUN: echo "dxv" > %T/dxv && chmod 754 %T/dxv && %clang_dxc --dxv-path=%T %s -Tlib_6_3 -### 2>&1 | FileCheck %s --check-prefix=DXV_PATH
-// DXV_PATH:dxv{{(.exe)?}}" "-" "-o" "{{.*}}.dxo"
+// RUN: echo "dxv" > %T/dxv && chmod 754 %T/dxv && %clang_dxc --dxv-path=%T %s -Tlib_6_3 -Fo %T/a.dxo -### 2>&1 | FileCheck %s --check-prefix=DXV_PATH
+// DXV_PATH:dxv{{(.exe)?}}" "{{.*}}.obj" "-o" "{{.*}}/a.dxo"
 
 // RUN: %clang_dxc -I test -Vd -Tlib_6_3  -### %s 2>&1 | FileCheck %s --check-prefix=VD
 // VD:"-cc1"{{.*}}"-triple" "dxilv1.3-unknown-shadermodel6.3-library"
diff --git a/clang/test/Driver/dxc_options.hlsl b/clang/test/Driver/dxc_options.hlsl
index 09fdba1c3dd5f..5026b75e52688 100644
--- a/clang/test/Driver/dxc_options.hlsl
+++ b/clang/test/Driver/dxc_options.hlsl
@@ -4,5 +4,4 @@
 // RUN: -fdiagnostics-color \
 // RUN: -fno-diagnostics-color \
 // RUN: -fdiagnostics-color=auto \
-// RUN: -Tlib_6_7 -Vd -fdriver-only -- %s 2>&1 |count 0
-
+// RUN: -Tlib_6_7 -fdriver-only -- %s 2>&1 |count 0
diff --git a/clang/test/Driver/hlsl-lang-targets.hlsl b/clang/test/Driver/hlsl-lang-targets.hlsl
index 7ce490a66df5f..b321b00548e9b 100644
--- a/clang/test/Driver/hlsl-lang-targets.hlsl
+++ b/clang/test/Driver/hlsl-lang-targets.hlsl
@@ -2,10 +2,10 @@
 
 // Supported targets
 //
-// RUN: %clang -target dxil--shadermodel6.2-pixel %s -S -o /dev/null 2>&1 | FileCheck --check-prefix=CHECK-VALID %s
-// RUN: %clang -target dxil-unknown-shadermodel6.2-pixel %s -S -o /dev/null 2>&1 | FileCheck --check-prefix=CHECK-VALID %s
-// RUN: %clang -target dxil--shadermodel6.2-library %s -S -o /dev/null 2>&1 | FileCheck --check-prefix=CHECK-VALID %s
-// RUN: %clang -target dxil-unknown-shadermodel6.2-library %s -S -o /dev/null 2>&1 | FileCheck --check-prefix=CHECK-VALID %s
+// RUN: %clang -target dxil--shadermodel6.2-pixel %s -S -o /dev/null 2>&1 | FileCheck --allow-empty --check-prefix=CHECK-VALID %s
+// RUN: %clang -target dxil-unknown-shadermodel6.2-pixel %s -S -o /dev/null 2>&1 | FileCheck --allow-empty --check-prefix=CHECK-VALID %s
+// RUN: %clang -target dxil--shadermodel6.2-library %s -S -o /dev/null 2>&1 | FileCheck --allow-empty --check-prefix=CHECK-VALID %s
+// RUN: %clang -target dxil-unknown-shadermodel6.2-library %s -S -o /dev/null 2>&1 | FileCheck --allow-empty --check-prefix=CHECK-VALID %s
 
 // Empty shader model
 //

From e19fcb72d7fbda6a1e67c45b85b399fe69d212ad Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1@linux.ibm.com>
Date: Wed, 16 Apr 2025 12:25:10 -0600
Subject: [PATCH 159/710] Fix 'unannotated fall-through between switch labels'
 warning. (#136000)

---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index fddb99d2f0b22..b596f6b8d6ce2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -5467,8 +5467,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
     break;
 
-  case ISD::STRICT_FMINIMUM: {
-  case ISD::STRICT_FMAXIMUM:
+  case ISD::STRICT_FMINIMUM:
+  case ISD::STRICT_FMAXIMUM: {
     SDValue InChain = Node->getOperand(0);
     SDVTList VTs = DAG.getVTList(NVT, MVT::Other);
     Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, VTs, InChain,

From 8a00efd26db21ef73df58b465b7741d1f889a681 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 11:25:55 -0700
Subject: [PATCH 160/710] [SystemZ] Fix warnings

This patch fixes:

  llvm/lib/Target/SystemZ/SystemZISelLowering.cpp:6916:7: error:
  unused variable 'RegVT' [-Werror,-Wunused-variable]

  llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp:1265:30: error: unused
  variable 'RC' [-Werror,-Wunused-variable]
---
 llvm/lib/Target/SystemZ/SystemZISelLowering.cpp | 1 +
 llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index fdbfc196e8fbc..75cd5a319557d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -6915,6 +6915,7 @@ SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
                                             SelectionDAG &DAG) const {
   EVT RegVT = Op.getValueType();
   assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
+  (void)RegVT;
 
   // Load as integer.
   SDLoc DL(Op);
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 1ae3994eb0e01..ae6ca55a36092 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -1267,6 +1267,7 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
   assert((Size * 8 == TRI->getRegSizeInBits(*RC) ||
           (RC == &SystemZ::FP16BitRegClass && Size == 4 && !STI.hasVector())) &&
          "Invalid size combination");
+  (void)RC;
 
   if ((Opcode == SystemZ::AHI || Opcode == SystemZ::AGHI) && OpNum == 0 &&
       isInt<8>(MI.getOperand(2).getImm())) {

From e77ef7b291a0024ae34eaa76dafb62aef06d3c95 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 11:29:34 -0700
Subject: [PATCH 161/710] [NFC][CFI] Dump test output to debug
 llvm-clang-win-x-aarch64 failure (#136002)

---
 clang/test/Driver/sanitizer-ld.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index a00ec029d3d46..c04831171bba9 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -839,6 +839,14 @@
 // CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-flto'
 // CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-fvisibility='
 
+// CFI by itself does not link runtime libraries.
+// RUN: %clang -fsanitize=cfi \
+// RUN:     -flto -fvisibility=hidden \
+// RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
+// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
+// RUN:     -### %s
+
 // CFI by itself does not link runtime libraries.
 // RUN: %clang -fsanitize=cfi \
 // RUN:     -flto -fvisibility=hidden \

From 4aca20c8b6dcf86696db03d860e635112601a7f9 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 16 Apr 2025 11:42:56 -0700
Subject: [PATCH 162/710] [SLP]Pre-cache the last instruction for all entries
 before vectorization

Need to pre-cache last instruction to avoid unexpected changes in the
last instruction detection during the vectorization, caused by adding
the new vector instructions, which add new uses and may affect the
analysis.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  8 +-
 .../entry-no-bundle-but-extra-use-on-vec.ll   | 91 +++++++++++++++++++
 2 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/entry-no-bundle-but-extra-use-on-vec.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 83252bdb51ea2..41ad43012d337 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18414,8 +18414,14 @@ Value *BoUpSLP::vectorizeTree(
   // need to rebuild it.
   EntryToLastInstruction.clear();
   // All blocks must be scheduled before any instructions are inserted.
-  for (auto &BSIter : BlocksSchedules) {
+  for (auto &BSIter : BlocksSchedules)
     scheduleBlock(BSIter.second.get());
+  // Cache last instructions for the nodes to avoid side effects, which may
+  // appear during vectorization, like extra uses, etc.
+  for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
+    if (TE->isGather())
+      continue;
+    (void)getLastInstructionInBundle(TE.get());
   }
 
   if (ReductionRoot)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/entry-no-bundle-but-extra-use-on-vec.ll b/llvm/test/Transforms/SLPVectorizer/X86/entry-no-bundle-but-extra-use-on-vec.ll
new file mode 100644
index 0000000000000..9d48e7f8a787a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/entry-no-bundle-but-extra-use-on-vec.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-generic-linux-gnu < %s | FileCheck %s
+
+define void @test(ptr %nExp, float %0, i1 %cmp, float %1) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[NEXP:%.*]], float [[TMP0:%.*]], i1 [[CMP:%.*]], float [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> <float 0.000000e+00, float 0x7FF8000000000000, float poison, float poison>, float [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[TMP0]], i32 3
+; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[NEXP]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <2 x i32> <i32 3, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x float> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float poison>, float [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP10]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    br label %[[IF_END]]
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <4 x float> [ [[TMP11]], %[[IF_THEN]] ], [ [[TMP3]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi <2 x float> [ [[TMP8]], %[[IF_THEN]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x float> [ zeroinitializer, %[[IF_THEN]] ], [ <float 0x7FF8000000000000, float 1.000000e+00>, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x float> [ [[TMP7]], %[[IF_THEN]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP14]], <2 x float> <float poison, float 0.000000e+00>, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <2 x float> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul <2 x float> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <4 x float> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[CALL25:%.*]] = load volatile ptr, ptr null, align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = fadd <2 x float> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul <2 x float> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = fadd <2 x float> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = fmul <4 x float> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = fadd <4 x float> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x float> [[TMP23]], <4 x float> [[TMP24]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP26:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> <float 0.000000e+00, float 1.000000e+00, float poison, float poison>, <2 x float> [[TMP22]], i64 2)
+; CHECK-NEXT:    [[TMP27:%.*]] = fadd <4 x float> [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    store <4 x float> [[TMP27]], ptr [[CALL25]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %div.i41 = fmul float %0, 0.000000e+00
+  %2 = load float, ptr %nExp, align 4
+  %div.1.i.i = fmul float %2, 0.000000e+00
+  %div.2.i.i = fmul float %0, 0.000000e+00
+  br label %if.end
+
+if.end:
+  %3 = phi float [ %1, %if.then ], [ %0, %entry ]
+  %4 = phi float [ 0.000000e+00, %if.then ], [ %1, %entry ]
+  %5 = phi float [ 0.000000e+00, %if.then ], [ 0x7FF8000000000000, %entry ]
+  %6 = phi float [ 0.000000e+00, %if.then ], [ 1.000000e+00, %entry ]
+  %fa.sroa.9.0 = phi float [ %div.2.i.i, %if.then ], [ 0.000000e+00, %entry ]
+  %fa.sroa.7.0 = phi float [ %div.1.i.i, %if.then ], [ 0.000000e+00, %entry ]
+  %fa.sroa.0.0 = phi float [ %div.i41, %if.then ], [ 0.000000e+00, %entry ]
+  %mul.1.i.i58 = fmul float %fa.sroa.7.0, %6
+  %mul.2.i.i60 = fmul float %fa.sroa.9.0, %6
+  %mul.1.i.i.i63 = fmul float %fa.sroa.0.0, %5
+  %mul.2.i.i.i65 = fmul float %fa.sroa.0.0, 0.000000e+00
+  %mul.i66 = fmul float %fa.sroa.0.0, 0.000000e+00
+  %add.1.i.i = fadd float %mul.1.i.i58, %mul.1.i.i.i63
+  %add.2.i.i = fadd float %mul.2.i.i60, %mul.2.i.i.i65
+  %mul.1.i.i74 = fmul float %add.1.i.i, 0.000000e+00
+  %mul.2.i.i76 = fmul float %add.2.i.i, 0.000000e+00
+  %mul.i.i.i78 = fmul float %mul.i66, 0.000000e+00
+  %add.1.i.i85 = fadd float %mul.1.i.i74, 0.000000e+00
+  %add.2.i.i86 = fadd float %mul.2.i.i76, 0.000000e+00
+  %mul.i.i.i97 = fmul float %5, 0.000000e+00
+  %mul.1.i.i.i99 = fmul float %4, 0.000000e+00
+  %mul.2.i.i.i101 = fmul float %3, 0.000000e+00
+  %add.i.i103 = fadd float %mul.i.i.i97, 0.000000e+00
+  %add.1.i.i104 = fadd float %mul.1.i.i.i99, 0.000000e+00
+  %add.2.i.i105 = fadd float %mul.2.i.i.i101, 0.000000e+00
+  %add = fadd float %mul.i.i.i78, 0.000000e+00
+  %add.i = fadd float %add.i.i103, 1.000000e+00
+  %add.1.i = fadd float %add.1.i.i104, %add.1.i.i85
+  %add.2.i = fadd float %add.2.i.i105, %add.2.i.i86
+  %call25 = load volatile ptr, ptr null, align 8
+  store float %add, ptr %call25, align 4
+  %__trans_tmp_29.sroa.5.0.call25.sroa_idx = getelementptr i8, ptr %call25, i64 4
+  store float %add.i, ptr %__trans_tmp_29.sroa.5.0.call25.sroa_idx, align 4
+  %__trans_tmp_29.sroa.6.0.call25.sroa_idx = getelementptr i8, ptr %call25, i64 8
+  store float %add.1.i, ptr %__trans_tmp_29.sroa.6.0.call25.sroa_idx, align 4
+  %__trans_tmp_29.sroa.7.0.call25.sroa_idx = getelementptr i8, ptr %call25, i64 12
+  store float %add.2.i, ptr %__trans_tmp_29.sroa.7.0.call25.sroa_idx, align 4
+  ret void
+}

From 4903a7b77b56c7d9a650205b6e7dca46581c7134 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Wed, 16 Apr 2025 12:10:08 -0700
Subject: [PATCH 163/710] [ctxprof][nfc] Move profile annotator to Analysis
 (#135871)

This moves the utility that propagates counter values such that we can reuse it elsewhere. Specifically, in a subsequent patch, it'll be used to guide ICP: we need to prioritize promoting indirect calls that dominate larger portions of the dynamic instruction count. We can compare them based on the dynamic count of IR instructions, and we can get that early with this counter propagation logic.

The patch is mostly a move of the existing logic, with a pimpl - style implementation to hide all the current complexity.
---
 llvm/include/llvm/Analysis/CtxProfAnalysis.h  |  28 ++
 llvm/lib/Analysis/CtxProfAnalysis.cpp         | 372 +++++++++++++++++
 .../Instrumentation/PGOCtxProfFlattening.cpp  | 375 ++----------------
 3 files changed, 424 insertions(+), 351 deletions(-)

diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
index 6f1c3696ca78c..aa582cfef1ad1 100644
--- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h
+++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
@@ -157,6 +157,34 @@ class CtxProfAnalysisPrinterPass
   const PrintMode Mode;
 };
 
+/// Utility that propagates counter values to each basic block and to each edge
+/// when a basic block has more than one outgoing edge, using an adaptation of
+/// PGOUseFunc::populateCounters.
+// FIXME(mtrofin): look into factoring the code to share one implementation.
+class ProfileAnnotatorImpl;
+class ProfileAnnotator {
+  std::unique_ptr<ProfileAnnotatorImpl> PImpl;
+
+public:
+  ProfileAnnotator(const Function &F, ArrayRef<uint64_t> RawCounters);
+  uint64_t getBBCount(const BasicBlock &BB) const;
+
+  // Finds the true and false counts for the given select instruction. Returns
+  // false if the select doesn't have instrumentation or if the count of the
+  // parent BB is 0.
+  bool getSelectInstrProfile(SelectInst &SI, uint64_t &TrueCount,
+                             uint64_t &FalseCount) const;
+  // Clears Profile and populates it with the edge weights, in the same order as
+  // they need to appear in the MD_prof metadata. Also computes the max of those
+  // weights an returns it in MaxCount. Returs false if:
+  //   - the BB has less than 2 successors
+  //   - the counts are 0
+  bool getOutgoingBranchWeights(BasicBlock &BB,
+                                SmallVectorImpl<uint64_t> &Profile,
+                                uint64_t &MaxCount) const;
+  ~ProfileAnnotator();
+};
+
 /// Assign a GUID to functions as metadata. GUID calculation takes linkage into
 /// account, which may change especially through and after thinlto. By
 /// pre-computing and assigning as metadata, this mechanism is resilient to such
diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp
index d203e277546ea..391631e15aa89 100644
--- a/llvm/lib/Analysis/CtxProfAnalysis.cpp
+++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp
@@ -14,7 +14,9 @@
 #include "llvm/Analysis/CtxProfAnalysis.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/CFG.h"
 #include "llvm/IR/Analysis.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
@@ -22,6 +24,8 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include <deque>
+#include <memory>
 
 #define DEBUG_TYPE "ctx_prof"
 
@@ -46,6 +50,374 @@ static cl::opt<bool> ForceIsInSpecializedModule(
 
 const char *AssignGUIDPass::GUIDMetadataName = "guid";
 
+namespace llvm {
+class ProfileAnnotatorImpl final {
+  friend class ProfileAnnotator;
+  class BBInfo;
+  struct EdgeInfo {
+    BBInfo *const Src;
+    BBInfo *const Dest;
+    std::optional<uint64_t> Count;
+
+    explicit EdgeInfo(BBInfo &Src, BBInfo &Dest) : Src(&Src), Dest(&Dest) {}
+  };
+
+  class BBInfo {
+    std::optional<uint64_t> Count;
+    // OutEdges is dimensioned to match the number of terminator operands.
+    // Entries in the vector match the index in the terminator operand list. In
+    // some cases - see `shouldExcludeEdge` and its implementation - an entry
+    // will be nullptr.
+    // InEdges doesn't have the above constraint.
+    SmallVector<EdgeInfo *> OutEdges;
+    SmallVector<EdgeInfo *> InEdges;
+    size_t UnknownCountOutEdges = 0;
+    size_t UnknownCountInEdges = 0;
+
+    // Pass AssumeAllKnown when we try to propagate counts from edges to BBs -
+    // because all the edge counters must be known.
+    // Return std::nullopt if there were no edges to sum. The user can decide
+    // how to interpret that.
+    std::optional<uint64_t> getEdgeSum(const SmallVector<EdgeInfo *> &Edges,
+                                       bool AssumeAllKnown) const {
+      std::optional<uint64_t> Sum;
+      for (const auto *E : Edges) {
+        // `Edges` may be `OutEdges`, case in which `E` could be nullptr.
+        if (E) {
+          if (!Sum.has_value())
+            Sum = 0;
+          *Sum += (AssumeAllKnown ? *E->Count : E->Count.value_or(0U));
+        }
+      }
+      return Sum;
+    }
+
+    bool computeCountFrom(const SmallVector<EdgeInfo *> &Edges) {
+      assert(!Count.has_value());
+      Count = getEdgeSum(Edges, true);
+      return Count.has_value();
+    }
+
+    void setSingleUnknownEdgeCount(SmallVector<EdgeInfo *> &Edges) {
+      uint64_t KnownSum = getEdgeSum(Edges, false).value_or(0U);
+      uint64_t EdgeVal = *Count > KnownSum ? *Count - KnownSum : 0U;
+      EdgeInfo *E = nullptr;
+      for (auto *I : Edges)
+        if (I && !I->Count.has_value()) {
+          E = I;
+#ifdef NDEBUG
+          break;
+#else
+          assert((!E || E == I) &&
+                 "Expected exactly one edge to have an unknown count, "
+                 "found a second one");
+          continue;
+#endif
+        }
+      assert(E && "Expected exactly one edge to have an unknown count");
+      assert(!E->Count.has_value());
+      E->Count = EdgeVal;
+      assert(E->Src->UnknownCountOutEdges > 0);
+      assert(E->Dest->UnknownCountInEdges > 0);
+      --E->Src->UnknownCountOutEdges;
+      --E->Dest->UnknownCountInEdges;
+    }
+
+  public:
+    BBInfo(size_t NumInEdges, size_t NumOutEdges, std::optional<uint64_t> Count)
+        : Count(Count) {
+      // For in edges, we just want to pre-allocate enough space, since we know
+      // it at this stage. For out edges, we will insert edges at the indices
+      // corresponding to positions in this BB's terminator instruction, so we
+      // construct a default (nullptr values)-initialized vector. A nullptr edge
+      // corresponds to those that are excluded (see shouldExcludeEdge).
+      InEdges.reserve(NumInEdges);
+      OutEdges.resize(NumOutEdges);
+    }
+
+    bool tryTakeCountFromKnownOutEdges(const BasicBlock &BB) {
+      if (!UnknownCountOutEdges) {
+        return computeCountFrom(OutEdges);
+      }
+      return false;
+    }
+
+    bool tryTakeCountFromKnownInEdges(const BasicBlock &BB) {
+      if (!UnknownCountInEdges) {
+        return computeCountFrom(InEdges);
+      }
+      return false;
+    }
+
+    void addInEdge(EdgeInfo &Info) {
+      InEdges.push_back(&Info);
+      ++UnknownCountInEdges;
+    }
+
+    // For the out edges, we care about the position we place them in, which is
+    // the position in terminator instruction's list (at construction). Later,
+    // we build branch_weights metadata with edge frequency values matching
+    // these positions.
+    void addOutEdge(size_t Index, EdgeInfo &Info) {
+      OutEdges[Index] = &Info;
+      ++UnknownCountOutEdges;
+    }
+
+    bool hasCount() const { return Count.has_value(); }
+
+    uint64_t getCount() const { return *Count; }
+
+    bool trySetSingleUnknownInEdgeCount() {
+      if (UnknownCountInEdges == 1) {
+        setSingleUnknownEdgeCount(InEdges);
+        return true;
+      }
+      return false;
+    }
+
+    bool trySetSingleUnknownOutEdgeCount() {
+      if (UnknownCountOutEdges == 1) {
+        setSingleUnknownEdgeCount(OutEdges);
+        return true;
+      }
+      return false;
+    }
+    size_t getNumOutEdges() const { return OutEdges.size(); }
+
+    uint64_t getEdgeCount(size_t Index) const {
+      if (auto *E = OutEdges[Index])
+        return *E->Count;
+      return 0U;
+    }
+  };
+
+  const Function &F;
+  ArrayRef<uint64_t> Counters;
+  // To be accessed through getBBInfo() after construction.
+  std::map<const BasicBlock *, BBInfo> BBInfos;
+  std::vector<EdgeInfo> EdgeInfos;
+
+  // The only criteria for exclusion is faux suspend -> exit edges in presplit
+  // coroutines. The API serves for readability, currently.
+  bool shouldExcludeEdge(const BasicBlock &Src, const BasicBlock &Dest) const {
+    return llvm::isPresplitCoroSuspendExitEdge(Src, Dest);
+  }
+
+  BBInfo &getBBInfo(const BasicBlock &BB) { return BBInfos.find(&BB)->second; }
+
+  const BBInfo &getBBInfo(const BasicBlock &BB) const {
+    return BBInfos.find(&BB)->second;
+  }
+
+  // validation function after we propagate the counters: all BBs and edges'
+  // counters must have a value.
+  bool allCountersAreAssigned() const {
+    for (const auto &BBInfo : BBInfos)
+      if (!BBInfo.second.hasCount())
+        return false;
+    for (const auto &EdgeInfo : EdgeInfos)
+      if (!EdgeInfo.Count.has_value())
+        return false;
+    return true;
+  }
+
+  /// Check that all paths from the entry basic block that use edges with
+  /// non-zero counts arrive at a basic block with no successors (i.e. "exit")
+  bool allTakenPathsExit() const {
+    std::deque<const BasicBlock *> Worklist;
+    DenseSet<const BasicBlock *> Visited;
+    Worklist.push_back(&F.getEntryBlock());
+    bool HitExit = false;
+    while (!Worklist.empty()) {
+      const auto *BB = Worklist.front();
+      Worklist.pop_front();
+      if (!Visited.insert(BB).second)
+        continue;
+      if (succ_size(BB) == 0) {
+        if (isa<UnreachableInst>(BB->getTerminator()))
+          return false;
+        HitExit = true;
+        continue;
+      }
+      if (succ_size(BB) == 1) {
+        Worklist.push_back(BB->getUniqueSuccessor());
+        continue;
+      }
+      const auto &BBInfo = getBBInfo(*BB);
+      bool HasAWayOut = false;
+      for (auto I = 0U; I < BB->getTerminator()->getNumSuccessors(); ++I) {
+        const auto *Succ = BB->getTerminator()->getSuccessor(I);
+        if (!shouldExcludeEdge(*BB, *Succ)) {
+          if (BBInfo.getEdgeCount(I) > 0) {
+            HasAWayOut = true;
+            Worklist.push_back(Succ);
+          }
+        }
+      }
+      if (!HasAWayOut)
+        return false;
+    }
+    return HitExit;
+  }
+
+  bool allNonColdSelectsHaveProfile() const {
+    for (const auto &BB : F) {
+      if (getBBInfo(BB).getCount() > 0) {
+        for (const auto &I : BB) {
+          if (const auto *SI = dyn_cast<SelectInst>(&I)) {
+            if (const auto *Inst = CtxProfAnalysis::getSelectInstrumentation(
+                    *const_cast<SelectInst *>(SI))) {
+              auto Index = Inst->getIndex()->getZExtValue();
+              assert(Index < Counters.size());
+              if (Counters[Index] == 0)
+                return false;
+            }
+          }
+        }
+      }
+    }
+    return true;
+  }
+
+  // This is an adaptation of PGOUseFunc::populateCounters.
+  // FIXME(mtrofin): look into factoring the code to share one implementation.
+  void propagateCounterValues() {
+    bool KeepGoing = true;
+    while (KeepGoing) {
+      KeepGoing = false;
+      for (const auto &BB : F) {
+        auto &Info = getBBInfo(BB);
+        if (!Info.hasCount())
+          KeepGoing |= Info.tryTakeCountFromKnownOutEdges(BB) ||
+                       Info.tryTakeCountFromKnownInEdges(BB);
+        if (Info.hasCount()) {
+          KeepGoing |= Info.trySetSingleUnknownOutEdgeCount();
+          KeepGoing |= Info.trySetSingleUnknownInEdgeCount();
+        }
+      }
+    }
+    assert(allCountersAreAssigned() &&
+           "[ctx-prof] Expected all counters have been assigned.");
+    assert(allTakenPathsExit() &&
+           "[ctx-prof] Encountered a BB with more than one successor, where "
+           "all outgoing edges have a 0 count. This occurs in non-exiting "
+           "functions (message pumps, usually) which are not supported in the "
+           "contextual profiling case");
+    assert(allNonColdSelectsHaveProfile() &&
+           "[ctx-prof] All non-cold select instructions were expected to have "
+           "a profile.");
+  }
+
+public:
+  ProfileAnnotatorImpl(const Function &F, ArrayRef<uint64_t> Counters)
+      : F(F), Counters(Counters) {
+    assert(!F.isDeclaration());
+    assert(!Counters.empty());
+    size_t NrEdges = 0;
+    for (const auto &BB : F) {
+      std::optional<uint64_t> Count;
+      if (auto *Ins = CtxProfAnalysis::getBBInstrumentation(
+              const_cast<BasicBlock &>(BB))) {
+        auto Index = Ins->getIndex()->getZExtValue();
+        assert(Index < Counters.size() &&
+               "The index must be inside the counters vector by construction - "
+               "tripping this assertion indicates a bug in how the contextual "
+               "profile is managed by IPO transforms");
+        (void)Index;
+        Count = Counters[Ins->getIndex()->getZExtValue()];
+      } else if (isa<UnreachableInst>(BB.getTerminator())) {
+        // The program presumably didn't crash.
+        Count = 0;
+      }
+      auto [It, Ins] =
+          BBInfos.insert({&BB, {pred_size(&BB), succ_size(&BB), Count}});
+      (void)Ins;
+      assert(Ins && "We iterate through the function's BBs, no reason to "
+                    "insert one more than once");
+      NrEdges += llvm::count_if(successors(&BB), [&](const auto *Succ) {
+        return !shouldExcludeEdge(BB, *Succ);
+      });
+    }
+    // Pre-allocate the vector, we want references to its contents to be stable.
+    EdgeInfos.reserve(NrEdges);
+    for (const auto &BB : F) {
+      auto &Info = getBBInfo(BB);
+      for (auto I = 0U; I < BB.getTerminator()->getNumSuccessors(); ++I) {
+        const auto *Succ = BB.getTerminator()->getSuccessor(I);
+        if (!shouldExcludeEdge(BB, *Succ)) {
+          auto &EI = EdgeInfos.emplace_back(getBBInfo(BB), getBBInfo(*Succ));
+          Info.addOutEdge(I, EI);
+          getBBInfo(*Succ).addInEdge(EI);
+        }
+      }
+    }
+    assert(EdgeInfos.capacity() == NrEdges &&
+           "The capacity of EdgeInfos should have stayed unchanged it was "
+           "populated, because we need pointers to its contents to be stable");
+    propagateCounterValues();
+  }
+
+  uint64_t getBBCount(const BasicBlock &BB) { return getBBInfo(BB).getCount(); }
+};
+
+} // namespace llvm
+
+ProfileAnnotator::ProfileAnnotator(const Function &F,
+                                   ArrayRef<uint64_t> RawCounters)
+    : PImpl(std::make_unique<ProfileAnnotatorImpl>(F, RawCounters)) {}
+
+ProfileAnnotator::~ProfileAnnotator() = default;
+
+uint64_t ProfileAnnotator::getBBCount(const BasicBlock &BB) const {
+  return PImpl->getBBCount(BB);
+}
+
+bool ProfileAnnotator::getSelectInstrProfile(SelectInst &SI,
+                                             uint64_t &TrueCount,
+                                             uint64_t &FalseCount) const {
+  const auto &BBInfo = PImpl->getBBInfo(*SI.getParent());
+  TrueCount = FalseCount = 0;
+  if (BBInfo.getCount() == 0)
+    return false;
+
+  auto *Step = CtxProfAnalysis::getSelectInstrumentation(SI);
+  if (!Step)
+    return false;
+  auto Index = Step->getIndex()->getZExtValue();
+  assert(Index < PImpl->Counters.size() &&
+         "The index of the step instruction must be inside the "
+         "counters vector by "
+         "construction - tripping this assertion indicates a bug in "
+         "how the contextual profile is managed by IPO transforms");
+  auto TotalCount = BBInfo.getCount();
+  TrueCount = PImpl->Counters[Index];
+  FalseCount = (TotalCount > TrueCount ? TotalCount - TrueCount : 0U);
+  return true;
+}
+
+bool ProfileAnnotator::getOutgoingBranchWeights(
+    BasicBlock &BB, SmallVectorImpl<uint64_t> &Profile,
+    uint64_t &MaxCount) const {
+  Profile.clear();
+
+  if (succ_size(&BB) < 2)
+    return false;
+
+  auto *Term = BB.getTerminator();
+  Profile.resize(Term->getNumSuccessors());
+
+  const auto &BBInfo = PImpl->getBBInfo(BB);
+  MaxCount = 0;
+  for (unsigned SuccIdx = 0, Size = BBInfo.getNumOutEdges(); SuccIdx < Size;
+       ++SuccIdx) {
+    uint64_t EdgeCount = BBInfo.getEdgeCount(SuccIdx);
+    if (EdgeCount > MaxCount)
+      MaxCount = EdgeCount;
+    Profile[SuccIdx] = EdgeCount;
+  }
+  return MaxCount > 0;
+}
+
 PreservedAnalyses AssignGUIDPass::run(Module &M, ModuleAnalysisManager &MAM) {
   for (auto &F : M.functions()) {
     if (F.isDeclaration())
diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
index 508a41684ed20..e47c9ab75ffe1 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
@@ -45,358 +45,33 @@ using namespace llvm;
 
 namespace {
 
-class ProfileAnnotator final {
-  class BBInfo;
-  struct EdgeInfo {
-    BBInfo *const Src;
-    BBInfo *const Dest;
-    std::optional<uint64_t> Count;
+/// Assign branch weights and function entry count. Also update the PSI
+/// builder.
+void assignProfileData(Function &F, ArrayRef<uint64_t> RawCounters) {
+  assert(!RawCounters.empty());
+  ProfileAnnotator PA(F, RawCounters);
 
-    explicit EdgeInfo(BBInfo &Src, BBInfo &Dest) : Src(&Src), Dest(&Dest) {}
-  };
+  F.setEntryCount(RawCounters[0]);
+  SmallVector<uint64_t, 2> ProfileHolder;
 
-  class BBInfo {
-    std::optional<uint64_t> Count;
-    // OutEdges is dimensioned to match the number of terminator operands.
-    // Entries in the vector match the index in the terminator operand list. In
-    // some cases - see `shouldExcludeEdge` and its implementation - an entry
-    // will be nullptr.
-    // InEdges doesn't have the above constraint.
-    SmallVector<EdgeInfo *> OutEdges;
-    SmallVector<EdgeInfo *> InEdges;
-    size_t UnknownCountOutEdges = 0;
-    size_t UnknownCountInEdges = 0;
-
-    // Pass AssumeAllKnown when we try to propagate counts from edges to BBs -
-    // because all the edge counters must be known.
-    // Return std::nullopt if there were no edges to sum. The user can decide
-    // how to interpret that.
-    std::optional<uint64_t> getEdgeSum(const SmallVector<EdgeInfo *> &Edges,
-                                       bool AssumeAllKnown) const {
-      std::optional<uint64_t> Sum;
-      for (const auto *E : Edges) {
-        // `Edges` may be `OutEdges`, case in which `E` could be nullptr.
-        if (E) {
-          if (!Sum.has_value())
-            Sum = 0;
-          *Sum += (AssumeAllKnown ? *E->Count : E->Count.value_or(0U));
-        }
-      }
-      return Sum;
-    }
-
-    bool computeCountFrom(const SmallVector<EdgeInfo *> &Edges) {
-      assert(!Count.has_value());
-      Count = getEdgeSum(Edges, true);
-      return Count.has_value();
-    }
-
-    void setSingleUnknownEdgeCount(SmallVector<EdgeInfo *> &Edges) {
-      uint64_t KnownSum = getEdgeSum(Edges, false).value_or(0U);
-      uint64_t EdgeVal = *Count > KnownSum ? *Count - KnownSum : 0U;
-      EdgeInfo *E = nullptr;
-      for (auto *I : Edges)
-        if (I && !I->Count.has_value()) {
-          E = I;
-#ifdef NDEBUG
-          break;
-#else
-          assert((!E || E == I) &&
-                 "Expected exactly one edge to have an unknown count, "
-                 "found a second one");
-          continue;
-#endif
-        }
-      assert(E && "Expected exactly one edge to have an unknown count");
-      assert(!E->Count.has_value());
-      E->Count = EdgeVal;
-      assert(E->Src->UnknownCountOutEdges > 0);
-      assert(E->Dest->UnknownCountInEdges > 0);
-      --E->Src->UnknownCountOutEdges;
-      --E->Dest->UnknownCountInEdges;
-    }
-
-  public:
-    BBInfo(size_t NumInEdges, size_t NumOutEdges, std::optional<uint64_t> Count)
-        : Count(Count) {
-      // For in edges, we just want to pre-allocate enough space, since we know
-      // it at this stage. For out edges, we will insert edges at the indices
-      // corresponding to positions in this BB's terminator instruction, so we
-      // construct a default (nullptr values)-initialized vector. A nullptr edge
-      // corresponds to those that are excluded (see shouldExcludeEdge).
-      InEdges.reserve(NumInEdges);
-      OutEdges.resize(NumOutEdges);
-    }
-
-    bool tryTakeCountFromKnownOutEdges(const BasicBlock &BB) {
-      if (!UnknownCountOutEdges) {
-        return computeCountFrom(OutEdges);
-      }
-      return false;
-    }
-
-    bool tryTakeCountFromKnownInEdges(const BasicBlock &BB) {
-      if (!UnknownCountInEdges) {
-        return computeCountFrom(InEdges);
-      }
-      return false;
-    }
-
-    void addInEdge(EdgeInfo &Info) {
-      InEdges.push_back(&Info);
-      ++UnknownCountInEdges;
-    }
-
-    // For the out edges, we care about the position we place them in, which is
-    // the position in terminator instruction's list (at construction). Later,
-    // we build branch_weights metadata with edge frequency values matching
-    // these positions.
-    void addOutEdge(size_t Index, EdgeInfo &Info) {
-      OutEdges[Index] = &Info;
-      ++UnknownCountOutEdges;
-    }
-
-    bool hasCount() const { return Count.has_value(); }
-
-    uint64_t getCount() const { return *Count; }
-
-    bool trySetSingleUnknownInEdgeCount() {
-      if (UnknownCountInEdges == 1) {
-        setSingleUnknownEdgeCount(InEdges);
-        return true;
-      }
-      return false;
-    }
-
-    bool trySetSingleUnknownOutEdgeCount() {
-      if (UnknownCountOutEdges == 1) {
-        setSingleUnknownEdgeCount(OutEdges);
-        return true;
-      }
-      return false;
-    }
-    size_t getNumOutEdges() const { return OutEdges.size(); }
-
-    uint64_t getEdgeCount(size_t Index) const {
-      if (auto *E = OutEdges[Index])
-        return *E->Count;
-      return 0U;
-    }
-  };
-
-  Function &F;
-  const SmallVectorImpl<uint64_t> &Counters;
-  // To be accessed through getBBInfo() after construction.
-  std::map<const BasicBlock *, BBInfo> BBInfos;
-  std::vector<EdgeInfo> EdgeInfos;
-
-  // This is an adaptation of PGOUseFunc::populateCounters.
-  // FIXME(mtrofin): look into factoring the code to share one implementation.
-  void propagateCounterValues(const SmallVectorImpl<uint64_t> &Counters) {
-    bool KeepGoing = true;
-    while (KeepGoing) {
-      KeepGoing = false;
-      for (const auto &BB : F) {
-        auto &Info = getBBInfo(BB);
-        if (!Info.hasCount())
-          KeepGoing |= Info.tryTakeCountFromKnownOutEdges(BB) ||
-                       Info.tryTakeCountFromKnownInEdges(BB);
-        if (Info.hasCount()) {
-          KeepGoing |= Info.trySetSingleUnknownOutEdgeCount();
-          KeepGoing |= Info.trySetSingleUnknownInEdgeCount();
-        }
-      }
-    }
-  }
-  // The only criteria for exclusion is faux suspend -> exit edges in presplit
-  // coroutines. The API serves for readability, currently.
-  bool shouldExcludeEdge(const BasicBlock &Src, const BasicBlock &Dest) const {
-    return llvm::isPresplitCoroSuspendExitEdge(Src, Dest);
-  }
-
-  BBInfo &getBBInfo(const BasicBlock &BB) { return BBInfos.find(&BB)->second; }
-
-  const BBInfo &getBBInfo(const BasicBlock &BB) const {
-    return BBInfos.find(&BB)->second;
-  }
-
-  // validation function after we propagate the counters: all BBs and edges'
-  // counters must have a value.
-  bool allCountersAreAssigned() const {
-    for (const auto &BBInfo : BBInfos)
-      if (!BBInfo.second.hasCount())
-        return false;
-    for (const auto &EdgeInfo : EdgeInfos)
-      if (!EdgeInfo.Count.has_value())
-        return false;
-    return true;
-  }
-
-  /// Check that all paths from the entry basic block that use edges with
-  /// non-zero counts arrive at a basic block with no successors (i.e. "exit")
-  bool allTakenPathsExit() const {
-    std::deque<const BasicBlock *> Worklist;
-    DenseSet<const BasicBlock *> Visited;
-    Worklist.push_back(&F.getEntryBlock());
-    bool HitExit = false;
-    while (!Worklist.empty()) {
-      const auto *BB = Worklist.front();
-      Worklist.pop_front();
-      if (!Visited.insert(BB).second)
-        continue;
-      if (succ_size(BB) == 0) {
-        if (isa<UnreachableInst>(BB->getTerminator()))
-          return false;
-        HitExit = true;
-        continue;
-      }
-      if (succ_size(BB) == 1) {
-        Worklist.push_back(BB->getUniqueSuccessor());
-        continue;
-      }
-      const auto &BBInfo = getBBInfo(*BB);
-      bool HasAWayOut = false;
-      for (auto I = 0U; I < BB->getTerminator()->getNumSuccessors(); ++I) {
-        const auto *Succ = BB->getTerminator()->getSuccessor(I);
-        if (!shouldExcludeEdge(*BB, *Succ)) {
-          if (BBInfo.getEdgeCount(I) > 0) {
-            HasAWayOut = true;
-            Worklist.push_back(Succ);
-          }
-        }
-      }
-      if (!HasAWayOut)
-        return false;
-    }
-    return HitExit;
-  }
-
-  bool allNonColdSelectsHaveProfile() const {
-    for (const auto &BB : F) {
-      if (getBBInfo(BB).getCount() > 0) {
-        for (const auto &I : BB) {
-          if (const auto *SI = dyn_cast<SelectInst>(&I)) {
-            if (!SI->getMetadata(LLVMContext::MD_prof)) {
-              return false;
-            }
-          }
-        }
-      }
-    }
-    return true;
-  }
-
-public:
-  ProfileAnnotator(Function &F, const SmallVectorImpl<uint64_t> &Counters)
-      : F(F), Counters(Counters) {
-    assert(!F.isDeclaration());
-    assert(!Counters.empty());
-    size_t NrEdges = 0;
-    for (const auto &BB : F) {
-      std::optional<uint64_t> Count;
-      if (auto *Ins = CtxProfAnalysis::getBBInstrumentation(
-              const_cast<BasicBlock &>(BB))) {
-        auto Index = Ins->getIndex()->getZExtValue();
-        assert(Index < Counters.size() &&
-               "The index must be inside the counters vector by construction - "
-               "tripping this assertion indicates a bug in how the contextual "
-               "profile is managed by IPO transforms");
-        (void)Index;
-        Count = Counters[Ins->getIndex()->getZExtValue()];
-      } else if (isa<UnreachableInst>(BB.getTerminator())) {
-        // The program presumably didn't crash.
-        Count = 0;
-      }
-      auto [It, Ins] =
-          BBInfos.insert({&BB, {pred_size(&BB), succ_size(&BB), Count}});
-      (void)Ins;
-      assert(Ins && "We iterate through the function's BBs, no reason to "
-                    "insert one more than once");
-      NrEdges += llvm::count_if(successors(&BB), [&](const auto *Succ) {
-        return !shouldExcludeEdge(BB, *Succ);
-      });
-    }
-    // Pre-allocate the vector, we want references to its contents to be stable.
-    EdgeInfos.reserve(NrEdges);
-    for (const auto &BB : F) {
-      auto &Info = getBBInfo(BB);
-      for (auto I = 0U; I < BB.getTerminator()->getNumSuccessors(); ++I) {
-        const auto *Succ = BB.getTerminator()->getSuccessor(I);
-        if (!shouldExcludeEdge(BB, *Succ)) {
-          auto &EI = EdgeInfos.emplace_back(getBBInfo(BB), getBBInfo(*Succ));
-          Info.addOutEdge(I, EI);
-          getBBInfo(*Succ).addInEdge(EI);
-        }
-      }
-    }
-    assert(EdgeInfos.capacity() == NrEdges &&
-           "The capacity of EdgeInfos should have stayed unchanged it was "
-           "populated, because we need pointers to its contents to be stable");
-  }
-
-  void setProfileForSelectInstructions(BasicBlock &BB, const BBInfo &BBInfo) {
-    if (BBInfo.getCount() == 0)
-      return;
-
-    for (auto &I : BB) {
+  for (auto &BB : F) {
+    for (auto &I : BB)
       if (auto *SI = dyn_cast<SelectInst>(&I)) {
-        if (auto *Step = CtxProfAnalysis::getSelectInstrumentation(*SI)) {
-          auto Index = Step->getIndex()->getZExtValue();
-          assert(Index < Counters.size() &&
-                 "The index of the step instruction must be inside the "
-                 "counters vector by "
-                 "construction - tripping this assertion indicates a bug in "
-                 "how the contextual profile is managed by IPO transforms");
-          auto TotalCount = BBInfo.getCount();
-          auto TrueCount = Counters[Index];
-          auto FalseCount =
-              (TotalCount > TrueCount ? TotalCount - TrueCount : 0U);
-          setProfMetadata(F.getParent(), SI, {TrueCount, FalseCount},
-                          std::max(TrueCount, FalseCount));
-        }
-      }
-    }
-  }
-
-  /// Assign branch weights and function entry count. Also update the PSI
-  /// builder.
-  void assignProfileData() {
-    assert(!Counters.empty());
-    propagateCounterValues(Counters);
-    F.setEntryCount(Counters[0]);
-
-    for (auto &BB : F) {
-      const auto &BBInfo = getBBInfo(BB);
-      setProfileForSelectInstructions(BB, BBInfo);
-      if (succ_size(&BB) < 2)
-        continue;
-      auto *Term = BB.getTerminator();
-      SmallVector<uint64_t, 2> EdgeCounts(Term->getNumSuccessors(), 0);
-      uint64_t MaxCount = 0;
-
-      for (unsigned SuccIdx = 0, Size = BBInfo.getNumOutEdges(); SuccIdx < Size;
-           ++SuccIdx) {
-        uint64_t EdgeCount = BBInfo.getEdgeCount(SuccIdx);
-        if (EdgeCount > MaxCount)
-          MaxCount = EdgeCount;
-        EdgeCounts[SuccIdx] = EdgeCount;
+        uint64_t TrueCount, FalseCount = 0;
+        if (!PA.getSelectInstrProfile(*SI, TrueCount, FalseCount))
+          continue;
+        setProfMetadata(F.getParent(), SI, {TrueCount, FalseCount},
+                        std::max(TrueCount, FalseCount));
       }
-
-      if (MaxCount != 0)
-        setProfMetadata(F.getParent(), Term, EdgeCounts, MaxCount);
-    }
-    assert(allCountersAreAssigned() &&
-           "[ctx-prof] Expected all counters have been assigned.");
-    assert(allTakenPathsExit() &&
-           "[ctx-prof] Encountered a BB with more than one successor, where "
-           "all outgoing edges have a 0 count. This occurs in non-exiting "
-           "functions (message pumps, usually) which are not supported in the "
-           "contextual profiling case");
-    assert(allNonColdSelectsHaveProfile() &&
-           "[ctx-prof] All non-cold select instructions were expected to have "
-           "a profile.");
+    if (succ_size(&BB) < 2)
+      continue;
+    uint64_t MaxCount = 0;
+    if (!PA.getOutgoingBranchWeights(BB, ProfileHolder, MaxCount))
+      continue;
+    assert(MaxCount > 0);
+    setProfMetadata(F.getParent(), BB.getTerminator(), ProfileHolder, MaxCount);
   }
-};
+}
 
 [[maybe_unused]] bool areAllBBsReachable(const Function &F,
                                          FunctionAnalysisManager &FAM) {
@@ -510,10 +185,8 @@ PreservedAnalyses PGOCtxProfFlatteningPass::run(Module &M,
     // If this function didn't appear in the contextual profile, it's cold.
     if (It == FlattenedProfile.end())
       clearColdFunctionProfile(F);
-    else {
-      ProfileAnnotator S(F, It->second);
-      S.assignProfileData();
-    }
+    else
+      assignProfileData(F, It->second);
   }
   InstrProfSummaryBuilder PB(ProfileSummaryBuilder::DefaultCutoffs);
   // use here the flat profiles just so the importer doesn't complain about

From fe94f11407453c2d166597ef6e58d31f5b27d46e Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Wed, 16 Apr 2025 16:27:24 -0300
Subject: [PATCH 164/710] [clang] Fix elaborated keyword canonicalization
 (#135916)

---
 .../modernize/UseConstraintsCheck.cpp         |  8 ++-
 clang/docs/ReleaseNotes.rst                   |  4 ++
 clang/include/clang/AST/Type.h                | 14 +++++
 clang/lib/AST/ASTContext.cpp                  | 49 +++++++++++----
 clang/lib/AST/Type.cpp                        |  8 +++
 clang/lib/AST/TypeLoc.cpp                     | 44 ++++++++-----
 clang/lib/Sema/SemaDecl.cpp                   |  6 +-
 clang/lib/Sema/SemaDeclCXX.cpp                | 60 +++++++++++-------
 clang/lib/Sema/SemaInit.cpp                   |  7 ++-
 clang/lib/Sema/SemaTemplate.cpp               | 18 +++---
 clang/test/Analysis/anonymous-decls.cpp       |  4 +-
 clang/test/CXX/drs/cwg23xx.cpp                |  4 +-
 .../dependent-template-recover.cpp            |  4 +-
 .../elaborated-type-specifier.cpp             | 63 ++++++++++++++++++-
 .../SemaTemplate/typename-specifier-3.cpp     | 18 ++++++
 15 files changed, 234 insertions(+), 77 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
index fb82efb4dd211..6040cddf0e52a 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
@@ -54,8 +54,10 @@ static std::optional<TemplateSpecializationTypeLoc>
 matchEnableIfSpecializationImplTypename(TypeLoc TheType) {
   if (const auto Dep = TheType.getAs<DependentNameTypeLoc>()) {
     const IdentifierInfo *Identifier = Dep.getTypePtr()->getIdentifier();
+    ElaboratedTypeKeyword Keyword = Dep.getTypePtr()->getKeyword();
     if (!Identifier || Identifier->getName() != "type" ||
-        Dep.getTypePtr()->getKeyword() != ElaboratedTypeKeyword::Typename) {
+        (Keyword != ElaboratedTypeKeyword::Typename &&
+         Keyword != ElaboratedTypeKeyword::None)) {
       return std::nullopt;
     }
     TheType = Dep.getQualifierLoc().getTypeLoc();
@@ -108,8 +110,10 @@ matchEnableIfSpecializationImplTrait(TypeLoc TheType) {
 
     if (const auto *AliasedType =
             dyn_cast<DependentNameType>(Specialization->getAliasedType())) {
+      ElaboratedTypeKeyword Keyword = AliasedType->getKeyword();
       if (AliasedType->getIdentifier()->getName() != "type" ||
-          AliasedType->getKeyword() != ElaboratedTypeKeyword::Typename) {
+          (Keyword != ElaboratedTypeKeyword::Typename &&
+           Keyword != ElaboratedTypeKeyword::None)) {
         return std::nullopt;
       }
     } else {
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0891fd058bb57..07ff1251fc1ad 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -486,6 +486,10 @@ Bug Fixes to C++ Support
 - Fixes matching of nested template template parameters. (#GH130362)
 - Correctly diagnoses template template paramters which have a pack parameter
   not in the last position.
+- Disallow overloading on struct vs class on dependent types, which is IFNDR, as
+  this makes the problem diagnosable.
+- Improved preservation of the presence or abscence of typename specifier when
+  printing types in diagnostics.
 - Clang now correctly parses ``if constexpr`` expressions in immediate function context. (#GH123524)
 - Fixed an assertion failure affecting code that uses C++23 "deducing this". (#GH130272)
 - Clang now properly instantiates destructors for initialized members within non-delegating constructors. (#GH93251)
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 5bf036e3347eb..1ecd64539e2de 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -2838,6 +2838,20 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
   /// immediately following this class.
   template <typename T> const T *getAs() const;
 
+  /// Look through sugar for an instance of TemplateSpecializationType which
+  /// is not a type alias, or null if there is no such type.
+  /// This is used when you want as-written template arguments or the template
+  /// name for a class template specialization.
+  const TemplateSpecializationType *
+  getAsNonAliasTemplateSpecializationType() const;
+
+  const TemplateSpecializationType *
+  castAsNonAliasTemplateSpecializationType() const {
+    const auto *TST = getAsNonAliasTemplateSpecializationType();
+    assert(TST && "not a TemplateSpecializationType");
+    return TST;
+  }
+
   /// Member-template getAsAdjusted<specific type>. Look through specific kinds
   /// of sugar (parens, attributes, etc) for an instance of \<specific type>.
   /// This is used when you need to walk over sugar nodes that represent some
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index c6ffe7bbf5257..bf24704e48eaa 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -5747,6 +5747,30 @@ ASTContext::getMacroQualifiedType(QualType UnderlyingTy,
   return QualType(newType, 0);
 }
 
+static ElaboratedTypeKeyword
+getCanonicalElaboratedTypeKeyword(ElaboratedTypeKeyword Keyword) {
+  switch (Keyword) {
+  // These are just themselves.
+  case ElaboratedTypeKeyword::None:
+  case ElaboratedTypeKeyword::Struct:
+  case ElaboratedTypeKeyword::Union:
+  case ElaboratedTypeKeyword::Enum:
+  case ElaboratedTypeKeyword::Interface:
+    return Keyword;
+
+  // These are equivalent.
+  case ElaboratedTypeKeyword::Typename:
+    return ElaboratedTypeKeyword::None;
+
+  // These are functionally equivalent, so relying on their equivalence is
+  // IFNDR. By making them equivalent, we disallow overloading, which at least
+  // can produce a diagnostic.
+  case ElaboratedTypeKeyword::Class:
+    return ElaboratedTypeKeyword::Struct;
+  }
+  llvm_unreachable("unexpected keyword kind");
+}
+
 QualType ASTContext::getDependentNameType(ElaboratedTypeKeyword Keyword,
                                           NestedNameSpecifier *NNS,
                                           const IdentifierInfo *Name) const {
@@ -5758,10 +5782,13 @@ QualType ASTContext::getDependentNameType(ElaboratedTypeKeyword Keyword,
           DependentNameTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(T, 0);
 
+  ElaboratedTypeKeyword CanonKeyword =
+      getCanonicalElaboratedTypeKeyword(Keyword);
+  NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
+
   QualType Canon;
-  if (NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
-      CanonNNS != NNS) {
-    Canon = getDependentNameType(Keyword, CanonNNS, Name);
+  if (CanonKeyword != Keyword || CanonNNS != NNS) {
+    Canon = getDependentNameType(CanonKeyword, CanonNNS, Name);
     [[maybe_unused]] DependentNameType *T =
         DependentNameTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!T && "broken canonicalization");
@@ -5800,19 +5827,19 @@ QualType ASTContext::getDependentTemplateSpecializationType(
 
   QualType Canon;
   if (!IsCanonical) {
-    ElaboratedTypeKeyword CanonKeyword = Keyword != ElaboratedTypeKeyword::None
-                                             ? Keyword
-                                             : ElaboratedTypeKeyword::Typename;
+    ElaboratedTypeKeyword CanonKeyword =
+        getCanonicalElaboratedTypeKeyword(Keyword);
     NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
     bool AnyNonCanonArgs = false;
     auto CanonArgs =
         ::getCanonicalTemplateArguments(*this, Args, AnyNonCanonArgs);
 
-    if (AnyNonCanonArgs || CanonNNS != NNS || !Name.hasTemplateKeyword() ||
-        CanonKeyword != Keyword) {
+    if (CanonKeyword != Keyword || AnyNonCanonArgs || CanonNNS != NNS ||
+        !Name.hasTemplateKeyword()) {
       Canon = getDependentTemplateSpecializationType(
           CanonKeyword, {CanonNNS, Name.getName(), /*HasTemplateKeyword=*/true},
-          CanonArgs, /*IsCanonical=*/true);
+          CanonArgs,
+          /*IsCanonical=*/true);
       // Find the insert position again.
       [[maybe_unused]] auto *Nothing =
           DependentTemplateSpecializationTypes.FindNodeOrInsertPos(ID,
@@ -5820,7 +5847,7 @@ QualType ASTContext::getDependentTemplateSpecializationType(
       assert(!Nothing && "canonical type broken");
     }
   } else {
-    assert(Keyword != ElaboratedTypeKeyword::None);
+    assert(Keyword == getCanonicalElaboratedTypeKeyword(Keyword));
     assert(Name.hasTemplateKeyword());
     assert(NNS == getCanonicalNestedNameSpecifier(NNS));
 #ifndef NDEBUG
@@ -7657,7 +7684,7 @@ ASTContext::getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) const {
     if (const auto *DTST = T->getAs<DependentTemplateSpecializationType>()) {
       const DependentTemplateStorage &DTN = DTST->getDependentTemplateName();
       QualType NewT = getDependentTemplateSpecializationType(
-          ElaboratedTypeKeyword::Typename,
+          ElaboratedTypeKeyword::None,
           {/*NNS=*/nullptr, DTN.getName(), /*HasTemplateKeyword=*/true},
           DTST->template_arguments(), /*IsCanonical=*/true);
       assert(NewT.isCanonical());
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 53620003c9655..42e94d66d1a13 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -1938,6 +1938,14 @@ TagDecl *Type::getAsTagDecl() const {
   return nullptr;
 }
 
+const TemplateSpecializationType *
+Type::getAsNonAliasTemplateSpecializationType() const {
+  const auto *TST = getAs<TemplateSpecializationType>();
+  while (TST && TST->isTypeAlias())
+    TST = TST->desugar()->getAs<TemplateSpecializationType>();
+  return TST;
+}
+
 bool Type::hasAttr(attr::Kind AK) const {
   const Type *Cur = this;
   while (const auto *AT = Cur->getAs<AttributedType>()) {
diff --git a/clang/lib/AST/TypeLoc.cpp b/clang/lib/AST/TypeLoc.cpp
index 24726901b8f55..3d1b5ca966b66 100644
--- a/clang/lib/AST/TypeLoc.cpp
+++ b/clang/lib/AST/TypeLoc.cpp
@@ -546,37 +546,47 @@ void UnaryTransformTypeLoc::initializeLocal(ASTContext &Context,
         Context.getTrivialTypeSourceInfo(getTypePtr()->getBaseType(), Loc));
 }
 
+template <class TL>
+static void initializeElaboratedKeyword(TL T, SourceLocation Loc) {
+  T.setElaboratedKeywordLoc(T.getTypePtr()->getKeyword() !=
+                                    ElaboratedTypeKeyword::None
+                                ? Loc
+                                : SourceLocation());
+}
+
+static NestedNameSpecifierLoc
+initializeQualifier(ASTContext &Context, NestedNameSpecifier *Qualifier,
+                    SourceLocation Loc) {
+  if (!Qualifier)
+    return NestedNameSpecifierLoc();
+  NestedNameSpecifierLocBuilder Builder;
+  Builder.MakeTrivial(Context, Qualifier, Loc);
+  return Builder.getWithLocInContext(Context);
+}
+
 void ElaboratedTypeLoc::initializeLocal(ASTContext &Context,
                                         SourceLocation Loc) {
   if (isEmpty())
     return;
-  setElaboratedKeywordLoc(Loc);
-  NestedNameSpecifierLocBuilder Builder;
-  Builder.MakeTrivial(Context, getTypePtr()->getQualifier(), Loc);
-  setQualifierLoc(Builder.getWithLocInContext(Context));
+  initializeElaboratedKeyword(*this, Loc);
+  setQualifierLoc(
+      initializeQualifier(Context, getTypePtr()->getQualifier(), Loc));
 }
 
 void DependentNameTypeLoc::initializeLocal(ASTContext &Context,
                                            SourceLocation Loc) {
-  setElaboratedKeywordLoc(Loc);
-  NestedNameSpecifierLocBuilder Builder;
-  Builder.MakeTrivial(Context, getTypePtr()->getQualifier(), Loc);
-  setQualifierLoc(Builder.getWithLocInContext(Context));
+  initializeElaboratedKeyword(*this, Loc);
+  setQualifierLoc(
+      initializeQualifier(Context, getTypePtr()->getQualifier(), Loc));
   setNameLoc(Loc);
 }
 
 void
 DependentTemplateSpecializationTypeLoc::initializeLocal(ASTContext &Context,
                                                         SourceLocation Loc) {
-  setElaboratedKeywordLoc(Loc);
-  if (NestedNameSpecifier *Qualifier =
-          getTypePtr()->getDependentTemplateName().getQualifier()) {
-    NestedNameSpecifierLocBuilder Builder;
-    Builder.MakeTrivial(Context, Qualifier, Loc);
-    setQualifierLoc(Builder.getWithLocInContext(Context));
-  } else {
-    setQualifierLoc(NestedNameSpecifierLoc());
-  }
+  initializeElaboratedKeyword(*this, Loc);
+  setQualifierLoc(initializeQualifier(
+      Context, getTypePtr()->getDependentTemplateName().getQualifier(), Loc));
   setTemplateKeywordLoc(Loc);
   setTemplateNameLoc(Loc);
   setLAngleLoc(Loc);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 127c0a4500a43..46933c5c43168 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -247,15 +247,15 @@ static ParsedType recoverFromTypeInKnownDependentBase(Sema &S,
     return nullptr;
 
   // We found some types in dependent base classes.  Recover as if the user
-  // wrote 'typename MyClass::II' instead of 'II'.  We'll fully resolve the
-  // lookup during template instantiation.
+  // wrote 'MyClass::II' instead of 'II', and this implicit typename was
+  // allowed.  We'll fully resolve the lookup during template instantiation.
   S.Diag(NameLoc, diag::ext_found_in_dependent_base) << &II;
 
   ASTContext &Context = S.Context;
   auto *NNS = NestedNameSpecifier::Create(
       Context, nullptr, cast<Type>(Context.getRecordType(RD)));
   QualType T =
-      Context.getDependentNameType(ElaboratedTypeKeyword::Typename, NNS, &II);
+      Context.getDependentNameType(ElaboratedTypeKeyword::None, NNS, &II);
 
   CXXScopeSpec SS;
   SS.MakeTrivial(Context, NNS, SourceRange(NameLoc));
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 05991228dbfc2..d4e48a14d13c2 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -12154,33 +12154,31 @@ static bool isStdClassTemplate(Sema &S, QualType SugaredType, QualType *TypeArg,
   };
 
   ClassTemplateDecl *Template = nullptr;
-  const TemplateArgument *Arguments = nullptr;
-
-  QualType Ty = S.Context.getCanonicalType(SugaredType);
-  if (const RecordType *RT = Ty->getAs<RecordType>()) {
-    ClassTemplateSpecializationDecl *Specialization =
-        dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl());
-    if (!Specialization) {
-      ReportMatchingNameAsMalformed(RT->getDecl());
-      return false;
-    }
-
-    Template = Specialization->getSpecializedTemplate();
-    Arguments = Specialization->getTemplateArgs().data();
-  } else {
-    const TemplateSpecializationType *TST = nullptr;
-    if (auto *ICN = Ty->getAs<InjectedClassNameType>())
-      TST = ICN->getInjectedTST();
-    else
-      TST = Ty->getAs<TemplateSpecializationType>();
+  ArrayRef<TemplateArgument> Arguments;
+  {
+    const TemplateSpecializationType *TST =
+        SugaredType->getAsNonAliasTemplateSpecializationType();
+    if (!TST)
+      if (const auto *ICN = SugaredType->getAs<InjectedClassNameType>())
+        TST = ICN->getInjectedTST();
     if (TST) {
       Template = dyn_cast_or_null<ClassTemplateDecl>(
           TST->getTemplateName().getAsTemplateDecl());
-      Arguments = TST->template_arguments().begin();
+      Arguments = TST->template_arguments();
+    } else if (const RecordType *RT = SugaredType->getAs<RecordType>()) {
+      ClassTemplateSpecializationDecl *Specialization =
+          dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl());
+      if (!Specialization) {
+        ReportMatchingNameAsMalformed(RT->getDecl());
+        return false;
+      }
+      Template = Specialization->getSpecializedTemplate();
+      Arguments = Specialization->getTemplateArgs().asArray();
     }
   }
+
   if (!Template) {
-    ReportMatchingNameAsMalformed(Ty->getAsTagDecl());
+    ReportMatchingNameAsMalformed(SugaredType->getAsTagDecl());
     return false;
   }
 
@@ -12200,7 +12198,8 @@ static bool isStdClassTemplate(Sema &S, QualType SugaredType, QualType *TypeArg,
     // template?
     TemplateParameterList *Params = Template->getTemplateParameters();
     if (Params->getMinRequiredArguments() != 1 ||
-        !isa<TemplateTypeParmDecl>(Params->getParam(0))) {
+        !isa<TemplateTypeParmDecl>(Params->getParam(0)) ||
+        Params->getParam(0)->isTemplateParameterPack()) {
       if (MalformedDecl)
         *MalformedDecl = TemplateClass;
       return false;
@@ -12214,8 +12213,21 @@ static bool isStdClassTemplate(Sema &S, QualType SugaredType, QualType *TypeArg,
     return false;
 
   // This is an instance of std::{ClassName}. Find the argument type.
-  if (TypeArg)
-    *TypeArg = Arguments[0].getAsType();
+  if (TypeArg) {
+    QualType ArgType = Arguments[0].getAsType();
+    // FIXME: Since TST only has as-written arguments, we have to perform the
+    // only kind of conversion applicable to type arguments; in Objective-C ARC:
+    // - If an explicitly-specified template argument type is a lifetime type
+    //   with no lifetime qualifier, the __strong lifetime qualifier is
+    //   inferred.
+    if (S.getLangOpts().ObjCAutoRefCount && ArgType->isObjCLifetimeType() &&
+        !ArgType.getObjCLifetime()) {
+      Qualifiers Qs;
+      Qs.setObjCLifetime(Qualifiers::OCL_Strong);
+      ArgType = S.Context.getQualifiedType(ArgType, Qs);
+    }
+    *TypeArg = ArgType;
+  }
 
   return true;
 }
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index a1e4bb4321d53..77d7f821f2011 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -9897,7 +9897,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
 
   auto TemplateName = DeducedTST->getTemplateName();
   if (TemplateName.isDependent())
-    return SubstAutoTypeDependent(TSInfo->getType());
+    return SubstAutoTypeSourceInfoDependent(TSInfo)->getType();
 
   // We can only perform deduction for class templates or alias templates.
   auto *Template =
@@ -9942,7 +9942,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     Diag(TSInfo->getTypeLoc().getBeginLoc(),
          diag::warn_cxx14_compat_class_template_argument_deduction)
         << TSInfo->getTypeLoc().getSourceRange() << 0;
-    return SubstAutoTypeDependent(TSInfo->getType());
+    return SubstAutoTypeSourceInfoDependent(TSInfo)->getType();
   }
 
   // FIXME: Perform "exact type" matching first, per CWG discussion?
@@ -10253,7 +10253,8 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
   //  The placeholder is replaced by the return type of the function selected
   //  by overload resolution for class template deduction.
   QualType DeducedType =
-      SubstAutoType(TSInfo->getType(), Best->Function->getReturnType());
+      SubstAutoTypeSourceInfo(TSInfo, Best->Function->getReturnType())
+          ->getType();
   Diag(TSInfo->getTypeLoc().getBeginLoc(),
        diag::warn_cxx14_compat_class_template_argument_deduction)
       << TSInfo->getTypeLoc().getSourceRange() << 1 << DeducedType;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 6b7892fa30989..894f072d84989 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -4892,7 +4892,7 @@ bool Sema::CheckTemplateTypeArgument(
 
         // Recover by synthesizing a type using the location information that we
         // already have.
-        ArgType = Context.getDependentNameType(ElaboratedTypeKeyword::Typename,
+        ArgType = Context.getDependentNameType(ElaboratedTypeKeyword::None,
                                                SS.getScopeRep(), II);
         TypeLocBuilder TLB;
         DependentNameTypeLoc TL = TLB.push<DependentNameTypeLoc>(ArgType);
@@ -10672,10 +10672,8 @@ TypeResult Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
   NestedNameSpecifierLoc QualifierLoc = SS.getWithLocInContext(Context);
   TypeSourceInfo *TSI = nullptr;
   QualType T =
-      CheckTypenameType((TypenameLoc.isValid() ||
-                         IsImplicitTypename == ImplicitTypenameContext::Yes)
-                            ? ElaboratedTypeKeyword::Typename
-                            : ElaboratedTypeKeyword::None,
+      CheckTypenameType(TypenameLoc.isValid() ? ElaboratedTypeKeyword::Typename
+                                              : ElaboratedTypeKeyword::None,
                         TypenameLoc, QualifierLoc, II, IdLoc, &TSI,
                         /*DeducedTSTContext=*/true);
   if (T.isNull())
@@ -10713,6 +10711,9 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
   TemplateArgumentListInfo TemplateArgs(LAngleLoc, RAngleLoc);
   translateTemplateArguments(TemplateArgsIn, TemplateArgs);
 
+  auto Keyword = TypenameLoc.isValid() ? ElaboratedTypeKeyword::Typename
+                                       : ElaboratedTypeKeyword::None;
+
   TemplateName Template = TemplateIn.get();
   if (DependentTemplateName *DTN = Template.getAsDependentTemplateName()) {
     // Construct a dependent template specialization type.
@@ -10726,7 +10727,7 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
     }
 
     QualType T = Context.getDependentTemplateSpecializationType(
-        ElaboratedTypeKeyword::Typename, *DTN, TemplateArgs.arguments());
+        Keyword, *DTN, TemplateArgs.arguments());
 
     // Create source-location information for this type.
     TypeLocBuilder Builder;
@@ -10758,8 +10759,7 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
   for (unsigned I = 0, N = TemplateArgs.size(); I != N; ++I)
     SpecTL.setArgLocInfo(I, TemplateArgs[I].getLocInfo());
 
-  T = Context.getElaboratedType(ElaboratedTypeKeyword::Typename,
-                                SS.getScopeRep(), T);
+  T = Context.getElaboratedType(Keyword, SS.getScopeRep(), T);
   ElaboratedTypeLoc TL = Builder.push<ElaboratedTypeLoc>(T);
   TL.setElaboratedKeywordLoc(TypenameLoc);
   TL.setQualifierLoc(SS.getWithLocInContext(Context));
@@ -10853,6 +10853,8 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword,
                         NestedNameSpecifierLoc QualifierLoc,
                         const IdentifierInfo &II,
                         SourceLocation IILoc, bool DeducedTSTContext) {
+  assert((Keyword != ElaboratedTypeKeyword::None) == KeywordLoc.isValid());
+
   CXXScopeSpec SS;
   SS.Adopt(QualifierLoc);
 
diff --git a/clang/test/Analysis/anonymous-decls.cpp b/clang/test/Analysis/anonymous-decls.cpp
index 211184523aa51..85449caa46972 100644
--- a/clang/test/Analysis/anonymous-decls.cpp
+++ b/clang/test/Analysis/anonymous-decls.cpp
@@ -74,13 +74,13 @@ int main() {
 // CHECK-NEXT:   4: * [B3.3] (OperatorCall)
 // CHECK-NEXT:   5: auto &;
 // CHECK-NEXT:   6: get<0UL>
-// CHECK-NEXT:   7: [B3.6] (ImplicitCastExpr, FunctionToPointerDecay, typename tuple_element<0L, pair<int, int> >::type (*)(pair<int, int> &))
+// CHECK-NEXT:   7: [B3.6] (ImplicitCastExpr, FunctionToPointerDecay, tuple_element<0L, pair<int, int> >::type (*)(pair<int, int> &))
 // CHECK-NEXT:   8: decomposition-a-b
 // CHECK-NEXT:   9: [B3.7]([B3.8])
 // CHECK-NEXT:  10: [B3.9]
 // CHECK-NEXT:  11: std::tuple_element<0, std::pair<int, int>>::type a = get<0UL>(decomposition-a-b);
 // CHECK-NEXT:  12: get<1UL>
-// CHECK-NEXT:  13: [B3.12] (ImplicitCastExpr, FunctionToPointerDecay, typename tuple_element<1L, pair<int, int> >::type (*)(pair<int, int> &))
+// CHECK-NEXT:  13: [B3.12] (ImplicitCastExpr, FunctionToPointerDecay, tuple_element<1L, pair<int, int> >::type (*)(pair<int, int> &))
 // CHECK-NEXT:  14: decomposition-a-b
 // CHECK-NEXT:  15: [B3.13]([B3.14])
 // CHECK-NEXT:  16: [B3.15]
diff --git a/clang/test/CXX/drs/cwg23xx.cpp b/clang/test/CXX/drs/cwg23xx.cpp
index 78cecb8b71bca..74e72f2371e2a 100644
--- a/clang/test/CXX/drs/cwg23xx.cpp
+++ b/clang/test/CXX/drs/cwg23xx.cpp
@@ -91,7 +91,7 @@ struct Y {};
 struct Z : W,
   X, check_derived_from<Z, X>, // #cwg2310-X
   check_derived_from<Z, Y>, Y  // #cwg2310-Y
-{  
+{
   // FIXME: It was properly rejected before, but we're crashing since Clang 11 in C++11 and C++14 modes.
   //        See https://github.com/llvm/llvm-project/issues/59920
 #if __cplusplus >= 201703L
@@ -188,7 +188,7 @@ struct InitListCtor {
 
 std::initializer_list<InitListCtor> i;
 auto j = std::initializer_list<InitListCtor>{ i };
-// since-cxx17-error@-1 {{conversion function from 'std::initializer_list<InitListCtor>' to 'const cwg2311::InitListCtor' invokes a deleted function}}
+// since-cxx17-error@-1 {{conversion function from 'std::initializer_list<InitListCtor>' to 'const InitListCtor' invokes a deleted function}}
 //   since-cxx17-note@#cwg2311-InitListCtor {{'InitListCtor' has been explicitly marked deleted here}}
 #endif
 } // namespace cwg2311
diff --git a/clang/test/SemaTemplate/dependent-template-recover.cpp b/clang/test/SemaTemplate/dependent-template-recover.cpp
index 251a8f9816417..21e6a963719bd 100644
--- a/clang/test/SemaTemplate/dependent-template-recover.cpp
+++ b/clang/test/SemaTemplate/dependent-template-recover.cpp
@@ -146,9 +146,9 @@ namespace templ_spec {
 
     // FIXME: Why error recovery for the non-typename case is so bad?
     A<T::template X<int>> t3; // expected-error {{did you forget 'typename'}}
-    // expected-error@-1 {{'A<typename T::X>' (aka 'void')}}
+    // expected-error@-1 {{'A<T::X>' (aka 'void')}}
 
     A<T::X<int>> t4; // expected-error {{use 'template' keyword}} expected-error {{did you forget 'typename'}}
-    // expected-error@-1 {{'A<typename T::X>' (aka 'void')}}
+    // expected-error@-1 {{'A<T::X>' (aka 'void')}}
   };
 } // namespace templ_spec
diff --git a/clang/test/SemaTemplate/elaborated-type-specifier.cpp b/clang/test/SemaTemplate/elaborated-type-specifier.cpp
index 27b3f36ee14dd..95c2aa9f60a39 100644
--- a/clang/test/SemaTemplate/elaborated-type-specifier.cpp
+++ b/clang/test/SemaTemplate/elaborated-type-specifier.cpp
@@ -10,7 +10,7 @@ namespace PR6915 {
   struct D1 {
     enum X { value };
   };
-  struct D2 { 
+  struct D2 {
     class X { }; // expected-note{{previous use is here}}
   };
   struct D3 { };
@@ -25,12 +25,12 @@ struct DeclOrDef {
   enum T::foo; // expected-error{{nested name specifier for a declaration cannot depend on a template parameter}}
                // expected-error@-1{{forward declaration of enum cannot have a nested name specifier}}
   enum T::bar { // expected-error{{nested name specifier for a declaration cannot depend on a template parameter}}
-    value 
+    value
   };
 };
 
 namespace PR6649 {
-  template <typename T> struct foo { 
+  template <typename T> struct foo {
     class T::bar;  // expected-error{{nested name specifier for a declaration cannot depend on a template parameter}}
                    // expected-error@-1{{forward declaration of class cannot have a nested name specifier}}
     class T::bar { int x; }; // expected-error{{nested name specifier for a declaration cannot depend on a template parameter}}
@@ -40,3 +40,60 @@ namespace PR6649 {
 namespace rdar8568507 {
   template <class T> struct A *makeA(T t);
 }
+
+namespace canon {
+  template <class T> void t1(struct T::X) {}
+  // expected-note@-1 {{previous definition is here}}
+  template <class T> void t1(class T::X) {}
+  // expected-error@-1 {{redefinition of 't1'}}
+
+  template <class T> void t2(struct T::template X<int>) {}
+  // expected-note@-1 {{previous definition is here}}
+  template <class T> void t2(class T::template X<int>) {}
+  // expected-error@-1 {{redefinition of 't2'}}
+
+  template <class T> constexpr int t3(typename T::X* = 0) { return 0; } // #canon-t3-0
+  template <class T> constexpr int t3(struct   T::X* = 0) { return 1; } // #canon-t3-1
+  template <class T> constexpr int t3(union    T::X* = 0) { return 2; } // #canon-t3-2
+  template <class T> constexpr int t3(enum     T::X* = 0) { return 3; } // #canon-t3-3
+
+  struct A { using X = int; };
+  static_assert(t3<A>() == 0);
+
+  struct B { struct X {}; };
+  static_assert(t3<B>() == 1);
+  // expected-error@-1 {{call to 't3' is ambiguous}}
+  // expected-note@#canon-t3-0 {{candidate function}}
+  // expected-note@#canon-t3-1 {{candidate function}}
+
+  struct C { union X {}; };
+  static_assert(t3<C>() == 2);
+  // expected-error@-1 {{call to 't3' is ambiguous}}
+  // expected-note@#canon-t3-0 {{candidate function}}
+  // expected-note@#canon-t3-2 {{candidate function}}
+
+  struct D { enum X {}; };
+  static_assert(t3<D>() == 3);
+  // expected-error@-1 {{call to 't3' is ambiguous}}
+  // expected-note@#canon-t3-0 {{candidate function}}
+  // expected-note@#canon-t3-3 {{candidate function}}
+
+  template <class T> constexpr int t4(typename T::template X<int>* = 0) { return 0; }
+  // expected-note@-1 3{{candidate function}}
+  template <class T> constexpr int t4(struct   T::template X<int>* = 0) { return 1; }
+  // expected-note@-1 3{{candidate function}}
+  template <class T> constexpr int t4(union    T::template X<int>* = 0) { return 2; }
+  // expected-note@-1 3{{candidate function}}
+
+  // FIXME: This should work.
+  struct E { template <class T> using X = T; };
+  static_assert(t4<E>() == 0); // expected-error {{call to 't4' is ambiguous}}
+
+  // FIXME: Should not match the union overload.
+  struct F { template <class> struct X {}; };
+  static_assert(t4<F>() == 1); // expected-error {{call to 't4' is ambiguous}}
+
+  // FIXME: Should not match the struct overload.
+  struct G { template <class> union X {}; };
+  static_assert(t4<G>() == 2); // expected-error {{call to 't4' is ambiguous}}
+} // namespace canon
diff --git a/clang/test/SemaTemplate/typename-specifier-3.cpp b/clang/test/SemaTemplate/typename-specifier-3.cpp
index 0140b1a479c2d..cdd065c98bb0a 100644
--- a/clang/test/SemaTemplate/typename-specifier-3.cpp
+++ b/clang/test/SemaTemplate/typename-specifier-3.cpp
@@ -75,3 +75,21 @@ namespace PR12884_fixed {
 
   A<int>::C::x a; // ok
 }
+
+namespace preserve_keyword {
+  template <class T> struct A {
+    using type = T;
+  };
+
+  template <class T> using B = A<T>::type*; // precxx17-warning {{missing 'typename'}}
+  void *t1 = *B<int>(); // expected-error {{lvalue of type 'A<int>::type' (aka 'int')}}
+
+  template <class T> using C = typename A<T>::type*;
+  void *t2 = *C<int>(); // expected-error {{lvalue of type 'typename A<int>::type' (aka 'int')}}
+
+  using D = A<int>::type*;
+  void *t3 = *D(); // expected-error {{lvalue of type 'A<int>::type' (aka 'int')}}
+
+  using D = typename A<int>::type*;
+  void *t4 = *D(); // expected-error {{lvalue of type 'typename A<int>::type' (aka 'int')}}
+} // namespace preserve_keyword

From c4e9901b5b660f7c64570c3440080436c8e8b32e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 12:28:47 -0700
Subject: [PATCH 165/710] [llvm] Use llvm::append_range (NFC) (#135931)

---
 llvm/include/llvm/IR/ModuleSummaryIndex.h                   | 6 ++++--
 llvm/lib/Analysis/MemoryProfileInfo.cpp                     | 6 ++----
 llvm/lib/Analysis/ScalarEvolution.cpp                       | 6 ++----
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp                   | 2 +-
 llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp                | 4 +---
 llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp           | 2 +-
 llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp  | 2 +-
 .../ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp    | 2 +-
 llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp              | 4 ++--
 llvm/lib/IR/DebugInfoMetadata.cpp                           | 2 +-
 llvm/lib/MC/DXContainerPSVInfo.cpp                          | 3 +--
 llvm/lib/MC/MCParser/MasmParser.cpp                         | 5 ++---
 llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp                         | 4 ++--
 llvm/lib/ProfileData/InstrProfReader.cpp                    | 2 +-
 llvm/lib/TargetParser/SubtargetFeature.cpp                  | 2 +-
 llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp           | 4 ++--
 llvm/unittests/DebugInfo/PDB/HashTableTest.cpp              | 2 +-
 llvm/unittests/Transforms/IPO/LowerTypeTests.cpp            | 2 +-
 llvm/utils/TableGen/Common/CodeGenRegisters.cpp             | 6 +++---
 llvm/utils/TableGen/RegisterInfoEmitter.cpp                 | 2 +-
 20 files changed, 31 insertions(+), 37 deletions(-)

diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index 7aa36345268cd..3f20d40722ca2 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -1319,8 +1319,10 @@ class CfiFunctionIndex {
 
   std::vector<StringRef> symbols() const {
     std::vector<StringRef> Symbols;
-    for (auto &[GUID, Syms] : Index)
-      Symbols.insert(Symbols.end(), Syms.begin(), Syms.end());
+    for (auto &[GUID, Syms] : Index) {
+      (void)GUID;
+      llvm::append_range(Symbols, Syms);
+    }
     return Symbols;
   }
 
diff --git a/llvm/lib/Analysis/MemoryProfileInfo.cpp b/llvm/lib/Analysis/MemoryProfileInfo.cpp
index 95138de592290..6ca5b5e492723 100644
--- a/llvm/lib/Analysis/MemoryProfileInfo.cpp
+++ b/llvm/lib/Analysis/MemoryProfileInfo.cpp
@@ -181,8 +181,7 @@ void CallStackTrie::addCallStack(
     Curr = New;
   }
   assert(Curr);
-  Curr->ContextSizeInfo.insert(Curr->ContextSizeInfo.end(),
-                               ContextSizeInfo.begin(), ContextSizeInfo.end());
+  llvm::append_range(Curr->ContextSizeInfo, ContextSizeInfo);
 }
 
 void CallStackTrie::addCallStack(MDNode *MIB) {
@@ -235,8 +234,7 @@ static MDNode *createMIBNode(LLVMContext &Ctx, ArrayRef<uint64_t> MIBCallStack,
 
 void CallStackTrie::collectContextSizeInfo(
     CallStackTrieNode *Node, std::vector<ContextTotalSize> &ContextSizeInfo) {
-  ContextSizeInfo.insert(ContextSizeInfo.end(), Node->ContextSizeInfo.begin(),
-                         Node->ContextSizeInfo.end());
+  llvm::append_range(ContextSizeInfo, Node->ContextSizeInfo);
   for (auto &Caller : Node->Callers)
     collectContextSizeInfo(Caller.second, ContextSizeInfo);
 }
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index d193c9e3210ea..5132ee13a9632 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -8503,10 +8503,8 @@ void ScalarEvolution::forgetLoop(const Loop *L) {
     }
 
     auto LoopUsersItr = LoopUsers.find(CurrL);
-    if (LoopUsersItr != LoopUsers.end()) {
-      ToForget.insert(ToForget.end(), LoopUsersItr->second.begin(),
-                LoopUsersItr->second.end());
-    }
+    if (LoopUsersItr != LoopUsers.end())
+      llvm::append_range(ToForget, LoopUsersItr->second);
 
     // Drop information about expressions based on loop-header PHIs.
     PushLoopPHIs(CurrL, Worklist, Visited);
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index ad15f13902e63..73bed85c65b3d 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -5098,7 +5098,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
       return;
     for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
       auto Defs = CfiIndex.forGuid(GUID);
-      Functions.insert(Functions.end(), Defs.begin(), Defs.end());
+      llvm::append_range(Functions, Defs);
     }
     if (Functions.empty())
       return;
diff --git a/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp b/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
index 642ab61756ea5..22137ea172240 100644
--- a/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
+++ b/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
@@ -621,9 +621,7 @@ void DwarfTransformer::parseCallSiteInfoFromDwarf(CUInfo &CUI, DWARFDie Die,
     if (!FI.CallSites)
       FI.CallSites = CallSiteInfoCollection();
     // Append parsed DWARF callsites:
-    FI.CallSites->CallSites.insert(FI.CallSites->CallSites.end(),
-                                   CSIC.CallSites.begin(),
-                                   CSIC.CallSites.end());
+    llvm::append_range(FI.CallSites->CallSites, CSIC.CallSites);
   }
 }
 
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp
index 5673ea7c2cd23..3cb2662f2f313 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp
@@ -230,7 +230,7 @@ Error LVCompare::execute(LVReader *ReferenceReader, LVReader *TargetReader) {
         }
         if (Pass == LVComparePass::Added)
           // Record all the current missing elements for this category.
-          Set.insert(Set.end(), Elements.begin(), Elements.end());
+          llvm::append_range(Set, Elements);
         if (options().getReportList()) {
           if (Elements.size()) {
             OS << "\n(" << Elements.size() << ") "
diff --git a/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
index 4f8f883a75f32..87675be1fc8e1 100644
--- a/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
+++ b/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
@@ -485,7 +485,7 @@ static GenericValue lle_X_fprintf(FunctionType *FT,
   char Buffer[10000];
   std::vector<GenericValue> NewArgs;
   NewArgs.push_back(PTOGV(Buffer));
-  NewArgs.insert(NewArgs.end(), Args.begin()+1, Args.end());
+  llvm::append_range(NewArgs, llvm::drop_begin(Args));
   GenericValue GV = lle_X_sprintf(FT, NewArgs);
 
   fputs(Buffer, (FILE *) GVTOP(Args[0]));
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
index 6a00b87dd0a6b..8793d6f8ab90b 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp
@@ -162,7 +162,7 @@ void VTuneSupportPlugin::notifyTransferringResources(JITDylib &JD,
     return;
 
   auto &Dest = LoadedMethodIDs[DstKey];
-  Dest.insert(Dest.end(), I->second.begin(), I->second.end());
+  llvm::append_range(Dest, I->second);
   LoadedMethodIDs.erase(SrcKey);
 }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
index 80f2a1304dde7..48b096f62ff29 100644
--- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
@@ -323,7 +323,7 @@ void LazyReexportsManager::handleTransferResources(JITDylib &JD,
     } else {
       auto &SrcAddrs = I->second;
       auto &DstAddrs = J->second;
-      DstAddrs.insert(DstAddrs.end(), SrcAddrs.begin(), SrcAddrs.end());
+      llvm::append_range(DstAddrs, SrcAddrs);
       KeyToReentryAddrs.erase(I);
     }
     if (L)
@@ -503,7 +503,7 @@ void SimpleLazyReexportsSpeculator::onLazyReexportsTransfered(
   } else {
     auto &SrcNames = J->second;
     auto &DstNames = K->second;
-    DstNames.insert(DstNames.end(), SrcNames.begin(), SrcNames.end());
+    llvm::append_range(DstNames, SrcNames);
     MapForJD.erase(J);
   }
 }
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 12aba7d2bd123..b8b824aed7178 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -1990,7 +1990,7 @@ DIExpression *DIExpression::appendOpsToArg(const DIExpression *Expr,
     }
     Op.appendToVector(NewOps);
     if (Op.getOp() == dwarf::DW_OP_LLVM_arg && Op.getArg(0) == ArgNo)
-      NewOps.insert(NewOps.end(), Ops.begin(), Ops.end());
+      llvm::append_range(NewOps, Ops);
   }
   if (StackValue)
     NewOps.push_back(dwarf::DW_OP_stack_value);
diff --git a/llvm/lib/MC/DXContainerPSVInfo.cpp b/llvm/lib/MC/DXContainerPSVInfo.cpp
index aeff693801397..f70c8b1af01b3 100644
--- a/llvm/lib/MC/DXContainerPSVInfo.cpp
+++ b/llvm/lib/MC/DXContainerPSVInfo.cpp
@@ -58,8 +58,7 @@ ProcessElementList(StringTableBuilder &StrTabBuilder,
     size_t Idx = FindSequence(IndexBuffer, El.Indices);
     if (Idx == npos) {
       FinalElement.IndicesOffset = static_cast<uint32_t>(IndexBuffer.size());
-      IndexBuffer.insert(IndexBuffer.end(), El.Indices.begin(),
-                         El.Indices.end());
+      llvm::append_range(IndexBuffer, El.Indices);
     } else
       FinalElement.IndicesOffset = static_cast<uint32_t>(Idx);
     FinalElements.push_back(FinalElement);
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index bbcdffd4d4fa8..f758020566465 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -3647,9 +3647,8 @@ bool MasmParser::parseFieldInitializer(const FieldInfo &Field,
                           std::to_string(Initializers.size()));
   }
   // Default-initialize all remaining values.
-  Initializers.insert(Initializers.end(),
-                      Contents.Initializers.begin() + Initializers.size(),
-                      Contents.Initializers.end());
+  llvm::append_range(Initializers, llvm::drop_begin(Contents.Initializers,
+                                                    Initializers.size()));
 
   Initializer = FieldInitializer(std::move(Initializers), Contents.Structure);
   return false;
diff --git a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
index b0ec215aec203..935f89ad76440 100644
--- a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
+++ b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
@@ -662,13 +662,13 @@ RemoveNoteDetail::updateData(ArrayRef<uint8_t> OldData,
   for (const DeletedRange &RemRange : ToRemove) {
     if (CurPos < RemRange.OldFrom) {
       auto Slice = OldData.slice(CurPos, RemRange.OldFrom - CurPos);
-      NewData.insert(NewData.end(), Slice.begin(), Slice.end());
+      llvm::append_range(NewData, Slice);
     }
     CurPos = RemRange.OldTo;
   }
   if (CurPos < OldData.size()) {
     auto Slice = OldData.slice(CurPos);
-    NewData.insert(NewData.end(), Slice.begin(), Slice.end());
+    llvm::append_range(NewData, Slice);
   }
   return NewData;
 }
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index cac1760d3ef80..4075b513c218d 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1096,7 +1096,7 @@ class llvm::InstrProfReaderItaniumRemapper
                                SmallVectorImpl<char> &Out) {
     Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size());
     Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin());
-    Out.insert(Out.end(), Replacement.begin(), Replacement.end());
+    llvm::append_range(Out, Replacement);
     Out.insert(Out.end(), ExtractedName.end(), OrigName.end());
   }
 
diff --git a/llvm/lib/TargetParser/SubtargetFeature.cpp b/llvm/lib/TargetParser/SubtargetFeature.cpp
index be42a42967332..36c67f661d9a5 100644
--- a/llvm/lib/TargetParser/SubtargetFeature.cpp
+++ b/llvm/lib/TargetParser/SubtargetFeature.cpp
@@ -43,7 +43,7 @@ void SubtargetFeatures::AddFeature(StringRef String, bool Enable) {
 
 void SubtargetFeatures::addFeaturesVector(
     const ArrayRef<std::string> OtherFeatures) {
-  Features.insert(Features.cend(), OtherFeatures.begin(), OtherFeatures.end());
+  llvm::append_range(Features, OtherFeatures);
 }
 
 SubtargetFeatures::SubtargetFeatures(StringRef Initial) {
diff --git a/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp b/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
index cf4a5f27585d0..f8d161d8c50b6 100644
--- a/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
@@ -243,7 +243,7 @@ void dwarfgen::LineTable::addByte(uint8_t Value) {
 void dwarfgen::LineTable::addStandardOpcode(uint8_t Opcode,
                                             ArrayRef<ValueAndLength> Operands) {
   Contents.push_back({Opcode, Byte});
-  Contents.insert(Contents.end(), Operands.begin(), Operands.end());
+  llvm::append_range(Contents, Operands);
 }
 
 void dwarfgen::LineTable::addExtendedOpcode(uint64_t Length, uint8_t Opcode,
@@ -251,7 +251,7 @@ void dwarfgen::LineTable::addExtendedOpcode(uint64_t Length, uint8_t Opcode,
   Contents.push_back({0, Byte});
   Contents.push_back({Length, ULEB});
   Contents.push_back({Opcode, Byte});
-  Contents.insert(Contents.end(), Operands.begin(), Operands.end());
+  llvm::append_range(Contents, Operands);
 }
 
 void dwarfgen::LineTable::generate(MCContext &MC, AsmPrinter &Asm) const {
diff --git a/llvm/unittests/DebugInfo/PDB/HashTableTest.cpp b/llvm/unittests/DebugInfo/PDB/HashTableTest.cpp
index 6d17332f49079..94e82ed02c398 100644
--- a/llvm/unittests/DebugInfo/PDB/HashTableTest.cpp
+++ b/llvm/unittests/DebugInfo/PDB/HashTableTest.cpp
@@ -233,7 +233,7 @@ struct FooBarHashTraits {
 
   uint32_t lookupKeyToStorageKey(StringRef S) {
     uint32_t N = Buffer.size();
-    Buffer.insert(Buffer.end(), S.begin(), S.end());
+    llvm::append_range(Buffer, S);
     Buffer.push_back('\0');
     return N;
   }
diff --git a/llvm/unittests/Transforms/IPO/LowerTypeTests.cpp b/llvm/unittests/Transforms/IPO/LowerTypeTests.cpp
index ba13378099ecb..1602826b7252c 100644
--- a/llvm/unittests/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/unittests/Transforms/IPO/LowerTypeTests.cpp
@@ -100,7 +100,7 @@ TEST(LowerTypeTests, GlobalLayoutBuilder) {
 
     std::vector<uint64_t> ComputedLayout;
     for (auto &&F : GLB.Fragments)
-      ComputedLayout.insert(ComputedLayout.end(), F.begin(), F.end());
+      llvm::append_range(ComputedLayout, F);
 
     EXPECT_EQ(T.WantLayout, ComputedLayout);
   }
diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
index 3a6e828a99f2d..eb142e66faf2f 100644
--- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
@@ -678,7 +678,7 @@ struct TupleExpander : SetTheory::Expander {
       // Take the cost list of the first register in the tuple.
       const ListInit *CostList = Proto->getValueAsListInit("CostPerUse");
       SmallVector<const Init *, 2> CostPerUse;
-      CostPerUse.insert(CostPerUse.end(), CostList->begin(), CostList->end());
+      llvm::append_range(CostPerUse, *CostList);
 
       const StringInit *AsmName = StringInit::get(RK, "");
       if (!RegNames.empty()) {
@@ -1186,7 +1186,7 @@ void CodeGenRegisterClass::extendSuperRegClasses(CodeGenSubRegIndex *SubIdx) {
     return;
 
   SmallVector<CodeGenRegisterClass *> MidRCs;
-  MidRCs.insert(MidRCs.end(), It->second.begin(), It->second.end());
+  llvm::append_range(MidRCs, It->second);
 
   for (CodeGenRegisterClass *MidRC : MidRCs) {
     for (auto &Pair : MidRC->SuperRegClasses) {
@@ -1244,7 +1244,7 @@ CodeGenRegBank::CodeGenRegBank(const RecordKeeper &Records,
     for (const Record *R : Records.getAllDerivedDefinitions("RegisterTuples")) {
       // Expand tuples and merge the vectors
       std::vector<const Record *> TupRegs = *Sets.expand(R);
-      Regs.insert(Regs.end(), TupRegs.begin(), TupRegs.end());
+      llvm::append_range(Regs, TupRegs);
     }
 
     llvm::sort(Regs, LessRecordRegister());
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index 45c6db94023b7..98f0d7eaaff38 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -1487,7 +1487,7 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS) {
   // each register. Fill with zero for values which are not explicitly given.
   for (const auto &Reg : Regs) {
     auto Costs = Reg.CostPerUse;
-    AllRegCostPerUse.insert(AllRegCostPerUse.end(), Costs.begin(), Costs.end());
+    llvm::append_range(AllRegCostPerUse, Costs);
     if (NumRegCosts > Costs.size())
       AllRegCostPerUse.insert(AllRegCostPerUse.end(),
                               NumRegCosts - Costs.size(), 0);

From 0f97cd87dee1ddc6aadc04369262739da2f55b5a Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 12:29:04 -0700
Subject: [PATCH 166/710] [Frontend] Use StringRef::ends_with (NFC) (#135988)

---
 clang/lib/Frontend/InitPreprocessor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 659af42478991..1f297f228fc1b 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -32,7 +32,7 @@ using namespace clang;
 static bool MacroBodyEndsInBackslash(StringRef MacroBody) {
   while (!MacroBody.empty() && isWhitespace(MacroBody.back()))
     MacroBody = MacroBody.drop_back();
-  return !MacroBody.empty() && MacroBody.back() == '\\';
+  return MacroBody.ends_with('\\');
 }
 
 // Append a #define line to Buf for Macro.  Macro should be of the form XXX,

From fa46d522fe66b77bdf3156be9255ce3b83010433 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 12:29:33 -0700
Subject: [PATCH 167/710] [llvm-xray] Use llvm::make_second_range (NFC)
 (#135989)

---
 llvm/tools/llvm-xray/xray-stacks.cpp | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/llvm/tools/llvm-xray/xray-stacks.cpp b/llvm/tools/llvm-xray/xray-stacks.cpp
index cbf6faeb32960..b11d732a4fcc0 100644
--- a/llvm/tools/llvm-xray/xray-stacks.cpp
+++ b/llvm/tools/llvm-xray/xray-stacks.cpp
@@ -495,15 +495,8 @@ class StackTrie {
   void printIgnoringThreads(raw_ostream &OS, FuncIdConversionHelper &FN) {
     RootVector RootValues;
 
-    // Function to pull the values out of a map iterator.
-    using RootsType = decltype(Roots.begin())::value_type;
-    auto MapValueFn = [](const RootsType &Value) { return Value.second; };
-
-    for (const auto &RootNodeRange :
-         make_range(map_iterator(Roots.begin(), MapValueFn),
-                    map_iterator(Roots.end(), MapValueFn))) {
+    for (const auto &RootNodeRange : make_second_range(Roots))
       llvm::append_range(RootValues, RootNodeRange);
-    }
 
     print(OS, FN, RootValues);
   }

From 4863d1ffbde77b8a3c580b4f7905ec21b1aba7e0 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 12:29:58 -0700
Subject: [PATCH 168/710] [Serialization] Use llvm::map_range (NFC) (#135990)

---
 clang/lib/Serialization/MultiOnDiskHashTable.h | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/clang/lib/Serialization/MultiOnDiskHashTable.h b/clang/lib/Serialization/MultiOnDiskHashTable.h
index fa18a29a9a98e..996e9b94287f4 100644
--- a/clang/lib/Serialization/MultiOnDiskHashTable.h
+++ b/clang/lib/Serialization/MultiOnDiskHashTable.h
@@ -103,11 +103,9 @@ template<typename Info> class MultiOnDiskHashTable {
 
   /// The current set of on-disk tables.
   table_range tables() {
-    auto Begin = Tables.begin(), End = Tables.end();
-    if (getMergedTable())
-      ++Begin;
-    return llvm::make_range(llvm::map_iterator(Begin, AsOnDiskTable()),
-                            llvm::map_iterator(End, AsOnDiskTable()));
+    unsigned DropBegin = getMergedTable() ? 1 : 0;
+    return llvm::map_range(llvm::drop_begin(Tables, DropBegin),
+                           AsOnDiskTable());
   }
 
   MergedTable *getMergedTable() const {

From d338bdc9fe72aef1d9d2043d569fd94eb57db111 Mon Sep 17 00:00:00 2001
From: Adam Siemieniuk <adam.siemieniuk@intel.com>
Date: Wed, 16 Apr 2025 22:02:52 +0200
Subject: [PATCH 169/710] [mlir][linalg][NFC] Update elementwise docs to match
 op name (#135999)

Updates linalg.elementwise op description to replace older abbreviated
mnemonic with its current form.
---
 .../mlir/Dialect/Linalg/IR/LinalgStructuredOps.td    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 308e39a9a51e1..b9edcc92e81a9 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -565,18 +565,18 @@ def ElementwiseOp : LinalgStructuredBase_Op<"elementwise", [
 
     Example:
 
-    Defining a unary linalg.elemwise with default indexing-map:
+    Defining a unary linalg.elementwise with default indexing-map:
     ```mlir
-    %exp = linalg.elemwise
-        kind=#linalg.elemwise_kind<exp>
+    %exp = linalg.elementwise
+        kind=#linalg.elementwise_kind<exp>
         ins(%x : tensor<4x16x8xf32>)
         outs(%y: tensor<4x16x8xf32>) -> tensor<4x16x8xf32>
     ```
 
-    Defining a binary linalg.elemwise with user-defined indexing-map:
+    Defining a binary linalg.elementwise with user-defined indexing-map:
     ```mlir
-    %add = linalg.elemwise
-        kind=#linalg.elemwise_kind<add>
+    %add = linalg.elementwise
+        kind=#linalg.elementwise_kind<add>
         indexing_maps = [#transpose, #broadcast, #identity]
         ins(%exp, %arg1 : tensor<4x16x8xf32>, tensor<4x16xf32>)
         outs(%arg2: tensor<4x8x16xf32>) -> tensor<4x8x16xf32>

From 598ec8ce2d1e5e20b45c56de8972f58a0caeb697 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Wed, 16 Apr 2025 13:07:58 -0700
Subject: [PATCH 170/710] [LLVM][TableGen] Parameterize NumToSkip in
 DecoderEmitter (#135882)

- Add command line option `num-to-skip-size` to parameterize the size of
`NumToSkip` bytes in the decoder table. Default value will be 2, and
targets that need larger size can use 3.
- Keep all existing targets, except AArch64, to use size 2, and change
AArch64 to use size 3 since it run into the "disassembler decoding table
too large" error with size 2.
- Following is a rough reduction in size for the decoder tables by
switching to size 2.

```
Target         Old Size   New Size   % Reduction
================================================
AArch64           153254     153254        0.00
AMDGPU            471566     412805       12.46
ARC                 5724       5061       11.58
ARM                84936      73831       13.07
AVR                 1497       1306       12.76
BPF                 2172       1927       11.28
CSKY               10064       8692       13.63
Hexagon            47967      41965       12.51
Lanai               1108        982       11.37
LoongArch          24446      21621       11.56
MSP430              4200       3716       11.52
Mips               36330      31415       13.53
PPC                31897      28098       11.91
RISCV              37979      32790       13.66
Sparc               8331       7252       12.95
SystemZ            36722      32248       12.18
VE                 48296      42873       11.23
XCore               2590       2316       10.58
Xtensa              3827       3316       13.35
```
---
 llvm/lib/Target/AArch64/CMakeLists.txt    |   2 +-
 llvm/test/TableGen/VarLenDecoder.td       |   4 +-
 llvm/test/TableGen/trydecode-emission.td  |  10 +-
 llvm/test/TableGen/trydecode-emission2.td |  16 +--
 llvm/test/TableGen/trydecode-emission3.td |   2 +-
 llvm/test/TableGen/trydecode-emission4.td |   2 +-
 llvm/utils/TableGen/DecoderEmitter.cpp    | 115 ++++++++++++----------
 7 files changed, 83 insertions(+), 68 deletions(-)

diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 2300e479bc110..ba1d1605ec104 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -7,7 +7,7 @@ tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
 tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
 tablegen(LLVM AArch64GenCallingConv.inc -gen-callingconv)
 tablegen(LLVM AArch64GenDAGISel.inc -gen-dag-isel)
-tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler)
+tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler --num-to-skip-size=3)
 tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel)
 tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel)
 tablegen(LLVM AArch64GenO0PreLegalizeGICombiner.inc -gen-global-isel-combiner
diff --git a/llvm/test/TableGen/VarLenDecoder.td b/llvm/test/TableGen/VarLenDecoder.td
index 5cf0bf8911859..b77702ff7c5c1 100644
--- a/llvm/test/TableGen/VarLenDecoder.td
+++ b/llvm/test/TableGen/VarLenDecoder.td
@@ -47,9 +47,9 @@ def FOO32 : MyVarInst<MemOp32> {
 }
 
 // CHECK:      MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
-// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, 0, // Skip to: 12
+// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, // Skip to: 11
 // CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
-// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, 0, // Skip to: 21
+// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, // Skip to: 19
 // CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
 // CHECK-NEXT: MCD::OPC_Fail,
 
diff --git a/llvm/test/TableGen/trydecode-emission.td b/llvm/test/TableGen/trydecode-emission.td
index 20d2446eeac7f..2b4239f4fbe65 100644
--- a/llvm/test/TableGen/trydecode-emission.td
+++ b/llvm/test/TableGen/trydecode-emission.td
@@ -34,10 +34,10 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
-// CHECK-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
-// CHECK-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
-// CHECK-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 26 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23
+// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19
+// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 19
+// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
+// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission2.td b/llvm/test/TableGen/trydecode-emission2.td
index 0584034e41233..7d30474058f73 100644
--- a/llvm/test/TableGen/trydecode-emission2.td
+++ b/llvm/test/TableGen/trydecode-emission2.td
@@ -31,14 +31,14 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 36, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 8 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
-// CHECK-NEXT: /* 11 */      MCD::OPC_FilterValue, 0, 28, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 16 */      MCD::OPC_CheckField, 0, 2, 3, 7, 0, 0, // Skip to: 30
-// CHECK-NEXT: /* 23 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 30
-// CHECK-NEXT: /* 30 */      MCD::OPC_CheckField, 3, 2, 0, 7, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 37 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstA, skip to: 44
-// CHECK-NEXT: /* 44 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 31, 0, // Skip to: 38
+// CHECK-NEXT: /* 7 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-NEXT: /* 10 */      MCD::OPC_FilterValue, 0, 24, 0, // Skip to: 38
+// CHECK-NEXT: /* 14 */      MCD::OPC_CheckField, 0, 2, 3, 6, 0, // Skip to: 26
+// CHECK-NEXT: /* 20 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 26
+// CHECK-NEXT: /* 26 */      MCD::OPC_CheckField, 3, 2, 0, 6, 0, // Skip to: 38
+// CHECK-NEXT: /* 32 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, // Opcode: InstA, skip to: 38
+// CHECK-NEXT: /* 38 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 // CHECK: if (!Check(S, DecodeInstA(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission3.td b/llvm/test/TableGen/trydecode-emission3.td
index 4c5be7e1af229..0abbe62fe337e 100644
--- a/llvm/test/TableGen/trydecode-emission3.td
+++ b/llvm/test/TableGen/trydecode-emission3.td
@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+ // RUN: llvm-tblgen -gen-disassembler --num-to-skip-size=3 -I %p/../../include %s  | FileCheck %s
 
 include "llvm/Target/Target.td"
 
diff --git a/llvm/test/TableGen/trydecode-emission4.td b/llvm/test/TableGen/trydecode-emission4.td
index 1e51ba5e40768..413e4a0d1275a 100644
--- a/llvm/test/TableGen/trydecode-emission4.td
+++ b/llvm/test/TableGen/trydecode-emission4.td
@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler --num-to-skip-size=3 -I %p/../../include %s | FileCheck %s
 
 // Test for OPC_ExtractField/OPC_CheckField with start bit > 255.
 // These large start values may arise for architectures with long instruction
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 9c6015cc24576..0735702abf339 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -32,8 +32,10 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/LEB128.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -76,6 +78,12 @@ static cl::opt<SuppressLevel> DecoderEmitterSuppressDuplicates(
             "significantly reducing Table Duplications")),
     cl::init(SUPPRESSION_DISABLE), cl::cat(DisassemblerEmitterCat));
 
+static cl::opt<uint32_t>
+    NumToSkipSizeInBytes("num-to-skip-size",
+                         cl::desc("number of bytes to use for num-to-skip "
+                                  "entries in the decoder table (2 or 3)"),
+                         cl::init(2), cl::cat(DisassemblerEmitterCat));
+
 STATISTIC(NumEncodings, "Number of encodings considered");
 STATISTIC(NumEncodingsLackingDisasm,
           "Number of encodings without disassembler info");
@@ -130,10 +138,29 @@ struct DecoderTable : public std::vector<uint8_t> {
   // in the table for patching.
   size_t insertNumToSkip() {
     size_t Size = size();
-    insert(end(), 3, 0);
+    insert(end(), NumToSkipSizeInBytes, 0);
     return Size;
   }
+
+  void patchNumToSkip(size_t FixupIdx, uint32_t DestIdx) {
+    // Calculate the distance from the byte following the fixup entry byte
+    // to the destination. The Target is calculated from after the
+    // `NumToSkipSizeInBytes`-byte NumToSkip entry itself, so subtract
+    // `NumToSkipSizeInBytes` from the displacement here to account for that.
+    assert(DestIdx > FixupIdx + NumToSkipSizeInBytes &&
+           "Expecting a forward jump in the decoding table");
+    uint32_t Delta = DestIdx - FixupIdx - NumToSkipSizeInBytes;
+    if (!isUIntN(8 * NumToSkipSizeInBytes, Delta))
+      PrintFatalError(
+          "disassembler decoding table too large, try --num-to-skip-size=3");
+
+    (*this)[FixupIdx] = static_cast<uint8_t>(Delta);
+    (*this)[FixupIdx + 1] = static_cast<uint8_t>(Delta >> 8);
+    if (NumToSkipSizeInBytes == 3)
+      (*this)[FixupIdx + 2] = static_cast<uint8_t>(Delta >> 16);
+  }
 };
+
 struct DecoderTableInfo {
   DecoderTable Table;
   FixupScopeList FixupStack;
@@ -690,19 +717,8 @@ static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups,
                                uint32_t DestIdx) {
   // Any NumToSkip fixups in the current scope can resolve to the
   // current location.
-  for (uint32_t FixupIdx : reverse(Fixups)) {
-    // Calculate the distance from the byte following the fixup entry byte
-    // to the destination. The Target is calculated from after the 24-bit
-    // NumToSkip entry itself, so subtract three from the displacement here
-    // to account for that.
-    uint32_t Delta = DestIdx - FixupIdx - 3;
-    // Our NumToSkip entries are 24-bits. Make sure our table isn't too
-    // big.
-    assert(isUInt<24>(Delta));
-    Table[FixupIdx] = (uint8_t)Delta;
-    Table[FixupIdx + 1] = (uint8_t)(Delta >> 8);
-    Table[FixupIdx + 2] = (uint8_t)(Delta >> 16);
-  }
+  for (uint32_t FixupIdx : Fixups)
+    Table.patchNumToSkip(FixupIdx, DestIdx);
 }
 
 // Emit table entries to decode instructions given a segment or segments
@@ -759,15 +775,9 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
     Delegate->emitTableEntries(TableInfo);
 
     // Now that we've emitted the body of the handler, update the NumToSkip
-    // of the filter itself to be able to skip forward when false. Subtract
-    // three as to account for the width of the NumToSkip field itself.
-    if (PrevFilter) {
-      uint32_t NumToSkip = Table.size() - PrevFilter - 3;
-      assert(isUInt<24>(NumToSkip) && "disassembler decoding table too large!");
-      Table[PrevFilter] = (uint8_t)NumToSkip;
-      Table[PrevFilter + 1] = (uint8_t)(NumToSkip >> 8);
-      Table[PrevFilter + 2] = (uint8_t)(NumToSkip >> 16);
-    }
+    // of the filter itself to be able to skip forward when false.
+    if (PrevFilter)
+      Table.patchNumToSkip(PrevFilter, Table.size());
   }
 
   // If there is no fallthrough, then the final filter should get fixed
@@ -814,7 +824,8 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     OS << (unsigned)*I++ << ", ";
   };
 
-  // Emit 24-bit numtoskip value to OS, returning the NumToSkip value.
+  // Emit `NumToSkipSizeInBytes`-byte numtoskip value to OS, returning the
+  // NumToSkip value.
   auto emitNumToSkip = [](DecoderTable::const_iterator &I,
                           formatted_raw_ostream &OS) {
     uint8_t Byte = *I++;
@@ -823,9 +834,11 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     Byte = *I++;
     OS << (unsigned)Byte << ", ";
     NumToSkip |= Byte << 8;
-    Byte = *I++;
-    OS << (unsigned)(Byte) << ", ";
-    NumToSkip |= Byte << 16;
+    if (NumToSkipSizeInBytes == 3) {
+      Byte = *I++;
+      OS << (unsigned)(Byte) << ", ";
+      NumToSkip |= Byte << 16;
+    }
     return NumToSkip;
   };
 
@@ -867,7 +880,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // The filter value is ULEB128 encoded.
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
+      // numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -883,7 +896,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // ULEB128 encoded field value.
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
+      // numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -893,7 +906,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       OS << Indent << "MCD::OPC_CheckPredicate, ";
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
+      // numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -925,7 +938,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
 
       // Fallthrough for OPC_TryDecode.
 
-      // 24-bit numtoskip value.
+      // numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
 
       OS << "// Opcode: " << NumberedEncodings[EncodingID]
@@ -1411,9 +1424,9 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
     TableInfo.Table.push_back(NumBits);
     TableInfo.Table.insertULEB128(Ilnd.FieldVal);
 
-    // The fixup is always 24-bits, so go ahead and allocate the space
-    // in the table so all our relative position calculations work OK even
-    // before we fully resolve the real value here.
+    // Allocate space in the table for fixup (NumToSkipSizeInBytes) so all
+    // our relative position calculations work OK even before we fully
+    // resolve the real value here.
 
     // Push location for NumToSkip backpatching.
     TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
@@ -2157,7 +2170,18 @@ insertBits(InsnType &field, uint64_t bits, unsigned startBit, unsigned numBits)
 // decodeInstruction().
 static void emitDecodeInstruction(formatted_raw_ostream &OS,
                                   bool IsVarLenInst) {
+  OS << formatv("\nconstexpr unsigned NumToSkipSizeInBytes = {};\n",
+                NumToSkipSizeInBytes);
+
   OS << R"(
+inline unsigned decodeNumToSkip(const uint8_t *&Ptr) {
+  unsigned NumToSkip = *Ptr++;
+  NumToSkip |= (*Ptr++) << 8;
+  if constexpr (NumToSkipSizeInBytes == 3)
+    NumToSkip |= (*Ptr++) << 16;
+  return NumToSkip;
+}
+
 template <typename InsnType>
 static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
                                       InsnType insn, uint64_t Address,
@@ -2195,10 +2219,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the field value.
       uint64_t Val = decodeULEB128AndIncUnsafe(++Ptr);
       bool Failed = Val != CurFieldValue;
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // Perform the filter operation.
       if (Failed)
@@ -2222,10 +2243,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       uint64_t ExpectedValue = decodeULEB128(++Ptr, &PtrLen);
       Ptr += PtrLen;
       bool Failed = ExpectedValue != FieldValue;
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // If the actual and expected values don't match, skip.
       if (Failed)
@@ -2240,10 +2258,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     case MCD::OPC_CheckPredicate: {
       // Decode the Predicate Index value.
       unsigned PIdx = decodeULEB128AndIncUnsafe(++Ptr);
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
       // Check the predicate.
       bool Failed = !checkDecoderPredicate(PIdx, Bits);
       if (Failed)
@@ -2278,10 +2293,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the Opcode value.
       unsigned Opc = decodeULEB128AndIncUnsafe(++Ptr);
       unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // Perform the decode operation.
       MCInst TmpMI;
@@ -2406,6 +2418,9 @@ handleHwModesUnrelatedEncodings(const CodeGenInstruction *Instr,
 
 // Emits disassembler code for instruction decoding.
 void DecoderEmitter::run(raw_ostream &o) {
+  if (NumToSkipSizeInBytes != 2 && NumToSkipSizeInBytes != 3)
+    PrintFatalError("Invalid value for num-to-skip-size, must be 2 or 3");
+
   formatted_raw_ostream OS(o);
   OS << R"(
 #include "llvm/MC/MCInst.h"

From 80855eb6f11b06c194939eb305761eb2b62822f9 Mon Sep 17 00:00:00 2001
From: Lei Wang <wlei@fb.com>
Date: Wed, 16 Apr 2025 13:11:55 -0700
Subject: [PATCH 171/710] [SampleFDO] Extend the function base name max size
 (#135863)

The function base name could be way long which overflows and leads to a
crash. Update to extend the max size.

Also changed to use heap allocation( `std::vector<char>` ) to avoid
stack overflow.
---
 llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
index d6d1b7c51d4c0..963c321772d6e 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
@@ -737,11 +737,11 @@ bool SampleProfileMatcher::functionMatchesProfileHelper(
     auto FunctionName = FName.str();
     if (Demangler.partialDemangle(FunctionName.c_str()))
       return std::string();
-    constexpr size_t MaxBaseNameSize = 4096;
-    char BaseNameBuf[MaxBaseNameSize] = {};
+    constexpr size_t MaxBaseNameSize = 65536;
+    std::vector<char> BaseNameBuf(MaxBaseNameSize, 0);
     size_t BaseNameSize = MaxBaseNameSize;
     char *BaseNamePtr =
-        Demangler.getFunctionBaseName(BaseNameBuf, &BaseNameSize);
+        Demangler.getFunctionBaseName(BaseNameBuf.data(), &BaseNameSize);
     return (BaseNamePtr && BaseNameSize)
                ? std::string(BaseNamePtr, BaseNameSize)
                : std::string();

From 05aa98955c697e52209d775327013e9cc2be6321 Mon Sep 17 00:00:00 2001
From: calebwat <107081575+calebwat@users.noreply.github.com>
Date: Wed, 16 Apr 2025 13:12:25 -0700
Subject: [PATCH 172/710] [NFCI] Explicitly delete unused copy constructor and
 assign copy for VPInterleavedAccessInfo (#134755)

VPInterleavedAccessInfo has a defined destructor freeing memory, but no
explicitly defined copy constructor or copy assignment op. These are not
used, so this patch marks them as deleted to avoid usage of the
implicitly defined implementations.
---
 llvm/lib/Transforms/Vectorize/VPlanSLP.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.h b/llvm/lib/Transforms/Vectorize/VPlanSLP.h
index 93f04e6e30a6f..7f123689170ad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.h
@@ -48,6 +48,8 @@ class VPInterleavedAccessInfo {
 
 public:
   VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI);
+  VPInterleavedAccessInfo(const VPInterleavedAccessInfo &) = delete;
+  VPInterleavedAccessInfo &operator=(const VPInterleavedAccessInfo &) = delete;
 
   ~VPInterleavedAccessInfo() {
     // Avoid releasing a pointer twice.

From 7fd0c8acd4659ccd0aef5486afe32c8ddf0f2957 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Wed, 16 Apr 2025 13:16:32 -0700
Subject: [PATCH 173/710] Revert "[LLVM][TableGen] Parameterize NumToSkip in
 DecoderEmitter" (#136017)

Reverts llvm/llvm-project#135882

Causing assert failures for AArch64 backend
---
 llvm/lib/Target/AArch64/CMakeLists.txt    |   2 +-
 llvm/test/TableGen/VarLenDecoder.td       |   4 +-
 llvm/test/TableGen/trydecode-emission.td  |  10 +-
 llvm/test/TableGen/trydecode-emission2.td |  16 +--
 llvm/test/TableGen/trydecode-emission3.td |   2 +-
 llvm/test/TableGen/trydecode-emission4.td |   2 +-
 llvm/utils/TableGen/DecoderEmitter.cpp    | 115 ++++++++++------------
 7 files changed, 68 insertions(+), 83 deletions(-)

diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index ba1d1605ec104..2300e479bc110 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -7,7 +7,7 @@ tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
 tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
 tablegen(LLVM AArch64GenCallingConv.inc -gen-callingconv)
 tablegen(LLVM AArch64GenDAGISel.inc -gen-dag-isel)
-tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler --num-to-skip-size=3)
+tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler)
 tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel)
 tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel)
 tablegen(LLVM AArch64GenO0PreLegalizeGICombiner.inc -gen-global-isel-combiner
diff --git a/llvm/test/TableGen/VarLenDecoder.td b/llvm/test/TableGen/VarLenDecoder.td
index b77702ff7c5c1..5cf0bf8911859 100644
--- a/llvm/test/TableGen/VarLenDecoder.td
+++ b/llvm/test/TableGen/VarLenDecoder.td
@@ -47,9 +47,9 @@ def FOO32 : MyVarInst<MemOp32> {
 }
 
 // CHECK:      MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
-// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, // Skip to: 11
+// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, 0, // Skip to: 12
 // CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
-// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, // Skip to: 19
+// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, 0, // Skip to: 21
 // CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
 // CHECK-NEXT: MCD::OPC_Fail,
 
diff --git a/llvm/test/TableGen/trydecode-emission.td b/llvm/test/TableGen/trydecode-emission.td
index 2b4239f4fbe65..20d2446eeac7f 100644
--- a/llvm/test/TableGen/trydecode-emission.td
+++ b/llvm/test/TableGen/trydecode-emission.td
@@ -34,10 +34,10 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23
-// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19
-// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 19
-// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
+// CHECK-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
+// CHECK-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
+// CHECK-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
+// CHECK-NEXT: /* 26 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission2.td b/llvm/test/TableGen/trydecode-emission2.td
index 7d30474058f73..0584034e41233 100644
--- a/llvm/test/TableGen/trydecode-emission2.td
+++ b/llvm/test/TableGen/trydecode-emission2.td
@@ -31,14 +31,14 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 31, 0, // Skip to: 38
-// CHECK-NEXT: /* 7 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
-// CHECK-NEXT: /* 10 */      MCD::OPC_FilterValue, 0, 24, 0, // Skip to: 38
-// CHECK-NEXT: /* 14 */      MCD::OPC_CheckField, 0, 2, 3, 6, 0, // Skip to: 26
-// CHECK-NEXT: /* 20 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 26
-// CHECK-NEXT: /* 26 */      MCD::OPC_CheckField, 3, 2, 0, 6, 0, // Skip to: 38
-// CHECK-NEXT: /* 32 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, // Opcode: InstA, skip to: 38
-// CHECK-NEXT: /* 38 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 36, 0, 0, // Skip to: 44
+// CHECK-NEXT: /* 8 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-NEXT: /* 11 */      MCD::OPC_FilterValue, 0, 28, 0, 0, // Skip to: 44
+// CHECK-NEXT: /* 16 */      MCD::OPC_CheckField, 0, 2, 3, 7, 0, 0, // Skip to: 30
+// CHECK-NEXT: /* 23 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 30
+// CHECK-NEXT: /* 30 */      MCD::OPC_CheckField, 3, 2, 0, 7, 0, 0, // Skip to: 44
+// CHECK-NEXT: /* 37 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstA, skip to: 44
+// CHECK-NEXT: /* 44 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 // CHECK: if (!Check(S, DecodeInstA(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission3.td b/llvm/test/TableGen/trydecode-emission3.td
index 0abbe62fe337e..4c5be7e1af229 100644
--- a/llvm/test/TableGen/trydecode-emission3.td
+++ b/llvm/test/TableGen/trydecode-emission3.td
@@ -1,4 +1,4 @@
- // RUN: llvm-tblgen -gen-disassembler --num-to-skip-size=3 -I %p/../../include %s  | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
 
 include "llvm/Target/Target.td"
 
diff --git a/llvm/test/TableGen/trydecode-emission4.td b/llvm/test/TableGen/trydecode-emission4.td
index 413e4a0d1275a..1e51ba5e40768 100644
--- a/llvm/test/TableGen/trydecode-emission4.td
+++ b/llvm/test/TableGen/trydecode-emission4.td
@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-disassembler --num-to-skip-size=3 -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
 
 // Test for OPC_ExtractField/OPC_CheckField with start bit > 255.
 // These large start values may arise for architectures with long instruction
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 0735702abf339..9c6015cc24576 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -32,10 +32,8 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/LEB128.h"
-#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -78,12 +76,6 @@ static cl::opt<SuppressLevel> DecoderEmitterSuppressDuplicates(
             "significantly reducing Table Duplications")),
     cl::init(SUPPRESSION_DISABLE), cl::cat(DisassemblerEmitterCat));
 
-static cl::opt<uint32_t>
-    NumToSkipSizeInBytes("num-to-skip-size",
-                         cl::desc("number of bytes to use for num-to-skip "
-                                  "entries in the decoder table (2 or 3)"),
-                         cl::init(2), cl::cat(DisassemblerEmitterCat));
-
 STATISTIC(NumEncodings, "Number of encodings considered");
 STATISTIC(NumEncodingsLackingDisasm,
           "Number of encodings without disassembler info");
@@ -138,29 +130,10 @@ struct DecoderTable : public std::vector<uint8_t> {
   // in the table for patching.
   size_t insertNumToSkip() {
     size_t Size = size();
-    insert(end(), NumToSkipSizeInBytes, 0);
+    insert(end(), 3, 0);
     return Size;
   }
-
-  void patchNumToSkip(size_t FixupIdx, uint32_t DestIdx) {
-    // Calculate the distance from the byte following the fixup entry byte
-    // to the destination. The Target is calculated from after the
-    // `NumToSkipSizeInBytes`-byte NumToSkip entry itself, so subtract
-    // `NumToSkipSizeInBytes` from the displacement here to account for that.
-    assert(DestIdx > FixupIdx + NumToSkipSizeInBytes &&
-           "Expecting a forward jump in the decoding table");
-    uint32_t Delta = DestIdx - FixupIdx - NumToSkipSizeInBytes;
-    if (!isUIntN(8 * NumToSkipSizeInBytes, Delta))
-      PrintFatalError(
-          "disassembler decoding table too large, try --num-to-skip-size=3");
-
-    (*this)[FixupIdx] = static_cast<uint8_t>(Delta);
-    (*this)[FixupIdx + 1] = static_cast<uint8_t>(Delta >> 8);
-    if (NumToSkipSizeInBytes == 3)
-      (*this)[FixupIdx + 2] = static_cast<uint8_t>(Delta >> 16);
-  }
 };
-
 struct DecoderTableInfo {
   DecoderTable Table;
   FixupScopeList FixupStack;
@@ -717,8 +690,19 @@ static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups,
                                uint32_t DestIdx) {
   // Any NumToSkip fixups in the current scope can resolve to the
   // current location.
-  for (uint32_t FixupIdx : Fixups)
-    Table.patchNumToSkip(FixupIdx, DestIdx);
+  for (uint32_t FixupIdx : reverse(Fixups)) {
+    // Calculate the distance from the byte following the fixup entry byte
+    // to the destination. The Target is calculated from after the 24-bit
+    // NumToSkip entry itself, so subtract three from the displacement here
+    // to account for that.
+    uint32_t Delta = DestIdx - FixupIdx - 3;
+    // Our NumToSkip entries are 24-bits. Make sure our table isn't too
+    // big.
+    assert(isUInt<24>(Delta));
+    Table[FixupIdx] = (uint8_t)Delta;
+    Table[FixupIdx + 1] = (uint8_t)(Delta >> 8);
+    Table[FixupIdx + 2] = (uint8_t)(Delta >> 16);
+  }
 }
 
 // Emit table entries to decode instructions given a segment or segments
@@ -775,9 +759,15 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
     Delegate->emitTableEntries(TableInfo);
 
     // Now that we've emitted the body of the handler, update the NumToSkip
-    // of the filter itself to be able to skip forward when false.
-    if (PrevFilter)
-      Table.patchNumToSkip(PrevFilter, Table.size());
+    // of the filter itself to be able to skip forward when false. Subtract
+    // three as to account for the width of the NumToSkip field itself.
+    if (PrevFilter) {
+      uint32_t NumToSkip = Table.size() - PrevFilter - 3;
+      assert(isUInt<24>(NumToSkip) && "disassembler decoding table too large!");
+      Table[PrevFilter] = (uint8_t)NumToSkip;
+      Table[PrevFilter + 1] = (uint8_t)(NumToSkip >> 8);
+      Table[PrevFilter + 2] = (uint8_t)(NumToSkip >> 16);
+    }
   }
 
   // If there is no fallthrough, then the final filter should get fixed
@@ -824,8 +814,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     OS << (unsigned)*I++ << ", ";
   };
 
-  // Emit `NumToSkipSizeInBytes`-byte numtoskip value to OS, returning the
-  // NumToSkip value.
+  // Emit 24-bit numtoskip value to OS, returning the NumToSkip value.
   auto emitNumToSkip = [](DecoderTable::const_iterator &I,
                           formatted_raw_ostream &OS) {
     uint8_t Byte = *I++;
@@ -834,11 +823,9 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     Byte = *I++;
     OS << (unsigned)Byte << ", ";
     NumToSkip |= Byte << 8;
-    if (NumToSkipSizeInBytes == 3) {
-      Byte = *I++;
-      OS << (unsigned)(Byte) << ", ";
-      NumToSkip |= Byte << 16;
-    }
+    Byte = *I++;
+    OS << (unsigned)(Byte) << ", ";
+    NumToSkip |= Byte << 16;
     return NumToSkip;
   };
 
@@ -880,7 +867,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // The filter value is ULEB128 encoded.
       emitULEB128(I, OS);
 
-      // numtoskip value.
+      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -896,7 +883,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // ULEB128 encoded field value.
       emitULEB128(I, OS);
 
-      // numtoskip value.
+      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -906,7 +893,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       OS << Indent << "MCD::OPC_CheckPredicate, ";
       emitULEB128(I, OS);
 
-      // numtoskip value.
+      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -938,7 +925,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
 
       // Fallthrough for OPC_TryDecode.
 
-      // numtoskip value.
+      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
 
       OS << "// Opcode: " << NumberedEncodings[EncodingID]
@@ -1424,9 +1411,9 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
     TableInfo.Table.push_back(NumBits);
     TableInfo.Table.insertULEB128(Ilnd.FieldVal);
 
-    // Allocate space in the table for fixup (NumToSkipSizeInBytes) so all
-    // our relative position calculations work OK even before we fully
-    // resolve the real value here.
+    // The fixup is always 24-bits, so go ahead and allocate the space
+    // in the table so all our relative position calculations work OK even
+    // before we fully resolve the real value here.
 
     // Push location for NumToSkip backpatching.
     TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
@@ -2170,18 +2157,7 @@ insertBits(InsnType &field, uint64_t bits, unsigned startBit, unsigned numBits)
 // decodeInstruction().
 static void emitDecodeInstruction(formatted_raw_ostream &OS,
                                   bool IsVarLenInst) {
-  OS << formatv("\nconstexpr unsigned NumToSkipSizeInBytes = {};\n",
-                NumToSkipSizeInBytes);
-
   OS << R"(
-inline unsigned decodeNumToSkip(const uint8_t *&Ptr) {
-  unsigned NumToSkip = *Ptr++;
-  NumToSkip |= (*Ptr++) << 8;
-  if constexpr (NumToSkipSizeInBytes == 3)
-    NumToSkip |= (*Ptr++) << 16;
-  return NumToSkip;
-}
-
 template <typename InsnType>
 static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
                                       InsnType insn, uint64_t Address,
@@ -2219,7 +2195,10 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the field value.
       uint64_t Val = decodeULEB128AndIncUnsafe(++Ptr);
       bool Failed = Val != CurFieldValue;
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      // NumToSkip is a plain 24-bit integer.
+      unsigned NumToSkip = *Ptr++;
+      NumToSkip |= (*Ptr++) << 8;
+      NumToSkip |= (*Ptr++) << 16;
 
       // Perform the filter operation.
       if (Failed)
@@ -2243,7 +2222,10 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       uint64_t ExpectedValue = decodeULEB128(++Ptr, &PtrLen);
       Ptr += PtrLen;
       bool Failed = ExpectedValue != FieldValue;
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      // NumToSkip is a plain 24-bit integer.
+      unsigned NumToSkip = *Ptr++;
+      NumToSkip |= (*Ptr++) << 8;
+      NumToSkip |= (*Ptr++) << 16;
 
       // If the actual and expected values don't match, skip.
       if (Failed)
@@ -2258,7 +2240,10 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     case MCD::OPC_CheckPredicate: {
       // Decode the Predicate Index value.
       unsigned PIdx = decodeULEB128AndIncUnsafe(++Ptr);
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      // NumToSkip is a plain 24-bit integer.
+      unsigned NumToSkip = *Ptr++;
+      NumToSkip |= (*Ptr++) << 8;
+      NumToSkip |= (*Ptr++) << 16;
       // Check the predicate.
       bool Failed = !checkDecoderPredicate(PIdx, Bits);
       if (Failed)
@@ -2293,7 +2278,10 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the Opcode value.
       unsigned Opc = decodeULEB128AndIncUnsafe(++Ptr);
       unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      // NumToSkip is a plain 24-bit integer.
+      unsigned NumToSkip = *Ptr++;
+      NumToSkip |= (*Ptr++) << 8;
+      NumToSkip |= (*Ptr++) << 16;
 
       // Perform the decode operation.
       MCInst TmpMI;
@@ -2418,9 +2406,6 @@ handleHwModesUnrelatedEncodings(const CodeGenInstruction *Instr,
 
 // Emits disassembler code for instruction decoding.
 void DecoderEmitter::run(raw_ostream &o) {
-  if (NumToSkipSizeInBytes != 2 && NumToSkipSizeInBytes != 3)
-    PrintFatalError("Invalid value for num-to-skip-size, must be 2 or 3");
-
   formatted_raw_ostream OS(o);
   OS << R"(
 #include "llvm/MC/MCInst.h"

From c7fae59ac540ced666f664c88c2a49e06352a8dc Mon Sep 17 00:00:00 2001
From: James Newling <james.newling@gmail.com>
Date: Wed, 16 Apr 2025 13:20:05 -0700
Subject: [PATCH 174/710] [mlir][vector] Move extract_strided_slice
 canonicalization to folding (#135676)

Folders are preferred:
https://mlir.llvm.org/docs/Canonicalization/#when-to-use-the-fold-method-vs-rewriterpatterns-for-canonicalizations

Included here : some missing `-----` between lit test file with mlir-opt
with `-split-input-file` flag
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp   | 165 ++++++++-------------
 mlir/test/Dialect/Vector/canonicalize.mlir |   7 +
 2 files changed, 69 insertions(+), 103 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 4b2fba03ce551..cea31276d10a8 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -3714,12 +3714,67 @@ foldExtractStridedOpFromInsertChain(ExtractStridedSliceOp op) {
   return failure();
 }
 
+// ExtractStridedSliceOp(non-splat ConstantOp) -> ConstantOp.
+static OpFoldResult
+foldExtractStridedSliceNonSplatConstant(ExtractStridedSliceOp op,
+                                        Attribute foldInput) {
+
+  auto dense = llvm::dyn_cast_if_present<DenseElementsAttr>(foldInput);
+  if (!dense)
+    return {};
+
+  // TODO: Handle non-unit strides when they become available.
+  if (op.hasNonUnitStrides())
+    return {};
+
+  VectorType sourceVecTy = op.getSourceVectorType();
+  ArrayRef<int64_t> sourceShape = sourceVecTy.getShape();
+  SmallVector<int64_t, 4> sourceStrides = computeStrides(sourceShape);
+
+  VectorType sliceVecTy = op.getType();
+  ArrayRef<int64_t> sliceShape = sliceVecTy.getShape();
+  int64_t rank = sliceVecTy.getRank();
+
+  // Expand offsets and sizes to match the vector rank.
+  SmallVector<int64_t, 4> offsets(rank, 0);
+  copy(getI64SubArray(op.getOffsets()), offsets.begin());
+
+  SmallVector<int64_t, 4> sizes(sourceShape);
+  copy(getI64SubArray(op.getSizes()), sizes.begin());
+
+  // Calculate the slice elements by enumerating all slice positions and
+  // linearizing them. The enumeration order is lexicographic which yields a
+  // sequence of monotonically increasing linearized position indices.
+  const auto denseValuesBegin = dense.value_begin<Attribute>();
+  SmallVector<Attribute> sliceValues;
+  sliceValues.reserve(sliceVecTy.getNumElements());
+  SmallVector<int64_t> currSlicePosition(offsets.begin(), offsets.end());
+  do {
+    int64_t linearizedPosition = linearize(currSlicePosition, sourceStrides);
+    assert(linearizedPosition < sourceVecTy.getNumElements() &&
+           "Invalid index");
+    sliceValues.push_back(*(denseValuesBegin + linearizedPosition));
+  } while (succeeded(incSlicePosition(currSlicePosition, sliceShape, offsets)));
+
+  assert(static_cast<int64_t>(sliceValues.size()) ==
+             sliceVecTy.getNumElements() &&
+         "Invalid number of slice elements");
+  return DenseElementsAttr::get(sliceVecTy, sliceValues);
+}
+
 OpFoldResult ExtractStridedSliceOp::fold(FoldAdaptor adaptor) {
   if (getSourceVectorType() == getResult().getType())
     return getVector();
   if (succeeded(foldExtractStridedOpFromInsertChain(*this)))
     return getResult();
-  return {};
+
+  // ExtractStridedSliceOp(splat ConstantOp) -> ConstantOp.
+  if (auto splat =
+          llvm::dyn_cast_if_present<SplatElementsAttr>(adaptor.getVector()))
+    DenseElementsAttr::get(getType(), splat.getSplatValue<Attribute>());
+
+  // ExtractStridedSliceOp(non-splat ConstantOp) -> ConstantOp.
+  return foldExtractStridedSliceNonSplatConstant(*this, adaptor.getVector());
 }
 
 void ExtractStridedSliceOp::getOffsets(SmallVectorImpl<int64_t> &results) {
@@ -3783,98 +3838,6 @@ class StridedSliceConstantMaskFolder final
   }
 };
 
-// Pattern to rewrite a ExtractStridedSliceOp(splat ConstantOp) -> ConstantOp.
-class StridedSliceSplatConstantFolder final
-    : public OpRewritePattern<ExtractStridedSliceOp> {
-public:
-  using OpRewritePattern::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(ExtractStridedSliceOp extractStridedSliceOp,
-                                PatternRewriter &rewriter) const override {
-    // Return if 'ExtractStridedSliceOp' operand is not defined by a splat
-    // ConstantOp.
-    Value sourceVector = extractStridedSliceOp.getVector();
-    Attribute vectorCst;
-    if (!matchPattern(sourceVector, m_Constant(&vectorCst)))
-      return failure();
-
-    auto splat = llvm::dyn_cast<SplatElementsAttr>(vectorCst);
-    if (!splat)
-      return failure();
-
-    auto newAttr = SplatElementsAttr::get(extractStridedSliceOp.getType(),
-                                          splat.getSplatValue<Attribute>());
-    rewriter.replaceOpWithNewOp<arith::ConstantOp>(extractStridedSliceOp,
-                                                   newAttr);
-    return success();
-  }
-};
-
-// Pattern to rewrite a ExtractStridedSliceOp(non-splat ConstantOp) ->
-// ConstantOp.
-class StridedSliceNonSplatConstantFolder final
-    : public OpRewritePattern<ExtractStridedSliceOp> {
-public:
-  using OpRewritePattern::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(ExtractStridedSliceOp extractStridedSliceOp,
-                                PatternRewriter &rewriter) const override {
-    // Return if 'ExtractStridedSliceOp' operand is not defined by a non-splat
-    // ConstantOp.
-    Value sourceVector = extractStridedSliceOp.getVector();
-    Attribute vectorCst;
-    if (!matchPattern(sourceVector, m_Constant(&vectorCst)))
-      return failure();
-
-    // The splat case is handled by `StridedSliceSplatConstantFolder`.
-    auto dense = llvm::dyn_cast<DenseElementsAttr>(vectorCst);
-    if (!dense || dense.isSplat())
-      return failure();
-
-    // TODO: Handle non-unit strides when they become available.
-    if (extractStridedSliceOp.hasNonUnitStrides())
-      return failure();
-
-    auto sourceVecTy = llvm::cast<VectorType>(sourceVector.getType());
-    ArrayRef<int64_t> sourceShape = sourceVecTy.getShape();
-    SmallVector<int64_t, 4> sourceStrides = computeStrides(sourceShape);
-
-    VectorType sliceVecTy = extractStridedSliceOp.getType();
-    ArrayRef<int64_t> sliceShape = sliceVecTy.getShape();
-    int64_t sliceRank = sliceVecTy.getRank();
-
-    // Expand offsets and sizes to match the vector rank.
-    SmallVector<int64_t, 4> offsets(sliceRank, 0);
-    copy(getI64SubArray(extractStridedSliceOp.getOffsets()), offsets.begin());
-
-    SmallVector<int64_t, 4> sizes(sourceShape);
-    copy(getI64SubArray(extractStridedSliceOp.getSizes()), sizes.begin());
-
-    // Calculate the slice elements by enumerating all slice positions and
-    // linearizing them. The enumeration order is lexicographic which yields a
-    // sequence of monotonically increasing linearized position indices.
-    auto denseValuesBegin = dense.value_begin<Attribute>();
-    SmallVector<Attribute> sliceValues;
-    sliceValues.reserve(sliceVecTy.getNumElements());
-    SmallVector<int64_t> currSlicePosition(offsets.begin(), offsets.end());
-    do {
-      int64_t linearizedPosition = linearize(currSlicePosition, sourceStrides);
-      assert(linearizedPosition < sourceVecTy.getNumElements() &&
-             "Invalid index");
-      sliceValues.push_back(*(denseValuesBegin + linearizedPosition));
-    } while (
-        succeeded(incSlicePosition(currSlicePosition, sliceShape, offsets)));
-
-    assert(static_cast<int64_t>(sliceValues.size()) ==
-               sliceVecTy.getNumElements() &&
-           "Invalid number of slice elements");
-    auto newAttr = DenseElementsAttr::get(sliceVecTy, sliceValues);
-    rewriter.replaceOpWithNewOp<arith::ConstantOp>(extractStridedSliceOp,
-                                                   newAttr);
-    return success();
-  }
-};
-
 // Pattern to rewrite an ExtractStridedSliceOp(BroadcastOp) to
 // BroadcastOp(ExtractStrideSliceOp).
 class StridedSliceBroadcast final
@@ -4018,8 +3981,7 @@ void ExtractStridedSliceOp::getCanonicalizationPatterns(
     RewritePatternSet &results, MLIRContext *context) {
   // Pattern to rewrite a ExtractStridedSliceOp(ConstantMaskOp) ->
   // ConstantMaskOp and ExtractStridedSliceOp(ConstantOp) -> ConstantOp.
-  results.add<StridedSliceConstantMaskFolder, StridedSliceSplatConstantFolder,
-              StridedSliceNonSplatConstantFolder, StridedSliceBroadcast,
+  results.add<StridedSliceConstantMaskFolder, StridedSliceBroadcast,
               StridedSliceSplat, ContiguousExtractStridedSliceToExtract>(
       context);
 }
@@ -5659,10 +5621,8 @@ OpFoldResult ShapeCastOp::fold(FoldAdaptor adaptor) {
 
   // shape_cast(constant) -> constant
   if (auto splatAttr =
-          llvm::dyn_cast_if_present<SplatElementsAttr>(adaptor.getSource())) {
-    return DenseElementsAttr::get(resultType,
-                                  splatAttr.getSplatValue<Attribute>());
-  }
+          llvm::dyn_cast_if_present<SplatElementsAttr>(adaptor.getSource()))
+    return splatAttr.reshape(getType());
 
   // shape_cast(poison) -> poison
   if (llvm::dyn_cast_if_present<ub::PoisonAttr>(adaptor.getSource())) {
@@ -6006,10 +5966,9 @@ void vector::TransposeOp::build(OpBuilder &builder, OperationState &result,
 
 OpFoldResult vector::TransposeOp::fold(FoldAdaptor adaptor) {
   // Eliminate splat constant transpose ops.
-  if (auto attr =
-          llvm::dyn_cast_if_present<DenseElementsAttr>(adaptor.getVector()))
-    if (attr.isSplat())
-      return attr.reshape(getResultVectorType());
+  if (auto splat =
+          llvm::dyn_cast_if_present<SplatElementsAttr>(adaptor.getVector()))
+    return splat.reshape(getResultVectorType());
 
   // Eliminate poison transpose ops.
   if (llvm::dyn_cast_if_present<ub::PoisonAttr>(adaptor.getVector()))
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index b24cf93707d8b..f50d92fc9c8b1 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -1121,6 +1121,8 @@ func.func @bitcast_folding(%I1: vector<4x8xf32>, %I2: vector<2xi32>) -> (vector<
   return %0, %2 : vector<4x8xf32>, vector<2xi32>
 }
 
+// -----
+
 // CHECK-LABEL: func @bitcast_f16_to_f32
 //              bit pattern: 0x40004000
 //       CHECK-DAG: %[[CST1:.+]] = arith.constant dense<2.00390625> : vector<4xf32>
@@ -1135,6 +1137,8 @@ func.func @bitcast_f16_to_f32() -> (vector<4xf32>, vector<4xf32>) {
   return %cast0, %cast1: vector<4xf32>, vector<4xf32>
 }
 
+// -----
+
 // CHECK-LABEL: func @bitcast_i8_to_i32
 //              bit pattern: 0xA0A0A0A0
 //       CHECK-DAG: %[[CST1:.+]] = arith.constant dense<-1600085856> : vector<4xi32>
@@ -1732,6 +1736,7 @@ func.func @vector_multi_reduction_unit_dimensions(%source: vector<5x1x4x1x20xf32
 }
 
 // -----
+
 // CHECK-LABEL:   func.func @vector_multi_reduction_scalable(
 // CHECK-SAME:     %[[VAL_0:.*]]: vector<1x[4]x1xf32>,
 // CHECK-SAME:     %[[VAL_1:.*]]: vector<1x[4]xf32>,
@@ -2249,6 +2254,8 @@ func.func @transpose_splat_constant() -> vector<8x4xf32> {
   return %0 : vector<8x4xf32>
 }
 
+// -----
+
 // CHECK-LABEL:   func @transpose_splat2(
 // CHECK-SAME:                           %[[VAL_0:.*]]: f32) -> vector<3x4xf32> {
 // CHECK:           %[[VAL_1:.*]] = vector.splat %[[VAL_0]] : vector<3x4xf32>

From cbbf6b487360a0926c88d512c986b3640136f91e Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 13:55:01 -0700
Subject: [PATCH 175/710] Revert "[NFC][CFI] Dump test output to debug
 llvm-clang-win-x-aarch64 failure" (#136029)

Reverts llvm/llvm-project#136002

Not needed any more.
---
 clang/test/Driver/sanitizer-ld.c | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index c04831171bba9..a00ec029d3d46 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -839,14 +839,6 @@
 // CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-flto'
 // CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-fvisibility='
 
-// CFI by itself does not link runtime libraries.
-// RUN: %clang -fsanitize=cfi \
-// RUN:     -flto -fvisibility=hidden \
-// RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \
-// RUN:     -resource-dir=%S/Inputs/resource_dir \
-// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
-// RUN:     -### %s
-
 // CFI by itself does not link runtime libraries.
 // RUN: %clang -fsanitize=cfi \
 // RUN:     -flto -fvisibility=hidden \

From 6b0c8c4acd31eba83bf20ab1cf8729b2153e383c Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 13:56:38 -0700
Subject: [PATCH 176/710] Revert "[NFC][CFI] Avoid clang error in CFI tests"
 (#136030)

Reverts llvm/llvm-project#135981

Fails with 'clang: error: --rtlib=libgcc requires --unwindlib=libgcc' on
some bots.
---
 clang/test/Driver/sanitizer-ld.c | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index a00ec029d3d46..67ca33d676d20 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -840,8 +840,7 @@
 // CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-fvisibility='
 
 // CFI by itself does not link runtime libraries.
-// RUN: %clang -fsanitize=cfi \
-// RUN:     -flto -fvisibility=hidden \
+// RUN: not %clang -fsanitize=cfi \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -850,8 +849,7 @@
 // CHECK-CFI-LINUX: "{{.*}}ld{{(.exe)?}}"
 
 // CFI with diagnostics links the UBSan runtime.
-// RUN: %clang -fsanitize=cfi -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
-// RUN:     -flto -fvisibility=hidden \
+// RUN: not %clang -fsanitize=cfi -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -861,8 +859,7 @@
 // CHECK-CFI-DIAG-LINUX: "--whole-archive" "{{[^"]*}}libclang_rt.ubsan_standalone.a" "--no-whole-archive"
 
 // Cross-DSO CFI links the CFI runtime.
-// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
-// RUN:     -flto -fvisibility=hidden \
+// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -873,8 +870,7 @@
 // CHECK-CFI-CROSS-DSO-LINUX: -export-dynamic
 
 // Cross-DSO CFI with diagnostics links just the CFI runtime.
-// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
-// RUN:     -flto -fvisibility=hidden \
+// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
 // RUN:     -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
@@ -886,8 +882,7 @@
 // CHECK-CFI-CROSS-DSO-DIAG-LINUX: -export-dynamic
 
 // Cross-DSO CFI on Android does not link runtime libraries.
-// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
-// RUN:     -flto -fvisibility=hidden \
+// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
 // RUN:     --target=aarch64-linux-android -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree \
@@ -896,8 +891,7 @@
 // CHECK-CFI-CROSS-DSO-ANDROID: "{{.*}}ld{{(.exe)?}}"
 
 // Cross-DSO CFI with diagnostics on Android links just the UBSAN runtime.
-// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
-// RUN:     -flto -fvisibility=hidden \
+// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
 // RUN:     -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=aarch64-linux-android -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \

From 41c1a7be3f1a2556e407e761acb766a5d103d691 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 16 Apr 2025 22:58:09 +0200
Subject: [PATCH 177/710] [LV] Don't add fixed-order recurrence phis to forced
 scalars.

Fixed-order recurrence phis cannot be forced to be scalar, they will
always be widened at the moment.

Make sure we don't add them to ForcedScalars, otherwise the legacy cost
model will compute incorrect costs.

This fixes an assertion reported with
https://github.com/llvm/llvm-project/pull/129645.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |   7 +-
 .../X86/fixed-order-recurrence.ll             | 100 +++++++++++++++++-
 2 files changed, 105 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index dd7f05465a50b..e9e1d0ac6c196 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6464,10 +6464,15 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
                  getMemoryInstructionCost(Member, ElementCount::getFixed(1))));
         }
       }
-    } else
+    } else {
+      // Cannot scalarize fixed-order recurrence phis at the moment.
+      if (isa<PHINode>(I) && Legal->isFixedOrderRecurrence(cast<PHINode>(I)))
+        continue;
+
       // Make sure I gets scalarized and a cost estimate without
       // scalarization overhead.
       ForcedScalars[VF].insert(I);
+    }
   }
 }
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index bc57aeb775fb1..838df4d0caf09 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -91,7 +91,7 @@ for.body:                                         ; preds = %for.body.preheader,
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
-define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapture noundef writeonly %y, i32 noundef %n) #0 {
+define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapture noundef writeonly %y, i32 noundef %n) {
 ; CHECK-LABEL: @thirdorderrec(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP38:%.*]] = icmp sgt i32 [[N:%.*]], 3
@@ -352,3 +352,101 @@ loop:
 exit:
   ret void
 }
+
+define void @test_for_tried_to_force_scalar(ptr noalias %A, ptr noalias %B, ptr noalias %C, i64 %n) #0 {
+; CHECK-LABEL: @test_for_tried_to_force_scalar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
+; CHECK-NEXT:    [[CONFLICT_RDX20:%.*]] = icmp ule i64 [[TMP0]], 8
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX20]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 8, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP4]]
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x ptr> poison, ptr [[A:%.*]], i32 3
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x ptr> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <4 x ptr> [[TMP17]], ptr [[TMP14]], i32 1
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x ptr> [[TMP18]], ptr [[TMP15]], i32 2
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <4 x ptr> [[TMP19]], ptr [[TMP16]], i32 3
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP21]], i32 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x ptr> [[TMP25]], ptr [[TMP22]], i32 1
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x ptr> [[TMP26]], ptr [[TMP23]], i32 2
+; CHECK-NEXT:    [[TMP28]] = insertelement <4 x ptr> [[TMP27]], ptr [[TMP24]], i32 3
+; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <4 x ptr> [[TMP20]], <4 x ptr> [[TMP28]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <12 x float>, ptr [[TMP21]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x float> [[WIDE_VEC]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x float> [[STRIDED_VEC]], i32 3
+; CHECK-NEXT:    store float [[TMP30]], ptr [[C:%.*]], align 4
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x ptr> [[TMP29]], i32 0
+; CHECK-NEXT:    [[TMP38:%.*]] = load float, ptr [[TMP31]], align 4
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <4 x ptr> [[TMP29]], i32 1
+; CHECK-NEXT:    [[TMP32:%.*]] = load float, ptr [[TMP33]], align 4
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x ptr> [[TMP29]], i32 2
+; CHECK-NEXT:    [[TMP34:%.*]] = load float, ptr [[TMP35]], align 4
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x ptr> [[TMP29]], i32 3
+; CHECK-NEXT:    [[TMP36:%.*]] = load float, ptr [[TMP37]], align 4
+; CHECK-NEXT:    store float [[TMP36]], ptr [[B:%.*]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi ptr [ [[TMP24]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PREV:%.*]] = phi ptr [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP40:%.*]] = load float, ptr [[NEXT]], align 4
+; CHECK-NEXT:    store float [[TMP40]], ptr [[C]], align 4
+; CHECK-NEXT:    [[TMP41:%.*]] = load float, ptr [[PREV]], align 4
+; CHECK-NEXT:    store float [[TMP41]], ptr [[B]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %prev = phi ptr [ %A, %entry ], [ %next, %loop ]
+  %next = getelementptr nusw [3 x float], ptr %A, i64 %iv
+  %0 = load float, ptr %next, align 4
+  store float %0, ptr %C, align 4
+  %1 = load float, ptr %prev, align 4
+  store float %1, ptr %B, align 4
+  %iv.next = add nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-cpu"="znver3" }

From 42ad82bb059f1ba454c6f7d882984f38d6099d88 Mon Sep 17 00:00:00 2001
From: Tai Ly <tai.ly@arm.com>
Date: Wed, 16 Apr 2025 15:58:42 -0500
Subject: [PATCH 178/710] [mlir][tosa] Add verifier check for Slice Op
 (#135853)

Add verifier check for Slice Op to make sure input1 and output have same
ranks.
Added test in verifier.mlir
Also moved existing slice verifier tests in invalid.mlir to verfier.mlir

Signed-off-by: Tai Ly <tai.ly@arm.com>
---
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp | 29 ++++++++++++-------
 mlir/test/Dialect/Tosa/invalid.mlir  | 32 ---------------------
 mlir/test/Dialect/Tosa/verifier.mlir | 43 ++++++++++++++++++++++++++++
 3 files changed, 61 insertions(+), 43 deletions(-)

diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 9a1dfd8e17b85..8b4f6ef0d0980 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -1446,19 +1446,26 @@ LogicalResult tosa::SliceOp::verify() {
                              /* outType = */ getOutput().getType())
           .failed())
     return failure();
-  auto inputType = llvm::dyn_cast<RankedTensorType>(getInput1().getType());
-  if (!inputType)
-    return success();
 
-  auto startShapeRank =
-      llvm::cast<tosa::shapeType>(getStart().getType()).getRank();
-  if (inputType.getRank() != startShapeRank)
-    return emitOpError("length of start is not equal to rank of input shape");
+  const ShapeAdaptor inputShape(getInput1().getType());
+  if (inputShape.hasRank()) {
+    const auto inputRank = inputShape.getRank();
+    const ShapeAdaptor outputShape(getOutput().getType());
+    if (outputShape.hasRank() && inputRank != outputShape.getRank())
+      return emitOpError(
+                 "expect input1 and output to have the same ranks, got ")
+             << inputRank << " and " << outputShape.getRank();
+
+    const auto startShapeRank =
+        llvm::cast<tosa::shapeType>(getStart().getType()).getRank();
+    if (inputRank != startShapeRank)
+      return emitOpError("length of start is not equal to rank of input shape");
 
-  auto sizeShapeRank =
-      llvm::cast<tosa::shapeType>(getSize().getType()).getRank();
-  if (inputType.getRank() != sizeShapeRank)
-    return emitOpError("length of size is not equal to rank of input shape");
+    const auto sizeShapeRank =
+        llvm::cast<tosa::shapeType>(getSize().getType()).getRank();
+    if (inputRank != sizeShapeRank)
+      return emitOpError("length of size is not equal to rank of input shape");
+  }
 
   return success();
 }
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index c0b251885de5c..fc98aa95ed5b3 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -660,28 +660,6 @@ func.func @test_variable_write_shape(%arg0: tensor<1x4x8xi8>) -> () {
 
 // -----
 
-func.func @test_slice_invalid_start() {
-  %0 = tensor.empty() : tensor<4x31x31xf32>
-  %start = tosa.const_shape {values = dense<[1, 1]> : tensor<2xindex>} : () -> !tosa.shape<2>
-  %size = tosa.const_shape {values = dense<[1, 1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
-  // expected-error@+1 {{'tosa.slice' op length of start is not equal to rank of input shape}}
-  %3 = tosa.slice %0, %start, %size : (tensor<4x31x31xf32>, !tosa.shape<2>, !tosa.shape<3>) -> tensor<*xf32>
-  return
-}
-
-// -----
-
-func.func @test_slice_invalid_size() {
-  %0 = tensor.empty() : tensor<4x31x31xf32>
-  %start = tosa.const_shape {values = dense<[1, 1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
-  %size = tosa.const_shape {values = dense<[1]> : tensor<1xindex>} : () -> !tosa.shape<1>
-  // expected-error@+1 {{'tosa.slice' op length of size is not equal to rank of input shape}}
-  %3 = tosa.slice %0, %start, %size : (tensor<4x31x31xf32>, !tosa.shape<3>, !tosa.shape<1>) -> tensor<*xf32>
-  return
-}
-
-// -----
-
 func.func @test_tile_invalid_multiples() {
   %0 = tensor.empty() : tensor<4x31x31xf32>
   %cst = tosa.const_shape { values = dense<1> : tensor<1xindex> } : () -> !tosa.shape<1>
@@ -1938,16 +1916,6 @@ func.func @test_scalar_reverse(%arg0: tensor<f32>) -> tensor<f32> {
 
 // -----
 
-func.func @test_scalar_slice(%arg0: tensor<f32>) -> tensor<f32> {
-  %0 = tosa.const_shape {values = dense<[]> : tensor<0xindex>} : () -> !tosa.shape<0>
-  %1 = tosa.const_shape {values = dense<[]> : tensor<0xindex>} : () -> !tosa.shape<0>
-  // expected-error@+1 {{'tosa.slice' op operand #0 must be tosa-conformant tensor of at least rank 1, but got 'tensor<f32>'}}
-  %2 = tosa.slice %arg0, %0, %1 : (tensor<f32>, !tosa.shape<0>, !tosa.shape<0>) -> tensor<f32>
-  return %2 : tensor<f32>
-}
-
-// -----
-
 func.func @test_scalar_tile(%arg0: tensor<f32>) -> tensor<*xf32> {
   %cst = tosa.const_shape { values = dense<[]> : tensor<0xindex> } : () -> !tosa.shape<0>
   // expected-error@+1 {{'tosa.tile' op operand #0 must be tosa-conformant tensor of at least rank 1, but got 'tensor<f32>'}}
diff --git a/mlir/test/Dialect/Tosa/verifier.mlir b/mlir/test/Dialect/Tosa/verifier.mlir
index c49cbecd25c78..efdd26a9346fb 100644
--- a/mlir/test/Dialect/Tosa/verifier.mlir
+++ b/mlir/test/Dialect/Tosa/verifier.mlir
@@ -124,3 +124,46 @@ func.func @test_scalar_output_transpose(%arg0: tensor<*xf32>) -> tensor<f32> {
   %1 = tosa.transpose %arg0 {perms = array<i32: 2, 0, 1>} : (tensor<*xf32>) -> tensor<f32>
   return %1 : tensor<f32>
 }
+
+// -----
+
+func.func @test_slice_invalid_output_rank() {
+  %0 = tensor.empty() : tensor<4x31x31xf32>
+  %start = tosa.const_shape {values = dense<[1, 1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %size = tosa.const_shape {values = dense<[1, 1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  // expected-error@+1 {{'tosa.slice' op expect input1 and output to have the same ranks, got 3 and 4}}
+  %3 = tosa.slice %0, %start, %size : (tensor<4x31x31xf32>, !tosa.shape<2>, !tosa.shape<3>) -> tensor<?x?x?x?xf32>
+  return
+}
+
+// -----
+
+func.func @test_slice_invalid_start() {
+  %0 = tensor.empty() : tensor<4x31x31xf32>
+  %start = tosa.const_shape {values = dense<[1, 1]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %size = tosa.const_shape {values = dense<[1, 1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  // expected-error@+1 {{'tosa.slice' op length of start is not equal to rank of input shape}}
+  %3 = tosa.slice %0, %start, %size : (tensor<4x31x31xf32>, !tosa.shape<2>, !tosa.shape<3>) -> tensor<*xf32>
+  return
+}
+
+// -----
+
+func.func @test_slice_invalid_size() {
+  %0 = tensor.empty() : tensor<4x31x31xf32>
+  %start = tosa.const_shape {values = dense<[1, 1, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %size = tosa.const_shape {values = dense<[1]> : tensor<1xindex>} : () -> !tosa.shape<1>
+  // expected-error@+1 {{'tosa.slice' op length of size is not equal to rank of input shape}}
+  %3 = tosa.slice %0, %start, %size : (tensor<4x31x31xf32>, !tosa.shape<3>, !tosa.shape<1>) -> tensor<*xf32>
+  return
+}
+
+// -----
+
+func.func @test_scalar_slice(%arg0: tensor<f32>) -> tensor<f32> {
+  %0 = tosa.const_shape {values = dense<[]> : tensor<0xindex>} : () -> !tosa.shape<0>
+  %1 = tosa.const_shape {values = dense<[]> : tensor<0xindex>} : () -> !tosa.shape<0>
+  // expected-error@+1 {{'tosa.slice' op operand #0 must be tosa-conformant tensor of at least rank 1, but got 'tensor<f32>'}}
+  %2 = tosa.slice %arg0, %0, %1 : (tensor<f32>, !tosa.shape<0>, !tosa.shape<0>) -> tensor<f32>
+  return %2 : tensor<f32>
+}

From 2afef58e40ba953c0848577106dee51819b9be8f Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Thu, 17 Apr 2025 00:03:34 +0300
Subject: [PATCH 179/710] [ARM] Use helper class for emitting CFI instructions
 into MIR (#135994)

Similar to #135845.

PR: https://github.com/llvm/llvm-project/pull/135994
---
 llvm/include/llvm/CodeGen/CFIInstBuilder.h  |  20 ++++
 llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp    |  89 +++--------------
 llvm/lib/Target/ARM/ARMBaseInstrInfo.h      |  10 --
 llvm/lib/Target/ARM/ARMFrameLowering.cpp    | 104 ++++++--------------
 llvm/lib/Target/ARM/Thumb1FrameLowering.cpp |  81 ++++-----------
 5 files changed, 83 insertions(+), 221 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/CFIInstBuilder.h b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
index e799b47a0c974..9025624c0d8ab 100644
--- a/llvm/include/llvm/CodeGen/CFIInstBuilder.h
+++ b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
@@ -45,6 +45,10 @@ class CFIInstBuilder {
     setInsertPoint(InsertPt);
   }
 
+  CFIInstBuilder(MachineBasicBlock *MBB, MachineInstr::MIFlag MIFlag,
+                 bool IsEH = true)
+      : CFIInstBuilder(*MBB, MBB->end(), MIFlag, IsEH) {}
+
   void setInsertPoint(MachineBasicBlock::iterator IP) { InsertPt = IP; }
 
   void insertCFIInst(const MCCFIInstruction &CFIInst) const {
@@ -72,11 +76,27 @@ class CFIInstBuilder {
         nullptr, TRI.getDwarfRegNum(Reg, IsEH), Offset));
   }
 
+  void buildRegister(MCRegister Reg1, MCRegister Reg2) const {
+    insertCFIInst(MCCFIInstruction::createRegister(
+        nullptr, TRI.getDwarfRegNum(Reg1, IsEH),
+        TRI.getDwarfRegNum(Reg2, IsEH)));
+  }
+
   void buildRestore(MCRegister Reg) const {
     insertCFIInst(MCCFIInstruction::createRestore(
         nullptr, TRI.getDwarfRegNum(Reg, IsEH)));
   }
 
+  void buildUndefined(MCRegister Reg) const {
+    insertCFIInst(MCCFIInstruction::createUndefined(
+        nullptr, TRI.getDwarfRegNum(Reg, IsEH)));
+  }
+
+  void buildSameValue(MCRegister Reg) const {
+    insertCFIInst(MCCFIInstruction::createSameValue(
+        nullptr, TRI.getDwarfRegNum(Reg, IsEH)));
+  }
+
   void buildEscape(StringRef Bytes, StringRef Comment = "") const {
     insertCFIInst(
         MCCFIInstruction::createEscape(nullptr, Bytes, SMLoc(), Comment));
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 6843ec895e69c..69bc84a6733c0 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -24,6 +24,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/DFAPacketizer.h"
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -6485,51 +6486,20 @@ void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
   if (!CFI)
     return;
 
-  MachineFunction &MF = *MBB.getParent();
-
   // Add a CFI, saying CFA is offset by Align bytes from SP.
-  int64_t StackPosEntry =
-      MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align));
-  BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-      .addCFIIndex(StackPosEntry)
-      .setMIFlags(MachineInstr::FrameSetup);
+  CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
+  CFIBuilder.buildDefCFAOffset(Align);
 
   // Add a CFI saying that the LR that we want to find is now higher than
   // before.
   int LROffset = Auth ? Align - 4 : Align;
-  const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
-  unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
-  int64_t LRPosEntry = MF.addFrameInst(
-      MCCFIInstruction::createOffset(nullptr, DwarfLR, -LROffset));
-  BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-      .addCFIIndex(LRPosEntry)
-      .setMIFlags(MachineInstr::FrameSetup);
+  CFIBuilder.buildOffset(ARM::LR, -LROffset);
   if (Auth) {
     // Add a CFI for the location of the return adddress PAC.
-    unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true);
-    int64_t RACPosEntry = MF.addFrameInst(
-        MCCFIInstruction::createOffset(nullptr, DwarfRAC, -Align));
-    BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-        .addCFIIndex(RACPosEntry)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIBuilder.buildOffset(ARM::RA_AUTH_CODE, -Align);
   }
 }
 
-void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB,
-                                             MachineBasicBlock::iterator It,
-                                             Register Reg) const {
-  MachineFunction &MF = *MBB.getParent();
-  const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
-  unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
-  unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
-
-  int64_t LRPosEntry = MF.addFrameInst(
-      MCCFIInstruction::createRegister(nullptr, DwarfLR, DwarfReg));
-  BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-      .addCFIIndex(LRPosEntry)
-      .setMIFlags(MachineInstr::FrameSetup);
-}
-
 void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator It,
                                           bool CFI, bool Auth) const {
@@ -6560,50 +6530,18 @@ void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
   }
 
   if (CFI) {
-    // Now stack has moved back up...
-    MachineFunction &MF = *MBB.getParent();
-    const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
-    unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
-    int64_t StackPosEntry =
-        MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
-    BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-        .addCFIIndex(StackPosEntry)
-        .setMIFlags(MachineInstr::FrameDestroy);
-
-    // ... and we have restored LR.
-    int64_t LRPosEntry =
-        MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
-    BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-        .addCFIIndex(LRPosEntry)
-        .setMIFlags(MachineInstr::FrameDestroy);
-
-    if (Auth) {
-      unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true);
-      int64_t Entry =
-          MF.addFrameInst(MCCFIInstruction::createUndefined(nullptr, DwarfRAC));
-      BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-          .addCFIIndex(Entry)
-          .setMIFlags(MachineInstr::FrameDestroy);
-    }
+    // Now stack has moved back up and we have restored LR.
+    CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameDestroy);
+    CFIBuilder.buildDefCFAOffset(0);
+    CFIBuilder.buildRestore(ARM::LR);
+    if (Auth)
+      CFIBuilder.buildUndefined(ARM::RA_AUTH_CODE);
   }
 
   if (Auth)
     BuildMI(MBB, It, DebugLoc(), get(ARM::t2AUT));
 }
 
-void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
-  MachineFunction &MF = *MBB.getParent();
-  const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
-  unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
-
-  int64_t LRPosEntry =
-      MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
-  BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-      .addCFIIndex(LRPosEntry)
-      .setMIFlags(MachineInstr::FrameDestroy);
-}
-
 void ARMBaseInstrInfo::buildOutlinedFrame(
     MachineBasicBlock &MBB, MachineFunction &MF,
     const outliner::OutlinedFunction &OF) const {
@@ -6722,11 +6660,12 @@ MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall(
     // Save and restore LR from that register.
     copyPhysReg(MBB, It, DebugLoc(), Reg, ARM::LR, true);
     if (!AFI.isLRSpilled())
-      emitCFIForLRSaveToReg(MBB, It, Reg);
+      CFIInstBuilder(MBB, It, MachineInstr::FrameSetup)
+          .buildRegister(ARM::LR, Reg);
     CallPt = MBB.insert(It, CallMIB);
     copyPhysReg(MBB, It, DebugLoc(), ARM::LR, Reg, true);
     if (!AFI.isLRSpilled())
-      emitCFIForLRRestoreFromReg(MBB, It);
+      CFIInstBuilder(MBB, It, MachineInstr::FrameDestroy).buildRestore(ARM::LR);
     It--;
     return CallPt;
   }
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 35edd5bf003ef..987f5a0e3d824 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -409,16 +409,6 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
                           MachineBasicBlock::iterator It, bool CFI,
                           bool Auth) const;
 
-  /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It,
-  /// for the case when the LR is saved in the register \p Reg.
-  void emitCFIForLRSaveToReg(MachineBasicBlock &MBB,
-                             MachineBasicBlock::iterator It,
-                             Register Reg) const;
-
-  /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It,
-  /// after the LR is was restored from a register.
-  void emitCFIForLRRestoreFromReg(MachineBasicBlock &MBB,
-                                  MachineBasicBlock::iterator It) const;
   /// \brief Sets the offsets on outlined instructions in \p MBB which use SP
   /// so that they will be valid post-outlining.
   ///
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 475f53fc03399..d3a6504c9100e 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -120,6 +120,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -140,10 +141,7 @@
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Function.h"
 #include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCInstrDesc.h"
-#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
@@ -760,21 +758,16 @@ struct StackAdjustingInsts {
     Info->SPAdjust += ExtraBytes;
   }
 
-  void emitDefCFAOffsets(MachineBasicBlock &MBB, const DebugLoc &dl,
-                         const ARMBaseInstrInfo &TII, bool HasFP) {
-    MachineFunction &MF = *MBB.getParent();
+  void emitDefCFAOffsets(MachineBasicBlock &MBB, bool HasFP) {
+    CFIInstBuilder CFIBuilder(MBB, MBB.end(), MachineInstr::FrameSetup);
     unsigned CFAOffset = 0;
     for (auto &Info : Insts) {
       if (HasFP && !Info.BeforeFPSet)
         return;
 
       CFAOffset += Info.SPAdjust;
-      unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset));
-      BuildMI(MBB, std::next(Info.I), dl,
-              TII.get(TargetOpcode::CFI_INSTRUCTION))
-              .addCFIIndex(CFIIndex)
-              .setMIFlags(MachineInstr::FrameSetup);
+      CFIBuilder.setInsertPoint(std::next(Info.I));
+      CFIBuilder.buildDefCFAOffset(CFAOffset);
     }
   }
 
@@ -890,9 +883,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo  &MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-  MCContext &Context = MF.getContext();
   const TargetMachine &TM = MF.getTarget();
-  const MCRegisterInfo *MRI = Context.getRegisterInfo();
   const ARMBaseRegisterInfo *RegInfo = STI.getRegisterInfo();
   const ARMBaseInstrInfo &TII = *STI.getInstrInfo();
   assert(!AFI->isThumb1OnlyFunction() &&
@@ -938,7 +929,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
       DefCFAOffsetCandidates.addInst(std::prev(MBBI), NumBytes, true);
     }
     if (!NeedsWinCFI)
-      DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
+      DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, HasFP);
     if (NeedsWinCFI && MBBI != MBB.begin()) {
       insertSEHRange(MBB, {}, MBBI, TII, MachineInstr::FrameSetup);
       BuildMI(MBB, MBBI, dl, TII.get(ARM::SEH_PrologEnd))
@@ -1245,21 +1236,11 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
     if (!NeedsWinCFI) {
       // Emit DWARF info to find the CFA using the frame pointer from this
       // point onward.
-      if (FPOffsetAfterPush != 0) {
-        unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
-            nullptr, MRI->getDwarfRegNum(FramePtr, true),
-            -MFI.getObjectOffset(FramePtrSpillFI)));
-        BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex)
-            .setMIFlags(MachineInstr::FrameSetup);
-      } else {
-        unsigned CFIIndex =
-            MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(
-                nullptr, MRI->getDwarfRegNum(FramePtr, true)));
-        BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex)
-            .setMIFlags(MachineInstr::FrameSetup);
-      }
+      CFIInstBuilder CFIBuilder(MBB, AfterPush, MachineInstr::FrameSetup);
+      if (FPOffsetAfterPush != 0)
+        CFIBuilder.buildDefCFA(FramePtr, -MFI.getObjectOffset(FramePtrSpillFI));
+      else
+        CFIBuilder.buildDefCFARegister(FramePtr);
     }
   }
 
@@ -1304,14 +1285,9 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
       }
 
       if (CFIPos.isValid()) {
-        int CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
-            nullptr,
-            MRI->getDwarfRegNum(Reg == ARM::R12 ? ARM::RA_AUTH_CODE : Reg,
-                                true),
-            MFI.getObjectOffset(FI)));
-        BuildMI(MBB, CFIPos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex)
-            .setMIFlags(MachineInstr::FrameSetup);
+        CFIInstBuilder(MBB, CFIPos, MachineInstr::FrameSetup)
+            .buildOffset(Reg == ARM::R12 ? ARM::RA_AUTH_CODE : Reg,
+                         MFI.getObjectOffset(FI));
       }
     }
   }
@@ -1322,7 +1298,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   // actually get emitted.
   if (!NeedsWinCFI) {
     LLVM_DEBUG(DefCFAOffsetCandidates.dump());
-    DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
+    DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, HasFP);
   }
 
   if (STI.isTargetELF() && hasFP(MF))
@@ -3155,7 +3131,6 @@ static const uint64_t kSplitStackAvailable = 256;
 void ARMFrameLowering::adjustForSegmentedStacks(
     MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
   unsigned Opcode;
-  unsigned CFIIndex;
   const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
   bool Thumb = ST->isThumb();
   bool Thumb2 = ST->isThumb2();
@@ -3168,8 +3143,6 @@ void ARMFrameLowering::adjustForSegmentedStacks(
     report_fatal_error("Segmented stacks not supported on this platform.");
 
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MCContext &Context = MF.getContext();
-  const MCRegisterInfo *MRI = Context.getRegisterInfo();
   const ARMBaseInstrInfo &TII =
       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
   ARMFunctionInfo *ARMFI = MF.getInfo<ARMFunctionInfo>();
@@ -3267,17 +3240,10 @@ void ARMFrameLowering::adjustForSegmentedStacks(
   // Emit the relevant DWARF information about the change in stack pointer as
   // well as where to find both r4 and r5 (the callee-save registers)
   if (!MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 8));
-    BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
-        nullptr, MRI->getDwarfRegNum(ScratchReg1, true), -4));
-    BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
-        nullptr, MRI->getDwarfRegNum(ScratchReg0, true), -8));
-    BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
+    CFIInstBuilder CFIBuilder(PrevStackMBB, MachineInstr::NoFlags);
+    CFIBuilder.buildDefCFAOffset(8);
+    CFIBuilder.buildOffset(ScratchReg1, -4);
+    CFIBuilder.buildOffset(ScratchReg0, -8);
   }
 
   // mov SR1, sp
@@ -3486,13 +3452,9 @@ void ARMFrameLowering::adjustForSegmentedStacks(
   // Emit the DWARF info about the change in stack as well as where to find the
   // previous link register
   if (!MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 12));
-    BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
-        nullptr, MRI->getDwarfRegNum(ARM::LR, true), -12));
-    BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
+    CFIInstBuilder CFIBuilder(AllocMBB, MachineInstr::NoFlags);
+    CFIBuilder.buildDefCFAOffset(12);
+    CFIBuilder.buildOffset(ARM::LR, -12);
   }
 
   // Call __morestack().
@@ -3549,11 +3511,8 @@ void ARMFrameLowering::adjustForSegmentedStacks(
   }
 
   // Update the CFA offset now that we've popped
-  if (!MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
-    BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
-  }
+  if (!MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
+    CFIInstBuilder(AllocMBB, MachineInstr::NoFlags).buildDefCFAOffset(0);
 
   // Return from this function.
   BuildMI(AllocMBB, DL, TII.get(ST->getReturnOpcode())).add(predOps(ARMCC::AL));
@@ -3576,20 +3535,13 @@ void ARMFrameLowering::adjustForSegmentedStacks(
 
   // Update the CFA offset now that we've popped
   if (!MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
-    BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
+    CFIInstBuilder CFIBuilder(PostStackMBB, MachineInstr::NoFlags);
+    CFIBuilder.buildDefCFAOffset(0);
 
     // Tell debuggers that r4 and r5 are now the same as they were in the
     // previous function, that they're the "Same Value".
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(
-        nullptr, MRI->getDwarfRegNum(ScratchReg0, true)));
-    BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
-    CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(
-        nullptr, MRI->getDwarfRegNum(ScratchReg1, true)));
-    BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
+    CFIBuilder.buildSameValue(ScratchReg0);
+    CFIBuilder.buildSameValue(ScratchReg1);
   }
 
   // Organizing MBB lists
diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
index a69e307a5da20..b04e20a0b6709 100644
--- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
+++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -34,9 +35,6 @@
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugLoc.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCDwarf.h"
-#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cassert>
@@ -150,7 +148,6 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
-  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
   const ThumbRegisterInfo *RegInfo =
       static_cast<const ThumbRegisterInfo *>(STI.getRegisterInfo());
   const Thumb1InstrInfo &TII =
@@ -180,16 +177,13 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
   // belongs to which callee-save spill areas.
   unsigned FRSize = 0, GPRCS1Size = 0, GPRCS2Size = 0, DPRCSSize = 0;
   int FramePtrSpillFI = 0;
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
 
   if (ArgRegsSaveSize) {
     emitPrologueEpilogueSPUpdate(MBB, MBBI, TII, dl, *RegInfo, -ArgRegsSaveSize,
                                  ARM::NoRegister, MachineInstr::FrameSetup);
     CFAOffset += ArgRegsSaveSize;
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset));
-    BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIBuilder.buildDefCFAOffset(CFAOffset);
   }
 
   if (!AFI->hasStackFrame()) {
@@ -198,11 +192,7 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
                                    -(NumBytes - ArgRegsSaveSize),
                                    ARM::NoRegister, MachineInstr::FrameSetup);
       CFAOffset += NumBytes - ArgRegsSaveSize;
-      unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset));
-      BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameSetup);
+      CFIBuilder.buildDefCFAOffset(CFAOffset);
     }
     return;
   }
@@ -340,20 +330,11 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
           .add(predOps(ARMCC::AL));
     }
 
-    if(FramePtrOffsetInBlock) {
-      unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
-          nullptr, MRI->getDwarfRegNum(FramePtr, true), (CFAOffset - FramePtrOffsetInBlock)));
-      BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameSetup);
-    } else {
-      unsigned CFIIndex =
-          MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(
-              nullptr, MRI->getDwarfRegNum(FramePtr, true)));
-      BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameSetup);
-    }
+    CFIBuilder.setInsertPoint(AfterPush);
+    if (FramePtrOffsetInBlock)
+      CFIBuilder.buildDefCFA(FramePtr, CFAOffset - FramePtrOffsetInBlock);
+    else
+      CFIBuilder.buildDefCFARegister(FramePtr);
     if (NumBytes > 508)
       // If offset is > 508 then sp cannot be adjusted in a single instruction,
       // try restoring from fp instead.
@@ -362,18 +343,11 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
 
   // Emit call frame information for the callee-saved low registers.
   if (GPRCS1Size > 0) {
-    MachineBasicBlock::iterator Pos = std::next(GPRCS1Push);
-    if (adjustedGPRCS1Size) {
-      unsigned CFIIndex =
-          MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset));
-      BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameSetup);
-    }
+    CFIBuilder.setInsertPoint(std::next(GPRCS1Push));
+    if (adjustedGPRCS1Size)
+      CFIBuilder.buildDefCFAOffset(CFAOffset);
     for (const CalleeSavedInfo &I : CSI) {
-      MCRegister Reg = I.getReg();
-      int FI = I.getFrameIdx();
-      switch (Reg) {
+      switch (I.getReg()) {
       case ARM::R8:
       case ARM::R9:
       case ARM::R10:
@@ -389,11 +363,8 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
       case ARM::R6:
       case ARM::R7:
       case ARM::LR:
-        unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
-            nullptr, MRI->getDwarfRegNum(Reg, true), MFI.getObjectOffset(FI)));
-        BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex)
-            .setMIFlags(MachineInstr::FrameSetup);
+        CFIBuilder.buildOffset(I.getReg(),
+                               MFI.getObjectOffset(I.getFrameIdx()));
         break;
       }
     }
@@ -401,23 +372,17 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
 
   // Emit call frame information for the callee-saved high registers.
   if (GPRCS2Size > 0) {
-    MachineBasicBlock::iterator Pos = std::next(GPRCS2Push);
+    CFIBuilder.setInsertPoint(std::next(GPRCS2Push));
     for (auto &I : CSI) {
-      MCRegister Reg = I.getReg();
-      int FI = I.getFrameIdx();
-      switch (Reg) {
+      switch (I.getReg()) {
       case ARM::R8:
       case ARM::R9:
       case ARM::R10:
       case ARM::R11:
-      case ARM::R12: {
-        unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
-            nullptr, MRI->getDwarfRegNum(Reg, true), MFI.getObjectOffset(FI)));
-        BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex)
-            .setMIFlags(MachineInstr::FrameSetup);
+      case ARM::R12:
+        CFIBuilder.buildOffset(I.getReg(),
+                               MFI.getObjectOffset(I.getFrameIdx()));
         break;
-      }
       default:
         break;
       }
@@ -442,11 +407,7 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF,
                                  ScratchRegister, MachineInstr::FrameSetup);
     if (!HasFP) {
       CFAOffset += NumBytes;
-      unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset));
-      BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameSetup);
+      CFIBuilder.buildDefCFAOffset(CFAOffset);
     }
   }
 

From a9827fbc86b3d2973b9eef7bfb8f726dd75f17a5 Mon Sep 17 00:00:00 2001
From: Prabhu Rajasekaran <prabhukr@google.com>
Date: Wed, 16 Apr 2025 14:14:43 -0700
Subject: [PATCH 180/710] [llvm] Handle CPI symbols for UEFI (#135652)

UEFI targets besides MSVC environment must support constant pool
symbols.
---
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 1d72a1e116ecd..4b05ad398f053 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -4113,7 +4113,8 @@ const MCExpr *AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
 
 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
 MCSymbol *AsmPrinter::GetCPISymbol(unsigned CPID) const {
-  if (getSubtargetInfo().getTargetTriple().isWindowsMSVCEnvironment()) {
+  if (getSubtargetInfo().getTargetTriple().isWindowsMSVCEnvironment() ||
+      getSubtargetInfo().getTargetTriple().isUEFI()) {
     const MachineConstantPoolEntry &CPE =
         MF->getConstantPool()->getConstants()[CPID];
     if (!CPE.isMachineConstantPoolEntry()) {

From 9dbe107219c7ab1c422300f9eeb9ca3f7fc87c53 Mon Sep 17 00:00:00 2001
From: Shubham Sandeep Rastogi <srastogi22@apple.com>
Date: Wed, 16 Apr 2025 14:41:20 -0700
Subject: [PATCH 181/710] disable test on older compilers (#136037)

---
 .../TestDataFormatterLibcxxInvalidVectorSimulator.py            | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
index 8788ea7be882d..3f58018b0fbd9 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
@@ -13,6 +13,8 @@
 class LibcxxInvalidVectorDataFormatterSimulatorTestCase(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
+
+    @skipIf(compiler="clang", compiler_version=['<', '15.0.1'])
     def test(self):
         self.build()
         lldbutil.run_to_source_breakpoint(self, "return 0", lldb.SBFileSpec("main.cpp"))

From c409da2223ad910d2a0fb491c19623c679e8aae3 Mon Sep 17 00:00:00 2001
From: Muzammil <55665739+Muzammiluddin-Syed-ECE@users.noreply.github.com>
Date: Wed, 16 Apr 2025 17:53:17 -0400
Subject: [PATCH 182/710] [mlir][ROCDL] Add permlanex16 op to allow subgroup
 reductions on gfx10+ (#135983)

Adding Permlanex16Op to ROCDL dialect to enable subgroup reduce lowering
to DPP ops for gfx 10+ devices.
See [this PR](https://github.com/llvm/llvm-project/pull/133204).

---------

Signed-off-by: Muzammiluddin Syed <muzasyed@amd.com>
---
 mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td | 16 ++++++++++++++++
 mlir/test/Dialect/LLVMIR/rocdl.mlir          | 10 ++++++++++
 mlir/test/Target/LLVMIR/rocdl.mlir           | 14 ++++++++++++++
 3 files changed, 40 insertions(+)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index 900155c274b4d..186a4f53f93cb 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -668,6 +668,22 @@ def ROCDL_DPPUpdateOp : ROCDL_IntrOp<"update.dpp", [], [0],
   }];
 }
 
+// PermLaneX16 intrinsic operation
+def ROCDL_PermlaneX16Op : ROCDL_IntrOp<"permlanex16", [], [0],
+    [AllTypesMatch<["res", "old", "src0"]>, AllTypesMatch<["src1", "src2"]>], 1, 0, 0,
+    [4, 5], ["fi", "boundControl"]>,
+  Arguments<(ins LLVM_Type:$old, LLVM_Type:$src0, LLVM_Type:$src1, LLVM_Type:$src2,
+             I1Attr:$fi, I1Attr:$boundControl)> {
+  let results = (outs LLVM_Type:$res);
+  let assemblyFormat = [{
+    attr-dict $old `,` $src0 `,` $src1 `,` $src2 `,` $fi `,` $boundControl `:` type($src0) `,` type($src1)
+  }];
+  let description = [{
+    Performs a `permlanex16` operation with the given operands, applying the
+    permutation specified by $fi to the provided inputs.
+  }];
+}
+
 def ROCDL_V2I16Type : FixedVectorOfLengthAndType<[2], [I16]>,
                         BuildableType<"::mlir::VectorType::get("
                           "{2},$_builder.getI16Type())">;
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index 828fd58544597..cda1a9ca5f1f6 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -889,6 +889,16 @@ llvm.func @rocdl.readlane(%src : f32) -> f32 {
 
 // -----
 
+llvm.func @rocdl.permlanex16(%src : f32) -> f32 {
+  %cst0 = llvm.mlir.constant(-1 : i32) : i32
+  // CHECK-LABEL: rocdl.permlanex16
+  // CHECK: rocdl.permlanex16 %{{.*}} %{{.*}}
+  %ret = rocdl.permlanex16 %src, %src, %cst0, %cst0, 0, -1 : f32, i32
+  llvm.return %ret : f32
+}
+
+// -----
+
 // expected-error@below {{attribute attached to unexpected op}}
 func.func private @expected_llvm_func() attributes { rocdl.kernel }
 
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index e70617bfff99e..3db1f7b2b6427 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -872,6 +872,20 @@ llvm.func @rocdl.make.buffer.rsrc.p7.p1(%ptr : !llvm.ptr<1>,
   llvm.return %rsrc : !llvm.ptr<7>
 }
 
+llvm.func @rocdl.permlanex16(%src0 : f32, %src1 : i32, %src2 : vector<2 x f32>, %src3 : vector<2 x i32>) -> f32 {
+  %cst0 = llvm.mlir.constant(-1 : i32) : i32
+  // CHECK-LABEL: rocdl.permlanex16
+  // CHECK: call float @llvm.amdgcn.permlanex16.f32(float %{{.*}}, float %{{.*}}, i32 -1, i32 -1, i1 false, i1 true)
+  %ret0 = rocdl.permlanex16 %src0, %src0, %cst0, %cst0, 0, -1 : f32, i32
+  // CHECK: call i32 @llvm.amdgcn.permlanex16.i32(i32 %{{.*}}, i32 %{{.*}}, i32 -1, i32 -1, i1 false, i1 true)
+  %ret1 = rocdl.permlanex16 %src1, %src1, %cst0, %cst0, 0, -1 : i32, i32
+  // CHECK: call <2 x float> @llvm.amdgcn.permlanex16.v2f32(<2 x float> %{{.*}}, <2 x float> %{{.*}}, i32 -1, i32 -1, i1 false, i1 true)
+  %ret2 = rocdl.permlanex16 %src2, %src2, %cst0, %cst0, 0, -1 : vector<2 x f32>, i32
+  // CHECK: call <2 x i32> @llvm.amdgcn.permlanex16.v2i32(<2 x i32> %{{.*}}, <2 x i32> %{{.*}}, i32 -1, i32 -1, i1 false, i1 true)
+  %ret3 = rocdl.permlanex16 %src3, %src3, %cst0, %cst0, 0, -1 : vector<2 x i32>, i32
+  llvm.return %ret0 : f32
+}
+
 llvm.func @rocdl.wmma.fp8(%arg0 : vector<2 x i32>, %arg1 : vector<8xf32>) -> vector<8xf32> {
   // CHECK: call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32(<2 x i32> %{{.*}}, <2 x i32> %{{.*}}, <8 x float> %{{.*}})
   %r0 = rocdl.wmma.f32.16x16x16.fp8_fp8 %arg0, %arg0, %arg1: (vector<2xi32>, vector<2xi32>, vector<8xf32>) -> vector<8xf32>

From 6727d588919a49ae14d0857d2a642099098c9194 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 14:59:18 -0700
Subject: [PATCH 183/710] [NFC][CFI] Remove unnecessary -rtlib=platform from
 tests (#136032)

Should allow to reland https://github.com/llvm/llvm-project/pull/135981
---
 clang/test/Driver/sanitizer-ld.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index 67ca33d676d20..0ace06bac3b31 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -831,7 +831,7 @@
 
 // CFI requirements.
 // RUN: not %clang -fsanitize=cfi \
-// RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \
+// RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
 // RUN:     -### %s 2>&1 \
@@ -841,7 +841,7 @@
 
 // CFI by itself does not link runtime libraries.
 // RUN: not %clang -fsanitize=cfi \
-// RUN:     --target=x86_64-unknown-linux -fuse-ld=ld -rtlib=platform \
+// RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
 // RUN:     -### %s 2>&1 \

From 7623501c056a38c665ccf718ad318fd16451e4cc Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 15:04:06 -0700
Subject: [PATCH 184/710] [asan] Fix build on fuchsia (#136042)

Does not link after #131756
---
 compiler-rt/lib/asan/asan_report.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compiler-rt/lib/asan/asan_report.cpp b/compiler-rt/lib/asan/asan_report.cpp
index e515f20548c00..8b22aebe7cafc 100644
--- a/compiler-rt/lib/asan/asan_report.cpp
+++ b/compiler-rt/lib/asan/asan_report.cpp
@@ -149,7 +149,9 @@ class ScopedInErrorReport {
     // 2. Subsequent error reporting avoids nested lock acquisition patterns.
     // 3. Eliminates the lock order inversion risk between libdl and ASan's
     //    thread registry.
+#if !SANITIZER_SYMBOLIZER_MARKUP
     Symbolizer::GetOrInit()->GetRefreshedListOfModules();
+#endif
 
     // Make sure the registry and sanitizer report mutexes are locked while
     // we're printing an error report.

From 697aa9995c24a977425e672d76a4a434384b16e3 Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental@gmail.com>
Date: Wed, 16 Apr 2025 18:17:09 -0400
Subject: [PATCH 185/710] [mlir][SMT] add python bindings (#135674)

This PR adds "rich" python bindings to SMT dialect.
---
 mlir/include/mlir-c/Dialect/SMT.h         | 69 ++++++++--------
 mlir/include/mlir-c/Target/ExportSMTLIB.h | 10 ++-
 mlir/lib/Bindings/Python/DialectSMT.cpp   | 83 +++++++++++++++++++
 mlir/lib/CAPI/Dialect/SMT.cpp             | 52 ++++++------
 mlir/lib/CAPI/Target/ExportSMTLIB.cpp     | 21 ++++-
 mlir/python/CMakeLists.txt                | 24 ++++++
 mlir/python/mlir/dialects/SMTOps.td       | 14 ++++
 mlir/python/mlir/dialects/smt.py          | 33 ++++++++
 mlir/test/CAPI/smt.c                      | 97 ++++++++++++-----------
 mlir/test/python/dialects/smt.py          | 87 ++++++++++++++++++++
 10 files changed, 378 insertions(+), 112 deletions(-)
 create mode 100644 mlir/lib/Bindings/Python/DialectSMT.cpp
 create mode 100644 mlir/python/mlir/dialects/SMTOps.td
 create mode 100644 mlir/python/mlir/dialects/smt.py
 create mode 100644 mlir/test/python/dialects/smt.py

diff --git a/mlir/include/mlir-c/Dialect/SMT.h b/mlir/include/mlir-c/Dialect/SMT.h
index d076dccce1b06..0ad64746f148b 100644
--- a/mlir/include/mlir-c/Dialect/SMT.h
+++ b/mlir/include/mlir-c/Dialect/SMT.h
@@ -26,82 +26,83 @@ MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(SMT, smt);
 //===----------------------------------------------------------------------===//
 
 /// Checks if the given type is any non-func SMT value type.
-MLIR_CAPI_EXPORTED bool smtTypeIsAnyNonFuncSMTValueType(MlirType type);
+MLIR_CAPI_EXPORTED bool mlirSMTTypeIsAnyNonFuncSMTValueType(MlirType type);
 
 /// Checks if the given type is any SMT value type.
-MLIR_CAPI_EXPORTED bool smtTypeIsAnySMTValueType(MlirType type);
+MLIR_CAPI_EXPORTED bool mlirSMTTypeIsAnySMTValueType(MlirType type);
 
 /// Checks if the given type is a smt::ArrayType.
-MLIR_CAPI_EXPORTED bool smtTypeIsAArray(MlirType type);
+MLIR_CAPI_EXPORTED bool mlirSMTTypeIsAArray(MlirType type);
 
 /// Creates an array type with the given domain and range types.
-MLIR_CAPI_EXPORTED MlirType smtTypeGetArray(MlirContext ctx,
-                                            MlirType domainType,
-                                            MlirType rangeType);
+MLIR_CAPI_EXPORTED MlirType mlirSMTTypeGetArray(MlirContext ctx,
+                                                MlirType domainType,
+                                                MlirType rangeType);
 
 /// Checks if the given type is a smt::BitVectorType.
-MLIR_CAPI_EXPORTED bool smtTypeIsABitVector(MlirType type);
+MLIR_CAPI_EXPORTED bool mlirSMTTypeIsABitVector(MlirType type);
 
 /// Creates a smt::BitVectorType with the given width.
-MLIR_CAPI_EXPORTED MlirType smtTypeGetBitVector(MlirContext ctx, int32_t width);
+MLIR_CAPI_EXPORTED MlirType mlirSMTTypeGetBitVector(MlirContext ctx,
+                                                    int32_t width);
 
 /// Checks if the given type is a smt::BoolType.
-MLIR_CAPI_EXPORTED bool smtTypeIsABool(MlirType type);
+MLIR_CAPI_EXPORTED bool mlirSMTTypeIsABool(MlirType type);
 
 /// Creates a smt::BoolType.
-MLIR_CAPI_EXPORTED MlirType smtTypeGetBool(MlirContext ctx);
+MLIR_CAPI_EXPORTED MlirType mlirSMTTypeGetBool(MlirContext ctx);
 
 /// Checks if the given type is a smt::IntType.
-MLIR_CAPI_EXPORTED bool smtTypeIsAInt(MlirType type);
+MLIR_CAPI_EXPORTED bool mlirSMTTypeIsAInt(MlirType type);
 
 /// Creates a smt::IntType.
-MLIR_CAPI_EXPORTED MlirType smtTypeGetInt(MlirContext ctx);
+MLIR_CAPI_EXPORTED MlirType mlirSMTTypeGetInt(MlirContext ctx);
 
 /// Checks if the given type is a smt::FuncType.
-MLIR_CAPI_EXPORTED bool smtTypeIsASMTFunc(MlirType type);
+MLIR_CAPI_EXPORTED bool mlirSMTTypeIsASMTFunc(MlirType type);
 
 /// Creates a smt::FuncType with the given domain and range types.
-MLIR_CAPI_EXPORTED MlirType smtTypeGetSMTFunc(MlirContext ctx,
-                                              size_t numberOfDomainTypes,
-                                              const MlirType *domainTypes,
-                                              MlirType rangeType);
+MLIR_CAPI_EXPORTED MlirType mlirSMTTypeGetSMTFunc(MlirContext ctx,
+                                                  size_t numberOfDomainTypes,
+                                                  const MlirType *domainTypes,
+                                                  MlirType rangeType);
 
 /// Checks if the given type is a smt::SortType.
-MLIR_CAPI_EXPORTED bool smtTypeIsASort(MlirType type);
+MLIR_CAPI_EXPORTED bool mlirSMTTypeIsASort(MlirType type);
 
 /// Creates a smt::SortType with the given identifier and sort parameters.
-MLIR_CAPI_EXPORTED MlirType smtTypeGetSort(MlirContext ctx,
-                                           MlirIdentifier identifier,
-                                           size_t numberOfSortParams,
-                                           const MlirType *sortParams);
+MLIR_CAPI_EXPORTED MlirType mlirSMTTypeGetSort(MlirContext ctx,
+                                               MlirIdentifier identifier,
+                                               size_t numberOfSortParams,
+                                               const MlirType *sortParams);
 
 //===----------------------------------------------------------------------===//
 // Attribute API.
 //===----------------------------------------------------------------------===//
 
 /// Checks if the given string is a valid smt::BVCmpPredicate.
-MLIR_CAPI_EXPORTED bool smtAttrCheckBVCmpPredicate(MlirContext ctx,
-                                                   MlirStringRef str);
+MLIR_CAPI_EXPORTED bool mlirSMTAttrCheckBVCmpPredicate(MlirContext ctx,
+                                                       MlirStringRef str);
 
 /// Checks if the given string is a valid smt::IntPredicate.
-MLIR_CAPI_EXPORTED bool smtAttrCheckIntPredicate(MlirContext ctx,
-                                                 MlirStringRef str);
+MLIR_CAPI_EXPORTED bool mlirSMTAttrCheckIntPredicate(MlirContext ctx,
+                                                     MlirStringRef str);
 
 /// Checks if the given attribute is a smt::SMTAttribute.
-MLIR_CAPI_EXPORTED bool smtAttrIsASMTAttribute(MlirAttribute attr);
+MLIR_CAPI_EXPORTED bool mlirSMTAttrIsASMTAttribute(MlirAttribute attr);
 
 /// Creates a smt::BitVectorAttr with the given value and width.
-MLIR_CAPI_EXPORTED MlirAttribute smtAttrGetBitVector(MlirContext ctx,
-                                                     uint64_t value,
-                                                     unsigned width);
+MLIR_CAPI_EXPORTED MlirAttribute mlirSMTAttrGetBitVector(MlirContext ctx,
+                                                         uint64_t value,
+                                                         unsigned width);
 
 /// Creates a smt::BVCmpPredicateAttr with the given string.
-MLIR_CAPI_EXPORTED MlirAttribute smtAttrGetBVCmpPredicate(MlirContext ctx,
-                                                          MlirStringRef str);
+MLIR_CAPI_EXPORTED MlirAttribute
+mlirSMTAttrGetBVCmpPredicate(MlirContext ctx, MlirStringRef str);
 
 /// Creates a smt::IntPredicateAttr with the given string.
-MLIR_CAPI_EXPORTED MlirAttribute smtAttrGetIntPredicate(MlirContext ctx,
-                                                        MlirStringRef str);
+MLIR_CAPI_EXPORTED MlirAttribute mlirSMTAttrGetIntPredicate(MlirContext ctx,
+                                                            MlirStringRef str);
 
 #ifdef __cplusplus
 }
diff --git a/mlir/include/mlir-c/Target/ExportSMTLIB.h b/mlir/include/mlir-c/Target/ExportSMTLIB.h
index 31f411c4a89c2..59beda54d289b 100644
--- a/mlir/include/mlir-c/Target/ExportSMTLIB.h
+++ b/mlir/include/mlir-c/Target/ExportSMTLIB.h
@@ -21,9 +21,13 @@ extern "C" {
 
 /// Emits SMTLIB for the specified module using the provided callback and user
 /// data
-MLIR_CAPI_EXPORTED MlirLogicalResult mlirExportSMTLIB(MlirModule,
-                                                      MlirStringCallback,
-                                                      void *userData);
+MLIR_CAPI_EXPORTED MlirLogicalResult
+mlirTranslateModuleToSMTLIB(MlirModule, MlirStringCallback, void *userData,
+                            bool inlineSingleUseValues, bool indentLetBody);
+
+MLIR_CAPI_EXPORTED MlirLogicalResult mlirTranslateOperationToSMTLIB(
+    MlirOperation, MlirStringCallback, void *userData,
+    bool inlineSingleUseValues, bool indentLetBody);
 
 #ifdef __cplusplus
 }
diff --git a/mlir/lib/Bindings/Python/DialectSMT.cpp b/mlir/lib/Bindings/Python/DialectSMT.cpp
new file mode 100644
index 0000000000000..4e7647729fb0a
--- /dev/null
+++ b/mlir/lib/Bindings/Python/DialectSMT.cpp
@@ -0,0 +1,83 @@
+//===- DialectSMT.cpp - Pybind module for SMT dialect API support ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NanobindUtils.h"
+
+#include "mlir-c/Dialect/SMT.h"
+#include "mlir-c/IR.h"
+#include "mlir-c/Support.h"
+#include "mlir-c/Target/ExportSMTLIB.h"
+#include "mlir/Bindings/Python/Diagnostics.h"
+#include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
+
+namespace nb = nanobind;
+
+using namespace nanobind::literals;
+
+using namespace mlir;
+using namespace mlir::python;
+using namespace mlir::python::nanobind_adaptors;
+
+void populateDialectSMTSubmodule(nanobind::module_ &m) {
+
+  auto smtBoolType = mlir_type_subclass(m, "BoolType", mlirSMTTypeIsABool)
+                         .def_classmethod(
+                             "get",
+                             [](const nb::object &, MlirContext context) {
+                               return mlirSMTTypeGetBool(context);
+                             },
+                             "cls"_a, "context"_a.none() = nb::none());
+  auto smtBitVectorType =
+      mlir_type_subclass(m, "BitVectorType", mlirSMTTypeIsABitVector)
+          .def_classmethod(
+              "get",
+              [](const nb::object &, int32_t width, MlirContext context) {
+                return mlirSMTTypeGetBitVector(context, width);
+              },
+              "cls"_a, "width"_a, "context"_a.none() = nb::none());
+
+  auto exportSMTLIB = [](MlirOperation module, bool inlineSingleUseValues,
+                         bool indentLetBody) {
+    mlir::python::CollectDiagnosticsToStringScope scope(
+        mlirOperationGetContext(module));
+    PyPrintAccumulator printAccum;
+    MlirLogicalResult result = mlirTranslateOperationToSMTLIB(
+        module, printAccum.getCallback(), printAccum.getUserData(),
+        inlineSingleUseValues, indentLetBody);
+    if (mlirLogicalResultIsSuccess(result))
+      return printAccum.join();
+    throw nb::value_error(
+        ("Failed to export smtlib.\nDiagnostic message " + scope.takeMessage())
+            .c_str());
+  };
+
+  m.def(
+      "export_smtlib",
+      [&exportSMTLIB](MlirOperation module, bool inlineSingleUseValues,
+                      bool indentLetBody) {
+        return exportSMTLIB(module, inlineSingleUseValues, indentLetBody);
+      },
+      "module"_a, "inline_single_use_values"_a = false,
+      "indent_let_body"_a = false);
+  m.def(
+      "export_smtlib",
+      [&exportSMTLIB](MlirModule module, bool inlineSingleUseValues,
+                      bool indentLetBody) {
+        return exportSMTLIB(mlirModuleGetOperation(module),
+                            inlineSingleUseValues, indentLetBody);
+      },
+      "module"_a, "inline_single_use_values"_a = false,
+      "indent_let_body"_a = false);
+}
+
+NB_MODULE(_mlirDialectsSMT, m) {
+  m.doc() = "MLIR SMT Dialect";
+
+  populateDialectSMTSubmodule(m);
+}
diff --git a/mlir/lib/CAPI/Dialect/SMT.cpp b/mlir/lib/CAPI/Dialect/SMT.cpp
index 3a4620df8ccdf..7e96bbb071533 100644
--- a/mlir/lib/CAPI/Dialect/SMT.cpp
+++ b/mlir/lib/CAPI/Dialect/SMT.cpp
@@ -25,46 +25,49 @@ MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(SMT, smt, mlir::smt::SMTDialect)
 // Type API.
 //===----------------------------------------------------------------------===//
 
-bool smtTypeIsAnyNonFuncSMTValueType(MlirType type) {
+bool mlirSMTTypeIsAnyNonFuncSMTValueType(MlirType type) {
   return isAnyNonFuncSMTValueType(unwrap(type));
 }
 
-bool smtTypeIsAnySMTValueType(MlirType type) {
+bool mlirSMTTypeIsAnySMTValueType(MlirType type) {
   return isAnySMTValueType(unwrap(type));
 }
 
-bool smtTypeIsAArray(MlirType type) { return isa<ArrayType>(unwrap(type)); }
+bool mlirSMTTypeIsAArray(MlirType type) { return isa<ArrayType>(unwrap(type)); }
 
-MlirType smtTypeGetArray(MlirContext ctx, MlirType domainType,
-                         MlirType rangeType) {
+MlirType mlirSMTTypeGetArray(MlirContext ctx, MlirType domainType,
+                             MlirType rangeType) {
   return wrap(
       ArrayType::get(unwrap(ctx), unwrap(domainType), unwrap(rangeType)));
 }
 
-bool smtTypeIsABitVector(MlirType type) {
+bool mlirSMTTypeIsABitVector(MlirType type) {
   return isa<BitVectorType>(unwrap(type));
 }
 
-MlirType smtTypeGetBitVector(MlirContext ctx, int32_t width) {
+MlirType mlirSMTTypeGetBitVector(MlirContext ctx, int32_t width) {
   return wrap(BitVectorType::get(unwrap(ctx), width));
 }
 
-bool smtTypeIsABool(MlirType type) { return isa<BoolType>(unwrap(type)); }
+bool mlirSMTTypeIsABool(MlirType type) { return isa<BoolType>(unwrap(type)); }
 
-MlirType smtTypeGetBool(MlirContext ctx) {
+MlirType mlirSMTTypeGetBool(MlirContext ctx) {
   return wrap(BoolType::get(unwrap(ctx)));
 }
 
-bool smtTypeIsAInt(MlirType type) { return isa<IntType>(unwrap(type)); }
+bool mlirSMTTypeIsAInt(MlirType type) { return isa<IntType>(unwrap(type)); }
 
-MlirType smtTypeGetInt(MlirContext ctx) {
+MlirType mlirSMTTypeGetInt(MlirContext ctx) {
   return wrap(IntType::get(unwrap(ctx)));
 }
 
-bool smtTypeIsASMTFunc(MlirType type) { return isa<SMTFuncType>(unwrap(type)); }
+bool mlirSMTTypeIsASMTFunc(MlirType type) {
+  return isa<SMTFuncType>(unwrap(type));
+}
 
-MlirType smtTypeGetSMTFunc(MlirContext ctx, size_t numberOfDomainTypes,
-                           const MlirType *domainTypes, MlirType rangeType) {
+MlirType mlirSMTTypeGetSMTFunc(MlirContext ctx, size_t numberOfDomainTypes,
+                               const MlirType *domainTypes,
+                               MlirType rangeType) {
   SmallVector<Type> domainTypesVec;
   domainTypesVec.reserve(numberOfDomainTypes);
 
@@ -74,10 +77,11 @@ MlirType smtTypeGetSMTFunc(MlirContext ctx, size_t numberOfDomainTypes,
   return wrap(SMTFuncType::get(unwrap(ctx), domainTypesVec, unwrap(rangeType)));
 }
 
-bool smtTypeIsASort(MlirType type) { return isa<SortType>(unwrap(type)); }
+bool mlirSMTTypeIsASort(MlirType type) { return isa<SortType>(unwrap(type)); }
 
-MlirType smtTypeGetSort(MlirContext ctx, MlirIdentifier identifier,
-                        size_t numberOfSortParams, const MlirType *sortParams) {
+MlirType mlirSMTTypeGetSort(MlirContext ctx, MlirIdentifier identifier,
+                            size_t numberOfSortParams,
+                            const MlirType *sortParams) {
   SmallVector<Type> sortParamsVec;
   sortParamsVec.reserve(numberOfSortParams);
 
@@ -91,31 +95,31 @@ MlirType smtTypeGetSort(MlirContext ctx, MlirIdentifier identifier,
 // Attribute API.
 //===----------------------------------------------------------------------===//
 
-bool smtAttrCheckBVCmpPredicate(MlirContext ctx, MlirStringRef str) {
+bool mlirSMTAttrCheckBVCmpPredicate(MlirContext ctx, MlirStringRef str) {
   return symbolizeBVCmpPredicate(unwrap(str)).has_value();
 }
 
-bool smtAttrCheckIntPredicate(MlirContext ctx, MlirStringRef str) {
+bool mlirSMTAttrCheckIntPredicate(MlirContext ctx, MlirStringRef str) {
   return symbolizeIntPredicate(unwrap(str)).has_value();
 }
 
-bool smtAttrIsASMTAttribute(MlirAttribute attr) {
+bool mlirSMTAttrIsASMTAttribute(MlirAttribute attr) {
   return isa<BitVectorAttr, BVCmpPredicateAttr, IntPredicateAttr>(unwrap(attr));
 }
 
-MlirAttribute smtAttrGetBitVector(MlirContext ctx, uint64_t value,
-                                  unsigned width) {
+MlirAttribute mlirSMTAttrGetBitVector(MlirContext ctx, uint64_t value,
+                                      unsigned width) {
   return wrap(BitVectorAttr::get(unwrap(ctx), value, width));
 }
 
-MlirAttribute smtAttrGetBVCmpPredicate(MlirContext ctx, MlirStringRef str) {
+MlirAttribute mlirSMTAttrGetBVCmpPredicate(MlirContext ctx, MlirStringRef str) {
   auto predicate = symbolizeBVCmpPredicate(unwrap(str));
   assert(predicate.has_value() && "invalid predicate");
 
   return wrap(BVCmpPredicateAttr::get(unwrap(ctx), predicate.value()));
 }
 
-MlirAttribute smtAttrGetIntPredicate(MlirContext ctx, MlirStringRef str) {
+MlirAttribute mlirSMTAttrGetIntPredicate(MlirContext ctx, MlirStringRef str) {
   auto predicate = symbolizeIntPredicate(unwrap(str));
   assert(predicate.has_value() && "invalid predicate");
 
diff --git a/mlir/lib/CAPI/Target/ExportSMTLIB.cpp b/mlir/lib/CAPI/Target/ExportSMTLIB.cpp
index c9ac7ce704af8..4326f967281e1 100644
--- a/mlir/lib/CAPI/Target/ExportSMTLIB.cpp
+++ b/mlir/lib/CAPI/Target/ExportSMTLIB.cpp
@@ -19,9 +19,24 @@
 
 using namespace mlir;
 
-MlirLogicalResult mlirExportSMTLIB(MlirModule module,
-                                   MlirStringCallback callback,
-                                   void *userData) {
+MlirLogicalResult mlirTranslateOperationToSMTLIB(MlirOperation module,
+                                                 MlirStringCallback callback,
+                                                 void *userData,
+                                                 bool inlineSingleUseValues,
+                                                 bool indentLetBody) {
   mlir::detail::CallbackOstream stream(callback, userData);
+  smt::SMTEmissionOptions options;
+  options.inlineSingleUseValues = inlineSingleUseValues;
+  options.indentLetBody = indentLetBody;
   return wrap(smt::exportSMTLIB(unwrap(module), stream));
 }
+
+MlirLogicalResult mlirTranslateModuleToSMTLIB(MlirModule module,
+                                              MlirStringCallback callback,
+                                              void *userData,
+                                              bool inlineSingleUseValues,
+                                              bool indentLetBody) {
+  return mlirTranslateOperationToSMTLIB(mlirModuleGetOperation(module),
+                                        callback, userData,
+                                        inlineSingleUseValues, indentLetBody);
+}
diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt
index fb115a5f43423..bbf6819608bb9 100644
--- a/mlir/python/CMakeLists.txt
+++ b/mlir/python/CMakeLists.txt
@@ -403,6 +403,15 @@ declare_mlir_dialect_python_bindings(
     "../../include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td"
 )
 
+declare_mlir_dialect_python_bindings(
+  ADD_TO_PARENT MLIRPythonSources.Dialects
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
+  TD_FILE dialects/SMTOps.td
+  GEN_ENUM_BINDINGS
+  SOURCES
+    dialects/smt.py
+  DIALECT_NAME smt)
+
 declare_mlir_dialect_python_bindings(
     ADD_TO_PARENT MLIRPythonSources.Dialects
     ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
@@ -664,6 +673,21 @@ declare_mlir_python_extension(MLIRPythonExtension.LinalgPasses
     MLIRCAPILinalg
 )
 
+declare_mlir_python_extension(MLIRPythonExtension.Dialects.SMT.Pybind
+  MODULE_NAME _mlirDialectsSMT
+  ADD_TO_PARENT MLIRPythonSources.Dialects.smt
+  ROOT_DIR "${PYTHON_SOURCE_DIR}"
+  PYTHON_BINDINGS_LIBRARY nanobind
+  SOURCES
+    DialectSMT.cpp
+  PRIVATE_LINK_LIBS
+    LLVMSupport
+  EMBED_CAPI_LINK_LIBS
+    MLIRCAPIIR
+    MLIRCAPISMT
+    MLIRCAPIExportSMTLIB
+)
+
 declare_mlir_python_extension(MLIRPythonExtension.SparseTensorDialectPasses
   MODULE_NAME _mlirSparseTensorPasses
   ADD_TO_PARENT MLIRPythonSources.Dialects.sparse_tensor
diff --git a/mlir/python/mlir/dialects/SMTOps.td b/mlir/python/mlir/dialects/SMTOps.td
new file mode 100644
index 0000000000000..e143f071eb658
--- /dev/null
+++ b/mlir/python/mlir/dialects/SMTOps.td
@@ -0,0 +1,14 @@
+//===- SMTOps.td - Entry point for SMT bindings ------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BINDINGS_PYTHON_SMT_OPS
+#define BINDINGS_PYTHON_SMT_OPS
+
+include "mlir/Dialect/SMT/IR/SMT.td"
+
+#endif // BINDINGS_PYTHON_SMT_OPS
diff --git a/mlir/python/mlir/dialects/smt.py b/mlir/python/mlir/dialects/smt.py
new file mode 100644
index 0000000000000..ae7a4c41cbc3a
--- /dev/null
+++ b/mlir/python/mlir/dialects/smt.py
@@ -0,0 +1,33 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from ._smt_ops_gen import *
+
+from .._mlir_libs._mlirDialectsSMT import *
+from ..extras.meta import region_op
+
+
+def bool_t():
+    return BoolType.get()
+
+
+def bv_t(width):
+    return BitVectorType.get(width)
+
+
+def _solver(
+    inputs=None,
+    results=None,
+    loc=None,
+    ip=None,
+):
+    if inputs is None:
+        inputs = []
+    if results is None:
+        results = []
+
+    return SolverOp(results, inputs, loc=loc, ip=ip)
+
+
+solver = region_op(_solver, terminator=YieldOp)
diff --git a/mlir/test/CAPI/smt.c b/mlir/test/CAPI/smt.c
index 77815d4f79657..95a9b55e3209b 100644
--- a/mlir/test/CAPI/smt.c
+++ b/mlir/test/CAPI/smt.c
@@ -34,7 +34,8 @@ void testExportSMTLIB(MlirContext ctx) {
   MlirModule module =
       mlirModuleCreateParse(ctx, mlirStringRefCreateFromCString(testSMT));
 
-  MlirLogicalResult result = mlirExportSMTLIB(module, dumpCallback, NULL);
+  MlirLogicalResult result =
+      mlirTranslateModuleToSMTLIB(module, dumpCallback, NULL, false, false);
   (void)result;
   assert(mlirLogicalResultIsSuccess(result));
 
@@ -44,13 +45,13 @@ void testExportSMTLIB(MlirContext ctx) {
 }
 
 void testSMTType(MlirContext ctx) {
-  MlirType boolType = smtTypeGetBool(ctx);
-  MlirType intType = smtTypeGetInt(ctx);
-  MlirType arrayType = smtTypeGetArray(ctx, intType, boolType);
-  MlirType bvType = smtTypeGetBitVector(ctx, 32);
+  MlirType boolType = mlirSMTTypeGetBool(ctx);
+  MlirType intType = mlirSMTTypeGetInt(ctx);
+  MlirType arrayType = mlirSMTTypeGetArray(ctx, intType, boolType);
+  MlirType bvType = mlirSMTTypeGetBitVector(ctx, 32);
   MlirType funcType =
-      smtTypeGetSMTFunc(ctx, 2, (MlirType[]){intType, boolType}, boolType);
-  MlirType sortType = smtTypeGetSort(
+      mlirSMTTypeGetSMTFunc(ctx, 2, (MlirType[]){intType, boolType}, boolType);
+  MlirType sortType = mlirSMTTypeGetSort(
       ctx, mlirIdentifierGet(ctx, mlirStringRefCreateFromCString("sort")), 0,
       NULL);
 
@@ -68,107 +69,107 @@ void testSMTType(MlirContext ctx) {
   mlirTypeDump(sortType);
 
   // CHECK: bool_is_any_non_func_smt_value_type
-  fprintf(stderr, smtTypeIsAnyNonFuncSMTValueType(boolType)
+  fprintf(stderr, mlirSMTTypeIsAnyNonFuncSMTValueType(boolType)
                       ? "bool_is_any_non_func_smt_value_type\n"
                       : "bool_is_func_smt_value_type\n");
   // CHECK: int_is_any_non_func_smt_value_type
-  fprintf(stderr, smtTypeIsAnyNonFuncSMTValueType(intType)
+  fprintf(stderr, mlirSMTTypeIsAnyNonFuncSMTValueType(intType)
                       ? "int_is_any_non_func_smt_value_type\n"
                       : "int_is_func_smt_value_type\n");
   // CHECK: array_is_any_non_func_smt_value_type
-  fprintf(stderr, smtTypeIsAnyNonFuncSMTValueType(arrayType)
+  fprintf(stderr, mlirSMTTypeIsAnyNonFuncSMTValueType(arrayType)
                       ? "array_is_any_non_func_smt_value_type\n"
                       : "array_is_func_smt_value_type\n");
   // CHECK: bit_vector_is_any_non_func_smt_value_type
-  fprintf(stderr, smtTypeIsAnyNonFuncSMTValueType(bvType)
+  fprintf(stderr, mlirSMTTypeIsAnyNonFuncSMTValueType(bvType)
                       ? "bit_vector_is_any_non_func_smt_value_type\n"
                       : "bit_vector_is_func_smt_value_type\n");
   // CHECK: sort_is_any_non_func_smt_value_type
-  fprintf(stderr, smtTypeIsAnyNonFuncSMTValueType(sortType)
+  fprintf(stderr, mlirSMTTypeIsAnyNonFuncSMTValueType(sortType)
                       ? "sort_is_any_non_func_smt_value_type\n"
                       : "sort_is_func_smt_value_type\n");
   // CHECK: smt_func_is_func_smt_value_type
-  fprintf(stderr, smtTypeIsAnyNonFuncSMTValueType(funcType)
+  fprintf(stderr, mlirSMTTypeIsAnyNonFuncSMTValueType(funcType)
                       ? "smt_func_is_any_non_func_smt_value_type\n"
                       : "smt_func_is_func_smt_value_type\n");
 
   // CHECK: bool_is_any_smt_value_type
-  fprintf(stderr, smtTypeIsAnySMTValueType(boolType)
+  fprintf(stderr, mlirSMTTypeIsAnySMTValueType(boolType)
                       ? "bool_is_any_smt_value_type\n"
                       : "bool_is_not_any_smt_value_type\n");
   // CHECK: int_is_any_smt_value_type
-  fprintf(stderr, smtTypeIsAnySMTValueType(intType)
+  fprintf(stderr, mlirSMTTypeIsAnySMTValueType(intType)
                       ? "int_is_any_smt_value_type\n"
                       : "int_is_not_any_smt_value_type\n");
   // CHECK: array_is_any_smt_value_type
-  fprintf(stderr, smtTypeIsAnySMTValueType(arrayType)
+  fprintf(stderr, mlirSMTTypeIsAnySMTValueType(arrayType)
                       ? "array_is_any_smt_value_type\n"
                       : "array_is_not_any_smt_value_type\n");
   // CHECK: array_is_any_smt_value_type
-  fprintf(stderr, smtTypeIsAnySMTValueType(bvType)
+  fprintf(stderr, mlirSMTTypeIsAnySMTValueType(bvType)
                       ? "array_is_any_smt_value_type\n"
                       : "array_is_not_any_smt_value_type\n");
   // CHECK: smt_func_is_any_smt_value_type
-  fprintf(stderr, smtTypeIsAnySMTValueType(funcType)
+  fprintf(stderr, mlirSMTTypeIsAnySMTValueType(funcType)
                       ? "smt_func_is_any_smt_value_type\n"
                       : "smt_func_is_not_any_smt_value_type\n");
   // CHECK: sort_is_any_smt_value_type
-  fprintf(stderr, smtTypeIsAnySMTValueType(sortType)
+  fprintf(stderr, mlirSMTTypeIsAnySMTValueType(sortType)
                       ? "sort_is_any_smt_value_type\n"
                       : "sort_is_not_any_smt_value_type\n");
 
   // CHECK: int_type_is_not_a_bool
-  fprintf(stderr, smtTypeIsABool(intType) ? "int_type_is_a_bool\n"
-                                          : "int_type_is_not_a_bool\n");
+  fprintf(stderr, mlirSMTTypeIsABool(intType) ? "int_type_is_a_bool\n"
+                                              : "int_type_is_not_a_bool\n");
   // CHECK: bool_type_is_not_a_int
-  fprintf(stderr, smtTypeIsAInt(boolType) ? "bool_type_is_a_int\n"
-                                          : "bool_type_is_not_a_int\n");
+  fprintf(stderr, mlirSMTTypeIsAInt(boolType) ? "bool_type_is_a_int\n"
+                                              : "bool_type_is_not_a_int\n");
   // CHECK: bv_type_is_not_a_array
-  fprintf(stderr, smtTypeIsAArray(bvType) ? "bv_type_is_a_array\n"
-                                          : "bv_type_is_not_a_array\n");
+  fprintf(stderr, mlirSMTTypeIsAArray(bvType) ? "bv_type_is_a_array\n"
+                                              : "bv_type_is_not_a_array\n");
   // CHECK: array_type_is_not_a_bit_vector
-  fprintf(stderr, smtTypeIsABitVector(arrayType)
+  fprintf(stderr, mlirSMTTypeIsABitVector(arrayType)
                       ? "array_type_is_a_bit_vector\n"
                       : "array_type_is_not_a_bit_vector\n");
   // CHECK: sort_type_is_not_a_smt_func
-  fprintf(stderr, smtTypeIsASMTFunc(sortType)
+  fprintf(stderr, mlirSMTTypeIsASMTFunc(sortType)
                       ? "sort_type_is_a_smt_func\n"
                       : "sort_type_is_not_a_smt_func\n");
   // CHECK: func_type_is_not_a_sort
-  fprintf(stderr, smtTypeIsASort(funcType) ? "func_type_is_a_sort\n"
-                                           : "func_type_is_not_a_sort\n");
+  fprintf(stderr, mlirSMTTypeIsASort(funcType) ? "func_type_is_a_sort\n"
+                                               : "func_type_is_not_a_sort\n");
 }
 
 void testSMTAttribute(MlirContext ctx) {
   // CHECK: slt_is_BVCmpPredicate
-  fprintf(stderr,
-          smtAttrCheckBVCmpPredicate(ctx, mlirStringRefCreateFromCString("slt"))
-              ? "slt_is_BVCmpPredicate\n"
-              : "slt_is_not_BVCmpPredicate\n");
+  fprintf(stderr, mlirSMTAttrCheckBVCmpPredicate(
+                      ctx, mlirStringRefCreateFromCString("slt"))
+                      ? "slt_is_BVCmpPredicate\n"
+                      : "slt_is_not_BVCmpPredicate\n");
   // CHECK: lt_is_not_BVCmpPredicate
-  fprintf(stderr,
-          smtAttrCheckBVCmpPredicate(ctx, mlirStringRefCreateFromCString("lt"))
-              ? "lt_is_BVCmpPredicate\n"
-              : "lt_is_not_BVCmpPredicate\n");
+  fprintf(stderr, mlirSMTAttrCheckBVCmpPredicate(
+                      ctx, mlirStringRefCreateFromCString("lt"))
+                      ? "lt_is_BVCmpPredicate\n"
+                      : "lt_is_not_BVCmpPredicate\n");
   // CHECK: slt_is_not_IntPredicate
-  fprintf(stderr,
-          smtAttrCheckIntPredicate(ctx, mlirStringRefCreateFromCString("slt"))
-              ? "slt_is_IntPredicate\n"
-              : "slt_is_not_IntPredicate\n");
+  fprintf(stderr, mlirSMTAttrCheckIntPredicate(
+                      ctx, mlirStringRefCreateFromCString("slt"))
+                      ? "slt_is_IntPredicate\n"
+                      : "slt_is_not_IntPredicate\n");
   // CHECK: lt_is_IntPredicate
-  fprintf(stderr,
-          smtAttrCheckIntPredicate(ctx, mlirStringRefCreateFromCString("lt"))
-              ? "lt_is_IntPredicate\n"
-              : "lt_is_not_IntPredicate\n");
+  fprintf(stderr, mlirSMTAttrCheckIntPredicate(
+                      ctx, mlirStringRefCreateFromCString("lt"))
+                      ? "lt_is_IntPredicate\n"
+                      : "lt_is_not_IntPredicate\n");
 
   // CHECK: #smt.bv<5> : !smt.bv<32>
-  mlirAttributeDump(smtAttrGetBitVector(ctx, 5, 32));
+  mlirAttributeDump(mlirSMTAttrGetBitVector(ctx, 5, 32));
   // CHECK: 0 : i64
   mlirAttributeDump(
-      smtAttrGetBVCmpPredicate(ctx, mlirStringRefCreateFromCString("slt")));
+      mlirSMTAttrGetBVCmpPredicate(ctx, mlirStringRefCreateFromCString("slt")));
   // CHECK: 0 : i64
   mlirAttributeDump(
-      smtAttrGetIntPredicate(ctx, mlirStringRefCreateFromCString("lt")));
+      mlirSMTAttrGetIntPredicate(ctx, mlirStringRefCreateFromCString("lt")));
 }
 
 int main(void) {
diff --git a/mlir/test/python/dialects/smt.py b/mlir/test/python/dialects/smt.py
new file mode 100644
index 0000000000000..6f0cd8835b65b
--- /dev/null
+++ b/mlir/test/python/dialects/smt.py
@@ -0,0 +1,87 @@
+# RUN: %PYTHON %s | FileCheck %s
+
+from mlir.dialects import smt, arith
+from mlir.ir import Context, Location, Module, InsertionPoint, F32Type
+
+
+def run(f):
+    print("\nTEST:", f.__name__)
+    with Context(), Location.unknown():
+        module = Module.create()
+        with InsertionPoint(module.body):
+            f(module)
+        print(module)
+        assert module.operation.verify()
+
+
+# CHECK-LABEL: TEST: test_smoke
+@run
+def test_smoke(_module):
+    true = smt.constant(True)
+    false = smt.constant(False)
+    # CHECK: smt.constant true
+    # CHECK: smt.constant false
+
+
+# CHECK-LABEL: TEST: test_types
+@run
+def test_types(_module):
+    bool_t = smt.bool_t()
+    bitvector_t = smt.bv_t(5)
+    # CHECK: !smt.bool
+    print(bool_t)
+    # CHECK: !smt.bv<5>
+    print(bitvector_t)
+
+
+# CHECK-LABEL: TEST: test_solver_op
+@run
+def test_solver_op(_module):
+    @smt.solver
+    def foo1():
+        true = smt.constant(True)
+        false = smt.constant(False)
+
+    # CHECK: smt.solver() : () -> () {
+    # CHECK:   %true = smt.constant true
+    # CHECK:   %false = smt.constant false
+    # CHECK: }
+
+    f32 = F32Type.get()
+
+    @smt.solver(results=[f32])
+    def foo2():
+        return arith.ConstantOp(f32, 1.0)
+
+    # CHECK: %{{.*}} = smt.solver() : () -> f32 {
+    # CHECK:   %[[CST1:.*]] = arith.constant 1.000000e+00 : f32
+    # CHECK:   smt.yield %[[CST1]] : f32
+    # CHECK: }
+
+    two = arith.ConstantOp(f32, 2.0)
+    # CHECK: %[[CST2:.*]] = arith.constant 2.000000e+00 : f32
+    print(two)
+
+    @smt.solver(inputs=[two], results=[f32])
+    def foo3(z: f32):
+        return z
+
+    # CHECK: %{{.*}} = smt.solver(%[[CST2]]) : (f32) -> f32 {
+    # CHECK: ^bb0(%[[ARG0:.*]]: f32):
+    # CHECK:   smt.yield %[[ARG0]] : f32
+    # CHECK: }
+
+
+# CHECK-LABEL: TEST: test_export_smtlib
+@run
+def test_export_smtlib(module):
+    @smt.solver
+    def foo1():
+        true = smt.constant(True)
+        smt.assert_(true)
+
+    query = smt.export_smtlib(module.operation)
+    # CHECK: ; solver scope 0
+    # CHECK: (assert true)
+    # CHECK: (reset)
+    print(query)

From 9c98a9801dff79ea7e440f2a45f669be2cdc965b Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 15:17:47 -0700
Subject: [PATCH 186/710] [NFC][Asan] CRLF to LF in a test

---
 .../asan/TestCases/asan_lsan_deadlock.cpp     | 144 +++++++++---------
 1 file changed, 72 insertions(+), 72 deletions(-)

diff --git a/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp b/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
index 4e1a2415ad013..28ce4e3e81b23 100644
--- a/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
+++ b/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
@@ -1,72 +1,72 @@
-// Test for potential deadlock in LeakSanitizer+AddressSanitizer.
-// REQUIRES: leak-detection
-//
-// RUN: %clangxx_asan -O0 %s -o %t
-// RUN: %env_asan_opts=detect_leaks=1 not %run %t 2>&1 | FileCheck %s
-
-/*
- * Purpose: Verify deadlock prevention between ASan error reporting and LSan leak checking.
- * 
- * Test Design:
- * 1. Creates contention scenario between:
- *    - ASan's error reporting (requires lock B -> lock A ordering)
- *    - LSan's leak check (requires lock A -> lock B ordering)
- * 2. Thread timing:
- *    - Main thread: Holds 'in' mutex -> Triggers LSan check (lock A then B) 
- *    - Worker thread: Triggers ASan OOB error (lock B then A via symbolization)
- * 
- * Deadlock Condition (if unfixed):
- * Circular lock dependency forms when:
- * [Main Thread] LSan: lock A -> requests lock B
- * [Worker Thread] ASan: lock B -> requests lock A
- * 
- * Success Criteria: 
- * With proper lock ordering enforcement, watchdog should NOT trigger - test exits normally.
- * If deadlock occurs, watchdog terminates via _exit(1) after 10s timeout.
- */
-
-#include <mutex>
-#include <sanitizer/lsan_interface.h>
-#include <stdio.h>
-#include <thread>
-#include <unistd.h>
-
-void Watchdog() {
-  // Safety mechanism: Turn infinite deadlock into finite test failure
-  usleep(10000000);
-  // CHECK-NOT: Timeout! Deadlock detected.
-  puts("Timeout! Deadlock detected.");
-  fflush(stdout);
-  _exit(1);
-}
-
-int main(int argc, char **argv) {
-  int arr[1] = {0};
-  std::mutex in;
-  in.lock();
-
-  std::thread w(Watchdog);
-  w.detach();
-
-  std::thread t([&]() {
-    in.unlock();
-    /* 
-     * Provoke ASan error: ASan's error reporting acquires: 
-     * 1. ASan's thread registry lock (B) during the reporting 
-     * 2. dl_iterate_phdr lock (A) during symbolization
-     */
-    // CHECK: SUMMARY: AddressSanitizer: stack-buffer-overflow
-    arr[argc] = 1; // Deliberate OOB access
-  });
-
-  in.lock();
-  /* 
-   * Critical section: LSan's check acquires: 
-   * 1. dl_iterate_phdr lock (A)
-   * 2. ASan's thread registry lock (B)
-   * before Stop The World.
-   */
-  __lsan_do_leak_check();
-  t.join();
-  return 0;
-}
+// Test for potential deadlock in LeakSanitizer+AddressSanitizer.
+// REQUIRES: leak-detection
+//
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=detect_leaks=1 not %run %t 2>&1 | FileCheck %s
+
+/*
+ * Purpose: Verify deadlock prevention between ASan error reporting and LSan leak checking.
+ * 
+ * Test Design:
+ * 1. Creates contention scenario between:
+ *    - ASan's error reporting (requires lock B -> lock A ordering)
+ *    - LSan's leak check (requires lock A -> lock B ordering)
+ * 2. Thread timing:
+ *    - Main thread: Holds 'in' mutex -> Triggers LSan check (lock A then B) 
+ *    - Worker thread: Triggers ASan OOB error (lock B then A via symbolization)
+ * 
+ * Deadlock Condition (if unfixed):
+ * Circular lock dependency forms when:
+ * [Main Thread] LSan: lock A -> requests lock B
+ * [Worker Thread] ASan: lock B -> requests lock A
+ * 
+ * Success Criteria: 
+ * With proper lock ordering enforcement, watchdog should NOT trigger - test exits normally.
+ * If deadlock occurs, watchdog terminates via _exit(1) after 10s timeout.
+ */
+
+#include <mutex>
+#include <sanitizer/lsan_interface.h>
+#include <stdio.h>
+#include <thread>
+#include <unistd.h>
+
+void Watchdog() {
+  // Safety mechanism: Turn infinite deadlock into finite test failure
+  usleep(10000000);
+  // CHECK-NOT: Timeout! Deadlock detected.
+  puts("Timeout! Deadlock detected.");
+  fflush(stdout);
+  _exit(1);
+}
+
+int main(int argc, char **argv) {
+  int arr[1] = {0};
+  std::mutex in;
+  in.lock();
+
+  std::thread w(Watchdog);
+  w.detach();
+
+  std::thread t([&]() {
+    in.unlock();
+    /* 
+     * Provoke ASan error: ASan's error reporting acquires: 
+     * 1. ASan's thread registry lock (B) during the reporting 
+     * 2. dl_iterate_phdr lock (A) during symbolization
+     */
+    // CHECK: SUMMARY: AddressSanitizer: stack-buffer-overflow
+    arr[argc] = 1; // Deliberate OOB access
+  });
+
+  in.lock();
+  /* 
+   * Critical section: LSan's check acquires: 
+   * 1. dl_iterate_phdr lock (A)
+   * 2. ASan's thread registry lock (B)
+   * before Stop The World.
+   */
+  __lsan_do_leak_check();
+  t.join();
+  return 0;
+}

From 91df4cce44ac33d2d169614c532868c1dde5df51 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Wed, 16 Apr 2025 15:19:54 -0700
Subject: [PATCH 187/710] [NFC][Asan] Disabled test dead-locking on Darwin

After #131756.
---
 compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp b/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
index 28ce4e3e81b23..7ca07edbaf18c 100644
--- a/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
+++ b/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
@@ -4,6 +4,9 @@
 // RUN: %clangxx_asan -O0 %s -o %t
 // RUN: %env_asan_opts=detect_leaks=1 not %run %t 2>&1 | FileCheck %s
 
+// Hangs for unknown reasons.
+// UNSUPPORTED: darwin
+
 /*
  * Purpose: Verify deadlock prevention between ASan error reporting and LSan leak checking.
  * 

From e64305096a3d1ec122953be6e7008fbf05467d73 Mon Sep 17 00:00:00 2001
From: David Truby <david.truby@arm.com>
Date: Wed, 16 Apr 2025 23:26:20 +0100
Subject: [PATCH 188/710] [flang] Complete alignment of -x language modes with
 gfortran (#133775)

---
 clang/include/clang/Driver/Types.def             |  4 ++--
 clang/test/Driver/fortran.f95                    |  4 ++--
 .../Driver/input-from-stdin/input-from-stdin.f90 |  2 +-
 flang/test/Driver/phases.f90                     | 12 ++++++------
 flang/test/Driver/pp-fixed-form.f90              | 16 ++++++++--------
 5 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/clang/include/clang/Driver/Types.def b/clang/include/clang/Driver/Types.def
index 214c5e7a789f9..76944ec656917 100644
--- a/clang/include/clang/Driver/Types.def
+++ b/clang/include/clang/Driver/Types.def
@@ -88,8 +88,8 @@ TYPE("assembler-with-cpp",       Asm,          PP_Asm,          "S",      phases
 // modules when Flang needs to emit pre-processed files. Therefore, the
 // `PP_TYPE` is set to `PP_Fortran` so that the driver is fine with
 // "pre-processing a pre-processed file".
-TYPE("f95",                      PP_Fortran,   PP_Fortran,      "i",      phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
-TYPE("f95-cpp-input",            Fortran,      PP_Fortran,      nullptr,  phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
+TYPE("f95",                      Fortran,      PP_Fortran,      nullptr,  phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
+TYPE("f95-cpp-input",            PP_Fortran,   PP_Fortran,      "i",      phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("java",                     Java,         INVALID,         nullptr,  phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 
 // LLVM IR/LTO types. We define separate types for IR and LTO because LTO
diff --git a/clang/test/Driver/fortran.f95 b/clang/test/Driver/fortran.f95
index 275b1886b2fda..1447312b8bb58 100644
--- a/clang/test/Driver/fortran.f95
+++ b/clang/test/Driver/fortran.f95
@@ -5,14 +5,14 @@
 ! RUN:   | FileCheck --check-prefix=CHECK-OBJECT %s
 ! CHECK-OBJECT: gcc
 ! CHECK-OBJECT: "-c"
-! CHECK-OBJECT: "-x" "f95"
+! CHECK-OBJECT: "-x" "f95-cpp-input"
 ! CHECK-OBJECT-NOT: "-cc1as"
 
 ! RUN: %clang --target=x86_64-unknown-linux-gnu -integrated-as -S %s -### 2>&1 \
 ! RUN:   | FileCheck --check-prefix=CHECK-ASM %s
 ! CHECK-ASM: gcc
 ! CHECK-ASM: "-S"
-! CHECK-ASM: "-x" "f95"
+! CHECK-ASM: "-x" "f95-cpp-input"
 ! CHECK-ASM-NOT: "-cc1"
 
 ! RUN: %clang -Wall --target=x86_64-unknown-linux-gnu -integrated-as %s -### 2>&1 | FileCheck --check-prefix=CHECK-WARN %s
diff --git a/flang/test/Driver/input-from-stdin/input-from-stdin.f90 b/flang/test/Driver/input-from-stdin/input-from-stdin.f90
index 1fcc0340a64ba..285f0751b35d8 100644
--- a/flang/test/Driver/input-from-stdin/input-from-stdin.f90
+++ b/flang/test/Driver/input-from-stdin/input-from-stdin.f90
@@ -6,7 +6,7 @@
 ! Input type is implicit
 ! RUN: cat %s | %flang -E -cpp - | FileCheck %s --check-prefix=PP-NOT-DEFINED
 ! RUN: cat %s | %flang -DNEW -E -cpp - | FileCheck %s --check-prefix=PP-DEFINED
-! RUN: cat %s | %flang -DNEW -E - | FileCheck %s --check-prefix=PP-DEFINED
+! RUN: cat %s | %flang -DNEW -E - | FileCheck %s --check-prefix=PP-NOT-DEFINED
 ! RUN: cat %s | %flang -DNEW -E -nocpp - | FileCheck %s --check-prefix=PP-NOT-DEFINED
 
 ! Input type is explicit
diff --git a/flang/test/Driver/phases.f90 b/flang/test/Driver/phases.f90
index b688600dae035..9346773c883ef 100644
--- a/flang/test/Driver/phases.f90
+++ b/flang/test/Driver/phases.f90
@@ -4,15 +4,15 @@
 ! RUN: %flang -fsyntax-only -ccc-print-phases %s 2>&1 | FileCheck %s --check-prefix=COMPILE
 ! RUN: %flang -c -ccc-print-phases %s 2>&1 | FileCheck %s --check-prefix=EMIT_OBJ
 
-! PP: +- 0: input, "{{.*}}phases.f90", f95-cpp-input
-! PP-NEXT: 1: preprocessor, {0}, f95
+! PP: +- 0: input, "{{.*}}phases.f90", f95
+! PP-NEXT: 1: preprocessor, {0}, f95-cpp-input
 
-! COMPILE: +- 0: input, "{{.*}}phases.f90", f95-cpp-input
-! COMPILE-NEXT: 1: preprocessor, {0}, f95
+! COMPILE: +- 0: input, "{{.*}}phases.f90", f95
+! COMPILE-NEXT: 1: preprocessor, {0}, f95-cpp-input
 ! COMPILE-NEXT: 2: compiler, {1}, none
 
-! EMIT_OBJ: +- 0: input, "{{.*}}phases.f90", f95-cpp-input
-! EMIT_OBJ-NEXT: 1: preprocessor, {0}, f95
+! EMIT_OBJ: +- 0: input, "{{.*}}phases.f90", f95
+! EMIT_OBJ-NEXT: 1: preprocessor, {0}, f95-cpp-input
 ! EMIT_OBJ-NEXT: 2: compiler, {1}, ir
 ! EMIT_OBJ-NEXT: +- 3: backend, {2}, assembler
 ! EMIT_OBJ-NEXT: 4: assembler, {3}, object
diff --git a/flang/test/Driver/pp-fixed-form.f90 b/flang/test/Driver/pp-fixed-form.f90
index 4695da78763ae..bb869cd3341a7 100644
--- a/flang/test/Driver/pp-fixed-form.f90
+++ b/flang/test/Driver/pp-fixed-form.f90
@@ -1,19 +1,19 @@
 !RUN: %flang -save-temps -### %S/Inputs/free-form-test.f90  2>&1 | FileCheck %s --check-prefix=FREE
-FREE:       "-fc1" {{.*}} "-o" "free-form-test.i" {{.*}} "-x" "f95-cpp-input" "{{.*}}/free-form-test.f90"
-FREE-NEXT:  "-fc1" {{.*}} "-ffixed-form" {{.*}} "-x" "f95" "free-form-test.i"
+FREE:       "-fc1" {{.*}} "-o" "free-form-test.i" {{.*}} "-x" "f95" "{{.*}}/free-form-test.f90"
+FREE-NEXT:  "-fc1" {{.*}} "-ffixed-form" {{.*}} "-x" "f95-cpp-input" "free-form-test.i"
 
 !RUN: %flang -save-temps -### %S/Inputs/fixed-form-test.f  2>&1 | FileCheck %s --check-prefix=FIXED
-FIXED:      "-fc1" {{.*}} "-o" "fixed-form-test.i" {{.*}} "-x" "f95-cpp-input" "{{.*}}/fixed-form-test.f"
-FIXED-NEXT: "-fc1" {{.*}} "-ffixed-form" {{.*}} "-x" "f95" "fixed-form-test.i"
+FIXED:      "-fc1" {{.*}} "-o" "fixed-form-test.i" {{.*}} "-x" "f95" "{{.*}}/fixed-form-test.f"
+FIXED-NEXT: "-fc1" {{.*}} "-ffixed-form" {{.*}} "-x" "f95-cpp-input" "fixed-form-test.i"
 
 !RUN: %flang -save-temps -### -ffree-form %S/Inputs/free-form-test.f90  2>&1 | FileCheck %s --check-prefix=FREE-FLAG
-FREE-FLAG:           "-fc1" {{.*}} "-o" "free-form-test.i" {{.*}} "-x" "f95-cpp-input" "{{.*}}/free-form-test.f90"
+FREE-FLAG:           "-fc1" {{.*}} "-o" "free-form-test.i" {{.*}} "-x" "f95" "{{.*}}/free-form-test.f90"
 FREE-FLAG-NEXT:      "-fc1" {{.*}} "-emit-llvm-bc" "-ffree-form"
 FREE-FLAG-NOT:       "-ffixed-form"
-FREE-FLAG-SAME:      "-x" "f95" "free-form-test.i"
+FREE-FLAG-SAME:      "-x" "f95-cpp-input" "free-form-test.i"
 
 !RUN: %flang -save-temps -### -ffixed-form %S/Inputs/fixed-form-test.f  2>&1 | FileCheck %s --check-prefix=FIXED-FLAG
-FIXED-FLAG:          "-fc1" {{.*}} "-o" "fixed-form-test.i" {{.*}} "-x" "f95-cpp-input" "{{.*}}/fixed-form-test.f"
+FIXED-FLAG:          "-fc1" {{.*}} "-o" "fixed-form-test.i" {{.*}} "-x" "f95" "{{.*}}/fixed-form-test.f"
 FIXED-FLAG-NEXT:     "-fc1" {{.*}} "-emit-llvm-bc" "-ffixed-form"
 FIXED-FLAG-NOT:      "-ffixed-form"
-FIXED-FLAG-SAME:     "-x" "f95" "fixed-form-test.i"
+FIXED-FLAG-SAME:     "-x" "f95-cpp-input" "fixed-form-test.i"

From 8ebdd9d8a19543992195f197de215c53d506fb72 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Wed, 16 Apr 2025 15:40:34 -0700
Subject: [PATCH 189/710] Reapply "[LLVM][TableGen] Parameterize NumToSkip in
 DecoderEmitter" (#136017) (#136019)

This reverts commit 7fd0c8acd4659ccd0aef5486afe32c8ddf0f2957, and fixes
the assert condition in `patchNumToSkip`.
---
 llvm/lib/Target/AArch64/CMakeLists.txt    |   2 +-
 llvm/test/TableGen/VarLenDecoder.td       |   4 +-
 llvm/test/TableGen/trydecode-emission.td  |  10 +-
 llvm/test/TableGen/trydecode-emission2.td |  16 +--
 llvm/test/TableGen/trydecode-emission3.td |   2 +-
 llvm/test/TableGen/trydecode-emission4.td |   2 +-
 llvm/utils/TableGen/DecoderEmitter.cpp    | 115 ++++++++++++----------
 7 files changed, 83 insertions(+), 68 deletions(-)

diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 2300e479bc110..ba1d1605ec104 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -7,7 +7,7 @@ tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
 tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
 tablegen(LLVM AArch64GenCallingConv.inc -gen-callingconv)
 tablegen(LLVM AArch64GenDAGISel.inc -gen-dag-isel)
-tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler)
+tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler --num-to-skip-size=3)
 tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel)
 tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel)
 tablegen(LLVM AArch64GenO0PreLegalizeGICombiner.inc -gen-global-isel-combiner
diff --git a/llvm/test/TableGen/VarLenDecoder.td b/llvm/test/TableGen/VarLenDecoder.td
index 5cf0bf8911859..b77702ff7c5c1 100644
--- a/llvm/test/TableGen/VarLenDecoder.td
+++ b/llvm/test/TableGen/VarLenDecoder.td
@@ -47,9 +47,9 @@ def FOO32 : MyVarInst<MemOp32> {
 }
 
 // CHECK:      MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
-// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, 0, // Skip to: 12
+// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, // Skip to: 11
 // CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
-// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, 0, // Skip to: 21
+// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, // Skip to: 19
 // CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
 // CHECK-NEXT: MCD::OPC_Fail,
 
diff --git a/llvm/test/TableGen/trydecode-emission.td b/llvm/test/TableGen/trydecode-emission.td
index 20d2446eeac7f..2b4239f4fbe65 100644
--- a/llvm/test/TableGen/trydecode-emission.td
+++ b/llvm/test/TableGen/trydecode-emission.td
@@ -34,10 +34,10 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
-// CHECK-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
-// CHECK-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
-// CHECK-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 26 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23
+// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19
+// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 19
+// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
+// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission2.td b/llvm/test/TableGen/trydecode-emission2.td
index 0584034e41233..7d30474058f73 100644
--- a/llvm/test/TableGen/trydecode-emission2.td
+++ b/llvm/test/TableGen/trydecode-emission2.td
@@ -31,14 +31,14 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 36, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 8 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
-// CHECK-NEXT: /* 11 */      MCD::OPC_FilterValue, 0, 28, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 16 */      MCD::OPC_CheckField, 0, 2, 3, 7, 0, 0, // Skip to: 30
-// CHECK-NEXT: /* 23 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 30
-// CHECK-NEXT: /* 30 */      MCD::OPC_CheckField, 3, 2, 0, 7, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 37 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstA, skip to: 44
-// CHECK-NEXT: /* 44 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 31, 0, // Skip to: 38
+// CHECK-NEXT: /* 7 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-NEXT: /* 10 */      MCD::OPC_FilterValue, 0, 24, 0, // Skip to: 38
+// CHECK-NEXT: /* 14 */      MCD::OPC_CheckField, 0, 2, 3, 6, 0, // Skip to: 26
+// CHECK-NEXT: /* 20 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 26
+// CHECK-NEXT: /* 26 */      MCD::OPC_CheckField, 3, 2, 0, 6, 0, // Skip to: 38
+// CHECK-NEXT: /* 32 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, // Opcode: InstA, skip to: 38
+// CHECK-NEXT: /* 38 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 // CHECK: if (!Check(S, DecodeInstA(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission3.td b/llvm/test/TableGen/trydecode-emission3.td
index 4c5be7e1af229..0abbe62fe337e 100644
--- a/llvm/test/TableGen/trydecode-emission3.td
+++ b/llvm/test/TableGen/trydecode-emission3.td
@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+ // RUN: llvm-tblgen -gen-disassembler --num-to-skip-size=3 -I %p/../../include %s  | FileCheck %s
 
 include "llvm/Target/Target.td"
 
diff --git a/llvm/test/TableGen/trydecode-emission4.td b/llvm/test/TableGen/trydecode-emission4.td
index 1e51ba5e40768..413e4a0d1275a 100644
--- a/llvm/test/TableGen/trydecode-emission4.td
+++ b/llvm/test/TableGen/trydecode-emission4.td
@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler --num-to-skip-size=3 -I %p/../../include %s | FileCheck %s
 
 // Test for OPC_ExtractField/OPC_CheckField with start bit > 255.
 // These large start values may arise for architectures with long instruction
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 9c6015cc24576..eff63c6b45bb3 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -32,8 +32,10 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/LEB128.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -76,6 +78,12 @@ static cl::opt<SuppressLevel> DecoderEmitterSuppressDuplicates(
             "significantly reducing Table Duplications")),
     cl::init(SUPPRESSION_DISABLE), cl::cat(DisassemblerEmitterCat));
 
+static cl::opt<uint32_t>
+    NumToSkipSizeInBytes("num-to-skip-size",
+                         cl::desc("number of bytes to use for num-to-skip "
+                                  "entries in the decoder table (2 or 3)"),
+                         cl::init(2), cl::cat(DisassemblerEmitterCat));
+
 STATISTIC(NumEncodings, "Number of encodings considered");
 STATISTIC(NumEncodingsLackingDisasm,
           "Number of encodings without disassembler info");
@@ -130,10 +138,29 @@ struct DecoderTable : public std::vector<uint8_t> {
   // in the table for patching.
   size_t insertNumToSkip() {
     size_t Size = size();
-    insert(end(), 3, 0);
+    insert(end(), NumToSkipSizeInBytes, 0);
     return Size;
   }
+
+  void patchNumToSkip(size_t FixupIdx, uint32_t DestIdx) {
+    // Calculate the distance from the byte following the fixup entry byte
+    // to the destination. The Target is calculated from after the
+    // `NumToSkipSizeInBytes`-byte NumToSkip entry itself, so subtract
+    // `NumToSkipSizeInBytes` from the displacement here to account for that.
+    assert(DestIdx >= FixupIdx + NumToSkipSizeInBytes &&
+           "Expecting a forward jump in the decoding table");
+    uint32_t Delta = DestIdx - FixupIdx - NumToSkipSizeInBytes;
+    if (!isUIntN(8 * NumToSkipSizeInBytes, Delta))
+      PrintFatalError(
+          "disassembler decoding table too large, try --num-to-skip-size=3");
+
+    (*this)[FixupIdx] = static_cast<uint8_t>(Delta);
+    (*this)[FixupIdx + 1] = static_cast<uint8_t>(Delta >> 8);
+    if (NumToSkipSizeInBytes == 3)
+      (*this)[FixupIdx + 2] = static_cast<uint8_t>(Delta >> 16);
+  }
 };
+
 struct DecoderTableInfo {
   DecoderTable Table;
   FixupScopeList FixupStack;
@@ -690,19 +717,8 @@ static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups,
                                uint32_t DestIdx) {
   // Any NumToSkip fixups in the current scope can resolve to the
   // current location.
-  for (uint32_t FixupIdx : reverse(Fixups)) {
-    // Calculate the distance from the byte following the fixup entry byte
-    // to the destination. The Target is calculated from after the 24-bit
-    // NumToSkip entry itself, so subtract three from the displacement here
-    // to account for that.
-    uint32_t Delta = DestIdx - FixupIdx - 3;
-    // Our NumToSkip entries are 24-bits. Make sure our table isn't too
-    // big.
-    assert(isUInt<24>(Delta));
-    Table[FixupIdx] = (uint8_t)Delta;
-    Table[FixupIdx + 1] = (uint8_t)(Delta >> 8);
-    Table[FixupIdx + 2] = (uint8_t)(Delta >> 16);
-  }
+  for (uint32_t FixupIdx : Fixups)
+    Table.patchNumToSkip(FixupIdx, DestIdx);
 }
 
 // Emit table entries to decode instructions given a segment or segments
@@ -759,15 +775,9 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
     Delegate->emitTableEntries(TableInfo);
 
     // Now that we've emitted the body of the handler, update the NumToSkip
-    // of the filter itself to be able to skip forward when false. Subtract
-    // three as to account for the width of the NumToSkip field itself.
-    if (PrevFilter) {
-      uint32_t NumToSkip = Table.size() - PrevFilter - 3;
-      assert(isUInt<24>(NumToSkip) && "disassembler decoding table too large!");
-      Table[PrevFilter] = (uint8_t)NumToSkip;
-      Table[PrevFilter + 1] = (uint8_t)(NumToSkip >> 8);
-      Table[PrevFilter + 2] = (uint8_t)(NumToSkip >> 16);
-    }
+    // of the filter itself to be able to skip forward when false.
+    if (PrevFilter)
+      Table.patchNumToSkip(PrevFilter, Table.size());
   }
 
   // If there is no fallthrough, then the final filter should get fixed
@@ -814,7 +824,8 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     OS << (unsigned)*I++ << ", ";
   };
 
-  // Emit 24-bit numtoskip value to OS, returning the NumToSkip value.
+  // Emit `NumToSkipSizeInBytes`-byte numtoskip value to OS, returning the
+  // NumToSkip value.
   auto emitNumToSkip = [](DecoderTable::const_iterator &I,
                           formatted_raw_ostream &OS) {
     uint8_t Byte = *I++;
@@ -823,9 +834,11 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     Byte = *I++;
     OS << (unsigned)Byte << ", ";
     NumToSkip |= Byte << 8;
-    Byte = *I++;
-    OS << (unsigned)(Byte) << ", ";
-    NumToSkip |= Byte << 16;
+    if (NumToSkipSizeInBytes == 3) {
+      Byte = *I++;
+      OS << (unsigned)(Byte) << ", ";
+      NumToSkip |= Byte << 16;
+    }
     return NumToSkip;
   };
 
@@ -867,7 +880,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // The filter value is ULEB128 encoded.
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
+      // numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -883,7 +896,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // ULEB128 encoded field value.
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
+      // numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -893,7 +906,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       OS << Indent << "MCD::OPC_CheckPredicate, ";
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
+      // numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -925,7 +938,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
 
       // Fallthrough for OPC_TryDecode.
 
-      // 24-bit numtoskip value.
+      // numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
 
       OS << "// Opcode: " << NumberedEncodings[EncodingID]
@@ -1411,9 +1424,9 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
     TableInfo.Table.push_back(NumBits);
     TableInfo.Table.insertULEB128(Ilnd.FieldVal);
 
-    // The fixup is always 24-bits, so go ahead and allocate the space
-    // in the table so all our relative position calculations work OK even
-    // before we fully resolve the real value here.
+    // Allocate space in the table for fixup (NumToSkipSizeInBytes) so all
+    // our relative position calculations work OK even before we fully
+    // resolve the real value here.
 
     // Push location for NumToSkip backpatching.
     TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
@@ -2157,7 +2170,18 @@ insertBits(InsnType &field, uint64_t bits, unsigned startBit, unsigned numBits)
 // decodeInstruction().
 static void emitDecodeInstruction(formatted_raw_ostream &OS,
                                   bool IsVarLenInst) {
+  OS << formatv("\nconstexpr unsigned NumToSkipSizeInBytes = {};\n",
+                NumToSkipSizeInBytes);
+
   OS << R"(
+inline unsigned decodeNumToSkip(const uint8_t *&Ptr) {
+  unsigned NumToSkip = *Ptr++;
+  NumToSkip |= (*Ptr++) << 8;
+  if constexpr (NumToSkipSizeInBytes == 3)
+    NumToSkip |= (*Ptr++) << 16;
+  return NumToSkip;
+}
+
 template <typename InsnType>
 static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
                                       InsnType insn, uint64_t Address,
@@ -2195,10 +2219,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the field value.
       uint64_t Val = decodeULEB128AndIncUnsafe(++Ptr);
       bool Failed = Val != CurFieldValue;
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // Perform the filter operation.
       if (Failed)
@@ -2222,10 +2243,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       uint64_t ExpectedValue = decodeULEB128(++Ptr, &PtrLen);
       Ptr += PtrLen;
       bool Failed = ExpectedValue != FieldValue;
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // If the actual and expected values don't match, skip.
       if (Failed)
@@ -2240,10 +2258,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     case MCD::OPC_CheckPredicate: {
       // Decode the Predicate Index value.
       unsigned PIdx = decodeULEB128AndIncUnsafe(++Ptr);
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
       // Check the predicate.
       bool Failed = !checkDecoderPredicate(PIdx, Bits);
       if (Failed)
@@ -2278,10 +2293,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the Opcode value.
       unsigned Opc = decodeULEB128AndIncUnsafe(++Ptr);
       unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // Perform the decode operation.
       MCInst TmpMI;
@@ -2406,6 +2418,9 @@ handleHwModesUnrelatedEncodings(const CodeGenInstruction *Instr,
 
 // Emits disassembler code for instruction decoding.
 void DecoderEmitter::run(raw_ostream &o) {
+  if (NumToSkipSizeInBytes != 2 && NumToSkipSizeInBytes != 3)
+    PrintFatalError("Invalid value for num-to-skip-size, must be 2 or 3");
+
   formatted_raw_ostream OS(o);
   OS << R"(
 #include "llvm/MC/MCInst.h"

From d35bf17e8a0cf37959149257c0eda7b9f912390e Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Wed, 16 Apr 2025 15:45:18 -0700
Subject: [PATCH 190/710] [HLSL] Add a warning for implicit bindings (#135909)

Implicit bindings will cause very confusing crashes in the backend at
present, so this is intended at least partially as a stop gap until we
get them implemented (see #110722).

However, I do think that this is useful in the longer term as well as an
off-by-default warning, as it is quite easy to miss a binding or two
when using explicit bindings and the results of that can be surprisingly
hard to debug. I've filed #135907 to track turning this into an
off-by-default warning or removing it eventually as we see fit.
---
 clang/include/clang/Basic/DiagnosticGroups.td |  3 ++
 .../clang/Basic/DiagnosticSemaKinds.td        |  1 +
 clang/lib/Parse/ParseHLSL.cpp                 |  2 +-
 clang/lib/Sema/SemaHLSL.cpp                   | 13 ++++++
 .../test/AST/HLSL/ByteAddressBuffers-AST.hlsl | 12 +++---
 clang/test/AST/HLSL/OutArgExpr.hlsl           |  6 +--
 .../test/AST/HLSL/StructuredBuffers-AST.hlsl  | 20 ++++-----
 clang/test/AST/HLSL/TypedBuffers-AST.hlsl     | 10 ++---
 .../AST/HLSL/ast-dump-comment-cbuffer.hlsl    |  4 +-
 clang/test/AST/HLSL/cbuffer.hlsl              |  8 ++--
 .../test/AST/HLSL/cbuffer_and_namespaces.hlsl |  8 ++--
 clang/test/AST/HLSL/default_cbuffer.hlsl      |  4 +-
 ...d_resource_element_compatible_concept.hlsl |  2 +-
 ...d_resource_element_compatible_concept.hlsl |  2 +-
 clang/test/AST/HLSL/packoffset.hlsl           |  2 +-
 clang/test/AST/HLSL/pch.hlsl                  |  9 ++--
 clang/test/AST/HLSL/pch_hlsl_buffer.hlsl      | 13 +++---
 clang/test/AST/HLSL/pch_with_buf.hlsl         |  7 ++--
 clang/test/AST/HLSL/private.hlsl              |  2 +-
 .../CodeGenHLSL/GlobalConstructorLib.hlsl     |  4 +-
 .../test/CodeGenHLSL/GlobalConstructors.hlsl  |  2 +-
 .../AppendStructuredBuffer-elementtype.hlsl   |  4 +-
 .../ConsumeStructuredBuffer-elementtype.hlsl  |  4 +-
 .../builtins/RWBuffer-elementtype.hlsl        |  4 +-
 .../builtins/RWBuffer-subscript.hlsl          |  4 +-
 .../RWStructuredBuffer-elementtype.hlsl       |  2 +-
 ...erOrderedStructuredBuffer-elementtype.hlsl |  2 +-
 .../StructuredBuffer-elementtype.hlsl         |  2 +-
 .../StructuredBuffers-methods-ps.hlsl         |  4 +-
 .../StructuredBuffers-subscripts.hlsl         |  2 +-
 clang/test/CodeGenHLSL/cbuffer.hlsl           |  3 +-
 clang/test/CodeGenHLSL/cbuffer_align.hlsl     |  4 +-
 .../CodeGenHLSL/cbuffer_and_namespaces.hlsl   |  4 +-
 ...uffer_with_static_global_and_function.hlsl |  3 +-
 clang/test/CodeGenHLSL/default_cbuffer.hlsl   |  5 +--
 .../default_cbuffer_with_layout.hlsl          |  7 ++--
 .../implicit-norecurse-attrib.hlsl            |  4 +-
 clang/test/CodeGenHLSL/inline-functions.hlsl  | 12 +++---
 clang/test/CodeGenHLSL/static-local-ctor.hlsl |  3 +-
 clang/test/ParserHLSL/cb_error.hlsl           |  2 +-
 .../ParserHLSL/hlsl_is_rov_attr_error.hlsl    |  2 +-
 .../hlsl_raw_buffer_attr_error.hlsl           |  2 +-
 .../ParserHLSL/hlsl_resource_class_attr.hlsl  |  2 +-
 .../hlsl_resource_handle_attrs.hlsl           |  2 +-
 clang/test/ParserHLSL/invalid_inside_cb.hlsl  |  2 +-
 clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl   |  2 +-
 .../SemaHLSL/BuiltIns/StructuredBuffers.hlsl  |  2 +-
 .../SemaHLSL/BuiltIns/hlsl_resource_t.hlsl    |  2 +-
 .../Language/AggregateSplatCast-errors.hlsl   |  4 +-
 clang/test/SemaHLSL/cb_error.hlsl             |  2 +-
 clang/test/SemaHLSL/export.hlsl               |  2 +-
 clang/test/SemaHLSL/packoffset-invalid.hlsl   |  2 +-
 .../SemaHLSL/resource_binding_attr_error.hlsl |  4 +-
 .../resource_binding_attr_error_basic.hlsl    |  2 +-
 .../resource_binding_attr_error_resource.hlsl |  2 +-
 ...urce_binding_attr_error_silence_diags.hlsl |  4 +-
 .../resource_binding_attr_error_space.hlsl    |  4 +-
 .../resource_binding_attr_error_udt.hlsl      |  2 +-
 .../SemaHLSL/resource_binding_implicit.hlsl   | 41 +++++++++++++++++++
 59 files changed, 168 insertions(+), 126 deletions(-)
 create mode 100644 clang/test/SemaHLSL/resource_binding_implicit.hlsl

diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index d97bbfee2e4d5..b234d60fee8fc 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1612,6 +1612,9 @@ def HLSLExtension : DiagGroup<"hlsl-extensions", [HLSL202y]>;
 // Warning for mix packoffset and non-packoffset.
 def HLSLMixPackOffset : DiagGroup<"mix-packoffset">;
 
+// Warning for implicit resource bindings.
+def HLSLImplicitBinding : DiagGroup<"hlsl-implicit-binding">;
+
 // Warnings for DXIL validation
 def DXILValidation : DiagGroup<"dxil-validation">;
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 3f7499d8656bd..6cbe8b60fe9bf 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12734,6 +12734,7 @@ def warn_hlsl_deprecated_register_type_i: Warning<"binding type 'i' ignored. The
 def err_hlsl_unsupported_register_number : Error<"register number should be an integer">;
 def err_hlsl_expected_space : Error<"invalid space specifier '%0' used; expected 'space' followed by an integer, like space1">;
 def err_hlsl_space_on_global_constant : Error<"register space cannot be specified on global constants">;
+def warn_hlsl_implicit_binding : Warning<"resource has implicit register binding">, InGroup<HLSLImplicitBinding>, DefaultError;
 def warn_hlsl_packoffset_mix : Warning<"cannot mix packoffset elements with nonpackoffset elements in a cbuffer">,
     InGroup<HLSLMixPackOffset>;
 def err_hlsl_packoffset_overlap : Error<"packoffset overlap between %0, %1">;
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index f4c109f9a81a2..b832f7b6792d9 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -75,6 +75,7 @@ Decl *Parser::ParseHLSLBuffer(SourceLocation &DeclEnd) {
   Decl *D = Actions.HLSL().ActOnStartBuffer(getCurScope(), IsCBuffer, BufferLoc,
                                             Identifier, IdentifierLoc,
                                             T.getOpenLocation());
+  Actions.ProcessDeclAttributeList(Actions.CurScope, D, Attrs);
 
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
     // FIXME: support attribute on constants inside cbuffer/tbuffer.
@@ -98,7 +99,6 @@ Decl *Parser::ParseHLSLBuffer(SourceLocation &DeclEnd) {
   BufferScope.Exit();
   Actions.HLSL().ActOnFinishBuffer(D, DeclEnd);
 
-  Actions.ProcessDeclAttributeList(Actions.CurScope, D, Attrs);
   return D;
 }
 
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 0b442b75d174d..76fb81d39d67c 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -305,6 +305,10 @@ static bool isResourceRecordTypeOrArrayOf(const Type *Ty) {
   return HLSLAttributedResourceType::findHandleTypeOnResource(Ty) != nullptr;
 }
 
+static bool isResourceRecordTypeOrArrayOf(VarDecl *VD) {
+  return isResourceRecordTypeOrArrayOf(VD->getType().getTypePtr());
+}
+
 // Returns true if the type is a leaf element type that is not valid to be
 // included in HLSL Buffer, such as a resource class, empty struct, zero-sized
 // array, or a builtin intangible type. Returns false it is a valid leaf element
@@ -541,6 +545,10 @@ void SemaHLSL::ActOnFinishBuffer(Decl *Dcl, SourceLocation RBrace) {
   // create buffer layout struct
   createHostLayoutStructForBuffer(SemaRef, BufDecl);
 
+  if (std::none_of(Dcl->attr_begin(), Dcl->attr_end(),
+                   [](Attr *A) { return isa<HLSLResourceBindingAttr>(A); }))
+    SemaRef.Diag(Dcl->getLocation(), diag::warn_hlsl_implicit_binding);
+
   SemaRef.PopDeclContext();
 }
 
@@ -3248,10 +3256,12 @@ void SemaHLSL::collectResourceBindingsOnVarDecl(VarDecl *VD) {
 void SemaHLSL::processExplicitBindingsOnDecl(VarDecl *VD) {
   assert(VD->hasGlobalStorage() && "expected global variable");
 
+  bool HasBinding = false;
   for (Attr *A : VD->attrs()) {
     HLSLResourceBindingAttr *RBA = dyn_cast<HLSLResourceBindingAttr>(A);
     if (!RBA)
       continue;
+    HasBinding = true;
 
     RegisterType RT = RBA->getRegisterType();
     assert(RT != RegisterType::I && "invalid or obsolete register type should "
@@ -3278,6 +3288,9 @@ void SemaHLSL::processExplicitBindingsOnDecl(VarDecl *VD) {
           << static_cast<int>(RT);
     }
   }
+
+  if (!HasBinding && isResourceRecordTypeOrArrayOf(VD))
+    SemaRef.Diag(VD->getLocation(), diag::warn_hlsl_implicit_binding);
 }
 
 static bool CastInitializer(Sema &S, ASTContext &Ctx, Expr *E,
diff --git a/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl b/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl
index 38e5b6281c42e..3a11aff5f651a 100644
--- a/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl
@@ -1,24 +1,24 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=ByteAddressBuffer %s | FileCheck -DRESOURCE=ByteAddressBuffer \
 // RUN:  -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=ByteAddressBuffer %s | FileCheck -DRESOURCE=ByteAddressBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-SRV,CHECK-NOSUBSCRIPT %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=RWByteAddressBuffer %s | FileCheck -DRESOURCE=RWByteAddressBuffer \
 // RUN:  -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=RWByteAddressBuffer %s | FileCheck -DRESOURCE=RWByteAddressBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=RasterizerOrderedByteAddressBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedByteAddressBuffer \
 // RUN:  -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=RasterizerOrderedByteAddressBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedByteAddressBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT %s
 
diff --git a/clang/test/AST/HLSL/OutArgExpr.hlsl b/clang/test/AST/HLSL/OutArgExpr.hlsl
index b07c2efadbf4a..1cdb05418a3dd 100644
--- a/clang/test/AST/HLSL/OutArgExpr.hlsl
+++ b/clang/test/AST/HLSL/OutArgExpr.hlsl
@@ -1,7 +1,7 @@
 // RUN: rm -f %t.pch
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-pch -finclude-default-header -o %t.pch %s
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -finclude-default-header -include-pch %t.pch %s -ast-dump | FileCheck --check-prefix=AST %s
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -finclude-default-header -include-pch %t.pch %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -emit-pch -finclude-default-header -o %t.pch %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -finclude-default-header -include-pch %t.pch %s -ast-dump | FileCheck --check-prefix=AST %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -finclude-default-header -include-pch %t.pch %s -ast-print | FileCheck %s
 
 
 #ifndef TEST_HLSL
diff --git a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
index dcead068f481e..5c4716a6b6ad0 100644
--- a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
@@ -1,40 +1,40 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=StructuredBuffer %s | FileCheck -DRESOURCE=StructuredBuffer \
 // RUN:  -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=StructuredBuffer %s | FileCheck -DRESOURCE=StructuredBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-SRV,CHECK-SUBSCRIPT,CHECK-LOAD %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=RWStructuredBuffer %s | FileCheck -DRESOURCE=RWStructuredBuffer \
 // RUN:  -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=RWStructuredBuffer %s | FileCheck -DRESOURCE=RWStructuredBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-SUBSCRIPT,CHECK-COUNTER,CHECK-LOAD %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=AppendStructuredBuffer %s | FileCheck -DRESOURCE=AppendStructuredBuffer \
 // RUN:  -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=AppendStructuredBuffer %s | FileCheck -DRESOURCE=AppendStructuredBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT,CHECK-APPEND %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=ConsumeStructuredBuffer %s | FileCheck -DRESOURCE=ConsumeStructuredBuffer \
 // RUN:  -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=ConsumeStructuredBuffer %s | FileCheck -DRESOURCE=ConsumeStructuredBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-NOSUBSCRIPT,CHECK-CONSUME %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:  -DRESOURCE=RasterizerOrderedStructuredBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedStructuredBuffer \
 // RUN:  -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=RasterizerOrderedStructuredBuffer %s | FileCheck -DRESOURCE=RasterizerOrderedStructuredBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-UAV,CHECK-ROV,CHECK-SUBSCRIPT,CHECK-LOAD %s
 
diff --git a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
index f665b06d691e8..fc345b79e4aa0 100644
--- a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY \
 // RUN:   -DRESOURCE=RWBuffer %s | FileCheck -DRESOURCE=RWBuffer -check-prefix=EMPTY %s
 //
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump \
 // RUN:   -DRESOURCE=RWBuffer %s | FileCheck -DRESOURCE=RWBuffer \
 // RUN:   -check-prefixes=CHECK,CHECK-UAV %s
 
@@ -66,7 +66,7 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
 // CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME: ' lvalue .__handle {{.*}} 
+// CHECK-SAME: ' lvalue .__handle {{.*}}
 // CHECK-NEXT: CXXThisExpr {{.*}} 'const [[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}}  'Index' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
@@ -81,7 +81,7 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
 // CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME: ' lvalue .__handle {{.*}} 
+// CHECK-SAME: ' lvalue .__handle {{.*}}
 // CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}}  'Index' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
@@ -96,7 +96,7 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
 // CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME: ' lvalue .__handle {{.*}} 
+// CHECK-SAME: ' lvalue .__handle {{.*}}
 // CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}}  'Index' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
diff --git a/clang/test/AST/HLSL/ast-dump-comment-cbuffer.hlsl b/clang/test/AST/HLSL/ast-dump-comment-cbuffer.hlsl
index 37946b7cedea2..4cca9cc742c07 100644
--- a/clang/test/AST/HLSL/ast-dump-comment-cbuffer.hlsl
+++ b/clang/test/AST/HLSL/ast-dump-comment-cbuffer.hlsl
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -Wdocumentation -ast-dump=json -x hlsl -triple dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=JSON
-// RUN: %clang_cc1 -Wdocumentation -ast-dump -x hlsl -triple dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=AST
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -Wdocumentation -ast-dump=json -x hlsl -triple dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=JSON
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -Wdocumentation -ast-dump -x hlsl -triple dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=AST
 
 // JSON:"kind": "HLSLBufferDecl",
 // JSON:"name": "A",
diff --git a/clang/test/AST/HLSL/cbuffer.hlsl b/clang/test/AST/HLSL/cbuffer.hlsl
index 5c5aa6fc5ab10..726183821e57f 100644
--- a/clang/test/AST/HLSL/cbuffer.hlsl
+++ b/clang/test/AST/HLSL/cbuffer.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
 
 struct EmptyStruct {
 };
@@ -55,14 +55,14 @@ cbuffer CB {
 }
 _Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(OneFloat, __cblayout_CB), "");
 
-// Check that buffer layout struct does not include resources or empty types 
+// Check that buffer layout struct does not include resources or empty types
 // CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 2]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 cbuffer CB {
   // CHECK: VarDecl {{.*}} used a2 'hlsl_constant float'
   float a2;
   // CHECK: VarDecl {{.*}} b2 'RWBuffer<float>':'hlsl::RWBuffer<float>'
-  RWBuffer<float> b2; 
+  RWBuffer<float> b2;
   // CHECK: VarDecl {{.*}} c2 'EmptyStruct'
   EmptyStruct c2;
   // CHECK: VarDecl {{.*}} d2 'float[0]'
@@ -123,7 +123,7 @@ _Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblay
 
 // check that layout struct is created for E because because its base struct
 // is empty and should be eliminated, and BTypedef should reuse the previously
-// defined '__cblayout_B' 
+// defined '__cblayout_B'
 // CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 2]]:9 cbuffer CB
 // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 cbuffer CB {
diff --git a/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl b/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
index b2860fe07216e..3589535923241 100644
--- a/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
+++ b/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
 
 // CHECK: CXXRecordDecl {{.*}} struct EmptyStruct definition
 struct EmptyStruct {
@@ -7,11 +7,11 @@ struct EmptyStruct {
 // CHECK: NamespaceDecl {{.*}} NS1
 namespace NS1 {
   // CHECK: CXXRecordDecl {{.*}} struct Foo definition
-  struct Foo { 
+  struct Foo {
     float a;
     EmptyStruct es;
   };
-  
+
   // CHECK: CXXRecordDecl {{.*}} struct Bar definition
   struct Bar {
     // CHECK: CXXRecordDecl {{.*}} struct Foo definition
@@ -56,7 +56,7 @@ struct CB1ExpectedShape {
 _Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(CB1ExpectedShape, __cblayout_CB1), "");
 
 namespace NS2 {
-  struct Foo { 
+  struct Foo {
     float d[4];
     EmptyStruct es;
   };
diff --git a/clang/test/AST/HLSL/default_cbuffer.hlsl b/clang/test/AST/HLSL/default_cbuffer.hlsl
index 9e0fce7cc53cf..03b3ebd0d9f2b 100644
--- a/clang/test/AST/HLSL/default_cbuffer.hlsl
+++ b/clang/test/AST/HLSL/default_cbuffer.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
 
 struct EmptyStruct {
 };
@@ -14,7 +14,7 @@ struct S {
 float a;
 
 // CHECK: VarDecl {{.*}} b 'RWBuffer<float>':'hlsl::RWBuffer<float>'
-RWBuffer<float> b; 
+RWBuffer<float> b;
 
 // CHECK: VarDecl {{.*}} c 'EmptyStruct'
 EmptyStruct c;
diff --git a/clang/test/AST/HLSL/is_structured_resource_element_compatible_concept.hlsl b/clang/test/AST/HLSL/is_structured_resource_element_compatible_concept.hlsl
index a4f6e6c44794e..38c28b8e65394 100644
--- a/clang/test/AST/HLSL/is_structured_resource_element_compatible_concept.hlsl
+++ b/clang/test/AST/HLSL/is_structured_resource_element_compatible_concept.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -ast-dump-filter=__is_structured_resource_element_compatible %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -ast-dump-filter=__is_structured_resource_element_compatible %s | FileCheck %s
 
 // CHECK: ConceptDecl {{.*}} __is_structured_resource_element_compatible
 // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 element_type
diff --git a/clang/test/AST/HLSL/is_typed_resource_element_compatible_concept.hlsl b/clang/test/AST/HLSL/is_typed_resource_element_compatible_concept.hlsl
index 24a57624e2e9d..e4aefd5ac6aca 100644
--- a/clang/test/AST/HLSL/is_typed_resource_element_compatible_concept.hlsl
+++ b/clang/test/AST/HLSL/is_typed_resource_element_compatible_concept.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -ast-dump-filter=__is_typed_resource_element_compatible %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -ast-dump-filter=__is_typed_resource_element_compatible %s | FileCheck %s
 
 // CHECK: ConceptDecl {{.*}} __is_typed_resource_element_compatible
 // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 element_type
diff --git a/clang/test/AST/HLSL/packoffset.hlsl b/clang/test/AST/HLSL/packoffset.hlsl
index 1f57540aeac2f..4fe8aed5cb31a 100644
--- a/clang/test/AST/HLSL/packoffset.hlsl
+++ b/clang/test/AST/HLSL/packoffset.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-unknown-shadermodel6.3-library -S -finclude-default-header -fnative-half-type -ast-dump  -x hlsl %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-unknown-shadermodel6.3-library -S -finclude-default-header -fnative-half-type -ast-dump  -x hlsl %s | FileCheck %s
 
 
 // CHECK: HLSLBufferDecl {{.*}} cbuffer A
diff --git a/clang/test/AST/HLSL/pch.hlsl b/clang/test/AST/HLSL/pch.hlsl
index 483af0f5b4c79..353f894b8bc25 100644
--- a/clang/test/AST/HLSL/pch.hlsl
+++ b/clang/test/AST/HLSL/pch.hlsl
@@ -1,12 +1,9 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl \
-// RUN:  -finclude-default-header -emit-pch -o %t %S/Inputs/pch.hlsl
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl \
-// RUN:  -finclude-default-header -include-pch %t -ast-dump-all %s \
-// RUN: | FileCheck  %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -finclude-default-header -emit-pch -o %t %S/Inputs/pch.hlsl
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -finclude-default-header -include-pch %t -ast-dump-all %s | FileCheck  %s
 
 // Make sure PCH works by using function declared in PCH header and declare a RWBuffer in current file.
 // CHECK:FunctionDecl 0x[[FOO:[0-9a-f]+]] <{{.*}}:2:1, line:4:1> line:2:8 imported used foo 'float2 (float2, float2)'
-// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:10:1, col:23> col:23 Buffer 'hlsl::RWBuffer<float>'
+// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:{{[0-9]+}}:1, col:23> col:23 Buffer 'hlsl::RWBuffer<float>'
 hlsl::RWBuffer<float> Buffer;
 
 float2 bar(float2 a, float2 b) {
diff --git a/clang/test/AST/HLSL/pch_hlsl_buffer.hlsl b/clang/test/AST/HLSL/pch_hlsl_buffer.hlsl
index bfb89b4833677..9c73f587b7210 100644
--- a/clang/test/AST/HLSL/pch_hlsl_buffer.hlsl
+++ b/clang/test/AST/HLSL/pch_hlsl_buffer.hlsl
@@ -1,8 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl \
-// RUN:   -emit-pch -o %t %s
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl \
-// RUN:   -include-pch %t -ast-dump-all %S/Inputs/empty.hlsl \
-// RUN: | FileCheck  %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-pch -o %t %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -include-pch %t -ast-dump-all %S/Inputs/empty.hlsl | FileCheck  %s
 
 cbuffer A {
   float a;
@@ -17,19 +14,19 @@ float foo() {
 }
 
 // Make sure cbuffer/tbuffer works for PCH.
-// CHECK: HLSLBufferDecl {{.*}} line:7:9 imported <undeserialized declarations> cbuffer A
+// CHECK: HLSLBufferDecl {{.*}} line:{{[0-9]+}}:9 imported <undeserialized declarations> cbuffer A
 // CHECK-NEXT: HLSLResourceClassAttr {{.*}} Implicit CBuffer
 // CHECK-NEXT: VarDecl 0x[[A:[0-9a-f]+]] {{.*}} imported used a 'hlsl_constant float'
 // CHECK-NEXT: CXXRecordDecl {{.*}} imported implicit <undeserialized declarations> struct __cblayout_A definition
 // CHECK: FieldDecl {{.*}} imported a 'float'
 
-// CHECK: HLSLBufferDecl {{.*}} line:11:9 imported <undeserialized declarations> tbuffer B
+// CHECK: HLSLBufferDecl {{.*}} line:{{[0-9]+}}:9 imported <undeserialized declarations> tbuffer B
 // CHECK-NEXT: HLSLResourceClassAttr {{.*}} Implicit SRV
 // CHECK-NEXT: VarDecl 0x[[B:[0-9a-f]+]] {{.*}} imported used b 'hlsl_constant float'
 // CHECK-NEXT: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} imported implicit <undeserialized declarations> struct __cblayout_B definition
 // CHECK: FieldDecl 0x{{[0-9a-f]+}} {{.*}} imported b 'float'
 
-// CHECK-NEXT: FunctionDecl {{.*}} line:15:7 imported foo 'float ()'
+// CHECK-NEXT: FunctionDecl {{.*}} line:{{[0-9]+}}:7 imported foo 'float ()'
 // CHECK-NEXT: CompoundStmt {{.*}}
 // CHECK-NEXT: ReturnStmt {{.*}}
 // CHECK-NEXT: BinaryOperator {{.*}} 'float' '+'
diff --git a/clang/test/AST/HLSL/pch_with_buf.hlsl b/clang/test/AST/HLSL/pch_with_buf.hlsl
index 7fb5e2a3812ea..4a0bced6bbc33 100644
--- a/clang/test/AST/HLSL/pch_with_buf.hlsl
+++ b/clang/test/AST/HLSL/pch_with_buf.hlsl
@@ -1,13 +1,12 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -finclude-default-header -emit-pch -o %t %S/Inputs/pch_with_buf.hlsl
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl \
-// RUN:  -finclude-default-header -include-pch %t -ast-dump-all %s | FileCheck  %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -finclude-default-header -emit-pch -o %t %S/Inputs/pch_with_buf.hlsl
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -finclude-default-header -include-pch %t -ast-dump-all %s | FileCheck  %s
 
 // Make sure PCH works by using function declared in PCH header.
 // CHECK:FunctionDecl 0x[[FOO:[0-9a-f]+]] <{{.*}}:2:1, line:4:1> line:2:8 imported used foo 'float2 (float2, float2)'
 // Make sure buffer defined in PCH works.
 // CHECK:VarDecl 0x{{[0-9a-f]+}} <line:6:1, col:17> col:17 imported Buf 'RWBuffer<float>'
 // Make sure declare a RWBuffer in current file works.
-// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:11:1, col:23> col:23 Buf2 'hlsl::RWBuffer<float>'
+// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:{{[0-9]+}}:1, col:23> col:23 Buf2 'hlsl::RWBuffer<float>'
 hlsl::RWBuffer<float> Buf2;
 
 float2 bar(float2 a, float2 b) {
diff --git a/clang/test/AST/HLSL/private.hlsl b/clang/test/AST/HLSL/private.hlsl
index e00afb8f5cbd8..4fd04792eaec0 100644
--- a/clang/test/AST/HLSL/private.hlsl
+++ b/clang/test/AST/HLSL/private.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
 
 // CHECK: VarDecl {{.*}} global_scalar 'hlsl_private int' static cinit
 static int global_scalar = 0;
diff --git a/clang/test/CodeGenHLSL/GlobalConstructorLib.hlsl b/clang/test/CodeGenHLSL/GlobalConstructorLib.hlsl
index 09c44f6242c53..b798c2a6d6c4b 100644
--- a/clang/test/CodeGenHLSL/GlobalConstructorLib.hlsl
+++ b/clang/test/CodeGenHLSL/GlobalConstructorLib.hlsl
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -O0 %s -o - | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -emit-llvm -O0 %s -o - | FileCheck %s --check-prefixes=CHECK,INLINE
 
 // Make sure global variable for ctors exist for lib profile.
 // CHECK:@llvm.global_ctors
diff --git a/clang/test/CodeGenHLSL/GlobalConstructors.hlsl b/clang/test/CodeGenHLSL/GlobalConstructors.hlsl
index 7b26dba0d1901..9d0d9d32aaea0 100644
--- a/clang/test/CodeGenHLSL/GlobalConstructors.hlsl
+++ b/clang/test/CodeGenHLSL/GlobalConstructors.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s
 
 RWBuffer<float> Buffer;
 
diff --git a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
index 85face8eaeb6c..7fc01e59d3f8c 100644
--- a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
 
 struct MyStruct {
   float4 a;
@@ -11,7 +11,7 @@ struct MyStruct {
 // DXIL: %"class.hlsl::AppendStructuredBuffer.2" = type { target("dx.RawBuffer", i32, 1, 0)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.3" = type { target("dx.RawBuffer", i64, 1, 0)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.4" = type { target("dx.RawBuffer", i64, 1, 0)
-// DXIL: %"class.hlsl::AppendStructuredBuffer.5" = type { target("dx.RawBuffer", half, 1, 0) 
+// DXIL: %"class.hlsl::AppendStructuredBuffer.5" = type { target("dx.RawBuffer", half, 1, 0)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.6" = type { target("dx.RawBuffer", float, 1, 0)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.7" = type { target("dx.RawBuffer", double, 1, 0)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.8" = type { target("dx.RawBuffer", <4 x i16>, 1, 0)
diff --git a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
index 5ed9e9ad8160f..338d73e9a3890 100644
--- a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
 
 struct MyStruct {
   float4 a;
@@ -11,7 +11,7 @@ struct MyStruct {
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.2" = type { target("dx.RawBuffer", i32, 1, 0)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.3" = type { target("dx.RawBuffer", i64, 1, 0)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.4" = type { target("dx.RawBuffer", i64, 1, 0)
-// DXIL: %"class.hlsl::ConsumeStructuredBuffer.5" = type { target("dx.RawBuffer", half, 1, 0) 
+// DXIL: %"class.hlsl::ConsumeStructuredBuffer.5" = type { target("dx.RawBuffer", half, 1, 0)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.6" = type { target("dx.RawBuffer", float, 1, 0)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.7" = type { target("dx.RawBuffer", double, 1, 0)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.8" = type { target("dx.RawBuffer", <4 x i16>, 1, 0)
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
index 0944ad59d5fb5..a8b6e543d82bd 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
-// RUN: %clang_cc1 -triple spirv-pc-vulkan-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPIRV
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple spirv-pc-vulkan-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPIRV
 
 // DXIL: %"class.hlsl::RWBuffer" = type { target("dx.TypedBuffer", i16, 1, 0, 1) }
 // DXIL: %"class.hlsl::RWBuffer.0" = type { target("dx.TypedBuffer", i16, 1, 0, 0) }
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
index 2ad5b82a02912..6f296b5609c67 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-llvm -o - -O0 %s | FileCheck %s --check-prefixes=DXC,CHECK
-// RUN: %clang_cc1 -triple spirv1.6-pc-vulkan1.3-compute -emit-llvm -o - -O0 %s | FileCheck %s --check-prefixes=SPIRV,CHECK
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -emit-llvm -o - -O0 %s | FileCheck %s --check-prefixes=DXC,CHECK
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple spirv1.6-pc-vulkan1.3-compute -emit-llvm -o - -O0 %s | FileCheck %s --check-prefixes=SPIRV,CHECK
 
 RWBuffer<int> In;
 RWBuffer<int> Out;
diff --git a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
index f2f6956ce1541..2fdeca2dda6fb 100644
--- a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
 
 // CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", i16, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.0" = type { target("dx.RawBuffer", i16, 1, 0) }
diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedStructuredBuffer-elementtype.hlsl
index 68d626de689f2..e50d97b119058 100644
--- a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
 
 struct MyStruct {
   float4 a;
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
index d322cdc0d0645..0986c9b3330f0 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
 
 // CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", i16, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.0" = type { target("dx.RawBuffer", i16, 0, 0) }
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-ps.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-ps.hlsl
index 5b1d8e3052eae..f2aea4e376b03 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-ps.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-methods-ps.hlsl
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-pixel -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
-// RUN-DISABLED: %clang_cc1 -triple spirv-vulkan-pixel -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-pixel -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
+// RUN-DISABLED: %clang_cc1 -Wno-hlsl-implicit-binding -triple spirv-vulkan-pixel -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV
 
 // NOTE: SPIRV codegen for resource methods is not yet implemented
 
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
index 2af7c3ed3219f..c86aca6f342e0 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffers-subscripts.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-llvm -o - -O0 %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -emit-llvm -o - -O0 %s | FileCheck %s
 
 StructuredBuffer<int> In;
 RWStructuredBuffer<int> Out1;
diff --git a/clang/test/CodeGenHLSL/cbuffer.hlsl b/clang/test/CodeGenHLSL/cbuffer.hlsl
index 0a0465cc44e91..e13999a9982e2 100644
--- a/clang/test/CodeGenHLSL/cbuffer.hlsl
+++ b/clang/test/CodeGenHLSL/cbuffer.hlsl
@@ -1,5 +1,4 @@
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-compute \
-// RUN:   -fnative-half-type -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -finclude-default-header -triple dxil-pc-shadermodel6.3-compute -fnative-half-type -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
 
 // CHECK: %__cblayout_CBScalars = type <{ float, double, half, i64, i32, i16, i32, i64 }>
 // CHECK: %__cblayout_CBVectors = type <{ <3 x float>, <3 x double>, <2 x half>, <3 x i64>, <4 x i32>, <3 x i16>, <3 x i64> }>
diff --git a/clang/test/CodeGenHLSL/cbuffer_align.hlsl b/clang/test/CodeGenHLSL/cbuffer_align.hlsl
index 25fe20da7a230..42abe4f3421d1 100644
--- a/clang/test/CodeGenHLSL/cbuffer_align.hlsl
+++ b/clang/test/CodeGenHLSL/cbuffer_align.hlsl
@@ -1,6 +1,4 @@
-// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
-// RUN:   dxil-pc-shadermodel6.3-library %s -fnative-half-type \
-// RUN:   -fsyntax-only -verify -verify-ignore-unexpected=warning
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -std=hlsl2021 -finclude-default-header -x hlsl -triple dxil-pc-shadermodel6.3-library %s -fnative-half-type -fsyntax-only -verify -verify-ignore-unexpected=warning
 
 struct S0 {
   half a;
diff --git a/clang/test/CodeGenHLSL/cbuffer_and_namespaces.hlsl b/clang/test/CodeGenHLSL/cbuffer_and_namespaces.hlsl
index 7cbde19b67d1f..188c9831044e6 100644
--- a/clang/test/CodeGenHLSL/cbuffer_and_namespaces.hlsl
+++ b/clang/test/CodeGenHLSL/cbuffer_and_namespaces.hlsl
@@ -1,6 +1,4 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   dxil-pc-shadermodel6.3-library %s \
-// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -finclude-default-header -x hlsl -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
 
 // Make sure cbuffer inside namespace works.
 
diff --git a/clang/test/CodeGenHLSL/cbuffer_with_static_global_and_function.hlsl b/clang/test/CodeGenHLSL/cbuffer_with_static_global_and_function.hlsl
index 5eecfc41579f2..611e041f0df63 100644
--- a/clang/test/CodeGenHLSL/cbuffer_with_static_global_and_function.hlsl
+++ b/clang/test/CodeGenHLSL/cbuffer_with_static_global_and_function.hlsl
@@ -1,5 +1,4 @@
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library %s \
-// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -finclude-default-header -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
 
 // CHECK: %__cblayout_A = type <{ float }>
 
diff --git a/clang/test/CodeGenHLSL/default_cbuffer.hlsl b/clang/test/CodeGenHLSL/default_cbuffer.hlsl
index 82dc01eb09be2..bec048a80d82b 100644
--- a/clang/test/CodeGenHLSL/default_cbuffer.hlsl
+++ b/clang/test/CodeGenHLSL/default_cbuffer.hlsl
@@ -1,5 +1,4 @@
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-compute \
-// RUN:   -fnative-half-type -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -finclude-default-header -triple dxil-pc-shadermodel6.3-compute -fnative-half-type -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
 
 // CHECK: %"__cblayout_$Globals" = type <{ float, float, target("dx.Layout", %__cblayout_S, 4, 0) }>
 // CHECK: %__cblayout_S = type <{ float }>
@@ -20,7 +19,7 @@ struct S {
 };
 
 float a;
-RWBuffer<float> b; 
+RWBuffer<float> b;
 EmptyStruct c;
 float d[0];
 RWBuffer<float> e[2];
diff --git a/clang/test/CodeGenHLSL/default_cbuffer_with_layout.hlsl b/clang/test/CodeGenHLSL/default_cbuffer_with_layout.hlsl
index 1400288ba7699..7e69e1c040935 100644
--- a/clang/test/CodeGenHLSL/default_cbuffer_with_layout.hlsl
+++ b/clang/test/CodeGenHLSL/default_cbuffer_with_layout.hlsl
@@ -1,5 +1,4 @@
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-compute \
-// RUN:            -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -finclude-default-header -triple dxil-pc-shadermodel6.3-compute -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
 
 // CHECK: %"__cblayout_$Globals" = type <{ i32, float, [4 x double], <4 x i32>, <4 x float>,
 // CHECK-SAME: target("dx.Layout", %S, 8, 0) }>
@@ -16,7 +15,7 @@
 
 struct S {
   float2 v;
-}; 
+};
 
 int a;
 float b : register(c1);
@@ -34,4 +33,4 @@ void main() {
 
 // CHECK: !hlsl.cbs = !{![[CB:.*]]}
 // CHECK: ![[CB]] = !{ptr @"$Globals.cb", ptr addrspace(2) @a, ptr addrspace(2) @b, ptr addrspace(2) @c,
-// CHECK-SAME: ptr addrspace(2) @d, ptr addrspace(2) @e, ptr addrspace(2) @s}
\ No newline at end of file
+// CHECK-SAME: ptr addrspace(2) @d, ptr addrspace(2) @e, ptr addrspace(2) @s}
diff --git a/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl b/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
index a8ab6ce98ae7e..ab5001a0496de 100644
--- a/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
+++ b/clang/test/CodeGenHLSL/implicit-norecurse-attrib.hlsl
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -x hlsl -triple dxil-pc-shadermodel6.3-library  -finclude-default-header %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
-// RUN: %clang_cc1 -x hlsl -triple dxil-pc-shadermodel6.0-compute  -finclude-default-header %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -x hlsl -triple dxil-pc-shadermodel6.3-library  -finclude-default-header %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -x hlsl -triple dxil-pc-shadermodel6.0-compute  -finclude-default-header %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
 
 // Verify that a few different function types all get the NoRecurse attribute
 
diff --git a/clang/test/CodeGenHLSL/inline-functions.hlsl b/clang/test/CodeGenHLSL/inline-functions.hlsl
index 4748eeee7475f..c69fee902e305 100644
--- a/clang/test/CodeGenHLSL/inline-functions.hlsl
+++ b/clang/test/CodeGenHLSL/inline-functions.hlsl
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE,OPT_ATTR
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,OPT_ATTR
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,NOOPT_ATTR
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,OPT_ATTR
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,NOOPT_ATTR
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE,OPT_ATTR
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,OPT_ATTR
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,NOOPT_ATTR
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,OPT_ATTR
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE,NOOPT_ATTR
 
 // Tests that user functions will always be inlined.
 // This includes exported functions and mangled entry point implementation functions.
diff --git a/clang/test/CodeGenHLSL/static-local-ctor.hlsl b/clang/test/CodeGenHLSL/static-local-ctor.hlsl
index eba37e3f4c6b8..7aeb5e987d6b2 100644
--- a/clang/test/CodeGenHLSL/static-local-ctor.hlsl
+++ b/clang/test/CodeGenHLSL/static-local-ctor.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-llvm -o - -disable-llvm-passes %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -emit-llvm -o - -disable-llvm-passes %s | FileCheck %s
 
 // Verify that no per variable _Init_thread instructions are emitted for non-trivial static locals
 // These would normally be emitted by the MicrosoftCXXABI, but the DirectX backend should exlude them
@@ -34,4 +34,3 @@ void main() {
   mybuf = buf[0];
   InitBuf(mybuf);
 }
-
diff --git a/clang/test/ParserHLSL/cb_error.hlsl b/clang/test/ParserHLSL/cb_error.hlsl
index 245bf0a03fac9..f6c2dd9fd26ab 100644
--- a/clang/test/ParserHLSL/cb_error.hlsl
+++ b/clang/test/ParserHLSL/cb_error.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
 
 // expected-error@+2 {{expected identifier}}
 // expected-error@+1 {{expected unqualified-id}}
diff --git a/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
index 3b2c12e7a96c5..9920b65561527 100644
--- a/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
@@ -8,7 +8,7 @@ __hlsl_resource_t [[hlsl::is_rov]] res1;
 
 // expected-error@+1{{'is_rov' attribute takes no arguments}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov(3)]] res2;
-  
+
 // expected-error@+1{{use of undeclared identifier 'gibberish'}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov(gibberish)]] res3;
 
diff --git a/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
index 77530cbf9e4d9..a638d1d3e156c 100644
--- a/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
@@ -5,7 +5,7 @@
 
 // expected-error@+1{{'raw_buffer' attribute takes no arguments}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(3)]] res2;
-  
+
 // expected-error@+1{{use of undeclared identifier 'gibberish'}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(gibberish)]] res3;
 
diff --git a/clang/test/ParserHLSL/hlsl_resource_class_attr.hlsl b/clang/test/ParserHLSL/hlsl_resource_class_attr.hlsl
index fbada8b4b99f7..36995e28b3b85 100644
--- a/clang/test/ParserHLSL/hlsl_resource_class_attr.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_class_attr.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -ast-dump -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -x hlsl -ast-dump -o - %s | FileCheck %s
 
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} struct MyBuffer definition
 // CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:3, col:51> col:51 h '__hlsl_resource_t
diff --git a/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl b/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
index b5737f5dac8a9..f2cc8918c0ff1 100644
--- a/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -ast-dump -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -ast-dump -o - %s | FileCheck %s
 
 // CHECK: ClassTemplateSpecializationDecl {{.*}} class RWBuffer definition implicit_instantiation
 // CHECK: TemplateArgument type 'float'
diff --git a/clang/test/ParserHLSL/invalid_inside_cb.hlsl b/clang/test/ParserHLSL/invalid_inside_cb.hlsl
index b74021fd22422..926f53e58bc7f 100644
--- a/clang/test/ParserHLSL/invalid_inside_cb.hlsl
+++ b/clang/test/ParserHLSL/invalid_inside_cb.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -std=hlsl202x -o - -fsyntax-only %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -std=hlsl202x -o - -fsyntax-only %s -verify
 
 // template not allowed inside cbuffer.
 cbuffer A {
diff --git a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
index 34930d8963688..91e96b995585f 100644
--- a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -fsyntax-only -verify %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -x hlsl -fsyntax-only -verify %s
 
 typedef vector<float, 3> float3;
 typedef vector<double, 2> double2;
diff --git a/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl b/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
index fb14429025d5a..991b04c80ac86 100644
--- a/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -fsyntax-only -verify %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.0-compute -x hlsl -fsyntax-only -verify %s
 
 typedef vector<float, 3> float3;
 
diff --git a/clang/test/SemaHLSL/BuiltIns/hlsl_resource_t.hlsl b/clang/test/SemaHLSL/BuiltIns/hlsl_resource_t.hlsl
index a17ec327ba9e7..83fad4b8deb7c 100644
--- a/clang/test/SemaHLSL/BuiltIns/hlsl_resource_t.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/hlsl_resource_t.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -triple dxil-unknown-shadermodel6.3-library %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -fsyntax-only -verify -triple dxil-unknown-shadermodel6.3-library %s
 
 // Note: As HLSL resource type are sizeless type, we don't exhaustively
 // test for cases covered by sizeless-1.c and similar tests.
diff --git a/clang/test/SemaHLSL/Language/AggregateSplatCast-errors.hlsl b/clang/test/SemaHLSL/Language/AggregateSplatCast-errors.hlsl
index 662dae27e8200..7dc3c3e1c3e6c 100644
--- a/clang/test/SemaHLSL/Language/AggregateSplatCast-errors.hlsl
+++ b/clang/test/SemaHLSL/Language/AggregateSplatCast-errors.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -verify -verify-ignore-unexpected=note
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -verify -verify-ignore-unexpected=note
 
 struct S {
   int A : 8;
@@ -46,4 +46,4 @@ struct X {
 export void cantCast5() {
   X x = (X)1;
   // expected-error@-1 {{no matching conversion for C-style cast from 'int' to 'X'}}
-}
\ No newline at end of file
+}
diff --git a/clang/test/SemaHLSL/cb_error.hlsl b/clang/test/SemaHLSL/cb_error.hlsl
index 95c917a9bb9ee..9640ab252b942 100644
--- a/clang/test/SemaHLSL/cb_error.hlsl
+++ b/clang/test/SemaHLSL/cb_error.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
 
 // expected-note@+1 {{declared here}}
 cbuffer a {
diff --git a/clang/test/SemaHLSL/export.hlsl b/clang/test/SemaHLSL/export.hlsl
index 2d19fa561fa0a..bda68db09287c 100644
--- a/clang/test/SemaHLSL/export.hlsl
+++ b/clang/test/SemaHLSL/export.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - %s -verify
 
 export void f1();
 
diff --git a/clang/test/SemaHLSL/packoffset-invalid.hlsl b/clang/test/SemaHLSL/packoffset-invalid.hlsl
index 526a511edf1f2..0b06d55ccd688 100644
--- a/clang/test/SemaHLSL/packoffset-invalid.hlsl
+++ b/clang/test/SemaHLSL/packoffset-invalid.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -fnative-half-type -verify %s
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -finclude-default-header -triple dxil-pc-shadermodel6.3-library -fnative-half-type -verify %s
 
 // expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
 cbuffer Mix
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error.hlsl
index 74aff79f0e37f..a3a91c3ddddb8 100644
--- a/clang/test/SemaHLSL/resource_binding_attr_error.hlsl
+++ b/clang/test/SemaHLSL/resource_binding_attr_error.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
 
 template<typename T>
 struct MyTemplatedSRV {
@@ -58,7 +58,7 @@ void bar(MyTemplatedSRV<float> U : register(u3)) {
 
 }
 
-struct S {  
+struct S {
   // expected-error@+1 {{'register' attribute only applies to cbuffer/tbuffer and external global variables}}
   MyTemplatedSRV<float> U : register(u3);
 };
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error_basic.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error_basic.hlsl
index 760c057630a7f..5d4059f9a9cf4 100644
--- a/clang/test/SemaHLSL/resource_binding_attr_error_basic.hlsl
+++ b/clang/test/SemaHLSL/resource_binding_attr_error_basic.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
 
 // expected-error@+1{{binding type 't' only applies to SRV resources}}
 float f1 : register(t0);
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error_resource.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error_resource.hlsl
index 4b6af47c0ab72..a003a11765105 100644
--- a/clang/test/SemaHLSL/resource_binding_attr_error_resource.hlsl
+++ b/clang/test/SemaHLSL/resource_binding_attr_error_resource.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
 
 // This test validates the diagnostics that are emitted when a variable with a "resource" type
 // is bound to a register using the register annotation
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error_silence_diags.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error_silence_diags.hlsl
index e63f264452da7..70b53524d3faa 100644
--- a/clang/test/SemaHLSL/resource_binding_attr_error_silence_diags.hlsl
+++ b/clang/test/SemaHLSL/resource_binding_attr_error_silence_diags.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only -Wno-legacy-constant-register-binding %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only -Wno-legacy-constant-register-binding %s -verify
 
 // expected-no-diagnostics
 float f2 : register(b9);
@@ -11,7 +11,7 @@ cbuffer g_cbuffer1 {
 
 
 struct Eg12{
-  RWBuffer<int> a;  
+  RWBuffer<int> a;
 };
 
 Eg12 e12 : register(c9);
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error_space.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error_space.hlsl
index 70e64e6ca7528..e66b1114ac320 100644
--- a/clang/test/SemaHLSL/resource_binding_attr_error_space.hlsl
+++ b/clang/test/SemaHLSL/resource_binding_attr_error_space.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
 
 // valid
 cbuffer cbuf {
@@ -30,7 +30,7 @@ cbuffer cbuf4 {
   // this test validates that no diagnostic is emitted on the space parameter, because
   // this register annotation is not in the global scope.
   // expected-error@+1 {{binding type 'u' only applies to UAV resources}}
-  float a : register(u2, space3); 
+  float a : register(u2, space3);
 }
 
 // expected-error@+1 {{invalid space specifier 's2' used; expected 'space' followed by an integer, like space1}}
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error_udt.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error_udt.hlsl
index 40517f393e128..3472ea762487b 100644
--- a/clang/test/SemaHLSL/resource_binding_attr_error_udt.hlsl
+++ b/clang/test/SemaHLSL/resource_binding_attr_error_udt.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+// RUN: %clang_cc1 -Wno-hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
 
 template<typename T>
 struct MyTemplatedUAV {
diff --git a/clang/test/SemaHLSL/resource_binding_implicit.hlsl b/clang/test/SemaHLSL/resource_binding_implicit.hlsl
new file mode 100644
index 0000000000000..8f0e721c7153f
--- /dev/null
+++ b/clang/test/SemaHLSL/resource_binding_implicit.hlsl
@@ -0,0 +1,41 @@
+// RUN: %clang_cc1 -Wno-error=hlsl-implicit-binding -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+
+// expected-warning@+1 {{resource has implicit register binding}}
+cbuffer cb0 {
+  int a;
+}
+
+// No warning - this is an element of the $Globals buffer not it's own binding.
+float b;
+
+// expected-warning@+1 {{resource has implicit register binding}}
+RWBuffer<int> c;
+
+// No warning - explicit binding.
+RWBuffer<float> d : register(u0);
+
+// TODO: Add this test once #135287 lands
+// TODO: ... @+1 {{resource has implicit register binding}}
+// TODO: RWBuffer<float> dd : register(space1);
+
+// No warning - explicit binding.
+RWBuffer<float> ddd : register(u3, space4);
+
+struct S { int x; };
+// expected-warning@+1 {{resource has implicit register binding}}
+StructuredBuffer<S> e;
+
+// No warning - __hlsl_resource_t isn't itself a resource object.
+__hlsl_resource_t [[hlsl::resource_class(SRV)]] f;
+
+struct CustomSRV {
+  __hlsl_resource_t [[hlsl::resource_class(SRV)]] x;
+};
+// expected-warning@+1 {{resource has implicit register binding}}
+CustomSRV g;
+
+// expected-warning@+1 {{resource has implicit register binding}}
+RWBuffer<float> h[10];
+
+// No warning - explicit binding.
+RWBuffer<float> hh[100] : register(u4);

From b9f1de04f65b062559d01c83dfd3948601924ee1 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 16 Apr 2025 15:47:16 -0700
Subject: [PATCH 191/710] [TableGen] Combine the two separate OperandMapping
 loops in PseudoLoweringEmitter. (#136007)

Previously we had one loop over the DAG for immediates and registers and
another loop over the destination operands for mapping from the source.

Now we have a single loop over the destination operands that handles immediates,
registers, and named operands. A helper method is added so we can handle
operands and sub-operands specified by a sub-dag.

My goal is to allow a named operand to appear in a sub-dag which wasn't
supported before. This will allow the destination instruction to have an
operand with sub-operands when the source does not have sub operands.

For RISC-V, I'm looking into using an operand with sub-operands to
represent an reg+offset memory address. I need to be able to lower a
pseudo instruction that only has a register operand to an instruction
that has a reg+offset operand. The offset will be filled in with 0
during expansion and the register will be copied from the source.

The expansion would look like this:
def PseudoCALLIndirect : Pseudo<(outs), (ins GPRJALR:$rs1),
                                [(riscv_call GPRJALR:$rs1)]>,
PseudoInstExpansion<(JALR X1, (ops GPR:$rs1, 0))>;
---
 llvm/utils/TableGen/PseudoLoweringEmitter.cpp | 194 +++++++++---------
 1 file changed, 98 insertions(+), 96 deletions(-)

diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index 96325eac95004..44a17a3906fe6 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -51,10 +51,11 @@ class PseudoLoweringEmitter {
 
   SmallVector<PseudoExpansion, 64> Expansions;
 
-  unsigned addDagOperandMapping(const Record *Rec, const DagInit *Dag,
-                                const CodeGenInstruction &Insn,
-                                IndexedMap<OpData> &OperandMap,
-                                unsigned BaseIdx);
+  void addOperandMapping(unsigned MIOpNo, unsigned NumOps, const Record *Rec,
+                         const DagInit *Dag, unsigned DagIdx,
+                         const Record *OpRec, IndexedMap<OpData> &OperandMap,
+                         const StringMap<unsigned> &SourceOperands,
+                         const CodeGenInstruction &SourceInsn);
   void evaluateExpansion(const Record *Pseudo);
   void emitLoweringEmitter(raw_ostream &o);
 
@@ -66,64 +67,67 @@ class PseudoLoweringEmitter {
 };
 } // End anonymous namespace
 
-// FIXME: This pass currently can only expand a pseudo to a single instruction.
-//        The pseudo expansion really should take a list of dags, not just
-//        a single dag, so we can do fancier things.
-unsigned PseudoLoweringEmitter::addDagOperandMapping(
-    const Record *Rec, const DagInit *Dag, const CodeGenInstruction &Insn,
-    IndexedMap<OpData> &OperandMap, unsigned BaseIdx) {
-  unsigned OpsAdded = 0;
-  for (unsigned i = 0, e = Dag->getNumArgs(); i != e; ++i) {
-    if (const DefInit *DI = dyn_cast<DefInit>(Dag->getArg(i))) {
-      // Physical register reference. Explicit check for the special case
-      // "zero_reg" definition.
-      if (DI->getDef()->isSubClassOf("Register") ||
-          DI->getDef()->getName() == "zero_reg") {
-        auto &Entry = OperandMap[BaseIdx + i];
-        Entry.Kind = OpData::Reg;
-        Entry.Data.Reg = DI->getDef();
-        ++OpsAdded;
-        continue;
-      }
+void PseudoLoweringEmitter::addOperandMapping(
+    unsigned MIOpNo, unsigned NumOps, const Record *Rec, const DagInit *Dag,
+    unsigned DagIdx, const Record *OpRec, IndexedMap<OpData> &OperandMap,
+    const StringMap<unsigned> &SourceOperands,
+    const CodeGenInstruction &SourceInsn) {
+  const Init *DagArg = Dag->getArg(DagIdx);
+  if (const DefInit *DI = dyn_cast<DefInit>(DagArg)) {
+    // Physical register reference. Explicit check for the special case
+    // "zero_reg" definition.
+    if (DI->getDef()->isSubClassOf("Register") ||
+        DI->getDef()->getName() == "zero_reg") {
+      auto &Entry = OperandMap[MIOpNo];
+      Entry.Kind = OpData::Reg;
+      Entry.Data.Reg = DI->getDef();
+      return;
+    }
 
-      // Normal operands should always have the same type, or we have a
-      // problem.
-      // FIXME: We probably shouldn't ever get a non-zero BaseIdx here.
-      assert(BaseIdx == 0 && "Named subargument in pseudo expansion?!");
-      if (DI->getDef() != Insn.Operands[BaseIdx + i].Rec)
-        PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
-                                 "', operand type '" + DI->getDef()->getName() +
-                                 "' does not match expansion operand type '" +
-                                 Insn.Operands[BaseIdx + i].Rec->getName() +
-                                 "'");
-      // Source operand maps to destination operand. The Data element
-      // will be filled in later, just set the Kind for now. Do it
-      // for each corresponding MachineInstr operand, not just the first.
-      for (unsigned I = 0, E = Insn.Operands[i].MINumOperands; I != E; ++I)
-        OperandMap[BaseIdx + i + I].Kind = OpData::Operand;
-      OpsAdded += Insn.Operands[i].MINumOperands;
-    } else if (const IntInit *II = dyn_cast<IntInit>(Dag->getArg(i))) {
-      auto &Entry = OperandMap[BaseIdx + i];
-      Entry.Kind = OpData::Imm;
-      Entry.Data.Imm = II->getValue();
-      ++OpsAdded;
-    } else if (const auto *BI = dyn_cast<BitsInit>(Dag->getArg(i))) {
-      auto &Entry = OperandMap[BaseIdx + i];
-      Entry.Kind = OpData::Imm;
-      Entry.Data.Imm = *BI->convertInitializerToInt();
-      ++OpsAdded;
-    } else if (const DagInit *SubDag = dyn_cast<DagInit>(Dag->getArg(i))) {
-      // Just add the operands recursively. This is almost certainly
-      // a constant value for a complex operand (> 1 MI operand).
-      unsigned NewOps =
-          addDagOperandMapping(Rec, SubDag, Insn, OperandMap, BaseIdx + i);
-      OpsAdded += NewOps;
-      // Since we added more than one, we also need to adjust the base.
-      BaseIdx += NewOps - 1;
-    } else
-      llvm_unreachable("Unhandled pseudo-expansion argument type!");
-  }
-  return OpsAdded;
+    if (DI->getDef() != OpRec)
+      PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                               "', operand type '" + DI->getDef()->getName() +
+                               "' does not match expansion operand type '" +
+                               OpRec->getName() + "'");
+
+    StringMap<unsigned>::const_iterator SourceOp =
+        SourceOperands.find(Dag->getArgNameStr(DagIdx));
+    if (SourceOp == SourceOperands.end())
+      PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                               "', output operand '" +
+                               Dag->getArgNameStr(DagIdx) +
+                               "' has no matching source operand");
+    const auto &SrcOpnd = SourceInsn.Operands[SourceOp->getValue()];
+    if (NumOps != SrcOpnd.MINumOperands)
+      PrintFatalError(
+          Rec,
+          "In pseudo instruction '" + Rec->getName() + "', output operand '" +
+              OpRec->getName() +
+              "' has a different number of sub operands than source operand '" +
+              SrcOpnd.Rec->getName() + "'");
+
+    // Source operand maps to destination operand. Do it for each corresponding
+    // MachineInstr operand, not just the first.
+    for (unsigned I = 0, E = NumOps; I != E; ++I) {
+      auto &Entry = OperandMap[MIOpNo + I];
+      Entry.Kind = OpData::Operand;
+      Entry.Data.Operand = SrcOpnd.MIOperandNo + I;
+    }
+
+    LLVM_DEBUG(dbgs() << "    " << SourceOp->getValue() << " ==> " << DagIdx
+                      << "\n");
+  } else if (const auto *II = dyn_cast<IntInit>(DagArg)) {
+    assert(NumOps == 1);
+    auto &Entry = OperandMap[MIOpNo];
+    Entry.Kind = OpData::Imm;
+    Entry.Data.Imm = II->getValue();
+  } else if (const auto *BI = dyn_cast<BitsInit>(DagArg)) {
+    assert(NumOps == 1);
+    auto &Entry = OperandMap[MIOpNo];
+    Entry.Kind = OpData::Imm;
+    Entry.Data.Imm = *BI->convertInitializerToInt();
+  } else
+    llvm_unreachable("Unhandled pseudo-expansion argument type!");
 }
 
 void PseudoLoweringEmitter::evaluateExpansion(const Record *Rec) {
@@ -157,14 +161,6 @@ void PseudoLoweringEmitter::evaluateExpansion(const Record *Rec) {
                              "', result operator '" + Operator->getName() +
                              "' has the wrong number of operands");
 
-  unsigned NumMIOperands = 0;
-  for (const auto &Op : Insn.Operands)
-    NumMIOperands += Op.MINumOperands;
-  IndexedMap<OpData> OperandMap;
-  OperandMap.grow(NumMIOperands);
-
-  addDagOperandMapping(Rec, Dag, Insn, OperandMap, 0);
-
   // If there are more operands that weren't in the DAG, they have to
   // be operands that have default values, or we have an error. Currently,
   // Operands that are a subclass of OperandWithDefaultOp have default values.
@@ -180,37 +176,43 @@ void PseudoLoweringEmitter::evaluateExpansion(const Record *Rec) {
   for (const auto &[Idx, SrcOp] : enumerate(SourceInsn.Operands))
     SourceOperands[SrcOp.Name] = Idx;
 
-  LLVM_DEBUG(dbgs() << "  Operand mapping:\n");
-  for (const auto &[Idx, Opnd] : enumerate(Insn.Operands)) {
-    // We've already handled constant values. Just map instruction operands
-    // here.
-    if (OperandMap[Opnd.MIOperandNo].Kind != OpData::Operand)
-      continue;
-    StringMap<unsigned>::iterator SourceOp =
-        SourceOperands.find(Dag->getArgNameStr(Idx));
-    if (SourceOp == SourceOperands.end())
-      PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
-                               "', output operand '" + Dag->getArgNameStr(Idx) +
-                               "' has no matching source operand");
-    const auto &SrcOpnd = SourceInsn.Operands[SourceOp->getValue()];
-    if (Opnd.MINumOperands != SrcOpnd.MINumOperands)
-      PrintFatalError(
-          Rec,
-          "In pseudo instruction '" + Rec->getName() + "', output operand '" +
-              Opnd.Rec->getName() +
-              "' has a different number of sub operands than source operand '" +
-              SrcOpnd.Rec->getName() + "'");
+  unsigned NumMIOperands = 0;
+  for (const auto &Op : Insn.Operands)
+    NumMIOperands += Op.MINumOperands;
+  IndexedMap<OpData> OperandMap;
+  OperandMap.grow(NumMIOperands);
 
-    // Map the source operand to the destination operand index for each
-    // MachineInstr operand.
-    for (unsigned I = 0, E = Opnd.MINumOperands; I != E; ++I)
-      OperandMap[Opnd.MIOperandNo + I].Data.Operand = SrcOpnd.MIOperandNo + I;
+  // FIXME: This pass currently can only expand a pseudo to a single
+  // instruction. The pseudo expansion really should take a list of dags, not
+  // just a single dag, so we can do fancier things.
+  LLVM_DEBUG(dbgs() << "  Operand mapping:\n");
+  for (const auto &[Idx, DstOp] : enumerate(Insn.Operands)) {
+    unsigned MIOpNo = DstOp.MIOperandNo;
 
-    LLVM_DEBUG(dbgs() << "    " << SourceOp->getValue() << " ==> " << Idx
-                      << "\n");
+    if (const auto *SubDag = dyn_cast<DagInit>(Dag->getArg(Idx))) {
+      if (!DstOp.MIOperandInfo || DstOp.MIOperandInfo->getNumArgs() == 0)
+        PrintFatalError(Rec, "In pseudo instruction '" + Rec->getName() +
+                                 "', operand '" + DstOp.Rec->getName() +
+                                 "' does not have suboperands");
+      if (DstOp.MINumOperands != SubDag->getNumArgs()) {
+        PrintFatalError(
+            Rec, "In pseudo instruction '" + Rec->getName() + "', '" +
+                     SubDag->getAsString() +
+                     "' has wrong number of operands for operand type '" +
+                     DstOp.Rec->getName() + "'");
+      }
+      for (unsigned I = 0, E = DstOp.MINumOperands; I != E; ++I) {
+        auto *OpndRec = cast<DefInit>(DstOp.MIOperandInfo->getArg(I))->getDef();
+        addOperandMapping(MIOpNo + I, 1, Rec, SubDag, I, OpndRec, OperandMap,
+                          SourceOperands, SourceInsn);
+      }
+    } else {
+      addOperandMapping(MIOpNo, DstOp.MINumOperands, Rec, Dag, Idx, DstOp.Rec,
+                        OperandMap, SourceOperands, SourceInsn);
+    }
   }
 
-  Expansions.push_back(PseudoExpansion(SourceInsn, Insn, OperandMap));
+  Expansions.emplace_back(SourceInsn, Insn, OperandMap);
 }
 
 void PseudoLoweringEmitter::emitLoweringEmitter(raw_ostream &o) {

From f7bdf30cb9b8a10847c29b9cad34e5e8abc1ecb8 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Wed, 16 Apr 2025 15:48:30 -0700
Subject: [PATCH 192/710] Add empty top level .clang-format-ignore (#136022)

Otherwise if the source tree is embedded in another project with a
.clang-format-ignore, some clang-format tests fail because they use that
.clang-format-ignore.
---
 .clang-format-ignore | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 .clang-format-ignore

diff --git a/.clang-format-ignore b/.clang-format-ignore
new file mode 100644
index 0000000000000..e69de29bb2d1d

From f8ea2ed59820a0bef3f23638ce7a5d10165f7109 Mon Sep 17 00:00:00 2001
From: Jorge Gorbe Moya <jgorbe@google.com>
Date: Wed, 16 Apr 2025 16:18:34 -0700
Subject: [PATCH 193/710] [bazel] Fix build for
 8ebdd9d8a19543992195f197de215c53d506fb72

---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index b77ddf634eec6..143304c24c241 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -2148,7 +2148,10 @@ llvm_target_lib_list = [lib for lib in [
                 "lib/Target/AArch64/AArch64GenSubtargetInfo.inc",
             ),
             (
-                ["-gen-disassembler"],
+                [
+                    "-gen-disassembler",
+                    "--num-to-skip-size=3",
+                ],
                 "lib/Target/AArch64/AArch64GenDisassemblerTables.inc",
             ),
             (

From f3bf844d2ff0a2984ca9bf976014decc0241d2b4 Mon Sep 17 00:00:00 2001
From: Asher Mancinelli <ashermancinelli@gmail.com>
Date: Wed, 16 Apr 2025 17:15:15 -0700
Subject: [PATCH 194/710] [flang] Unwrap sequence types when checking for
 descriptor members (#136039)

The TBAA generation gives conservative TBAA metadata when handling an
access of a record type with a descriptor member, since the access may
be a regular data access OR another descriptor. Array members were being
incorrectly identified as non-descriptor-members, and were giving
incorrect TBAA metadata which led to bugs showing up in the optimizer
when LLVM encountered mismatching TBAA.

`fir::isRecordWithDescriptorMember` now unwraps sequence types before
checking for descriptor members.
---
 flang/lib/Optimizer/Dialect/FIRType.cpp |  1 +
 flang/test/Fir/tbaa-codegen-records.fir | 30 +++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 flang/test/Fir/tbaa-codegen-records.fir

diff --git a/flang/lib/Optimizer/Dialect/FIRType.cpp b/flang/lib/Optimizer/Dialect/FIRType.cpp
index 4d40c2618a0d0..b76856d72a017 100644
--- a/flang/lib/Optimizer/Dialect/FIRType.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRType.cpp
@@ -435,6 +435,7 @@ bool isRecordWithDescriptorMember(mlir::Type ty) {
   ty = unwrapSequenceType(ty);
   if (auto recTy = mlir::dyn_cast<fir::RecordType>(ty))
     for (auto [field, memTy] : recTy.getTypeList()) {
+      memTy = unwrapSequenceType(memTy);
       if (mlir::isa<fir::BaseBoxType>(memTy))
         return true;
       if (mlir::isa<fir::RecordType>(memTy) &&
diff --git a/flang/test/Fir/tbaa-codegen-records.fir b/flang/test/Fir/tbaa-codegen-records.fir
new file mode 100644
index 0000000000000..336354098f0f8
--- /dev/null
+++ b/flang/test/Fir/tbaa-codegen-records.fir
@@ -0,0 +1,30 @@
+// RUN: fir-opt --split-input-file --pass-pipeline="builtin.module(fir-to-llvm-ir{apply-tbaa=true})" %s | FileCheck %s
+
+// Ensure that records with array members are identified as having descriptor members,
+// as reflected by the TBAA metadata.
+
+func.func @record_array_member(%arg0 : !fir.ref<!fir.type<_QFTt2{y:!fir.array<1x!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>}>>) {
+  %c0_i64 = arith.constant 0 : i64
+  %c1 = arith.constant 1 : index
+  %1 = fir.alloca !fir.type<_QFTt2{y:!fir.array<1x!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>}> {bindc_name = "z", uniq_name = "_QFB1Ez"}
+  fir.copy %arg0 to %1 no_overlap : !fir.ref<!fir.type<_QFTt2{y:!fir.array<1x!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>}>>, !fir.ref<!fir.type<_QFTt2{y:!fir.array<1x!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>}>>
+  %3 = fir.coordinate_of %1, y : (!fir.ref<!fir.type<_QFTt2{y:!fir.array<1x!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>}>>) -> !fir.ref<!fir.array<1x!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>
+  %4 = fircg.ext_array_coor %3(%c1)<%c1> : (!fir.ref<!fir.array<1x!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>, index, index) -> !fir.ref<!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+  %5 = fir.coordinate_of %4, x : (!fir.ref<!fir.type<_QFTt{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %6 = fir.load %5 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  return
+}
+
+// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root record_array_member">
+// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#tbaa_root, 0>}>
+// CHECK: #[[$ATTR_2:.+]] = #llvm.tbaa_tag<base_type = #tbaa_type_desc, access_type = #tbaa_type_desc, offset = 0>
+// CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "descriptor member", members = {<#tbaa_type_desc, 0>}>
+// CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_tag<base_type = #tbaa_type_desc1, access_type = #tbaa_type_desc1, offset = 0>
+
+// CHECK-LABEL:   llvm.func @record_array_member(
+// CHECK-SAME:                                   %[[ARG0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !llvm.ptr) {
+// CHECK:           %[[X_VAL:.*]] = llvm.alloca %{{.+}} x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
+// CHECK:           %[[Z_VAL:.*]] = llvm.alloca %{{.+}} x !llvm.struct<"_QFTt2", (array<1 x struct<"_QFTt", (struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>)>>)> {bindc_name = "z"} : (i64) -> !llvm.ptr
+// CHECK:           "llvm.intr.memcpy"(%[[Z_VAL]], %[[ARG0]], %{{.+}}) <{isVolatile = false, tbaa = [#[[$ATTR_2]]]}> : (!llvm.ptr, !llvm.ptr, i64) -> ()
+
+// CHECK:           "llvm.intr.memcpy"(%[[X_VAL]], %{{.+}}, %{{.+}}) <{isVolatile = false, tbaa = [#[[$ATTR_4]]]}> : (!llvm.ptr, !llvm.ptr, i32) -> ()

From eef978290ccb847c67bf0431e5fdd1dc4c7d408d Mon Sep 17 00:00:00 2001
From: Mohamed Emad <73320969+hulxv@users.noreply.github.com>
Date: Thu, 17 Apr 2025 02:16:12 +0200
Subject: [PATCH 195/710]  [clang-doc][NFC] clean unused variable in HTML
 generator  (#135505)

While reading the code, I found some dead variables that are not used
anymore but it still declared without removing them.
---
 clang-tools-extra/clang-doc/HTMLGenerator.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index cb10f16804024..aceb83e8c4c57 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -716,7 +716,6 @@ genHTML(const EnumInfo &I, const ClangDocContext &CDCtx) {
 
   maybeWriteSourceFileRef(Out, CDCtx, I.DefLoc);
 
-  std::string Description;
   if (!I.Description.empty())
     Out.emplace_back(genHTML(I.Description));
 
@@ -759,7 +758,6 @@ genHTML(const FunctionInfo &I, const ClangDocContext &CDCtx,
 
   maybeWriteSourceFileRef(Out, CDCtx, I.DefLoc);
 
-  std::string Description;
   if (!I.Description.empty())
     Out.emplace_back(genHTML(I.Description));
 
@@ -777,7 +775,6 @@ genHTML(const NamespaceInfo &I, Index &InfoIndex, const ClangDocContext &CDCtx,
 
   Out.emplace_back(std::make_unique<TagNode>(HTMLTag::TAG_H1, InfoTitle));
 
-  std::string Description;
   if (!I.Description.empty())
     Out.emplace_back(genHTML(I.Description));
 
@@ -820,7 +817,6 @@ genHTML(const RecordInfo &I, Index &InfoIndex, const ClangDocContext &CDCtx,
 
   maybeWriteSourceFileRef(Out, CDCtx, I.DefLoc);
 
-  std::string Description;
   if (!I.Description.empty())
     Out.emplace_back(genHTML(I.Description));
 

From 218531821a6b6f1132a245a1f52e125610dc7f6a Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <peter@pcc.me.uk>
Date: Wed, 16 Apr 2025 17:19:50 -0700
Subject: [PATCH 196/710] LowerTypeTests: Fix quadratic complexity (try 2).

Currently we have quadratic complexity in LowerTypeTests because
ScopedSaveAliaseesAndUsed loops over all aliases for each disjoint
set, and the number of aliases and number of disjoint sets is
roughly proportional to the program size. Fix that by moving
ScopedSaveAliaseesAndUsed to LowerTypeTestsModule::lower() so that
we do this only once.

Reland of #135875 with fix for bug that caused check-lld test failures.
The fix is to only remove functions from llvm.used/llvm.compiler.used
because buildBitSetsFromGlobalVariables, which now runs while
ScopedSaveAliaseesAndUsed is in scope, will delete global variables,
which would otherwise lead to a use-after-free when they are added
back to llvm.used or llvm.compiler.used.

Reviewers: fmayer, vitalybuka

Reviewed By: fmayer

Pull Request: https://github.com/llvm/llvm-project/pull/136053
---
 llvm/lib/Transforms/IPO/LowerTypeTests.cpp | 194 ++++++++++++---------
 1 file changed, 107 insertions(+), 87 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 7cf7d74acfcfa..38da7329bbd58 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -352,6 +352,30 @@ struct ScopedSaveAliaseesAndUsed {
   std::vector<std::pair<GlobalAlias *, Function *>> FunctionAliases;
   std::vector<std::pair<GlobalIFunc *, Function *>> ResolverIFuncs;
 
+  // This function only removes functions from llvm.used and llvm.compiler.used.
+  // We cannot remove global variables because they need to follow RAUW, as
+  // they may be deleted by buildBitSetsFromGlobalVariables.
+  void collectAndEraseUsedFunctions(Module &M,
+                                    SmallVectorImpl<GlobalValue *> &Vec,
+                                    bool CompilerUsed) {
+    auto *GV = collectUsedGlobalVariables(M, Vec, CompilerUsed);
+    if (!GV)
+      return;
+    // There's no API to only remove certain array elements from
+    // llvm.used/llvm.compiler.used, so we remove all of them and add back only
+    // the non-functions.
+    GV->eraseFromParent();
+    auto NonFuncBegin =
+        std::stable_partition(Vec.begin(), Vec.end(), [](GlobalValue *GV) {
+          return isa<Function>(GV);
+        });
+    if (CompilerUsed)
+      appendToCompilerUsed(M, {NonFuncBegin, Vec.end()});
+    else
+      appendToUsed(M, {NonFuncBegin, Vec.end()});
+    Vec.resize(NonFuncBegin - Vec.begin());
+  }
+
   ScopedSaveAliaseesAndUsed(Module &M) : M(M) {
     // The users of this class want to replace all function references except
     // for aliases and llvm.used/llvm.compiler.used with references to a jump
@@ -365,10 +389,8 @@ struct ScopedSaveAliaseesAndUsed {
     // llvm.used/llvm.compiler.used and aliases, erase the used lists, let RAUW
     // replace the aliasees and then set them back to their original values at
     // the end.
-    if (GlobalVariable *GV = collectUsedGlobalVariables(M, Used, false))
-      GV->eraseFromParent();
-    if (GlobalVariable *GV = collectUsedGlobalVariables(M, CompilerUsed, true))
-      GV->eraseFromParent();
+    collectAndEraseUsedFunctions(M, Used, false);
+    collectAndEraseUsedFunctions(M, CompilerUsed, true);
 
     for (auto &GA : M.aliases()) {
       // FIXME: This should look past all aliases not just interposable ones,
@@ -1669,61 +1691,55 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
 
   lowerTypeTestCalls(TypeIds, JumpTable, GlobalLayout);
 
-  {
-    ScopedSaveAliaseesAndUsed S(M);
+  // Build aliases pointing to offsets into the jump table, and replace
+  // references to the original functions with references to the aliases.
+  for (unsigned I = 0; I != Functions.size(); ++I) {
+    Function *F = cast<Function>(Functions[I]->getGlobal());
+    bool IsJumpTableCanonical = Functions[I]->isJumpTableCanonical();
 
-    // Build aliases pointing to offsets into the jump table, and replace
-    // references to the original functions with references to the aliases.
-    for (unsigned I = 0; I != Functions.size(); ++I) {
-      Function *F = cast<Function>(Functions[I]->getGlobal());
-      bool IsJumpTableCanonical = Functions[I]->isJumpTableCanonical();
-
-      Constant *CombinedGlobalElemPtr = ConstantExpr::getInBoundsGetElementPtr(
-          JumpTableType, JumpTable,
-          ArrayRef<Constant *>{ConstantInt::get(IntPtrTy, 0),
-                               ConstantInt::get(IntPtrTy, I)});
-
-      const bool IsExported = Functions[I]->isExported();
-      if (!IsJumpTableCanonical) {
-        GlobalValue::LinkageTypes LT = IsExported
-                                           ? GlobalValue::ExternalLinkage
-                                           : GlobalValue::InternalLinkage;
-        GlobalAlias *JtAlias = GlobalAlias::create(F->getValueType(), 0, LT,
-                                                   F->getName() + ".cfi_jt",
-                                                   CombinedGlobalElemPtr, &M);
-        if (IsExported)
-          JtAlias->setVisibility(GlobalValue::HiddenVisibility);
-        else
-          appendToUsed(M, {JtAlias});
-      }
+    Constant *CombinedGlobalElemPtr = ConstantExpr::getInBoundsGetElementPtr(
+        JumpTableType, JumpTable,
+        ArrayRef<Constant *>{ConstantInt::get(IntPtrTy, 0),
+                             ConstantInt::get(IntPtrTy, I)});
+
+    const bool IsExported = Functions[I]->isExported();
+    if (!IsJumpTableCanonical) {
+      GlobalValue::LinkageTypes LT = IsExported ? GlobalValue::ExternalLinkage
+                                                : GlobalValue::InternalLinkage;
+      GlobalAlias *JtAlias = GlobalAlias::create(F->getValueType(), 0, LT,
+                                                 F->getName() + ".cfi_jt",
+                                                 CombinedGlobalElemPtr, &M);
+      if (IsExported)
+        JtAlias->setVisibility(GlobalValue::HiddenVisibility);
+      else
+        appendToUsed(M, {JtAlias});
+    }
 
-      if (IsExported) {
-        if (IsJumpTableCanonical)
-          ExportSummary->cfiFunctionDefs().emplace(F->getName());
-        else
-          ExportSummary->cfiFunctionDecls().emplace(F->getName());
-      }
+    if (IsExported) {
+      if (IsJumpTableCanonical)
+        ExportSummary->cfiFunctionDefs().emplace(F->getName());
+      else
+        ExportSummary->cfiFunctionDecls().emplace(F->getName());
+    }
 
-      if (!IsJumpTableCanonical) {
-        if (F->hasExternalWeakLinkage())
-          replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr,
-                                                 IsJumpTableCanonical);
-        else
-          replaceCfiUses(F, CombinedGlobalElemPtr, IsJumpTableCanonical);
-      } else {
-        assert(F->getType()->getAddressSpace() == 0);
-
-        GlobalAlias *FAlias =
-            GlobalAlias::create(F->getValueType(), 0, F->getLinkage(), "",
-                                CombinedGlobalElemPtr, &M);
-        FAlias->setVisibility(F->getVisibility());
-        FAlias->takeName(F);
-        if (FAlias->hasName())
-          F->setName(FAlias->getName() + ".cfi");
-        replaceCfiUses(F, FAlias, IsJumpTableCanonical);
-        if (!F->hasLocalLinkage())
-          F->setVisibility(GlobalVariable::HiddenVisibility);
-      }
+    if (!IsJumpTableCanonical) {
+      if (F->hasExternalWeakLinkage())
+        replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr,
+                                               IsJumpTableCanonical);
+      else
+        replaceCfiUses(F, CombinedGlobalElemPtr, IsJumpTableCanonical);
+    } else {
+      assert(F->getType()->getAddressSpace() == 0);
+
+      GlobalAlias *FAlias = GlobalAlias::create(
+          F->getValueType(), 0, F->getLinkage(), "", CombinedGlobalElemPtr, &M);
+      FAlias->setVisibility(F->getVisibility());
+      FAlias->takeName(F);
+      if (FAlias->hasName())
+        F->setName(FAlias->getName() + ".cfi");
+      replaceCfiUses(F, FAlias, IsJumpTableCanonical);
+      if (!F->hasLocalLinkage())
+        F->setVisibility(GlobalVariable::HiddenVisibility);
     }
   }
 
@@ -2339,39 +2355,43 @@ bool LowerTypeTestsModule::lower() {
   if (GlobalClasses.empty())
     return false;
 
-  // For each disjoint set we found...
-  for (const auto &C : GlobalClasses) {
-    if (!C->isLeader())
-      continue;
-
-    ++NumTypeIdDisjointSets;
-    // Build the list of type identifiers in this disjoint set.
-    std::vector<Metadata *> TypeIds;
-    std::vector<GlobalTypeMember *> Globals;
-    std::vector<ICallBranchFunnel *> ICallBranchFunnels;
-    for (auto M : GlobalClasses.members(*C)) {
-      if (isa<Metadata *>(M))
-        TypeIds.push_back(cast<Metadata *>(M));
-      else if (isa<GlobalTypeMember *>(M))
-        Globals.push_back(cast<GlobalTypeMember *>(M));
-      else
-        ICallBranchFunnels.push_back(cast<ICallBranchFunnel *>(M));
-    }
-
-    // Order type identifiers by unique ID for determinism. This ordering is
-    // stable as there is a one-to-one mapping between metadata and unique IDs.
-    llvm::sort(TypeIds, [&](Metadata *M1, Metadata *M2) {
-      return TypeIdInfo[M1].UniqueId < TypeIdInfo[M2].UniqueId;
-    });
+  {
+    ScopedSaveAliaseesAndUsed S(M);
+    // For each disjoint set we found...
+    for (const auto &C : GlobalClasses) {
+      if (!C->isLeader())
+        continue;
 
-    // Same for the branch funnels.
-    llvm::sort(ICallBranchFunnels,
-               [&](ICallBranchFunnel *F1, ICallBranchFunnel *F2) {
-                 return F1->UniqueId < F2->UniqueId;
-               });
+      ++NumTypeIdDisjointSets;
+      // Build the list of type identifiers in this disjoint set.
+      std::vector<Metadata *> TypeIds;
+      std::vector<GlobalTypeMember *> Globals;
+      std::vector<ICallBranchFunnel *> ICallBranchFunnels;
+      for (auto M : GlobalClasses.members(*C)) {
+        if (isa<Metadata *>(M))
+          TypeIds.push_back(cast<Metadata *>(M));
+        else if (isa<GlobalTypeMember *>(M))
+          Globals.push_back(cast<GlobalTypeMember *>(M));
+        else
+          ICallBranchFunnels.push_back(cast<ICallBranchFunnel *>(M));
+      }
 
-    // Build bitsets for this disjoint set.
-    buildBitSetsFromDisjointSet(TypeIds, Globals, ICallBranchFunnels);
+      // Order type identifiers by unique ID for determinism. This ordering is
+      // stable as there is a one-to-one mapping between metadata and unique
+      // IDs.
+      llvm::sort(TypeIds, [&](Metadata *M1, Metadata *M2) {
+        return TypeIdInfo[M1].UniqueId < TypeIdInfo[M2].UniqueId;
+      });
+
+      // Same for the branch funnels.
+      llvm::sort(ICallBranchFunnels,
+                 [&](ICallBranchFunnel *F1, ICallBranchFunnel *F2) {
+                   return F1->UniqueId < F2->UniqueId;
+                 });
+
+      // Build bitsets for this disjoint set.
+      buildBitSetsFromDisjointSet(TypeIds, Globals, ICallBranchFunnels);
+    }
   }
 
   allocateByteArrays();

From 18855ece3c34a0d76a2126538d60760ddeee2de8 Mon Sep 17 00:00:00 2001
From: Dave Lee <davelee.com@gmail.com>
Date: Wed, 16 Apr 2025 17:31:01 -0700
Subject: [PATCH 197/710] [lldb] Add summary for NSIndirectTaggedPointerString
 (#136025)

rdar://143164164
---
 .../source/Plugins/Language/ObjC/NSString.cpp | 51 ++++++++++++++++---
 lldb/source/Plugins/Language/ObjC/NSString.h  |  4 ++
 .../Plugins/Language/ObjC/ObjCLanguage.cpp    |  4 ++
 .../objc/foundation/tagged/strings/Makefile   |  3 ++
 .../tagged/strings/TestObjCTaggedStrings.py   | 12 +++++
 .../objc/foundation/tagged/strings/main.m     | 17 +++++++
 6 files changed, 85 insertions(+), 6 deletions(-)
 create mode 100644 lldb/test/API/lang/objc/foundation/tagged/strings/Makefile
 create mode 100644 lldb/test/API/lang/objc/foundation/tagged/strings/TestObjCTaggedStrings.py
 create mode 100644 lldb/test/API/lang/objc/foundation/tagged/strings/main.m

diff --git a/lldb/source/Plugins/Language/ObjC/NSString.cpp b/lldb/source/Plugins/Language/ObjC/NSString.cpp
index a99d042572bfe..2626b9a3f7b8a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSString.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSString.cpp
@@ -63,12 +63,17 @@ bool lldb_private::formatters::NSStringSummaryProvider(
   if (class_name.empty())
     return false;
 
-  bool is_tagged_ptr = class_name == "NSTaggedPointerString" &&
-                       descriptor->GetTaggedPointerInfo();
-  // for a tagged pointer, the descriptor has everything we need
-  if (is_tagged_ptr)
-    return NSTaggedString_SummaryProvider(valobj, descriptor, stream,
-                                          summary_options);
+  // For tagged pointers, the descriptor has everything needed.
+  bool is_tagged = descriptor->GetTaggedPointerInfo();
+  if (is_tagged) {
+    if (class_name == "NSTaggedPointerString")
+      return NSTaggedString_SummaryProvider(valobj, descriptor, stream,
+                                            summary_options);
+
+    if (class_name == "NSIndirectTaggedPointerString")
+      return NSIndirectTaggedString_SummaryProvider(valobj, descriptor, stream,
+                                                    summary_options);
+  }
 
   auto &additionals_map(NSString_Additionals::GetAdditionalSummaries());
   auto iter = additionals_map.find(class_name_cs), end = additionals_map.end();
@@ -368,3 +373,37 @@ bool lldb_private::formatters::NSTaggedString_SummaryProvider(
   stream << suffix;
   return true;
 }
+
+bool lldb_private::formatters::NSIndirectTaggedString_SummaryProvider(
+    ValueObject &valobj, ObjCLanguageRuntime::ClassDescriptorSP descriptor,
+    Stream &stream, const TypeSummaryOptions &summary_options) {
+  if (!descriptor)
+    return false;
+
+  uint64_t payload = 0;
+  if (!descriptor->GetTaggedPointerInfo(nullptr, nullptr, &payload))
+    return false;
+
+  // First 47 bits are the address of the contents.
+  addr_t ptr = payload & 0x7fffffffffffULL;
+  // Next 13 bits are the string's length.
+  size_t size = (payload >> 47) & 0x1fff;
+
+  Status status;
+  std::vector<char> buf(size);
+  if (auto process_sp = valobj.GetProcessSP())
+    if (process_sp->ReadMemory(ptr, buf.data(), size, status)) {
+      llvm::StringRef prefix, suffix;
+      if (auto *language = Language::FindPlugin(summary_options.GetLanguage()))
+        std::tie(prefix, suffix) =
+            language->GetFormatterPrefixSuffix("NSString");
+      stream << prefix << '"';
+      stream.PutCString({buf.data(), size});
+      stream << '"' << suffix;
+      return true;
+    }
+
+  if (status.Fail())
+    stream.Format("<{0}>", status);
+  return false;
+}
diff --git a/lldb/source/Plugins/Language/ObjC/NSString.h b/lldb/source/Plugins/Language/ObjC/NSString.h
index 8c9fcf955f1f8..5d405b30b6817 100644
--- a/lldb/source/Plugins/Language/ObjC/NSString.h
+++ b/lldb/source/Plugins/Language/ObjC/NSString.h
@@ -25,6 +25,10 @@ bool NSTaggedString_SummaryProvider(
     ValueObject &valobj, ObjCLanguageRuntime::ClassDescriptorSP descriptor,
     Stream &stream, const TypeSummaryOptions &summary_options);
 
+bool NSIndirectTaggedString_SummaryProvider(
+    ValueObject &valobj, ObjCLanguageRuntime::ClassDescriptorSP descriptor,
+    Stream &stream, const TypeSummaryOptions &summary_options);
+
 bool NSAttributedStringSummaryProvider(ValueObject &valobj, Stream &stream,
                                        const TypeSummaryOptions &options);
 
diff --git a/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp b/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp
index c835b439a64dd..3b8e21cbb9269 100644
--- a/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp
+++ b/lldb/source/Plugins/Language/ObjC/ObjCLanguage.cpp
@@ -691,6 +691,10 @@ static void LoadObjCFormatters(TypeCategoryImplSP objc_category_sp) {
   AddCXXSummary(
       objc_category_sp, lldb_private::formatters::NSStringSummaryProvider,
       "NSString summary provider", "NSTaggedPointerString", appkit_flags);
+  AddCXXSummary(objc_category_sp,
+                lldb_private::formatters::NSStringSummaryProvider,
+                "NSString summary provider", "NSIndirectTaggedPointerString",
+                appkit_flags);
 
   AddCXXSummary(objc_category_sp,
                 lldb_private::formatters::NSAttributedStringSummaryProvider,
diff --git a/lldb/test/API/lang/objc/foundation/tagged/strings/Makefile b/lldb/test/API/lang/objc/foundation/tagged/strings/Makefile
new file mode 100644
index 0000000000000..a3198db9e8e88
--- /dev/null
+++ b/lldb/test/API/lang/objc/foundation/tagged/strings/Makefile
@@ -0,0 +1,3 @@
+OBJC_SOURCES := main.m
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/test/API/lang/objc/foundation/tagged/strings/TestObjCTaggedStrings.py b/lldb/test/API/lang/objc/foundation/tagged/strings/TestObjCTaggedStrings.py
new file mode 100644
index 0000000000000..66dc895b6a9ed
--- /dev/null
+++ b/lldb/test/API/lang/objc/foundation/tagged/strings/TestObjCTaggedStrings.py
@@ -0,0 +1,12 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+    def test(self):
+        """Verify summary formatter for tagged strings."""
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.m"))
+        self.expect("v str1 str2", patterns=['@"nineDigit"', '@"tenDigitXX"'])
diff --git a/lldb/test/API/lang/objc/foundation/tagged/strings/main.m b/lldb/test/API/lang/objc/foundation/tagged/strings/main.m
new file mode 100644
index 0000000000000..c2b5575986a99
--- /dev/null
+++ b/lldb/test/API/lang/objc/foundation/tagged/strings/main.m
@@ -0,0 +1,17 @@
+#import <Foundation/Foundation.h>
+
+@interface NSObject (Fake)
+// 9 digit selector
+- (void)nineDigit;
+// 10 digit selector
+- (void)tenDigitXX;
+@end
+
+int main() {
+  SEL sel1 = @selector(nineDigit);
+  NSString *str1 = NSStringFromSelector(sel1);
+  SEL sel2 = @selector(tenDigitXX);
+  NSString *str2 = NSStringFromSelector(sel2);
+  NSLog(@"break here %@, %@", str1, str2);
+  return 0;
+}

From 78671dbb26628f072cb94de784349c50201c1e0f Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Thu, 17 Apr 2025 03:53:08 +0300
Subject: [PATCH 198/710] [Sparc] Use helper class for emitting CFI
 instructions into MIR (#136027)

Also, guard emission by `needsFrameMoves()` check. There are no changes
in tests because cfi instructions are currently ignored by AsmPrinter
when they don't need to be printed/encoded.

PR: https://github.com/llvm/llvm-project/pull/136027
---
 llvm/include/llvm/CodeGen/CFIInstBuilder.h   |  4 +++
 llvm/lib/Target/Sparc/SparcFrameLowering.cpp | 34 ++++----------------
 2 files changed, 11 insertions(+), 27 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/CFIInstBuilder.h b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
index 9025624c0d8ab..bf7102eeea497 100644
--- a/llvm/include/llvm/CodeGen/CFIInstBuilder.h
+++ b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
@@ -82,6 +82,10 @@ class CFIInstBuilder {
         TRI.getDwarfRegNum(Reg2, IsEH)));
   }
 
+  void buildWindowSave() const {
+    insertCFIInst(MCCFIInstruction::createWindowSave(nullptr));
+  }
+
   void buildRestore(MCRegister Reg) const {
     insertCFIInst(MCCFIInstruction::createRestore(
         nullptr, TRI.getDwarfRegNum(Reg, IsEH)));
diff --git a/llvm/lib/Target/Sparc/SparcFrameLowering.cpp b/llvm/lib/Target/Sparc/SparcFrameLowering.cpp
index 14233a526eec1..2934c88b6bffc 100644
--- a/llvm/lib/Target/Sparc/SparcFrameLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcFrameLowering.cpp
@@ -14,6 +14,7 @@
 #include "SparcInstrInfo.h"
 #include "SparcMachineFunctionInfo.h"
 #include "SparcSubtarget.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -88,14 +89,7 @@ void SparcFrameLowering::emitPrologue(MachineFunction &MF,
   assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
   MachineFrameInfo &MFI = MF.getFrameInfo();
   const SparcSubtarget &Subtarget = MF.getSubtarget<SparcSubtarget>();
-  const SparcInstrInfo &TII =
-      *static_cast<const SparcInstrInfo *>(Subtarget.getInstrInfo());
-  const SparcRegisterInfo &RegInfo =
-      *static_cast<const SparcRegisterInfo *>(Subtarget.getRegisterInfo());
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  // Debug location must be unknown since the first debug location is used
-  // to determine the end of the prologue.
-  DebugLoc dl;
 
   // Get the number of bytes to allocate from the FrameInfo
   int NumBytes = (int) MFI.getStackSize();
@@ -141,26 +135,12 @@ void SparcFrameLowering::emitPrologue(MachineFunction &MF,
 
   emitSPAdjustment(MF, MBB, MBBI, -NumBytes, SAVErr, SAVEri);
 
-  unsigned regFP = RegInfo.getDwarfRegNum(SP::I6, true);
-
-  // Emit ".cfi_def_cfa_register 30".
-  unsigned CFIIndex =
-      MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, regFP));
-  BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex);
-
-  // Emit ".cfi_window_save".
-  CFIIndex = MF.addFrameInst(MCCFIInstruction::createWindowSave(nullptr));
-  BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex);
-
-  unsigned regInRA = RegInfo.getDwarfRegNum(SP::I7, true);
-  unsigned regOutRA = RegInfo.getDwarfRegNum(SP::O7, true);
-  // Emit ".cfi_register 15, 31".
-  CFIIndex = MF.addFrameInst(
-      MCCFIInstruction::createRegister(nullptr, regOutRA, regInRA));
-  BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex);
+  if (MF.needsFrameMoves()) {
+    CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::NoFlags);
+    CFIBuilder.buildDefCFARegister(SP::I6);
+    CFIBuilder.buildWindowSave();
+    CFIBuilder.buildRegister(SP::O7, SP::I7);
+  }
 }
 
 MachineBasicBlock::iterator SparcFrameLowering::

From 6d8bf3cf3dcc5d85bec7b1e70a59a02cdfdaa1b4 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Wed, 16 Apr 2025 18:24:10 -0700
Subject: [PATCH 199/710] Revert "Reapply "[LLVM][TableGen] Parameterize
 NumToSkip in DecoderEmitter" (#136017)" (#136068)

Reverts llvm/llvm-project#136019

Expensive checks tests are failing, so reverting.
---
 llvm/lib/Target/AArch64/CMakeLists.txt    |   2 +-
 llvm/test/TableGen/VarLenDecoder.td       |   4 +-
 llvm/test/TableGen/trydecode-emission.td  |  10 +-
 llvm/test/TableGen/trydecode-emission2.td |  16 +--
 llvm/test/TableGen/trydecode-emission3.td |   2 +-
 llvm/test/TableGen/trydecode-emission4.td |   2 +-
 llvm/utils/TableGen/DecoderEmitter.cpp    | 115 ++++++++++------------
 7 files changed, 68 insertions(+), 83 deletions(-)

diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index ba1d1605ec104..2300e479bc110 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -7,7 +7,7 @@ tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
 tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
 tablegen(LLVM AArch64GenCallingConv.inc -gen-callingconv)
 tablegen(LLVM AArch64GenDAGISel.inc -gen-dag-isel)
-tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler --num-to-skip-size=3)
+tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler)
 tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel)
 tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel)
 tablegen(LLVM AArch64GenO0PreLegalizeGICombiner.inc -gen-global-isel-combiner
diff --git a/llvm/test/TableGen/VarLenDecoder.td b/llvm/test/TableGen/VarLenDecoder.td
index b77702ff7c5c1..5cf0bf8911859 100644
--- a/llvm/test/TableGen/VarLenDecoder.td
+++ b/llvm/test/TableGen/VarLenDecoder.td
@@ -47,9 +47,9 @@ def FOO32 : MyVarInst<MemOp32> {
 }
 
 // CHECK:      MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
-// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, // Skip to: 11
+// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, 0, // Skip to: 12
 // CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
-// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, // Skip to: 19
+// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, 0, // Skip to: 21
 // CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
 // CHECK-NEXT: MCD::OPC_Fail,
 
diff --git a/llvm/test/TableGen/trydecode-emission.td b/llvm/test/TableGen/trydecode-emission.td
index 2b4239f4fbe65..20d2446eeac7f 100644
--- a/llvm/test/TableGen/trydecode-emission.td
+++ b/llvm/test/TableGen/trydecode-emission.td
@@ -34,10 +34,10 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23
-// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19
-// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 19
-// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
+// CHECK-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
+// CHECK-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
+// CHECK-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
+// CHECK-NEXT: /* 26 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission2.td b/llvm/test/TableGen/trydecode-emission2.td
index 7d30474058f73..0584034e41233 100644
--- a/llvm/test/TableGen/trydecode-emission2.td
+++ b/llvm/test/TableGen/trydecode-emission2.td
@@ -31,14 +31,14 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 31, 0, // Skip to: 38
-// CHECK-NEXT: /* 7 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
-// CHECK-NEXT: /* 10 */      MCD::OPC_FilterValue, 0, 24, 0, // Skip to: 38
-// CHECK-NEXT: /* 14 */      MCD::OPC_CheckField, 0, 2, 3, 6, 0, // Skip to: 26
-// CHECK-NEXT: /* 20 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 26
-// CHECK-NEXT: /* 26 */      MCD::OPC_CheckField, 3, 2, 0, 6, 0, // Skip to: 38
-// CHECK-NEXT: /* 32 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, // Opcode: InstA, skip to: 38
-// CHECK-NEXT: /* 38 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 36, 0, 0, // Skip to: 44
+// CHECK-NEXT: /* 8 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-NEXT: /* 11 */      MCD::OPC_FilterValue, 0, 28, 0, 0, // Skip to: 44
+// CHECK-NEXT: /* 16 */      MCD::OPC_CheckField, 0, 2, 3, 7, 0, 0, // Skip to: 30
+// CHECK-NEXT: /* 23 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 30
+// CHECK-NEXT: /* 30 */      MCD::OPC_CheckField, 3, 2, 0, 7, 0, 0, // Skip to: 44
+// CHECK-NEXT: /* 37 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstA, skip to: 44
+// CHECK-NEXT: /* 44 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 // CHECK: if (!Check(S, DecodeInstA(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission3.td b/llvm/test/TableGen/trydecode-emission3.td
index 0abbe62fe337e..4c5be7e1af229 100644
--- a/llvm/test/TableGen/trydecode-emission3.td
+++ b/llvm/test/TableGen/trydecode-emission3.td
@@ -1,4 +1,4 @@
- // RUN: llvm-tblgen -gen-disassembler --num-to-skip-size=3 -I %p/../../include %s  | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
 
 include "llvm/Target/Target.td"
 
diff --git a/llvm/test/TableGen/trydecode-emission4.td b/llvm/test/TableGen/trydecode-emission4.td
index 413e4a0d1275a..1e51ba5e40768 100644
--- a/llvm/test/TableGen/trydecode-emission4.td
+++ b/llvm/test/TableGen/trydecode-emission4.td
@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-disassembler --num-to-skip-size=3 -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
 
 // Test for OPC_ExtractField/OPC_CheckField with start bit > 255.
 // These large start values may arise for architectures with long instruction
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index eff63c6b45bb3..9c6015cc24576 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -32,10 +32,8 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/LEB128.h"
-#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -78,12 +76,6 @@ static cl::opt<SuppressLevel> DecoderEmitterSuppressDuplicates(
             "significantly reducing Table Duplications")),
     cl::init(SUPPRESSION_DISABLE), cl::cat(DisassemblerEmitterCat));
 
-static cl::opt<uint32_t>
-    NumToSkipSizeInBytes("num-to-skip-size",
-                         cl::desc("number of bytes to use for num-to-skip "
-                                  "entries in the decoder table (2 or 3)"),
-                         cl::init(2), cl::cat(DisassemblerEmitterCat));
-
 STATISTIC(NumEncodings, "Number of encodings considered");
 STATISTIC(NumEncodingsLackingDisasm,
           "Number of encodings without disassembler info");
@@ -138,29 +130,10 @@ struct DecoderTable : public std::vector<uint8_t> {
   // in the table for patching.
   size_t insertNumToSkip() {
     size_t Size = size();
-    insert(end(), NumToSkipSizeInBytes, 0);
+    insert(end(), 3, 0);
     return Size;
   }
-
-  void patchNumToSkip(size_t FixupIdx, uint32_t DestIdx) {
-    // Calculate the distance from the byte following the fixup entry byte
-    // to the destination. The Target is calculated from after the
-    // `NumToSkipSizeInBytes`-byte NumToSkip entry itself, so subtract
-    // `NumToSkipSizeInBytes` from the displacement here to account for that.
-    assert(DestIdx >= FixupIdx + NumToSkipSizeInBytes &&
-           "Expecting a forward jump in the decoding table");
-    uint32_t Delta = DestIdx - FixupIdx - NumToSkipSizeInBytes;
-    if (!isUIntN(8 * NumToSkipSizeInBytes, Delta))
-      PrintFatalError(
-          "disassembler decoding table too large, try --num-to-skip-size=3");
-
-    (*this)[FixupIdx] = static_cast<uint8_t>(Delta);
-    (*this)[FixupIdx + 1] = static_cast<uint8_t>(Delta >> 8);
-    if (NumToSkipSizeInBytes == 3)
-      (*this)[FixupIdx + 2] = static_cast<uint8_t>(Delta >> 16);
-  }
 };
-
 struct DecoderTableInfo {
   DecoderTable Table;
   FixupScopeList FixupStack;
@@ -717,8 +690,19 @@ static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups,
                                uint32_t DestIdx) {
   // Any NumToSkip fixups in the current scope can resolve to the
   // current location.
-  for (uint32_t FixupIdx : Fixups)
-    Table.patchNumToSkip(FixupIdx, DestIdx);
+  for (uint32_t FixupIdx : reverse(Fixups)) {
+    // Calculate the distance from the byte following the fixup entry byte
+    // to the destination. The Target is calculated from after the 24-bit
+    // NumToSkip entry itself, so subtract three from the displacement here
+    // to account for that.
+    uint32_t Delta = DestIdx - FixupIdx - 3;
+    // Our NumToSkip entries are 24-bits. Make sure our table isn't too
+    // big.
+    assert(isUInt<24>(Delta));
+    Table[FixupIdx] = (uint8_t)Delta;
+    Table[FixupIdx + 1] = (uint8_t)(Delta >> 8);
+    Table[FixupIdx + 2] = (uint8_t)(Delta >> 16);
+  }
 }
 
 // Emit table entries to decode instructions given a segment or segments
@@ -775,9 +759,15 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
     Delegate->emitTableEntries(TableInfo);
 
     // Now that we've emitted the body of the handler, update the NumToSkip
-    // of the filter itself to be able to skip forward when false.
-    if (PrevFilter)
-      Table.patchNumToSkip(PrevFilter, Table.size());
+    // of the filter itself to be able to skip forward when false. Subtract
+    // three as to account for the width of the NumToSkip field itself.
+    if (PrevFilter) {
+      uint32_t NumToSkip = Table.size() - PrevFilter - 3;
+      assert(isUInt<24>(NumToSkip) && "disassembler decoding table too large!");
+      Table[PrevFilter] = (uint8_t)NumToSkip;
+      Table[PrevFilter + 1] = (uint8_t)(NumToSkip >> 8);
+      Table[PrevFilter + 2] = (uint8_t)(NumToSkip >> 16);
+    }
   }
 
   // If there is no fallthrough, then the final filter should get fixed
@@ -824,8 +814,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     OS << (unsigned)*I++ << ", ";
   };
 
-  // Emit `NumToSkipSizeInBytes`-byte numtoskip value to OS, returning the
-  // NumToSkip value.
+  // Emit 24-bit numtoskip value to OS, returning the NumToSkip value.
   auto emitNumToSkip = [](DecoderTable::const_iterator &I,
                           formatted_raw_ostream &OS) {
     uint8_t Byte = *I++;
@@ -834,11 +823,9 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     Byte = *I++;
     OS << (unsigned)Byte << ", ";
     NumToSkip |= Byte << 8;
-    if (NumToSkipSizeInBytes == 3) {
-      Byte = *I++;
-      OS << (unsigned)(Byte) << ", ";
-      NumToSkip |= Byte << 16;
-    }
+    Byte = *I++;
+    OS << (unsigned)(Byte) << ", ";
+    NumToSkip |= Byte << 16;
     return NumToSkip;
   };
 
@@ -880,7 +867,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // The filter value is ULEB128 encoded.
       emitULEB128(I, OS);
 
-      // numtoskip value.
+      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -896,7 +883,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // ULEB128 encoded field value.
       emitULEB128(I, OS);
 
-      // numtoskip value.
+      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -906,7 +893,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       OS << Indent << "MCD::OPC_CheckPredicate, ";
       emitULEB128(I, OS);
 
-      // numtoskip value.
+      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -938,7 +925,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
 
       // Fallthrough for OPC_TryDecode.
 
-      // numtoskip value.
+      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
 
       OS << "// Opcode: " << NumberedEncodings[EncodingID]
@@ -1424,9 +1411,9 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
     TableInfo.Table.push_back(NumBits);
     TableInfo.Table.insertULEB128(Ilnd.FieldVal);
 
-    // Allocate space in the table for fixup (NumToSkipSizeInBytes) so all
-    // our relative position calculations work OK even before we fully
-    // resolve the real value here.
+    // The fixup is always 24-bits, so go ahead and allocate the space
+    // in the table so all our relative position calculations work OK even
+    // before we fully resolve the real value here.
 
     // Push location for NumToSkip backpatching.
     TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
@@ -2170,18 +2157,7 @@ insertBits(InsnType &field, uint64_t bits, unsigned startBit, unsigned numBits)
 // decodeInstruction().
 static void emitDecodeInstruction(formatted_raw_ostream &OS,
                                   bool IsVarLenInst) {
-  OS << formatv("\nconstexpr unsigned NumToSkipSizeInBytes = {};\n",
-                NumToSkipSizeInBytes);
-
   OS << R"(
-inline unsigned decodeNumToSkip(const uint8_t *&Ptr) {
-  unsigned NumToSkip = *Ptr++;
-  NumToSkip |= (*Ptr++) << 8;
-  if constexpr (NumToSkipSizeInBytes == 3)
-    NumToSkip |= (*Ptr++) << 16;
-  return NumToSkip;
-}
-
 template <typename InsnType>
 static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
                                       InsnType insn, uint64_t Address,
@@ -2219,7 +2195,10 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the field value.
       uint64_t Val = decodeULEB128AndIncUnsafe(++Ptr);
       bool Failed = Val != CurFieldValue;
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      // NumToSkip is a plain 24-bit integer.
+      unsigned NumToSkip = *Ptr++;
+      NumToSkip |= (*Ptr++) << 8;
+      NumToSkip |= (*Ptr++) << 16;
 
       // Perform the filter operation.
       if (Failed)
@@ -2243,7 +2222,10 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       uint64_t ExpectedValue = decodeULEB128(++Ptr, &PtrLen);
       Ptr += PtrLen;
       bool Failed = ExpectedValue != FieldValue;
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      // NumToSkip is a plain 24-bit integer.
+      unsigned NumToSkip = *Ptr++;
+      NumToSkip |= (*Ptr++) << 8;
+      NumToSkip |= (*Ptr++) << 16;
 
       // If the actual and expected values don't match, skip.
       if (Failed)
@@ -2258,7 +2240,10 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     case MCD::OPC_CheckPredicate: {
       // Decode the Predicate Index value.
       unsigned PIdx = decodeULEB128AndIncUnsafe(++Ptr);
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      // NumToSkip is a plain 24-bit integer.
+      unsigned NumToSkip = *Ptr++;
+      NumToSkip |= (*Ptr++) << 8;
+      NumToSkip |= (*Ptr++) << 16;
       // Check the predicate.
       bool Failed = !checkDecoderPredicate(PIdx, Bits);
       if (Failed)
@@ -2293,7 +2278,10 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the Opcode value.
       unsigned Opc = decodeULEB128AndIncUnsafe(++Ptr);
       unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      // NumToSkip is a plain 24-bit integer.
+      unsigned NumToSkip = *Ptr++;
+      NumToSkip |= (*Ptr++) << 8;
+      NumToSkip |= (*Ptr++) << 16;
 
       // Perform the decode operation.
       MCInst TmpMI;
@@ -2418,9 +2406,6 @@ handleHwModesUnrelatedEncodings(const CodeGenInstruction *Instr,
 
 // Emits disassembler code for instruction decoding.
 void DecoderEmitter::run(raw_ostream &o) {
-  if (NumToSkipSizeInBytes != 2 && NumToSkipSizeInBytes != 3)
-    PrintFatalError("Invalid value for num-to-skip-size, must be 2 or 3");
-
   formatted_raw_ostream OS(o);
   OS << R"(
 #include "llvm/MC/MCInst.h"

From 6b8d072cfd41f647f2c241f0a1a0843a279d049b Mon Sep 17 00:00:00 2001
From: Wu Yingcong <yingcong.wu@intel.com>
Date: Thu, 17 Apr 2025 09:33:48 +0800
Subject: [PATCH 200/710] [libc] Fix incorrect unsigned comparison (#135595)

There is a problem with such unsigned comparison pattern:
```
if(unsigned_a - unsigned_b > 0) { /* only NOT go here when unsigned_a==unsigned_b */ }
```
When `unsigned_a` < `unsigned_b`, the result will still be `>0` due to
underflow.
This patch fixes two of the occurrences I found.
Also remove two redundant `if` where its condition is guaranteed by
outer `if`.
---
 libc/src/stdio/printf_core/float_dec_converter.h | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/libc/src/stdio/printf_core/float_dec_converter.h b/libc/src/stdio/printf_core/float_dec_converter.h
index ee5549825a6f2..ed004f9a26a13 100644
--- a/libc/src/stdio/printf_core/float_dec_converter.h
+++ b/libc/src/stdio/printf_core/float_dec_converter.h
@@ -186,13 +186,12 @@ template <WriteMode write_mode> class FloatWriter {
     if (total_digits_written < digits_before_decimal &&
         total_digits_written + buffered_digits >= digits_before_decimal &&
         has_decimal_point) {
+      // digits_to_write > 0 guaranteed by outer if
       size_t digits_to_write = digits_before_decimal - total_digits_written;
-      if (digits_to_write > 0) {
-        // Write the digits before the decimal point.
-        RET_IF_RESULT_NEGATIVE(writer->write({block_buffer, digits_to_write}));
-      }
+      // Write the digits before the decimal point.
+      RET_IF_RESULT_NEGATIVE(writer->write({block_buffer, digits_to_write}));
       RET_IF_RESULT_NEGATIVE(writer->write(DECIMAL_POINT));
-      if (buffered_digits - digits_to_write > 0) {
+      if (buffered_digits > digits_to_write) {
         // Write the digits after the decimal point.
         RET_IF_RESULT_NEGATIVE(
             writer->write({block_buffer + digits_to_write,
@@ -217,12 +216,11 @@ template <WriteMode write_mode> class FloatWriter {
         total_digits_written + BLOCK_SIZE * max_block_count >=
             digits_before_decimal &&
         has_decimal_point) {
+      // digits_to_write > 0 guaranteed by outer if
       size_t digits_to_write = digits_before_decimal - total_digits_written;
-      if (digits_to_write > 0) {
-        RET_IF_RESULT_NEGATIVE(writer->write(MAX_BLOCK_DIGIT, digits_to_write));
-      }
+      RET_IF_RESULT_NEGATIVE(writer->write(MAX_BLOCK_DIGIT, digits_to_write));
       RET_IF_RESULT_NEGATIVE(writer->write(DECIMAL_POINT));
-      if ((BLOCK_SIZE * max_block_count) - digits_to_write > 0) {
+      if ((BLOCK_SIZE * max_block_count) > digits_to_write) {
         RET_IF_RESULT_NEGATIVE(writer->write(
             MAX_BLOCK_DIGIT, (BLOCK_SIZE * max_block_count) - digits_to_write));
       }

From 53eae22e228532fe3349890b6c7fc10b9c10dbee Mon Sep 17 00:00:00 2001
From: joaosaffran <126493771+joaosaffran@users.noreply.github.com>
Date: Wed, 16 Apr 2025 18:38:53 -0700
Subject: [PATCH 201/710] [DirectX] adding support in obj2yaml and yaml2obj to
 root constants (#127840)

Adding support for Root Constant in MC, Object and obj2yaml and
yaml2obj, this PR adds:
  - new structures to dxbc definition.
  - serialize and desirialize logic from dxcontainer to yaml
  - tests validating against dxc
  - adding support to multiple parts.

Closes: https://github.com/llvm/llvm-project/issues/126633

---------

Co-authored-by: joaosaffran <joao.saffran@microsoft.com>
---
 llvm/include/llvm/BinaryFormat/DXContainer.h  |  77 ++++++++++-
 .../BinaryFormat/DXContainerConstants.def     |  18 +++
 .../llvm/MC/DXContainerRootSignature.h        |  24 +++-
 llvm/include/llvm/Object/DXContainer.h        |  72 +++++++++-
 .../include/llvm/ObjectYAML/DXContainerYAML.h |  38 +++++-
 llvm/lib/BinaryFormat/DXContainer.cpp         |  20 +++
 llvm/lib/MC/DXContainerRootSignature.cpp      |  76 ++++++++++-
 llvm/lib/Object/DXContainer.cpp               |  31 ++---
 llvm/lib/ObjectYAML/DXContainerEmitter.cpp    |  19 ++-
 llvm/lib/ObjectYAML/DXContainerYAML.cpp       |  84 ++++++++++--
 llvm/lib/Target/DirectX/DXILRootSignature.cpp |  17 ++-
 .../ContainerData/RootSignature-Flags.ll      |   3 +-
 .../RootSignature-MultipleEntryFunctions.ll   |   8 +-
 .../DXContainer/RootSignature-Flags.yaml      |  18 +--
 .../RootSignature-InvalidType.yaml            |  29 +++++
 .../RootSignature-InvalidVisibility.yaml      |  33 +++++
 .../RootSignature-MultipleParameters.yaml     |  59 +++++++++
 llvm/tools/obj2yaml/dxcontainer2yaml.cpp      |   9 +-
 llvm/unittests/Object/DXContainerTest.cpp     | 123 ++++++++++++++----
 .../ObjectYAML/DXContainerYAMLTest.cpp        | 111 +++++++++++++++-
 20 files changed, 764 insertions(+), 105 deletions(-)
 create mode 100644 llvm/test/ObjectYAML/DXContainer/RootSignature-InvalidType.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/RootSignature-InvalidVisibility.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/RootSignature-MultipleParameters.yaml

diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index 28905e27837a7..455657980bf40 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -14,6 +14,7 @@
 #define LLVM_BINARYFORMAT_DXCONTAINER_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -157,6 +158,40 @@ enum class RootElementFlag : uint32_t {
 #include "DXContainerConstants.def"
 };
 
+#define ROOT_PARAMETER(Val, Enum) Enum = Val,
+enum class RootParameterType : uint32_t {
+#include "DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<RootParameterType>> getRootParameterTypes();
+
+#define ROOT_PARAMETER(Val, Enum)                                              \
+  case Val:                                                                    \
+    return true;
+inline bool isValidParameterType(uint32_t V) {
+  switch (V) {
+#include "DXContainerConstants.def"
+  }
+  return false;
+}
+
+#define SHADER_VISIBILITY(Val, Enum) Enum = Val,
+enum class ShaderVisibility : uint32_t {
+#include "DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<ShaderVisibility>> getShaderVisibility();
+
+#define SHADER_VISIBILITY(Val, Enum)                                           \
+  case Val:                                                                    \
+    return true;
+inline bool isValidShaderVisibility(uint32_t V) {
+  switch (V) {
+#include "DXContainerConstants.def"
+  }
+  return false;
+}
+
 PartType parsePartType(StringRef S);
 
 struct VertexPSVInfo {
@@ -546,15 +581,49 @@ struct ProgramSignatureElement {
 static_assert(sizeof(ProgramSignatureElement) == 32,
               "ProgramSignatureElement is misaligned");
 
-struct RootSignatureValidations {
+// following dx12 naming
+// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ns-d3d12-d3d12_root_constants
+struct RootConstants {
+  uint32_t ShaderRegister;
+  uint32_t RegisterSpace;
+  uint32_t Num32BitValues;
+
+  void swapBytes() {
+    sys::swapByteOrder(ShaderRegister);
+    sys::swapByteOrder(RegisterSpace);
+    sys::swapByteOrder(Num32BitValues);
+  }
+};
 
-  static bool isValidRootFlag(uint32_t Flags) { return (Flags & ~0xfff) == 0; }
+struct RootParameterHeader {
+  uint32_t ParameterType;
+  uint32_t ShaderVisibility;
+  uint32_t ParameterOffset;
 
-  static bool isValidVersion(uint32_t Version) {
-    return (Version == 1 || Version == 2);
+  void swapBytes() {
+    sys::swapByteOrder(ParameterType);
+    sys::swapByteOrder(ShaderVisibility);
+    sys::swapByteOrder(ParameterOffset);
   }
 };
 
+struct RootSignatureHeader {
+  uint32_t Version;
+  uint32_t NumParameters;
+  uint32_t ParametersOffset;
+  uint32_t NumStaticSamplers;
+  uint32_t StaticSamplerOffset;
+  uint32_t Flags;
+
+  void swapBytes() {
+    sys::swapByteOrder(Version);
+    sys::swapByteOrder(NumParameters);
+    sys::swapByteOrder(ParametersOffset);
+    sys::swapByteOrder(NumStaticSamplers);
+    sys::swapByteOrder(StaticSamplerOffset);
+    sys::swapByteOrder(Flags);
+  }
+};
 } // namespace dxbc
 } // namespace llvm
 
diff --git a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
index 6d44ea14df444..590ded5e8c899 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
+++ b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
@@ -72,6 +72,24 @@ ROOT_ELEMENT_FLAG(11, SamplerHeapDirectlyIndexed)
 #undef ROOT_ELEMENT_FLAG
 #endif // ROOT_ELEMENT_FLAG
 
+#ifdef ROOT_PARAMETER
+
+ROOT_PARAMETER(1, Constants32Bit)
+#undef ROOT_PARAMETER
+#endif // ROOT_PARAMETER
+
+#ifdef SHADER_VISIBILITY
+
+SHADER_VISIBILITY(0, All)
+SHADER_VISIBILITY(1, Vertex)
+SHADER_VISIBILITY(2, Hull)
+SHADER_VISIBILITY(3, Domain)
+SHADER_VISIBILITY(4, Geometry)
+SHADER_VISIBILITY(5, Pixel)
+SHADER_VISIBILITY(6, Amplification)
+SHADER_VISIBILITY(7, Mesh)
+#undef SHADER_VISIBILITY
+#endif // SHADER_VISIBILITY
 
 #ifdef DXIL_MODULE_FLAG
 
diff --git a/llvm/include/llvm/MC/DXContainerRootSignature.h b/llvm/include/llvm/MC/DXContainerRootSignature.h
index e414112498798..fee799249b255 100644
--- a/llvm/include/llvm/MC/DXContainerRootSignature.h
+++ b/llvm/include/llvm/MC/DXContainerRootSignature.h
@@ -6,23 +6,33 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/BinaryFormat/DXContainer.h"
 #include <cstdint>
 #include <limits>
 
 namespace llvm {
 
 class raw_ostream;
-
 namespace mcdxbc {
+
+struct RootParameter {
+  dxbc::RootParameterHeader Header;
+  union {
+    dxbc::RootConstants Constants;
+  };
+};
 struct RootSignatureDesc {
-  uint32_t Version = 2;
-  uint32_t NumParameters = 0;
-  uint32_t RootParametersOffset = 0;
-  uint32_t NumStaticSamplers = 0;
-  uint32_t StaticSamplersOffset = 0;
-  uint32_t Flags = 0;
+
+  uint32_t Version = 2U;
+  uint32_t Flags = 0U;
+  uint32_t RootParameterOffset = 0U;
+  uint32_t StaticSamplersOffset = 0u;
+  uint32_t NumStaticSamplers = 0u;
+  SmallVector<mcdxbc::RootParameter> Parameters;
 
   void write(raw_ostream &OS) const;
+
+  size_t getSize() const;
 };
 } // namespace mcdxbc
 } // namespace llvm
diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h
index c3a2f756bd683..e8287ce078365 100644
--- a/llvm/include/llvm/Object/DXContainer.h
+++ b/llvm/include/llvm/Object/DXContainer.h
@@ -18,10 +18,12 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/DXContainer.h"
+#include "llvm/Object/Error.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/TargetParser/Triple.h"
 #include <array>
+#include <cstddef>
 #include <variant>
 
 namespace llvm {
@@ -116,6 +118,40 @@ template <typename T> struct ViewArray {
 };
 
 namespace DirectX {
+struct RootParameterView {
+  const dxbc::RootParameterHeader &Header;
+  StringRef ParamData;
+  RootParameterView(const dxbc::RootParameterHeader &H, StringRef P)
+      : Header(H), ParamData(P) {}
+
+  template <typename T> Expected<T> readParameter() {
+    T Struct;
+    if (sizeof(T) != ParamData.size())
+      return make_error<GenericBinaryError>(
+          "Reading structure out of file bounds", object_error::parse_failed);
+
+    memcpy(&Struct, ParamData.data(), sizeof(T));
+    // DXContainer is always little endian
+    if (sys::IsBigEndianHost)
+      Struct.swapBytes();
+    return Struct;
+  }
+};
+
+struct RootConstantView : RootParameterView {
+  static bool classof(const RootParameterView *V) {
+    return V->Header.ParameterType ==
+           (uint32_t)dxbc::RootParameterType::Constants32Bit;
+  }
+
+  llvm::Expected<dxbc::RootConstants> read() {
+    return readParameter<dxbc::RootConstants>();
+  }
+};
+
+static Error parseFailed(const Twine &Msg) {
+  return make_error<GenericBinaryError>(Msg.str(), object_error::parse_failed);
+}
 
 class RootSignature {
 private:
@@ -125,17 +161,49 @@ class RootSignature {
   uint32_t NumStaticSamplers;
   uint32_t StaticSamplersOffset;
   uint32_t Flags;
+  ViewArray<dxbc::RootParameterHeader> ParametersHeaders;
+  StringRef PartData;
+
+  using param_header_iterator = ViewArray<dxbc::RootParameterHeader>::iterator;
 
 public:
-  RootSignature() {}
+  RootSignature(StringRef PD) : PartData(PD) {}
 
-  Error parse(StringRef Data);
+  Error parse();
   uint32_t getVersion() const { return Version; }
   uint32_t getNumParameters() const { return NumParameters; }
   uint32_t getRootParametersOffset() const { return RootParametersOffset; }
   uint32_t getNumStaticSamplers() const { return NumStaticSamplers; }
   uint32_t getStaticSamplersOffset() const { return StaticSamplersOffset; }
+  uint32_t getNumRootParameters() const { return ParametersHeaders.size(); }
+  llvm::iterator_range<param_header_iterator> param_headers() const {
+    return llvm::make_range(ParametersHeaders.begin(), ParametersHeaders.end());
+  }
   uint32_t getFlags() const { return Flags; }
+
+  llvm::Expected<RootParameterView>
+  getParameter(const dxbc::RootParameterHeader &Header) const {
+    size_t DataSize;
+
+    if (!dxbc::isValidParameterType(Header.ParameterType))
+      return parseFailed("invalid parameter type");
+
+    switch (static_cast<dxbc::RootParameterType>(Header.ParameterType)) {
+    case dxbc::RootParameterType::Constants32Bit:
+      DataSize = sizeof(dxbc::RootConstants);
+      break;
+    }
+    size_t EndOfSectionByte = getNumStaticSamplers() == 0
+                                  ? PartData.size()
+                                  : getStaticSamplersOffset();
+
+    if (Header.ParameterOffset + DataSize > EndOfSectionByte)
+      return parseFailed("Reading structure out of file bounds");
+
+    StringRef Buff = PartData.substr(Header.ParameterOffset, DataSize);
+    RootParameterView View = RootParameterView(Header, Buff);
+    return View;
+  }
 };
 
 class PSVRuntimeInfo {
diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
index ecad35e82b155..393bba9c79bf8 100644
--- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
@@ -74,18 +74,43 @@ struct ShaderHash {
 };
 
 #define ROOT_ELEMENT_FLAG(Num, Val) bool Val = false;
+
+struct RootConstantsYaml {
+  uint32_t ShaderRegister;
+  uint32_t RegisterSpace;
+  uint32_t Num32BitValues;
+};
+
+struct RootParameterYamlDesc {
+  uint32_t Type;
+  uint32_t Visibility;
+  uint32_t Offset;
+
+  union {
+    RootConstantsYaml Constants;
+  };
+};
+
 struct RootSignatureYamlDesc {
   RootSignatureYamlDesc() = default;
-  RootSignatureYamlDesc(const object::DirectX::RootSignature &Data);
 
   uint32_t Version;
-  uint32_t NumParameters;
+  uint32_t NumRootParameters;
   uint32_t RootParametersOffset;
   uint32_t NumStaticSamplers;
   uint32_t StaticSamplersOffset;
 
+  SmallVector<RootParameterYamlDesc> Parameters;
+
   uint32_t getEncodedFlags();
 
+  iterator_range<RootParameterYamlDesc *> params() {
+    return make_range(Parameters.begin(), Parameters.end());
+  }
+
+  static llvm::Expected<DXContainerYAML::RootSignatureYamlDesc>
+  create(const object::DirectX::RootSignature &Data);
+
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
@@ -192,6 +217,7 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DXContainerYAML::ResourceBindInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DXContainerYAML::SignatureElement)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DXContainerYAML::PSVInfo::MaskVector)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DXContainerYAML::SignatureParameter)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DXContainerYAML::RootParameterYamlDesc)
 LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::dxbc::PSV::SemanticKind)
 LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::dxbc::PSV::ComponentType)
 LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::dxbc::PSV::InterpolationMode)
@@ -264,6 +290,14 @@ template <> struct MappingTraits<DXContainerYAML::RootSignatureYamlDesc> {
                       DXContainerYAML::RootSignatureYamlDesc &RootSignature);
 };
 
+template <> struct MappingTraits<llvm::DXContainerYAML::RootParameterYamlDesc> {
+  static void mapping(IO &IO, llvm::DXContainerYAML::RootParameterYamlDesc &P);
+};
+
+template <> struct MappingTraits<llvm::DXContainerYAML::RootConstantsYaml> {
+  static void mapping(IO &IO, llvm::DXContainerYAML::RootConstantsYaml &C);
+};
+
 } // namespace yaml
 
 } // namespace llvm
diff --git a/llvm/lib/BinaryFormat/DXContainer.cpp b/llvm/lib/BinaryFormat/DXContainer.cpp
index 97ceb16ccf53f..8e7b7d313706a 100644
--- a/llvm/lib/BinaryFormat/DXContainer.cpp
+++ b/llvm/lib/BinaryFormat/DXContainer.cpp
@@ -60,6 +60,26 @@ ArrayRef<EnumEntry<SigComponentType>> dxbc::getSigComponentTypes() {
   return ArrayRef(SigComponentTypes);
 }
 
+#define SHADER_VISIBILITY(Val, Enum) {#Enum, ShaderVisibility::Enum},
+
+static const EnumEntry<ShaderVisibility> ShaderVisibilityValues[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<ShaderVisibility>> dxbc::getShaderVisibility() {
+  return ArrayRef(ShaderVisibilityValues);
+}
+
+#define ROOT_PARAMETER(Val, Enum) {#Enum, RootParameterType::Enum},
+
+static const EnumEntry<RootParameterType> RootParameterTypes[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<RootParameterType>> dxbc::getRootParameterTypes() {
+  return ArrayRef(RootParameterTypes);
+}
+
 #define SEMANTIC_KIND(Val, Enum) {#Enum, PSV::SemanticKind::Enum},
 
 static const EnumEntry<PSV::SemanticKind> SemanticKindNames[] = {
diff --git a/llvm/lib/MC/DXContainerRootSignature.cpp b/llvm/lib/MC/DXContainerRootSignature.cpp
index b6f2b85bac74e..c2731d95c955e 100644
--- a/llvm/lib/MC/DXContainerRootSignature.cpp
+++ b/llvm/lib/MC/DXContainerRootSignature.cpp
@@ -7,17 +7,81 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/DXContainerRootSignature.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/EndianStream.h"
 
 using namespace llvm;
 using namespace llvm::mcdxbc;
 
+static uint32_t writePlaceholder(raw_svector_ostream &Stream) {
+  const uint32_t DummyValue = std::numeric_limits<uint32_t>::max();
+  uint32_t Offset = Stream.tell();
+  support::endian::write(Stream, DummyValue, llvm::endianness::little);
+  return Offset;
+}
+
+static void rewriteOffsetToCurrentByte(raw_svector_ostream &Stream,
+                                       uint32_t Offset) {
+  uint32_t Value =
+      support::endian::byte_swap<uint32_t, llvm::endianness::little>(
+          Stream.tell());
+  Stream.pwrite(reinterpret_cast<const char *>(&Value), sizeof(Value), Offset);
+}
+
+size_t RootSignatureDesc::getSize() const {
+  size_t Size = sizeof(dxbc::RootSignatureHeader) +
+                Parameters.size() * sizeof(dxbc::RootParameterHeader);
+
+  for (const mcdxbc::RootParameter &P : Parameters) {
+    switch (P.Header.ParameterType) {
+    case llvm::to_underlying(dxbc::RootParameterType::Constants32Bit):
+      Size += sizeof(dxbc::RootConstants);
+      break;
+    }
+  }
+  return Size;
+}
+
 void RootSignatureDesc::write(raw_ostream &OS) const {
+  SmallString<256> Storage;
+  raw_svector_ostream BOS(Storage);
+  BOS.reserveExtraSpace(getSize());
+
+  const uint32_t NumParameters = Parameters.size();
+
+  support::endian::write(BOS, Version, llvm::endianness::little);
+  support::endian::write(BOS, NumParameters, llvm::endianness::little);
+  support::endian::write(BOS, RootParameterOffset, llvm::endianness::little);
+  support::endian::write(BOS, NumStaticSamplers, llvm::endianness::little);
+  support::endian::write(BOS, StaticSamplersOffset, llvm::endianness::little);
+  support::endian::write(BOS, Flags, llvm::endianness::little);
+
+  SmallVector<uint32_t> ParamsOffsets;
+  for (const mcdxbc::RootParameter &P : Parameters) {
+    support::endian::write(BOS, P.Header.ParameterType,
+                           llvm::endianness::little);
+    support::endian::write(BOS, P.Header.ShaderVisibility,
+                           llvm::endianness::little);
+
+    ParamsOffsets.push_back(writePlaceholder(BOS));
+  }
+
+  assert(NumParameters == ParamsOffsets.size());
+  for (size_t I = 0; I < NumParameters; ++I) {
+    rewriteOffsetToCurrentByte(BOS, ParamsOffsets[I]);
+    const mcdxbc::RootParameter &P = Parameters[I];
 
-  support::endian::write(OS, Version, llvm::endianness::little);
-  support::endian::write(OS, NumParameters, llvm::endianness::little);
-  support::endian::write(OS, RootParametersOffset, llvm::endianness::little);
-  support::endian::write(OS, NumStaticSamplers, llvm::endianness::little);
-  support::endian::write(OS, StaticSamplersOffset, llvm::endianness::little);
-  support::endian::write(OS, Flags, llvm::endianness::little);
+    switch (P.Header.ParameterType) {
+    case llvm::to_underlying(dxbc::RootParameterType::Constants32Bit):
+      support::endian::write(BOS, P.Constants.ShaderRegister,
+                             llvm::endianness::little);
+      support::endian::write(BOS, P.Constants.RegisterSpace,
+                             llvm::endianness::little);
+      support::endian::write(BOS, P.Constants.Num32BitValues,
+                             llvm::endianness::little);
+      break;
+    }
+  }
+  assert(Storage.size() == getSize());
+  OS.write(Storage.data(), Storage.size());
 }
diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp
index 1eb1453c65147..95f6788e75aa6 100644
--- a/llvm/lib/Object/DXContainer.cpp
+++ b/llvm/lib/Object/DXContainer.cpp
@@ -20,10 +20,6 @@ static Error parseFailed(const Twine &Msg) {
   return make_error<GenericBinaryError>(Msg.str(), object_error::parse_failed);
 }
 
-static Error validationFailed(const Twine &Msg) {
-  return make_error<StringError>(Msg.str(), inconvertibleErrorCode());
-}
-
 template <typename T>
 static Error readStruct(StringRef Buffer, const char *Src, T &Struct) {
   // Don't read before the beginning or past the end of the file
@@ -100,8 +96,8 @@ Error DXContainer::parseHash(StringRef Part) {
 Error DXContainer::parseRootSignature(StringRef Part) {
   if (RootSignature)
     return parseFailed("More than one RTS0 part is present in the file");
-  RootSignature = DirectX::RootSignature();
-  if (Error Err = RootSignature->parse(Part))
+  RootSignature = DirectX::RootSignature(Part);
+  if (Error Err = RootSignature->parse())
     return Err;
   return Error::success();
 }
@@ -246,23 +242,17 @@ void DXContainer::PartIterator::updateIteratorImpl(const uint32_t Offset) {
   IteratorState.Offset = Offset;
 }
 
-Error DirectX::RootSignature::parse(StringRef Data) {
-  const char *Current = Data.begin();
+Error DirectX::RootSignature::parse() {
+  const char *Current = PartData.begin();
 
   // Root Signature headers expects 6 integers to be present.
-  if (Data.size() < 6 * sizeof(uint32_t))
+  if (PartData.size() < 6 * sizeof(uint32_t))
     return parseFailed(
         "Invalid root signature, insufficient space for header.");
 
-  uint32_t VValue =
-      support::endian::read<uint32_t, llvm::endianness::little>(Current);
+  Version = support::endian::read<uint32_t, llvm::endianness::little>(Current);
   Current += sizeof(uint32_t);
 
-  if (!dxbc::RootSignatureValidations::isValidVersion(VValue))
-    return validationFailed("unsupported root signature version read: " +
-                            llvm::Twine(VValue));
-  Version = VValue;
-
   NumParameters =
       support::endian::read<uint32_t, llvm::endianness::little>(Current);
   Current += sizeof(uint32_t);
@@ -279,14 +269,11 @@ Error DirectX::RootSignature::parse(StringRef Data) {
       support::endian::read<uint32_t, llvm::endianness::little>(Current);
   Current += sizeof(uint32_t);
 
-  uint32_t FValue =
-      support::endian::read<uint32_t, llvm::endianness::little>(Current);
+  Flags = support::endian::read<uint32_t, llvm::endianness::little>(Current);
   Current += sizeof(uint32_t);
 
-  if (!dxbc::RootSignatureValidations::isValidRootFlag(FValue))
-    return validationFailed("unsupported root signature flag value read: " +
-                            llvm::Twine(FValue));
-  Flags = FValue;
+  ParametersHeaders.Data = PartData.substr(
+      RootParametersOffset, NumParameters * sizeof(dxbc::RootParameterHeader));
 
   return Error::success();
 }
diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
index f6ed09c857bb7..86e24eae4abc6 100644
--- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
@@ -269,11 +269,26 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
       mcdxbc::RootSignatureDesc RS;
       RS.Flags = P.RootSignature->getEncodedFlags();
       RS.Version = P.RootSignature->Version;
-      RS.NumParameters = P.RootSignature->NumParameters;
-      RS.RootParametersOffset = P.RootSignature->RootParametersOffset;
+      RS.RootParameterOffset = P.RootSignature->RootParametersOffset;
       RS.NumStaticSamplers = P.RootSignature->NumStaticSamplers;
       RS.StaticSamplersOffset = P.RootSignature->StaticSamplersOffset;
 
+      for (const auto &Param : P.RootSignature->Parameters) {
+        mcdxbc::RootParameter NewParam;
+        NewParam.Header = dxbc::RootParameterHeader{
+            Param.Type, Param.Visibility, Param.Offset};
+
+        switch (Param.Type) {
+        case llvm::to_underlying(dxbc::RootParameterType::Constants32Bit):
+          NewParam.Constants.Num32BitValues = Param.Constants.Num32BitValues;
+          NewParam.Constants.RegisterSpace = Param.Constants.RegisterSpace;
+          NewParam.Constants.ShaderRegister = Param.Constants.ShaderRegister;
+          break;
+        }
+
+        RS.Parameters.push_back(NewParam);
+      }
+
       RS.write(OS);
       break;
     }
diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
index f03c7da65999d..59914fe30082d 100644
--- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
@@ -12,9 +12,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ObjectYAML/DXContainerYAML.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/BinaryFormat/DXContainer.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/ScopedPrinter.h"
+#include <cstdint>
+#include <system_error>
 
 namespace llvm {
 
@@ -29,16 +33,60 @@ DXContainerYAML::ShaderFeatureFlags::ShaderFeatureFlags(uint64_t FlagData) {
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
-DXContainerYAML::RootSignatureYamlDesc::RootSignatureYamlDesc(
-    const object::DirectX::RootSignature &Data)
-    : Version(Data.getVersion()), NumParameters(Data.getNumParameters()),
-      RootParametersOffset(Data.getRootParametersOffset()),
-      NumStaticSamplers(Data.getNumStaticSamplers()),
-      StaticSamplersOffset(Data.getStaticSamplersOffset()) {
+llvm::Expected<DXContainerYAML::RootSignatureYamlDesc>
+DXContainerYAML::RootSignatureYamlDesc::create(
+    const object::DirectX::RootSignature &Data) {
+
+  RootSignatureYamlDesc RootSigDesc;
+
+  RootSigDesc.Version = Data.getVersion();
+  RootSigDesc.NumStaticSamplers = Data.getNumStaticSamplers();
+  RootSigDesc.StaticSamplersOffset = Data.getStaticSamplersOffset();
+  RootSigDesc.NumRootParameters = Data.getNumRootParameters();
+  RootSigDesc.RootParametersOffset = Data.getRootParametersOffset();
+
   uint32_t Flags = Data.getFlags();
+  for (const dxbc::RootParameterHeader &PH : Data.param_headers()) {
+
+    RootParameterYamlDesc NewP;
+    NewP.Offset = PH.ParameterOffset;
+
+    if (!dxbc::isValidParameterType(PH.ParameterType))
+      return createStringError(std::errc::invalid_argument,
+                               "Invalid value for parameter type");
+
+    NewP.Type = PH.ParameterType;
+
+    if (!dxbc::isValidShaderVisibility(PH.ShaderVisibility))
+      return createStringError(std::errc::invalid_argument,
+                               "Invalid value for shader visibility");
+
+    NewP.Visibility = PH.ShaderVisibility;
+
+    llvm::Expected<object::DirectX::RootParameterView> ParamViewOrErr =
+        Data.getParameter(PH);
+    if (Error E = ParamViewOrErr.takeError())
+      return std::move(E);
+    object::DirectX::RootParameterView ParamView = ParamViewOrErr.get();
+
+    if (auto *RCV = dyn_cast<object::DirectX::RootConstantView>(&ParamView)) {
+      llvm::Expected<dxbc::RootConstants> ConstantsOrErr = RCV->read();
+      if (Error E = ConstantsOrErr.takeError())
+        return std::move(E);
+
+      auto Constants = *ConstantsOrErr;
+
+      NewP.Constants.Num32BitValues = Constants.Num32BitValues;
+      NewP.Constants.ShaderRegister = Constants.ShaderRegister;
+      NewP.Constants.RegisterSpace = Constants.RegisterSpace;
+    }
+    RootSigDesc.Parameters.push_back(NewP);
+  }
 #define ROOT_ELEMENT_FLAG(Num, Val)                                            \
-  Val = (Flags & (uint32_t)dxbc::RootElementFlag::Val) > 0;
+  RootSigDesc.Val =                                                            \
+      (Flags & llvm::to_underlying(dxbc::RootElementFlag::Val)) > 0;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
+  return RootSigDesc;
 }
 
 uint32_t DXContainerYAML::RootSignatureYamlDesc::getEncodedFlags() {
@@ -212,14 +260,34 @@ void MappingTraits<DXContainerYAML::Signature>::mapping(
 void MappingTraits<DXContainerYAML::RootSignatureYamlDesc>::mapping(
     IO &IO, DXContainerYAML::RootSignatureYamlDesc &S) {
   IO.mapRequired("Version", S.Version);
-  IO.mapRequired("NumParameters", S.NumParameters);
+  IO.mapRequired("NumRootParameters", S.NumRootParameters);
   IO.mapRequired("RootParametersOffset", S.RootParametersOffset);
   IO.mapRequired("NumStaticSamplers", S.NumStaticSamplers);
   IO.mapRequired("StaticSamplersOffset", S.StaticSamplersOffset);
+  IO.mapRequired("Parameters", S.Parameters);
 #define ROOT_ELEMENT_FLAG(Num, Val) IO.mapOptional(#Val, S.Val, false);
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
+void MappingTraits<llvm::DXContainerYAML::RootConstantsYaml>::mapping(
+    IO &IO, llvm::DXContainerYAML::RootConstantsYaml &C) {
+  IO.mapRequired("Num32BitValues", C.Num32BitValues);
+  IO.mapRequired("RegisterSpace", C.RegisterSpace);
+  IO.mapRequired("ShaderRegister", C.ShaderRegister);
+}
+
+void MappingTraits<llvm::DXContainerYAML::RootParameterYamlDesc>::mapping(
+    IO &IO, llvm::DXContainerYAML::RootParameterYamlDesc &P) {
+  IO.mapRequired("ParameterType", P.Type);
+  IO.mapRequired("ShaderVisibility", P.Visibility);
+
+  switch (P.Type) {
+  case llvm::to_underlying(dxbc::RootParameterType::Constants32Bit):
+    IO.mapRequired("Constants", P.Constants);
+    break;
+  }
+}
+
 void MappingTraits<DXContainerYAML::Part>::mapping(IO &IO,
                                                    DXContainerYAML::Part &P) {
   IO.mapRequired("Name", P.Name);
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.cpp b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
index fd390cdbf9057..3ba0535e0114b 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.cpp
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
@@ -92,8 +92,10 @@ static bool parse(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD,
   return HasError;
 }
 
+static bool verifyRootFlag(uint32_t Flags) { return (Flags & ~0xfff) == 0; }
+
 static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
-  if (!dxbc::RootSignatureValidations::isValidRootFlag(RSD.Flags)) {
+  if (!verifyRootFlag(RSD.Flags)) {
     return reportError(Ctx, "Invalid Root Signature flag value");
   }
   return false;
@@ -189,6 +191,8 @@ PreservedAnalyses RootSignatureAnalysisPrinter::run(Module &M,
 
   SmallDenseMap<const Function *, mcdxbc::RootSignatureDesc> &RSDMap =
       AM.getResult<RootSignatureAnalysis>(M);
+
+  const size_t RSHSize = sizeof(dxbc::RootSignatureHeader);
   OS << "Root Signature Definitions"
      << "\n";
   uint8_t Space = 0;
@@ -203,12 +207,11 @@ PreservedAnalyses RootSignatureAnalysisPrinter::run(Module &M,
     Space++;
     OS << indent(Space) << "Flags: " << format_hex(RS.Flags, 8) << ":\n";
     OS << indent(Space) << "Version: " << RS.Version << ":\n";
-    OS << indent(Space) << "NumParameters: " << RS.NumParameters << ":\n";
-    OS << indent(Space) << "RootParametersOffset: " << RS.RootParametersOffset
-       << ":\n";
-    OS << indent(Space) << "NumStaticSamplers: " << RS.NumStaticSamplers
-       << ":\n";
-    OS << indent(Space) << "StaticSamplersOffset: " << RS.StaticSamplersOffset
+    OS << indent(Space) << "NumParameters: " << RS.Parameters.size() << ":\n";
+    OS << indent(Space) << "RootParametersOffset: " << RSHSize << ":\n";
+    OS << indent(Space) << "NumStaticSamplers: " << 0 << ":\n";
+    OS << indent(Space)
+       << "StaticSamplersOffset: " << RSHSize + RS.Parameters.size_in_bytes()
        << ":\n";
     Space--;
     // end root signature header
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags.ll
index 3f5bb166ad0e5..ef2b97860bfae 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags.ll
@@ -22,8 +22,9 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 ; DXC-NEXT:    Size:            24
 ; DXC-NEXT:    RootSignature:
 ; DXC-NEXT:      Version:         2
-; DXC-NEXT:      NumParameters:   0
+; DXC-NEXT:      NumRootParameters: 0
 ; DXC-NEXT:      RootParametersOffset: 0
 ; DXC-NEXT:      NumStaticSamplers: 0
 ; DXC-NEXT:      StaticSamplersOffset: 0
+; DXC-NEXT:      Parameters: []
 ; DXC-NEXT:      AllowInputAssemblerInputLayout: true
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-MultipleEntryFunctions.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-MultipleEntryFunctions.ll
index 7adb17d0b022f..581ac9aaec110 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-MultipleEntryFunctions.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-MultipleEntryFunctions.ll
@@ -27,14 +27,14 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 ; CHECK-NEXT:   Flags: 0x000001
 ; CHECK-NEXT:   Version: 2
 ; CHECK-NEXT:   NumParameters: 0
-; CHECK-NEXT:   RootParametersOffset: 0
+; CHECK-NEXT:   RootParametersOffset: 24
 ; CHECK-NEXT:   NumStaticSamplers: 0
-; CHECK-NEXT:   StaticSamplersOffset: 0
+; CHECK-NEXT:   StaticSamplersOffset: 24
 
 ; CHECK-LABEL: Definition for 'anotherMain':
 ; CHECK-NEXT:   Flags: 0x000002
 ; CHECK-NEXT:   Version: 2
 ; CHECK-NEXT:   NumParameters: 0
-; CHECK-NEXT:   RootParametersOffset: 0
+; CHECK-NEXT:   RootParametersOffset: 24
 ; CHECK-NEXT:   NumStaticSamplers: 0
-; CHECK-NEXT:   StaticSamplersOffset: 0
+; CHECK-NEXT:   StaticSamplersOffset: 24
diff --git a/llvm/test/ObjectYAML/DXContainer/RootSignature-Flags.yaml b/llvm/test/ObjectYAML/DXContainer/RootSignature-Flags.yaml
index b0a3e6945f454..74816d403183a 100644
--- a/llvm/test/ObjectYAML/DXContainer/RootSignature-Flags.yaml
+++ b/llvm/test/ObjectYAML/DXContainer/RootSignature-Flags.yaml
@@ -14,10 +14,11 @@ Parts:
     Size:            24
     RootSignature:
       Version: 2
-      NumParameters: 1
-      RootParametersOffset: 3
-      NumStaticSamplers: 4
-      StaticSamplersOffset: 5
+      NumRootParameters: 0
+      RootParametersOffset: 24
+      NumStaticSamplers: 0
+      StaticSamplersOffset: 60
+      Parameters: []
       AllowInputAssemblerInputLayout: true
       DenyGeometryShaderRootAccess: true
 
@@ -25,9 +26,10 @@ Parts:
 # CHECK-NEXT:    Size:            24
 # CHECK-NEXT:    RootSignature:
 # CHECK-NEXT:      Version: 2
-# CHECK-NEXT:      NumParameters: 1
-# CHECK-NEXT:      RootParametersOffset: 3
-# CHECK-NEXT:      NumStaticSamplers: 4
-# CHECK-NEXT:      StaticSamplersOffset: 5
+# CHECK-NEXT:      NumRootParameters: 0
+# CHECK-NEXT:      RootParametersOffset: 24
+# CHECK-NEXT:      NumStaticSamplers: 0
+# CHECK-NEXT:      StaticSamplersOffset: 60
+# CHECK-NEXT:      Parameters: []
 # CHECK-NEXT:      AllowInputAssemblerInputLayout: true
 # CHECK-NEXT:      DenyGeometryShaderRootAccess: true
diff --git a/llvm/test/ObjectYAML/DXContainer/RootSignature-InvalidType.yaml b/llvm/test/ObjectYAML/DXContainer/RootSignature-InvalidType.yaml
new file mode 100644
index 0000000000000..091e70789d956
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/RootSignature-InvalidType.yaml
@@ -0,0 +1,29 @@
+# RUN: yaml2obj %s -o %t
+# RUN: not obj2yaml 2>&1 %t | FileCheck %s -DFILE=%t
+
+# CHECK: Error reading file: [[FILE]]: Invalid value for parameter type
+
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       1
+  PartOffsets:     [ 60 ]
+Parts:
+  - Name:            RTS0
+    Size:            80
+    RootSignature:
+      Version: 2
+      NumRootParameters: 2
+      RootParametersOffset: 24
+      NumStaticSamplers: 0
+      StaticSamplersOffset: 64
+      Parameters:         
+      - ParameterType: 255 # INVALID
+        ShaderVisibility: 2 # Hull
+      AllowInputAssemblerInputLayout: true
+      DenyGeometryShaderRootAccess: true
diff --git a/llvm/test/ObjectYAML/DXContainer/RootSignature-InvalidVisibility.yaml b/llvm/test/ObjectYAML/DXContainer/RootSignature-InvalidVisibility.yaml
new file mode 100644
index 0000000000000..1acaf6e4e08a4
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/RootSignature-InvalidVisibility.yaml
@@ -0,0 +1,33 @@
+# RUN: yaml2obj %s -o %t
+# RUN: not obj2yaml 2>&1 %t | FileCheck %s -DFILE=%t
+
+# CHECK: Error reading file: [[FILE]]: Invalid value for shader visibility
+
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       1
+  PartOffsets:     [ 60 ]
+Parts:
+  - Name:            RTS0
+    Size:            80
+    RootSignature:
+      Version: 2
+      NumRootParameters: 2
+      RootParametersOffset: 24
+      NumStaticSamplers: 0
+      StaticSamplersOffset: 64
+      Parameters:         
+      - ParameterType: 1 # Constants32Bit
+        ShaderVisibility: 255 # INVALID
+        Constants:
+          Num32BitValues: 21
+          ShaderRegister: 22
+          RegisterSpace: 23   
+      AllowInputAssemblerInputLayout: true
+      DenyGeometryShaderRootAccess: true
diff --git a/llvm/test/ObjectYAML/DXContainer/RootSignature-MultipleParameters.yaml b/llvm/test/ObjectYAML/DXContainer/RootSignature-MultipleParameters.yaml
new file mode 100644
index 0000000000000..f366d71714359
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/RootSignature-MultipleParameters.yaml
@@ -0,0 +1,59 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       1
+  PartOffsets:     [ 60 ]
+Parts:
+  - Name:            RTS0
+    Size:            80
+    RootSignature:
+      Version: 2
+      NumRootParameters: 2
+      RootParametersOffset: 24
+      NumStaticSamplers: 0
+      StaticSamplersOffset: 60
+      Parameters:         
+      - ParameterType: 1 # Constants32Bit
+        ShaderVisibility: 2 # Hull
+        Constants:
+          Num32BitValues: 16
+          ShaderRegister: 15
+          RegisterSpace: 14
+      - ParameterType: 1 # Constants32Bit
+        ShaderVisibility: 4 # Geometry
+        Constants:
+          Num32BitValues: 21
+          ShaderRegister: 22
+          RegisterSpace: 23     
+      AllowInputAssemblerInputLayout: true
+      DenyGeometryShaderRootAccess: true
+
+# CHECK:  - Name:            RTS0
+# CHECK-NEXT:    Size:            80
+# CHECK-NEXT:    RootSignature:
+# CHECK-NEXT:      Version:         2
+# CHECK-NEXT:      NumRootParameters: 2
+# CHECK-NEXT:      RootParametersOffset: 24
+# CHECK-NEXT:      NumStaticSamplers: 0
+# CHECK-NEXT:      StaticSamplersOffset: 60
+# CHECK-NEXT:      Parameters:
+# CHECK-NEXT:        - ParameterType:   1
+# CHECK-NEXT:          ShaderVisibility: 2
+# CHECK-NEXT:          Constants:
+# CHECK-NEXT:            Num32BitValues:  16
+# CHECK-NEXT:            RegisterSpace:   14
+# CHECK-NEXT:            ShaderRegister:  15
+# CHECK-NEXT:        - ParameterType:   1
+# CHECK-NEXT:          ShaderVisibility: 4
+# CHECK-NEXT:          Constants:
+# CHECK-NEXT:            Num32BitValues:  21
+# CHECK-NEXT:            RegisterSpace:   23
+# CHECK-NEXT:            ShaderRegister:  22
+# CHECK-NEXT:      AllowInputAssemblerInputLayout: true
+# CHECK-NEXT:      DenyGeometryShaderRootAccess: true
diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
index f3ef1b6a27bcf..c727595406767 100644
--- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
+++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
@@ -155,8 +155,13 @@ dumpDXContainer(MemoryBufferRef Source) {
       break;
     case dxbc::PartType::RTS0:
       std::optional<DirectX::RootSignature> RS = Container.getRootSignature();
-      if (RS.has_value())
-        NewPart.RootSignature = DXContainerYAML::RootSignatureYamlDesc(*RS);
+      if (RS.has_value()) {
+        auto RootSigDescOrErr =
+            DXContainerYAML::RootSignatureYamlDesc::create(*RS);
+        if (Error E = RootSigDescOrErr.takeError())
+          return std::move(E);
+        NewPart.RootSignature = RootSigDescOrErr.get();
+      }
       break;
     }
   }
diff --git a/llvm/unittests/Object/DXContainerTest.cpp b/llvm/unittests/Object/DXContainerTest.cpp
index 5a73f32ab7c32..62ef8e385373f 100644
--- a/llvm/unittests/Object/DXContainerTest.cpp
+++ b/llvm/unittests/Object/DXContainerTest.cpp
@@ -11,6 +11,7 @@
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/ObjectYAML/DXContainerYAML.h"
 #include "llvm/ObjectYAML/yaml2obj.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
@@ -822,6 +823,57 @@ TEST(DXCFile, MalformedSignature) {
   }
 }
 
+TEST(RootSignature, RootParameters) {
+  {
+    // Root Parameters offset has changed to 36, additional padding was added,
+    // as well as fixing the parameter offset.
+    uint8_t Buffer[] = {
+        0x44, 0x58, 0x42, 0x43, 0x32, 0x9a, 0x53, 0xd8, 0xec, 0xbe, 0x35, 0x6f,
+        0x05, 0x39, 0xe1, 0xfe, 0x31, 0x20, 0xf0, 0xc1, 0x01, 0x00, 0x00, 0x00,
+        0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+        0x52, 0x54, 0x53, 0x30, 0x59, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+        0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x2c, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+        0x02, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00,
+        0x0e, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00};
+
+    DXContainer C =
+        llvm::cantFail(DXContainer::create(getMemoryBuffer<144>(Buffer)));
+
+    auto MaybeRS = C.getRootSignature();
+    ASSERT_TRUE(MaybeRS.has_value());
+    const auto &RS = MaybeRS.value();
+    ASSERT_EQ(RS.getVersion(), 2u);
+    ASSERT_EQ(RS.getNumParameters(), 1u);
+    ASSERT_EQ(RS.getRootParametersOffset(), 36u);
+    ASSERT_EQ(RS.getNumStaticSamplers(), 0u);
+    ASSERT_EQ(RS.getStaticSamplersOffset(), 44u);
+    ASSERT_EQ(RS.getFlags(), 17u);
+
+    auto RootParam = *RS.param_headers().begin();
+    ASSERT_EQ((unsigned)RootParam.ParameterType, 1u);
+    ASSERT_EQ((unsigned)RootParam.ShaderVisibility, 2u);
+    auto ParamView = RS.getParameter(RootParam);
+    ASSERT_THAT_ERROR(ParamView.takeError(), Succeeded());
+
+    DirectX::RootConstantView *RootConstantsView =
+        dyn_cast<DirectX::RootConstantView>(&*ParamView);
+    ASSERT_TRUE(RootConstantsView != nullptr);
+    auto Constants = RootConstantsView->read();
+
+    ASSERT_THAT_ERROR(Constants.takeError(), Succeeded());
+
+    ASSERT_EQ(Constants->ShaderRegister, 15u);
+    ASSERT_EQ(Constants->RegisterSpace, 14u);
+    ASSERT_EQ(Constants->Num32BitValues, 16u);
+  }
+}
+
 TEST(RootSignature, ParseRootFlags) {
   {
     uint8_t Buffer[] = {
@@ -829,7 +881,7 @@ TEST(RootSignature, ParseRootFlags) {
         0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00, 0x00, 0x00,
         0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
         0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
         0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
     };
     DXContainer C =
@@ -839,12 +891,11 @@ TEST(RootSignature, ParseRootFlags) {
     ASSERT_TRUE(RS.has_value());
     ASSERT_EQ(RS->getVersion(), 2u);
     ASSERT_EQ(RS->getNumParameters(), 0u);
-    ASSERT_EQ(RS->getRootParametersOffset(), 0u);
+    ASSERT_EQ(RS->getRootParametersOffset(), 24u);
     ASSERT_EQ(RS->getNumStaticSamplers(), 0u);
     ASSERT_EQ(RS->getStaticSamplersOffset(), 0u);
     ASSERT_EQ(RS->getFlags(), 0x01u);
   }
-
   {
     // this parameter has the root signature definition missing some values.
     uint8_t Buffer[] = {
@@ -860,33 +911,51 @@ TEST(RootSignature, ParseRootFlags) {
         FailedWithMessage(
             "Invalid root signature, insufficient space for header."));
   }
+}
+
+TEST(RootSignature, ParseRootConstant) {
   {
-    // Version has been changed to an invalid number.
     uint8_t Buffer[] = {
-        0x44, 0x58, 0x42, 0x43, 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F,
-        0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00, 0x00, 0x00,
-        0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
-        0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00,
+        0x44, 0x58, 0x42, 0x43, 0x32, 0x9a, 0x53, 0xd8, 0xec, 0xbe, 0x35, 0x6f,
+        0x05, 0x39, 0xe1, 0xfe, 0x31, 0x20, 0xf0, 0xc1, 0x01, 0x00, 0x00, 0x00,
+        0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+        0x52, 0x54, 0x53, 0x30, 0x59, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+        0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x2c, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+        0x02, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00,
+        0x0e, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
-    };
-    EXPECT_THAT_EXPECTED(
-        DXContainer::create(getMemoryBuffer<100>(Buffer)),
-        FailedWithMessage("unsupported root signature version read: 3"));
-  }
-  {
-    // Flag has been set to an invalid value
-    uint8_t Buffer[] = {
-        0x44, 0x58, 0x42, 0x43, 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F,
-        0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00, 0x00, 0x00,
-        0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
-        0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xFF,
-    };
-    EXPECT_THAT_EXPECTED(
-        DXContainer::create(getMemoryBuffer<100>(Buffer)),
-        FailedWithMessage(
-            "unsupported root signature flag value read: 4278190081"));
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00};
+    DXContainer C =
+        llvm::cantFail(DXContainer::create(getMemoryBuffer<133>(Buffer)));
+
+    auto MaybeRS = C.getRootSignature();
+    ASSERT_TRUE(MaybeRS.has_value());
+    const auto &RS = MaybeRS.value();
+    ASSERT_EQ(RS.getVersion(), 2u);
+    ASSERT_EQ(RS.getNumParameters(), 1u);
+    ASSERT_EQ(RS.getRootParametersOffset(), 24u);
+    ASSERT_EQ(RS.getNumStaticSamplers(), 0u);
+    ASSERT_EQ(RS.getStaticSamplersOffset(), 44u);
+    ASSERT_EQ(RS.getFlags(), 17u);
+
+    auto RootParam = *RS.param_headers().begin();
+    ASSERT_EQ((unsigned)RootParam.ParameterType, 1u);
+    ASSERT_EQ((unsigned)RootParam.ShaderVisibility, 2u);
+    auto ParamView = RS.getParameter(RootParam);
+    ASSERT_THAT_ERROR(ParamView.takeError(), Succeeded());
+
+    DirectX::RootConstantView *RootConstantsView =
+        dyn_cast<DirectX::RootConstantView>(&*ParamView);
+    ASSERT_TRUE(RootConstantsView != nullptr);
+    auto Constants = RootConstantsView->read();
+
+    ASSERT_THAT_ERROR(Constants.takeError(), Succeeded());
+
+    ASSERT_EQ(Constants->ShaderRegister, 15u);
+    ASSERT_EQ(Constants->RegisterSpace, 14u);
+    ASSERT_EQ(Constants->Num32BitValues, 16u);
   }
 }
diff --git a/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp b/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
index b48cd9ce53987..61390049bc0df 100644
--- a/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
+++ b/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
@@ -127,10 +127,11 @@ TEST(RootSignature, ParseRootFlags) {
       Size:            24
       RootSignature:
         Version:         2
-        NumParameters:   0
-        RootParametersOffset: 0
+        NumRootParameters: 0
+        RootParametersOffset: 24
         NumStaticSamplers: 0
         StaticSamplersOffset: 0
+        Parameters: []
         AllowInputAssemblerInputLayout: true
     )"));
 
@@ -139,10 +140,114 @@ TEST(RootSignature, ParseRootFlags) {
       0x05, 0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1, 0x01, 0x00, 0x00, 0x00,
       0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
       0x52, 0x54, 0x53, 0x30, 0x18, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
-      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
       0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
   };
 
   EXPECT_EQ(Storage.size(), 68u);
   EXPECT_TRUE(memcmp(Buffer, Storage.data(), 68u) == 0);
 }
+
+TEST(RootSignature, HeaderData) {
+  SmallString<128> Storage;
+
+  // First read a fully explicit yaml with all sizes and offsets provided
+  ASSERT_TRUE(convert(Storage, R"(--- !dxcontainer
+  Header:
+      Hash:            [ 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F, 0x5, 
+                        0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1 ]
+      Version:
+        Major:           1
+        Minor:           0
+      FileSize:        133
+      PartCount:       1
+      PartOffsets:     [ 36 ]
+  Parts:
+    - Name:            RTS0
+      Size:            89
+      RootSignature:
+        Version: 2
+        NumRootParameters: 1
+        RootParametersOffset: 255
+        NumStaticSamplers: 0
+        StaticSamplersOffset: 0
+        Parameters:
+          - ParameterType: 1
+            ShaderVisibility: 2
+            Constants:
+              Num32BitValues: 16
+              ShaderRegister: 15
+              RegisterSpace: 14
+        AllowInputAssemblerInputLayout: true
+        DenyGeometryShaderRootAccess: true
+    )"));
+
+  uint8_t Buffer[] = {
+      0x44, 0x58, 0x42, 0x43, 0x32, 0x9a, 0x53, 0xd8, 0xec, 0xbe, 0x35, 0x6f,
+      0x05, 0x39, 0xe1, 0xfe, 0x31, 0x20, 0xf0, 0xc1, 0x01, 0x00, 0x00, 0x00,
+      0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+      0x52, 0x54, 0x53, 0x30, 0x59, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+      0x01, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+      0x02, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00,
+      0x0e, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00};
+
+  EXPECT_EQ(Storage.size(), 133u);
+  EXPECT_TRUE(memcmp(Buffer, Storage.data(), 133u) == 0);
+}
+
+TEST(RootSignature, ParseRootConstants) {
+  SmallString<128> Storage;
+
+  // First read a fully explicit yaml with all sizes and offsets provided
+  ASSERT_TRUE(convert(Storage, R"(--- !dxcontainer
+  Header:
+      Hash:            [ 0x32, 0x9A, 0x53, 0xD8, 0xEC, 0xBE, 0x35, 0x6F, 0x5, 
+                        0x39, 0xE1, 0xFE, 0x31, 0x20, 0xF0, 0xC1 ]
+      Version:
+        Major:           1
+        Minor:           0
+      FileSize:        133
+      PartCount:       1
+      PartOffsets:     [ 36 ]
+  Parts:
+    - Name:            RTS0
+      Size:            89
+      RootSignature:
+        Version: 2
+        NumRootParameters: 1
+        RootParametersOffset: 36
+        NumStaticSamplers: 0
+        StaticSamplersOffset: 0
+        Parameters:
+          - ParameterType: 1
+            ShaderVisibility: 2
+            Constants:
+              Num32BitValues: 16
+              ShaderRegister: 15
+              RegisterSpace: 14
+        AllowInputAssemblerInputLayout: true
+        DenyGeometryShaderRootAccess: true
+    )"));
+
+  uint8_t Buffer[] = {
+      0x44, 0x58, 0x42, 0x43, 0x32, 0x9a, 0x53, 0xd8, 0xec, 0xbe, 0x35, 0x6f,
+      0x05, 0x39, 0xe1, 0xfe, 0x31, 0x20, 0xf0, 0xc1, 0x01, 0x00, 0x00, 0x00,
+      0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+      0x52, 0x54, 0x53, 0x30, 0x59, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+      0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+      0x02, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00,
+      0x0e, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00};
+
+  EXPECT_EQ(Storage.size(), 133u);
+  EXPECT_TRUE(memcmp(Buffer, Storage.data(), 133u) == 0);
+}

From 9bd0c8726a5e3fd4f76e84692bd920dfca7a8d7f Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Wed, 16 Apr 2025 18:48:10 -0700
Subject: [PATCH 202/710] [clang-format] Fix a bug in BWACS_MultiLine (#135906)

Fix #51940
---
 clang/lib/Format/TokenAnnotator.cpp         | 15 ++++++-
 clang/lib/Format/UnwrappedLineFormatter.cpp | 45 ++++-----------------
 clang/lib/Format/UnwrappedLineParser.cpp    | 11 ++++-
 clang/unittests/Format/FormatTest.cpp       | 11 +++++
 4 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index ef5f07e2c62ee..144983f675828 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -4153,8 +4153,18 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const {
                              ChildSize + Current->SpacesRequiredBefore;
     }
 
-    if (Current->is(TT_CtorInitializerColon))
+    if (Current->is(TT_ControlStatementLBrace)) {
+      if (Style.ColumnLimit > 0 &&
+          Style.BraceWrapping.AfterControlStatement ==
+              FormatStyle::BWACS_MultiLine &&
+          Line.Level * Style.IndentWidth + Line.Last->TotalLength >
+              Style.ColumnLimit) {
+        Current->CanBreakBefore = true;
+        Current->MustBreakBefore = true;
+      }
+    } else if (Current->is(TT_CtorInitializerColon)) {
       InFunctionDecl = false;
+    }
 
     // FIXME: Only calculate this if CanBreakBefore is true once static
     // initializers etc. are sorted out.
@@ -5586,12 +5596,13 @@ static bool isAllmanLambdaBrace(const FormatToken &Tok) {
 
 bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
                                      const FormatToken &Right) const {
-  const FormatToken &Left = *Right.Previous;
   if (Right.NewlinesBefore > 1 && Style.MaxEmptyLinesToKeep > 0 &&
       (!Style.RemoveEmptyLinesInUnwrappedLines || &Right == Line.First)) {
     return true;
   }
 
+  const FormatToken &Left = *Right.Previous;
+
   if (Style.BreakFunctionDefinitionParameters && Line.MightBeFunctionDecl &&
       Line.mightBeFunctionDefinition() && Left.MightBeFunctionDeclParen &&
       Left.ParameterCount > 0) {
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 617d46ad281d5..6806ab18312ea 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -424,43 +424,14 @@ class LineJoiner {
                  : 0;
     }
     // Try to merge a control statement block with left brace wrapped.
-    if (NextLine.First->is(tok::l_brace)) {
-      if ((TheLine->First->isOneOf(tok::kw_if, tok::kw_else, tok::kw_while,
-                                   tok::kw_for, tok::kw_switch, tok::kw_try,
-                                   tok::kw_do, TT_ForEachMacro) ||
-           (TheLine->First->is(tok::r_brace) && TheLine->First->Next &&
-            TheLine->First->Next->isOneOf(tok::kw_else, tok::kw_catch))) &&
-          Style.BraceWrapping.AfterControlStatement ==
-              FormatStyle::BWACS_MultiLine) {
-        // If possible, merge the next line's wrapped left brace with the
-        // current line. Otherwise, leave it on the next line, as this is a
-        // multi-line control statement.
-        return (Style.ColumnLimit == 0 || TheLine->Level * Style.IndentWidth +
-                                                  TheLine->Last->TotalLength <=
-                                              Style.ColumnLimit)
-                   ? 1
-                   : 0;
-      }
-      if (TheLine->First->isOneOf(tok::kw_if, tok::kw_else, tok::kw_while,
-                                  tok::kw_for, TT_ForEachMacro)) {
-        return (Style.BraceWrapping.AfterControlStatement ==
-                FormatStyle::BWACS_Always)
-                   ? tryMergeSimpleBlock(I, E, Limit)
-                   : 0;
-      }
-      if (TheLine->First->isOneOf(tok::kw_else, tok::kw_catch) &&
-          Style.BraceWrapping.AfterControlStatement ==
-              FormatStyle::BWACS_MultiLine) {
-        // This case if different from the upper BWACS_MultiLine processing
-        // in that a preceding r_brace is not on the same line as else/catch
-        // most likely because of BeforeElse/BeforeCatch set to true.
-        // If the line length doesn't fit ColumnLimit, leave l_brace on the
-        // next line to respect the BWACS_MultiLine.
-        return (Style.ColumnLimit == 0 ||
-                TheLine->Last->TotalLength <= Style.ColumnLimit)
-                   ? 1
-                   : 0;
-      }
+    if (NextLine.First->is(TT_ControlStatementLBrace)) {
+      // If possible, merge the next line's wrapped left brace with the
+      // current line. Otherwise, leave it on the next line, as this is a
+      // multi-line control statement.
+      return Style.BraceWrapping.AfterControlStatement ==
+                     FormatStyle::BWACS_Always
+                 ? tryMergeSimpleBlock(I, E, Limit)
+                 : 0;
     }
     if (PreviousLine && TheLine->First->is(tok::l_brace)) {
       switch (PreviousLine->First->Tok.getKind()) {
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 5fe65cb9a47e7..b9430d4389feb 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -135,7 +135,8 @@ class CompoundStatementIndenter {
   CompoundStatementIndenter(UnwrappedLineParser *Parser,
                             const FormatStyle &Style, unsigned &LineLevel)
       : CompoundStatementIndenter(Parser, LineLevel,
-                                  Style.BraceWrapping.AfterControlStatement,
+                                  Style.BraceWrapping.AfterControlStatement ==
+                                      FormatStyle::BWACS_Always,
                                   Style.BraceWrapping.IndentBraces) {}
   CompoundStatementIndenter(UnwrappedLineParser *Parser, unsigned &LineLevel,
                             bool WrapBrace, bool IndentBrace)
@@ -3067,7 +3068,7 @@ void UnwrappedLineParser::parseTryCatch() {
     parseStructuralElement();
     --Line->Level;
   }
-  while (true) {
+  for (bool SeenCatch = false;;) {
     if (FormatTok->is(tok::at))
       nextToken();
     if (!(FormatTok->isOneOf(tok::kw_catch, Keywords.kw___except,
@@ -3077,6 +3078,8 @@ void UnwrappedLineParser::parseTryCatch() {
            FormatTok->is(Keywords.kw_finally)))) {
       break;
     }
+    if (FormatTok->is(tok::kw_catch))
+      SeenCatch = true;
     nextToken();
     while (FormatTok->isNot(tok::l_brace)) {
       if (FormatTok->is(tok::l_paren)) {
@@ -3090,6 +3093,10 @@ void UnwrappedLineParser::parseTryCatch() {
       }
       nextToken();
     }
+    if (SeenCatch) {
+      FormatTok->setFinalizedType(TT_ControlStatementLBrace);
+      SeenCatch = false;
+    }
     NeedsUnwrappedLine = false;
     Line->MustBeDeclaration = false;
     CompoundStatementIndenter Indenter(this, Style, Line->Level);
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index b62d49e17c83f..49284c7f51e27 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -3419,6 +3419,17 @@ TEST_F(FormatTest, MultiLineControlStatements) {
                "{\n"
                "};",
                Style);
+
+  Style = getLLVMStyle();
+  Style.AllowShortBlocksOnASingleLine = FormatStyle::SBS_Always;
+  Style.AllowShortIfStatementsOnASingleLine = FormatStyle::SIS_WithoutElse;
+  Style.AllowShortLoopsOnASingleLine = true;
+  Style.BreakBeforeBraces = FormatStyle::BS_Custom;
+  Style.BraceWrapping.AfterControlStatement = FormatStyle::BWACS_MultiLine;
+  verifyFormat("if (true) { return; }", Style);
+  verifyFormat("while (true) { return; }", Style);
+  // Failing test in https://reviews.llvm.org/D114521#3151727
+  verifyFormat("for (;;) { bar(); }", Style);
 }
 
 TEST_F(FormatTest, BeforeWhile) {

From de528d689f3a2305c051528993fe30295f2a0cd9 Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Thu, 17 Apr 2025 10:08:50 +0800
Subject: [PATCH 203/710] [Clang] Handle default template arguments for alias
 CTAD guides (#134807)

It's possible that some deduced template arguments come from default
arguments, not just from the return type. So we need to recursively
visit the default arguments of the parameter if it's referenced, thereby
the template parameter referenced by the defualt arguments could come
along to the synthesized deduction guide.

Fixes https://github.com/llvm/llvm-project/issues/134471
---
 clang/docs/ReleaseNotes.rst                   |   1 +
 clang/lib/Sema/SemaTemplateDeductionGuide.cpp |  20 +++
 clang/test/SemaTemplate/deduction-guide.cpp   | 117 +++++++++++++++++-
 3 files changed, 137 insertions(+), 1 deletion(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 07ff1251fc1ad..0bf5e07ec5dfd 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -471,6 +471,7 @@ Bug Fixes to C++ Support
 - Clang no longer crashes when trying to unify the types of arrays with
   certain differences in qualifiers (this could happen during template argument
   deduction or when building a ternary operator). (#GH97005)
+- Fixed type alias CTAD issues involving default template arguments. (#GH134471)
 - The initialization kind of elements of structured bindings
   direct-list-initialized from an array is corrected to direct-initialization.
 - Clang no longer crashes when a coroutine is declared ``[[noreturn]]``. (#GH127327)
diff --git a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
index b4863cefc3fb4..29c5736a9bf9e 100644
--- a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
+++ b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
@@ -690,6 +690,26 @@ SmallVector<unsigned> TemplateParamsReferencedInTemplateArgumentList(
   SemaRef.MarkUsedTemplateParameters(
       DeducedArgs, TemplateParamsList->getDepth(), ReferencedTemplateParams);
 
+  auto MarkDefaultArgs = [&](auto *Param) {
+    if (!Param->hasDefaultArgument())
+      return;
+    SemaRef.MarkUsedTemplateParameters(
+        Param->getDefaultArgument().getArgument(),
+        TemplateParamsList->getDepth(), ReferencedTemplateParams);
+  };
+
+  for (unsigned Index = 0; Index < TemplateParamsList->size(); ++Index) {
+    if (!ReferencedTemplateParams[Index])
+      continue;
+    auto *Param = TemplateParamsList->getParam(Index);
+    if (auto *TTPD = dyn_cast<TemplateTypeParmDecl>(Param))
+      MarkDefaultArgs(TTPD);
+    else if (auto *NTTPD = dyn_cast<NonTypeTemplateParmDecl>(Param))
+      MarkDefaultArgs(NTTPD);
+    else
+      MarkDefaultArgs(cast<TemplateTemplateParmDecl>(Param));
+  }
+
   SmallVector<unsigned> Results;
   for (unsigned Index = 0; Index < TemplateParamsList->size(); ++Index) {
     if (ReferencedTemplateParams[Index])
diff --git a/clang/test/SemaTemplate/deduction-guide.cpp b/clang/test/SemaTemplate/deduction-guide.cpp
index 2888453a12f5b..dabd0cf12f77e 100644
--- a/clang/test/SemaTemplate/deduction-guide.cpp
+++ b/clang/test/SemaTemplate/deduction-guide.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2a -verify -ast-dump -ast-dump-decl-types -ast-dump-filter "deduction guide" %s | FileCheck %s --strict-whitespace
+// RUN: %clang_cc1 -std=c++2a -verify -ast-dump -ast-dump-decl-types -ast-dump-filter "deduction guide" %s | FileCheck %s --strict-whitespace -dump-input=always
 
 template<auto ...> struct X {};
 template<template<typename X, X> typename> struct Y {};
@@ -771,3 +771,118 @@ D d(24);
 // CHECK-NEXT:  `-ParmVarDecl {{.+}} 'U'
 
 } // namespace GH132616_DeductionGuide
+
+namespace GH133132 {
+
+template <class _Ty>
+struct A {};
+
+template <class T = int, class U = T>
+using AA = A<U>;
+
+AA a{};
+
+// CHECK-LABEL: Dumping GH133132::<deduction guide for AA>:
+// CHECK-NEXT:  FunctionTemplateDecl {{.+}} implicit <deduction guide for AA>
+// CHECK-NEXT:  |-TemplateTypeParmDecl {{.+}} class depth 0 index 0 T
+// CHECK-NEXT:  | `-TemplateArgument type 'int'
+// CHECK-NEXT:  |   `-BuiltinType {{.+}} 'int'
+// CHECK-NEXT:  |-TemplateTypeParmDecl {{.+}} class depth 0 index 1 U
+// CHECK-NEXT:  | `-TemplateArgument type 'T':'type-parameter-0-0'
+// CHECK-NEXT:  |   `-TemplateTypeParmType {{.+}} 'T' dependent depth 0 index 0
+// CHECK-NEXT:  |     `-TemplateTypeParm {{.+}} 'T'
+// CHECK-NEXT:  |-TypeTraitExpr {{.+}} 'bool' __is_deducible
+// CHECK-NEXT:  | |-DeducedTemplateSpecializationType {{.+}} 'GH133132::AA' dependent
+// CHECK-NEXT:  | | `-name: 'GH133132::AA'
+// CHECK-NEXT:  | |   `-TypeAliasTemplateDecl {{.+}} AA
+// CHECK-NEXT:  | `-TemplateSpecializationType {{.+}} 'A<U>' dependent
+// CHECK-NEXT:  |   |-name: 'A':'GH133132::A' qualified
+// CHECK-NEXT:  |   | `-ClassTemplateDecl {{.+}} A
+// CHECK-NEXT:  |   `-TemplateArgument type 'U':'type-parameter-0-1'
+// CHECK-NEXT:  |     `-SubstTemplateTypeParmType {{.+}} 'U' sugar dependent class depth 0 index 0 _Ty
+// CHECK-NEXT:  |       |-FunctionTemplate {{.+}} '<deduction guide for A>'
+// CHECK-NEXT:  |       `-TemplateTypeParmType {{.+}} 'U' dependent depth 0 index 1
+// CHECK-NEXT:  |         `-TemplateTypeParm {{.+}} 'U'
+// CHECK-NEXT:  |-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for AA> 'auto () -> A<U>'
+// CHECK-NEXT:  `-CXXDeductionGuideDecl {{.+}} implicit used <deduction guide for AA> 'auto () -> A<int>' implicit_instantiation
+// CHECK-NEXT:    |-TemplateArgument type 'int'
+// CHECK-NEXT:    | `-BuiltinType {{.+}} 'int'
+// CHECK-NEXT:    `-TemplateArgument type 'int'
+// CHECK-NEXT:      `-BuiltinType {{.+}} 'int'
+
+template <template <class> class _X>
+struct B {};
+
+template <template <class> class _X = A, template <class> class _Y = _X>
+using BB = B<_Y>;
+
+BB b{};
+
+// CHECK-LABEL: Dumping GH133132::<deduction guide for BB>:
+// CHECK-NEXT:  FunctionTemplateDecl {{.+}} implicit <deduction guide for BB>
+// CHECK-NEXT:  |-TemplateTemplateParmDecl {{.+}} depth 0 index 0 _X
+// CHECK-NEXT:  | |-TemplateTypeParmDecl {{.+}} class depth 0 index 0
+// CHECK-NEXT:  | `-TemplateArgument {{.+}} template 'A':'GH133132::A' qualified
+// CHECK-NEXT:  |   `-ClassTemplateDecl {{.+}} A
+// CHECK-NEXT:  |-TemplateTemplateParmDecl {{.+}} depth 0 index 1 _Y
+// CHECK-NEXT:  | |-TemplateTypeParmDecl {{.+}} class depth 0 index 0
+// CHECK-NEXT:  | `-TemplateArgument {{.+}} template '_X':'template-parameter-0-0' qualified
+// CHECK-NEXT:  |   `-TemplateTemplateParmDecl {{.+}} depth 0 index 0 _X
+// CHECK-NEXT:  |-TypeTraitExpr {{.+}} 'bool' __is_deducible
+// CHECK-NEXT:  | |-DeducedTemplateSpecializationType {{.+}} 'GH133132::BB' dependent
+// CHECK-NEXT:  | | `-name: 'GH133132::BB'
+// CHECK-NEXT:  | |   `-TypeAliasTemplateDecl {{.+}} BB
+// CHECK-NEXT:  | `-TemplateSpecializationType {{.+}} 'B<_Y>' dependent
+// CHECK-NEXT:  |   |-name: 'B':'GH133132::B' qualified
+// CHECK-NEXT:  |   | `-ClassTemplateDecl {{.+}} B
+// CHECK-NEXT:  |   `-TemplateArgument template '_Y':'template-parameter-0-1' subst index 0
+// CHECK-NEXT:  |     |-parameter: TemplateTemplateParmDecl {{.+}} depth 0 index 0 _X
+// CHECK-NEXT:  |     |-associated FunctionTemplate {{.+}} '<deduction guide for B>'
+// CHECK-NEXT:  |     `-replacement: '_Y':'template-parameter-0-1' qualified
+// CHECK-NEXT:  |       `-TemplateTemplateParmDecl {{.+}} depth 0 index 1 _Y
+// CHECK-NEXT:  |-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for BB> 'auto () -> B<_Y>'
+// CHECK-NEXT:  `-CXXDeductionGuideDecl {{.+}} implicit used <deduction guide for BB> 'auto () -> B<GH133132::A>' implicit_instantiation
+// CHECK-NEXT:    |-TemplateArgument template 'GH133132::A'
+// CHECK-NEXT:    | `-ClassTemplateDecl {{.+}} A
+// CHECK-NEXT:    `-TemplateArgument template 'GH133132::A'
+// CHECK-NEXT:      `-ClassTemplateDecl {{.+}} A
+
+template <int N = 42, class U = A<decltype(N)>>
+using CC = A<U>;
+
+CC c{};
+
+// CHECK-LABEL: Dumping GH133132::<deduction guide for CC>:
+// CHECK-NEXT:  FunctionTemplateDecl {{.+}} implicit <deduction guide for CC>
+// CHECK-NEXT:  |-NonTypeTemplateParmDecl {{.+}} 'int' depth 0 index 0 N
+// CHECK-NEXT:  | `-TemplateArgument {{.+}} expr '42'
+// CHECK-NEXT:  |   `-IntegerLiteral {{.+}} 'int' 42
+// CHECK-NEXT:  |-TemplateTypeParmDecl {{.+}} class depth 0 index 1 U
+// CHECK-NEXT:  | `-TemplateArgument type 'A<decltype(N)>'
+// CHECK-NEXT:  |   `-ElaboratedType {{.+}} 'A<decltype(N)>' sugar dependent
+// CHECK-NEXT:  |     `-TemplateSpecializationType {{.+}} 'A<decltype(N)>' dependent
+// CHECK-NEXT:  |       |-name: 'A':'GH133132::A' qualified
+// CHECK-NEXT:  |       | `-ClassTemplateDecl {{.+}} A
+// CHECK-NEXT:  |       `-TemplateArgument type 'decltype(N)'
+// CHECK-NEXT:  |         `-DecltypeType {{.+}} 'decltype(N)' dependent
+// CHECK-NEXT:  |           `-DeclRefExpr {{.+}} 'int' NonTypeTemplateParm {{.+}} 'N' 'int'
+// CHECK-NEXT:  |-TypeTraitExpr {{.+}} 'bool' __is_deducible
+// CHECK-NEXT:  | |-DeducedTemplateSpecializationType {{.+}} 'GH133132::CC' dependent
+// CHECK-NEXT:  | | `-name: 'GH133132::CC'
+// CHECK-NEXT:  | |   `-TypeAliasTemplateDecl {{.+}} CC
+// CHECK-NEXT:  | `-TemplateSpecializationType {{.+}} 'A<U>' dependent
+// CHECK-NEXT:  |   |-name: 'A':'GH133132::A' qualified
+// CHECK-NEXT:  |   | `-ClassTemplateDecl {{.+}} A
+// CHECK-NEXT:  |   `-TemplateArgument type 'U':'type-parameter-0-1'
+// CHECK-NEXT:  |     `-SubstTemplateTypeParmType {{.+}} 'U' sugar dependent class depth 0 index 0 _Ty
+// CHECK-NEXT:  |       |-FunctionTemplate {{.+}} '<deduction guide for A>'
+// CHECK-NEXT:  |       `-TemplateTypeParmType {{.+}} 'U' dependent depth 0 index 1
+// CHECK-NEXT:  |         `-TemplateTypeParm {{.+}} 'U'
+// CHECK-NEXT:  |-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for CC> 'auto () -> A<U>'
+// CHECK-NEXT:  `-CXXDeductionGuideDecl {{.+}} implicit used <deduction guide for CC> 'auto () -> A<GH133132::A<int>>' implicit_instantiation
+// CHECK-NEXT:    |-TemplateArgument integral '42'
+// CHECK-NEXT:    `-TemplateArgument type 'GH133132::A<int>'
+// CHECK-NEXT:      `-RecordType {{.+}} 'GH133132::A<int>'
+// CHECK-NEXT:        `-ClassTemplateSpecialization {{.+}} 'A'
+
+}

From ee17ca77e50a2170dd79f31e109f6751235e098e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 19:29:30 -0700
Subject: [PATCH 204/710] [llvm] Construct SmallVector with ArrayRef (NFC)
 (#136063)

---
 llvm/lib/IR/IRBuilder.cpp                     | 26 ++++++-------------
 .../deltas/ReduceOperandsToArgs.cpp           |  3 +--
 llvm/tools/lto/lto.cpp                        |  3 +--
 3 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index e5a2f08c393c9..b448c0372eb0e 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -747,21 +747,13 @@ getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
                      std::optional<ArrayRef<T2>> DeoptArgs,
                      ArrayRef<T3> GCArgs) {
   std::vector<OperandBundleDef> Rval;
-  if (DeoptArgs) {
-    SmallVector<Value*, 16> DeoptValues;
-    llvm::append_range(DeoptValues, *DeoptArgs);
-    Rval.emplace_back("deopt", DeoptValues);
-  }
-  if (TransitionArgs) {
-    SmallVector<Value*, 16> TransitionValues;
-    llvm::append_range(TransitionValues, *TransitionArgs);
-    Rval.emplace_back("gc-transition", TransitionValues);
-  }
-  if (GCArgs.size()) {
-    SmallVector<Value*, 16> LiveValues;
-    llvm::append_range(LiveValues, GCArgs);
-    Rval.emplace_back("gc-live", LiveValues);
-  }
+  if (DeoptArgs)
+    Rval.emplace_back("deopt", SmallVector<Value *, 16>(*DeoptArgs));
+  if (TransitionArgs)
+    Rval.emplace_back("gc-transition",
+                      SmallVector<Value *, 16>(*TransitionArgs));
+  if (GCArgs.size())
+    Rval.emplace_back("gc-live", SmallVector<Value *, 16>(GCArgs));
   return Rval;
 }
 
@@ -1091,9 +1083,7 @@ CallInst *IRBuilderBase::CreateConstrainedFPCall(
     Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
     std::optional<RoundingMode> Rounding,
     std::optional<fp::ExceptionBehavior> Except) {
-  llvm::SmallVector<Value *, 6> UseArgs;
-
-  append_range(UseArgs, Args);
+  llvm::SmallVector<Value *, 6> UseArgs(Args);
 
   if (Intrinsic::hasConstrainedFPRoundingModeOperand(Callee->getIntrinsicID()))
     UseArgs.push_back(getConstrainedFPRounding(Rounding));
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceOperandsToArgs.cpp b/llvm/tools/llvm-reduce/deltas/ReduceOperandsToArgs.cpp
index 3f877b525549f..cb27d5473fd73 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceOperandsToArgs.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceOperandsToArgs.cpp
@@ -129,8 +129,7 @@ static void substituteOperandWithArgument(Function *OldF,
     UniqueValues.insert(Op->get());
 
   // Determine the new function's signature.
-  SmallVector<Type *> NewArgTypes;
-  llvm::append_range(NewArgTypes, OldF->getFunctionType()->params());
+  SmallVector<Type *> NewArgTypes(OldF->getFunctionType()->params());
   size_t ArgOffset = NewArgTypes.size();
   for (Value *V : UniqueValues)
     NewArgTypes.push_back(V->getType());
diff --git a/llvm/tools/lto/lto.cpp b/llvm/tools/lto/lto.cpp
index 29219c9114522..bb64d42ccced1 100644
--- a/llvm/tools/lto/lto.cpp
+++ b/llvm/tools/lto/lto.cpp
@@ -496,8 +496,7 @@ void lto_codegen_debug_options_array(lto_code_gen_t cg,
                                      const char *const *options, int number) {
   assert(optionParsingState != OptParsingState::Early &&
          "early option processing already happened");
-  SmallVector<StringRef, 4> Options;
-  llvm::append_range(Options, ArrayRef(options, number));
+  SmallVector<StringRef, 4> Options(ArrayRef(options, number));
   unwrap(cg)->setCodeGenDebugOptions(ArrayRef(Options));
 }
 

From 799916ae1047cac6f6af8aed1499d8bf1b921f82 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 19:29:47 -0700
Subject: [PATCH 205/710] [llvm] Construct SmallVector with iterator ranges
 (NFC) (#136064)

---
 llvm/lib/Analysis/VectorUtils.cpp                        | 4 +---
 llvm/lib/IR/Verifier.cpp                                 | 3 +--
 llvm/lib/Transforms/IPO/ArgumentPromotion.cpp            | 3 +--
 llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 5 ++---
 4 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 62f1555fdd150..6448c372f5d5d 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -1703,9 +1703,7 @@ void InterleaveGroup<InstT>::addMetadata(InstT *NewInst) const {
 namespace llvm {
 template <>
 void InterleaveGroup<Instruction>::addMetadata(Instruction *NewInst) const {
-  SmallVector<Value *, 4> VL;
-  std::transform(Members.begin(), Members.end(), std::back_inserter(VL),
-                 [](std::pair<int, Instruction *> p) { return p.second; });
+  SmallVector<Value *, 4> VL(make_second_range(Members));
   propagateMetadata(NewInst, VL);
 }
 } // namespace llvm
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index e3f6c1ad5a65b..765befc95032c 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -720,8 +720,7 @@ static void forEachUser(const Value *User,
   if (!Visited.insert(User).second)
     return;
 
-  SmallVector<const Value *> WorkList;
-  append_range(WorkList, User->materialized_users());
+  SmallVector<const Value *> WorkList(User->materialized_users());
   while (!WorkList.empty()) {
    const Value *Cur = WorkList.pop_back_val();
     if (!Visited.insert(Cur).second)
diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index 80a0df7d64205..0ec5202b8cfe7 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -382,9 +382,8 @@ doPromotion(Function *F, FunctionAnalysisManager &FAM,
     // Cleanup the code from the dead instructions: GEPs and BitCasts in between
     // the original argument and its users: loads and stores. Retarget every
     // user to the new created alloca.
-    SmallVector<Value *, 16> Worklist;
+    SmallVector<Value *, 16> Worklist(Arg.users());
     SmallVector<Instruction *, 16> DeadInsts;
-    append_range(Worklist, Arg.users());
     while (!Worklist.empty()) {
       Value *V = Worklist.pop_back_val();
       if (isa<GetElementPtrInst>(V)) {
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index cc9e52499d285..f807f5f4519fc 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -2500,9 +2500,8 @@ Instruction *InstCombinerImpl::visitGEPOfGEP(GetElementPtrInst &GEP,
       return nullptr;
 
     GEPNoWrapFlags NW = getMergedGEPNoWrapFlags(*Src, *cast<GEPOperator>(&GEP));
-    SmallVector<Value *> Indices;
-    append_range(Indices, drop_end(Src->indices(),
-                                   Src->getNumIndices() - NumVarIndices));
+    SmallVector<Value *> Indices(
+        drop_end(Src->indices(), Src->getNumIndices() - NumVarIndices));
     for (const APInt &Idx : drop_begin(ConstIndices, !IsFirstType)) {
       Indices.push_back(ConstantInt::get(GEP.getContext(), Idx));
       // Even if the total offset is inbounds, we may end up representing it

From 47d8fec9b8bd5425af17711317a41743a30a8cef Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 16 Apr 2025 19:30:01 -0700
Subject: [PATCH 206/710] [llvm] Use llvm::append_range (NFC) (#136066)

This patch replaces:

  llvm::copy(Src, std::back_inserter(Dst));

with:

  llvm::append_range(Dst, Src);

for breavity.

One side benefit is that llvm::append_range eventually calls
llvm::SmallVector::reserve if Dst is of llvm::SmallVector.
---
 llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h | 3 +--
 llvm/include/llvm/Bitcode/BitcodeConvenience.h      | 2 +-
 llvm/include/llvm/IR/DiagnosticInfo.h               | 2 +-
 llvm/lib/Analysis/LoopAccessAnalysis.cpp            | 2 +-
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp        | 2 +-
 llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp     | 6 +++---
 llvm/lib/CodeGen/ModuloSchedule.cpp                 | 4 ++--
 llvm/lib/ExecutionEngine/JITLink/JITLink.cpp        | 4 ++--
 llvm/lib/Transforms/Scalar/GVNSink.cpp              | 6 +++---
 llvm/tools/llvm-jitlink/llvm-jitlink.cpp            | 2 +-
 llvm/utils/TableGen/SearchableTableEmitter.cpp      | 3 +--
 11 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 66ea5ecbc4bfb..57c408968b19a 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -1159,8 +1159,7 @@ void BlockFrequencyInfoImpl<BT>::setBlockFreq(const BlockT *BB,
 
 template <class BT> void BlockFrequencyInfoImpl<BT>::initializeRPOT() {
   const BlockT *Entry = &F->front();
-  RPOT.reserve(F->size());
-  std::copy(po_begin(Entry), po_end(Entry), std::back_inserter(RPOT));
+  llvm::append_range(RPOT, post_order(Entry));
   std::reverse(RPOT.begin(), RPOT.end());
 
   assert(RPOT.size() - 1 <= BlockNode::getMaxIndex() &&
diff --git a/llvm/include/llvm/Bitcode/BitcodeConvenience.h b/llvm/include/llvm/Bitcode/BitcodeConvenience.h
index 009a086a339cb..b7f63664409c2 100644
--- a/llvm/include/llvm/Bitcode/BitcodeConvenience.h
+++ b/llvm/include/llvm/Bitcode/BitcodeConvenience.h
@@ -266,7 +266,7 @@ template <typename ElementTy> class BCRecordCoding<BCArray<ElementTy>> {
       ElementTy::assertValid(element);
 #endif
     buffer.reserve(buffer.size() + std::distance(array.begin(), array.end()));
-    std::copy(array.begin(), array.end(), std::back_inserter(buffer));
+    llvm::append_range(buffer, array);
     Stream.EmitRecordWithAbbrev(code, buffer);
   }
 
diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h
index 779c88993b71c..8743f8058c382 100644
--- a/llvm/include/llvm/IR/DiagnosticInfo.h
+++ b/llvm/include/llvm/IR/DiagnosticInfo.h
@@ -693,7 +693,7 @@ class DiagnosticInfoIROptimization : public DiagnosticInfoOptimizationBase {
             Orig.RemarkName, Orig.getFunction(), Orig.getLocation()),
         CodeRegion(Orig.getCodeRegion()) {
     *this << Prepend;
-    std::copy(Orig.Args.begin(), Orig.Args.end(), std::back_inserter(Args));
+    llvm::append_range(Args, Orig.Args);
   }
 
   /// Legacy interface.
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 0cb1332eb337b..9a7d361b5b512 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -563,7 +563,7 @@ void RuntimePointerChecking::groupChecks(
 
     // We've computed the grouped checks for this partition.
     // Save the results and continue with the next one.
-    llvm::copy(Groups, std::back_inserter(CheckingGroups));
+    llvm::append_range(CheckingGroups, Groups);
   }
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 85a6d67609798..931e4fe19e69a 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -232,7 +232,7 @@ ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
     unsigned Idx = 0;
     while (auto Elt = C.getAggregateElement(Idx++)) {
       auto EltRegs = getOrCreateVRegs(*Elt);
-      llvm::copy(EltRegs, std::back_inserter(*VRegs));
+      llvm::append_range(*VRegs, EltRegs);
     }
   } else {
     assert(SplitTys.size() == 1 && "unexpectedly split LLT");
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index ee271234d3119..0aa853389bf1a 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -592,7 +592,7 @@ llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
         isLibCallInTailPosition(Result, *MI, MIRBuilder.getTII(),
                                 *MIRBuilder.getMRI());
 
-  std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
+  llvm::append_range(Info.OrigArgs, Args);
   if (!CLI.lowerCall(MIRBuilder, Info))
     return LegalizerHelper::UnableToLegalize;
 
@@ -708,7 +708,7 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
       MI.getOperand(MI.getNumOperands() - 1).getImm() &&
       isLibCallInTailPosition(Info.OrigRet, MI, MIRBuilder.getTII(), MRI);
 
-  std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
+  llvm::append_range(Info.OrigArgs, Args);
   if (!CLI.lowerCall(MIRBuilder, Info))
     return LegalizerHelper::UnableToLegalize;
 
@@ -855,7 +855,7 @@ createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI) {
   Info.Callee = MachineOperand::CreateES(Name);
   Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0);
 
-  std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
+  llvm::append_range(Info.OrigArgs, Args);
   if (!CLI.lowerCall(MIRBuilder, Info))
     return LegalizerHelper::UnableToLegalize;
 
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index 9fd7443dc0d00..f028f01c58290 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -1855,9 +1855,9 @@ void PeelingModuloScheduleExpander::peelPrologAndEpilogs() {
 
   // Create a list of all blocks in order.
   SmallVector<MachineBasicBlock *, 8> Blocks;
-  llvm::copy(PeeledFront, std::back_inserter(Blocks));
+  llvm::append_range(Blocks, PeeledFront);
   Blocks.push_back(BB);
-  llvm::copy(PeeledBack, std::back_inserter(Blocks));
+  llvm::append_range(Blocks, PeeledBack);
 
   // Iterate in reverse order over all instructions, remapping as we go.
   for (MachineBasicBlock *B : reverse(Blocks)) {
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
index 15a8fcf312ade..e1209e1e95496 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
@@ -313,7 +313,7 @@ void LinkGraph::dump(raw_ostream &OS) {
     OS << "section " << Sec->getName() << ":\n\n";
 
     std::vector<Block *> SortedBlocks;
-    llvm::copy(Sec->blocks(), std::back_inserter(SortedBlocks));
+    llvm::append_range(SortedBlocks, Sec->blocks());
     llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) {
       return LHS->getAddress() < RHS->getAddress();
     });
@@ -339,7 +339,7 @@ void LinkGraph::dump(raw_ostream &OS) {
       if (!B->edges_empty()) {
         OS << "    edges:\n";
         std::vector<Edge> SortedEdges;
-        llvm::copy(B->edges(), std::back_inserter(SortedEdges));
+        llvm::append_range(SortedEdges, B->edges());
         llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) {
           return LHS.getOffset() < RHS.getOffset();
         });
diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index d0f844d7cd36a..6f88408abfdbc 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -199,8 +199,8 @@ class ModelledPHI {
               SmallSetVector<BasicBlock *, 4> &B,
               const DenseMap<const BasicBlock *, unsigned> &BlockOrder) {
     // The order of Values and Blocks are already ordered by the caller.
-    llvm::copy(V, std::back_inserter(Values));
-    llvm::copy(B, std::back_inserter(Blocks));
+    llvm::append_range(Values, V);
+    llvm::append_range(Blocks, B);
     verifyModelledPHI(BlockOrder);
   }
 
@@ -208,7 +208,7 @@ class ModelledPHI {
   /// TODO: Figure out a way to verifyModelledPHI in this constructor.
   ModelledPHI(ArrayRef<Instruction *> Insts, unsigned OpNum,
               SmallSetVector<BasicBlock *, 4> &B) {
-    llvm::copy(B, std::back_inserter(Blocks));
+    llvm::append_range(Blocks, B);
     for (auto *I : Insts)
       Values.push_back(I->getOperand(OpNum));
   }
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 8a17abd58e98a..54b90cd7c7506 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -2360,7 +2360,7 @@ static Error addLibraries(Session &S,
         SmallVector<char, 256> LibPath;
         LibPath.reserve(SearchPath.size() + strlen("lib") + LL.LibName.size() +
                         LibExt.size() + 2); // +2 for pathsep, null term.
-        llvm::copy(SearchPath, std::back_inserter(LibPath));
+        llvm::append_range(LibPath, SearchPath);
         if (LibExt != ".lib" && LibExt != ".dll")
           sys::path::append(LibPath, "lib" + LL.LibName + LibExt);
         else
diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp
index e91baf98e9ffc..7251a1ba545d5 100644
--- a/llvm/utils/TableGen/SearchableTableEmitter.cpp
+++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp
@@ -701,8 +701,7 @@ void SearchableTableEmitter::collectTableEntries(
   }
 
   SearchIndex Idx;
-  std::copy(Table.Fields.begin(), Table.Fields.end(),
-            std::back_inserter(Idx.Fields));
+  llvm::append_range(Idx.Fields, Table.Fields);
   llvm::sort(Table.Entries, [&](const Record *LHS, const Record *RHS) {
     return compareBy(LHS, RHS, Idx);
   });

From de893f87d2893c0de4dd730bcd91a79797078fa7 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Thu, 17 Apr 2025 11:31:30 +0900
Subject: [PATCH 207/710] Revert "[bazel] Fix build for
 8ebdd9d8a19543992195f197de215c53d506fb72"

This reverts commit f8ea2ed59820a0bef3f23638ce7a5d10165f7109.
---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 143304c24c241..b77ddf634eec6 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -2148,10 +2148,7 @@ llvm_target_lib_list = [lib for lib in [
                 "lib/Target/AArch64/AArch64GenSubtargetInfo.inc",
             ),
             (
-                [
-                    "-gen-disassembler",
-                    "--num-to-skip-size=3",
-                ],
+                ["-gen-disassembler"],
                 "lib/Target/AArch64/AArch64GenDisassemblerTables.inc",
             ),
             (

From b6c99cb56bcd7f08d6c94d0feca88afacb25302f Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Wed, 16 Apr 2025 19:48:56 -0700
Subject: [PATCH 208/710] [gn] port 6d03f51f0c59

---
 llvm/utils/gn/secondary/compiler-rt/lib/builtins/sources.gni | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/sources.gni b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/sources.gni
index b693b75e04582..ba151075c0f9d 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/sources.gni
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/sources.gni
@@ -55,6 +55,7 @@ builtins_sources = [
   "divti3.c",
   "extendhfsf2.c",
   "extendsfdf2.c",
+  "extendhfdf2.c",
   "ffsdi2.c",
   "ffssi2.c",
   "ffsti2.c",

From 638c386b618a286ab1f30f628485e495f21ce124 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Wed, 16 Apr 2025 19:52:19 -0700
Subject: [PATCH 209/710] [gn] port ece59a8cb9c8

---
 llvm/lib/Support/CMakeLists.txt                         | 2 +-
 llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn       | 1 +
 llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 98ffd829d80b8..14991b1cc5bb4 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -221,7 +221,7 @@ add_llvm_component_library(LLVMSupport
   MD5.cpp
   MSP430Attributes.cpp
   MSP430AttributeParser.cpp
-  Mustache.cpp      
+  Mustache.cpp
   NativeFormatting.cpp
   OptimizedStructLayout.cpp
   Optional.cpp
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 3a9f43b1070a7..aa48edfaa87ab 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -109,6 +109,7 @@ static_library("Support") {
     "LockFileManager.cpp",
     "MD5.cpp",
     "MSP430AttributeParser.cpp",
+    "Mustache.cpp",
     "MSP430Attributes.cpp",
     "ManagedStatic.cpp",
     "MathExtras.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index bf6a0b7523279..3185ece413dea 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -65,6 +65,7 @@ unittest("SupportTests") {
     "MemoryBufferTest.cpp",
     "MemoryTest.cpp",
     "ModRefTest.cpp",
+    "MustacheTest.cpp",
     "NativeFormatTests.cpp",
     "OptimizedStructLayoutTest.cpp",
     "ParallelTest.cpp",

From ebf9883b82aed98aacdfc77824655fec8dc46988 Mon Sep 17 00:00:00 2001
From: tangaac <tangyan01@loongson.cn>
Date: Thu, 17 Apr 2025 11:08:18 +0800
Subject: [PATCH 210/710] [LoongArch] Lower vector shuffle as byte rotate (if
 possible) (#135157)

---
 .../LoongArch/LoongArchISelLowering.cpp       | 137 ++++++++++++++++++
 .../LoongArch/lasx/vec-shuffle-byte-rotate.ll |  74 +++++-----
 .../LoongArch/lsx/vec-shuffle-byte-rotate.ll  |  74 +++++-----
 3 files changed, 203 insertions(+), 82 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index f72b55e1d175c..7e37bbd652114 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -714,6 +714,139 @@ static void computeZeroableShuffleElements(ArrayRef<int> Mask, SDValue V1,
   }
 }
 
+/// Test whether a shuffle mask is equivalent within each sub-lane.
+///
+/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
+/// non-trivial to compute in the face of undef lanes. The representation is
+/// suitable for use with existing 128-bit shuffles as entries from the second
+/// vector have been remapped to [LaneSize, 2*LaneSize).
+static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
+                                  ArrayRef<int> Mask,
+                                  SmallVectorImpl<int> &RepeatedMask) {
+  auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
+  RepeatedMask.assign(LaneSize, -1);
+  int Size = Mask.size();
+  for (int i = 0; i < Size; ++i) {
+    assert(Mask[i] == -1 || Mask[i] >= 0);
+    if (Mask[i] < 0)
+      continue;
+    if ((Mask[i] % Size) / LaneSize != i / LaneSize)
+      // This entry crosses lanes, so there is no way to model this shuffle.
+      return false;
+
+    // Ok, handle the in-lane shuffles by detecting if and when they repeat.
+    // Adjust second vector indices to start at LaneSize instead of Size.
+    int LocalM =
+        Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
+    if (RepeatedMask[i % LaneSize] < 0)
+      // This is the first non-undef entry in this slot of a 128-bit lane.
+      RepeatedMask[i % LaneSize] = LocalM;
+    else if (RepeatedMask[i % LaneSize] != LocalM)
+      // Found a mismatch with the repeated mask.
+      return false;
+  }
+  return true;
+}
+
+/// Attempts to match vector shuffle as byte rotation.
+static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
+                                    ArrayRef<int> Mask) {
+
+  SDValue Lo, Hi;
+  SmallVector<int, 16> RepeatedMask;
+
+  if (!isRepeatedShuffleMask(128, VT, Mask, RepeatedMask))
+    return -1;
+
+  int NumElts = RepeatedMask.size();
+  int Rotation = 0;
+  int Scale = 16 / NumElts;
+
+  for (int i = 0; i < NumElts; ++i) {
+    int M = RepeatedMask[i];
+    assert((M == -1 || (0 <= M && M < (2 * NumElts))) &&
+           "Unexpected mask index.");
+    if (M < 0)
+      continue;
+
+    // Determine where a rotated vector would have started.
+    int StartIdx = i - (M % NumElts);
+    if (StartIdx == 0)
+      return -1;
+
+    // If we found the tail of a vector the rotation must be the missing
+    // front. If we found the head of a vector, it must be how much of the
+    // head.
+    int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
+
+    if (Rotation == 0)
+      Rotation = CandidateRotation;
+    else if (Rotation != CandidateRotation)
+      return -1;
+
+    // Compute which value this mask is pointing at.
+    SDValue MaskV = M < NumElts ? V1 : V2;
+
+    // Compute which of the two target values this index should be assigned
+    // to. This reflects whether the high elements are remaining or the low
+    // elements are remaining.
+    SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
+
+    // Either set up this value if we've not encountered it before, or check
+    // that it remains consistent.
+    if (!TargetV)
+      TargetV = MaskV;
+    else if (TargetV != MaskV)
+      return -1;
+  }
+
+  // Check that we successfully analyzed the mask, and normalize the results.
+  assert(Rotation != 0 && "Failed to locate a viable rotation!");
+  assert((Lo || Hi) && "Failed to find a rotated input vector!");
+  if (!Lo)
+    Lo = Hi;
+  else if (!Hi)
+    Hi = Lo;
+
+  V1 = Lo;
+  V2 = Hi;
+
+  return Rotation * Scale;
+}
+
+/// Lower VECTOR_SHUFFLE as byte rotate (if possible).
+///
+/// For example:
+///   %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b,
+///                            <2 x i32> <i32 3, i32 0>
+/// is lowered to:
+///      (VBSRL_V $v1, $v1, 8)
+///      (VBSLL_V $v0, $v0, 8)
+///      (VOR_V $v0, $V0, $v1)
+static SDValue lowerVECTOR_SHUFFLEAsByteRotate(const SDLoc &DL,
+                                               ArrayRef<int> Mask, MVT VT,
+                                               SDValue V1, SDValue V2,
+                                               SelectionDAG &DAG) {
+
+  SDValue Lo = V1, Hi = V2;
+  int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask);
+  if (ByteRotation <= 0)
+    return SDValue();
+
+  MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
+  Lo = DAG.getBitcast(ByteVT, Lo);
+  Hi = DAG.getBitcast(ByteVT, Hi);
+
+  int LoByteShift = 16 - ByteRotation;
+  int HiByteShift = ByteRotation;
+
+  SDValue LoShift = DAG.getNode(LoongArchISD::VBSLL, DL, ByteVT, Lo,
+                                DAG.getConstant(LoByteShift, DL, MVT::i64));
+  SDValue HiShift = DAG.getNode(LoongArchISD::VBSRL, DL, ByteVT, Hi,
+                                DAG.getConstant(HiByteShift, DL, MVT::i64));
+  return DAG.getBitcast(VT, DAG.getNode(ISD::OR, DL, ByteVT, LoShift, HiShift));
+}
+
 /// Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
 ///
 /// For example:
@@ -1230,6 +1363,8 @@ static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
   if ((Result =
            lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Zeroable)))
     return Result;
+  if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG)))
+    return Result;
   if ((Result = lowerVECTOR_SHUFFLE_VSHUF(DL, Mask, VT, V1, V2, DAG)))
     return Result;
   return SDValue();
@@ -1666,6 +1801,8 @@ static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
   if ((Result =
            lowerVECTOR_SHUFFLEAsShift(DL, NewMask, VT, V1, V2, DAG, Zeroable)))
     return Result;
+  if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, NewMask, VT, V1, V2, DAG)))
+    return Result;
   if ((Result = lowerVECTOR_SHUFFLE_XVSHUF(DL, NewMask, VT, V1, V2, DAG)))
     return Result;
 
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-shuffle-byte-rotate.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-shuffle-byte-rotate.ll
index 88d9037e3a9e9..b697a2fd07435 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/vec-shuffle-byte-rotate.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-shuffle-byte-rotate.ll
@@ -6,9 +6,9 @@
 define <32 x i8> @byte_rotate_v32_i8_1(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK-LABEL: byte_rotate_v32_i8_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    xvshuf.b $xr0, $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvbsrl.v $xr0, $xr0, 1
+; CHECK-NEXT:    xvbsll.v $xr1, $xr1, 15
+; CHECK-NEXT:    xvor.v $xr0, $xr1, $xr0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48>
   ret <32 x i8> %shuffle
@@ -17,9 +17,9 @@ define <32 x i8> @byte_rotate_v32_i8_1(<32 x i8> %a, <32 x i8> %b) {
 define <32 x i8> @byte_rotate_v32_i8_2(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK-LABEL: byte_rotate_v32_i8_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI1_0)
-; CHECK-NEXT:    xvshuf.b $xr0, $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 13
+; CHECK-NEXT:    xvbsll.v $xr0, $xr0, 3
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 45, i32 46, i32 47, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 61, i32 62, i32 63, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28>
   ret <32 x i8> %shuffle
@@ -28,9 +28,9 @@ define <32 x i8> @byte_rotate_v32_i8_2(<32 x i8> %a, <32 x i8> %b) {
 define <32 x i8> @byte_rotate_v32_i8_3(<32 x i8> %a) {
 ; CHECK-LABEL: byte_rotate_v32_i8_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
-; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI2_0)
-; CHECK-NEXT:    xvshuf.b $xr0, $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr0, 1
+; CHECK-NEXT:    xvbsll.v $xr0, $xr0, 15
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16>
   ret <32 x i8> %shuffle
@@ -40,10 +40,9 @@ define <32 x i8> @byte_rotate_v32_i8_3(<32 x i8> %a) {
 define <16 x i16> @byte_rotate_v16i16_1(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v16i16_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI3_0)
-; CHECK-NEXT:    xvshuf.h $xr2, $xr1, $xr0
-; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
+; CHECK-NEXT:    xvbsrl.v $xr0, $xr0, 6
+; CHECK-NEXT:    xvbsll.v $xr1, $xr1, 10
+; CHECK-NEXT:    xvor.v $xr0, $xr1, $xr0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26>
   ret <16 x i16> %shuffle
@@ -52,10 +51,9 @@ define <16 x i16> @byte_rotate_v16i16_1(<16 x i16> %a, <16 x i16> %b) nounwind {
 define <16 x i16> @byte_rotate_v16i16_2(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v16i16_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    xvshuf.h $xr2, $xr1, $xr0
-; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 10
+; CHECK-NEXT:    xvbsll.v $xr0, $xr0, 6
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 21, i32 22, i32 23, i32 0, i32 1, i32 2,i32 3, i32 4, i32 29, i32 30, i32 31, i32 8, i32 9, i32 10, i32 11, i32 12>
   ret <16 x i16> %shuffle
@@ -64,10 +62,9 @@ define <16 x i16> @byte_rotate_v16i16_2(<16 x i16> %a, <16 x i16> %b) nounwind {
 define <16 x i16> @byte_rotate_v16i16_3(<16 x i16> %a) nounwind {
 ; CHECK-LABEL: byte_rotate_v16i16_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
-; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI5_0)
-; CHECK-NEXT:    xvshuf.h $xr1, $xr0, $xr0
-; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr0, 6
+; CHECK-NEXT:    xvbsll.v $xr0, $xr0, 10
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10>
   ret <16 x i16> %shuffle
@@ -76,10 +73,9 @@ define <16 x i16> @byte_rotate_v16i16_3(<16 x i16> %a) nounwind {
 define <8 x i32> @byte_rotate_v8i32_1(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v8i32_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI6_0)
-; CHECK-NEXT:    xvshuf.w $xr2, $xr1, $xr0
-; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
+; CHECK-NEXT:    xvbsrl.v $xr0, $xr0, 4
+; CHECK-NEXT:    xvbsll.v $xr1, $xr1, 12
+; CHECK-NEXT:    xvor.v $xr0, $xr1, $xr0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 8, i32 5, i32 6, i32 7, i32 12>
   ret <8 x i32> %shuffle
@@ -88,10 +84,9 @@ define <8 x i32> @byte_rotate_v8i32_1(<8 x i32> %a, <8 x i32> %b) nounwind {
 define <8 x i32> @byte_rotate_v8i32_2(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v8i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI7_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI7_0)
-; CHECK-NEXT:    xvshuf.w $xr2, $xr1, $xr0
-; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 12
+; CHECK-NEXT:    xvbsll.v $xr0, $xr0, 4
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 11, i32 0, i32 1, i32 2, i32 15, i32 4, i32 5, i32 6>
   ret <8 x i32> %shuffle
@@ -109,10 +104,9 @@ define <8 x i32> @byte_rotate_v8i32_3(<8 x i32> %a) nounwind {
 define <4 x i64> @byte_rotate_v4i64_1(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v4i64_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI9_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI9_0)
-; CHECK-NEXT:    xvshuf.d $xr2, $xr1, $xr0
-; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
+; CHECK-NEXT:    xvbsrl.v $xr0, $xr0, 8
+; CHECK-NEXT:    xvbsll.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvor.v $xr0, $xr1, $xr0
 ; CHECK-NEXT:    ret
     %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 4, i32 3, i32 6>
     ret <4 x i64> %shuffle
@@ -121,10 +115,9 @@ define <4 x i64> @byte_rotate_v4i64_1(<4 x i64> %a, <4 x i64> %b) nounwind {
 define <4 x i64> @byte_rotate_v4i64_2(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v4i64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI10_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI10_0)
-; CHECK-NEXT:    xvshuf.d $xr2, $xr1, $xr0
-; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvbsll.v $xr0, $xr0, 8
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
     %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 5, i32 0, i32 7, i32 2>
     ret <4 x i64> %shuffle
@@ -133,10 +126,9 @@ define <4 x i64> @byte_rotate_v4i64_2(<4 x i64> %a, <4 x i64> %b) nounwind {
 define <4 x i64> @byte_rotate_v4i64_3(<4 x i64> %a) nounwind {
 ; CHECK-LABEL: byte_rotate_v4i64_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI11_0)
-; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI11_0)
-; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr0
-; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr0, 8
+; CHECK-NEXT:    xvbsll.v $xr0, $xr0, 8
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
 ; CHECK-NEXT:    ret
     %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
     ret <4 x i64> %shuffle
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-shuffle-byte-rotate.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-shuffle-byte-rotate.ll
index eb92d236f43a0..7b2bb47424fee 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/vec-shuffle-byte-rotate.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-shuffle-byte-rotate.ll
@@ -6,9 +6,9 @@
 define <16 x i8> @byte_rotate_v16i8_1(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v16i8_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
-; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    vshuf.b $vr0, $vr1, $vr0, $vr2
+; CHECK-NEXT:    vbsrl.v $vr0, $vr0, 5
+; CHECK-NEXT:    vbsll.v $vr1, $vr1, 11
+; CHECK-NEXT:    vor.v $vr0, $vr1, $vr0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
   ret <16 x i8> %shuffle
@@ -17,9 +17,9 @@ define <16 x i8> @byte_rotate_v16i8_1(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <16 x i8> @byte_rotate_v16i8_2(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v16i8_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; CHECK-NEXT:    vld $vr1, $a0, %pc_lo12(.LCPI1_0)
-; CHECK-NEXT:    vshuf.b $vr0, $vr0, $vr0, $vr1
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 11
+; CHECK-NEXT:    vbsll.v $vr0, $vr0, 5
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10>
   ret <16 x i8> %shuffle
@@ -28,9 +28,9 @@ define <16 x i8> @byte_rotate_v16i8_2(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <16 x i8> @byte_rotate_v16i8_3(<16 x i8> %a) nounwind {
 ; CHECK-LABEL: byte_rotate_v16i8_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
-; CHECK-NEXT:    vld $vr1, $a0, %pc_lo12(.LCPI2_0)
-; CHECK-NEXT:    vshuf.b $vr0, $vr0, $vr0, $vr1
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 11
+; CHECK-NEXT:    vbsll.v $vr0, $vr0, 5
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> poison, <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10>
   ret <16 x i8> %shuffle
@@ -39,10 +39,9 @@ define <16 x i8> @byte_rotate_v16i8_3(<16 x i8> %a) nounwind {
 define <8 x i16> @byte_rotate_v8i16_1(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v8i16_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
-; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI3_0)
-; CHECK-NEXT:    vshuf.h $vr2, $vr1, $vr0
-; CHECK-NEXT:    vori.b $vr0, $vr2, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr1, 10
+; CHECK-NEXT:    vbsll.v $vr0, $vr0, 6
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4>
   ret <8 x i16> %shuffle
@@ -51,10 +50,9 @@ define <8 x i16> @byte_rotate_v8i16_1(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <8 x i16> @byte_rotate_v8i16_2(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v8i16_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    vshuf.h $vr2, $vr1, $vr0
-; CHECK-NEXT:    vori.b $vr0, $vr2, 0
+; CHECK-NEXT:    vbsrl.v $vr0, $vr0, 6
+; CHECK-NEXT:    vbsll.v $vr1, $vr1, 10
+; CHECK-NEXT:    vor.v $vr0, $vr1, $vr0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10>
   ret <8 x i16> %shuffle
@@ -63,10 +61,9 @@ define <8 x i16> @byte_rotate_v8i16_2(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <8 x i16> @byte_rotate_v8i16_3(<8 x i16> %a) nounwind {
 ; CHECK-LABEL: byte_rotate_v8i16_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
-; CHECK-NEXT:    vld $vr1, $a0, %pc_lo12(.LCPI5_0)
-; CHECK-NEXT:    vshuf.h $vr1, $vr0, $vr0
-; CHECK-NEXT:    vori.b $vr0, $vr1, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 10
+; CHECK-NEXT:    vbsll.v $vr0, $vr0, 6
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4>
   ret <8 x i16> %shuffle
@@ -75,10 +72,9 @@ define <8 x i16> @byte_rotate_v8i16_3(<8 x i16> %a) nounwind {
 define <4 x i32> @byte_rotate_v4i32_1(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v4i32_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
-; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI6_0)
-; CHECK-NEXT:    vshuf.w $vr2, $vr1, $vr0
-; CHECK-NEXT:    vori.b $vr0, $vr2, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr1, 12
+; CHECK-NEXT:    vbsll.v $vr0, $vr0, 4
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
     %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 7, i32 0, i32 1, i32 2>
     ret <4 x i32> %shuffle
@@ -87,10 +83,9 @@ define <4 x i32> @byte_rotate_v4i32_1(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <4 x i32> @byte_rotate_v4i32_2(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v4i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI7_0)
-; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI7_0)
-; CHECK-NEXT:    vshuf.w $vr2, $vr1, $vr0
-; CHECK-NEXT:    vori.b $vr0, $vr2, 0
+; CHECK-NEXT:    vbsrl.v $vr0, $vr0, 4
+; CHECK-NEXT:    vbsll.v $vr1, $vr1, 12
+; CHECK-NEXT:    vor.v $vr0, $vr1, $vr0
 ; CHECK-NEXT:    ret
     %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
     ret <4 x i32> %shuffle
@@ -108,10 +103,9 @@ define <4 x i32> @byte_rotate_v4i32_3(<4 x i32> %a) nounwind {
 define <2 x i64> @byte_rotate_v2i64_1(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v2i64_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI9_0)
-; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI9_0)
-; CHECK-NEXT:    vshuf.d $vr2, $vr1, $vr0
-; CHECK-NEXT:    vori.b $vr0, $vr2, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr1, 8
+; CHECK-NEXT:    vbsll.v $vr0, $vr0, 8
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
     %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 3, i32 0>
     ret <2 x i64> %shuffle
@@ -120,10 +114,9 @@ define <2 x i64> @byte_rotate_v2i64_1(<2 x i64> %a, <2 x i64> %b) nounwind {
 define <2 x i64> @byte_rotate_v2i64_2(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; CHECK-LABEL: byte_rotate_v2i64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI10_0)
-; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI10_0)
-; CHECK-NEXT:    vshuf.d $vr2, $vr1, $vr0
-; CHECK-NEXT:    vori.b $vr0, $vr2, 0
+; CHECK-NEXT:    vbsrl.v $vr0, $vr0, 8
+; CHECK-NEXT:    vbsll.v $vr1, $vr1, 8
+; CHECK-NEXT:    vor.v $vr0, $vr1, $vr0
 ; CHECK-NEXT:    ret
     %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 1, i32 2>
     ret <2 x i64> %shuffle
@@ -132,10 +125,9 @@ define <2 x i64> @byte_rotate_v2i64_2(<2 x i64> %a, <2 x i64> %b) nounwind {
 define <2 x i64> @byte_rotate_v2i64_3(<2 x i64> %a) nounwind {
 ; CHECK-LABEL: byte_rotate_v2i64_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI11_0)
-; CHECK-NEXT:    vld $vr1, $a0, %pc_lo12(.LCPI11_0)
-; CHECK-NEXT:    vshuf.d $vr1, $vr0, $vr0
-; CHECK-NEXT:    vori.b $vr0, $vr1, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vbsll.v $vr0, $vr0, 8
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
 ; CHECK-NEXT:    ret
     %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
     ret <2 x i64> %shuffle

From b99ac8b3d37ee2410d57c4c943a4bcff806dc5d6 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 17 Apr 2025 03:09:18 +0000
Subject: [PATCH 211/710] [gn build] Port 3de88fe40fd0

---
 llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
index f1732ebe09250..cb62a0b57a173 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
@@ -4,5 +4,8 @@ static_library("HLSL") {
     "//llvm/lib/IR",
     "//llvm/lib/Support",
   ]
-  sources = [ "HLSLResource.cpp" ]
+  sources = [
+    "CBuffer.cpp",
+    "HLSLResource.cpp",
+  ]
 }

From bf7f602e7d549d9933a8a5f073e65917e7c877de Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 17 Apr 2025 03:09:19 +0000
Subject: [PATCH 212/710] [gn build] Port b07c88563feb

---
 llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index 3185ece413dea..802f603ade2bc 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -52,6 +52,7 @@ unittest("SupportTests") {
     "HashBuilderTest.cpp",
     "IndexedAccessorTest.cpp",
     "InstructionCostTest.cpp",
+    "InterleavedRangeTest.cpp",
     "JSONTest.cpp",
     "KnownBitsTest.cpp",
     "LEB128Test.cpp",

From e016a90e0bcfbb61f5523696c6b2ad3d6db2579b Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental@gmail.com>
Date: Wed, 16 Apr 2025 23:54:35 -0400
Subject: [PATCH 213/710] [mlir][SMT] add missing ExportSMTLIB tests (#136069)

Whoops forgot these -export-smtlib tests.

---------

Co-authored-by: Bea Healy <beahealy22@gmail.com>
Co-authored-by: Martin Erhart <maerhart@outlook.com>
Co-authored-by: Mike Urbach <mikeurbach@gmail.com>
Co-authored-by: Will Dietz <will.dietz@sifive.com>
Co-authored-by: fzi-hielscher <hielscher@fzi.de>
Co-authored-by: Fehr Mathieu <mathieu.fehr@gmail.com>
Co-authored-by: Clo91eaf <Clo91eaf@qq.com>
---
 mlir/test/Target/ExportSMTLIB/attributes.mlir | 143 +++++++++++++
 mlir/test/Target/ExportSMTLIB/basic.mlir      | 191 ++++++++++++++++++
 mlir/test/Target/ExportSMTLIB/lit.local.cfg   |   5 +
 3 files changed, 339 insertions(+)
 create mode 100644 mlir/test/Target/ExportSMTLIB/attributes.mlir
 create mode 100644 mlir/test/Target/ExportSMTLIB/basic.mlir
 create mode 100644 mlir/test/Target/ExportSMTLIB/lit.local.cfg

diff --git a/mlir/test/Target/ExportSMTLIB/attributes.mlir b/mlir/test/Target/ExportSMTLIB/attributes.mlir
new file mode 100644
index 0000000000000..2ecbd145acbe3
--- /dev/null
+++ b/mlir/test/Target/ExportSMTLIB/attributes.mlir
@@ -0,0 +1,143 @@
+// REQUIRES: z3-prover
+// RUN: mlir-translate --export-smtlib %s | z3 -in 2>&1 | FileCheck %s
+// RUN: mlir-translate --export-smtlib --smtlibexport-inline-single-use-values %s | z3 -in 2>&1 | FileCheck %s
+
+// Quantifiers Attributes
+smt.solver () : () -> () {
+
+  %true = smt.constant true
+
+  %7 = smt.exists weight 2 {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %arg3 : !smt.int
+    %5 = smt.implies %4, %true
+    smt.yield %5 : !smt.bool
+  }
+  smt.assert %7
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+// Quantifiers Attributes
+smt.solver () : () -> () {
+
+  %true = smt.constant true
+
+  %7 = smt.exists weight 2 {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %arg3 : !smt.int
+    %5 = smt.implies %4, %true
+    smt.yield %5 : !smt.bool
+  } patterns {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %arg3 : !smt.int
+    smt.yield %4: !smt.bool
+  }
+  smt.assert %7
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+// Quantifiers Attributes
+smt.solver () : () -> () {
+
+  %true = smt.constant true
+
+  %7 = smt.exists {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %arg3 : !smt.int
+    %5 = smt.implies %4, %true
+    smt.yield %5 : !smt.bool
+  } patterns {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %arg3 : !smt.int
+    smt.yield %4: !smt.bool
+  }
+  smt.assert %7
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+smt.solver () : () -> () {
+
+  %true = smt.constant true
+  %three = smt.int.constant 3
+  %four = smt.int.constant 4
+
+  %7 = smt.exists {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %three: !smt.int
+    %5 = smt.eq %arg3, %four: !smt.int
+    %6 = smt.eq %4, %5: !smt.bool
+    smt.yield %6 : !smt.bool
+  } patterns {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %three: !smt.int
+    smt.yield %4: !smt.bool}, {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %5 = smt.eq %arg3, %four: !smt.int
+    smt.yield %5: !smt.bool
+  }
+  smt.assert %7
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+smt.solver () : () -> () {
+
+  %true = smt.constant true
+  %three = smt.int.constant 3
+  %four = smt.int.constant 4
+
+  %10 = smt.exists {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %three: !smt.int
+    %5 = smt.eq %arg3, %four: !smt.int
+    %9 = smt.eq %4, %5: !smt.bool
+    smt.yield %9 : !smt.bool
+  } patterns {
+    ^bb0(%arg2: !smt.int, %arg3: !smt.int):
+    %4 = smt.eq %arg2, %three: !smt.int
+    %5 = smt.eq %arg3, %four: !smt.int
+    smt.yield %4, %5: !smt.bool, !smt.bool
+  }
+  smt.assert %10
+
+  smt.check sat {} unknown {} unsat {}
+
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
diff --git a/mlir/test/Target/ExportSMTLIB/basic.mlir b/mlir/test/Target/ExportSMTLIB/basic.mlir
new file mode 100644
index 0000000000000..81c23cde0f9fd
--- /dev/null
+++ b/mlir/test/Target/ExportSMTLIB/basic.mlir
@@ -0,0 +1,191 @@
+// REQUIRES: z3-prover
+// RUN: mlir-translate --export-smtlib %s | z3 -in 2>&1 | FileCheck %s
+// RUN: mlir-translate --export-smtlib --smtlibexport-inline-single-use-values %s | z3 -in 2>&1 | FileCheck %s
+
+// Function and constant symbol declarations, uninterpreted sorts
+smt.solver () : () -> () {
+  %0 = smt.declare_fun "b" : !smt.bv<32>
+  %1 = smt.declare_fun : !smt.int
+  %2 = smt.declare_fun : !smt.func<(!smt.int, !smt.bv<32>) !smt.bool>
+  %3 = smt.apply_func %2(%1, %0) : !smt.func<(!smt.int, !smt.bv<32>) !smt.bool>
+  smt.assert %3
+
+  %4 = smt.declare_fun : !smt.sort<"uninterpreted">
+  %5 = smt.declare_fun : !smt.sort<"other"[!smt.sort<"uninterpreted">]>
+  %6 = smt.declare_fun : !smt.func<(!smt.sort<"other"[!smt.sort<"uninterpreted">]>, !smt.sort<"uninterpreted">) !smt.bool>
+  %7 = smt.apply_func %6(%5, %4) : !smt.func<(!smt.sort<"other"[!smt.sort<"uninterpreted">]>, !smt.sort<"uninterpreted">) !smt.bool>
+  smt.assert %7
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+// Big expression
+smt.solver () : () -> () {
+  %true = smt.constant true
+  %false = smt.constant false
+  %0 = smt.not %true
+  %1 = smt.and %0, %true, %false
+  %2 = smt.or %1, %0, %true
+  %3 = smt.xor %0, %2
+  %4 = smt.implies %3, %false
+  %5 = smt.implies %4, %true
+  smt.assert %5
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+// Constant array
+smt.solver () : () -> () {
+  %true = smt.constant true
+  %false = smt.constant false
+  %c = smt.int.constant 0
+  %0 = smt.array.broadcast %true : !smt.array<[!smt.int -> !smt.bool]>
+  %1 = smt.array.store %0[%c], %false : !smt.array<[!smt.int -> !smt.bool]>
+  %2 = smt.array.select %1[%c] : !smt.array<[!smt.int -> !smt.bool]>
+  smt.assert %2
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK: unsat
+
+// BitVector extract and concat, and constants
+smt.solver () : () -> () {
+  %xf = smt.bv.constant #smt.bv<0xf> : !smt.bv<4>
+  %x0 = smt.bv.constant #smt.bv<0> : !smt.bv<4>
+  %xff = smt.bv.constant #smt.bv<0xff> : !smt.bv<8>
+
+  %0 = smt.bv.concat %xf, %x0 : !smt.bv<4>, !smt.bv<4>
+  %1 = smt.bv.extract %0 from 4 : (!smt.bv<8>) -> !smt.bv<4>
+  %2 = smt.bv.repeat 2 times %1 : !smt.bv<4>
+  %3 = smt.eq %2, %xff : !smt.bv<8>
+  smt.assert %3
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+// Quantifiers
+smt.solver () : () -> () {
+  %1 = smt.forall ["a"] {
+  ^bb0(%arg2: !smt.int):
+    %c2_1 = smt.int.constant 2
+    %2 = smt.int.mul %arg2, %c2_1
+
+    %3 = smt.exists {
+    ^bb0(%arg1: !smt.int):
+      %c2 = smt.int.constant 2
+      %4 = smt.int.mul %c2, %arg1
+      %5 = smt.eq %4, %2 : !smt.int
+      smt.yield %5 : !smt.bool
+    }
+
+    smt.yield %3 : !smt.bool
+  }
+  smt.assert %1
+
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+// Push and pop
+smt.solver () : () -> () {
+  %three = smt.int.constant 3
+  %four = smt.int.constant 4
+  %unsat_eq = smt.eq %three, %four : !smt.int
+  %sat_eq = smt.eq %three, %three : !smt.int
+
+  smt.push 1
+  smt.assert %unsat_eq
+  smt.check sat {} unknown {} unsat {}
+  smt.pop 1
+  smt.assert %sat_eq
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: sat
+// CHECK: unsat
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+// Reset
+smt.solver () : () -> () {
+  %three = smt.int.constant 3
+  %four = smt.int.constant 4
+  %unsat_eq = smt.eq %three, %four : !smt.int
+  %sat_eq = smt.eq %three, %three : !smt.int
+
+  smt.assert %unsat_eq
+  smt.check sat {} unknown {} unsat {}
+  smt.reset
+  smt.assert %sat_eq
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: sat
+// CHECK: unsat
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK: sat
+
+smt.solver () : () -> () {
+  smt.set_logic "HORN"
+  %c = smt.declare_fun : !smt.int
+  %c4 = smt.int.constant 4
+  %eq = smt.eq %c, %c4 : !smt.int
+  smt.assert %eq
+  smt.check sat {} unknown {} unsat {}
+}
+
+// CHECK-NOT: WARNING
+// CHECK-NOT: warning
+// CHECK-NOT: ERROR
+// CHECK-NOT: error
+// CHECK-NOT: unsat
+// CHECK-NOT: sat
+// CHECK: unknown
diff --git a/mlir/test/Target/ExportSMTLIB/lit.local.cfg b/mlir/test/Target/ExportSMTLIB/lit.local.cfg
new file mode 100644
index 0000000000000..ecc28e19f5b95
--- /dev/null
+++ b/mlir/test/Target/ExportSMTLIB/lit.local.cfg
@@ -0,0 +1,5 @@
+import lit.util
+
+if lit.util.which("z3", config.environment["PATH"]):
+    config.available_features.add("z3-prover")
+

From c11c2fe1ae2fe7b88135c51c32a66d0e30fe5431 Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Wed, 16 Apr 2025 21:14:00 -0700
Subject: [PATCH 214/710] [NVPTX] Lower i1 select with logical ops in the
 general case (#135868)

Update i1 select lowering to use an expansion based on logical ops,
unless the selected operands are truncations. This can improve generated
code quality by exposing additional potential optimizations.
---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp |  52 ++--
 llvm/lib/Target/NVPTX/NVPTXISelLowering.h   |   2 -
 llvm/test/CodeGen/NVPTX/bug22246.ll         |  20 +-
 llvm/test/CodeGen/NVPTX/i1-select.ll        | 128 +++++++++
 llvm/test/CodeGen/NVPTX/i128.ll             | 300 ++++++++++----------
 5 files changed, 325 insertions(+), 177 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/i1-select.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 9bde2a976e164..abe4c27009698 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -2850,6 +2850,40 @@ static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG,
   return DAG.getSelect(DL, Ty, IsInf, X, Sub);
 }
 
+static SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) {
+  assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1");
+
+  SDValue Cond = Op->getOperand(0);
+  SDValue TrueVal = Op->getOperand(1);
+  SDValue FalseVal = Op->getOperand(2);
+  SDLoc DL(Op);
+
+  // If both operands are truncated, we push the select through the truncates.
+  if (TrueVal.getOpcode() == ISD::TRUNCATE &&
+      FalseVal.getOpcode() == ISD::TRUNCATE) {
+    TrueVal = TrueVal.getOperand(0);
+    FalseVal = FalseVal.getOperand(0);
+
+    EVT VT = TrueVal.getSimpleValueType().bitsLE(FalseVal.getSimpleValueType())
+                 ? TrueVal.getValueType()
+                 : FalseVal.getValueType();
+    TrueVal = DAG.getAnyExtOrTrunc(TrueVal, DL, VT);
+    FalseVal = DAG.getAnyExtOrTrunc(FalseVal, DL, VT);
+    SDValue Select = DAG.getSelect(DL, VT, Cond, TrueVal, FalseVal);
+    return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Select);
+  }
+
+  // Otherwise, expand the select into a series of logical operations. These
+  // often can be folded into other operations either by us or ptxas.
+  TrueVal = DAG.getFreeze(TrueVal);
+  FalseVal = DAG.getFreeze(FalseVal);
+  SDValue And1 = DAG.getNode(ISD::AND, DL, MVT::i1, Cond, TrueVal);
+  SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
+  SDValue And2 = DAG.getNode(ISD::AND, DL, MVT::i1, NotCond, FalseVal);
+  SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i1, And1, And2);
+  return Or;
+}
+
 SDValue
 NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
@@ -2889,7 +2923,7 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SRL_PARTS:
     return LowerShiftRightParts(Op, DAG);
   case ISD::SELECT:
-    return LowerSelect(Op, DAG);
+    return lowerSELECT(Op, DAG);
   case ISD::FROUND:
     return LowerFROUND(Op, DAG);
   case ISD::FCOPYSIGN:
@@ -3056,22 +3090,6 @@ SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
                       MachinePointerInfo(SV));
 }
 
-SDValue NVPTXTargetLowering::LowerSelect(SDValue Op, SelectionDAG &DAG) const {
-  SDValue Op0 = Op->getOperand(0);
-  SDValue Op1 = Op->getOperand(1);
-  SDValue Op2 = Op->getOperand(2);
-  SDLoc DL(Op.getNode());
-
-  assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1");
-
-  Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
-  Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
-  SDValue Select = DAG.getNode(ISD::SELECT, DL, MVT::i32, Op0, Op1, Op2);
-  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Select);
-
-  return Trunc;
-}
-
 SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   if (Op.getValueType() == MVT::i1)
     return LowerLOADi1(Op, DAG);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index dd90746f6d9d6..7a8bf3bf33a94 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -324,8 +324,6 @@ class NVPTXTargetLowering : public TargetLowering {
   SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
 
-  SDValue LowerSelect(SDValue Op, SelectionDAG &DAG) const;
-
   SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/NVPTX/bug22246.ll b/llvm/test/CodeGen/NVPTX/bug22246.ll
index e0051d33f0b30..0080aafcf5631 100644
--- a/llvm/test/CodeGen/NVPTX/bug22246.ll
+++ b/llvm/test/CodeGen/NVPTX/bug22246.ll
@@ -1,13 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
 target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
 target triple = "nvptx64-nvidia-cuda"
 
-; CHECK-LABEL: _Z3foobbbPb
 define void @_Z3foobbbPb(i1 zeroext %p1, i1 zeroext %p2, i1 zeroext %p3, ptr nocapture %output) {
+; CHECK-LABEL: _Z3foobbbPb(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<7>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.u8 %rs1, [_Z3foobbbPb_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.u8 %rs3, [_Z3foobbbPb_param_1];
+; CHECK-NEXT:    ld.param.u8 %rs4, [_Z3foobbbPb_param_2];
+; CHECK-NEXT:    selp.b16 %rs5, %rs3, %rs4, %p1;
+; CHECK-NEXT:    and.b16 %rs6, %rs5, 1;
+; CHECK-NEXT:    ld.param.u64 %rd1, [_Z3foobbbPb_param_3];
+; CHECK-NEXT:    st.u8 [%rd1], %rs6;
+; CHECK-NEXT:    ret;
 entry:
-; CHECK: selp.b32       %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %p{{[0-9]+}}
   %.sink.v = select i1 %p1, i1 %p2, i1 %p3
   %frombool5 = zext i1 %.sink.v to i8
   store i8 %frombool5, ptr %output, align 1
diff --git a/llvm/test/CodeGen/NVPTX/i1-select.ll b/llvm/test/CodeGen/NVPTX/i1-select.ll
new file mode 100644
index 0000000000000..d24b06c4d721c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/i1-select.ll
@@ -0,0 +1,128 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
+
+target triple = "nvptx-nvidia-cuda"
+
+define i32 @test_select_i1_trunc(i32 %a, i32 %b, i32 %c, i32 %true, i32 %false) {
+; CHECK-LABEL: test_select_i1_trunc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<3>;
+; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [test_select_i1_trunc_param_0];
+; CHECK-NEXT:    and.b32 %r2, %r1, 1;
+; CHECK-NEXT:    setp.ne.b32 %p1, %r2, 0;
+; CHECK-NEXT:    ld.param.u32 %r3, [test_select_i1_trunc_param_1];
+; CHECK-NEXT:    ld.param.u32 %r4, [test_select_i1_trunc_param_2];
+; CHECK-NEXT:    ld.param.u32 %r5, [test_select_i1_trunc_param_3];
+; CHECK-NEXT:    selp.b32 %r6, %r3, %r4, %p1;
+; CHECK-NEXT:    and.b32 %r7, %r6, 1;
+; CHECK-NEXT:    setp.ne.b32 %p2, %r7, 0;
+; CHECK-NEXT:    ld.param.u32 %r8, [test_select_i1_trunc_param_4];
+; CHECK-NEXT:    selp.b32 %r9, %r5, %r8, %p2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-NEXT:    ret;
+  %a_trunc = trunc i32 %a to i1
+  %b_trunc = trunc i32 %b to i1
+  %c_trunc = trunc i32 %c to i1
+  %select_i1 = select i1 %a_trunc, i1 %b_trunc, i1 %c_trunc
+  %select_ret = select i1 %select_i1, i32 %true, i32 %false
+  ret i32 %select_ret
+}
+
+define i32 @test_select_i1_trunc_2(i64 %a, i16 %b, i32 %c, i32 %true, i32 %false) {
+; CHECK-LABEL: test_select_i1_trunc_2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<3>;
+; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_select_i1_trunc_2_param_0];
+; CHECK-NEXT:    and.b64 %rd2, %rd1, 1;
+; CHECK-NEXT:    setp.ne.b64 %p1, %rd2, 0;
+; CHECK-NEXT:    ld.param.u16 %rs1, [test_select_i1_trunc_2_param_1];
+; CHECK-NEXT:    ld.param.u16 %rs2, [test_select_i1_trunc_2_param_2];
+; CHECK-NEXT:    ld.param.u32 %r1, [test_select_i1_trunc_2_param_3];
+; CHECK-NEXT:    selp.b16 %rs3, %rs1, %rs2, %p1;
+; CHECK-NEXT:    and.b16 %rs4, %rs3, 1;
+; CHECK-NEXT:    setp.ne.b16 %p2, %rs4, 0;
+; CHECK-NEXT:    ld.param.u32 %r2, [test_select_i1_trunc_2_param_4];
+; CHECK-NEXT:    selp.b32 %r3, %r1, %r2, %p2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %a_trunc = trunc i64 %a to i1
+  %b_trunc = trunc i16 %b to i1
+  %c_trunc = trunc i32 %c to i1
+  %select_i1 = select i1 %a_trunc, i1 %b_trunc, i1 %c_trunc
+  %select_ret = select i1 %select_i1, i32 %true, i32 %false
+  ret i32 %select_ret
+}
+
+define i32 @test_select_i1_basic(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %false) {
+; CHECK-LABEL: test_select_i1_basic(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b32 %r<12>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [test_select_i1_basic_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [test_select_i1_basic_param_1];
+; CHECK-NEXT:    or.b32 %r4, %r1, %r2;
+; CHECK-NEXT:    setp.ne.s32 %p1, %r1, 0;
+; CHECK-NEXT:    ld.param.u32 %r5, [test_select_i1_basic_param_2];
+; CHECK-NEXT:    setp.eq.s32 %p2, %r5, 0;
+; CHECK-NEXT:    ld.param.u32 %r7, [test_select_i1_basic_param_3];
+; CHECK-NEXT:    setp.eq.s32 %p3, %r4, 0;
+; CHECK-NEXT:    ld.param.u32 %r8, [test_select_i1_basic_param_4];
+; CHECK-NEXT:    selp.b32 %r9, %r7, %r8, %p2;
+; CHECK-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; CHECK-NEXT:    selp.b32 %r11, %r7, %r10, %p3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r11;
+; CHECK-NEXT:    ret;
+  %b1 = icmp eq i32 %v1, 0
+  %b2 = icmp eq i32 %v2, 0
+  %b3 = icmp eq i32 %v3, 0
+  %select_i1 = select i1 %b1, i1 %b2, i1 %b3
+  %select_ret = select i1 %select_i1, i32 %true, i32 %false
+  ret i32 %select_ret
+}
+
+define i32 @test_select_i1_basic_folding(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %false) {
+; CHECK-LABEL: test_select_i1_basic_folding(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<13>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [test_select_i1_basic_folding_param_0];
+; CHECK-NEXT:    setp.eq.s32 %p1, %r1, 0;
+; CHECK-NEXT:    ld.param.u32 %r2, [test_select_i1_basic_folding_param_1];
+; CHECK-NEXT:    setp.ne.s32 %p2, %r2, 0;
+; CHECK-NEXT:    setp.eq.s32 %p3, %r2, 0;
+; CHECK-NEXT:    ld.param.u32 %r3, [test_select_i1_basic_folding_param_2];
+; CHECK-NEXT:    setp.eq.s32 %p4, %r3, 0;
+; CHECK-NEXT:    ld.param.u32 %r4, [test_select_i1_basic_folding_param_3];
+; CHECK-NEXT:    xor.pred %p6, %p1, %p3;
+; CHECK-NEXT:    ld.param.u32 %r5, [test_select_i1_basic_folding_param_4];
+; CHECK-NEXT:    and.pred %p7, %p6, %p4;
+; CHECK-NEXT:    and.pred %p9, %p2, %p4;
+; CHECK-NEXT:    and.pred %p10, %p3, %p7;
+; CHECK-NEXT:    or.pred %p11, %p10, %p9;
+; CHECK-NEXT:    xor.pred %p12, %p11, %p3;
+; CHECK-NEXT:    selp.b32 %r6, %r4, %r5, %p12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
+  %b1 = icmp eq i32 %v1, 0
+  %b2 = icmp eq i32 %v2, 0
+  %b3 = icmp eq i32 %v3, 0
+  %b4 = xor i1 %b1, %b2
+  %b5 = and i1 %b4, %b3
+  %select_i1 = select i1 %b2, i1 %b5, i1 %b3
+  %b6 = xor i1 %select_i1, %b2
+  %select_ret = select i1 %b6, i32 %true, i32 %false
+  ret i32 %select_ret
+}
diff --git a/llvm/test/CodeGen/NVPTX/i128.ll b/llvm/test/CodeGen/NVPTX/i128.ll
index bf6189c280191..64786e601c4b5 100644
--- a/llvm/test/CodeGen/NVPTX/i128.ll
+++ b/llvm/test/CodeGen/NVPTX/i128.ll
@@ -5,8 +5,8 @@
 define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-LABEL: srem_i128(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<19>;
-; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-NEXT:    .reg .pred %p<20>;
+; CHECK-NEXT:    .reg .b32 %r<12>;
 ; CHECK-NEXT:    .reg .b64 %rd<127>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
@@ -45,51 +45,48 @@ define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    selp.b64 %rd65, %rd62, %rd64, %p7;
 ; CHECK-NEXT:    sub.cc.s64 %rd66, %rd61, %rd65;
 ; CHECK-NEXT:    subc.cc.s64 %rd67, %rd117, 0;
-; CHECK-NEXT:    setp.eq.s64 %p8, %rd67, 0;
-; CHECK-NEXT:    setp.ne.s64 %p9, %rd67, 0;
-; CHECK-NEXT:    selp.b32 %r5, -1, 0, %p9;
-; CHECK-NEXT:    setp.gt.u64 %p10, %rd66, 127;
-; CHECK-NEXT:    selp.b32 %r6, -1, 0, %p10;
-; CHECK-NEXT:    selp.b32 %r7, %r6, %r5, %p8;
-; CHECK-NEXT:    and.b32 %r8, %r7, 1;
-; CHECK-NEXT:    setp.ne.b32 %p11, %r8, 0;
-; CHECK-NEXT:    or.pred %p12, %p5, %p11;
+; CHECK-NEXT:    setp.gt.u64 %p8, %rd66, 127;
+; CHECK-NEXT:    setp.eq.s64 %p9, %rd67, 0;
+; CHECK-NEXT:    and.pred %p10, %p9, %p8;
+; CHECK-NEXT:    setp.ne.s64 %p11, %rd67, 0;
+; CHECK-NEXT:    or.pred %p12, %p10, %p11;
+; CHECK-NEXT:    or.pred %p13, %p5, %p12;
 ; CHECK-NEXT:    xor.b64 %rd68, %rd66, 127;
 ; CHECK-NEXT:    or.b64 %rd69, %rd68, %rd67;
-; CHECK-NEXT:    setp.eq.s64 %p13, %rd69, 0;
-; CHECK-NEXT:    selp.b64 %rd126, 0, %rd4, %p12;
-; CHECK-NEXT:    selp.b64 %rd125, 0, %rd3, %p12;
-; CHECK-NEXT:    or.pred %p14, %p12, %p13;
-; CHECK-NEXT:    @%p14 bra $L__BB0_5;
+; CHECK-NEXT:    setp.eq.s64 %p14, %rd69, 0;
+; CHECK-NEXT:    selp.b64 %rd126, 0, %rd4, %p13;
+; CHECK-NEXT:    selp.b64 %rd125, 0, %rd3, %p13;
+; CHECK-NEXT:    or.pred %p15, %p13, %p14;
+; CHECK-NEXT:    @%p15 bra $L__BB0_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
 ; CHECK-NEXT:    add.cc.s64 %rd119, %rd66, 1;
 ; CHECK-NEXT:    addc.cc.s64 %rd120, %rd67, 0;
 ; CHECK-NEXT:    or.b64 %rd72, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.s64 %p15, %rd72, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd66;
-; CHECK-NEXT:    sub.s32 %r10, 127, %r9;
-; CHECK-NEXT:    shl.b64 %rd73, %rd4, %r10;
-; CHECK-NEXT:    sub.s32 %r11, 64, %r10;
-; CHECK-NEXT:    shr.u64 %rd74, %rd3, %r11;
+; CHECK-NEXT:    setp.eq.s64 %p16, %rd72, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd66;
+; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
+; CHECK-NEXT:    shl.b64 %rd73, %rd4, %r6;
+; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
+; CHECK-NEXT:    shr.u64 %rd74, %rd3, %r7;
 ; CHECK-NEXT:    or.b64 %rd75, %rd73, %rd74;
-; CHECK-NEXT:    sub.s32 %r12, 63, %r9;
-; CHECK-NEXT:    shl.b64 %rd76, %rd3, %r12;
-; CHECK-NEXT:    setp.gt.s32 %p16, %r10, 63;
-; CHECK-NEXT:    selp.b64 %rd124, %rd76, %rd75, %p16;
-; CHECK-NEXT:    shl.b64 %rd123, %rd3, %r10;
+; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
+; CHECK-NEXT:    shl.b64 %rd76, %rd3, %r8;
+; CHECK-NEXT:    setp.gt.s32 %p17, %r6, 63;
+; CHECK-NEXT:    selp.b64 %rd124, %rd76, %rd75, %p17;
+; CHECK-NEXT:    shl.b64 %rd123, %rd3, %r6;
 ; CHECK-NEXT:    mov.b64 %rd114, %rd117;
-; CHECK-NEXT:    @%p15 bra $L__BB0_4;
+; CHECK-NEXT:    @%p16 bra $L__BB0_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r13, %rd119;
-; CHECK-NEXT:    shr.u64 %rd79, %rd3, %r13;
-; CHECK-NEXT:    sub.s32 %r14, 64, %r13;
-; CHECK-NEXT:    shl.b64 %rd80, %rd4, %r14;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd119;
+; CHECK-NEXT:    shr.u64 %rd79, %rd3, %r9;
+; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
+; CHECK-NEXT:    shl.b64 %rd80, %rd4, %r10;
 ; CHECK-NEXT:    or.b64 %rd81, %rd79, %rd80;
-; CHECK-NEXT:    add.s32 %r15, %r13, -64;
-; CHECK-NEXT:    shr.u64 %rd82, %rd4, %r15;
-; CHECK-NEXT:    setp.gt.s32 %p17, %r13, 63;
-; CHECK-NEXT:    selp.b64 %rd121, %rd82, %rd81, %p17;
-; CHECK-NEXT:    shr.u64 %rd122, %rd4, %r13;
+; CHECK-NEXT:    add.s32 %r11, %r9, -64;
+; CHECK-NEXT:    shr.u64 %rd82, %rd4, %r11;
+; CHECK-NEXT:    setp.gt.s32 %p18, %r9, 63;
+; CHECK-NEXT:    selp.b64 %rd121, %rd82, %rd81, %p18;
+; CHECK-NEXT:    shr.u64 %rd122, %rd4, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd35, %rd5, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd36, %rd6, -1;
 ; CHECK-NEXT:    mov.b64 %rd114, 0;
@@ -119,8 +116,8 @@ define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    add.cc.s64 %rd119, %rd119, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd120, %rd120, -1;
 ; CHECK-NEXT:    or.b64 %rd98, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.s64 %p18, %rd98, 0;
-; CHECK-NEXT:    @%p18 bra $L__BB0_4;
+; CHECK-NEXT:    setp.eq.s64 %p19, %rd98, 0;
+; CHECK-NEXT:    @%p19 bra $L__BB0_4;
 ; CHECK-NEXT:    bra.uni $L__BB0_2;
 ; CHECK-NEXT:  $L__BB0_4: // %udiv-loop-exit
 ; CHECK-NEXT:    shr.u64 %rd99, %rd123, 63;
@@ -149,8 +146,8 @@ define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-LABEL: urem_i128(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<17>;
-; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-NEXT:    .reg .pred %p<18>;
+; CHECK-NEXT:    .reg .b32 %r<12>;
 ; CHECK-NEXT:    .reg .b64 %rd<113>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
@@ -178,51 +175,48 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    mov.b64 %rd103, 0;
 ; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
 ; CHECK-NEXT:    subc.cc.s64 %rd57, %rd103, 0;
-; CHECK-NEXT:    setp.eq.s64 %p6, %rd57, 0;
-; CHECK-NEXT:    setp.ne.s64 %p7, %rd57, 0;
-; CHECK-NEXT:    selp.b32 %r5, -1, 0, %p7;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd56, 127;
-; CHECK-NEXT:    selp.b32 %r6, -1, 0, %p8;
-; CHECK-NEXT:    selp.b32 %r7, %r6, %r5, %p6;
-; CHECK-NEXT:    and.b32 %r8, %r7, 1;
-; CHECK-NEXT:    setp.ne.b32 %p9, %r8, 0;
-; CHECK-NEXT:    or.pred %p10, %p3, %p9;
+; CHECK-NEXT:    setp.gt.u64 %p6, %rd56, 127;
+; CHECK-NEXT:    setp.eq.s64 %p7, %rd57, 0;
+; CHECK-NEXT:    and.pred %p8, %p7, %p6;
+; CHECK-NEXT:    setp.ne.s64 %p9, %rd57, 0;
+; CHECK-NEXT:    or.pred %p10, %p8, %p9;
+; CHECK-NEXT:    or.pred %p11, %p3, %p10;
 ; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
 ; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
-; CHECK-NEXT:    setp.eq.s64 %p11, %rd59, 0;
-; CHECK-NEXT:    selp.b64 %rd112, 0, %rd42, %p10;
-; CHECK-NEXT:    selp.b64 %rd111, 0, %rd41, %p10;
-; CHECK-NEXT:    or.pred %p12, %p10, %p11;
-; CHECK-NEXT:    @%p12 bra $L__BB1_5;
+; CHECK-NEXT:    setp.eq.s64 %p12, %rd59, 0;
+; CHECK-NEXT:    selp.b64 %rd112, 0, %rd42, %p11;
+; CHECK-NEXT:    selp.b64 %rd111, 0, %rd41, %p11;
+; CHECK-NEXT:    or.pred %p13, %p11, %p12;
+; CHECK-NEXT:    @%p13 bra $L__BB1_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
 ; CHECK-NEXT:    add.cc.s64 %rd105, %rd56, 1;
 ; CHECK-NEXT:    addc.cc.s64 %rd106, %rd57, 0;
 ; CHECK-NEXT:    or.b64 %rd62, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.s64 %p13, %rd62, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd56;
-; CHECK-NEXT:    sub.s32 %r10, 127, %r9;
-; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r10;
-; CHECK-NEXT:    sub.s32 %r11, 64, %r10;
-; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r11;
+; CHECK-NEXT:    setp.eq.s64 %p14, %rd62, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd56;
+; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
+; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r6;
+; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
+; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r7;
 ; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
-; CHECK-NEXT:    sub.s32 %r12, 63, %r9;
-; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r12;
-; CHECK-NEXT:    setp.gt.s32 %p14, %r10, 63;
-; CHECK-NEXT:    selp.b64 %rd110, %rd66, %rd65, %p14;
-; CHECK-NEXT:    shl.b64 %rd109, %rd41, %r10;
+; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
+; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r8;
+; CHECK-NEXT:    setp.gt.s32 %p15, %r6, 63;
+; CHECK-NEXT:    selp.b64 %rd110, %rd66, %rd65, %p15;
+; CHECK-NEXT:    shl.b64 %rd109, %rd41, %r6;
 ; CHECK-NEXT:    mov.b64 %rd100, %rd103;
-; CHECK-NEXT:    @%p13 bra $L__BB1_4;
+; CHECK-NEXT:    @%p14 bra $L__BB1_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r13, %rd105;
-; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r13;
-; CHECK-NEXT:    sub.s32 %r14, 64, %r13;
-; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r14;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd105;
+; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r9;
+; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
+; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r10;
 ; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
-; CHECK-NEXT:    add.s32 %r15, %r13, -64;
-; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r15;
-; CHECK-NEXT:    setp.gt.s32 %p15, %r13, 63;
-; CHECK-NEXT:    selp.b64 %rd107, %rd72, %rd71, %p15;
-; CHECK-NEXT:    shr.u64 %rd108, %rd42, %r13;
+; CHECK-NEXT:    add.s32 %r11, %r9, -64;
+; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r11;
+; CHECK-NEXT:    setp.gt.s32 %p16, %r9, 63;
+; CHECK-NEXT:    selp.b64 %rd107, %rd72, %rd71, %p16;
+; CHECK-NEXT:    shr.u64 %rd108, %rd42, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd33, %rd3, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd34, %rd4, -1;
 ; CHECK-NEXT:    mov.b64 %rd100, 0;
@@ -252,8 +246,8 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    add.cc.s64 %rd105, %rd105, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd106, %rd106, -1;
 ; CHECK-NEXT:    or.b64 %rd88, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.s64 %p16, %rd88, 0;
-; CHECK-NEXT:    @%p16 bra $L__BB1_4;
+; CHECK-NEXT:    setp.eq.s64 %p17, %rd88, 0;
+; CHECK-NEXT:    @%p17 bra $L__BB1_4;
 ; CHECK-NEXT:    bra.uni $L__BB1_2;
 ; CHECK-NEXT:  $L__BB1_4: // %udiv-loop-exit
 ; CHECK-NEXT:    shr.u64 %rd89, %rd109, 63;
@@ -313,8 +307,8 @@ define i128 @urem_i128_pow2k(i128 %lhs) {
 define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-LABEL: sdiv_i128(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<19>;
-; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-NEXT:    .reg .pred %p<20>;
+; CHECK-NEXT:    .reg .b32 %r<12>;
 ; CHECK-NEXT:    .reg .b64 %rd<122>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
@@ -354,51 +348,48 @@ define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    selp.b64 %rd66, %rd63, %rd65, %p7;
 ; CHECK-NEXT:    sub.cc.s64 %rd67, %rd62, %rd66;
 ; CHECK-NEXT:    subc.cc.s64 %rd68, %rd112, 0;
-; CHECK-NEXT:    setp.eq.s64 %p8, %rd68, 0;
-; CHECK-NEXT:    setp.ne.s64 %p9, %rd68, 0;
-; CHECK-NEXT:    selp.b32 %r5, -1, 0, %p9;
-; CHECK-NEXT:    setp.gt.u64 %p10, %rd67, 127;
-; CHECK-NEXT:    selp.b32 %r6, -1, 0, %p10;
-; CHECK-NEXT:    selp.b32 %r7, %r6, %r5, %p8;
-; CHECK-NEXT:    and.b32 %r8, %r7, 1;
-; CHECK-NEXT:    setp.ne.b32 %p11, %r8, 0;
-; CHECK-NEXT:    or.pred %p12, %p5, %p11;
+; CHECK-NEXT:    setp.gt.u64 %p8, %rd67, 127;
+; CHECK-NEXT:    setp.eq.s64 %p9, %rd68, 0;
+; CHECK-NEXT:    and.pred %p10, %p9, %p8;
+; CHECK-NEXT:    setp.ne.s64 %p11, %rd68, 0;
+; CHECK-NEXT:    or.pred %p12, %p10, %p11;
+; CHECK-NEXT:    or.pred %p13, %p5, %p12;
 ; CHECK-NEXT:    xor.b64 %rd69, %rd67, 127;
 ; CHECK-NEXT:    or.b64 %rd70, %rd69, %rd68;
-; CHECK-NEXT:    setp.eq.s64 %p13, %rd70, 0;
-; CHECK-NEXT:    selp.b64 %rd121, 0, %rd2, %p12;
-; CHECK-NEXT:    selp.b64 %rd120, 0, %rd1, %p12;
-; CHECK-NEXT:    or.pred %p14, %p12, %p13;
-; CHECK-NEXT:    @%p14 bra $L__BB4_5;
+; CHECK-NEXT:    setp.eq.s64 %p14, %rd70, 0;
+; CHECK-NEXT:    selp.b64 %rd121, 0, %rd2, %p13;
+; CHECK-NEXT:    selp.b64 %rd120, 0, %rd1, %p13;
+; CHECK-NEXT:    or.pred %p15, %p13, %p14;
+; CHECK-NEXT:    @%p15 bra $L__BB4_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
 ; CHECK-NEXT:    add.cc.s64 %rd114, %rd67, 1;
 ; CHECK-NEXT:    addc.cc.s64 %rd115, %rd68, 0;
 ; CHECK-NEXT:    or.b64 %rd73, %rd114, %rd115;
-; CHECK-NEXT:    setp.eq.s64 %p15, %rd73, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd67;
-; CHECK-NEXT:    sub.s32 %r10, 127, %r9;
-; CHECK-NEXT:    shl.b64 %rd74, %rd2, %r10;
-; CHECK-NEXT:    sub.s32 %r11, 64, %r10;
-; CHECK-NEXT:    shr.u64 %rd75, %rd1, %r11;
+; CHECK-NEXT:    setp.eq.s64 %p16, %rd73, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd67;
+; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
+; CHECK-NEXT:    shl.b64 %rd74, %rd2, %r6;
+; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
+; CHECK-NEXT:    shr.u64 %rd75, %rd1, %r7;
 ; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
-; CHECK-NEXT:    sub.s32 %r12, 63, %r9;
-; CHECK-NEXT:    shl.b64 %rd77, %rd1, %r12;
-; CHECK-NEXT:    setp.gt.s32 %p16, %r10, 63;
-; CHECK-NEXT:    selp.b64 %rd119, %rd77, %rd76, %p16;
-; CHECK-NEXT:    shl.b64 %rd118, %rd1, %r10;
+; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
+; CHECK-NEXT:    shl.b64 %rd77, %rd1, %r8;
+; CHECK-NEXT:    setp.gt.s32 %p17, %r6, 63;
+; CHECK-NEXT:    selp.b64 %rd119, %rd77, %rd76, %p17;
+; CHECK-NEXT:    shl.b64 %rd118, %rd1, %r6;
 ; CHECK-NEXT:    mov.b64 %rd109, %rd112;
-; CHECK-NEXT:    @%p15 bra $L__BB4_4;
+; CHECK-NEXT:    @%p16 bra $L__BB4_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r13, %rd114;
-; CHECK-NEXT:    shr.u64 %rd80, %rd1, %r13;
-; CHECK-NEXT:    sub.s32 %r14, 64, %r13;
-; CHECK-NEXT:    shl.b64 %rd81, %rd2, %r14;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd114;
+; CHECK-NEXT:    shr.u64 %rd80, %rd1, %r9;
+; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
+; CHECK-NEXT:    shl.b64 %rd81, %rd2, %r10;
 ; CHECK-NEXT:    or.b64 %rd82, %rd80, %rd81;
-; CHECK-NEXT:    add.s32 %r15, %r13, -64;
-; CHECK-NEXT:    shr.u64 %rd83, %rd2, %r15;
-; CHECK-NEXT:    setp.gt.s32 %p17, %r13, 63;
-; CHECK-NEXT:    selp.b64 %rd116, %rd83, %rd82, %p17;
-; CHECK-NEXT:    shr.u64 %rd117, %rd2, %r13;
+; CHECK-NEXT:    add.s32 %r11, %r9, -64;
+; CHECK-NEXT:    shr.u64 %rd83, %rd2, %r11;
+; CHECK-NEXT:    setp.gt.s32 %p18, %r9, 63;
+; CHECK-NEXT:    selp.b64 %rd116, %rd83, %rd82, %p18;
+; CHECK-NEXT:    shr.u64 %rd117, %rd2, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd35, %rd3, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd36, %rd4, -1;
 ; CHECK-NEXT:    mov.b64 %rd109, 0;
@@ -428,8 +419,8 @@ define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    add.cc.s64 %rd114, %rd114, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd115, %rd115, -1;
 ; CHECK-NEXT:    or.b64 %rd99, %rd114, %rd115;
-; CHECK-NEXT:    setp.eq.s64 %p18, %rd99, 0;
-; CHECK-NEXT:    @%p18 bra $L__BB4_4;
+; CHECK-NEXT:    setp.eq.s64 %p19, %rd99, 0;
+; CHECK-NEXT:    @%p19 bra $L__BB4_4;
 ; CHECK-NEXT:    bra.uni $L__BB4_2;
 ; CHECK-NEXT:  $L__BB4_4: // %udiv-loop-exit
 ; CHECK-NEXT:    shr.u64 %rd100, %rd118, 63;
@@ -452,8 +443,8 @@ define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-LABEL: udiv_i128(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<17>;
-; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-NEXT:    .reg .pred %p<18>;
+; CHECK-NEXT:    .reg .b32 %r<12>;
 ; CHECK-NEXT:    .reg .b64 %rd<107>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
@@ -481,51 +472,48 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    mov.b64 %rd97, 0;
 ; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
 ; CHECK-NEXT:    subc.cc.s64 %rd57, %rd97, 0;
-; CHECK-NEXT:    setp.eq.s64 %p6, %rd57, 0;
-; CHECK-NEXT:    setp.ne.s64 %p7, %rd57, 0;
-; CHECK-NEXT:    selp.b32 %r5, -1, 0, %p7;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd56, 127;
-; CHECK-NEXT:    selp.b32 %r6, -1, 0, %p8;
-; CHECK-NEXT:    selp.b32 %r7, %r6, %r5, %p6;
-; CHECK-NEXT:    and.b32 %r8, %r7, 1;
-; CHECK-NEXT:    setp.ne.b32 %p9, %r8, 0;
-; CHECK-NEXT:    or.pred %p10, %p3, %p9;
+; CHECK-NEXT:    setp.gt.u64 %p6, %rd56, 127;
+; CHECK-NEXT:    setp.eq.s64 %p7, %rd57, 0;
+; CHECK-NEXT:    and.pred %p8, %p7, %p6;
+; CHECK-NEXT:    setp.ne.s64 %p9, %rd57, 0;
+; CHECK-NEXT:    or.pred %p10, %p8, %p9;
+; CHECK-NEXT:    or.pred %p11, %p3, %p10;
 ; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
 ; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
-; CHECK-NEXT:    setp.eq.s64 %p11, %rd59, 0;
-; CHECK-NEXT:    selp.b64 %rd106, 0, %rd42, %p10;
-; CHECK-NEXT:    selp.b64 %rd105, 0, %rd41, %p10;
-; CHECK-NEXT:    or.pred %p12, %p10, %p11;
-; CHECK-NEXT:    @%p12 bra $L__BB5_5;
+; CHECK-NEXT:    setp.eq.s64 %p12, %rd59, 0;
+; CHECK-NEXT:    selp.b64 %rd106, 0, %rd42, %p11;
+; CHECK-NEXT:    selp.b64 %rd105, 0, %rd41, %p11;
+; CHECK-NEXT:    or.pred %p13, %p11, %p12;
+; CHECK-NEXT:    @%p13 bra $L__BB5_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
 ; CHECK-NEXT:    add.cc.s64 %rd99, %rd56, 1;
 ; CHECK-NEXT:    addc.cc.s64 %rd100, %rd57, 0;
 ; CHECK-NEXT:    or.b64 %rd62, %rd99, %rd100;
-; CHECK-NEXT:    setp.eq.s64 %p13, %rd62, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd56;
-; CHECK-NEXT:    sub.s32 %r10, 127, %r9;
-; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r10;
-; CHECK-NEXT:    sub.s32 %r11, 64, %r10;
-; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r11;
+; CHECK-NEXT:    setp.eq.s64 %p14, %rd62, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd56;
+; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
+; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r6;
+; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
+; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r7;
 ; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
-; CHECK-NEXT:    sub.s32 %r12, 63, %r9;
-; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r12;
-; CHECK-NEXT:    setp.gt.s32 %p14, %r10, 63;
-; CHECK-NEXT:    selp.b64 %rd104, %rd66, %rd65, %p14;
-; CHECK-NEXT:    shl.b64 %rd103, %rd41, %r10;
+; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
+; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r8;
+; CHECK-NEXT:    setp.gt.s32 %p15, %r6, 63;
+; CHECK-NEXT:    selp.b64 %rd104, %rd66, %rd65, %p15;
+; CHECK-NEXT:    shl.b64 %rd103, %rd41, %r6;
 ; CHECK-NEXT:    mov.b64 %rd94, %rd97;
-; CHECK-NEXT:    @%p13 bra $L__BB5_4;
+; CHECK-NEXT:    @%p14 bra $L__BB5_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r13, %rd99;
-; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r13;
-; CHECK-NEXT:    sub.s32 %r14, 64, %r13;
-; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r14;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd99;
+; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r9;
+; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
+; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r10;
 ; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
-; CHECK-NEXT:    add.s32 %r15, %r13, -64;
-; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r15;
-; CHECK-NEXT:    setp.gt.s32 %p15, %r13, 63;
-; CHECK-NEXT:    selp.b64 %rd101, %rd72, %rd71, %p15;
-; CHECK-NEXT:    shr.u64 %rd102, %rd42, %r13;
+; CHECK-NEXT:    add.s32 %r11, %r9, -64;
+; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r11;
+; CHECK-NEXT:    setp.gt.s32 %p16, %r9, 63;
+; CHECK-NEXT:    selp.b64 %rd101, %rd72, %rd71, %p16;
+; CHECK-NEXT:    shr.u64 %rd102, %rd42, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd33, %rd43, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd34, %rd44, -1;
 ; CHECK-NEXT:    mov.b64 %rd94, 0;
@@ -555,8 +543,8 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    add.cc.s64 %rd99, %rd99, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd100, %rd100, -1;
 ; CHECK-NEXT:    or.b64 %rd88, %rd99, %rd100;
-; CHECK-NEXT:    setp.eq.s64 %p16, %rd88, 0;
-; CHECK-NEXT:    @%p16 bra $L__BB5_4;
+; CHECK-NEXT:    setp.eq.s64 %p17, %rd88, 0;
+; CHECK-NEXT:    @%p17 bra $L__BB5_4;
 ; CHECK-NEXT:    bra.uni $L__BB5_2;
 ; CHECK-NEXT:  $L__BB5_4: // %udiv-loop-exit
 ; CHECK-NEXT:    shr.u64 %rd89, %rd103, 63;

From d647d66da697952f600d7a0d84fba94612fe51db Mon Sep 17 00:00:00 2001
From: Shoreshen <372660931@qq.com>
Date: Thu, 17 Apr 2025 12:14:34 +0800
Subject: [PATCH 215/710] [AMDGPU] Add illegal type convertion (#135729)

Add more bit-convert tests for illegal types conversion
---
 .../CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll   |  1466 +
 .../CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll   |  3246 +-
 .../CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll   |  1759 ++
 .../CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll   |  2047 ++
 .../CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll   |  7796 +++--
 .../CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll   |  2343 ++
 .../CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll   |  4954 ++-
 .../CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll   |  6080 +++-
 .../CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll   | 13506 ++++++++-
 .../CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll   | 15188 +++++++++-
 .../CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll   | 16936 ++++++++++-
 .../CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll   | 18939 +++++++++++-
 .../CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll   | 21001 ++++++++++++-
 .../CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll   | 22946 +++++++++++++-
 .../CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll   | 25063 +++++++++++++++-
 15 files changed, 159928 insertions(+), 3342 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
index ee6f2708990bb..25bf7b2255e5c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
@@ -176,3 +176,1469 @@ end:
   %phi = phi <5 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <5 x i32> %phi
 }
+
+define <10 x i16> @bitcast_v5i32_to_v10i16(<5 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v5i32_to_v10i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB2_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB2_4
+; GCN-NEXT:  .LBB2_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB2_3: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v9, s4, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
+; GCN-NEXT:  .LBB2_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_alignbit_b32 v9, s4, v8, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v5i32_to_v10i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v5i32_to_v10i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v5i32_to_v10i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <5 x i32> %a, splat (i32 3)
+  %a2 = bitcast <5 x i32> %a1 to <10 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <5 x i32> %a to <10 x i16>
+  br label %end
+
+end:
+  %phi = phi <10 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x i16> %phi
+}
+
+define <5 x i32> @bitcast_v10i16_to_v5i32(<10 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i16_to_v5i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v13, v4
+; GCN-NEXT:    v_mov_b32_e32 v12, v2
+; GCN-NEXT:    v_mov_b32_e32 v11, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_4
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB3_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v10
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v14
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v5
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  .LBB3_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v11
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v8
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_or_b32_e32 v0, v10, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v14, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 0x30000, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0x30000, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0x30000, v4
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i16_to_v5i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v6, 3
+; VI-NEXT:    v_add_u16_e32 v5, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_u16_e32 v5, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v5, v3
+; VI-NEXT:    v_add_u16_e32 v5, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v5, v2
+; VI-NEXT:    v_add_u16_e32 v5, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v5, v1
+; VI-NEXT:    v_add_u16_e32 v5, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v5, v0
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i16_to_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i16_to_v5i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i16> %a, splat (i16 3)
+  %a2 = bitcast <10 x i16> %a1 to <5 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i16> %a to <5 x i32>
+  br label %end
+
+end:
+  %phi = phi <5 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <5 x i32> %phi
+}
+
+define <10 x half> @bitcast_v5i32_to_v10f16(<5 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v5i32_to_v10f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v14, v4
+; GCN-NEXT:    v_mov_b32_e32 v13, v3
+; GCN-NEXT:    v_mov_b32_e32 v12, v2
+; GCN-NEXT:    v_mov_b32_e32 v11, v1
+; GCN-NEXT:    v_mov_b32_e32 v10, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB4_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB4_4
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB4_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v16, 16, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v10
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  .LBB4_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v5i32_to_v10f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v5i32_to_v10f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v5i32_to_v10f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <5 x i32> %a, splat (i32 3)
+  %a2 = bitcast <5 x i32> %a1 to <10 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <5 x i32> %a to <10 x half>
+  br label %end
+
+end:
+  %phi = phi <10 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x half> %phi
+}
+
+define <5 x i32> @bitcast_v10f16_to_v5i32(<10 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f16_to_v5i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v8
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB5_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB5_4
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB5_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v0, v14, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v12, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v10, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v6, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  .LBB5_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v8, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v6, v9
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v7
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f16_to_v5i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v5, 0x200
+; VI-NEXT:    v_add_f16_sdwa v6, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v5, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v6
+; VI-NEXT:    v_or_b32_e32 v0, v0, v5
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f16_to_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f16_to_v5i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <10 x half> %a1 to <5 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x half> %a to <5 x i32>
+  br label %end
+
+end:
+  %phi = phi <5 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <5 x i32> %phi
+}
+
+define <10 x i16> @bitcast_v5f32_to_v10i16(<5 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v5f32_to_v10i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB6_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB6_4
+; GCN-NEXT:  .LBB6_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB6_3: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v9, s4, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  .LBB6_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_alignbit_b32 v9, s4, v8, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v5f32_to_v10i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v5f32_to_v10i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v5f32_to_v10i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <5 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <5 x float> %a1 to <10 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <5 x float> %a to <10 x i16>
+  br label %end
+
+end:
+  %phi = phi <10 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x i16> %phi
+}
+
+define <5 x float> @bitcast_v10i16_to_v5f32(<10 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i16_to_v5f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v13, v4
+; GCN-NEXT:    v_mov_b32_e32 v12, v2
+; GCN-NEXT:    v_mov_b32_e32 v11, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_4
+; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v10
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v14
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v5
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  .LBB7_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v11
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v8
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_or_b32_e32 v0, v10, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v14, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 0x30000, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0x30000, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0x30000, v4
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i16_to_v5f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v6, 3
+; VI-NEXT:    v_add_u16_e32 v5, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_u16_e32 v5, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v5, v3
+; VI-NEXT:    v_add_u16_e32 v5, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v5, v2
+; VI-NEXT:    v_add_u16_e32 v5, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v5, v1
+; VI-NEXT:    v_add_u16_e32 v5, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v5, v0
+; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i16_to_v5f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i16_to_v5f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i16> %a, splat (i16 3)
+  %a2 = bitcast <10 x i16> %a1 to <5 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i16> %a to <5 x float>
+  br label %end
+
+end:
+  %phi = phi <5 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <5 x float> %phi
+}
+
+define <10 x half> @bitcast_v5f32_to_v10f16(<5 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v5f32_to_v10f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v14, v4
+; GCN-NEXT:    v_mov_b32_e32 v13, v3
+; GCN-NEXT:    v_mov_b32_e32 v12, v2
+; GCN-NEXT:    v_mov_b32_e32 v11, v1
+; GCN-NEXT:    v_mov_b32_e32 v10, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_4
+; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB8_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v16, 16, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v10
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  .LBB8_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v5f32_to_v10f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v5f32_to_v10f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v5f32_to_v10f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <5 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <5 x float> %a1 to <10 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <5 x float> %a to <10 x half>
+  br label %end
+
+end:
+  %phi = phi <10 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x half> %phi
+}
+
+define <5 x float> @bitcast_v10f16_to_v5f32(<10 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f16_to_v5f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v8
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_4
+; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB9_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v0, v14, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v12, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v10, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v6, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  .LBB9_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v8, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v6, v9
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v7
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f16_to_v5f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v5, 0x200
+; VI-NEXT:    v_add_f16_sdwa v6, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v5, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v6
+; VI-NEXT:    v_or_b32_e32 v0, v0, v5
+; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f16_to_v5f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f16_to_v5f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <10 x half> %a1 to <5 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x half> %a to <5 x float>
+  br label %end
+
+end:
+  %phi = phi <5 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <5 x float> %phi
+}
+
+define <10 x half> @bitcast_v10i16_to_v10f16(<10 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i16_to_v10f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v19, v9
+; GCN-NEXT:    v_mov_b32_e32 v18, v8
+; GCN-NEXT:    v_mov_b32_e32 v17, v7
+; GCN-NEXT:    v_mov_b32_e32 v16, v6
+; GCN-NEXT:    v_mov_b32_e32 v15, v5
+; GCN-NEXT:    v_mov_b32_e32 v14, v4
+; GCN-NEXT:    v_mov_b32_e32 v13, v3
+; GCN-NEXT:    v_mov_b32_e32 v12, v2
+; GCN-NEXT:    v_mov_b32_e32 v11, v1
+; GCN-NEXT:    v_mov_b32_e32 v20, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB10_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB10_4
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB10_3: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  .LBB10_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i16_to_v10f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v5, 3
+; VI-NEXT:    v_add_u16_sdwa v6, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v7, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v8, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v9, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v5, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_or_b32_e32 v3, v3, v9
+; VI-NEXT:    v_or_b32_e32 v2, v2, v8
+; VI-NEXT:    v_or_b32_e32 v1, v1, v7
+; VI-NEXT:    v_or_b32_e32 v0, v0, v6
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i16_to_v10f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i16_to_v10f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i16> %a, splat (i16 3)
+  %a2 = bitcast <10 x i16> %a1 to <10 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i16> %a to <10 x half>
+  br label %end
+
+end:
+  %phi = phi <10 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x half> %phi
+}
+
+define <10 x i16> @bitcast_v10f16_to_v10i16(<10 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f16_to_v10i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v10
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v11
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v12
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v5, 16
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f16_to_v10i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v6, 0x200
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v0
+; VI-NEXT:    v_add_f16_sdwa v0, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v1, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v3
+; VI-NEXT:    v_add_f16_sdwa v3, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v4
+; VI-NEXT:    v_add_f16_sdwa v4, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v10, v4
+; VI-NEXT:    v_or_b32_e32 v3, v9, v3
+; VI-NEXT:    v_or_b32_e32 v2, v8, v2
+; VI-NEXT:    v_or_b32_e32 v1, v7, v1
+; VI-NEXT:    v_or_b32_e32 v0, v5, v0
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f16_to_v10i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f16_to_v10i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v5
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <10 x half> %a1 to <10 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x half> %a to <10 x i16>
+  br label %end
+
+end:
+  %phi = phi <10 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll
index ecaf64567500f..32d21e19e8e01 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll
@@ -541,6 +541,639 @@ end:
   ret <6 x i32> %phi
 }
 
+define <12 x i16> @bitcast_v6i32_to_v12i16(<6 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6i32_to_v12i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v12, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v12, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v12, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v6, v12
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6i32_to_v12i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6i32_to_v12i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6i32_to_v12i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <6 x i32> %a, splat (i32 3)
+  %a2 = bitcast <6 x i32> %a1 to <12 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x i32> %a to <12 x i16>
+  br label %end
+
+end:
+  %phi = phi <12 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i16> %phi
+}
+
+define <6 x i32> @bitcast_v12i16_to_v6i32(<12 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i16_to_v6i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v15, v4
+; GCN-NEXT:    v_mov_b32_e32 v14, v2
+; GCN-NEXT:    v_mov_b32_e32 v13, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_4
+; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v12
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v16
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v17
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  .LBB7_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v13
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_or_b32_e32 v0, v12, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v16, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v17, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0x30000, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0x30000, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0x30000, v5
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i16_to_v6i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v7, 3
+; VI-NEXT:    v_add_u16_e32 v6, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_u16_e32 v6, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v6, v4
+; VI-NEXT:    v_add_u16_e32 v6, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v6, v3
+; VI-NEXT:    v_add_u16_e32 v6, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v6, v2
+; VI-NEXT:    v_add_u16_e32 v6, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v6, v1
+; VI-NEXT:    v_add_u16_e32 v6, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v6, v0
+; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i16_to_v6i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i16_to_v6i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i16> %a, splat (i16 3)
+  %a2 = bitcast <12 x i16> %a1 to <6 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i16> %a to <6 x i32>
+  br label %end
+
+end:
+  %phi = phi <6 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x i32> %phi
+}
+
+define <12 x half> @bitcast_v6i32_to_v12f16(<6 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6i32_to_v12f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v17, v5
+; GCN-NEXT:    v_mov_b32_e32 v16, v4
+; GCN-NEXT:    v_mov_b32_e32 v15, v3
+; GCN-NEXT:    v_mov_b32_e32 v14, v2
+; GCN-NEXT:    v_mov_b32_e32 v13, v1
+; GCN-NEXT:    v_mov_b32_e32 v12, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_4
+; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB8_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  .LBB8_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6i32_to_v12f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6i32_to_v12f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6i32_to_v12f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <6 x i32> %a, splat (i32 3)
+  %a2 = bitcast <6 x i32> %a1 to <12 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x i32> %a to <12 x half>
+  br label %end
+
+end:
+  %phi = phi <12 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x half> %phi
+}
+
+define <6 x i32> @bitcast_v12f16_to_v6i32(<12 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f16_to_v6i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v10
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_4
+; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB9_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v16, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v14, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v13, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v12, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  .LBB9_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v8
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f16_to_v6i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v6, 0x200
+; VI-NEXT:    v_add_f16_sdwa v7, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v6, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v7
+; VI-NEXT:    v_or_b32_e32 v0, v0, v6
+; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f16_to_v6i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f16_to_v6i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <12 x half> %a1 to <6 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x half> %a to <6 x i32>
+  br label %end
+
+end:
+  %phi = phi <6 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x i32> %phi
+}
+
 define <3 x i64> @bitcast_v6f32_to_v3i64(<6 x float> %a, i32 %b) {
 ; GCN-LABEL: bitcast_v6f32_to_v3i64:
 ; GCN:       ; %bb.0:
@@ -549,37 +1182,1735 @@ define <3 x i64> @bitcast_v6f32_to_v3i64(<6 x float> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB6_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GCN-NEXT:  .LBB6_2: ; %end
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6f32_to_v3i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6f32_to_v3i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6f32_to_v3i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <6 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <6 x float> %a1 to <3 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x float> %a to <3 x i64>
+  br label %end
+
+end:
+  %phi = phi <3 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <3 x i64> %phi
+}
+
+define <6 x float> @bitcast_v3i64_to_v6f32(<3 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v3i64_to_v6f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v3i64_to_v6f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v3i64_to_v6f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v3i64_to_v6f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <3 x i64> %a, splat (i64 3)
+  %a2 = bitcast <3 x i64> %a1 to <6 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <3 x i64> %a to <6 x float>
+  br label %end
+
+end:
+  %phi = phi <6 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x float> %phi
+}
+
+define <3 x double> @bitcast_v6f32_to_v3f64(<6 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6f32_to_v3f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6f32_to_v3f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6f32_to_v3f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6f32_to_v3f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <6 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <6 x float> %a1 to <3 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x float> %a to <3 x double>
+  br label %end
+
+end:
+  %phi = phi <3 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <3 x double> %phi
+}
+
+define <6 x float> @bitcast_v3f64_to_v6f32(<3 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v3f64_to_v6f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v3f64_to_v6f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v3f64_to_v6f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v3f64_to_v6f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <3 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <3 x double> %a1 to <6 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <3 x double> %a to <6 x float>
+  br label %end
+
+end:
+  %phi = phi <6 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x float> %phi
+}
+
+define <12 x i16> @bitcast_v6f32_to_v12i16(<6 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6f32_to_v12i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v12, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v12, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v12, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v6, v12
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6f32_to_v12i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6f32_to_v12i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6f32_to_v12i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <6 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <6 x float> %a1 to <12 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x float> %a to <12 x i16>
+  br label %end
+
+end:
+  %phi = phi <12 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i16> %phi
+}
+
+define <6 x float> @bitcast_v12i16_to_v6f32(<12 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i16_to_v6f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v15, v4
+; GCN-NEXT:    v_mov_b32_e32 v14, v2
+; GCN-NEXT:    v_mov_b32_e32 v13, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB15_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB15_4
+; GCN-NEXT:  .LBB15_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB15_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v12
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v16
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v17
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  .LBB15_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v13
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_or_b32_e32 v0, v12, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v16, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v17, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0x30000, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0x30000, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0x30000, v5
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i16_to_v6f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v7, 3
+; VI-NEXT:    v_add_u16_e32 v6, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_u16_e32 v6, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v6, v4
+; VI-NEXT:    v_add_u16_e32 v6, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v6, v3
+; VI-NEXT:    v_add_u16_e32 v6, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v6, v2
+; VI-NEXT:    v_add_u16_e32 v6, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v6, v1
+; VI-NEXT:    v_add_u16_e32 v6, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v6, v0
+; VI-NEXT:  .LBB15_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i16_to_v6f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i16_to_v6f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i16> %a, splat (i16 3)
+  %a2 = bitcast <12 x i16> %a1 to <6 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i16> %a to <6 x float>
+  br label %end
+
+end:
+  %phi = phi <6 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x float> %phi
+}
+
+define <12 x half> @bitcast_v6f32_to_v12f16(<6 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6f32_to_v12f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v17, v5
+; GCN-NEXT:    v_mov_b32_e32 v16, v4
+; GCN-NEXT:    v_mov_b32_e32 v15, v3
+; GCN-NEXT:    v_mov_b32_e32 v14, v2
+; GCN-NEXT:    v_mov_b32_e32 v13, v1
+; GCN-NEXT:    v_mov_b32_e32 v12, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB16_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB16_4
+; GCN-NEXT:  .LBB16_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB16_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  .LBB16_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6f32_to_v12f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6f32_to_v12f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6f32_to_v12f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <6 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <6 x float> %a1 to <12 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x float> %a to <12 x half>
+  br label %end
+
+end:
+  %phi = phi <12 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x half> %phi
+}
+
+define <6 x float> @bitcast_v12f16_to_v6f32(<12 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f16_to_v6f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v10
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB17_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB17_4
+; GCN-NEXT:  .LBB17_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB17_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v16, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v14, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v13, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v12, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  .LBB17_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v8
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f16_to_v6f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v6, 0x200
+; VI-NEXT:    v_add_f16_sdwa v7, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v6, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v7
+; VI-NEXT:    v_or_b32_e32 v0, v0, v6
+; VI-NEXT:  .LBB17_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f16_to_v6f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f16_to_v6f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <12 x half> %a1 to <6 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x half> %a to <6 x float>
+  br label %end
+
+end:
+  %phi = phi <6 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x float> %phi
+}
+
+define <3 x double> @bitcast_v3i64_to_v3f64(<3 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v3i64_to_v3f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v3i64_to_v3f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v3i64_to_v3f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v3i64_to_v3f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <3 x i64> %a, splat (i64 3)
+  %a2 = bitcast <3 x i64> %a1 to <3 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <3 x i64> %a to <3 x double>
+  br label %end
+
+end:
+  %phi = phi <3 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <3 x double> %phi
+}
+
+define <3 x i64> @bitcast_v3f64_to_v3i64(<3 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v3f64_to_v3i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v3f64_to_v3i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v3f64_to_v3i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v3f64_to_v3i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <3 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <3 x double> %a1 to <3 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <3 x double> %a to <3 x i64>
+  br label %end
+
+end:
+  %phi = phi <3 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <3 x i64> %phi
+}
+
+define <12 x i16> @bitcast_v3i64_to_v12i16(<3 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v3i64_to_v12i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v12, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v12, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v12, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v6, v12
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v3i64_to_v12i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v3i64_to_v12i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v3i64_to_v12i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <3 x i64> %a, splat (i64 3)
+  %a2 = bitcast <3 x i64> %a1 to <12 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <3 x i64> %a to <12 x i16>
+  br label %end
+
+end:
+  %phi = phi <12 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i16> %phi
+}
+
+define <3 x i64> @bitcast_v12i16_to_v3i64(<12 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i16_to_v3i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v15, v4
+; GCN-NEXT:    v_mov_b32_e32 v14, v2
+; GCN-NEXT:    v_mov_b32_e32 v13, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB21_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB21_4
+; GCN-NEXT:  .LBB21_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB21_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v12
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v16
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v17
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  .LBB21_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v13
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_or_b32_e32 v0, v12, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v16, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v17, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0x30000, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0x30000, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0x30000, v5
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i16_to_v3i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v7, 3
+; VI-NEXT:    v_add_u16_e32 v6, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_u16_e32 v6, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v6, v4
+; VI-NEXT:    v_add_u16_e32 v6, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v6, v3
+; VI-NEXT:    v_add_u16_e32 v6, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v6, v2
+; VI-NEXT:    v_add_u16_e32 v6, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v6, v1
+; VI-NEXT:    v_add_u16_e32 v6, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v6, v0
+; VI-NEXT:  .LBB21_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i16_to_v3i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i16_to_v3i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i16> %a, splat (i16 3)
+  %a2 = bitcast <12 x i16> %a1 to <3 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i16> %a to <3 x i64>
+  br label %end
+
+end:
+  %phi = phi <3 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <3 x i64> %phi
+}
+
+define <12 x half> @bitcast_v3i64_to_v12f16(<3 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v3i64_to_v12f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v14, v5
+; GCN-NEXT:    v_mov_b32_e32 v13, v4
+; GCN-NEXT:    v_mov_b32_e32 v16, v3
+; GCN-NEXT:    v_mov_b32_e32 v15, v2
+; GCN-NEXT:    v_mov_b32_e32 v17, v1
+; GCN-NEXT:    v_mov_b32_e32 v12, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB22_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB22_4
+; GCN-NEXT:  .LBB22_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB22_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  .LBB22_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v14, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v12
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v3i64_to_v12f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v3i64_to_v12f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v3i64_to_v12f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <3 x i64> %a, splat (i64 3)
+  %a2 = bitcast <3 x i64> %a1 to <12 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <3 x i64> %a to <12 x half>
+  br label %end
+
+end:
+  %phi = phi <12 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x half> %phi
+}
+
+define <3 x i64> @bitcast_v12f16_to_v3i64(<12 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f16_to_v3i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v10
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB23_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB23_4
+; GCN-NEXT:  .LBB23_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB23_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v16, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v14, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v13, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v12, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  .LBB23_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v8
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v6f32_to_v3i64:
+; VI-LABEL: bitcast_v12f16_to_v3i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v6, 0x200
+; VI-NEXT:    v_add_f16_sdwa v7, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v6, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v7
+; VI-NEXT:    v_or_b32_e32 v0, v0, v6
+; VI-NEXT:  .LBB23_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v6f32_to_v3i64:
+; GFX9-LABEL: bitcast_v12f16_to_v3i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
@@ -587,17 +2918,18 @@ define <3 x i64> @bitcast_v6f32_to_v3i64(<6 x float> %a, i32 %b) {
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:  ; %bb.2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v6f32_to_v3i64:
+; GFX11-LABEL: bitcast_v12f16_to_v3i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -606,9 +2938,12 @@ define <3 x i64> @bitcast_v6f32_to_v3i64(<6 x float> %a, i32 %b) {
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
-; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
-; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
 ; GFX11-NEXT:  ; %bb.2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -616,12 +2951,12 @@ define <3 x i64> @bitcast_v6f32_to_v3i64(<6 x float> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <6 x float> %a, splat (float 1.000000e+00)
-  %a2 = bitcast <6 x float> %a1 to <3 x i64>
+  %a1 = fadd <12 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <12 x half> %a1 to <3 x i64>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <6 x float> %a to <3 x i64>
+  %a3 = bitcast <12 x half> %a to <3 x i64>
   br label %end
 
 end:
@@ -629,63 +2964,89 @@ end:
   ret <3 x i64> %phi
 }
 
-define <6 x float> @bitcast_v3i64_to_v6f32(<3 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v3i64_to_v6f32:
+define <12 x i16> @bitcast_v3f64_to_v12i16(<3 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v3f64_to_v12i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v13, v5
+; GCN-NEXT:    v_mov_b32_e32 v12, v4
+; GCN-NEXT:    v_mov_b32_e32 v15, v3
+; GCN-NEXT:    v_mov_b32_e32 v14, v2
+; GCN-NEXT:    v_mov_b32_e32 v17, v1
+; GCN-NEXT:    v_mov_b32_e32 v16, v0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v9, v13, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v15, v14, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v17, v16, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v17
+; GCN-NEXT:  .LBB24_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB7_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_alignbit_b32 v9, v13, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v15, v14, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v17, v16, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v17
+; GCN-NEXT:  .LBB24_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v0, v16
+; GCN-NEXT:    v_mov_b32_e32 v2, v17
+; GCN-NEXT:    v_mov_b32_e32 v4, v14
+; GCN-NEXT:    v_mov_b32_e32 v6, v15
+; GCN-NEXT:    v_mov_b32_e32 v8, v12
+; GCN-NEXT:    v_mov_b32_e32 v10, v13
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v3i64_to_v6f32:
+; VI-LABEL: bitcast_v3f64_to_v12i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB24_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v3i64_to_v6f32:
+; GFX9-LABEL: bitcast_v3f64_to_v12i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB24_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v3i64_to_v6f32:
+; GFX11-LABEL: bitcast_v3f64_to_v12i16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -693,74 +3054,144 @@ define <6 x float> @bitcast_v3i64_to_v6f32(<3 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB24_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <3 x i64> %a, splat (i64 3)
-  %a2 = bitcast <3 x i64> %a1 to <6 x float>
+  %a1 = fadd <3 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <3 x double> %a1 to <12 x i16>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <3 x i64> %a to <6 x float>
+  %a3 = bitcast <3 x double> %a to <12 x i16>
   br label %end
 
 end:
-  %phi = phi <6 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <6 x float> %phi
+  %phi = phi <12 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i16> %phi
 }
 
-define <3 x double> @bitcast_v6f32_to_v3f64(<6 x float> %a, i32 %b) {
-; GCN-LABEL: bitcast_v6f32_to_v3f64:
+define <3 x double> @bitcast_v12i16_to_v3f64(<12 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i16_to_v3f64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    v_mov_b32_e32 v15, v4
+; GCN-NEXT:    v_mov_b32_e32 v14, v2
+; GCN-NEXT:    v_mov_b32_e32 v13, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB25_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB8_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB25_4
+; GCN-NEXT:  .LBB25_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB25_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v12
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v16
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v17
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  .LBB25_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v13
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_or_b32_e32 v0, v12, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v16, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v17, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0x30000, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0x30000, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0x30000, v5
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v6f32_to_v3f64:
+; VI-LABEL: bitcast_v12i16_to_v3f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v7, 3
+; VI-NEXT:    v_add_u16_e32 v6, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_u16_e32 v6, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v6, v4
+; VI-NEXT:    v_add_u16_e32 v6, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v6, v3
+; VI-NEXT:    v_add_u16_e32 v6, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v6, v2
+; VI-NEXT:    v_add_u16_e32 v6, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v6, v1
+; VI-NEXT:    v_add_u16_e32 v6, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v6, v0
+; VI-NEXT:  .LBB25_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v6f32_to_v3f64:
+; GFX9-LABEL: bitcast_v12i16_to_v3f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
@@ -768,17 +3199,17 @@ define <3 x double> @bitcast_v6f32_to_v3f64(<6 x float> %a, i32 %b) {
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
 ; GFX9-NEXT:  ; %bb.2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v6f32_to_v3f64:
+; GFX11-LABEL: bitcast_v12i16_to_v3f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -787,9 +3218,12 @@ define <3 x double> @bitcast_v6f32_to_v3f64(<6 x float> %a, i32 %b) {
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
-; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
-; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
 ; GFX11-NEXT:  ; %bb.2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -797,12 +3231,12 @@ define <3 x double> @bitcast_v6f32_to_v3f64(<6 x float> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <6 x float> %a, splat (float 1.000000e+00)
-  %a2 = bitcast <6 x float> %a1 to <3 x double>
+  %a1 = add <12 x i16> %a, splat (i16 3)
+  %a2 = bitcast <12 x i16> %a1 to <3 x double>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <6 x float> %a to <3 x double>
+  %a3 = bitcast <12 x i16> %a to <3 x double>
   br label %end
 
 end:
@@ -810,56 +3244,116 @@ end:
   ret <3 x double> %phi
 }
 
-define <6 x float> @bitcast_v3f64_to_v6f32(<3 x double> %a, i32 %b) {
-; GCN-LABEL: bitcast_v3f64_to_v6f32:
+define <12 x half> @bitcast_v3f64_to_v12f16(<3 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v3f64_to_v12f16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v12, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v16, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:  .LBB26_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB9_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v0
+; GCN-NEXT:  .LBB26_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v0, v12
+; GCN-NEXT:    v_mov_b32_e32 v1, v17
+; GCN-NEXT:    v_mov_b32_e32 v2, v13
+; GCN-NEXT:    v_mov_b32_e32 v3, v16
+; GCN-NEXT:    v_mov_b32_e32 v4, v14
+; GCN-NEXT:    v_mov_b32_e32 v5, v15
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v3f64_to_v6f32:
+; VI-LABEL: bitcast_v3f64_to_v12f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:    s_cbranch_execz .LBB26_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:  .LBB26_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v3f64_to_v6f32:
+; GFX9-LABEL: bitcast_v3f64_to_v12f16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:  .LBB9_2: ; %end
+; GFX9-NEXT:  .LBB26_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v3f64_to_v6f32:
+; GFX11-LABEL: bitcast_v3f64_to_v12f16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -867,12 +3361,12 @@ define <6 x float> @bitcast_v3f64_to_v6f32(<3 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:  .LBB26_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -880,57 +3374,157 @@ define <6 x float> @bitcast_v3f64_to_v6f32(<3 x double> %a, i32 %b) {
 
 cmp.true:
   %a1 = fadd <3 x double> %a, splat (double 1.000000e+00)
-  %a2 = bitcast <3 x double> %a1 to <6 x float>
+  %a2 = bitcast <3 x double> %a1 to <12 x half>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <3 x double> %a to <6 x float>
+  %a3 = bitcast <3 x double> %a to <12 x half>
   br label %end
 
 end:
-  %phi = phi <6 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <6 x float> %phi
+  %phi = phi <12 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x half> %phi
 }
 
-define <3 x double> @bitcast_v3i64_to_v3f64(<3 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v3i64_to_v3f64:
+define <3 x double> @bitcast_v12f16_to_v3f64(<12 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f16_to_v3f64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v10
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB27_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB10_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB27_4
+; GCN-NEXT:  .LBB27_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB27_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v16, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v14, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v13, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v12, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  .LBB27_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v8
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v3i64_to_v3f64:
+; VI-LABEL: bitcast_v12f16_to_v3f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v6, 0x200
+; VI-NEXT:    v_add_f16_sdwa v7, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v6, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v7
+; VI-NEXT:    v_or_b32_e32 v0, v0, v6
+; VI-NEXT:  .LBB27_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v3i64_to_v3f64:
+; GFX9-LABEL: bitcast_v12f16_to_v3f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
@@ -938,17 +3532,18 @@ define <3 x double> @bitcast_v3i64_to_v3f64(<3 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:  ; %bb.2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v3i64_to_v3f64:
+; GFX11-LABEL: bitcast_v12f16_to_v3f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -957,14 +3552,12 @@ define <3 x double> @bitcast_v3i64_to_v3f64(<3 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
 ; GFX11-NEXT:  ; %bb.2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -972,12 +3565,12 @@ define <3 x double> @bitcast_v3i64_to_v3f64(<3 x i64> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <3 x i64> %a, splat (i64 3)
-  %a2 = bitcast <3 x i64> %a1 to <3 x double>
+  %a1 = fadd <12 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <12 x half> %a1 to <3 x double>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <3 x i64> %a to <3 x double>
+  %a3 = bitcast <12 x half> %a to <3 x double>
   br label %end
 
 end:
@@ -985,56 +3578,313 @@ end:
   ret <3 x double> %phi
 }
 
-define <3 x i64> @bitcast_v3f64_to_v3i64(<3 x double> %a, i32 %b) {
-; GCN-LABEL: bitcast_v3f64_to_v3i64:
+define <12 x half> @bitcast_v12i16_to_v12f16(<12 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i16_to_v12f16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v11
+; GCN-NEXT:    v_mov_b32_e32 v22, v10
+; GCN-NEXT:    v_mov_b32_e32 v21, v9
+; GCN-NEXT:    v_mov_b32_e32 v20, v8
+; GCN-NEXT:    v_mov_b32_e32 v19, v7
+; GCN-NEXT:    v_mov_b32_e32 v18, v6
+; GCN-NEXT:    v_mov_b32_e32 v17, v5
+; GCN-NEXT:    v_mov_b32_e32 v16, v4
+; GCN-NEXT:    v_mov_b32_e32 v15, v3
+; GCN-NEXT:    v_mov_b32_e32 v14, v2
+; GCN-NEXT:    v_mov_b32_e32 v13, v1
+; GCN-NEXT:    v_mov_b32_e32 v24, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB28_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:    s_cbranch_execnz .LBB28_4
+; GCN-NEXT:  .LBB28_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB28_3: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  .LBB28_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i16_to_v12f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v6, 3
+; VI-NEXT:    v_add_u16_sdwa v7, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v8, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v9, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v10, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v11, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v6, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_or_b32_e32 v4, v4, v11
+; VI-NEXT:    v_or_b32_e32 v3, v3, v10
+; VI-NEXT:    v_or_b32_e32 v2, v2, v9
+; VI-NEXT:    v_or_b32_e32 v1, v1, v8
+; VI-NEXT:    v_or_b32_e32 v0, v0, v7
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i16_to_v12f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i16_to_v12f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v6
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i16> %a, splat (i16 3)
+  %a2 = bitcast <12 x i16> %a1 to <12 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i16> %a to <12 x half>
+  br label %end
+
+end:
+  %phi = phi <12 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x half> %phi
+}
+
+define <12 x i16> @bitcast_v12f16_to_v12i16(<12 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f16_to_v12i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v12
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v14
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v9, 16
+; GCN-NEXT:  .LBB29_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v3f64_to_v3i64:
+; VI-LABEL: bitcast_v12f16_to_v12i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:    s_cbranch_execz .LBB29_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v7, 0x200
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v0
+; VI-NEXT:    v_add_f16_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v3
+; VI-NEXT:    v_add_f16_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v4
+; VI-NEXT:    v_add_f16_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v5
+; VI-NEXT:    v_add_f16_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v12, v5
+; VI-NEXT:    v_or_b32_e32 v4, v11, v4
+; VI-NEXT:    v_or_b32_e32 v3, v10, v3
+; VI-NEXT:    v_or_b32_e32 v2, v9, v2
+; VI-NEXT:    v_or_b32_e32 v1, v8, v1
+; VI-NEXT:    v_or_b32_e32 v0, v6, v0
+; VI-NEXT:  .LBB29_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v3f64_to_v3i64:
+; GFX9-LABEL: bitcast_v12f16_to_v12i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v3f64_to_v3i64:
+; GFX11-LABEL: bitcast_v12f16_to_v12i16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -1042,27 +3892,29 @@ define <3 x i64> @bitcast_v3f64_to_v3i64(<3 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  ; %bb.2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <3 x double> %a, splat (double 1.000000e+00)
-  %a2 = bitcast <3 x double> %a1 to <3 x i64>
+  %a1 = fadd <12 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <12 x half> %a1 to <12 x i16>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <3 x double> %a to <3 x i64>
+  %a3 = bitcast <12 x half> %a to <12 x i16>
   br label %end
 
 end:
-  %phi = phi <3 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <3 x i64> %phi
+  %phi = phi <12 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i16> %phi
 }
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
index d2e46475487c2..3ec705baa9c82 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
@@ -192,3 +192,1762 @@ end:
   %phi = phi <7 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <7 x i32> %phi
 }
+
+define <14 x i16> @bitcast_v7i32_to_v14i16(<7 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7i32_to_v14i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB2_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB2_4
+; GCN-NEXT:  .LBB2_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB2_3: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v13, s4, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
+; GCN-NEXT:  .LBB2_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_alignbit_b32 v13, s4, v12, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7i32_to_v14i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7i32_to_v14i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7i32_to_v14i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB2_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <7 x i32> %a, splat (i32 3)
+  %a2 = bitcast <7 x i32> %a1 to <14 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x i32> %a to <14 x i16>
+  br label %end
+
+end:
+  %phi = phi <14 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i16> %phi
+}
+
+define <7 x i32> @bitcast_v14i16_to_v7i32(<14 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i16_to_v7i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v18, v6
+; GCN-NEXT:    v_mov_b32_e32 v17, v4
+; GCN-NEXT:    v_mov_b32_e32 v16, v2
+; GCN-NEXT:    v_mov_b32_e32 v15, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_4
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB3_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v14
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v19
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v20
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  .LBB3_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v15
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_or_b32_e32 v0, v14, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v19, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v20, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0x30000, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0x30000, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 0x30000, v6
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i16_to_v7i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v8, 3
+; VI-NEXT:    v_add_u16_e32 v7, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_u16_e32 v7, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v7, v5
+; VI-NEXT:    v_add_u16_e32 v7, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v7, v4
+; VI-NEXT:    v_add_u16_e32 v7, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v7, v3
+; VI-NEXT:    v_add_u16_e32 v7, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v7, v2
+; VI-NEXT:    v_add_u16_e32 v7, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v7, v1
+; VI-NEXT:    v_add_u16_e32 v7, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v7, v0
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i16_to_v7i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i16_to_v7i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i16> %a, splat (i16 3)
+  %a2 = bitcast <14 x i16> %a1 to <7 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i16> %a to <7 x i32>
+  br label %end
+
+end:
+  %phi = phi <7 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x i32> %phi
+}
+
+define <14 x half> @bitcast_v7i32_to_v14f16(<7 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7i32_to_v14f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v20, v6
+; GCN-NEXT:    v_mov_b32_e32 v19, v5
+; GCN-NEXT:    v_mov_b32_e32 v18, v4
+; GCN-NEXT:    v_mov_b32_e32 v17, v3
+; GCN-NEXT:    v_mov_b32_e32 v16, v2
+; GCN-NEXT:    v_mov_b32_e32 v15, v1
+; GCN-NEXT:    v_mov_b32_e32 v14, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB4_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB4_4
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB4_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v22, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  .LBB4_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7i32_to_v14f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7i32_to_v14f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7i32_to_v14f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <7 x i32> %a, splat (i32 3)
+  %a2 = bitcast <7 x i32> %a1 to <14 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x i32> %a to <14 x half>
+  br label %end
+
+end:
+  %phi = phi <14 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x half> %phi
+}
+
+define <7 x i32> @bitcast_v14f16_to_v7i32(<14 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f16_to_v7i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v12
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB5_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB5_4
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB5_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v0, v20, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v18, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v16, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v15, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v14, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  .LBB5_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v10, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v9
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f16_to_v7i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v7, 0x200
+; VI-NEXT:    v_add_f16_sdwa v8, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v7, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v8
+; VI-NEXT:    v_or_b32_e32 v0, v0, v7
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f16_to_v7i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f16_to_v7i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <14 x half> %a1 to <7 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x half> %a to <7 x i32>
+  br label %end
+
+end:
+  %phi = phi <7 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x i32> %phi
+}
+
+define <14 x i16> @bitcast_v7f32_to_v14i16(<7 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7f32_to_v14i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB6_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB6_4
+; GCN-NEXT:  .LBB6_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB6_3: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v13, s4, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  .LBB6_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_alignbit_b32 v13, s4, v12, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7f32_to_v14i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7f32_to_v14i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7f32_to_v14i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v6, 1.0, v6 :: v_dual_add_f32 v5, 1.0, v5
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <7 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <7 x float> %a1 to <14 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x float> %a to <14 x i16>
+  br label %end
+
+end:
+  %phi = phi <14 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i16> %phi
+}
+
+define <7 x float> @bitcast_v14i16_to_v7f32(<14 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i16_to_v7f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v18, v6
+; GCN-NEXT:    v_mov_b32_e32 v17, v4
+; GCN-NEXT:    v_mov_b32_e32 v16, v2
+; GCN-NEXT:    v_mov_b32_e32 v15, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_4
+; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v14
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v19
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v20
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  .LBB7_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v15
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_or_b32_e32 v0, v14, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v19, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v20, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0x30000, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0x30000, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 0x30000, v6
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i16_to_v7f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v8, 3
+; VI-NEXT:    v_add_u16_e32 v7, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_u16_e32 v7, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v7, v5
+; VI-NEXT:    v_add_u16_e32 v7, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v7, v4
+; VI-NEXT:    v_add_u16_e32 v7, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v7, v3
+; VI-NEXT:    v_add_u16_e32 v7, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v7, v2
+; VI-NEXT:    v_add_u16_e32 v7, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v7, v1
+; VI-NEXT:    v_add_u16_e32 v7, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v7, v0
+; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i16_to_v7f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i16_to_v7f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i16> %a, splat (i16 3)
+  %a2 = bitcast <14 x i16> %a1 to <7 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i16> %a to <7 x float>
+  br label %end
+
+end:
+  %phi = phi <7 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x float> %phi
+}
+
+define <14 x half> @bitcast_v7f32_to_v14f16(<7 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7f32_to_v14f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v20, v6
+; GCN-NEXT:    v_mov_b32_e32 v19, v5
+; GCN-NEXT:    v_mov_b32_e32 v18, v4
+; GCN-NEXT:    v_mov_b32_e32 v17, v3
+; GCN-NEXT:    v_mov_b32_e32 v16, v2
+; GCN-NEXT:    v_mov_b32_e32 v15, v1
+; GCN-NEXT:    v_mov_b32_e32 v14, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_4
+; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB8_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v22, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  .LBB8_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7f32_to_v14f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7f32_to_v14f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7f32_to_v14f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v6, 1.0, v6 :: v_dual_add_f32 v5, 1.0, v5
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <7 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <7 x float> %a1 to <14 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x float> %a to <14 x half>
+  br label %end
+
+end:
+  %phi = phi <14 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x half> %phi
+}
+
+define <7 x float> @bitcast_v14f16_to_v7f32(<14 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f16_to_v7f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v12
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_4
+; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB9_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v0, v20, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v18, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v16, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v15, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v14, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  .LBB9_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v10, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v9
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f16_to_v7f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v7, 0x200
+; VI-NEXT:    v_add_f16_sdwa v8, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v7, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v8
+; VI-NEXT:    v_or_b32_e32 v0, v0, v7
+; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f16_to_v7f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f16_to_v7f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <14 x half> %a1 to <7 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x half> %a to <7 x float>
+  br label %end
+
+end:
+  %phi = phi <7 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x float> %phi
+}
+
+define <14 x half> @bitcast_v14i16_to_v14f16(<14 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i16_to_v14f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v27, v13
+; GCN-NEXT:    v_mov_b32_e32 v26, v12
+; GCN-NEXT:    v_mov_b32_e32 v25, v11
+; GCN-NEXT:    v_mov_b32_e32 v24, v10
+; GCN-NEXT:    v_mov_b32_e32 v23, v9
+; GCN-NEXT:    v_mov_b32_e32 v22, v8
+; GCN-NEXT:    v_mov_b32_e32 v21, v7
+; GCN-NEXT:    v_mov_b32_e32 v20, v6
+; GCN-NEXT:    v_mov_b32_e32 v19, v5
+; GCN-NEXT:    v_mov_b32_e32 v18, v4
+; GCN-NEXT:    v_mov_b32_e32 v17, v3
+; GCN-NEXT:    v_mov_b32_e32 v16, v2
+; GCN-NEXT:    v_mov_b32_e32 v15, v1
+; GCN-NEXT:    v_mov_b32_e32 v28, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB10_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB10_4
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB10_3: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  .LBB10_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i16_to_v14f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v7, 3
+; VI-NEXT:    v_add_u16_sdwa v8, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v9, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v10, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v11, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v12, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v13, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v7, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_or_b32_e32 v5, v5, v13
+; VI-NEXT:    v_or_b32_e32 v4, v4, v12
+; VI-NEXT:    v_or_b32_e32 v3, v3, v11
+; VI-NEXT:    v_or_b32_e32 v2, v2, v10
+; VI-NEXT:    v_or_b32_e32 v1, v1, v9
+; VI-NEXT:    v_or_b32_e32 v0, v0, v8
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i16_to_v14f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i16_to_v14f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i16> %a, splat (i16 3)
+  %a2 = bitcast <14 x i16> %a1 to <14 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i16> %a to <14 x half>
+  br label %end
+
+end:
+  %phi = phi <14 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x half> %phi
+}
+
+define <14 x i16> @bitcast_v14f16_to_v14i16(<14 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f16_to_v14i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v9
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v14
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v15
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v16
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v17
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v9, 16
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f16_to_v14i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v8, 0x200
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v0
+; VI-NEXT:    v_add_f16_sdwa v0, v0, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v1, v1, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v2, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v3
+; VI-NEXT:    v_add_f16_sdwa v3, v3, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v4
+; VI-NEXT:    v_add_f16_sdwa v4, v4, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v5
+; VI-NEXT:    v_add_f16_sdwa v5, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v6, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v14, v6
+; VI-NEXT:    v_or_b32_e32 v5, v13, v5
+; VI-NEXT:    v_or_b32_e32 v4, v12, v4
+; VI-NEXT:    v_or_b32_e32 v3, v11, v3
+; VI-NEXT:    v_or_b32_e32 v2, v10, v2
+; VI-NEXT:    v_or_b32_e32 v1, v9, v1
+; VI-NEXT:    v_or_b32_e32 v0, v7, v0
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f16_to_v14i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f16_to_v14i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v7
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <14 x half> %a1 to <14 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x half> %a to <14 x i16>
+  br label %end
+
+end:
+  %phi = phi <14 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
index 76f16189b7f97..86ac4354e3f17 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
@@ -207,3 +207,2050 @@ end:
   %phi = phi <9 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <9 x i32> %phi
 }
+
+define <18 x i16> @bitcast_v9i32_to_v18i16(<9 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9i32_to_v18i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB2_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB2_4
+; GCN-NEXT:  .LBB2_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB2_3: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v17, s4, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
+; GCN-NEXT:  .LBB2_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_alignbit_b32 v17, s4, v16, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9i32_to_v18i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9i32_to_v18i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9i32_to_v18i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB2_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <9 x i32> %a, splat (i32 3)
+  %a2 = bitcast <9 x i32> %a1 to <18 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x i32> %a to <18 x i16>
+  br label %end
+
+end:
+  %phi = phi <18 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x i16> %phi
+}
+
+define <9 x i32> @bitcast_v18i16_to_v9i32(<18 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18i16_to_v9i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v23, v8
+; GCN-NEXT:    v_mov_b32_e32 v22, v6
+; GCN-NEXT:    v_mov_b32_e32 v21, v4
+; GCN-NEXT:    v_mov_b32_e32 v20, v2
+; GCN-NEXT:    v_mov_b32_e32 v19, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_4
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB3_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v25
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v26
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v18
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v24
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  .LBB3_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v19
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v25, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v26, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v18, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v24, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 0x30000, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 0x30000, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x30000, v8
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18i16_to_v9i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v10, 3
+; VI-NEXT:    v_add_u16_e32 v9, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_u16_e32 v9, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v9, v7
+; VI-NEXT:    v_add_u16_e32 v9, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v9, v6
+; VI-NEXT:    v_add_u16_e32 v9, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v9, v5
+; VI-NEXT:    v_add_u16_e32 v9, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v9, v4
+; VI-NEXT:    v_add_u16_e32 v9, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v9, v3
+; VI-NEXT:    v_add_u16_e32 v9, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v9, v2
+; VI-NEXT:    v_add_u16_e32 v9, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v9, v1
+; VI-NEXT:    v_add_u16_e32 v9, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v9, v0
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18i16_to_v9i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18i16_to_v9i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <18 x i16> %a, splat (i16 3)
+  %a2 = bitcast <18 x i16> %a1 to <9 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x i16> %a to <9 x i32>
+  br label %end
+
+end:
+  %phi = phi <9 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x i32> %phi
+}
+
+define <18 x half> @bitcast_v9i32_to_v18f16(<9 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9i32_to_v18f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v26, v8
+; GCN-NEXT:    v_mov_b32_e32 v25, v7
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v5
+; GCN-NEXT:    v_mov_b32_e32 v22, v4
+; GCN-NEXT:    v_mov_b32_e32 v21, v3
+; GCN-NEXT:    v_mov_b32_e32 v20, v2
+; GCN-NEXT:    v_mov_b32_e32 v19, v1
+; GCN-NEXT:    v_mov_b32_e32 v18, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB4_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB4_4
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB4_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  .LBB4_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9i32_to_v18f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9i32_to_v18f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9i32_to_v18f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <9 x i32> %a, splat (i32 3)
+  %a2 = bitcast <9 x i32> %a1 to <18 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x i32> %a to <18 x half>
+  br label %end
+
+end:
+  %phi = phi <18 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x half> %phi
+}
+
+define <9 x i32> @bitcast_v18f16_to_v9i32(<18 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18f16_to_v9i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v16
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB5_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB5_4
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB5_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v0, v29, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v27, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v2, v23, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v21, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v19, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v18, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v12, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v10, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  .LBB5_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v27
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v14, v8
+; GCN-NEXT:    v_or_b32_e32 v6, v12, v15
+; GCN-NEXT:    v_or_b32_e32 v7, v10, v13
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v11
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18f16_to_v9i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v9, 0x200
+; VI-NEXT:    v_add_f16_sdwa v10, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v7, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v6, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v5, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v2, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v1, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v9, v0, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v10
+; VI-NEXT:    v_or_b32_e32 v0, v0, v9
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18f16_to_v9i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18f16_to_v9i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <18 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <18 x half> %a1 to <9 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x half> %a to <9 x i32>
+  br label %end
+
+end:
+  %phi = phi <9 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x i32> %phi
+}
+
+define <18 x i16> @bitcast_v9f32_to_v18i16(<9 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9f32_to_v18i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB6_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB6_4
+; GCN-NEXT:  .LBB6_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB6_3: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v17, s4, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  .LBB6_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_alignbit_b32 v17, s4, v16, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9f32_to_v18i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9f32_to_v18i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9f32_to_v18i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v8, 1.0, v8 :: v_dual_add_f32 v7, 1.0, v7
+; GFX11-NEXT:    v_dual_add_f32 v6, 1.0, v6 :: v_dual_add_f32 v5, 1.0, v5
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <9 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <9 x float> %a1 to <18 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x float> %a to <18 x i16>
+  br label %end
+
+end:
+  %phi = phi <18 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x i16> %phi
+}
+
+define <9 x float> @bitcast_v18i16_to_v9f32(<18 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18i16_to_v9f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v23, v8
+; GCN-NEXT:    v_mov_b32_e32 v22, v6
+; GCN-NEXT:    v_mov_b32_e32 v21, v4
+; GCN-NEXT:    v_mov_b32_e32 v20, v2
+; GCN-NEXT:    v_mov_b32_e32 v19, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_4
+; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v25
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v26
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v18
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v24
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  .LBB7_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v19
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_or_b32_e32 v0, v25, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v26, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v18, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v24, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 0x30000, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 0x30000, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x30000, v8
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18i16_to_v9f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v10, 3
+; VI-NEXT:    v_add_u16_e32 v9, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_u16_e32 v9, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v9, v7
+; VI-NEXT:    v_add_u16_e32 v9, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v9, v6
+; VI-NEXT:    v_add_u16_e32 v9, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v9, v5
+; VI-NEXT:    v_add_u16_e32 v9, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v9, v4
+; VI-NEXT:    v_add_u16_e32 v9, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v9, v3
+; VI-NEXT:    v_add_u16_e32 v9, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v9, v2
+; VI-NEXT:    v_add_u16_e32 v9, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v9, v1
+; VI-NEXT:    v_add_u16_e32 v9, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v9, v0
+; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18i16_to_v9f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18i16_to_v9f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <18 x i16> %a, splat (i16 3)
+  %a2 = bitcast <18 x i16> %a1 to <9 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x i16> %a to <9 x float>
+  br label %end
+
+end:
+  %phi = phi <9 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x float> %phi
+}
+
+define <18 x half> @bitcast_v9f32_to_v18f16(<9 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9f32_to_v18f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v26, v8
+; GCN-NEXT:    v_mov_b32_e32 v25, v7
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v5
+; GCN-NEXT:    v_mov_b32_e32 v22, v4
+; GCN-NEXT:    v_mov_b32_e32 v21, v3
+; GCN-NEXT:    v_mov_b32_e32 v20, v2
+; GCN-NEXT:    v_mov_b32_e32 v19, v1
+; GCN-NEXT:    v_mov_b32_e32 v18, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_4
+; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB8_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  .LBB8_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9f32_to_v18f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  ; %bb.2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9f32_to_v18f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9f32_to_v18f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v8, 1.0, v8 :: v_dual_add_f32 v7, 1.0, v7
+; GFX11-NEXT:    v_dual_add_f32 v6, 1.0, v6 :: v_dual_add_f32 v5, 1.0, v5
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <9 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <9 x float> %a1 to <18 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x float> %a to <18 x half>
+  br label %end
+
+end:
+  %phi = phi <18 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x half> %phi
+}
+
+define <9 x float> @bitcast_v18f16_to_v9f32(<18 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18f16_to_v9f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v16
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_4
+; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB9_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v0, v29, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v27, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v2, v23, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v21, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v19, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v18, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v12, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v10, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  .LBB9_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v27
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v14, v8
+; GCN-NEXT:    v_or_b32_e32 v6, v12, v15
+; GCN-NEXT:    v_or_b32_e32 v7, v10, v13
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v11
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18f16_to_v9f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v9, 0x200
+; VI-NEXT:    v_add_f16_sdwa v10, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v7, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v6, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v5, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v2, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v1, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v9, v0, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v10
+; VI-NEXT:    v_or_b32_e32 v0, v0, v9
+; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18f16_to_v9f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18f16_to_v9f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <18 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <18 x half> %a1 to <9 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x half> %a to <9 x float>
+  br label %end
+
+end:
+  %phi = phi <9 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x float> %phi
+}
+
+define <18 x half> @bitcast_v18i16_to_v18f16(<18 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18i16_to_v18f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v35, v17
+; GCN-NEXT:    v_mov_b32_e32 v34, v16
+; GCN-NEXT:    v_mov_b32_e32 v33, v15
+; GCN-NEXT:    v_mov_b32_e32 v32, v14
+; GCN-NEXT:    v_mov_b32_e32 v31, v13
+; GCN-NEXT:    v_mov_b32_e32 v30, v12
+; GCN-NEXT:    v_mov_b32_e32 v29, v11
+; GCN-NEXT:    v_mov_b32_e32 v28, v10
+; GCN-NEXT:    v_mov_b32_e32 v27, v9
+; GCN-NEXT:    v_mov_b32_e32 v26, v8
+; GCN-NEXT:    v_mov_b32_e32 v25, v7
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v5
+; GCN-NEXT:    v_mov_b32_e32 v22, v4
+; GCN-NEXT:    v_mov_b32_e32 v21, v3
+; GCN-NEXT:    v_mov_b32_e32 v20, v2
+; GCN-NEXT:    v_mov_b32_e32 v19, v1
+; GCN-NEXT:    v_mov_b32_e32 v36, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB10_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB10_4
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB10_3: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  .LBB10_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18i16_to_v18f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v9, 3
+; VI-NEXT:    v_add_u16_sdwa v10, v0, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v11, v1, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v12, v2, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v13, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v14, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v15, v5, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v16, v6, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v17, v7, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v9, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_or_b32_e32 v7, v7, v17
+; VI-NEXT:    v_or_b32_e32 v6, v6, v16
+; VI-NEXT:    v_or_b32_e32 v5, v5, v15
+; VI-NEXT:    v_or_b32_e32 v4, v4, v14
+; VI-NEXT:    v_or_b32_e32 v3, v3, v13
+; VI-NEXT:    v_or_b32_e32 v2, v2, v12
+; VI-NEXT:    v_or_b32_e32 v1, v1, v11
+; VI-NEXT:    v_or_b32_e32 v0, v0, v10
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18i16_to_v18f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  ; %bb.2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18i16_to_v18f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <18 x i16> %a, splat (i16 3)
+  %a2 = bitcast <18 x i16> %a1 to <18 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x i16> %a to <18 x half>
+  br label %end
+
+end:
+  %phi = phi <18 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x half> %phi
+}
+
+define <18 x i16> @bitcast_v18f16_to_v18i16(<18 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18f16_to_v18i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v9
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v19
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v18
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v13, 16
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18f16_to_v18i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v10, 0x200
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v0
+; VI-NEXT:    v_add_f16_sdwa v0, v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v1, v1, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v2, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v3
+; VI-NEXT:    v_add_f16_sdwa v3, v3, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v4
+; VI-NEXT:    v_add_f16_sdwa v4, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v5
+; VI-NEXT:    v_add_f16_sdwa v5, v5, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v7, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v8, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v18, v8
+; VI-NEXT:    v_or_b32_e32 v7, v17, v7
+; VI-NEXT:    v_or_b32_e32 v6, v16, v6
+; VI-NEXT:    v_or_b32_e32 v5, v15, v5
+; VI-NEXT:    v_or_b32_e32 v4, v14, v4
+; VI-NEXT:    v_or_b32_e32 v3, v13, v3
+; VI-NEXT:    v_or_b32_e32 v2, v12, v2
+; VI-NEXT:    v_or_b32_e32 v1, v11, v1
+; VI-NEXT:    v_or_b32_e32 v0, v9, v0
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18f16_to_v18i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18f16_to_v18i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v9
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <18 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <18 x half> %a1 to <18 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x half> %a to <18 x i16>
+  br label %end
+
+end:
+  %phi = phi <18 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index 05412efc100f6..467e7740d24cf 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -219,6 +219,384 @@ end:
   ret <10 x i32> %phi
 }
 
+define <20 x i16> @bitcast_v10i32_to_v20i16(<10 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i32_to_v20i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v20, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v20, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB2_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB2_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v20, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB2_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v10, v20
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i32_to_v20i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB2_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB2_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i32_to_v20i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB2_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i32_to_v20i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB2_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i32> %a, splat (i32 3)
+  %a2 = bitcast <10 x i32> %a1 to <20 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i32> %a to <20 x i16>
+  br label %end
+
+end:
+  %phi = phi <20 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i16> %phi
+}
+
+define <10 x i32> @bitcast_v20i16_to_v10i32(<20 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i16_to_v10i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v25, v8
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v4
+; GCN-NEXT:    v_mov_b32_e32 v22, v2
+; GCN-NEXT:    v_mov_b32_e32 v21, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_4
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB3_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v20
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v26
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v27
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v28
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  .LBB3_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v21
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_or_b32_e32 v0, v20, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v26, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v27, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v28, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v29, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 0x30000, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x30000, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x30000, v9
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20i16_to_v10i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v11, 3
+; VI-NEXT:    v_add_u16_e32 v10, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_u16_e32 v10, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v10, v8
+; VI-NEXT:    v_add_u16_e32 v10, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v10, v7
+; VI-NEXT:    v_add_u16_e32 v10, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v10, v6
+; VI-NEXT:    v_add_u16_e32 v10, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v10, v5
+; VI-NEXT:    v_add_u16_e32 v10, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v10, v4
+; VI-NEXT:    v_add_u16_e32 v10, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v10, v3
+; VI-NEXT:    v_add_u16_e32 v10, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v10, v2
+; VI-NEXT:    v_add_u16_e32 v10, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v10, v1
+; VI-NEXT:    v_add_u16_e32 v10, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v10, v0
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20i16_to_v10i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20i16_to_v10i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <20 x i16> %a, splat (i16 3)
+  %a2 = bitcast <20 x i16> %a1 to <10 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x i16> %a to <10 x i32>
+  br label %end
+
+end:
+  %phi = phi <10 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x i32> %phi
+}
+
 define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; GCN-LABEL: bitcast_v10i32_to_v20f16:
 ; GCN:       ; %bb.0:
@@ -256,14 +634,14 @@ define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB2_3
+; GCN-NEXT:    s_cbranch_execnz .LBB4_3
 ; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB2_4
-; GCN-NEXT:  .LBB2_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB4_4
+; GCN-NEXT:  .LBB4_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
-; GCN-NEXT:  .LBB2_3: ; %cmp.false
+; GCN-NEXT:  .LBB4_3: ; %cmp.false
 ; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
 ; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v28
 ; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v27
@@ -305,8 +683,8 @@ define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr28
 ; GCN-NEXT:    ; implicit-def: $vgpr29
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB2_2
-; GCN-NEXT:  .LBB2_4: ; %cmp.true
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  .LBB4_4: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v20
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v21
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v22
@@ -357,7 +735,7 @@ define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB2_2
+; VI-NEXT:    s_cbranch_execz .LBB4_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
@@ -369,7 +747,7 @@ define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB2_2: ; %end
+; VI-NEXT:  .LBB4_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -380,7 +758,7 @@ define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
 ; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
@@ -392,7 +770,7 @@ define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB2_2: ; %end
+; GFX9-NEXT:  .LBB4_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -404,7 +782,7 @@ define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
 ; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
@@ -416,7 +794,7 @@ define <20 x half> @bitcast_v10i32_to_v20f16(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB2_2: ; %end
+; GFX11-NEXT:  .LBB4_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -464,14 +842,14 @@ define <10 x i32> @bitcast_v20f16_to_v10i32(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB3_3
+; GCN-NEXT:    s_cbranch_execnz .LBB5_3
 ; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB3_4
-; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB5_4
+; GCN-NEXT:  .LBB5_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
-; GCN-NEXT:  .LBB3_3: ; %cmp.false
+; GCN-NEXT:  .LBB5_3: ; %cmp.false
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v33
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v32
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v31
@@ -513,8 +891,8 @@ define <10 x i32> @bitcast_v20f16_to_v10i32(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr13
 ; GCN-NEXT:    ; implicit-def: $vgpr10
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB3_2
-; GCN-NEXT:  .LBB3_4: ; %cmp.true
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  .LBB5_4: ; %cmp.true
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v33
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
@@ -605,7 +983,7 @@ define <10 x i32> @bitcast_v20f16_to_v10i32(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:    s_cbranch_execz .LBB5_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_mov_b32_e32 v10, 0x200
 ; VI-NEXT:    v_add_f16_sdwa v11, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -638,7 +1016,7 @@ define <10 x i32> @bitcast_v20f16_to_v10i32(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
 ; VI-NEXT:    v_or_b32_e32 v1, v1, v11
 ; VI-NEXT:    v_or_b32_e32 v0, v0, v10
-; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:  .LBB5_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -649,7 +1027,7 @@ define <10 x i32> @bitcast_v20f16_to_v10i32(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    s_movk_i32 s6, 0x200
 ; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
@@ -662,7 +1040,7 @@ define <10 x i32> @bitcast_v20f16_to_v10i32(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:  .LBB5_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -674,7 +1052,7 @@ define <10 x i32> @bitcast_v20f16_to_v10i32(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
@@ -686,7 +1064,7 @@ define <10 x i32> @bitcast_v20f16_to_v10i32(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
-; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:  .LBB5_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -743,7 +1121,7 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr16
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_alignbit_b32 v11, v10, v9, 24
 ; GCN-NEXT:    v_alignbit_b32 v12, v10, v9, 16
@@ -775,9 +1153,9 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
-; GCN-NEXT:  .LBB4_2: ; %Flow
+; GCN-NEXT:  .LBB6_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB4_4
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
@@ -819,7 +1197,7 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
-; GCN-NEXT:  .LBB4_4: ; %end
+; GCN-NEXT:  .LBB6_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    v_and_b32_e32 v49, 0xff, v1
 ; GCN-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
@@ -969,7 +1347,7 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr11
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:    s_cbranch_execz .LBB6_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; VI-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -1001,9 +1379,9 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB4_2: ; %Flow
+; VI-NEXT:  .LBB6_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB4_4
+; VI-NEXT:    s_cbranch_execz .LBB6_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
 ; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
@@ -1045,7 +1423,7 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB4_4: ; %end
+; VI-NEXT:  .LBB6_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -1155,7 +1533,7 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr11
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX9-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -1187,9 +1565,9 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB4_2: ; %Flow
+; GFX9-NEXT:  .LBB6_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB4_4
+; GFX9-NEXT:    s_cbranch_execz .LBB6_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
 ; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
@@ -1231,7 +1609,7 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB4_4: ; %end
+; GFX9-NEXT:  .LBB6_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -1333,7 +1711,7 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX11-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX11-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -1365,9 +1743,9 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB4_2: ; %Flow
+; GFX11-NEXT:  .LBB6_2: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB4_4
+; GFX11-NEXT:    s_cbranch_execz .LBB6_4
 ; GFX11-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
 ; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
@@ -1409,7 +1787,7 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB4_4: ; %end
+; GFX11-NEXT:  .LBB6_4: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
@@ -1572,7 +1950,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v32
@@ -1684,9 +2062,9 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr23
 ; GCN-NEXT:    ; implicit-def: $vgpr53
 ; GCN-NEXT:    ; implicit-def: $vgpr25
-; GCN-NEXT:  .LBB5_2: ; %Flow
+; GCN-NEXT:  .LBB7_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB5_4
+; GCN-NEXT:    s_cbranch_execz .LBB7_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
 ; GCN-NEXT:    s_movk_i32 s6, 0x300
@@ -1800,7 +2178,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x3000000, v8
 ; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x3000000, v9
-; GCN-NEXT:  .LBB5_4: ; %end
+; GCN-NEXT:  .LBB7_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -1860,7 +2238,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:    s_cbranch_execz .LBB7_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_or_b32_sdwa v0, v31, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v32, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -1933,9 +2311,9 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr15
 ; VI-NEXT:    ; implicit-def: $vgpr13
 ; VI-NEXT:    ; implicit-def: $vgpr11
-; VI-NEXT:  .LBB5_2: ; %Flow
+; VI-NEXT:  .LBB7_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB5_4
+; VI-NEXT:    s_cbranch_execz .LBB7_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v31
 ; VI-NEXT:    v_add_u16_e32 v1, 3, v32
@@ -2009,7 +2387,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_add_u16_sdwa v9, v11, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v9, v10, v9
-; VI-NEXT:  .LBB5_4: ; %end
+; VI-NEXT:  .LBB7_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -2072,7 +2450,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v31, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -2145,9 +2523,9 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr15
 ; GFX9-NEXT:    ; implicit-def: $vgpr13
 ; GFX9-NEXT:    ; implicit-def: $vgpr11
-; GFX9-NEXT:  .LBB5_2: ; %Flow
+; GFX9-NEXT:  .LBB7_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB5_4
+; GFX9-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v31
 ; GFX9-NEXT:    v_add_u16_e32 v1, 3, v32
@@ -2221,7 +2599,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
-; GFX9-NEXT:  .LBB5_4: ; %end
+; GFX9-NEXT:  .LBB7_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -2279,15 +2657,15 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB5_3
+; GFX11-NEXT:    s_cbranch_execnz .LBB7_3
 ; GFX11-NEXT:  ; %bb.1: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB5_4
-; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_cbranch_execnz .LBB7_4
+; GFX11-NEXT:  .LBB7_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-; GFX11-NEXT:  .LBB5_3: ; %cmp.false
+; GFX11-NEXT:  .LBB7_3: ; %cmp.false
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v32
 ; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v33
@@ -2400,8 +2778,8 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    ; implicit-def: $vgpr17
 ; GFX11-NEXT:    ; implicit-def: $vgpr19
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB5_2
-; GFX11-NEXT:  .LBB5_4: ; %cmp.true
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  .LBB7_4: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u16 v0, v31, 3
 ; GFX11-NEXT:    v_add_nc_u16 v1, v32, 3
 ; GFX11-NEXT:    v_add_nc_u16 v2, v33, 3
@@ -2540,7 +2918,7 @@ define <5 x double> @bitcast_v10i32_to_v5f64(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
@@ -2552,7 +2930,7 @@ define <5 x double> @bitcast_v10i32_to_v5f64(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB6_2: ; %end
+; GCN-NEXT:  .LBB8_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2563,7 +2941,7 @@ define <5 x double> @bitcast_v10i32_to_v5f64(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:    s_cbranch_execz .LBB8_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
@@ -2575,7 +2953,7 @@ define <5 x double> @bitcast_v10i32_to_v5f64(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB6_2: ; %end
+; VI-NEXT:  .LBB8_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2586,7 +2964,7 @@ define <5 x double> @bitcast_v10i32_to_v5f64(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
 ; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
@@ -2598,7 +2976,7 @@ define <5 x double> @bitcast_v10i32_to_v5f64(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB6_2: ; %end
+; GFX9-NEXT:  .LBB8_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2610,7 +2988,7 @@ define <5 x double> @bitcast_v10i32_to_v5f64(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
 ; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
@@ -2622,7 +3000,7 @@ define <5 x double> @bitcast_v10i32_to_v5f64(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB6_2: ; %end
+; GFX11-NEXT:  .LBB8_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -2650,14 +3028,14 @@ define <10 x i32> @bitcast_v5f64_to_v10i32(<5 x double> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:  .LBB9_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2668,14 +3046,14 @@ define <10 x i32> @bitcast_v5f64_to_v10i32(<5 x double> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:    s_cbranch_execz .LBB9_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:  .LBB9_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2686,14 +3064,14 @@ define <10 x i32> @bitcast_v5f64_to_v10i32(<5 x double> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:  .LBB7_2: ; %end
+; GFX9-NEXT:  .LBB9_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2705,14 +3083,14 @@ define <10 x i32> @bitcast_v5f64_to_v10i32(<5 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:  .LBB9_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -2740,7 +3118,7 @@ define <5 x i64> @bitcast_v10i32_to_v5i64(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
@@ -2752,7 +3130,7 @@ define <5 x i64> @bitcast_v10i32_to_v5i64(<10 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:  .LBB10_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2763,7 +3141,7 @@ define <5 x i64> @bitcast_v10i32_to_v5i64(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:    s_cbranch_execz .LBB10_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
@@ -2775,7 +3153,7 @@ define <5 x i64> @bitcast_v10i32_to_v5i64(<10 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB8_2: ; %end
+; VI-NEXT:  .LBB10_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2786,7 +3164,7 @@ define <5 x i64> @bitcast_v10i32_to_v5i64(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
 ; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
@@ -2798,7 +3176,7 @@ define <5 x i64> @bitcast_v10i32_to_v5i64(<10 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB8_2: ; %end
+; GFX9-NEXT:  .LBB10_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2810,7 +3188,7 @@ define <5 x i64> @bitcast_v10i32_to_v5i64(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
 ; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
@@ -2822,7 +3200,7 @@ define <5 x i64> @bitcast_v10i32_to_v5i64(<10 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB8_2: ; %end
+; GFX11-NEXT:  .LBB10_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -2850,7 +3228,7 @@ define <10 x i32> @bitcast_v5i64_to_v10i32(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
@@ -2862,7 +3240,7 @@ define <10 x i32> @bitcast_v5i64_to_v10i32(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:  .LBB11_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2873,7 +3251,7 @@ define <10 x i32> @bitcast_v5i64_to_v10i32(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:    s_cbranch_execz .LBB11_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
@@ -2885,7 +3263,7 @@ define <10 x i32> @bitcast_v5i64_to_v10i32(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:  .LBB11_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2896,7 +3274,7 @@ define <10 x i32> @bitcast_v5i64_to_v10i32(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
 ; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
@@ -2908,7 +3286,7 @@ define <10 x i32> @bitcast_v5i64_to_v10i32(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:  .LBB9_2: ; %end
+; GFX9-NEXT:  .LBB11_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2920,7 +3298,7 @@ define <10 x i32> @bitcast_v5i64_to_v10i32(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -2935,7 +3313,7 @@ define <10 x i32> @bitcast_v5i64_to_v10i32(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:  .LBB11_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -2955,145 +3333,81 @@ end:
   ret <10 x i32> %phi
 }
 
-define <20 x half> @bitcast_v10f32_to_v20f16(<10 x float> %a, i32 %b) {
-; GCN-LABEL: bitcast_v10f32_to_v20f16:
+define <20 x i16> @bitcast_v10f32_to_v20i16(<10 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f32_to_v20i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v29, v9
-; GCN-NEXT:    v_mov_b32_e32 v28, v8
-; GCN-NEXT:    v_mov_b32_e32 v27, v7
-; GCN-NEXT:    v_mov_b32_e32 v26, v6
-; GCN-NEXT:    v_mov_b32_e32 v25, v5
-; GCN-NEXT:    v_mov_b32_e32 v24, v4
-; GCN-NEXT:    v_mov_b32_e32 v23, v3
-; GCN-NEXT:    v_mov_b32_e32 v22, v2
-; GCN-NEXT:    v_mov_b32_e32 v21, v1
-; GCN-NEXT:    v_mov_b32_e32 v20, v0
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v20, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
-; GCN-NEXT:    ; implicit-def: $vgpr0
 ; GCN-NEXT:    ; implicit-def: $vgpr1
-; GCN-NEXT:    ; implicit-def: $vgpr2
 ; GCN-NEXT:    ; implicit-def: $vgpr3
-; GCN-NEXT:    ; implicit-def: $vgpr4
 ; GCN-NEXT:    ; implicit-def: $vgpr5
-; GCN-NEXT:    ; implicit-def: $vgpr6
 ; GCN-NEXT:    ; implicit-def: $vgpr7
-; GCN-NEXT:    ; implicit-def: $vgpr8
 ; GCN-NEXT:    ; implicit-def: $vgpr9
-; GCN-NEXT:    ; implicit-def: $vgpr10
 ; GCN-NEXT:    ; implicit-def: $vgpr11
-; GCN-NEXT:    ; implicit-def: $vgpr12
 ; GCN-NEXT:    ; implicit-def: $vgpr13
-; GCN-NEXT:    ; implicit-def: $vgpr14
 ; GCN-NEXT:    ; implicit-def: $vgpr15
-; GCN-NEXT:    ; implicit-def: $vgpr16
 ; GCN-NEXT:    ; implicit-def: $vgpr17
-; GCN-NEXT:    ; implicit-def: $vgpr18
 ; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB10_3
-; GCN-NEXT:  ; %bb.1: ; %Flow
-; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB10_4
-; GCN-NEXT:  .LBB10_2: ; %end
-; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_setpc_b64 s[30:31]
-; GCN-NEXT:  .LBB10_3: ; %cmp.false
-; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
-; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v28
-; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v27
-; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v26
-; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v25
-; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v24
-; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v23
-; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
-; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v21
-; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
-; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v29
-; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v28
-; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v27
-; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v26
-; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v25
-; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v24
-; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v23
-; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v22
-; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v21
-; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v0
-; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v1
-; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v3
-; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v5
-; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v7
-; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v30
-; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v31
-; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
-; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
-; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v20
-; GCN-NEXT:    ; implicit-def: $vgpr20
-; GCN-NEXT:    ; implicit-def: $vgpr21
-; GCN-NEXT:    ; implicit-def: $vgpr22
-; GCN-NEXT:    ; implicit-def: $vgpr23
-; GCN-NEXT:    ; implicit-def: $vgpr24
-; GCN-NEXT:    ; implicit-def: $vgpr25
-; GCN-NEXT:    ; implicit-def: $vgpr26
-; GCN-NEXT:    ; implicit-def: $vgpr27
-; GCN-NEXT:    ; implicit-def: $vgpr28
-; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v20, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB12_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB10_2
-; GCN-NEXT:  .LBB10_4: ; %cmp.true
-; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v20
-; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v21
-; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v22
-; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v23
-; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v24
-; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v25
-; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v26
-; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v27
-; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v28
-; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v29
-; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v19
-; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v17
-; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v15
-; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
-; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
-; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
-; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
-; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
-; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
-; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
-; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
-; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
-; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
-; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
-; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
-; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_cbranch_execz .LBB12_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v20, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB12_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v10, v20
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v10f32_to_v20f16:
+; VI-LABEL: bitcast_v10f32_to_v20i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:    s_cbranch_execz .LBB12_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
 ; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
@@ -3105,18 +3419,18 @@ define <20 x half> @bitcast_v10f32_to_v20f16(<10 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
 ; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
 ; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:  .LBB12_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v10f32_to_v20f16:
+; GFX9-LABEL: bitcast_v10f32_to_v20i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
 ; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
@@ -3128,11 +3442,11 @@ define <20 x half> @bitcast_v10f32_to_v20f16(<10 x float> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
 ; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
 ; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:  .LBB12_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v10f32_to_v20f16:
+; GFX11-LABEL: bitcast_v10f32_to_v20i16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -3154,249 +3468,204 @@ define <20 x half> @bitcast_v10f32_to_v20f16(<10 x float> %a, i32 %b) {
 
 cmp.true:
   %a1 = fadd <10 x float> %a, splat (float 1.000000e+00)
-  %a2 = bitcast <10 x float> %a1 to <20 x half>
+  %a2 = bitcast <10 x float> %a1 to <20 x i16>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <10 x float> %a to <20 x half>
+  %a3 = bitcast <10 x float> %a to <20 x i16>
   br label %end
 
 end:
-  %phi = phi <20 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <20 x half> %phi
+  %phi = phi <20 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i16> %phi
 }
 
-define <10 x float> @bitcast_v20f16_to_v10f32(<20 x half> %a, i32 %b) {
-; GCN-LABEL: bitcast_v20f16_to_v10f32:
+define <10 x float> @bitcast_v20i16_to_v10f32(<20 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i16_to_v10f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v25, v8
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v4
+; GCN-NEXT:    v_mov_b32_e32 v22, v2
+; GCN-NEXT:    v_mov_b32_e32 v21, v0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
-; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v1
-; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v0
-; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v3
-; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v2
-; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v5
-; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v4
-; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v7
-; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v6
-; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v9
-; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v8
-; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v11
-; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v10
-; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v13
-; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v12
-; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v14
-; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v17
-; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v16
-; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v19
-; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB11_3
+; GCN-NEXT:    s_cbranch_execnz .LBB13_3
 ; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB11_4
-; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB13_4
+; GCN-NEXT:  .LBB13_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
-; GCN-NEXT:  .LBB11_3: ; %cmp.false
-; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v33
-; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v32
-; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v31
-; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v30
-; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v28
-; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v26
-; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v24
-; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v15
-; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v14
-; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v13
-; GCN-NEXT:    v_or_b32_e32 v0, v29, v0
-; GCN-NEXT:    v_or_b32_e32 v1, v27, v1
-; GCN-NEXT:    v_or_b32_e32 v2, v25, v2
-; GCN-NEXT:    v_or_b32_e32 v3, v23, v3
-; GCN-NEXT:    v_or_b32_e32 v4, v22, v4
-; GCN-NEXT:    v_or_b32_e32 v5, v21, v5
-; GCN-NEXT:    v_or_b32_e32 v6, v20, v6
-; GCN-NEXT:    v_or_b32_e32 v7, v12, v7
-; GCN-NEXT:    v_or_b32_e32 v8, v11, v8
-; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
-; GCN-NEXT:    ; implicit-def: $vgpr33
-; GCN-NEXT:    ; implicit-def: $vgpr29
-; GCN-NEXT:    ; implicit-def: $vgpr32
-; GCN-NEXT:    ; implicit-def: $vgpr27
-; GCN-NEXT:    ; implicit-def: $vgpr31
-; GCN-NEXT:    ; implicit-def: $vgpr25
-; GCN-NEXT:    ; implicit-def: $vgpr30
-; GCN-NEXT:    ; implicit-def: $vgpr23
-; GCN-NEXT:    ; implicit-def: $vgpr28
-; GCN-NEXT:    ; implicit-def: $vgpr22
-; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:  .LBB13_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v20
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v26
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v27
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v28
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
 ; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
 ; GCN-NEXT:    ; implicit-def: $vgpr24
-; GCN-NEXT:    ; implicit-def: $vgpr20
-; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr10
 ; GCN-NEXT:    ; implicit-def: $vgpr12
 ; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
 ; GCN-NEXT:    ; implicit-def: $vgpr11
 ; GCN-NEXT:    ; implicit-def: $vgpr13
-; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB11_2
-; GCN-NEXT:  .LBB11_4: ; %cmp.true
-; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v33
-; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
-; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
-; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v27
-; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v31
-; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v25
-; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v30
-; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v23
-; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v28
-; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v22
-; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v26
-; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v21
-; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v24
-; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v20
-; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
-; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
-; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
-; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
-; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
-; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
-; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
-; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
-; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
-; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
-; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
-; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
-; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
-; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
-; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
-; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
-; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
-; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
-; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
-; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
-; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
-; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
-; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
-; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
-; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
-; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
-; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
-; GCN-NEXT:    v_or_b32_e32 v5, v17, v16
-; GCN-NEXT:    v_or_b32_e32 v6, v19, v18
-; GCN-NEXT:    v_or_b32_e32 v7, v12, v15
-; GCN-NEXT:    v_or_b32_e32 v8, v11, v14
-; GCN-NEXT:    v_or_b32_e32 v9, v10, v13
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  .LBB13_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v21
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_or_b32_e32 v0, v20, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v26, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v27, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v28, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v29, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 0x30000, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x30000, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x30000, v9
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v20f16_to_v10f32:
+; VI-LABEL: bitcast_v20i16_to_v10f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:    s_cbranch_execz .LBB13_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_mov_b32_e32 v10, 0x200
-; VI-NEXT:    v_add_f16_sdwa v11, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
-; VI-NEXT:    v_or_b32_e32 v9, v9, v11
-; VI-NEXT:    v_add_f16_sdwa v11, v8, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
-; VI-NEXT:    v_or_b32_e32 v8, v8, v11
-; VI-NEXT:    v_add_f16_sdwa v11, v7, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
-; VI-NEXT:    v_or_b32_e32 v7, v7, v11
-; VI-NEXT:    v_add_f16_sdwa v11, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
-; VI-NEXT:    v_or_b32_e32 v6, v6, v11
-; VI-NEXT:    v_add_f16_sdwa v11, v5, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
-; VI-NEXT:    v_or_b32_e32 v5, v5, v11
-; VI-NEXT:    v_add_f16_sdwa v11, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
-; VI-NEXT:    v_or_b32_e32 v4, v4, v11
-; VI-NEXT:    v_add_f16_sdwa v11, v3, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
-; VI-NEXT:    v_or_b32_e32 v3, v3, v11
-; VI-NEXT:    v_add_f16_sdwa v11, v2, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
-; VI-NEXT:    v_or_b32_e32 v2, v2, v11
-; VI-NEXT:    v_add_f16_sdwa v11, v1, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
-; VI-NEXT:    v_add_f16_sdwa v10, v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
-; VI-NEXT:    v_or_b32_e32 v1, v1, v11
-; VI-NEXT:    v_or_b32_e32 v0, v0, v10
-; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v11, 3
+; VI-NEXT:    v_add_u16_e32 v10, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_u16_e32 v10, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v10, v8
+; VI-NEXT:    v_add_u16_e32 v10, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v10, v7
+; VI-NEXT:    v_add_u16_e32 v10, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v10, v6
+; VI-NEXT:    v_add_u16_e32 v10, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v10, v5
+; VI-NEXT:    v_add_u16_e32 v10, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v10, v4
+; VI-NEXT:    v_add_u16_e32 v10, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v10, v3
+; VI-NEXT:    v_add_u16_e32 v10, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v10, v2
+; VI-NEXT:    v_add_u16_e32 v10, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v10, v1
+; VI-NEXT:    v_add_u16_e32 v10, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v10, v0
+; VI-NEXT:  .LBB13_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v20f16_to_v10f32:
+; GFX9-LABEL: bitcast_v20i16_to_v10f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    s_movk_i32 s6, 0x200
-; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB13_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v20f16_to_v10f32:
+; GFX11-LABEL: bitcast_v20i16_to_v10f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -3404,31 +3673,31 @@ define <10 x float> @bitcast_v20f16_to_v10f32(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
-; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
-; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB13_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <20 x half> %a, splat (half 0xH0200)
-  %a2 = bitcast <20 x half> %a1 to <10 x float>
+  %a1 = add <20 x i16> %a, splat (i16 3)
+  %a2 = bitcast <20 x i16> %a1 to <10 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <20 x half> %a to <10 x float>
+  %a3 = bitcast <20 x i16> %a to <10 x float>
   br label %end
 
 end:
@@ -3436,420 +3705,3797 @@ end:
   ret <10 x float> %phi
 }
 
-define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
-; GCN-LABEL: bitcast_v10f32_to_v40i8:
+define <20 x half> @bitcast_v10f32_to_v20f16(<10 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f32_to_v20f16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
-; GCN-NEXT:    ; implicit-def: $vgpr35
-; GCN-NEXT:    ; implicit-def: $vgpr33
-; GCN-NEXT:    ; implicit-def: $vgpr32
-; GCN-NEXT:    ; implicit-def: $vgpr48
-; GCN-NEXT:    ; implicit-def: $vgpr39
-; GCN-NEXT:    ; implicit-def: $vgpr38
-; GCN-NEXT:    ; implicit-def: $vgpr29
-; GCN-NEXT:    ; implicit-def: $vgpr27
-; GCN-NEXT:    ; implicit-def: $vgpr26
-; GCN-NEXT:    ; implicit-def: $vgpr37
-; GCN-NEXT:    ; implicit-def: $vgpr36
-; GCN-NEXT:    ; implicit-def: $vgpr34
-; GCN-NEXT:    ; implicit-def: $vgpr23
-; GCN-NEXT:    ; implicit-def: $vgpr21
-; GCN-NEXT:    ; implicit-def: $vgpr20
-; GCN-NEXT:    ; implicit-def: $vgpr31
-; GCN-NEXT:    ; implicit-def: $vgpr30
-; GCN-NEXT:    ; implicit-def: $vgpr28
-; GCN-NEXT:    ; implicit-def: $vgpr17
-; GCN-NEXT:    ; implicit-def: $vgpr15
-; GCN-NEXT:    ; implicit-def: $vgpr14
-; GCN-NEXT:    ; implicit-def: $vgpr25
-; GCN-NEXT:    ; implicit-def: $vgpr24
-; GCN-NEXT:    ; implicit-def: $vgpr22
-; GCN-NEXT:    ; implicit-def: $vgpr13
-; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    v_mov_b32_e32 v29, v9
+; GCN-NEXT:    v_mov_b32_e32 v28, v8
+; GCN-NEXT:    v_mov_b32_e32 v27, v7
+; GCN-NEXT:    v_mov_b32_e32 v26, v6
+; GCN-NEXT:    v_mov_b32_e32 v25, v5
+; GCN-NEXT:    v_mov_b32_e32 v24, v4
+; GCN-NEXT:    v_mov_b32_e32 v23, v3
+; GCN-NEXT:    v_mov_b32_e32 v22, v2
+; GCN-NEXT:    v_mov_b32_e32 v21, v1
+; GCN-NEXT:    v_mov_b32_e32 v20, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
 ; GCN-NEXT:    ; implicit-def: $vgpr11
-; GCN-NEXT:    ; implicit-def: $vgpr19
-; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
 ; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:    s_cbranch_execnz .LBB14_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB14_4
+; GCN-NEXT:  .LBB14_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB14_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  .LBB14_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f32_to_v20f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB14_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f32_to_v20f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB14_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f32_to_v20f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <10 x float> %a1 to <20 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x float> %a to <20 x half>
+  br label %end
+
+end:
+  %phi = phi <20 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x half> %phi
+}
+
+define <10 x float> @bitcast_v20f16_to_v10f32(<20 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20f16_to_v10f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v18
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB15_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB15_4
+; GCN-NEXT:  .LBB15_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB15_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v0, v29, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v27, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v25, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v23, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v22, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v21, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v20, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v12, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  .LBB15_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v5, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v6, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v7, v12, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v14
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v13
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20f16_to_v10f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v10, 0x200
+; VI-NEXT:    v_add_f16_sdwa v11, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v8, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v7, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v5, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v3, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v2, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v1, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v10, v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v11
+; VI-NEXT:    v_or_b32_e32 v0, v0, v10
+; VI-NEXT:  .LBB15_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20f16_to_v10f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20f16_to_v10f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <20 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <20 x half> %a1 to <10 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x half> %a to <10 x float>
+  br label %end
+
+end:
+  %phi = phi <10 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x float> %phi
+}
+
+define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f32_to_v40i8:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v11, v10, v9, 24
+; GCN-NEXT:    v_alignbit_b32 v12, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v10, v9, 8
+; GCN-NEXT:    v_alignbit_b32 v14, v8, v7, 24
+; GCN-NEXT:    v_alignbit_b32 v15, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v8, v7, 8
+; GCN-NEXT:    v_alignbit_b32 v20, v6, v5, 24
+; GCN-NEXT:    v_alignbit_b32 v21, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v6, v5, 8
+; GCN-NEXT:    v_alignbit_b32 v26, v4, v3, 24
+; GCN-NEXT:    v_alignbit_b32 v27, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v4, v3, 8
+; GCN-NEXT:    v_alignbit_b32 v32, v2, v1, 24
+; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v2, v1, 8
+; GCN-NEXT:    v_lshrrev_b32_e32 v16, 24, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 8, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 24, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 8, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 24, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 8, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
+; GCN-NEXT:  .LBB16_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_alignbit_b32 v11, v10, v9, 24
+; GCN-NEXT:    v_alignbit_b32 v12, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v10, v9, 8
+; GCN-NEXT:    v_alignbit_b32 v14, v8, v7, 24
+; GCN-NEXT:    v_alignbit_b32 v15, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v8, v7, 8
+; GCN-NEXT:    v_alignbit_b32 v20, v6, v5, 24
+; GCN-NEXT:    v_alignbit_b32 v21, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v6, v5, 8
+; GCN-NEXT:    v_alignbit_b32 v26, v4, v3, 24
+; GCN-NEXT:    v_alignbit_b32 v27, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v4, v3, 8
+; GCN-NEXT:    v_alignbit_b32 v32, v2, v1, 24
+; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v2, v1, 8
+; GCN-NEXT:    v_lshrrev_b32_e32 v16, 24, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 8, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 24, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 8, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 24, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 8, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
+; GCN-NEXT:  .LBB16_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v49, 0xff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
+; GCN-NEXT:    v_and_b32_e32 v33, 0xff, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 24, v32
+; GCN-NEXT:    v_and_b32_e32 v50, 0xff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 8, v48
+; GCN-NEXT:    v_and_b32_e32 v39, 0xff, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 24, v38
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 8, v29
+; GCN-NEXT:    v_and_b32_e32 v27, 0xff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 24, v26
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 8, v37
+; GCN-NEXT:    v_and_b32_e32 v36, 0xff, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 24, v34
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 8, v23
+; GCN-NEXT:    v_and_b32_e32 v21, 0xff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 24, v20
+; GCN-NEXT:    v_or_b32_e32 v35, v49, v35
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 8, v31
+; GCN-NEXT:    v_and_b32_e32 v30, 0xff, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 24, v28
+; GCN-NEXT:    v_or_b32_e32 v48, v50, v48
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 8, v17
+; GCN-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 24, v14
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 8, v25
+; GCN-NEXT:    v_and_b32_e32 v24, 0xff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 24, v22
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 8, v13
+; GCN-NEXT:    v_and_b32_e32 v12, 0xff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 24, v11
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 8, v19
+; GCN-NEXT:    v_and_b32_e32 v18, 0xff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 24, v16
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 36, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v18
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v35
+; GCN-NEXT:    v_or_b32_e32 v19, v32, v33
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v25, v38, v39
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v27
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_or_b32_e32 v27, v34, v36
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v21
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_or_b32_e32 v21, v28, v30
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v15
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v17
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v12
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_or_b32_e32 v12, v16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v18, v19
+; GCN-NEXT:    v_or_b32_e32 v16, v24, v25
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v26
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v27
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v20
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v21
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v14
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v15
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v11
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v12
+; GCN-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v31, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f32_to_v40i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    ; implicit-def: $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr15
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr14
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr13
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr12
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    ; implicit-def: $vgpr17
+; VI-NEXT:    ; implicit-def: $vgpr11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
+; VI-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
+; VI-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
+; VI-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
+; VI-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
+; VI-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 8, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 8, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 24, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 8, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 8, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 24, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 8, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 8, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 24, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
+; VI-NEXT:  .LBB16_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
+; VI-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
+; VI-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 8, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 8, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 24, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 8, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 8, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 24, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 8, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 8, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 24, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
+; VI-NEXT:  .LBB16_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v39
+; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v37
+; VI-NEXT:    v_or_b32_sdwa v2, v38, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 4, v0
+; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v36
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v14
+; VI-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v35, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 8, v0
+; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v34
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v32
+; VI-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v33, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 12, v0
+; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v31
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v13
+; VI-NEXT:    v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v30, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 16, v0
+; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v27
+; VI-NEXT:    v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v28, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 20, v0
+; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v26
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v12
+; VI-NEXT:    v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v25, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 24, v0
+; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v24
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v22
+; VI-NEXT:    v_or_b32_sdwa v1, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v23, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 28, v0
+; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v21
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v11
+; VI-NEXT:    v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v20, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 32, v0
+; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v19
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v17
+; VI-NEXT:    v_or_b32_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v18, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; VI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f32_to_v40i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    ; implicit-def: $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr15
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr14
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr13
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr12
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr17
+; GFX9-NEXT:    ; implicit-def: $vgpr11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
+; GFX9-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
+; GFX9-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
+; GFX9-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
+; GFX9-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 8, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 8, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 24, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 8, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 8, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 24, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 8, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 8, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 24, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
+; GFX9-NEXT:  .LBB16_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
+; GFX9-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 8, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 8, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 24, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 8, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 8, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 24, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 8, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 8, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 24, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
+; GFX9-NEXT:  .LBB16_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
+; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
+; GFX9-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v39
+; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v37
+; GFX9-NEXT:    v_or_b32_sdwa v2, v38, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v36
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v14
+; GFX9-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v35, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v34
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v32
+; GFX9-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v33, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v31
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v13
+; GFX9-NEXT:    v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v30, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v27
+; GFX9-NEXT:    v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v28, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v26
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v12
+; GFX9-NEXT:    v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v25, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v24
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v22
+; GFX9-NEXT:    v_or_b32_sdwa v1, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v23, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v21
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v11
+; GFX9-NEXT:    v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v20, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v19
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v17
+; GFX9-NEXT:    v_or_b32_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v18, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f32_to_v40i8:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v11
+; GFX11-NEXT:    ; implicit-def: $vgpr16
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr15
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr14
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr13
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr12
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr11
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    ; implicit-def: $vgpr17
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
+; GFX11-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
+; GFX11-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
+; GFX11-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
+; GFX11-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 8, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 8, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 24, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 8, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 8, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 24, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 8, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 8, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 24, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
+; GFX11-NEXT:  .LBB16_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v9, 1.0, v9
+; GFX11-NEXT:    v_dual_add_f32 v10, 1.0, v10 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v7, 1.0, v7
+; GFX11-NEXT:    v_dual_add_f32 v8, 1.0, v8 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v6, 1.0, v6 :: v_dual_add_f32 v5, 1.0, v5
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
+; GFX11-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; GFX11-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
+; GFX11-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
+; GFX11-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 8, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 8, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 24, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 8, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 8, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 24, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 8, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 8, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 24, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
+; GFX11-NEXT:  .LBB16_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v16
+; GFX11-NEXT:    v_and_b32_e32 v48, 0xff, v48
+; GFX11-NEXT:    v_lshlrev_b16 v15, 8, v15
+; GFX11-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-NEXT:    v_lshlrev_b16 v36, 8, v36
+; GFX11-NEXT:    v_and_b32_e32 v35, 0xff, v35
+; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v14
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; GFX11-NEXT:    v_lshlrev_b16 v31, 8, v31
+; GFX11-NEXT:    v_and_b32_e32 v30, 0xff, v30
+; GFX11-NEXT:    v_lshlrev_b16 v13, 8, v13
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT:    v_lshlrev_b16 v39, 8, v39
+; GFX11-NEXT:    v_and_b32_e32 v38, 0xff, v38
+; GFX11-NEXT:    v_lshlrev_b16 v37, 8, v37
+; GFX11-NEXT:    v_or_b32_e32 v1, v1, v16
+; GFX11-NEXT:    v_or_b32_e32 v15, v48, v15
+; GFX11-NEXT:    v_or_b32_e32 v3, v3, v36
+; GFX11-NEXT:    v_or_b32_e32 v14, v35, v14
+; GFX11-NEXT:    v_or_b32_e32 v5, v5, v31
+; GFX11-NEXT:    v_or_b32_e32 v13, v30, v13
+; GFX11-NEXT:    v_or_b32_e32 v2, v2, v39
+; GFX11-NEXT:    v_or_b32_e32 v16, v38, v37
+; GFX11-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-NEXT:    v_or_b32_e32 v1, v1, v15
+; GFX11-NEXT:    v_or_b32_e32 v3, v3, v14
+; GFX11-NEXT:    v_or_b32_e32 v5, v5, v13
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GFX11-NEXT:    v_lshlrev_b16 v13, 8, v29
+; GFX11-NEXT:    v_and_b32_e32 v14, 0xff, v28
+; GFX11-NEXT:    v_lshlrev_b16 v15, 8, v27
+; GFX11-NEXT:    v_or_b32_e32 v2, v2, v16
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v26
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; GFX11-NEXT:    v_lshlrev_b16 v34, 8, v34
+; GFX11-NEXT:    v_and_b32_e32 v33, 0xff, v33
+; GFX11-NEXT:    v_lshlrev_b16 v32, 8, v32
+; GFX11-NEXT:    v_and_b32_e32 v25, 0xff, v25
+; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v12
+; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; GFX11-NEXT:    v_lshlrev_b16 v24, 8, v24
+; GFX11-NEXT:    v_or_b32_e32 v6, v6, v13
+; GFX11-NEXT:    v_or_b32_e32 v13, v14, v15
+; GFX11-NEXT:    v_and_b32_e32 v14, 0xff, v23
+; GFX11-NEXT:    v_lshlrev_b16 v15, 8, v22
+; GFX11-NEXT:    v_or_b32_e32 v7, v7, v16
+; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v21
+; GFX11-NEXT:    v_and_b32_e32 v20, 0xff, v20
+; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v11
+; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; GFX11-NEXT:    v_lshlrev_b16 v19, 8, v19
+; GFX11-NEXT:    v_and_b32_e32 v18, 0xff, v18
+; GFX11-NEXT:    v_lshlrev_b16 v17, 8, v17
+; GFX11-NEXT:    v_or_b32_e32 v4, v4, v34
+; GFX11-NEXT:    v_or_b32_e32 v32, v33, v32
+; GFX11-NEXT:    v_or_b32_e32 v12, v25, v12
+; GFX11-NEXT:    v_or_b32_e32 v8, v8, v24
+; GFX11-NEXT:    v_or_b32_e32 v14, v14, v15
+; GFX11-NEXT:    v_or_b32_e32 v9, v9, v16
+; GFX11-NEXT:    v_or_b32_e32 v11, v20, v11
+; GFX11-NEXT:    v_or_b32_e32 v10, v10, v19
+; GFX11-NEXT:    v_or_b32_e32 v15, v18, v17
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GFX11-NEXT:    v_lshlrev_b32_e32 v30, 16, v32
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GFX11-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GFX11-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GFX11-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-NEXT:    v_or_b32_e32 v4, v4, v30
+; GFX11-NEXT:    v_or_b32_e32 v6, v6, v13
+; GFX11-NEXT:    v_or_b32_e32 v7, v7, v12
+; GFX11-NEXT:    v_or_b32_e32 v8, v8, v14
+; GFX11-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX11-NEXT:    v_or_b32_e32 v10, v10, v15
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    scratch_store_b128 v0, v[1:4], off
+; GFX11-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
+; GFX11-NEXT:    scratch_store_b64 v0, v[9:10], off offset:32
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <10 x float> %a1 to <40 x i8>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x float> %a to <40 x i8>
+  br label %end
+
+end:
+  %phi = phi <40 x i8> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x i8> %phi
+}
+
+define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40i8_to_v10f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v35, v8
+; GCN-NEXT:    v_mov_b32_e32 v34, v6
+; GCN-NEXT:    v_mov_b32_e32 v33, v4
+; GCN-NEXT:    v_mov_b32_e32 v32, v2
+; GCN-NEXT:    v_mov_b32_e32 v31, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 8, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 24, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 8, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 24, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 8, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 24, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 8, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 24, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 8, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 24, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 8, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 24, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 8, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 24, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 8, v29
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 24, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 8, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 24, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 8, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 24, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v31
+; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v32
+; GCN-NEXT:    v_and_b32_e32 v2, 0xff, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xff, v34
+; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v35
+; GCN-NEXT:    v_and_b32_e32 v5, 0xff, v10
+; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xff, v20
+; GCN-NEXT:    v_and_b32_e32 v12, 0xff, v22
+; GCN-NEXT:    v_and_b32_e32 v14, 0xff, v24
+; GCN-NEXT:    v_and_b32_e32 v16, 0xff, v26
+; GCN-NEXT:    v_and_b32_e32 v18, 0xff, v28
+; GCN-NEXT:    v_and_b32_e32 v20, 0xff, v30
+; GCN-NEXT:    v_and_b32_e32 v22, 0xff, v41
+; GCN-NEXT:    v_and_b32_e32 v24, 0xff, v40
+; GCN-NEXT:    v_and_b32_e32 v26, 0xff, v55
+; GCN-NEXT:    v_and_b32_e32 v28, 0xff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v37, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v13, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v15, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v12
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v14
+; GCN-NEXT:    v_or_b32_e32 v13, v19, v16
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v18
+; GCN-NEXT:    v_or_b32_e32 v15, v21, v20
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v22
+; GCN-NEXT:    v_or_b32_e32 v17, v23, v24
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v25, v27
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v2, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v3, v6, v7
+; GCN-NEXT:    v_or_b32_e32 v4, v8, v9
+; GCN-NEXT:    v_or_b32_e32 v5, v10, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v12, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v14, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v16, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v18, v19
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
+; GCN-NEXT:    s_movk_i32 s6, 0x300
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v32
+; GCN-NEXT:    s_mov_b32 s7, 0x3000000
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v41
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v55
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v54
+; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; GCN-NEXT:    v_and_b32_e32 v12, 0xff, v12
+; GCN-NEXT:    v_and_b32_e32 v14, 0xff, v14
+; GCN-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; GCN-NEXT:    v_and_b32_e32 v18, 0xff, v18
+; GCN-NEXT:    v_and_b32_e32 v20, 0xff, v20
+; GCN-NEXT:    v_and_b32_e32 v22, 0xff, v22
+; GCN-NEXT:    v_and_b32_e32 v24, 0xff, v24
+; GCN-NEXT:    v_and_b32_e32 v26, 0xff, v26
+; GCN-NEXT:    v_and_b32_e32 v28, 0xff, v28
+; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v39, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v48, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v49, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v50, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v51, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v14, v52, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v18, v27, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v22, v29, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v26, v53, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v37, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v13, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v15, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v14
+; GCN-NEXT:    v_or_b32_e32 v13, v19, v16
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v18
+; GCN-NEXT:    v_or_b32_e32 v15, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v22
+; GCN-NEXT:    v_or_b32_e32 v17, v23, v24
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 0x300, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v25, v27
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s7, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s7, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s7, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s7, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x3000000, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x3000000, v9
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40i8_to_v10f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v35, v8
+; VI-NEXT:    v_mov_b32_e32 v34, v6
+; VI-NEXT:    v_mov_b32_e32 v33, v4
+; VI-NEXT:    v_mov_b32_e32 v32, v2
+; VI-NEXT:    v_mov_b32_e32 v31, v0
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
+; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:4
+; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v9
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v11
+; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v13
+; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v17
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v19
+; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v21
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v23
+; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
+; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v29
+; VI-NEXT:    s_waitcnt vmcnt(9)
+; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v0
+; VI-NEXT:    s_waitcnt vmcnt(8)
+; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v2
+; VI-NEXT:    s_waitcnt vmcnt(7)
+; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v4
+; VI-NEXT:    s_waitcnt vmcnt(6)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_lshlrev_b16_e32 v11, 8, v8
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_or_b32_sdwa v0, v31, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v32, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v33, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v34, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v35, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v10, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v12, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v14, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v18, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v20, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v22, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v26, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v28, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v30, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v8, v39, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v38, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v37, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v36, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr10
+; VI-NEXT:    ; implicit-def: $vgpr12
+; VI-NEXT:    ; implicit-def: $vgpr14
+; VI-NEXT:    ; implicit-def: $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr17
+; VI-NEXT:    ; implicit-def: $vgpr15
+; VI-NEXT:    ; implicit-def: $vgpr13
+; VI-NEXT:    ; implicit-def: $vgpr11
+; VI-NEXT:  .LBB17_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u16_e32 v0, 3, v31
+; VI-NEXT:    v_add_u16_e32 v1, 3, v32
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_mov_b32_e32 v9, 0x300
+; VI-NEXT:    v_add_u16_e32 v0, 0x300, v0
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v33
+; VI-NEXT:    v_add_u16_e32 v2, 3, v34
+; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v2, v40, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 0x300, v1
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v35
+; VI-NEXT:    v_add_u16_e32 v3, 3, v10
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v2, 0x300, v2
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v12
+; VI-NEXT:    v_add_u16_e32 v4, 3, v14
+; VI-NEXT:    v_or_b32_sdwa v3, v53, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v4, v52, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v3, 0x300, v3
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v16
+; VI-NEXT:    v_add_u16_e32 v5, 3, v18
+; VI-NEXT:    v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v4, 0x300, v4
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v20
+; VI-NEXT:    v_add_u16_e32 v6, 3, v22
+; VI-NEXT:    v_or_b32_sdwa v5, v49, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v6, v48, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v5, 0x300, v5
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v24
+; VI-NEXT:    v_add_u16_e32 v7, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v6, v25, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v23, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v6, 0x300, v6
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v28
+; VI-NEXT:    v_add_u16_e32 v8, 3, v30
+; VI-NEXT:    v_or_b32_sdwa v7, v21, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v8, v19, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u16_e32 v8, 3, v39
+; VI-NEXT:    v_add_u16_e32 v10, 3, v38
+; VI-NEXT:    v_or_b32_sdwa v8, v17, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v37
+; VI-NEXT:    v_add_u16_e32 v12, 3, v36
+; VI-NEXT:    v_or_b32_sdwa v10, v13, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
+; VI-NEXT:    v_add_u16_sdwa v9, v11, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:  .LBB17_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40i8_to_v10f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v35, v8
+; GFX9-NEXT:    v_mov_b32_e32 v34, v6
+; GFX9-NEXT:    v_mov_b32_e32 v33, v4
+; GFX9-NEXT:    v_mov_b32_e32 v32, v2
+; GFX9-NEXT:    v_mov_b32_e32 v31, v0
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v11
+; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v13
+; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v15
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v17
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v19
+; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v21
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v23
+; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
+; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v29
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v11, 8, v8
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_or_b32_sdwa v0, v31, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v33, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v34, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v35, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v10, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v12, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v14, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v5, v18, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v5, v20, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v22, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v26, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v28, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v30, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v8, v39, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v38, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v37, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v36, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr10
+; GFX9-NEXT:    ; implicit-def: $vgpr12
+; GFX9-NEXT:    ; implicit-def: $vgpr14
+; GFX9-NEXT:    ; implicit-def: $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr17
+; GFX9-NEXT:    ; implicit-def: $vgpr15
+; GFX9-NEXT:    ; implicit-def: $vgpr13
+; GFX9-NEXT:    ; implicit-def: $vgpr11
+; GFX9-NEXT:  .LBB17_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v31
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v32
+; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_movk_i32 s6, 0x300
+; GFX9-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
+; GFX9-NEXT:    v_add_u16_sdwa v1, v1, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v33
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v34
+; GFX9-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v2, v40, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v1, 0x300, v1
+; GFX9-NEXT:    v_add_u16_sdwa v2, v2, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v35
+; GFX9-NEXT:    v_add_u16_e32 v3, 3, v10
+; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v2
+; GFX9-NEXT:    v_add_u16_sdwa v3, v3, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX9-NEXT:    v_add_u16_e32 v3, 3, v12
+; GFX9-NEXT:    v_add_u16_e32 v4, 3, v14
+; GFX9-NEXT:    v_or_b32_sdwa v3, v53, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v4, v52, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v3
+; GFX9-NEXT:    v_add_u16_sdwa v4, v4, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX9-NEXT:    v_add_u16_e32 v4, 3, v16
+; GFX9-NEXT:    v_add_u16_e32 v5, 3, v18
+; GFX9-NEXT:    v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v4
+; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX9-NEXT:    v_add_u16_e32 v5, 3, v20
+; GFX9-NEXT:    v_add_u16_e32 v6, 3, v22
+; GFX9-NEXT:    v_or_b32_sdwa v5, v49, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v6, v48, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v5, 0x300, v5
+; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v5, v5, v6
+; GFX9-NEXT:    v_add_u16_e32 v6, 3, v24
+; GFX9-NEXT:    v_add_u16_e32 v7, 3, v26
+; GFX9-NEXT:    v_or_b32_sdwa v6, v25, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v23, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v6
+; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v6, v6, v7
+; GFX9-NEXT:    v_add_u16_e32 v7, 3, v28
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v30
+; GFX9-NEXT:    v_or_b32_sdwa v7, v21, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v8, v19, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
+; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v39
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v8, v17, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v9, v15, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
+; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v37
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v36
+; GFX9-NEXT:    v_or_b32_sdwa v9, v13, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
+; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
+; GFX9-NEXT:  .LBB17_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40i8_to_v10f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v35, v8 :: v_dual_mov_b32 v34, v6
+; GFX11-NEXT:    v_dual_mov_b32 v33, v4 :: v_dual_mov_b32 v32, v2
+; GFX11-NEXT:    v_mov_b32_e32 v31, v0
+; GFX11-NEXT:    s_clause 0x9
+; GFX11-NEXT:    scratch_load_b32 v0, off, s32 offset:36
+; GFX11-NEXT:    scratch_load_u16 v2, off, s32
+; GFX11-NEXT:    scratch_load_u16 v4, off, s32 offset:8
+; GFX11-NEXT:    scratch_load_u16 v6, off, s32 offset:16
+; GFX11-NEXT:    scratch_load_u16 v8, off, s32 offset:24
+; GFX11-NEXT:    scratch_load_u16 v66, off, s32 offset:32
+; GFX11-NEXT:    scratch_load_u16 v36, off, s32 offset:28
+; GFX11-NEXT:    scratch_load_u16 v37, off, s32 offset:20
+; GFX11-NEXT:    scratch_load_u16 v38, off, s32 offset:12
+; GFX11-NEXT:    scratch_load_u16 v39, off, s32 offset:4
+; GFX11-NEXT:    v_lshlrev_b16 v53, 8, v1
+; GFX11-NEXT:    v_lshlrev_b16 v54, 8, v3
+; GFX11-NEXT:    v_lshlrev_b16 v55, 8, v5
+; GFX11-NEXT:    v_lshlrev_b16 v64, 8, v7
+; GFX11-NEXT:    v_lshlrev_b16 v65, 8, v9
+; GFX11-NEXT:    v_lshlrev_b16 v48, 8, v11
+; GFX11-NEXT:    v_lshlrev_b16 v49, 8, v13
+; GFX11-NEXT:    v_lshlrev_b16 v50, 8, v15
+; GFX11-NEXT:    v_lshlrev_b16 v51, 8, v17
+; GFX11-NEXT:    v_lshlrev_b16 v52, 8, v19
+; GFX11-NEXT:    v_lshlrev_b16 v21, 8, v21
+; GFX11-NEXT:    v_lshlrev_b16 v23, 8, v23
+; GFX11-NEXT:    v_lshlrev_b16 v25, 8, v25
+; GFX11-NEXT:    v_lshlrev_b16 v27, 8, v27
+; GFX11-NEXT:    v_lshlrev_b16 v29, 8, v29
+; GFX11-NEXT:    s_waitcnt vmcnt(9)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    s_waitcnt vmcnt(8)
+; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v2
+; GFX11-NEXT:    s_waitcnt vmcnt(7)
+; GFX11-NEXT:    v_lshlrev_b16 v13, 8, v4
+; GFX11-NEXT:    s_waitcnt vmcnt(6)
+; GFX11-NEXT:    v_lshlrev_b16 v15, 8, v6
+; GFX11-NEXT:    s_waitcnt vmcnt(5)
+; GFX11-NEXT:    v_lshlrev_b16 v17, 8, v8
+; GFX11-NEXT:    s_waitcnt vmcnt(4)
+; GFX11-NEXT:    v_lshlrev_b16 v19, 8, v66
+; GFX11-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execnz .LBB17_3
+; GFX11-NEXT:  ; %bb.1: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execnz .LBB17_4
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-NEXT:  .LBB17_3: ; %cmp.false
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v31
+; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v32
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v33
+; GFX11-NEXT:    v_and_b32_e32 v3, 0xff, v34
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v35
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v10
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v12
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v14
+; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v16
+; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v18
+; GFX11-NEXT:    v_or_b32_e32 v0, v0, v53
+; GFX11-NEXT:    v_or_b32_e32 v1, v1, v54
+; GFX11-NEXT:    v_or_b32_e32 v2, v2, v55
+; GFX11-NEXT:    v_or_b32_e32 v3, v3, v64
+; GFX11-NEXT:    v_or_b32_e32 v4, v4, v65
+; GFX11-NEXT:    v_or_b32_e32 v5, v5, v48
+; GFX11-NEXT:    v_or_b32_e32 v6, v6, v49
+; GFX11-NEXT:    v_or_b32_e32 v7, v7, v50
+; GFX11-NEXT:    v_or_b32_e32 v8, v8, v51
+; GFX11-NEXT:    v_or_b32_e32 v9, v9, v52
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GFX11-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GFX11-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT:    v_or_b32_e32 v1, v2, v3
+; GFX11-NEXT:    v_or_b32_e32 v2, v4, v5
+; GFX11-NEXT:    v_or_b32_e32 v3, v6, v7
+; GFX11-NEXT:    v_or_b32_e32 v4, v8, v9
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v20
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v22
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v24
+; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v26
+; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v28
+; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v30
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v12, 0xff, v39
+; GFX11-NEXT:    v_and_b32_e32 v14, 0xff, v38
+; GFX11-NEXT:    v_and_b32_e32 v16, 0xff, v37
+; GFX11-NEXT:    v_and_b32_e32 v18, 0xff, v36
+; GFX11-NEXT:    v_or_b32_e32 v5, v5, v21
+; GFX11-NEXT:    v_or_b32_e32 v6, v6, v23
+; GFX11-NEXT:    v_or_b32_e32 v7, v7, v25
+; GFX11-NEXT:    v_or_b32_e32 v8, v8, v27
+; GFX11-NEXT:    v_or_b32_e32 v9, v9, v29
+; GFX11-NEXT:    v_or_b32_e32 v10, v10, v11
+; GFX11-NEXT:    v_or_b32_e32 v11, v12, v13
+; GFX11-NEXT:    v_or_b32_e32 v12, v14, v15
+; GFX11-NEXT:    v_or_b32_e32 v13, v16, v17
+; GFX11-NEXT:    v_or_b32_e32 v14, v18, v19
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GFX11-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GFX11-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GFX11-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-NEXT:    v_or_b32_e32 v5, v5, v6
+; GFX11-NEXT:    v_or_b32_e32 v6, v7, v8
+; GFX11-NEXT:    v_or_b32_e32 v7, v9, v10
+; GFX11-NEXT:    v_or_b32_e32 v8, v11, v12
+; GFX11-NEXT:    v_or_b32_e32 v9, v13, v14
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr10
+; GFX11-NEXT:    ; implicit-def: $vgpr12
+; GFX11-NEXT:    ; implicit-def: $vgpr14
+; GFX11-NEXT:    ; implicit-def: $vgpr16
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr11
+; GFX11-NEXT:    ; implicit-def: $vgpr13
+; GFX11-NEXT:    ; implicit-def: $vgpr15
+; GFX11-NEXT:    ; implicit-def: $vgpr17
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  .LBB17_4: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u16 v0, v31, 3
+; GFX11-NEXT:    v_add_nc_u16 v1, v32, 3
+; GFX11-NEXT:    v_add_nc_u16 v2, v33, 3
+; GFX11-NEXT:    v_add_nc_u16 v3, v34, 3
+; GFX11-NEXT:    v_add_nc_u16 v4, v35, 3
+; GFX11-NEXT:    v_add_nc_u16 v5, v10, 3
+; GFX11-NEXT:    v_add_nc_u16 v6, v12, 3
+; GFX11-NEXT:    v_add_nc_u16 v7, v14, 3
+; GFX11-NEXT:    v_add_nc_u16 v8, v16, 3
+; GFX11-NEXT:    v_add_nc_u16 v9, v18, 3
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GFX11-NEXT:    v_or_b32_e32 v0, v53, v0
+; GFX11-NEXT:    v_or_b32_e32 v1, v54, v1
+; GFX11-NEXT:    v_or_b32_e32 v2, v55, v2
+; GFX11-NEXT:    v_or_b32_e32 v3, v64, v3
+; GFX11-NEXT:    v_or_b32_e32 v4, v65, v4
+; GFX11-NEXT:    v_or_b32_e32 v5, v48, v5
+; GFX11-NEXT:    v_or_b32_e32 v6, v49, v6
+; GFX11-NEXT:    v_or_b32_e32 v7, v50, v7
+; GFX11-NEXT:    v_or_b32_e32 v8, v51, v8
+; GFX11-NEXT:    v_or_b32_e32 v9, v52, v9
+; GFX11-NEXT:    v_add_nc_u16 v0, 0x300, v0
+; GFX11-NEXT:    v_add_nc_u16 v1, 0x300, v1
+; GFX11-NEXT:    v_add_nc_u16 v2, 0x300, v2
+; GFX11-NEXT:    v_add_nc_u16 v3, 0x300, v3
+; GFX11-NEXT:    v_add_nc_u16 v4, 0x300, v4
+; GFX11-NEXT:    v_add_nc_u16 v5, 0x300, v5
+; GFX11-NEXT:    v_add_nc_u16 v6, 0x300, v6
+; GFX11-NEXT:    v_add_nc_u16 v7, 0x300, v7
+; GFX11-NEXT:    v_add_nc_u16 v8, 0x300, v8
+; GFX11-NEXT:    v_add_nc_u16 v9, 0x300, v9
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GFX11-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GFX11-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT:    v_or_b32_e32 v1, v2, v3
+; GFX11-NEXT:    v_or_b32_e32 v2, v4, v5
+; GFX11-NEXT:    v_or_b32_e32 v3, v6, v7
+; GFX11-NEXT:    v_or_b32_e32 v4, v8, v9
+; GFX11-NEXT:    v_add_nc_u16 v5, v20, 3
+; GFX11-NEXT:    v_add_nc_u16 v6, v22, 3
+; GFX11-NEXT:    v_add_nc_u16 v7, v24, 3
+; GFX11-NEXT:    v_add_nc_u16 v8, v26, 3
+; GFX11-NEXT:    v_add_nc_u16 v9, v28, 3
+; GFX11-NEXT:    v_add_nc_u16 v10, v30, 3
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_add_nc_u16 v12, v39, 3
+; GFX11-NEXT:    v_add_nc_u16 v14, v38, 3
+; GFX11-NEXT:    v_add_nc_u16 v16, v37, 3
+; GFX11-NEXT:    v_add_nc_u16 v18, v36, 3
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; GFX11-NEXT:    v_and_b32_e32 v12, 0xff, v12
+; GFX11-NEXT:    v_and_b32_e32 v14, 0xff, v14
+; GFX11-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; GFX11-NEXT:    v_and_b32_e32 v18, 0xff, v18
+; GFX11-NEXT:    v_or_b32_e32 v5, v21, v5
+; GFX11-NEXT:    v_or_b32_e32 v6, v23, v6
+; GFX11-NEXT:    v_or_b32_e32 v7, v25, v7
+; GFX11-NEXT:    v_or_b32_e32 v8, v27, v8
+; GFX11-NEXT:    v_or_b32_e32 v9, v29, v9
+; GFX11-NEXT:    v_or_b32_e32 v10, v11, v10
+; GFX11-NEXT:    v_or_b32_e32 v11, v13, v12
+; GFX11-NEXT:    v_or_b32_e32 v12, v15, v14
+; GFX11-NEXT:    v_or_b32_e32 v13, v17, v16
+; GFX11-NEXT:    v_or_b32_e32 v14, v19, v18
+; GFX11-NEXT:    v_add_nc_u16 v5, 0x300, v5
+; GFX11-NEXT:    v_add_nc_u16 v6, 0x300, v6
+; GFX11-NEXT:    v_add_nc_u16 v7, 0x300, v7
+; GFX11-NEXT:    v_add_nc_u16 v8, 0x300, v8
+; GFX11-NEXT:    v_add_nc_u16 v9, 0x300, v9
+; GFX11-NEXT:    v_add_nc_u16 v10, 0x300, v10
+; GFX11-NEXT:    v_add_nc_u16 v11, 0x300, v11
+; GFX11-NEXT:    v_add_nc_u16 v12, 0x300, v12
+; GFX11-NEXT:    v_add_nc_u16 v13, 0x300, v13
+; GFX11-NEXT:    v_add_nc_u16 v14, 0x300, v14
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GFX11-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GFX11-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GFX11-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-NEXT:    v_or_b32_e32 v5, v5, v6
+; GFX11-NEXT:    v_or_b32_e32 v6, v7, v8
+; GFX11-NEXT:    v_or_b32_e32 v7, v9, v10
+; GFX11-NEXT:    v_or_b32_e32 v8, v11, v12
+; GFX11-NEXT:    v_or_b32_e32 v9, v13, v14
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <40 x i8> %a, splat (i8 3)
+  %a2 = bitcast <40 x i8> %a1 to <10 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x i8> %a to <10 x float>
+  br label %end
+
+end:
+  %phi = phi <10 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x float> %phi
+}
+
+define <5 x double> @bitcast_v10f32_to_v5f64(<10 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f32_to_v5f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f32_to_v5f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f32_to_v5f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f32_to_v5f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <10 x float> %a1 to <5 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x float> %a to <5 x double>
+  br label %end
+
+end:
+  %phi = phi <5 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <5 x double> %phi
+}
+
+define <10 x float> @bitcast_v5f64_to_v10f32(<5 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v5f64_to_v10f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v5f64_to_v10f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v5f64_to_v10f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v5f64_to_v10f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <5 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <5 x double> %a1 to <10 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <5 x double> %a to <10 x float>
+  br label %end
+
+end:
+  %phi = phi <10 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x float> %phi
+}
+
+define <5 x i64> @bitcast_v10f32_to_v5i64(<10 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f32_to_v5i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB20_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f32_to_v5i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB20_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f32_to_v5i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB20_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f32_to_v5i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <10 x float> %a1 to <5 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x float> %a to <5 x i64>
+  br label %end
+
+end:
+  %phi = phi <5 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <5 x i64> %phi
+}
+
+define <10 x float> @bitcast_v5i64_to_v10f32(<5 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v5i64_to_v10f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB21_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v5i64_to_v10f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB21_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v5i64_to_v10f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB21_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v5i64_to_v10f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <5 x i64> %a, splat (i64 3)
+  %a2 = bitcast <5 x i64> %a1 to <10 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <5 x i64> %a to <10 x float>
+  br label %end
+
+end:
+  %phi = phi <10 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x float> %phi
+}
+
+define <20 x half> @bitcast_v20i16_to_v20f16(<20 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i16_to_v20f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v39, v19
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v37, v17
+; GCN-NEXT:    v_mov_b32_e32 v36, v16
+; GCN-NEXT:    v_mov_b32_e32 v35, v15
+; GCN-NEXT:    v_mov_b32_e32 v34, v14
+; GCN-NEXT:    v_mov_b32_e32 v33, v13
+; GCN-NEXT:    v_mov_b32_e32 v32, v12
+; GCN-NEXT:    v_mov_b32_e32 v31, v11
+; GCN-NEXT:    v_mov_b32_e32 v30, v10
+; GCN-NEXT:    v_mov_b32_e32 v29, v9
+; GCN-NEXT:    v_mov_b32_e32 v28, v8
+; GCN-NEXT:    v_mov_b32_e32 v27, v7
+; GCN-NEXT:    v_mov_b32_e32 v26, v6
+; GCN-NEXT:    v_mov_b32_e32 v25, v5
+; GCN-NEXT:    v_mov_b32_e32 v24, v4
+; GCN-NEXT:    v_mov_b32_e32 v23, v3
+; GCN-NEXT:    v_mov_b32_e32 v22, v2
+; GCN-NEXT:    v_mov_b32_e32 v21, v1
+; GCN-NEXT:    v_mov_b32_e32 v48, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB22_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB22_4
+; GCN-NEXT:  .LBB22_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB22_3: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  .LBB22_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20i16_to_v20f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v10, 3
+; VI-NEXT:    v_add_u16_sdwa v11, v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v12, v1, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v13, v2, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v14, v3, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v15, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v16, v5, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v17, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v18, v7, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v19, v8, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v10, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_or_b32_e32 v8, v8, v19
+; VI-NEXT:    v_or_b32_e32 v7, v7, v18
+; VI-NEXT:    v_or_b32_e32 v6, v6, v17
+; VI-NEXT:    v_or_b32_e32 v5, v5, v16
+; VI-NEXT:    v_or_b32_e32 v4, v4, v15
+; VI-NEXT:    v_or_b32_e32 v3, v3, v14
+; VI-NEXT:    v_or_b32_e32 v2, v2, v13
+; VI-NEXT:    v_or_b32_e32 v1, v1, v12
+; VI-NEXT:    v_or_b32_e32 v0, v0, v11
+; VI-NEXT:  .LBB22_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20i16_to_v20f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB22_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20i16_to_v20f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB22_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <20 x i16> %a, splat (i16 3)
+  %a2 = bitcast <20 x i16> %a1 to <20 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x i16> %a to <20 x half>
+  br label %end
+
+end:
+  %phi = phi <20 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x half> %phi
+}
+
+define <20 x i16> @bitcast_v20f16_to_v20i16(<20 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20f16_to_v20i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v9
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v17
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v17, 16
+; GCN-NEXT:  .LBB23_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20f16_to_v20i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v11, 0x200
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v0
+; VI-NEXT:    v_add_f16_sdwa v0, v0, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v1, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v3
+; VI-NEXT:    v_add_f16_sdwa v3, v3, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v4
+; VI-NEXT:    v_add_f16_sdwa v4, v4, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v5
+; VI-NEXT:    v_add_f16_sdwa v5, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v8, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_sdwa v11, v9, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v11
+; VI-NEXT:    v_or_b32_e32 v8, v19, v8
+; VI-NEXT:    v_or_b32_e32 v7, v18, v7
+; VI-NEXT:    v_or_b32_e32 v6, v17, v6
+; VI-NEXT:    v_or_b32_e32 v5, v16, v5
+; VI-NEXT:    v_or_b32_e32 v4, v15, v4
+; VI-NEXT:    v_or_b32_e32 v3, v14, v3
+; VI-NEXT:    v_or_b32_e32 v2, v13, v2
+; VI-NEXT:    v_or_b32_e32 v1, v12, v1
+; VI-NEXT:    v_or_b32_e32 v0, v10, v0
+; VI-NEXT:  .LBB23_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20f16_to_v20i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20f16_to_v20i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v10
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <20 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <20 x half> %a1 to <20 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x half> %a to <20 x i16>
+  br label %end
+
+end:
+  %phi = phi <20 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i16> %phi
+}
+
+define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i16_to_v40i8:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v12
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v20
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; kill: killed $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
-; GCN-NEXT:    v_alignbit_b32 v11, v10, v9, 24
-; GCN-NEXT:    v_alignbit_b32 v12, v10, v9, 16
-; GCN-NEXT:    v_alignbit_b32 v13, v10, v9, 8
-; GCN-NEXT:    v_alignbit_b32 v14, v8, v7, 24
-; GCN-NEXT:    v_alignbit_b32 v15, v8, v7, 16
-; GCN-NEXT:    v_alignbit_b32 v17, v8, v7, 8
-; GCN-NEXT:    v_alignbit_b32 v20, v6, v5, 24
-; GCN-NEXT:    v_alignbit_b32 v21, v6, v5, 16
-; GCN-NEXT:    v_alignbit_b32 v23, v6, v5, 8
-; GCN-NEXT:    v_alignbit_b32 v26, v4, v3, 24
-; GCN-NEXT:    v_alignbit_b32 v27, v4, v3, 16
-; GCN-NEXT:    v_alignbit_b32 v29, v4, v3, 8
-; GCN-NEXT:    v_alignbit_b32 v32, v2, v1, 24
-; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
-; GCN-NEXT:    v_alignbit_b32 v35, v2, v1, 8
-; GCN-NEXT:    v_lshrrev_b32_e32 v16, 24, v10
-; GCN-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
-; GCN-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
-; GCN-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
-; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v8
-; GCN-NEXT:    v_lshrrev_b32_e32 v25, 8, v8
-; GCN-NEXT:    v_lshrrev_b32_e32 v28, 24, v6
-; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
-; GCN-NEXT:    v_lshrrev_b32_e32 v31, 8, v6
-; GCN-NEXT:    v_lshrrev_b32_e32 v34, 24, v4
-; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
-; GCN-NEXT:    v_lshrrev_b32_e32 v37, 8, v4
-; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
-; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
-; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
-; GCN-NEXT:  .LBB12_2: ; %Flow
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v51, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v34, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v16
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v20
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_bfe_u32 v54, v4, 8, 8
+; GCN-NEXT:    v_bfe_u32 v36, v8, 8, 8
+; GCN-NEXT:    v_bfe_u32 v29, v12, 8, 8
+; GCN-NEXT:    v_bfe_u32 v24, v16, 8, 8
+; GCN-NEXT:    v_or_b32_e32 v50, v1, v56
+; GCN-NEXT:    v_or_b32_e32 v39, v2, v57
+; GCN-NEXT:    v_or_b32_e32 v32, v3, v58
+; GCN-NEXT:    v_or_b32_e32 v28, v5, v59
+; GCN-NEXT:    v_or_b32_e32 v23, v6, v60
+; GCN-NEXT:    v_or_b32_e32 v21, v7, v61
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v9, v62
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v63
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v18
+; GCN-NEXT:    v_or_b32_e32 v2, v13, v22
+; GCN-NEXT:    v_alignbit_b32 v41, v39, v50, 24
+; GCN-NEXT:    v_alignbit_b32 v43, v39, v50, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v39, v50, 8
+; GCN-NEXT:    v_alignbit_b32 v53, v28, v32, 24
+; GCN-NEXT:    v_alignbit_b32 v55, v28, v32, 16
+; GCN-NEXT:    v_alignbit_b32 v42, v28, v32, 8
+; GCN-NEXT:    v_alignbit_b32 v37, v21, v23, 24
+; GCN-NEXT:    v_alignbit_b32 v48, v21, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v21, v23, 8
+; GCN-NEXT:    v_alignbit_b32 v31, v10, v14, 24
+; GCN-NEXT:    v_alignbit_b32 v33, v10, v14, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v10, v14, 8
+; GCN-NEXT:    v_alignbit_b32 v25, v2, v6, 24
+; GCN-NEXT:    v_alignbit_b32 v27, v2, v6, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v2, v6, 8
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 8, v39
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 8, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 8, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 8, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 8, v2
+; GCN-NEXT:    v_bfe_u32 v38, v20, 8, 8
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:  .LBB24_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB12_4
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
-; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GCN-NEXT:    v_alignbit_b32 v11, v10, v9, 24
-; GCN-NEXT:    v_alignbit_b32 v12, v10, v9, 16
-; GCN-NEXT:    v_alignbit_b32 v13, v10, v9, 8
-; GCN-NEXT:    v_alignbit_b32 v14, v8, v7, 24
-; GCN-NEXT:    v_alignbit_b32 v15, v8, v7, 16
-; GCN-NEXT:    v_alignbit_b32 v17, v8, v7, 8
-; GCN-NEXT:    v_alignbit_b32 v20, v6, v5, 24
-; GCN-NEXT:    v_alignbit_b32 v21, v6, v5, 16
-; GCN-NEXT:    v_alignbit_b32 v23, v6, v5, 8
-; GCN-NEXT:    v_alignbit_b32 v26, v4, v3, 24
-; GCN-NEXT:    v_alignbit_b32 v27, v4, v3, 16
-; GCN-NEXT:    v_alignbit_b32 v29, v4, v3, 8
-; GCN-NEXT:    v_alignbit_b32 v32, v2, v1, 24
-; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
-; GCN-NEXT:    v_alignbit_b32 v35, v2, v1, 8
-; GCN-NEXT:    v_lshrrev_b32_e32 v16, 24, v10
-; GCN-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
-; GCN-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
-; GCN-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
-; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v8
-; GCN-NEXT:    v_lshrrev_b32_e32 v25, 8, v8
-; GCN-NEXT:    v_lshrrev_b32_e32 v28, 24, v6
-; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
-; GCN-NEXT:    v_lshrrev_b32_e32 v31, 8, v6
-; GCN-NEXT:    v_lshrrev_b32_e32 v34, 24, v4
-; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
-; GCN-NEXT:    v_lshrrev_b32_e32 v37, 8, v4
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v17
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v18, v2
+; GCN-NEXT:    v_or_b32_e32 v4, v22, v4
+; GCN-NEXT:    v_or_b32_e32 v11, v62, v6
+; GCN-NEXT:    v_or_b32_e32 v8, v63, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v60, v9
+; GCN-NEXT:    v_or_b32_e32 v12, v61, v10
+; GCN-NEXT:    v_or_b32_e32 v5, v58, v5
+; GCN-NEXT:    v_or_b32_e32 v7, v59, v7
+; GCN-NEXT:    v_or_b32_e32 v1, v56, v1
+; GCN-NEXT:    v_or_b32_e32 v3, v57, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 0x30000, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, s6, v3
+; GCN-NEXT:    v_alignbit_b32 v41, v39, v50, 24
+; GCN-NEXT:    v_alignbit_b32 v43, v39, v50, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v39, v50, 8
+; GCN-NEXT:    v_alignbit_b32 v53, v28, v32, 24
+; GCN-NEXT:    v_alignbit_b32 v55, v28, v32, 16
+; GCN-NEXT:    v_alignbit_b32 v42, v28, v32, 8
+; GCN-NEXT:    v_alignbit_b32 v37, v21, v23, 24
+; GCN-NEXT:    v_alignbit_b32 v48, v21, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v21, v23, 8
+; GCN-NEXT:    v_alignbit_b32 v31, v10, v14, 24
+; GCN-NEXT:    v_alignbit_b32 v33, v10, v14, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v10, v14, 8
+; GCN-NEXT:    v_alignbit_b32 v25, v2, v6, 24
+; GCN-NEXT:    v_alignbit_b32 v27, v2, v6, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v2, v6, 8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 24, v39
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v39
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 8, v39
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 24, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 8, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 24, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 8, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 24, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 8, v10
 ; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
-; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
-; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
-; GCN-NEXT:  .LBB12_4: ; %end
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 8, v2
+; GCN-NEXT:  .LBB24_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    v_and_b32_e32 v49, 0xff, v1
-; GCN-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
-; GCN-NEXT:    v_and_b32_e32 v33, 0xff, v33
-; GCN-NEXT:    v_lshlrev_b32_e32 v32, 24, v32
-; GCN-NEXT:    v_and_b32_e32 v50, 0xff, v2
-; GCN-NEXT:    v_lshlrev_b32_e32 v48, 8, v48
-; GCN-NEXT:    v_and_b32_e32 v39, 0xff, v39
-; GCN-NEXT:    v_lshlrev_b32_e32 v38, 24, v38
+; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 8, v44
+; GCN-NEXT:    v_and_b32_e32 v7, 0xff, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 24, v41
+; GCN-NEXT:    v_and_b32_e32 v9, 0xff, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 8, v47
+; GCN-NEXT:    v_and_b32_e32 v12, 0xff, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 24, v54
+; GCN-NEXT:    s_waitcnt expcnt(0)
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
-; GCN-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GCN-NEXT:    v_lshlrev_b32_e32 v29, 8, v29
-; GCN-NEXT:    v_and_b32_e32 v27, 0xff, v27
-; GCN-NEXT:    v_lshlrev_b32_e32 v26, 24, v26
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
-; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GCN-NEXT:    v_lshlrev_b32_e32 v37, 8, v37
-; GCN-NEXT:    v_and_b32_e32 v36, 0xff, v36
-; GCN-NEXT:    v_lshlrev_b32_e32 v34, 24, v34
-; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
-; GCN-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GCN-NEXT:    v_lshlrev_b32_e32 v23, 8, v23
+; GCN-NEXT:    v_and_b32_e32 v15, 0xff, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 8, v42
+; GCN-NEXT:    v_and_b32_e32 v17, 0xff, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 24, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 8, v46
+; GCN-NEXT:    v_and_b32_e32 v22, 0xff, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 24, v36
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 8, v52
+; GCN-NEXT:    v_and_b32_e32 v36, 0xff, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 24, v37
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 16, v0
 ; GCN-NEXT:    v_and_b32_e32 v21, 0xff, v21
-; GCN-NEXT:    v_lshlrev_b32_e32 v20, 24, v20
-; GCN-NEXT:    v_or_b32_e32 v35, v49, v35
-; GCN-NEXT:    v_add_i32_e32 v49, vcc, 16, v0
-; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GCN-NEXT:    v_lshlrev_b32_e32 v31, 8, v31
-; GCN-NEXT:    v_and_b32_e32 v30, 0xff, v30
-; GCN-NEXT:    v_lshlrev_b32_e32 v28, 24, v28
-; GCN-NEXT:    v_or_b32_e32 v48, v50, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 8, v45
+; GCN-NEXT:    v_and_b32_e32 v26, 0xff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 24, v29
 ; GCN-NEXT:    v_add_i32_e32 v50, vcc, 20, v0
-; GCN-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; GCN-NEXT:    v_lshlrev_b32_e32 v17, 8, v17
-; GCN-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; GCN-NEXT:    v_lshlrev_b32_e32 v14, 24, v14
-; GCN-NEXT:    v_or_b32_e32 v3, v3, v29
-; GCN-NEXT:    v_add_i32_e32 v29, vcc, 24, v0
-; GCN-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; GCN-NEXT:    v_lshlrev_b32_e32 v25, 8, v25
-; GCN-NEXT:    v_and_b32_e32 v24, 0xff, v24
-; GCN-NEXT:    v_lshlrev_b32_e32 v22, 24, v22
-; GCN-NEXT:    v_or_b32_e32 v4, v4, v37
-; GCN-NEXT:    v_add_i32_e32 v37, vcc, 28, v0
-; GCN-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; GCN-NEXT:    v_lshlrev_b32_e32 v13, 8, v13
-; GCN-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; GCN-NEXT:    v_lshlrev_b32_e32 v11, 24, v11
-; GCN-NEXT:    v_or_b32_e32 v5, v5, v23
-; GCN-NEXT:    v_add_i32_e32 v23, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
+; GCN-NEXT:    v_and_b32_e32 v33, 0xff, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 24, v31
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 24, v0
 ; GCN-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; GCN-NEXT:    v_lshlrev_b32_e32 v19, 8, v19
-; GCN-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; GCN-NEXT:    v_lshlrev_b32_e32 v16, 24, v16
-; GCN-NEXT:    v_or_b32_e32 v6, v6, v31
-; GCN-NEXT:    v_add_i32_e32 v31, vcc, 36, v0
-; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GCN-NEXT:    v_or_b32_e32 v7, v7, v17
-; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GCN-NEXT:    v_or_b32_e32 v8, v8, v25
-; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v24
-; GCN-NEXT:    v_or_b32_e32 v9, v9, v13
-; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GCN-NEXT:    v_or_b32_e32 v10, v10, v19
-; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v18
-; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v35
-; GCN-NEXT:    v_or_b32_e32 v19, v32, v33
-; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v48
-; GCN-NEXT:    v_or_b32_e32 v25, v38, v39
-; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GCN-NEXT:    v_or_b32_e32 v26, v26, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 8, v40
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v53, 0xff, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 24, v24
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 8, v30
+; GCN-NEXT:    v_and_b32_e32 v27, 0xff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 24, v25
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 8, v49
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v40, 0xff, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 24, v38
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 36, v0
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v9, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v15, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v17
+; GCN-NEXT:    v_or_b32_e32 v15, v19, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v17, v23, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v40
 ; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GCN-NEXT:    v_or_b32_e32 v27, v34, v36
-; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GCN-NEXT:    v_or_b32_e32 v20, v20, v21
-; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GCN-NEXT:    v_or_b32_e32 v21, v28, v30
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v5
 ; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GCN-NEXT:    v_or_b32_e32 v14, v14, v15
-; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GCN-NEXT:    v_or_b32_e32 v15, v22, v17
-; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GCN-NEXT:    v_or_b32_e32 v11, v11, v12
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v9
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v12
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v15
+; GCN-NEXT:    v_or_b32_e32 v13, v28, v16
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v17
+; GCN-NEXT:    v_or_b32_e32 v16, v37, v19
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v20
+; GCN-NEXT:    v_or_b32_e32 v18, v29, v21
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_or_b32_e32 v19, v31, v22
 ; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GCN-NEXT:    v_or_b32_e32 v12, v16, v13
-; GCN-NEXT:    v_or_b32_e32 v13, v18, v19
-; GCN-NEXT:    v_or_b32_e32 v16, v24, v25
-; GCN-NEXT:    v_or_b32_e32 v3, v3, v26
-; GCN-NEXT:    v_or_b32_e32 v4, v4, v27
-; GCN-NEXT:    v_or_b32_e32 v5, v5, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v24, v23
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_or_b32_e32 v21, v25, v26
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_or_b32_e32 v22, v38, v27
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v7, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v9, v11
+; GCN-NEXT:    v_or_b32_e32 v8, v12, v13
+; GCN-NEXT:    v_or_b32_e32 v9, v15, v16
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v18
+; GCN-NEXT:    v_or_b32_e32 v12, v14, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v20
 ; GCN-NEXT:    v_or_b32_e32 v6, v6, v21
-; GCN-NEXT:    v_or_b32_e32 v7, v7, v14
-; GCN-NEXT:    v_or_b32_e32 v8, v8, v15
-; GCN-NEXT:    v_or_b32_e32 v9, v9, v11
-; GCN-NEXT:    v_or_b32_e32 v10, v10, v12
-; GCN-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v16, v1, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v5, v49, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v6, v50, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v7, v29, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v8, v37, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v9, v23, s[0:3], 0 offen
-; GCN-NEXT:    buffer_store_dword v10, v31, s[0:3], 0 offen
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v22
+; GCN-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v10f32_to_v40i8:
+; VI-LABEL: bitcast_v20i16_to_v40i8:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
-; VI-NEXT:    ; implicit-def: $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    v_lshrrev_b32_e32 v17, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v1
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr40
 ; VI-NEXT:    ; implicit-def: $vgpr15
-; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr14
 ; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr50
 ; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr13
+; VI-NEXT:    ; implicit-def: $vgpr49
 ; VI-NEXT:    ; implicit-def: $vgpr35
-; VI-NEXT:    ; implicit-def: $vgpr14
-; VI-NEXT:    ; implicit-def: $vgpr34
-; VI-NEXT:    ; implicit-def: $vgpr33
-; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr55
 ; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr12
+; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr13
-; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr43
 ; VI-NEXT:    ; implicit-def: $vgpr28
-; VI-NEXT:    ; implicit-def: $vgpr27
-; VI-NEXT:    ; implicit-def: $vgpr26
-; VI-NEXT:    ; implicit-def: $vgpr25
-; VI-NEXT:    ; implicit-def: $vgpr12
-; VI-NEXT:    ; implicit-def: $vgpr24
-; VI-NEXT:    ; implicit-def: $vgpr23
-; VI-NEXT:    ; implicit-def: $vgpr22
-; VI-NEXT:    ; implicit-def: $vgpr21
-; VI-NEXT:    ; implicit-def: $vgpr20
-; VI-NEXT:    ; implicit-def: $vgpr19
-; VI-NEXT:    ; implicit-def: $vgpr18
-; VI-NEXT:    ; implicit-def: $vgpr17
 ; VI-NEXT:    ; implicit-def: $vgpr11
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr29
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:    s_cbranch_execz .LBB24_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; VI-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
 ; VI-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
 ; VI-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 24, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 8, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 8, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 24, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 8, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 8, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 24, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 8, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 8, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 8, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 8, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 24, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 8, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 8, v1
 ; VI-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
-; VI-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
-; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
-; VI-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
-; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
-; VI-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
-; VI-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
-; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
-; VI-NEXT:    v_lshrrev_b32_e32 v24, 8, v8
-; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
-; VI-NEXT:    v_lshrrev_b32_e32 v26, 8, v7
-; VI-NEXT:    v_lshrrev_b32_e32 v27, 24, v6
-; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
-; VI-NEXT:    v_lshrrev_b32_e32 v29, 8, v6
-; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
-; VI-NEXT:    v_lshrrev_b32_e32 v31, 8, v5
-; VI-NEXT:    v_lshrrev_b32_e32 v32, 24, v4
-; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
-; VI-NEXT:    v_lshrrev_b32_e32 v34, 8, v4
-; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
-; VI-NEXT:    v_lshrrev_b32_e32 v36, 8, v3
-; VI-NEXT:    v_lshrrev_b32_e32 v37, 24, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
-; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB12_2: ; %Flow
+; VI-NEXT:    v_mov_b32_e32 v34, v1
+; VI-NEXT:    v_mov_b32_e32 v32, v2
+; VI-NEXT:    v_mov_b32_e32 v38, v3
+; VI-NEXT:    v_mov_b32_e32 v37, v4
+; VI-NEXT:    v_mov_b32_e32 v50, v5
+; VI-NEXT:    v_mov_b32_e32 v49, v6
+; VI-NEXT:    v_mov_b32_e32 v55, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v43, v9
+; VI-NEXT:    v_mov_b32_e32 v42, v10
+; VI-NEXT:    ; implicit-def: $vgpr1
+; VI-NEXT:    ; implicit-def: $vgpr3
+; VI-NEXT:    ; implicit-def: $vgpr5
+; VI-NEXT:    ; implicit-def: $vgpr7
+; VI-NEXT:    ; implicit-def: $vgpr9
+; VI-NEXT:  .LBB24_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB12_4
+; VI-NEXT:    s_cbranch_execz .LBB24_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
-; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_mov_b32_e32 v11, 3
+; VI-NEXT:    v_add_u16_sdwa v17, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v20, v9, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v18, v8, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v22, v7, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v42, 3, v10
+; VI-NEXT:    v_lshlrev_b32_e32 v10, 16, v17
+; VI-NEXT:    v_add_u16_e32 v43, 3, v9
+; VI-NEXT:    v_lshlrev_b32_e32 v9, 16, v20
+; VI-NEXT:    v_add_u16_sdwa v19, v6, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v24, v5, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v53, 3, v8
+; VI-NEXT:    v_lshlrev_b32_e32 v8, 16, v18
+; VI-NEXT:    v_add_u16_e32 v55, 3, v7
+; VI-NEXT:    v_lshlrev_b32_e32 v7, 16, v22
+; VI-NEXT:    v_or_b32_e32 v10, v42, v10
+; VI-NEXT:    v_or_b32_e32 v9, v43, v9
+; VI-NEXT:    v_add_u16_sdwa v23, v2, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v26, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v21, v4, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v25, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v49, 3, v6
+; VI-NEXT:    v_lshlrev_b32_e32 v6, 16, v19
+; VI-NEXT:    v_add_u16_e32 v50, 3, v5
+; VI-NEXT:    v_lshlrev_b32_e32 v5, 16, v24
+; VI-NEXT:    v_or_b32_e32 v8, v53, v8
+; VI-NEXT:    v_or_b32_e32 v7, v55, v7
 ; VI-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
-; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_u16_e32 v37, 3, v4
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 16, v21
+; VI-NEXT:    v_add_u16_e32 v38, 3, v3
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 16, v25
+; VI-NEXT:    v_or_b32_e32 v6, v49, v6
+; VI-NEXT:    v_or_b32_e32 v5, v50, v5
 ; VI-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
-; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_u16_e32 v32, 3, v2
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 16, v23
+; VI-NEXT:    v_add_u16_e32 v34, 3, v1
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 16, v26
+; VI-NEXT:    v_or_b32_e32 v4, v37, v4
+; VI-NEXT:    v_or_b32_e32 v3, v38, v3
 ; VI-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
-; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_or_b32_e32 v2, v32, v2
+; VI-NEXT:    v_or_b32_e32 v1, v34, v1
 ; VI-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
 ; VI-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
-; VI-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
-; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
-; VI-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
-; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
-; VI-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
-; VI-NEXT:    v_lshrrev_b32_e32 v22, 24, v8
-; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
-; VI-NEXT:    v_lshrrev_b32_e32 v24, 8, v8
-; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
-; VI-NEXT:    v_lshrrev_b32_e32 v26, 8, v7
-; VI-NEXT:    v_lshrrev_b32_e32 v27, 24, v6
-; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
-; VI-NEXT:    v_lshrrev_b32_e32 v29, 8, v6
-; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
-; VI-NEXT:    v_lshrrev_b32_e32 v31, 8, v5
-; VI-NEXT:    v_lshrrev_b32_e32 v32, 24, v4
-; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
-; VI-NEXT:    v_lshrrev_b32_e32 v34, 8, v4
-; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
-; VI-NEXT:    v_lshrrev_b32_e32 v36, 8, v3
-; VI-NEXT:    v_lshrrev_b32_e32 v37, 24, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
-; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB12_4: ; %end
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 8, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 8, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 8, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 8, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 8, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 8, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 8, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 8, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 8, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 8, v1
+; VI-NEXT:    v_bfe_u32 v29, v17, 8, 8
+; VI-NEXT:    v_bfe_u32 v33, v18, 8, 8
+; VI-NEXT:    v_bfe_u32 v39, v19, 8, 8
+; VI-NEXT:    v_bfe_u32 v52, v21, 8, 8
+; VI-NEXT:    v_bfe_u32 v41, v23, 8, 8
+; VI-NEXT:  .LBB24_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
-; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
-; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v40
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v15
+; VI-NEXT:    v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v26, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v39
-; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v37
-; VI-NEXT:    v_or_b32_sdwa v2, v38, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v54
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v41
+; VI-NEXT:    v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v23, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 4, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v36
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v51
 ; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v14
-; VI-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v35, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v25, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 8, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v34
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v32
-; VI-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v33, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v48
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v52
+; VI-NEXT:    v_or_b32_sdwa v1, v37, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v21, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 12, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v31
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v36
 ; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v13
-; VI-NEXT:    v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v30, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v24, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 16, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v27
-; VI-NEXT:    v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v28, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v35
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v39
+; VI-NEXT:    v_or_b32_sdwa v1, v49, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v19, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 20, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v26
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v31
 ; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v12
-; VI-NEXT:    v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v25, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v55, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v22, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 24, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v24
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v22
-; VI-NEXT:    v_or_b32_sdwa v1, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v23, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v30
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v33
+; VI-NEXT:    v_or_b32_sdwa v1, v53, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v18, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 28, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v21
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v28
 ; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v11
-; VI-NEXT:    v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v43, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v2, v20, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 32, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v19
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v17
-; VI-NEXT:    v_or_b32_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v18, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v29
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v17, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; VI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v10f32_to_v40i8:
+; GFX9-LABEL: bitcast_v20i16_to_v40i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
@@ -3885,7 +7531,7 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr11
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX9-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -3917,23 +7563,23 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB12_2: ; %Flow
+; GFX9-NEXT:  .LBB24_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB12_4
+; GFX9-NEXT:    s_cbranch_execz .LBB24_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
-; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
-; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
-; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
-; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
 ; GFX9-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
@@ -3961,7 +7607,7 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB12_4: ; %end
+; GFX9-NEXT:  .LBB24_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -4026,7 +7672,7 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v10f32_to_v40i8:
+; GFX11-LABEL: bitcast_v20i16_to_v40i8:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v11
@@ -4063,7 +7709,7 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX11-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX11-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -4095,23 +7741,27 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB12_2: ; %Flow
+; GFX11-NEXT:  .LBB24_2: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB12_4
+; GFX11-NEXT:    s_cbranch_execz .LBB24_4
 ; GFX11-NEXT:  ; %bb.3: ; %cmp.true
-; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v9, 1.0, v9
-; GFX11-NEXT:    v_dual_add_f32 v10, 1.0, v10 :: v_dual_add_f32 v1, 1.0, v1
-; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v7, 1.0, v7
-; GFX11-NEXT:    v_dual_add_f32 v8, 1.0, v8 :: v_dual_add_f32 v3, 1.0, v3
-; GFX11-NEXT:    v_dual_add_f32 v6, 1.0, v6 :: v_dual_add_f32 v5, 1.0, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
 ; GFX11-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX11-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
-; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
-; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
 ; GFX11-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
 ; GFX11-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
 ; GFX11-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 8, v10
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v9
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 8, v9
@@ -4135,7 +7785,7 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB12_4: ; %end
+; GFX11-NEXT:  .LBB24_4: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
@@ -4237,12 +7887,12 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <10 x float> %a, splat (float 1.000000e+00)
-  %a2 = bitcast <10 x float> %a1 to <40 x i8>
+  %a1 = add <20 x i16> %a, splat (i16 3)
+  %a2 = bitcast <20 x i16> %a1 to <40 x i8>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <10 x float> %a to <40 x i8>
+  %a3 = bitcast <20 x i16> %a to <40 x i8>
   br label %end
 
 end:
@@ -4250,131 +7900,166 @@ end:
   ret <40 x i8> %phi
 }
 
-define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
-; GCN-LABEL: bitcast_v40i8_to_v10f32:
+define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40i8_to_v20i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GCN-NEXT:    v_mov_b32_e32 v35, v8
-; GCN-NEXT:    v_mov_b32_e32 v34, v6
-; GCN-NEXT:    v_mov_b32_e32 v33, v4
-; GCN-NEXT:    v_mov_b32_e32 v32, v2
-; GCN-NEXT:    v_mov_b32_e32 v31, v0
-; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
-; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:28
-; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
-; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:20
-; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; GCN-NEXT:    s_waitcnt expcnt(1)
-; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12
-; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
-; GCN-NEXT:    s_waitcnt expcnt(0)
-; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
-; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32
-; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:36
-; GCN-NEXT:    v_lshlrev_b32_e32 v38, 8, v1
-; GCN-NEXT:    v_lshlrev_b32_e32 v36, 24, v3
-; GCN-NEXT:    v_lshlrev_b32_e32 v39, 8, v5
-; GCN-NEXT:    v_lshlrev_b32_e32 v37, 24, v7
-; GCN-NEXT:    v_lshlrev_b32_e32 v48, 8, v9
-; GCN-NEXT:    v_lshlrev_b32_e32 v11, 24, v11
-; GCN-NEXT:    v_lshlrev_b32_e32 v49, 8, v13
-; GCN-NEXT:    v_lshlrev_b32_e32 v13, 24, v15
-; GCN-NEXT:    v_lshlrev_b32_e32 v50, 8, v17
-; GCN-NEXT:    v_lshlrev_b32_e32 v15, 24, v19
-; GCN-NEXT:    v_lshlrev_b32_e32 v51, 8, v21
-; GCN-NEXT:    v_lshlrev_b32_e32 v17, 24, v23
-; GCN-NEXT:    v_lshlrev_b32_e32 v52, 8, v25
-; GCN-NEXT:    v_lshlrev_b32_e32 v19, 24, v27
-; GCN-NEXT:    v_lshlrev_b32_e32 v27, 8, v29
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:32
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 8, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 24, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 24, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 8, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 24, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 24, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 8, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 24, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 24, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 8, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 24, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 8, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 8, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 8, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 8, v25
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v53
-; GCN-NEXT:    v_lshlrev_b32_e32 v21, 24, v8
-; GCN-NEXT:    v_lshlrev_b32_e32 v29, 8, v6
-; GCN-NEXT:    v_lshlrev_b32_e32 v23, 24, v4
-; GCN-NEXT:    v_lshlrev_b32_e32 v53, 8, v2
-; GCN-NEXT:    v_lshlrev_b32_e32 v25, 24, v0
-; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 24, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 8, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 24, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 24, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 8, v34
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
-; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v31
-; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v32
-; GCN-NEXT:    v_and_b32_e32 v2, 0xff, v33
-; GCN-NEXT:    v_and_b32_e32 v3, 0xff, v34
-; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v35
-; GCN-NEXT:    v_and_b32_e32 v5, 0xff, v10
-; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v12
-; GCN-NEXT:    v_and_b32_e32 v7, 0xff, v14
-; GCN-NEXT:    v_and_b32_e32 v8, 0xff, v16
-; GCN-NEXT:    v_and_b32_e32 v9, 0xff, v18
-; GCN-NEXT:    v_and_b32_e32 v10, 0xff, v20
-; GCN-NEXT:    v_and_b32_e32 v12, 0xff, v22
-; GCN-NEXT:    v_and_b32_e32 v14, 0xff, v24
-; GCN-NEXT:    v_and_b32_e32 v16, 0xff, v26
-; GCN-NEXT:    v_and_b32_e32 v18, 0xff, v28
-; GCN-NEXT:    v_and_b32_e32 v20, 0xff, v30
-; GCN-NEXT:    v_and_b32_e32 v22, 0xff, v41
-; GCN-NEXT:    v_and_b32_e32 v24, 0xff, v40
-; GCN-NEXT:    v_and_b32_e32 v26, 0xff, v55
-; GCN-NEXT:    v_and_b32_e32 v28, 0xff, v54
-; GCN-NEXT:    v_or_b32_e32 v0, v0, v38
-; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GCN-NEXT:    v_or_b32_e32 v2, v2, v39
+; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v4
+; GCN-NEXT:    v_and_b32_e32 v3, 0xff, v6
+; GCN-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v12
+; GCN-NEXT:    v_and_b32_e32 v5, 0xff, v14
+; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v10
+; GCN-NEXT:    v_and_b32_e32 v7, 0xff, v20
+; GCN-NEXT:    v_and_b32_e32 v9, 0xff, v22
+; GCN-NEXT:    v_and_b32_e32 v10, 0xff, v18
+; GCN-NEXT:    v_and_b32_e32 v11, 0xff, v28
+; GCN-NEXT:    v_and_b32_e32 v12, 0xff, v30
+; GCN-NEXT:    v_and_b32_e32 v13, 0xff, v26
+; GCN-NEXT:    v_and_b32_e32 v14, 0xff, v58
+; GCN-NEXT:    v_and_b32_e32 v15, 0xff, v57
+; GCN-NEXT:    v_and_b32_e32 v17, 0xff, v59
+; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; GCN-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; GCN-NEXT:    v_and_b32_e32 v18, 0xff, v24
+; GCN-NEXT:    v_and_b32_e32 v19, 0xff, v56
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v53
 ; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GCN-NEXT:    v_or_b32_e32 v4, v4, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v55
 ; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GCN-NEXT:    v_or_b32_e32 v6, v6, v49
-; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GCN-NEXT:    v_or_b32_e32 v8, v8, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v40
 ; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GCN-NEXT:    v_or_b32_e32 v10, v10, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v42
 ; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GCN-NEXT:    v_or_b32_e32 v14, v14, v52
-; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GCN-NEXT:    v_or_b32_e32 v18, v18, v27
-; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GCN-NEXT:    v_or_b32_e32 v22, v22, v29
-; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GCN-NEXT:    v_or_b32_e32 v26, v26, v53
-; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
-; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
-; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GCN-NEXT:    v_or_b32_e32 v3, v37, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v41
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v43
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v44
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v45
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v47
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_or_b32_e32 v3, v36, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v37, v2
 ; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
-; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GCN-NEXT:    v_or_b32_e32 v7, v13, v7
+; GCN-NEXT:    v_or_b32_e32 v5, v38, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v39, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_or_b32_e32 v9, v48, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v49, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v51, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v50, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_or_b32_e32 v20, v52, v15
+; GCN-NEXT:    v_or_b32_e32 v17, v54, v17
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GCN-NEXT:    v_or_b32_e32 v9, v15, v9
-; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GCN-NEXT:    v_or_b32_e32 v11, v17, v12
-; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v14
-; GCN-NEXT:    v_or_b32_e32 v13, v19, v16
-; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v18
-; GCN-NEXT:    v_or_b32_e32 v15, v21, v20
-; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v22
-; GCN-NEXT:    v_or_b32_e32 v17, v23, v24
-; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v26
-; GCN-NEXT:    v_or_b32_e32 v19, v25, v27
-; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
-; GCN-NEXT:    v_or_b32_e32 v1, v2, v3
-; GCN-NEXT:    v_or_b32_e32 v2, v4, v5
-; GCN-NEXT:    v_or_b32_e32 v3, v6, v7
-; GCN-NEXT:    v_or_b32_e32 v4, v8, v9
-; GCN-NEXT:    v_or_b32_e32 v5, v10, v11
-; GCN-NEXT:    v_or_b32_e32 v6, v12, v13
-; GCN-NEXT:    v_or_b32_e32 v7, v14, v15
-; GCN-NEXT:    v_or_b32_e32 v8, v16, v17
-; GCN-NEXT:    v_or_b32_e32 v9, v18, v19
-; GCN-NEXT:    ; implicit-def: $vgpr31
-; GCN-NEXT:    ; implicit-def: $vgpr32
-; GCN-NEXT:    ; implicit-def: $vgpr33
-; GCN-NEXT:    ; implicit-def: $vgpr34
-; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v19
+; GCN-NEXT:    v_or_b32_e32 v23, v1, v3
+; GCN-NEXT:    v_or_b32_e32 v27, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v31, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v33, v11, v12
+; GCN-NEXT:    v_or_b32_e32 v35, v14, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v0, v2
+; GCN-NEXT:    v_or_b32_e32 v25, v8, v6
+; GCN-NEXT:    v_or_b32_e32 v29, v15, v10
+; GCN-NEXT:    v_or_b32_e32 v32, v16, v13
+; GCN-NEXT:    v_or_b32_e32 v34, v18, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v12
+; GCN-NEXT:    v_alignbit_b32 v1, v23, v2, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v27, v6, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v31, v10, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v33, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v35, v17, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v20
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
 ; GCN-NEXT:    ; implicit-def: $vgpr10
 ; GCN-NEXT:    ; implicit-def: $vgpr12
 ; GCN-NEXT:    ; implicit-def: $vgpr14
@@ -4386,247 +8071,283 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr26
 ; GCN-NEXT:    ; implicit-def: $vgpr28
 ; GCN-NEXT:    ; implicit-def: $vgpr30
-; GCN-NEXT:    ; implicit-def: $vgpr41
-; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
 ; GCN-NEXT:    ; implicit-def: $vgpr55
-; GCN-NEXT:    ; implicit-def: $vgpr54
 ; GCN-NEXT:    ; implicit-def: $vgpr38
-; GCN-NEXT:    ; implicit-def: $vgpr36
 ; GCN-NEXT:    ; implicit-def: $vgpr39
-; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr40
 ; GCN-NEXT:    ; implicit-def: $vgpr48
-; GCN-NEXT:    ; implicit-def: $vgpr11
 ; GCN-NEXT:    ; implicit-def: $vgpr49
-; GCN-NEXT:    ; implicit-def: $vgpr13
-; GCN-NEXT:    ; implicit-def: $vgpr50
-; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr42
 ; GCN-NEXT:    ; implicit-def: $vgpr51
-; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr46
 ; GCN-NEXT:    ; implicit-def: $vgpr52
-; GCN-NEXT:    ; implicit-def: $vgpr19
-; GCN-NEXT:    ; implicit-def: $vgpr27
-; GCN-NEXT:    ; implicit-def: $vgpr21
-; GCN-NEXT:    ; implicit-def: $vgpr29
-; GCN-NEXT:    ; implicit-def: $vgpr23
-; GCN-NEXT:    ; implicit-def: $vgpr53
-; GCN-NEXT:    ; implicit-def: $vgpr25
-; GCN-NEXT:  .LBB13_2: ; %Flow
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:  .LBB25_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB13_4
+; GCN-NEXT:    s_cbranch_execz .LBB25_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v56
 ; GCN-NEXT:    s_movk_i32 s6, 0x300
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v59
 ; GCN-NEXT:    s_mov_b32 s7, 0x3000000
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v33
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v34
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v35
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v10
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
-; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
-; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
-; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v22
-; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v24
-; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v26
-; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v28
-; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v30
-; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v41
-; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v40
-; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v55
-; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v54
-; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v58
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v57
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GCN-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GCN-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GCN-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; GCN-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; GCN-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; GCN-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GCN-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; GCN-NEXT:    v_and_b32_e32 v13, 0xff, v13
+; GCN-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xff, v19
+; GCN-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; GCN-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; GCN-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; GCN-NEXT:    v_and_b32_e32 v14, 0xff, v14
-; GCN-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; GCN-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; GCN-NEXT:    v_and_b32_e32 v20, 0xff, v20
-; GCN-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; GCN-NEXT:    v_and_b32_e32 v24, 0xff, v24
-; GCN-NEXT:    v_and_b32_e32 v26, 0xff, v26
-; GCN-NEXT:    v_and_b32_e32 v28, 0xff, v28
-; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
-; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GCN-NEXT:    v_or_b32_e32 v2, v39, v2
+; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GCN-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; GCN-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GCN-NEXT:    v_or_b32_e32 v1, v47, v1
 ; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GCN-NEXT:    v_or_b32_e32 v4, v48, v4
-; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GCN-NEXT:    v_or_b32_e32 v6, v49, v6
-; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GCN-NEXT:    v_or_b32_e32 v8, v50, v8
-; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GCN-NEXT:    v_or_b32_e32 v10, v51, v10
-; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GCN-NEXT:    v_or_b32_e32 v14, v52, v14
-; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GCN-NEXT:    v_or_b32_e32 v18, v27, v18
-; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GCN-NEXT:    v_or_b32_e32 v22, v29, v22
-; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GCN-NEXT:    v_or_b32_e32 v26, v53, v26
-; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
-; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; GCN-NEXT:    v_or_b32_e32 v3, v37, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
-; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
-; GCN-NEXT:    v_or_b32_e32 v7, v13, v7
+; GCN-NEXT:    v_or_b32_e32 v5, v46, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v9, v45, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v13, v42, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v44, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v40, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v8, v43, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v12, v55, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v0, v41, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v4, v53, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
+; GCN-NEXT:    v_or_b32_e32 v3, v54, v3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_or_b32_e32 v7, v52, v7
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_or_b32_e32 v11, v50, v11
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_or_b32_e32 v15, v51, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v49, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
-; GCN-NEXT:    v_or_b32_e32 v9, v15, v9
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
-; GCN-NEXT:    v_or_b32_e32 v11, v17, v12
-; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v14
-; GCN-NEXT:    v_or_b32_e32 v13, v19, v16
-; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v18
-; GCN-NEXT:    v_or_b32_e32 v15, v21, v20
-; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v22
-; GCN-NEXT:    v_or_b32_e32 v17, v23, v24
-; GCN-NEXT:    v_add_i32_e32 v18, vcc, 0x300, v26
-; GCN-NEXT:    v_or_b32_e32 v19, v25, v27
-; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_or_b32_e32 v10, v39, v10
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_or_b32_e32 v14, v38, v14
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT:    v_or_b32_e32 v2, v37, v2
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_or_b32_e32 v6, v36, v6
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
-; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
-; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
-; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
-; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
-; GCN-NEXT:    v_or_b32_e32 v5, v11, v10
-; GCN-NEXT:    v_or_b32_e32 v6, v13, v12
-; GCN-NEXT:    v_or_b32_e32 v7, v15, v14
-; GCN-NEXT:    v_or_b32_e32 v8, v17, v16
-; GCN-NEXT:    v_or_b32_e32 v9, v19, v18
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, s7, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, s7, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, s7, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, s7, v6
-; GCN-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
-; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x3000000, v8
-; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x3000000, v9
-; GCN-NEXT:  .LBB13_4: ; %end
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v1
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v9
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v13
+; GCN-NEXT:    v_or_b32_e32 v9, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v11, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v8, v10, v8
+; GCN-NEXT:    v_or_b32_e32 v10, v14, v12
+; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
+; GCN-NEXT:    v_or_b32_e32 v2, v6, v4
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, s7, v1
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, s7, v3
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, s7, v5
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, s7, v7
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, s7, v9
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, s7, v11
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s7, v8
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, s7, v10
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s7, v0
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s7, v2
+; GCN-NEXT:    v_alignbit_b32 v1, v23, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v27, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v31, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v33, v32, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v35, v34, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v33
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v35
+; GCN-NEXT:  .LBB25_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    v_mov_b32_e32 v0, v21
+; GCN-NEXT:    v_mov_b32_e32 v2, v23
+; GCN-NEXT:    v_mov_b32_e32 v4, v25
+; GCN-NEXT:    v_mov_b32_e32 v6, v27
+; GCN-NEXT:    v_mov_b32_e32 v8, v29
+; GCN-NEXT:    v_mov_b32_e32 v10, v31
+; GCN-NEXT:    v_mov_b32_e32 v12, v32
+; GCN-NEXT:    v_mov_b32_e32 v14, v33
+; GCN-NEXT:    v_mov_b32_e32 v16, v34
+; GCN-NEXT:    v_mov_b32_e32 v18, v35
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v40i8_to_v10f32:
+; VI-LABEL: bitcast_v40i8_to_v20i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; VI-NEXT:    v_mov_b32_e32 v35, v8
-; VI-NEXT:    v_mov_b32_e32 v34, v6
-; VI-NEXT:    v_mov_b32_e32 v33, v4
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v34, v10
+; VI-NEXT:    v_mov_b32_e32 v33, v8
+; VI-NEXT:    v_mov_b32_e32 v35, v6
+; VI-NEXT:    v_mov_b32_e32 v38, v4
 ; VI-NEXT:    v_mov_b32_e32 v32, v2
-; VI-NEXT:    v_mov_b32_e32 v31, v0
+; VI-NEXT:    v_mov_b32_e32 v36, v0
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
 ; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:4
-; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v7
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v9
-; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v11
-; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v13
-; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v15
-; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v17
-; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v19
-; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v21
-; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v23
+; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:4
+; VI-NEXT:    v_mov_b32_e32 v31, v14
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v9
+; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v11
+; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v13
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v17
+; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v19
+; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
+; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v23
 ; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
-; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
-; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v29
+; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v2
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v4
+; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v4
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_lshlrev_b16_e32 v11, 8, v8
-; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:    s_cbranch_execz .LBB25_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
-; VI-NEXT:    v_or_b32_sdwa v0, v31, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v1, v32, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v36, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v32, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v1, v33, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v34, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v38, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v35, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v2, v35, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v3, v10, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v33, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v34, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v3, v12, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v4, v14, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v37, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v31, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v4, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v5, v18, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v5, v20, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v6, v22, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v6, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v7, v26, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v7, v28, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v30, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v8, v39, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v38, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v53, v45 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v37, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v36, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr36
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr35
 ; VI-NEXT:    ; implicit-def: $vgpr33
 ; VI-NEXT:    ; implicit-def: $vgpr34
-; VI-NEXT:    ; implicit-def: $vgpr35
-; VI-NEXT:    ; implicit-def: $vgpr10
-; VI-NEXT:    ; implicit-def: $vgpr12
-; VI-NEXT:    ; implicit-def: $vgpr14
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr31
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; implicit-def: $vgpr18
 ; VI-NEXT:    ; implicit-def: $vgpr20
@@ -4635,210 +8356,222 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr26
 ; VI-NEXT:    ; implicit-def: $vgpr28
 ; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr39
-; VI-NEXT:    ; implicit-def: $vgpr38
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr36
-; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr42
-; VI-NEXT:    ; implicit-def: $vgpr41
-; VI-NEXT:    ; implicit-def: $vgpr40
-; VI-NEXT:    ; implicit-def: $vgpr55
-; VI-NEXT:    ; implicit-def: $vgpr54
-; VI-NEXT:    ; implicit-def: $vgpr53
-; VI-NEXT:    ; implicit-def: $vgpr52
 ; VI-NEXT:    ; implicit-def: $vgpr51
-; VI-NEXT:    ; implicit-def: $vgpr50
-; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr39
 ; VI-NEXT:    ; implicit-def: $vgpr48
-; VI-NEXT:    ; implicit-def: $vgpr25
-; VI-NEXT:    ; implicit-def: $vgpr23
-; VI-NEXT:    ; implicit-def: $vgpr21
-; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr17
-; VI-NEXT:    ; implicit-def: $vgpr15
-; VI-NEXT:    ; implicit-def: $vgpr13
-; VI-NEXT:    ; implicit-def: $vgpr11
-; VI-NEXT:  .LBB13_2: ; %Flow
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:  .LBB25_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB13_4
+; VI-NEXT:    s_cbranch_execz .LBB25_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
-; VI-NEXT:    v_add_u16_e32 v0, 3, v31
-; VI-NEXT:    v_add_u16_e32 v1, 3, v32
+; VI-NEXT:    v_add_u16_e32 v0, 3, v55
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_mov_b32_e32 v1, 0x300
+; VI-NEXT:    v_add_u16_sdwa v9, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v54
+; VI-NEXT:    v_or_b32_sdwa v10, v46, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v53
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v8, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v51
+; VI-NEXT:    v_or_b32_sdwa v11, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v30
 ; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_mov_b32_e32 v9, 0x300
-; VI-NEXT:    v_add_u16_e32 v0, 0x300, v0
-; VI-NEXT:    v_add_u16_sdwa v1, v1, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v0, v0, v1
-; VI-NEXT:    v_add_u16_e32 v1, 3, v33
-; VI-NEXT:    v_add_u16_e32 v2, 3, v34
-; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v2, v40, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v7, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v0, 3, v28
+; VI-NEXT:    v_or_b32_sdwa v12, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v6, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v0, 3, v24
+; VI-NEXT:    v_or_b32_sdwa v13, v25, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v23, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v5, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v0, 3, v20
+; VI-NEXT:    v_or_b32_sdwa v14, v21, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v19, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v4, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v0, 3, v16
+; VI-NEXT:    v_or_b32_sdwa v15, v17, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v31
+; VI-NEXT:    v_or_b32_sdwa v0, v42, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v0, 3, v37
+; VI-NEXT:    v_or_b32_sdwa v16, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v34
+; VI-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v0, 3, v33
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v35
+; VI-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v18, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v0, 3, v38
+; VI-NEXT:    v_or_b32_sdwa v19, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v32
+; VI-NEXT:    v_or_b32_sdwa v0, v48, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v1, 3, v36
+; VI-NEXT:    v_or_b32_sdwa v1, v39, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v1, 0x300, v1
-; VI-NEXT:    v_add_u16_sdwa v2, v2, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v1, v1, v2
-; VI-NEXT:    v_add_u16_e32 v2, 3, v35
-; VI-NEXT:    v_add_u16_e32 v3, 3, v10
-; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v2, 0x300, v2
-; VI-NEXT:    v_add_u16_sdwa v3, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v2, v2, v3
-; VI-NEXT:    v_add_u16_e32 v3, 3, v12
-; VI-NEXT:    v_add_u16_e32 v4, 3, v14
-; VI-NEXT:    v_or_b32_sdwa v3, v53, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v4, v52, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v3, 0x300, v3
-; VI-NEXT:    v_add_u16_sdwa v4, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v3, v3, v4
-; VI-NEXT:    v_add_u16_e32 v4, 3, v16
-; VI-NEXT:    v_add_u16_e32 v5, 3, v18
-; VI-NEXT:    v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v4, 0x300, v4
-; VI-NEXT:    v_add_u16_sdwa v5, v5, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v4, v4, v5
-; VI-NEXT:    v_add_u16_e32 v5, 3, v20
-; VI-NEXT:    v_add_u16_e32 v6, 3, v22
-; VI-NEXT:    v_or_b32_sdwa v5, v49, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v6, v48, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v5, 0x300, v5
-; VI-NEXT:    v_add_u16_sdwa v6, v6, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v5, v5, v6
-; VI-NEXT:    v_add_u16_e32 v6, 3, v24
-; VI-NEXT:    v_add_u16_e32 v7, 3, v26
-; VI-NEXT:    v_or_b32_sdwa v6, v25, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v7, v23, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v6, 0x300, v6
-; VI-NEXT:    v_add_u16_sdwa v7, v7, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v6, v6, v7
-; VI-NEXT:    v_add_u16_e32 v7, 3, v28
-; VI-NEXT:    v_add_u16_e32 v8, 3, v30
-; VI-NEXT:    v_or_b32_sdwa v7, v21, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v8, v19, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
-; VI-NEXT:    v_add_u16_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v8, 3, v39
-; VI-NEXT:    v_add_u16_e32 v10, 3, v38
-; VI-NEXT:    v_or_b32_sdwa v8, v17, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
-; VI-NEXT:    v_add_u16_sdwa v10, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v8, v8, v10
-; VI-NEXT:    v_add_u16_e32 v10, 3, v37
-; VI-NEXT:    v_add_u16_e32 v12, 3, v36
-; VI-NEXT:    v_or_b32_sdwa v10, v13, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_u16_e32 v1, 0x300, v19
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v17
+; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
+; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
+; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
+; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
+; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
+; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
-; VI-NEXT:    v_add_u16_sdwa v9, v11, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v18
+; VI-NEXT:    v_or_b32_e32 v2, v17, v2
+; VI-NEXT:    v_or_b32_e32 v3, v16, v3
+; VI-NEXT:    v_or_b32_e32 v4, v15, v4
+; VI-NEXT:    v_or_b32_e32 v5, v14, v5
+; VI-NEXT:    v_or_b32_e32 v6, v13, v6
+; VI-NEXT:    v_or_b32_e32 v7, v12, v7
+; VI-NEXT:    v_or_b32_e32 v8, v11, v8
 ; VI-NEXT:    v_or_b32_e32 v9, v10, v9
-; VI-NEXT:  .LBB13_4: ; %end
+; VI-NEXT:  .LBB25_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v40i8_to_v10f32:
+; GFX9-LABEL: bitcast_v40i8_to_v20i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_mov_b32_e32 v35, v8
-; GFX9-NEXT:    v_mov_b32_e32 v34, v6
-; GFX9-NEXT:    v_mov_b32_e32 v33, v4
-; GFX9-NEXT:    v_mov_b32_e32 v32, v2
-; GFX9-NEXT:    v_mov_b32_e32 v31, v0
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v31, v10
+; GFX9-NEXT:    v_mov_b32_e32 v32, v8
+; GFX9-NEXT:    v_mov_b32_e32 v38, v6
+; GFX9-NEXT:    v_mov_b32_e32 v35, v4
+; GFX9-NEXT:    v_mov_b32_e32 v33, v2
+; GFX9-NEXT:    v_mov_b32_e32 v36, v0
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
 ; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v7
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v9
-; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v11
-; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v13
-; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v15
-; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v17
-; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v19
-; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v21
-; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v23
+; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    v_mov_b32_e32 v34, v12
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v11
+; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v13
+; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v15
+; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v17
+; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v19
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
+; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v23
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v29
+; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v11, 8, v8
-; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v47, 8, v10
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
-; GFX9-NEXT:    v_or_b32_sdwa v0, v31, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v1, v33, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v2, v34, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v2, v35, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v3, v10, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v3, v12, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v4, v14, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v4, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v5, v18, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v5, v20, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v6, v22, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v36, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v33, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s6
+; GFX9-NEXT:    v_or_b32_sdwa v1, v35, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v38, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v1, v2, v1, s6
+; GFX9-NEXT:    v_or_b32_sdwa v2, v32, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v31, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v2, v3, v2, s6
+; GFX9-NEXT:    v_or_b32_sdwa v3, v34, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v37, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v3, v4, v3, s6
+; GFX9-NEXT:    v_or_b32_sdwa v4, v16, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v5, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
+; GFX9-NEXT:    v_or_b32_sdwa v5, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v22, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v5, v6, v5, s6
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v7, v26, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v7, v28, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v30, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v26, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v6, v7, v6, s6
+; GFX9-NEXT:    v_or_b32_sdwa v7, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v30, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v7, v8, v7, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v8, v39, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v38, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v37, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v36, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr31
-; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    v_or_b32_sdwa v8, v55, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v53, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
+; GFX9-NEXT:    v_or_b32_sdwa v9, v42, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v54, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr36
 ; GFX9-NEXT:    ; implicit-def: $vgpr33
-; GFX9-NEXT:    ; implicit-def: $vgpr34
 ; GFX9-NEXT:    ; implicit-def: $vgpr35
-; GFX9-NEXT:    ; implicit-def: $vgpr10
-; GFX9-NEXT:    ; implicit-def: $vgpr12
-; GFX9-NEXT:    ; implicit-def: $vgpr14
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr37
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr18
 ; GFX9-NEXT:    ; implicit-def: $vgpr20
@@ -4847,208 +8580,203 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr26
 ; GFX9-NEXT:    ; implicit-def: $vgpr28
 ; GFX9-NEXT:    ; implicit-def: $vgpr30
-; GFX9-NEXT:    ; implicit-def: $vgpr39
-; GFX9-NEXT:    ; implicit-def: $vgpr38
-; GFX9-NEXT:    ; implicit-def: $vgpr37
-; GFX9-NEXT:    ; implicit-def: $vgpr36
-; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr42
-; GFX9-NEXT:    ; implicit-def: $vgpr41
-; GFX9-NEXT:    ; implicit-def: $vgpr40
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
-; GFX9-NEXT:    ; implicit-def: $vgpr54
 ; GFX9-NEXT:    ; implicit-def: $vgpr53
-; GFX9-NEXT:    ; implicit-def: $vgpr52
-; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    ; implicit-def: $vgpr49
-; GFX9-NEXT:    ; implicit-def: $vgpr48
-; GFX9-NEXT:    ; implicit-def: $vgpr25
-; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr17
 ; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
-; GFX9-NEXT:    ; implicit-def: $vgpr17
-; GFX9-NEXT:    ; implicit-def: $vgpr15
-; GFX9-NEXT:    ; implicit-def: $vgpr13
-; GFX9-NEXT:    ; implicit-def: $vgpr11
-; GFX9-NEXT:  .LBB13_2: ; %Flow
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:  .LBB25_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB13_4
+; GFX9-NEXT:    s_cbranch_execz .LBB25_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v31
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v32
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v42
+; GFX9-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v54
+; GFX9-NEXT:    v_or_b32_sdwa v0, v46, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v55
+; GFX9-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v53
+; GFX9-NEXT:    v_or_b32_sdwa v0, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v28
+; GFX9-NEXT:    v_or_b32_sdwa v0, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v30
+; GFX9-NEXT:    v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v24
+; GFX9-NEXT:    v_or_b32_sdwa v0, v25, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v26
+; GFX9-NEXT:    v_or_b32_sdwa v0, v23, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v20
+; GFX9-NEXT:    v_or_b32_sdwa v0, v21, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v5, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v22
+; GFX9-NEXT:    v_or_b32_sdwa v0, v19, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v16
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v18
+; GFX9-NEXT:    v_or_b32_sdwa v0, v17, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v34
+; GFX9-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v37
+; GFX9-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v32
+; GFX9-NEXT:    v_or_b32_sdwa v0, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v31
+; GFX9-NEXT:    v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v17, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v35
+; GFX9-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v1, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v36
+; GFX9-NEXT:    v_add_u16_e32 v19, 3, v33
+; GFX9-NEXT:    v_or_b32_sdwa v0, v48, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v19, v39, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
-; GFX9-NEXT:    v_add_u16_sdwa v1, v1, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v33
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v34
-; GFX9-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v2, v40, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v1, 0x300, v1
-; GFX9-NEXT:    v_add_u16_sdwa v2, v2, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v35
-; GFX9-NEXT:    v_add_u16_e32 v3, 3, v10
-; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v2
-; GFX9-NEXT:    v_add_u16_sdwa v3, v3, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT:    v_add_u16_e32 v3, 3, v12
-; GFX9-NEXT:    v_add_u16_e32 v4, 3, v14
-; GFX9-NEXT:    v_or_b32_sdwa v3, v53, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v4, v52, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v3
-; GFX9-NEXT:    v_add_u16_sdwa v4, v4, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v3, v3, v4
-; GFX9-NEXT:    v_add_u16_e32 v4, 3, v16
-; GFX9-NEXT:    v_add_u16_e32 v5, 3, v18
-; GFX9-NEXT:    v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v4
-; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v4, v4, v5
-; GFX9-NEXT:    v_add_u16_e32 v5, 3, v20
-; GFX9-NEXT:    v_add_u16_e32 v6, 3, v22
-; GFX9-NEXT:    v_or_b32_sdwa v5, v49, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v6, v48, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v5, 0x300, v5
-; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v5, v5, v6
-; GFX9-NEXT:    v_add_u16_e32 v6, 3, v24
-; GFX9-NEXT:    v_add_u16_e32 v7, 3, v26
-; GFX9-NEXT:    v_or_b32_sdwa v6, v25, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v7, v23, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v6
-; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v6, v6, v7
-; GFX9-NEXT:    v_add_u16_e32 v7, 3, v28
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v30
-; GFX9-NEXT:    v_or_b32_sdwa v7, v21, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v8, v19, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
-; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v39
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v38
-; GFX9-NEXT:    v_or_b32_sdwa v8, v17, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v9, v15, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
-; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v37
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v36
-; GFX9-NEXT:    v_or_b32_sdwa v9, v13, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
-; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
-; GFX9-NEXT:  .LBB13_4: ; %end
+; GFX9-NEXT:    v_add_u16_e32 v19, 0x300, v19
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v19, v0, s6
+; GFX9-NEXT:    v_perm_b32 v1, v18, v1, s6
+; GFX9-NEXT:    v_perm_b32 v2, v17, v2, s6
+; GFX9-NEXT:    v_perm_b32 v3, v16, v3, s6
+; GFX9-NEXT:    v_perm_b32 v4, v15, v4, s6
+; GFX9-NEXT:    v_perm_b32 v5, v14, v5, s6
+; GFX9-NEXT:    v_perm_b32 v6, v13, v6, s6
+; GFX9-NEXT:    v_perm_b32 v7, v12, v7, s6
+; GFX9-NEXT:    v_perm_b32 v8, v11, v8, s6
+; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
+; GFX9-NEXT:  .LBB25_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v40i8_to_v10f32:
+; GFX11-LABEL: bitcast_v40i8_to_v20i16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v35, v8 :: v_dual_mov_b32 v34, v6
-; GFX11-NEXT:    v_dual_mov_b32 v33, v4 :: v_dual_mov_b32 v32, v2
-; GFX11-NEXT:    v_mov_b32_e32 v31, v0
+; GFX11-NEXT:    v_dual_mov_b32 v33, v10 :: v_dual_mov_b32 v34, v8
+; GFX11-NEXT:    v_dual_mov_b32 v38, v6 :: v_dual_mov_b32 v35, v4
+; GFX11-NEXT:    v_dual_mov_b32 v37, v2 :: v_dual_mov_b32 v36, v0
 ; GFX11-NEXT:    s_clause 0x9
 ; GFX11-NEXT:    scratch_load_b32 v0, off, s32 offset:36
 ; GFX11-NEXT:    scratch_load_u16 v2, off, s32
 ; GFX11-NEXT:    scratch_load_u16 v4, off, s32 offset:8
 ; GFX11-NEXT:    scratch_load_u16 v6, off, s32 offset:16
 ; GFX11-NEXT:    scratch_load_u16 v8, off, s32 offset:24
-; GFX11-NEXT:    scratch_load_u16 v66, off, s32 offset:32
-; GFX11-NEXT:    scratch_load_u16 v36, off, s32 offset:28
-; GFX11-NEXT:    scratch_load_u16 v37, off, s32 offset:20
-; GFX11-NEXT:    scratch_load_u16 v38, off, s32 offset:12
-; GFX11-NEXT:    scratch_load_u16 v39, off, s32 offset:4
-; GFX11-NEXT:    v_lshlrev_b16 v53, 8, v1
-; GFX11-NEXT:    v_lshlrev_b16 v54, 8, v3
-; GFX11-NEXT:    v_lshlrev_b16 v55, 8, v5
-; GFX11-NEXT:    v_lshlrev_b16 v64, 8, v7
-; GFX11-NEXT:    v_lshlrev_b16 v65, 8, v9
-; GFX11-NEXT:    v_lshlrev_b16 v48, 8, v11
-; GFX11-NEXT:    v_lshlrev_b16 v49, 8, v13
-; GFX11-NEXT:    v_lshlrev_b16 v50, 8, v15
-; GFX11-NEXT:    v_lshlrev_b16 v51, 8, v17
-; GFX11-NEXT:    v_lshlrev_b16 v52, 8, v19
+; GFX11-NEXT:    scratch_load_u16 v10, off, s32 offset:32
+; GFX11-NEXT:    scratch_load_u16 v66, off, s32 offset:28
+; GFX11-NEXT:    scratch_load_u16 v68, off, s32 offset:20
+; GFX11-NEXT:    scratch_load_u16 v65, off, s32 offset:12
+; GFX11-NEXT:    scratch_load_u16 v67, off, s32 offset:4
+; GFX11-NEXT:    v_dual_mov_b32 v31, v14 :: v_dual_mov_b32 v32, v12
+; GFX11-NEXT:    v_lshlrev_b16 v48, 8, v1
+; GFX11-NEXT:    v_lshlrev_b16 v49, 8, v3
+; GFX11-NEXT:    v_lshlrev_b16 v51, 8, v5
+; GFX11-NEXT:    v_lshlrev_b16 v50, 8, v7
+; GFX11-NEXT:    v_lshlrev_b16 v55, 8, v9
+; GFX11-NEXT:    v_lshlrev_b16 v39, 8, v11
+; GFX11-NEXT:    v_lshlrev_b16 v53, 8, v13
+; GFX11-NEXT:    v_lshlrev_b16 v52, 8, v15
+; GFX11-NEXT:    v_lshlrev_b16 v54, 8, v17
+; GFX11-NEXT:    v_lshlrev_b16 v17, 8, v19
 ; GFX11-NEXT:    v_lshlrev_b16 v21, 8, v21
-; GFX11-NEXT:    v_lshlrev_b16 v23, 8, v23
-; GFX11-NEXT:    v_lshlrev_b16 v25, 8, v25
-; GFX11-NEXT:    v_lshlrev_b16 v27, 8, v27
-; GFX11-NEXT:    v_lshlrev_b16 v29, 8, v29
+; GFX11-NEXT:    v_lshlrev_b16 v19, 8, v23
+; GFX11-NEXT:    v_lshlrev_b16 v64, 8, v25
+; GFX11-NEXT:    v_lshlrev_b16 v25, 8, v27
+; GFX11-NEXT:    v_lshlrev_b16 v69, 8, v29
 ; GFX11-NEXT:    s_waitcnt vmcnt(9)
 ; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-NEXT:    s_waitcnt vmcnt(8)
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v2
+; GFX11-NEXT:    v_lshlrev_b16 v23, 8, v2
 ; GFX11-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-NEXT:    v_lshlrev_b16 v13, 8, v4
+; GFX11-NEXT:    v_lshlrev_b16 v27, 8, v4
 ; GFX11-NEXT:    s_waitcnt vmcnt(6)
-; GFX11-NEXT:    v_lshlrev_b16 v15, 8, v6
+; GFX11-NEXT:    v_lshlrev_b16 v29, 8, v6
 ; GFX11-NEXT:    s_waitcnt vmcnt(5)
-; GFX11-NEXT:    v_lshlrev_b16 v17, 8, v8
+; GFX11-NEXT:    v_lshlrev_b16 v70, 8, v8
 ; GFX11-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-NEXT:    v_lshlrev_b16 v19, 8, v66
-; GFX11-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-NEXT:    v_lshlrev_b16 v71, 8, v10
+; GFX11-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB13_3
+; GFX11-NEXT:    s_cbranch_execnz .LBB25_3
 ; GFX11-NEXT:  ; %bb.1: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB13_4
-; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_cbranch_execnz .LBB25_4
+; GFX11-NEXT:  .LBB25_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-; GFX11-NEXT:  .LBB13_3: ; %cmp.false
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v31
-; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v32
-; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v33
-; GFX11-NEXT:    v_and_b32_e32 v3, 0xff, v34
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v35
-; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v10
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v12
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v14
+; GFX11-NEXT:  .LBB25_3: ; %cmp.false
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v36
+; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v37
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v35
+; GFX11-NEXT:    v_and_b32_e32 v3, 0xff, v38
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v34
+; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v33
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v32
+; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v31
 ; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v16
 ; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v18
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v53
-; GFX11-NEXT:    v_or_b32_e32 v1, v1, v54
-; GFX11-NEXT:    v_or_b32_e32 v2, v2, v55
-; GFX11-NEXT:    v_or_b32_e32 v3, v3, v64
-; GFX11-NEXT:    v_or_b32_e32 v4, v4, v65
-; GFX11-NEXT:    v_or_b32_e32 v5, v5, v48
-; GFX11-NEXT:    v_or_b32_e32 v6, v6, v49
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v50
-; GFX11-NEXT:    v_or_b32_e32 v8, v8, v51
-; GFX11-NEXT:    v_or_b32_e32 v9, v9, v52
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT:    v_or_b32_e32 v1, v2, v3
-; GFX11-NEXT:    v_or_b32_e32 v2, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v3, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v4, v8, v9
+; GFX11-NEXT:    v_or_b32_e32 v0, v0, v48
+; GFX11-NEXT:    v_or_b32_e32 v1, v1, v49
+; GFX11-NEXT:    v_or_b32_e32 v2, v2, v51
+; GFX11-NEXT:    v_or_b32_e32 v3, v3, v50
+; GFX11-NEXT:    v_or_b32_e32 v4, v4, v55
+; GFX11-NEXT:    v_or_b32_e32 v5, v5, v39
+; GFX11-NEXT:    v_or_b32_e32 v6, v6, v53
+; GFX11-NEXT:    v_or_b32_e32 v7, v7, v52
+; GFX11-NEXT:    v_or_b32_e32 v8, v8, v54
+; GFX11-NEXT:    v_or_b32_e32 v9, v9, v17
+; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v3, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v5, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v7, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v9, v8, 0x5040100
 ; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v20
 ; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v22
 ; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v24
@@ -5056,43 +8784,33 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v28
 ; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v30
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v12, 0xff, v39
-; GFX11-NEXT:    v_and_b32_e32 v14, 0xff, v38
-; GFX11-NEXT:    v_and_b32_e32 v16, 0xff, v37
-; GFX11-NEXT:    v_and_b32_e32 v18, 0xff, v36
+; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v67
+; GFX11-NEXT:    v_and_b32_e32 v12, 0xff, v65
+; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v68
+; GFX11-NEXT:    v_and_b32_e32 v14, 0xff, v66
 ; GFX11-NEXT:    v_or_b32_e32 v5, v5, v21
-; GFX11-NEXT:    v_or_b32_e32 v6, v6, v23
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v25
-; GFX11-NEXT:    v_or_b32_e32 v8, v8, v27
-; GFX11-NEXT:    v_or_b32_e32 v9, v9, v29
-; GFX11-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX11-NEXT:    v_or_b32_e32 v11, v12, v13
-; GFX11-NEXT:    v_or_b32_e32 v12, v14, v15
-; GFX11-NEXT:    v_or_b32_e32 v13, v16, v17
-; GFX11-NEXT:    v_or_b32_e32 v14, v18, v19
-; GFX11-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-NEXT:    v_or_b32_e32 v5, v5, v6
-; GFX11-NEXT:    v_or_b32_e32 v6, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v7, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v8, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v9, v13, v14
-; GFX11-NEXT:    ; implicit-def: $vgpr31
-; GFX11-NEXT:    ; implicit-def: $vgpr32
-; GFX11-NEXT:    ; implicit-def: $vgpr33
-; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    v_or_b32_e32 v6, v6, v19
+; GFX11-NEXT:    v_or_b32_e32 v7, v7, v64
+; GFX11-NEXT:    v_or_b32_e32 v8, v8, v25
+; GFX11-NEXT:    v_or_b32_e32 v9, v9, v69
+; GFX11-NEXT:    v_or_b32_e32 v10, v10, v23
+; GFX11-NEXT:    v_or_b32_e32 v11, v11, v27
+; GFX11-NEXT:    v_or_b32_e32 v12, v12, v29
+; GFX11-NEXT:    v_or_b32_e32 v13, v13, v70
+; GFX11-NEXT:    v_or_b32_e32 v14, v14, v71
+; GFX11-NEXT:    v_perm_b32 v5, v6, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v8, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v10, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v12, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v14, v13, 0x5040100
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr37
 ; GFX11-NEXT:    ; implicit-def: $vgpr35
-; GFX11-NEXT:    ; implicit-def: $vgpr10
-; GFX11-NEXT:    ; implicit-def: $vgpr12
-; GFX11-NEXT:    ; implicit-def: $vgpr14
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
 ; GFX11-NEXT:    ; implicit-def: $vgpr16
 ; GFX11-NEXT:    ; implicit-def: $vgpr18
 ; GFX11-NEXT:    ; implicit-def: $vgpr20
@@ -5101,144 +8819,128 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    ; implicit-def: $vgpr26
 ; GFX11-NEXT:    ; implicit-def: $vgpr28
 ; GFX11-NEXT:    ; implicit-def: $vgpr30
-; GFX11-NEXT:    ; implicit-def: $vgpr39
-; GFX11-NEXT:    ; implicit-def: $vgpr38
-; GFX11-NEXT:    ; implicit-def: $vgpr37
-; GFX11-NEXT:    ; implicit-def: $vgpr36
-; GFX11-NEXT:    ; implicit-def: $vgpr53
-; GFX11-NEXT:    ; implicit-def: $vgpr54
-; GFX11-NEXT:    ; implicit-def: $vgpr55
-; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr67
 ; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr66
 ; GFX11-NEXT:    ; implicit-def: $vgpr48
 ; GFX11-NEXT:    ; implicit-def: $vgpr49
-; GFX11-NEXT:    ; implicit-def: $vgpr50
 ; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr53
 ; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr17
 ; GFX11-NEXT:    ; implicit-def: $vgpr21
-; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr64
 ; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr23
 ; GFX11-NEXT:    ; implicit-def: $vgpr27
 ; GFX11-NEXT:    ; implicit-def: $vgpr29
-; GFX11-NEXT:    ; implicit-def: $vgpr11
-; GFX11-NEXT:    ; implicit-def: $vgpr13
-; GFX11-NEXT:    ; implicit-def: $vgpr15
-; GFX11-NEXT:    ; implicit-def: $vgpr17
-; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr71
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB13_2
-; GFX11-NEXT:  .LBB13_4: ; %cmp.true
-; GFX11-NEXT:    v_add_nc_u16 v0, v31, 3
-; GFX11-NEXT:    v_add_nc_u16 v1, v32, 3
-; GFX11-NEXT:    v_add_nc_u16 v2, v33, 3
-; GFX11-NEXT:    v_add_nc_u16 v3, v34, 3
-; GFX11-NEXT:    v_add_nc_u16 v4, v35, 3
-; GFX11-NEXT:    v_add_nc_u16 v5, v10, 3
-; GFX11-NEXT:    v_add_nc_u16 v6, v12, 3
-; GFX11-NEXT:    v_add_nc_u16 v7, v14, 3
-; GFX11-NEXT:    v_add_nc_u16 v8, v16, 3
-; GFX11-NEXT:    v_add_nc_u16 v9, v18, 3
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  .LBB25_4: ; %cmp.true
+; GFX11-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-NEXT:    v_add_nc_u16 v0, v68, 3
+; GFX11-NEXT:    v_add_nc_u16 v1, v66, 3
+; GFX11-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NEXT:    v_add_nc_u16 v3, v65, 3
+; GFX11-NEXT:    v_add_nc_u16 v4, v28, 3
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_add_nc_u16 v2, v67, 3
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT:    v_or_b32_e32 v0, v70, v0
+; GFX11-NEXT:    v_or_b32_e32 v1, v71, v1
+; GFX11-NEXT:    v_or_b32_e32 v3, v29, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_or_b32_e32 v2, v27, v2
+; GFX11-NEXT:    v_add_nc_u16 v9, 0x300, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_add_nc_u16 v10, 0x300, v1
+; GFX11-NEXT:    v_or_b32_e32 v0, v69, v4
+; GFX11-NEXT:    v_add_nc_u16 v1, v30, 3
+; GFX11-NEXT:    v_add_nc_u16 v11, 0x300, v3
+; GFX11-NEXT:    v_add_nc_u16 v3, v20, 3
+; GFX11-NEXT:    v_add_nc_u16 v8, 0x300, v2
+; GFX11-NEXT:    v_add_nc_u16 v7, 0x300, v0
+; GFX11-NEXT:    v_add_nc_u16 v0, v26, 3
+; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-NEXT:    v_add_nc_u16 v2, v24, 3
 ; GFX11-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-NEXT:    v_add_nc_u16 v4, v22, 3
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX11-NEXT:    v_or_b32_e32 v1, v23, v1
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT:    v_perm_b32 v8, v11, v8, 0x5040100
 ; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; GFX11-NEXT:    v_or_b32_e32 v0, v53, v0
-; GFX11-NEXT:    v_or_b32_e32 v1, v54, v1
+; GFX11-NEXT:    v_or_b32_e32 v0, v25, v0
+; GFX11-NEXT:    v_add_nc_u16 v12, 0x300, v1
+; GFX11-NEXT:    v_or_b32_e32 v1, v21, v3
+; GFX11-NEXT:    v_or_b32_e32 v2, v64, v2
+; GFX11-NEXT:    v_add_nc_u16 v3, v18, 3
+; GFX11-NEXT:    v_add_nc_u16 v13, 0x300, v0
+; GFX11-NEXT:    v_add_nc_u16 v0, v16, 3
+; GFX11-NEXT:    v_add_nc_u16 v5, 0x300, v1
+; GFX11-NEXT:    v_add_nc_u16 v1, v32, 3
+; GFX11-NEXT:    v_add_nc_u16 v6, 0x300, v2
+; GFX11-NEXT:    v_or_b32_e32 v2, v19, v4
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX11-NEXT:    v_add_nc_u16 v4, v31, 3
+; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX11-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-NEXT:    v_add_nc_u16 v14, 0x300, v2
+; GFX11-NEXT:    v_or_b32_e32 v0, v54, v0
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; GFX11-NEXT:    v_or_b32_e32 v1, v53, v1
+; GFX11-NEXT:    v_add_nc_u16 v2, v34, 3
+; GFX11-NEXT:    v_add_nc_u16 v16, v36, 3
+; GFX11-NEXT:    v_add_nc_u16 v15, 0x300, v0
+; GFX11-NEXT:    v_or_b32_e32 v0, v17, v3
+; GFX11-NEXT:    v_add_nc_u16 v3, 0x300, v1
+; GFX11-NEXT:    v_or_b32_e32 v1, v52, v4
+; GFX11-NEXT:    v_add_nc_u16 v4, v35, 3
+; GFX11-NEXT:    v_add_nc_u16 v17, v37, 3
+; GFX11-NEXT:    v_add_nc_u16 v18, v38, 3
+; GFX11-NEXT:    v_add_nc_u16 v19, v33, 3
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; GFX11-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; GFX11-NEXT:    v_and_b32_e32 v17, 0xff, v17
+; GFX11-NEXT:    v_and_b32_e32 v18, 0xff, v18
+; GFX11-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; GFX11-NEXT:    v_or_b32_e32 v2, v55, v2
-; GFX11-NEXT:    v_or_b32_e32 v3, v64, v3
-; GFX11-NEXT:    v_or_b32_e32 v4, v65, v4
-; GFX11-NEXT:    v_or_b32_e32 v5, v48, v5
-; GFX11-NEXT:    v_or_b32_e32 v6, v49, v6
-; GFX11-NEXT:    v_or_b32_e32 v7, v50, v7
-; GFX11-NEXT:    v_or_b32_e32 v8, v51, v8
-; GFX11-NEXT:    v_or_b32_e32 v9, v52, v9
-; GFX11-NEXT:    v_add_nc_u16 v0, 0x300, v0
-; GFX11-NEXT:    v_add_nc_u16 v1, 0x300, v1
+; GFX11-NEXT:    v_or_b32_e32 v4, v51, v4
+; GFX11-NEXT:    v_or_b32_e32 v16, v48, v16
+; GFX11-NEXT:    v_or_b32_e32 v17, v49, v17
+; GFX11-NEXT:    v_or_b32_e32 v18, v50, v18
+; GFX11-NEXT:    v_or_b32_e32 v19, v39, v19
 ; GFX11-NEXT:    v_add_nc_u16 v2, 0x300, v2
-; GFX11-NEXT:    v_add_nc_u16 v3, 0x300, v3
 ; GFX11-NEXT:    v_add_nc_u16 v4, 0x300, v4
-; GFX11-NEXT:    v_add_nc_u16 v5, 0x300, v5
-; GFX11-NEXT:    v_add_nc_u16 v6, 0x300, v6
-; GFX11-NEXT:    v_add_nc_u16 v7, 0x300, v7
-; GFX11-NEXT:    v_add_nc_u16 v8, 0x300, v8
-; GFX11-NEXT:    v_add_nc_u16 v9, 0x300, v9
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT:    v_or_b32_e32 v1, v2, v3
-; GFX11-NEXT:    v_or_b32_e32 v2, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v3, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v4, v8, v9
-; GFX11-NEXT:    v_add_nc_u16 v5, v20, 3
-; GFX11-NEXT:    v_add_nc_u16 v6, v22, 3
-; GFX11-NEXT:    v_add_nc_u16 v7, v24, 3
-; GFX11-NEXT:    v_add_nc_u16 v8, v26, 3
-; GFX11-NEXT:    v_add_nc_u16 v9, v28, 3
-; GFX11-NEXT:    v_add_nc_u16 v10, v30, 3
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_add_nc_u16 v12, v39, 3
-; GFX11-NEXT:    v_add_nc_u16 v14, v38, 3
-; GFX11-NEXT:    v_add_nc_u16 v16, v37, 3
-; GFX11-NEXT:    v_add_nc_u16 v18, v36, 3
-; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; GFX11-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; GFX11-NEXT:    v_and_b32_e32 v14, 0xff, v14
-; GFX11-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; GFX11-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; GFX11-NEXT:    v_or_b32_e32 v5, v21, v5
-; GFX11-NEXT:    v_or_b32_e32 v6, v23, v6
-; GFX11-NEXT:    v_or_b32_e32 v7, v25, v7
-; GFX11-NEXT:    v_or_b32_e32 v8, v27, v8
-; GFX11-NEXT:    v_or_b32_e32 v9, v29, v9
-; GFX11-NEXT:    v_or_b32_e32 v10, v11, v10
-; GFX11-NEXT:    v_or_b32_e32 v11, v13, v12
-; GFX11-NEXT:    v_or_b32_e32 v12, v15, v14
-; GFX11-NEXT:    v_or_b32_e32 v13, v17, v16
-; GFX11-NEXT:    v_or_b32_e32 v14, v19, v18
-; GFX11-NEXT:    v_add_nc_u16 v5, 0x300, v5
-; GFX11-NEXT:    v_add_nc_u16 v6, 0x300, v6
-; GFX11-NEXT:    v_add_nc_u16 v7, 0x300, v7
-; GFX11-NEXT:    v_add_nc_u16 v8, 0x300, v8
-; GFX11-NEXT:    v_add_nc_u16 v9, 0x300, v9
-; GFX11-NEXT:    v_add_nc_u16 v10, 0x300, v10
-; GFX11-NEXT:    v_add_nc_u16 v11, 0x300, v11
-; GFX11-NEXT:    v_add_nc_u16 v12, 0x300, v12
-; GFX11-NEXT:    v_add_nc_u16 v13, 0x300, v13
-; GFX11-NEXT:    v_add_nc_u16 v14, 0x300, v14
-; GFX11-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-NEXT:    v_or_b32_e32 v5, v5, v6
-; GFX11-NEXT:    v_or_b32_e32 v6, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v7, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v8, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v9, v13, v14
+; GFX11-NEXT:    v_add_nc_u16 v16, 0x300, v16
+; GFX11-NEXT:    v_add_nc_u16 v17, 0x300, v17
+; GFX11-NEXT:    v_add_nc_u16 v18, 0x300, v18
+; GFX11-NEXT:    v_add_nc_u16 v19, 0x300, v19
+; GFX11-NEXT:    v_add_nc_u16 v20, 0x300, v1
+; GFX11-NEXT:    v_add_nc_u16 v21, 0x300, v0
+; GFX11-NEXT:    v_perm_b32 v0, v17, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v18, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v19, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v14, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v13, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v12, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v10, v9, 0x5040100
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -5246,89 +8948,207 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 
 cmp.true:
   %a1 = add <40 x i8> %a, splat (i8 3)
-  %a2 = bitcast <40 x i8> %a1 to <10 x float>
+  %a2 = bitcast <40 x i8> %a1 to <20 x i16>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <40 x i8> %a to <10 x float>
+  %a3 = bitcast <40 x i8> %a to <20 x i16>
   br label %end
 
 end:
-  %phi = phi <10 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <10 x float> %phi
+  %phi = phi <20 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i16> %phi
 }
 
-define <5 x double> @bitcast_v10f32_to_v5f64(<10 x float> %a, i32 %b) {
-; GCN-LABEL: bitcast_v10f32_to_v5f64:
+define <5 x double> @bitcast_v20i16_to_v5f64(<20 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i16_to_v5f64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_mov_b32_e32 v28, v14
+; GCN-NEXT:    v_mov_b32_e32 v27, v12
+; GCN-NEXT:    v_mov_b32_e32 v26, v10
+; GCN-NEXT:    v_mov_b32_e32 v21, v8
+; GCN-NEXT:    v_mov_b32_e32 v22, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v4
+; GCN-NEXT:    v_mov_b32_e32 v24, v2
+; GCN-NEXT:    v_mov_b32_e32 v25, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB26_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB14_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GCN-NEXT:  .LBB14_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB26_4
+; GCN-NEXT:  .LBB26_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB26_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v24
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v35
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v20
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v29
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v30
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v31
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v32
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v33
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  .LBB26_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v25
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v35, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v20, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v29, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v30, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v31, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v32, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v33, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x30000, v9
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v10f32_to_v5f64:
+; VI-LABEL: bitcast_v20i16_to_v5f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:    s_cbranch_execz .LBB26_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; VI-NEXT:  .LBB14_2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v11, 3
+; VI-NEXT:    v_add_u16_e32 v10, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_u16_e32 v10, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v10, v8
+; VI-NEXT:    v_add_u16_e32 v10, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v10, v7
+; VI-NEXT:    v_add_u16_e32 v10, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v10, v6
+; VI-NEXT:    v_add_u16_e32 v10, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v10, v5
+; VI-NEXT:    v_add_u16_e32 v10, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v10, v4
+; VI-NEXT:    v_add_u16_e32 v10, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v10, v3
+; VI-NEXT:    v_add_u16_e32 v10, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v10, v2
+; VI-NEXT:    v_add_u16_e32 v10, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v10, v1
+; VI-NEXT:    v_add_u16_e32 v10, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v10, v0
+; VI-NEXT:  .LBB26_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v10f32_to_v5f64:
+; GFX9-LABEL: bitcast_v20i16_to_v5f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GFX9-NEXT:  .LBB14_2: ; %end
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB26_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v10f32_to_v5f64:
+; GFX11-LABEL: bitcast_v20i16_to_v5f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -5336,25 +9156,31 @@ define <5 x double> @bitcast_v10f32_to_v5f64(<10 x float> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
-; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
-; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
-; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
-; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
-; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB26_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <10 x float> %a, splat (float 1.000000e+00)
-  %a2 = bitcast <10 x float> %a1 to <5 x double>
+  %a1 = add <20 x i16> %a, splat (i16 3)
+  %a2 = bitcast <20 x i16> %a1 to <5 x double>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <10 x float> %a to <5 x double>
+  %a3 = bitcast <20 x i16> %a to <5 x double>
   br label %end
 
 end:
@@ -5362,62 +9188,113 @@ end:
   ret <5 x double> %phi
 }
 
-define <10 x float> @bitcast_v5f64_to_v10f32(<5 x double> %a, i32 %b) {
-; GCN-LABEL: bitcast_v5f64_to_v10f32:
+define <20 x i16> @bitcast_v5f64_to_v20i16(<5 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v5f64_to_v20i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v28, v9
+; GCN-NEXT:    v_mov_b32_e32 v27, v8
+; GCN-NEXT:    v_mov_b32_e32 v26, v7
+; GCN-NEXT:    v_mov_b32_e32 v25, v6
+; GCN-NEXT:    v_mov_b32_e32 v24, v5
+; GCN-NEXT:    v_mov_b32_e32 v23, v4
+; GCN-NEXT:    v_mov_b32_e32 v22, v3
+; GCN-NEXT:    v_mov_b32_e32 v21, v2
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v17, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v1, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT:  .LBB27_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB15_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GCN-NEXT:  .LBB15_2: ; %end
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_add_f64 v[25:26], v[25:26], 1.0
+; GCN-NEXT:    v_add_f64 v[27:28], v[27:28], 1.0
+; GCN-NEXT:    v_alignbit_b32 v17, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v1, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT:  .LBB27_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_mov_b32_e32 v4, v21
+; GCN-NEXT:    v_mov_b32_e32 v6, v22
+; GCN-NEXT:    v_mov_b32_e32 v8, v23
+; GCN-NEXT:    v_mov_b32_e32 v10, v24
+; GCN-NEXT:    v_mov_b32_e32 v12, v25
+; GCN-NEXT:    v_mov_b32_e32 v14, v26
+; GCN-NEXT:    v_mov_b32_e32 v16, v27
+; GCN-NEXT:    v_mov_b32_e32 v18, v28
+; GCN-NEXT:    v_mov_b32_e32 v1, v20
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v5f64_to_v10f32:
+; VI-LABEL: bitcast_v5f64_to_v20i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:    s_cbranch_execz .LBB27_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT:  .LBB15_2: ; %end
+; VI-NEXT:  .LBB27_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v5f64_to_v10f32:
+; GFX9-LABEL: bitcast_v5f64_to_v20i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:  .LBB15_2: ; %end
+; GFX9-NEXT:  .LBB27_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v5f64_to_v10f32:
+; GFX11-LABEL: bitcast_v5f64_to_v20i16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -5425,14 +9302,14 @@ define <10 x float> @bitcast_v5f64_to_v10f32(<5 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:  .LBB27_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -5440,89 +9317,207 @@ define <10 x float> @bitcast_v5f64_to_v10f32(<5 x double> %a, i32 %b) {
 
 cmp.true:
   %a1 = fadd <5 x double> %a, splat (double 1.000000e+00)
-  %a2 = bitcast <5 x double> %a1 to <10 x float>
+  %a2 = bitcast <5 x double> %a1 to <20 x i16>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <5 x double> %a to <10 x float>
+  %a3 = bitcast <5 x double> %a to <20 x i16>
   br label %end
 
 end:
-  %phi = phi <10 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <10 x float> %phi
+  %phi = phi <20 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i16> %phi
 }
 
-define <5 x i64> @bitcast_v10f32_to_v5i64(<10 x float> %a, i32 %b) {
-; GCN-LABEL: bitcast_v10f32_to_v5i64:
+define <5 x i64> @bitcast_v20i16_to_v5i64(<20 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i16_to_v5i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_mov_b32_e32 v28, v14
+; GCN-NEXT:    v_mov_b32_e32 v27, v12
+; GCN-NEXT:    v_mov_b32_e32 v26, v10
+; GCN-NEXT:    v_mov_b32_e32 v21, v8
+; GCN-NEXT:    v_mov_b32_e32 v22, v6
+; GCN-NEXT:    v_mov_b32_e32 v23, v4
+; GCN-NEXT:    v_mov_b32_e32 v24, v2
+; GCN-NEXT:    v_mov_b32_e32 v25, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB28_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB16_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GCN-NEXT:  .LBB16_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB28_4
+; GCN-NEXT:  .LBB28_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB28_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v24
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v35
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v20
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v29
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v30
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v31
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v32
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v33
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  .LBB28_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v25
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v35, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v20, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v29, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v30, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v31, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v32, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v33, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x30000, v9
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v10f32_to_v5i64:
+; VI-LABEL: bitcast_v20i16_to_v5i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:    s_cbranch_execz .LBB28_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; VI-NEXT:  .LBB16_2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v11, 3
+; VI-NEXT:    v_add_u16_e32 v10, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_u16_e32 v10, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v10, v8
+; VI-NEXT:    v_add_u16_e32 v10, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v10, v7
+; VI-NEXT:    v_add_u16_e32 v10, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v10, v6
+; VI-NEXT:    v_add_u16_e32 v10, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v10, v5
+; VI-NEXT:    v_add_u16_e32 v10, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v10, v4
+; VI-NEXT:    v_add_u16_e32 v10, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v10, v3
+; VI-NEXT:    v_add_u16_e32 v10, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v10, v2
+; VI-NEXT:    v_add_u16_e32 v10, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v10, v1
+; VI-NEXT:    v_add_u16_e32 v10, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v10, v0
+; VI-NEXT:  .LBB28_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v10f32_to_v5i64:
+; GFX9-LABEL: bitcast_v20i16_to_v5i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GFX9-NEXT:  .LBB16_2: ; %end
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB28_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v10f32_to_v5i64:
+; GFX11-LABEL: bitcast_v20i16_to_v5i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -5530,25 +9525,31 @@ define <5 x i64> @bitcast_v10f32_to_v5i64(<10 x float> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
-; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
-; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
-; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
-; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
-; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB28_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <10 x float> %a, splat (float 1.000000e+00)
-  %a2 = bitcast <10 x float> %a1 to <5 x i64>
+  %a1 = add <20 x i16> %a, splat (i16 3)
+  %a2 = bitcast <20 x i16> %a1 to <5 x i64>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <10 x float> %a to <5 x i64>
+  %a3 = bitcast <20 x i16> %a to <5 x i64>
   br label %end
 
 end:
@@ -5556,38 +9557,81 @@ end:
   ret <5 x i64> %phi
 }
 
-define <10 x float> @bitcast_v5i64_to_v10f32(<5 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v5i64_to_v10f32:
+define <20 x i16> @bitcast_v5i64_to_v20i16(<5 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v5i64_to_v20i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v20, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v20, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB29_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB17_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    s_cbranch_execz .LBB29_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:  .LBB17_2: ; %end
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v20, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB29_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v10, v20
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v5i64_to_v10f32:
+; VI-LABEL: bitcast_v5i64_to_v20i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:    s_cbranch_execz .LBB29_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
@@ -5599,18 +9643,18 @@ define <10 x float> @bitcast_v5i64_to_v10f32(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:  .LBB17_2: ; %end
+; VI-NEXT:  .LBB29_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v5i64_to_v10f32:
+; GFX9-LABEL: bitcast_v5i64_to_v20i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
 ; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
@@ -5622,11 +9666,11 @@ define <10 x float> @bitcast_v5i64_to_v10f32(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:  .LBB17_2: ; %end
+; GFX9-NEXT:  .LBB29_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v5i64_to_v10f32:
+; GFX11-LABEL: bitcast_v5i64_to_v20i16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -5634,7 +9678,7 @@ define <10 x float> @bitcast_v5i64_to_v10f32(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -5649,7 +9693,7 @@ define <10 x float> @bitcast_v5i64_to_v10f32(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:  .LBB29_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -5657,16 +9701,16 @@ define <10 x float> @bitcast_v5i64_to_v10f32(<5 x i64> %a, i32 %b) {
 
 cmp.true:
   %a1 = add <5 x i64> %a, splat (i64 3)
-  %a2 = bitcast <5 x i64> %a1 to <10 x float>
+  %a2 = bitcast <5 x i64> %a1 to <20 x i16>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <5 x i64> %a to <10 x float>
+  %a3 = bitcast <5 x i64> %a to <20 x i16>
   br label %end
 
 end:
-  %phi = phi <10 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <10 x float> %phi
+  %phi = phi <20 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i16> %phi
 }
 
 define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
@@ -5743,7 +9787,7 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr22
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:    s_cbranch_execz .LBB30_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v55
 ; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v21
@@ -5805,9 +9849,9 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr47
 ; GCN-NEXT:    ; implicit-def: $vgpr45
 ; GCN-NEXT:    ; implicit-def: $vgpr44
-; GCN-NEXT:  .LBB18_2: ; %Flow
+; GCN-NEXT:  .LBB30_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB18_4
+; GCN-NEXT:    s_cbranch_execz .LBB30_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v47
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v45
@@ -5914,7 +9958,7 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    v_lshrrev_b32_e32 v30, 8, v9
 ; GCN-NEXT:    v_lshrrev_b32_e32 v23, 8, v6
 ; GCN-NEXT:    v_bfe_u32 v22, v1, 8, 8
-; GCN-NEXT:  .LBB18_4: ; %end
+; GCN-NEXT:  .LBB30_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    v_and_b32_e32 v29, 0xff, v29
 ; GCN-NEXT:    v_lshlrev_b32_e32 v38, 8, v38
@@ -6072,7 +10116,7 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr11
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:    s_cbranch_execz .LBB30_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; VI-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -6094,9 +10138,9 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    v_lshrrev_b32_e32 v48, 24, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB18_2: ; %Flow
+; VI-NEXT:  .LBB30_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB18_4
+; VI-NEXT:    s_cbranch_execz .LBB30_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    v_mov_b32_e32 v11, 0x200
 ; VI-NEXT:    v_add_f16_sdwa v23, v2, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -6159,7 +10203,7 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    v_bfe_u32 v35, v19, 8, 8
 ; VI-NEXT:    v_bfe_u32 v38, v21, 8, 8
 ; VI-NEXT:    v_bfe_u32 v48, v23, 8, 8
-; VI-NEXT:  .LBB18_4: ; %end
+; VI-NEXT:  .LBB30_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -6269,7 +10313,7 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr11
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:    s_cbranch_execz .LBB30_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX9-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -6301,9 +10345,9 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB18_2: ; %Flow
+; GFX9-NEXT:  .LBB30_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB18_4
+; GFX9-NEXT:    s_cbranch_execz .LBB30_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    s_movk_i32 s6, 0x200
 ; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
@@ -6346,7 +10390,7 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB18_4: ; %end
+; GFX9-NEXT:  .LBB30_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -6448,7 +10492,7 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:    s_cbranch_execz .LBB30_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX11-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX11-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -6480,9 +10524,9 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB18_2: ; %Flow
+; GFX11-NEXT:  .LBB30_2: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB18_4
+; GFX11-NEXT:    s_cbranch_execz .LBB30_4
 ; GFX11-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
@@ -6524,7 +10568,7 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB18_4: ; %end
+; GFX11-NEXT:  .LBB30_4: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
@@ -6710,7 +10754,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:    s_cbranch_execz .LBB31_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v2
@@ -6812,9 +10856,9 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr45
 ; GCN-NEXT:    ; implicit-def: $vgpr46
 ; GCN-NEXT:    ; implicit-def: $vgpr47
-; GCN-NEXT:  .LBB19_2: ; %Flow
+; GCN-NEXT:  .LBB31_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB19_4
+; GCN-NEXT:    s_cbranch_execz .LBB31_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v59
 ; GCN-NEXT:    s_movk_i32 s6, 0x300
@@ -6917,7 +10961,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v24
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v22
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v20
-; GCN-NEXT:  .LBB19_4: ; %end
+; GCN-NEXT:  .LBB31_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v0, v31
 ; GCN-NEXT:    v_mov_b32_e32 v2, v23
@@ -7003,7 +11047,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:    s_cbranch_execz .LBB31_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_or_b32_sdwa v0, v36, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v32, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -7076,9 +11120,9 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr45
 ; VI-NEXT:    ; implicit-def: $vgpr46
 ; VI-NEXT:    ; implicit-def: $vgpr47
-; VI-NEXT:  .LBB19_2: ; %Flow
+; VI-NEXT:  .LBB31_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB19_4
+; VI-NEXT:    s_cbranch_execz .LBB31_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v55
 ; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -7154,7 +11198,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v7, v12, v7
 ; VI-NEXT:    v_or_b32_e32 v8, v11, v8
 ; VI-NEXT:    v_or_b32_e32 v9, v10, v9
-; VI-NEXT:  .LBB19_4: ; %end
+; VI-NEXT:  .LBB31_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -7226,7 +11270,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:    s_cbranch_execz .LBB31_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v36, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v33, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -7300,9 +11344,9 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr47
 ; GFX9-NEXT:    ; implicit-def: $vgpr46
-; GFX9-NEXT:  .LBB19_2: ; %Flow
+; GFX9-NEXT:  .LBB31_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB19_4
+; GFX9-NEXT:    s_cbranch_execz .LBB31_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v42
@@ -7377,7 +11421,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v7, v12, v7, s6
 ; GFX9-NEXT:    v_perm_b32 v8, v11, v8, s6
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
-; GFX9-NEXT:  .LBB19_4: ; %end
+; GFX9-NEXT:  .LBB31_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -7439,15 +11483,15 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB19_3
+; GFX11-NEXT:    s_cbranch_execnz .LBB31_3
 ; GFX11-NEXT:  ; %bb.1: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB19_4
-; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_cbranch_execnz .LBB31_4
+; GFX11-NEXT:  .LBB31_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-; GFX11-NEXT:  .LBB19_3: ; %cmp.false
+; GFX11-NEXT:  .LBB31_3: ; %cmp.false
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v36
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v37
 ; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v35
@@ -7540,8 +11584,8 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    ; implicit-def: $vgpr70
 ; GFX11-NEXT:    ; implicit-def: $vgpr71
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB19_2
-; GFX11-NEXT:  .LBB19_4: ; %cmp.true
+; GFX11-NEXT:    s_cbranch_execz .LBB31_2
+; GFX11-NEXT:  .LBB31_4: ; %cmp.true
 ; GFX11-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-NEXT:    v_add_nc_u16 v0, v68, 3
 ; GFX11-NEXT:    v_add_nc_u16 v1, v66, 3
@@ -7684,14 +11728,14 @@ define <5 x double> @bitcast_v20f16_to_v5f64(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB20_3
+; GCN-NEXT:    s_cbranch_execnz .LBB32_3
 ; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB20_4
-; GCN-NEXT:  .LBB20_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB32_4
+; GCN-NEXT:  .LBB32_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
-; GCN-NEXT:  .LBB20_3: ; %cmp.false
+; GCN-NEXT:  .LBB32_3: ; %cmp.false
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v36
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v34
 ; GCN-NEXT:    v_or_b32_e32 v0, v35, v0
@@ -7733,8 +11777,8 @@ define <5 x double> @bitcast_v20f16_to_v5f64(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    ; implicit-def: $vgpr16
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB20_2
-; GCN-NEXT:  .LBB20_4: ; %cmp.true
+; GCN-NEXT:    s_cbranch_execz .LBB32_2
+; GCN-NEXT:  .LBB32_4: ; %cmp.true
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v36
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
@@ -7825,7 +11869,7 @@ define <5 x double> @bitcast_v20f16_to_v5f64(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:    s_cbranch_execz .LBB32_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_mov_b32_e32 v10, 0x200
 ; VI-NEXT:    v_add_f16_sdwa v11, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -7858,7 +11902,7 @@ define <5 x double> @bitcast_v20f16_to_v5f64(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
 ; VI-NEXT:    v_or_b32_e32 v1, v1, v11
 ; VI-NEXT:    v_or_b32_e32 v0, v0, v10
-; VI-NEXT:  .LBB20_2: ; %end
+; VI-NEXT:  .LBB32_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7869,7 +11913,7 @@ define <5 x double> @bitcast_v20f16_to_v5f64(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:    s_cbranch_execz .LBB32_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    s_movk_i32 s6, 0x200
 ; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
@@ -7882,7 +11926,7 @@ define <5 x double> @bitcast_v20f16_to_v5f64(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:  .LBB20_2: ; %end
+; GFX9-NEXT:  .LBB32_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7894,7 +11938,7 @@ define <5 x double> @bitcast_v20f16_to_v5f64(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:    s_cbranch_execz .LBB32_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
@@ -7906,7 +11950,7 @@ define <5 x double> @bitcast_v20f16_to_v5f64(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
-; GFX11-NEXT:  .LBB20_2: ; %end
+; GFX11-NEXT:  .LBB32_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -7953,7 +11997,7 @@ define <20 x half> @bitcast_v5f64_to_v20f16(<5 x double> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:    s_cbranch_execz .LBB33_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
 ; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
@@ -7990,9 +12034,9 @@ define <20 x half> @bitcast_v5f64_to_v20f16(<5 x double> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr4
 ; GCN-NEXT:    ; implicit-def: $vgpr6
 ; GCN-NEXT:    ; implicit-def: $vgpr8
-; GCN-NEXT:  .LBB21_2: ; %Flow
+; GCN-NEXT:  .LBB33_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB21_4
+; GCN-NEXT:    s_cbranch_execz .LBB33_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
 ; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
@@ -8029,7 +12073,7 @@ define <20 x half> @bitcast_v5f64_to_v20f16(<5 x double> %a, i32 %b) {
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
-; GCN-NEXT:  .LBB21_4: ; %end
+; GCN-NEXT:  .LBB33_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v0, v24
 ; GCN-NEXT:    v_mov_b32_e32 v1, v29
@@ -8050,14 +12094,14 @@ define <20 x half> @bitcast_v5f64_to_v20f16(<5 x double> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:    s_cbranch_execz .LBB33_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT:  .LBB21_2: ; %end
+; VI-NEXT:  .LBB33_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8068,14 +12112,14 @@ define <20 x half> @bitcast_v5f64_to_v20f16(<5 x double> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:    s_cbranch_execz .LBB33_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:  .LBB21_2: ; %end
+; GFX9-NEXT:  .LBB33_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8087,14 +12131,14 @@ define <20 x half> @bitcast_v5f64_to_v20f16(<5 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:    s_cbranch_execz .LBB33_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
 ; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:  .LBB33_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -8142,14 +12186,14 @@ define <5 x i64> @bitcast_v20f16_to_v5i64(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB22_3
+; GCN-NEXT:    s_cbranch_execnz .LBB34_3
 ; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB22_4
-; GCN-NEXT:  .LBB22_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB34_4
+; GCN-NEXT:  .LBB34_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
-; GCN-NEXT:  .LBB22_3: ; %cmp.false
+; GCN-NEXT:  .LBB34_3: ; %cmp.false
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v36
 ; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v34
 ; GCN-NEXT:    v_or_b32_e32 v0, v35, v0
@@ -8191,8 +12235,8 @@ define <5 x i64> @bitcast_v20f16_to_v5i64(<20 x half> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    ; implicit-def: $vgpr16
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB22_2
-; GCN-NEXT:  .LBB22_4: ; %cmp.true
+; GCN-NEXT:    s_cbranch_execz .LBB34_2
+; GCN-NEXT:  .LBB34_4: ; %cmp.true
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v36
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
 ; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
@@ -8283,7 +12327,7 @@ define <5 x i64> @bitcast_v20f16_to_v5i64(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:    s_cbranch_execz .LBB34_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_mov_b32_e32 v10, 0x200
 ; VI-NEXT:    v_add_f16_sdwa v11, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -8316,7 +12360,7 @@ define <5 x i64> @bitcast_v20f16_to_v5i64(<20 x half> %a, i32 %b) {
 ; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
 ; VI-NEXT:    v_or_b32_e32 v1, v1, v11
 ; VI-NEXT:    v_or_b32_e32 v0, v0, v10
-; VI-NEXT:  .LBB22_2: ; %end
+; VI-NEXT:  .LBB34_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8327,7 +12371,7 @@ define <5 x i64> @bitcast_v20f16_to_v5i64(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:    s_cbranch_execz .LBB34_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    s_movk_i32 s6, 0x200
 ; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
@@ -8340,7 +12384,7 @@ define <5 x i64> @bitcast_v20f16_to_v5i64(<20 x half> %a, i32 %b) {
 ; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
-; GFX9-NEXT:  .LBB22_2: ; %end
+; GFX9-NEXT:  .LBB34_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8352,7 +12396,7 @@ define <5 x i64> @bitcast_v20f16_to_v5i64(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:    s_cbranch_execz .LBB34_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
@@ -8364,7 +12408,7 @@ define <5 x i64> @bitcast_v20f16_to_v5i64(<20 x half> %a, i32 %b) {
 ; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
 ; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
-; GFX11-NEXT:  .LBB22_2: ; %end
+; GFX11-NEXT:  .LBB34_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -8421,14 +12465,14 @@ define <20 x half> @bitcast_v5i64_to_v20f16(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr19
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB23_3
+; GCN-NEXT:    s_cbranch_execnz .LBB35_3
 ; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB23_4
-; GCN-NEXT:  .LBB23_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB35_4
+; GCN-NEXT:  .LBB35_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
-; GCN-NEXT:  .LBB23_3: ; %cmp.false
+; GCN-NEXT:  .LBB35_3: ; %cmp.false
 ; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v22
 ; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v21
 ; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v24
@@ -8470,8 +12514,8 @@ define <20 x half> @bitcast_v5i64_to_v20f16(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr21
 ; GCN-NEXT:    ; implicit-def: $vgpr22
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB23_2
-; GCN-NEXT:  .LBB23_4: ; %cmp.true
+; GCN-NEXT:    s_cbranch_execz .LBB35_2
+; GCN-NEXT:  .LBB35_4: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v20
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v29, vcc
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v27
@@ -8522,7 +12566,7 @@ define <20 x half> @bitcast_v5i64_to_v20f16(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:    s_cbranch_execz .LBB35_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
@@ -8534,7 +12578,7 @@ define <20 x half> @bitcast_v5i64_to_v20f16(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:  .LBB23_2: ; %end
+; VI-NEXT:  .LBB35_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8545,7 +12589,7 @@ define <20 x half> @bitcast_v5i64_to_v20f16(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:    s_cbranch_execz .LBB35_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
 ; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
@@ -8557,7 +12601,7 @@ define <20 x half> @bitcast_v5i64_to_v20f16(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:  .LBB23_2: ; %end
+; GFX9-NEXT:  .LBB35_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8569,7 +12613,7 @@ define <20 x half> @bitcast_v5i64_to_v20f16(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:    s_cbranch_execz .LBB35_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -8584,7 +12628,7 @@ define <20 x half> @bitcast_v5i64_to_v20f16(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:  .LBB35_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -8663,7 +12707,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:    s_cbranch_execz .LBB36_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v32
@@ -8775,9 +12819,9 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr25
 ; GCN-NEXT:    ; implicit-def: $vgpr43
 ; GCN-NEXT:    ; implicit-def: $vgpr27
-; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:  .LBB36_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:    s_cbranch_execz .LBB36_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
 ; GCN-NEXT:    s_movk_i32 s6, 0x300
@@ -8891,7 +12935,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
 ; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x3000000, v9
-; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:  .LBB36_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -8964,7 +13008,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:    s_cbranch_execz .LBB36_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_or_b32_sdwa v0, v31, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v32, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -9037,9 +13081,9 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr21
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr17
-; VI-NEXT:  .LBB24_2: ; %Flow
+; VI-NEXT:  .LBB36_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB24_4
+; VI-NEXT:    s_cbranch_execz .LBB36_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v31
 ; VI-NEXT:    v_add_u16_e32 v1, 3, v32
@@ -9113,7 +13157,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_add_u16_sdwa v9, v11, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v9, v10, v9
-; VI-NEXT:  .LBB24_4: ; %end
+; VI-NEXT:  .LBB36_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -9187,7 +13231,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:    s_cbranch_execz .LBB36_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v31, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -9260,9 +13304,9 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
-; GFX9-NEXT:  .LBB24_2: ; %Flow
+; GFX9-NEXT:  .LBB36_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB24_4
+; GFX9-NEXT:    s_cbranch_execz .LBB36_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v31
 ; GFX9-NEXT:    v_add_u16_e32 v1, 3, v32
@@ -9336,7 +13380,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
-; GFX9-NEXT:  .LBB24_4: ; %end
+; GFX9-NEXT:  .LBB36_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -9399,15 +13443,15 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB24_3
+; GFX11-NEXT:    s_cbranch_execnz .LBB36_3
 ; GFX11-NEXT:  ; %bb.1: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB24_4
-; GFX11-NEXT:  .LBB24_2: ; %end
+; GFX11-NEXT:    s_cbranch_execnz .LBB36_4
+; GFX11-NEXT:  .LBB36_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-; GFX11-NEXT:  .LBB24_3: ; %cmp.false
+; GFX11-NEXT:  .LBB36_3: ; %cmp.false
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v32
 ; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v33
@@ -9520,8 +13564,8 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    ; implicit-def: $vgpr23
 ; GFX11-NEXT:    ; implicit-def: $vgpr25
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB24_2
-; GFX11-NEXT:  .LBB24_4: ; %cmp.true
+; GFX11-NEXT:    s_cbranch_execz .LBB36_2
+; GFX11-NEXT:  .LBB36_4: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u16 v0, v31, 3
 ; GFX11-NEXT:    v_add_nc_u16 v1, v32, 3
 ; GFX11-NEXT:    v_add_nc_u16 v2, v33, 3
@@ -9689,7 +13733,7 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr20
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:    s_cbranch_execz .LBB37_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_alignbit_b32 v11, v10, v9, 24
 ; GCN-NEXT:    v_alignbit_b32 v12, v10, v9, 16
@@ -9721,9 +13765,9 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
-; GCN-NEXT:  .LBB25_2: ; %Flow
+; GCN-NEXT:  .LBB37_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB25_4
+; GCN-NEXT:    s_cbranch_execz .LBB37_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
 ; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
@@ -9760,7 +13804,7 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
-; GCN-NEXT:  .LBB25_4: ; %end
+; GCN-NEXT:  .LBB37_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    v_and_b32_e32 v49, 0xff, v1
 ; GCN-NEXT:    v_lshlrev_b32_e32 v50, 8, v35
@@ -9910,7 +13954,7 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr11
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:    s_cbranch_execz .LBB37_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; VI-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -9942,9 +13986,9 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB25_2: ; %Flow
+; VI-NEXT:  .LBB37_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB25_4
+; VI-NEXT:    s_cbranch_execz .LBB37_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
 ; VI-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
@@ -9981,7 +14025,7 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB25_4: ; %end
+; VI-NEXT:  .LBB37_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -10091,7 +14135,7 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr11
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:    s_cbranch_execz .LBB37_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX9-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -10123,9 +14167,9 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB25_2: ; %Flow
+; GFX9-NEXT:  .LBB37_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB25_4
+; GFX9-NEXT:    s_cbranch_execz .LBB37_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
 ; GFX9-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
@@ -10162,7 +14206,7 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB25_4: ; %end
+; GFX9-NEXT:  .LBB37_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -10264,7 +14308,7 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:    s_cbranch_execz .LBB37_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX11-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX11-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -10296,9 +14340,9 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB25_2: ; %Flow
+; GFX11-NEXT:  .LBB37_2: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB25_4
+; GFX11-NEXT:    s_cbranch_execz .LBB37_4
 ; GFX11-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX11-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
 ; GFX11-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
@@ -10335,7 +14379,7 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB25_4: ; %end
+; GFX11-NEXT:  .LBB37_4: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
@@ -10509,7 +14553,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:    s_cbranch_execz .LBB38_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; GCN-NEXT:    v_and_b32_e32 v1, 0xff, v32
@@ -10621,9 +14665,9 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr25
 ; GCN-NEXT:    ; implicit-def: $vgpr43
 ; GCN-NEXT:    ; implicit-def: $vgpr27
-; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:  .LBB38_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:    s_cbranch_execz .LBB38_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
 ; GCN-NEXT:    s_movk_i32 s6, 0x300
@@ -10737,7 +14781,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
 ; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x3000000, v9
-; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:  .LBB38_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -10810,7 +14854,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:    s_cbranch_execz .LBB38_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_or_b32_sdwa v0, v31, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v32, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -10883,9 +14927,9 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr21
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr17
-; VI-NEXT:  .LBB26_2: ; %Flow
+; VI-NEXT:  .LBB38_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB26_4
+; VI-NEXT:    s_cbranch_execz .LBB38_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v31
 ; VI-NEXT:    v_add_u16_e32 v1, 3, v32
@@ -10959,7 +15003,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_add_u16_sdwa v9, v11, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v9, v10, v9
-; VI-NEXT:  .LBB26_4: ; %end
+; VI-NEXT:  .LBB38_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -11033,7 +15077,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:    s_cbranch_execz .LBB38_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v31, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -11106,9 +15150,9 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
-; GFX9-NEXT:  .LBB26_2: ; %Flow
+; GFX9-NEXT:  .LBB38_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB26_4
+; GFX9-NEXT:    s_cbranch_execz .LBB38_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v31
 ; GFX9-NEXT:    v_add_u16_e32 v1, 3, v32
@@ -11182,7 +15226,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
-; GFX9-NEXT:  .LBB26_4: ; %end
+; GFX9-NEXT:  .LBB38_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -11245,15 +15289,15 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB26_3
+; GFX11-NEXT:    s_cbranch_execnz .LBB38_3
 ; GFX11-NEXT:  ; %bb.1: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execnz .LBB26_4
-; GFX11-NEXT:  .LBB26_2: ; %end
+; GFX11-NEXT:    s_cbranch_execnz .LBB38_4
+; GFX11-NEXT:  .LBB38_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-; GFX11-NEXT:  .LBB26_3: ; %cmp.false
+; GFX11-NEXT:  .LBB38_3: ; %cmp.false
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v32
 ; GFX11-NEXT:    v_and_b32_e32 v2, 0xff, v33
@@ -11366,8 +15410,8 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX11-NEXT:    ; implicit-def: $vgpr23
 ; GFX11-NEXT:    ; implicit-def: $vgpr25
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB26_2
-; GFX11-NEXT:  .LBB26_4: ; %cmp.true
+; GFX11-NEXT:    s_cbranch_execz .LBB38_2
+; GFX11-NEXT:  .LBB38_4: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u16 v0, v31, 3
 ; GFX11-NEXT:    v_add_nc_u16 v1, v32, 3
 ; GFX11-NEXT:    v_add_nc_u16 v2, v33, 3
@@ -11535,7 +15579,7 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    ; implicit-def: $vgpr17
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:    s_cbranch_execz .LBB39_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.false
 ; GCN-NEXT:    v_alignbit_b32 v11, v10, v9, 24
 ; GCN-NEXT:    v_alignbit_b32 v12, v10, v9, 16
@@ -11567,9 +15611,9 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
-; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:  .LBB39_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:    s_cbranch_execz .LBB39_4
 ; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
@@ -11611,7 +15655,7 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    v_lshrrev_b32_e32 v38, 24, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
 ; GCN-NEXT:    v_lshrrev_b32_e32 v48, 8, v2
-; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:  .LBB39_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    v_and_b32_e32 v49, 0xff, v1
 ; GCN-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
@@ -11761,7 +15805,7 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr11
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:    s_cbranch_execz .LBB39_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; VI-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -11793,9 +15837,9 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB27_2: ; %Flow
+; VI-NEXT:  .LBB39_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB27_4
+; VI-NEXT:    s_cbranch_execz .LBB39_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
@@ -11837,7 +15881,7 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; VI-NEXT:  .LBB27_4: ; %end
+; VI-NEXT:  .LBB39_4: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -11947,7 +15991,7 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr11
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:    s_cbranch_execz .LBB39_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX9-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -11979,9 +16023,9 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB27_2: ; %Flow
+; GFX9-NEXT:  .LBB39_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB27_4
+; GFX9-NEXT:    s_cbranch_execz .LBB39_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, 3, v1
 ; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
@@ -12023,7 +16067,7 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX9-NEXT:  .LBB27_4: ; %end
+; GFX9-NEXT:  .LBB39_4: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v15
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v16
@@ -12125,7 +16169,7 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:    s_cbranch_execz .LBB39_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX11-NEXT:    v_lshrrev_b64 v[11:12], 24, v[9:10]
 ; GFX11-NEXT:    v_lshrrev_b64 v[12:13], 24, v[7:8]
@@ -12157,9 +16201,9 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB27_2: ; %Flow
+; GFX11-NEXT:  .LBB39_2: ; %Flow
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB27_4
+; GFX11-NEXT:    s_cbranch_execz .LBB39_4
 ; GFX11-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, v3, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -12204,7 +16248,7 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 8, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
-; GFX11-NEXT:  .LBB27_4: ; %end
+; GFX11-NEXT:  .LBB39_4: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
@@ -12327,14 +16371,14 @@ define <5 x i64> @bitcast_v5f64_to_v5i64(<5 x double> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:    s_cbranch_execz .LBB40_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
 ; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GCN-NEXT:  .LBB28_2: ; %end
+; GCN-NEXT:  .LBB40_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -12345,14 +16389,14 @@ define <5 x i64> @bitcast_v5f64_to_v5i64(<5 x double> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:    s_cbranch_execz .LBB40_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
 ; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:  .LBB40_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -12363,14 +16407,14 @@ define <5 x i64> @bitcast_v5f64_to_v5i64(<5 x double> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:    s_cbranch_execz .LBB40_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
 ; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:  .LBB40_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -12382,14 +16426,14 @@ define <5 x i64> @bitcast_v5f64_to_v5i64(<5 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:    s_cbranch_execz .LBB40_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
 ; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
 ; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:  .LBB40_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -12417,7 +16461,7 @@ define <5 x double> @bitcast_v5i64_to_v5f64(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:    s_cbranch_execz .LBB41_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -12429,7 +16473,7 @@ define <5 x double> @bitcast_v5i64_to_v5f64(<5 x i64> %a, i32 %b) {
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:  .LBB41_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -12440,7 +16484,7 @@ define <5 x double> @bitcast_v5i64_to_v5f64(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:    s_cbranch_execz .LBB41_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -12452,7 +16496,7 @@ define <5 x double> @bitcast_v5i64_to_v5f64(<5 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:  .LBB41_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -12463,7 +16507,7 @@ define <5 x double> @bitcast_v5i64_to_v5f64(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:    s_cbranch_execz .LBB41_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
@@ -12475,7 +16519,7 @@ define <5 x double> @bitcast_v5i64_to_v5f64(<5 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
 ; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:  .LBB41_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -12487,7 +16531,7 @@ define <5 x double> @bitcast_v5i64_to_v5f64(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:    s_cbranch_execz .LBB41_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -12502,7 +16546,7 @@ define <5 x double> @bitcast_v5i64_to_v5f64(<5 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:  .LBB41_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
index fa1f3bd96ad2f..2a7315c055a54 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
@@ -226,3 +226,2346 @@ end:
   %phi = phi <11 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <11 x i32> %phi
 }
+
+define <22 x i16> @bitcast_v11i32_to_v22i16(<11 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11i32_to_v22i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v20, v10
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB2_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB2_4
+; GCN-NEXT:  .LBB2_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB2_3: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, s4, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
+; GCN-NEXT:  .LBB2_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_alignbit_b32 v21, s4, v20, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11i32_to_v22i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB2_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB2_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11i32_to_v22i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB2_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11i32_to_v22i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB2_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <11 x i32> %a, splat (i32 3)
+  %a2 = bitcast <11 x i32> %a1 to <22 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x i32> %a to <22 x i16>
+  br label %end
+
+end:
+  %phi = phi <22 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x i16> %phi
+}
+
+define <11 x i32> @bitcast_v22i16_to_v11i32(<22 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22i16_to_v11i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v28, v10
+; GCN-NEXT:    v_mov_b32_e32 v27, v8
+; GCN-NEXT:    v_mov_b32_e32 v26, v6
+; GCN-NEXT:    v_mov_b32_e32 v25, v4
+; GCN-NEXT:    v_mov_b32_e32 v24, v2
+; GCN-NEXT:    v_mov_b32_e32 v23, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB3_4
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB3_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v22
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v29
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v30
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v31
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v32
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  .LBB3_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v23
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_or_b32_e32 v0, v22, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v29, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v30, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v31, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v32, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x30000, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x30000, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 0x30000, v10
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22i16_to_v11i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v12, 3
+; VI-NEXT:    v_add_u16_e32 v11, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_u16_e32 v11, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v11, v9
+; VI-NEXT:    v_add_u16_e32 v11, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v11, v8
+; VI-NEXT:    v_add_u16_e32 v11, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v11, v7
+; VI-NEXT:    v_add_u16_e32 v11, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v11, v6
+; VI-NEXT:    v_add_u16_e32 v11, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v11, v5
+; VI-NEXT:    v_add_u16_e32 v11, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v11, v4
+; VI-NEXT:    v_add_u16_e32 v11, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v11, v3
+; VI-NEXT:    v_add_u16_e32 v11, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v11, v2
+; VI-NEXT:    v_add_u16_e32 v11, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v11, v1
+; VI-NEXT:    v_add_u16_e32 v11, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v11, v0
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22i16_to_v11i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22i16_to_v11i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <22 x i16> %a, splat (i16 3)
+  %a2 = bitcast <22 x i16> %a1 to <11 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x i16> %a to <11 x i32>
+  br label %end
+
+end:
+  %phi = phi <11 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x i32> %phi
+}
+
+define <22 x half> @bitcast_v11i32_to_v22f16(<11 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11i32_to_v22f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v32, v10
+; GCN-NEXT:    v_mov_b32_e32 v31, v9
+; GCN-NEXT:    v_mov_b32_e32 v30, v8
+; GCN-NEXT:    v_mov_b32_e32 v29, v7
+; GCN-NEXT:    v_mov_b32_e32 v28, v6
+; GCN-NEXT:    v_mov_b32_e32 v27, v5
+; GCN-NEXT:    v_mov_b32_e32 v26, v4
+; GCN-NEXT:    v_mov_b32_e32 v25, v3
+; GCN-NEXT:    v_mov_b32_e32 v24, v2
+; GCN-NEXT:    v_mov_b32_e32 v23, v1
+; GCN-NEXT:    v_mov_b32_e32 v22, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB4_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB4_4
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB4_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  .LBB4_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11i32_to_v22f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB4_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11i32_to_v22f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB4_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11i32_to_v22f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <11 x i32> %a, splat (i32 3)
+  %a2 = bitcast <11 x i32> %a1 to <22 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x i32> %a to <22 x half>
+  br label %end
+
+end:
+  %phi = phi <22 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x half> %phi
+}
+
+define <11 x i32> @bitcast_v22f16_to_v11i32(<22 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22f16_to_v11i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v20
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB5_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB5_4
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB5_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v0, v32, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v30, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v28, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v26, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v24, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v23, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v22, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v12, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  .LBB5_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v5, v18, v10
+; GCN-NEXT:    v_or_b32_e32 v6, v20, v19
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v21
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v12, v16
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v14
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22f16_to_v11i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v11, 0x200
+; VI-NEXT:    v_add_f16_sdwa v12, v10, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v9, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v8, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v4, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v3, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v11, v0, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v12
+; VI-NEXT:    v_or_b32_e32 v0, v0, v11
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22f16_to_v11i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22f16_to_v11i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <22 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <22 x half> %a1 to <11 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x half> %a to <11 x i32>
+  br label %end
+
+end:
+  %phi = phi <11 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x i32> %phi
+}
+
+define <22 x i16> @bitcast_v11f32_to_v22i16(<11 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11f32_to_v22i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v20, v10
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB6_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB6_4
+; GCN-NEXT:  .LBB6_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB6_3: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, s4, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  .LBB6_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_alignbit_b32 v21, s4, v20, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11f32_to_v22i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB6_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11f32_to_v22i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB6_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11f32_to_v22i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v10, 1.0, v10 :: v_dual_add_f32 v9, 1.0, v9
+; GFX11-NEXT:    v_dual_add_f32 v8, 1.0, v8 :: v_dual_add_f32 v7, 1.0, v7
+; GFX11-NEXT:    v_dual_add_f32 v6, 1.0, v6 :: v_dual_add_f32 v5, 1.0, v5
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <11 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <11 x float> %a1 to <22 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x float> %a to <22 x i16>
+  br label %end
+
+end:
+  %phi = phi <22 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x i16> %phi
+}
+
+define <11 x float> @bitcast_v22i16_to_v11f32(<22 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22i16_to_v11f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v28, v10
+; GCN-NEXT:    v_mov_b32_e32 v27, v8
+; GCN-NEXT:    v_mov_b32_e32 v26, v6
+; GCN-NEXT:    v_mov_b32_e32 v25, v4
+; GCN-NEXT:    v_mov_b32_e32 v24, v2
+; GCN-NEXT:    v_mov_b32_e32 v23, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_4
+; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v22
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v29
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v30
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v31
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v32
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v11
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  .LBB7_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v23
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_or_b32_e32 v0, v22, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v29, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v30, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v31, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v32, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x30000, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x30000, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 0x30000, v10
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22i16_to_v11f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v12, 3
+; VI-NEXT:    v_add_u16_e32 v11, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_u16_e32 v11, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v11, v9
+; VI-NEXT:    v_add_u16_e32 v11, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v11, v8
+; VI-NEXT:    v_add_u16_e32 v11, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v11, v7
+; VI-NEXT:    v_add_u16_e32 v11, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v11, v6
+; VI-NEXT:    v_add_u16_e32 v11, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v11, v5
+; VI-NEXT:    v_add_u16_e32 v11, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v11, v4
+; VI-NEXT:    v_add_u16_e32 v11, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v11, v3
+; VI-NEXT:    v_add_u16_e32 v11, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v11, v2
+; VI-NEXT:    v_add_u16_e32 v11, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v11, v1
+; VI-NEXT:    v_add_u16_e32 v11, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v11, v0
+; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22i16_to_v11f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22i16_to_v11f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <22 x i16> %a, splat (i16 3)
+  %a2 = bitcast <22 x i16> %a1 to <11 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x i16> %a to <11 x float>
+  br label %end
+
+end:
+  %phi = phi <11 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x float> %phi
+}
+
+define <22 x half> @bitcast_v11f32_to_v22f16(<11 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11f32_to_v22f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v32, v10
+; GCN-NEXT:    v_mov_b32_e32 v31, v9
+; GCN-NEXT:    v_mov_b32_e32 v30, v8
+; GCN-NEXT:    v_mov_b32_e32 v29, v7
+; GCN-NEXT:    v_mov_b32_e32 v28, v6
+; GCN-NEXT:    v_mov_b32_e32 v27, v5
+; GCN-NEXT:    v_mov_b32_e32 v26, v4
+; GCN-NEXT:    v_mov_b32_e32 v25, v3
+; GCN-NEXT:    v_mov_b32_e32 v24, v2
+; GCN-NEXT:    v_mov_b32_e32 v23, v1
+; GCN-NEXT:    v_mov_b32_e32 v22, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_4
+; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB8_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  .LBB8_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v29
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v30
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v31
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11f32_to_v22f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB8_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11f32_to_v22f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB8_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11f32_to_v22f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v10, 1.0, v10 :: v_dual_add_f32 v9, 1.0, v9
+; GFX11-NEXT:    v_dual_add_f32 v8, 1.0, v8 :: v_dual_add_f32 v7, 1.0, v7
+; GFX11-NEXT:    v_dual_add_f32 v6, 1.0, v6 :: v_dual_add_f32 v5, 1.0, v5
+; GFX11-NEXT:    v_dual_add_f32 v4, 1.0, v4 :: v_dual_add_f32 v3, 1.0, v3
+; GFX11-NEXT:    v_dual_add_f32 v2, 1.0, v2 :: v_dual_add_f32 v1, 1.0, v1
+; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <11 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <11 x float> %a1 to <22 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x float> %a to <22 x half>
+  br label %end
+
+end:
+  %phi = phi <22 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x half> %phi
+}
+
+define <11 x float> @bitcast_v22f16_to_v11f32(<22 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22f16_to_v11f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v20
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_4
+; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB9_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v0, v32, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v30, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v28, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v26, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v24, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v23, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v22, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v12, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  .LBB9_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v5, v18, v10
+; GCN-NEXT:    v_or_b32_e32 v6, v20, v19
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v21
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v12, v16
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v14
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22f16_to_v11f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v11, 0x200
+; VI-NEXT:    v_add_f16_sdwa v12, v10, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v9, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v8, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v4, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v3, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v11, v0, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v12
+; VI-NEXT:    v_or_b32_e32 v0, v0, v11
+; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22f16_to_v11f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22f16_to_v11f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <22 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <22 x half> %a1 to <11 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x half> %a to <11 x float>
+  br label %end
+
+end:
+  %phi = phi <11 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x float> %phi
+}
+
+define <22 x half> @bitcast_v22i16_to_v22f16(<22 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22i16_to_v22f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v51, v21
+; GCN-NEXT:    v_mov_b32_e32 v50, v20
+; GCN-NEXT:    v_mov_b32_e32 v49, v19
+; GCN-NEXT:    v_mov_b32_e32 v48, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v17
+; GCN-NEXT:    v_mov_b32_e32 v38, v16
+; GCN-NEXT:    v_mov_b32_e32 v37, v15
+; GCN-NEXT:    v_mov_b32_e32 v36, v14
+; GCN-NEXT:    v_mov_b32_e32 v35, v13
+; GCN-NEXT:    v_mov_b32_e32 v34, v12
+; GCN-NEXT:    v_mov_b32_e32 v33, v11
+; GCN-NEXT:    v_mov_b32_e32 v32, v10
+; GCN-NEXT:    v_mov_b32_e32 v31, v9
+; GCN-NEXT:    v_mov_b32_e32 v30, v8
+; GCN-NEXT:    v_mov_b32_e32 v29, v7
+; GCN-NEXT:    v_mov_b32_e32 v28, v6
+; GCN-NEXT:    v_mov_b32_e32 v27, v5
+; GCN-NEXT:    v_mov_b32_e32 v26, v4
+; GCN-NEXT:    v_mov_b32_e32 v25, v3
+; GCN-NEXT:    v_mov_b32_e32 v24, v2
+; GCN-NEXT:    v_mov_b32_e32 v23, v1
+; GCN-NEXT:    v_mov_b32_e32 v52, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB10_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB10_4
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB10_3: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  .LBB10_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22i16_to_v22f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v11, 3
+; VI-NEXT:    v_add_u16_sdwa v19, v10, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v9, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v12, v0, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v13, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v14, v2, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v15, v3, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v16, v4, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v17, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v18, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v11, v8, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_or_b32_e32 v8, v8, v11
+; VI-NEXT:    v_or_b32_e32 v7, v7, v19
+; VI-NEXT:    v_or_b32_e32 v6, v6, v18
+; VI-NEXT:    v_or_b32_e32 v5, v5, v17
+; VI-NEXT:    v_or_b32_e32 v4, v4, v16
+; VI-NEXT:    v_or_b32_e32 v3, v3, v15
+; VI-NEXT:    v_or_b32_e32 v2, v2, v14
+; VI-NEXT:    v_or_b32_e32 v1, v1, v13
+; VI-NEXT:    v_or_b32_e32 v0, v0, v12
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22i16_to_v22f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22i16_to_v22f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <22 x i16> %a, splat (i16 3)
+  %a2 = bitcast <22 x i16> %a1 to <22 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x i16> %a to <22 x half>
+  br label %end
+
+end:
+  %phi = phi <22 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x half> %phi
+}
+
+define <22 x i16> @bitcast_v22f16_to_v22i16(<22 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22f16_to_v22i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v9
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v17
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v17, 16
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22f16_to_v22i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v12, 0x200
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v10, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v19, v10
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v9
+; VI-NEXT:    v_add_f16_sdwa v9, v9, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v0
+; VI-NEXT:    v_add_f16_sdwa v0, v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v1, v1, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v2, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v3
+; VI-NEXT:    v_add_f16_sdwa v3, v3, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v4
+; VI-NEXT:    v_add_f16_sdwa v4, v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v5
+; VI-NEXT:    v_add_f16_sdwa v5, v5, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v6, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v19, v9
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v7, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_sdwa v12, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v12
+; VI-NEXT:    v_or_b32_e32 v7, v19, v7
+; VI-NEXT:    v_or_b32_e32 v6, v18, v6
+; VI-NEXT:    v_or_b32_e32 v5, v17, v5
+; VI-NEXT:    v_or_b32_e32 v4, v16, v4
+; VI-NEXT:    v_or_b32_e32 v3, v15, v3
+; VI-NEXT:    v_or_b32_e32 v2, v14, v2
+; VI-NEXT:    v_or_b32_e32 v1, v13, v1
+; VI-NEXT:    v_or_b32_e32 v0, v11, v0
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22f16_to_v22i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22f16_to_v22i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v11
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <22 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <22 x half> %a1 to <22 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x half> %a to <22 x i16>
+  br label %end
+
+end:
+  %phi = phi <22 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll
index c7af60f324892..264e2b2bf0122 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll
@@ -685,83 +685,2966 @@ end:
   ret <12 x i32> %phi
 }
 
+define <24 x i16> @bitcast_v12i32_to_v24i16(<12 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i32_to_v24i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v22, v11
+; GCN-NEXT:    v_mov_b32_e32 v20, v10
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v12, v24
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i32_to_v24i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB6_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i32_to_v24i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB6_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i32_to_v24i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB6_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i32> %a, splat (i32 3)
+  %a2 = bitcast <12 x i32> %a1 to <24 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i32> %a to <24 x i16>
+  br label %end
+
+end:
+  %phi = phi <24 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i16> %phi
+}
+
+define <12 x i32> @bitcast_v24i16_to_v12i32(<24 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i16_to_v12i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v30, v10
+; GCN-NEXT:    v_mov_b32_e32 v29, v8
+; GCN-NEXT:    v_mov_b32_e32 v28, v6
+; GCN-NEXT:    v_mov_b32_e32 v27, v4
+; GCN-NEXT:    v_mov_b32_e32 v26, v2
+; GCN-NEXT:    v_mov_b32_e32 v25, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_4
+; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v35
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v30
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v22
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v24
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v31
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v32
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v33
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  .LBB7_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v25
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v22
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v35, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v24, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v31, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v32, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v33, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v11
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x30000, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 0x30000, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 0x30000, v11
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24i16_to_v12i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v13, 3
+; VI-NEXT:    v_add_u16_e32 v12, 3, v11
+; VI-NEXT:    v_add_u16_sdwa v11, v11, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_u16_e32 v12, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v12, v10
+; VI-NEXT:    v_add_u16_e32 v12, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v12, v9
+; VI-NEXT:    v_add_u16_e32 v12, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v12, v8
+; VI-NEXT:    v_add_u16_e32 v12, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v12, v7
+; VI-NEXT:    v_add_u16_e32 v12, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v12, v6
+; VI-NEXT:    v_add_u16_e32 v12, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v12, v5
+; VI-NEXT:    v_add_u16_e32 v12, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v12, v4
+; VI-NEXT:    v_add_u16_e32 v12, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v12, v3
+; VI-NEXT:    v_add_u16_e32 v12, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v12, v2
+; VI-NEXT:    v_add_u16_e32 v12, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v12, v1
+; VI-NEXT:    v_add_u16_e32 v12, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v12, v0
+; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24i16_to_v12i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24i16_to_v12i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <24 x i16> %a, splat (i16 3)
+  %a2 = bitcast <24 x i16> %a1 to <12 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x i16> %a to <12 x i32>
+  br label %end
+
+end:
+  %phi = phi <12 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i32> %phi
+}
+
+define <24 x half> @bitcast_v12i32_to_v24f16(<12 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i32_to_v24f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v35, v11
+; GCN-NEXT:    v_mov_b32_e32 v34, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v9
+; GCN-NEXT:    v_mov_b32_e32 v32, v8
+; GCN-NEXT:    v_mov_b32_e32 v31, v7
+; GCN-NEXT:    v_mov_b32_e32 v30, v6
+; GCN-NEXT:    v_mov_b32_e32 v29, v5
+; GCN-NEXT:    v_mov_b32_e32 v28, v4
+; GCN-NEXT:    v_mov_b32_e32 v27, v3
+; GCN-NEXT:    v_mov_b32_e32 v26, v2
+; GCN-NEXT:    v_mov_b32_e32 v25, v1
+; GCN-NEXT:    v_mov_b32_e32 v24, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_4
+; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB8_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v33
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  .LBB8_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i32_to_v24f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB8_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i32_to_v24f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB8_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i32_to_v24f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB8_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i32> %a, splat (i32 3)
+  %a2 = bitcast <12 x i32> %a1 to <24 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i32> %a to <24 x half>
+  br label %end
+
+end:
+  %phi = phi <24 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x half> %phi
+}
+
+define <12 x i32> @bitcast_v24f16_to_v12i32(<24 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f16_to_v12i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v22
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB9_4
+; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB9_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v0, v39, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v37, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v2, v32, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v30, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v28, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v26, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v25, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v24, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v14, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v13, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  .LBB9_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v37
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v20, v18
+; GCN-NEXT:    v_or_b32_e32 v8, v16, v21
+; GCN-NEXT:    v_or_b32_e32 v9, v14, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v13, v17
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v15
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24f16_to_v12i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v12, 0x200
+; VI-NEXT:    v_add_f16_sdwa v13, v11, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v10, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v9, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v7, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v6, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v5, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v3, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v2, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v1, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v12, v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v13
+; VI-NEXT:    v_or_b32_e32 v0, v0, v12
+; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24f16_to_v12i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24f16_to_v12i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <24 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <24 x half> %a1 to <12 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x half> %a to <12 x i32>
+  br label %end
+
+end:
+  %phi = phi <12 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i32> %phi
+}
+
 define <6 x double> @bitcast_v12f32_to_v6f64(<12 x float> %a, i32 %b) {
 ; GCN-LABEL: bitcast_v12f32_to_v6f64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f32_to_v6f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f32_to_v6f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f32_to_v6f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <12 x float> %a1 to <6 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x float> %a to <6 x double>
+  br label %end
+
+end:
+  %phi = phi <6 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x double> %phi
+}
+
+define <12 x float> @bitcast_v6f64_to_v12f32(<6 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6f64_to_v12f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6f64_to_v12f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6f64_to_v12f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6f64_to_v12f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <6 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <6 x double> %a1 to <12 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x double> %a to <12 x float>
+  br label %end
+
+end:
+  %phi = phi <12 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x float> %phi
+}
+
+define <6 x i64> @bitcast_v12f32_to_v6i64(<12 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f32_to_v6i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f32_to_v6i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f32_to_v6i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f32_to_v6i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <12 x float> %a1 to <6 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x float> %a to <6 x i64>
+  br label %end
+
+end:
+  %phi = phi <6 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x i64> %phi
+}
+
+define <12 x float> @bitcast_v6i64_to_v12f32(<6 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6i64_to_v12f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6i64_to_v12f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6i64_to_v12f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6i64_to_v12f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <6 x i64> %a, splat (i64 3)
+  %a2 = bitcast <6 x i64> %a1 to <12 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x i64> %a to <12 x float>
+  br label %end
+
+end:
+  %phi = phi <12 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x float> %phi
+}
+
+define <24 x i16> @bitcast_v12f32_to_v24i16(<12 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f32_to_v24i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v22, v11
+; GCN-NEXT:    v_mov_b32_e32 v20, v10
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v12, v24
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f32_to_v24i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB14_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f32_to_v24i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB14_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f32_to_v24i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <12 x float> %a1 to <24 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x float> %a to <24 x i16>
+  br label %end
+
+end:
+  %phi = phi <24 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i16> %phi
+}
+
+define <12 x float> @bitcast_v24i16_to_v12f32(<24 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i16_to_v12f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v30, v10
+; GCN-NEXT:    v_mov_b32_e32 v29, v8
+; GCN-NEXT:    v_mov_b32_e32 v28, v6
+; GCN-NEXT:    v_mov_b32_e32 v27, v4
+; GCN-NEXT:    v_mov_b32_e32 v26, v2
+; GCN-NEXT:    v_mov_b32_e32 v25, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB15_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB15_4
+; GCN-NEXT:  .LBB15_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB15_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v35
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v30
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v22
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v24
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v31
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v32
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v33
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  .LBB15_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v25
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v22
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v35, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v24, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v31, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v32, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v33, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v11
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x30000, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 0x30000, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 0x30000, v11
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24i16_to_v12f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v13, 3
+; VI-NEXT:    v_add_u16_e32 v12, 3, v11
+; VI-NEXT:    v_add_u16_sdwa v11, v11, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_u16_e32 v12, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v12, v10
+; VI-NEXT:    v_add_u16_e32 v12, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v12, v9
+; VI-NEXT:    v_add_u16_e32 v12, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v12, v8
+; VI-NEXT:    v_add_u16_e32 v12, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v12, v7
+; VI-NEXT:    v_add_u16_e32 v12, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v12, v6
+; VI-NEXT:    v_add_u16_e32 v12, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v12, v5
+; VI-NEXT:    v_add_u16_e32 v12, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v12, v4
+; VI-NEXT:    v_add_u16_e32 v12, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v12, v3
+; VI-NEXT:    v_add_u16_e32 v12, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v12, v2
+; VI-NEXT:    v_add_u16_e32 v12, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v12, v1
+; VI-NEXT:    v_add_u16_e32 v12, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v12, v0
+; VI-NEXT:  .LBB15_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24i16_to_v12f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24i16_to_v12f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <24 x i16> %a, splat (i16 3)
+  %a2 = bitcast <24 x i16> %a1 to <12 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x i16> %a to <12 x float>
+  br label %end
+
+end:
+  %phi = phi <12 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x float> %phi
+}
+
+define <24 x half> @bitcast_v12f32_to_v24f16(<12 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f32_to_v24f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v35, v11
+; GCN-NEXT:    v_mov_b32_e32 v34, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v9
+; GCN-NEXT:    v_mov_b32_e32 v32, v8
+; GCN-NEXT:    v_mov_b32_e32 v31, v7
+; GCN-NEXT:    v_mov_b32_e32 v30, v6
+; GCN-NEXT:    v_mov_b32_e32 v29, v5
+; GCN-NEXT:    v_mov_b32_e32 v28, v4
+; GCN-NEXT:    v_mov_b32_e32 v27, v3
+; GCN-NEXT:    v_mov_b32_e32 v26, v2
+; GCN-NEXT:    v_mov_b32_e32 v25, v1
+; GCN-NEXT:    v_mov_b32_e32 v24, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB16_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB16_4
+; GCN-NEXT:  .LBB16_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB16_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v33
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  .LBB16_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v29
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v30
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v31
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v32
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v33
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v34
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f32_to_v24f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB16_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f32_to_v24f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB16_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f32_to_v24f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <12 x float> %a1 to <24 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x float> %a to <24 x half>
+  br label %end
+
+end:
+  %phi = phi <24 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x half> %phi
+}
+
+define <12 x float> @bitcast_v24f16_to_v12f32(<24 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f16_to_v12f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v22
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB17_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB17_4
+; GCN-NEXT:  .LBB17_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB17_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v0, v39, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v37, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v2, v32, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v30, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v28, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v26, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v25, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v24, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v14, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v13, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  .LBB17_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v37
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v20, v18
+; GCN-NEXT:    v_or_b32_e32 v8, v16, v21
+; GCN-NEXT:    v_or_b32_e32 v9, v14, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v13, v17
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v15
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24f16_to_v12f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v12, 0x200
+; VI-NEXT:    v_add_f16_sdwa v13, v11, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v10, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v9, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v7, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v6, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v5, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v3, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v2, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v1, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v12, v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v13
+; VI-NEXT:    v_or_b32_e32 v0, v0, v12
+; VI-NEXT:  .LBB17_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24f16_to_v12f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24f16_to_v12f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <24 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <24 x half> %a1 to <12 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x half> %a to <12 x float>
+  br label %end
+
+end:
+  %phi = phi <12 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x float> %phi
+}
+
+define <6 x i64> @bitcast_v6f64_to_v6i64(<6 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6f64_to_v6i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6f64_to_v6i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6f64_to_v6i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6f64_to_v6i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <6 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <6 x double> %a1 to <6 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x double> %a to <6 x i64>
+  br label %end
+
+end:
+  %phi = phi <6 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x i64> %phi
+}
+
+define <6 x double> @bitcast_v6i64_to_v6f64(<6 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6i64_to_v6f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6i64_to_v6f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6i64_to_v6f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6i64_to_v6f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <6 x i64> %a, splat (i64 3)
+  %a2 = bitcast <6 x i64> %a1 to <6 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x i64> %a to <6 x double>
+  br label %end
+
+end:
+  %phi = phi <6 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x double> %phi
+}
+
+define <24 x i16> @bitcast_v6f64_to_v24i16(<6 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6f64_to_v24i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v34, v11
+; GCN-NEXT:    v_mov_b32_e32 v33, v10
+; GCN-NEXT:    v_mov_b32_e32 v32, v9
+; GCN-NEXT:    v_mov_b32_e32 v31, v8
+; GCN-NEXT:    v_mov_b32_e32 v30, v7
+; GCN-NEXT:    v_mov_b32_e32 v29, v6
+; GCN-NEXT:    v_mov_b32_e32 v28, v5
+; GCN-NEXT:    v_mov_b32_e32 v27, v4
+; GCN-NEXT:    v_mov_b32_e32 v26, v3
+; GCN-NEXT:    v_mov_b32_e32 v25, v2
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, v34, v33, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v32, v31, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v1, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[25:26], v[25:26], 1.0
+; GCN-NEXT:    v_add_f64 v[27:28], v[27:28], 1.0
+; GCN-NEXT:    v_add_f64 v[29:30], v[29:30], 1.0
+; GCN-NEXT:    v_add_f64 v[31:32], v[31:32], 1.0
+; GCN-NEXT:    v_add_f64 v[33:34], v[33:34], 1.0
+; GCN-NEXT:    v_alignbit_b32 v21, v34, v33, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v32, v31, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v1, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_mov_b32_e32 v4, v25
+; GCN-NEXT:    v_mov_b32_e32 v6, v26
+; GCN-NEXT:    v_mov_b32_e32 v8, v27
+; GCN-NEXT:    v_mov_b32_e32 v10, v28
+; GCN-NEXT:    v_mov_b32_e32 v12, v29
+; GCN-NEXT:    v_mov_b32_e32 v14, v30
+; GCN-NEXT:    v_mov_b32_e32 v16, v31
+; GCN-NEXT:    v_mov_b32_e32 v18, v32
+; GCN-NEXT:    v_mov_b32_e32 v20, v33
+; GCN-NEXT:    v_mov_b32_e32 v22, v34
+; GCN-NEXT:    v_mov_b32_e32 v1, v24
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6f64_to_v24i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB20_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6f64_to_v24i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB20_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6f64_to_v24i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB20_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <6 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <6 x double> %a1 to <24 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x double> %a to <24 x i16>
+  br label %end
+
+end:
+  %phi = phi <24 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i16> %phi
+}
+
+define <6 x double> @bitcast_v24i16_to_v6f64(<24 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i16_to_v6f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v32, v14
+; GCN-NEXT:    v_mov_b32_e32 v31, v12
+; GCN-NEXT:    v_mov_b32_e32 v30, v10
+; GCN-NEXT:    v_mov_b32_e32 v29, v8
+; GCN-NEXT:    v_mov_b32_e32 v28, v6
+; GCN-NEXT:    v_mov_b32_e32 v25, v4
+; GCN-NEXT:    v_mov_b32_e32 v26, v2
+; GCN-NEXT:    v_mov_b32_e32 v27, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB21_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB6_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
-; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GCN-NEXT:  .LBB6_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB21_4
+; GCN-NEXT:  .LBB21_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB21_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v38
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v39
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v30
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v31
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v22
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v24
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v33
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v34
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v36
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v37
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v23
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  .LBB21_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v27
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v22
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v39, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v24, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v33, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v34, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v35, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v36, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v37, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v11
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 0x30000, v11
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v12f32_to_v6f64:
+; VI-LABEL: bitcast_v24i16_to_v6f64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:    s_cbranch_execz .LBB21_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
-; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; VI-NEXT:  .LBB6_2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v13, 3
+; VI-NEXT:    v_add_u16_e32 v12, 3, v11
+; VI-NEXT:    v_add_u16_sdwa v11, v11, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_u16_e32 v12, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v12, v10
+; VI-NEXT:    v_add_u16_e32 v12, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v12, v9
+; VI-NEXT:    v_add_u16_e32 v12, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v12, v8
+; VI-NEXT:    v_add_u16_e32 v12, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v12, v7
+; VI-NEXT:    v_add_u16_e32 v12, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v12, v6
+; VI-NEXT:    v_add_u16_e32 v12, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v12, v5
+; VI-NEXT:    v_add_u16_e32 v12, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v12, v4
+; VI-NEXT:    v_add_u16_e32 v12, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v12, v3
+; VI-NEXT:    v_add_u16_e32 v12, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v12, v2
+; VI-NEXT:    v_add_u16_e32 v12, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v12, v1
+; VI-NEXT:    v_add_u16_e32 v12, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v12, v0
+; VI-NEXT:  .LBB21_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v12f32_to_v6f64:
+; GFX9-LABEL: bitcast_v24i16_to_v6f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
-; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GFX9-NEXT:  .LBB6_2: ; %end
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v12f32_to_v6f64:
+; GFX11-LABEL: bitcast_v24i16_to_v6f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -769,26 +3652,33 @@ define <6 x double> @bitcast_v12f32_to_v6f64(<12 x float> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
-; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
-; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
-; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
-; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
-; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
-; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <12 x float> %a, splat (float 1.000000e+00)
-  %a2 = bitcast <12 x float> %a1 to <6 x double>
+  %a1 = add <24 x i16> %a, splat (i16 3)
+  %a2 = bitcast <24 x i16> %a1 to <6 x double>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <12 x float> %a to <6 x double>
+  %a3 = bitcast <24 x i16> %a to <6 x double>
   br label %end
 
 end:
@@ -796,34 +3686,151 @@ end:
   ret <6 x double> %phi
 }
 
-define <12 x float> @bitcast_v6f64_to_v12f32(<6 x double> %a, i32 %b) {
-; GCN-LABEL: bitcast_v6f64_to_v12f32:
+define <24 x half> @bitcast_v6f64_to_v24f16(<6 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6f64_to_v24f16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:  .LBB22_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB7_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    s_cbranch_execz .LBB22_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
 ; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:  .LBB22_4: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v0, v29
+; GCN-NEXT:    v_mov_b32_e32 v1, v35
+; GCN-NEXT:    v_mov_b32_e32 v2, v24
+; GCN-NEXT:    v_mov_b32_e32 v3, v34
+; GCN-NEXT:    v_mov_b32_e32 v4, v25
+; GCN-NEXT:    v_mov_b32_e32 v5, v33
+; GCN-NEXT:    v_mov_b32_e32 v6, v26
+; GCN-NEXT:    v_mov_b32_e32 v7, v32
+; GCN-NEXT:    v_mov_b32_e32 v8, v27
+; GCN-NEXT:    v_mov_b32_e32 v9, v31
+; GCN-NEXT:    v_mov_b32_e32 v10, v28
+; GCN-NEXT:    v_mov_b32_e32 v11, v30
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v6f64_to_v12f32:
+; VI-LABEL: bitcast_v6f64_to_v24f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:    s_cbranch_execz .LBB22_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
 ; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
@@ -831,18 +3838,18 @@ define <12 x float> @bitcast_v6f64_to_v12f32(<6 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:  .LBB22_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v6f64_to_v12f32:
+; GFX9-LABEL: bitcast_v6f64_to_v24f16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
 ; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
@@ -850,11 +3857,11 @@ define <12 x float> @bitcast_v6f64_to_v12f32(<6 x double> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:  .LBB7_2: ; %end
+; GFX9-NEXT:  .LBB22_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v6f64_to_v12f32:
+; GFX11-LABEL: bitcast_v6f64_to_v24f16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -862,7 +3869,7 @@ define <12 x float> @bitcast_v6f64_to_v12f32(<6 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
 ; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
@@ -870,7 +3877,7 @@ define <12 x float> @bitcast_v6f64_to_v12f32(<6 x double> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
 ; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:  .LBB22_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -878,95 +3885,713 @@ define <12 x float> @bitcast_v6f64_to_v12f32(<6 x double> %a, i32 %b) {
 
 cmp.true:
   %a1 = fadd <6 x double> %a, splat (double 1.000000e+00)
-  %a2 = bitcast <6 x double> %a1 to <12 x float>
+  %a2 = bitcast <6 x double> %a1 to <24 x half>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <6 x double> %a to <12 x float>
+  %a3 = bitcast <6 x double> %a to <24 x half>
   br label %end
 
 end:
-  %phi = phi <12 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <12 x float> %phi
+  %phi = phi <24 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x half> %phi
 }
 
-define <6 x i64> @bitcast_v12f32_to_v6i64(<12 x float> %a, i32 %b) {
-; GCN-LABEL: bitcast_v12f32_to_v6i64:
+define <6 x double> @bitcast_v24f16_to_v6f64(<24 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f16_to_v6f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v22
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB23_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB23_4
+; GCN-NEXT:  .LBB23_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB23_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v0, v49, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v39, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v2, v34, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v30, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v28, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v27, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v25, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v24, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v18, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v17, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v16, v11
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  .LBB23_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v39
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v8, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v9, v18, v20
+; GCN-NEXT:    v_or_b32_e32 v10, v17, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v16, v19
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24f16_to_v6f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v12, 0x200
+; VI-NEXT:    v_add_f16_sdwa v13, v11, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v10, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v9, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v7, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v6, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v5, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v3, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v2, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v1, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v12, v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v13
+; VI-NEXT:    v_or_b32_e32 v0, v0, v12
+; VI-NEXT:  .LBB23_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24f16_to_v6f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24f16_to_v6f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <24 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <24 x half> %a1 to <6 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x half> %a to <6 x double>
+  br label %end
+
+end:
+  %phi = phi <6 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <6 x double> %phi
+}
+
+define <24 x i16> @bitcast_v6i64_to_v24i16(<6 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6i64_to_v24i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v22, v11
+; GCN-NEXT:    v_mov_b32_e32 v20, v10
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v14, v7
+; GCN-NEXT:    v_mov_b32_e32 v24, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v12, v24
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v6i64_to_v24i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB24_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v6i64_to_v24i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB24_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v6i64_to_v24i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB24_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <6 x i64> %a, splat (i64 3)
+  %a2 = bitcast <6 x i64> %a1 to <24 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <6 x i64> %a to <24 x i16>
+  br label %end
+
+end:
+  %phi = phi <24 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i16> %phi
+}
+
+define <6 x i64> @bitcast_v24i16_to_v6i64(<24 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i16_to_v6i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_mov_b32_e32 v32, v14
+; GCN-NEXT:    v_mov_b32_e32 v31, v12
+; GCN-NEXT:    v_mov_b32_e32 v30, v10
+; GCN-NEXT:    v_mov_b32_e32 v29, v8
+; GCN-NEXT:    v_mov_b32_e32 v28, v6
+; GCN-NEXT:    v_mov_b32_e32 v25, v4
+; GCN-NEXT:    v_mov_b32_e32 v26, v2
+; GCN-NEXT:    v_mov_b32_e32 v27, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB25_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB8_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
-; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB25_4
+; GCN-NEXT:  .LBB25_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB25_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v38
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v39
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v30
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v31
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v22
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v24
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v33
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v34
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v36
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v37
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v23
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  .LBB25_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v27
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v22
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v39, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v24, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v33, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v34, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v35, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v36, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v37, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v11
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 0x30000, v11
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v12f32_to_v6i64:
+; VI-LABEL: bitcast_v24i16_to_v6i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:    s_cbranch_execz .LBB25_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
-; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; VI-NEXT:  .LBB8_2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v13, 3
+; VI-NEXT:    v_add_u16_e32 v12, 3, v11
+; VI-NEXT:    v_add_u16_sdwa v11, v11, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_u16_e32 v12, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v12, v10
+; VI-NEXT:    v_add_u16_e32 v12, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v12, v9
+; VI-NEXT:    v_add_u16_e32 v12, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v12, v8
+; VI-NEXT:    v_add_u16_e32 v12, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v12, v7
+; VI-NEXT:    v_add_u16_e32 v12, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v12, v6
+; VI-NEXT:    v_add_u16_e32 v12, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v12, v5
+; VI-NEXT:    v_add_u16_e32 v12, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v12, v4
+; VI-NEXT:    v_add_u16_e32 v12, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v12, v3
+; VI-NEXT:    v_add_u16_e32 v12, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v12, v2
+; VI-NEXT:    v_add_u16_e32 v12, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v12, v1
+; VI-NEXT:    v_add_u16_e32 v12, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v12, v0
+; VI-NEXT:  .LBB25_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v12f32_to_v6i64:
+; GFX9-LABEL: bitcast_v24i16_to_v6i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
-; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
-; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
-; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
-; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
-; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
-; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
-; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
-; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
-; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
-; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
-; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
-; GFX9-NEXT:  .LBB8_2: ; %end
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v12f32_to_v6i64:
+; GFX11-LABEL: bitcast_v24i16_to_v6i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -974,26 +4599,33 @@ define <6 x i64> @bitcast_v12f32_to_v6i64(<12 x float> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
-; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
-; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
-; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
-; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
-; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
-; GFX11-NEXT:  ; %bb.2: ; %end
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <12 x float> %a, splat (float 1.000000e+00)
-  %a2 = bitcast <12 x float> %a1 to <6 x i64>
+  %a1 = add <24 x i16> %a, splat (i16 3)
+  %a2 = bitcast <24 x i16> %a1 to <6 x i64>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <12 x float> %a to <6 x i64>
+  %a3 = bitcast <24 x i16> %a to <6 x i64>
   br label %end
 
 end:
@@ -1001,40 +4633,167 @@ end:
   ret <6 x i64> %phi
 }
 
-define <12 x float> @bitcast_v6i64_to_v12f32(<6 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v6i64_to_v12f32:
+define <24 x half> @bitcast_v6i64_to_v24f16(<6 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v6i64_to_v24f16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v26, v11
+; GCN-NEXT:    v_mov_b32_e32 v25, v10
+; GCN-NEXT:    v_mov_b32_e32 v28, v9
+; GCN-NEXT:    v_mov_b32_e32 v27, v8
+; GCN-NEXT:    v_mov_b32_e32 v30, v7
+; GCN-NEXT:    v_mov_b32_e32 v29, v6
+; GCN-NEXT:    v_mov_b32_e32 v32, v5
+; GCN-NEXT:    v_mov_b32_e32 v31, v4
+; GCN-NEXT:    v_mov_b32_e32 v34, v3
+; GCN-NEXT:    v_mov_b32_e32 v33, v2
+; GCN-NEXT:    v_mov_b32_e32 v35, v1
+; GCN-NEXT:    v_mov_b32_e32 v24, v0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB26_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB9_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:  .LBB9_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB26_4
+; GCN-NEXT:  .LBB26_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB26_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v33
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v35
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  .LBB26_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v35, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v33
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v34, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v31
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v32, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v29
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v30, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v27
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v28, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v25
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v26, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v24
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v6i64_to_v12f32:
+; VI-LABEL: bitcast_v6i64_to_v24f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:    s_cbranch_execz .LBB26_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
 ; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
@@ -1048,18 +4807,18 @@ define <12 x float> @bitcast_v6i64_to_v12f32(<6 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:  .LBB26_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v6i64_to_v12f32:
+; GFX9-LABEL: bitcast_v6i64_to_v24f16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
 ; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
@@ -1073,11 +4832,11 @@ define <12 x float> @bitcast_v6i64_to_v12f32(<6 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:  .LBB9_2: ; %end
+; GFX9-NEXT:  .LBB26_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v6i64_to_v12f32:
+; GFX11-LABEL: bitcast_v6i64_to_v24f16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -1085,7 +4844,7 @@ define <12 x float> @bitcast_v6i64_to_v12f32(<6 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1102,7 +4861,7 @@ define <12 x float> @bitcast_v6i64_to_v12f32(<6 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:  .LBB26_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -1110,77 +4869,285 @@ define <12 x float> @bitcast_v6i64_to_v12f32(<6 x i64> %a, i32 %b) {
 
 cmp.true:
   %a1 = add <6 x i64> %a, splat (i64 3)
-  %a2 = bitcast <6 x i64> %a1 to <12 x float>
+  %a2 = bitcast <6 x i64> %a1 to <24 x half>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <6 x i64> %a to <12 x float>
+  %a3 = bitcast <6 x i64> %a to <24 x half>
   br label %end
 
 end:
-  %phi = phi <12 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <12 x float> %phi
+  %phi = phi <24 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x half> %phi
 }
 
-define <6 x i64> @bitcast_v6f64_to_v6i64(<6 x double> %a, i32 %b) {
-; GCN-LABEL: bitcast_v6f64_to_v6i64:
+define <6 x i64> @bitcast_v24f16_to_v6i64(<24 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f16_to_v6i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v22
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB27_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB10_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_cbranch_execnz .LBB27_4
+; GCN-NEXT:  .LBB27_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB27_3: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v0, v49, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v39, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v2, v34, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v30, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v28, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v27, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v25, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v24, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v18, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v17, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v16, v11
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  .LBB27_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v39
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v8, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v9, v18, v20
+; GCN-NEXT:    v_or_b32_e32 v10, v17, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v16, v19
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v6f64_to_v6i64:
+; VI-LABEL: bitcast_v24f16_to_v6i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:    s_cbranch_execz .LBB27_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v12, 0x200
+; VI-NEXT:    v_add_f16_sdwa v13, v11, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v10, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v9, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v7, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v6, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v5, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v3, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v2, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v1, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v12, v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v13
+; VI-NEXT:    v_or_b32_e32 v0, v0, v12
+; VI-NEXT:  .LBB27_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v6f64_to_v6i64:
+; GFX9-LABEL: bitcast_v24f16_to_v6i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v6f64_to_v6i64:
+; GFX11-LABEL: bitcast_v24f16_to_v6i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -1188,27 +5155,33 @@ define <6 x i64> @bitcast_v6f64_to_v6i64(<6 x double> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <6 x double> %a, splat (double 1.000000e+00)
-  %a2 = bitcast <6 x double> %a1 to <6 x i64>
+  %a1 = fadd <24 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <24 x half> %a1 to <6 x i64>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <6 x double> %a to <6 x i64>
+  %a3 = bitcast <24 x half> %a to <6 x i64>
   br label %end
 
 end:
@@ -1216,83 +5189,505 @@ end:
   ret <6 x i64> %phi
 }
 
-define <6 x double> @bitcast_v6i64_to_v6f64(<6 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v6i64_to_v6f64:
+define <24 x half> @bitcast_v24i16_to_v24f16(<24 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i16_to_v24f16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v55, v23
+; GCN-NEXT:    v_mov_b32_e32 v54, v22
+; GCN-NEXT:    v_mov_b32_e32 v53, v21
+; GCN-NEXT:    v_mov_b32_e32 v52, v20
+; GCN-NEXT:    v_mov_b32_e32 v51, v19
+; GCN-NEXT:    v_mov_b32_e32 v50, v18
+; GCN-NEXT:    v_mov_b32_e32 v49, v17
+; GCN-NEXT:    v_mov_b32_e32 v48, v16
+; GCN-NEXT:    v_mov_b32_e32 v39, v15
+; GCN-NEXT:    v_mov_b32_e32 v38, v14
+; GCN-NEXT:    v_mov_b32_e32 v37, v13
+; GCN-NEXT:    v_mov_b32_e32 v36, v12
+; GCN-NEXT:    v_mov_b32_e32 v35, v11
+; GCN-NEXT:    v_mov_b32_e32 v34, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v9
+; GCN-NEXT:    v_mov_b32_e32 v32, v8
+; GCN-NEXT:    v_mov_b32_e32 v31, v7
+; GCN-NEXT:    v_mov_b32_e32 v30, v6
+; GCN-NEXT:    v_mov_b32_e32 v29, v5
+; GCN-NEXT:    v_mov_b32_e32 v28, v4
+; GCN-NEXT:    v_mov_b32_e32 v27, v3
+; GCN-NEXT:    v_mov_b32_e32 v26, v2
+; GCN-NEXT:    v_mov_b32_e32 v25, v1
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v40, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:  .LBB28_2: ; %Flow
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v55
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24i16_to_v24f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v12, 3
+; VI-NEXT:    v_add_u16_sdwa v19, v11, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v10, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v9, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v13, v0, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v14, v1, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v15, v2, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v16, v3, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v17, v4, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v18, v5, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v6, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v12, v7, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_or_b32_e32 v7, v7, v12
+; VI-NEXT:    v_or_b32_e32 v6, v6, v19
+; VI-NEXT:    v_or_b32_e32 v5, v5, v18
+; VI-NEXT:    v_or_b32_e32 v4, v4, v17
+; VI-NEXT:    v_or_b32_e32 v3, v3, v16
+; VI-NEXT:    v_or_b32_e32 v2, v2, v15
+; VI-NEXT:    v_or_b32_e32 v1, v1, v14
+; VI-NEXT:    v_or_b32_e32 v0, v0, v13
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24i16_to_v24f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24i16_to_v24f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v12
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <24 x i16> %a, splat (i16 3)
+  %a2 = bitcast <24 x i16> %a1 to <24 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x i16> %a to <24 x half>
+  br label %end
+
+end:
+  %phi = phi <24 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x half> %phi
+}
+
+define <24 x i16> @bitcast_v24f16_to_v24i16(<24 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f16_to_v24i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v9
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v17
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v21
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v21, 16
+; GCN-NEXT:  .LBB29_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v6i64_to_v6f64:
+; VI-LABEL: bitcast_v24f16_to_v24i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:    s_cbranch_execz .LBB29_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    v_mov_b32_e32 v13, 0x200
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v11, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v19, v11
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v19, v10
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v9
+; VI-NEXT:    v_add_f16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v19, v9
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v8, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v0
+; VI-NEXT:    v_add_f16_sdwa v0, v0, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v1, v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v2, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v3
+; VI-NEXT:    v_add_f16_sdwa v3, v3, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v4
+; VI-NEXT:    v_add_f16_sdwa v4, v4, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v5
+; VI-NEXT:    v_add_f16_sdwa v5, v5, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v19, v8
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v6, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_sdwa v13, v7, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v13
+; VI-NEXT:    v_or_b32_e32 v6, v19, v6
+; VI-NEXT:    v_or_b32_e32 v5, v18, v5
+; VI-NEXT:    v_or_b32_e32 v4, v17, v4
+; VI-NEXT:    v_or_b32_e32 v3, v16, v3
+; VI-NEXT:    v_or_b32_e32 v2, v15, v2
+; VI-NEXT:    v_or_b32_e32 v1, v14, v1
+; VI-NEXT:    v_or_b32_e32 v0, v12, v0
+; VI-NEXT:  .LBB29_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v6i64_to_v6f64:
+; GFX9-LABEL: bitcast_v24f16_to_v24i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB29_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v6i64_to_v6f64:
+; GFX11-LABEL: bitcast_v24f16_to_v24i16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -1300,39 +5695,36 @@ define <6 x double> @bitcast_v6i64_to_v6f64(<6 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB29_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <6 x i64> %a, splat (i64 3)
-  %a2 = bitcast <6 x i64> %a1 to <6 x double>
+  %a1 = fadd <24 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <24 x half> %a1 to <24 x i16>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <6 x i64> %a to <6 x double>
+  %a3 = bitcast <24 x half> %a to <24 x i16>
   br label %end
 
 end:
-  %phi = phi <6 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <6 x double> %phi
+  %phi = phi <24 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i16> %phi
 }
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll
index b0fa6a21cd5f1..4b2b9560e5927 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll
@@ -5,8 +5,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
 
-define <14 x i32> @bitcast_v7i64_to_v14i32(<7 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v7i64_to_v14i32:
+define <14 x float> @bitcast_v14i32_to_v14f32(<14 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i32_to_v14f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
@@ -15,25 +15,25 @@ define <14 x i32> @bitcast_v7i64_to_v14i32(<7 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:  .LBB0_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v7i64_to_v14i32:
+; VI-LABEL: bitcast_v14i32_to_v14f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
@@ -42,25 +42,25 @@ define <14 x i32> @bitcast_v7i64_to_v14i32(<7 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB0_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
-; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:  .LBB0_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v7i64_to_v14i32:
+; GFX9-LABEL: bitcast_v14i32_to_v14f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
@@ -69,25 +69,25 @@ define <14 x i32> @bitcast_v7i64_to_v14i32(<7 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
 ; GFX9-NEXT:  .LBB0_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v7i64_to_v14i32:
+; GFX11-LABEL: bitcast_v14i32_to_v14f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -97,24 +97,20 @@ define <14 x i32> @bitcast_v7i64_to_v14i32(<7 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
 ; GFX11-NEXT:  .LBB0_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -122,12 +118,131 @@ define <14 x i32> @bitcast_v7i64_to_v14i32(<7 x i64> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <7 x i64> %a, splat (i64 3)
-  %a2 = bitcast <7 x i64> %a1 to <14 x i32>
+  %a1 = add <14 x i32> %a, splat (i32 3)
+  %a2 = bitcast <14 x i32> %a1 to <14 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <7 x i64> %a to <14 x i32>
+  %a3 = bitcast <14 x i32> %a to <14 x float>
+  br label %end
+
+end:
+  %phi = phi <14 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x float> %phi
+}
+
+define <14 x i32> @bitcast_v14f32_to_v14i32(<14 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f32_to_v14i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f32_to_v14i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f32_to_v14i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f32_to_v14i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <14 x float> %a1 to <14 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x float> %a to <14 x i32>
   br label %end
 
 end:
@@ -143,7 +258,7 @@ define <7 x i64> @bitcast_v14i32_to_v7i64(<14 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
@@ -159,7 +274,7 @@ define <7 x i64> @bitcast_v14i32_to_v7i64(<14 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:  .LBB2_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -170,7 +285,7 @@ define <7 x i64> @bitcast_v14i32_to_v7i64(<14 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:    s_cbranch_execz .LBB2_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
@@ -186,7 +301,7 @@ define <7 x i64> @bitcast_v14i32_to_v7i64(<14 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:  .LBB2_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -197,7 +312,7 @@ define <7 x i64> @bitcast_v14i32_to_v7i64(<14 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
 ; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
@@ -213,7 +328,7 @@ define <7 x i64> @bitcast_v14i32_to_v7i64(<14 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:  .LBB2_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -225,7 +340,7 @@ define <7 x i64> @bitcast_v14i32_to_v7i64(<14 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
 ; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
@@ -241,7 +356,7 @@ define <7 x i64> @bitcast_v14i32_to_v7i64(<14 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:  .LBB2_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -261,89 +376,89 @@ end:
   ret <7 x i64> %phi
 }
 
-define <7 x double> @bitcast_v7i64_to_v7f64(<7 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v7i64_to_v7f64:
+define <14 x i32> @bitcast_v7i64_to_v14i32(<7 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7i64_to_v14i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB2_2
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
-; GCN-NEXT:  .LBB2_2: ; %end
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB3_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v7i64_to_v7f64:
+; VI-LABEL: bitcast_v7i64_to_v14i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB2_2
+; VI-NEXT:    s_cbranch_execz .LBB3_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
 ; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
-; VI-NEXT:  .LBB2_2: ; %end
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB3_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v7i64_to_v7f64:
+; GFX9-LABEL: bitcast_v7i64_to_v14i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
 ; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:  .LBB2_2: ; %end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: bitcast_v7i64_to_v7f64:
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7i64_to_v14i32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -351,124 +466,26 @@ define <7 x double> @bitcast_v7i64_to_v7f64(<7 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:  .LBB2_2: ; %end
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
-  %cmp = icmp eq i32 %b, 0
-  br i1 %cmp, label %cmp.true, label %cmp.false
-
-cmp.true:
-  %a1 = add <7 x i64> %a, splat (i64 3)
-  %a2 = bitcast <7 x i64> %a1 to <7 x double>
-  br label %end
-
-cmp.false:
-  %a3 = bitcast <7 x i64> %a to <7 x double>
-  br label %end
-
-end:
-  %phi = phi <7 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <7 x double> %phi
-}
-
-define <7 x i64> @bitcast_v7f64_to_v7i64(<7 x double> %a, i32 %b) {
-; GCN-LABEL: bitcast_v7f64_to_v7i64:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
-; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB3_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
-; GCN-NEXT:  .LBB3_2: ; %end
-; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_setpc_b64 s[30:31]
-;
-; VI-LABEL: bitcast_v7f64_to_v7i64:
-; VI:       ; %bb.0:
-; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
-; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB3_2
-; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
-; VI-NEXT:  .LBB3_2: ; %end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: bitcast_v7f64_to_v7i64:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
-; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB3_2
-; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
-; GFX9-NEXT:  .LBB3_2: ; %end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: bitcast_v7f64_to_v7i64:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
-; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB3_2
-; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
-; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
-; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
-; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
-; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX11-NEXT:  .LBB3_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -476,17 +493,17 @@ define <7 x i64> @bitcast_v7f64_to_v7i64(<7 x double> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = fadd <7 x double> %a, splat (double 1.000000e+00)
-  %a2 = bitcast <7 x double> %a1 to <7 x i64>
+  %a1 = add <7 x i64> %a, splat (i64 3)
+  %a2 = bitcast <7 x i64> %a1 to <14 x i32>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <7 x double> %a to <7 x i64>
+  %a3 = bitcast <7 x i64> %a to <14 x i32>
   br label %end
 
 end:
-  %phi = phi <7 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <7 x i64> %phi
+  %phi = phi <14 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i32> %phi
 }
 
 define <7 x double> @bitcast_v14i32_to_v7f64(<14 x i32> %a, i32 %b) {
@@ -712,3 +729,5610 @@ end:
   %phi = phi <14 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <14 x i32> %phi
 }
+
+define <28 x i16> @bitcast_v14i32_to_v28i16(<14 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i32_to_v28i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v26, v13
+; GCN-NEXT:    v_mov_b32_e32 v24, v12
+; GCN-NEXT:    v_mov_b32_e32 v22, v11
+; GCN-NEXT:    v_mov_b32_e32 v20, v10
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v28, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v25, v26, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v28, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_alignbit_b32 v25, v26, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v28, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v14, v28
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i32_to_v28i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB6_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i32_to_v28i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB6_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i32_to_v28i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB6_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i32> %a, splat (i32 3)
+  %a2 = bitcast <14 x i32> %a1 to <28 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i32> %a to <28 x i16>
+  br label %end
+
+end:
+  %phi = phi <28 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i16> %phi
+}
+
+define <14 x i32> @bitcast_v28i16_to_v14i32(<28 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i16_to_v14i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v36, v14
+; GCN-NEXT:    v_mov_b32_e32 v35, v12
+; GCN-NEXT:    v_mov_b32_e32 v34, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v8
+; GCN-NEXT:    v_mov_b32_e32 v32, v6
+; GCN-NEXT:    v_mov_b32_e32 v31, v4
+; GCN-NEXT:    v_mov_b32_e32 v30, v2
+; GCN-NEXT:    v_mov_b32_e32 v29, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB7_4
+; GCN-NEXT:  .LBB7_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v30
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v31
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v28
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v37
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v38
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v48
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v50
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v51
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v25
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v27
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  .LBB7_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v29
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v26
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_or_b32_e32 v0, v28, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v37, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v38, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v39, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v48, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v49, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v50, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v51, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v25, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v27, v13
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 0x30000, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x30000, v13
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28i16_to_v14i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v15, 3
+; VI-NEXT:    v_add_u16_e32 v14, 3, v13
+; VI-NEXT:    v_add_u16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_u16_e32 v14, 3, v12
+; VI-NEXT:    v_add_u16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v14, v12
+; VI-NEXT:    v_add_u16_e32 v14, 3, v11
+; VI-NEXT:    v_add_u16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v14, v11
+; VI-NEXT:    v_add_u16_e32 v14, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v14, v10
+; VI-NEXT:    v_add_u16_e32 v14, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v14, v9
+; VI-NEXT:    v_add_u16_e32 v14, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v14, v8
+; VI-NEXT:    v_add_u16_e32 v14, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v14, v7
+; VI-NEXT:    v_add_u16_e32 v14, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v14, v6
+; VI-NEXT:    v_add_u16_e32 v14, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v14, v5
+; VI-NEXT:    v_add_u16_e32 v14, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v14, v4
+; VI-NEXT:    v_add_u16_e32 v14, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v14, v3
+; VI-NEXT:    v_add_u16_e32 v14, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v14, v2
+; VI-NEXT:    v_add_u16_e32 v14, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v14, v1
+; VI-NEXT:    v_add_u16_e32 v14, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v14, v0
+; VI-NEXT:  .LBB7_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28i16_to_v14i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28i16_to_v14i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <28 x i16> %a, splat (i16 3)
+  %a2 = bitcast <28 x i16> %a1 to <14 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x i16> %a to <14 x i32>
+  br label %end
+
+end:
+  %phi = phi <14 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i32> %phi
+}
+
+define <28 x half> @bitcast_v14i32_to_v28f16(<14 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i32_to_v28f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v29, v13
+; GCN-NEXT:    v_mov_b32_e32 v30, v12
+; GCN-NEXT:    v_mov_b32_e32 v31, v11
+; GCN-NEXT:    v_mov_b32_e32 v32, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v9
+; GCN-NEXT:    v_mov_b32_e32 v34, v8
+; GCN-NEXT:    v_mov_b32_e32 v35, v7
+; GCN-NEXT:    v_mov_b32_e32 v36, v6
+; GCN-NEXT:    v_mov_b32_e32 v37, v5
+; GCN-NEXT:    v_mov_b32_e32 v38, v4
+; GCN-NEXT:    v_mov_b32_e32 v39, v3
+; GCN-NEXT:    v_mov_b32_e32 v48, v2
+; GCN-NEXT:    v_mov_b32_e32 v49, v1
+; GCN-NEXT:    v_mov_b32_e32 v28, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB8_4
+; GCN-NEXT:  .LBB8_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB8_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v33
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v35
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v36
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v37
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v38
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v39
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v48
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v49
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  .LBB8_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i32_to_v28f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB8_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i32_to_v28f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB8_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i32_to_v28f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB8_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i32> %a, splat (i32 3)
+  %a2 = bitcast <14 x i32> %a1 to <28 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i32> %a to <28 x half>
+  br label %end
+
+end:
+  %phi = phi <28 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x half> %phi
+}
+
+define <14 x i32> @bitcast_v28f16_to_v14i32(<28 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f16_to_v14i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v26
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v51, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v49, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v39, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v37, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v35, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v33, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v32, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v30, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v29, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v28, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v19, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v16, v13
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:  .LBB9_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v8, v24, v22
+; GCN-NEXT:    v_or_b32_e32 v9, v26, v25
+; GCN-NEXT:    v_or_b32_e32 v10, v19, v27
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v21
+; GCN-NEXT:    v_or_b32_e32 v13, v16, v20
+; GCN-NEXT:  .LBB9_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f16_to_v14i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v14, 0x200
+; VI-NEXT:    v_add_f16_sdwa v15, v13, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_or_b32_e32 v13, v13, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v12, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_or_b32_e32 v12, v12, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v11, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v10, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v9, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v8, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v7, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v6, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v5, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v4, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v3, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v2, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v1, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v14, v0, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v15
+; VI-NEXT:    v_or_b32_e32 v0, v0, v14
+; VI-NEXT:  .LBB9_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f16_to_v14i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f16_to_v14i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <28 x half> %a1 to <14 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x half> %a to <14 x i32>
+  br label %end
+
+end:
+  %phi = phi <14 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i32> %phi
+}
+
+define <7 x i64> @bitcast_v14f32_to_v7i64(<14 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f32_to_v7i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f32_to_v7i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f32_to_v7i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f32_to_v7i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <14 x float> %a1 to <7 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x float> %a to <7 x i64>
+  br label %end
+
+end:
+  %phi = phi <7 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x i64> %phi
+}
+
+define <14 x float> @bitcast_v7i64_to_v14f32(<7 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7i64_to_v14f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7i64_to_v14f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7i64_to_v14f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7i64_to_v14f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <7 x i64> %a, splat (i64 3)
+  %a2 = bitcast <7 x i64> %a1 to <14 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x i64> %a to <14 x float>
+  br label %end
+
+end:
+  %phi = phi <14 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x float> %phi
+}
+
+define <7 x double> @bitcast_v14f32_to_v7f64(<14 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f32_to_v7f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f32_to_v7f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f32_to_v7f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f32_to_v7f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB12_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <14 x float> %a1 to <7 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x float> %a to <7 x double>
+  br label %end
+
+end:
+  %phi = phi <7 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x double> %phi
+}
+
+define <14 x float> @bitcast_v7f64_to_v14f32(<7 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7f64_to_v14f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7f64_to_v14f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7f64_to_v14f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7f64_to_v14f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <7 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <7 x double> %a1 to <14 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x double> %a to <14 x float>
+  br label %end
+
+end:
+  %phi = phi <14 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x float> %phi
+}
+
+define <28 x i16> @bitcast_v14f32_to_v28i16(<14 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f32_to_v28i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v26, v13
+; GCN-NEXT:    v_mov_b32_e32 v24, v12
+; GCN-NEXT:    v_mov_b32_e32 v22, v11
+; GCN-NEXT:    v_mov_b32_e32 v20, v10
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v28, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v25, v26, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v28, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_alignbit_b32 v25, v26, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v28, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v14, v28
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f32_to_v28i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB14_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f32_to_v28i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB14_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f32_to_v28i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB14_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <14 x float> %a1 to <28 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x float> %a to <28 x i16>
+  br label %end
+
+end:
+  %phi = phi <28 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i16> %phi
+}
+
+define <14 x float> @bitcast_v28i16_to_v14f32(<28 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i16_to_v14f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v36, v14
+; GCN-NEXT:    v_mov_b32_e32 v35, v12
+; GCN-NEXT:    v_mov_b32_e32 v34, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v8
+; GCN-NEXT:    v_mov_b32_e32 v32, v6
+; GCN-NEXT:    v_mov_b32_e32 v31, v4
+; GCN-NEXT:    v_mov_b32_e32 v30, v2
+; GCN-NEXT:    v_mov_b32_e32 v29, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB15_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB15_4
+; GCN-NEXT:  .LBB15_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB15_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v30
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v31
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v28
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v37
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v38
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v48
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v50
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v51
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v25
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v27
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  .LBB15_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v29
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v26
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_or_b32_e32 v0, v28, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v37, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v38, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v39, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v48, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v49, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v50, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v51, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v25, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v27, v13
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 0x30000, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x30000, v13
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28i16_to_v14f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v15, 3
+; VI-NEXT:    v_add_u16_e32 v14, 3, v13
+; VI-NEXT:    v_add_u16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_u16_e32 v14, 3, v12
+; VI-NEXT:    v_add_u16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v14, v12
+; VI-NEXT:    v_add_u16_e32 v14, 3, v11
+; VI-NEXT:    v_add_u16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v14, v11
+; VI-NEXT:    v_add_u16_e32 v14, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v14, v10
+; VI-NEXT:    v_add_u16_e32 v14, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v14, v9
+; VI-NEXT:    v_add_u16_e32 v14, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v14, v8
+; VI-NEXT:    v_add_u16_e32 v14, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v14, v7
+; VI-NEXT:    v_add_u16_e32 v14, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v14, v6
+; VI-NEXT:    v_add_u16_e32 v14, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v14, v5
+; VI-NEXT:    v_add_u16_e32 v14, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v14, v4
+; VI-NEXT:    v_add_u16_e32 v14, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v14, v3
+; VI-NEXT:    v_add_u16_e32 v14, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v14, v2
+; VI-NEXT:    v_add_u16_e32 v14, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v14, v1
+; VI-NEXT:    v_add_u16_e32 v14, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v14, v0
+; VI-NEXT:  .LBB15_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28i16_to_v14f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28i16_to_v14f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <28 x i16> %a, splat (i16 3)
+  %a2 = bitcast <28 x i16> %a1 to <14 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x i16> %a to <14 x float>
+  br label %end
+
+end:
+  %phi = phi <14 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x float> %phi
+}
+
+define <28 x half> @bitcast_v14f32_to_v28f16(<14 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f32_to_v28f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v29, v13
+; GCN-NEXT:    v_mov_b32_e32 v30, v12
+; GCN-NEXT:    v_mov_b32_e32 v31, v11
+; GCN-NEXT:    v_mov_b32_e32 v32, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v9
+; GCN-NEXT:    v_mov_b32_e32 v34, v8
+; GCN-NEXT:    v_mov_b32_e32 v35, v7
+; GCN-NEXT:    v_mov_b32_e32 v36, v6
+; GCN-NEXT:    v_mov_b32_e32 v37, v5
+; GCN-NEXT:    v_mov_b32_e32 v38, v4
+; GCN-NEXT:    v_mov_b32_e32 v39, v3
+; GCN-NEXT:    v_mov_b32_e32 v48, v2
+; GCN-NEXT:    v_mov_b32_e32 v49, v1
+; GCN-NEXT:    v_mov_b32_e32 v28, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB16_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB16_4
+; GCN-NEXT:  .LBB16_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB16_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v33
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v35
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v36
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v37
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v38
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v39
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v48
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v49
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  .LBB16_4: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v49
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v48
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v39
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v38
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v37
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v36
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v35
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v34
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v33
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v32
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v31
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v30
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f32_to_v28f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB16_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f32_to_v28f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB16_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f32_to_v28f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB16_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <14 x float> %a1 to <28 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x float> %a to <28 x half>
+  br label %end
+
+end:
+  %phi = phi <28 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x half> %phi
+}
+
+define <14 x float> @bitcast_v28f16_to_v14f32(<28 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f16_to_v14f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v26
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v51, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v49, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v39, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v37, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v35, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v33, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v32, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v30, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v29, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v28, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v19, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v16, v13
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v8, v24, v22
+; GCN-NEXT:    v_or_b32_e32 v9, v26, v25
+; GCN-NEXT:    v_or_b32_e32 v10, v19, v27
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v21
+; GCN-NEXT:    v_or_b32_e32 v13, v16, v20
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f16_to_v14f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v14, 0x200
+; VI-NEXT:    v_add_f16_sdwa v15, v13, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_or_b32_e32 v13, v13, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v12, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_or_b32_e32 v12, v12, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v11, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v10, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v9, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v8, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v7, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v6, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v5, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v4, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v3, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v2, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v1, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v14, v0, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v15
+; VI-NEXT:    v_or_b32_e32 v0, v0, v14
+; VI-NEXT:  .LBB17_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f16_to_v14f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f16_to_v14f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <28 x half> %a1 to <14 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x half> %a to <14 x float>
+  br label %end
+
+end:
+  %phi = phi <14 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x float> %phi
+}
+
+define <7 x double> @bitcast_v7i64_to_v7f64(<7 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7i64_to_v7f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7i64_to_v7f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7i64_to_v7f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7i64_to_v7f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <7 x i64> %a, splat (i64 3)
+  %a2 = bitcast <7 x i64> %a1 to <7 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x i64> %a to <7 x double>
+  br label %end
+
+end:
+  %phi = phi <7 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x double> %phi
+}
+
+define <7 x i64> @bitcast_v7f64_to_v7i64(<7 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7f64_to_v7i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7f64_to_v7i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7f64_to_v7i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7f64_to_v7i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <7 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <7 x double> %a1 to <7 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x double> %a to <7 x i64>
+  br label %end
+
+end:
+  %phi = phi <7 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x i64> %phi
+}
+
+define <28 x i16> @bitcast_v7i64_to_v28i16(<7 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7i64_to_v28i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v26, v13
+; GCN-NEXT:    v_mov_b32_e32 v24, v12
+; GCN-NEXT:    v_mov_b32_e32 v22, v11
+; GCN-NEXT:    v_mov_b32_e32 v20, v10
+; GCN-NEXT:    v_mov_b32_e32 v18, v9
+; GCN-NEXT:    v_mov_b32_e32 v16, v8
+; GCN-NEXT:    v_mov_b32_e32 v28, v7
+; GCN-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NEXT:    v_mov_b32_e32 v6, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v25, v26, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v28, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v28, vcc, 0, v28, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v26, vcc, 0, v26, vcc
+; GCN-NEXT:    v_alignbit_b32 v25, v26, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v16, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v28, v12, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v8, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v4, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v14, v28
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7i64_to_v28i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB20_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7i64_to_v28i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB20_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7i64_to_v28i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB20_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <7 x i64> %a, splat (i64 3)
+  %a2 = bitcast <7 x i64> %a1 to <28 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x i64> %a to <28 x i16>
+  br label %end
+
+end:
+  %phi = phi <28 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i16> %phi
+}
+
+define <7 x i64> @bitcast_v28i16_to_v7i64(<28 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i16_to_v7i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v36, v14
+; GCN-NEXT:    v_mov_b32_e32 v35, v12
+; GCN-NEXT:    v_mov_b32_e32 v34, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v8
+; GCN-NEXT:    v_mov_b32_e32 v32, v6
+; GCN-NEXT:    v_mov_b32_e32 v31, v4
+; GCN-NEXT:    v_mov_b32_e32 v30, v2
+; GCN-NEXT:    v_mov_b32_e32 v29, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB21_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB21_4
+; GCN-NEXT:  .LBB21_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB21_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v30
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v31
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v28
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v37
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v38
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v48
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v50
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v51
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v25
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v27
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  .LBB21_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v29
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v26
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_or_b32_e32 v0, v28, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v37, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v38, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v39, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v48, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v49, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v50, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v51, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v25, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v27, v13
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 0x30000, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x30000, v13
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28i16_to_v7i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v15, 3
+; VI-NEXT:    v_add_u16_e32 v14, 3, v13
+; VI-NEXT:    v_add_u16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_u16_e32 v14, 3, v12
+; VI-NEXT:    v_add_u16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v14, v12
+; VI-NEXT:    v_add_u16_e32 v14, 3, v11
+; VI-NEXT:    v_add_u16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v14, v11
+; VI-NEXT:    v_add_u16_e32 v14, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v14, v10
+; VI-NEXT:    v_add_u16_e32 v14, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v14, v9
+; VI-NEXT:    v_add_u16_e32 v14, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v14, v8
+; VI-NEXT:    v_add_u16_e32 v14, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v14, v7
+; VI-NEXT:    v_add_u16_e32 v14, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v14, v6
+; VI-NEXT:    v_add_u16_e32 v14, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v14, v5
+; VI-NEXT:    v_add_u16_e32 v14, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v14, v4
+; VI-NEXT:    v_add_u16_e32 v14, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v14, v3
+; VI-NEXT:    v_add_u16_e32 v14, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v14, v2
+; VI-NEXT:    v_add_u16_e32 v14, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v14, v1
+; VI-NEXT:    v_add_u16_e32 v14, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v14, v0
+; VI-NEXT:  .LBB21_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28i16_to_v7i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28i16_to_v7i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <28 x i16> %a, splat (i16 3)
+  %a2 = bitcast <28 x i16> %a1 to <7 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x i16> %a to <7 x i64>
+  br label %end
+
+end:
+  %phi = phi <7 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x i64> %phi
+}
+
+define <28 x half> @bitcast_v7i64_to_v28f16(<7 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7i64_to_v28f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v30, v13
+; GCN-NEXT:    v_mov_b32_e32 v29, v12
+; GCN-NEXT:    v_mov_b32_e32 v32, v11
+; GCN-NEXT:    v_mov_b32_e32 v31, v10
+; GCN-NEXT:    v_mov_b32_e32 v34, v9
+; GCN-NEXT:    v_mov_b32_e32 v33, v8
+; GCN-NEXT:    v_mov_b32_e32 v36, v7
+; GCN-NEXT:    v_mov_b32_e32 v35, v6
+; GCN-NEXT:    v_mov_b32_e32 v38, v5
+; GCN-NEXT:    v_mov_b32_e32 v37, v4
+; GCN-NEXT:    v_mov_b32_e32 v48, v3
+; GCN-NEXT:    v_mov_b32_e32 v39, v2
+; GCN-NEXT:    v_mov_b32_e32 v49, v1
+; GCN-NEXT:    v_mov_b32_e32 v28, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB22_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB22_4
+; GCN-NEXT:  .LBB22_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB22_3: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v5, 16, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v33
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v36
+; GCN-NEXT:    v_lshrrev_b32_e32 v13, 16, v35
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v38
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v37
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v48
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v39
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v49
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  .LBB22_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v28
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v49, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v39
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v48, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v37
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v38, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v35
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v36, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v33
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v34, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v31
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v32, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v29
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v30, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7i64_to_v28f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB22_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7i64_to_v28f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB22_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7i64_to_v28f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB22_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <7 x i64> %a, splat (i64 3)
+  %a2 = bitcast <7 x i64> %a1 to <28 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x i64> %a to <28 x half>
+  br label %end
+
+end:
+  %phi = phi <28 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x half> %phi
+}
+
+define <7 x i64> @bitcast_v28f16_to_v7i64(<28 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f16_to_v7i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v26
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v51, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v49, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v39, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v37, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v35, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v33, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v32, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v30, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v29, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v28, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v19, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v16, v13
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:  .LBB23_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v8, v24, v22
+; GCN-NEXT:    v_or_b32_e32 v9, v26, v25
+; GCN-NEXT:    v_or_b32_e32 v10, v19, v27
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v21
+; GCN-NEXT:    v_or_b32_e32 v13, v16, v20
+; GCN-NEXT:  .LBB23_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f16_to_v7i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v14, 0x200
+; VI-NEXT:    v_add_f16_sdwa v15, v13, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_or_b32_e32 v13, v13, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v12, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_or_b32_e32 v12, v12, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v11, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v10, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v9, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v8, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v7, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v6, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v5, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v4, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v3, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v2, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v1, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v14, v0, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v15
+; VI-NEXT:    v_or_b32_e32 v0, v0, v14
+; VI-NEXT:  .LBB23_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f16_to_v7i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f16_to_v7i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <28 x half> %a1 to <7 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x half> %a to <7 x i64>
+  br label %end
+
+end:
+  %phi = phi <7 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x i64> %phi
+}
+
+define <28 x i16> @bitcast_v7f64_to_v28i16(<7 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7f64_to_v28i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v49, v13
+; GCN-NEXT:    v_mov_b32_e32 v48, v12
+; GCN-NEXT:    v_mov_b32_e32 v38, v11
+; GCN-NEXT:    v_mov_b32_e32 v37, v10
+; GCN-NEXT:    v_mov_b32_e32 v36, v9
+; GCN-NEXT:    v_mov_b32_e32 v35, v8
+; GCN-NEXT:    v_mov_b32_e32 v34, v7
+; GCN-NEXT:    v_mov_b32_e32 v33, v6
+; GCN-NEXT:    v_mov_b32_e32 v32, v5
+; GCN-NEXT:    v_mov_b32_e32 v31, v4
+; GCN-NEXT:    v_mov_b32_e32 v30, v3
+; GCN-NEXT:    v_mov_b32_e32 v29, v2
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v25, v49, v48, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v38, v37, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v36, v35, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v34, v33, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v32, v31, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v1, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v49
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v38
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[29:30], v[29:30], 1.0
+; GCN-NEXT:    v_add_f64 v[31:32], v[31:32], 1.0
+; GCN-NEXT:    v_add_f64 v[33:34], v[33:34], 1.0
+; GCN-NEXT:    v_add_f64 v[35:36], v[35:36], 1.0
+; GCN-NEXT:    v_add_f64 v[37:38], v[37:38], 1.0
+; GCN-NEXT:    v_add_f64 v[48:49], v[48:49], 1.0
+; GCN-NEXT:    v_alignbit_b32 v25, v49, v48, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v38, v37, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v36, v35, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v34, v33, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v32, v31, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v1, v0, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v49
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v38
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GCN-NEXT:    v_lshrrev_b32_e32 v11, 16, v32
+; GCN-NEXT:    v_lshrrev_b32_e32 v7, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_mov_b32_e32 v4, v29
+; GCN-NEXT:    v_mov_b32_e32 v6, v30
+; GCN-NEXT:    v_mov_b32_e32 v8, v31
+; GCN-NEXT:    v_mov_b32_e32 v10, v32
+; GCN-NEXT:    v_mov_b32_e32 v12, v33
+; GCN-NEXT:    v_mov_b32_e32 v14, v34
+; GCN-NEXT:    v_mov_b32_e32 v16, v35
+; GCN-NEXT:    v_mov_b32_e32 v18, v36
+; GCN-NEXT:    v_mov_b32_e32 v20, v37
+; GCN-NEXT:    v_mov_b32_e32 v22, v38
+; GCN-NEXT:    v_mov_b32_e32 v24, v48
+; GCN-NEXT:    v_mov_b32_e32 v26, v49
+; GCN-NEXT:    v_mov_b32_e32 v1, v28
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7f64_to_v28i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB24_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7f64_to_v28i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB24_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7f64_to_v28i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB24_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <7 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <7 x double> %a1 to <28 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x double> %a to <28 x i16>
+  br label %end
+
+end:
+  %phi = phi <28 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i16> %phi
+}
+
+define <7 x double> @bitcast_v28i16_to_v7f64(<28 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i16_to_v7f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v36, v14
+; GCN-NEXT:    v_mov_b32_e32 v35, v12
+; GCN-NEXT:    v_mov_b32_e32 v34, v10
+; GCN-NEXT:    v_mov_b32_e32 v33, v8
+; GCN-NEXT:    v_mov_b32_e32 v32, v6
+; GCN-NEXT:    v_mov_b32_e32 v31, v4
+; GCN-NEXT:    v_mov_b32_e32 v30, v2
+; GCN-NEXT:    v_mov_b32_e32 v29, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB25_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB25_4
+; GCN-NEXT:  .LBB25_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB25_3: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v30
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v31
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v26
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v28
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v37
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v38
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v48
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v50
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v51
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v17
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v19
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v21
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v25
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v27
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  .LBB25_4: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v29
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v26
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_or_b32_e32 v0, v28, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v37, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v38, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v39, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v48, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v49, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v50, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v51, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v19, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v21, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v25, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v27, v13
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 0x30000, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x30000, v13
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28i16_to_v7f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v15, 3
+; VI-NEXT:    v_add_u16_e32 v14, 3, v13
+; VI-NEXT:    v_add_u16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_u16_e32 v14, 3, v12
+; VI-NEXT:    v_add_u16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v14, v12
+; VI-NEXT:    v_add_u16_e32 v14, 3, v11
+; VI-NEXT:    v_add_u16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v14, v11
+; VI-NEXT:    v_add_u16_e32 v14, 3, v10
+; VI-NEXT:    v_add_u16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v14, v10
+; VI-NEXT:    v_add_u16_e32 v14, 3, v9
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v14, v9
+; VI-NEXT:    v_add_u16_e32 v14, 3, v8
+; VI-NEXT:    v_add_u16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v14, v8
+; VI-NEXT:    v_add_u16_e32 v14, 3, v7
+; VI-NEXT:    v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v14, v7
+; VI-NEXT:    v_add_u16_e32 v14, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v14, v6
+; VI-NEXT:    v_add_u16_e32 v14, 3, v5
+; VI-NEXT:    v_add_u16_sdwa v5, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v14, v5
+; VI-NEXT:    v_add_u16_e32 v14, 3, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v14, v4
+; VI-NEXT:    v_add_u16_e32 v14, 3, v3
+; VI-NEXT:    v_add_u16_sdwa v3, v3, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v14, v3
+; VI-NEXT:    v_add_u16_e32 v14, 3, v2
+; VI-NEXT:    v_add_u16_sdwa v2, v2, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v14, v2
+; VI-NEXT:    v_add_u16_e32 v14, 3, v1
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v14, v1
+; VI-NEXT:    v_add_u16_e32 v14, 3, v0
+; VI-NEXT:    v_add_u16_sdwa v0, v0, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v14, v0
+; VI-NEXT:  .LBB25_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28i16_to_v7f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28i16_to_v7f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <28 x i16> %a, splat (i16 3)
+  %a2 = bitcast <28 x i16> %a1 to <7 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x i16> %a to <7 x double>
+  br label %end
+
+end:
+  %phi = phi <7 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x double> %phi
+}
+
+define <28 x half> @bitcast_v7f64_to_v28f16(<7 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v7f64_to_v28f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v0, v34
+; GCN-NEXT:    v_mov_b32_e32 v1, v49
+; GCN-NEXT:    v_mov_b32_e32 v2, v28
+; GCN-NEXT:    v_mov_b32_e32 v3, v48
+; GCN-NEXT:    v_mov_b32_e32 v4, v29
+; GCN-NEXT:    v_mov_b32_e32 v5, v39
+; GCN-NEXT:    v_mov_b32_e32 v6, v30
+; GCN-NEXT:    v_mov_b32_e32 v7, v38
+; GCN-NEXT:    v_mov_b32_e32 v8, v31
+; GCN-NEXT:    v_mov_b32_e32 v9, v37
+; GCN-NEXT:    v_mov_b32_e32 v10, v32
+; GCN-NEXT:    v_mov_b32_e32 v11, v36
+; GCN-NEXT:    v_mov_b32_e32 v12, v33
+; GCN-NEXT:    v_mov_b32_e32 v13, v35
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v7f64_to_v28f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB26_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v7f64_to_v28f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB26_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v7f64_to_v28f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB26_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <7 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <7 x double> %a1 to <28 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <7 x double> %a to <28 x half>
+  br label %end
+
+end:
+  %phi = phi <28 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x half> %phi
+}
+
+define <7 x double> @bitcast_v28f16_to_v7f64(<28 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f16_to_v7f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v26
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v51, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v49, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v39, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v37, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v35, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v33, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v32, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v30, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v29, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v28, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v19, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v16, v13
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v5, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v6, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v8, v24, v22
+; GCN-NEXT:    v_or_b32_e32 v9, v26, v25
+; GCN-NEXT:    v_or_b32_e32 v10, v19, v27
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v21
+; GCN-NEXT:    v_or_b32_e32 v13, v16, v20
+; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f16_to_v7f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v14, 0x200
+; VI-NEXT:    v_add_f16_sdwa v15, v13, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_or_b32_e32 v13, v13, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v12, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_or_b32_e32 v12, v12, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v11, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v10, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v9, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v8, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v7, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v6, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_or_b32_e32 v6, v6, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v5, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v4, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_or_b32_e32 v4, v4, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v3, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_or_b32_e32 v3, v3, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v2, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_or_b32_e32 v2, v2, v15
+; VI-NEXT:    v_add_f16_sdwa v15, v1, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v14, v0, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_or_b32_e32 v1, v1, v15
+; VI-NEXT:    v_or_b32_e32 v0, v0, v14
+; VI-NEXT:  .LBB27_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f16_to_v7f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f16_to_v7f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <28 x half> %a1 to <7 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x half> %a to <7 x double>
+  br label %end
+
+end:
+  %phi = phi <7 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <7 x double> %phi
+}
+
+define <28 x half> @bitcast_v28i16_to_v28f16(<28 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i16_to_v28f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_mov_b32_e32 v47, v27
+; GCN-NEXT:    v_mov_b32_e32 v46, v26
+; GCN-NEXT:    v_mov_b32_e32 v45, v25
+; GCN-NEXT:    v_mov_b32_e32 v44, v24
+; GCN-NEXT:    v_mov_b32_e32 v43, v23
+; GCN-NEXT:    v_mov_b32_e32 v42, v22
+; GCN-NEXT:    v_mov_b32_e32 v41, v21
+; GCN-NEXT:    v_mov_b32_e32 v40, v20
+; GCN-NEXT:    v_mov_b32_e32 v55, v19
+; GCN-NEXT:    v_mov_b32_e32 v54, v18
+; GCN-NEXT:    v_mov_b32_e32 v53, v17
+; GCN-NEXT:    v_mov_b32_e32 v52, v16
+; GCN-NEXT:    v_mov_b32_e32 v51, v15
+; GCN-NEXT:    v_mov_b32_e32 v50, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v13
+; GCN-NEXT:    v_mov_b32_e32 v48, v12
+; GCN-NEXT:    v_mov_b32_e32 v39, v11
+; GCN-NEXT:    v_mov_b32_e32 v38, v10
+; GCN-NEXT:    v_mov_b32_e32 v37, v9
+; GCN-NEXT:    v_mov_b32_e32 v36, v8
+; GCN-NEXT:    v_mov_b32_e32 v35, v7
+; GCN-NEXT:    v_mov_b32_e32 v34, v6
+; GCN-NEXT:    v_mov_b32_e32 v33, v5
+; GCN-NEXT:    v_mov_b32_e32 v32, v4
+; GCN-NEXT:    v_mov_b32_e32 v31, v3
+; GCN-NEXT:    v_mov_b32_e32 v30, v2
+; GCN-NEXT:    v_mov_b32_e32 v29, v1
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v56, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:  .LBB28_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v47
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v42
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v41
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v55
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28i16_to_v28f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v14, 3
+; VI-NEXT:    v_add_u16_sdwa v19, v13, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v13, 3, v13
+; VI-NEXT:    v_or_b32_e32 v13, v13, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v12, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v12, 3, v12
+; VI-NEXT:    v_or_b32_e32 v12, v12, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v11, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_or_b32_e32 v11, v11, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v10, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_or_b32_e32 v10, v10, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v9, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_or_b32_e32 v9, v9, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v8, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_or_b32_e32 v8, v8, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v7, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_or_b32_e32 v7, v7, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v6, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_sdwa v15, v0, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v16, v1, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v17, v2, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v18, v3, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v19
+; VI-NEXT:    v_add_u16_sdwa v19, v4, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_sdwa v14, v5, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_or_b32_e32 v5, v5, v14
+; VI-NEXT:    v_or_b32_e32 v4, v4, v19
+; VI-NEXT:    v_or_b32_e32 v3, v3, v18
+; VI-NEXT:    v_or_b32_e32 v2, v2, v17
+; VI-NEXT:    v_or_b32_e32 v1, v1, v16
+; VI-NEXT:    v_or_b32_e32 v0, v0, v15
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28i16_to_v28f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28i16_to_v28f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <28 x i16> %a, splat (i16 3)
+  %a2 = bitcast <28 x i16> %a1 to <28 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x i16> %a to <28 x half>
+  br label %end
+
+end:
+  %phi = phi <28 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x half> %phi
+}
+
+define <28 x i16> @bitcast_v28f16_to_v28i16(<28 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f16_to_v28i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v5
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v9
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v17
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v21
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v25
+; GCN-NEXT:    v_alignbit_b32 v1, v2, v1, 16
+; GCN-NEXT:    v_alignbit_b32 v5, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v26, v25, 16
+; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f16_to_v28i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v15, 0x200
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v13
+; VI-NEXT:    v_add_f16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v19, v13
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v12
+; VI-NEXT:    v_add_f16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v19, v12
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v11
+; VI-NEXT:    v_add_f16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v19, v11
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v10
+; VI-NEXT:    v_add_f16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v19, v10
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v9
+; VI-NEXT:    v_add_f16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v19, v9
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v8
+; VI-NEXT:    v_add_f16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v19, v8
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v7
+; VI-NEXT:    v_add_f16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v19, v7
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v6
+; VI-NEXT:    v_add_f16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v0
+; VI-NEXT:    v_add_f16_sdwa v0, v0, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v1
+; VI-NEXT:    v_add_f16_sdwa v1, v1, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v2, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v3
+; VI-NEXT:    v_add_f16_sdwa v3, v3, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v19, v6
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v4
+; VI-NEXT:    v_add_f16_sdwa v4, v4, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_sdwa v15, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_or_b32_e32 v5, v5, v15
+; VI-NEXT:    v_or_b32_e32 v4, v19, v4
+; VI-NEXT:    v_or_b32_e32 v3, v18, v3
+; VI-NEXT:    v_or_b32_e32 v2, v17, v2
+; VI-NEXT:    v_or_b32_e32 v1, v16, v1
+; VI-NEXT:    v_or_b32_e32 v0, v14, v0
+; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f16_to_v28i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_movk_i32 s6, 0x200
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s6 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f16_to_v28i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v14
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <28 x half> %a1 to <28 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x half> %a to <28 x i16>
+  br label %end
+
+end:
+  %phi = phi <28 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
index 110c6109b1556..1ef7d358d8cae 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
@@ -5,8 +5,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
 
-define <18 x i32> @bitcast_v9i64_to_v18i32(<9 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v9i64_to_v18i32:
+define <18 x float> @bitcast_v18i32_to_v18f32(<18 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18i32_to_v18f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
@@ -15,29 +15,29 @@ define <18 x i32> @bitcast_v9i64_to_v18i32(<9 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:  .LBB0_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v9i64_to_v18i32:
+; VI-LABEL: bitcast_v18i32_to_v18f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
@@ -46,29 +46,29 @@ define <18 x i32> @bitcast_v9i64_to_v18i32(<9 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB0_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
 ; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
-; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
 ; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
-; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
-; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:  .LBB0_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v9i64_to_v18i32:
+; GFX9-LABEL: bitcast_v18i32_to_v18f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
@@ -77,29 +77,29 @@ define <18 x i32> @bitcast_v9i64_to_v18i32(<9 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
-; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
-; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
 ; GFX9-NEXT:  .LBB0_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v9i64_to_v18i32:
+; GFX11-LABEL: bitcast_v18i32_to_v18f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -109,29 +109,24 @@ define <18 x i32> @bitcast_v9i64_to_v18i32(<9 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
 ; GFX11-NEXT:  .LBB0_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -139,12 +134,145 @@ define <18 x i32> @bitcast_v9i64_to_v18i32(<9 x i64> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <9 x i64> %a, splat (i64 3)
-  %a2 = bitcast <9 x i64> %a1 to <18 x i32>
+  %a1 = add <18 x i32> %a, splat (i32 3)
+  %a2 = bitcast <18 x i32> %a1 to <18 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <9 x i64> %a to <18 x i32>
+  %a3 = bitcast <18 x i32> %a to <18 x float>
+  br label %end
+
+end:
+  %phi = phi <18 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x float> %phi
+}
+
+define <18 x i32> @bitcast_v18f32_to_v18i32(<18 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18f32_to_v18i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18f32_to_v18i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18f32_to_v18i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18f32_to_v18i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <18 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <18 x float> %a1 to <18 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x float> %a to <18 x i32>
   br label %end
 
 end:
@@ -160,7 +288,7 @@ define <9 x i64> @bitcast_v18i32_to_v9i64(<18 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
@@ -180,7 +308,7 @@ define <9 x i64> @bitcast_v18i32_to_v9i64(<18 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:  .LBB2_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -191,7 +319,7 @@ define <9 x i64> @bitcast_v18i32_to_v9i64(<18 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:    s_cbranch_execz .LBB2_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
 ; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
@@ -211,7 +339,7 @@ define <9 x i64> @bitcast_v18i32_to_v9i64(<18 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:  .LBB2_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -222,7 +350,7 @@ define <9 x i64> @bitcast_v18i32_to_v9i64(<18 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
 ; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
@@ -242,7 +370,7 @@ define <9 x i64> @bitcast_v18i32_to_v9i64(<18 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:  .LBB2_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -254,7 +382,7 @@ define <9 x i64> @bitcast_v18i32_to_v9i64(<18 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
 ; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
@@ -274,7 +402,7 @@ define <9 x i64> @bitcast_v18i32_to_v9i64(<18 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:  .LBB2_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -293,3 +421,13231 @@ end:
   %phi = phi <9 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <9 x i64> %phi
 }
+
+define <18 x i32> @bitcast_v9i64_to_v18i32(<9 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9i64_to_v18i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9i64_to_v18i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9i64_to_v18i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9i64_to_v18i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <9 x i64> %a, splat (i64 3)
+  %a2 = bitcast <9 x i64> %a1 to <18 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x i64> %a to <18 x i32>
+  br label %end
+
+end:
+  %phi = phi <18 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x i32> %phi
+}
+
+define <9 x double> @bitcast_v18i32_to_v9f64(<18 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18i32_to_v9f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18i32_to_v9f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB4_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18i32_to_v9f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB4_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18i32_to_v9f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <18 x i32> %a, splat (i32 3)
+  %a2 = bitcast <18 x i32> %a1 to <9 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x i32> %a to <9 x double>
+  br label %end
+
+end:
+  %phi = phi <9 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x double> %phi
+}
+
+define <18 x i32> @bitcast_v9f64_to_v18i32(<9 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9f64_to_v18i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9f64_to_v18i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9f64_to_v18i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9f64_to_v18i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <9 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <9 x double> %a1 to <18 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x double> %a to <18 x i32>
+  br label %end
+
+end:
+  %phi = phi <18 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x i32> %phi
+}
+
+define <36 x i16> @bitcast_v18i32_to_v36i16(<18 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18i32_to_v36i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v19
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v19, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_alignbit_b32 v19, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v33
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v21
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v19
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x44, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v28, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v23, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18i32_to_v36i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB6_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB6_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18i32_to_v36i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB6_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB6_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v35, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v34, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v33, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v30, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v29, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v28, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v27, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v26, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v25, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v23, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v22, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v21, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v20, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v19, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v18, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18i32_to_v36i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB6_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB6_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v35, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v26, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v25, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v24, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v23, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v22, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v21, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v20, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v19, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v18, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <18 x i32> %a, splat (i32 3)
+  %a2 = bitcast <18 x i32> %a1 to <36 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x i32> %a to <36 x i16>
+  br label %end
+
+end:
+  %phi = phi <36 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x i16> %phi
+}
+
+define <18 x i32> @bitcast_v36i16_to_v18i32(<36 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36i16_to_v18i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v29
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v40
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v41
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v62
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v18
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v18
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v43
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v44
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v45
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v46
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v47
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v56
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v57
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v58
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v59
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v60
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v61
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:  .LBB7_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_or_b32_e32 v0, v40, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v41, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v18, v4
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v18, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v43, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v44, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v45, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v46, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v47, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v56, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v57, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v58, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v59, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v60, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v61, v17
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36i16_to_v18i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v17
+; VI-NEXT:    v_mov_b32_e32 v33, v16
+; VI-NEXT:    v_mov_b32_e32 v34, v15
+; VI-NEXT:    v_mov_b32_e32 v35, v14
+; VI-NEXT:    v_mov_b32_e32 v36, v13
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_mov_b32_e32 v38, v11
+; VI-NEXT:    v_mov_b32_e32 v39, v10
+; VI-NEXT:    v_mov_b32_e32 v48, v9
+; VI-NEXT:    v_mov_b32_e32 v49, v8
+; VI-NEXT:    v_mov_b32_e32 v50, v7
+; VI-NEXT:    v_mov_b32_e32 v51, v6
+; VI-NEXT:    v_mov_b32_e32 v52, v5
+; VI-NEXT:    v_mov_b32_e32 v53, v4
+; VI-NEXT:    v_mov_b32_e32 v54, v3
+; VI-NEXT:    v_mov_b32_e32 v55, v2
+; VI-NEXT:    v_mov_b32_e32 v40, v1
+; VI-NEXT:    v_mov_b32_e32 v41, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v17, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v17, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v17, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v17, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v17, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v17, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v17, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v17, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v17, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v17, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v17, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v17, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v17, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v17, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v17, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v53, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v51, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v49, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v39, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v37, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v33, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB7_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v17, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v1, v41, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v2, v40, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v3, v55, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v4, v54, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v5, v53, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v6, v52, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v7, v51, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v8, v50, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v9, v49, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v10, v48, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v11, v39, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v12, v38, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v13, v37, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v14, v36, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v15, v35, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v16, v34, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v18, v33, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v17, v32, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:  .LBB7_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36i16_to_v18i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v17
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v16
+; GFX9-NEXT:    v_mov_b32_e32 v41, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v14
+; GFX9-NEXT:    v_mov_b32_e32 v36, v13
+; GFX9-NEXT:    v_mov_b32_e32 v37, v12
+; GFX9-NEXT:    v_mov_b32_e32 v38, v11
+; GFX9-NEXT:    v_mov_b32_e32 v39, v10
+; GFX9-NEXT:    v_mov_b32_e32 v48, v9
+; GFX9-NEXT:    v_mov_b32_e32 v49, v8
+; GFX9-NEXT:    v_mov_b32_e32 v50, v7
+; GFX9-NEXT:    v_mov_b32_e32 v51, v6
+; GFX9-NEXT:    v_mov_b32_e32 v52, v5
+; GFX9-NEXT:    v_mov_b32_e32 v53, v4
+; GFX9-NEXT:    v_mov_b32_e32 v54, v3
+; GFX9-NEXT:    v_mov_b32_e32 v55, v2
+; GFX9-NEXT:    v_mov_b32_e32 v40, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v41
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:  .LBB7_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36i16_to_v18i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_perm_b32 v4, v32, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v33, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v35, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v31, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v30, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v29, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v28, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v27, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v26, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v25, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v24, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v23, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v22, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v20, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v19, v17, 0x5040100
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <36 x i16> %a, splat (i16 3)
+  %a2 = bitcast <36 x i16> %a1 to <18 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x i16> %a to <18 x i32>
+  br label %end
+
+end:
+  %phi = phi <18 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x i32> %phi
+}
+
+define <36 x half> @bitcast_v18i32_to_v36f16(<18 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18i32_to_v36f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v19
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:  .LBB8_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:  .LBB8_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v42
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v53
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v51
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v49
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v39
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v37
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v6, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v14, v13
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v10, v35, v37
+; GCN-NEXT:    v_or_b32_e32 v11, v33, v49
+; GCN-NEXT:    v_or_b32_e32 v13, v31, v52
+; GCN-NEXT:    v_or_b32_e32 v14, v29, v50
+; GCN-NEXT:    v_or_b32_e32 v16, v27, v48
+; GCN-NEXT:    v_or_b32_e32 v17, v25, v38
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v36
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v34
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v32
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v26
+; GCN-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18i32_to_v36f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB8_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB8_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18i32_to_v36f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB8_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB8_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v35, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v34, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v33, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v30, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v29, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v28, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v27, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v26, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v25, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v23, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v22, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v21, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v20, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v19, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v18, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18i32_to_v36f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB8_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB8_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v35, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v26, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v25, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v24, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v23, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v22, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v21, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v20, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v19, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v18, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <18 x i32> %a, splat (i32 3)
+  %a2 = bitcast <18 x i32> %a1 to <36 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x i32> %a to <36 x half>
+  br label %end
+
+end:
+  %phi = phi <36 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x half> %phi
+}
+
+define <18 x i32> @bitcast_v36f16_to_v18i32(<36 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36f16_to_v18i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v29
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v32, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v2, v62, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v3, v60, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v46, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v44, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v40, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v54, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v52, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v51, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v48, v11
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v18, v12
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v18, v13
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v18, v14
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v18, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v37, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v36, v17
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:  .LBB9_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v62
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v60
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v55
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v53
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v50
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v49
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v36
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v9, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v10, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v11, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v12, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v13, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v14, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v15, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v16, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v17, v31, v30
+; GCN-NEXT:  .LBB9_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36f16_to_v18i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v17
+; VI-NEXT:    v_mov_b32_e32 v33, v16
+; VI-NEXT:    v_mov_b32_e32 v34, v15
+; VI-NEXT:    v_mov_b32_e32 v35, v14
+; VI-NEXT:    v_mov_b32_e32 v36, v13
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_mov_b32_e32 v38, v11
+; VI-NEXT:    v_mov_b32_e32 v39, v10
+; VI-NEXT:    v_mov_b32_e32 v48, v9
+; VI-NEXT:    v_mov_b32_e32 v49, v8
+; VI-NEXT:    v_mov_b32_e32 v50, v7
+; VI-NEXT:    v_mov_b32_e32 v51, v6
+; VI-NEXT:    v_mov_b32_e32 v52, v5
+; VI-NEXT:    v_mov_b32_e32 v53, v4
+; VI-NEXT:    v_mov_b32_e32 v54, v3
+; VI-NEXT:    v_mov_b32_e32 v55, v2
+; VI-NEXT:    v_mov_b32_e32 v40, v1
+; VI-NEXT:    v_mov_b32_e32 v41, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v17, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v17, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v17, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v17, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v17, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v17, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v17, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v17, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v17, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v17, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v17, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v17, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v17, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v17, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v17, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v53, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v51, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v49, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v39, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v37, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v33, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB9_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v17, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v41, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v40, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v55, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v54, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v53, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v52, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v51, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v50, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v49, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v48, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v39, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v38, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v37, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v36, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v35, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v34, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v33, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v16, v18, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v32, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:  .LBB9_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36f16_to_v18i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v17
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v16
+; GFX9-NEXT:    v_mov_b32_e32 v41, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v14
+; GFX9-NEXT:    v_mov_b32_e32 v36, v13
+; GFX9-NEXT:    v_mov_b32_e32 v37, v12
+; GFX9-NEXT:    v_mov_b32_e32 v38, v11
+; GFX9-NEXT:    v_mov_b32_e32 v39, v10
+; GFX9-NEXT:    v_mov_b32_e32 v48, v9
+; GFX9-NEXT:    v_mov_b32_e32 v49, v8
+; GFX9-NEXT:    v_mov_b32_e32 v50, v7
+; GFX9-NEXT:    v_mov_b32_e32 v51, v6
+; GFX9-NEXT:    v_mov_b32_e32 v52, v5
+; GFX9-NEXT:    v_mov_b32_e32 v53, v4
+; GFX9-NEXT:    v_mov_b32_e32 v54, v3
+; GFX9-NEXT:    v_mov_b32_e32 v55, v2
+; GFX9-NEXT:    v_mov_b32_e32 v40, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v41
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:  .LBB9_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36f16_to_v18i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_perm_b32 v4, v32, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v33, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v35, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v31, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v30, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v29, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v28, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v27, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v26, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v25, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v24, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v23, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v22, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v20, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v19, v17, 0x5040100
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <36 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <36 x half> %a1 to <18 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x half> %a to <18 x i32>
+  br label %end
+
+end:
+  %phi = phi <18 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x i32> %phi
+}
+
+define <9 x i64> @bitcast_v18f32_to_v9i64(<18 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18f32_to_v9i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18f32_to_v9i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18f32_to_v9i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18f32_to_v9i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <18 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <18 x float> %a1 to <9 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x float> %a to <9 x i64>
+  br label %end
+
+end:
+  %phi = phi <9 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x i64> %phi
+}
+
+define <18 x float> @bitcast_v9i64_to_v18f32(<9 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9i64_to_v18f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9i64_to_v18f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9i64_to_v18f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9i64_to_v18f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <9 x i64> %a, splat (i64 3)
+  %a2 = bitcast <9 x i64> %a1 to <18 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x i64> %a to <18 x float>
+  br label %end
+
+end:
+  %phi = phi <18 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x float> %phi
+}
+
+define <9 x double> @bitcast_v18f32_to_v9f64(<18 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18f32_to_v9f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18f32_to_v9f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18f32_to_v9f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18f32_to_v9f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB12_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <18 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <18 x float> %a1 to <9 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x float> %a to <9 x double>
+  br label %end
+
+end:
+  %phi = phi <9 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x double> %phi
+}
+
+define <18 x float> @bitcast_v9f64_to_v18f32(<9 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9f64_to_v18f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9f64_to_v18f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9f64_to_v18f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9f64_to_v18f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <9 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <9 x double> %a1 to <18 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x double> %a to <18 x float>
+  br label %end
+
+end:
+  %phi = phi <18 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x float> %phi
+}
+
+define <36 x i16> @bitcast_v18f32_to_v36i16(<18 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18f32_to_v36i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v19
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v19, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_alignbit_b32 v19, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v33
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v21
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v19
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x44, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v28, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v23, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18f32_to_v36i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB14_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB14_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18f32_to_v36i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB14_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB14_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v35, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v34, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v33, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v30, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v29, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v28, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v27, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v26, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v25, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v23, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v22, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v21, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v20, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v19, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v18, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18f32_to_v36i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB14_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB14_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v35, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v26, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v25, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v24, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v23, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v22, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v21, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v20, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v19, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v18, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <18 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <18 x float> %a1 to <36 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x float> %a to <36 x i16>
+  br label %end
+
+end:
+  %phi = phi <36 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x i16> %phi
+}
+
+define <18 x float> @bitcast_v36i16_to_v18f32(<36 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36i16_to_v18f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v29
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v40
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v41
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v62
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v18
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v18
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v43
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v44
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v45
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v46
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v47
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v56
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v57
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v58
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v59
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v60
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v61
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:  .LBB15_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_or_b32_e32 v0, v40, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v41, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v18, v4
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v18, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v43, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v44, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v45, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v46, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v47, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v56, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v57, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v58, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v59, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v60, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v61, v17
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:  .LBB15_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36i16_to_v18f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v17
+; VI-NEXT:    v_mov_b32_e32 v33, v16
+; VI-NEXT:    v_mov_b32_e32 v34, v15
+; VI-NEXT:    v_mov_b32_e32 v35, v14
+; VI-NEXT:    v_mov_b32_e32 v36, v13
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_mov_b32_e32 v38, v11
+; VI-NEXT:    v_mov_b32_e32 v39, v10
+; VI-NEXT:    v_mov_b32_e32 v48, v9
+; VI-NEXT:    v_mov_b32_e32 v49, v8
+; VI-NEXT:    v_mov_b32_e32 v50, v7
+; VI-NEXT:    v_mov_b32_e32 v51, v6
+; VI-NEXT:    v_mov_b32_e32 v52, v5
+; VI-NEXT:    v_mov_b32_e32 v53, v4
+; VI-NEXT:    v_mov_b32_e32 v54, v3
+; VI-NEXT:    v_mov_b32_e32 v55, v2
+; VI-NEXT:    v_mov_b32_e32 v40, v1
+; VI-NEXT:    v_mov_b32_e32 v41, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v17, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v17, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v17, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v17, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v17, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v17, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v17, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v17, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v17, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v17, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v17, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v17, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v17, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v17, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v17, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v53, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v51, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v49, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v39, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v37, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v33, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB15_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v17, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v1, v41, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v2, v40, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v3, v55, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v4, v54, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v5, v53, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v6, v52, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v7, v51, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v8, v50, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v9, v49, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v10, v48, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v11, v39, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v12, v38, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v13, v37, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v14, v36, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v15, v35, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v16, v34, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v18, v33, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v17, v32, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:  .LBB15_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36i16_to_v18f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v17
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v16
+; GFX9-NEXT:    v_mov_b32_e32 v41, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v14
+; GFX9-NEXT:    v_mov_b32_e32 v36, v13
+; GFX9-NEXT:    v_mov_b32_e32 v37, v12
+; GFX9-NEXT:    v_mov_b32_e32 v38, v11
+; GFX9-NEXT:    v_mov_b32_e32 v39, v10
+; GFX9-NEXT:    v_mov_b32_e32 v48, v9
+; GFX9-NEXT:    v_mov_b32_e32 v49, v8
+; GFX9-NEXT:    v_mov_b32_e32 v50, v7
+; GFX9-NEXT:    v_mov_b32_e32 v51, v6
+; GFX9-NEXT:    v_mov_b32_e32 v52, v5
+; GFX9-NEXT:    v_mov_b32_e32 v53, v4
+; GFX9-NEXT:    v_mov_b32_e32 v54, v3
+; GFX9-NEXT:    v_mov_b32_e32 v55, v2
+; GFX9-NEXT:    v_mov_b32_e32 v40, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v41
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:  .LBB15_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36i16_to_v18f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_perm_b32 v4, v32, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v33, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v35, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v31, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v30, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v29, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v28, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v27, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v26, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v25, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v24, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v23, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v22, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v20, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v19, v17, 0x5040100
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <36 x i16> %a, splat (i16 3)
+  %a2 = bitcast <36 x i16> %a1 to <18 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x i16> %a to <18 x float>
+  br label %end
+
+end:
+  %phi = phi <18 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x float> %phi
+}
+
+define <36 x half> @bitcast_v18f32_to_v36f16(<18 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v18f32_to_v36f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v19
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:  .LBB16_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:  .LBB16_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v42
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v53
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v51
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v49
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v39
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v37
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v6, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v14, v13
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v10, v35, v37
+; GCN-NEXT:    v_or_b32_e32 v11, v33, v49
+; GCN-NEXT:    v_or_b32_e32 v13, v31, v52
+; GCN-NEXT:    v_or_b32_e32 v14, v29, v50
+; GCN-NEXT:    v_or_b32_e32 v16, v27, v48
+; GCN-NEXT:    v_or_b32_e32 v17, v25, v38
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v36
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v34
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v32
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v26
+; GCN-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v18f32_to_v36f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB16_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB16_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v18f32_to_v36f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB16_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB16_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v35, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v34, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v33, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v30, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v29, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v28, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v27, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v26, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v25, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v23, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v22, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v21, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v20, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v19, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v18, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v18f32_to_v36f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB16_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB16_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v35, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v26, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v25, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v24, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v23, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v22, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v21, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v20, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v19, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v18, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <18 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <18 x float> %a1 to <36 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <18 x float> %a to <36 x half>
+  br label %end
+
+end:
+  %phi = phi <36 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x half> %phi
+}
+
+define <18 x float> @bitcast_v36f16_to_v18f32(<36 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36f16_to_v18f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v29
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v32, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v2, v62, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v3, v60, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v46, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v44, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v40, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v54, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v52, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v51, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v48, v11
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v18, v12
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v18, v13
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v18, v14
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v18, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v37, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v36, v17
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v62
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v60
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v55
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v53
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v50
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v49
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v36
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v9, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v10, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v11, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v12, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v13, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v14, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v15, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v16, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v17, v31, v30
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36f16_to_v18f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v17
+; VI-NEXT:    v_mov_b32_e32 v33, v16
+; VI-NEXT:    v_mov_b32_e32 v34, v15
+; VI-NEXT:    v_mov_b32_e32 v35, v14
+; VI-NEXT:    v_mov_b32_e32 v36, v13
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_mov_b32_e32 v38, v11
+; VI-NEXT:    v_mov_b32_e32 v39, v10
+; VI-NEXT:    v_mov_b32_e32 v48, v9
+; VI-NEXT:    v_mov_b32_e32 v49, v8
+; VI-NEXT:    v_mov_b32_e32 v50, v7
+; VI-NEXT:    v_mov_b32_e32 v51, v6
+; VI-NEXT:    v_mov_b32_e32 v52, v5
+; VI-NEXT:    v_mov_b32_e32 v53, v4
+; VI-NEXT:    v_mov_b32_e32 v54, v3
+; VI-NEXT:    v_mov_b32_e32 v55, v2
+; VI-NEXT:    v_mov_b32_e32 v40, v1
+; VI-NEXT:    v_mov_b32_e32 v41, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v17, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v17, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v17, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v17, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v17, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v17, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v17, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v17, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v17, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v17, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v17, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v17, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v17, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v17, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v17, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v53, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v51, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v49, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v39, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v37, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v33, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB17_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v17, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v41, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v40, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v55, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v54, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v53, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v52, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v51, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v50, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v49, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v48, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v39, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v38, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v37, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v36, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v35, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v34, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v33, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v16, v18, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v32, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:  .LBB17_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36f16_to_v18f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v17
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v16
+; GFX9-NEXT:    v_mov_b32_e32 v41, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v14
+; GFX9-NEXT:    v_mov_b32_e32 v36, v13
+; GFX9-NEXT:    v_mov_b32_e32 v37, v12
+; GFX9-NEXT:    v_mov_b32_e32 v38, v11
+; GFX9-NEXT:    v_mov_b32_e32 v39, v10
+; GFX9-NEXT:    v_mov_b32_e32 v48, v9
+; GFX9-NEXT:    v_mov_b32_e32 v49, v8
+; GFX9-NEXT:    v_mov_b32_e32 v50, v7
+; GFX9-NEXT:    v_mov_b32_e32 v51, v6
+; GFX9-NEXT:    v_mov_b32_e32 v52, v5
+; GFX9-NEXT:    v_mov_b32_e32 v53, v4
+; GFX9-NEXT:    v_mov_b32_e32 v54, v3
+; GFX9-NEXT:    v_mov_b32_e32 v55, v2
+; GFX9-NEXT:    v_mov_b32_e32 v40, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v41
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:  .LBB17_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36f16_to_v18f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_perm_b32 v4, v32, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v33, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v35, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v31, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v30, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v29, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v28, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v27, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v26, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v25, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v24, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v23, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v22, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v20, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v19, v17, 0x5040100
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <36 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <36 x half> %a1 to <18 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x half> %a to <18 x float>
+  br label %end
+
+end:
+  %phi = phi <18 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <18 x float> %phi
+}
+
+define <9 x double> @bitcast_v9i64_to_v9f64(<9 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9i64_to_v9f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9i64_to_v9f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9i64_to_v9f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9i64_to_v9f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <9 x i64> %a, splat (i64 3)
+  %a2 = bitcast <9 x i64> %a1 to <9 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x i64> %a to <9 x double>
+  br label %end
+
+end:
+  %phi = phi <9 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x double> %phi
+}
+
+define <9 x i64> @bitcast_v9f64_to_v9i64(<9 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9f64_to_v9i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9f64_to_v9i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9f64_to_v9i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9f64_to_v9i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <9 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <9 x double> %a1 to <9 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x double> %a to <9 x i64>
+  br label %end
+
+end:
+  %phi = phi <9 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x i64> %phi
+}
+
+define <36 x i16> @bitcast_v9i64_to_v36i16(<9 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9i64_to_v36i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v19
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v19, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_alignbit_b32 v19, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v33
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v21
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v19
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x44, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v28, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v23, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9i64_to_v36i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB20_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB20_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9i64_to_v36i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB20_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB20_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v35, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v34, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v33, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v30, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v29, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v28, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v27, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v26, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v25, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v23, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v22, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v21, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v20, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v19, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v18, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9i64_to_v36i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB20_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB20_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v35, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v26, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v25, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v24, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v23, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v22, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v21, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v20, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v19, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v18, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <9 x i64> %a, splat (i64 3)
+  %a2 = bitcast <9 x i64> %a1 to <36 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x i64> %a to <36 x i16>
+  br label %end
+
+end:
+  %phi = phi <36 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x i16> %phi
+}
+
+define <9 x i64> @bitcast_v36i16_to_v9i64(<36 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36i16_to_v9i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v29
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v40
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v41
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v62
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v18
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v18
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v43
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v44
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v45
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v46
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v47
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v56
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v57
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v58
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v59
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v60
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v61
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:  .LBB21_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_or_b32_e32 v0, v40, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v41, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v18, v4
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v18, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v43, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v44, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v45, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v46, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v47, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v56, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v57, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v58, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v59, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v60, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v61, v17
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:  .LBB21_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36i16_to_v9i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v17
+; VI-NEXT:    v_mov_b32_e32 v33, v16
+; VI-NEXT:    v_mov_b32_e32 v34, v15
+; VI-NEXT:    v_mov_b32_e32 v35, v14
+; VI-NEXT:    v_mov_b32_e32 v36, v13
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_mov_b32_e32 v38, v11
+; VI-NEXT:    v_mov_b32_e32 v39, v10
+; VI-NEXT:    v_mov_b32_e32 v48, v9
+; VI-NEXT:    v_mov_b32_e32 v49, v8
+; VI-NEXT:    v_mov_b32_e32 v50, v7
+; VI-NEXT:    v_mov_b32_e32 v51, v6
+; VI-NEXT:    v_mov_b32_e32 v52, v5
+; VI-NEXT:    v_mov_b32_e32 v53, v4
+; VI-NEXT:    v_mov_b32_e32 v54, v3
+; VI-NEXT:    v_mov_b32_e32 v55, v2
+; VI-NEXT:    v_mov_b32_e32 v40, v1
+; VI-NEXT:    v_mov_b32_e32 v41, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v17, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v17, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v17, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v17, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v17, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v17, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v17, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v17, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v17, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v17, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v17, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v17, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v17, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v17, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v17, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v53, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v51, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v49, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v39, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v37, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v33, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB21_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v17, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v1, v41, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v2, v40, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v3, v55, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v4, v54, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v5, v53, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v6, v52, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v7, v51, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v8, v50, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v9, v49, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v10, v48, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v11, v39, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v12, v38, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v13, v37, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v14, v36, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v15, v35, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v16, v34, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v18, v33, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v17, v32, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:  .LBB21_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36i16_to_v9i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v17
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v16
+; GFX9-NEXT:    v_mov_b32_e32 v41, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v14
+; GFX9-NEXT:    v_mov_b32_e32 v36, v13
+; GFX9-NEXT:    v_mov_b32_e32 v37, v12
+; GFX9-NEXT:    v_mov_b32_e32 v38, v11
+; GFX9-NEXT:    v_mov_b32_e32 v39, v10
+; GFX9-NEXT:    v_mov_b32_e32 v48, v9
+; GFX9-NEXT:    v_mov_b32_e32 v49, v8
+; GFX9-NEXT:    v_mov_b32_e32 v50, v7
+; GFX9-NEXT:    v_mov_b32_e32 v51, v6
+; GFX9-NEXT:    v_mov_b32_e32 v52, v5
+; GFX9-NEXT:    v_mov_b32_e32 v53, v4
+; GFX9-NEXT:    v_mov_b32_e32 v54, v3
+; GFX9-NEXT:    v_mov_b32_e32 v55, v2
+; GFX9-NEXT:    v_mov_b32_e32 v40, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v41
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:  .LBB21_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36i16_to_v9i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_perm_b32 v4, v32, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v33, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v35, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v31, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v30, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v29, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v28, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v27, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v26, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v25, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v24, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v23, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v22, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v20, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v19, v17, 0x5040100
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <36 x i16> %a, splat (i16 3)
+  %a2 = bitcast <36 x i16> %a1 to <9 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x i16> %a to <9 x i64>
+  br label %end
+
+end:
+  %phi = phi <9 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x i64> %phi
+}
+
+define <36 x half> @bitcast_v9i64_to_v36f16(<9 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9i64_to_v36f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v19
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:  .LBB22_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:  .LBB22_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v42
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v53
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v51
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v49
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v39
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v37
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v6, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v14, v13
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v10, v35, v37
+; GCN-NEXT:    v_or_b32_e32 v11, v33, v49
+; GCN-NEXT:    v_or_b32_e32 v13, v31, v52
+; GCN-NEXT:    v_or_b32_e32 v14, v29, v50
+; GCN-NEXT:    v_or_b32_e32 v16, v27, v48
+; GCN-NEXT:    v_or_b32_e32 v17, v25, v38
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v36
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v34
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v32
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v26
+; GCN-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9i64_to_v36f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB22_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB22_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9i64_to_v36f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB22_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB22_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v35, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v34, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v33, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v30, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v29, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v28, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v27, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v26, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v25, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v23, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v22, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v21, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v20, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v19, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v18, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9i64_to_v36f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB22_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB22_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v35, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v26, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v25, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v24, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v23, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v22, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v21, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v20, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v19, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v18, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <9 x i64> %a, splat (i64 3)
+  %a2 = bitcast <9 x i64> %a1 to <36 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x i64> %a to <36 x half>
+  br label %end
+
+end:
+  %phi = phi <36 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x half> %phi
+}
+
+define <9 x i64> @bitcast_v36f16_to_v9i64(<36 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36f16_to_v9i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v29
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v32, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v2, v62, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v3, v60, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v46, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v44, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v40, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v54, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v52, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v51, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v48, v11
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v18, v12
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v18, v13
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v18, v14
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v18, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v37, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v36, v17
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:  .LBB23_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v62
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v60
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v55
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v53
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v50
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v49
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v36
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v9, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v10, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v11, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v12, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v13, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v14, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v15, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v16, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v17, v31, v30
+; GCN-NEXT:  .LBB23_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36f16_to_v9i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v17
+; VI-NEXT:    v_mov_b32_e32 v33, v16
+; VI-NEXT:    v_mov_b32_e32 v34, v15
+; VI-NEXT:    v_mov_b32_e32 v35, v14
+; VI-NEXT:    v_mov_b32_e32 v36, v13
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_mov_b32_e32 v38, v11
+; VI-NEXT:    v_mov_b32_e32 v39, v10
+; VI-NEXT:    v_mov_b32_e32 v48, v9
+; VI-NEXT:    v_mov_b32_e32 v49, v8
+; VI-NEXT:    v_mov_b32_e32 v50, v7
+; VI-NEXT:    v_mov_b32_e32 v51, v6
+; VI-NEXT:    v_mov_b32_e32 v52, v5
+; VI-NEXT:    v_mov_b32_e32 v53, v4
+; VI-NEXT:    v_mov_b32_e32 v54, v3
+; VI-NEXT:    v_mov_b32_e32 v55, v2
+; VI-NEXT:    v_mov_b32_e32 v40, v1
+; VI-NEXT:    v_mov_b32_e32 v41, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v17, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v17, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v17, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v17, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v17, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v17, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v17, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v17, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v17, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v17, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v17, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v17, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v17, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v17, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v17, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v53, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v51, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v49, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v39, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v37, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v33, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB23_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v17, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v41, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v40, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v55, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v54, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v53, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v52, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v51, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v50, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v49, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v48, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v39, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v38, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v37, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v36, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v35, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v34, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v33, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v16, v18, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v32, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:  .LBB23_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36f16_to_v9i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v17
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v16
+; GFX9-NEXT:    v_mov_b32_e32 v41, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v14
+; GFX9-NEXT:    v_mov_b32_e32 v36, v13
+; GFX9-NEXT:    v_mov_b32_e32 v37, v12
+; GFX9-NEXT:    v_mov_b32_e32 v38, v11
+; GFX9-NEXT:    v_mov_b32_e32 v39, v10
+; GFX9-NEXT:    v_mov_b32_e32 v48, v9
+; GFX9-NEXT:    v_mov_b32_e32 v49, v8
+; GFX9-NEXT:    v_mov_b32_e32 v50, v7
+; GFX9-NEXT:    v_mov_b32_e32 v51, v6
+; GFX9-NEXT:    v_mov_b32_e32 v52, v5
+; GFX9-NEXT:    v_mov_b32_e32 v53, v4
+; GFX9-NEXT:    v_mov_b32_e32 v54, v3
+; GFX9-NEXT:    v_mov_b32_e32 v55, v2
+; GFX9-NEXT:    v_mov_b32_e32 v40, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v41
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:  .LBB23_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36f16_to_v9i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_perm_b32 v4, v32, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v33, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v35, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v31, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v30, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v29, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v28, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v27, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v26, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v25, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v24, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v23, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v22, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v20, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v19, v17, 0x5040100
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <36 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <36 x half> %a1 to <9 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x half> %a to <9 x i64>
+  br label %end
+
+end:
+  %phi = phi <9 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x i64> %phi
+}
+
+define <36 x i16> @bitcast_v9f64_to_v36i16(<9 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9f64_to_v36i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v19
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v19, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v2
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_alignbit_b32 v19, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v2
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 4, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 8, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 12, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 20, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 24, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 28, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 32, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 36, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 40, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v21
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 52, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 56, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v19
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 0x44, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v28, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v25, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9f64_to_v36i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB24_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB24_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9f64_to_v36i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB24_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB24_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v35, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v34, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v33, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v30, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v29, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v28, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v27, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v26, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v25, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v23, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v22, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v21, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v20, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v19, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v18, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9f64_to_v36i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB24_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB24_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v35, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v26, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v25, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v24, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v23, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v22, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v21, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v20, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v19, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v18, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <9 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <9 x double> %a1 to <36 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x double> %a to <36 x i16>
+  br label %end
+
+end:
+  %phi = phi <36 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x i16> %phi
+}
+
+define <9 x double> @bitcast_v36i16_to_v9f64(<36 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36i16_to_v9f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v29
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v40
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v41
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v62
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v18
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v18
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v43
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v44
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v45
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v46
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v47
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v56
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v57
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v58
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v59
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v60
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v61
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:  .LBB25_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_or_b32_e32 v0, v40, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v41, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v18, v4
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v18, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v43, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v44, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v45, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v46, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v47, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v56, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v57, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v58, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v59, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v60, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v61, v17
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:  .LBB25_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36i16_to_v9f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v17
+; VI-NEXT:    v_mov_b32_e32 v33, v16
+; VI-NEXT:    v_mov_b32_e32 v34, v15
+; VI-NEXT:    v_mov_b32_e32 v35, v14
+; VI-NEXT:    v_mov_b32_e32 v36, v13
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_mov_b32_e32 v38, v11
+; VI-NEXT:    v_mov_b32_e32 v39, v10
+; VI-NEXT:    v_mov_b32_e32 v48, v9
+; VI-NEXT:    v_mov_b32_e32 v49, v8
+; VI-NEXT:    v_mov_b32_e32 v50, v7
+; VI-NEXT:    v_mov_b32_e32 v51, v6
+; VI-NEXT:    v_mov_b32_e32 v52, v5
+; VI-NEXT:    v_mov_b32_e32 v53, v4
+; VI-NEXT:    v_mov_b32_e32 v54, v3
+; VI-NEXT:    v_mov_b32_e32 v55, v2
+; VI-NEXT:    v_mov_b32_e32 v40, v1
+; VI-NEXT:    v_mov_b32_e32 v41, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v17, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v17, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v17, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v17, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v17, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v17, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v17, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v17, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v17, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v17, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v17, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v17, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v17, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v17, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v17, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v53, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v51, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v49, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v39, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v37, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v33, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB25_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v17, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v1, v41, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v2, v40, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v3, v55, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v4, v54, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v5, v53, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v6, v52, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v7, v51, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v8, v50, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v9, v49, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v10, v48, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v11, v39, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v12, v38, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v13, v37, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v14, v36, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v15, v35, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v16, v34, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v18, v33, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v17, v32, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:  .LBB25_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36i16_to_v9f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v17
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v16
+; GFX9-NEXT:    v_mov_b32_e32 v41, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v14
+; GFX9-NEXT:    v_mov_b32_e32 v36, v13
+; GFX9-NEXT:    v_mov_b32_e32 v37, v12
+; GFX9-NEXT:    v_mov_b32_e32 v38, v11
+; GFX9-NEXT:    v_mov_b32_e32 v39, v10
+; GFX9-NEXT:    v_mov_b32_e32 v48, v9
+; GFX9-NEXT:    v_mov_b32_e32 v49, v8
+; GFX9-NEXT:    v_mov_b32_e32 v50, v7
+; GFX9-NEXT:    v_mov_b32_e32 v51, v6
+; GFX9-NEXT:    v_mov_b32_e32 v52, v5
+; GFX9-NEXT:    v_mov_b32_e32 v53, v4
+; GFX9-NEXT:    v_mov_b32_e32 v54, v3
+; GFX9-NEXT:    v_mov_b32_e32 v55, v2
+; GFX9-NEXT:    v_mov_b32_e32 v40, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v41
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:  .LBB25_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36i16_to_v9f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_perm_b32 v4, v32, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v33, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v35, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v31, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v30, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v29, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v28, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v27, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v26, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v25, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v24, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v23, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v22, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v20, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v19, v17, 0x5040100
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <36 x i16> %a, splat (i16 3)
+  %a2 = bitcast <36 x i16> %a1 to <9 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x i16> %a to <9 x double>
+  br label %end
+
+end:
+  %phi = phi <9 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x double> %phi
+}
+
+define <36 x half> @bitcast_v9f64_to_v36f16(<9 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v9f64_to_v36f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v19
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v42
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v53
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v51
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v49
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v39
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v37
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v6, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v14, v13
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v10, v35, v37
+; GCN-NEXT:    v_or_b32_e32 v11, v33, v49
+; GCN-NEXT:    v_or_b32_e32 v13, v31, v52
+; GCN-NEXT:    v_or_b32_e32 v14, v29, v50
+; GCN-NEXT:    v_or_b32_e32 v16, v27, v48
+; GCN-NEXT:    v_or_b32_e32 v17, v25, v38
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v36
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v34
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v32
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v26
+; GCN-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v9f64_to_v36f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr19
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB26_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:  .LBB26_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v9f64_to_v36f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr19
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB26_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX9-NEXT:  .LBB26_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v35, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v34, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v33, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v30, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v29, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v28, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v27, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v26, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v25, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v23, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v22, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v21, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v20, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v19, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v18, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v9f64_to_v36f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    ; implicit-def: $vgpr19
+; GFX11-NEXT:    ; implicit-def: $vgpr18
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB26_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-NEXT:  .LBB26_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v35, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v26, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v25, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v24, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v23, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v22, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v21, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v20, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v19, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v18, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <9 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <9 x double> %a1 to <36 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <9 x double> %a to <36 x half>
+  br label %end
+
+end:
+  %phi = phi <36 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x half> %phi
+}
+
+define <9 x double> @bitcast_v36f16_to_v9f64(<36 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36f16_to_v9f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v29
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v32, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v2, v62, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v3, v60, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v46, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v44, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v40, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v54, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v52, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v51, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v48, v11
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v18, v12
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v18, v13
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v18, v14
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v18, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v37, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v36, v17
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; kill: killed $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v62
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v60
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v55
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v53
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v50
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v49
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v36
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v8, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v9, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v10, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v11, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v12, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v13, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v14, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v15, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v16, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v17, v31, v30
+; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36f16_to_v9f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v17
+; VI-NEXT:    v_mov_b32_e32 v33, v16
+; VI-NEXT:    v_mov_b32_e32 v34, v15
+; VI-NEXT:    v_mov_b32_e32 v35, v14
+; VI-NEXT:    v_mov_b32_e32 v36, v13
+; VI-NEXT:    v_mov_b32_e32 v37, v12
+; VI-NEXT:    v_mov_b32_e32 v38, v11
+; VI-NEXT:    v_mov_b32_e32 v39, v10
+; VI-NEXT:    v_mov_b32_e32 v48, v9
+; VI-NEXT:    v_mov_b32_e32 v49, v8
+; VI-NEXT:    v_mov_b32_e32 v50, v7
+; VI-NEXT:    v_mov_b32_e32 v51, v6
+; VI-NEXT:    v_mov_b32_e32 v52, v5
+; VI-NEXT:    v_mov_b32_e32 v53, v4
+; VI-NEXT:    v_mov_b32_e32 v54, v3
+; VI-NEXT:    v_mov_b32_e32 v55, v2
+; VI-NEXT:    v_mov_b32_e32 v40, v1
+; VI-NEXT:    v_mov_b32_e32 v41, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v17, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v17, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v17, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v17, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v17, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v17, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v17, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v17, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v17, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v17, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v17, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v17, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v17, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v17, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v17, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v53, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v51, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v49, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v39, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v37, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v33, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB27_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v17, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v41, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v40, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v55, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v54, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v53, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v52, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v51, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v50, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v49, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v48, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v39, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v38, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v37, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v36, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v35, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v34, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v33, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v16, v18, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v32, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:  .LBB27_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36f16_to_v9f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v17
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v16
+; GFX9-NEXT:    v_mov_b32_e32 v41, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v14
+; GFX9-NEXT:    v_mov_b32_e32 v36, v13
+; GFX9-NEXT:    v_mov_b32_e32 v37, v12
+; GFX9-NEXT:    v_mov_b32_e32 v38, v11
+; GFX9-NEXT:    v_mov_b32_e32 v39, v10
+; GFX9-NEXT:    v_mov_b32_e32 v48, v9
+; GFX9-NEXT:    v_mov_b32_e32 v49, v8
+; GFX9-NEXT:    v_mov_b32_e32 v50, v7
+; GFX9-NEXT:    v_mov_b32_e32 v51, v6
+; GFX9-NEXT:    v_mov_b32_e32 v52, v5
+; GFX9-NEXT:    v_mov_b32_e32 v53, v4
+; GFX9-NEXT:    v_mov_b32_e32 v54, v3
+; GFX9-NEXT:    v_mov_b32_e32 v55, v2
+; GFX9-NEXT:    v_mov_b32_e32 v40, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v41
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; kill: killed $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:  .LBB27_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v45, v41, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v44, v40, s6
+; GFX9-NEXT:    v_perm_b32 v2, v43, v55, s6
+; GFX9-NEXT:    v_perm_b32 v3, v42, v54, s6
+; GFX9-NEXT:    v_perm_b32 v4, v63, v53, s6
+; GFX9-NEXT:    v_perm_b32 v5, v62, v52, s6
+; GFX9-NEXT:    v_perm_b32 v6, v61, v51, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v50, s6
+; GFX9-NEXT:    v_perm_b32 v8, v59, v49, s6
+; GFX9-NEXT:    v_perm_b32 v9, v58, v48, s6
+; GFX9-NEXT:    v_perm_b32 v10, v57, v39, s6
+; GFX9-NEXT:    v_perm_b32 v11, v56, v38, s6
+; GFX9-NEXT:    v_perm_b32 v12, v47, v37, s6
+; GFX9-NEXT:    v_perm_b32 v13, v46, v36, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36f16_to_v9f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_perm_b32 v4, v32, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v33, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v34, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v35, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v31, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v30, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v29, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v28, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v27, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v26, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v25, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v24, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v23, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v22, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v21, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v20, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v19, v17, 0x5040100
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <36 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <36 x half> %a1 to <9 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x half> %a to <9 x double>
+  br label %end
+
+end:
+  %phi = phi <9 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <9 x double> %phi
+}
+
+define <36 x half> @bitcast_v36i16_to_v36f16(<36 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36i16_to_v36f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v30
+; GCN-NEXT:    s_waitcnt vmcnt(9)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v39
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:  .LBB28_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 3, v39
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 3, v38
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 3, v37
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 3, v36
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 3, v35
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v41
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v42
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v43
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v44
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v45
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v46
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v47
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v56
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v57
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v48
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v49
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v50
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v51
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v6, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v8, v7
+; GCN-NEXT:    v_or_b32_e32 v6, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v7, v14, v13
+; GCN-NEXT:    v_or_b32_e32 v8, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v19
+; GCN-NEXT:    v_or_b32_e32 v11, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v25
+; GCN-NEXT:    v_or_b32_e32 v14, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v16, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v17, v38, v37
+; GCN-NEXT:    v_or_b32_e32 v19, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v20, v52, v51
+; GCN-NEXT:    v_or_b32_e32 v22, v53, v42
+; GCN-NEXT:    v_or_b32_e32 v23, v54, v31
+; GCN-NEXT:    v_or_b32_e32 v25, v55, v32
+; GCN-NEXT:    v_or_b32_e32 v26, v40, v33
+; GCN-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36i16_to_v36f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_add_u16_e32 v35, 3, v35
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v34, 3, v34
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v33, 3, v33
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v32, 3, v32
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v18, 3, v18
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v31, 3, v31
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v30, 3, v30
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v29, 3, v29
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v28, 3, v28
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_e32 v27, 3, v27
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_add_u16_e32 v26, 3, v26
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_add_u16_e32 v25, 3, v25
+; VI-NEXT:    v_add_u16_e32 v12, 3, v12
+; VI-NEXT:    v_add_u16_e32 v24, 3, v24
+; VI-NEXT:    v_add_u16_e32 v13, 3, v13
+; VI-NEXT:    v_add_u16_e32 v23, 3, v23
+; VI-NEXT:    v_add_u16_e32 v14, 3, v14
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    v_add_u16_e32 v21, 3, v21
+; VI-NEXT:    v_add_u16_e32 v16, 3, v16
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_add_u16_e32 v17, 3, v17
+; VI-NEXT:    v_add_u16_e32 v19, 3, v19
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v25
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v23
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v21
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36i16_to_v36f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v17, v35, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v34, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v33, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v32, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v31, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v30, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v29, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v28, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v27, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v26, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v25, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v24, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v22, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v23, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v21, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v20, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v19, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v18, v0, s6
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v18, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v19, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v20, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v21, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v23, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v22, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v24, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v25, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v26, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v27, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v28, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v29, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v30, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v31, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v32, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v33, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v34, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v35, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36i16_to_v36f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v17, v36, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v34, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v32, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v30, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v28, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v26, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v25, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v24, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v19, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v20, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v22, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v23, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v17
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v19, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v20, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v22, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v23, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v24, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v25, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v26, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v28, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v30, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v32, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v34, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v36, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <36 x i16> %a, splat (i16 3)
+  %a2 = bitcast <36 x i16> %a1 to <36 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x i16> %a to <36 x half>
+  br label %end
+
+end:
+  %phi = phi <36 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x half> %phi
+}
+
+define <36 x i16> @bitcast_v36f16_to_v36i16(<36 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v36f16_to_v36i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v30
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v55
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v44
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v31
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v31, v21, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v33
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v33, v21, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v35
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v35, v21, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v52, v22, v21
+; GCN-NEXT:    v_or_b32_e32 v50, v24, v23
+; GCN-NEXT:    v_or_b32_e32 v48, v26, v25
+; GCN-NEXT:    v_or_b32_e32 v38, v28, v27
+; GCN-NEXT:    v_or_b32_e32 v37, v30, v29
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v20
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v19
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v17
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v15
+; GCN-NEXT:    v_alignbit_b32 v54, v35, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v33, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v31, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v11, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v9, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v7, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v19, v5, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v17, v3, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v15, v1, v15, 16
+; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v22
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v30, 0xffff, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v32, 0xffff, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v32, v32, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v34, 0xffff, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v34, v34, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v19
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v17
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v15
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 0x44, v0
+; GCN-NEXT:    buffer_store_dword v21, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v36f16_to_v36i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_add_f16_e32 v35, 0x200, v35
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_e32 v34, 0x200, v34
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_add_f16_e32 v33, 0x200, v33
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_add_f16_e32 v32, 0x200, v32
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v18
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_add_f16_e32 v31, 0x200, v31
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v30
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_add_f16_e32 v29, 0x200, v29
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v28
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v27
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v26
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v25
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v24
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v23
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v14
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v22
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v15
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v21
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v16
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v20
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v17
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v19
+; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v25
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v23
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v21
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v20
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v36f16_to_v36i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v17, v35, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v34, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v33, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v32, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v31, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v30, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v29, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v28, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v27, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v26, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v25, v7, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v6, v24, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v22, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v23, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v21, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v20, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v19, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v18, v0, s6
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v18, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v19, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v20, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v21, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v23, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v22, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v24, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v25, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v26, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v27, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v28, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v29, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v30, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v31, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v32, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v33, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v34, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v35, v17, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v36f16_to_v36i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v18
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v17, v36, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v34, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v32, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v30, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v28, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v26, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v25, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v24, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v19, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v20, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v22, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v23, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v17
+; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v19, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v20, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v22, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v23, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v24, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v25, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v26, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v27, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v28, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v30, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v32, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v34, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v36, v17, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <36 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <36 x half> %a1 to <36 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <36 x half> %a to <36 x i16>
+  br label %end
+
+end:
+  %phi = phi <36 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <36 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
index bea2243e8087c..8f9de9e898301 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
@@ -5,8 +5,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
 
-define <20 x i32> @bitcast_v10i64_to_v20i32(<10 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v10i64_to_v20i32:
+define <20 x float> @bitcast_v20i32_to_v20f32(<20 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i32_to_v20f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
@@ -15,31 +15,31 @@ define <20 x i32> @bitcast_v10i64_to_v20i32(<10 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:  .LBB0_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v10i64_to_v20i32:
+; VI-LABEL: bitcast_v20i32_to_v20f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
@@ -48,31 +48,31 @@ define <20 x i32> @bitcast_v10i64_to_v20i32(<10 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB0_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
 ; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
-; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
 ; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
-; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
 ; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
-; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
-; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:  .LBB0_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v10i64_to_v20i32:
+; GFX9-LABEL: bitcast_v20i32_to_v20f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
@@ -81,31 +81,31 @@ define <20 x i32> @bitcast_v10i64_to_v20i32(<10 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
-; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
-; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
-; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
 ; GFX9-NEXT:  .LBB0_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v10i64_to_v20i32:
+; GFX11-LABEL: bitcast_v20i32_to_v20f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -115,31 +115,26 @@ define <20 x i32> @bitcast_v10i64_to_v20i32(<10 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
 ; GFX11-NEXT:  .LBB0_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -147,12 +142,152 @@ define <20 x i32> @bitcast_v10i64_to_v20i32(<10 x i64> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <10 x i64> %a, splat (i64 3)
-  %a2 = bitcast <10 x i64> %a1 to <20 x i32>
+  %a1 = add <20 x i32> %a, splat (i32 3)
+  %a2 = bitcast <20 x i32> %a1 to <20 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <10 x i64> %a to <20 x i32>
+  %a3 = bitcast <20 x i32> %a to <20 x float>
+  br label %end
+
+end:
+  %phi = phi <20 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x float> %phi
+}
+
+define <20 x i32> @bitcast_v20f32_to_v20i32(<20 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20f32_to_v20i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20f32_to_v20i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20f32_to_v20i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20f32_to_v20i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <20 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <20 x float> %a1 to <20 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x float> %a to <20 x i32>
   br label %end
 
 end:
@@ -168,7 +303,7 @@ define <10 x i64> @bitcast_v20i32_to_v10i64(<20 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
@@ -190,7 +325,7 @@ define <10 x i64> @bitcast_v20i32_to_v10i64(<20 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:  .LBB2_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -201,7 +336,7 @@ define <10 x i64> @bitcast_v20i32_to_v10i64(<20 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:    s_cbranch_execz .LBB2_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
 ; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
@@ -223,7 +358,7 @@ define <10 x i64> @bitcast_v20i32_to_v10i64(<20 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:  .LBB2_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -234,7 +369,7 @@ define <10 x i64> @bitcast_v20i32_to_v10i64(<20 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
 ; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
@@ -256,7 +391,7 @@ define <10 x i64> @bitcast_v20i32_to_v10i64(<20 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:  .LBB2_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -268,7 +403,7 @@ define <10 x i64> @bitcast_v20i32_to_v10i64(<20 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
 ; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
@@ -290,7 +425,7 @@ define <10 x i64> @bitcast_v20i32_to_v10i64(<20 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:  .LBB2_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -309,3 +444,14894 @@ end:
   %phi = phi <10 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <10 x i64> %phi
 }
+
+define <20 x i32> @bitcast_v10i64_to_v20i32(<10 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i64_to_v20i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i64_to_v20i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i64_to_v20i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i64_to_v20i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i64> %a, splat (i64 3)
+  %a2 = bitcast <10 x i64> %a1 to <20 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i64> %a to <20 x i32>
+  br label %end
+
+end:
+  %phi = phi <20 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i32> %phi
+}
+
+define <10 x double> @bitcast_v20i32_to_v10f64(<20 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i32_to_v10f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20i32_to_v10f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB4_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20i32_to_v10f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB4_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20i32_to_v10f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <20 x i32> %a, splat (i32 3)
+  %a2 = bitcast <20 x i32> %a1 to <10 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x i32> %a to <10 x double>
+  br label %end
+
+end:
+  %phi = phi <10 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x double> %phi
+}
+
+define <20 x i32> @bitcast_v10f64_to_v20i32(<10 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f64_to_v20i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f64_to_v20i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f64_to_v20i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f64_to_v20i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <10 x double> %a1 to <20 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x double> %a to <20 x i32>
+  br label %end
+
+end:
+  %phi = phi <20 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i32> %phi
+}
+
+define <40 x i16> @bitcast_v20i32_to_v40i16(<20 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i32_to_v40i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_alignbit_b32 v21, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 0x4c, v0
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v22
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v21
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v24
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v28, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20i32_to_v40i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB6_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB6_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20i32_to_v40i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB6_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB6_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v39, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v38, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v37, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v36, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v35, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v32, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v31, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v30, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v29, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v28, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v27, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v26, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v25, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v24, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v23, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v22, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v21, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20i32_to_v40i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB6_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB6_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v39, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v37, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v28, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v27, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v26, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v25, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v24, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v23, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v22, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v21, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v20, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <20 x i32> %a, splat (i32 3)
+  %a2 = bitcast <20 x i32> %a1 to <40 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x i32> %a to <40 x i16>
+  br label %end
+
+end:
+  %phi = phi <40 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x i16> %phi
+}
+
+define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40i16_to_v20i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v29
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v45
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v46
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v44
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v43
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v42
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v41
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v62
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v20
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v47
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v56
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v57
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v58
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v59
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v60
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v61
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:  .LBB7_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_or_b32_e32 v0, v45, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v46, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v44, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v43, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v42, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v41, v5
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v20, v6
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v20, v7
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v20, v8
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v20, v9
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v10
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v20, v11
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v47, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v56, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v57, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v58, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v59, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v60, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v61, v19
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40i16_to_v20i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v19
+; VI-NEXT:    v_mov_b32_e32 v33, v18
+; VI-NEXT:    v_mov_b32_e32 v34, v17
+; VI-NEXT:    v_mov_b32_e32 v35, v16
+; VI-NEXT:    v_mov_b32_e32 v36, v15
+; VI-NEXT:    v_mov_b32_e32 v37, v14
+; VI-NEXT:    v_mov_b32_e32 v38, v13
+; VI-NEXT:    v_mov_b32_e32 v39, v12
+; VI-NEXT:    v_mov_b32_e32 v48, v11
+; VI-NEXT:    v_mov_b32_e32 v49, v10
+; VI-NEXT:    v_mov_b32_e32 v50, v9
+; VI-NEXT:    v_mov_b32_e32 v51, v8
+; VI-NEXT:    v_mov_b32_e32 v52, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v6
+; VI-NEXT:    v_mov_b32_e32 v54, v5
+; VI-NEXT:    v_mov_b32_e32 v55, v4
+; VI-NEXT:    v_mov_b32_e32 v40, v3
+; VI-NEXT:    v_mov_b32_e32 v41, v2
+; VI-NEXT:    v_mov_b32_e32 v42, v1
+; VI-NEXT:    v_mov_b32_e32 v43, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v19, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v19, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v19, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v19, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v19, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v19, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v19, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v19, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v19, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v19, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v19, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v19, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v19, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v19, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v19, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v19, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v19, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v19, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB7_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v19, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v1, v43, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v2, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v3, v41, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v4, v40, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v5, v55, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v6, v54, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v7, v53, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v8, v52, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v9, v51, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v10, v50, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v11, v49, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v12, v48, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v13, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v14, v38, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v15, v37, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v16, v36, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v17, v35, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v18, v34, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v20, v33, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v19, v32, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:  .LBB7_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40i16_to_v20i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v19
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v18
+; GFX9-NEXT:    v_mov_b32_e32 v43, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v12
+; GFX9-NEXT:    v_mov_b32_e32 v48, v11
+; GFX9-NEXT:    v_mov_b32_e32 v49, v10
+; GFX9-NEXT:    v_mov_b32_e32 v50, v9
+; GFX9-NEXT:    v_mov_b32_e32 v51, v8
+; GFX9-NEXT:    v_mov_b32_e32 v52, v7
+; GFX9-NEXT:    v_mov_b32_e32 v53, v6
+; GFX9-NEXT:    v_mov_b32_e32 v54, v5
+; GFX9-NEXT:    v_mov_b32_e32 v55, v4
+; GFX9-NEXT:    v_mov_b32_e32 v40, v3
+; GFX9-NEXT:    v_mov_b32_e32 v41, v2
+; GFX9-NEXT:    v_mov_b32_e32 v42, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:  .LBB7_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40i16_to_v20i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    v_perm_b32 v0, v37, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v39, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v36, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v35, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v34, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v33, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v32, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v31, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v29, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v28, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v27, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v26, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v25, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v24, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v23, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v22, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v21, v19, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <40 x i16> %a, splat (i16 3)
+  %a2 = bitcast <40 x i16> %a1 to <20 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x i16> %a to <20 x i32>
+  br label %end
+
+end:
+  %phi = phi <20 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i32> %phi
+}
+
+define <40 x half> @bitcast_v20i32_to_v40f16(<20 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20i32_to_v40f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:  .LBB8_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:  .LBB8_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v56
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v42
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v40
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v54
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v52
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v50
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v19
+; GCN-NEXT:    v_or_b32_e32 v13, v38, v50
+; GCN-NEXT:    v_or_b32_e32 v14, v36, v54
+; GCN-NEXT:    v_or_b32_e32 v15, v34, v41
+; GCN-NEXT:    v_or_b32_e32 v16, v32, v55
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v53
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v51
+; GCN-NEXT:    v_or_b32_e32 v20, v27, v49
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v39
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v37
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v35
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v33
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v31
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v29
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20i32_to_v40f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB8_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB8_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20i32_to_v40f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB8_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB8_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v39, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v38, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v37, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v36, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v35, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v32, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v31, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v30, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v29, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v28, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v27, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v26, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v25, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v24, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v23, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v22, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v21, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20i32_to_v40f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB8_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB8_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v39, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v37, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v28, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v27, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v26, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v25, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v24, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v23, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v22, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v21, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v20, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <20 x i32> %a, splat (i32 3)
+  %a2 = bitcast <20 x i32> %a1 to <40 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x i32> %a to <40 x half>
+  br label %end
+
+end:
+  %phi = phi <40 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x half> %phi
+}
+
+define <20 x i32> @bitcast_v40f16_to_v20i32(<40 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40f16_to_v20i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v23
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v25
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v2, v34, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v4, v62, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v5, v60, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v41
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v6, v46, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v44, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v42, v8
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v20, v9
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v10
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v20, v11
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v12
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v20, v13
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v20, v14
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v20, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v53, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v51, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v49, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:  .LBB9_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v34
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v32
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v62
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v60
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v56
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v47
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v45
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v43
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v41
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v40
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v48
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v13, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v14, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v15, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v16, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v17, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v18, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v19, v33, v32
+; GCN-NEXT:  .LBB9_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40f16_to_v20i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v19
+; VI-NEXT:    v_mov_b32_e32 v33, v18
+; VI-NEXT:    v_mov_b32_e32 v34, v17
+; VI-NEXT:    v_mov_b32_e32 v35, v16
+; VI-NEXT:    v_mov_b32_e32 v36, v15
+; VI-NEXT:    v_mov_b32_e32 v37, v14
+; VI-NEXT:    v_mov_b32_e32 v38, v13
+; VI-NEXT:    v_mov_b32_e32 v39, v12
+; VI-NEXT:    v_mov_b32_e32 v48, v11
+; VI-NEXT:    v_mov_b32_e32 v49, v10
+; VI-NEXT:    v_mov_b32_e32 v50, v9
+; VI-NEXT:    v_mov_b32_e32 v51, v8
+; VI-NEXT:    v_mov_b32_e32 v52, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v6
+; VI-NEXT:    v_mov_b32_e32 v54, v5
+; VI-NEXT:    v_mov_b32_e32 v55, v4
+; VI-NEXT:    v_mov_b32_e32 v40, v3
+; VI-NEXT:    v_mov_b32_e32 v41, v2
+; VI-NEXT:    v_mov_b32_e32 v42, v1
+; VI-NEXT:    v_mov_b32_e32 v43, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v19, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v19, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v19, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v19, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v19, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v19, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v19, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v19, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v19, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v19, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v19, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v19, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v19, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v19, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v19, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v19, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v19, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v19, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB9_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v19, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v43, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v41, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v40, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v55, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v54, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v53, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v52, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v51, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v50, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v49, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v48, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v38, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v37, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v36, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v35, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v34, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v33, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v18, v20, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v32, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:  .LBB9_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40f16_to_v20i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v19
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v18
+; GFX9-NEXT:    v_mov_b32_e32 v43, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v12
+; GFX9-NEXT:    v_mov_b32_e32 v48, v11
+; GFX9-NEXT:    v_mov_b32_e32 v49, v10
+; GFX9-NEXT:    v_mov_b32_e32 v50, v9
+; GFX9-NEXT:    v_mov_b32_e32 v51, v8
+; GFX9-NEXT:    v_mov_b32_e32 v52, v7
+; GFX9-NEXT:    v_mov_b32_e32 v53, v6
+; GFX9-NEXT:    v_mov_b32_e32 v54, v5
+; GFX9-NEXT:    v_mov_b32_e32 v55, v4
+; GFX9-NEXT:    v_mov_b32_e32 v40, v3
+; GFX9-NEXT:    v_mov_b32_e32 v41, v2
+; GFX9-NEXT:    v_mov_b32_e32 v42, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:  .LBB9_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40f16_to_v20i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    v_perm_b32 v0, v37, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v39, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v36, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v35, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v34, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v33, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v32, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v31, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v29, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v28, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v27, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v26, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v25, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v24, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v23, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v22, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v21, v19, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <40 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <40 x half> %a1 to <20 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x half> %a to <20 x i32>
+  br label %end
+
+end:
+  %phi = phi <20 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x i32> %phi
+}
+
+define <10 x i64> @bitcast_v20f32_to_v10i64(<20 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20f32_to_v10i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20f32_to_v10i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20f32_to_v10i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20f32_to_v10i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <20 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <20 x float> %a1 to <10 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x float> %a to <10 x i64>
+  br label %end
+
+end:
+  %phi = phi <10 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x i64> %phi
+}
+
+define <20 x float> @bitcast_v10i64_to_v20f32(<10 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i64_to_v20f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i64_to_v20f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i64_to_v20f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i64_to_v20f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i64> %a, splat (i64 3)
+  %a2 = bitcast <10 x i64> %a1 to <20 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i64> %a to <20 x float>
+  br label %end
+
+end:
+  %phi = phi <20 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x float> %phi
+}
+
+define <10 x double> @bitcast_v20f32_to_v10f64(<20 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20f32_to_v10f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20f32_to_v10f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20f32_to_v10f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20f32_to_v10f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB12_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <20 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <20 x float> %a1 to <10 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x float> %a to <10 x double>
+  br label %end
+
+end:
+  %phi = phi <10 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x double> %phi
+}
+
+define <20 x float> @bitcast_v10f64_to_v20f32(<10 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f64_to_v20f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f64_to_v20f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f64_to_v20f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f64_to_v20f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <10 x double> %a1 to <20 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x double> %a to <20 x float>
+  br label %end
+
+end:
+  %phi = phi <20 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x float> %phi
+}
+
+define <40 x i16> @bitcast_v20f32_to_v40i16(<20 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20f32_to_v40i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_alignbit_b32 v21, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 0x4c, v0
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v22
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v21
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v24
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v28, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20f32_to_v40i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB14_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB14_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20f32_to_v40i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB14_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB14_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v39, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v38, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v37, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v36, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v35, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v32, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v31, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v30, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v29, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v28, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v27, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v26, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v25, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v24, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v23, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v22, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v21, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20f32_to_v40i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB14_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB14_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v39, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v37, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v28, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v27, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v26, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v25, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v24, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v23, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v22, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v21, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v20, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <20 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <20 x float> %a1 to <40 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x float> %a to <40 x i16>
+  br label %end
+
+end:
+  %phi = phi <40 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x i16> %phi
+}
+
+define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40i16_to_v20f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v29
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v45
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v46
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v44
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v43
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v42
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v41
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v62
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v20
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v47
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v56
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v57
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v58
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v59
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v60
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v61
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:  .LBB15_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_or_b32_e32 v0, v45, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v46, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v44, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v43, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v42, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v41, v5
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v20, v6
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v20, v7
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v20, v8
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v20, v9
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v10
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v20, v11
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v47, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v56, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v57, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v58, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v59, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v60, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v61, v19
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:  .LBB15_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40i16_to_v20f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v19
+; VI-NEXT:    v_mov_b32_e32 v33, v18
+; VI-NEXT:    v_mov_b32_e32 v34, v17
+; VI-NEXT:    v_mov_b32_e32 v35, v16
+; VI-NEXT:    v_mov_b32_e32 v36, v15
+; VI-NEXT:    v_mov_b32_e32 v37, v14
+; VI-NEXT:    v_mov_b32_e32 v38, v13
+; VI-NEXT:    v_mov_b32_e32 v39, v12
+; VI-NEXT:    v_mov_b32_e32 v48, v11
+; VI-NEXT:    v_mov_b32_e32 v49, v10
+; VI-NEXT:    v_mov_b32_e32 v50, v9
+; VI-NEXT:    v_mov_b32_e32 v51, v8
+; VI-NEXT:    v_mov_b32_e32 v52, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v6
+; VI-NEXT:    v_mov_b32_e32 v54, v5
+; VI-NEXT:    v_mov_b32_e32 v55, v4
+; VI-NEXT:    v_mov_b32_e32 v40, v3
+; VI-NEXT:    v_mov_b32_e32 v41, v2
+; VI-NEXT:    v_mov_b32_e32 v42, v1
+; VI-NEXT:    v_mov_b32_e32 v43, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v19, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v19, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v19, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v19, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v19, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v19, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v19, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v19, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v19, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v19, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v19, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v19, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v19, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v19, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v19, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v19, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v19, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v19, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB15_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v19, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v1, v43, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v2, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v3, v41, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v4, v40, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v5, v55, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v6, v54, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v7, v53, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v8, v52, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v9, v51, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v10, v50, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v11, v49, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v12, v48, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v13, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v14, v38, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v15, v37, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v16, v36, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v17, v35, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v18, v34, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v20, v33, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v19, v32, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:  .LBB15_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40i16_to_v20f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v19
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v18
+; GFX9-NEXT:    v_mov_b32_e32 v43, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v12
+; GFX9-NEXT:    v_mov_b32_e32 v48, v11
+; GFX9-NEXT:    v_mov_b32_e32 v49, v10
+; GFX9-NEXT:    v_mov_b32_e32 v50, v9
+; GFX9-NEXT:    v_mov_b32_e32 v51, v8
+; GFX9-NEXT:    v_mov_b32_e32 v52, v7
+; GFX9-NEXT:    v_mov_b32_e32 v53, v6
+; GFX9-NEXT:    v_mov_b32_e32 v54, v5
+; GFX9-NEXT:    v_mov_b32_e32 v55, v4
+; GFX9-NEXT:    v_mov_b32_e32 v40, v3
+; GFX9-NEXT:    v_mov_b32_e32 v41, v2
+; GFX9-NEXT:    v_mov_b32_e32 v42, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:  .LBB15_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40i16_to_v20f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    v_perm_b32 v0, v37, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v39, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v36, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v35, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v34, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v33, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v32, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v31, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v29, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v28, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v27, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v26, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v25, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v24, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v23, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v22, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v21, v19, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <40 x i16> %a, splat (i16 3)
+  %a2 = bitcast <40 x i16> %a1 to <20 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x i16> %a to <20 x float>
+  br label %end
+
+end:
+  %phi = phi <20 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x float> %phi
+}
+
+define <40 x half> @bitcast_v20f32_to_v40f16(<20 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v20f32_to_v40f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:  .LBB16_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:  .LBB16_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v56
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v42
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v40
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v54
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v52
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v50
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v19
+; GCN-NEXT:    v_or_b32_e32 v13, v38, v50
+; GCN-NEXT:    v_or_b32_e32 v14, v36, v54
+; GCN-NEXT:    v_or_b32_e32 v15, v34, v41
+; GCN-NEXT:    v_or_b32_e32 v16, v32, v55
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v53
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v51
+; GCN-NEXT:    v_or_b32_e32 v20, v27, v49
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v39
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v37
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v35
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v33
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v31
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v29
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v20f32_to_v40f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB16_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB16_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v20f32_to_v40f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB16_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB16_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v39, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v38, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v37, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v36, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v35, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v32, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v31, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v30, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v29, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v28, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v27, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v26, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v25, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v24, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v23, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v22, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v21, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v20f32_to_v40f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB16_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB16_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v39, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v37, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v28, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v27, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v26, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v25, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v24, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v23, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v22, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v21, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v20, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <20 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <20 x float> %a1 to <40 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <20 x float> %a to <40 x half>
+  br label %end
+
+end:
+  %phi = phi <40 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x half> %phi
+}
+
+define <20 x float> @bitcast_v40f16_to_v20f32(<40 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40f16_to_v20f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v23
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v25
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v2, v34, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v4, v62, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v5, v60, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v41
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v6, v46, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v44, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v42, v8
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v20, v9
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v10
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v20, v11
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v12
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v20, v13
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v20, v14
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v20, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v53, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v51, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v49, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v34
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v32
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v62
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v60
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v56
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v47
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v45
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v43
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v41
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v40
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v48
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v13, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v14, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v15, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v16, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v17, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v18, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v19, v33, v32
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40f16_to_v20f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v19
+; VI-NEXT:    v_mov_b32_e32 v33, v18
+; VI-NEXT:    v_mov_b32_e32 v34, v17
+; VI-NEXT:    v_mov_b32_e32 v35, v16
+; VI-NEXT:    v_mov_b32_e32 v36, v15
+; VI-NEXT:    v_mov_b32_e32 v37, v14
+; VI-NEXT:    v_mov_b32_e32 v38, v13
+; VI-NEXT:    v_mov_b32_e32 v39, v12
+; VI-NEXT:    v_mov_b32_e32 v48, v11
+; VI-NEXT:    v_mov_b32_e32 v49, v10
+; VI-NEXT:    v_mov_b32_e32 v50, v9
+; VI-NEXT:    v_mov_b32_e32 v51, v8
+; VI-NEXT:    v_mov_b32_e32 v52, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v6
+; VI-NEXT:    v_mov_b32_e32 v54, v5
+; VI-NEXT:    v_mov_b32_e32 v55, v4
+; VI-NEXT:    v_mov_b32_e32 v40, v3
+; VI-NEXT:    v_mov_b32_e32 v41, v2
+; VI-NEXT:    v_mov_b32_e32 v42, v1
+; VI-NEXT:    v_mov_b32_e32 v43, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v19, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v19, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v19, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v19, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v19, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v19, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v19, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v19, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v19, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v19, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v19, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v19, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v19, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v19, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v19, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v19, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v19, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v19, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB17_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v19, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v43, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v41, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v40, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v55, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v54, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v53, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v52, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v51, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v50, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v49, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v48, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v38, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v37, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v36, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v35, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v34, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v33, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v18, v20, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v32, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:  .LBB17_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40f16_to_v20f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v19
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v18
+; GFX9-NEXT:    v_mov_b32_e32 v43, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v12
+; GFX9-NEXT:    v_mov_b32_e32 v48, v11
+; GFX9-NEXT:    v_mov_b32_e32 v49, v10
+; GFX9-NEXT:    v_mov_b32_e32 v50, v9
+; GFX9-NEXT:    v_mov_b32_e32 v51, v8
+; GFX9-NEXT:    v_mov_b32_e32 v52, v7
+; GFX9-NEXT:    v_mov_b32_e32 v53, v6
+; GFX9-NEXT:    v_mov_b32_e32 v54, v5
+; GFX9-NEXT:    v_mov_b32_e32 v55, v4
+; GFX9-NEXT:    v_mov_b32_e32 v40, v3
+; GFX9-NEXT:    v_mov_b32_e32 v41, v2
+; GFX9-NEXT:    v_mov_b32_e32 v42, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:  .LBB17_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40f16_to_v20f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    v_perm_b32 v0, v37, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v39, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v36, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v35, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v34, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v33, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v32, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v31, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v29, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v28, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v27, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v26, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v25, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v24, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v23, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v22, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v21, v19, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <40 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <40 x half> %a1 to <20 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x half> %a to <20 x float>
+  br label %end
+
+end:
+  %phi = phi <20 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <20 x float> %phi
+}
+
+define <10 x double> @bitcast_v10i64_to_v10f64(<10 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i64_to_v10f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i64_to_v10f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i64_to_v10f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i64_to_v10f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i64> %a, splat (i64 3)
+  %a2 = bitcast <10 x i64> %a1 to <10 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i64> %a to <10 x double>
+  br label %end
+
+end:
+  %phi = phi <10 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x double> %phi
+}
+
+define <10 x i64> @bitcast_v10f64_to_v10i64(<10 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f64_to_v10i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f64_to_v10i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f64_to_v10i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f64_to_v10i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <10 x double> %a1 to <10 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x double> %a to <10 x i64>
+  br label %end
+
+end:
+  %phi = phi <10 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x i64> %phi
+}
+
+define <40 x i16> @bitcast_v10i64_to_v40i16(<10 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i64_to_v40i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_alignbit_b32 v21, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 0x4c, v0
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v22
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v27
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v21
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v25
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v28, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i64_to_v40i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB20_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB20_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i64_to_v40i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB20_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB20_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v39, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v38, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v37, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v36, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v35, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v32, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v31, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v30, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v29, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v28, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v27, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v26, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v25, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v24, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v23, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v22, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v21, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i64_to_v40i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB20_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB20_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v39, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v37, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v28, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v27, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v26, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v25, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v24, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v23, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v22, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v21, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v20, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i64> %a, splat (i64 3)
+  %a2 = bitcast <10 x i64> %a1 to <40 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i64> %a to <40 x i16>
+  br label %end
+
+end:
+  %phi = phi <40 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x i16> %phi
+}
+
+define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40i16_to_v10i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v29
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v45
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v46
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v44
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v43
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v42
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v41
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v62
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v20
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v47
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v56
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v57
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v58
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v59
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v60
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v61
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:  .LBB21_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_or_b32_e32 v0, v45, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v46, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v44, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v43, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v42, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v41, v5
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v20, v6
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v20, v7
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v20, v8
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v20, v9
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v10
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v20, v11
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v47, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v56, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v57, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v58, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v59, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v60, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v61, v19
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:  .LBB21_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40i16_to_v10i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v19
+; VI-NEXT:    v_mov_b32_e32 v33, v18
+; VI-NEXT:    v_mov_b32_e32 v34, v17
+; VI-NEXT:    v_mov_b32_e32 v35, v16
+; VI-NEXT:    v_mov_b32_e32 v36, v15
+; VI-NEXT:    v_mov_b32_e32 v37, v14
+; VI-NEXT:    v_mov_b32_e32 v38, v13
+; VI-NEXT:    v_mov_b32_e32 v39, v12
+; VI-NEXT:    v_mov_b32_e32 v48, v11
+; VI-NEXT:    v_mov_b32_e32 v49, v10
+; VI-NEXT:    v_mov_b32_e32 v50, v9
+; VI-NEXT:    v_mov_b32_e32 v51, v8
+; VI-NEXT:    v_mov_b32_e32 v52, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v6
+; VI-NEXT:    v_mov_b32_e32 v54, v5
+; VI-NEXT:    v_mov_b32_e32 v55, v4
+; VI-NEXT:    v_mov_b32_e32 v40, v3
+; VI-NEXT:    v_mov_b32_e32 v41, v2
+; VI-NEXT:    v_mov_b32_e32 v42, v1
+; VI-NEXT:    v_mov_b32_e32 v43, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v19, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v19, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v19, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v19, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v19, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v19, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v19, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v19, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v19, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v19, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v19, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v19, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v19, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v19, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v19, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v19, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v19, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v19, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB21_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v19, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v1, v43, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v2, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v3, v41, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v4, v40, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v5, v55, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v6, v54, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v7, v53, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v8, v52, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v9, v51, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v10, v50, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v11, v49, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v12, v48, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v13, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v14, v38, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v15, v37, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v16, v36, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v17, v35, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v18, v34, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v20, v33, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v19, v32, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:  .LBB21_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40i16_to_v10i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v19
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v18
+; GFX9-NEXT:    v_mov_b32_e32 v43, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v12
+; GFX9-NEXT:    v_mov_b32_e32 v48, v11
+; GFX9-NEXT:    v_mov_b32_e32 v49, v10
+; GFX9-NEXT:    v_mov_b32_e32 v50, v9
+; GFX9-NEXT:    v_mov_b32_e32 v51, v8
+; GFX9-NEXT:    v_mov_b32_e32 v52, v7
+; GFX9-NEXT:    v_mov_b32_e32 v53, v6
+; GFX9-NEXT:    v_mov_b32_e32 v54, v5
+; GFX9-NEXT:    v_mov_b32_e32 v55, v4
+; GFX9-NEXT:    v_mov_b32_e32 v40, v3
+; GFX9-NEXT:    v_mov_b32_e32 v41, v2
+; GFX9-NEXT:    v_mov_b32_e32 v42, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:  .LBB21_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40i16_to_v10i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    v_perm_b32 v0, v37, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v39, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v36, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v35, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v34, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v33, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v32, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v31, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v29, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v28, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v27, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v26, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v25, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v24, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v23, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v22, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v21, v19, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <40 x i16> %a, splat (i16 3)
+  %a2 = bitcast <40 x i16> %a1 to <10 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x i16> %a to <10 x i64>
+  br label %end
+
+end:
+  %phi = phi <10 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x i64> %phi
+}
+
+define <40 x half> @bitcast_v10i64_to_v40f16(<10 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10i64_to_v40f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:  .LBB22_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:  .LBB22_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v56
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v42
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v40
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v54
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v52
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v50
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v19
+; GCN-NEXT:    v_or_b32_e32 v13, v38, v50
+; GCN-NEXT:    v_or_b32_e32 v14, v36, v54
+; GCN-NEXT:    v_or_b32_e32 v15, v34, v41
+; GCN-NEXT:    v_or_b32_e32 v16, v32, v55
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v53
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v51
+; GCN-NEXT:    v_or_b32_e32 v20, v27, v49
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v39
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v37
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v35
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v33
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v31
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v29
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10i64_to_v40f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB22_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB22_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10i64_to_v40f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB22_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB22_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v39, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v38, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v37, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v36, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v35, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v32, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v31, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v30, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v29, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v28, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v27, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v26, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v25, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v24, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v23, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v22, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v21, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10i64_to_v40f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB22_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB22_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v39, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v37, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v28, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v27, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v26, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v25, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v24, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v23, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v22, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v21, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v20, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <10 x i64> %a, splat (i64 3)
+  %a2 = bitcast <10 x i64> %a1 to <40 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x i64> %a to <40 x half>
+  br label %end
+
+end:
+  %phi = phi <40 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x half> %phi
+}
+
+define <10 x i64> @bitcast_v40f16_to_v10i64(<40 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40f16_to_v10i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v23
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v25
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v2, v34, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v4, v62, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v5, v60, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v41
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v6, v46, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v44, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v42, v8
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v20, v9
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v10
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v20, v11
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v12
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v20, v13
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v20, v14
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v20, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v53, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v51, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v49, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:  .LBB23_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v34
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v32
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v62
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v60
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v56
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v47
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v45
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v43
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v41
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v40
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v48
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v13, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v14, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v15, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v16, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v17, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v18, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v19, v33, v32
+; GCN-NEXT:  .LBB23_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40f16_to_v10i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v19
+; VI-NEXT:    v_mov_b32_e32 v33, v18
+; VI-NEXT:    v_mov_b32_e32 v34, v17
+; VI-NEXT:    v_mov_b32_e32 v35, v16
+; VI-NEXT:    v_mov_b32_e32 v36, v15
+; VI-NEXT:    v_mov_b32_e32 v37, v14
+; VI-NEXT:    v_mov_b32_e32 v38, v13
+; VI-NEXT:    v_mov_b32_e32 v39, v12
+; VI-NEXT:    v_mov_b32_e32 v48, v11
+; VI-NEXT:    v_mov_b32_e32 v49, v10
+; VI-NEXT:    v_mov_b32_e32 v50, v9
+; VI-NEXT:    v_mov_b32_e32 v51, v8
+; VI-NEXT:    v_mov_b32_e32 v52, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v6
+; VI-NEXT:    v_mov_b32_e32 v54, v5
+; VI-NEXT:    v_mov_b32_e32 v55, v4
+; VI-NEXT:    v_mov_b32_e32 v40, v3
+; VI-NEXT:    v_mov_b32_e32 v41, v2
+; VI-NEXT:    v_mov_b32_e32 v42, v1
+; VI-NEXT:    v_mov_b32_e32 v43, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v19, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v19, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v19, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v19, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v19, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v19, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v19, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v19, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v19, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v19, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v19, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v19, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v19, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v19, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v19, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v19, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v19, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v19, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB23_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v19, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v43, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v41, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v40, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v55, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v54, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v53, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v52, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v51, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v50, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v49, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v48, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v38, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v37, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v36, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v35, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v34, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v33, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v18, v20, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v32, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:  .LBB23_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40f16_to_v10i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v19
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v18
+; GFX9-NEXT:    v_mov_b32_e32 v43, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v12
+; GFX9-NEXT:    v_mov_b32_e32 v48, v11
+; GFX9-NEXT:    v_mov_b32_e32 v49, v10
+; GFX9-NEXT:    v_mov_b32_e32 v50, v9
+; GFX9-NEXT:    v_mov_b32_e32 v51, v8
+; GFX9-NEXT:    v_mov_b32_e32 v52, v7
+; GFX9-NEXT:    v_mov_b32_e32 v53, v6
+; GFX9-NEXT:    v_mov_b32_e32 v54, v5
+; GFX9-NEXT:    v_mov_b32_e32 v55, v4
+; GFX9-NEXT:    v_mov_b32_e32 v40, v3
+; GFX9-NEXT:    v_mov_b32_e32 v41, v2
+; GFX9-NEXT:    v_mov_b32_e32 v42, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:  .LBB23_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40f16_to_v10i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    v_perm_b32 v0, v37, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v39, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v36, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v35, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v34, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v33, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v32, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v31, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v29, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v28, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v27, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v26, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v25, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v24, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v23, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v22, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v21, v19, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <40 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <40 x half> %a1 to <10 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x half> %a to <10 x i64>
+  br label %end
+
+end:
+  %phi = phi <10 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x i64> %phi
+}
+
+define <40 x i16> @bitcast_v10f64_to_v40i16(<10 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f64_to_v40i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v21, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_alignbit_b32 v21, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v49, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_and_b32_e32 v50, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v38, v49, v38
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v48, v50, v48
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 20, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 24, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 28, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 32, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 36, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 40, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 52, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 56, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x4c, v0
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v22
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v30
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v21
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    buffer_store_dword v38, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v31, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f64_to_v40i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB24_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB24_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f64_to_v40i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB24_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB24_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v39, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v38, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v37, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v36, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v35, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v32, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v31, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v30, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v29, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v28, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v27, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v26, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v25, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v24, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v23, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v22, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v21, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f64_to_v40i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB24_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB24_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v39, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v37, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v28, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v27, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v26, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v25, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v24, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v23, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v22, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v21, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v20, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <10 x double> %a1 to <40 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x double> %a to <40 x i16>
+  br label %end
+
+end:
+  %phi = phi <40 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x i16> %phi
+}
+
+define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40i16_to_v10f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v29
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v45
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v46
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v44
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v43
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v42
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v41
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v62
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v20
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v47
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v56
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v57
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v58
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v59
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v60
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v61
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:  .LBB25_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_or_b32_e32 v0, v45, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v46, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v44, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v43, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v42, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v41, v5
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v20, v6
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v20, v7
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v20, v8
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v20, v9
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v10
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v20, v11
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v47, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v56, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v57, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v58, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v59, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v60, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v61, v19
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:  .LBB25_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40i16_to_v10f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v19
+; VI-NEXT:    v_mov_b32_e32 v33, v18
+; VI-NEXT:    v_mov_b32_e32 v34, v17
+; VI-NEXT:    v_mov_b32_e32 v35, v16
+; VI-NEXT:    v_mov_b32_e32 v36, v15
+; VI-NEXT:    v_mov_b32_e32 v37, v14
+; VI-NEXT:    v_mov_b32_e32 v38, v13
+; VI-NEXT:    v_mov_b32_e32 v39, v12
+; VI-NEXT:    v_mov_b32_e32 v48, v11
+; VI-NEXT:    v_mov_b32_e32 v49, v10
+; VI-NEXT:    v_mov_b32_e32 v50, v9
+; VI-NEXT:    v_mov_b32_e32 v51, v8
+; VI-NEXT:    v_mov_b32_e32 v52, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v6
+; VI-NEXT:    v_mov_b32_e32 v54, v5
+; VI-NEXT:    v_mov_b32_e32 v55, v4
+; VI-NEXT:    v_mov_b32_e32 v40, v3
+; VI-NEXT:    v_mov_b32_e32 v41, v2
+; VI-NEXT:    v_mov_b32_e32 v42, v1
+; VI-NEXT:    v_mov_b32_e32 v43, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v19, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v19, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v19, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v19, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v19, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v19, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v19, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v19, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v19, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v19, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v19, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v19, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v19, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v19, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v19, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v19, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v19, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v19, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB25_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v19, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v1, v43, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v2, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v3, v41, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v4, v40, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v5, v55, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v6, v54, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v7, v53, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v8, v52, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v9, v51, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v10, v50, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v11, v49, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v12, v48, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v13, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v14, v38, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v15, v37, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v16, v36, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v17, v35, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v18, v34, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v20, v33, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v19, v32, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:  .LBB25_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40i16_to_v10f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v19
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v18
+; GFX9-NEXT:    v_mov_b32_e32 v43, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v12
+; GFX9-NEXT:    v_mov_b32_e32 v48, v11
+; GFX9-NEXT:    v_mov_b32_e32 v49, v10
+; GFX9-NEXT:    v_mov_b32_e32 v50, v9
+; GFX9-NEXT:    v_mov_b32_e32 v51, v8
+; GFX9-NEXT:    v_mov_b32_e32 v52, v7
+; GFX9-NEXT:    v_mov_b32_e32 v53, v6
+; GFX9-NEXT:    v_mov_b32_e32 v54, v5
+; GFX9-NEXT:    v_mov_b32_e32 v55, v4
+; GFX9-NEXT:    v_mov_b32_e32 v40, v3
+; GFX9-NEXT:    v_mov_b32_e32 v41, v2
+; GFX9-NEXT:    v_mov_b32_e32 v42, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:  .LBB25_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40i16_to_v10f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    v_perm_b32 v0, v37, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v39, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v36, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v35, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v34, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v33, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v32, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v31, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v29, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v28, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v27, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v26, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v25, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v24, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v23, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v22, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v21, v19, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <40 x i16> %a, splat (i16 3)
+  %a2 = bitcast <40 x i16> %a1 to <10 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x i16> %a to <10 x double>
+  br label %end
+
+end:
+  %phi = phi <10 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x double> %phi
+}
+
+define <40 x half> @bitcast_v10f64_to_v40f16(<10 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v10f64_to_v40f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v21
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v56
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v42
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v40
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v54
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v52
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v50
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v19
+; GCN-NEXT:    v_or_b32_e32 v13, v38, v50
+; GCN-NEXT:    v_or_b32_e32 v14, v36, v54
+; GCN-NEXT:    v_or_b32_e32 v15, v34, v41
+; GCN-NEXT:    v_or_b32_e32 v16, v32, v55
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v53
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v51
+; GCN-NEXT:    v_or_b32_e32 v20, v27, v49
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v39
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v37
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v35
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v33
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v31
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v29
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v10f64_to_v40f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB26_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:  .LBB26_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v10f64_to_v40f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB26_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX9-NEXT:  .LBB26_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v39, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v38, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v37, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v36, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v35, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v32, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v31, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v30, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v29, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v28, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v27, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v26, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v25, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v24, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v23, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v22, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v21, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v10f64_to_v40f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    ; implicit-def: $vgpr21
+; GFX11-NEXT:    ; implicit-def: $vgpr20
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB26_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-NEXT:  .LBB26_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v39, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v37, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v36, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v29, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v28, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v27, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v26, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v25, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v24, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v23, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v22, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v21, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v20, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <10 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <10 x double> %a1 to <40 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <10 x double> %a to <40 x half>
+  br label %end
+
+end:
+  %phi = phi <40 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x half> %phi
+}
+
+define <10 x double> @bitcast_v40f16_to_v10f64(<40 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40f16_to_v10f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v23
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v25
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v2, v34, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v4, v62, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v5, v60, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v41
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v6, v46, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v44, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v42, v8
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v20, v9
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v20, v10
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v20, v11
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v12
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v20, v13
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v20, v14
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v20, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v53, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v51, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v49, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; kill: killed $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v34
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v32
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v62
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v60
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v56
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v47
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v45
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v43
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v41
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v40
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v48
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v13, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v14, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v15, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v16, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v17, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v18, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v19, v33, v32
+; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40f16_to_v10f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v19
+; VI-NEXT:    v_mov_b32_e32 v33, v18
+; VI-NEXT:    v_mov_b32_e32 v34, v17
+; VI-NEXT:    v_mov_b32_e32 v35, v16
+; VI-NEXT:    v_mov_b32_e32 v36, v15
+; VI-NEXT:    v_mov_b32_e32 v37, v14
+; VI-NEXT:    v_mov_b32_e32 v38, v13
+; VI-NEXT:    v_mov_b32_e32 v39, v12
+; VI-NEXT:    v_mov_b32_e32 v48, v11
+; VI-NEXT:    v_mov_b32_e32 v49, v10
+; VI-NEXT:    v_mov_b32_e32 v50, v9
+; VI-NEXT:    v_mov_b32_e32 v51, v8
+; VI-NEXT:    v_mov_b32_e32 v52, v7
+; VI-NEXT:    v_mov_b32_e32 v53, v6
+; VI-NEXT:    v_mov_b32_e32 v54, v5
+; VI-NEXT:    v_mov_b32_e32 v55, v4
+; VI-NEXT:    v_mov_b32_e32 v40, v3
+; VI-NEXT:    v_mov_b32_e32 v41, v2
+; VI-NEXT:    v_mov_b32_e32 v42, v1
+; VI-NEXT:    v_mov_b32_e32 v43, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v19, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v19, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v19, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v19, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v19, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v19, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v19, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v19, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v19, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v19, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v19, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v19, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v19, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v19, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v19, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v19, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v19, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v19, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB27_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v19, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v43, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v42, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v41, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v40, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v55, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v54, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v53, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v52, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v51, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v50, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v49, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v48, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v39, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v38, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v37, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v36, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v35, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v34, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v33, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v18, v20, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v32, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:  .LBB27_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40f16_to_v10f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v19
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v18
+; GFX9-NEXT:    v_mov_b32_e32 v43, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v12
+; GFX9-NEXT:    v_mov_b32_e32 v48, v11
+; GFX9-NEXT:    v_mov_b32_e32 v49, v10
+; GFX9-NEXT:    v_mov_b32_e32 v50, v9
+; GFX9-NEXT:    v_mov_b32_e32 v51, v8
+; GFX9-NEXT:    v_mov_b32_e32 v52, v7
+; GFX9-NEXT:    v_mov_b32_e32 v53, v6
+; GFX9-NEXT:    v_mov_b32_e32 v54, v5
+; GFX9-NEXT:    v_mov_b32_e32 v55, v4
+; GFX9-NEXT:    v_mov_b32_e32 v40, v3
+; GFX9-NEXT:    v_mov_b32_e32 v41, v2
+; GFX9-NEXT:    v_mov_b32_e32 v42, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; kill: killed $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:  .LBB27_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v43, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v2, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v3, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v4, v47, v55, s6
+; GFX9-NEXT:    v_perm_b32 v5, v46, v54, s6
+; GFX9-NEXT:    v_perm_b32 v6, v45, v53, s6
+; GFX9-NEXT:    v_perm_b32 v7, v44, v52, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v51, s6
+; GFX9-NEXT:    v_perm_b32 v9, v62, v50, s6
+; GFX9-NEXT:    v_perm_b32 v10, v61, v49, s6
+; GFX9-NEXT:    v_perm_b32 v11, v60, v48, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40f16_to_v10f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX11-NEXT:    v_perm_b32 v0, v37, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v38, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v39, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v36, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v35, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v34, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v33, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v32, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v31, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v29, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v28, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v27, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v26, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v25, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v24, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v23, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v22, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v21, v19, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <40 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <40 x half> %a1 to <10 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x half> %a to <10 x double>
+  br label %end
+
+end:
+  %phi = phi <10 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <10 x double> %phi
+}
+
+define <40 x half> @bitcast_v40i16_to_v40f16(<40 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40i16_to_v40f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; kill: killed $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v24
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v30
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v39
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v55
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:  .LBB28_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 3, v55
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 3, v54
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 3, v53
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 3, v52
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 3, v51
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 3, v50
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 3, v49
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v24
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v30
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v43
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v44
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v45
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v3
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v46
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v47
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v56
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v57
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v58
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v59
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v60
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v61
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v62
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v63
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v8, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v10, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v11, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v12, v20, v19
+; GCN-NEXT:    v_or_b32_e32 v13, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v25
+; GCN-NEXT:    v_or_b32_e32 v15, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v16, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v17, v50, v49
+; GCN-NEXT:    v_or_b32_e32 v19, v53, v52
+; GCN-NEXT:    v_or_b32_e32 v20, v55, v31
+; GCN-NEXT:    v_or_b32_e32 v22, v44, v32
+; GCN-NEXT:    v_or_b32_e32 v23, v46, v33
+; GCN-NEXT:    v_or_b32_e32 v25, v56, v34
+; GCN-NEXT:    v_or_b32_e32 v26, v40, v35
+; GCN-NEXT:    v_or_b32_e32 v28, v41, v36
+; GCN-NEXT:    v_or_b32_e32 v29, v42, v37
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40i16_to_v40f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_add_u16_e32 v39, 3, v39
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v38, 3, v38
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v37, 3, v37
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v36, 3, v36
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v35, 3, v35
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v34, 3, v34
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v33, 3, v33
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v32, 3, v32
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_e32 v31, 3, v31
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_add_u16_e32 v30, 3, v30
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_add_u16_e32 v29, 3, v29
+; VI-NEXT:    v_add_u16_e32 v12, 3, v12
+; VI-NEXT:    v_add_u16_e32 v28, 3, v28
+; VI-NEXT:    v_add_u16_e32 v13, 3, v13
+; VI-NEXT:    v_add_u16_e32 v27, 3, v27
+; VI-NEXT:    v_add_u16_e32 v14, 3, v14
+; VI-NEXT:    v_add_u16_e32 v26, 3, v26
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    v_add_u16_e32 v25, 3, v25
+; VI-NEXT:    v_add_u16_e32 v16, 3, v16
+; VI-NEXT:    v_add_u16_e32 v24, 3, v24
+; VI-NEXT:    v_add_u16_e32 v17, 3, v17
+; VI-NEXT:    v_add_u16_e32 v23, 3, v23
+; VI-NEXT:    v_add_u16_e32 v18, 3, v18
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v19, 3, v19
+; VI-NEXT:    v_add_u16_e32 v21, 3, v21
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v25
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v23
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40i16_to_v40f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v19, v39, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v38, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v37, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v36, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v35, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v34, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v33, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v32, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v31, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v30, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v29, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v28, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v27, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v26, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v24, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v25, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v23, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v22, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v21, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v20, v0, s6
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v19
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v20, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v21, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v22, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v23, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v25, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v24, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v26, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v27, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v28, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v29, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v30, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v31, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v32, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v33, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v34, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v35, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v36, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v37, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v38, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v39, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40i16_to_v40f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v39, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v37, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v35, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v33, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v31, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v29, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v27, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v26, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v21, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v23, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v24, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v25, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v21, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v23, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v24, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v25, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v26, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v27, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v29, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v31, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v33, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v35, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v37, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v39, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <40 x i16> %a, splat (i16 3)
+  %a2 = bitcast <40 x i16> %a1 to <40 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x i16> %a to <40 x half>
+  br label %end
+
+end:
+  %phi = phi <40 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x half> %phi
+}
+
+define <40 x i16> @bitcast_v40f16_to_v40i16(<40 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v40f16_to_v40i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v30
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v40
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v53, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v51, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v49, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v39, v39, v30
+; GCN-NEXT:    v_or_b32_e32 v37, v37, v48
+; GCN-NEXT:    v_or_b32_e32 v36, v36, v38
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v23
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v22
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v20
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v18
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v54
+; GCN-NEXT:    v_or_b32_e32 v35, v35, v52
+; GCN-NEXT:    v_or_b32_e32 v34, v34, v55
+; GCN-NEXT:    v_or_b32_e32 v33, v33, v50
+; GCN-NEXT:    v_alignbit_b32 v55, v33, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v34, v26, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v35, v28, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v15, v30, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v14, v48, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v13, v38, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v12, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v11, v22, 16
+; GCN-NEXT:    v_alignbit_b32 v20, v10, v20, 16
+; GCN-NEXT:    v_alignbit_b32 v18, v9, v18, 16
+; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v55
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v31
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v54
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v33, 0xffff, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v49, 0xffff, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v35, 0xffff, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v39, 0xffff, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v37, 0xffff, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v49, v49, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v8, v35, v8
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v36, 0xffff, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v39, v39, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v7, v15, v7
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v37, v37, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v6, v14, v6
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v36, v36, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v5, v13, v5
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v4, v12, v4
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v3, v11, v3
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 0x4c, v0
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v20
+; GCN-NEXT:    v_or_b32_e32 v2, v10, v2
+; GCN-NEXT:    v_or_b32_e32 v10, v16, v18
+; GCN-NEXT:    v_or_b32_e32 v1, v9, v1
+; GCN-NEXT:    buffer_store_dword v24, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v28, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v49, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v1, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v40f16_to_v40i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_add_f16_e32 v39, 0x200, v39
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_e32 v38, 0x200, v38
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_add_f16_e32 v37, 0x200, v37
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_add_f16_e32 v36, 0x200, v36
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v20
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_add_f16_e32 v35, 0x200, v35
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_add_f16_e32 v34, 0x200, v34
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_add_f16_e32 v33, 0x200, v33
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_add_f16_e32 v32, 0x200, v32
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_add_f16_e32 v31, 0x200, v31
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v30
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_add_f16_e32 v29, 0x200, v29
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v28
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v27
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v14
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v26
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v15
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v25
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v16
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v24
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v17
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v23
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v18
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v22
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v19
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v21
+; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v25
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v23
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v22
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v40f16_to_v40i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v19, v39, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v38, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v37, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v36, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v35, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v34, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v33, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v32, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v31, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v30, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v29, v9, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v8, v28, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v27, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v26, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v24, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v25, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v23, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v22, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v21, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v20, v0, s6
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v19
+; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v20, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v21, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v22, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v23, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v25, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v24, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v26, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v27, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v28, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v29, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v30, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v31, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v32, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v33, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v34, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v35, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v36, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v37, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v38, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v39, v19, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v40f16_to_v40i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v20
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v39, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v37, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v35, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v33, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v31, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v29, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v27, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v26, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v21, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v23, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v24, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v25, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v21, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v23, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v24, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v25, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v26, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v27, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v28, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v29, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v30, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v31, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v33, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v35, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v37, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v39, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <40 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <40 x half> %a1 to <40 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <40 x half> %a to <40 x i16>
+  br label %end
+
+end:
+  %phi = phi <40 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <40 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
index 05fb285362f09..a0fe407022d81 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
@@ -5,8 +5,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
 
-define <22 x i32> @bitcast_v11i64_to_v22i32(<11 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v11i64_to_v22i32:
+define <22 x float> @bitcast_v22i32_to_v22f32(<22 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22i32_to_v22f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
@@ -15,33 +15,33 @@ define <22 x i32> @bitcast_v11i64_to_v22i32(<11 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:  .LBB0_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v11i64_to_v22i32:
+; VI-LABEL: bitcast_v22i32_to_v22f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
@@ -50,33 +50,33 @@ define <22 x i32> @bitcast_v11i64_to_v22i32(<11 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB0_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
-; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
 ; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
-; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
 ; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
-; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
 ; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
-; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
-; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:  .LBB0_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v11i64_to_v22i32:
+; GFX9-LABEL: bitcast_v22i32_to_v22f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
@@ -85,33 +85,33 @@ define <22 x i32> @bitcast_v11i64_to_v22i32(<11 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
-; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
-; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
-; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
-; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
 ; GFX9-NEXT:  .LBB0_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v11i64_to_v22i32:
+; GFX11-LABEL: bitcast_v22i32_to_v22f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -121,34 +121,28 @@ define <22 x i32> @bitcast_v11i64_to_v22i32(<11 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
 ; GFX11-NEXT:  .LBB0_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -156,12 +150,159 @@ define <22 x i32> @bitcast_v11i64_to_v22i32(<11 x i64> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <11 x i64> %a, splat (i64 3)
-  %a2 = bitcast <11 x i64> %a1 to <22 x i32>
+  %a1 = add <22 x i32> %a, splat (i32 3)
+  %a2 = bitcast <22 x i32> %a1 to <22 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <11 x i64> %a to <22 x i32>
+  %a3 = bitcast <22 x i32> %a to <22 x float>
+  br label %end
+
+end:
+  %phi = phi <22 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x float> %phi
+}
+
+define <22 x i32> @bitcast_v22f32_to_v22i32(<22 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22f32_to_v22i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22f32_to_v22i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22f32_to_v22i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22f32_to_v22i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <22 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <22 x float> %a1 to <22 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x float> %a to <22 x i32>
   br label %end
 
 end:
@@ -177,7 +318,7 @@ define <11 x i64> @bitcast_v22i32_to_v11i64(<22 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
@@ -201,7 +342,7 @@ define <11 x i64> @bitcast_v22i32_to_v11i64(<22 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:  .LBB2_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -212,7 +353,7 @@ define <11 x i64> @bitcast_v22i32_to_v11i64(<22 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:    s_cbranch_execz .LBB2_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
@@ -236,7 +377,7 @@ define <11 x i64> @bitcast_v22i32_to_v11i64(<22 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:  .LBB2_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -247,7 +388,7 @@ define <11 x i64> @bitcast_v22i32_to_v11i64(<22 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
 ; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
@@ -271,7 +412,7 @@ define <11 x i64> @bitcast_v22i32_to_v11i64(<22 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:  .LBB2_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -283,7 +424,7 @@ define <11 x i64> @bitcast_v22i32_to_v11i64(<22 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
 ; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
@@ -307,7 +448,7 @@ define <11 x i64> @bitcast_v22i32_to_v11i64(<22 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:  .LBB2_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -326,3 +467,16622 @@ end:
   %phi = phi <11 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <11 x i64> %phi
 }
+
+define <22 x i32> @bitcast_v11i64_to_v22i32(<11 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11i64_to_v22i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11i64_to_v22i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11i64_to_v22i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11i64_to_v22i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <11 x i64> %a, splat (i64 3)
+  %a2 = bitcast <11 x i64> %a1 to <22 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x i64> %a to <22 x i32>
+  br label %end
+
+end:
+  %phi = phi <22 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x i32> %phi
+}
+
+define <11 x double> @bitcast_v22i32_to_v11f64(<22 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22i32_to_v11f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22i32_to_v11f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB4_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22i32_to_v11f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB4_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22i32_to_v11f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <22 x i32> %a, splat (i32 3)
+  %a2 = bitcast <22 x i32> %a1 to <11 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x i32> %a to <11 x double>
+  br label %end
+
+end:
+  %phi = phi <11 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x double> %phi
+}
+
+define <22 x i32> @bitcast_v11f64_to_v22i32(<11 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11f64_to_v22i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11f64_to_v22i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11f64_to_v22i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11f64_to_v22i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <11 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <11 x double> %a1 to <22 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x double> %a to <22 x i32>
+  br label %end
+
+end:
+  %phi = phi <22 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x i32> %phi
+}
+
+define <44 x i16> @bitcast_v22i32_to_v44i16(<22 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22i32_to_v44i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v23
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v23, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_alignbit_b32 v23, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v49
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x54, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v28, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v27, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22i32_to_v44i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB6_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB6_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22i32_to_v44i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB6_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB6_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v51, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v50, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v49, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v48, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v39, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v38, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v37, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v34, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v33, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v32, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v31, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v30, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v29, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v27, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v26, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v25, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v24, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v23, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22i32_to_v44i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB6_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB6_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v51, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v49, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v39, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v29, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v28, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v27, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v26, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v25, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v24, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v23, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v22, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <22 x i32> %a, splat (i32 3)
+  %a2 = bitcast <22 x i32> %a1 to <44 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x i32> %a to <44 x i16>
+  br label %end
+
+end:
+  %phi = phi <44 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x i16> %phi
+}
+
+define <22 x i32> @bitcast_v44i16_to_v22i32(<44 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44i16_to_v22i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v32, v30
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:52
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v56
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v57
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v47
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v45
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v44
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v43
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v41
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v59
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v58
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v46
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v62
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v61
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v60
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v22
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:  .LBB7_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v59
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v58
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_or_b32_e32 v0, v56, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v57, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v47, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v45, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v44, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v43, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v41, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v61, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v60, v9
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v22, v10
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v22, v11
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v22, v12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v22, v13
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v22, v14
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v15
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v22, v16
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v22, v17
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v22, v18
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v22, v19
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v22, v20
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v22, v21
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44i16_to_v22i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v21
+; VI-NEXT:    v_mov_b32_e32 v33, v20
+; VI-NEXT:    v_mov_b32_e32 v34, v19
+; VI-NEXT:    v_mov_b32_e32 v35, v18
+; VI-NEXT:    v_mov_b32_e32 v36, v17
+; VI-NEXT:    v_mov_b32_e32 v37, v16
+; VI-NEXT:    v_mov_b32_e32 v38, v15
+; VI-NEXT:    v_mov_b32_e32 v39, v14
+; VI-NEXT:    v_mov_b32_e32 v48, v13
+; VI-NEXT:    v_mov_b32_e32 v49, v12
+; VI-NEXT:    v_mov_b32_e32 v50, v11
+; VI-NEXT:    v_mov_b32_e32 v51, v10
+; VI-NEXT:    v_mov_b32_e32 v52, v9
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v54, v7
+; VI-NEXT:    v_mov_b32_e32 v55, v6
+; VI-NEXT:    v_mov_b32_e32 v40, v5
+; VI-NEXT:    v_mov_b32_e32 v41, v4
+; VI-NEXT:    v_mov_b32_e32 v42, v3
+; VI-NEXT:    v_mov_b32_e32 v43, v2
+; VI-NEXT:    v_mov_b32_e32 v44, v1
+; VI-NEXT:    v_mov_b32_e32 v45, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v21, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v21, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v21, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v21, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v21, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v21, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v21, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v21, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v21, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v21, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v21, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v21, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v21, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v21, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v21, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v21, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v21, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v21, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v21, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v21, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v43, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v41, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v55, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v53, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v51, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v49, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v35, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v33, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB7_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v21, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v1, v45, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v2, v44, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v3, v43, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v4, v42, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v5, v41, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v6, v40, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v7, v55, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v8, v54, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v9, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v10, v52, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v11, v51, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v12, v50, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v13, v49, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v14, v48, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v15, v39, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v16, v38, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v17, v37, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v18, v36, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v19, v35, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v20, v34, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v22, v33, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v21, v32, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:  .LBB7_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44i16_to_v22i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v21
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v20
+; GFX9-NEXT:    v_mov_b32_e32 v45, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v10
+; GFX9-NEXT:    v_mov_b32_e32 v52, v9
+; GFX9-NEXT:    v_mov_b32_e32 v53, v8
+; GFX9-NEXT:    v_mov_b32_e32 v54, v7
+; GFX9-NEXT:    v_mov_b32_e32 v55, v6
+; GFX9-NEXT:    v_mov_b32_e32 v40, v5
+; GFX9-NEXT:    v_mov_b32_e32 v41, v4
+; GFX9-NEXT:    v_mov_b32_e32 v42, v3
+; GFX9-NEXT:    v_mov_b32_e32 v43, v2
+; GFX9-NEXT:    v_mov_b32_e32 v44, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v45
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v51
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v50
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v49
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v48
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:  .LBB7_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44i16_to_v22i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    v_perm_b32 v0, v49, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v51, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v48, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v39, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v38, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v37, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v36, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v35, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v34, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v31, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v30, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v29, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v28, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v27, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v26, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v25, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v24, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v23, v21, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <44 x i16> %a, splat (i16 3)
+  %a2 = bitcast <44 x i16> %a1 to <22 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x i16> %a to <22 x i32>
+  br label %end
+
+end:
+  %phi = phi <22 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x i32> %phi
+}
+
+define <44 x half> @bitcast_v22i32_to_v44f16(<22 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22i32_to_v44f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v23
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:  .LBB8_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:  .LBB8_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v61
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v56
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v46
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v63
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v44
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v42
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v40
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v54
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v52
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v50
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v25, v48, v28
+; GCN-NEXT:    v_or_b32_e32 v26, v38, v52
+; GCN-NEXT:    v_or_b32_e32 v27, v37, v40
+; GCN-NEXT:    v_or_b32_e32 v28, v35, v41
+; GCN-NEXT:    v_or_b32_e32 v33, v33, v55
+; GCN-NEXT:    v_or_b32_e32 v31, v31, v53
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v51
+; GCN-NEXT:    v_or_b32_e32 v35, v46, v49
+; GCN-NEXT:    v_or_b32_e32 v37, v47, v39
+; GCN-NEXT:    v_or_b32_e32 v36, v56, v36
+; GCN-NEXT:    v_or_b32_e32 v34, v57, v34
+; GCN-NEXT:    v_or_b32_e32 v32, v58, v32
+; GCN-NEXT:    v_or_b32_e32 v29, v59, v29
+; GCN-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v35, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22i32_to_v44f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB8_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB8_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22i32_to_v44f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB8_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB8_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v51, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v50, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v49, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v48, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v39, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v38, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v37, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v34, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v33, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v32, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v31, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v30, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v29, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v27, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v26, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v25, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v24, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v23, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22i32_to_v44f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB8_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB8_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v51, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v49, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v39, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v29, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v28, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v27, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v26, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v25, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v24, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v23, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v22, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <22 x i32> %a, splat (i32 3)
+  %a2 = bitcast <22 x i32> %a1 to <44 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x i32> %a to <44 x half>
+  br label %end
+
+end:
+  %phi = phi <44 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x half> %phi
+}
+
+define <22 x i32> @bitcast_v44f16_to_v22i32(<44 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44f16_to_v22i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v19
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(12)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v0, v50, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v48, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v2, v38, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v3, v36, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v34, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v5, v32, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v6, v62, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v7, v60, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v57
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v54
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v22, v8
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v22, v9
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v22, v10
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v22, v11
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v22, v12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v22, v13
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v22, v14
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v45, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v43, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v41, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v55, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v53, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v52, v21
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:  .LBB9_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v48
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v38
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v36
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v34
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v32
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v62
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v60
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v59
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v58
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v57
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v56
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v52
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v13, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v14, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v15, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v16, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v17, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v18, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v19, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v21, v35, v34
+; GCN-NEXT:  .LBB9_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44f16_to_v22i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v21
+; VI-NEXT:    v_mov_b32_e32 v33, v20
+; VI-NEXT:    v_mov_b32_e32 v34, v19
+; VI-NEXT:    v_mov_b32_e32 v35, v18
+; VI-NEXT:    v_mov_b32_e32 v36, v17
+; VI-NEXT:    v_mov_b32_e32 v37, v16
+; VI-NEXT:    v_mov_b32_e32 v38, v15
+; VI-NEXT:    v_mov_b32_e32 v39, v14
+; VI-NEXT:    v_mov_b32_e32 v48, v13
+; VI-NEXT:    v_mov_b32_e32 v49, v12
+; VI-NEXT:    v_mov_b32_e32 v50, v11
+; VI-NEXT:    v_mov_b32_e32 v51, v10
+; VI-NEXT:    v_mov_b32_e32 v52, v9
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v54, v7
+; VI-NEXT:    v_mov_b32_e32 v55, v6
+; VI-NEXT:    v_mov_b32_e32 v40, v5
+; VI-NEXT:    v_mov_b32_e32 v41, v4
+; VI-NEXT:    v_mov_b32_e32 v42, v3
+; VI-NEXT:    v_mov_b32_e32 v43, v2
+; VI-NEXT:    v_mov_b32_e32 v44, v1
+; VI-NEXT:    v_mov_b32_e32 v45, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v21, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v21, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v21, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v21, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v21, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v21, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v21, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v21, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v21, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v21, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v21, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v21, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v21, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v21, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v21, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v21, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v21, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v21, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v21, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v21, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v43, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v41, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v55, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v53, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v51, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v49, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v35, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v33, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB9_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v21, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v45, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v44, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v43, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v42, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v41, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v40, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v55, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v54, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v52, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v51, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v50, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v49, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v48, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v39, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v38, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v37, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v36, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v35, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v34, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v33, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v20, v22, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v32, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:  .LBB9_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44f16_to_v22i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v21
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v20
+; GFX9-NEXT:    v_mov_b32_e32 v45, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v10
+; GFX9-NEXT:    v_mov_b32_e32 v52, v9
+; GFX9-NEXT:    v_mov_b32_e32 v53, v8
+; GFX9-NEXT:    v_mov_b32_e32 v54, v7
+; GFX9-NEXT:    v_mov_b32_e32 v55, v6
+; GFX9-NEXT:    v_mov_b32_e32 v40, v5
+; GFX9-NEXT:    v_mov_b32_e32 v41, v4
+; GFX9-NEXT:    v_mov_b32_e32 v42, v3
+; GFX9-NEXT:    v_mov_b32_e32 v43, v2
+; GFX9-NEXT:    v_mov_b32_e32 v44, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v45
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v51
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v50
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v49
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v48
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:  .LBB9_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44f16_to_v22i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    v_perm_b32 v0, v49, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v51, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v48, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v39, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v38, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v37, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v36, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v35, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v34, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v31, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v30, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v29, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v28, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v27, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v26, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v25, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v24, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v23, v21, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <44 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <44 x half> %a1 to <22 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x half> %a to <22 x i32>
+  br label %end
+
+end:
+  %phi = phi <22 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x i32> %phi
+}
+
+define <11 x i64> @bitcast_v22f32_to_v11i64(<22 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22f32_to_v11i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22f32_to_v11i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22f32_to_v11i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22f32_to_v11i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <22 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <22 x float> %a1 to <11 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x float> %a to <11 x i64>
+  br label %end
+
+end:
+  %phi = phi <11 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x i64> %phi
+}
+
+define <22 x float> @bitcast_v11i64_to_v22f32(<11 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11i64_to_v22f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11i64_to_v22f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11i64_to_v22f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11i64_to_v22f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <11 x i64> %a, splat (i64 3)
+  %a2 = bitcast <11 x i64> %a1 to <22 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x i64> %a to <22 x float>
+  br label %end
+
+end:
+  %phi = phi <22 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x float> %phi
+}
+
+define <11 x double> @bitcast_v22f32_to_v11f64(<22 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22f32_to_v11f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22f32_to_v11f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22f32_to_v11f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22f32_to_v11f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB12_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <22 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <22 x float> %a1 to <11 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x float> %a to <11 x double>
+  br label %end
+
+end:
+  %phi = phi <11 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x double> %phi
+}
+
+define <22 x float> @bitcast_v11f64_to_v22f32(<11 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11f64_to_v22f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11f64_to_v22f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11f64_to_v22f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11f64_to_v22f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <11 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <11 x double> %a1 to <22 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x double> %a to <22 x float>
+  br label %end
+
+end:
+  %phi = phi <22 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x float> %phi
+}
+
+define <44 x i16> @bitcast_v22f32_to_v44i16(<22 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22f32_to_v44i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v23
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v23, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_alignbit_b32 v23, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v49
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x54, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v28, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v27, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22f32_to_v44i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB14_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB14_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22f32_to_v44i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB14_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB14_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v51, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v50, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v49, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v48, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v39, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v38, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v37, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v34, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v33, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v32, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v31, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v30, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v29, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v27, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v26, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v25, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v24, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v23, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22f32_to_v44i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB14_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB14_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v51, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v49, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v39, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v29, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v28, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v27, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v26, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v25, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v24, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v23, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v22, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <22 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <22 x float> %a1 to <44 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x float> %a to <44 x i16>
+  br label %end
+
+end:
+  %phi = phi <44 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x i16> %phi
+}
+
+define <22 x float> @bitcast_v44i16_to_v22f32(<44 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44i16_to_v22f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v32, v30
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:52
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v56
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v57
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v47
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v45
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v44
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v43
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v41
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v59
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v58
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v46
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v62
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v61
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v60
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v22
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:  .LBB15_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v59
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v58
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_or_b32_e32 v0, v56, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v57, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v47, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v45, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v44, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v43, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v41, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v61, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v60, v9
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v22, v10
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v22, v11
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v22, v12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v22, v13
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v22, v14
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v15
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v22, v16
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v22, v17
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v22, v18
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v22, v19
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v22, v20
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v22, v21
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:  .LBB15_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44i16_to_v22f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v21
+; VI-NEXT:    v_mov_b32_e32 v33, v20
+; VI-NEXT:    v_mov_b32_e32 v34, v19
+; VI-NEXT:    v_mov_b32_e32 v35, v18
+; VI-NEXT:    v_mov_b32_e32 v36, v17
+; VI-NEXT:    v_mov_b32_e32 v37, v16
+; VI-NEXT:    v_mov_b32_e32 v38, v15
+; VI-NEXT:    v_mov_b32_e32 v39, v14
+; VI-NEXT:    v_mov_b32_e32 v48, v13
+; VI-NEXT:    v_mov_b32_e32 v49, v12
+; VI-NEXT:    v_mov_b32_e32 v50, v11
+; VI-NEXT:    v_mov_b32_e32 v51, v10
+; VI-NEXT:    v_mov_b32_e32 v52, v9
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v54, v7
+; VI-NEXT:    v_mov_b32_e32 v55, v6
+; VI-NEXT:    v_mov_b32_e32 v40, v5
+; VI-NEXT:    v_mov_b32_e32 v41, v4
+; VI-NEXT:    v_mov_b32_e32 v42, v3
+; VI-NEXT:    v_mov_b32_e32 v43, v2
+; VI-NEXT:    v_mov_b32_e32 v44, v1
+; VI-NEXT:    v_mov_b32_e32 v45, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v21, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v21, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v21, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v21, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v21, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v21, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v21, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v21, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v21, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v21, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v21, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v21, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v21, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v21, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v21, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v21, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v21, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v21, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v21, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v21, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v43, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v41, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v55, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v53, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v51, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v49, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v35, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v33, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB15_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v21, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v1, v45, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v2, v44, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v3, v43, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v4, v42, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v5, v41, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v6, v40, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v7, v55, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v8, v54, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v9, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v10, v52, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v11, v51, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v12, v50, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v13, v49, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v14, v48, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v15, v39, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v16, v38, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v17, v37, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v18, v36, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v19, v35, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v20, v34, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v22, v33, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v21, v32, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:  .LBB15_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44i16_to_v22f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v21
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v20
+; GFX9-NEXT:    v_mov_b32_e32 v45, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v10
+; GFX9-NEXT:    v_mov_b32_e32 v52, v9
+; GFX9-NEXT:    v_mov_b32_e32 v53, v8
+; GFX9-NEXT:    v_mov_b32_e32 v54, v7
+; GFX9-NEXT:    v_mov_b32_e32 v55, v6
+; GFX9-NEXT:    v_mov_b32_e32 v40, v5
+; GFX9-NEXT:    v_mov_b32_e32 v41, v4
+; GFX9-NEXT:    v_mov_b32_e32 v42, v3
+; GFX9-NEXT:    v_mov_b32_e32 v43, v2
+; GFX9-NEXT:    v_mov_b32_e32 v44, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v45
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v51
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v50
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v49
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v48
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:  .LBB15_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44i16_to_v22f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    v_perm_b32 v0, v49, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v51, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v48, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v39, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v38, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v37, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v36, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v35, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v34, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v31, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v30, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v29, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v28, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v27, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v26, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v25, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v24, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v23, v21, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <44 x i16> %a, splat (i16 3)
+  %a2 = bitcast <44 x i16> %a1 to <22 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x i16> %a to <22 x float>
+  br label %end
+
+end:
+  %phi = phi <22 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x float> %phi
+}
+
+define <44 x half> @bitcast_v22f32_to_v44f16(<22 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v22f32_to_v44f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v23
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:  .LBB16_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:  .LBB16_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v61
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v56
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v46
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v63
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v44
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v42
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v40
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v54
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v52
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v50
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v25, v48, v28
+; GCN-NEXT:    v_or_b32_e32 v26, v38, v52
+; GCN-NEXT:    v_or_b32_e32 v27, v37, v40
+; GCN-NEXT:    v_or_b32_e32 v28, v35, v41
+; GCN-NEXT:    v_or_b32_e32 v33, v33, v55
+; GCN-NEXT:    v_or_b32_e32 v31, v31, v53
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v51
+; GCN-NEXT:    v_or_b32_e32 v35, v46, v49
+; GCN-NEXT:    v_or_b32_e32 v37, v47, v39
+; GCN-NEXT:    v_or_b32_e32 v36, v56, v36
+; GCN-NEXT:    v_or_b32_e32 v34, v57, v34
+; GCN-NEXT:    v_or_b32_e32 v32, v58, v32
+; GCN-NEXT:    v_or_b32_e32 v29, v59, v29
+; GCN-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v35, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v22f32_to_v44f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB16_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB16_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v22f32_to_v44f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB16_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB16_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v51, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v50, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v49, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v48, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v39, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v38, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v37, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v34, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v33, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v32, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v31, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v30, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v29, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v27, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v26, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v25, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v24, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v23, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v22f32_to_v44f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB16_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB16_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v51, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v49, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v39, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v29, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v28, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v27, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v26, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v25, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v24, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v23, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v22, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <22 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <22 x float> %a1 to <44 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <22 x float> %a to <44 x half>
+  br label %end
+
+end:
+  %phi = phi <44 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x half> %phi
+}
+
+define <22 x float> @bitcast_v44f16_to_v22f32(<44 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44f16_to_v22f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v19
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(12)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v0, v50, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v48, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v2, v38, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v3, v36, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v34, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v5, v32, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v6, v62, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v7, v60, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v57
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v54
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v22, v8
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v22, v9
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v22, v10
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v22, v11
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v22, v12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v22, v13
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v22, v14
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v45, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v43, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v41, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v55, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v53, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v52, v21
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v48
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v38
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v36
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v34
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v32
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v62
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v60
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v59
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v58
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v57
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v56
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v52
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v13, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v14, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v15, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v16, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v17, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v18, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v19, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v21, v35, v34
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44f16_to_v22f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v21
+; VI-NEXT:    v_mov_b32_e32 v33, v20
+; VI-NEXT:    v_mov_b32_e32 v34, v19
+; VI-NEXT:    v_mov_b32_e32 v35, v18
+; VI-NEXT:    v_mov_b32_e32 v36, v17
+; VI-NEXT:    v_mov_b32_e32 v37, v16
+; VI-NEXT:    v_mov_b32_e32 v38, v15
+; VI-NEXT:    v_mov_b32_e32 v39, v14
+; VI-NEXT:    v_mov_b32_e32 v48, v13
+; VI-NEXT:    v_mov_b32_e32 v49, v12
+; VI-NEXT:    v_mov_b32_e32 v50, v11
+; VI-NEXT:    v_mov_b32_e32 v51, v10
+; VI-NEXT:    v_mov_b32_e32 v52, v9
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v54, v7
+; VI-NEXT:    v_mov_b32_e32 v55, v6
+; VI-NEXT:    v_mov_b32_e32 v40, v5
+; VI-NEXT:    v_mov_b32_e32 v41, v4
+; VI-NEXT:    v_mov_b32_e32 v42, v3
+; VI-NEXT:    v_mov_b32_e32 v43, v2
+; VI-NEXT:    v_mov_b32_e32 v44, v1
+; VI-NEXT:    v_mov_b32_e32 v45, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v21, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v21, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v21, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v21, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v21, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v21, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v21, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v21, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v21, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v21, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v21, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v21, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v21, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v21, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v21, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v21, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v21, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v21, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v21, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v21, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v43, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v41, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v55, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v53, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v51, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v49, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v35, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v33, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB17_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v21, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v45, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v44, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v43, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v42, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v41, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v40, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v55, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v54, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v52, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v51, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v50, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v49, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v48, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v39, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v38, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v37, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v36, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v35, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v34, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v33, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v20, v22, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v32, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:  .LBB17_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44f16_to_v22f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v21
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v20
+; GFX9-NEXT:    v_mov_b32_e32 v45, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v10
+; GFX9-NEXT:    v_mov_b32_e32 v52, v9
+; GFX9-NEXT:    v_mov_b32_e32 v53, v8
+; GFX9-NEXT:    v_mov_b32_e32 v54, v7
+; GFX9-NEXT:    v_mov_b32_e32 v55, v6
+; GFX9-NEXT:    v_mov_b32_e32 v40, v5
+; GFX9-NEXT:    v_mov_b32_e32 v41, v4
+; GFX9-NEXT:    v_mov_b32_e32 v42, v3
+; GFX9-NEXT:    v_mov_b32_e32 v43, v2
+; GFX9-NEXT:    v_mov_b32_e32 v44, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v45
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v51
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v50
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v49
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v48
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:  .LBB17_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44f16_to_v22f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    v_perm_b32 v0, v49, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v51, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v48, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v39, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v38, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v37, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v36, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v35, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v34, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v31, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v30, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v29, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v28, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v27, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v26, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v25, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v24, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v23, v21, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <44 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <44 x half> %a1 to <22 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x half> %a to <22 x float>
+  br label %end
+
+end:
+  %phi = phi <22 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <22 x float> %phi
+}
+
+define <11 x double> @bitcast_v11i64_to_v11f64(<11 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11i64_to_v11f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11i64_to_v11f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11i64_to_v11f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11i64_to_v11f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <11 x i64> %a, splat (i64 3)
+  %a2 = bitcast <11 x i64> %a1 to <11 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x i64> %a to <11 x double>
+  br label %end
+
+end:
+  %phi = phi <11 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x double> %phi
+}
+
+define <11 x i64> @bitcast_v11f64_to_v11i64(<11 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11f64_to_v11i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11f64_to_v11i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11f64_to_v11i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11f64_to_v11i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <11 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <11 x double> %a1 to <11 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x double> %a to <11 x i64>
+  br label %end
+
+end:
+  %phi = phi <11 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x i64> %phi
+}
+
+define <44 x i16> @bitcast_v11i64_to_v44i16(<11 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11i64_to_v44i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v23
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v23, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_alignbit_b32 v23, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v49
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 0x54, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v28, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11i64_to_v44i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB20_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB20_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11i64_to_v44i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB20_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB20_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v51, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v50, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v49, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v48, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v39, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v38, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v37, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v34, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v33, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v32, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v31, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v30, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v29, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v27, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v26, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v25, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v24, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v23, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11i64_to_v44i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB20_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB20_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v51, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v49, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v39, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v29, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v28, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v27, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v26, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v25, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v24, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v23, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v22, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <11 x i64> %a, splat (i64 3)
+  %a2 = bitcast <11 x i64> %a1 to <44 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x i64> %a to <44 x i16>
+  br label %end
+
+end:
+  %phi = phi <44 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x i16> %phi
+}
+
+define <11 x i64> @bitcast_v44i16_to_v11i64(<44 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44i16_to_v11i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v32, v30
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:52
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v56
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v57
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v47
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v45
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v44
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v43
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v41
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v59
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v58
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v46
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v62
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v61
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v60
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v22
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:  .LBB21_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v59
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v58
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_or_b32_e32 v0, v56, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v57, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v47, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v45, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v44, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v43, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v41, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v61, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v60, v9
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v22, v10
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v22, v11
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v22, v12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v22, v13
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v22, v14
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v15
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v22, v16
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v22, v17
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v22, v18
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v22, v19
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v22, v20
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v22, v21
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:  .LBB21_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44i16_to_v11i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v21
+; VI-NEXT:    v_mov_b32_e32 v33, v20
+; VI-NEXT:    v_mov_b32_e32 v34, v19
+; VI-NEXT:    v_mov_b32_e32 v35, v18
+; VI-NEXT:    v_mov_b32_e32 v36, v17
+; VI-NEXT:    v_mov_b32_e32 v37, v16
+; VI-NEXT:    v_mov_b32_e32 v38, v15
+; VI-NEXT:    v_mov_b32_e32 v39, v14
+; VI-NEXT:    v_mov_b32_e32 v48, v13
+; VI-NEXT:    v_mov_b32_e32 v49, v12
+; VI-NEXT:    v_mov_b32_e32 v50, v11
+; VI-NEXT:    v_mov_b32_e32 v51, v10
+; VI-NEXT:    v_mov_b32_e32 v52, v9
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v54, v7
+; VI-NEXT:    v_mov_b32_e32 v55, v6
+; VI-NEXT:    v_mov_b32_e32 v40, v5
+; VI-NEXT:    v_mov_b32_e32 v41, v4
+; VI-NEXT:    v_mov_b32_e32 v42, v3
+; VI-NEXT:    v_mov_b32_e32 v43, v2
+; VI-NEXT:    v_mov_b32_e32 v44, v1
+; VI-NEXT:    v_mov_b32_e32 v45, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v21, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v21, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v21, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v21, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v21, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v21, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v21, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v21, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v21, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v21, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v21, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v21, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v21, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v21, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v21, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v21, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v21, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v21, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v21, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v21, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v43, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v41, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v55, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v53, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v51, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v49, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v35, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v33, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB21_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v21, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v1, v45, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v2, v44, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v3, v43, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v4, v42, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v5, v41, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v6, v40, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v7, v55, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v8, v54, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v9, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v10, v52, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v11, v51, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v12, v50, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v13, v49, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v14, v48, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v15, v39, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v16, v38, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v17, v37, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v18, v36, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v19, v35, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v20, v34, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v22, v33, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v21, v32, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:  .LBB21_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44i16_to_v11i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v21
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v20
+; GFX9-NEXT:    v_mov_b32_e32 v45, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v10
+; GFX9-NEXT:    v_mov_b32_e32 v52, v9
+; GFX9-NEXT:    v_mov_b32_e32 v53, v8
+; GFX9-NEXT:    v_mov_b32_e32 v54, v7
+; GFX9-NEXT:    v_mov_b32_e32 v55, v6
+; GFX9-NEXT:    v_mov_b32_e32 v40, v5
+; GFX9-NEXT:    v_mov_b32_e32 v41, v4
+; GFX9-NEXT:    v_mov_b32_e32 v42, v3
+; GFX9-NEXT:    v_mov_b32_e32 v43, v2
+; GFX9-NEXT:    v_mov_b32_e32 v44, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v45
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v51
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v50
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v49
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v48
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:  .LBB21_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44i16_to_v11i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    v_perm_b32 v0, v49, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v51, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v48, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v39, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v38, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v37, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v36, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v35, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v34, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v31, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v30, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v29, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v28, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v27, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v26, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v25, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v24, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v23, v21, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <44 x i16> %a, splat (i16 3)
+  %a2 = bitcast <44 x i16> %a1 to <11 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x i16> %a to <11 x i64>
+  br label %end
+
+end:
+  %phi = phi <11 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x i64> %phi
+}
+
+define <44 x half> @bitcast_v11i64_to_v44f16(<11 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11i64_to_v44f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v23
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:  .LBB22_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:  .LBB22_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v61
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v56
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v46
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v63
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v44
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v42
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v40
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v54
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v52
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v50
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v25, v48, v28
+; GCN-NEXT:    v_or_b32_e32 v26, v38, v52
+; GCN-NEXT:    v_or_b32_e32 v27, v37, v40
+; GCN-NEXT:    v_or_b32_e32 v28, v35, v41
+; GCN-NEXT:    v_or_b32_e32 v33, v33, v55
+; GCN-NEXT:    v_or_b32_e32 v31, v31, v53
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v51
+; GCN-NEXT:    v_or_b32_e32 v35, v46, v49
+; GCN-NEXT:    v_or_b32_e32 v37, v47, v39
+; GCN-NEXT:    v_or_b32_e32 v36, v56, v36
+; GCN-NEXT:    v_or_b32_e32 v34, v57, v34
+; GCN-NEXT:    v_or_b32_e32 v32, v58, v32
+; GCN-NEXT:    v_or_b32_e32 v29, v59, v29
+; GCN-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v35, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11i64_to_v44f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB22_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB22_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11i64_to_v44f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB22_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB22_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v51, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v50, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v49, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v48, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v39, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v38, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v37, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v34, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v33, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v32, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v31, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v30, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v29, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v27, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v26, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v25, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v24, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v23, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11i64_to_v44f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB22_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB22_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v51, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v49, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v39, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v29, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v28, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v27, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v26, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v25, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v24, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v23, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v22, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <11 x i64> %a, splat (i64 3)
+  %a2 = bitcast <11 x i64> %a1 to <44 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x i64> %a to <44 x half>
+  br label %end
+
+end:
+  %phi = phi <44 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x half> %phi
+}
+
+define <11 x i64> @bitcast_v44f16_to_v11i64(<44 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44f16_to_v11i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v19
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(12)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v0, v50, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v48, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v2, v38, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v3, v36, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v34, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v5, v32, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v6, v62, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v7, v60, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v57
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v54
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v22, v8
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v22, v9
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v22, v10
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v22, v11
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v22, v12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v22, v13
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v22, v14
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v45, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v43, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v41, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v55, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v53, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v52, v21
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:  .LBB23_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v48
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v38
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v36
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v34
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v32
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v62
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v60
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v59
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v58
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v57
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v56
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v52
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v13, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v14, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v15, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v16, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v17, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v18, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v19, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v21, v35, v34
+; GCN-NEXT:  .LBB23_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44f16_to_v11i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v21
+; VI-NEXT:    v_mov_b32_e32 v33, v20
+; VI-NEXT:    v_mov_b32_e32 v34, v19
+; VI-NEXT:    v_mov_b32_e32 v35, v18
+; VI-NEXT:    v_mov_b32_e32 v36, v17
+; VI-NEXT:    v_mov_b32_e32 v37, v16
+; VI-NEXT:    v_mov_b32_e32 v38, v15
+; VI-NEXT:    v_mov_b32_e32 v39, v14
+; VI-NEXT:    v_mov_b32_e32 v48, v13
+; VI-NEXT:    v_mov_b32_e32 v49, v12
+; VI-NEXT:    v_mov_b32_e32 v50, v11
+; VI-NEXT:    v_mov_b32_e32 v51, v10
+; VI-NEXT:    v_mov_b32_e32 v52, v9
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v54, v7
+; VI-NEXT:    v_mov_b32_e32 v55, v6
+; VI-NEXT:    v_mov_b32_e32 v40, v5
+; VI-NEXT:    v_mov_b32_e32 v41, v4
+; VI-NEXT:    v_mov_b32_e32 v42, v3
+; VI-NEXT:    v_mov_b32_e32 v43, v2
+; VI-NEXT:    v_mov_b32_e32 v44, v1
+; VI-NEXT:    v_mov_b32_e32 v45, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v21, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v21, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v21, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v21, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v21, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v21, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v21, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v21, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v21, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v21, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v21, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v21, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v21, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v21, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v21, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v21, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v21, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v21, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v21, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v21, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v43, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v41, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v55, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v53, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v51, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v49, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v35, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v33, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB23_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v21, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v45, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v44, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v43, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v42, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v41, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v40, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v55, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v54, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v52, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v51, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v50, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v49, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v48, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v39, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v38, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v37, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v36, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v35, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v34, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v33, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v20, v22, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v32, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:  .LBB23_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44f16_to_v11i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v21
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v20
+; GFX9-NEXT:    v_mov_b32_e32 v45, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v10
+; GFX9-NEXT:    v_mov_b32_e32 v52, v9
+; GFX9-NEXT:    v_mov_b32_e32 v53, v8
+; GFX9-NEXT:    v_mov_b32_e32 v54, v7
+; GFX9-NEXT:    v_mov_b32_e32 v55, v6
+; GFX9-NEXT:    v_mov_b32_e32 v40, v5
+; GFX9-NEXT:    v_mov_b32_e32 v41, v4
+; GFX9-NEXT:    v_mov_b32_e32 v42, v3
+; GFX9-NEXT:    v_mov_b32_e32 v43, v2
+; GFX9-NEXT:    v_mov_b32_e32 v44, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v45
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v51
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v50
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v49
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v48
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:  .LBB23_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44f16_to_v11i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    v_perm_b32 v0, v49, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v51, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v48, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v39, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v38, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v37, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v36, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v35, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v34, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v31, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v30, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v29, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v28, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v27, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v26, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v25, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v24, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v23, v21, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <44 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <44 x half> %a1 to <11 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x half> %a to <11 x i64>
+  br label %end
+
+end:
+  %phi = phi <11 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x i64> %phi
+}
+
+define <44 x i16> @bitcast_v11f64_to_v44i16(<11 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11f64_to_v44i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v23
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v23, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v2
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_alignbit_b32 v23, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v2
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 4, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 8, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 12, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 20, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 24, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 28, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 32, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 36, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v28
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 40, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v27
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 52, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 56, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 0x48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x4c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x50, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x54, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v28, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v31, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11f64_to_v44i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB24_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB24_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11f64_to_v44i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB24_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB24_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v51, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v50, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v49, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v48, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v39, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v38, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v37, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v34, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v33, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v32, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v31, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v30, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v29, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v27, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v26, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v25, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v24, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v23, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11f64_to_v44i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB24_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB24_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v51, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v49, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v39, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v29, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v28, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v27, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v26, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v25, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v24, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v23, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v22, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <11 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <11 x double> %a1 to <44 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x double> %a to <44 x i16>
+  br label %end
+
+end:
+  %phi = phi <44 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x i16> %phi
+}
+
+define <11 x double> @bitcast_v44i16_to_v11f64(<44 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44i16_to_v11f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v32, v30
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:52
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v56
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v57
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v47
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v45
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v44
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v43
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v41
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v59
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v58
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v46
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v63
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v62
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v61
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v60
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v22
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v22
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:  .LBB25_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v59
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v58
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v62
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_or_b32_e32 v0, v56, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v57, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v47, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v45, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v44, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v43, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v42, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v41, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v61, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v60, v9
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v22, v10
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v22, v11
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v22, v12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v22, v13
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v22, v14
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v15
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v22, v16
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v22, v17
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v22, v18
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v22, v19
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v22, v20
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v22, v21
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:  .LBB25_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44i16_to_v11f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v21
+; VI-NEXT:    v_mov_b32_e32 v33, v20
+; VI-NEXT:    v_mov_b32_e32 v34, v19
+; VI-NEXT:    v_mov_b32_e32 v35, v18
+; VI-NEXT:    v_mov_b32_e32 v36, v17
+; VI-NEXT:    v_mov_b32_e32 v37, v16
+; VI-NEXT:    v_mov_b32_e32 v38, v15
+; VI-NEXT:    v_mov_b32_e32 v39, v14
+; VI-NEXT:    v_mov_b32_e32 v48, v13
+; VI-NEXT:    v_mov_b32_e32 v49, v12
+; VI-NEXT:    v_mov_b32_e32 v50, v11
+; VI-NEXT:    v_mov_b32_e32 v51, v10
+; VI-NEXT:    v_mov_b32_e32 v52, v9
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v54, v7
+; VI-NEXT:    v_mov_b32_e32 v55, v6
+; VI-NEXT:    v_mov_b32_e32 v40, v5
+; VI-NEXT:    v_mov_b32_e32 v41, v4
+; VI-NEXT:    v_mov_b32_e32 v42, v3
+; VI-NEXT:    v_mov_b32_e32 v43, v2
+; VI-NEXT:    v_mov_b32_e32 v44, v1
+; VI-NEXT:    v_mov_b32_e32 v45, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v21, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v21, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v21, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v21, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v21, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v21, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v21, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v21, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v21, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v21, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v21, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v21, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v21, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v21, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v21, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v21, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v21, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v21, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v21, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v21, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v43, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v41, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v55, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v53, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v51, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v49, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v35, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v33, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB25_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v21, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v1, v45, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v2, v44, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v3, v43, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v4, v42, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v5, v41, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v6, v40, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v7, v55, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v8, v54, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v9, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v10, v52, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v11, v51, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v12, v50, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v13, v49, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v14, v48, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v15, v39, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v16, v38, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v17, v37, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v18, v36, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v19, v35, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v20, v34, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v22, v33, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v21, v32, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:  .LBB25_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44i16_to_v11f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v21
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v20
+; GFX9-NEXT:    v_mov_b32_e32 v45, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v10
+; GFX9-NEXT:    v_mov_b32_e32 v52, v9
+; GFX9-NEXT:    v_mov_b32_e32 v53, v8
+; GFX9-NEXT:    v_mov_b32_e32 v54, v7
+; GFX9-NEXT:    v_mov_b32_e32 v55, v6
+; GFX9-NEXT:    v_mov_b32_e32 v40, v5
+; GFX9-NEXT:    v_mov_b32_e32 v41, v4
+; GFX9-NEXT:    v_mov_b32_e32 v42, v3
+; GFX9-NEXT:    v_mov_b32_e32 v43, v2
+; GFX9-NEXT:    v_mov_b32_e32 v44, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v45
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v51
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v50
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v49
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v48
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:  .LBB25_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44i16_to_v11f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    v_perm_b32 v0, v49, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v51, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v48, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v39, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v38, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v37, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v36, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v35, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v34, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v31, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v30, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v29, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v28, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v27, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v26, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v25, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v24, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v23, v21, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <44 x i16> %a, splat (i16 3)
+  %a2 = bitcast <44 x i16> %a1 to <11 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x i16> %a to <11 x double>
+  br label %end
+
+end:
+  %phi = phi <11 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x double> %phi
+}
+
+define <44 x half> @bitcast_v11f64_to_v44f16(<11 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v11f64_to_v44f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v23
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v61
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v56
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v46
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v63
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v44
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v42
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v40
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v54
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v52
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v50
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v25, v48, v28
+; GCN-NEXT:    v_or_b32_e32 v26, v38, v52
+; GCN-NEXT:    v_or_b32_e32 v27, v37, v40
+; GCN-NEXT:    v_or_b32_e32 v28, v35, v41
+; GCN-NEXT:    v_or_b32_e32 v33, v33, v55
+; GCN-NEXT:    v_or_b32_e32 v31, v31, v53
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v51
+; GCN-NEXT:    v_or_b32_e32 v35, v46, v49
+; GCN-NEXT:    v_or_b32_e32 v37, v47, v39
+; GCN-NEXT:    v_or_b32_e32 v36, v56, v36
+; GCN-NEXT:    v_or_b32_e32 v34, v57, v34
+; GCN-NEXT:    v_or_b32_e32 v32, v58, v32
+; GCN-NEXT:    v_or_b32_e32 v29, v59, v29
+; GCN-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v35, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v11f64_to_v44f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr22
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB26_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; VI-NEXT:  .LBB26_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v11f64_to_v44f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB26_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX9-NEXT:  .LBB26_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v51, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v50, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v49, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v48, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v39, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v38, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v37, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v34, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v33, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v32, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v31, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v30, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v29, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v27, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v26, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v25, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v24, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v23, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v11f64_to_v44f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    ; implicit-def: $vgpr23
+; GFX11-NEXT:    ; implicit-def: $vgpr22
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB26_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v0
+; GFX11-NEXT:  .LBB26_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v51, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v49, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v48, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v39, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v32, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v31, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v29, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v28, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v27, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v26, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v25, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v24, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v23, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v22, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <11 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <11 x double> %a1 to <44 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <11 x double> %a to <44 x half>
+  br label %end
+
+end:
+  %phi = phi <44 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x half> %phi
+}
+
+define <11 x double> @bitcast_v44f16_to_v11f64(<44 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44f16_to_v11f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v19
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(12)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v0, v50, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v48, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v2, v38, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v3, v36, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v4, v34, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v5, v32, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v6, v62, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v7, v60, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v57
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v54
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v22, v8
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v22, v9
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v22, v10
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v22, v11
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v22, v12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v22, v13
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v22, v14
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v22, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v45, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v43, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v41, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v55, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v53, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v52, v21
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; kill: killed $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v48
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v38
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v36
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v34
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v32
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v62
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v60
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v59
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v58
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v57
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v56
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v52
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v12, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v13, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v14, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v15, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v16, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v17, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v18, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v19, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v21, v35, v34
+; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44f16_to_v11f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v21
+; VI-NEXT:    v_mov_b32_e32 v33, v20
+; VI-NEXT:    v_mov_b32_e32 v34, v19
+; VI-NEXT:    v_mov_b32_e32 v35, v18
+; VI-NEXT:    v_mov_b32_e32 v36, v17
+; VI-NEXT:    v_mov_b32_e32 v37, v16
+; VI-NEXT:    v_mov_b32_e32 v38, v15
+; VI-NEXT:    v_mov_b32_e32 v39, v14
+; VI-NEXT:    v_mov_b32_e32 v48, v13
+; VI-NEXT:    v_mov_b32_e32 v49, v12
+; VI-NEXT:    v_mov_b32_e32 v50, v11
+; VI-NEXT:    v_mov_b32_e32 v51, v10
+; VI-NEXT:    v_mov_b32_e32 v52, v9
+; VI-NEXT:    v_mov_b32_e32 v53, v8
+; VI-NEXT:    v_mov_b32_e32 v54, v7
+; VI-NEXT:    v_mov_b32_e32 v55, v6
+; VI-NEXT:    v_mov_b32_e32 v40, v5
+; VI-NEXT:    v_mov_b32_e32 v41, v4
+; VI-NEXT:    v_mov_b32_e32 v42, v3
+; VI-NEXT:    v_mov_b32_e32 v43, v2
+; VI-NEXT:    v_mov_b32_e32 v44, v1
+; VI-NEXT:    v_mov_b32_e32 v45, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v21, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v21, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v21, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v21, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v21, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v21, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v21, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v21, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v21, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v21, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v21, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v21, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v21, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v21, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v21, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v21, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v21, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v21, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v21, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v21, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v43, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v41, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v55, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v53, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v51, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v49, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v37, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v35, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v33, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB27_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v21, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v45, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v44, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v43, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v42, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v41, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v40, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v55, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v54, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v52, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v51, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v50, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v49, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v48, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v39, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v38, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v37, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v36, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v35, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v34, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v33, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v20, v22, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v32, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:  .LBB27_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44f16_to_v11f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v32, v21
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v20
+; GFX9-NEXT:    v_mov_b32_e32 v45, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v32
+; GFX9-NEXT:    v_mov_b32_e32 v34, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v10
+; GFX9-NEXT:    v_mov_b32_e32 v52, v9
+; GFX9-NEXT:    v_mov_b32_e32 v53, v8
+; GFX9-NEXT:    v_mov_b32_e32 v54, v7
+; GFX9-NEXT:    v_mov_b32_e32 v55, v6
+; GFX9-NEXT:    v_mov_b32_e32 v40, v5
+; GFX9-NEXT:    v_mov_b32_e32 v41, v4
+; GFX9-NEXT:    v_mov_b32_e32 v42, v3
+; GFX9-NEXT:    v_mov_b32_e32 v43, v2
+; GFX9-NEXT:    v_mov_b32_e32 v44, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v45
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v51
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v50
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v49
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v48
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v39
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v38
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v37
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v36
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v33
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr22
+; GFX9-NEXT:    ; kill: killed $vgpr22
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:  .LBB27_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v45, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v46, v44, s6
+; GFX9-NEXT:    v_perm_b32 v2, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v3, v62, v42, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v41, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v40, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v55, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v54, s6
+; GFX9-NEXT:    v_perm_b32 v8, v57, v53, s6
+; GFX9-NEXT:    v_perm_b32 v9, v56, v52, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v32, s6
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44f16_to_v11f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX11-NEXT:    v_perm_b32 v0, v49, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v51, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v48, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v39, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v38, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v37, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v36, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v35, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v34, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v31, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v30, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v29, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v28, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v27, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v26, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v25, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v24, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v23, v21, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <44 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <44 x half> %a1 to <11 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x half> %a to <11 x double>
+  br label %end
+
+end:
+  %phi = phi <11 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <11 x double> %phi
+}
+
+define <44 x half> @bitcast_v44i16_to_v44f16(<44 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44i16_to_v44f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; kill: killed $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; kill: killed $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; kill: killed $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; kill: killed $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; kill: killed $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; kill: killed $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v24
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v30
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v51
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v52
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v53
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v55
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v43
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v44
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v45
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v47
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:  .LBB28_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 3, v47
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 3, v46
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 3, v45
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 3, v44
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 3, v43
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 3, v42
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 3, v41
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 3, v55
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v24
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v30
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v52
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v53
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v43
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v44
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v45
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v47
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v59
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v60
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v61
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v3
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v62
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v63
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v5
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v31
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v32
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v7
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v34
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v9
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v35
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v36
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v37
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v38
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v58
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v57
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v25, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v26, v32, v31
+; GCN-NEXT:    v_or_b32_e32 v27, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v28, v38, v37
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v39
+; GCN-NEXT:    v_or_b32_e32 v31, v53, v48
+; GCN-NEXT:    v_or_b32_e32 v32, v55, v49
+; GCN-NEXT:    v_or_b32_e32 v34, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v35, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v37, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v38, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v39, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v48, v59, v58
+; GCN-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v35, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44i16_to_v44f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v1
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v51, 3, v51
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v50, 3, v50
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v49, 3, v49
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v48, 3, v48
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v39, 3, v39
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v38, 3, v38
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v37, 3, v37
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v36, 3, v36
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_e32 v35, 3, v35
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_add_u16_e32 v34, 3, v34
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_add_u16_e32 v33, 3, v33
+; VI-NEXT:    v_add_u16_e32 v12, 3, v12
+; VI-NEXT:    v_add_u16_e32 v32, 3, v32
+; VI-NEXT:    v_add_u16_e32 v13, 3, v13
+; VI-NEXT:    v_add_u16_e32 v31, 3, v31
+; VI-NEXT:    v_add_u16_e32 v14, 3, v14
+; VI-NEXT:    v_add_u16_e32 v30, 3, v30
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    v_add_u16_e32 v29, 3, v29
+; VI-NEXT:    v_add_u16_e32 v16, 3, v16
+; VI-NEXT:    v_add_u16_e32 v28, 3, v28
+; VI-NEXT:    v_add_u16_e32 v17, 3, v17
+; VI-NEXT:    v_add_u16_e32 v27, 3, v27
+; VI-NEXT:    v_add_u16_e32 v18, 3, v18
+; VI-NEXT:    v_add_u16_e32 v26, 3, v26
+; VI-NEXT:    v_add_u16_e32 v19, 3, v19
+; VI-NEXT:    v_add_u16_e32 v25, 3, v25
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_add_u16_e32 v24, 3, v24
+; VI-NEXT:    v_add_u16_e32 v21, 3, v21
+; VI-NEXT:    v_add_u16_e32 v23, 3, v23
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44i16_to_v44f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v1
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v21, v51, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v50, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v49, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v48, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v39, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v38, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v37, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v36, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v35, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v34, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v33, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v32, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v31, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v30, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v29, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v28, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v27, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v26, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v25, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v24, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v23, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v22, v0, s6
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v21
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v22, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v23, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v24, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v25, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v26, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v27, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v28, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v29, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v30, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v31, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v32, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v33, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v34, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v35, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v36, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v37, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v38, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v39, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v48, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v49, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v50, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v51, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44i16_to_v44f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v21, v52, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v51, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v50, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v48, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v38, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v36, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v34, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v32, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v30, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v28, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v24, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v25, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v26, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v27, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v21
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v24, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v25, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v26, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v27, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v28, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v30, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v32, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v34, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v36, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v38, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v48, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v50, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v51, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v52, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <44 x i16> %a, splat (i16 3)
+  %a2 = bitcast <44 x i16> %a1 to <44 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x i16> %a to <44 x half>
+  br label %end
+
+end:
+  %phi = phi <44 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x half> %phi
+}
+
+define <44 x i16> @bitcast_v44f16_to_v44i16(<44 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v44f16_to_v44i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v30
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v41
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v42
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v6
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v27
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v31
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v31, v27, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v33
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v33, v27, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v35
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v35, v27, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v52, 0x38000000, v52
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v54, v28, v27
+; GCN-NEXT:    v_or_b32_e32 v52, v30, v29
+; GCN-NEXT:    v_or_b32_e32 v50, v50, v53
+; GCN-NEXT:    v_or_b32_e32 v48, v48, v51
+; GCN-NEXT:    v_or_b32_e32 v38, v38, v49
+; GCN-NEXT:    v_or_b32_e32 v37, v37, v39
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v25
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v23
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v21
+; GCN-NEXT:    v_alignbit_b32 v40, v35, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v33, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v31, v53, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v15, v51, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v12, v49, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v8, v39, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v3, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v11, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v7, v26, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v4, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v21, v1, v21, 16
+; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v28
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v29
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v30, 0xffff, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v33, 0xffff, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v33, v33, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v36, 0xffff, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v36, v36, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v31, 0xffff, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v31, v31, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v48, 0xffff, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v48, v48, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v38, 0xffff, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v38, v38, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v37, 0xffff, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v37, v37, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v25
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v23
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v13
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v21
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v0
+; GCN-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v44f16_to_v44i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v1
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v22
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_e32 v51, 0x200, v51
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_add_f16_e32 v50, 0x200, v50
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_add_f16_e32 v49, 0x200, v49
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_add_f16_e32 v48, 0x200, v48
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_add_f16_e32 v39, 0x200, v39
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_add_f16_e32 v38, 0x200, v38
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_add_f16_e32 v37, 0x200, v37
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_add_f16_e32 v36, 0x200, v36
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_add_f16_e32 v35, 0x200, v35
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_add_f16_e32 v34, 0x200, v34
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_add_f16_e32 v33, 0x200, v33
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_add_f16_e32 v32, 0x200, v32
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_add_f16_e32 v31, 0x200, v31
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v14
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v30
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v15
+; VI-NEXT:    v_add_f16_e32 v29, 0x200, v29
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v16
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v28
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v17
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v27
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v18
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v26
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v19
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v25
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v20
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v24
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v21
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v23
+; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v44f16_to_v44i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v1
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v21, v51, v21, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v20, v50, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v49, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v48, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v39, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v38, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v37, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v36, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v35, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v34, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v33, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v32, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v31, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v30, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v29, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v28, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v27, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v26, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v25, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v24, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v23, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v22, v0, s6
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v21
+; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v22, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v23, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v24, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v25, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v26, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v27, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v28, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v29, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v30, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v31, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v32, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v33, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v34, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v35, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v36, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v37, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v38, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v39, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v48, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v49, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v50, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v51, v21, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v44f16_to_v44i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v22
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v21, v52, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v51, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v50, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v48, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v38, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v36, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v34, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v32, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v30, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v28, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v24, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v25, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v26, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v27, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v21
+; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v24, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v25, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v26, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v27, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v28, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v29, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v30, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v31, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v32, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v33, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v34, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v36, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v38, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v48, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v50, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v51, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v52, v21, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <44 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <44 x half> %a1 to <44 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <44 x half> %a to <44 x i16>
+  br label %end
+
+end:
+  %phi = phi <44 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <44 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
index 45e9c321d4aac..87fa5af74c596 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
@@ -5,8 +5,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
 
-define <24 x i32> @bitcast_v12i64_to_v24i32(<12 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v12i64_to_v24i32:
+define <24 x float> @bitcast_v24i32_to_v24f32(<24 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i32_to_v24f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
@@ -15,35 +15,35 @@ define <24 x i32> @bitcast_v12i64_to_v24i32(<12 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:  .LBB0_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v12i64_to_v24i32:
+; VI-LABEL: bitcast_v24i32_to_v24f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
@@ -52,35 +52,35 @@ define <24 x i32> @bitcast_v12i64_to_v24i32(<12 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB0_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
 ; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
-; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
-; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
 ; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
-; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
 ; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
-; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
 ; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
-; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
-; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:  .LBB0_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v12i64_to_v24i32:
+; GFX9-LABEL: bitcast_v24i32_to_v24f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
@@ -89,35 +89,35 @@ define <24 x i32> @bitcast_v12i64_to_v24i32(<12 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
-; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
-; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
-; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
-; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
-; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
 ; GFX9-NEXT:  .LBB0_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v12i64_to_v24i32:
+; GFX11-LABEL: bitcast_v24i32_to_v24f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -127,36 +127,30 @@ define <24 x i32> @bitcast_v12i64_to_v24i32(<12 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
 ; GFX11-NEXT:  .LBB0_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -164,12 +158,166 @@ define <24 x i32> @bitcast_v12i64_to_v24i32(<12 x i64> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <12 x i64> %a, splat (i64 3)
-  %a2 = bitcast <12 x i64> %a1 to <24 x i32>
+  %a1 = add <24 x i32> %a, splat (i32 3)
+  %a2 = bitcast <24 x i32> %a1 to <24 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <12 x i64> %a to <24 x i32>
+  %a3 = bitcast <24 x i32> %a to <24 x float>
+  br label %end
+
+end:
+  %phi = phi <24 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x float> %phi
+}
+
+define <24 x i32> @bitcast_v24f32_to_v24i32(<24 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f32_to_v24i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24f32_to_v24i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24f32_to_v24i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24f32_to_v24i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <24 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <24 x float> %a1 to <24 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x float> %a to <24 x i32>
   br label %end
 
 end:
@@ -185,7 +333,7 @@ define <12 x i64> @bitcast_v24i32_to_v12i64(<24 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
@@ -211,7 +359,7 @@ define <12 x i64> @bitcast_v24i32_to_v12i64(<24 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:  .LBB2_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -222,7 +370,7 @@ define <12 x i64> @bitcast_v24i32_to_v12i64(<24 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:    s_cbranch_execz .LBB2_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
 ; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
@@ -248,7 +396,7 @@ define <12 x i64> @bitcast_v24i32_to_v12i64(<24 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:  .LBB2_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -259,7 +407,7 @@ define <12 x i64> @bitcast_v24i32_to_v12i64(<24 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
 ; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
@@ -285,7 +433,7 @@ define <12 x i64> @bitcast_v24i32_to_v12i64(<24 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:  .LBB2_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -297,7 +445,7 @@ define <12 x i64> @bitcast_v24i32_to_v12i64(<24 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
 ; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
@@ -323,7 +471,7 @@ define <12 x i64> @bitcast_v24i32_to_v12i64(<24 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:  .LBB2_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -342,3 +490,18606 @@ end:
   %phi = phi <12 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <12 x i64> %phi
 }
+
+define <24 x i32> @bitcast_v12i64_to_v24i32(<12 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i64_to_v24i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i64_to_v24i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i64_to_v24i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i64_to_v24i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i64> %a, splat (i64 3)
+  %a2 = bitcast <12 x i64> %a1 to <24 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i64> %a to <24 x i32>
+  br label %end
+
+end:
+  %phi = phi <24 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i32> %phi
+}
+
+define <12 x double> @bitcast_v24i32_to_v12f64(<24 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i32_to_v12f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24i32_to_v12f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB4_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24i32_to_v12f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB4_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24i32_to_v12f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <24 x i32> %a, splat (i32 3)
+  %a2 = bitcast <24 x i32> %a1 to <12 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x i32> %a to <12 x double>
+  br label %end
+
+end:
+  %phi = phi <12 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x double> %phi
+}
+
+define <24 x i32> @bitcast_v12f64_to_v24i32(<12 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f64_to_v24i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f64_to_v24i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f64_to_v24i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f64_to_v24i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <12 x double> %a1 to <24 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x double> %a to <24 x i32>
+  br label %end
+
+end:
+  %phi = phi <24 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i32> %phi
+}
+
+define <48 x i16> @bitcast_v24i32_to_v48i16(<24 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i32_to_v48i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v25, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_alignbit_b32 v25, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v41, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_and_b32_e32 v42, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v53, v41, v53
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v40, v42, v40
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v32
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v39
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v30
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v37
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v28
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v35
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v27
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v33
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v31
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v25
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v29
+; GCN-NEXT:    buffer_store_dword v53, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24i32_to_v48i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB6_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB6_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24i32_to_v48i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB6_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB6_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v55, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v54, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v53, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v52, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v51, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v50, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v49, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v48, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v39, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v36, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v35, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v34, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v33, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v32, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v31, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v30, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v29, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v28, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v27, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v26, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v25, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24i32_to_v48i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB6_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB6_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v53, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v50, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v49, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v48, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v32, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v31, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v30, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v29, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v28, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v27, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v26, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v25, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v24, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <24 x i32> %a, splat (i32 3)
+  %a2 = bitcast <24 x i32> %a1 to <48 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x i32> %a to <48 x i16>
+  br label %end
+
+end:
+  %phi = phi <48 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x i16> %phi
+}
+
+define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48i16_to_v24i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v59
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v60
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v58
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v56
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v47
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v46
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v45
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v44
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v43
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v41
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v62
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v61
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v57
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v42
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v63
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v24
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:  .LBB7_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v62
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v61
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v57
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v42
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v63
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_or_b32_e32 v0, v59, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v60, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v58, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v56, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v47, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v46, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v45, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v44, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v43, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v41, v9
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v24, v10
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v24, v11
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v24, v12
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v24, v13
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v24, v14
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v24, v15
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v24, v16
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v24, v17
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v24, v18
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v24, v19
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v24, v20
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v24, v21
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v24, v22
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v24, v23
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48i16_to_v24i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v23
+; VI-NEXT:    v_mov_b32_e32 v33, v22
+; VI-NEXT:    v_mov_b32_e32 v34, v21
+; VI-NEXT:    v_mov_b32_e32 v35, v20
+; VI-NEXT:    v_mov_b32_e32 v36, v19
+; VI-NEXT:    v_mov_b32_e32 v37, v18
+; VI-NEXT:    v_mov_b32_e32 v38, v17
+; VI-NEXT:    v_mov_b32_e32 v39, v16
+; VI-NEXT:    v_mov_b32_e32 v48, v15
+; VI-NEXT:    v_mov_b32_e32 v49, v14
+; VI-NEXT:    v_mov_b32_e32 v50, v13
+; VI-NEXT:    v_mov_b32_e32 v51, v12
+; VI-NEXT:    v_mov_b32_e32 v52, v11
+; VI-NEXT:    v_mov_b32_e32 v53, v10
+; VI-NEXT:    v_mov_b32_e32 v54, v9
+; VI-NEXT:    v_mov_b32_e32 v55, v8
+; VI-NEXT:    v_mov_b32_e32 v40, v7
+; VI-NEXT:    v_mov_b32_e32 v41, v6
+; VI-NEXT:    v_mov_b32_e32 v42, v5
+; VI-NEXT:    v_mov_b32_e32 v43, v4
+; VI-NEXT:    v_mov_b32_e32 v44, v3
+; VI-NEXT:    v_mov_b32_e32 v45, v2
+; VI-NEXT:    v_mov_b32_e32 v46, v1
+; VI-NEXT:    v_mov_b32_e32 v47, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v23, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v23, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v23, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v23, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v23, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v23, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v23, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v23, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v23, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v23, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v23, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v23, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v23, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v23, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v23, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v23, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v23, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v23, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v23, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v23, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v23, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v23, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v23, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB7_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v23, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v1, v47, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v2, v46, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v3, v45, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v4, v44, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v5, v43, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v6, v42, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v7, v41, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v8, v40, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v9, v55, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v10, v54, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v11, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v12, v52, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v13, v51, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v14, v50, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v15, v49, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v16, v48, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v17, v39, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v18, v38, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v19, v37, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v20, v36, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v21, v35, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v22, v34, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v24, v33, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v23, v32, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:  .LBB7_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48i16_to_v24i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v22
+; GFX9-NEXT:    v_mov_b32_e32 v47, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v34, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v9
+; GFX9-NEXT:    v_mov_b32_e32 v55, v8
+; GFX9-NEXT:    v_mov_b32_e32 v40, v7
+; GFX9-NEXT:    v_mov_b32_e32 v41, v6
+; GFX9-NEXT:    v_mov_b32_e32 v42, v5
+; GFX9-NEXT:    v_mov_b32_e32 v43, v4
+; GFX9-NEXT:    v_mov_b32_e32 v44, v3
+; GFX9-NEXT:    v_mov_b32_e32 v45, v2
+; GFX9-NEXT:    v_mov_b32_e32 v46, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v47
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:  .LBB7_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48i16_to_v24i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    v_perm_b32 v0, v53, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v55, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v52, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v51, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v50, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v49, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v48, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v39, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v38, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v37, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v36, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v35, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v34, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v33, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v32, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v31, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v30, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v29, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v28, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v27, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v26, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v25, v23, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <48 x i16> %a, splat (i16 3)
+  %a2 = bitcast <48 x i16> %a1 to <24 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x i16> %a to <24 x i32>
+  br label %end
+
+end:
+  %phi = phi <24 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i32> %phi
+}
+
+define <48 x half> @bitcast_v24i32_to_v48f16(<24 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24i32_to_v48f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v12
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v25
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v26
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v27
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v28
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v29
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v37
+; GCN-NEXT:    v_mov_b32_e32 v37, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v39
+; GCN-NEXT:    v_mov_b32_e32 v39, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v49
+; GCN-NEXT:    v_mov_b32_e32 v49, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:  .LBB8_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:  .LBB8_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v61
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v59
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v57
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v47
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v46
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v44
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v42
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v40
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v55
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v53
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v41
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v26, v24
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v25, v27, v25
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v37, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v39, v51, v50
+; GCN-NEXT:    v_or_b32_e32 v48, v54, v53
+; GCN-NEXT:    v_or_b32_e32 v49, v55, v52
+; GCN-NEXT:    v_or_b32_e32 v50, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v51, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v52, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v53, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v54, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v55, v59, v58
+; GCN-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v49, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v51, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24i32_to_v48f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB8_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB8_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24i32_to_v48f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB8_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB8_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v55, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v54, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v53, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v52, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v51, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v50, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v49, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v48, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v39, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v36, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v35, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v34, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v33, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v32, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v31, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v30, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v29, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v28, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v27, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v26, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v25, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24i32_to_v48f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB8_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB8_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v53, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v50, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v49, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v48, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v32, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v31, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v30, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v29, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v28, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v27, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v26, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v25, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v24, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <24 x i32> %a, splat (i32 3)
+  %a2 = bitcast <24 x i32> %a1 to <48 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x i32> %a to <48 x half>
+  br label %end
+
+end:
+  %phi = phi <48 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x half> %phi
+}
+
+define <24 x i32> @bitcast_v48f16_to_v24i32(<48 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48f16_to_v24i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v32
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v63
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v0, v54, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v52, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v2, v50, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v3, v48, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v38, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v5, v36, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v34, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v62
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v60
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v42
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v24, v10
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v24, v11
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v24, v12
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v24, v13
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v24, v14
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v24, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v61, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v59, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v57, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v47, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v32, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v43, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v41, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v40, v23
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:  .LBB9_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v52
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v50
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v48
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v36
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v34
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v40
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v13, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v14, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v15, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v16, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v17, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v18, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v20, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v21, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v22, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v23, v37, v36
+; GCN-NEXT:  .LBB9_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48f16_to_v24i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v23
+; VI-NEXT:    v_mov_b32_e32 v33, v22
+; VI-NEXT:    v_mov_b32_e32 v34, v21
+; VI-NEXT:    v_mov_b32_e32 v35, v20
+; VI-NEXT:    v_mov_b32_e32 v36, v19
+; VI-NEXT:    v_mov_b32_e32 v37, v18
+; VI-NEXT:    v_mov_b32_e32 v38, v17
+; VI-NEXT:    v_mov_b32_e32 v39, v16
+; VI-NEXT:    v_mov_b32_e32 v48, v15
+; VI-NEXT:    v_mov_b32_e32 v49, v14
+; VI-NEXT:    v_mov_b32_e32 v50, v13
+; VI-NEXT:    v_mov_b32_e32 v51, v12
+; VI-NEXT:    v_mov_b32_e32 v52, v11
+; VI-NEXT:    v_mov_b32_e32 v53, v10
+; VI-NEXT:    v_mov_b32_e32 v54, v9
+; VI-NEXT:    v_mov_b32_e32 v55, v8
+; VI-NEXT:    v_mov_b32_e32 v40, v7
+; VI-NEXT:    v_mov_b32_e32 v41, v6
+; VI-NEXT:    v_mov_b32_e32 v42, v5
+; VI-NEXT:    v_mov_b32_e32 v43, v4
+; VI-NEXT:    v_mov_b32_e32 v44, v3
+; VI-NEXT:    v_mov_b32_e32 v45, v2
+; VI-NEXT:    v_mov_b32_e32 v46, v1
+; VI-NEXT:    v_mov_b32_e32 v47, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v23, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v23, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v23, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v23, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v23, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v23, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v23, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v23, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v23, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v23, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v23, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v23, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v23, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v23, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v23, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v23, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v23, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v23, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v23, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v23, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v23, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v23, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v23, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB9_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v23, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v47, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v46, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v45, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v44, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v43, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v42, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v41, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v40, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v55, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v54, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v52, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v51, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v50, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v49, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v48, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v39, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v38, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v37, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v36, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v35, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v34, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v33, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v22, v24, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v32, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:  .LBB9_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48f16_to_v24i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v22
+; GFX9-NEXT:    v_mov_b32_e32 v47, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v34, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v9
+; GFX9-NEXT:    v_mov_b32_e32 v55, v8
+; GFX9-NEXT:    v_mov_b32_e32 v40, v7
+; GFX9-NEXT:    v_mov_b32_e32 v41, v6
+; GFX9-NEXT:    v_mov_b32_e32 v42, v5
+; GFX9-NEXT:    v_mov_b32_e32 v43, v4
+; GFX9-NEXT:    v_mov_b32_e32 v44, v3
+; GFX9-NEXT:    v_mov_b32_e32 v45, v2
+; GFX9-NEXT:    v_mov_b32_e32 v46, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v47
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:  .LBB9_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48f16_to_v24i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    v_perm_b32 v0, v53, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v55, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v52, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v51, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v50, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v49, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v48, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v39, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v38, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v37, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v36, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v35, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v34, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v33, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v32, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v31, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v30, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v29, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v28, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v27, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v26, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v25, v23, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <48 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <48 x half> %a1 to <24 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x half> %a to <24 x i32>
+  br label %end
+
+end:
+  %phi = phi <24 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x i32> %phi
+}
+
+define <12 x i64> @bitcast_v24f32_to_v12i64(<24 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f32_to_v12i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24f32_to_v12i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24f32_to_v12i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24f32_to_v12i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <24 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <24 x float> %a1 to <12 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x float> %a to <12 x i64>
+  br label %end
+
+end:
+  %phi = phi <12 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i64> %phi
+}
+
+define <24 x float> @bitcast_v12i64_to_v24f32(<12 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i64_to_v24f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i64_to_v24f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i64_to_v24f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i64_to_v24f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i64> %a, splat (i64 3)
+  %a2 = bitcast <12 x i64> %a1 to <24 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i64> %a to <24 x float>
+  br label %end
+
+end:
+  %phi = phi <24 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x float> %phi
+}
+
+define <12 x double> @bitcast_v24f32_to_v12f64(<24 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f32_to_v12f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24f32_to_v12f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24f32_to_v12f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24f32_to_v12f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB12_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <24 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <24 x float> %a1 to <12 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x float> %a to <12 x double>
+  br label %end
+
+end:
+  %phi = phi <12 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x double> %phi
+}
+
+define <24 x float> @bitcast_v12f64_to_v24f32(<12 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f64_to_v24f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f64_to_v24f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f64_to_v24f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f64_to_v24f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <12 x double> %a1 to <24 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x double> %a to <24 x float>
+  br label %end
+
+end:
+  %phi = phi <24 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x float> %phi
+}
+
+define <48 x i16> @bitcast_v24f32_to_v48i16(<24 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f32_to_v48i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v25, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_alignbit_b32 v25, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v41, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_and_b32_e32 v42, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v53, v41, v53
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v40, v42, v40
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v32
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v39
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v30
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v37
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v28
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v35
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v27
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v33
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v31
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v25
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v29
+; GCN-NEXT:    buffer_store_dword v53, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24f32_to_v48i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB14_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB14_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24f32_to_v48i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB14_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB14_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v55, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v54, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v53, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v52, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v51, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v50, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v49, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v48, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v39, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v36, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v35, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v34, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v33, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v32, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v31, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v30, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v29, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v28, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v27, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v26, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v25, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24f32_to_v48i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB14_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB14_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v53, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v50, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v49, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v48, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v32, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v31, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v30, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v29, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v28, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v27, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v26, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v25, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v24, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <24 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <24 x float> %a1 to <48 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x float> %a to <48 x i16>
+  br label %end
+
+end:
+  %phi = phi <48 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x i16> %phi
+}
+
+define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48i16_to_v24f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v59
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v60
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v58
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v56
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v47
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v46
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v45
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v44
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v43
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v41
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v62
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v61
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v57
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v42
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v63
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v24
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:  .LBB15_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v62
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v61
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v57
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v42
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v63
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_or_b32_e32 v0, v59, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v60, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v58, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v56, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v47, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v46, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v45, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v44, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v43, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v41, v9
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v24, v10
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v24, v11
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v24, v12
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v24, v13
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v24, v14
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v24, v15
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v24, v16
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v24, v17
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v24, v18
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v24, v19
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v24, v20
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v24, v21
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v24, v22
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v24, v23
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:  .LBB15_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48i16_to_v24f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v23
+; VI-NEXT:    v_mov_b32_e32 v33, v22
+; VI-NEXT:    v_mov_b32_e32 v34, v21
+; VI-NEXT:    v_mov_b32_e32 v35, v20
+; VI-NEXT:    v_mov_b32_e32 v36, v19
+; VI-NEXT:    v_mov_b32_e32 v37, v18
+; VI-NEXT:    v_mov_b32_e32 v38, v17
+; VI-NEXT:    v_mov_b32_e32 v39, v16
+; VI-NEXT:    v_mov_b32_e32 v48, v15
+; VI-NEXT:    v_mov_b32_e32 v49, v14
+; VI-NEXT:    v_mov_b32_e32 v50, v13
+; VI-NEXT:    v_mov_b32_e32 v51, v12
+; VI-NEXT:    v_mov_b32_e32 v52, v11
+; VI-NEXT:    v_mov_b32_e32 v53, v10
+; VI-NEXT:    v_mov_b32_e32 v54, v9
+; VI-NEXT:    v_mov_b32_e32 v55, v8
+; VI-NEXT:    v_mov_b32_e32 v40, v7
+; VI-NEXT:    v_mov_b32_e32 v41, v6
+; VI-NEXT:    v_mov_b32_e32 v42, v5
+; VI-NEXT:    v_mov_b32_e32 v43, v4
+; VI-NEXT:    v_mov_b32_e32 v44, v3
+; VI-NEXT:    v_mov_b32_e32 v45, v2
+; VI-NEXT:    v_mov_b32_e32 v46, v1
+; VI-NEXT:    v_mov_b32_e32 v47, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v23, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v23, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v23, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v23, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v23, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v23, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v23, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v23, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v23, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v23, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v23, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v23, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v23, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v23, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v23, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v23, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v23, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v23, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v23, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v23, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v23, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v23, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v23, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB15_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v23, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v1, v47, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v2, v46, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v3, v45, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v4, v44, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v5, v43, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v6, v42, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v7, v41, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v8, v40, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v9, v55, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v10, v54, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v11, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v12, v52, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v13, v51, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v14, v50, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v15, v49, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v16, v48, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v17, v39, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v18, v38, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v19, v37, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v20, v36, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v21, v35, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v22, v34, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v24, v33, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v23, v32, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:  .LBB15_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48i16_to_v24f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v22
+; GFX9-NEXT:    v_mov_b32_e32 v47, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v34, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v9
+; GFX9-NEXT:    v_mov_b32_e32 v55, v8
+; GFX9-NEXT:    v_mov_b32_e32 v40, v7
+; GFX9-NEXT:    v_mov_b32_e32 v41, v6
+; GFX9-NEXT:    v_mov_b32_e32 v42, v5
+; GFX9-NEXT:    v_mov_b32_e32 v43, v4
+; GFX9-NEXT:    v_mov_b32_e32 v44, v3
+; GFX9-NEXT:    v_mov_b32_e32 v45, v2
+; GFX9-NEXT:    v_mov_b32_e32 v46, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v47
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:  .LBB15_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48i16_to_v24f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    v_perm_b32 v0, v53, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v55, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v52, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v51, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v50, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v49, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v48, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v39, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v38, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v37, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v36, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v35, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v34, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v33, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v32, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v31, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v30, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v29, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v28, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v27, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v26, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v25, v23, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <48 x i16> %a, splat (i16 3)
+  %a2 = bitcast <48 x i16> %a1 to <24 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x i16> %a to <24 x float>
+  br label %end
+
+end:
+  %phi = phi <24 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x float> %phi
+}
+
+define <48 x half> @bitcast_v24f32_to_v48f16(<24 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v24f32_to_v48f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v12
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v25
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v26
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v27
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v28
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v29
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v37
+; GCN-NEXT:    v_mov_b32_e32 v37, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v39
+; GCN-NEXT:    v_mov_b32_e32 v39, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v49
+; GCN-NEXT:    v_mov_b32_e32 v49, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:  .LBB16_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:  .LBB16_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v61
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v59
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v57
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v47
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v46
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v44
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v42
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v40
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v55
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v53
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v41
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v26, v24
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v25, v27, v25
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v37, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v39, v51, v50
+; GCN-NEXT:    v_or_b32_e32 v48, v54, v53
+; GCN-NEXT:    v_or_b32_e32 v49, v55, v52
+; GCN-NEXT:    v_or_b32_e32 v50, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v51, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v52, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v53, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v54, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v55, v59, v58
+; GCN-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v49, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v51, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v24f32_to_v48f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB16_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB16_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v24f32_to_v48f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB16_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB16_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v55, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v54, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v53, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v52, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v51, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v50, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v49, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v48, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v39, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v36, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v35, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v34, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v33, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v32, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v31, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v30, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v29, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v28, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v27, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v26, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v25, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v24f32_to_v48f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB16_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB16_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v53, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v50, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v49, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v48, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v32, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v31, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v30, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v29, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v28, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v27, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v26, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v25, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v24, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <24 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <24 x float> %a1 to <48 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <24 x float> %a to <48 x half>
+  br label %end
+
+end:
+  %phi = phi <48 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x half> %phi
+}
+
+define <24 x float> @bitcast_v48f16_to_v24f32(<48 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48f16_to_v24f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v32
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v63
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v0, v54, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v52, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v2, v50, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v3, v48, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v38, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v5, v36, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v34, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v62
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v60
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v42
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v24, v10
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v24, v11
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v24, v12
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v24, v13
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v24, v14
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v24, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v61, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v59, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v57, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v47, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v32, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v43, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v41, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v40, v23
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v52
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v50
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v48
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v36
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v34
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v40
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v13, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v14, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v15, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v16, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v17, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v18, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v20, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v21, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v22, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v23, v37, v36
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48f16_to_v24f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v23
+; VI-NEXT:    v_mov_b32_e32 v33, v22
+; VI-NEXT:    v_mov_b32_e32 v34, v21
+; VI-NEXT:    v_mov_b32_e32 v35, v20
+; VI-NEXT:    v_mov_b32_e32 v36, v19
+; VI-NEXT:    v_mov_b32_e32 v37, v18
+; VI-NEXT:    v_mov_b32_e32 v38, v17
+; VI-NEXT:    v_mov_b32_e32 v39, v16
+; VI-NEXT:    v_mov_b32_e32 v48, v15
+; VI-NEXT:    v_mov_b32_e32 v49, v14
+; VI-NEXT:    v_mov_b32_e32 v50, v13
+; VI-NEXT:    v_mov_b32_e32 v51, v12
+; VI-NEXT:    v_mov_b32_e32 v52, v11
+; VI-NEXT:    v_mov_b32_e32 v53, v10
+; VI-NEXT:    v_mov_b32_e32 v54, v9
+; VI-NEXT:    v_mov_b32_e32 v55, v8
+; VI-NEXT:    v_mov_b32_e32 v40, v7
+; VI-NEXT:    v_mov_b32_e32 v41, v6
+; VI-NEXT:    v_mov_b32_e32 v42, v5
+; VI-NEXT:    v_mov_b32_e32 v43, v4
+; VI-NEXT:    v_mov_b32_e32 v44, v3
+; VI-NEXT:    v_mov_b32_e32 v45, v2
+; VI-NEXT:    v_mov_b32_e32 v46, v1
+; VI-NEXT:    v_mov_b32_e32 v47, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v23, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v23, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v23, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v23, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v23, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v23, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v23, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v23, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v23, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v23, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v23, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v23, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v23, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v23, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v23, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v23, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v23, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v23, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v23, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v23, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v23, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v23, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v23, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB17_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v23, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v47, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v46, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v45, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v44, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v43, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v42, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v41, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v40, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v55, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v54, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v52, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v51, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v50, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v49, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v48, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v39, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v38, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v37, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v36, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v35, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v34, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v33, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v22, v24, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v32, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:  .LBB17_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48f16_to_v24f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v22
+; GFX9-NEXT:    v_mov_b32_e32 v47, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v34, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v9
+; GFX9-NEXT:    v_mov_b32_e32 v55, v8
+; GFX9-NEXT:    v_mov_b32_e32 v40, v7
+; GFX9-NEXT:    v_mov_b32_e32 v41, v6
+; GFX9-NEXT:    v_mov_b32_e32 v42, v5
+; GFX9-NEXT:    v_mov_b32_e32 v43, v4
+; GFX9-NEXT:    v_mov_b32_e32 v44, v3
+; GFX9-NEXT:    v_mov_b32_e32 v45, v2
+; GFX9-NEXT:    v_mov_b32_e32 v46, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v47
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:  .LBB17_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48f16_to_v24f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    v_perm_b32 v0, v53, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v55, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v52, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v51, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v50, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v49, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v48, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v39, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v38, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v37, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v36, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v35, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v34, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v33, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v32, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v31, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v30, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v29, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v28, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v27, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v26, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v25, v23, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <48 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <48 x half> %a1 to <24 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x half> %a to <24 x float>
+  br label %end
+
+end:
+  %phi = phi <24 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <24 x float> %phi
+}
+
+define <12 x double> @bitcast_v12i64_to_v12f64(<12 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i64_to_v12f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i64_to_v12f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i64_to_v12f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i64_to_v12f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i64> %a, splat (i64 3)
+  %a2 = bitcast <12 x i64> %a1 to <12 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i64> %a to <12 x double>
+  br label %end
+
+end:
+  %phi = phi <12 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x double> %phi
+}
+
+define <12 x i64> @bitcast_v12f64_to_v12i64(<12 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f64_to_v12i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f64_to_v12i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f64_to_v12i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f64_to_v12i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <12 x double> %a1 to <12 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x double> %a to <12 x i64>
+  br label %end
+
+end:
+  %phi = phi <12 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i64> %phi
+}
+
+define <48 x i16> @bitcast_v12i64_to_v48i16(<12 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i64_to_v48i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v25, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_addc_u32_e32 v24, vcc, 0, v24, vcc
+; GCN-NEXT:    v_alignbit_b32 v25, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v41, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_and_b32_e32 v42, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v53, v41, v53
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v40, v42, v40
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v32
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v39
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v29
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v37
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v28
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v35
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v27
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v33
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v31
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v25
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v30
+; GCN-NEXT:    buffer_store_dword v53, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i64_to_v48i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB20_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB20_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i64_to_v48i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB20_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB20_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v55, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v54, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v53, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v52, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v51, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v50, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v49, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v48, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v39, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v36, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v35, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v34, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v33, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v32, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v31, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v30, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v29, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v28, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v27, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v26, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v25, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i64_to_v48i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB20_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB20_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v53, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v50, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v49, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v48, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v32, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v31, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v30, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v29, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v28, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v27, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v26, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v25, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v24, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i64> %a, splat (i64 3)
+  %a2 = bitcast <12 x i64> %a1 to <48 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i64> %a to <48 x i16>
+  br label %end
+
+end:
+  %phi = phi <48 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x i16> %phi
+}
+
+define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48i16_to_v12i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v59
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v60
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v58
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v56
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v47
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v46
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v45
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v44
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v43
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v41
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v62
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v61
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v57
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v42
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v63
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v24
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:  .LBB21_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v62
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v61
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v57
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v42
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v63
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_or_b32_e32 v0, v59, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v60, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v58, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v56, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v47, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v46, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v45, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v44, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v43, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v41, v9
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v24, v10
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v24, v11
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v24, v12
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v24, v13
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v24, v14
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v24, v15
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v24, v16
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v24, v17
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v24, v18
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v24, v19
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v24, v20
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v24, v21
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v24, v22
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v24, v23
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:  .LBB21_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48i16_to_v12i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v23
+; VI-NEXT:    v_mov_b32_e32 v33, v22
+; VI-NEXT:    v_mov_b32_e32 v34, v21
+; VI-NEXT:    v_mov_b32_e32 v35, v20
+; VI-NEXT:    v_mov_b32_e32 v36, v19
+; VI-NEXT:    v_mov_b32_e32 v37, v18
+; VI-NEXT:    v_mov_b32_e32 v38, v17
+; VI-NEXT:    v_mov_b32_e32 v39, v16
+; VI-NEXT:    v_mov_b32_e32 v48, v15
+; VI-NEXT:    v_mov_b32_e32 v49, v14
+; VI-NEXT:    v_mov_b32_e32 v50, v13
+; VI-NEXT:    v_mov_b32_e32 v51, v12
+; VI-NEXT:    v_mov_b32_e32 v52, v11
+; VI-NEXT:    v_mov_b32_e32 v53, v10
+; VI-NEXT:    v_mov_b32_e32 v54, v9
+; VI-NEXT:    v_mov_b32_e32 v55, v8
+; VI-NEXT:    v_mov_b32_e32 v40, v7
+; VI-NEXT:    v_mov_b32_e32 v41, v6
+; VI-NEXT:    v_mov_b32_e32 v42, v5
+; VI-NEXT:    v_mov_b32_e32 v43, v4
+; VI-NEXT:    v_mov_b32_e32 v44, v3
+; VI-NEXT:    v_mov_b32_e32 v45, v2
+; VI-NEXT:    v_mov_b32_e32 v46, v1
+; VI-NEXT:    v_mov_b32_e32 v47, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v23, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v23, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v23, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v23, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v23, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v23, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v23, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v23, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v23, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v23, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v23, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v23, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v23, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v23, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v23, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v23, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v23, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v23, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v23, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v23, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v23, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v23, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v23, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB21_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v23, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v1, v47, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v2, v46, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v3, v45, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v4, v44, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v5, v43, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v6, v42, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v7, v41, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v8, v40, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v9, v55, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v10, v54, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v11, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v12, v52, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v13, v51, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v14, v50, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v15, v49, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v16, v48, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v17, v39, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v18, v38, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v19, v37, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v20, v36, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v21, v35, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v22, v34, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v24, v33, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v23, v32, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:  .LBB21_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48i16_to_v12i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v22
+; GFX9-NEXT:    v_mov_b32_e32 v47, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v34, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v9
+; GFX9-NEXT:    v_mov_b32_e32 v55, v8
+; GFX9-NEXT:    v_mov_b32_e32 v40, v7
+; GFX9-NEXT:    v_mov_b32_e32 v41, v6
+; GFX9-NEXT:    v_mov_b32_e32 v42, v5
+; GFX9-NEXT:    v_mov_b32_e32 v43, v4
+; GFX9-NEXT:    v_mov_b32_e32 v44, v3
+; GFX9-NEXT:    v_mov_b32_e32 v45, v2
+; GFX9-NEXT:    v_mov_b32_e32 v46, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v47
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:  .LBB21_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48i16_to_v12i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    v_perm_b32 v0, v53, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v55, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v52, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v51, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v50, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v49, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v48, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v39, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v38, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v37, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v36, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v35, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v34, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v33, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v32, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v31, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v30, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v29, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v28, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v27, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v26, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v25, v23, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <48 x i16> %a, splat (i16 3)
+  %a2 = bitcast <48 x i16> %a1 to <12 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x i16> %a to <12 x i64>
+  br label %end
+
+end:
+  %phi = phi <12 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i64> %phi
+}
+
+define <48 x half> @bitcast_v12i64_to_v48f16(<12 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12i64_to_v48f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v12
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v25
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v26
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v27
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v28
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v29
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v37
+; GCN-NEXT:    v_mov_b32_e32 v37, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v39
+; GCN-NEXT:    v_mov_b32_e32 v39, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v49
+; GCN-NEXT:    v_mov_b32_e32 v49, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:  .LBB22_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_addc_u32_e32 v24, vcc, 0, v24, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:  .LBB22_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v61
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v59
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v57
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v47
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v46
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v44
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v42
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v40
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v55
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v53
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v41
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v26, v24
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v25, v27, v25
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v37, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v39, v51, v50
+; GCN-NEXT:    v_or_b32_e32 v48, v54, v53
+; GCN-NEXT:    v_or_b32_e32 v49, v55, v52
+; GCN-NEXT:    v_or_b32_e32 v50, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v51, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v52, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v53, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v54, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v55, v59, v58
+; GCN-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v49, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v51, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12i64_to_v48f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB22_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB22_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12i64_to_v48f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB22_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB22_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v55, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v54, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v53, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v52, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v51, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v50, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v49, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v48, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v39, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v36, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v35, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v34, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v33, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v32, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v31, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v30, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v29, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v28, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v27, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v26, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v25, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12i64_to_v48f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB22_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB22_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v53, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v50, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v49, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v48, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v32, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v31, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v30, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v29, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v28, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v27, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v26, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v25, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v24, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <12 x i64> %a, splat (i64 3)
+  %a2 = bitcast <12 x i64> %a1 to <48 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x i64> %a to <48 x half>
+  br label %end
+
+end:
+  %phi = phi <48 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x half> %phi
+}
+
+define <12 x i64> @bitcast_v48f16_to_v12i64(<48 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48f16_to_v12i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v32
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v63
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v0, v54, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v52, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v2, v50, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v3, v48, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v38, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v5, v36, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v34, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v62
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v60
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v42
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v24, v10
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v24, v11
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v24, v12
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v24, v13
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v24, v14
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v24, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v61, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v59, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v57, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v47, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v32, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v43, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v41, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v40, v23
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:  .LBB23_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v52
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v50
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v48
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v36
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v34
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v40
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v13, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v14, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v15, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v16, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v17, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v18, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v20, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v21, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v22, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v23, v37, v36
+; GCN-NEXT:  .LBB23_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48f16_to_v12i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v23
+; VI-NEXT:    v_mov_b32_e32 v33, v22
+; VI-NEXT:    v_mov_b32_e32 v34, v21
+; VI-NEXT:    v_mov_b32_e32 v35, v20
+; VI-NEXT:    v_mov_b32_e32 v36, v19
+; VI-NEXT:    v_mov_b32_e32 v37, v18
+; VI-NEXT:    v_mov_b32_e32 v38, v17
+; VI-NEXT:    v_mov_b32_e32 v39, v16
+; VI-NEXT:    v_mov_b32_e32 v48, v15
+; VI-NEXT:    v_mov_b32_e32 v49, v14
+; VI-NEXT:    v_mov_b32_e32 v50, v13
+; VI-NEXT:    v_mov_b32_e32 v51, v12
+; VI-NEXT:    v_mov_b32_e32 v52, v11
+; VI-NEXT:    v_mov_b32_e32 v53, v10
+; VI-NEXT:    v_mov_b32_e32 v54, v9
+; VI-NEXT:    v_mov_b32_e32 v55, v8
+; VI-NEXT:    v_mov_b32_e32 v40, v7
+; VI-NEXT:    v_mov_b32_e32 v41, v6
+; VI-NEXT:    v_mov_b32_e32 v42, v5
+; VI-NEXT:    v_mov_b32_e32 v43, v4
+; VI-NEXT:    v_mov_b32_e32 v44, v3
+; VI-NEXT:    v_mov_b32_e32 v45, v2
+; VI-NEXT:    v_mov_b32_e32 v46, v1
+; VI-NEXT:    v_mov_b32_e32 v47, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v23, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v23, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v23, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v23, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v23, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v23, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v23, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v23, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v23, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v23, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v23, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v23, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v23, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v23, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v23, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v23, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v23, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v23, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v23, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v23, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v23, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v23, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v23, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB23_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v23, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v47, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v46, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v45, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v44, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v43, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v42, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v41, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v40, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v55, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v54, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v52, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v51, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v50, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v49, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v48, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v39, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v38, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v37, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v36, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v35, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v34, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v33, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v22, v24, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v32, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:  .LBB23_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48f16_to_v12i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v22
+; GFX9-NEXT:    v_mov_b32_e32 v47, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v34, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v9
+; GFX9-NEXT:    v_mov_b32_e32 v55, v8
+; GFX9-NEXT:    v_mov_b32_e32 v40, v7
+; GFX9-NEXT:    v_mov_b32_e32 v41, v6
+; GFX9-NEXT:    v_mov_b32_e32 v42, v5
+; GFX9-NEXT:    v_mov_b32_e32 v43, v4
+; GFX9-NEXT:    v_mov_b32_e32 v44, v3
+; GFX9-NEXT:    v_mov_b32_e32 v45, v2
+; GFX9-NEXT:    v_mov_b32_e32 v46, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v47
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:  .LBB23_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48f16_to_v12i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    v_perm_b32 v0, v53, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v55, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v52, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v51, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v50, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v49, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v48, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v39, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v38, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v37, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v36, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v35, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v34, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v33, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v32, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v31, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v30, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v29, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v28, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v27, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v26, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v25, v23, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <48 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <48 x half> %a1 to <12 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x half> %a to <12 x i64>
+  br label %end
+
+end:
+  %phi = phi <12 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x i64> %phi
+}
+
+define <48 x i16> @bitcast_v12f64_to_v48i16(<12 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f64_to_v48i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v25, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_alignbit_b32 v25, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v41, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_and_b32_e32 v42, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v43, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v44, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_and_b32_e32 v45, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_and_b32_e32 v46, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_and_b32_e32 v47, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v53, v41, v53
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v40, v42, v40
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v50, v43, v50
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 56, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v55, v44, v55
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v48, v45, v48
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v54, v46, v54
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 0x44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v39, v47, v39
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 0x48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v52, v56, v52
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 0x4c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x50, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x58, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v30
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v36
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v29
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v35
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v28
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v34
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v27
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v33
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v32
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v25
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v31
+; GCN-NEXT:    buffer_store_dword v53, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f64_to_v48i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB24_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB24_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f64_to_v48i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB24_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB24_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v55, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v54, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v53, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v52, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v51, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v50, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v49, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v48, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v39, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v36, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v35, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v34, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v33, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v32, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v31, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v30, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v29, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v28, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v27, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v26, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v25, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f64_to_v48i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB24_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB24_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v53, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v50, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v49, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v48, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v32, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v31, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v30, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v29, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v28, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v27, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v26, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v25, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v24, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <12 x double> %a1 to <48 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x double> %a to <48 x i16>
+  br label %end
+
+end:
+  %phi = phi <48 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x i16> %phi
+}
+
+define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48i16_to_v12f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v33, v28
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:68
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v59
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v60
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v58
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v56
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v47
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v46
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v45
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v44
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v43
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v41
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v62
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v61
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v57
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v42
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v63
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v24
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v24
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:  .LBB25_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v33
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v62
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v61
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v57
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v42
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v63
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_or_b32_e32 v0, v59, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v60, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v58, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v56, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v47, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v46, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v45, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v44, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v43, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v41, v9
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v24, v10
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v24, v11
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v24, v12
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v24, v13
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v24, v14
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v24, v15
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v24, v16
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v24, v17
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v24, v18
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v24, v19
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v24, v20
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v24, v21
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v24, v22
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v24, v23
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:  .LBB25_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48i16_to_v12f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v23
+; VI-NEXT:    v_mov_b32_e32 v33, v22
+; VI-NEXT:    v_mov_b32_e32 v34, v21
+; VI-NEXT:    v_mov_b32_e32 v35, v20
+; VI-NEXT:    v_mov_b32_e32 v36, v19
+; VI-NEXT:    v_mov_b32_e32 v37, v18
+; VI-NEXT:    v_mov_b32_e32 v38, v17
+; VI-NEXT:    v_mov_b32_e32 v39, v16
+; VI-NEXT:    v_mov_b32_e32 v48, v15
+; VI-NEXT:    v_mov_b32_e32 v49, v14
+; VI-NEXT:    v_mov_b32_e32 v50, v13
+; VI-NEXT:    v_mov_b32_e32 v51, v12
+; VI-NEXT:    v_mov_b32_e32 v52, v11
+; VI-NEXT:    v_mov_b32_e32 v53, v10
+; VI-NEXT:    v_mov_b32_e32 v54, v9
+; VI-NEXT:    v_mov_b32_e32 v55, v8
+; VI-NEXT:    v_mov_b32_e32 v40, v7
+; VI-NEXT:    v_mov_b32_e32 v41, v6
+; VI-NEXT:    v_mov_b32_e32 v42, v5
+; VI-NEXT:    v_mov_b32_e32 v43, v4
+; VI-NEXT:    v_mov_b32_e32 v44, v3
+; VI-NEXT:    v_mov_b32_e32 v45, v2
+; VI-NEXT:    v_mov_b32_e32 v46, v1
+; VI-NEXT:    v_mov_b32_e32 v47, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v23, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v23, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v23, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v23, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v23, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v23, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v23, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v23, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v23, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v23, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v23, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v23, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v23, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v23, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v23, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v23, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v23, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v23, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v23, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v23, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v23, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v23, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v23, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB25_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v23, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v1, v47, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v2, v46, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v3, v45, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v4, v44, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v5, v43, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v6, v42, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v7, v41, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v8, v40, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v9, v55, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v10, v54, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v11, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v12, v52, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v13, v51, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v14, v50, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v15, v49, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v16, v48, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v17, v39, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v18, v38, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v19, v37, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v20, v36, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v21, v35, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v22, v34, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v24, v33, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v23, v32, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:  .LBB25_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48i16_to_v12f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v22
+; GFX9-NEXT:    v_mov_b32_e32 v47, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v34, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v9
+; GFX9-NEXT:    v_mov_b32_e32 v55, v8
+; GFX9-NEXT:    v_mov_b32_e32 v40, v7
+; GFX9-NEXT:    v_mov_b32_e32 v41, v6
+; GFX9-NEXT:    v_mov_b32_e32 v42, v5
+; GFX9-NEXT:    v_mov_b32_e32 v43, v4
+; GFX9-NEXT:    v_mov_b32_e32 v44, v3
+; GFX9-NEXT:    v_mov_b32_e32 v45, v2
+; GFX9-NEXT:    v_mov_b32_e32 v46, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v47
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:  .LBB25_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48i16_to_v12f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    v_perm_b32 v0, v53, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v55, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v52, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v51, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v50, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v49, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v48, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v39, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v38, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v37, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v36, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v35, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v34, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v33, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v32, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v31, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v30, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v29, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v28, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v27, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v26, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v25, v23, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <48 x i16> %a, splat (i16 3)
+  %a2 = bitcast <48 x i16> %a1 to <12 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x i16> %a to <12 x double>
+  br label %end
+
+end:
+  %phi = phi <12 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x double> %phi
+}
+
+define <48 x half> @bitcast_v12f64_to_v48f16(<12 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v12f64_to_v48f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; kill: killed $vgpr34
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v12
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v25
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v26
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v27
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v28
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v29
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v37
+; GCN-NEXT:    v_mov_b32_e32 v37, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v39
+; GCN-NEXT:    v_mov_b32_e32 v39, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v49
+; GCN-NEXT:    v_mov_b32_e32 v49, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v61
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v59
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v57
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v47
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v46
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v44
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v42
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v40
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v55
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v53
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v41
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v54
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v26, v24
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v25, v27, v25
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v37, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v39, v51, v50
+; GCN-NEXT:    v_or_b32_e32 v48, v54, v53
+; GCN-NEXT:    v_or_b32_e32 v49, v55, v52
+; GCN-NEXT:    v_or_b32_e32 v50, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v51, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v52, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v53, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v54, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v55, v59, v58
+; GCN-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v49, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v51, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v12f64_to_v48f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr25
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB26_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:  .LBB26_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v12f64_to_v48f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB26_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX9-NEXT:  .LBB26_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v55, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v54, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v53, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v52, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v51, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v50, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v49, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v48, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v39, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v36, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v35, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v34, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v33, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v32, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v31, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v30, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v29, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v28, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v27, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v26, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v25, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v12f64_to_v48f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    ; implicit-def: $vgpr25
+; GFX11-NEXT:    ; implicit-def: $vgpr24
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB26_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; GFX11-NEXT:  .LBB26_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v53, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v50, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v49, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v48, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v35, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v34, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v33, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v32, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v31, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v30, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v29, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v28, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v27, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v26, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v25, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v24, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <12 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <12 x double> %a1 to <48 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <12 x double> %a to <48 x half>
+  br label %end
+
+end:
+  %phi = phi <48 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x half> %phi
+}
+
+define <12 x double> @bitcast_v48f16_to_v12f64(<48 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48f16_to_v12f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v32
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v63
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v0, v54, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v52, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v2, v50, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v3, v48, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v4, v38, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v5, v36, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v6, v34, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v62
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v60
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v42
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v24, v10
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v24, v11
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v24, v12
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v24, v13
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v24, v14
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v24, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v61, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v59, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v57, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v47, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v32, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v43, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v41, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v40, v23
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; kill: killed $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v52
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v50
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v48
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v36
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v34
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v33
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v40
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v13, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v14, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v15, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v16, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v17, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v18, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v19, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v20, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v21, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v22, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v23, v37, v36
+; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48f16_to_v12f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v23
+; VI-NEXT:    v_mov_b32_e32 v33, v22
+; VI-NEXT:    v_mov_b32_e32 v34, v21
+; VI-NEXT:    v_mov_b32_e32 v35, v20
+; VI-NEXT:    v_mov_b32_e32 v36, v19
+; VI-NEXT:    v_mov_b32_e32 v37, v18
+; VI-NEXT:    v_mov_b32_e32 v38, v17
+; VI-NEXT:    v_mov_b32_e32 v39, v16
+; VI-NEXT:    v_mov_b32_e32 v48, v15
+; VI-NEXT:    v_mov_b32_e32 v49, v14
+; VI-NEXT:    v_mov_b32_e32 v50, v13
+; VI-NEXT:    v_mov_b32_e32 v51, v12
+; VI-NEXT:    v_mov_b32_e32 v52, v11
+; VI-NEXT:    v_mov_b32_e32 v53, v10
+; VI-NEXT:    v_mov_b32_e32 v54, v9
+; VI-NEXT:    v_mov_b32_e32 v55, v8
+; VI-NEXT:    v_mov_b32_e32 v40, v7
+; VI-NEXT:    v_mov_b32_e32 v41, v6
+; VI-NEXT:    v_mov_b32_e32 v42, v5
+; VI-NEXT:    v_mov_b32_e32 v43, v4
+; VI-NEXT:    v_mov_b32_e32 v44, v3
+; VI-NEXT:    v_mov_b32_e32 v45, v2
+; VI-NEXT:    v_mov_b32_e32 v46, v1
+; VI-NEXT:    v_mov_b32_e32 v47, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v23, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v23, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v23, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v23, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v23, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v23, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v23, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v23, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v23, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v23, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v23, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v23, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v23, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v23, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v23, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v23, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v23, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v23, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v23, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v23, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v23, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v23, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v23, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB27_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v23, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v47, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v46, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v45, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v44, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v43, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v42, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v41, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v40, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v55, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v54, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v52, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v51, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v50, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v49, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v48, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v39, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v38, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v37, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v36, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v35, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v34, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v33, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v22, v24, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v32, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:  .LBB27_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48f16_to_v12f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v33, v22
+; GFX9-NEXT:    v_mov_b32_e32 v47, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v34, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v33
+; GFX9-NEXT:    v_mov_b32_e32 v35, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v34
+; GFX9-NEXT:    v_mov_b32_e32 v36, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v9
+; GFX9-NEXT:    v_mov_b32_e32 v55, v8
+; GFX9-NEXT:    v_mov_b32_e32 v40, v7
+; GFX9-NEXT:    v_mov_b32_e32 v41, v6
+; GFX9-NEXT:    v_mov_b32_e32 v42, v5
+; GFX9-NEXT:    v_mov_b32_e32 v43, v4
+; GFX9-NEXT:    v_mov_b32_e32 v44, v3
+; GFX9-NEXT:    v_mov_b32_e32 v45, v2
+; GFX9-NEXT:    v_mov_b32_e32 v46, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v47
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v35
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v34
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; kill: killed $vgpr24
+; GFX9-NEXT:  .LBB27_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v32, v47, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v62, v46, s6
+; GFX9-NEXT:    v_perm_b32 v2, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v3, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v4, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v5, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v6, v57, v41, s6
+; GFX9-NEXT:    v_perm_b32 v7, v56, v40, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v55, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v34, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v33, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48f16_to_v12f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX11-NEXT:    v_perm_b32 v0, v53, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v55, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v52, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v51, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v50, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v49, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v48, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v39, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v38, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v37, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v36, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v35, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v34, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v33, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v32, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v31, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v30, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v29, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v28, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v27, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v26, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v25, v23, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <48 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <48 x half> %a1 to <12 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x half> %a to <12 x double>
+  br label %end
+
+end:
+  %phi = phi <12 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <12 x double> %phi
+}
+
+define <48 x half> @bitcast_v48i16_to_v48f16(<48 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48i16_to_v48f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v32
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; kill: killed $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v24
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v47
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v56
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v57
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v59
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v60
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v61
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v62
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v43
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v44
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v45
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:  .LBB28_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 3, v42
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 3, v41
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 3, v40
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_add_i32_e32 v62, vcc, 3, v62
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v61, vcc, 3, v61
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v60, vcc, 3, v60
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 3, v59
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 3, v58
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 3, v57
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 3, v56
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 3, v47
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 3, v55
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v24
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v47
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v56
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v57
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v59
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v60
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v61
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v62
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v43
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v44
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v45
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v34
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v35
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v36
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v3
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v37
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v38
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v5
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v39
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v48
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v7
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v49
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v50
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v9
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v51
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v52
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v11
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v53
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 56, v0
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 60, v0
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 64, v0
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 0x44, v0
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v33, v38, v33
+; GCN-NEXT:    v_or_b32_e32 v32, v39, v32
+; GCN-NEXT:    v_or_b32_e32 v38, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v39, v51, v50
+; GCN-NEXT:    v_or_b32_e32 v48, v53, v52
+; GCN-NEXT:    v_or_b32_e32 v49, v55, v54
+; GCN-NEXT:    v_or_b32_e32 v50, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v51, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v52, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v53, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v54, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v55, v59, v58
+; GCN-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v49, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v51, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48i16_to_v48f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_add_u16_e32 v55, 3, v55
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v54, 3, v54
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v53, 3, v53
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v52, 3, v52
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v24, 3, v24
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v51, 3, v51
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v50, 3, v50
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v49, 3, v49
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v48, 3, v48
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_e32 v39, 3, v39
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_add_u16_e32 v38, 3, v38
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_add_u16_e32 v37, 3, v37
+; VI-NEXT:    v_add_u16_e32 v12, 3, v12
+; VI-NEXT:    v_add_u16_e32 v36, 3, v36
+; VI-NEXT:    v_add_u16_e32 v13, 3, v13
+; VI-NEXT:    v_add_u16_e32 v35, 3, v35
+; VI-NEXT:    v_add_u16_e32 v14, 3, v14
+; VI-NEXT:    v_add_u16_e32 v34, 3, v34
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    v_add_u16_e32 v33, 3, v33
+; VI-NEXT:    v_add_u16_e32 v16, 3, v16
+; VI-NEXT:    v_add_u16_e32 v32, 3, v32
+; VI-NEXT:    v_add_u16_e32 v17, 3, v17
+; VI-NEXT:    v_add_u16_e32 v31, 3, v31
+; VI-NEXT:    v_add_u16_e32 v18, 3, v18
+; VI-NEXT:    v_add_u16_e32 v30, 3, v30
+; VI-NEXT:    v_add_u16_e32 v19, 3, v19
+; VI-NEXT:    v_add_u16_e32 v29, 3, v29
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_add_u16_e32 v28, 3, v28
+; VI-NEXT:    v_add_u16_e32 v21, 3, v21
+; VI-NEXT:    v_add_u16_e32 v27, 3, v27
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v26, 3, v26
+; VI-NEXT:    v_add_u16_e32 v23, 3, v23
+; VI-NEXT:    v_add_u16_e32 v25, 3, v25
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48i16_to_v48f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v23, v55, v23, s6
+; GFX9-NEXT:    v_perm_b32 v22, v54, v22, s6
+; GFX9-NEXT:    v_perm_b32 v21, v53, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v52, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v51, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v50, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v49, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v48, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v39, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v38, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v37, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v36, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v35, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v34, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v33, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v31, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v30, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v28, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v29, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v27, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v26, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v25, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v24, v0, s6
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v23
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v24, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v25, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v26, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v27, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v29, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v28, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v30, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v31, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v32, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v33, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v34, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v35, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v36, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v37, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v38, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v39, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v48, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v49, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v50, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v51, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v52, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v53, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v54, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v55, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48i16_to_v48f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v23, v64, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v55, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v54, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v53, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v52, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v51, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v49, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v39, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v37, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v35, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v33, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v31, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v25, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v26, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v27, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v28, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v29, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v23
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v25, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v26, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v27, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v28, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v29, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v31, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v33, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v35, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v37, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v39, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v49, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v51, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v52, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v53, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v54, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v55, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v64, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <48 x i16> %a, splat (i16 3)
+  %a2 = bitcast <48 x i16> %a1 to <48 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x i16> %a to <48 x half>
+  br label %end
+
+end:
+  %phi = phi <48 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x half> %phi
+}
+
+define <48 x i16> @bitcast_v48f16_to_v48i16(<48 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v48f16_to_v48i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:36
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:32
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:28
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:72
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_add_f32_e32 v40, 0x38000000, v40
+; GCN-NEXT:    v_add_f32_e32 v53, 0x38000000, v53
+; GCN-NEXT:    v_add_f32_e32 v55, 0x38000000, v55
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_add_f32_e32 v54, 0x38000000, v54
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v52, 0x38000000, v52
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v53, v53, v31
+; GCN-NEXT:    v_or_b32_e32 v51, v51, v32
+; GCN-NEXT:    v_or_b32_e32 v49, v49, v54
+; GCN-NEXT:    v_or_b32_e32 v39, v39, v52
+; GCN-NEXT:    v_or_b32_e32 v37, v37, v50
+; GCN-NEXT:    v_or_b32_e32 v36, v36, v48
+; GCN-NEXT:    v_or_b32_e32 v35, v35, v38
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v30
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v29
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v27
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v25
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v55
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v40
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v42
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v43
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v44
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v46
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v47
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v56
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v57
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v58
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v59
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v60
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v31, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v3, v32, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v4, v54, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v14, v52, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v16, v50, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v18, v48, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v19, v38, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v30, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v17, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v27, v15, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v13, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v12, v23, 16
+; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v53, 0xffff, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v51, 0xffff, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v49, 0xffff, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v39, 0xffff, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v37, 0xffff, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v36, 0xffff, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v35, 0xffff, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v53, v53, v40
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v1, v2, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v51, v51, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v7
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v49, v49, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v39, v39, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v9, v14, v9
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v37, v37, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v10, v16, v10
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v36, v36, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v11, v18, v11
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 0x5c, v0
+; GCN-NEXT:    v_or_b32_e32 v35, v35, v38
+; GCN-NEXT:    v_or_b32_e32 v6, v19, v6
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v30
+; GCN-NEXT:    v_or_b32_e32 v5, v20, v5
+; GCN-NEXT:    v_or_b32_e32 v20, v26, v29
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v34
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v27
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v33
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v25
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v45
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v23
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v41
+; GCN-NEXT:    buffer_store_dword v53, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v1, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v51, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v49, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v39, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v35, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v48f16_to_v48i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v24, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_add_f16_e32 v55, 0x200, v55
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_e32 v54, 0x200, v54
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_add_f16_e32 v53, 0x200, v53
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_add_f16_e32 v52, 0x200, v52
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v24
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_add_f16_e32 v51, 0x200, v51
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_add_f16_e32 v50, 0x200, v50
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_add_f16_e32 v49, 0x200, v49
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_add_f16_e32 v48, 0x200, v48
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_add_f16_e32 v39, 0x200, v39
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_add_f16_e32 v38, 0x200, v38
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_add_f16_e32 v37, 0x200, v37
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_add_f16_e32 v36, 0x200, v36
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_add_f16_e32 v35, 0x200, v35
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v14
+; VI-NEXT:    v_add_f16_e32 v34, 0x200, v34
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v15
+; VI-NEXT:    v_add_f16_e32 v33, 0x200, v33
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v16
+; VI-NEXT:    v_add_f16_e32 v32, 0x200, v32
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v17
+; VI-NEXT:    v_add_f16_e32 v31, 0x200, v31
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v18
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v30
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v19
+; VI-NEXT:    v_add_f16_e32 v29, 0x200, v29
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v20
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v28
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v21
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v27
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v22
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v26
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v23
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v25
+; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v48f16_to_v48i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v23, v55, v23, s6
+; GFX9-NEXT:    v_perm_b32 v22, v54, v22, s6
+; GFX9-NEXT:    v_perm_b32 v21, v53, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v52, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v51, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v50, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v49, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v48, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v39, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v38, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v37, v13, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v12, v36, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v35, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v34, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v33, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v31, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v30, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v28, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v29, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v27, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v26, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v25, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v24, v0, s6
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v23
+; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v24, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v25, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v26, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v27, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v29, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v28, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v30, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v31, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v32, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v33, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v34, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v35, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v36, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v37, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v38, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v39, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v48, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v49, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v50, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v51, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v52, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v53, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v54, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v55, v23, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v48f16_to_v48i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v24
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v23, v64, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v55, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v54, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v53, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v52, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v51, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v49, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v39, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v37, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v35, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v33, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v31, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v25, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v26, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v27, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v28, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v29, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v23
+; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v25, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v26, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v27, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v28, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v29, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v30, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v31, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v32, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v33, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v35, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v36, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v37, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v39, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v49, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v51, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v52, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v53, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v54, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v55, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v64, v23, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <48 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <48 x half> %a1 to <48 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <48 x half> %a to <48 x i16>
+  br label %end
+
+end:
+  %phi = phi <48 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <48 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
index 4dcfaee680984..e4f8a96c482c6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
@@ -5,8 +5,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
 
-define <26 x i32> @bitcast_v13i64_to_v26i32(<13 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v13i64_to_v26i32:
+define <26 x float> @bitcast_v26i32_to_v26f32(<26 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26i32_to_v26f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
@@ -15,37 +15,37 @@ define <26 x i32> @bitcast_v13i64_to_v26i32(<13 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:  .LBB0_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v13i64_to_v26i32:
+; VI-LABEL: bitcast_v26i32_to_v26f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
@@ -54,37 +54,37 @@ define <26 x i32> @bitcast_v13i64_to_v26i32(<13 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB0_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
 ; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
-; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
 ; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
-; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
-; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
 ; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
-; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
 ; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
-; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
 ; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
-; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
-; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:  .LBB0_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v13i64_to_v26i32:
+; GFX9-LABEL: bitcast_v26i32_to_v26f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
@@ -93,37 +93,37 @@ define <26 x i32> @bitcast_v13i64_to_v26i32(<13 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
-; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
-; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
-; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
-; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
-; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
-; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
 ; GFX9-NEXT:  .LBB0_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v13i64_to_v26i32:
+; GFX11-LABEL: bitcast_v26i32_to_v26f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -133,39 +133,32 @@ define <26 x i32> @bitcast_v13i64_to_v26i32(<13 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
 ; GFX11-NEXT:  .LBB0_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -173,12 +166,173 @@ define <26 x i32> @bitcast_v13i64_to_v26i32(<13 x i64> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <13 x i64> %a, splat (i64 3)
-  %a2 = bitcast <13 x i64> %a1 to <26 x i32>
+  %a1 = add <26 x i32> %a, splat (i32 3)
+  %a2 = bitcast <26 x i32> %a1 to <26 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <13 x i64> %a to <26 x i32>
+  %a3 = bitcast <26 x i32> %a to <26 x float>
+  br label %end
+
+end:
+  %phi = phi <26 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x float> %phi
+}
+
+define <26 x i32> @bitcast_v26f32_to_v26i32(<26 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26f32_to_v26i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v26f32_to_v26i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v26f32_to_v26i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v26f32_to_v26i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <26 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <26 x float> %a1 to <26 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <26 x float> %a to <26 x i32>
   br label %end
 
 end:
@@ -194,7 +348,7 @@ define <13 x i64> @bitcast_v26i32_to_v13i64(<26 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
@@ -222,7 +376,7 @@ define <13 x i64> @bitcast_v26i32_to_v13i64(<26 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:  .LBB2_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -233,7 +387,7 @@ define <13 x i64> @bitcast_v26i32_to_v13i64(<26 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:    s_cbranch_execz .LBB2_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
 ; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
@@ -261,7 +415,7 @@ define <13 x i64> @bitcast_v26i32_to_v13i64(<26 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:  .LBB2_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -272,7 +426,7 @@ define <13 x i64> @bitcast_v26i32_to_v13i64(<26 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
 ; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
@@ -300,7 +454,7 @@ define <13 x i64> @bitcast_v26i32_to_v13i64(<26 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:  .LBB2_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -312,7 +466,7 @@ define <13 x i64> @bitcast_v26i32_to_v13i64(<26 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
 ; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
@@ -340,7 +494,7 @@ define <13 x i64> @bitcast_v26i32_to_v13i64(<26 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:  .LBB2_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -359,3 +513,20648 @@ end:
   %phi = phi <13 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <13 x i64> %phi
 }
+
+define <26 x i32> @bitcast_v13i64_to_v26i32(<13 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13i64_to_v26i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13i64_to_v26i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13i64_to_v26i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13i64_to_v26i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <13 x i64> %a, splat (i64 3)
+  %a2 = bitcast <13 x i64> %a1 to <26 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x i64> %a to <26 x i32>
+  br label %end
+
+end:
+  %phi = phi <26 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x i32> %phi
+}
+
+define <13 x double> @bitcast_v26i32_to_v13f64(<26 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26i32_to_v13f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v26i32_to_v13f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB4_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v26i32_to_v13f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB4_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v26i32_to_v13f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <26 x i32> %a, splat (i32 3)
+  %a2 = bitcast <26 x i32> %a1 to <13 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <26 x i32> %a to <13 x double>
+  br label %end
+
+end:
+  %phi = phi <13 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x double> %phi
+}
+
+define <26 x i32> @bitcast_v13f64_to_v26i32(<13 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13f64_to_v26i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13f64_to_v26i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13f64_to_v26i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13f64_to_v26i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <13 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <13 x double> %a1 to <26 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x double> %a to <26 x i32>
+  br label %end
+
+end:
+  %phi = phi <26 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x i32> %phi
+}
+
+define <52 x i16> @bitcast_v26i32_to_v52i16(<26 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26i32_to_v52i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v27
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v27, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_alignbit_b32 v27, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_and_b32_e32 v45, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_and_b32_e32 v46, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_and_b32_e32 v47, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v57, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v40
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v44, v46, v44
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v54, v47, v54
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v43, v56, v43
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v52, v57, v52
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x64, v0
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v37
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v29
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v35
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v28
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v33
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v27
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v31
+; GCN-NEXT:    buffer_store_dword v40, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v26i32_to_v52i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB6_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB6_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v26i32_to_v52i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB6_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB6_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v43, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v42, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v41, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v40, v3, s4
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v4, v55, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v54, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v53, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v52, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v51, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v50, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v49, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v38, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v37, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v36, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v35, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v34, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v33, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v32, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v31, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v30, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v29, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v28, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v27, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v26i32_to_v52i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB6_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB6_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v67, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v65, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v55, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v54, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v53, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v52, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v51, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v50, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v34, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v33, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v32, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v31, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v30, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v29, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v28, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v27, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v26, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <26 x i32> %a, splat (i32 3)
+  %a2 = bitcast <26 x i32> %a1 to <52 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <26 x i32> %a to <52 x i16>
+  br label %end
+
+end:
+  %phi = phi <52 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x i16> %phi
+}
+
+define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52i16_to_v26i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:84
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v62
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v63
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v61
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v59
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v58
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v57
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v56
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v47
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v46
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v44
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v43
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v42
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v41
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v60
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v45
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v26
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:  .LBB7_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v41
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v60
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v45
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_or_b32_e32 v0, v62, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v63, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v61, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v59, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v58, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v57, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v56, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v47, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v46, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v44, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v43, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v42, v11
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v26, v12
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v13
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v14
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v26, v15
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v26, v16
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v26, v17
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v26, v18
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v26, v19
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v26, v20
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v26, v21
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v26, v22
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v26, v23
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v26, v24
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v26, v25
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 0x30000, v25
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52i16_to_v26i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v25
+; VI-NEXT:    v_mov_b32_e32 v33, v24
+; VI-NEXT:    v_mov_b32_e32 v34, v23
+; VI-NEXT:    v_mov_b32_e32 v35, v22
+; VI-NEXT:    v_mov_b32_e32 v36, v21
+; VI-NEXT:    v_mov_b32_e32 v37, v20
+; VI-NEXT:    v_mov_b32_e32 v38, v19
+; VI-NEXT:    v_mov_b32_e32 v39, v18
+; VI-NEXT:    v_mov_b32_e32 v48, v17
+; VI-NEXT:    v_mov_b32_e32 v49, v16
+; VI-NEXT:    v_mov_b32_e32 v50, v15
+; VI-NEXT:    v_mov_b32_e32 v51, v14
+; VI-NEXT:    v_mov_b32_e32 v52, v13
+; VI-NEXT:    v_mov_b32_e32 v53, v12
+; VI-NEXT:    v_mov_b32_e32 v54, v11
+; VI-NEXT:    v_mov_b32_e32 v55, v10
+; VI-NEXT:    v_mov_b32_e32 v40, v9
+; VI-NEXT:    v_mov_b32_e32 v41, v8
+; VI-NEXT:    v_mov_b32_e32 v42, v7
+; VI-NEXT:    v_mov_b32_e32 v43, v6
+; VI-NEXT:    v_mov_b32_e32 v44, v5
+; VI-NEXT:    v_mov_b32_e32 v45, v4
+; VI-NEXT:    v_mov_b32_e32 v46, v3
+; VI-NEXT:    v_mov_b32_e32 v47, v2
+; VI-NEXT:    v_mov_b32_e32 v56, v1
+; VI-NEXT:    v_mov_b32_e32 v57, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v25, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v25, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v25, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v25, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v25, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v25, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v25, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v25, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v25, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v25, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v25, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v25, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v25, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v25, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v25, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v25, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v25, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v25, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v25, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v25, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v25, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v25, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v25, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v47, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v45, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v43, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v41, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v53, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v51, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v49, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v39, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v35, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v33, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB7_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v25, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v1, v57, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v2, v56, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v3, v47, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v4, v46, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v5, v45, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v6, v44, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v7, v43, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v8, v42, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v9, v41, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v10, v40, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v11, v55, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v12, v54, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v13, v53, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v14, v52, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v15, v51, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v16, v50, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v17, v49, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v18, v48, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v19, v39, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v20, v38, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v21, v37, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v22, v36, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v23, v35, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v24, v34, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v26, v33, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v25, v32, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:  .LBB7_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52i16_to_v26i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v57, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v35, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v36, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v9
+; GFX9-NEXT:    v_mov_b32_e32 v41, v8
+; GFX9-NEXT:    v_mov_b32_e32 v42, v7
+; GFX9-NEXT:    v_mov_b32_e32 v43, v6
+; GFX9-NEXT:    v_mov_b32_e32 v44, v5
+; GFX9-NEXT:    v_mov_b32_e32 v45, v4
+; GFX9-NEXT:    v_mov_b32_e32 v46, v3
+; GFX9-NEXT:    v_mov_b32_e32 v47, v2
+; GFX9-NEXT:    v_mov_b32_e32 v56, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v57
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v35
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:  .LBB7_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52i16_to_v26i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    v_perm_b32 v0, v65, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v67, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v64, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v55, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v54, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v53, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v52, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v51, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v50, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v49, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v48, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v39, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v38, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v37, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v36, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v35, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v34, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v33, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v32, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v31, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v30, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v29, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v28, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v27, v25, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <52 x i16> %a, splat (i16 3)
+  %a2 = bitcast <52 x i16> %a1 to <26 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x i16> %a to <26 x i32>
+  br label %end
+
+end:
+  %phi = phi <26 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x i32> %phi
+}
+
+define <52 x half> @bitcast_v26i32_to_v52f16(<26 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26i32_to_v52f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v27
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v29
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v31
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v51
+; GCN-NEXT:    v_mov_b32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v53
+; GCN-NEXT:    v_mov_b32_e32 v53, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v55
+; GCN-NEXT:    v_mov_b32_e32 v55, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v41
+; GCN-NEXT:    v_mov_b32_e32 v41, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v52
+; GCN-NEXT:    v_mov_b32_e32 v52, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v54
+; GCN-NEXT:    v_mov_b32_e32 v54, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v43
+; GCN-NEXT:    v_mov_b32_e32 v43, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v45
+; GCN-NEXT:    v_mov_b32_e32 v45, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v47
+; GCN-NEXT:    v_mov_b32_e32 v47, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:  .LBB8_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v53
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:  .LBB8_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v30
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v28
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v27
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v62
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v60
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v58
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v32
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v31
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v29
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v63
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v61
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v59
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v56
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v55
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v53
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v51
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v41
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_or_b32_e32 v53, v55, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v55, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v41, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v42, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v43, v59, v58
+; GCN-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v41, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v26i32_to_v52f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB8_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB8_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v26i32_to_v52f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB8_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB8_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v43, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v42, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v41, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v40, v3, s4
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v4, v55, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v54, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v53, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v52, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v51, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v50, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v49, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v38, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v37, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v36, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v35, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v34, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v33, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v32, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v31, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v30, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v29, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v28, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v27, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v26i32_to_v52f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB8_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB8_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v67, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v65, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v55, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v54, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v53, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v52, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v51, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v50, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v34, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v33, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v32, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v31, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v30, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v29, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v28, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v27, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v26, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <26 x i32> %a, splat (i32 3)
+  %a2 = bitcast <26 x i32> %a1 to <52 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <26 x i32> %a to <52 x half>
+  br label %end
+
+end:
+  %phi = phi <52 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x half> %phi
+}
+
+define <26 x i32> @bitcast_v52f16_to_v26i32(<52 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52f16_to_v26i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:56
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:44
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:84
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v38
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v35
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v34
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v63
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v44
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v0, v42, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v40, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v2, v54, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v3, v52, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v50, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v49
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v61
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v45
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v26, v12
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v13
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v14
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v26, v15
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v26, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v48, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v38, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v36, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v34, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v32, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v33, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v35, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v37, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v39, v25
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:  .LBB9_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v40
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v54
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v52
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v50
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v49
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v48
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v38
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v15, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v16, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v17, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v18, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v20, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v21, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v22, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v23, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v24, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v25, v39, v38
+; GCN-NEXT:  .LBB9_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52f16_to_v26i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v25
+; VI-NEXT:    v_mov_b32_e32 v33, v24
+; VI-NEXT:    v_mov_b32_e32 v34, v23
+; VI-NEXT:    v_mov_b32_e32 v35, v22
+; VI-NEXT:    v_mov_b32_e32 v36, v21
+; VI-NEXT:    v_mov_b32_e32 v37, v20
+; VI-NEXT:    v_mov_b32_e32 v38, v19
+; VI-NEXT:    v_mov_b32_e32 v39, v18
+; VI-NEXT:    v_mov_b32_e32 v48, v17
+; VI-NEXT:    v_mov_b32_e32 v49, v16
+; VI-NEXT:    v_mov_b32_e32 v50, v15
+; VI-NEXT:    v_mov_b32_e32 v51, v14
+; VI-NEXT:    v_mov_b32_e32 v52, v13
+; VI-NEXT:    v_mov_b32_e32 v53, v12
+; VI-NEXT:    v_mov_b32_e32 v54, v11
+; VI-NEXT:    v_mov_b32_e32 v55, v10
+; VI-NEXT:    v_mov_b32_e32 v40, v9
+; VI-NEXT:    v_mov_b32_e32 v41, v8
+; VI-NEXT:    v_mov_b32_e32 v42, v7
+; VI-NEXT:    v_mov_b32_e32 v43, v6
+; VI-NEXT:    v_mov_b32_e32 v44, v5
+; VI-NEXT:    v_mov_b32_e32 v45, v4
+; VI-NEXT:    v_mov_b32_e32 v46, v3
+; VI-NEXT:    v_mov_b32_e32 v47, v2
+; VI-NEXT:    v_mov_b32_e32 v56, v1
+; VI-NEXT:    v_mov_b32_e32 v57, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v25, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v25, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v25, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v25, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v25, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v25, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v25, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v25, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v25, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v25, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v25, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v25, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v25, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v25, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v25, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v25, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v25, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v25, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v25, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v25, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v25, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v25, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v25, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v47, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v45, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v43, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v41, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v53, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v51, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v49, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v39, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v35, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v33, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB9_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v25, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v57, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v56, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v47, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v46, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v45, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v44, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v43, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v42, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v41, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v40, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v55, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v54, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v53, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v52, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v51, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v50, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v49, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v48, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v39, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v38, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v37, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v36, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v35, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v34, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v33, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v24, v26, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v32, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:  .LBB9_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52f16_to_v26i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v57, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v35, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v36, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v9
+; GFX9-NEXT:    v_mov_b32_e32 v41, v8
+; GFX9-NEXT:    v_mov_b32_e32 v42, v7
+; GFX9-NEXT:    v_mov_b32_e32 v43, v6
+; GFX9-NEXT:    v_mov_b32_e32 v44, v5
+; GFX9-NEXT:    v_mov_b32_e32 v45, v4
+; GFX9-NEXT:    v_mov_b32_e32 v46, v3
+; GFX9-NEXT:    v_mov_b32_e32 v47, v2
+; GFX9-NEXT:    v_mov_b32_e32 v56, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v57
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v35
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:  .LBB9_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52f16_to_v26i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    v_perm_b32 v0, v65, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v67, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v64, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v55, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v54, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v53, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v52, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v51, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v50, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v49, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v48, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v39, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v38, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v37, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v36, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v35, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v34, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v33, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v32, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v31, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v30, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v29, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v28, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v27, v25, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <52 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <52 x half> %a1 to <26 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x half> %a to <26 x i32>
+  br label %end
+
+end:
+  %phi = phi <26 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x i32> %phi
+}
+
+define <13 x i64> @bitcast_v26f32_to_v13i64(<26 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26f32_to_v13i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v26f32_to_v13i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v26f32_to_v13i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v26f32_to_v13i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <26 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <26 x float> %a1 to <13 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <26 x float> %a to <13 x i64>
+  br label %end
+
+end:
+  %phi = phi <13 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x i64> %phi
+}
+
+define <26 x float> @bitcast_v13i64_to_v26f32(<13 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13i64_to_v26f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13i64_to_v26f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13i64_to_v26f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13i64_to_v26f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <13 x i64> %a, splat (i64 3)
+  %a2 = bitcast <13 x i64> %a1 to <26 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x i64> %a to <26 x float>
+  br label %end
+
+end:
+  %phi = phi <26 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x float> %phi
+}
+
+define <13 x double> @bitcast_v26f32_to_v13f64(<26 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26f32_to_v13f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v26f32_to_v13f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v26f32_to_v13f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v26f32_to_v13f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB12_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <26 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <26 x float> %a1 to <13 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <26 x float> %a to <13 x double>
+  br label %end
+
+end:
+  %phi = phi <13 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x double> %phi
+}
+
+define <26 x float> @bitcast_v13f64_to_v26f32(<13 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13f64_to_v26f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13f64_to_v26f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13f64_to_v26f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13f64_to_v26f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <13 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <13 x double> %a1 to <26 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x double> %a to <26 x float>
+  br label %end
+
+end:
+  %phi = phi <26 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x float> %phi
+}
+
+define <52 x i16> @bitcast_v26f32_to_v52i16(<26 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26f32_to_v52i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v27
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v27, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_alignbit_b32 v27, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_and_b32_e32 v45, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_and_b32_e32 v46, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_and_b32_e32 v47, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v57, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v40
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v44, v46, v44
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v54, v47, v54
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v43, v56, v43
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v52, v57, v52
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x64, v0
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v37
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v29
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v35
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v28
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v33
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v27
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v31
+; GCN-NEXT:    buffer_store_dword v40, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v26f32_to_v52i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB14_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB14_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v26f32_to_v52i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB14_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB14_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v43, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v42, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v41, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v40, v3, s4
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v4, v55, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v54, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v53, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v52, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v51, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v50, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v49, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v38, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v37, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v36, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v35, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v34, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v33, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v32, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v31, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v30, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v29, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v28, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v27, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v26f32_to_v52i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB14_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB14_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v67, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v65, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v55, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v54, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v53, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v52, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v51, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v50, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v34, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v33, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v32, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v31, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v30, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v29, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v28, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v27, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v26, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <26 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <26 x float> %a1 to <52 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <26 x float> %a to <52 x i16>
+  br label %end
+
+end:
+  %phi = phi <52 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x i16> %phi
+}
+
+define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52i16_to_v26f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:84
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v62
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v63
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v61
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v59
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v58
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v57
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v56
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v47
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v46
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v44
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v43
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v42
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v41
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v60
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v45
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v26
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:  .LBB15_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v41
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v60
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v45
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_or_b32_e32 v0, v62, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v63, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v61, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v59, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v58, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v57, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v56, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v47, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v46, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v44, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v43, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v42, v11
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v26, v12
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v13
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v14
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v26, v15
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v26, v16
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v26, v17
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v26, v18
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v26, v19
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v26, v20
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v26, v21
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v26, v22
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v26, v23
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v26, v24
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v26, v25
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 0x30000, v25
+; GCN-NEXT:  .LBB15_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52i16_to_v26f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v25
+; VI-NEXT:    v_mov_b32_e32 v33, v24
+; VI-NEXT:    v_mov_b32_e32 v34, v23
+; VI-NEXT:    v_mov_b32_e32 v35, v22
+; VI-NEXT:    v_mov_b32_e32 v36, v21
+; VI-NEXT:    v_mov_b32_e32 v37, v20
+; VI-NEXT:    v_mov_b32_e32 v38, v19
+; VI-NEXT:    v_mov_b32_e32 v39, v18
+; VI-NEXT:    v_mov_b32_e32 v48, v17
+; VI-NEXT:    v_mov_b32_e32 v49, v16
+; VI-NEXT:    v_mov_b32_e32 v50, v15
+; VI-NEXT:    v_mov_b32_e32 v51, v14
+; VI-NEXT:    v_mov_b32_e32 v52, v13
+; VI-NEXT:    v_mov_b32_e32 v53, v12
+; VI-NEXT:    v_mov_b32_e32 v54, v11
+; VI-NEXT:    v_mov_b32_e32 v55, v10
+; VI-NEXT:    v_mov_b32_e32 v40, v9
+; VI-NEXT:    v_mov_b32_e32 v41, v8
+; VI-NEXT:    v_mov_b32_e32 v42, v7
+; VI-NEXT:    v_mov_b32_e32 v43, v6
+; VI-NEXT:    v_mov_b32_e32 v44, v5
+; VI-NEXT:    v_mov_b32_e32 v45, v4
+; VI-NEXT:    v_mov_b32_e32 v46, v3
+; VI-NEXT:    v_mov_b32_e32 v47, v2
+; VI-NEXT:    v_mov_b32_e32 v56, v1
+; VI-NEXT:    v_mov_b32_e32 v57, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v25, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v25, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v25, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v25, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v25, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v25, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v25, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v25, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v25, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v25, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v25, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v25, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v25, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v25, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v25, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v25, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v25, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v25, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v25, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v25, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v25, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v25, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v25, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v47, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v45, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v43, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v41, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v53, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v51, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v49, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v39, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v35, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v33, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB15_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v25, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v1, v57, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v2, v56, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v3, v47, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v4, v46, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v5, v45, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v6, v44, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v7, v43, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v8, v42, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v9, v41, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v10, v40, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v11, v55, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v12, v54, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v13, v53, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v14, v52, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v15, v51, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v16, v50, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v17, v49, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v18, v48, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v19, v39, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v20, v38, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v21, v37, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v22, v36, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v23, v35, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v24, v34, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v26, v33, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v25, v32, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:  .LBB15_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52i16_to_v26f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v57, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v35, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v36, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v9
+; GFX9-NEXT:    v_mov_b32_e32 v41, v8
+; GFX9-NEXT:    v_mov_b32_e32 v42, v7
+; GFX9-NEXT:    v_mov_b32_e32 v43, v6
+; GFX9-NEXT:    v_mov_b32_e32 v44, v5
+; GFX9-NEXT:    v_mov_b32_e32 v45, v4
+; GFX9-NEXT:    v_mov_b32_e32 v46, v3
+; GFX9-NEXT:    v_mov_b32_e32 v47, v2
+; GFX9-NEXT:    v_mov_b32_e32 v56, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v57
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v35
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:  .LBB15_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52i16_to_v26f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    v_perm_b32 v0, v65, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v67, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v64, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v55, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v54, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v53, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v52, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v51, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v50, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v49, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v48, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v39, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v38, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v37, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v36, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v35, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v34, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v33, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v32, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v31, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v30, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v29, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v28, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v27, v25, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <52 x i16> %a, splat (i16 3)
+  %a2 = bitcast <52 x i16> %a1 to <26 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x i16> %a to <26 x float>
+  br label %end
+
+end:
+  %phi = phi <26 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x float> %phi
+}
+
+define <52 x half> @bitcast_v26f32_to_v52f16(<26 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v26f32_to_v52f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v27
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v29
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v31
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v51
+; GCN-NEXT:    v_mov_b32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v53
+; GCN-NEXT:    v_mov_b32_e32 v53, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v55
+; GCN-NEXT:    v_mov_b32_e32 v55, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v41
+; GCN-NEXT:    v_mov_b32_e32 v41, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v52
+; GCN-NEXT:    v_mov_b32_e32 v52, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v54
+; GCN-NEXT:    v_mov_b32_e32 v54, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v43
+; GCN-NEXT:    v_mov_b32_e32 v43, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v45
+; GCN-NEXT:    v_mov_b32_e32 v45, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v47
+; GCN-NEXT:    v_mov_b32_e32 v47, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:  .LBB16_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v53
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:  .LBB16_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v30
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v28
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v27
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v62
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v60
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v58
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v32
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v31
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v29
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v63
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v61
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v59
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v56
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v55
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v53
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v51
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v41
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_or_b32_e32 v53, v55, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v55, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v41, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v42, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v43, v59, v58
+; GCN-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v41, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v26f32_to_v52f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB16_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB16_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v26f32_to_v52f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB16_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB16_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v43, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v42, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v41, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v40, v3, s4
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v4, v55, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v54, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v53, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v52, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v51, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v50, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v49, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v38, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v37, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v36, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v35, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v34, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v33, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v32, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v31, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v30, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v29, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v28, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v27, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v26f32_to_v52f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB16_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB16_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v67, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v65, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v55, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v54, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v53, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v52, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v51, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v50, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v34, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v33, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v32, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v31, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v30, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v29, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v28, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v27, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v26, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <26 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <26 x float> %a1 to <52 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <26 x float> %a to <52 x half>
+  br label %end
+
+end:
+  %phi = phi <52 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x half> %phi
+}
+
+define <26 x float> @bitcast_v52f16_to_v26f32(<52 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52f16_to_v26f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:56
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:44
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:84
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v38
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v35
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v34
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v63
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v44
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v0, v42, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v40, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v2, v54, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v3, v52, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v50, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v49
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v61
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v45
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v26, v12
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v13
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v14
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v26, v15
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v26, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v48, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v38, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v36, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v34, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v32, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v33, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v35, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v37, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v39, v25
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v40
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v54
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v52
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v50
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v49
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v48
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v38
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v15, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v16, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v17, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v18, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v20, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v21, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v22, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v23, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v24, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v25, v39, v38
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52f16_to_v26f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v25
+; VI-NEXT:    v_mov_b32_e32 v33, v24
+; VI-NEXT:    v_mov_b32_e32 v34, v23
+; VI-NEXT:    v_mov_b32_e32 v35, v22
+; VI-NEXT:    v_mov_b32_e32 v36, v21
+; VI-NEXT:    v_mov_b32_e32 v37, v20
+; VI-NEXT:    v_mov_b32_e32 v38, v19
+; VI-NEXT:    v_mov_b32_e32 v39, v18
+; VI-NEXT:    v_mov_b32_e32 v48, v17
+; VI-NEXT:    v_mov_b32_e32 v49, v16
+; VI-NEXT:    v_mov_b32_e32 v50, v15
+; VI-NEXT:    v_mov_b32_e32 v51, v14
+; VI-NEXT:    v_mov_b32_e32 v52, v13
+; VI-NEXT:    v_mov_b32_e32 v53, v12
+; VI-NEXT:    v_mov_b32_e32 v54, v11
+; VI-NEXT:    v_mov_b32_e32 v55, v10
+; VI-NEXT:    v_mov_b32_e32 v40, v9
+; VI-NEXT:    v_mov_b32_e32 v41, v8
+; VI-NEXT:    v_mov_b32_e32 v42, v7
+; VI-NEXT:    v_mov_b32_e32 v43, v6
+; VI-NEXT:    v_mov_b32_e32 v44, v5
+; VI-NEXT:    v_mov_b32_e32 v45, v4
+; VI-NEXT:    v_mov_b32_e32 v46, v3
+; VI-NEXT:    v_mov_b32_e32 v47, v2
+; VI-NEXT:    v_mov_b32_e32 v56, v1
+; VI-NEXT:    v_mov_b32_e32 v57, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v25, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v25, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v25, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v25, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v25, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v25, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v25, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v25, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v25, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v25, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v25, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v25, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v25, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v25, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v25, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v25, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v25, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v25, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v25, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v25, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v25, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v25, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v25, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v47, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v45, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v43, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v41, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v53, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v51, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v49, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v39, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v35, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v33, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB17_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v25, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v57, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v56, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v47, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v46, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v45, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v44, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v43, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v42, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v41, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v40, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v55, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v54, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v53, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v52, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v51, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v50, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v49, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v48, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v39, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v38, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v37, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v36, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v35, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v34, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v33, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v24, v26, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v32, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:  .LBB17_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52f16_to_v26f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v57, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v35, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v36, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v9
+; GFX9-NEXT:    v_mov_b32_e32 v41, v8
+; GFX9-NEXT:    v_mov_b32_e32 v42, v7
+; GFX9-NEXT:    v_mov_b32_e32 v43, v6
+; GFX9-NEXT:    v_mov_b32_e32 v44, v5
+; GFX9-NEXT:    v_mov_b32_e32 v45, v4
+; GFX9-NEXT:    v_mov_b32_e32 v46, v3
+; GFX9-NEXT:    v_mov_b32_e32 v47, v2
+; GFX9-NEXT:    v_mov_b32_e32 v56, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v57
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v35
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:  .LBB17_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52f16_to_v26f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    v_perm_b32 v0, v65, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v67, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v64, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v55, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v54, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v53, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v52, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v51, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v50, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v49, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v48, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v39, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v38, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v37, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v36, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v35, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v34, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v33, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v32, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v31, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v30, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v29, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v28, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v27, v25, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <52 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <52 x half> %a1 to <26 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x half> %a to <26 x float>
+  br label %end
+
+end:
+  %phi = phi <26 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <26 x float> %phi
+}
+
+define <13 x double> @bitcast_v13i64_to_v13f64(<13 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13i64_to_v13f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13i64_to_v13f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13i64_to_v13f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13i64_to_v13f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <13 x i64> %a, splat (i64 3)
+  %a2 = bitcast <13 x i64> %a1 to <13 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x i64> %a to <13 x double>
+  br label %end
+
+end:
+  %phi = phi <13 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x double> %phi
+}
+
+define <13 x i64> @bitcast_v13f64_to_v13i64(<13 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13f64_to_v13i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13f64_to_v13i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13f64_to_v13i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13f64_to_v13i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <13 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <13 x double> %a1 to <13 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x double> %a to <13 x i64>
+  br label %end
+
+end:
+  %phi = phi <13 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x i64> %phi
+}
+
+define <52 x i16> @bitcast_v13i64_to_v52i16(<13 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13i64_to_v52i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v27
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v27, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_addc_u32_e32 v24, vcc, 0, v24, vcc
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_addc_u32_e32 v26, vcc, 0, v26, vcc
+; GCN-NEXT:    v_alignbit_b32 v27, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_and_b32_e32 v45, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_and_b32_e32 v46, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_and_b32_e32 v47, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v57, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v40
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v44, v46, v44
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v54, v47, v54
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v43, v56, v43
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v52, v57, v52
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x64, v0
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v37
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v29
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v36
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v28
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v34
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v27
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v32
+; GCN-NEXT:    buffer_store_dword v40, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13i64_to_v52i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB20_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB20_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13i64_to_v52i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB20_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB20_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v43, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v42, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v41, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v40, v3, s4
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v4, v55, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v54, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v53, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v52, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v51, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v50, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v49, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v38, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v37, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v36, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v35, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v34, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v33, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v32, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v31, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v30, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v29, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v28, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v27, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13i64_to_v52i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB20_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB20_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v67, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v65, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v55, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v54, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v53, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v52, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v51, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v50, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v34, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v33, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v32, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v31, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v30, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v29, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v28, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v27, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v26, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <13 x i64> %a, splat (i64 3)
+  %a2 = bitcast <13 x i64> %a1 to <52 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x i64> %a to <52 x i16>
+  br label %end
+
+end:
+  %phi = phi <52 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x i16> %phi
+}
+
+define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52i16_to_v13i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:84
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v62
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v63
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v61
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v59
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v58
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v57
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v56
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v47
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v46
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v44
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v43
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v42
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v41
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v60
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v45
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v26
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:  .LBB21_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v41
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v60
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v45
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_or_b32_e32 v0, v62, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v63, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v61, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v59, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v58, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v57, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v56, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v47, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v46, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v44, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v43, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v42, v11
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v26, v12
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v13
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v14
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v26, v15
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v26, v16
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v26, v17
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v26, v18
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v26, v19
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v26, v20
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v26, v21
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v26, v22
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v26, v23
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v26, v24
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v26, v25
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 0x30000, v25
+; GCN-NEXT:  .LBB21_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52i16_to_v13i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v25
+; VI-NEXT:    v_mov_b32_e32 v33, v24
+; VI-NEXT:    v_mov_b32_e32 v34, v23
+; VI-NEXT:    v_mov_b32_e32 v35, v22
+; VI-NEXT:    v_mov_b32_e32 v36, v21
+; VI-NEXT:    v_mov_b32_e32 v37, v20
+; VI-NEXT:    v_mov_b32_e32 v38, v19
+; VI-NEXT:    v_mov_b32_e32 v39, v18
+; VI-NEXT:    v_mov_b32_e32 v48, v17
+; VI-NEXT:    v_mov_b32_e32 v49, v16
+; VI-NEXT:    v_mov_b32_e32 v50, v15
+; VI-NEXT:    v_mov_b32_e32 v51, v14
+; VI-NEXT:    v_mov_b32_e32 v52, v13
+; VI-NEXT:    v_mov_b32_e32 v53, v12
+; VI-NEXT:    v_mov_b32_e32 v54, v11
+; VI-NEXT:    v_mov_b32_e32 v55, v10
+; VI-NEXT:    v_mov_b32_e32 v40, v9
+; VI-NEXT:    v_mov_b32_e32 v41, v8
+; VI-NEXT:    v_mov_b32_e32 v42, v7
+; VI-NEXT:    v_mov_b32_e32 v43, v6
+; VI-NEXT:    v_mov_b32_e32 v44, v5
+; VI-NEXT:    v_mov_b32_e32 v45, v4
+; VI-NEXT:    v_mov_b32_e32 v46, v3
+; VI-NEXT:    v_mov_b32_e32 v47, v2
+; VI-NEXT:    v_mov_b32_e32 v56, v1
+; VI-NEXT:    v_mov_b32_e32 v57, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v25, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v25, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v25, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v25, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v25, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v25, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v25, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v25, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v25, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v25, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v25, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v25, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v25, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v25, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v25, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v25, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v25, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v25, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v25, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v25, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v25, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v25, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v25, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v47, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v45, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v43, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v41, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v53, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v51, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v49, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v39, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v35, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v33, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB21_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v25, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v1, v57, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v2, v56, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v3, v47, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v4, v46, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v5, v45, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v6, v44, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v7, v43, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v8, v42, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v9, v41, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v10, v40, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v11, v55, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v12, v54, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v13, v53, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v14, v52, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v15, v51, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v16, v50, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v17, v49, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v18, v48, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v19, v39, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v20, v38, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v21, v37, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v22, v36, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v23, v35, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v24, v34, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v26, v33, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v25, v32, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:  .LBB21_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52i16_to_v13i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v57, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v35, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v36, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v9
+; GFX9-NEXT:    v_mov_b32_e32 v41, v8
+; GFX9-NEXT:    v_mov_b32_e32 v42, v7
+; GFX9-NEXT:    v_mov_b32_e32 v43, v6
+; GFX9-NEXT:    v_mov_b32_e32 v44, v5
+; GFX9-NEXT:    v_mov_b32_e32 v45, v4
+; GFX9-NEXT:    v_mov_b32_e32 v46, v3
+; GFX9-NEXT:    v_mov_b32_e32 v47, v2
+; GFX9-NEXT:    v_mov_b32_e32 v56, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v57
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v35
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:  .LBB21_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52i16_to_v13i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    v_perm_b32 v0, v65, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v67, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v64, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v55, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v54, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v53, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v52, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v51, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v50, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v49, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v48, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v39, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v38, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v37, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v36, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v35, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v34, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v33, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v32, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v31, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v30, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v29, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v28, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v27, v25, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <52 x i16> %a, splat (i16 3)
+  %a2 = bitcast <52 x i16> %a1 to <13 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x i16> %a to <13 x i64>
+  br label %end
+
+end:
+  %phi = phi <13 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x i64> %phi
+}
+
+define <52 x half> @bitcast_v13i64_to_v52f16(<13 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13i64_to_v52f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v27
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v29
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v31
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v51
+; GCN-NEXT:    v_mov_b32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v53
+; GCN-NEXT:    v_mov_b32_e32 v53, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v55
+; GCN-NEXT:    v_mov_b32_e32 v55, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v41
+; GCN-NEXT:    v_mov_b32_e32 v41, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v52
+; GCN-NEXT:    v_mov_b32_e32 v52, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v54
+; GCN-NEXT:    v_mov_b32_e32 v54, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v43
+; GCN-NEXT:    v_mov_b32_e32 v43, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v45
+; GCN-NEXT:    v_mov_b32_e32 v45, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v47
+; GCN-NEXT:    v_mov_b32_e32 v47, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:  .LBB22_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_addc_u32_e32 v24, vcc, 0, v24, vcc
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_addc_u32_e32 v26, vcc, 0, v26, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v53
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:  .LBB22_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v30
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v28
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v27
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v62
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v60
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v58
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v32
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v31
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v29
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v63
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v61
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v59
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v56
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v55
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v53
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v51
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v41
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_or_b32_e32 v53, v55, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v55, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v41, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v42, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v43, v59, v58
+; GCN-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v41, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13i64_to_v52f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB22_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB22_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13i64_to_v52f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB22_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB22_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v43, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v42, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v41, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v40, v3, s4
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v4, v55, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v54, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v53, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v52, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v51, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v50, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v49, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v38, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v37, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v36, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v35, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v34, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v33, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v32, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v31, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v30, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v29, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v28, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v27, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13i64_to_v52f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB22_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB22_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v67, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v65, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v55, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v54, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v53, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v52, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v51, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v50, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v34, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v33, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v32, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v31, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v30, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v29, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v28, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v27, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v26, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <13 x i64> %a, splat (i64 3)
+  %a2 = bitcast <13 x i64> %a1 to <52 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x i64> %a to <52 x half>
+  br label %end
+
+end:
+  %phi = phi <52 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x half> %phi
+}
+
+define <13 x i64> @bitcast_v52f16_to_v13i64(<52 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52f16_to_v13i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:56
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:44
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:84
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v38
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v35
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v34
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v63
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v44
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v0, v42, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v40, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v2, v54, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v3, v52, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v50, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v49
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v61
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v45
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v26, v12
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v13
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v14
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v26, v15
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v26, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v48, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v38, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v36, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v34, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v32, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v33, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v35, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v37, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v39, v25
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:  .LBB23_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v40
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v54
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v52
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v50
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v49
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v48
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v38
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v15, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v16, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v17, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v18, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v20, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v21, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v22, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v23, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v24, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v25, v39, v38
+; GCN-NEXT:  .LBB23_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52f16_to_v13i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v25
+; VI-NEXT:    v_mov_b32_e32 v33, v24
+; VI-NEXT:    v_mov_b32_e32 v34, v23
+; VI-NEXT:    v_mov_b32_e32 v35, v22
+; VI-NEXT:    v_mov_b32_e32 v36, v21
+; VI-NEXT:    v_mov_b32_e32 v37, v20
+; VI-NEXT:    v_mov_b32_e32 v38, v19
+; VI-NEXT:    v_mov_b32_e32 v39, v18
+; VI-NEXT:    v_mov_b32_e32 v48, v17
+; VI-NEXT:    v_mov_b32_e32 v49, v16
+; VI-NEXT:    v_mov_b32_e32 v50, v15
+; VI-NEXT:    v_mov_b32_e32 v51, v14
+; VI-NEXT:    v_mov_b32_e32 v52, v13
+; VI-NEXT:    v_mov_b32_e32 v53, v12
+; VI-NEXT:    v_mov_b32_e32 v54, v11
+; VI-NEXT:    v_mov_b32_e32 v55, v10
+; VI-NEXT:    v_mov_b32_e32 v40, v9
+; VI-NEXT:    v_mov_b32_e32 v41, v8
+; VI-NEXT:    v_mov_b32_e32 v42, v7
+; VI-NEXT:    v_mov_b32_e32 v43, v6
+; VI-NEXT:    v_mov_b32_e32 v44, v5
+; VI-NEXT:    v_mov_b32_e32 v45, v4
+; VI-NEXT:    v_mov_b32_e32 v46, v3
+; VI-NEXT:    v_mov_b32_e32 v47, v2
+; VI-NEXT:    v_mov_b32_e32 v56, v1
+; VI-NEXT:    v_mov_b32_e32 v57, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v25, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v25, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v25, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v25, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v25, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v25, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v25, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v25, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v25, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v25, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v25, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v25, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v25, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v25, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v25, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v25, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v25, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v25, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v25, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v25, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v25, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v25, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v25, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v47, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v45, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v43, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v41, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v53, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v51, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v49, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v39, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v35, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v33, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB23_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v25, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v57, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v56, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v47, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v46, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v45, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v44, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v43, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v42, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v41, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v40, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v55, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v54, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v53, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v52, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v51, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v50, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v49, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v48, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v39, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v38, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v37, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v36, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v35, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v34, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v33, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v24, v26, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v32, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:  .LBB23_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52f16_to_v13i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v57, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v35, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v36, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v9
+; GFX9-NEXT:    v_mov_b32_e32 v41, v8
+; GFX9-NEXT:    v_mov_b32_e32 v42, v7
+; GFX9-NEXT:    v_mov_b32_e32 v43, v6
+; GFX9-NEXT:    v_mov_b32_e32 v44, v5
+; GFX9-NEXT:    v_mov_b32_e32 v45, v4
+; GFX9-NEXT:    v_mov_b32_e32 v46, v3
+; GFX9-NEXT:    v_mov_b32_e32 v47, v2
+; GFX9-NEXT:    v_mov_b32_e32 v56, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v57
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v35
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:  .LBB23_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52f16_to_v13i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    v_perm_b32 v0, v65, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v67, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v64, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v55, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v54, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v53, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v52, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v51, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v50, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v49, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v48, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v39, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v38, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v37, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v36, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v35, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v34, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v33, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v32, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v31, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v30, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v29, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v28, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v27, v25, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <52 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <52 x half> %a1 to <13 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x half> %a to <13 x i64>
+  br label %end
+
+end:
+  %phi = phi <13 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x i64> %phi
+}
+
+define <52 x i16> @bitcast_v13f64_to_v52i16(<13 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13f64_to_v52i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v27
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v27, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v2
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_add_f64 v[25:26], v[25:26], 1.0
+; GCN-NEXT:    v_alignbit_b32 v27, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v2
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_and_b32_e32 v45, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_and_b32_e32 v46, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_and_b32_e32 v47, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v57, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_and_b32_e32 v58, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 28, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v40
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 32, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v44, v46, v44
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 36, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v55, v47, v55
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 40, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v43, v56, v43
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v54, v57, v54
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v42, v58, v42
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 52, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 56, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x4c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x50, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 0x54, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x58, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x64, v0
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v30
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v39
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v29
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v38
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v28
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v37
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v27
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v36
+; GCN-NEXT:    buffer_store_dword v40, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13f64_to_v52i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB24_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB24_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13f64_to_v52i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB24_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB24_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v43, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v42, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v41, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v40, v3, s4
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v4, v55, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v54, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v53, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v52, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v51, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v50, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v49, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v38, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v37, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v36, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v35, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v34, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v33, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v32, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v31, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v30, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v29, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v28, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v27, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13f64_to_v52i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB24_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB24_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v67, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v65, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v55, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v54, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v53, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v52, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v51, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v50, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v34, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v33, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v32, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v31, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v30, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v29, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v28, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v27, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v26, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <13 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <13 x double> %a1 to <52 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x double> %a to <52 x i16>
+  br label %end
+
+end:
+  %phi = phi <52 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x i16> %phi
+}
+
+define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52i16_to_v13f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:84
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v62
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v63
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v61
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v59
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v58
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v57
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v56
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v47
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v46
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v44
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v43
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v42
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v41
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v32
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v60
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v45
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v26
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v26
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:  .LBB25_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v41
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v60
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v45
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_or_b32_e32 v0, v62, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v63, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v61, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v59, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v58, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v57, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v56, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v47, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v46, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v44, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v43, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v42, v11
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v26, v12
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v13
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v14
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v26, v15
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v26, v16
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v26, v17
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v26, v18
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v26, v19
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v26, v20
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v26, v21
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v26, v22
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v26, v23
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v26, v24
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v26, v25
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 0x30000, v25
+; GCN-NEXT:  .LBB25_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52i16_to_v13f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v25
+; VI-NEXT:    v_mov_b32_e32 v33, v24
+; VI-NEXT:    v_mov_b32_e32 v34, v23
+; VI-NEXT:    v_mov_b32_e32 v35, v22
+; VI-NEXT:    v_mov_b32_e32 v36, v21
+; VI-NEXT:    v_mov_b32_e32 v37, v20
+; VI-NEXT:    v_mov_b32_e32 v38, v19
+; VI-NEXT:    v_mov_b32_e32 v39, v18
+; VI-NEXT:    v_mov_b32_e32 v48, v17
+; VI-NEXT:    v_mov_b32_e32 v49, v16
+; VI-NEXT:    v_mov_b32_e32 v50, v15
+; VI-NEXT:    v_mov_b32_e32 v51, v14
+; VI-NEXT:    v_mov_b32_e32 v52, v13
+; VI-NEXT:    v_mov_b32_e32 v53, v12
+; VI-NEXT:    v_mov_b32_e32 v54, v11
+; VI-NEXT:    v_mov_b32_e32 v55, v10
+; VI-NEXT:    v_mov_b32_e32 v40, v9
+; VI-NEXT:    v_mov_b32_e32 v41, v8
+; VI-NEXT:    v_mov_b32_e32 v42, v7
+; VI-NEXT:    v_mov_b32_e32 v43, v6
+; VI-NEXT:    v_mov_b32_e32 v44, v5
+; VI-NEXT:    v_mov_b32_e32 v45, v4
+; VI-NEXT:    v_mov_b32_e32 v46, v3
+; VI-NEXT:    v_mov_b32_e32 v47, v2
+; VI-NEXT:    v_mov_b32_e32 v56, v1
+; VI-NEXT:    v_mov_b32_e32 v57, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v25, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v25, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v25, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v25, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v25, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v25, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v25, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v25, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v25, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v25, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v25, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v25, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v25, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v25, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v25, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v25, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v25, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v25, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v25, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v25, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v25, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v25, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v25, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v47, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v45, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v43, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v41, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v53, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v51, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v49, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v39, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v35, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v33, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB25_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v25, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v1, v57, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v1, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v2, v56, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v1, v1, v2
+; VI-NEXT:    v_add_u16_e32 v2, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v3, v47, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v4, v46, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v5, v45, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v6, v44, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v7, v43, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v8, v42, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v9, v41, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v10, v40, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v11, v55, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v12, v54, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v13, v53, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v14, v52, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v15, v51, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v16, v50, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v17, v49, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v18, v48, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v19, v39, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v20, v38, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v21, v37, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v22, v36, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v23, v35, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v24, v34, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v26, v33, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v25, v32, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:  .LBB25_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52i16_to_v13f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v57, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v35, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v36, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v9
+; GFX9-NEXT:    v_mov_b32_e32 v41, v8
+; GFX9-NEXT:    v_mov_b32_e32 v42, v7
+; GFX9-NEXT:    v_mov_b32_e32 v43, v6
+; GFX9-NEXT:    v_mov_b32_e32 v44, v5
+; GFX9-NEXT:    v_mov_b32_e32 v45, v4
+; GFX9-NEXT:    v_mov_b32_e32 v46, v3
+; GFX9-NEXT:    v_mov_b32_e32 v47, v2
+; GFX9-NEXT:    v_mov_b32_e32 v56, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v57
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v35
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:  .LBB25_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52i16_to_v13f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    v_perm_b32 v0, v65, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v67, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v64, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v55, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v54, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v53, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v52, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v51, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v50, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v49, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v48, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v39, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v38, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v37, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v36, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v35, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v34, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v33, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v32, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v31, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v30, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v29, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v28, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v27, v25, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <52 x i16> %a, splat (i16 3)
+  %a2 = bitcast <52 x i16> %a1 to <13 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x i16> %a to <13 x double>
+  br label %end
+
+end:
+  %phi = phi <13 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x double> %phi
+}
+
+define <52 x half> @bitcast_v13f64_to_v52f16(<13 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v13f64_to_v52f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v27
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; kill: killed $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; kill: killed $vgpr50
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v29
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v31
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v51
+; GCN-NEXT:    v_mov_b32_e32 v51, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v53
+; GCN-NEXT:    v_mov_b32_e32 v53, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v55
+; GCN-NEXT:    v_mov_b32_e32 v55, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v41
+; GCN-NEXT:    v_mov_b32_e32 v41, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v52
+; GCN-NEXT:    v_mov_b32_e32 v52, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v54
+; GCN-NEXT:    v_mov_b32_e32 v54, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v43
+; GCN-NEXT:    v_mov_b32_e32 v43, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v45
+; GCN-NEXT:    v_mov_b32_e32 v45, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v47
+; GCN-NEXT:    v_mov_b32_e32 v47, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_add_f64 v[25:26], v[25:26], 1.0
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v53
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v30
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v28
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v27
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v62
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v60
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v58
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v32
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v31
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v29
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v63
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v61
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v59
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v56
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v55
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v53
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v51
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v41
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_or_b32_e32 v53, v55, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v55, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v41, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v42, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v43, v59, v58
+; GCN-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v41, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v13f64_to_v52f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB26_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; VI-NEXT:  .LBB26_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v13f64_to_v52f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB26_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v0
+; GFX9-NEXT:  .LBB26_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v43, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v42, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v41, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v40, v3, s4
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v4, v55, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v54, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v53, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v52, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v51, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v50, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v49, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v38, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v37, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v36, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v35, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v34, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v33, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v32, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v31, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v30, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v29, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v28, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v27, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v13f64_to_v52f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    ; implicit-def: $vgpr27
+; GFX11-NEXT:    ; implicit-def: $vgpr26
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB26_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v0
+; GFX11-NEXT:  .LBB26_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v67, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v65, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v64, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v55, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v54, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v53, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v52, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v51, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v50, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v38, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v37, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v36, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v35, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v34, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v33, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v32, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v31, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v30, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v29, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v28, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v27, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v26, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <13 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <13 x double> %a1 to <52 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <13 x double> %a to <52 x half>
+  br label %end
+
+end:
+  %phi = phi <52 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x half> %phi
+}
+
+define <13 x double> @bitcast_v52f16_to_v13f64(<52 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52f16_to_v13f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:56
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:44
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:40
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:84
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v38
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v35
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v36
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v34
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v63
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v44
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v0, v42, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v40, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v2, v54, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v3, v52, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v50, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v49
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v63
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v61
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v45
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v26, v12
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v26, v13
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v26, v14
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v26, v15
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v26, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v48, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v38, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v36, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v34, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v32, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v33, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v35, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v37, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v39, v25
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; kill: killed $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v40
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v54
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v52
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v50
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v49
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v48
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v38
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v15, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v16, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v17, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v18, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v19, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v20, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v21, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v22, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v23, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v24, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v25, v39, v38
+; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52f16_to_v13f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v25
+; VI-NEXT:    v_mov_b32_e32 v33, v24
+; VI-NEXT:    v_mov_b32_e32 v34, v23
+; VI-NEXT:    v_mov_b32_e32 v35, v22
+; VI-NEXT:    v_mov_b32_e32 v36, v21
+; VI-NEXT:    v_mov_b32_e32 v37, v20
+; VI-NEXT:    v_mov_b32_e32 v38, v19
+; VI-NEXT:    v_mov_b32_e32 v39, v18
+; VI-NEXT:    v_mov_b32_e32 v48, v17
+; VI-NEXT:    v_mov_b32_e32 v49, v16
+; VI-NEXT:    v_mov_b32_e32 v50, v15
+; VI-NEXT:    v_mov_b32_e32 v51, v14
+; VI-NEXT:    v_mov_b32_e32 v52, v13
+; VI-NEXT:    v_mov_b32_e32 v53, v12
+; VI-NEXT:    v_mov_b32_e32 v54, v11
+; VI-NEXT:    v_mov_b32_e32 v55, v10
+; VI-NEXT:    v_mov_b32_e32 v40, v9
+; VI-NEXT:    v_mov_b32_e32 v41, v8
+; VI-NEXT:    v_mov_b32_e32 v42, v7
+; VI-NEXT:    v_mov_b32_e32 v43, v6
+; VI-NEXT:    v_mov_b32_e32 v44, v5
+; VI-NEXT:    v_mov_b32_e32 v45, v4
+; VI-NEXT:    v_mov_b32_e32 v46, v3
+; VI-NEXT:    v_mov_b32_e32 v47, v2
+; VI-NEXT:    v_mov_b32_e32 v56, v1
+; VI-NEXT:    v_mov_b32_e32 v57, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v25, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v25, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v25, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v25, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v25, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v25, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v25, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v25, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v25, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v25, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v25, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v25, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v25, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v25, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v25, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v25, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v25, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v25, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v25, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v25, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v25, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v25, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v25, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v47, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v45, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v43, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v41, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v55, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v53, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v51, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v49, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v39, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v35, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v33, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB27_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v25, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v57, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_sdwa v1, v56, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v1, v2, v1
+; VI-NEXT:    v_add_f16_sdwa v2, v47, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v46, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v45, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v44, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v43, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v42, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v41, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v40, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v55, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v54, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v53, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v52, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v51, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v50, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v49, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v48, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v39, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v38, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v37, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v36, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v35, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v34, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v33, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v24, v26, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v32, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:  .LBB27_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52f16_to_v13f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v57, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v35, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v36, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v35
+; GFX9-NEXT:    v_mov_b32_e32 v37, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v36
+; GFX9-NEXT:    v_mov_b32_e32 v38, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v9
+; GFX9-NEXT:    v_mov_b32_e32 v41, v8
+; GFX9-NEXT:    v_mov_b32_e32 v42, v7
+; GFX9-NEXT:    v_mov_b32_e32 v43, v6
+; GFX9-NEXT:    v_mov_b32_e32 v44, v5
+; GFX9-NEXT:    v_mov_b32_e32 v45, v4
+; GFX9-NEXT:    v_mov_b32_e32 v46, v3
+; GFX9-NEXT:    v_mov_b32_e32 v47, v2
+; GFX9-NEXT:    v_mov_b32_e32 v56, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v57
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v37
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v36
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v35
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; kill: killed $vgpr26
+; GFX9-NEXT:  .LBB27_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v34, v57, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v62, v56, s6
+; GFX9-NEXT:    v_perm_b32 v2, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v46, s6
+; GFX9-NEXT:    v_perm_b32 v4, v61, v45, s6
+; GFX9-NEXT:    v_perm_b32 v5, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v6, v59, v43, s6
+; GFX9-NEXT:    v_perm_b32 v7, v58, v42, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v41, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v36, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v35, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52f16_to_v13f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v26
+; GFX11-NEXT:    v_perm_b32 v0, v65, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v66, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v67, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v64, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v55, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v54, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v53, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v52, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v51, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v50, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v49, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v48, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v39, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v38, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v37, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v36, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v35, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v34, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v33, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v32, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v31, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v30, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v29, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v28, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v27, v25, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <52 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <52 x half> %a1 to <13 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x half> %a to <13 x double>
+  br label %end
+
+end:
+  %phi = phi <13 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <13 x double> %phi
+}
+
+define <52 x half> @bitcast_v52i16_to_v52f16(<52 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52i16_to_v52f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; kill: killed $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v42
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v43
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v44
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v45
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v47
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v56
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v57
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v59
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v60
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v61
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v62
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:  .LBB28_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v63, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v62, vcc, 3, v62
+; GCN-NEXT:    v_add_i32_e32 v61, vcc, 3, v61
+; GCN-NEXT:    v_add_i32_e32 v60, vcc, 3, v60
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 3, v59
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 3, v58
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 3, v57
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 3, v56
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 3, v38
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 3, v37
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 3, v36
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 3, v35
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 3, v34
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 3, v33
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 3, v47
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 3, v42
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v14
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v18
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v42
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v43
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v44
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v45
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v47
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v56
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v57
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v59
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v60
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v61
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v62
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v50
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v51
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v52
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v3
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v53
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v54
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v5
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v55
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v40
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v7
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v41
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v49
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v48
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 40, v0
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 44, v0
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 48, v0
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 52, v0
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 56, v0
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 60, v0
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 64, v0
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x44, v0
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_or_b32_e32 v53, v55, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v41, v40
+; GCN-NEXT:    v_or_b32_e32 v55, v43, v42
+; GCN-NEXT:    v_or_b32_e32 v40, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v41, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v42, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v43, v59, v58
+; GCN-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v53, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v41, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52i16_to_v52f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_add_u16_e32 v44, 3, v44
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v43, 3, v43
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v42, 3, v42
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v41, 3, v41
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v40, 3, v40
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v55, 3, v55
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v54, 3, v54
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v53, 3, v53
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v52, 3, v52
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_e32 v51, 3, v51
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_add_u16_e32 v50, 3, v50
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_add_u16_e32 v49, 3, v49
+; VI-NEXT:    v_add_u16_e32 v12, 3, v12
+; VI-NEXT:    v_add_u16_e32 v48, 3, v48
+; VI-NEXT:    v_add_u16_e32 v13, 3, v13
+; VI-NEXT:    v_add_u16_e32 v39, 3, v39
+; VI-NEXT:    v_add_u16_e32 v14, 3, v14
+; VI-NEXT:    v_add_u16_e32 v38, 3, v38
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    v_add_u16_e32 v37, 3, v37
+; VI-NEXT:    v_add_u16_e32 v16, 3, v16
+; VI-NEXT:    v_add_u16_e32 v36, 3, v36
+; VI-NEXT:    v_add_u16_e32 v17, 3, v17
+; VI-NEXT:    v_add_u16_e32 v35, 3, v35
+; VI-NEXT:    v_add_u16_e32 v18, 3, v18
+; VI-NEXT:    v_add_u16_e32 v34, 3, v34
+; VI-NEXT:    v_add_u16_e32 v19, 3, v19
+; VI-NEXT:    v_add_u16_e32 v33, 3, v33
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_add_u16_e32 v32, 3, v32
+; VI-NEXT:    v_add_u16_e32 v21, 3, v21
+; VI-NEXT:    v_add_u16_e32 v31, 3, v31
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v30, 3, v30
+; VI-NEXT:    v_add_u16_e32 v23, 3, v23
+; VI-NEXT:    v_add_u16_e32 v29, 3, v29
+; VI-NEXT:    v_add_u16_e32 v24, 3, v24
+; VI-NEXT:    v_add_u16_e32 v28, 3, v28
+; VI-NEXT:    v_add_u16_e32 v25, 3, v25
+; VI-NEXT:    v_add_u16_e32 v27, 3, v27
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v44
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v43
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v42
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v41
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v40
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v55
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v54
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v53
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v52
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52i16_to_v52f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v25, v44, v25, s6
+; GFX9-NEXT:    v_perm_b32 v24, v43, v24, s6
+; GFX9-NEXT:    v_perm_b32 v23, v42, v23, s6
+; GFX9-NEXT:    v_perm_b32 v22, v41, v22, s6
+; GFX9-NEXT:    v_perm_b32 v21, v40, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v55, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v54, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v53, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v52, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v50, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v48, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v38, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v36, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v34, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v32, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v30, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v29, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v28, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v27, v0, s6
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v25
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v21, v40, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v41, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v42, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v43, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v44, v25, s4
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v0, v27, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v28, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v29, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v30, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v32, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v34, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v36, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v38, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v48, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v50, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v52, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v53, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v54, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v55, v20, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52i16_to_v52f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v25, v68, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v67, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v66, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v65, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v64, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v55, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v54, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v53, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v52, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v50, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v48, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v38, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v36, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v34, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v32, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v27, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v28, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v29, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v30, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v25
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v27, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v28, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v29, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v30, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v32, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v34, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v36, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v38, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v48, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v50, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v52, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v53, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v54, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v55, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v64, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v65, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v66, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v67, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v68, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <52 x i16> %a, splat (i16 3)
+  %a2 = bitcast <52 x i16> %a1 to <52 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x i16> %a to <52 x half>
+  br label %end
+
+end:
+  %phi = phi <52 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x half> %phi
+}
+
+define <52 x i16> @bitcast_v52f16_to_v52i16(<52 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v52f16_to_v52i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:72
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:64
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:56
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:88
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v20
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v23
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v24
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v30
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v63, v9
+; GCN-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_add_f32_e32 v44, 0x38000000, v44
+; GCN-NEXT:    v_add_f32_e32 v41, 0x38000000, v41
+; GCN-NEXT:    v_add_f32_e32 v43, 0x38000000, v43
+; GCN-NEXT:    v_add_f32_e32 v40, 0x38000000, v40
+; GCN-NEXT:    v_add_f32_e32 v42, 0x38000000, v42
+; GCN-NEXT:    v_add_f32_e32 v54, 0x38000000, v54
+; GCN-NEXT:    v_add_f32_e32 v55, 0x38000000, v55
+; GCN-NEXT:    v_add_f32_e32 v52, 0x38000000, v52
+; GCN-NEXT:    v_add_f32_e32 v53, 0x38000000, v53
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v46, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v47, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v56, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v61
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v63
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v12, v8, v12
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v41, v41, v22
+; GCN-NEXT:    v_or_b32_e32 v40, v40, v36
+; GCN-NEXT:    v_or_b32_e32 v54, v54, v42
+; GCN-NEXT:    v_or_b32_e32 v52, v52, v55
+; GCN-NEXT:    v_or_b32_e32 v8, v50, v53
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v51
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v49
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v34
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v30
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v28
+; GCN-NEXT:    v_or_b32_e32 v39, v23, v26
+; GCN-NEXT:    v_or_b32_e32 v35, v32, v24
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_or_b32_e32 v56, v31, v33
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v46
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v47
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v58
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v59
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v60
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v62
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v43
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v44
+; GCN-NEXT:    v_alignbit_b32 v44, v3, v22, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v4, v36, 16
+; GCN-NEXT:    v_alignbit_b32 v42, v5, v42, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v6, v55, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v7, v53, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v19, v51, 16
+; GCN-NEXT:    v_alignbit_b32 v22, v17, v49, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v18, v34, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v16, v30, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v15, v28, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v14, v26, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v13, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v12, v33, 16
+; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v49, 0xffff, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v44
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v40, 0xffff, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v43
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v54, 0xffff, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v52, 0xffff, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v49, v49, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v40, v40, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_or_b32_e32 v2, v4, v2
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v54, v54, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v43
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v52, v52, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v11
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v47
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v39, 0xffff, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v61
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v58
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v35, 0xffff, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v23, v29, v23
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x64, v0
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v28
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v37
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v26
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v59
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v24
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v57
+; GCN-NEXT:    v_or_b32_e32 v28, v56, v36
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v45
+; GCN-NEXT:    buffer_store_dword v49, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v1, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v52f16_to_v52i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_add_f16_e32 v44, 0x200, v44
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_e32 v43, 0x200, v43
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_add_f16_e32 v42, 0x200, v42
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_add_f16_e32 v41, 0x200, v41
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_add_f16_e32 v40, 0x200, v40
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_add_f16_e32 v55, 0x200, v55
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_add_f16_e32 v54, 0x200, v54
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_add_f16_e32 v53, 0x200, v53
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_add_f16_e32 v52, 0x200, v52
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_add_f16_e32 v51, 0x200, v51
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_add_f16_e32 v50, 0x200, v50
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_add_f16_e32 v49, 0x200, v49
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_add_f16_e32 v48, 0x200, v48
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_add_f16_e32 v39, 0x200, v39
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v14
+; VI-NEXT:    v_add_f16_e32 v38, 0x200, v38
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v15
+; VI-NEXT:    v_add_f16_e32 v37, 0x200, v37
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v16
+; VI-NEXT:    v_add_f16_e32 v36, 0x200, v36
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v17
+; VI-NEXT:    v_add_f16_e32 v35, 0x200, v35
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v18
+; VI-NEXT:    v_add_f16_e32 v34, 0x200, v34
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v19
+; VI-NEXT:    v_add_f16_e32 v33, 0x200, v33
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v20
+; VI-NEXT:    v_add_f16_e32 v32, 0x200, v32
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v21
+; VI-NEXT:    v_add_f16_e32 v31, 0x200, v31
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v22
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v30
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v23
+; VI-NEXT:    v_add_f16_e32 v29, 0x200, v29
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v24
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v28
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v25
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v27
+; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v44
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v43
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v42
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v41
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v40
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v55
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v54
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v53
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v52
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v52f16_to_v52i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v25, v44, v25, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v24, v43, v24, s6
+; GFX9-NEXT:    v_perm_b32 v23, v42, v23, s6
+; GFX9-NEXT:    v_perm_b32 v22, v41, v22, s6
+; GFX9-NEXT:    v_perm_b32 v21, v40, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v55, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v54, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v53, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v52, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v50, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v48, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v38, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v36, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v34, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v32, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v30, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v29, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v28, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v27, v0, s6
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v25
+; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v21, v40, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v41, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v42, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v43, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v44, v25, s4
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v0, v27, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v28, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v29, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v30, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v31, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v32, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v33, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v34, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v35, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v36, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v37, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v38, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v39, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v48, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v50, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v52, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v53, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v54, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v55, v20, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v52f16_to_v52i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v26
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v25, v68, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v67, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v66, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v65, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v64, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v55, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v54, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v53, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v52, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v50, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v48, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v38, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v36, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v34, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v32, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v27, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v28, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v29, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v30, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v27, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v25
+; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v27, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v28, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v29, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v30, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v31, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v32, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v33, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v34, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v36, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v37, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v38, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v39, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v48, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v50, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v52, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v53, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v54, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v55, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v64, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v65, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v66, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v67, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v68, v25, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <52 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <52 x half> %a1 to <52 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <52 x half> %a to <52 x i16>
+  br label %end
+
+end:
+  %phi = phi <52 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <52 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
index 37cf5b81b81e3..d1531b389ac42 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
@@ -5,8 +5,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
 
-define <28 x i32> @bitcast_v14i64_to_v28i32(<14 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v14i64_to_v28i32:
+define <28 x float> @bitcast_v28i32_to_v28f32(<28 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i32_to_v28f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
@@ -15,39 +15,39 @@ define <28 x i32> @bitcast_v14i64_to_v28i32(<14 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
 ; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
-; GCN-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:  .LBB0_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v14i64_to_v28i32:
+; VI-LABEL: bitcast_v28i32_to_v28f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
@@ -56,39 +56,39 @@ define <28 x i32> @bitcast_v14i64_to_v28i32(<14 x i64> %a, i32 %b) {
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB0_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
 ; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
-; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
 ; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
-; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
 ; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
-; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
-; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
 ; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
-; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
 ; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
-; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
 ; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
-; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
 ; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
-; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
 ; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:  .LBB0_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v14i64_to_v28i32:
+; GFX9-LABEL: bitcast_v28i32_to_v28f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
@@ -97,39 +97,39 @@ define <28 x i32> @bitcast_v14i64_to_v28i32(<14 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
-; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
-; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
-; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
-; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
-; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
-; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
-; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
+; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
 ; GFX9-NEXT:  .LBB0_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v14i64_to_v28i32:
+; GFX11-LABEL: bitcast_v28i32_to_v28f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -139,41 +139,34 @@ define <28 x i32> @bitcast_v14i64_to_v28i32(<14 x i64> %a, i32 %b) {
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
+; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
 ; GFX11-NEXT:  .LBB0_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
@@ -181,12 +174,180 @@ define <28 x i32> @bitcast_v14i64_to_v28i32(<14 x i64> %a, i32 %b) {
   br i1 %cmp, label %cmp.true, label %cmp.false
 
 cmp.true:
-  %a1 = add <14 x i64> %a, splat (i64 3)
-  %a2 = bitcast <14 x i64> %a1 to <28 x i32>
+  %a1 = add <28 x i32> %a, splat (i32 3)
+  %a2 = bitcast <28 x i32> %a1 to <28 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <14 x i64> %a to <28 x i32>
+  %a3 = bitcast <28 x i32> %a to <28 x float>
+  br label %end
+
+end:
+  %phi = phi <28 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x float> %phi
+}
+
+define <28 x i32> @bitcast_v28f32_to_v28i32(<28 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f32_to_v28i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f32_to_v28i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f32_to_v28i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f32_to_v28i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <28 x float> %a1 to <28 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x float> %a to <28 x i32>
   br label %end
 
 end:
@@ -202,7 +363,7 @@ define <14 x i64> @bitcast_v28i32_to_v14i64(<28 x i32> %a, i32 %b) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
 ; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
@@ -232,7 +393,7 @@ define <14 x i64> @bitcast_v28i32_to_v14i64(<28 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:  .LBB2_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -243,7 +404,7 @@ define <14 x i64> @bitcast_v28i32_to_v14i64(<28 x i32> %a, i32 %b) {
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:    s_cbranch_execz .LBB2_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
 ; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
@@ -273,7 +434,7 @@ define <14 x i64> @bitcast_v28i32_to_v14i64(<28 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:  .LBB2_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -284,7 +445,7 @@ define <14 x i64> @bitcast_v28i32_to_v14i64(<28 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
@@ -314,7 +475,7 @@ define <14 x i64> @bitcast_v28i32_to_v14i64(<28 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:  .LBB2_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -326,7 +487,7 @@ define <14 x i64> @bitcast_v28i32_to_v14i64(<28 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
 ; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
@@ -356,7 +517,7 @@ define <14 x i64> @bitcast_v28i32_to_v14i64(<28 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:  .LBB2_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -375,3 +536,22574 @@ end:
   %phi = phi <14 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
   ret <14 x i64> %phi
 }
+
+define <28 x i32> @bitcast_v14i64_to_v28i32(<14 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i64_to_v28i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i64_to_v28i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i64_to_v28i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i64_to_v28i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i64> %a, splat (i64 3)
+  %a2 = bitcast <14 x i64> %a1 to <28 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i64> %a to <28 x i32>
+  br label %end
+
+end:
+  %phi = phi <28 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i32> %phi
+}
+
+define <14 x double> @bitcast_v28i32_to_v14f64(<28 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i32_to_v14f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28i32_to_v14f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB4_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28i32_to_v14f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
+; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB4_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28i32_to_v14f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
+; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <28 x i32> %a, splat (i32 3)
+  %a2 = bitcast <28 x i32> %a1 to <14 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x i32> %a to <14 x double>
+  br label %end
+
+end:
+  %phi = phi <14 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x double> %phi
+}
+
+define <28 x i32> @bitcast_v14f64_to_v28i32(<14 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f64_to_v28i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f64_to_v28i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f64_to_v28i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f64_to_v28i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <14 x double> %a1 to <28 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x double> %a to <28 x i32>
+  br label %end
+
+end:
+  %phi = phi <28 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i32> %phi
+}
+
+define <56 x i16> @bitcast_v28i32_to_v56i16(<28 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i32_to_v56i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v29
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v29, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v45, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_alignbit_b32 v29, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v45, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_and_b32_e32 v57, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_and_b32_e32 v58, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v59, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v60, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_and_b32_e32 v61, 0xffff, v5
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v5, v57, v45
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v45, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v41, v58, v56
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v59, v43
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v58, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v43, v60, v47
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v47, v61, v62
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v45, v45, v46
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v56, v56, v57
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v44, v58, v44
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v40
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x64, v0
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x68, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x6c, v0
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v30
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v36
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v29
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v34
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v41, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v45, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28i32_to_v56i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB6_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB6_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28i32_to_v56i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB6_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
+; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB6_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v46, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v45, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v44, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v43, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v42, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v41, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v40, v7, s4
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v8, v55, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v54, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v53, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v52, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v51, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v48, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v39, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v38, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v37, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v35, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v34, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v33, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v32, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v31, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v30, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v29, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28i32_to_v56i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB6_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
+; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB6_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v71, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v69, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v67, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v66, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v65, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v64, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v55, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v54, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v53, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v52, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v37, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v36, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v35, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v34, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v33, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v32, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v31, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v30, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v29, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v28, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <28 x i32> %a, splat (i32 3)
+  %a2 = bitcast <28 x i32> %a1 to <56 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x i32> %a to <56 x i16>
+  br label %end
+
+end:
+  %phi = phi <56 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x i16> %phi
+}
+
+define <28 x i32> @bitcast_v56i16_to_v28i32(<56 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56i16_to_v28i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:100
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v41
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v42
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v63
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v62
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v61
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v60
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v59
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v58
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v56
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v47
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v46
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v45
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v44
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v43
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v57
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v28
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:  .LBB7_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v57
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_or_b32_e32 v0, v41, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v42, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v63, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v62, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v61, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v60, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v59, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v58, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v56, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v47, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v46, v11
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v28, v12
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v28, v13
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v28, v14
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v28, v15
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v28, v16
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v28, v17
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v28, v18
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v19
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v28, v20
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v28, v21
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v28, v22
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v28, v23
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v28, v24
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v28, v25
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v28, v26
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v28, v27
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x30000, v27
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56i16_to_v28i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v27
+; VI-NEXT:    v_mov_b32_e32 v33, v26
+; VI-NEXT:    v_mov_b32_e32 v34, v25
+; VI-NEXT:    v_mov_b32_e32 v35, v24
+; VI-NEXT:    v_mov_b32_e32 v36, v23
+; VI-NEXT:    v_mov_b32_e32 v37, v22
+; VI-NEXT:    v_mov_b32_e32 v38, v21
+; VI-NEXT:    v_mov_b32_e32 v39, v20
+; VI-NEXT:    v_mov_b32_e32 v48, v19
+; VI-NEXT:    v_mov_b32_e32 v49, v18
+; VI-NEXT:    v_mov_b32_e32 v50, v17
+; VI-NEXT:    v_mov_b32_e32 v51, v16
+; VI-NEXT:    v_mov_b32_e32 v52, v15
+; VI-NEXT:    v_mov_b32_e32 v53, v14
+; VI-NEXT:    v_mov_b32_e32 v54, v13
+; VI-NEXT:    v_mov_b32_e32 v55, v12
+; VI-NEXT:    v_mov_b32_e32 v40, v11
+; VI-NEXT:    v_mov_b32_e32 v41, v10
+; VI-NEXT:    v_mov_b32_e32 v42, v9
+; VI-NEXT:    v_mov_b32_e32 v43, v8
+; VI-NEXT:    v_mov_b32_e32 v44, v7
+; VI-NEXT:    v_mov_b32_e32 v45, v6
+; VI-NEXT:    v_mov_b32_e32 v46, v5
+; VI-NEXT:    v_mov_b32_e32 v47, v4
+; VI-NEXT:    v_mov_b32_e32 v56, v3
+; VI-NEXT:    v_mov_b32_e32 v57, v2
+; VI-NEXT:    v_mov_b32_e32 v58, v1
+; VI-NEXT:    v_mov_b32_e32 v59, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v27, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v27, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v27, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v27, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v27, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v27, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v27, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v27, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v27, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v27, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v27, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v27, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v27, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v27, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v27, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v27, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v27, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v27, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v27, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v27, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v27, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v27, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v27, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v27, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v27, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v27, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB7_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v27, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v59
+; VI-NEXT:    v_add_u16_sdwa v1, v59, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v2, 3, v58
+; VI-NEXT:    v_add_u16_sdwa v3, v58, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v3, v57, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v4, v56, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v5, v47, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v6, v46, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v7, v45, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v8, v44, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v9, v43, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v10, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v11, v41, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v12, v40, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v13, v55, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v14, v54, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v15, v53, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v16, v52, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v17, v51, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v18, v50, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v19, v49, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v20, v48, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v21, v39, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v22, v38, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v23, v37, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v24, v36, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v25, v35, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v25
+; VI-NEXT:    v_add_u16_e32 v25, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v26, v34, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v25, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v28, v33, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v26, v26, v28
+; VI-NEXT:    v_add_u16_e32 v28, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v27, v32, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:  .LBB7_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56i16_to_v28i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v59, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v37, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v38, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v9
+; GFX9-NEXT:    v_mov_b32_e32 v43, v8
+; GFX9-NEXT:    v_mov_b32_e32 v44, v7
+; GFX9-NEXT:    v_mov_b32_e32 v45, v6
+; GFX9-NEXT:    v_mov_b32_e32 v46, v5
+; GFX9-NEXT:    v_mov_b32_e32 v47, v4
+; GFX9-NEXT:    v_mov_b32_e32 v56, v3
+; GFX9-NEXT:    v_mov_b32_e32 v57, v2
+; GFX9-NEXT:    v_mov_b32_e32 v58, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v59
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v37
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:  .LBB7_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56i16_to_v28i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    v_perm_b32 v0, v69, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v71, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v68, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v67, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v66, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v65, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v64, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v55, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v54, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v53, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v52, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v51, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v50, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v49, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v48, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v39, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v38, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v37, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v36, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v35, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v34, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v33, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v32, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v31, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v30, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v29, v27, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <56 x i16> %a, splat (i16 3)
+  %a2 = bitcast <56 x i16> %a1 to <28 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x i16> %a to <28 x i32>
+  br label %end
+
+end:
+  %phi = phi <28 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i32> %phi
+}
+
+define <56 x half> @bitcast_v28i32_to_v56f16(<28 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28i32_to_v56f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v29
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v40
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v44
+; GCN-NEXT:    v_mov_b32_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v46
+; GCN-NEXT:    v_mov_b32_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v56
+; GCN-NEXT:    v_mov_b32_e32 v56, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v58
+; GCN-NEXT:    v_mov_b32_e32 v58, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v60
+; GCN-NEXT:    v_mov_b32_e32 v60, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v62
+; GCN-NEXT:    v_mov_b32_e32 v62, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v43
+; GCN-NEXT:    v_mov_b32_e32 v43, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v45
+; GCN-NEXT:    v_mov_b32_e32 v45, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v47
+; GCN-NEXT:    v_mov_b32_e32 v47, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v57
+; GCN-NEXT:    v_mov_b32_e32 v57, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v59
+; GCN-NEXT:    v_mov_b32_e32 v59, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v61
+; GCN-NEXT:    v_mov_b32_e32 v61, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:  .LBB8_2: ; %Flow
+; GCN-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v42, v63
+; GCN-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:  .LBB8_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v52
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v39
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v37
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v49
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v48
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v38
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v36
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v34
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v32
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 56, v0
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v61
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v59
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v57
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v47
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v45
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v43
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v42
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v44
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x5c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v46
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v56
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v58
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x68, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v60
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x6c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v62
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v45, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v47, v59, v58
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v45, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28i32_to_v56f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB8_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB8_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28i32_to_v56f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB8_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
+; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB8_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v46, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v45, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v44, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v43, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v42, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v41, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v40, v7, s4
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v8, v55, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v54, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v53, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v52, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v51, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v48, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v39, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v38, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v37, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v35, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v34, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v33, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v32, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v31, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v30, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v29, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28i32_to_v56f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB8_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
+; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB8_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v71, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v69, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v67, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v66, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v65, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v64, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v55, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v54, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v53, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v52, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v37, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v36, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v35, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v34, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v33, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v32, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v31, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v30, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v29, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v28, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <28 x i32> %a, splat (i32 3)
+  %a2 = bitcast <28 x i32> %a1 to <56 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x i32> %a to <56 x half>
+  br label %end
+
+end:
+  %phi = phi <56 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x half> %phi
+}
+
+define <28 x i32> @bitcast_v56f16_to_v28i32(<56 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56f16_to_v28i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v6
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v9
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v8
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v11
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v55
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v53
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v54
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v51
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v49
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v36
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v35
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v34
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v31
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v0, v46, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v44, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v2, v42, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v41
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v51
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v28, v14
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v28, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v55, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v52, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v50, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v38, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v36, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v34, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v33, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v37, v25
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v26
+; GCN-NEXT:    v_or_b32_e32 v27, v49, v27
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:  .LBB9_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v44
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v42
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v41
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v55
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v52
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v50
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v48
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v38
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v36
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v17, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v18, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v19, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v20, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v21, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v22, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v23, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v25, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v27, v49, v48
+; GCN-NEXT:  .LBB9_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56f16_to_v28i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v27
+; VI-NEXT:    v_mov_b32_e32 v33, v26
+; VI-NEXT:    v_mov_b32_e32 v34, v25
+; VI-NEXT:    v_mov_b32_e32 v35, v24
+; VI-NEXT:    v_mov_b32_e32 v36, v23
+; VI-NEXT:    v_mov_b32_e32 v37, v22
+; VI-NEXT:    v_mov_b32_e32 v38, v21
+; VI-NEXT:    v_mov_b32_e32 v39, v20
+; VI-NEXT:    v_mov_b32_e32 v48, v19
+; VI-NEXT:    v_mov_b32_e32 v49, v18
+; VI-NEXT:    v_mov_b32_e32 v50, v17
+; VI-NEXT:    v_mov_b32_e32 v51, v16
+; VI-NEXT:    v_mov_b32_e32 v52, v15
+; VI-NEXT:    v_mov_b32_e32 v53, v14
+; VI-NEXT:    v_mov_b32_e32 v54, v13
+; VI-NEXT:    v_mov_b32_e32 v55, v12
+; VI-NEXT:    v_mov_b32_e32 v40, v11
+; VI-NEXT:    v_mov_b32_e32 v41, v10
+; VI-NEXT:    v_mov_b32_e32 v42, v9
+; VI-NEXT:    v_mov_b32_e32 v43, v8
+; VI-NEXT:    v_mov_b32_e32 v44, v7
+; VI-NEXT:    v_mov_b32_e32 v45, v6
+; VI-NEXT:    v_mov_b32_e32 v46, v5
+; VI-NEXT:    v_mov_b32_e32 v47, v4
+; VI-NEXT:    v_mov_b32_e32 v56, v3
+; VI-NEXT:    v_mov_b32_e32 v57, v2
+; VI-NEXT:    v_mov_b32_e32 v58, v1
+; VI-NEXT:    v_mov_b32_e32 v59, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v27, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v27, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v27, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v27, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v27, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v27, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v27, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v27, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v27, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v27, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v27, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v27, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v27, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v27, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v27, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v27, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v27, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v27, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v27, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v27, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v27, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v27, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v27, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v27, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v27, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v27, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB9_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v27, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v59, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v59
+; VI-NEXT:    v_add_f16_sdwa v2, v58, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v58
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v57, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v56, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v47, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v46, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v45, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v44, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v43, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v41, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v40, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v55, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v54, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v53, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v52, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v51, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v50, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v49, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v48, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v39, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v38, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v37, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v36, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v35, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v24, v25, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v34, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:    v_add_f16_sdwa v26, v33, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v26, v28, v26
+; VI-NEXT:    v_add_f16_sdwa v27, v32, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:  .LBB9_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56f16_to_v28i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v59, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v37, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v38, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v9
+; GFX9-NEXT:    v_mov_b32_e32 v43, v8
+; GFX9-NEXT:    v_mov_b32_e32 v44, v7
+; GFX9-NEXT:    v_mov_b32_e32 v45, v6
+; GFX9-NEXT:    v_mov_b32_e32 v46, v5
+; GFX9-NEXT:    v_mov_b32_e32 v47, v4
+; GFX9-NEXT:    v_mov_b32_e32 v56, v3
+; GFX9-NEXT:    v_mov_b32_e32 v57, v2
+; GFX9-NEXT:    v_mov_b32_e32 v58, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v59
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v37
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:  .LBB9_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56f16_to_v28i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    v_perm_b32 v0, v69, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v71, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v68, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v67, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v66, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v65, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v64, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v55, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v54, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v53, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v52, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v51, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v50, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v49, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v48, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v39, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v38, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v37, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v36, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v35, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v34, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v33, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v32, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v31, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v30, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v29, v27, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <56 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <56 x half> %a1 to <28 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x half> %a to <28 x i32>
+  br label %end
+
+end:
+  %phi = phi <28 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x i32> %phi
+}
+
+define <14 x i64> @bitcast_v28f32_to_v14i64(<28 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f32_to_v14i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f32_to_v14i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f32_to_v14i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f32_to_v14i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <28 x float> %a1 to <14 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x float> %a to <14 x i64>
+  br label %end
+
+end:
+  %phi = phi <14 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i64> %phi
+}
+
+define <28 x float> @bitcast_v14i64_to_v28f32(<14 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i64_to_v28f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i64_to_v28f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i64_to_v28f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i64_to_v28f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i64> %a, splat (i64 3)
+  %a2 = bitcast <14 x i64> %a1 to <28 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i64> %a to <28 x float>
+  br label %end
+
+end:
+  %phi = phi <28 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x float> %phi
+}
+
+define <14 x double> @bitcast_v28f32_to_v14f64(<28 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f32_to_v14f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f32_to_v14f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f32_to_v14f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f32_to_v14f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB12_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <28 x float> %a1 to <14 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x float> %a to <14 x double>
+  br label %end
+
+end:
+  %phi = phi <14 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x double> %phi
+}
+
+define <28 x float> @bitcast_v14f64_to_v28f32(<14 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f64_to_v28f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f64_to_v28f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f64_to_v28f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f64_to_v28f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <14 x double> %a1 to <28 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x double> %a to <28 x float>
+  br label %end
+
+end:
+  %phi = phi <28 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x float> %phi
+}
+
+define <56 x i16> @bitcast_v28f32_to_v56i16(<28 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f32_to_v56i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v29
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v29, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v45, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_alignbit_b32 v29, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v45, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_and_b32_e32 v57, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_and_b32_e32 v58, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v59, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v60, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_and_b32_e32 v61, 0xffff, v5
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v5, v57, v45
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v45, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v41, v58, v56
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v59, v43
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v58, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v43, v60, v47
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v47, v61, v62
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v45, v45, v46
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v56, v56, v57
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v44, v58, v44
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v40
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x64, v0
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x68, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x6c, v0
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v30
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v36
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v29
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v34
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v41, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v45, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f32_to_v56i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB14_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB14_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f32_to_v56i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB14_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB14_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v46, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v45, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v44, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v43, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v42, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v41, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v40, v7, s4
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v8, v55, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v54, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v53, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v52, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v51, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v48, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v39, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v38, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v37, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v35, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v34, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v33, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v32, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v31, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v30, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v29, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f32_to_v56i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB14_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB14_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v71, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v69, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v67, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v66, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v65, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v64, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v55, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v54, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v53, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v52, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v37, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v36, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v35, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v34, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v33, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v32, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v31, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v30, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v29, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v28, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <28 x float> %a1 to <56 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x float> %a to <56 x i16>
+  br label %end
+
+end:
+  %phi = phi <56 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x i16> %phi
+}
+
+define <28 x float> @bitcast_v56i16_to_v28f32(<56 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56i16_to_v28f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:100
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v41
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v42
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v63
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v62
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v61
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v60
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v59
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v58
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v56
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v47
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v46
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v45
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v44
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v43
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v57
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v28
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:  .LBB15_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v57
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_or_b32_e32 v0, v41, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v42, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v63, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v62, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v61, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v60, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v59, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v58, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v56, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v47, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v46, v11
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v28, v12
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v28, v13
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v28, v14
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v28, v15
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v28, v16
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v28, v17
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v28, v18
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v19
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v28, v20
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v28, v21
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v28, v22
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v28, v23
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v28, v24
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v28, v25
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v28, v26
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v28, v27
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x30000, v27
+; GCN-NEXT:  .LBB15_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56i16_to_v28f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v27
+; VI-NEXT:    v_mov_b32_e32 v33, v26
+; VI-NEXT:    v_mov_b32_e32 v34, v25
+; VI-NEXT:    v_mov_b32_e32 v35, v24
+; VI-NEXT:    v_mov_b32_e32 v36, v23
+; VI-NEXT:    v_mov_b32_e32 v37, v22
+; VI-NEXT:    v_mov_b32_e32 v38, v21
+; VI-NEXT:    v_mov_b32_e32 v39, v20
+; VI-NEXT:    v_mov_b32_e32 v48, v19
+; VI-NEXT:    v_mov_b32_e32 v49, v18
+; VI-NEXT:    v_mov_b32_e32 v50, v17
+; VI-NEXT:    v_mov_b32_e32 v51, v16
+; VI-NEXT:    v_mov_b32_e32 v52, v15
+; VI-NEXT:    v_mov_b32_e32 v53, v14
+; VI-NEXT:    v_mov_b32_e32 v54, v13
+; VI-NEXT:    v_mov_b32_e32 v55, v12
+; VI-NEXT:    v_mov_b32_e32 v40, v11
+; VI-NEXT:    v_mov_b32_e32 v41, v10
+; VI-NEXT:    v_mov_b32_e32 v42, v9
+; VI-NEXT:    v_mov_b32_e32 v43, v8
+; VI-NEXT:    v_mov_b32_e32 v44, v7
+; VI-NEXT:    v_mov_b32_e32 v45, v6
+; VI-NEXT:    v_mov_b32_e32 v46, v5
+; VI-NEXT:    v_mov_b32_e32 v47, v4
+; VI-NEXT:    v_mov_b32_e32 v56, v3
+; VI-NEXT:    v_mov_b32_e32 v57, v2
+; VI-NEXT:    v_mov_b32_e32 v58, v1
+; VI-NEXT:    v_mov_b32_e32 v59, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v27, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v27, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v27, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v27, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v27, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v27, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v27, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v27, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v27, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v27, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v27, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v27, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v27, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v27, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v27, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v27, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v27, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v27, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v27, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v27, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v27, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v27, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v27, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v27, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v27, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v27, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB15_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v27, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v59
+; VI-NEXT:    v_add_u16_sdwa v1, v59, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v2, 3, v58
+; VI-NEXT:    v_add_u16_sdwa v3, v58, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v3, v57, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v4, v56, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v5, v47, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v6, v46, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v7, v45, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v8, v44, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v9, v43, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v10, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v11, v41, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v12, v40, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v13, v55, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v14, v54, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v15, v53, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v16, v52, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v17, v51, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v18, v50, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v19, v49, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v20, v48, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v21, v39, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v22, v38, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v23, v37, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v24, v36, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v25, v35, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v25
+; VI-NEXT:    v_add_u16_e32 v25, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v26, v34, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v25, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v28, v33, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v26, v26, v28
+; VI-NEXT:    v_add_u16_e32 v28, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v27, v32, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:  .LBB15_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56i16_to_v28f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v59, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v37, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v38, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v9
+; GFX9-NEXT:    v_mov_b32_e32 v43, v8
+; GFX9-NEXT:    v_mov_b32_e32 v44, v7
+; GFX9-NEXT:    v_mov_b32_e32 v45, v6
+; GFX9-NEXT:    v_mov_b32_e32 v46, v5
+; GFX9-NEXT:    v_mov_b32_e32 v47, v4
+; GFX9-NEXT:    v_mov_b32_e32 v56, v3
+; GFX9-NEXT:    v_mov_b32_e32 v57, v2
+; GFX9-NEXT:    v_mov_b32_e32 v58, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v59
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v37
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:  .LBB15_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56i16_to_v28f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    v_perm_b32 v0, v69, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v71, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v68, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v67, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v66, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v65, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v64, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v55, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v54, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v53, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v52, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v51, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v50, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v49, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v48, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v39, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v38, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v37, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v36, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v35, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v34, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v33, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v32, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v31, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v30, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v29, v27, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <56 x i16> %a, splat (i16 3)
+  %a2 = bitcast <56 x i16> %a1 to <28 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x i16> %a to <28 x float>
+  br label %end
+
+end:
+  %phi = phi <28 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x float> %phi
+}
+
+define <56 x half> @bitcast_v28f32_to_v56f16(<28 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v28f32_to_v56f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v29
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v40
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v44
+; GCN-NEXT:    v_mov_b32_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v46
+; GCN-NEXT:    v_mov_b32_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v56
+; GCN-NEXT:    v_mov_b32_e32 v56, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v58
+; GCN-NEXT:    v_mov_b32_e32 v58, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v60
+; GCN-NEXT:    v_mov_b32_e32 v60, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v62
+; GCN-NEXT:    v_mov_b32_e32 v62, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v43
+; GCN-NEXT:    v_mov_b32_e32 v43, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v45
+; GCN-NEXT:    v_mov_b32_e32 v45, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v47
+; GCN-NEXT:    v_mov_b32_e32 v47, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v57
+; GCN-NEXT:    v_mov_b32_e32 v57, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v59
+; GCN-NEXT:    v_mov_b32_e32 v59, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v61
+; GCN-NEXT:    v_mov_b32_e32 v61, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:  .LBB16_2: ; %Flow
+; GCN-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v42, v63
+; GCN-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:  .LBB16_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v52
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v39
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v37
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v49
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v48
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v38
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v36
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v34
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v32
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 56, v0
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v61
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v59
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v57
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v47
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v45
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v43
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v42
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v44
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x5c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v46
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v56
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v58
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x68, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v60
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x6c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v62
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v45, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v47, v59, v58
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v45, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v28f32_to_v56f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB16_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB16_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v28f32_to_v56f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB16_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB16_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v46, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v45, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v44, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v43, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v42, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v41, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v40, v7, s4
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v8, v55, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v54, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v53, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v52, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v51, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v48, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v39, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v38, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v37, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v35, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v34, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v33, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v32, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v31, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v30, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v29, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v28f32_to_v56f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB16_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB16_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v71, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v69, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v67, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v66, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v65, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v64, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v55, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v54, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v53, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v52, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v37, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v36, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v35, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v34, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v33, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v32, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v31, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v30, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v29, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v28, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <28 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <28 x float> %a1 to <56 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <28 x float> %a to <56 x half>
+  br label %end
+
+end:
+  %phi = phi <56 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x half> %phi
+}
+
+define <28 x float> @bitcast_v56f16_to_v28f32(<56 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56f16_to_v28f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v6
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v9
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v8
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v11
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v55
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v53
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v54
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v51
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v49
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v36
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v35
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v34
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v31
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v0, v46, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v44, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v2, v42, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v41
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v51
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v28, v14
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v28, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v55, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v52, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v50, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v38, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v36, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v34, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v33, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v37, v25
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v26
+; GCN-NEXT:    v_or_b32_e32 v27, v49, v27
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v44
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v42
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v41
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v55
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v52
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v50
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v48
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v38
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v36
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v17, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v18, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v19, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v20, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v21, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v22, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v23, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v25, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v27, v49, v48
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56f16_to_v28f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v27
+; VI-NEXT:    v_mov_b32_e32 v33, v26
+; VI-NEXT:    v_mov_b32_e32 v34, v25
+; VI-NEXT:    v_mov_b32_e32 v35, v24
+; VI-NEXT:    v_mov_b32_e32 v36, v23
+; VI-NEXT:    v_mov_b32_e32 v37, v22
+; VI-NEXT:    v_mov_b32_e32 v38, v21
+; VI-NEXT:    v_mov_b32_e32 v39, v20
+; VI-NEXT:    v_mov_b32_e32 v48, v19
+; VI-NEXT:    v_mov_b32_e32 v49, v18
+; VI-NEXT:    v_mov_b32_e32 v50, v17
+; VI-NEXT:    v_mov_b32_e32 v51, v16
+; VI-NEXT:    v_mov_b32_e32 v52, v15
+; VI-NEXT:    v_mov_b32_e32 v53, v14
+; VI-NEXT:    v_mov_b32_e32 v54, v13
+; VI-NEXT:    v_mov_b32_e32 v55, v12
+; VI-NEXT:    v_mov_b32_e32 v40, v11
+; VI-NEXT:    v_mov_b32_e32 v41, v10
+; VI-NEXT:    v_mov_b32_e32 v42, v9
+; VI-NEXT:    v_mov_b32_e32 v43, v8
+; VI-NEXT:    v_mov_b32_e32 v44, v7
+; VI-NEXT:    v_mov_b32_e32 v45, v6
+; VI-NEXT:    v_mov_b32_e32 v46, v5
+; VI-NEXT:    v_mov_b32_e32 v47, v4
+; VI-NEXT:    v_mov_b32_e32 v56, v3
+; VI-NEXT:    v_mov_b32_e32 v57, v2
+; VI-NEXT:    v_mov_b32_e32 v58, v1
+; VI-NEXT:    v_mov_b32_e32 v59, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v27, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v27, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v27, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v27, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v27, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v27, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v27, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v27, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v27, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v27, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v27, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v27, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v27, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v27, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v27, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v27, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v27, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v27, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v27, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v27, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v27, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v27, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v27, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v27, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v27, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v27, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB17_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v27, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v59, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v59
+; VI-NEXT:    v_add_f16_sdwa v2, v58, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v58
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v57, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v56, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v47, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v46, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v45, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v44, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v43, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v41, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v40, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v55, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v54, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v53, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v52, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v51, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v50, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v49, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v48, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v39, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v38, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v37, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v36, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v35, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v24, v25, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v34, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:    v_add_f16_sdwa v26, v33, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v26, v28, v26
+; VI-NEXT:    v_add_f16_sdwa v27, v32, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:  .LBB17_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56f16_to_v28f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v59, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v37, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v38, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v9
+; GFX9-NEXT:    v_mov_b32_e32 v43, v8
+; GFX9-NEXT:    v_mov_b32_e32 v44, v7
+; GFX9-NEXT:    v_mov_b32_e32 v45, v6
+; GFX9-NEXT:    v_mov_b32_e32 v46, v5
+; GFX9-NEXT:    v_mov_b32_e32 v47, v4
+; GFX9-NEXT:    v_mov_b32_e32 v56, v3
+; GFX9-NEXT:    v_mov_b32_e32 v57, v2
+; GFX9-NEXT:    v_mov_b32_e32 v58, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v59
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v37
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:  .LBB17_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56f16_to_v28f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    v_perm_b32 v0, v69, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v71, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v68, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v67, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v66, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v65, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v64, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v55, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v54, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v53, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v52, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v51, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v50, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v49, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v48, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v39, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v38, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v37, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v36, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v35, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v34, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v33, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v32, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v31, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v30, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v29, v27, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <56 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <56 x half> %a1 to <28 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x half> %a to <28 x float>
+  br label %end
+
+end:
+  %phi = phi <28 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <28 x float> %phi
+}
+
+define <14 x double> @bitcast_v14i64_to_v14f64(<14 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i64_to_v14f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i64_to_v14f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i64_to_v14f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i64_to_v14f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i64> %a, splat (i64 3)
+  %a2 = bitcast <14 x i64> %a1 to <14 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i64> %a to <14 x double>
+  br label %end
+
+end:
+  %phi = phi <14 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x double> %phi
+}
+
+define <14 x i64> @bitcast_v14f64_to_v14i64(<14 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f64_to_v14i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f64_to_v14i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f64_to_v14i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f64_to_v14i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <14 x double> %a1 to <14 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x double> %a to <14 x i64>
+  br label %end
+
+end:
+  %phi = phi <14 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i64> %phi
+}
+
+define <56 x i16> @bitcast_v14i64_to_v56i16(<14 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i64_to_v56i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v29
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v29, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v45, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_addc_u32_e32 v24, vcc, 0, v24, vcc
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_addc_u32_e32 v26, vcc, 0, v26, vcc
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_addc_u32_e32 v28, vcc, 0, v28, vcc
+; GCN-NEXT:    v_alignbit_b32 v29, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v54, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v40, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v45, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_and_b32_e32 v57, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_and_b32_e32 v58, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v59, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v60, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_and_b32_e32 v61, 0xffff, v5
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v5, v57, v45
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v45, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v40, v58, v56
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v59, v43
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v58, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v43, v60, v47
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v47, v61, v62
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v45, v45, v46
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v56, v56, v57
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v44, v58, v44
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x64, v0
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x68, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x6c, v0
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v30
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v36
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v29
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v34
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v45, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i64_to_v56i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB20_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB20_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i64_to_v56i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB20_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB20_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v46, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v45, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v44, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v43, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v42, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v41, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v40, v7, s4
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v8, v55, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v54, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v53, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v52, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v51, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v48, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v39, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v38, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v37, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v35, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v34, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v33, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v32, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v31, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v30, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v29, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i64_to_v56i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB20_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB20_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v71, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v69, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v67, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v66, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v65, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v64, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v55, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v54, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v53, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v52, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v37, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v36, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v35, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v34, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v33, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v32, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v31, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v30, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v29, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v28, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i64> %a, splat (i64 3)
+  %a2 = bitcast <14 x i64> %a1 to <56 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i64> %a to <56 x i16>
+  br label %end
+
+end:
+  %phi = phi <56 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x i16> %phi
+}
+
+define <14 x i64> @bitcast_v56i16_to_v14i64(<56 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56i16_to_v14i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:100
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v41
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v42
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v63
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v62
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v61
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v60
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v59
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v58
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v56
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v47
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v46
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v45
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v44
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v43
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v57
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v28
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:  .LBB21_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v57
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_or_b32_e32 v0, v41, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v42, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v63, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v62, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v61, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v60, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v59, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v58, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v56, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v47, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v46, v11
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v28, v12
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v28, v13
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v28, v14
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v28, v15
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v28, v16
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v28, v17
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v28, v18
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v19
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v28, v20
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v28, v21
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v28, v22
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v28, v23
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v28, v24
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v28, v25
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v28, v26
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v28, v27
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x30000, v27
+; GCN-NEXT:  .LBB21_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56i16_to_v14i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v27
+; VI-NEXT:    v_mov_b32_e32 v33, v26
+; VI-NEXT:    v_mov_b32_e32 v34, v25
+; VI-NEXT:    v_mov_b32_e32 v35, v24
+; VI-NEXT:    v_mov_b32_e32 v36, v23
+; VI-NEXT:    v_mov_b32_e32 v37, v22
+; VI-NEXT:    v_mov_b32_e32 v38, v21
+; VI-NEXT:    v_mov_b32_e32 v39, v20
+; VI-NEXT:    v_mov_b32_e32 v48, v19
+; VI-NEXT:    v_mov_b32_e32 v49, v18
+; VI-NEXT:    v_mov_b32_e32 v50, v17
+; VI-NEXT:    v_mov_b32_e32 v51, v16
+; VI-NEXT:    v_mov_b32_e32 v52, v15
+; VI-NEXT:    v_mov_b32_e32 v53, v14
+; VI-NEXT:    v_mov_b32_e32 v54, v13
+; VI-NEXT:    v_mov_b32_e32 v55, v12
+; VI-NEXT:    v_mov_b32_e32 v40, v11
+; VI-NEXT:    v_mov_b32_e32 v41, v10
+; VI-NEXT:    v_mov_b32_e32 v42, v9
+; VI-NEXT:    v_mov_b32_e32 v43, v8
+; VI-NEXT:    v_mov_b32_e32 v44, v7
+; VI-NEXT:    v_mov_b32_e32 v45, v6
+; VI-NEXT:    v_mov_b32_e32 v46, v5
+; VI-NEXT:    v_mov_b32_e32 v47, v4
+; VI-NEXT:    v_mov_b32_e32 v56, v3
+; VI-NEXT:    v_mov_b32_e32 v57, v2
+; VI-NEXT:    v_mov_b32_e32 v58, v1
+; VI-NEXT:    v_mov_b32_e32 v59, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v27, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v27, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v27, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v27, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v27, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v27, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v27, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v27, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v27, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v27, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v27, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v27, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v27, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v27, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v27, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v27, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v27, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v27, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v27, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v27, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v27, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v27, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v27, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v27, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v27, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v27, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB21_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v27, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v59
+; VI-NEXT:    v_add_u16_sdwa v1, v59, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v2, 3, v58
+; VI-NEXT:    v_add_u16_sdwa v3, v58, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v3, v57, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v4, v56, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v5, v47, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v6, v46, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v7, v45, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v8, v44, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v9, v43, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v10, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v11, v41, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v12, v40, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v13, v55, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v14, v54, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v15, v53, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v16, v52, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v17, v51, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v18, v50, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v19, v49, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v20, v48, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v21, v39, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v22, v38, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v23, v37, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v24, v36, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v25, v35, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v25
+; VI-NEXT:    v_add_u16_e32 v25, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v26, v34, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v25, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v28, v33, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v26, v26, v28
+; VI-NEXT:    v_add_u16_e32 v28, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v27, v32, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:  .LBB21_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56i16_to_v14i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v59, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v37, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v38, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v9
+; GFX9-NEXT:    v_mov_b32_e32 v43, v8
+; GFX9-NEXT:    v_mov_b32_e32 v44, v7
+; GFX9-NEXT:    v_mov_b32_e32 v45, v6
+; GFX9-NEXT:    v_mov_b32_e32 v46, v5
+; GFX9-NEXT:    v_mov_b32_e32 v47, v4
+; GFX9-NEXT:    v_mov_b32_e32 v56, v3
+; GFX9-NEXT:    v_mov_b32_e32 v57, v2
+; GFX9-NEXT:    v_mov_b32_e32 v58, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v59
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v37
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:  .LBB21_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56i16_to_v14i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    v_perm_b32 v0, v69, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v71, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v68, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v67, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v66, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v65, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v64, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v55, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v54, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v53, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v52, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v51, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v50, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v49, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v48, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v39, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v38, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v37, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v36, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v35, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v34, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v33, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v32, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v31, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v30, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v29, v27, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <56 x i16> %a, splat (i16 3)
+  %a2 = bitcast <56 x i16> %a1 to <14 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x i16> %a to <14 x i64>
+  br label %end
+
+end:
+  %phi = phi <14 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i64> %phi
+}
+
+define <56 x half> @bitcast_v14i64_to_v56f16(<14 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14i64_to_v56f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v29
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v40
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v44
+; GCN-NEXT:    v_mov_b32_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v46
+; GCN-NEXT:    v_mov_b32_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v56
+; GCN-NEXT:    v_mov_b32_e32 v56, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v58
+; GCN-NEXT:    v_mov_b32_e32 v58, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v60
+; GCN-NEXT:    v_mov_b32_e32 v60, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v62
+; GCN-NEXT:    v_mov_b32_e32 v62, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v43
+; GCN-NEXT:    v_mov_b32_e32 v43, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v45
+; GCN-NEXT:    v_mov_b32_e32 v45, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v47
+; GCN-NEXT:    v_mov_b32_e32 v47, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v57
+; GCN-NEXT:    v_mov_b32_e32 v57, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v59
+; GCN-NEXT:    v_mov_b32_e32 v59, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v61
+; GCN-NEXT:    v_mov_b32_e32 v61, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:  .LBB22_2: ; %Flow
+; GCN-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v42, v63
+; GCN-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_addc_u32_e32 v24, vcc, 0, v24, vcc
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_addc_u32_e32 v26, vcc, 0, v26, vcc
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_addc_u32_e32 v28, vcc, 0, v28, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:  .LBB22_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v52
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v39
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v37
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v49
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v48
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v38
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v36
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v34
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v32
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 56, v0
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v61
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v59
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v57
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v47
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v45
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v43
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v42
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v44
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x5c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v46
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v56
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v58
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x68, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v60
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x6c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v62
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v45, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v47, v59, v58
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v45, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14i64_to_v56f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB22_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB22_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14i64_to_v56f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB22_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB22_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v46, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v45, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v44, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v43, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v42, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v41, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v40, v7, s4
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v8, v55, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v54, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v53, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v52, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v51, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v48, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v39, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v38, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v37, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v35, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v34, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v33, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v32, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v31, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v30, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v29, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14i64_to_v56f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB22_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB22_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v71, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v69, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v67, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v66, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v65, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v64, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v55, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v54, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v53, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v52, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v37, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v36, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v35, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v34, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v33, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v32, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v31, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v30, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v29, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v28, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <14 x i64> %a, splat (i64 3)
+  %a2 = bitcast <14 x i64> %a1 to <56 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x i64> %a to <56 x half>
+  br label %end
+
+end:
+  %phi = phi <56 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x half> %phi
+}
+
+define <14 x i64> @bitcast_v56f16_to_v14i64(<56 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56f16_to_v14i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v6
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v9
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v8
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v11
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v55
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v53
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v54
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v51
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v49
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v36
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v35
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v34
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v31
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v0, v46, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v44, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v2, v42, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v41
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v51
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v28, v14
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v28, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v55, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v52, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v50, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v38, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v36, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v34, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v33, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v37, v25
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v26
+; GCN-NEXT:    v_or_b32_e32 v27, v49, v27
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:  .LBB23_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v44
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v42
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v41
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v55
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v52
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v50
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v48
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v38
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v36
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v17, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v18, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v19, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v20, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v21, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v22, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v23, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v25, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v27, v49, v48
+; GCN-NEXT:  .LBB23_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56f16_to_v14i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v27
+; VI-NEXT:    v_mov_b32_e32 v33, v26
+; VI-NEXT:    v_mov_b32_e32 v34, v25
+; VI-NEXT:    v_mov_b32_e32 v35, v24
+; VI-NEXT:    v_mov_b32_e32 v36, v23
+; VI-NEXT:    v_mov_b32_e32 v37, v22
+; VI-NEXT:    v_mov_b32_e32 v38, v21
+; VI-NEXT:    v_mov_b32_e32 v39, v20
+; VI-NEXT:    v_mov_b32_e32 v48, v19
+; VI-NEXT:    v_mov_b32_e32 v49, v18
+; VI-NEXT:    v_mov_b32_e32 v50, v17
+; VI-NEXT:    v_mov_b32_e32 v51, v16
+; VI-NEXT:    v_mov_b32_e32 v52, v15
+; VI-NEXT:    v_mov_b32_e32 v53, v14
+; VI-NEXT:    v_mov_b32_e32 v54, v13
+; VI-NEXT:    v_mov_b32_e32 v55, v12
+; VI-NEXT:    v_mov_b32_e32 v40, v11
+; VI-NEXT:    v_mov_b32_e32 v41, v10
+; VI-NEXT:    v_mov_b32_e32 v42, v9
+; VI-NEXT:    v_mov_b32_e32 v43, v8
+; VI-NEXT:    v_mov_b32_e32 v44, v7
+; VI-NEXT:    v_mov_b32_e32 v45, v6
+; VI-NEXT:    v_mov_b32_e32 v46, v5
+; VI-NEXT:    v_mov_b32_e32 v47, v4
+; VI-NEXT:    v_mov_b32_e32 v56, v3
+; VI-NEXT:    v_mov_b32_e32 v57, v2
+; VI-NEXT:    v_mov_b32_e32 v58, v1
+; VI-NEXT:    v_mov_b32_e32 v59, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v27, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v27, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v27, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v27, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v27, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v27, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v27, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v27, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v27, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v27, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v27, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v27, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v27, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v27, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v27, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v27, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v27, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v27, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v27, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v27, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v27, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v27, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v27, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v27, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v27, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v27, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB23_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v27, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v59, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v59
+; VI-NEXT:    v_add_f16_sdwa v2, v58, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v58
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v57, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v56, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v47, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v46, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v45, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v44, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v43, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v41, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v40, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v55, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v54, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v53, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v52, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v51, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v50, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v49, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v48, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v39, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v38, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v37, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v36, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v35, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v24, v25, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v34, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:    v_add_f16_sdwa v26, v33, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v26, v28, v26
+; VI-NEXT:    v_add_f16_sdwa v27, v32, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:  .LBB23_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56f16_to_v14i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v59, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v37, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v38, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v9
+; GFX9-NEXT:    v_mov_b32_e32 v43, v8
+; GFX9-NEXT:    v_mov_b32_e32 v44, v7
+; GFX9-NEXT:    v_mov_b32_e32 v45, v6
+; GFX9-NEXT:    v_mov_b32_e32 v46, v5
+; GFX9-NEXT:    v_mov_b32_e32 v47, v4
+; GFX9-NEXT:    v_mov_b32_e32 v56, v3
+; GFX9-NEXT:    v_mov_b32_e32 v57, v2
+; GFX9-NEXT:    v_mov_b32_e32 v58, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v59
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v37
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:  .LBB23_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56f16_to_v14i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    v_perm_b32 v0, v69, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v71, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v68, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v67, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v66, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v65, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v64, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v55, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v54, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v53, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v52, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v51, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v50, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v49, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v48, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v39, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v38, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v37, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v36, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v35, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v34, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v33, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v32, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v31, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v30, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v29, v27, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <56 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <56 x half> %a1 to <14 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x half> %a to <14 x i64>
+  br label %end
+
+end:
+  %phi = phi <14 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x i64> %phi
+}
+
+define <56 x i16> @bitcast_v14f64_to_v56i16(<14 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f64_to_v56i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v29
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v29, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v6, v5, 16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_alignbit_b32 v45, v4, v3, 16
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_alignbit_b32 v46, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_add_f64 v[25:26], v[25:26], 1.0
+; GCN-NEXT:    v_add_f64 v[27:28], v[27:28], 1.0
+; GCN-NEXT:    v_alignbit_b32 v29, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v31, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v6, v5, 16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_alignbit_b32 v45, v4, v3, 16
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_alignbit_b32 v46, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v57, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_and_b32_e32 v58, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 12, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v46, v57, v46
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v56, v58, v56
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 20, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v45
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 24, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v47
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 28, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 32, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v44
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 36, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 40, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v43
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 52, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 56, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v40
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 0x44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x4c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x50, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x54, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x58, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x5c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x68, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 0x6c, v0
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v30
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v49
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v29
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v39
+; GCN-NEXT:    buffer_store_dword v46, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f64_to_v56i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB24_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB24_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f64_to_v56i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB24_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB24_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v46, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v45, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v44, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v43, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v42, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v41, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v40, v7, s4
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v8, v55, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v54, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v53, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v52, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v51, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v48, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v39, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v38, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v37, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v35, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v34, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v33, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v32, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v31, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v30, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v29, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f64_to_v56i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB24_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB24_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v71, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v69, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v67, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v66, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v65, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v64, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v55, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v54, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v53, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v52, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v37, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v36, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v35, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v34, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v33, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v32, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v31, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v30, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v29, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v28, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <14 x double> %a1 to <56 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x double> %a to <56 x i16>
+  br label %end
+
+end:
+  %phi = phi <56 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x i16> %phi
+}
+
+define <14 x double> @bitcast_v56i16_to_v14f64(<56 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56i16_to_v14f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v34, v26
+; GCN-NEXT:    v_mov_b32_e32 v35, v24
+; GCN-NEXT:    v_mov_b32_e32 v36, v22
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:60
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:52
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:100
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v26
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v41
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v42
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v63
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v62
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v61
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v60
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v59
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v58
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v56
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v47
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v36
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v46
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v35
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v13
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v45
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v44
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v43
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v33
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v57
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v28
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:  .LBB25_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v34
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v57
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_or_b32_e32 v0, v41, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v42, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v63, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v62, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v61, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v60, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v59, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v58, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v56, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v47, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v46, v11
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v28, v12
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v28, v13
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v28, v14
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v28, v15
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v28, v16
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v28, v17
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v28, v18
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v28, v19
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v28, v20
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v28, v21
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v28, v22
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v28, v23
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v28, v24
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v28, v25
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v28, v26
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v28, v27
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 0x30000, v27
+; GCN-NEXT:  .LBB25_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56i16_to_v14f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v27
+; VI-NEXT:    v_mov_b32_e32 v33, v26
+; VI-NEXT:    v_mov_b32_e32 v34, v25
+; VI-NEXT:    v_mov_b32_e32 v35, v24
+; VI-NEXT:    v_mov_b32_e32 v36, v23
+; VI-NEXT:    v_mov_b32_e32 v37, v22
+; VI-NEXT:    v_mov_b32_e32 v38, v21
+; VI-NEXT:    v_mov_b32_e32 v39, v20
+; VI-NEXT:    v_mov_b32_e32 v48, v19
+; VI-NEXT:    v_mov_b32_e32 v49, v18
+; VI-NEXT:    v_mov_b32_e32 v50, v17
+; VI-NEXT:    v_mov_b32_e32 v51, v16
+; VI-NEXT:    v_mov_b32_e32 v52, v15
+; VI-NEXT:    v_mov_b32_e32 v53, v14
+; VI-NEXT:    v_mov_b32_e32 v54, v13
+; VI-NEXT:    v_mov_b32_e32 v55, v12
+; VI-NEXT:    v_mov_b32_e32 v40, v11
+; VI-NEXT:    v_mov_b32_e32 v41, v10
+; VI-NEXT:    v_mov_b32_e32 v42, v9
+; VI-NEXT:    v_mov_b32_e32 v43, v8
+; VI-NEXT:    v_mov_b32_e32 v44, v7
+; VI-NEXT:    v_mov_b32_e32 v45, v6
+; VI-NEXT:    v_mov_b32_e32 v46, v5
+; VI-NEXT:    v_mov_b32_e32 v47, v4
+; VI-NEXT:    v_mov_b32_e32 v56, v3
+; VI-NEXT:    v_mov_b32_e32 v57, v2
+; VI-NEXT:    v_mov_b32_e32 v58, v1
+; VI-NEXT:    v_mov_b32_e32 v59, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v27, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v27, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v27, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v27, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v27, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v27, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v27, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v27, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v27, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v27, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v27, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v27, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v27, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v27, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v27, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v27, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v27, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v27, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v27, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v27, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v27, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v27, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v27, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v27, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v27, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v27, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB25_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v27, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v59
+; VI-NEXT:    v_add_u16_sdwa v1, v59, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v2, 3, v58
+; VI-NEXT:    v_add_u16_sdwa v3, v58, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v3, v57, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v4, v56, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v5, v47, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v6, v46, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v7, v45, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v8, v44, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v9, v43, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v10, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v11, v41, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v12, v40, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v13, v55, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v14, v54, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v15, v53, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v16, v52, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v17, v51, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v18, v50, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v19, v49, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v20, v48, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v21, v39, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v22, v38, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v23, v37, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v24, v36, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v25, v35, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v25
+; VI-NEXT:    v_add_u16_e32 v25, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v26, v34, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v25, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v28, v33, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v26, v26, v28
+; VI-NEXT:    v_add_u16_e32 v28, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v27, v32, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:  .LBB25_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56i16_to_v14f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v59, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v37, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v38, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v9
+; GFX9-NEXT:    v_mov_b32_e32 v43, v8
+; GFX9-NEXT:    v_mov_b32_e32 v44, v7
+; GFX9-NEXT:    v_mov_b32_e32 v45, v6
+; GFX9-NEXT:    v_mov_b32_e32 v46, v5
+; GFX9-NEXT:    v_mov_b32_e32 v47, v4
+; GFX9-NEXT:    v_mov_b32_e32 v56, v3
+; GFX9-NEXT:    v_mov_b32_e32 v57, v2
+; GFX9-NEXT:    v_mov_b32_e32 v58, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v59
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v37
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:  .LBB25_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56i16_to_v14f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    v_perm_b32 v0, v69, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v71, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v68, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v67, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v66, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v65, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v64, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v55, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v54, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v53, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v52, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v51, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v50, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v49, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v48, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v39, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v38, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v37, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v36, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v35, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v34, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v33, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v32, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v31, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v30, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v29, v27, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <56 x i16> %a, splat (i16 3)
+  %a2 = bitcast <56 x i16> %a1 to <14 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x i16> %a to <14 x double>
+  br label %end
+
+end:
+  %phi = phi <14 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x double> %phi
+}
+
+define <56 x half> @bitcast_v14f64_to_v56f16(<14 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v14f64_to_v56f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v29
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; kill: killed $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v40
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v44
+; GCN-NEXT:    v_mov_b32_e32 v44, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v46
+; GCN-NEXT:    v_mov_b32_e32 v46, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v56
+; GCN-NEXT:    v_mov_b32_e32 v56, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v58
+; GCN-NEXT:    v_mov_b32_e32 v58, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v60
+; GCN-NEXT:    v_mov_b32_e32 v60, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v62
+; GCN-NEXT:    v_mov_b32_e32 v62, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v43
+; GCN-NEXT:    v_mov_b32_e32 v43, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v45
+; GCN-NEXT:    v_mov_b32_e32 v45, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v47
+; GCN-NEXT:    v_mov_b32_e32 v47, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v57
+; GCN-NEXT:    v_mov_b32_e32 v57, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v59
+; GCN-NEXT:    v_mov_b32_e32 v59, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v61
+; GCN-NEXT:    v_mov_b32_e32 v61, v13
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v42, v63
+; GCN-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_add_f64 v[25:26], v[25:26], 1.0
+; GCN-NEXT:    v_add_f64 v[27:28], v[27:28], 1.0
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v7
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v39
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v37
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v49
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v48
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v38
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v36
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v34
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v32
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 56, v0
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v61
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v59
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v57
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v47
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v45
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v43
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v42
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v44
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x5c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v46
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v56
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v58
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x68, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v60
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x6c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v62
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v45, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v47, v59, v58
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v45, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v14f64_to_v56f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB26_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; VI-NEXT:  .LBB26_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v14f64_to_v56f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB26_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v0
+; GFX9-NEXT:  .LBB26_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v47, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v46, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v45, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v44, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v43, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v42, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v41, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v40, v7, s4
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v8, v55, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v54, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v53, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v52, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v51, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v49, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v48, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v39, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v38, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v37, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v35, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v34, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v33, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v32, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v31, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v30, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v29, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v14f64_to_v56f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    ; implicit-def: $vgpr29
+; GFX11-NEXT:    ; implicit-def: $vgpr28
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB26_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-NEXT:  .LBB26_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v71, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v69, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v68, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v67, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v66, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v65, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v64, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v55, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v54, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v53, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v52, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v49, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v48, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v39, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v38, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v37, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v36, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v35, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v34, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v33, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v32, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v31, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v30, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v29, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v28, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <14 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <14 x double> %a1 to <56 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <14 x double> %a to <56 x half>
+  br label %end
+
+end:
+  %phi = phi <56 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x half> %phi
+}
+
+define <14 x double> @bitcast_v56f16_to_v14f64(<56 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56f16_to_v14f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:100
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v6
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v9
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v8
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v11
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v40
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v55
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v53
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v54
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v51
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v49
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v36
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v35
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v34
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v31
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v0, v46, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v44, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v2, v42, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v41
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v51
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v28, v14
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v28, v15
+; GCN-NEXT:    v_or_b32_e32 v16, v55, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v52, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v50, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v48, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v38, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v36, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v34, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v33, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v37, v25
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v26
+; GCN-NEXT:    v_or_b32_e32 v27, v49, v27
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; kill: killed $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v44
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v42
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v41
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v55
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v52
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v50
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v48
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v38
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v36
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v17, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v18, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v19, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v20, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v21, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v22, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v23, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v24, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v25, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v26, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v27, v49, v48
+; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56f16_to_v14f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v27
+; VI-NEXT:    v_mov_b32_e32 v33, v26
+; VI-NEXT:    v_mov_b32_e32 v34, v25
+; VI-NEXT:    v_mov_b32_e32 v35, v24
+; VI-NEXT:    v_mov_b32_e32 v36, v23
+; VI-NEXT:    v_mov_b32_e32 v37, v22
+; VI-NEXT:    v_mov_b32_e32 v38, v21
+; VI-NEXT:    v_mov_b32_e32 v39, v20
+; VI-NEXT:    v_mov_b32_e32 v48, v19
+; VI-NEXT:    v_mov_b32_e32 v49, v18
+; VI-NEXT:    v_mov_b32_e32 v50, v17
+; VI-NEXT:    v_mov_b32_e32 v51, v16
+; VI-NEXT:    v_mov_b32_e32 v52, v15
+; VI-NEXT:    v_mov_b32_e32 v53, v14
+; VI-NEXT:    v_mov_b32_e32 v54, v13
+; VI-NEXT:    v_mov_b32_e32 v55, v12
+; VI-NEXT:    v_mov_b32_e32 v40, v11
+; VI-NEXT:    v_mov_b32_e32 v41, v10
+; VI-NEXT:    v_mov_b32_e32 v42, v9
+; VI-NEXT:    v_mov_b32_e32 v43, v8
+; VI-NEXT:    v_mov_b32_e32 v44, v7
+; VI-NEXT:    v_mov_b32_e32 v45, v6
+; VI-NEXT:    v_mov_b32_e32 v46, v5
+; VI-NEXT:    v_mov_b32_e32 v47, v4
+; VI-NEXT:    v_mov_b32_e32 v56, v3
+; VI-NEXT:    v_mov_b32_e32 v57, v2
+; VI-NEXT:    v_mov_b32_e32 v58, v1
+; VI-NEXT:    v_mov_b32_e32 v59, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v27, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v27, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v27, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v27, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v27, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v27, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v27, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v27, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v27, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v27, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v27, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v27, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v27, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v27, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v27, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v27, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v27, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v27, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v27, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v27, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v27, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v27, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v27, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v27, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v27, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v27, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB27_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v27, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v59, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v59
+; VI-NEXT:    v_add_f16_sdwa v2, v58, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v58
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v57, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v56, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v47, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v46, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v45, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v44, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v43, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v41, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v40, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v55, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v54, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v53, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v52, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v51, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v50, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v49, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v48, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v39, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v38, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v37, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v36, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v35, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v24, v25, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v34, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:    v_add_f16_sdwa v26, v33, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v26, v28, v26
+; VI-NEXT:    v_add_f16_sdwa v27, v32, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:  .LBB27_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56f16_to_v14f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v59, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v37, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v38, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v37
+; GFX9-NEXT:    v_mov_b32_e32 v39, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v38
+; GFX9-NEXT:    v_mov_b32_e32 v48, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v9
+; GFX9-NEXT:    v_mov_b32_e32 v43, v8
+; GFX9-NEXT:    v_mov_b32_e32 v44, v7
+; GFX9-NEXT:    v_mov_b32_e32 v45, v6
+; GFX9-NEXT:    v_mov_b32_e32 v46, v5
+; GFX9-NEXT:    v_mov_b32_e32 v47, v4
+; GFX9-NEXT:    v_mov_b32_e32 v56, v3
+; GFX9-NEXT:    v_mov_b32_e32 v57, v2
+; GFX9-NEXT:    v_mov_b32_e32 v58, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v60, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v61, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v59
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v39
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v37
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; kill: killed $vgpr28
+; GFX9-NEXT:  .LBB27_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v36, v59, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v35, v58, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v57, s6
+; GFX9-NEXT:    v_perm_b32 v3, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v47, s6
+; GFX9-NEXT:    v_perm_b32 v5, v61, v46, s6
+; GFX9-NEXT:    v_perm_b32 v6, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v7, v60, v44, s6
+; GFX9-NEXT:    v_perm_b32 v8, v63, v43, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v38, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v37, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56f16_to_v14f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v28
+; GFX11-NEXT:    v_perm_b32 v0, v69, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v70, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v71, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v68, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v67, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v66, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v65, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v64, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v55, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v54, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v53, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v52, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v51, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v50, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v49, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v48, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v39, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v38, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v37, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v36, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v35, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v34, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v33, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v32, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v31, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v30, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v29, v27, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <56 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <56 x half> %a1 to <14 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x half> %a to <14 x double>
+  br label %end
+
+end:
+  %phi = phi <14 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <14 x double> %phi
+}
+
+define <56 x half> @bitcast_v56i16_to_v56f16(<56 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56i16_to_v56f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:100
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:92
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:72
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:104
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:8
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; kill: killed $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v47
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v39
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v49
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v52
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v53
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v56
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v57
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v58
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v59
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v60
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v61
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v62
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:  .LBB28_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
+; GCN-NEXT:    v_add_i32_e32 v63, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v62, vcc, 3, v62
+; GCN-NEXT:    v_add_i32_e32 v61, vcc, 3, v61
+; GCN-NEXT:    v_add_i32_e32 v60, vcc, 3, v60
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 3, v59
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 3, v58
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 3, v57
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 3, v56
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 3, v54
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 3, v53
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 3, v52
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 3, v51
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 3, v50
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 3, v49
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 3, v48
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 3, v47
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v6
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v10
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v47
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v39
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v49
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v52
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v53
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v56
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v57
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v58
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v59
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v60
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v61
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v62
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v42
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v43
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v44
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v3
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v45
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v41
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v40
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 24, v0
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 28, v0
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 32, v0
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 36, v0
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 40, v0
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 44, v0
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 48, v0
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 52, v0
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 56, v0
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 60, v0
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 64, v0
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x44, v0
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x68, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x6c, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_or_b32_e32 v45, v47, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v57, v56
+; GCN-NEXT:    v_or_b32_e32 v47, v59, v58
+; GCN-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v2, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v4, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v45, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56i16_to_v56f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_add_u16_e32 v56, 3, v56
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v47, 3, v47
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v46, 3, v46
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v45, 3, v45
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v44, 3, v44
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v43, 3, v43
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v42, 3, v42
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v41, 3, v41
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v40, 3, v40
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_e32 v55, 3, v55
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_add_u16_e32 v54, 3, v54
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_add_u16_e32 v53, 3, v53
+; VI-NEXT:    v_add_u16_e32 v12, 3, v12
+; VI-NEXT:    v_add_u16_e32 v52, 3, v52
+; VI-NEXT:    v_add_u16_e32 v13, 3, v13
+; VI-NEXT:    v_add_u16_e32 v51, 3, v51
+; VI-NEXT:    v_add_u16_e32 v14, 3, v14
+; VI-NEXT:    v_add_u16_e32 v50, 3, v50
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    v_add_u16_e32 v49, 3, v49
+; VI-NEXT:    v_add_u16_e32 v16, 3, v16
+; VI-NEXT:    v_add_u16_e32 v48, 3, v48
+; VI-NEXT:    v_add_u16_e32 v17, 3, v17
+; VI-NEXT:    v_add_u16_e32 v39, 3, v39
+; VI-NEXT:    v_add_u16_e32 v18, 3, v18
+; VI-NEXT:    v_add_u16_e32 v38, 3, v38
+; VI-NEXT:    v_add_u16_e32 v19, 3, v19
+; VI-NEXT:    v_add_u16_e32 v37, 3, v37
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_add_u16_e32 v36, 3, v36
+; VI-NEXT:    v_add_u16_e32 v21, 3, v21
+; VI-NEXT:    v_add_u16_e32 v35, 3, v35
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v34, 3, v34
+; VI-NEXT:    v_add_u16_e32 v23, 3, v23
+; VI-NEXT:    v_add_u16_e32 v33, 3, v33
+; VI-NEXT:    v_add_u16_e32 v24, 3, v24
+; VI-NEXT:    v_add_u16_e32 v32, 3, v32
+; VI-NEXT:    v_add_u16_e32 v25, 3, v25
+; VI-NEXT:    v_add_u16_e32 v31, 3, v31
+; VI-NEXT:    v_add_u16_e32 v26, 3, v26
+; VI-NEXT:    v_add_u16_e32 v30, 3, v30
+; VI-NEXT:    v_add_u16_e32 v27, 3, v27
+; VI-NEXT:    v_add_u16_e32 v29, 3, v29
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v56
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v47
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v46
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v45
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v44
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v43
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v42
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v41
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v40
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v55
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v54
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v53
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56i16_to_v56f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v27, v56, v27, s6
+; GFX9-NEXT:    v_perm_b32 v26, v47, v26, s6
+; GFX9-NEXT:    v_perm_b32 v25, v46, v25, s6
+; GFX9-NEXT:    v_perm_b32 v24, v45, v24, s6
+; GFX9-NEXT:    v_perm_b32 v23, v44, v23, s6
+; GFX9-NEXT:    v_perm_b32 v22, v43, v22, s6
+; GFX9-NEXT:    v_perm_b32 v21, v42, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v41, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v40, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v55, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v54, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v53, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v51, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v49, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v39, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v37, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v35, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v31, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v30, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v29, v0, s6
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v27
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v19, v40, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v41, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v42, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v43, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v44, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v45, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v46, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v47, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v56, v27, s4
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v0, v29, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v30, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v31, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v33, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v35, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v37, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v39, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v49, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v51, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v53, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v54, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v55, v18, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56i16_to_v56f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v27, v80, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v71, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v70, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v69, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v68, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v67, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v66, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v65, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v64, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v55, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v54, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v53, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v51, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v49, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v39, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v37, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v35, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v29, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v30, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v31, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v33, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v27
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v29, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v30, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v31, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v33, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v35, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v37, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v39, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v49, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v51, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v53, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v54, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v55, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v64, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v65, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v66, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v67, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v68, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v69, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v70, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v71, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v80, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <56 x i16> %a, splat (i16 3)
+  %a2 = bitcast <56 x i16> %a1 to <56 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x i16> %a to <56 x half>
+  br label %end
+
+end:
+  %phi = phi <56 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x half> %phi
+}
+
+define <56 x i16> @bitcast_v56f16_to_v56i16(<56 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v56f16_to_v56i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:96
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:88
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:80
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:104
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v4
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v7
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v8
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v11
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v12
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v15
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v16
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v24
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v28
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v29
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v30
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v52
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v49
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v48
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v35
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v34
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v38
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v61
+; GCN-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v62, v40
+; GCN-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v13
+; GCN-NEXT:    v_mov_b32_e32 v11, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v8
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v61
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v11
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v15
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v16
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v17
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_add_f32_e32 v41, 0x38000000, v56
+; GCN-NEXT:    v_add_f32_e32 v46, 0x38000000, v46
+; GCN-NEXT:    v_add_f32_e32 v47, 0x38000000, v47
+; GCN-NEXT:    v_add_f32_e32 v44, 0x38000000, v44
+; GCN-NEXT:    v_add_f32_e32 v45, 0x38000000, v45
+; GCN-NEXT:    v_add_f32_e32 v42, 0x38000000, v42
+; GCN-NEXT:    v_add_f32_e32 v43, 0x38000000, v43
+; GCN-NEXT:    v_add_f32_e32 v40, 0x38000000, v40
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v54, 0x38000000, v60
+; GCN-NEXT:    v_add_f32_e32 v55, 0x38000000, v55
+; GCN-NEXT:    v_add_f32_e32 v52, 0x38000000, v52
+; GCN-NEXT:    v_add_f32_e32 v53, 0x38000000, v53
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v56, 0x38000000, v58
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v58, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v60, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v61, v32
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v63, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v48
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v62
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v61
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v33
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v22
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_or_b32_e32 v61, v13, v3
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v41, v2, v15
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v57
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v57, v62
+; GCN-NEXT:    v_mov_b32_e32 v62, v18
+; GCN-NEXT:    v_or_b32_e32 v15, v4, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v62
+; GCN-NEXT:    v_or_b32_e32 v16, v8, v17
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v19
+; GCN-NEXT:    v_or_b32_e32 v17, v9, v3
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v12
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v12, v31
+; GCN-NEXT:    v_or_b32_e32 v18, v10, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v19, v11, v4
+; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v8
+; GCN-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v2
+; GCN-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v46, v46, v24
+; GCN-NEXT:    v_or_b32_e32 v44, v44, v47
+; GCN-NEXT:    v_or_b32_e32 v3, v42, v45
+; GCN-NEXT:    v_or_b32_e32 v11, v40, v43
+; GCN-NEXT:    v_or_b32_e32 v2, v54, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v52, v55
+; GCN-NEXT:    v_or_b32_e32 v8, v50, v53
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v51
+; GCN-NEXT:    v_or_b32_e32 v22, v56, v49
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v39
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v30
+; GCN-NEXT:    v_or_b32_e32 v54, v59, v28
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v38, v23, v58
+; GCN-NEXT:    v_or_b32_e32 v31, v34, v60
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v4
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v9
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v13
+; GCN-NEXT:    v_or_b32_e32 v1, v48, v26
+; GCN-NEXT:    v_alignbit_b32 v56, v1, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v47, v6, v47, 16
+; GCN-NEXT:    v_alignbit_b32 v45, v7, v45, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v5, v43, 16
+; GCN-NEXT:    v_alignbit_b32 v13, v14, v37, 16
+; GCN-NEXT:    v_alignbit_b32 v4, v21, v55, 16
+; GCN-NEXT:    v_alignbit_b32 v26, v20, v53, 16
+; GCN-NEXT:    v_alignbit_b32 v9, v19, v51, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v18, v49, 16
+; GCN-NEXT:    v_alignbit_b32 v23, v17, v39, 16
+; GCN-NEXT:    v_alignbit_b32 v30, v16, v30, 16
+; GCN-NEXT:    v_alignbit_b32 v28, v15, v28, 16
+; GCN-NEXT:    v_alignbit_b32 v63, v41, v58, 16
+; GCN-NEXT:    v_alignbit_b32 v59, v61, v60, 16
+; GCN-NEXT:    v_mov_b32_e32 v60, v61
+; GCN-NEXT:    v_mov_b32_e32 v61, v31
+; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v35, 0xffff, v46
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v56
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v39, 0xffff, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v47
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v42, v35, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v37, v1, v37
+; GCN-NEXT:    v_mov_b32_e32 v55, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v43, v39, v49
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v35, v6, v50
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v51
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v52
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v53
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v4, v10, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v62
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GCN-NEXT:    v_or_b32_e32 v9, v25, v9
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v24
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v44
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v48, 0xffff, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v23, v29, v23
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v45
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v38, 0xffff, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v63
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v30
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v41, 0xffff, v41
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v46
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 0x64, v0
+; GCN-NEXT:    v_and_b32_e32 v54, 0xffff, v61
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v59
+; GCN-NEXT:    v_or_b32_e32 v28, v48, v28
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x68, v0
+; GCN-NEXT:    v_and_b32_e32 v59, 0xffff, v60
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x6c, v0
+; GCN-NEXT:    v_or_b32_e32 v38, v38, v47
+; GCN-NEXT:    v_or_b32_e32 v41, v41, v56
+; GCN-NEXT:    v_or_b32_e32 v54, v54, v58
+; GCN-NEXT:    v_or_b32_e32 v47, v59, v57
+; GCN-NEXT:    buffer_store_dword v42, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v35, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v1, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v24, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v30, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v41, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v47, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v56f16_to_v56i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_add_f16_e32 v56, 0x200, v56
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_e32 v47, 0x200, v47
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_add_f16_e32 v46, 0x200, v46
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_add_f16_e32 v45, 0x200, v45
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_add_f16_e32 v44, 0x200, v44
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_add_f16_e32 v43, 0x200, v43
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_add_f16_e32 v42, 0x200, v42
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_add_f16_e32 v41, 0x200, v41
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_add_f16_e32 v40, 0x200, v40
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_add_f16_e32 v55, 0x200, v55
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_add_f16_e32 v54, 0x200, v54
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_add_f16_e32 v53, 0x200, v53
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_add_f16_e32 v52, 0x200, v52
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_add_f16_e32 v51, 0x200, v51
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v14
+; VI-NEXT:    v_add_f16_e32 v50, 0x200, v50
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v15
+; VI-NEXT:    v_add_f16_e32 v49, 0x200, v49
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v16
+; VI-NEXT:    v_add_f16_e32 v48, 0x200, v48
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v17
+; VI-NEXT:    v_add_f16_e32 v39, 0x200, v39
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v18
+; VI-NEXT:    v_add_f16_e32 v38, 0x200, v38
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v19
+; VI-NEXT:    v_add_f16_e32 v37, 0x200, v37
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v20
+; VI-NEXT:    v_add_f16_e32 v36, 0x200, v36
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v21
+; VI-NEXT:    v_add_f16_e32 v35, 0x200, v35
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v22
+; VI-NEXT:    v_add_f16_e32 v34, 0x200, v34
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v23
+; VI-NEXT:    v_add_f16_e32 v33, 0x200, v33
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v24
+; VI-NEXT:    v_add_f16_e32 v32, 0x200, v32
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v25
+; VI-NEXT:    v_add_f16_e32 v31, 0x200, v31
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v26
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v30
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v27
+; VI-NEXT:    v_add_f16_e32 v29, 0x200, v29
+; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v56
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v47
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v46
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v45
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v44
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v43
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v42
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v41
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v40
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v55
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v54
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v53
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v56f16_to_v56i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v28
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v27, v56, v27, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v26, v47, v26, s6
+; GFX9-NEXT:    v_perm_b32 v25, v46, v25, s6
+; GFX9-NEXT:    v_perm_b32 v24, v45, v24, s6
+; GFX9-NEXT:    v_perm_b32 v23, v44, v23, s6
+; GFX9-NEXT:    v_perm_b32 v22, v43, v22, s6
+; GFX9-NEXT:    v_perm_b32 v21, v42, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v41, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v40, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v55, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v54, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v53, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v51, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v49, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v39, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v37, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v35, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v33, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v31, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v30, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v29, v0, s6
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v29, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v27
+; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v19, v40, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v41, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v42, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v43, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v44, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v45, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v46, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v47, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v56, v27, s4
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v0, v29, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v30, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v31, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v32, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v33, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v34, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v35, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v36, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v37, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v38, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v39, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v48, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v49, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v50, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v51, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v53, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v54, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v55, v18, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v56f16_to_v56i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v28
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v27, v80, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v71, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v70, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v69, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v68, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v67, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v66, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v65, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v64, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v55, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v54, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v53, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v51, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v49, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v39, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v37, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v35, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v29, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v30, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v31, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v33, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v29, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v27
+; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v29, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v30, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v31, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v32, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v33, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v34, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v35, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v37, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v38, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v39, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v48, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v49, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v50, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v51, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v53, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v54, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v55, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v64, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v65, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v66, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v67, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v68, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v69, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v70, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v71, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v80, v27, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <56 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <56 x half> %a1 to <56 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <56 x half> %a to <56 x i16>
+  br label %end
+
+end:
+  %phi = phi <56 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <56 x i16> %phi
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
index ca0c0bb0d4ca2..b60649cc23590 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
@@ -5,8 +5,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
 
-define <30 x i32> @bitcast_v15i64_to_v30i32(<15 x i64> %a, i32 %b) {
-; GCN-LABEL: bitcast_v15i64_to_v30i32:
+define <30 x float> @bitcast_v30i32_to_v30f32(<30 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30i32_to_v30f32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
@@ -15,204 +15,6 @@ define <30 x i32> @bitcast_v15i64_to_v30i32(<15 x i64> %a, i32 %b) {
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %cmp.true
-; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
-; GCN-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
-; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
-; GCN-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
-; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
-; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
-; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
-; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
-; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
-; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
-; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
-; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:  .LBB0_2: ; %end
-; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_setpc_b64 s[30:31]
-;
-; VI-LABEL: bitcast_v15i64_to_v30i32:
-; VI:       ; %bb.0:
-; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB0_2
-; VI-NEXT:  ; %bb.1: ; %cmp.true
-; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
-; VI-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
-; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
-; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
-; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
-; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
-; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
-; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
-; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
-; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
-; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
-; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
-; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
-; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
-; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
-; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
-; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
-; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
-; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
-; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
-; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:  .LBB0_2: ; %end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX9-LABEL: bitcast_v15i64_to_v30i32:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB0_2
-; GFX9-NEXT:  ; %bb.1: ; %cmp.true
-; GFX9-NEXT:    v_add_co_u32_e32 v28, vcc, 3, v28
-; GFX9-NEXT:    v_addc_co_u32_e32 v29, vcc, 0, v29, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
-; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
-; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
-; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
-; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
-; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
-; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
-; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT:  .LBB0_2: ; %end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: bitcast_v15i64_to_v30i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
-; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB0_2
-; GFX11-NEXT:  ; %bb.1: ; %cmp.true
-; GFX11-NEXT:    v_add_co_u32 v28, vcc_lo, v28, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v29, null, 0, v29, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT:  .LBB0_2: ; %end
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
-  %cmp = icmp eq i32 %b, 0
-  br i1 %cmp, label %cmp.true, label %cmp.false
-
-cmp.true:
-  %a1 = add <15 x i64> %a, splat (i64 3)
-  %a2 = bitcast <15 x i64> %a1 to <30 x i32>
-  br label %end
-
-cmp.false:
-  %a3 = bitcast <15 x i64> %a to <30 x i32>
-  br label %end
-
-end:
-  %phi = phi <30 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <30 x i32> %phi
-}
-
-define <15 x i64> @bitcast_v30i32_to_v15i64(<30 x i32> %a, i32 %b) {
-; GCN-LABEL: bitcast_v30i32_to_v15i64:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execz .LBB1_2
-; GCN-NEXT:  ; %bb.1: ; %cmp.true
 ; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
 ; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
 ; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
@@ -243,18 +45,18 @@ define <15 x i64> @bitcast_v30i32_to_v15i64(<30 x i32> %a, i32 %b) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
-; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:  .LBB0_2: ; %end
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; VI-LABEL: bitcast_v30i32_to_v15i64:
+; VI-LABEL: bitcast_v30i32_to_v30f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:    s_cbranch_execz .LBB0_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.true
 ; VI-NEXT:    v_add_u32_e32 v29, vcc, 3, v29
 ; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
@@ -286,18 +88,18 @@ define <15 x i64> @bitcast_v30i32_to_v15i64(<30 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:  .LBB0_2: ; %end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: bitcast_v30i32_to_v15i64:
+; GFX9-LABEL: bitcast_v30i32_to_v30f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX9-NEXT:    v_add_u32_e32 v29, 3, v29
 ; GFX9-NEXT:    v_add_u32_e32 v28, 3, v28
@@ -329,11 +131,11 @@ define <15 x i64> @bitcast_v30i32_to_v15i64(<30 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:  .LBB0_2: ; %end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: bitcast_v30i32_to_v15i64:
+; GFX11-LABEL: bitcast_v30i32_to_v30f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, exec_lo
@@ -341,7 +143,7 @@ define <15 x i64> @bitcast_v30i32_to_v15i64(<30 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX11-NEXT:  ; %bb.1: ; %cmp.true
 ; GFX11-NEXT:    v_add_nc_u32_e32 v29, 3, v29
 ; GFX11-NEXT:    v_add_nc_u32_e32 v28, 3, v28
@@ -373,7 +175,7 @@ define <15 x i64> @bitcast_v30i32_to_v15i64(<30 x i32> %a, i32 %b) {
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
-; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:  .LBB0_2: ; %end
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %b, 0
@@ -381,14 +183,24847 @@ define <15 x i64> @bitcast_v30i32_to_v15i64(<30 x i32> %a, i32 %b) {
 
 cmp.true:
   %a1 = add <30 x i32> %a, splat (i32 3)
-  %a2 = bitcast <30 x i32> %a1 to <15 x i64>
+  %a2 = bitcast <30 x i32> %a1 to <30 x float>
   br label %end
 
 cmp.false:
-  %a3 = bitcast <30 x i32> %a to <15 x i64>
+  %a3 = bitcast <30 x i32> %a to <30 x float>
   br label %end
 
 end:
-  %phi = phi <15 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
-  ret <15 x i64> %phi
+  %phi = phi <30 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x float> %phi
+}
+
+define <30 x i32> @bitcast_v30f32_to_v30i32(<30 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30f32_to_v30i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GCN-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30f32_to_v30i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB1_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; VI-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB1_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30f32_to_v30i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB1_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GFX9-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB1_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30f32_to_v30i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB1_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v29, 1.0, v29 :: v_dual_add_f32 v28, 1.0, v28
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB1_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <30 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <30 x float> %a1 to <30 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x float> %a to <30 x i32>
+  br label %end
+
+end:
+  %phi = phi <30 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x i32> %phi
+}
+
+define <15 x i64> @bitcast_v30i32_to_v15i64(<30 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30i32_to_v15i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB2_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:  .LBB2_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30i32_to_v15i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB2_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v29, vcc, 3, v29
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB2_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30i32_to_v15i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v29, 3, v29
+; GFX9-NEXT:    v_add_u32_e32 v28, 3, v28
+; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
+; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB2_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30i32_to_v15i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v29, 3, v29
+; GFX11-NEXT:    v_add_nc_u32_e32 v28, 3, v28
+; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
+; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB2_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <30 x i32> %a, splat (i32 3)
+  %a2 = bitcast <30 x i32> %a1 to <15 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x i32> %a to <15 x i64>
+  br label %end
+
+end:
+  %phi = phi <15 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x i64> %phi
+}
+
+define <30 x i32> @bitcast_v15i64_to_v30i32(<15 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15i64_to_v30i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB3_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB3_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15i64_to_v30i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB3_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB3_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15i64_to_v30i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB3_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v28, vcc, 3, v28
+; GFX9-NEXT:    v_addc_co_u32_e32 v29, vcc, 0, v29, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB3_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15i64_to_v30i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB3_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v28, vcc_lo, v28, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v29, null, 0, v29, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB3_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <15 x i64> %a, splat (i64 3)
+  %a2 = bitcast <15 x i64> %a1 to <30 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x i64> %a to <30 x i32>
+  br label %end
+
+end:
+  %phi = phi <30 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x i32> %phi
+}
+
+define <15 x double> @bitcast_v30i32_to_v15f64(<30 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30i32_to_v15f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB4_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:  .LBB4_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30i32_to_v15f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB4_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v29, vcc, 3, v29
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:  .LBB4_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30i32_to_v15f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB4_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v29, 3, v29
+; GFX9-NEXT:    v_add_u32_e32 v28, 3, v28
+; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
+; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:  .LBB4_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30i32_to_v15f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB4_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v29, 3, v29
+; GFX11-NEXT:    v_add_nc_u32_e32 v28, 3, v28
+; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
+; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:  .LBB4_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <30 x i32> %a, splat (i32 3)
+  %a2 = bitcast <30 x i32> %a1 to <15 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x i32> %a to <15 x double>
+  br label %end
+
+end:
+  %phi = phi <15 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x double> %phi
+}
+
+define <30 x i32> @bitcast_v15f64_to_v30i32(<15 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15f64_to_v30i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB5_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GCN-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB5_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15f64_to_v30i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB5_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB5_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15f64_to_v30i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB5_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB5_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15f64_to_v30i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB5_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB5_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <15 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <15 x double> %a1 to <30 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x double> %a to <30 x i32>
+  br label %end
+
+end:
+  %phi = phi <30 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x i32> %phi
+}
+
+define <60 x i16> @bitcast_v30i32_to_v60i16(<30 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30i32_to_v60i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v31, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v6, v5, 16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_alignbit_b32 v46, v4, v3, 16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_alignbit_b32 v56, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v2
+; GCN-NEXT:  .LBB6_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB6_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_alignbit_b32 v31, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v6, v5, 16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_alignbit_b32 v46, v4, v3, 16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_alignbit_b32 v56, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v2
+; GCN-NEXT:  .LBB6_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v56
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v60
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v56
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v46
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v59
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v44
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v58
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v57
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v47
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v45
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v43
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v40
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x64, v0
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x68, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x6c, v0
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x70, v0
+; GCN-NEXT:    v_and_b32_e32 v30, 0xffff, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30i32_to_v60i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB6_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB6_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v29, vcc, 3, v29
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB6_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; VI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; VI-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; VI-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30i32_to_v60i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB6_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB6_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v29, 3, v29
+; GFX9-NEXT:    v_add_u32_e32 v28, 3, v28
+; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
+; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB6_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v58, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v57, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v56, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v47, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v46, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v45, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v44, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v43, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v42, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v41, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v40, v11, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v12, v55, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v54, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v53, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v50, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v49, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v48, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v39, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v38, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v37, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v36, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v35, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v34, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v33, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v32, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v31, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30i32_to_v60i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    ; implicit-def: $vgpr83
+; GFX11-NEXT:    ; implicit-def: $vgpr82
+; GFX11-NEXT:    ; implicit-def: $vgpr81
+; GFX11-NEXT:    ; implicit-def: $vgpr80
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB6_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB6_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v29, 3, v29
+; GFX11-NEXT:    v_add_nc_u32_e32 v28, 3, v28
+; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
+; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB6_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v83, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v81, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v71, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v70, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v69, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v68, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v67, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v66, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v65, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v64, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v55, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v54, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v39, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v38, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v37, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v36, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v35, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v34, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v33, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v32, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v31, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v30, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <30 x i32> %a, splat (i32 3)
+  %a2 = bitcast <30 x i32> %a1 to <60 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x i32> %a to <60 x i16>
+  br label %end
+
+end:
+  %phi = phi <60 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x i16> %phi
+}
+
+define <30 x i32> @bitcast_v60i16_to_v30i32(<60 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60i16_to_v30i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:112
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:104
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(6) expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v42
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v41
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v63
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v62
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v61
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v60
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v59
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v58
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v57
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v47
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v15
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v16
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v44
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v46
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v45
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v43
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v56
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v30
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:  .LBB7_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v56
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v42, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v41, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v63, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v62, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v61, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v60, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v59, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v58, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v57, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v47, v12
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v30, v13
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v30, v14
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v30, v15
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v30, v16
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v17
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v30, v18
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v30, v19
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v30, v20
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v30, v21
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v30, v22
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v30, v23
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v30, v24
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v30, v25
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v30, v26
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v30, v27
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v28, v30, v28
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v29, v30, v29
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 0x30000, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x30000, v29
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60i16_to_v30i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v29
+; VI-NEXT:    v_mov_b32_e32 v33, v28
+; VI-NEXT:    v_mov_b32_e32 v34, v27
+; VI-NEXT:    v_mov_b32_e32 v35, v26
+; VI-NEXT:    v_mov_b32_e32 v36, v25
+; VI-NEXT:    v_mov_b32_e32 v37, v24
+; VI-NEXT:    v_mov_b32_e32 v38, v23
+; VI-NEXT:    v_mov_b32_e32 v39, v22
+; VI-NEXT:    v_mov_b32_e32 v48, v21
+; VI-NEXT:    v_mov_b32_e32 v49, v20
+; VI-NEXT:    v_mov_b32_e32 v50, v19
+; VI-NEXT:    v_mov_b32_e32 v51, v18
+; VI-NEXT:    v_mov_b32_e32 v52, v17
+; VI-NEXT:    v_mov_b32_e32 v53, v16
+; VI-NEXT:    v_mov_b32_e32 v54, v15
+; VI-NEXT:    v_mov_b32_e32 v55, v14
+; VI-NEXT:    v_mov_b32_e32 v40, v13
+; VI-NEXT:    v_mov_b32_e32 v41, v12
+; VI-NEXT:    v_mov_b32_e32 v42, v11
+; VI-NEXT:    v_mov_b32_e32 v43, v10
+; VI-NEXT:    v_mov_b32_e32 v44, v9
+; VI-NEXT:    v_mov_b32_e32 v45, v8
+; VI-NEXT:    v_mov_b32_e32 v46, v7
+; VI-NEXT:    v_mov_b32_e32 v47, v6
+; VI-NEXT:    v_mov_b32_e32 v56, v5
+; VI-NEXT:    v_mov_b32_e32 v57, v4
+; VI-NEXT:    v_mov_b32_e32 v58, v3
+; VI-NEXT:    v_mov_b32_e32 v59, v2
+; VI-NEXT:    v_mov_b32_e32 v60, v1
+; VI-NEXT:    v_mov_b32_e32 v61, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v29, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v29, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v29, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v29, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v29, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v29, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v29, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v29, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v29, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v29, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v29, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v29, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v29, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v29, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v29, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v29, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v29, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v29, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v29, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v29, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v29, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v29, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v29, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v29, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v29, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v29, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v29, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v28, v29, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v29, v29, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v60, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v59, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v58, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v57, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v56, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v47, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v46, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v44, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v43, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v41, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v40, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v55, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v54, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v53, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v51, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v50, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v38, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v36, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v35, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v34, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v33, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v32, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr60
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB7_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB7_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v29, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v61
+; VI-NEXT:    v_add_u16_sdwa v1, v61, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v2, 3, v60
+; VI-NEXT:    v_add_u16_sdwa v3, v60, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v59
+; VI-NEXT:    v_add_u16_sdwa v3, v59, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v58
+; VI-NEXT:    v_add_u16_sdwa v4, v58, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v5, v57, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v6, v56, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v7, v47, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v8, v46, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v9, v45, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v10, v44, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v11, v43, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v12, v42, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v13, v41, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v14, v40, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v15, v55, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v16, v54, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v17, v53, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v18, v52, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v19, v51, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v20, v50, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v21, v49, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v22, v48, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v23, v39, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v24, v38, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v25, v37, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v25
+; VI-NEXT:    v_add_u16_e32 v25, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v26, v36, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v25, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v27, v35, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v26, v26, v27
+; VI-NEXT:    v_add_u16_e32 v27, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v28, v34, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v27, v27, v28
+; VI-NEXT:    v_add_u16_e32 v28, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v30, v33, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v28, v28, v30
+; VI-NEXT:    v_add_u16_e32 v30, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v29, v32, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v29, v30, v29
+; VI-NEXT:  .LBB7_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60i16_to_v30i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v61, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GFX9-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v39, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v48, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    v_mov_b32_e32 v43, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_mov_b32_e32 v44, v9
+; GFX9-NEXT:    v_mov_b32_e32 v45, v8
+; GFX9-NEXT:    v_mov_b32_e32 v46, v7
+; GFX9-NEXT:    v_mov_b32_e32 v47, v6
+; GFX9-NEXT:    v_mov_b32_e32 v56, v5
+; GFX9-NEXT:    v_mov_b32_e32 v57, v4
+; GFX9-NEXT:    v_mov_b32_e32 v58, v3
+; GFX9-NEXT:    v_mov_b32_e32 v59, v2
+; GFX9-NEXT:    v_mov_b32_e32 v60, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v43
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v59
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v60
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v61
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v39
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:  .LBB7_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB7_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB7_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60i16_to_v30i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    v_perm_b32 v0, v81, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v83, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v84, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v80, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v71, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v70, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v69, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v68, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v67, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v66, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v65, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v64, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v55, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v54, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v53, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v52, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v51, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v50, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v49, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v48, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v39, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v38, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v37, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v36, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v35, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v34, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v33, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v32, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v31, v29, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB7_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB7_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <60 x i16> %a, splat (i16 3)
+  %a2 = bitcast <60 x i16> %a1 to <30 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x i16> %a to <30 x i32>
+  br label %end
+
+end:
+  %phi = phi <30 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x i32> %phi
+}
+
+define <60 x half> @bitcast_v30i32_to_v60f16(<30 x i32> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30i32_to_v60f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v31
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v36
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v50
+; GCN-NEXT:    v_mov_b32_e32 v50, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v61
+; GCN-NEXT:    v_mov_b32_e32 v61, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v63
+; GCN-NEXT:    v_mov_b32_e32 v63, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:  .LBB8_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB8_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v23
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:  .LBB8_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v2, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v46
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v45
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v43
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v42
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v41
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v40
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v54
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v52
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v51
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v49
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v39
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v37
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v35
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v35, v30
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v33, v36, v33
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v31, v37, v31
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v62
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v48, v34
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v49, v32
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v61
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v63
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x68, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v47, v46
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 0x6c, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v56, v57, v56
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x70, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v58, v59, v58
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v58, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30i32_to_v60f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB8_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB8_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v29, vcc, 3, v29
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_add_u32_e32 v27, vcc, 3, v27
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_add_u32_e32 v25, vcc, 3, v25
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_add_u32_e32 v23, vcc, 3, v23
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_add_u32_e32 v21, vcc, 3, v21
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_add_u32_e32 v19, vcc, 3, v19
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_add_u32_e32 v17, vcc, 3, v17
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_add_u32_e32 v15, vcc, 3, v15
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_add_u32_e32 v13, vcc, 3, v13
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_add_u32_e32 v11, vcc, 3, v11
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 3, v9
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v7
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 3, v5
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 3, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB8_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; VI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; VI-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; VI-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30i32_to_v60f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB8_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB8_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_u32_e32 v29, 3, v29
+; GFX9-NEXT:    v_add_u32_e32 v28, 3, v28
+; GFX9-NEXT:    v_add_u32_e32 v27, 3, v27
+; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
+; GFX9-NEXT:    v_add_u32_e32 v25, 3, v25
+; GFX9-NEXT:    v_add_u32_e32 v24, 3, v24
+; GFX9-NEXT:    v_add_u32_e32 v23, 3, v23
+; GFX9-NEXT:    v_add_u32_e32 v22, 3, v22
+; GFX9-NEXT:    v_add_u32_e32 v21, 3, v21
+; GFX9-NEXT:    v_add_u32_e32 v20, 3, v20
+; GFX9-NEXT:    v_add_u32_e32 v19, 3, v19
+; GFX9-NEXT:    v_add_u32_e32 v18, 3, v18
+; GFX9-NEXT:    v_add_u32_e32 v17, 3, v17
+; GFX9-NEXT:    v_add_u32_e32 v16, 3, v16
+; GFX9-NEXT:    v_add_u32_e32 v15, 3, v15
+; GFX9-NEXT:    v_add_u32_e32 v14, 3, v14
+; GFX9-NEXT:    v_add_u32_e32 v13, 3, v13
+; GFX9-NEXT:    v_add_u32_e32 v12, 3, v12
+; GFX9-NEXT:    v_add_u32_e32 v11, 3, v11
+; GFX9-NEXT:    v_add_u32_e32 v10, 3, v10
+; GFX9-NEXT:    v_add_u32_e32 v9, 3, v9
+; GFX9-NEXT:    v_add_u32_e32 v8, 3, v8
+; GFX9-NEXT:    v_add_u32_e32 v7, 3, v7
+; GFX9-NEXT:    v_add_u32_e32 v6, 3, v6
+; GFX9-NEXT:    v_add_u32_e32 v5, 3, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, 3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, 3, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB8_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v58, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v57, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v56, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v47, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v46, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v45, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v44, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v43, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v42, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v41, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v40, v11, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v12, v55, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v54, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v53, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v50, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v49, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v48, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v39, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v38, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v37, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v36, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v35, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v34, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v33, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v32, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v31, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30i32_to_v60f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    ; implicit-def: $vgpr83
+; GFX11-NEXT:    ; implicit-def: $vgpr82
+; GFX11-NEXT:    ; implicit-def: $vgpr81
+; GFX11-NEXT:    ; implicit-def: $vgpr80
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB8_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB8_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_nc_u32_e32 v29, 3, v29
+; GFX11-NEXT:    v_add_nc_u32_e32 v28, 3, v28
+; GFX11-NEXT:    v_add_nc_u32_e32 v27, 3, v27
+; GFX11-NEXT:    v_add_nc_u32_e32 v26, 3, v26
+; GFX11-NEXT:    v_add_nc_u32_e32 v25, 3, v25
+; GFX11-NEXT:    v_add_nc_u32_e32 v24, 3, v24
+; GFX11-NEXT:    v_add_nc_u32_e32 v23, 3, v23
+; GFX11-NEXT:    v_add_nc_u32_e32 v22, 3, v22
+; GFX11-NEXT:    v_add_nc_u32_e32 v21, 3, v21
+; GFX11-NEXT:    v_add_nc_u32_e32 v20, 3, v20
+; GFX11-NEXT:    v_add_nc_u32_e32 v19, 3, v19
+; GFX11-NEXT:    v_add_nc_u32_e32 v18, 3, v18
+; GFX11-NEXT:    v_add_nc_u32_e32 v17, 3, v17
+; GFX11-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_add_nc_u32_e32 v14, 3, v14
+; GFX11-NEXT:    v_add_nc_u32_e32 v13, 3, v13
+; GFX11-NEXT:    v_add_nc_u32_e32 v12, 3, v12
+; GFX11-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-NEXT:    v_add_nc_u32_e32 v10, 3, v10
+; GFX11-NEXT:    v_add_nc_u32_e32 v9, 3, v9
+; GFX11-NEXT:    v_add_nc_u32_e32 v8, 3, v8
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 3, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 3, v6
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 3, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 3, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, 3, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB8_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v83, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v81, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v71, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v70, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v69, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v68, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v67, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v66, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v65, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v64, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v55, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v54, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v39, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v38, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v37, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v36, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v35, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v34, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v33, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v32, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v31, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v30, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <30 x i32> %a, splat (i32 3)
+  %a2 = bitcast <30 x i32> %a1 to <60 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x i32> %a to <60 x half>
+  br label %end
+
+end:
+  %phi = phi <60 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x half> %phi
+}
+
+define <30 x i32> @bitcast_v60f16_to_v30i32(<60 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60f16_to_v30i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:112
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:104
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:92
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44
+; GCN-NEXT:    s_waitcnt vmcnt(10)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v48
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v5
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v4
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v7
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v6
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v9
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v8
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v11
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v38
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v35
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v36
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v46
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v47
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v44
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v45
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v48
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v60
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v0, v58, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v56, v1
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v50
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v30, v16
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v46, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v44, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v43, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v41, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v48, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v38, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v36, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v25
+; GCN-NEXT:    v_or_b32_e32 v26, v37, v26
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v27
+; GCN-NEXT:    v_or_b32_e32 v28, v49, v28
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v29
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:  .LBB9_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB9_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v56
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v15, v16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v46
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v44
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v19, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v20, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v21, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v23, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v24, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v26, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v28, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v50
+; GCN-NEXT:  .LBB9_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60f16_to_v30i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v29
+; VI-NEXT:    v_mov_b32_e32 v33, v28
+; VI-NEXT:    v_mov_b32_e32 v34, v27
+; VI-NEXT:    v_mov_b32_e32 v35, v26
+; VI-NEXT:    v_mov_b32_e32 v36, v25
+; VI-NEXT:    v_mov_b32_e32 v37, v24
+; VI-NEXT:    v_mov_b32_e32 v38, v23
+; VI-NEXT:    v_mov_b32_e32 v39, v22
+; VI-NEXT:    v_mov_b32_e32 v48, v21
+; VI-NEXT:    v_mov_b32_e32 v49, v20
+; VI-NEXT:    v_mov_b32_e32 v50, v19
+; VI-NEXT:    v_mov_b32_e32 v51, v18
+; VI-NEXT:    v_mov_b32_e32 v52, v17
+; VI-NEXT:    v_mov_b32_e32 v53, v16
+; VI-NEXT:    v_mov_b32_e32 v54, v15
+; VI-NEXT:    v_mov_b32_e32 v55, v14
+; VI-NEXT:    v_mov_b32_e32 v40, v13
+; VI-NEXT:    v_mov_b32_e32 v41, v12
+; VI-NEXT:    v_mov_b32_e32 v42, v11
+; VI-NEXT:    v_mov_b32_e32 v43, v10
+; VI-NEXT:    v_mov_b32_e32 v44, v9
+; VI-NEXT:    v_mov_b32_e32 v45, v8
+; VI-NEXT:    v_mov_b32_e32 v46, v7
+; VI-NEXT:    v_mov_b32_e32 v47, v6
+; VI-NEXT:    v_mov_b32_e32 v56, v5
+; VI-NEXT:    v_mov_b32_e32 v57, v4
+; VI-NEXT:    v_mov_b32_e32 v58, v3
+; VI-NEXT:    v_mov_b32_e32 v59, v2
+; VI-NEXT:    v_mov_b32_e32 v60, v1
+; VI-NEXT:    v_mov_b32_e32 v61, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v29, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v29, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v29, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v29, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v29, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v29, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v29, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v29, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v29, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v29, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v29, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v29, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v29, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v29, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v29, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v29, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v29, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v29, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v29, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v29, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v29, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v29, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v29, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v29, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v29, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v29, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v29, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v28, v29, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v29, v29, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v60, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v59, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v58, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v57, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v56, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v47, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v46, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v44, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v43, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v41, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v40, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v55, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v54, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v53, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v51, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v50, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v38, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v36, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v35, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v34, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v33, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v32, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr60
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB9_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB9_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v29, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v61, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v61
+; VI-NEXT:    v_add_f16_sdwa v2, v60, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v60
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v59, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v59
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v58, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v58
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v57, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v56, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v47, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v46, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v45, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v44, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v43, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v42, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v41, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v40, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v55, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v54, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v53, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v52, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v51, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v50, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v49, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v48, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v39, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v38, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v37, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v24, v25, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v36, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:    v_add_f16_sdwa v26, v35, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v26, v27, v26
+; VI-NEXT:    v_add_f16_sdwa v27, v34, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:    v_add_f16_sdwa v28, v33, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v28, v30, v28
+; VI-NEXT:    v_add_f16_sdwa v29, v32, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v29, v30, v29
+; VI-NEXT:  .LBB9_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60f16_to_v30i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v61, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GFX9-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v39, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v48, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    v_mov_b32_e32 v43, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_mov_b32_e32 v44, v9
+; GFX9-NEXT:    v_mov_b32_e32 v45, v8
+; GFX9-NEXT:    v_mov_b32_e32 v46, v7
+; GFX9-NEXT:    v_mov_b32_e32 v47, v6
+; GFX9-NEXT:    v_mov_b32_e32 v56, v5
+; GFX9-NEXT:    v_mov_b32_e32 v57, v4
+; GFX9-NEXT:    v_mov_b32_e32 v58, v3
+; GFX9-NEXT:    v_mov_b32_e32 v59, v2
+; GFX9-NEXT:    v_mov_b32_e32 v60, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v43
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v59
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v60
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v61
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v39
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:  .LBB9_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB9_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v28, v28, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    v_pk_add_f16 v29, v29, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB9_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60f16_to_v30i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    v_perm_b32 v0, v81, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v83, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v84, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v80, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v71, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v70, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v69, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v68, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v67, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v66, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v65, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v64, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v55, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v54, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v53, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v52, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v51, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v50, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v49, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v48, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v39, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v38, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v37, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v36, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v35, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v34, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v33, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v32, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v31, v29, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB9_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v28, 0x200, v28 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v29, 0x200, v29 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB9_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <60 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <60 x half> %a1 to <30 x i32>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x half> %a to <30 x i32>
+  br label %end
+
+end:
+  %phi = phi <30 x i32> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x i32> %phi
+}
+
+define <15 x i64> @bitcast_v30f32_to_v15i64(<30 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30f32_to_v15i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB10_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GCN-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB10_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30f32_to_v15i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB10_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; VI-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB10_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30f32_to_v15i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GFX9-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB10_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30f32_to_v15i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB10_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v29, 1.0, v29 :: v_dual_add_f32 v28, 1.0, v28
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB10_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <30 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <30 x float> %a1 to <15 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x float> %a to <15 x i64>
+  br label %end
+
+end:
+  %phi = phi <15 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x i64> %phi
+}
+
+define <30 x float> @bitcast_v15i64_to_v30f32(<15 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15i64_to_v30f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB11_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:  .LBB11_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15i64_to_v30f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB11_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:  .LBB11_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15i64_to_v30f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB11_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v28, vcc, 3, v28
+; GFX9-NEXT:    v_addc_co_u32_e32 v29, vcc, 0, v29, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:  .LBB11_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15i64_to_v30f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB11_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v28, vcc_lo, v28, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v29, null, 0, v29, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:  .LBB11_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <15 x i64> %a, splat (i64 3)
+  %a2 = bitcast <15 x i64> %a1 to <30 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x i64> %a to <30 x float>
+  br label %end
+
+end:
+  %phi = phi <30 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x float> %phi
+}
+
+define <15 x double> @bitcast_v30f32_to_v15f64(<30 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30f32_to_v15f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB12_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GCN-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:  .LBB12_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30f32_to_v15f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB12_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; VI-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:  .LBB12_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30f32_to_v15f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB12_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GFX9-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:  .LBB12_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30f32_to_v15f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB12_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v29, 1.0, v29 :: v_dual_add_f32 v28, 1.0, v28
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:  .LBB12_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <30 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <30 x float> %a1 to <15 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x float> %a to <15 x double>
+  br label %end
+
+end:
+  %phi = phi <15 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x double> %phi
+}
+
+define <30 x float> @bitcast_v15f64_to_v30f32(<15 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15f64_to_v30f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB13_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GCN-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:  .LBB13_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15f64_to_v30f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB13_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:  .LBB13_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15f64_to_v30f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB13_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:  .LBB13_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15f64_to_v30f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB13_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:  .LBB13_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <15 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <15 x double> %a1 to <30 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x double> %a to <30 x float>
+  br label %end
+
+end:
+  %phi = phi <30 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x float> %phi
+}
+
+define <60 x i16> @bitcast_v30f32_to_v60i16(<30 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30f32_to_v60i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v31, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v6, v5, 16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_alignbit_b32 v46, v4, v3, 16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_alignbit_b32 v56, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v2
+; GCN-NEXT:  .LBB14_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB14_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v30, 1.0, v30
+; GCN-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GCN-NEXT:    v_alignbit_b32 v31, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v51, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v6, v5, 16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_alignbit_b32 v46, v4, v3, 16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_alignbit_b32 v56, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v2
+; GCN-NEXT:  .LBB14_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v56
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v60
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v56
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v46
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v59
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v44
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v58
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v57
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v47
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v45
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v43
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v40
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x64, v0
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x68, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x6c, v0
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x70, v0
+; GCN-NEXT:    v_and_b32_e32 v30, 0xffff, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30f32_to_v60i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB14_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB14_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; VI-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB14_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; VI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; VI-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; VI-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30f32_to_v60i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB14_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB14_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GFX9-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB14_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v58, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v57, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v56, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v47, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v46, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v45, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v44, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v43, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v42, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v41, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v40, v11, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v12, v55, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v54, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v53, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v50, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v49, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v48, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v39, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v38, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v37, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v36, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v35, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v34, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v33, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v32, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v31, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30f32_to_v60i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    ; implicit-def: $vgpr83
+; GFX11-NEXT:    ; implicit-def: $vgpr82
+; GFX11-NEXT:    ; implicit-def: $vgpr81
+; GFX11-NEXT:    ; implicit-def: $vgpr80
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB14_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB14_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v29, 1.0, v29 :: v_dual_add_f32 v28, 1.0, v28
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB14_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v83, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v81, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v71, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v70, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v69, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v68, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v67, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v66, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v65, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v64, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v55, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v54, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v39, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v38, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v37, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v36, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v35, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v34, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v33, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v32, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v31, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v30, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <30 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <30 x float> %a1 to <60 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x float> %a to <60 x i16>
+  br label %end
+
+end:
+  %phi = phi <60 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x i16> %phi
+}
+
+define <30 x float> @bitcast_v60i16_to_v30f32(<60 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60i16_to_v30f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:112
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:104
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(6) expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v42
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v41
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v63
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v62
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v61
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v60
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v59
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v58
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v57
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v47
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v15
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v16
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v44
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v46
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v45
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v43
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v56
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v30
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:  .LBB15_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB15_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v56
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v42, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v41, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v63, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v62, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v61, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v60, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v59, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v58, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v57, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v47, v12
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v30, v13
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v30, v14
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v30, v15
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v30, v16
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v17
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v30, v18
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v30, v19
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v30, v20
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v30, v21
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v30, v22
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v30, v23
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v30, v24
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v30, v25
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v30, v26
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v30, v27
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v28, v30, v28
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v29, v30, v29
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 0x30000, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x30000, v29
+; GCN-NEXT:  .LBB15_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60i16_to_v30f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v29
+; VI-NEXT:    v_mov_b32_e32 v33, v28
+; VI-NEXT:    v_mov_b32_e32 v34, v27
+; VI-NEXT:    v_mov_b32_e32 v35, v26
+; VI-NEXT:    v_mov_b32_e32 v36, v25
+; VI-NEXT:    v_mov_b32_e32 v37, v24
+; VI-NEXT:    v_mov_b32_e32 v38, v23
+; VI-NEXT:    v_mov_b32_e32 v39, v22
+; VI-NEXT:    v_mov_b32_e32 v48, v21
+; VI-NEXT:    v_mov_b32_e32 v49, v20
+; VI-NEXT:    v_mov_b32_e32 v50, v19
+; VI-NEXT:    v_mov_b32_e32 v51, v18
+; VI-NEXT:    v_mov_b32_e32 v52, v17
+; VI-NEXT:    v_mov_b32_e32 v53, v16
+; VI-NEXT:    v_mov_b32_e32 v54, v15
+; VI-NEXT:    v_mov_b32_e32 v55, v14
+; VI-NEXT:    v_mov_b32_e32 v40, v13
+; VI-NEXT:    v_mov_b32_e32 v41, v12
+; VI-NEXT:    v_mov_b32_e32 v42, v11
+; VI-NEXT:    v_mov_b32_e32 v43, v10
+; VI-NEXT:    v_mov_b32_e32 v44, v9
+; VI-NEXT:    v_mov_b32_e32 v45, v8
+; VI-NEXT:    v_mov_b32_e32 v46, v7
+; VI-NEXT:    v_mov_b32_e32 v47, v6
+; VI-NEXT:    v_mov_b32_e32 v56, v5
+; VI-NEXT:    v_mov_b32_e32 v57, v4
+; VI-NEXT:    v_mov_b32_e32 v58, v3
+; VI-NEXT:    v_mov_b32_e32 v59, v2
+; VI-NEXT:    v_mov_b32_e32 v60, v1
+; VI-NEXT:    v_mov_b32_e32 v61, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v29, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v29, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v29, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v29, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v29, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v29, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v29, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v29, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v29, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v29, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v29, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v29, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v29, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v29, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v29, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v29, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v29, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v29, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v29, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v29, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v29, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v29, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v29, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v29, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v29, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v29, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v29, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v28, v29, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v29, v29, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v60, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v59, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v58, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v57, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v56, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v47, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v46, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v44, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v43, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v41, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v40, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v55, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v54, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v53, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v51, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v50, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v38, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v36, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v35, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v34, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v33, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v32, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr60
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB15_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB15_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v29, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v61
+; VI-NEXT:    v_add_u16_sdwa v1, v61, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v2, 3, v60
+; VI-NEXT:    v_add_u16_sdwa v3, v60, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v59
+; VI-NEXT:    v_add_u16_sdwa v3, v59, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v58
+; VI-NEXT:    v_add_u16_sdwa v4, v58, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v5, v57, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v6, v56, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v7, v47, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v8, v46, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v9, v45, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v10, v44, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v11, v43, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v12, v42, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v13, v41, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v14, v40, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v15, v55, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v16, v54, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v17, v53, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v18, v52, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v19, v51, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v20, v50, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v21, v49, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v22, v48, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v23, v39, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v24, v38, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v25, v37, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v25
+; VI-NEXT:    v_add_u16_e32 v25, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v26, v36, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v25, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v27, v35, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v26, v26, v27
+; VI-NEXT:    v_add_u16_e32 v27, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v28, v34, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v27, v27, v28
+; VI-NEXT:    v_add_u16_e32 v28, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v30, v33, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v28, v28, v30
+; VI-NEXT:    v_add_u16_e32 v30, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v29, v32, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v29, v30, v29
+; VI-NEXT:  .LBB15_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60i16_to_v30f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v61, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GFX9-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v39, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v48, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    v_mov_b32_e32 v43, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_mov_b32_e32 v44, v9
+; GFX9-NEXT:    v_mov_b32_e32 v45, v8
+; GFX9-NEXT:    v_mov_b32_e32 v46, v7
+; GFX9-NEXT:    v_mov_b32_e32 v47, v6
+; GFX9-NEXT:    v_mov_b32_e32 v56, v5
+; GFX9-NEXT:    v_mov_b32_e32 v57, v4
+; GFX9-NEXT:    v_mov_b32_e32 v58, v3
+; GFX9-NEXT:    v_mov_b32_e32 v59, v2
+; GFX9-NEXT:    v_mov_b32_e32 v60, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v43
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v59
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v60
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v61
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v39
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:  .LBB15_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB15_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB15_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60i16_to_v30f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    v_perm_b32 v0, v81, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v83, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v84, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v80, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v71, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v70, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v69, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v68, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v67, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v66, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v65, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v64, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v55, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v54, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v53, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v52, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v51, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v50, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v49, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v48, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v39, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v38, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v37, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v36, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v35, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v34, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v33, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v32, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v31, v29, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB15_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB15_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <60 x i16> %a, splat (i16 3)
+  %a2 = bitcast <60 x i16> %a1 to <30 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x i16> %a to <30 x float>
+  br label %end
+
+end:
+  %phi = phi <30 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x float> %phi
+}
+
+define <60 x half> @bitcast_v30f32_to_v60f16(<30 x float> %a, i32 %b) {
+; GCN-LABEL: bitcast_v30f32_to_v60f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v31
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v36
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v50
+; GCN-NEXT:    v_mov_b32_e32 v50, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v61
+; GCN-NEXT:    v_mov_b32_e32 v61, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v63
+; GCN-NEXT:    v_mov_b32_e32 v63, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:  .LBB16_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB16_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 1.0, v30
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v23
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:  .LBB16_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v2, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v46
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v45
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v43
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v42
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v41
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v40
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v54
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v52
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v51
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v49
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v39
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v37
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v35
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v35, v30
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v33, v36, v33
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v31, v37, v31
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v62
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v48, v34
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v49, v32
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v61
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v63
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x68, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v47, v46
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 0x6c, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v56, v57, v56
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x70, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v58, v59, v58
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v58, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v30f32_to_v60f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB16_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB16_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; VI-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; VI-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; VI-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; VI-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; VI-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; VI-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; VI-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; VI-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; VI-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; VI-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; VI-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; VI-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; VI-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; VI-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; VI-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; VI-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; VI-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; VI-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; VI-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; VI-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; VI-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; VI-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; VI-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; VI-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; VI-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; VI-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; VI-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB16_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; VI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; VI-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; VI-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v30f32_to_v60f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB16_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB16_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f32_e32 v29, 1.0, v29
+; GFX9-NEXT:    v_add_f32_e32 v28, 1.0, v28
+; GFX9-NEXT:    v_add_f32_e32 v27, 1.0, v27
+; GFX9-NEXT:    v_add_f32_e32 v26, 1.0, v26
+; GFX9-NEXT:    v_add_f32_e32 v25, 1.0, v25
+; GFX9-NEXT:    v_add_f32_e32 v24, 1.0, v24
+; GFX9-NEXT:    v_add_f32_e32 v23, 1.0, v23
+; GFX9-NEXT:    v_add_f32_e32 v22, 1.0, v22
+; GFX9-NEXT:    v_add_f32_e32 v21, 1.0, v21
+; GFX9-NEXT:    v_add_f32_e32 v20, 1.0, v20
+; GFX9-NEXT:    v_add_f32_e32 v19, 1.0, v19
+; GFX9-NEXT:    v_add_f32_e32 v18, 1.0, v18
+; GFX9-NEXT:    v_add_f32_e32 v17, 1.0, v17
+; GFX9-NEXT:    v_add_f32_e32 v16, 1.0, v16
+; GFX9-NEXT:    v_add_f32_e32 v15, 1.0, v15
+; GFX9-NEXT:    v_add_f32_e32 v14, 1.0, v14
+; GFX9-NEXT:    v_add_f32_e32 v13, 1.0, v13
+; GFX9-NEXT:    v_add_f32_e32 v12, 1.0, v12
+; GFX9-NEXT:    v_add_f32_e32 v11, 1.0, v11
+; GFX9-NEXT:    v_add_f32_e32 v10, 1.0, v10
+; GFX9-NEXT:    v_add_f32_e32 v9, 1.0, v9
+; GFX9-NEXT:    v_add_f32_e32 v8, 1.0, v8
+; GFX9-NEXT:    v_add_f32_e32 v7, 1.0, v7
+; GFX9-NEXT:    v_add_f32_e32 v6, 1.0, v6
+; GFX9-NEXT:    v_add_f32_e32 v5, 1.0, v5
+; GFX9-NEXT:    v_add_f32_e32 v4, 1.0, v4
+; GFX9-NEXT:    v_add_f32_e32 v3, 1.0, v3
+; GFX9-NEXT:    v_add_f32_e32 v2, 1.0, v2
+; GFX9-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB16_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v58, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v57, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v56, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v47, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v46, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v45, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v44, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v43, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v42, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v41, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v40, v11, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v12, v55, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v54, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v53, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v50, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v49, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v48, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v39, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v38, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v37, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v36, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v35, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v34, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v33, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v32, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v31, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v30f32_to_v60f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    ; implicit-def: $vgpr83
+; GFX11-NEXT:    ; implicit-def: $vgpr82
+; GFX11-NEXT:    ; implicit-def: $vgpr81
+; GFX11-NEXT:    ; implicit-def: $vgpr80
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB16_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB16_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_dual_add_f32 v29, 1.0, v29 :: v_dual_add_f32 v28, 1.0, v28
+; GFX11-NEXT:    v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v26, 1.0, v26
+; GFX11-NEXT:    v_dual_add_f32 v25, 1.0, v25 :: v_dual_add_f32 v24, 1.0, v24
+; GFX11-NEXT:    v_dual_add_f32 v23, 1.0, v23 :: v_dual_add_f32 v22, 1.0, v22
+; GFX11-NEXT:    v_dual_add_f32 v21, 1.0, v21 :: v_dual_add_f32 v20, 1.0, v20
+; GFX11-NEXT:    v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v18, 1.0, v18
+; GFX11-NEXT:    v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-NEXT:    v_dual_add_f32 v15, 1.0, v15 :: v_dual_add_f32 v14, 1.0, v14
+; GFX11-NEXT:    v_dual_add_f32 v13, 1.0, v13 :: v_dual_add_f32 v12, 1.0, v12
+; GFX11-NEXT:    v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-NEXT:    v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
+; GFX11-NEXT:    v_dual_add_f32 v7, 1.0, v7 :: v_dual_add_f32 v6, 1.0, v6
+; GFX11-NEXT:    v_dual_add_f32 v5, 1.0, v5 :: v_dual_add_f32 v4, 1.0, v4
+; GFX11-NEXT:    v_dual_add_f32 v3, 1.0, v3 :: v_dual_add_f32 v2, 1.0, v2
+; GFX11-NEXT:    v_dual_add_f32 v1, 1.0, v1 :: v_dual_add_f32 v0, 1.0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB16_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v83, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v81, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v71, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v70, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v69, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v68, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v67, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v66, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v65, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v64, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v55, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v54, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v39, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v38, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v37, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v36, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v35, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v34, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v33, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v32, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v31, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v30, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <30 x float> %a, splat (float 1.000000e+00)
+  %a2 = bitcast <30 x float> %a1 to <60 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <30 x float> %a to <60 x half>
+  br label %end
+
+end:
+  %phi = phi <60 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x half> %phi
+}
+
+define <30 x float> @bitcast_v60f16_to_v30f32(<60 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60f16_to_v30f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:112
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:104
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:92
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44
+; GCN-NEXT:    s_waitcnt vmcnt(10)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v48
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v5
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v4
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v7
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v6
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v9
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v8
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v11
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v38
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v35
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v36
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v46
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v47
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v44
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v45
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v48
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v60
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v0, v58, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v56, v1
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v50
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v30, v16
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v46, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v44, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v43, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v41, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v48, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v38, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v36, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v25
+; GCN-NEXT:    v_or_b32_e32 v26, v37, v26
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v27
+; GCN-NEXT:    v_or_b32_e32 v28, v49, v28
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v29
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:  .LBB17_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB17_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v56
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v15, v16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v46
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v44
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v19, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v20, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v21, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v23, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v24, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v26, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v28, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v50
+; GCN-NEXT:  .LBB17_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60f16_to_v30f32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v29
+; VI-NEXT:    v_mov_b32_e32 v33, v28
+; VI-NEXT:    v_mov_b32_e32 v34, v27
+; VI-NEXT:    v_mov_b32_e32 v35, v26
+; VI-NEXT:    v_mov_b32_e32 v36, v25
+; VI-NEXT:    v_mov_b32_e32 v37, v24
+; VI-NEXT:    v_mov_b32_e32 v38, v23
+; VI-NEXT:    v_mov_b32_e32 v39, v22
+; VI-NEXT:    v_mov_b32_e32 v48, v21
+; VI-NEXT:    v_mov_b32_e32 v49, v20
+; VI-NEXT:    v_mov_b32_e32 v50, v19
+; VI-NEXT:    v_mov_b32_e32 v51, v18
+; VI-NEXT:    v_mov_b32_e32 v52, v17
+; VI-NEXT:    v_mov_b32_e32 v53, v16
+; VI-NEXT:    v_mov_b32_e32 v54, v15
+; VI-NEXT:    v_mov_b32_e32 v55, v14
+; VI-NEXT:    v_mov_b32_e32 v40, v13
+; VI-NEXT:    v_mov_b32_e32 v41, v12
+; VI-NEXT:    v_mov_b32_e32 v42, v11
+; VI-NEXT:    v_mov_b32_e32 v43, v10
+; VI-NEXT:    v_mov_b32_e32 v44, v9
+; VI-NEXT:    v_mov_b32_e32 v45, v8
+; VI-NEXT:    v_mov_b32_e32 v46, v7
+; VI-NEXT:    v_mov_b32_e32 v47, v6
+; VI-NEXT:    v_mov_b32_e32 v56, v5
+; VI-NEXT:    v_mov_b32_e32 v57, v4
+; VI-NEXT:    v_mov_b32_e32 v58, v3
+; VI-NEXT:    v_mov_b32_e32 v59, v2
+; VI-NEXT:    v_mov_b32_e32 v60, v1
+; VI-NEXT:    v_mov_b32_e32 v61, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v29, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v29, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v29, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v29, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v29, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v29, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v29, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v29, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v29, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v29, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v29, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v29, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v29, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v29, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v29, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v29, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v29, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v29, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v29, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v29, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v29, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v29, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v29, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v29, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v29, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v29, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v29, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v28, v29, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v29, v29, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v60, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v59, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v58, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v57, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v56, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v47, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v46, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v44, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v43, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v41, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v40, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v55, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v54, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v53, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v51, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v50, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v38, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v36, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v35, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v34, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v33, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v32, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr60
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB17_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB17_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v29, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v61, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v61
+; VI-NEXT:    v_add_f16_sdwa v2, v60, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v60
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v59, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v59
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v58, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v58
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v57, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v56, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v47, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v46, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v45, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v44, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v43, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v42, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v41, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v40, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v55, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v54, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v53, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v52, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v51, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v50, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v49, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v48, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v39, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v38, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v37, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v24, v25, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v36, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:    v_add_f16_sdwa v26, v35, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v26, v27, v26
+; VI-NEXT:    v_add_f16_sdwa v27, v34, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:    v_add_f16_sdwa v28, v33, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v28, v30, v28
+; VI-NEXT:    v_add_f16_sdwa v29, v32, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v29, v30, v29
+; VI-NEXT:  .LBB17_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60f16_to_v30f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v61, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GFX9-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v39, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v48, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    v_mov_b32_e32 v43, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_mov_b32_e32 v44, v9
+; GFX9-NEXT:    v_mov_b32_e32 v45, v8
+; GFX9-NEXT:    v_mov_b32_e32 v46, v7
+; GFX9-NEXT:    v_mov_b32_e32 v47, v6
+; GFX9-NEXT:    v_mov_b32_e32 v56, v5
+; GFX9-NEXT:    v_mov_b32_e32 v57, v4
+; GFX9-NEXT:    v_mov_b32_e32 v58, v3
+; GFX9-NEXT:    v_mov_b32_e32 v59, v2
+; GFX9-NEXT:    v_mov_b32_e32 v60, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v43
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v59
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v60
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v61
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v39
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:  .LBB17_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB17_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v28, v28, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    v_pk_add_f16 v29, v29, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB17_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60f16_to_v30f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    v_perm_b32 v0, v81, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v83, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v84, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v80, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v71, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v70, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v69, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v68, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v67, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v66, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v65, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v64, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v55, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v54, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v53, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v52, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v51, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v50, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v49, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v48, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v39, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v38, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v37, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v36, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v35, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v34, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v33, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v32, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v31, v29, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB17_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v28, 0x200, v28 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v29, 0x200, v29 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB17_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <60 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <60 x half> %a1 to <30 x float>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x half> %a to <30 x float>
+  br label %end
+
+end:
+  %phi = phi <30 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <30 x float> %phi
+}
+
+define <15 x double> @bitcast_v15i64_to_v15f64(<15 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15i64_to_v15f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB18_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
+; GCN-NEXT:  .LBB18_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15i64_to_v15f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB18_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
+; VI-NEXT:  .LBB18_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15i64_to_v15f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB18_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v28, vcc, 3, v28
+; GFX9-NEXT:    v_addc_co_u32_e32 v29, vcc, 0, v29, vcc
+; GFX9-NEXT:  .LBB18_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15i64_to_v15f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB18_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v28, vcc_lo, v28, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v29, null, 0, v29, vcc_lo
+; GFX11-NEXT:  .LBB18_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <15 x i64> %a, splat (i64 3)
+  %a2 = bitcast <15 x i64> %a1 to <15 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x i64> %a to <15 x double>
+  br label %end
+
+end:
+  %phi = phi <15 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x double> %phi
+}
+
+define <15 x i64> @bitcast_v15f64_to_v15i64(<15 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15f64_to_v15i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB19_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GCN-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GCN-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GCN-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GCN-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GCN-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GCN-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GCN-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GCN-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GCN-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GCN-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GCN-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GCN-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GCN-NEXT:  .LBB19_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15f64_to_v15i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB19_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; VI-NEXT:  .LBB19_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15f64_to_v15i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB19_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX9-NEXT:  .LBB19_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15f64_to_v15i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB19_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX11-NEXT:  .LBB19_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <15 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <15 x double> %a1 to <15 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x double> %a to <15 x i64>
+  br label %end
+
+end:
+  %phi = phi <15 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x i64> %phi
+}
+
+define <60 x i16> @bitcast_v15i64_to_v60i16(<15 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15i64_to_v60i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v31, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v6, v5, 16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_alignbit_b32 v46, v4, v3, 16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_alignbit_b32 v56, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v2
+; GCN-NEXT:  .LBB20_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB20_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_addc_u32_e32 v24, vcc, 0, v24, vcc
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_addc_u32_e32 v26, vcc, 0, v26, vcc
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_addc_u32_e32 v28, vcc, 0, v28, vcc
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_addc_u32_e32 v30, vcc, 0, v30, vcc
+; GCN-NEXT:    v_alignbit_b32 v31, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v50, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v43, v6, v5, 16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_alignbit_b32 v46, v4, v3, 16
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_alignbit_b32 v56, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v2
+; GCN-NEXT:  .LBB20_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v56
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v60
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v56
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v46
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v59
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v43
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v58
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 20, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v57
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 28, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v47
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v45
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v44
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v40
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x64, v0
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x68, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x6c, v0
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x70, v0
+; GCN-NEXT:    v_and_b32_e32 v30, 0xffff, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15i64_to_v60i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB20_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB20_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB20_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; VI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; VI-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; VI-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15i64_to_v60i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB20_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB20_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v28, vcc, 3, v28
+; GFX9-NEXT:    v_addc_co_u32_e32 v29, vcc, 0, v29, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB20_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v58, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v57, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v56, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v47, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v46, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v45, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v44, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v43, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v42, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v41, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v40, v11, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v12, v55, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v54, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v53, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v50, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v49, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v48, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v39, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v38, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v37, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v36, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v35, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v34, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v33, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v32, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v31, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15i64_to_v60i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    ; implicit-def: $vgpr83
+; GFX11-NEXT:    ; implicit-def: $vgpr82
+; GFX11-NEXT:    ; implicit-def: $vgpr81
+; GFX11-NEXT:    ; implicit-def: $vgpr80
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB20_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB20_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v28, vcc_lo, v28, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v29, null, 0, v29, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB20_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v83, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v81, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v71, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v70, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v69, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v68, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v67, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v66, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v65, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v64, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v55, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v54, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v39, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v38, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v37, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v36, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v35, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v34, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v33, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v32, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v31, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v30, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <15 x i64> %a, splat (i64 3)
+  %a2 = bitcast <15 x i64> %a1 to <60 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x i64> %a to <60 x i16>
+  br label %end
+
+end:
+  %phi = phi <60 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x i16> %phi
+}
+
+define <15 x i64> @bitcast_v60i16_to_v15i64(<60 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60i16_to_v15i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:112
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:104
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(6) expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v42
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v41
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v63
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v62
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v61
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v60
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v59
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v58
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v57
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v47
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v15
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v16
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v44
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v46
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v45
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v43
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v56
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v30
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:  .LBB21_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB21_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v56
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v42, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v41, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v63, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v62, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v61, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v60, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v59, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v58, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v57, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v47, v12
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v30, v13
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v30, v14
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v30, v15
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v30, v16
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v17
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v30, v18
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v30, v19
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v30, v20
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v30, v21
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v30, v22
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v30, v23
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v30, v24
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v30, v25
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v30, v26
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v30, v27
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v28, v30, v28
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v29, v30, v29
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 0x30000, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x30000, v29
+; GCN-NEXT:  .LBB21_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60i16_to_v15i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v29
+; VI-NEXT:    v_mov_b32_e32 v33, v28
+; VI-NEXT:    v_mov_b32_e32 v34, v27
+; VI-NEXT:    v_mov_b32_e32 v35, v26
+; VI-NEXT:    v_mov_b32_e32 v36, v25
+; VI-NEXT:    v_mov_b32_e32 v37, v24
+; VI-NEXT:    v_mov_b32_e32 v38, v23
+; VI-NEXT:    v_mov_b32_e32 v39, v22
+; VI-NEXT:    v_mov_b32_e32 v48, v21
+; VI-NEXT:    v_mov_b32_e32 v49, v20
+; VI-NEXT:    v_mov_b32_e32 v50, v19
+; VI-NEXT:    v_mov_b32_e32 v51, v18
+; VI-NEXT:    v_mov_b32_e32 v52, v17
+; VI-NEXT:    v_mov_b32_e32 v53, v16
+; VI-NEXT:    v_mov_b32_e32 v54, v15
+; VI-NEXT:    v_mov_b32_e32 v55, v14
+; VI-NEXT:    v_mov_b32_e32 v40, v13
+; VI-NEXT:    v_mov_b32_e32 v41, v12
+; VI-NEXT:    v_mov_b32_e32 v42, v11
+; VI-NEXT:    v_mov_b32_e32 v43, v10
+; VI-NEXT:    v_mov_b32_e32 v44, v9
+; VI-NEXT:    v_mov_b32_e32 v45, v8
+; VI-NEXT:    v_mov_b32_e32 v46, v7
+; VI-NEXT:    v_mov_b32_e32 v47, v6
+; VI-NEXT:    v_mov_b32_e32 v56, v5
+; VI-NEXT:    v_mov_b32_e32 v57, v4
+; VI-NEXT:    v_mov_b32_e32 v58, v3
+; VI-NEXT:    v_mov_b32_e32 v59, v2
+; VI-NEXT:    v_mov_b32_e32 v60, v1
+; VI-NEXT:    v_mov_b32_e32 v61, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v29, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v29, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v29, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v29, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v29, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v29, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v29, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v29, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v29, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v29, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v29, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v29, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v29, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v29, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v29, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v29, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v29, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v29, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v29, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v29, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v29, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v29, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v29, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v29, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v29, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v29, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v29, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v28, v29, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v29, v29, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v60, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v59, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v58, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v57, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v56, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v47, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v46, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v44, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v43, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v41, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v40, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v55, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v54, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v53, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v51, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v50, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v38, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v36, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v35, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v34, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v33, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v32, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr60
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB21_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB21_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v29, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v61
+; VI-NEXT:    v_add_u16_sdwa v1, v61, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v2, 3, v60
+; VI-NEXT:    v_add_u16_sdwa v3, v60, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v59
+; VI-NEXT:    v_add_u16_sdwa v3, v59, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v58
+; VI-NEXT:    v_add_u16_sdwa v4, v58, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v5, v57, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v6, v56, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v7, v47, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v8, v46, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v9, v45, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v10, v44, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v11, v43, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v12, v42, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v13, v41, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v14, v40, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v15, v55, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v16, v54, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v17, v53, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v18, v52, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v19, v51, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v20, v50, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v21, v49, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v22, v48, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v23, v39, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v24, v38, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v25, v37, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v25
+; VI-NEXT:    v_add_u16_e32 v25, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v26, v36, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v25, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v27, v35, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v26, v26, v27
+; VI-NEXT:    v_add_u16_e32 v27, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v28, v34, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v27, v27, v28
+; VI-NEXT:    v_add_u16_e32 v28, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v30, v33, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v28, v28, v30
+; VI-NEXT:    v_add_u16_e32 v30, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v29, v32, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v29, v30, v29
+; VI-NEXT:  .LBB21_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60i16_to_v15i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v61, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GFX9-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v39, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v48, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    v_mov_b32_e32 v43, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_mov_b32_e32 v44, v9
+; GFX9-NEXT:    v_mov_b32_e32 v45, v8
+; GFX9-NEXT:    v_mov_b32_e32 v46, v7
+; GFX9-NEXT:    v_mov_b32_e32 v47, v6
+; GFX9-NEXT:    v_mov_b32_e32 v56, v5
+; GFX9-NEXT:    v_mov_b32_e32 v57, v4
+; GFX9-NEXT:    v_mov_b32_e32 v58, v3
+; GFX9-NEXT:    v_mov_b32_e32 v59, v2
+; GFX9-NEXT:    v_mov_b32_e32 v60, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v43
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v59
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v60
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v61
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v39
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:  .LBB21_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB21_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB21_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60i16_to_v15i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    v_perm_b32 v0, v81, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v83, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v84, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v80, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v71, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v70, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v69, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v68, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v67, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v66, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v65, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v64, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v55, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v54, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v53, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v52, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v51, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v50, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v49, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v48, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v39, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v38, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v37, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v36, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v35, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v34, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v33, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v32, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v31, v29, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB21_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB21_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <60 x i16> %a, splat (i16 3)
+  %a2 = bitcast <60 x i16> %a1 to <15 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x i16> %a to <15 x i64>
+  br label %end
+
+end:
+  %phi = phi <15 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x i64> %phi
+}
+
+define <60 x half> @bitcast_v15i64_to_v60f16(<15 x i64> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15i64_to_v60f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; kill: killed $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; kill: killed $vgpr36
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v5
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v31
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v33
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v34
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v36
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v50
+; GCN-NEXT:    v_mov_b32_e32 v50, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v61
+; GCN-NEXT:    v_mov_b32_e32 v61, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v63
+; GCN-NEXT:    v_mov_b32_e32 v63, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v1
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:  .LBB22_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB22_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_addc_u32_e32 v12, vcc, 0, v12, vcc
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_addc_u32_e32 v18, vcc, 0, v18, vcc
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_addc_u32_e32 v22, vcc, 0, v22, vcc
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_addc_u32_e32 v24, vcc, 0, v24, vcc
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_addc_u32_e32 v26, vcc, 0, v26, vcc
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_addc_u32_e32 v28, vcc, 0, v28, vcc
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_addc_u32_e32 v30, vcc, 0, v30, vcc
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v52, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v23
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v61, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:  .LBB22_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v2, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v46
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v45
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v43
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v42
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v41
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v40
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v54
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v52
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v51
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v49
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v39
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v37
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v35
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v35, v30
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v33, v36, v33
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v31, v37, v31
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v62
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v48, v34
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v49, v32
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v61
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v63
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x68, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v47, v46
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 0x6c, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v56, v57, v56
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x70, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v58, v59, v58
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v58, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15i64_to_v60f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB22_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB22_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_u32_e32 v28, vcc, 3, v28
+; VI-NEXT:    v_addc_u32_e32 v29, vcc, 0, v29, vcc
+; VI-NEXT:    v_add_u32_e32 v26, vcc, 3, v26
+; VI-NEXT:    v_addc_u32_e32 v27, vcc, 0, v27, vcc
+; VI-NEXT:    v_add_u32_e32 v24, vcc, 3, v24
+; VI-NEXT:    v_addc_u32_e32 v25, vcc, 0, v25, vcc
+; VI-NEXT:    v_add_u32_e32 v22, vcc, 3, v22
+; VI-NEXT:    v_addc_u32_e32 v23, vcc, 0, v23, vcc
+; VI-NEXT:    v_add_u32_e32 v20, vcc, 3, v20
+; VI-NEXT:    v_addc_u32_e32 v21, vcc, 0, v21, vcc
+; VI-NEXT:    v_add_u32_e32 v18, vcc, 3, v18
+; VI-NEXT:    v_addc_u32_e32 v19, vcc, 0, v19, vcc
+; VI-NEXT:    v_add_u32_e32 v16, vcc, 3, v16
+; VI-NEXT:    v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; VI-NEXT:    v_add_u32_e32 v14, vcc, 3, v14
+; VI-NEXT:    v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; VI-NEXT:    v_add_u32_e32 v12, vcc, 3, v12
+; VI-NEXT:    v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v10
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 3, v6
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB22_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; VI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; VI-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; VI-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15i64_to_v60f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB22_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB22_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_co_u32_e32 v28, vcc, 3, v28
+; GFX9-NEXT:    v_addc_co_u32_e32 v29, vcc, 0, v29, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v26, vcc, 3, v26
+; GFX9-NEXT:    v_addc_co_u32_e32 v27, vcc, 0, v27, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v24, vcc, 3, v24
+; GFX9-NEXT:    v_addc_co_u32_e32 v25, vcc, 0, v25, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, 3, v22
+; GFX9-NEXT:    v_addc_co_u32_e32 v23, vcc, 0, v23, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, 3, v20
+; GFX9-NEXT:    v_addc_co_u32_e32 v21, vcc, 0, v21, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, 3, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, 0, v19, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, 3, v16
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v14, vcc, 3, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, 3, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, 3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, 3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, 3, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 3, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 3, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB22_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v58, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v57, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v56, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v47, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v46, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v45, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v44, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v43, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v42, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v41, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v40, v11, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v12, v55, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v54, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v53, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v50, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v49, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v48, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v39, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v38, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v37, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v36, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v35, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v34, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v33, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v32, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v31, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15i64_to_v60f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    ; implicit-def: $vgpr83
+; GFX11-NEXT:    ; implicit-def: $vgpr82
+; GFX11-NEXT:    ; implicit-def: $vgpr81
+; GFX11-NEXT:    ; implicit-def: $vgpr80
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB22_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB22_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_co_u32 v28, vcc_lo, v28, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v29, null, 0, v29, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v26, vcc_lo, v26, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v24, vcc_lo, v24, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v25, null, 0, v25, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v22, vcc_lo, v22, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v23, null, 0, v23, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v20, vcc_lo, v20, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v21, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v16, vcc_lo, v16, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v14, vcc_lo, v14, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v15, null, 0, v15, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v13, null, 0, v13, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v10, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 3
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB22_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v83, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v81, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v71, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v70, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v69, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v68, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v67, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v66, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v65, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v64, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v55, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v54, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v39, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v38, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v37, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v36, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v35, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v34, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v33, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v32, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v31, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v30, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <15 x i64> %a, splat (i64 3)
+  %a2 = bitcast <15 x i64> %a1 to <60 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x i64> %a to <60 x half>
+  br label %end
+
+end:
+  %phi = phi <60 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x half> %phi
+}
+
+define <15 x i64> @bitcast_v60f16_to_v15i64(<60 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60f16_to_v15i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:112
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:104
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:92
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44
+; GCN-NEXT:    s_waitcnt vmcnt(10)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v48
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v5
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v4
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v7
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v6
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v9
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v8
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v11
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v38
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v35
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v36
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v46
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v47
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v44
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v45
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v48
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v60
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v0, v58, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v56, v1
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v50
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v30, v16
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v46, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v44, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v43, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v41, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v48, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v38, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v36, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v25
+; GCN-NEXT:    v_or_b32_e32 v26, v37, v26
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v27
+; GCN-NEXT:    v_or_b32_e32 v28, v49, v28
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v29
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:  .LBB23_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB23_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v56
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v15, v16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v46
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v44
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v19, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v20, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v21, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v23, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v24, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v26, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v28, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v50
+; GCN-NEXT:  .LBB23_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60f16_to_v15i64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v29
+; VI-NEXT:    v_mov_b32_e32 v33, v28
+; VI-NEXT:    v_mov_b32_e32 v34, v27
+; VI-NEXT:    v_mov_b32_e32 v35, v26
+; VI-NEXT:    v_mov_b32_e32 v36, v25
+; VI-NEXT:    v_mov_b32_e32 v37, v24
+; VI-NEXT:    v_mov_b32_e32 v38, v23
+; VI-NEXT:    v_mov_b32_e32 v39, v22
+; VI-NEXT:    v_mov_b32_e32 v48, v21
+; VI-NEXT:    v_mov_b32_e32 v49, v20
+; VI-NEXT:    v_mov_b32_e32 v50, v19
+; VI-NEXT:    v_mov_b32_e32 v51, v18
+; VI-NEXT:    v_mov_b32_e32 v52, v17
+; VI-NEXT:    v_mov_b32_e32 v53, v16
+; VI-NEXT:    v_mov_b32_e32 v54, v15
+; VI-NEXT:    v_mov_b32_e32 v55, v14
+; VI-NEXT:    v_mov_b32_e32 v40, v13
+; VI-NEXT:    v_mov_b32_e32 v41, v12
+; VI-NEXT:    v_mov_b32_e32 v42, v11
+; VI-NEXT:    v_mov_b32_e32 v43, v10
+; VI-NEXT:    v_mov_b32_e32 v44, v9
+; VI-NEXT:    v_mov_b32_e32 v45, v8
+; VI-NEXT:    v_mov_b32_e32 v46, v7
+; VI-NEXT:    v_mov_b32_e32 v47, v6
+; VI-NEXT:    v_mov_b32_e32 v56, v5
+; VI-NEXT:    v_mov_b32_e32 v57, v4
+; VI-NEXT:    v_mov_b32_e32 v58, v3
+; VI-NEXT:    v_mov_b32_e32 v59, v2
+; VI-NEXT:    v_mov_b32_e32 v60, v1
+; VI-NEXT:    v_mov_b32_e32 v61, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v29, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v29, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v29, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v29, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v29, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v29, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v29, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v29, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v29, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v29, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v29, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v29, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v29, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v29, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v29, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v29, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v29, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v29, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v29, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v29, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v29, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v29, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v29, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v29, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v29, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v29, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v29, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v28, v29, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v29, v29, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v60, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v59, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v58, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v57, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v56, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v47, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v46, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v44, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v43, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v41, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v40, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v55, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v54, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v53, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v51, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v50, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v38, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v36, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v35, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v34, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v33, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v32, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr60
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB23_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB23_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v29, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v61, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v61
+; VI-NEXT:    v_add_f16_sdwa v2, v60, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v60
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v59, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v59
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v58, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v58
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v57, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v56, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v47, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v46, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v45, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v44, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v43, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v42, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v41, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v40, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v55, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v54, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v53, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v52, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v51, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v50, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v49, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v48, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v39, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v38, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v37, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v24, v25, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v36, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:    v_add_f16_sdwa v26, v35, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v26, v27, v26
+; VI-NEXT:    v_add_f16_sdwa v27, v34, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:    v_add_f16_sdwa v28, v33, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v28, v30, v28
+; VI-NEXT:    v_add_f16_sdwa v29, v32, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v29, v30, v29
+; VI-NEXT:  .LBB23_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60f16_to_v15i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v61, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GFX9-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v39, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v48, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    v_mov_b32_e32 v43, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_mov_b32_e32 v44, v9
+; GFX9-NEXT:    v_mov_b32_e32 v45, v8
+; GFX9-NEXT:    v_mov_b32_e32 v46, v7
+; GFX9-NEXT:    v_mov_b32_e32 v47, v6
+; GFX9-NEXT:    v_mov_b32_e32 v56, v5
+; GFX9-NEXT:    v_mov_b32_e32 v57, v4
+; GFX9-NEXT:    v_mov_b32_e32 v58, v3
+; GFX9-NEXT:    v_mov_b32_e32 v59, v2
+; GFX9-NEXT:    v_mov_b32_e32 v60, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v43
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v59
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v60
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v61
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v39
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:  .LBB23_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB23_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v28, v28, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    v_pk_add_f16 v29, v29, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB23_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60f16_to_v15i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    v_perm_b32 v0, v81, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v83, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v84, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v80, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v71, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v70, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v69, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v68, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v67, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v66, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v65, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v64, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v55, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v54, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v53, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v52, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v51, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v50, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v49, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v48, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v39, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v38, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v37, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v36, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v35, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v34, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v33, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v32, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v31, v29, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB23_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v28, 0x200, v28 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v29, 0x200, v29 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB23_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <60 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <60 x half> %a1 to <15 x i64>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x half> %a to <15 x i64>
+  br label %end
+
+end:
+  %phi = phi <15 x i64> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x i64> %phi
+}
+
+define <60 x i16> @bitcast_v15f64_to_v60i16(<15 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15f64_to_v60i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_alignbit_b32 v31, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v14
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v12
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v2
+; GCN-NEXT:  .LBB24_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB24_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_add_f64 v[25:26], v[25:26], 1.0
+; GCN-NEXT:    v_add_f64 v[27:28], v[27:28], 1.0
+; GCN-NEXT:    v_add_f64 v[29:30], v[29:30], 1.0
+; GCN-NEXT:    v_alignbit_b32 v31, v30, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v32, v28, v27, 16
+; GCN-NEXT:    v_alignbit_b32 v33, v26, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v34, v24, v23, 16
+; GCN-NEXT:    v_alignbit_b32 v35, v22, v21, 16
+; GCN-NEXT:    v_alignbit_b32 v36, v20, v19, 16
+; GCN-NEXT:    v_alignbit_b32 v37, v18, v17, 16
+; GCN-NEXT:    v_alignbit_b32 v38, v16, v15, 16
+; GCN-NEXT:    v_alignbit_b32 v39, v14, v13, 16
+; GCN-NEXT:    v_alignbit_b32 v48, v12, v11, 16
+; GCN-NEXT:    v_alignbit_b32 v49, v10, v9, 16
+; GCN-NEXT:    v_alignbit_b32 v52, v8, v7, 16
+; GCN-NEXT:    v_alignbit_b32 v55, v6, v5, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v4, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v2, v1, 16
+; GCN-NEXT:    v_lshrrev_b32_e32 v50, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v40, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v16
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v14
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v12
+; GCN-NEXT:    s_waitcnt expcnt(4)
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v10
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v2
+; GCN-NEXT:  .LBB24_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_and_b32_e32 v30, 0xffff, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v60
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v44
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 4, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 8, v0
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v59
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 12, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v55
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v58
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 20, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 24, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v57
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 28, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v49
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 32, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v56
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 36, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v48
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 40, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v47
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v39
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v46
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 52, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v38
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 56, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v45
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v37
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v43
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x44, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v36
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 0x48, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v42
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 0x4c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v35
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x50, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v40
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 0x54, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v34
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 0x58, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v54
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x5c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v33
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 0x60, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x64, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 0x68, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v51
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x6c, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v31
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 0x70, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v3, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v46, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v40, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v34, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15f64_to_v60i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB24_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB24_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB24_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; VI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; VI-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; VI-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15f64_to_v60i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB24_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB24_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB24_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v58, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v57, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v56, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v47, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v46, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v45, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v44, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v43, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v42, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v41, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v40, v11, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v12, v55, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v54, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v53, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v50, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v49, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v48, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v39, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v38, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v37, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v36, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v35, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v34, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v33, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v32, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v31, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15f64_to_v60i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    ; implicit-def: $vgpr83
+; GFX11-NEXT:    ; implicit-def: $vgpr82
+; GFX11-NEXT:    ; implicit-def: $vgpr81
+; GFX11-NEXT:    ; implicit-def: $vgpr80
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB24_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB24_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB24_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v83, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v81, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v71, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v70, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v69, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v68, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v67, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v66, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v65, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v64, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v55, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v54, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v39, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v38, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v37, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v36, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v35, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v34, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v33, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v32, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v31, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v30, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <15 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <15 x double> %a1 to <60 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x double> %a to <60 x i16>
+  br label %end
+
+end:
+  %phi = phi <60 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x i16> %phi
+}
+
+define <15 x double> @bitcast_v60i16_to_v15f64(<60 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60i16_to_v15f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    v_mov_b32_e32 v37, v20
+; GCN-NEXT:    v_mov_b32_e32 v38, v18
+; GCN-NEXT:    v_mov_b32_e32 v39, v16
+; GCN-NEXT:    v_mov_b32_e32 v48, v14
+; GCN-NEXT:    v_mov_b32_e32 v49, v12
+; GCN-NEXT:    v_mov_b32_e32 v50, v10
+; GCN-NEXT:    v_mov_b32_e32 v51, v8
+; GCN-NEXT:    v_mov_b32_e32 v52, v6
+; GCN-NEXT:    v_mov_b32_e32 v53, v4
+; GCN-NEXT:    v_mov_b32_e32 v54, v2
+; GCN-NEXT:    v_mov_b32_e32 v55, v0
+; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:112
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:104
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:92
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:84
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v63, 16, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v62, 16, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v61, 16, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v60, 16, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v59, 16, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v57, 16, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(6) expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v55
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v42
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v53
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v33
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v52
+; GCN-NEXT:    v_or_b32_e32 v3, v3, v32
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v51
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v41
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v50
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v63
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v49
+; GCN-NEXT:    v_or_b32_e32 v6, v6, v62
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v48
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v61
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v39
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v60
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v38
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v59
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v37
+; GCN-NEXT:    v_or_b32_e32 v10, v10, v58
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v57
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v12, v47
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v14, v15
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v16
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v44
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v35
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v46
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v36
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v45
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v43
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v40
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v56
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v16, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v18, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v20, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v22, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v24, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v25, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v26, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v27, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v30
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v29, v29, v30
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:  .LBB25_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB25_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
+; GCN-NEXT:    s_mov_b32 s6, 0x30000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v37
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v44
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v46
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v45
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v43
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v40
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v56
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GCN-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; GCN-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; GCN-NEXT:    v_and_b32_e32 v26, 0xffff, v26
+; GCN-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; GCN-NEXT:    v_or_b32_e32 v0, v34, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v42, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v32, v3
+; GCN-NEXT:    v_or_b32_e32 v4, v41, v4
+; GCN-NEXT:    v_or_b32_e32 v5, v63, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v62, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v61, v7
+; GCN-NEXT:    v_or_b32_e32 v8, v60, v8
+; GCN-NEXT:    v_or_b32_e32 v9, v59, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v58, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v57, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v47, v12
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v30, v13
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v30, v14
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v30, v15
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v30, v16
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v17
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v18, v30, v18
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v19, v30, v19
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v20, v30, v20
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v21, v30, v21
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v22, v30, v22
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v23, v30, v23
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v24, v30, v24
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v25, v30, v25
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v26, v30, v26
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v27, v30, v27
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v28, v30, v28
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v29, v30, v29
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 0x30000, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 0x30000, v29
+; GCN-NEXT:  .LBB25_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60i16_to_v15f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v29
+; VI-NEXT:    v_mov_b32_e32 v33, v28
+; VI-NEXT:    v_mov_b32_e32 v34, v27
+; VI-NEXT:    v_mov_b32_e32 v35, v26
+; VI-NEXT:    v_mov_b32_e32 v36, v25
+; VI-NEXT:    v_mov_b32_e32 v37, v24
+; VI-NEXT:    v_mov_b32_e32 v38, v23
+; VI-NEXT:    v_mov_b32_e32 v39, v22
+; VI-NEXT:    v_mov_b32_e32 v48, v21
+; VI-NEXT:    v_mov_b32_e32 v49, v20
+; VI-NEXT:    v_mov_b32_e32 v50, v19
+; VI-NEXT:    v_mov_b32_e32 v51, v18
+; VI-NEXT:    v_mov_b32_e32 v52, v17
+; VI-NEXT:    v_mov_b32_e32 v53, v16
+; VI-NEXT:    v_mov_b32_e32 v54, v15
+; VI-NEXT:    v_mov_b32_e32 v55, v14
+; VI-NEXT:    v_mov_b32_e32 v40, v13
+; VI-NEXT:    v_mov_b32_e32 v41, v12
+; VI-NEXT:    v_mov_b32_e32 v42, v11
+; VI-NEXT:    v_mov_b32_e32 v43, v10
+; VI-NEXT:    v_mov_b32_e32 v44, v9
+; VI-NEXT:    v_mov_b32_e32 v45, v8
+; VI-NEXT:    v_mov_b32_e32 v46, v7
+; VI-NEXT:    v_mov_b32_e32 v47, v6
+; VI-NEXT:    v_mov_b32_e32 v56, v5
+; VI-NEXT:    v_mov_b32_e32 v57, v4
+; VI-NEXT:    v_mov_b32_e32 v58, v3
+; VI-NEXT:    v_mov_b32_e32 v59, v2
+; VI-NEXT:    v_mov_b32_e32 v60, v1
+; VI-NEXT:    v_mov_b32_e32 v61, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v29, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v29, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v29, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v29, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v29, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v29, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v29, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v29, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v29, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v29, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v29, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v29, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v29, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v29, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v29, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v29, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v29, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v29, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v29, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v29, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v29, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v29, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v29, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v29, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v29, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v29, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v29, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v28, v29, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v29, v29, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v60, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v59, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v58, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v57, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v56, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v47, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v46, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v44, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v43, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v41, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v40, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v55, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v54, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v53, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v51, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v50, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v38, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v36, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v35, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v34, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v33, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v32, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr60
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB25_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB25_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v29, 3
+; VI-NEXT:    v_add_u16_e32 v0, 3, v61
+; VI-NEXT:    v_add_u16_sdwa v1, v61, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v2, 3, v60
+; VI-NEXT:    v_add_u16_sdwa v3, v60, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    v_or_b32_e32 v1, v2, v3
+; VI-NEXT:    v_add_u16_e32 v2, 3, v59
+; VI-NEXT:    v_add_u16_sdwa v3, v59, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v2, v2, v3
+; VI-NEXT:    v_add_u16_e32 v3, 3, v58
+; VI-NEXT:    v_add_u16_sdwa v4, v58, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v3, v3, v4
+; VI-NEXT:    v_add_u16_e32 v4, 3, v57
+; VI-NEXT:    v_add_u16_sdwa v5, v57, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v4, v4, v5
+; VI-NEXT:    v_add_u16_e32 v5, 3, v56
+; VI-NEXT:    v_add_u16_sdwa v6, v56, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v5, v5, v6
+; VI-NEXT:    v_add_u16_e32 v6, 3, v47
+; VI-NEXT:    v_add_u16_sdwa v7, v47, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v6, v6, v7
+; VI-NEXT:    v_add_u16_e32 v7, 3, v46
+; VI-NEXT:    v_add_u16_sdwa v8, v46, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v7, v7, v8
+; VI-NEXT:    v_add_u16_e32 v8, 3, v45
+; VI-NEXT:    v_add_u16_sdwa v9, v45, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v8, v8, v9
+; VI-NEXT:    v_add_u16_e32 v9, 3, v44
+; VI-NEXT:    v_add_u16_sdwa v10, v44, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v9, v9, v10
+; VI-NEXT:    v_add_u16_e32 v10, 3, v43
+; VI-NEXT:    v_add_u16_sdwa v11, v43, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v10, v10, v11
+; VI-NEXT:    v_add_u16_e32 v11, 3, v42
+; VI-NEXT:    v_add_u16_sdwa v12, v42, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v11, v11, v12
+; VI-NEXT:    v_add_u16_e32 v12, 3, v41
+; VI-NEXT:    v_add_u16_sdwa v13, v41, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v12, v12, v13
+; VI-NEXT:    v_add_u16_e32 v13, 3, v40
+; VI-NEXT:    v_add_u16_sdwa v14, v40, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v13, v13, v14
+; VI-NEXT:    v_add_u16_e32 v14, 3, v55
+; VI-NEXT:    v_add_u16_sdwa v15, v55, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v14, v14, v15
+; VI-NEXT:    v_add_u16_e32 v15, 3, v54
+; VI-NEXT:    v_add_u16_sdwa v16, v54, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
+; VI-NEXT:    v_add_u16_e32 v16, 3, v53
+; VI-NEXT:    v_add_u16_sdwa v17, v53, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v16, v16, v17
+; VI-NEXT:    v_add_u16_e32 v17, 3, v52
+; VI-NEXT:    v_add_u16_sdwa v18, v52, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v17, v17, v18
+; VI-NEXT:    v_add_u16_e32 v18, 3, v51
+; VI-NEXT:    v_add_u16_sdwa v19, v51, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v18, v18, v19
+; VI-NEXT:    v_add_u16_e32 v19, 3, v50
+; VI-NEXT:    v_add_u16_sdwa v20, v50, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v19, v19, v20
+; VI-NEXT:    v_add_u16_e32 v20, 3, v49
+; VI-NEXT:    v_add_u16_sdwa v21, v49, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v20, v20, v21
+; VI-NEXT:    v_add_u16_e32 v21, 3, v48
+; VI-NEXT:    v_add_u16_sdwa v22, v48, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v21, v21, v22
+; VI-NEXT:    v_add_u16_e32 v22, 3, v39
+; VI-NEXT:    v_add_u16_sdwa v23, v39, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v22, v22, v23
+; VI-NEXT:    v_add_u16_e32 v23, 3, v38
+; VI-NEXT:    v_add_u16_sdwa v24, v38, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v23, v23, v24
+; VI-NEXT:    v_add_u16_e32 v24, 3, v37
+; VI-NEXT:    v_add_u16_sdwa v25, v37, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v24, v24, v25
+; VI-NEXT:    v_add_u16_e32 v25, 3, v36
+; VI-NEXT:    v_add_u16_sdwa v26, v36, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v25, v25, v26
+; VI-NEXT:    v_add_u16_e32 v26, 3, v35
+; VI-NEXT:    v_add_u16_sdwa v27, v35, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v26, v26, v27
+; VI-NEXT:    v_add_u16_e32 v27, 3, v34
+; VI-NEXT:    v_add_u16_sdwa v28, v34, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v27, v27, v28
+; VI-NEXT:    v_add_u16_e32 v28, 3, v33
+; VI-NEXT:    v_add_u16_sdwa v30, v33, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v28, v28, v30
+; VI-NEXT:    v_add_u16_e32 v30, 3, v32
+; VI-NEXT:    v_add_u16_sdwa v29, v32, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v29, v30, v29
+; VI-NEXT:  .LBB25_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60i16_to_v15f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v61, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GFX9-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v39, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v48, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    v_mov_b32_e32 v43, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_mov_b32_e32 v44, v9
+; GFX9-NEXT:    v_mov_b32_e32 v45, v8
+; GFX9-NEXT:    v_mov_b32_e32 v46, v7
+; GFX9-NEXT:    v_mov_b32_e32 v47, v6
+; GFX9-NEXT:    v_mov_b32_e32 v56, v5
+; GFX9-NEXT:    v_mov_b32_e32 v57, v4
+; GFX9-NEXT:    v_mov_b32_e32 v58, v3
+; GFX9-NEXT:    v_mov_b32_e32 v59, v2
+; GFX9-NEXT:    v_mov_b32_e32 v60, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v43
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v59
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v60
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v61
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v39
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:  .LBB25_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB25_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB25_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60i16_to_v15f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    v_perm_b32 v0, v81, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v83, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v84, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v80, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v71, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v70, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v69, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v68, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v67, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v66, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v65, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v64, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v55, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v54, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v53, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v52, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v51, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v50, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v49, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v48, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v39, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v38, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v37, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v36, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v35, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v34, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v33, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v32, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v31, v29, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB25_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:  .LBB25_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <60 x i16> %a, splat (i16 3)
+  %a2 = bitcast <60 x i16> %a1 to <15 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x i16> %a to <15 x double>
+  br label %end
+
+end:
+  %phi = phi <15 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x double> %phi
+}
+
+define <60 x half> @bitcast_v15f64_to_v60f16(<15 x double> %a, i32 %b) {
+; GCN-LABEL: bitcast_v15f64_to_v60f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; kill: killed $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; kill: killed $vgpr38
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v30
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v21
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v19
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v9
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
+; GCN-NEXT:    s_waitcnt expcnt(5)
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v7
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v6
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v3
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v58
+; GCN-NEXT:    v_mov_b32_e32 v58, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v60
+; GCN-NEXT:    v_mov_b32_e32 v60, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v62
+; GCN-NEXT:    v_mov_b32_e32 v62, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v63
+; GCN-NEXT:    v_mov_b32_e32 v63, v5
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v3
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:  .LBB26_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB26_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_f64 v[1:2], v[1:2], 1.0
+; GCN-NEXT:    v_add_f64 v[3:4], v[3:4], 1.0
+; GCN-NEXT:    v_add_f64 v[5:6], v[5:6], 1.0
+; GCN-NEXT:    v_add_f64 v[7:8], v[7:8], 1.0
+; GCN-NEXT:    v_add_f64 v[9:10], v[9:10], 1.0
+; GCN-NEXT:    v_add_f64 v[11:12], v[11:12], 1.0
+; GCN-NEXT:    v_add_f64 v[13:14], v[13:14], 1.0
+; GCN-NEXT:    v_add_f64 v[15:16], v[15:16], 1.0
+; GCN-NEXT:    v_add_f64 v[17:18], v[17:18], 1.0
+; GCN-NEXT:    v_add_f64 v[19:20], v[19:20], 1.0
+; GCN-NEXT:    v_add_f64 v[21:22], v[21:22], 1.0
+; GCN-NEXT:    v_add_f64 v[23:24], v[23:24], 1.0
+; GCN-NEXT:    v_add_f64 v[25:26], v[25:26], 1.0
+; GCN-NEXT:    v_add_f64 v[27:28], v[27:28], 1.0
+; GCN-NEXT:    v_add_f64 v[29:30], v[29:30], 1.0
+; GCN-NEXT:    s_waitcnt expcnt(6)
+; GCN-NEXT:    v_lshrrev_b32_e32 v57, 16, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v47, 16, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v46, 16, v4
+; GCN-NEXT:    v_lshrrev_b32_e32 v45, 16, v5
+; GCN-NEXT:    v_lshrrev_b32_e32 v44, 16, v6
+; GCN-NEXT:    v_lshrrev_b32_e32 v43, 16, v7
+; GCN-NEXT:    v_lshrrev_b32_e32 v42, 16, v8
+; GCN-NEXT:    v_lshrrev_b32_e32 v41, 16, v9
+; GCN-NEXT:    v_lshrrev_b32_e32 v54, 16, v10
+; GCN-NEXT:    v_lshrrev_b32_e32 v53, 16, v11
+; GCN-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GCN-NEXT:    v_lshrrev_b32_e32 v49, 16, v13
+; GCN-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; GCN-NEXT:    v_lshrrev_b32_e32 v37, 16, v15
+; GCN-NEXT:    v_lshrrev_b32_e32 v35, 16, v16
+; GCN-NEXT:    v_lshrrev_b32_e32 v33, 16, v17
+; GCN-NEXT:    v_lshrrev_b32_e32 v32, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v56, 16, v19
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_lshrrev_b32_e32 v61, 16, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v59, 16, v21
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v63, 16, v22
+; GCN-NEXT:    v_lshrrev_b32_e32 v58, 16, v23
+; GCN-NEXT:    v_lshrrev_b32_e32 v60, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v62, 16, v25
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GCN-NEXT:    v_lshrrev_b32_e32 v34, 16, v27
+; GCN-NEXT:    v_lshrrev_b32_e32 v36, 16, v28
+; GCN-NEXT:    v_lshrrev_b32_e32 v38, 16, v29
+; GCN-NEXT:    v_lshrrev_b32_e32 v48, 16, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; GCN-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v52, v3
+; GCN-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v62, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v60
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v56
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v54, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v46
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; GCN-NEXT:  .LBB26_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v2, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 4, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 12, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v45
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 16, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v44
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 20, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v43
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 24, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v42
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 28, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v41
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 32, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v54
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 36, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v53
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 40, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v51
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v49
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v39
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 52, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v37
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 56, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v35
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v35, v30
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 60, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; GCN-NEXT:    v_or_b32_e32 v33, v37, v33
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 64, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v38, v32
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 0x44, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v39, v36
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x48, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v48, v34
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 0x4c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_or_b32_e32 v31, v49, v31
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x50, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v63
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v58
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v60
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x5c, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v62
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x68, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v47, v46
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 0x6c, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v56, v57, v56
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x70, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v58, v59, v58
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v33, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v48, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v58, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v15f64_to_v60f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; implicit-def: $vgpr31
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB26_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB26_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; VI-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; VI-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; VI-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; VI-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; VI-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; VI-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; VI-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; VI-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; VI-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; VI-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; VI-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; VI-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; VI-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; VI-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; VI-NEXT:  .LBB26_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v59, 16, v59
+; VI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; VI-NEXT:    v_lshlrev_b32_e32 v57, 16, v57
+; VI-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; VI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; VI-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v45, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; VI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; VI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; VI-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; VI-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; VI-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; VI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; VI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; VI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; VI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; VI-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
+; VI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v15f64_to_v60f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB26_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB26_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX9-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX9-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX9-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX9-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX9-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX9-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX9-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX9-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX9-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX9-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX9-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v0
+; GFX9-NEXT:  .LBB26_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v59, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v58, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v57, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v56, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v47, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v46, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v45, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v44, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v43, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v42, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v41, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v40, v11, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v12, v55, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v54, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v53, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v52, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v51, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v50, v17, s4
+; GFX9-NEXT:    v_perm_b32 v18, v49, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v48, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v39, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v38, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v37, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v36, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v35, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v34, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v33, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v32, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v31, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v15f64_to_v60f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    ; implicit-def: $vgpr83
+; GFX11-NEXT:    ; implicit-def: $vgpr82
+; GFX11-NEXT:    ; implicit-def: $vgpr81
+; GFX11-NEXT:    ; implicit-def: $vgpr80
+; GFX11-NEXT:    ; implicit-def: $vgpr71
+; GFX11-NEXT:    ; implicit-def: $vgpr70
+; GFX11-NEXT:    ; implicit-def: $vgpr69
+; GFX11-NEXT:    ; implicit-def: $vgpr68
+; GFX11-NEXT:    ; implicit-def: $vgpr67
+; GFX11-NEXT:    ; implicit-def: $vgpr66
+; GFX11-NEXT:    ; implicit-def: $vgpr65
+; GFX11-NEXT:    ; implicit-def: $vgpr64
+; GFX11-NEXT:    ; implicit-def: $vgpr55
+; GFX11-NEXT:    ; implicit-def: $vgpr54
+; GFX11-NEXT:    ; implicit-def: $vgpr53
+; GFX11-NEXT:    ; implicit-def: $vgpr52
+; GFX11-NEXT:    ; implicit-def: $vgpr51
+; GFX11-NEXT:    ; implicit-def: $vgpr50
+; GFX11-NEXT:    ; implicit-def: $vgpr49
+; GFX11-NEXT:    ; implicit-def: $vgpr48
+; GFX11-NEXT:    ; implicit-def: $vgpr39
+; GFX11-NEXT:    ; implicit-def: $vgpr38
+; GFX11-NEXT:    ; implicit-def: $vgpr37
+; GFX11-NEXT:    ; implicit-def: $vgpr36
+; GFX11-NEXT:    ; implicit-def: $vgpr35
+; GFX11-NEXT:    ; implicit-def: $vgpr34
+; GFX11-NEXT:    ; implicit-def: $vgpr33
+; GFX11-NEXT:    ; implicit-def: $vgpr32
+; GFX11-NEXT:    ; implicit-def: $vgpr31
+; GFX11-NEXT:    ; implicit-def: $vgpr30
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB26_2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB26_4
+; GFX11-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-NEXT:    v_add_f64 v[28:29], v[28:29], 1.0
+; GFX11-NEXT:    v_add_f64 v[26:27], v[26:27], 1.0
+; GFX11-NEXT:    v_add_f64 v[24:25], v[24:25], 1.0
+; GFX11-NEXT:    v_add_f64 v[22:23], v[22:23], 1.0
+; GFX11-NEXT:    v_add_f64 v[20:21], v[20:21], 1.0
+; GFX11-NEXT:    v_add_f64 v[18:19], v[18:19], 1.0
+; GFX11-NEXT:    v_add_f64 v[16:17], v[16:17], 1.0
+; GFX11-NEXT:    v_add_f64 v[14:15], v[14:15], 1.0
+; GFX11-NEXT:    v_add_f64 v[12:13], v[12:13], 1.0
+; GFX11-NEXT:    v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-NEXT:    v_add_f64 v[8:9], v[8:9], 1.0
+; GFX11-NEXT:    v_add_f64 v[6:7], v[6:7], 1.0
+; GFX11-NEXT:    v_add_f64 v[4:5], v[4:5], 1.0
+; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
+; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX11-NEXT:  .LBB26_4: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v0, v83, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v81, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v80, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v71, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v70, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v69, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v68, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v67, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v66, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v65, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v64, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v55, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v54, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v52, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v51, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v50, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v49, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v48, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v39, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v38, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v37, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v36, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v35, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v34, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v33, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v32, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v31, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v30, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <15 x double> %a, splat (double 1.000000e+00)
+  %a2 = bitcast <15 x double> %a1 to <60 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <15 x double> %a to <60 x half>
+  br label %end
+
+end:
+  %phi = phi <60 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x half> %phi
+}
+
+define <15 x double> @bitcast_v60f16_to_v15f64(<60 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60f16_to_v15f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:112
+; GCN-NEXT:    s_waitcnt expcnt(3)
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:104
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:100
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:92
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44
+; GCN-NEXT:    s_waitcnt vmcnt(10)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v48
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v5
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v4
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v7
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v6
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v9
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v8
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v11
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v10
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v13
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v12
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v15
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v14
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v17
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v16
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v19
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v18
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v21
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v20
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v23
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v22
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v25
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v24
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v26
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v29
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v28
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v30
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v38
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v35
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v36
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v46
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v47
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v44
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v45
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v48
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v61
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v60
+; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v57
+; GCN-NEXT:    v_or_b32_e32 v0, v58, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v56, v1
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v15, v16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v21, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v23, 16, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v50
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v16, v30, v16
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v17, v30, v17
+; GCN-NEXT:    v_or_b32_e32 v18, v46, v18
+; GCN-NEXT:    v_or_b32_e32 v19, v44, v19
+; GCN-NEXT:    v_or_b32_e32 v20, v43, v20
+; GCN-NEXT:    v_or_b32_e32 v21, v41, v21
+; GCN-NEXT:    v_or_b32_e32 v22, v48, v22
+; GCN-NEXT:    v_or_b32_e32 v23, v38, v23
+; GCN-NEXT:    v_or_b32_e32 v24, v36, v24
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v25
+; GCN-NEXT:    v_or_b32_e32 v26, v37, v26
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v27
+; GCN-NEXT:    v_or_b32_e32 v28, v49, v28
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v29
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; kill: killed $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:  .LBB27_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB27_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v59
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v57
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v56
+; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v5, v6, v5
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GCN-NEXT:    v_or_b32_e32 v7, v8, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GCN-NEXT:    v_or_b32_e32 v9, v10, v9
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GCN-NEXT:    v_or_b32_e32 v11, v12, v11
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GCN-NEXT:    v_or_b32_e32 v13, v14, v13
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GCN-NEXT:    v_or_b32_e32 v15, v16, v15
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v46
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v44
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v43
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v45
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v48
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v40
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_or_b32_e32 v17, v19, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v21, v20
+; GCN-NEXT:    v_or_b32_e32 v19, v23, v22
+; GCN-NEXT:    v_or_b32_e32 v20, v25, v24
+; GCN-NEXT:    v_or_b32_e32 v21, v27, v26
+; GCN-NEXT:    v_or_b32_e32 v22, v29, v28
+; GCN-NEXT:    v_or_b32_e32 v23, v31, v30
+; GCN-NEXT:    v_or_b32_e32 v24, v33, v32
+; GCN-NEXT:    v_or_b32_e32 v25, v35, v34
+; GCN-NEXT:    v_or_b32_e32 v26, v37, v36
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v38
+; GCN-NEXT:    v_or_b32_e32 v28, v49, v48
+; GCN-NEXT:    v_or_b32_e32 v29, v51, v50
+; GCN-NEXT:  .LBB27_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60f16_to_v15f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_mov_b32_e32 v32, v29
+; VI-NEXT:    v_mov_b32_e32 v33, v28
+; VI-NEXT:    v_mov_b32_e32 v34, v27
+; VI-NEXT:    v_mov_b32_e32 v35, v26
+; VI-NEXT:    v_mov_b32_e32 v36, v25
+; VI-NEXT:    v_mov_b32_e32 v37, v24
+; VI-NEXT:    v_mov_b32_e32 v38, v23
+; VI-NEXT:    v_mov_b32_e32 v39, v22
+; VI-NEXT:    v_mov_b32_e32 v48, v21
+; VI-NEXT:    v_mov_b32_e32 v49, v20
+; VI-NEXT:    v_mov_b32_e32 v50, v19
+; VI-NEXT:    v_mov_b32_e32 v51, v18
+; VI-NEXT:    v_mov_b32_e32 v52, v17
+; VI-NEXT:    v_mov_b32_e32 v53, v16
+; VI-NEXT:    v_mov_b32_e32 v54, v15
+; VI-NEXT:    v_mov_b32_e32 v55, v14
+; VI-NEXT:    v_mov_b32_e32 v40, v13
+; VI-NEXT:    v_mov_b32_e32 v41, v12
+; VI-NEXT:    v_mov_b32_e32 v42, v11
+; VI-NEXT:    v_mov_b32_e32 v43, v10
+; VI-NEXT:    v_mov_b32_e32 v44, v9
+; VI-NEXT:    v_mov_b32_e32 v45, v8
+; VI-NEXT:    v_mov_b32_e32 v46, v7
+; VI-NEXT:    v_mov_b32_e32 v47, v6
+; VI-NEXT:    v_mov_b32_e32 v56, v5
+; VI-NEXT:    v_mov_b32_e32 v57, v4
+; VI-NEXT:    v_mov_b32_e32 v58, v3
+; VI-NEXT:    v_mov_b32_e32 v59, v2
+; VI-NEXT:    v_mov_b32_e32 v60, v1
+; VI-NEXT:    v_mov_b32_e32 v61, v0
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_2
+; VI-NEXT:  ; %bb.1: ; %cmp.false
+; VI-NEXT:    v_mov_b32_e32 v29, 16
+; VI-NEXT:    v_lshlrev_b32_sdwa v0, v29, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v1, v29, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v2, v29, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v3, v29, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v4, v29, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v5, v29, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v6, v29, v47 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v7, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v8, v29, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v9, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v10, v29, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v11, v29, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v12, v29, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v13, v29, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v14, v29, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v15, v29, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v16, v29, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v17, v29, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v18, v29, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v19, v29, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v20, v29, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v21, v29, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v22, v29, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v23, v29, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v24, v29, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v25, v29, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v26, v29, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v27, v29, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v28, v29, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_lshlrev_b32_sdwa v29, v29, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v60, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v59, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v58, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v57, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v5, v56, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v47, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v46, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v44, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v43, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v41, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v40, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v55, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v54, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v53, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v17, v52, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v18, v51, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v19, v50, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v23, v38, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v25, v36, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v26, v35, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v27, v34, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v28, v33, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v29, v32, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr60
+; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr44
+; VI-NEXT:    ; implicit-def: $vgpr43
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr52
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr36
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:  .LBB27_2: ; %Flow
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB27_4
+; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    v_mov_b32_e32 v29, 0x200
+; VI-NEXT:    v_add_f16_sdwa v0, v61, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v61
+; VI-NEXT:    v_add_f16_sdwa v2, v60, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v60
+; VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; VI-NEXT:    v_or_b32_e32 v1, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v2, v59, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v59
+; VI-NEXT:    v_or_b32_e32 v2, v3, v2
+; VI-NEXT:    v_add_f16_sdwa v3, v58, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v58
+; VI-NEXT:    v_or_b32_e32 v3, v4, v3
+; VI-NEXT:    v_add_f16_sdwa v4, v57, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v57
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_add_f16_sdwa v5, v56, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v56
+; VI-NEXT:    v_or_b32_e32 v5, v6, v5
+; VI-NEXT:    v_add_f16_sdwa v6, v47, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v47
+; VI-NEXT:    v_or_b32_e32 v6, v7, v6
+; VI-NEXT:    v_add_f16_sdwa v7, v46, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v46
+; VI-NEXT:    v_or_b32_e32 v7, v8, v7
+; VI-NEXT:    v_add_f16_sdwa v8, v45, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v45
+; VI-NEXT:    v_or_b32_e32 v8, v9, v8
+; VI-NEXT:    v_add_f16_sdwa v9, v44, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v44
+; VI-NEXT:    v_or_b32_e32 v9, v10, v9
+; VI-NEXT:    v_add_f16_sdwa v10, v43, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v43
+; VI-NEXT:    v_or_b32_e32 v10, v11, v10
+; VI-NEXT:    v_add_f16_sdwa v11, v42, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v42
+; VI-NEXT:    v_or_b32_e32 v11, v12, v11
+; VI-NEXT:    v_add_f16_sdwa v12, v41, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v41
+; VI-NEXT:    v_or_b32_e32 v12, v13, v12
+; VI-NEXT:    v_add_f16_sdwa v13, v40, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v40
+; VI-NEXT:    v_or_b32_e32 v13, v14, v13
+; VI-NEXT:    v_add_f16_sdwa v14, v55, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v55
+; VI-NEXT:    v_or_b32_e32 v14, v15, v14
+; VI-NEXT:    v_add_f16_sdwa v15, v54, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v54
+; VI-NEXT:    v_or_b32_e32 v15, v16, v15
+; VI-NEXT:    v_add_f16_sdwa v16, v53, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v53
+; VI-NEXT:    v_or_b32_e32 v16, v17, v16
+; VI-NEXT:    v_add_f16_sdwa v17, v52, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v52
+; VI-NEXT:    v_or_b32_e32 v17, v18, v17
+; VI-NEXT:    v_add_f16_sdwa v18, v51, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v51
+; VI-NEXT:    v_or_b32_e32 v18, v19, v18
+; VI-NEXT:    v_add_f16_sdwa v19, v50, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v50
+; VI-NEXT:    v_or_b32_e32 v19, v20, v19
+; VI-NEXT:    v_add_f16_sdwa v20, v49, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v49
+; VI-NEXT:    v_or_b32_e32 v20, v21, v20
+; VI-NEXT:    v_add_f16_sdwa v21, v48, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v48
+; VI-NEXT:    v_or_b32_e32 v21, v22, v21
+; VI-NEXT:    v_add_f16_sdwa v22, v39, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v39
+; VI-NEXT:    v_or_b32_e32 v22, v23, v22
+; VI-NEXT:    v_add_f16_sdwa v23, v38, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v38
+; VI-NEXT:    v_or_b32_e32 v23, v24, v23
+; VI-NEXT:    v_add_f16_sdwa v24, v37, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v37
+; VI-NEXT:    v_or_b32_e32 v24, v25, v24
+; VI-NEXT:    v_add_f16_sdwa v25, v36, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v36
+; VI-NEXT:    v_or_b32_e32 v25, v26, v25
+; VI-NEXT:    v_add_f16_sdwa v26, v35, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v35
+; VI-NEXT:    v_or_b32_e32 v26, v27, v26
+; VI-NEXT:    v_add_f16_sdwa v27, v34, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v34
+; VI-NEXT:    v_or_b32_e32 v27, v28, v27
+; VI-NEXT:    v_add_f16_sdwa v28, v33, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v33
+; VI-NEXT:    v_or_b32_e32 v28, v30, v28
+; VI-NEXT:    v_add_f16_sdwa v29, v32, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v32
+; VI-NEXT:    v_or_b32_e32 v29, v30, v29
+; VI-NEXT:  .LBB27_4: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60f16_to_v15f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_mov_b32_e32 v61, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v29
+; GFX9-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v25
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v24
+; GFX9-NEXT:    v_mov_b32_e32 v39, v22
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v23
+; GFX9-NEXT:    v_mov_b32_e32 v48, v21
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v39
+; GFX9-NEXT:    v_mov_b32_e32 v49, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v48
+; GFX9-NEXT:    v_mov_b32_e32 v50, v19
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v49
+; GFX9-NEXT:    v_mov_b32_e32 v51, v18
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v50
+; GFX9-NEXT:    v_mov_b32_e32 v52, v17
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v51
+; GFX9-NEXT:    v_mov_b32_e32 v53, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v52
+; GFX9-NEXT:    v_mov_b32_e32 v54, v15
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v53
+; GFX9-NEXT:    v_mov_b32_e32 v55, v14
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v54
+; GFX9-NEXT:    v_mov_b32_e32 v40, v13
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v55
+; GFX9-NEXT:    v_mov_b32_e32 v41, v12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT:    v_mov_b32_e32 v42, v11
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT:    v_mov_b32_e32 v43, v10
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v42
+; GFX9-NEXT:    v_mov_b32_e32 v44, v9
+; GFX9-NEXT:    v_mov_b32_e32 v45, v8
+; GFX9-NEXT:    v_mov_b32_e32 v46, v7
+; GFX9-NEXT:    v_mov_b32_e32 v47, v6
+; GFX9-NEXT:    v_mov_b32_e32 v56, v5
+; GFX9-NEXT:    v_mov_b32_e32 v57, v4
+; GFX9-NEXT:    v_mov_b32_e32 v58, v3
+; GFX9-NEXT:    v_mov_b32_e32 v59, v2
+; GFX9-NEXT:    v_mov_b32_e32 v60, v1
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v43
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v45
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v46
+; GFX9-NEXT:    v_lshrrev_b32_e32 v63, 16, v47
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v56
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v57
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v58
+; GFX9-NEXT:    v_lshrrev_b32_e32 v62, 16, v59
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v60
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v61
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.false
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v44
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v43
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v42
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v41
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v40
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v55
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v54
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v53
+; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v52
+; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v51
+; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v50
+; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v49
+; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v48
+; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v39
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr60
+; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr56
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr44
+; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; kill: killed $vgpr30
+; GFX9-NEXT:  .LBB27_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB27_4
+; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v0, v38, v61, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v1, v37, v60, s6
+; GFX9-NEXT:    v_perm_b32 v2, v62, v59, s6
+; GFX9-NEXT:    v_perm_b32 v3, v36, v58, s6
+; GFX9-NEXT:    v_perm_b32 v4, v35, v57, s6
+; GFX9-NEXT:    v_perm_b32 v5, v34, v56, s6
+; GFX9-NEXT:    v_perm_b32 v6, v63, v47, s6
+; GFX9-NEXT:    v_perm_b32 v7, v33, v46, s6
+; GFX9-NEXT:    v_perm_b32 v8, v32, v45, s6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(15)
+; GFX9-NEXT:    v_perm_b32 v9, v9, v44, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_perm_b32 v10, v10, v43, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_perm_b32 v11, v11, v42, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    v_perm_b32 v12, v12, v41, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
+; GFX9-NEXT:    v_perm_b32 v13, v13, v40, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_perm_b32 v14, v14, v55, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_perm_b32 v15, v15, v54, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_perm_b32 v16, v16, v53, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_perm_b32 v17, v17, v52, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_perm_b32 v18, v18, v51, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_perm_b32 v19, v19, v50, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_perm_b32 v20, v20, v49, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_perm_b32 v21, v21, v48, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_perm_b32 v22, v22, v39, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v27, v28, v27, s6
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_pk_add_f16 v28, v28, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
+; GFX9-NEXT:    v_pk_add_f16 v29, v29, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:  .LBB27_4: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60f16_to_v15f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v30
+; GFX11-NEXT:    v_perm_b32 v0, v81, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v82, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v83, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v84, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v80, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v71, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v70, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v69, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v68, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v67, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v66, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v65, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v64, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v55, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v54, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v53, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v52, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v51, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v50, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v49, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v48, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v39, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v38, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v37, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v36, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v35, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v34, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v33, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v32, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v31, v29, 0x5040100
+; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB27_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v28, 0x200, v28 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v29, 0x200, v29 op_sel_hi:[0,1]
+; GFX11-NEXT:  .LBB27_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <60 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <60 x half> %a1 to <15 x double>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x half> %a to <15 x double>
+  br label %end
+
+end:
+  %phi = phi <15 x double> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <15 x double> %phi
+}
+
+define <60 x half> @bitcast_v60i16_to_v60f16(<60 x i16> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60i16_to_v60f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:116
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:112
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:108
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:104
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:100
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:92
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:56
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:48
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:24
+; GCN-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr57
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr56
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    ; kill: killed $vgpr47
+; GCN-NEXT:    ; implicit-def: $vgpr47
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.false
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v59
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v60
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v61
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v62
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v40
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v43
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v44
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v45
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v39
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v49
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v52
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v53
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    ; implicit-def: $vgpr1
+; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    ; implicit-def: $vgpr3
+; GCN-NEXT:    ; implicit-def: $vgpr4
+; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr7
+; GCN-NEXT:    ; implicit-def: $vgpr8
+; GCN-NEXT:    ; implicit-def: $vgpr9
+; GCN-NEXT:    ; implicit-def: $vgpr10
+; GCN-NEXT:    ; implicit-def: $vgpr11
+; GCN-NEXT:    ; implicit-def: $vgpr12
+; GCN-NEXT:    ; implicit-def: $vgpr13
+; GCN-NEXT:    ; implicit-def: $vgpr14
+; GCN-NEXT:    ; implicit-def: $vgpr15
+; GCN-NEXT:    ; implicit-def: $vgpr16
+; GCN-NEXT:    ; implicit-def: $vgpr17
+; GCN-NEXT:    ; implicit-def: $vgpr18
+; GCN-NEXT:    ; implicit-def: $vgpr19
+; GCN-NEXT:    ; implicit-def: $vgpr20
+; GCN-NEXT:    ; implicit-def: $vgpr21
+; GCN-NEXT:    ; implicit-def: $vgpr22
+; GCN-NEXT:    ; implicit-def: $vgpr23
+; GCN-NEXT:    ; implicit-def: $vgpr24
+; GCN-NEXT:    ; implicit-def: $vgpr25
+; GCN-NEXT:    ; implicit-def: $vgpr26
+; GCN-NEXT:    ; implicit-def: $vgpr27
+; GCN-NEXT:    ; implicit-def: $vgpr28
+; GCN-NEXT:    ; implicit-def: $vgpr29
+; GCN-NEXT:    ; implicit-def: $vgpr30
+; GCN-NEXT:    ; implicit-def: $vgpr58
+; GCN-NEXT:    ; implicit-def: $vgpr59
+; GCN-NEXT:    ; implicit-def: $vgpr60
+; GCN-NEXT:    ; implicit-def: $vgpr61
+; GCN-NEXT:    ; implicit-def: $vgpr62
+; GCN-NEXT:    ; implicit-def: $vgpr63
+; GCN-NEXT:    ; implicit-def: $vgpr55
+; GCN-NEXT:    ; implicit-def: $vgpr40
+; GCN-NEXT:    ; implicit-def: $vgpr41
+; GCN-NEXT:    ; implicit-def: $vgpr42
+; GCN-NEXT:    ; implicit-def: $vgpr43
+; GCN-NEXT:    ; implicit-def: $vgpr44
+; GCN-NEXT:    ; implicit-def: $vgpr45
+; GCN-NEXT:    ; implicit-def: $vgpr46
+; GCN-NEXT:    ; implicit-def: $vgpr31
+; GCN-NEXT:    ; implicit-def: $vgpr32
+; GCN-NEXT:    ; implicit-def: $vgpr33
+; GCN-NEXT:    ; implicit-def: $vgpr34
+; GCN-NEXT:    ; implicit-def: $vgpr35
+; GCN-NEXT:    ; implicit-def: $vgpr36
+; GCN-NEXT:    ; implicit-def: $vgpr37
+; GCN-NEXT:    ; implicit-def: $vgpr38
+; GCN-NEXT:    ; implicit-def: $vgpr39
+; GCN-NEXT:    ; implicit-def: $vgpr48
+; GCN-NEXT:    ; implicit-def: $vgpr49
+; GCN-NEXT:    ; implicit-def: $vgpr50
+; GCN-NEXT:    ; implicit-def: $vgpr51
+; GCN-NEXT:    ; implicit-def: $vgpr52
+; GCN-NEXT:    ; implicit-def: $vgpr53
+; GCN-NEXT:    ; implicit-def: $vgpr54
+; GCN-NEXT:  .LBB28_2: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB28_4
+; GCN-NEXT:  ; %bb.3: ; %cmp.true
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 3, v54
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 3, v53
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 3, v52
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 3, v51
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 3, v50
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 3, v49
+; GCN-NEXT:    v_add_i32_e32 v48, vcc, 3, v48
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 3, v39
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 3, v38
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 3, v37
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 3, v36
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 3, v35
+; GCN-NEXT:    v_add_i32_e32 v34, vcc, 3, v34
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 3, v33
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
+; GCN-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NEXT:    v_add_i32_e32 v46, vcc, 3, v46
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 3, v45
+; GCN-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 3, v44
+; GCN-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 3, v43
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 3, v42
+; GCN-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 3, v41
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_add_i32_e32 v40, vcc, 3, v40
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 3, v55
+; GCN-NEXT:    v_add_i32_e32 v63, vcc, 3, v63
+; GCN-NEXT:    v_add_i32_e32 v62, vcc, 3, v62
+; GCN-NEXT:    v_add_i32_e32 v61, vcc, 3, v61
+; GCN-NEXT:    v_add_i32_e32 v60, vcc, 3, v60
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 3, v59
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 3, v58
+; GCN-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
+; GCN-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; GCN-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v2
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v3
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v6
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v8
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v9
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v10
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v11
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v12
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v13
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v14
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v15
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v17
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v18
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v19
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v20
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v21
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v22
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v23
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v24
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v25
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v26
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v27
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v28
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v29
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v58
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v59
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v60
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v61
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v62
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v63
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v55
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v40
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v41
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v42
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v43
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v44
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v45
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v46
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v31
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v32
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v33
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v39
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v48
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v49
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v51
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v52
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v53
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v54
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:  .LBB28_4: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v57
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v2, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v56
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 4, v0
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v5, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v6, v7, v6
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 12, v0
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v8, v9, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, 16, v0
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v10, v11, v10
+; GCN-NEXT:    v_add_i32_e32 v11, vcc, 20, v0
+; GCN-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v12, v13, v12
+; GCN-NEXT:    v_add_i32_e32 v13, vcc, 24, v0
+; GCN-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v14, v15, v14
+; GCN-NEXT:    v_add_i32_e32 v15, vcc, 28, v0
+; GCN-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v16, v17, v16
+; GCN-NEXT:    v_add_i32_e32 v17, vcc, 32, v0
+; GCN-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v18, v19, v18
+; GCN-NEXT:    v_add_i32_e32 v19, vcc, 36, v0
+; GCN-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v20, v21, v20
+; GCN-NEXT:    v_add_i32_e32 v21, vcc, 40, v0
+; GCN-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v22, v23, v22
+; GCN-NEXT:    v_add_i32_e32 v23, vcc, 44, v0
+; GCN-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v25, v24
+; GCN-NEXT:    v_add_i32_e32 v25, vcc, 48, v0
+; GCN-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v26, v27, v26
+; GCN-NEXT:    v_add_i32_e32 v27, vcc, 52, v0
+; GCN-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; GCN-NEXT:    v_or_b32_e32 v28, v29, v28
+; GCN-NEXT:    v_add_i32_e32 v29, vcc, 56, v0
+; GCN-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GCN-NEXT:    v_or_b32_e32 v30, v31, v30
+; GCN-NEXT:    v_add_i32_e32 v31, vcc, 60, v0
+; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 64, v0
+; GCN-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
+; GCN-NEXT:    v_add_i32_e32 v35, vcc, 0x44, v0
+; GCN-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
+; GCN-NEXT:    v_add_i32_e32 v37, vcc, 0x48, v0
+; GCN-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x4c, v0
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
+; GCN-NEXT:    v_or_b32_e32 v48, v49, v48
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x50, v0
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_or_b32_e32 v50, v51, v50
+; GCN-NEXT:    v_add_i32_e32 v51, vcc, 0x54, v0
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v52, v53, v52
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x58, v0
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v54
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GCN-NEXT:    v_or_b32_e32 v54, v55, v54
+; GCN-NEXT:    v_add_i32_e32 v55, vcc, 0x5c, v0
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_or_b32_e32 v40, v41, v40
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x60, v0
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; GCN-NEXT:    v_or_b32_e32 v42, v43, v42
+; GCN-NEXT:    v_add_i32_e32 v43, vcc, 0x64, v0
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v44, v45, v44
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 0x68, v0
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v46, 16, v46
+; GCN-NEXT:    v_or_b32_e32 v46, v47, v46
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 0x6c, v0
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v56, v57, v56
+; GCN-NEXT:    v_add_i32_e32 v57, vcc, 0x70, v0
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_or_b32_e32 v58, v59, v58
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v8, v9, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v10, v11, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v12, v13, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v14, v15, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v16, v17, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v18, v19, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v20, v21, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v22, v23, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v25, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v26, v27, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v29, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v31, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v32, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v35, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v36, v37, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v38, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v50, v51, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v52, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v54, v55, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v42, v43, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v44, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v56, v57, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v58, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60i16_to_v60f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v1
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB28_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_u16_e32 v0, 3, v0
+; VI-NEXT:    v_add_u16_e32 v30, 3, v30
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_add_u16_e32 v59, 3, v59
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    v_add_u16_e32 v58, 3, v58
+; VI-NEXT:    v_add_u16_e32 v3, 3, v3
+; VI-NEXT:    v_add_u16_e32 v57, 3, v57
+; VI-NEXT:    v_add_u16_e32 v4, 3, v4
+; VI-NEXT:    v_add_u16_e32 v56, 3, v56
+; VI-NEXT:    v_add_u16_e32 v5, 3, v5
+; VI-NEXT:    v_add_u16_e32 v47, 3, v47
+; VI-NEXT:    v_add_u16_e32 v6, 3, v6
+; VI-NEXT:    v_add_u16_e32 v46, 3, v46
+; VI-NEXT:    v_add_u16_e32 v7, 3, v7
+; VI-NEXT:    v_add_u16_e32 v45, 3, v45
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_add_u16_e32 v44, 3, v44
+; VI-NEXT:    v_add_u16_e32 v9, 3, v9
+; VI-NEXT:    v_add_u16_e32 v43, 3, v43
+; VI-NEXT:    v_add_u16_e32 v10, 3, v10
+; VI-NEXT:    v_add_u16_e32 v42, 3, v42
+; VI-NEXT:    v_add_u16_e32 v11, 3, v11
+; VI-NEXT:    v_add_u16_e32 v41, 3, v41
+; VI-NEXT:    v_add_u16_e32 v12, 3, v12
+; VI-NEXT:    v_add_u16_e32 v40, 3, v40
+; VI-NEXT:    v_add_u16_e32 v13, 3, v13
+; VI-NEXT:    v_add_u16_e32 v55, 3, v55
+; VI-NEXT:    v_add_u16_e32 v14, 3, v14
+; VI-NEXT:    v_add_u16_e32 v54, 3, v54
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    v_add_u16_e32 v53, 3, v53
+; VI-NEXT:    v_add_u16_e32 v16, 3, v16
+; VI-NEXT:    v_add_u16_e32 v52, 3, v52
+; VI-NEXT:    v_add_u16_e32 v17, 3, v17
+; VI-NEXT:    v_add_u16_e32 v51, 3, v51
+; VI-NEXT:    v_add_u16_e32 v18, 3, v18
+; VI-NEXT:    v_add_u16_e32 v50, 3, v50
+; VI-NEXT:    v_add_u16_e32 v19, 3, v19
+; VI-NEXT:    v_add_u16_e32 v49, 3, v49
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_add_u16_e32 v48, 3, v48
+; VI-NEXT:    v_add_u16_e32 v21, 3, v21
+; VI-NEXT:    v_add_u16_e32 v39, 3, v39
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v38, 3, v38
+; VI-NEXT:    v_add_u16_e32 v23, 3, v23
+; VI-NEXT:    v_add_u16_e32 v37, 3, v37
+; VI-NEXT:    v_add_u16_e32 v24, 3, v24
+; VI-NEXT:    v_add_u16_e32 v36, 3, v36
+; VI-NEXT:    v_add_u16_e32 v25, 3, v25
+; VI-NEXT:    v_add_u16_e32 v35, 3, v35
+; VI-NEXT:    v_add_u16_e32 v26, 3, v26
+; VI-NEXT:    v_add_u16_e32 v34, 3, v34
+; VI-NEXT:    v_add_u16_e32 v27, 3, v27
+; VI-NEXT:    v_add_u16_e32 v33, 3, v33
+; VI-NEXT:    v_add_u16_e32 v28, 3, v28
+; VI-NEXT:    v_add_u16_e32 v32, 3, v32
+; VI-NEXT:    v_add_u16_e32 v29, 3, v29
+; VI-NEXT:    v_add_u16_e32 v31, 3, v31
+; VI-NEXT:  .LBB28_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v59
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v58
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v57
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v56
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v47
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v46
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v45
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v44
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v43
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v42
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v41
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v40
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v55
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v54
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v53
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v52
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60i16_to_v60f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v1
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB28_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v29, v59, v29, s6
+; GFX9-NEXT:    v_perm_b32 v28, v58, v28, s6
+; GFX9-NEXT:    v_perm_b32 v27, v57, v27, s6
+; GFX9-NEXT:    v_perm_b32 v26, v56, v26, s6
+; GFX9-NEXT:    v_perm_b32 v25, v47, v25, s6
+; GFX9-NEXT:    v_perm_b32 v24, v46, v24, s6
+; GFX9-NEXT:    v_perm_b32 v23, v45, v23, s6
+; GFX9-NEXT:    v_perm_b32 v22, v44, v22, s6
+; GFX9-NEXT:    v_perm_b32 v21, v43, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v42, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v41, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v40, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v55, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v54, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v53, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v52, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v51, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v50, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v49, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v48, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v39, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v38, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v37, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v36, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v35, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v34, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v33, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v32, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v31, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v30, v0, s6
+; GFX9-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v29
+; GFX9-NEXT:  .LBB28_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v18, v40, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v41, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v42, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v43, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v44, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v45, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v46, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v47, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v56, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v57, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v58, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v59, v29, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v0, v30, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v31, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v32, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v33, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v34, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v35, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v36, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v37, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v38, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v39, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v48, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v49, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v50, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v51, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v52, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v53, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v54, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v55, v17, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60i16_to_v60f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v29, v84, v29, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v83, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v82, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v81, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v80, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v71, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v70, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v69, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v68, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v67, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v66, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v65, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v64, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v55, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v54, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v52, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v50, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v48, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v38, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v36, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v31, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v32, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v34, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v29
+; GFX11-NEXT:  .LBB28_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v31, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v32, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v34, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v36, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v38, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v48, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v50, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v52, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v54, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v55, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v64, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v65, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v66, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v67, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v68, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v69, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v70, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v71, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v80, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v81, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v82, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v83, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v84, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = add <60 x i16> %a, splat (i16 3)
+  %a2 = bitcast <60 x i16> %a1 to <60 x half>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x i16> %a to <60 x half>
+  br label %end
+
+end:
+  %phi = phi <60 x half> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x half> %phi
+}
+
+define <60 x i16> @bitcast_v60f16_to_v60i16(<60 x half> %a, i32 %b) {
+; GCN-LABEL: bitcast_v60f16_to_v60i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:108
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:104
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:100
+; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:96
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:92
+; GCN-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:88
+; GCN-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:20
+; GCN-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:16
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12
+; GCN-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:8
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:120
+; GCN-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:84
+; GCN-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:80
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:76
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:72
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:68
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:64
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:56
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:52
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v4
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v5
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v8
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v13
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v14
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v16
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v17
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v8, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v20
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v22
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v27
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v28
+; GCN-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v30
+; GCN-NEXT:    s_waitcnt vmcnt(14)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v50
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:44
+; GCN-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:40
+; GCN-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
+; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:32
+; GCN-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:28
+; GCN-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:24
+; GCN-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:116
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v51
+; GCN-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v54
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v52
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v54, v39
+; GCN-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:112
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v63
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v62
+; GCN-NEXT:    v_cvt_f16_f32_e32 v62, v61
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v50
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v61, v27
+; GCN-NEXT:    v_mov_b32_e32 v63, v2
+; GCN-NEXT:    v_mov_b32_e32 v45, v1
+; GCN-NEXT:    v_mov_b32_e32 v56, v8
+; GCN-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB29_2
+; GCN-NEXT:  ; %bb.1: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v52
+; GCN-NEXT:    v_cvt_f32_f16_e32 v48, v48
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
+; GCN-NEXT:    v_cvt_f16_f32_e32 v52, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v48, v48, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v54
+; GCN-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v54, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v54, v50, v54
+; GCN-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GCN-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GCN-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; GCN-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; GCN-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GCN-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GCN-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GCN-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; GCN-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; GCN-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GCN-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GCN-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GCN-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; GCN-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; GCN-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GCN-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GCN-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GCN-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; GCN-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; GCN-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; GCN-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GCN-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GCN-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; GCN-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; GCN-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; GCN-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v22, v22
+; GCN-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; GCN-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; GCN-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; GCN-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; GCN-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v50
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v8, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GCN-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; GCN-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v8
+; GCN-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GCN-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GCN-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; GCN-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; GCN-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GCN-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GCN-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GCN-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; GCN-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; GCN-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GCN-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v32, v32
+; GCN-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; GCN-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; GCN-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; GCN-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; GCN-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v31, v31, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; GCN-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; GCN-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; GCN-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; GCN-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; GCN-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v33, v33, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; GCN-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; GCN-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
+; GCN-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; GCN-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; GCN-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v35, v35, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v38, v38
+; GCN-NEXT:    v_cvt_f32_f16_e32 v37, v37
+; GCN-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; GCN-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; GCN-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; GCN-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v37, v37, v50
+; GCN-NEXT:    v_cvt_f32_f16_e32 v50, v60
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v40, v59
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v43, v2
+; GCN-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v46, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v56, v56
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v57, v27
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cvt_f32_f16_e32 v58, v27
+; GCN-NEXT:    v_cvt_f32_f16_e32 v59, v63
+; GCN-NEXT:    v_cvt_f32_f16_e32 v26, v26
+; GCN-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; GCN-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; GCN-NEXT:    v_cvt_f32_f16_e32 v60, v61
+; GCN-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; GCN-NEXT:    v_cvt_f32_f16_e32 v55, v55
+; GCN-NEXT:    v_cvt_f32_f16_e32 v44, v44
+; GCN-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; GCN-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; GCN-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; GCN-NEXT:    v_cvt_f32_f16_e32 v53, v53
+; GCN-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; GCN-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; GCN-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; GCN-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; GCN-NEXT:    v_cvt_f32_f16_e32 v49, v49
+; GCN-NEXT:    v_cvt_f32_f16_e32 v27, v62
+; GCN-NEXT:    v_cvt_f32_f16_e32 v39, v39
+; GCN-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; GCN-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; GCN-NEXT:    v_add_f32_e32 v40, 0x38000000, v40
+; GCN-NEXT:    v_add_f32_e32 v43, 0x38000000, v43
+; GCN-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; GCN-NEXT:    v_add_f32_e32 v45, 0x38000000, v45
+; GCN-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; GCN-NEXT:    v_add_f32_e32 v46, 0x38000000, v46
+; GCN-NEXT:    v_add_f32_e32 v56, 0x38000000, v56
+; GCN-NEXT:    v_add_f32_e32 v57, 0x38000000, v57
+; GCN-NEXT:    v_add_f32_e32 v58, 0x38000000, v58
+; GCN-NEXT:    v_add_f32_e32 v59, 0x38000000, v59
+; GCN-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
+; GCN-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; GCN-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; GCN-NEXT:    v_add_f32_e32 v60, 0x38000000, v60
+; GCN-NEXT:    v_add_f32_e32 v47, 0x38000000, v47
+; GCN-NEXT:    v_add_f32_e32 v55, 0x38000000, v55
+; GCN-NEXT:    v_add_f32_e32 v44, 0x38000000, v44
+; GCN-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
+; GCN-NEXT:    v_add_f32_e32 v41, 0x38000000, v41
+; GCN-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; GCN-NEXT:    v_add_f32_e32 v53, 0x38000000, v53
+; GCN-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; GCN-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; GCN-NEXT:    v_add_f32_e32 v42, 0x38000000, v42
+; GCN-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; GCN-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; GCN-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; GCN-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; GCN-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v40, v40
+; GCN-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; GCN-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; GCN-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; GCN-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; GCN-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; GCN-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; GCN-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; GCN-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; GCN-NEXT:    v_cvt_f16_f32_e32 v23, v23
+; GCN-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; GCN-NEXT:    v_cvt_f16_f32_e32 v60, v60
+; GCN-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; GCN-NEXT:    v_cvt_f16_f32_e32 v55, v55
+; GCN-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; GCN-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; GCN-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; GCN-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; GCN-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; GCN-NEXT:    v_cvt_f16_f32_e32 v28, v28
+; GCN-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; GCN-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; GCN-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; GCN-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; GCN-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; GCN-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v56, 16, v56
+; GCN-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v50
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v1, v43, v40
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GCN-NEXT:    v_or_b32_e32 v45, v45, v3
+; GCN-NEXT:    v_or_b32_e32 v43, v46, v2
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v43, v57, v56
+; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GCN-NEXT:    v_or_b32_e32 v63, v59, v58
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v26
+; GCN-NEXT:    v_or_b32_e32 v61, v60, v25
+; GCN-NEXT:    v_or_b32_e32 v55, v55, v47
+; GCN-NEXT:    v_or_b32_e32 v51, v51, v44
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v41
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v53
+; GCN-NEXT:    v_or_b32_e32 v42, v42, v24
+; GCN-NEXT:    v_or_b32_e32 v49, v49, v29
+; GCN-NEXT:    v_or_b32_e32 v39, v39, v27
+; GCN-NEXT:    v_alignbit_b32 v60, v37, v50, 16
+; GCN-NEXT:    v_alignbit_b32 v59, v35, v40, 16
+; GCN-NEXT:    v_alignbit_b32 v3, v33, v3, 16
+; GCN-NEXT:    v_alignbit_b32 v1, v31, v2, 16
+; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-NEXT:    v_alignbit_b32 v56, v15, v56, 16
+; GCN-NEXT:    v_alignbit_b32 v2, v11, v58, 16
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GCN-NEXT:    v_alignbit_b32 v26, v7, v26, 16
+; GCN-NEXT:    v_alignbit_b32 v25, v21, v25, 16
+; GCN-NEXT:    v_alignbit_b32 v47, v19, v47, 16
+; GCN-NEXT:    v_alignbit_b32 v44, v17, v44, 16
+; GCN-NEXT:    v_alignbit_b32 v41, v13, v41, 16
+; GCN-NEXT:    v_alignbit_b32 v53, v9, v53, 16
+; GCN-NEXT:    v_alignbit_b32 v24, v5, v24, 16
+; GCN-NEXT:    v_alignbit_b32 v29, v54, v29, 16
+; GCN-NEXT:    v_alignbit_b32 v62, v48, v27, 16
+; GCN-NEXT:  .LBB29_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt expcnt(1)
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v60
+; GCN-NEXT:    v_or_b32_e32 v57, v1, v50
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v37
+; GCN-NEXT:    v_lshlrev_b32_e32 v37, 16, v38
+; GCN-NEXT:    v_or_b32_e32 v37, v1, v37
+; GCN-NEXT:    v_add_i32_e32 v38, vcc, 4, v0
+; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v59
+; GCN-NEXT:    v_or_b32_e32 v46, v1, v50
+; GCN-NEXT:    v_add_i32_e32 v59, vcc, 8, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v35
+; GCN-NEXT:    v_lshlrev_b32_e32 v35, 16, v36
+; GCN-NEXT:    v_or_b32_e32 v35, v1, v35
+; GCN-NEXT:    v_add_i32_e32 v36, vcc, 12, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v45
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v43, v1, v3
+; GCN-NEXT:    v_add_i32_e32 v58, vcc, 16, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v34
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v3
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 20, v0
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v33, 0xffff, v2
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_add_i32_e32 v33, vcc, 24, v0
+; GCN-NEXT:    v_and_b32_e32 v31, 0xffff, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GCN-NEXT:    v_or_b32_e32 v31, v31, v32
+; GCN-NEXT:    v_add_i32_e32 v32, vcc, 28, v0
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v34, 0xffff, v27
+; GCN-NEXT:    v_lshlrev_b32_e32 v50, 16, v56
+; GCN-NEXT:    v_or_b32_e32 v34, v34, v50
+; GCN-NEXT:    v_add_i32_e32 v50, vcc, 32, v0
+; GCN-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v16
+; GCN-NEXT:    v_add_i32_e32 v16, vcc, 36, v0
+; GCN-NEXT:    v_and_b32_e32 v40, 0xffff, v63
+; GCN-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v45, 16, v27
+; GCN-NEXT:    v_or_b32_e32 v40, v40, v45
+; GCN-NEXT:    v_add_i32_e32 v45, vcc, 40, v0
+; GCN-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GCN-NEXT:    v_or_b32_e32 v11, v11, v12
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, 44, v0
+; GCN-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; GCN-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GCN-NEXT:    v_or_b32_e32 v23, v23, v26
+; GCN-NEXT:    v_add_i32_e32 v26, vcc, 48, v0
+; GCN-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v7, v7, v8
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, 52, v0
+; GCN-NEXT:    v_and_b32_e32 v56, 0xffff, v61
+; GCN-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GCN-NEXT:    v_or_b32_e32 v25, v56, v25
+; GCN-NEXT:    v_add_i32_e32 v56, vcc, 56, v0
+; GCN-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; GCN-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GCN-NEXT:    v_or_b32_e32 v21, v21, v22
+; GCN-NEXT:    v_add_i32_e32 v22, vcc, 60, v0
+; GCN-NEXT:    v_and_b32_e32 v55, 0xffff, v55
+; GCN-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; GCN-NEXT:    v_or_b32_e32 v55, v55, v47
+; GCN-NEXT:    v_add_i32_e32 v47, vcc, 64, v0
+; GCN-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; GCN-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_or_b32_e32 v19, v19, v20
+; GCN-NEXT:    v_add_i32_e32 v20, vcc, 0x44, v0
+; GCN-NEXT:    v_and_b32_e32 v51, 0xffff, v51
+; GCN-NEXT:    v_lshlrev_b32_e32 v44, 16, v44
+; GCN-NEXT:    v_or_b32_e32 v51, v51, v44
+; GCN-NEXT:    v_add_i32_e32 v44, vcc, 0x48, v0
+; GCN-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; GCN-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_or_b32_e32 v17, v17, v18
+; GCN-NEXT:    v_add_i32_e32 v18, vcc, 0x4c, v0
+; GCN-NEXT:    v_and_b32_e32 v30, 0xffff, v30
+; GCN-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; GCN-NEXT:    v_or_b32_e32 v30, v30, v41
+; GCN-NEXT:    v_add_i32_e32 v41, vcc, 0x50, v0
+; GCN-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GCN-NEXT:    v_or_b32_e32 v13, v13, v14
+; GCN-NEXT:    v_add_i32_e32 v14, vcc, 0x54, v0
+; GCN-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; GCN-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GCN-NEXT:    v_or_b32_e32 v28, v28, v53
+; GCN-NEXT:    v_add_i32_e32 v53, vcc, 0x58, v0
+; GCN-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GCN-NEXT:    v_or_b32_e32 v9, v9, v10
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, 0x5c, v0
+; GCN-NEXT:    v_and_b32_e32 v42, 0xffff, v42
+; GCN-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_or_b32_e32 v24, v42, v24
+; GCN-NEXT:    v_add_i32_e32 v42, vcc, 0x60, v0
+; GCN-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GCN-NEXT:    v_or_b32_e32 v5, v5, v6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, 0x64, v0
+; GCN-NEXT:    v_and_b32_e32 v49, 0xffff, v49
+; GCN-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GCN-NEXT:    v_or_b32_e32 v29, v49, v29
+; GCN-NEXT:    v_add_i32_e32 v49, vcc, 0x68, v0
+; GCN-NEXT:    v_and_b32_e32 v54, 0xffff, v54
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_or_b32_e32 v4, v54, v4
+; GCN-NEXT:    v_add_i32_e32 v54, vcc, 0x6c, v0
+; GCN-NEXT:    v_and_b32_e32 v39, 0xffff, v39
+; GCN-NEXT:    v_lshlrev_b32_e32 v27, 16, v62
+; GCN-NEXT:    v_or_b32_e32 v27, v39, v27
+; GCN-NEXT:    v_add_i32_e32 v39, vcc, 0x70, v0
+; GCN-NEXT:    v_and_b32_e32 v48, 0xffff, v48
+; GCN-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GCN-NEXT:    v_or_b32_e32 v48, v48, v52
+; GCN-NEXT:    v_add_i32_e32 v52, vcc, 0x74, v0
+; GCN-NEXT:    buffer_store_dword v57, v0, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v37, v38, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v46, v59, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v35, v36, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v43, v58, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v1, v3, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v2, v33, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v31, v32, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v34, v50, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v15, v16, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v40, v45, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v11, v12, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v23, v26, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v7, v8, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v25, v56, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v21, v22, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v55, v47, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v19, v20, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v51, v44, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v17, v18, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v30, v41, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v13, v14, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v28, v53, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v9, v10, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v24, v42, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v5, v6, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v29, v49, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v4, v54, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v27, v39, s[0:3], 0 offen
+; GCN-NEXT:    buffer_store_dword v48, v52, s[0:3], 0 offen
+; GCN-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v60f16_to_v60i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v34, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v24
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; VI-NEXT:    v_lshrrev_b32_e32 v38, 16, v22
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v21
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v20
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v19
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v18
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v17
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v16
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; VI-NEXT:    v_lshrrev_b32_e32 v54, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v13
+; VI-NEXT:    v_lshrrev_b32_e32 v40, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v11
+; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v8
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v1
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; VI-NEXT:    v_lshrrev_b32_e32 v30, 16, v0
+; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT:    s_cbranch_execz .LBB29_2
+; VI-NEXT:  ; %bb.1: ; %cmp.true
+; VI-NEXT:    v_add_f16_e32 v0, 0x200, v0
+; VI-NEXT:    v_add_f16_e32 v30, 0x200, v30
+; VI-NEXT:    v_add_f16_e32 v1, 0x200, v1
+; VI-NEXT:    v_add_f16_e32 v59, 0x200, v59
+; VI-NEXT:    v_add_f16_e32 v2, 0x200, v2
+; VI-NEXT:    v_add_f16_e32 v58, 0x200, v58
+; VI-NEXT:    v_add_f16_e32 v3, 0x200, v3
+; VI-NEXT:    v_add_f16_e32 v57, 0x200, v57
+; VI-NEXT:    v_add_f16_e32 v4, 0x200, v4
+; VI-NEXT:    v_add_f16_e32 v56, 0x200, v56
+; VI-NEXT:    v_add_f16_e32 v5, 0x200, v5
+; VI-NEXT:    v_add_f16_e32 v47, 0x200, v47
+; VI-NEXT:    v_add_f16_e32 v6, 0x200, v6
+; VI-NEXT:    v_add_f16_e32 v46, 0x200, v46
+; VI-NEXT:    v_add_f16_e32 v7, 0x200, v7
+; VI-NEXT:    v_add_f16_e32 v45, 0x200, v45
+; VI-NEXT:    v_add_f16_e32 v8, 0x200, v8
+; VI-NEXT:    v_add_f16_e32 v44, 0x200, v44
+; VI-NEXT:    v_add_f16_e32 v9, 0x200, v9
+; VI-NEXT:    v_add_f16_e32 v43, 0x200, v43
+; VI-NEXT:    v_add_f16_e32 v10, 0x200, v10
+; VI-NEXT:    v_add_f16_e32 v42, 0x200, v42
+; VI-NEXT:    v_add_f16_e32 v11, 0x200, v11
+; VI-NEXT:    v_add_f16_e32 v41, 0x200, v41
+; VI-NEXT:    v_add_f16_e32 v12, 0x200, v12
+; VI-NEXT:    v_add_f16_e32 v40, 0x200, v40
+; VI-NEXT:    v_add_f16_e32 v13, 0x200, v13
+; VI-NEXT:    v_add_f16_e32 v55, 0x200, v55
+; VI-NEXT:    v_add_f16_e32 v14, 0x200, v14
+; VI-NEXT:    v_add_f16_e32 v54, 0x200, v54
+; VI-NEXT:    v_add_f16_e32 v15, 0x200, v15
+; VI-NEXT:    v_add_f16_e32 v53, 0x200, v53
+; VI-NEXT:    v_add_f16_e32 v16, 0x200, v16
+; VI-NEXT:    v_add_f16_e32 v52, 0x200, v52
+; VI-NEXT:    v_add_f16_e32 v17, 0x200, v17
+; VI-NEXT:    v_add_f16_e32 v51, 0x200, v51
+; VI-NEXT:    v_add_f16_e32 v18, 0x200, v18
+; VI-NEXT:    v_add_f16_e32 v50, 0x200, v50
+; VI-NEXT:    v_add_f16_e32 v19, 0x200, v19
+; VI-NEXT:    v_add_f16_e32 v49, 0x200, v49
+; VI-NEXT:    v_add_f16_e32 v20, 0x200, v20
+; VI-NEXT:    v_add_f16_e32 v48, 0x200, v48
+; VI-NEXT:    v_add_f16_e32 v21, 0x200, v21
+; VI-NEXT:    v_add_f16_e32 v39, 0x200, v39
+; VI-NEXT:    v_add_f16_e32 v22, 0x200, v22
+; VI-NEXT:    v_add_f16_e32 v38, 0x200, v38
+; VI-NEXT:    v_add_f16_e32 v23, 0x200, v23
+; VI-NEXT:    v_add_f16_e32 v37, 0x200, v37
+; VI-NEXT:    v_add_f16_e32 v24, 0x200, v24
+; VI-NEXT:    v_add_f16_e32 v36, 0x200, v36
+; VI-NEXT:    v_add_f16_e32 v25, 0x200, v25
+; VI-NEXT:    v_add_f16_e32 v35, 0x200, v35
+; VI-NEXT:    v_add_f16_e32 v26, 0x200, v26
+; VI-NEXT:    v_add_f16_e32 v34, 0x200, v34
+; VI-NEXT:    v_add_f16_e32 v27, 0x200, v27
+; VI-NEXT:    v_add_f16_e32 v33, 0x200, v33
+; VI-NEXT:    v_add_f16_e32 v28, 0x200, v28
+; VI-NEXT:    v_add_f16_e32 v32, 0x200, v32
+; VI-NEXT:    v_add_f16_e32 v29, 0x200, v29
+; VI-NEXT:    v_add_f16_e32 v31, 0x200, v31
+; VI-NEXT:  .LBB29_2: ; %end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v59
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v58
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v57
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v56
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v47
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v46
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v45
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v44
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v43
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v42
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v41
+; VI-NEXT:    v_or_b32_sdwa v11, v11, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v40
+; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v12, v12, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v55
+; VI-NEXT:    v_or_b32_sdwa v13, v13, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v54
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v53
+; VI-NEXT:    v_or_b32_sdwa v15, v15, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v52
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v51
+; VI-NEXT:    v_or_b32_sdwa v17, v17, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v50
+; VI-NEXT:    v_or_b32_sdwa v18, v18, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v49
+; VI-NEXT:    v_or_b32_sdwa v19, v19, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v48
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v39
+; VI-NEXT:    v_or_b32_sdwa v21, v21, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v38
+; VI-NEXT:    v_or_b32_sdwa v22, v22, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v37
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v36
+; VI-NEXT:    v_or_b32_sdwa v24, v24, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v35
+; VI-NEXT:    v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v26, v26, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v33
+; VI-NEXT:    v_or_b32_sdwa v27, v27, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v32
+; VI-NEXT:    v_or_b32_sdwa v28, v28, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v30, 16, v31
+; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bitcast_v60f16_to_v60i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v29
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v1
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB29_2
+; GFX9-NEXT:  ; %bb.1: ; %cmp.true
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v29, v59, v29, s6
+; GFX9-NEXT:    s_movk_i32 s7, 0x200
+; GFX9-NEXT:    v_perm_b32 v28, v58, v28, s6
+; GFX9-NEXT:    v_perm_b32 v27, v57, v27, s6
+; GFX9-NEXT:    v_perm_b32 v26, v56, v26, s6
+; GFX9-NEXT:    v_perm_b32 v25, v47, v25, s6
+; GFX9-NEXT:    v_perm_b32 v24, v46, v24, s6
+; GFX9-NEXT:    v_perm_b32 v23, v45, v23, s6
+; GFX9-NEXT:    v_perm_b32 v22, v44, v22, s6
+; GFX9-NEXT:    v_perm_b32 v21, v43, v21, s6
+; GFX9-NEXT:    v_perm_b32 v20, v42, v20, s6
+; GFX9-NEXT:    v_perm_b32 v19, v41, v19, s6
+; GFX9-NEXT:    v_perm_b32 v18, v40, v18, s6
+; GFX9-NEXT:    v_perm_b32 v17, v55, v17, s6
+; GFX9-NEXT:    v_perm_b32 v16, v54, v16, s6
+; GFX9-NEXT:    v_perm_b32 v15, v53, v15, s6
+; GFX9-NEXT:    v_perm_b32 v14, v52, v14, s6
+; GFX9-NEXT:    v_perm_b32 v13, v51, v13, s6
+; GFX9-NEXT:    v_perm_b32 v12, v50, v12, s6
+; GFX9-NEXT:    v_perm_b32 v11, v49, v11, s6
+; GFX9-NEXT:    v_perm_b32 v10, v48, v10, s6
+; GFX9-NEXT:    v_perm_b32 v9, v39, v9, s6
+; GFX9-NEXT:    v_perm_b32 v8, v38, v8, s6
+; GFX9-NEXT:    v_perm_b32 v7, v37, v7, s6
+; GFX9-NEXT:    v_perm_b32 v6, v36, v6, s6
+; GFX9-NEXT:    v_perm_b32 v5, v35, v5, s6
+; GFX9-NEXT:    v_perm_b32 v4, v34, v4, s6
+; GFX9-NEXT:    v_perm_b32 v3, v33, v3, s6
+; GFX9-NEXT:    v_perm_b32 v2, v32, v2, s6
+; GFX9-NEXT:    v_perm_b32 v1, v31, v1, s6
+; GFX9-NEXT:    v_perm_b32 v0, v30, v0, s6
+; GFX9-NEXT:    v_pk_add_f16 v29, v29, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v28, v28, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v27, v27, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v26, v26, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v25, v25, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v24, v24, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v23, v23, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v22, v22, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v21, v21, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v20, v20, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v19, v19, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v18, v18, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v17, v17, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v16, v16, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v15, v15, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v14, v14, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v13, v13, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v12, v12, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v11, v11, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v10, v10, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v9, v9, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v8, v8, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v7, v7, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v6, v6, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v5, v5, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v4, v4, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, s7 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v34, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v36, 16, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v38, 16, v8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v48, 16, v10
+; GFX9-NEXT:    v_lshrrev_b32_e32 v49, 16, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v50, 16, v12
+; GFX9-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GFX9-NEXT:    v_lshrrev_b32_e32 v52, 16, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v53, 16, v15
+; GFX9-NEXT:    v_lshrrev_b32_e32 v54, 16, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
+; GFX9-NEXT:    v_lshrrev_b32_e32 v40, 16, v18
+; GFX9-NEXT:    v_lshrrev_b32_e32 v41, 16, v19
+; GFX9-NEXT:    v_lshrrev_b32_e32 v42, 16, v20
+; GFX9-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
+; GFX9-NEXT:    v_lshrrev_b32_e32 v44, 16, v22
+; GFX9-NEXT:    v_lshrrev_b32_e32 v45, 16, v23
+; GFX9-NEXT:    v_lshrrev_b32_e32 v46, 16, v24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v47, 16, v25
+; GFX9-NEXT:    v_lshrrev_b32_e32 v56, 16, v26
+; GFX9-NEXT:    v_lshrrev_b32_e32 v57, 16, v27
+; GFX9-NEXT:    v_lshrrev_b32_e32 v58, 16, v28
+; GFX9-NEXT:    v_lshrrev_b32_e32 v59, 16, v29
+; GFX9-NEXT:  .LBB29_2: ; %end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_perm_b32 v18, v40, v18, s4
+; GFX9-NEXT:    v_perm_b32 v19, v41, v19, s4
+; GFX9-NEXT:    v_perm_b32 v20, v42, v20, s4
+; GFX9-NEXT:    v_perm_b32 v21, v43, v21, s4
+; GFX9-NEXT:    v_perm_b32 v22, v44, v22, s4
+; GFX9-NEXT:    v_perm_b32 v23, v45, v23, s4
+; GFX9-NEXT:    v_perm_b32 v24, v46, v24, s4
+; GFX9-NEXT:    v_perm_b32 v25, v47, v25, s4
+; GFX9-NEXT:    v_perm_b32 v26, v56, v26, s4
+; GFX9-NEXT:    v_perm_b32 v27, v57, v27, s4
+; GFX9-NEXT:    v_perm_b32 v28, v58, v28, s4
+; GFX9-NEXT:    v_perm_b32 v29, v59, v29, s4
+; GFX9-NEXT:    buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_perm_b32 v0, v30, v0, s4
+; GFX9-NEXT:    v_perm_b32 v1, v31, v1, s4
+; GFX9-NEXT:    v_perm_b32 v2, v32, v2, s4
+; GFX9-NEXT:    v_perm_b32 v3, v33, v3, s4
+; GFX9-NEXT:    v_perm_b32 v4, v34, v4, s4
+; GFX9-NEXT:    v_perm_b32 v5, v35, v5, s4
+; GFX9-NEXT:    v_perm_b32 v6, v36, v6, s4
+; GFX9-NEXT:    v_perm_b32 v7, v37, v7, s4
+; GFX9-NEXT:    v_perm_b32 v8, v38, v8, s4
+; GFX9-NEXT:    v_perm_b32 v9, v39, v9, s4
+; GFX9-NEXT:    v_perm_b32 v10, v48, v10, s4
+; GFX9-NEXT:    v_perm_b32 v11, v49, v11, s4
+; GFX9-NEXT:    v_perm_b32 v12, v50, v12, s4
+; GFX9-NEXT:    v_perm_b32 v13, v51, v13, s4
+; GFX9-NEXT:    v_perm_b32 v14, v52, v14, s4
+; GFX9-NEXT:    v_perm_b32 v15, v53, v15, s4
+; GFX9-NEXT:    v_perm_b32 v16, v54, v16, s4
+; GFX9-NEXT:    v_perm_b32 v17, v55, v17, s4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: bitcast_v60f16_to_v60i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v29
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v30
+; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1: ; %cmp.true
+; GFX11-NEXT:    v_perm_b32 v29, v84, v29, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v83, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v82, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v81, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v80, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v71, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v70, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v69, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v68, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v67, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v66, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v65, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v64, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v55, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v54, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v52, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v50, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v48, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v38, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v36, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v0, v31, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v32, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v34, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_pk_add_f16 v29, 0x200, v29 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v28, 0x200, v28 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
+; GFX11-NEXT:    v_lshrrev_b32_e32 v31, 16, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v34, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v36, 16, v5
+; GFX11-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v38, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v48, 16, v9
+; GFX11-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v50, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v51, 16, v12
+; GFX11-NEXT:    v_lshrrev_b32_e32 v52, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v53, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v54, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX11-NEXT:    v_lshrrev_b32_e32 v64, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v65, 16, v18
+; GFX11-NEXT:    v_lshrrev_b32_e32 v66, 16, v19
+; GFX11-NEXT:    v_lshrrev_b32_e32 v67, 16, v20
+; GFX11-NEXT:    v_lshrrev_b32_e32 v68, 16, v21
+; GFX11-NEXT:    v_lshrrev_b32_e32 v69, 16, v22
+; GFX11-NEXT:    v_lshrrev_b32_e32 v70, 16, v23
+; GFX11-NEXT:    v_lshrrev_b32_e32 v71, 16, v24
+; GFX11-NEXT:    v_lshrrev_b32_e32 v80, 16, v25
+; GFX11-NEXT:    v_lshrrev_b32_e32 v81, 16, v26
+; GFX11-NEXT:    v_lshrrev_b32_e32 v82, 16, v27
+; GFX11-NEXT:    v_lshrrev_b32_e32 v83, 16, v28
+; GFX11-NEXT:    v_lshrrev_b32_e32 v84, 16, v29
+; GFX11-NEXT:  .LBB29_2: ; %end
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    v_perm_b32 v0, v31, v0, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v1, v32, v1, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v2, v33, v2, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v3, v34, v3, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v35, v4, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v5, v36, v5, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v7, v38, v7, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v8, v39, v8, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v9, v48, v9, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v10, v49, v10, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v11, v50, v11, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v12, v51, v12, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v13, v52, v13, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v14, v53, v14, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v15, v54, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v16, v55, v16, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v17, v64, v17, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v18, v65, v18, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v19, v66, v19, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v20, v67, v20, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v21, v68, v21, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v22, v69, v22, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v23, v70, v23, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v24, v71, v24, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v25, v80, v25, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v26, v81, v26, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v27, v82, v27, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v28, v83, v28, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v29, v84, v29, 0x5040100
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+  %a1 = fadd <60 x half> %a, splat (half 0xH0200)
+  %a2 = bitcast <60 x half> %a1 to <60 x i16>
+  br label %end
+
+cmp.false:
+  %a3 = bitcast <60 x half> %a to <60 x i16>
+  br label %end
+
+end:
+  %phi = phi <60 x i16> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+  ret <60 x i16> %phi
 }

From adba24aa3c52c7e2673104fc16bd19e639221077 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 17 Apr 2025 06:19:37 +0200
Subject: [PATCH 216/710] [clang][bytecode] Add missing __builtin_memcpy checks
 (#135975)

Add a test for type punning and tests and the necessary checks for
non-trivially-copyable types and incomplete types.
---
 clang/lib/AST/ByteCode/InterpBuiltin.cpp      | 18 +++++++++-
 clang/test/AST/ByteCode/builtin-functions.cpp | 35 +++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index b694a34e47ee0..31d97d9060142 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -1872,7 +1872,23 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
     return false;
   }
 
-  // Check if we have enough elements to read from and write to/
+  if (DestElemType->isIncompleteType() ||
+      DestPtr.getType()->isIncompleteType()) {
+    QualType DiagType =
+        DestElemType->isIncompleteType() ? DestElemType : DestPtr.getType();
+    S.FFDiag(S.Current->getSource(OpPC),
+             diag::note_constexpr_memcpy_incomplete_type)
+        << Move << DiagType;
+    return false;
+  }
+
+  if (!DestElemType.isTriviallyCopyableType(ASTCtx)) {
+    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_nontrivial)
+        << Move << DestElemType;
+    return false;
+  }
+
+  // Check if we have enough elements to read from and write to.
   size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
   size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
   if (Size.ugt(RemainingDestBytes) || Size.ugt(RemainingSrcBytes)) {
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index a57b4530d2264..a4c8ec4856ecc 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -33,6 +33,30 @@ extern "C" {
   extern wchar_t *wmemcpy(wchar_t *d, const wchar_t *s, size_t n);
 }
 
+
+constexpr int test_address_of_incomplete_array_type() { // both-error {{never produces a constant expression}}
+  extern int arr[];
+  __builtin_memmove(&arr, &arr, 4 * sizeof(arr[0])); // both-note 2{{cannot constant evaluate 'memmove' between objects of incomplete type 'int[]'}}
+  return arr[0] * 1000 + arr[1] * 100 + arr[2] * 10 + arr[3];
+}
+static_assert(test_address_of_incomplete_array_type() == 1234, ""); // both-error {{constant}} \
+                                                                    // both-note {{in call}}
+
+
+  struct NonTrivial {
+    constexpr NonTrivial() : n(0) {}
+    constexpr NonTrivial(const NonTrivial &) : n(1) {}
+    int n;
+  };
+  constexpr bool test_nontrivial_memcpy() { // ref-error {{never produces a constant}}
+    NonTrivial arr[3] = {};
+    __builtin_memcpy(arr, arr + 1, sizeof(NonTrivial)); // both-note {{non-trivially-copyable}} \
+                                                        // ref-note {{non-trivially-copyable}}
+    return true;
+  }
+  static_assert(test_nontrivial_memcpy()); // both-error {{constant}} \
+                                           // both-note {{in call}}
+
 namespace strcmp {
   constexpr char kFoobar[6] = {'f','o','o','b','a','r'};
   constexpr char kFoobazfoobar[12] = {'f','o','o','b','a','z','f','o','o','b','a','r'};
@@ -1349,6 +1373,17 @@ namespace BuiltinMemcpy {
                                                                                   // both-note {{source of 'memcpy' is (void *)123}}
   static_assert(__builtin_memcpy(fold(reinterpret_cast<wchar_t*>(123)), &global, sizeof(wchar_t))); // both-error {{not an integral constant expression}} \
                                                                                                     // both-note {{destination of 'memcpy' is (void *)123}}
+
+
+  constexpr float type_pun(const unsigned &n) {
+    float f = 0.0f;
+    __builtin_memcpy(&f, &n, 4); // both-note {{cannot constant evaluate 'memcpy' from object of type 'const unsigned int' to object of type 'float'}}
+    return f;
+  }
+  static_assert(type_pun(0x3f800000) == 1.0f); // both-error {{constant}} \
+                                               // both-note {{in call}}
+
+
 }
 
 namespace Memcmp {

From 84082223c819426a8a6486f9740210e956a4cea8 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke@amd.com>
Date: Thu, 17 Apr 2025 10:08:22 +0530
Subject: [PATCH 217/710] [AMDGPU][NPM] Cleanup AMDGPUPassRegistry.def
 (#130071)

Finishing up AMDGPU specific passes. Only ones remaining are assembly
printer, virt reg rewriter and PEI.
---
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def         | 8 +-------
 llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp        | 1 +
 3 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 60d27a7fbef29..98a1147ef6d66 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -104,6 +104,7 @@ MACHINE_FUNCTION_PASS("amdgpu-reserve-wwm-regs", AMDGPUReserveWWMRegsPass())
 MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass())
 MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
 MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
+MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
 MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
 MACHINE_FUNCTION_PASS("gcn-create-vopd", GCNCreateVOPDPass())
 MACHINE_FUNCTION_PASS("gcn-dpp-combine", GCNDPPCombinePass())
@@ -133,13 +134,6 @@ MACHINE_FUNCTION_PASS("si-wqm", SIWholeQuadModePass())
 #undef MACHINE_FUNCTION_PASS
 
 #define DUMMY_MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
-DUMMY_MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
-DUMMY_MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass())
-
-// TODO: Move amdgpu-preload-kern-arg-prolog to MACHINE_FUNCTION_PASS since it
-// already exists.
-DUMMY_MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
-
 // Global ISel passes
 DUMMY_MACHINE_FUNCTION_PASS("amdgpu-prelegalizer-combiner", AMDGPUPreLegalizerCombinerPass())
 DUMMY_MACHINE_FUNCTION_PASS("amdgpu-postlegalizer-combiner", AMDGPUPostLegalizerCombinerPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp
index b3a2139dfd24e..40094518dce0a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp
@@ -207,5 +207,5 @@ AMDGPUPreloadKernArgPrologPass::run(MachineFunction &MF,
   if (!AMDGPUPreloadKernArgProlog(MF).run())
     return PreservedAnalyses::all();
 
-  return PreservedAnalyses::none();
+  return getMachineFunctionPassPreservedAnalyses();
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 34dacd5f9209d..90e3489ced923 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -24,6 +24,7 @@
 #include "AMDGPUISelDAGToDAG.h"
 #include "AMDGPUMacroFusion.h"
 #include "AMDGPUPerfHintAnalysis.h"
+#include "AMDGPUPreloadKernArgProlog.h"
 #include "AMDGPURemoveIncompatibleFunctions.h"
 #include "AMDGPUReserveWWMRegs.h"
 #include "AMDGPUSplitModule.h"

From dfed3d235fe00b5ad7befe584fc0cad8359afde2 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Thu, 17 Apr 2025 06:38:14 +0200
Subject: [PATCH 218/710] [lldb][test]
 TestDataFormatterLibcxxInvalidVectorSimulator.py: fix inline namespace
 warnings

Fixes:
```
/Users/ec2-user/jenkins/workspace/apple-llvm-project-pr-macos/branch-swift/release/6.2/llvm-project/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp:5:11: warning: inline namespace reopened as a non-inline namespace [-Winline-namespace-reopened-noninline]
    5 | namespace __1 {
      |           ^
```

Drive-by: compile test as C++20 (in an attempt to fix another buildbot issue)
---
 .../libcxx-simulators/invalid-vector/Makefile               | 2 +-
 .../libcxx-simulators/invalid-vector/main.cpp               | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/Makefile
index 38cfa81053488..460d78b9c880a 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/Makefile
@@ -1,3 +1,3 @@
 CXX_SOURCES := main.cpp
-override CXXFLAGS_EXTRAS += -std=c++14
+override CXXFLAGS_EXTRAS += -std=c++20
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
index c9f04f60ec24d..3a745883fb49c 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
@@ -2,7 +2,7 @@
 #include <libcxx-simulators-common/compressed_pair.h>
 
 namespace std {
-namespace __1 {
+inline namespace __1 {
 template <typename T> struct vector {
   T *__begin_;
   T *__end_;
@@ -10,11 +10,11 @@ template <typename T> struct vector {
 };
 } // namespace __1
 
-namespace __2 {
+inline namespace __2 {
 template <typename T> struct vector {};
 } // namespace __2
 
-namespace __3 {
+inline namespace __3 {
 template <typename T> struct vector {
   T *__begin_;
   T *__end_;

From bc1725ef3a2707c244d8c2a401d7545755f96cbd Mon Sep 17 00:00:00 2001
From: Sameer Sahasrabuddhe <sameer.sahasrabuddhe@amd.com>
Date: Thu, 17 Apr 2025 05:13:54 +0000
Subject: [PATCH 219/710] [clang] convergent attribute does not require "all
 threads" (#135803)

The documentation for the `convergent` attribute claims that OpenCL and
CUDA require "all threads" in a group to call the same convergent
operation. This is true only for OpenCL, and in general, the
`convergent` attribute is used in LLVM IR on operations that have no
such constraint.
---
 clang/include/clang/Basic/AttrDocs.td | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 76f805ef373dd..29d1d2c69e824 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1680,10 +1680,6 @@ translated into the LLVM ``convergent`` attribute, which indicates that the call
 instructions of a function with this attribute cannot be made control-dependent
 on any additional values.
 
-In languages designed for SPMD/SIMT programming model, e.g. OpenCL or CUDA,
-the call instructions of a function with this attribute must be executed by
-all work items or threads in a work group or sub group.
-
 This attribute is different from ``noduplicate`` because it allows duplicating
 function calls if it can be proved that the duplicated function calls are
 not made control-dependent on any additional values, e.g., unrolling a loop

From 4d84d3dfba38ee4151350ef2ec8c1d92ba8f6d36 Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental@gmail.com>
Date: Thu, 17 Apr 2025 01:31:19 -0400
Subject: [PATCH 220/710] [mlir][python][smt] fix DialectSMT (include
 NanobindUtils.h)

---
 mlir/python/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt
index bbf6819608bb9..e3934fc9f3511 100644
--- a/mlir/python/CMakeLists.txt
+++ b/mlir/python/CMakeLists.txt
@@ -680,6 +680,8 @@ declare_mlir_python_extension(MLIRPythonExtension.Dialects.SMT.Pybind
   PYTHON_BINDINGS_LIBRARY nanobind
   SOURCES
     DialectSMT.cpp
+    # Headers must be included explicitly so they are installed.
+    NanobindUtils.h
   PRIVATE_LINK_LIBS
     LLVMSupport
   EMBED_CAPI_LINK_LIBS

From 1fd7e4c517141ddfb527e7560e02fd5856244971 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Thu, 17 Apr 2025 05:53:31 +0000
Subject: [PATCH 221/710] Revert "[CI] monolithic-linux improvements (#135499)"

This reverts commit a399c6926a8701083c767cbb041e22ff92e9d717.

This is causing some premerge workflow failures.

Examople:
https://buildkite.com/llvm-project/github-pull-requests/builds/169129#01963d1d-dc75-4b4c-9952-fb60efbf91b4
---
 .ci/compute_projects.py      | 10 ++---
 .ci/compute_projects_test.py | 13 -------
 .ci/monolithic-linux.sh      | 72 ++++++++++++++++++++----------------
 lldb/test/requirements.txt   |  2 -
 4 files changed, 44 insertions(+), 53 deletions(-)

diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py
index 17a2136a270d5..ff43547c9bbe5 100644
--- a/.ci/compute_projects.py
+++ b/.ci/compute_projects.py
@@ -52,9 +52,6 @@
     "clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
     "clang-tools-extra": {"libc"},
     "mlir": {"flang"},
-    # Test everything if ci scripts are changed.
-    # FIXME: Figure out what is missing and add here.
-    ".ci": {"llvm", "clang", "lld", "lldb"},
 }
 
 DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
@@ -133,11 +130,12 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
 def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     projects_to_test = set()
     for modified_project in modified_projects:
+        # Skip all projects where we cannot run tests.
+        if modified_project not in PROJECT_CHECK_TARGETS:
+            continue
         if modified_project in RUNTIMES:
             continue
-        # Skip all projects where we cannot run tests.
-        if modified_project in PROJECT_CHECK_TARGETS:
-            projects_to_test.add(modified_project)
+        projects_to_test.add(modified_project)
         if modified_project not in DEPENDENTS_TO_TEST:
             continue
         for dependent_project in DEPENDENTS_TO_TEST[modified_project]:
diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py
index 1ab1c82498932..e787fd8133c86 100644
--- a/.ci/compute_projects_test.py
+++ b/.ci/compute_projects_test.py
@@ -188,19 +188,6 @@ def test_exclude_gn(self):
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
 
-     def test_ci(self):
-        env_variables = compute_projects.get_env_variables(
-            [".ci/compute_projects.py"], "Linux"
-        )
-        self.assertEqual(env_variables["projects_to_build"],
-                         "clang;lld;llvm;lldb")
-        self.assertEqual(env_variables["project_check_targets"], "check-clang
-                         check-lld check-llvm check-lldb")
-        self.assertEqual(env_variables["runtimes_to_build"],
-                         "libcxx;libcxxabi;libunwind")
-        self.assertEqual(env_variables["runtimes_check_targets"], "check-cxx
-                         check-cxxabi check-unwind")
-
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index f81a14cca6cb3..6461c9d40ad59 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -18,6 +18,7 @@ set -o pipefail
 
 MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
 BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
+INSTALL_DIR="${BUILD_DIR}/install"
 rm -rf "${BUILD_DIR}"
 
 ccache --zero-stats
@@ -27,14 +28,10 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
   ccache --clear
 fi
 
-mkdir -p artifacts/reproducers
-
-# Make sure any clang reproducers will end up as artifacts.
-export CLANG_CRASH_DIAGNOSTICS_DIR=`realpath artifacts/reproducers`
-
 function at-exit {
   retcode=$?
 
+  mkdir -p artifacts
   ccache --print-stats > artifacts/ccache_stats.txt
   cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
 
@@ -53,28 +50,17 @@ trap at-exit EXIT
 
 projects="${1}"
 targets="${2}"
-runtimes="${3}"
 
 lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
 
 echo "--- cmake"
-
 export PIP_BREAK_SYSTEM_PACKAGES=1
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
-
-# Set the system llvm-symbolizer as preferred.
-export LLVM_SYMBOLIZER_PATH=`which llvm-symbolizer`
-[[ ! -f "${LLVM_SYMBOLIZER_PATH}" ]] && echo "llvm-symbolizer not found!"
-
-# Set up all runtimes either way. libcxx is a dependency of LLDB.
-# If it ends up being unused, not much harm.
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
-      -D LLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
       -G Ninja \
-      -D CMAKE_PREFIX_PATH="${HOME}/.local" \
       -D CMAKE_BUILD_TYPE=Release \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
@@ -83,47 +69,69 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_LLD=ON \
       -D CMAKE_CXX_FLAGS=-gmlt \
       -D LLVM_CCACHE_BUILD=ON \
-      -D LIBCXX_CXX_ABI=libcxxabi \
       -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
-      -D LLDB_ENABLE_PYTHON=ON \
-      -D LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=ON
+      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
 
 echo "--- ninja"
-
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
 
+runtimes="${3}"
 runtime_targets="${4}"
 
-# Run runtimes tests.
-# We don't need to do a clean separate build of runtimes, because runtimes
-# will be built against just built clang, and because LIBCXX_TEST_PARAMS
-# and LIBCXXABI_TEST_PARAMS only affect lit configuration, which successfully
-# propagates without a clean build. Other that those two variables, builds
-# are supposed to be the same.
+# Compiling runtimes with just-built Clang and running their tests
+# as an additional testing for Clang.
 if [[ "${runtimes}" != "" ]]; then
   if [[ "${runtime_targets}" == "" ]]; then
     echo "Runtimes to build are specified, but targets are not."
     exit 1
   fi
 
+  echo "--- ninja install-clang"
+
+  ninja -C ${BUILD_DIR} install-clang install-clang-resource-headers
+
+  RUNTIMES_BUILD_DIR="${MONOREPO_ROOT}/build-runtimes"
+  INSTALL_DIR="${BUILD_DIR}/install"
+  mkdir -p ${RUNTIMES_BUILD_DIR}
+
   echo "--- cmake runtimes C++26"
 
-  cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
+  rm -rf "${RUNTIMES_BUILD_DIR}"
+  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
+      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
+      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
+      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
+      -D LIBCXX_CXX_ABI=libcxxabi \
+      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
+      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
       -D LIBCXX_TEST_PARAMS="std=c++26" \
-      -D LIBCXXABI_TEST_PARAMS="std=c++26"
+      -D LIBCXXABI_TEST_PARAMS="std=c++26" \
+      -D LLVM_LIT_ARGS="${lit_args}"
 
   echo "--- ninja runtimes C++26"
 
-  ninja -vC "${BUILD_DIR}" ${runtime_targets}
+  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
 
   echo "--- cmake runtimes clang modules"
 
-  cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
+  # We don't need to do a clean build of runtimes, because LIBCXX_TEST_PARAMS
+  # and LIBCXXABI_TEST_PARAMS only affect lit configuration, which successfully
+  # propagates without a clean build. Other that those two variables, builds
+  # are supposed to be the same.
+
+  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
+      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
+      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
+      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
+      -D LIBCXX_CXX_ABI=libcxxabi \
+      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
+      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
       -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
-      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang"
+      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
+      -D LLVM_LIT_ARGS="${lit_args}"
 
   echo "--- ninja runtimes clang modules"
 
-  ninja -vC "${BUILD_DIR}" ${runtime_targets}
+  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
 fi
diff --git a/lldb/test/requirements.txt b/lldb/test/requirements.txt
index c187180697d0b..708f2327a6a04 100644
--- a/lldb/test/requirements.txt
+++ b/lldb/test/requirements.txt
@@ -5,5 +5,3 @@ psutil>=5.9.4
 # See llvm.org/pr22274.
 pexpect>=4.9.0; sys_platform != 'win32'
 packaging
-# Required for python tests
-swig

From a2029ee91ddf0738f8b521ec8159068b6d65010f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 16 Apr 2025 22:41:25 -0700
Subject: [PATCH 222/710] [RISCV] Rename fields in RegRegOp. NFC

Rename Reg1->BaseReg, Reg2->OffsetReg.
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 013944787ff2d..f8df7ee4c5019 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -395,8 +395,8 @@ struct RISCVOperand final : public MCParsedAsmOperand {
   };
 
   struct RegRegOp {
-    MCRegister Reg1;
-    MCRegister Reg2;
+    MCRegister BaseReg;
+    MCRegister OffsetReg;
   };
 
   SMLoc StartLoc, EndLoc;
@@ -1043,8 +1043,8 @@ struct RISCVOperand final : public MCParsedAsmOperand {
       OS << '>';
       break;
     case KindTy::RegReg:
-      OS << "<RegReg:  Reg1 " << RegName(RegReg.Reg1);
-      OS << " Reg2 " << RegName(RegReg.Reg2);
+      OS << "<RegReg: BaseReg " << RegName(RegReg.BaseReg) << " OffsetReg "
+         << RegName(RegReg.OffsetReg);
       break;
     }
   }
@@ -1129,11 +1129,11 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     return Op;
   }
 
-  static std::unique_ptr<RISCVOperand> createRegReg(MCRegister Reg1,
-                                                    MCRegister Reg2, SMLoc S) {
+  static std::unique_ptr<RISCVOperand>
+  createRegReg(MCRegister BaseReg, MCRegister OffsetReg, SMLoc S) {
     auto Op = std::make_unique<RISCVOperand>(KindTy::RegReg);
-    Op->RegReg.Reg1 = Reg1;
-    Op->RegReg.Reg2 = Reg2;
+    Op->RegReg.BaseReg = BaseReg;
+    Op->RegReg.OffsetReg = OffsetReg;
     Op->StartLoc = S;
     Op->EndLoc = S;
     return Op;
@@ -1213,8 +1213,8 @@ struct RISCVOperand final : public MCParsedAsmOperand {
 
   void addRegRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 2 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(RegReg.Reg1));
-    Inst.addOperand(MCOperand::createReg(RegReg.Reg2));
+    Inst.addOperand(MCOperand::createReg(RegReg.BaseReg));
+    Inst.addOperand(MCOperand::createReg(RegReg.OffsetReg));
   }
 
   void addStackAdjOperands(MCInst &Inst, unsigned N) const {

From dbb79c30c9f3578b7afd9ea0ec33f82e70e472c7 Mon Sep 17 00:00:00 2001
From: wangjue <wangjue@zhcomputing.com>
Date: Thu, 17 Apr 2025 14:01:00 +0800
Subject: [PATCH 223/710] [BOLT][Instrumentation] Initial instrumentation
 support for RISCV64 (#133882)

This patch adds code generation for RISCV64 instrumentation.The work
    involved includes the following three points:

a) Implements support for instrumenting direct function call and jump
    on RISC-V which relies on , Atomic instructions
    (used to increment counters) are only available on RISC-V when the A
    extension is used.

b) Implements support for instrumenting direct function inderect call
    by implementing the createInstrumentedIndCallHandlerEntryBB and
createInstrumentedIndCallHandlerExitBB interfaces. In this process, we
    need to accurately record the target address and IndCallID to ensure
    the correct recording of the indirect call counters.

c)Implemented the RISCV64 Bolt runtime library, implemented some system
call interfaces through embedded assembly. Get the difference between
runtime addrress of .text section andstatic address in section header
table, which in turn can be used to search for indirect call
description.

However, the community code currently has problems with relocation in
    some scenarios, but this has nothing to do with instrumentation. We
    may continue to submit patches to fix the related bugs.
---
 bolt/CMakeLists.txt                           |   3 +-
 bolt/lib/Core/Relocation.cpp                  |   4 +
 bolt/lib/Rewrite/RewriteInstance.cpp          |  10 +-
 bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp  | 378 ++++++++++++++
 bolt/runtime/CMakeLists.txt                   |  14 +-
 bolt/runtime/common.h                         |   4 +-
 bolt/runtime/instr.cpp                        |  48 ++
 bolt/runtime/sys_riscv64.h                    | 460 ++++++++++++++++++
 .../runtime/RISCV/basic-instrumentation.s     |  33 ++
 .../runtime/RISCV/instrumentation-ind-call.c  |  39 ++
 bolt/test/runtime/RISCV/lit.local.cfg         |   2 +
 11 files changed, 984 insertions(+), 11 deletions(-)
 create mode 100644 bolt/runtime/sys_riscv64.h
 create mode 100644 bolt/test/runtime/RISCV/basic-instrumentation.s
 create mode 100644 bolt/test/runtime/RISCV/instrumentation-ind-call.c
 create mode 100644 bolt/test/runtime/RISCV/lit.local.cfg

diff --git a/bolt/CMakeLists.txt b/bolt/CMakeLists.txt
index 73c3ab54722e9..52c796518ac05 100644
--- a/bolt/CMakeLists.txt
+++ b/bolt/CMakeLists.txt
@@ -82,7 +82,8 @@ endforeach()
 
 set(BOLT_ENABLE_RUNTIME_default OFF)
 if ((CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64"
-    OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)$")
+    OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)$"
+    OR CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64")
     AND (CMAKE_SYSTEM_NAME STREQUAL "Linux"
       OR CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     AND (NOT CMAKE_CROSSCOMPILING))
diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index ff1681f823987..f099dfa46f3d4 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -123,6 +123,7 @@ static bool isSupportedRISCV(uint32_t Type) {
   case ELF::R_RISCV_LO12_S:
   case ELF::R_RISCV_64:
   case ELF::R_RISCV_TLS_GOT_HI20:
+  case ELF::R_RISCV_TLS_GD_HI20:
   case ELF::R_RISCV_TPREL_HI20:
   case ELF::R_RISCV_TPREL_ADD:
   case ELF::R_RISCV_TPREL_LO12_I:
@@ -236,6 +237,7 @@ static size_t getSizeForTypeRISCV(uint32_t Type) {
   case ELF::R_RISCV_64:
   case ELF::R_RISCV_GOT_HI20:
   case ELF::R_RISCV_TLS_GOT_HI20:
+  case ELF::R_RISCV_TLS_GD_HI20:
     // See extractValueRISCV for why this is necessary.
     return 8;
   }
@@ -491,6 +493,7 @@ static uint64_t extractValueRISCV(uint32_t Type, uint64_t Contents,
     return extractBImmRISCV(Contents);
   case ELF::R_RISCV_GOT_HI20:
   case ELF::R_RISCV_TLS_GOT_HI20:
+  case ELF::R_RISCV_TLS_GD_HI20:
     // We need to know the exact address of the GOT entry so we extract the
     // value from both the AUIPC and L[D|W]. We cannot rely on the symbol in the
     // relocation for this since it simply refers to the object that is stored
@@ -707,6 +710,7 @@ static bool isPCRelativeRISCV(uint32_t Type) {
   case ELF::R_RISCV_RVC_BRANCH:
   case ELF::R_RISCV_32_PCREL:
   case ELF::R_RISCV_TLS_GOT_HI20:
+  case ELF::R_RISCV_TLS_GD_HI20:
     return true;
   }
 }
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index a03df3de39e83..37dcfa868c211 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -2926,12 +2926,12 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
 
     if (BinaryData *BD = BC->getBinaryDataContainingAddress(SymbolAddress)) {
       // Note: this assertion is trying to check sanity of BinaryData objects
-      // but AArch64 has inferred and incomplete object locations coming from
-      // GOT/TLS or any other non-trivial relocation (that requires creation
-      // of sections and whose symbol address is not really what should be
-      // encoded in the instruction). So we essentially disabled this check
+      // but AArch64 and RISCV has inferred and incomplete object locations
+      // coming from GOT/TLS or any other non-trivial relocation (that requires
+      // creation of sections and whose symbol address is not really what should
+      // be encoded in the instruction). So we essentially disabled this check
       // for AArch64 and live with bogus names for objects.
-      assert((IsAArch64 || IsSectionRelocation ||
+      assert((IsAArch64 || BC->isRISCV() || IsSectionRelocation ||
               BD->nameStartsWith(SymbolName) ||
               BD->nameStartsWith("PG" + SymbolName) ||
               (BD->nameStartsWith("ANONYMOUS") &&
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index 4320c679acd54..0e27d29019e95 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -14,7 +14,9 @@
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "bolt/Core/MCPlusBuilder.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 
@@ -72,6 +74,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     case ELF::R_RISCV_LO12_I:
     case ELF::R_RISCV_LO12_S:
     case ELF::R_RISCV_TLS_GOT_HI20:
+    case ELF::R_RISCV_TLS_GD_HI20:
       return true;
     default:
       llvm_unreachable("Unexpected RISCV relocation type in code");
@@ -252,6 +255,11 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     return createCall(RISCV::PseudoCALL, Inst, Target, Ctx);
   }
 
+  void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
+                          MCContext *Ctx) override {
+    createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true);
+  }
+
   void createTailCall(MCInst &Inst, const MCSymbol *Target,
                       MCContext *Ctx) override {
     return createCall(RISCV::PseudoTAIL, Inst, Target, Ctx);
@@ -424,6 +432,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
       return Expr;
     case ELF::R_RISCV_GOT_HI20:
     case ELF::R_RISCV_TLS_GOT_HI20:
+    case ELF::R_RISCV_TLS_GD_HI20:
       // The GOT is reused so no need to create GOT relocations
     case ELF::R_RISCV_PCREL_HI20:
       return RISCVMCExpr::create(Expr, RISCVMCExpr::VK_PCREL_HI, Ctx);
@@ -483,6 +492,375 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
       return 2;
     return 4;
   }
+
+  void createStackPointerIncrement(
+      MCInst &Inst, int imm,
+      bool NoFlagsClobber = false /*unused for RISCV*/) const override {
+    Inst = MCInstBuilder(RISCV::ADDI)
+               .addReg(RISCV::X2)
+               .addReg(RISCV::X2)
+               .addImm(-imm);
+  }
+
+  void createStackPointerDecrement(
+      MCInst &Inst, int imm,
+      bool NoFlagsClobber = false /*unused for RISCV*/) const override {
+    Inst = MCInstBuilder(RISCV::ADDI)
+               .addReg(RISCV::X2)
+               .addReg(RISCV::X2)
+               .addImm(imm);
+  }
+
+  void loadReg(MCInst &Inst, MCPhysReg To, MCPhysReg From,
+               int64_t offset) const {
+    Inst = MCInstBuilder(RISCV::LD).addReg(To).addReg(From).addImm(offset);
+  }
+
+  void storeReg(MCInst &Inst, MCPhysReg From, MCPhysReg To,
+                int64_t offset) const {
+    Inst = MCInstBuilder(RISCV::SD).addReg(From).addReg(To).addImm(offset);
+  }
+
+  void spillRegs(InstructionListType &Insts,
+                 const SmallVector<unsigned> &Regs) const {
+    Insts.emplace_back();
+    createStackPointerIncrement(Insts.back(), Regs.size() * 8);
+
+    int64_t Offset = 0;
+    for (auto Reg : Regs) {
+      Insts.emplace_back();
+      storeReg(Insts.back(), Reg, RISCV::X2, Offset);
+      Offset += 8;
+    }
+  }
+
+  void reloadRegs(InstructionListType &Insts,
+                  const SmallVector<unsigned> &Regs) const {
+    int64_t Offset = 0;
+    for (auto Reg : Regs) {
+      Insts.emplace_back();
+      loadReg(Insts.back(), Reg, RISCV::X2, Offset);
+      Offset += 8;
+    }
+
+    Insts.emplace_back();
+    createStackPointerDecrement(Insts.back(), Regs.size() * 8);
+  }
+
+  void atomicAdd(MCInst &Inst, MCPhysReg RegAtomic, MCPhysReg RegTo,
+                 MCPhysReg RegCnt) const {
+    Inst = MCInstBuilder(RISCV::AMOADD_D)
+               .addReg(RegAtomic)
+               .addReg(RegTo)
+               .addReg(RegCnt);
+  }
+
+  InstructionListType createCmpJE(MCPhysReg RegNo, MCPhysReg RegTmp,
+                                  const MCSymbol *Target,
+                                  MCContext *Ctx) const {
+    InstructionListType Insts;
+    Insts.emplace_back(
+        MCInstBuilder(RISCV::SUB).addReg(RegTmp).addReg(RegNo).addReg(RegNo));
+    Insts.emplace_back(MCInstBuilder(RISCV::BEQ)
+                           .addReg(RegNo)
+                           .addReg(RegTmp)
+                           .addExpr(MCSymbolRefExpr::create(
+                               Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    return Insts;
+  }
+
+  void createTrap(MCInst &Inst) const override {
+    Inst.clear();
+    Inst.setOpcode(RISCV::EBREAK);
+  }
+
+  void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
+                      MCContext *Ctx, bool IsTailCall) override {
+    // The sequence of instructions we create here is the following:
+    //  auipc   a5, hi20(Target)
+    //  addi    a5, a5, low12(Target)
+    //  jr x5 => jalr x0, x5, 0
+    MCPhysReg Reg = RISCV::X5;
+    InstructionListType Insts = materializeAddress(Target, Ctx, Reg);
+    Insts.emplace_back();
+    MCInst &Inst = Insts.back();
+    Inst.clear();
+    Inst = MCInstBuilder(RISCV::JALR).addReg(RISCV::X0).addReg(Reg).addImm(0);
+    if (IsTailCall)
+      setTailCall(Inst);
+    Seq.swap(Insts);
+  }
+
+  InstructionListType createGetter(MCContext *Ctx, const char *name) const {
+    InstructionListType Insts(4);
+    MCSymbol *Locs = Ctx->getOrCreateSymbol(name);
+    InstructionListType Addr = materializeAddress(Locs, Ctx, RISCV::X10);
+    std::copy(Addr.begin(), Addr.end(), Insts.begin());
+    loadReg(Insts[2], RISCV::X10, RISCV::X10, 0);
+    createReturn(Insts[3]);
+    return Insts;
+  }
+
+  InstructionListType createIncMemory(MCPhysReg RegTo, MCPhysReg RegCnt,
+                                      MCPhysReg RegAtomic) const {
+    InstructionListType Insts;
+    Insts.emplace_back();
+    Insts.back() =
+        MCInstBuilder(RISCV::ADDI).addReg(RegCnt).addReg(RegAtomic).addImm(1);
+    Insts.emplace_back();
+    atomicAdd(Insts.back(), RegAtomic, RegTo, RegCnt);
+    return Insts;
+  }
+
+  InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx,
+                                         MCPhysReg RegName,
+                                         int64_t Addend = 0) const override {
+    // Get the symbol address by auipc + addi
+    InstructionListType Insts(2);
+    MCSymbol *AuipcLabel = Ctx->createNamedTempSymbol("pcrel_hi");
+    Insts[0] = MCInstBuilder(RISCV::AUIPC).addReg(RegName).addImm(0);
+    setOperandToSymbolRef(Insts[0], /* OpNum */ 1, Target, Addend, Ctx,
+                          ELF::R_RISCV_PCREL_HI20);
+    setInstLabel(Insts[0], AuipcLabel);
+
+    Insts[1] =
+        MCInstBuilder(RISCV::ADDI).addReg(RegName).addReg(RegName).addImm(0);
+    setOperandToSymbolRef(Insts[1], /* OpNum */ 2, AuipcLabel, Addend, Ctx,
+                          ELF::R_RISCV_PCREL_LO12_I);
+    return Insts;
+  }
+
+  InstructionListType
+  createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
+                       unsigned CodePointerSize) const override {
+    // We need 2 scratch registers: one for the target address (x10), and one
+    // for the increment value (x11).
+    // addi sp, sp, -16
+    // sd x10, 0(sp)
+    // sd x11, 8(sp)
+    // la x10, target         # 1: auipc x10, %pcrel_hi(target)
+    //                        # addi x10, x10, %pcrel_lo(1b)
+    // li x11, 1              # addi x11, zero, 1
+    // amoadd.d zero, x10, x11
+    // ld x10, 0(sp)
+    // ld x11, 8(sp)
+    // addi sp, sp, 16
+
+    InstructionListType Insts;
+    spillRegs(Insts, {RISCV::X10, RISCV::X11});
+    InstructionListType Addr = materializeAddress(Target, Ctx, RISCV::X10);
+    Insts.insert(Insts.end(), Addr.begin(), Addr.end());
+    InstructionListType IncInsts =
+        createIncMemory(RISCV::X10, RISCV::X11, RISCV::X0);
+    Insts.insert(Insts.end(), IncInsts.begin(), IncInsts.end());
+    reloadRegs(Insts, {RISCV::X10, RISCV::X11});
+    return Insts;
+  }
+
+  void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
+                        bool IsTailCall) override {
+    Inst.setOpcode(RISCV::JAL);
+    Inst.clear();
+    if (IsTailCall) {
+      Inst.addOperand(MCOperand::createReg(RISCV::X0));
+      Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
+          Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
+          *Ctx, 0)));
+      convertJmpToTailCall(Inst);
+    } else {
+      Inst.addOperand(MCOperand::createReg(RISCV::X1));
+      Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
+          Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
+          *Ctx, 0)));
+    }
+  }
+
+  void createIndirectCallInst(MCInst &Inst, bool IsTailCall, MCPhysReg Reg,
+                              int64_t Disp) const {
+    Inst.clear();
+    Inst.setOpcode(RISCV::JALR);
+    Inst.clear();
+    if (IsTailCall) {
+      Inst.addOperand(MCOperand::createReg(RISCV::X0));
+      Inst.addOperand(MCOperand::createReg(Reg));
+      Inst.addOperand(MCOperand::createImm(Disp));
+    } else {
+      Inst.addOperand(MCOperand::createReg(RISCV::X1));
+      Inst.addOperand(MCOperand::createReg(Reg));
+      Inst.addOperand(MCOperand::createImm(Disp));
+    }
+  }
+
+  InstructionListType
+  createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
+                                          const MCSymbol *IndCallHandler,
+                                          MCContext *Ctx) override {
+    // Code sequence used to check whether InstrTampoline was initialized
+    // and call it if so, returns via IndCallHandler
+    //   sp      -16(sp)
+    //   sd      x10, 0(sp)
+    //   sd      x11, 0(sp)
+    //   la      x10, InstrTrampoline -> auipc + addi
+    //   ld      x10, [x10]
+    //   beq     x10, x11, IndCallHandler
+    //   sp      -16(sp)
+    //   sd      x1, 0(sp)
+    //   jalr    x1,x10,0
+    //   ld      x1, [sp], #16
+    //   sp      16(sp)
+    //   jal     x0, IndCallHandler
+
+    InstructionListType Insts;
+    spillRegs(Insts, {RISCV::X10, RISCV::X11});
+    InstructionListType Addr =
+        materializeAddress(InstrTrampoline, Ctx, RISCV::X10);
+    Insts.insert(Insts.end(), Addr.begin(), Addr.end());
+    Insts.emplace_back();
+    loadReg(Insts.back(), RISCV::X10, RISCV::X10, 0);
+    InstructionListType cmpJmp =
+        createCmpJE(RISCV::X10, RISCV::X11, IndCallHandler, Ctx);
+    Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end());
+    Insts.emplace_back();
+    createStackPointerIncrement(Insts.back(), 16);
+    Insts.emplace_back();
+    storeReg(Insts.back(), RISCV::X1, RISCV::X2, 0);
+    Insts.emplace_back();
+    createIndirectCallInst(Insts.back(), /*IsTailCall*/ false, RISCV::X10, 0);
+    Insts.emplace_back();
+    loadReg(Insts.back(), RISCV::X1, RISCV::X2, 0);
+    Insts.emplace_back();
+    createStackPointerDecrement(Insts.back(), 16);
+    Insts.emplace_back();
+    createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true);
+    return Insts;
+  }
+
+  InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
+    InstructionListType Insts;
+    reloadRegs(Insts, {RISCV::X10, RISCV::X11});
+    Insts.emplace_back();
+    loadReg(Insts.back(), RISCV::X5, RISCV::X2, 0);
+    Insts.emplace_back();
+    createStackPointerDecrement(Insts.back(), 16);
+    reloadRegs(Insts, {RISCV::X10, RISCV::X11});
+    Insts.emplace_back();
+    createIndirectCallInst(Insts.back(), /*IsTailCall*/ true, RISCV::X5, 0);
+    return Insts;
+  }
+
+  InstructionListType
+  createInstrumentedIndTailCallHandlerExitBB() const override {
+    return createInstrumentedIndCallHandlerExitBB();
+  }
+
+  std::vector<MCInst> createSymbolTrampoline(const MCSymbol *TgtSym,
+                                             MCContext *Ctx) override {
+    std::vector<MCInst> Insts;
+    createShortJmp(Insts, TgtSym, Ctx, /*IsTailCall*/ true);
+    return Insts;
+  }
+
+  InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
+    return createGetter(Ctx, "__bolt_num_counters");
+  }
+
+  InstructionListType
+  createInstrLocationsGetter(MCContext *Ctx) const override {
+    return createGetter(Ctx, "__bolt_instr_locations");
+  }
+
+  InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
+    return createGetter(Ctx, "__bolt_instr_tables");
+  }
+
+  InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
+    return createGetter(Ctx, "__bolt_instr_num_funcs");
+  }
+
+  void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg,
+                                 MCPhysReg ZeroReg) const {
+    bool IsTailCall = isTailCall(Inst);
+    if (IsTailCall)
+      removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
+    Inst.setOpcode(RISCV::ADD);
+    Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
+    Inst.insert(Inst.begin() + 1, MCOperand::createReg(ZeroReg));
+    return;
+  }
+
+  InstructionListType createLoadImmediate(const MCPhysReg Dest,
+                                          uint64_t Imm) const override {
+    InstructionListType Insts;
+    // get IMM higher 32bit
+    Insts.emplace_back(
+        MCInstBuilder(RISCV::LUI).addReg(Dest).addImm((Imm >> 44) & 0xFFFFF));
+    Insts.emplace_back(MCInstBuilder(RISCV::LUI)
+                           .addReg(RISCV::X5)
+                           .addImm((Imm >> 32) & 0xFFF));
+    Insts.emplace_back(MCInstBuilder(RISCV::SRLI)
+                           .addReg(RISCV::X5)
+                           .addReg(RISCV::X5)
+                           .addImm(12));
+    Insts.emplace_back(
+        MCInstBuilder(RISCV::OR).addReg(Dest).addReg(Dest).addReg(RISCV::X5));
+    Insts.emplace_back(
+        MCInstBuilder(RISCV::SLLI).addReg(Dest).addReg(Dest).addImm(32));
+
+    // get IMM lower 32bit
+    Insts.emplace_back(MCInstBuilder(RISCV::LUI)
+                           .addReg(RISCV::X5)
+                           .addImm((Imm >> 12) & 0xFFFFF));
+    Insts.emplace_back(
+        MCInstBuilder(RISCV::LUI).addReg(RISCV::X6).addImm((Imm)&0xFFF));
+    Insts.emplace_back(MCInstBuilder(RISCV::SRLI)
+                           .addReg(RISCV::X6)
+                           .addReg(RISCV::X6)
+                           .addImm(12));
+    Insts.emplace_back(
+        MCInstBuilder(RISCV::OR).addReg(RISCV::X5).addReg(RISCV::X5).addReg(
+            RISCV::X6));
+
+    // get 64bit IMM
+    Insts.emplace_back(
+        MCInstBuilder(RISCV::OR).addReg(Dest).addReg(Dest).addReg(RISCV::X5));
+    return Insts;
+  }
+
+  InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,
+                                                     MCSymbol *HandlerFuncAddr,
+                                                     int CallSiteID,
+                                                     MCContext *Ctx) override {
+    // Code sequence used to enter indirect call instrumentation helper:
+    //   addi  sp, sp, -0x10
+    //   sd  a0, 0x0(sp)
+    //   sd  a1, 0x8(sp)
+    //   mov target x0  convertIndirectCallToLoad -> add a0, zero, target
+    //   mov x1 CallSiteID createLoadImmediate
+    //   addi  sp, sp, -0x10
+    //   sd  a0, 0x0(sp)
+    //   sd  a1, 0x8(sp)
+    //   la x0 *HandlerFuncAddr -> auipc + addi
+    //   jalr x0
+
+    InstructionListType Insts;
+    spillRegs(Insts, {RISCV::X10, RISCV::X11});
+    Insts.emplace_back(CallInst);
+    convertIndirectCallToLoad(Insts.back(), RISCV::X10, RISCV::X0);
+    InstructionListType LoadImm = createLoadImmediate(RISCV::X11, CallSiteID);
+    Insts.insert(Insts.end(), LoadImm.begin(), LoadImm.end());
+    spillRegs(Insts, {RISCV::X10, RISCV::X11});
+    InstructionListType Addr =
+        materializeAddress(HandlerFuncAddr, Ctx, RISCV::X5);
+    Insts.insert(Insts.end(), Addr.begin(), Addr.end());
+    Insts.emplace_back();
+    createIndirectCallInst(Insts.back(), isTailCall(CallInst), RISCV::X5, 0);
+
+    // // Carry over metadata including tail call marker if present.
+    stripAnnotations(Insts.back());
+    moveAnnotations(std::move(CallInst), Insts.back());
+
+    return Insts;
+  }
 };
 
 } // end anonymous namespace
diff --git a/bolt/runtime/CMakeLists.txt b/bolt/runtime/CMakeLists.txt
index 0deb69a27d435..87cc44812da11 100644
--- a/bolt/runtime/CMakeLists.txt
+++ b/bolt/runtime/CMakeLists.txt
@@ -35,15 +35,21 @@ set(BOLT_RT_FLAGS
   -fno-exceptions
   -fno-rtti
   -fno-stack-protector
-  -fPIC
-  -mgeneral-regs-only)
+  -fPIC)
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
-  set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS} "-mno-sse")
+  set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS} 
+    -mno-sse 
+    -mgeneral-regs-only)
+endif()
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
+  set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS})
 endif()
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
   check_cxx_compiler_flag("-mno-outline-atomics" CXX_SUPPORTS_OUTLINE_ATOMICS)
   if (CXX_SUPPORTS_OUTLINE_ATOMICS)
-    set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS} "-mno-outline-atomics")
+    set(BOLT_RT_FLAGS ${BOLT_RT_FLAGS} 
+      -mno-outline-atomics 
+      -mgeneral-regs-only)
   endif()
 endif()
 
diff --git a/bolt/runtime/common.h b/bolt/runtime/common.h
index 27d0830071067..3461f8c545f9f 100644
--- a/bolt/runtime/common.h
+++ b/bolt/runtime/common.h
@@ -153,10 +153,12 @@ struct timespec {
 
 #if defined(__aarch64__) || defined(__arm64__)
 #include "sys_aarch64.h"
+#elif defined(__riscv)
+#include "sys_riscv64.h"
 #elif defined(__x86_64__)
 #include "sys_x86_64.h"
 #else
-#error "For AArch64/ARM64 and X86_64 only."
+#error "For AArch64/ARM64,X86_64 AND RISCV64 only."
 #endif
 
 constexpr uint32_t BufSize = 10240;
diff --git a/bolt/runtime/instr.cpp b/bolt/runtime/instr.cpp
index d1f8a216badcf..ae356e71cbe41 100644
--- a/bolt/runtime/instr.cpp
+++ b/bolt/runtime/instr.cpp
@@ -1674,6 +1674,19 @@ extern "C" __attribute((naked)) void __bolt_instr_indirect_call()
                        "ret\n"
                        :::);
   // clang-format on
+#elif defined(__riscv)
+  // clang-format off
+  __asm__ __volatile__(
+                      SAVE_ALL
+                      "addi sp, sp, 288\n"
+                      "ld x10, 0(sp)\n"
+                      "ld x11, 8(sp)\n"
+                      "addi sp, sp, -288\n"
+                      "jal x1, instrumentIndirectCall\n"
+                      RESTORE_ALL
+                      "ret\n"
+                      :::);
+  // clang-format on
 #else
   // clang-format off
   __asm__ __volatile__(SAVE_ALL
@@ -1698,6 +1711,18 @@ extern "C" __attribute((naked)) void __bolt_instr_indirect_tailcall()
                        "ret\n"
                        :::);
   // clang-format on
+#elif defined(__riscv)
+  // clang-format off
+  __asm__ __volatile__(SAVE_ALL
+                      "addi sp, sp, 288\n"
+                      "ld x10, 0(sp)\n"
+                      "ld x11, 8(sp)\n"
+                      "addi sp, sp, -288\n"
+                      "jal x1, instrumentIndirectCall\n"
+                      RESTORE_ALL
+                      "ret\n"
+                      :::);
+  // clang-format on
 #else
   // clang-format off
   __asm__ __volatile__(SAVE_ALL
@@ -1724,6 +1749,18 @@ extern "C" __attribute((naked)) void __bolt_instr_start()
                        "br x16\n"
                        :::);
   // clang-format on
+#elif defined(__riscv)
+  // clang-format off
+  __asm__ __volatile__(
+                      SAVE_ALL
+                      "jal x1, __bolt_instr_setup\n"
+                      RESTORE_ALL
+                      "setup_symbol:\n"
+                      "auipc x5, %%pcrel_hi(__bolt_start_trampoline)\n" 
+                      "addi x5, x5, %%pcrel_lo(setup_symbol)\n"
+                      "jr x5\n"
+                      :::);
+  // clang-format on
 #else
   // clang-format off
   __asm__ __volatile__(SAVE_ALL
@@ -1746,6 +1783,17 @@ extern "C" void __bolt_instr_fini() {
                        RESTORE_ALL
                        :::);
   // clang-format on
+#elif defined(__riscv)
+  // clang-format off
+  __asm__ __volatile__(
+                      SAVE_ALL
+                      "fini_symbol:\n"
+                      "auipc x5, %%pcrel_hi(__bolt_fini_trampoline)\n" 
+                      "addi x5, x5, %%pcrel_lo(fini_symbol)\n" 
+                      "jalr x1, 0(x5)\n"
+                      RESTORE_ALL
+                      :::);
+  // clang-format on
 #else
   __asm__ __volatile__("call __bolt_fini_trampoline\n" :::);
 #endif
diff --git a/bolt/runtime/sys_riscv64.h b/bolt/runtime/sys_riscv64.h
new file mode 100644
index 0000000000000..00a21e4945f0f
--- /dev/null
+++ b/bolt/runtime/sys_riscv64.h
@@ -0,0 +1,460 @@
+#ifndef LLVM_TOOLS_LLVM_BOLT_SYS_RISCV
+#define LLVM_TOOLS_LLVM_BOLT_SYS_RISCV
+
+// Save all registers while keeping 16B stack alignment
+#define SAVE_ALL                                                               \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x0, 0(sp)\n"                                                             \
+  "sd x1, 8(sp)\n"                                                             \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x2, 0(sp)\n"                                                             \
+  "sd x3, 8(sp)\n"                                                             \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x4, 0(sp)\n"                                                             \
+  "sd x5, 8(sp)\n"                                                             \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x6, 0(sp)\n"                                                             \
+  "sd x7, 8(sp)\n"                                                             \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x8, 0(sp)\n"                                                             \
+  "sd x9, 8(sp)\n"                                                             \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x10, 0(sp)\n"                                                            \
+  "sd x11, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x12, 0(sp)\n"                                                            \
+  "sd x13, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x14, 0(sp)\n"                                                            \
+  "sd x15, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x16, 0(sp)\n"                                                            \
+  "sd x17, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x18, 0(sp)\n"                                                            \
+  "sd x19, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x20, 0(sp)\n"                                                            \
+  "sd x21, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x22, 0(sp)\n"                                                            \
+  "sd x23, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x24, 0(sp)\n"                                                            \
+  "sd x25, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x26, 0(sp)\n"                                                            \
+  "sd x27, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x28, 0(sp)\n"                                                            \
+  "sd x29, 8(sp)\n"                                                            \
+  "addi sp, sp, -16\n"                                                         \
+  "sd x30, 0(sp)\n"                                                            \
+  "sd x31, 8(sp)\n"
+// Mirrors SAVE_ALL
+#define RESTORE_ALL                                                            \
+  "ld  x30, 0(sp)\n"                                                           \
+  "ld  x31, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x28, 0(sp)\n"                                                           \
+  "ld  x29, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x26, 0(sp)\n"                                                           \
+  "ld  x27, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x24, 0(sp)\n"                                                           \
+  "ld  x25, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x22, 0(sp)\n"                                                           \
+  "ld  x23, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x20, 0(sp)\n"                                                           \
+  "ld  x21, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x18, 0(sp)\n"                                                           \
+  "ld  x19, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x16, 0(sp)\n"                                                           \
+  "ld  x17, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x14, 0(sp)\n"                                                           \
+  "ld  x15, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x12, 0(sp)\n"                                                           \
+  "ld  x13, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x10, 0(sp)\n"                                                           \
+  "ld  x11, 8(sp)\n"                                                           \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x8, 0(sp)\n"                                                            \
+  "ld  x9, 8(sp)\n"                                                            \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x6, 0(sp)\n"                                                            \
+  "ld  x7, 8(sp)\n"                                                            \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x4, 0(sp)\n"                                                            \
+  "ld  x5, 8(sp)\n"                                                            \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x2, 0(sp)\n"                                                            \
+  "ld  x3, 8(sp)\n"                                                            \
+  "addi sp, sp,  16\n"                                                         \
+  "ld  x0, 0(sp)\n"                                                            \
+  "ld  x1, 8(sp)\n"                                                            \
+  "addi sp, sp,  16\n"
+
+// Anonymous namespace covering everything but our library entry point
+namespace {
+
+// Get the difference between runtime addrress of .text section and
+// static address in section header table. Can be extracted from arbitrary
+// pc value recorded at runtime to get the corresponding static address, which
+// in turn can be used to search for indirect call description. Needed because
+// indirect call descriptions are read-only non-relocatable data.
+uint64_t getTextBaseAddress() {
+  uint64_t DynAddr;
+  uint64_t StaticAddr;
+  __asm__ volatile("lla %0, __hot_end\n\t"
+                   "lui %1, %%hi(__hot_end)\n\t"
+                   "addi %1, %1, %%lo(__hot_end)\n\t"
+                   : "=r"(DynAddr), "=r"(StaticAddr));
+  return DynAddr - StaticAddr;
+}
+
+uint64_t __read(uint64_t fd, const void *buf, uint64_t count) {
+  uint64_t ret;
+  register uint64_t a0 __asm__("a0") = fd;
+  register const void *a1 __asm__("a1") = buf;
+  register uint64_t a2 __asm__("a2") = count;
+  register uint64_t a7 __asm__("a7") =
+      63; // Assuming 63 is the syscall number for read
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret)
+                       : "r"(a0), "r"(a1), "r"(a2), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __write(uint64_t fd, const void *buf, uint64_t count) {
+  uint64_t ret;
+  register uint64_t a0 __asm__("a0") = fd;
+  register const void *a1 __asm__("a1") = buf;
+  register uint64_t a2 __asm__("a2") = count;
+  register uint32_t a7 __asm__("a7") =
+      64; // Assuming 64 is the syscall number for write
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret)
+                       : "r"(a0), "r"(a1), "r"(a2), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+void *__mmap(uint64_t addr, uint64_t size, uint64_t prot, uint64_t flags,
+             uint64_t fd, uint64_t offset) {
+  void *ret;
+  register uint64_t a0 __asm__("a0") = addr;
+  register uint64_t a1 __asm__("a1") = size;
+  register uint64_t a2 __asm__("a2") = prot;
+  register uint64_t a3 __asm__("a3") = flags;
+  register uint64_t a4 __asm__("a4") = fd;
+  register uint64_t a5 __asm__("a5") = offset;
+  register uint32_t a7 __asm__("a7") =
+      222; // Assuming 222 is the syscall number for mmap
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret)
+                       : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
+                         "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __munmap(void *addr, uint64_t size) {
+  uint64_t ret;
+  register void *a0 __asm__("a0") = addr;
+  register uint64_t a1 __asm__("a1") = size;
+  register uint32_t a7 __asm__("a7") = 215;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __exit(uint64_t code) {
+  uint64_t ret;
+  register uint64_t a0 __asm__("a0") = code;
+  register uint32_t a7 __asm__("a7") = 94;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __open(const char *pathname, uint64_t flags, uint64_t mode) {
+  uint64_t ret;
+  register int a0 __asm__("a0") =
+      -100; // Assuming -100 is an invalid file descriptor
+  register const char *a1 __asm__("a1") = pathname;
+  register uint64_t a2 __asm__("a2") = flags;
+  register uint64_t a3 __asm__("a3") = mode;
+  register uint64_t a7 __asm__("a7") =
+      56; // Assuming 56 is the syscall number for open
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret)
+                       : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+long __getdents64(unsigned int fd, dirent64 *dirp, size_t count) {
+  long ret;
+  register unsigned int a0 __asm__("a0") = fd;
+  register dirent64 *a1 __asm__("a1") = dirp;
+  register size_t a2 __asm__("a2") = count;
+  register uint32_t a7 __asm__("a7") = 61;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a2), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __readlink(const char *pathname, char *buf, size_t bufsize) {
+  uint64_t ret;
+  register int a0 __asm__("a0") = -100;
+  register const char *a1 __asm__("a1") = pathname;
+  register char *a2 __asm__("a2") = buf;
+  register size_t a3 __asm__("a3") = bufsize;
+  register uint32_t a7 __asm__("a7") = 78; // readlinkat
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a2), "r"(a3), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) {
+  uint64_t ret;
+  register uint64_t a0 __asm__("a0") = fd;
+  register uint64_t a1 __asm__("a1") = pos;
+  register uint64_t a2 __asm__("a2") = whence;
+  register uint32_t a7 __asm__("a7") = 62;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a2), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __ftruncate(uint64_t fd, uint64_t length) {
+  int ret;
+  register uint64_t a0 __asm__("a0") = fd;
+  register uint64_t a1 __asm__("a1") = length;
+  register uint32_t a7 __asm__("a7") = 46;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __close(uint64_t fd) {
+  int ret;
+  register uint64_t a0 __asm__("a0") = fd;
+  register uint32_t a7 __asm__("a7") =
+      57; // Assuming 57 is the syscall number for close
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret)
+                       : "r"(a0), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __madvise(void *addr, size_t length, int advice) {
+  int ret;
+  register void *a0 __asm__("a0") = addr;
+  register size_t a1 __asm__("a1") = length;
+  register int a2 __asm__("a2") = advice;
+  register uint32_t a7 __asm__("a7") = 233;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a2), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __uname(struct UtsNameTy *buf) {
+  int ret;
+  register UtsNameTy *a0 __asm__("a0") = buf;
+  register uint32_t a7 __asm__("a7") = 160;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __nanosleep(const timespec *req, timespec *rem) {
+  uint64_t ret;
+  register const timespec *a0 __asm__("a0") = req;
+  register timespec *a1 __asm__("a1") = rem;
+  register uint32_t a7 __asm__("a7") = 101;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int64_t __fork() {
+  uint64_t ret;
+  // clone instead of fork with flags
+  // "CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD"
+  register uint64_t a0 __asm__("a0") = 0x1200011;
+  register uint64_t a1 __asm__("a1") = 0;
+  register uint64_t a2 __asm__("a2") = 0;
+  register uint64_t a3 __asm__("a3") = 0;
+  register uint64_t a4 __asm__("a4") = 0;
+  register uint32_t a7 __asm__("a7") = 220;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a2), "r"(a3), "r"(a4), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __mprotect(void *addr, size_t len, int prot) {
+  int ret;
+  register void *a0 __asm__("a0") = addr;
+  register size_t a1 __asm__("a1") = len;
+  register int a2 __asm__("a2") = prot;
+  register uint32_t a7 __asm__("a7") = 226;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a2), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __getpid() {
+  uint64_t ret;
+  register uint32_t a7 __asm__("a7") = 172;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __getppid() {
+  uint64_t ret;
+  register uint32_t a7 __asm__("a7") = 173;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __setpgid(uint64_t pid, uint64_t pgid) {
+  int ret;
+  register uint64_t a0 __asm__("a0") = pid;
+  register uint64_t a1 __asm__("a1") = pgid;
+  register uint32_t a7 __asm__("a7") = 154;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __getpgid(uint64_t pid) {
+  uint64_t ret;
+  register uint64_t a0 __asm__("a0") = pid;
+  register uint32_t a7 __asm__("a7") = 155;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __kill(uint64_t pid, int sig) {
+  int ret;
+  register uint64_t a0 __asm__("a0") = pid;
+  register int a1 __asm__("a1") = sig;
+  register uint32_t a7 __asm__("a7") = 129;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __fsync(int fd) {
+  int ret;
+  register int a0 __asm__("a0") = fd;
+  register uint32_t a7 __asm__("a7") = 82;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0)
+                       : "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+uint64_t __sigprocmask(int how, const void *set, void *oldset) {
+  uint64_t ret;
+  register int a0 __asm__("a0") = how;
+  register const void *a1 __asm__("a1") = set;
+  register void *a2 __asm__("a2") = oldset;
+  register long a3 asm("a3") = 8;
+  register uint32_t a7 __asm__("a7") = 135;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a2), "r"(a3), "r"(a7)
+                       : "memory");
+  return ret;
+}
+
+int __prctl(int option, unsigned long arg2, unsigned long arg3,
+            unsigned long arg4, unsigned long arg5) {
+  int ret;
+  register int a0 __asm__("a0") = option;
+  register unsigned long a1 __asm__("a1") = arg2;
+  register unsigned long a2 __asm__("a2") = arg3;
+  register unsigned long a3 __asm__("a3") = arg4;
+  register unsigned long a4 __asm__("a4") = arg5;
+  register uint32_t a7 __asm__("a7") = 167;
+  __asm__ __volatile__("ecall\n\t"
+                       "mv %0, a0"
+                       : "=r"(ret), "+r"(a0), "+r"(a1)
+                       : "r"(a2), "r"(a3), "r"(a4), "r"(a7)
+                       : "cc", "memory");
+  return ret;
+}
+} // anonymous namespace
+
+#endif
\ No newline at end of file
diff --git a/bolt/test/runtime/RISCV/basic-instrumentation.s b/bolt/test/runtime/RISCV/basic-instrumentation.s
new file mode 100644
index 0000000000000..e926f98cef43b
--- /dev/null
+++ b/bolt/test/runtime/RISCV/basic-instrumentation.s
@@ -0,0 +1,33 @@
+# REQUIRES: system-linux,bolt-runtime
+
+# RUN: %clang %cflags -Wl,-q -o %t.exe %s
+# RUN: llvm-bolt --instrument --instrumentation-file=%t.fdata -o %t.instr %t.exe
+
+## Run the profiled binary and check that the profile reports at least that `f`
+## has been called.
+# RUN: rm -f %t.fdata
+# RUN: %t.instr
+# RUN: cat %t.fdata | FileCheck %s
+# CHECK: f 0 0 1{{$}}
+
+## Check BOLT works with this profile
+# RUN: llvm-bolt --data %t.fdata --reorder-blocks=cache -o %t.bolt %t.exe
+
+    .text
+    .globl main
+    .type main, @function
+main:
+    addi sp, sp, -8
+    sd ra, 0(sp)
+    call f
+    ld ra, 0(sp)
+    addi sp, sp, 8
+    li a0, 0
+    ret
+    .size main, .-main
+
+    .globl f
+    .type f, @function
+f:
+    ret
+    .size f, .-f
diff --git a/bolt/test/runtime/RISCV/instrumentation-ind-call.c b/bolt/test/runtime/RISCV/instrumentation-ind-call.c
new file mode 100644
index 0000000000000..1fd49a774c942
--- /dev/null
+++ b/bolt/test/runtime/RISCV/instrumentation-ind-call.c
@@ -0,0 +1,39 @@
+/*
+REQUIRES: system-linux,bolt-runtime
+
+RUN: %clang %cflags %s -o %t.exe -Wl,-q 
+
+RUN: llvm-bolt %t.exe --instrument --instrumentation-file=%t.fdata \
+RUN:   -o %t.instrumented
+
+# Instrumented program needs to finish returning zero
+RUN: %t.instrumented | FileCheck %s -check-prefix=CHECK-OUTPUT
+
+# Test that the instrumented data makes sense
+RUN:  llvm-bolt %t.exe -o %t.bolted --data %t.fdata \
+RUN:    --reorder-blocks=ext-tsp --reorder-functions=hfsort+ \
+RUN:    --print-only=main --print-finalized | FileCheck %s
+
+RUN: %t.bolted | FileCheck %s -check-prefix=CHECK-OUTPUT
+
+CHECK-OUTPUT: The sum is: 30
+
+# Check that our indirect call has 1 hit recorded in the fdata file and that
+# this was processed correctly by BOLT
+CHECK:         jalr    a2 # CallProfile: 1 (0 misses) :
+CHECK-NEXT:    { add: 1 (0 misses) }
+*/
+
+#include <stdio.h>
+
+typedef int (*func_ptr)(int, int);
+
+int add(int a, int b) { return a + b; }
+
+int main() {
+  func_ptr fun;
+  fun = add;
+  int sum = fun(10, 20); // indirect call to 'add'
+  printf("The sum is: %d\n", sum);
+  return 0;
+}
diff --git a/bolt/test/runtime/RISCV/lit.local.cfg b/bolt/test/runtime/RISCV/lit.local.cfg
new file mode 100644
index 0000000000000..c0627d905ab3b
--- /dev/null
+++ b/bolt/test/runtime/RISCV/lit.local.cfg
@@ -0,0 +1,2 @@
+if config.host_arch not in ["riscv64"]:
+    config.unsupported = True

From 377ec36b323ea99ca316cb5cf79c0a0c93eebc37 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Thu, 17 Apr 2025 08:09:55 +0200
Subject: [PATCH 224/710] [Clang] Bypass TAD during overload resolution if a
 perfect match exists (#136018)

This implements the same overload resolution behavior as GCC,
as described in https://wg21.link/p3606 (section 1-2, not 3)

If during overload resolution, there is a non-template candidate
that would be always be picked - because each of the argument
is a perfect match (ie the source and target types are the same),
we do not perform deduction for any template candidate
that might exists.

The goal is to be able to merge
https://github.com/llvm/llvm-project/pull/122423 without being too
disruptive.

This change means that the selection of the best viable candidate and
template argument deduction become interleaved.

To avoid rewriting half of Clang we store in `OverloadCandidateSet`
enough information to be able to deduce template candidates from
`OverloadCandidateSet::BestViableFunction`. Which means
the lifetime of any object used by template argument must outlive
a call to `Add*Template*Candidate`.

This two phase resolution is not performed for some initialization
as there are cases where template candidate are better match
in these cases per the standard. It's also bypassed for code completion.

The change has a nice impact on compile times

https://llvm-compile-time-tracker.com/compare.php?from=719b029c16eeb1035da522fd641dfcc4cee6be74&to=bf7041045c9408490c395230047c5461de72fc39&stat=instructions%3Au

Fixes https://github.com/llvm/llvm-project/issues/62096
Fixes https://github.com/llvm/llvm-project/issues/74581

Reapplies #133426
---
 clang/docs/ReleaseNotes.rst                   |   6 +
 clang/include/clang/Sema/Overload.h           | 232 +++++++-
 clang/lib/Sema/SemaCodeComplete.cpp           |   6 +-
 clang/lib/Sema/SemaInit.cpp                   |  18 +-
 clang/lib/Sema/SemaOverload.cpp               | 532 ++++++++++++++----
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   4 +-
 .../constrant-satisfaction-conversions.cpp    |   8 +-
 clang/test/SemaCUDA/function-overload.cu      |   3 -
 .../SemaCXX/implicit-member-functions.cpp     |  21 +-
 ...overload-resolution-deferred-templates.cpp | 200 +++++++
 .../instantiate-function-params.cpp           |   7 +-
 .../Templight/templight-empty-entries-fix.cpp | 126 ++---
 12 files changed, 944 insertions(+), 219 deletions(-)
 create mode 100644 clang/test/SemaCXX/overload-resolution-deferred-templates.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0bf5e07ec5dfd..2bc7054e3c794 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -96,6 +96,12 @@ C++ Language Changes
       asm((std::string_view("nop")) ::: (std::string_view("memory")));
     }
 
+- Clang now implements the changes to overload resolution proposed by section 1 and 2 of
+  `P3606 <https://wg21.link/P3606R0>`_. If a non-template candidate exists in an overload set that is
+  a perfect match (all conversion sequences are identity conversions) template candidates are not instantiated.
+  Diagnostics that would have resulted from the instantiation of these template candidates are no longer
+  produced. This aligns Clang closer to the behavior of GCC, and fixes (#GH62096), (#GH74581), and (#GH74581).
+
 C++2c Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 6e08762dcc6d7..e667147bfac7e 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -407,6 +407,26 @@ class Sema;
              Third == ICK_Identity;
     }
 
+    /// A conversion sequence is perfect if it is an identity conversion and
+    /// the type of the source is the same as the type of the target.
+    bool isPerfect(const ASTContext &C) const {
+      if (!isIdentityConversion())
+        return false;
+      // If we are not performing a reference binding, we can skip comparing
+      // the types, which has a noticeable performance impact.
+      if (!ReferenceBinding) {
+        // The types might differ if there is an array-to-pointer conversion
+        // or lvalue-to-rvalue conversion.
+        assert(First || C.hasSameUnqualifiedType(getFromType(), getToType(2)));
+        return true;
+      }
+      if (!C.hasSameType(getFromType(), getToType(2)))
+        return false;
+      if (BindsToRvalue && IsLvalueReference)
+        return false;
+      return true;
+    }
+
     ImplicitConversionRank getRank() const;
     NarrowingKind
     getNarrowingKind(ASTContext &Context, const Expr *Converted,
@@ -743,6 +763,12 @@ class Sema;
       Standard.setAllToTypes(T);
     }
 
+    /// A conversion sequence is perfect if it is an identity conversion and
+    /// the type of the source is the same as the type of the target.
+    bool isPerfect(const ASTContext &C) const {
+      return isStandard() && Standard.isPerfect(C);
+    }
+
     // True iff this is a conversion sequence from an initializer list to an
     // array or std::initializer.
     bool hasInitializerListContainerType() const {
@@ -939,6 +965,10 @@ class Sema;
     LLVM_PREFERRED_TYPE(CallExpr::ADLCallKind)
     unsigned IsADLCandidate : 1;
 
+    /// Whether FinalConversion has been set.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasFinalConversion : 1;
+
     /// Whether this is a rewritten candidate, and if so, of what kind?
     LLVM_PREFERRED_TYPE(OverloadCandidateRewriteKind)
     unsigned RewriteKind : 2;
@@ -979,6 +1009,20 @@ class Sema;
       return false;
     }
 
+    // An overload is a perfect match if the conversion
+    // sequences for each argument are perfect.
+    bool isPerfectMatch(const ASTContext &Ctx) const {
+      if (!Viable)
+        return false;
+      for (const auto &C : Conversions) {
+        if (!C.isInitialized() || !C.isPerfect(Ctx))
+          return false;
+      }
+      if (HasFinalConversion)
+        return FinalConversion.isPerfect(Ctx);
+      return true;
+    }
+
     bool TryToFixBadConversion(unsigned Idx, Sema &S) {
       bool CanFix = Fix.tryToFixConversion(
                       Conversions[Idx].Bad.FromExpr,
@@ -1012,8 +1056,67 @@ class Sema;
         : IsSurrogate(false), IgnoreObjectArgument(false),
           TookAddressOfOverload(false), StrictPackMatch(false),
           IsADLCandidate(llvm::to_underlying(CallExpr::NotADL)),
-          RewriteKind(CRK_None) {}
+          HasFinalConversion(false), RewriteKind(CRK_None) {}
+  };
+
+  struct DeferredTemplateOverloadCandidate {
+
+    // intrusive linked list support for allocateDeferredCandidate
+    DeferredTemplateOverloadCandidate *Next = nullptr;
+
+    enum Kind { Function, Method, Conversion };
+
+    LLVM_PREFERRED_TYPE(Kind)
+    unsigned Kind : 2;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowObjCConversionOnExplicit : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowResultConversion : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowExplicit : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned SuppressUserConversions : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned PartialOverloading : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AggregateCandidateDeduction : 1;
+  };
+
+  struct DeferredFunctionTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    ArrayRef<Expr *> Args;
+    CallExpr::ADLCallKind IsADLCandidate;
+    OverloadCandidateParamOrder PO;
+  };
+  static_assert(std::is_trivially_destructible_v<
+                DeferredFunctionTemplateOverloadCandidate>);
+
+  struct DeferredMethodTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    ArrayRef<Expr *> Args;
+    CXXRecordDecl *ActingContext;
+    Expr::Classification ObjectClassification;
+    QualType ObjectType;
+    OverloadCandidateParamOrder PO;
   };
+  static_assert(std::is_trivially_destructible_v<
+                DeferredMethodTemplateOverloadCandidate>);
+
+  struct DeferredConversionTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    CXXRecordDecl *ActingContext;
+    Expr *From;
+    QualType ToType;
+  };
+
+  static_assert(std::is_trivially_destructible_v<
+                DeferredConversionTemplateOverloadCandidate>);
 
   /// OverloadCandidateSet - A set of overload candidates, used in C++
   /// overload resolution (C++ 13.3).
@@ -1043,6 +1146,11 @@ class Sema;
       /// C++ [over.match.call.general]
       /// Resolve a call through the address of an overload set.
       CSK_AddressOfOverloadSet,
+
+      /// When doing overload resolution during code completion,
+      /// we want to show all viable candidates, including otherwise
+      /// deferred template candidates.
+      CSK_CodeCompletion,
     };
 
     /// Information about operator rewrites to consider when adding operator
@@ -1117,7 +1225,15 @@ class Sema;
     SmallVector<OverloadCandidate, 16> Candidates;
     llvm::SmallPtrSet<uintptr_t, 16> Functions;
 
-    // Allocator for ConversionSequenceLists. We store the first few of these
+    DeferredTemplateOverloadCandidate *FirstDeferredCandidate = nullptr;
+    unsigned DeferredCandidatesCount : 8 * sizeof(unsigned) - 2;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasDeferredTemplateConstructors : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned ResolutionByPerfectCandidateIsDisabled : 1;
+
+    // Allocator for ConversionSequenceLists and deferred candidate args.
+    // We store the first few of these
     // inline to avoid allocation for small sets.
     llvm::BumpPtrAllocator SlabAllocator;
 
@@ -1125,8 +1241,11 @@ class Sema;
     CandidateSetKind Kind;
     OperatorRewriteInfo RewriteInfo;
 
+    /// Small storage size for ImplicitConversionSequences
+    /// and the persisted arguments of deferred candidates.
     constexpr static unsigned NumInlineBytes =
-        24 * sizeof(ImplicitConversionSequence);
+        32 * sizeof(ImplicitConversionSequence);
+
     unsigned NumInlineBytesUsed = 0;
     alignas(void *) char InlineSpace[NumInlineBytes];
 
@@ -1137,15 +1256,13 @@ class Sema;
     /// from the slab allocator.
     /// FIXME: It would probably be nice to have a SmallBumpPtrAllocator
     /// instead.
-    /// FIXME: Now that this only allocates ImplicitConversionSequences, do we
-    /// want to un-generalize this?
     template <typename T>
     T *slabAllocate(unsigned N) {
       // It's simpler if this doesn't need to consider alignment.
       static_assert(alignof(T) == alignof(void *),
                     "Only works for pointer-aligned types.");
-      static_assert(std::is_trivial<T>::value ||
-                        std::is_same<ImplicitConversionSequence, T>::value,
+      static_assert(std::is_trivially_destructible_v<T> ||
+                        (std::is_same_v<ImplicitConversionSequence, T>),
                     "Add destruction logic to OverloadCandidateSet::clear().");
 
       unsigned NBytes = sizeof(T) * N;
@@ -1159,12 +1276,34 @@ class Sema;
       return reinterpret_cast<T *>(FreeSpaceStart);
     }
 
+    // Because the size of OverloadCandidateSet has a noticeable impact on
+    // performance, we store each deferred template candidate in the slab
+    // allocator such that deferred candidates are ultimately a singly-linked
+    // intrusive linked list. This ends up being much more efficient than a
+    // SmallVector that is empty in the common case.
+    template <typename T> T *allocateDeferredCandidate() {
+      T *C = slabAllocate<T>(1);
+      if (!FirstDeferredCandidate)
+        FirstDeferredCandidate = C;
+      else {
+        auto *F = FirstDeferredCandidate;
+        while (F->Next)
+          F = F->Next;
+        F->Next = C;
+      }
+      DeferredCandidatesCount++;
+      return C;
+    }
+
     void destroyCandidates();
 
   public:
     OverloadCandidateSet(SourceLocation Loc, CandidateSetKind CSK,
                          OperatorRewriteInfo RewriteInfo = {})
-        : Loc(Loc), Kind(CSK), RewriteInfo(RewriteInfo) {}
+        : FirstDeferredCandidate(nullptr), DeferredCandidatesCount(0),
+          HasDeferredTemplateConstructors(false),
+          ResolutionByPerfectCandidateIsDisabled(false), Loc(Loc), Kind(CSK),
+          RewriteInfo(RewriteInfo) {}
     OverloadCandidateSet(const OverloadCandidateSet &) = delete;
     OverloadCandidateSet &operator=(const OverloadCandidateSet &) = delete;
     ~OverloadCandidateSet() { destroyCandidates(); }
@@ -1176,6 +1315,9 @@ class Sema;
     /// Whether diagnostics should be deferred.
     bool shouldDeferDiags(Sema &S, ArrayRef<Expr *> Args, SourceLocation OpLoc);
 
+    // Whether the resolution of template candidates should be deferred
+    bool shouldDeferTemplateArgumentDeduction(const LangOptions &Opts) const;
+
     /// Determine when this overload candidate will be new to the
     /// overload set.
     bool isNewCandidate(Decl *F, OverloadCandidateParamOrder PO =
@@ -1199,8 +1341,10 @@ class Sema;
     iterator begin() { return Candidates.begin(); }
     iterator end() { return Candidates.end(); }
 
-    size_t size() const { return Candidates.size(); }
-    bool empty() const { return Candidates.empty(); }
+    size_t size() const { return Candidates.size() + DeferredCandidatesCount; }
+    bool empty() const {
+      return Candidates.empty() && DeferredCandidatesCount == 0;
+    }
 
     /// Allocate storage for conversion sequences for NumConversions
     /// conversions.
@@ -1216,6 +1360,24 @@ class Sema;
       return ConversionSequenceList(Conversions, NumConversions);
     }
 
+    /// Provide storage for any Expr* arg that must be preserved
+    /// until deferred template candidates are deduced.
+    /// Typically this should be used for reversed operator arguments
+    /// and any time the argument array is transformed while adding
+    /// a template candidate.
+    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(unsigned N) {
+      Expr **Exprs = slabAllocate<Expr *>(N);
+      return llvm::MutableArrayRef<Expr *>(Exprs, N);
+    }
+
+    template <typename... T>
+    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(T *...Exprs) {
+      llvm::MutableArrayRef<Expr *> Arr =
+          getPersistentArgsArray(sizeof...(Exprs));
+      llvm::copy(std::initializer_list<Expr *>{Exprs...}, Arr.data());
+      return Arr;
+    }
+
     /// Add a new candidate with NumConversions conversion sequence slots
     /// to the overload set.
     OverloadCandidate &addCandidate(unsigned NumConversions = 0,
@@ -1231,6 +1393,32 @@ class Sema;
       return C;
     }
 
+    void AddDeferredTemplateCandidate(
+        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+        ArrayRef<Expr *> Args, bool SuppressUserConversions,
+        bool PartialOverloading, bool AllowExplicit,
+        CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+        bool AggregateCandidateDeduction);
+
+    void AddDeferredMethodTemplateCandidate(
+        FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+        CXXRecordDecl *ActingContext, QualType ObjectType,
+        Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+        bool SuppressUserConversions, bool PartialOverloading,
+        OverloadCandidateParamOrder PO);
+
+    void AddDeferredConversionTemplateCandidate(
+        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+        CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+        bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+        bool AllowResultConversion);
+
+    void InjectNonDeducedTemplateCandidates(Sema &S);
+
+    void DisableResolutionByPerfectCandidate() {
+      ResolutionByPerfectCandidateIsDisabled = true;
+    }
+
     /// Find the best viable function on this overload set, if it exists.
     OverloadingResult BestViableFunction(Sema &S, SourceLocation Loc,
                                          OverloadCandidateSet::iterator& Best);
@@ -1263,6 +1451,15 @@ class Sema;
       DestAS = AS;
     }
 
+  private:
+    OverloadingResult ResultForBestCandidate(const iterator &Best);
+    void CudaExcludeWrongSideCandidates(
+        Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates);
+    OverloadingResult
+    BestViableFunctionImpl(Sema &S, SourceLocation Loc,
+                           OverloadCandidateSet::iterator &Best);
+    void PerfectViableFunction(Sema &S, SourceLocation Loc,
+                               OverloadCandidateSet::iterator &Best);
   };
 
   bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
@@ -1311,6 +1508,21 @@ class Sema;
   // parameter.
   bool shouldEnforceArgLimit(bool PartialOverloading, FunctionDecl *Function);
 
+  inline bool OverloadCandidateSet::shouldDeferTemplateArgumentDeduction(
+      const LangOptions &Opts) const {
+    return
+        // For user defined conversion we need to check against different
+        // combination of CV qualifiers and look at any explicit specifier, so
+        // always deduce template candidates.
+        Kind != CSK_InitByUserDefinedConversion
+        // When doing code completion, we want to see all the
+        // viable candidates.
+        && Kind != CSK_CodeCompletion
+        // CUDA may prefer template candidates even when a non-candidate
+        // is a perfect match
+        && !Opts.CUDA;
+  }
+
 } // namespace clang
 
 #endif // LLVM_CLANG_SEMA_OVERLOAD_H
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index f6ec4cb0f069e..45405d4709e14 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -6354,7 +6354,8 @@ SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
   Expr *NakedFn = Fn->IgnoreParenCasts();
   // Build an overload candidate set based on the functions we find.
   SourceLocation Loc = Fn->getExprLoc();
-  OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
+  OverloadCandidateSet CandidateSet(Loc,
+                                    OverloadCandidateSet::CSK_CodeCompletion);
 
   if (auto ULE = dyn_cast<UnresolvedLookupExpr>(NakedFn)) {
     SemaRef.AddOverloadedCallCandidates(ULE, ArgsWithoutDependentTypes,
@@ -6557,7 +6558,8 @@ QualType SemaCodeCompletion::ProduceConstructorSignatureHelp(
   // FIXME: Provide support for variadic template constructors.
 
   if (CRD) {
-    OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
+    OverloadCandidateSet CandidateSet(Loc,
+                                      OverloadCandidateSet::CSK_CodeCompletion);
     for (NamedDecl *C : SemaRef.LookupConstructors(CRD)) {
       if (auto *FD = dyn_cast<FunctionDecl>(C)) {
         // FIXME: we can't yet provide correct signature help for initializer
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 77d7f821f2011..9e802a175bb05 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -5231,7 +5231,7 @@ static OverloadingResult TryRefInitWithConversionFunction(
 
   // Add the final conversion sequence, if necessary.
   if (NewRefRelationship == Sema::Ref_Incompatible) {
-    assert(!isa<CXXConstructorDecl>(Function) &&
+    assert(Best->HasFinalConversion && !isa<CXXConstructorDecl>(Function) &&
            "should not have conversion after constructor");
 
     ImplicitConversionSequence ICS;
@@ -6200,6 +6200,7 @@ static void TryUserDefinedConversion(Sema &S,
 
   // If the conversion following the call to the conversion function
   // is interesting, add it as a separate step.
+  assert(Best->HasFinalConversion);
   if (Best->FinalConversion.First || Best->FinalConversion.Second ||
       Best->FinalConversion.Third) {
     ImplicitConversionSequence ICS;
@@ -10029,12 +10030,19 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     //   When [...] the constructor [...] is a candidate by
     //    - [over.match.copy] (in all cases)
     if (TD) {
-      SmallVector<Expr *, 8> TmpInits;
-      for (Expr *E : Inits)
+
+      // As template candidates are not deduced immediately,
+      // persist the array in the overload set.
+      MutableArrayRef<Expr *> TmpInits =
+          Candidates.getPersistentArgsArray(Inits.size());
+
+      for (auto [I, E] : llvm::enumerate(Inits)) {
         if (auto *DI = dyn_cast<DesignatedInitExpr>(E))
-          TmpInits.push_back(DI->getInit());
+          TmpInits[I] = DI->getInit();
         else
-          TmpInits.push_back(E);
+          TmpInits[I] = E;
+      }
+
       AddTemplateOverloadCandidate(
           TD, FoundDecl, /*ExplicitArgs=*/nullptr, TmpInits, Candidates,
           /*SuppressUserConversions=*/false,
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 55634aa75ae25..b7a981e08ead9 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1123,6 +1123,10 @@ void OverloadCandidateSet::clear(CandidateSetKind CSK) {
   Candidates.clear();
   Functions.clear();
   Kind = CSK;
+  FirstDeferredCandidate = nullptr;
+  DeferredCandidatesCount = 0;
+  HasDeferredTemplateConstructors = false;
+  ResolutionByPerfectCandidateIsDisabled = false;
 }
 
 namespace {
@@ -4075,6 +4079,9 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
     }
     if (CXXConversionDecl *Conversion
                  = dyn_cast<CXXConversionDecl>(Best->Function)) {
+
+      assert(Best->HasFinalConversion);
+
       // C++ [over.ics.user]p1:
       //
       //   [...] If the user-defined conversion is specified by a
@@ -5158,6 +5165,9 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
   case OR_Success:
+
+    assert(Best->HasFinalConversion);
+
     // C++ [over.ics.ref]p1:
     //
     //   [...] If the parameter binds directly to the result of
@@ -7795,15 +7805,14 @@ void Sema::AddMethodCandidate(
   }
 }
 
-void Sema::AddMethodTemplateCandidate(
+static void AddMethodTemplateCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
     CXXRecordDecl *ActingContext,
     TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
     Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, OverloadCandidateParamOrder PO) {
-  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
-    return;
+    bool SuppressUserConversions, bool PartialOverloading,
+    OverloadCandidateParamOrder PO) {
 
   // C++ [over.match.funcs]p7:
   //   In each case where a candidate is a function template, candidate
@@ -7817,12 +7826,12 @@ void Sema::AddMethodTemplateCandidate(
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           MethodTmpl, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, /*AggregateDeductionCandidate=*/false,
           /*PartialOrdering=*/false, ObjectType, ObjectClassification,
           [&](ArrayRef<QualType> ParamTypes) {
-            return CheckNonDependentConversions(
+            return S.CheckNonDependentConversions(
                 MethodTmpl, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, ActingContext, ObjectType,
                 ObjectClassification, PO);
@@ -7844,8 +7853,8 @@ void Sema::AddMethodTemplateCandidate(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                            Info);
+      Candidate.DeductionFailure =
+          MakeDeductionFailureInfo(S.Context, Result, Info);
     }
     return;
   }
@@ -7855,10 +7864,34 @@ void Sema::AddMethodTemplateCandidate(
   assert(Specialization && "Missing member function template specialization?");
   assert(isa<CXXMethodDecl>(Specialization) &&
          "Specialization is not a member function?");
-  AddMethodCandidate(cast<CXXMethodDecl>(Specialization), FoundDecl,
-                     ActingContext, ObjectType, ObjectClassification, Args,
-                     CandidateSet, SuppressUserConversions, PartialOverloading,
-                     Conversions, PO, Info.hasStrictPackMatch());
+  S.AddMethodCandidate(
+      cast<CXXMethodDecl>(Specialization), FoundDecl, ActingContext, ObjectType,
+      ObjectClassification, Args, CandidateSet, SuppressUserConversions,
+      PartialOverloading, Conversions, PO, Info.hasStrictPackMatch());
+}
+
+void Sema::AddMethodTemplateCandidate(
+    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext,
+    TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
+    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, OverloadCandidateParamOrder PO) {
+  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
+    return;
+
+  if (ExplicitTemplateArgs ||
+      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts())) {
+    AddMethodTemplateCandidateImmediately(
+        *this, CandidateSet, MethodTmpl, FoundDecl, ActingContext,
+        ExplicitTemplateArgs, ObjectType, ObjectClassification, Args,
+        SuppressUserConversions, PartialOverloading, PO);
+    return;
+  }
+
+  CandidateSet.AddDeferredMethodTemplateCandidate(
+      MethodTmpl, FoundDecl, ActingContext, ObjectType, ObjectClassification,
+      Args, SuppressUserConversions, PartialOverloading, PO);
 }
 
 /// Determine whether a given function template has a simple explicit specifier
@@ -7867,14 +7900,18 @@ static bool isNonDependentlyExplicit(FunctionTemplateDecl *FTD) {
   return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).isExplicit();
 }
 
-void Sema::AddTemplateOverloadCandidate(
+static bool hasDependentExplicit(FunctionTemplateDecl *FTD) {
+  return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).getKind() ==
+         ExplicitSpecKind::Unresolved;
+}
+
+static void AddTemplateOverloadCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
     TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
-    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
-  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
-    return;
+    bool SuppressUserConversions, bool PartialOverloading, bool AllowExplicit,
+    Sema::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+    bool AggregateCandidateDeduction) {
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -7901,14 +7938,14 @@ void Sema::AddTemplateOverloadCandidate(
                              FunctionTemplate->getTemplateDepth());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           FunctionTemplate, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, AggregateCandidateDeduction,
           /*PartialOrdering=*/false,
           /*ObjectType=*/QualType(),
           /*ObjectClassification=*/Expr::Classification(),
           [&](ArrayRef<QualType> ParamTypes) {
-            return CheckNonDependentConversions(
+            return S.CheckNonDependentConversions(
                 FunctionTemplate, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, nullptr, QualType(), {}, PO);
           });
@@ -7932,8 +7969,8 @@ void Sema::AddTemplateOverloadCandidate(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                            Info);
+      Candidate.DeductionFailure =
+          MakeDeductionFailureInfo(S.Context, Result, Info);
     }
     return;
   }
@@ -7941,7 +7978,7 @@ void Sema::AddTemplateOverloadCandidate(
   // Add the function template specialization produced by template argument
   // deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  AddOverloadCandidate(
+  S.AddOverloadCandidate(
       Specialization, FoundDecl, Args, CandidateSet, SuppressUserConversions,
       PartialOverloading, AllowExplicit,
       /*AllowExplicitConversions=*/false, IsADLCandidate, Conversions, PO,
@@ -7949,6 +7986,38 @@ void Sema::AddTemplateOverloadCandidate(
       Info.hasStrictPackMatch());
 }
 
+void Sema::AddTemplateOverloadCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
+    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
+  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
+    return;
+
+  bool DependentExplicitSpecifier = hasDependentExplicit(FunctionTemplate);
+
+  if (ExplicitTemplateArgs ||
+      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
+      (isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl()) &&
+       DependentExplicitSpecifier)) {
+
+    AddTemplateOverloadCandidateImmediately(
+        *this, CandidateSet, FunctionTemplate, FoundDecl, ExplicitTemplateArgs,
+        Args, SuppressUserConversions, PartialOverloading, AllowExplicit,
+        IsADLCandidate, PO, AggregateCandidateDeduction);
+
+    if (DependentExplicitSpecifier)
+      CandidateSet.DisableResolutionByPerfectCandidate();
+    return;
+  }
+
+  CandidateSet.AddDeferredTemplateCandidate(
+      FunctionTemplate, FoundDecl, Args, SuppressUserConversions,
+      PartialOverloading, AllowExplicit, IsADLCandidate, PO,
+      AggregateCandidateDeduction);
+}
+
 bool Sema::CheckNonDependentConversions(
     FunctionTemplateDecl *FunctionTemplate, ArrayRef<QualType> ParamTypes,
     ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet,
@@ -8111,6 +8180,7 @@ void Sema::AddConversionCandidate(
   Candidate.FinalConversion.setAsIdentityConversion();
   Candidate.FinalConversion.setFromType(ConvType);
   Candidate.FinalConversion.setAllToTypes(ToType);
+  Candidate.HasFinalConversion = true;
   Candidate.Viable = true;
   Candidate.ExplicitCallArguments = 1;
   Candidate.StrictPackMatch = StrictPackMatch;
@@ -8215,6 +8285,7 @@ void Sema::AddConversionCandidate(
   switch (ICS.getKind()) {
   case ImplicitConversionSequence::StandardConversion:
     Candidate.FinalConversion = ICS.Standard;
+    Candidate.HasFinalConversion = true;
 
     // C++ [over.ics.user]p3:
     //   If the user-defined conversion is specified by a specialization of a
@@ -8264,16 +8335,12 @@ void Sema::AddConversionCandidate(
   }
 }
 
-void Sema::AddTemplateConversionCandidate(
+static void AddTemplateConversionCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
-    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
-    bool AllowExplicit, bool AllowResultConversion) {
-  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
-         "Only conversion function templates permitted here");
-
-  if (!CandidateSet.isNewCandidate(FunctionTemplate))
-    return;
+    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+    bool AllowResultConversion) {
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -8288,11 +8355,11 @@ void Sema::AddTemplateConversionCandidate(
   }
 
   QualType ObjectType = From->getType();
-  Expr::Classification ObjectClassification = From->Classify(getASTContext());
+  Expr::Classification ObjectClassification = From->Classify(S.Context);
 
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   CXXConversionDecl *Specialization = nullptr;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           FunctionTemplate, ObjectType, ObjectClassification, ToType,
           Specialization, Info);
       Result != TemplateDeductionResult::Success) {
@@ -8302,18 +8369,47 @@ void Sema::AddTemplateConversionCandidate(
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_deduction;
     Candidate.ExplicitCallArguments = 1;
-    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                          Info);
+    Candidate.DeductionFailure =
+        MakeDeductionFailureInfo(S.Context, Result, Info);
     return;
   }
 
   // Add the conversion function template specialization produced by
   // template argument deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  AddConversionCandidate(Specialization, FoundDecl, ActingDC, From, ToType,
-                         CandidateSet, AllowObjCConversionOnExplicit,
-                         AllowExplicit, AllowResultConversion,
-                         Info.hasStrictPackMatch());
+  S.AddConversionCandidate(Specialization, FoundDecl, ActingContext, From,
+                           ToType, CandidateSet, AllowObjCConversionOnExplicit,
+                           AllowExplicit, AllowResultConversion,
+                           Info.hasStrictPackMatch());
+}
+
+void Sema::AddTemplateConversionCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
+    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
+    bool AllowExplicit, bool AllowResultConversion) {
+  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
+         "Only conversion function templates permitted here");
+
+  if (!CandidateSet.isNewCandidate(FunctionTemplate))
+    return;
+
+  if (!CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
+      CandidateSet.getKind() ==
+          OverloadCandidateSet::CSK_InitByUserDefinedConversion ||
+      CandidateSet.getKind() == OverloadCandidateSet::CSK_InitByConstructor) {
+    AddTemplateConversionCandidateImmediately(
+        *this, CandidateSet, FunctionTemplate, FoundDecl, ActingDC, From,
+        ToType, AllowObjCConversionOnExplicit, AllowExplicit,
+        AllowResultConversion);
+
+    CandidateSet.DisableResolutionByPerfectCandidate();
+    return;
+  }
+
+  CandidateSet.AddDeferredConversionTemplateCandidate(
+      FunctionTemplate, FoundDecl, ActingDC, From, ToType,
+      AllowObjCConversionOnExplicit, AllowExplicit, AllowResultConversion);
 }
 
 void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
@@ -8463,11 +8559,17 @@ void Sema::AddNonMemberOperatorCandidates(
     if (FunTmpl) {
       AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
                                    FunctionArgs, CandidateSet);
-      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD))
-        AddTemplateOverloadCandidate(
-            FunTmpl, F.getPair(), ExplicitTemplateArgs,
-            {FunctionArgs[1], FunctionArgs[0]}, CandidateSet, false, false,
-            true, ADLCallKind::NotADL, OverloadCandidateParamOrder::Reversed);
+      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD)) {
+
+        // As template candidates are not deduced immediately,
+        // persist the array in the overload set.
+        ArrayRef<Expr *> Reversed = CandidateSet.getPersistentArgsArray(
+            FunctionArgs[1], FunctionArgs[0]);
+        AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
+                                     Reversed, CandidateSet, false, false, true,
+                                     ADLCallKind::NotADL,
+                                     OverloadCandidateParamOrder::Reversed);
+      }
     } else {
       if (ExplicitTemplateArgs)
         continue;
@@ -10199,6 +10301,8 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
   // FIXME: Pass in the explicit template arguments?
   ArgumentDependentLookup(Name, Loc, Args, Fns);
 
+  ArrayRef<Expr *> ReversedArgs;
+
   // Erase all of the candidates we already knew about.
   for (OverloadCandidateSet::iterator Cand = CandidateSet.begin(),
                                    CandEnd = CandidateSet.end();
@@ -10238,9 +10342,15 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
           /*AllowExplicit=*/true, ADLCallKind::UsesADL);
       if (CandidateSet.getRewriteInfo().shouldAddReversed(
               *this, Args, FTD->getTemplatedDecl())) {
+
+        // As template candidates are not deduced immediately,
+        // persist the array in the overload set.
+        if (ReversedArgs.empty())
+          ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
+
         AddTemplateOverloadCandidate(
-            FTD, FoundDecl, ExplicitTemplateArgs, {Args[1], Args[0]},
-            CandidateSet, /*SuppressUserConversions=*/false, PartialOverloading,
+            FTD, FoundDecl, ExplicitTemplateArgs, ReversedArgs, CandidateSet,
+            /*SuppressUserConversions=*/false, PartialOverloading,
             /*AllowExplicit=*/true, ADLCallKind::UsesADL,
             OverloadCandidateParamOrder::Reversed);
       }
@@ -10654,6 +10764,8 @@ bool clang::isBetterOverloadCandidate(
       Cand1.Function && Cand2.Function &&
       isa<CXXConversionDecl>(Cand1.Function) &&
       isa<CXXConversionDecl>(Cand2.Function)) {
+
+    assert(Cand1.HasFinalConversion && Cand2.HasFinalConversion);
     // First check whether we prefer one of the conversion functions over the
     // other. This only distinguishes the results in non-standard, extension
     // cases such as the conversion from a lambda closure type to a function
@@ -10913,23 +11025,147 @@ bool OverloadCandidate::NotValidBecauseConstraintExprHasError() const {
              ->Satisfaction.ContainsErrors;
 }
 
-/// Computes the best viable function (C++ 13.3.3)
-/// within an overload candidate set.
-///
-/// \param Loc The location of the function name (or operator symbol) for
-/// which overload resolution occurs.
-///
-/// \param Best If overload resolution was successful or found a deleted
-/// function, \p Best points to the candidate function found.
-///
-/// \returns The result of overload resolution.
+void OverloadCandidateSet::AddDeferredTemplateCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    ArrayRef<Expr *> Args, bool SuppressUserConversions,
+    bool PartialOverloading, bool AllowExplicit,
+    CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+    bool AggregateCandidateDeduction) {
+
+  auto *C =
+      allocateDeferredCandidate<DeferredFunctionTemplateOverloadCandidate>();
+
+  C = new (C) DeferredFunctionTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Function,
+       /*AllowObjCConversionOnExplicit=*/false,
+       /*AllowResultConversion=*/false, AllowExplicit, SuppressUserConversions,
+       PartialOverloading, AggregateCandidateDeduction},
+      FunctionTemplate,
+      FoundDecl,
+      Args,
+      IsADLCandidate,
+      PO};
+
+  HasDeferredTemplateConstructors |=
+      isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl());
+}
+
+void OverloadCandidateSet::AddDeferredMethodTemplateCandidate(
+    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext, QualType ObjectType,
+    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+    bool SuppressUserConversions, bool PartialOverloading,
+    OverloadCandidateParamOrder PO) {
+
+  assert(!isa<CXXConstructorDecl>(MethodTmpl->getTemplatedDecl()));
+
+  auto *C =
+      allocateDeferredCandidate<DeferredMethodTemplateOverloadCandidate>();
+
+  C = new (C) DeferredMethodTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Method,
+       /*AllowObjCConversionOnExplicit=*/false,
+       /*AllowResultConversion=*/false,
+       /*AllowExplicit=*/false, SuppressUserConversions, PartialOverloading,
+       /*AggregateCandidateDeduction=*/false},
+      MethodTmpl,
+      FoundDecl,
+      Args,
+      ActingContext,
+      ObjectClassification,
+      ObjectType,
+      PO};
+}
+
+void OverloadCandidateSet::AddDeferredConversionTemplateCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+    bool AllowResultConversion) {
+
+  auto *C =
+      allocateDeferredCandidate<DeferredConversionTemplateOverloadCandidate>();
+
+  C = new (C) DeferredConversionTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Conversion,
+       AllowObjCConversionOnExplicit, AllowResultConversion,
+       /*AllowExplicit=*/false,
+       /*SuppressUserConversions=*/false,
+       /*PartialOverloading*/ false,
+       /*AggregateCandidateDeduction=*/false},
+      FunctionTemplate,
+      FoundDecl,
+      ActingContext,
+      From,
+      ToType};
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredMethodTemplateOverloadCandidate &C) {
+
+  AddMethodTemplateCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext,
+      /*ExplicitTemplateArgs=*/nullptr, C.ObjectType, C.ObjectClassification,
+      C.Args, C.SuppressUserConversions, C.PartialOverloading, C.PO);
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredFunctionTemplateOverloadCandidate &C) {
+  AddTemplateOverloadCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl,
+      /*ExplicitTemplateArgs=*/nullptr, C.Args, C.SuppressUserConversions,
+      C.PartialOverloading, C.AllowExplicit, C.IsADLCandidate, C.PO,
+      C.AggregateCandidateDeduction);
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredConversionTemplateOverloadCandidate &C) {
+  return AddTemplateConversionCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext, C.From,
+      C.ToType, C.AllowObjCConversionOnExplicit, C.AllowExplicit,
+      C.AllowResultConversion);
+}
+
+void OverloadCandidateSet::InjectNonDeducedTemplateCandidates(Sema &S) {
+  Candidates.reserve(Candidates.size() + DeferredCandidatesCount);
+  DeferredTemplateOverloadCandidate *Cand = FirstDeferredCandidate;
+  while (Cand) {
+    switch (Cand->Kind) {
+    case DeferredTemplateOverloadCandidate::Function:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredFunctionTemplateOverloadCandidate *>(Cand));
+      break;
+    case DeferredTemplateOverloadCandidate::Method:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredMethodTemplateOverloadCandidate *>(Cand));
+      break;
+    case DeferredTemplateOverloadCandidate::Conversion:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredConversionTemplateOverloadCandidate *>(Cand));
+      break;
+    }
+    Cand = Cand->Next;
+  }
+  FirstDeferredCandidate = nullptr;
+  DeferredCandidatesCount = 0;
+}
+
 OverloadingResult
-OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
-                                         iterator &Best) {
-  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
-  std::transform(begin(), end(), std::back_inserter(Candidates),
-                 [](OverloadCandidate &Cand) { return &Cand; });
+OverloadCandidateSet::ResultForBestCandidate(const iterator &Best) {
+  Best->Best = true;
+  if (Best->Function && Best->Function->isDeleted())
+    return OR_Deleted;
+  return OR_Success;
+}
 
+void OverloadCandidateSet::CudaExcludeWrongSideCandidates(
+    Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates) {
   // [CUDA] HD->H or HD->D calls are technically not allowed by CUDA but
   // are accepted by both clang and NVCC. However, during a particular
   // compilation mode only one call variant is viable. We need to
@@ -10941,27 +11177,112 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   // -fgpu-exclude-wrong-side-overloads is off. When
   // -fgpu-exclude-wrong-side-overloads is on, all candidates are compared
   // uniformly in isBetterOverloadCandidate.
-  if (S.getLangOpts().CUDA && !S.getLangOpts().GPUExcludeWrongSideOverloads) {
-    const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
-    bool ContainsSameSideCandidate =
-        llvm::any_of(Candidates, [&](OverloadCandidate *Cand) {
-          // Check viable function only.
-          return Cand->Viable && Cand->Function &&
-                 S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-                     SemaCUDA::CFP_SameSide;
-        });
-    if (ContainsSameSideCandidate) {
-      auto IsWrongSideCandidate = [&](OverloadCandidate *Cand) {
-        // Check viable function only to avoid unnecessary data copying/moving.
+  if (!S.getLangOpts().CUDA || S.getLangOpts().GPUExcludeWrongSideOverloads)
+    return;
+  const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
+
+  bool ContainsSameSideCandidate =
+      llvm::any_of(Candidates, [&](const OverloadCandidate *Cand) {
+        // Check viable function only.
         return Cand->Viable && Cand->Function &&
                S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-                   SemaCUDA::CFP_WrongSide;
-      };
-      llvm::erase_if(Candidates, IsWrongSideCandidate);
+                   SemaCUDA::CFP_SameSide;
+      });
+
+  if (!ContainsSameSideCandidate)
+    return;
+
+  auto IsWrongSideCandidate = [&](const OverloadCandidate *Cand) {
+    // Check viable function only to avoid unnecessary data copying/moving.
+    return Cand->Viable && Cand->Function &&
+           S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
+               SemaCUDA::CFP_WrongSide;
+  };
+  llvm::erase_if(Candidates, IsWrongSideCandidate);
+}
+
+/// Computes the best viable function (C++ 13.3.3)
+/// within an overload candidate set.
+///
+/// \param Loc The location of the function name (or operator symbol) for
+/// which overload resolution occurs.
+///
+/// \param Best If overload resolution was successful or found a deleted
+/// function, \p Best points to the candidate function found.
+///
+/// \returns The result of overload resolution.
+OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
+                                                           SourceLocation Loc,
+                                                           iterator &Best) {
+
+  assert(shouldDeferTemplateArgumentDeduction(S.getLangOpts()) ||
+         DeferredCandidatesCount == 0 &&
+             "Unexpected deferred template candidates");
+
+  bool TwoPhaseResolution =
+      DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
+
+  if (TwoPhaseResolution) {
+
+    PerfectViableFunction(S, Loc, Best);
+    if (Best != end())
+      return ResultForBestCandidate(Best);
+  }
+
+  InjectNonDeducedTemplateCandidates(S);
+  return BestViableFunctionImpl(S, Loc, Best);
+}
+
+void OverloadCandidateSet::PerfectViableFunction(
+    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
+
+  Best = end();
+  for (auto It = begin(); It != end(); ++It) {
+
+    if (!It->isPerfectMatch(S.getASTContext()))
+      continue;
+
+    // We found a suitable conversion function
+    // but if there is a template constructor in the target class
+    // we might prefer that instead.
+    if (HasDeferredTemplateConstructors &&
+        isa_and_nonnull<CXXConversionDecl>(It->Function)) {
+      Best = end();
+      break;
     }
+
+    if (Best == end()) {
+      Best = It;
+      continue;
+    }
+    if (Best->Function && It->Function) {
+      FunctionDecl *D =
+          S.getMoreConstrainedFunction(Best->Function, It->Function);
+      if (D == nullptr) {
+        Best = end();
+        break;
+      }
+      if (D == It->Function)
+        Best = It;
+      continue;
+    }
+    // ambiguous
+    Best = end();
+    break;
   }
+}
+
+OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
+    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
+
+  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
+  Candidates.reserve(this->Candidates.size());
+  std::transform(begin(), end(), std::back_inserter(Candidates),
+                 [](OverloadCandidate &Cand) { return &Cand; });
+
+  if (S.getLangOpts().CUDA)
+    CudaExcludeWrongSideCandidates(S, Candidates);
 
-  // Find the best viable function.
   Best = end();
   for (auto *Cand : Candidates) {
     Cand->Best = false;
@@ -10983,9 +11304,8 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   if (Best == end())
     return OR_No_Viable_Function;
 
+  llvm::SmallVector<OverloadCandidate *, 4> PendingBest;
   llvm::SmallVector<const NamedDecl *, 4> EquivalentCands;
-
-  llvm::SmallVector<OverloadCandidate*, 4> PendingBest;
   PendingBest.push_back(&*Best);
   Best->Best = true;
 
@@ -11008,25 +11328,15 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
     }
   }
 
-  // If we found more than one best candidate, this is ambiguous.
   if (Best == end())
     return OR_Ambiguous;
 
-  // Best is the best viable function.
-  if (Best->Function && Best->Function->isDeleted())
-    return OR_Deleted;
-
-  if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
-      Kind == CSK_AddressOfOverloadSet && M &&
-      M->isImplicitObjectMemberFunction()) {
-    return OR_No_Viable_Function;
-  }
+  OverloadingResult R = ResultForBestCandidate(Best);
 
   if (!EquivalentCands.empty())
     S.diagnoseEquivalentInternalLinkageDeclarations(Loc, Best->Function,
                                                     EquivalentCands);
-
-  return OR_Success;
+  return R;
 }
 
 namespace {
@@ -12733,6 +13043,9 @@ SmallVector<OverloadCandidate *, 32> OverloadCandidateSet::CompleteCandidates(
     Sema &S, OverloadCandidateDisplayKind OCD, ArrayRef<Expr *> Args,
     SourceLocation OpLoc,
     llvm::function_ref<bool(OverloadCandidate &)> Filter) {
+
+  InjectNonDeducedTemplateCandidates(S);
+
   // Sort the candidates by viability and position.  Sorting directly would
   // be prohibitive, so we make a set of pointers and sort those.
   SmallVector<OverloadCandidate*, 32> Cands;
@@ -14354,10 +14667,12 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
                                          Expr *ExecConfig,
                                          bool AllowTypoCorrection,
                                          bool CalleesAddressIsTaken) {
-  OverloadCandidateSet CandidateSet(
-      Fn->getExprLoc(), CalleesAddressIsTaken
-                            ? OverloadCandidateSet::CSK_AddressOfOverloadSet
-                            : OverloadCandidateSet::CSK_Normal);
+
+  OverloadCandidateSet::CandidateSetKind CSK =
+      CalleesAddressIsTaken ? OverloadCandidateSet::CSK_AddressOfOverloadSet
+                            : OverloadCandidateSet::CSK_Normal;
+
+  OverloadCandidateSet CandidateSet(Fn->getExprLoc(), CSK);
   ExprResult result;
 
   if (buildOverloadedCallSet(S, Fn, ULE, Args, LParenLoc, &CandidateSet,
@@ -14373,6 +14688,17 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
   OverloadingResult OverloadResult =
       CandidateSet.BestViableFunction(*this, Fn->getBeginLoc(), Best);
 
+  // [C++23][over.call.func]
+  // if overload resolution selects a non-static member function,
+  // the call is ill-formed;
+  if (CSK == OverloadCandidateSet::CSK_AddressOfOverloadSet &&
+      Best != CandidateSet.end()) {
+    if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
+        M && M->isImplicitObjectMemberFunction()) {
+      OverloadResult = OR_No_Viable_Function;
+    }
+  }
+
   // Model the case with a call to a templated function whose definition
   // encloses the call and whose return type contains a placeholder type as if
   // the UnresolvedLookupExpr was type-dependent.
@@ -14708,18 +15034,24 @@ void Sema::LookupOverloadedBinOp(OverloadCandidateSet &CandidateSet,
   // rewritten candidates using these functions if necessary.
   AddNonMemberOperatorCandidates(Fns, Args, CandidateSet);
 
+  // As template candidates are not deduced immediately,
+  // persist the array in the overload set.
+  ArrayRef<Expr *> ReversedArgs;
+  if (CandidateSet.getRewriteInfo().allowsReversed(Op) ||
+      CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
+    ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
+
   // Add operator candidates that are member functions.
   AddMemberOperatorCandidates(Op, OpLoc, Args, CandidateSet);
   if (CandidateSet.getRewriteInfo().allowsReversed(Op))
-    AddMemberOperatorCandidates(Op, OpLoc, {Args[1], Args[0]}, CandidateSet,
+    AddMemberOperatorCandidates(Op, OpLoc, ReversedArgs, CandidateSet,
                                 OverloadCandidateParamOrder::Reversed);
 
   // In C++20, also add any rewritten member candidates.
   if (ExtraOp) {
     AddMemberOperatorCandidates(ExtraOp, OpLoc, Args, CandidateSet);
     if (CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
-      AddMemberOperatorCandidates(ExtraOp, OpLoc, {Args[1], Args[0]},
-                                  CandidateSet,
+      AddMemberOperatorCandidates(ExtraOp, OpLoc, ReversedArgs, CandidateSet,
                                   OverloadCandidateParamOrder::Reversed);
   }
 
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 772962ac653f7..0ecdbb3ffb89f 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -6142,9 +6142,9 @@ FunctionDecl *Sema::getMoreConstrainedFunction(FunctionDecl *FD1,
   assert(!FD1->getDescribedTemplate() && !FD2->getDescribedTemplate() &&
          "not for function templates");
   assert(!FD1->isFunctionTemplateSpecialization() ||
-         isa<CXXConversionDecl>(FD1));
+         (isa<CXXConversionDecl, CXXConstructorDecl>(FD1)));
   assert(!FD2->isFunctionTemplateSpecialization() ||
-         isa<CXXConversionDecl>(FD2));
+         (isa<CXXConversionDecl, CXXConstructorDecl>(FD2)));
 
   FunctionDecl *F1 = FD1;
   if (FunctionDecl *P = FD1->getTemplateInstantiationPattern(false))
diff --git a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
index ba8e2dc372e98..083e743818121 100644
--- a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
+++ b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
@@ -14,7 +14,7 @@ template<typename T> struct S {
 // expected-note@#FINST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{})
 void f(T);
-void f(int);
+void f(long);
 
 // Ensure this applies to operator && as well.
 // expected-error@+3{{atomic constraint must be of type 'bool' (found 'S<int>')}}
@@ -22,7 +22,7 @@ void f(int);
 // expected-note@#F2INST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{} && true)
 void f2(T);
-void f2(int);
+void f2(long);
 
 template<typename T> requires requires {
   requires S<T>{};
@@ -36,12 +36,12 @@ template<typename T> requires requires {
   //
 }
 void f3(T);
-void f3(int);
+void f3(long);
 
 // Doesn't diagnose, since this is no longer a compound requirement.
 template<typename T> requires (bool(1 && 2))
 void f4(T);
-void f4(int);
+void f4(long);
 
 void g() {
   f(0); // #FINST
diff --git a/clang/test/SemaCUDA/function-overload.cu b/clang/test/SemaCUDA/function-overload.cu
index 4710c81763adf..3d05839af7528 100644
--- a/clang/test/SemaCUDA/function-overload.cu
+++ b/clang/test/SemaCUDA/function-overload.cu
@@ -1,6 +1,3 @@
-// REQUIRES: x86-registered-target
-// REQUIRES: nvptx-registered-target
-
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu -fsyntax-only \
 // RUN:   -verify=host,hostdefer,devdefer,expected %s
 // RUN: %clang_cc1 -std=c++14 -triple nvptx64-nvidia-cuda -fsyntax-only \
diff --git a/clang/test/SemaCXX/implicit-member-functions.cpp b/clang/test/SemaCXX/implicit-member-functions.cpp
index 1554b1af5d59a..8350eac5b88a0 100644
--- a/clang/test/SemaCXX/implicit-member-functions.cpp
+++ b/clang/test/SemaCXX/implicit-member-functions.cpp
@@ -54,31 +54,24 @@ namespace PR7594 {
 namespace Recursion {
   template<typename T> struct InvokeCopyConstructor {
     static const T &get();
-    typedef decltype(T(get())) type; // expected-error {{no matching conver}}
+    typedef decltype(T(get())) type;
   };
   struct B;
   struct A {
-    // expected-note@-1 {{while substituting deduced template arguments}}
     typedef B type;
     template<typename T,
              typename = typename InvokeCopyConstructor<typename T::type>::type>
-    // expected-note@-1 {{in instantiation of template class}}
     A(const T &);
-    // expected-note@-1 {{in instantiation of default argument}}
   };
-  struct B { // expected-note {{while declaring the implicit copy constructor for 'B'}}
-    // expected-note@-1 {{candidate constructor (the implicit move }}
-    B(); // expected-note {{candidate constructor not viable}}
+  struct B {
+    B();
     A a;
   };
   // Triggering the declaration of B's copy constructor causes overload
-  // resolution to occur for A's copying constructor, which instantiates
-  // InvokeCopyConstructor<B>, which triggers the declaration of B's copy
-  // constructor. Notionally, this happens when we get to the end of the
-  // definition of 'struct B', so there is no declared copy constructor yet.
-  //
-  // This behavior is g++-compatible, but isn't exactly right; the class is
-  // supposed to be incomplete when we implicitly declare its special members.
+  // resolution to occur for A's copying constructor, which picks
+  // the implicit copy constructor of A.
+  // Because that copy constructor is always a perfect match the template
+  // candidate is not instantiated.
   B b = B();
 
 
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
new file mode 100644
index 0000000000000..9139fd2688790
--- /dev/null
+++ b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -0,0 +1,200 @@
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++11 %s
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++20 %s
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++2c %s
+
+template <typename T>
+struct Invalid { static_assert(false, "instantiated Invalid"); }; // #err-invalid
+
+template <typename T>
+int f(T a, Invalid<T> = {}); // #note-f
+
+// sanity check
+int e1 = f(0);
+//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
+//expected-note@-2 {{in instantiation of default function argument expression for 'f<int>' required here}}
+//expected-note@#note-f {{in instantiation of template class 'Invalid<int>' requested here}}
+//expected-note@#note-f {{passing argument to parameter here}}
+
+int f(int);
+int ok1 = f(0);
+int e4 = f((const int&)(ok1));
+
+int f(int, int = 0);
+int ok2 = f(0, 0);
+
+int e2  = f(0L);
+//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
+//expected-note@-2 {{in instantiation of default function argument expression for 'f<long>' required here}}
+//expected-note@#note-f {{in instantiation of template class 'Invalid<long>' requested here}}
+//expected-note@#note-f {{passing argument to parameter here}}
+
+int f(long);
+int ok3 = f(0L);
+
+template <typename T>
+struct Invalid2 { static_assert(false, "instantiated Invalid2"); }; // #err-qualifiers
+
+template <typename T>
+int ref(T a, Invalid2<T> = {}); // expected-note 2{{here}}
+int ref(int&);
+int ref1 = ref(ok3);
+int ref2 = ref((const int&)ok3); // expected-note {{here}}
+//expected-error@#err-qualifiers {{static assertion failed: instantiated Invalid2}}
+
+
+template <typename T>
+int f_alias(T a, Invalid<T> = {});
+using Alias = int;
+int f_alias(Alias);
+int ok4 = f_alias(0);
+
+#if __cplusplus >= 202002
+
+struct Copyable {
+  template <typename T>
+  requires __is_constructible(Copyable, T)
+  explicit Copyable(T op) noexcept; // #1
+  Copyable(const Copyable&) noexcept = default; // #2
+};
+static_assert(__is_constructible(Copyable, const Copyable&));
+
+struct ImplicitlyCopyable {
+  template <typename T>
+  requires __is_constructible(ImplicitlyCopyable, T)
+  explicit ImplicitlyCopyable(T op) = delete; // #1
+};
+static_assert(__is_constructible(ImplicitlyCopyable, const ImplicitlyCopyable&));
+
+
+struct Movable {
+  template <typename T>
+  requires __is_constructible(Movable, T) // #err-self-constraint-1
+  explicit Movable(T op) noexcept; // #1
+  Movable(Movable&&) noexcept = default; // #2
+};
+static_assert(__is_constructible(Movable, Movable&&));
+static_assert(__is_constructible(Movable, const Movable&));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(Movable, const Movable &)'}}
+
+static_assert(__is_constructible(Movable, int));
+// expected-error@-1{{static assertion failed due to requirement '__is_constructible(Movable, int)'}} \
+// expected-note@-1 2{{}}
+// expected-error@#err-self-constraint-1{{satisfaction of constraint '__is_constructible(Movable, T)' depends on itself}}
+// expected-note@#err-self-constraint-1 4{{}}
+
+template <typename T>
+struct Members {
+    constexpr auto f(auto) {
+        static_assert(false, "");
+    }
+    constexpr auto f(int) { return 1; }
+    constexpr auto f(int) requires true { return 2; }
+
+    constexpr auto g(auto) {
+        static_assert(false, "instantiated member"); //#err-qualified-member
+        return 0;
+    }
+    constexpr auto g(int) & { return 1; }
+
+    static constexpr auto s(auto) {
+        static_assert(false, "");
+    }
+    static constexpr auto s(int) {
+        return 1;
+    }
+    static constexpr auto s(int) requires true {
+        return 2;
+    }
+};
+
+static_assert(Members<int[1]>{}.f(0) == 2);
+static_assert(Members<int[2]>{}.g(0) == 0);
+// expected-error@#err-qualified-member {{static assertion failed: instantiated member}} \
+// expected-note@-1{{in instantiation of function template specialization 'Members<int[2]>::g<int>' }}
+Members<int[3]> m1;
+static_assert(m1.g(0) == 1);
+static_assert(Members<int[3]>{}.s(0) == 2);
+
+
+namespace ConstructorInit{
+struct S {
+  template <typename T>
+  S(T&&) {}
+};
+struct Test {
+  operator S() = delete;
+};
+
+static_assert(__is_constructible(S, Test));
+}
+
+namespace RefBinding {
+
+template <typename> struct remove_reference;
+template <typename _Tp> struct remove_reference<_Tp &> {
+  using type = _Tp;
+};
+template <typename _Tp> remove_reference<_Tp>::type move(_Tp &&);
+template <typename _Head> struct _Head_base {
+  _Head_base(_Head &__h) : _M_head_impl(__h) {}
+  template <typename _UHead> _Head_base(_UHead &&);
+  _Head _M_head_impl;
+};
+
+template <typename _Elements> void forward_as_tuple(_Elements &&) {
+  _Head_base<_Elements &&>(_Elements{});
+}
+struct StringRef {
+  void operator[](const StringRef __k) { forward_as_tuple((move)(__k)); }
+};
+
+}
+
+template <class> struct tuple {};
+struct BonkersBananas {
+  template <class T> operator T();
+  template <class = void> explicit operator tuple<int>() = delete;
+};
+static_assert(!__is_constructible(tuple<int>, BonkersBananas));
+
+namespace GH62096 {
+template <typename T>
+struct Oops {
+  static_assert(sizeof(T) == 0); // #GH62096-err
+  static constexpr bool value = true;
+};
+
+template <class OPERATOR>
+concept Operator = Oops<OPERATOR>::value; // #GH62096-note1
+
+template <Operator OP> void f(OP op); // // #GH62096-note2
+void f(int);
+
+void g(int n) { f(n); } // OK
+void h(short n) { f(n); }
+// expected-error@#GH62096-err {{static assertion failed due to requirement 'sizeof(short) == 0'}} \
+// expected-note@-1{{in instantiation of function template specialization}} \
+// expected-note@-1{{while checking constraint satisfaction for template}}
+// expected-note@#GH62096-note1{{in instantiation}}
+// expected-note@#GH62096-note1{{while substituting template arguments into constraint expression here}}
+// expected-note@#GH62096-note2{{while substituting template arguments into constraint expression here}}
+// expected-note@#GH62096-note2{{while checking the satisfaction of concept}}
+// expected-note@#GH62096-err {{expression evaluates}}
+}
+
+#endif
+
+template<typename ...Ts>
+struct t1 {
+};
+struct t6 {
+  template<typename T = int>
+  operator t1<float>() {
+    return {};
+  }
+};
+
+int main() {
+  t6 v6;
+  v6.operator t1<float>();
+}
diff --git a/clang/test/SemaTemplate/instantiate-function-params.cpp b/clang/test/SemaTemplate/instantiate-function-params.cpp
index 7dd5595de58a3..eb2a7c5d4e8d6 100644
--- a/clang/test/SemaTemplate/instantiate-function-params.cpp
+++ b/clang/test/SemaTemplate/instantiate-function-params.cpp
@@ -6,13 +6,12 @@ template<typename T1> struct if_ {
   typedef if_c< static_cast<bool>(T1::value)> almost_type_; // expected-note 7{{in instantiation}}
 };
 template <class Model, void (Model::*)()> struct wrap_constraints { };
-template <class Model> 
+template <class Model>
 inline char has_constraints_(Model* , // expected-note 3{{candidate template ignored}}
-                               wrap_constraints<Model,&Model::constraints>* = 0); // expected-note 4{{in instantiation}}
-
+                               wrap_constraints<Model,&Model::constraints>* = 0);
 template <class Model> struct not_satisfied {
   static const bool value = sizeof( has_constraints_((Model*)0)  == 1); // expected-error 3{{no matching function}} \
-  // expected-note 4{{while substituting deduced template arguments into function template 'has_constraints_' [with }}
+  // expected-note 4{{in instantiation}}
 };
 template <class ModelFn> struct requirement_;
 template <void(*)()> struct instantiate {
diff --git a/clang/test/Templight/templight-empty-entries-fix.cpp b/clang/test/Templight/templight-empty-entries-fix.cpp
index d13b748068efe..7f34b10134929 100644
--- a/clang/test/Templight/templight-empty-entries-fix.cpp
+++ b/clang/test/Templight/templight-empty-entries-fix.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -templight-dump -Wno-unused-value %s 2>&1 | FileCheck %s
 
-void a() {
+void a(long) {
   [] {};
 }
 
@@ -17,14 +17,14 @@ void a() {
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 // CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 
-template <int = 0> void a() { a(); }
+template <int = 0> void a(long) { a(0); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -42,29 +42,29 @@ template <int = 0> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 
 template <int> struct b { typedef int c; };
-template <bool d = true, class = typename b<d>::c> void a() { a(); }
+template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -130,25 +130,25 @@ template <bool d = true, class = typename b<d>::c> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -166,34 +166,10 @@ template <bool d = true, class = typename b<d>::c> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 
 template <bool = true> void d(int = 0) { d(); }
 
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
@@ -249,41 +225,41 @@ void e() {
 }
 
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
 
 
 template <template<typename> class>
@@ -299,71 +275,71 @@ void foo() {
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}

From 78b37ca2a3af50de1daea09295b6f4ea99e7edaa Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Thu, 17 Apr 2025 14:28:53 +0800
Subject: [PATCH 225/710] [Hexagon] Pre-commit tests for PR130742. NFC.
 (#135604)

Needed by https://github.com/llvm/llvm-project/pull/130742.
---
 llvm/test/CodeGen/Hexagon/64bit_tstbit.ll                   | 4 ++--
 llvm/test/CodeGen/Hexagon/always-ext.ll                     | 2 +-
 llvm/test/CodeGen/Hexagon/autohvx/isel-concat-multiple.ll   | 2 +-
 .../CodeGen/Hexagon/autohvx/isel-q-legalization-loop.ll     | 4 ++--
 .../test/CodeGen/Hexagon/autohvx/vector-align-terminator.ll | 6 +++---
 .../Hexagon/autohvx/vector-align-use-in-different-block.ll  | 4 ++--
 llvm/test/CodeGen/Hexagon/reg-scavengebug-2.ll              | 2 +-
 llvm/test/CodeGen/Hexagon/swp-const-tc1.ll                  | 2 +-
 8 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/llvm/test/CodeGen/Hexagon/64bit_tstbit.ll b/llvm/test/CodeGen/Hexagon/64bit_tstbit.ll
index c61e5a7fed685..f050e7fd845d4 100644
--- a/llvm/test/CodeGen/Hexagon/64bit_tstbit.ll
+++ b/llvm/test/CodeGen/Hexagon/64bit_tstbit.ll
@@ -14,9 +14,9 @@ target triple = "hexagon-unknown-unknown-elf"
 
 declare dso_local void @panic(ptr, ...) local_unnamed_addr
 
-define dso_local fastcc void @elv_rqhash_find() unnamed_addr {
+define dso_local fastcc void @elv_rqhash_find(ptr %p) unnamed_addr {
 entry:
-  %cmd_flags = getelementptr inbounds %struct.hlist_node.45.966.3115.3729.4036.4650.4957.6492.6799.7413.7720.9562.10790.11097.11404.11711.14474.17192, ptr null, i32 -5
+  %cmd_flags = getelementptr inbounds %struct.hlist_node.45.966.3115.3729.4036.4650.4957.6492.6799.7413.7720.9562.10790.11097.11404.11711.14474.17192, ptr %p, i32 -5
   %0 = load i64, ptr %cmd_flags, align 8
   %1 = and i64 %0, 4294967296
   %tobool10 = icmp eq i64 %1, 0
diff --git a/llvm/test/CodeGen/Hexagon/always-ext.ll b/llvm/test/CodeGen/Hexagon/always-ext.ll
index 7b64b2346db87..880514aa7dfbd 100644
--- a/llvm/test/CodeGen/Hexagon/always-ext.ll
+++ b/llvm/test/CodeGen/Hexagon/always-ext.ll
@@ -18,7 +18,7 @@
 
 declare void @_Assert()
 
-define void @CuSuiteAddSuite() nounwind {
+define void @CuSuiteAddSuite() nounwind null_pointer_is_valid {
 entry:
   br i1 undef, label %for.body.us, label %for.end
 
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/isel-concat-multiple.ll b/llvm/test/CodeGen/Hexagon/autohvx/isel-concat-multiple.ll
index 362918e0ca53c..1d6e2a4199ba7 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/isel-concat-multiple.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/isel-concat-multiple.ll
@@ -26,7 +26,7 @@ b0:
   %v15 = lshr <8 x i32> %v14, <i32 18, i32 18, i32 18, i32 18, i32 18, i32 18, i32 18, i32 18>
   %v16 = and <8 x i32> %v15, %v14
   %v17 = extractelement <8 x i32> %v16, i32 5
-  %v18 = getelementptr inbounds i8, ptr null, i32 %v17
+  %v18 = getelementptr inbounds i8, ptr %a0, i32 %v17
   %v19 = load i8, ptr %v18, align 1
   store i8 %v19, ptr %a2, align 1
   ret void
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/isel-q-legalization-loop.ll b/llvm/test/CodeGen/Hexagon/autohvx/isel-q-legalization-loop.ll
index b6dc87009e86d..fd3078e4ba0a7 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/isel-q-legalization-loop.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/isel-q-legalization-loop.ll
@@ -15,10 +15,10 @@ declare <64 x i32> @llvm.hexagon.V6.vdealvdd.128B(<32 x i32>, <32 x i32>, i32) #
 declare <32 x i32> @llvm.hexagon.V6.hi.128B(<64 x i32>) #0
 
 ; Function Attrs: nounwind
-define void @f0() local_unnamed_addr #1 {
+define void @f0(ptr %p) local_unnamed_addr #1 {
 b0:
   %v0 = tail call <128 x i1> @llvm.hexagon.V6.vandvrt.128B(<32 x i32> undef, i32 16843009)
-  %v1 = getelementptr inbounds %s.0, ptr null, i32 0, i32 0, i32 3
+  %v1 = getelementptr inbounds %s.0, ptr %p, i32 0, i32 0, i32 3
   br label %b1
 
 b1:                                               ; preds = %b1, %b0
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-terminator.ll b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-terminator.ll
index 590b4fe9aa6f3..d47bc27757cc3 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-terminator.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-terminator.ll
@@ -5,18 +5,18 @@
 
 target triple = "hexagon"
 
-define void @f0() #0 {
+define void @f0(ptr %p) #0 {
 b0:
   br label %b1
 
 b1:                                               ; preds = %b0, %b1
   %v0 = phi i32 [ %v9, %b1 ], [ 0, %b0 ]
   %v1 = zext i32 %v0 to i64
-  %v2 = getelementptr inbounds float, ptr null, i64 %v1
+  %v2 = getelementptr inbounds float, ptr %p, i64 %v1
   store float poison, ptr %v2, align 16
   %v3 = or i32 %v0, 3
   %v4 = zext i32 %v3 to i64
-  %v5 = getelementptr inbounds float, ptr null, i64 %v4
+  %v5 = getelementptr inbounds float, ptr %p, i64 %v4
   store float poison, ptr %v5, align 4
   %v6 = add nuw nsw i32 %v0, 4
   %v7 = icmp ult i32 %v3, 63
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-use-in-different-block.ll b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-use-in-different-block.ll
index 1fdb75f701fe7..f85c81b9bceb2 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-use-in-different-block.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-use-in-different-block.ll
@@ -8,7 +8,7 @@
 target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1124:1024:1024-v2048:2048:2048"
 target triple = "hexagon"
 
-define dso_local <32 x i32> @f0(i32 %a0, i32 %a1) local_unnamed_addr #0 {
+define dso_local <32 x i32> @f0(i32 %a0, i32 %a1, ptr %p) local_unnamed_addr #0 {
 ; CHECK-LABEL: f0:
 ; CHECK:     = vmem({{.*}})
 ; CHECK:     = vmem({{.*}})
@@ -17,7 +17,7 @@ b0:
 
 b1:                                               ; preds = %b0
   %v0 = mul nsw i32 -4, %a0
-  %v1 = getelementptr inbounds i8, ptr null, i32 %v0
+  %v1 = getelementptr inbounds i8, ptr %p, i32 %v0
   %v2 = getelementptr inbounds i8, ptr %v1, i32 -64
   %v4 = load <16 x i32>, ptr %v2, align 64
   %v5 = getelementptr inbounds i8, ptr %v1, i32 64
diff --git a/llvm/test/CodeGen/Hexagon/reg-scavengebug-2.ll b/llvm/test/CodeGen/Hexagon/reg-scavengebug-2.ll
index e58bd86846293..ae6a4fd391cdc 100644
--- a/llvm/test/CodeGen/Hexagon/reg-scavengebug-2.ll
+++ b/llvm/test/CodeGen/Hexagon/reg-scavengebug-2.ll
@@ -112,7 +112,7 @@ declare <16 x i32> @llvm.hexagon.V6.lo(<32 x i32>) #1
 ; Function Attrs: nounwind readnone
 declare <32 x i32> @llvm.hexagon.V6.vunpackub(<16 x i32>) #1
 
-attributes #0 = { nounwind "target-cpu"="hexagonv60" "target-features"="+hvxv60,+hvx-length64b" }
+attributes #0 = { null_pointer_is_valid nounwind "target-cpu"="hexagonv60" "target-features"="+hvxv60,+hvx-length64b" }
 attributes #1 = { nounwind readnone }
 
 !0 = !{!1, !1, i64 0}
diff --git a/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll b/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
index 4b8d756e93312..26bbd71ad44de 100644
--- a/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
@@ -61,7 +61,7 @@ b6:                                               ; preds = %b5
   ret void
 }
 
-attributes #0 = { nounwind optsize "target-cpu"="hexagonv55" }
+attributes #0 = { null_pointer_is_valid nounwind optsize "target-cpu"="hexagonv55" }
 
 !0 = !{!1, !1, i64 0}
 !1 = !{!"short", !2}

From 8e67d8fdf3a31b42f52ec12d995490f866b4449b Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Thu, 17 Apr 2025 14:29:09 +0800
Subject: [PATCH 226/710] [Clang][CodeGen][OpenCL] Add
 `-fno-delete-null-pointer-checks` to avoid UB. NFC. (#135602)

Needed by https://github.com/llvm/llvm-project/pull/130742.
---
 clang/test/CodeGenOpenCL/spir32_target.cl | 2 +-
 clang/test/CodeGenOpenCL/spir64_target.cl | 2 +-
 clang/test/CodeGenOpenCL/spirv_target.cl  | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/test/CodeGenOpenCL/spir32_target.cl b/clang/test/CodeGenOpenCL/spir32_target.cl
index 924b2c12f5537..cccdf3635ea63 100644
--- a/clang/test/CodeGenOpenCL/spir32_target.cl
+++ b/clang/test/CodeGenOpenCL/spir32_target.cl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple "spir-unknown-unknown" -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple "spir-unknown-unknown" -fno-delete-null-pointer-checks -emit-llvm -o - | FileCheck %s
 
 // CHECK: target triple = "spir-unknown-unknown"
 
diff --git a/clang/test/CodeGenOpenCL/spir64_target.cl b/clang/test/CodeGenOpenCL/spir64_target.cl
index ba4a66d7158fd..67880f47adb34 100644
--- a/clang/test/CodeGenOpenCL/spir64_target.cl
+++ b/clang/test/CodeGenOpenCL/spir64_target.cl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple "spir64-unknown-unknown" -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple "spir64-unknown-unknown" -emit-llvm -fno-delete-null-pointer-checks -o - | FileCheck %s
 
 // CHECK: target triple = "spir64-unknown-unknown"
 
diff --git a/clang/test/CodeGenOpenCL/spirv_target.cl b/clang/test/CodeGenOpenCL/spirv_target.cl
index 2aeed195449a8..f9e92a79d5617 100644
--- a/clang/test/CodeGenOpenCL/spirv_target.cl
+++ b/clang/test/CodeGenOpenCL/spirv_target.cl
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 %s -triple "spirv32-unknown-unknown" -verify -emit-llvm -o - | FileCheck %s -check-prefix=SPIRV32
-// RUN: %clang_cc1 %s -triple "spirv64-unknown-unknown" -verify -emit-llvm -o - | FileCheck %s -check-prefix=SPIRV64
+// RUN: %clang_cc1 %s -triple "spirv32-unknown-unknown" -fno-delete-null-pointer-checks -verify -emit-llvm -o - | FileCheck %s -check-prefix=SPIRV32
+// RUN: %clang_cc1 %s -triple "spirv64-unknown-unknown" -fno-delete-null-pointer-checks -verify -emit-llvm -o - | FileCheck %s -check-prefix=SPIRV64
 
 // SPIRV32: target triple = "spirv32-unknown-unknown"
 // SPIRV64: target triple = "spirv64-unknown-unknown"

From d1a80deae674300d1011ccb6d6ee7030eaf8e713 Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Thu, 17 Apr 2025 14:34:32 +0800
Subject: [PATCH 227/710] Reapply "[Clang] Fix dependent local class
 instantiation bugs" (#135914)

This reapplies #134038

Since the last patch, this fixes a null pointer dereference where the
TSI of the destructor wasn't properly propagated into the
DeclarationNameInfo. We now construct a LocInfoType for dependent cases,
as done elsewhere in getDestructorName, such that GetTypeFromParser can
correctly obtain the TSI.

---

This patch fixes two long-standing bugs that prevent Clang from
instantiating local class members inside a dependent context. These bugs
were introduced in commits
https://github.com/llvm/llvm-project/commit/21eb1af469c3257606aec2270d544e0e8ecf77b2
and
https://github.com/llvm/llvm-project/commit/919df9d75ac2a721a8072327c803f34486884571.


https://github.com/llvm/llvm-project/commit/21eb1af469c3257606aec2270d544e0e8ecf77b2
introduced a concept called eligible methods such that it did an attempt
to skip past ineligible method instantiation when instantiating class
members. Unfortunately, this broke the instantiation chain for local
classes - getTemplateInstantiationPattern() would fail to find the
correct definition pattern if the class was defined within a partially
transformed dependent context.


https://github.com/llvm/llvm-project/commit/919df9d75ac2a721a8072327c803f34486884571
introduced a separate issue by incorrectly copying the
DeclarationNameInfo during function definition instantiation from the
template pattern, even though that DNI might contain a transformed
TypeSourceInfo. Since that TSI was already updated when the declaration
was instantiated, this led to inconsistencies. As a result, the final
instantiated function could lose track of the transformed declarations,
hence we crash: https://compiler-explorer.com/z/vjvoG76Tf.

This PR corrects them by

1. Removing the bypass logic for method instantiation. The eligible flag
is independent of instantiation and can be updated properly afterward,
so skipping instantiation is unnecessary.

2. Carefully handling TypeSourceInfo by creating a new instance that
preserves the pattern's source location while using the already
transformed type.
---
 clang/docs/ReleaseNotes.rst                   |  1 +
 clang/lib/Sema/SemaExprCXX.cpp                |  6 +-
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  3 -
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  | 59 ++++++++++++++++-
 .../CodeGenCXX/local-class-instantiation.cpp  | 64 +++++++++++++++++++
 .../SemaTemplate/instantiate-local-class.cpp  | 34 ++++++++++
 6 files changed, 161 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/CodeGenCXX/local-class-instantiation.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 2bc7054e3c794..fc36962b317e8 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -474,6 +474,7 @@ Bug Fixes to C++ Support
   by template argument deduction.
 - Clang is now better at instantiating the function definition after its use inside
   of a constexpr lambda. (#GH125747)
+- Fixed a local class member function instantiation bug inside dependent lambdas. (#GH59734), (#GH132208)
 - Clang no longer crashes when trying to unify the types of arrays with
   certain differences in qualifiers (this could happen during template argument
   deduction or when building a ternary operator). (#GH97005)
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 8df590fa624cf..394e56c465c24 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -344,10 +344,12 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II,
     // We didn't find our type, but that's OK: it's dependent anyway.
 
     // FIXME: What if we have no nested-name-specifier?
+    TypeSourceInfo *TSI = nullptr;
     QualType T =
         CheckTypenameType(ElaboratedTypeKeyword::None, SourceLocation(),
-                          SS.getWithLocInContext(Context), II, NameLoc);
-    return ParsedType::make(T);
+                          SS.getWithLocInContext(Context), II, NameLoc, &TSI,
+                          /*DeducedTSTContext=*/true);
+    return CreateParsedType(T, TSI);
   }
 
   // The remaining cases are all non-standard extensions imitating the behavior
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index d2408a94ad0ab..0e81804f8c1e7 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -4126,9 +4126,6 @@ Sema::InstantiateClassMembers(SourceLocation PointOfInstantiation,
       if (FunctionDecl *Pattern =
               Function->getInstantiatedFromMemberFunction()) {
 
-        if (Function->isIneligibleOrNotSelected())
-          continue;
-
         if (Function->getTrailingRequiresClause()) {
           ConstraintSatisfaction Satisfaction;
           if (CheckFunctionConstraints(Function, Satisfaction) ||
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 5c80077f294c6..76c055d28f091 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -5597,7 +5597,64 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   Function->setLocation(PatternDecl->getLocation());
   Function->setInnerLocStart(PatternDecl->getInnerLocStart());
   Function->setRangeEnd(PatternDecl->getEndLoc());
-  Function->setDeclarationNameLoc(PatternDecl->getNameInfo().getInfo());
+  // Let the instantiation use the Pattern's DeclarationNameLoc, due to the
+  // following awkwardness:
+  //
+  //   1. There are out-of-tree users of getNameInfo().getSourceRange(), who
+  //   expect the source range of the instantiated declaration to be set to
+  //   point to the definition.
+  //
+  //   2. That getNameInfo().getSourceRange() might return the TypeLocInfo's
+  //   location it tracked.
+  //
+  //   3. Function might come from an (implicit) declaration, while the pattern
+  //   comes from a definition. In these cases, we need the PatternDecl's source
+  //   location.
+  //
+  // To that end, we need to more or less tweak the DeclarationNameLoc. However,
+  // we can't blindly copy the DeclarationNameLoc from the PatternDecl to the
+  // function, since it contains associated TypeLocs that should have already
+  // been transformed. So, we rebuild the TypeLoc for that purpose. Technically,
+  // we should create a new function declaration and assign everything we need,
+  // but InstantiateFunctionDefinition updates the declaration in place.
+  auto NameLocPointsToPattern = [&] {
+    DeclarationNameInfo PatternName = PatternDecl->getNameInfo();
+    DeclarationNameLoc PatternNameLoc = PatternName.getInfo();
+    switch (PatternName.getName().getNameKind()) {
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
+      break;
+    default:
+      // Cases where DeclarationNameLoc doesn't matter, as it merely contains a
+      // source range.
+      return PatternNameLoc;
+    }
+
+    TypeSourceInfo *TSI = Function->getNameInfo().getNamedTypeInfo();
+    // TSI might be null if the function is named by a constructor template id.
+    // E.g. S<T>() {} for class template S with a template parameter T.
+    if (!TSI) {
+      // We don't care about the DeclarationName of the instantiated function,
+      // but only the DeclarationNameLoc. So if the TypeLoc is absent, we do
+      // nothing.
+      return PatternNameLoc;
+    }
+
+    QualType InstT = TSI->getType();
+    // We want to use a TypeLoc that reflects the transformed type while
+    // preserving the source location from the pattern.
+    TypeLocBuilder TLB;
+    TypeSourceInfo *PatternTSI = PatternName.getNamedTypeInfo();
+    assert(PatternTSI && "Pattern is supposed to have an associated TSI");
+    // FIXME: PatternTSI is not trivial. We should copy the source location
+    // along the TypeLoc chain. However a trivial TypeLoc is sufficient for
+    // getNameInfo().getSourceRange().
+    TLB.pushTrivial(Context, InstT, PatternTSI->getTypeLoc().getBeginLoc());
+    return DeclarationNameLoc::makeNamedTypeLoc(
+        TLB.getTypeSourceInfo(Context, InstT));
+  };
+  Function->setDeclarationNameLoc(NameLocPointsToPattern());
 
   EnterExpressionEvaluationContext EvalContext(
       *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
diff --git a/clang/test/CodeGenCXX/local-class-instantiation.cpp b/clang/test/CodeGenCXX/local-class-instantiation.cpp
new file mode 100644
index 0000000000000..34103a1ee55ef
--- /dev/null
+++ b/clang/test/CodeGenCXX/local-class-instantiation.cpp
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -std=c++17 %s -emit-llvm -triple %itanium_abi_triple -o - | FileCheck %s
+
+namespace LambdaContainingLocalClasses {
+
+template <typename F>
+void GH59734() {
+  [&](auto param) {
+    struct Guard {
+      Guard() {
+        // Check that we're able to create DeclRefExpr to param at this point.
+        static_assert(__is_same(decltype(param), int), "");
+      }
+      ~Guard() {
+        static_assert(__is_same(decltype(param), int), "");
+      }
+      operator decltype(param)() {
+        return decltype(param)();
+      }
+    };
+    Guard guard;
+    param = guard;
+  }(42);
+}
+
+// Guard::Guard():
+// CHECK-DAG: define {{.*}} @_ZZZN28LambdaContainingLocalClasses7GH59734IiEEvvENKUlT_E_clIiEEDaS1_EN5GuardC2Ev
+// Guard::operator int():
+// CHECK-DAG: define {{.*}} @_ZZZN28LambdaContainingLocalClasses7GH59734IiEEvvENKUlT_E_clIiEEDaS1_EN5GuardcviEv
+// Guard::~Guard():
+// CHECK-DAG: define {{.*}} @_ZZZN28LambdaContainingLocalClasses7GH59734IiEEvvENKUlT_E_clIiEEDaS1_EN5GuardD2Ev
+
+struct S {};
+
+template <class T = void>
+auto GH132208 = [](auto param) {
+  struct OnScopeExit {
+    OnScopeExit() {
+      static_assert(__is_same(decltype(param), S), "");
+    }
+    ~OnScopeExit() {
+      static_assert(__is_same(decltype(param), S), "");
+    }
+    operator decltype(param)() {
+      return decltype(param)();
+    }
+  } pending;
+
+  param = pending;
+};
+
+void bar() {
+  GH59734<int>();
+
+  GH132208<void>(S{});
+}
+
+// OnScopeExit::OnScopeExit():
+// CHECK-DAG: define {{.*}} @_ZZNK28LambdaContainingLocalClasses8GH132208IvEMUlT_E_clINS_1SEEEDaS2_EN11OnScopeExitC2Ev
+// OnScopeExit::operator S():
+// CHECK-DAG: define {{.*}} @_ZZNK28LambdaContainingLocalClasses8GH132208IvEMUlT_E_clINS_1SEEEDaS2_EN11OnScopeExitcvS5_Ev
+// OnScopeExit::~OnScopeExit():
+// CHECK-DAG: define {{.*}} @_ZZNK28LambdaContainingLocalClasses8GH132208IvEMUlT_E_clINS_1SEEEDaS2_EN11OnScopeExitD2Ev
+
+} // namespace LambdaContainingLocalClasses
diff --git a/clang/test/SemaTemplate/instantiate-local-class.cpp b/clang/test/SemaTemplate/instantiate-local-class.cpp
index 298233739900f..810ce9329eae7 100644
--- a/clang/test/SemaTemplate/instantiate-local-class.cpp
+++ b/clang/test/SemaTemplate/instantiate-local-class.cpp
@@ -535,3 +535,37 @@ void foo() {
 } // namespace local_recursive_lambda
 
 #endif
+
+namespace PR134038_Regression {
+
+template <class T> class G {
+public:
+  template <class> class Iter {
+  public:
+    Iter();
+    ~Iter();
+
+    operator G<T>();
+  };
+};
+
+template <class ObserverType>
+template <class ContainerType>
+G<ObserverType>::Iter<ContainerType>::Iter() {}
+
+template <class ObserverType>
+template <class ContainerType>
+G<ObserverType>::Iter<ContainerType>::~Iter() {}
+
+template <class ObserverType>
+template <class ContainerType>
+G<ObserverType>::Iter<ContainerType>::operator G<ObserverType>() {
+  return G<ObserverType>{};
+}
+
+void NotifySettingChanged()  {
+  G<int>::Iter<int> Iter;
+  G<int> g = Iter;
+}
+
+}

From 1a09ffea31e22172ce3de2fd553b770f52add576 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Thu, 17 Apr 2025 08:48:12 +0200
Subject: [PATCH 228/710] [mlir][ArmSME][NFC] Check early for unsupported mask
 ops (#135955)

This is to avoid rollbacks in the dialect conversion, which are
expensive.

Note: This is in preparation of the One-Shot Dialect Conversion
refactoring.
---
 mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 6ed29903ea407..630414030d98b 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -77,11 +77,6 @@ FailureOr<scf::ForOp> createLoadStoreForOverTileSlices(
   Value upperBound;
   if (mask) {
     auto createMaskOp = mask.getDefiningOp<vector::CreateMaskOp>();
-    if (!createMaskOp)
-      return rewriter.notifyMatchFailure(
-          loc, "unsupported mask op, only 'vector.create_mask' is "
-               "currently supported");
-
     auto maskDim0 = createMaskOp.getOperands()[0];
     auto maskDim1 = createMaskOp.getOperands()[1];
 
@@ -184,6 +179,10 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
 
     Value initTile;
     if (mask) {
+      if (!mask.getDefiningOp<vector::CreateMaskOp>())
+        return rewriter.notifyMatchFailure(
+            loc, "unsupported mask op, only 'vector.create_mask' is "
+                 "currently supported");
       auto padOp = tileLoadOp.getPadding();
       assert(padOp && "expected padding when masking!");
 
@@ -373,6 +372,14 @@ struct TileStoreOpConversion : public OpRewritePattern<arm_sme::TileStoreOp> {
 
   LogicalResult matchAndRewrite(arm_sme::TileStoreOp tileStoreOp,
                                 PatternRewriter &rewriter) const override {
+    if (Value mask = tileStoreOp.getMask()) {
+      if (!mask.getDefiningOp<vector::CreateMaskOp>())
+        return rewriter.notifyMatchFailure(
+            tileStoreOp.getLoc(),
+            "unsupported mask op, only 'vector.create_mask' is "
+            "currently supported");
+    }
+
     // Create a loop that stores each active ZA tile slice from memory.
     return createLoadStoreForOverTileSlices(
         rewriter, tileStoreOp.getLoc(), tileStoreOp.getVectorType(),

From 1906c18ce01092aed9524f1cb8344c11920ba144 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Thu, 17 Apr 2025 08:48:29 +0200
Subject: [PATCH 229/710] [mlir][memref] Mark result memref as "strided"
 (#135939)

We already check this in the verifier, but this makes it explicit in the
Tablegen.
---
 mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index 3edc2433c85ea..0cea19964f5f4 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -1372,7 +1372,7 @@ def MemRef_ReinterpretCastOp
                        DenseI64ArrayAttr:$static_offsets,
                        DenseI64ArrayAttr:$static_sizes,
                        DenseI64ArrayAttr:$static_strides);
-  let results = (outs AnyMemRef:$result);
+  let results = (outs AnyStridedMemRef:$result);
 
   let assemblyFormat = [{
     $source `to` `offset` `` `:`

From 62d32c2c27a83261af9f2529a961a22605df8a2b Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Thu, 17 Apr 2025 08:58:48 +0200
Subject: [PATCH 230/710] [mlir][memref][NFC] Simplify `constifyIndexValues`
 (#135940)

Simplify the code by removing function pointers.
---
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp | 149 ++++++++---------------
 1 file changed, 48 insertions(+), 101 deletions(-)

diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 63f5251398716..6f10a31c15626 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -88,101 +88,30 @@ SmallVector<OpFoldResult> memref::getMixedSizes(OpBuilder &builder,
 // Utility functions for propagating static information
 //===----------------------------------------------------------------------===//
 
-/// Helper function that infers the constant values from a list of \p values,
-/// a \p memRefTy, and another helper function \p getAttributes.
-/// The inferred constant values replace the related `OpFoldResult` in
-/// \p values.
+/// Helper function that sets values[i] to constValues[i] if the latter is a
+/// static value, as indicated by ShapedType::kDynamic.
 ///
-/// \note This function shouldn't be used directly, instead, use the
-/// `getConstifiedMixedXXX` methods from the related operations.
-///
-/// \p getAttributes retuns a list of potentially constant values, as determined
-/// by \p isDynamic, from the given \p memRefTy. The returned list must have as
-/// many elements as \p values or be empty.
-///
-/// E.g., consider the following example:
-/// ```
-/// memref.reinterpret_cast %base to <...> strides: [2, %dyn_stride] :
-///     memref<f32> to memref<?x?xf32, strided<[?, 1], offset: ?>>
-/// ```
-/// `ReinterpretCastOp::getMixedStrides()` will return `[2, %dyn_stride]`.
-/// Now using this helper function with:
-/// - `values == [2, %dyn_stride]`,
-/// - `memRefTy == memref<?x?xf32, strided<[?, 1], offset: ?>>`
-/// - `getAttributes == getConstantStrides` (i.e., a wrapper around
-/// `getStridesAndOffset`), and
-/// - `isDynamic == ShapedType::isDynamic`
-/// Will yield: `values == [2, 1]`
-static void constifyIndexValues(
-    SmallVectorImpl<OpFoldResult> &values, MemRefType memRefTy,
-    MLIRContext *ctxt,
-    llvm::function_ref<SmallVector<int64_t>(MemRefType)> getAttributes,
-    llvm::function_ref<bool(int64_t)> isDynamic) {
-  SmallVector<int64_t> constValues = getAttributes(memRefTy);
-  Builder builder(ctxt);
-  for (const auto &it : llvm::enumerate(constValues)) {
-    int64_t constValue = it.value();
-    if (!isDynamic(constValue))
-      values[it.index()] = builder.getIndexAttr(constValue);
-  }
-  for (OpFoldResult &ofr : values) {
-    if (auto attr = dyn_cast<Attribute>(ofr)) {
-      // FIXME: We shouldn't need to do that, but right now, the static indices
-      // are created with the wrong type: `i64` instead of `index`.
-      // As a result, if we were to keep the attribute as is, we may fail to see
-      // that two attributes are equal because one would have the i64 type and
-      // the other the index type.
-      // The alternative would be to create constant indices with getI64Attr in
-      // this and the previous loop, but it doesn't logically make sense (we are
-      // dealing with indices here) and would only strenghten the inconsistency
-      // around how static indices are created (some places use getI64Attr,
-      // others use getIndexAttr).
-      // The workaround here is to stick to the IndexAttr type for all the
-      // values, hence we recreate the attribute even when it is already static
-      // to make sure the type is consistent.
-      ofr = builder.getIndexAttr(llvm::cast<IntegerAttr>(attr).getInt());
+/// If constValues[i] is dynamic, tries to extract a constant value from
+/// value[i] to allow for additional folding opportunities. Also convertes all
+/// existing attributes to index attributes. (They may be i64 attributes.)
+static void constifyIndexValues(SmallVectorImpl<OpFoldResult> &values,
+                                ArrayRef<int64_t> constValues) {
+  assert(constValues.size() == values.size() &&
+         "incorrect number of const values");
+  for (auto [i, cstVal] : llvm::enumerate(constValues)) {
+    Builder builder(values[i].getContext());
+    if (!ShapedType::isDynamic(cstVal)) {
+      // Constant value is known, use it directly.
+      values[i] = builder.getIndexAttr(cstVal);
       continue;
     }
-    std::optional<int64_t> maybeConstant =
-        getConstantIntValue(cast<Value>(ofr));
-    if (maybeConstant)
-      ofr = builder.getIndexAttr(*maybeConstant);
+    if (std::optional<int64_t> cst = getConstantIntValue(values[i])) {
+      // Try to extract a constant or convert an existing to index.
+      values[i] = builder.getIndexAttr(*cst);
+    }
   }
 }
 
-/// Wrapper around `getShape` that conforms to the function signature
-/// expected for `getAttributes` in `constifyIndexValues`.
-static SmallVector<int64_t> getConstantSizes(MemRefType memRefTy) {
-  ArrayRef<int64_t> sizes = memRefTy.getShape();
-  return SmallVector<int64_t>(sizes);
-}
-
-/// Wrapper around `getStridesAndOffset` that returns only the offset and
-/// conforms to the function signature expected for `getAttributes` in
-/// `constifyIndexValues`.
-static SmallVector<int64_t> getConstantOffset(MemRefType memrefType) {
-  SmallVector<int64_t> strides;
-  int64_t offset;
-  LogicalResult hasStaticInformation =
-      memrefType.getStridesAndOffset(strides, offset);
-  if (failed(hasStaticInformation))
-    return SmallVector<int64_t>();
-  return SmallVector<int64_t>(1, offset);
-}
-
-/// Wrapper around `getStridesAndOffset` that returns only the strides and
-/// conforms to the function signature expected for `getAttributes` in
-/// `constifyIndexValues`.
-static SmallVector<int64_t> getConstantStrides(MemRefType memrefType) {
-  SmallVector<int64_t> strides;
-  int64_t offset;
-  LogicalResult hasStaticInformation =
-      memrefType.getStridesAndOffset(strides, offset);
-  if (failed(hasStaticInformation))
-    return SmallVector<int64_t>();
-  return strides;
-}
-
 //===----------------------------------------------------------------------===//
 // AllocOp / AllocaOp
 //===----------------------------------------------------------------------===//
@@ -1445,24 +1374,34 @@ ExtractStridedMetadataOp::fold(FoldAdaptor adaptor,
 
 SmallVector<OpFoldResult> ExtractStridedMetadataOp::getConstifiedMixedSizes() {
   SmallVector<OpFoldResult> values = getAsOpFoldResult(getSizes());
-  constifyIndexValues(values, getSource().getType(), getContext(),
-                      getConstantSizes, ShapedType::isDynamic);
+  constifyIndexValues(values, getSource().getType().getShape());
   return values;
 }
 
 SmallVector<OpFoldResult>
 ExtractStridedMetadataOp::getConstifiedMixedStrides() {
   SmallVector<OpFoldResult> values = getAsOpFoldResult(getStrides());
-  constifyIndexValues(values, getSource().getType(), getContext(),
-                      getConstantStrides, ShapedType::isDynamic);
+  SmallVector<int64_t> staticValues;
+  int64_t unused;
+  LogicalResult status =
+      getSource().getType().getStridesAndOffset(staticValues, unused);
+  (void)status;
+  assert(succeeded(status) && "could not get strides from type");
+  constifyIndexValues(values, staticValues);
   return values;
 }
 
 OpFoldResult ExtractStridedMetadataOp::getConstifiedMixedOffset() {
   OpFoldResult offsetOfr = getAsOpFoldResult(getOffset());
   SmallVector<OpFoldResult> values(1, offsetOfr);
-  constifyIndexValues(values, getSource().getType(), getContext(),
-                      getConstantOffset, ShapedType::isDynamic);
+  SmallVector<int64_t> staticValues, unused;
+  int64_t offset;
+  LogicalResult status =
+      getSource().getType().getStridesAndOffset(unused, offset);
+  (void)status;
+  assert(succeeded(status) && "could not get offset from type");
+  staticValues.push_back(offset);
+  constifyIndexValues(values, staticValues);
   return values[0];
 }
 
@@ -1975,15 +1914,18 @@ OpFoldResult ReinterpretCastOp::fold(FoldAdaptor /*operands*/) {
 
 SmallVector<OpFoldResult> ReinterpretCastOp::getConstifiedMixedSizes() {
   SmallVector<OpFoldResult> values = getMixedSizes();
-  constifyIndexValues(values, getType(), getContext(), getConstantSizes,
-                      ShapedType::isDynamic);
+  constifyIndexValues(values, getType().getShape());
   return values;
 }
 
 SmallVector<OpFoldResult> ReinterpretCastOp::getConstifiedMixedStrides() {
   SmallVector<OpFoldResult> values = getMixedStrides();
-  constifyIndexValues(values, getType(), getContext(), getConstantStrides,
-                      ShapedType::isDynamic);
+  SmallVector<int64_t> staticValues;
+  int64_t unused;
+  LogicalResult status = getType().getStridesAndOffset(staticValues, unused);
+  (void)status;
+  assert(succeeded(status) && "could not get strides from type");
+  constifyIndexValues(values, staticValues);
   return values;
 }
 
@@ -1991,8 +1933,13 @@ OpFoldResult ReinterpretCastOp::getConstifiedMixedOffset() {
   SmallVector<OpFoldResult> values = getMixedOffsets();
   assert(values.size() == 1 &&
          "reinterpret_cast must have one and only one offset");
-  constifyIndexValues(values, getType(), getContext(), getConstantOffset,
-                      ShapedType::isDynamic);
+  SmallVector<int64_t> staticValues, unused;
+  int64_t offset;
+  LogicalResult status = getType().getStridesAndOffset(unused, offset);
+  (void)status;
+  assert(succeeded(status) && "could not get offset from type");
+  staticValues.push_back(offset);
+  constifyIndexValues(values, staticValues);
   return values[0];
 }
 

From 34f9ddf1ce2d775d8df1ca9c9806710f3ba8361d Mon Sep 17 00:00:00 2001
From: Ricardo Jesus <rjj@nvidia.com>
Date: Thu, 17 Apr 2025 08:41:17 +0100
Subject: [PATCH 231/710] [AArch64][SVE] Fold ADD+CNTB to INCB/DECB (#118280)

Currently, given:
```cpp
uint64_t incb(uint64_t x) {
  return x+svcntb();
}
```
LLVM generates:
```gas
incb:
        addvl   x0, x0, #1
        ret
```
Which is equivalent to:
```gas
incb:
        incb    x0
        ret
```

However, on microarchitectures like the Neoverse V2 and Neoverse V3,
the second form (with INCB) can have significantly better latency and
throughput (according to their SWOG). On the Neoverse V2, for example,
ADDVL has a latency and throughput of 2, whereas some forms of INCB
have a latency of 1 and a throughput of 4. The same applies to DECB.
This patch adds patterns to prefer the cheaper INCB/DECB forms over
ADDVL where applicable.
---
 llvm/lib/Target/AArch64/AArch64Features.td    |   5 +
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   2 +
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  23 +++
 .../CodeGen/AArch64/sme-framelower-use-bp.ll  |   6 +-
 llvm/test/CodeGen/AArch64/sve-lsrchain.ll     |   2 +-
 llvm/test/CodeGen/AArch64/sve-vl-arith.ll     | 134 +++++++++++++++++-
 .../AArch64/sve2p1-intrinsics-ld1-single.ll   |   4 +-
 .../AArch64/sve2p1-intrinsics-st1-single.ll   |   4 +-
 .../AArch64/vscale-fixups.ll                  |  65 ++++-----
 9 files changed, 203 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index f4f931a5cdab1..469c76752c78c 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -820,6 +820,11 @@ def FeatureUseFixedOverScalableIfEqualCost : SubtargetFeature<"use-fixed-over-sc
 def FeatureAvoidLDAPUR : SubtargetFeature<"avoid-ldapur", "AvoidLDAPUR", "true",
   "Prefer add+ldapr to offset ldapur">;
 
+// Some INC/DEC forms have better latency and throughput than ADDVL.
+def FeatureDisableFastIncVL : SubtargetFeature<"disable-fast-inc-vl",
+                                               "HasDisableFastIncVL", "true",
+                                               "Do not prefer INC/DEC, ALL, { 1, 2, 4 } over ADDVL">;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index a7a01ed785afa..083b9410f7c11 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -389,6 +389,8 @@ def UseScalarIncVL : Predicate<"Subtarget->useScalarIncVL()">;
 
 def NoUseScalarIncVL : Predicate<"!Subtarget->useScalarIncVL()">;
 
+def HasFastIncVL : Predicate<"!Subtarget->hasDisableFastIncVL()">;
+
 def UseSVEFPLD1R : Predicate<"!Subtarget->noSVEFPLD1R()">;
 
 def UseLDAPUR : Predicate<"!Subtarget->avoidLDAPUR()">;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index b40c82a25e7ba..4830aebc2739d 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2677,6 +2677,29 @@ let Predicates = [HasSVE_or_SME] in {
               (DECD_ZPiI ZPR:$op, 31, $imm)>;
   }
 
+  // Some INCB/DECB forms have better latency and throughput than ADDVL, so we
+  // prefer using them here.
+  // We could extend this to other INC/DEC (scalar) instructions.
+  let Predicates = [HasSVE_or_SME, UseScalarIncVL, HasFastIncVL], AddedComplexity = 6 in {
+    foreach imm = [ 1, 2, 4 ] in {
+      def : Pat<(add GPR64:$op, (vscale !mul(imm, 16))),
+                (INCB_XPiI GPR64:$op, 31, imm)>;
+
+      def : Pat<(add GPR32:$op, (i32 (trunc (vscale !mul(imm, 16))))),
+                (EXTRACT_SUBREG (INCB_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
+                                            GPR32:$op, sub_32), 31, imm),
+                                 sub_32)>;
+
+      def : Pat<(add GPR64:$op, (vscale !mul(imm, -16))),
+                (DECB_XPiI GPR64:$op, 31, imm)>;
+
+      def : Pat<(add GPR32:$op, (i32 (trunc (vscale !mul(imm, -16))))),
+                (EXTRACT_SUBREG (DECB_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
+                                            GPR32:$op, sub_32), 31, imm),
+                                 sub_32)>;
+      }
+  }
+
   let Predicates = [HasSVE_or_SME, UseScalarIncVL], AddedComplexity = 5 in {
     def : Pat<(add GPR64:$op, (vscale (sve_rdvl_imm i32:$imm))),
               (ADDVL_XXI GPR64:$op, $imm)>;
diff --git a/llvm/test/CodeGen/AArch64/sme-framelower-use-bp.ll b/llvm/test/CodeGen/AArch64/sme-framelower-use-bp.ll
index f49bb910b5bd1..99c65b090adb0 100644
--- a/llvm/test/CodeGen/AArch64/sme-framelower-use-bp.ll
+++ b/llvm/test/CodeGen/AArch64/sme-framelower-use-bp.ll
@@ -65,7 +65,8 @@ define void @quux() #1 {
 ; CHECK-NEXT:    mov sp, x9
 ; CHECK-NEXT:    sub x10, x29, #104
 ; CHECK-NEXT:    stur x9, [x10, #-256] // 8-byte Folded Spill
-; CHECK-NEXT:    addvl x9, x8, #1
+; CHECK-NEXT:    mov x9, x8
+; CHECK-NEXT:    incb x9
 ; CHECK-NEXT:    mov w0, w9
 ; CHECK-NEXT:    // implicit-def: $x9
 ; CHECK-NEXT:    mov w9, w0
@@ -160,7 +161,8 @@ define void @quux() #1 {
 ; CHECK-NEXT:    mov x9, sp
 ; CHECK-NEXT:    subs x9, x9, #16
 ; CHECK-NEXT:    mov sp, x9
-; CHECK-NEXT:    addvl x9, x8, #2
+; CHECK-NEXT:    mov x9, x8
+; CHECK-NEXT:    incb x9, all, mul #2
 ; CHECK-NEXT:    mov w0, w9
 ; CHECK-NEXT:    // implicit-def: $x9
 ; CHECK-NEXT:    mov w9, w0
diff --git a/llvm/test/CodeGen/AArch64/sve-lsrchain.ll b/llvm/test/CodeGen/AArch64/sve-lsrchain.ll
index 78f93f1ecbb26..d94fa6433bb7f 100644
--- a/llvm/test/CodeGen/AArch64/sve-lsrchain.ll
+++ b/llvm/test/CodeGen/AArch64/sve-lsrchain.ll
@@ -85,7 +85,7 @@ define void @test(ptr nocapture noundef readonly %kernel, i32 noundef %kw, float
 ; CHECK-NEXT:    ldr z5, [x4, #3, mul vl]
 ; CHECK-NEXT:    fmla z4.h, p0/m, z5.h, z3.h
 ; CHECK-NEXT:    str z4, [x16, #3, mul vl]
-; CHECK-NEXT:    addvl x16, x16, #4
+; CHECK-NEXT:    incb x16, all, mul #4
 ; CHECK-NEXT:    cmp x16, x11
 ; CHECK-NEXT:    b.lo .LBB0_4
 ; CHECK-NEXT:  // %bb.5: // %while.cond.i..exit_crit_edge.us
diff --git a/llvm/test/CodeGen/AArch64/sve-vl-arith.ll b/llvm/test/CodeGen/AArch64/sve-vl-arith.ll
index a6c0e5aa70583..0a9f4948a5f77 100644
--- a/llvm/test/CodeGen/AArch64/sve-vl-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vl-arith.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -verify-machineinstrs < %s | FileCheck %s -check-prefix=NO_SCALAR_INC
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -sve-use-scalar-inc-vl=true -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,disable-fast-inc-vl -sve-use-scalar-inc-vl=true -verify-machineinstrs < %s | FileCheck %s -check-prefix=NO_FAST_INC
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -sve-use-scalar-inc-vl=false -verify-machineinstrs < %s | FileCheck %s -check-prefix=NO_SCALAR_INC
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2,disable-fast-inc-vl -verify-machineinstrs < %s | FileCheck %s -check-prefix=NO_FAST_INC
 
 define <vscale x 8 x i16> @inch_vec(<vscale x 8 x i16> %a) {
 ; NO_SCALAR_INC-LABEL: inch_vec:
@@ -14,6 +16,11 @@ define <vscale x 8 x i16> @inch_vec(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    inch z0.h
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: inch_vec:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    inch z0.h
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i16 @llvm.vscale.i16()
   %mul = mul i16 %vscale, 8
   %vl = insertelement <vscale x 8 x i16> poison, i16 %mul, i32 0
@@ -32,6 +39,11 @@ define <vscale x 4 x i32> @incw_vec(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    incw z0.s
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: incw_vec:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    incw z0.s
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i32 @llvm.vscale.i32()
   %mul = mul i32 %vscale, 4
   %vl = insertelement <vscale x 4 x i32> poison, i32 %mul, i32 0
@@ -50,6 +62,11 @@ define <vscale x 2 x i64> @incd_vec(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    incd z0.d
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: incd_vec:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    incd z0.d
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 2
   %vl = insertelement <vscale x 2 x i64> poison, i64 %mul, i32 0
@@ -68,6 +85,11 @@ define <vscale x 8 x i16> @dech_vec(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    dech z0.h, all, mul #2
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: dech_vec:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    dech z0.h, all, mul #2
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i16 @llvm.vscale.i16()
   %mul = mul i16 %vscale, 16
   %vl = insertelement <vscale x 8 x i16> poison, i16 %mul, i32 0
@@ -86,6 +108,11 @@ define <vscale x 4 x i32> @decw_vec(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    decw z0.s, all, mul #4
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: decw_vec:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    decw z0.s, all, mul #4
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i32 @llvm.vscale.i32()
   %mul = mul i32 %vscale, 16
   %vl = insertelement <vscale x 4 x i32> poison, i32 %mul, i32 0
@@ -104,6 +131,11 @@ define <vscale x 2 x i64> @decd_vec(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    decd z0.d, all, mul #8
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: decd_vec:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    decd z0.d, all, mul #8
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 16
   %vl = insertelement <vscale x 2 x i64> poison, i64 %mul, i32 0
@@ -123,8 +155,13 @@ define i64 @incb_scalar_i64(i64 %a) {
 ;
 ; CHECK-LABEL: incb_scalar_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    addvl x0, x0, #1
+; CHECK-NEXT:    incb x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: incb_scalar_i64:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    addvl x0, x0, #1
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 16
   %add = add i64 %a, %mul
@@ -142,6 +179,11 @@ define i64 @inch_scalar_i64(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    inch x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: inch_scalar_i64:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    inch x0
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 8
   %add = add i64 %a, %mul
@@ -159,6 +201,11 @@ define i64 @incw_scalar_i64(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    incw x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: incw_scalar_i64:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    incw x0
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 4
   %add = add i64 %a, %mul
@@ -176,6 +223,11 @@ define i64 @incd_scalar_i64(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    incd x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: incd_scalar_i64:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    incd x0
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 2
   %add = add i64 %a, %mul
@@ -193,8 +245,13 @@ define i64 @decb_scalar_i64(i64 %a) {
 ;
 ; CHECK-LABEL: decb_scalar_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    addvl x0, x0, #-2
+; CHECK-NEXT:    decb x0, all, mul #2
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: decb_scalar_i64:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    addvl x0, x0, #-2
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 32
   %sub = sub i64 %a, %mul
@@ -212,6 +269,11 @@ define i64 @dech_scalar_i64(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    dech x0, all, mul #3
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: dech_scalar_i64:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    dech x0, all, mul #3
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 24
   %sub = sub i64 %a, %mul
@@ -229,6 +291,11 @@ define i64 @decw_scalar_i64(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    decw x0, all, mul #3
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: decw_scalar_i64:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    decw x0, all, mul #3
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 12
   %sub = sub i64 %a, %mul
@@ -246,6 +313,11 @@ define i64 @decd_scalar_i64(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    decd x0, all, mul #3
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: decd_scalar_i64:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    decd x0, all, mul #3
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 6
   %sub = sub i64 %a, %mul
@@ -267,6 +339,13 @@ define i32 @incb_scalar_i32(i32 %a) {
 ; CHECK-NEXT:    addvl x0, x0, #3
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: incb_scalar_i32:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 def $x0
+; NO_FAST_INC-NEXT:    addvl x0, x0, #3
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; NO_FAST_INC-NEXT:    ret
 
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 48
@@ -288,6 +367,13 @@ define i32 @inch_scalar_i32(i32 %a) {
 ; CHECK-NEXT:    inch x0, all, mul #7
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: inch_scalar_i32:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 def $x0
+; NO_FAST_INC-NEXT:    inch x0, all, mul #7
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; NO_FAST_INC-NEXT:    ret
 
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 56
@@ -309,6 +395,13 @@ define i32 @incw_scalar_i32(i32 %a) {
 ; CHECK-NEXT:    incw x0, all, mul #7
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: incw_scalar_i32:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 def $x0
+; NO_FAST_INC-NEXT:    incw x0, all, mul #7
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; NO_FAST_INC-NEXT:    ret
 
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 28
@@ -330,6 +423,13 @@ define i32 @incd_scalar_i32(i32 %a) {
 ; CHECK-NEXT:    incd x0, all, mul #7
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: incd_scalar_i32:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 def $x0
+; NO_FAST_INC-NEXT:    incd x0, all, mul #7
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; NO_FAST_INC-NEXT:    ret
 
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 14
@@ -350,9 +450,16 @@ define i32 @decb_scalar_i32(i32 %a) {
 ; CHECK-LABEL: decb_scalar_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    addvl x0, x0, #-4
+; CHECK-NEXT:    decb x0, all, mul #4
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: decb_scalar_i32:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 def $x0
+; NO_FAST_INC-NEXT:    addvl x0, x0, #-4
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; NO_FAST_INC-NEXT:    ret
 
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 64
@@ -374,6 +481,13 @@ define i32 @dech_scalar_i32(i32 %a) {
 ; CHECK-NEXT:    dech x0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: dech_scalar_i32:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 def $x0
+; NO_FAST_INC-NEXT:    dech x0
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; NO_FAST_INC-NEXT:    ret
 
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 8
@@ -395,6 +509,13 @@ define i32 @decw_scalar_i32(i32 %a) {
 ; CHECK-NEXT:    decw x0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: decw_scalar_i32:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 def $x0
+; NO_FAST_INC-NEXT:    decw x0
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; NO_FAST_INC-NEXT:    ret
 
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 4
@@ -416,6 +537,13 @@ define i32 @decd_scalar_i32(i32 %a) {
 ; CHECK-NEXT:    decd x0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; NO_FAST_INC-LABEL: decd_scalar_i32:
+; NO_FAST_INC:       // %bb.0:
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 def $x0
+; NO_FAST_INC-NEXT:    decd x0
+; NO_FAST_INC-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; NO_FAST_INC-NEXT:    ret
   %vscale = call i64 @llvm.vscale.i64()
   %mul = mul i64 %vscale, 2
   %vl = trunc i64 %mul to i32
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-ld1-single.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-ld1-single.ll
index 39ee4510d51b4..abb5f2aa07fc8 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-ld1-single.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-ld1-single.ll
@@ -33,8 +33,8 @@ define <vscale x 4 x i32> @test_svld1uwq_i32_si(<vscale x 1 x i1> %pred, ptr %ba
 define <vscale x 4 x i32> @test_svld1uwq_i32_out_of_bound(<vscale x 1 x i1> %pred, ptr %base) {
 ; CHECK-LABEL: test_svld1uwq_i32_out_of_bound:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    addvl x8, x0, #2
-; CHECK-NEXT:    ld1w { z0.q }, p0/z, [x8]
+; CHECK-NEXT:    incb x0, all, mul #2
+; CHECK-NEXT:    ld1w { z0.q }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %gep = getelementptr inbounds <vscale x 1 x i32>, ptr %base, i64 8
   %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1uwq.nxv4i32(<vscale x 1 x i1> %pred, ptr %gep)
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-st1-single.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-st1-single.ll
index 4ffc0b42d0711..d4c77328be478 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-st1-single.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-st1-single.ll
@@ -30,8 +30,8 @@ define void @test_svst1wq_i32_si(<vscale x 4 x i32> %zt, <vscale x 1 x i1> %pred
 define void @test_svst1wq_i32_out_of_bound(<vscale x 4 x i32> %zt, <vscale x 1 x i1> %pred, ptr %base) {
 ; CHECK-LABEL: test_svst1wq_i32_out_of_bound:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    addvl x8, x0, #2
-; CHECK-NEXT:    st1w { z0.q }, p0, [x8]
+; CHECK-NEXT:    incb x0, all, mul #2
+; CHECK-NEXT:    st1w { z0.q }, p0, [x0]
 ; CHECK-NEXT:    ret
   %gep = getelementptr inbounds <vscale x 1 x i32>, ptr %base, i64 8
   call void @llvm.aarch64.sve.st1wq.nxv4i32(<vscale x 4 x i32> %zt, <vscale x 1 x i1> %pred, ptr %gep)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
index bae69ef590f52..aa954aeb0ad07 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
@@ -23,7 +23,7 @@ define void @mulvl123_addressing(ptr %src, ptr %dst, i64 %count) #0 {
 ; COMMON-NEXT:    umax z1.b, p0/m, z1.b, z3.b
 ; COMMON-NEXT:    umax z0.b, p0/m, z0.b, z1.b
 ; COMMON-NEXT:    st1b { z0.b }, p0, [x1, x8]
-; COMMON-NEXT:    addvl x8, x8, #1
+; COMMON-NEXT:    incb x8
 ; COMMON-NEXT:    cmp x8, x2
 ; COMMON-NEXT:    b.lo .LBB0_1
 ; COMMON-NEXT:  // %bb.2: // %for.exit
@@ -71,13 +71,13 @@ define void @many_mulvl1_addressing(ptr %src_rows, ptr %dst_rows, i64 %stride, i
 ; COMMON-NEXT:    ld1b { z1.b }, p0/z, [x0, x2]
 ; COMMON-NEXT:    ldr z2, [x0, #1, mul vl]
 ; COMMON-NEXT:    ldr z3, [x8, #1, mul vl]
+; COMMON-NEXT:    incb x0, all, mul #2
 ; COMMON-NEXT:    subs x3, x3, #1
-; COMMON-NEXT:    addvl x0, x0, #2
 ; COMMON-NEXT:    add z0.b, z0.b, z1.b
 ; COMMON-NEXT:    add z1.b, z2.b, z3.b
 ; COMMON-NEXT:    st1b { z0.h }, p1, [x1]
 ; COMMON-NEXT:    st1b { z1.h }, p1, [x1, #1, mul vl]
-; COMMON-NEXT:    addvl x1, x1, #2
+; COMMON-NEXT:    incb x1, all, mul #2
 ; COMMON-NEXT:    b.ne .LBB1_1
 ; COMMON-NEXT:  // %bb.2: // %for.exit
 ; COMMON-NEXT:    ret
@@ -156,56 +156,56 @@ for.exit:
 define void @mixed_offsets_scalable_then_fixed(ptr %src, ptr %dst, i64 %count) #0 {
 ; BASE-LABEL: mixed_offsets_scalable_then_fixed:
 ; BASE:       // %bb.0: // %entry
+; BASE-NEXT:    incb x0, all, mul #4
 ; BASE-NEXT:    ptrue p0.s
-; BASE-NEXT:    addvl x8, x0, #4
-; BASE-NEXT:    mov x9, #8 // =0x8
+; BASE-NEXT:    mov x8, #8 // =0x8
 ; BASE-NEXT:  .LBB3_1: // %for.body
 ; BASE-NEXT:    // =>This Inner Loop Header: Depth=1
-; BASE-NEXT:    ldr z0, [x8, #-4, mul vl]
-; BASE-NEXT:    ldr z1, [x8]
+; BASE-NEXT:    ldr z0, [x0, #-4, mul vl]
+; BASE-NEXT:    ldr z1, [x0]
 ; BASE-NEXT:    decw x2
-; BASE-NEXT:    ld1w { z2.s }, p0/z, [x8, x9, lsl #2]
-; BASE-NEXT:    addvl x8, x8, #1
+; BASE-NEXT:    ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; BASE-NEXT:    incb x0
 ; BASE-NEXT:    add z0.s, z0.s, z1.s
 ; BASE-NEXT:    add z0.s, z0.s, z2.s
 ; BASE-NEXT:    str z0, [x1]
-; BASE-NEXT:    addvl x1, x1, #1
+; BASE-NEXT:    incb x1
 ; BASE-NEXT:    cbnz x2, .LBB3_1
 ; BASE-NEXT:  // %bb.2: // %for.exit
 ; BASE-NEXT:    ret
 ;
 ; PREINDEX-LABEL: mixed_offsets_scalable_then_fixed:
 ; PREINDEX:       // %bb.0: // %entry
+; PREINDEX-NEXT:    incb x0, all, mul #4
 ; PREINDEX-NEXT:    ptrue p0.s
-; PREINDEX-NEXT:    addvl x8, x0, #4
-; PREINDEX-NEXT:    mov x9, #8 // =0x8
+; PREINDEX-NEXT:    mov x8, #8 // =0x8
 ; PREINDEX-NEXT:  .LBB3_1: // %for.body
 ; PREINDEX-NEXT:    // =>This Inner Loop Header: Depth=1
-; PREINDEX-NEXT:    ldr z0, [x8, #-4, mul vl]
-; PREINDEX-NEXT:    ldr z1, [x8]
+; PREINDEX-NEXT:    ldr z0, [x0, #-4, mul vl]
+; PREINDEX-NEXT:    ldr z1, [x0]
 ; PREINDEX-NEXT:    decw x2
-; PREINDEX-NEXT:    ld1w { z2.s }, p0/z, [x8, x9, lsl #2]
-; PREINDEX-NEXT:    addvl x8, x8, #1
+; PREINDEX-NEXT:    ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; PREINDEX-NEXT:    incb x0
 ; PREINDEX-NEXT:    add z0.s, z0.s, z1.s
 ; PREINDEX-NEXT:    add z0.s, z0.s, z2.s
 ; PREINDEX-NEXT:    str z0, [x1]
-; PREINDEX-NEXT:    addvl x1, x1, #1
+; PREINDEX-NEXT:    incb x1
 ; PREINDEX-NEXT:    cbnz x2, .LBB3_1
 ; PREINDEX-NEXT:  // %bb.2: // %for.exit
 ; PREINDEX-NEXT:    ret
 ;
 ; POSTINDEX-LABEL: mixed_offsets_scalable_then_fixed:
 ; POSTINDEX:       // %bb.0: // %entry
+; POSTINDEX-NEXT:    incb x0, all, mul #4
 ; POSTINDEX-NEXT:    ptrue p0.s
 ; POSTINDEX-NEXT:    mov x8, xzr
-; POSTINDEX-NEXT:    addvl x9, x0, #4
-; POSTINDEX-NEXT:    mov x10, #8 // =0x8
+; POSTINDEX-NEXT:    mov x9, #8 // =0x8
 ; POSTINDEX-NEXT:  .LBB3_1: // %for.body
 ; POSTINDEX-NEXT:    // =>This Inner Loop Header: Depth=1
-; POSTINDEX-NEXT:    ldr z0, [x9, #-4, mul vl]
-; POSTINDEX-NEXT:    ldr z1, [x9]
-; POSTINDEX-NEXT:    ld1w { z2.s }, p0/z, [x9, x10, lsl #2]
-; POSTINDEX-NEXT:    addvl x9, x9, #1
+; POSTINDEX-NEXT:    ldr z0, [x0, #-4, mul vl]
+; POSTINDEX-NEXT:    ldr z1, [x0]
+; POSTINDEX-NEXT:    ld1w { z2.s }, p0/z, [x0, x9, lsl #2]
+; POSTINDEX-NEXT:    incb x0
 ; POSTINDEX-NEXT:    add z0.s, z0.s, z1.s
 ; POSTINDEX-NEXT:    add z0.s, z0.s, z2.s
 ; POSTINDEX-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
@@ -243,11 +243,12 @@ for.exit:
 define void @mixed_offsets_fixed_then_scalable(ptr %src, ptr %dst, i64 %count) #0 {
 ; COMMON-LABEL: mixed_offsets_fixed_then_scalable:
 ; COMMON:       // %bb.0: // %entry
-; COMMON-NEXT:    addvl x9, x0, #4
+; COMMON-NEXT:    mov x9, x0
 ; COMMON-NEXT:    ptrue p0.s
 ; COMMON-NEXT:    mov x8, xzr
-; COMMON-NEXT:    add x9, x9, #32
+; COMMON-NEXT:    incb x9, all, mul #4
 ; COMMON-NEXT:    mov x10, #8 // =0x8
+; COMMON-NEXT:    add x9, x9, #32
 ; COMMON-NEXT:  .LBB4_1: // %for.body
 ; COMMON-NEXT:    // =>This Inner Loop Header: Depth=1
 ; COMMON-NEXT:    add x11, x0, x8, lsl #2
@@ -304,11 +305,11 @@ define void @three_access_wide_gap(ptr %src, ptr %dst, i64 %count) #0 {
 ; BASE-NEXT:    ldr z1, [x0, #4, mul vl]
 ; BASE-NEXT:    decw x2
 ; BASE-NEXT:    ldr z2, [x0, #8, mul vl]
-; BASE-NEXT:    addvl x0, x0, #1
+; BASE-NEXT:    incb x0
 ; BASE-NEXT:    add z0.s, z0.s, z1.s
 ; BASE-NEXT:    add z0.s, z0.s, z2.s
 ; BASE-NEXT:    str z0, [x1]
-; BASE-NEXT:    addvl x1, x1, #1
+; BASE-NEXT:    incb x1
 ; BASE-NEXT:    cbnz x2, .LBB5_1
 ; BASE-NEXT:  // %bb.2: // %for.exit
 ; BASE-NEXT:    ret
@@ -321,11 +322,11 @@ define void @three_access_wide_gap(ptr %src, ptr %dst, i64 %count) #0 {
 ; PREINDEX-NEXT:    ldr z1, [x0, #4, mul vl]
 ; PREINDEX-NEXT:    decw x2
 ; PREINDEX-NEXT:    ldr z2, [x0, #8, mul vl]
-; PREINDEX-NEXT:    addvl x0, x0, #1
+; PREINDEX-NEXT:    incb x0
 ; PREINDEX-NEXT:    add z0.s, z0.s, z1.s
 ; PREINDEX-NEXT:    add z0.s, z0.s, z2.s
 ; PREINDEX-NEXT:    str z0, [x1]
-; PREINDEX-NEXT:    addvl x1, x1, #1
+; PREINDEX-NEXT:    incb x1
 ; PREINDEX-NEXT:    cbnz x2, .LBB5_1
 ; PREINDEX-NEXT:  // %bb.2: // %for.exit
 ; PREINDEX-NEXT:    ret
@@ -339,7 +340,7 @@ define void @three_access_wide_gap(ptr %src, ptr %dst, i64 %count) #0 {
 ; POSTINDEX-NEXT:    ldr z0, [x0]
 ; POSTINDEX-NEXT:    ldr z1, [x0, #4, mul vl]
 ; POSTINDEX-NEXT:    ldr z2, [x0, #8, mul vl]
-; POSTINDEX-NEXT:    addvl x0, x0, #1
+; POSTINDEX-NEXT:    incb x0
 ; POSTINDEX-NEXT:    add z0.s, z0.s, z1.s
 ; POSTINDEX-NEXT:    add z0.s, z0.s, z2.s
 ; POSTINDEX-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
@@ -396,9 +397,9 @@ define void @vscale_squared_offset(ptr %alloc) #0 {
 ; COMMON-NEXT:    // =>This Inner Loop Header: Depth=1
 ; COMMON-NEXT:    add x11, x0, x9
 ; COMMON-NEXT:    st1w { z0.s }, p0, [x0]
-; COMMON-NEXT:    add x8, x8, #1
+; COMMON-NEXT:    incb x0
 ; COMMON-NEXT:    st1w { z1.s }, p0, [x11]
-; COMMON-NEXT:    addvl x0, x0, #1
+; COMMON-NEXT:    add x8, x8, #1
 ; COMMON-NEXT:    cmp x8, x10
 ; COMMON-NEXT:    b.lt .LBB6_1
 ; COMMON-NEXT:  .LBB6_2: // %for.exit

From 1cf9acdb750612985674a0349ed607768e1edc6d Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Thu, 17 Apr 2025 08:44:47 +0100
Subject: [PATCH 232/710] [TLI] Use AArch64 vector calling convention for ArmPL
 routines (#135790)

Some ArmPL vector math routines that operate on fixed-width
vector types obey the aarch64_vector_pcs calling convention,
e.g. armpl_vsinq_f64, etc. This patch adds support for
functions in the TLI to specify an optional calling
convention to be added to the declared function. Once this
patch lands I intend to follow up with a vectoriser patch
that adds this optional calling convention when invoking the
vector library variant from the loop vectoriser.
---
 .../include/llvm/Analysis/TargetLibraryInfo.h |   7 +-
 llvm/include/llvm/Analysis/VecFuncs.def       | 601 +++++++++---------
 llvm/lib/Analysis/TargetLibraryInfo.cpp       |  14 +-
 .../Transforms/Utils/InjectTLIMappings.cpp    |   2 +
 llvm/test/Transforms/Util/add-TLI-mappings.ll |  16 +-
 .../Analysis/ReplaceWithVecLibTest.cpp        |   6 +-
 6 files changed, 326 insertions(+), 320 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
index f51d2bb9d50a2..33f2ab4fa90b0 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
@@ -45,20 +45,23 @@ class VecDesc {
   ElementCount VectorizationFactor;
   bool Masked;
   StringRef VABIPrefix;
+  std::optional<CallingConv::ID> CC;
 
 public:
   VecDesc() = delete;
   VecDesc(StringRef ScalarFnName, StringRef VectorFnName,
-          ElementCount VectorizationFactor, bool Masked, StringRef VABIPrefix)
+          ElementCount VectorizationFactor, bool Masked, StringRef VABIPrefix,
+          std::optional<CallingConv::ID> Conv)
       : ScalarFnName(ScalarFnName), VectorFnName(VectorFnName),
         VectorizationFactor(VectorizationFactor), Masked(Masked),
-        VABIPrefix(VABIPrefix) {}
+        VABIPrefix(VABIPrefix), CC(Conv) {}
 
   StringRef getScalarFnName() const { return ScalarFnName; }
   StringRef getVectorFnName() const { return VectorFnName; }
   ElementCount getVectorizationFactor() const { return VectorizationFactor; }
   bool isMasked() const { return Masked; }
   StringRef getVABIPrefix() const { return VABIPrefix; }
+  std::optional<CallingConv::ID> getCallingConv() const { return CC; }
 
   /// Returns a vector function ABI variant string on the form:
   ///    _ZGV<isa><mask><vlen><vparams>_<scalarname>(<vectorname>)
diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def
index 251331b9f860c..b9f7a38943b05 100644
--- a/llvm/include/llvm/Analysis/VecFuncs.def
+++ b/llvm/include/llvm/Analysis/VecFuncs.def
@@ -16,9 +16,10 @@
 #define SCALABLE(NL) ElementCount::getScalable(NL)
 #define NOMASK false
 #define MASKED true
+#define NOCC std::nullopt
 
 #if !(defined(TLI_DEFINE_VECFUNC))
-#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) {SCAL, VEC, VF, NOMASK, VABI_PREFIX},
+#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) {SCAL, VEC, VF, NOMASK, VABI_PREFIX, NOCC},
 #endif
 
 #if defined(TLI_DEFINE_ACCELERATE_VECFUNCS)
@@ -1067,305 +1068,305 @@ TLI_DEFINE_VECFUNC("tgammaf", "Sleef_tgammafx_u10rvvm2", SCALABLE(4), NOMASK, "_
 
 #elif defined(TLI_DEFINE_ARMPL_VECFUNCS)
 
-TLI_DEFINE_VECFUNC("acos", "armpl_vacosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("acosf", "armpl_vacosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("acos", "armpl_svacos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("acosf", "armpl_svacos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.acos.f64", "armpl_vacosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.acos.f32", "armpl_vacosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.acos.f64", "armpl_svacos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.acos.f32", "armpl_svacos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("acosh", "armpl_vacoshq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("acoshf", "armpl_vacoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("acosh", "armpl_svacosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("acoshf", "armpl_svacosh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("asin", "armpl_vasinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("asinf", "armpl_vasinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("asin", "armpl_svasin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("asinf", "armpl_svasin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.asin.f64", "armpl_vasinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.asin.f32", "armpl_vasinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.asin.f64", "armpl_svasin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.asin.f32", "armpl_svasin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("asinh", "armpl_vasinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("asinhf", "armpl_vasinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("asinh", "armpl_svasinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("asinhf", "armpl_svasinh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("atan", "armpl_vatanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("atanf", "armpl_vatanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("atan", "armpl_svatan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("atanf", "armpl_svatan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.atan.f64", "armpl_vatanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.atan.f32", "armpl_vatanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.atan.f64", "armpl_svatan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.atan.f32", "armpl_svatan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("atan2", "armpl_vatan2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("atan2f", "armpl_vatan2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("atan2", "armpl_svatan2_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("atan2f", "armpl_svatan2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("llvm.atan2.f64", "armpl_vatan2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("llvm.atan2.f32", "armpl_vatan2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("llvm.atan2.f64", "armpl_svatan2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("llvm.atan2.f32", "armpl_svatan2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("atanh", "armpl_vatanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("atanhf", "armpl_vatanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("atanh", "armpl_svatanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("atanhf", "armpl_svatanh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("cbrt", "armpl_vcbrtq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("cbrtf", "armpl_vcbrtq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("cbrt", "armpl_svcbrt_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("cbrtf", "armpl_svcbrt_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("copysign", "armpl_vcopysignq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("copysignf", "armpl_vcopysignq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("copysign", "armpl_svcopysign_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("copysignf", "armpl_svcopysign_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("cos", "armpl_vcosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("cosf", "armpl_vcosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("cos", "armpl_svcos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("cosf", "armpl_svcos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.cos.f64", "armpl_vcosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.cos.f32", "armpl_vcosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.cos.f64", "armpl_svcos_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.cos.f32", "armpl_svcos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("cosh", "armpl_vcoshq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("coshf", "armpl_vcoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("cosh", "armpl_svcosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("coshf", "armpl_svcosh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.cosh.f64", "armpl_vcoshq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.cosh.f32", "armpl_vcoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.cosh.f64", "armpl_svcosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.cosh.f32", "armpl_svcosh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("cospi", "armpl_vcospiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("cospif", "armpl_vcospiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("cospi", "armpl_svcospi_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("cospif", "armpl_svcospi_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("erf", "armpl_verfq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("erff", "armpl_verfq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("erf", "armpl_sverf_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("erff", "armpl_sverf_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("erfc", "armpl_verfcq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("erfcf", "armpl_verfcq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("erfc", "armpl_sverfc_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("erfcf", "armpl_sverfc_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("exp", "armpl_vexpq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("expf", "armpl_vexpq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("exp", "armpl_svexp_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("expf", "armpl_svexp_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.exp.f64", "armpl_vexpq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.exp.f32", "armpl_vexpq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.exp.f64", "armpl_svexp_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.exp.f32", "armpl_svexp_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("exp10", "armpl_vexp10q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("exp10f", "armpl_vexp10q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("exp10", "armpl_svexp10_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("exp10f", "armpl_svexp10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.exp10.f64", "armpl_vexp10q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.exp10.f32", "armpl_vexp10q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.exp10.f64", "armpl_svexp10_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.exp10.f32", "armpl_svexp10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("exp2", "armpl_vexp2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("exp2f", "armpl_vexp2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("exp2", "armpl_svexp2_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("exp2f", "armpl_svexp2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.exp2.f64", "armpl_vexp2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.exp2.f32", "armpl_vexp2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.exp2.f64", "armpl_svexp2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.exp2.f32", "armpl_svexp2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("expm1", "armpl_vexpm1q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("expm1f", "armpl_vexpm1q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("expm1", "armpl_svexpm1_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("expm1f", "armpl_svexpm1_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("fdim", "armpl_vfdimq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("fdimf", "armpl_vfdimq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("fdim", "armpl_svfdim_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("fdimf", "armpl_svfdim_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("fma", "armpl_vfmaq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vvv")
-TLI_DEFINE_VECFUNC("fmaf", "armpl_vfmaq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vvv")
-TLI_DEFINE_VECFUNC("fma", "armpl_svfma_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvvv")
-TLI_DEFINE_VECFUNC("fmaf", "armpl_svfma_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvvv")
-
-TLI_DEFINE_VECFUNC("fmax", "armpl_vfmaxq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("fmaxf", "armpl_vfmaxq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("fmax", "armpl_svfmax_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("fmaxf", "armpl_svfmax_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("fmin", "armpl_vfminq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("fminf", "armpl_vfminq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("fmin", "armpl_svfmin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("fminf", "armpl_svfmin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("fmod", "armpl_vfmodq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("fmodf", "armpl_vfmodq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("fmod", "armpl_svfmod_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("fmodf", "armpl_svfmod_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("hypot", "armpl_vhypotq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("hypotf", "armpl_vhypotq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("hypot", "armpl_svhypot_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("hypotf", "armpl_svhypot_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("ilogb", "armpl_vilogbq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("ilogbf", "armpl_vilogbq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("ilogb", "armpl_svilogb_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("ilogbf", "armpl_svilogb_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("ldexp", "armpl_vldexpq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("ldexpf", "armpl_vldexpq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("ldexp", "armpl_svldexp_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("ldexpf", "armpl_svldexp_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("lgamma", "armpl_vlgammaq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("lgammaf", "armpl_vlgammaq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("lgamma", "armpl_svlgamma_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("lgammaf", "armpl_svlgamma_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("log", "armpl_vlogq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("logf", "armpl_vlogq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("log", "armpl_svlog_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("logf", "armpl_svlog_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.log.f64", "armpl_vlogq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.log.f32", "armpl_vlogq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.log.f64", "armpl_svlog_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.log.f32", "armpl_svlog_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("log10", "armpl_vlog10q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("log10f", "armpl_vlog10q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("log10", "armpl_svlog10_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("log10f", "armpl_svlog10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.log10.f64", "armpl_vlog10q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.log10.f32", "armpl_vlog10q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.log10.f64", "armpl_svlog10_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.log10.f32", "armpl_svlog10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("log1p", "armpl_vlog1pq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("log1pf", "armpl_vlog1pq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("log1p", "armpl_svlog1p_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("log1pf", "armpl_svlog1p_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("log2", "armpl_vlog2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("log2f", "armpl_vlog2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("log2", "armpl_svlog2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("log2f", "armpl_svlog2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.log2.f64", "armpl_vlog2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.log2.f32", "armpl_vlog2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.log2.f64", "armpl_svlog2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.log2.f32", "armpl_svlog2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("modf", "armpl_vmodfq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8")
-TLI_DEFINE_VECFUNC("modff", "armpl_vmodfq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4")
-TLI_DEFINE_VECFUNC("modf", "armpl_svmodf_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8")
-TLI_DEFINE_VECFUNC("modff", "armpl_svmodf_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4")
-
-TLI_DEFINE_VECFUNC("nextafter", "armpl_vnextafterq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("nextafterf", "armpl_vnextafterq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("nextafter", "armpl_svnextafter_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("nextafterf", "armpl_svnextafter_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("pow", "armpl_vpowq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("powf", "armpl_vpowq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("pow", "armpl_svpow_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("powf", "armpl_svpow_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("llvm.pow.f64", "armpl_vpowq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv")
-TLI_DEFINE_VECFUNC("llvm.pow.f32", "armpl_vpowq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv")
-TLI_DEFINE_VECFUNC("llvm.pow.f64", "armpl_svpow_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvv")
-TLI_DEFINE_VECFUNC("llvm.pow.f32", "armpl_svpow_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv")
-
-TLI_DEFINE_VECFUNC("sin", "armpl_vsinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("sinf", "armpl_vsinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("sin", "armpl_svsin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("sinf", "armpl_svsin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.sin.f64", "armpl_vsinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.sin.f32", "armpl_vsinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.sin.f64", "armpl_svsin_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.sin.f32", "armpl_svsin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("sincos", "armpl_vsincosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8l8")
-TLI_DEFINE_VECFUNC("sincosf", "armpl_vsincosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4")
-TLI_DEFINE_VECFUNC("sincos", "armpl_svsincos_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8l8")
-TLI_DEFINE_VECFUNC("sincosf", "armpl_svsincos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4l4")
-
-TLI_DEFINE_VECFUNC("sincospi", "armpl_vsincospiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8l8")
-TLI_DEFINE_VECFUNC("sincospif", "armpl_vsincospiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4")
-TLI_DEFINE_VECFUNC("sincospi", "armpl_svsincospi_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8l8")
-TLI_DEFINE_VECFUNC("sincospif", "armpl_svsincospi_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4l4")
-
-TLI_DEFINE_VECFUNC("sinh", "armpl_vsinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("sinhf", "armpl_vsinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("sinh", "armpl_svsinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("sinhf", "armpl_svsinh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.sinh.f64", "armpl_vsinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.sinh.f32", "armpl_vsinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.sinh.f64", "armpl_svsinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.sinh.f32", "armpl_svsinh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("sinpi", "armpl_vsinpiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("sinpif", "armpl_vsinpiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("sinpi", "armpl_svsinpi_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("sinpif", "armpl_svsinpi_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("sqrt", "armpl_vsqrtq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("sqrtf", "armpl_vsqrtq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("sqrt", "armpl_svsqrt_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("sqrtf", "armpl_svsqrt_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("tan", "armpl_vtanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("tanf", "armpl_vtanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("tan", "armpl_svtan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("tanf", "armpl_svtan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.tan.f64", "armpl_vtanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.tan.f32", "armpl_vtanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.tan.f64", "armpl_svtan_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.tan.f32", "armpl_svtan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("tanh", "armpl_vtanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("tanhf", "armpl_vtanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("tanh", "armpl_svtanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("tanhf", "armpl_svtanh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("llvm.tanh.f64", "armpl_vtanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("llvm.tanh.f32", "armpl_vtanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("llvm.tanh.f64", "armpl_svtanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("llvm.tanh.f32", "armpl_svtanh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
-
-TLI_DEFINE_VECFUNC("tgamma", "armpl_vtgammaq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v")
-TLI_DEFINE_VECFUNC("tgammaf", "armpl_vtgammaq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v")
-TLI_DEFINE_VECFUNC("tgamma", "armpl_svtgamma_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv")
-TLI_DEFINE_VECFUNC("tgammaf", "armpl_svtgamma_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv")
+TLI_DEFINE_VECFUNC("acos", "armpl_vacosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("acosf", "armpl_vacosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("acos", "armpl_svacos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("acosf", "armpl_svacos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.acos.f64", "armpl_vacosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.acos.f32", "armpl_vacosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.acos.f64", "armpl_svacos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.acos.f32", "armpl_svacos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("acosh", "armpl_vacoshq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("acoshf", "armpl_vacoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("acosh", "armpl_svacosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("acoshf", "armpl_svacosh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("asin", "armpl_vasinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("asinf", "armpl_vasinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("asin", "armpl_svasin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("asinf", "armpl_svasin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.asin.f64", "armpl_vasinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.asin.f32", "armpl_vasinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.asin.f64", "armpl_svasin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.asin.f32", "armpl_svasin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("asinh", "armpl_vasinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("asinhf", "armpl_vasinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("asinh", "armpl_svasinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("asinhf", "armpl_svasinh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("atan", "armpl_vatanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("atanf", "armpl_vatanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("atan", "armpl_svatan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("atanf", "armpl_svatan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.atan.f64", "armpl_vatanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.atan.f32", "armpl_vatanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.atan.f64", "armpl_svatan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.atan.f32", "armpl_svatan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("atan2", "armpl_vatan2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("atan2f", "armpl_vatan2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("atan2", "armpl_svatan2_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("atan2f", "armpl_svatan2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.atan2.f64", "armpl_vatan2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.atan2.f32", "armpl_vatan2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.atan2.f64", "armpl_svatan2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.atan2.f32", "armpl_svatan2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("atanh", "armpl_vatanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("atanhf", "armpl_vatanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("atanh", "armpl_svatanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("atanhf", "armpl_svatanh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("cbrt", "armpl_vcbrtq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("cbrtf", "armpl_vcbrtq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("cbrt", "armpl_svcbrt_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("cbrtf", "armpl_svcbrt_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("copysign", "armpl_vcopysignq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("copysignf", "armpl_vcopysignq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("copysign", "armpl_svcopysign_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("copysignf", "armpl_svcopysign_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("cos", "armpl_vcosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("cosf", "armpl_vcosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("cos", "armpl_svcos_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("cosf", "armpl_svcos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.cos.f64", "armpl_vcosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.cos.f32", "armpl_vcosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.cos.f64", "armpl_svcos_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.cos.f32", "armpl_svcos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("cosh", "armpl_vcoshq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("coshf", "armpl_vcoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("cosh", "armpl_svcosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("coshf", "armpl_svcosh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.cosh.f64", "armpl_vcoshq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.cosh.f32", "armpl_vcoshq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.cosh.f64", "armpl_svcosh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.cosh.f32", "armpl_svcosh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("cospi", "armpl_vcospiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("cospif", "armpl_vcospiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("cospi", "armpl_svcospi_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("cospif", "armpl_svcospi_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("erf", "armpl_verfq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("erff", "armpl_verfq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("erf", "armpl_sverf_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("erff", "armpl_sverf_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("erfc", "armpl_verfcq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("erfcf", "armpl_verfcq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("erfc", "armpl_sverfc_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("erfcf", "armpl_sverfc_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("exp", "armpl_vexpq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("expf", "armpl_vexpq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("exp", "armpl_svexp_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("expf", "armpl_svexp_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.exp.f64", "armpl_vexpq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.exp.f32", "armpl_vexpq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.exp.f64", "armpl_svexp_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.exp.f32", "armpl_svexp_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("exp10", "armpl_vexp10q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("exp10f", "armpl_vexp10q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("exp10", "armpl_svexp10_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("exp10f", "armpl_svexp10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.exp10.f64", "armpl_vexp10q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.exp10.f32", "armpl_vexp10q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.exp10.f64", "armpl_svexp10_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.exp10.f32", "armpl_svexp10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("exp2", "armpl_vexp2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("exp2f", "armpl_vexp2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("exp2", "armpl_svexp2_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("exp2f", "armpl_svexp2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.exp2.f64", "armpl_vexp2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.exp2.f32", "armpl_vexp2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.exp2.f64", "armpl_svexp2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.exp2.f32", "armpl_svexp2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("expm1", "armpl_vexpm1q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("expm1f", "armpl_vexpm1q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("expm1", "armpl_svexpm1_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("expm1f", "armpl_svexpm1_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("fdim", "armpl_vfdimq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fdimf", "armpl_vfdimq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fdim", "armpl_svfdim_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("fdimf", "armpl_svfdim_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("fma", "armpl_vfmaq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vvv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fmaf", "armpl_vfmaq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vvv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fma", "armpl_svfma_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvvv", NOCC)
+TLI_DEFINE_VECFUNC("fmaf", "armpl_svfma_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvvv", NOCC)
+
+TLI_DEFINE_VECFUNC("fmax", "armpl_vfmaxq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fmaxf", "armpl_vfmaxq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fmax", "armpl_svfmax_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("fmaxf", "armpl_svfmax_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("fmin", "armpl_vfminq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fminf", "armpl_vfminq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fmin", "armpl_svfmin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("fminf", "armpl_svfmin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("fmod", "armpl_vfmodq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fmodf", "armpl_vfmodq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("fmod", "armpl_svfmod_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("fmodf", "armpl_svfmod_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("hypot", "armpl_vhypotq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("hypotf", "armpl_vhypotq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("hypot", "armpl_svhypot_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("hypotf", "armpl_svhypot_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("ilogb", "armpl_vilogbq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("ilogbf", "armpl_vilogbq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("ilogb", "armpl_svilogb_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("ilogbf", "armpl_svilogb_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("ldexp", "armpl_vldexpq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("ldexpf", "armpl_vldexpq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("ldexp", "armpl_svldexp_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("ldexpf", "armpl_svldexp_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("lgamma", "armpl_vlgammaq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("lgammaf", "armpl_vlgammaq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("lgamma", "armpl_svlgamma_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("lgammaf", "armpl_svlgamma_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("log", "armpl_vlogq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("logf", "armpl_vlogq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("log", "armpl_svlog_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("logf", "armpl_svlog_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.log.f64", "armpl_vlogq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.log.f32", "armpl_vlogq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.log.f64", "armpl_svlog_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.log.f32", "armpl_svlog_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("log10", "armpl_vlog10q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("log10f", "armpl_vlog10q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("log10", "armpl_svlog10_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("log10f", "armpl_svlog10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.log10.f64", "armpl_vlog10q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.log10.f32", "armpl_vlog10q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.log10.f64", "armpl_svlog10_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.log10.f32", "armpl_svlog10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("log1p", "armpl_vlog1pq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("log1pf", "armpl_vlog1pq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("log1p", "armpl_svlog1p_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("log1pf", "armpl_svlog1p_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("log2", "armpl_vlog2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("log2f", "armpl_vlog2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("log2", "armpl_svlog2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("log2f", "armpl_svlog2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.log2.f64", "armpl_vlog2q_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.log2.f32", "armpl_vlog2q_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.log2.f64", "armpl_svlog2_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.log2.f32", "armpl_svlog2_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("modf", "armpl_vmodfq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("modff", "armpl_vmodfq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("modf", "armpl_svmodf_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8", NOCC)
+TLI_DEFINE_VECFUNC("modff", "armpl_svmodf_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4", NOCC)
+
+TLI_DEFINE_VECFUNC("nextafter", "armpl_vnextafterq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("nextafterf", "armpl_vnextafterq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("nextafter", "armpl_svnextafter_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("nextafterf", "armpl_svnextafter_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("pow", "armpl_vpowq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("powf", "armpl_vpowq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("pow", "armpl_svpow_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("powf", "armpl_svpow_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.pow.f64", "armpl_vpowq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.pow.f32", "armpl_vpowq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.pow.f64", "armpl_svpow_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.pow.f32", "armpl_svpow_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvv", NOCC)
+
+TLI_DEFINE_VECFUNC("sin", "armpl_vsinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sinf", "armpl_vsinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sin", "armpl_svsin_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("sinf", "armpl_svsin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.sin.f64", "armpl_vsinq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.sin.f32", "armpl_vsinq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.sin.f64", "armpl_svsin_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.sin.f32", "armpl_svsin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("sincos", "armpl_vsincosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8l8", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sincosf", "armpl_vsincosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sincos", "armpl_svsincos_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8l8", NOCC)
+TLI_DEFINE_VECFUNC("sincosf", "armpl_svsincos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4l4", NOCC)
+
+TLI_DEFINE_VECFUNC("sincospi", "armpl_vsincospiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8l8", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sincospif", "armpl_vsincospiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sincospi", "armpl_svsincospi_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8l8", NOCC)
+TLI_DEFINE_VECFUNC("sincospif", "armpl_svsincospi_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4l4", NOCC)
+
+TLI_DEFINE_VECFUNC("sinh", "armpl_vsinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sinhf", "armpl_vsinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sinh", "armpl_svsinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("sinhf", "armpl_svsinh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.sinh.f64", "armpl_vsinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.sinh.f32", "armpl_vsinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.sinh.f64", "armpl_svsinh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.sinh.f32", "armpl_svsinh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("sinpi", "armpl_vsinpiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sinpif", "armpl_vsinpiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sinpi", "armpl_svsinpi_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("sinpif", "armpl_svsinpi_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("sqrt", "armpl_vsqrtq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sqrtf", "armpl_vsqrtq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("sqrt", "armpl_svsqrt_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("sqrtf", "armpl_svsqrt_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("tan", "armpl_vtanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("tanf", "armpl_vtanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("tan", "armpl_svtan_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("tanf", "armpl_svtan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "armpl_vtanq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "armpl_vtanq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.tan.f64", "armpl_svtan_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.tan.f32", "armpl_svtan_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("tanh", "armpl_vtanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("tanhf", "armpl_vtanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("tanh", "armpl_svtanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("tanhf", "armpl_svtanh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("llvm.tanh.f64", "armpl_vtanhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.tanh.f32", "armpl_vtanhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("llvm.tanh.f64", "armpl_svtanh_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("llvm.tanh.f32", "armpl_svtanh_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
+
+TLI_DEFINE_VECFUNC("tgamma", "armpl_vtgammaq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("tgammaf", "armpl_vtgammaq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v", CallingConv::AArch64_VectorCall)
+TLI_DEFINE_VECFUNC("tgamma", "armpl_svtgamma_f64_x",  SCALABLE(2), MASKED, "_ZGVsMxv", NOCC)
+TLI_DEFINE_VECFUNC("tgammaf", "armpl_svtgamma_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv", NOCC)
 
 #elif defined(TLI_DEFINE_AMDLIBM_VECFUNCS)
 TLI_DEFINE_VECFUNC("sinf", "amd_vrs16_sinf", FIXED(16), NOMASK, "_ZGV_LLVM_N16v")
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 3a8cdf946da37..df4720c66f723 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -1306,21 +1306,21 @@ static const VecDesc VecFuncs_SVML[] = {
 static const VecDesc VecFuncs_SLEEFGNUABI_VF2[] = {
 #define TLI_DEFINE_SLEEFGNUABI_VF2_VECFUNCS
 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX)                         \
-  {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX},
+  {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX, /* CC = */ std::nullopt},
 #include "llvm/Analysis/VecFuncs.def"
 #undef TLI_DEFINE_SLEEFGNUABI_VF2_VECFUNCS
 };
 static const VecDesc VecFuncs_SLEEFGNUABI_VF4[] = {
 #define TLI_DEFINE_SLEEFGNUABI_VF4_VECFUNCS
 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX)                         \
-  {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX},
+  {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX, /* CC = */ std::nullopt},
 #include "llvm/Analysis/VecFuncs.def"
 #undef TLI_DEFINE_SLEEFGNUABI_VF4_VECFUNCS
 };
 static const VecDesc VecFuncs_SLEEFGNUABI_VFScalable[] = {
 #define TLI_DEFINE_SLEEFGNUABI_SCALABLE_VECFUNCS
 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX)                   \
-  {SCAL, VEC, VF, MASK, VABI_PREFIX},
+  {SCAL, VEC, VF, MASK, VABI_PREFIX, /* CC = */ std::nullopt},
 #include "llvm/Analysis/VecFuncs.def"
 #undef TLI_DEFINE_SLEEFGNUABI_SCALABLE_VECFUNCS
 };
@@ -1328,15 +1328,15 @@ static const VecDesc VecFuncs_SLEEFGNUABI_VFScalable[] = {
 static const VecDesc VecFuncs_SLEEFGNUABI_VFScalableRISCV[] = {
 #define TLI_DEFINE_SLEEFGNUABI_SCALABLE_VECFUNCS_RISCV
 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX)                   \
-  {SCAL, VEC, VF, MASK, VABI_PREFIX},
+  {SCAL, VEC, VF, MASK, VABI_PREFIX, /* CC = */ std::nullopt},
 #include "llvm/Analysis/VecFuncs.def"
 #undef TLI_DEFINE_SLEEFGNUABI_SCALABLE_VECFUNCS_RISCV
 };
 
 static const VecDesc VecFuncs_ArmPL[] = {
 #define TLI_DEFINE_ARMPL_VECFUNCS
-#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX)                   \
-  {SCAL, VEC, VF, MASK, VABI_PREFIX},
+#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX, CC)               \
+  {SCAL, VEC, VF, MASK, VABI_PREFIX, CC},
 #include "llvm/Analysis/VecFuncs.def"
 #undef TLI_DEFINE_ARMPL_VECFUNCS
 };
@@ -1344,7 +1344,7 @@ static const VecDesc VecFuncs_ArmPL[] = {
 const VecDesc VecFuncs_AMDLIBM[] = {
 #define TLI_DEFINE_AMDLIBM_VECFUNCS
 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX)                   \
-  {SCAL, VEC, VF, MASK, VABI_PREFIX},
+  {SCAL, VEC, VF, MASK, VABI_PREFIX, /* CC = */ std::nullopt},
 #include "llvm/Analysis/VecFuncs.def"
 #undef TLI_DEFINE_AMDLIBM_VECFUNCS
 };
diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
index afe834ee201bb..ca2a02bfd031a 100644
--- a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
+++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
@@ -55,6 +55,8 @@ static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
   Function *VecFunc =
       Function::Create(VectorFTy, Function::ExternalLinkage, VFName, M);
   VecFunc->copyAttributesFrom(CI.getCalledFunction());
+  if (auto CC = VD->getCallingConv())
+    VecFunc->setCallingConv(*CC);
   ++NumVFDeclAdded;
   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
                     << "` of type " << *VectorFTy << "\n");
diff --git a/llvm/test/Transforms/Util/add-TLI-mappings.ll b/llvm/test/Transforms/Util/add-TLI-mappings.ll
index 39caf008f924f..b7eef89304c0d 100644
--- a/llvm/test/Transforms/Util/add-TLI-mappings.ll
+++ b/llvm/test/Transforms/Util/add-TLI-mappings.ll
@@ -218,21 +218,21 @@ declare float @llvm.log10.f32(float) #0
 ; SLEEFGNUABI_RISCV: declare <vscale x 2 x double> @Sleef_sindx_u10rvvm2(<vscale x 2 x double>)
 ; SLEEFGNUABI_RISCV: declare <vscale x 4 x float> @Sleef_log10fx_u10rvvm2(<vscale x 4 x float>)
 
-; ARMPL: declare <2 x double> @armpl_vmodfq_f64(<2 x double>, ptr)
+; ARMPL: declare aarch64_vector_pcs <2 x double> @armpl_vmodfq_f64(<2 x double>, ptr)
 ; ARMPL: declare <vscale x 2 x double> @armpl_svmodf_f64_x(<vscale x 2 x double>, ptr, <vscale x 2 x i1>)
-; ARMPL: declare <4 x float> @armpl_vmodfq_f32(<4 x float>, ptr)
+; ARMPL: declare aarch64_vector_pcs <4 x float> @armpl_vmodfq_f32(<4 x float>, ptr)
 ; ARMPL: declare <vscale x 4 x float> @armpl_svmodf_f32_x(<vscale x 4 x float>, ptr, <vscale x 4 x i1>)
-; ARMPL: declare <2 x double> @armpl_vsinq_f64(<2 x double>)
+; ARMPL: declare aarch64_vector_pcs <2 x double> @armpl_vsinq_f64(<2 x double>)
 ; ARMPL: declare <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double>, <vscale x 2 x i1>)
-; ARMPL: declare void @armpl_vsincosq_f64(<2 x double>, ptr, ptr)
+; ARMPL: declare aarch64_vector_pcs void @armpl_vsincosq_f64(<2 x double>, ptr, ptr)
 ; ARMPL: declare void @armpl_svsincos_f64_x(<vscale x 2 x double>, ptr, ptr, <vscale x 2 x i1>)
-; ARMPL: declare void @armpl_vsincosq_f32(<4 x float>, ptr, ptr)
+; ARMPL: declare aarch64_vector_pcs void @armpl_vsincosq_f32(<4 x float>, ptr, ptr)
 ; ARMPL: declare void @armpl_svsincos_f32_x(<vscale x 4 x float>, ptr, ptr, <vscale x 4 x i1>)
-; ARMPL: declare void @armpl_vsincospiq_f64(<2 x double>, ptr, ptr)
+; ARMPL: declare aarch64_vector_pcs void @armpl_vsincospiq_f64(<2 x double>, ptr, ptr)
 ; ARMPL: declare void @armpl_svsincospi_f64_x(<vscale x 2 x double>, ptr, ptr, <vscale x 2 x i1>)
-; ARMPL: declare void @armpl_vsincospiq_f32(<4 x float>, ptr, ptr)
+; ARMPL: declare aarch64_vector_pcs void @armpl_vsincospiq_f32(<4 x float>, ptr, ptr)
 ; ARMPL: declare void @armpl_svsincospi_f32_x(<vscale x 4 x float>, ptr, ptr, <vscale x 4 x i1>)
-; ARMPL: declare <4 x float> @armpl_vlog10q_f32(<4 x float>)
+; ARMPL: declare aarch64_vector_pcs <4 x float> @armpl_vlog10q_f32(<4 x float>)
 ; ARMPL: declare <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float>, <vscale x 4 x i1>)
 
 attributes #0 = { nounwind readnone }
diff --git a/llvm/unittests/Analysis/ReplaceWithVecLibTest.cpp b/llvm/unittests/Analysis/ReplaceWithVecLibTest.cpp
index d97549cb69d55..31e0263ad64d7 100644
--- a/llvm/unittests/Analysis/ReplaceWithVecLibTest.cpp
+++ b/llvm/unittests/Analysis/ReplaceWithVecLibTest.cpp
@@ -97,7 +97,7 @@ declare <vscale x 4 x float> @llvm.powi.f32.i32(<vscale x 4 x float>, i32) #0
 TEST_F(ReplaceWithVecLibTest, TestValidMapping) {
   VecDesc CorrectVD = {"llvm.powi.f32.i32", "_ZGVsMxvu_powi",
                        ElementCount::getScalable(4), /*Masked*/ true,
-                       "_ZGVsMxvu"};
+                       "_ZGVsMxvu", /* CC = */ std::nullopt};
   EXPECT_EQ(run(CorrectVD, IR),
             "Intrinsic calls replaced with vector libraries: 1");
 }
@@ -107,9 +107,9 @@ TEST_F(ReplaceWithVecLibTest, TestValidMapping) {
 TEST_F(ReplaceWithVecLibTest, TestInvalidMapping) {
   VecDesc IncorrectVD = {"llvm.powi.f32.i32", "_ZGVsMxvv_powi",
                          ElementCount::getScalable(4), /*Masked*/ true,
-                         "_ZGVsMxvv"};
+                         "_ZGVsMxvv", /* CC = */ std::nullopt};
   EXPECT_EQ(run(IncorrectVD, IR),
             "replace-with-veclib: Will not replace: llvm.powi.f32.i32. Wrong "
             "type at index 1: i32");
 }
-#endif
\ No newline at end of file
+#endif

From d41bd1fb4056f2bb649a76b09b8a5cc258453818 Mon Sep 17 00:00:00 2001
From: Jerry-Ge <jerry.ge@arm.com>
Date: Thu, 17 Apr 2025 00:59:34 -0700
Subject: [PATCH 233/710] [mlir][tosa] Remove trailing whitespaces (#136031)

Trivial format cleanup

Signed-off-by: Jerry Ge <jerry.ge@arm.com>
---
 mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
index fd8ff039d1d0c..0aef4653b74ff 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
@@ -250,9 +250,9 @@ def Tosa_EXT_DYNAMIC      : I32EnumAttrCase<"dynamic", 11>;
 
 def Tosa_ExtensionAttr
     : Tosa_I32EnumAttr<"Extension", "supported TOSA extensions", "ext", [
-      Tosa_EXT_NONE, Tosa_EXT_INT16, Tosa_EXT_INT4, Tosa_EXT_BF16, 
-      Tosa_EXT_FP8E4M3, Tosa_EXT_FP8E5M2, Tosa_EXT_FFT, Tosa_EXT_VARIABLE, 
-      Tosa_EXT_CONTROLFLOW, Tosa_EXT_DOUBLEROUND, Tosa_EXT_INEXACTROUND, 
+      Tosa_EXT_NONE, Tosa_EXT_INT16, Tosa_EXT_INT4, Tosa_EXT_BF16,
+      Tosa_EXT_FP8E4M3, Tosa_EXT_FP8E5M2, Tosa_EXT_FFT, Tosa_EXT_VARIABLE,
+      Tosa_EXT_CONTROLFLOW, Tosa_EXT_DOUBLEROUND, Tosa_EXT_INEXACTROUND,
       Tosa_EXT_DYNAMIC
     ]>;
 

From f351172d4a840dfbf533319b62925747a10b762f Mon Sep 17 00:00:00 2001
From: lntue <lntue@google.com>
Date: Thu, 17 Apr 2025 16:02:44 +0800
Subject: [PATCH 234/710] [libc] Export standard C symbols in the public
 packages for MacOS instead of namespaced C++ symbols. (#136100)

---
 libc/src/__support/common.h | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/libc/src/__support/common.h b/libc/src/__support/common.h
index 42e8a79187fac..15209b76978af 100644
--- a/libc/src/__support/common.h
+++ b/libc/src/__support/common.h
@@ -37,17 +37,27 @@
 
 #define LLVM_LIBC_ATTR(name) EXPAND_THEN_SECOND(LLVM_LIBC_FUNCTION_ATTR_##name)
 
-// MacOS needs to be excluded because it does not support aliasing.
-#if defined(LIBC_COPT_PUBLIC_PACKAGING) && (!defined(__APPLE__))
+// At the moment, [[gnu::alias()]] is not supported on MacOS, and it is needed
+// to cleanly export and alias the C++ symbol `LIBC_NAMESPACE::func` with the C
+// symbol `func`.  So for public packaging on MacOS, we will only export the C
+// symbol.  Moreover, a C symbol `func` in macOS is mangled as `_func`.
+#if defined(LIBC_COPT_PUBLIC_PACKAGING)
+#ifndef __APPLE__
 #define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist)                           \
   LLVM_LIBC_ATTR(name)                                                         \
   LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name)                       \
       __##name##_impl__ __asm__(#name);                                        \
   decltype(LIBC_NAMESPACE::name) name [[gnu::alias(#name)]];                   \
   type __##name##_impl__ arglist
-#else
+#else // __APPLE__
+#define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist)                           \
+  LLVM_LIBC_ATTR(name)                                                         \
+  LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name) name asm("_" #name);  \
+  type name arglist
+#endif // __APPLE__
+#else  // LIBC_COPT_PUBLIC_PACKAGING
 #define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist) type name arglist
-#endif
+#endif // LIBC_COPT_PUBLIC_PACKAGING
 
 // This extra layer of macro allows `name` to be a macro to rename a function.
 #define LLVM_LIBC_FUNCTION(type, name, arglist)                                \

From bf6986f9f09f79da38006a83c339226c429bb686 Mon Sep 17 00:00:00 2001
From: Camsyn <camsyn@foxmail.com>
Date: Thu, 17 Apr 2025 16:09:07 +0800
Subject: [PATCH 235/710] [TSan, SanitizerBinaryMetadata] Improve instrument
 for derived pointers via phis/selects (#132752)

ThreadSanitizer.cpp and SanitizerBinaryMetadata.cpp previously used
`getUnderlyingObject` to check if pointers originate from stack objects.

However, `getUnderlyingObject()` by default only looks through linear
chains, not selects/phis. In particular, this means that we miss cases
involving pointer induction variables.

For instance,
```llvm
%stkobj = alloca [2 x i32], align 8
; getUnderlyingObject(%derived) = %derived
%derived = getelementptr inbounds i32, ptr %stkobj, i64 1
```

This will result in redundant instrumentation of TSan, resulting in
greater performance costs, especially when there are loops, referring to
this [godbolt page](https://godbolt.org/z/eaT1fPjTW) for details.
```cpp
char loop(int x) {
    char buf[10];
    char *p = buf;
    for (int i = 0; i < x && i < 10; i++) {
      // Should not instrument, as its base object is a non-captured stack
      // variable.
      // However, currectly, it is instrumented due to %p = %phi ...
      *p++ = i;
    }

    // Use buf to prevent it from being eliminated by optimization
    return buf[9];
}
```

There are TWO APIs `getUnderlyingObjectAggressive` and
`findAllocaForValue` that can backtrack the pointer via tree traversal,
supporting phis/selects.

This patch replaces `getUnderlyingObject` with `findAllocaForValue`
which:
1. Properly tracks through PHINodes and select operations
2. Directly identifies if a pointer comes from a `AllocaInst`

Performance impact:
- Compilation: Moderate cost increase due to wider value tracing, but...
- Runtime: Significant wins for code with pointer induction variables
derived from stack allocas, especially for loop-heavy code, as
instrumentation can now be safely omitted.
---
 .../SanitizerBinaryMetadata.cpp               |  4 +--
 .../Instrumentation/ThreadSanitizer.cpp       |  4 +--
 .../ThreadSanitizer/capture.ll                | 31 +++++++++++++++++++
 3 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
index ab5cdc08664d7..b6787fd8cb21e 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
@@ -392,8 +392,8 @@ bool maybeSharedMutable(const Value *Addr) {
   if (!Addr)
     return true;
 
-  if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
-      !PointerMayBeCaptured(Addr, /*ReturnCaptures=*/true))
+  const AllocaInst *AI = findAllocaForValue(Addr);
+  if (AI && !PointerMayBeCaptured(Addr, /*ReturnCaptures=*/true))
     return false; // Object is on stack but does not escape.
 
   Addr = Addr->stripInBoundsOffsets();
diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 1811d145f9907..7f846d25b8133 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -448,8 +448,8 @@ void ThreadSanitizer::chooseInstructionsToInstrument(
       }
     }
 
-    if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
-        !PointerMayBeCaptured(Addr, /*ReturnCaptures=*/true)) {
+    const AllocaInst *AI = findAllocaForValue(Addr);
+    if (AI && !PointerMayBeCaptured(Addr, /*ReturnCaptures=*/true)) {
       // The variable is addressable but not captured, so it cannot be
       // referenced from a different thread and participate in a data race
       // (see llvm/Analysis/CaptureTracking.h for details).
diff --git a/llvm/test/Instrumentation/ThreadSanitizer/capture.ll b/llvm/test/Instrumentation/ThreadSanitizer/capture.ll
index 8edf310df9823..9cd5d77f4753e 100644
--- a/llvm/test/Instrumentation/ThreadSanitizer/capture.ll
+++ b/llvm/test/Instrumentation/ThreadSanitizer/capture.ll
@@ -88,4 +88,35 @@ entry:
 ; CHECK: __tsan_write
 ; CHECK: ret void
 
+define void @notcaptured3(i1 %cond) nounwind uwtable sanitize_thread {
+entry:
+  %stkobj = alloca [2 x i32], align 8
+  %derived = getelementptr inbounds i32, ptr %stkobj, i64 1
+  %ptr = select i1 %cond, ptr %derived, ptr %stkobj
+  store i32 42, ptr %ptr, align 4
+  ret void
+}
+; CHECK-LABEL: define void @notcaptured3
+; CHECK-NOT: call void @__tsan_write4(ptr %ptr)
+; CHECK: ret void
 
+define void @notcaptured4() nounwind uwtable sanitize_thread {
+entry:
+  %stkobj = alloca [10 x i8], align 1
+  br label %loop
+
+exit:
+  ret void
+
+loop:
+  %count = phi i32 [ 0, %entry ], [ %addone, %loop ]
+  %derived = phi ptr [ %stkobj, %entry ], [ %ptraddone, %loop ]
+  store i32 %count, ptr %derived, align 4
+  %ptraddone = getelementptr inbounds i32, ptr %derived, i64 1
+  %addone = add nuw nsw i32 %count, 1
+  %eq10 = icmp eq i32 %addone, 10
+  br i1 %eq10, label %exit, label %loop
+}
+; CHECK-LABEL: define void @notcaptured4
+; CHECK: ret void
+; CHECK-NOT: call void @__tsan_write4(ptr %derived)

From 1588aab6ed2d02db2ffb23ca7f339d38a4d8c5e9 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Thu, 17 Apr 2025 09:25:57 +0100
Subject: [PATCH 236/710] [AArch64] Generalize integer FPR lane stores for all
 types (#134117)

This rewrites the fold from #129756 to apply to all types, including
stores of i8s. This required adding a new `aarch64mfp8` MVT to represent
FPR8 types on AArch64, which can be used to extract and store 8-bit
values using b sub-registers.

Follow on from: #129756
Closes: #131793
---
 llvm/include/llvm/CodeGen/ValueTypes.td       |   2 +
 llvm/lib/CodeGen/ValueTypes.cpp               |   4 +
 .../Target/AArch64/AArch64ISelLowering.cpp    |  79 ++++--
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  44 ++-
 .../lib/Target/AArch64/AArch64RegisterInfo.td |   2 +-
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  16 ++
 .../CodeGen/AArch64/aarch64-sve-ldst-one.ll   | 161 ++++++++---
 llvm/test/CodeGen/AArch64/add.ll              |  19 +-
 llvm/test/CodeGen/AArch64/andorxor.ll         |  57 ++--
 .../test/CodeGen/AArch64/arm64-collect-loh.ll |   9 +-
 llvm/test/CodeGen/AArch64/arm64-neon-copy.ll  |   6 +-
 .../AArch64/arm64-neon-simd-ldst-one.ll       |  14 +-
 llvm/test/CodeGen/AArch64/arm64-rev.ll        |   4 +-
 llvm/test/CodeGen/AArch64/arm64-st1.ll        | 251 +++++++++++++-----
 llvm/test/CodeGen/AArch64/bitcast-v2i8.ll     |  18 +-
 llvm/test/CodeGen/AArch64/concat-vector.ll    |   4 +-
 llvm/test/CodeGen/AArch64/ctlz.ll             |  19 +-
 llvm/test/CodeGen/AArch64/ctpop.ll            |  19 +-
 llvm/test/CodeGen/AArch64/cttz.ll             |  19 +-
 llvm/test/CodeGen/AArch64/dp1.ll              |   3 +-
 .../CodeGen/AArch64/extract-vector-cmp.ll     |   7 +-
 llvm/test/CodeGen/AArch64/insertextract.ll    |   4 +-
 llvm/test/CodeGen/AArch64/mul.ll              |  19 +-
 llvm/test/CodeGen/AArch64/neon-truncstore.ll  |  14 +-
 llvm/test/CodeGen/AArch64/nontemporal-load.ll |   3 +-
 llvm/test/CodeGen/AArch64/pr-cf624b2.ll       |  60 ++---
 llvm/test/CodeGen/AArch64/sadd_sat_vec.ll     |   9 +-
 .../CodeGen/AArch64/setcc-type-mismatch.ll    |   3 +-
 llvm/test/CodeGen/AArch64/ssub_sat_vec.ll     |   9 +-
 llvm/test/CodeGen/AArch64/store.ll            |  31 ++-
 llvm/test/CodeGen/AArch64/sub.ll              |  19 +-
 .../AArch64/sve-fixed-length-permute-rev.ll   |  64 +++--
 ...-streaming-mode-fixed-length-ld2-alloca.ll |   6 +-
 ...mode-fixed-length-masked-gather-scatter.ll |  12 +-
 ...g-mode-fixed-length-permute-zip-uzp-trn.ll | 172 ++++++------
 llvm/test/CodeGen/AArch64/tbl-loops.ll        |   7 +-
 llvm/test/CodeGen/AArch64/trunc-to-tbl.ll     |  41 +--
 llvm/test/CodeGen/AArch64/uadd_sat_vec.ll     |   9 +-
 llvm/test/CodeGen/AArch64/usub_sat_vec.ll     |   9 +-
 .../vec-combine-compare-truncate-store.ll     |  23 +-
 .../AArch64/vec3-loads-ext-trunc-stores.ll    | 202 +++++++-------
 llvm/test/CodeGen/AArch64/vec_uaddo.ll        |  39 ++-
 llvm/test/CodeGen/AArch64/vec_umulo.ll        |  44 ++-
 llvm/test/CodeGen/AArch64/vector-compress.ll  |  64 ++---
 llvm/test/CodeGen/AArch64/zext-to-tbl.ll      | 210 +++++++--------
 45 files changed, 1035 insertions(+), 795 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td
index fc1a95e33380b..28216a7a55398 100644
--- a/llvm/include/llvm/CodeGen/ValueTypes.td
+++ b/llvm/include/llvm/CodeGen/ValueTypes.td
@@ -338,6 +338,8 @@ def amdgpuBufferFatPointer : ValueType<160, 234>;
 // FIXME: Remove this and the getPointerType() override if MVT::i82 is added.
 def amdgpuBufferStridedPointer : ValueType<192, 235>;
 
+def aarch64mfp8 : ValueType<8,  236>;  // 8-bit value in FPR (AArch64)
+
 let isNormalValueType = false in {
 def token      : ValueType<0, 504>;  // TokenTy
 def MetadataVT : ValueType<0, 505> { // Metadata
diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp
index 0554b6387c5e6..10970b719fcae 100644
--- a/llvm/lib/CodeGen/ValueTypes.cpp
+++ b/llvm/lib/CodeGen/ValueTypes.cpp
@@ -198,6 +198,8 @@ std::string EVT::getEVTString() const {
     return "amdgpuBufferFatPointer";
   case MVT::amdgpuBufferStridedPointer:
     return "amdgpuBufferStridedPointer";
+  case MVT::aarch64mfp8:
+    return "aarch64mfp8";
   }
 }
 
@@ -221,6 +223,8 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
   case MVT::x86mmx:  return llvm::FixedVectorType::get(llvm::IntegerType::get(Context, 64), 1);
   case MVT::aarch64svcount:
     return TargetExtType::get(Context, "aarch64.svcount");
+  case MVT::aarch64mfp8:
+    return FixedVectorType::get(IntegerType::get(Context, 8), 1);
   case MVT::x86amx:  return Type::getX86_AMXTy(Context);
   case MVT::i64x8:   return IntegerType::get(Context, 512);
   case MVT::amdgpuBufferFatPointer:  return IntegerType::get(Context, 160);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a95d8d343adf2..771eee1b3fecf 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -400,6 +400,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   }
 
   if (Subtarget->hasFPARMv8()) {
+    addRegisterClass(MVT::aarch64mfp8, &AArch64::FPR8RegClass);
     addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
     addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
     addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
@@ -23930,6 +23931,8 @@ static SDValue combineI8TruncStore(StoreSDNode *ST, SelectionDAG &DAG,
 static unsigned getFPSubregForVT(EVT VT) {
   assert(VT.isSimple() && "Expected simple VT");
   switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::aarch64mfp8:
+    return AArch64::bsub;
   case MVT::f16:
     return AArch64::hsub;
   case MVT::f32:
@@ -24019,39 +24022,65 @@ static SDValue performSTORECombine(SDNode *N,
     SDValue ExtIdx = Value.getOperand(1);
     EVT VectorVT = Vector.getValueType();
     EVT ElemVT = VectorVT.getVectorElementType();
-    if (!ValueVT.isInteger() || ElemVT == MVT::i8 || MemVT == MVT::i8)
+
+    if (!ValueVT.isInteger())
+      return SDValue();
+
+    // Propagate zero constants (applying this fold may miss optimizations).
+    if (ISD::isConstantSplatVectorAllZeros(Vector.getNode())) {
+      SDValue ZeroElt = DAG.getConstant(0, DL, ValueVT);
+      DAG.ReplaceAllUsesWith(Value, ZeroElt);
       return SDValue();
+    }
+
     if (ValueVT != MemVT && !ST->isTruncatingStore())
       return SDValue();
 
-    // Heuristic: If there are other users of integer scalars extracted from
-    // this vector that won't fold into the store -- abandon folding. Applying
-    // this fold may extend the vector lifetime and disrupt paired stores.
-    for (const auto &Use : Vector->uses()) {
-      if (Use.getResNo() != Vector.getResNo())
-        continue;
-      const SDNode *User = Use.getUser();
-      if (User->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
-          (!User->hasOneUse() ||
-           (*User->user_begin())->getOpcode() != ISD::STORE))
-        return SDValue();
-    }
+    // This could generate an additional extract if the index is non-zero and
+    // the extracted value has multiple uses.
+    auto *ExtCst = dyn_cast<ConstantSDNode>(ExtIdx);
+    if ((!ExtCst || !ExtCst->isZero()) && !Value.hasOneUse())
+      return SDValue();
 
-    EVT FPElemVT = EVT::getFloatingPointVT(ElemVT.getSizeInBits());
-    EVT FPVectorVT = VectorVT.changeVectorElementType(FPElemVT);
-    SDValue Cast = DAG.getNode(ISD::BITCAST, DL, FPVectorVT, Vector);
-    SDValue Ext =
-        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, FPElemVT, Cast, ExtIdx);
+    // These can lower to st1, which is preferable if we're unlikely to fold the
+    // addressing into the store.
+    if (Subtarget->isNeonAvailable() && ElemVT == MemVT &&
+        (VectorVT.is64BitVector() || VectorVT.is128BitVector()) && ExtCst &&
+        !ExtCst->isZero() && ST->getBasePtr().getOpcode() != ISD::ADD)
+      return SDValue();
 
-    EVT FPMemVT = EVT::getFloatingPointVT(MemVT.getSizeInBits());
-    if (ST->isTruncatingStore() && FPMemVT != FPElemVT) {
-      SDValue Trunc = DAG.getTargetExtractSubreg(getFPSubregForVT(FPMemVT), DL,
-                                                 FPMemVT, Ext);
-      return DAG.getStore(ST->getChain(), DL, Trunc, ST->getBasePtr(),
-                          ST->getMemOperand());
+    if (MemVT == MVT::i64 || MemVT == MVT::i32) {
+      // Heuristic: If there are other users of w/x integer scalars extracted
+      // from this vector that won't fold into the store -- abandon folding.
+      // Applying this fold may disrupt paired stores.
+      for (const auto &Use : Vector->uses()) {
+        if (Use.getResNo() != Vector.getResNo())
+          continue;
+        const SDNode *User = Use.getUser();
+        if (User->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+            (!User->hasOneUse() ||
+             (*User->user_begin())->getOpcode() != ISD::STORE))
+          return SDValue();
+      }
     }
 
-    return DAG.getStore(ST->getChain(), DL, Ext, ST->getBasePtr(),
+    SDValue ExtVector = Vector;
+    if (!ExtCst || !ExtCst->isZero()) {
+      // Handle extracting from lanes != 0.
+      SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
+                                Value.getValueType(), Vector, ExtIdx);
+      SDValue Zero = DAG.getVectorIdxConstant(0, DL);
+      ExtVector = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VectorVT,
+                              DAG.getUNDEF(VectorVT), Ext, Zero);
+    }
+
+    EVT FPMemVT = MemVT == MVT::i8
+                      ? MVT::aarch64mfp8
+                      : EVT::getFloatingPointVT(MemVT.getSizeInBits());
+    SDValue FPSubreg = DAG.getTargetExtractSubreg(getFPSubregForVT(FPMemVT), DL,
+                                                  FPMemVT, ExtVector);
+
+    return DAG.getStore(ST->getChain(), DL, FPSubreg, ST->getBasePtr(),
                         ST->getMemOperand());
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 083b9410f7c11..a060a2f597ccd 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3590,7 +3590,7 @@ defm LDRW : LoadUI<0b10, 0, 0b01, GPR32z, uimm12s4, "ldr",
                          (load (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset)))]>;
 let Predicates = [HasFPARMv8] in {
 defm LDRB : LoadUI<0b00, 1, 0b01, FPR8Op, uimm12s1, "ldr",
-                   [(set FPR8Op:$Rt,
+                   [(set (i8 FPR8Op:$Rt),
                          (load (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset)))]>;
 defm LDRH : LoadUI<0b01, 1, 0b01, FPR16Op, uimm12s2, "ldr",
                    [(set (f16 FPR16Op:$Rt),
@@ -3778,7 +3778,7 @@ defm LDURW : LoadUnscaled<0b10, 0, 0b01, GPR32z, "ldur",
                           (load (am_unscaled32 GPR64sp:$Rn, simm9:$offset)))]>;
 let Predicates = [HasFPARMv8] in {
 defm LDURB : LoadUnscaled<0b00, 1, 0b01, FPR8Op, "ldur",
-                    [(set FPR8Op:$Rt,
+                    [(set (i8 FPR8Op:$Rt),
                           (load (am_unscaled8 GPR64sp:$Rn, simm9:$offset)))]>;
 defm LDURH : LoadUnscaled<0b01, 1, 0b01, FPR16Op, "ldur",
                     [(set (f16 FPR16Op:$Rt),
@@ -4348,7 +4348,7 @@ defm STRW : StoreUIz<0b10, 0, 0b00, GPR32z, uimm12s4, "str",
                             (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))]>;
 let Predicates = [HasFPARMv8] in {
 defm STRB : StoreUI<0b00, 1, 0b00, FPR8Op, uimm12s1, "str",
-                    [(store FPR8Op:$Rt,
+                    [(store (i8 FPR8Op:$Rt),
                             (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))]>;
 defm STRH : StoreUI<0b01, 1, 0b00, FPR16Op, uimm12s2, "str",
                     [(store (f16 FPR16Op:$Rt),
@@ -4484,7 +4484,7 @@ defm STURW : StoreUnscaled<0b10, 0, 0b00, GPR32z, "stur",
                                  (am_unscaled32 GPR64sp:$Rn, simm9:$offset))]>;
 let Predicates = [HasFPARMv8] in {
 defm STURB : StoreUnscaled<0b00, 1, 0b00, FPR8Op, "stur",
-                         [(store FPR8Op:$Rt,
+                         [(store (i8 FPR8Op:$Rt),
                                  (am_unscaled8 GPR64sp:$Rn, simm9:$offset))]>;
 defm STURH : StoreUnscaled<0b01, 1, 0b00, FPR16Op, "stur",
                          [(store (f16 FPR16Op:$Rt),
@@ -4604,6 +4604,12 @@ def : Pat<(truncstorei16 GPR64:$Rt, (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
 def : Pat<(truncstorei8 GPR64:$Rt, (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
   (STURBBi (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, simm9:$offset)>;
 
+// aarch64mfp8 (bsub) stores
+def : Pat<(store aarch64mfp8:$Rt, (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
+          (STURBi FPR8:$Rt, GPR64sp:$Rn, simm9:$offset)>;
+def : Pat<(store aarch64mfp8:$Rt, (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset)),
+          (STRBui FPR8:$Rt, GPR64sp:$Rn, uimm12s1:$offset)>;
+
 // Match stores from lane 0 to the appropriate subreg's store.
 multiclass VecStoreULane0Pat<SDPatternOperator StoreOp,
                              ValueType VTy, ValueType STy,
@@ -7245,8 +7251,15 @@ def : Pat<(v2i64 (int_aarch64_neon_vcopy_lane
 
 // Move elements between vectors
 multiclass Neon_INS_elt_pattern<ValueType VT128, ValueType VT64, ValueType VTSVE,
-                                ValueType VTScal, Operand SVEIdxTy, Instruction INS> {
+                                ValueType VTScal, Operand SVEIdxTy, Instruction INS, Instruction DUP, SubRegIndex DUPSub> {
   // Extracting from the lowest 128-bits of an SVE vector
+  def : Pat<(VT128 (vector_insert undef,
+                      (VTScal (vector_extract VTSVE:$Rm, (i64 SVEIdxTy:$Immn))),
+                      (i64 0))),
+            (INSERT_SUBREG (VT128 (IMPLICIT_DEF)),
+                (DUP (VT128 (EXTRACT_SUBREG VTSVE:$Rm, zsub)), SVEIdxTy:$Immn),
+                 DUPSub)>;
+
   def : Pat<(VT128 (vector_insert VT128:$Rn,
                       (VTScal (vector_extract VTSVE:$Rm, (i64 SVEIdxTy:$Immn))),
                       (i64 imm:$Immd))),
@@ -7265,6 +7278,11 @@ multiclass Neon_INS_elt_pattern<ValueType VT128, ValueType VT64, ValueType VTSVE
                         (i64 imm:$Immd))),
             (INS V128:$src, imm:$Immd, V128:$Rn, imm:$Immn)>;
 
+  def : Pat<(VT128 (vector_insert undef,
+                        (VTScal (vector_extract (VT128 V128:$Rn), (i64 imm:$Immn))),
+                        (i64 0))),
+            (INSERT_SUBREG (VT128 (IMPLICIT_DEF)), (DUP V128:$Rn, imm:$Immn), DUPSub)>;
+
   def : Pat<(VT128 (vector_insert V128:$src,
                         (VTScal (vector_extract (VT64 V64:$Rn), (i64 imm:$Immn))),
                         (i64 imm:$Immd))),
@@ -7287,15 +7305,15 @@ multiclass Neon_INS_elt_pattern<ValueType VT128, ValueType VT64, ValueType VTSVE
                 dsub)>;
 }
 
-defm : Neon_INS_elt_pattern<v8f16,  v4f16,  nxv8f16,  f16,  VectorIndexH, INSvi16lane>;
-defm : Neon_INS_elt_pattern<v8bf16, v4bf16, nxv8bf16, bf16, VectorIndexH, INSvi16lane>;
-defm : Neon_INS_elt_pattern<v4f32,  v2f32,  nxv4f32,  f32,  VectorIndexS, INSvi32lane>;
-defm : Neon_INS_elt_pattern<v2f64,  v1f64,  nxv2f64,  f64,  VectorIndexD, INSvi64lane>;
+defm : Neon_INS_elt_pattern<v8f16,  v4f16,  nxv8f16,  f16,  VectorIndexH, INSvi16lane, DUPi16, hsub>;
+defm : Neon_INS_elt_pattern<v8bf16, v4bf16, nxv8bf16, bf16, VectorIndexH, INSvi16lane, DUPi16, hsub>;
+defm : Neon_INS_elt_pattern<v4f32,  v2f32,  nxv4f32,  f32,  VectorIndexS, INSvi32lane, DUPi32, ssub>;
+defm : Neon_INS_elt_pattern<v2f64,  v1f64,  nxv2f64,  f64,  VectorIndexD, INSvi64lane, DUPi64, dsub>;
 
-defm : Neon_INS_elt_pattern<v16i8,  v8i8,   nxv16i8,  i32,  VectorIndexB, INSvi8lane>;
-defm : Neon_INS_elt_pattern<v8i16,  v4i16,  nxv8i16,  i32,  VectorIndexH, INSvi16lane>;
-defm : Neon_INS_elt_pattern<v4i32,  v2i32,  nxv4i32,  i32,  VectorIndexS, INSvi32lane>;
-defm : Neon_INS_elt_pattern<v2i64,  v1i64,  nxv2i64,  i64,  VectorIndexD, INSvi64lane>;
+defm : Neon_INS_elt_pattern<v16i8,  v8i8,   nxv16i8,  i32,  VectorIndexB, INSvi8lane,  DUPi8,  bsub>;
+defm : Neon_INS_elt_pattern<v8i16,  v4i16,  nxv8i16,  i32,  VectorIndexH, INSvi16lane, DUPi16, hsub>;
+defm : Neon_INS_elt_pattern<v4i32,  v2i32,  nxv4i32,  i32,  VectorIndexS, INSvi32lane, DUPi32, ssub>;
+defm : Neon_INS_elt_pattern<v2i64,  v1i64,  nxv2i64,  i64,  VectorIndexD, INSvi64lane, DUPi64, dsub>;
 
 // Insert from bitcast
 // vector_insert(bitcast(f32 src), n, lane) -> INSvi32lane(src, lane, INSERT_SUBREG(-, n), 0)
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index 40553aff04919..d3252ea54321e 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -497,7 +497,7 @@ def Q30   : AArch64Reg<30, "q30", [D30, D30_HI], ["v30", ""]>, DwarfRegAlias<B30
 def Q31   : AArch64Reg<31, "q31", [D31, D31_HI], ["v31", ""]>, DwarfRegAlias<B31>;
 }
 
-def FPR8  : RegisterClass<"AArch64", [i8], 8, (sequence "B%u", 0, 31)> {
+def FPR8  : RegisterClass<"AArch64", [i8, aarch64mfp8], 8, (sequence "B%u", 0, 31)> {
   let Size = 8;
   let DecoderMethod = "DecodeSimpleRegisterClass<AArch64::FPR8RegClassID, 0, 32>";
 }
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 4830aebc2739d..e5d99037b6c63 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3498,6 +3498,22 @@ let Predicates = [HasSVE_or_SME] in {
             (EXTRACT_SUBREG ZPR:$Zs, dsub)>;
   }
 
+  multiclass sve_insert_extract_elt<ValueType VT, ValueType VTScalar, Instruction DUP, Operand IdxTy> {
+    // NOP pattern (needed to avoid pointless DUPs being added by the second pattern).
+    def : Pat<(VT (vector_insert undef,
+                  (VTScalar (vector_extract VT:$vec, (i64 0))), (i64 0))),
+              (VT $vec)>;
+
+    def : Pat<(VT (vector_insert undef,
+                    (VTScalar (vector_extract VT:$vec, (i64 IdxTy:$Idx))), (i64 0))),
+              (DUP ZPR:$vec, IdxTy:$Idx)>;
+  }
+
+  defm : sve_insert_extract_elt<nxv16i8, i32, DUP_ZZI_B, sve_elm_idx_extdup_b>;
+  defm : sve_insert_extract_elt<nxv8i16, i32, DUP_ZZI_H, sve_elm_idx_extdup_h>;
+  defm : sve_insert_extract_elt<nxv4i32, i32, DUP_ZZI_S, sve_elm_idx_extdup_s>;
+  defm : sve_insert_extract_elt<nxv2i64, i64, DUP_ZZI_D, sve_elm_idx_extdup_d>;
+
   multiclass sve_predicated_add<SDNode extend, int value> {
     def : Pat<(nxv16i8 (add ZPR:$op, (extend nxv16i1:$pred))),
               (ADD_ZPmZ_B PPR:$pred, ZPR:$op, (DUP_ZI_B value, 0))>;
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sve-ldst-one.ll b/llvm/test/CodeGen/AArch64/aarch64-sve-ldst-one.ll
index d39c9bf760621..eb215898a7ad5 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sve-ldst-one.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sve-ldst-one.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -verify-machineinstrs -mattr=+sve -global-isel=0 | FileCheck %s --check-prefixes=CHECK,CHECK-NONSTREAMING
-; RUN: llc < %s -verify-machineinstrs -mattr=+sme -global-isel=0 -force-streaming | FileCheck %s --check-prefixes=CHECK,STREAMING-COMPAT
-; RUN: llc < %s -verify-machineinstrs -mattr=+sve -global-isel=0 -force-streaming-compatible | FileCheck %s --check-prefixes=CHECK,STREAMING-COMPAT
+; RUN: llc < %s -verify-machineinstrs -mattr=+sve -global-isel=0 | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mattr=+sme -global-isel=0 -force-streaming | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mattr=+sve -global-isel=0 -force-streaming-compatible | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -106,19 +106,11 @@ entry:
 }
 
 define void @test_str_lane_s8(ptr %a, <vscale x 16 x i8> %b) {
-; CHECK-NONSTREAMING-LABEL: test_str_lane_s8:
-; CHECK-NONSTREAMING:       // %bb.0: // %entry
-; CHECK-NONSTREAMING-NEXT:    umov w8, v0.b[7]
-; CHECK-NONSTREAMING-NEXT:    strb w8, [x0]
-; CHECK-NONSTREAMING-NEXT:    ret
-;
-; STREAMING-COMPAT-LABEL: test_str_lane_s8:
-; STREAMING-COMPAT:       // %bb.0: // %entry
-; STREAMING-COMPAT-NEXT:    mov z0.b, z0.b[7]
-; STREAMING-COMPAT-NEXT:    fmov w8, s0
-; STREAMING-COMPAT-NEXT:    strb w8, [x0]
-; STREAMING-COMPAT-NEXT:    ret
-
+; CHECK-LABEL: test_str_lane_s8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.b, z0.b[7]
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    ret
 entry:
   %0 = extractelement <vscale x 16 x i8> %b, i32 7
   store i8 %0, ptr %a, align 1
@@ -128,10 +120,8 @@ entry:
 define void @test_str_lane0_s8(ptr %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: test_str_lane0_s8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
-
 entry:
   %0 = extractelement <vscale x 16 x i8> %b, i32 0
   store i8 %0, ptr %a, align 1
@@ -201,6 +191,19 @@ define void @test_str_reduction_i32_to_i16(ptr %ptr, <vscale x 4 x i1> %p0, <vsc
   ret void
 }
 
+define void @test_str_reduction_i32_to_i8(ptr %ptr, <vscale x 4 x i1> %p0, <vscale x 4 x i32> %v) {
+; CHECK-LABEL: test_str_reduction_i32_to_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uaddv d0, p0, z0.s
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    ret
+
+  %reduce = tail call i64 @llvm.aarch64.sve.uaddv.nxv4i32(<vscale x 4 x i1> %p0, <vscale x 4 x i32> %v)
+  %trunc = trunc i64 %reduce to i8
+  store i8 %trunc, ptr %ptr, align 1
+  ret void
+}
+
 define void @test_str_reduction_i32_to_i32_negative_offset(ptr %ptr, <vscale x 4 x i1> %p0, <vscale x 4 x i32> %v) {
 ; CHECK-LABEL: test_str_reduction_i32_to_i32_negative_offset:
 ; CHECK:       // %bb.0:
@@ -242,6 +245,20 @@ define void @test_str_reduction_i32_to_i16_negative_offset(ptr %ptr, <vscale x 4
   ret void
 }
 
+define void @test_str_reduction_i32_to_i8_negative_offset(ptr %ptr, <vscale x 4 x i1> %p0, <vscale x 4 x i32> %v) {
+; CHECK-LABEL: test_str_reduction_i32_to_i8_negative_offset:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uaddv d0, p0, z0.s
+; CHECK-NEXT:    stur b0, [x0, #-8]
+; CHECK-NEXT:    ret
+
+  %reduce = tail call i64 @llvm.aarch64.sve.uaddv.nxv4i32(<vscale x 4 x i1> %p0, <vscale x 4 x i32> %v)
+  %trunc = trunc i64 %reduce to i8
+  %out_ptr = getelementptr inbounds i8, ptr %ptr, i64 -8
+  store i8 %trunc, ptr %out_ptr, align 1
+  ret void
+}
+
 define void @test_str_lane_s32_negative_offset(ptr %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: test_str_lane_s32_negative_offset:
 ; CHECK:       // %bb.0: // %entry
@@ -297,19 +314,11 @@ entry:
 }
 
 define void @test_str_lane_s8_negative_offset(ptr %a, <vscale x 16 x i8> %b) {
-; CHECK-NONSTREAMING-LABEL: test_str_lane_s8_negative_offset:
-; CHECK-NONSTREAMING:       // %bb.0: // %entry
-; CHECK-NONSTREAMING-NEXT:    umov w8, v0.b[7]
-; CHECK-NONSTREAMING-NEXT:    sturb w8, [x0, #-8]
-; CHECK-NONSTREAMING-NEXT:    ret
-;
-; STREAMING-COMPAT-LABEL: test_str_lane_s8_negative_offset:
-; STREAMING-COMPAT:       // %bb.0: // %entry
-; STREAMING-COMPAT-NEXT:    mov z0.b, z0.b[7]
-; STREAMING-COMPAT-NEXT:    fmov w8, s0
-; STREAMING-COMPAT-NEXT:    sturb w8, [x0, #-8]
-; STREAMING-COMPAT-NEXT:    ret
-
+; CHECK-LABEL: test_str_lane_s8_negative_offset:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.b, z0.b[7]
+; CHECK-NEXT:    stur b0, [x0, #-8]
+; CHECK-NEXT:    ret
 entry:
   %0 = extractelement <vscale x 16 x i8> %b, i32 7
   %out_ptr = getelementptr inbounds i8, ptr %a, i64 -8
@@ -320,10 +329,8 @@ entry:
 define void @test_str_lane0_s8_negative_offset(ptr %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: test_str_lane0_s8_negative_offset:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    sturb w8, [x0, #-8]
+; CHECK-NEXT:    stur b0, [x0, #-8]
 ; CHECK-NEXT:    ret
-
 entry:
   %0 = extractelement <vscale x 16 x i8> %b, i32 0
   %out_ptr = getelementptr inbounds i8, ptr %a, i64 -8
@@ -385,6 +392,47 @@ entry:
   ret void
 }
 
+
+define void @test_str_trunc_lane_s32_to_s8(ptr %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: test_str_trunc_lane_s32_to_s8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, z0.s[3]
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = extractelement <vscale x 4 x i32> %b, i32 3
+  %trunc = trunc i32 %0 to i8
+  store i8 %trunc, ptr %a, align 1
+  ret void
+}
+
+define void @test_str_trunc_lane0_s32_to_s8(ptr %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: test_str_trunc_lane0_s32_to_s8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    ret
+
+entry:
+  %0 = extractelement <vscale x 4 x i32> %b, i32 0
+  %trunc = trunc i32 %0 to i8
+  store i8 %trunc, ptr %a, align 1
+  ret void
+}
+
+define void @test_str_trunc_lane_s64_to_s8(ptr %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: test_str_trunc_lane_s64_to_s8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z0.d[3]
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    ret
+
+entry:
+  %0 = extractelement <vscale x 2 x i64> %b, i32 3
+  %trunc = trunc i64 %0 to i8
+  store i8 %trunc, ptr %a, align 1
+  ret void
+}
+
 define void @test_str_trunc_lane_s32_to_s16_negative_offset(ptr %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: test_str_trunc_lane_s32_to_s16_negative_offset:
 ; CHECK:       // %bb.0: // %entry
@@ -413,3 +461,46 @@ entry:
   store i16 %trunc, ptr %out_ptr, align 2
   ret void
 }
+
+define void @test_str_trunc_lane_s32_to_s8_negative_offset(ptr %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: test_str_trunc_lane_s32_to_s8_negative_offset:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.s, z0.s[3]
+; CHECK-NEXT:    stur b0, [x0, #-8]
+; CHECK-NEXT:    ret
+entry:
+  %0 = extractelement <vscale x 4 x i32> %b, i32 3
+  %trunc = trunc i32 %0 to i8
+  %out_ptr = getelementptr inbounds i8, ptr %a, i64 -8
+  store i8 %trunc, ptr %out_ptr, align 1
+  ret void
+}
+
+define void @test_str_trunc_lane0_s32_to_s8_negative_offset(ptr %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: test_str_trunc_lane0_s32_to_s8_negative_offset:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stur b0, [x0, #-8]
+; CHECK-NEXT:    ret
+
+entry:
+  %0 = extractelement <vscale x 4 x i32> %b, i32 0
+  %trunc = trunc i32 %0 to i8
+  %out_ptr = getelementptr inbounds i8, ptr %a, i64 -8
+  store i8 %trunc, ptr %out_ptr, align 1
+  ret void
+}
+
+define void @test_str_trunc_lane_s64_to_s8_negative_offset(ptr %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: test_str_trunc_lane_s64_to_s8_negative_offset:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, z0.d[3]
+; CHECK-NEXT:    stur b0, [x0, #-8]
+; CHECK-NEXT:    ret
+
+entry:
+  %0 = extractelement <vscale x 2 x i64> %b, i32 3
+  %trunc = trunc i64 %0 to i8
+  %out_ptr = getelementptr inbounds i8, ptr %a, i64 -8
+  store i8 %trunc, ptr %out_ptr, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/add.ll b/llvm/test/CodeGen/AArch64/add.ll
index fc0ba336b21cc..d5bd1b712a2a6 100644
--- a/llvm/test/CodeGen/AArch64/add.ll
+++ b/llvm/test/CodeGen/AArch64/add.ll
@@ -63,10 +63,9 @@ define void @v2i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x9]
 ; CHECK-SD-NEXT:    add v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -101,11 +100,11 @@ define void @v3i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    zip1 v1.8b, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    add v0.4h, v0.4h, v1.4h
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -263,10 +262,10 @@ define void @v3i16(ptr %p1, ptr %p2) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr d0, [x0]
 ; CHECK-SD-NEXT:    ldr d1, [x1]
-; CHECK-SD-NEXT:    add x8, x0, #4
 ; CHECK-SD-NEXT:    add v0.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h1, v0.h[2]
 ; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    str h1, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v3i16:
diff --git a/llvm/test/CodeGen/AArch64/andorxor.ll b/llvm/test/CodeGen/AArch64/andorxor.ll
index 0384848082caa..f7df1092287bd 100644
--- a/llvm/test/CodeGen/AArch64/andorxor.ll
+++ b/llvm/test/CodeGen/AArch64/andorxor.ll
@@ -183,10 +183,9 @@ define void @and_v2i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x9]
 ; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: and_v2i8:
@@ -220,10 +219,9 @@ define void @or_v2i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x9]
 ; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: or_v2i8:
@@ -257,10 +255,9 @@ define void @xor_v2i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x9]
 ; CHECK-SD-NEXT:    eor v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: xor_v2i8:
@@ -295,11 +292,11 @@ define void @and_v3i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    zip1 v1.8b, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -343,11 +340,11 @@ define void @or_v3i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    zip1 v1.8b, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -391,11 +388,11 @@ define void @xor_v3i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    zip1 v1.8b, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    eor v0.8b, v0.8b, v1.8b
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -802,8 +799,8 @@ define void @and_v3i16(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    and x8, x8, x9
 ; CHECK-SD-NEXT:    fmov d0, x8
 ; CHECK-SD-NEXT:    str w8, [x0]
-; CHECK-SD-NEXT:    add x8, x0, #4
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
+; CHECK-SD-NEXT:    str h0, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: and_v3i16:
@@ -839,8 +836,8 @@ define void @or_v3i16(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    orr x8, x8, x9
 ; CHECK-SD-NEXT:    fmov d0, x8
 ; CHECK-SD-NEXT:    str w8, [x0]
-; CHECK-SD-NEXT:    add x8, x0, #4
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
+; CHECK-SD-NEXT:    str h0, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: or_v3i16:
@@ -876,8 +873,8 @@ define void @xor_v3i16(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    eor x8, x8, x9
 ; CHECK-SD-NEXT:    fmov d0, x8
 ; CHECK-SD-NEXT:    str w8, [x0]
-; CHECK-SD-NEXT:    add x8, x0, #4
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
+; CHECK-SD-NEXT:    str h0, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: xor_v3i16:
diff --git a/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll b/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
index 2c065e0051cd7..7f2bebf584d8f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-collect-loh.ll
@@ -615,11 +615,10 @@ define <1 x i8> @getL() {
 ; CHECK-NEXT: ; kill
 ; CHECK-NEXT: [[LDRGOT_LABEL:Lloh[0-9]+]]:
 ; CHECK-NEXT: ldr {{[xw]}}[[LDRGOT_REG:[0-9]+]], [[[ADRP_REG]], _L@GOTPAGEOFF]
-; Ultimately we should generate str b0, but right now, we match the vector
-; variant which does not allow to fold the immediate into the store.
-; CHECK-NEXT: st1.b { v0 }[0], [x[[LDRGOT_REG]]]
+; CHECK-NEXT: [[STR_LABEL:Lloh[0-9]+]]:
+; CHECK-NEXT: str b0, [x[[LDRGOT_REG]]]
 ; CHECK-NEXT: ret
-; CHECK: .loh AdrpLdrGot [[ADRP_LABEL]], [[LDRGOT_LABEL]]
+; CHECK: .loh AdrpLdrGotStr [[ADRP_LABEL]], [[LDRGOT_LABEL]], [[STR_LABEL]]
 define void @setL(<1 x i8> %t) {
   store <1 x i8> %t, ptr @L, align 4
   ret void
@@ -678,6 +677,6 @@ if.end.i:
   call void (ptr, ...) @callee(ptr @.str.89, ptr @.str.90, double %sub)
   unreachable
 }
-declare void @callee(ptr nocapture readonly, ...) 
+declare void @callee(ptr nocapture readonly, ...)
 
 attributes #0 = { "target-cpu"="cyclone" }
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
index c0d91c1e0c836..2a085dc0e72bf 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
@@ -2062,7 +2062,7 @@ define <8 x i16> @test_concat_v8i16_v8i16_v4i16(<8 x i16> %x, <4 x i16> %y) #0 {
 ;
 ; CHECK-GI-LABEL: test_concat_v8i16_v8i16_v4i16:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v2.h[0], v0.h[0]
+; CHECK-GI-NEXT:    mov h2, v0.h[0]
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-GI-NEXT:    mov v2.h[1], v0.h[1]
 ; CHECK-GI-NEXT:    mov v2.h[2], v0.h[2]
@@ -2189,7 +2189,7 @@ define <4 x i32> @test_concat_v4i32_v4i32_v2i32(<4 x i32> %x, <2 x i32> %y) #0 {
 ;
 ; CHECK-GI-LABEL: test_concat_v4i32_v4i32_v2i32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v2.s[0], v0.s[0]
+; CHECK-GI-NEXT:    mov s2, v0.s[0]
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-GI-NEXT:    mov v2.s[1], v0.s[1]
 ; CHECK-GI-NEXT:    mov v2.s[2], v1.s[0]
@@ -2252,7 +2252,7 @@ define <2 x i64> @test_concat_v2i64_v2i64_v1i64(<2 x i64> %x, <1 x i64> %y) #0 {
 ;
 ; CHECK-GI-LABEL: test_concat_v2i64_v2i64_v1i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.d[0], v0.d[0]
+; CHECK-GI-NEXT:    mov d0, v0.d[0]
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-GI-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-simd-ldst-one.ll b/llvm/test/CodeGen/AArch64/arm64-neon-simd-ldst-one.ll
index 7d87be0ce8e1c..7721616be436c 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-simd-ldst-one.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-simd-ldst-one.ll
@@ -674,16 +674,10 @@ entry:
 }
 
 define void @test_vst1_lane_s64(ptr %a, <1 x i64> %b) {
-; CHECK-GI-LABEL: test_vst1_lane_s64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    str d0, [x0]
-; CHECK-GI-NEXT:    ret
-;
-; CHECK-SD-LABEL: test_vst1_lane_s64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str d0, [x0]
-; CHECK-SD-NEXT:    ret
+; CHECK-LABEL: test_vst1_lane_s64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
 entry:
   %0 = extractelement <1 x i64> %b, i32 0
   store i64 %0, ptr %a, align 8
diff --git a/llvm/test/CodeGen/AArch64/arm64-rev.ll b/llvm/test/CodeGen/AArch64/arm64-rev.ll
index 14ab7b5108125..6bdd5f998a3b9 100644
--- a/llvm/test/CodeGen/AArch64/arm64-rev.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-rev.ll
@@ -462,9 +462,9 @@ define void @test_vrev64(ptr nocapture %source, ptr nocapture %dst) nounwind ssp
 ; CHECK-SD-LABEL: test_vrev64:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr q0, [x0]
-; CHECK-SD-NEXT:    add x8, x1, #2
-; CHECK-SD-NEXT:    st1.h { v0 }[5], [x8]
+; CHECK-SD-NEXT:    mov h1, v0[5]
 ; CHECK-SD-NEXT:    st1.h { v0 }[6], [x1]
+; CHECK-SD-NEXT:    str h1, [x1, #2]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: test_vrev64:
diff --git a/llvm/test/CodeGen/AArch64/arm64-st1.ll b/llvm/test/CodeGen/AArch64/arm64-st1.ll
index 02797f3ed186c..d6abf829bc989 100644
--- a/llvm/test/CodeGen/AArch64/arm64-st1.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-st1.ll
@@ -5,10 +5,17 @@
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs -mcpu=exynos-m3 | FileCheck --check-prefixes=CHECK,EXYNOS %s
 
 define void @st1lane_16b(<16 x i8> %A, ptr %D) {
-; CHECK-LABEL: st1lane_16b:
-; CHECK:    add x8, x0, #1
-; CHECK:    st1.b { v0 }[1], [x8]
-
+; SD-CHECK-LABEL: st1lane_16b:
+; SD-CHECK:    mov b0, v0[1]
+; SD-CHECK:    stur b0, [x0, #1]
+;
+; GI-CHECK-LABEL: st1lane_16b:
+; GI-CHECK:    add x8, x0, #1
+; GI-CHECK:    st1.b { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_16b:
+; EXYNOS:    mov b0, v0[1]
+; EXYNOS:    stur b0, [x0, #1]
   %ptr = getelementptr i8, ptr %D, i64 1
   %tmp = extractelement <16 x i8> %A, i32 1
   store i8 %tmp, ptr %ptr
@@ -16,9 +23,15 @@ define void @st1lane_16b(<16 x i8> %A, ptr %D) {
 }
 
 define void @st1lane0_16b(<16 x i8> %A, ptr %D) {
-; CHECK-LABEL: st1lane0_16b:
-; CHECK:    add x8, x0, #1
-; CHECK:    st1.b { v0 }[0], [x8]
+; SD-CHECK-LABEL: st1lane0_16b:
+; SD-CHECK:    stur b0, [x0, #1]
+;
+; GI-CHECK-LABEL: st1lane0_16b:
+; GI-CHECK:    add x8, x0, #1
+; GI-CHECK:    st1.b { v0 }[0], [x8]
+;
+; EXYNOS-LABEL: st1lane0_16b:
+; EXYNOS:    stur b0, [x0, #1]
 
   %ptr = getelementptr i8, ptr %D, i64 1
   %tmp = extractelement <16 x i8> %A, i32 0
@@ -27,10 +40,15 @@ define void @st1lane0_16b(<16 x i8> %A, ptr %D) {
 }
 
 define void @st1lane0u_16b(<16 x i8> %A, ptr %D) {
-; CHECK-LABEL: st1lane0u_16b:
-; CHECK:    sub x8, x0, #1
-; CHECK:    st1.b { v0 }[0], [x8]
-
+; SD-CHECK-LABEL: st1lane0u_16b:
+; SD-CHECK:    stur b0, [x0, #-1]
+;
+; GI-CHECK-LABEL: st1lane0u_16b:
+; GI-CHECK:    sub x8, x0, #1
+; GI-CHECK:    st1.b { v0 }[0], [x8]
+;
+; EXYNOS-LABEL: st1lane0u_16b:
+; EXYNOS:    stur b0, [x0, #-1]
   %ptr = getelementptr i8, ptr %D, i64 -1
   %tmp = extractelement <16 x i8> %A, i32 0
   store i8 %tmp, ptr %ptr
@@ -38,10 +56,17 @@ define void @st1lane0u_16b(<16 x i8> %A, ptr %D) {
 }
 
 define void @st1lane_ro_16b(<16 x i8> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane_ro_16b:
-; CHECK:    add x8, x0, x1
-; CHECK:    st1.b { v0 }[1], [x8]
-
+; SD-CHECK-LABEL: st1lane_ro_16b:
+; SD-CHECK:    mov b0, v0[1]
+; SD-CHECK:    str b0, [x0, x1]
+;
+; GI-CHECK-LABEL: st1lane_ro_16b:
+; GI-CHECK:    add x8, x0, x1
+; GI-CHECK:    st1.b { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_ro_16b:
+; EXYNOS:    mov b0, v0[1]
+; EXYNOS:    str b0, [x0, x1]
   %ptr = getelementptr i8, ptr %D, i64 %offset
   %tmp = extractelement <16 x i8> %A, i32 1
   store i8 %tmp, ptr %ptr
@@ -49,10 +74,15 @@ define void @st1lane_ro_16b(<16 x i8> %A, ptr %D, i64 %offset) {
 }
 
 define void @st1lane0_ro_16b(<16 x i8> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane0_ro_16b:
-; CHECK:    add x8, x0, x1
-; CHECK:    st1.b { v0 }[0], [x8]
-
+; SD-CHECK-LABEL: st1lane0_ro_16b:
+; SD-CHECK:    str b0, [x0, x1]
+;
+; GI-CHECK-LABEL: st1lane0_ro_16b:
+; GI-CHECK:    add x8, x0, x1
+; GI-CHECK:    st1.b { v0 }[0], [x8]
+;
+; EXYNOS-LABEL: st1lane0_ro_16b:
+; EXYNOS:    str b0, [x0, x1]
   %ptr = getelementptr i8, ptr %D, i64 %offset
   %tmp = extractelement <16 x i8> %A, i32 0
   store i8 %tmp, ptr %ptr
@@ -60,9 +90,17 @@ define void @st1lane0_ro_16b(<16 x i8> %A, ptr %D, i64 %offset) {
 }
 
 define void @st1lane_8h(<8 x i16> %A, ptr %D) {
-; CHECK-LABEL: st1lane_8h:
-; CHECK:    add x8, x0, #2
-; CHECK:    st1.h { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_8h:
+; SD-CHECK:    mov h0, v0[1]
+; SD-CHECK:    str h0, [x0, #2]
+;
+; GI-CHECK-LABEL: st1lane_8h:
+; GI-CHECK:    add x8, x0, #2
+; GI-CHECK:    st1.h { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_8h:
+; EXYNOS:    mov h0, v0[1]
+; EXYNOS:    str h0, [x0, #2]
   %ptr = getelementptr i16, ptr %D, i64 1
   %tmp = extractelement <8 x i16> %A, i32 1
   store i16 %tmp, ptr %ptr
@@ -88,9 +126,17 @@ define void @st1lane0u_8h(<8 x i16> %A, ptr %D) {
 }
 
 define void @st1lane_ro_8h(<8 x i16> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane_ro_8h:
-; CHECK:    add x8, x0, x1, lsl #1
-; CHECK:    st1.h { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_ro_8h:
+; SD-CHECK:    mov h0, v0[1]
+; SD-CHECK:    str h0, [x0, x1, lsl #1]
+;
+; GI-CHECK-LABEL: st1lane_ro_8h:
+; GI-CHECK:    add x8, x0, x1, lsl #1
+; GI-CHECK:    st1.h { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_ro_8h:
+; EXYNOS:    mov h0, v0[1]
+; EXYNOS:    str h0, [x0, x1, lsl #1]
   %ptr = getelementptr i16, ptr %D, i64 %offset
   %tmp = extractelement <8 x i16> %A, i32 1
   store i16 %tmp, ptr %ptr
@@ -107,9 +153,17 @@ define void @st1lane0_ro_8h(<8 x i16> %A, ptr %D, i64 %offset) {
 }
 
 define void @st1lane_4s(<4 x i32> %A, ptr %D) {
-; CHECK-LABEL: st1lane_4s:
-; CHECK:    add x8, x0, #4
-; CHECK:    st1.s { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_4s:
+; SD-CHECK:    mov s0, v0[1]
+; SD-CHECK:    str s0, [x0, #4]
+;
+; GI-CHECK-LABEL: st1lane_4s:
+; GI-CHECK:    add x8, x0, #4
+; GI-CHECK:    st1.s { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_4s:
+; EXYNOS:    mov s0, v0[1]
+; EXYNOS:    str s0, [x0, #4]
   %ptr = getelementptr i32, ptr %D, i64 1
   %tmp = extractelement <4 x i32> %A, i32 1
   store i32 %tmp, ptr %ptr
@@ -135,9 +189,17 @@ define void @st1lane0u_4s(<4 x i32> %A, ptr %D) {
 }
 
 define void @st1lane_ro_4s(<4 x i32> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane_ro_4s:
-; CHECK:    add x8, x0, x1, lsl #2
-; CHECK:    st1.s { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_ro_4s:
+; SD-CHECK:    mov s0, v0[1]
+; SD-CHECK:    str s0, [x0, x1, lsl #2]
+;
+; GI-CHECK-LABEL: st1lane_ro_4s:
+; GI-CHECK:    add x8, x0, x1, lsl #2
+; GI-CHECK:    st1.s { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_ro_4s:
+; EXYNOS:    mov s0, v0[1]
+; EXYNOS:    str s0, [x0, x1, lsl #2]
   %ptr = getelementptr i32, ptr %D, i64 %offset
   %tmp = extractelement <4 x i32> %A, i32 1
   store i32 %tmp, ptr %ptr
@@ -201,9 +263,17 @@ define void @st1lane0_ro_4s_float(<4 x float> %A, ptr %D, i64 %offset) {
 }
 
 define void @st1lane_2d(<2 x i64> %A, ptr %D) {
-; CHECK-LABEL: st1lane_2d:
-; CHECK:    add x8, x0, #8
-; CHECK:    st1.d { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_2d:
+; SD-CHECK:    mov d0, v0[1]
+; SD-CHECK:    str d0, [x0, #8]
+;
+; GI-CHECK-LABEL: st1lane_2d:
+; GI-CHECK:    add x8, x0, #8
+; GI-CHECK:    st1.d { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_2d:
+; EXYNOS:    mov d0, v0[1]
+; EXYNOS:    str d0, [x0, #8]
   %ptr = getelementptr i64, ptr %D, i64 1
   %tmp = extractelement <2 x i64> %A, i32 1
   store i64 %tmp, ptr %ptr
@@ -229,9 +299,17 @@ define void @st1lane0u_2d(<2 x i64> %A, ptr %D) {
 }
 
 define void @st1lane_ro_2d(<2 x i64> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane_ro_2d:
-; CHECK:    add x8, x0, x1, lsl #3
-; CHECK:    st1.d { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_ro_2d:
+; SD-CHECK:    mov d0, v0[1]
+; SD-CHECK:    str d0, [x0, x1, lsl #3]
+;
+; GI-CHECK-LABEL: st1lane_ro_2d:
+; GI-CHECK:    add x8, x0, x1, lsl #3
+; GI-CHECK:    st1.d { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_ro_2d:
+; EXYNOS:    mov d0, v0[1]
+; EXYNOS:    str d0, [x0, x1, lsl #3]
   %ptr = getelementptr i64, ptr %D, i64 %offset
   %tmp = extractelement <2 x i64> %A, i32 1
   store i64 %tmp, ptr %ptr
@@ -295,10 +373,17 @@ define void @st1lane0_ro_2d_double(<2 x double> %A, ptr %D, i64 %offset) {
 }
 
 define void @st1lane_8b(<8 x i8> %A, ptr %D) {
-; CHECK-LABEL: st1lane_8b:
-; CHECK:    add x8, x0, #1
-; CHECK:    st1.b { v0 }[1], [x8]
-
+; SD-CHECK-LABEL: st1lane_8b:
+; SD-CHECK:    mov b0, v0[1]
+; SD-CHECK:    stur b0, [x0, #1]
+;
+; GI-CHECK-LABEL: st1lane_8b:
+; GI-CHECK:    add x8, x0, #1
+; GI-CHECK:    st1.b { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_8b:
+; EXYNOS:    mov b0, v0[1]
+; EXYNOS:    stur b0, [x0, #1]
   %ptr = getelementptr i8, ptr %D, i64 1
   %tmp = extractelement <8 x i8> %A, i32 1
   store i8 %tmp, ptr %ptr
@@ -306,10 +391,17 @@ define void @st1lane_8b(<8 x i8> %A, ptr %D) {
 }
 
 define void @st1lane_ro_8b(<8 x i8> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane_ro_8b:
-; CHECK:    add x8, x0, x1
-; CHECK:    st1.b { v0 }[1], [x8]
-
+; SD-CHECK-LABEL: st1lane_ro_8b:
+; SD-CHECK:    mov b0, v0[1]
+; SD-CHECK:    str b0, [x0, x1]
+;
+; GI-CHECK-LABEL: st1lane_ro_8b:
+; GI-CHECK:    add x8, x0, x1
+; GI-CHECK:    st1.b { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_ro_8b:
+; EXYNOS:    mov b0, v0[1]
+; EXYNOS:    str b0, [x0, x1]
   %ptr = getelementptr i8, ptr %D, i64 %offset
   %tmp = extractelement <8 x i8> %A, i32 1
   store i8 %tmp, ptr %ptr
@@ -317,10 +409,15 @@ define void @st1lane_ro_8b(<8 x i8> %A, ptr %D, i64 %offset) {
 }
 
 define void @st1lane0_ro_8b(<8 x i8> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane0_ro_8b:
-; CHECK:    add x8, x0, x1
-; CHECK:    st1.b { v0 }[0], [x8]
-
+; SD-CHECK-LABEL: st1lane0_ro_8b:
+; SD-CHECK:    str b0, [x0, x1]
+;
+; GI-CHECK-LABEL: st1lane0_ro_8b:
+; GI-CHECK:    add x8, x0, x1
+; GI-CHECK:    st1.b { v0 }[0], [x8]
+;
+; EXYNOS-LABEL: st1lane0_ro_8b:
+; EXYNOS:    str b0, [x0, x1]
   %ptr = getelementptr i8, ptr %D, i64 %offset
   %tmp = extractelement <8 x i8> %A, i32 0
   store i8 %tmp, ptr %ptr
@@ -328,9 +425,17 @@ define void @st1lane0_ro_8b(<8 x i8> %A, ptr %D, i64 %offset) {
 }
 
 define void @st1lane_4h(<4 x i16> %A, ptr %D) {
-; CHECK-LABEL: st1lane_4h:
-; CHECK:    add x8, x0, #2
-; CHECK:    st1.h { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_4h:
+; SD-CHECK:    mov h0, v0[1]
+; SD-CHECK:    str h0, [x0, #2]
+;
+; GI-CHECK-LABEL: st1lane_4h:
+; GI-CHECK:    add x8, x0, #2
+; GI-CHECK:    st1.h { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_4h:
+; EXYNOS:    mov h0, v0[1]
+; EXYNOS:    str h0, [x0, #2]
   %ptr = getelementptr i16, ptr %D, i64 1
   %tmp = extractelement <4 x i16> %A, i32 1
   store i16 %tmp, ptr %ptr
@@ -356,9 +461,17 @@ define void @st1lane0u_4h(<4 x i16> %A, ptr %D) {
 }
 
 define void @st1lane_ro_4h(<4 x i16> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane_ro_4h:
-; CHECK:    add x8, x0, x1, lsl #1
-; CHECK:    st1.h { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_ro_4h:
+; SD-CHECK:    mov h0, v0[1]
+; SD-CHECK:    str h0, [x0, x1, lsl #1]
+;
+; GI-CHECK-LABEL: st1lane_ro_4h:
+; GI-CHECK:    add x8, x0, x1, lsl #1
+; GI-CHECK:    st1.h { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_ro_4h:
+; EXYNOS:    mov h0, v0[1]
+; EXYNOS:    str h0, [x0, x1, lsl #1]
   %ptr = getelementptr i16, ptr %D, i64 %offset
   %tmp = extractelement <4 x i16> %A, i32 1
   store i16 %tmp, ptr %ptr
@@ -375,9 +488,17 @@ define void @st1lane0_ro_4h(<4 x i16> %A, ptr %D, i64 %offset) {
 }
 
 define void @st1lane_2s(<2 x i32> %A, ptr %D) {
-; CHECK-LABEL: st1lane_2s:
-; CHECK:    add x8, x0, #4
-; CHECK:    st1.s { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_2s:
+; SD-CHECK:    mov s0, v0[1]
+; SD-CHECK:    str s0, [x0, #4]
+;
+; GI-CHECK-LABEL: st1lane_2s:
+; GI-CHECK:    add x8, x0, #4
+; GI-CHECK:    st1.s { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_2s:
+; EXYNOS:    mov s0, v0[1]
+; EXYNOS:    str s0, [x0, #4]
   %ptr = getelementptr i32, ptr %D, i64 1
   %tmp = extractelement <2 x i32> %A, i32 1
   store i32 %tmp, ptr %ptr
@@ -403,9 +524,17 @@ define void @st1lane0u_2s(<2 x i32> %A, ptr %D) {
 }
 
 define void @st1lane_ro_2s(<2 x i32> %A, ptr %D, i64 %offset) {
-; CHECK-LABEL: st1lane_ro_2s:
-; CHECK:    add x8, x0, x1, lsl #2
-; CHECK:    st1.s { v0 }[1], [x8]
+; SD-CHECK-LABEL: st1lane_ro_2s:
+; SD-CHECK:    mov s0, v0[1]
+; SD-CHECK:    str s0, [x0, x1, lsl #2]
+;
+; GI-CHECK-LABEL: st1lane_ro_2s:
+; GI-CHECK:    add x8, x0, x1, lsl #2
+; GI-CHECK:    st1.s { v0 }[1], [x8]
+;
+; EXYNOS-LABEL: st1lane_ro_2s:
+; EXYNOS:    mov s0, v0[1]
+; EXYNOS:    str s0, [x0, x1, lsl #2]
   %ptr = getelementptr i32, ptr %D, i64 %offset
   %tmp = extractelement <2 x i32> %A, i32 1
   store i32 %tmp, ptr %ptr
diff --git a/llvm/test/CodeGen/AArch64/bitcast-v2i8.ll b/llvm/test/CodeGen/AArch64/bitcast-v2i8.ll
index aff3ffc70a711..2866214e1e473 100644
--- a/llvm/test/CodeGen/AArch64/bitcast-v2i8.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast-v2i8.ll
@@ -1,14 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck %s
 
 ; Part of PR21549: going through the stack isn't ideal but is correct.
 
 define i16 @test_bitcast_v2i8_to_i16(<2 x i8> %a) {
-; CHECK-LABEL: test_bitcast_v2i8_to_i16
-; CHECK:      mov.s   [[WREG_HI:w[0-9]+]], v0[1]
-; CHECK-NEXT: fmov    [[WREG_LO:w[0-9]+]], s0
-; CHECK-NEXT: strb    [[WREG_HI]], [sp, #15]
-; CHECK-NEXT: strb    [[WREG_LO]], [sp, #14]
-; CHECK-NEXT: ldrh    w0, [sp, #14]
+; CHECK-LABEL: test_bitcast_v2i8_to_i16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov s1, v0[1]
+; CHECK-NEXT:    str b0, [sp, #14]
+; CHECK-NEXT:    stur b1, [sp, #15]
+; CHECK-NEXT:    ldrh w0, [sp, #14]
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
 
   %aa = bitcast <2 x i8> %a to i16
   ret i16 %aa
diff --git a/llvm/test/CodeGen/AArch64/concat-vector.ll b/llvm/test/CodeGen/AArch64/concat-vector.ll
index 0daa6e7f16202..05a40453833ee 100644
--- a/llvm/test/CodeGen/AArch64/concat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/concat-vector.ll
@@ -480,7 +480,7 @@ define <2 x i64> @concat_high_high_v2i64(<2 x i64> %a_vec, <2 x i64> %b_vec) {
 ;
 ; CHECK-GI-LABEL: concat_high_high_v2i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.d[0], v0.d[1]
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
 ; CHECK-GI-NEXT:    mov v0.d[1], v1.d[1]
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -498,7 +498,7 @@ define <2 x double> @concat_high_high_v2f64(<2 x double> %a_vec, <2 x double> %b
 ;
 ; CHECK-GI-LABEL: concat_high_high_v2f64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.d[0], v0.d[1]
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
 ; CHECK-GI-NEXT:    mov v0.d[1], v1.d[1]
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/ctlz.ll b/llvm/test/CodeGen/AArch64/ctlz.ll
index 742433c50d390..fcd1fa2983420 100644
--- a/llvm/test/CodeGen/AArch64/ctlz.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz.ll
@@ -13,10 +13,9 @@ define void @v2i8(ptr %p1) {
 ; CHECK-SD-NEXT:    mov v1.s[1], w9
 ; CHECK-SD-NEXT:    clz v1.2s, v1.2s
 ; CHECK-SD-NEXT:    sub v0.2s, v1.2s, v0.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -49,11 +48,11 @@ define void @v3i8(ptr %p1) {
 ; CHECK-SD-NEXT:    clz v1.4h, v1.4h
 ; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -179,10 +178,10 @@ define void @v3i16(ptr %p1) {
 ; CHECK-SD-LABEL: v3i16:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr d0, [x0]
-; CHECK-SD-NEXT:    add x8, x0, #4
 ; CHECK-SD-NEXT:    clz v0.4h, v0.4h
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h1, v0.h[2]
 ; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    str h1, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v3i16:
diff --git a/llvm/test/CodeGen/AArch64/ctpop.ll b/llvm/test/CodeGen/AArch64/ctpop.ll
index c7c378d3e67cd..10ec1d0c1982a 100644
--- a/llvm/test/CodeGen/AArch64/ctpop.ll
+++ b/llvm/test/CodeGen/AArch64/ctpop.ll
@@ -13,10 +13,9 @@ define void @v2i8(ptr %p1) {
 ; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
 ; CHECK-SD-NEXT:    uaddlp v0.4h, v0.8b
 ; CHECK-SD-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -48,11 +47,11 @@ define void @v3i8(ptr %p1) {
 ; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
 ; CHECK-SD-NEXT:    uaddlp v0.4h, v0.8b
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -178,11 +177,11 @@ define void @v3i16(ptr %p1) {
 ; CHECK-SD-LABEL: v3i16:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr d0, [x0]
-; CHECK-SD-NEXT:    add x8, x0, #4
 ; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
 ; CHECK-SD-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h1, v0.h[2]
 ; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    str h1, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v3i16:
diff --git a/llvm/test/CodeGen/AArch64/cttz.ll b/llvm/test/CodeGen/AArch64/cttz.ll
index 41843e03cb81e..60125f8a19811 100644
--- a/llvm/test/CodeGen/AArch64/cttz.ll
+++ b/llvm/test/CodeGen/AArch64/cttz.ll
@@ -16,10 +16,9 @@ define void @v2i8(ptr %p1) {
 ; CHECK-SD-NEXT:    movi v1.2s, #32
 ; CHECK-SD-NEXT:    clz v0.2s, v0.2s
 ; CHECK-SD-NEXT:    sub v0.2s, v1.2s, v0.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -59,11 +58,11 @@ define void @v3i8(ptr %p1) {
 ; CHECK-SD-NEXT:    clz v0.4h, v0.4h
 ; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -263,14 +262,14 @@ define void @v3i16(ptr %p1) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    movi v0.4h, #1
 ; CHECK-SD-NEXT:    ldr d1, [x0]
-; CHECK-SD-NEXT:    add x8, x0, #4
 ; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
 ; CHECK-SD-NEXT:    bic v0.8b, v0.8b, v1.8b
 ; CHECK-SD-NEXT:    movi v1.4h, #16
 ; CHECK-SD-NEXT:    clz v0.4h, v0.4h
 ; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h1, v0.h[2]
 ; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    str h1, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v3i16:
diff --git a/llvm/test/CodeGen/AArch64/dp1.ll b/llvm/test/CodeGen/AArch64/dp1.ll
index 4f48aac72ebc3..e904f4b6d247a 100644
--- a/llvm/test/CodeGen/AArch64/dp1.ll
+++ b/llvm/test/CodeGen/AArch64/dp1.ll
@@ -205,8 +205,7 @@ define void @ctpop_i32() {
 ; CHECK-SDAG-NEXT:    fmov d0, x9
 ; CHECK-SDAG-NEXT:    cnt v0.8b, v0.8b
 ; CHECK-SDAG-NEXT:    addv b0, v0.8b
-; CHECK-SDAG-NEXT:    fmov w9, s0
-; CHECK-SDAG-NEXT:    str w9, [x8]
+; CHECK-SDAG-NEXT:    str s0, [x8]
 ; CHECK-SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: ctpop_i32:
diff --git a/llvm/test/CodeGen/AArch64/extract-vector-cmp.ll b/llvm/test/CodeGen/AArch64/extract-vector-cmp.ll
index 8345fdfa46b4c..f076ee12427d8 100644
--- a/llvm/test/CodeGen/AArch64/extract-vector-cmp.ll
+++ b/llvm/test/CodeGen/AArch64/extract-vector-cmp.ll
@@ -184,17 +184,16 @@ define i1 @extract_icmp_v4i32_splat_rhs_mul_use(<4 x i32> %a, ptr %p) {
 ; CHECK-LABEL: extract_icmp_v4i32_splat_rhs_mul_use:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #235
-; CHECK-NEXT:    adrp x9, .LCPI8_0
+; CHECK-NEXT:    adrp x8, .LCPI8_0
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI8_0]
 ; CHECK-NEXT:    mov x8, x0
-; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI8_0]
 ; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    xtn v1.4h, v0.4s
 ; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    umov w9, v1.h[1]
-; CHECK-NEXT:    fmov w10, s0
+; CHECK-NEXT:    str b0, [x8]
 ; CHECK-NEXT:    and w0, w9, #0x1
-; CHECK-NEXT:    strb w10, [x8]
 ; CHECK-NEXT:    ret
   %icmp = icmp ult <4 x i32> %a, splat(i32 235)
   %ext = extractelement <4 x i1> %icmp, i32 1
diff --git a/llvm/test/CodeGen/AArch64/insertextract.ll b/llvm/test/CodeGen/AArch64/insertextract.ll
index 54ee693db1239..aa4f31fb5f53e 100644
--- a/llvm/test/CodeGen/AArch64/insertextract.ll
+++ b/llvm/test/CodeGen/AArch64/insertextract.ll
@@ -271,7 +271,7 @@ define <3 x float> @insert_v3f32_2(<3 x float> %a, float %b, i32 %c) {
 ;
 ; CHECK-GI-LABEL: insert_v3f32_2:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v2.s[0], v0.s[0]
+; CHECK-GI-NEXT:    mov s2, v0.s[0]
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 def $q1
 ; CHECK-GI-NEXT:    mov v2.s[1], v0.s[1]
 ; CHECK-GI-NEXT:    mov v2.s[2], v1.s[0]
@@ -992,7 +992,7 @@ define <3 x i32> @insert_v3i32_2(<3 x i32> %a, i32 %b, i32 %c) {
 ;
 ; CHECK-GI-LABEL: insert_v3i32_2:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    mov s1, v0.s[0]
 ; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
 ; CHECK-GI-NEXT:    mov v1.s[2], w0
 ; CHECK-GI-NEXT:    mov v0.16b, v1.16b
diff --git a/llvm/test/CodeGen/AArch64/mul.ll b/llvm/test/CodeGen/AArch64/mul.ll
index 500379d1cfdec..8d9a6e6b92914 100644
--- a/llvm/test/CodeGen/AArch64/mul.ll
+++ b/llvm/test/CodeGen/AArch64/mul.ll
@@ -75,10 +75,9 @@ define void @v2i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x9]
 ; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -113,11 +112,11 @@ define void @v3i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    zip1 v1.8b, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.4h
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -275,10 +274,10 @@ define void @v3i16(ptr %p1, ptr %p2) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr d0, [x0]
 ; CHECK-SD-NEXT:    ldr d1, [x1]
-; CHECK-SD-NEXT:    add x8, x0, #4
 ; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h1, v0.h[2]
 ; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    str h1, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v3i16:
diff --git a/llvm/test/CodeGen/AArch64/neon-truncstore.ll b/llvm/test/CodeGen/AArch64/neon-truncstore.ll
index 3d3362d314a99..f25c90af7968e 100644
--- a/llvm/test/CodeGen/AArch64/neon-truncstore.ll
+++ b/llvm/test/CodeGen/AArch64/neon-truncstore.ll
@@ -89,10 +89,9 @@ define void @v2i32_v2i8(<2 x i32> %a, ptr %result) {
 ; CHECK-LABEL: v2i32_v2i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    strb w9, [x0]
-; CHECK-NEXT:    strb w8, [x0, #1]
+; CHECK-NEXT:    mov s1, v0.s[1]
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    stur b1, [x0, #1]
 ; CHECK-NEXT:    ret
   %b = trunc <2 x i32> %a to <2 x i8>
   store <2 x i8> %b, ptr %result
@@ -156,10 +155,9 @@ define void @v2i16_v2i8(<2 x i16> %a, ptr %result) {
 ; CHECK-LABEL: v2i16_v2i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    strb w9, [x0]
-; CHECK-NEXT:    strb w8, [x0, #1]
+; CHECK-NEXT:    mov s1, v0.s[1]
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    stur b1, [x0, #1]
 ; CHECK-NEXT:    ret
   %b = trunc <2 x i16> %a to <2 x i8>
   store <2 x i8> %b, ptr %result
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-load.ll b/llvm/test/CodeGen/AArch64/nontemporal-load.ll
index 959ac7f68e351..adb209c0c6348 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-load.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-load.ll
@@ -449,10 +449,9 @@ define <33 x i8> @test_ldnp_v33i8(ptr %A) {
 ; CHECK-LABEL: test_ldnp_v33i8:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ldnp q0, q1, [x0]
-; CHECK-NEXT:    add x9, x8, #32
 ; CHECK-NEXT:    ldr b2, [x0, #32]
 ; CHECK-NEXT:    stp q0, q1, [x8]
-; CHECK-NEXT:    st1.b { v2 }[0], [x9]
+; CHECK-NEXT:    stur b2, [x8, #32]
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: test_ldnp_v33i8:
diff --git a/llvm/test/CodeGen/AArch64/pr-cf624b2.ll b/llvm/test/CodeGen/AArch64/pr-cf624b2.ll
index ea9588e9e3db7..02375b07b3482 100644
--- a/llvm/test/CodeGen/AArch64/pr-cf624b2.ll
+++ b/llvm/test/CodeGen/AArch64/pr-cf624b2.ll
@@ -11,45 +11,31 @@ define linkonce_odr void @_ZN1y2beEPiRK1vPmPS1_(<8 x i8> %0, ptr %agg.tmp.i) {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov b1, v0.b[7]
+; CHECK-NEXT:    mov b2, v0.b[6]
+; CHECK-NEXT:    stur b0, [sp, #15]
+; CHECK-NEXT:    stur b0, [sp, #14]
+; CHECK-NEXT:    stur b0, [sp, #13]
+; CHECK-NEXT:    stur b0, [sp, #12]
+; CHECK-NEXT:    stur b1, [sp, #7]
+; CHECK-NEXT:    mov b1, v0.b[5]
+; CHECK-NEXT:    stur b2, [sp, #6]
+; CHECK-NEXT:    mov b2, v0.b[4]
+; CHECK-NEXT:    stur b0, [sp, #11]
+; CHECK-NEXT:    stur b0, [sp, #10]
+; CHECK-NEXT:    stur b1, [sp, #5]
+; CHECK-NEXT:    mov b1, v0.b[3]
+; CHECK-NEXT:    stur b0, [sp, #9]
+; CHECK-NEXT:    stur b2, [sp, #4]
+; CHECK-NEXT:    mov b2, v0.b[2]
+; CHECK-NEXT:    str b0, [sp]
+; CHECK-NEXT:    mov b0, v0.b[1]
+; CHECK-NEXT:    stur b1, [sp, #3]
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    orr x9, x8, #0xf
-; CHECK-NEXT:    orr x10, x8, #0xe
-; CHECK-NEXT:    st1 { v0.b }[0], [x8]
-; CHECK-NEXT:    st1 { v0.b }[15], [x9]
-; CHECK-NEXT:    orr x9, x8, #0xc
-; CHECK-NEXT:    st1 { v0.b }[12], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x8
-; CHECK-NEXT:    st1 { v0.b }[8], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x7
-; CHECK-NEXT:    st1 { v0.b }[7], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x6
-; CHECK-NEXT:    st1 { v0.b }[6], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x4
-; CHECK-NEXT:    st1 { v0.b }[4], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x3
-; CHECK-NEXT:    st1 { v0.b }[3], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x2
-; CHECK-NEXT:    st1 { v0.b }[14], [x10]
-; CHECK-NEXT:    mov w10, #13 // =0xd
-; CHECK-NEXT:    st1 { v0.b }[2], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x1
-; CHECK-NEXT:    st1 { v0.b }[1], [x9]
-; CHECK-NEXT:    orr x9, x8, x10
-; CHECK-NEXT:    mov w10, #11 // =0xb
-; CHECK-NEXT:    st1 { v0.b }[13], [x9]
-; CHECK-NEXT:    orr x9, x8, x10
-; CHECK-NEXT:    mov w10, #10 // =0xa
-; CHECK-NEXT:    st1 { v0.b }[11], [x9]
-; CHECK-NEXT:    orr x9, x8, x10
-; CHECK-NEXT:    mov w10, #9 // =0x9
-; CHECK-NEXT:    st1 { v0.b }[10], [x9]
-; CHECK-NEXT:    orr x9, x8, x10
-; CHECK-NEXT:    st1 { v0.b }[9], [x9]
-; CHECK-NEXT:    mov w9, #5 // =0x5
-; CHECK-NEXT:    orr x8, x8, x9
-; CHECK-NEXT:    st1 { v0.b }[5], [x8]
+; CHECK-NEXT:    stur b2, [sp, #2]
+; CHECK-NEXT:    stur b0, [sp, #8]
+; CHECK-NEXT:    stur b0, [sp, #1]
 ; CHECK-NEXT:    ldr q0, [sp]
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    add sp, sp, #16
diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
index 4d76994be204f..d54dde3c86364 100644
--- a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
@@ -200,10 +200,9 @@ define void @v2i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-SD-NEXT:    sqadd v0.2s, v0.2s, v1.2s
 ; CHECK-SD-NEXT:    ushr v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x2]
-; CHECK-SD-NEXT:    strb w8, [x2, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x2]
+; CHECK-SD-NEXT:    stur b1, [x2, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -325,7 +324,7 @@ define void @v1i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD-NEXT:    ldr b0, [x0]
 ; CHECK-SD-NEXT:    ldr b1, [x1]
 ; CHECK-SD-NEXT:    sqadd v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    st1 { v0.b }[0], [x2]
+; CHECK-SD-NEXT:    str b0, [x2]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v1i8:
diff --git a/llvm/test/CodeGen/AArch64/setcc-type-mismatch.ll b/llvm/test/CodeGen/AArch64/setcc-type-mismatch.ll
index c0a728014e390..950ac92a8b12f 100644
--- a/llvm/test/CodeGen/AArch64/setcc-type-mismatch.ll
+++ b/llvm/test/CodeGen/AArch64/setcc-type-mismatch.ll
@@ -12,8 +12,7 @@ define void @test_mismatched_setcc(<4 x i22> %l, <4 x i22> %r, ptr %addr) {
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI0_0]
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 
   %tst = icmp eq <4 x i22> %l, %r
diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
index ae2a16929e254..dc39ad0571b14 100644
--- a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
@@ -201,10 +201,9 @@ define void @v2i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-SD-NEXT:    sqsub v0.2s, v0.2s, v1.2s
 ; CHECK-SD-NEXT:    ushr v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x2]
-; CHECK-SD-NEXT:    strb w8, [x2, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x2]
+; CHECK-SD-NEXT:    stur b1, [x2, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -326,7 +325,7 @@ define void @v1i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD-NEXT:    ldr b0, [x0]
 ; CHECK-SD-NEXT:    ldr b1, [x1]
 ; CHECK-SD-NEXT:    sqsub v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    st1 { v0.b }[0], [x2]
+; CHECK-SD-NEXT:    str b0, [x2]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v1i8:
diff --git a/llvm/test/CodeGen/AArch64/store.ll b/llvm/test/CodeGen/AArch64/store.ll
index 37a6ad08d4cb3..3a9f12b838702 100644
--- a/llvm/test/CodeGen/AArch64/store.ll
+++ b/llvm/test/CodeGen/AArch64/store.ll
@@ -110,10 +110,9 @@ define void @store_v2i8(<2 x i8> %a, ptr %ptr){
 ; CHECK-SD-LABEL: store_v2i8:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: store_v2i8:
@@ -231,12 +230,12 @@ define void @store_v3i8(<3 x i8> %a, ptr %ptr){
 define void @store_v7i8(<7 x i8> %a, ptr %ptr){
 ; CHECK-SD-LABEL: store_v7i8:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    add x8, x0, #6
-; CHECK-SD-NEXT:    add x9, x0, #4
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov b1, v0.b[6]
+; CHECK-SD-NEXT:    mov h2, v0.h[2]
 ; CHECK-SD-NEXT:    str s0, [x0]
-; CHECK-SD-NEXT:    st1 { v0.b }[6], [x8]
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x9]
+; CHECK-SD-NEXT:    stur b1, [x0, #6]
+; CHECK-SD-NEXT:    str h2, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: store_v7i8:
@@ -263,10 +262,10 @@ define void @store_v7i8(<7 x i8> %a, ptr %ptr){
 define void @store_v3i16(<3 x i16> %a, ptr %ptr){
 ; CHECK-SD-LABEL: store_v3i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    add x8, x0, #4
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov h1, v0.h[2]
 ; CHECK-SD-NEXT:    str s0, [x0]
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    str h1, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: store_v3i16:
@@ -285,11 +284,11 @@ define void @store_v3i16(<3 x i16> %a, ptr %ptr){
 define void @store_v7i16(<7 x i16> %a, ptr %ptr){
 ; CHECK-SD-LABEL: store_v7i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    add x8, x0, #12
-; CHECK-SD-NEXT:    add x9, x0, #8
+; CHECK-SD-NEXT:    mov h1, v0.h[6]
+; CHECK-SD-NEXT:    mov s2, v0.s[2]
 ; CHECK-SD-NEXT:    str d0, [x0]
-; CHECK-SD-NEXT:    st1 { v0.h }[6], [x8]
-; CHECK-SD-NEXT:    st1 { v0.s }[2], [x9]
+; CHECK-SD-NEXT:    str h1, [x0, #12]
+; CHECK-SD-NEXT:    str s2, [x0, #8]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: store_v7i16:
@@ -315,9 +314,9 @@ define void @store_v7i16(<7 x i16> %a, ptr %ptr){
 define void @store_v3i32(<3 x i32> %a, ptr %ptr){
 ; CHECK-SD-LABEL: store_v3i32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    add x8, x0, #8
+; CHECK-SD-NEXT:    mov s1, v0.s[2]
 ; CHECK-SD-NEXT:    str d0, [x0]
-; CHECK-SD-NEXT:    st1 { v0.s }[2], [x8]
+; CHECK-SD-NEXT:    str s1, [x0, #8]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: store_v3i32:
diff --git a/llvm/test/CodeGen/AArch64/sub.ll b/llvm/test/CodeGen/AArch64/sub.ll
index 8183a82f21cb5..7a436eddb23a6 100644
--- a/llvm/test/CodeGen/AArch64/sub.ll
+++ b/llvm/test/CodeGen/AArch64/sub.ll
@@ -63,10 +63,9 @@ define void @v2i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x9]
 ; CHECK-SD-NEXT:    sub v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x0]
-; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x0]
+; CHECK-SD-NEXT:    stur b1, [x0, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -101,11 +100,11 @@ define void @v3i8(ptr %p1, ptr %p2) {
 ; CHECK-SD-NEXT:    zip1 v1.8b, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    sub v0.4h, v0.4h, v1.4h
 ; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    mov h0, v0.h[2]
 ; CHECK-SD-NEXT:    str s1, [sp, #12]
-; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
-; CHECK-SD-NEXT:    strb w8, [x0, #2]
-; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    ldrh w8, [sp, #12]
+; CHECK-SD-NEXT:    stur b0, [x0, #2]
+; CHECK-SD-NEXT:    strh w8, [x0]
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
@@ -263,10 +262,10 @@ define void @v3i16(ptr %p1, ptr %p2) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr d0, [x0]
 ; CHECK-SD-NEXT:    ldr d1, [x1]
-; CHECK-SD-NEXT:    add x8, x0, #4
 ; CHECK-SD-NEXT:    sub v0.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    mov h1, v0.h[2]
 ; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    str h1, [x0, #4]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v3i16:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
index 797f953591b11..0d0b5cbc776c4 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
@@ -376,43 +376,39 @@ define void @test_revv8i16v8i16(ptr %a, ptr %b, ptr %c) #1 {
 ; CHECK-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-NEXT:    .cfi_offset w30, -8
 ; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ldr q0, [x1]
-; CHECK-NEXT:    ldr q1, [x0]
-; CHECK-NEXT:    orr x9, x8, #0x1e
-; CHECK-NEXT:    orr x10, x8, #0x1c
+; CHECK-NEXT:    ldr q5, [x0]
+; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    st1 { v0.h }[4], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x18
-; CHECK-NEXT:    st1 { v0.h }[7], [x9]
-; CHECK-NEXT:    orr x9, x8, #0xe
-; CHECK-NEXT:    st1 { v1.h }[4], [x9]
-; CHECK-NEXT:    orr x9, x8, #0xc
-; CHECK-NEXT:    st1 { v1.h }[5], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x8
-; CHECK-NEXT:    st1 { v0.h }[5], [x10]
-; CHECK-NEXT:    orr x10, x8, #0x10
-; CHECK-NEXT:    st1 { v1.h }[7], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x4
-; CHECK-NEXT:    st1 { v0.h }[3], [x10]
-; CHECK-NEXT:    mov w10, #26 // =0x1a
-; CHECK-NEXT:    st1 { v1.h }[1], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x2
-; CHECK-NEXT:    st1 { v1.h }[2], [x9]
-; CHECK-NEXT:    orr x9, x8, x10
-; CHECK-NEXT:    mov w10, #20 // =0x14
-; CHECK-NEXT:    st1 { v0.h }[6], [x9]
-; CHECK-NEXT:    orr x9, x8, x10
-; CHECK-NEXT:    mov w10, #18 // =0x12
-; CHECK-NEXT:    st1 { v0.h }[1], [x9]
-; CHECK-NEXT:    orr x9, x8, x10
-; CHECK-NEXT:    st1 { v0.h }[2], [x9]
-; CHECK-NEXT:    mov w9, #10 // =0xa
-; CHECK-NEXT:    orr x9, x8, x9
-; CHECK-NEXT:    st1 { v1.h }[3], [x8]
-; CHECK-NEXT:    st1 { v1.h }[6], [x9]
+; CHECK-NEXT:    mov h1, v0.h[4]
+; CHECK-NEXT:    mov h2, v0.h[5]
+; CHECK-NEXT:    mov h3, v0.h[6]
+; CHECK-NEXT:    mov h4, v0.h[7]
 ; CHECK-NEXT:    str h0, [sp, #22]
-; CHECK-NEXT:    str h1, [sp, #6]
+; CHECK-NEXT:    st1 { v5.h }[3], [x8]
+; CHECK-NEXT:    str h5, [sp, #6]
+; CHECK-NEXT:    str h1, [sp, #30]
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    str h2, [sp, #28]
+; CHECK-NEXT:    mov h2, v0.h[2]
+; CHECK-NEXT:    mov h0, v0.h[3]
+; CHECK-NEXT:    str h3, [sp, #26]
+; CHECK-NEXT:    mov h3, v5.h[2]
+; CHECK-NEXT:    str h4, [sp, #24]
+; CHECK-NEXT:    str h1, [sp, #20]
+; CHECK-NEXT:    mov h1, v5.h[4]
+; CHECK-NEXT:    str h2, [sp, #18]
+; CHECK-NEXT:    mov h2, v5.h[5]
+; CHECK-NEXT:    str h0, [sp, #16]
+; CHECK-NEXT:    mov h0, v5.h[6]
+; CHECK-NEXT:    str h3, [sp, #2]
+; CHECK-NEXT:    str h1, [sp, #14]
+; CHECK-NEXT:    mov h1, v5.h[7]
+; CHECK-NEXT:    str h2, [sp, #12]
+; CHECK-NEXT:    mov h2, v5.h[1]
+; CHECK-NEXT:    str h0, [sp, #10]
+; CHECK-NEXT:    str h1, [sp, #8]
+; CHECK-NEXT:    str h2, [sp, #4]
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
 ; CHECK-NEXT:    mov sp, x29
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
index aa1adfd306a4c..e8c9704940c70 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
@@ -119,11 +119,11 @@ define void @alloc_v32i8(ptr %st_ptr) nounwind {
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    bl def
 ; CHECK-NEXT:    adrp x8, .LCPI2_0
-; CHECK-NEXT:    ldp q0, q2, [sp]
+; CHECK-NEXT:    ldr q0, [sp]
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI2_0]
 ; CHECK-NEXT:    tbl z0.b, { z0.b }, z1.b
-; CHECK-NEXT:    fmov w8, s2
-; CHECK-NEXT:    strb w8, [x19, #8]
+; CHECK-NEXT:    ldr q1, [sp, #16]
+; CHECK-NEXT:    stur b1, [x19, #8]
 ; CHECK-NEXT:    str d0, [x19]
 ; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-gather-scatter.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-gather-scatter.ll
index d9f8482a3c503..b1ac9469c0573 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-gather-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-gather-scatter.ll
@@ -20,9 +20,8 @@ define <2 x i64> @masked_gather_v2i64(ptr %a, ptr %b) vscale_range(2, 2) {
 ; CHECK-NEXT:    ldr q1, [x1]
 ; CHECK-NEXT:    uaddv d0, p0, z0.d
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    strb w8, [sp, #12]
-; CHECK-NEXT:    and w8, w8, #0xff
+; CHECK-NEXT:    str b0, [sp, #12]
+; CHECK-NEXT:    ldrb w8, [sp, #12]
 ; CHECK-NEXT:    tbz w8, #0, .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %cond.load
 ; CHECK-NEXT:    fmov x9, d1
@@ -109,11 +108,10 @@ define void @masked_scatter_v2i64(ptr %a, ptr %b) vscale_range(2, 2) {
 ; CHECK-NEXT:    cmpeq p1.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    mov z2.d, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    and z1.d, z2.d, z1.d
-; CHECK-NEXT:    uaddv d1, p0, z1.d
-; CHECK-NEXT:    fmov x8, d1
+; CHECK-NEXT:    uaddv d2, p0, z1.d
 ; CHECK-NEXT:    ldr q1, [x1]
-; CHECK-NEXT:    strb w8, [sp, #12]
-; CHECK-NEXT:    and w8, w8, #0xff
+; CHECK-NEXT:    str b2, [sp, #12]
+; CHECK-NEXT:    ldrb w8, [sp, #12]
 ; CHECK-NEXT:    tbnz w8, #0, .LBB1_3
 ; CHECK-NEXT:  // %bb.1: // %else
 ; CHECK-NEXT:    tbnz w8, #1, .LBB1_4
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
index a20a330b39bb4..3d9f407c3064c 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
@@ -159,26 +159,26 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z4.h, z3.h[7]
 ; CHECK-NEXT:    mov z6.h, z3.h[6]
 ; CHECK-NEXT:    mov z16.h, z3.h[5]
-; CHECK-NEXT:    mov z20.h, z2.h[7]
-; CHECK-NEXT:    mov z21.h, z1.h[7]
 ; CHECK-NEXT:    mov z18.h, z3.h[4]
 ; CHECK-NEXT:    mov z19.h, z0.h[4]
+; CHECK-NEXT:    mov z20.h, z2.h[7]
+; CHECK-NEXT:    mov z21.h, z1.h[7]
 ; CHECK-NEXT:    mov z22.h, z2.h[6]
 ; CHECK-NEXT:    mov z23.h, z1.h[6]
 ; CHECK-NEXT:    zip1 z24.h, z5.h, z4.h
 ; CHECK-NEXT:    zip1 z25.h, z7.h, z6.h
-; CHECK-NEXT:    zip1 z17.h, z17.h, z16.h
+; CHECK-NEXT:    zip1 z16.h, z17.h, z16.h
 ; CHECK-NEXT:    ldp q4, q6, [x0, #32]
-; CHECK-NEXT:    zip1 z16.h, z21.h, z20.h
+; CHECK-NEXT:    zip1 z17.h, z19.h, z18.h
 ; CHECK-NEXT:    ldp q5, q7, [x1, #32]
-; CHECK-NEXT:    zip1 z18.h, z19.h, z18.h
-; CHECK-NEXT:    zip1 z19.s, z25.s, z24.s
+; CHECK-NEXT:    zip1 z18.h, z21.h, z20.h
+; CHECK-NEXT:    zip1 z21.s, z25.s, z24.s
 ; CHECK-NEXT:    zip1 z22.h, z23.h, z22.h
 ; CHECK-NEXT:    mov z23.h, z2.h[5]
-; CHECK-NEXT:    mov z21.h, z6.h[7]
+; CHECK-NEXT:    mov z20.h, z6.h[7]
 ; CHECK-NEXT:    mov z24.h, z1.h[5]
 ; CHECK-NEXT:    mov z25.h, z2.h[4]
-; CHECK-NEXT:    mov z20.h, z7.h[7]
+; CHECK-NEXT:    mov z19.h, z7.h[7]
 ; CHECK-NEXT:    mov z26.h, z1.h[4]
 ; CHECK-NEXT:    mov z27.h, z6.h[6]
 ; CHECK-NEXT:    mov z28.h, z7.h[5]
@@ -187,8 +187,8 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z31.h, z6.h[4]
 ; CHECK-NEXT:    mov z8.h, z5.h[7]
 ; CHECK-NEXT:    mov z9.h, z4.h[7]
-; CHECK-NEXT:    zip1 z20.h, z21.h, z20.h
-; CHECK-NEXT:    mov z21.h, z7.h[6]
+; CHECK-NEXT:    zip1 z19.h, z20.h, z19.h
+; CHECK-NEXT:    mov z20.h, z7.h[6]
 ; CHECK-NEXT:    mov z10.h, z5.h[6]
 ; CHECK-NEXT:    mov z11.h, z4.h[6]
 ; CHECK-NEXT:    mov z12.h, z5.h[5]
@@ -196,7 +196,7 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z14.h, z5.h[4]
 ; CHECK-NEXT:    mov z15.h, z4.h[4]
 ; CHECK-NEXT:    zip1 z23.h, z24.h, z23.h
-; CHECK-NEXT:    zip1 z21.h, z27.h, z21.h
+; CHECK-NEXT:    zip1 z20.h, z27.h, z20.h
 ; CHECK-NEXT:    zip1 z27.h, z29.h, z28.h
 ; CHECK-NEXT:    zip1 z28.h, z31.h, z30.h
 ; CHECK-NEXT:    zip1 z24.h, z26.h, z25.h
@@ -207,23 +207,23 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    zip1 z30.h, z15.h, z14.h
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    zip1 z17.s, z18.s, z17.s
-; CHECK-NEXT:    zip1 z18.s, z21.s, z20.s
-; CHECK-NEXT:    zip1 z20.s, z28.s, z27.s
-; CHECK-NEXT:    zip1 z16.s, z22.s, z16.s
-; CHECK-NEXT:    zip1 z21.s, z24.s, z23.s
+; CHECK-NEXT:    zip1 z16.s, z17.s, z16.s
+; CHECK-NEXT:    zip1 z17.s, z20.s, z19.s
+; CHECK-NEXT:    zip1 z19.s, z28.s, z27.s
+; CHECK-NEXT:    zip1 z18.s, z22.s, z18.s
+; CHECK-NEXT:    zip1 z20.s, z24.s, z23.s
 ; CHECK-NEXT:    zip1 z0.h, z0.h, z3.h
 ; CHECK-NEXT:    zip1 z3.s, z26.s, z25.s
 ; CHECK-NEXT:    zip1 z22.s, z30.s, z29.s
 ; CHECK-NEXT:    zip1 z6.h, z6.h, z7.h
-; CHECK-NEXT:    zip1 z7.d, z17.d, z19.d
-; CHECK-NEXT:    zip1 z17.d, z20.d, z18.d
+; CHECK-NEXT:    zip1 z7.d, z16.d, z21.d
+; CHECK-NEXT:    zip1 z16.d, z19.d, z17.d
 ; CHECK-NEXT:    zip1 z1.h, z1.h, z2.h
 ; CHECK-NEXT:    zip1 z2.h, z4.h, z5.h
-; CHECK-NEXT:    zip1 z4.d, z21.d, z16.d
+; CHECK-NEXT:    zip1 z4.d, z20.d, z18.d
 ; CHECK-NEXT:    zip1 z3.d, z22.d, z3.d
 ; CHECK-NEXT:    add z0.h, z0.h, z6.h
-; CHECK-NEXT:    add z5.h, z7.h, z17.h
+; CHECK-NEXT:    add z5.h, z7.h, z16.h
 ; CHECK-NEXT:    add z1.h, z1.h, z2.h
 ; CHECK-NEXT:    add z2.h, z4.h, z3.h
 ; CHECK-NEXT:    stp q0, q5, [x0, #32]
@@ -1476,44 +1476,44 @@ define void @uzp_v32i8(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    zip1 z20.b, z24.b, z23.b
 ; CHECK-NEXT:    zip1 z21.b, z26.b, z25.b
 ; CHECK-NEXT:    zip1 z22.b, z28.b, z27.b
+; CHECK-NEXT:    zip1 z23.b, z17.b, z29.b
 ; CHECK-NEXT:    mov z24.b, z2.b[14]
 ; CHECK-NEXT:    mov z25.b, z2.b[12]
 ; CHECK-NEXT:    mov z26.b, z2.b[10]
 ; CHECK-NEXT:    mov z27.b, z2.b[8]
-; CHECK-NEXT:    zip1 z23.b, z17.b, z29.b
 ; CHECK-NEXT:    zip1 z3.h, z4.h, z3.h
 ; CHECK-NEXT:    zip1 z4.h, z6.h, z5.h
 ; CHECK-NEXT:    zip1 z5.h, z7.h, z18.h
 ; CHECK-NEXT:    zip1 z6.h, z19.h, z16.h
 ; CHECK-NEXT:    zip1 z7.h, z21.h, z20.h
+; CHECK-NEXT:    zip1 z16.h, z23.h, z22.h
 ; CHECK-NEXT:    zip1 z18.b, z25.b, z24.b
 ; CHECK-NEXT:    zip1 z19.b, z27.b, z26.b
 ; CHECK-NEXT:    mov z20.b, z2.b[6]
 ; CHECK-NEXT:    mov z21.b, z2.b[4]
+; CHECK-NEXT:    mov z23.b, z17.b[15]
+; CHECK-NEXT:    mov z24.b, z17.b[13]
 ; CHECK-NEXT:    mov z29.b, z17.b[3]
 ; CHECK-NEXT:    mov z30.b, z17.b[1]
 ; CHECK-NEXT:    mov z31.b, z2.b[15]
 ; CHECK-NEXT:    mov z8.b, z2.b[13]
-; CHECK-NEXT:    zip1 z16.h, z23.h, z22.h
 ; CHECK-NEXT:    mov z22.b, z2.b[2]
-; CHECK-NEXT:    mov z23.b, z17.b[15]
-; CHECK-NEXT:    mov z24.b, z17.b[13]
 ; CHECK-NEXT:    mov z25.b, z17.b[11]
 ; CHECK-NEXT:    mov z26.b, z17.b[9]
 ; CHECK-NEXT:    mov z27.b, z17.b[7]
 ; CHECK-NEXT:    mov z28.b, z17.b[5]
 ; CHECK-NEXT:    zip1 z17.h, z19.h, z18.h
-; CHECK-NEXT:    zip1 z21.b, z21.b, z20.b
-; CHECK-NEXT:    zip1 z19.b, z30.b, z29.b
-; CHECK-NEXT:    zip1 z20.b, z8.b, z31.b
+; CHECK-NEXT:    zip1 z18.b, z21.b, z20.b
+; CHECK-NEXT:    zip1 z20.b, z24.b, z23.b
+; CHECK-NEXT:    zip1 z23.b, z30.b, z29.b
+; CHECK-NEXT:    zip1 z24.b, z8.b, z31.b
 ; CHECK-NEXT:    mov z29.b, z1.b[15]
 ; CHECK-NEXT:    mov z30.b, z1.b[13]
 ; CHECK-NEXT:    mov z31.b, z1.b[11]
 ; CHECK-NEXT:    mov z8.b, z1.b[9]
-; CHECK-NEXT:    zip1 z22.b, z2.b, z22.b
-; CHECK-NEXT:    zip1 z23.b, z24.b, z23.b
-; CHECK-NEXT:    zip1 z24.b, z26.b, z25.b
-; CHECK-NEXT:    zip1 z18.b, z28.b, z27.b
+; CHECK-NEXT:    zip1 z19.b, z2.b, z22.b
+; CHECK-NEXT:    zip1 z21.b, z26.b, z25.b
+; CHECK-NEXT:    zip1 z22.b, z28.b, z27.b
 ; CHECK-NEXT:    mov z25.b, z2.b[11]
 ; CHECK-NEXT:    mov z26.b, z2.b[9]
 ; CHECK-NEXT:    mov z27.b, z2.b[7]
@@ -1538,25 +1538,25 @@ define void @uzp_v32i8(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    zip1 z25.b, z26.b, z25.b
 ; CHECK-NEXT:    zip1 z26.b, z28.b, z27.b
 ; CHECK-NEXT:    zip1 z2.b, z2.b, z8.b
-; CHECK-NEXT:    zip1 z21.h, z22.h, z21.h
-; CHECK-NEXT:    zip1 z22.h, z24.h, z23.h
-; CHECK-NEXT:    zip1 z23.h, z31.h, z29.h
+; CHECK-NEXT:    zip1 z18.h, z19.h, z18.h
+; CHECK-NEXT:    zip1 z19.h, z21.h, z20.h
+; CHECK-NEXT:    zip1 z20.h, z31.h, z29.h
 ; CHECK-NEXT:    zip1 z1.h, z1.h, z9.h
 ; CHECK-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    zip1 z24.h, z10.h, z11.h
+; CHECK-NEXT:    zip1 z21.h, z10.h, z11.h
 ; CHECK-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    zip1 z0.h, z30.h, z0.h
-; CHECK-NEXT:    zip1 z18.h, z19.h, z18.h
-; CHECK-NEXT:    zip1 z19.h, z25.h, z20.h
+; CHECK-NEXT:    zip1 z22.h, z23.h, z22.h
+; CHECK-NEXT:    zip1 z23.h, z25.h, z24.h
 ; CHECK-NEXT:    zip1 z2.h, z2.h, z26.h
 ; CHECK-NEXT:    zip1 z3.s, z4.s, z3.s
 ; CHECK-NEXT:    zip1 z4.s, z6.s, z5.s
 ; CHECK-NEXT:    zip1 z5.s, z16.s, z7.s
-; CHECK-NEXT:    zip1 z1.s, z1.s, z23.s
-; CHECK-NEXT:    zip1 z6.s, z21.s, z17.s
-; CHECK-NEXT:    zip1 z0.s, z0.s, z24.s
-; CHECK-NEXT:    zip1 z7.s, z18.s, z22.s
-; CHECK-NEXT:    zip1 z2.s, z2.s, z19.s
+; CHECK-NEXT:    zip1 z1.s, z1.s, z20.s
+; CHECK-NEXT:    zip1 z6.s, z18.s, z17.s
+; CHECK-NEXT:    zip1 z0.s, z0.s, z21.s
+; CHECK-NEXT:    zip1 z7.s, z22.s, z19.s
+; CHECK-NEXT:    zip1 z2.s, z2.s, z23.s
 ; CHECK-NEXT:    zip1 z3.d, z4.d, z3.d
 ; CHECK-NEXT:    zip1 z0.d, z0.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z6.d, z5.d
@@ -1752,67 +1752,67 @@ define void @uzp_v16i16(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset b8, -16
-; CHECK-NEXT:    ldp q1, q6, [x0]
-; CHECK-NEXT:    ldp q0, q2, [x1]
-; CHECK-NEXT:    mov z3.h, z6.h[6]
-; CHECK-NEXT:    mov z4.h, z6.h[4]
-; CHECK-NEXT:    mov z5.h, z6.h[2]
+; CHECK-NEXT:    ldp q1, q0, [x0]
+; CHECK-NEXT:    ldp q5, q6, [x1]
+; CHECK-NEXT:    mov z2.h, z0.h[6]
+; CHECK-NEXT:    mov z3.h, z0.h[4]
+; CHECK-NEXT:    mov z4.h, z0.h[2]
 ; CHECK-NEXT:    mov z7.h, z1.h[6]
 ; CHECK-NEXT:    mov z16.h, z1.h[4]
 ; CHECK-NEXT:    mov z17.h, z1.h[2]
-; CHECK-NEXT:    mov z18.h, z2.h[6]
-; CHECK-NEXT:    mov z19.h, z2.h[4]
-; CHECK-NEXT:    mov z20.h, z2.h[2]
-; CHECK-NEXT:    mov z21.h, z0.h[6]
-; CHECK-NEXT:    mov z22.h, z0.h[4]
-; CHECK-NEXT:    zip1 z3.h, z4.h, z3.h
-; CHECK-NEXT:    zip1 z4.h, z6.h, z5.h
-; CHECK-NEXT:    zip1 z5.h, z16.h, z7.h
+; CHECK-NEXT:    mov z18.h, z6.h[6]
+; CHECK-NEXT:    mov z19.h, z6.h[4]
+; CHECK-NEXT:    mov z20.h, z6.h[2]
+; CHECK-NEXT:    mov z21.h, z5.h[6]
+; CHECK-NEXT:    mov z22.h, z5.h[4]
+; CHECK-NEXT:    zip1 z2.h, z3.h, z2.h
+; CHECK-NEXT:    zip1 z3.h, z0.h, z4.h
+; CHECK-NEXT:    zip1 z4.h, z16.h, z7.h
 ; CHECK-NEXT:    zip1 z7.h, z1.h, z17.h
 ; CHECK-NEXT:    zip1 z16.h, z19.h, z18.h
-; CHECK-NEXT:    zip1 z18.h, z2.h, z20.h
-; CHECK-NEXT:    mov z19.h, z0.h[2]
-; CHECK-NEXT:    zip1 z17.h, z22.h, z21.h
-; CHECK-NEXT:    mov z20.h, z6.h[7]
-; CHECK-NEXT:    mov z21.h, z6.h[5]
-; CHECK-NEXT:    mov z22.h, z6.h[3]
-; CHECK-NEXT:    mov z6.h, z6.h[1]
+; CHECK-NEXT:    zip1 z17.h, z6.h, z20.h
+; CHECK-NEXT:    mov z19.h, z5.h[2]
+; CHECK-NEXT:    zip1 z18.h, z22.h, z21.h
+; CHECK-NEXT:    mov z20.h, z0.h[7]
+; CHECK-NEXT:    mov z21.h, z0.h[5]
+; CHECK-NEXT:    mov z22.h, z0.h[3]
+; CHECK-NEXT:    mov z0.h, z0.h[1]
 ; CHECK-NEXT:    mov z23.h, z1.h[7]
 ; CHECK-NEXT:    mov z24.h, z1.h[5]
 ; CHECK-NEXT:    mov z25.h, z1.h[3]
 ; CHECK-NEXT:    mov z1.h, z1.h[1]
-; CHECK-NEXT:    mov z26.h, z2.h[7]
-; CHECK-NEXT:    mov z27.h, z2.h[5]
-; CHECK-NEXT:    mov z28.h, z2.h[3]
-; CHECK-NEXT:    mov z2.h, z2.h[1]
-; CHECK-NEXT:    mov z29.h, z0.h[7]
-; CHECK-NEXT:    mov z30.h, z0.h[5]
-; CHECK-NEXT:    mov z31.h, z0.h[3]
-; CHECK-NEXT:    mov z8.h, z0.h[1]
-; CHECK-NEXT:    zip1 z0.h, z0.h, z19.h
+; CHECK-NEXT:    mov z26.h, z6.h[7]
+; CHECK-NEXT:    mov z27.h, z6.h[5]
+; CHECK-NEXT:    mov z28.h, z6.h[3]
+; CHECK-NEXT:    mov z6.h, z6.h[1]
+; CHECK-NEXT:    mov z29.h, z5.h[7]
+; CHECK-NEXT:    mov z30.h, z5.h[5]
+; CHECK-NEXT:    mov z31.h, z5.h[3]
+; CHECK-NEXT:    mov z8.h, z5.h[1]
+; CHECK-NEXT:    zip1 z5.h, z5.h, z19.h
 ; CHECK-NEXT:    zip1 z19.h, z21.h, z20.h
-; CHECK-NEXT:    zip1 z6.h, z6.h, z22.h
+; CHECK-NEXT:    zip1 z0.h, z0.h, z22.h
 ; CHECK-NEXT:    zip1 z20.h, z24.h, z23.h
 ; CHECK-NEXT:    zip1 z1.h, z1.h, z25.h
 ; CHECK-NEXT:    zip1 z21.h, z27.h, z26.h
-; CHECK-NEXT:    zip1 z2.h, z2.h, z28.h
+; CHECK-NEXT:    zip1 z6.h, z6.h, z28.h
 ; CHECK-NEXT:    zip1 z22.h, z30.h, z29.h
 ; CHECK-NEXT:    zip1 z23.h, z8.h, z31.h
-; CHECK-NEXT:    zip1 z3.s, z4.s, z3.s
-; CHECK-NEXT:    zip1 z4.s, z7.s, z5.s
-; CHECK-NEXT:    zip1 z5.s, z18.s, z16.s
-; CHECK-NEXT:    zip1 z6.s, z6.s, z19.s
+; CHECK-NEXT:    zip1 z2.s, z3.s, z2.s
+; CHECK-NEXT:    zip1 z3.s, z7.s, z4.s
+; CHECK-NEXT:    zip1 z4.s, z17.s, z16.s
+; CHECK-NEXT:    zip1 z0.s, z0.s, z19.s
 ; CHECK-NEXT:    zip1 z1.s, z1.s, z20.s
-; CHECK-NEXT:    zip1 z0.s, z0.s, z17.s
-; CHECK-NEXT:    zip1 z2.s, z2.s, z21.s
+; CHECK-NEXT:    zip1 z5.s, z5.s, z18.s
+; CHECK-NEXT:    zip1 z6.s, z6.s, z21.s
 ; CHECK-NEXT:    zip1 z7.s, z23.s, z22.s
-; CHECK-NEXT:    zip1 z3.d, z4.d, z3.d
-; CHECK-NEXT:    zip1 z1.d, z1.d, z6.d
-; CHECK-NEXT:    zip1 z0.d, z0.d, z5.d
-; CHECK-NEXT:    zip1 z2.d, z7.d, z2.d
-; CHECK-NEXT:    add z1.h, z3.h, z1.h
-; CHECK-NEXT:    add z0.h, z0.h, z2.h
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    zip1 z2.d, z3.d, z2.d
+; CHECK-NEXT:    zip1 z0.d, z1.d, z0.d
+; CHECK-NEXT:    zip1 z1.d, z5.d, z4.d
+; CHECK-NEXT:    zip1 z3.d, z7.d, z6.d
+; CHECK-NEXT:    add z0.h, z2.h, z0.h
+; CHECK-NEXT:    add z1.h, z1.h, z3.h
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ldr d8, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index 0ad9900865518..b5d64112db727 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -346,7 +346,6 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
 ; CHECK-NEXT:  .LBB2_4: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ld3 { v2.4s, v3.4s, v4.4s }, [x1], #48
-; CHECK-NEXT:    add x13, x0, #8
 ; CHECK-NEXT:    subs x12, x12, #4
 ; CHECK-NEXT:    fcmgt v5.4s, v2.4s, v0.4s
 ; CHECK-NEXT:    fcmgt v6.4s, v3.4s, v0.4s
@@ -367,8 +366,10 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
 ; CHECK-NEXT:    xtn v6.4h, v4.4s
 ; CHECK-NEXT:    xtn v7.4h, v2.4s
 ; CHECK-NEXT:    tbl v2.16b, { v5.16b, v6.16b, v7.16b }, v1.16b
-; CHECK-NEXT:    st1 { v2.s }[2], [x13]
-; CHECK-NEXT:    str d2, [x0], #12
+; CHECK-NEXT:    mov s3, v2.s[2]
+; CHECK-NEXT:    str d2, [x0]
+; CHECK-NEXT:    str s3, [x0, #8]
+; CHECK-NEXT:    add x0, x0, #12
 ; CHECK-NEXT:    b.ne .LBB2_4
 ; CHECK-NEXT:  // %bb.5: // %middle.block
 ; CHECK-NEXT:    cmp x11, x10
diff --git a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
index 184e8fff154b9..fd23f3da18cd7 100644
--- a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
@@ -706,11 +706,10 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-NEXT:  LBB6_1: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldp q4, q0, [x0, #48]
-; CHECK-NEXT:    add x9, x1, #10
-; CHECK-NEXT:    ldr d1, [x0, #80]
+; CHECK-NEXT:    subs x8, x8, #1
 ; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    ldr d1, [x0, #80]
 ; CHECK-NEXT:    ldr q5, [x0, #32]
-; CHECK-NEXT:    subs x8, x8, #1
 ; CHECK-NEXT:    add x0, x0, #128
 ; CHECK-NEXT:    uzp1.4s v0, v0, v1
 ; CHECK-NEXT:    uzp1.4s v1, v5, v4
@@ -719,10 +718,12 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-NEXT:    uzp1.8h v1, v2, v1
 ; CHECK-NEXT:    uzp1.8b v2, v0, v0
 ; CHECK-NEXT:    uzp1.16b v0, v1, v0
-; CHECK-NEXT:    st1.b { v2 }[2], [x9]
-; CHECK-NEXT:    add x9, x1, #8
-; CHECK-NEXT:    st1.h { v0 }[4], [x9]
-; CHECK-NEXT:    str d0, [x1], #16
+; CHECK-NEXT:    mov b1, v2[2]
+; CHECK-NEXT:    mov h2, v0[4]
+; CHECK-NEXT:    str d0, [x1]
+; CHECK-NEXT:    stur b1, [x1, #10]
+; CHECK-NEXT:    str h2, [x1, #8]
+; CHECK-NEXT:    add x1, x1, #16
 ; CHECK-NEXT:    b.eq LBB6_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
 ; CHECK-NEXT:    ret
@@ -742,11 +743,10 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-BE-NEXT:    ld1 { v2.2d }, [x9]
 ; CHECK-BE-NEXT:    ldr d5, [x0, #80]
 ; CHECK-BE-NEXT:    ld1 { v4.2d }, [x10]
-; CHECK-BE-NEXT:    add x9, x1, #10
 ; CHECK-BE-NEXT:    subs x8, x8, #1
+; CHECK-BE-NEXT:    add x0, x0, #128
 ; CHECK-BE-NEXT:    uzp1 v1.4s, v3.4s, v1.4s
 ; CHECK-BE-NEXT:    uzp1 v0.4s, v0.4s, v5.4s
-; CHECK-BE-NEXT:    add x0, x0, #128
 ; CHECK-BE-NEXT:    uzp1 v2.4s, v4.4s, v2.4s
 ; CHECK-BE-NEXT:    xtn v0.4h, v0.4s
 ; CHECK-BE-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
@@ -754,10 +754,12 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-BE-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-BE-NEXT:    rev16 v2.16b, v1.16b
 ; CHECK-BE-NEXT:    rev64 v1.16b, v1.16b
-; CHECK-BE-NEXT:    st1 { v0.b }[2], [x9]
-; CHECK-BE-NEXT:    add x9, x1, #8
-; CHECK-BE-NEXT:    st1 { v2.h }[4], [x9]
-; CHECK-BE-NEXT:    str d1, [x1], #16
+; CHECK-BE-NEXT:    mov b0, v0.b[2]
+; CHECK-BE-NEXT:    mov h2, v2.h[4]
+; CHECK-BE-NEXT:    str d1, [x1]
+; CHECK-BE-NEXT:    stur b0, [x1, #10]
+; CHECK-BE-NEXT:    str h2, [x1, #8]
+; CHECK-BE-NEXT:    add x1, x1, #16
 ; CHECK-BE-NEXT:    b.eq .LBB6_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit
 ; CHECK-BE-NEXT:    ret
@@ -777,11 +779,10 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-DISABLE-NEXT:    ld1 { v2.2d }, [x9]
 ; CHECK-DISABLE-NEXT:    ldr d5, [x0, #80]
 ; CHECK-DISABLE-NEXT:    ld1 { v4.2d }, [x10]
-; CHECK-DISABLE-NEXT:    add x9, x1, #10
 ; CHECK-DISABLE-NEXT:    subs x8, x8, #1
+; CHECK-DISABLE-NEXT:    add x0, x0, #128
 ; CHECK-DISABLE-NEXT:    uzp1 v1.4s, v3.4s, v1.4s
 ; CHECK-DISABLE-NEXT:    uzp1 v0.4s, v0.4s, v5.4s
-; CHECK-DISABLE-NEXT:    add x0, x0, #128
 ; CHECK-DISABLE-NEXT:    uzp1 v2.4s, v4.4s, v2.4s
 ; CHECK-DISABLE-NEXT:    xtn v0.4h, v0.4s
 ; CHECK-DISABLE-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
@@ -789,10 +790,12 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-DISABLE-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-DISABLE-NEXT:    rev16 v2.16b, v1.16b
 ; CHECK-DISABLE-NEXT:    rev64 v1.16b, v1.16b
-; CHECK-DISABLE-NEXT:    st1 { v0.b }[2], [x9]
-; CHECK-DISABLE-NEXT:    add x9, x1, #8
-; CHECK-DISABLE-NEXT:    st1 { v2.h }[4], [x9]
-; CHECK-DISABLE-NEXT:    str d1, [x1], #16
+; CHECK-DISABLE-NEXT:    mov b0, v0.b[2]
+; CHECK-DISABLE-NEXT:    mov h2, v2.h[4]
+; CHECK-DISABLE-NEXT:    str d1, [x1]
+; CHECK-DISABLE-NEXT:    stur b0, [x1, #10]
+; CHECK-DISABLE-NEXT:    str h2, [x1, #8]
+; CHECK-DISABLE-NEXT:    add x1, x1, #16
 ; CHECK-DISABLE-NEXT:    b.eq .LBB6_1
 ; CHECK-DISABLE-NEXT:  // %bb.2: // %exit
 ; CHECK-DISABLE-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
index d0173307bd830..14a578fa317d0 100644
--- a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
@@ -198,10 +198,9 @@ define void @v2i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD-NEXT:    mov v1.s[1], w11
 ; CHECK-SD-NEXT:    add v0.2s, v0.2s, v1.2s
 ; CHECK-SD-NEXT:    umin v0.2s, v0.2s, v2.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x2]
-; CHECK-SD-NEXT:    strb w8, [x2, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x2]
+; CHECK-SD-NEXT:    stur b1, [x2, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -324,7 +323,7 @@ define void @v1i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD-NEXT:    ldr b0, [x0]
 ; CHECK-SD-NEXT:    ldr b1, [x1]
 ; CHECK-SD-NEXT:    uqadd v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    st1 { v0.b }[0], [x2]
+; CHECK-SD-NEXT:    str b0, [x2]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v1i8:
diff --git a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
index dc3ebfb0682ca..ddb3332abf5d0 100644
--- a/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
@@ -197,10 +197,9 @@ define void @v2i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD-NEXT:    mov v0.s[1], w10
 ; CHECK-SD-NEXT:    mov v1.s[1], w11
 ; CHECK-SD-NEXT:    uqsub v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    strb w9, [x2]
-; CHECK-SD-NEXT:    strb w8, [x2, #1]
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str b0, [x2]
+; CHECK-SD-NEXT:    stur b1, [x2, #1]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v2i8:
@@ -321,7 +320,7 @@ define void @v1i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD-NEXT:    ldr b0, [x0]
 ; CHECK-SD-NEXT:    ldr b1, [x1]
 ; CHECK-SD-NEXT:    uqsub v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    st1 { v0.b }[0], [x2]
+; CHECK-SD-NEXT:    str b0, [x2]
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: v1i8:
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll
index dd7a9c6d7768b..3c42079dc8d8a 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll
@@ -35,8 +35,7 @@ define void @store_8_elements(<8 x i16> %vec, ptr %out) {
 ; CHECK-NEXT:    ldr q1, [x8, lCPI1_0@PAGEOFF]
 ; CHECK-NEXT:    bic.16b v0, v1, v0
 ; CHECK-NEXT:    addv.8h h0, v0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh2, Lloh3
 
@@ -56,8 +55,7 @@ define void @store_4_elements(<4 x i32> %vec, ptr %out) {
 ; CHECK-NEXT:    ldr q1, [x8, lCPI2_0@PAGEOFF]
 ; CHECK-NEXT:    bic.16b v0, v1, v0
 ; CHECK-NEXT:    addv.4s s0, v0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh4, Lloh5
 
@@ -77,8 +75,7 @@ define void @store_2_elements(<2 x i64> %vec, ptr %out) {
 ; CHECK-NEXT:    ldr q1, [x8, lCPI3_0@PAGEOFF]
 ; CHECK-NEXT:    bic.16b v0, v1, v0
 ; CHECK-NEXT:    addp.2d d0, v0
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh6, Lloh7
 
@@ -99,8 +96,7 @@ define void @add_trunc_compare_before_store(<4 x i32> %vec, ptr %out) {
 ; CHECK-NEXT:    cmlt.4s v0, v0, #0
 ; CHECK-NEXT:    and.16b v0, v0, v1
 ; CHECK-NEXT:    addv.4s s0, v0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh8, Lloh9
 
@@ -121,8 +117,7 @@ define void @add_trunc_mask_unknown_vector_type(<4 x i1> %vec, ptr %out) {
 ; CHECK-NEXT:    cmlt.4h v0, v0, #0
 ; CHECK-NEXT:    and.8b v0, v0, v1
 ; CHECK-NEXT:    addv.4h h0, v0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh10, Lloh11
 
@@ -141,7 +136,7 @@ define void @store_8_elements_64_bit_vector(<8 x i8> %vec, ptr %out) {
 ; CHECK-NEXT:    ldr d1, [x8, lCPI6_0@PAGEOFF]
 ; CHECK-NEXT:    bic.8b v0, v1, v0
 ; CHECK-NEXT:    addv.8b b0, v0
-; CHECK-NEXT:    st1.b { v0 }[0], [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh12, Lloh13
 
@@ -161,8 +156,7 @@ define void @store_4_elements_64_bit_vector(<4 x i16> %vec, ptr %out) {
 ; CHECK-NEXT:    ldr d1, [x8, lCPI7_0@PAGEOFF]
 ; CHECK-NEXT:    bic.8b v0, v1, v0
 ; CHECK-NEXT:    addv.4h h0, v0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh14, Lloh15
 
@@ -182,8 +176,7 @@ define void @store_2_elements_64_bit_vector(<2 x i32> %vec, ptr %out) {
 ; CHECK-NEXT:    ldr d1, [x8, lCPI8_0@PAGEOFF]
 ; CHECK-NEXT:    bic.8b v0, v1, v0
 ; CHECK-NEXT:    addp.2s v0, v0, v0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b0, [x0]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh16, Lloh17
 
diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
index 45b7a2759b0b3..7d3f5bc270d6b 100644
--- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
@@ -355,14 +355,14 @@ define <3 x i32> @load_v3i8_sext_to_3xi32(ptr %src) {
 define void @store_trunc_from_64bits(ptr %src, ptr %dst) {
 ; CHECK-LABEL: store_trunc_from_64bits:
 ; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    add x8, x0, #4
+; CHECK-NEXT:    ld1r.4h { v0 }, [x8]
 ; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    add x9, x0, #4
-; CHECK-NEXT:    ld1r.4h { v0 }, [x9]
 ; CHECK-NEXT:    lsr w9, w8, #16
 ; CHECK-NEXT:    strb w8, [x1]
-; CHECK-NEXT:    add x8, x1, #2
+; CHECK-NEXT:    mov b0, v0[4]
 ; CHECK-NEXT:    strb w9, [x1, #1]
-; CHECK-NEXT:    st1.b { v0 }[4], [x8]
+; CHECK-NEXT:    stur b0, [x1, #2]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: store_trunc_from_64bits:
@@ -397,13 +397,13 @@ define void @store_trunc_add_from_64bits(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    adrp x8, lCPI11_0@PAGE
 ; CHECK-NEXT:  Lloh1:
 ; CHECK-NEXT:    ldr d1, [x8, lCPI11_0@PAGEOFF]
-; CHECK-NEXT:    add x8, x1, #1
 ; CHECK-NEXT:    ld1.h { v0 }[2], [x9]
-; CHECK-NEXT:    add x9, x1, #2
 ; CHECK-NEXT:    add.4h v0, v0, v1
-; CHECK-NEXT:    st1.b { v0 }[2], [x8]
-; CHECK-NEXT:    st1.b { v0 }[4], [x9]
-; CHECK-NEXT:    st1.b { v0 }[0], [x1]
+; CHECK-NEXT:    mov b1, v0[2]
+; CHECK-NEXT:    mov b2, v0[4]
+; CHECK-NEXT:    str b0, [x1]
+; CHECK-NEXT:    stur b1, [x1, #1]
+; CHECK-NEXT:    stur b2, [x1, #2]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh0, Lloh1
 ;
@@ -420,12 +420,12 @@ define void @store_trunc_add_from_64bits(ptr %src, ptr %dst) {
 ; BE-NEXT:    ld1 { v1.4h }, [x8]
 ; BE-NEXT:    add v0.4h, v0.4h, v1.4h
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #12]
-; BE-NEXT:    ldrh w9, [sp, #12]
-; BE-NEXT:    strb w8, [x1, #2]
-; BE-NEXT:    strh w9, [x1]
+; BE-NEXT:    ldrh w8, [sp, #12]
+; BE-NEXT:    stur b0, [x1, #2]
+; BE-NEXT:    strh w8, [x1]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
 entry:
@@ -443,11 +443,11 @@ define void @load_ext_to_64bits(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    ldrh w9, [x0]
 ; CHECK-NEXT:    orr w8, w9, w8, lsl #16
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    add x8, x1, #4
 ; CHECK-NEXT:    zip1.8b v0, v0, v0
 ; CHECK-NEXT:    bic.4h v0, #255, lsl #8
-; CHECK-NEXT:    st1.h { v0 }[2], [x8]
+; CHECK-NEXT:    mov h1, v0[2]
 ; CHECK-NEXT:    str s0, [x1]
+; CHECK-NEXT:    str h1, [x1, #4]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: load_ext_to_64bits:
@@ -461,11 +461,11 @@ define void @load_ext_to_64bits(ptr %src, ptr %dst) {
 ; BE-NEXT:    rev32 v0.8b, v0.8b
 ; BE-NEXT:    ushll v0.8h, v0.8b, #0
 ; BE-NEXT:    ld1 { v0.b }[4], [x8]
-; BE-NEXT:    add x8, x1, #4
 ; BE-NEXT:    bic v0.4h, #255, lsl #8
 ; BE-NEXT:    rev32 v1.8h, v0.8h
-; BE-NEXT:    st1 { v0.h }[2], [x8]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    str s1, [x1]
+; BE-NEXT:    str h0, [x1, #4]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
 entry:
@@ -479,23 +479,23 @@ define void @load_ext_to_64bits_default_align(ptr %src, ptr %dst) {
 ; CHECK-LABEL: load_ext_to_64bits_default_align:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    add x8, x1, #4
 ; CHECK-NEXT:    zip1.8b v0, v0, v0
 ; CHECK-NEXT:    bic.4h v0, #255, lsl #8
-; CHECK-NEXT:    st1.h { v0 }[2], [x8]
+; CHECK-NEXT:    mov h1, v0[2]
 ; CHECK-NEXT:    str s0, [x1]
+; CHECK-NEXT:    str h1, [x1, #4]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: load_ext_to_64bits_default_align:
 ; BE:       // %bb.0: // %entry
 ; BE-NEXT:    ldr s0, [x0]
-; BE-NEXT:    add x8, x1, #4
 ; BE-NEXT:    rev32 v0.8b, v0.8b
 ; BE-NEXT:    zip1 v0.8b, v0.8b, v0.8b
 ; BE-NEXT:    bic v0.4h, #255, lsl #8
 ; BE-NEXT:    rev32 v1.8h, v0.8h
-; BE-NEXT:    st1 { v0.h }[2], [x8]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    str s1, [x1]
+; BE-NEXT:    str h0, [x1, #4]
 ; BE-NEXT:    ret
 entry:
   %l = load <3 x i8>, ptr %src
@@ -508,23 +508,23 @@ define void @load_ext_to_64bits_align_4(ptr %src, ptr %dst) {
 ; CHECK-LABEL: load_ext_to_64bits_align_4:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    add x8, x1, #4
 ; CHECK-NEXT:    zip1.8b v0, v0, v0
 ; CHECK-NEXT:    bic.4h v0, #255, lsl #8
-; CHECK-NEXT:    st1.h { v0 }[2], [x8]
+; CHECK-NEXT:    mov h1, v0[2]
 ; CHECK-NEXT:    str s0, [x1]
+; CHECK-NEXT:    str h1, [x1, #4]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: load_ext_to_64bits_align_4:
 ; BE:       // %bb.0: // %entry
 ; BE-NEXT:    ldr s0, [x0]
-; BE-NEXT:    add x8, x1, #4
 ; BE-NEXT:    rev32 v0.8b, v0.8b
 ; BE-NEXT:    zip1 v0.8b, v0.8b, v0.8b
 ; BE-NEXT:    bic v0.4h, #255, lsl #8
 ; BE-NEXT:    rev32 v1.8h, v0.8h
-; BE-NEXT:    st1 { v0.h }[2], [x8]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    str s1, [x1]
+; BE-NEXT:    str h0, [x1, #4]
 ; BE-NEXT:    ret
 entry:
   %l = load <3 x i8>, ptr %src, align 4
@@ -542,14 +542,14 @@ define void @load_ext_add_to_64bits(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    adrp x8, lCPI15_0@PAGE
 ; CHECK-NEXT:  Lloh3:
 ; CHECK-NEXT:    ldr d1, [x8, lCPI15_0@PAGEOFF]
-; CHECK-NEXT:    add x8, x1, #4
 ; CHECK-NEXT:    orr w9, w10, w9, lsl #16
 ; CHECK-NEXT:    fmov s0, w9
 ; CHECK-NEXT:    zip1.8b v0, v0, v0
 ; CHECK-NEXT:    bic.4h v0, #255, lsl #8
 ; CHECK-NEXT:    add.4h v0, v0, v1
-; CHECK-NEXT:    st1.h { v0 }[2], [x8]
+; CHECK-NEXT:    mov h1, v0[2]
 ; CHECK-NEXT:    str s0, [x1]
+; CHECK-NEXT:    str h1, [x1, #4]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh2, Lloh3
 ;
@@ -567,12 +567,12 @@ define void @load_ext_add_to_64bits(ptr %src, ptr %dst) {
 ; BE-NEXT:    adrp x8, .LCPI15_0
 ; BE-NEXT:    add x8, x8, :lo12:.LCPI15_0
 ; BE-NEXT:    ld1 { v1.4h }, [x8]
-; BE-NEXT:    add x8, x1, #4
 ; BE-NEXT:    bic v0.4h, #255, lsl #8
 ; BE-NEXT:    add v0.4h, v0.4h, v1.4h
 ; BE-NEXT:    rev32 v1.8h, v0.8h
-; BE-NEXT:    st1 { v0.h }[2], [x8]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    str s1, [x1]
+; BE-NEXT:    str h0, [x1, #4]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
 entry:
@@ -587,12 +587,12 @@ define void @shift_trunc_store(ptr %src, ptr %dst) {
 ; CHECK-LABEL: shift_trunc_store:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    add x8, x1, #1
-; CHECK-NEXT:    add x9, x1, #2
 ; CHECK-NEXT:    ushr.4s v0, v0, #16
-; CHECK-NEXT:    st1.b { v0 }[4], [x8]
-; CHECK-NEXT:    st1.b { v0 }[8], [x9]
-; CHECK-NEXT:    st1.b { v0 }[0], [x1]
+; CHECK-NEXT:    mov b1, v0[4]
+; CHECK-NEXT:    mov b2, v0[8]
+; CHECK-NEXT:    str b0, [x1]
+; CHECK-NEXT:    stur b1, [x1, #1]
+; CHECK-NEXT:    stur b2, [x1, #2]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: shift_trunc_store:
@@ -602,12 +602,12 @@ define void @shift_trunc_store(ptr %src, ptr %dst) {
 ; BE-NEXT:    ld1 { v0.4s }, [x0]
 ; BE-NEXT:    shrn v0.4h, v0.4s, #16
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #12]
-; BE-NEXT:    ldrh w9, [sp, #12]
-; BE-NEXT:    strb w8, [x1, #2]
-; BE-NEXT:    strh w9, [x1]
+; BE-NEXT:    ldrh w8, [sp, #12]
+; BE-NEXT:    stur b0, [x1, #2]
+; BE-NEXT:    strh w8, [x1]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
   %l = load <3 x i32>, ptr %src
@@ -621,12 +621,12 @@ define void @shift_trunc_store_default_align(ptr %src, ptr %dst) {
 ; CHECK-LABEL: shift_trunc_store_default_align:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    add x8, x1, #1
-; CHECK-NEXT:    add x9, x1, #2
 ; CHECK-NEXT:    ushr.4s v0, v0, #16
-; CHECK-NEXT:    st1.b { v0 }[4], [x8]
-; CHECK-NEXT:    st1.b { v0 }[8], [x9]
-; CHECK-NEXT:    st1.b { v0 }[0], [x1]
+; CHECK-NEXT:    mov b1, v0[4]
+; CHECK-NEXT:    mov b2, v0[8]
+; CHECK-NEXT:    str b0, [x1]
+; CHECK-NEXT:    stur b1, [x1, #1]
+; CHECK-NEXT:    stur b2, [x1, #2]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: shift_trunc_store_default_align:
@@ -636,12 +636,12 @@ define void @shift_trunc_store_default_align(ptr %src, ptr %dst) {
 ; BE-NEXT:    ld1 { v0.4s }, [x0]
 ; BE-NEXT:    shrn v0.4h, v0.4s, #16
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #12]
-; BE-NEXT:    ldrh w9, [sp, #12]
-; BE-NEXT:    strb w8, [x1, #2]
-; BE-NEXT:    strh w9, [x1]
+; BE-NEXT:    ldrh w8, [sp, #12]
+; BE-NEXT:    stur b0, [x1, #2]
+; BE-NEXT:    strh w8, [x1]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
   %l = load <3 x i32>, ptr %src
@@ -655,12 +655,12 @@ define void @shift_trunc_store_align_4(ptr %src, ptr %dst) {
 ; CHECK-LABEL: shift_trunc_store_align_4:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    add x8, x1, #1
-; CHECK-NEXT:    add x9, x1, #2
 ; CHECK-NEXT:    ushr.4s v0, v0, #16
-; CHECK-NEXT:    st1.b { v0 }[4], [x8]
-; CHECK-NEXT:    st1.b { v0 }[8], [x9]
-; CHECK-NEXT:    st1.b { v0 }[0], [x1]
+; CHECK-NEXT:    mov b1, v0[4]
+; CHECK-NEXT:    mov b2, v0[8]
+; CHECK-NEXT:    str b0, [x1]
+; CHECK-NEXT:    stur b1, [x1, #1]
+; CHECK-NEXT:    stur b2, [x1, #2]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: shift_trunc_store_align_4:
@@ -670,12 +670,12 @@ define void @shift_trunc_store_align_4(ptr %src, ptr %dst) {
 ; BE-NEXT:    ld1 { v0.4s }, [x0]
 ; BE-NEXT:    shrn v0.4h, v0.4s, #16
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #12]
-; BE-NEXT:    ldrh w9, [sp, #12]
-; BE-NEXT:    strb w8, [x1, #2]
-; BE-NEXT:    strh w9, [x1]
+; BE-NEXT:    ldrh w8, [sp, #12]
+; BE-NEXT:    stur b0, [x1, #2]
+; BE-NEXT:    strh w8, [x1]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
   %l = load <3 x i32>, ptr %src
@@ -689,13 +689,12 @@ define void @shift_trunc_store_const_offset_1(ptr %src, ptr %dst) {
 ; CHECK-LABEL: shift_trunc_store_const_offset_1:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    add x8, x1, #2
-; CHECK-NEXT:    add x9, x1, #3
 ; CHECK-NEXT:    ushr.4s v0, v0, #16
-; CHECK-NEXT:    st1.b { v0 }[4], [x8]
-; CHECK-NEXT:    add x8, x1, #1
-; CHECK-NEXT:    st1.b { v0 }[8], [x9]
-; CHECK-NEXT:    st1.b { v0 }[0], [x8]
+; CHECK-NEXT:    mov b1, v0[4]
+; CHECK-NEXT:    mov b2, v0[8]
+; CHECK-NEXT:    stur b0, [x1, #1]
+; CHECK-NEXT:    stur b1, [x1, #2]
+; CHECK-NEXT:    stur b2, [x1, #3]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: shift_trunc_store_const_offset_1:
@@ -705,12 +704,12 @@ define void @shift_trunc_store_const_offset_1(ptr %src, ptr %dst) {
 ; BE-NEXT:    ld1 { v0.4s }, [x0]
 ; BE-NEXT:    shrn v0.4h, v0.4s, #16
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #12]
-; BE-NEXT:    ldrh w9, [sp, #12]
-; BE-NEXT:    strb w8, [x1, #3]
-; BE-NEXT:    sturh w9, [x1, #1]
+; BE-NEXT:    ldrh w8, [sp, #12]
+; BE-NEXT:    stur b0, [x1, #3]
+; BE-NEXT:    sturh w8, [x1, #1]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
   %l = load <3 x i32>, ptr %src
@@ -725,13 +724,12 @@ define void @shift_trunc_store_const_offset_3(ptr %src, ptr %dst) {
 ; CHECK-LABEL: shift_trunc_store_const_offset_3:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    add x8, x1, #4
-; CHECK-NEXT:    add x9, x1, #5
 ; CHECK-NEXT:    ushr.4s v0, v0, #16
-; CHECK-NEXT:    st1.b { v0 }[4], [x8]
-; CHECK-NEXT:    add x8, x1, #3
-; CHECK-NEXT:    st1.b { v0 }[8], [x9]
-; CHECK-NEXT:    st1.b { v0 }[0], [x8]
+; CHECK-NEXT:    mov b1, v0[4]
+; CHECK-NEXT:    mov b2, v0[8]
+; CHECK-NEXT:    stur b0, [x1, #3]
+; CHECK-NEXT:    stur b1, [x1, #4]
+; CHECK-NEXT:    stur b2, [x1, #5]
 ; CHECK-NEXT:    ret
 ;
 ; BE-LABEL: shift_trunc_store_const_offset_3:
@@ -741,12 +739,12 @@ define void @shift_trunc_store_const_offset_3(ptr %src, ptr %dst) {
 ; BE-NEXT:    ld1 { v0.4s }, [x0]
 ; BE-NEXT:    shrn v0.4h, v0.4s, #16
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #12]
-; BE-NEXT:    ldrh w9, [sp, #12]
-; BE-NEXT:    strb w8, [x1, #5]
-; BE-NEXT:    sturh w9, [x1, #3]
+; BE-NEXT:    ldrh w8, [sp, #12]
+; BE-NEXT:    stur b0, [x1, #5]
+; BE-NEXT:    sturh w8, [x1, #3]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
   %l = load <3 x i32>, ptr %src
@@ -765,11 +763,11 @@ define void @shift_trunc_volatile_store(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    shrn.4h v0, v0, #16
 ; CHECK-NEXT:    uzp1.8b v1, v0, v0
-; CHECK-NEXT:    umov.h w8, v0[2]
+; CHECK-NEXT:    mov h0, v0[2]
 ; CHECK-NEXT:    str s1, [sp, #12]
-; CHECK-NEXT:    ldrh w9, [sp, #12]
-; CHECK-NEXT:    strb w8, [x1, #2]
-; CHECK-NEXT:    strh w9, [x1]
+; CHECK-NEXT:    ldrh w8, [sp, #12]
+; CHECK-NEXT:    stur b0, [x1, #2]
+; CHECK-NEXT:    strh w8, [x1]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
 ;
@@ -780,12 +778,12 @@ define void @shift_trunc_volatile_store(ptr %src, ptr %dst) {
 ; BE-NEXT:    ld1 { v0.4s }, [x0]
 ; BE-NEXT:    shrn v0.4h, v0.4s, #16
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #12]
-; BE-NEXT:    ldrh w9, [sp, #12]
-; BE-NEXT:    strb w8, [x1, #2]
-; BE-NEXT:    strh w9, [x1]
+; BE-NEXT:    ldrh w8, [sp, #12]
+; BE-NEXT:    stur b0, [x1, #2]
+; BE-NEXT:    strh w8, [x1]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
   %l = load <3 x i32>, ptr %src
@@ -804,15 +802,15 @@ define void @load_v3i8_zext_to_3xi32_add_trunc_store(ptr %src) {
 ; CHECK-NEXT:    adrp x8, lCPI22_0@PAGE
 ; CHECK-NEXT:  Lloh5:
 ; CHECK-NEXT:    ldr q1, [x8, lCPI22_0@PAGEOFF]
-; CHECK-NEXT:    add x8, x0, #2
 ; CHECK-NEXT:    orr w9, w10, w9, lsl #16
 ; CHECK-NEXT:    fmov s0, w9
 ; CHECK-NEXT:    zip1.8b v0, v0, v0
 ; CHECK-NEXT:    uaddw.4s v0, v1, v0
-; CHECK-NEXT:    st1.b { v0 }[8], [x8]
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    st1.b { v0 }[0], [x0]
-; CHECK-NEXT:    st1.b { v0 }[4], [x8]
+; CHECK-NEXT:    mov b1, v0[8]
+; CHECK-NEXT:    mov b2, v0[4]
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    stur b1, [x0, #2]
+; CHECK-NEXT:    stur b2, [x0, #1]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh4, Lloh5
 ;
@@ -832,12 +830,12 @@ define void @load_v3i8_zext_to_3xi32_add_trunc_store(ptr %src) {
 ; BE-NEXT:    ld1 { v0.b }[4], [x9]
 ; BE-NEXT:    add v0.4h, v0.4h, v1.4h
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #8]
-; BE-NEXT:    ldrh w9, [sp, #8]
-; BE-NEXT:    strb w8, [x0, #2]
-; BE-NEXT:    strh w9, [x0]
+; BE-NEXT:    ldrh w8, [sp, #8]
+; BE-NEXT:    stur b0, [x0, #2]
+; BE-NEXT:    strh w8, [x0]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
   %l = load <3 x i8>, ptr %src, align 1
@@ -857,15 +855,15 @@ define void @load_v3i8_sext_to_3xi32_add_trunc_store(ptr %src) {
 ; CHECK-NEXT:    adrp x8, lCPI23_0@PAGE
 ; CHECK-NEXT:  Lloh7:
 ; CHECK-NEXT:    ldr q1, [x8, lCPI23_0@PAGEOFF]
-; CHECK-NEXT:    add x8, x0, #2
 ; CHECK-NEXT:    orr w9, w10, w9, lsl #16
 ; CHECK-NEXT:    fmov s0, w9
 ; CHECK-NEXT:    zip1.8b v0, v0, v0
 ; CHECK-NEXT:    uaddw.4s v0, v1, v0
-; CHECK-NEXT:    st1.b { v0 }[8], [x8]
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    st1.b { v0 }[0], [x0]
-; CHECK-NEXT:    st1.b { v0 }[4], [x8]
+; CHECK-NEXT:    mov b1, v0[8]
+; CHECK-NEXT:    mov b2, v0[4]
+; CHECK-NEXT:    str b0, [x0]
+; CHECK-NEXT:    stur b1, [x0, #2]
+; CHECK-NEXT:    stur b2, [x0, #1]
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .loh AdrpLdr Lloh6, Lloh7
 ;
@@ -885,12 +883,12 @@ define void @load_v3i8_sext_to_3xi32_add_trunc_store(ptr %src) {
 ; BE-NEXT:    ld1 { v0.b }[4], [x9]
 ; BE-NEXT:    add v0.4h, v0.4h, v1.4h
 ; BE-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; BE-NEXT:    umov w8, v0.h[2]
+; BE-NEXT:    mov h0, v0.h[2]
 ; BE-NEXT:    rev32 v1.16b, v1.16b
 ; BE-NEXT:    str s1, [sp, #8]
-; BE-NEXT:    ldrh w9, [sp, #8]
-; BE-NEXT:    strb w8, [x0, #2]
-; BE-NEXT:    strh w9, [x0]
+; BE-NEXT:    ldrh w8, [sp, #8]
+; BE-NEXT:    stur b0, [x0, #2]
+; BE-NEXT:    strh w8, [x0]
 ; BE-NEXT:    add sp, sp, #16
 ; BE-NEXT:    ret
   %l = load <3 x i8>, ptr %src, align 1
diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
index 37c6374215d81..b29195eed9149 100644
--- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
@@ -50,10 +50,10 @@ define <3 x i32> @uaddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, ptr %p2) nounwind {
 ; CHECK-LABEL: uaddo_v3i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    add x8, x0, #8
+; CHECK-NEXT:    mov s2, v1.s[2]
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    st1 { v1.s }[2], [x8]
 ; CHECK-NEXT:    str d1, [x0]
+; CHECK-NEXT:    str s2, [x0, #8]
 ; CHECK-NEXT:    ret
   %t = call {<3 x i32>, <3 x i1>} @llvm.uadd.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
   %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
@@ -212,26 +212,26 @@ define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    bic v1.4s, #255, lsl #24
 ; CHECK-NEXT:    bic v0.4s, #255, lsl #24
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    mov v1.16b, v0.16b
-; CHECK-NEXT:    mov w8, v0.s[3]
-; CHECK-NEXT:    mov w9, v0.s[2]
-; CHECK-NEXT:    mov w10, v0.s[1]
-; CHECK-NEXT:    fmov w11, s0
-; CHECK-NEXT:    bic v1.4s, #1, lsl #24
+; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    mov w8, v1.s[3]
+; CHECK-NEXT:    mov w10, v1.s[1]
+; CHECK-NEXT:    mov w9, v1.s[2]
+; CHECK-NEXT:    str h1, [x0]
+; CHECK-NEXT:    bic v0.4s, #1, lsl #24
 ; CHECK-NEXT:    sturh w8, [x0, #9]
 ; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    strh w9, [x0, #6]
-; CHECK-NEXT:    lsr w9, w9, #16
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v0.4s
+; CHECK-NEXT:    sturh w10, [x0, #3]
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    strb w8, [x0, #11]
 ; CHECK-NEXT:    lsr w8, w10, #16
-; CHECK-NEXT:    strb w9, [x0, #8]
-; CHECK-NEXT:    lsr w9, w11, #16
-; CHECK-NEXT:    sturh w10, [x0, #3]
-; CHECK-NEXT:    mvn v0.16b, v1.16b
-; CHECK-NEXT:    strh w11, [x0]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    strh w9, [x0, #6]
+; CHECK-NEXT:    lsr w9, w9, #16
 ; CHECK-NEXT:    strb w8, [x0, #5]
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    strb w9, [x0, #8]
+; CHECK-NEXT:    lsr w9, w10, #16
 ; CHECK-NEXT:    strb w9, [x0, #2]
 ; CHECK-NEXT:    ret
   %t = call {<4 x i24>, <4 x i1>} @llvm.uadd.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
@@ -249,15 +249,14 @@ define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    adrp x8, .LCPI10_0
 ; CHECK-NEXT:    shl v1.4h, v2.4h, #15
-; CHECK-NEXT:    ldr d2, [x8, :lo12:.LCPI10_0]
 ; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    ldr d2, [x8, :lo12:.LCPI10_0]
 ; CHECK-NEXT:    cmlt v1.4h, v1.4h, #0
 ; CHECK-NEXT:    shl v0.4s, v0.4s, #31
 ; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
 ; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
 ; CHECK-NEXT:    addv h1, v1.4h
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    str b1, [x0]
 ; CHECK-NEXT:    ret
   %t = call {<4 x i1>, <4 x i1>} @llvm.uadd.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
   %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vec_umulo.ll b/llvm/test/CodeGen/AArch64/vec_umulo.ll
index 3a481efd9785a..12ea8862a03cd 100644
--- a/llvm/test/CodeGen/AArch64/vec_umulo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_umulo.ll
@@ -55,13 +55,12 @@ define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, ptr %p2) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umull2 v2.2d, v0.4s, v1.4s
 ; CHECK-NEXT:    umull v3.2d, v0.2s, v1.2s
-; CHECK-NEXT:    add x8, x0, #8
 ; CHECK-NEXT:    mul v1.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    uzp2 v2.4s, v3.4s, v2.4s
-; CHECK-NEXT:    st1 { v1.s }[2], [x8]
 ; CHECK-NEXT:    str d1, [x0]
-; CHECK-NEXT:    cmtst v2.4s, v2.4s, v2.4s
-; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    cmtst v0.4s, v2.4s, v2.4s
+; CHECK-NEXT:    mov s2, v1.s[2]
+; CHECK-NEXT:    str s2, [x0, #8]
 ; CHECK-NEXT:    ret
   %t = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
   %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
@@ -260,27 +259,27 @@ define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {
 ; CHECK-NEXT:    bic v0.4s, #255, lsl #24
 ; CHECK-NEXT:    umull2 v2.2d, v0.4s, v1.4s
 ; CHECK-NEXT:    umull v3.2d, v0.2s, v1.2s
-; CHECK-NEXT:    mul v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp2 v1.4s, v3.4s, v2.4s
-; CHECK-NEXT:    ushr v2.4s, v0.4s, #24
-; CHECK-NEXT:    mov w8, v0.s[3]
-; CHECK-NEXT:    mov w9, v0.s[2]
-; CHECK-NEXT:    mov w10, v0.s[1]
-; CHECK-NEXT:    fmov w11, s0
+; CHECK-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mov w8, v1.s[3]
+; CHECK-NEXT:    uzp2 v0.4s, v3.4s, v2.4s
+; CHECK-NEXT:    ushr v2.4s, v1.4s, #24
+; CHECK-NEXT:    mov w10, v1.s[1]
+; CHECK-NEXT:    mov w9, v1.s[2]
+; CHECK-NEXT:    str h1, [x0]
 ; CHECK-NEXT:    cmtst v2.4s, v2.4s, v2.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, #0
 ; CHECK-NEXT:    sturh w8, [x0, #9]
 ; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    strh w9, [x0, #6]
-; CHECK-NEXT:    lsr w9, w9, #16
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    sturh w10, [x0, #3]
 ; CHECK-NEXT:    strb w8, [x0, #11]
 ; CHECK-NEXT:    lsr w8, w10, #16
-; CHECK-NEXT:    orn v0.16b, v2.16b, v1.16b
-; CHECK-NEXT:    strb w9, [x0, #8]
-; CHECK-NEXT:    lsr w9, w11, #16
-; CHECK-NEXT:    sturh w10, [x0, #3]
-; CHECK-NEXT:    strh w11, [x0]
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    strh w9, [x0, #6]
+; CHECK-NEXT:    lsr w9, w9, #16
+; CHECK-NEXT:    orn v0.16b, v2.16b, v0.16b
 ; CHECK-NEXT:    strb w8, [x0, #5]
+; CHECK-NEXT:    strb w9, [x0, #8]
+; CHECK-NEXT:    lsr w9, w10, #16
 ; CHECK-NEXT:    strb w9, [x0, #2]
 ; CHECK-NEXT:    ret
   %t = call {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
@@ -299,11 +298,10 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI10_0]
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #15
 ; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    addv h1, v0.4h
+; CHECK-NEXT:    and v1.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    addv h1, v1.4h
+; CHECK-NEXT:    str b1, [x0]
 ; CHECK-NEXT:    ret
   %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
   %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vector-compress.ll b/llvm/test/CodeGen/AArch64/vector-compress.ll
index 710ea70d678c5..a580913d40d95 100644
--- a/llvm/test/CodeGen/AArch64/vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/vector-compress.ll
@@ -109,7 +109,7 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask) {
 ; CHECK-NEXT:    shl.16b v1, v1, #7
 ; CHECK-NEXT:    mov x12, sp
 ; CHECK-NEXT:    mov x8, sp
-; CHECK-NEXT:    st1.b { v0 }[0], [x8]
+; CHECK-NEXT:    str b0, [sp]
 ; CHECK-NEXT:    mov x13, sp
 ; CHECK-NEXT:    cmlt.16b v1, v1, #0
 ; CHECK-NEXT:    umov.b w9, v1[0]
@@ -209,44 +209,44 @@ define <8 x i32> @test_compress_large(<8 x i32> %vec, <8 x i1> %mask) {
 ; CHECK-NEXT:    ; kill: def $d2 killed $d2 def $q2
 ; CHECK-NEXT:    umov.b w9, v2[0]
 ; CHECK-NEXT:    umov.b w10, v2[1]
-; CHECK-NEXT:    mov x12, sp
-; CHECK-NEXT:    umov.b w11, v2[2]
+; CHECK-NEXT:    mov x11, sp
+; CHECK-NEXT:    umov.b w12, v2[2]
 ; CHECK-NEXT:    umov.b w13, v2[3]
 ; CHECK-NEXT:    mov x8, sp
-; CHECK-NEXT:    umov.b w14, v2[4]
+; CHECK-NEXT:    umov.b w15, v2[4]
 ; CHECK-NEXT:    str s0, [sp]
+; CHECK-NEXT:    mov s3, v0[3]
 ; CHECK-NEXT:    and x10, x10, #0x1
-; CHECK-NEXT:    and x15, x9, #0x1
-; CHECK-NEXT:    bfi x12, x9, #2, #1
-; CHECK-NEXT:    and x9, x11, #0x1
-; CHECK-NEXT:    add x10, x15, x10
-; CHECK-NEXT:    umov.b w11, v2[5]
-; CHECK-NEXT:    add x9, x10, x9
-; CHECK-NEXT:    orr x15, x8, x10, lsl #2
-; CHECK-NEXT:    umov.b w10, v2[6]
-; CHECK-NEXT:    st1.s { v0 }[1], [x12]
-; CHECK-NEXT:    add x12, x8, x9, lsl #2
-; CHECK-NEXT:    and x13, x13, #0x1
-; CHECK-NEXT:    st1.s { v0 }[2], [x15]
-; CHECK-NEXT:    add x9, x9, x13
-; CHECK-NEXT:    st1.s { v0 }[3], [x12]
-; CHECK-NEXT:    and x12, x14, #0x1
-; CHECK-NEXT:    and x11, x11, #0x1
+; CHECK-NEXT:    and x14, x9, #0x1
+; CHECK-NEXT:    bfi x11, x9, #2, #1
+; CHECK-NEXT:    add x9, x14, x10
+; CHECK-NEXT:    umov.b w10, v2[5]
+; CHECK-NEXT:    st1.s { v0 }[1], [x11]
+; CHECK-NEXT:    and x11, x12, #0x1
+; CHECK-NEXT:    orr x14, x8, x9, lsl #2
+; CHECK-NEXT:    and x12, x13, #0x1
+; CHECK-NEXT:    add x9, x9, x11
+; CHECK-NEXT:    umov.b w11, v2[6]
+; CHECK-NEXT:    and x13, x15, #0x1
 ; CHECK-NEXT:    add x12, x9, x12
-; CHECK-NEXT:    and w10, w10, #0x1
-; CHECK-NEXT:    and x9, x9, #0x7
-; CHECK-NEXT:    add x11, x12, x11
+; CHECK-NEXT:    st1.s { v0 }[2], [x14]
+; CHECK-NEXT:    str s3, [x8, x9, lsl #2]
+; CHECK-NEXT:    and x9, x10, #0x1
+; CHECK-NEXT:    add x10, x12, x13
 ; CHECK-NEXT:    and x12, x12, #0x7
-; CHECK-NEXT:    str s1, [x8, x9, lsl #2]
-; CHECK-NEXT:    add w10, w11, w10
-; CHECK-NEXT:    and x11, x11, #0x7
-; CHECK-NEXT:    add x12, x8, x12, lsl #2
+; CHECK-NEXT:    add x9, x10, x9
 ; CHECK-NEXT:    and x10, x10, #0x7
-; CHECK-NEXT:    add x9, x8, x11, lsl #2
-; CHECK-NEXT:    add x8, x8, x10, lsl #2
-; CHECK-NEXT:    st1.s { v1 }[1], [x12]
-; CHECK-NEXT:    st1.s { v1 }[2], [x9]
-; CHECK-NEXT:    st1.s { v1 }[3], [x8]
+; CHECK-NEXT:    str s1, [x8, x12, lsl #2]
+; CHECK-NEXT:    and x12, x9, #0x7
+; CHECK-NEXT:    mov s0, v1[3]
+; CHECK-NEXT:    and w11, w11, #0x1
+; CHECK-NEXT:    add x10, x8, x10, lsl #2
+; CHECK-NEXT:    add x12, x8, x12, lsl #2
+; CHECK-NEXT:    add w9, w9, w11
+; CHECK-NEXT:    and x9, x9, #0x7
+; CHECK-NEXT:    st1.s { v1 }[1], [x10]
+; CHECK-NEXT:    st1.s { v1 }[2], [x12]
+; CHECK-NEXT:    str s0, [x8, x9, lsl #2]
 ; CHECK-NEXT:    ldp q0, q1, [sp], #32
 ; CHECK-NEXT:    ret
     %out = call <8 x i32> @llvm.experimental.vector.compress(<8 x i32> %vec, <8 x i1> %mask, <8 x i32> undef)
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index 6536f0c355b47..eb83aa5a13e52 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1570,36 +1570,36 @@ define void @zext_v8i8_to_v8i128_in_loop(ptr %src, ptr %dst) {
 ; CHECK-NEXT:  LBB16_1: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr d0, [x0, x8]
-; CHECK-NEXT:    add x9, x1, #112
 ; CHECK-NEXT:    add x8, x8, #16
 ; CHECK-NEXT:    str xzr, [x1, #120]
+; CHECK-NEXT:    str xzr, [x1, #104]
 ; CHECK-NEXT:    cmp x8, #128
 ; CHECK-NEXT:    ushll.8h v0, v0, #0
-; CHECK-NEXT:    str xzr, [x1, #104]
 ; CHECK-NEXT:    str xzr, [x1, #88]
 ; CHECK-NEXT:    str xzr, [x1, #72]
+; CHECK-NEXT:    str xzr, [x1, #56]
 ; CHECK-NEXT:    ushll2.4s v1, v0, #0
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    str xzr, [x1, #56]
 ; CHECK-NEXT:    str xzr, [x1, #40]
 ; CHECK-NEXT:    str xzr, [x1, #24]
+; CHECK-NEXT:    str xzr, [x1, #8]
 ; CHECK-NEXT:    ushll2.2d v2, v1, #0
 ; CHECK-NEXT:    ushll.2d v1, v1, #0
 ; CHECK-NEXT:    ushll2.2d v3, v0, #0
 ; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    str xzr, [x1, #8]
-; CHECK-NEXT:    st1.d { v2 }[1], [x9]
-; CHECK-NEXT:    add x9, x1, #80
-; CHECK-NEXT:    st1.d { v1 }[1], [x9]
-; CHECK-NEXT:    add x9, x1, #48
 ; CHECK-NEXT:    str d2, [x1, #96]
-; CHECK-NEXT:    st1.d { v3 }[1], [x9]
-; CHECK-NEXT:    add x9, x1, #16
+; CHECK-NEXT:    mov d2, v2[1]
 ; CHECK-NEXT:    str d1, [x1, #64]
+; CHECK-NEXT:    mov d1, v1[1]
 ; CHECK-NEXT:    str d3, [x1, #32]
+; CHECK-NEXT:    mov d3, v3[1]
 ; CHECK-NEXT:    str d0, [x1]
+; CHECK-NEXT:    mov d0, v0[1]
+; CHECK-NEXT:    str d2, [x1, #112]
+; CHECK-NEXT:    str d1, [x1, #80]
+; CHECK-NEXT:    str d3, [x1, #48]
+; CHECK-NEXT:    str d0, [x1, #16]
 ; CHECK-NEXT:    add x1, x1, #256
-; CHECK-NEXT:    st1.d { v0 }[1], [x9]
 ; CHECK-NEXT:    b.ne LBB16_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
 ; CHECK-NEXT:    ret
@@ -1612,10 +1612,9 @@ define void @zext_v8i8_to_v8i128_in_loop(ptr %src, ptr %dst) {
 ; CHECK-BE-NEXT:    add x9, x0, x8
 ; CHECK-BE-NEXT:    add x8, x8, #16
 ; CHECK-BE-NEXT:    ld1 { v0.8b }, [x9]
-; CHECK-BE-NEXT:    add x9, x1, #120
 ; CHECK-BE-NEXT:    str xzr, [x1, #112]
-; CHECK-BE-NEXT:    str xzr, [x1, #96]
 ; CHECK-BE-NEXT:    cmp x8, #128
+; CHECK-BE-NEXT:    str xzr, [x1, #96]
 ; CHECK-BE-NEXT:    str xzr, [x1, #80]
 ; CHECK-BE-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-BE-NEXT:    str xzr, [x1, #64]
@@ -1629,18 +1628,19 @@ define void @zext_v8i8_to_v8i128_in_loop(ptr %src, ptr %dst) {
 ; CHECK-BE-NEXT:    ushll v1.2d, v1.2s, #0
 ; CHECK-BE-NEXT:    ushll2 v3.2d, v0.4s, #0
 ; CHECK-BE-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-BE-NEXT:    st1 { v2.d }[1], [x9]
-; CHECK-BE-NEXT:    add x9, x1, #88
-; CHECK-BE-NEXT:    st1 { v1.d }[1], [x9]
-; CHECK-BE-NEXT:    add x9, x1, #56
 ; CHECK-BE-NEXT:    str d2, [x1, #104]
-; CHECK-BE-NEXT:    st1 { v3.d }[1], [x9]
-; CHECK-BE-NEXT:    add x9, x1, #24
+; CHECK-BE-NEXT:    mov d2, v2.d[1]
 ; CHECK-BE-NEXT:    str d1, [x1, #72]
+; CHECK-BE-NEXT:    mov d1, v1.d[1]
 ; CHECK-BE-NEXT:    str d3, [x1, #40]
+; CHECK-BE-NEXT:    mov d3, v3.d[1]
 ; CHECK-BE-NEXT:    str d0, [x1, #8]
+; CHECK-BE-NEXT:    mov d0, v0.d[1]
+; CHECK-BE-NEXT:    str d2, [x1, #120]
+; CHECK-BE-NEXT:    str d1, [x1, #88]
+; CHECK-BE-NEXT:    str d3, [x1, #56]
+; CHECK-BE-NEXT:    str d0, [x1, #24]
 ; CHECK-BE-NEXT:    add x1, x1, #256
-; CHECK-BE-NEXT:    st1 { v0.d }[1], [x9]
 ; CHECK-BE-NEXT:    b.ne .LBB16_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit
 ; CHECK-BE-NEXT:    ret
@@ -2031,58 +2031,7 @@ exit:
   ret void
 }
 
-; CHECK-LABEL:  lCPI20_0:
-; CHECK-NEXT:	.byte	0                               ; 0x0
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	1                               ; 0x1
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	2                               ; 0x2
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	3                               ; 0x3
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	4                               ; 0x4
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	5                               ; 0x5
-; CHECK-NEXT:lCPI20_1:
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	6                               ; 0x6
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	7                               ; 0x7
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	8                               ; 0x8
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	9                               ; 0x9
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	10                              ; 0xa
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:lCPI20_2:
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	11                              ; 0xb
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	12                              ; 0xc
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	13                              ; 0xd
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	14                              ; 0xe
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	15                              ; 0xf
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:	.byte	255                             ; 0xff
-; CHECK-NEXT:lCPI20_3:
+; CHECK-LABEL: lCPI20_0:
 ; CHECK-NEXT:	.byte	0                               ; 0x0
 ; CHECK-NEXT:	.byte	255                             ; 0xff
 ; CHECK-NEXT:	.byte	255                             ; 0xff
@@ -2099,6 +2048,57 @@ exit:
 ; CHECK-NEXT:	.byte	255                             ; 0xff
 ; CHECK-NEXT:	.byte	255                             ; 0xff
 ; CHECK-NEXT:	.byte	255                             ; 0xff
+; CHECK-NEXT: lCPI20_1:
+; CHECK-NEXT: .byte	0                               ; 0x0
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	1                               ; 0x1
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	2                               ; 0x2
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	3                               ; 0x3
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	4                               ; 0x4
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	5                               ; 0x5
+; CHECK-NEXT: lCPI20_2:
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	6                               ; 0x6
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	7                               ; 0x7
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	8                               ; 0x8
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	9                               ; 0x9
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	10                              ; 0xa
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: lCPI20_3:
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	11                              ; 0xb
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	12                              ; 0xc
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	13                              ; 0xd
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	14                              ; 0xe
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	15                              ; 0xf
+; CHECK-NEXT: .byte	255                             ; 0xff
+; CHECK-NEXT: .byte	255                             ; 0xff
 
 ; CHECK-BE-LABEL: .LCPI20_0:
 ; CHECK-BE-NEXT:  	.byte	255                             // 0xff
@@ -2193,18 +2193,18 @@ define void @zext_v20i8_to_v20i24_in_loop(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add x9, x0, x8
 ; CHECK-NEXT:    add x8, x8, #16
-; CHECK-NEXT:    ldp q5, q4, [x9]
-; CHECK-NEXT:    add x9, x1, #56
+; CHECK-NEXT:    ldp q4, q5, [x9]
 ; CHECK-NEXT:    cmp x8, #128
-; CHECK-NEXT:    tbl.16b v4, { v4 }, v3
-; CHECK-NEXT:    tbl.16b v6, { v5 }, v2
-; CHECK-NEXT:    tbl.16b v7, { v5 }, v1
 ; CHECK-NEXT:    tbl.16b v5, { v5 }, v0
+; CHECK-NEXT:    tbl.16b v6, { v4 }, v3
+; CHECK-NEXT:    tbl.16b v7, { v4 }, v2
+; CHECK-NEXT:    tbl.16b v4, { v4 }, v1
 ; CHECK-NEXT:    stp q7, q6, [x1, #16]
-; CHECK-NEXT:    str q5, [x1]
-; CHECK-NEXT:    str d4, [x1, #48]
+; CHECK-NEXT:    mov s6, v5[2]
+; CHECK-NEXT:    str q4, [x1]
+; CHECK-NEXT:    str d5, [x1, #48]
+; CHECK-NEXT:    str s6, [x1, #56]
 ; CHECK-NEXT:    add x1, x1, #64
-; CHECK-NEXT:    st1.s { v4 }[2], [x9]
 ; CHECK-NEXT:    b.ne LBB20_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
 ; CHECK-NEXT:    ret
@@ -2239,19 +2239,19 @@ define void @zext_v20i8_to_v20i24_in_loop(ptr %src, ptr %dst) {
 ; CHECK-BE-NEXT:    ld1 { v4.16b }, [x10]
 ; CHECK-BE-NEXT:    cmp x8, #128
 ; CHECK-BE-NEXT:    tbl v6.16b, { v5.16b }, v3.16b
-; CHECK-BE-NEXT:    tbl v7.16b, { v5.16b }, v2.16b
+; CHECK-BE-NEXT:    tbl v16.16b, { v5.16b }, v2.16b
 ; CHECK-BE-NEXT:    tbl v5.16b, { v5.16b }, v1.16b
 ; CHECK-BE-NEXT:    tbl v4.16b, { v4.16b }, v0.16b
 ; CHECK-BE-NEXT:    st1 { v6.16b }, [x9]
 ; CHECK-BE-NEXT:    add x9, x1, #16
-; CHECK-BE-NEXT:    rev32 v16.16b, v4.16b
+; CHECK-BE-NEXT:    rev32 v7.16b, v4.16b
 ; CHECK-BE-NEXT:    rev64 v4.16b, v4.16b
-; CHECK-BE-NEXT:    st1 { v7.16b }, [x9]
-; CHECK-BE-NEXT:    add x9, x1, #56
 ; CHECK-BE-NEXT:    st1 { v5.16b }, [x1]
+; CHECK-BE-NEXT:    st1 { v16.16b }, [x9]
+; CHECK-BE-NEXT:    mov s6, v7.s[2]
 ; CHECK-BE-NEXT:    str d4, [x1, #48]
+; CHECK-BE-NEXT:    str s6, [x1, #56]
 ; CHECK-BE-NEXT:    add x1, x1, #64
-; CHECK-BE-NEXT:    st1 { v16.s }[2], [x9]
 ; CHECK-BE-NEXT:    b.ne .LBB20_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit
 ; CHECK-BE-NEXT:    ret
@@ -2592,36 +2592,36 @@ define void @zext_v23i8_to_v23i48_in_loop(ptr %src, ptr %dst) {
 ; CHECK-BE-NEXT:    ld1 { v7.16b }, [x9]
 ; CHECK-BE-NEXT:    add x9, x9, #16
 ; CHECK-BE-NEXT:    cmp x8, #128
-; CHECK-BE-NEXT:    ld1 { v17.16b }, [x9]
+; CHECK-BE-NEXT:    ld1 { v16.16b }, [x9]
 ; CHECK-BE-NEXT:    add x9, x1, #80
-; CHECK-BE-NEXT:    tbl v16.16b, { v7.16b }, v6.16b
+; CHECK-BE-NEXT:    tbl v17.16b, { v7.16b }, v6.16b
 ; CHECK-BE-NEXT:    tbl v18.16b, { v7.16b }, v5.16b
-; CHECK-BE-NEXT:    tbl v19.16b, { v7.16b }, v4.16b
-; CHECK-BE-NEXT:    tbl v20.16b, { v7.16b }, v3.16b
-; CHECK-BE-NEXT:    tbl v21.16b, { v17.16b }, v0.16b
-; CHECK-BE-NEXT:    st1 { v16.16b }, [x9]
+; CHECK-BE-NEXT:    tbl v20.16b, { v7.16b }, v4.16b
+; CHECK-BE-NEXT:    tbl v19.16b, { v16.16b }, v0.16b
+; CHECK-BE-NEXT:    tbl v21.16b, { v7.16b }, v3.16b
+; CHECK-BE-NEXT:    st1 { v17.16b }, [x9]
 ; CHECK-BE-NEXT:    add x9, x1, #64
-; CHECK-BE-NEXT:    tbl v16.16b, { v7.16b }, v2.16b
+; CHECK-BE-NEXT:    tbl v17.16b, { v7.16b }, v2.16b
 ; CHECK-BE-NEXT:    st1 { v18.16b }, [x9]
 ; CHECK-BE-NEXT:    add x9, x1, #48
-; CHECK-BE-NEXT:    tbl v18.16b, { v17.16b }, v2.16b
-; CHECK-BE-NEXT:    st1 { v19.16b }, [x9]
-; CHECK-BE-NEXT:    add x9, x1, #32
-; CHECK-BE-NEXT:    tbl v17.16b, { v17.16b }, v1.16b
+; CHECK-BE-NEXT:    rev16 v18.16b, v19.16b
 ; CHECK-BE-NEXT:    st1 { v20.16b }, [x9]
+; CHECK-BE-NEXT:    add x9, x1, #32
+; CHECK-BE-NEXT:    tbl v20.16b, { v16.16b }, v2.16b
+; CHECK-BE-NEXT:    st1 { v21.16b }, [x9]
 ; CHECK-BE-NEXT:    add x9, x1, #16
-; CHECK-BE-NEXT:    rev64 v19.16b, v21.16b
-; CHECK-BE-NEXT:    st1 { v16.16b }, [x9]
-; CHECK-BE-NEXT:    rev16 v16.16b, v21.16b
+; CHECK-BE-NEXT:    tbl v16.16b, { v16.16b }, v1.16b
+; CHECK-BE-NEXT:    st1 { v17.16b }, [x9]
+; CHECK-BE-NEXT:    rev64 v17.16b, v19.16b
 ; CHECK-BE-NEXT:    add x9, x1, #112
-; CHECK-BE-NEXT:    st1 { v18.16b }, [x9]
-; CHECK-BE-NEXT:    add x9, x1, #96
 ; CHECK-BE-NEXT:    tbl v7.16b, { v7.16b }, v1.16b
-; CHECK-BE-NEXT:    st1 { v17.16b }, [x9]
-; CHECK-BE-NEXT:    add x9, x1, #136
-; CHECK-BE-NEXT:    st1 { v16.h }[4], [x9]
-; CHECK-BE-NEXT:    fmov x9, d19
+; CHECK-BE-NEXT:    mov h18, v18.h[4]
+; CHECK-BE-NEXT:    st1 { v20.16b }, [x9]
+; CHECK-BE-NEXT:    add x9, x1, #96
+; CHECK-BE-NEXT:    st1 { v16.16b }, [x9]
+; CHECK-BE-NEXT:    fmov x9, d17
 ; CHECK-BE-NEXT:    st1 { v7.16b }, [x1]
+; CHECK-BE-NEXT:    str h18, [x1, #136]
 ; CHECK-BE-NEXT:    str x9, [x1, #128]!
 ; CHECK-BE-NEXT:    b.ne .LBB21_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit

From 5c97397c2c414d7ac8e43e87c6d2a79a87e802fb Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 17 Apr 2025 10:42:07 +0200
Subject: [PATCH 237/710] [SROA] Support load-only promotion with dynamic
 offset loads (#135609)

If we do load-only promotion, it is okay if we leave some loads alone.
We only need to know all stores that affect a specific location.

As such, we can handle loads with unknown offset via the "escaped
read-only" code path.

This is something we already support in LICM load-only promotion, but
doing this in SROA is much better from a phase ordering perspective.

Fixes https://github.com/llvm/llvm-project/issues/134513.
---
 llvm/lib/Transforms/Scalar/SROA.cpp            | 4 +++-
 llvm/test/Transforms/SROA/readonlynocapture.ll | 8 ++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 7d49b63a8e4f6..a4e373d395b90 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -1114,8 +1114,10 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
     assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
            "All simple FCA loads should have been pre-split");
 
+    // If there is a load with an unknown offset, we can still perform store
+    // to load forwarding for other known-offset loads.
     if (!IsOffsetKnown)
-      return PI.setAborted(&LI);
+      return PI.setEscapedReadOnly(&LI);
 
     TypeSize Size = DL.getTypeStoreSize(LI.getType());
     if (Size.isScalable())
diff --git a/llvm/test/Transforms/SROA/readonlynocapture.ll b/llvm/test/Transforms/SROA/readonlynocapture.ll
index 90ca72462f7c1..5752fadd76d48 100644
--- a/llvm/test/Transforms/SROA/readonlynocapture.ll
+++ b/llvm/test/Transforms/SROA/readonlynocapture.ll
@@ -501,6 +501,8 @@ define i32 @twoalloc_with_lifetimes() {
 
 declare void @use.i32(i32)
 
+; We can promote the %i load, even though there is an unknown offset load
+; in the loop. It is sufficient that we know all stores.
 define void @load_dyn_offset(ptr %ary) {
 ; CHECK-LABEL: @load_dyn_offset(
 ; CHECK-NEXT:    [[A:%.*]] = alloca { i64, [4 x i32] }, align 8
@@ -509,8 +511,8 @@ define void @load_dyn_offset(ptr %ary) {
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[GEP]], ptr [[ARY:%.*]], i64 16, i1 false)
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = load i64, ptr [[A]], align 4
-; CHECK-NEXT:    [[I_NEXT:%.*]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
 ; CHECK-NEXT:    store i64 [[I_NEXT]], ptr [[A]], align 4
 ; CHECK-NEXT:    [[GEP_I:%.*]] = getelementptr i32, ptr [[GEP]], i64 [[I]]
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[GEP_I]], align 4
@@ -540,6 +542,8 @@ exit:
   ret void
 }
 
+; Same as previous test, but with an unknown-offset store. We can't promote in
+; that case.
 define void @store_dyn_offset(ptr %ary) {
 ; CHECK-LABEL: @store_dyn_offset(
 ; CHECK-NEXT:    [[A:%.*]] = alloca { i64, [4 x i32] }, align 8

From 046a1e629c49365b1cd5f1ff17f650fcce186494 Mon Sep 17 00:00:00 2001
From: Will Froom <willfroom@google.com>
Date: Thu, 17 Apr 2025 09:45:43 +0100
Subject: [PATCH 238/710] [MLIR:LLVM] Add UWTableKind attribute (#135811)

Add `UWTableKind` enum and corresponding attribute to `llvm.func`
including translation to `llvm::Function` attribute.
---
 .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td       |  9 ++++++++
 mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td | 21 +++++++++++++++++++
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   |  3 ++-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |  7 +++++++
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  |  3 +++
 mlir/test/Target/LLVMIR/Import/uwtable.ll     |  7 +++++++
 mlir/test/Target/LLVMIR/uwtable.mlir          |  8 +++++++
 7 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 mlir/test/Target/LLVMIR/Import/uwtable.ll
 create mode 100644 mlir/test/Target/LLVMIR/uwtable.mlir

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index bb528fec8c684..0f195ff82c3ff 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -1365,4 +1365,13 @@ def LLVM_DependentLibrariesAttr
   let assemblyFormat = "`<` $libs `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// UWTableKindAttr
+//===----------------------------------------------------------------------===//
+
+def UWTableKindAttr : LLVM_Attr<"UWTableKind", "uwtableKind"> {
+  let parameters = (ins "uwtable::UWTableKind":$uwtableKind);
+  let assemblyFormat = "`<` $uwtableKind `>`";
+}
+
 #endif // LLVMIR_ATTRDEFS
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
index 34a30a00790ea..45ccf30644920 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
@@ -855,4 +855,25 @@ def ModFlagBehaviorAttr : LLVM_EnumAttr<
   let cppNamespace = "::mlir::LLVM";
 }
 
+//===----------------------------------------------------------------------===//
+// UWTableKind
+//===----------------------------------------------------------------------===//
+
+def UWTableKindNone
+    : LLVM_EnumAttrCase<"None", "none", "None", 0>;
+def UWTableKindSync
+    : LLVM_EnumAttrCase<"Sync", "sync", "Sync", 1>;
+def UWTableKindAsync
+    : LLVM_EnumAttrCase<"Async", "async", "Async", 2>;
+
+// UWTableKind::Default is unsupported as the llvm enum value is the same as async  
+// which the generated enum converters can't deal with.
+def UWTableKindEnum : LLVM_EnumAttr<
+    "UWTableKind",
+    "::llvm::UWTableKind",
+    "LLVM Unwind Behavior",
+    [UWTableKindNone, UWTableKindSync, UWTableKindAsync]> {
+  let cppNamespace = "::mlir::LLVM::uwtable";
+}
+
 #endif // LLVMIR_ENUMS
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index e89e78aec7147..76a2ec47b3a22 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1842,7 +1842,8 @@ def LLVM_LLVMFuncOp : LLVM_Op<"func", [
     OptionalAttr<LLVM_VecTypeHintAttr>:$vec_type_hint,
     OptionalAttr<DenseI32ArrayAttr>:$work_group_size_hint,
     OptionalAttr<DenseI32ArrayAttr>:$reqd_work_group_size,
-    OptionalAttr<I32Attr>:$intel_reqd_sub_group_size
+    OptionalAttr<I32Attr>:$intel_reqd_sub_group_size,
+    OptionalAttr<UWTableKindAttr>:$uwtable_kind
   );
 
   let regions = (region AnyRegion:$body);
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 8e7a7ab95b6b6..481d719787397 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -2075,6 +2075,7 @@ static constexpr std::array kExplicitAttributes{
     StringLiteral("target-features"),
     StringLiteral("tune-cpu"),
     StringLiteral("unsafe-fp-math"),
+    StringLiteral("uwtable"),
     StringLiteral("vscale_range"),
     StringLiteral("willreturn"),
 };
@@ -2237,6 +2238,12 @@ void ModuleImport::processFunctionAttributes(llvm::Function *func,
   if (llvm::Attribute attr = func->getFnAttribute("fp-contract");
       attr.isStringAttribute())
     funcOp.setFpContractAttr(StringAttr::get(context, attr.getValueAsString()));
+
+  if (func->hasUWTable()) {
+    ::llvm::UWTableKind uwtableKind = func->getUWTableKind();
+    funcOp.setUwtableKindAttr(LLVM::UWTableKindAttr::get(
+        funcOp.getContext(), convertUWTableKindFromLLVM(uwtableKind)));
+  }
 }
 
 DictionaryAttr
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index ee7dc3a5231f4..d39ec0e801437 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -1602,6 +1602,9 @@ static void convertFunctionAttributes(LLVMFuncOp func,
   if (FramePointerKindAttr fpAttr = func.getFramePointerAttr())
     llvmFunc->addFnAttr("frame-pointer", stringifyFramePointerKind(
                                              fpAttr.getFramePointerKind()));
+  if (UWTableKindAttr uwTableKindAttr = func.getUwtableKindAttr())
+    llvmFunc->setUWTableKind(
+        convertUWTableKindToLLVM(uwTableKindAttr.getUwtableKind()));
   convertFunctionMemoryAttributes(func, llvmFunc);
 }
 
diff --git a/mlir/test/Target/LLVMIR/Import/uwtable.ll b/mlir/test/Target/LLVMIR/Import/uwtable.ll
new file mode 100644
index 0000000000000..e5f5884240857
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/uwtable.ll
@@ -0,0 +1,7 @@
+; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s
+
+; CHECK-LABEL: llvm.func @uwtable_func
+; CHECK-SAME: attributes {uwtable_kind = #llvm.uwtableKind<sync>}
+define void @uwtable_func() uwtable(sync) {
+  ret void
+}
diff --git a/mlir/test/Target/LLVMIR/uwtable.mlir b/mlir/test/Target/LLVMIR/uwtable.mlir
new file mode 100644
index 0000000000000..bd9a208ea75bb
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/uwtable.mlir
@@ -0,0 +1,8 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK-LABEL: define void @uwtable_func() 
+// CHECK-SAME: #[[ATTRS:[0-9]+]]
+llvm.func @uwtable_func() attributes {uwtable_kind = #llvm.uwtableKind<sync>}  {
+  llvm.return
+}
+// CHECK: attributes #[[ATTRS]] = { uwtable(sync) }

From 2a91d04b022f1295fe2057bc88dc89987e6a6e04 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Thu, 17 Apr 2025 11:00:56 +0200
Subject: [PATCH 239/710] Revert "[Clang] Bypass TAD during overload resolution
 if a perfect match exists" (#136113)

Reverts llvm/llvm-project#136018

Still some bots failing
https://lab.llvm.org/buildbot/#/builders/52/builds/7643
---
 clang/docs/ReleaseNotes.rst                   |   6 -
 clang/include/clang/Sema/Overload.h           | 232 +-------
 clang/lib/Sema/SemaCodeComplete.cpp           |   6 +-
 clang/lib/Sema/SemaInit.cpp                   |  18 +-
 clang/lib/Sema/SemaOverload.cpp               | 532 ++++--------------
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   4 +-
 .../constrant-satisfaction-conversions.cpp    |   8 +-
 clang/test/SemaCUDA/function-overload.cu      |   3 +
 .../SemaCXX/implicit-member-functions.cpp     |  21 +-
 ...overload-resolution-deferred-templates.cpp | 200 -------
 .../instantiate-function-params.cpp           |   7 +-
 .../Templight/templight-empty-entries-fix.cpp | 126 +++--
 12 files changed, 219 insertions(+), 944 deletions(-)
 delete mode 100644 clang/test/SemaCXX/overload-resolution-deferred-templates.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index fc36962b317e8..4f640697e1817 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -96,12 +96,6 @@ C++ Language Changes
       asm((std::string_view("nop")) ::: (std::string_view("memory")));
     }
 
-- Clang now implements the changes to overload resolution proposed by section 1 and 2 of
-  `P3606 <https://wg21.link/P3606R0>`_. If a non-template candidate exists in an overload set that is
-  a perfect match (all conversion sequences are identity conversions) template candidates are not instantiated.
-  Diagnostics that would have resulted from the instantiation of these template candidates are no longer
-  produced. This aligns Clang closer to the behavior of GCC, and fixes (#GH62096), (#GH74581), and (#GH74581).
-
 C++2c Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index e667147bfac7e..6e08762dcc6d7 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -407,26 +407,6 @@ class Sema;
              Third == ICK_Identity;
     }
 
-    /// A conversion sequence is perfect if it is an identity conversion and
-    /// the type of the source is the same as the type of the target.
-    bool isPerfect(const ASTContext &C) const {
-      if (!isIdentityConversion())
-        return false;
-      // If we are not performing a reference binding, we can skip comparing
-      // the types, which has a noticeable performance impact.
-      if (!ReferenceBinding) {
-        // The types might differ if there is an array-to-pointer conversion
-        // or lvalue-to-rvalue conversion.
-        assert(First || C.hasSameUnqualifiedType(getFromType(), getToType(2)));
-        return true;
-      }
-      if (!C.hasSameType(getFromType(), getToType(2)))
-        return false;
-      if (BindsToRvalue && IsLvalueReference)
-        return false;
-      return true;
-    }
-
     ImplicitConversionRank getRank() const;
     NarrowingKind
     getNarrowingKind(ASTContext &Context, const Expr *Converted,
@@ -763,12 +743,6 @@ class Sema;
       Standard.setAllToTypes(T);
     }
 
-    /// A conversion sequence is perfect if it is an identity conversion and
-    /// the type of the source is the same as the type of the target.
-    bool isPerfect(const ASTContext &C) const {
-      return isStandard() && Standard.isPerfect(C);
-    }
-
     // True iff this is a conversion sequence from an initializer list to an
     // array or std::initializer.
     bool hasInitializerListContainerType() const {
@@ -965,10 +939,6 @@ class Sema;
     LLVM_PREFERRED_TYPE(CallExpr::ADLCallKind)
     unsigned IsADLCandidate : 1;
 
-    /// Whether FinalConversion has been set.
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned HasFinalConversion : 1;
-
     /// Whether this is a rewritten candidate, and if so, of what kind?
     LLVM_PREFERRED_TYPE(OverloadCandidateRewriteKind)
     unsigned RewriteKind : 2;
@@ -1009,20 +979,6 @@ class Sema;
       return false;
     }
 
-    // An overload is a perfect match if the conversion
-    // sequences for each argument are perfect.
-    bool isPerfectMatch(const ASTContext &Ctx) const {
-      if (!Viable)
-        return false;
-      for (const auto &C : Conversions) {
-        if (!C.isInitialized() || !C.isPerfect(Ctx))
-          return false;
-      }
-      if (HasFinalConversion)
-        return FinalConversion.isPerfect(Ctx);
-      return true;
-    }
-
     bool TryToFixBadConversion(unsigned Idx, Sema &S) {
       bool CanFix = Fix.tryToFixConversion(
                       Conversions[Idx].Bad.FromExpr,
@@ -1056,67 +1012,8 @@ class Sema;
         : IsSurrogate(false), IgnoreObjectArgument(false),
           TookAddressOfOverload(false), StrictPackMatch(false),
           IsADLCandidate(llvm::to_underlying(CallExpr::NotADL)),
-          HasFinalConversion(false), RewriteKind(CRK_None) {}
-  };
-
-  struct DeferredTemplateOverloadCandidate {
-
-    // intrusive linked list support for allocateDeferredCandidate
-    DeferredTemplateOverloadCandidate *Next = nullptr;
-
-    enum Kind { Function, Method, Conversion };
-
-    LLVM_PREFERRED_TYPE(Kind)
-    unsigned Kind : 2;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned AllowObjCConversionOnExplicit : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned AllowResultConversion : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned AllowExplicit : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned SuppressUserConversions : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned PartialOverloading : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned AggregateCandidateDeduction : 1;
-  };
-
-  struct DeferredFunctionTemplateOverloadCandidate
-      : public DeferredTemplateOverloadCandidate {
-    FunctionTemplateDecl *FunctionTemplate;
-    DeclAccessPair FoundDecl;
-    ArrayRef<Expr *> Args;
-    CallExpr::ADLCallKind IsADLCandidate;
-    OverloadCandidateParamOrder PO;
-  };
-  static_assert(std::is_trivially_destructible_v<
-                DeferredFunctionTemplateOverloadCandidate>);
-
-  struct DeferredMethodTemplateOverloadCandidate
-      : public DeferredTemplateOverloadCandidate {
-    FunctionTemplateDecl *FunctionTemplate;
-    DeclAccessPair FoundDecl;
-    ArrayRef<Expr *> Args;
-    CXXRecordDecl *ActingContext;
-    Expr::Classification ObjectClassification;
-    QualType ObjectType;
-    OverloadCandidateParamOrder PO;
+          RewriteKind(CRK_None) {}
   };
-  static_assert(std::is_trivially_destructible_v<
-                DeferredMethodTemplateOverloadCandidate>);
-
-  struct DeferredConversionTemplateOverloadCandidate
-      : public DeferredTemplateOverloadCandidate {
-    FunctionTemplateDecl *FunctionTemplate;
-    DeclAccessPair FoundDecl;
-    CXXRecordDecl *ActingContext;
-    Expr *From;
-    QualType ToType;
-  };
-
-  static_assert(std::is_trivially_destructible_v<
-                DeferredConversionTemplateOverloadCandidate>);
 
   /// OverloadCandidateSet - A set of overload candidates, used in C++
   /// overload resolution (C++ 13.3).
@@ -1146,11 +1043,6 @@ class Sema;
       /// C++ [over.match.call.general]
       /// Resolve a call through the address of an overload set.
       CSK_AddressOfOverloadSet,
-
-      /// When doing overload resolution during code completion,
-      /// we want to show all viable candidates, including otherwise
-      /// deferred template candidates.
-      CSK_CodeCompletion,
     };
 
     /// Information about operator rewrites to consider when adding operator
@@ -1225,15 +1117,7 @@ class Sema;
     SmallVector<OverloadCandidate, 16> Candidates;
     llvm::SmallPtrSet<uintptr_t, 16> Functions;
 
-    DeferredTemplateOverloadCandidate *FirstDeferredCandidate = nullptr;
-    unsigned DeferredCandidatesCount : 8 * sizeof(unsigned) - 2;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned HasDeferredTemplateConstructors : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned ResolutionByPerfectCandidateIsDisabled : 1;
-
-    // Allocator for ConversionSequenceLists and deferred candidate args.
-    // We store the first few of these
+    // Allocator for ConversionSequenceLists. We store the first few of these
     // inline to avoid allocation for small sets.
     llvm::BumpPtrAllocator SlabAllocator;
 
@@ -1241,11 +1125,8 @@ class Sema;
     CandidateSetKind Kind;
     OperatorRewriteInfo RewriteInfo;
 
-    /// Small storage size for ImplicitConversionSequences
-    /// and the persisted arguments of deferred candidates.
     constexpr static unsigned NumInlineBytes =
-        32 * sizeof(ImplicitConversionSequence);
-
+        24 * sizeof(ImplicitConversionSequence);
     unsigned NumInlineBytesUsed = 0;
     alignas(void *) char InlineSpace[NumInlineBytes];
 
@@ -1256,13 +1137,15 @@ class Sema;
     /// from the slab allocator.
     /// FIXME: It would probably be nice to have a SmallBumpPtrAllocator
     /// instead.
+    /// FIXME: Now that this only allocates ImplicitConversionSequences, do we
+    /// want to un-generalize this?
     template <typename T>
     T *slabAllocate(unsigned N) {
       // It's simpler if this doesn't need to consider alignment.
       static_assert(alignof(T) == alignof(void *),
                     "Only works for pointer-aligned types.");
-      static_assert(std::is_trivially_destructible_v<T> ||
-                        (std::is_same_v<ImplicitConversionSequence, T>),
+      static_assert(std::is_trivial<T>::value ||
+                        std::is_same<ImplicitConversionSequence, T>::value,
                     "Add destruction logic to OverloadCandidateSet::clear().");
 
       unsigned NBytes = sizeof(T) * N;
@@ -1276,34 +1159,12 @@ class Sema;
       return reinterpret_cast<T *>(FreeSpaceStart);
     }
 
-    // Because the size of OverloadCandidateSet has a noticeable impact on
-    // performance, we store each deferred template candidate in the slab
-    // allocator such that deferred candidates are ultimately a singly-linked
-    // intrusive linked list. This ends up being much more efficient than a
-    // SmallVector that is empty in the common case.
-    template <typename T> T *allocateDeferredCandidate() {
-      T *C = slabAllocate<T>(1);
-      if (!FirstDeferredCandidate)
-        FirstDeferredCandidate = C;
-      else {
-        auto *F = FirstDeferredCandidate;
-        while (F->Next)
-          F = F->Next;
-        F->Next = C;
-      }
-      DeferredCandidatesCount++;
-      return C;
-    }
-
     void destroyCandidates();
 
   public:
     OverloadCandidateSet(SourceLocation Loc, CandidateSetKind CSK,
                          OperatorRewriteInfo RewriteInfo = {})
-        : FirstDeferredCandidate(nullptr), DeferredCandidatesCount(0),
-          HasDeferredTemplateConstructors(false),
-          ResolutionByPerfectCandidateIsDisabled(false), Loc(Loc), Kind(CSK),
-          RewriteInfo(RewriteInfo) {}
+        : Loc(Loc), Kind(CSK), RewriteInfo(RewriteInfo) {}
     OverloadCandidateSet(const OverloadCandidateSet &) = delete;
     OverloadCandidateSet &operator=(const OverloadCandidateSet &) = delete;
     ~OverloadCandidateSet() { destroyCandidates(); }
@@ -1315,9 +1176,6 @@ class Sema;
     /// Whether diagnostics should be deferred.
     bool shouldDeferDiags(Sema &S, ArrayRef<Expr *> Args, SourceLocation OpLoc);
 
-    // Whether the resolution of template candidates should be deferred
-    bool shouldDeferTemplateArgumentDeduction(const LangOptions &Opts) const;
-
     /// Determine when this overload candidate will be new to the
     /// overload set.
     bool isNewCandidate(Decl *F, OverloadCandidateParamOrder PO =
@@ -1341,10 +1199,8 @@ class Sema;
     iterator begin() { return Candidates.begin(); }
     iterator end() { return Candidates.end(); }
 
-    size_t size() const { return Candidates.size() + DeferredCandidatesCount; }
-    bool empty() const {
-      return Candidates.empty() && DeferredCandidatesCount == 0;
-    }
+    size_t size() const { return Candidates.size(); }
+    bool empty() const { return Candidates.empty(); }
 
     /// Allocate storage for conversion sequences for NumConversions
     /// conversions.
@@ -1360,24 +1216,6 @@ class Sema;
       return ConversionSequenceList(Conversions, NumConversions);
     }
 
-    /// Provide storage for any Expr* arg that must be preserved
-    /// until deferred template candidates are deduced.
-    /// Typically this should be used for reversed operator arguments
-    /// and any time the argument array is transformed while adding
-    /// a template candidate.
-    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(unsigned N) {
-      Expr **Exprs = slabAllocate<Expr *>(N);
-      return llvm::MutableArrayRef<Expr *>(Exprs, N);
-    }
-
-    template <typename... T>
-    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(T *...Exprs) {
-      llvm::MutableArrayRef<Expr *> Arr =
-          getPersistentArgsArray(sizeof...(Exprs));
-      llvm::copy(std::initializer_list<Expr *>{Exprs...}, Arr.data());
-      return Arr;
-    }
-
     /// Add a new candidate with NumConversions conversion sequence slots
     /// to the overload set.
     OverloadCandidate &addCandidate(unsigned NumConversions = 0,
@@ -1393,32 +1231,6 @@ class Sema;
       return C;
     }
 
-    void AddDeferredTemplateCandidate(
-        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-        ArrayRef<Expr *> Args, bool SuppressUserConversions,
-        bool PartialOverloading, bool AllowExplicit,
-        CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
-        bool AggregateCandidateDeduction);
-
-    void AddDeferredMethodTemplateCandidate(
-        FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
-        CXXRecordDecl *ActingContext, QualType ObjectType,
-        Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-        bool SuppressUserConversions, bool PartialOverloading,
-        OverloadCandidateParamOrder PO);
-
-    void AddDeferredConversionTemplateCandidate(
-        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-        CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
-        bool AllowObjCConversionOnExplicit, bool AllowExplicit,
-        bool AllowResultConversion);
-
-    void InjectNonDeducedTemplateCandidates(Sema &S);
-
-    void DisableResolutionByPerfectCandidate() {
-      ResolutionByPerfectCandidateIsDisabled = true;
-    }
-
     /// Find the best viable function on this overload set, if it exists.
     OverloadingResult BestViableFunction(Sema &S, SourceLocation Loc,
                                          OverloadCandidateSet::iterator& Best);
@@ -1451,15 +1263,6 @@ class Sema;
       DestAS = AS;
     }
 
-  private:
-    OverloadingResult ResultForBestCandidate(const iterator &Best);
-    void CudaExcludeWrongSideCandidates(
-        Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates);
-    OverloadingResult
-    BestViableFunctionImpl(Sema &S, SourceLocation Loc,
-                           OverloadCandidateSet::iterator &Best);
-    void PerfectViableFunction(Sema &S, SourceLocation Loc,
-                               OverloadCandidateSet::iterator &Best);
   };
 
   bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
@@ -1508,21 +1311,6 @@ class Sema;
   // parameter.
   bool shouldEnforceArgLimit(bool PartialOverloading, FunctionDecl *Function);
 
-  inline bool OverloadCandidateSet::shouldDeferTemplateArgumentDeduction(
-      const LangOptions &Opts) const {
-    return
-        // For user defined conversion we need to check against different
-        // combination of CV qualifiers and look at any explicit specifier, so
-        // always deduce template candidates.
-        Kind != CSK_InitByUserDefinedConversion
-        // When doing code completion, we want to see all the
-        // viable candidates.
-        && Kind != CSK_CodeCompletion
-        // CUDA may prefer template candidates even when a non-candidate
-        // is a perfect match
-        && !Opts.CUDA;
-  }
-
 } // namespace clang
 
 #endif // LLVM_CLANG_SEMA_OVERLOAD_H
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 45405d4709e14..f6ec4cb0f069e 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -6354,8 +6354,7 @@ SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
   Expr *NakedFn = Fn->IgnoreParenCasts();
   // Build an overload candidate set based on the functions we find.
   SourceLocation Loc = Fn->getExprLoc();
-  OverloadCandidateSet CandidateSet(Loc,
-                                    OverloadCandidateSet::CSK_CodeCompletion);
+  OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
 
   if (auto ULE = dyn_cast<UnresolvedLookupExpr>(NakedFn)) {
     SemaRef.AddOverloadedCallCandidates(ULE, ArgsWithoutDependentTypes,
@@ -6558,8 +6557,7 @@ QualType SemaCodeCompletion::ProduceConstructorSignatureHelp(
   // FIXME: Provide support for variadic template constructors.
 
   if (CRD) {
-    OverloadCandidateSet CandidateSet(Loc,
-                                      OverloadCandidateSet::CSK_CodeCompletion);
+    OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
     for (NamedDecl *C : SemaRef.LookupConstructors(CRD)) {
       if (auto *FD = dyn_cast<FunctionDecl>(C)) {
         // FIXME: we can't yet provide correct signature help for initializer
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 9e802a175bb05..77d7f821f2011 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -5231,7 +5231,7 @@ static OverloadingResult TryRefInitWithConversionFunction(
 
   // Add the final conversion sequence, if necessary.
   if (NewRefRelationship == Sema::Ref_Incompatible) {
-    assert(Best->HasFinalConversion && !isa<CXXConstructorDecl>(Function) &&
+    assert(!isa<CXXConstructorDecl>(Function) &&
            "should not have conversion after constructor");
 
     ImplicitConversionSequence ICS;
@@ -6200,7 +6200,6 @@ static void TryUserDefinedConversion(Sema &S,
 
   // If the conversion following the call to the conversion function
   // is interesting, add it as a separate step.
-  assert(Best->HasFinalConversion);
   if (Best->FinalConversion.First || Best->FinalConversion.Second ||
       Best->FinalConversion.Third) {
     ImplicitConversionSequence ICS;
@@ -10030,19 +10029,12 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     //   When [...] the constructor [...] is a candidate by
     //    - [over.match.copy] (in all cases)
     if (TD) {
-
-      // As template candidates are not deduced immediately,
-      // persist the array in the overload set.
-      MutableArrayRef<Expr *> TmpInits =
-          Candidates.getPersistentArgsArray(Inits.size());
-
-      for (auto [I, E] : llvm::enumerate(Inits)) {
+      SmallVector<Expr *, 8> TmpInits;
+      for (Expr *E : Inits)
         if (auto *DI = dyn_cast<DesignatedInitExpr>(E))
-          TmpInits[I] = DI->getInit();
+          TmpInits.push_back(DI->getInit());
         else
-          TmpInits[I] = E;
-      }
-
+          TmpInits.push_back(E);
       AddTemplateOverloadCandidate(
           TD, FoundDecl, /*ExplicitArgs=*/nullptr, TmpInits, Candidates,
           /*SuppressUserConversions=*/false,
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index b7a981e08ead9..55634aa75ae25 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1123,10 +1123,6 @@ void OverloadCandidateSet::clear(CandidateSetKind CSK) {
   Candidates.clear();
   Functions.clear();
   Kind = CSK;
-  FirstDeferredCandidate = nullptr;
-  DeferredCandidatesCount = 0;
-  HasDeferredTemplateConstructors = false;
-  ResolutionByPerfectCandidateIsDisabled = false;
 }
 
 namespace {
@@ -4079,9 +4075,6 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
     }
     if (CXXConversionDecl *Conversion
                  = dyn_cast<CXXConversionDecl>(Best->Function)) {
-
-      assert(Best->HasFinalConversion);
-
       // C++ [over.ics.user]p1:
       //
       //   [...] If the user-defined conversion is specified by a
@@ -5165,9 +5158,6 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
   case OR_Success:
-
-    assert(Best->HasFinalConversion);
-
     // C++ [over.ics.ref]p1:
     //
     //   [...] If the parameter binds directly to the result of
@@ -7805,14 +7795,15 @@ void Sema::AddMethodCandidate(
   }
 }
 
-static void AddMethodTemplateCandidateImmediately(
-    Sema &S, OverloadCandidateSet &CandidateSet,
+void Sema::AddMethodTemplateCandidate(
     FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
     CXXRecordDecl *ActingContext,
     TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
     Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    bool SuppressUserConversions, bool PartialOverloading,
-    OverloadCandidateParamOrder PO) {
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, OverloadCandidateParamOrder PO) {
+  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
+    return;
 
   // C++ [over.match.funcs]p7:
   //   In each case where a candidate is a function template, candidate
@@ -7826,12 +7817,12 @@ static void AddMethodTemplateCandidateImmediately(
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = DeduceTemplateArguments(
           MethodTmpl, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, /*AggregateDeductionCandidate=*/false,
           /*PartialOrdering=*/false, ObjectType, ObjectClassification,
           [&](ArrayRef<QualType> ParamTypes) {
-            return S.CheckNonDependentConversions(
+            return CheckNonDependentConversions(
                 MethodTmpl, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, ActingContext, ObjectType,
                 ObjectClassification, PO);
@@ -7853,8 +7844,8 @@ static void AddMethodTemplateCandidateImmediately(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure =
-          MakeDeductionFailureInfo(S.Context, Result, Info);
+      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
+                                                            Info);
     }
     return;
   }
@@ -7864,34 +7855,10 @@ static void AddMethodTemplateCandidateImmediately(
   assert(Specialization && "Missing member function template specialization?");
   assert(isa<CXXMethodDecl>(Specialization) &&
          "Specialization is not a member function?");
-  S.AddMethodCandidate(
-      cast<CXXMethodDecl>(Specialization), FoundDecl, ActingContext, ObjectType,
-      ObjectClassification, Args, CandidateSet, SuppressUserConversions,
-      PartialOverloading, Conversions, PO, Info.hasStrictPackMatch());
-}
-
-void Sema::AddMethodTemplateCandidate(
-    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext,
-    TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
-    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, OverloadCandidateParamOrder PO) {
-  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
-    return;
-
-  if (ExplicitTemplateArgs ||
-      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts())) {
-    AddMethodTemplateCandidateImmediately(
-        *this, CandidateSet, MethodTmpl, FoundDecl, ActingContext,
-        ExplicitTemplateArgs, ObjectType, ObjectClassification, Args,
-        SuppressUserConversions, PartialOverloading, PO);
-    return;
-  }
-
-  CandidateSet.AddDeferredMethodTemplateCandidate(
-      MethodTmpl, FoundDecl, ActingContext, ObjectType, ObjectClassification,
-      Args, SuppressUserConversions, PartialOverloading, PO);
+  AddMethodCandidate(cast<CXXMethodDecl>(Specialization), FoundDecl,
+                     ActingContext, ObjectType, ObjectClassification, Args,
+                     CandidateSet, SuppressUserConversions, PartialOverloading,
+                     Conversions, PO, Info.hasStrictPackMatch());
 }
 
 /// Determine whether a given function template has a simple explicit specifier
@@ -7900,18 +7867,14 @@ static bool isNonDependentlyExplicit(FunctionTemplateDecl *FTD) {
   return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).isExplicit();
 }
 
-static bool hasDependentExplicit(FunctionTemplateDecl *FTD) {
-  return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).getKind() ==
-         ExplicitSpecKind::Unresolved;
-}
-
-static void AddTemplateOverloadCandidateImmediately(
-    Sema &S, OverloadCandidateSet &CandidateSet,
+void Sema::AddTemplateOverloadCandidate(
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
     TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
-    bool SuppressUserConversions, bool PartialOverloading, bool AllowExplicit,
-    Sema::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
-    bool AggregateCandidateDeduction) {
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
+    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
+  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
+    return;
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -7938,14 +7901,14 @@ static void AddTemplateOverloadCandidateImmediately(
                              FunctionTemplate->getTemplateDepth());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = DeduceTemplateArguments(
           FunctionTemplate, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, AggregateCandidateDeduction,
           /*PartialOrdering=*/false,
           /*ObjectType=*/QualType(),
           /*ObjectClassification=*/Expr::Classification(),
           [&](ArrayRef<QualType> ParamTypes) {
-            return S.CheckNonDependentConversions(
+            return CheckNonDependentConversions(
                 FunctionTemplate, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, nullptr, QualType(), {}, PO);
           });
@@ -7969,8 +7932,8 @@ static void AddTemplateOverloadCandidateImmediately(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure =
-          MakeDeductionFailureInfo(S.Context, Result, Info);
+      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
+                                                            Info);
     }
     return;
   }
@@ -7978,7 +7941,7 @@ static void AddTemplateOverloadCandidateImmediately(
   // Add the function template specialization produced by template argument
   // deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  S.AddOverloadCandidate(
+  AddOverloadCandidate(
       Specialization, FoundDecl, Args, CandidateSet, SuppressUserConversions,
       PartialOverloading, AllowExplicit,
       /*AllowExplicitConversions=*/false, IsADLCandidate, Conversions, PO,
@@ -7986,38 +7949,6 @@ static void AddTemplateOverloadCandidateImmediately(
       Info.hasStrictPackMatch());
 }
 
-void Sema::AddTemplateOverloadCandidate(
-    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
-    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
-  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
-    return;
-
-  bool DependentExplicitSpecifier = hasDependentExplicit(FunctionTemplate);
-
-  if (ExplicitTemplateArgs ||
-      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
-      (isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl()) &&
-       DependentExplicitSpecifier)) {
-
-    AddTemplateOverloadCandidateImmediately(
-        *this, CandidateSet, FunctionTemplate, FoundDecl, ExplicitTemplateArgs,
-        Args, SuppressUserConversions, PartialOverloading, AllowExplicit,
-        IsADLCandidate, PO, AggregateCandidateDeduction);
-
-    if (DependentExplicitSpecifier)
-      CandidateSet.DisableResolutionByPerfectCandidate();
-    return;
-  }
-
-  CandidateSet.AddDeferredTemplateCandidate(
-      FunctionTemplate, FoundDecl, Args, SuppressUserConversions,
-      PartialOverloading, AllowExplicit, IsADLCandidate, PO,
-      AggregateCandidateDeduction);
-}
-
 bool Sema::CheckNonDependentConversions(
     FunctionTemplateDecl *FunctionTemplate, ArrayRef<QualType> ParamTypes,
     ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet,
@@ -8180,7 +8111,6 @@ void Sema::AddConversionCandidate(
   Candidate.FinalConversion.setAsIdentityConversion();
   Candidate.FinalConversion.setFromType(ConvType);
   Candidate.FinalConversion.setAllToTypes(ToType);
-  Candidate.HasFinalConversion = true;
   Candidate.Viable = true;
   Candidate.ExplicitCallArguments = 1;
   Candidate.StrictPackMatch = StrictPackMatch;
@@ -8285,7 +8215,6 @@ void Sema::AddConversionCandidate(
   switch (ICS.getKind()) {
   case ImplicitConversionSequence::StandardConversion:
     Candidate.FinalConversion = ICS.Standard;
-    Candidate.HasFinalConversion = true;
 
     // C++ [over.ics.user]p3:
     //   If the user-defined conversion is specified by a specialization of a
@@ -8335,12 +8264,16 @@ void Sema::AddConversionCandidate(
   }
 }
 
-static void AddTemplateConversionCandidateImmediately(
-    Sema &S, OverloadCandidateSet &CandidateSet,
+void Sema::AddTemplateConversionCandidate(
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
-    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
-    bool AllowResultConversion) {
+    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
+    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
+    bool AllowExplicit, bool AllowResultConversion) {
+  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
+         "Only conversion function templates permitted here");
+
+  if (!CandidateSet.isNewCandidate(FunctionTemplate))
+    return;
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -8355,11 +8288,11 @@ static void AddTemplateConversionCandidateImmediately(
   }
 
   QualType ObjectType = From->getType();
-  Expr::Classification ObjectClassification = From->Classify(S.Context);
+  Expr::Classification ObjectClassification = From->Classify(getASTContext());
 
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   CXXConversionDecl *Specialization = nullptr;
-  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = DeduceTemplateArguments(
           FunctionTemplate, ObjectType, ObjectClassification, ToType,
           Specialization, Info);
       Result != TemplateDeductionResult::Success) {
@@ -8369,47 +8302,18 @@ static void AddTemplateConversionCandidateImmediately(
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_deduction;
     Candidate.ExplicitCallArguments = 1;
-    Candidate.DeductionFailure =
-        MakeDeductionFailureInfo(S.Context, Result, Info);
+    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
+                                                          Info);
     return;
   }
 
   // Add the conversion function template specialization produced by
   // template argument deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  S.AddConversionCandidate(Specialization, FoundDecl, ActingContext, From,
-                           ToType, CandidateSet, AllowObjCConversionOnExplicit,
-                           AllowExplicit, AllowResultConversion,
-                           Info.hasStrictPackMatch());
-}
-
-void Sema::AddTemplateConversionCandidate(
-    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
-    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
-    bool AllowExplicit, bool AllowResultConversion) {
-  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
-         "Only conversion function templates permitted here");
-
-  if (!CandidateSet.isNewCandidate(FunctionTemplate))
-    return;
-
-  if (!CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
-      CandidateSet.getKind() ==
-          OverloadCandidateSet::CSK_InitByUserDefinedConversion ||
-      CandidateSet.getKind() == OverloadCandidateSet::CSK_InitByConstructor) {
-    AddTemplateConversionCandidateImmediately(
-        *this, CandidateSet, FunctionTemplate, FoundDecl, ActingDC, From,
-        ToType, AllowObjCConversionOnExplicit, AllowExplicit,
-        AllowResultConversion);
-
-    CandidateSet.DisableResolutionByPerfectCandidate();
-    return;
-  }
-
-  CandidateSet.AddDeferredConversionTemplateCandidate(
-      FunctionTemplate, FoundDecl, ActingDC, From, ToType,
-      AllowObjCConversionOnExplicit, AllowExplicit, AllowResultConversion);
+  AddConversionCandidate(Specialization, FoundDecl, ActingDC, From, ToType,
+                         CandidateSet, AllowObjCConversionOnExplicit,
+                         AllowExplicit, AllowResultConversion,
+                         Info.hasStrictPackMatch());
 }
 
 void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
@@ -8559,17 +8463,11 @@ void Sema::AddNonMemberOperatorCandidates(
     if (FunTmpl) {
       AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
                                    FunctionArgs, CandidateSet);
-      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD)) {
-
-        // As template candidates are not deduced immediately,
-        // persist the array in the overload set.
-        ArrayRef<Expr *> Reversed = CandidateSet.getPersistentArgsArray(
-            FunctionArgs[1], FunctionArgs[0]);
-        AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
-                                     Reversed, CandidateSet, false, false, true,
-                                     ADLCallKind::NotADL,
-                                     OverloadCandidateParamOrder::Reversed);
-      }
+      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD))
+        AddTemplateOverloadCandidate(
+            FunTmpl, F.getPair(), ExplicitTemplateArgs,
+            {FunctionArgs[1], FunctionArgs[0]}, CandidateSet, false, false,
+            true, ADLCallKind::NotADL, OverloadCandidateParamOrder::Reversed);
     } else {
       if (ExplicitTemplateArgs)
         continue;
@@ -10301,8 +10199,6 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
   // FIXME: Pass in the explicit template arguments?
   ArgumentDependentLookup(Name, Loc, Args, Fns);
 
-  ArrayRef<Expr *> ReversedArgs;
-
   // Erase all of the candidates we already knew about.
   for (OverloadCandidateSet::iterator Cand = CandidateSet.begin(),
                                    CandEnd = CandidateSet.end();
@@ -10342,15 +10238,9 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
           /*AllowExplicit=*/true, ADLCallKind::UsesADL);
       if (CandidateSet.getRewriteInfo().shouldAddReversed(
               *this, Args, FTD->getTemplatedDecl())) {
-
-        // As template candidates are not deduced immediately,
-        // persist the array in the overload set.
-        if (ReversedArgs.empty())
-          ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
-
         AddTemplateOverloadCandidate(
-            FTD, FoundDecl, ExplicitTemplateArgs, ReversedArgs, CandidateSet,
-            /*SuppressUserConversions=*/false, PartialOverloading,
+            FTD, FoundDecl, ExplicitTemplateArgs, {Args[1], Args[0]},
+            CandidateSet, /*SuppressUserConversions=*/false, PartialOverloading,
             /*AllowExplicit=*/true, ADLCallKind::UsesADL,
             OverloadCandidateParamOrder::Reversed);
       }
@@ -10764,8 +10654,6 @@ bool clang::isBetterOverloadCandidate(
       Cand1.Function && Cand2.Function &&
       isa<CXXConversionDecl>(Cand1.Function) &&
       isa<CXXConversionDecl>(Cand2.Function)) {
-
-    assert(Cand1.HasFinalConversion && Cand2.HasFinalConversion);
     // First check whether we prefer one of the conversion functions over the
     // other. This only distinguishes the results in non-standard, extension
     // cases such as the conversion from a lambda closure type to a function
@@ -11025,147 +10913,23 @@ bool OverloadCandidate::NotValidBecauseConstraintExprHasError() const {
              ->Satisfaction.ContainsErrors;
 }
 
-void OverloadCandidateSet::AddDeferredTemplateCandidate(
-    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    ArrayRef<Expr *> Args, bool SuppressUserConversions,
-    bool PartialOverloading, bool AllowExplicit,
-    CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
-    bool AggregateCandidateDeduction) {
-
-  auto *C =
-      allocateDeferredCandidate<DeferredFunctionTemplateOverloadCandidate>();
-
-  C = new (C) DeferredFunctionTemplateOverloadCandidate{
-      {nullptr, DeferredFunctionTemplateOverloadCandidate::Function,
-       /*AllowObjCConversionOnExplicit=*/false,
-       /*AllowResultConversion=*/false, AllowExplicit, SuppressUserConversions,
-       PartialOverloading, AggregateCandidateDeduction},
-      FunctionTemplate,
-      FoundDecl,
-      Args,
-      IsADLCandidate,
-      PO};
-
-  HasDeferredTemplateConstructors |=
-      isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl());
-}
-
-void OverloadCandidateSet::AddDeferredMethodTemplateCandidate(
-    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext, QualType ObjectType,
-    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    bool SuppressUserConversions, bool PartialOverloading,
-    OverloadCandidateParamOrder PO) {
-
-  assert(!isa<CXXConstructorDecl>(MethodTmpl->getTemplatedDecl()));
-
-  auto *C =
-      allocateDeferredCandidate<DeferredMethodTemplateOverloadCandidate>();
-
-  C = new (C) DeferredMethodTemplateOverloadCandidate{
-      {nullptr, DeferredFunctionTemplateOverloadCandidate::Method,
-       /*AllowObjCConversionOnExplicit=*/false,
-       /*AllowResultConversion=*/false,
-       /*AllowExplicit=*/false, SuppressUserConversions, PartialOverloading,
-       /*AggregateCandidateDeduction=*/false},
-      MethodTmpl,
-      FoundDecl,
-      Args,
-      ActingContext,
-      ObjectClassification,
-      ObjectType,
-      PO};
-}
-
-void OverloadCandidateSet::AddDeferredConversionTemplateCandidate(
-    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
-    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
-    bool AllowResultConversion) {
-
-  auto *C =
-      allocateDeferredCandidate<DeferredConversionTemplateOverloadCandidate>();
-
-  C = new (C) DeferredConversionTemplateOverloadCandidate{
-      {nullptr, DeferredFunctionTemplateOverloadCandidate::Conversion,
-       AllowObjCConversionOnExplicit, AllowResultConversion,
-       /*AllowExplicit=*/false,
-       /*SuppressUserConversions=*/false,
-       /*PartialOverloading*/ false,
-       /*AggregateCandidateDeduction=*/false},
-      FunctionTemplate,
-      FoundDecl,
-      ActingContext,
-      From,
-      ToType};
-}
-
-static void
-AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
-                             DeferredMethodTemplateOverloadCandidate &C) {
-
-  AddMethodTemplateCandidateImmediately(
-      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext,
-      /*ExplicitTemplateArgs=*/nullptr, C.ObjectType, C.ObjectClassification,
-      C.Args, C.SuppressUserConversions, C.PartialOverloading, C.PO);
-}
-
-static void
-AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
-                             DeferredFunctionTemplateOverloadCandidate &C) {
-  AddTemplateOverloadCandidateImmediately(
-      S, CandidateSet, C.FunctionTemplate, C.FoundDecl,
-      /*ExplicitTemplateArgs=*/nullptr, C.Args, C.SuppressUserConversions,
-      C.PartialOverloading, C.AllowExplicit, C.IsADLCandidate, C.PO,
-      C.AggregateCandidateDeduction);
-}
-
-static void
-AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
-                             DeferredConversionTemplateOverloadCandidate &C) {
-  return AddTemplateConversionCandidateImmediately(
-      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext, C.From,
-      C.ToType, C.AllowObjCConversionOnExplicit, C.AllowExplicit,
-      C.AllowResultConversion);
-}
-
-void OverloadCandidateSet::InjectNonDeducedTemplateCandidates(Sema &S) {
-  Candidates.reserve(Candidates.size() + DeferredCandidatesCount);
-  DeferredTemplateOverloadCandidate *Cand = FirstDeferredCandidate;
-  while (Cand) {
-    switch (Cand->Kind) {
-    case DeferredTemplateOverloadCandidate::Function:
-      AddTemplateOverloadCandidate(
-          S, *this,
-          *static_cast<DeferredFunctionTemplateOverloadCandidate *>(Cand));
-      break;
-    case DeferredTemplateOverloadCandidate::Method:
-      AddTemplateOverloadCandidate(
-          S, *this,
-          *static_cast<DeferredMethodTemplateOverloadCandidate *>(Cand));
-      break;
-    case DeferredTemplateOverloadCandidate::Conversion:
-      AddTemplateOverloadCandidate(
-          S, *this,
-          *static_cast<DeferredConversionTemplateOverloadCandidate *>(Cand));
-      break;
-    }
-    Cand = Cand->Next;
-  }
-  FirstDeferredCandidate = nullptr;
-  DeferredCandidatesCount = 0;
-}
-
+/// Computes the best viable function (C++ 13.3.3)
+/// within an overload candidate set.
+///
+/// \param Loc The location of the function name (or operator symbol) for
+/// which overload resolution occurs.
+///
+/// \param Best If overload resolution was successful or found a deleted
+/// function, \p Best points to the candidate function found.
+///
+/// \returns The result of overload resolution.
 OverloadingResult
-OverloadCandidateSet::ResultForBestCandidate(const iterator &Best) {
-  Best->Best = true;
-  if (Best->Function && Best->Function->isDeleted())
-    return OR_Deleted;
-  return OR_Success;
-}
+OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
+                                         iterator &Best) {
+  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
+  std::transform(begin(), end(), std::back_inserter(Candidates),
+                 [](OverloadCandidate &Cand) { return &Cand; });
 
-void OverloadCandidateSet::CudaExcludeWrongSideCandidates(
-    Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates) {
   // [CUDA] HD->H or HD->D calls are technically not allowed by CUDA but
   // are accepted by both clang and NVCC. However, during a particular
   // compilation mode only one call variant is viable. We need to
@@ -11177,112 +10941,27 @@ void OverloadCandidateSet::CudaExcludeWrongSideCandidates(
   // -fgpu-exclude-wrong-side-overloads is off. When
   // -fgpu-exclude-wrong-side-overloads is on, all candidates are compared
   // uniformly in isBetterOverloadCandidate.
-  if (!S.getLangOpts().CUDA || S.getLangOpts().GPUExcludeWrongSideOverloads)
-    return;
-  const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
-
-  bool ContainsSameSideCandidate =
-      llvm::any_of(Candidates, [&](const OverloadCandidate *Cand) {
-        // Check viable function only.
+  if (S.getLangOpts().CUDA && !S.getLangOpts().GPUExcludeWrongSideOverloads) {
+    const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
+    bool ContainsSameSideCandidate =
+        llvm::any_of(Candidates, [&](OverloadCandidate *Cand) {
+          // Check viable function only.
+          return Cand->Viable && Cand->Function &&
+                 S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
+                     SemaCUDA::CFP_SameSide;
+        });
+    if (ContainsSameSideCandidate) {
+      auto IsWrongSideCandidate = [&](OverloadCandidate *Cand) {
+        // Check viable function only to avoid unnecessary data copying/moving.
         return Cand->Viable && Cand->Function &&
                S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-                   SemaCUDA::CFP_SameSide;
-      });
-
-  if (!ContainsSameSideCandidate)
-    return;
-
-  auto IsWrongSideCandidate = [&](const OverloadCandidate *Cand) {
-    // Check viable function only to avoid unnecessary data copying/moving.
-    return Cand->Viable && Cand->Function &&
-           S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-               SemaCUDA::CFP_WrongSide;
-  };
-  llvm::erase_if(Candidates, IsWrongSideCandidate);
-}
-
-/// Computes the best viable function (C++ 13.3.3)
-/// within an overload candidate set.
-///
-/// \param Loc The location of the function name (or operator symbol) for
-/// which overload resolution occurs.
-///
-/// \param Best If overload resolution was successful or found a deleted
-/// function, \p Best points to the candidate function found.
-///
-/// \returns The result of overload resolution.
-OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
-                                                           SourceLocation Loc,
-                                                           iterator &Best) {
-
-  assert(shouldDeferTemplateArgumentDeduction(S.getLangOpts()) ||
-         DeferredCandidatesCount == 0 &&
-             "Unexpected deferred template candidates");
-
-  bool TwoPhaseResolution =
-      DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
-
-  if (TwoPhaseResolution) {
-
-    PerfectViableFunction(S, Loc, Best);
-    if (Best != end())
-      return ResultForBestCandidate(Best);
-  }
-
-  InjectNonDeducedTemplateCandidates(S);
-  return BestViableFunctionImpl(S, Loc, Best);
-}
-
-void OverloadCandidateSet::PerfectViableFunction(
-    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
-
-  Best = end();
-  for (auto It = begin(); It != end(); ++It) {
-
-    if (!It->isPerfectMatch(S.getASTContext()))
-      continue;
-
-    // We found a suitable conversion function
-    // but if there is a template constructor in the target class
-    // we might prefer that instead.
-    if (HasDeferredTemplateConstructors &&
-        isa_and_nonnull<CXXConversionDecl>(It->Function)) {
-      Best = end();
-      break;
-    }
-
-    if (Best == end()) {
-      Best = It;
-      continue;
-    }
-    if (Best->Function && It->Function) {
-      FunctionDecl *D =
-          S.getMoreConstrainedFunction(Best->Function, It->Function);
-      if (D == nullptr) {
-        Best = end();
-        break;
-      }
-      if (D == It->Function)
-        Best = It;
-      continue;
+                   SemaCUDA::CFP_WrongSide;
+      };
+      llvm::erase_if(Candidates, IsWrongSideCandidate);
     }
-    // ambiguous
-    Best = end();
-    break;
   }
-}
-
-OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
-    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
-
-  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
-  Candidates.reserve(this->Candidates.size());
-  std::transform(begin(), end(), std::back_inserter(Candidates),
-                 [](OverloadCandidate &Cand) { return &Cand; });
-
-  if (S.getLangOpts().CUDA)
-    CudaExcludeWrongSideCandidates(S, Candidates);
 
+  // Find the best viable function.
   Best = end();
   for (auto *Cand : Candidates) {
     Cand->Best = false;
@@ -11304,8 +10983,9 @@ OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
   if (Best == end())
     return OR_No_Viable_Function;
 
-  llvm::SmallVector<OverloadCandidate *, 4> PendingBest;
   llvm::SmallVector<const NamedDecl *, 4> EquivalentCands;
+
+  llvm::SmallVector<OverloadCandidate*, 4> PendingBest;
   PendingBest.push_back(&*Best);
   Best->Best = true;
 
@@ -11328,15 +11008,25 @@ OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
     }
   }
 
+  // If we found more than one best candidate, this is ambiguous.
   if (Best == end())
     return OR_Ambiguous;
 
-  OverloadingResult R = ResultForBestCandidate(Best);
+  // Best is the best viable function.
+  if (Best->Function && Best->Function->isDeleted())
+    return OR_Deleted;
+
+  if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
+      Kind == CSK_AddressOfOverloadSet && M &&
+      M->isImplicitObjectMemberFunction()) {
+    return OR_No_Viable_Function;
+  }
 
   if (!EquivalentCands.empty())
     S.diagnoseEquivalentInternalLinkageDeclarations(Loc, Best->Function,
                                                     EquivalentCands);
-  return R;
+
+  return OR_Success;
 }
 
 namespace {
@@ -13043,9 +12733,6 @@ SmallVector<OverloadCandidate *, 32> OverloadCandidateSet::CompleteCandidates(
     Sema &S, OverloadCandidateDisplayKind OCD, ArrayRef<Expr *> Args,
     SourceLocation OpLoc,
     llvm::function_ref<bool(OverloadCandidate &)> Filter) {
-
-  InjectNonDeducedTemplateCandidates(S);
-
   // Sort the candidates by viability and position.  Sorting directly would
   // be prohibitive, so we make a set of pointers and sort those.
   SmallVector<OverloadCandidate*, 32> Cands;
@@ -14667,12 +14354,10 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
                                          Expr *ExecConfig,
                                          bool AllowTypoCorrection,
                                          bool CalleesAddressIsTaken) {
-
-  OverloadCandidateSet::CandidateSetKind CSK =
-      CalleesAddressIsTaken ? OverloadCandidateSet::CSK_AddressOfOverloadSet
-                            : OverloadCandidateSet::CSK_Normal;
-
-  OverloadCandidateSet CandidateSet(Fn->getExprLoc(), CSK);
+  OverloadCandidateSet CandidateSet(
+      Fn->getExprLoc(), CalleesAddressIsTaken
+                            ? OverloadCandidateSet::CSK_AddressOfOverloadSet
+                            : OverloadCandidateSet::CSK_Normal);
   ExprResult result;
 
   if (buildOverloadedCallSet(S, Fn, ULE, Args, LParenLoc, &CandidateSet,
@@ -14688,17 +14373,6 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
   OverloadingResult OverloadResult =
       CandidateSet.BestViableFunction(*this, Fn->getBeginLoc(), Best);
 
-  // [C++23][over.call.func]
-  // if overload resolution selects a non-static member function,
-  // the call is ill-formed;
-  if (CSK == OverloadCandidateSet::CSK_AddressOfOverloadSet &&
-      Best != CandidateSet.end()) {
-    if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
-        M && M->isImplicitObjectMemberFunction()) {
-      OverloadResult = OR_No_Viable_Function;
-    }
-  }
-
   // Model the case with a call to a templated function whose definition
   // encloses the call and whose return type contains a placeholder type as if
   // the UnresolvedLookupExpr was type-dependent.
@@ -15034,24 +14708,18 @@ void Sema::LookupOverloadedBinOp(OverloadCandidateSet &CandidateSet,
   // rewritten candidates using these functions if necessary.
   AddNonMemberOperatorCandidates(Fns, Args, CandidateSet);
 
-  // As template candidates are not deduced immediately,
-  // persist the array in the overload set.
-  ArrayRef<Expr *> ReversedArgs;
-  if (CandidateSet.getRewriteInfo().allowsReversed(Op) ||
-      CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
-    ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
-
   // Add operator candidates that are member functions.
   AddMemberOperatorCandidates(Op, OpLoc, Args, CandidateSet);
   if (CandidateSet.getRewriteInfo().allowsReversed(Op))
-    AddMemberOperatorCandidates(Op, OpLoc, ReversedArgs, CandidateSet,
+    AddMemberOperatorCandidates(Op, OpLoc, {Args[1], Args[0]}, CandidateSet,
                                 OverloadCandidateParamOrder::Reversed);
 
   // In C++20, also add any rewritten member candidates.
   if (ExtraOp) {
     AddMemberOperatorCandidates(ExtraOp, OpLoc, Args, CandidateSet);
     if (CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
-      AddMemberOperatorCandidates(ExtraOp, OpLoc, ReversedArgs, CandidateSet,
+      AddMemberOperatorCandidates(ExtraOp, OpLoc, {Args[1], Args[0]},
+                                  CandidateSet,
                                   OverloadCandidateParamOrder::Reversed);
   }
 
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 0ecdbb3ffb89f..772962ac653f7 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -6142,9 +6142,9 @@ FunctionDecl *Sema::getMoreConstrainedFunction(FunctionDecl *FD1,
   assert(!FD1->getDescribedTemplate() && !FD2->getDescribedTemplate() &&
          "not for function templates");
   assert(!FD1->isFunctionTemplateSpecialization() ||
-         (isa<CXXConversionDecl, CXXConstructorDecl>(FD1)));
+         isa<CXXConversionDecl>(FD1));
   assert(!FD2->isFunctionTemplateSpecialization() ||
-         (isa<CXXConversionDecl, CXXConstructorDecl>(FD2)));
+         isa<CXXConversionDecl>(FD2));
 
   FunctionDecl *F1 = FD1;
   if (FunctionDecl *P = FD1->getTemplateInstantiationPattern(false))
diff --git a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
index 083e743818121..ba8e2dc372e98 100644
--- a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
+++ b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
@@ -14,7 +14,7 @@ template<typename T> struct S {
 // expected-note@#FINST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{})
 void f(T);
-void f(long);
+void f(int);
 
 // Ensure this applies to operator && as well.
 // expected-error@+3{{atomic constraint must be of type 'bool' (found 'S<int>')}}
@@ -22,7 +22,7 @@ void f(long);
 // expected-note@#F2INST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{} && true)
 void f2(T);
-void f2(long);
+void f2(int);
 
 template<typename T> requires requires {
   requires S<T>{};
@@ -36,12 +36,12 @@ template<typename T> requires requires {
   //
 }
 void f3(T);
-void f3(long);
+void f3(int);
 
 // Doesn't diagnose, since this is no longer a compound requirement.
 template<typename T> requires (bool(1 && 2))
 void f4(T);
-void f4(long);
+void f4(int);
 
 void g() {
   f(0); // #FINST
diff --git a/clang/test/SemaCUDA/function-overload.cu b/clang/test/SemaCUDA/function-overload.cu
index 3d05839af7528..4710c81763adf 100644
--- a/clang/test/SemaCUDA/function-overload.cu
+++ b/clang/test/SemaCUDA/function-overload.cu
@@ -1,3 +1,6 @@
+// REQUIRES: x86-registered-target
+// REQUIRES: nvptx-registered-target
+
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu -fsyntax-only \
 // RUN:   -verify=host,hostdefer,devdefer,expected %s
 // RUN: %clang_cc1 -std=c++14 -triple nvptx64-nvidia-cuda -fsyntax-only \
diff --git a/clang/test/SemaCXX/implicit-member-functions.cpp b/clang/test/SemaCXX/implicit-member-functions.cpp
index 8350eac5b88a0..1554b1af5d59a 100644
--- a/clang/test/SemaCXX/implicit-member-functions.cpp
+++ b/clang/test/SemaCXX/implicit-member-functions.cpp
@@ -54,24 +54,31 @@ namespace PR7594 {
 namespace Recursion {
   template<typename T> struct InvokeCopyConstructor {
     static const T &get();
-    typedef decltype(T(get())) type;
+    typedef decltype(T(get())) type; // expected-error {{no matching conver}}
   };
   struct B;
   struct A {
+    // expected-note@-1 {{while substituting deduced template arguments}}
     typedef B type;
     template<typename T,
              typename = typename InvokeCopyConstructor<typename T::type>::type>
+    // expected-note@-1 {{in instantiation of template class}}
     A(const T &);
+    // expected-note@-1 {{in instantiation of default argument}}
   };
-  struct B {
-    B();
+  struct B { // expected-note {{while declaring the implicit copy constructor for 'B'}}
+    // expected-note@-1 {{candidate constructor (the implicit move }}
+    B(); // expected-note {{candidate constructor not viable}}
     A a;
   };
   // Triggering the declaration of B's copy constructor causes overload
-  // resolution to occur for A's copying constructor, which picks
-  // the implicit copy constructor of A.
-  // Because that copy constructor is always a perfect match the template
-  // candidate is not instantiated.
+  // resolution to occur for A's copying constructor, which instantiates
+  // InvokeCopyConstructor<B>, which triggers the declaration of B's copy
+  // constructor. Notionally, this happens when we get to the end of the
+  // definition of 'struct B', so there is no declared copy constructor yet.
+  //
+  // This behavior is g++-compatible, but isn't exactly right; the class is
+  // supposed to be incomplete when we implicitly declare its special members.
   B b = B();
 
 
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
deleted file mode 100644
index 9139fd2688790..0000000000000
--- a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
+++ /dev/null
@@ -1,200 +0,0 @@
-// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++11 %s
-// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++20 %s
-// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++2c %s
-
-template <typename T>
-struct Invalid { static_assert(false, "instantiated Invalid"); }; // #err-invalid
-
-template <typename T>
-int f(T a, Invalid<T> = {}); // #note-f
-
-// sanity check
-int e1 = f(0);
-//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
-//expected-note@-2 {{in instantiation of default function argument expression for 'f<int>' required here}}
-//expected-note@#note-f {{in instantiation of template class 'Invalid<int>' requested here}}
-//expected-note@#note-f {{passing argument to parameter here}}
-
-int f(int);
-int ok1 = f(0);
-int e4 = f((const int&)(ok1));
-
-int f(int, int = 0);
-int ok2 = f(0, 0);
-
-int e2  = f(0L);
-//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
-//expected-note@-2 {{in instantiation of default function argument expression for 'f<long>' required here}}
-//expected-note@#note-f {{in instantiation of template class 'Invalid<long>' requested here}}
-//expected-note@#note-f {{passing argument to parameter here}}
-
-int f(long);
-int ok3 = f(0L);
-
-template <typename T>
-struct Invalid2 { static_assert(false, "instantiated Invalid2"); }; // #err-qualifiers
-
-template <typename T>
-int ref(T a, Invalid2<T> = {}); // expected-note 2{{here}}
-int ref(int&);
-int ref1 = ref(ok3);
-int ref2 = ref((const int&)ok3); // expected-note {{here}}
-//expected-error@#err-qualifiers {{static assertion failed: instantiated Invalid2}}
-
-
-template <typename T>
-int f_alias(T a, Invalid<T> = {});
-using Alias = int;
-int f_alias(Alias);
-int ok4 = f_alias(0);
-
-#if __cplusplus >= 202002
-
-struct Copyable {
-  template <typename T>
-  requires __is_constructible(Copyable, T)
-  explicit Copyable(T op) noexcept; // #1
-  Copyable(const Copyable&) noexcept = default; // #2
-};
-static_assert(__is_constructible(Copyable, const Copyable&));
-
-struct ImplicitlyCopyable {
-  template <typename T>
-  requires __is_constructible(ImplicitlyCopyable, T)
-  explicit ImplicitlyCopyable(T op) = delete; // #1
-};
-static_assert(__is_constructible(ImplicitlyCopyable, const ImplicitlyCopyable&));
-
-
-struct Movable {
-  template <typename T>
-  requires __is_constructible(Movable, T) // #err-self-constraint-1
-  explicit Movable(T op) noexcept; // #1
-  Movable(Movable&&) noexcept = default; // #2
-};
-static_assert(__is_constructible(Movable, Movable&&));
-static_assert(__is_constructible(Movable, const Movable&));
-// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(Movable, const Movable &)'}}
-
-static_assert(__is_constructible(Movable, int));
-// expected-error@-1{{static assertion failed due to requirement '__is_constructible(Movable, int)'}} \
-// expected-note@-1 2{{}}
-// expected-error@#err-self-constraint-1{{satisfaction of constraint '__is_constructible(Movable, T)' depends on itself}}
-// expected-note@#err-self-constraint-1 4{{}}
-
-template <typename T>
-struct Members {
-    constexpr auto f(auto) {
-        static_assert(false, "");
-    }
-    constexpr auto f(int) { return 1; }
-    constexpr auto f(int) requires true { return 2; }
-
-    constexpr auto g(auto) {
-        static_assert(false, "instantiated member"); //#err-qualified-member
-        return 0;
-    }
-    constexpr auto g(int) & { return 1; }
-
-    static constexpr auto s(auto) {
-        static_assert(false, "");
-    }
-    static constexpr auto s(int) {
-        return 1;
-    }
-    static constexpr auto s(int) requires true {
-        return 2;
-    }
-};
-
-static_assert(Members<int[1]>{}.f(0) == 2);
-static_assert(Members<int[2]>{}.g(0) == 0);
-// expected-error@#err-qualified-member {{static assertion failed: instantiated member}} \
-// expected-note@-1{{in instantiation of function template specialization 'Members<int[2]>::g<int>' }}
-Members<int[3]> m1;
-static_assert(m1.g(0) == 1);
-static_assert(Members<int[3]>{}.s(0) == 2);
-
-
-namespace ConstructorInit{
-struct S {
-  template <typename T>
-  S(T&&) {}
-};
-struct Test {
-  operator S() = delete;
-};
-
-static_assert(__is_constructible(S, Test));
-}
-
-namespace RefBinding {
-
-template <typename> struct remove_reference;
-template <typename _Tp> struct remove_reference<_Tp &> {
-  using type = _Tp;
-};
-template <typename _Tp> remove_reference<_Tp>::type move(_Tp &&);
-template <typename _Head> struct _Head_base {
-  _Head_base(_Head &__h) : _M_head_impl(__h) {}
-  template <typename _UHead> _Head_base(_UHead &&);
-  _Head _M_head_impl;
-};
-
-template <typename _Elements> void forward_as_tuple(_Elements &&) {
-  _Head_base<_Elements &&>(_Elements{});
-}
-struct StringRef {
-  void operator[](const StringRef __k) { forward_as_tuple((move)(__k)); }
-};
-
-}
-
-template <class> struct tuple {};
-struct BonkersBananas {
-  template <class T> operator T();
-  template <class = void> explicit operator tuple<int>() = delete;
-};
-static_assert(!__is_constructible(tuple<int>, BonkersBananas));
-
-namespace GH62096 {
-template <typename T>
-struct Oops {
-  static_assert(sizeof(T) == 0); // #GH62096-err
-  static constexpr bool value = true;
-};
-
-template <class OPERATOR>
-concept Operator = Oops<OPERATOR>::value; // #GH62096-note1
-
-template <Operator OP> void f(OP op); // // #GH62096-note2
-void f(int);
-
-void g(int n) { f(n); } // OK
-void h(short n) { f(n); }
-// expected-error@#GH62096-err {{static assertion failed due to requirement 'sizeof(short) == 0'}} \
-// expected-note@-1{{in instantiation of function template specialization}} \
-// expected-note@-1{{while checking constraint satisfaction for template}}
-// expected-note@#GH62096-note1{{in instantiation}}
-// expected-note@#GH62096-note1{{while substituting template arguments into constraint expression here}}
-// expected-note@#GH62096-note2{{while substituting template arguments into constraint expression here}}
-// expected-note@#GH62096-note2{{while checking the satisfaction of concept}}
-// expected-note@#GH62096-err {{expression evaluates}}
-}
-
-#endif
-
-template<typename ...Ts>
-struct t1 {
-};
-struct t6 {
-  template<typename T = int>
-  operator t1<float>() {
-    return {};
-  }
-};
-
-int main() {
-  t6 v6;
-  v6.operator t1<float>();
-}
diff --git a/clang/test/SemaTemplate/instantiate-function-params.cpp b/clang/test/SemaTemplate/instantiate-function-params.cpp
index eb2a7c5d4e8d6..7dd5595de58a3 100644
--- a/clang/test/SemaTemplate/instantiate-function-params.cpp
+++ b/clang/test/SemaTemplate/instantiate-function-params.cpp
@@ -6,12 +6,13 @@ template<typename T1> struct if_ {
   typedef if_c< static_cast<bool>(T1::value)> almost_type_; // expected-note 7{{in instantiation}}
 };
 template <class Model, void (Model::*)()> struct wrap_constraints { };
-template <class Model>
+template <class Model> 
 inline char has_constraints_(Model* , // expected-note 3{{candidate template ignored}}
-                               wrap_constraints<Model,&Model::constraints>* = 0);
+                               wrap_constraints<Model,&Model::constraints>* = 0); // expected-note 4{{in instantiation}}
+
 template <class Model> struct not_satisfied {
   static const bool value = sizeof( has_constraints_((Model*)0)  == 1); // expected-error 3{{no matching function}} \
-  // expected-note 4{{in instantiation}}
+  // expected-note 4{{while substituting deduced template arguments into function template 'has_constraints_' [with }}
 };
 template <class ModelFn> struct requirement_;
 template <void(*)()> struct instantiate {
diff --git a/clang/test/Templight/templight-empty-entries-fix.cpp b/clang/test/Templight/templight-empty-entries-fix.cpp
index 7f34b10134929..d13b748068efe 100644
--- a/clang/test/Templight/templight-empty-entries-fix.cpp
+++ b/clang/test/Templight/templight-empty-entries-fix.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -templight-dump -Wno-unused-value %s 2>&1 | FileCheck %s
 
-void a(long) {
+void a() {
   [] {};
 }
 
@@ -17,14 +17,14 @@ void a(long) {
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 // CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 
-template <int = 0> void a(long) { a(0); }
+template <int = 0> void a() { a(); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -42,29 +42,29 @@ template <int = 0> void a(long) { a(0); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
 
 template <int> struct b { typedef int c; };
-template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
+template <bool d = true, class = typename b<d>::c> void a() { a(); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -130,25 +130,25 @@ template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -166,10 +166,34 @@ template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 
 template <bool = true> void d(int = 0) { d(); }
 
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+a$}}
+// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
+// CHECK: {{^event:[ ]+Begin$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+a$}}
+// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
+// CHECK: {{^event:[ ]+End$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+a$}}
+// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
+// CHECK: {{^event:[ ]+Begin$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+a$}}
+// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
+// CHECK: {{^event:[ ]+End$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
@@ -225,41 +249,41 @@ void e() {
 }
 
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
 
 
 template <template<typename> class>
@@ -275,71 +299,71 @@ void foo() {
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}

From 2ee7fc04568f835be00629cb4130171b14b2e7df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu@arm.com>
Date: Thu, 17 Apr 2025 10:02:53 +0100
Subject: [PATCH 240/710] [SLP] More OOP to simplify vectorizeStores() (NFC)
 (#134605)

This moves more code into the RelatedStoreInsts helper class. The
FillStoresSet lambda is now only a couple of lines and is easier to
read.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 138 ++++++++++--------
 1 file changed, 81 insertions(+), 57 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 41ad43012d337..fd23fb6c81c2c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -20859,9 +20859,16 @@ namespace {
 /// A group of stores that we'll try to bundle together using vector ops.
 /// They are ordered using the signed distance of their address operand to the
 /// address of this group's BaseInstr.
-struct RelatedStoreInsts {
-  RelatedStoreInsts(unsigned BaseInstrIdx) { reset(BaseInstrIdx); }
+class RelatedStoreInsts {
+public:
+  RelatedStoreInsts(unsigned BaseInstrIdx, ArrayRef<StoreInst *> AllStores)
+      : AllStores(AllStores) {
+    reset(BaseInstrIdx);
+  }
+
   void reset(unsigned NewBaseInstr) {
+    assert(NewBaseInstr < AllStores.size() &&
+           "Instruction index out of bounds");
     BaseInstrIdx = NewBaseInstr;
     Instrs.clear();
     insertOrLookup(NewBaseInstr, 0);
@@ -20876,12 +20883,58 @@ struct RelatedStoreInsts {
     return Inserted ? std::nullopt : std::optional<unsigned>(It->second);
   }
 
+  using DistToInstMap = std::map<int, unsigned>;
+  const DistToInstMap &getStores() const { return Instrs; }
+
+  /// If \p SI is related to this group of stores, return the distance of its
+  /// pointer operand to the one the group's BaseInstr.
+  std::optional<int> getPointerDiff(StoreInst &SI, const DataLayout &DL,
+                                    ScalarEvolution &SE) const {
+    StoreInst &BaseStore = *AllStores[BaseInstrIdx];
+    return getPointersDiff(
+        BaseStore.getValueOperand()->getType(), BaseStore.getPointerOperand(),
+        SI.getValueOperand()->getType(), SI.getPointerOperand(), DL, SE,
+        /*StrictCheck=*/true);
+  }
+
+  /// Recompute the pointer distances to be based on \p NewBaseInstIdx.
+  /// Stores whose index is less than \p MinSafeIdx will be dropped.
+  void rebase(unsigned MinSafeIdx, unsigned NewBaseInstIdx,
+              int DistFromCurBase) {
+    DistToInstMap PrevSet = std::move(Instrs);
+    reset(NewBaseInstIdx);
+
+    // Re-insert stores that come after MinSafeIdx to try and vectorize them
+    // again. Their distance will be "rebased" to use NewBaseInstIdx as
+    // reference.
+    for (auto [Dist, InstIdx] : PrevSet) {
+      if (InstIdx >= MinSafeIdx)
+        insertOrLookup(InstIdx, Dist - DistFromCurBase);
+    }
+  }
+
+  /// Remove all stores that have been vectorized from this group.
+  void clearVectorizedStores(const BoUpSLP::ValueSet &VectorizedStores) {
+    DistToInstMap::reverse_iterator LastVectorizedStore = find_if(
+        reverse(Instrs), [&](const std::pair<int, unsigned> &DistAndIdx) {
+          return VectorizedStores.contains(AllStores[DistAndIdx.second]);
+        });
+
+    // Get a forward iterator pointing after the last vectorized store and erase
+    // all stores before it so we don't try to vectorize them again.
+    DistToInstMap::iterator VectorizedStoresEnd = LastVectorizedStore.base();
+    Instrs.erase(Instrs.begin(), VectorizedStoresEnd);
+  }
+
+private:
   /// The index of the Base instruction, i.e. the one with a 0 pointer distance.
   unsigned BaseInstrIdx;
 
   /// Maps a pointer distance from \p BaseInstrIdx to an instruction index.
-  using DistToInstMap = std::map<int, unsigned>;
   DistToInstMap Instrs;
+
+  /// Reference to all the stores in the BB being analyzed.
+  ArrayRef<StoreInst *> AllStores;
 };
 
 } // end anonymous namespace
@@ -21165,14 +21218,7 @@ bool SLPVectorizerPass::vectorizeStores(
     }
   };
 
-  // Stores pair (first: index of the store into Stores array ref, address of
-  // which taken as base, second: sorted set of pairs {index, dist}, which are
-  // indices of stores in the set and their store location distances relative to
-  // the base address).
-
-  // Need to store the index of the very first store separately, since the set
-  // may be reordered after the insertion and the first store may be moved. This
-  // container allows to reduce number of calls of getPointersDiff() function.
+  /// Groups of stores to vectorize
   SmallVector<RelatedStoreInsts> SortedStores;
 
   // Inserts the specified store SI with the given index Idx to the set of the
@@ -21208,52 +21254,30 @@ bool SLPVectorizerPass::vectorizeStores(
   // dependencies and no need to waste compile time to try to vectorize them.
   // - Try to vectorize the sequence {1, {1, 0}, {3, 2}}.
   auto FillStoresSet = [&](unsigned Idx, StoreInst *SI) {
-    for (RelatedStoreInsts &StoreSeq : SortedStores) {
-      std::optional<int> Diff = getPointersDiff(
-          Stores[StoreSeq.BaseInstrIdx]->getValueOperand()->getType(),
-          Stores[StoreSeq.BaseInstrIdx]->getPointerOperand(),
-          SI->getValueOperand()->getType(), SI->getPointerOperand(), *DL, *SE,
-          /*StrictCheck=*/true);
-      if (!Diff)
-        continue;
-      std::optional<unsigned> PrevInst =
-          StoreSeq.insertOrLookup(/*InstrIdx=*/Idx, /*PtrDist=*/*Diff);
-      if (!PrevInst) {
-        // No store was associated to that distance. Keep collecting.
-        return;
-      }
-      // Try to vectorize the first found set to avoid duplicate analysis.
-      TryToVectorize(StoreSeq.Instrs);
-      RelatedStoreInsts::DistToInstMap PrevSet;
-      copy_if(StoreSeq.Instrs, std::inserter(PrevSet, PrevSet.end()),
-              [&](const std::pair<int, unsigned> &DistAndIdx) {
-                return DistAndIdx.second > *PrevInst;
-              });
-      StoreSeq.reset(Idx);
-      // Insert stores that followed previous match to try to vectorize them
-      // with this store.
-      unsigned StartIdx = *PrevInst + 1;
-      SmallBitVector UsedStores(Idx - StartIdx);
-      // Distances to previously found dup store (or this store, since they
-      // store to the same addresses).
-      SmallVector<int> Dists(Idx - StartIdx, 0);
-      for (auto [PtrDist, InstIdx] : reverse(PrevSet)) {
-        // Do not try to vectorize sequences, we already tried.
-        if (VectorizedStores.contains(Stores[InstIdx]))
-          break;
-        unsigned BI = InstIdx - StartIdx;
-        UsedStores.set(BI);
-        Dists[BI] = PtrDist - *Diff;
-      }
-      for (unsigned I = StartIdx; I < Idx; ++I) {
-        unsigned BI = I - StartIdx;
-        if (UsedStores.test(BI))
-          StoreSeq.insertOrLookup(I, Dists[BI]);
-      }
+    std::optional<int> PtrDist;
+    auto *RelatedStores = find_if(
+        SortedStores, [&PtrDist, SI, this](const RelatedStoreInsts &StoreSeq) {
+          PtrDist = StoreSeq.getPointerDiff(*SI, *DL, *SE);
+          return PtrDist.has_value();
+        });
+
+    // We did not find a comparable store, start a new group.
+    if (RelatedStores == SortedStores.end()) {
+      SortedStores.emplace_back(Idx, Stores);
       return;
     }
-    // We did not find a comparable store, start a new sequence.
-    SortedStores.emplace_back(Idx);
+
+    // If there is already a store in the group with the same PtrDiff, try to
+    // vectorize the existing instructions before adding the current store.
+    // Otherwise, insert this store and keep collecting.
+    if (std::optional<unsigned> PrevInst =
+            RelatedStores->insertOrLookup(Idx, *PtrDist)) {
+      TryToVectorize(RelatedStores->getStores());
+      RelatedStores->clearVectorizedStores(VectorizedStores);
+      RelatedStores->rebase(/*MinSafeIdx=*/*PrevInst + 1,
+                            /*NewBaseInstIdx=*/Idx,
+                            /*DistFromCurBase=*/*PtrDist);
+    }
   };
   Type *PrevValTy = nullptr;
   for (auto [I, SI] : enumerate(Stores)) {
@@ -21264,7 +21288,7 @@ bool SLPVectorizerPass::vectorizeStores(
     // Check that we do not try to vectorize stores of different types.
     if (PrevValTy != SI->getValueOperand()->getType()) {
       for (RelatedStoreInsts &StoreSeq : SortedStores)
-        TryToVectorize(StoreSeq.Instrs);
+        TryToVectorize(StoreSeq.getStores());
       SortedStores.clear();
       PrevValTy = SI->getValueOperand()->getType();
     }
@@ -21273,7 +21297,7 @@ bool SLPVectorizerPass::vectorizeStores(
 
   // Final vectorization attempt.
   for (RelatedStoreInsts &StoreSeq : SortedStores)
-    TryToVectorize(StoreSeq.Instrs);
+    TryToVectorize(StoreSeq.getStores());
 
   return Changed;
 }

From 121cd7c6f0270540ce976b98061ba765edc1675d Mon Sep 17 00:00:00 2001
From: Shoreshen <372660931@qq.com>
Date: Thu, 17 Apr 2025 17:03:32 +0800
Subject: [PATCH 241/710] Re apply 130577 narrow math for and operand (#133896)

Re-apply https://github.com/llvm/llvm-project/pull/130577

Which is reverted in https://github.com/llvm/llvm-project/pull/133880

The old application failed in address sanitizer due to
`tryNarrowMathIfNoOverflow` was called after `I.eraseFromParent();` in
`AMDGPUCodeGenPrepareImpl::visitBinaryOperator`, it create a use after
free failure.

To fix this, `tryNarrowMathIfNoOverflow` will be called before and
directly return if `tryNarrowMathIfNoOverflow` result in true.
---
 .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp    |  77 ++++++
 .../AMDGPU/amdgpu-codegenprepare-mul24.ll     |   5 +-
 .../atomic_optimizations_global_pointer.ll    |  52 ++--
 .../CodeGen/AMDGPU/memcpy-crash-issue63986.ll |  10 +-
 .../CodeGen/AMDGPU/narrow_math_for_and.ll     | 225 ++++++++++++++++++
 llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll  |  25 +-
 6 files changed, 346 insertions(+), 48 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/narrow_math_for_and.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index df92847c1ba71..a37128b0d745a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -1559,6 +1559,80 @@ void AMDGPUCodeGenPrepareImpl::expandDivRem64(BinaryOperator &I) const {
   llvm_unreachable("not a division");
 }
 
+/*
+This will cause non-byte load in consistency, for example:
+```
+    %load = load i1, ptr addrspace(4) %arg, align 4
+    %zext = zext i1 %load to
+    i64 %add = add i64 %zext
+```
+Instead of creating `s_and_b32 s0, s0, 1`,
+it will create `s_and_b32 s0, s0, 0xff`.
+We accept this change since the non-byte load assumes the upper bits
+within the byte are all 0.
+*/
+static bool tryNarrowMathIfNoOverflow(Instruction *I,
+                                      const SITargetLowering *TLI,
+                                      const TargetTransformInfo &TTI,
+                                      const DataLayout &DL) {
+  unsigned Opc = I->getOpcode();
+  Type *OldType = I->getType();
+
+  if (Opc != Instruction::Add && Opc != Instruction::Mul)
+    return false;
+
+  unsigned OrigBit = OldType->getScalarSizeInBits();
+
+  if (Opc != Instruction::Add && Opc != Instruction::Mul)
+    llvm_unreachable("Unexpected opcode, only valid for Instruction::Add and "
+                     "Instruction::Mul.");
+
+  unsigned MaxBitsNeeded = computeKnownBits(I, DL).countMaxActiveBits();
+
+  MaxBitsNeeded = std::max<unsigned>(bit_ceil(MaxBitsNeeded), 8);
+  Type *NewType = DL.getSmallestLegalIntType(I->getContext(), MaxBitsNeeded);
+  if (!NewType)
+    return false;
+  unsigned NewBit = NewType->getIntegerBitWidth();
+  if (NewBit >= OrigBit)
+    return false;
+  NewType = I->getType()->getWithNewBitWidth(NewBit);
+
+  // Old cost
+  InstructionCost OldCost =
+      TTI.getArithmeticInstrCost(Opc, OldType, TTI::TCK_RecipThroughput);
+  // New cost of new op
+  InstructionCost NewCost =
+      TTI.getArithmeticInstrCost(Opc, NewType, TTI::TCK_RecipThroughput);
+  // New cost of narrowing 2 operands (use trunc)
+  int NumOfNonConstOps = 2;
+  if (isa<Constant>(I->getOperand(0)) || isa<Constant>(I->getOperand(1))) {
+    // Cannot be both constant, should be propagated
+    NumOfNonConstOps = 1;
+  }
+  NewCost += NumOfNonConstOps * TTI.getCastInstrCost(Instruction::Trunc,
+                                                     NewType, OldType,
+                                                     TTI.getCastContextHint(I),
+                                                     TTI::TCK_RecipThroughput);
+  // New cost of zext narrowed result to original type
+  NewCost +=
+      TTI.getCastInstrCost(Instruction::ZExt, OldType, NewType,
+                           TTI.getCastContextHint(I), TTI::TCK_RecipThroughput);
+  if (NewCost >= OldCost)
+    return false;
+
+  IRBuilder<> Builder(I);
+  Value *Trunc0 = Builder.CreateTrunc(I->getOperand(0), NewType);
+  Value *Trunc1 = Builder.CreateTrunc(I->getOperand(1), NewType);
+  Value *Arith =
+      Builder.CreateBinOp((Instruction::BinaryOps)Opc, Trunc0, Trunc1);
+
+  Value *Zext = Builder.CreateZExt(Arith, OldType);
+  I->replaceAllUsesWith(Zext);
+  I->eraseFromParent();
+  return true;
+}
+
 bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
   if (foldBinOpIntoSelect(I))
     return true;
@@ -1569,6 +1643,9 @@ bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
 
   if (UseMul24Intrin && replaceMulWithMul24(I))
     return true;
+  if (tryNarrowMathIfNoOverflow(&I, ST.getTargetLowering(),
+                                TM.getTargetTransformInfo(F), DL))
+    return true;
 
   bool Changed = false;
   Instruction::BinaryOps Opc = I.getOpcode();
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll
index 296b817bc8f75..d7c35a8b007c6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll
@@ -414,7 +414,10 @@ define i64 @umul24_i64_2(i64 %lhs, i64 %rhs) {
 ; DISABLED-LABEL: @umul24_i64_2(
 ; DISABLED-NEXT:    [[LHS24:%.*]] = and i64 [[LHS:%.*]], 65535
 ; DISABLED-NEXT:    [[RHS24:%.*]] = and i64 [[RHS:%.*]], 65535
-; DISABLED-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
+; DISABLED-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
+; DISABLED-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP1]], [[TMP2]]
+; DISABLED-NEXT:    [[MUL:%.*]] = zext i32 [[TMP3]] to i64
 ; DISABLED-NEXT:    ret i64 [[MUL]]
 ;
   %lhs24 = and i64 %lhs, 65535
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 62083b3e67ab6..e2dfcf55b7856 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -1823,22 +1823,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1264:       ; %bb.0: ; %entry
 ; GFX1264-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX1264-NEXT:    s_mov_b64 s[6:7], exec
-; GFX1264-NEXT:    s_mov_b32 s9, 0
-; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    s_mov_b64 s[4:5], exec
+; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v2, s7, v0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1264-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1264-NEXT:    s_cbranch_execz .LBB3_2
 ; GFX1264-NEXT:  ; %bb.1:
-; GFX1264-NEXT:    s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1264-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1264-NEXT:    s_wait_alu 0xfffe
+; GFX1264-NEXT:    s_mul_i32 s6, s6, 5
 ; GFX1264-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT:    s_mul_u64 s[6:7], s[8:9], 5
-; GFX1264-NEXT:    s_mov_b32 s10, -1
 ; GFX1264-NEXT:    s_wait_alu 0xfffe
 ; GFX1264-NEXT:    v_mov_b32_e32 v0, s6
-; GFX1264-NEXT:    v_mov_b32_e32 v1, s7
+; GFX1264-NEXT:    s_mov_b32 s10, -1
 ; GFX1264-NEXT:    s_wait_kmcnt 0x0
 ; GFX1264-NEXT:    s_mov_b32 s8, s2
 ; GFX1264-NEXT:    s_mov_b32 s9, s3
@@ -1860,20 +1860,19 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-LABEL: add_i64_constant:
 ; GFX1232:       ; %bb.0: ; %entry
 ; GFX1232-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT:    s_mov_b32 s7, exec_lo
-; GFX1232-NEXT:    s_mov_b32 s5, 0
-; GFX1232-NEXT:    v_mbcnt_lo_u32_b32 v2, s7, 0
 ; GFX1232-NEXT:    s_mov_b32 s6, exec_lo
+; GFX1232-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1232-NEXT:    v_mbcnt_lo_u32_b32 v2, s6, 0
 ; GFX1232-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1232-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1232-NEXT:    s_cbranch_execz .LBB3_2
 ; GFX1232-NEXT:  ; %bb.1:
-; GFX1232-NEXT:    s_bcnt1_i32_b32 s4, s7
+; GFX1232-NEXT:    s_bcnt1_i32_b32 s5, s6
 ; GFX1232-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX1232-NEXT:    s_mul_u64 s[4:5], s[4:5], 5
+; GFX1232-NEXT:    s_mul_i32 s5, s5, 5
 ; GFX1232-NEXT:    s_mov_b32 s10, -1
-; GFX1232-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1232-NEXT:    v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
 ; GFX1232-NEXT:    s_mov_b32 s8, s2
 ; GFX1232-NEXT:    s_mov_b32 s9, s3
@@ -1881,8 +1880,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-NEXT:    s_wait_loadcnt 0x0
 ; GFX1232-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX1232-NEXT:  .LBB3_2:
-; GFX1232-NEXT:    s_wait_alu 0xfffe
-; GFX1232-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX1232-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
 ; GFX1232-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX1232-NEXT:    v_readfirstlane_b32 s2, v0
@@ -5372,22 +5370,22 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1264:       ; %bb.0: ; %entry
 ; GFX1264-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX1264-NEXT:    s_mov_b64 s[6:7], exec
-; GFX1264-NEXT:    s_mov_b32 s9, 0
-; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    s_mov_b64 s[4:5], exec
+; GFX1264-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1264-NEXT:    v_mbcnt_hi_u32_b32 v2, s7, v0
 ; GFX1264-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1264-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1264-NEXT:    s_cbranch_execz .LBB9_2
 ; GFX1264-NEXT:  ; %bb.1:
-; GFX1264-NEXT:    s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1264-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1264-NEXT:    s_wait_alu 0xfffe
+; GFX1264-NEXT:    s_mul_i32 s6, s6, 5
 ; GFX1264-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT:    s_mul_u64 s[6:7], s[8:9], 5
-; GFX1264-NEXT:    s_mov_b32 s10, -1
 ; GFX1264-NEXT:    s_wait_alu 0xfffe
 ; GFX1264-NEXT:    v_mov_b32_e32 v0, s6
-; GFX1264-NEXT:    v_mov_b32_e32 v1, s7
+; GFX1264-NEXT:    s_mov_b32 s10, -1
 ; GFX1264-NEXT:    s_wait_kmcnt 0x0
 ; GFX1264-NEXT:    s_mov_b32 s8, s2
 ; GFX1264-NEXT:    s_mov_b32 s9, s3
@@ -5412,20 +5410,19 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-LABEL: sub_i64_constant:
 ; GFX1232:       ; %bb.0: ; %entry
 ; GFX1232-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT:    s_mov_b32 s7, exec_lo
-; GFX1232-NEXT:    s_mov_b32 s5, 0
-; GFX1232-NEXT:    v_mbcnt_lo_u32_b32 v2, s7, 0
 ; GFX1232-NEXT:    s_mov_b32 s6, exec_lo
+; GFX1232-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1232-NEXT:    v_mbcnt_lo_u32_b32 v2, s6, 0
 ; GFX1232-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1232-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1232-NEXT:    s_cbranch_execz .LBB9_2
 ; GFX1232-NEXT:  ; %bb.1:
-; GFX1232-NEXT:    s_bcnt1_i32_b32 s4, s7
+; GFX1232-NEXT:    s_bcnt1_i32_b32 s5, s6
 ; GFX1232-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX1232-NEXT:    s_mul_u64 s[4:5], s[4:5], 5
+; GFX1232-NEXT:    s_mul_i32 s5, s5, 5
 ; GFX1232-NEXT:    s_mov_b32 s10, -1
-; GFX1232-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1232-NEXT:    v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
 ; GFX1232-NEXT:    s_mov_b32 s8, s2
 ; GFX1232-NEXT:    s_mov_b32 s9, s3
@@ -5433,8 +5430,7 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-NEXT:    s_wait_loadcnt 0x0
 ; GFX1232-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX1232-NEXT:  .LBB9_2:
-; GFX1232-NEXT:    s_wait_alu 0xfffe
-; GFX1232-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX1232-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
 ; GFX1232-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1232-NEXT:    v_mul_u32_u24_e32 v0, 5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index 29d9164f95d99..c92c672dda2ad 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -170,13 +170,13 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
 ; CHECK-NEXT:    s_cbranch_execnz .LBB1_8
 ; CHECK-NEXT:  .LBB1_5: ; %loop-memcpy-residual.preheader
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s4
-; CHECK-NEXT:    s_mov_b64 s[6:7], 0
+; CHECK-NEXT:    s_mov_b64 s[8:9], 0
+; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v1, s5
 ; CHECK-NEXT:  .LBB1_6: ; %loop-memcpy-residual
-; CHECK-NEXT:    s_add_u32 s4, s6, 1
-; CHECK-NEXT:    s_addc_u32 s5, s7, 0
-; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
-; CHECK-NEXT:    s_mov_b64 s[6:7], 1
+; CHECK-NEXT:    s_add_i32 s6, s8, 1
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
+; CHECK-NEXT:    s_mov_b64 s[8:9], 1
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB1_6
 ; CHECK-NEXT:  ; %bb.7: ; %Flow
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/narrow_math_for_and.ll b/llvm/test/CodeGen/AMDGPU/narrow_math_for_and.ll
new file mode 100644
index 0000000000000..151456e82ae51
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/narrow_math_for_and.ll
@@ -0,0 +1,225 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck %s
+
+define i64 @narrow_add(i64 %a, i64 %b) {
+; CHECK-LABEL: narrow_add:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v0
+; CHECK-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v2
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v1
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and i64 %a, 2147483647
+  %zext1 = and i64 %b, 2147483647
+  %add = add i64 %zext0, %zext1
+  ret i64 %add
+}
+
+define i64 @narrow_add_1(i64 %a, i64 %b) {
+; CHECK-LABEL: narrow_add_1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_bfi_b32 v0, 0x7fffffff, v0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and i64 %a, 2147483647
+  %zext1 = and i64 %b, 2147483648
+  %add = add i64 %zext0, %zext1
+  ret i64 %add
+}
+
+define <2 x i64> @narrow_add_vec(<2 x i64> %a, <2 x i64> %b) #0 {
+; CHECK-LABEL: narrow_add_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 30, v2
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v0
+; CHECK-NEXT:    v_and_b32_e32 v2, 0x7fffffff, v4
+; CHECK-NEXT:    v_and_b32_e32 v3, 0x7ffffffe, v6
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, v0, v2
+; CHECK-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_add_nc_u32 v2, v1, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and <2 x i64> %a, <i64 2147483647, i64 30>
+  %zext1 = and <2 x i64> %b, <i64 2147483647, i64 2147483646>
+  %add = add <2 x i64> %zext0, %zext1
+  ret <2 x i64> %add
+}
+
+define <2 x i32> @narrow_add_vec_1(<2 x i32> %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: narrow_add_vec_1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 0x3fff, v1
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x4000, v0
+; CHECK-NEXT:    v_and_b32_e32 v2, 0x4000, v2
+; CHECK-NEXT:    v_and_b32_e32 v3, 0x4001, v3
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, v0, v2
+; CHECK-NEXT:    v_add_nc_u32_e32 v1, v1, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and <2 x i32> %a, <i32 16384, i32 16383>
+  %zext1 = and <2 x i32> %b, <i32 16384, i32 16385>
+  %add = add <2 x i32> %zext0, %zext1
+  ret <2 x i32> %add
+}
+
+define i64 @narrow_mul(i64 %a, i64 %b) {
+; CHECK-LABEL: narrow_mul:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 2, v2
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT:    v_mul_lo_u32 v0, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and i64 %a, 2147483647
+  %zext1 = and i64 %b, 2
+  %mul = mul i64 %zext0, %zext1
+  ret i64 %mul
+}
+
+define i64 @narrow_mul_1(i64 %a, i64 %b) {
+; CHECK-LABEL: narrow_mul_1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 0xf73594, v0
+; CHECK-NEXT:    v_and_b32_e32 v2, 0x100, v2
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT:    v_mul_u32_u24_e32 v0, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32_u24_e32 v1, v1, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and i64 %a, 16201108
+  %zext1 = and i64 %b, 256
+  %mul = mul i64 %zext0, %zext1
+  ret i64 %mul
+}
+
+define <2 x i64> @narrow_mul_vec(<2 x i64> %a, <2 x i64> %b) #0 {
+; CHECK-LABEL: narrow_mul_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x2d48aff, v0
+; CHECK-NEXT:    v_and_b32_e32 v1, 0x50, v4
+; CHECK-NEXT:    v_and_b32_e32 v2, 50, v2
+; CHECK-NEXT:    v_and_b32_e32 v3, 20, v6
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; CHECK-NEXT:    v_mul_lo_u32 v0, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mul_u32_u24_e32 v2, v2, v3
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and <2 x i64> %a, <i64 47483647, i64 50>
+  %zext1 = and <2 x i64> %b, <i64 80, i64 20>
+  %mul = mul <2 x i64> %zext0, %zext1
+  ret <2 x i64> %mul
+}
+
+define <2 x i32> @narrow_add_mul_1(<2 x i32> %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: narrow_add_mul_1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 0x4000, v1
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x4000, v0
+; CHECK-NEXT:    v_and_b32_e32 v2, 3, v2
+; CHECK-NEXT:    v_and_b32_e32 v3, 2, v3
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
+; CHECK-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and <2 x i32> %a, <i32 16384, i32 16384>
+  %zext1 = and <2 x i32> %b, <i32 3, i32 2>
+  %mul = mul <2 x i32> %zext0, %zext1
+  ret <2 x i32> %mul
+}
+
+define i64 @no_narrow_add(i64 %a, i64 %b) {
+; CHECK-LABEL: no_narrow_add:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x80000000, v0
+; CHECK-NEXT:    v_and_b32_e32 v1, 0x80000000, v2
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_add_co_u32 v0, s0, v0, v1
+; CHECK-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, 0, s0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and i64 %a, 2147483648
+  %zext1 = and i64 %b, 2147483648
+  %add = add i64 %zext0, %zext1
+  ret i64 %add
+}
+
+define i64 @no_narrow_add_1(i64 %a, i64 %b) {
+; CHECK-LABEL: no_narrow_add_1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 1, v2
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_add_co_u32 v0, s0, v0, v1
+; CHECK-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, 0, s0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and i64 %a, 4294967295
+  %zext1 = and i64 %b, 1
+  %add = add i64 %zext0, %zext1
+  ret i64 %add
+}
+
+define <2 x i64> @no_narrow_add_vec(<2 x i64> %a, <2 x i64> %b) #0 {
+; CHECK-LABEL: no_narrow_add_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x80000000, v0
+; CHECK-NEXT:    v_and_b32_e32 v1, 0x80000000, v4
+; CHECK-NEXT:    v_and_b32_e32 v2, 30, v2
+; CHECK-NEXT:    v_and_b32_e32 v3, 0x7ffffffe, v6
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_add_co_u32 v0, s0, v0, v1
+; CHECK-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, 0, s0
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_add_co_u32 v2, s0, v2, v3
+; CHECK-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and <2 x i64> %a, <i64 2147483648, i64 30>
+  %zext1 = and <2 x i64> %b, <i64 2147483648, i64 2147483646>
+  %add = add <2 x i64> %zext0, %zext1
+  ret <2 x i64> %add
+}
+
+define i64 @no_narrow_mul(i64 %a, i64 %b) {
+; CHECK-LABEL: no_narrow_mul:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x80000000, v0
+; CHECK-NEXT:    v_and_b32_e32 v1, 2, v2
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and i64 %a, 2147483648
+  %zext1 = and i64 %b, 2
+  %mul = mul i64 %zext0, %zext1
+  ret i64 %mul
+}
+
+define <2 x i64> @no_narrow_mul_vec(<2 x i64> %a, <2 x i64> %b) #0 {
+; CHECK-LABEL: no_narrow_mul_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, 0x8000, v0
+; CHECK-NEXT:    v_and_b32_e32 v3, 0x20000, v4
+; CHECK-NEXT:    v_and_b32_e32 v4, 50, v2
+; CHECK-NEXT:    v_and_b32_e32 v5, 20, v6
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; CHECK-NEXT:    v_mul_u32_u24_e32 v0, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32_u24_e32 v1, v1, v3
+; CHECK-NEXT:    v_mul_u32_u24_e32 v2, v4, v5
+; CHECK-NEXT:    v_mul_hi_u32_u24_e32 v3, v4, v5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %zext0 = and <2 x i64> %a, <i64 32768, i64 50>
+  %zext1 = and <2 x i64> %b, <i64 131072, i64 20>
+  %mul = mul <2 x i64> %zext0, %zext1
+  ret <2 x i64> %mul
+}
diff --git a/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll b/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
index 4290590e99711..bac70b69650cd 100644
--- a/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
@@ -508,17 +508,16 @@ define amdgpu_kernel void @widen_i1_zext_to_i64_constant_load(ptr addrspace(4) %
 ; SI-LABEL: widen_i1_zext_to_i64_constant_load:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dword s2, s[0:1], 0x0
 ; SI-NEXT:    s_mov_b64 s[0:1], 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_and_b32 s2, s2, 1
-; SI-NEXT:    s_add_u32 s4, s2, 0x3e7
-; SI-NEXT:    s_addc_u32 s5, 0, 0
-; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    s_and_b32 s2, s2, 0xff
+; SI-NEXT:    s_addk_i32 s2, 0x3e7
+; SI-NEXT:    v_mov_b32_e32 v0, s2
 ; SI-NEXT:    s_mov_b32 s2, -1
-; SI-NEXT:    v_mov_b32_e32 v1, s5
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -527,13 +526,12 @@ define amdgpu_kernel void @widen_i1_zext_to_i64_constant_load(ptr addrspace(4) %
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; VI-NEXT:    v_mov_b32_e32 v0, 0
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    v_mov_b32_e32 v3, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_and_b32 s0, s0, 1
-; VI-NEXT:    s_add_u32 s0, s0, 0x3e7
-; VI-NEXT:    s_addc_u32 s1, 0, 0
-; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_and_b32 s0, s0, 0xff
+; VI-NEXT:    s_addk_i32 s0, 0x3e7
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
@@ -541,14 +539,13 @@ define amdgpu_kernel void @widen_i1_zext_to_i64_constant_load(ptr addrspace(4) %
 ; GFX11-LABEL: widen_i1_zext_to_i64_constant_load:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_and_b32 s0, s0, 1
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_add_u32 s0, s0, 0x3e7
-; GFX11-NEXT:    s_addc_u32 s1, 0, 0
-; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT:    s_and_b32 s0, s0, 0xff
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_addk_i32 s0, 0x3e7
 ; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
 ; GFX11-NEXT:    global_store_b64 v[0:1], v[2:3], off
 ; GFX11-NEXT:    s_endpgm

From c5e112eed78a8ddfef5d16f6df6030c3ec8ca6ef Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Thu, 17 Apr 2025 10:08:07 +0100
Subject: [PATCH 242/710] [flang][OpenMP][Semantics] Disallow NOWAIT and
 ORDERED with CANCEL (#135991)

NOWAIT was a tricky one because the clause can be on either the start or
the end directive. I couldn't find a convenient way to access the end
directive from the CANCEL directive nested inside of the construct, but
there are convenient ways to access the start directive. I have added a
list to the start directive context containing the clauses from the end
directive.
---
 .../lib/Semantics/check-directive-structure.h |  1 +
 flang/lib/Semantics/check-omp-structure.cpp   | 46 ++++++++++++++++-
 flang/lib/Semantics/check-omp-structure.h     |  2 +
 flang/test/Semantics/OpenMP/cancel.f90        | 51 +++++++++++++++++++
 4 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/check-directive-structure.h b/flang/lib/Semantics/check-directive-structure.h
index 91ffda6404c23..4a4893fe805a2 100644
--- a/flang/lib/Semantics/check-directive-structure.h
+++ b/flang/lib/Semantics/check-directive-structure.h
@@ -202,6 +202,7 @@ class DirectiveStructureChecker : public virtual BaseChecker {
     const PC *clause{nullptr};
     ClauseMapTy clauseInfo;
     std::list<C> actualClauses;
+    std::list<C> endDirectiveClauses;
     std::list<C> crtGroup;
     Symbol *loopIV{nullptr};
   };
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 717982f66027c..bc4651584b9a8 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -15,6 +15,7 @@
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/openmp-modifiers.h"
 #include "flang/Semantics/tools.h"
+#include "llvm/ADT/STLExtras.h"
 #include <variant>
 
 namespace Fortran::semantics {
@@ -682,11 +683,20 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclarativeConstruct &x) {
   ExitDirectiveNest(DeclarativeNest);
 }
 
+void OmpStructureChecker::AddEndDirectiveClauses(
+    const parser::OmpClauseList &clauses) {
+  for (const parser::OmpClause &clause : clauses.v) {
+    GetContext().endDirectiveClauses.push_back(clause.Id());
+  }
+}
+
 void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
   loopStack_.push_back(&x);
   const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
   const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
 
+  PushContextAndClauseSets(beginDir.source, beginDir.v);
+
   // check matching, End directive is optional
   if (const auto &endLoopDir{
           std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
@@ -694,9 +704,10 @@ void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
         std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
 
     CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
+
+    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
   }
 
-  PushContextAndClauseSets(beginDir.source, beginDir.v);
   if (llvm::omp::allSimdSet.test(GetContext().directive)) {
     EnterDirectiveNest(SIMDNest);
   }
@@ -1429,6 +1440,8 @@ void OmpStructureChecker::Enter(const parser::OpenMPSectionsConstruct &x) {
   CheckMatching<parser::OmpSectionsDirective>(beginDir, endDir);
 
   PushContextAndClauseSets(beginDir.source, beginDir.v);
+  AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endSectionsDir.t));
+
   const auto &sectionBlocks{std::get<parser::OmpSectionBlocks>(x.t)};
   for (const parser::OpenMPConstruct &block : sectionBlocks.v) {
     CheckNoBranching(std::get<parser::OpenMPSectionConstruct>(block.u).v,
@@ -2288,6 +2301,37 @@ void OmpStructureChecker::Enter(const parser::OpenMPCancelConstruct &x) {
   if (auto maybeConstruct{GetCancelType(
           llvm::omp::Directive::OMPD_cancel, x.source, maybeClauses)}) {
     CheckCancellationNest(dirName.source, *maybeConstruct);
+
+    if (CurrentDirectiveIsNested()) {
+      // nowait can be put on the end directive rather than the start directive
+      // so we need to check both
+      auto getParentClauses{[&]() {
+        const DirectiveContext &parent{GetContextParent()};
+        return llvm::concat<const llvm::omp::Clause>(
+            parent.actualClauses, parent.endDirectiveClauses);
+      }};
+
+      if (llvm::omp::nestedCancelDoAllowedSet.test(*maybeConstruct)) {
+        for (llvm::omp::Clause clause : getParentClauses()) {
+          if (clause == llvm::omp::Clause::OMPC_nowait) {
+            context_.Say(dirName.source,
+                "The CANCEL construct cannot be nested inside of a worksharing construct with the NOWAIT clause"_err_en_US);
+          }
+          if (clause == llvm::omp::Clause::OMPC_ordered) {
+            context_.Say(dirName.source,
+                "The CANCEL construct cannot be nested inside of a worksharing construct with the ORDERED clause"_err_en_US);
+          }
+        }
+      } else if (llvm::omp::nestedCancelSectionsAllowedSet.test(
+                     *maybeConstruct)) {
+        for (llvm::omp::Clause clause : getParentClauses()) {
+          if (clause == llvm::omp::Clause::OMPC_nowait) {
+            context_.Say(dirName.source,
+                "The CANCEL construct cannot be nested inside of a worksharing construct with the NOWAIT clause"_err_en_US);
+          }
+        }
+      }
+    }
   }
 }
 
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index a8869561cf5ea..5af3b505937c1 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -318,6 +318,8 @@ class OmpStructureChecker
 
   void CheckAlignValue(const parser::OmpClause &);
 
+  void AddEndDirectiveClauses(const parser::OmpClauseList &clauses);
+
   void EnterDirectiveNest(const int index) { directiveNest_[index]++; }
   void ExitDirectiveNest(const int index) { directiveNest_[index]--; }
   int GetDirectiveNest(const int index) { return directiveNest_[index]; }
diff --git a/flang/test/Semantics/OpenMP/cancel.f90 b/flang/test/Semantics/OpenMP/cancel.f90
index 581c4bdb97646..6f3a255312ccf 100644
--- a/flang/test/Semantics/OpenMP/cancel.f90
+++ b/flang/test/Semantics/OpenMP/cancel.f90
@@ -27,3 +27,54 @@ subroutine f03
 !$omp cancellation point parallel parallel
 !$omp end parallel
 end
+
+subroutine do_nowait1
+!$omp parallel
+!$omp do nowait
+  do i=1,2
+!ERROR: The CANCEL construct cannot be nested inside of a worksharing construct with the NOWAIT clause
+    !$omp cancel do
+  enddo
+!$omp end do
+!$omp end parallel
+end subroutine
+
+subroutine do_nowait2
+!$omp parallel
+!$omp do
+  do i=1,2
+!ERROR: The CANCEL construct cannot be nested inside of a worksharing construct with the NOWAIT clause
+    !$omp cancel do
+  enddo
+!$omp end do nowait
+!$omp end parallel
+end subroutine
+
+subroutine do_ordered
+!$omp parallel do ordered
+  do i=1,2
+!ERROR: The CANCEL construct cannot be nested inside of a worksharing construct with the ORDERED clause
+    !$omp cancel do
+  enddo
+!$omp end parallel do
+end subroutine
+
+subroutine sections_nowait1
+!$omp parallel
+!$omp sections nowait
+  !$omp section
+!ERROR: The CANCEL construct cannot be nested inside of a worksharing construct with the NOWAIT clause
+    !$omp cancel sections
+!$omp end sections
+!$omp end parallel
+end subroutine
+
+subroutine sections_nowait2
+!$omp parallel
+!$omp sections
+  !$omp section
+!ERROR: The CANCEL construct cannot be nested inside of a worksharing construct with the NOWAIT clause
+    !$omp cancel sections
+!$omp end sections nowait
+!$omp end parallel
+end subroutine

From bd49bbaaafc98433a2cb4e95ce25b7a201baf5a5 Mon Sep 17 00:00:00 2001
From: Kashyap Chamarthy <kchamart@redhat.com>
Date: Thu, 17 Apr 2025 11:09:18 +0200
Subject: [PATCH 243/710] [ORC][unittests] Skip the ReOptimizeLayerTest for
 RISC-V (#134702)

David Abdurachmanov reports[1] that we hit this error on P550 hardware:

/builddir/build/BUILD/llvm-20.1.1-build/llvm-project-20.1.1.src/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp:140:
Failure
    Value of: llvm::detail::TakeError(RM.takeError())
    Expected: succeeded
Actual: failed (Architecture not supported) (of type
llvm::detail::ErrorHolder)

Tom Stellard noted[2] that he's seen this before on other architectures
and suggested to skip it.

[1] https://src.fedoraproject.org/rpms/llvm/pull-request/408#comment-255547
[2] https://src.fedoraproject.org/rpms/llvm/pull-request/408#comment-255557

Signed-off-by: Kashyap Chamarthy <kchamart@redhat.com>
---
 llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
index 78dc0f24c0f99..a7edac29d1e99 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
@@ -55,6 +55,10 @@ class ReOptimizeLayerTest : public testing::Test {
     if (Triple.isPPC())
       GTEST_SKIP();
 
+    // RISC-V is not supported yet
+    if (Triple.isRISCV())
+      GTEST_SKIP();
+
     auto EPC = SelfExecutorProcessControl::Create();
     if (!EPC) {
       consumeError(EPC.takeError());

From c3c0b27f2d34cd106df278afc378c30fe493f513 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 17 Apr 2025 11:11:00 +0200
Subject: [PATCH 244/710] [Intrinsics] Add support for range attributes
 (#135642)

Add support for specifying range attributes in Intrinsics.td. Use this
to specify the ucmp/scmp range [-1,2).

This case is trickier than existing intrinsic attributes, because we
need to create the attribute with the correct bitwidth. As such, the
attribute construction now needs to be aware of the function type.

We also need to be careful to no longer assign attributes on intrinsics
with invalid signatures, as we'd make invalid assumptions about the
number of arguments etc otherwise.

Fixes https://github.com/llvm/llvm-project/issues/130179.
---
 llvm/include/llvm/IR/Intrinsics.h             |  2 +-
 llvm/include/llvm/IR/Intrinsics.td            | 12 +++++++--
 llvm/lib/IR/AutoUpgrade.cpp                   |  9 +++++--
 llvm/lib/IR/Function.cpp                      | 11 ++++++--
 llvm/lib/IR/Intrinsics.cpp                    |  1 +
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  3 ++-
 .../Scalar/RewriteStatepointsForGC.cpp        |  2 +-
 .../aarch64-intrinsics-attributes.ll          |  4 +--
 .../autoupgrade-invalid-mem-intrinsics.ll     | 12 ++++-----
 llvm/test/TableGen/intrinsic-attrs.td         | 11 +++++---
 .../CorrelatedValuePropagation/uscmp.ll       | 12 ++++-----
 .../TableGen/Basic/CodeGenIntrinsics.cpp      | 13 +++++++---
 llvm/utils/TableGen/Basic/CodeGenIntrinsics.h | 13 +++++++---
 .../utils/TableGen/Basic/IntrinsicEmitter.cpp | 25 ++++++++++++++++---
 14 files changed, 90 insertions(+), 40 deletions(-)

diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index ccca7a06f43aa..6fb1bf9359b9a 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -86,7 +86,7 @@ namespace Intrinsic {
   ID lookupIntrinsicID(StringRef Name);
 
   /// Return the attributes for an intrinsic.
-  AttributeList getAttributes(LLVMContext &C, ID id);
+  AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT);
 
   /// Return the function attributes for an intrinsic.
   AttributeSet getFnAttributes(LLVMContext &C, ID id);
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index d10b07ccd91c2..a174ccbf61002 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -134,6 +134,14 @@ class ReadNone<ArgIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
+// The return value or argument is in the range [lower, upper),
+// where lower and upper are interpreted as signed integers.
+class Range<AttrIndex idx, int lower, int upper> : IntrinsicProperty {
+  int ArgNo = idx.Value;
+  int Lower = lower;
+  int Upper = upper;
+}
+
 def IntrNoReturn : IntrinsicProperty;
 
 // Applied by default.
@@ -1620,10 +1628,10 @@ def int_umin : DefaultAttrsIntrinsic<
     [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
 def int_scmp : DefaultAttrsIntrinsic<
     [llvm_anyint_ty], [llvm_anyint_ty, LLVMMatchType<1>],
-    [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
+    [IntrNoMem, IntrSpeculatable, IntrWillReturn, Range<RetIndex, -1, 2>]>;
 def int_ucmp : DefaultAttrsIntrinsic<
     [llvm_anyint_ty], [llvm_anyint_ty, LLVMMatchType<1>],
-    [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
+    [IntrNoMem, IntrSpeculatable, IntrWillReturn, Range<RetIndex, -1, 2>]>;
 
 //===------------------------- Memory Use Markers -------------------------===//
 //
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 0b329d91c3c7c..699191ceb8d4d 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1519,8 +1519,13 @@ bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
   // Upgrade intrinsic attributes.  This does not change the function.
   if (NewFn)
     F = NewFn;
-  if (Intrinsic::ID id = F->getIntrinsicID())
-    F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
+  if (Intrinsic::ID id = F->getIntrinsicID()) {
+    // Only do this if the intrinsic signature is valid.
+    SmallVector<Type *> OverloadTys;
+    if (Intrinsic::getIntrinsicSignature(id, F->getFunctionType(), OverloadTys))
+      F->setAttributes(
+          Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
+  }
   return Upgraded;
 }
 
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 3644fab913b10..ce0f71046e822 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -514,8 +514,15 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace,
   // Ensure intrinsics have the right parameter attributes.
   // Note, the IntID field will have been set in Value::setName if this function
   // name is a valid intrinsic ID.
-  if (IntID)
-    setAttributes(Intrinsic::getAttributes(getContext(), IntID));
+  if (IntID) {
+    // Don't set the attributes if the intrinsic signature is invalid. This
+    // case will either be auto-upgraded or fail verification.
+    SmallVector<Type *> OverloadTys;
+    if (!Intrinsic::getIntrinsicSignature(IntID, Ty, OverloadTys))
+      return;
+
+    setAttributes(Intrinsic::getAttributes(getContext(), IntID, Ty));
+  }
 }
 
 Function::~Function() {
diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp
index a0375c6508ec9..b3e8adb66ed6f 100644
--- a/llvm/lib/IR/Intrinsics.cpp
+++ b/llvm/lib/IR/Intrinsics.cpp
@@ -13,6 +13,7 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringTable.h"
+#include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index bd95bcd89e183..985af4a86009f 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16506,7 +16506,8 @@ Align SITargetLowering::computeKnownAlignForTargetInstr(
     // site specifies a lower alignment?
     Intrinsic::ID IID = GI->getIntrinsicID();
     LLVMContext &Ctx = VT.getMachineFunction().getFunction().getContext();
-    AttributeList Attrs = Intrinsic::getAttributes(Ctx, IID);
+    AttributeList Attrs =
+        Intrinsic::getAttributes(Ctx, IID, Intrinsic::getType(Ctx, IID));
     if (MaybeAlign RetAlign = Attrs.getRetAlignment())
       return *RetAlign;
   }
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index f98a693804645..329bd45902242 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -2903,7 +2903,7 @@ static void stripNonValidAttributesFromPrototype(Function &F) {
   // assumes that the attributes defined in Intrinsic.td are conservatively
   // correct for both physical and abstract model.
   if (Intrinsic::ID id = F.getIntrinsicID()) {
-    F.setAttributes(Intrinsic::getAttributes(Ctx, id));
+    F.setAttributes(Intrinsic::getAttributes(Ctx, id, F.getFunctionType()));
     return;
   }
 
diff --git a/llvm/test/Assembler/aarch64-intrinsics-attributes.ll b/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
index 0d0111216e8f9..31b7101fba116 100644
--- a/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
+++ b/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
@@ -10,8 +10,8 @@ declare i64 @llvm.aarch64.ldxr.p0(ptr)
 ; CHECK: declare i32 @llvm.aarch64.stxp(i64, i64, ptr) [[NOFREE_NOUNWIND_WILLRETURN]]
 declare i32 @llvm.aarch64.stxp(i64, i64, ptr)
 
-; CHECK: declare i32 @llvm.aarch64.dsb(i32) [[NOFREE_NOUNWIND_WILLRETURN]]
-declare i32 @llvm.aarch64.dsb(i32)
+; CHECK: declare void @llvm.aarch64.dsb(i32) [[NOFREE_NOUNWIND_WILLRETURN]]
+declare void @llvm.aarch64.dsb(i32)
 
 ; CHECK: declare i64 @llvm.aarch64.neon.sqdmulls.scalar(i32, i32) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_READNONE_WILLRETURN:#[0-9]+]]
 declare i64 @llvm.aarch64.neon.sqdmulls.scalar(i32, i32)
diff --git a/llvm/test/Assembler/autoupgrade-invalid-mem-intrinsics.ll b/llvm/test/Assembler/autoupgrade-invalid-mem-intrinsics.ll
index e9bd4f0e9661b..0ea683fb44dce 100644
--- a/llvm/test/Assembler/autoupgrade-invalid-mem-intrinsics.ll
+++ b/llvm/test/Assembler/autoupgrade-invalid-mem-intrinsics.ll
@@ -1,15 +1,13 @@
-; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
 
 ; Check that remangling code doesn't fail on an intrinsic with wrong signature
+; TODO: This should probably produce an error.
 
-; CHECK: Attribute after last parameter!
-; CHECK-NEXT: ptr @llvm.memset.i64
+; CHECK: declare void @llvm.memset.i64
 declare void @llvm.memset.i64(ptr nocapture, i8, i64) nounwind
 
-; CHECK: Attribute after last parameter!
-; CHECK-NEXT: ptr @llvm.memcpy.i64
+; CHECK: declare void @llvm.memcpy.i64
 declare void @llvm.memcpy.i64(ptr nocapture, i8, i64) nounwind
 
-; CHECK: Attribute after last parameter!
-; CHECK-NEXT: ptr @llvm.memmove.i64
+; CHECK: declare void @llvm.memmove.i64
 declare void @llvm.memmove.i64(ptr nocapture, i8, i64) nounwind
diff --git a/llvm/test/TableGen/intrinsic-attrs.td b/llvm/test/TableGen/intrinsic-attrs.td
index 689351a79237a..18309d7419994 100644
--- a/llvm/test/TableGen/intrinsic-attrs.td
+++ b/llvm/test/TableGen/intrinsic-attrs.td
@@ -6,8 +6,10 @@ def int_random_gen   : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrHasSideEffec
 
 def int_deref_ptr_ret : Intrinsic<[llvm_ptr_ty], [], [Dereferenceable<RetIndex, 16>]>;
 
-// CHECK: static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) {
-// CHECK-NEXT:   switch (ID) {
+// CHECK: static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID,
+// CHECK-NEXT: Type *ArgType) {
+// CHECK-NEXT: unsigned BitWidth = ArgType->getScalarSizeInBits();
+// CHECK-NEXT: switch (ID) {
 // CHECK-NEXT: default: llvm_unreachable("Invalid attribute set number");
 // CHECK-NEXT: case 0:
 // CHECK-NEXT:     return AttributeSet::get(C, {
@@ -26,13 +28,14 @@ def int_deref_ptr_ret : Intrinsic<[llvm_ptr_ty], [], [Dereferenceable<RetIndex,
 // CHECK: 0 << 8 | 0, // llvm.deref.ptr.ret
 // CHECK: 1 << 8 | 1, // llvm.random.gen
 
-// CHECK: getAttributes(LLVMContext &C, ID id)
+// CHECK: getAttributes(LLVMContext &C, ID id,
+// CHECK-NEXT: FunctionType *FT) {
 // CHECK: case 1:
 // CHECK-NEXT: return AttributeList::get(C, {
 // CHECK-NEXT:   {AttributeList::FunctionIndex, getIntrinsicFnAttributeSet(C, FnAttrID)}
 // CHECK-NEXT: });
 // CHECK-NEXT: case 0:
 // CHECK-NEXT: return AttributeList::get(C, {
-// CHECK-NEXT:   {0, getIntrinsicArgAttributeSet(C, 0)},
+// CHECK-NEXT:   {0, getIntrinsicArgAttributeSet(C, 0, FT->getContainedType(0))},
 // CHECK-NEXT:   {AttributeList::FunctionIndex, getIntrinsicFnAttributeSet(C, FnAttrID)}
 // CHECK-NEXT: });
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/uscmp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/uscmp.ll
index a4fb9e7f5369a..72012e4c09db8 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/uscmp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/uscmp.ll
@@ -261,11 +261,9 @@ define i8 @ucmp_switch(i32 %x, i32 %y) {
 ; CHECK-LABEL: @ucmp_switch(
 ; CHECK-NEXT:    [[CMP:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
 ; CHECK-NEXT:    switch i8 [[CMP]], label [[DEFAULT_UNREACHABLE:%.*]] [
-; CHECK-NEXT:      i8 -2, label [[BB_NEG2:%.*]]
+; CHECK-NEXT:      i8 1, label [[BB_1:%.*]]
 ; CHECK-NEXT:      i8 -1, label [[BB_NEG1:%.*]]
 ; CHECK-NEXT:      i8 0, label [[BB_0:%.*]]
-; CHECK-NEXT:      i8 1, label [[BB_1:%.*]]
-; CHECK-NEXT:      i8 2, label [[BB_2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       bb.neg2:
 ; CHECK-NEXT:    ret i8 -2
@@ -277,6 +275,8 @@ define i8 @ucmp_switch(i32 %x, i32 %y) {
 ; CHECK-NEXT:    ret i8 1
 ; CHECK:       bb.2:
 ; CHECK-NEXT:    ret i8 2
+; CHECK:       default.unreachable:
+; CHECK-NEXT:    unreachable
 ; CHECK:       default:
 ; CHECK-NEXT:    ret i8 123
 ;
@@ -312,11 +312,9 @@ define i8 @scmp_switch(i32 %x, i32 %y) {
 ; CHECK-LABEL: @scmp_switch(
 ; CHECK-NEXT:    [[CMP:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
 ; CHECK-NEXT:    switch i8 [[CMP]], label [[DEFAULT_UNREACHABLE:%.*]] [
-; CHECK-NEXT:      i8 -2, label [[BB_NEG2:%.*]]
+; CHECK-NEXT:      i8 1, label [[BB_1:%.*]]
 ; CHECK-NEXT:      i8 -1, label [[BB_NEG1:%.*]]
 ; CHECK-NEXT:      i8 0, label [[BB_0:%.*]]
-; CHECK-NEXT:      i8 1, label [[BB_1:%.*]]
-; CHECK-NEXT:      i8 2, label [[BB_2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       bb.neg2:
 ; CHECK-NEXT:    ret i8 -2
@@ -328,6 +326,8 @@ define i8 @scmp_switch(i32 %x, i32 %y) {
 ; CHECK-NEXT:    ret i8 1
 ; CHECK:       bb.2:
 ; CHECK-NEXT:    ret i8 2
+; CHECK:       default.unreachable:
+; CHECK-NEXT:    unreachable
 ; CHECK:       default:
 ; CHECK-NEXT:    ret i8 123
 ;
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
index 0846f66ea6452..24dc3272d8071 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
@@ -439,6 +439,11 @@ void CodeGenIntrinsic::setProperty(const Record *R) {
     unsigned ArgNo = R->getValueAsInt("ArgNo");
     uint64_t Bytes = R->getValueAsInt("Bytes");
     addArgAttribute(ArgNo, Dereferenceable, Bytes);
+  } else if (R->isSubClassOf("Range")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    int64_t Lower = R->getValueAsInt("Lower");
+    int64_t Upper = R->getValueAsInt("Upper");
+    addArgAttribute(ArgNo, Range, Lower, Upper);
   } else
     llvm_unreachable("Unknown property!");
 }
@@ -455,14 +460,14 @@ bool CodeGenIntrinsic::isParamImmArg(unsigned ParamIdx) const {
   ++ParamIdx;
   if (ParamIdx >= ArgumentAttributes.size())
     return false;
-  ArgAttribute Val{ImmArg, 0};
+  ArgAttribute Val{ImmArg, 0, 0};
   return std::binary_search(ArgumentAttributes[ParamIdx].begin(),
                             ArgumentAttributes[ParamIdx].end(), Val);
 }
 
-void CodeGenIntrinsic::addArgAttribute(unsigned Idx, ArgAttrKind AK,
-                                       uint64_t V) {
+void CodeGenIntrinsic::addArgAttribute(unsigned Idx, ArgAttrKind AK, uint64_t V,
+                                       uint64_t V2) {
   if (Idx >= ArgumentAttributes.size())
     ArgumentAttributes.resize(Idx + 1);
-  ArgumentAttributes[Idx].emplace_back(AK, V);
+  ArgumentAttributes[Idx].emplace_back(AK, V, V2);
 }
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
index 8428d09a94009..676f575b2749d 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
@@ -128,24 +128,29 @@ struct CodeGenIntrinsic {
     ReadNone,
     ImmArg,
     Alignment,
-    Dereferenceable
+    Dereferenceable,
+    Range,
   };
 
   struct ArgAttribute {
     ArgAttrKind Kind;
     uint64_t Value;
+    uint64_t Value2;
 
-    ArgAttribute(ArgAttrKind K, uint64_t V) : Kind(K), Value(V) {}
+    ArgAttribute(ArgAttrKind K, uint64_t V, uint64_t V2)
+        : Kind(K), Value(V), Value2(V2) {}
 
     bool operator<(const ArgAttribute &Other) const {
-      return std::tie(Kind, Value) < std::tie(Other.Kind, Other.Value);
+      return std::tie(Kind, Value, Value2) <
+             std::tie(Other.Kind, Other.Value, Other.Value2);
     }
   };
 
   /// Vector of attributes for each argument.
   SmallVector<SmallVector<ArgAttribute, 0>> ArgumentAttributes;
 
-  void addArgAttribute(unsigned Idx, ArgAttrKind AK, uint64_t V = 0);
+  void addArgAttribute(unsigned Idx, ArgAttrKind AK, uint64_t V = 0,
+                       uint64_t V2 = 0);
 
   bool hasProperty(enum SDNP Prop) const { return Properties & (1 << Prop); }
 
diff --git a/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp b/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
index 2497a5bf8529f..e7fc5250bf9d8 100644
--- a/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
@@ -493,6 +493,8 @@ static StringRef getArgAttrEnumName(CodeGenIntrinsic::ArgAttrKind Kind) {
     return "Alignment";
   case CodeGenIntrinsic::Dereferenceable:
     return "Dereferenceable";
+  case CodeGenIntrinsic::Range:
+    return "Range";
   }
   llvm_unreachable("Unknown CodeGenIntrinsic::ArgAttrKind enum");
 }
@@ -502,7 +504,9 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
                                       raw_ostream &OS) {
   OS << R"(// Add parameter attributes that are not common to all intrinsics.
 #ifdef GET_INTRINSIC_ATTRIBUTES
-static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) {
+static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID,
+                                                Type *ArgType) {
+  unsigned BitWidth = ArgType->getScalarSizeInBits();
   switch (ID) {
   default: llvm_unreachable("Invalid attribute set number");)";
   // Compute unique argument attribute sets.
@@ -535,6 +539,17 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) {
             Attr.Kind == CodeGenIntrinsic::Dereferenceable)
           OS << formatv("      Attribute::get(C, Attribute::{}, {}),\n",
                         AttrName, Attr.Value);
+        else if (Attr.Kind == CodeGenIntrinsic::Range)
+          // This allows implicitTrunc because the range may only fit the
+          // type based on rules implemented in the IR verifier. E.g. the
+          // [-1, 1] range for ucmp/scmp intrinsics requires a minimum i2 type.
+          // Give the verifier a chance to diagnose this instead of asserting
+          // here.
+          OS << formatv("      Attribute::get(C, Attribute::{}, "
+                        "ConstantRange(APInt(BitWidth, {}, /*isSigned=*/true, "
+                        "/*implicitTrunc=*/true), APInt(BitWidth, {}, "
+                        "/*isSigned=*/true, /*implicitTrunc=*/true))),\n",
+                        AttrName, (int64_t)Attr.Value, (int64_t)Attr.Value2);
         else
           OS << formatv("      Attribute::get(C, Attribute::{}),\n", AttrName);
       }
@@ -635,7 +650,8 @@ static constexpr uint16_t IntrinsicsToAttributesMap[] = {)";
   OS << R"(
 };
 
-AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {)";
+AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id,
+                                       FunctionType *FT) {)";
 
   OS << formatv(R"(
   if (id == 0)
@@ -669,8 +685,9 @@ AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {)";
 
       unsigned ArgAttrID = UniqArgAttributes.find(Attrs)->second;
       OS << LS
-         << formatv("      {{{}, getIntrinsicArgAttributeSet(C, {})}", AttrIdx,
-                    ArgAttrID);
+         << formatv("      {{{}, getIntrinsicArgAttributeSet(C, {}, "
+                    "FT->getContainedType({}))}",
+                    AttrIdx, ArgAttrID, AttrIdx);
     }
 
     if (hasFnAttributes(Int)) {

From 35f4cdbf59fca82b97869cce7e9e5d5009144938 Mon Sep 17 00:00:00 2001
From: Prakhar Dixit <dixitprakhar11@gmail.com>
Date: Thu, 17 Apr 2025 14:47:34 +0530
Subject: [PATCH 245/710] [mlir][arith] Add constraints to the MulIOp for
 preventing type mismatch while folding (#136093)

Fixes #135289
The original version didn't check if the types of lhs, rhs, and the
result matched, which could cause type errors.
This fix adds type checks to make sure the constants attributes have
the same type as the SSA values before applying the simplification.
---
 .../Dialect/Arith/IR/ArithCanonicalization.td  |  4 +++-
 mlir/test/Dialect/Arith/canonicalize.mlir      | 18 ++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
index 6d7ac2be951dd..7e212df9029d1 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
+++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
@@ -90,7 +90,9 @@ def MulIMulIConstant :
           (Arith_MulIOp $x, (ConstantLikeMatcher APIntAttr:$c0), $ovf1),
           (ConstantLikeMatcher APIntAttr:$c1), $ovf2),
         (Arith_MulIOp $x, (Arith_ConstantOp (MulIntAttrs $res, $c0, $c1)),
-            (MergeOverflow $ovf1, $ovf2))>;
+            (MergeOverflow $ovf1, $ovf2)),
+             [(Constraint<CPred<"$0.getType() == cast<IntegerAttr>($1).getType()">> $res, $c0),
+              (Constraint<CPred<"$0.getType() == cast<IntegerAttr>($1).getType()">> $res, $c1)]>;
 
 //===----------------------------------------------------------------------===//
 // AddUIExtendedOp
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index f0b2731707d18..d62c5b18fd041 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -1234,6 +1234,24 @@ func.func @doubleAddSub2(%arg0: index, %arg1 : index) -> index {
   return %add : index
 }
 
+// Negative test case to ensure no further folding is performed when there's a type mismatch between the values and the result.
+// CHECK-LABEL:   func.func @nested_muli() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = "test.constant"() <{value = 2147483647 : i64}> : () -> i32
+// CHECK:           %[[VAL_1:.*]] = "test.constant"() <{value = -2147483648 : i64}> : () -> i32
+// CHECK:           %[[VAL_2:.*]] = "test.constant"() <{value = 2147483648 : i64}> : () -> i32
+// CHECK:           %[[VAL_3:.*]] = arith.muli %[[VAL_0]], %[[VAL_1]] : i32
+// CHECK:           %[[VAL_4:.*]] = arith.muli %[[VAL_3]], %[[VAL_2]] : i32
+// CHECK:           return %[[VAL_4]] : i32
+// CHECK:         }
+func.func @nested_muli() -> (i32) {
+  %0 = "test.constant"() {value = 0x7fffffff} : () -> i32
+  %1 = "test.constant"() {value = -2147483648} : () -> i32
+  %2 = "test.constant"() {value = 0x80000000} : () -> i32
+  %4 = arith.muli %0, %1 : i32
+  %5 = arith.muli %4, %2 : i32
+  return %5 : i32
+}
+
 // CHECK-LABEL: @tripleMulIMulIIndex
 //       CHECK:   %[[cres:.+]] = arith.constant 15 : index
 //       CHECK:   %[[muli:.+]] = arith.muli %arg0, %[[cres]] : index

From 7e5317139d5a96837ede208635daf031ab53eaee Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Thu, 17 Apr 2025 17:52:49 +0800
Subject: [PATCH 246/710] [PowerPC] Pre-commit tests for PR130742. NFC.
 (#135606)

Needed by https://github.com/llvm/llvm-project/pull/130742.
---
 .../PowerPC/loop-instr-form-non-inc.ll        |  15 +--
 llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll   |  80 +++++++------
 llvm/test/CodeGen/PowerPC/sms-cpy-1.ll        | 107 +++++++++---------
 llvm/test/CodeGen/PowerPC/sms-phi.ll          |   4 +-
 .../PowerPC/p10-respect-unroll-pragma.ll      | 103 +++++++++--------
 5 files changed, 160 insertions(+), 149 deletions(-)

diff --git a/llvm/test/CodeGen/PowerPC/loop-instr-form-non-inc.ll b/llvm/test/CodeGen/PowerPC/loop-instr-form-non-inc.ll
index e87d6392c4c7b..287376dc319e8 100644
--- a/llvm/test/CodeGen/PowerPC/loop-instr-form-non-inc.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-instr-form-non-inc.ll
@@ -2,21 +2,22 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr9 < %s | FileCheck %s
 
-define dso_local void @test_no_inc(i32 signext %a) local_unnamed_addr nounwind align 2 {
+define dso_local void @test_no_inc(i32 signext %a, ptr %p) local_unnamed_addr nounwind align 2 {
 ; CHECK-LABEL: test_no_inc:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    srawi 4, 3, 31
+; CHECK-NEXT:    srawi 5, 3, 31
 ; CHECK-NEXT:    cmpwi 3, 0
 ; CHECK-NEXT:    li 6, 1
 ; CHECK-NEXT:    li 7, 0
-; CHECK-NEXT:    andc 4, 3, 4
-; CHECK-NEXT:    addi 5, 4, 1
+; CHECK-NEXT:    andc 5, 3, 5
+; CHECK-NEXT:    add 4, 5, 4
+; CHECK-NEXT:    addi 4, 4, 1
 ; CHECK-NEXT:    b .LBB0_2
 ; CHECK-NEXT:    .p2align 5
 ; CHECK-NEXT:  .LBB0_1: # %for.cond.cleanup
 ; CHECK-NEXT:    #
-; CHECK-NEXT:    stb 7, 0(5)
-; CHECK-NEXT:    add 5, 5, 4
+; CHECK-NEXT:    stb 7, 0(4)
+; CHECK-NEXT:    add 4, 4, 5
 ; CHECK-NEXT:  .LBB0_2: # %for.cond
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    bc 4, 1, .LBB0_1
@@ -38,7 +39,7 @@ for.body.preheader:                               ; preds = %for.cond
 
 for.cond.cleanup:                                 ; preds = %for.body.preheader, %for.cond
   %g.1.lcssa = phi i32 [ %g.0, %for.cond ], [ %0, %for.body.preheader ]
-  %arrayidx5 = getelementptr inbounds i8, ptr null, i32 %g.1.lcssa
+  %arrayidx5 = getelementptr inbounds i8, ptr %p, i32 %g.1.lcssa
   store i8 0, ptr %arrayidx5, align 1
   br label %for.cond
 }
diff --git a/llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll b/llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll
index f13ba7765e398..e99ea59bfacac 100644
--- a/llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-spill-crgt.ll
@@ -15,7 +15,7 @@
 ; bit of any CR field is spilled. We need to test the spilling of a CR bit
 ; other than the LT bit. Hence this test case is rather complex.
 
-define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
+define dso_local fastcc void @P10_Spill_CR_GT(ptr %p) unnamed_addr {
 ; CHECK-LABEL: P10_Spill_CR_GT:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    mfcr r12
@@ -25,32 +25,35 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-NEXT:    stdu r1, -64(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r28, -32
 ; CHECK-NEXT:    .cfi_offset r29, -24
 ; CHECK-NEXT:    .cfi_offset r30, -16
 ; CHECK-NEXT:    .cfi_offset cr2, 8
 ; CHECK-NEXT:    .cfi_offset cr3, 8
 ; CHECK-NEXT:    .cfi_offset cr4, 8
-; CHECK-NEXT:    lwz r3, 0(r3)
-; CHECK-NEXT:    std r29, 40(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    lwz r4, 0(r3)
 ; CHECK-NEXT:    std r30, 48(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    addi r30, r3, -1
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    std r28, 32(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r29, 40(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    crxor 4*cr2+eq, 4*cr2+eq, 4*cr2+eq
-; CHECK-NEXT:    paddi r29, 0, .LJTI0_0@PCREL, 1
-; CHECK-NEXT:    srwi r4, r3, 4
-; CHECK-NEXT:    srwi r3, r3, 5
+; CHECK-NEXT:    paddi r28, 0, .LJTI0_0@PCREL, 1
+; CHECK-NEXT:    sldi r29, r3, 2
+; CHECK-NEXT:    srwi r5, r4, 4
+; CHECK-NEXT:    srwi r4, r4, 5
+; CHECK-NEXT:    andi. r5, r5, 1
+; CHECK-NEXT:    crmove 4*cr2+gt, gt
 ; CHECK-NEXT:    andi. r4, r4, 1
 ; CHECK-NEXT:    li r4, 0
-; CHECK-NEXT:    crmove 4*cr2+gt, gt
-; CHECK-NEXT:    andi. r3, r3, 1
-; CHECK-NEXT:    li r3, 0
 ; CHECK-NEXT:    crmove 4*cr2+lt, gt
-; CHECK-NEXT:    sldi r30, r3, 2
 ; CHECK-NEXT:    b .LBB0_2
 ; CHECK-NEXT:  .LBB0_1: # %bb43
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    bl call_1@notoc
-; CHECK-NEXT:    setnbc r3, 4*cr3+eq
-; CHECK-NEXT:    li r4, 0
-; CHECK-NEXT:    stb r4, 0(r3)
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    isel r4, r30, r3, 4*cr3+eq
+; CHECK-NEXT:    stb r3, 0(r4)
 ; CHECK-NEXT:    li r4, 0
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %bb5
@@ -65,8 +68,8 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-NEXT:    lwz r5, 0(r3)
 ; CHECK-NEXT:    rlwinm r4, r5, 0, 21, 22
 ; CHECK-NEXT:    cmpwi cr3, r4, 512
-; CHECK-NEXT:    lwax r4, r29, r30
-; CHECK-NEXT:    add r4, r29, r4
+; CHECK-NEXT:    lwax r4, r28, r29
+; CHECK-NEXT:    add r4, r28, r4
 ; CHECK-NEXT:    mtctr r4
 ; CHECK-NEXT:    li r4, 0
 ; CHECK-NEXT:    bctr
@@ -177,6 +180,7 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-NEXT:  .LBB0_31: # %bb9
 ; CHECK-NEXT:    ld r30, 48(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 40(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r28, 32(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 64
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    lwz r12, 8(r1)
@@ -187,10 +191,10 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-NEXT:    blr
 ; CHECK-NEXT:  .LBB0_32: # %bb29
 ; CHECK-NEXT:    crmove eq, 4*cr3+eq
+; CHECK-NEXT:    li r29, 0
 ; CHECK-NEXT:    cmpwi cr3, r5, 366
 ; CHECK-NEXT:    cmpwi cr4, r3, 0
-; CHECK-NEXT:    li r29, 0
-; CHECK-NEXT:    setnbc r30, eq
+; CHECK-NEXT:    iseleq r30, r30, r29
 ; CHECK-NEXT:    bc 12, 4*cr2+lt, .LBB0_36
 ; CHECK-NEXT:    .p2align 5
 ; CHECK-NEXT:  .LBB0_33: # %bb36
@@ -216,34 +220,37 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-BE-NEXT:    stdu r1, -144(r1)
 ; CHECK-BE-NEXT:    .cfi_def_cfa_offset 144
 ; CHECK-BE-NEXT:    .cfi_offset lr, 16
+; CHECK-BE-NEXT:    .cfi_offset r28, -32
 ; CHECK-BE-NEXT:    .cfi_offset r29, -24
 ; CHECK-BE-NEXT:    .cfi_offset r30, -16
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
-; CHECK-BE-NEXT:    lwz r3, 0(r3)
-; CHECK-BE-NEXT:    std r29, 120(r1) # 8-byte Folded Spill
+; CHECK-BE-NEXT:    lwz r4, 0(r3)
 ; CHECK-BE-NEXT:    std r30, 128(r1) # 8-byte Folded Spill
+; CHECK-BE-NEXT:    addi r30, r3, -1
+; CHECK-BE-NEXT:    li r3, 0
+; CHECK-BE-NEXT:    std r28, 112(r1) # 8-byte Folded Spill
+; CHECK-BE-NEXT:    std r29, 120(r1) # 8-byte Folded Spill
 ; CHECK-BE-NEXT:    crxor 4*cr2+eq, 4*cr2+eq, 4*cr2+eq
-; CHECK-BE-NEXT:    srwi r4, r3, 4
-; CHECK-BE-NEXT:    srwi r3, r3, 5
+; CHECK-BE-NEXT:    sldi r29, r3, 2
+; CHECK-BE-NEXT:    addis r3, r2, .LC0@toc@ha
+; CHECK-BE-NEXT:    ld r28, .LC0@toc@l(r3)
+; CHECK-BE-NEXT:    srwi r5, r4, 4
+; CHECK-BE-NEXT:    srwi r4, r4, 5
+; CHECK-BE-NEXT:    andi. r5, r5, 1
+; CHECK-BE-NEXT:    crmove 4*cr2+gt, gt
 ; CHECK-BE-NEXT:    andi. r4, r4, 1
 ; CHECK-BE-NEXT:    li r4, 0
-; CHECK-BE-NEXT:    crmove 4*cr2+gt, gt
-; CHECK-BE-NEXT:    andi. r3, r3, 1
-; CHECK-BE-NEXT:    li r3, 0
 ; CHECK-BE-NEXT:    crmove 4*cr2+lt, gt
-; CHECK-BE-NEXT:    sldi r30, r3, 2
-; CHECK-BE-NEXT:    addis r3, r2, .LC0@toc@ha
-; CHECK-BE-NEXT:    ld r29, .LC0@toc@l(r3)
 ; CHECK-BE-NEXT:    b .LBB0_2
 ; CHECK-BE-NEXT:  .LBB0_1: # %bb43
 ; CHECK-BE-NEXT:    #
 ; CHECK-BE-NEXT:    bl call_1
 ; CHECK-BE-NEXT:    nop
-; CHECK-BE-NEXT:    setnbc r3, 4*cr3+eq
-; CHECK-BE-NEXT:    li r4, 0
-; CHECK-BE-NEXT:    stb r4, 0(r3)
+; CHECK-BE-NEXT:    li r3, 0
+; CHECK-BE-NEXT:    isel r4, r30, r3, 4*cr3+eq
+; CHECK-BE-NEXT:    stb r3, 0(r4)
 ; CHECK-BE-NEXT:    li r4, 0
 ; CHECK-BE-NEXT:    .p2align 4
 ; CHECK-BE-NEXT:  .LBB0_2: # %bb5
@@ -258,8 +265,8 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-BE-NEXT:    lwz r5, 0(r3)
 ; CHECK-BE-NEXT:    rlwinm r4, r5, 0, 21, 22
 ; CHECK-BE-NEXT:    cmpwi cr3, r4, 512
-; CHECK-BE-NEXT:    lwax r4, r29, r30
-; CHECK-BE-NEXT:    add r4, r29, r4
+; CHECK-BE-NEXT:    lwax r4, r28, r29
+; CHECK-BE-NEXT:    add r4, r28, r4
 ; CHECK-BE-NEXT:    mtctr r4
 ; CHECK-BE-NEXT:    li r4, 0
 ; CHECK-BE-NEXT:    bctr
@@ -370,6 +377,7 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-BE-NEXT:  .LBB0_31: # %bb9
 ; CHECK-BE-NEXT:    ld r30, 128(r1) # 8-byte Folded Reload
 ; CHECK-BE-NEXT:    ld r29, 120(r1) # 8-byte Folded Reload
+; CHECK-BE-NEXT:    ld r28, 112(r1) # 8-byte Folded Reload
 ; CHECK-BE-NEXT:    addi r1, r1, 144
 ; CHECK-BE-NEXT:    ld r0, 16(r1)
 ; CHECK-BE-NEXT:    lwz r12, 8(r1)
@@ -380,10 +388,10 @@ define dso_local fastcc void @P10_Spill_CR_GT() unnamed_addr {
 ; CHECK-BE-NEXT:    blr
 ; CHECK-BE-NEXT:  .LBB0_32: # %bb29
 ; CHECK-BE-NEXT:    crmove eq, 4*cr3+eq
+; CHECK-BE-NEXT:    li r29, 0
 ; CHECK-BE-NEXT:    cmpwi cr3, r5, 366
 ; CHECK-BE-NEXT:    cmpwi cr4, r3, 0
-; CHECK-BE-NEXT:    li r29, 0
-; CHECK-BE-NEXT:    setnbc r30, eq
+; CHECK-BE-NEXT:    iseleq r30, r30, r29
 ; CHECK-BE-NEXT:    bc 12, 4*cr2+lt, .LBB0_36
 ; CHECK-BE-NEXT:    .p2align 4
 ; CHECK-BE-NEXT:  .LBB0_33: # %bb36
@@ -528,7 +536,7 @@ bb32:                                             ; preds = %bb40, %bb29
   br i1 %tmp7, label %bb33, label %bb36
 
 bb33:                                             ; preds = %bb32
-  %tmp34 = getelementptr inbounds i8, ptr null, i64 -1
+  %tmp34 = getelementptr inbounds i8, ptr %p, i64 -1
   %tmp35 = select i1 %tmp12, ptr %tmp34, ptr null
   store i8 0, ptr %tmp35, align 1
   br label %bb36
@@ -558,7 +566,7 @@ bb42:                                             ; preds = %bb42, %bb41
 
 bb43:                                             ; preds = %bb10, %bb10
   call void @call_1()
-  %tmp44 = getelementptr inbounds i8, ptr null, i64 -1
+  %tmp44 = getelementptr inbounds i8, ptr %p, i64 -1
   %tmp45 = select i1 %tmp12, ptr %tmp44, ptr null
   store i8 0, ptr %tmp45, align 1
   br label %bb63
diff --git a/llvm/test/CodeGen/PowerPC/sms-cpy-1.ll b/llvm/test/CodeGen/PowerPC/sms-cpy-1.ll
index 334379cda07ec..9bf619983ee78 100644
--- a/llvm/test/CodeGen/PowerPC/sms-cpy-1.ll
+++ b/llvm/test/CodeGen/PowerPC/sms-cpy-1.ll
@@ -4,78 +4,77 @@
 
 @.str.28 = external unnamed_addr constant [69 x i8], align 1
 
-define void @print_res() nounwind {
+define void @print_res(ptr %p) nounwind {
 ; CHECK-LABEL: print_res:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lwz 3, 0(3)
-; CHECK-NEXT:    mflr 0
-; CHECK-NEXT:    addi 3, 3, -1
-; CHECK-NEXT:    clrldi 4, 3, 32
-; CHECK-NEXT:    cmplwi 3, 3
-; CHECK-NEXT:    li 3, 3
-; CHECK-NEXT:    isellt 3, 4, 3
-; CHECK-NEXT:    li 4, 1
-; CHECK-NEXT:    cmpldi 3, 1
-; CHECK-NEXT:    iselgt 3, 3, 4
-; CHECK-NEXT:    li 4, 0
-; CHECK-NEXT:    mtctr 3
-; CHECK-NEXT:    stdu 1, -128(1)
+; CHECK-NEXT:    lwz 4, 0(3)
+; CHECK-NEXT:    addi 4, 4, -1
+; CHECK-NEXT:    clrldi 5, 4, 32
+; CHECK-NEXT:    cmplwi 4, 3
+; CHECK-NEXT:    li 4, 3
+; CHECK-NEXT:    isellt 4, 5, 4
+; CHECK-NEXT:    li 5, 1
+; CHECK-NEXT:    cmpldi 4, 1
+; CHECK-NEXT:    iselgt 4, 4, 5
 ; CHECK-NEXT:    li 5, 0
-; CHECK-NEXT:    std 0, 144(1)
-; CHECK-NEXT:    li 3, 1
-; CHECK-NEXT:    li 7, -1
-; CHECK-NEXT:    lbz 5, 0(5)
+; CHECK-NEXT:    mtctr 4
+; CHECK-NEXT:    li 8, -1
+; CHECK-NEXT:    lbz 6, 0(3)
+; CHECK-NEXT:    li 4, 1
 ; CHECK-NEXT:    bdz .LBB0_6
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    xori 6, 5, 84
-; CHECK-NEXT:    clrldi 5, 7, 32
-; CHECK-NEXT:    addi 3, 3, 1
-; CHECK-NEXT:    addi 8, 7, -1
-; CHECK-NEXT:    lbz 5, 0(5)
+; CHECK-NEXT:    xori 7, 6, 84
+; CHECK-NEXT:    clrldi 6, 8, 32
+; CHECK-NEXT:    addi 4, 4, 1
+; CHECK-NEXT:    addi 9, 8, -1
+; CHECK-NEXT:    lbzx 6, 3, 6
 ; CHECK-NEXT:    bdz .LBB0_5
 ; CHECK-NEXT:  # %bb.2:
-; CHECK-NEXT:    cntlzw 6, 6
-; CHECK-NEXT:    addi 3, 3, 1
-; CHECK-NEXT:    srwi 7, 6, 5
-; CHECK-NEXT:    xori 6, 5, 84
-; CHECK-NEXT:    clrldi 5, 8, 32
-; CHECK-NEXT:    addi 8, 8, -1
-; CHECK-NEXT:    lbz 5, 0(5)
+; CHECK-NEXT:    cntlzw 7, 7
+; CHECK-NEXT:    addi 4, 4, 1
+; CHECK-NEXT:    srwi 8, 7, 5
+; CHECK-NEXT:    xori 7, 6, 84
+; CHECK-NEXT:    clrldi 6, 9, 32
+; CHECK-NEXT:    addi 9, 9, -1
+; CHECK-NEXT:    lbzx 6, 3, 6
 ; CHECK-NEXT:    bdz .LBB0_4
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3:
-; CHECK-NEXT:    clrldi 10, 8, 32
-; CHECK-NEXT:    cntlzw 9, 6
-; CHECK-NEXT:    xori 6, 5, 84
-; CHECK-NEXT:    addi 8, 8, -1
-; CHECK-NEXT:    lbz 5, 0(10)
-; CHECK-NEXT:    addi 3, 3, 1
-; CHECK-NEXT:    add 4, 4, 7
-; CHECK-NEXT:    srwi 7, 9, 5
+; CHECK-NEXT:    clrldi 11, 9, 32
+; CHECK-NEXT:    cntlzw 10, 7
+; CHECK-NEXT:    xori 7, 6, 84
+; CHECK-NEXT:    addi 9, 9, -1
+; CHECK-NEXT:    lbzx 6, 3, 11
+; CHECK-NEXT:    addi 4, 4, 1
+; CHECK-NEXT:    add 5, 5, 8
+; CHECK-NEXT:    srwi 8, 10, 5
 ; CHECK-NEXT:    bdnz .LBB0_3
 ; CHECK-NEXT:  .LBB0_4:
-; CHECK-NEXT:    add 4, 4, 7
+; CHECK-NEXT:    add 5, 5, 8
 ; CHECK-NEXT:  .LBB0_5:
-; CHECK-NEXT:    cntlzw 6, 6
-; CHECK-NEXT:    srwi 6, 6, 5
-; CHECK-NEXT:    add 4, 4, 6
+; CHECK-NEXT:    cntlzw 3, 7
+; CHECK-NEXT:    srwi 3, 3, 5
+; CHECK-NEXT:    add 5, 5, 3
 ; CHECK-NEXT:  .LBB0_6:
-; CHECK-NEXT:    xori 5, 5, 84
-; CHECK-NEXT:    clrldi 3, 3, 32
-; CHECK-NEXT:    li 7, 0
-; CHECK-NEXT:    li 8, 3
-; CHECK-NEXT:    std 3, 104(1)
-; CHECK-NEXT:    cntlzw 5, 5
+; CHECK-NEXT:    xori 3, 6, 84
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    cntlzw 3, 3
+; CHECK-NEXT:    srwi 3, 3, 5
+; CHECK-NEXT:    add 3, 5, 3
+; CHECK-NEXT:    stdu 1, -128(1)
+; CHECK-NEXT:    clrldi 6, 3, 32
 ; CHECK-NEXT:    addis 3, 2, .LC0@toc@ha
-; CHECK-NEXT:    li 10, 0
-; CHECK-NEXT:    ld 3, .LC0@toc@l(3)
-; CHECK-NEXT:    srwi 5, 5, 5
-; CHECK-NEXT:    add 4, 4, 5
 ; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    std 0, 144(1)
+; CHECK-NEXT:    ld 3, .LC0@toc@l(3)
 ; CHECK-NEXT:    std 5, 120(1)
 ; CHECK-NEXT:    li 5, 3
-; CHECK-NEXT:    clrldi 6, 4, 32
+; CHECK-NEXT:    clrldi 4, 4, 32
+; CHECK-NEXT:    std 4, 104(1)
 ; CHECK-NEXT:    li 4, 3
+; CHECK-NEXT:    li 7, 0
+; CHECK-NEXT:    li 8, 3
+; CHECK-NEXT:    li 10, 0
 ; CHECK-NEXT:    std 5, 96(1)
 ; CHECK-NEXT:    li 5, 0
 ; CHECK-NEXT:    bl printf
@@ -92,7 +91,7 @@ define void @print_res() nounwind {
   %8 = trunc i64 %6 to i32
   %9 = sub i32 0, %8
   %10 = zext i32 %9 to i64
-  %11 = getelementptr inbounds i8, ptr null, i64 %10
+  %11 = getelementptr inbounds i8, ptr %p, i64 %10
   %12 = load i8, ptr %11, align 1
   %13 = icmp eq i8 %12, 84
   %14 = zext i1 %13 to i32
diff --git a/llvm/test/CodeGen/PowerPC/sms-phi.ll b/llvm/test/CodeGen/PowerPC/sms-phi.ll
index 53a3f13c0597d..5d71423dc781e 100644
--- a/llvm/test/CodeGen/PowerPC/sms-phi.ll
+++ b/llvm/test/CodeGen/PowerPC/sms-phi.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs\
 ; RUN:       -mcpu=pwr9 --ppc-enable-pipeliner -debug-only=pipeliner 2>&1 \
 ; RUN:       >/dev/null | FileCheck %s
-define dso_local void @sha512() #0 {
+define dso_local void @sha512(ptr %p) #0 {
 ;CHECK: prolog:
 ;CHECK:        %{{[0-9]+}}:g8rc = ADD8 %{{[0-9]+}}:g8rc, %{{[0-9]+}}:g8rc
 ;CHECK: epilog:
@@ -15,7 +15,7 @@ define dso_local void @sha512() #0 {
   %2 = phi i64 [ 0, %0 ], [ %12, %1 ]
   %3 = phi i64 [ undef, %0 ], [ %11, %1 ]
   %4 = phi i64 [ undef, %0 ], [ %3, %1 ]
-  %5 = getelementptr inbounds [80 x i64], ptr null, i64 0, i64 %2
+  %5 = getelementptr inbounds [80 x i64], ptr %p, i64 0, i64 %2
   %6 = load i64, ptr %5, align 8
   %7 = add i64 0, %6
   %8 = and i64 %3, %4
diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/p10-respect-unroll-pragma.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/p10-respect-unroll-pragma.ll
index bf9cf59aedd1d..12d4e285cdfc2 100644
--- a/llvm/test/Transforms/LoopUnroll/PowerPC/p10-respect-unroll-pragma.ll
+++ b/llvm/test/Transforms/LoopUnroll/PowerPC/p10-respect-unroll-pragma.ll
@@ -7,104 +7,107 @@
 define dso_local void @test(ptr %arg) #0 {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[I19:%.*]] = getelementptr i8, ptr [[ARG:%.*]], i64 -32
+; CHECK-NEXT:    [[I21:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 48
+; CHECK-NEXT:    [[I21_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 64
 ; CHECK-NEXT:    br label [[BB16:%.*]]
 ; CHECK:       bb16:
-; CHECK-NEXT:    [[I20:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    [[I20:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20]])
 ; CHECK-NEXT:    [[I24_ELT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_1:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_1]])
 ; CHECK-NEXT:    [[I24_ELT_1:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_1]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_1]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_1]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_1:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_1]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_1]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_2:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_1]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_2:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_2]])
 ; CHECK-NEXT:    [[I24_ELT_2:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_2]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_2]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_2]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_2:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_2]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_2]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_3:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_2]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_3:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_3:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_3]])
 ; CHECK-NEXT:    [[I24_ELT_3:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_3]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_3]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_3]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_3:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_3]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_3]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_4:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_3]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_4:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_4:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_4]])
 ; CHECK-NEXT:    [[I24_ELT_4:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_4]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_4]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_4]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_4:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_4]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_4]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_5:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_4]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_5:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_5:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_5]])
 ; CHECK-NEXT:    [[I24_ELT_5:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_5]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_5]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_5]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_5:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_5]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_5]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_6:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_5]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_6:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_6:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_6]])
 ; CHECK-NEXT:    [[I24_ELT_6:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_6]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_6]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_6]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_6:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_6]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_6]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_7:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_6]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_7:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_7:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_7]])
 ; CHECK-NEXT:    [[I24_ELT_7:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_7]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_7]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_7]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_7:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_7]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_7]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_8:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_7]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_8:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_8:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_8]])
 ; CHECK-NEXT:    [[I24_ELT_8:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_8]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_8]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_8]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_8:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_8]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_8]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_9:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_8]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_9:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_9:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_9]])
 ; CHECK-NEXT:    [[I24_ELT_9:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_9]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_9]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_9]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_9:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_9]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_9]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_10:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_9]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_10:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_10:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_10]])
 ; CHECK-NEXT:    [[I24_ELT_10:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_10]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_10]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_10]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_10:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_10]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_10]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_11:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_10]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_11:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_11:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_11]])
 ; CHECK-NEXT:    [[I24_ELT_11:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_11]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_11]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_11]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_11:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_11]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_11]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_12:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_11]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_12:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_12:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_12]])
 ; CHECK-NEXT:    [[I24_ELT_12:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_12]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_12]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_12]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_12:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_12]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_12]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_13:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_12]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_13:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_13:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_13]])
 ; CHECK-NEXT:    [[I24_ELT_13:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_13]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_13]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_13]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_13:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_13]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_13]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_14:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_13]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_14:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_14:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_14]])
 ; CHECK-NEXT:    [[I24_ELT_14:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_14]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_14]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_14]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_14:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_14]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_14]], ptr inttoptr (i64 64 to ptr), align 64
-; CHECK-NEXT:    [[I20_15:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr nonnull inttoptr (i64 -32 to ptr))
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_14]], ptr [[I21_REPACK1]], align 16
+; CHECK-NEXT:    [[I20_15:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[I19]])
 ; CHECK-NEXT:    [[I24_15:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[I20_15]])
 ; CHECK-NEXT:    [[I24_ELT_15:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_15]], 0
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT_15]], ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT_15]], ptr [[I21]], align 16
 ; CHECK-NEXT:    [[I24_ELT1_15:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[I24_15]], 1
-; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_15]], ptr inttoptr (i64 64 to ptr), align 64
+; CHECK-NEXT:    store <16 x i8> [[I24_ELT1_15]], ptr [[I21_REPACK1]], align 16
 ; CHECK-NEXT:    br label [[BB16]], !llvm.loop [[LOOP0:![0-9]+]]
 ;
 bb:
@@ -131,10 +134,10 @@ bb:
 bb16:                                             ; preds = %bb16, %bb
   %i17 = load i64, ptr %i5, align 8
   %i18 = icmp sge i64 %i17, 1
-  %i19 = getelementptr i8, ptr null, i64 -32
+  %i19 = getelementptr i8, ptr %arg, i64 -32
   %i20 = call <256 x i1> @llvm.ppc.vsx.lxvp(ptr %i19)
   store <256 x i1> %i20, ptr %i7, align 32
-  %i21 = getelementptr inbounds i8, ptr null, i64 48
+  %i21 = getelementptr inbounds i8, ptr %arg, i64 48
   %i23 = load <256 x i1>, ptr %i7, align 32
   %i24 = call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> %i23)
   store { <16 x i8>, <16 x i8> } %i24, ptr %i21, align 16

From a09fd9c6536b746deea651b796978285ed161a00 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke@amd.com>
Date: Thu, 17 Apr 2025 15:24:46 +0530
Subject: [PATCH 247/710] [CodeGen][NPM] Port MachineBlockPlacementStats to NPM
 (#129853)

---
 .../llvm/CodeGen/MachineBlockPlacement.h      |  9 ++++
 llvm/include/llvm/InitializePasses.h          |  2 +-
 .../llvm/Passes/MachinePassRegistry.def       |  2 +-
 llvm/lib/CodeGen/CodeGen.cpp                  |  2 +-
 llvm/lib/CodeGen/MachineBlockPlacement.cpp    | 45 ++++++++++++++-----
 5 files changed, 45 insertions(+), 15 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachineBlockPlacement.h b/llvm/include/llvm/CodeGen/MachineBlockPlacement.h
index 733d24ab719a8..f88e72c2a177b 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockPlacement.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockPlacement.h
@@ -30,6 +30,15 @@ class MachineBlockPlacementPass
                 function_ref<StringRef(StringRef)> MapClassName2PassName) const;
 };
 
+class MachineBlockPlacementStatsPass
+    : public PassInfoMixin<MachineBlockPlacementStatsPass> {
+
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+  static bool isRequired() { return true; }
+};
+
 } // namespace llvm
 
 #endif // LLVM_CODEGEN_MACHINEBLOCKPLACEMENT_H
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index ea6afbfdc7861..9fd8f8228a717 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -185,7 +185,7 @@ void initializeMIRNamerPass(PassRegistry &);
 void initializeMIRPrintingPassPass(PassRegistry &);
 void initializeMachineBlockFrequencyInfoWrapperPassPass(PassRegistry &);
 void initializeMachineBlockPlacementLegacyPass(PassRegistry &);
-void initializeMachineBlockPlacementStatsPass(PassRegistry &);
+void initializeMachineBlockPlacementStatsLegacyPass(PassRegistry &);
 void initializeMachineBranchProbabilityInfoWrapperPassPass(PassRegistry &);
 void initializeMachineCFGPrinterPass(PassRegistry &);
 void initializeMachineCSELegacyPass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 3ba2d259fe78c..fff5083caad1a 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -137,6 +137,7 @@ MACHINE_FUNCTION_ANALYSIS("virtregmap", VirtRegMapAnalysis())
 #ifndef MACHINE_FUNCTION_PASS
 #define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
 #endif
+MACHINE_FUNCTION_PASS("block-placement-stats", MachineBlockPlacementStatsPass())
 MACHINE_FUNCTION_PASS("branch-relaxation", BranchRelaxationPass())
 MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass())
 MACHINE_FUNCTION_PASS("detect-dead-lanes", DetectDeadLanesPass())
@@ -281,7 +282,6 @@ DUMMY_MACHINE_MODULE_PASS("mir-strip-debug", StripDebugMachineModulePass)
 #endif
 DUMMY_MACHINE_FUNCTION_PASS("bbsections-prepare", BasicBlockSectionsPass)
 DUMMY_MACHINE_FUNCTION_PASS("bbsections-profile-reader", BasicBlockSectionsProfileReaderPass)
-DUMMY_MACHINE_FUNCTION_PASS("block-placement-stats", MachineBlockPlacementStatsPass)
 DUMMY_MACHINE_FUNCTION_PASS("break-false-deps", BreakFalseDepsPass)
 DUMMY_MACHINE_FUNCTION_PASS("cfguard-longjmp", CFGuardLongjmpPass)
 DUMMY_MACHINE_FUNCTION_PASS("cfi-fixup", CFIFixupPass)
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index dd04b2e81a2a7..2b0e93803971f 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -73,7 +73,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeMIRProfileLoaderPassPass(Registry);
   initializeMachineBlockFrequencyInfoWrapperPassPass(Registry);
   initializeMachineBlockPlacementLegacyPass(Registry);
-  initializeMachineBlockPlacementStatsPass(Registry);
+  initializeMachineBlockPlacementStatsLegacyPass(Registry);
   initializeMachineCFGPrinterPass(Registry);
   initializeMachineCSELegacyPass(Registry);
   initializeMachineCombinerPass(Registry);
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index 47213ede3a33e..bd777db0791b7 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -3839,21 +3839,35 @@ namespace {
 /// placement. This is separate from the actual placement pass so that they can
 /// be computed in the absence of any placement transformations or when using
 /// alternative placement strategies.
-class MachineBlockPlacementStats : public MachineFunctionPass {
+class MachineBlockPlacementStats {
   /// A handle to the branch probability pass.
   const MachineBranchProbabilityInfo *MBPI;
 
   /// A handle to the function-wide block frequency pass.
   const MachineBlockFrequencyInfo *MBFI;
 
+public:
+  MachineBlockPlacementStats(const MachineBranchProbabilityInfo *MBPI,
+                             const MachineBlockFrequencyInfo *MBFI)
+      : MBPI(MBPI), MBFI(MBFI) {}
+  bool run(MachineFunction &MF);
+};
+
+class MachineBlockPlacementStatsLegacy : public MachineFunctionPass {
 public:
   static char ID; // Pass identification, replacement for typeid
 
-  MachineBlockPlacementStats() : MachineFunctionPass(ID) {
-    initializeMachineBlockPlacementStatsPass(*PassRegistry::getPassRegistry());
+  MachineBlockPlacementStatsLegacy() : MachineFunctionPass(ID) {
+    initializeMachineBlockPlacementStatsLegacyPass(
+        *PassRegistry::getPassRegistry());
   }
 
-  bool runOnMachineFunction(MachineFunction &F) override;
+  bool runOnMachineFunction(MachineFunction &F) override {
+    auto *MBPI =
+        &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
+    auto *MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
+    return MachineBlockPlacementStats(MBPI, MBFI).run(F);
+  }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
@@ -3865,18 +3879,28 @@ class MachineBlockPlacementStats : public MachineFunctionPass {
 
 } // end anonymous namespace
 
-char MachineBlockPlacementStats::ID = 0;
+char MachineBlockPlacementStatsLegacy::ID = 0;
 
-char &llvm::MachineBlockPlacementStatsID = MachineBlockPlacementStats::ID;
+char &llvm::MachineBlockPlacementStatsID = MachineBlockPlacementStatsLegacy::ID;
 
-INITIALIZE_PASS_BEGIN(MachineBlockPlacementStats, "block-placement-stats",
+INITIALIZE_PASS_BEGIN(MachineBlockPlacementStatsLegacy, "block-placement-stats",
                       "Basic Block Placement Stats", false, false)
 INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
-INITIALIZE_PASS_END(MachineBlockPlacementStats, "block-placement-stats",
+INITIALIZE_PASS_END(MachineBlockPlacementStatsLegacy, "block-placement-stats",
                     "Basic Block Placement Stats", false, false)
 
-bool MachineBlockPlacementStats::runOnMachineFunction(MachineFunction &F) {
+PreservedAnalyses
+MachineBlockPlacementStatsPass::run(MachineFunction &MF,
+                                    MachineFunctionAnalysisManager &MFAM) {
+  auto &MBPI = MFAM.getResult<MachineBranchProbabilityAnalysis>(MF);
+  auto &MBFI = MFAM.getResult<MachineBlockFrequencyAnalysis>(MF);
+
+  MachineBlockPlacementStats(&MBPI, &MBFI).run(MF);
+  return PreservedAnalyses::all();
+}
+
+bool MachineBlockPlacementStats::run(MachineFunction &F) {
   // Check for single-block functions and skip them.
   if (std::next(F.begin()) == F.end())
     return false;
@@ -3884,9 +3908,6 @@ bool MachineBlockPlacementStats::runOnMachineFunction(MachineFunction &F) {
   if (!isFunctionInPrintList(F.getName()))
     return false;
 
-  MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
-  MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
-
   for (MachineBasicBlock &MBB : F) {
     BlockFrequency BlockFreq = MBFI->getBlockFreq(&MBB);
     Statistic &NumBranches =

From 6f91bfcc8aebe61ba4469c48270928f82ee89027 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Thu, 17 Apr 2025 11:43:10 +0100
Subject: [PATCH 248/710] [LLVM][AArch64ISel] Fix IsLE predicate setting so it
 does not affect BE codegen. (#135978)

Ensure little endian specific patterns, not just their multi-classes,
are protected by IsLE.
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 27 +++++-----
 .../sve-ld1-addressing-mode-reg-reg.ll        | 54 +++++++++++++------
 .../sve-st1-addressing-mode-reg-reg.ll        | 54 +++++++++++++------
 3 files changed, 90 insertions(+), 45 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index e5d99037b6c63..d13728ec930c8 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2752,7 +2752,7 @@ let Predicates = [HasSVE_or_SME] in {
 
   // For big endian, only BITCASTs involving same sized vector types with same
   // size vector elements can be isel'd directly.
-  let Predicates = [IsLE] in
+  let Predicates = [HasSVE_or_SME, IsLE] in
     foreach VT = [ nxv16i8, nxv8i16, nxv4i32, nxv2i64, nxv8f16, nxv4f32, nxv2f64, nxv8bf16 ] in
       foreach VT2 = [ nxv16i8, nxv8i16, nxv4i32, nxv2i64, nxv8f16, nxv4f32, nxv2f64, nxv8bf16 ] in
         if !ne(VT,VT2) then
@@ -3002,24 +3002,25 @@ let Predicates = [HasSVE_or_SME] in {
   defm : unpred_load<        load, nxv2f32,  LD1W_D,  LD1W_D_IMM, PTRUE_D, am_sve_regreg_lsl2>;
   defm : unpred_load<        load, nxv2f64,    LD1D,    LD1D_IMM, PTRUE_D, am_sve_regreg_lsl3>;
 
-  // Allow using the reg+reg form of ld1b/st1b for memory accesses with the
-  // same width as nxv16i8.  This saves an add in cases where we would
-  // otherwise compute the address separately.
-  multiclass unpred_loadstore_bitcast<ValueType Ty> {
-    let Predicates = [IsLE] in {
+  let Predicates = [HasSVE_or_SME, IsLE] in {
+    // Allow using the reg+reg form of ld1b/st1b for memory accesses with the
+    // same width as nxv16i8.  This saves an add in cases where we would
+    // otherwise compute the address separately.
+    multiclass unpred_loadstore_bitcast<ValueType Ty> {
       def : Pat<(Ty (load (am_sve_regreg_lsl0 GPR64sp:$base, GPR64:$offset))),
                 (LD1B (PTRUE_B 31), GPR64sp:$base, GPR64:$offset)>;
       def : Pat<(store Ty:$val, (am_sve_regreg_lsl0 GPR64sp:$base, GPR64:$offset)),
                 (ST1B ZPR:$val, (PTRUE_B 31), GPR64sp:$base, GPR64:$offset)>;
     }
+
+    defm : unpred_loadstore_bitcast<nxv8i16>;
+    defm : unpred_loadstore_bitcast<nxv8f16>;
+    defm : unpred_loadstore_bitcast<nxv8bf16>;
+    defm : unpred_loadstore_bitcast<nxv4f32>;
+    defm : unpred_loadstore_bitcast<nxv4i32>;
+    defm : unpred_loadstore_bitcast<nxv2i64>;
+    defm : unpred_loadstore_bitcast<nxv2f64>;
   }
-  defm : unpred_loadstore_bitcast<nxv8i16>;
-  defm : unpred_loadstore_bitcast<nxv8f16>;
-  defm : unpred_loadstore_bitcast<nxv8bf16>;
-  defm : unpred_loadstore_bitcast<nxv4f32>;
-  defm : unpred_loadstore_bitcast<nxv4i32>;
-  defm : unpred_loadstore_bitcast<nxv2i64>;
-  defm : unpred_loadstore_bitcast<nxv2f64>;
 
   // Allow using LDR/STR to avoid the predicate dependence.
   let Predicates = [HasSVE_or_SME, IsLE, AllowMisalignedMemAccesses] in
diff --git a/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-reg.ll b/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-reg.ll
index 3f31917b125b7..05abfa319d389 100644
--- a/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-reg.ll
+++ b/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-reg.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -mtriple=aarch64_be-linux-gnu -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 
 ; LD1B
 
@@ -15,33 +16,54 @@ define <vscale x 16 x i8> @ld1_nxv16i8(ptr %addr, i64 %off) {
 }
 
 define <vscale x 8 x i16> @ld1_nxv16i8_bitcast_to_i16(ptr %addr, i64 %off) {
-; CHECK-LABEL: ld1_nxv16i8_bitcast_to_i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0, x1]
-; CHECK-NEXT:    ret
+; CHECK-LE-LABEL: ld1_nxv16i8_bitcast_to_i16:
+; CHECK-LE:       // %bb.0:
+; CHECK-LE-NEXT:    ptrue p0.b
+; CHECK-LE-NEXT:    ld1b { z0.b }, p0/z, [x0, x1]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: ld1_nxv16i8_bitcast_to_i16:
+; CHECK-BE:       // %bb.0:
+; CHECK-BE-NEXT:    ptrue p0.h
+; CHECK-BE-NEXT:    add x8, x0, x1
+; CHECK-BE-NEXT:    ld1h { z0.h }, p0/z, [x8]
+; CHECK-BE-NEXT:    ret
   %ptr = getelementptr inbounds i8, ptr %addr, i64 %off
   %val = load volatile <vscale x 8 x i16>, ptr %ptr
   ret <vscale x 8 x i16> %val
 }
 
 define <vscale x 4 x i32> @ld1_nxv16i8_bitcast_to_i32(ptr %addr, i64 %off) {
-; CHECK-LABEL: ld1_nxv16i8_bitcast_to_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0, x1]
-; CHECK-NEXT:    ret
+; CHECK-LE-LABEL: ld1_nxv16i8_bitcast_to_i32:
+; CHECK-LE:       // %bb.0:
+; CHECK-LE-NEXT:    ptrue p0.b
+; CHECK-LE-NEXT:    ld1b { z0.b }, p0/z, [x0, x1]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: ld1_nxv16i8_bitcast_to_i32:
+; CHECK-BE:       // %bb.0:
+; CHECK-BE-NEXT:    ptrue p0.s
+; CHECK-BE-NEXT:    add x8, x0, x1
+; CHECK-BE-NEXT:    ld1w { z0.s }, p0/z, [x8]
+; CHECK-BE-NEXT:    ret
   %ptr = getelementptr inbounds i8, ptr %addr, i64 %off
   %val = load volatile <vscale x 4 x i32>, ptr %ptr
   ret <vscale x 4 x i32> %val
 }
 
 define <vscale x 2 x i64> @ld1_nxv16i8_bitcast_to_i64(ptr %addr, i64 %off) {
-; CHECK-LABEL: ld1_nxv16i8_bitcast_to_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0, x1]
-; CHECK-NEXT:    ret
+; CHECK-LE-LABEL: ld1_nxv16i8_bitcast_to_i64:
+; CHECK-LE:       // %bb.0:
+; CHECK-LE-NEXT:    ptrue p0.b
+; CHECK-LE-NEXT:    ld1b { z0.b }, p0/z, [x0, x1]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: ld1_nxv16i8_bitcast_to_i64:
+; CHECK-BE:       // %bb.0:
+; CHECK-BE-NEXT:    ptrue p0.d
+; CHECK-BE-NEXT:    add x8, x0, x1
+; CHECK-BE-NEXT:    ld1d { z0.d }, p0/z, [x8]
+; CHECK-BE-NEXT:    ret
   %ptr = getelementptr inbounds i8, ptr %addr, i64 %off
   %val = load volatile <vscale x 2 x i64>, ptr %ptr
   ret <vscale x 2 x i64> %val
diff --git a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-reg.ll b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-reg.ll
index d859bbb567ebb..0bf6b12a5d020 100644
--- a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-reg.ll
+++ b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-reg.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -mtriple=aarch64_be-linux-gnu -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 
 ; ST1B
 
@@ -15,33 +16,54 @@ define void @st1_nxv16i8(ptr %addr, i64 %off, <vscale x 16 x i8> %val) {
 }
 
 define void @st1_nxv16i8_bitcast_from_i16(ptr %addr, i64 %off, <vscale x 8 x i16> %val) {
-; CHECK-LABEL: st1_nxv16i8_bitcast_from_i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    st1b { z0.b }, p0, [x0, x1]
-; CHECK-NEXT:    ret
+; CHECK-LE-LABEL: st1_nxv16i8_bitcast_from_i16:
+; CHECK-LE:       // %bb.0:
+; CHECK-LE-NEXT:    ptrue p0.b
+; CHECK-LE-NEXT:    st1b { z0.b }, p0, [x0, x1]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: st1_nxv16i8_bitcast_from_i16:
+; CHECK-BE:       // %bb.0:
+; CHECK-BE-NEXT:    ptrue p0.h
+; CHECK-BE-NEXT:    add x8, x0, x1
+; CHECK-BE-NEXT:    st1h { z0.h }, p0, [x8]
+; CHECK-BE-NEXT:    ret
   %ptr = getelementptr inbounds i8, ptr %addr, i64 %off
   store <vscale x 8 x i16> %val, ptr %ptr
   ret void
 }
 
 define void @st1_nxv16i8_bitcast_from_i32(ptr %addr, i64 %off, <vscale x 4 x i32> %val) {
-; CHECK-LABEL: st1_nxv16i8_bitcast_from_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    st1b { z0.b }, p0, [x0, x1]
-; CHECK-NEXT:    ret
+; CHECK-LE-LABEL: st1_nxv16i8_bitcast_from_i32:
+; CHECK-LE:       // %bb.0:
+; CHECK-LE-NEXT:    ptrue p0.b
+; CHECK-LE-NEXT:    st1b { z0.b }, p0, [x0, x1]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: st1_nxv16i8_bitcast_from_i32:
+; CHECK-BE:       // %bb.0:
+; CHECK-BE-NEXT:    ptrue p0.s
+; CHECK-BE-NEXT:    add x8, x0, x1
+; CHECK-BE-NEXT:    st1w { z0.s }, p0, [x8]
+; CHECK-BE-NEXT:    ret
   %ptr = getelementptr inbounds i8, ptr %addr, i64 %off
   store <vscale x 4 x i32> %val, ptr %ptr
   ret void
 }
 
 define void @st1_nxv16i8_bitcast_from_i64(ptr %addr, i64 %off, <vscale x 2 x i64> %val) {
-; CHECK-LABEL: st1_nxv16i8_bitcast_from_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    st1b { z0.b }, p0, [x0, x1]
-; CHECK-NEXT:    ret
+; CHECK-LE-LABEL: st1_nxv16i8_bitcast_from_i64:
+; CHECK-LE:       // %bb.0:
+; CHECK-LE-NEXT:    ptrue p0.b
+; CHECK-LE-NEXT:    st1b { z0.b }, p0, [x0, x1]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: st1_nxv16i8_bitcast_from_i64:
+; CHECK-BE:       // %bb.0:
+; CHECK-BE-NEXT:    ptrue p0.d
+; CHECK-BE-NEXT:    add x8, x0, x1
+; CHECK-BE-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-BE-NEXT:    ret
   %ptr = getelementptr inbounds i8, ptr %addr, i64 %off
   store <vscale x 2 x i64> %val, ptr %ptr
   ret void

From d69ee885cccecb49f0b288ec634186c35c8ecfb5 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 17 Apr 2025 12:44:57 +0200
Subject: [PATCH 249/710] [CaptureTracking] Remove dereferenceable_or_null
 special case (#135613)

Remove the special case where comparing a dereferenceable_or_null
pointer with null results in captures(none) instead of
captures(address_is_null).

This special case is not entirely correct. Let's say we have an
allocated object of size 2 at address 1 and have a pointer `%p` pointing
either to address 1 or 2. Then passing `gep p, -1` to a
`dereferenceable_or_null(1)` function is well-defined, and allows us to
distinguish between the two possible pointers, capturing information
about the address.

Now that we ignore address captures in alias analysis, I think we're
ready to drop this special case. Additionally, if there are regressions
in other places, the fact that this is inferred as address_is_null
should allow us to easily address them if necessary.
---
 llvm/include/llvm/Analysis/CaptureTracking.h  | 12 +------
 llvm/lib/Analysis/CaptureTracking.cpp         | 33 ++-----------------
 .../Transforms/IPO/AttributorAttributes.cpp   | 20 ++---------
 .../lib/Transforms/Scalar/MemCpyOptimizer.cpp |  9 +----
 .../test/Transforms/Attributor/nocapture-1.ll |  2 +-
 .../test/Transforms/Attributor/nocapture-2.ll | 26 +++++++--------
 .../Transforms/FunctionAttrs/nocapture.ll     |  2 +-
 7 files changed, 22 insertions(+), 82 deletions(-)

diff --git a/llvm/include/llvm/Analysis/CaptureTracking.h b/llvm/include/llvm/Analysis/CaptureTracking.h
index bd8d2bb568876..c0cea8c9fadb7 100644
--- a/llvm/include/llvm/Analysis/CaptureTracking.h
+++ b/llvm/include/llvm/Analysis/CaptureTracking.h
@@ -168,11 +168,6 @@ namespace llvm {
     /// Return one of Stop, Continue or ContinueIgnoringReturn to control
     /// further traversal.
     virtual Action captured(const Use *U, UseCaptureInfo CI) = 0;
-
-    /// isDereferenceableOrNull - Overload to allow clients with additional
-    /// knowledge about pointer dereferenceability to provide it and thereby
-    /// avoid conservative responses when a pointer is compared to null.
-    virtual bool isDereferenceableOrNull(Value *O, const DataLayout &DL);
   };
 
   /// Determine what kind of capture behaviour \p U may exhibit.
@@ -183,12 +178,7 @@ namespace llvm {
   ///
   /// \p Base is the starting value of the capture analysis, which is
   /// relevant for address_is_null captures.
-  /// The \p IsDereferenceableOrNull callback is used to rule out capturing for
-  /// certain comparisons.
-  UseCaptureInfo
-  DetermineUseCaptureKind(const Use &U, const Value *Base,
-                          llvm::function_ref<bool(Value *, const DataLayout &)>
-                              IsDereferenceableOrNull);
+  UseCaptureInfo DetermineUseCaptureKind(const Use &U, const Value *Base);
 
   /// PointerMayBeCaptured - Visit the value and the values derived from it and
   /// find values which appear to be capturing the pointer value. This feeds
diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp
index aa53a27537567..2550b7a4732c1 100644
--- a/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/llvm/lib/Analysis/CaptureTracking.cpp
@@ -56,21 +56,6 @@ CaptureTracker::~CaptureTracker() = default;
 
 bool CaptureTracker::shouldExplore(const Use *U) { return true; }
 
-bool CaptureTracker::isDereferenceableOrNull(Value *O, const DataLayout &DL) {
-  // We want comparisons to null pointers to not be considered capturing,
-  // but need to guard against cases like gep(p, -ptrtoint(p2)) == null,
-  // which are equivalent to p == p2 and would capture the pointer.
-  //
-  // A dereferenceable pointer is a case where this is known to be safe,
-  // because the pointer resulting from such a construction would not be
-  // dereferenceable.
-  //
-  // It is not sufficient to check for inbounds GEP here, because GEP with
-  // zero offset is always inbounds.
-  bool CanBeNull, CanBeFreed;
-  return O->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
-}
-
 namespace {
 struct SimpleCaptureTracker : public CaptureTracker {
   explicit SimpleCaptureTracker(bool ReturnCaptures, CaptureComponents Mask,
@@ -281,9 +266,7 @@ Instruction *llvm::FindEarliestCapture(const Value *V, Function &F,
   return CB.EarliestCapture;
 }
 
-UseCaptureInfo llvm::DetermineUseCaptureKind(
-    const Use &U, const Value *Base,
-    function_ref<bool(Value *, const DataLayout &)> IsDereferenceableOrNull) {
+UseCaptureInfo llvm::DetermineUseCaptureKind(const Use &U, const Value *Base) {
   Instruction *I = dyn_cast<Instruction>(U.getUser());
 
   // TODO: Investigate non-instruction uses.
@@ -391,15 +374,6 @@ UseCaptureInfo llvm::DetermineUseCaptureKind(
       if (U->getType()->getPointerAddressSpace() == 0)
         if (isNoAliasCall(U.get()->stripPointerCasts()))
           return CaptureComponents::None;
-      if (!I->getFunction()->nullPointerIsDefined()) {
-        auto *O = I->getOperand(Idx)->stripPointerCastsSameRepresentation();
-        // Comparing a dereferenceable_or_null pointer against null cannot
-        // lead to pointer escapes, because if it is not null it must be a
-        // valid (in-bounds) pointer.
-        const DataLayout &DL = I->getDataLayout();
-        if (IsDereferenceableOrNull && IsDereferenceableOrNull(O, DL))
-          return CaptureComponents::None;
-      }
 
       // Check whether this is a comparison of the base pointer against
       // null.
@@ -447,12 +421,9 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
   if (!AddUses(V))
     return;
 
-  auto IsDereferenceableOrNull = [Tracker](Value *V, const DataLayout &DL) {
-    return Tracker->isDereferenceableOrNull(V, DL);
-  };
   while (!Worklist.empty()) {
     const Use *U = Worklist.pop_back_val();
-    UseCaptureInfo CI = DetermineUseCaptureKind(*U, V, IsDereferenceableOrNull);
+    UseCaptureInfo CI = DetermineUseCaptureKind(*U, V);
     if (capturesAnything(CI.UseCC)) {
       switch (Tracker->captured(U, CI)) {
       case CaptureTracker::Stop:
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index cc6e846f4f211..ac56df3823e20 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -3928,12 +3928,6 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl {
     // (iii) There is no other pointer argument which could alias with the
     //       value.
 
-    auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
-      const auto *DerefAA = A.getAAFor<AADereferenceable>(
-          *this, IRPosition::value(*O), DepClassTy::OPTIONAL);
-      return DerefAA ? DerefAA->getAssumedDereferenceableBytes() : 0;
-    };
-
     const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
     const Function *ScopeFn = VIRP.getAnchorScope();
     // Check whether the value is captured in the scope using AANoCapture.
@@ -3973,8 +3967,7 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl {
       //       is CGSCC runs. For those we would need to "allow" AANoCapture for
       //       a value in the module slice.
       // TODO(captures): Make this more precise.
-      UseCaptureInfo CI =
-          DetermineUseCaptureKind(U, /*Base=*/nullptr, IsDereferenceableOrNull);
+      UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
       if (capturesNothing(CI))
         return true;
       if (CI.isPassthrough()) {
@@ -6035,16 +6028,9 @@ ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
     }
   }
 
-  auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
-    const auto *DerefAA = A.getAAFor<AADereferenceable>(
-        *this, IRPosition::value(*O), DepClassTy::OPTIONAL);
-    return DerefAA && DerefAA->getAssumedDereferenceableBytes();
-  };
-
   auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
     // TODO(captures): Make this more precise.
-    UseCaptureInfo CI =
-        DetermineUseCaptureKind(U, /*Base=*/nullptr, IsDereferenceableOrNull);
+    UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
     if (capturesNothing(CI))
       return true;
     if (CI.isPassthrough()) {
@@ -12177,7 +12163,7 @@ struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
     auto UsePred = [&](const Use &U, bool &Follow) -> bool {
       Uses.insert(&U);
       // TODO(captures): Make this more precise.
-      UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr, nullptr);
+      UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
       if (CI.isPassthrough()) {
         Follow = true;
         return true;
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 0d03395f0ff20..17a35c66c5966 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -1519,12 +1519,6 @@ bool MemCpyOptPass::performStackMoveOptzn(Instruction *Load, Instruction *Store,
   SmallSet<Instruction *, 4> AAMetadataInstrs;
   bool SrcNotDom = false;
 
-  // Recursively track the user and check whether modified alias exist.
-  auto IsDereferenceableOrNull = [](Value *V, const DataLayout &DL) -> bool {
-    bool CanBeNull, CanBeFreed;
-    return V->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
-  };
-
   auto CaptureTrackingWithModRef =
       [&](Instruction *AI,
           function_ref<bool(Instruction *)> ModRefCallback) -> bool {
@@ -1551,8 +1545,7 @@ bool MemCpyOptPass::performStackMoveOptzn(Instruction *Load, Instruction *Store,
         }
         if (!Visited.insert(&U).second)
           continue;
-        UseCaptureInfo CI =
-            DetermineUseCaptureKind(U, AI, IsDereferenceableOrNull);
+        UseCaptureInfo CI = DetermineUseCaptureKind(U, AI);
         // TODO(captures): Make this more precise.
         if (capturesAnything(CI.UseCC))
           return false;
diff --git a/llvm/test/Transforms/Attributor/nocapture-1.ll b/llvm/test/Transforms/Attributor/nocapture-1.ll
index c6097b3868d16..1f1c442fbc5f7 100644
--- a/llvm/test/Transforms/Attributor/nocapture-1.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-1.ll
@@ -741,7 +741,7 @@ define i1 @nocaptureInboundsGEPICmpRev(ptr %x) {
 define i1 @nocaptureDereferenceableOrNullICmp(ptr dereferenceable_or_null(4) %x) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@nocaptureDereferenceableOrNullICmp
-; CHECK-SAME: (ptr nofree noundef readnone captures(none) dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[X]], null
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
diff --git a/llvm/test/Transforms/Attributor/nocapture-2.ll b/llvm/test/Transforms/Attributor/nocapture-2.ll
index 475121f6bd357..6242a1e20a2fe 100644
--- a/llvm/test/Transforms/Attributor/nocapture-2.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-2.ll
@@ -164,14 +164,14 @@ entry:
 define ptr @scc_A(ptr dereferenceable_or_null(4) %a) {
 ; CHECK: Function Attrs: nofree nosync nounwind memory(none)
 ; CHECK-LABEL: define noundef dereferenceable_or_null(4) ptr @scc_A
-; CHECK-SAME: (ptr nofree noundef readnone returned dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef readnone returned dereferenceable_or_null(4) [[A:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne ptr [[A]], null
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.true:
-; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) ptr @scc_C(ptr noalias nofree noundef nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
-; CHECK-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(8) ptr @scc_B(ptr noalias nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
-; CHECK-NEXT:    [[CALL2:%.*]] = call ptr @scc_A(ptr noalias nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) ptr @scc_C(ptr noalias nofree noundef nonnull readnone dereferenceable(4) [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(8) ptr @scc_B(ptr noalias nofree noundef readnone dereferenceable_or_null(8) [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL2:%.*]] = call ptr @scc_A(ptr noalias nofree noundef readnone dereferenceable_or_null(8) [[A]]) #[[ATTR2]]
 ; CHECK-NEXT:    br label [[COND_END:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    br label [[COND_END]]
@@ -201,14 +201,14 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 define ptr @scc_B(ptr dereferenceable_or_null(8) %a) {
 ; CHECK: Function Attrs: nofree nosync nounwind memory(none)
 ; CHECK-LABEL: define noundef dereferenceable_or_null(8) ptr @scc_B
-; CHECK-SAME: (ptr nofree noundef readnone returned dereferenceable_or_null(8) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef readnone returned dereferenceable_or_null(8) [[A:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne ptr [[A]], null
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.true:
-; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) ptr @scc_A(ptr noalias nofree noundef nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
-; CHECK-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(8) ptr @scc_B(ptr noalias nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
-; CHECK-NEXT:    [[CALL2:%.*]] = call ptr @scc_C(ptr noalias nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) ptr @scc_A(ptr noalias nofree noundef nonnull readnone dereferenceable(8) [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(8) ptr @scc_B(ptr noalias nofree noundef readnone dereferenceable_or_null(8) [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL2:%.*]] = call ptr @scc_C(ptr noalias nofree noundef readnone dereferenceable_or_null(8) [[A]]) #[[ATTR2]]
 ; CHECK-NEXT:    br label [[COND_END:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    br label [[COND_END]]
@@ -237,20 +237,20 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 define ptr @scc_C(ptr dereferenceable_or_null(2) %a) {
 ; CHECK: Function Attrs: nofree nosync nounwind memory(none)
 ; CHECK-LABEL: define noundef dereferenceable_or_null(4) ptr @scc_C
-; CHECK-SAME: (ptr nofree noundef readnone returned dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
+; CHECK-SAME: (ptr nofree noundef readnone returned dereferenceable_or_null(4) [[A:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) ptr @scc_A(ptr noalias nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call ptr @scc_A(ptr noalias nofree noundef readnone dereferenceable_or_null(4) [[A]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne ptr [[A]], null
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.true:
-; CHECK-NEXT:    [[CALL1:%.*]] = call ptr @scc_B(ptr noalias nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL1:%.*]] = call ptr @scc_B(ptr noalias nofree noundef readnone dereferenceable_or_null(8) [[A]]) #[[ATTR2]]
 ; CHECK-NEXT:    br label [[COND_END:%.*]]
 ; CHECK:       cond.false:
-; CHECK-NEXT:    [[CALL2:%.*]] = call ptr @scc_C(ptr noalias nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL2:%.*]] = call ptr @scc_C(ptr noalias nofree noundef readnone dereferenceable_or_null(4) [[A]]) #[[ATTR2]]
 ; CHECK-NEXT:    br label [[COND_END]]
 ; CHECK:       cond.end:
 ; CHECK-NEXT:    [[COND:%.*]] = phi ptr [ [[A]], [[COND_TRUE]] ], [ [[A]], [[COND_FALSE]] ]
-; CHECK-NEXT:    [[CALL3:%.*]] = call ptr @scc_A(ptr noalias nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL3:%.*]] = call ptr @scc_A(ptr noalias nofree noundef readnone dereferenceable_or_null(4) [[A]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret ptr [[A]]
 ;
 entry:
diff --git a/llvm/test/Transforms/FunctionAttrs/nocapture.ll b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
index dc1fdb6100aeb..50dcb74d973a5 100644
--- a/llvm/test/Transforms/FunctionAttrs/nocapture.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
@@ -960,7 +960,7 @@ define i1 @inboundsGEPICmpNullPointerDefined(ptr %x) null_pointer_is_valid {
 define i1 @nocaptureDereferenceableOrNullICmp(ptr dereferenceable_or_null(4) %x) {
 ; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 ; FNATTRS-LABEL: define noundef i1 @nocaptureDereferenceableOrNullICmp
-; FNATTRS-SAME: (ptr readnone captures(none) dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
+; FNATTRS-SAME: (ptr readnone captures(address_is_null) dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[X]], null
 ; FNATTRS-NEXT:    ret i1 [[TMP1]]
 ;

From 90ddb5444030b8d7cca6e91a27994e4fa9a6525d Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 17 Apr 2025 12:50:28 +0200
Subject: [PATCH 250/710] [clang][bytecode] Enter a non-constant context when
 revisiting (#136104)

Otherwise, things like __builtin_is_constant_evaluated() return the
wrong value.
---
 clang/lib/AST/ByteCode/Compiler.cpp            | 5 +++++
 clang/lib/AST/ByteCode/Interp.h                | 9 +++++++++
 clang/lib/AST/ByteCode/InterpState.h           | 2 +-
 clang/lib/AST/ByteCode/Opcodes.td              | 3 +++
 clang/test/AST/ByteCode/builtin-constant-p.cpp | 8 ++++++++
 5 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 157e306e5cdb3..d3eabc513a9ac 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -6466,8 +6466,13 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
 
   // In case we need to re-visit a declaration.
   auto revisit = [&](const VarDecl *VD) -> bool {
+    if (!this->emitPushCC(VD->hasConstantInitialization(), E))
+      return false;
     auto VarState = this->visitDecl(VD, /*IsConstexprUnknown=*/true);
 
+    if (!this->emitPopCC(E))
+      return false;
+
     if (VarState.notCreated())
       return true;
     if (!VarState)
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index bd58c2a88e9d9..49e2326186bf8 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -2848,6 +2848,15 @@ inline bool EndSpeculation(InterpState &S, CodePtr OpPC) {
   return true;
 }
 
+inline bool PushCC(InterpState &S, CodePtr OpPC, bool Value) {
+  S.ConstantContextOverride = Value;
+  return true;
+}
+inline bool PopCC(InterpState &S, CodePtr OpPC) {
+  S.ConstantContextOverride = std::nullopt;
+  return true;
+}
+
 /// Do nothing and just abort execution.
 inline bool Error(InterpState &S, CodePtr OpPC) { return false; }
 
diff --git a/clang/lib/AST/ByteCode/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
index 74001b80d9c00..528c1a24e7b05 100644
--- a/clang/lib/AST/ByteCode/InterpState.h
+++ b/clang/lib/AST/ByteCode/InterpState.h
@@ -127,7 +127,6 @@ class InterpState final : public State, public SourceMapper {
   SourceMapper *M;
   /// Allocator used for dynamic allocations performed via the program.
   DynamicAllocator Alloc;
-  std::optional<bool> ConstantContextOverride;
 
 public:
   /// Reference to the module containing all bytecode.
@@ -147,6 +146,7 @@ class InterpState final : public State, public SourceMapper {
   /// Things needed to do speculative execution.
   SmallVectorImpl<PartialDiagnosticAt> *PrevDiags = nullptr;
   unsigned SpeculationDepth = 0;
+  std::optional<bool> ConstantContextOverride;
 
   llvm::SmallVector<
       std::pair<const Expr *, const LifetimeExtendedTemporaryDecl *>>
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 5a9079fea0846..8451e54ad0c41 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -872,3 +872,6 @@ def GetTypeidPtr : Opcode { let Args = [ArgTypePtr]; }
 def DiagTypeid : Opcode;
 
 def CheckDestruction : Opcode;
+
+def PushCC : Opcode { let Args = [ArgBool]; }
+def PopCC : Opcode;
diff --git a/clang/test/AST/ByteCode/builtin-constant-p.cpp b/clang/test/AST/ByteCode/builtin-constant-p.cpp
index ed9e606ed16aa..f5b16761bfdc9 100644
--- a/clang/test/AST/ByteCode/builtin-constant-p.cpp
+++ b/clang/test/AST/ByteCode/builtin-constant-p.cpp
@@ -121,3 +121,11 @@ constexpr int mutate6(bool mutate) {
 static_assert(mutate6(false) == 11);
 static_assert(mutate6(true) == 21); // ref-error {{static assertion failed}} \
                                     // ref-note {{evaluates to '10 == 21'}}
+
+#define fold(x) (__builtin_constant_p(x) ? (x) : (x))
+void g() {
+  /// f will be revisited when evaluating the static_assert, since it's
+  /// a local variable. But it should be visited in a non-constant context.
+  const float f = __builtin_is_constant_evaluated();
+  static_assert(fold(f == 0.0f));
+}

From eafbb879f676ccd65fe7efae6b5280ae750ef20a Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn@arm.com>
Date: Thu, 17 Apr 2025 11:50:49 +0100
Subject: [PATCH 251/710] [LoopVectorize] Don't replicate blocks with optsize
 (#129265)

Any VPlan we generate that contains a replicator region will result in
replicated blocks in the output, causing a large code size increase.
Reject such VPlans when optimizing for size, as the code size impact is
usually worse than having a scalar epilogue, which we already forbid
with optsize.

This change requires a lot of test changes. For tests of optsize
specifically I've updated the test with the new output, otherwise the
tests have been adjusted to not rely on optsize.

Fixes #66652
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  23 +++
 .../AArch64/aarch64-predication.ll            |  12 +-
 .../AArch64/conditional-branches-cost.ll      |  58 ++----
 .../LoopVectorize/AArch64/optsize_minsize.ll  | 191 +-----------------
 .../LoopVectorize/ARM/optsize_minsize.ll      | 187 +----------------
 .../LoopVectorize/X86/tail_loop_folding.ll    |   5 +-
 .../pr45679-fold-tail-by-masking.ll           |   2 +-
 .../Transforms/LoopVectorize/struct-return.ll |   2 +-
 .../tail-folding-counting-down.ll             |   2 +-
 .../vplan-sink-scalars-and-merge.ll           |  41 +++-
 10 files changed, 94 insertions(+), 429 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e9e1d0ac6c196..d8a510f59a59f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4526,6 +4526,12 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
   return false;
 }
 
+static bool hasReplicatorRegion(VPlan &Plan) {
+  return any_of(VPBlockUtils::blocksOnly<VPRegionBlock>(vp_depth_first_shallow(
+                    Plan.getVectorLoopRegion()->getEntry())),
+                [](auto *VPRB) { return VPRB->isReplicator(); });
+}
+
 #ifndef NDEBUG
 VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
   InstructionCost ExpectedCost = CM.expectedCost(ElementCount::getFixed(1));
@@ -4598,6 +4604,15 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
         continue;
       }
 
+      if (CM.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
+        LLVM_DEBUG(
+            dbgs()
+            << "LV: Not considering vector loop of width " << VF
+            << " because it would cause replicated blocks to be generated,"
+            << " which isn't allowed when optimizing for size.\n");
+        continue;
+      }
+
       if (isMoreProfitable(Candidate, ChosenFactor, P->hasScalarTail()))
         ChosenFactor = Candidate;
     }
@@ -7776,6 +7791,14 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
             << " because it will not generate any vector instructions.\n");
         continue;
       }
+      if (CM.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
+        LLVM_DEBUG(
+            dbgs()
+            << "LV: Not considering vector loop of width " << VF
+            << " because it would cause replicated blocks to be generated,"
+            << " which isn't allowed when optimizing for size.\n");
+        continue;
+      }
 
       InstructionCost Cost = cost(*P, VF);
       VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
index e6d93ea192e56..c18f9f2fae06b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
@@ -1,22 +1,22 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; REQUIRES: asserts
-; RUN: opt < %s -passes=loop-vectorize -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefix=COST
-; RUN: opt < %s -passes=loop-vectorize,instcombine,simplifycfg -force-vector-width=2 -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefix=COST
+; RUN: opt < %s -passes=loop-vectorize,instcombine,simplifycfg -force-vector-interleave=1 -force-vector-width=2 -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
 
 ; This test checks that we correctly compute the scalarized operands for a
-; user-specified vectorization factor when interleaving is disabled. We use the
-; "optsize" attribute to disable all interleaving calculations.  A cost of 4
-; for %var4 indicates that we would scalarize it's operand (%var3), giving
+; user-specified vectorization factor when interleaving is disabled. We use
+; -force-vector-interleave=1 to disable all interleaving calculations. A cost of
+; 4 for %var4 indicates that we would scalarize it's operand (%var3), giving
 ; %var4 a lower scalarization overhead.
 ;
 ; COST-LABEL:  predicated_udiv_scalarized_operand
 ; COST:        Cost of 5 for VF 2: profitable to scalarize   %var4 = udiv i64 %var2, %var3
 ;
 ;
-define i64 @predicated_udiv_scalarized_operand(ptr %a, i64 %x) optsize {
+define i64 @predicated_udiv_scalarized_operand(ptr %a, i64 %x) {
 ; CHECK-LABEL: @predicated_udiv_scalarized_operand(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 4435c31e3b189..b96a768bba24d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1472,55 +1472,29 @@ exit:
   ret void
 }
 
-define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) optsize {
+define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) {
 ; DEFAULT-LABEL: define void @redundant_branch_and_tail_folding(
-; DEFAULT-SAME: ptr [[DST:%.*]], i1 [[C:%.*]]) #[[ATTR4:[0-9]+]] {
+; DEFAULT-SAME: ptr [[DST:%.*]], i1 [[C:%.*]]) {
 ; DEFAULT-NEXT:  [[ENTRY:.*]]:
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; DEFAULT:       [[VECTOR_BODY]]:
-; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
-; DEFAULT-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
-; DEFAULT-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], splat (i64 20)
-; DEFAULT-NEXT:    [[TMP1:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)
-; DEFAULT-NEXT:    [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i32>
-; DEFAULT-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
-; DEFAULT-NEXT:    br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; DEFAULT:       [[PRED_STORE_IF]]:
-; DEFAULT-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; DEFAULT-NEXT:    store i32 [[TMP4]], ptr [[DST]], align 4
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE]]
-; DEFAULT:       [[PRED_STORE_CONTINUE]]:
-; DEFAULT-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
-; DEFAULT-NEXT:    br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; DEFAULT:       [[PRED_STORE_IF1]]:
-; DEFAULT-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
-; DEFAULT-NEXT:    store i32 [[TMP6]], ptr [[DST]], align 4
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
-; DEFAULT:       [[PRED_STORE_CONTINUE2]]:
-; DEFAULT-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
-; DEFAULT-NEXT:    br i1 [[TMP7]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; DEFAULT:       [[PRED_STORE_IF3]]:
-; DEFAULT-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
-; DEFAULT-NEXT:    store i32 [[TMP8]], ptr [[DST]], align 4
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
-; DEFAULT:       [[PRED_STORE_CONTINUE4]]:
-; DEFAULT-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
-; DEFAULT-NEXT:    br i1 [[TMP9]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
-; DEFAULT:       [[PRED_STORE_IF5]]:
-; DEFAULT-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; DEFAULT-NEXT:    store i32 [[TMP10]], ptr [[DST]], align 4
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
-; DEFAULT:       [[PRED_STORE_CONTINUE6]]:
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; DEFAULT-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
-; DEFAULT-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT:    [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; DEFAULT-NEXT:    [[TMP0:%.*]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 1)
+; DEFAULT-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[TMP0]] to <4 x i32>
+; DEFAULT-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; DEFAULT-NEXT:    store i32 [[TMP2]], ptr [[DST]], align 4
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; DEFAULT-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; DEFAULT-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT:    br label %[[EXIT:.*]]
+; DEFAULT-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
+; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; DEFAULT:       [[LOOP_HEADER]]:
 ; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
@@ -1537,7 +1511,7 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) optsize {
 ; DEFAULT-NEXT:    ret void
 ;
 ; PRED-LABEL: define void @redundant_branch_and_tail_folding(
-; PRED-SAME: ptr [[DST:%.*]], i1 [[C:%.*]]) #[[ATTR4:[0-9]+]] {
+; PRED-SAME: ptr [[DST:%.*]], i1 [[C:%.*]]) {
 ; PRED-NEXT:  [[ENTRY:.*]]:
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
index 3239d2c2e9388..5c876b760e943 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
@@ -228,7 +228,6 @@ for.cond.cleanup:
 
 ; This should be vectorized and tail predicated without optsize, as that's
 ; faster, but not with optsize, as it's much larger.
-; FIXME: Currently we avoid tail predication only with minsize
 define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n) {
 ; DEFAULT-LABEL: define void @tail_predicate_without_optsize(
 ; DEFAULT-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) {
@@ -428,182 +427,9 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
 ; OPTSIZE-LABEL: define void @tail_predicate_without_optsize(
 ; OPTSIZE-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
 ; OPTSIZE-NEXT:  [[ENTRY:.*]]:
-; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; OPTSIZE:       [[VECTOR_PH]]:
-; OPTSIZE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[A]], i64 0
-; OPTSIZE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
-; OPTSIZE-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <16 x i8> poison, i8 [[B]], i64 0
-; OPTSIZE-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT3]], <16 x i8> poison, <16 x i32> zeroinitializer
-; OPTSIZE-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <16 x i8> poison, i8 [[C]], i64 0
-; OPTSIZE-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT5]], <16 x i8> poison, <16 x i32> zeroinitializer
-; OPTSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
-; OPTSIZE:       [[VECTOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE36:.*]] ]
-; OPTSIZE-NEXT:    [[VEC_IND:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE36]] ]
-; OPTSIZE-NEXT:    [[VEC_IND1:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[PRED_STORE_CONTINUE36]] ]
-; OPTSIZE-NEXT:    [[TMP72:%.*]] = icmp ule <16 x i8> [[VEC_IND]], splat (i8 14)
-; OPTSIZE-NEXT:    [[TMP1:%.*]] = mul <16 x i8> [[BROADCAST_SPLAT]], [[VEC_IND1]]
-; OPTSIZE-NEXT:    [[TMP2:%.*]] = lshr <16 x i8> [[VEC_IND1]], splat (i8 1)
-; OPTSIZE-NEXT:    [[TMP3:%.*]] = mul <16 x i8> [[TMP2]], [[BROADCAST_SPLAT4]]
-; OPTSIZE-NEXT:    [[TMP4:%.*]] = add <16 x i8> [[TMP3]], [[TMP1]]
-; OPTSIZE-NEXT:    [[TMP5:%.*]] = lshr <16 x i8> [[VEC_IND1]], splat (i8 2)
-; OPTSIZE-NEXT:    [[TMP6:%.*]] = mul <16 x i8> [[TMP5]], [[BROADCAST_SPLAT6]]
-; OPTSIZE-NEXT:    [[TMP7:%.*]] = add <16 x i8> [[TMP4]], [[TMP6]]
-; OPTSIZE-NEXT:    [[TMP8:%.*]] = extractelement <16 x i1> [[TMP72]], i32 0
-; OPTSIZE-NEXT:    br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; OPTSIZE:       [[PRED_STORE_IF]]:
-; OPTSIZE-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
-; OPTSIZE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP9]]
-; OPTSIZE-NEXT:    [[TMP11:%.*]] = extractelement <16 x i8> [[TMP7]], i32 0
-; OPTSIZE-NEXT:    store i8 [[TMP11]], ptr [[TMP10]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE]]:
-; OPTSIZE-NEXT:    [[TMP12:%.*]] = extractelement <16 x i1> [[TMP72]], i32 1
-; OPTSIZE-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; OPTSIZE:       [[PRED_STORE_IF7]]:
-; OPTSIZE-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 1
-; OPTSIZE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP13]]
-; OPTSIZE-NEXT:    [[TMP15:%.*]] = extractelement <16 x i8> [[TMP7]], i32 1
-; OPTSIZE-NEXT:    store i8 [[TMP15]], ptr [[TMP14]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE8]]:
-; OPTSIZE-NEXT:    [[TMP16:%.*]] = extractelement <16 x i1> [[TMP72]], i32 2
-; OPTSIZE-NEXT:    br i1 [[TMP16]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; OPTSIZE:       [[PRED_STORE_IF9]]:
-; OPTSIZE-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 2
-; OPTSIZE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP17]]
-; OPTSIZE-NEXT:    [[TMP19:%.*]] = extractelement <16 x i8> [[TMP7]], i32 2
-; OPTSIZE-NEXT:    store i8 [[TMP19]], ptr [[TMP18]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE10]]:
-; OPTSIZE-NEXT:    [[TMP20:%.*]] = extractelement <16 x i1> [[TMP72]], i32 3
-; OPTSIZE-NEXT:    br i1 [[TMP20]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; OPTSIZE:       [[PRED_STORE_IF11]]:
-; OPTSIZE-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX]], 3
-; OPTSIZE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP21]]
-; OPTSIZE-NEXT:    [[TMP23:%.*]] = extractelement <16 x i8> [[TMP7]], i32 3
-; OPTSIZE-NEXT:    store i8 [[TMP23]], ptr [[TMP22]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE12]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE12]]:
-; OPTSIZE-NEXT:    [[TMP24:%.*]] = extractelement <16 x i1> [[TMP72]], i32 4
-; OPTSIZE-NEXT:    br i1 [[TMP24]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; OPTSIZE:       [[PRED_STORE_IF13]]:
-; OPTSIZE-NEXT:    [[TMP25:%.*]] = add i64 [[INDEX]], 4
-; OPTSIZE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP25]]
-; OPTSIZE-NEXT:    [[TMP27:%.*]] = extractelement <16 x i8> [[TMP7]], i32 4
-; OPTSIZE-NEXT:    store i8 [[TMP27]], ptr [[TMP26]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE14]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE14]]:
-; OPTSIZE-NEXT:    [[TMP28:%.*]] = extractelement <16 x i1> [[TMP72]], i32 5
-; OPTSIZE-NEXT:    br i1 [[TMP28]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
-; OPTSIZE:       [[PRED_STORE_IF15]]:
-; OPTSIZE-NEXT:    [[TMP29:%.*]] = add i64 [[INDEX]], 5
-; OPTSIZE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP29]]
-; OPTSIZE-NEXT:    [[TMP31:%.*]] = extractelement <16 x i8> [[TMP7]], i32 5
-; OPTSIZE-NEXT:    store i8 [[TMP31]], ptr [[TMP30]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE16]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE16]]:
-; OPTSIZE-NEXT:    [[TMP32:%.*]] = extractelement <16 x i1> [[TMP72]], i32 6
-; OPTSIZE-NEXT:    br i1 [[TMP32]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
-; OPTSIZE:       [[PRED_STORE_IF17]]:
-; OPTSIZE-NEXT:    [[TMP33:%.*]] = add i64 [[INDEX]], 6
-; OPTSIZE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP33]]
-; OPTSIZE-NEXT:    [[TMP35:%.*]] = extractelement <16 x i8> [[TMP7]], i32 6
-; OPTSIZE-NEXT:    store i8 [[TMP35]], ptr [[TMP34]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE18]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE18]]:
-; OPTSIZE-NEXT:    [[TMP36:%.*]] = extractelement <16 x i1> [[TMP72]], i32 7
-; OPTSIZE-NEXT:    br i1 [[TMP36]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
-; OPTSIZE:       [[PRED_STORE_IF19]]:
-; OPTSIZE-NEXT:    [[TMP37:%.*]] = add i64 [[INDEX]], 7
-; OPTSIZE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP37]]
-; OPTSIZE-NEXT:    [[TMP39:%.*]] = extractelement <16 x i8> [[TMP7]], i32 7
-; OPTSIZE-NEXT:    store i8 [[TMP39]], ptr [[TMP38]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE20]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE20]]:
-; OPTSIZE-NEXT:    [[TMP40:%.*]] = extractelement <16 x i1> [[TMP72]], i32 8
-; OPTSIZE-NEXT:    br i1 [[TMP40]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
-; OPTSIZE:       [[PRED_STORE_IF21]]:
-; OPTSIZE-NEXT:    [[TMP41:%.*]] = add i64 [[INDEX]], 8
-; OPTSIZE-NEXT:    [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP41]]
-; OPTSIZE-NEXT:    [[TMP43:%.*]] = extractelement <16 x i8> [[TMP7]], i32 8
-; OPTSIZE-NEXT:    store i8 [[TMP43]], ptr [[TMP42]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE22]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE22]]:
-; OPTSIZE-NEXT:    [[TMP44:%.*]] = extractelement <16 x i1> [[TMP72]], i32 9
-; OPTSIZE-NEXT:    br i1 [[TMP44]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
-; OPTSIZE:       [[PRED_STORE_IF23]]:
-; OPTSIZE-NEXT:    [[TMP45:%.*]] = add i64 [[INDEX]], 9
-; OPTSIZE-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP45]]
-; OPTSIZE-NEXT:    [[TMP47:%.*]] = extractelement <16 x i8> [[TMP7]], i32 9
-; OPTSIZE-NEXT:    store i8 [[TMP47]], ptr [[TMP46]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE24]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE24]]:
-; OPTSIZE-NEXT:    [[TMP48:%.*]] = extractelement <16 x i1> [[TMP72]], i32 10
-; OPTSIZE-NEXT:    br i1 [[TMP48]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
-; OPTSIZE:       [[PRED_STORE_IF25]]:
-; OPTSIZE-NEXT:    [[TMP49:%.*]] = add i64 [[INDEX]], 10
-; OPTSIZE-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP49]]
-; OPTSIZE-NEXT:    [[TMP51:%.*]] = extractelement <16 x i8> [[TMP7]], i32 10
-; OPTSIZE-NEXT:    store i8 [[TMP51]], ptr [[TMP50]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE26]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE26]]:
-; OPTSIZE-NEXT:    [[TMP52:%.*]] = extractelement <16 x i1> [[TMP72]], i32 11
-; OPTSIZE-NEXT:    br i1 [[TMP52]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
-; OPTSIZE:       [[PRED_STORE_IF27]]:
-; OPTSIZE-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX]], 11
-; OPTSIZE-NEXT:    [[TMP54:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP53]]
-; OPTSIZE-NEXT:    [[TMP55:%.*]] = extractelement <16 x i8> [[TMP7]], i32 11
-; OPTSIZE-NEXT:    store i8 [[TMP55]], ptr [[TMP54]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE28]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE28]]:
-; OPTSIZE-NEXT:    [[TMP56:%.*]] = extractelement <16 x i1> [[TMP72]], i32 12
-; OPTSIZE-NEXT:    br i1 [[TMP56]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
-; OPTSIZE:       [[PRED_STORE_IF29]]:
-; OPTSIZE-NEXT:    [[TMP57:%.*]] = add i64 [[INDEX]], 12
-; OPTSIZE-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP57]]
-; OPTSIZE-NEXT:    [[TMP59:%.*]] = extractelement <16 x i8> [[TMP7]], i32 12
-; OPTSIZE-NEXT:    store i8 [[TMP59]], ptr [[TMP58]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE30]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE30]]:
-; OPTSIZE-NEXT:    [[TMP60:%.*]] = extractelement <16 x i1> [[TMP72]], i32 13
-; OPTSIZE-NEXT:    br i1 [[TMP60]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
-; OPTSIZE:       [[PRED_STORE_IF31]]:
-; OPTSIZE-NEXT:    [[TMP61:%.*]] = add i64 [[INDEX]], 13
-; OPTSIZE-NEXT:    [[TMP62:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP61]]
-; OPTSIZE-NEXT:    [[TMP63:%.*]] = extractelement <16 x i8> [[TMP7]], i32 13
-; OPTSIZE-NEXT:    store i8 [[TMP63]], ptr [[TMP62]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE32]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE32]]:
-; OPTSIZE-NEXT:    [[TMP64:%.*]] = extractelement <16 x i1> [[TMP72]], i32 14
-; OPTSIZE-NEXT:    br i1 [[TMP64]], label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
-; OPTSIZE:       [[PRED_STORE_IF33]]:
-; OPTSIZE-NEXT:    [[TMP65:%.*]] = add i64 [[INDEX]], 14
-; OPTSIZE-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP65]]
-; OPTSIZE-NEXT:    [[TMP67:%.*]] = extractelement <16 x i8> [[TMP7]], i32 14
-; OPTSIZE-NEXT:    store i8 [[TMP67]], ptr [[TMP66]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE34]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE34]]:
-; OPTSIZE-NEXT:    [[TMP68:%.*]] = extractelement <16 x i1> [[TMP72]], i32 15
-; OPTSIZE-NEXT:    br i1 [[TMP68]], label %[[PRED_STORE_IF35:.*]], label %[[PRED_STORE_CONTINUE36]]
-; OPTSIZE:       [[PRED_STORE_IF35]]:
-; OPTSIZE-NEXT:    [[TMP69:%.*]] = add i64 [[INDEX]], 15
-; OPTSIZE-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP69]]
-; OPTSIZE-NEXT:    [[TMP71:%.*]] = extractelement <16 x i8> [[TMP7]], i32 15
-; OPTSIZE-NEXT:    store i8 [[TMP71]], ptr [[TMP70]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE36]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE36]]:
-; OPTSIZE-NEXT:    [[VEC_IND_NEXT]] = add <16 x i8> [[VEC_IND]], splat (i8 16)
-; OPTSIZE-NEXT:    [[VEC_IND_NEXT2]] = add <16 x i8> [[VEC_IND1]], splat (i8 16)
-; OPTSIZE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; OPTSIZE-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; OPTSIZE:       [[MIDDLE_BLOCK]]:
-; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
-; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i8
 ; OPTSIZE-NEXT:    [[MUL:%.*]] = mul i8 [[A]], [[TMP0]]
 ; OPTSIZE-NEXT:    [[SHR:%.*]] = lshr i8 [[TMP0]], 1
@@ -616,7 +442,7 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
 ; OPTSIZE-NEXT:    store i8 [[ADD10]], ptr [[ARRAYIDX]], align 1
 ; OPTSIZE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; OPTSIZE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 15
-; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]]
 ; OPTSIZE:       [[FOR_COND_CLEANUP]]:
 ; OPTSIZE-NEXT:    ret void
 ;
@@ -800,7 +626,7 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; OPTSIZE-NEXT:    [[TMP24:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
 ; OPTSIZE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[DOTSPLAT]]
 ; OPTSIZE-NEXT:    [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP24]], i32 0
-; OPTSIZE-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; OPTSIZE:       [[SCALAR_PH]]:
@@ -820,7 +646,7 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; OPTSIZE-NEXT:    store i8 [[ADD10]], ptr [[ARRAYIDX]], align 1
 ; OPTSIZE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; OPTSIZE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 15
-; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; OPTSIZE:       [[FOR_COND_CLEANUP]]:
 ; OPTSIZE-NEXT:    ret void
 ;
@@ -1008,7 +834,7 @@ define void @dont_vectorize_with_minsize() {
 ; OPTSIZE-NEXT:    store <8 x i16> [[TMP9]], ptr [[TMP7]], align 2
 ; OPTSIZE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; OPTSIZE-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; OPTSIZE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
 ; OPTSIZE:       [[SCALAR_PH]]:
@@ -1028,7 +854,7 @@ define void @dont_vectorize_with_minsize() {
 ; OPTSIZE-NEXT:    store i16 [[ADD]], ptr [[ARRAYIDX4]], align 2
 ; OPTSIZE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; OPTSIZE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
-; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; OPTSIZE:       [[FOR_COND_CLEANUP]]:
 ; OPTSIZE-NEXT:    ret void
 ;
@@ -1187,7 +1013,7 @@ define void @vectorization_forced_minsize_reduce_width() {
 ; OPTSIZE-NEXT:    store <8 x i16> [[TMP9]], ptr [[TMP7]], align 2
 ; OPTSIZE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; OPTSIZE-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; OPTSIZE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
 ; OPTSIZE:       [[SCALAR_PH]]:
@@ -1207,7 +1033,7 @@ define void @vectorization_forced_minsize_reduce_width() {
 ; OPTSIZE-NEXT:    store i16 [[ADD]], ptr [[ARRAYIDX4]], align 2
 ; OPTSIZE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; OPTSIZE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
-; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; OPTSIZE:       [[FOR_COND_CLEANUP]]:
 ; OPTSIZE-NEXT:    ret void
 ;
@@ -1285,4 +1111,3 @@ attributes #0 = { "target-features"="+sve" }
 
 !0 = distinct !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
-
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
index 1e91fc9c7c56d..1ce272bcfb783 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
@@ -224,7 +224,6 @@ for.cond.cleanup:
 
 ; This should be vectorized and tail predicated without optsize, as that's
 ; faster, but not with optsize, as it's much larger.
-; FIXME: Currently we avoid tail predication only with minsize
 define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n) {
 ; DEFAULT-LABEL: define void @tail_predicate_without_optsize(
 ; DEFAULT-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) {
@@ -424,182 +423,9 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
 ; OPTSIZE-LABEL: define void @tail_predicate_without_optsize(
 ; OPTSIZE-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
 ; OPTSIZE-NEXT:  [[ENTRY:.*]]:
-; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; OPTSIZE:       [[VECTOR_PH]]:
-; OPTSIZE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[A]], i64 0
-; OPTSIZE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
-; OPTSIZE-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <16 x i8> poison, i8 [[B]], i64 0
-; OPTSIZE-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT3]], <16 x i8> poison, <16 x i32> zeroinitializer
-; OPTSIZE-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <16 x i8> poison, i8 [[C]], i64 0
-; OPTSIZE-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT5]], <16 x i8> poison, <16 x i32> zeroinitializer
-; OPTSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
-; OPTSIZE:       [[VECTOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE36:.*]] ]
-; OPTSIZE-NEXT:    [[VEC_IND:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE36]] ]
-; OPTSIZE-NEXT:    [[VEC_IND1:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[PRED_STORE_CONTINUE36]] ]
-; OPTSIZE-NEXT:    [[TMP72:%.*]] = icmp ule <16 x i8> [[VEC_IND]], splat (i8 14)
-; OPTSIZE-NEXT:    [[TMP1:%.*]] = mul <16 x i8> [[BROADCAST_SPLAT]], [[VEC_IND1]]
-; OPTSIZE-NEXT:    [[TMP2:%.*]] = lshr <16 x i8> [[VEC_IND1]], splat (i8 1)
-; OPTSIZE-NEXT:    [[TMP3:%.*]] = mul <16 x i8> [[TMP2]], [[BROADCAST_SPLAT4]]
-; OPTSIZE-NEXT:    [[TMP4:%.*]] = add <16 x i8> [[TMP3]], [[TMP1]]
-; OPTSIZE-NEXT:    [[TMP5:%.*]] = lshr <16 x i8> [[VEC_IND1]], splat (i8 2)
-; OPTSIZE-NEXT:    [[TMP6:%.*]] = mul <16 x i8> [[TMP5]], [[BROADCAST_SPLAT6]]
-; OPTSIZE-NEXT:    [[TMP7:%.*]] = add <16 x i8> [[TMP4]], [[TMP6]]
-; OPTSIZE-NEXT:    [[TMP8:%.*]] = extractelement <16 x i1> [[TMP72]], i32 0
-; OPTSIZE-NEXT:    br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; OPTSIZE:       [[PRED_STORE_IF]]:
-; OPTSIZE-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
-; OPTSIZE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP9]]
-; OPTSIZE-NEXT:    [[TMP11:%.*]] = extractelement <16 x i8> [[TMP7]], i32 0
-; OPTSIZE-NEXT:    store i8 [[TMP11]], ptr [[TMP10]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE]]:
-; OPTSIZE-NEXT:    [[TMP12:%.*]] = extractelement <16 x i1> [[TMP72]], i32 1
-; OPTSIZE-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; OPTSIZE:       [[PRED_STORE_IF7]]:
-; OPTSIZE-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 1
-; OPTSIZE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP13]]
-; OPTSIZE-NEXT:    [[TMP15:%.*]] = extractelement <16 x i8> [[TMP7]], i32 1
-; OPTSIZE-NEXT:    store i8 [[TMP15]], ptr [[TMP14]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE8]]:
-; OPTSIZE-NEXT:    [[TMP16:%.*]] = extractelement <16 x i1> [[TMP72]], i32 2
-; OPTSIZE-NEXT:    br i1 [[TMP16]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; OPTSIZE:       [[PRED_STORE_IF9]]:
-; OPTSIZE-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 2
-; OPTSIZE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP17]]
-; OPTSIZE-NEXT:    [[TMP19:%.*]] = extractelement <16 x i8> [[TMP7]], i32 2
-; OPTSIZE-NEXT:    store i8 [[TMP19]], ptr [[TMP18]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE10]]:
-; OPTSIZE-NEXT:    [[TMP20:%.*]] = extractelement <16 x i1> [[TMP72]], i32 3
-; OPTSIZE-NEXT:    br i1 [[TMP20]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; OPTSIZE:       [[PRED_STORE_IF11]]:
-; OPTSIZE-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX]], 3
-; OPTSIZE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP21]]
-; OPTSIZE-NEXT:    [[TMP23:%.*]] = extractelement <16 x i8> [[TMP7]], i32 3
-; OPTSIZE-NEXT:    store i8 [[TMP23]], ptr [[TMP22]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE12]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE12]]:
-; OPTSIZE-NEXT:    [[TMP24:%.*]] = extractelement <16 x i1> [[TMP72]], i32 4
-; OPTSIZE-NEXT:    br i1 [[TMP24]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; OPTSIZE:       [[PRED_STORE_IF13]]:
-; OPTSIZE-NEXT:    [[TMP25:%.*]] = add i64 [[INDEX]], 4
-; OPTSIZE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP25]]
-; OPTSIZE-NEXT:    [[TMP27:%.*]] = extractelement <16 x i8> [[TMP7]], i32 4
-; OPTSIZE-NEXT:    store i8 [[TMP27]], ptr [[TMP26]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE14]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE14]]:
-; OPTSIZE-NEXT:    [[TMP28:%.*]] = extractelement <16 x i1> [[TMP72]], i32 5
-; OPTSIZE-NEXT:    br i1 [[TMP28]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
-; OPTSIZE:       [[PRED_STORE_IF15]]:
-; OPTSIZE-NEXT:    [[TMP29:%.*]] = add i64 [[INDEX]], 5
-; OPTSIZE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP29]]
-; OPTSIZE-NEXT:    [[TMP31:%.*]] = extractelement <16 x i8> [[TMP7]], i32 5
-; OPTSIZE-NEXT:    store i8 [[TMP31]], ptr [[TMP30]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE16]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE16]]:
-; OPTSIZE-NEXT:    [[TMP32:%.*]] = extractelement <16 x i1> [[TMP72]], i32 6
-; OPTSIZE-NEXT:    br i1 [[TMP32]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
-; OPTSIZE:       [[PRED_STORE_IF17]]:
-; OPTSIZE-NEXT:    [[TMP33:%.*]] = add i64 [[INDEX]], 6
-; OPTSIZE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP33]]
-; OPTSIZE-NEXT:    [[TMP35:%.*]] = extractelement <16 x i8> [[TMP7]], i32 6
-; OPTSIZE-NEXT:    store i8 [[TMP35]], ptr [[TMP34]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE18]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE18]]:
-; OPTSIZE-NEXT:    [[TMP36:%.*]] = extractelement <16 x i1> [[TMP72]], i32 7
-; OPTSIZE-NEXT:    br i1 [[TMP36]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
-; OPTSIZE:       [[PRED_STORE_IF19]]:
-; OPTSIZE-NEXT:    [[TMP37:%.*]] = add i64 [[INDEX]], 7
-; OPTSIZE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP37]]
-; OPTSIZE-NEXT:    [[TMP39:%.*]] = extractelement <16 x i8> [[TMP7]], i32 7
-; OPTSIZE-NEXT:    store i8 [[TMP39]], ptr [[TMP38]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE20]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE20]]:
-; OPTSIZE-NEXT:    [[TMP40:%.*]] = extractelement <16 x i1> [[TMP72]], i32 8
-; OPTSIZE-NEXT:    br i1 [[TMP40]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
-; OPTSIZE:       [[PRED_STORE_IF21]]:
-; OPTSIZE-NEXT:    [[TMP41:%.*]] = add i64 [[INDEX]], 8
-; OPTSIZE-NEXT:    [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP41]]
-; OPTSIZE-NEXT:    [[TMP43:%.*]] = extractelement <16 x i8> [[TMP7]], i32 8
-; OPTSIZE-NEXT:    store i8 [[TMP43]], ptr [[TMP42]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE22]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE22]]:
-; OPTSIZE-NEXT:    [[TMP44:%.*]] = extractelement <16 x i1> [[TMP72]], i32 9
-; OPTSIZE-NEXT:    br i1 [[TMP44]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
-; OPTSIZE:       [[PRED_STORE_IF23]]:
-; OPTSIZE-NEXT:    [[TMP45:%.*]] = add i64 [[INDEX]], 9
-; OPTSIZE-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP45]]
-; OPTSIZE-NEXT:    [[TMP47:%.*]] = extractelement <16 x i8> [[TMP7]], i32 9
-; OPTSIZE-NEXT:    store i8 [[TMP47]], ptr [[TMP46]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE24]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE24]]:
-; OPTSIZE-NEXT:    [[TMP48:%.*]] = extractelement <16 x i1> [[TMP72]], i32 10
-; OPTSIZE-NEXT:    br i1 [[TMP48]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
-; OPTSIZE:       [[PRED_STORE_IF25]]:
-; OPTSIZE-NEXT:    [[TMP49:%.*]] = add i64 [[INDEX]], 10
-; OPTSIZE-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP49]]
-; OPTSIZE-NEXT:    [[TMP51:%.*]] = extractelement <16 x i8> [[TMP7]], i32 10
-; OPTSIZE-NEXT:    store i8 [[TMP51]], ptr [[TMP50]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE26]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE26]]:
-; OPTSIZE-NEXT:    [[TMP52:%.*]] = extractelement <16 x i1> [[TMP72]], i32 11
-; OPTSIZE-NEXT:    br i1 [[TMP52]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
-; OPTSIZE:       [[PRED_STORE_IF27]]:
-; OPTSIZE-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX]], 11
-; OPTSIZE-NEXT:    [[TMP54:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP53]]
-; OPTSIZE-NEXT:    [[TMP55:%.*]] = extractelement <16 x i8> [[TMP7]], i32 11
-; OPTSIZE-NEXT:    store i8 [[TMP55]], ptr [[TMP54]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE28]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE28]]:
-; OPTSIZE-NEXT:    [[TMP56:%.*]] = extractelement <16 x i1> [[TMP72]], i32 12
-; OPTSIZE-NEXT:    br i1 [[TMP56]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
-; OPTSIZE:       [[PRED_STORE_IF29]]:
-; OPTSIZE-NEXT:    [[TMP57:%.*]] = add i64 [[INDEX]], 12
-; OPTSIZE-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP57]]
-; OPTSIZE-NEXT:    [[TMP59:%.*]] = extractelement <16 x i8> [[TMP7]], i32 12
-; OPTSIZE-NEXT:    store i8 [[TMP59]], ptr [[TMP58]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE30]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE30]]:
-; OPTSIZE-NEXT:    [[TMP60:%.*]] = extractelement <16 x i1> [[TMP72]], i32 13
-; OPTSIZE-NEXT:    br i1 [[TMP60]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
-; OPTSIZE:       [[PRED_STORE_IF31]]:
-; OPTSIZE-NEXT:    [[TMP61:%.*]] = add i64 [[INDEX]], 13
-; OPTSIZE-NEXT:    [[TMP62:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP61]]
-; OPTSIZE-NEXT:    [[TMP63:%.*]] = extractelement <16 x i8> [[TMP7]], i32 13
-; OPTSIZE-NEXT:    store i8 [[TMP63]], ptr [[TMP62]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE32]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE32]]:
-; OPTSIZE-NEXT:    [[TMP64:%.*]] = extractelement <16 x i1> [[TMP72]], i32 14
-; OPTSIZE-NEXT:    br i1 [[TMP64]], label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
-; OPTSIZE:       [[PRED_STORE_IF33]]:
-; OPTSIZE-NEXT:    [[TMP65:%.*]] = add i64 [[INDEX]], 14
-; OPTSIZE-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP65]]
-; OPTSIZE-NEXT:    [[TMP67:%.*]] = extractelement <16 x i8> [[TMP7]], i32 14
-; OPTSIZE-NEXT:    store i8 [[TMP67]], ptr [[TMP66]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE34]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE34]]:
-; OPTSIZE-NEXT:    [[TMP68:%.*]] = extractelement <16 x i1> [[TMP72]], i32 15
-; OPTSIZE-NEXT:    br i1 [[TMP68]], label %[[PRED_STORE_IF35:.*]], label %[[PRED_STORE_CONTINUE36]]
-; OPTSIZE:       [[PRED_STORE_IF35]]:
-; OPTSIZE-NEXT:    [[TMP69:%.*]] = add i64 [[INDEX]], 15
-; OPTSIZE-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP69]]
-; OPTSIZE-NEXT:    [[TMP71:%.*]] = extractelement <16 x i8> [[TMP7]], i32 15
-; OPTSIZE-NEXT:    store i8 [[TMP71]], ptr [[TMP70]], align 1
-; OPTSIZE-NEXT:    br label %[[PRED_STORE_CONTINUE36]]
-; OPTSIZE:       [[PRED_STORE_CONTINUE36]]:
-; OPTSIZE-NEXT:    [[VEC_IND_NEXT]] = add <16 x i8> [[VEC_IND]], splat (i8 16)
-; OPTSIZE-NEXT:    [[VEC_IND_NEXT2]] = add <16 x i8> [[VEC_IND1]], splat (i8 16)
-; OPTSIZE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; OPTSIZE-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; OPTSIZE:       [[MIDDLE_BLOCK]]:
-; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
-; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i8
 ; OPTSIZE-NEXT:    [[MUL:%.*]] = mul i8 [[A]], [[TMP0]]
 ; OPTSIZE-NEXT:    [[SHR:%.*]] = lshr i8 [[TMP0]], 1
@@ -612,7 +438,7 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
 ; OPTSIZE-NEXT:    store i8 [[ADD10]], ptr [[ARRAYIDX]], align 1
 ; OPTSIZE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; OPTSIZE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 15
-; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]]
 ; OPTSIZE:       [[FOR_COND_CLEANUP]]:
 ; OPTSIZE-NEXT:    ret void
 ;
@@ -737,7 +563,7 @@ define void @dont_vectorize_with_minsize() {
 ; OPTSIZE-NEXT:    store <4 x i16> [[TMP9]], ptr [[TMP7]], align 2
 ; OPTSIZE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
 ; OPTSIZE-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; OPTSIZE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
 ; OPTSIZE:       [[SCALAR_PH]]:
@@ -757,7 +583,7 @@ define void @dont_vectorize_with_minsize() {
 ; OPTSIZE-NEXT:    store i16 [[ADD]], ptr [[ARRAYIDX4]], align 2
 ; OPTSIZE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; OPTSIZE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
-; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; OPTSIZE:       [[FOR_COND_CLEANUP]]:
 ; OPTSIZE-NEXT:    ret void
 ;
@@ -906,7 +732,7 @@ define void @vectorization_forced() {
 ; OPTSIZE-NEXT:    store <4 x i16> [[TMP9]], ptr [[TMP7]], align 2
 ; OPTSIZE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
 ; OPTSIZE-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
-; OPTSIZE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
 ; OPTSIZE:       [[SCALAR_PH]]:
@@ -926,7 +752,7 @@ define void @vectorization_forced() {
 ; OPTSIZE-NEXT:    store i16 [[ADD]], ptr [[ARRAYIDX4]], align 2
 ; OPTSIZE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; OPTSIZE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
-; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; OPTSIZE:       [[FOR_COND_CLEANUP]]:
 ; OPTSIZE-NEXT:    ret void
 ;
@@ -1002,4 +828,3 @@ for.cond.cleanup:
 
 !0 = distinct !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
-
diff --git a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
index c38410091d7bc..5fc9e64147801 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
@@ -220,7 +220,7 @@ for.body:
   %sum.1 = add nuw nsw i32 %add, %sum.0
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, %N
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !6
 
 for.cond.cleanup:
   ret i32 %sum.1
@@ -233,6 +233,5 @@ attributes #0 = { nounwind optsize uwtable "target-cpu"="core-avx2" "target-feat
 !7 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
 !8 = !{!"llvm.loop.vectorize.enable", i1 true}
 
-!10 = distinct !{!10, !11, !12}
+!10 = distinct !{!10, !11}
 !11 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}
-!12 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
index 0b6b789c1dcea..b6aea9c6c27c5 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
@@ -10,7 +10,7 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 ; -force-vector-interleave, but is a multiple of the internally computed MaxVF;
 ; e.g., when all types are i32 lead to MaxVF=1.
 
-define void @pr45679(ptr %A) optsize {
+define void @pr45679(ptr %A) {
 ; CHECK-LABEL: @pr45679(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return.ll b/llvm/test/Transforms/LoopVectorize/struct-return.ll
index 1b2a809a552d8..57d0aa3787c45 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return.ll
@@ -211,7 +211,7 @@ exit:
 ; Test crafted to exercise computePredInstDiscount with struct results
 ; (mainly it does not crash).
 ; CHECK-REMARKS: remark: {{.*}} vectorized loop
-define void @scalarized_predicated_struct_return(ptr %a) optsize {
+define void @scalarized_predicated_struct_return(ptr %a) {
 ; CHECK-LABEL: define void @scalarized_predicated_struct_return
 ; CHECK:  vector.body:
 ; CHECK:  pred.store.if:
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-counting-down.ll
index d2d99827d5f35..40a1eb477a212 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-counting-down.ll
@@ -43,7 +43,7 @@ while.end:
 ; Make sure a loop is successfully vectorized with fold-tail when the backedge
 ; taken count is constant and used inside the loop. Issue revealed by D76992.
 ;
-define void @reuse_const_btc(ptr %A) optsize {
+define void @reuse_const_btc(ptr %A) {
 ; CHECK-LABEL: @reuse_const_btc
 ; CHECK: {{%.*}} = icmp ule <4 x i32> {{%.*}}, splat (i32 13)
 ; CHECK: {{%.*}} = select <4 x i1> {{%.*}}, <4 x i32> splat (i32 12), <4 x i32> splat (i32 13)
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index cf9991d68fce6..43172cc2f170a 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -1035,32 +1035,32 @@ exit:
 
 ; Test case with a dead GEP between the load and store regions. Dead recipes
 ; need to be removed before merging.
-define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr noalias %dst) optsize {
+define void @merge_with_dead_gep_between_regions(i32 %n, i32 %k, ptr noalias %src, ptr noalias %dst) {
 ; CHECK-LABEL: LV: Checking a loop in 'merge_with_dead_gep_between_regions'
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF
 ; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
 ; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<entry>:
 ; CHECK-NEXT: Successor(s): vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
+; CHECK-NEXT:   vp<[[END:%.+]]> = DERIVED-IV ir<%n> + vp<[[VEC_TC]]> * ir<-1>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:     ir<%iv> = WIDEN-INDUCTION ir<%n>, ir<-1>, vp<[[VF]]>
 ; CHECK-NEXT:     vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT:     EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
-; CHECK-NEXT:     EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_IV]]>, vp<[[BTC]]>
+; CHECK-NEXT:     WIDEN ir<%cond> = icmp ult ir<%iv>, ir<%k>
 ; CHECK-NEXT:   Successor(s): pred.store
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   <xVFxUF> pred.store: {
 ; CHECK-NEXT:     pred.store.entry:
-; CHECK-NEXT:       BRANCH-ON-MASK vp<[[MASK]]>
+; CHECK-NEXT:       BRANCH-ON-MASK ir<%cond>
 ; CHECK-NEXT:     Successor(s): pred.store.if, pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:     pred.store.if:
@@ -1074,19 +1074,31 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
 ; CHECK-NEXT:     pred.store.continue:
 ; CHECK-NEXT:     No successors
 ; CHECK-NEXT:   }
-; CHECK-NEXT:   Successor(s): loop.1
+; CHECK-NEXT:   Successor(s): loop.then.1
 ; CHECK-EMPTY:
-; CHECK-NEXT:   loop.1:
-; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:   loop.then.1:
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count  vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-NEXT:   EMIT vp<[[CMP:%.+]]> = icmp eq ir<%n>, vp<[[VEC_TC]]>
+; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT:   EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[END]]>, ir<%n>
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT:   IR   %iv = phi i32 [ %n, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<[[RESUME]]> from scalar.ph)
+; CHECK-NEXT:   IR   %iv.next = add nsw i32 %iv, -1
+; CHECK-NEXT:   IR   %cond = icmp ult i32 %iv, %k
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: ir-bb<exit>:
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ;
@@ -1094,13 +1106,20 @@ entry:
   br label %loop
 
 loop:
-  %iv = phi i32[ %n, %entry ], [ %iv.next, %loop ]
+  %iv = phi i32[ %n, %entry ], [ %iv.next, %loop.latch ]
   %iv.next = add nsw i32 %iv, -1
+  %cond = icmp ult i32 %iv, %k
+  br i1 %cond, label %loop.then, label %loop.latch
+
+loop.then:
   %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv
   %l = load i32, ptr %gep.src, align 16
   %dead_gep = getelementptr inbounds i32, ptr %dst, i64 1
   %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv
   store i32 %l, ptr %gep.dst, align 16
+  br label %loop.latch
+
+loop.latch:
   %ec = icmp eq i32 %iv.next, 0
   br i1 %ec, label %exit, label %loop
 

From c536967af123c30b43f186133e8719e0090f24a6 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Thu, 17 Apr 2025 07:11:51 -0400
Subject: [PATCH 252/710] Split _Countof tests into two files; NFC

Post-commit review feedback during the language WG meeting requested
that I try to generalize the testing for this rather than only test on
a single target as we previously did.

The tests which are hard to generalize are the VLA tests, so those
still have specific triples in the RUN line, but have more coverage and
a comment explaining that the test should generalize to all targets.
---
 clang/test/C/C2y/n3369_2.c | 114 +++----------------------
 clang/test/C/C2y/n3369_3.c | 166 +++++++++++++++++++++++++++++++++++++
 2 files changed, 178 insertions(+), 102 deletions(-)
 create mode 100644 clang/test/C/C2y/n3369_3.c

diff --git a/clang/test/C/C2y/n3369_2.c b/clang/test/C/C2y/n3369_2.c
index 0cbf362d38d4e..9cab828b461b5 100644
--- a/clang/test/C/C2y/n3369_2.c
+++ b/clang/test/C/C2y/n3369_2.c
@@ -1,116 +1,26 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
-// RUN: %clang_cc1 -std=c2y -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -std=c2y -emit-llvm -o - %s | FileCheck %s
 
-// This tests the codegen behavior for _Countof.
-// CHECK-LABEL: define dso_local i32 @test1(
+// This tests the non-VLA codegen behavior for _Countof.
+
+typedef typeof(sizeof(0)) size_t;
+
+// CHECK-LABEL: define dso_local i64 @test1(
 // CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[ARRAY:%.*]] = alloca [12 x i32], align 16
-// CHECK-NEXT:    ret i32 12
+// CHECK-NEXT:    [[ARRAY:%.*]] = alloca [12 x i32], align
+// CHECK-NEXT:    ret i64 12
 //
-int test1() {
+size_t test1() {
   int array[12];
   return _Countof(array);
 }
 
-// CHECK-LABEL: define dso_local i32 @test2(
-// CHECK-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SAVED_STACK:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    [[__VLA_EXPR0:%.*]] = alloca i64, align 8
-// CHECK-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
-// CHECK-NEXT:    [[TMP2:%.*]] = call ptr @llvm.stacksave.p0()
-// CHECK-NEXT:    store ptr [[TMP2]], ptr [[SAVED_STACK]], align 8
-// CHECK-NEXT:    [[VLA:%.*]] = alloca i32, i64 [[TMP1]], align 16
-// CHECK-NEXT:    store i64 [[TMP1]], ptr [[__VLA_EXPR0]], align 8
-// CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP1]] to i32
-// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[SAVED_STACK]], align 8
-// CHECK-NEXT:    call void @llvm.stackrestore.p0(ptr [[TMP3]])
-// CHECK-NEXT:    ret i32 [[CONV]]
-//
-int test2(int n) {
-  int array[n];
-  return _Countof(array);
-}
-
-// CHECK-LABEL: define dso_local i32 @test3(
-// CHECK-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
-// CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP1]] to i32
-// CHECK-NEXT:    ret i32 [[CONV]]
-//
-int test3(int n) {
-  return _Countof(int[n]);
-}
-
-// CHECK-LABEL: define dso_local i32 @test4(
+// CHECK-LABEL: define dso_local i64 @test2(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    ret i32 100
+// CHECK-NEXT:    ret i64 100
 //
-int test4() {
+size_t test2() {
   return _Countof(float[100]);
 }
-
-// CHECK-LABEL: define dso_local i32 @test5(
-// CHECK-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SAVED_STACK:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    [[__VLA_EXPR0:%.*]] = alloca i64, align 8
-// CHECK-NEXT:    [[X:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
-// CHECK-NEXT:    [[TMP2:%.*]] = call ptr @llvm.stacksave.p0()
-// CHECK-NEXT:    store ptr [[TMP2]], ptr [[SAVED_STACK]], align 8
-// CHECK-NEXT:    [[VLA:%.*]] = alloca [7 x i32], i64 [[TMP1]], align 16
-// CHECK-NEXT:    store i64 [[TMP1]], ptr [[__VLA_EXPR0]], align 8
-// CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP1]] to i32
-// CHECK-NEXT:    store i32 [[CONV]], ptr [[X]], align 4
-// CHECK-NEXT:    store i32 7, ptr [[Y]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[X]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[Y]], align 4
-// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[SAVED_STACK]], align 8
-// CHECK-NEXT:    call void @llvm.stackrestore.p0(ptr [[TMP5]])
-// CHECK-NEXT:    ret i32 [[ADD]]
-//
-int test5(int n) {
-  int array[n][7];
-  int x = _Countof(array);
-  int y = _Countof(*array);
-  return x + y;
-}
-
-// CHECK-LABEL: define dso_local void @test6(
-// CHECK-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[X:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    store i32 7, ptr [[X]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-// CHECK-NEXT:    store i32 [[INC]], ptr [[N_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
-// CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP1]] to i32
-// CHECK-NEXT:    store i32 [[CONV]], ptr [[Y]], align 4
-// CHECK-NEXT:    ret void
-//
-void test6(int n) {
-  // n should not be evaluated in this case because the operator does not need
-  // to evaluate it to know the result is 7.
-  int x = _Countof(int[7][n++]);
-  // n should be evaluated in this case, however.
-  int y = _Countof(int[n++][7]);
-}
diff --git a/clang/test/C/C2y/n3369_3.c b/clang/test/C/C2y/n3369_3.c
new file mode 100644
index 0000000000000..d3909b2e5bc92
--- /dev/null
+++ b/clang/test/C/C2y/n3369_3.c
@@ -0,0 +1,166 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -std=c2y -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-64
+// RUN: %clang_cc1 -std=c2y -triple aarch64-unknown-unknown -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-64
+// RUN: %clang_cc1 -std=c2y -triple ppc64le-unknown-unknown -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-64
+// RUN: %clang_cc1 -std=c2y -triple i686-unknown-unknown -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-32
+
+// This tests the codegen behavior for VLA use in _Countof, which builds upon
+// the code generation for sizeof. It tests a limited set of triples due to
+// variations in VLA handling between platforms, but this test should
+// generalize to all targets.
+
+typedef typeof(sizeof(0)) size_t;
+
+// CHECK-64-LABEL: define dso_local i64 @test1(
+// CHECK-64-SAME: i32 noundef{{( signext)?}} [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-64-NEXT:  [[ENTRY:.*:]]
+// CHECK-64-NEXT:    [[N_ADDR:%.*]] = alloca i32, align
+// CHECK-64-NEXT:    [[SAVED_STACK:%.*]] = alloca ptr, align
+// CHECK-64-NEXT:    [[__VLA_EXPR0:%.*]] = alloca i64, align
+// CHECK-64-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+// CHECK-64-NEXT:    [[TMP2:%.*]] = call ptr @llvm.stacksave.p0()
+// CHECK-64-NEXT:    store ptr [[TMP2]], ptr [[SAVED_STACK]], align
+// CHECK-64-NEXT:    [[VLA:%.*]] = alloca i32, i64 [[TMP1]], align
+// CHECK-64-NEXT:    store i64 [[TMP1]], ptr [[__VLA_EXPR0]], align
+// CHECK-64-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[SAVED_STACK]], align
+// CHECK-64-NEXT:    call void @llvm.stackrestore.p0(ptr [[TMP3]])
+// CHECK-64-NEXT:    ret i64 [[TMP1]]
+//
+// CHECK-32-LABEL: define dso_local i32 @test1(
+// CHECK-32-SAME: i32 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-32-NEXT:  [[ENTRY:.*:]]
+// CHECK-32-NEXT:    [[N_ADDR:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    [[SAVED_STACK:%.*]] = alloca ptr, align
+// CHECK-32-NEXT:    [[__VLA_EXPR0:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    [[TMP1:%.*]] = call ptr @llvm.stacksave.p0()
+// CHECK-32-NEXT:    store ptr [[TMP1]], ptr [[SAVED_STACK]], align
+// CHECK-32-NEXT:    [[VLA:%.*]] = alloca i32, i32 [[TMP0]], align
+// CHECK-32-NEXT:    store i32 [[TMP0]], ptr [[__VLA_EXPR0]], align
+// CHECK-32-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[SAVED_STACK]], align
+// CHECK-32-NEXT:    call void @llvm.stackrestore.p0(ptr [[TMP2]])
+// CHECK-32-NEXT:    ret i32 [[TMP0]]
+//
+size_t test1(int n) {
+  int array[n];
+  return _Countof(array);
+}
+
+// CHECK-64-LABEL: define dso_local i64 @test2(
+// CHECK-64-SAME: i32 noundef{{( signext)?}} [[N:%.*]]) #[[ATTR0]] {
+// CHECK-64-NEXT:  [[ENTRY:.*:]]
+// CHECK-64-NEXT:    [[N_ADDR:%.*]] = alloca i32, align
+// CHECK-64-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+// CHECK-64-NEXT:    ret i64 [[TMP1]]
+//
+// CHECK-32-LABEL: define dso_local i32 @test2(
+// CHECK-32-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
+// CHECK-32-NEXT:  [[ENTRY:.*:]]
+// CHECK-32-NEXT:    [[N_ADDR:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    ret i32 [[TMP0]]
+//
+size_t test2(int n) {
+  return _Countof(int[n]);
+}
+
+// CHECK-64-LABEL: define dso_local i64 @test3(
+// CHECK-64-SAME: i32 noundef{{( signext)?}} [[N:%.*]]) #[[ATTR0]] {
+// CHECK-64-NEXT:  [[ENTRY:.*:]]
+// CHECK-64-NEXT:    [[N_ADDR:%.*]] = alloca i32, align
+// CHECK-64-NEXT:    [[SAVED_STACK:%.*]] = alloca ptr, align
+// CHECK-64-NEXT:    [[__VLA_EXPR0:%.*]] = alloca i64, align
+// CHECK-64-NEXT:    [[X:%.*]] = alloca i32, align
+// CHECK-64-NEXT:    [[Y:%.*]] = alloca i32, align
+// CHECK-64-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+// CHECK-64-NEXT:    [[TMP2:%.*]] = call ptr @llvm.stacksave.p0()
+// CHECK-64-NEXT:    store ptr [[TMP2]], ptr [[SAVED_STACK]], align
+// CHECK-64-NEXT:    [[VLA:%.*]] = alloca [7 x i32], i64 [[TMP1]], align
+// CHECK-64-NEXT:    store i64 [[TMP1]], ptr [[__VLA_EXPR0]], align
+// CHECK-64-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP1]] to i32
+// CHECK-64-NEXT:    store i32 [[CONV]], ptr [[X]], align
+// CHECK-64-NEXT:    store i32 7, ptr [[Y]], align
+// CHECK-64-NEXT:    [[TMP3:%.*]] = load i32, ptr [[X]], align
+// CHECK-64-NEXT:    [[TMP4:%.*]] = load i32, ptr [[Y]], align
+// CHECK-64-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK-64-NEXT:    [[CONV1:%.*]] = sext i32 [[ADD]] to i64
+// CHECK-64-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[SAVED_STACK]], align
+// CHECK-64-NEXT:    call void @llvm.stackrestore.p0(ptr [[TMP5]])
+// CHECK-64-NEXT:    ret i64 [[CONV1]]
+//
+// CHECK-32-LABEL: define dso_local i32 @test3(
+// CHECK-32-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
+// CHECK-32-NEXT:  [[ENTRY:.*:]]
+// CHECK-32-NEXT:    [[N_ADDR:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    [[SAVED_STACK:%.*]] = alloca ptr, align
+// CHECK-32-NEXT:    [[__VLA_EXPR0:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    [[X:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    [[Y:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    [[TMP1:%.*]] = call ptr @llvm.stacksave.p0()
+// CHECK-32-NEXT:    store ptr [[TMP1]], ptr [[SAVED_STACK]], align
+// CHECK-32-NEXT:    [[VLA:%.*]] = alloca [7 x i32], i32 [[TMP0]], align
+// CHECK-32-NEXT:    store i32 [[TMP0]], ptr [[__VLA_EXPR0]], align
+// CHECK-32-NEXT:    store i32 [[TMP0]], ptr [[X]], align
+// CHECK-32-NEXT:    store i32 7, ptr [[Y]], align
+// CHECK-32-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align
+// CHECK-32-NEXT:    [[TMP3:%.*]] = load i32, ptr [[Y]], align
+// CHECK-32-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK-32-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[SAVED_STACK]], align
+// CHECK-32-NEXT:    call void @llvm.stackrestore.p0(ptr [[TMP4]])
+// CHECK-32-NEXT:    ret i32 [[ADD]]
+//
+size_t test3(int n) {
+  int array[n][7];
+  int x = _Countof(array);
+  int y = _Countof(*array);
+  return x + y;
+}
+
+// CHECK-64-LABEL: define dso_local void @test4(
+// CHECK-64-SAME: i32 noundef{{( signext)?}} [[N:%.*]]) #[[ATTR0]] {
+// CHECK-64-NEXT:  [[ENTRY:.*:]]
+// CHECK-64-NEXT:    [[N_ADDR:%.*]] = alloca i32, align
+// CHECK-64-NEXT:    [[X:%.*]] = alloca i32, align
+// CHECK-64-NEXT:    [[Y:%.*]] = alloca i32, align
+// CHECK-64-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    store i32 7, ptr [[X]], align
+// CHECK-64-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
+// CHECK-64-NEXT:    store i32 [[INC]], ptr [[N_ADDR]], align
+// CHECK-64-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+// CHECK-64-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP1]] to i32
+// CHECK-64-NEXT:    store i32 [[CONV]], ptr [[Y]], align
+// CHECK-64-NEXT:    ret void
+//
+// CHECK-32-LABEL: define dso_local void @test4(
+// CHECK-32-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
+// CHECK-32-NEXT:  [[ENTRY:.*:]]
+// CHECK-32-NEXT:    [[N_ADDR:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    [[X:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    [[Y:%.*]] = alloca i32, align
+// CHECK-32-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    store i32 7, ptr [[X]], align
+// CHECK-32-NEXT:    [[TMP0:%.*]] = load i32, ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
+// CHECK-32-NEXT:    store i32 [[INC]], ptr [[N_ADDR]], align
+// CHECK-32-NEXT:    store i32 [[TMP0]], ptr [[Y]], align
+// CHECK-32-NEXT:    ret void
+//
+void test4(int n) {
+  // n should not be evaluated in this case because the operator does not need
+  // to evaluate it to know the result is 7.
+  int x = _Countof(int[7][n++]);
+  // n should be evaluated in this case, however.
+  int y = _Countof(int[n++][7]);
+}
+

From 4bd9b9e9adf9db1a326e2f2fa616c714beb83c4f Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Thu, 17 Apr 2025 07:32:12 -0400
Subject: [PATCH 253/710] Fix failing bot with changes to _Countof testing

This addresses the issue found by:
https://lab.llvm.org/buildbot/#/builders/190/builds/18484
---
 clang/test/C/C2y/n3369_2.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/test/C/C2y/n3369_2.c b/clang/test/C/C2y/n3369_2.c
index 9cab828b461b5..3e9e14283497e 100644
--- a/clang/test/C/C2y/n3369_2.c
+++ b/clang/test/C/C2y/n3369_2.c
@@ -5,7 +5,7 @@
 
 typedef typeof(sizeof(0)) size_t;
 
-// CHECK-LABEL: define dso_local i64 @test1(
+// CHECK-LABEL: define{{( dso_local)?}} i64 @test1(
 // CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[ARRAY:%.*]] = alloca [12 x i32], align
@@ -16,7 +16,7 @@ size_t test1() {
   return _Countof(array);
 }
 
-// CHECK-LABEL: define dso_local i64 @test2(
+// CHECK-LABEL: define{{( dso_local)?}} i64 @test2(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    ret i64 100

From a84a6f7dd68b218757e192fe21a806c80ef0b63d Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Thu, 17 Apr 2025 07:36:48 -0400
Subject: [PATCH 254/710] Fix this _Countof test for 32 vs 64-bits

This should address the issues found by:
https://lab.llvm.org/buildbot/#/builders/154/builds/14958
---
 clang/test/C/C2y/n3369_2.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/test/C/C2y/n3369_2.c b/clang/test/C/C2y/n3369_2.c
index 3e9e14283497e..d0862c591a3a7 100644
--- a/clang/test/C/C2y/n3369_2.c
+++ b/clang/test/C/C2y/n3369_2.c
@@ -5,21 +5,21 @@
 
 typedef typeof(sizeof(0)) size_t;
 
-// CHECK-LABEL: define{{( dso_local)?}} i64 @test1(
+// CHECK-LABEL: define{{( dso_local)?}} i{{64|32}} @test1(
 // CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[ARRAY:%.*]] = alloca [12 x i32], align
-// CHECK-NEXT:    ret i64 12
+// CHECK-NEXT:    ret i{{64|32}} 12
 //
 size_t test1() {
   int array[12];
   return _Countof(array);
 }
 
-// CHECK-LABEL: define{{( dso_local)?}} i64 @test2(
+// CHECK-LABEL: define{{( dso_local)?}} i{{64|32}} @test2(
 // CHECK-SAME: ) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    ret i64 100
+// CHECK-NEXT:    ret i{{64|32}} 100
 //
 size_t test2() {
   return _Countof(float[100]);

From 51295d6d56f938b878ca0e2bb0a749eb801e8202 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 17 Apr 2025 13:39:27 +0200
Subject: [PATCH 255/710] [clang][bytecode] Reject assignments in C (#136126)

Similar to what the current interpreter does.
---
 clang/lib/AST/ByteCode/Compiler.cpp |  6 +++++-
 clang/test/AST/ByteCode/c2y.c       | 23 +++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/AST/ByteCode/c2y.c

diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index d3eabc513a9ac..3e53f2a2c8557 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -863,8 +863,12 @@ bool Compiler<Emitter>::VisitBinaryOperator(const BinaryOperator *BO) {
       return this->VisitPointerArithBinOp(BO);
   }
 
-  // Assignmentes require us to evalute the RHS first.
+  // Assignments require us to evalute the RHS first.
   if (BO->getOpcode() == BO_Assign) {
+    // We don't support assignments in C.
+    if (!Ctx.getLangOpts().CPlusPlus)
+      return this->emitInvalid(BO);
+
     if (!visit(RHS) || !visit(LHS))
       return false;
     if (!this->emitFlip(*LT, *RT, BO))
diff --git a/clang/test/AST/ByteCode/c2y.c b/clang/test/AST/ByteCode/c2y.c
new file mode 100644
index 0000000000000..bbb163ab83833
--- /dev/null
+++ b/clang/test/AST/ByteCode/c2y.c
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple x86_64-linux %s -std=c2y -verify=expected,both -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -triple x86_64-linux %s -std=c2y -verify=ref,both
+
+// both-no-diagnostics
+
+struct S {
+  int x;
+  char c;
+  float f;
+};
+
+#define DECL_BUFFER(Ty, Name) alignas(Ty) unsigned char Name[sizeof(Ty)]
+
+struct T {
+  DECL_BUFFER(struct S, buffer);
+};
+
+int quorble() {
+  DECL_BUFFER(struct T, buffer);
+  ((struct S *)((struct T *)buffer)->buffer)->x = 12;
+  const struct S *s_ptr = (struct S *)((struct T *)buffer)->buffer;
+  return s_ptr->x;
+}

From d744596f7da7bb80536d231bd00080870a12112a Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Thu, 17 Apr 2025 07:51:54 -0400
Subject: [PATCH 256/710] [libc][workflows] fix fullbuild by installing
 clang-20/clang++-20 (#136074)

---
 .github/workflows/libc-fullbuild-tests.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/libc-fullbuild-tests.yml b/.github/workflows/libc-fullbuild-tests.yml
index 00d2696e7e586..54d0df99b687e 100644
--- a/.github/workflows/libc-fullbuild-tests.yml
+++ b/.github/workflows/libc-fullbuild-tests.yml
@@ -19,13 +19,13 @@ jobs:
         include:
           - os: ubuntu-24.04
             ccache-variant: sccache
-            c_compiler: clang
-            cpp_compiler: clang++
+            c_compiler: clang-20
+            cpp_compiler: clang++-20
           # TODO: remove ccache logic when https://github.com/hendrikmuhs/ccache-action/issues/279 is resolved.
           - os: ubuntu-24.04-arm
             ccache-variant: ccache
-            c_compiler: clang
-            cpp_compiler: clang++
+            c_compiler: clang-20
+            cpp_compiler: clang++-20
           # TODO: add back gcc build when it is fixed
           # - c_compiler: gcc
           #   cpp_compiler: g++
@@ -51,6 +51,9 @@ jobs:
     #   For more information, see https://wiki.debian.org/Multiarch/LibraryPathOverview
     - name: Prepare dependencies (Ubuntu)
       run: |
+        wget https://apt.llvm.org/llvm.sh
+        chmod +x llvm.sh
+        sudo ./llvm.sh 20
         sudo apt-get update
         sudo apt-get install -y libmpfr-dev libgmp-dev libmpc-dev ninja-build linux-libc-dev
         sudo ln -sf /usr/include/$(uname -p)-linux-gnu/asm /usr/include/asm

From f135ce6a9329124f1e508fcad2eedf5326a668fa Mon Sep 17 00:00:00 2001
From: Charles Zablit <zablitcharles@gmail.com>
Date: Thu, 17 Apr 2025 12:55:18 +0100
Subject: [PATCH 257/710] [lldb] Remove CompilerType::GetIndexOfFieldWithName
 (#135963)

This patch removes the unused `CompilerType::GetIndexOfFieldWithName` API (it wasn't used apart from in a single testcase). Given we have so many similarly named APIs already, it's best not to maintain this API that's not really used (and isnt tested).
---
 lldb/include/lldb/Symbol/CompilerType.h       |  6 ------
 lldb/source/Symbol/CompilerType.cpp           | 19 -------------------
 .../Platform/PlatformSiginfoTest.cpp          |  8 +++++---
 3 files changed, 5 insertions(+), 28 deletions(-)

diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 41a1676dabd76..3561bc70887e6 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -433,12 +433,6 @@ class CompilerType {
 
   CompilerDecl GetStaticFieldWithName(llvm::StringRef name) const;
 
-  uint32_t GetIndexOfFieldWithName(const char *name,
-                                   CompilerType *field_compiler_type = nullptr,
-                                   uint64_t *bit_offset_ptr = nullptr,
-                                   uint32_t *bitfield_bit_size_ptr = nullptr,
-                                   bool *is_bitfield_ptr = nullptr) const;
-
   llvm::Expected<CompilerType> GetChildCompilerTypeAtIndex(
       ExecutionContext *exe_ctx, size_t idx, bool transparent_pointers,
       bool omit_empty_base_classes, bool ignore_array_bounds,
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 22fdd24fc7cd5..8e89d006d08d3 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -893,25 +893,6 @@ CompilerDecl CompilerType::GetStaticFieldWithName(llvm::StringRef name) const {
   return CompilerDecl();
 }
 
-uint32_t CompilerType::GetIndexOfFieldWithName(
-    const char *name, CompilerType *field_compiler_type_ptr,
-    uint64_t *bit_offset_ptr, uint32_t *bitfield_bit_size_ptr,
-    bool *is_bitfield_ptr) const {
-  unsigned count = GetNumFields();
-  std::string field_name;
-  for (unsigned index = 0; index < count; index++) {
-    CompilerType field_compiler_type(
-        GetFieldAtIndex(index, field_name, bit_offset_ptr,
-                        bitfield_bit_size_ptr, is_bitfield_ptr));
-    if (strcmp(field_name.c_str(), name) == 0) {
-      if (field_compiler_type_ptr)
-        *field_compiler_type_ptr = field_compiler_type;
-      return index;
-    }
-  }
-  return UINT32_MAX;
-}
-
 llvm::Expected<CompilerType> CompilerType::GetChildCompilerTypeAtIndex(
     ExecutionContext *exe_ctx, size_t idx, bool transparent_pointers,
     bool omit_empty_base_classes, bool ignore_array_bounds,
diff --git a/lldb/unittests/Platform/PlatformSiginfoTest.cpp b/lldb/unittests/Platform/PlatformSiginfoTest.cpp
index 4b2c93a68a94a..b9f1fe4bbaadf 100644
--- a/lldb/unittests/Platform/PlatformSiginfoTest.cpp
+++ b/lldb/unittests/Platform/PlatformSiginfoTest.cpp
@@ -60,9 +60,11 @@ class PlatformSiginfoTest : public ::testing::Test {
     uint64_t total_offset = 0;
     for (auto field_name : llvm::split(path, '.')) {
       uint64_t bit_offset;
-      ASSERT_NE(field_type.GetIndexOfFieldWithName(field_name.str().c_str(),
-                                                   &field_type, &bit_offset),
-                UINT32_MAX);
+      std::string name;
+      field_type = field_type.GetFieldAtIndex(
+          field_type.GetIndexOfChildWithName(field_name, false), name,
+          &bit_offset, nullptr, nullptr);
+      ASSERT_TRUE(field_type);
       total_offset += bit_offset;
     }
 

From f9c01b59e3d5239a4e42f94d5edbeb3698a81032 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Thu, 17 Apr 2025 13:26:15 +0100
Subject: [PATCH 258/710] [LV] Fix '-1U' bits for smallest type in
 getSmallestAndWidestTypes (#135783)

For loops without loads/stores, where the smallest/widest types are
calculated from the reduction, the smallest type returned is always -1U
and it actually returns the smallest type as the widest type. This PR
fixes the calculation.

This follows from
https://github.com/llvm/llvm-project/pull/132190#discussion_r2044232607
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp          | 9 ++++-----
 .../LoopVectorize/AArch64/smallest-and-widest-types.ll   | 6 +++---
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d8a510f59a59f..4c1ed15ee700f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4813,17 +4813,16 @@ LoopVectorizationCostModel::getSmallestAndWidestTypes() {
   // if there are no loads/stores in the loop. In this case, check through the
   // reduction variables to determine the maximum width.
   if (ElementTypesInLoop.empty() && !Legal->getReductionVars().empty()) {
-    // Reset MaxWidth so that we can find the smallest type used by recurrences
-    // in the loop.
-    MaxWidth = -1U;
     for (const auto &PhiDescriptorPair : Legal->getReductionVars()) {
       const RecurrenceDescriptor &RdxDesc = PhiDescriptorPair.second;
       // When finding the min width used by the recurrence we need to account
       // for casts on the input operands of the recurrence.
-      MaxWidth = std::min<unsigned>(
-          MaxWidth, std::min<unsigned>(
+      MinWidth = std::min<unsigned>(
+          MinWidth, std::min<unsigned>(
                         RdxDesc.getMinWidthCastToRecurrenceTypeInBits(),
                         RdxDesc.getRecurrenceType()->getScalarSizeInBits()));
+      MaxWidth = std::max<unsigned>(
+          MaxWidth, RdxDesc.getRecurrenceType()->getScalarSizeInBits());
     }
   } else {
     for (Type *T : ElementTypesInLoop) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
index 269562fa70549..b34ba5e38811a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
@@ -37,7 +37,7 @@ for.end:
 ; chosen. The following 3 cases check different combinations of widths.
 
 ; CHECK-LABEL: Checking a loop in 'no_loads_stores_32'
-; CHECK: The Smallest and Widest types: 4294967295 / 32 bits
+; CHECK: The Smallest and Widest types: 32 / 64 bits
 ; CHECK: Selecting VF: 4
 
 define double @no_loads_stores_32(i32 %n) {
@@ -60,7 +60,7 @@ for.end:
 }
 
 ; CHECK-LABEL: Checking a loop in 'no_loads_stores_16'
-; CHECK: The Smallest and Widest types: 4294967295 / 16 bits
+; CHECK: The Smallest and Widest types: 16 / 64 bits
 ; CHECK: Selecting VF: 8
 
 define double @no_loads_stores_16() {
@@ -82,7 +82,7 @@ for.end:
 }
 
 ; CHECK-LABEL: Checking a loop in 'no_loads_stores_8'
-; CHECK: The Smallest and Widest types: 4294967295 / 8 bits
+; CHECK: The Smallest and Widest types: 8 / 32 bits
 ; CHECK: Selecting VF: 16
 
 define float @no_loads_stores_8() {

From 23e3cbb2e82b62586266116c8ab77ce68e412cf8 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Thu, 17 Apr 2025 14:37:20 +0200
Subject: [PATCH 259/710] [mlir][Transforms] Delete 1:N dialect conversion
 driver (#121389)

The 1:N dialect conversion driver has been deprecated. Use the regular
dialect conversion driver instead. This commit deletes the 1:N dialect
conversion driver.

Note for LLVM integration: If you are already using the regular dialect conversion, but still have argument materializations in your code base, simply delete all `addArgumentMaterialization` calls.

For details, see
https://discourse.llvm.org/t/rfc-merging-1-1-and-1-n-dialect-conversions/82513.
---
 .../Func/Transforms/OneToNFuncConversions.h   |  26 -
 .../mlir/Dialect/SCF/Transforms/Patterns.h    |   9 -
 .../SPIRV/Transforms/SPIRVConversion.h        |   1 -
 .../mlir/Transforms/DialectConversion.h       |  20 -
 .../mlir/Transforms/OneToNTypeConversion.h    | 285 -----------
 .../Dialect/Func/Transforms/CMakeLists.txt    |   1 -
 .../Func/Transforms/OneToNFuncConversions.cpp |  87 ----
 .../lib/Dialect/SCF/Transforms/CMakeLists.txt |   1 -
 .../SCF/Transforms/OneToNTypeConversion.cpp   | 215 --------
 .../SPIRV/Transforms/SPIRVConversion.cpp      |   7 +-
 mlir/lib/Transforms/Utils/CMakeLists.txt      |   1 -
 .../Transforms/Utils/DialectConversion.cpp    |  11 -
 .../Transforms/Utils/OneToNTypeConversion.cpp | 458 ------------------
 .../one-to-n-type-conversion.mlir             | 140 ------
 ...f-structural-one-to-n-type-conversion.mlir | 183 -------
 .../decompose-call-graph-types.mlir           |  53 --
 mlir/test/lib/Conversion/CMakeLists.txt       |   1 -
 .../OneToNTypeConversion/CMakeLists.txt       |  23 -
 .../TestOneToNTypeConversionPass.cpp          | 261 ----------
 mlir/tools/mlir-opt/CMakeLists.txt            |   1 -
 mlir/tools/mlir-opt/mlir-opt.cpp              |   2 -
 .../llvm-project-overlay/mlir/BUILD.bazel     |   2 -
 .../mlir/test/BUILD.bazel                     |  17 -
 23 files changed, 4 insertions(+), 1801 deletions(-)
 delete mode 100644 mlir/include/mlir/Dialect/Func/Transforms/OneToNFuncConversions.h
 delete mode 100644 mlir/include/mlir/Transforms/OneToNTypeConversion.h
 delete mode 100644 mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
 delete mode 100644 mlir/lib/Dialect/SCF/Transforms/OneToNTypeConversion.cpp
 delete mode 100644 mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
 delete mode 100644 mlir/test/Conversion/OneToNTypeConversion/one-to-n-type-conversion.mlir
 delete mode 100644 mlir/test/Conversion/OneToNTypeConversion/scf-structural-one-to-n-type-conversion.mlir
 delete mode 100644 mlir/test/lib/Conversion/OneToNTypeConversion/CMakeLists.txt
 delete mode 100644 mlir/test/lib/Conversion/OneToNTypeConversion/TestOneToNTypeConversionPass.cpp

diff --git a/mlir/include/mlir/Dialect/Func/Transforms/OneToNFuncConversions.h b/mlir/include/mlir/Dialect/Func/Transforms/OneToNFuncConversions.h
deleted file mode 100644
index c9e407daf9bf8..0000000000000
--- a/mlir/include/mlir/Dialect/Func/Transforms/OneToNFuncConversions.h
+++ /dev/null
@@ -1,26 +0,0 @@
-//===- OneToNTypeFuncConversions.h - 1:N type conv. for Func ----*- C++ -*-===//
-//
-// Licensed under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_DIALECT_FUNC_TRANSFORMS_ONETONTYPEFUNCCONVERSIONS_H
-#define MLIR_DIALECT_FUNC_TRANSFORMS_ONETONTYPEFUNCCONVERSIONS_H
-
-namespace mlir {
-class TypeConverter;
-class RewritePatternSet;
-} // namespace mlir
-
-namespace mlir {
-
-// Populates the provided pattern set with patterns that do 1:N type conversions
-// on func ops. This is intended to be used with `applyPartialOneToNConversion`.
-void populateFuncTypeConversionPatterns(const TypeConverter &typeConverter,
-                                        RewritePatternSet &patterns);
-
-} // namespace mlir
-
-#endif // MLIR_DIALECT_FUNC_TRANSFORMS_ONETONTYPEFUNCCONVERSIONS_H
diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/Patterns.h b/mlir/include/mlir/Dialect/SCF/Transforms/Patterns.h
index 18c9dfd205de3..00c8a5c0c517b 100644
--- a/mlir/include/mlir/Dialect/SCF/Transforms/Patterns.h
+++ b/mlir/include/mlir/Dialect/SCF/Transforms/Patterns.h
@@ -63,15 +63,6 @@ void populateSCFStructuralTypeConversions(const TypeConverter &typeConverter,
 void populateSCFStructuralTypeConversionTarget(
     const TypeConverter &typeConverter, ConversionTarget &target);
 
-/// Populates the provided pattern set with patterns that do 1:N type
-/// conversions on (some) SCF ops. This is intended to be used with
-/// applyPartialOneToNConversion.
-/// FIXME: The 1:N dialect conversion is deprecated and will be removed soon.
-/// 1:N support has been added to the regular dialect conversion driver.
-/// Use populateSCFStructuralTypeConversions() instead.
-void populateSCFStructuralOneToNTypeConversions(
-    const TypeConverter &typeConverter, RewritePatternSet &patterns);
-
 /// Populate patterns for SCF software pipelining transformation. See the
 /// ForLoopPipeliningPattern for the transformation details.
 void populateSCFLoopPipeliningPatterns(RewritePatternSet &patterns,
diff --git a/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h b/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
index bed4d66ccd6cb..3d22ec918f4c5 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
+++ b/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
@@ -20,7 +20,6 @@
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/OneToNTypeConversion.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/LogicalResult.h"
 
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index ecfa5248b7559..b6ab252456e70 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -45,13 +45,11 @@ class TypeConverter {
   // Copy the registered conversions, but not the caches
   TypeConverter(const TypeConverter &other)
       : conversions(other.conversions),
-        argumentMaterializations(other.argumentMaterializations),
         sourceMaterializations(other.sourceMaterializations),
         targetMaterializations(other.targetMaterializations),
         typeAttributeConversions(other.typeAttributeConversions) {}
   TypeConverter &operator=(const TypeConverter &other) {
     conversions = other.conversions;
-    argumentMaterializations = other.argumentMaterializations;
     sourceMaterializations = other.sourceMaterializations;
     targetMaterializations = other.targetMaterializations;
     typeAttributeConversions = other.typeAttributeConversions;
@@ -180,21 +178,6 @@ class TypeConverter {
   /// can be a TypeRange; in that case, the function must return a
   /// SmallVector<Value>.
 
-  /// This method registers a materialization that will be called when
-  /// converting (potentially multiple) block arguments that were the result of
-  /// a signature conversion of a single block argument, to a single SSA value
-  /// with the old block argument type.
-  ///
-  /// Note: Argument materializations are used only with the 1:N dialect
-  /// conversion driver. The 1:N dialect conversion driver will be removed soon
-  /// and so will be argument materializations.
-  template <typename FnT, typename T = typename llvm::function_traits<
-                              std::decay_t<FnT>>::template arg_t<1>>
-  void addArgumentMaterialization(FnT &&callback) {
-    argumentMaterializations.emplace_back(
-        wrapMaterialization<T>(std::forward<FnT>(callback)));
-  }
-
   /// This method registers a materialization that will be called when
   /// converting a replacement value back to its original source type.
   /// This is used when some uses of the original value persist beyond the main
@@ -322,8 +305,6 @@ class TypeConverter {
   /// generating a cast sequence of some kind. See the respective
   /// `add*Materialization` for more information on the context for these
   /// methods.
-  Value materializeArgumentConversion(OpBuilder &builder, Location loc,
-                                      Type resultType, ValueRange inputs) const;
   Value materializeSourceConversion(OpBuilder &builder, Location loc,
                                     Type resultType, ValueRange inputs) const;
   Value materializeTargetConversion(OpBuilder &builder, Location loc,
@@ -510,7 +491,6 @@ class TypeConverter {
   SmallVector<ConversionCallbackFn, 4> conversions;
 
   /// The list of registered materialization functions.
-  SmallVector<MaterializationCallbackFn, 2> argumentMaterializations;
   SmallVector<MaterializationCallbackFn, 2> sourceMaterializations;
   SmallVector<TargetMaterializationCallbackFn, 2> targetMaterializations;
 
diff --git a/mlir/include/mlir/Transforms/OneToNTypeConversion.h b/mlir/include/mlir/Transforms/OneToNTypeConversion.h
deleted file mode 100644
index 37a326818d645..0000000000000
--- a/mlir/include/mlir/Transforms/OneToNTypeConversion.h
+++ /dev/null
@@ -1,285 +0,0 @@
-//===-- OneToNTypeConversion.h - Utils for 1:N type conversion --*- C++ -*-===//
-//
-// Licensed under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Note: The 1:N dialect conversion is deprecated and will be removed soon.
-// 1:N support has been added to the regular dialect conversion driver.
-//
-// This file provides utils for implementing (poor-man's) dialect conversion
-// passes with 1:N type conversions.
-//
-// The main function, `applyPartialOneToNConversion`, first applies a set of
-// `RewritePattern`s, which produce unrealized casts to convert the operands and
-// results from and to the source types, and then replaces all newly added
-// unrealized casts by user-provided materializations. For this to work, the
-// main function requires a special `TypeConverter`, a special
-// `PatternRewriter`, and special RewritePattern`s, which extend their
-// respective base classes for 1:N type converions.
-//
-// Note that this is much more simple-minded than the "real" dialect conversion,
-// which checks for legality before applying patterns and does probably many
-// other additional things. Ideally, some of the extensions here could be
-// integrated there.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_TRANSFORMS_ONETONTYPECONVERSION_H
-#define MLIR_TRANSFORMS_ONETONTYPECONVERSION_H
-
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/Transforms/DialectConversion.h"
-#include "llvm/ADT/SmallVector.h"
-
-namespace mlir {
-
-/// Stores a 1:N mapping of types and provides several useful accessors. This
-/// class extends `SignatureConversion`, which already supports 1:N type
-/// mappings but lacks some accessors into the mapping as well as access to the
-/// original types.
-class OneToNTypeMapping : public TypeConverter::SignatureConversion {
-public:
-  OneToNTypeMapping(TypeRange originalTypes)
-      : TypeConverter::SignatureConversion(originalTypes.size()),
-        originalTypes(originalTypes) {}
-
-  using TypeConverter::SignatureConversion::getConvertedTypes;
-
-  /// Returns the list of types that corresponds to the original type at the
-  /// given index.
-  TypeRange getConvertedTypes(unsigned originalTypeNo) const;
-
-  /// Returns the list of original types.
-  TypeRange getOriginalTypes() const { return originalTypes; }
-
-  /// Returns the slice of converted values that corresponds the original value
-  /// at the given index.
-  ValueRange getConvertedValues(ValueRange convertedValues,
-                                unsigned originalValueNo) const;
-
-  /// Fills the given result vector with as many copies of the location of the
-  /// original value as the number of values it is converted to.
-  void convertLocation(Value originalValue, unsigned originalValueNo,
-                       llvm::SmallVectorImpl<Location> &result) const;
-
-  /// Fills the given result vector with as many copies of the lociation of each
-  /// original value as the number of values they are respectively converted to.
-  void convertLocations(ValueRange originalValues,
-                        llvm::SmallVectorImpl<Location> &result) const;
-
-  /// Returns true iff at least one type conversion maps an input type to a type
-  /// that is different from itself.
-  bool hasNonIdentityConversion() const;
-
-private:
-  llvm::SmallVector<Type> originalTypes;
-};
-
-/// Extends the basic `RewritePattern` class with a type converter member and
-/// some accessors to it. This is useful for patterns that are not
-/// `ConversionPattern`s but still require access to a type converter.
-class RewritePatternWithConverter : public mlir::RewritePattern {
-public:
-  /// Construct a conversion pattern with the given converter, and forward the
-  /// remaining arguments to RewritePattern.
-  template <typename... Args>
-  RewritePatternWithConverter(const TypeConverter &typeConverter,
-                              Args &&...args)
-      : RewritePattern(std::forward<Args>(args)...),
-        typeConverter(&typeConverter) {}
-
-  /// Return the type converter held by this pattern, or nullptr if the pattern
-  /// does not require type conversion.
-  const TypeConverter *getTypeConverter() const { return typeConverter; }
-
-  template <typename ConverterTy>
-  std::enable_if_t<std::is_base_of<TypeConverter, ConverterTy>::value,
-                   const ConverterTy *>
-  getTypeConverter() const {
-    return static_cast<const ConverterTy *>(typeConverter);
-  }
-
-protected:
-  /// A type converter for use by this pattern.
-  const TypeConverter *const typeConverter;
-};
-
-/// Specialization of `PatternRewriter` that `OneToNConversionPattern`s use. The
-/// class provides additional rewrite methods that are specific to 1:N type
-/// conversions.
-class OneToNPatternRewriter : public PatternRewriter {
-public:
-  OneToNPatternRewriter(MLIRContext *context,
-                        OpBuilder::Listener *listener = nullptr)
-      : PatternRewriter(context, listener) {}
-
-  /// Replaces the results of the operation with the specified list of values
-  /// mapped back to the original types as specified in the provided type
-  /// mapping. That type mapping must match the replaced op (i.e., the original
-  /// types must be the same as the result types of the op) and the new values
-  /// (i.e., the converted types must be the same as the types of the new
-  /// values).
-  /// FIXME: The 1:N dialect conversion is deprecated and will be removed soon.
-  /// Use replaceOpWithMultiple() instead.
-  void replaceOp(Operation *op, ValueRange newValues,
-                 const OneToNTypeMapping &resultMapping);
-  using PatternRewriter::replaceOp;
-
-  /// Applies the given argument conversion to the given block. This consists of
-  /// replacing each original argument with N arguments as specified in the
-  /// argument conversion and inserting unrealized casts from the converted
-  /// values to the original types, which are then used in lieu of the original
-  /// ones. (Eventually, `applyPartialOneToNConversion` replaces these casts
-  /// with a user-provided argument materialization if necessary.) This is
-  /// similar to `ArgConverter::applySignatureConversion` but (1) handles 1:N
-  /// type conversion properly and probably (2) doesn't handle many other edge
-  /// cases.
-  Block *applySignatureConversion(Block *block,
-                                  OneToNTypeMapping &argumentConversion);
-};
-
-/// Base class for patterns with 1:N type conversions. Derived classes have to
-/// overwrite the `matchAndRewrite` overlaod that provides additional
-/// information for 1:N type conversions.
-class OneToNConversionPattern : public RewritePatternWithConverter {
-public:
-  using RewritePatternWithConverter::RewritePatternWithConverter;
-
-  /// This function has to be implemented by derived classes and is called from
-  /// the usual overloads. Like in "normal" `DialectConversion`, the function is
-  /// provided with the converted operands (which thus have target types). Since
-  /// 1:N conversions are supported, there is usually no 1:1 relationship
-  /// between the original and the converted operands. Instead, the provided
-  /// `operandMapping` can be used to access the converted operands that
-  /// correspond to a particular original operand. Similarly, `resultMapping`
-  /// is provided to help with assembling the result values, which may have 1:N
-  /// correspondences as well. In that case, the original op should be replaced
-  /// with the overload of `replaceOp` that takes the provided `resultMapping`
-  /// in order to deal with the mapping of converted result values to their
-  /// usages in the original types correctly.
-  virtual LogicalResult matchAndRewrite(Operation *op,
-                                        OneToNPatternRewriter &rewriter,
-                                        const OneToNTypeMapping &operandMapping,
-                                        const OneToNTypeMapping &resultMapping,
-                                        ValueRange convertedOperands) const = 0;
-
-  LogicalResult matchAndRewrite(Operation *op,
-                                PatternRewriter &rewriter) const final;
-};
-
-/// This class is a wrapper around `OneToNConversionPattern` for matching
-/// against instances of a particular op class.
-template <typename SourceOp>
-class OneToNOpConversionPattern : public OneToNConversionPattern {
-public:
-  OneToNOpConversionPattern(const TypeConverter &typeConverter,
-                            MLIRContext *context, PatternBenefit benefit = 1,
-                            ArrayRef<StringRef> generatedNames = {})
-      : OneToNConversionPattern(typeConverter, SourceOp::getOperationName(),
-                                benefit, context, generatedNames) {}
-  /// Generic adaptor around the root op of this pattern using the converted
-  /// operands. Importantly, each operand is represented as a *range* of values,
-  /// namely the N values each original operand gets converted to. Concretely,
-  /// this makes the result type of the accessor functions of the adaptor class
-  /// be a `ValueRange`.
-  class OpAdaptor
-      : public SourceOp::template GenericAdaptor<ArrayRef<ValueRange>> {
-  public:
-    using RangeT = ArrayRef<ValueRange>;
-    using BaseT = typename SourceOp::template GenericAdaptor<RangeT>;
-    using Properties = typename SourceOp::template InferredProperties<SourceOp>;
-
-    OpAdaptor(const OneToNTypeMapping *operandMapping,
-              const OneToNTypeMapping *resultMapping,
-              const ValueRange *convertedOperands, RangeT values, SourceOp op)
-        : BaseT(values, op), operandMapping(operandMapping),
-          resultMapping(resultMapping), convertedOperands(convertedOperands) {}
-
-    /// Get the type mapping of the original operands to the converted operands.
-    const OneToNTypeMapping &getOperandMapping() const {
-      return *operandMapping;
-    }
-
-    /// Get the type mapping of the original results to the converted results.
-    const OneToNTypeMapping &getResultMapping() const { return *resultMapping; }
-
-    /// Get a flat range of all converted operands. Unlike `getOperands`, which
-    /// returns an `ArrayRef` with one `ValueRange` for each original operand,
-    /// this function returns a `ValueRange` that contains all converted
-    /// operands irrespectively of which operand they originated from.
-    ValueRange getFlatOperands() const { return *convertedOperands; }
-
-  private:
-    const OneToNTypeMapping *operandMapping;
-    const OneToNTypeMapping *resultMapping;
-    const ValueRange *convertedOperands;
-  };
-
-  using OneToNConversionPattern::matchAndRewrite;
-
-  /// Overload that derived classes have to override for their op type.
-  virtual LogicalResult
-  matchAndRewrite(SourceOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const = 0;
-
-  LogicalResult matchAndRewrite(Operation *op, OneToNPatternRewriter &rewriter,
-                                const OneToNTypeMapping &operandMapping,
-                                const OneToNTypeMapping &resultMapping,
-                                ValueRange convertedOperands) const final {
-    // Wrap converted operands and type mappings into an adaptor.
-    SmallVector<ValueRange> valueRanges;
-    for (int64_t i = 0; i < op->getNumOperands(); i++) {
-      auto values = operandMapping.getConvertedValues(convertedOperands, i);
-      valueRanges.push_back(values);
-    }
-    OpAdaptor adaptor(&operandMapping, &resultMapping, &convertedOperands,
-                      valueRanges, cast<SourceOp>(op));
-
-    // Call overload implemented by the derived class.
-    return matchAndRewrite(cast<SourceOp>(op), adaptor, rewriter);
-  }
-};
-
-/// Applies the given set of patterns recursively on the given op and adds user
-/// materializations where necessary. The patterns are expected to be
-/// `OneToNConversionPattern`, which help converting the types of the operands
-/// and results of the matched ops. The provided type converter is used to
-/// convert the operands of matched ops from their original types to operands
-/// with different types. Unlike in `DialectConversion`, this supports 1:N type
-/// conversions. Those conversions at the "boundary" of the pattern application,
-/// where converted results are not consumed by replaced ops that expect the
-/// converted operands or vice versa, the function inserts user materializations
-/// from the type converter. Also unlike `DialectConversion`, there are no legal
-/// or illegal types; the function simply applies the given patterns and does
-/// not fail if some ops or types remain unconverted (i.e., the conversion is
-/// only "partial").
-/// FIXME: The 1:N dialect conversion is deprecated and will be removed soon.
-/// 1:N support has been added to the regular dialect conversion driver.
-/// Use applyPartialConversion() instead.
-LogicalResult
-applyPartialOneToNConversion(Operation *op, TypeConverter &typeConverter,
-                             const FrozenRewritePatternSet &patterns);
-
-/// Add a pattern to the given pattern list to convert the signature of a
-/// FunctionOpInterface op with the given type converter. This only supports
-/// ops which use FunctionType to represent their type. This is intended to be
-/// used with the 1:N dialect conversion.
-/// FIXME: The 1:N dialect conversion is deprecated and will be removed soon.
-/// 1:N support has been added to the regular dialect conversion driver.
-/// Use populateFunctionOpInterfaceTypeConversionPattern() instead.
-void populateOneToNFunctionOpInterfaceTypeConversionPattern(
-    StringRef functionLikeOpName, const TypeConverter &converter,
-    RewritePatternSet &patterns);
-template <typename FuncOpT>
-void populateOneToNFunctionOpInterfaceTypeConversionPattern(
-    const TypeConverter &converter, RewritePatternSet &patterns) {
-  populateOneToNFunctionOpInterfaceTypeConversionPattern(
-      FuncOpT::getOperationName(), converter, patterns);
-}
-
-} // namespace mlir
-
-#endif // MLIR_TRANSFORMS_ONETONTYPECONVERSION_H
diff --git a/mlir/lib/Dialect/Func/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Func/Transforms/CMakeLists.txt
index 6384d25ee7027..0bed59e109503 100644
--- a/mlir/lib/Dialect/Func/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Func/Transforms/CMakeLists.txt
@@ -1,7 +1,6 @@
 add_mlir_dialect_library(MLIRFuncTransforms
   DuplicateFunctionElimination.cpp
   FuncConversions.cpp
-  OneToNFuncConversions.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Func/Transforms
diff --git a/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
deleted file mode 100644
index 3b8982257a9c9..0000000000000
--- a/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
+++ /dev/null
@@ -1,87 +0,0 @@
-//===-- OneToNTypeFuncConversions.cpp - Func 1:N type conversion-*- C++ -*-===//
-//
-// Licensed under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// The patterns in this file are heavily inspired (and copied from)
-// convertFuncOpTypes in lib/Transforms/Utils/DialectConversion.cpp and the
-// patterns in lib/Dialect/Func/Transforms/FuncConversions.cpp but work for 1:N
-// type conversions.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/Func/Transforms/OneToNFuncConversions.h"
-
-#include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Transforms/OneToNTypeConversion.h"
-
-using namespace mlir;
-using namespace mlir::func;
-
-namespace {
-
-class ConvertTypesInFuncCallOp : public OneToNOpConversionPattern<CallOp> {
-public:
-  using OneToNOpConversionPattern<CallOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(CallOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    Location loc = op->getLoc();
-    const OneToNTypeMapping &resultMapping = adaptor.getResultMapping();
-
-    // Nothing to do if the op doesn't have any non-identity conversions for its
-    // operands or results.
-    if (!adaptor.getOperandMapping().hasNonIdentityConversion() &&
-        !resultMapping.hasNonIdentityConversion())
-      return failure();
-
-    // Create new CallOp.
-    auto newOp =
-        rewriter.create<CallOp>(loc, resultMapping.getConvertedTypes(),
-                                adaptor.getFlatOperands(), op->getAttrs());
-
-    rewriter.replaceOp(op, newOp->getResults(), resultMapping);
-    return success();
-  }
-};
-
-class ConvertTypesInFuncReturnOp : public OneToNOpConversionPattern<ReturnOp> {
-public:
-  using OneToNOpConversionPattern<ReturnOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(ReturnOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    // Nothing to do if there is no non-identity conversion.
-    if (!adaptor.getOperandMapping().hasNonIdentityConversion())
-      return failure();
-
-    // Convert operands.
-    rewriter.modifyOpInPlace(
-        op, [&] { op->setOperands(adaptor.getFlatOperands()); });
-
-    return success();
-  }
-};
-
-} // namespace
-
-namespace mlir {
-
-void populateFuncTypeConversionPatterns(const TypeConverter &typeConverter,
-                                        RewritePatternSet &patterns) {
-  patterns.add<
-      // clang-format off
-      ConvertTypesInFuncCallOp,
-      ConvertTypesInFuncReturnOp
-      // clang-format on
-      >(typeConverter, patterns.getContext());
-  populateOneToNFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(
-      typeConverter, patterns);
-}
-
-} // namespace mlir
diff --git a/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
index e99b5d0cc26fc..84dd992bec53a 100644
--- a/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
@@ -8,7 +8,6 @@ add_mlir_dialect_library(MLIRSCFTransforms
   LoopPipelining.cpp
   LoopRangeFolding.cpp
   LoopSpecialization.cpp
-  OneToNTypeConversion.cpp
   ParallelLoopCollapsing.cpp
   ParallelLoopFusion.cpp
   ParallelLoopTiling.cpp
diff --git a/mlir/lib/Dialect/SCF/Transforms/OneToNTypeConversion.cpp b/mlir/lib/Dialect/SCF/Transforms/OneToNTypeConversion.cpp
deleted file mode 100644
index 4cd17f77dfb94..0000000000000
--- a/mlir/lib/Dialect/SCF/Transforms/OneToNTypeConversion.cpp
+++ /dev/null
@@ -1,215 +0,0 @@
-//===-- OneToNTypeConversion.cpp - SCF 1:N type conversion ------*- C++ -*-===//
-//
-// Licensed under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// The patterns in this file are heavily inspired (and copied from)
-// lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp but work for 1:N
-// type conversions.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/SCF/Transforms/Transforms.h"
-
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Transforms/OneToNTypeConversion.h"
-
-using namespace mlir;
-using namespace mlir::scf;
-
-class ConvertTypesInSCFIfOp : public OneToNOpConversionPattern<IfOp> {
-public:
-  using OneToNOpConversionPattern<IfOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(IfOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    Location loc = op->getLoc();
-    const OneToNTypeMapping &resultMapping = adaptor.getResultMapping();
-
-    // Nothing to do if there is no non-identity conversion.
-    if (!resultMapping.hasNonIdentityConversion())
-      return failure();
-
-    // Create new IfOp.
-    TypeRange convertedResultTypes = resultMapping.getConvertedTypes();
-    auto newOp = rewriter.create<IfOp>(loc, convertedResultTypes,
-                                       op.getCondition(), true);
-    newOp->setAttrs(op->getAttrs());
-
-    // We do not need the empty blocks created by rewriter.
-    rewriter.eraseBlock(newOp.elseBlock());
-    rewriter.eraseBlock(newOp.thenBlock());
-
-    // Inlines block from the original operation.
-    rewriter.inlineRegionBefore(op.getThenRegion(), newOp.getThenRegion(),
-                                newOp.getThenRegion().end());
-    rewriter.inlineRegionBefore(op.getElseRegion(), newOp.getElseRegion(),
-                                newOp.getElseRegion().end());
-
-    rewriter.replaceOp(op, newOp->getResults(), resultMapping);
-    return success();
-  }
-};
-
-class ConvertTypesInSCFWhileOp : public OneToNOpConversionPattern<WhileOp> {
-public:
-  using OneToNOpConversionPattern<WhileOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(WhileOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    Location loc = op->getLoc();
-
-    const OneToNTypeMapping &operandMapping = adaptor.getOperandMapping();
-    const OneToNTypeMapping &resultMapping = adaptor.getResultMapping();
-
-    // Nothing to do if the op doesn't have any non-identity conversions for its
-    // operands or results.
-    if (!operandMapping.hasNonIdentityConversion() &&
-        !resultMapping.hasNonIdentityConversion())
-      return failure();
-
-    // Create new WhileOp.
-    TypeRange convertedResultTypes = resultMapping.getConvertedTypes();
-
-    auto newOp = rewriter.create<WhileOp>(loc, convertedResultTypes,
-                                          adaptor.getFlatOperands());
-    newOp->setAttrs(op->getAttrs());
-
-    // Update block signatures.
-    std::array<OneToNTypeMapping, 2> blockMappings = {operandMapping,
-                                                      resultMapping};
-    for (unsigned int i : {0u, 1u}) {
-      Region *region = &op.getRegion(i);
-      Block *block = &region->front();
-
-      rewriter.applySignatureConversion(block, blockMappings[i]);
-
-      // Move updated region to new WhileOp.
-      Region &dstRegion = newOp.getRegion(i);
-      rewriter.inlineRegionBefore(op.getRegion(i), dstRegion, dstRegion.end());
-    }
-
-    rewriter.replaceOp(op, newOp->getResults(), resultMapping);
-    return success();
-  }
-};
-
-class ConvertTypesInSCFYieldOp : public OneToNOpConversionPattern<YieldOp> {
-public:
-  using OneToNOpConversionPattern<YieldOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(YieldOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    // Nothing to do if there is no non-identity conversion.
-    if (!adaptor.getOperandMapping().hasNonIdentityConversion())
-      return failure();
-
-    // Convert operands.
-    rewriter.modifyOpInPlace(
-        op, [&] { op->setOperands(adaptor.getFlatOperands()); });
-
-    return success();
-  }
-};
-
-class ConvertTypesInSCFConditionOp
-    : public OneToNOpConversionPattern<ConditionOp> {
-public:
-  using OneToNOpConversionPattern<ConditionOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(ConditionOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    // Nothing to do if there is no non-identity conversion.
-    if (!adaptor.getOperandMapping().hasNonIdentityConversion())
-      return failure();
-
-    // Convert operands.
-    rewriter.modifyOpInPlace(
-        op, [&] { op->setOperands(adaptor.getFlatOperands()); });
-
-    return success();
-  }
-};
-
-class ConvertTypesInSCFForOp final : public OneToNOpConversionPattern<ForOp> {
-public:
-  using OneToNOpConversionPattern<ForOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    const OneToNTypeMapping &operandMapping = adaptor.getOperandMapping();
-    const OneToNTypeMapping &resultMapping = adaptor.getResultMapping();
-
-    // Nothing to do if there is no non-identity conversion.
-    if (!operandMapping.hasNonIdentityConversion() &&
-        !resultMapping.hasNonIdentityConversion())
-      return failure();
-
-    // If the lower-bound, upper-bound, or step were expanded, abort the
-    // conversion. This conversion does not know what to do in such cases.
-    ValueRange lbs = adaptor.getLowerBound();
-    ValueRange ubs = adaptor.getUpperBound();
-    ValueRange steps = adaptor.getStep();
-    if (lbs.size() != 1 || ubs.size() != 1 || steps.size() != 1)
-      return rewriter.notifyMatchFailure(
-          forOp, "index operands converted to multiple values");
-
-    Location loc = forOp.getLoc();
-
-    Region *region = &forOp.getRegion();
-    Block *block = &region->front();
-
-    // Construct the new for-op with an empty body.
-    ValueRange newInits = adaptor.getFlatOperands().drop_front(3);
-    auto newOp =
-        rewriter.create<ForOp>(loc, lbs[0], ubs[0], steps[0], newInits);
-    newOp->setAttrs(forOp->getAttrs());
-
-    // We do not need the empty blocks created by rewriter.
-    rewriter.eraseBlock(newOp.getBody());
-
-    // Convert the signature of the body region.
-    OneToNTypeMapping bodyTypeMapping(block->getArgumentTypes());
-    if (failed(typeConverter->convertSignatureArgs(block->getArgumentTypes(),
-                                                   bodyTypeMapping)))
-      return failure();
-
-    // Perform signature conversion on the body block.
-    rewriter.applySignatureConversion(block, bodyTypeMapping);
-
-    // Splice the old body region into the new for-op.
-    Region &dstRegion = newOp.getBodyRegion();
-    rewriter.inlineRegionBefore(forOp.getRegion(), dstRegion, dstRegion.end());
-
-    rewriter.replaceOp(forOp, newOp.getResults(), resultMapping);
-
-    return success();
-  }
-};
-
-namespace mlir {
-namespace scf {
-
-void populateSCFStructuralOneToNTypeConversions(
-    const TypeConverter &typeConverter, RewritePatternSet &patterns) {
-  patterns.add<
-      // clang-format off
-      ConvertTypesInSCFConditionOp,
-      ConvertTypesInSCFForOp,
-      ConvertTypesInSCFIfOp,
-      ConvertTypesInSCFWhileOp,
-      ConvertTypesInSCFYieldOp
-      // clang-format on
-      >(typeConverter, patterns.getContext());
-}
-
-} // namespace scf
-} // namespace mlir
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index a60410d01ac57..19b9af146f4a4 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -29,7 +29,6 @@
 #include "mlir/Support/LLVM.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/OneToNTypeConversion.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -933,7 +932,8 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
     OpBuilder::InsertionGuard guard(rewriter);
     rewriter.setInsertionPointToStart(&entryBlock);
 
-    OneToNTypeMapping oneToNTypeMapping(fnType.getInputs());
+    TypeConverter::SignatureConversion oneToNTypeMapping(
+        fnType.getInputs().size());
 
     // For arguments that are of illegal types and require unrolling.
     // `unrolledInputNums` stores the indices of arguments that result from
@@ -1073,7 +1073,8 @@ struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
       return failure();
 
     FunctionType fnType = funcOp.getFunctionType();
-    OneToNTypeMapping oneToNTypeMapping(fnType.getResults());
+    TypeConverter::SignatureConversion oneToNTypeMapping(
+        fnType.getResults().size());
     Location loc = returnOp.getLoc();
 
     // For the new return op.
diff --git a/mlir/lib/Transforms/Utils/CMakeLists.txt b/mlir/lib/Transforms/Utils/CMakeLists.txt
index 72eb34f36cf5f..3ca16239ba33c 100644
--- a/mlir/lib/Transforms/Utils/CMakeLists.txt
+++ b/mlir/lib/Transforms/Utils/CMakeLists.txt
@@ -8,7 +8,6 @@ add_mlir_library(MLIRTransformUtils
   Inliner.cpp
   InliningUtils.cpp
   LoopInvariantCodeMotionUtils.cpp
-  OneToNTypeConversion.cpp
   RegionUtils.cpp
   WalkPatternRewriteDriver.cpp
 
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 566dab7f30adb..a56fca25e1697 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2931,17 +2931,6 @@ TypeConverter::convertSignatureArgs(TypeRange types,
   return success();
 }
 
-Value TypeConverter::materializeArgumentConversion(OpBuilder &builder,
-                                                   Location loc,
-                                                   Type resultType,
-                                                   ValueRange inputs) const {
-  for (const MaterializationCallbackFn &fn :
-       llvm::reverse(argumentMaterializations))
-    if (Value result = fn(builder, resultType, inputs, loc))
-      return result;
-  return nullptr;
-}
-
 Value TypeConverter::materializeSourceConversion(OpBuilder &builder,
                                                  Location loc, Type resultType,
                                                  ValueRange inputs) const {
diff --git a/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp b/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
deleted file mode 100644
index 6474c59595eb4..0000000000000
--- a/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
+++ /dev/null
@@ -1,458 +0,0 @@
-//===-- OneToNTypeConversion.cpp - Utils for 1:N type conversion-*- C++ -*-===//
-//
-// Licensed under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Transforms/OneToNTypeConversion.h"
-
-#include "mlir/Interfaces/FunctionInterfaces.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/SmallSet.h"
-
-#include <unordered_map>
-
-using namespace llvm;
-using namespace mlir;
-
-TypeRange OneToNTypeMapping::getConvertedTypes(unsigned originalTypeNo) const {
-  TypeRange convertedTypes = getConvertedTypes();
-  if (auto mapping = getInputMapping(originalTypeNo))
-    return convertedTypes.slice(mapping->inputNo, mapping->size);
-  return {};
-}
-
-ValueRange
-OneToNTypeMapping::getConvertedValues(ValueRange convertedValues,
-                                      unsigned originalValueNo) const {
-  if (auto mapping = getInputMapping(originalValueNo))
-    return convertedValues.slice(mapping->inputNo, mapping->size);
-  return {};
-}
-
-void OneToNTypeMapping::convertLocation(
-    Value originalValue, unsigned originalValueNo,
-    llvm::SmallVectorImpl<Location> &result) const {
-  if (auto mapping = getInputMapping(originalValueNo))
-    result.append(mapping->size, originalValue.getLoc());
-}
-
-void OneToNTypeMapping::convertLocations(
-    ValueRange originalValues, llvm::SmallVectorImpl<Location> &result) const {
-  assert(originalValues.size() == getOriginalTypes().size());
-  for (auto [i, value] : llvm::enumerate(originalValues))
-    convertLocation(value, i, result);
-}
-
-static bool isIdentityConversion(Type originalType, TypeRange convertedTypes) {
-  return convertedTypes.size() == 1 && convertedTypes[0] == originalType;
-}
-
-bool OneToNTypeMapping::hasNonIdentityConversion() const {
-  // XXX: I think that the original types and the converted types are the same
-  //      iff there was no non-identity type conversion. If that is true, the
-  //      patterns could actually test whether there is anything useful to do
-  //      without having access to the signature conversion.
-  for (auto [i, originalType] : llvm::enumerate(originalTypes)) {
-    TypeRange types = getConvertedTypes(i);
-    if (!isIdentityConversion(originalType, types)) {
-      assert(TypeRange(originalTypes) != getConvertedTypes());
-      return true;
-    }
-  }
-  assert(TypeRange(originalTypes) == getConvertedTypes());
-  return false;
-}
-
-namespace {
-enum class CastKind {
-  // Casts block arguments in the target type back to the source type. (If
-  // necessary, this cast becomes an argument materialization.)
-  Argument,
-
-  // Casts other values in the target type back to the source type. (If
-  // necessary, this cast becomes a source materialization.)
-  Source,
-
-  // Casts values in the source type to the target type. (If necessary, this
-  // cast becomes a target materialization.)
-  Target
-};
-} // namespace
-
-/// Mapping of enum values to string values.
-StringRef getCastKindName(CastKind kind) {
-  static const std::unordered_map<CastKind, StringRef> castKindNames = {
-      {CastKind::Argument, "argument"},
-      {CastKind::Source, "source"},
-      {CastKind::Target, "target"}};
-  return castKindNames.at(kind);
-}
-
-/// Attribute name that is used to annotate inserted unrealized casts with their
-/// kind (source, argument, or target).
-static const char *const castKindAttrName =
-    "__one-to-n-type-conversion_cast-kind__";
-
-/// Builds an `UnrealizedConversionCastOp` from the given inputs to the given
-/// result types. Returns the result values of the cast.
-static ValueRange buildUnrealizedCast(OpBuilder &builder, TypeRange resultTypes,
-                                      ValueRange inputs, CastKind kind) {
-  // Special case: 1-to-N conversion with N = 0. No need to build an
-  // UnrealizedConversionCastOp because the op will always be dead.
-  if (resultTypes.empty())
-    return ValueRange();
-
-  // Create cast.
-  Location loc = builder.getUnknownLoc();
-  if (!inputs.empty())
-    loc = inputs.front().getLoc();
-  auto castOp =
-      builder.create<UnrealizedConversionCastOp>(loc, resultTypes, inputs);
-
-  // Store cast kind as attribute.
-  auto kindAttr = StringAttr::get(builder.getContext(), getCastKindName(kind));
-  castOp->setAttr(castKindAttrName, kindAttr);
-
-  return castOp->getResults();
-}
-
-/// Builds one `UnrealizedConversionCastOp` for each of the given original
-/// values using the respective target types given in the provided conversion
-/// mapping and returns the results of these casts. If the conversion mapping of
-/// a value maps a type to itself (i.e., is an identity conversion), then no
-/// cast is inserted and the original value is returned instead.
-/// Note that these unrealized casts are different from target materializations
-/// in that they are *always* inserted, even if they immediately fold away, such
-/// that patterns always see valid intermediate IR, whereas materializations are
-/// only used in the places where the unrealized casts *don't* fold away.
-static SmallVector<Value>
-buildUnrealizedForwardCasts(ValueRange originalValues,
-                            OneToNTypeMapping &conversion,
-                            RewriterBase &rewriter, CastKind kind) {
-
-  // Convert each operand one by one.
-  SmallVector<Value> convertedValues;
-  convertedValues.reserve(conversion.getConvertedTypes().size());
-  for (auto [idx, originalValue] : llvm::enumerate(originalValues)) {
-    TypeRange convertedTypes = conversion.getConvertedTypes(idx);
-
-    // Identity conversion: keep operand as is.
-    if (isIdentityConversion(originalValue.getType(), convertedTypes)) {
-      convertedValues.push_back(originalValue);
-      continue;
-    }
-
-    // Non-identity conversion: materialize target types.
-    ValueRange castResult =
-        buildUnrealizedCast(rewriter, convertedTypes, originalValue, kind);
-    convertedValues.append(castResult.begin(), castResult.end());
-  }
-
-  return convertedValues;
-}
-
-/// Builds one `UnrealizedConversionCastOp` for each sequence of the given
-/// original values to one value of the type they originated from, i.e., a
-/// "reverse" conversion from N converted values back to one value of the
-/// original type, using the given (forward) type conversion. If a given value
-/// was mapped to a value of the same type (i.e., the conversion in the mapping
-/// is an identity conversion), then the "converted" value is returned without
-/// cast.
-/// Note that these unrealized casts are different from source materializations
-/// in that they are *always* inserted, even if they immediately fold away, such
-/// that patterns always see valid intermediate IR, whereas materializations are
-/// only used in the places where the unrealized casts *don't* fold away.
-static SmallVector<Value>
-buildUnrealizedBackwardsCasts(ValueRange convertedValues,
-                              const OneToNTypeMapping &typeConversion,
-                              RewriterBase &rewriter) {
-  assert(typeConversion.getConvertedTypes() == convertedValues.getTypes());
-
-  // Create unrealized cast op for each converted result of the op.
-  SmallVector<Value> recastValues;
-  TypeRange originalTypes = typeConversion.getOriginalTypes();
-  recastValues.reserve(originalTypes.size());
-  auto convertedValueIt = convertedValues.begin();
-  for (auto [idx, originalType] : llvm::enumerate(originalTypes)) {
-    TypeRange convertedTypes = typeConversion.getConvertedTypes(idx);
-    size_t numConvertedValues = convertedTypes.size();
-    if (isIdentityConversion(originalType, convertedTypes)) {
-      // Identity conversion: take result as is.
-      recastValues.push_back(*convertedValueIt);
-    } else {
-      // Non-identity conversion: cast back to source type.
-      ValueRange recastValue = buildUnrealizedCast(
-          rewriter, originalType,
-          ValueRange{convertedValueIt, convertedValueIt + numConvertedValues},
-          CastKind::Source);
-      assert(recastValue.size() == 1);
-      recastValues.push_back(recastValue.front());
-    }
-    convertedValueIt += numConvertedValues;
-  }
-
-  return recastValues;
-}
-
-void OneToNPatternRewriter::replaceOp(Operation *op, ValueRange newValues,
-                                      const OneToNTypeMapping &resultMapping) {
-  // Create a cast back to the original types and replace the results of the
-  // original op with those.
-  assert(newValues.size() == resultMapping.getConvertedTypes().size());
-  assert(op->getResultTypes() == resultMapping.getOriginalTypes());
-  PatternRewriter::InsertionGuard g(*this);
-  setInsertionPointAfter(op);
-  SmallVector<Value> castResults =
-      buildUnrealizedBackwardsCasts(newValues, resultMapping, *this);
-  replaceOp(op, castResults);
-}
-
-Block *OneToNPatternRewriter::applySignatureConversion(
-    Block *block, OneToNTypeMapping &argumentConversion) {
-  PatternRewriter::InsertionGuard g(*this);
-
-  // Split the block at the beginning to get a new block to use for the
-  // updated signature.
-  SmallVector<Location> locs;
-  argumentConversion.convertLocations(block->getArguments(), locs);
-  Block *newBlock =
-      createBlock(block, argumentConversion.getConvertedTypes(), locs);
-  replaceAllUsesWith(block, newBlock);
-
-  // Create necessary casts in new block.
-  SmallVector<Value> castResults;
-  for (auto [i, arg] : llvm::enumerate(block->getArguments())) {
-    TypeRange convertedTypes = argumentConversion.getConvertedTypes(i);
-    ValueRange newArgs =
-        argumentConversion.getConvertedValues(newBlock->getArguments(), i);
-    if (isIdentityConversion(arg.getType(), convertedTypes)) {
-      // Identity conversion: take argument as is.
-      assert(newArgs.size() == 1);
-      castResults.push_back(newArgs.front());
-    } else {
-      // Non-identity conversion: cast the converted arguments to the original
-      // type.
-      PatternRewriter::InsertionGuard g(*this);
-      setInsertionPointToStart(newBlock);
-      ValueRange castResult = buildUnrealizedCast(*this, arg.getType(), newArgs,
-                                                  CastKind::Argument);
-      assert(castResult.size() == 1);
-      castResults.push_back(castResult.front());
-    }
-  }
-
-  // Merge old block into new block such that we only have the latter with the
-  // new signature.
-  mergeBlocks(block, newBlock, castResults);
-
-  return newBlock;
-}
-
-LogicalResult
-OneToNConversionPattern::matchAndRewrite(Operation *op,
-                                         PatternRewriter &rewriter) const {
-  auto *typeConverter = getTypeConverter();
-
-  // Construct conversion mapping for results.
-  Operation::result_type_range originalResultTypes = op->getResultTypes();
-  OneToNTypeMapping resultMapping(originalResultTypes);
-  if (failed(typeConverter->convertSignatureArgs(originalResultTypes,
-                                                 resultMapping)))
-    return failure();
-
-  // Construct conversion mapping for operands.
-  Operation::operand_type_range originalOperandTypes = op->getOperandTypes();
-  OneToNTypeMapping operandMapping(originalOperandTypes);
-  if (failed(typeConverter->convertSignatureArgs(originalOperandTypes,
-                                                 operandMapping)))
-    return failure();
-
-  // Cast operands to target types.
-  SmallVector<Value> convertedOperands = buildUnrealizedForwardCasts(
-      op->getOperands(), operandMapping, rewriter, CastKind::Target);
-
-  // Create a `OneToNPatternRewriter` for the pattern, which provides additional
-  // functionality.
-  // TODO(ingomueller): I guess it would be better to use only one rewriter
-  //                    throughout the whole pass, but that would require to
-  //                    drive the pattern application ourselves, which is a lot
-  //                    of additional boilerplate code. This seems to work fine,
-  //                    so I leave it like this for the time being.
-  OneToNPatternRewriter oneToNPatternRewriter(rewriter.getContext(),
-                                              rewriter.getListener());
-  oneToNPatternRewriter.restoreInsertionPoint(rewriter.saveInsertionPoint());
-
-  // Apply actual pattern.
-  if (failed(matchAndRewrite(op, oneToNPatternRewriter, operandMapping,
-                             resultMapping, convertedOperands)))
-    return failure();
-
-  return success();
-}
-
-namespace mlir {
-
-// This function applies the provided patterns using
-// `applyPatternsGreedily` and then replaces all newly inserted
-// `UnrealizedConversionCastOps` that haven't folded away. ("Backward" casts
-// from target to source types inserted by a `OneToNConversionPattern` normally
-// fold away with the "forward" casts from source to target types inserted by
-// the next pattern.) To understand which casts are "newly inserted", all casts
-// inserted by this pass are annotated with a string attribute that also
-// documents which kind of the cast (source, argument, or target).
-LogicalResult
-applyPartialOneToNConversion(Operation *op, TypeConverter &typeConverter,
-                             const FrozenRewritePatternSet &patterns) {
-#ifndef NDEBUG
-  // Remember existing unrealized casts. This data structure is only used in
-  // asserts; building it only for that purpose may be an overkill.
-  SmallSet<UnrealizedConversionCastOp, 4> existingCasts;
-  op->walk([&](UnrealizedConversionCastOp castOp) {
-    assert(!castOp->hasAttr(castKindAttrName));
-    existingCasts.insert(castOp);
-  });
-#endif // NDEBUG
-
-  // Apply provided conversion patterns.
-  if (failed(applyPatternsGreedily(op, patterns))) {
-    emitError(op->getLoc()) << "failed to apply conversion patterns";
-    return failure();
-  }
-
-  // Find all unrealized casts inserted by the pass that haven't folded away.
-  SmallVector<UnrealizedConversionCastOp> worklist;
-  op->walk([&](UnrealizedConversionCastOp castOp) {
-    if (castOp->hasAttr(castKindAttrName)) {
-      assert(!existingCasts.contains(castOp));
-      worklist.push_back(castOp);
-    }
-  });
-
-  // Replace new casts with user materializations.
-  IRRewriter rewriter(op->getContext());
-  for (UnrealizedConversionCastOp castOp : worklist) {
-    TypeRange resultTypes = castOp->getResultTypes();
-    ValueRange operands = castOp->getOperands();
-    StringRef castKind =
-        castOp->getAttrOfType<StringAttr>(castKindAttrName).getValue();
-    rewriter.setInsertionPoint(castOp);
-
-#ifndef NDEBUG
-    // Determine whether operands or results are already legal to test some
-    // assumptions for the different kind of materializations. These properties
-    // are only used it asserts and it may be overkill to compute them.
-    bool areOperandTypesLegal = llvm::all_of(
-        operands.getTypes(), [&](Type t) { return typeConverter.isLegal(t); });
-    bool areResultsTypesLegal = llvm::all_of(
-        resultTypes, [&](Type t) { return typeConverter.isLegal(t); });
-#endif // NDEBUG
-
-    // Add materialization and remember materialized results.
-    SmallVector<Value> materializedResults;
-    if (castKind == getCastKindName(CastKind::Target)) {
-      // Target materialization.
-      assert(!areOperandTypesLegal && areResultsTypesLegal &&
-             operands.size() == 1 && "found unexpected target cast");
-      materializedResults = typeConverter.materializeTargetConversion(
-          rewriter, castOp->getLoc(), resultTypes, operands.front());
-      if (materializedResults.empty()) {
-        emitError(castOp->getLoc())
-            << "failed to create target materialization";
-        return failure();
-      }
-    } else {
-      // Source and argument materializations.
-      assert(areOperandTypesLegal && !areResultsTypesLegal &&
-             resultTypes.size() == 1 && "found unexpected cast");
-      std::optional<Value> maybeResult;
-      if (castKind == getCastKindName(CastKind::Source)) {
-        // Source materialization.
-        maybeResult = typeConverter.materializeSourceConversion(
-            rewriter, castOp->getLoc(), resultTypes.front(),
-            castOp.getOperands());
-      } else {
-        // Argument materialization.
-        assert(castKind == getCastKindName(CastKind::Argument) &&
-               "unexpected value of cast kind attribute");
-        assert(llvm::all_of(operands, llvm::IsaPred<BlockArgument>));
-        maybeResult = typeConverter.materializeArgumentConversion(
-            rewriter, castOp->getLoc(), resultTypes.front(),
-            castOp.getOperands());
-      }
-      if (!maybeResult.has_value() || !maybeResult.value()) {
-        emitError(castOp->getLoc())
-            << "failed to create " << castKind << " materialization";
-        return failure();
-      }
-      materializedResults = {maybeResult.value()};
-    }
-
-    // Replace the cast with the result of the materialization.
-    rewriter.replaceOp(castOp, materializedResults);
-  }
-
-  return success();
-}
-
-namespace {
-class FunctionOpInterfaceSignatureConversion : public OneToNConversionPattern {
-public:
-  FunctionOpInterfaceSignatureConversion(StringRef functionLikeOpName,
-                                         MLIRContext *ctx,
-                                         const TypeConverter &converter)
-      : OneToNConversionPattern(converter, functionLikeOpName, /*benefit=*/1,
-                                ctx) {}
-
-  LogicalResult matchAndRewrite(Operation *op, OneToNPatternRewriter &rewriter,
-                                const OneToNTypeMapping &operandMapping,
-                                const OneToNTypeMapping &resultMapping,
-                                ValueRange convertedOperands) const override {
-    auto funcOp = cast<FunctionOpInterface>(op);
-    auto *typeConverter = getTypeConverter();
-
-    // Construct mapping for function arguments.
-    OneToNTypeMapping argumentMapping(funcOp.getArgumentTypes());
-    if (failed(typeConverter->convertSignatureArgs(funcOp.getArgumentTypes(),
-                                                   argumentMapping)))
-      return failure();
-
-    // Construct mapping for function results.
-    OneToNTypeMapping funcResultMapping(funcOp.getResultTypes());
-    if (failed(typeConverter->convertSignatureArgs(funcOp.getResultTypes(),
-                                                   funcResultMapping)))
-      return failure();
-
-    // Nothing to do if the op doesn't have any non-identity conversions for its
-    // operands or results.
-    if (!argumentMapping.hasNonIdentityConversion() &&
-        !funcResultMapping.hasNonIdentityConversion())
-      return failure();
-
-    // Update the function signature in-place.
-    auto newType = FunctionType::get(rewriter.getContext(),
-                                     argumentMapping.getConvertedTypes(),
-                                     funcResultMapping.getConvertedTypes());
-    rewriter.modifyOpInPlace(op, [&] { funcOp.setType(newType); });
-
-    // Update block signatures.
-    if (!funcOp.isExternal()) {
-      Region *region = &funcOp.getFunctionBody();
-      Block *block = &region->front();
-      rewriter.applySignatureConversion(block, argumentMapping);
-    }
-
-    return success();
-  }
-};
-} // namespace
-
-void populateOneToNFunctionOpInterfaceTypeConversionPattern(
-    StringRef functionLikeOpName, const TypeConverter &converter,
-    RewritePatternSet &patterns) {
-  patterns.add<FunctionOpInterfaceSignatureConversion>(
-      functionLikeOpName, patterns.getContext(), converter);
-}
-} // namespace mlir
diff --git a/mlir/test/Conversion/OneToNTypeConversion/one-to-n-type-conversion.mlir b/mlir/test/Conversion/OneToNTypeConversion/one-to-n-type-conversion.mlir
deleted file mode 100644
index 611ec0265cd37..0000000000000
--- a/mlir/test/Conversion/OneToNTypeConversion/one-to-n-type-conversion.mlir
+++ /dev/null
@@ -1,140 +0,0 @@
-// RUN: mlir-opt %s -split-input-file \
-// RUN:   -test-one-to-n-type-conversion="convert-tuple-ops" \
-// RUN: | FileCheck --check-prefix=CHECK-TUP %s
-
-// RUN: mlir-opt %s -split-input-file \
-// RUN:   -test-one-to-n-type-conversion="convert-func-ops" \
-// RUN: | FileCheck --check-prefix=CHECK-FUNC %s
-
-// RUN: mlir-opt %s -split-input-file \
-// RUN:   -test-one-to-n-type-conversion="convert-func-ops convert-tuple-ops" \
-// RUN: | FileCheck --check-prefix=CHECK-BOTH %s
-
-// Test case: Matching nested packs and unpacks just disappear.
-
-// CHECK-TUP-LABEL: func.func @pack_unpack(
-// CHECK-TUP-SAME:                          %[[ARG0:.*]]: i1,
-// CHECK-TUP-SAME:                          %[[ARG1:.*]]: i2) -> (i1, i2) {
-// CHECK-TUP-DAG:     return %[[ARG0]], %[[ARG1]] : i1, i2
-func.func @pack_unpack(%arg0: i1, %arg1: i2) -> (i1, i2) {
-  %0 = "test.make_tuple"() : () -> tuple<>
-  %1 = "test.make_tuple"(%arg1) : (i2) -> tuple<i2>
-  %2 = "test.make_tuple"(%1) : (tuple<i2>) -> tuple<tuple<i2>>
-  %3 = "test.make_tuple"(%0, %arg0, %2) : (tuple<>, i1, tuple<tuple<i2>>) -> tuple<tuple<>, i1, tuple<tuple<i2>>>
-  %4 = "test.get_tuple_element"(%3) {index = 0 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-  %5 = "test.get_tuple_element"(%3) {index = 1 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-  %6 = "test.get_tuple_element"(%3) {index = 2 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-  %7 = "test.get_tuple_element"(%6) {index = 0 : i32} : (tuple<tuple<i2>>) -> tuple<i2>
-  %8 = "test.get_tuple_element"(%7) {index = 0 : i32} : (tuple<i2>) -> i2
-  return %5, %8 : i1, i2
-}
-
-// -----
-
-// Test case: Appropriate materializations are created depending on which ops
-// are converted.
-
-// If we only convert the tuple ops, the original `get_tuple_element` ops will
-// disappear but one target materialization will be inserted from the
-// unconverted function arguments to each of the return values (which have
-// redundancy among themselves).
-//
-// CHECK-TUP-LABEL: func.func @materializations_tuple_args(
-// CHECK-TUP-SAME:                                         %[[ARG0:.*]]: tuple<tuple<>, i1, tuple<tuple<i2>>>) -> (i1, i2) {
-// CHECK-TUP-DAG:     %[[V0:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 0 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-// CHECK-TUP-DAG:     %[[V1:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 1 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-// CHECK-TUP-DAG:     %[[V2:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 2 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK-TUP-DAG:     %[[V3:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK-TUP-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
-// CHECK-TUP-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 0 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-// CHECK-TUP-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 1 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-// CHECK-TUP-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 2 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK-TUP-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK-TUP-DAG:     %[[V9:.*]] = "test.get_tuple_element"(%[[V8]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
-// CHECK-TUP-DAG:     return %[[V1]], %[[V9]] : i1, i2
-
-// If we only convert the func ops, argument materializations are created from
-// the converted tuple elements back to the tuples that the `get_tuple_element`
-// ops expect.
-//
-// CHECK-FUNC-LABEL: func.func @materializations_tuple_args(
-// CHECK-FUNC-SAME:                                         %[[ARG0:.*]]: i1,
-// CHECK-FUNC-SAME:                                         %[[ARG1:.*]]: i2) -> (i1, i2) {
-// CHECK-FUNC-DAG:     %[[V0:.*]] = "test.make_tuple"() : () -> tuple<>
-// CHECK-FUNC-DAG:     %[[V1:.*]] = "test.make_tuple"(%[[ARG1]]) : (i2) -> tuple<i2>
-// CHECK-FUNC-DAG:     %[[V2:.*]] = "test.make_tuple"(%[[V1]]) : (tuple<i2>) -> tuple<tuple<i2>>
-// CHECK-FUNC-DAG:     %[[V3:.*]] = "test.make_tuple"(%[[V0]], %[[ARG0]], %[[V2]]) : (tuple<>, i1, tuple<tuple<i2>>) -> tuple<tuple<>, i1, tuple<tuple<i2>>>
-// CHECK-FUNC-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-// CHECK-FUNC-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-// CHECK-FUNC-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 2 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK-FUNC-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK-FUNC-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
-// CHECK-FUNC-DAG:     return %[[V5]], %[[V8]] : i1, i2
-
-// If we convert both tuple and func ops, basically everything disappears.
-//
-// CHECK-BOTH-LABEL: func.func @materializations_tuple_args(
-// CHECK-BOTH-SAME:                                         %[[ARG0:.*]]: i1,
-// CHECK-BOTH-SAME:                                         %[[ARG1:.*]]: i2) -> (i1, i2) {
-// CHECK-BOTH-DAG:     return %[[ARG0]], %[[ARG1]] : i1, i2
-
-func.func @materializations_tuple_args(%arg0: tuple<tuple<>, i1, tuple<tuple<i2>>>) -> (i1, i2) {
-  %0 = "test.get_tuple_element"(%arg0) {index = 0 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-  %1 = "test.get_tuple_element"(%arg0) {index = 1 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-  %2 = "test.get_tuple_element"(%arg0) {index = 2 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-  %3 = "test.get_tuple_element"(%2) {index = 0 : i32} : (tuple<tuple<i2>>) -> tuple<i2>
-  %4 = "test.get_tuple_element"(%3) {index = 0 : i32} : (tuple<i2>) -> i2
-  return %1, %4 : i1, i2
-}
-// -----
-
-// Test case: Appropriate materializations are created depending on which ops
-// are converted.
-
-// If we only convert the tuple ops, the original `make_tuple` ops will
-// disappear but a source materialization will be inserted from the result of
-// conversion (which, for `make_tuple`, are the original ops that get forwarded)
-// to the operands of the unconverted op with the original type (i.e.,
-// `return`).
-
-// CHECK-TUP-LABEL: func.func @materializations_tuple_return(
-// CHECK-TUP-SAME:                                           %[[ARG0:.*]]: i1,
-// CHECK-TUP-SAME:                                           %[[ARG1:.*]]: i2) -> tuple<tuple<>, i1, tuple<tuple<i2>>> {
-// CHECK-TUP-DAG:     %[[V0:.*]] = "test.make_tuple"() : () -> tuple<>
-// CHECK-TUP-DAG:     %[[V1:.*]] = "test.make_tuple"(%[[ARG1]]) : (i2) -> tuple<i2>
-// CHECK-TUP-DAG:     %[[V2:.*]] = "test.make_tuple"(%[[V1]]) : (tuple<i2>) -> tuple<tuple<i2>>
-// CHECK-TUP-DAG:     %[[V3:.*]] = "test.make_tuple"(%[[V0]], %[[ARG0]], %[[V2]]) : (tuple<>, i1, tuple<tuple<i2>>) -> tuple<tuple<>, i1, tuple<tuple<i2>>>
-// CHECK-TUP-DAG:     return %[[V3]] : tuple<tuple<>, i1, tuple<tuple<i2>>>
-
-// If we only convert the func ops, target materializations are created from
-// original tuples produced by `make_tuple` to its constituent elements that the
-// converted op (i.e., `return`) expect.
-//
-// CHECK-FUNC-LABEL: func.func @materializations_tuple_return(
-// CHECK-FUNC-SAME:                                           %[[ARG0:.*]]: i1,
-// CHECK-FUNC-SAME:                                           %[[ARG1:.*]]: i2) -> (i1, i2) {
-// CHECK-FUNC-DAG:     %[[V0:.*]] = "test.make_tuple"() : () -> tuple<>
-// CHECK-FUNC-DAG:     %[[V1:.*]] = "test.make_tuple"(%[[ARG1]]) : (i2) -> tuple<i2>
-// CHECK-FUNC-DAG:     %[[V2:.*]] = "test.make_tuple"(%[[V1]]) : (tuple<i2>) -> tuple<tuple<i2>>
-// CHECK-FUNC-DAG:     %[[V3:.*]] = "test.make_tuple"(%[[V0]], %[[ARG0]], %[[V2]]) : (tuple<>, i1, tuple<tuple<i2>>) -> tuple<tuple<>, i1, tuple<tuple<i2>>>
-// CHECK-FUNC-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-// CHECK-FUNC-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-// CHECK-FUNC-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 2 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK-FUNC-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK-FUNC-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
-// CHECK-FUNC-DAG:     return %[[V5]], %[[V8]] : i1, i2
-
-// If we convert both tuple and func ops, basically everything disappears.
-//
-// CHECK-BOTH-LABEL: func.func @materializations_tuple_return(
-// CHECK-BOTH-SAME:                                           %[[ARG0:.*]]: i1,
-// CHECK-BOTH-SAME:                                           %[[ARG1:.*]]: i2) -> (i1, i2) {
-// CHECK-BOTH-DAG:     return %[[ARG0]], %[[ARG1]] : i1, i2
-
-func.func @materializations_tuple_return(%arg0: i1, %arg1: i2) -> tuple<tuple<>, i1, tuple<tuple<i2>>> {
-  %0 = "test.make_tuple"() : () -> tuple<>
-  %1 = "test.make_tuple"(%arg1) : (i2) -> tuple<i2>
-  %2 = "test.make_tuple"(%1) : (tuple<i2>) -> tuple<tuple<i2>>
-  %3 = "test.make_tuple"(%0, %arg0, %2) : (tuple<>, i1, tuple<tuple<i2>>) -> tuple<tuple<>, i1, tuple<tuple<i2>>>
-  return %3 : tuple<tuple<>, i1, tuple<tuple<i2>>>
-}
diff --git a/mlir/test/Conversion/OneToNTypeConversion/scf-structural-one-to-n-type-conversion.mlir b/mlir/test/Conversion/OneToNTypeConversion/scf-structural-one-to-n-type-conversion.mlir
deleted file mode 100644
index 535ab68e8d893..0000000000000
--- a/mlir/test/Conversion/OneToNTypeConversion/scf-structural-one-to-n-type-conversion.mlir
+++ /dev/null
@@ -1,183 +0,0 @@
-// RUN: mlir-opt %s -split-input-file \
-// RUN:   -test-one-to-n-type-conversion="convert-func-ops convert-scf-ops" \
-// RUN: | FileCheck %s
-
-// Test case: Nested 1:N type conversion is carried through scf.if and
-// scf.yield.
-
-// CHECK-LABEL: func.func @if_result(
-// CHECK-SAME:                       %[[ARG0:.*]]: i1,
-// CHECK-SAME:                       %[[ARG1:.*]]: i2,
-// CHECK-SAME:                       %[[ARG2:.*]]: i1) -> (i1, i2) {
-// CHECK-NEXT:    %[[V0:.*]]:2 = scf.if %[[ARG2]] -> (i1, i2) {
-// CHECK-NEXT:     scf.yield %[[ARG0]], %[[ARG1]] : i1, i2
-// CHECK-NEXT:   } else {
-// CHECK-NEXT:     scf.yield %[[ARG0]], %[[ARG1]] : i1, i2
-// CHECK-NEXT:   }
-// CHECK-NEXT:   return %[[V0]]#0, %[[V0]]#1 : i1, i2
-func.func @if_result(%arg0: tuple<tuple<>, i1, tuple<i2>>, %arg1: i1) -> tuple<tuple<>, i1, tuple<i2>> {
-  %0 = scf.if %arg1 -> (tuple<tuple<>, i1, tuple<i2>>) {
-    scf.yield %arg0 : tuple<tuple<>, i1, tuple<i2>>
-  } else {
-    scf.yield %arg0 : tuple<tuple<>, i1, tuple<i2>>
-  }
-  return %0 : tuple<tuple<>, i1, tuple<i2>>
-}
-
-// -----
-
-// Test case: Nested 1:N type conversion is carried through scf.if and
-// scf.yield and unconverted ops inside have proper materializations.
-
-// CHECK-LABEL: func.func @if_tuple_ops(
-// CHECK-SAME:                          %[[ARG0:.*]]: i1,
-// CHECK-SAME:                          %[[ARG1:.*]]: i1) -> i1 {
-// CHECK-NEXT:    %[[V0:.*]] = "test.make_tuple"() : () -> tuple<>
-// CHECK-NEXT:    %[[V1:.*]] = "test.make_tuple"(%[[V0]], %[[ARG0]]) : (tuple<>, i1) -> tuple<tuple<>, i1>
-// CHECK-NEXT:    %[[V2:.*]] = scf.if %[[ARG1]] -> (i1) {
-// CHECK-NEXT:      %[[V3:.*]] = "test.op"(%[[V1]]) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:      %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
-// CHECK-NEXT:      scf.yield %[[V5]] : i1
-// CHECK-NEXT:    } else {
-// CHECK-NEXT:      %[[V6:.*]] = "test.source"() : () -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:      %[[V8:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
-// CHECK-NEXT:      scf.yield %[[V8]] : i1
-// CHECK-NEXT:    }
-// CHECK-NEXT:    return %[[V2]] : i1
-func.func @if_tuple_ops(%arg0: tuple<tuple<>, i1>, %arg1: i1) -> tuple<tuple<>, i1> {
-  %0 = scf.if %arg1 -> (tuple<tuple<>, i1>) {
-    %1 = "test.op"(%arg0) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-    scf.yield %1 : tuple<tuple<>, i1>
-  } else {
-    %1 = "test.source"() : () -> tuple<tuple<>, i1>
-    scf.yield %1 : tuple<tuple<>, i1>
-  }
-  return %0 : tuple<tuple<>, i1>
-}
-// -----
-
-// Test case: Nested 1:N type conversion is carried through scf.while,
-// scf.condition, and scf.yield.
-
-// CHECK-LABEL: func.func @while_operands_results(
-// CHECK-SAME:                                    %[[ARG0:.*]]: i1,
-// CHECK-SAME:                                    %[[ARG1:.*]]: i2,
-// CHECK-SAME:                                    %[[ARG2:.*]]: i1) -> (i1, i2) {
-//   %[[V0:.*]]:2 = scf.while (%[[ARG3:.*]] = %[[ARG0]], %[[ARG4:.*]] = %[[ARG1]]) : (i1, i2) -> (i1, i2) {
-//     scf.condition(%arg2) %[[ARG3]], %[[ARG4]] : i1, i2
-//   } do {
-//   ^bb0(%[[ARG5:.*]]: i1, %[[ARG6:.*]]: i2):
-//     scf.yield %[[ARG5]], %[[ARG4]] : i1, i2
-//   }
-//   return %[[V0]]#0, %[[V0]]#1 : i1, i2
-func.func @while_operands_results(%arg0: tuple<tuple<>, i1, tuple<i2>>, %arg1: i1) -> tuple<tuple<>, i1, tuple<i2>> {
-  %0 = scf.while (%arg2 = %arg0) : (tuple<tuple<>, i1, tuple<i2>>) -> tuple<tuple<>, i1, tuple<i2>> {
-    scf.condition(%arg1) %arg2 : tuple<tuple<>, i1, tuple<i2>>
-  } do {
-  ^bb0(%arg2: tuple<tuple<>, i1, tuple<i2>>):
-    scf.yield %arg2 : tuple<tuple<>, i1, tuple<i2>>
-  }
-  return %0 : tuple<tuple<>, i1, tuple<i2>>
-}
-
-// -----
-
-// Test case: Nested 1:N type conversion is carried through scf.while,
-// scf.condition, and unconverted ops inside have proper materializations.
-
-// CHECK-LABEL: func.func @while_tuple_ops(
-// CHECK-SAME:                             %[[ARG0:.*]]: i1,
-// CHECK-SAME:                             %[[ARG1:.*]]: i1) -> i1 {
-// CHECK-NEXT:    %[[V0:.*]] = scf.while (%[[ARG2:.*]] = %[[ARG0]]) : (i1) -> i1 {
-// CHECK-NEXT:      %[[V1:.*]] = "test.make_tuple"() : () -> tuple<>
-// CHECK-NEXT:      %[[V2:.*]] = "test.make_tuple"(%[[V1]], %[[ARG2]]) : (tuple<>, i1) -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V3:.*]] = "test.op"(%[[V2]]) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:      %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
-// CHECK-NEXT:      scf.condition(%[[ARG1]]) %[[V5]] : i1
-// CHECK-NEXT:    } do {
-// CHECK-NEXT:    ^bb0(%[[ARG3:.*]]: i1):
-// CHECK-NEXT:      %[[V6:.*]] = "test.source"() : () -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:      %[[V8:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
-// CHECK-NEXT:      scf.yield %[[V8]] : i1
-// CHECK-NEXT:    }
-// CHECK-NEXT:    return %[[V0]] : i1
-func.func @while_tuple_ops(%arg0: tuple<tuple<>, i1>, %arg1: i1) -> tuple<tuple<>, i1> {
-  %0 = scf.while (%arg2 = %arg0) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1> {
-    %1 = "test.op"(%arg2) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-    scf.condition(%arg1) %1 : tuple<tuple<>, i1>
-  } do {
-  ^bb0(%arg2: tuple<tuple<>, i1>):
-    %1 = "test.source"() : () -> tuple<tuple<>, i1>
-    scf.yield %1 : tuple<tuple<>, i1>
-  }
-  return %0 : tuple<tuple<>, i1>
-}
-
-// -----
-
-// Test case: Nested 1:N type conversion is carried through scf.for and scf.yield.
-
-// CHECK-LABEL: func.func @for_operands_results(
-// CHECK-SAME:                                    %[[ARG0:.*]]: i1,
-// CHECK-SAME:                                    %[[ARG1:.*]]: i2) -> (i1, i2) {
-// CHECK-NEXT:    %[[C0:.+]] = arith.constant 0 : index
-// CHECK-NEXT:    %[[C1:.+]] = arith.constant 1 : index
-// CHECK-NEXT:    %[[C10:.+]] = arith.constant 10 : index
-// CHECK-NEXT:    %[[OUT:.+]]:2 = scf.for %arg2 = %[[C0]] to %[[C10]] step %[[C1]] iter_args(%[[ITER0:.+]] = %[[ARG0]], %[[ITER1:.+]] = %[[ARG1]]) -> (i1, i2) {
-// CHECK-NEXT:     scf.yield %[[ITER0]], %[[ITER1]] : i1, i2
-// CHECK-NEXT:    }
-// CHECK-NEXT:    return %[[OUT]]#0, %[[OUT]]#1 : i1, i2
-
-func.func @for_operands_results(%arg0: tuple<tuple<>, i1, tuple<i2>>) -> tuple<tuple<>, i1, tuple<i2>> {
-  %c0 = arith.constant 0 : index
-  %c1 = arith.constant 1 : index
-  %c10 = arith.constant 10 : index
-
-  %0 = scf.for %i = %c0 to %c10 step %c1 iter_args(%acc = %arg0) -> tuple<tuple<>, i1, tuple<i2>> {
-    scf.yield %acc : tuple<tuple<>, i1, tuple<i2>>
-  }
-
-  return %0 : tuple<tuple<>, i1, tuple<i2>>
-}
-
-// -----
-
-// Test case: Nested 1:N type conversion is carried through scf.for and scf.yield
-
-// CHECK-LABEL: func.func @for_tuple_ops(
-// CHECK-SAME:                             %[[ARG0:.+]]: i1) -> i1 {
-// CHECK-NEXT: %[[C0:.+]] = arith.constant 0 : index
-// CHECK-NEXT: %[[C1:.+]] = arith.constant 1 : index
-// CHECK-NEXT: %[[C10:.+]] = arith.constant 10 : index
-// CHECK-NEXT: %[[FOR:.+]] = scf.for %arg1 = %[[C0]] to %[[C10]] step %[[C1]] iter_args(%[[ITER:.+]] = %[[ARG0]]) -> (i1) {
-// CHECK-NEXT:   %[[V1:.+]] = "test.make_tuple"() : () -> tuple<>
-// CHECK-NEXT:   %[[V2:.+]] = "test.make_tuple"(%[[V1]], %[[ITER]]) : (tuple<>, i1) -> tuple<tuple<>, i1>
-// CHECK-NEXT:   %[[V3:.+]] = "test.op"(%[[V2]]) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-// CHECK-NEXT:   %[[V4:.+]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:   %[[V5:.+]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
-// CHECK-NEXT:   scf.yield %[[V5]] : i1
-// CHECK-NEXT: }
-// CHECK-NEXT: %[[V6:.+]] = "test.make_tuple"() : () -> tuple<>
-// CHECK-NEXT: %[[V7:.+]] = "test.make_tuple"(%[[V6]], %[[FOR]]) : (tuple<>, i1) -> tuple<tuple<>, i1>
-// CHECK-NEXT: %[[V8:.+]] = "test.op"(%[[V7]]) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-// CHECK-NEXT: %[[V9:.+]] = "test.get_tuple_element"(%[[V8]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT: %[[V10:.+]] = "test.get_tuple_element"(%[[V8]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
-// CHECK-NEXT: return %[[V10]] : i1
-
-func.func @for_tuple_ops(%arg0: tuple<tuple<>, i1>) -> tuple<tuple<>, i1> {
-  %c0 = arith.constant 0 : index
-  %c1 = arith.constant 1 : index
-  %c10 = arith.constant 10 : index
-
-  %0 = scf.for %i = %c0 to %c10 step %c1 iter_args(%acc = %arg0) -> tuple<tuple<>, i1> {
-    %1 = "test.op"(%acc) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-    scf.yield %1 : tuple<tuple<>, i1>
-  } 
-
-  %1 = "test.op"(%0) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-  return %1 : tuple<tuple<>, i1>
-}
diff --git a/mlir/test/Transforms/decompose-call-graph-types.mlir b/mlir/test/Transforms/decompose-call-graph-types.mlir
index 4e641317ac2f3..55d78d9fedebb 100644
--- a/mlir/test/Transforms/decompose-call-graph-types.mlir
+++ b/mlir/test/Transforms/decompose-call-graph-types.mlir
@@ -1,19 +1,11 @@
 // RUN: mlir-opt %s -split-input-file -test-decompose-call-graph-types | FileCheck %s
 
-// RUN: mlir-opt %s -split-input-file \
-// RUN:   -test-one-to-n-type-conversion="convert-func-ops" \
-// RUN: | FileCheck %s --check-prefix=CHECK-12N
-
 // Test case: Most basic case of a 1:N decomposition, an identity function.
 
 // CHECK-LABEL:   func @identity(
 // CHECK-SAME:                   %[[ARG0:.*]]: i1,
 // CHECK-SAME:                   %[[ARG1:.*]]: i32) -> (i1, i32) {
 // CHECK:           return %[[ARG0]], %[[ARG1]] : i1, i32
-// CHECK-12N-LABEL:   func @identity(
-// CHECK-12N-SAME:                   %[[ARG0:.*]]: i1,
-// CHECK-12N-SAME:                   %[[ARG1:.*]]: i32) -> (i1, i32) {
-// CHECK-12N:           return %[[ARG0]], %[[ARG1]] : i1, i32
 func.func @identity(%arg0: tuple<i1, i32>) -> tuple<i1, i32> {
   return %arg0 : tuple<i1, i32>
 }
@@ -25,9 +17,6 @@ func.func @identity(%arg0: tuple<i1, i32>) -> tuple<i1, i32> {
 // CHECK-LABEL:   func @identity_1_to_1_no_materializations(
 // CHECK-SAME:                                              %[[ARG0:.*]]: i1) -> i1 {
 // CHECK:           return %[[ARG0]] : i1
-// CHECK-12N-LABEL:   func @identity_1_to_1_no_materializations(
-// CHECK-12N-SAME:                                              %[[ARG0:.*]]: i1) -> i1 {
-// CHECK-12N:           return %[[ARG0]] : i1
 func.func @identity_1_to_1_no_materializations(%arg0: tuple<i1>) -> tuple<i1> {
   return %arg0 : tuple<i1>
 }
@@ -39,9 +28,6 @@ func.func @identity_1_to_1_no_materializations(%arg0: tuple<i1>) -> tuple<i1> {
 // CHECK-LABEL:   func @recursive_decomposition(
 // CHECK-SAME:                                   %[[ARG0:.*]]: i1) -> i1 {
 // CHECK:           return %[[ARG0]] : i1
-// CHECK-12N-LABEL:   func @recursive_decomposition(
-// CHECK-12N-SAME:                                   %[[ARG0:.*]]: i1) -> i1 {
-// CHECK-12N:           return %[[ARG0]] : i1
 func.func @recursive_decomposition(%arg0: tuple<tuple<tuple<i1>>>) -> tuple<tuple<tuple<i1>>> {
   return %arg0 : tuple<tuple<tuple<i1>>>
 }
@@ -54,10 +40,6 @@ func.func @recursive_decomposition(%arg0: tuple<tuple<tuple<i1>>>) -> tuple<tupl
 // CHECK-SAME:                 %[[ARG0:.*]]: i1,
 // CHECK-SAME:                 %[[ARG1:.*]]: i2) -> (i1, i2) {
 // CHECK:           return %[[ARG0]], %[[ARG1]] : i1, i2
-// CHECK-12N-LABEL:   func @mixed_recursive_decomposition(
-// CHECK-12N-SAME:                 %[[ARG0:.*]]: i1,
-// CHECK-12N-SAME:                 %[[ARG1:.*]]: i2) -> (i1, i2) {
-// CHECK-12N:           return %[[ARG0]], %[[ARG1]] : i1, i2
 func.func @mixed_recursive_decomposition(%arg0: tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>> {
   return %arg0 : tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>
 }
@@ -67,7 +49,6 @@ func.func @mixed_recursive_decomposition(%arg0: tuple<tuple<>, tuple<i1>, tuple<
 // Test case: Check decomposition of calls.
 
 // CHECK-LABEL:   func private @callee(i1, i32) -> (i1, i32)
-// CHECK-12N-LABEL:   func private @callee(i1, i32) -> (i1, i32)
 func.func private @callee(tuple<i1, i32>) -> tuple<i1, i32>
 
 // CHECK-LABEL:   func @caller(
@@ -75,11 +56,6 @@ func.func private @callee(tuple<i1, i32>) -> tuple<i1, i32>
 // CHECK-SAME:                 %[[ARG1:.*]]: i32) -> (i1, i32) {
 // CHECK:           %[[V0:.*]]:2 = call @callee(%[[ARG0]], %[[ARG1]]) : (i1, i32) -> (i1, i32)
 // CHECK:           return %[[V0]]#0, %[[V0]]#1 : i1, i32
-// CHECK-12N-LABEL:   func @caller(
-// CHECK-12N-SAME:                 %[[ARG0:.*]]: i1,
-// CHECK-12N-SAME:                 %[[ARG1:.*]]: i32) -> (i1, i32) {
-// CHECK-12N:           %[[V0:.*]]:2 = call @callee(%[[ARG0]], %[[ARG1]]) : (i1, i32) -> (i1, i32)
-// CHECK-12N:           return %[[V0]]#0, %[[V0]]#1 : i1, i32
 func.func @caller(%arg0: tuple<i1, i32>) -> tuple<i1, i32> {
   %0 = call @callee(%arg0) : (tuple<i1, i32>) -> tuple<i1, i32>
   return %0 : tuple<i1, i32>
@@ -90,15 +66,11 @@ func.func @caller(%arg0: tuple<i1, i32>) -> tuple<i1, i32> {
 // Test case: Type that decomposes to nothing (that is, a 1:0 decomposition).
 
 // CHECK-LABEL:   func private @callee()
-// CHECK-12N-LABEL:   func private @callee()
 func.func private @callee(tuple<>) -> tuple<>
 
 // CHECK-LABEL:   func @caller() {
 // CHECK:           call @callee() : () -> ()
 // CHECK:           return
-// CHECK-12N-LABEL:   func @caller() {
-// CHECK-12N:           call @callee() : () -> ()
-// CHECK-12N:           return
 func.func @caller(%arg0: tuple<>) -> tuple<> {
   %0 = call @callee(%arg0) : (tuple<>) -> (tuple<>)
   return %0 : tuple<>
@@ -114,11 +86,6 @@ func.func @caller(%arg0: tuple<>) -> tuple<> {
 // CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
 // CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
 // CHECK:           return %[[RET0]], %[[RET1]] : i1, i32
-// CHECK-12N-LABEL:   func @unconverted_op_result() -> (i1, i32) {
-// CHECK-12N:           %[[UNCONVERTED_VALUE:.*]] = "test.source"() : () -> tuple<i1, i32>
-// CHECK-12N:           %[[RET0:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
-// CHECK-12N:           %[[RET1:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
-// CHECK-12N:           return %[[RET0]], %[[RET1]] : i1, i32
 func.func @unconverted_op_result() -> tuple<i1, i32> {
   %0 = "test.source"() : () -> (tuple<i1, i32>)
   return %0 : tuple<i1, i32>
@@ -139,16 +106,6 @@ func.func @unconverted_op_result() -> tuple<i1, i32> {
 // CHECK:           %[[V4:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 1 : i32}> : (tuple<i1, tuple<i32>>) -> tuple<i32>
 // CHECK:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 0 : i32}> : (tuple<i32>) -> i32
 // CHECK:           return %[[V3]], %[[V5]] : i1, i32
-// CHECK-12N-LABEL:   func @nested_unconverted_op_result(
-// CHECK-12N-SAME:                 %[[ARG0:.*]]: i1,
-// CHECK-12N-SAME:                 %[[ARG1:.*]]: i32) -> (i1, i32) {
-// CHECK-12N:           %[[V0:.*]] = "test.make_tuple"(%[[ARG1]]) : (i32) -> tuple<i32>
-// CHECK-12N:           %[[V1:.*]] = "test.make_tuple"(%[[ARG0]], %[[V0]]) : (i1, tuple<i32>) -> tuple<i1, tuple<i32>>
-// CHECK-12N:           %[[V2:.*]] = "test.op"(%[[V1]]) : (tuple<i1, tuple<i32>>) -> tuple<i1, tuple<i32>>
-// CHECK-12N:           %[[V3:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 0 : i32}> : (tuple<i1, tuple<i32>>) -> i1
-// CHECK-12N:           %[[V4:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 1 : i32}> : (tuple<i1, tuple<i32>>) -> tuple<i32>
-// CHECK-12N:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 0 : i32}> : (tuple<i32>) -> i32
-// CHECK-12N:           return %[[V3]], %[[V5]] : i1, i32
 func.func @nested_unconverted_op_result(%arg: tuple<i1, tuple<i32>>) -> tuple<i1, tuple<i32>> {
   %0 = "test.op"(%arg) : (tuple<i1, tuple<i32>>) -> (tuple<i1, tuple<i32>>)
   return %0 : tuple<i1, tuple<i32>>
@@ -160,7 +117,6 @@ func.func @nested_unconverted_op_result(%arg: tuple<i1, tuple<i32>>) -> tuple<i1
 // This makes sure to test the cases if 1:0, 1:1, and 1:N decompositions.
 
 // CHECK-LABEL:   func private @callee(i1, i2, i3, i4, i5, i6) -> (i1, i2, i3, i4, i5, i6)
-// CHECK-12N-LABEL:   func private @callee(i1, i2, i3, i4, i5, i6) -> (i1, i2, i3, i4, i5, i6)
 func.func private @callee(tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6) -> (tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6)
 
 // CHECK-LABEL:   func @caller(
@@ -172,15 +128,6 @@ func.func private @callee(tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6) -> (tup
 // CHECK-SAME:                 %[[I6:.*]]: i6) -> (i1, i2, i3, i4, i5, i6) {
 // CHECK:           %[[CALL:.*]]:6 = call @callee(%[[I1]], %[[I2]], %[[I3]], %[[I4]], %[[I5]], %[[I6]]) : (i1, i2, i3, i4, i5, i6) -> (i1, i2, i3, i4, i5, i6)
 // CHECK:           return %[[CALL]]#0, %[[CALL]]#1, %[[CALL]]#2, %[[CALL]]#3, %[[CALL]]#4, %[[CALL]]#5 : i1, i2, i3, i4, i5, i6
-// CHECK-12N-LABEL:   func @caller(
-// CHECK-12N-SAME:                 %[[I1:.*]]: i1,
-// CHECK-12N-SAME:                 %[[I2:.*]]: i2,
-// CHECK-12N-SAME:                 %[[I3:.*]]: i3,
-// CHECK-12N-SAME:                 %[[I4:.*]]: i4,
-// CHECK-12N-SAME:                 %[[I5:.*]]: i5,
-// CHECK-12N-SAME:                 %[[I6:.*]]: i6) -> (i1, i2, i3, i4, i5, i6) {
-// CHECK-12N:           %[[CALL:.*]]:6 = call @callee(%[[I1]], %[[I2]], %[[I3]], %[[I4]], %[[I5]], %[[I6]]) : (i1, i2, i3, i4, i5, i6) -> (i1, i2, i3, i4, i5, i6)
-// CHECK-12N:           return %[[CALL]]#0, %[[CALL]]#1, %[[CALL]]#2, %[[CALL]]#3, %[[CALL]]#4, %[[CALL]]#5 : i1, i2, i3, i4, i5, i6
 func.func @caller(%arg0: tuple<>, %arg1: i1, %arg2: tuple<i2>, %arg3: i3, %arg4: tuple<i4, i5>, %arg5: i6) -> (tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6) {
   %0, %1, %2, %3, %4, %5 = call @callee(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) : (tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6) -> (tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6)
   return %0, %1, %2, %3, %4, %5 : tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6
diff --git a/mlir/test/lib/Conversion/CMakeLists.txt b/mlir/test/lib/Conversion/CMakeLists.txt
index 19975f671b081..c09496be729be 100644
--- a/mlir/test/lib/Conversion/CMakeLists.txt
+++ b/mlir/test/lib/Conversion/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_subdirectory(ConvertToSPIRV)
 add_subdirectory(FuncToLLVM)
 add_subdirectory(MathToVCIX)
-add_subdirectory(OneToNTypeConversion)
 add_subdirectory(VectorToSPIRV)
diff --git a/mlir/test/lib/Conversion/OneToNTypeConversion/CMakeLists.txt b/mlir/test/lib/Conversion/OneToNTypeConversion/CMakeLists.txt
deleted file mode 100644
index f0ea87946a983..0000000000000
--- a/mlir/test/lib/Conversion/OneToNTypeConversion/CMakeLists.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-add_mlir_library(MLIRTestOneToNTypeConversionPass
-  TestOneToNTypeConversionPass.cpp
-
-  EXCLUDE_FROM_LIBMLIR
-
-  LINK_LIBS PUBLIC
-  MLIRTestDialect
- )
-mlir_target_link_libraries(MLIRTestOneToNTypeConversionPass PUBLIC
-  MLIRFuncDialect
-  MLIRFuncTransforms
-  MLIRIR
-  MLIRPass
-  MLIRSCFDialect
-  MLIRSCFTransforms
-  MLIRTransformUtils
- )
-
-target_include_directories(MLIRTestOneToNTypeConversionPass
-  PRIVATE
-  ${CMAKE_CURRENT_SOURCE_DIR}/../../Dialect/Test
-  ${CMAKE_CURRENT_BINARY_DIR}/../../Dialect/Test
-  )
diff --git a/mlir/test/lib/Conversion/OneToNTypeConversion/TestOneToNTypeConversionPass.cpp b/mlir/test/lib/Conversion/OneToNTypeConversion/TestOneToNTypeConversionPass.cpp
deleted file mode 100644
index b18dfd8bb22cb..0000000000000
--- a/mlir/test/lib/Conversion/OneToNTypeConversion/TestOneToNTypeConversionPass.cpp
+++ /dev/null
@@ -1,261 +0,0 @@
-//===- TestOneToNTypeConversionPass.cpp - Test pass 1:N type conv. utils --===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "TestDialect.h"
-#include "TestOps.h"
-#include "mlir/Dialect/Func/Transforms/OneToNFuncConversions.h"
-#include "mlir/Dialect/SCF/Transforms/Patterns.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Transforms/OneToNTypeConversion.h"
-
-using namespace mlir;
-
-namespace {
-/// Test pass that exercises the (poor-man's) 1:N type conversion mechanisms
-/// in `applyPartialOneToNConversion` by converting built-in tuples to the
-/// elements they consist of as well as some dummy ops operating on these
-/// tuples.
-struct TestOneToNTypeConversionPass
-    : public PassWrapper<TestOneToNTypeConversionPass,
-                         OperationPass<ModuleOp>> {
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestOneToNTypeConversionPass)
-
-  TestOneToNTypeConversionPass() = default;
-  TestOneToNTypeConversionPass(const TestOneToNTypeConversionPass &pass)
-      : PassWrapper(pass) {}
-
-  void getDependentDialects(DialectRegistry &registry) const override {
-    registry.insert<test::TestDialect>();
-  }
-
-  StringRef getArgument() const final {
-    return "test-one-to-n-type-conversion";
-  }
-
-  StringRef getDescription() const final {
-    return "Test pass for 1:N type conversion";
-  }
-
-  Option<bool> convertFuncOps{*this, "convert-func-ops",
-                              llvm::cl::desc("Enable conversion on func ops"),
-                              llvm::cl::init(false)};
-
-  Option<bool> convertSCFOps{*this, "convert-scf-ops",
-                             llvm::cl::desc("Enable conversion on scf ops"),
-                             llvm::cl::init(false)};
-
-  Option<bool> convertTupleOps{*this, "convert-tuple-ops",
-                               llvm::cl::desc("Enable conversion on tuple ops"),
-                               llvm::cl::init(false)};
-
-  void runOnOperation() override;
-};
-
-} // namespace
-
-namespace mlir {
-namespace test {
-void registerTestOneToNTypeConversionPass() {
-  PassRegistration<TestOneToNTypeConversionPass>();
-}
-} // namespace test
-} // namespace mlir
-
-namespace {
-
-/// Test pattern on for the `make_tuple` op from the test dialect that converts
-/// this kind of op into it's "decomposed" form, i.e., the elements of the tuple
-/// that is being produced by `test.make_tuple`, which are really just the
-/// operands of this op.
-class ConvertMakeTupleOp
-    : public OneToNOpConversionPattern<::test::MakeTupleOp> {
-public:
-  using OneToNOpConversionPattern<
-      ::test::MakeTupleOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(::test::MakeTupleOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    // Simply replace the current op with the converted operands.
-    rewriter.replaceOp(op, adaptor.getFlatOperands(),
-                       adaptor.getResultMapping());
-    return success();
-  }
-};
-
-/// Test pattern on for the `get_tuple_element` op from the test dialect that
-/// converts this kind of op into it's "decomposed" form, i.e., instead of
-/// "physically" extracting one element from the tuple, we forward the one
-/// element of the decomposed form that is being extracted (or the several
-/// elements in case that element is a nested tuple).
-class ConvertGetTupleElementOp
-    : public OneToNOpConversionPattern<::test::GetTupleElementOp> {
-public:
-  using OneToNOpConversionPattern<
-      ::test::GetTupleElementOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(::test::GetTupleElementOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    // Construct mapping for tuple element types.
-    auto stateType = cast<TupleType>(op->getOperand(0).getType());
-    TypeRange originalElementTypes = stateType.getTypes();
-    OneToNTypeMapping elementMapping(originalElementTypes);
-    if (failed(typeConverter->convertSignatureArgs(originalElementTypes,
-                                                   elementMapping)))
-      return failure();
-
-    // Compute converted operands corresponding to original input tuple.
-    assert(adaptor.getOperands().size() == 1 &&
-           "expected 'get_tuple_element' to have one operand");
-    ValueRange convertedTuple = adaptor.getOperands()[0];
-
-    // Got those converted operands that correspond to the index-th element ofq
-    // the original input tuple.
-    size_t index = op.getIndex();
-    ValueRange extractedElement =
-        elementMapping.getConvertedValues(convertedTuple, index);
-
-    rewriter.replaceOp(op, extractedElement, adaptor.getResultMapping());
-
-    return success();
-  }
-};
-
-} // namespace
-
-static void
-populateDecomposeTuplesTestPatterns(const TypeConverter &typeConverter,
-                                    RewritePatternSet &patterns) {
-  patterns.add<
-      // clang-format off
-      ConvertMakeTupleOp,
-      ConvertGetTupleElementOp
-      // clang-format on
-      >(typeConverter, patterns.getContext());
-}
-
-/// Creates a sequence of `test.get_tuple_element` ops for all elements of a
-/// given tuple value. If some tuple elements are, in turn, tuples, the elements
-/// of those are extracted recursively such that the returned values have the
-/// same types as `resultTypes.getFlattenedTypes()`.
-///
-/// This function has been copied (with small adaptions) from
-/// TestDecomposeCallGraphTypes.cpp.
-static SmallVector<Value> buildGetTupleElementOps(OpBuilder &builder,
-                                                  TypeRange resultTypes,
-                                                  ValueRange inputs,
-                                                  Location loc) {
-  if (inputs.size() != 1)
-    return {};
-  Value input = inputs.front();
-
-  TupleType inputType = dyn_cast<TupleType>(input.getType());
-  if (!inputType)
-    return {};
-
-  SmallVector<Value> values;
-  for (auto [idx, elementType] : llvm::enumerate(inputType.getTypes())) {
-    Value element = builder.create<::test::GetTupleElementOp>(
-        loc, elementType, input, builder.getI32IntegerAttr(idx));
-    if (auto nestedTupleType = dyn_cast<TupleType>(elementType)) {
-      // Recurse if the current element is also a tuple.
-      SmallVector<Type> flatRecursiveTypes;
-      nestedTupleType.getFlattenedTypes(flatRecursiveTypes);
-      std::optional<SmallVector<Value>> resursiveValues =
-          buildGetTupleElementOps(builder, flatRecursiveTypes, element, loc);
-      if (!resursiveValues.has_value())
-        return {};
-      values.append(resursiveValues.value());
-    } else {
-      values.push_back(element);
-    }
-  }
-  return values;
-}
-
-/// Creates a `test.make_tuple` op out of the given inputs building a tuple of
-/// type `resultType`. If that type is nested, each nested tuple is built
-/// recursively with another `test.make_tuple` op.
-///
-/// This function has been copied (with small adaptions) from
-/// TestDecomposeCallGraphTypes.cpp.
-static Value buildMakeTupleOp(OpBuilder &builder, TupleType resultType,
-                              ValueRange inputs, Location loc) {
-  // Build one value for each element at this nesting level.
-  SmallVector<Value> elements;
-  elements.reserve(resultType.getTypes().size());
-  ValueRange::iterator inputIt = inputs.begin();
-  for (Type elementType : resultType.getTypes()) {
-    if (auto nestedTupleType = dyn_cast<TupleType>(elementType)) {
-      // Determine how many input values are needed for the nested elements of
-      // the nested TupleType and advance inputIt by that number.
-      // TODO: We only need the *number* of nested types, not the types itself.
-      //       Maybe it's worth adding a more efficient overload?
-      SmallVector<Type> nestedFlattenedTypes;
-      nestedTupleType.getFlattenedTypes(nestedFlattenedTypes);
-      size_t numNestedFlattenedTypes = nestedFlattenedTypes.size();
-      ValueRange nestedFlattenedelements(inputIt,
-                                         inputIt + numNestedFlattenedTypes);
-      inputIt += numNestedFlattenedTypes;
-
-      // Recurse on the values for the nested TupleType.
-      Value res = buildMakeTupleOp(builder, nestedTupleType,
-                                   nestedFlattenedelements, loc);
-      if (!res)
-        return Value();
-
-      // The tuple constructed by the conversion is the element value.
-      elements.push_back(res);
-    } else {
-      // Base case: take one input as is.
-      elements.push_back(*inputIt++);
-    }
-  }
-
-  // Assemble the tuple from the elements.
-  return builder.create<::test::MakeTupleOp>(loc, resultType, elements);
-}
-
-void TestOneToNTypeConversionPass::runOnOperation() {
-  ModuleOp module = getOperation();
-  auto *context = &getContext();
-
-  // Assemble type converter.
-  TypeConverter typeConverter;
-
-  typeConverter.addConversion([](Type type) { return type; });
-  typeConverter.addConversion(
-      [](TupleType tupleType, SmallVectorImpl<Type> &types) {
-        tupleType.getFlattenedTypes(types);
-        return success();
-      });
-
-  typeConverter.addArgumentMaterialization(buildMakeTupleOp);
-  typeConverter.addSourceMaterialization(buildMakeTupleOp);
-  typeConverter.addTargetMaterialization(buildGetTupleElementOps);
-  // Test the other target materialization variant that takes the original type
-  // as additional argument. This materialization function always fails.
-  typeConverter.addTargetMaterialization(
-      [](OpBuilder &builder, TypeRange resultTypes, ValueRange inputs,
-         Location loc, Type originalType) -> SmallVector<Value> { return {}; });
-
-  // Assemble patterns.
-  RewritePatternSet patterns(context);
-  if (convertTupleOps)
-    populateDecomposeTuplesTestPatterns(typeConverter, patterns);
-  if (convertFuncOps)
-    populateFuncTypeConversionPatterns(typeConverter, patterns);
-  if (convertSCFOps)
-    scf::populateSCFStructuralOneToNTypeConversions(typeConverter, patterns);
-
-  // Run conversion.
-  if (failed(applyPartialOneToNConversion(module, typeConverter,
-                                          std::move(patterns))))
-    return signalPassFailure();
-}
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index 68b31e563ba35..a5a442909fc6d 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -40,7 +40,6 @@ if(MLIR_INCLUDE_TESTS)
     MLIRTestDialect
     MLIRTestDynDialect
     MLIRTestIR
-    MLIRTestOneToNTypeConversionPass
     MLIRTestPass
     MLIRTestReducer
     MLIRTestTransforms
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 91bbfca88c3c8..344576a44ca41 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -135,7 +135,6 @@ void registerTestMeshSimplificationsPass();
 void registerTestMultiBuffering();
 void registerTestNextAccessPass();
 void registerTestNVGPULowerings();
-void registerTestOneToNTypeConversionPass();
 void registerTestOpaqueLoc();
 void registerTestOpLoweringPasses();
 void registerTestPadFusion();
@@ -279,7 +278,6 @@ void registerTestPasses() {
   mlir::test::registerTestMultiBuffering();
   mlir::test::registerTestNextAccessPass();
   mlir::test::registerTestNVGPULowerings();
-  mlir::test::registerTestOneToNTypeConversionPass();
   mlir::test::registerTestOpaqueLoc();
   mlir::test::registerTestOpLoweringPasses();
   mlir::test::registerTestPadFusion();
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index f7b379c2cd6d8..010605f9a4c6d 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -7206,7 +7206,6 @@ cc_library(
         "include/mlir/Transforms/GreedyPatternRewriteDriver.h",
         "include/mlir/Transforms/Inliner.h",
         "include/mlir/Transforms/LoopInvariantCodeMotionUtils.h",
-        "include/mlir/Transforms/OneToNTypeConversion.h",
         "include/mlir/Transforms/RegionUtils.h",
         "include/mlir/Transforms/WalkPatternRewriteDriver.h",
     ],
@@ -8950,7 +8949,6 @@ cc_binary(
         "//mlir/test:TestMemRef",
         "//mlir/test:TestMesh",
         "//mlir/test:TestNVGPU",
-        "//mlir/test:TestOneToNTypeConversion",
         "//mlir/test:TestPDLL",
         "//mlir/test:TestPass",
         "//mlir/test:TestReducer",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 59410286f22d8..b07c6f98a5ea0 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -615,23 +615,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "TestOneToNTypeConversion",
-    srcs = glob(["lib/Conversion/OneToNTypeConversion/*.cpp"]),
-    includes = ["lib/Dialect/Test"],
-    deps = [
-        ":TestDialect",
-        "//mlir:FuncDialect",
-        "//mlir:FuncTransforms",
-        "//mlir:IR",
-        "//mlir:Pass",
-        "//mlir:SCFDialect",
-        "//mlir:SCFTransforms",
-        "//mlir:TransformUtils",
-        "//mlir:Transforms",
-    ],
-)
-
 cc_library(
     name = "TestVectorToSPIRV",
     srcs = glob(["lib/Conversion/VectorToSPIRV/*.cpp"]),

From 5a993558c5fa52c605d984e0effdc1cd3b452476 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Thu, 17 Apr 2025 20:44:46 +0800
Subject: [PATCH 260/710] [InstSimplify] Fold `getelementptr inbounds null, idx
 -> null` (#130742)

Proof: https://alive2.llvm.org/ce/z/5ZkPx-
See also https://github.com/llvm/llvm-project/pull/130734 for the motivation.
---
 llvm/lib/Analysis/InstructionSimplify.cpp     |   8 +
 .../CodeGen/AMDGPU/memcpy-crash-issue63986.ll | 193 +++++++++---------
 .../Transforms/InstCombine/getelementptr.ll   |  37 ----
 llvm/test/Transforms/InstCombine/store.ll     |   3 +-
 llvm/test/Transforms/InstCombine/sub-gep.ll   |   4 +-
 llvm/test/Transforms/InstSimplify/gep.ll      |  59 ++++++
 6 files changed, 168 insertions(+), 136 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 718d272dd0ac7..8ffb471070d91 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5042,6 +5042,14 @@ static Value *simplifyGEPInst(Type *SrcTy, Value *Ptr,
   if (Q.isUndefValue(Ptr))
     return UndefValue::get(GEPTy);
 
+  // getelementptr inbounds null, idx -> null
+  if (NW.isInBounds() && Q.IIQ.UseInstrInfo && Q.CxtI) {
+    if (auto *BaseC = dyn_cast<Constant>(Ptr))
+      if (BaseC->isNullValue() &&
+          !NullPointerIsDefined(Q.CxtI->getFunction(), AS))
+        return Constant::getNullValue(GEPTy);
+  }
+
   bool IsScalableVec =
       SrcTy->isScalableTy() || any_of(Indices, [](const Value *V) {
         return isa<ScalableVectorType>(V->getType());
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index c92c672dda2ad..94056913f3a33 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -7,138 +7,141 @@ define void @issue63986(i64 %0, i64 %idxprom, ptr inreg %ptr) {
 ; CHECK-LABEL: issue63986:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshlrev_b64 v[4:5], 6, v[2:3]
-; CHECK-NEXT:    v_mov_b32_e32 v6, s17
-; CHECK-NEXT:    v_add_co_u32_e32 v8, vcc, s16, v4
-; CHECK-NEXT:    v_addc_co_u32_e32 v9, vcc, v6, v5, vcc
+; CHECK-NEXT:    v_lshlrev_b64 v[8:9], 6, v[2:3]
+; CHECK-NEXT:    v_mov_b32_e32 v4, s17
+; CHECK-NEXT:    v_add_co_u32_e32 v10, vcc, s16, v8
+; CHECK-NEXT:    v_addc_co_u32_e32 v11, vcc, v4, v9, vcc
+; CHECK-NEXT:  ; %bb.1: ; %entry.loop-memcpy-expansion_crit_edge
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
 ; CHECK-NEXT:    s_mov_b64 s[4:5], 0
-; CHECK-NEXT:  .LBB0_1: ; %loop-memcpy-expansion
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:  .LBB0_2: ; %loop-memcpy-expansion
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    v_mov_b32_e32 v7, s5
-; CHECK-NEXT:    v_mov_b32_e32 v6, s4
-; CHECK-NEXT:    flat_load_dwordx4 v[10:13], v[6:7]
-; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s4, v8
+; CHECK-NEXT:    v_add_co_u32_e32 v12, vcc, s4, v10
 ; CHECK-NEXT:    s_add_u32 s4, s4, 16
+; CHECK-NEXT:    v_mov_b32_e32 v13, s5
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
 ; CHECK-NEXT:    v_cmp_ge_u64_e64 s[6:7], s[4:5], 32
-; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v9, v7, vcc
+; CHECK-NEXT:    v_addc_co_u32_e32 v13, vcc, v11, v13, vcc
 ; CHECK-NEXT:    s_and_b64 vcc, exec, s[6:7]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[6:7], v[10:13]
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_1
-; CHECK-NEXT:  ; %bb.2: ; %loop-memcpy-residual-header
-; CHECK-NEXT:    s_branch .LBB0_4
-; CHECK-NEXT:  ; %bb.3:
-; CHECK-NEXT:    ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[12:13], v[4:7]
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_2
+; CHECK-NEXT:  ; %bb.3: ; %loop-memcpy-residual-header
 ; CHECK-NEXT:    s_branch .LBB0_5
-; CHECK-NEXT:  .LBB0_4: ; %loop-memcpy-residual-header.post-loop-memcpy-expansion_crit_edge
-; CHECK-NEXT:    v_lshlrev_b64 v[6:7], 6, v[2:3]
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_8
-; CHECK-NEXT:  .LBB0_5: ; %loop-memcpy-residual.preheader
-; CHECK-NEXT:    s_add_u32 s4, s16, 32
-; CHECK-NEXT:    s_addc_u32 s5, s17, 0
-; CHECK-NEXT:    v_mov_b32_e32 v3, s5
-; CHECK-NEXT:    v_add_co_u32_e32 v2, vcc, s4, v4
-; CHECK-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v5, vcc
+; CHECK-NEXT:  ; %bb.4:
+; CHECK-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT:    s_branch .LBB0_6
+; CHECK-NEXT:  .LBB0_5: ; %loop-memcpy-residual-header.post-loop-memcpy-expansion_crit_edge
+; CHECK-NEXT:    v_lshlrev_b64 v[2:3], 6, v[2:3]
+; CHECK-NEXT:    s_cbranch_execnz .LBB0_9
+; CHECK-NEXT:  .LBB0_6: ; %loop-memcpy-residual-header.loop-memcpy-residual_crit_edge
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    flat_load_ubyte v2, v[2:3]
+; CHECK-NEXT:    s_add_u32 s6, s16, 32
+; CHECK-NEXT:    s_addc_u32 s4, s17, 0
+; CHECK-NEXT:    v_mov_b32_e32 v4, s4
+; CHECK-NEXT:    v_add_co_u32_e32 v3, vcc, s6, v8
 ; CHECK-NEXT:    s_mov_b64 s[4:5], 0
-; CHECK-NEXT:  ; %bb.6: ; %loop-memcpy-residual
-; CHECK-NEXT:    s_add_u32 s6, 32, s4
-; CHECK-NEXT:    s_addc_u32 s7, 0, s5
-; CHECK-NEXT:    v_mov_b32_e32 v6, s6
-; CHECK-NEXT:    v_mov_b32_e32 v7, s7
-; CHECK-NEXT:    flat_load_ubyte v10, v[6:7]
-; CHECK-NEXT:    v_mov_b32_e32 v7, s5
-; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s4, v2
-; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v3, v7, vcc
+; CHECK-NEXT:    v_addc_co_u32_e32 v4, vcc, v4, v9, vcc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:  ; %bb.7: ; %loop-memcpy-residual
+; CHECK-NEXT:    v_mov_b32_e32 v6, s5
+; CHECK-NEXT:    v_add_co_u32_e32 v5, vcc, s4, v3
 ; CHECK-NEXT:    s_add_u32 s4, s4, 1
+; CHECK-NEXT:    v_addc_co_u32_e32 v6, vcc, v4, v6, vcc
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[6:7], v10
-; CHECK-NEXT:  ; %bb.7:
-; CHECK-NEXT:    v_mov_b32_e32 v7, v5
-; CHECK-NEXT:    v_mov_b32_e32 v6, v4
-; CHECK-NEXT:  .LBB0_8: ; %post-loop-memcpy-expansion
-; CHECK-NEXT:    v_and_b32_e32 v2, 15, v0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[5:6], v2
+; CHECK-NEXT:  ; %bb.8:
+; CHECK-NEXT:    v_mov_b32_e32 v2, v8
+; CHECK-NEXT:    v_mov_b32_e32 v3, v9
+; CHECK-NEXT:  .LBB0_9: ; %post-loop-memcpy-expansion
+; CHECK-NEXT:    v_and_b32_e32 v6, 15, v0
 ; CHECK-NEXT:    v_and_b32_e32 v0, -16, v0
-; CHECK-NEXT:    v_add_co_u32_e32 v4, vcc, v6, v0
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0
-; CHECK-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v1, vcc
+; CHECK-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v0
+; CHECK-NEXT:    v_mov_b32_e32 v7, 0
+; CHECK-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
 ; CHECK-NEXT:    v_cmp_ne_u64_e64 s[4:5], 0, v[0:1]
-; CHECK-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[2:3]
-; CHECK-NEXT:    v_mov_b32_e32 v6, s17
-; CHECK-NEXT:    v_add_co_u32_e32 v4, vcc, s16, v4
-; CHECK-NEXT:    v_addc_co_u32_e32 v5, vcc, v6, v5, vcc
-; CHECK-NEXT:    s_branch .LBB0_11
-; CHECK-NEXT:  .LBB0_9: ; %Flow14
-; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT:    v_cmp_ne_u64_e64 s[6:7], 0, v[6:7]
+; CHECK-NEXT:    v_mov_b32_e32 v4, s17
+; CHECK-NEXT:    v_mov_b32_e32 v8, 0
+; CHECK-NEXT:    v_add_co_u32_e32 v12, vcc, s16, v2
+; CHECK-NEXT:    v_mov_b32_e32 v9, 0
+; CHECK-NEXT:    v_addc_co_u32_e32 v13, vcc, v4, v3, vcc
+; CHECK-NEXT:    s_branch .LBB0_12
+; CHECK-NEXT:  .LBB0_10: ; %Flow14
+; CHECK-NEXT:    ; in Loop: Header=BB0_12 Depth=1
 ; CHECK-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], 0
-; CHECK-NEXT:  .LBB0_10: ; %Flow16
-; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT:  .LBB0_11: ; %Flow16
+; CHECK-NEXT:    ; in Loop: Header=BB0_12 Depth=1
 ; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_19
-; CHECK-NEXT:  .LBB0_11: ; %while.cond
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_20
+; CHECK-NEXT:  .LBB0_12: ; %while.cond
 ; CHECK-NEXT:    ; =>This Loop Header: Depth=1
-; CHECK-NEXT:    ; Child Loop BB0_13 Depth 2
-; CHECK-NEXT:    ; Child Loop BB0_17 Depth 2
+; CHECK-NEXT:    ; Child Loop BB0_14 Depth 2
+; CHECK-NEXT:    ; Child Loop BB0_18 Depth 2
 ; CHECK-NEXT:    s_and_saveexec_b64 s[8:9], s[4:5]
-; CHECK-NEXT:    s_cbranch_execz .LBB0_14
-; CHECK-NEXT:  ; %bb.12: ; %loop-memcpy-expansion2.preheader
-; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT:    s_cbranch_execz .LBB0_15
+; CHECK-NEXT:  ; %bb.13: ; %while.cond.loop-memcpy-expansion2_crit_edge
+; CHECK-NEXT:    ; in Loop: Header=BB0_12 Depth=1
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], 0
 ; CHECK-NEXT:    s_mov_b64 s[12:13], 0
-; CHECK-NEXT:  .LBB0_13: ; %loop-memcpy-expansion2
-; CHECK-NEXT:    ; Parent Loop BB0_11 Depth=1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:  .LBB0_14: ; %loop-memcpy-expansion2
+; CHECK-NEXT:    ; Parent Loop BB0_12 Depth=1
 ; CHECK-NEXT:    ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    v_mov_b32_e32 v6, s12
-; CHECK-NEXT:    v_mov_b32_e32 v7, s13
-; CHECK-NEXT:    flat_load_dwordx4 v[10:13], v[6:7]
-; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s12, v8
+; CHECK-NEXT:    v_mov_b32_e32 v15, s13
+; CHECK-NEXT:    v_add_co_u32_e32 v14, vcc, s12, v10
 ; CHECK-NEXT:    s_add_u32 s12, s12, 16
-; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v9, v7, vcc
+; CHECK-NEXT:    v_addc_co_u32_e32 v15, vcc, v11, v15, vcc
 ; CHECK-NEXT:    s_addc_u32 s13, s13, 0
 ; CHECK-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[14:15], v[2:5]
 ; CHECK-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[6:7], v[10:13]
 ; CHECK-NEXT:    s_andn2_b64 exec, exec, s[10:11]
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_13
-; CHECK-NEXT:  .LBB0_14: ; %Flow15
-; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT:    s_cbranch_execnz .LBB0_14
+; CHECK-NEXT:  .LBB0_15: ; %Flow15
+; CHECK-NEXT:    ; in Loop: Header=BB0_12 Depth=1
 ; CHECK-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], -1
-; CHECK-NEXT:    s_cbranch_execz .LBB0_10
-; CHECK-NEXT:  ; %bb.15: ; %loop-memcpy-residual-header5
-; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT:    s_cbranch_execz .LBB0_11
+; CHECK-NEXT:  ; %bb.16: ; %loop-memcpy-residual-header5
+; CHECK-NEXT:    ; in Loop: Header=BB0_12 Depth=1
 ; CHECK-NEXT:    s_and_saveexec_b64 s[8:9], s[6:7]
 ; CHECK-NEXT:    s_xor_b64 s[10:11], exec, s[8:9]
-; CHECK-NEXT:    s_cbranch_execz .LBB0_9
-; CHECK-NEXT:  ; %bb.16: ; %loop-memcpy-residual4.preheader
-; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT:    s_cbranch_execz .LBB0_10
+; CHECK-NEXT:  ; %bb.17: ; %loop-memcpy-residual-header5.loop-memcpy-residual4_crit_edge
+; CHECK-NEXT:    ; in Loop: Header=BB0_12 Depth=1
+; CHECK-NEXT:    flat_load_ubyte v2, v[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[12:13], 0
 ; CHECK-NEXT:    s_mov_b64 s[14:15], 0
-; CHECK-NEXT:  .LBB0_17: ; %loop-memcpy-residual4
-; CHECK-NEXT:    ; Parent Loop BB0_11 Depth=1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:  .LBB0_18: ; %loop-memcpy-residual4
+; CHECK-NEXT:    ; Parent Loop BB0_12 Depth=1
 ; CHECK-NEXT:    ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    v_mov_b32_e32 v10, s15
-; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s14, v0
-; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v1, v10, vcc
-; CHECK-NEXT:    flat_load_ubyte v11, v[6:7]
-; CHECK-NEXT:    v_add_co_u32_e32 v6, vcc, s14, v4
+; CHECK-NEXT:    v_add_co_u32_e32 v3, vcc, s14, v12
 ; CHECK-NEXT:    s_add_u32 s14, s14, 1
+; CHECK-NEXT:    v_mov_b32_e32 v4, s15
 ; CHECK-NEXT:    s_addc_u32 s15, s15, 0
-; CHECK-NEXT:    v_cmp_ge_u64_e64 s[8:9], s[14:15], v[2:3]
-; CHECK-NEXT:    v_addc_co_u32_e32 v7, vcc, v5, v10, vcc
+; CHECK-NEXT:    v_cmp_ge_u64_e64 s[8:9], s[14:15], v[6:7]
+; CHECK-NEXT:    v_addc_co_u32_e32 v4, vcc, v13, v4, vcc
 ; CHECK-NEXT:    s_or_b64 s[12:13], s[8:9], s[12:13]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[6:7], v11
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[3:4], v2
 ; CHECK-NEXT:    s_andn2_b64 exec, exec, s[12:13]
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_17
-; CHECK-NEXT:  ; %bb.18: ; %Flow
-; CHECK-NEXT:    ; in Loop: Header=BB0_11 Depth=1
+; CHECK-NEXT:    s_cbranch_execnz .LBB0_18
+; CHECK-NEXT:  ; %bb.19: ; %Flow
+; CHECK-NEXT:    ; in Loop: Header=BB0_12 Depth=1
 ; CHECK-NEXT:    s_or_b64 exec, exec, s[12:13]
-; CHECK-NEXT:    s_branch .LBB0_9
-; CHECK-NEXT:  .LBB0_19: ; %DummyReturnBlock
+; CHECK-NEXT:    s_branch .LBB0_10
+; CHECK-NEXT:  .LBB0_20: ; %DummyReturnBlock
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index ec03d9a2dae2b..feba952919b9a 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -1326,42 +1326,6 @@ define ptr @PR45084_extra_use(i1 %cond, ptr %p) {
   ret ptr %sel
 }
 
-define ptr @gep_null_inbounds(i64 %idx) {
-; CHECK-LABEL: @gep_null_inbounds(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr null, i64 [[IDX:%.*]]
-; CHECK-NEXT:    ret ptr [[GEP]]
-;
-  %gep = getelementptr inbounds i8, ptr null, i64 %idx
-  ret ptr %gep
-}
-
-define ptr @gep_null_not_inbounds(i64 %idx) {
-; CHECK-LABEL: @gep_null_not_inbounds(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr null, i64 [[IDX:%.*]]
-; CHECK-NEXT:    ret ptr [[GEP]]
-;
-  %gep = getelementptr i8, ptr null, i64 %idx
-  ret ptr %gep
-}
-
-define ptr @gep_null_defined(i64 %idx) null_pointer_is_valid {
-; CHECK-LABEL: @gep_null_defined(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr null, i64 [[IDX:%.*]]
-; CHECK-NEXT:    ret ptr [[GEP]]
-;
-  %gep = getelementptr inbounds i8, ptr null, i64 %idx
-  ret ptr %gep
-}
-
-define ptr @gep_null_inbounds_different_type(i64 %idx1, i64 %idx2) {
-; CHECK-LABEL: @gep_null_inbounds_different_type(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [0 x i8], ptr null, i64 0, i64 [[IDX2:%.*]]
-; CHECK-NEXT:    ret ptr [[GEP]]
-;
-  %gep = getelementptr inbounds [0 x i8], ptr null, i64 %idx1, i64 %idx2
-  ret ptr %gep
-}
-
 define ptr @D98588(ptr %c1, i64 %offset) {
 ; CHECK-LABEL: @D98588(
 ; CHECK-NEXT:    [[C2_NEXT_IDX:%.*]] = shl nsw i64 [[OFFSET:%.*]], 3
@@ -2019,5 +1983,4 @@ define ptr @gep_merge_nusw_const(ptr %p, i64 %idx, i64 %idx2) {
   ret ptr %gep
 }
 
-
 !0 = !{!"branch_weights", i32 2, i32 10}
diff --git a/llvm/test/Transforms/InstCombine/store.ll b/llvm/test/Transforms/InstCombine/store.ll
index daa40da1828b5..48c63c6f24c72 100644
--- a/llvm/test/Transforms/InstCombine/store.ll
+++ b/llvm/test/Transforms/InstCombine/store.ll
@@ -49,8 +49,7 @@ define void @test2(ptr %P) {
 
 define void @store_at_gep_off_null_inbounds(i64 %offset) {
 ; CHECK-LABEL: @store_at_gep_off_null_inbounds(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds i32, ptr null, i64 [[OFFSET:%.*]]
-; CHECK-NEXT:    store i32 poison, ptr [[PTR]], align 4
+; CHECK-NEXT:    store i32 poison, ptr null, align 4
 ; CHECK-NEXT:    ret void
 ;
   %ptr = getelementptr inbounds i32, ptr null, i64 %offset
diff --git a/llvm/test/Transforms/InstCombine/sub-gep.ll b/llvm/test/Transforms/InstCombine/sub-gep.ll
index 3f8728d3a4381..c86a1a37bd7ad 100644
--- a/llvm/test/Transforms/InstCombine/sub-gep.ll
+++ b/llvm/test/Transforms/InstCombine/sub-gep.ll
@@ -741,7 +741,7 @@ define i64 @nullptrtoint_scalable_c() {
 ; CHECK-NEXT:    ret i64 [[PTR_IDX]]
 ;
 entry:
-  %ptr = getelementptr inbounds <vscale x 4 x i32>, ptr null, i64 8
+  %ptr = getelementptr nusw <vscale x 4 x i32>, ptr null, i64 8
   %ret = ptrtoint ptr %ptr to i64
   ret i64 %ret
 }
@@ -755,7 +755,7 @@ define i64 @nullptrtoint_scalable_x(i64 %x) {
 ; CHECK-NEXT:    ret i64 [[PTR_IDX]]
 ;
 entry:
-  %ptr = getelementptr inbounds <vscale x 4 x i32>, ptr null, i64 %x
+  %ptr = getelementptr nusw <vscale x 4 x i32>, ptr null, i64 %x
   %ret = ptrtoint ptr %ptr to i64
   ret i64 %ret
 }
diff --git a/llvm/test/Transforms/InstSimplify/gep.ll b/llvm/test/Transforms/InstSimplify/gep.ll
index a330f5cbc9268..a73c902fac647 100644
--- a/llvm/test/Transforms/InstSimplify/gep.ll
+++ b/llvm/test/Transforms/InstSimplify/gep.ll
@@ -386,3 +386,62 @@ define i64 @gep_array_of_scalable_vectors_ptrdiff(ptr %ptr) {
   %diff = sub i64 %c2.int, %c1.int
   ret i64 %diff
 }
+
+define ptr @gep_null_inbounds(i64 %idx) {
+; CHECK-LABEL: @gep_null_inbounds(
+; CHECK-NEXT:    ret ptr null
+;
+  %gep = getelementptr inbounds i8, ptr null, i64 %idx
+  ret ptr %gep
+}
+
+define ptr @gep_null_not_inbounds(i64 %idx) {
+; CHECK-LABEL: @gep_null_not_inbounds(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr null, i64 [[IDX:%.*]]
+; CHECK-NEXT:    ret ptr [[GEP]]
+;
+  %gep = getelementptr i8, ptr null, i64 %idx
+  ret ptr %gep
+}
+
+define ptr @gep_null_defined(i64 %idx) null_pointer_is_valid {
+; CHECK-LABEL: @gep_null_defined(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr null, i64 [[IDX:%.*]]
+; CHECK-NEXT:    ret ptr [[GEP]]
+;
+  %gep = getelementptr inbounds i8, ptr null, i64 %idx
+  ret ptr %gep
+}
+
+define ptr @gep_null_inbounds_different_type(i64 %idx1, i64 %idx2) {
+; CHECK-LABEL: @gep_null_inbounds_different_type(
+; CHECK-NEXT:    ret ptr null
+;
+  %gep = getelementptr inbounds [0 x i8], ptr null, i64 %idx1, i64 %idx2
+  ret ptr %gep
+}
+
+define <2 x ptr> @gep_inbounds_null_vec(i64 %idx) {
+; CHECK-LABEL: @gep_inbounds_null_vec(
+; CHECK-NEXT:    ret <2 x ptr> zeroinitializer
+;
+  %p = getelementptr inbounds i8, <2 x ptr> zeroinitializer, i64 %idx
+  ret <2 x ptr> %p
+}
+
+define <2 x ptr> @gep_inbounds_null_vec_broadcast(<2 x i64> %idx) {
+; CHECK-LABEL: @gep_inbounds_null_vec_broadcast(
+; CHECK-NEXT:    ret <2 x ptr> zeroinitializer
+;
+  %p = getelementptr inbounds i8, ptr null, <2 x i64> %idx
+  ret <2 x ptr> %p
+}
+
+define ptr @gep_noinbounds_null(i64 %idx) {
+; CHECK-LABEL: @gep_noinbounds_null(
+; CHECK-NEXT:    [[P:%.*]] = getelementptr i8, ptr null, i64 [[IDX:%.*]]
+; CHECK-NEXT:    ret ptr [[P]]
+;
+  %p = getelementptr i8, ptr null, i64 %idx
+  ret ptr %p
+}

From 47f4f39265b31e2249536b74d33d63508cdfb457 Mon Sep 17 00:00:00 2001
From: Zhuoran Yin <zhuoryin@amd.com>
Date: Thu, 17 Apr 2025 08:50:31 -0400
Subject: [PATCH 261/710] [MLIR][AMDGPU] Fixing word alignment check for
 bufferload fastpath (#135982)

`delta_bytes % (32 ceilDiv elementBitwidth) != 0` condition is incorrect
in https://github.com/llvm/llvm-project/pull/135014

For example, last load is issued to load only one last element of fp16.
Then `delta bytes = 2`, `(32 ceildiv 16) = 2`. In this case it will be
judged as word aligned. It will send to fast path but get all zeros for
the fp16 because it cross the word boundary.

In reality the equation should be just `delta_bytes % 4` , since a word
is 4 bytes. This PR fix the bug by amending the mod target to 4.
---
 .../AMDGPU/Transforms/TransferReadToLoad.cpp     |  9 ++-------
 .../Dialect/AMDGPU/transfer-read-to-load.mlir    | 16 ++++++----------
 .../bazel/llvm-project-overlay/mlir/BUILD.bazel  |  2 ++
 3 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
index f665c1794cdd4..9f64abb5a8860 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
@@ -11,12 +11,10 @@
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -225,15 +223,12 @@ struct TransferReadLowering final : OpRewritePattern<vector::TransferReadOp> {
     Value isOutofBounds = rewriter.create<arith::CmpIOp>(
         loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
 
-    // 2) check if (detla_bytes % (32 / elementBitwidth) != 0)
-    Value deltaBytes = rewriter.create<arith::MulIOp>(
-        loc, delta,
-        rewriter.create<arith::ConstantIndexOp>(loc, elementBitWidth / 8));
+    // 2) check if (detla % elements_per_word != 0)
     Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
         loc, llvm::divideCeil(32, elementBitWidth));
     Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
         loc, arith::CmpIPredicate::ne,
-        rewriter.create<arith::RemUIOp>(loc, deltaBytes, elementsPerWord),
+        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
         rewriter.create<arith::ConstantIndexOp>(loc, 0));
 
     // We take the fallback of transfer_read default lowering only it is both
diff --git a/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir b/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
index d0805b6b8a973..22b425680fb05 100644
--- a/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
+++ b/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
@@ -10,8 +10,7 @@ func.func @transfer_to_maskedload_fatrawbuffer(%mem : memref<8x8xf32, #amdgpu.ad
   return %res : vector<4xf32>
 }
 
-// CHECK: %[[FALSE:.*]] = arith.constant false
-// CHECK: %[[IF:.*]] = scf.if %[[FALSE]] -> (vector<4xf32>) {
+// CHECK: %[[IF:.*]] = scf.if
 // CHECK: vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]]
 
 // CHECK: } else {
@@ -35,14 +34,13 @@ func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgp
 // CHECK-DAG: %[[C0:.*]] = arith.constant 0
 // CHECK-DAG: %[[SIZE:.*]] = arith.constant 64
 // CHECK-DAG: %[[BYTES:.*]] = arith.constant 2
-// CHECK-DAG: %[[VECTORSIZE:.*]] = arith.constant 4
+// CHECK-DAG: %[[C4:.*]] = arith.constant 4
 
 // CHECK: %[[LINEAR:.*]] = affine.apply #map()[%[[ARG1]], %[[ARG2]]]
 // CHECK: %[[DELTA:.*]] = arith.subi %[[SIZE]], %[[LINEAR]]
-// CHECK: %[[COND1:.*]] = arith.cmpi ult, %[[DELTA]], %[[VECTORSIZE]]
+// CHECK: %[[COND1:.*]] = arith.cmpi ult, %[[DELTA]], %[[C4]]
 
-// CHECK: %[[DELTABYTES:.*]] = arith.muli %[[DELTA]], %[[BYTES]]
-// CHECK: %[[REM:.*]] = arith.remui %[[DELTABYTES]], %[[BYTES]]
+// CHECK: %[[REM:.*]] = arith.remui %[[DELTA]], %[[BYTES]]
 // CHECK: %[[COND2:.*]] = arith.cmpi ne, %[[REM]], %[[C0]]
 
 // CHECK: %[[COND:.*]] = arith.andi %[[COND1]], %[[COND2]]
@@ -120,8 +118,7 @@ func.func @transfer_broadcasting(%mem : memref<8x8xf32, #amdgpu.address_space<fa
   return %res : vector<4xf32>
 }
 // CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
-// CHECK: %[[FALSE:.*]] = arith.constant false
-// CHECK: %[[IF:.*]] = scf.if %[[FALSE]] -> (vector<4xf32>) {
+// CHECK: %[[IF:.*]] = scf.if
 // CHECK: %[[LOAD:.*]] = vector.load %arg0[%arg1, %arg1]
 // CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[CST]]
 // CHECK: %[[BROADCAST:.*]] = vector.broadcast %[[SELECT]] : vector<1xf32> to vector<4xf32>
@@ -140,7 +137,6 @@ func.func @transfer_scalar(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_
   return %res : vector<1xf32>
 }
 // CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
-// CHECK: %[[FALSE:.*]] = arith.constant false
-// CHECK: %[[IF:.*]] = scf.if %[[FALSE]] -> (vector<1xf32>) {
+// CHECK: %[[IF:.*]] = scf.if
 // CHECK: %[[LOAD:.*]] = vector.load %[[ARG0]][%[[ARG1]], %[[ARG1]]]
 // CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[CST]]
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 010605f9a4c6d..b24160dc20e44 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -1559,6 +1559,7 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/AMDGPU/Transforms/*.h"]),
     includes = ["include"],
     deps = [
+        ":AffineDialect",
         ":AMDGPUDialect",
         ":AMDGPUPassIncGen",
         ":AMDGPUUtils",
@@ -1569,6 +1570,7 @@ cc_library(
         ":FuncDialect",
         ":GPUDialect",
         ":IR",
+        ":LLVMSupportHeaders",
         ":MemRefDialect",
         ":MemRefUtils",
         ":Pass",

From fb00fa56b51b191d026eec104905e416bf34bbda Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 17 Apr 2025 14:52:23 +0200
Subject: [PATCH 262/710] [clang] Implement dump() for MemberPointer APValues
 (#136130)

Print the member pointer decl and the path.
---
 clang/lib/AST/TextNodeDumper.cpp           | 16 ++++++++++++++--
 clang/test/AST/ast-dump-APValue-lvalue.cpp | 18 ++++++++++++++++--
 clang/test/AST/ast-dump-APValue-todo.cpp   | 22 ----------------------
 3 files changed, 30 insertions(+), 26 deletions(-)
 delete mode 100644 clang/test/AST/ast-dump-APValue-todo.cpp

diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index c8b459ee78e6b..89567425f0d5f 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -829,9 +829,21 @@ void TextNodeDumper::Visit(const APValue &Value, QualType Ty) {
 
     return;
   }
-  case APValue::MemberPointer:
-    OS << "MemberPointer <todo>";
+  case APValue::MemberPointer: {
+    OS << "MemberPointer ";
+    auto Path = Value.getMemberPointerPath();
+    for (const CXXRecordDecl *D : Path) {
+      {
+        ColorScope Color(OS, ShowColors, DeclNameColor);
+        OS << D->getDeclName();
+      }
+      OS << "::";
+    }
+
+    ColorScope Color(OS, ShowColors, DeclNameColor);
+    OS << Value.getMemberPointerDecl()->getDeclName();
     return;
+  }
   case APValue::AddrLabelDiff:
     OS << "AddrLabelDiff <todo>";
     return;
diff --git a/clang/test/AST/ast-dump-APValue-lvalue.cpp b/clang/test/AST/ast-dump-APValue-lvalue.cpp
index 333f7aa419377..51d22a5ba8b6d 100644
--- a/clang/test/AST/ast-dump-APValue-lvalue.cpp
+++ b/clang/test/AST/ast-dump-APValue-lvalue.cpp
@@ -27,6 +27,16 @@ namespace std {
   class type_info;
 }
 
+struct P {
+  int x;
+};
+struct Q {
+  float m;
+};
+struct MP : P, Q {
+  int i;
+};
+
 void Test(int (&arr)[10]) {
   constexpr int *pi = &i;
   // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pi 'int *const' constexpr cinit
@@ -53,6 +63,10 @@ void Test(int (&arr)[10]) {
   // CHECK-NEXT:  |   |-value: LValue Base=null, Null=1, Offset=0, HasPath=1, PathLength=0, Path=()
 
   constexpr const std::type_info* pti = &typeid(int);
-  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pti 'const std::type_info *const' constexpr cinit
-  // CHECK-NEXT:      |-value: LValue Base=TypeInfoLValue typeid(int), Null=0, Offset=0, HasPath=1, PathLength=0, Path=()
+  // CHECK:  | `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pti 'const std::type_info *const' constexpr cinit
+  // CHECK-NEXT:  |   |-value: LValue Base=TypeInfoLValue typeid(int), Null=0, Offset=0, HasPath=1, PathLength=0, Path=()
+
+  constexpr int(MP::*pmi) = (int MP::*)&P::x;
+  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pmi 'int (MP::*const)' constexpr cinit
+  // CHECK-NEXT:      |-value: MemberPointer MP::x
 }
diff --git a/clang/test/AST/ast-dump-APValue-todo.cpp b/clang/test/AST/ast-dump-APValue-todo.cpp
deleted file mode 100644
index acaa82ba53b6f..0000000000000
--- a/clang/test/AST/ast-dump-APValue-todo.cpp
+++ /dev/null
@@ -1,22 +0,0 @@
-// Test without serialization:
-// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
-// RUN:            -ast-dump %s -ast-dump-filter Test \
-// RUN: | FileCheck --strict-whitespace --match-full-lines %s
-//
-// Test with serialization:
-// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 -emit-pch -o %t %s
-// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -Wno-unused-value -std=gnu++17 \
-// RUN:           -include-pch %t -ast-dump-all -ast-dump-filter Test /dev/null \
-// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
-// RUN: | FileCheck --strict-whitespace --match-full-lines %s
-
-int i;
-struct S {
-  int i;
-};
-
-void Test() {
-  constexpr int(S::*pmi) = &S::i;
-  // CHECK:    `-VarDecl {{.*}} <col:{{.*}}, col:{{.*}}> col:{{.*}} pmi 'int (S::*const)' constexpr cinit
-  // CHECK-NEXT:      |-value: MemberPointer <todo>
-}

From 0348ff515854438cab8a48b79e8839cb99d48701 Mon Sep 17 00:00:00 2001
From: Tom Honermann <tom.honermann@intel.com>
Date: Thu, 17 Apr 2025 06:14:45 -0700
Subject: [PATCH 263/710] [SYCL] Basic code generation for SYCL kernel caller
 offload entry point functions. (#133030)

A function declared with the `sycl_kernel_entry_point` attribute,
sometimes called a SYCL kernel entry point function, specifies a pattern
from which the parameters and body of an offload entry point function,
sometimes called a SYCL kernel caller function, are derived.

SYCL kernel caller functions are emitted during SYCL device compilation.
Their parameters and body are derived from the `SYCLKernelCallStmt`
statement and `OutlinedFunctionDecl` declaration associated with their
corresponding SYCL kernel entry point function. A distinct SYCL kernel
caller function is generated for each SYCL kernel entry point function
defined as a non-inline function or ODR-used in the translation unit.

The name of each SYCL kernel caller function is parameterized by the
SYCL kernel name type specified by the `sycl_kernel_entry_point`
attribute attached to the corresponding SYCL kernel entry point
function. For the moment, the Itanium ABI mangled name for typeinfo data
(`_ZTS<type>`) is used to name these functions; a future change will
switch to a more appropriate naming scheme.

The calling convention used for a SYCL kernel caller function is target
dependent. Support for AMDGCN, NVPTX, and SPIR targets is currently
provided. These functions are required to observe the language
restrictions for SYCL devices as specified by the SYCL 2020
specification; this includes a forward progress guarantee and prohibits
recursion.

Only SYCL kernel caller functions, functions declared as
`SYCL_EXTERNAL`, and functions directly or indirectly referenced from
those functions should be emitted during device compilation. Pruning of
other declarations has not yet been implemented.

---------

Co-authored-by: Elizabeth Andrews <elizabeth.andrews@intel.com>
---
 clang/include/clang/AST/SYCLKernelInfo.h      |   8 +-
 clang/lib/AST/ASTContext.cpp                  |  44 ++++-
 clang/lib/CodeGen/CGCall.cpp                  |  11 ++
 clang/lib/CodeGen/CMakeLists.txt              |   1 +
 clang/lib/CodeGen/CodeGenModule.cpp           |  25 +++
 clang/lib/CodeGen/CodeGenModule.h             |   5 +
 clang/lib/CodeGen/CodeGenSYCL.cpp             |  72 +++++++
 clang/lib/CodeGen/CodeGenTypes.h              |   7 +
 clang/lib/CodeGen/Targets/NVPTX.cpp           |   4 +
 .../CodeGenSYCL/kernel-caller-entry-point.cpp | 184 ++++++++++++++++++
 10 files changed, 355 insertions(+), 6 deletions(-)
 create mode 100644 clang/lib/CodeGen/CodeGenSYCL.cpp
 create mode 100644 clang/test/CodeGenSYCL/kernel-caller-entry-point.cpp

diff --git a/clang/include/clang/AST/SYCLKernelInfo.h b/clang/include/clang/AST/SYCLKernelInfo.h
index 4a4827e601053..3825af86c14e3 100644
--- a/clang/include/clang/AST/SYCLKernelInfo.h
+++ b/clang/include/clang/AST/SYCLKernelInfo.h
@@ -22,9 +22,10 @@ namespace clang {
 class SYCLKernelInfo {
 public:
   SYCLKernelInfo(CanQualType KernelNameType,
-                 const FunctionDecl *KernelEntryPointDecl)
+                 const FunctionDecl *KernelEntryPointDecl,
+                 const std::string &KernelName)
       : KernelNameType(KernelNameType),
-        KernelEntryPointDecl(KernelEntryPointDecl) {}
+        KernelEntryPointDecl(KernelEntryPointDecl), KernelName(KernelName) {}
 
   CanQualType getKernelNameType() const { return KernelNameType; }
 
@@ -32,9 +33,12 @@ class SYCLKernelInfo {
     return KernelEntryPointDecl;
   }
 
+  const std::string &GetKernelName() const { return KernelName; }
+
 private:
   CanQualType KernelNameType;
   const FunctionDecl *KernelEntryPointDecl;
+  std::string KernelName;
 };
 
 } // namespace clang
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index bf24704e48eaa..860e6ec0fb47e 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -12825,6 +12825,15 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
     if (!FD->doesThisDeclarationHaveABody())
       return FD->doesDeclarationForceExternallyVisibleDefinition();
 
+    // Function definitions with the sycl_kernel_entry_point attribute are
+    // required during device compilation so that SYCL kernel caller offload
+    // entry points are emitted.
+    if (LangOpts.SYCLIsDevice && FD->hasAttr<SYCLKernelEntryPointAttr>())
+      return true;
+
+    // FIXME: Functions declared with SYCL_EXTERNAL are required during
+    // device compilation.
+
     // Constructors and destructors are required.
     if (FD->hasAttr<ConstructorAttr>() || FD->hasAttr<DestructorAttr>())
       return true;
@@ -14832,9 +14841,36 @@ void ASTContext::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
   }
 }
 
-static SYCLKernelInfo BuildSYCLKernelInfo(CanQualType KernelNameType,
+static SYCLKernelInfo BuildSYCLKernelInfo(ASTContext &Context,
+                                          CanQualType KernelNameType,
                                           const FunctionDecl *FD) {
-  return {KernelNameType, FD};
+  // Host and device compilation may use different ABIs and different ABIs
+  // may allocate name mangling discriminators differently. A discriminator
+  // override is used to ensure consistent discriminator allocation across
+  // host and device compilation.
+  auto DeviceDiscriminatorOverrider =
+      [](ASTContext &Ctx, const NamedDecl *ND) -> UnsignedOrNone {
+    if (const auto *RD = dyn_cast<CXXRecordDecl>(ND))
+      if (RD->isLambda())
+        return RD->getDeviceLambdaManglingNumber();
+    return std::nullopt;
+  };
+  std::unique_ptr<MangleContext> MC{ItaniumMangleContext::create(
+      Context, Context.getDiagnostics(), DeviceDiscriminatorOverrider)};
+
+  // Construct a mangled name for the SYCL kernel caller offload entry point.
+  // FIXME: The Itanium typeinfo mangling (_ZTS<type>) is currently used to
+  // name the SYCL kernel caller offload entry point function. This mangling
+  // does not suffice to clearly identify symbols that correspond to SYCL
+  // kernel caller functions, nor is this mangling natural for targets that
+  // use a non-Itanium ABI.
+  std::string Buffer;
+  Buffer.reserve(128);
+  llvm::raw_string_ostream Out(Buffer);
+  MC->mangleCanonicalTypeName(KernelNameType, Out);
+  std::string KernelName = Out.str();
+
+  return {KernelNameType, FD, KernelName};
 }
 
 void ASTContext::registerSYCLEntryPointFunction(FunctionDecl *FD) {
@@ -14855,8 +14891,8 @@ void ASTContext::registerSYCLEntryPointFunction(FunctionDecl *FD) {
           declaresSameEntity(FD, IT->second.getKernelEntryPointDecl())) &&
          "SYCL kernel name conflict");
   (void)IT;
-  SYCLKernels.insert(
-      std::make_pair(KernelNameType, BuildSYCLKernelInfo(KernelNameType, FD)));
+  SYCLKernels.insert(std::make_pair(
+      KernelNameType, BuildSYCLKernelInfo(*this, KernelNameType, FD)));
 }
 
 const SYCLKernelInfo &ASTContext::getSYCLKernelInfo(QualType T) const {
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index bc1035163a8eb..8cb27420dd911 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -739,6 +739,17 @@ CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
                                  RequiredArgs::All);
 }
 
+const CGFunctionInfo &
+CodeGenTypes::arrangeSYCLKernelCallerDeclaration(QualType resultType,
+                                                 const FunctionArgList &args) {
+  CanQualTypeList argTypes = getArgTypesForDeclaration(Context, args);
+
+  return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,
+                                 argTypes,
+                                 FunctionType::ExtInfo(CC_OpenCLKernel),
+                                 /*paramInfos=*/{}, RequiredArgs::All);
+}
+
 /// Arrange a call to a C++ method, passing the given arguments.
 ///
 /// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
diff --git a/clang/lib/CodeGen/CMakeLists.txt b/clang/lib/CodeGen/CMakeLists.txt
index dc5b2a35583b4..c377ac0786747 100644
--- a/clang/lib/CodeGen/CMakeLists.txt
+++ b/clang/lib/CodeGen/CMakeLists.txt
@@ -102,6 +102,7 @@ add_clang_library(clangCodeGen
   CodeGenFunction.cpp
   CodeGenModule.cpp
   CodeGenPGO.cpp
+  CodeGenSYCL.cpp
   CodeGenTBAA.cpp
   CodeGenTypes.cpp
   ConstantInitBuilder.cpp
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 26e09fe239242..83d8d4f758195 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -3309,6 +3309,27 @@ void CodeGenModule::EmitDeferred() {
   CurDeclsToEmit.swap(DeferredDeclsToEmit);
 
   for (GlobalDecl &D : CurDeclsToEmit) {
+    // Functions declared with the sycl_kernel_entry_point attribute are
+    // emitted normally during host compilation. During device compilation,
+    // a SYCL kernel caller offload entry point function is generated and
+    // emitted in place of each of these functions.
+    if (const auto *FD = D.getDecl()->getAsFunction()) {
+      if (LangOpts.SYCLIsDevice && FD->hasAttr<SYCLKernelEntryPointAttr>() &&
+          FD->isDefined()) {
+        // Functions with an invalid sycl_kernel_entry_point attribute are
+        // ignored during device compilation.
+        if (!FD->getAttr<SYCLKernelEntryPointAttr>()->isInvalidAttr()) {
+          // Generate and emit the SYCL kernel caller function.
+          EmitSYCLKernelCaller(FD, getContext());
+          // Recurse to emit any symbols directly or indirectly referenced
+          // by the SYCL kernel caller function.
+          EmitDeferred();
+        }
+        // Do not emit the sycl_kernel_entry_point attributed function.
+        continue;
+      }
+    }
+
     // We should call GetAddrOfGlobal with IsForDefinition set to true in order
     // to get GlobalValue with exactly the type we need, not something that
     // might had been created for another decl with the same mangled name but
@@ -3644,6 +3665,10 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) {
     // Defer until all versions have been semantically checked.
     if (FD->hasAttr<TargetVersionAttr>() && !FD->isMultiVersion())
       return false;
+    // Defer emission of SYCL kernel entry point functions during device
+    // compilation.
+    if (LangOpts.SYCLIsDevice && FD->hasAttr<SYCLKernelEntryPointAttr>())
+      return false;
   }
   if (const auto *VD = dyn_cast<VarDecl>(Global)) {
     if (Context.getInlineVariableDefinitionKind(VD) ==
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 46de3d868f901..9a0bc675e0baa 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -1972,6 +1972,11 @@ class CodeGenModule : public CodeGenTypeCache {
   /// .gcda files in a way that persists in .bc files.
   void EmitCoverageFile();
 
+  /// Given a sycl_kernel_entry_point attributed function, emit the
+  /// corresponding SYCL kernel caller offload entry point function.
+  void EmitSYCLKernelCaller(const FunctionDecl *KernelEntryPointFn,
+                            ASTContext &Ctx);
+
   /// Determine whether the definition must be emitted; if this returns \c
   /// false, the definition can be emitted lazily if it's used.
   bool MustBeEmitted(const ValueDecl *D);
diff --git a/clang/lib/CodeGen/CodeGenSYCL.cpp b/clang/lib/CodeGen/CodeGenSYCL.cpp
new file mode 100644
index 0000000000000..b9a96fe8ab838
--- /dev/null
+++ b/clang/lib/CodeGen/CodeGenSYCL.cpp
@@ -0,0 +1,72 @@
+//===--------- CodeGenSYCL.cpp - Code for SYCL kernel generation ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code required for generation of SYCL kernel caller offload
+// entry point functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeGenFunction.h"
+#include "CodeGenModule.h"
+
+using namespace clang;
+using namespace CodeGen;
+
+static void SetSYCLKernelAttributes(llvm::Function *Fn, CodeGenFunction &CGF) {
+  // SYCL 2020 device language restrictions require forward progress and
+  // disallow recursion.
+  Fn->setDoesNotRecurse();
+  if (CGF.checkIfFunctionMustProgress())
+    Fn->addFnAttr(llvm::Attribute::MustProgress);
+}
+
+void CodeGenModule::EmitSYCLKernelCaller(const FunctionDecl *KernelEntryPointFn,
+                                         ASTContext &Ctx) {
+  assert(Ctx.getLangOpts().SYCLIsDevice &&
+         "SYCL kernel caller offload entry point functions can only be emitted"
+         " during device compilation");
+
+  const auto *KernelEntryPointAttr =
+      KernelEntryPointFn->getAttr<SYCLKernelEntryPointAttr>();
+  assert(KernelEntryPointAttr && "Missing sycl_kernel_entry_point attribute");
+  assert(!KernelEntryPointAttr->isInvalidAttr() &&
+         "sycl_kernel_entry_point attribute is invalid");
+
+  // Find the SYCLKernelCallStmt.
+  SYCLKernelCallStmt *KernelCallStmt =
+      cast<SYCLKernelCallStmt>(KernelEntryPointFn->getBody());
+
+  // Retrieve the SYCL kernel caller parameters from the OutlinedFunctionDecl.
+  FunctionArgList Args;
+  const OutlinedFunctionDecl *OutlinedFnDecl =
+      KernelCallStmt->getOutlinedFunctionDecl();
+  Args.append(OutlinedFnDecl->param_begin(), OutlinedFnDecl->param_end());
+
+  // Compute the function info and LLVM function type.
+  const CGFunctionInfo &FnInfo =
+      getTypes().arrangeSYCLKernelCallerDeclaration(Ctx.VoidTy, Args);
+  llvm::FunctionType *FnTy = getTypes().GetFunctionType(FnInfo);
+
+  // Retrieve the generated name for the SYCL kernel caller function.
+  CanQualType KernelNameType =
+      Ctx.getCanonicalType(KernelEntryPointAttr->getKernelName());
+  const SYCLKernelInfo &KernelInfo = Ctx.getSYCLKernelInfo(KernelNameType);
+  auto *Fn = llvm::Function::Create(FnTy, llvm::Function::ExternalLinkage,
+                                    KernelInfo.GetKernelName(), &getModule());
+
+  // Emit the SYCL kernel caller function.
+  CodeGenFunction CGF(*this);
+  SetLLVMFunctionAttributes(GlobalDecl(), FnInfo, Fn, false);
+  SetSYCLKernelAttributes(Fn, CGF);
+  CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, FnInfo, Args,
+                    SourceLocation(), SourceLocation());
+  CGF.EmitFunctionBody(OutlinedFnDecl->getBody());
+  setDSOLocal(Fn);
+  SetLLVMFunctionAttributesForDefinition(cast<Decl>(OutlinedFnDecl), Fn);
+  CGF.FinishFunction();
+}
diff --git a/clang/lib/CodeGen/CodeGenTypes.h b/clang/lib/CodeGen/CodeGenTypes.h
index 307048bcc510d..29f6f1ec80bc3 100644
--- a/clang/lib/CodeGen/CodeGenTypes.h
+++ b/clang/lib/CodeGen/CodeGenTypes.h
@@ -229,6 +229,13 @@ class CodeGenTypes {
   const CGFunctionInfo &arrangeBuiltinFunctionCall(QualType resultType,
                                                    const CallArgList &args);
 
+  /// A SYCL kernel caller function is an offload device entry point function
+  /// with a target device dependent calling convention such as amdgpu_kernel,
+  /// ptx_kernel, or spir_kernel.
+  const CGFunctionInfo &
+  arrangeSYCLKernelCallerDeclaration(QualType resultType,
+                                     const FunctionArgList &args);
+
   /// Objective-C methods are C functions with some implicit parameters.
   const CGFunctionInfo &arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD);
   const CGFunctionInfo &arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
diff --git a/clang/lib/CodeGen/Targets/NVPTX.cpp b/clang/lib/CodeGen/Targets/NVPTX.cpp
index f617e645a9eaf..25ab28c54b659 100644
--- a/clang/lib/CodeGen/Targets/NVPTX.cpp
+++ b/clang/lib/CodeGen/Targets/NVPTX.cpp
@@ -77,6 +77,10 @@ class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo {
     return true;
   }
 
+  unsigned getOpenCLKernelCallingConv() const override {
+    return llvm::CallingConv::PTX_Kernel;
+  }
+
   // Adds a NamedMDNode with GV, Name, and Operand as operands, and adds the
   // resulting MDNode to the nvvm.annotations MDNode.
   static void addNVVMMetadata(llvm::GlobalValue *GV, StringRef Name,
diff --git a/clang/test/CodeGenSYCL/kernel-caller-entry-point.cpp b/clang/test/CodeGenSYCL/kernel-caller-entry-point.cpp
new file mode 100644
index 0000000000000..195f1d9d26d7d
--- /dev/null
+++ b/clang/test/CodeGenSYCL/kernel-caller-entry-point.cpp
@@ -0,0 +1,184 @@
+// RUN: %clang_cc1 -fsycl-is-host -emit-llvm -triple x86_64-unknown-linux-gnu -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-HOST,CHECK-HOST-LINUX %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-unknown-linux-gnu -triple amdgcn-amd-amdhsa -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-AMDGCN %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-unknown-linux-gnu -triple nvptx-nvidia-cuda -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-NVPTX %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-unknown-linux-gnu -triple nvptx64-nvidia-cuda -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-NVPTX %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-unknown-linux-gnu -triple spir-unknown-unknown -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-SPIR %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-unknown-linux-gnu -triple spir64-unknown-unknown -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-SPIR %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-unknown-linux-gnu -triple spirv32-unknown-unknown -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-SPIR %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-unknown-linux-gnu -triple spirv64-unknown-unknown -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-SPIR %s
+// RUN: %clang_cc1 -fsycl-is-host -emit-llvm -triple x86_64-pc-windows-msvc -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-HOST,CHECK-HOST-WINDOWS %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-pc-windows-msvc -triple amdgcn-amd-amdhsa -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-AMDGCN %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-pc-windows-msvc -triple nvptx-nvidia-cuda -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-NVPTX %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-pc-windows-msvc -triple nvptx64-nvidia-cuda -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-NVPTX %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-pc-windows-msvc -triple spir-unknown-unknown -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-SPIR %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-pc-windows-msvc -triple spir64-unknown-unknown -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-SPIR %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-pc-windows-msvc -triple spirv32-unknown-unknown -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-SPIR %s
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -aux-triple x86_64-pc-windows-msvc -triple spirv64-unknown-unknown -std=c++17 %s -o - | FileCheck --check-prefixes=CHECK-DEVICE,CHECK-SPIR %s
+
+// Test the generation of SYCL kernel caller functions. These functions are
+// generated from functions declared with the sycl_kernel_entry_point attribute
+// and emited during device compilation. They are not emitted during device
+// compilation.
+
+struct single_purpose_kernel_name;
+struct single_purpose_kernel {
+  void operator()() const {}
+};
+
+[[clang::sycl_kernel_entry_point(single_purpose_kernel_name)]]
+void single_purpose_kernel_task(single_purpose_kernel kernelFunc) {
+  kernelFunc();
+}
+
+template <typename KernelName, typename KernelType>
+[[clang::sycl_kernel_entry_point(KernelName)]]
+void kernel_single_task(KernelType kernelFunc) {
+  kernelFunc(42);
+}
+
+int main() {
+  single_purpose_kernel obj;
+  single_purpose_kernel_task(obj);
+  int capture;
+  auto lambda = [=](auto) { (void) capture; };
+  kernel_single_task<decltype(lambda)>(lambda);
+}
+
+// Verify that SYCL kernel caller functions are not emitted during host
+// compilation.
+//
+// CHECK-HOST-NOT: _ZTS26single_purpose_kernel_name
+// CHECK-HOST-NOT: _ZTSZ4mainE18lambda_kernel_name
+
+// Verify that sycl_kernel_entry_point attributed functions are not emitted
+// during device compilation.
+//
+// CHECK-DEVICE-NOT: single_purpose_kernel_task
+// CHECK-DEVICE-NOT: kernel_single_task
+
+// Verify that no code is generated for the bodies of sycl_kernel_entry_point
+// attributed functions during host compilation. ODR-use of these functions may
+// require them to be emitted, but they have no effect if called.
+//
+// CHECK-HOST-LINUX:      define dso_local void @_Z26single_purpose_kernel_task21single_purpose_kernel() #{{[0-9]+}} {
+// CHECK-HOST-LINUX-NEXT: entry:
+// CHECK-HOST-LINUX-NEXT:   %kernelFunc = alloca %struct.single_purpose_kernel, align 1
+// CHECK-HOST-LINUX-NEXT:   ret void
+// CHECK-HOST-LINUX-NEXT: }
+//
+// CHECK-HOST-LINUX:      define internal void @_Z18kernel_single_taskIZ4mainEUlT_E_S1_EvT0_(i32 %kernelFunc.coerce) #{{[0-9]+}} {
+// CHECK-HOST-LINUX-NEXT: entry:
+// CHECK-HOST-LINUX-NEXT:   %kernelFunc = alloca %class.anon, align 4
+// CHECK-HOST-LINUX-NEXT:   %coerce.dive = getelementptr inbounds nuw %class.anon, ptr %kernelFunc, i32 0, i32 0
+// CHECK-HOST-LINUX-NEXT:   store i32 %kernelFunc.coerce, ptr %coerce.dive, align 4
+// CHECK-HOST-LINUX-NEXT:   ret void
+// CHECK-HOST-LINUX-NEXT: }
+//
+// CHECK-HOST-WINDOWS:      define dso_local void @"?single_purpose_kernel_task@@YAXUsingle_purpose_kernel@@@Z"(i8 %kernelFunc.coerce) #{{[0-9]+}} {
+// CHECK-HOST-WINDOWS-NEXT: entry:
+// CHECK-HOST-WINDOWS-NEXT:   %kernelFunc = alloca %struct.single_purpose_kernel, align 1
+// CHECK-HOST-WINDOWS-NEXT:   %coerce.dive = getelementptr inbounds nuw %struct.single_purpose_kernel, ptr %kernelFunc, i32 0, i32 0
+// CHECK-HOST-WINDOWS-NEXT:   store i8 %kernelFunc.coerce, ptr %coerce.dive, align 1
+// CHECK-HOST-WINDOWS-NEXT:   ret void
+// CHECK-HOST-WINDOWS-NEXT: }
+//
+// CHECK-HOST-WINDOWS:      define internal void @"??$kernel_single_task@V<lambda_1>@?0??main@@9@V1?0??2@9@@@YAXV<lambda_1>@?0??main@@9@@Z"(i32 %kernelFunc.coerce) #{{[0-9]+}} {
+// CHECK-HOST-WINDOWS-NEXT: entry:
+// CHECK-HOST-WINDOWS-NEXT:   %kernelFunc = alloca %class.anon, align 4
+// CHECK-HOST-WINDOWS-NEXT:   %coerce.dive = getelementptr inbounds nuw %class.anon, ptr %kernelFunc, i32 0, i32 0
+// CHECK-HOST-WINDOWS-NEXT:   store i32 %kernelFunc.coerce, ptr %coerce.dive, align 4
+// CHECK-HOST-WINDOWS-NEXT:   ret void
+// CHECK-HOST-WINDOWS-NEXT: }
+
+// Verify that SYCL kernel caller functions are emitted for each device target.
+//
+// FIXME: The following set of matches are used to skip over the declaration of
+// main(). main() shouldn't be emitted in device code, but that pruning isn't
+// performed yet.
+// CHECK-DEVICE:      Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+// CHECK-DEVICE-NEXT: define {{[a-z_ ]*}}noundef i32 @main() #0
+
+// IR for the SYCL kernel caller function generated for
+// single_purpose_kernel_task with single_purpose_kernel_name as the SYCL kernel
+// name type.
+//
+// CHECK-AMDGCN:      Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+// CHECK-AMDGCN-NEXT: define dso_local amdgpu_kernel void @_ZTS26single_purpose_kernel_name
+// CHECK-AMDGCN-SAME:   (ptr addrspace(4) noundef byref(%struct.single_purpose_kernel) align 1 %0) #[[AMDGCN_ATTR0:[0-9]+]] {
+// CHECK-AMDGCN-NEXT: entry:
+// CHECK-AMDGCN-NEXT:   %coerce = alloca %struct.single_purpose_kernel, align 1, addrspace(5)
+// CHECK-AMDGCN-NEXT:   %kernelFunc = addrspacecast ptr addrspace(5) %coerce to ptr
+// CHECK-AMDGCN-NEXT:   call void @llvm.memcpy.p0.p4.i64(ptr align 1 %kernelFunc, ptr addrspace(4) align 1 %0, i64 1, i1 false)
+// CHECK-AMDGCN-NEXT:   call void @_ZNK21single_purpose_kernelclEv
+// CHECK-AMDGCN-SAME:     (ptr noundef nonnull align 1 dereferenceable(1) %kernelFunc) #[[AMDGCN_ATTR1:[0-9]+]]
+// CHECK-AMDGCN-NEXT:   ret void
+// CHECK-AMDGCN-NEXT: }
+// CHECK-AMDGCN:      define linkonce_odr void @_ZNK21single_purpose_kernelclEv
+//
+// CHECK-NVPTX:       Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+// CHECK-NVPTX-NEXT:  define dso_local ptx_kernel void @_ZTS26single_purpose_kernel_name
+// CHECK-NVPTX-SAME:    (ptr noundef byval(%struct.single_purpose_kernel) align 1 %kernelFunc) #[[NVPTX_ATTR0:[0-9]+]] {
+// CHECK-NVPTX-NEXT:  entry:
+// CHECK-NVPTX-NEXT:    call void @_ZNK21single_purpose_kernelclEv
+// CHECK-NVPTX-SAME:      (ptr noundef nonnull align 1 dereferenceable(1) %kernelFunc) #[[NVPTX_ATTR1:[0-9]+]]
+// CHECK-NVPTX-NEXT:    ret void
+// CHECK-NVPTX-NEXT:  }
+// CHECK-NVPTX:       define linkonce_odr void @_ZNK21single_purpose_kernelclEv
+//
+// CHECK-SPIR:        Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+// CHECK-SPIR-NEXT:   define {{[a-z_ ]*}}spir_kernel void @_ZTS26single_purpose_kernel_name
+// CHECK-SPIR-SAME:     (ptr noundef byval(%struct.single_purpose_kernel) align 1 %kernelFunc) #[[SPIR_ATTR0:[0-9]+]] {
+// CHECK-SPIR-NEXT:   entry:
+// CHECK-SPIR-NEXT:     %kernelFunc.ascast = addrspacecast ptr %kernelFunc to ptr addrspace(4)
+// CHECK-SPIR-NEXT:     call spir_func void @_ZNK21single_purpose_kernelclEv
+// CHECK-SPIR-SAME:       (ptr addrspace(4) noundef align 1 dereferenceable_or_null(1) %kernelFunc.ascast) #[[SPIR_ATTR1:[0-9]+]]
+// CHECK-SPIR-NEXT:     ret void
+// CHECK-SPIR-NEXT:   }
+// CHECK-SPIR:        define linkonce_odr spir_func void @_ZNK21single_purpose_kernelclEv
+
+// IR for the SYCL kernel caller function generated for kernel_single_task with
+// lambda_kernel_name as the SYCL kernel name type.
+//
+// CHECK-AMDGCN:      Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+// CHECK-AMDGCN-NEXT: define dso_local amdgpu_kernel void @_ZTSZ4mainEUlT_E_
+// CHECK-AMDGCN-SAME:   (i32 %kernelFunc.coerce) #[[AMDGCN_ATTR0]] {
+// CHECK-AMDGCN-NEXT: entry:
+// CHECK-AMDGCN-NEXT:   %kernelFunc = alloca %class.anon, align 4, addrspace(5)
+// CHECK-AMDGCN-NEXT:   %kernelFunc1 = addrspacecast ptr addrspace(5) %kernelFunc to ptr
+// CHECK-AMDGCN-NEXT:   %coerce.dive = getelementptr inbounds nuw %class.anon, ptr %kernelFunc1, i32 0, i32 0
+// CHECK-AMDGCN-NEXT:   store i32 %kernelFunc.coerce, ptr %coerce.dive, align 4
+// CHECK-AMDGCN-NEXT:   call void @_ZZ4mainENKUlT_E_clIiEEDaS_
+// CHECK-AMDGCN-SAME:     (ptr noundef nonnull align 4 dereferenceable(4) %kernelFunc1, i32 noundef 42) #[[AMDGCN_ATTR1]]
+// CHECK-AMDGCN-NEXT:   ret void
+// CHECK-AMDGCN-NEXT: }
+// CHECK-AMDGCN:      define internal void @_ZZ4mainENKUlT_E_clIiEEDaS_
+//
+// CHECK-NVPTX:       Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+// CHECK-NVPTX-NEXT:  define dso_local ptx_kernel void @_ZTSZ4mainEUlT_E_
+// CHECK-NVPTX-SAME:    (ptr noundef byval(%class.anon) align 4 %kernelFunc) #[[NVPTX_ATTR0]] {
+// CHECK-NVPTX-NEXT:  entry:
+// CHECK-NVPTX-NEXT:    call void @_ZZ4mainENKUlT_E_clIiEEDaS_
+// CHECK-NVPTX-SAME:      (ptr noundef nonnull align 4 dereferenceable(4) %kernelFunc, i32 noundef 42) #[[NVPTX_ATTR1]]
+// CHECK-NVPTX-NEXT:    ret void
+// CHECK-NVPTX-NEXT:  }
+// CHECK-NVPTX:       define internal void @_ZZ4mainENKUlT_E_clIiEEDaS_
+//
+// CHECK-SPIR:        Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
+// CHECK-SPIR-NEXT:   define {{[a-z_ ]*}}spir_kernel void @_ZTSZ4mainEUlT_E_
+// CHECK-SPIR-SAME:     (ptr noundef byval(%class.anon) align 4 %kernelFunc) #[[SPIR_ATTR0]] {
+// CHECK-SPIR-NEXT:   entry:
+// CHECK-SPIR-NEXT:     %kernelFunc.ascast = addrspacecast ptr %kernelFunc to ptr addrspace(4)
+// CHECK-SPIR-NEXT:     call spir_func void @_ZZ4mainENKUlT_E_clIiEEDaS_
+// CHECK-SPIR-SAME:       (ptr addrspace(4) noundef align 4 dereferenceable_or_null(4) %kernelFunc.ascast, i32 noundef 42) #[[SPIR_ATTR1]]
+// CHECK-SPIR-NEXT:     ret void
+// CHECK-SPIR-NEXT:   }
+// CHECK-SPIR:        define internal spir_func void @_ZZ4mainENKUlT_E_clIiEEDaS_
+
+// CHECK-AMDGCN: #[[AMDGCN_ATTR0]] = { convergent mustprogress noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK-AMDGCN: #[[AMDGCN_ATTR1]] = { convergent nounwind }
+//
+// CHECK-NVPTX: #[[NVPTX_ATTR0]] = { convergent mustprogress noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ptx32" }
+// CHECK-NVPTX: #[[NVPTX_ATTR1]] = { convergent nounwind }
+//
+// CHECK-SPIR: #[[SPIR_ATTR0]] = { convergent mustprogress noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK-SPIR: #[[SPIR_ATTR1]] = { convergent nounwind }

From 40417915a161e87b398f1cc3e9b7c159207abd77 Mon Sep 17 00:00:00 2001
From: Mariya Podchishchaeva <mariya.podchishchaeva@intel.com>
Date: Thu, 17 Apr 2025 15:15:07 +0200
Subject: [PATCH 264/710] [clang] Implement StmtPrinter for EmbedExpr (#135957)

Tries to avoid memory leaks previously caused by saving filename by
allocating memory in the preprocessor.

Fixes https://github.com/llvm/llvm-project/issues/132641
Fixes https://github.com/llvm/llvm-project/issues/107869

---------

Co-authored-by: Aaron Ballman <aaron@aaronballman.com>
---
 clang/docs/ReleaseNotes.rst             |  3 ++
 clang/include/clang/AST/Expr.h          |  4 +++
 clang/include/clang/Lex/Preprocessor.h  |  3 +-
 clang/include/clang/Sema/Sema.h         |  2 +-
 clang/lib/AST/StmtPrinter.cpp           |  6 +++-
 clang/lib/Lex/PPDirectives.cpp          | 16 ++++++++--
 clang/lib/Parse/ParseInit.cpp           |  2 +-
 clang/lib/Sema/SemaExpr.cpp             |  3 +-
 clang/test/Preprocessor/embed_weird.cpp | 40 +++++++++++++++++++++++++
 9 files changed, 72 insertions(+), 7 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 4f640697e1817..0ea8498351c24 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -426,6 +426,9 @@ Bug Fixes in This Version
   using C++23 "deducing this" did not have a diagnostic location (#GH135522)
 
 - Fixed a crash when a ``friend`` function is redefined as deleted. (#GH135506)
+- Fixed a crash when ``#embed`` appears as a part of a failed constant
+  evaluation. The crashes were happening during diagnostics emission due to
+  unimplemented statement printer. (#GH132641)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 529c6228bfa19..a83320a7ddec2 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -4961,6 +4961,9 @@ class SourceLocExpr final : public Expr {
 /// Stores data related to a single #embed directive.
 struct EmbedDataStorage {
   StringLiteral *BinaryData;
+  // FileName string already includes braces, i.e. it is <files/my_file> for a
+  // directive #embed <files/my_file>.
+  StringRef FileName;
   size_t getDataElementCount() const { return BinaryData->getByteLength(); }
 };
 
@@ -5007,6 +5010,7 @@ class EmbedExpr final : public Expr {
   SourceLocation getEndLoc() const { return EmbedKeywordLoc; }
 
   StringLiteral *getDataStringLiteral() const { return Data->BinaryData; }
+  StringRef getFileName() const { return Data->FileName; }
   EmbedDataStorage *getData() const { return Data; }
 
   unsigned getStartingElementPos() const { return Begin; }
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 24bb524783e93..19d54edf23faf 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -2762,7 +2762,7 @@ class Preprocessor {
                             const FileEntry *LookupFromFile = nullptr);
   void HandleEmbedDirectiveImpl(SourceLocation HashLoc,
                                 const LexEmbedParametersResult &Params,
-                                StringRef BinaryContents);
+                                StringRef BinaryContents, StringRef FileName);
 
   // File inclusion.
   void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
@@ -3066,6 +3066,7 @@ class EmptylineHandler {
 /// preprocessor to the parser through an annotation token.
 struct EmbedAnnotationData {
   StringRef BinaryData;
+  StringRef FileName;
 };
 
 /// Registry of pragma handlers added by plugins
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index fe37fd7701ce3..c167f8df28acd 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -7256,7 +7256,7 @@ class Sema final : public SemaBase {
 
   // #embed
   ExprResult ActOnEmbedExpr(SourceLocation EmbedKeywordLoc,
-                            StringLiteral *BinaryData);
+                            StringLiteral *BinaryData, StringRef FileName);
 
   // Build a potentially resolved SourceLocExpr.
   ExprResult BuildSourceLocExpr(SourceLocIdentKind Kind, QualType ResultTy,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index aae10fd3bd885..c6c49c6c1ba4d 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -1284,7 +1284,11 @@ void StmtPrinter::VisitSourceLocExpr(SourceLocExpr *Node) {
 }
 
 void StmtPrinter::VisitEmbedExpr(EmbedExpr *Node) {
-  llvm::report_fatal_error("Not implemented");
+  // FIXME: Embed parameters are not reflected in the AST, so there is no way to
+  // print them yet.
+  OS << "#embed ";
+  OS << Node->getFileName();
+  OS << NL;
 }
 
 void StmtPrinter::VisitConstantExpr(ConstantExpr *Node) {
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 8411526019f3e..49a4e24923a5e 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -3909,7 +3909,7 @@ Preprocessor::LexEmbedParameters(Token &CurTok, bool ForHasEmbed) {
 
 void Preprocessor::HandleEmbedDirectiveImpl(
     SourceLocation HashLoc, const LexEmbedParametersResult &Params,
-    StringRef BinaryContents) {
+    StringRef BinaryContents, StringRef FileName) {
   if (BinaryContents.empty()) {
     // If we have no binary contents, the only thing we need to emit are the
     // if_empty tokens, if any.
@@ -3940,6 +3940,7 @@ void Preprocessor::HandleEmbedDirectiveImpl(
 
   EmbedAnnotationData *Data = new (BP) EmbedAnnotationData;
   Data->BinaryData = BinaryContents;
+  Data->FileName = FileName;
 
   Toks[CurIdx].startToken();
   Toks[CurIdx].setKind(tok::annot_embed);
@@ -4049,5 +4050,16 @@ void Preprocessor::HandleEmbedDirective(SourceLocation HashLoc, Token &EmbedTok,
   if (Callbacks)
     Callbacks->EmbedDirective(HashLoc, Filename, isAngled, MaybeFileRef,
                               *Params);
-  HandleEmbedDirectiveImpl(HashLoc, *Params, BinaryContents);
+  // getSpelling() may return a buffer from the token itself or it may use the
+  // SmallString buffer we provided. getSpelling() may also return a string that
+  // is actually longer than FilenameTok.getLength(), so we first pass a
+  // locally created buffer to getSpelling() to get the string of real length
+  // and then we allocate a long living buffer because the buffer we used
+  // previously will only live till the end of this function and we need
+  // filename info to live longer.
+  void *Mem = BP.Allocate(OriginalFilename.size(), alignof(char *));
+  memcpy(Mem, OriginalFilename.data(), OriginalFilename.size());
+  StringRef FilenameToGo =
+      StringRef(static_cast<char *>(Mem), OriginalFilename.size());
+  HandleEmbedDirectiveImpl(HashLoc, *Params, BinaryContents, FilenameToGo);
 }
diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp
index 471b3eaf28287..86696329ea02e 100644
--- a/clang/lib/Parse/ParseInit.cpp
+++ b/clang/lib/Parse/ParseInit.cpp
@@ -451,7 +451,7 @@ ExprResult Parser::createEmbedExpr() {
 
     StringLiteral *BinaryDataArg = CreateStringLiteralFromStringRef(
         Data->BinaryData, Context.UnsignedCharTy);
-    Res = Actions.ActOnEmbedExpr(StartLoc, BinaryDataArg);
+    Res = Actions.ActOnEmbedExpr(StartLoc, BinaryDataArg, Data->FileName);
   }
   return Res;
 }
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 6830bb5c01c7d..01117e7785bc4 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16903,9 +16903,10 @@ ExprResult Sema::BuildSourceLocExpr(SourceLocIdentKind Kind, QualType ResultTy,
 }
 
 ExprResult Sema::ActOnEmbedExpr(SourceLocation EmbedKeywordLoc,
-                                StringLiteral *BinaryData) {
+                                StringLiteral *BinaryData, StringRef FileName) {
   EmbedDataStorage *Data = new (Context) EmbedDataStorage;
   Data->BinaryData = BinaryData;
+  Data->FileName = FileName;
   return new (Context)
       EmbedExpr(Context, EmbedKeywordLoc, Data, /*NumOfElements=*/0,
                 Data->getDataElementCount());
diff --git a/clang/test/Preprocessor/embed_weird.cpp b/clang/test/Preprocessor/embed_weird.cpp
index 9a984e40d4aa2..e44aff55c59e2 100644
--- a/clang/test/Preprocessor/embed_weird.cpp
+++ b/clang/test/Preprocessor/embed_weird.cpp
@@ -136,3 +136,43 @@ constexpr struct HasChar c = {
                                 c-error {{constexpr initializer evaluates to 255 which is not exactly representable in type 'signed char'}}
 
 };
+
+#if __cplusplus
+namespace std {
+typedef decltype(sizeof(int)) size_t;
+
+template <class _E> class initializer_list {
+  const _E *__begin_;
+  size_t __size_;
+
+  constexpr initializer_list(const _E *__b, size_t __s)
+      : __begin_(__b), __size_(__s) {}
+
+public:
+  constexpr initializer_list() : __begin_(nullptr), __size_(0) {}
+};
+} // namespace std
+
+class S2 {
+public:
+  constexpr S2(std::initializer_list<char>)  { // cxx-error {{constexpr constructor never produces a constant expression}}
+    1/0; // cxx-warning {{division by zero is undefined}}
+         // cxx-warning@-1 {{unused}}
+         // cxx-note@-2 4{{division by zero}}
+  }
+};
+
+
+constexpr S2 s2 { // cxx-error {{must be initialized by a constant expression}}
+                  // cxx-note-re@-1 {{in call to 'S2{{.*}} #embed <jk.txt>}}
+#embed <jk.txt> prefix(0x2c, 0x20, )limit(5)
+};
+constexpr S2 s3 {1, // cxx-error {{must be initialized by a constant expression}}
+                    // cxx-note-re@-1 {{in call to 'S2{{.*}} #embed "jk.txt"}}
+#embed "jk.txt"
+};
+constexpr S2 s4 { // cxx-error {{must be initialized by a constant expression}}
+                  // cxx-note-re@-1 {{in call to 'S2{{.*}}"jk"}}
+#embed "jk.txt"
+};
+#endif

From 3dfdb4dad5da316a9de88828e9feebca99ba7ae4 Mon Sep 17 00:00:00 2001
From: zhijian lin <zhijian@ca.ibm.com>
Date: Thu, 17 Apr 2025 09:23:14 -0400
Subject: [PATCH 265/710] [SelectionDAG] Propagate poison in getNode with two
 operands if the input is poison. (#135387)

Propagation to poison in function `SDValue
SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,SDValue
N1, SDValue N2, const SDNodeFlags Flags) ` if one of the input is
poison.

 The patch also revert the test cases
 llvm/test/CodeGen/X86/pr119158.ll
 llvm/test/CodeGen/X86/half.ll

which are mentioned in
https://github.com/llvm/llvm-project/pull/125883#discussion_r2021390919

---------

Co-authored-by: Amy Kwan <amy.kwan1@ibm.com>
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 19 +++++++++++----
 llvm/test/CodeGen/X86/half.ll                 | 24 +++++++++----------
 llvm/test/CodeGen/X86/poison-ops.ll           | 20 ----------------
 llvm/test/CodeGen/X86/pr119158.ll             |  7 +++---
 4 files changed, 29 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 8682c40898046..b45369ecf7bfd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -7578,7 +7578,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     } else {
       switch (Opcode) {
       case ISD::SUB:
-        return getUNDEF(VT);     // fold op(undef, arg2) -> undef
+        // fold op(undef, arg2) -> undef, fold op(poison, arg2) ->poison.
+        return N1.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
       case ISD::SIGN_EXTEND_INREG:
       case ISD::UDIV:
       case ISD::SDIV:
@@ -7586,7 +7587,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
       case ISD::SREM:
       case ISD::SSUBSAT:
       case ISD::USUBSAT:
-        return getConstant(0, DL, VT);    // fold op(undef, arg2) -> 0
+        // fold op(undef, arg2) -> 0, fold op(poison, arg2) -> poison.
+        return N1.getOpcode() == ISD::POISON ? getPOISON(VT)
+                                             : getConstant(0, DL, VT);
       }
     }
   }
@@ -7606,16 +7609,22 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     case ISD::SDIV:
     case ISD::UREM:
     case ISD::SREM:
-      return getUNDEF(VT);       // fold op(arg1, undef) -> undef
+      // fold op(arg1, undef) -> undef, fold op(arg1, poison) -> poison.
+      return N2.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
     case ISD::MUL:
     case ISD::AND:
     case ISD::SSUBSAT:
     case ISD::USUBSAT:
-      return getConstant(0, DL, VT);  // fold op(arg1, undef) -> 0
+      // fold op(arg1, undef) -> 0, fold op(arg1, poison) -> poison.
+      return N2.getOpcode() == ISD::POISON ? getPOISON(VT)
+                                           : getConstant(0, DL, VT);
     case ISD::OR:
     case ISD::SADDSAT:
     case ISD::UADDSAT:
-      return getAllOnesConstant(DL, VT);
+      // fold op(arg1, undef) -> an all-ones constant, fold op(arg1, poison) ->
+      // poison.
+      return N2.getOpcode() == ISD::POISON ? getPOISON(VT)
+                                           : getAllOnesConstant(DL, VT);
     }
   }
 
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index a64238170cef9..1b98886ba24e7 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -1991,8 +1991,8 @@ define void @pr63114() {
 ; CHECK-LIBCALL-LABEL: pr63114:
 ; CHECK-LIBCALL:       # %bb.0:
 ; CHECK-LIBCALL-NEXT:    movdqu (%rax), %xmm4
-; CHECK-LIBCALL-NEXT:    pshufhw {{.*#+}} xmm0 = xmm4[0,1,2,3,4,5,7,7]
-; CHECK-LIBCALL-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-LIBCALL-NEXT:    pshuflw {{.*#+}} xmm0 = xmm4[0,1,3,3,4,5,6,7]
+; CHECK-LIBCALL-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,1]
 ; CHECK-LIBCALL-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,0,65535,65535,65535,65535]
 ; CHECK-LIBCALL-NEXT:    pand %xmm1, %xmm0
 ; CHECK-LIBCALL-NEXT:    movq {{.*#+}} xmm2 = [0,0,0,15360,0,0,0,0]
@@ -2001,8 +2001,8 @@ define void @pr63114() {
 ; CHECK-LIBCALL-NEXT:    pand %xmm3, %xmm0
 ; CHECK-LIBCALL-NEXT:    movdqa {{.*#+}} xmm5 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,60]
 ; CHECK-LIBCALL-NEXT:    por %xmm5, %xmm0
-; CHECK-LIBCALL-NEXT:    pshuflw {{.*#+}} xmm6 = xmm4[0,1,3,3,4,5,6,7]
-; CHECK-LIBCALL-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,0,2,1]
+; CHECK-LIBCALL-NEXT:    pshufhw {{.*#+}} xmm6 = xmm4[0,1,2,3,4,5,7,7]
+; CHECK-LIBCALL-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
 ; CHECK-LIBCALL-NEXT:    pand %xmm1, %xmm6
 ; CHECK-LIBCALL-NEXT:    por %xmm2, %xmm6
 ; CHECK-LIBCALL-NEXT:    pand %xmm3, %xmm6
@@ -2020,8 +2020,8 @@ define void @pr63114() {
 ; CHECK-LIBCALL-NEXT:    por %xmm5, %xmm7
 ; CHECK-LIBCALL-NEXT:    movdqu %xmm7, 0
 ; CHECK-LIBCALL-NEXT:    movdqu %xmm4, 32
-; CHECK-LIBCALL-NEXT:    movdqu %xmm6, 16
-; CHECK-LIBCALL-NEXT:    movdqu %xmm0, 48
+; CHECK-LIBCALL-NEXT:    movdqu %xmm6, 48
+; CHECK-LIBCALL-NEXT:    movdqu %xmm0, 16
 ; CHECK-LIBCALL-NEXT:    retq
 ;
 ; BWON-F16C-LABEL: pr63114:
@@ -2056,8 +2056,8 @@ define void @pr63114() {
 ; CHECK-I686-LABEL: pr63114:
 ; CHECK-I686:       # %bb.0:
 ; CHECK-I686-NEXT:    movdqu (%eax), %xmm6
-; CHECK-I686-NEXT:    pshufhw {{.*#+}} xmm0 = xmm6[0,1,2,3,4,5,7,7]
-; CHECK-I686-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; CHECK-I686-NEXT:    pshuflw {{.*#+}} xmm0 = xmm6[0,1,3,3,4,5,6,7]
+; CHECK-I686-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,1]
 ; CHECK-I686-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,0,65535,65535,65535,65535]
 ; CHECK-I686-NEXT:    pand %xmm1, %xmm0
 ; CHECK-I686-NEXT:    movq {{.*#+}} xmm2 = [0,0,0,15360,0,0,0,0]
@@ -2066,8 +2066,8 @@ define void @pr63114() {
 ; CHECK-I686-NEXT:    pand %xmm3, %xmm0
 ; CHECK-I686-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,60]
 ; CHECK-I686-NEXT:    por %xmm4, %xmm0
-; CHECK-I686-NEXT:    pshuflw {{.*#+}} xmm5 = xmm6[0,1,3,3,4,5,6,7]
-; CHECK-I686-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,0,2,1]
+; CHECK-I686-NEXT:    pshufhw {{.*#+}} xmm5 = xmm6[0,1,2,3,4,5,7,7]
+; CHECK-I686-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
 ; CHECK-I686-NEXT:    pand %xmm1, %xmm5
 ; CHECK-I686-NEXT:    por %xmm2, %xmm5
 ; CHECK-I686-NEXT:    pand %xmm3, %xmm5
@@ -2085,8 +2085,8 @@ define void @pr63114() {
 ; CHECK-I686-NEXT:    por %xmm4, %xmm7
 ; CHECK-I686-NEXT:    movdqu %xmm7, 0
 ; CHECK-I686-NEXT:    movdqu %xmm6, 32
-; CHECK-I686-NEXT:    movdqu %xmm5, 16
-; CHECK-I686-NEXT:    movdqu %xmm0, 48
+; CHECK-I686-NEXT:    movdqu %xmm5, 48
+; CHECK-I686-NEXT:    movdqu %xmm0, 16
 ; CHECK-I686-NEXT:    retl
   %1 = load <24 x half>, ptr poison, align 2
   %2 = shufflevector <24 x half> %1, <24 x half> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
diff --git a/llvm/test/CodeGen/X86/poison-ops.ll b/llvm/test/CodeGen/X86/poison-ops.ll
index 3cd2ceb125ce8..636a42fd06e26 100644
--- a/llvm/test/CodeGen/X86/poison-ops.ll
+++ b/llvm/test/CodeGen/X86/poison-ops.ll
@@ -68,7 +68,6 @@ define <4 x i32> @sub_poison_lhs_vec(<4 x i32> %x) {
 define i32 @mul_poison_rhs(i32 %x) {
 ; CHECK-LABEL: mul_poison_rhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %r = mul i32 %x, poison
   ret i32 %r
@@ -77,7 +76,6 @@ define i32 @mul_poison_rhs(i32 %x) {
 define <4 x i32> @mul_poison_rhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: mul_poison_rhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = mul <4 x i32> %x, poison
   ret <4 x i32> %r
@@ -86,7 +84,6 @@ define <4 x i32> @mul_poison_rhs_vec(<4 x i32> %x) {
 define i32 @mul_poison_lhs(i32 %x) {
 ; CHECK-LABEL: mul_poison_lhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %r = mul i32 poison, %x
   ret i32 %r
@@ -95,7 +92,6 @@ define i32 @mul_poison_lhs(i32 %x) {
 define <4 x i32> @mul_poison_lhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: mul_poison_lhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = mul <4 x i32> poison, %x
   ret <4 x i32> %r
@@ -120,7 +116,6 @@ define <4 x i32> @sdiv_poison_rhs_vec(<4 x i32> %x) {
 define i32 @sdiv_poison_lhs(i32 %x) {
 ; CHECK-LABEL: sdiv_poison_lhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %r = sdiv i32 poison, %x
   ret i32 %r
@@ -129,7 +124,6 @@ define i32 @sdiv_poison_lhs(i32 %x) {
 define <4 x i32> @sdiv_poison_lhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: sdiv_poison_lhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = sdiv <4 x i32> poison, %x
   ret <4 x i32> %r
@@ -154,7 +148,6 @@ define <4 x i32> @udiv_poison_rhs_vec(<4 x i32> %x) {
 define i32 @udiv_poison_lhs(i32 %x) {
 ; CHECK-LABEL: udiv_poison_lhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %r = udiv i32 poison, %x
   ret i32 %r
@@ -163,7 +156,6 @@ define i32 @udiv_poison_lhs(i32 %x) {
 define <4 x i32> @udiv_poison_lhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: udiv_poison_lhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = udiv <4 x i32> poison, %x
   ret <4 x i32> %r
@@ -188,7 +180,6 @@ define <4 x i32> @srem_poison_rhs_vec(<4 x i32> %x) {
 define i32 @srem_poison_lhs(i32 %x) {
 ; CHECK-LABEL: srem_poison_lhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %r = srem i32 poison, %x
   ret i32 %r
@@ -197,7 +188,6 @@ define i32 @srem_poison_lhs(i32 %x) {
 define <4 x i32> @srem_poison_lhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: srem_poison_lhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = srem <4 x i32> poison, %x
   ret <4 x i32> %r
@@ -222,7 +212,6 @@ define <4 x i32> @urem_poison_rhs_vec(<4 x i32> %x) {
 define i32 @urem_poison_lhs(i32 %x) {
 ; CHECK-LABEL: urem_poison_lhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %r = urem i32 poison, %x
   ret i32 %r
@@ -231,7 +220,6 @@ define i32 @urem_poison_lhs(i32 %x) {
 define <4 x i32> @urem_poison_lhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: urem_poison_lhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = urem <4 x i32> poison, %x
   ret <4 x i32> %r
@@ -342,7 +330,6 @@ define <4 x i32> @shl_poison_lhs_vec(<4 x i32> %x) {
 define i32 @and_poison_rhs(i32 %x) {
 ; CHECK-LABEL: and_poison_rhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %r = and i32 %x, poison
   ret i32 %r
@@ -351,7 +338,6 @@ define i32 @and_poison_rhs(i32 %x) {
 define <4 x i32> @and_poison_rhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: and_poison_rhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = and <4 x i32> %x, poison
   ret <4 x i32> %r
@@ -360,7 +346,6 @@ define <4 x i32> @and_poison_rhs_vec(<4 x i32> %x) {
 define i32 @and_poison_lhs(i32 %x) {
 ; CHECK-LABEL: and_poison_lhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %r = and i32 poison, %x
   ret i32 %r
@@ -369,7 +354,6 @@ define i32 @and_poison_lhs(i32 %x) {
 define <4 x i32> @and_poison_lhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: and_poison_lhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = and <4 x i32> poison, %x
   ret <4 x i32> %r
@@ -378,7 +362,6 @@ define <4 x i32> @and_poison_lhs_vec(<4 x i32> %x) {
 define i32 @or_poison_rhs(i32 %x) {
 ; CHECK-LABEL: or_poison_rhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl $-1, %eax
 ; CHECK-NEXT:    retq
   %r = or i32 %x, poison
   ret i32 %r
@@ -387,7 +370,6 @@ define i32 @or_poison_rhs(i32 %x) {
 define <4 x i32> @or_poison_rhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: or_poison_rhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = or <4 x i32> %x, poison
   ret <4 x i32> %r
@@ -396,7 +378,6 @@ define <4 x i32> @or_poison_rhs_vec(<4 x i32> %x) {
 define i32 @or_poison_lhs(i32 %x) {
 ; CHECK-LABEL: or_poison_lhs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl $-1, %eax
 ; CHECK-NEXT:    retq
   %r = or i32 poison, %x
   ret i32 %r
@@ -405,7 +386,6 @@ define i32 @or_poison_lhs(i32 %x) {
 define <4 x i32> @or_poison_lhs_vec(<4 x i32> %x) {
 ; CHECK-LABEL: or_poison_lhs_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %r = or <4 x i32> poison, %x
   ret <4 x i32> %r
diff --git a/llvm/test/CodeGen/X86/pr119158.ll b/llvm/test/CodeGen/X86/pr119158.ll
index 4a1da30ca6c25..ca31df802c913 100644
--- a/llvm/test/CodeGen/X86/pr119158.ll
+++ b/llvm/test/CodeGen/X86/pr119158.ll
@@ -5,10 +5,9 @@ define dso_local void @foo() #1 {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %newFuncRoot
 ; CHECK-NEXT:    vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [18,0,18,0,18,0,18,0,18,0,18,0,18,0,18,0]
-; CHECK-NEXT:    vpmaddwd        %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; CHECK-NEXT:    vpsrld $7, %ymm0, %ymm0
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64]
+; CHECK-NEXT:    vpdpwssd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1
+; CHECK-NEXT:    vpsrld $7, %ymm1, %ymm0
 ; CHECK-NEXT:    vpackusdw %ymm0, %ymm0, %ymm0
 ; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; CHECK-NEXT:    vmovdqu %ymm0, (%rax)

From ea8b1bf8895aa6b8a5c668577f845e6a13ddc2e9 Mon Sep 17 00:00:00 2001
From: Antonio Frighetto <me@antoniofrighetto.com>
Date: Thu, 17 Apr 2025 15:36:02 +0200
Subject: [PATCH 266/710] =?UTF-8?q?[X86][FastISel]=C2=A0Refactor=20bitcast?=
 =?UTF-8?q?=20selection=20into=20separate=20function=20(NFC)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 llvm/lib/Target/X86/X86FastISel.cpp | 63 +++++++++++++++--------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index afe5d7f4bc7ed..d7cb93bdb7376 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -124,6 +124,7 @@ class X86FastISel final : public FastISel {
   bool X86SelectSIToFP(const Instruction *I);
   bool X86SelectUIToFP(const Instruction *I);
   bool X86SelectIntToFP(const Instruction *I, bool IsSigned);
+  bool X86SelectBitCast(const Instruction *I);
 
   const X86InstrInfo *getInstrInfo() const {
     return Subtarget->getInstrInfo();
@@ -2546,6 +2547,36 @@ bool X86FastISel::X86SelectTrunc(const Instruction *I) {
   return true;
 }
 
+bool X86FastISel::X86SelectBitCast(const Instruction *I) {
+  // Select SSE2/AVX bitcasts between 128/256/512 bit vector types.
+  MVT SrcVT, DstVT;
+  if (!Subtarget->hasSSE2() ||
+      !isTypeLegal(I->getOperand(0)->getType(), SrcVT) ||
+      !isTypeLegal(I->getType(), DstVT))
+    return false;
+
+  // Only allow vectors that use xmm/ymm/zmm.
+  if (!SrcVT.isVector() || !DstVT.isVector() ||
+      SrcVT.getVectorElementType() == MVT::i1 ||
+      DstVT.getVectorElementType() == MVT::i1)
+    return false;
+
+  Register Reg = getRegForValue(I->getOperand(0));
+  if (!Reg)
+    return false;
+
+  // Emit a reg-reg copy so we don't propagate cached known bits information
+  // with the wrong VT if we fall out of fast isel after selecting this.
+  const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
+  Register ResultReg = createResultReg(DstClass);
+  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
+          ResultReg)
+      .addReg(Reg);
+
+  updateValueMap(I, ResultReg);
+  return true;
+}
+
 bool X86FastISel::IsMemcpySmall(uint64_t Len) {
   return Len <= (Subtarget->is64Bit() ? 32 : 16);
 }
@@ -3693,36 +3724,8 @@ X86FastISel::fastSelectInstruction(const Instruction *I)  {
     updateValueMap(I, Reg);
     return true;
   }
-  case Instruction::BitCast: {
-    // Select SSE2/AVX bitcasts between 128/256/512 bit vector types.
-    if (!Subtarget->hasSSE2())
-      return false;
-
-    MVT SrcVT, DstVT;
-    if (!isTypeLegal(I->getOperand(0)->getType(), SrcVT) ||
-        !isTypeLegal(I->getType(), DstVT))
-      return false;
-
-    // Only allow vectors that use xmm/ymm/zmm.
-    if (!SrcVT.isVector() || !DstVT.isVector() ||
-        SrcVT.getVectorElementType() == MVT::i1 ||
-        DstVT.getVectorElementType() == MVT::i1)
-      return false;
-
-    Register Reg = getRegForValue(I->getOperand(0));
-    if (!Reg)
-      return false;
-
-    // Emit a reg-reg copy so we don't propagate cached known bits information
-    // with the wrong VT if we fall out of fast isel after selecting this.
-    const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
-    Register ResultReg = createResultReg(DstClass);
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
-              TII.get(TargetOpcode::COPY), ResultReg).addReg(Reg);
-
-    updateValueMap(I, ResultReg);
-    return true;
-  }
+  case Instruction::BitCast:
+    return X86SelectBitCast(I);
   }
 
   return false;

From e6f23b383e0c08ecc380d879035e594019f34211 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Thu, 17 Apr 2025 09:44:29 -0400
Subject: [PATCH 267/710] Remove names for unused parameters; NFC

Fixes #21393
---
 llvm/include/llvm/Support/ErrorOr.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Support/ErrorOr.h b/llvm/include/llvm/Support/ErrorOr.h
index 1daac9a436c91..4afe56690ca07 100644
--- a/llvm/include/llvm/Support/ErrorOr.h
+++ b/llvm/include/llvm/Support/ErrorOr.h
@@ -185,7 +185,7 @@ class ErrorOr {
   }
 
   template <class T1, class T2>
-  static bool compareThisIfSameType(const T1 &a, const T2 &b) {
+  static bool compareThisIfSameType(const T1 &, const T2 &) {
     return false;
   }
 

From 927a0cb8d654fe11dc010dc3d05e200772709ce6 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Thu, 17 Apr 2025 15:07:33 +0100
Subject: [PATCH 268/710] [LV][NFC] Regenerate AArch64/veclib-* test CHECK
 lines (#136138)

---
 .../AArch64/veclib-function-calls.ll          | 960 +++++++++---------
 .../AArch64/veclib-intrinsic-calls.ll         | 464 ++++-----
 2 files changed, 712 insertions(+), 712 deletions(-)

diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
index 4ad9987b09069..12de7ac0471b8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
@@ -21,28 +21,28 @@ declare float @acosf(float)
 define void @acos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @acos_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_acos(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_acos(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @acos_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @acos_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acos(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @acos_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acos(double [[IN:%.*]])
 ;
   entry:
@@ -66,28 +66,28 @@ define void @acos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @acos_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @acos_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_acosf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_acosf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @acos_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @acos_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acosf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @acos_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acosf(float [[IN:%.*]])
 ;
   entry:
@@ -114,28 +114,28 @@ declare float @acoshf(float)
 define void @acosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @acosh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_acosh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_acosh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @acosh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @acosh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acosh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acosh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acosh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vacoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vacoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svacosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svacosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @acosh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svacosh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svacosh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acosh(double [[IN:%.*]])
 ;
   entry:
@@ -159,28 +159,28 @@ define void @acosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @acosh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @acosh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_acoshf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_acoshf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @acosh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acoshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acoshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @acosh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acoshf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acoshf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acoshf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vacoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vacoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svacosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svacosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @acosh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svacosh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svacosh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acoshf(float [[IN:%.*]])
 ;
   entry:
@@ -207,28 +207,28 @@ declare float @asinf(float)
 define void @asin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @asin_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_asin(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_asin(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @asin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @asin_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asin(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @asin_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asin(double [[IN:%.*]])
 ;
   entry:
@@ -252,28 +252,28 @@ define void @asin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @asin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @asin_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_asinf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_asinf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @asin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @asin_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @asin_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinf(float [[IN:%.*]])
 ;
   entry:
@@ -300,28 +300,28 @@ declare float @asinhf(float)
 define void @asinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @asinh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_asinh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_asinh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @asinh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @asinh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asinh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asinh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asinh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vasinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vasinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svasinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svasinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @asinh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svasinh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svasinh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asinh(double [[IN:%.*]])
 ;
   entry:
@@ -345,28 +345,28 @@ define void @asinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @asinh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @asinh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_asinhf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_asinhf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @asinh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @asinh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinhf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vasinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vasinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svasinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svasinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @asinh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svasinh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svasinh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinhf(float [[IN:%.*]])
 ;
   entry:
@@ -393,28 +393,28 @@ declare float @atanf(float)
 define void @atan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atan_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_atan(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_atan(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @atan_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atan_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atan_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan(double [[IN:%.*]])
 ;
   entry:
@@ -438,28 +438,28 @@ define void @atan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @atan_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atan_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_atanf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_atanf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @atan_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atan_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atan_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanf(float [[IN:%.*]])
 ;
   entry:
@@ -486,28 +486,28 @@ declare float @atan2f(float, float)
 define void @atan2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atan2_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_atan2(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_atan2(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @atan2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atan2_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan2(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atan2_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan2(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -531,28 +531,28 @@ define void @atan2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @atan2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atan2_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_atan2f(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_atan2f(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @atan2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atan2_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atan2f(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atan2_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atan2f(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -579,28 +579,28 @@ declare float @atanhf(float)
 define void @atanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atanh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_atanh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_atanh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @atanh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atanh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atanh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vatanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svatanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atanh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svatanh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svatanh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atanh(double [[IN:%.*]])
 ;
   entry:
@@ -624,28 +624,28 @@ define void @atanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @atanh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atanh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_atanhf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_atanhf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @atanh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atanh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanhf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vatanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svatanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atanh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svatanh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svatanh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanhf(float [[IN:%.*]])
 ;
   entry:
@@ -672,28 +672,28 @@ declare float @cbrtf(float)
 define void @cbrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cbrt_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_cbrt(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cbrt(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cbrt_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cbrt(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cbrt(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cbrt_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cbrt(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cbrt(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cbrt(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cbrt_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vcbrtq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcbrtq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cbrt_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcbrt_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svcbrt_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cbrt_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcbrt_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svcbrt_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cbrt(double [[IN:%.*]])
 ;
   entry:
@@ -717,28 +717,28 @@ define void @cbrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @cbrt_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cbrt_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_cbrtf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_cbrtf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cbrt_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cbrtf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cbrtf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cbrt_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cbrtf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cbrtf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cbrtf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cbrt_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vcbrtq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcbrtq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cbrt_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcbrt_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svcbrt_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cbrt_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcbrt_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svcbrt_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cbrtf(float [[IN:%.*]])
 ;
   entry:
@@ -765,28 +765,28 @@ declare float @copysignf(float, float)
 define void @copysign_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @copysign_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_copysign(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_copysign(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @copysign_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_copysign(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_copysign(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @copysign_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_copysign(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_copysign(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @copysign(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vcopysignq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcopysignq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcopysign_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svcopysign_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @copysign_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcopysign_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svcopysign_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @copysign(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -810,28 +810,28 @@ define void @copysign_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @copysign_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @copysign_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_copysignf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_copysignf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @copysign_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_copysignf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_copysignf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @copysign_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_copysignf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_copysignf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @copysignf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vcopysignq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcopysignq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcopysign_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svcopysign_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @copysign_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcopysign_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svcopysign_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @copysignf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -858,28 +858,28 @@ declare float @cosf(float)
 define void @cos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cos_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_cos(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cos(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cos_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cos_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cos(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vcosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cos_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cos(double [[IN:%.*]])
 ;
   entry:
@@ -903,28 +903,28 @@ define void @cos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @cos_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cos_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_cosf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_cosf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cos_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cos_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cosf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vcosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cos_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cosf(float [[IN:%.*]])
 ;
   entry:
@@ -951,28 +951,28 @@ declare float @coshf(float)
 define void @cosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cosh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_cosh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cosh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cosh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cosh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cosh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cosh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cosh(double [[IN:%.*]])
 ;
   entry:
@@ -996,28 +996,28 @@ define void @cosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @cosh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cosh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_coshf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_coshf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cosh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cosh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @coshf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cosh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @coshf(float [[IN:%.*]])
 ;
   entry:
@@ -1044,28 +1044,28 @@ declare float @cospif(float)
 define void @cospi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cospi_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_cospi(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cospi(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cospi_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cospi(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cospi(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cospi_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cospi(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cospi(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cospi(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cospi_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vcospiq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcospiq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cospi_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcospi_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svcospi_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cospi_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcospi_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svcospi_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cospi(double [[IN:%.*]])
 ;
   entry:
@@ -1089,28 +1089,28 @@ define void @cospi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @cospi_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cospi_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_cospif(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_cospif(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cospi_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cospif(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cospif(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cospi_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cospif(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cospif(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cospif(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cospi_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vcospiq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcospiq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cospi_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcospi_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svcospi_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cospi_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcospi_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svcospi_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cospif(float [[IN:%.*]])
 ;
   entry:
@@ -1137,28 +1137,28 @@ declare float @erff(float)
 define void @erf_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @erf_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_erf(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_erf(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @erf_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erf(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erf(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @erf_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erf(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erf(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erf(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @erf_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_verfq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_verfq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @erf_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_sverf_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_sverf_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @erf_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_sverf_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_sverf_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erf(double [[IN:%.*]])
 ;
   entry:
@@ -1182,28 +1182,28 @@ define void @erf_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @erf_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @erf_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_erff(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_erff(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @erf_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erff(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erff(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @erf_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erff(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erff(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erff(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @erf_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_verfq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_verfq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @erf_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_sverf_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_sverf_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @erf_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_sverf_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_sverf_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erff(float [[IN:%.*]])
 ;
   entry:
@@ -1230,28 +1230,28 @@ declare float @erfcf(float)
 define void @erfc_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @erfc_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_erfc(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_erfc(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @erfc_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erfc(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erfc(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @erfc_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erfc(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erfc(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erfc(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @erfc_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_verfcq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_verfcq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @erfc_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_sverfc_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_sverfc_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @erfc_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_sverfc_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_sverfc_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erfc(double [[IN:%.*]])
 ;
   entry:
@@ -1275,28 +1275,28 @@ define void @erfc_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @erfc_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @erfc_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_erfcf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_erfcf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @erfc_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erfcf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erfcf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @erfc_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erfcf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erfcf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erfcf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @erfc_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_verfcq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_verfcq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @erfc_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_sverfc_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_sverfc_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @erfc_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_sverfc_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_sverfc_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erfcf(float [[IN:%.*]])
 ;
   entry:
@@ -1323,28 +1323,28 @@ declare float @expf(float)
 define void @exp_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_exp(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vexpq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexpq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp(double [[IN:%.*]])
 ;
   entry:
@@ -1368,28 +1368,28 @@ define void @exp_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @exp_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_expf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_expf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vexpq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexpq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expf(float [[IN:%.*]])
 ;
   entry:
@@ -1416,28 +1416,28 @@ declare float @exp10f(float)
 define void @exp10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp10_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_exp10(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp10(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp10_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp10_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp10(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vexp10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexp10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp10_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp10(double [[IN:%.*]])
 ;
   entry:
@@ -1461,28 +1461,28 @@ define void @exp10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @exp10_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp10_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_exp10f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_exp10f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp10_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp10_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp10f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vexp10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexp10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp10_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp10f(float [[IN:%.*]])
 ;
   entry:
@@ -1509,28 +1509,28 @@ declare float @exp2f(float)
 define void @exp2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp2_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_exp2(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp2(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp2_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp2(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vexp2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexp2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp2_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp2(double [[IN:%.*]])
 ;
   entry:
@@ -1554,28 +1554,28 @@ define void @exp2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @exp2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp2_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_exp2f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_exp2f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp2_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp2f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vexp2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexp2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp2_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp2f(float [[IN:%.*]])
 ;
   entry:
@@ -1602,28 +1602,28 @@ declare float @expm1f(float)
 define void @expm1_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @expm1_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_expm1(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_expm1(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @expm1_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_expm1(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_expm1(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @expm1_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_expm1(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_expm1(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @expm1(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @expm1_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vexpm1q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexpm1q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @expm1_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexpm1_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svexpm1_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @expm1_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svexpm1_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svexpm1_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @expm1(double [[IN:%.*]])
 ;
   entry:
@@ -1647,28 +1647,28 @@ define void @expm1_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @expm1_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @expm1_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_expm1f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_expm1f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @expm1_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expm1f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expm1f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @expm1_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expm1f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expm1f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expm1f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @expm1_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vexpm1q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexpm1q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @expm1_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexpm1_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svexpm1_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @expm1_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svexpm1_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svexpm1_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expm1f(float [[IN:%.*]])
 ;
   entry:
@@ -1695,28 +1695,28 @@ declare float @fdimf(float, float)
 define void @fdim_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fdim_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_fdim(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_fdim(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fdim_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fdim(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fdim(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fdim_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fdim(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fdim(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fdim(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fdim_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vfdimq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfdimq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fdim_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfdim_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svfdim_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fdim_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfdim_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svfdim_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fdim(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -1740,28 +1740,28 @@ define void @fdim_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @fdim_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fdim_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_fdimf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_fdimf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fdim_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fdimf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fdimf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fdim_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fdimf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fdimf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fdimf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fdim_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vfdimq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfdimq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fdim_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfdim_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svfdim_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fdim_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfdim_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svfdim_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fdimf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -1788,28 +1788,28 @@ declare float @fmaf(float, float, float)
 define void @fma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fma_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vvv_fma(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vvv_fma(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fma_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvvv_fma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvvv_fma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fma_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvvv_fma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvvv_fma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fma(double [[IN:%.*]], double [[IN]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vfmaq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfmaq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svfma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fma_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svfma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fma(double [[IN:%.*]], double [[IN]], double [[IN]])
 ;
   entry:
@@ -1833,28 +1833,28 @@ define void @fma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @fma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fma_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vvv_fmaf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vvv_fmaf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fma_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvvv_fmaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvvv_fmaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fma_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvvv_fmaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvvv_fmaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaf(float [[IN:%.*]], float [[IN]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vfmaq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfmaq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svfma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fma_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svfma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaf(float [[IN:%.*]], float [[IN]], float [[IN]])
 ;
   entry:
@@ -1881,28 +1881,28 @@ declare float @fmaxf(float, float)
 define void @fmax_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fmax_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_fmax(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_fmax(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fmax_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmax(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmax(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmax_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmax(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmax(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmax(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmax_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vfmaxq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfmaxq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmax_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfmax_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svfmax_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmax_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfmax_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svfmax_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmax(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -1926,28 +1926,28 @@ define void @fmax_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @fmax_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fmax_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_fmaxf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_fmaxf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fmax_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmaxf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmaxf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmax_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmaxf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmaxf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaxf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmax_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vfmaxq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfmaxq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmax_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfmax_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svfmax_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmax_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfmax_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svfmax_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaxf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -1974,28 +1974,28 @@ declare float @fminf(float, float)
 define void @fmin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fmin_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_fmin(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_fmin(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fmin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmin_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmin(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vfminq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfminq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfmin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svfmin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmin_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfmin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svfmin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmin(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -2019,28 +2019,28 @@ define void @fmin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @fmin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fmin_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_fminf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_fminf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fmin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fminf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fminf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmin_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fminf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fminf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fminf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vfminq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfminq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfmin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svfmin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmin_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfmin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svfmin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fminf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -2067,28 +2067,28 @@ declare float @fmodf(float, float)
 define void @fmod_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fmod_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_fmod(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_fmod(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fmod_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmod_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmod(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmod_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vfmodq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfmodq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmod_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfmod_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svfmod_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmod_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfmod_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svfmod_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmod(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -2112,28 +2112,28 @@ define void @fmod_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @fmod_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fmod_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_fmodf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_fmodf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fmod_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmod_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmodf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmod_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vfmodq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfmodq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmod_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfmod_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svfmod_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmod_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfmod_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svfmod_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmodf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -2160,28 +2160,28 @@ declare float @hypotf(float, float)
 define void @hypot_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @hypot_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_hypot(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_hypot(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @hypot_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_hypot(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_hypot(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @hypot_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_hypot(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_hypot(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @hypot(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @hypot_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vhypotq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vhypotq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @hypot_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svhypot_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svhypot_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @hypot_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svhypot_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svhypot_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @hypot(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -2205,28 +2205,28 @@ define void @hypot_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @hypot_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @hypot_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_hypotf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_hypotf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @hypot_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_hypotf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_hypotf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @hypot_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_hypotf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_hypotf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @hypotf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @hypot_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vhypotq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vhypotq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @hypot_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svhypot_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svhypot_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @hypot_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svhypot_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svhypot_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @hypotf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -2253,28 +2253,28 @@ declare i32 @ilogbf(float)
 define void @ilogb_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @ilogb_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x i32> @_ZGVnN2v_ilogb(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x i32> @_ZGVnN2v_ilogb(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @ilogb_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x i32> @_ZGVsMxv_ilogb(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x i32> @_ZGVsMxv_ilogb(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @ilogb_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x i32> @_ZGVsMxv_ilogb(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x i32> @_ZGVsMxv_ilogb(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogb(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ilogb_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x i32> @armpl_vilogbq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x i32> @armpl_vilogbq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ilogb_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x i32> @armpl_svilogb_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x i32> @armpl_svilogb_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @ilogb_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x i32> @armpl_svilogb_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x i32> @armpl_svilogb_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogb(double [[IN:%.*]])
 ;
   entry:
@@ -2298,28 +2298,28 @@ define void @ilogb_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @ilogb_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @ilogb_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x i32> @_ZGVnN4v_ilogbf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x i32> @_ZGVnN4v_ilogbf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @ilogb_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x i32> @_ZGVsMxv_ilogbf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x i32> @_ZGVsMxv_ilogbf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @ilogb_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x i32> @_ZGVsMxv_ilogbf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x i32> @_ZGVsMxv_ilogbf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogbf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ilogb_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x i32> @armpl_vilogbq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x i32> @armpl_vilogbq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ilogb_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x i32> @armpl_svilogb_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x i32> @armpl_svilogb_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @ilogb_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x i32> @armpl_svilogb_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x i32> @armpl_svilogb_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogbf(float [[IN:%.*]])
 ;
   entry:
@@ -2346,28 +2346,28 @@ declare float @ldexpf(float, i32)
 define void @ldexp_f64(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @ldexp_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP5:%.*]] = call <2 x double> @_ZGVnN2vv_ldexp(<2 x double> [[WIDE_LOAD:%.*]], <2 x i32> [[WIDE_LOAD1:%.*]])
+; SLEEF-NEON:    [[TMP4:%.*]] = call <2 x double> @_ZGVnN2vv_ldexp(<2 x double> [[WIDE_LOAD:%.*]], <2 x i32> [[WIDE_LOAD1:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @ldexp_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_ldexp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP14:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_ldexp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @ldexp_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP11:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_ldexp(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_LOAD1:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP10:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_ldexp(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_LOAD1:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @ldexp(double [[IN1:%.*]], i32 [[IN2:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ldexp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP5:%.*]] = call <2 x double> @armpl_vldexpq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x i32> [[WIDE_LOAD1:%.*]])
+; ARMPL-NEON:    [[TMP4:%.*]] = call <2 x double> @armpl_vldexpq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x i32> [[WIDE_LOAD1:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ldexp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svldexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP14:%.*]] = call <vscale x 2 x double> @armpl_svldexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @ldexp_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP11:%.*]] = call <vscale x 2 x double> @armpl_svldexp_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_LOAD1:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP10:%.*]] = call <vscale x 2 x double> @armpl_svldexp_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_LOAD1:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @ldexp(double [[IN1:%.*]], i32 [[IN2:%.*]])
 ;
   entry:
@@ -2393,28 +2393,28 @@ define void @ldexp_f64(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %
 define void @ldexp_f32(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @ldexp_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP5:%.*]] = call <4 x float> @_ZGVnN4vv_ldexpf(<4 x float> [[WIDE_LOAD:%.*]], <4 x i32> [[WIDE_LOAD1:%.*]])
+; SLEEF-NEON:    [[TMP4:%.*]] = call <4 x float> @_ZGVnN4vv_ldexpf(<4 x float> [[WIDE_LOAD:%.*]], <4 x i32> [[WIDE_LOAD1:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @ldexp_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_ldexpf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP14:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_ldexpf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @ldexp_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP11:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_ldexpf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_LOAD1:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP10:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_ldexpf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_LOAD1:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @ldexpf(float [[IN1:%.*]], i32 [[IN2:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ldexp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP5:%.*]] = call <4 x float> @armpl_vldexpq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x i32> [[WIDE_LOAD1:%.*]])
+; ARMPL-NEON:    [[TMP4:%.*]] = call <4 x float> @armpl_vldexpq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x i32> [[WIDE_LOAD1:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ldexp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svldexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP14:%.*]] = call <vscale x 4 x float> @armpl_svldexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @ldexp_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP11:%.*]] = call <vscale x 4 x float> @armpl_svldexp_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_LOAD1:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP10:%.*]] = call <vscale x 4 x float> @armpl_svldexp_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_LOAD1:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @ldexpf(float [[IN1:%.*]], i32 [[IN2:%.*]])
 ;
   entry:
@@ -2443,28 +2443,28 @@ declare float @lgammaf(float)
 define void @lgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @lgamma_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_lgamma(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_lgamma(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @lgamma_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @lgamma_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @lgamma(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @lgamma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vlgammaq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlgammaq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @lgamma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlgamma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svlgamma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @lgamma_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlgamma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svlgamma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @lgamma(double [[IN:%.*]])
 ;
   entry:
@@ -2488,28 +2488,28 @@ define void @lgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @lgamma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @lgamma_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_lgammaf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_lgammaf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @lgamma_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @lgamma_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @lgammaf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @lgamma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vlgammaq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlgammaq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @lgamma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlgamma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svlgamma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @lgamma_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlgamma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svlgamma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @lgammaf(float [[IN:%.*]])
 ;
   entry:
@@ -2536,28 +2536,28 @@ declare float @logf(float)
 define void @log_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_log(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vlogq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlogq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log(double [[IN:%.*]])
 ;
   entry:
@@ -2581,28 +2581,28 @@ define void @log_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @log_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_logf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_logf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @logf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vlogq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlogq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @logf(float [[IN:%.*]])
 ;
   entry:
@@ -2629,28 +2629,28 @@ declare float @log10f(float)
 define void @log10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log10_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_log10(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log10(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log10_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log10_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log10(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vlog10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log10_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log10(double [[IN:%.*]])
 ;
   entry:
@@ -2674,28 +2674,28 @@ define void @log10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @log10_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log10_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_log10f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log10f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log10_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log10_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log10f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vlog10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log10_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log10f(float [[IN:%.*]])
 ;
   entry:
@@ -2722,28 +2722,28 @@ declare float @log1pf(float)
 define void @log1p_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log1p_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_log1p(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log1p(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log1p_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log1p(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log1p(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log1p_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log1p(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log1p(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log1p(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log1p_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vlog1pq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog1pq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log1p_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog1p_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svlog1p_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log1p_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlog1p_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svlog1p_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log1p(double [[IN:%.*]])
 ;
   entry:
@@ -2767,28 +2767,28 @@ define void @log1p_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @log1p_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log1p_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_log1pf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log1pf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log1p_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log1pf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log1pf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log1p_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log1pf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log1pf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log1pf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log1p_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vlog1pq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog1pq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log1p_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog1p_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svlog1p_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log1p_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlog1p_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svlog1p_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log1pf(float [[IN:%.*]])
 ;
   entry:
@@ -2815,28 +2815,28 @@ declare float @log2f(float)
 define void @log2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log2_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_log2(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log2(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log2_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log2(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vlog2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log2_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log2(double [[IN:%.*]])
 ;
   entry:
@@ -2860,28 +2860,28 @@ define void @log2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @log2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log2_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_log2f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log2f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log2_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log2f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vlog2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log2_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log2f(float [[IN:%.*]])
 ;
   entry:
@@ -3002,28 +3002,28 @@ declare float @nextafterf(float, float)
 define void @nextafter_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @nextafter_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_nextafter(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_nextafter(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @nextafter_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_nextafter(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_nextafter(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @nextafter_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_nextafter(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_nextafter(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @nextafter(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @nextafter_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vnextafterq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vnextafterq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @nextafter_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svnextafter_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svnextafter_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @nextafter_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svnextafter_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svnextafter_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @nextafter(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -3047,28 +3047,28 @@ define void @nextafter_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @nextafter_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @nextafter_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_nextafterf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_nextafterf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @nextafter_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_nextafterf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_nextafterf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @nextafter_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_nextafterf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_nextafterf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @nextafterf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @nextafter_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vnextafterq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vnextafterq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @nextafter_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svnextafter_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svnextafter_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @nextafter_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svnextafter_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svnextafter_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @nextafterf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -3095,28 +3095,28 @@ declare float @powf(float, float)
 define void @pow_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @pow_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_pow(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_pow(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @pow_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @pow_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @pow(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vpowq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vpowq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @pow_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @pow(double [[IN:%.*]], double [[IN]])
 ;
   entry:
@@ -3140,28 +3140,28 @@ define void @pow_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @pow_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @pow_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_powf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_powf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @pow_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @pow_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @powf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vpowq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vpowq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @pow_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @powf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
@@ -3188,28 +3188,28 @@ declare float @sinf(float)
 define void @sin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sin_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_sin(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sin(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sin_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sin(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vsinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sin_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sin(double [[IN:%.*]])
 ;
   entry:
@@ -3233,28 +3233,28 @@ define void @sin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @sin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sin_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_sinf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sinf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sin_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vsinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sin_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinf(float [[IN:%.*]])
 ;
   entry:
@@ -3465,28 +3465,28 @@ declare float @sinhf(float)
 define void @sinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sinh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_sinh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sinh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sinh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sinh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sinh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinh(double [[IN:%.*]])
 ;
   entry:
@@ -3510,28 +3510,28 @@ define void @sinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @sinh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sinh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_sinhf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sinhf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sinh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sinh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinhf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sinh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinhf(float [[IN:%.*]])
 ;
   entry:
@@ -3558,28 +3558,28 @@ declare float @sinpif(float)
 define void @sinpi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sinpi_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_sinpi(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sinpi(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sinpi_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinpi(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinpi(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sinpi_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinpi(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinpi(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinpi(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinpi_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vsinpiq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinpiq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinpi_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsinpi_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svsinpi_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sinpi_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svsinpi_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svsinpi_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinpi(double [[IN:%.*]])
 ;
   entry:
@@ -3603,28 +3603,28 @@ define void @sinpi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @sinpi_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sinpi_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_sinpif(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sinpif(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sinpi_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinpif(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinpif(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sinpi_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinpif(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinpif(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinpif(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinpi_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vsinpiq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinpiq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinpi_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsinpi_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svsinpi_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sinpi_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svsinpi_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svsinpi_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinpif(float [[IN:%.*]])
 ;
   entry:
@@ -3651,28 +3651,28 @@ declare float @sqrtf(float)
 define void @sqrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sqrt_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_sqrt(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sqrt(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sqrt_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sqrt_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sqrt(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vsqrtq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsqrtq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsqrt_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svsqrt_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sqrt_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svsqrt_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svsqrt_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sqrt(double [[IN:%.*]])
 ;
   entry:
@@ -3696,28 +3696,28 @@ define void @sqrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @sqrt_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sqrt_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_sqrtf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sqrtf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sqrt_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sqrt_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sqrtf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vsqrtq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsqrtq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsqrt_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svsqrt_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sqrt_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svsqrt_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svsqrt_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sqrtf(float [[IN:%.*]])
 ;
   entry:
@@ -3744,28 +3744,28 @@ declare float @tanf(float)
 define void @tan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tan_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_tan(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_tan(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tan_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tan_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tan(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tan_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tan(double [[IN:%.*]])
 ;
   entry:
@@ -3789,28 +3789,28 @@ define void @tan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @tan_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tan_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_tanf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_tanf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tan_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tan_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tan_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanf(float [[IN:%.*]])
 ;
   entry:
@@ -3837,28 +3837,28 @@ declare float @tanhf(float)
 define void @tanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tanh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_tanh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_tanh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tanh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tanh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tanh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tanh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tanh(double [[IN:%.*]])
 ;
   entry:
@@ -3882,28 +3882,28 @@ define void @tanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @tanh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tanh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_tanhf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_tanhf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tanh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tanh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanhf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tanh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanhf(float [[IN:%.*]])
 ;
   entry:
@@ -3930,28 +3930,28 @@ declare float @tgammaf(float)
 define void @tgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tgamma_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_tgamma(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_tgamma(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tgamma_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tgamma_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tgamma(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tgamma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vtgammaq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtgammaq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tgamma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtgamma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svtgamma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tgamma_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svtgamma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 2 x double> @armpl_svtgamma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tgamma(double [[IN:%.*]])
 ;
   entry:
@@ -3975,28 +3975,28 @@ define void @tgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 define void @tgamma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tgamma_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_tgammaf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_tgammaf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tgamma_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tgamma_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; SLEEF-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tgammaf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tgamma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vtgammaq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtgammaq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tgamma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtgamma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svtgamma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tgamma_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svtgamma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
+; ARMPL-SVE-NOPRED:    [[TMP8:%.*]] = call <vscale x 4 x float> @armpl_svtgamma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> splat (i1 true))
 ; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tgammaf(float [[IN:%.*]])
 ;
   entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
index 30c53bf0177e6..d0a98bddf08ab 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
@@ -18,19 +18,19 @@ declare float @llvm.acos.f32(float)
 define void @acos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @acos_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_acos(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_acos(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @acos_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -53,19 +53,19 @@ define void @acos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @acos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @acos_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_acosf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_acosf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @acos_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -91,19 +91,19 @@ declare float @llvm.asin.f32(float)
 define void @asin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @asin_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_asin(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_asin(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @asin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -126,19 +126,19 @@ define void @asin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @asin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @asin_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_asinf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_asinf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @asin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -164,19 +164,19 @@ declare float @llvm.atan.f32(float)
 define void @atan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atan_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_atan(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_atan(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @atan_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -199,19 +199,19 @@ define void @atan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @atan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atan_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_atanf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_atanf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @atan_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -237,19 +237,19 @@ declare float @llvm.atan2.f32(float, float)
 define void @atan2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atan2_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_atan2(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_atan2(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @atan2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -272,19 +272,19 @@ define void @atan2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @atan2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @atan2_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_atan2f(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_atan2f(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @atan2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -310,19 +310,19 @@ declare float @llvm.ceil.f32(float)
 define void @ceil_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @ceil_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @ceil_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ceil_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ceil_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -345,19 +345,19 @@ define void @ceil_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @ceil_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @ceil_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @ceil_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ceil_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ceil_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -383,19 +383,19 @@ declare float @llvm.copysign.f32(float, float)
 define void @copysign_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @copysign_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @copysign_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -418,19 +418,19 @@ define void @copysign_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @copysign_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @copysign_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @copysign_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -456,19 +456,19 @@ declare float @llvm.cos.f32(float)
 define void @cos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cos_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_cos(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cos(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cos_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vcosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -491,19 +491,19 @@ define void @cos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @cos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cos_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_cosf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_cosf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cos_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vcosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -529,19 +529,19 @@ declare float @llvm.cosh.f32(float)
 define void @cosh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cosh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_cosh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cosh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cosh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -564,19 +564,19 @@ define void @cosh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @cosh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @cosh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_coshf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_coshf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @cosh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -602,19 +602,19 @@ declare float @llvm.exp.f32(float)
 define void @exp_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_exp(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vexpq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexpq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -637,19 +637,19 @@ define void @exp_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @exp_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_expf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_expf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vexpq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexpq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -675,19 +675,19 @@ declare float @llvm.exp10.f32(float)
 define void @exp10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp10_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_exp10(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp10(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp10_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vexp10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexp10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -710,19 +710,19 @@ define void @exp10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @exp10_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp10_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_exp10f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_exp10f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp10_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vexp10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexp10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -748,19 +748,19 @@ declare float @llvm.exp2.f32(float)
 define void @exp2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp2_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_exp2(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp2(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vexp2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexp2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -783,19 +783,19 @@ define void @exp2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @exp2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @exp2_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_exp2f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_exp2f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @exp2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vexp2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexp2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -821,19 +821,19 @@ declare float @llvm.fabs.f32(float)
 define void @fabs_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fabs_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @fabs_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @fabs_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @fabs_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -856,19 +856,19 @@ define void @fabs_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @fabs_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fabs_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @fabs_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @fabs_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @fabs_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -894,19 +894,19 @@ declare float @llvm.floor.f32(float)
 define void @floor_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @floor_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @floor_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @floor_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @floor_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -929,19 +929,19 @@ define void @floor_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @floor_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @floor_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @floor_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @floor_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @floor_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -967,19 +967,19 @@ declare float @llvm.fma.f32(float, float, float)
 define void @fma_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fma_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fma_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1002,19 +1002,19 @@ define void @fma_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @fma_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @fma_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @fma_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1040,19 +1040,19 @@ declare float @llvm.log.f32(float)
 define void @log_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_log(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vlogq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlogq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1075,19 +1075,19 @@ define void @log_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @log_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_logf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_logf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vlogq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlogq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1113,19 +1113,19 @@ declare float @llvm.log10.f32(float)
 define void @log10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log10_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_log10(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log10(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log10_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vlog10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1148,19 +1148,19 @@ define void @log10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @log10_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log10_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_log10f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log10f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log10_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vlog10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1186,19 +1186,19 @@ declare float @llvm.log2.f32(float)
 define void @log2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log2_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_log2(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log2(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vlog2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1221,19 +1221,19 @@ define void @log2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @log2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @log2_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_log2f(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log2f(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @log2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vlog2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1259,19 +1259,19 @@ declare float @llvm.maxnum.f32(float, float)
 define void @maxnum_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @maxnum_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.maxnum.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.maxnum.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @maxnum_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @maxnum_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.maxnum.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.maxnum.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @maxnum_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1294,19 +1294,19 @@ define void @maxnum_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @maxnum_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @maxnum_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @maxnum_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @maxnum_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @maxnum_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1332,19 +1332,19 @@ declare float @llvm.minnum.f32(float, float)
 define void @minnum_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @minnum_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @minnum_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @minnum_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @minnum_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1367,19 +1367,19 @@ define void @minnum_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @minnum_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @minnum_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @minnum_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @minnum_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @minnum_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1405,19 +1405,19 @@ declare float @llvm.nearbyint.f32(float)
 define void @nearbyint_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @nearbyint_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @nearbyint_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @nearbyint_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @nearbyint_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1440,19 +1440,19 @@ define void @nearbyint_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @nearbyint_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @nearbyint_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @nearbyint_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @nearbyint_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @nearbyint_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1478,19 +1478,19 @@ declare float @llvm.pow.f32(float, float)
 define void @pow_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @pow_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2vv_pow(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_pow(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @pow_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vpowq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vpowq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1513,19 +1513,19 @@ define void @pow_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @pow_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @pow_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4vv_powf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_powf(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; SLEEF-SVE-LABEL: define void @pow_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vpowq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vpowq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1551,19 +1551,19 @@ declare float @llvm.rint.f32(float)
 define void @rint_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @rint_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @rint_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @rint_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @rint_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1586,19 +1586,19 @@ define void @rint_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @rint_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @rint_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @rint_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @rint_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @rint_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1624,19 +1624,19 @@ declare float @llvm.round.f32(float)
 define void @round_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @round_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @round_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @round_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @round_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1659,19 +1659,19 @@ define void @round_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @round_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @round_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.round.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.round.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @round_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @round_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.round.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.round.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @round_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1697,19 +1697,19 @@ declare float @llvm.sin.f32(float)
 define void @sin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sin_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_sin(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sin(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vsinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1732,19 +1732,19 @@ define void @sin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @sin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sin_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_sinf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sinf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vsinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1770,19 +1770,19 @@ declare float @llvm.sinh.f32(float)
 define void @sinh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sinh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_sinh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sinh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sinh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1805,19 +1805,19 @@ define void @sinh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @sinh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sinh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_sinhf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sinhf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sinh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1843,19 +1843,19 @@ declare float @llvm.sqrt.f32(float)
 define void @sqrt_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sqrt_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sqrt_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1878,19 +1878,19 @@ define void @sqrt_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @sqrt_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @sqrt_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @sqrt_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1916,19 +1916,19 @@ declare float @llvm.tan.f32(float)
 define void @tan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tan_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_tan(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_tan(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tan_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1951,19 +1951,19 @@ define void @tan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @tan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tan_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_tanf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_tanf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tan_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1989,19 +1989,19 @@ declare float @llvm.tanh.f32(float)
 define void @tanh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tanh_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @_ZGVnN2v_tanh(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_tanh(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tanh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -2024,19 +2024,19 @@ define void @tanh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @tanh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @tanh_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @_ZGVnN4v_tanhf(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_tanhf(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @tanh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -2062,19 +2062,19 @@ declare float @llvm.trunc.f32(float)
 define void @trunc_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @trunc_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @trunc_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @trunc_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @trunc_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -2097,19 +2097,19 @@ define void @trunc_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 define void @trunc_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ; SLEEF-NEON-LABEL: define void @trunc_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; SLEEF-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; SLEEF-SVE-LABEL: define void @trunc_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @trunc_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP3:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @trunc_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body

From bd0b903a91721570e0bc98f7ed8b0aaf2f2a628b Mon Sep 17 00:00:00 2001
From: Zibi Sarbinowski <zibi@ca.ibm.com>
Date: Thu, 17 Apr 2025 10:08:57 -0400
Subject: [PATCH 269/710] Mark the file opened by
 DeserializedDeclsSourceRangePrinter as a text file (#135842)

This PR will fix the following lit failure seeing on z/OS and most likely on Windows:

`FAIL: Clang :: Frontend/dump-minimization-hints.cpp`

Without `OF_TextWithCRLF` flag, a file is treated as binary and is read improperly, at least on z/OS.
---
 clang/lib/Frontend/FrontendAction.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp
index bd084aa94fc37..1c4dec08575d1 100644
--- a/clang/lib/Frontend/FrontendAction.cpp
+++ b/clang/lib/Frontend/FrontendAction.cpp
@@ -312,7 +312,7 @@ FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI,
     std::error_code ErrorCode;
     auto FileStream = std::make_unique<llvm::raw_fd_ostream>(
         DumpDeserializedDeclarationRangesPath, ErrorCode,
-        llvm::sys::fs::OF_None);
+        llvm::sys::fs::OF_TextWithCRLF);
     if (!ErrorCode) {
       Consumers.push_back(std::make_unique<DeserializedDeclsSourceRangePrinter>(
           CI.getSourceManager(), std::move(FileStream)));

From 728f6de4177a7e4d8030cb37ace525e2af97d247 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 17 Apr 2025 14:13:21 +0000
Subject: [PATCH 270/710] [gn build] Port 0348ff515854

---
 llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
index 85ad5303a9246..54199b3fdbc21 100644
--- a/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
@@ -90,6 +90,7 @@ static_library("CodeGen") {
     "CodeGenFunction.cpp",
     "CodeGenModule.cpp",
     "CodeGenPGO.cpp",
+    "CodeGenSYCL.cpp",
     "CodeGenTBAA.cpp",
     "CodeGenTypes.cpp",
     "ConstantInitBuilder.cpp",

From 9ee4fdf4990f7dbdc0cfc30a8f733f676dbdcb51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Thu, 17 Apr 2025 07:25:48 -0700
Subject: [PATCH 271/710] [flang][cuda] Introduce stream cast op (#136050)

Cast a stream object reference as a GPU async token. This is useful to
be able to connect the stream representation of CUDA Fortran and the
async mechanism of the GPU dialect.
This op will later become a no op.
---
 .../flang/Optimizer/Dialect/CUF/CUFOps.td     | 22 +++++++++++++++++++
 .../include/flang/Optimizer/Support/InitFIR.h |  2 +-
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    | 11 ++++++++++
 flang/test/Fir/CUDA/cuda-stream.mlir          | 21 ++++++++++++++++++
 flang/tools/fir-opt/fir-opt.cpp               |  2 --
 5 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Fir/CUDA/cuda-stream.mlir

diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
index feef5485194f8..f55f3e8a4466d 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
@@ -18,6 +18,7 @@ include "flang/Optimizer/Dialect/CUF/CUFDialect.td"
 include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td"
 include "flang/Optimizer/Dialect/FIRTypes.td"
 include "flang/Optimizer/Dialect/FIRAttr.td"
+include "mlir/Dialect/GPU/IR/GPUBase.td"
 include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
 include "mlir/Interfaces/LoopLikeInterface.td"
 include "mlir/IR/BuiltinAttributes.td"
@@ -370,4 +371,25 @@ def cuf_SharedMemoryOp
       CArg<"llvm::ArrayRef<mlir::NamedAttribute>", "{}">:$attributes)>];
 }
 
+def cuf_StreamCastOp : cuf_Op<"stream_cast", [NoMemoryEffect]> {
+  let summary = "Adapt a stream value to a GPU async token";
+
+  let description = [{
+    Cast a stream object reference as a GPU async token. This is useful to be
+    able to connect the stream representation of CUDA Fortran and the async
+    mechanism of the GPU dialect.
+    Later in the lowering this will become a no op.
+  }];
+
+  let arguments = (ins fir_ReferenceType:$stream);
+
+  let results = (outs GPU_AsyncToken:$token);
+
+  let assemblyFormat = [{
+    $stream attr-dict `:` type($stream)
+  }];
+
+  let hasVerifier = 1;
+}
+
 #endif // FORTRAN_DIALECT_CUF_CUF_OPS
diff --git a/flang/include/flang/Optimizer/Support/InitFIR.h b/flang/include/flang/Optimizer/Support/InitFIR.h
index 4c57e01c28c93..1868fbb201970 100644
--- a/flang/include/flang/Optimizer/Support/InitFIR.h
+++ b/flang/include/flang/Optimizer/Support/InitFIR.h
@@ -40,7 +40,7 @@ namespace fir::support {
       mlir::cf::ControlFlowDialect, mlir::func::FuncDialect,                   \
       mlir::vector::VectorDialect, mlir::math::MathDialect,                    \
       mlir::complex::ComplexDialect, mlir::DLTIDialect, cuf::CUFDialect,       \
-      mlir::NVVM::NVVMDialect
+      mlir::NVVM::NVVMDialect, mlir::gpu::GPUDialect
 
 #define FLANG_CODEGEN_DIALECT_LIST FIRCodeGenDialect, mlir::LLVM::LLVMDialect
 
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index 957e4c01fb4a1..ce197d48d4860 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -319,6 +319,17 @@ void cuf::SharedMemoryOp::build(
   result.addAttributes(attributes);
 }
 
+//===----------------------------------------------------------------------===//
+// StreamCastOp
+//===----------------------------------------------------------------------===//
+
+llvm::LogicalResult cuf::StreamCastOp::verify() {
+  auto refTy = mlir::dyn_cast<fir::ReferenceType>(getStream().getType());
+  if (!refTy.getEleTy().isInteger(64))
+    return emitOpError("stream is expected to be a i64 reference");
+  return mlir::success();
+}
+
 // Tablegen operators
 
 #define GET_OP_CLASSES
diff --git a/flang/test/Fir/CUDA/cuda-stream.mlir b/flang/test/Fir/CUDA/cuda-stream.mlir
new file mode 100644
index 0000000000000..50f230467854b
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-stream.mlir
@@ -0,0 +1,21 @@
+// RUN: fir-opt --split-input-file %s | FileCheck %s
+
+module attributes {gpu.container_module} {
+  gpu.module @cuda_device_mod {
+    gpu.func @_QMmod1Psub1() kernel {
+      gpu.return
+    }
+  }
+  func.func @_QMmod1Phost_sub() {
+    %0 = fir.alloca i64
+    %1 = arith.constant 1 : index
+    %asyncTok = cuf.stream_cast %0 : !fir.ref<i64>
+    gpu.launch_func [%asyncTok] @cuda_device_mod::@_QMmod1Psub1 blocks in (%1, %1, %1) threads in (%1, %1, %1) args() {cuf.proc_attr = #cuf.cuda_proc<grid_global>}
+    return
+  }
+}
+
+// CHECK-LABEL: func.func @_QMmod1Phost_sub()
+// CHECK: %[[STREAM:.*]] = fir.alloca i64
+// CHECK: %[[TOKEN:.*]] = cuf.stream_cast %[[STREAM]] : <i64>
+// CHECK: gpu.launch_func [%[[TOKEN]]] @cuda_device_mod::@_QMmod1Psub1
diff --git a/flang/tools/fir-opt/fir-opt.cpp b/flang/tools/fir-opt/fir-opt.cpp
index ef510ff77ad25..d66fc3f08bdf8 100644
--- a/flang/tools/fir-opt/fir-opt.cpp
+++ b/flang/tools/fir-opt/fir-opt.cpp
@@ -44,8 +44,6 @@ int main(int argc, char **argv) {
 #endif
   DialectRegistry registry;
   fir::support::registerDialects(registry);
-  registry.insert<mlir::gpu::GPUDialect>();
-  registry.insert<mlir::NVVM::NVVMDialect>();
   fir::support::addFIRExtensions(registry);
   return failed(MlirOptMain(argc, argv, "FIR modular optimizer driver\n",
       registry));

From 10f75b8ef7edc80759b9e916e72e17b1c2e9abcb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Csan=C3=A1d=20Hajd=C3=BA?= <csanad.hajdu@arm.com>
Date: Thu, 17 Apr 2025 16:26:11 +0200
Subject: [PATCH 272/710] [LLD][ELF][AArch64] Mark .plt and .iplt with PURECODE
 flag (#134798)

Mark the synthetic sections `.plt` and `.iplt` with the
`SHF_AARCH64_PURECODE` section flag, allowing them to be placed in an
executable-only segment.
---
 lld/ELF/SyntheticSections.cpp           |  10 +++
 lld/test/ELF/aarch64-execute-only-plt.s |  35 ++++++++
 lld/test/ELF/aarch64-gnu-ifunc-plt.s    |  36 +++++----
 lld/test/ELF/aarch64-plt.s              | 103 ++++++------------------
 4 files changed, 90 insertions(+), 94 deletions(-)
 create mode 100644 lld/test/ELF/aarch64-execute-only-plt.s

diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 3bb9815336a7c..196ecc4fee8c8 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -2594,6 +2594,11 @@ PltSection::PltSection(Ctx &ctx)
     : SyntheticSection(ctx, ".plt", SHT_PROGBITS, SHF_ALLOC | SHF_EXECINSTR,
                        16),
       headerSize(ctx.target->pltHeaderSize) {
+  // On AArch64, PLT entries only do loads from the .got.plt section, so the
+  // .plt section can be marked with the SHF_AARCH64_PURECODE section flag.
+  if (ctx.arg.emachine == EM_AARCH64)
+    this->flags |= SHF_AARCH64_PURECODE;
+
   // On PowerPC, this section contains lazy symbol resolvers.
   if (ctx.arg.emachine == EM_PPC64) {
     name = ".glink";
@@ -2654,6 +2659,11 @@ void PltSection::addSymbols() {
 IpltSection::IpltSection(Ctx &ctx)
     : SyntheticSection(ctx, ".iplt", SHT_PROGBITS, SHF_ALLOC | SHF_EXECINSTR,
                        16) {
+  // On AArch64, PLT entries only do loads from the .got.plt section, so the
+  // .iplt section can be marked with the SHF_AARCH64_PURECODE section flag.
+  if (ctx.arg.emachine == EM_AARCH64)
+    this->flags |= SHF_AARCH64_PURECODE;
+
   if (ctx.arg.emachine == EM_PPC || ctx.arg.emachine == EM_PPC64) {
     name = ".glink";
     addralign = 4;
diff --git a/lld/test/ELF/aarch64-execute-only-plt.s b/lld/test/ELF/aarch64-execute-only-plt.s
new file mode 100644
index 0000000000000..b74df4cf0cdb1
--- /dev/null
+++ b/lld/test/ELF/aarch64-execute-only-plt.s
@@ -0,0 +1,35 @@
+// REQUIRES: aarch64
+
+// RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu %s -o %t.o
+// RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu %S/Inputs/plt-aarch64.s -o %t2.o
+// RUN: ld.lld -shared %t2.o -soname=t2.so -o %t2.so
+// RUN: ld.lld %t.o %t2.so -o %t
+// RUN: llvm-readelf -S -l %t | FileCheck %s
+
+// CHECK-LABEL: Section Headers:
+// CHECK:         .text PROGBITS 00000000002102a0 0002a0 000010 00 AXy  0   0  4
+// CHECK:         .plt  PROGBITS 00000000002102b0 0002b0 000030 00 AXy  0   0 16
+// CHECK:         .iplt PROGBITS 00000000002102e0 0002e0 000010 00 AXy  0   0 16
+
+// CHECK-LABEL: Program Headers:
+// CHECK:         PHDR
+// CHECK-NEXT:    LOAD
+// CHECK-NEXT:    LOAD 0x0002a0 0x00000000002102a0 0x00000000002102a0 0x000050 0x000050   E 0x10000
+
+// CHECK-LABEL: Section to Segment mapping:
+/// Index should match the index of the LOAD segment above.
+// CHECK:         02   .text .plt .iplt
+
+.global bar
+
+.section .text,"axy",@progbits,unique,0
+.global _start
+_start:
+  bl foo
+  bl bar
+  ret
+
+.globl foo
+.type foo STT_GNU_IFUNC
+foo:
+  ret
diff --git a/lld/test/ELF/aarch64-gnu-ifunc-plt.s b/lld/test/ELF/aarch64-gnu-ifunc-plt.s
index bde6f1420540f..73ecf58ee76bf 100644
--- a/lld/test/ELF/aarch64-gnu-ifunc-plt.s
+++ b/lld/test/ELF/aarch64-gnu-ifunc-plt.s
@@ -5,7 +5,7 @@
 // RUN: ld.lld --hash-style=sysv %t.so %t.o -o %tout
 // RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn %tout | FileCheck %s --check-prefix=DISASM
 // RUN: llvm-objdump -s %tout | FileCheck %s --check-prefix=GOTPLT
-// RUN: llvm-readobj --dynamic-table -r %tout | FileCheck %s
+// RUN: llvm-readelf -S --dynamic-table -r %tout | FileCheck %s
 
 // RUN: llvm-mc -filetype=obj -triple=aarch64_be %S/Inputs/shared2.s -o %t1.be.o
 // RUN: ld.lld %t1.be.o --shared --soname=t.so -o %t.be.so
@@ -13,24 +13,26 @@
 // RUN: ld.lld --hash-style=sysv %t.be.so %t.be.o -o %t.be
 // RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn %t.be | FileCheck %s --check-prefix=DISASM
 // RUN: llvm-objdump -s %t.be | FileCheck %s --check-prefix=GOTPLT_BE
-// RUN: llvm-readobj --dynamic-table -r %t.be | FileCheck %s
+// RUN: llvm-readelf -S --dynamic-table -r %t.be | FileCheck %s
 
-// Check that the PLTRELSZ tag does not include the IRELATIVE relocations
-// CHECK: DynamicSection [
-// CHECK:   0x0000000000000008 RELASZ               48 (bytes)
-// CHECK:   0x0000000000000002 PLTRELSZ             48 (bytes)
+// CHECK-LABEL: Section Headers:
+///       Name  Type     Address          Off    Size   ES Flg Lk Inf Al
+// CHECK: .iplt PROGBITS 0000000000210330 000330 000020 00 AXy  0   0 16
 
-// Check that the IRELATIVE relocations are after the JUMP_SLOT in the plt
-// CHECK: Relocations [
-// CHECK-NEXT:   Section (4) .rela.dyn {
-// CHECK-NEXT:     0x230468 R_AARCH64_IRELATIVE - 0x2102D8
-// CHECK-NEXT:     0x230470 R_AARCH64_IRELATIVE - 0x2102DC
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Section (5) .rela.plt {
-// CHECK-NEXT:     0x230458 R_AARCH64_JUMP_SLOT bar2 0x0
-// CHECK-NEXT:     0x230460 R_AARCH64_JUMP_SLOT zed2 0x0
-// CHECK-NEXT:   }
-// CHECK-NEXT: ]
+/// Check that the PLTRELSZ tag does not include the IRELATIVE relocations
+// CHECK-LABEL: Dynamic section at offset 0x350 contains 15 entries:
+// CHECK: 0x0000000000000008 (RELASZ)   48 (bytes)
+// CHECK: 0x0000000000000002 (PLTRELSZ) 48 (bytes)
+
+/// Check that the IRELATIVE relocations are after the JUMP_SLOT in the plt
+// CHECK-LABEL: Relocation section '.rela.dyn' at offset 0x278 contains 2 entries:
+// CHECK-NEXT:      Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+// CHECK-NEXT:  0000000000230468  0000000000000408 R_AARCH64_IRELATIVE                     2102d8
+// CHECK-NEXT:  0000000000230470  0000000000000408 R_AARCH64_IRELATIVE                     2102dc
+// CHECK-LABEL: Relocation section '.rela.plt' at offset 0x2a8 contains 2 entries:
+// CHECK-NEXT:      Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+// CHECK-NEXT:  0000000000230458  0000000100000402 R_AARCH64_JUMP_SLOT    0000000000000000 bar2 + 0
+// CHECK-NEXT:  0000000000230460  0000000200000402 R_AARCH64_JUMP_SLOT    0000000000000000 zed2 + 0
 
 // Check that .got.plt entries point back to PLT header
 // GOTPLT: Contents of section .got.plt:
diff --git a/lld/test/ELF/aarch64-plt.s b/lld/test/ELF/aarch64-plt.s
index 7d83a40dfc278..420df721b8f93 100644
--- a/lld/test/ELF/aarch64-plt.s
+++ b/lld/test/ELF/aarch64-plt.s
@@ -4,52 +4,26 @@
 // RUN: ld.lld -shared %t2.o -soname=t2.so -o %t2.so
 // RUN: ld.lld -shared %t.o %t2.so -o %t.so
 // RUN: ld.lld %t.o %t2.so -o %t.exe
-// RUN: llvm-readobj -S -r %t.so | FileCheck --check-prefix=CHECKDSO %s
+// RUN: llvm-readelf -S -r %t.so | FileCheck --check-prefix=CHECKDSO %s
 // RUN: llvm-objdump -s --section=.got.plt %t.so | FileCheck --check-prefix=DUMPDSO %s
 // RUN: llvm-objdump -d --no-show-raw-insn --print-imm-hex %t.so | FileCheck --check-prefix=DISASMDSO %s
-// RUN: llvm-readobj -S -r %t.exe | FileCheck --check-prefix=CHECKEXE %s
+// RUN: llvm-readelf -S -r %t.exe | FileCheck --check-prefix=CHECKEXE %s
 // RUN: llvm-objdump -s --section=.got.plt %t.exe | FileCheck --check-prefix=DUMPEXE %s
 // RUN: llvm-objdump -d --no-show-raw-insn --print-imm-hex %t.exe | FileCheck --check-prefix=DISASMEXE %s
 
-// CHECKDSO:     Name: .plt
-// CHECKDSO-NEXT:     Type: SHT_PROGBITS
-// CHECKDSO-NEXT:     Flags [
-// CHECKDSO-NEXT:       SHF_ALLOC
-// CHECKDSO-NEXT:       SHF_EXECINSTR
-// CHECKDSO-NEXT:     ]
-// CHECKDSO-NEXT:     Address: 0x10340
-// CHECKDSO-NEXT:     Offset:
-// CHECKDSO-NEXT:     Size: 80
-// CHECKDSO-NEXT:     Link:
-// CHECKDSO-NEXT:     Info:
-// CHECKDSO-NEXT:     AddressAlignment: 16
-
-// CHECKDSO:     Name: .got.plt
-// CHECKDSO-NEXT:     Type: SHT_PROGBITS
-// CHECKDSO-NEXT:     Flags [
-// CHECKDSO-NEXT:       SHF_ALLOC
-// CHECKDSO-NEXT:       SHF_WRITE
-// CHECKDSO-NEXT:     ]
-// CHECKDSO-NEXT:     Address: 0x30450
-// CHECKDSO-NEXT:     Offset:
-// CHECKDSO-NEXT:     Size: 48
-// CHECKDSO-NEXT:     Link:
-// CHECKDSO-NEXT:     Info:
-// CHECKDSO-NEXT:     AddressAlignment: 8
-
-// CHECKDSO: Relocations [
-// CHECKDSO-NEXT:   Section ({{.*}}) .rela.plt {
-
-// &(.got.plt[3]) = 0x30450 + 3 * 8 = 0x30468
-// CHECKDSO-NEXT:     0x30468 R_AARCH64_JUMP_SLOT foo
-
-// &(.got.plt[4]) = 0x30450 + 4 * 8 = 0x30470
-// CHECKDSO-NEXT:     0x30470 R_AARCH64_JUMP_SLOT bar
-
-// &(.got.plt[5]) = 0x30000 + 5 * 8 = 0x30470
-// CHECKDSO-NEXT:     0x30478 R_AARCH64_JUMP_SLOT weak
-// CHECKDSO-NEXT:   }
-// CHECKDSO-NEXT: ]
+// CHECKDSO-LABEL: Section Headers:
+///          Name     Type     Address          Off    Size   ES Flg Lk Inf Al
+// CHECKDSO: .plt     PROGBITS 0000000000010340 000340 000050 00 AXy  0   0 16
+// CHECKDSO: .got.plt PROGBITS 0000000000030450 000450 000030 00  WA  0   0  8
+
+// CHECKDSO-LABEL: Relocation section '.rela.plt' at offset 0x2e8 contains 3 entries:
+// CHECKDSO-NEXT:      Offset             Info             Type            Symbol's Value  Symbol's Name + Addend
+/// &(.got.plt[3]) = 0x30450 + 3 * 8 = 0x30468
+// CHECKDSO-NEXT:  0000000000030468  0000000400000402 R_AARCH64_JUMP_SLOT 000000000001033c foo + 0
+/// &(.got.plt[4]) = 0x30450 + 4 * 8 = 0x30470
+// CHECKDSO-NEXT:  0000000000030470  0000000100000402 R_AARCH64_JUMP_SLOT 0000000000000000 bar + 0
+/// &(.got.plt[5]) = 0x30000 + 5 * 8 = 0x30478
+// CHECKDSO-NEXT:  0000000000030478  0000000200000402 R_AARCH64_JUMP_SLOT 0000000000000000 weak + 0
 
 // DUMPDSO: Contents of section .got.plt:
 // .got.plt[0..2] = 0 (reserved)
@@ -106,42 +80,17 @@
 // DISASMDSO-NEXT:     10388: add     x16, x16, #0x478
 // DISASMDSO-NEXT:     1038c: br      x17
 
-// CHECKEXE:     Name: .plt
-// CHECKEXE-NEXT:     Type: SHT_PROGBITS
-// CHECKEXE-NEXT:     Flags [
-// CHECKEXE-NEXT:       SHF_ALLOC
-// CHECKEXE-NEXT:       SHF_EXECINSTR
-// CHECKEXE-NEXT:     ]
-// CHECKEXE-NEXT:     Address: 0x2102E0
-// CHECKEXE-NEXT:     Offset:
-// CHECKEXE-NEXT:     Size: 64
-// CHECKEXE-NEXT:     Link:
-// CHECKEXE-NEXT:     Info:
-// CHECKEXE-NEXT:     AddressAlignment: 16
-
-// CHECKEXE:     Name: .got.plt
-// CHECKEXE-NEXT:     Type: SHT_PROGBITS
-// CHECKEXE-NEXT:     Flags [
-// CHECKEXE-NEXT:       SHF_ALLOC
-// CHECKEXE-NEXT:       SHF_WRITE
-// CHECKEXE-NEXT:     ]
-// CHECKEXE-NEXT:     Address: 0x2303F0
-// CHECKEXE-NEXT:     Offset:
-// CHECKEXE-NEXT:     Size: 40
-// CHECKEXE-NEXT:     Link:
-// CHECKEXE-NEXT:     Info:
-// CHECKEXE-NEXT:     AddressAlignment: 8
-
-// CHECKEXE: Relocations [
-// CHECKEXE-NEXT:   Section ({{.*}}) .rela.plt {
-
-// &(.got.plt[3]) = 0x2303f0 + 3 * 8 = 0x230408
-// CHECKEXE-NEXT:     0x230408 R_AARCH64_JUMP_SLOT bar 0x0
-
-// &(.got.plt[4]) = 0x2303f0 + 4 * 8 = 0x230410
-// CHECKEXE-NEXT:     0x230410 R_AARCH64_JUMP_SLOT weak 0x0
-// CHECKEXE-NEXT:   }
-// CHECKEXE-NEXT: ]
+// CHECKEXE-LABEL: Section Headers:
+///          Name     Type     Address          Off    Size   ES Flg Lk Inf Al
+// CHECKEXE: .plt     PROGBITS 00000000002102e0 0002e0 000040 00 AXy  0   0 16
+// CHECKEXE: .got.plt PROGBITS 00000000002303f0 0003f0 000028 00  WA  0   0  8
+
+// CHECKEXE-LABEL: Relocation section '.rela.plt' at offset 0x298 contains 2 entries:
+// CHECKEXE-NEXT:      Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+/// &(.got.plt[3]) = 0x2303f0 + 3 * 8 = 0x230408
+// CHECKEXE-NEXT:  0000000000230408  0000000100000402 R_AARCH64_JUMP_SLOT    0000000000000000 bar + 0
+/// &(.got.plt[4]) = 0x2303f0 + 4 * 8 = 0x230410
+// CHECKEXE-NEXT:  0000000000230410  0000000200000402 R_AARCH64_JUMP_SLOT    0000000000000000 weak + 0
 
 // DUMPEXE: Contents of section .got.plt:
 // .got.plt[0..2] = 0 (reserved)

From 45f27167bcd312a8691365a63719fab136972692 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Csan=C3=A1d=20Hajd=C3=BA?= <csanad.hajdu@arm.com>
Date: Thu, 17 Apr 2025 16:27:09 +0200
Subject: [PATCH 273/710] [AArch64][Docs] Add release note for execute-only
 support on AArch64 (#134799)

---
 llvm/docs/ReleaseNotes.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 526d6b4002bba..48c4663d63410 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -85,6 +85,14 @@ Changes to Interprocedural Optimizations
 Changes to the AArch64 Backend
 ------------------------------
 
+* Added the `execute-only` target feature, which indicates that the generated
+  program code doesn't contain any inline data, and there are no data accesses
+  to code sections. On ELF targets this property is indicated by the
+  `SHF_AARCH64_PURECODE` section flag.
+  ([#125687](https://github.com/llvm/llvm-project/pull/125687),
+  [#132196](https://github.com/llvm/llvm-project/pull/132196),
+  [#133084](https://github.com/llvm/llvm-project/pull/133084))
+
 Changes to the AMDGPU Backend
 -----------------------------
 

From 41675fa5b88dc703b3b1d96de668281582f21f3b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Thu, 17 Apr 2025 16:31:14 +0200
Subject: [PATCH 274/710] [VPlan] Simplify vp.merge true, (or x, y), x ->
 vp.merge y, true, x (#135017)

With EVL tail folding an AnyOf reduction will emit an i1 vp.merge like

vp.merge true, (or phi, cond), phi, evl

We can remove the or and optimise this to

vp.merge cond, true, phi, evl

Which makes it slightly easier to pattern match in #134898.

This also adds a pattern matcher for calls to help match this.

Blended AnyOf reductions will use an and instead of an or, which we may
also be able to simplify in a later patch.
---
 .../Transforms/Vectorize/VPlanPatternMatch.h  | 118 ++++++++++++++++++
 .../Transforms/Vectorize/VPlanTransforms.cpp  |  12 ++
 ...ze-force-tail-with-evl-inloop-reduction.ll |   6 +-
 ...vectorize-force-tail-with-evl-reduction.ll |   6 +-
 4 files changed, 134 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 7934c47ee5ba3..f38eb3cc8d43f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -166,11 +166,29 @@ template <typename LTy, typename RTy> struct match_combine_or {
   }
 };
 
+template <typename LTy, typename RTy> struct match_combine_and {
+  LTy L;
+  RTy R;
+
+  match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
+
+  template <typename ITy> bool match(ITy *V) const {
+    return L.match(V) && R.match(V);
+  }
+};
+
+/// Combine two pattern matchers matching L || R
 template <typename LTy, typename RTy>
 inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) {
   return match_combine_or<LTy, RTy>(L, R);
 }
 
+/// Combine two pattern matchers matching L && R
+template <typename LTy, typename RTy>
+inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) {
+  return match_combine_and<LTy, RTy>(L, R);
+}
+
 /// Match a VPValue, capturing it if we match.
 inline bind_ty<VPValue> m_VPValue(VPValue *&V) { return V; }
 
@@ -469,6 +487,106 @@ m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
   return VPDerivedIV_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2});
 }
 
+/// Match a call argument at a given argument index.
+template <typename Opnd_t> struct Argument_match {
+  /// Call argument index to match.
+  unsigned OpI;
+  Opnd_t Val;
+
+  Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
+
+  template <typename OpTy> bool match(OpTy *V) const {
+    if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))
+      return Val.match(R->getOperand(OpI));
+    if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))
+      return Val.match(R->getOperand(OpI));
+    if (const auto *R = dyn_cast<VPReplicateRecipe>(V))
+      if (isa<CallInst>(R->getUnderlyingInstr()))
+        return Val.match(R->getOperand(OpI + 1));
+    return false;
+  }
+};
+
+/// Match a call argument.
+template <unsigned OpI, typename Opnd_t>
+inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
+  return Argument_match<Opnd_t>(OpI, Op);
+}
+
+/// Intrinsic matchers.
+struct IntrinsicID_match {
+  unsigned ID;
+
+  IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
+
+  template <typename OpTy> bool match(OpTy *V) const {
+    if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))
+      return R->getVectorIntrinsicID() == ID;
+    if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))
+      return R->getCalledScalarFunction()->getIntrinsicID() == ID;
+    if (const auto *R = dyn_cast<VPReplicateRecipe>(V))
+      if (const auto *CI = dyn_cast<CallInst>(R->getUnderlyingInstr()))
+        if (const auto *F = CI->getCalledFunction())
+          return F->getIntrinsicID() == ID;
+    return false;
+  }
+};
+
+/// Intrinsic matches are combinations of ID matchers, and argument
+/// matchers. Higher arity matcher are defined recursively in terms of and-ing
+/// them with lower arity matchers. Here's some convenient typedefs for up to
+/// several arguments, and more can be added as needed
+template <typename T0 = void, typename T1 = void, typename T2 = void,
+          typename T3 = void>
+struct m_Intrinsic_Ty;
+template <typename T0> struct m_Intrinsic_Ty<T0> {
+  using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;
+};
+template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> {
+  using Ty =
+      match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>;
+};
+template <typename T0, typename T1, typename T2>
+struct m_Intrinsic_Ty<T0, T1, T2> {
+  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty,
+                               Argument_match<T2>>;
+};
+template <typename T0, typename T1, typename T2, typename T3>
+struct m_Intrinsic_Ty {
+  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,
+                               Argument_match<T3>>;
+};
+
+/// Match intrinsic calls like this:
+/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
+template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() {
+  return IntrinsicID_match(IntrID);
+}
+
+template <Intrinsic::ID IntrID, typename T0>
+inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) {
+  return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0));
+}
+
+template <Intrinsic::ID IntrID, typename T0, typename T1>
+inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0,
+                                                       const T1 &Op1) {
+  return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1));
+}
+
+template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2>
+inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty
+m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) {
+  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2));
+}
+
+template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
+          typename T3>
+inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty
+m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
+  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
+}
+
 } // namespace VPlanPatternMatch
 } // namespace llvm
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d0cb1c3ac590f..9d8f1706cf61b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1026,6 +1026,18 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
     if (TypeInfo.inferScalarType(X) != WideStepTy)
       X = VPBuilder(&R).createWidenCast(Instruction::Trunc, X, WideStepTy);
     R.getVPSingleValue()->replaceAllUsesWith(X);
+    return;
+  }
+
+  // For i1 vp.merges produced by AnyOf reductions:
+  // vp.merge true, (or x, y), x, evl -> vp.merge y, true, x, evl
+  if (match(&R, m_Intrinsic<Intrinsic::vp_merge>(m_True(), m_VPValue(A),
+                                                 m_VPValue(X), m_VPValue())) &&
+      match(A, m_c_BinaryOr(m_Specific(X), m_VPValue(Y))) &&
+      TypeInfo.inferScalarType(R.getVPSingleValue())->isIntegerTy(1)) {
+    R.setOperand(1, R.getOperand(0));
+    R.setOperand(0, Y);
+    return;
   }
 }
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
index 1fb84dbc79c49..990ac0bdb72ad 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
@@ -1899,8 +1899,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 3)
-; IF-EVL-NEXT:    [[TMP15:%.*]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP14]]
-; IF-EVL-NEXT:    [[TMP16]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP10]])
+; IF-EVL-NEXT:    [[TMP16]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP10]])
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP10]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
@@ -2024,8 +2023,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = fcmp fast olt <vscale x 4 x float> [[VP_OP_LOAD]], splat (float 3.000000e+00)
-; IF-EVL-NEXT:    [[TMP15:%.*]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP14]]
-; IF-EVL-NEXT:    [[TMP16]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP10]])
+; IF-EVL-NEXT:    [[TMP16]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP10]])
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP10]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
index a9a9bf7294cca..7660d06a35d43 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
@@ -1953,8 +1953,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 3)
-; IF-EVL-NEXT:    [[TMP14:%.*]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP13]]
-; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
@@ -2078,8 +2077,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = fcmp fast olt <vscale x 4 x float> [[VP_OP_LOAD]], splat (float 3.000000e+00)
-; IF-EVL-NEXT:    [[TMP14:%.*]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP13]]
-; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]

From 8d4b19e3d4a8cee8899d5a293972aef48f9bdb7f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 17 Apr 2025 07:34:38 -0700
Subject: [PATCH 275/710] [RISCV] Remove skipFunction from
 RISCVVMV0Elimination. (#136080)

skipFunction disables the pass for functions with optnone attribute.
This pass is important to correct register allocation so I don't think
it should be disabled.
---
 llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp b/llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp
index 9270a5b98a142..6dbae5bf525db 100644
--- a/llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp
@@ -79,9 +79,6 @@ static bool isVMV0(const MCOperandInfo &MCOI) {
 }
 
 bool RISCVVMV0Elimination::runOnMachineFunction(MachineFunction &MF) {
-  if (skipFunction(MF.getFunction()))
-    return false;
-
   // Skip if the vector extension is not enabled.
   const RISCVSubtarget *ST = &MF.getSubtarget<RISCVSubtarget>();
   if (!ST->hasVInstructions())

From 427a7792b15d28a7962e42b5f677bfde9383e975 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Csan=C3=A1d=20Hajd=C3=BA?= <csanad.hajdu@arm.com>
Date: Thu, 17 Apr 2025 16:36:02 +0200
Subject: [PATCH 276/710] [LLD][Docs] Add release notes for execute-only
 support on AArch64 (#134804)

---
 lld/docs/ReleaseNotes.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 36028514cba27..87466e49cb8d5 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -47,6 +47,17 @@ ELF Improvements
   default value is ``--no-xosegment``.
   (`#132412 <https://github.com/llvm/llvm-project/pull/132412>`_)
 
+* For AArch64, added support for the ``SHF_AARCH64_PURECODE`` section flag,
+  which indicates that the section only contains program code and no data.
+  An output section will only have this flag set if all input sections also
+  have it set. (`#125689 <https://github.com/llvm/llvm-project/pull/125689>`_,
+  `#134798 <https://github.com/llvm/llvm-project/pull/134798>`_)
+
+* For AArch64 and ARM, added ``-zexecute-only-report``, which checks for
+  missing ``SHF_AARCH64_PURECODE`` and ``SHF_ARM_PURECODE`` section flags
+  on executable sections.
+  (`#128883 <https://github.com/llvm/llvm-project/pull/128883>`_)
+
 Breaking changes
 ----------------
 * Executable-only and readable-executable sections are now allowed to be placed

From 7daa5010ab6a2ae77ac88ffd84e2cc37a2e11faa Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Thu, 17 Apr 2025 07:37:24 -0700
Subject: [PATCH 277/710] [NVPTX] Cleanup and document nvvm.fabs intrinsics,
 adding f16 support (#135644)

This change unifies the NVVM intrinsics for floating point absolute
value into two new overloaded intrinsics "llvm.nvvm.fabs.*" and
"llvm.nvvm.fabs.ftz.*". Documentation has been added specifying the
semantics of these intrinsics to clarify how they differ from
"llvm.fabs.*". In addition, support for these new intrinsics is
extended to cover the f16 variants.
---
 clang/include/clang/Basic/BuiltinsNVPTX.td    |   5 +
 clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp    |  15 ++
 .../CodeGen/builtins-nvptx-native-half-type.c |  41 +++--
 clang/test/CodeGen/builtins-nvptx.c           |  12 +-
 llvm/docs/NVPTXUsage.rst                      |  53 ++++++
 llvm/include/llvm/IR/IntrinsicsNVVM.td        |  14 +-
 llvm/lib/IR/AutoUpgrade.cpp                   |  16 +-
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td       |  19 +-
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td      | 162 +++++++++---------
 .../Target/NVPTX/NVPTXTargetTransformInfo.cpp |   2 -
 llvm/test/CodeGen/NVPTX/fabs-intrinsics.ll    | 142 +++++++++++++++
 .../math-intrins-sm80-ptx70-autoupgrade.ll    |   3 +
 .../InstCombine/NVPTX/nvvm-intrins.ll         |   2 +-
 13 files changed, 368 insertions(+), 118 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/fabs-intrinsics.ll

diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.td b/clang/include/clang/Basic/BuiltinsNVPTX.td
index bdbdfa2cea6c6..f797e29fe66a3 100644
--- a/clang/include/clang/Basic/BuiltinsNVPTX.td
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.td
@@ -321,6 +321,11 @@ def __nvvm_fabs_ftz_f : NVPTXBuiltin<"float(float)">;
 def __nvvm_fabs_f : NVPTXBuiltin<"float(float)">;
 def __nvvm_fabs_d : NVPTXBuiltin<"double(double)">;
 
+def __nvvm_fabs_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16)", SM_53, PTX65>;
+def __nvvm_fabs_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>)", SM_53, PTX65>;
+def __nvvm_fabs_ftz_f16 : NVPTXBuiltinSMAndPTX<"__fp16(__fp16)", SM_53, PTX65>;
+def __nvvm_fabs_ftz_f16x2 : NVPTXBuiltinSMAndPTX<"_Vector<2, __fp16>(_Vector<2, __fp16>)", SM_53, PTX65>;
+
 // Round
 
 def __nvvm_round_ftz_f : NVPTXBuiltin<"float(float)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index 0f7ab9fd3b099..002af4f931c09 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -1034,6 +1034,21 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
   case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
     return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
                         *this);
+  case NVPTX::BI__nvvm_fabs_f:
+  case NVPTX::BI__nvvm_abs_bf16:
+  case NVPTX::BI__nvvm_abs_bf16x2:
+  case NVPTX::BI__nvvm_fabs_f16:
+  case NVPTX::BI__nvvm_fabs_f16x2:
+    return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs,
+                                        EmitScalarExpr(E->getArg(0)));
+  case NVPTX::BI__nvvm_fabs_ftz_f:
+  case NVPTX::BI__nvvm_fabs_ftz_f16:
+  case NVPTX::BI__nvvm_fabs_ftz_f16x2:
+    return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs_ftz,
+                                        EmitScalarExpr(E->getArg(0)));
+  case NVPTX::BI__nvvm_fabs_d:
+    return Builder.CreateUnaryIntrinsic(Intrinsic::fabs,
+                                        EmitScalarExpr(E->getArg(0)));
   case NVPTX::BI__nvvm_ldg_h:
   case NVPTX::BI__nvvm_ldg_h2:
     return MakeHalfType(Intrinsic::not_intrinsic, BuiltinID, E, *this);
diff --git a/clang/test/CodeGen/builtins-nvptx-native-half-type.c b/clang/test/CodeGen/builtins-nvptx-native-half-type.c
index 511497702ff7f..01a004efd71e4 100644
--- a/clang/test/CodeGen/builtins-nvptx-native-half-type.c
+++ b/clang/test/CodeGen/builtins-nvptx-native-half-type.c
@@ -26,14 +26,14 @@
 // RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX72_SM86 %s
 
 // RUN: %clang_cc1 -ffp-contract=off -triple nvptx-unknown-unknown -target-cpu \
-// RUN:   sm_53 -target-feature +ptx42 -fcuda-is-device -fnative-half-type \
+// RUN:   sm_53 -target-feature +ptx65 -fcuda-is-device -fnative-half-type \
 // RUN:   -emit-llvm -o - -x cuda %s \
-// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX42_SM53 %s
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX65_SM53 %s
 
 // RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown \
-// RUN:   -target-cpu sm_53 -target-feature +ptx42 -fcuda-is-device \
+// RUN:   -target-cpu sm_53 -target-feature +ptx65 -fcuda-is-device \
 // RUN:   -fnative-half-type -emit-llvm -o - -x cuda %s \
-// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX42_SM53 %s
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX65_SM53 %s
 
 #define __device__ __attribute__((device))
 
@@ -108,25 +108,25 @@ __device__ void nvvm_fma_f16_f16x2_sm80() {
 // CHECK-LABEL: nvvm_fma_f16_f16x2_sm53
 __device__ void nvvm_fma_f16_f16x2_sm53() {
 #if __CUDA_ARCH__ >= 530
-  // CHECK_PTX42_SM53: call half @llvm.nvvm.fma.rn.f16
+  // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.rn.f16
   __nvvm_fma_rn_f16(0.1f16, 0.1f16, 0.1f16);
-  // CHECK_PTX42_SM53: call half @llvm.nvvm.fma.rn.ftz.f16
+  // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.rn.ftz.f16
   __nvvm_fma_rn_ftz_f16(0.1f16, 0.1f16, 0.1f16);
-  // CHECK_PTX42_SM53: call half @llvm.nvvm.fma.rn.sat.f16
+  // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.rn.sat.f16
   __nvvm_fma_rn_sat_f16(0.1f16, 0.1f16, 0.1f16);
-  // CHECK_PTX42_SM53: call half @llvm.nvvm.fma.rn.ftz.sat.f16
+  // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.rn.ftz.sat.f16
   __nvvm_fma_rn_ftz_sat_f16(0.1f16, 0.1f16, 0.1f16);
 
-  // CHECK_PTX42_SM53: call <2 x half> @llvm.nvvm.fma.rn.f16x2
+  // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.rn.f16x2
   __nvvm_fma_rn_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
                       {0.1f16, 0.7f16});
-  // CHECK_PTX42_SM53: call <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2
+  // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2
   __nvvm_fma_rn_ftz_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
                           {0.1f16, 0.7f16});
-  // CHECK_PTX42_SM53: call <2 x half> @llvm.nvvm.fma.rn.sat.f16x2
+  // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.rn.sat.f16x2
   __nvvm_fma_rn_sat_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
                           {0.1f16, 0.7f16});
-  // CHECK_PTX42_SM53: call <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2
+  // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2
   __nvvm_fma_rn_ftz_sat_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
                               {0.1f16, 0.7f16});
 #endif
@@ -173,6 +173,23 @@ __device__ void nvvm_min_max_sm86() {
   // CHECK: ret void
 }
 
+// CHECK-LABEL: nvvm_fabs_f16
+__device__ void nvvm_fabs_f16() {
+#if __CUDA_ARCH__ >= 530
+  // CHECK: call half @llvm.nvvm.fabs.f16
+  __nvvm_fabs_f16(0.1f16);
+  // CHECK: call half @llvm.nvvm.fabs.ftz.f16
+  __nvvm_fabs_ftz_f16(0.1f16);
+  // CHECK: call <2 x half> @llvm.nvvm.fabs.v2f16
+  __nvvm_fabs_f16x2({0.1f16, 0.7f16});
+  // CHECK: call <2 x half> @llvm.nvvm.fabs.ftz.v2f16
+  __nvvm_fabs_ftz_f16x2({0.1f16, 0.7f16});
+#endif
+  // CHECK: ret void
+}
+
+
+
 typedef __fp16 __fp16v2 __attribute__((ext_vector_type(2)));
 
 // CHECK-LABEL: nvvm_ldg_native_half_types
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 7404ce01c535c..639c18190f436 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -245,6 +245,14 @@ __device__ void nvvm_math(float f1, float f2, double d1, double d2) {
 // CHECK: call double @llvm.nvvm.rcp.rn.d
   double td4 = __nvvm_rcp_rn_d(d2);
 
+// CHECK: call float @llvm.nvvm.fabs.f32
+  float t6 = __nvvm_fabs_f(f1);
+// CHECK: call float @llvm.nvvm.fabs.ftz.f32
+  float t7 = __nvvm_fabs_ftz_f(f2);
+
+// CHECK: call double @llvm.fabs.f64
+  double td5 = __nvvm_fabs_d(d1);
+
 // CHECK: call void @llvm.nvvm.membar.cta()
   __nvvm_membar_cta();
 // CHECK: call void @llvm.nvvm.membar.gl()
@@ -1181,9 +1189,9 @@ __device__ void nvvm_cvt_sm100a_sm101a_sm120a() {
 __device__ void nvvm_abs_neg_bf16_bf16x2_sm80() {
 #if __CUDA_ARCH__ >= 800
 
-  // CHECK_PTX70_SM80: call bfloat @llvm.nvvm.abs.bf16(bfloat 0xR3DCD)
+  // CHECK_PTX70_SM80: call bfloat @llvm.nvvm.fabs.bf16(bfloat 0xR3DCD)
   __nvvm_abs_bf16(BF16);
-  // CHECK_PTX70_SM80: call <2 x bfloat> @llvm.nvvm.abs.bf16x2(<2 x bfloat> splat (bfloat 0xR3DCD))
+  // CHECK_PTX70_SM80: call <2 x bfloat> @llvm.nvvm.fabs.v2bf16(<2 x bfloat> splat (bfloat 0xR3DCD))
   __nvvm_abs_bf16x2(BF16X2);
 
   // CHECK_PTX70_SM80: call bfloat @llvm.nvvm.neg.bf16(bfloat 0xR3DCD)
diff --git a/llvm/docs/NVPTXUsage.rst b/llvm/docs/NVPTXUsage.rst
index 621879fc5648b..ef6e888286def 100644
--- a/llvm/docs/NVPTXUsage.rst
+++ b/llvm/docs/NVPTXUsage.rst
@@ -309,6 +309,59 @@ space casted to this space), 1 is returned, otherwise 0 is returned.
 Arithmetic Intrinsics
 ---------------------
 
+'``llvm.nvvm.fabs.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+    declare float @llvm.nvvm.fabs.f32(float %a)
+    declare double @llvm.nvvm.fabs.f64(double %a)
+    declare half @llvm.nvvm.fabs.f16(half %a)
+    declare <2 x half> @llvm.nvvm.fabs.v2f16(<2 x half> %a)
+    declare bfloat @llvm.nvvm.fabs.bf16(bfloat %a)
+    declare <2 x bfloat> @llvm.nvvm.fabs.v2bf16(<2 x bfloat> %a)
+
+Overview:
+"""""""""
+
+The '``llvm.nvvm.fabs.*``' intrinsics return the absolute value of the operand.
+
+Semantics:
+""""""""""
+
+Unlike, '``llvm.fabs.*``', these intrinsics do not perfectly preserve NaN
+values. Instead, a NaN input yeilds an unspecified NaN output.
+
+
+'``llvm.nvvm.fabs.ftz.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+    declare float @llvm.nvvm.fabs.ftz.f32(float %a)
+    declare half @llvm.nvvm.fabs.ftz.f16(half %a)
+    declare <2 x half> @llvm.nvvm.fabs.ftz.v2f16(<2 x half> %a)
+
+Overview:
+"""""""""
+
+The '``llvm.nvvm.fabs.ftz.*``' intrinsics return the absolute value of the
+operand, flushing subnormals to sign preserving zero.
+
+Semantics:
+""""""""""
+
+Before the absolute value is taken, the input is flushed to sign preserving
+zero if it is a subnormal. In addtion, unlike '``llvm.fabs.*``', a NaN input
+yields an unspecified NaN output.
+
+
 '``llvm.nvvm.idp2a.[us].[us]``' Intrinsics
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 7af051d22f9b7..93895048faaf3 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1039,18 +1039,18 @@ let TargetPrefix = "nvvm" in {
 // Abs
 //
 
-  def int_nvvm_fabs_ftz_f : ClangBuiltin<"__nvvm_fabs_ftz_f">,
-      DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>;
-  def int_nvvm_fabs_f : ClangBuiltin<"__nvvm_fabs_f">,
-      DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>;
-  def int_nvvm_fabs_d : ClangBuiltin<"__nvvm_fabs_d">,
-      DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty], [IntrNoMem, IntrSpeculatable]>;
+  def int_nvvm_fabs_ftz :
+      DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>],
+                            [IntrNoMem, IntrSpeculatable]>;
 
+  def int_nvvm_fabs :
+      DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>],
+                            [IntrNoMem, IntrSpeculatable]>;
 //
 // Abs, Neg bf16, bf16x2
 //
 
-  foreach unary = ["abs", "neg"] in {
+  foreach unary = ["neg"] in {
     def int_nvvm_ # unary # _bf16 :
       ClangBuiltin<!strconcat("__nvvm_", unary, "_bf16")>,
       DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_bfloat_ty], [IntrNoMem]>;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 699191ceb8d4d..7a91da5f9f7eb 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -939,12 +939,6 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
 }
 
 static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
-  if (Name.consume_front("abs."))
-    return StringSwitch<Intrinsic::ID>(Name)
-        .Case("bf16", Intrinsic::nvvm_abs_bf16)
-        .Case("bf16x2", Intrinsic::nvvm_abs_bf16x2)
-        .Default(Intrinsic::not_intrinsic);
-
   if (Name.consume_front("fma.rn."))
     return StringSwitch<Intrinsic::ID>(Name)
         .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
@@ -1291,7 +1285,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
       bool Expand = false;
       if (Name.consume_front("abs."))
         // nvvm.abs.{i,ii}
-        Expand = Name == "i" || Name == "ll";
+        Expand =
+            Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
       else if (Name == "clz.ll" || Name == "popc.ll" || Name == "h2f" ||
                Name == "swap.lo.hi.b64")
         Expand = true;
@@ -2316,6 +2311,13 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
     Value *Cmp = Builder.CreateICmpSGE(
         Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
     Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
+  } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
+    Type *Ty = (Name == "abs.bf16")
+                   ? Builder.getBFloatTy()
+                   : FixedVectorType::get(Builder.getBFloatTy(), 2);
+    Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
+    Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
+    Rep = Builder.CreateBitCast(Abs, CI->getType());
   } else if (Name.starts_with("atomic.load.add.f32.p") ||
              Name.starts_with("atomic.load.add.f64.p")) {
     Value *Ptr = CI->getArgOperand(0);
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 46549d850cf0a..ee6380a8a89c4 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -226,14 +226,17 @@ class RegTyInfo<ValueType ty, NVPTXRegClass rc, Operand imm, SDNode imm_node,
   int Size = ty.Size;
 }
 
-def I16RT : RegTyInfo<i16, Int16Regs, i16imm, imm>;
-def I32RT : RegTyInfo<i32, Int32Regs, i32imm, imm>;
-def I64RT : RegTyInfo<i64, Int64Regs, i64imm, imm>;
-
-def F32RT : RegTyInfo<f32, Float32Regs, f32imm, fpimm>;
-def F64RT : RegTyInfo<f64, Float64Regs, f64imm, fpimm>;
-def F16RT : RegTyInfo<f16, Int16Regs, f16imm, fpimm, supports_imm = 0>;
-def BF16RT : RegTyInfo<bf16, Int16Regs, bf16imm, fpimm, supports_imm = 0>;
+def I16RT    : RegTyInfo<i16, Int16Regs, i16imm, imm>;
+def I32RT    : RegTyInfo<i32, Int32Regs, i32imm, imm>;
+def I64RT    : RegTyInfo<i64, Int64Regs, i64imm, imm>;
+
+def F32RT    : RegTyInfo<f32, Float32Regs, f32imm, fpimm>;
+def F64RT    : RegTyInfo<f64, Float64Regs, f64imm, fpimm>;
+def F16RT    : RegTyInfo<f16, Int16Regs, f16imm, fpimm, supports_imm = 0>;
+def BF16RT   : RegTyInfo<bf16, Int16Regs, bf16imm, fpimm, supports_imm = 0>;
+
+def F16X2RT  : RegTyInfo<v2f16, Int32Regs, ?, ?, supports_imm = 0>;
+def BF16X2RT : RegTyInfo<v2bf16, Int32Regs, ?, ?, supports_imm = 0>;
 
 
 // Template for instructions which take three int64, int32, or int16 args.
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index b5d3c9a05ec34..9634ad8eb8da2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -983,12 +983,13 @@ def : Pat<(int_nvvm_fmin_d
 
 // We need a full string for OpcStr here because we need to deal with case like
 // INT_PTX_RECIP.
-class F_MATH_1<string OpcStr, NVPTXRegClass target_regclass,
-  NVPTXRegClass src_regclass, Intrinsic IntOP, list<Predicate> Preds = []>
-            : NVPTXInst<(outs target_regclass:$dst), (ins src_regclass:$src0),
-            OpcStr,
-        [(set target_regclass:$dst, (IntOP src_regclass:$src0))]>,
-        Requires<Preds>;
+class F_MATH_1<string OpcStr, RegTyInfo dst, RegTyInfo src, Intrinsic IntOP,
+               list<Predicate> Preds = []>
+  : NVPTXInst<(outs dst.RC:$dst),
+              (ins src.RC:$src0),
+              OpcStr,
+              [(set dst.Ty:$dst, (IntOP src.Ty:$src0))]>,
+    Requires<Preds>;
 
 // We need a full string for OpcStr here because we need to deal with the case
 // like INT_PTX_NATIVE_POWR_F.
@@ -1307,13 +1308,20 @@ def : Pat<(int_nvvm_ceil_d f64:$a),
 // Abs
 //
 
-def INT_NVVM_FABS_FTZ_F : F_MATH_1<"abs.ftz.f32 \t$dst, $src0;", Float32Regs,
-  Float32Regs, int_nvvm_fabs_ftz_f>;
-def INT_NVVM_FABS_F : F_MATH_1<"abs.f32 \t$dst, $src0;", Float32Regs,
-  Float32Regs, int_nvvm_fabs_f>;
+multiclass F_ABS<string suffix, RegTyInfo RT, bit support_ftz, list<Predicate> preds = []> {
+  def "" : F_MATH_1<"abs." # suffix # " \t$dst, $src0;", RT, RT, int_nvvm_fabs, preds>;
+  if support_ftz then
+    def _FTZ : F_MATH_1<"abs.ftz." # suffix # " \t$dst, $src0;", RT, RT, int_nvvm_fabs_ftz, preds>;
+}
+
+defm ABS_F16 : F_ABS<"f16", F16RT, support_ftz = true, preds = [hasPTX<65>, hasSM<53>]>;
+defm ABS_F16X2 : F_ABS<"f16x2", F16X2RT, support_ftz = true, preds = [hasPTX<65>, hasSM<53>]>;
+
+defm ABS_BF16 : F_ABS<"bf16", BF16RT, support_ftz = false, preds = [hasPTX<70>, hasSM<80>]>;
+defm ABS_BF16X2 : F_ABS<"bf16x2", BF16X2RT, support_ftz = false, preds = [hasPTX<70>, hasSM<80>]>;
 
-def INT_NVVM_FABS_D : F_MATH_1<"abs.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_fabs_d>;
+defm ABS_F32 : F_ABS<"f32", F32RT, support_ftz = true>;
+defm ABS_F64 : F_ABS<"f64", F64RT, support_ftz = false>;
 
 //
 // copysign
@@ -1332,17 +1340,13 @@ def COPYSIGN_D :
               [(set f64:$dst, (fcopysign_nvptx f64:$src1, f64:$src0))]>;
 
 //
-// Abs, Neg bf16, bf16x2
+// Neg bf16, bf16x2
 //
 
-def INT_NVVM_ABS_BF16 : F_MATH_1<"abs.bf16 \t$dst, $src0;", Int16Regs,
-  Int16Regs, int_nvvm_abs_bf16, [hasPTX<70>, hasSM<80>]>;
-def INT_NVVM_ABS_BF16X2 : F_MATH_1<"abs.bf16x2 \t$dst, $src0;", Int32Regs,
-  Int32Regs, int_nvvm_abs_bf16x2, [hasPTX<70>, hasSM<80>]>;
-def INT_NVVM_NEG_BF16 : F_MATH_1<"neg.bf16 \t$dst, $src0;", Int16Regs,
-  Int16Regs, int_nvvm_neg_bf16, [hasPTX<70>, hasSM<80>]>;
-def INT_NVVM_NEG_BF16X2 : F_MATH_1<"neg.bf16x2 \t$dst, $src0;", Int32Regs,
-  Int32Regs, int_nvvm_neg_bf16x2, [hasPTX<70>, hasSM<80>]>;
+def INT_NVVM_NEG_BF16 : F_MATH_1<"neg.bf16 \t$dst, $src0;", BF16RT,
+  BF16RT, int_nvvm_neg_bf16, [hasPTX<70>, hasSM<80>]>;
+def INT_NVVM_NEG_BF16X2 : F_MATH_1<"neg.bf16x2 \t$dst, $src0;", BF16X2RT,
+  BF16X2RT, int_nvvm_neg_bf16x2, [hasPTX<70>, hasSM<80>]>;
 
 //
 // Round
@@ -1382,16 +1386,16 @@ def : Pat<(int_nvvm_saturate_d f64:$a),
 //
 
 def INT_NVVM_EX2_APPROX_FTZ_F : F_MATH_1<"ex2.approx.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_ex2_approx_ftz_f>;
+  F32RT, F32RT, int_nvvm_ex2_approx_ftz_f>;
 def INT_NVVM_EX2_APPROX_F : F_MATH_1<"ex2.approx.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_ex2_approx_f>;
+  F32RT, F32RT, int_nvvm_ex2_approx_f>;
 def INT_NVVM_EX2_APPROX_D : F_MATH_1<"ex2.approx.f64 \t$dst, $src0;",
-  Float64Regs, Float64Regs, int_nvvm_ex2_approx_d>;
+  F64RT, F64RT, int_nvvm_ex2_approx_d>;
 
 def INT_NVVM_EX2_APPROX_F16 : F_MATH_1<"ex2.approx.f16 \t$dst, $src0;",
-  Int16Regs, Int16Regs, int_nvvm_ex2_approx_f16, [hasPTX<70>, hasSM<75>]>;
+  F16RT, F16RT, int_nvvm_ex2_approx_f16, [hasPTX<70>, hasSM<75>]>;
 def INT_NVVM_EX2_APPROX_F16X2 : F_MATH_1<"ex2.approx.f16x2 \t$dst, $src0;",
-  Int32Regs, Int32Regs, int_nvvm_ex2_approx_f16x2, [hasPTX<70>, hasSM<75>]>;
+  F16X2RT, F16X2RT, int_nvvm_ex2_approx_f16x2, [hasPTX<70>, hasSM<75>]>;
 
 def : Pat<(fexp2 f32:$a),
           (INT_NVVM_EX2_APPROX_FTZ_F $a)>, Requires<[doF32FTZ]>;
@@ -1403,11 +1407,11 @@ def : Pat<(fexp2 v2f16:$a),
           (INT_NVVM_EX2_APPROX_F16X2 $a)>, Requires<[useFP16Math]>;
 
 def INT_NVVM_LG2_APPROX_FTZ_F : F_MATH_1<"lg2.approx.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_lg2_approx_ftz_f>;
+  F32RT, F32RT, int_nvvm_lg2_approx_ftz_f>;
 def INT_NVVM_LG2_APPROX_F : F_MATH_1<"lg2.approx.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_lg2_approx_f>;
+  F32RT, F32RT, int_nvvm_lg2_approx_f>;
 def INT_NVVM_LG2_APPROX_D : F_MATH_1<"lg2.approx.f64 \t$dst, $src0;",
-  Float64Regs, Float64Regs, int_nvvm_lg2_approx_d>;
+  F64RT, F64RT, int_nvvm_lg2_approx_d>;
 
 def : Pat<(flog2 f32:$a), (INT_NVVM_LG2_APPROX_FTZ_F $a)>,
           Requires<[doF32FTZ]>;
@@ -1419,14 +1423,14 @@ def : Pat<(flog2 f32:$a), (INT_NVVM_LG2_APPROX_F $a)>,
 //
 
 def INT_NVVM_SIN_APPROX_FTZ_F : F_MATH_1<"sin.approx.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_sin_approx_ftz_f>;
+  F32RT, F32RT, int_nvvm_sin_approx_ftz_f>;
 def INT_NVVM_SIN_APPROX_F : F_MATH_1<"sin.approx.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_sin_approx_f>;
+  F32RT, F32RT, int_nvvm_sin_approx_f>;
 
 def INT_NVVM_COS_APPROX_FTZ_F : F_MATH_1<"cos.approx.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_cos_approx_ftz_f>;
+  F32RT, F32RT, int_nvvm_cos_approx_ftz_f>;
 def INT_NVVM_COS_APPROX_F : F_MATH_1<"cos.approx.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_cos_approx_f>;
+  F32RT, F32RT, int_nvvm_cos_approx_f>;
 
 //
 // Fma
@@ -1511,69 +1515,69 @@ defm INT_NVVM_FMA : FMA_INST;
 //
 
 def INT_NVVM_RCP_RN_FTZ_F : F_MATH_1<"rcp.rn.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_rn_ftz_f>;
+  F32RT, F32RT, int_nvvm_rcp_rn_ftz_f>;
 def INT_NVVM_RCP_RN_F : F_MATH_1<"rcp.rn.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_rn_f>;
+  F32RT, F32RT, int_nvvm_rcp_rn_f>;
 def INT_NVVM_RCP_RZ_FTZ_F : F_MATH_1<"rcp.rz.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_rz_ftz_f>;
+  F32RT, F32RT, int_nvvm_rcp_rz_ftz_f>;
 def INT_NVVM_RCP_RZ_F : F_MATH_1<"rcp.rz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_rz_f>;
+  F32RT, F32RT, int_nvvm_rcp_rz_f>;
 def INT_NVVM_RCP_RM_FTZ_F : F_MATH_1<"rcp.rm.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_rm_ftz_f>;
+  F32RT, F32RT, int_nvvm_rcp_rm_ftz_f>;
 def INT_NVVM_RCP_RM_F : F_MATH_1<"rcp.rm.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_rm_f>;
+  F32RT, F32RT, int_nvvm_rcp_rm_f>;
 def INT_NVVM_RCP_RP_FTZ_F : F_MATH_1<"rcp.rp.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_rp_ftz_f>;
+  F32RT, F32RT, int_nvvm_rcp_rp_ftz_f>;
 def INT_NVVM_RCP_RP_F : F_MATH_1<"rcp.rp.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_rp_f>;
+  F32RT, F32RT, int_nvvm_rcp_rp_f>;
 
-def INT_NVVM_RCP_RN_D : F_MATH_1<"rcp.rn.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_rcp_rn_d>;
-def INT_NVVM_RCP_RZ_D : F_MATH_1<"rcp.rz.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_rcp_rz_d>;
-def INT_NVVM_RCP_RM_D : F_MATH_1<"rcp.rm.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_rcp_rm_d>;
-def INT_NVVM_RCP_RP_D : F_MATH_1<"rcp.rp.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_rcp_rp_d>;
+def INT_NVVM_RCP_RN_D : F_MATH_1<"rcp.rn.f64 \t$dst, $src0;", F64RT,
+  F64RT, int_nvvm_rcp_rn_d>;
+def INT_NVVM_RCP_RZ_D : F_MATH_1<"rcp.rz.f64 \t$dst, $src0;", F64RT,
+  F64RT, int_nvvm_rcp_rz_d>;
+def INT_NVVM_RCP_RM_D : F_MATH_1<"rcp.rm.f64 \t$dst, $src0;", F64RT,
+  F64RT, int_nvvm_rcp_rm_d>;
+def INT_NVVM_RCP_RP_D : F_MATH_1<"rcp.rp.f64 \t$dst, $src0;", F64RT,
+  F64RT, int_nvvm_rcp_rp_d>;
 
 def INT_NVVM_RCP_APPROX_FTZ_F : F_MATH_1<"rcp.approx.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rcp_approx_ftz_f>;
+  F32RT, F32RT, int_nvvm_rcp_approx_ftz_f>;
 def INT_NVVM_RCP_APPROX_FTZ_D : F_MATH_1<"rcp.approx.ftz.f64 \t$dst, $src0;",
-  Float64Regs, Float64Regs, int_nvvm_rcp_approx_ftz_d>;
+  F64RT, F64RT, int_nvvm_rcp_approx_ftz_d>;
 
 //
 // Sqrt
 //
 
 def INT_NVVM_SQRT_RN_FTZ_F : F_MATH_1<"sqrt.rn.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_sqrt_rn_ftz_f>;
-def INT_NVVM_SQRT_RN_F : F_MATH_1<"sqrt.rn.f32 \t$dst, $src0;", Float32Regs,
-  Float32Regs, int_nvvm_sqrt_rn_f>;
+  F32RT, F32RT, int_nvvm_sqrt_rn_ftz_f>;
+def INT_NVVM_SQRT_RN_F : F_MATH_1<"sqrt.rn.f32 \t$dst, $src0;", F32RT,
+  F32RT, int_nvvm_sqrt_rn_f>;
 def INT_NVVM_SQRT_RZ_FTZ_F : F_MATH_1<"sqrt.rz.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_sqrt_rz_ftz_f>;
-def INT_NVVM_SQRT_RZ_F : F_MATH_1<"sqrt.rz.f32 \t$dst, $src0;", Float32Regs,
-  Float32Regs, int_nvvm_sqrt_rz_f>;
+  F32RT, F32RT, int_nvvm_sqrt_rz_ftz_f>;
+def INT_NVVM_SQRT_RZ_F : F_MATH_1<"sqrt.rz.f32 \t$dst, $src0;", F32RT,
+  F32RT, int_nvvm_sqrt_rz_f>;
 def INT_NVVM_SQRT_RM_FTZ_F : F_MATH_1<"sqrt.rm.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_sqrt_rm_ftz_f>;
-def INT_NVVM_SQRT_RM_F : F_MATH_1<"sqrt.rm.f32 \t$dst, $src0;", Float32Regs,
-  Float32Regs, int_nvvm_sqrt_rm_f>;
+  F32RT, F32RT, int_nvvm_sqrt_rm_ftz_f>;
+def INT_NVVM_SQRT_RM_F : F_MATH_1<"sqrt.rm.f32 \t$dst, $src0;", F32RT,
+  F32RT, int_nvvm_sqrt_rm_f>;
 def INT_NVVM_SQRT_RP_FTZ_F : F_MATH_1<"sqrt.rp.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_sqrt_rp_ftz_f>;
-def INT_NVVM_SQRT_RP_F : F_MATH_1<"sqrt.rp.f32 \t$dst, $src0;", Float32Regs,
-  Float32Regs, int_nvvm_sqrt_rp_f>;
+  F32RT, F32RT, int_nvvm_sqrt_rp_ftz_f>;
+def INT_NVVM_SQRT_RP_F : F_MATH_1<"sqrt.rp.f32 \t$dst, $src0;", F32RT,
+  F32RT, int_nvvm_sqrt_rp_f>;
 def INT_NVVM_SQRT_APPROX_FTZ_F : F_MATH_1<"sqrt.approx.ftz.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_sqrt_approx_ftz_f>;
+  F32RT, F32RT, int_nvvm_sqrt_approx_ftz_f>;
 def INT_NVVM_SQRT_APPROX_F : F_MATH_1<"sqrt.approx.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_sqrt_approx_f>;
+  F32RT, F32RT, int_nvvm_sqrt_approx_f>;
 
-def INT_NVVM_SQRT_RN_D : F_MATH_1<"sqrt.rn.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_sqrt_rn_d>;
-def INT_NVVM_SQRT_RZ_D : F_MATH_1<"sqrt.rz.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_sqrt_rz_d>;
-def INT_NVVM_SQRT_RM_D : F_MATH_1<"sqrt.rm.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_sqrt_rm_d>;
-def INT_NVVM_SQRT_RP_D : F_MATH_1<"sqrt.rp.f64 \t$dst, $src0;", Float64Regs,
-  Float64Regs, int_nvvm_sqrt_rp_d>;
+def INT_NVVM_SQRT_RN_D : F_MATH_1<"sqrt.rn.f64 \t$dst, $src0;", F64RT,
+  F64RT, int_nvvm_sqrt_rn_d>;
+def INT_NVVM_SQRT_RZ_D : F_MATH_1<"sqrt.rz.f64 \t$dst, $src0;", F64RT,
+  F64RT, int_nvvm_sqrt_rz_d>;
+def INT_NVVM_SQRT_RM_D : F_MATH_1<"sqrt.rm.f64 \t$dst, $src0;", F64RT,
+  F64RT, int_nvvm_sqrt_rm_d>;
+def INT_NVVM_SQRT_RP_D : F_MATH_1<"sqrt.rp.f64 \t$dst, $src0;", F64RT,
+  F64RT, int_nvvm_sqrt_rp_d>;
 
 // nvvm_sqrt intrinsic
 def : Pat<(int_nvvm_sqrt_f f32:$a),
@@ -1590,16 +1594,16 @@ def : Pat<(int_nvvm_sqrt_f f32:$a),
 //
 
 def INT_NVVM_RSQRT_APPROX_FTZ_F
-  : F_MATH_1<"rsqrt.approx.ftz.f32 \t$dst, $src0;", Float32Regs, Float32Regs,
+  : F_MATH_1<"rsqrt.approx.ftz.f32 \t$dst, $src0;", F32RT, F32RT,
     int_nvvm_rsqrt_approx_ftz_f>;
 def INT_NVVM_RSQRT_APPROX_FTZ_D
-  : F_MATH_1<"rsqrt.approx.ftz.f64 \t$dst, $src0;", Float64Regs, Float64Regs,
+  : F_MATH_1<"rsqrt.approx.ftz.f64 \t$dst, $src0;", F64RT, F64RT,
     int_nvvm_rsqrt_approx_ftz_d>;
 
 def INT_NVVM_RSQRT_APPROX_F : F_MATH_1<"rsqrt.approx.f32 \t$dst, $src0;",
-  Float32Regs, Float32Regs, int_nvvm_rsqrt_approx_f>;
+  F32RT, F32RT, int_nvvm_rsqrt_approx_f>;
 def INT_NVVM_RSQRT_APPROX_D : F_MATH_1<"rsqrt.approx.f64 \t$dst, $src0;",
-  Float64Regs, Float64Regs, int_nvvm_rsqrt_approx_d>;
+  F64RT, F64RT, int_nvvm_rsqrt_approx_d>;
 
 // 1.0f / sqrt_approx -> rsqrt_approx
 def: Pat<(fdiv FloatConst1, (int_nvvm_sqrt_approx_f f32:$a)),
@@ -1815,13 +1819,13 @@ def INT_NVVM_D2I_LO : F_MATH_1<
              ".reg .b32 %temp; \n\t",
              "mov.b64 \t{$dst, %temp}, $src0;\n\t",
              "}}"),
-  Int32Regs, Float64Regs, int_nvvm_d2i_lo>;
+  I32RT, F64RT, int_nvvm_d2i_lo>;
 def INT_NVVM_D2I_HI : F_MATH_1<
   !strconcat("{{\n\t",
              ".reg .b32 %temp; \n\t",
              "mov.b64 \t{%temp, $dst}, $src0;\n\t",
              "}}"),
-  Int32Regs, Float64Regs, int_nvvm_d2i_hi>;
+  I32RT, F64RT, int_nvvm_d2i_hi>;
 
 def : Pat<(int_nvvm_f2ll_rn_ftz f32:$a),
           (CVT_s64_f32 $a, CvtRNI_FTZ)>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 81ad01bea8867..72245fe83491d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -185,8 +185,6 @@ static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
       return {Intrinsic::ceil, FTZ_MustBeOff};
     case Intrinsic::nvvm_ceil_ftz_f:
       return {Intrinsic::ceil, FTZ_MustBeOn};
-    case Intrinsic::nvvm_fabs_d:
-      return {Intrinsic::fabs, FTZ_Any};
     case Intrinsic::nvvm_floor_d:
       return {Intrinsic::floor, FTZ_Any};
     case Intrinsic::nvvm_floor_f:
diff --git a/llvm/test/CodeGen/NVPTX/fabs-intrinsics.ll b/llvm/test/CodeGen/NVPTX/fabs-intrinsics.ll
new file mode 100644
index 0000000000000..dd9ef220a9b47
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fabs-intrinsics.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_80 -mattr=+ptx70 | FileCheck --check-prefixes=CHECK %s
+; RUN: %if ptxas-11.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | %ptxas-verify -arch=sm_80 %}
+
+target triple = "nvptx-nvidia-cuda"
+
+declare float @llvm.nvvm.fabs.f32(float)
+declare float @llvm.nvvm.fabs.ftz.f32(float)
+declare double @llvm.nvvm.fabs.f64(double)
+declare half @llvm.nvvm.fabs.f16(half)
+declare half @llvm.nvvm.fabs.ftz.f16(half)
+declare <2 x half> @llvm.nvvm.fabs.v2f16(<2 x half>)
+declare <2 x half> @llvm.nvvm.fabs.ftz.v2f16(<2 x half>)
+declare bfloat @llvm.nvvm.fabs.bf16(bfloat)
+declare <2 x bfloat> @llvm.nvvm.fabs.v2bf16(<2 x bfloat>)
+
+
+define float @fabs_float(float %a) {
+; CHECK-LABEL: fabs_float(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [fabs_float_param_0];
+; CHECK-NEXT:    abs.f32 %f2, %f1;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f2;
+; CHECK-NEXT:    ret;
+  %ret = call float @llvm.nvvm.fabs.f32(float %a)
+  ret float %ret
+}
+
+define float @fabs_float_ftz(float %a) {
+; CHECK-LABEL: fabs_float_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [fabs_float_ftz_param_0];
+; CHECK-NEXT:    abs.ftz.f32 %f2, %f1;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f2;
+; CHECK-NEXT:    ret;
+  %ret = call float @llvm.nvvm.fabs.ftz.f32(float %a)
+  ret float %ret
+}
+
+define double @fabs_double(double %a) {
+; CHECK-LABEL: fabs_double(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f64 %fd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f64 %fd1, [fabs_double_param_0];
+; CHECK-NEXT:    abs.f64 %fd2, %fd1;
+; CHECK-NEXT:    st.param.f64 [func_retval0], %fd2;
+; CHECK-NEXT:    ret;
+  %ret = call double @llvm.nvvm.fabs.f64(double %a)
+  ret double %ret
+}
+
+define half @fabs_half(half %a) {
+; CHECK-LABEL: fabs_half(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [fabs_half_param_0];
+; CHECK-NEXT:    abs.f16 %rs2, %rs1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs2;
+; CHECK-NEXT:    ret;
+  %ret = call half @llvm.nvvm.fabs.f16(half %a)
+  ret half %ret
+}
+
+define half @fabs_half_ftz(half %a) {
+; CHECK-LABEL: fabs_half_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [fabs_half_ftz_param_0];
+; CHECK-NEXT:    abs.ftz.f16 %rs2, %rs1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs2;
+; CHECK-NEXT:    ret;
+  %ret = call half @llvm.nvvm.fabs.ftz.f16(half %a)
+  ret half %ret
+}
+
+define <2 x half> @fabs_v2half(<2 x half> %a) {
+; CHECK-LABEL: fabs_v2half(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fabs_v2half_param_0];
+; CHECK-NEXT:    abs.f16x2 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
+  %ret = call <2 x half> @llvm.nvvm.fabs.v2f16(<2 x half> %a)
+  ret <2 x half> %ret
+}
+
+define <2 x half> @fabs_v2half_ftz(<2 x half> %a) {
+; CHECK-LABEL: fabs_v2half_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fabs_v2half_ftz_param_0];
+; CHECK-NEXT:    abs.ftz.f16x2 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
+  %ret = call <2 x half> @llvm.nvvm.fabs.ftz.v2f16(<2 x half> %a)
+  ret <2 x half> %ret
+}
+
+define bfloat @fabs_bf16(bfloat %a) {
+; CHECK-LABEL: fabs_bf16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [fabs_bf16_param_0];
+; CHECK-NEXT:    abs.bf16 %rs2, %rs1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs2;
+; CHECK-NEXT:    ret;
+  %ret = call bfloat @llvm.nvvm.fabs.bf16(bfloat %a)
+  ret bfloat %ret
+}
+
+define <2 x bfloat> @fabs_v2bf16(<2 x bfloat> %a) {
+; CHECK-LABEL: fabs_v2bf16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fabs_v2bf16_param_0];
+; CHECK-NEXT:    abs.bf16x2 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
+  %ret = call <2 x bfloat> @llvm.nvvm.fabs.v2bf16(<2 x bfloat> %a)
+  ret <2 x bfloat> %ret
+}
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-autoupgrade.ll b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-autoupgrade.ll
index 7faac51ff27ca..c04fd07ec5da1 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-autoupgrade.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-autoupgrade.ll
@@ -1,6 +1,9 @@
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s
 ; RUN: %if ptxas-11.0 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | %ptxas-verify -arch=sm_80 %}
 
+declare bfloat @llvm.nvvm.abs.bf16(bfloat)
+declare <2 x bfloat> @llvm.nvvm.abs.bf16x2(<2 x bfloat>)
+
 ; CHECK-LABEL: abs_bf16
 define bfloat @abs_bf16(bfloat %0) {
   ; CHECK-NOT: call
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index a1517409e1ee1..1819f4ed181c0 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -43,7 +43,7 @@ define float @ceil_float_ftz(float %a) #0 {
 
 ; CHECK-LABEL: @fabs_double
 define double @fabs_double(double %a) #0 {
-; CHECK: call double @llvm.fabs.f64
+; CHECK: call double @llvm.nvvm.fabs.f64
   %ret = call double @llvm.nvvm.fabs.d(double %a)
   ret double %ret
 }

From 78857e7263ba555fb40b286c6b40fcd35a85a65a Mon Sep 17 00:00:00 2001
From: Sirui Mu <msrlancern@gmail.com>
Date: Thu, 17 Apr 2025 22:38:37 +0800
Subject: [PATCH 278/710] [CIR] cir.call with scalar return type (#135552)

This PR introduces support for calling functions with a scalar return
type to the upstream. This PR also includes an initial version of
`CIRGenTargetInfo` and related definitions which are essential for the
CIRGen of call ops.

Related to #132487 .
---
 clang/include/clang/CIR/ABIArgInfo.h          | 92 +++++++++++++++++++
 .../CIR/Dialect/Builder/CIRBaseBuilder.h      |  8 +-
 clang/include/clang/CIR/Dialect/IR/CIROps.td  |  6 +-
 clang/include/clang/CIR/MissingFeatures.h     |  5 +
 clang/lib/CIR/CodeGen/ABIInfo.h               | 32 +++++++
 clang/lib/CIR/CodeGen/CIRGenCall.cpp          | 79 ++++++++++++++--
 clang/lib/CIR/CodeGen/CIRGenCall.h            |  4 +
 clang/lib/CIR/CodeGen/CIRGenExpr.cpp          | 24 ++++-
 clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp    |  7 +-
 clang/lib/CIR/CodeGen/CIRGenFunction.h        | 15 ++-
 clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h    | 39 +++++++-
 clang/lib/CIR/CodeGen/CIRGenModule.cpp        | 28 ++++++
 clang/lib/CIR/CodeGen/CIRGenModule.h          |  6 ++
 clang/lib/CIR/CodeGen/CIRGenTypes.cpp         | 24 ++++-
 clang/lib/CIR/CodeGen/CIRGenTypes.h           |  9 +-
 clang/lib/CIR/CodeGen/CMakeLists.txt          |  1 +
 clang/lib/CIR/CodeGen/TargetInfo.cpp          | 59 ++++++++++++
 clang/lib/CIR/CodeGen/TargetInfo.h            | 85 +++++++++++++++++
 clang/lib/CIR/Dialect/IR/CIRDialect.cpp       | 29 +++++-
 clang/test/CIR/CodeGen/call.cpp               | 10 ++
 clang/test/CIR/IR/call.cir                    | 14 +++
 clang/test/CIR/IR/invalid-call.cir            | 43 +++++++++
 22 files changed, 579 insertions(+), 40 deletions(-)
 create mode 100644 clang/include/clang/CIR/ABIArgInfo.h
 create mode 100644 clang/lib/CIR/CodeGen/ABIInfo.h
 create mode 100644 clang/lib/CIR/CodeGen/TargetInfo.cpp
 create mode 100644 clang/lib/CIR/CodeGen/TargetInfo.h
 create mode 100644 clang/test/CIR/IR/invalid-call.cir

diff --git a/clang/include/clang/CIR/ABIArgInfo.h b/clang/include/clang/CIR/ABIArgInfo.h
new file mode 100644
index 0000000000000..b8d10445f9586
--- /dev/null
+++ b/clang/include/clang/CIR/ABIArgInfo.h
@@ -0,0 +1,92 @@
+//==-- ABIArgInfo.h - Abstract info regarding ABI-specific arguments -------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines ABIArgInfo and associated types used by CIR to track information
+// regarding ABI-coerced types for function arguments and return values. This
+// was moved to the common library as it might be used by both CIRGen and
+// passes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CIR_ABIARGINFO_H
+#define CLANG_CIR_ABIARGINFO_H
+
+#include "mlir/IR/Types.h"
+#include "clang/CIR/MissingFeatures.h"
+
+namespace cir {
+
+class ABIArgInfo {
+public:
+  enum Kind : uint8_t {
+    /// Pass the argument directly using the normal converted CIR type,
+    /// or by coercing to another specified type stored in 'CoerceToType'). If
+    /// an offset is specified (in UIntData), then the argument passed is offset
+    /// by some number of bytes in the memory representation. A dummy argument
+    /// is emitted before the real argument if the specified type stored in
+    /// "PaddingType" is not zero.
+    Direct,
+
+    /// Ignore the argument (treat as void). Useful for void and empty
+    /// structs.
+    Ignore,
+
+    // TODO: more argument kinds will be added as the upstreaming proceeds.
+  };
+
+private:
+  mlir::Type typeData;
+  struct DirectAttrInfo {
+    unsigned offset;
+    unsigned align;
+  };
+  union {
+    DirectAttrInfo directAttr;
+  };
+  Kind theKind;
+
+public:
+  ABIArgInfo(Kind k = Direct) : directAttr{0, 0}, theKind(k) {}
+
+  static ABIArgInfo getDirect(mlir::Type ty = nullptr) {
+    ABIArgInfo info(Direct);
+    info.setCoerceToType(ty);
+    assert(!cir::MissingFeatures::abiArgInfo());
+    return info;
+  }
+
+  static ABIArgInfo getIgnore() { return ABIArgInfo(Ignore); }
+
+  Kind getKind() const { return theKind; }
+  bool isDirect() const { return theKind == Direct; }
+  bool isIgnore() const { return theKind == Ignore; }
+
+  bool canHaveCoerceToType() const {
+    assert(!cir::MissingFeatures::abiArgInfo());
+    return isDirect();
+  }
+
+  unsigned getDirectOffset() const {
+    assert(!cir::MissingFeatures::abiArgInfo());
+    return directAttr.offset;
+  }
+
+  mlir::Type getCoerceToType() const {
+    assert(canHaveCoerceToType() && "invalid kind!");
+    return typeData;
+  }
+
+  void setCoerceToType(mlir::Type ty) {
+    assert(canHaveCoerceToType() && "invalid kind!");
+    typeData = ty;
+  }
+};
+
+} // namespace cir
+
+#endif // CLANG_CIR_ABIARGINFO_H
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index a787908fb9f8d..d2a241964f34f 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -207,13 +207,15 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   // Call operators
   //===--------------------------------------------------------------------===//
 
-  cir::CallOp createCallOp(mlir::Location loc, mlir::SymbolRefAttr callee) {
-    auto op = create<cir::CallOp>(loc, callee);
+  cir::CallOp createCallOp(mlir::Location loc, mlir::SymbolRefAttr callee,
+                           mlir::Type returnType) {
+    auto op = create<cir::CallOp>(loc, callee, returnType);
     return op;
   }
 
   cir::CallOp createCallOp(mlir::Location loc, cir::FuncOp callee) {
-    return createCallOp(loc, mlir::SymbolRefAttr::get(callee));
+    return createCallOp(loc, mlir::SymbolRefAttr::get(callee),
+                        callee.getFunctionType().getReturnType());
   }
 
   //===--------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 0d3c2065cd58c..5ba4b33dc1a12 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -1408,10 +1408,14 @@ def CallOp : CIR_CallOpBase<"call", [NoRegionArguments]> {
     ```
   }];
 
+  let results = (outs Optional<CIR_AnyType>:$result);
   let arguments = commonArgs;
 
-  let builders = [OpBuilder<(ins "mlir::SymbolRefAttr":$callee), [{
+  let builders = [OpBuilder<(ins "mlir::SymbolRefAttr":$callee,
+                                 "mlir::Type":$resType), [{
       $_state.addAttribute("callee", callee);
+      if (resType && !isa<VoidType>(resType))
+        $_state.addTypes(resType);
     }]>];
 }
 
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 5ba55c53dfc4d..0b66ed31d01f1 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -114,6 +114,9 @@ struct MissingFeatures {
 
   // Misc
   static bool cxxABI() { return false; }
+  static bool cirgenABIInfo() { return false; }
+  static bool cirgenTargetInfo() { return false; }
+  static bool abiArgInfo() { return false; }
   static bool tryEmitAsConstant() { return false; }
   static bool constructABIArgDirectExtend() { return false; }
   static bool opGlobalViewAttr() { return false; }
@@ -132,6 +135,8 @@ struct MissingFeatures {
   static bool fpConstraints() { return false; }
   static bool sanitizers() { return false; }
   static bool addHeapAllocSiteMetadata() { return false; }
+  static bool targetCIRGenInfoArch() { return false; }
+  static bool targetCIRGenInfoOS() { return false; }
   static bool targetCodeGenInfoGetNullPointer() { return false; }
   static bool loopInfoStack() { return false; }
   static bool requiresCleanups() { return false; }
diff --git a/clang/lib/CIR/CodeGen/ABIInfo.h b/clang/lib/CIR/CodeGen/ABIInfo.h
new file mode 100644
index 0000000000000..157e80f67a67c
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/ABIInfo.h
@@ -0,0 +1,32 @@
+//===----- ABIInfo.h - ABI information access & encapsulation ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CIR_ABIINFO_H
+#define LLVM_CLANG_LIB_CIR_ABIINFO_H
+
+namespace clang::CIRGen {
+
+class CIRGenFunctionInfo;
+class CIRGenTypes;
+
+class ABIInfo {
+  ABIInfo() = delete;
+
+public:
+  CIRGenTypes &cgt;
+
+  ABIInfo(CIRGenTypes &cgt) : cgt(cgt) {}
+
+  virtual ~ABIInfo();
+
+  virtual void computeInfo(CIRGenFunctionInfo &funcInfo) const = 0;
+};
+
+} // namespace clang::CIRGen
+
+#endif // LLVM_CLANG_LIB_CIR_ABIINFO_H
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.cpp b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
index 1a936458782ea..69266f79a88a5 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
@@ -18,9 +18,12 @@
 using namespace clang;
 using namespace clang::CIRGen;
 
-CIRGenFunctionInfo *CIRGenFunctionInfo::create() {
-  // For now we just create an empty CIRGenFunctionInfo.
-  CIRGenFunctionInfo *fi = new CIRGenFunctionInfo();
+CIRGenFunctionInfo *CIRGenFunctionInfo::create(CanQualType resultType) {
+  void *buffer = operator new(totalSizeToAlloc<ArgInfo>(1));
+
+  CIRGenFunctionInfo *fi = new (buffer) CIRGenFunctionInfo();
+  fi->getArgsBuffer()[0].type = resultType;
+
   return fi;
 }
 
@@ -29,13 +32,29 @@ CIRGenCallee CIRGenCallee::prepareConcreteCallee(CIRGenFunction &cgf) const {
   return *this;
 }
 
-static const CIRGenFunctionInfo &arrangeFreeFunctionLikeCall(CIRGenTypes &cgt) {
+static const CIRGenFunctionInfo &
+arrangeFreeFunctionLikeCall(CIRGenTypes &cgt, CIRGenModule &cgm,
+                            const FunctionType *fnType) {
+  if (const auto *proto = dyn_cast<FunctionProtoType>(fnType)) {
+    if (proto->isVariadic())
+      cgm.errorNYI("call to variadic function");
+    if (proto->hasExtParameterInfos())
+      cgm.errorNYI("call to functions with extra parameter info");
+  } else if (cgm.getTargetCIRGenInfo().isNoProtoCallVariadic(
+                 cast<FunctionNoProtoType>(fnType)))
+    cgm.errorNYI("call to function without a prototype");
+
   assert(!cir::MissingFeatures::opCallArgs());
-  return cgt.arrangeCIRFunctionInfo();
+
+  CanQualType retType = fnType->getReturnType()
+                            ->getCanonicalTypeUnqualified()
+                            .getUnqualifiedType();
+  return cgt.arrangeCIRFunctionInfo(retType);
 }
 
-const CIRGenFunctionInfo &CIRGenTypes::arrangeFreeFunctionCall() {
-  return arrangeFreeFunctionLikeCall(*this);
+const CIRGenFunctionInfo &
+CIRGenTypes::arrangeFreeFunctionCall(const FunctionType *fnType) {
+  return arrangeFreeFunctionLikeCall(*this, cgm, fnType);
 }
 
 static cir::CIRCallOpInterface emitCallLikeOp(CIRGenFunction &cgf,
@@ -54,8 +73,12 @@ static cir::CIRCallOpInterface emitCallLikeOp(CIRGenFunction &cgf,
 
 RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
                                 const CIRGenCallee &callee,
+                                ReturnValueSlot returnValue,
                                 cir::CIRCallOpInterface *callOp,
                                 mlir::Location loc) {
+  QualType retTy = funcInfo.getReturnType();
+  const cir::ABIArgInfo &retInfo = funcInfo.getReturnInfo();
+
   assert(!cir::MissingFeatures::opCallArgs());
   assert(!cir::MissingFeatures::emitLifetimeMarkers());
 
@@ -87,9 +110,45 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
   assert(!cir::MissingFeatures::opCallMustTail());
   assert(!cir::MissingFeatures::opCallReturn());
 
-  // For now we just return nothing because we don't have support for return
-  // values yet.
-  RValue ret = RValue::get(nullptr);
+  RValue ret;
+  switch (retInfo.getKind()) {
+  case cir::ABIArgInfo::Direct: {
+    mlir::Type retCIRTy = convertType(retTy);
+    if (retInfo.getCoerceToType() == retCIRTy &&
+        retInfo.getDirectOffset() == 0) {
+      switch (getEvaluationKind(retTy)) {
+      case cir::TEK_Scalar: {
+        mlir::ResultRange results = theCall->getOpResults();
+        assert(results.size() == 1 && "unexpected number of returns");
+
+        // If the argument doesn't match, perform a bitcast to coerce it. This
+        // can happen due to trivial type mismatches.
+        if (results[0].getType() != retCIRTy)
+          cgm.errorNYI(loc, "bitcast on function return value");
+
+        mlir::Region *region = builder.getBlock()->getParent();
+        if (region != theCall->getParentRegion())
+          cgm.errorNYI(loc, "function calls with cleanup");
+
+        return RValue::get(results[0]);
+      }
+      default:
+        cgm.errorNYI(loc,
+                     "unsupported evaluation kind of function call result");
+      }
+    } else
+      cgm.errorNYI(loc, "unsupported function call form");
+
+    break;
+  }
+  case cir::ABIArgInfo::Ignore:
+    // If we are ignoring an argument that had a result, make sure to construct
+    // the appropriate return value for our caller.
+    ret = getUndefRValue(retTy);
+    break;
+  default:
+    cgm.errorNYI(loc, "unsupported return value information");
+  }
 
   return ret;
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.h b/clang/lib/CIR/CodeGen/CIRGenCall.h
index 76fefdca9e45e..4427fda863d7e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.h
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.h
@@ -81,6 +81,10 @@ struct CallArg {};
 
 class CallArgList : public llvm::SmallVector<CallArg, 8> {};
 
+/// Contains the address where the return value of a function can be stored, and
+/// whether the address is volatile or not.
+class ReturnValueSlot {};
+
 } // namespace clang::CIRGen
 
 #endif // CLANG_LIB_CODEGEN_CIRGENCALL_H
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index cffe5c5cd1ec3..aca26526b79f2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -695,23 +695,36 @@ static CIRGenCallee emitDirectCallee(CIRGenModule &cgm, GlobalDecl gd) {
   return CIRGenCallee::forDirect(callee, gd);
 }
 
+RValue CIRGenFunction::getUndefRValue(QualType ty) {
+  if (ty->isVoidType())
+    return RValue::get(nullptr);
+
+  cgm.errorNYI("unsupported type for undef rvalue");
+  return RValue::get(nullptr);
+}
+
 RValue CIRGenFunction::emitCall(clang::QualType calleeTy,
                                 const CIRGenCallee &callee,
-                                const clang::CallExpr *e) {
+                                const clang::CallExpr *e,
+                                ReturnValueSlot returnValue) {
   // Get the actual function type. The callee type will always be a pointer to
   // function type or a block pointer type.
   assert(calleeTy->isFunctionPointerType() &&
          "Callee must have function pointer type!");
 
   calleeTy = getContext().getCanonicalType(calleeTy);
+  auto pointeeTy = cast<PointerType>(calleeTy)->getPointeeType();
 
   if (getLangOpts().CPlusPlus)
     assert(!cir::MissingFeatures::sanitizers());
 
+  const auto *fnType = cast<FunctionType>(pointeeTy);
+
   assert(!cir::MissingFeatures::sanitizers());
   assert(!cir::MissingFeatures::opCallArgs());
 
-  const CIRGenFunctionInfo &funcInfo = cgm.getTypes().arrangeFreeFunctionCall();
+  const CIRGenFunctionInfo &funcInfo =
+      cgm.getTypes().arrangeFreeFunctionCall(fnType);
 
   assert(!cir::MissingFeatures::opCallNoPrototypeFunc());
   assert(!cir::MissingFeatures::opCallChainCall());
@@ -720,7 +733,7 @@ RValue CIRGenFunction::emitCall(clang::QualType calleeTy,
 
   cir::CIRCallOpInterface callOp;
   RValue callResult =
-      emitCall(funcInfo, callee, &callOp, getLoc(e->getExprLoc()));
+      emitCall(funcInfo, callee, returnValue, &callOp, getLoc(e->getExprLoc()));
 
   assert(!cir::MissingFeatures::generateDebugInfo());
 
@@ -746,7 +759,8 @@ CIRGenCallee CIRGenFunction::emitCallee(const clang::Expr *e) {
   return {};
 }
 
-RValue CIRGenFunction::emitCallExpr(const clang::CallExpr *e) {
+RValue CIRGenFunction::emitCallExpr(const clang::CallExpr *e,
+                                    ReturnValueSlot returnValue) {
   assert(!cir::MissingFeatures::objCBlocks());
 
   if (isa<CXXMemberCallExpr>(e)) {
@@ -778,7 +792,7 @@ RValue CIRGenFunction::emitCallExpr(const clang::CallExpr *e) {
   }
   assert(!cir::MissingFeatures::opCallPseudoDtor());
 
-  return emitCall(e->getCallee()->getType(), callee, e);
+  return emitCall(e->getCallee()->getType(), callee, e, returnValue);
 }
 
 /// Emit code to compute the specified expression, ignoring the result.
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 38104f8533c7d..3dae26dc86f85 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -1519,11 +1519,8 @@ mlir::Value ScalarExprEmitter::VisitCastExpr(CastExpr *ce) {
 }
 
 mlir::Value ScalarExprEmitter::VisitCallExpr(const CallExpr *e) {
-  if (e->getCallReturnType(cgf.getContext())->isReferenceType()) {
-    cgf.getCIRGenModule().errorNYI(
-        e->getSourceRange(), "call to function with non-void return type");
-    return {};
-  }
+  if (e->getCallReturnType(cgf.getContext())->isReferenceType())
+    return emitLoadOfLValue(e);
 
   auto v = cgf.emitCallExpr(e).getScalarVal();
   assert(!cir::MissingFeatures::emitLValueAlignmentAssumption());
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index a96d277d0bc0b..01abd84ce1c85 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -269,6 +269,12 @@ class CIRGenFunction : public CIRGenTypeCache {
     return LValue::makeAddr(addr, ty, baseInfo);
   }
 
+  /// Get an appropriate 'undef' rvalue for the given type.
+  /// TODO: What's the equivalent for MLIR? Currently we're only using this for
+  /// void types so it just returns RValue::get(nullptr) but it'll need
+  /// addressed later.
+  RValue getUndefRValue(clang::QualType ty);
+
   cir::FuncOp generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
                            cir::FuncType funcType);
 
@@ -451,11 +457,12 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitBreakStmt(const clang::BreakStmt &s);
 
   RValue emitCall(const CIRGenFunctionInfo &funcInfo,
-                  const CIRGenCallee &callee, cir::CIRCallOpInterface *callOp,
-                  mlir::Location loc);
+                  const CIRGenCallee &callee, ReturnValueSlot returnValue,
+                  cir::CIRCallOpInterface *callOp, mlir::Location loc);
   RValue emitCall(clang::QualType calleeTy, const CIRGenCallee &callee,
-                  const clang::CallExpr *e);
-  RValue emitCallExpr(const clang::CallExpr *e);
+                  const clang::CallExpr *e, ReturnValueSlot returnValue);
+  RValue emitCallExpr(const clang::CallExpr *e,
+                      ReturnValueSlot returnValue = ReturnValueSlot());
   CIRGenCallee emitCallee(const clang::Expr *e);
 
   mlir::LogicalResult emitContinueStmt(const clang::ContinueStmt &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h b/clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h
index da73e7a7a9059..c4a2b238c96ae 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h
@@ -15,18 +15,49 @@
 #ifndef LLVM_CLANG_CIR_CIRGENFUNCTIONINFO_H
 #define LLVM_CLANG_CIR_CIRGENFUNCTIONINFO_H
 
+#include "clang/AST/CanonicalType.h"
+#include "clang/CIR/ABIArgInfo.h"
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/Support/TrailingObjects.h"
 
 namespace clang::CIRGen {
 
-class CIRGenFunctionInfo final : public llvm::FoldingSetNode {
+struct CIRGenFunctionInfoArgInfo {
+  CanQualType type;
+  cir::ABIArgInfo info;
+};
+
+class CIRGenFunctionInfo final
+    : public llvm::FoldingSetNode,
+      private llvm::TrailingObjects<CIRGenFunctionInfo,
+                                    CIRGenFunctionInfoArgInfo> {
+  using ArgInfo = CIRGenFunctionInfoArgInfo;
+
+  ArgInfo *getArgsBuffer() { return getTrailingObjects<ArgInfo>(); }
+  const ArgInfo *getArgsBuffer() const { return getTrailingObjects<ArgInfo>(); }
+
 public:
-  static CIRGenFunctionInfo *create();
+  static CIRGenFunctionInfo *create(CanQualType resultType);
+
+  void operator delete(void *p) { ::operator delete(p); }
+
+  // Friending class TrailingObjects is apparantly not good enough for MSVC, so
+  // these have to be public.
+  friend class TrailingObjects;
 
   // This function has to be CamelCase because llvm::FoldingSet requires so.
   // NOLINTNEXTLINE(readability-identifier-naming)
-  static void Profile(llvm::FoldingSetNodeID &id) {
-    // We don't have anything to profile yet.
+  static void Profile(llvm::FoldingSetNodeID &id, CanQualType resultType) {
+    resultType.Profile(id);
+  }
+
+  void Profile(llvm::FoldingSetNodeID &id) { getReturnType().Profile(id); }
+
+  CanQualType getReturnType() const { return getArgsBuffer()[0].type; }
+
+  cir::ABIArgInfo &getReturnInfo() { return getArgsBuffer()[0].info; }
+  const cir::ABIArgInfo &getReturnInfo() const {
+    return getArgsBuffer()[0].info;
   }
 };
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index ddcdc0d269708..1b504546162c1 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -129,6 +129,34 @@ CharUnits CIRGenModule::getNaturalTypeAlignment(QualType t,
   return alignment;
 }
 
+const TargetCIRGenInfo &CIRGenModule::getTargetCIRGenInfo() {
+  if (theTargetCIRGenInfo)
+    return *theTargetCIRGenInfo;
+
+  const llvm::Triple &triple = getTarget().getTriple();
+  switch (triple.getArch()) {
+  default:
+    assert(!cir::MissingFeatures::targetCIRGenInfoArch());
+
+    // Currently we just fall through to x86_64.
+    [[fallthrough]];
+
+  case llvm::Triple::x86_64: {
+    switch (triple.getOS()) {
+    default:
+      assert(!cir::MissingFeatures::targetCIRGenInfoOS());
+
+      // Currently we just fall through to x86_64.
+      [[fallthrough]];
+
+    case llvm::Triple::Linux:
+      theTargetCIRGenInfo = createX8664TargetCIRGenInfo(genTypes);
+      return *theTargetCIRGenInfo;
+    }
+  }
+  }
+}
+
 mlir::Location CIRGenModule::getLoc(SourceLocation cLoc) {
   assert(cLoc.isValid() && "expected valid source location");
   const SourceManager &sm = astContext.getSourceManager();
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 764ad1d7592aa..1e0d6623c4f40 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -21,6 +21,7 @@
 #include "clang/AST/CharUnits.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
 
+#include "TargetInfo.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/MLIRContext.h"
@@ -60,6 +61,8 @@ class CIRGenModule : public CIRGenTypeCache {
   ~CIRGenModule() = default;
 
 private:
+  mutable std::unique_ptr<TargetCIRGenInfo> theTargetCIRGenInfo;
+
   CIRGenBuilderTy builder;
 
   /// Hold Clang AST information.
@@ -86,6 +89,7 @@ class CIRGenModule : public CIRGenTypeCache {
   mlir::ModuleOp getModule() const { return theModule; }
   CIRGenBuilderTy &getBuilder() { return builder; }
   clang::ASTContext &getASTContext() const { return astContext; }
+  const clang::TargetInfo &getTarget() const { return target; }
   const clang::CodeGenOptions &getCodeGenOpts() const { return codeGenOpts; }
   CIRGenTypes &getTypes() { return genTypes; }
   const clang::LangOptions &getLangOpts() const { return langOpts; }
@@ -116,6 +120,8 @@ class CIRGenModule : public CIRGenTypeCache {
   getAddrOfGlobalVar(const VarDecl *d, mlir::Type ty = {},
                      ForDefinition_t isForDefinition = NotForDefinition);
 
+  const TargetCIRGenInfo &getTargetCIRGenInfo();
+
   /// Helpers to convert the presumed location of Clang's SourceLocation to an
   /// MLIR Location.
   mlir::Location getLoc(clang::SourceLocation cLoc);
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index ec77c4428d43b..a896b9dce14d6 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -14,9 +14,13 @@ using namespace clang::CIRGen;
 
 CIRGenTypes::CIRGenTypes(CIRGenModule &genModule)
     : cgm(genModule), astContext(genModule.getASTContext()),
-      builder(cgm.getBuilder()) {}
+      builder(cgm.getBuilder()),
+      theABIInfo(cgm.getTargetCIRGenInfo().getABIInfo()) {}
 
-CIRGenTypes::~CIRGenTypes() {}
+CIRGenTypes::~CIRGenTypes() {
+  for (auto i = functionInfos.begin(), e = functionInfos.end(); i != e;)
+    delete &*i++;
+}
 
 mlir::MLIRContext &CIRGenTypes::getMLIRContext() const {
   return *builder.getContext();
@@ -392,10 +396,11 @@ bool CIRGenTypes::isZeroInitializable(clang::QualType t) {
   return true;
 }
 
-const CIRGenFunctionInfo &CIRGenTypes::arrangeCIRFunctionInfo() {
+const CIRGenFunctionInfo &
+CIRGenTypes::arrangeCIRFunctionInfo(CanQualType returnType) {
   // Lookup or create unique function info.
   llvm::FoldingSetNodeID id;
-  CIRGenFunctionInfo::Profile(id);
+  CIRGenFunctionInfo::Profile(id, returnType);
 
   void *insertPos = nullptr;
   CIRGenFunctionInfo *fi = functionInfos.FindNodeOrInsertPos(id, insertPos);
@@ -405,7 +410,7 @@ const CIRGenFunctionInfo &CIRGenTypes::arrangeCIRFunctionInfo() {
   assert(!cir::MissingFeatures::opCallCallConv());
 
   // Construction the function info. We co-allocate the ArgInfos.
-  fi = CIRGenFunctionInfo::create();
+  fi = CIRGenFunctionInfo::create(returnType);
   functionInfos.InsertNode(fi, insertPos);
 
   bool inserted = functionsBeingProcessed.insert(fi).second;
@@ -413,6 +418,15 @@ const CIRGenFunctionInfo &CIRGenTypes::arrangeCIRFunctionInfo() {
   assert(inserted && "Are functions being processed recursively?");
 
   assert(!cir::MissingFeatures::opCallCallConv());
+  getABIInfo().computeInfo(*fi);
+
+  // Loop over all of the computed argument and return value info. If any of
+  // them are direct or extend without a specified coerce type, specify the
+  // default now.
+  cir::ABIArgInfo &retInfo = fi->getReturnInfo();
+  if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
+    retInfo.setCoerceToType(convertType(fi->getReturnType()));
+
   assert(!cir::MissingFeatures::opCallArgs());
 
   bool erased = functionsBeingProcessed.erase(fi);
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.h b/clang/lib/CIR/CodeGen/CIRGenTypes.h
index 38e4bb2f688ab..fc6e9cf621cb3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENTYPES_H
 #define LLVM_CLANG_LIB_CODEGEN_CODEGENTYPES_H
 
+#include "ABIInfo.h"
 #include "CIRGenFunctionInfo.h"
 #include "CIRGenRecordLayout.h"
 
@@ -46,6 +47,8 @@ class CIRGenTypes {
   clang::ASTContext &astContext;
   CIRGenBuilderTy &builder;
 
+  const ABIInfo &theABIInfo;
+
   /// Contains the CIR type for any converted RecordDecl.
   llvm::DenseMap<const clang::Type *, std::unique_ptr<CIRGenRecordLayout>>
       cirGenRecordLayouts;
@@ -88,6 +91,8 @@ class CIRGenTypes {
 
   bool noRecordsBeingLaidOut() const { return recordsBeingLaidOut.empty(); }
 
+  const ABIInfo &getABIInfo() const { return theABIInfo; }
+
   /// Convert a Clang type into a mlir::Type.
   mlir::Type convertType(clang::QualType type);
 
@@ -110,9 +115,9 @@ class CIRGenTypes {
   /// LLVM zeroinitializer.
   bool isZeroInitializable(clang::QualType ty);
 
-  const CIRGenFunctionInfo &arrangeFreeFunctionCall();
+  const CIRGenFunctionInfo &arrangeFreeFunctionCall(const FunctionType *fnType);
 
-  const CIRGenFunctionInfo &arrangeCIRFunctionInfo();
+  const CIRGenFunctionInfo &arrangeCIRFunctionInfo(CanQualType returnType);
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index 418bc2db408cb..400af0237d588 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -22,6 +22,7 @@ add_clang_library(clangCIR
   CIRGenStmt.cpp
   CIRGenStmtOpenACC.cpp
   CIRGenTypes.cpp
+  TargetInfo.cpp
 
   DEPENDS
   MLIRCIR
diff --git a/clang/lib/CIR/CodeGen/TargetInfo.cpp b/clang/lib/CIR/CodeGen/TargetInfo.cpp
new file mode 100644
index 0000000000000..0d0ffb93d4e7e
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/TargetInfo.cpp
@@ -0,0 +1,59 @@
+#include "TargetInfo.h"
+#include "ABIInfo.h"
+#include "CIRGenFunctionInfo.h"
+#include "clang/CIR/MissingFeatures.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+static bool testIfIsVoidTy(QualType ty) {
+  const auto *builtinTy = ty->getAs<BuiltinType>();
+  return builtinTy && builtinTy->getKind() == BuiltinType::Void;
+}
+
+namespace {
+
+class X8664ABIInfo : public ABIInfo {
+public:
+  X8664ABIInfo(CIRGenTypes &cgt) : ABIInfo(cgt) {}
+
+  void computeInfo(CIRGenFunctionInfo &funcInfo) const override;
+};
+
+class X8664TargetCIRGenInfo : public TargetCIRGenInfo {
+public:
+  X8664TargetCIRGenInfo(CIRGenTypes &cgt)
+      : TargetCIRGenInfo(std::make_unique<X8664ABIInfo>(cgt)) {}
+};
+
+} // namespace
+
+void X8664ABIInfo::computeInfo(CIRGenFunctionInfo &funcInfo) const {
+  // Top level CIR has unlimited arguments and return types. Lowering for ABI
+  // specific concerns should happen during a lowering phase. Assume everything
+  // is direct for now.
+  assert(!cir::MissingFeatures::opCallArgs());
+
+  CanQualType retTy = funcInfo.getReturnType();
+  if (testIfIsVoidTy(retTy))
+    funcInfo.getReturnInfo() = cir::ABIArgInfo::getIgnore();
+  else
+    funcInfo.getReturnInfo() =
+        cir::ABIArgInfo::getDirect(cgt.convertType(retTy));
+}
+
+std::unique_ptr<TargetCIRGenInfo>
+clang::CIRGen::createX8664TargetCIRGenInfo(CIRGenTypes &cgt) {
+  return std::make_unique<X8664TargetCIRGenInfo>(cgt);
+}
+
+ABIInfo::~ABIInfo() noexcept = default;
+
+bool TargetCIRGenInfo::isNoProtoCallVariadic(
+    const FunctionNoProtoType *fnType) const {
+  // The following conventions are known to require this to be false:
+  //   x86_stdcall
+  //   MIPS
+  // For everything else, we just prefer false unless we opt out.
+  return false;
+}
diff --git a/clang/lib/CIR/CodeGen/TargetInfo.h b/clang/lib/CIR/CodeGen/TargetInfo.h
new file mode 100644
index 0000000000000..d31d1ee82d90a
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/TargetInfo.h
@@ -0,0 +1,85 @@
+//===---- TargetInfo.h - Encapsulate target details -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// These classes wrap the information about a call or function definition used
+// to handle ABI compliancy.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CIR_TARGETINFO_H
+#define LLVM_CLANG_LIB_CIR_TARGETINFO_H
+
+#include "ABIInfo.h"
+#include "CIRGenTypes.h"
+
+#include <memory>
+#include <utility>
+
+namespace clang::CIRGen {
+
+class TargetCIRGenInfo {
+  std::unique_ptr<ABIInfo> info;
+
+public:
+  TargetCIRGenInfo(std::unique_ptr<ABIInfo> info) : info(std::move(info)) {}
+
+  virtual ~TargetCIRGenInfo() = default;
+
+  /// Returns ABI info helper for the target.
+  const ABIInfo &getABIInfo() const { return *info; }
+
+  /// Determine whether a call to an unprototyped functions under
+  /// the given calling convention should use the variadic
+  /// convention or the non-variadic convention.
+  ///
+  /// There's a good reason to make a platform's variadic calling
+  /// convention be different from its non-variadic calling
+  /// convention: the non-variadic arguments can be passed in
+  /// registers (better for performance), and the variadic arguments
+  /// can be passed on the stack (also better for performance).  If
+  /// this is done, however, unprototyped functions *must* use the
+  /// non-variadic convention, because C99 states that a call
+  /// through an unprototyped function type must succeed if the
+  /// function was defined with a non-variadic prototype with
+  /// compatible parameters.  Therefore, splitting the conventions
+  /// makes it impossible to call a variadic function through an
+  /// unprototyped type.  Since function prototypes came out in the
+  /// late 1970s, this is probably an acceptable trade-off.
+  /// Nonetheless, not all platforms are willing to make it, and in
+  /// particularly x86-64 bends over backwards to make the
+  /// conventions compatible.
+  ///
+  /// The default is false.  This is correct whenever:
+  ///   - the conventions are exactly the same, because it does not
+  ///     matter and the resulting IR will be somewhat prettier in
+  ///     certain cases; or
+  ///   - the conventions are substantively different in how they pass
+  ///     arguments, because in this case using the variadic convention
+  ///     will lead to C99 violations.
+  ///
+  /// However, some platforms make the conventions identical except
+  /// for passing additional out-of-band information to a variadic
+  /// function: for example, x86-64 passes the number of SSE
+  /// arguments in %al.  On these platforms, it is desirable to
+  /// call unprototyped functions using the variadic convention so
+  /// that unprototyped calls to varargs functions still succeed.
+  ///
+  /// Relatedly, platforms which pass the fixed arguments to this:
+  ///   A foo(B, C, D);
+  /// differently than they would pass them to this:
+  ///   A foo(B, C, D, ...);
+  /// may need to adjust the debugger-support code in Sema to do the
+  /// right thing when calling a function with no know signature.
+  virtual bool isNoProtoCallVariadic(const FunctionNoProtoType *fnType) const;
+};
+
+std::unique_ptr<TargetCIRGenInfo> createX8664TargetCIRGenInfo(CIRGenTypes &cgt);
+
+} // namespace clang::CIRGen
+
+#endif // LLVM_CLANG_LIB_CIR_TARGETINFO_H
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 1f4232b9e29ec..d2313e75870b4 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -449,6 +449,7 @@ OpFoldResult cir::CastOp::fold(FoldAdaptor adaptor) {
 static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
                                          mlir::OperationState &result) {
   mlir::FlatSymbolRefAttr calleeAttr;
+  llvm::ArrayRef<mlir::Type> allResultTypes;
 
   if (!parser.parseOptionalAttribute(calleeAttr, "callee", result.attributes)
            .has_value())
@@ -473,6 +474,9 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
   if (parser.parseType(opsFnTy))
     return mlir::failure();
 
+  allResultTypes = opsFnTy.getResults();
+  result.addTypes(allResultTypes);
+
   return mlir::success();
 }
 
@@ -515,9 +519,32 @@ verifyCallCommInSymbolUses(mlir::Operation *op,
     return op->emitOpError() << "'" << fnAttr.getValue()
                              << "' does not reference a valid function";
 
-  // TODO(cir): verify function arguments and return type
+  auto callIf = dyn_cast<cir::CIRCallOpInterface>(op);
+  assert(callIf && "expected CIR call interface to be always available");
+
+  // Verify that the operand and result types match the callee. Note that
+  // argument-checking is disabled for functions without a prototype.
+  auto fnType = fn.getFunctionType();
+
+  // TODO(cir): verify function arguments
   assert(!cir::MissingFeatures::opCallArgs());
 
+  // Void function must not return any results.
+  if (fnType.hasVoidReturn() && op->getNumResults() != 0)
+    return op->emitOpError("callee returns void but call has results");
+
+  // Non-void function calls must return exactly one result.
+  if (!fnType.hasVoidReturn() && op->getNumResults() != 1)
+    return op->emitOpError("incorrect number of results for callee");
+
+  // Parent function and return value types must match.
+  if (!fnType.hasVoidReturn() &&
+      op->getResultTypes().front() != fnType.getReturnType()) {
+    return op->emitOpError("result type mismatch: expected ")
+           << fnType.getReturnType() << ", but provided "
+           << op->getResult(0).getType();
+  }
+
   return mlir::success();
 }
 
diff --git a/clang/test/CIR/CodeGen/call.cpp b/clang/test/CIR/CodeGen/call.cpp
index e69b347c2ca99..9082fbc9f6860 100644
--- a/clang/test/CIR/CodeGen/call.cpp
+++ b/clang/test/CIR/CodeGen/call.cpp
@@ -7,3 +7,13 @@ void f2() {
 
 // CHECK-LABEL: cir.func @f2
 // CHECK:         cir.call @f1() : () -> ()
+
+int f3();
+int f4() {
+  int x = f3();
+  return x;
+}
+
+// CHECK-LABEL: cir.func @f4() -> !s32i
+// CHECK:         %[[#x:]] = cir.call @f3() : () -> !s32i
+// CHECK-NEXT:    cir.store %[[#x]], %{{.+}} : !s32i, !cir.ptr<!s32i>
diff --git a/clang/test/CIR/IR/call.cir b/clang/test/CIR/IR/call.cir
index 8630bb80eb14a..3c3fbf3d4d987 100644
--- a/clang/test/CIR/IR/call.cir
+++ b/clang/test/CIR/IR/call.cir
@@ -1,5 +1,7 @@
 // RUN: cir-opt %s | FileCheck %s
 
+!s32i = !cir.int<s, 32>
+
 module {
 
 cir.func @f1()
@@ -14,4 +16,16 @@ cir.func @f2() {
 // CHECK-NEXT:   cir.return
 // CHECK-NEXT: }
 
+cir.func @f3() -> !s32i
+
+cir.func @f4() -> !s32i {
+  %0 = cir.call @f3() : () -> !s32i
+  cir.return %0 : !s32i
+}
+
+// CHECK:      cir.func @f4() -> !s32i {
+// CHECK-NEXT:   %[[#x:]] = cir.call @f3() : () -> !s32i
+// CHECK-NEXT:   cir.return %[[#x]] : !s32i
+// CHECK-NEXT: }
+
 }
diff --git a/clang/test/CIR/IR/invalid-call.cir b/clang/test/CIR/IR/invalid-call.cir
new file mode 100644
index 0000000000000..64b6d56e0fa88
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-call.cir
@@ -0,0 +1,43 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+!u32i = !cir.int<u, 32>
+
+cir.func @f1() {
+  // expected-error @below {{'f' does not reference a valid function}}
+  cir.call @f() : () -> ()
+  cir.return
+}
+
+// -----
+
+!u32i = !cir.int<u, 32>
+
+cir.func @f2()
+cir.func @f3() {
+  // expected-error @below {{callee returns void but call has results}}
+  %0 = cir.call @f2() : () -> !u32i
+  cir.return
+}
+
+// -----
+
+!u32i = !cir.int<u, 32>
+
+cir.func @f4() -> !u32i
+cir.func @f5() {
+  // expected-error @below {{incorrect number of results for callee}}
+  cir.call @f4() : () -> ()
+  cir.return
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!u32i = !cir.int<u, 32>
+
+cir.func @f6() -> !u32i
+cir.func @f7() {
+  // expected-error @below {{result type mismatch}}
+  %0 = cir.call @f6() : () -> !s32i
+  cir.return
+}

From d83b639b4c62924deef504f46e573e7d995ea10d Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777@gmail.com>
Date: Thu, 17 Apr 2025 22:40:47 +0800
Subject: [PATCH 279/710] Reland [clang] Unify `SourceLocation` and
 `IdentifierInfo*` pair-like data structures to `IdentifierLoc` (#136077)

This PR reland https://github.com/llvm/llvm-project/pull/135808, fixed
some missed changes in LLDB.
I found this issue when I working on
https://github.com/llvm/llvm-project/pull/107168.

Currently we have many similiar data structures like:
- std::pair<IdentifierInfo *, SourceLocation>.
- Element type of ModuleIdPath.
- IdentifierLocPair.
- IdentifierLoc.

This PR unify these data structures to IdentifierLoc, moved
IdentifierLoc definition to SourceLocation.h, and deleted other similer
data structures.

---------

Signed-off-by: yronglin <yronglin777@gmail.com>
---
 .../pp-trace/PPCallbacksTracker.cpp           |   4 +-
 clang/include/clang/AST/OpenACCClause.h       |  20 +--
 clang/include/clang/Basic/IdentifierTable.h   |  26 +++-
 clang/include/clang/Lex/ModuleLoader.h        |   3 +-
 clang/include/clang/Lex/PPCallbacks.h         |   1 +
 clang/include/clang/Lex/Preprocessor.h        |   9 +-
 clang/include/clang/Parse/LoopHint.h          |   2 +-
 clang/include/clang/Parse/Parser.h            |  13 +-
 clang/include/clang/Sema/ParsedAttr.h         |  10 --
 clang/include/clang/Sema/Sema.h               |   2 +-
 clang/include/clang/Sema/SemaCodeCompletion.h |   3 +-
 clang/include/clang/Sema/SemaObjC.h           |   4 +-
 clang/include/clang/Sema/SemaOpenACC.h        |   2 +-
 clang/lib/AST/OpenACCClause.cpp               |   4 +-
 clang/lib/AST/TextNodeDumper.cpp              |   4 +-
 clang/lib/Frontend/CompilerInstance.cpp       |  53 ++++----
 clang/lib/Frontend/FrontendActions.cpp        |   4 +-
 clang/lib/Lex/PPDirectives.cpp                |  22 ++--
 clang/lib/Lex/PPLexerChange.cpp               |   6 +-
 clang/lib/Lex/Pragma.cpp                      |  73 +++++------
 clang/lib/Lex/Preprocessor.cpp                |  16 +--
 clang/lib/Parse/ParseDecl.cpp                 |  28 ++--
 clang/lib/Parse/ParseExpr.cpp                 |   7 +-
 clang/lib/Parse/ParseHLSL.cpp                 |   2 +-
 clang/lib/Parse/ParseObjc.cpp                 |  38 +++---
 clang/lib/Parse/ParseOpenACC.cpp              |  12 +-
 clang/lib/Parse/ParsePragma.cpp               |  15 +--
 clang/lib/Parse/ParseStmt.cpp                 |   6 +-
 clang/lib/Parse/Parser.cpp                    |  19 ++-
 clang/lib/Sema/ParsedAttr.cpp                 |   8 --
 clang/lib/Sema/SemaARM.cpp                    |   2 +-
 clang/lib/Sema/SemaCodeComplete.cpp           |   8 +-
 clang/lib/Sema/SemaDeclAttr.cpp               | 124 +++++++++---------
 clang/lib/Sema/SemaDeclObjC.cpp               |  35 ++---
 clang/lib/Sema/SemaHLSL.cpp                   |  12 +-
 clang/lib/Sema/SemaModule.cpp                 |  42 +++---
 clang/lib/Sema/SemaObjC.cpp                   |  45 +++----
 clang/lib/Sema/SemaOpenACCClause.cpp          |  11 +-
 clang/lib/Sema/SemaStmtAttr.cpp               |  29 ++--
 clang/lib/Sema/SemaSwift.cpp                  |  24 ++--
 clang/lib/Sema/SemaTemplateVariadic.cpp       |  10 +-
 clang/lib/Sema/SemaType.cpp                   |  13 +-
 clang/lib/Serialization/ASTReader.cpp         |   2 +-
 clang/lib/Serialization/ASTWriter.cpp         |   8 +-
 .../DependencyScanning/ModuleDepCollector.cpp |   2 +-
 .../Clang/ClangExpressionParser.cpp           |   6 +-
 .../Clang/ClangModulesDeclVendor.cpp          |  17 ++-
 47 files changed, 411 insertions(+), 395 deletions(-)

diff --git a/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp b/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
index 3bb30fd15b2e1..4c916fa30685b 100644
--- a/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
+++ b/clang-tools-extra/pp-trace/PPCallbacksTracker.cpp
@@ -547,8 +547,8 @@ void PPCallbacksTracker::appendArgument(const char *Name, ModuleIdPath Value) {
     if (I)
       SS << ", ";
     SS << "{"
-       << "Name: " << Value[I].first->getName() << ", "
-       << "Loc: " << getSourceLocationString(PP, Value[I].second) << "}";
+       << "Name: " << Value[I].getIdentifierInfo()->getName() << ", "
+       << "Loc: " << getSourceLocationString(PP, Value[I].getLoc()) << "}";
   }
   SS << "]";
   appendArgument(Name, SS.str());
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 681567228cbb0..f18a6cf62f2c5 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -258,7 +258,7 @@ inline bool operator!=(const OpenACCBindClause &LHS,
   return !(LHS == RHS);
 }
 
-using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
+using DeviceTypeArgument = IdentifierLoc;
 /// A 'device_type' or 'dtype' clause, takes a list of either an 'asterisk' or
 /// an identifier. The 'asterisk' means 'the rest'.
 class OpenACCDeviceTypeClause final
@@ -280,16 +280,16 @@ class OpenACCDeviceTypeClause final
         "Invalid clause kind for device-type");
 
     assert(!llvm::any_of(Archs, [](const DeviceTypeArgument &Arg) {
-      return Arg.second.isInvalid();
+      return Arg.getLoc().isInvalid();
     }) && "Invalid SourceLocation for an argument");
 
-    assert(
-        (Archs.size() == 1 || !llvm::any_of(Archs,
-                                            [](const DeviceTypeArgument &Arg) {
-                                              return Arg.first == nullptr;
-                                            })) &&
-        "Only a single asterisk version is permitted, and must be the "
-        "only one");
+    assert((Archs.size() == 1 ||
+            !llvm::any_of(Archs,
+                          [](const DeviceTypeArgument &Arg) {
+                            return Arg.getIdentifierInfo() == nullptr;
+                          })) &&
+           "Only a single asterisk version is permitted, and must be the "
+           "only one");
 
     std::uninitialized_copy(Archs.begin(), Archs.end(),
                             getTrailingObjects<DeviceTypeArgument>());
@@ -302,7 +302,7 @@ class OpenACCDeviceTypeClause final
   }
   bool hasAsterisk() const {
     return getArchitectures().size() > 0 &&
-           getArchitectures()[0].first == nullptr;
+           getArchitectures()[0].getIdentifierInfo() == nullptr;
   }
 
   ArrayRef<DeviceTypeArgument> getArchitectures() const {
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 0347880244a40..1275b056227b5 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -18,6 +18,7 @@
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/DiagnosticIDs.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/FoldingSet.h"
@@ -76,9 +77,6 @@ inline bool isReservedInAllContexts(ReservedIdentifierStatus Status) {
          Status != ReservedIdentifierStatus::StartsWithUnderscoreAndIsExternC;
 }
 
-/// A simple pair of identifier info and location.
-using IdentifierLocPair = std::pair<IdentifierInfo *, SourceLocation>;
-
 /// IdentifierInfo and other related classes are aligned to
 /// 8 bytes so that DeclarationName can use the lower 3 bits
 /// of a pointer to one of these classes.
@@ -1165,6 +1163,28 @@ class SelectorTable {
   static std::string getPropertyNameFromSetterSelector(Selector Sel);
 };
 
+/// A simple pair of identifier info and location.
+class IdentifierLoc {
+  SourceLocation Loc;
+  IdentifierInfo *II = nullptr;
+
+public:
+  IdentifierLoc() = default;
+  IdentifierLoc(SourceLocation L, IdentifierInfo *Ident) : Loc(L), II(Ident) {}
+
+  void setLoc(SourceLocation L) { Loc = L; }
+  void setIdentifierInfo(IdentifierInfo *Ident) { II = Ident; }
+  SourceLocation getLoc() const { return Loc; }
+  IdentifierInfo *getIdentifierInfo() const { return II; }
+
+  bool operator==(const IdentifierLoc &X) const {
+    return Loc == X.Loc && II == X.II;
+  }
+
+  bool operator!=(const IdentifierLoc &X) const {
+    return Loc != X.Loc || II != X.II;
+  }
+};
 }  // namespace clang
 
 namespace llvm {
diff --git a/clang/include/clang/Lex/ModuleLoader.h b/clang/include/clang/Lex/ModuleLoader.h
index f880a9091a2ed..a58407200c41c 100644
--- a/clang/include/clang/Lex/ModuleLoader.h
+++ b/clang/include/clang/Lex/ModuleLoader.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_LEX_MODULELOADER_H
 #define LLVM_CLANG_LEX_MODULELOADER_H
 
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceLocation.h"
@@ -29,7 +30,7 @@ class IdentifierInfo;
 
 /// A sequence of identifier/location pairs used to describe a particular
 /// module or submodule, e.g., std.vector.
-using ModuleIdPath = ArrayRef<std::pair<IdentifierInfo *, SourceLocation>>;
+using ModuleIdPath = ArrayRef<IdentifierLoc>;
 
 /// Describes the result of attempting to load a module.
 class ModuleLoadResult {
diff --git a/clang/include/clang/Lex/PPCallbacks.h b/clang/include/clang/Lex/PPCallbacks.h
index 46cc564086f1c..313b730afbab8 100644
--- a/clang/include/clang/Lex/PPCallbacks.h
+++ b/clang/include/clang/Lex/PPCallbacks.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_LEX_PPCALLBACKS_H
 
 #include "clang/Basic/DiagnosticIDs.h"
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/ModuleLoader.h"
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 19d54edf23faf..10260c61bdf11 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -327,7 +327,7 @@ class Preprocessor {
   SourceLocation ModuleImportLoc;
 
   /// The import path for named module that we're currently processing.
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> NamedModuleImportPath;
+  SmallVector<IdentifierLoc, 2> NamedModuleImportPath;
 
   llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
   unsigned CheckPointCounter = 0;
@@ -622,7 +622,7 @@ class Preprocessor {
 
   /// The identifier and source location of the currently-active
   /// \#pragma clang arc_cf_code_audited begin.
-  std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
+  IdentifierLoc PragmaARCCFCodeAuditedInfo;
 
   /// The source location of the currently-active
   /// \#pragma clang assume_nonnull begin.
@@ -1998,8 +1998,7 @@ class Preprocessor {
   /// arc_cf_code_audited begin.
   ///
   /// Returns an invalid location if there is no such pragma active.
-  std::pair<IdentifierInfo *, SourceLocation>
-  getPragmaARCCFCodeAuditedInfo() const {
+  IdentifierLoc getPragmaARCCFCodeAuditedInfo() const {
     return PragmaARCCFCodeAuditedInfo;
   }
 
@@ -2007,7 +2006,7 @@ class Preprocessor {
   /// arc_cf_code_audited begin.  An invalid location ends the pragma.
   void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident,
                                      SourceLocation Loc) {
-    PragmaARCCFCodeAuditedInfo = {Ident, Loc};
+    PragmaARCCFCodeAuditedInfo = IdentifierLoc(Loc, Ident);
   }
 
   /// The location of the currently-active \#pragma clang
diff --git a/clang/include/clang/Parse/LoopHint.h b/clang/include/clang/Parse/LoopHint.h
index cec5605ea3615..72be043d3c5a4 100644
--- a/clang/include/clang/Parse/LoopHint.h
+++ b/clang/include/clang/Parse/LoopHint.h
@@ -9,12 +9,12 @@
 #ifndef LLVM_CLANG_PARSE_LOOPHINT_H
 #define LLVM_CLANG_PARSE_LOOPHINT_H
 
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 
 namespace clang {
 
 class Expr;
-struct IdentifierLoc;
 
 /// Loop optimization hint for loop and unroll pragmas.
 struct LoopHint {
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 9ebcf144ba59e..662f54d0e8d8a 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -1725,8 +1725,8 @@ class Parser : public CodeCompletionHandler {
   ObjCTypeParamList *parseObjCTypeParamList();
   ObjCTypeParamList *parseObjCTypeParamListOrProtocolRefs(
       ObjCTypeParamListScope &Scope, SourceLocation &lAngleLoc,
-      SmallVectorImpl<IdentifierLocPair> &protocolIdents,
-      SourceLocation &rAngleLoc, bool mayBeProtocolList = true);
+      SmallVectorImpl<IdentifierLoc> &protocolIdents, SourceLocation &rAngleLoc,
+      bool mayBeProtocolList = true);
 
   void HelperActionsForIvarDeclarations(ObjCContainerDecl *interfaceDecl,
                                         SourceLocation atLoc,
@@ -3818,8 +3818,7 @@ class Parser : public CodeCompletionHandler {
                                SourceLocation Loc,
                                llvm::SmallVectorImpl<Expr *> &IntExprs);
   /// Parses the 'device-type-list', which is a list of identifiers.
-  bool ParseOpenACCDeviceTypeList(
-      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs);
+  bool ParseOpenACCDeviceTypeList(llvm::SmallVector<IdentifierLoc> &Archs);
   /// Parses the 'async-argument', which is an integral value with two
   /// 'special' values that are likely negative (but come from Macros).
   OpenACCIntExprParseResult ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK,
@@ -3951,10 +3950,8 @@ class Parser : public CodeCompletionHandler {
     return false;
   }
 
-  bool ParseModuleName(
-      SourceLocation UseLoc,
-      SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
-      bool IsImport);
+  bool ParseModuleName(SourceLocation UseLoc,
+                       SmallVectorImpl<IdentifierLoc> &Path, bool IsImport);
 
   //===--------------------------------------------------------------------===//
   // C++11/G++: Type Traits [Type-Traits.html in the GCC manual]
diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h
index b88b871dc8821..428d3111de80d 100644
--- a/clang/include/clang/Sema/ParsedAttr.h
+++ b/clang/include/clang/Sema/ParsedAttr.h
@@ -40,7 +40,6 @@ class LangOptions;
 class Sema;
 class Stmt;
 class TargetInfo;
-struct IdentifierLoc;
 
 /// Represents information about a change in availability for
 /// an entity, which is part of the encoding of the 'availability'
@@ -99,15 +98,6 @@ struct PropertyData {
 
 } // namespace detail
 
-/// Wraps an identifier and optional source location for the identifier.
-struct IdentifierLoc {
-  SourceLocation Loc;
-  IdentifierInfo *Ident;
-
-  static IdentifierLoc *create(ASTContext &Ctx, SourceLocation Loc,
-                               IdentifierInfo *Ident);
-};
-
 /// A union of the various pointer types that can be passed to an
 /// ParsedAttr as an argument.
 using ArgsUnion = llvm::PointerUnion<Expr *, IdentifierLoc *>;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index c167f8df28acd..a757f4c6430ae 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -143,7 +143,7 @@ enum class LangAS : unsigned int;
 class LocalInstantiationScope;
 class LookupResult;
 class MangleNumberingContext;
-typedef ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> ModuleIdPath;
+typedef ArrayRef<IdentifierLoc> ModuleIdPath;
 class ModuleLoader;
 class MultiLevelTemplateArgumentList;
 struct NormalizedConstraint;
diff --git a/clang/include/clang/Sema/SemaCodeCompletion.h b/clang/include/clang/Sema/SemaCodeCompletion.h
index 72159de3a6e72..3029e56e5cfe2 100644
--- a/clang/include/clang/Sema/SemaCodeCompletion.h
+++ b/clang/include/clang/Sema/SemaCodeCompletion.h
@@ -193,8 +193,7 @@ class SemaCodeCompletion : public SemaBase {
   void CodeCompleteObjCForCollection(Scope *S, DeclGroupPtrTy IterationVar);
   void CodeCompleteObjCSelector(Scope *S,
                                 ArrayRef<const IdentifierInfo *> SelIdents);
-  void
-  CodeCompleteObjCProtocolReferences(ArrayRef<IdentifierLocPair> Protocols);
+  void CodeCompleteObjCProtocolReferences(ArrayRef<IdentifierLoc> Protocols);
   void CodeCompleteObjCProtocolDecl(Scope *S);
   void CodeCompleteObjCInterfaceDecl(Scope *S);
   void CodeCompleteObjCClassForwardDecl(Scope *S);
diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
index 791a7f45b832f..4cda41a82b61f 100644
--- a/clang/include/clang/Sema/SemaObjC.h
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -307,11 +307,11 @@ class SemaObjC : public SemaBase {
 
   DeclGroupPtrTy
   ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc,
-                                  ArrayRef<IdentifierLocPair> IdentList,
+                                  ArrayRef<IdentifierLoc> IdentList,
                                   const ParsedAttributesView &attrList);
 
   void FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer,
-                               ArrayRef<IdentifierLocPair> ProtocolId,
+                               ArrayRef<IdentifierLoc> ProtocolId,
                                SmallVectorImpl<Decl *> &Protocols);
 
   void DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId,
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 4c3a13a3b044f..8d31d46444c7e 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -212,7 +212,7 @@ class SemaOpenACC : public SemaBase {
   } LoopWithoutSeqInfo;
 
   // Redeclaration of the version in OpenACCClause.h.
-  using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
+  using DeviceTypeArgument = IdentifierLoc;
 
   /// A type to represent all the data for an OpenACC Clause that has been
   /// parsed, but not yet created/semantically analyzed. This is effectively a
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index d7cbb51335359..2820d7b288658 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -891,10 +891,10 @@ void OpenACCClausePrinter::VisitDeviceTypeClause(
   OS << "(";
   llvm::interleaveComma(C.getArchitectures(), OS,
                         [&](const DeviceTypeArgument &Arch) {
-                          if (Arch.first == nullptr)
+                          if (Arch.getIdentifierInfo() == nullptr)
                             OS << "*";
                           else
-                            OS << Arch.first->getName();
+                            OS << Arch.getIdentifierInfo()->getName();
                         });
   OS << ")";
 }
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 89567425f0d5f..5f8417ac57b8b 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -500,10 +500,10 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       llvm::interleaveComma(
           cast<OpenACCDeviceTypeClause>(C)->getArchitectures(), OS,
           [&](const DeviceTypeArgument &Arch) {
-            if (Arch.first == nullptr)
+            if (Arch.getIdentifierInfo() == nullptr)
               OS << "*";
             else
-              OS << Arch.first->getName();
+              OS << Arch.getIdentifierInfo()->getName();
           });
       OS << ")";
       break;
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 243e0a3c15b05..93e4e31c2891d 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -35,6 +35,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/GlobalModuleIndex.h"
@@ -2009,8 +2010,8 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
                              Module::NameVisibilityKind Visibility,
                              bool IsInclusionDirective) {
   // Determine what file we're searching from.
-  StringRef ModuleName = Path[0].first->getName();
-  SourceLocation ModuleNameLoc = Path[0].second;
+  StringRef ModuleName = Path[0].getIdentifierInfo()->getName();
+  SourceLocation ModuleNameLoc = Path[0].getLoc();
 
   // If we've already handled this import, just return the cached result.
   // This one-element cache is important to eliminate redundant diagnostics
@@ -2026,7 +2027,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
   // If we don't already have information on this module, load the module now.
   Module *Module = nullptr;
   ModuleMap &MM = getPreprocessor().getHeaderSearchInfo().getModuleMap();
-  if (auto MaybeModule = MM.getCachedModuleLoad(*Path[0].first)) {
+  if (auto MaybeModule = MM.getCachedModuleLoad(*Path[0].getIdentifierInfo())) {
     // Use the cached result, which may be nullptr.
     Module = *MaybeModule;
     // Config macros are already checked before building a module, but they need
@@ -2046,7 +2047,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     // * `Preprocessor::HandleHeaderIncludeOrImport` will never call this
     //   function as the `#include` or `#import` is textual.
 
-    MM.cacheModuleLoad(*Path[0].first, Module);
+    MM.cacheModuleLoad(*Path[0].getIdentifierInfo(), Module);
   } else {
     ModuleLoadResult Result = findOrCompileModuleAndReadAST(
         ModuleName, ImportLoc, ModuleNameLoc, IsInclusionDirective);
@@ -2055,7 +2056,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (!Result)
       DisableGeneratingGlobalModuleIndex = true;
     Module = Result;
-    MM.cacheModuleLoad(*Path[0].first, Module);
+    MM.cacheModuleLoad(*Path[0].getIdentifierInfo(), Module);
   }
 
   // If we never found the module, fail.  Otherwise, verify the module and link
@@ -2067,7 +2068,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
   // a submodule.
   bool MapPrivateSubModToTopLevel = false;
   for (unsigned I = 1, N = Path.size(); I != N; ++I) {
-    StringRef Name = Path[I].first->getName();
+    StringRef Name = Path[I].getIdentifierInfo()->getName();
     clang::Module *Sub = Module->findSubmodule(Name);
 
     // If the user is requesting Foo.Private and it doesn't exist, try to
@@ -2078,10 +2079,10 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
       SmallString<128> PrivateModule(Module->Name);
       PrivateModule.append("_Private");
 
-      SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> PrivPath;
+      SmallVector<IdentifierLoc, 2> PrivPath;
       auto &II = PP->getIdentifierTable().get(
           PrivateModule, PP->getIdentifierInfo(Module->Name)->getTokenID());
-      PrivPath.push_back(std::make_pair(&II, Path[0].second));
+      PrivPath.emplace_back(Path[0].getLoc(), &II);
 
       std::string FileName;
       // If there is a modulemap module or prebuilt module, load it.
@@ -2095,11 +2096,12 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
         PP->markClangModuleAsAffecting(Module);
         if (!getDiagnostics().isIgnored(
                 diag::warn_no_priv_submodule_use_toplevel, ImportLoc)) {
-          getDiagnostics().Report(Path[I].second,
+          getDiagnostics().Report(Path[I].getLoc(),
                                   diag::warn_no_priv_submodule_use_toplevel)
-              << Path[I].first << Module->getFullModuleName() << PrivateModule
-              << SourceRange(Path[0].second, Path[I].second)
-              << FixItHint::CreateReplacement(SourceRange(Path[0].second),
+              << Path[I].getIdentifierInfo() << Module->getFullModuleName()
+              << PrivateModule
+              << SourceRange(Path[0].getLoc(), Path[I].getLoc())
+              << FixItHint::CreateReplacement(SourceRange(Path[0].getLoc()),
                                               PrivateModule);
           getDiagnostics().Report(Sub->DefinitionLoc,
                                   diag::note_private_top_level_defined);
@@ -2128,10 +2130,11 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
 
       // If there was a clear winner, user it.
       if (Best.size() == 1) {
-        getDiagnostics().Report(Path[I].second, diag::err_no_submodule_suggest)
-            << Path[I].first << Module->getFullModuleName() << Best[0]
-            << SourceRange(Path[0].second, Path[I - 1].second)
-            << FixItHint::CreateReplacement(SourceRange(Path[I].second),
+        getDiagnostics().Report(Path[I].getLoc(),
+                                diag::err_no_submodule_suggest)
+            << Path[I].getIdentifierInfo() << Module->getFullModuleName()
+            << Best[0] << SourceRange(Path[0].getLoc(), Path[I - 1].getLoc())
+            << FixItHint::CreateReplacement(SourceRange(Path[I].getLoc()),
                                             Best[0]);
 
         Sub = Module->findSubmodule(Best[0]);
@@ -2141,9 +2144,9 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (!Sub) {
       // No submodule by this name. Complain, and don't look for further
       // submodules.
-      getDiagnostics().Report(Path[I].second, diag::err_no_submodule)
-          << Path[I].first << Module->getFullModuleName()
-          << SourceRange(Path[0].second, Path[I - 1].second);
+      getDiagnostics().Report(Path[I].getLoc(), diag::err_no_submodule)
+          << Path[I].getIdentifierInfo() << Module->getFullModuleName()
+          << SourceRange(Path[0].getLoc(), Path[I - 1].getLoc());
       break;
     }
 
@@ -2161,8 +2164,8 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
       // FIXME: Should we detect this at module load time? It seems fairly
       // expensive (and rare).
       getDiagnostics().Report(ImportLoc, diag::warn_missing_submodule)
-        << Module->getFullModuleName()
-        << SourceRange(Path.front().second, Path.back().second);
+          << Module->getFullModuleName()
+          << SourceRange(Path.front().getLoc(), Path.back().getLoc());
 
       return ModuleLoadResult(Module, ModuleLoadResult::MissingExpected);
     }
@@ -2171,7 +2174,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     if (Preprocessor::checkModuleIsAvailable(getLangOpts(), getTarget(),
                                              *Module, getDiagnostics())) {
       getDiagnostics().Report(ImportLoc, diag::note_module_import_here)
-        << SourceRange(Path.front().second, Path.back().second);
+          << SourceRange(Path.front().getLoc(), Path.back().getLoc());
       LastModuleImportLoc = ImportLoc;
       LastModuleImportResult = ModuleLoadResult();
       return ModuleLoadResult();
@@ -2296,9 +2299,9 @@ GlobalModuleIndex *CompilerInstance::loadGlobalModuleIndex(
       Module *TheModule = I->second;
       OptionalFileEntryRef Entry = TheModule->getASTFile();
       if (!Entry) {
-        SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
-        Path.push_back(std::make_pair(
-            getPreprocessor().getIdentifierInfo(TheModule->Name), TriggerLoc));
+        SmallVector<IdentifierLoc, 2> Path;
+        Path.emplace_back(TriggerLoc,
+                          getPreprocessor().getIdentifierInfo(TheModule->Name));
         std::reverse(Path.begin(), Path.end());
         // Load a module as hidden.  This also adds it to the global index.
         loadModule(TheModule->DefinitionLoc, Path, Module::Hidden, false);
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index c5aeb92c7af73..e6c7b9f32c29b 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -1216,9 +1216,9 @@ void GetDependenciesByModuleNameAction::ExecuteAction() {
   SourceManager &SM = PP.getSourceManager();
   FileID MainFileID = SM.getMainFileID();
   SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+  SmallVector<IdentifierLoc, 2> Path;
   IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
-  Path.push_back(std::make_pair(ModuleID, FileStart));
+  Path.emplace_back(FileStart, ModuleID);
   auto ModResult = CI.loadModule(FileStart, Path, Module::Hidden, false);
   PPCallbacks *CB = PP.getPPCallbacks();
   CB->moduleImport(SourceLocation(), Path, ModResult);
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 49a4e24923a5e..6c337801a8435 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -1916,15 +1916,15 @@ void Preprocessor::EnterAnnotationToken(SourceRange Range,
 
 /// Produce a diagnostic informing the user that a #include or similar
 /// was implicitly treated as a module import.
-static void diagnoseAutoModuleImport(
-    Preprocessor &PP, SourceLocation HashLoc, Token &IncludeTok,
-    ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> Path,
-    SourceLocation PathEnd) {
+static void diagnoseAutoModuleImport(Preprocessor &PP, SourceLocation HashLoc,
+                                     Token &IncludeTok,
+                                     ArrayRef<IdentifierLoc> Path,
+                                     SourceLocation PathEnd) {
   SmallString<128> PathString;
   for (size_t I = 0, N = Path.size(); I != N; ++I) {
     if (I)
       PathString += '.';
-    PathString += Path[I].first->getName();
+    PathString += Path[I].getIdentifierInfo()->getName();
   }
 
   int IncludeKind = 0;
@@ -2273,12 +2273,12 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
   SourceLocation StartLoc = IsImportDecl ? IncludeTok.getLocation() : HashLoc;
 
   // Complain about attempts to #include files in an audit pragma.
-  if (PragmaARCCFCodeAuditedInfo.second.isValid()) {
+  if (PragmaARCCFCodeAuditedInfo.getLoc().isValid()) {
     Diag(StartLoc, diag::err_pp_include_in_arc_cf_code_audited) << IsImportDecl;
-    Diag(PragmaARCCFCodeAuditedInfo.second, diag::note_pragma_entered_here);
+    Diag(PragmaARCCFCodeAuditedInfo.getLoc(), diag::note_pragma_entered_here);
 
     // Immediately leave the pragma.
-    PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
+    PragmaARCCFCodeAuditedInfo = IdentifierLoc();
   }
 
   // Complain about attempts to #include files in an assume-nonnull pragma.
@@ -2403,10 +2403,10 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
     // Compute the module access path corresponding to this module.
     // FIXME: Should we have a second loadModule() overload to avoid this
     // extra lookup step?
-    SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+    SmallVector<IdentifierLoc, 2> Path;
     for (Module *Mod = ModuleToImport; Mod; Mod = Mod->Parent)
-      Path.push_back(std::make_pair(getIdentifierInfo(Mod->Name),
-                                    FilenameTok.getLocation()));
+      Path.emplace_back(FilenameTok.getLocation(),
+                        getIdentifierInfo(Mod->Name));
     std::reverse(Path.begin(), Path.end());
 
     // Warn that we're replacing the include/import with a module import.
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index a373a52506a24..db6069e31fa46 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -409,13 +409,13 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
   // Complain about reaching a true EOF within arc_cf_code_audited.
   // We don't want to complain about reaching the end of a macro
   // instantiation or a _Pragma.
-  if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro &&
+  if (PragmaARCCFCodeAuditedInfo.getLoc().isValid() && !isEndOfMacro &&
       !(CurLexer && CurLexer->Is_PragmaLexer)) {
-    Diag(PragmaARCCFCodeAuditedInfo.second,
+    Diag(PragmaARCCFCodeAuditedInfo.getLoc(),
          diag::err_pp_eof_in_arc_cf_code_audited);
 
     // Recover by leaving immediately.
-    PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
+    PragmaARCCFCodeAuditedInfo = IdentifierLoc();
   }
 
   // Complain about reaching a true EOF within assume_nonnull.
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
index 91c1619e35623..5b6a29bdad910 100644
--- a/clang/lib/Lex/Pragma.cpp
+++ b/clang/lib/Lex/Pragma.cpp
@@ -763,20 +763,19 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) {
 
 // Lex a component of a module name: either an identifier or a string literal;
 // for components that can be expressed both ways, the two forms are equivalent.
-static bool LexModuleNameComponent(
-    Preprocessor &PP, Token &Tok,
-    std::pair<IdentifierInfo *, SourceLocation> &ModuleNameComponent,
-    bool First) {
+static bool LexModuleNameComponent(Preprocessor &PP, Token &Tok,
+                                   IdentifierLoc &ModuleNameComponent,
+                                   bool First) {
   PP.LexUnexpandedToken(Tok);
   if (Tok.is(tok::string_literal) && !Tok.hasUDSuffix()) {
     StringLiteralParser Literal(Tok, PP);
     if (Literal.hadError)
       return true;
-    ModuleNameComponent = std::make_pair(
-        PP.getIdentifierInfo(Literal.GetString()), Tok.getLocation());
+    ModuleNameComponent = IdentifierLoc(
+        Tok.getLocation(), PP.getIdentifierInfo(Literal.GetString()));
   } else if (!Tok.isAnnotation() && Tok.getIdentifierInfo()) {
     ModuleNameComponent =
-        std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation());
+        IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
   } else {
     PP.Diag(Tok.getLocation(), diag::err_pp_expected_module_name) << First;
     return true;
@@ -784,12 +783,10 @@ static bool LexModuleNameComponent(
   return false;
 }
 
-static bool LexModuleName(
-    Preprocessor &PP, Token &Tok,
-    llvm::SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>>
-        &ModuleName) {
+static bool LexModuleName(Preprocessor &PP, Token &Tok,
+                          llvm::SmallVectorImpl<IdentifierLoc> &ModuleName) {
   while (true) {
-    std::pair<IdentifierInfo*, SourceLocation> NameComponent;
+    IdentifierLoc NameComponent;
     if (LexModuleNameComponent(PP, Tok, NameComponent, ModuleName.empty()))
       return true;
     ModuleName.push_back(NameComponent);
@@ -803,10 +800,10 @@ static bool LexModuleName(
 void Preprocessor::HandlePragmaModuleBuild(Token &Tok) {
   SourceLocation Loc = Tok.getLocation();
 
-  std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc;
+  IdentifierLoc ModuleNameLoc;
   if (LexModuleNameComponent(*this, Tok, ModuleNameLoc, true))
     return;
-  IdentifierInfo *ModuleName = ModuleNameLoc.first;
+  IdentifierInfo *ModuleName = ModuleNameLoc.getIdentifierInfo();
 
   LexUnexpandedToken(Tok);
   if (Tok.isNot(tok::eod)) {
@@ -1109,17 +1106,17 @@ struct PragmaDebugHandler : public PragmaHandler {
         PP.Diag(MacroName, diag::warn_pragma_debug_missing_argument)
             << II->getName();
     } else if (II->isStr("module_map")) {
-      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
-          ModuleName;
+      llvm::SmallVector<IdentifierLoc, 8> ModuleName;
       if (LexModuleName(PP, Tok, ModuleName))
         return;
       ModuleMap &MM = PP.getHeaderSearchInfo().getModuleMap();
       Module *M = nullptr;
       for (auto IIAndLoc : ModuleName) {
-        M = MM.lookupModuleQualified(IIAndLoc.first->getName(), M);
+        M = MM.lookupModuleQualified(IIAndLoc.getIdentifierInfo()->getName(),
+                                     M);
         if (!M) {
-          PP.Diag(IIAndLoc.second, diag::warn_pragma_debug_unknown_module)
-              << IIAndLoc.first->getName();
+          PP.Diag(IIAndLoc.getLoc(), diag::warn_pragma_debug_unknown_module)
+              << IIAndLoc.getIdentifierInfo()->getName();
           return;
         }
       }
@@ -1707,8 +1704,7 @@ struct PragmaModuleImportHandler : public PragmaHandler {
     SourceLocation ImportLoc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
-        ModuleName;
+    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1723,7 +1719,7 @@ struct PragmaModuleImportHandler : public PragmaHandler {
       return;
 
     PP.makeModuleVisible(Imported, ImportLoc);
-    PP.EnterAnnotationToken(SourceRange(ImportLoc, ModuleName.back().second),
+    PP.EnterAnnotationToken(SourceRange(ImportLoc, ModuleName.back().getLoc()),
                             tok::annot_module_include, Imported);
     if (auto *CB = PP.getPPCallbacks())
       CB->moduleImport(ImportLoc, ModuleName, Imported);
@@ -1744,8 +1740,7 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
     SourceLocation BeginLoc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
-        ModuleName;
+    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1754,10 +1749,11 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
 
     // We can only enter submodules of the current module.
     StringRef Current = PP.getLangOpts().CurrentModule;
-    if (ModuleName.front().first->getName() != Current) {
-      PP.Diag(ModuleName.front().second, diag::err_pp_module_begin_wrong_module)
-        << ModuleName.front().first << (ModuleName.size() > 1)
-        << Current.empty() << Current;
+    if (ModuleName.front().getIdentifierInfo()->getName() != Current) {
+      PP.Diag(ModuleName.front().getLoc(),
+              diag::err_pp_module_begin_wrong_module)
+          << ModuleName.front().getIdentifierInfo() << (ModuleName.size() > 1)
+          << Current.empty() << Current;
       return;
     }
 
@@ -1765,17 +1761,19 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
     // be loaded or implicitly loadable.
     auto &HSI = PP.getHeaderSearchInfo();
     auto &MM = HSI.getModuleMap();
-    Module *M = HSI.lookupModule(Current, ModuleName.front().second);
+    Module *M = HSI.lookupModule(Current, ModuleName.front().getLoc());
     if (!M) {
-      PP.Diag(ModuleName.front().second,
-              diag::err_pp_module_begin_no_module_map) << Current;
+      PP.Diag(ModuleName.front().getLoc(),
+              diag::err_pp_module_begin_no_module_map)
+          << Current;
       return;
     }
     for (unsigned I = 1; I != ModuleName.size(); ++I) {
-      auto *NewM = MM.findOrInferSubmodule(M, ModuleName[I].first->getName());
+      auto *NewM = MM.findOrInferSubmodule(
+          M, ModuleName[I].getIdentifierInfo()->getName());
       if (!NewM) {
-        PP.Diag(ModuleName[I].second, diag::err_pp_module_begin_no_submodule)
-          << M->getFullModuleName() << ModuleName[I].first;
+        PP.Diag(ModuleName[I].getLoc(), diag::err_pp_module_begin_no_submodule)
+            << M->getFullModuleName() << ModuleName[I].getIdentifierInfo();
         return;
       }
       M = NewM;
@@ -1791,7 +1789,7 @@ struct PragmaModuleBeginHandler : public PragmaHandler {
 
     // Enter the scope of the submodule.
     PP.EnterSubmodule(M, BeginLoc, /*ForPragma*/true);
-    PP.EnterAnnotationToken(SourceRange(BeginLoc, ModuleName.back().second),
+    PP.EnterAnnotationToken(SourceRange(BeginLoc, ModuleName.back().getLoc()),
                             tok::annot_module_begin, M);
   }
 };
@@ -1835,8 +1833,7 @@ struct PragmaModuleLoadHandler : public PragmaHandler {
     SourceLocation Loc = Tok.getLocation();
 
     // Read the module name.
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 8>
-        ModuleName;
+    llvm::SmallVector<IdentifierLoc, 8> ModuleName;
     if (LexModuleName(PP, Tok, ModuleName))
       return;
 
@@ -1901,7 +1898,7 @@ struct PragmaARCCFCodeAuditedHandler : public PragmaHandler {
       PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma";
 
     // The start location of the active audit.
-    SourceLocation BeginLoc = PP.getPragmaARCCFCodeAuditedInfo().second;
+    SourceLocation BeginLoc = PP.getPragmaARCCFCodeAuditedInfo().getLoc();
 
     // The start location we want after processing this.
     SourceLocation NewLoc;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index c25a3efd899e0..4c050bf1f5bb2 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -1159,8 +1159,8 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
     if (Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
       std::string Name = ModuleDeclState.getPrimaryName().str();
       Name += ":";
-      NamedModuleImportPath.push_back(
-          {getIdentifierInfo(Name), Result.getLocation()});
+      NamedModuleImportPath.emplace_back(Result.getLocation(),
+                                         getIdentifierInfo(Name));
       CurLexerCallback = CLK_LexAfterModuleImport;
       return true;
     }
@@ -1258,8 +1258,8 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
   if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
     // We expected to see an identifier here, and we did; continue handling
     // identifiers.
-    NamedModuleImportPath.push_back(
-        std::make_pair(Result.getIdentifierInfo(), Result.getLocation()));
+    NamedModuleImportPath.emplace_back(Result.getLocation(),
+                                       Result.getIdentifierInfo());
     ModuleImportExpectsIdentifier = false;
     CurLexerCallback = CLK_LexAfterModuleImport;
     return true;
@@ -1302,12 +1302,12 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
       // If the FlatModuleName ends with colon, it implies it is a partition.
       if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
         FlatModuleName += ".";
-      FlatModuleName += Piece.first->getName();
+      FlatModuleName += Piece.getIdentifierInfo()->getName();
     }
-    SourceLocation FirstPathLoc = NamedModuleImportPath[0].second;
+    SourceLocation FirstPathLoc = NamedModuleImportPath[0].getLoc();
     NamedModuleImportPath.clear();
-    NamedModuleImportPath.push_back(
-        std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
+    NamedModuleImportPath.emplace_back(FirstPathLoc,
+                                       getIdentifierInfo(FlatModuleName));
   }
 
   Module *Imported = nullptr;
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 8fa74ecff19aa..8444ff3332e08 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -432,9 +432,8 @@ static bool attributeParsedArgsUnevaluated(const IdentifierInfo &II,
 
 IdentifierLoc *Parser::ParseIdentifierLoc() {
   assert(Tok.is(tok::identifier) && "expected an identifier");
-  IdentifierLoc *IL = IdentifierLoc::create(Actions.Context,
-                                            Tok.getLocation(),
-                                            Tok.getIdentifierInfo());
+  IdentifierLoc *IL = new (Actions.Context)
+      IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
   ConsumeToken();
   return IL;
 }
@@ -1353,20 +1352,21 @@ void Parser::ParseAvailabilityAttribute(
     return;
   }
   IdentifierLoc *Platform = ParseIdentifierLoc();
-  if (const IdentifierInfo *const Ident = Platform->Ident) {
+  if (const IdentifierInfo *const Ident = Platform->getIdentifierInfo()) {
     // Disallow xrOS for availability attributes.
     if (Ident->getName().contains("xrOS") || Ident->getName().contains("xros"))
-      Diag(Platform->Loc, diag::warn_availability_unknown_platform) << Ident;
+      Diag(Platform->getLoc(), diag::warn_availability_unknown_platform)
+          << Ident;
     // Canonicalize platform name from "macosx" to "macos".
     else if (Ident->getName() == "macosx")
-      Platform->Ident = PP.getIdentifierInfo("macos");
+      Platform->setIdentifierInfo(PP.getIdentifierInfo("macos"));
     // Canonicalize platform name from "macosx_app_extension" to
     // "macos_app_extension".
     else if (Ident->getName() == "macosx_app_extension")
-      Platform->Ident = PP.getIdentifierInfo("macos_app_extension");
+      Platform->setIdentifierInfo(PP.getIdentifierInfo("macos_app_extension"));
     else
-      Platform->Ident = PP.getIdentifierInfo(
-          AvailabilityAttr::canonicalizePlatformName(Ident->getName()));
+      Platform->setIdentifierInfo(PP.getIdentifierInfo(
+          AvailabilityAttr::canonicalizePlatformName(Ident->getName())));
   }
 
   // Parse the ',' following the platform name.
@@ -1418,8 +1418,8 @@ void Parser::ParseAvailabilityAttribute(
       continue;
     }
 
-    if (Keyword == Ident_deprecated && Platform->Ident &&
-        Platform->Ident->isStr("swift")) {
+    if (Keyword == Ident_deprecated && Platform->getIdentifierInfo() &&
+        Platform->getIdentifierInfo()->isStr("swift")) {
       // For swift, we deprecate for all versions.
       if (Changes[Deprecated].KeywordLoc.isValid()) {
         Diag(KeywordLoc, diag::err_availability_redundant)
@@ -1436,7 +1436,7 @@ void Parser::ParseAvailabilityAttribute(
     if (Keyword == Ident_environment) {
       if (EnvironmentLoc != nullptr) {
         Diag(KeywordLoc, diag::err_availability_redundant)
-            << Keyword << SourceRange(EnvironmentLoc->Loc);
+            << Keyword << SourceRange(EnvironmentLoc->getLoc());
       }
     }
 
@@ -1792,8 +1792,8 @@ void Parser::ParseSwiftNewTypeAttribute(
     return;
   }
 
-  auto *SwiftType = IdentifierLoc::create(Actions.Context, Tok.getLocation(),
-                                          Tok.getIdentifierInfo());
+  auto *SwiftType = new (Actions.Context)
+      IdentifierLoc(Tok.getLocation(), Tok.getIdentifierInfo());
   ConsumeToken();
 
   // Closing ')'
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 0a22f7372a9f9..1416d52157dca 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -4006,19 +4006,20 @@ std::optional<AvailabilitySpec> Parser::ParseAvailabilitySpec() {
     if (Version.empty())
       return std::nullopt;
 
-    StringRef GivenPlatform = PlatformIdentifier->Ident->getName();
+    StringRef GivenPlatform =
+        PlatformIdentifier->getIdentifierInfo()->getName();
     StringRef Platform =
         AvailabilityAttr::canonicalizePlatformName(GivenPlatform);
 
     if (AvailabilityAttr::getPrettyPlatformName(Platform).empty() ||
         (GivenPlatform.contains("xros") || GivenPlatform.contains("xrOS"))) {
-      Diag(PlatformIdentifier->Loc,
+      Diag(PlatformIdentifier->getLoc(),
            diag::err_avail_query_unrecognized_platform_name)
           << GivenPlatform;
       return std::nullopt;
     }
 
-    return AvailabilitySpec(Version, Platform, PlatformIdentifier->Loc,
+    return AvailabilitySpec(Version, Platform, PlatformIdentifier->getLoc(),
                             VersionRange.getEnd());
   }
 }
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index b832f7b6792d9..a36d324affc57 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -115,7 +115,7 @@ static void fixSeparateAttrArgAndNumber(StringRef ArgStr, SourceLocation ArgLoc,
       << FixedArg
       << FixItHint::CreateReplacement(SourceRange(ArgLoc, EndNumLoc), FixedArg);
   ArgsUnion &Slot = ArgExprs.back();
-  Slot = IdentifierLoc::create(Ctx, ArgLoc, PP.getIdentifierInfo(FixedArg));
+  Slot = new (Ctx) IdentifierLoc(ArgLoc, PP.getIdentifierInfo(FixedArg));
 }
 
 void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index bcbf4dfbabafa..d872177b3d7aa 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -261,7 +261,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
   // case, LAngleLoc will be valid and ProtocolIdents will capture the
   // protocol references (that have not yet been resolved).
   SourceLocation LAngleLoc, EndProtoLoc;
-  SmallVector<IdentifierLocPair, 8> ProtocolIdents;
+  SmallVector<IdentifierLoc, 8> ProtocolIdents;
   ObjCTypeParamList *typeParameterList = nullptr;
   ObjCTypeParamListScope typeParamScope(Actions, getCurScope());
   if (Tok.is(tok::less))
@@ -361,8 +361,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
     if (!ProtocolIdents.empty()) {
       // We already parsed the protocols named when we thought we had a
       // type parameter list. Translate them into actual protocol references.
-      for (const auto &pair : ProtocolIdents) {
-        protocolLocs.push_back(pair.second);
+      for (const auto &Loc : ProtocolIdents) {
+        protocolLocs.push_back(Loc.getLoc());
       }
       Actions.ObjC().FindProtocolDeclaration(/*WarnOnDeclarations=*/true,
                                              /*ForObjCContainer=*/true,
@@ -459,8 +459,8 @@ static void addContextSensitiveTypeNullability(Parser &P,
 /// \param rAngleLoc The location of the ending '>'.
 ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     ObjCTypeParamListScope &Scope, SourceLocation &lAngleLoc,
-    SmallVectorImpl<IdentifierLocPair> &protocolIdents,
-    SourceLocation &rAngleLoc, bool mayBeProtocolList) {
+    SmallVectorImpl<IdentifierLoc> &protocolIdents, SourceLocation &rAngleLoc,
+    bool mayBeProtocolList) {
   assert(Tok.is(tok::less) && "Not at the beginning of a type parameter list");
 
   // Within the type parameter list, don't treat '>' as an operator.
@@ -474,7 +474,8 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     for (const auto &pair : protocolIdents) {
       DeclResult typeParam = Actions.ObjC().actOnObjCTypeParam(
           getCurScope(), ObjCTypeParamVariance::Invariant, SourceLocation(),
-          index++, pair.first, pair.second, SourceLocation(), nullptr);
+          index++, pair.getIdentifierInfo(), pair.getLoc(), SourceLocation(),
+          nullptr);
       if (typeParam.isUsable())
         typeParams.push_back(typeParam.get());
     }
@@ -546,7 +547,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     } else if (mayBeProtocolList) {
       // If this could still be a protocol list, just capture the identifier.
       // We don't want to turn it into a parameter.
-      protocolIdents.push_back(std::make_pair(paramName, paramLoc));
+      protocolIdents.emplace_back(paramLoc, paramName);
       continue;
     }
 
@@ -606,7 +607,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
 /// Parse an objc-type-parameter-list.
 ObjCTypeParamList *Parser::parseObjCTypeParamList() {
   SourceLocation lAngleLoc;
-  SmallVector<IdentifierLocPair, 1> protocolIdents;
+  SmallVector<IdentifierLoc, 1> protocolIdents;
   SourceLocation rAngleLoc;
 
   ObjCTypeParamListScope Scope(Actions, getCurScope());
@@ -1598,7 +1599,7 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
 
   LAngleLoc = ConsumeToken(); // the "<"
 
-  SmallVector<IdentifierLocPair, 8> ProtocolIdents;
+  SmallVector<IdentifierLoc, 8> ProtocolIdents;
 
   while (true) {
     if (Tok.is(tok::code_completion)) {
@@ -1612,8 +1613,7 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
       SkipUntil(tok::greater, StopAtSemi);
       return true;
     }
-    ProtocolIdents.push_back(std::make_pair(Tok.getIdentifierInfo(),
-                                       Tok.getLocation()));
+    ProtocolIdents.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
     ProtocolLocs.push_back(Tok.getLocation());
     ConsumeToken();
 
@@ -1693,10 +1693,9 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
 
     if (Tok.is(tok::code_completion)) {
       // FIXME: Also include types here.
-      SmallVector<IdentifierLocPair, 4> identifierLocPairs;
+      SmallVector<IdentifierLoc, 4> identifierLocPairs;
       for (unsigned i = 0, n = identifiers.size(); i != n; ++i) {
-        identifierLocPairs.push_back(IdentifierLocPair(identifiers[i],
-                                                       identifierLocs[i]));
+        identifierLocPairs.emplace_back(identifierLocs[i], identifiers[i]);
       }
 
       QualType BaseT = Actions.GetTypeFromParser(baseType);
@@ -2094,7 +2093,7 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   SourceLocation nameLoc = ConsumeToken();
 
   if (TryConsumeToken(tok::semi)) { // forward declaration of one protocol.
-    IdentifierLocPair ProtoInfo(protocolName, nameLoc);
+    IdentifierLoc ProtoInfo(nameLoc, protocolName);
     return Actions.ObjC().ActOnForwardProtocolDeclaration(AtLoc, ProtoInfo,
                                                           attrs);
   }
@@ -2102,8 +2101,8 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   CheckNestedObjCContexts(AtLoc);
 
   if (Tok.is(tok::comma)) { // list of forward declarations.
-    SmallVector<IdentifierLocPair, 8> ProtocolRefs;
-    ProtocolRefs.push_back(std::make_pair(protocolName, nameLoc));
+    SmallVector<IdentifierLoc, 8> ProtocolRefs;
+    ProtocolRefs.emplace_back(nameLoc, protocolName);
 
     // Parse the list of forward declarations.
     while (true) {
@@ -2112,8 +2111,7 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
         SkipUntil(tok::semi);
         return nullptr;
       }
-      ProtocolRefs.push_back(IdentifierLocPair(Tok.getIdentifierInfo(),
-                                               Tok.getLocation()));
+      ProtocolRefs.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
       ConsumeToken(); // the identifier
 
       if (Tok.isNot(tok::comma))
@@ -2196,7 +2194,7 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
   // permitted here. Parse and diagnose them.
   if (Tok.is(tok::less)) {
     SourceLocation lAngleLoc, rAngleLoc;
-    SmallVector<IdentifierLocPair, 8> protocolIdents;
+    SmallVector<IdentifierLoc, 8> protocolIdents;
     SourceLocation diagLoc = Tok.getLocation();
     ObjCTypeParamListScope typeParamScope(Actions, getCurScope());
     if (parseObjCTypeParamListOrProtocolRefs(typeParamScope, lAngleLoc,
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 64916995907c5..337b3eca49764 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -15,6 +15,7 @@
 #include "clang/Basic/OpenACCKinds.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/SemaOpenACC.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -814,7 +815,7 @@ bool Parser::ParseOpenACCIntExprList(OpenACCDirectiveKind DK,
 ///
 /// The device_type clause may be abbreviated to dtype.
 bool Parser::ParseOpenACCDeviceTypeList(
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs) {
+    llvm::SmallVector<IdentifierLoc> &Archs) {
 
   if (expectIdentifierOrKeyword(*this)) {
     SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
@@ -822,7 +823,7 @@ bool Parser::ParseOpenACCDeviceTypeList(
     return true;
   }
   IdentifierInfo *Ident = getCurToken().getIdentifierInfo();
-  Archs.emplace_back(Ident, ConsumeToken());
+  Archs.emplace_back(ConsumeToken(), Ident);
 
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     ExpectAndConsume(tok::comma);
@@ -833,7 +834,7 @@ bool Parser::ParseOpenACCDeviceTypeList(
       return true;
     }
     Ident = getCurToken().getIdentifierInfo();
-    Archs.emplace_back(Ident, ConsumeToken());
+    Archs.emplace_back(ConsumeToken(), Ident);
   }
   return false;
 }
@@ -1154,11 +1155,12 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
     }
     case OpenACCClauseKind::DType:
     case OpenACCClauseKind::DeviceType: {
-      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> Archs;
+      llvm::SmallVector<IdentifierLoc> Archs;
       if (getCurToken().is(tok::star)) {
         // FIXME: We want to mark that this is an 'everything else' type of
         // device_type in Sema.
-        ParsedClause.setDeviceTypeDetails({{nullptr, ConsumeToken()}});
+        ParsedClause.setDeviceTypeDetails(
+            {IdentifierLoc(ConsumeToken(), nullptr)});
       } else if (!ParseOpenACCDeviceTypeList(Archs)) {
         ParsedClause.setDeviceTypeDetails(std::move(Archs));
       } else {
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index 21ebff1e50559..17b2b30942582 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -1419,16 +1419,16 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
       static_cast<PragmaLoopHintInfo *>(Tok.getAnnotationValue());
 
   IdentifierInfo *PragmaNameInfo = Info->PragmaName.getIdentifierInfo();
-  Hint.PragmaNameLoc = IdentifierLoc::create(
-      Actions.Context, Info->PragmaName.getLocation(), PragmaNameInfo);
+  Hint.PragmaNameLoc = new (Actions.Context)
+      IdentifierLoc(Info->PragmaName.getLocation(), PragmaNameInfo);
 
   // It is possible that the loop hint has no option identifier, such as
   // #pragma unroll(4).
   IdentifierInfo *OptionInfo = Info->Option.is(tok::identifier)
                                    ? Info->Option.getIdentifierInfo()
                                    : nullptr;
-  Hint.OptionLoc = IdentifierLoc::create(
-      Actions.Context, Info->Option.getLocation(), OptionInfo);
+  Hint.OptionLoc = new (Actions.Context)
+      IdentifierLoc(Info->Option.getLocation(), OptionInfo);
 
   llvm::ArrayRef<Token> Toks = Info->Toks;
 
@@ -1508,7 +1508,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
     if (Toks.size() > 2)
       Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
           << PragmaLoopHintString(Info->PragmaName, Info->Option);
-    Hint.StateLoc = IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
+    Hint.StateLoc = new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
   } else if (OptionInfo && OptionInfo->getName() == "vectorize_width") {
     PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false,
                         /*IsReinject=*/false);
@@ -1529,8 +1529,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
           ConsumeAnyToken();
       }
 
-      Hint.StateLoc =
-          IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
+      Hint.StateLoc = new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
 
       ConsumeToken(); // Consume the constant expression eof terminator.
     } else {
@@ -1554,7 +1553,7 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
           Arg2Error = true;
         } else
           Hint.StateLoc =
-              IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
+              new (Actions.Context) IdentifierLoc(StateLoc, StateInfo);
 
         PP.Lex(Tok); // Identifier
       }
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index e8ec140fbe3e5..4a82d57fe566b 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -2545,9 +2545,9 @@ StmtResult Parser::ParsePragmaLoopHint(StmtVector &Stmts,
 
     ArgsUnion ArgHints[] = {Hint.PragmaNameLoc, Hint.OptionLoc, Hint.StateLoc,
                             ArgsUnion(Hint.ValueExpr)};
-    TempAttrs.addNew(Hint.PragmaNameLoc->Ident, Hint.Range, nullptr,
-                     Hint.PragmaNameLoc->Loc, ArgHints, 4,
-                     ParsedAttr::Form::Pragma());
+    TempAttrs.addNew(Hint.PragmaNameLoc->getIdentifierInfo(), Hint.Range,
+                     /*scopeName=*/nullptr, Hint.PragmaNameLoc->getLoc(),
+                     ArgHints, /*numArgs=*/4, ParsedAttr::Form::Pragma());
   }
 
   // Get the next statement.
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index f3191762b1244..d528664bca352 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -2541,17 +2541,17 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
     return Actions.ActOnPrivateModuleFragmentDecl(ModuleLoc, PrivateLoc);
   }
 
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+  SmallVector<IdentifierLoc, 2> Path;
   if (ParseModuleName(ModuleLoc, Path, /*IsImport*/ false))
     return nullptr;
 
   // Parse the optional module-partition.
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Partition;
+  SmallVector<IdentifierLoc, 2> Partition;
   if (Tok.is(tok::colon)) {
     SourceLocation ColonLoc = ConsumeToken();
     if (!getLangOpts().CPlusPlusModules)
       Diag(ColonLoc, diag::err_unsupported_module_partition)
-          << SourceRange(ColonLoc, Partition.back().second);
+          << SourceRange(ColonLoc, Partition.back().getLoc());
     // Recover by ignoring the partition name.
     else if (ParseModuleName(ModuleLoc, Partition, /*IsImport*/ false))
       return nullptr;
@@ -2600,7 +2600,7 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
   SourceLocation ImportLoc = ConsumeToken();
 
   // For C++20 modules, we can have "name" or ":Partition name" as valid input.
-  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+  SmallVector<IdentifierLoc, 2> Path;
   bool IsPartition = false;
   Module *HeaderUnit = nullptr;
   if (Tok.is(tok::header_name)) {
@@ -2616,7 +2616,7 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
     SourceLocation ColonLoc = ConsumeToken();
     if (!getLangOpts().CPlusPlusModules)
       Diag(ColonLoc, diag::err_unsupported_module_partition)
-          << SourceRange(ColonLoc, Path.back().second);
+          << SourceRange(ColonLoc, Path.back().getLoc());
     // Recover by leaving partition empty.
     else if (ParseModuleName(ColonLoc, Path, /*IsImport*/ true))
       return nullptr;
@@ -2718,10 +2718,9 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
 ///           module-name-qualifier[opt] identifier
 ///         module-name-qualifier:
 ///           module-name-qualifier[opt] identifier '.'
-bool Parser::ParseModuleName(
-    SourceLocation UseLoc,
-    SmallVectorImpl<std::pair<IdentifierInfo *, SourceLocation>> &Path,
-    bool IsImport) {
+bool Parser::ParseModuleName(SourceLocation UseLoc,
+                             SmallVectorImpl<IdentifierLoc> &Path,
+                             bool IsImport) {
   // Parse the module path.
   while (true) {
     if (!Tok.is(tok::identifier)) {
@@ -2737,7 +2736,7 @@ bool Parser::ParseModuleName(
     }
 
     // Record this part of the module path.
-    Path.push_back(std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation()));
+    Path.emplace_back(Tok.getLocation(), Tok.getIdentifierInfo());
     ConsumeToken();
 
     if (Tok.isNot(tok::period))
diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp
index b19a02b8c1a09..c149cef478539 100644
--- a/clang/lib/Sema/ParsedAttr.cpp
+++ b/clang/lib/Sema/ParsedAttr.cpp
@@ -23,14 +23,6 @@
 
 using namespace clang;
 
-IdentifierLoc *IdentifierLoc::create(ASTContext &Ctx, SourceLocation Loc,
-                                     IdentifierInfo *Ident) {
-  IdentifierLoc *Result = new (Ctx) IdentifierLoc;
-  Result->Loc = Loc;
-  Result->Ident = Ident;
-  return Result;
-}
-
 size_t ParsedAttr::allocated_size() const {
   if (IsAvailability) return AttributeFactory::AvailabilityAllocSize;
   else if (IsTypeTagForDatatype)
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 3f53fb200a93d..5bcbe78e9d633 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -1178,7 +1178,7 @@ void SemaARM::handleBuiltinAliasAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Ident = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *Ident = AL.getArgAsIdent(0)->getIdentifierInfo();
   unsigned BuiltinID = Ident->getBuiltinID();
   StringRef AliasName = cast<FunctionDecl>(D)->getIdentifier()->getName();
 
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index f6ec4cb0f069e..1e4e6fdc78351 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -8718,7 +8718,7 @@ static void AddProtocolResults(DeclContext *Ctx, DeclContext *CurContext,
 }
 
 void SemaCodeCompletion::CodeCompleteObjCProtocolReferences(
-    ArrayRef<IdentifierLocPair> Protocols) {
+    ArrayRef<IdentifierLoc> Protocols) {
   ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCProtocolName);
@@ -8729,9 +8729,9 @@ void SemaCodeCompletion::CodeCompleteObjCProtocolReferences(
     // Tell the result set to ignore all of the protocols we have
     // already seen.
     // FIXME: This doesn't work when caching code-completion results.
-    for (const IdentifierLocPair &Pair : Protocols)
-      if (ObjCProtocolDecl *Protocol =
-              SemaRef.ObjC().LookupProtocol(Pair.first, Pair.second))
+    for (const IdentifierLoc &Pair : Protocols)
+      if (ObjCProtocolDecl *Protocol = SemaRef.ObjC().LookupProtocol(
+              Pair.getIdentifierInfo(), Pair.getLoc()))
         Results.Ignore(Protocol);
 
     // Add all protocols.
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index bc891fb009410..7dd20a8795fc9 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -135,13 +135,13 @@ bool Sema::checkStringLiteralArgumentAttr(const ParsedAttr &AL, unsigned ArgNum,
   // Look for identifiers. If we have one emit a hint to fix it to a literal.
   if (AL.isArgIdent(ArgNum)) {
     IdentifierLoc *Loc = AL.getArgAsIdent(ArgNum);
-    Diag(Loc->Loc, diag::err_attribute_argument_type)
+    Diag(Loc->getLoc(), diag::err_attribute_argument_type)
         << AL << AANT_ArgumentString
-        << FixItHint::CreateInsertion(Loc->Loc, "\"")
-        << FixItHint::CreateInsertion(getLocForEndOfToken(Loc->Loc), "\"");
-    Str = Loc->Ident->getName();
+        << FixItHint::CreateInsertion(Loc->getLoc(), "\"")
+        << FixItHint::CreateInsertion(getLocForEndOfToken(Loc->getLoc()), "\"");
+    Str = Loc->getIdentifierInfo()->getName();
     if (ArgLocation)
-      *ArgLocation = Loc->Loc;
+      *ArgLocation = Loc->getLoc();
     return true;
   }
 
@@ -768,7 +768,7 @@ static void handleDiagnoseAsBuiltinAttr(Sema &S, Decl *D,
       auto Union = AL.getArg(Index - 1);
       if (auto *E = dyn_cast<Expr *>(Union))
         return E->getBeginLoc();
-      return cast<IdentifierLoc *>(Union)->Loc;
+      return cast<IdentifierLoc *>(Union)->getLoc();
     }();
 
     S.Diag(Loc, diag::err_attribute_argument_n_type) << AL << Index << T;
@@ -960,10 +960,10 @@ static void handleConsumableAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
-    if (!ConsumableAttr::ConvertStrToConsumedState(IL->Ident->getName(),
-                                                   DefaultState)) {
-      S.Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL
-                                                               << IL->Ident;
+    if (!ConsumableAttr::ConvertStrToConsumedState(
+            IL->getIdentifierInfo()->getName(), DefaultState)) {
+      S.Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
+          << AL << IL->getIdentifierInfo();
       return;
     }
   } else {
@@ -1005,8 +1005,8 @@ static void handleCallableWhenAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     SourceLocation Loc;
     if (AL.isArgIdent(ArgIndex)) {
       IdentifierLoc *Ident = AL.getArgAsIdent(ArgIndex);
-      StateString = Ident->Ident->getName();
-      Loc = Ident->Loc;
+      StateString = Ident->getIdentifierInfo()->getName();
+      Loc = Ident->getLoc();
     } else {
       if (!S.checkStringLiteralArgumentAttr(AL, ArgIndex, StateString, &Loc))
         return;
@@ -1030,11 +1030,11 @@ static void handleParamTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef StateString = Ident->Ident->getName();
+    StringRef StateString = Ident->getIdentifierInfo()->getName();
 
     if (!ParamTypestateAttr::ConvertStrToConsumedState(StateString,
                                                        ParamState)) {
-      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported)
+      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
           << AL << StateString;
       return;
     }
@@ -1064,10 +1064,10 @@ static void handleReturnTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
-    if (!ReturnTypestateAttr::ConvertStrToConsumedState(IL->Ident->getName(),
-                                                        ReturnState)) {
-      S.Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL
-                                                               << IL->Ident;
+    if (!ReturnTypestateAttr::ConvertStrToConsumedState(
+            IL->getIdentifierInfo()->getName(), ReturnState)) {
+      S.Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
+          << AL << IL->getIdentifierInfo();
       return;
     }
   } else {
@@ -1111,10 +1111,10 @@ static void handleSetTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   SetTypestateAttr::ConsumedState NewState;
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef Param = Ident->Ident->getName();
+    StringRef Param = Ident->getIdentifierInfo()->getName();
     if (!SetTypestateAttr::ConvertStrToConsumedState(Param, NewState)) {
-      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported) << AL
-                                                                  << Param;
+      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
+          << AL << Param;
       return;
     }
   } else {
@@ -1133,10 +1133,10 @@ static void handleTestTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   TestTypestateAttr::ConsumedState TestState;
   if (AL.isArgIdent(0)) {
     IdentifierLoc *Ident = AL.getArgAsIdent(0);
-    StringRef Param = Ident->Ident->getName();
+    StringRef Param = Ident->getIdentifierInfo()->getName();
     if (!TestTypestateAttr::ConvertStrToConsumedState(Param, TestState)) {
-      S.Diag(Ident->Loc, diag::warn_attribute_type_not_supported) << AL
-                                                                  << Param;
+      S.Diag(Ident->getLoc(), diag::warn_attribute_type_not_supported)
+          << AL << Param;
       return;
     }
   } else {
@@ -1497,7 +1497,7 @@ static void handleOwnershipAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Module = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *Module = AL.getArgAsIdent(0)->getIdentifierInfo();
 
   StringRef ModuleName = Module->getName();
   if (normalizeName(ModuleName)) {
@@ -1864,10 +1864,10 @@ static void handleCPUSpecificAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *CPUArg = AL.getArgAsIdent(ArgNo);
-    StringRef CPUName = CPUArg->Ident->getName().trim();
+    StringRef CPUName = CPUArg->getIdentifierInfo()->getName().trim();
 
     if (!S.Context.getTargetInfo().validateCPUSpecificCPUDispatch(CPUName)) {
-      S.Diag(CPUArg->Loc, diag::err_invalid_cpu_specific_dispatch_value)
+      S.Diag(CPUArg->getLoc(), diag::err_invalid_cpu_specific_dispatch_value)
           << CPUName << (AL.getKind() == ParsedAttr::AT_CPUDispatch);
       return;
     }
@@ -1880,7 +1880,7 @@ static void handleCPUSpecificAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
       S.Diag(AL.getLoc(), diag::warn_multiversion_duplicate_entries);
       return;
     }
-    CPUs.push_back(CPUArg->Ident);
+    CPUs.push_back(CPUArg->getIdentifierInfo());
   }
 
   FD->setIsMultiVersion(true);
@@ -2358,10 +2358,10 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   IdentifierLoc *Platform = AL.getArgAsIdent(0);
 
-  IdentifierInfo *II = Platform->Ident;
+  IdentifierInfo *II = Platform->getIdentifierInfo();
   if (AvailabilityAttr::getPrettyPlatformName(II->getName()).empty())
-    S.Diag(Platform->Loc, diag::warn_availability_unknown_platform)
-      << Platform->Ident;
+    S.Diag(Platform->getLoc(), diag::warn_availability_unknown_platform)
+        << Platform->getIdentifierInfo();
 
   auto *ND = dyn_cast<NamedDecl>(D);
   if (!ND) // We warned about this already, so just return.
@@ -2410,14 +2410,16 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   IdentifierInfo *IIEnvironment = nullptr;
   if (EnvironmentLoc) {
     if (S.getLangOpts().HLSL) {
-      IIEnvironment = EnvironmentLoc->Ident;
+      IIEnvironment = EnvironmentLoc->getIdentifierInfo();
       if (AvailabilityAttr::getEnvironmentType(
-              EnvironmentLoc->Ident->getName()) ==
+              EnvironmentLoc->getIdentifierInfo()->getName()) ==
           llvm::Triple::EnvironmentType::UnknownEnvironment)
-        S.Diag(EnvironmentLoc->Loc, diag::warn_availability_unknown_environment)
-            << EnvironmentLoc->Ident;
+        S.Diag(EnvironmentLoc->getLoc(),
+               diag::warn_availability_unknown_environment)
+            << EnvironmentLoc->getIdentifierInfo();
     } else {
-      S.Diag(EnvironmentLoc->Loc, diag::err_availability_unexpected_parameter)
+      S.Diag(EnvironmentLoc->getLoc(),
+             diag::err_availability_unexpected_parameter)
           << "environment" << /* C/C++ */ 1;
     }
   }
@@ -3630,7 +3632,7 @@ static void handleEnumExtensibilityAttr(Sema &S, Decl *D,
   }
 
   EnumExtensibilityAttr::Kind ExtensibilityKind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   if (!EnumExtensibilityAttr::ConvertStrToKind(II->getName(),
                                                ExtensibilityKind)) {
     S.Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
@@ -3853,7 +3855,7 @@ static bool handleFormatAttrCommon(Sema &S, Decl *D, const ParsedAttr &AL,
   bool HasImplicitThisParam = isInstanceMethod(D);
   Info->NumArgs = getFunctionOrMethodNumParams(D) + HasImplicitThisParam;
 
-  Info->Identifier = AL.getArgAsIdent(0)->Ident;
+  Info->Identifier = AL.getArgAsIdent(0)->getIdentifierInfo();
   StringRef Format = Info->Identifier->getName();
 
   if (normalizeName(Format)) {
@@ -4017,14 +4019,14 @@ static void handleCallbackAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
     if (AL.isArgIdent(I)) {
       IdentifierLoc *IdLoc = AL.getArgAsIdent(I);
-      auto It = NameIdxMapping.find(IdLoc->Ident->getName());
+      auto It = NameIdxMapping.find(IdLoc->getIdentifierInfo()->getName());
       if (It == UnknownName) {
         S.Diag(AL.getLoc(), diag::err_callback_attribute_argument_unknown)
-            << IdLoc->Ident << IdLoc->Loc;
+            << IdLoc->getIdentifierInfo() << IdLoc->getLoc();
         return;
       }
 
-      SR = SourceRange(IdLoc->Loc);
+      SR = SourceRange(IdLoc->getLoc());
       ArgIdx = It->second;
     } else if (AL.isArgExpr(I)) {
       Expr *IdxExpr = AL.getArgAsExpr(I);
@@ -4142,13 +4144,14 @@ LifetimeCaptureByAttr *Sema::ParseLifetimeCaptureByAttr(const ParsedAttr &AL,
     }
     assert(AL.isArgIdent(I));
     IdentifierLoc *IdLoc = AL.getArgAsIdent(I);
-    if (IdLoc->Ident->getName() == ParamName) {
-      Diag(IdLoc->Loc, diag::err_capture_by_references_itself) << IdLoc->Loc;
+    if (IdLoc->getIdentifierInfo()->getName() == ParamName) {
+      Diag(IdLoc->getLoc(), diag::err_capture_by_references_itself)
+          << IdLoc->getLoc();
       IsValid = false;
       continue;
     }
-    ParamIdents[I] = IdLoc->Ident;
-    ParamLocs[I] = IdLoc->Loc;
+    ParamIdents[I] = IdLoc->getIdentifierInfo();
+    ParamLocs[I] = IdLoc->getLoc();
   }
   if (!IsValid)
     return nullptr;
@@ -4754,7 +4757,7 @@ static void handleModeAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Name = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *Name = AL.getArgAsIdent(0)->getIdentifierInfo();
 
   S.AddModeAttr(D, AL, Name);
 }
@@ -5727,8 +5730,8 @@ static void handleArgumentWithTypeTagAttr(Sema &S, Decl *D,
   }
 
   D->addAttr(::new (S.Context) ArgumentWithTypeTagAttr(
-      S.Context, AL, AL.getArgAsIdent(0)->Ident, ArgumentIdx, TypeTagIdx,
-      IsPointer));
+      S.Context, AL, AL.getArgAsIdent(0)->getIdentifierInfo(), ArgumentIdx,
+      TypeTagIdx, IsPointer));
 }
 
 static void handleTypeTagForDatatypeAttr(Sema &S, Decl *D,
@@ -5748,7 +5751,7 @@ static void handleTypeTagForDatatypeAttr(Sema &S, Decl *D,
     return;
   }
 
-  IdentifierInfo *PointerKind = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *PointerKind = AL.getArgAsIdent(0)->getIdentifierInfo();
   TypeSourceInfo *MatchingCTypeLoc = nullptr;
   S.GetTypeFromParser(AL.getMatchingCType(), &MatchingCTypeLoc);
   assert(MatchingCTypeLoc && "no type source info for attribute argument");
@@ -5819,7 +5822,7 @@ static void handleBuiltinAliasAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *Ident = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *Ident = AL.getArgAsIdent(0)->getIdentifierInfo();
   unsigned BuiltinID = Ident->getBuiltinID();
   StringRef AliasName = cast<FunctionDecl>(D)->getIdentifier()->getName();
 
@@ -6585,7 +6588,7 @@ static void handleCFGuardAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 
   CFGuardAttr::GuardArg Arg;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   if (!CFGuardAttr::ConvertStrToGuardArg(II->getName(), Arg)) {
     S.Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
     return;
@@ -6687,8 +6690,9 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
   if (AL.isArgIdent(0)) {
     IdentifierLoc *IL = AL.getArgAsIdent(0);
     if (!VTablePointerAuthenticationAttr::ConvertStrToVPtrAuthKeyType(
-            IL->Ident->getName(), KeyType)) {
-      S.Diag(IL->Loc, diag::err_invalid_authentication_key) << IL->Ident;
+            IL->getIdentifierInfo()->getName(), KeyType)) {
+      S.Diag(IL->getLoc(), diag::err_invalid_authentication_key)
+          << IL->getIdentifierInfo();
       AL.setInvalid();
     }
     if (KeyType == VTablePointerAuthenticationAttr::DefaultKey &&
@@ -6708,15 +6712,16 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
     if (AL.isArgIdent(1)) {
       IdentifierLoc *IL = AL.getArgAsIdent(1);
       if (!VTablePointerAuthenticationAttr::
-              ConvertStrToAddressDiscriminationMode(IL->Ident->getName(),
-                                                    AddressDiversityMode)) {
-        S.Diag(IL->Loc, diag::err_invalid_address_discrimination) << IL->Ident;
+              ConvertStrToAddressDiscriminationMode(
+                  IL->getIdentifierInfo()->getName(), AddressDiversityMode)) {
+        S.Diag(IL->getLoc(), diag::err_invalid_address_discrimination)
+            << IL->getIdentifierInfo();
         AL.setInvalid();
       }
       if (AddressDiversityMode ==
               VTablePointerAuthenticationAttr::DefaultAddressDiscrimination &&
           !S.getLangOpts().PointerAuthCalls) {
-        S.Diag(IL->Loc, diag::err_no_default_vtable_pointer_auth) << 1;
+        S.Diag(IL->getLoc(), diag::err_no_default_vtable_pointer_auth) << 1;
         AL.setInvalid();
       }
     } else {
@@ -6731,8 +6736,9 @@ static void handleVTablePointerAuthentication(Sema &S, Decl *D,
     if (AL.isArgIdent(2)) {
       IdentifierLoc *IL = AL.getArgAsIdent(2);
       if (!VTablePointerAuthenticationAttr::ConvertStrToExtraDiscrimination(
-              IL->Ident->getName(), ED)) {
-        S.Diag(IL->Loc, diag::err_invalid_extra_discrimination) << IL->Ident;
+              IL->getIdentifierInfo()->getName(), ED)) {
+        S.Diag(IL->getLoc(), diag::err_invalid_extra_discrimination)
+            << IL->getIdentifierInfo();
         AL.setInvalid();
       }
       if (ED == VTablePointerAuthenticationAttr::DefaultExtraDiscrimination &&
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index ba9d3dcf19617..0a14ce23a396e 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -1310,24 +1310,26 @@ static bool NestedProtocolHasNoDefinition(ObjCProtocolDecl *PDecl,
 /// protocol declarations in its 'Protocols' argument.
 void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
                                        bool ForObjCContainer,
-                                       ArrayRef<IdentifierLocPair> ProtocolId,
+                                       ArrayRef<IdentifierLoc> ProtocolId,
                                        SmallVectorImpl<Decl *> &Protocols) {
-  for (const IdentifierLocPair &Pair : ProtocolId) {
-    ObjCProtocolDecl *PDecl = LookupProtocol(Pair.first, Pair.second);
+  for (const IdentifierLoc &Pair : ProtocolId) {
+    ObjCProtocolDecl *PDecl =
+        LookupProtocol(Pair.getIdentifierInfo(), Pair.getLoc());
     if (!PDecl) {
       DeclFilterCCC<ObjCProtocolDecl> CCC{};
-      TypoCorrection Corrected =
-          SemaRef.CorrectTypo(DeclarationNameInfo(Pair.first, Pair.second),
-                              Sema::LookupObjCProtocolName, SemaRef.TUScope,
-                              nullptr, CCC, Sema::CTK_ErrorRecovery);
+      TypoCorrection Corrected = SemaRef.CorrectTypo(
+          DeclarationNameInfo(Pair.getIdentifierInfo(), Pair.getLoc()),
+          Sema::LookupObjCProtocolName, SemaRef.TUScope, nullptr, CCC,
+          Sema::CTK_ErrorRecovery);
       if ((PDecl = Corrected.getCorrectionDeclAs<ObjCProtocolDecl>()))
         SemaRef.diagnoseTypo(Corrected,
                              PDiag(diag::err_undeclared_protocol_suggest)
-                                 << Pair.first);
+                                 << Pair.getIdentifierInfo());
     }
 
     if (!PDecl) {
-      Diag(Pair.second, diag::err_undeclared_protocol) << Pair.first;
+      Diag(Pair.getLoc(), diag::err_undeclared_protocol)
+          << Pair.getIdentifierInfo();
       continue;
     }
     // If this is a forward protocol declaration, get its definition.
@@ -1337,7 +1339,7 @@ void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
     // For an objc container, delay protocol reference checking until after we
     // can set the objc decl as the availability context, otherwise check now.
     if (!ForObjCContainer) {
-      (void)SemaRef.DiagnoseUseOfDecl(PDecl, Pair.second);
+      (void)SemaRef.DiagnoseUseOfDecl(PDecl, Pair.getLoc());
     }
 
     // If this is a forward declaration and we are supposed to warn in this
@@ -1347,7 +1349,8 @@ void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
 
     if (WarnOnDeclarations &&
         NestedProtocolHasNoDefinition(PDecl, UndefinedProtocol)) {
-      Diag(Pair.second, diag::warn_undef_protocolref) << Pair.first;
+      Diag(Pair.getLoc(), diag::warn_undef_protocolref)
+          << Pair.getIdentifierInfo();
       Diag(UndefinedProtocol->getLocation(), diag::note_protocol_decl_undefined)
         << UndefinedProtocol;
     }
@@ -1784,17 +1787,17 @@ void SemaObjC::DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
 
 /// ActOnForwardProtocolDeclaration - Handle \@protocol foo;
 SemaObjC::DeclGroupPtrTy SemaObjC::ActOnForwardProtocolDeclaration(
-    SourceLocation AtProtocolLoc, ArrayRef<IdentifierLocPair> IdentList,
+    SourceLocation AtProtocolLoc, ArrayRef<IdentifierLoc> IdentList,
     const ParsedAttributesView &attrList) {
   ASTContext &Context = getASTContext();
   SmallVector<Decl *, 8> DeclsInGroup;
-  for (const IdentifierLocPair &IdentPair : IdentList) {
-    IdentifierInfo *Ident = IdentPair.first;
+  for (const IdentifierLoc &IdentPair : IdentList) {
+    IdentifierInfo *Ident = IdentPair.getIdentifierInfo();
     ObjCProtocolDecl *PrevDecl = LookupProtocol(
-        Ident, IdentPair.second, SemaRef.forRedeclarationInCurContext());
+        Ident, IdentPair.getLoc(), SemaRef.forRedeclarationInCurContext());
     ObjCProtocolDecl *PDecl =
         ObjCProtocolDecl::Create(Context, SemaRef.CurContext, Ident,
-                                 IdentPair.second, AtProtocolLoc, PrevDecl);
+                                 IdentPair.getLoc(), AtProtocolLoc, PrevDecl);
 
     SemaRef.PushOnScopeChains(PDecl, SemaRef.TUScope);
     CheckObjCDeclScope(PDecl);
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 76fb81d39d67c..10c22817b082c 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -1282,8 +1282,8 @@ bool SemaHLSL::handleResourceTypeAttr(QualType T, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *Loc = AL.getArgAsIdent(0);
-    StringRef Identifier = Loc->Ident->getName();
-    SourceLocation ArgLoc = Loc->Loc;
+    StringRef Identifier = Loc->getIdentifierInfo()->getName();
+    SourceLocation ArgLoc = Loc->getLoc();
 
     // Validate resource class value
     ResourceClass RC;
@@ -1542,8 +1542,8 @@ void SemaHLSL::handleResourceBindingAttr(Decl *TheDecl, const ParsedAttr &AL) {
   }
 
   IdentifierLoc *Loc = AL.getArgAsIdent(0);
-  StringRef Str = Loc->Ident->getName();
-  SourceLocation ArgLoc = Loc->Loc;
+  StringRef Str = Loc->getIdentifierInfo()->getName();
+  SourceLocation ArgLoc = Loc->getLoc();
 
   SourceLocation SpaceArgLoc;
   bool SpecifiedSpace = false;
@@ -1557,8 +1557,8 @@ void SemaHLSL::handleResourceBindingAttr(Decl *TheDecl, const ParsedAttr &AL) {
     }
 
     IdentifierLoc *Loc = AL.getArgAsIdent(1);
-    Space = Loc->Ident->getName();
-    SpaceArgLoc = Loc->Loc;
+    Space = Loc->getIdentifierInfo()->getName();
+    SpaceArgLoc = Loc->getLoc();
   } else {
     Slot = Str;
   }
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index 76589bff40be9..4bba57193ded6 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/StringExtras.h"
 
@@ -68,7 +69,7 @@ static std::string stringFromPath(ModuleIdPath Path) {
   for (auto &Piece : Path) {
     if (!Name.empty())
       Name += ".";
-    Name += Piece.first->getName();
+    Name += Piece.getIdentifierInfo()->getName();
   }
   return Name;
 }
@@ -350,17 +351,18 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
 
   // Test the first part of the path to see if it's std[0-9]+ but allow the
   // name in a system header.
-  StringRef FirstComponentName = Path[0].first->getName();
-  if (!getSourceManager().isInSystemHeader(Path[0].second) &&
+  StringRef FirstComponentName = Path[0].getIdentifierInfo()->getName();
+  if (!getSourceManager().isInSystemHeader(Path[0].getLoc()) &&
       (FirstComponentName == "std" ||
        (FirstComponentName.starts_with("std") &&
         llvm::all_of(FirstComponentName.drop_front(3), &llvm::isDigit))))
-    Diag(Path[0].second, diag::warn_reserved_module_name) << Path[0].first;
+    Diag(Path[0].getLoc(), diag::warn_reserved_module_name)
+        << Path[0].getIdentifierInfo();
 
   // Then test all of the components in the path to see if any of them are
   // using another kind of reserved or invalid identifier.
   for (auto Part : Path) {
-    if (DiagReservedModuleName(*this, Part.first, Part.second))
+    if (DiagReservedModuleName(*this, Part.getIdentifierInfo(), Part.getLoc()))
       return nullptr;
   }
 
@@ -376,10 +378,10 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
   // correct.
   if (!getLangOpts().CurrentModule.empty() &&
       getLangOpts().CurrentModule != ModuleName) {
-    Diag(Path.front().second, diag::err_current_module_name_mismatch)
-        << SourceRange(Path.front().second, IsPartition
-                                                ? Partition.back().second
-                                                : Path.back().second)
+    Diag(Path.front().getLoc(), diag::err_current_module_name_mismatch)
+        << SourceRange(Path.front().getLoc(), IsPartition
+                                                  ? Partition.back().getLoc()
+                                                  : Path.back().getLoc())
         << getLangOpts().CurrentModule;
     return nullptr;
   }
@@ -394,7 +396,7 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
     // We can't have parsed or imported a definition of this module or parsed a
     // module map defining it already.
     if (auto *M = Map.findModule(ModuleName)) {
-      Diag(Path[0].second, diag::err_module_redefinition) << ModuleName;
+      Diag(Path[0].getLoc(), diag::err_module_redefinition) << ModuleName;
       if (M->DefinitionLoc.isValid())
         Diag(M->DefinitionLoc, diag::note_prev_module_definition);
       else if (OptionalFileEntryRef FE = M->getASTFile())
@@ -417,8 +419,8 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
     // keyword nor a module-partition implicitly imports the primary
     // module interface unit of the module as if by a module-import-
     // declaration.
-    std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc(
-        PP.getIdentifierInfo(ModuleName), Path[0].second);
+    IdentifierLoc ModuleNameLoc(Path[0].getLoc(),
+                                PP.getIdentifierInfo(ModuleName));
 
     // The module loader will assume we're trying to import the module that
     // we're building if `LangOpts.CurrentModule` equals to 'ModuleName'.
@@ -490,7 +492,7 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
 
     // Make the import decl for the interface in the impl module.
     ImportDecl *Import = ImportDecl::Create(Context, CurContext, ModuleLoc,
-                                            Interface, Path[0].second);
+                                            Interface, Path[0].getLoc());
     CurContext->addDecl(Import);
 
     // Sequence initialization of the imported module before that of the current
@@ -579,7 +581,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
 
   // For a C++20 module name, flatten into a single identifier with the source
   // location of the first component.
-  std::pair<IdentifierInfo *, SourceLocation> ModuleNameLoc;
+  IdentifierLoc ModuleNameLoc;
 
   std::string ModuleName;
   if (IsPartition) {
@@ -591,11 +593,13 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
     ModuleName = NamedMod->getPrimaryModuleInterfaceName().str();
     ModuleName += ":";
     ModuleName += stringFromPath(Path);
-    ModuleNameLoc = {PP.getIdentifierInfo(ModuleName), Path[0].second};
+    ModuleNameLoc =
+        IdentifierLoc(Path[0].getLoc(), PP.getIdentifierInfo(ModuleName));
     Path = ModuleIdPath(ModuleNameLoc);
   } else if (getLangOpts().CPlusPlusModules) {
     ModuleName = stringFromPath(Path);
-    ModuleNameLoc = {PP.getIdentifierInfo(ModuleName), Path[0].second};
+    ModuleNameLoc =
+        IdentifierLoc(Path[0].getLoc(), PP.getIdentifierInfo(ModuleName));
     Path = ModuleIdPath(ModuleNameLoc);
   }
 
@@ -680,7 +684,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
       IdentifierLocs.push_back(SourceLocation());
   } else if (getLangOpts().CPlusPlusModules && !Mod->Parent) {
     // A single identifier for the whole name.
-    IdentifierLocs.push_back(Path[0].second);
+    IdentifierLocs.push_back(Path[0].getLoc());
   } else {
     Module *ModCheck = Mod;
     for (unsigned I = 0, N = Path.size(); I != N; ++I) {
@@ -690,7 +694,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
         break;
       ModCheck = ModCheck->Parent;
 
-      IdentifierLocs.push_back(Path[I].second);
+      IdentifierLocs.push_back(Path[I].getLoc());
     }
   }
 
@@ -707,7 +711,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
   if (getLangOpts().CPlusPlusModules && ExportLoc.isValid() &&
       Mod->Kind == Module::ModuleKind::ModulePartitionImplementation) {
     Diag(ExportLoc, diag::err_export_partition_impl)
-        << SourceRange(ExportLoc, Path.back().second);
+        << SourceRange(ExportLoc, Path.back().getLoc());
   } else if (!ModuleScopes.empty() && !currentModuleIsImplementation()) {
     // Re-export the module if the imported module is exported.
     // Note that we don't need to add re-exported module to Imports field
diff --git a/clang/lib/Sema/SemaObjC.cpp b/clang/lib/Sema/SemaObjC.cpp
index 073d9791d037b..9b24b5f052119 100644
--- a/clang/lib/Sema/SemaObjC.cpp
+++ b/clang/lib/Sema/SemaObjC.cpp
@@ -1446,10 +1446,8 @@ SemaObjC::ObjCSubscriptKind SemaObjC::CheckSubscriptingKind(Expr *FromE) {
 
 void SemaObjC::AddCFAuditedAttribute(Decl *D) {
   ASTContext &Context = getASTContext();
-  IdentifierInfo *Ident;
-  SourceLocation Loc;
-  std::tie(Ident, Loc) = SemaRef.PP.getPragmaARCCFCodeAuditedInfo();
-  if (!Loc.isValid())
+  auto IdLoc = SemaRef.PP.getPragmaARCCFCodeAuditedInfo();
+  if (!IdLoc.getLoc().isValid())
     return;
 
   // Don't add a redundant or conflicting attribute.
@@ -1457,7 +1455,8 @@ void SemaObjC::AddCFAuditedAttribute(Decl *D) {
       D->hasAttr<CFUnknownTransferAttr>())
     return;
 
-  AttributeCommonInfo Info(Ident, SourceRange(Loc),
+  AttributeCommonInfo Info(IdLoc.getIdentifierInfo(),
+                           SourceRange(IdLoc.getLoc()),
                            AttributeCommonInfo::Form::Pragma());
   D->addAttr(CFAuditedTransferAttr::CreateImplicit(Context, Info));
 }
@@ -1642,8 +1641,10 @@ void SemaObjC::handleMethodFamilyAttr(Decl *D, const ParsedAttr &AL) {
 
   IdentifierLoc *IL = AL.getArgAsIdent(0);
   ObjCMethodFamilyAttr::FamilyKind F;
-  if (!ObjCMethodFamilyAttr::ConvertStrToFamilyKind(IL->Ident->getName(), F)) {
-    Diag(IL->Loc, diag::warn_attribute_type_not_supported) << AL << IL->Ident;
+  if (!ObjCMethodFamilyAttr::ConvertStrToFamilyKind(
+          IL->getIdentifierInfo()->getName(), F)) {
+    Diag(IL->getLoc(), diag::warn_attribute_type_not_supported)
+        << AL << IL->getIdentifierInfo();
     return;
   }
 
@@ -1706,7 +1707,7 @@ void SemaObjC::handleBlocksAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   BlocksAttr::BlockType type;
   if (!BlocksAttr::ConvertStrToBlockType(II->getName(), type)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
@@ -1998,7 +1999,7 @@ void SemaObjC::handleNSErrorDomain(Decl *D, const ParsedAttr &Attr) {
 
   IdentifierLoc *IdentLoc =
       Attr.isArgIdent(0) ? Attr.getArgAsIdent(0) : nullptr;
-  if (!IdentLoc || !IdentLoc->Ident) {
+  if (!IdentLoc || !IdentLoc->getIdentifierInfo()) {
     // Try to locate the argument directly.
     SourceLocation Loc = Attr.getLoc();
     if (Attr.isArgExpr(0) && Attr.getArgAsExpr(0))
@@ -2009,18 +2010,18 @@ void SemaObjC::handleNSErrorDomain(Decl *D, const ParsedAttr &Attr) {
   }
 
   // Verify that the identifier is a valid decl in the C decl namespace.
-  LookupResult Result(SemaRef, DeclarationName(IdentLoc->Ident),
+  LookupResult Result(SemaRef, DeclarationName(IdentLoc->getIdentifierInfo()),
                       SourceLocation(),
                       Sema::LookupNameKind::LookupOrdinaryName);
   if (!SemaRef.LookupName(Result, SemaRef.TUScope) ||
       !Result.getAsSingle<VarDecl>()) {
-    Diag(IdentLoc->Loc, diag::err_nserrordomain_invalid_decl)
-        << 1 << IdentLoc->Ident;
+    Diag(IdentLoc->getLoc(), diag::err_nserrordomain_invalid_decl)
+        << 1 << IdentLoc->getIdentifierInfo();
     return;
   }
 
-  D->addAttr(::new (getASTContext())
-                 NSErrorDomainAttr(getASTContext(), Attr, IdentLoc->Ident));
+  D->addAttr(::new (getASTContext()) NSErrorDomainAttr(
+      getASTContext(), Attr, IdentLoc->getIdentifierInfo()));
 }
 
 void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
@@ -2033,7 +2034,7 @@ void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
 
   // Typedefs only allow objc_bridge(id) and have some additional checking.
   if (const auto *TD = dyn_cast<TypedefNameDecl>(D)) {
-    if (!Parm->Ident->isStr("id")) {
+    if (!Parm->getIdentifierInfo()->isStr("id")) {
       Diag(AL.getLoc(), diag::err_objc_attr_typedef_not_id) << AL;
       return;
     }
@@ -2046,8 +2047,8 @@ void SemaObjC::handleBridgeAttr(Decl *D, const ParsedAttr &AL) {
     }
   }
 
-  D->addAttr(::new (getASTContext())
-                 ObjCBridgeAttr(getASTContext(), AL, Parm->Ident));
+  D->addAttr(::new (getASTContext()) ObjCBridgeAttr(getASTContext(), AL,
+                                                    Parm->getIdentifierInfo()));
 }
 
 void SemaObjC::handleBridgeMutableAttr(Decl *D, const ParsedAttr &AL) {
@@ -2058,21 +2059,21 @@ void SemaObjC::handleBridgeMutableAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  D->addAttr(::new (getASTContext())
-                 ObjCBridgeMutableAttr(getASTContext(), AL, Parm->Ident));
+  D->addAttr(::new (getASTContext()) ObjCBridgeMutableAttr(
+      getASTContext(), AL, Parm->getIdentifierInfo()));
 }
 
 void SemaObjC::handleBridgeRelatedAttr(Decl *D, const ParsedAttr &AL) {
   IdentifierInfo *RelatedClass =
-      AL.isArgIdent(0) ? AL.getArgAsIdent(0)->Ident : nullptr;
+      AL.isArgIdent(0) ? AL.getArgAsIdent(0)->getIdentifierInfo() : nullptr;
   if (!RelatedClass) {
     Diag(D->getBeginLoc(), diag::err_objc_attr_not_id) << AL << 0;
     return;
   }
   IdentifierInfo *ClassMethod =
-      AL.getArgAsIdent(1) ? AL.getArgAsIdent(1)->Ident : nullptr;
+      AL.getArgAsIdent(1) ? AL.getArgAsIdent(1)->getIdentifierInfo() : nullptr;
   IdentifierInfo *InstanceMethod =
-      AL.getArgAsIdent(2) ? AL.getArgAsIdent(2)->Ident : nullptr;
+      AL.getArgAsIdent(2) ? AL.getArgAsIdent(2)->getIdentifierInfo() : nullptr;
   D->addAttr(::new (getASTContext()) ObjCBridgeRelatedAttr(
       getASTContext(), AL, RelatedClass, ClassMethod, InstanceMethod));
 }
diff --git a/clang/lib/Sema/SemaOpenACCClause.cpp b/clang/lib/Sema/SemaOpenACCClause.cpp
index ab25dcfd1a081..049baead031a1 100644
--- a/clang/lib/Sema/SemaOpenACCClause.cpp
+++ b/clang/lib/Sema/SemaOpenACCClause.cpp
@@ -1343,7 +1343,7 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
   // the limitation, since the Dialect requires this.
   if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Set &&
       Clause.getDeviceTypeArchitectures().size() > 1) {
-    SemaRef.Diag(Clause.getDeviceTypeArchitectures()[1].second,
+    SemaRef.Diag(Clause.getDeviceTypeArchitectures()[1].getLoc(),
                  diag::err_acc_device_type_multiple_archs);
     return nullptr;
   }
@@ -1369,16 +1369,17 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
   bool Diagnosed = false;
   auto FilterPred = [&](const DeviceTypeArgument &Arch) {
     // The '*' case.
-    if (!Arch.first)
+    if (!Arch.getIdentifierInfo())
       return false;
     return llvm::find_if(ValidValues, [&](StringRef RHS) {
-             return Arch.first->getName().equals_insensitive(RHS);
+             return Arch.getIdentifierInfo()->getName().equals_insensitive(RHS);
            }) == ValidValues.end();
   };
 
   auto Diagnose = [&](const DeviceTypeArgument &Arch) {
-    Diagnosed = SemaRef.Diag(Arch.second, diag::err_acc_invalid_default_type)
-                << Arch.first << Clause.getClauseKind() << ValidValuesString;
+    Diagnosed = SemaRef.Diag(Arch.getLoc(), diag::err_acc_invalid_default_type)
+                << Arch.getIdentifierInfo() << Clause.getClauseKind()
+                << ValidValuesString;
   };
 
   // There aren't stable enumertor versions of 'for-each-then-erase', so do it
diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index 2f719c6d7a21e..a09626c3a9a8c 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -79,9 +79,10 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
   Expr *ValueExpr = A.getArgAsExpr(3);
 
   StringRef PragmaName =
-      llvm::StringSwitch<StringRef>(PragmaNameLoc->Ident->getName())
+      llvm::StringSwitch<StringRef>(
+          PragmaNameLoc->getIdentifierInfo()->getName())
           .Cases("unroll", "nounroll", "unroll_and_jam", "nounroll_and_jam",
-                 PragmaNameLoc->Ident->getName())
+                 PragmaNameLoc->getIdentifierInfo()->getName())
           .Default("clang loop");
 
   // This could be handled automatically by adding a Subjects definition in
@@ -127,10 +128,10 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
       SetHints(LoopHintAttr::UnrollAndJam, LoopHintAttr::Enable);
   } else {
     // #pragma clang loop ...
-    assert(OptionLoc && OptionLoc->Ident &&
+    assert(OptionLoc && OptionLoc->getIdentifierInfo() &&
            "Attribute must have valid option info.");
     Option = llvm::StringSwitch<LoopHintAttr::OptionType>(
-                 OptionLoc->Ident->getName())
+                 OptionLoc->getIdentifierInfo()->getName())
                  .Case("vectorize", LoopHintAttr::Vectorize)
                  .Case("vectorize_width", LoopHintAttr::VectorizeWidth)
                  .Case("interleave", LoopHintAttr::Interleave)
@@ -144,12 +145,13 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                  .Case("distribute", LoopHintAttr::Distribute)
                  .Default(LoopHintAttr::Vectorize);
     if (Option == LoopHintAttr::VectorizeWidth) {
-      assert((ValueExpr || (StateLoc && StateLoc->Ident)) &&
+      assert((ValueExpr || (StateLoc && StateLoc->getIdentifierInfo())) &&
              "Attribute must have a valid value expression or argument.");
       if (ValueExpr && S.CheckLoopHintExpr(ValueExpr, St->getBeginLoc(),
                                            /*AllowZero=*/false))
         return nullptr;
-      if (StateLoc && StateLoc->Ident && StateLoc->Ident->isStr("scalable"))
+      if (StateLoc && StateLoc->getIdentifierInfo() &&
+          StateLoc->getIdentifierInfo()->isStr("scalable"))
         State = LoopHintAttr::ScalableWidth;
       else
         State = LoopHintAttr::FixedWidth;
@@ -167,14 +169,15 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                Option == LoopHintAttr::Unroll ||
                Option == LoopHintAttr::Distribute ||
                Option == LoopHintAttr::PipelineDisabled) {
-      assert(StateLoc && StateLoc->Ident && "Loop hint must have an argument");
-      if (StateLoc->Ident->isStr("disable"))
+      assert(StateLoc && StateLoc->getIdentifierInfo() &&
+             "Loop hint must have an argument");
+      if (StateLoc->getIdentifierInfo()->isStr("disable"))
         State = LoopHintAttr::Disable;
-      else if (StateLoc->Ident->isStr("assume_safety"))
+      else if (StateLoc->getIdentifierInfo()->isStr("assume_safety"))
         State = LoopHintAttr::AssumeSafety;
-      else if (StateLoc->Ident->isStr("full"))
+      else if (StateLoc->getIdentifierInfo()->isStr("full"))
         State = LoopHintAttr::Full;
-      else if (StateLoc->Ident->isStr("enable"))
+      else if (StateLoc->getIdentifierInfo()->isStr("enable"))
         State = LoopHintAttr::Enable;
       else
         llvm_unreachable("bad loop hint argument");
@@ -644,8 +647,8 @@ static Attr *handleAtomicAttr(Sema &S, Stmt *St, const ParsedAttr &AL,
     }
 
     IdentifierLoc *Ident = AL.getArgAsIdent(ArgIndex);
-    OptionString = Ident->Ident->getName();
-    Loc = Ident->Loc;
+    OptionString = Ident->getIdentifierInfo()->getName();
+    Loc = Ident->getLoc();
     if (!AtomicAttr::ConvertStrToConsumedOption(OptionString, Option)) {
       S.Diag(Loc, diag::err_attribute_invalid_atomic_argument) << OptionString;
       return nullptr;
diff --git a/clang/lib/Sema/SemaSwift.cpp b/clang/lib/Sema/SemaSwift.cpp
index fe72d6c85c37a..4aae855a24b8f 100644
--- a/clang/lib/Sema/SemaSwift.cpp
+++ b/clang/lib/Sema/SemaSwift.cpp
@@ -148,8 +148,8 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
       return true;
 
     S.Diag(AL.getLoc(), diag::err_attr_swift_error_return_type)
-        << AL << AL.getArgAsIdent(0)->Ident->getName() << isa<ObjCMethodDecl>(D)
-        << /*pointer*/ 1;
+        << AL << AL.getArgAsIdent(0)->getIdentifierInfo()->getName()
+        << isa<ObjCMethodDecl>(D) << /*pointer*/ 1;
     return false;
   };
 
@@ -159,8 +159,8 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
       return true;
 
     S.Diag(AL.getLoc(), diag::err_attr_swift_error_return_type)
-        << AL << AL.getArgAsIdent(0)->Ident->getName() << isa<ObjCMethodDecl>(D)
-        << /*integral*/ 0;
+        << AL << AL.getArgAsIdent(0)->getIdentifierInfo()->getName()
+        << isa<ObjCMethodDecl>(D) << /*integral*/ 0;
     return false;
   };
 
@@ -169,10 +169,10 @@ void SemaSwift::handleError(Decl *D, const ParsedAttr &AL) {
 
   IdentifierLoc *Loc = AL.getArgAsIdent(0);
   SwiftErrorAttr::ConventionKind Convention;
-  if (!SwiftErrorAttr::ConvertStrToConventionKind(Loc->Ident->getName(),
-                                                  Convention)) {
+  if (!SwiftErrorAttr::ConvertStrToConventionKind(
+          Loc->getIdentifierInfo()->getName(), Convention)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported)
-        << AL << Loc->Ident;
+        << AL << Loc->getIdentifierInfo();
     return;
   }
 
@@ -287,10 +287,10 @@ static void checkSwiftAsyncErrorBlock(Sema &S, Decl *D,
 void SemaSwift::handleAsyncError(Decl *D, const ParsedAttr &AL) {
   IdentifierLoc *IDLoc = AL.getArgAsIdent(0);
   SwiftAsyncErrorAttr::ConventionKind ConvKind;
-  if (!SwiftAsyncErrorAttr::ConvertStrToConventionKind(IDLoc->Ident->getName(),
-                                                       ConvKind)) {
+  if (!SwiftAsyncErrorAttr::ConvertStrToConventionKind(
+          IDLoc->getIdentifierInfo()->getName(), ConvKind)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported)
-        << AL << IDLoc->Ident;
+        << AL << IDLoc->getIdentifierInfo();
     return;
   }
 
@@ -643,7 +643,7 @@ void SemaSwift::handleNewType(Decl *D, const ParsedAttr &AL) {
   }
 
   SwiftNewTypeAttr::NewtypeKind Kind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   if (!SwiftNewTypeAttr::ConvertStrToNewtypeKind(II->getName(), Kind)) {
     Diag(AL.getLoc(), diag::warn_attribute_type_not_supported) << AL << II;
     return;
@@ -667,7 +667,7 @@ void SemaSwift::handleAsyncAttr(Decl *D, const ParsedAttr &AL) {
   }
 
   SwiftAsyncAttr::Kind Kind;
-  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->getIdentifierInfo();
   if (!SwiftAsyncAttr::ConvertStrToKind(II->getName(), Kind)) {
     Diag(AL.getLoc(), diag::err_swift_async_no_access) << AL << II;
     return;
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index dc7e3a0bf8875..87682233c5246 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -14,6 +14,7 @@
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Sema/Lookup.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
@@ -755,7 +756,7 @@ bool Sema::CheckParameterPacksForExpansion(
     bool &RetainExpansion, UnsignedOrNone &NumExpansions) {
   ShouldExpand = true;
   RetainExpansion = false;
-  std::pair<IdentifierInfo *, SourceLocation> FirstPack;
+  IdentifierLoc FirstPack;
   bool HaveFirstPack = false;
   UnsignedOrNone NumPartialExpansions = std::nullopt;
   SourceLocation PartiallySubstitutedPackLoc;
@@ -867,8 +868,7 @@ bool Sema::CheckParameterPacksForExpansion(
       // This is the first pack we've seen for which we have an argument.
       // Record it.
       NumExpansions = NewPackSize;
-      FirstPack.first = Name;
-      FirstPack.second = ParmPack.second;
+      FirstPack = IdentifierLoc(ParmPack.second, Name);
       HaveFirstPack = true;
       continue;
     }
@@ -905,9 +905,9 @@ bool Sema::CheckParameterPacksForExpansion(
       //   the same number of arguments specified.
       if (HaveFirstPack)
         Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict)
-            << FirstPack.first << Name << *NumExpansions
+            << FirstPack.getIdentifierInfo() << Name << *NumExpansions
             << (LeastNewPackSize != NewPackSize) << LeastNewPackSize
-            << SourceRange(FirstPack.second) << SourceRange(ParmPack.second);
+            << SourceRange(FirstPack.getLoc()) << SourceRange(ParmPack.second);
       else
         Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict_multilevel)
             << Name << *NumExpansions << (LeastNewPackSize != NewPackSize)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index eba7267904fb2..6e7ee8b5506ff 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -99,8 +99,8 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
   StringRef name = attr.getAttrName()->getName();
 
   // The GC attributes are usually written with macros;  special-case them.
-  IdentifierInfo *II = attr.isArgIdent(0) ? attr.getArgAsIdent(0)->Ident
-                                          : nullptr;
+  IdentifierInfo *II =
+      attr.isArgIdent(0) ? attr.getArgAsIdent(0)->getIdentifierInfo() : nullptr;
   if (useExpansionLoc && loc.isMacroID() && II) {
     if (II->isStr("strong")) {
       if (S.findMacroSpelling(loc, "__strong")) name = "__strong";
@@ -5732,8 +5732,7 @@ static void transferARCOwnershipToDeclaratorChunk(TypeProcessingState &state,
   }
 
   IdentifierLoc *Arg = new (S.Context) IdentifierLoc;
-  Arg->Ident = &S.Context.Idents.get(attrStr);
-  Arg->Loc = SourceLocation();
+  Arg->setIdentifierInfo(&S.Context.Idents.get(attrStr));
 
   ArgsUnion Args(Arg);
 
@@ -6633,7 +6632,7 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
     return true;
   }
 
-  IdentifierInfo *II = attr.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = attr.getArgAsIdent(0)->getIdentifierInfo();
   Qualifiers::ObjCLifetime lifetime;
   if (II->isStr("none"))
     lifetime = Qualifiers::OCL_ExplicitNone;
@@ -6811,7 +6810,7 @@ static bool handleObjCGCTypeAttr(TypeProcessingState &state, ParsedAttr &attr,
     return true;
   }
 
-  IdentifierInfo *II = attr.getArgAsIdent(0)->Ident;
+  IdentifierInfo *II = attr.getArgAsIdent(0)->getIdentifierInfo();
   if (II->isStr("weak"))
     GCAttr = Qualifiers::Weak;
   else if (II->isStr("strong"))
@@ -7541,7 +7540,7 @@ static Attr *getCCTypeAttr(ASTContext &Ctx, ParsedAttr &Attr) {
     if (Attr.isArgExpr(0))
       Str = cast<StringLiteral>(Attr.getArgAsExpr(0))->getString();
     else
-      Str = Attr.getArgAsIdent(0)->Ident->getName();
+      Str = Attr.getArgAsIdent(0)->getIdentifierInfo()->getName();
     PcsAttr::PCSType Type;
     if (!PcsAttr::ConvertStrToPCSType(Str, Type))
       llvm_unreachable("already validated the attribute");
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 58d3d33b31644..d683b7930caf5 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12818,7 +12818,7 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     for (unsigned I = 0; I < NumArchs; ++I) {
       IdentifierInfo *Ident = readBool() ? readIdentifier() : nullptr;
       SourceLocation Loc = readSourceLocation();
-      Archs.emplace_back(Ident, Loc);
+      Archs.emplace_back(Loc, Ident);
     }
 
     return OpenACCDeviceTypeClause::Create(getContext(), ClauseKind, BeginLoc,
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 8c261e13d5ea4..5b59087a5a164 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8780,10 +8780,10 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeSourceLocation(DTC->getLParenLoc());
     writeUInt32(DTC->getArchitectures().size());
     for (const DeviceTypeArgument &Arg : DTC->getArchitectures()) {
-      writeBool(Arg.first);
-      if (Arg.first)
-        AddIdentifierRef(Arg.first);
-      writeSourceLocation(Arg.second);
+      writeBool(Arg.getIdentifierInfo());
+      if (Arg.getIdentifierInfo())
+        AddIdentifierRef(Arg.getIdentifierInfo());
+      writeSourceLocation(Arg.getLoc());
     }
     return;
   }
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 429bf823616da..07856dbdba4b4 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -664,7 +664,7 @@ void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
                                         const Module *Imported) {
   if (MDC.ScanInstance.getPreprocessor().isInImportingCXXNamedModules()) {
     P1689ModuleInfo RequiredModule;
-    RequiredModule.ModuleName = Path[0].first->getName().str();
+    RequiredModule.ModuleName = Path[0].getIdentifierInfo()->getName().str();
     RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
     MDC.RequiredStdCXXModules.push_back(RequiredModule);
     return;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
index 5a78ce8fa7688..078c9312ce4a7 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
@@ -13,6 +13,7 @@
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/DarwinSDKInfo.h"
 #include "clang/Basic/DiagnosticIDs.h"
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Version.h"
@@ -135,8 +136,9 @@ class ClangExpressionParser::LLDBPreprocessorCallbacks : public PPCallbacks {
 
     SourceModule module;
 
-    for (const std::pair<IdentifierInfo *, SourceLocation> &component : path)
-      module.path.push_back(ConstString(component.first->getName()));
+    for (const IdentifierLoc &component : path)
+      module.path.push_back(
+          ConstString(component.getIdentifierInfo()->getName()));
 
     StreamString error_stream;
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
index 887db0b8fc575..5b1353454a1c6 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
@@ -8,6 +8,7 @@
 
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticFrontend.h"
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendActions.h"
@@ -334,20 +335,18 @@ bool ClangModulesDeclVendorImpl::AddModule(const SourceModule &module,
     return false;
   }
 
-  llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
-                    4>
-      clang_path;
+  llvm::SmallVector<clang::IdentifierLoc, 4> clang_path;
 
   {
     clang::SourceManager &source_manager =
         m_compiler_instance->getASTContext().getSourceManager();
 
     for (ConstString path_component : module.path) {
-      clang_path.push_back(std::make_pair(
-          &m_compiler_instance->getASTContext().Idents.get(
-              path_component.GetStringRef()),
+      clang_path.emplace_back(
           source_manager.getLocForStartOfFile(source_manager.getMainFileID())
-              .getLocWithOffset(m_source_location_index++)));
+              .getLocWithOffset(m_source_location_index++),
+          &m_compiler_instance->getASTContext().Idents.get(
+              path_component.GetStringRef()));
     }
   }
 
@@ -629,8 +628,8 @@ ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
 
   const bool is_inclusion_directive = false;
 
-  return m_compiler_instance->loadModule(path.front().second, path, visibility,
-                                         is_inclusion_directive);
+  return m_compiler_instance->loadModule(path.front().getLoc(), path,
+                                         visibility, is_inclusion_directive);
 }
 
 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";

From 1467b3b0a86fcd4a6b3225fbafd959ee781b5b5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Csan=C3=A1d=20Hajd=C3=BA?= <csanad.hajdu@arm.com>
Date: Thu, 17 Apr 2025 16:40:57 +0200
Subject: [PATCH 280/710] [Clang][Docs] Add release note for execute-only
 support on AArch64 (#134802)

---
 clang/docs/ReleaseNotes.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0ea8498351c24..0a46bbea96f6c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -219,6 +219,8 @@ Modified Compiler Flags
 
 - `-Wpadded` option implemented for the `x86_64-windows-msvc` target. Fixes #61702
 
+- The ``-mexecute-only`` and ``-mpure-code`` flags are now accepted for AArch64 targets. (#GH125688)
+
 Removed Compiler Flags
 -------------------------
 
@@ -569,6 +571,9 @@ Arm and AArch64 Support
 
 -  Support for __ptrauth type qualifier has been added.
 
+- For AArch64, added support for generating executable-only code sections by using the
+  ``-mexecute-only`` or ``-mpure-code`` compiler flags. (#GH125688)
+
 Android Support
 ^^^^^^^^^^^^^^^
 

From 2007dcfeb86fe272fcd16283f9bca45292e3c630 Mon Sep 17 00:00:00 2001
From: Snehasish Kumar <snehasishk@google.com>
Date: Thu, 17 Apr 2025 08:22:19 -0700
Subject: [PATCH 281/710] Reapply [Metadata] Preserve MD_prof when merging
 instructions when one is missing. (#135418)

Preserve branch weight metadata when merging instructions if one of the
instructions is missing metadata. This is similar in behaviour to what
we do today for other types of metadata such as mmra, memprof and
callsite metadata.

Also add a legality check when merging prof metadata based on
instruction type. Without this check GVN PRE optimizations result in
prof metadata on phi nodes which break the module verifier.

Build failure caught by
https://lab.llvm.org/buildbot/#/builders/113/builds/6621
```
!9185 = !{!"branch_weights", i32 3912, i32 802}
Wrong number of operands
!9185 = !{!"branch_weights", i32 3912, i32 802}
fatal error: error in backend: Broken module found, compilation aborted!
```

Reverts #134200 with additional changes.
---
 llvm/lib/IR/Metadata.cpp                      | 20 ++++++
 llvm/lib/Transforms/Utils/Local.cpp           | 19 ++++--
 .../GVN/pre-invalid-prof-metadata.ll          | 60 ++++++++++++++++++
 ...rect-call-branch-weights-preserve-hoist.ll | 62 ++++++++++++++++++
 ...irect-call-branch-weights-preserve-sink.ll | 63 +++++++++++++++++++
 5 files changed, 218 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/Transforms/GVN/pre-invalid-prof-metadata.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-preserve-hoist.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-preserve-sink.ll

diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index bc722c7ed3e31..8e78cd9cc573a 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -1223,6 +1223,26 @@ MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
 MDNode *MDNode::getMergedProfMetadata(MDNode *A, MDNode *B,
                                       const Instruction *AInstr,
                                       const Instruction *BInstr) {
+  // Check that it is legal to merge prof metadata based on the opcode.
+  auto IsLegal = [](const Instruction &I) -> bool {
+    switch (I.getOpcode()) {
+    case Instruction::Invoke:
+    case Instruction::Br:
+    case Instruction::Switch:
+    case Instruction::Call:
+    case Instruction::IndirectBr:
+    case Instruction::Select:
+    case Instruction::CallBr:
+      return true;
+    default:
+      return false;
+    }
+  };
+  if (AInstr && !IsLegal(*AInstr))
+    return nullptr;
+  if (BInstr && !IsLegal(*BInstr))
+    return nullptr;
+
   if (!(A && B)) {
     return A ? A : B;
   }
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 2f3ea2266e07f..b3204da93049b 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -3353,9 +3353,10 @@ static void combineMetadata(Instruction *K, const Instruction *J,
       case LLVMContext::MD_invariant_group:
         // Preserve !invariant.group in K.
         break;
-      // Keep empty cases for mmra, memprof, and callsite to prevent them from
-      // being removed as unknown metadata. The actual merging is handled
+      // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
+      // from being removed as unknown metadata. The actual merging is handled
       // separately below.
+      case LLVMContext::MD_prof:
       case LLVMContext::MD_mmra:
       case LLVMContext::MD_memprof:
       case LLVMContext::MD_callsite:
@@ -3384,10 +3385,6 @@ static void combineMetadata(Instruction *K, const Instruction *J,
         if (!AAOnly)
           K->setMetadata(Kind, JMD);
         break;
-      case LLVMContext::MD_prof:
-        if (!AAOnly && DoesKMove)
-          K->setMetadata(Kind, MDNode::getMergedProfMetadata(KMD, JMD, K, J));
-        break;
       case LLVMContext::MD_noalias_addrspace:
         if (DoesKMove)
           K->setMetadata(Kind,
@@ -3434,6 +3431,16 @@ static void combineMetadata(Instruction *K, const Instruction *J,
     K->setMetadata(LLVMContext::MD_callsite,
                    MDNode::getMergedCallsiteMetadata(KCallSite, JCallSite));
   }
+
+  // Merge prof metadata.
+  // Handle separately to support cases where only one instruction has the
+  // metadata.
+  auto *JProf = J->getMetadata(LLVMContext::MD_prof);
+  auto *KProf = K->getMetadata(LLVMContext::MD_prof);
+  if (!AAOnly && (JProf || KProf)) {
+    K->setMetadata(LLVMContext::MD_prof,
+                   MDNode::getMergedProfMetadata(KProf, JProf, K, J));
+  }
 }
 
 void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
diff --git a/llvm/test/Transforms/GVN/pre-invalid-prof-metadata.ll b/llvm/test/Transforms/GVN/pre-invalid-prof-metadata.ll
new file mode 100644
index 0000000000000..978bfd424ae52
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pre-invalid-prof-metadata.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=gvn -S < %s | FileCheck %s
+
+; Make sure that performing PRE does not add prof metadata onto newly created
+; phi nodes.
+define fastcc ptr @foo(i8 %__pred.1.val) {
+; CHECK-LABEL: define fastcc ptr @foo(
+; CHECK-SAME: i8 [[__PRED_1_VAL:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    switch i64 0, label %[[SW_BB:.*]] [
+; CHECK-NEXT:      i64 1, label %[[ENTRY_SW_BB26_CRIT_EDGE:.*]]
+; CHECK-NEXT:      i64 0, label %[[ENTRY_SW_BB21_CRIT_EDGE:.*]]
+; CHECK-NEXT:    ]
+; CHECK:       [[ENTRY_SW_BB21_CRIT_EDGE]]:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = trunc i8 [[__PRED_1_VAL]] to i1
+; CHECK-NEXT:    [[DOTPRE1:%.*]] = select i1 [[DOTPRE]], i64 0, i64 4
+; CHECK-NEXT:    br label %[[SW_BB21:.*]]
+; CHECK:       [[ENTRY_SW_BB26_CRIT_EDGE]]:
+; CHECK-NEXT:    [[DOTPRE2:%.*]] = trunc i8 [[__PRED_1_VAL]] to i1
+; CHECK-NEXT:    [[DOTPRE3:%.*]] = select i1 [[DOTPRE2]], i64 0, i64 4, !prof [[PROF0:![0-9]+]]
+; CHECK-NEXT:    br label %[[SW_BB26:.*]]
+; CHECK:       [[SW_BB]]:
+; CHECK-NEXT:    [[L78:%.*]] = trunc i8 [[__PRED_1_VAL]] to i1
+; CHECK-NEXT:    [[C79:%.*]] = select i1 [[L78]], i64 0, i64 4
+; CHECK-NEXT:    br label %[[SW_BB21]]
+; CHECK:       [[SW_BB21]]:
+; CHECK-NEXT:    [[C84_PRE_PHI:%.*]] = phi i64 [ [[DOTPRE1]], %[[ENTRY_SW_BB21_CRIT_EDGE]] ], [ [[C79]], %[[SW_BB]] ]
+; CHECK-NEXT:    [[L83_PRE_PHI:%.*]] = phi i1 [ [[DOTPRE]], %[[ENTRY_SW_BB21_CRIT_EDGE]] ], [ [[L78]], %[[SW_BB]] ]
+; CHECK-NEXT:    br label %[[SW_BB26]]
+; CHECK:       [[SW_BB26]]:
+; CHECK-NEXT:    [[C89_PRE_PHI:%.*]] = phi i64 [ [[DOTPRE3]], %[[ENTRY_SW_BB26_CRIT_EDGE]] ], [ [[C84_PRE_PHI]], %[[SW_BB21]] ]
+; CHECK-NEXT:    [[L88_PRE_PHI:%.*]] = phi i1 [ [[DOTPRE2]], %[[ENTRY_SW_BB26_CRIT_EDGE]] ], [ [[L83_PRE_PHI]], %[[SW_BB21]] ]
+; CHECK-NEXT:    ret ptr null
+;
+entry:
+  switch i64 0, label %sw.bb [
+  i64 1, label %sw.bb26
+  i64 0, label %sw.bb21
+  ]
+
+sw.bb:                                            ; preds = %entry
+  %l78 = trunc i8 %__pred.1.val to i1
+  %c79 = select i1 %l78, i64 0, i64 4
+  br label %sw.bb21
+
+sw.bb21:                                          ; preds = %sw.bb, %entry
+  %l83 = trunc i8 %__pred.1.val to i1
+  %c84 = select i1 %l83, i64 0, i64 4
+  br label %sw.bb26
+
+sw.bb26:                                          ; preds = %sw.bb21, %entry
+  %l88 = trunc i8 %__pred.1.val to i1
+  %c89 = select i1 %l88, i64 0, i64 4, !prof !0
+  ret ptr null
+}
+
+!0 = !{!"branch_weights", i32 3994, i32 883}
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 3994, i32 883}
+;.
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-preserve-hoist.ll b/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-preserve-hoist.ll
new file mode 100644
index 0000000000000..d6058134f5285
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-preserve-hoist.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 2
+; RUN: opt < %s -passes='simplifycfg<no-sink-common-insts;hoist-common-insts>' -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s --check-prefix=HOIST
+
+; Test case based on C++ code with manualy annotated !prof metadata.
+; This is to test that when calls to 'func1' from 'if.then' block
+; and 'if.else' block are hoisted, the branch_weights are merged and
+; attached to merged call rather than dropped.
+;
+; int func1(int a, int b) ;
+; int func2(int a, int b) ;
+
+; int func(int a, int b, bool c) {
+;    int sum= 0;
+;    if(c) {
+;        sum += func1(a, b);
+;    } else {
+;        sum += func1(a, b);
+;        sum -= func2(a, b);
+;    }
+;    return sum;
+; }
+define i32 @_Z4funciib(i32 %a, i32 %b, i1 %c) {
+; HOIST-LABEL: define i32 @_Z4funciib
+; HOIST-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i1 [[C:%.*]]) {
+; HOIST-NEXT:  entry:
+; HOIST-NEXT:    [[CALL:%.*]] = tail call i32 @_Z5func1ii(i32 [[A]], i32 [[B]]), !prof [[PROF0:![0-9]+]]
+; HOIST-NEXT:    br i1 [[C]], label [[IF_END:%.*]], label [[IF_ELSE:%.*]]
+; HOIST:       if.else:
+; HOIST-NEXT:    [[CALL3:%.*]] = tail call i32 @_Z5func2ii(i32 [[A]], i32 [[B]])
+; HOIST-NEXT:    [[SUB:%.*]] = sub i32 [[CALL]], [[CALL3]]
+; HOIST-NEXT:    br label [[IF_END]]
+; HOIST:       if.end:
+; HOIST-NEXT:    [[SUM_0:%.*]] = phi i32 [ [[SUB]], [[IF_ELSE]] ], [ [[CALL]], [[ENTRY:%.*]] ]
+; HOIST-NEXT:    ret i32 [[SUM_0]]
+;
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %call = tail call i32 @_Z5func1ii(i32 %a, i32 %b)
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call1 = tail call i32 @_Z5func1ii(i32 %a, i32 %b), !prof !0
+  %call3 = tail call i32 @_Z5func2ii(i32 %a, i32 %b)
+  %sub = sub i32 %call1, %call3
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %sum.0 = phi i32 [ %call, %if.then ], [ %sub, %if.else ]
+  ret i32 %sum.0
+}
+
+declare i32 @_Z5func1ii(i32, i32)
+
+declare i32 @_Z5func2ii(i32, i32)
+
+!0 = !{!"branch_weights", i32 10}
+
+;.
+; HOIST: [[PROF0]] = !{!"branch_weights", i32 10}
+;.
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-preserve-sink.ll b/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-preserve-sink.ll
new file mode 100644
index 0000000000000..c4aed5eb95888
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-preserve-sink.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 2
+; RUN: opt < %s -passes='simplifycfg<sink-common-insts;no-hoist-common-insts>' -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s --check-prefix=SINK
+
+
+; Test case based on the following C++ code with manualy annotated !prof metadata.
+; This is to test that when calls to 'func1' from 'if.then' and 'if.else' are
+; sinked, the branch weights are merged and attached to sinked call.
+;
+; int func1(int a, int b) ;
+; int func2(int a, int b) ;
+
+; int func(int a, int b, bool c) {
+;    int sum = 0;
+;    if (c) {
+;        sum += func1(a,b);
+;    } else {
+;        b -= func2(a,b);
+;        sum += func1(a,b);
+;    }
+;    return sum;
+; }
+
+define i32 @_Z4funciib(i32 %a, i32 %b, i1 %c) {
+; SINK-LABEL: define i32 @_Z4funciib
+; SINK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i1 [[C:%.*]]) {
+; SINK-NEXT:  entry:
+; SINK-NEXT:    br i1 [[C]], label [[IF_END:%.*]], label [[IF_ELSE:%.*]]
+; SINK:       if.else:
+; SINK-NEXT:    [[CALL1:%.*]] = tail call i32 @_Z5func2ii(i32 [[A]], i32 [[B]])
+; SINK-NEXT:    [[SUB:%.*]] = sub i32 [[B]], [[CALL1]]
+; SINK-NEXT:    br label [[IF_END]]
+; SINK:       if.end:
+; SINK-NEXT:    [[SUB_SINK:%.*]] = phi i32 [ [[SUB]], [[IF_ELSE]] ], [ [[B]], [[ENTRY:%.*]] ]
+; SINK-NEXT:    [[CALL2:%.*]] = tail call i32 @_Z5func1ii(i32 [[A]], i32 [[SUB_SINK]]), !prof [[PROF0:![0-9]+]]
+; SINK-NEXT:    ret i32 [[CALL2]]
+;
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %call = tail call i32 @_Z5func1ii(i32 %a, i32 %b), !prof !0
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call1 = tail call i32 @_Z5func2ii(i32 %a, i32 %b)
+  %sub = sub i32 %b, %call1
+  %call2 = tail call i32 @_Z5func1ii(i32 %a, i32 %sub)
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %sum.0 = phi i32 [ %call, %if.then ], [ %call2, %if.else ]
+  ret i32 %sum.0
+}
+
+declare i32 @_Z5func1ii(i32, i32)
+
+declare i32 @_Z5func2ii(i32, i32)
+
+!0 = !{!"branch_weights", i32 10}
+
+;.
+; SINK: [[PROF0]] = !{!"branch_weights", i32 10}
+;.

From 909a9feda9292be8a7322f07dd167a603309b541 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Thu, 17 Apr 2025 23:23:30 +0800
Subject: [PATCH 282/710] [Reland][Clang][CodeGen][UBSan] Add more precise
 attributes to recoverable ubsan handlers (#135135)

This patch relands https://github.com/llvm/llvm-project/pull/130990.
If the check value is passed by reference, add `memory(read)`.

Original PR description:

This patch adds `memory(argmem: read, inaccessiblemem: readwrite)` to
**recoverable** ubsan handlers in order to unblock some
memory/loop optimizations. It provides an average of 3% performance
improvement on llvm-test-suite (except for 49 test failures due to ubsan
diagnostics).
---
 clang/lib/CodeGen/CGExpr.cpp                  |  31 ++-
 clang/lib/CodeGen/CodeGenFunction.h           |   4 +-
 .../AArch64/ubsan-handler-pass-by-ref.c       |  22 ++
 clang/test/CodeGen/allow-ubsan-check.c        |  28 +--
 clang/test/CodeGen/attr-counted-by.c          | 232 +++++++++---------
 clang/test/CodeGen/ubsan-attr.cpp             | 100 ++++++++
 6 files changed, 280 insertions(+), 137 deletions(-)
 create mode 100644 clang/test/CodeGen/AArch64/ubsan-handler-pass-by-ref.c
 create mode 100644 clang/test/CodeGen/ubsan-attr.cpp

diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index abb88477062fc..1efc626a8f5a6 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3486,7 +3486,8 @@ llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
   return GV;
 }
 
-llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
+llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V,
+                                             bool &MayReadFromPtrToInt) {
   llvm::Type *TargetTy = IntPtrTy;
 
   if (V->getType() == TargetTy)
@@ -3512,6 +3513,7 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
     Builder.CreateStore(V, Ptr);
     V = Ptr.getPointer();
   }
+  MayReadFromPtrToInt = true;
   return Builder.CreatePtrToInt(V, TargetTy);
 }
 
@@ -3617,7 +3619,8 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF,
                                  ArrayRef<llvm::Value *> FnArgs,
                                  SanitizerHandler CheckHandler,
                                  CheckRecoverableKind RecoverKind, bool IsFatal,
-                                 llvm::BasicBlock *ContBB, bool NoMerge) {
+                                 llvm::BasicBlock *ContBB, bool NoMerge,
+                                 bool MayReadFromPtrToInt) {
   assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
   std::optional<ApplyDebugLocation> DL;
   if (!CGF.Builder.getCurrentDebugLocation()) {
@@ -3645,6 +3648,20 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF,
         .addAttribute(llvm::Attribute::NoUnwind);
   }
   B.addUWTableAttr(llvm::UWTableKind::Default);
+  // Add more precise attributes to recoverable ubsan handlers for better
+  // optimizations.
+  if (CGF.CGM.getCodeGenOpts().OptimizationLevel > 0 && MayReturn) {
+    // __ubsan_handle_dynamic_type_cache_miss reads the vtable, which is also
+    // accessible by the current module.
+    if (CheckHandler != SanitizerHandler::DynamicTypeCacheMiss) {
+      llvm::MemoryEffects ME =
+          llvm::MemoryEffects::argMemOnly(llvm::ModRefInfo::Ref) |
+          llvm::MemoryEffects::inaccessibleMemOnly();
+      if (MayReadFromPtrToInt)
+        ME |= llvm::MemoryEffects::readOnly();
+      B.addMemoryAttr(ME);
+    }
+  }
 
   llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
       FnType, FnName,
@@ -3741,6 +3758,7 @@ void CodeGenFunction::EmitCheck(
   // representing operand values.
   SmallVector<llvm::Value *, 4> Args;
   SmallVector<llvm::Type *, 4> ArgTypes;
+  bool MayReadFromPtrToInt = false;
   if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
     Args.reserve(DynamicArgs.size() + 1);
     ArgTypes.reserve(DynamicArgs.size() + 1);
@@ -3760,7 +3778,7 @@ void CodeGenFunction::EmitCheck(
     }
 
     for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
-      Args.push_back(EmitCheckValue(DynamicArgs[i]));
+      Args.push_back(EmitCheckValue(DynamicArgs[i], MayReadFromPtrToInt));
       ArgTypes.push_back(IntPtrTy);
     }
   }
@@ -3772,7 +3790,8 @@ void CodeGenFunction::EmitCheck(
     // Simple case: we need to generate a single handler call, either
     // fatal, or non-fatal.
     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,
-                         (FatalCond != nullptr), Cont, NoMerge);
+                         (FatalCond != nullptr), Cont, NoMerge,
+                         MayReadFromPtrToInt);
   } else {
     // Emit two handler calls: first one for set of unrecoverable checks,
     // another one for recoverable.
@@ -3782,10 +3801,10 @@ void CodeGenFunction::EmitCheck(
     Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
     EmitBlock(FatalHandlerBB);
     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,
-                         NonFatalHandlerBB, NoMerge);
+                         NonFatalHandlerBB, NoMerge, MayReadFromPtrToInt);
     EmitBlock(NonFatalHandlerBB);
     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,
-                         Cont, NoMerge);
+                         Cont, NoMerge, MayReadFromPtrToInt);
   }
 
   EmitBlock(Cont);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 4c5e8a8a44926..9254c7077237f 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5244,7 +5244,9 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// Convert a value into a format suitable for passing to a runtime
   /// sanitizer handler.
-  llvm::Value *EmitCheckValue(llvm::Value *V);
+  /// If the check value is a pointer or passed by reference, set \p
+  /// MayReadFromPtrToInt to true.
+  llvm::Value *EmitCheckValue(llvm::Value *V, bool &MayReadFromPtrToInt);
 
   /// Emit a description of a source location in a format suitable for
   /// passing to a runtime sanitizer handler.
diff --git a/clang/test/CodeGen/AArch64/ubsan-handler-pass-by-ref.c b/clang/test/CodeGen/AArch64/ubsan-handler-pass-by-ref.c
new file mode 100644
index 0000000000000..a8ff941a124a4
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/ubsan-handler-pass-by-ref.c
@@ -0,0 +1,22 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -triple armv7-linux-androideabi24 -emit-llvm -fsanitize=signed-integer-overflow -O3 %s -o - -fsanitize-recover=signed-integer-overflow -w | FileCheck %s --check-prefixes=RECOVER
+// REQUIRES: aarch64-registered-target
+
+#define LLONG_MIN  (-__LONG_LONG_MAX__-1LL)
+
+// RECOVER-LABEL: define dso_local noundef i32 @main(
+// RECOVER-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// RECOVER-NEXT:  [[ENTRY:.*:]]
+// RECOVER-NEXT:    [[TMP:%.*]] = alloca i64, align 8
+// RECOVER-NEXT:    store i64 -9223372036854775808, ptr [[TMP]], align 8, !nosanitize [[META3:![0-9]+]]
+// RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[TMP]] to i32, !nosanitize [[META3]]
+// RECOVER-NEXT:    call void @__ubsan_handle_negate_overflow(ptr nonnull @[[GLOB1:[0-9]+]], i32 [[TMP0]]) #[[ATTR2:[0-9]+]], !nosanitize [[META3]]
+// RECOVER-NEXT:    ret i32 0
+//
+int main() {
+  __builtin_llabs(LLONG_MIN);
+  return 0;
+}
+//.
+// RECOVER: [[META3]] = !{}
+//.
diff --git a/clang/test/CodeGen/allow-ubsan-check.c b/clang/test/CodeGen/allow-ubsan-check.c
index e225fb63f08eb..6358425aa40be 100644
--- a/clang/test/CodeGen/allow-ubsan-check.c
+++ b/clang/test/CodeGen/allow-ubsan-check.c
@@ -29,7 +29,7 @@
 // CHECK:       [[HANDLER_DIVREM_OVERFLOW]]:
 // CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// CHECK-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR6:[0-9]+]], !nosanitize [[META2]]
+// CHECK-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR8:[0-9]+]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 // CHECK:       [[CONT]]:
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[X]], [[Y]]
@@ -75,7 +75,7 @@
 // REC:       [[HANDLER_DIVREM_OVERFLOW]]:
 // REC-NEXT:    [[TMP10:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // REC-NEXT:    [[TMP11:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// REC-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR6:[0-9]+]], !nosanitize [[META2]]
+// REC-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR8:[0-9]+]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[CONT]], !nosanitize [[META2]]
 // REC:       [[CONT]]:
 // REC-NEXT:    [[DIV:%.*]] = sdiv i32 [[X]], [[Y]]
@@ -86,7 +86,7 @@ int div(int x, int y) {
 }
 
 // CHECK-LABEL: define dso_local i32 @null(
-// CHECK-SAME: ptr noundef readonly captures(address_is_null) [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: ptr noundef readonly captures(address_is_null) [[X:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = icmp eq ptr [[X]], null, !nosanitize [[META2]]
 //
@@ -95,7 +95,7 @@ int div(int x, int y) {
 // CHECK-NEXT:    [[DOTNOT1:%.*]] = and i1 [[TMP0]], [[TMP1]]
 // CHECK-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_TYPE_MISMATCH:.*]], label %[[CONT:.*]], !prof [[PROF4:![0-9]+]], !nosanitize [[META2]]
 // CHECK:       [[HANDLER_TYPE_MISMATCH]]:
-// CHECK-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR6]], !nosanitize [[META2]]
+// CHECK-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 // CHECK:       [[CONT]]:
 // CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA5:![0-9]+]]
@@ -116,14 +116,14 @@ int div(int x, int y) {
 // TR-NEXT:    ret i32 [[TMP2]]
 //
 // REC-LABEL: define dso_local i32 @null(
-// REC-SAME: ptr noundef readonly captures(address_is_null) [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// REC-SAME: ptr noundef readonly captures(address_is_null) [[X:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // REC-NEXT:  [[ENTRY:.*:]]
 // REC-NEXT:    [[TMP0:%.*]] = icmp eq ptr [[X]], null, !nosanitize [[META2]]
 // REC-NEXT:    [[TMP1:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 29), !nosanitize [[META2]]
 // REC-NEXT:    [[DOTNOT1:%.*]] = and i1 [[TMP0]], [[TMP1]]
 // REC-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_TYPE_MISMATCH:.*]], label %[[CONT:.*]], !prof [[PROF4:![0-9]+]], !nosanitize [[META2]]
 // REC:       [[HANDLER_TYPE_MISMATCH]]:
-// REC-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR6]], !nosanitize [[META2]]
+// REC-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[CONT]], !nosanitize [[META2]]
 // REC:       [[CONT]]:
 // REC-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA5:![0-9]+]]
@@ -146,7 +146,7 @@ int null(int* x) {
 // CHECK:       [[HANDLER_ADD_OVERFLOW]]:
 // CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// CHECK-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP3]], i64 [[TMP4]]) #[[ATTR6]], !nosanitize [[META2]]
+// CHECK-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP3]], i64 [[TMP4]]) #[[ATTR8]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 // CHECK:       [[CONT]]:
 // CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
@@ -178,7 +178,7 @@ int null(int* x) {
 // REC:       [[HANDLER_ADD_OVERFLOW]]:
 // REC-NEXT:    [[TMP3:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // REC-NEXT:    [[TMP4:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// REC-NEXT:    tail call void @__ubsan_handle_add_overflow(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP3]], i64 [[TMP4]]) #[[ATTR6]], !nosanitize [[META2]]
+// REC-NEXT:    tail call void @__ubsan_handle_add_overflow(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP3]], i64 [[TMP4]]) #[[ATTR8]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[CONT]], !nosanitize [[META2]]
 // REC:       [[CONT]]:
 // REC-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
@@ -191,11 +191,11 @@ int overflow(int x, int y) {
 void use(double*);
 
 // CHECK-LABEL: define dso_local double @lbounds(
-// CHECK-SAME: i32 noundef [[B:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-SAME: i32 noundef [[B:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[B]] to i64
 // CHECK-NEXT:    [[VLA:%.*]] = alloca double, i64 [[TMP0]], align 16
-// CHECK-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR7:[0-9]+]]
+// CHECK-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR9:[0-9]+]]
 // CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[TMP0]], [[IDXPROM]]
 //
@@ -208,7 +208,7 @@ void use(double*);
 // CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA9:![0-9]+]]
 // CHECK-NEXT:    ret double [[TMP5]]
 // CHECK:       [[TRAP]]:
-// CHECK-NEXT:    call void @__ubsan_handle_local_out_of_bounds_abort() #[[ATTR6]], !nosanitize [[META2]]
+// CHECK-NEXT:    call void @__ubsan_handle_local_out_of_bounds_abort() #[[ATTR8]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // TR-LABEL: define dso_local double @lbounds(
@@ -231,11 +231,11 @@ void use(double*);
 // TR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // REC-LABEL: define dso_local double @lbounds(
-// REC-SAME: i32 noundef [[B:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// REC-SAME: i32 noundef [[B:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
 // REC-NEXT:  [[ENTRY:.*:]]
 // REC-NEXT:    [[TMP0:%.*]] = zext i32 [[B]] to i64
 // REC-NEXT:    [[VLA:%.*]] = alloca double, i64 [[TMP0]], align 16
-// REC-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR5:[0-9]+]]
+// REC-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR7:[0-9]+]]
 // REC-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // REC-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[TMP0]], [[IDXPROM]]
 // REC-NEXT:    [[TMP2:%.*]] = call i1 @llvm.allow.ubsan.check(i8 71), !nosanitize [[META2]]
@@ -246,7 +246,7 @@ void use(double*);
 // REC-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA9:![0-9]+]]
 // REC-NEXT:    ret double [[TMP5]]
 // REC:       [[TRAP]]:
-// REC-NEXT:    call void @__ubsan_handle_local_out_of_bounds() #[[ATTR6]], !nosanitize [[META2]]
+// REC-NEXT:    call void @__ubsan_handle_local_out_of_bounds() #[[ATTR8]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[BB4]], !nosanitize [[META2]]
 //
 double lbounds(int b, int i) {
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
index dfdf06587f0e2..8e062629464dd 100644
--- a/clang/test/CodeGen/attr-counted-by.c
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -68,7 +68,7 @@ struct anon_struct {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8:[0-9]+]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12:[0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -116,7 +116,7 @@ void test1(struct annotated *p, int index, int val) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -159,7 +159,7 @@ void test2(struct annotated *p, size_t index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934592, 8589934589) i64 @test2_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
@@ -181,7 +181,7 @@ void test2(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test2_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -203,7 +203,7 @@ size_t test2_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -242,7 +242,7 @@ void test3(struct annotated *p, size_t index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test3_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -252,7 +252,7 @@ void test3(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test3_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -275,7 +275,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 2
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 3) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 3) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl i32 [[DOTCOUNTED_BY_LOAD]], 2
@@ -283,7 +283,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT12:%.*]], label [[HANDLER_OUT_OF_BOUNDS8:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds8:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont12:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[DOTCOUNTED_BY_LOAD]], 2
@@ -295,7 +295,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT81:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 3
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT81]], label [[HANDLER_OUT_OF_BOUNDS18:%.*]], label [[CONT19:%.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds18:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 4) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 4) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont19:
 // SANITIZE-WITH-ATTR-NEXT:    [[ADD:%.*]] = add nsw i32 [[INDEX]], 1
@@ -303,7 +303,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[IDXPROM31]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP5]], label [[CONT38:%.*]], label [[HANDLER_OUT_OF_BOUNDS34:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds34:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM31]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM31]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont38:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[DOTCOUNTED_BY_LOAD]], 3
@@ -318,7 +318,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp ugt i64 [[IDXPROM42]], [[TMP8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS45:%.*]], label [[CONT46:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds45:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM42]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM42]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont46:
 // SANITIZE-WITH-ATTR-NEXT:    [[ADD59:%.*]] = add nsw i32 [[INDEX]], 2
@@ -326,7 +326,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = icmp ult i64 [[IDXPROM60]], [[TMP8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP9]], label [[CONT67:%.*]], label [[HANDLER_OUT_OF_BOUNDS63:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds63:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[IDXPROM60]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[IDXPROM60]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont67:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX65:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM60]]
@@ -422,7 +422,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 8589934589) i64 @test4_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
@@ -431,7 +431,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp ugt i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], label [[CONT1:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[DOTCOUNTED_BY_LOAD]] to i64
@@ -456,7 +456,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP4]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test4_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -478,7 +478,7 @@ size_t test4_bdos(struct annotated *p, int index) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[DOTCOUNTED_BY_LOAD]], [[IDXPROM]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB12:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB12:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
@@ -518,7 +518,7 @@ void test5(struct anon_struct *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test5_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -528,7 +528,7 @@ void test5(struct anon_struct *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test5_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -550,7 +550,7 @@ size_t test5_bdos(struct anon_struct *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[COUNTED_BY_LOAD]], [[IDXPROM]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
@@ -598,7 +598,7 @@ void test6(struct anon_struct *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, -9223372036854775808) i64 @test6_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i64, ptr [[COUNTED_BY_GEP]], align 4
@@ -616,7 +616,7 @@ void test6(struct anon_struct *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test6_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -639,7 +639,7 @@ size_t test6_bdos(struct anon_struct *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
 // SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
@@ -679,7 +679,7 @@ void test7(struct union_of_fams *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test7_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -689,7 +689,7 @@ void test7(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test7_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -712,7 +712,7 @@ size_t test7_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT14:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont14:
 // SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
@@ -754,7 +754,7 @@ void test8(struct union_of_fams *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 256) i64 @test8_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
@@ -770,7 +770,7 @@ void test8(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[COUNT]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test8_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -793,7 +793,7 @@ size_t test8_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
 // SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -833,7 +833,7 @@ void test9(struct union_of_fams *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test9_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -843,7 +843,7 @@ void test9(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test9_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -866,7 +866,7 @@ size_t test9_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT14:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB19:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB19:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont14:
 // SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -912,7 +912,7 @@ void test10(struct union_of_fams *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 2147483648) i64 @test10_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -930,7 +930,7 @@ void test10(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test10_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -953,7 +953,7 @@ size_t test10_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -1003,7 +1003,7 @@ void test11(struct annotated *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934584, 8589934597) i64 @test11_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
@@ -1027,7 +1027,7 @@ void test11(struct annotated *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test11_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1056,16 +1056,16 @@ struct hang {
 int test12_a, test12_b;
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test12(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR9:[0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR13:[0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT10:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB22:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB22:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
@@ -1075,10 +1075,10 @@ int test12_a, test12_b;
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 0
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds4:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB24:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB24:[0-9]+]], i64 0) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.type_mismatch6:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB25:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB25:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test12(
@@ -1098,16 +1098,16 @@ int test12_a, test12_b;
 // NO-SANITIZE-WITH-ATTR-NEXT:    br label [[FOR_COND]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test12(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR7:[0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR11:[0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META9:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP1]]) #[[ATTR8:[0-9]+]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP1]]) #[[ATTR12:[0-9]+]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
@@ -1117,10 +1117,10 @@ int test12_a, test12_b;
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 0
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF10:![0-9]+]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds4:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 0) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.type_mismatch6:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test12(
@@ -1159,7 +1159,7 @@ struct test13_bar {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test13(
-// SANITIZE-WITH-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr @test13_f, align 8, !tbaa [[TBAA11:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
@@ -1168,7 +1168,7 @@ struct test13_bar {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[INDEX]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB28:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB28:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
@@ -1186,7 +1186,7 @@ struct test13_bar {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 0
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test13(
-// SANITIZE-WITHOUT-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr @test13_f, align 8, !tbaa [[TBAA11:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
@@ -1195,7 +1195,7 @@ struct test13_bar {
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[INDEX]], [[TMP1]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont5:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
@@ -1223,13 +1223,13 @@ struct test14_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test14(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB29:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB29:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 undef
@@ -1248,13 +1248,13 @@ struct test14_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test14(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont3:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 undef
@@ -1277,13 +1277,13 @@ int test14(int idx) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test15(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB31:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB31:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 undef
@@ -1297,13 +1297,13 @@ int test14(int idx) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont1:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 undef
@@ -1326,14 +1326,14 @@ int test15(int idx) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test19(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 680
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 1
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB32:[0-9]+]], i64 2) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB32:[0-9]+]], i64 2) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
@@ -1344,7 +1344,7 @@ int test15(int idx) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test19(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1359,7 +1359,7 @@ size_t test19(struct annotated *p) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test20(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1369,7 +1369,7 @@ size_t test19(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test20(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1384,7 +1384,7 @@ size_t test20(struct annotated *p) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test21(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1394,7 +1394,7 @@ size_t test20(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test21(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1409,7 +1409,7 @@ size_t test21(struct annotated *p) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test22(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1419,7 +1419,7 @@ size_t test21(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test22(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1434,7 +1434,7 @@ size_t test22(struct annotated *p) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test23(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1444,7 +1444,7 @@ size_t test22(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test23(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1464,14 +1464,14 @@ struct tests_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test24(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 40
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 10
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT4:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB33:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB33:[0-9]+]], i64 10) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont4:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
@@ -1486,7 +1486,7 @@ struct tests_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test24(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA2]]
@@ -1505,14 +1505,14 @@ int test24(int c, struct tests_foo *var) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test25(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA17:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 10
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB34:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB34:[0-9]+]], i64 10) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
@@ -1528,7 +1528,7 @@ int test24(int c, struct tests_foo *var) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test25(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA17:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
@@ -1555,7 +1555,7 @@ struct test26_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test26(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[S:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[C]] to i64
@@ -1564,7 +1564,7 @@ struct test26_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB35:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB35:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
@@ -1582,7 +1582,7 @@ struct test26_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test26(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[C]] to i64
@@ -1626,7 +1626,7 @@ struct test27_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local ptr @test27(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -1635,7 +1635,7 @@ struct test27_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB37:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB37:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
@@ -1657,7 +1657,7 @@ struct test27_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret ptr [[ARRAYIDX2]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local ptr @test27(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
@@ -1689,7 +1689,7 @@ struct test28_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test28(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA21:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA21]]
@@ -1701,7 +1701,7 @@ struct test28_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT17:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont17:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
@@ -1722,7 +1722,7 @@ struct test28_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP3]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test28(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA21:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA21]]
@@ -1757,13 +1757,13 @@ struct annotated_struct_array {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test29(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR7]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 10
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB41:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB41:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
@@ -1775,7 +1775,7 @@ struct annotated_struct_array {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM27]], [[TMP3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT32:%.*]], label [[HANDLER_OUT_OF_BOUNDS28:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds28:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB42:[0-9]+]], i64 [[IDXPROM27]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB42:[0-9]+]], i64 [[IDXPROM27]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont32:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
@@ -1802,13 +1802,13 @@ struct annotated_struct_array {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test29(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR7]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 10
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT21:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont21:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
@@ -1847,10 +1847,10 @@ struct test30_struct {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR9:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB44:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB44:[0-9]+]], i64 [[TMP0]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
@@ -1863,10 +1863,10 @@ struct test30_struct {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test30(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR9:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test30(
@@ -1891,7 +1891,7 @@ struct test31_struct {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test31(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 -1
 //
@@ -1901,7 +1901,7 @@ struct test31_struct {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test31(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 -1
 //
@@ -1927,7 +1927,7 @@ struct annotated_with_array {
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX2]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB46:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB46:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
@@ -1937,7 +1937,7 @@ struct annotated_with_array {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[IDXPROM4]], [[TMP2]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP3]], label [[CONT9:%.*]], label [[HANDLER_OUT_OF_BOUNDS5:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds5:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB48:[0-9]+]], i64 [[IDXPROM4]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB48:[0-9]+]], i64 [[IDXPROM4]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont9:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 344
@@ -1979,7 +1979,7 @@ struct annotated_with_array {
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX2]] to i64, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB17:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB17:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont7:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 344
@@ -2002,13 +2002,13 @@ void test32(struct annotated_with_array *ptr, int idx1, int idx2) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 17179869521) i64 @test32_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 43
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB49:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB49:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
@@ -2039,13 +2039,13 @@ void test32(struct annotated_with_array *ptr, int idx1, int idx2) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP4]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test32_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 43
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont1:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
@@ -2060,7 +2060,7 @@ size_t test32_bdos(struct annotated_with_array *ptr, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test33(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -2070,7 +2070,7 @@ size_t test32_bdos(struct annotated_with_array *ptr, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test33(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -2091,20 +2091,20 @@ struct multi_subscripts {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test34(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR8]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 42
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB51:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB51:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[IDX2]], 43
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS2:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds2:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[IDX2]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB52:[0-9]+]], i64 [[TMP3]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB52:[0-9]+]], i64 [[TMP3]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
@@ -2115,20 +2115,20 @@ struct multi_subscripts {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test34(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 42
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont1:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[IDX2]], 43
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS2:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds2:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[IDX2]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB21:[0-9]+]], i64 [[TMP3]]) #[[ATTR8]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB21:[0-9]+]], i64 [[TMP3]]) #[[ATTR12]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont3:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
@@ -2151,7 +2151,7 @@ size_t test34(struct multi_subscripts *ptr, int idx1, int idx2) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB53:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB53:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -2188,7 +2188,7 @@ void test35(struct annotated *p, size_t index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test35_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 0
 //
@@ -2198,7 +2198,7 @@ void test35(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 0
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test35_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 0
 //
@@ -2225,7 +2225,7 @@ struct {
 } x;
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test36(
-// SANITIZE-WITH-ATTR-SAME: ) local_unnamed_addr #[[ATTR6:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: ) local_unnamed_addr #[[ATTR10:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -2235,7 +2235,7 @@ struct {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test36(
-// SANITIZE-WITHOUT-ATTR-SAME: ) local_unnamed_addr #[[ATTR6:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ) local_unnamed_addr #[[ATTR10:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
diff --git a/clang/test/CodeGen/ubsan-attr.cpp b/clang/test/CodeGen/ubsan-attr.cpp
new file mode 100644
index 0000000000000..48f1b83307660
--- /dev/null
+++ b/clang/test/CodeGen/ubsan-attr.cpp
@@ -0,0 +1,100 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 5
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -fsanitize=integer-divide-by-zero -O3 %s -o - -fsanitize-recover=integer-divide-by-zero | FileCheck %s --check-prefixes=RECOVER
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -fsanitize=integer-divide-by-zero -O3 %s -o - | FileCheck %s --check-prefixes=ABORT
+
+// RECOVER: Function Attrs: mustprogress nounwind memory(read, argmem: readwrite, inaccessiblemem: readwrite)
+// RECOVER-LABEL: define dso_local noundef range(i32 -32768, 32768) i32 @_Z4testRiRs(
+// RECOVER-SAME: ptr noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr noundef nonnull readonly align 2 captures(none) dereferenceable(2) [[C:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// RECOVER-NEXT:  [[ENTRY:.*:]]
+// RECOVER-NEXT:    [[TMP0:%.*]] = load i16, ptr [[C]], align 2, !tbaa [[TBAA2:![0-9]+]]
+// RECOVER-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4, !tbaa [[TBAA6:![0-9]+]]
+// RECOVER-NEXT:    [[DOTNOT:%.*]] = icmp eq i16 [[TMP0]], 0
+// RECOVER-NEXT:    br i1 [[DOTNOT]], label %[[HANDLER_DIVREM_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META9:![0-9]+]]
+// RECOVER:       [[HANDLER_DIVREM_OVERFLOW]]:
+// RECOVER-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64, !nosanitize [[META9]]
+// RECOVER-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @{{.+}}, i64 [[TMP2]], i64 0) #[[ATTR3:[0-9]+]], !nosanitize [[META9]]
+// RECOVER-NEXT:    br label %[[CONT]], !nosanitize [[META9]]
+// RECOVER:       [[CONT]]:
+// RECOVER-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
+// RECOVER-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP1]], [[CONV]]
+// RECOVER-NEXT:    store i32 [[DIV]], ptr [[A]], align 4, !tbaa [[TBAA6]]
+// RECOVER-NEXT:    ret i32 [[CONV]]
+//
+// ABORT: Function Attrs: mustprogress nounwind
+// ABORT-LABEL: define dso_local noundef range(i32 -32768, 32768) i32 @_Z4testRiRs(
+// ABORT-SAME: ptr noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr noundef nonnull readonly align 2 captures(none) dereferenceable(2) [[C:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// ABORT-NEXT:  [[ENTRY:.*:]]
+// ABORT-NEXT:    [[TMP0:%.*]] = load i16, ptr [[C]], align 2, !tbaa [[TBAA2:![0-9]+]]
+// ABORT-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4, !tbaa [[TBAA6:![0-9]+]]
+// ABORT-NEXT:    [[DOTNOT:%.*]] = icmp eq i16 [[TMP0]], 0
+// ABORT-NEXT:    br i1 [[DOTNOT]], label %[[HANDLER_DIVREM_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META9:![0-9]+]]
+// ABORT:       [[HANDLER_DIVREM_OVERFLOW]]:
+// ABORT-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64, !nosanitize [[META9]]
+// ABORT-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @{{.+}}, i64 [[TMP2]], i64 0) #[[ATTR3:[0-9]+]], !nosanitize [[META9]]
+// ABORT-NEXT:    unreachable, !nosanitize [[META9]]
+// ABORT:       [[CONT]]:
+// ABORT-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
+// ABORT-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP1]], [[CONV]]
+// ABORT-NEXT:    store i32 [[DIV]], ptr [[A]], align 4, !tbaa [[TBAA6]]
+// ABORT-NEXT:    ret i32 [[CONV]]
+//
+int test(int &a, short &c) {
+  a /= c;
+  return c;
+}
+
+// Make sure that we don't eliminate the call to ubsan handler.
+// RECOVER: Function Attrs: mustprogress noreturn nounwind memory(read, argmem: none, inaccessiblemem: readwrite)
+// RECOVER-LABEL: define dso_local noundef i32 @_Z16test_unreachablev(
+// RECOVER-SAME: ) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// RECOVER-NEXT:  [[ENTRY:.*:]]
+// RECOVER-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @{{.+}}, i64 1, i64 0) #[[ATTR3]], !nosanitize [[META9]]
+// RECOVER-NEXT:    unreachable
+//
+// ABORT: Function Attrs: mustprogress noreturn nounwind
+// ABORT-LABEL: define dso_local noundef i32 @_Z16test_unreachablev(
+// ABORT-SAME: ) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// ABORT-NEXT:  [[ENTRY:.*:]]
+// ABORT-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @{{.+}}, i64 1, i64 0) #[[ATTR3]], !nosanitize [[META9]]
+// ABORT-NEXT:    unreachable, !nosanitize [[META9]]
+//
+int test_unreachable() {
+  int a = 1;
+  int b = 0;
+  if (a / b) return 1;
+  return 0;
+}
+
+//.
+// RECOVER: attributes #[[ATTR0]] = { mustprogress nounwind memory(read, argmem: readwrite, inaccessiblemem: readwrite) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// RECOVER: attributes #[[ATTR1:[0-9]+]] = { memory(argmem: read, inaccessiblemem: readwrite) uwtable }
+// RECOVER: attributes #[[ATTR2]] = { mustprogress noreturn nounwind memory(read, argmem: none, inaccessiblemem: readwrite) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// RECOVER: attributes #[[ATTR3]] = { nomerge nounwind }
+//.
+// ABORT: attributes #[[ATTR0]] = { mustprogress nounwind "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// ABORT: attributes #[[ATTR1:[0-9]+]] = { noreturn nounwind uwtable }
+// ABORT: attributes #[[ATTR2]] = { mustprogress noreturn nounwind "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// ABORT: attributes #[[ATTR3]] = { nomerge noreturn nounwind }
+//.
+// RECOVER: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// RECOVER: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+// RECOVER: [[TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0}
+// RECOVER: [[META3]] = !{!"short", [[META4:![0-9]+]], i64 0}
+// RECOVER: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
+// RECOVER: [[META5]] = !{!"Simple C++ TBAA"}
+// RECOVER: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+// RECOVER: [[META7]] = !{!"int", [[META4]], i64 0}
+// RECOVER: [[PROF8]] = !{!"branch_weights", i32 1, i32 1048575}
+// RECOVER: [[META9]] = !{}
+//.
+// ABORT: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// ABORT: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+// ABORT: [[TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0}
+// ABORT: [[META3]] = !{!"short", [[META4:![0-9]+]], i64 0}
+// ABORT: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
+// ABORT: [[META5]] = !{!"Simple C++ TBAA"}
+// ABORT: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+// ABORT: [[META7]] = !{!"int", [[META4]], i64 0}
+// ABORT: [[PROF8]] = !{!"branch_weights", i32 1, i32 1048575}
+// ABORT: [[META9]] = !{}
+//.

From ecbd2d5e148cc1f2e5795d13d752e2a1c79cce1a Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 17 Apr 2025 17:27:19 +0200
Subject: [PATCH 283/710] [clang][ExprConst] Diagnose ptr subs with non-zero
 offset (#135938)

The attached test case was missing the note.
---
 clang/lib/AST/ExprConstant.cpp                   | 3 ---
 clang/test/AST/ByteCode/arrays.cpp               | 2 +-
 clang/test/SemaCXX/constant-expression-cxx11.cpp | 4 ++++
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index d1cc722fb7945..b14ff21a8ebc2 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -14764,9 +14764,6 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
     // Reject differing bases from the normal codepath; we special-case
     // comparisons to null.
     if (!HasSameBase(LHSValue, RHSValue)) {
-      // Handle &&A - &&B.
-      if (!LHSValue.Offset.isZero() || !RHSValue.Offset.isZero())
-        return Error(E);
       const Expr *LHSExpr = LHSValue.Base.dyn_cast<const Expr *>();
       const Expr *RHSExpr = RHSValue.Base.dyn_cast<const Expr *>();
 
diff --git a/clang/test/AST/ByteCode/arrays.cpp b/clang/test/AST/ByteCode/arrays.cpp
index f60cc19b09bd2..e50839e0f0877 100644
--- a/clang/test/AST/ByteCode/arrays.cpp
+++ b/clang/test/AST/ByteCode/arrays.cpp
@@ -107,7 +107,7 @@ static_assert(k1 == 1, "");
 static_assert((&arr[0] - &arr[1]) == -1, "");
 
 constexpr int k2 = &arr2[1] - &arr[0]; // both-error {{must be initialized by a constant expression}} \
-                                       // expected-note {{arithmetic involving unrelated objects}}
+                                       // both-note {{arithmetic involving unrelated objects}}
 
 static_assert((arr + 0) == arr, "");
 static_assert(&arr[0] == arr, "");
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index c35f3a5632a05..28016da925ef9 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -409,6 +409,10 @@ constexpr int a = 0;
 constexpr int b = 1;
 constexpr int n = &b - &a; // expected-error {{must be initialized by a constant expression}} \
                            // expected-note {{arithmetic involving unrelated objects '&b' and '&a' has unspecified value}}
+constexpr static int arrk[2] = {1,2};
+constexpr static int arrk2[2] = {3,4};
+constexpr int k2 = &arrk[1] - &arrk2[0]; // expected-error {{must be initialized by a constant expression}} \
+                                         // expected-note {{arithmetic involving unrelated objects}}
 
 namespace MaterializeTemporary {
 

From 70e2acf0c5650b8862c41f2697a9e0485ecb17be Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 17 Apr 2025 17:30:22 +0200
Subject: [PATCH 284/710] [clang][bytecode] Check if operator delete calls are
 in the right frame (#136141)

This is only permitted in a std::allocator::deallocate frame.
---
 clang/lib/AST/ByteCode/InterpBuiltin.cpp | 35 ++++++++++++++++++++++++
 clang/test/AST/ByteCode/new-delete.cpp   | 12 ++++++++
 2 files changed, 47 insertions(+)

diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 31d97d9060142..34553301ef630 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -1651,6 +1651,41 @@ static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
   const Expr *Source = nullptr;
   const Block *BlockToDelete = nullptr;
 
+  if (S.checkingPotentialConstantExpression())
+    return false;
+
+  // This is permitted only within a call to std::allocator<T>::deallocate.
+  bool DeallocateFrameFound = false;
+  for (const InterpFrame *F = Frame; F; F = F->Caller) {
+    const Function *Func = F->getFunction();
+    if (!Func)
+      continue;
+    const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Func->getDecl());
+    if (!MD)
+      continue;
+    const IdentifierInfo *FnII = MD->getIdentifier();
+    if (!FnII || !FnII->isStr("deallocate"))
+      continue;
+
+    const auto *CTSD =
+        dyn_cast<ClassTemplateSpecializationDecl>(MD->getParent());
+    if (!CTSD)
+      continue;
+
+    const IdentifierInfo *ClassII = CTSD->getIdentifier();
+    const TemplateArgumentList &TAL = CTSD->getTemplateArgs();
+    if (CTSD->isInStdNamespace() && ClassII && ClassII->isStr("allocator") &&
+        TAL.size() >= 1 && TAL[0].getKind() == TemplateArgument::Type) {
+      DeallocateFrameFound = true;
+      break;
+    }
+  }
+
+  if (!DeallocateFrameFound) {
+    S.FFDiag(Call);
+    return true;
+  }
+
   {
     const Pointer &Ptr = S.Stk.peek<Pointer>();
 
diff --git a/clang/test/AST/ByteCode/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
index 5ddd7070f6710..e1b81e9a7963e 100644
--- a/clang/test/AST/ByteCode/new-delete.cpp
+++ b/clang/test/AST/ByteCode/new-delete.cpp
@@ -992,6 +992,18 @@ namespace ZeroSizeSub {
                                    // both-note {{in call to}}
 }
 
+namespace WrongFrame {
+  constexpr int foo() {
+    int *p = nullptr;
+    __builtin_operator_delete(p); // both-note {{subexpression not valid in a constant expression}}
+
+    return 1;
+  }
+  static_assert(foo()); // both-error {{not an integral constant expression}} \
+                        // both-note {{in call to}}
+
+}
+
 #else
 /// Make sure we reject this prior to C++20
 constexpr int a() { // both-error {{never produces a constant expression}}

From a9dff35ad251cd20376ab25b26d1e5394e18ff4c Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer@sony.com>
Date: Thu, 17 Apr 2025 17:37:42 +0200
Subject: [PATCH 285/710] [Clang] Enable -fextend-lifetimes at -Og (#118026)

Recently, a new flag -fextend-variable-liveness was added that prevents
optimizations from removing the values of source variables in some cases,
improving the quality of debugging for optimized builds where it is
enabled.

Following the inclusion of the flag, this patch enables it by default
when `-Og` is set. Currently, `-Og` is equivalent to `-O1` - it is
effectively just an alias. By enabling `-fextend-lifetimes`, this patch
changes the code generated by Clang with `-Og` to have reduced
optimization and greater debuggability than `-O1`, differentiating the
two according to their respective purposes. This idea was discussed
previously on Discourse where there was general agreement with the
principle of this change:

https://discourse.llvm.org/t/rfc-redefine-og-o1-and-add-a-new-level-of-og
---
 clang/docs/CommandGuide/clang.rst            | 7 +++++--
 clang/docs/ReleaseNotes.rst                  | 4 ++++
 clang/lib/Driver/ToolChains/Clang.cpp        | 8 +++++++-
 clang/test/Driver/extend-variable-liveness.c | 3 ++-
 4 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/clang/docs/CommandGuide/clang.rst b/clang/docs/CommandGuide/clang.rst
index 42aac7b25d93c..a1e738f1afec8 100644
--- a/clang/docs/CommandGuide/clang.rst
+++ b/clang/docs/CommandGuide/clang.rst
@@ -443,8 +443,11 @@ Code Generation Options
     :option:`-Oz` Like :option:`-Os` (and thus :option:`-O2`), but reduces code
     size further.
 
-    :option:`-Og` Like :option:`-O1`. In future versions, this option might
-    disable different optimizations in order to improve debuggability.
+    :option:`-Og` Similar to :option:`-O1`, but with slightly reduced
+    optimization and better variable visibility. The same optimizations are run
+    as at :option:`-O1`, but the ``-fextend-variable-liveness`` flag is
+    also set, which tries to prevent optimizations from reducing the liveness of
+    user variables, improving their availability when debugging.
 
     :option:`-O` Equivalent to :option:`-O1`.
 
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0a46bbea96f6c..c75d83a6d1a7a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -221,6 +221,10 @@ Modified Compiler Flags
 
 - The ``-mexecute-only`` and ``-mpure-code`` flags are now accepted for AArch64 targets. (#GH125688)
 
+- The ``-Og`` optimization flag now sets ``-fextend-variable-liveness``,
+  reducing performance slightly while reducing the number of optimized-out
+  variables.
+
 Removed Compiler Flags
 -------------------------
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 8506a5c00e7bc..b2dd4b3b54869 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7681,7 +7681,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_fretain_comments_from_system_headers))
     CmdArgs.push_back("-fretain-comments-from-system-headers");
 
-  Args.AddLastArg(CmdArgs, options::OPT_fextend_variable_liveness_EQ);
+  if (Arg *A = Args.getLastArg(options::OPT_fextend_variable_liveness_EQ)) {
+    A->render(Args, CmdArgs);
+  } else if (Arg *A = Args.getLastArg(options::OPT_O_Group);
+             A && A->containsValue("g")) {
+    // Set -fextend-variable-liveness=all by default at -Og.
+    CmdArgs.push_back("-fextend-variable-liveness=all");
+  }
 
   // Forward -fcomment-block-commands to -cc1.
   Args.AddAllArgs(CmdArgs, options::OPT_fcomment_block_commands);
diff --git a/clang/test/Driver/extend-variable-liveness.c b/clang/test/Driver/extend-variable-liveness.c
index bbfb2ece6f297..99a5409ceccea 100644
--- a/clang/test/Driver/extend-variable-liveness.c
+++ b/clang/test/Driver/extend-variable-liveness.c
@@ -1,7 +1,8 @@
 // Tests that -fextend-variable-liveness and its aliases are correctly passed
-// by the driver.
+// by the driver, and are set by default at -Og.
 
 // RUN: %clang -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,DEFAULT
+// RUN: %clang -### -Og -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALL
 // RUN: %clang -fextend-variable-liveness=none -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NONE
 // RUN: %clang -fextend-variable-liveness=this -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,THIS
 // RUN: %clang -fextend-variable-liveness=all -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALL

From 1fcf78d15318e3fbe384c635a0ae651be89d0686 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 17 Apr 2025 08:02:52 -0700
Subject: [PATCH 286/710] [SLP]Cache data for compressed loads before codegen

Need to cache and use cached data for compressed loads before codegen to
avoid side-effects, caused by the earlier vectorization, which may
affect the analysis.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 19 ++++----
 .../masked-loads-side-effects-after-vec.ll    | 48 +++++++++++++++++++
 2 files changed, 58 insertions(+), 9 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/masked-loads-side-effects-after-vec.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fd23fb6c81c2c..0cf89ea4a9dc8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1889,6 +1889,7 @@ class BoUpSLP {
     LoadEntriesToVectorize.clear();
     IsGraphTransformMode = false;
     GatheredLoadsEntriesFirst.reset();
+    CompressEntryToData.clear();
     ExternalUses.clear();
     ExternalUsesAsOriginalScalar.clear();
     for (auto &Iter : BlocksSchedules) {
@@ -4308,6 +4309,11 @@ class BoUpSLP {
   /// The index of the first gathered load entry in the VectorizeTree.
   std::optional<unsigned> GatheredLoadsEntriesFirst;
 
+  /// Maps compress entries to their mask data for the final codegen.
+  SmallDenseMap<const TreeEntry *,
+                std::tuple<SmallVector<int>, VectorType *, unsigned, bool>>
+      CompressEntryToData;
+
   /// This POD struct describes one external user in the vectorized tree.
   struct ExternalUser {
     ExternalUser(Value *S, llvm::User *U, const TreeEntry &E, int L)
@@ -13428,6 +13434,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
             *TLI, [](Value *) { return true; }, IsMasked, InterleaveFactor,
             CompressMask, LoadVecTy);
         assert(IsVectorized && "Expected to be vectorized");
+        CompressEntryToData.try_emplace(E, CompressMask, LoadVecTy,
+                                        InterleaveFactor, IsMasked);
         Align CommonAlignment;
         if (IsMasked)
           CommonAlignment = computeCommonAlignment<LoadInst>(VL);
@@ -17963,10 +17971,6 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       if (E->State == TreeEntry::Vectorize) {
         NewLI = Builder.CreateAlignedLoad(VecTy, PO, LI->getAlign());
       } else if (E->State == TreeEntry::CompressVectorize) {
-        bool IsMasked;
-        unsigned InterleaveFactor;
-        SmallVector<int> CompressMask;
-        VectorType *LoadVecTy;
         SmallVector<Value *> Scalars(E->Scalars.begin(), E->Scalars.end());
         if (!E->ReorderIndices.empty()) {
           SmallVector<int> Mask(E->ReorderIndices.begin(),
@@ -17976,11 +17980,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         SmallVector<Value *> PointerOps(Scalars.size());
         for (auto [I, V] : enumerate(Scalars))
           PointerOps[I] = cast<LoadInst>(V)->getPointerOperand();
-        [[maybe_unused]] bool IsVectorized = isMaskedLoadCompress(
-            Scalars, PointerOps, E->ReorderIndices, *TTI, *DL, *SE, *AC, *DT,
-            *TLI, [](Value *) { return true; }, IsMasked, InterleaveFactor,
-            CompressMask, LoadVecTy);
-        assert(IsVectorized && "Expected to be vectorized");
+        auto [CompressMask, LoadVecTy, InterleaveFactor, IsMasked] =
+            CompressEntryToData.at(E);
         Align CommonAlignment;
         if (IsMasked)
           CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/masked-loads-side-effects-after-vec.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/masked-loads-side-effects-after-vec.ll
new file mode 100644
index 0000000000000..ca3c8bbac6366
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/masked-loads-side-effects-after-vec.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+declare noalias ptr @malloc()
+
+define void @test() {
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT:    [[TMP1:%.*]] = call dereferenceable_or_null(16) ptr @malloc()
+; CHECK-NEXT:    [[TMP2:%.*]] = load volatile ptr, ptr null, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <15 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <15 x i8> [[TMP3]], <15 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    store <8 x i8> [[TMP4]], ptr [[TMP2]], align 1
+; CHECK-NEXT:    ret void
+;
+  %1 = call dereferenceable_or_null(16) ptr @malloc()
+  %2 = load volatile ptr, ptr null, align 8
+  %3 = load i8, ptr %1, align 1
+  store i8 %3, ptr %2, align 1
+  %4 = getelementptr i8, ptr %1, i64 2
+  %5 = load i8, ptr %4, align 1
+  %6 = getelementptr i8, ptr %2, i64 1
+  store i8 %5, ptr %6, align 1
+  %7 = getelementptr i8, ptr %1, i64 4
+  %8 = load i8, ptr %7, align 1
+  %9 = getelementptr i8, ptr %2, i64 2
+  store i8 %8, ptr %9, align 1
+  %10 = getelementptr i8, ptr %1, i64 6
+  %11 = load i8, ptr %10, align 1
+  %12 = getelementptr i8, ptr %2, i64 3
+  store i8 %11, ptr %12, align 1
+  %13 = getelementptr i8, ptr %1, i64 8
+  %14 = load i8, ptr %13, align 1
+  %15 = getelementptr i8, ptr %2, i64 4
+  store i8 %14, ptr %15, align 1
+  %16 = getelementptr i8, ptr %1, i64 10
+  %17 = load i8, ptr %16, align 1
+  %18 = getelementptr i8, ptr %2, i64 5
+  store i8 %17, ptr %18, align 1
+  %19 = getelementptr i8, ptr %1, i64 12
+  %20 = load i8, ptr %19, align 1
+  %21 = getelementptr i8, ptr %2, i64 6
+  store i8 %20, ptr %21, align 1
+  %22 = getelementptr i8, ptr %1, i64 14
+  %23 = load i8, ptr %22, align 1
+  %24 = getelementptr i8, ptr %2, i64 7
+  store i8 %23, ptr %24, align 1
+  ret void
+}

From 83116209331ad6a5a45f1b8735ad5bce9e6ef761 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 17 Apr 2025 17:57:00 +0200
Subject: [PATCH 287/710] [lldb] Fix lock inversion between statusline mutex
 and output mutex (#135956)

Fix a deadlock between the statusline mutex (in Debugger) and the output
file mutex (in LockedStreamFile). The deadlock occurs when the main
thread is calling the statusline callback while holding the output mutex
in Editline, while the default event thread is trying to update the
statusline.

Extend the uncritical section so we can redraw the statusline there.
The loop in Editline::GetCharacter should be unnecessary. It would only
loop if we had a successful read with length zero, which shouldn't be
possible or when we can't convert a partial UTF-8 character, in which
case we bail out.

rdar://149251156
---
 lldb/source/Host/common/Editline.cpp          | 93 +++++++++----------
 .../statusline/TestStatusline.py              | 17 ++++
 2 files changed, 63 insertions(+), 47 deletions(-)

diff --git a/lldb/source/Host/common/Editline.cpp b/lldb/source/Host/common/Editline.cpp
index 29abaf7c65f28..ff71cd0cdb241 100644
--- a/lldb/source/Host/common/Editline.cpp
+++ b/lldb/source/Host/common/Editline.cpp
@@ -567,9 +567,6 @@ int Editline::GetCharacter(EditLineGetCharType *c) {
     m_needs_prompt_repaint = false;
   }
 
-  if (m_redraw_callback)
-    m_redraw_callback();
-
   if (m_multiline_enabled) {
     // Detect when the number of rows used for this input line changes due to
     // an edit
@@ -585,54 +582,56 @@ int Editline::GetCharacter(EditLineGetCharType *c) {
     m_current_line_rows = new_line_rows;
   }
 
+  if (m_terminal_size_has_changed)
+    ApplyTerminalSizeChange();
+
+  // This mutex is locked by our caller (GetLine). Unlock it while we read a
+  // character (blocking operation), so we do not hold the mutex
+  // indefinitely. This gives a chance for someone to interrupt us. After
+  // Read returns, immediately lock the mutex again and check if we were
+  // interrupted.
+  m_locked_output.reset();
+
+  if (m_redraw_callback)
+    m_redraw_callback();
+
   // Read an actual character
-  while (true) {
-    lldb::ConnectionStatus status = lldb::eConnectionStatusSuccess;
-    char ch = 0;
-
-    if (m_terminal_size_has_changed)
-      ApplyTerminalSizeChange();
-
-    // This mutex is locked by our caller (GetLine). Unlock it while we read a
-    // character (blocking operation), so we do not hold the mutex
-    // indefinitely. This gives a chance for someone to interrupt us. After
-    // Read returns, immediately lock the mutex again and check if we were
-    // interrupted.
-    m_locked_output.reset();
-    int read_count =
-        m_input_connection.Read(&ch, 1, std::nullopt, status, nullptr);
-    m_locked_output.emplace(m_output_stream_sp->Lock());
-    if (m_editor_status == EditorStatus::Interrupted) {
-      while (read_count > 0 && status == lldb::eConnectionStatusSuccess)
-        read_count =
-            m_input_connection.Read(&ch, 1, std::nullopt, status, nullptr);
-      lldbassert(status == lldb::eConnectionStatusInterrupted);
-      return 0;
-    }
+  lldb::ConnectionStatus status = lldb::eConnectionStatusSuccess;
+  char ch = 0;
+  int read_count =
+      m_input_connection.Read(&ch, 1, std::nullopt, status, nullptr);
+
+  // Re-lock the output mutex to protected m_editor_status here and in the
+  // switch below.
+  m_locked_output.emplace(m_output_stream_sp->Lock());
+  if (m_editor_status == EditorStatus::Interrupted) {
+    while (read_count > 0 && status == lldb::eConnectionStatusSuccess)
+      read_count =
+          m_input_connection.Read(&ch, 1, std::nullopt, status, nullptr);
+    lldbassert(status == lldb::eConnectionStatusInterrupted);
+    return 0;
+  }
 
-    if (read_count) {
-      if (CompleteCharacter(ch, *c))
-        return 1;
-    } else {
-      switch (status) {
-      case lldb::eConnectionStatusSuccess: // Success
-        break;
+  if (read_count) {
+    if (CompleteCharacter(ch, *c))
+      return 1;
+    return 0;
+  }
 
-      case lldb::eConnectionStatusInterrupted:
-        llvm_unreachable("Interrupts should have been handled above.");
-
-      case lldb::eConnectionStatusError:        // Check GetError() for details
-      case lldb::eConnectionStatusTimedOut:     // Request timed out
-      case lldb::eConnectionStatusEndOfFile:    // End-of-file encountered
-      case lldb::eConnectionStatusNoConnection: // No connection
-      case lldb::eConnectionStatusLostConnection: // Lost connection while
-                                                  // connected to a valid
-                                                  // connection
-        m_editor_status = EditorStatus::EndOfInput;
-        return 0;
-      }
-    }
+  switch (status) {
+  case lldb::eConnectionStatusSuccess:
+    llvm_unreachable("Success should have resulted in positive read_count.");
+  case lldb::eConnectionStatusInterrupted:
+    llvm_unreachable("Interrupts should have been handled above.");
+  case lldb::eConnectionStatusError:
+  case lldb::eConnectionStatusTimedOut:
+  case lldb::eConnectionStatusEndOfFile:
+  case lldb::eConnectionStatusNoConnection:
+  case lldb::eConnectionStatusLostConnection:
+    m_editor_status = EditorStatus::EndOfInput;
   }
+
+  return 0;
 }
 
 const char *Editline::Prompt() {
diff --git a/lldb/test/API/functionalities/statusline/TestStatusline.py b/lldb/test/API/functionalities/statusline/TestStatusline.py
index 747a7a14e0629..489c0371b11d8 100644
--- a/lldb/test/API/functionalities/statusline/TestStatusline.py
+++ b/lldb/test/API/functionalities/statusline/TestStatusline.py
@@ -77,3 +77,20 @@ def test_no_color(self):
                 "\x1b[7m",
             ],
         )
+
+    def test_deadlock(self):
+        """Regression test for lock inversion between the statusline mutex and
+        the output mutex."""
+        self.build()
+        self.launch(extra_args=["-o", "settings set use-color false"])
+        self.child.expect("(lldb)")
+
+        # Change the terminal dimensions.
+        terminal_height = 10
+        terminal_width = 60
+        self.child.setwinsize(terminal_height, terminal_width)
+
+        exe = self.getBuildArtifact("a.out")
+
+        self.expect("file {}".format(exe), ["Current executable"])
+        self.expect("help", ["Debugger commands"])

From 168092e2ba432c0995806470790fa5aef3d7315b Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi@microsoft.com>
Date: Thu, 17 Apr 2025 12:02:05 -0400
Subject: [PATCH 288/710] [DirectX] Legalize Freeze instruction (#136043)

fixes #135719

LLVM 3.7 did not have a freeze instruction
Further this instruction is really only used as syntactic sugar
in LLVM's optimizer passes to not aggressively optimize things that
could be undef or poison ie x*2 to x+x.
Most backends treat it as a no-op so we will do the same
by removing it and replacing its uses with its input.
---
 llvm/lib/Target/DirectX/DXILLegalizePass.cpp | 12 ++++++++++++
 llvm/test/CodeGen/DirectX/legalize-freeze.ll | 17 +++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 llvm/test/CodeGen/DirectX/legalize-freeze.ll

diff --git a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
index 395311e430fbb..b62ff4c52f70c 100644
--- a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
+++ b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
@@ -20,6 +20,17 @@
 
 using namespace llvm;
 
+static void legalizeFreeze(Instruction &I,
+                           SmallVectorImpl<Instruction *> &ToRemove,
+                           DenseMap<Value *, Value *>) {
+  auto *FI = dyn_cast<FreezeInst>(&I);
+  if (!FI)
+    return;
+
+  FI->replaceAllUsesWith(FI->getOperand(0));
+  ToRemove.push_back(FI);
+}
+
 static void fixI8TruncUseChain(Instruction &I,
                                SmallVectorImpl<Instruction *> &ToRemove,
                                DenseMap<Value *, Value *> &ReplacedValues) {
@@ -169,6 +180,7 @@ class DXILLegalizationPipeline {
   void initializeLegalizationPipeline() {
     LegalizationPipeline.push_back(fixI8TruncUseChain);
     LegalizationPipeline.push_back(downcastI64toI32InsertExtractElements);
+    LegalizationPipeline.push_back(legalizeFreeze);
   }
 };
 
diff --git a/llvm/test/CodeGen/DirectX/legalize-freeze.ll b/llvm/test/CodeGen/DirectX/legalize-freeze.ll
new file mode 100644
index 0000000000000..29446adeef215
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/legalize-freeze.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+
+; RUN: opt -S -passes='dxil-legalize' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+
+
+define i32 @test_remove_freeze(i32 %x) {
+; CHECK-LABEL: define i32 @test_remove_freeze(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[Y:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+entry:
+  %f = freeze i32 %x
+  %y = add i32 %f, 1
+  ret i32 %y
+}

From 1267facdc6f254fd9bd85cd8a033e7128b6e053d Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 17 Apr 2025 09:06:41 -0700
Subject: [PATCH 289/710] [bazel] Remove duplicate dependency (#136152)

AffineDialect was duplicated in the deps list since
47f4f39265b31e2249536b74d33d63508cdfb457
---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index b24160dc20e44..18e26f7a1b7a1 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -1559,7 +1559,6 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/AMDGPU/Transforms/*.h"]),
     includes = ["include"],
     deps = [
-        ":AffineDialect",
         ":AMDGPUDialect",
         ":AMDGPUPassIncGen",
         ":AMDGPUUtils",
@@ -6349,7 +6348,6 @@ cc_library(
     ],
 )
 
-
 td_library(
     name = "SMTTdFiles",
     srcs = [
@@ -6365,11 +6363,11 @@ td_library(
     includes = ["include"],
     deps = [
         ":AttrTdFiles",
-        ":OpBaseTdFiles",
-        ":SideEffectInterfacesTdFiles",
         ":BuiltinDialectTdFiles",
         ":ControlFlowInterfacesTdFiles",
         ":InferTypeOpInterfaceTdFiles",
+        ":OpBaseTdFiles",
+        ":SideEffectInterfacesTdFiles",
     ],
 )
 
@@ -6378,7 +6376,7 @@ gentbl_cc_library(
     tbl_outs = {
         "include/mlir/Dialect/SMT/IR/SMT.h.inc": ["-gen-op-decls"],
         "include/mlir/Dialect/SMT/IR/SMT.cpp.inc": ["-gen-op-defs"],
-          "include/mlir/Dialect/SMT/IR/SMTDialect.h.inc": [
+        "include/mlir/Dialect/SMT/IR/SMTDialect.h.inc": [
             "-gen-dialect-decls",
             "-dialect=smt",
         ],
@@ -8853,6 +8851,7 @@ cc_library(
         ":SCFToGPU",
         ":SCFTransformOps",
         ":SCFTransforms",
+        ":SMTDialect",
         ":SPIRVDialect",
         ":SPIRVPassIncGen",
         ":SPIRVTarget",
@@ -8862,7 +8861,6 @@ cc_library(
         ":ShapeToStandard",
         ":ShapeTransforms",
         ":ShapeTransformsPassIncGen",
-        ":SMTDialect",
         ":SparseTensorDialect",
         ":SparseTensorPipelines",
         ":SparseTensorTransformOps",

From 5fe91f1b5948591b75416ce529b3179234c8d93d Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 17 Apr 2025 09:03:20 -0700
Subject: [PATCH 290/710] [SLP]Check for catchswitch block before doing the
 analysis of the instructions

Need to skip the analysis of the catchswitch blocks to avoid a compiler
crash when trying to get the first instruction in the block.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  3 +-
 .../X86/catchswitch-block-in-use.ll           | 64 +++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/catchswitch-block-in-use.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0cf89ea4a9dc8..d496989cd0581 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14133,7 +14133,8 @@ InstructionCost BoUpSLP::getSpillCost() {
       Budget += BlockSize;
       if (Budget > BudgetLimit)
         return Res;
-      if (!CheckForNonVecCallsInSameBlock(&*BB->getFirstNonPHIOrDbgOrAlloca(),
+      if (!isa<CatchSwitchInst>(BB->getTerminator()) &&
+          !CheckForNonVecCallsInSameBlock(&*BB->getFirstNonPHIOrDbgOrAlloca(),
                                           BB->getTerminator()))
         return Res;
       Worklist.append(pred_begin(BB), pred_end(BB));
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/catchswitch-block-in-use.ll b/llvm/test/Transforms/SLPVectorizer/X86/catchswitch-block-in-use.ll
new file mode 100644
index 0000000000000..8ecb5af404a72
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/catchswitch-block-in-use.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-pc-windows-msvc19.40.33811 < %s | FileCheck %s
+
+%"class.boost::execution_exception" = type { i32, %"class.boost::unit_test::basic_cstring", %"struct.boost::execution_exception::location" }
+%"class.boost::unit_test::basic_cstring" = type { ptr, ptr }
+%"struct.boost::execution_exception::location" = type { %"class.boost::unit_test::basic_cstring", i64, %"class.boost::unit_test::basic_cstring" }
+
+define void @test() personality ptr null {
+; CHECK-LABEL: define void @test() personality ptr null {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_COND109:.*]]
+; CHECK:       [[FOR_COND109]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x ptr> [ zeroinitializer, %[[ENTRY]] ], [ zeroinitializer, %[[IF_END153:.*]] ]
+; CHECK-NEXT:    br label %[[INIT_ATTEMPT_I664:.*]]
+; CHECK:       [[INIT_ATTEMPT_I664]]:
+; CHECK-NEXT:    [[CALL_I666:%.*]] = invoke ptr null(ptr null)
+; CHECK-NEXT:            to label %[[INVOKE_CONT_I668:.*]] unwind label %[[CATCH_DISPATCH:.*]]
+; CHECK:       [[INVOKE_CONT_I668]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[CATCH_DISPATCH]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = catchswitch within none [label %catch] unwind to caller
+; CHECK:       [[CATCH:.*]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = catchpad within [[TMP1]] [ptr null, i32 0, ptr null]
+; CHECK-NEXT:    br i1 false, label %[[IF_END153]], label %[[INVOKE_CONT149:.*]]
+; CHECK:       [[INVOKE_CONT149]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x ptr>, ptr null, align 8
+; CHECK-NEXT:    br label %[[IF_END153]]
+; CHECK:       [[IF_END153]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x ptr> [ [[TMP0]], %[[CATCH]] ], [ [[TMP3]], %[[INVOKE_CONT149]] ]
+; CHECK-NEXT:    catchret from [[TMP2]] to label %[[FOR_COND109]]
+;
+entry:
+  br label %for.cond109
+
+for.cond109:
+  %setup_error.sroa.0.1 = phi ptr [ null, %entry ], [ null, %if.end153 ]
+  %setup_error.sroa.6.1 = phi ptr [ null, %entry ], [ null, %if.end153 ]
+  br label %init.attempt.i664
+
+init.attempt.i664:
+  %call.i666 = invoke ptr null(ptr null)
+  to label %invoke.cont.i668 unwind label %catch.dispatch
+
+invoke.cont.i668:
+  ret void
+
+catch.dispatch:
+  %0 = catchswitch within none [label %catch] unwind to caller
+
+catch:
+  %1 = catchpad within %0 [ptr null, i32 0, ptr null]
+  br i1 false, label %if.end153, label %invoke.cont149
+
+invoke.cont149:
+  %m_begin2.i.i = getelementptr %"class.boost::execution_exception", ptr null, i64 0, i32 1, i32 0
+  %2 = load ptr, ptr %m_begin2.i.i, align 8
+  %3 = load ptr, ptr null, align 8
+  br label %if.end153
+
+if.end153:
+  %setup_error.sroa.0.2 = phi ptr [ %setup_error.sroa.0.1, %catch ], [ %2, %invoke.cont149 ]
+  %setup_error.sroa.6.2 = phi ptr [ %setup_error.sroa.6.1, %catch ], [ %3, %invoke.cont149 ]
+  catchret from %1 to label %for.cond109
+}

From 1f224d889dfa89a293e3efa5b04740f634454370 Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer@sony.com>
Date: Thu, 17 Apr 2025 18:30:09 +0200
Subject: [PATCH 291/710] [DebugInfo][IndVarSimplify] Propagate source loc when
 simplifying rem (#135399)

When IndVarSimplify simplifies a rem of the induction variable to a cmp
and select, only the select currently receives the rem's source
location; this patch propagates it to the cmp as well.

Found using https://github.com/llvm/llvm-project/pull/107279.
---
 llvm/lib/Transforms/Utils/SimplifyIndVar.cpp  |  1 +
 .../IndVarSimplify/debugloc-rem-subst.ll      | 59 +++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll

diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 5a76bec017655..03dbf9513d140 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -350,6 +350,7 @@ void SimplifyIndvar::replaceRemWithNumeratorOrZero(BinaryOperator *Rem) {
   auto *T = Rem->getType();
   auto *N = Rem->getOperand(0), *D = Rem->getOperand(1);
   ICmpInst *ICmp = new ICmpInst(Rem->getIterator(), ICmpInst::ICMP_EQ, N, D);
+  ICmp->setDebugLoc(Rem->getDebugLoc());
   SelectInst *Sel =
       SelectInst::Create(ICmp, ConstantInt::get(T, 0), N, "iv.rem", Rem->getIterator());
   Rem->replaceAllUsesWith(Sel);
diff --git a/llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll b/llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll
new file mode 100644
index 0000000000000..121eec75c1b3c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -p=indvars -S | FileCheck %s
+
+;; When that when IndVarSimplify simplifies the rem to a cmp and select, we
+;; propagate the rem's source location to both the new instructions.
+
+define i32 @widget() !dbg !5 {
+; CHECK-LABEL: define i32 @widget(
+; CHECK-SAME: ) !dbg [[DBG5:![0-9]+]] {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    br label %[[BB1:.*]]
+; CHECK:       [[BB1_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    br label %[[BB2:.*]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, %[[BB1]] ], [ [[ADD:%.*]], %[[BB2]] ]
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 1, [[PHI]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[ADD]], 3, !dbg [[DBG8:![0-9]+]]
+; CHECK-NEXT:    [[IV_REM:%.*]] = select i1 [[TMP0]], i32 0, i32 [[ADD]], !dbg [[DBG8]]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[IV_REM]] to i64
+; CHECK-NEXT:    br i1 false, label %[[BB2]], label %[[BB1_LOOPEXIT]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb2, %bb
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb1
+  %phi = phi i32 [ 0, %bb1 ], [ %add, %bb2 ]
+  %add = add i32 1, %phi
+  %urem = urem i32 %add, 3, !dbg !8
+  %zext = zext i32 %urem to i64
+  %icmp = icmp ult i32 %phi, 2
+  br i1 %icmp, label %bb2, label %bb1
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll", directory: "/")
+!2 = !{i32 8}
+!3 = !{i32 0}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "widget", linkageName: "widget", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 1, column: 1, scope: !5)
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META1:![0-9]+]], producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+; CHECK: [[META1]] = !DIFile(filename: "llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll", directory: {{.*}})
+; CHECK: [[DBG5]] = distinct !DISubprogram(name: "widget", linkageName: "widget", scope: null, file: [[META1]], line: 1, type: [[META6:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]])
+; CHECK: [[META6]] = !DISubroutineType(types: [[META7:![0-9]+]])
+; CHECK: [[META7]] = !{}
+; CHECK: [[DBG8]] = !DILocation(line: 1, column: 1, scope: [[DBG5]])
+;.

From 8210ca019839fc5430b3a95d7caf5c829df3232a Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Thu, 17 Apr 2025 23:37:00 +0700
Subject: [PATCH 292/710] [SPARC] Use lzcnt to implement CTLZ when we have VIS3

Reviewers: s-barannikov, brad0, rorth

Reviewed By: s-barannikov

Pull Request: https://github.com/llvm/llvm-project/pull/135715
---
 llvm/lib/Target/Sparc/SparcISelLowering.cpp |  27 +-
 llvm/lib/Target/Sparc/SparcISelLowering.h   |   6 +
 llvm/lib/Target/Sparc/SparcInstrVIS.td      |  10 +
 llvm/test/CodeGen/SPARC/ctlz.ll             | 405 ++++++++++++++++++++
 4 files changed, 444 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPARC/ctlz.ll

diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 098e5f22834f4..4b61f588f18c8 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/DiagnosticInfo.h"
@@ -1752,8 +1753,7 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
 
     setOperationAction(ISD::CTPOP, MVT::i64,
                        Subtarget->usePopc() ? Legal : Expand);
-    setOperationAction(ISD::CTTZ , MVT::i64, Expand);
-    setOperationAction(ISD::CTLZ , MVT::i64, Expand);
+    setOperationAction(ISD::CTTZ, MVT::i64, Expand);
     setOperationAction(ISD::BSWAP, MVT::i64, Expand);
     setOperationAction(ISD::ROTL , MVT::i64, Expand);
     setOperationAction(ISD::ROTR , MVT::i64, Expand);
@@ -1814,8 +1814,7 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
   setOperationAction(ISD::FREM , MVT::f32, Expand);
   setOperationAction(ISD::FMA  , MVT::f32, Expand);
-  setOperationAction(ISD::CTTZ , MVT::i32, Expand);
-  setOperationAction(ISD::CTLZ , MVT::i32, Expand);
+  setOperationAction(ISD::CTTZ, MVT::i32, Expand);
   setOperationAction(ISD::ROTL , MVT::i32, Expand);
   setOperationAction(ISD::ROTR , MVT::i32, Expand);
   setOperationAction(ISD::BSWAP, MVT::i32, Expand);
@@ -1986,6 +1985,24 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
   if (Subtarget->hasLeonCycleCounter())
     setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
 
+  if (Subtarget->isVIS3()) {
+    setOperationAction(ISD::CTLZ, MVT::i32, Legal);
+    setOperationAction(ISD::CTLZ, MVT::i64, Legal);
+    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Legal);
+    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Legal);
+  } else if (Subtarget->usePopc()) {
+    setOperationAction(ISD::CTLZ, MVT::i32, Expand);
+    setOperationAction(ISD::CTLZ, MVT::i64, Expand);
+    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand);
+    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand);
+  } else {
+    setOperationAction(ISD::CTLZ, MVT::i32, Expand);
+    setOperationAction(ISD::CTLZ, MVT::i64, Expand);
+    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32,
+                       Subtarget->is64Bit() ? Promote : LibCall);
+    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, LibCall);
+  }
+
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 
   setMinFunctionAlignment(Align(4));
@@ -3571,6 +3588,8 @@ bool SparcTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
          Imm.isZero();
 }
 
+bool SparcTargetLowering::isCtlzFast() const { return Subtarget->isVIS3(); }
+
 // Override to disable global variable loading on Linux.
 void SparcTargetLowering::insertSSPDeclarations(Module &M) const {
   if (!Subtarget->isTargetLinux())
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.h b/llvm/lib/Target/Sparc/SparcISelLowering.h
index c09e465f5d05e..ef87e20094c98 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.h
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.h
@@ -210,6 +210,12 @@ namespace llvm {
     bool isFPImmLegal(const APFloat &Imm, EVT VT,
                       bool ForCodeSize) const override;
 
+    bool isCtlzFast() const override;
+
+    bool isCheapToSpeculateCtlz(Type *Ty) const override {
+      return isCtlzFast();
+    }
+
     bool shouldInsertFencesForAtomic(const Instruction *I) const override {
       // FIXME: We insert fences for each atomics and generate
       // sub-optimal code for PSO/TSO. (Approximately nobody uses any
diff --git a/llvm/lib/Target/Sparc/SparcInstrVIS.td b/llvm/lib/Target/Sparc/SparcInstrVIS.td
index ee24d8a54fe8e..78f50888b5508 100644
--- a/llvm/lib/Target/Sparc/SparcInstrVIS.td
+++ b/llvm/lib/Target/Sparc/SparcInstrVIS.td
@@ -294,4 +294,14 @@ def : Pat<(f32 fpnegimm0), (FNEGS (FZEROS))>;
 // VIS3 instruction patterns.
 let Predicates = [HasVIS3] in {
 def : Pat<(i64 (adde i64:$lhs, i64:$rhs)), (ADDXCCC $lhs, $rhs)>;
+
+def : Pat<(i64 (ctlz i64:$src)), (LZCNT $src)>;
+def : Pat<(i64 (ctlz_zero_undef i64:$src)), (LZCNT $src)>;
+// 32-bit LZCNT.
+// The zero extension will leave us with 32 extra leading zeros,
+// so we need to compensate for it.
+// FIXME remove this when the codegen supports using 64-bit values directly
+// in V8+ mode.
+def : Pat<(i32 (ctlz i32:$src)), (ADDri (LZCNT (SRLri $src, 0)), (i32 -32))>;
+def : Pat<(i32 (ctlz_zero_undef i32:$src)), (ADDri (LZCNT (SRLri $src, 0)), (i32 -32))>;
 } // Predicates = [HasVIS3]
diff --git a/llvm/test/CodeGen/SPARC/ctlz.ll b/llvm/test/CodeGen/SPARC/ctlz.ll
new file mode 100644
index 0000000000000..72505f221469e
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/ctlz.ll
@@ -0,0 +1,405 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=sparc | FileCheck %s -check-prefix=SPARC
+; RUN: llc < %s -mtriple=sparc -mattr=popc | FileCheck %s -check-prefix=SPARC-POPC
+; RUN: llc < %s -mtriple=sparc -mattr=vis3 | FileCheck %s -check-prefix=SPARC-VIS3
+; RUN: llc < %s -mtriple=sparcv9 | FileCheck %s -check-prefix=SPARC64
+; RUN: llc < %s -mtriple=sparcv9 -mattr=popc | FileCheck %s -check-prefix=SPARC64-POPC
+; RUN: llc < %s -mtriple=sparcv9 -mattr=vis3 | FileCheck %s -check-prefix=SPARC64-VIS3
+
+define i32 @i32_nopoison(i32 %x) nounwind {
+; SPARC-LABEL: i32_nopoison:
+; SPARC:       ! %bb.0:
+; SPARC-NEXT:    save %sp, -96, %sp
+; SPARC-NEXT:    cmp %i0, 0
+; SPARC-NEXT:    be .LBB0_2
+; SPARC-NEXT:    nop
+; SPARC-NEXT:  ! %bb.1: ! %cond.false
+; SPARC-NEXT:    call __clzsi2
+; SPARC-NEXT:    mov %i0, %o0
+; SPARC-NEXT:    ret
+; SPARC-NEXT:    restore %g0, %o0, %o0
+; SPARC-NEXT:  .LBB0_2:
+; SPARC-NEXT:    ret
+; SPARC-NEXT:    restore %g0, 32, %o0
+;
+; SPARC-POPC-LABEL: i32_nopoison:
+; SPARC-POPC:       ! %bb.0:
+; SPARC-POPC-NEXT:    save %sp, -96, %sp
+; SPARC-POPC-NEXT:    cmp %i0, 0
+; SPARC-POPC-NEXT:    be .LBB0_2
+; SPARC-POPC-NEXT:    nop
+; SPARC-POPC-NEXT:  ! %bb.1: ! %cond.false
+; SPARC-POPC-NEXT:    call __clzsi2
+; SPARC-POPC-NEXT:    mov %i0, %o0
+; SPARC-POPC-NEXT:    ret
+; SPARC-POPC-NEXT:    restore %g0, %o0, %o0
+; SPARC-POPC-NEXT:  .LBB0_2:
+; SPARC-POPC-NEXT:    ret
+; SPARC-POPC-NEXT:    restore %g0, 32, %o0
+;
+; SPARC-VIS3-LABEL: i32_nopoison:
+; SPARC-VIS3:       ! %bb.0:
+; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
+; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
+; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    add %o0, -32, %o0
+;
+; SPARC64-LABEL: i32_nopoison:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    cmp %i0, 0
+; SPARC64-NEXT:    be %icc, .LBB0_2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:  ! %bb.1: ! %cond.false
+; SPARC64-NEXT:    call __clzdi2
+; SPARC64-NEXT:    sllx %i0, 32, %o0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore %g0, %o0, %o0
+; SPARC64-NEXT:  .LBB0_2:
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore %g0, 32, %o0
+;
+; SPARC64-POPC-LABEL: i32_nopoison:
+; SPARC64-POPC:       ! %bb.0:
+; SPARC64-POPC-NEXT:    cmp %o0, 0
+; SPARC64-POPC-NEXT:    be %icc, .LBB0_2
+; SPARC64-POPC-NEXT:    nop
+; SPARC64-POPC-NEXT:  ! %bb.1: ! %cond.false
+; SPARC64-POPC-NEXT:    srl %o0, 1, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 2, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 4, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 8, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 16, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    xor %o0, -1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 0, %o0
+; SPARC64-POPC-NEXT:    retl
+; SPARC64-POPC-NEXT:    popc %o0, %o0
+; SPARC64-POPC-NEXT:  .LBB0_2:
+; SPARC64-POPC-NEXT:    retl
+; SPARC64-POPC-NEXT:    mov 32, %o0
+;
+; SPARC64-VIS3-LABEL: i32_nopoison:
+; SPARC64-VIS3:       ! %bb.0:
+; SPARC64-VIS3-NEXT:    srl %o0, 0, %o0
+; SPARC64-VIS3-NEXT:    lzcnt %o0, %o0
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    add %o0, -32, %o0
+  %ret = call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  ret i32 %ret
+}
+
+define i32 @i32_poison(i32 %x) nounwind {
+; SPARC-LABEL: i32_poison:
+; SPARC:       ! %bb.0:
+; SPARC-NEXT:    save %sp, -96, %sp
+; SPARC-NEXT:    call __clzsi2
+; SPARC-NEXT:    mov %i0, %o0
+; SPARC-NEXT:    ret
+; SPARC-NEXT:    restore %g0, %o0, %o0
+;
+; SPARC-POPC-LABEL: i32_poison:
+; SPARC-POPC:       ! %bb.0:
+; SPARC-POPC-NEXT:    save %sp, -96, %sp
+; SPARC-POPC-NEXT:    call __clzsi2
+; SPARC-POPC-NEXT:    mov %i0, %o0
+; SPARC-POPC-NEXT:    ret
+; SPARC-POPC-NEXT:    restore %g0, %o0, %o0
+;
+; SPARC-VIS3-LABEL: i32_poison:
+; SPARC-VIS3:       ! %bb.0:
+; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
+; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
+; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    add %o0, -32, %o0
+;
+; SPARC64-LABEL: i32_poison:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __clzdi2
+; SPARC64-NEXT:    sllx %i0, 32, %o0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore %g0, %o0, %o0
+;
+; SPARC64-POPC-LABEL: i32_poison:
+; SPARC64-POPC:       ! %bb.0:
+; SPARC64-POPC-NEXT:    srl %o0, 1, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 2, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 4, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 8, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 16, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    xor %o0, -1, %o0
+; SPARC64-POPC-NEXT:    srl %o0, 0, %o0
+; SPARC64-POPC-NEXT:    retl
+; SPARC64-POPC-NEXT:    popc %o0, %o0
+;
+; SPARC64-VIS3-LABEL: i32_poison:
+; SPARC64-VIS3:       ! %bb.0:
+; SPARC64-VIS3-NEXT:    srl %o0, 0, %o0
+; SPARC64-VIS3-NEXT:    lzcnt %o0, %o0
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    add %o0, -32, %o0
+  %ret = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  ret i32 %ret
+}
+
+define i64 @i64_nopoison(i64 %x) nounwind {
+; SPARC-LABEL: i64_nopoison:
+; SPARC:       ! %bb.0:
+; SPARC-NEXT:    save %sp, -96, %sp
+; SPARC-NEXT:    call __clzsi2
+; SPARC-NEXT:    mov %i0, %o0
+; SPARC-NEXT:    cmp %i0, 0
+; SPARC-NEXT:    bne .LBB2_2
+; SPARC-NEXT:    nop
+; SPARC-NEXT:  ! %bb.1:
+; SPARC-NEXT:    srl %i1, 1, %i0
+; SPARC-NEXT:    or %i1, %i0, %i0
+; SPARC-NEXT:    srl %i0, 2, %i1
+; SPARC-NEXT:    or %i0, %i1, %i0
+; SPARC-NEXT:    srl %i0, 4, %i1
+; SPARC-NEXT:    or %i0, %i1, %i0
+; SPARC-NEXT:    srl %i0, 8, %i1
+; SPARC-NEXT:    or %i0, %i1, %i0
+; SPARC-NEXT:    srl %i0, 16, %i1
+; SPARC-NEXT:    or %i0, %i1, %i0
+; SPARC-NEXT:    xor %i0, -1, %i0
+; SPARC-NEXT:    srl %i0, 1, %i1
+; SPARC-NEXT:    sethi 1398101, %i2
+; SPARC-NEXT:    or %i2, 341, %i2
+; SPARC-NEXT:    and %i1, %i2, %i1
+; SPARC-NEXT:    sub %i0, %i1, %i0
+; SPARC-NEXT:    sethi 838860, %i1
+; SPARC-NEXT:    or %i1, 819, %i1
+; SPARC-NEXT:    and %i0, %i1, %i2
+; SPARC-NEXT:    srl %i0, 2, %i0
+; SPARC-NEXT:    and %i0, %i1, %i0
+; SPARC-NEXT:    add %i2, %i0, %i0
+; SPARC-NEXT:    srl %i0, 4, %i1
+; SPARC-NEXT:    add %i0, %i1, %i0
+; SPARC-NEXT:    sethi 246723, %i1
+; SPARC-NEXT:    or %i1, 783, %i1
+; SPARC-NEXT:    and %i0, %i1, %i0
+; SPARC-NEXT:    sll %i0, 8, %i1
+; SPARC-NEXT:    add %i0, %i1, %i0
+; SPARC-NEXT:    sll %i0, 16, %i1
+; SPARC-NEXT:    add %i0, %i1, %i0
+; SPARC-NEXT:    srl %i0, 24, %i0
+; SPARC-NEXT:    add %i0, 32, %o0
+; SPARC-NEXT:  .LBB2_2:
+; SPARC-NEXT:    mov %g0, %i0
+; SPARC-NEXT:    ret
+; SPARC-NEXT:    restore %g0, %o0, %o1
+;
+; SPARC-POPC-LABEL: i64_nopoison:
+; SPARC-POPC:       ! %bb.0:
+; SPARC-POPC-NEXT:    save %sp, -96, %sp
+; SPARC-POPC-NEXT:    call __clzsi2
+; SPARC-POPC-NEXT:    mov %i0, %o0
+; SPARC-POPC-NEXT:    cmp %i0, 0
+; SPARC-POPC-NEXT:    bne .LBB2_2
+; SPARC-POPC-NEXT:    nop
+; SPARC-POPC-NEXT:  ! %bb.1:
+; SPARC-POPC-NEXT:    srl %i1, 1, %i0
+; SPARC-POPC-NEXT:    or %i1, %i0, %i0
+; SPARC-POPC-NEXT:    srl %i0, 2, %i1
+; SPARC-POPC-NEXT:    or %i0, %i1, %i0
+; SPARC-POPC-NEXT:    srl %i0, 4, %i1
+; SPARC-POPC-NEXT:    or %i0, %i1, %i0
+; SPARC-POPC-NEXT:    srl %i0, 8, %i1
+; SPARC-POPC-NEXT:    or %i0, %i1, %i0
+; SPARC-POPC-NEXT:    srl %i0, 16, %i1
+; SPARC-POPC-NEXT:    or %i0, %i1, %i0
+; SPARC-POPC-NEXT:    xor %i0, -1, %i0
+; SPARC-POPC-NEXT:    srl %i0, 1, %i1
+; SPARC-POPC-NEXT:    sethi 1398101, %i2
+; SPARC-POPC-NEXT:    or %i2, 341, %i2
+; SPARC-POPC-NEXT:    and %i1, %i2, %i1
+; SPARC-POPC-NEXT:    sub %i0, %i1, %i0
+; SPARC-POPC-NEXT:    sethi 838860, %i1
+; SPARC-POPC-NEXT:    or %i1, 819, %i1
+; SPARC-POPC-NEXT:    and %i0, %i1, %i2
+; SPARC-POPC-NEXT:    srl %i0, 2, %i0
+; SPARC-POPC-NEXT:    and %i0, %i1, %i0
+; SPARC-POPC-NEXT:    add %i2, %i0, %i0
+; SPARC-POPC-NEXT:    srl %i0, 4, %i1
+; SPARC-POPC-NEXT:    add %i0, %i1, %i0
+; SPARC-POPC-NEXT:    sethi 246723, %i1
+; SPARC-POPC-NEXT:    or %i1, 783, %i1
+; SPARC-POPC-NEXT:    and %i0, %i1, %i0
+; SPARC-POPC-NEXT:    sll %i0, 8, %i1
+; SPARC-POPC-NEXT:    add %i0, %i1, %i0
+; SPARC-POPC-NEXT:    sll %i0, 16, %i1
+; SPARC-POPC-NEXT:    add %i0, %i1, %i0
+; SPARC-POPC-NEXT:    srl %i0, 24, %i0
+; SPARC-POPC-NEXT:    add %i0, 32, %o0
+; SPARC-POPC-NEXT:  .LBB2_2:
+; SPARC-POPC-NEXT:    mov %g0, %i0
+; SPARC-POPC-NEXT:    ret
+; SPARC-POPC-NEXT:    restore %g0, %o0, %o1
+;
+; SPARC-VIS3-LABEL: i64_nopoison:
+; SPARC-VIS3:       ! %bb.0:
+; SPARC-VIS3-NEXT:    cmp %o0, 0
+; SPARC-VIS3-NEXT:    bne .LBB2_2
+; SPARC-VIS3-NEXT:    nop
+; SPARC-VIS3-NEXT:  ! %bb.1:
+; SPARC-VIS3-NEXT:    srl %o1, 0, %o0
+; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
+; SPARC-VIS3-NEXT:    add %o0, -32, %o0
+; SPARC-VIS3-NEXT:    add %o0, 32, %o1
+; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    mov %g0, %o0
+; SPARC-VIS3-NEXT:  .LBB2_2:
+; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
+; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
+; SPARC-VIS3-NEXT:    add %o0, -32, %o1
+; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    mov %g0, %o0
+;
+; SPARC64-LABEL: i64_nopoison:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    brz %i0, .LBB2_2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:  ! %bb.1: ! %cond.false
+; SPARC64-NEXT:    call __clzdi2
+; SPARC64-NEXT:    mov %i0, %o0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore %g0, %o0, %o0
+; SPARC64-NEXT:  .LBB2_2:
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore %g0, 64, %o0
+;
+; SPARC64-POPC-LABEL: i64_nopoison:
+; SPARC64-POPC:       ! %bb.0:
+; SPARC64-POPC-NEXT:    brz %o0, .LBB2_2
+; SPARC64-POPC-NEXT:    nop
+; SPARC64-POPC-NEXT:  ! %bb.1: ! %cond.false
+; SPARC64-POPC-NEXT:    srlx %o0, 1, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 2, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 4, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 8, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 16, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 32, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    xor %o0, -1, %o0
+; SPARC64-POPC-NEXT:    retl
+; SPARC64-POPC-NEXT:    popc %o0, %o0
+; SPARC64-POPC-NEXT:  .LBB2_2:
+; SPARC64-POPC-NEXT:    retl
+; SPARC64-POPC-NEXT:    mov 64, %o0
+;
+; SPARC64-VIS3-LABEL: i64_nopoison:
+; SPARC64-VIS3:       ! %bb.0:
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    lzcnt %o0, %o0
+  %ret = call i64 @llvm.ctlz.i64(i64 %x, i1 false)
+  ret i64 %ret
+}
+
+define i64 @i64_poison(i64 %x) nounwind {
+; SPARC-LABEL: i64_poison:
+; SPARC:       ! %bb.0:
+; SPARC-NEXT:    save %sp, -96, %sp
+; SPARC-NEXT:    mov %i1, %o0
+; SPARC-NEXT:    call __clzsi2
+; SPARC-NEXT:    mov %i0, %i1
+; SPARC-NEXT:    mov %o0, %i0
+; SPARC-NEXT:    call __clzsi2
+; SPARC-NEXT:    mov %i1, %o0
+; SPARC-NEXT:    cmp %i1, 0
+; SPARC-NEXT:    bne .LBB3_2
+; SPARC-NEXT:    nop
+; SPARC-NEXT:  ! %bb.1:
+; SPARC-NEXT:    add %i0, 32, %o0
+; SPARC-NEXT:  .LBB3_2:
+; SPARC-NEXT:    mov %g0, %i0
+; SPARC-NEXT:    ret
+; SPARC-NEXT:    restore %g0, %o0, %o1
+;
+; SPARC-POPC-LABEL: i64_poison:
+; SPARC-POPC:       ! %bb.0:
+; SPARC-POPC-NEXT:    save %sp, -96, %sp
+; SPARC-POPC-NEXT:    mov %i1, %o0
+; SPARC-POPC-NEXT:    call __clzsi2
+; SPARC-POPC-NEXT:    mov %i0, %i1
+; SPARC-POPC-NEXT:    mov %o0, %i0
+; SPARC-POPC-NEXT:    call __clzsi2
+; SPARC-POPC-NEXT:    mov %i1, %o0
+; SPARC-POPC-NEXT:    cmp %i1, 0
+; SPARC-POPC-NEXT:    bne .LBB3_2
+; SPARC-POPC-NEXT:    nop
+; SPARC-POPC-NEXT:  ! %bb.1:
+; SPARC-POPC-NEXT:    add %i0, 32, %o0
+; SPARC-POPC-NEXT:  .LBB3_2:
+; SPARC-POPC-NEXT:    mov %g0, %i0
+; SPARC-POPC-NEXT:    ret
+; SPARC-POPC-NEXT:    restore %g0, %o0, %o1
+;
+; SPARC-VIS3-LABEL: i64_poison:
+; SPARC-VIS3:       ! %bb.0:
+; SPARC-VIS3-NEXT:    cmp %o0, 0
+; SPARC-VIS3-NEXT:    bne .LBB3_2
+; SPARC-VIS3-NEXT:    nop
+; SPARC-VIS3-NEXT:  ! %bb.1:
+; SPARC-VIS3-NEXT:    srl %o1, 0, %o0
+; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
+; SPARC-VIS3-NEXT:    add %o0, -32, %o0
+; SPARC-VIS3-NEXT:    add %o0, 32, %o1
+; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    mov %g0, %o0
+; SPARC-VIS3-NEXT:  .LBB3_2:
+; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
+; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
+; SPARC-VIS3-NEXT:    add %o0, -32, %o1
+; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    mov %g0, %o0
+;
+; SPARC64-LABEL: i64_poison:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __clzdi2
+; SPARC64-NEXT:    mov %i0, %o0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore %g0, %o0, %o0
+;
+; SPARC64-POPC-LABEL: i64_poison:
+; SPARC64-POPC:       ! %bb.0:
+; SPARC64-POPC-NEXT:    srlx %o0, 1, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 2, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 4, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 8, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 16, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    srlx %o0, 32, %o1
+; SPARC64-POPC-NEXT:    or %o0, %o1, %o0
+; SPARC64-POPC-NEXT:    xor %o0, -1, %o0
+; SPARC64-POPC-NEXT:    retl
+; SPARC64-POPC-NEXT:    popc %o0, %o0
+;
+; SPARC64-VIS3-LABEL: i64_poison:
+; SPARC64-VIS3:       ! %bb.0:
+; SPARC64-VIS3-NEXT:    retl
+; SPARC64-VIS3-NEXT:    lzcnt %o0, %o0
+  %ret = call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  ret i64 %ret
+}

From 2ef01048624dc2a564d5925d082a72a51bb97c48 Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Thu, 17 Apr 2025 23:39:46 +0700
Subject: [PATCH 293/710] [SPARC] Use native bitcast instructions when we have
 VIS3

Reviewers: brad0, s-barannikov, rorth

Reviewed By: s-barannikov

Pull Request: https://github.com/llvm/llvm-project/pull/135716
---
 llvm/lib/Target/Sparc/SparcISelLowering.cpp |  12 +-
 llvm/lib/Target/Sparc/SparcInstrVIS.td      |  19 ++-
 llvm/test/CodeGen/SPARC/bitcast.ll          | 139 ++++++++++++++++++++
 3 files changed, 160 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPARC/bitcast.ll

diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 4b61f588f18c8..83108653397fb 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1705,8 +1705,10 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::FP16_TO_FP, MVT::f128, Expand);
   setOperationAction(ISD::FP_TO_FP16, MVT::f128, Expand);
 
-  setOperationAction(ISD::BITCAST, MVT::f32, Expand);
-  setOperationAction(ISD::BITCAST, MVT::i32, Expand);
+  setOperationAction(ISD::BITCAST, MVT::f32,
+                     Subtarget->isVIS3() ? Legal : Expand);
+  setOperationAction(ISD::BITCAST, MVT::i32,
+                     Subtarget->isVIS3() ? Legal : Expand);
 
   // Sparc has no select or setcc: expand to SELECT_CC.
   setOperationAction(ISD::SELECT, MVT::i32, Expand);
@@ -1744,8 +1746,10 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
   }
 
   if (Subtarget->is64Bit()) {
-    setOperationAction(ISD::BITCAST, MVT::f64, Expand);
-    setOperationAction(ISD::BITCAST, MVT::i64, Expand);
+    setOperationAction(ISD::BITCAST, MVT::f64,
+                       Subtarget->isVIS3() ? Legal : Expand);
+    setOperationAction(ISD::BITCAST, MVT::i64,
+                       Subtarget->isVIS3() ? Legal : Expand);
     setOperationAction(ISD::SELECT, MVT::i64, Expand);
     setOperationAction(ISD::SETCC, MVT::i64, Expand);
     setOperationAction(ISD::BR_CC, MVT::i64, Custom);
diff --git a/llvm/lib/Target/Sparc/SparcInstrVIS.td b/llvm/lib/Target/Sparc/SparcInstrVIS.td
index 78f50888b5508..fdb3468a70392 100644
--- a/llvm/lib/Target/Sparc/SparcInstrVIS.td
+++ b/llvm/lib/Target/Sparc/SparcInstrVIS.td
@@ -259,14 +259,14 @@ def LZCNT     : VISInstFormat<0b000010111, (outs I64Regs:$rd),
                    (ins I64Regs:$rs2), "lzcnt $rs2, $rd">;
 
 let rs1 = 0 in {
-def MOVSTOSW : VISInstFormat<0b100010011, (outs I64Regs:$rd),
-                   (ins DFPRegs:$rs2), "movstosw $rs2, $rd">;
-def MOVSTOUW : VISInstFormat<0b100010001, (outs I64Regs:$rd),
-                   (ins DFPRegs:$rs2), "movstouw $rs2, $rd">;
+def MOVSTOSW : VISInstFormat<0b100010011, (outs IntRegs:$rd),
+                   (ins FPRegs:$rs2), "movstosw $rs2, $rd">;
+def MOVSTOUW : VISInstFormat<0b100010001, (outs IntRegs:$rd),
+                   (ins FPRegs:$rs2), "movstouw $rs2, $rd">;
 def MOVDTOX  : VISInstFormat<0b100010000, (outs I64Regs:$rd),
                    (ins DFPRegs:$rs2), "movdtox $rs2, $rd">;
-def MOVWTOS  :  VISInstFormat<0b100011001, (outs DFPRegs:$rd),
-                   (ins I64Regs:$rs2), "movwtos $rs2, $rd">;
+def MOVWTOS  :  VISInstFormat<0b100011001, (outs FPRegs:$rd),
+                   (ins IntRegs:$rs2), "movwtos $rs2, $rd">;
 def MOVXTOD  :  VISInstFormat<0b100011000, (outs DFPRegs:$rd),
                    (ins I64Regs:$rs2), "movxtod $rs2, $rd">;
 }
@@ -304,4 +304,11 @@ def : Pat<(i64 (ctlz_zero_undef i64:$src)), (LZCNT $src)>;
 // in V8+ mode.
 def : Pat<(i32 (ctlz i32:$src)), (ADDri (LZCNT (SRLri $src, 0)), (i32 -32))>;
 def : Pat<(i32 (ctlz_zero_undef i32:$src)), (ADDri (LZCNT (SRLri $src, 0)), (i32 -32))>;
+
+def : Pat<(i32 (bitconvert f32:$src)), (MOVSTOUW $src)>;
+def : Pat<(i64 (zanyext (i32 (bitconvert f32:$src)))), (MOVSTOUW $src)>;
+def : Pat<(i64 (sext (i32 (bitconvert f32:$src)))), (MOVSTOSW $src)>;
+def : Pat<(f32 (bitconvert i32:$src)), (MOVWTOS $src)>;
+def : Pat<(i64 (bitconvert f64:$src)), (MOVDTOX $src)>;
+def : Pat<(f64 (bitconvert i64:$src)), (MOVXTOD $src)>;
 } // Predicates = [HasVIS3]
diff --git a/llvm/test/CodeGen/SPARC/bitcast.ll b/llvm/test/CodeGen/SPARC/bitcast.ll
new file mode 100644
index 0000000000000..ab13f86105cfe
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/bitcast.ll
@@ -0,0 +1,139 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=sparcv9 | FileCheck %s -check-prefix=V9
+; RUN: llc < %s -mtriple=sparcv9 -mattr=vis3 | FileCheck %s -check-prefix=VIS3
+
+define i32 @stow(float %float) nounwind {
+; V9-LABEL: stow:
+; V9:       ! %bb.0:
+; V9-NEXT:    add %sp, -144, %sp
+; V9-NEXT:    st %f1, [%sp+2187]
+; V9-NEXT:    ld [%sp+2187], %o0
+; V9-NEXT:    retl
+; V9-NEXT:    add %sp, 144, %sp
+;
+; VIS3-LABEL: stow:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    movstouw %f1, %o0
+  %w = bitcast float %float to i32
+  ret i32 %w
+}
+
+define zeroext i32 @stouw(float %float) nounwind {
+; V9-LABEL: stouw:
+; V9:       ! %bb.0:
+; V9-NEXT:    add %sp, -144, %sp
+; V9-NEXT:    st %f1, [%sp+2187]
+; V9-NEXT:    ld [%sp+2187], %o0
+; V9-NEXT:    retl
+; V9-NEXT:    add %sp, 144, %sp
+;
+; VIS3-LABEL: stouw:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    movstouw %f1, %o0
+  %uw = bitcast float %float to i32
+  ret i32 %uw
+}
+
+define signext i32 @stosw(float %float) nounwind {
+; V9-LABEL: stosw:
+; V9:       ! %bb.0:
+; V9-NEXT:    add %sp, -144, %sp
+; V9-NEXT:    st %f1, [%sp+2187]
+; V9-NEXT:    ldsw [%sp+2187], %o0
+; V9-NEXT:    retl
+; V9-NEXT:    add %sp, 144, %sp
+;
+; VIS3-LABEL: stosw:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    movstosw %f1, %o0
+  %sw = bitcast float %float to i32
+  ret i32 %sw
+}
+
+define float @wtos(i32 %w) nounwind {
+; V9-LABEL: wtos:
+; V9:       ! %bb.0:
+; V9-NEXT:    add %sp, -144, %sp
+; V9-NEXT:    st %o0, [%sp+2187]
+; V9-NEXT:    ld [%sp+2187], %f0
+; V9-NEXT:    retl
+; V9-NEXT:    add %sp, 144, %sp
+;
+; VIS3-LABEL: wtos:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    movwtos %o0, %f0
+  %float = bitcast i32 %w to float
+  ret float %float
+}
+
+define float @uwtos(i32 zeroext %uw) nounwind {
+; V9-LABEL: uwtos:
+; V9:       ! %bb.0:
+; V9-NEXT:    add %sp, -144, %sp
+; V9-NEXT:    st %o0, [%sp+2187]
+; V9-NEXT:    ld [%sp+2187], %f0
+; V9-NEXT:    retl
+; V9-NEXT:    add %sp, 144, %sp
+;
+; VIS3-LABEL: uwtos:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    movwtos %o0, %f0
+  %float = bitcast i32 %uw to float
+  ret float %float
+}
+
+define float @swtos(i32 signext %sw) nounwind {
+; V9-LABEL: swtos:
+; V9:       ! %bb.0:
+; V9-NEXT:    add %sp, -144, %sp
+; V9-NEXT:    st %o0, [%sp+2187]
+; V9-NEXT:    ld [%sp+2187], %f0
+; V9-NEXT:    retl
+; V9-NEXT:    add %sp, 144, %sp
+;
+; VIS3-LABEL: swtos:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    movwtos %o0, %f0
+  %float = bitcast i32 %sw to float
+  ret float %float
+}
+
+define i64 @dtox(double %double) nounwind {
+; V9-LABEL: dtox:
+; V9:       ! %bb.0:
+; V9-NEXT:    add %sp, -144, %sp
+; V9-NEXT:    std %f0, [%sp+2183]
+; V9-NEXT:    ldx [%sp+2183], %o0
+; V9-NEXT:    retl
+; V9-NEXT:    add %sp, 144, %sp
+;
+; VIS3-LABEL: dtox:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    movdtox %f0, %o0
+  %x = bitcast double %double to i64
+  ret i64 %x
+}
+
+define double @xtod(i64 %x) nounwind {
+; V9-LABEL: xtod:
+; V9:       ! %bb.0:
+; V9-NEXT:    add %sp, -144, %sp
+; V9-NEXT:    stx %o0, [%sp+2183]
+; V9-NEXT:    ldd [%sp+2183], %f0
+; V9-NEXT:    retl
+; V9-NEXT:    add %sp, 144, %sp
+;
+; VIS3-LABEL: xtod:
+; VIS3:       ! %bb.0:
+; VIS3-NEXT:    retl
+; VIS3-NEXT:    movxtod %o0, %f0
+  %double = bitcast i64 %x to double
+  ret double %double
+}

From 52e10e6c3bad782380c8a931aabca2800b53a85d Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <arvind.sudarsanam@intel.com>
Date: Thu, 17 Apr 2025 12:48:08 -0400
Subject: [PATCH 294/710] [SYCL] Add clang-linker-wrapper changes to call
 clang-sycl-linker for SYCL offloads (#135683)

This PR is one of the many PRs in the SYCL upstreaming effort focusing
on device code linking during the SYCL offload compilation process. RFC:
https://discourse.llvm.org/t/rfc-offloading-design-for-sycl-offload-kind-and-spir-targets/74088

Approved PRs so far:
1. [Clang][SYCL] Introduce clang-sycl-linker to link SYCL offloading
device code (Part 1 of many) -
[Link](https://github.com/llvm/llvm-project/pull/112245)
2. [clang-sycl-linker] Replace llvm-link with API calls -
[Link](https://github.com/llvm/llvm-project/pull/133797)
3. [SYCL][SPIR-V Backend][clang-sycl-linker] Add SPIR-V backend support
inside clang-sycl-linker -
[Link](https://github.com/llvm/llvm-project/pull/133967)

This PR adds SYCL device code linking support to clang-linker-wrapper.

### Summary for this PR

Device code linking happens inside clang-linker-wrapper. In the current
implementation, clang-linker-wrapper does the following:

1. Extracts device code. Input_1, Input_2,.....
5. Group device code according to target devices Inputs[triple_1] = ....
Inputs[triple_2] = ....
6. For each group, i.e. Inputs[triple_i], a. Gather all the offload
kinds found inside those inputs in ActiveOffloadKinds b. Link all images
inside Inputs[triple_i] by calling clang --target=triple_i .... c.
Create a copy of that linked image for each offload kind and add it to
Output[Kind] list.

In SYCL compilation flow, there is a deviation in Step 3b. We call
device code splitting inside the 'clang --target=triple_i ....' call and
the output is now a 'packaged' file containing multiple device images.
This deviation requires us to capture the OffloadKind during the linking
stage and pass it along to the linking function (clang), so that clang
can be called with a unique option '--sycl-link' that will help us to
call 'clang-sycl-linker' under the hood (clang-sycl-linker will do SYCL
specific linking).

Our current objective is to implement an end-to-end SYCL offloading flow
and get it working. We will eventually merge our approach with the
community flow.

Thanks

---------

Signed-off-by: Arvind Sudarsanam <arvind.sudarsanam@intel.com>
---
 clang/docs/ClangOffloadPackager.rst           |  2 +
 clang/test/Driver/clang-sycl-linker-test.cpp  |  4 +-
 clang/test/Driver/linker-wrapper.c            | 10 +++
 .../ClangLinkerWrapper.cpp                    | 31 ++++++++--
 .../clang-sycl-linker/ClangSYCLLinker.cpp     | 62 ++++++++++++++++---
 llvm/lib/Object/OffloadBinary.cpp             |  3 +
 6 files changed, 98 insertions(+), 14 deletions(-)

diff --git a/clang/docs/ClangOffloadPackager.rst b/clang/docs/ClangOffloadPackager.rst
index 2b985e260e302..481069b5e4235 100644
--- a/clang/docs/ClangOffloadPackager.rst
+++ b/clang/docs/ClangOffloadPackager.rst
@@ -112,6 +112,8 @@ the following values for the :ref:`offload kind<table-offload_kind>` and the
     +------------+-------+---------------------------------------+
     | OFK_HIP    | 0x03  | The producer was HIP                  |
     +------------+-------+---------------------------------------+
+    | OFK_SYCL   | 0x04  | The producer was SYCL                 |
+    +------------+-------+---------------------------------------+
 
 The flags are used to signify certain conditions, such as the presence of
 debugging information or whether or not LTO was used. The string entry table is
diff --git a/clang/test/Driver/clang-sycl-linker-test.cpp b/clang/test/Driver/clang-sycl-linker-test.cpp
index c399689653784..1dfba391fbc3e 100644
--- a/clang/test/Driver/clang-sycl-linker-test.cpp
+++ b/clang/test/Driver/clang-sycl-linker-test.cpp
@@ -8,7 +8,7 @@
 // RUN: clang-sycl-linker --dry-run -v -triple=spirv64 %t_1.bc %t_2.bc -o a.spv 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=SIMPLE-FO
 // SIMPLE-FO: sycl-device-link: inputs: {{.*}}.bc, {{.*}}.bc  libfiles:  output: [[LLVMLINKOUT:.*]].bc
-// SIMPLE-FO-NEXT: SPIR-V Backend: input: [[LLVMLINKOUT]].bc, output: a.spv
+// SIMPLE-FO-NEXT: SPIR-V Backend: input: [[LLVMLINKOUT]].bc, output: a_0.spv
 //
 // Test the dry run of a simple case with device library files specified.
 // RUN: touch %T/lib1.bc
@@ -16,7 +16,7 @@
 // RUN: clang-sycl-linker --dry-run -v -triple=spirv64 %t_1.bc %t_2.bc --library-path=%T --device-libs=lib1.bc,lib2.bc -o a.spv 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=DEVLIBS
 // DEVLIBS: sycl-device-link: inputs: {{.*}}.bc  libfiles: {{.*}}lib1.bc, {{.*}}lib2.bc  output: [[LLVMLINKOUT:.*]].bc
-// DEVLIBS-NEXT: SPIR-V Backend: input: [[LLVMLINKOUT]].bc, output: a.spv
+// DEVLIBS-NEXT: SPIR-V Backend: input: [[LLVMLINKOUT]].bc, output: a_0.spv
 //
 // Test a simple case with a random file (not bitcode) as input.
 // RUN: touch %t.o
diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c
index 0c77c2b34216a..a7e98e7351d98 100644
--- a/clang/test/Driver/linker-wrapper.c
+++ b/clang/test/Driver/linker-wrapper.c
@@ -2,6 +2,7 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: nvptx-registered-target
 // REQUIRES: amdgpu-registered-target
+// REQUIRES: spirv-registered-target
 
 // An externally visible variable so static libraries extract.
 __attribute__((visibility("protected"), used)) int x;
@@ -9,6 +10,7 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.elf.o
 // RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -o %t.nvptx.bc
 // RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -o %t.amdgpu.bc
+// RUN: %clang -cc1 %s -triple spirv64-unknown-unknown -emit-llvm-bc -o %t.spirv.bc
 
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
@@ -49,6 +51,14 @@ __attribute__((visibility("protected"), used)) int x;
 
 // AMDGPU-LTO-TEMPS: clang{{.*}} --target=amdgcn-amd-amdhsa -mcpu=gfx1030 -flto {{.*}}-save-temps
 
+// RUN: clang-offload-packager -o %t.out \
+// RUN:   --image=file=%t.spirv.bc,kind=sycl,triple=spirv64-unknown-unknown,arch=generic
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: not clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
+// RUN:   --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=SPIRV-LINK
+
+// SPIRV-LINK: clang{{.*}} -o {{.*}}.img --target=spirv64-unknown-unknown {{.*}}.o --sycl-link -Xlinker -triple=spirv64-unknown-unknown -Xlinker -arch=
+
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=x86_64-unknown-linux-gnu \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=x86_64-unknown-linux-gnu
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 082355e6c716f..4e716e818a232 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -464,7 +464,8 @@ fatbinary(ArrayRef<std::pair<StringRef, StringRef>> InputFiles,
 } // namespace amdgcn
 
 namespace generic {
-Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
+Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args,
+                          uint16_t ActiveOffloadKindMask) {
   llvm::TimeTraceScope TimeScope("Clang");
   // Use `clang` to invoke the appropriate device tools.
   Expected<std::string> ClangPath =
@@ -554,6 +555,17 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
   if (Args.hasArg(OPT_embed_bitcode))
     CmdArgs.push_back("-Wl,--lto-emit-llvm");
 
+  // For linking device code with the SYCL offload kind, special handling is
+  // required. Passing --sycl-link to clang results in a call to
+  // clang-sycl-linker. Additional linker flags required by clang-sycl-linker
+  // will be communicated via the -Xlinker option.
+  if (ActiveOffloadKindMask & OFK_SYCL) {
+    CmdArgs.push_back("--sycl-link");
+    CmdArgs.append(
+        {"-Xlinker", Args.MakeArgString("-triple=" + Triple.getTriple())});
+    CmdArgs.append({"-Xlinker", Args.MakeArgString("-arch=" + Arch)});
+  }
+
   for (StringRef Arg : Args.getAllArgValues(OPT_linker_arg_EQ))
     CmdArgs.append({"-Xlinker", Args.MakeArgString(Arg)});
   for (StringRef Arg : Args.getAllArgValues(OPT_compiler_arg_EQ))
@@ -567,7 +579,8 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
 } // namespace generic
 
 Expected<StringRef> linkDevice(ArrayRef<StringRef> InputFiles,
-                               const ArgList &Args) {
+                               const ArgList &Args,
+                               uint16_t ActiveOffloadKindMask) {
   const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
   switch (Triple.getArch()) {
   case Triple::nvptx:
@@ -582,7 +595,7 @@ Expected<StringRef> linkDevice(ArrayRef<StringRef> InputFiles,
   case Triple::spirv64:
   case Triple::systemz:
   case Triple::loongarch64:
-    return generic::clang(InputFiles, Args);
+    return generic::clang(InputFiles, Args, ActiveOffloadKindMask);
   default:
     return createStringError(Triple.getArchName() +
                              " linking is not supported");
@@ -792,6 +805,7 @@ bundleLinkedOutput(ArrayRef<OffloadingImage> Images, const ArgList &Args,
   llvm::TimeTraceScope TimeScope("Bundle linked output");
   switch (Kind) {
   case OFK_OpenMP:
+  case OFK_SYCL:
     return bundleOpenMP(Images);
   case OFK_Cuda:
     return bundleCuda(Images, Args);
@@ -927,6 +941,14 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
     for (const auto &File : Input)
       ActiveOffloadKindMask |= File.getBinary()->getOffloadKind();
 
+    // Linking images of SYCL offload kind with images of other kind is not
+    // supported.
+    // TODO: Remove the above limitation.
+    if ((ActiveOffloadKindMask & OFK_SYCL) &&
+        ((ActiveOffloadKindMask ^ OFK_SYCL) != 0))
+      return createStringError("Linking images of SYCL offload kind with "
+                               "images of any other kind is not supported");
+
     // Write any remaining device inputs to an output file.
     SmallVector<StringRef> InputFiles;
     for (const OffloadFile &File : Input) {
@@ -937,7 +959,8 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
     }
 
     // Link the remaining device files using the device linker.
-    auto OutputOrErr = linkDevice(InputFiles, LinkerArgs);
+    auto OutputOrErr =
+        linkDevice(InputFiles, LinkerArgs, ActiveOffloadKindMask);
     if (!OutputOrErr)
       return OutputOrErr.takeError();
 
diff --git a/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp b/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp
index c640deddc9e74..79747b0550417 100644
--- a/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp
+++ b/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp
@@ -70,6 +70,8 @@ static StringRef OutputFile;
 /// Directory to dump SPIR-V IR if requested by user.
 static SmallString<128> SPIRVDumpDir;
 
+using OffloadingImage = OffloadBinary::OffloadingImage;
+
 static void printVersion(raw_ostream &OS) {
   OS << clang::getClangToolFullVersion("clang-sycl-linker") << '\n';
 }
@@ -278,8 +280,8 @@ Expected<StringRef> linkDeviceCode(ArrayRef<std::string> InputFiles,
 /// Converts 'File' from LLVM bitcode to SPIR-V format using SPIR-V backend.
 /// 'Args' encompasses all arguments required for linking device code and will
 /// be parsed to generate options required to be passed into the backend.
-static Expected<StringRef> runSPIRVCodeGen(StringRef File, const ArgList &Args,
-                                           LLVMContext &C) {
+static Error runSPIRVCodeGen(StringRef File, const ArgList &Args,
+                             StringRef OutputFile, LLVMContext &C) {
   llvm::TimeTraceScope TimeScope("SPIR-V code generation");
 
   // Parse input module.
@@ -289,7 +291,7 @@ static Expected<StringRef> runSPIRVCodeGen(StringRef File, const ArgList &Args,
     return createStringError(Err.getMessage());
 
   if (Error Err = M->materializeAll())
-    return std::move(Err);
+    return Err;
 
   Triple TargetTriple(Args.getLastArgValue(OPT_triple_EQ));
   M->setTargetTriple(TargetTriple);
@@ -333,7 +335,7 @@ static Expected<StringRef> runSPIRVCodeGen(StringRef File, const ArgList &Args,
     errs() << formatv("SPIR-V Backend: input: {0}, output: {1}\n", File,
                       OutputFile);
 
-  return OutputFile;
+  return Error::success();
 }
 
 /// Performs the following steps:
@@ -349,10 +351,54 @@ Error runSYCLLink(ArrayRef<std::string> Files, const ArgList &Args) {
   if (!LinkedFile)
     reportError(LinkedFile.takeError());
 
+  // TODO: SYCL post link functionality involves device code splitting and will
+  // result in multiple bitcode codes.
+  // The following lines are placeholders to represent multiple files and will
+  // be refactored once SYCL post link support is available.
+  SmallVector<std::string> SplitModules;
+  SplitModules.emplace_back(*LinkedFile);
+
   // SPIR-V code generation step.
-  auto SPVFile = runSPIRVCodeGen(*LinkedFile, Args, C);
-  if (!SPVFile)
-    return SPVFile.takeError();
+  for (size_t I = 0, E = SplitModules.size(); I != E; ++I) {
+    auto Stem = OutputFile.rsplit('.').first;
+    std::string SPVFile(Stem);
+    SPVFile.append("_" + utostr(I) + ".spv");
+    auto Err = runSPIRVCodeGen(SplitModules[I], Args, SPVFile, C);
+    if (Err)
+      return std::move(Err);
+    SplitModules[I] = SPVFile;
+  }
+
+  // Write the final output into file.
+  int FD = -1;
+  if (std::error_code EC = sys::fs::openFileForWrite(OutputFile, FD))
+    return errorCodeToError(EC);
+  llvm::raw_fd_ostream FS(FD, /*shouldClose=*/true);
+
+  for (size_t I = 0, E = SplitModules.size(); I != E; ++I) {
+    auto File = SplitModules[I];
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileOrErr =
+        llvm::MemoryBuffer::getFileOrSTDIN(File);
+    if (std::error_code EC = FileOrErr.getError()) {
+      if (DryRun)
+        FileOrErr = MemoryBuffer::getMemBuffer("");
+      else
+        return createFileError(File, EC);
+    }
+    OffloadingImage TheImage{};
+    TheImage.TheImageKind = IMG_Object;
+    TheImage.TheOffloadKind = OFK_SYCL;
+    TheImage.StringData["triple"] =
+        Args.MakeArgString(Args.getLastArgValue(OPT_triple_EQ));
+    TheImage.StringData["arch"] =
+        Args.MakeArgString(Args.getLastArgValue(OPT_arch_EQ));
+    TheImage.Image = std::move(*FileOrErr);
+
+    llvm::SmallString<0> Buffer = OffloadBinary::write(TheImage);
+    if (Buffer.size() % OffloadBinary::getAlignment() != 0)
+      return createStringError("Offload binary has invalid size alignment");
+    FS << Buffer;
+  }
   return Error::success();
 }
 
@@ -394,7 +440,7 @@ int main(int argc, char **argv) {
   DryRun = Args.hasArg(OPT_dry_run);
   SaveTemps = Args.hasArg(OPT_save_temps);
 
-  OutputFile = "a.spv";
+  OutputFile = "a.out";
   if (Args.hasArg(OPT_o))
     OutputFile = Args.getLastArgValue(OPT_o);
 
diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp
index 56687c9acb653..3fff6b6a09e08 100644
--- a/llvm/lib/Object/OffloadBinary.cpp
+++ b/llvm/lib/Object/OffloadBinary.cpp
@@ -301,6 +301,7 @@ OffloadKind object::getOffloadKind(StringRef Name) {
       .Case("openmp", OFK_OpenMP)
       .Case("cuda", OFK_Cuda)
       .Case("hip", OFK_HIP)
+      .Case("sycl", OFK_SYCL)
       .Default(OFK_None);
 }
 
@@ -312,6 +313,8 @@ StringRef object::getOffloadKindName(OffloadKind Kind) {
     return "cuda";
   case OFK_HIP:
     return "hip";
+  case OFK_SYCL:
+    return "sycl";
   default:
     return "none";
   }

From 9ed4c705ac1c5c8797f328694f6cd22fbcdae03b Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Thu, 17 Apr 2025 09:52:41 -0700
Subject: [PATCH 295/710] [MTE] decide whether to tag global in AsmPrinter
 (#135891)

there are llvm passes that would invalidate the decision we make in
clang.
---
 clang/lib/CodeGen/SanitizerMetadata.cpp    | 45 +++------------------
 clang/test/CodeGen/memtag-globals.cpp      |  7 ++--
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 47 ++++++++++++++++++++++
 llvm/lib/IR/Verifier.cpp                   |  4 --
 llvm/unittests/IR/VerifierTest.cpp         | 21 ----------
 5 files changed, 56 insertions(+), 68 deletions(-)

diff --git a/clang/lib/CodeGen/SanitizerMetadata.cpp b/clang/lib/CodeGen/SanitizerMetadata.cpp
index b7b212ba46efd..288f7f6415f44 100644
--- a/clang/lib/CodeGen/SanitizerMetadata.cpp
+++ b/clang/lib/CodeGen/SanitizerMetadata.cpp
@@ -32,37 +32,6 @@ static SanitizerMask expandKernelSanitizerMasks(SanitizerMask Mask) {
   return Mask;
 }
 
-static bool shouldTagGlobal(const llvm::GlobalVariable &G) {
-  // For now, don't instrument constant data, as it'll be in .rodata anyway. It
-  // may be worth instrumenting these in future to stop them from being used as
-  // gadgets.
-  if (G.getName().starts_with("llvm.") || G.isThreadLocal() || G.isConstant())
-    return false;
-
-  // Globals can be placed implicitly or explicitly in sections. There's two
-  // different types of globals that meet this criteria that cause problems:
-  //  1. Function pointers that are going into various init arrays (either
-  //     explicitly through `__attribute__((section(<foo>)))` or implicitly
-  //     through `__attribute__((constructor)))`, such as ".(pre)init(_array)",
-  //     ".fini(_array)", ".ctors", and ".dtors". These function pointers end up
-  //     overaligned and overpadded, making iterating over them problematic, and
-  //     each function pointer is individually tagged (so the iteration over
-  //     them causes SIGSEGV/MTE[AS]ERR).
-  //  2. Global variables put into an explicit section, where the section's name
-  //     is a valid C-style identifier. The linker emits a `__start_<name>` and
-  //     `__stop_<name>` symbol for the section, so that you can iterate over
-  //     globals within this section. Unfortunately, again, these globals would
-  //     be tagged and so iteration causes SIGSEGV/MTE[AS]ERR.
-  //
-  // To mitigate both these cases, and because specifying a section is rare
-  // outside of these two cases, disable MTE protection for globals in any
-  // section.
-  if (G.hasSection())
-    return false;
-
-  return true;
-}
-
 void SanitizerMetadata::reportGlobal(llvm::GlobalVariable *GV,
                                      SourceLocation Loc, StringRef Name,
                                      QualType Ty,
@@ -89,15 +58,11 @@ void SanitizerMetadata::reportGlobal(llvm::GlobalVariable *GV,
   Meta.NoHWAddress |= CGM.isInNoSanitizeList(
       FsanitizeArgument.Mask & SanitizerKind::HWAddress, GV, Loc, Ty);
 
-  if (shouldTagGlobal(*GV)) {
-    Meta.Memtag |= static_cast<bool>(FsanitizeArgument.Mask &
-                                     SanitizerKind::MemtagGlobals);
-    Meta.Memtag &= !NoSanitizeAttrSet.hasOneOf(SanitizerKind::MemTag);
-    Meta.Memtag &= !CGM.isInNoSanitizeList(
-        FsanitizeArgument.Mask & SanitizerKind::MemTag, GV, Loc, Ty);
-  } else {
-    Meta.Memtag = false;
-  }
+  Meta.Memtag |=
+      static_cast<bool>(FsanitizeArgument.Mask & SanitizerKind::MemtagGlobals);
+  Meta.Memtag &= !NoSanitizeAttrSet.hasOneOf(SanitizerKind::MemTag);
+  Meta.Memtag &= !CGM.isInNoSanitizeList(
+      FsanitizeArgument.Mask & SanitizerKind::MemTag, GV, Loc, Ty);
 
   Meta.IsDynInit = IsDynInit && !Meta.NoAddress &&
                    FsanitizeArgument.has(SanitizerKind::Address) &&
diff --git a/clang/test/CodeGen/memtag-globals.cpp b/clang/test/CodeGen/memtag-globals.cpp
index 0d7fe22a7b0f0..407eea1c37cfa 100644
--- a/clang/test/CodeGen/memtag-globals.cpp
+++ b/clang/test/CodeGen/memtag-globals.cpp
@@ -25,8 +25,9 @@ void func() {
 // CHECK: @{{.*}}extra_global{{.*}} ={{.*}} sanitize_memtag
 // CHECK: @{{.*}}global{{.*}} ={{.*}} sanitize_memtag
 
-// CHECK:     @{{.*}}section_global{{.*}} =
-// CHECK-NOT: sanitize_memtag
+// This DOES NOT mean we are instrumenting the section global,
+// but we are ignoring it in AsmPrinter rather than in clang.
+// CHECK:     @{{.*}}section_global{{.*}} ={{.*}} sanitize_memtag
 // CHECK:     @{{.*}}attributed_global{{.*}} =
 // CHECK-NOT: sanitize_memtag
 // CHECK:     @{{.*}}disable_instrumentation_global{{.*}} =
@@ -35,7 +36,7 @@ void func() {
 // CHECK-NOT: sanitize_memtag
 
 // CHECK: @{{.*}}static_var{{.*}} ={{.*}} sanitize_memtag
-// CHECK: @{{.*}} = {{.*}} c"Hello, world!\00"{{.*}}
+// CHECK: @{{.*}} = {{.*}} c"Hello, world!\00"{{.*}} sanitize_memtag
 // CHECK: @{{.*}}external_global{{.*}} ={{.*}} sanitize_memtag
 
 // IGNORELIST: @{{.*}}extra_global{{.*}} ={{.*}} sanitize_memtag
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 4b05ad398f053..6b1f3df12ee44 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2398,6 +2398,41 @@ void AsmPrinter::emitRemarksSection(remarks::RemarkStreamer &RS) {
   OutStreamer->emitBinaryData(Buf);
 }
 
+static bool shouldTagGlobal(const llvm::GlobalVariable &G) {
+  // We used to do this in clang, but there are optimization passes that turn
+  // non-constant globals into constants. So now, clang only tells us whether
+  // it would *like* a global to be tagged, but we still make the decision here.
+  //
+  // For now, don't instrument constant data, as it'll be in .rodata anyway. It
+  // may be worth instrumenting these in future to stop them from being used as
+  // gadgets.
+  if (G.getName().starts_with("llvm.") || G.isThreadLocal() || G.isConstant())
+    return false;
+
+  // Globals can be placed implicitly or explicitly in sections. There's two
+  // different types of globals that meet this criteria that cause problems:
+  //  1. Function pointers that are going into various init arrays (either
+  //     explicitly through `__attribute__((section(<foo>)))` or implicitly
+  //     through `__attribute__((constructor)))`, such as ".(pre)init(_array)",
+  //     ".fini(_array)", ".ctors", and ".dtors". These function pointers end up
+  //     overaligned and overpadded, making iterating over them problematic, and
+  //     each function pointer is individually tagged (so the iteration over
+  //     them causes SIGSEGV/MTE[AS]ERR).
+  //  2. Global variables put into an explicit section, where the section's name
+  //     is a valid C-style identifier. The linker emits a `__start_<name>` and
+  //     `__stop_<name>` symbol for the section, so that you can iterate over
+  //     globals within this section. Unfortunately, again, these globals would
+  //     be tagged and so iteration causes SIGSEGV/MTE[AS]ERR.
+  //
+  // To mitigate both these cases, and because specifying a section is rare
+  // outside of these two cases, disable MTE protection for globals in any
+  // section.
+  if (G.hasSection())
+    return false;
+
+  return true;
+}
+
 static void tagGlobalDefinition(Module &M, GlobalVariable *G) {
   Constant *Initializer = G->getInitializer();
   uint64_t SizeInBytes =
@@ -2430,6 +2465,12 @@ static void tagGlobalDefinition(Module &M, GlobalVariable *G) {
   G->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
 }
 
+static void removeMemtagFromGlobal(GlobalVariable &G) {
+  auto Meta = G.getSanitizerMetadata();
+  Meta.Memtag = false;
+  G.setSanitizerMetadata(Meta);
+}
+
 bool AsmPrinter::doFinalization(Module &M) {
   // Set the MachineFunction to nullptr so that we can catch attempted
   // accesses to MF specific features at the module level and so that
@@ -2440,6 +2481,12 @@ bool AsmPrinter::doFinalization(Module &M) {
   for (GlobalVariable &G : M.globals()) {
     if (G.isDeclaration() || !G.isTagged())
       continue;
+    if (!shouldTagGlobal(G)) {
+      assert(G.hasSanitizerMetadata()); // because isTagged.
+      removeMemtagFromGlobal(G);
+      assert(!G.isTagged());
+      continue;
+    }
     GlobalsToTag.push_back(&G);
   }
   for (GlobalVariable *G : GlobalsToTag)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 765befc95032c..35c4d60cf325e 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -807,10 +807,6 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
           "visibility must be dso_local!",
           &GV);
 
-  if (GV.isTagged()) {
-    Check(!GV.hasSection(), "tagged GlobalValue must not be in section.", &GV);
-  }
-
   forEachUser(&GV, GlobalValueVisited, [&](const Value *V) -> bool {
     if (const Instruction *I = dyn_cast<Instruction>(V)) {
       if (!I->getParent() || !I->getParent()->getParent())
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 2b51f3c4ea2c2..7a136e6a382a7 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -416,26 +416,5 @@ TEST(VerifierTest, GetElementPtrInst) {
       << Error;
 }
 
-TEST(VerifierTest, DetectTaggedGlobalInSection) {
-  LLVMContext C;
-  Module M("M", C);
-  GlobalVariable *GV = new GlobalVariable(
-      Type::getInt64Ty(C), false, GlobalValue::InternalLinkage,
-      ConstantInt::get(Type::getInt64Ty(C), 1));
-  GV->setDSOLocal(true);
-  GlobalValue::SanitizerMetadata MD{};
-  MD.Memtag = true;
-  GV->setSanitizerMetadata(MD);
-  GV->setSection("foo");
-  M.insertGlobalVariable(GV);
-
-  std::string Error;
-  raw_string_ostream ErrorOS(Error);
-  EXPECT_TRUE(verifyModule(M, &ErrorOS));
-  EXPECT_TRUE(
-      StringRef(Error).starts_with("tagged GlobalValue must not be in section"))
-      << Error;
-}
-
 } // end anonymous namespace
 } // end namespace llvm

From 6bac20b391edce2bde348e59f5be2143157304b5 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Thu, 17 Apr 2025 09:55:41 -0700
Subject: [PATCH 296/710] [MTE] do not tag zero sized globals (#136020)

---
 clang/test/CodeGen/memtag-globals-asm.cpp  |  6 ++++++
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 13 +++++++++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/clang/test/CodeGen/memtag-globals-asm.cpp b/clang/test/CodeGen/memtag-globals-asm.cpp
index 57e3cedc083fd..fb3958dd8bcb6 100644
--- a/clang/test/CodeGen/memtag-globals-asm.cpp
+++ b/clang/test/CodeGen/memtag-globals-asm.cpp
@@ -21,6 +21,7 @@
 // RUN: FileCheck %s --input-file=%t.out --check-prefix=CHECK-P
 // RUN: FileCheck %s --input-file=%t.out --check-prefix=CHECK-Q
 // RUN: FileCheck %s --input-file=%t.out --check-prefix=CHECK-R
+// RUN: FileCheck %s --input-file=%t.out --check-prefix=CHECK-S
 
 // RUN: %clang_cc1 -O3 -S -x c++ -std=c++11 -triple aarch64-linux-android31 \
 // RUN:   -fsanitize=memtag-globals -o %t.out %s
@@ -43,6 +44,7 @@
 // RUN: FileCheck %s --input-file=%t.out --check-prefix=CHECK-P
 // RUN: FileCheck %s --input-file=%t.out --check-prefix=CHECK-Q
 // RUN: FileCheck %s --input-file=%t.out --check-prefix=CHECK-R
+// RUN: FileCheck %s --input-file=%t.out --check-prefix=CHECK-S
 
 /// Ensure that emulated TLS also doesn't get sanitized.
 // RUN: %clang_cc1 -S -x c++ -std=c++11 -triple aarch64-linux-android31 \
@@ -99,6 +101,10 @@ static char* global_buffer_local_end = &global_buffer[16];
 // CHECK-H: .size global_buffer_global_end, 16
 char* global_buffer_global_end = &global_buffer[16];
 
+// CHECK-S-NOT: .memtag zero_sized
+struct empty {};
+char zero_sized[0];
+
 class MyClass {
  public:
   virtual ~MyClass() {}
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 6b1f3df12ee44..bdcd54a135da9 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2398,6 +2398,12 @@ void AsmPrinter::emitRemarksSection(remarks::RemarkStreamer &RS) {
   OutStreamer->emitBinaryData(Buf);
 }
 
+static uint64_t globalSize(const llvm::GlobalVariable &G) {
+  const Constant *Initializer = G.getInitializer();
+  return G.getParent()->getDataLayout().getTypeAllocSize(
+      Initializer->getType());
+}
+
 static bool shouldTagGlobal(const llvm::GlobalVariable &G) {
   // We used to do this in clang, but there are optimization passes that turn
   // non-constant globals into constants. So now, clang only tells us whether
@@ -2430,19 +2436,18 @@ static bool shouldTagGlobal(const llvm::GlobalVariable &G) {
   if (G.hasSection())
     return false;
 
-  return true;
+  return globalSize(G) > 0;
 }
 
 static void tagGlobalDefinition(Module &M, GlobalVariable *G) {
-  Constant *Initializer = G->getInitializer();
-  uint64_t SizeInBytes =
-      M.getDataLayout().getTypeAllocSize(Initializer->getType());
+  uint64_t SizeInBytes = globalSize(*G);
 
   uint64_t NewSize = alignTo(SizeInBytes, 16);
   if (SizeInBytes != NewSize) {
     // Pad the initializer out to the next multiple of 16 bytes.
     llvm::SmallVector<uint8_t> Init(NewSize - SizeInBytes, 0);
     Constant *Padding = ConstantDataArray::get(M.getContext(), Init);
+    Constant *Initializer = G->getInitializer();
     Initializer = ConstantStruct::getAnon({Initializer, Padding});
     auto *NewGV = new GlobalVariable(
         M, Initializer->getType(), G->isConstant(), G->getLinkage(),

From 386cc00d8d0a8c211b3d25d08fce4c7e6ae36059 Mon Sep 17 00:00:00 2001
From: Chao Chen <chao.chen@intel.com>
Date: Thu, 17 Apr 2025 12:02:08 -0500
Subject: [PATCH 297/710] [MLIR][XeGPU] Update XeGPU doc (#136155)

---
 .../mlir/Dialect/XeGPU/IR/XeGPUAttrs.td       | 95 ++++++++++---------
 .../mlir/Dialect/XeGPU/IR/XeGPUDialect.td     | 19 +++-
 2 files changed, 62 insertions(+), 52 deletions(-)

diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
index ab5fb4a4a7de9..f1bed70253ef3 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
@@ -183,53 +183,54 @@ def XeGPU_LayoutAttr : XeGPUAttr<"Layout", "layout"> {
       1-dimensional layout. The first dimension in the order list is the fastest-changing dimension. If it
       is not present, the default value is [1, 0].
 
-    ### Examples:
-      1. Subgroup level layout:
-      ```mlir
-      #xegpu.layout<lane_layout = [2, 8], lane_data = [1, 1]>
-      ```
-      In this example, there are 16 work-items per subgroup, and is organized as
-      [[0, 1, 2, .., 7],[8, 9, .., 15]]. The distribution unit is 1x1.
-
-      2. Subgroup level layout with order:
-      ```mlir
-      #xegpu.layout<lane_layout = [2, 8], lane_data = [1, 1], order = [0, 1]>
-      ```
-      In this example, there are 16 work-items per subgroup, and is organized as
-      [[0, 2, 4, ..., 14], [1, 3, 5, ..., 15]]. The distribution unit is 1x1.
-
-      3. Subgroup level layout with inst_data
-      ```mlir
-      #xegpu.layout<inst_data = [8, 16], lane_layout = [2, 8], lane_data = [2, 2]>
-      ```
-      In this example, the original problem size is partitioned into smaller subproblems of dimensions [8, 16],
-      which are then distributed among 16 work-items arranged as [[0, 1, 2, ..., 7], [8, 9, ..., 15]]. Each
-      work-item is assigned four 2x2 blocks in a round-robin manner.
-
-      4. Workgroup level layout:
-      ```mlir
-      #xegpu.layout<sg_layout = [2, 4], sg_data = [16, 16], lane_layout = [2, 8], lane_data = [1, 1]>
-      ```
-      In this example, the layout represents a workgroup distribution. A workgroup consists of 8 subgroups
-      arranged as [[0, 1, 2, 3], [4, 5, 6, 7]]. Each subgroup accesses a 16x16 block per instruction, which
-      is further distributed to 16 work items which is organized as [[0, 1, 2, .., 7],[8, 9, .., 15]].
-
-      5. Workgroup level layout with order:
-      ```mlir
-      #xegpu.layout<sg_layout = [2, 4], sg_data = [16, 16], lane_layout = [2, 8], lane_data = [1, 1], order = [0, 1]>
-      ```
-      In this example, the layout represents a workgroup distribution. A workgroup consists of 8 subgroups
-      arranged as [[0, 2, 4, 6], [1, 3, 5, 7]]. Each subgroup accesses a 16x16 block per instruction, which
-      is further distributed to 16 work items which is organized as [[0, 2, 4, ..., 14], [1, 3, 5, ..., 15]].
-
-      6. Workgroup level layout with inst_data:
-      ```mlir
-      #xegpu.layout<sg_layout = [2, 4], sg_data = [16, 16], inst_data = [8, 16], lane_layout = [2, 8], lane_data = [1, 1]>
-      ```
-      This example is similar to the previous ones, but the `inst_data` parameter divides `sg_data` into two instructions,
-      each processing an 8x16 block. These blocks are further distributed across 16 work-items with a distribution unit of 1x1.
-      Unlike the 2x2 distribution unit in example 3, which results in accessing contiguous 2x2 blocks, the 1x1 distribution
-      unit may result in non-contiguous access.
+    Examples:
+
+    1. Subgroup level layout:
+    ```mlir
+    #xegpu.layout<lane_layout = [2, 8], lane_data = [1, 1]>
+    ```
+    In this example, there are 16 work-items per subgroup, and is organized as
+    [[0, 1, 2, .., 7],[8, 9, .., 15]]. The distribution unit is 1x1.
+
+    2. Subgroup level layout with order:
+    ```mlir
+    #xegpu.layout<lane_layout = [2, 8], lane_data = [1, 1], order = [0, 1]>
+    ```
+    In this example, there are 16 work-items per subgroup, and is organized as
+    [[0, 2, 4, ..., 14], [1, 3, 5, ..., 15]]. The distribution unit is 1x1.
+
+    3. Subgroup level layout with inst_data
+    ```mlir
+    #xegpu.layout<inst_data = [8, 16], lane_layout = [2, 8], lane_data = [2, 2]>
+    ```
+    In this example, the original problem size is partitioned into smaller subproblems of dimensions [8, 16],
+    which are then distributed among 16 work-items arranged as [[0, 1, 2, ..., 7], [8, 9, ..., 15]]. Each
+    work-item is assigned four 2x2 blocks in a round-robin manner.
+
+    4. Workgroup level layout:
+    ```mlir
+    #xegpu.layout<sg_layout = [2, 4], sg_data = [16, 16], lane_layout = [2, 8], lane_data = [1, 1]>
+    ```
+    In this example, the layout represents a workgroup distribution. A workgroup consists of 8 subgroups
+    arranged as [[0, 1, 2, 3], [4, 5, 6, 7]]. Each subgroup accesses a 16x16 block per instruction, which
+    is further distributed to 16 work items which is organized as [[0, 1, 2, .., 7],[8, 9, .., 15]].
+
+    5. Workgroup level layout with order:
+    ```mlir
+    #xegpu.layout<sg_layout = [2, 4], sg_data = [16, 16], lane_layout = [2, 8], lane_data = [1, 1], order = [0, 1]>
+    ```
+    In this example, the layout represents a workgroup distribution. A workgroup consists of 8 subgroups
+    arranged as [[0, 2, 4, 6], [1, 3, 5, 7]]. Each subgroup accesses a 16x16 block per instruction, which
+    is further distributed to 16 work items which is organized as [[0, 2, 4, ..., 14], [1, 3, 5, ..., 15]].
+
+    6. Workgroup level layout with inst_data:
+    ```mlir
+    #xegpu.layout<sg_layout = [2, 4], sg_data = [16, 16], inst_data = [8, 16], lane_layout = [2, 8], lane_data = [1, 1]>
+    ```
+    This example is similar to the previous ones, but the `inst_data` parameter divides `sg_data` into two instructions,
+    each processing an 8x16 block. These blocks are further distributed across 16 work-items with a distribution unit of 1x1.
+    Unlike the 2x2 distribution unit in example 3, which results in accessing contiguous 2x2 blocks, the 1x1 distribution
+    unit may result in non-contiguous access.
   }];
 
   let parameters = (ins
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td
index 765f218f95d26..fb5a1e6f1db0c 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td
@@ -16,11 +16,20 @@ def XeGPU_Dialect : Dialect {
     let cppNamespace = "::mlir::xegpu";
     let summary = "The XeGPU dialect that models Intel GPU's ISA";
     let description = [{
-      The XeGPU dialect models Intel Xe ISA semantics but works at vector and
-      TensorDesc data type. It provides 1:1 mappings to match Xe instructions
-      like DPAS and 2D block load. The matrix size being processed at this level
-      exactly matches the hardware instructions or the intrinsic supported by
-      the lower-level GPU compiler.
+      The XeGPU dialect closely models a subset of the Xe GPU's ISA, providing an
+      abstraction to support high-performance GEMM code generation. It serves as a
+      bridge dialect in the MLIR gradual lowering process, working with MLIR memref
+      and vector types, and complements the Arith, Math, Vector, and Memref dialects.
+      XeGPU operations are introduced for special Xe instructions not modeled by the
+      LLVM/SPIR-V dialect, such as DPAS and 2D block load and store.
+
+      It supports a tile-based programming model, decomposing the GEMM kernel into
+      large predefined tile sizes at the subgroup and workgroup levels. XeGPU allows
+      the high-level GEMM algorithm to be easily expressed. Underneath, it uses
+      target-specific recipes and hardware features to achieve optimal performance
+      on specific hardware. By decomposing GEMM at submatrix granularity and mapping it
+      to registers, it naturally supports optimizations like fusing with neighboring
+      operations.
     }];
 
     let dependentDialects = ["arith::ArithDialect"];

From 7866fc2bd9f5e87ffdccecd06f5f877b81218bcd Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 17 Apr 2025 10:06:43 -0700
Subject: [PATCH 298/710] [RISCV] Rewrite vrgather.vx undef, (vmv.s.x), 0, v0
 as vmv.v.x (#136010)

This extends the DAG combine introduced in 336b2909 to handle the case
where the prior value is defined by a vmv.s.x instead of a vmv.v.x. If
the vrgather splats the single source element, and has no passthru we
can replace it with a vmv.v.x - which will in turn usually get folded
into a vmerge if a select follows.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 30 +++++++++++++++++--
 .../RISCV/rvv/fixed-vectors-shuffle-fp.ll     | 19 ++++--------
 .../RISCV/rvv/fixed-vectors-shuffle-int.ll    | 17 ++++-------
 3 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index beea13d5f8f3e..5d4df039a0ed8 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -19710,20 +19710,46 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
       return V;
     break;
   case RISCVISD::VRGATHER_VX_VL: {
-    // Drop a redundant vrgather_vx.
     // Note this assumes that out of bounds indices produce poison
     // and can thus be replaced without having to prove them inbounds..
+    EVT VT = N->getValueType(0);
     SDValue Src = N->getOperand(0);
+    SDValue Idx = N->getOperand(1);
     SDValue Passthru = N->getOperand(2);
     SDValue VL = N->getOperand(4);
+
+    // Warning: Unlike most cases we strip an insert_subvector, this one
+    // does not require the first operand to be undef.
+    if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
+        isNullConstant(Src.getOperand(2)))
+      Src = Src.getOperand(1);
+
     switch (Src.getOpcode()) {
     default:
       break;
     case RISCVISD::VMV_V_X_VL:
     case RISCVISD::VFMV_V_F_VL:
-      if (Passthru.isUndef() && VL == Src.getOperand(2))
+      // Drop a redundant vrgather_vx.
+      // TODO: Remove the type restriction if we find a motivating
+      // test case?
+      if (Passthru.isUndef() && VL == Src.getOperand(2) &&
+          Src.getValueType() == VT)
         return Src;
       break;
+    case RISCVISD::VMV_S_X_VL:
+    case RISCVISD::VFMV_S_F_VL:
+      // If this use only demands lane zero from the source vmv.s.x, and
+      // doesn't have a passthru, then this vrgather.vi/vx is equivalent to
+      // a vmv.v.x.  Note that there can be other uses of the original
+      // vmv.s.x and thus we can't eliminate it.  (vfmv.s.f is analogous)
+      if (isNullConstant(Idx) && Passthru.isUndef() &&
+          VL == Src.getOperand(2)) {
+        unsigned Opc =
+            VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
+        return DAG.getNode(Opc, DL, VT, DAG.getUNDEF(VT), Src.getOperand(1),
+                           VL);
+      }
+      break;
     }
     break;
   }
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-fp.ll
index 5aac2687122ae..f580b1b993395 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-fp.ll
@@ -96,13 +96,11 @@ define <8 x float> @vmerge_vxm(<8 x float> %v, float %s) {
 ; CHECK-LABEL: vmerge_vxm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 25
-; CHECK-NEXT:    vsetivli zero, 8, e32, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vsetivli zero, 1, e32, m4, tu, ma
 ; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vmv2r.v v10, v8
-; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; CHECK-NEXT:    vrgather.vi v10, v8, 0, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vfmerge.vfm v8, v8, fa0, v0
 ; CHECK-NEXT:    ret
   %ins = insertelement <8 x float> %v, float %s, i32 0
   %shuf = shufflevector <8 x float> %ins, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 0, i32 0, i32 5, i32 6, i32 7>
@@ -112,15 +110,10 @@ define <8 x float> @vmerge_vxm(<8 x float> %v, float %s) {
 define <8 x float> @vmerge_vxm2(<8 x float> %v, float %s) {
 ; CHECK-LABEL: vmerge_vxm2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e32, m4, tu, ma
-; CHECK-NEXT:    vmv1r.v v12, v8
-; CHECK-NEXT:    vmv2r.v v10, v8
 ; CHECK-NEXT:    li a0, 25
-; CHECK-NEXT:    vfmv.s.f v12, fa0
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vmv1r.v v10, v12
-; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
-; CHECK-NEXT:    vrgather.vi v8, v10, 0, v0.t
+; CHECK-NEXT:    vfmerge.vfm v8, v8, fa0, v0
 ; CHECK-NEXT:    ret
   %ins = insertelement <8 x float> %v, float %s, i32 0
   %shuf = shufflevector <8 x float> %v, <8 x float> %ins, <8 x i32> <i32 8, i32 1, i32 2, i32 8, i32 8, i32 5, i32 6, i32 7>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
index 5c4ef29d7d5b7..8676803e20e3b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
@@ -1448,13 +1448,11 @@ define <8 x i8> @vmerge_vxm(<8 x i8> %v, i8 %s) {
 ; CHECK-LABEL: vmerge_vxm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a1, 25
-; CHECK-NEXT:    vsetivli zero, 8, e8, m1, tu, ma
-; CHECK-NEXT:    vmv.s.x v8, a0
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vi v9, v8, 0, v0.t
-; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    vmv.s.x v8, a0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vmerge.vxm v8, v8, a0, v0
 ; CHECK-NEXT:    ret
   %ins = insertelement <8 x i8> %v, i8 %s, i32 0
   %shuf = shufflevector <8 x i8> %ins, <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 0, i32 0, i32 5, i32 6, i32 7>
@@ -1465,12 +1463,9 @@ define <8 x i8> @vmerge_vxm2(<8 x i8> %v, i8 %s) {
 ; CHECK-LABEL: vmerge_vxm2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a1, 25
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, tu, ma
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a1
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vmv.s.x v9, a0
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vi v8, v9, 0, v0.t
+; CHECK-NEXT:    vmerge.vxm v8, v8, a0, v0
 ; CHECK-NEXT:    ret
   %ins = insertelement <8 x i8> %v, i8 %s, i32 0
   %shuf = shufflevector <8 x i8> %v, <8 x i8> %ins, <8 x i32> <i32 8, i32 1, i32 2, i32 8, i32 8, i32 5, i32 6, i32 7>

From 3133c956c57c5b952d3289323891dcdc252ea333 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 17 Apr 2025 10:19:37 -0700
Subject: [PATCH 299/710] [clang-sycl-linker] Fix a warning

This patch fixes:

  clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp:368:14: error:
  moving a local object in a return statement prevents copy elision
  [-Werror,-Wpessimizing-move]
---
 clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp b/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp
index 79747b0550417..61ec4dbc1489d 100644
--- a/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp
+++ b/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp
@@ -365,7 +365,7 @@ Error runSYCLLink(ArrayRef<std::string> Files, const ArgList &Args) {
     SPVFile.append("_" + utostr(I) + ".spv");
     auto Err = runSPIRVCodeGen(SplitModules[I], Args, SPVFile, C);
     if (Err)
-      return std::move(Err);
+      return Err;
     SplitModules[I] = SPVFile;
   }
 

From f9729859446180c1f0afc386513f7476f5f220e0 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 17 Apr 2025 18:40:48 +0100
Subject: [PATCH 300/710] [X86] combineGatherScatter - use
 FoldConstantArithmetic for truncation of constant build vectors (#136033)

No need to explicitly check with BuildVectorSDNode::isConstant - FoldConstantArithmetic can handle this, including through bitcasts etc.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3adeb4628eabf..024b653f78cb5 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56529,16 +56529,13 @@ static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
     if (IndexWidth > 32 && DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
       EVT NewVT = IndexVT.changeVectorElementType(MVT::i32);
 
-      // FIXME: We could support more than just constant vectors, but we need to
+      // FIXME: We could support more than just constant fold, but we need to
       // careful with costing. A truncate that can be optimized out would be
       // fine. Otherwise we might only want to create a truncate if it avoids a
       // split.
-      if (auto *BV = dyn_cast<BuildVectorSDNode>(Index)) {
-        if (BV->isConstant()) {
-          Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
-          return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
-        }
-      }
+      if (SDValue TruncIndex =
+              DAG.FoldConstantArithmetic(ISD::TRUNCATE, DL, NewVT, Index))
+        return rebuildGatherScatter(GorS, TruncIndex, Base, Scale, DAG);
 
       // Shrink any sign/zero extends from 32 or smaller to larger than 32 if
       // there are sufficient sign bits. Only do this before legalize types to

From d56afced16f4274c1b142a77a3282760d051cea0 Mon Sep 17 00:00:00 2001
From: Nicolas van Kempen <nvankemp@gmail.com>
Date: Thu, 17 Apr 2025 13:54:31 -0400
Subject: [PATCH 301/710] [NFC][clang-tidy] Remove {{^}} clauses in some tests
 (2/N) (#135824)

`check_clang_tidy` now matches full lines only, so `{{^}}` clauses are
no longer necessary.

I am splitting those changes over multiple PRs to make review easier.
Numbering them but the actual order doesn't matter.
---
 .../misplaced-operator-in-strlen-in-alloc.c   | 20 ++++++-------
 .../checkers/google/explicit-constructor.cpp  | 30 +++++++++----------
 .../checkers/misc/definitions-in-headers.hpp  |  2 +-
 .../clang-tidy/checkers/misc/static-assert.c  |  6 ++--
 .../checkers/misc/unused-alias-decls.cpp      |  2 +-
 .../checkers/misc/unused-using-decls.cpp      | 10 +++----
 .../raw-string-literal-delimiter.cpp          |  4 +--
 .../raw-string-literal-replace-shorter.cpp    |  4 +--
 .../checkers/modernize/shrink-to-fit.cpp      | 24 +++++++--------
 .../modernize/use-transparent-functors.cpp    | 10 +++----
 .../checkers/modernize/use-using.cpp          |  8 ++---
 11 files changed, 60 insertions(+), 60 deletions(-)

diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/misplaced-operator-in-strlen-in-alloc.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/misplaced-operator-in-strlen-in-alloc.c
index c5deb25e9ac06..f8fd2ddbfdcd3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/misplaced-operator-in-strlen-in-alloc.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/misplaced-operator-in-strlen-in-alloc.c
@@ -19,43 +19,43 @@ size_t wcsnlen_s(const wchar_t *, size_t);
 void bad_malloc(char *name) {
   char *new_name = (char *)malloc(strlen(name + 1));
   // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: addition operator is applied to the argument of strlen
-  // CHECK-FIXES: {{^  char \*new_name = \(char \*\)malloc\(}}strlen(name) + 1{{\);$}}
+  // CHECK-FIXES: char *new_name = (char *)malloc(strlen(name) + 1);
   new_name = (char *)malloc(strnlen(name + 1, 10));
   // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: addition operator is applied to the argument of strnlen
-  // CHECK-FIXES: {{^  new_name = \(char \*\)malloc\(}}strnlen(name, 10) + 1{{\);$}}
+  // CHECK-FIXES: new_name = (char *)malloc(strnlen(name, 10) + 1);
   new_name = (char *)malloc(strnlen_s(name + 1, 10));
   // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: addition operator is applied to the argument of strnlen_s
-  // CHECK-FIXES: {{^  new_name = \(char \*\)malloc\(}}strnlen_s(name, 10) + 1{{\);$}}
+  // CHECK-FIXES: new_name = (char *)malloc(strnlen_s(name, 10) + 1);
 }
 
 void bad_malloc_wide(wchar_t *name) {
   wchar_t *new_name = (wchar_t *)malloc(wcslen(name + 1));
   // CHECK-MESSAGES: :[[@LINE-1]]:34: warning: addition operator is applied to the argument of wcslen
-  // CHECK-FIXES: {{^  wchar_t \*new_name = \(wchar_t \*\)malloc\(}}wcslen(name) + 1{{\);$}}
+  // CHECK-FIXES: wchar_t *new_name = (wchar_t *)malloc(wcslen(name) + 1);
   new_name = (wchar_t *)malloc(wcsnlen(name + 1, 10));
   // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: addition operator is applied to the argument of wcsnlen
-  // CHECK-FIXES: {{^  new_name = \(wchar_t \*\)malloc\(}}wcsnlen(name, 10) + 1{{\);$}}
+  // CHECK-FIXES: new_name = (wchar_t *)malloc(wcsnlen(name, 10) + 1);
   new_name = (wchar_t *)malloc(wcsnlen_s(name + 1, 10));
   // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: addition operator is applied to the argument of wcsnlen_s
-  // CHECK-FIXES: {{^  new_name = \(wchar_t \*\)malloc\(}}wcsnlen_s(name, 10) + 1{{\);$}}
+  // CHECK-FIXES: new_name = (wchar_t *)malloc(wcsnlen_s(name, 10) + 1);
 }
 
 void bad_alloca(char *name) {
   char *new_name = (char *)alloca(strlen(name + 1));
   // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: addition operator is applied to the argument of strlen
-  // CHECK-FIXES: {{^  char \*new_name = \(char \*\)alloca\(}}strlen(name) + 1{{\);$}}
+  // CHECK-FIXES: char *new_name = (char *)alloca(strlen(name) + 1);
 }
 
 void bad_calloc(char *name) {
   char *new_names = (char *)calloc(2, strlen(name + 1));
   // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: addition operator is applied to the argument of strlen
-  // CHECK-FIXES: {{^  char \*new_names = \(char \*\)calloc\(2, }}strlen(name) + 1{{\);$}}
+  // CHECK-FIXES: char *new_names = (char *)calloc(2, strlen(name) + 1);
 }
 
 void bad_realloc(char *old_name, char *name) {
   char *new_name = (char *)realloc(old_name, strlen(name + 1));
   // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: addition operator is applied to the argument of strlen
-  // CHECK-FIXES: {{^  char \*new_name = \(char \*\)realloc\(old_name, }}strlen(name) + 1{{\);$}}
+  // CHECK-FIXES: char *new_name = (char *)realloc(old_name, strlen(name) + 1);
 }
 
 void intentional1(char *name) {
@@ -81,5 +81,5 @@ void (*(*const alloc_ptr)(size_t)) = malloc;
 void bad_indirect_alloc(char *name) {
   char *new_name = (char *)alloc_ptr(strlen(name + 1));
   // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: addition operator is applied to the argument of strlen
-  // CHECK-FIXES: {{^  char \*new_name = \(char \*\)alloc_ptr\(}}strlen(name) + 1{{\);$}}
+  // CHECK-FIXES: char *new_name = (char *)alloc_ptr(strlen(name) + 1);
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/google/explicit-constructor.cpp b/clang-tools-extra/test/clang-tidy/checkers/google/explicit-constructor.cpp
index 921237a436c6d..e8174b1aebb20 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/google/explicit-constructor.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/google/explicit-constructor.cpp
@@ -44,20 +44,20 @@ struct A {
 
   explicit A(const A& a) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: copy constructor should not be declared explicit [google-explicit-constructor]
-  // CHECK-FIXES: {{^  }}A(const A& a) {}
+  // CHECK-FIXES: A(const A& a) {}
 
   A(int x1);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: single-argument constructors must be marked explicit to avoid unintentional implicit conversions [google-explicit-constructor]
-  // CHECK-FIXES: {{^  }}explicit A(int x1);
+  // CHECK-FIXES: explicit A(int x1);
 
   A(double x2, double y = 3.14) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: constructors that are callable with a single argument must be marked explicit to avoid unintentional implicit conversions [google-explicit-constructor]
-  // CHECK-FIXES: {{^  }}explicit A(double x2, double y = 3.14) {}
+  // CHECK-FIXES: explicit A(double x2, double y = 3.14) {}
 
   template <typename... T>
   A(T&&... args);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: constructors that are callable with a single argument
-  // CHECK-FIXES: {{^  }}explicit A(T&&... args);
+  // CHECK-FIXES: explicit A(T&&... args);
 };
 
 inline A::A(int x1) {}
@@ -69,23 +69,23 @@ struct B {
 
   operator bool() const { return true; }
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: 'operator bool' must be marked explicit to avoid unintentional implicit conversions [google-explicit-constructor]
-  // CHECK-FIXES: {{^  }}explicit operator bool() const { return true; }
+  // CHECK-FIXES: explicit operator bool() const { return true; }
 
   operator double() const;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: 'operator double' must be marked explicit to avoid unintentional implicit conversions [google-explicit-constructor]
-  // CHECK-FIXES: {{^  }}explicit operator double() const;
+  // CHECK-FIXES: explicit operator double() const;
 
   explicit B(::std::initializer_list<double> list4) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: initializer-list constructor should not be declared explicit [google-explicit-constructor]
-  // CHECK-FIXES: {{^  }}B(::std::initializer_list<double> list4) {}
+  // CHECK-FIXES: B(::std::initializer_list<double> list4) {}
 
   explicit B(const ::std::initializer_list<char> &list5) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: initializer-list constructor
-  // CHECK-FIXES: {{^  }}B(const ::std::initializer_list<char> &list5) {}
+  // CHECK-FIXES: B(const ::std::initializer_list<char> &list5) {}
 
   explicit B(::std::initializer_list<char> &&list6) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: initializer-list constructor
-  // CHECK-FIXES: {{^  }}B(::std::initializer_list<char> &&list6) {}
+  // CHECK-FIXES: B(::std::initializer_list<char> &&list6) {}
 };
 
 inline B::operator double() const { return 0.0; }
@@ -110,7 +110,7 @@ struct C2 {
 
   explicit C2(initializer_list<double> list4) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: initializer-list constructor
-  // CHECK-FIXES: {{^  }}C2(initializer_list<double> list4) {}
+  // CHECK-FIXES: C2(initializer_list<double> list4) {}
 };
 
 template <typename T>
@@ -132,11 +132,11 @@ template <typename T>
 struct E {
   E(T *pt) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: single-argument constructors
-  // CHECK-FIXES: {{^  }}explicit E(T *pt) {}
+  // CHECK-FIXES: explicit E(T *pt) {}
   template <typename U>
   E(U *pu) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: single-argument constructors
-  // CHECK-FIXES: {{^  }}explicit E(U *pu) {}
+  // CHECK-FIXES: explicit E(U *pu) {}
 
   explicit E(T t) {}
   template <typename U>
@@ -156,14 +156,14 @@ template<typename T>
 struct G {
   operator bool() const;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: 'operator bool' must be marked
-  // CHECK-FIXES: {{^}}  explicit operator bool() const;
+  // CHECK-FIXES: explicit operator bool() const;
   operator F<T>() const;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: 'operator F<type-parameter-0-0>' must be marked
-  // CHECK-FIXES: {{^}}  explicit operator F<T>() const;
+  // CHECK-FIXES: explicit operator F<T>() const;
   template<typename U>
   operator F<U>*() const;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: 'operator F<type-parameter-1-0> *' must be marked
-  // CHECK-FIXES: {{^}}  explicit operator F<U>*() const;
+  // CHECK-FIXES: explicit operator F<U>*() const;
 };
 
 void f2() {
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/definitions-in-headers.hpp b/clang-tools-extra/test/clang-tidy/checkers/misc/definitions-in-headers.hpp
index eabcd17817020..c48a2ee9276be 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/definitions-in-headers.hpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/definitions-in-headers.hpp
@@ -203,4 +203,4 @@ constexpr bool f13<void, int> = false;
 
 int main() {}
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: function 'main' defined in a header file;
-// CHECK-FIXES: {{^}}int main() {
+// CHECK-FIXES: int main() {}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.c b/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.c
index a3edf8398e383..77262c219df03 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.c
@@ -13,15 +13,15 @@ void abort(void) {}
 void f(void) {
   int x = 1;
   assert(x == 0);
-  // CHECK-FIXES: {{^  }}assert(x == 0);
+  // CHECK-FIXES: assert(x == 0);
 
   #define static_assert(x, msg) _Static_assert(x, msg)
   assert(11 == 5 + 6);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES: {{^  }}static_assert(11 == 5 + 6, "");
+  // CHECK-FIXES: static_assert(11 == 5 + 6, "");
   #undef static_assert
 
   assert(10 == 5 + 5);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES: {{^  }}static_assert(10 == 5 + 5, "");
+  // CHECK-FIXES: static_assert(10 == 5 + 5, "");
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-alias-decls.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-alias-decls.cpp
index 6d63df4047a4b..efe1c01f42921 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-alias-decls.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-alias-decls.cpp
@@ -6,7 +6,7 @@ class C {};
 
 namespace unused_alias = ::my_namespace; // eol-comments aren't removed (yet)
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: namespace alias decl 'unused_alias' is unused
-// CHECK-FIXES: {{^}}// eol-comments aren't removed (yet)
+// CHECK-FIXES: // eol-comments aren't removed (yet)
 
 namespace used_alias = ::my_namespace;
 void f() { used_alias::C c; }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.cpp
index 62aa17b0b1c22..f6fd321a613dc 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.cpp
@@ -82,13 +82,13 @@ T ff() { T t; return t; }
 using n::A; // A
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: using decl 'A' is unused
 // CHECK-MESSAGES: :[[@LINE-2]]:10: note: remove the using
-// CHECK-FIXES: {{^}}// A
+// CHECK-FIXES: // A
 using n::B;
 using n::C;
 using n::D;
 using n::E; // E
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: using decl 'E' is unused
-// CHECK-FIXES: {{^}}// E
+// CHECK-FIXES: // E
 using n::F;
 using n::G;
 using n::H;
@@ -103,10 +103,10 @@ using n::UsedFunc;
 using n::UsedTemplateFunc;
 using n::UnusedInstance; // UnusedInstance
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: using decl 'UnusedInstance' is unused
-// CHECK-FIXES: {{^}}// UnusedInstance
+// CHECK-FIXES: // UnusedInstance
 using n::UnusedFunc; // UnusedFunc
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: using decl 'UnusedFunc' is unused
-// CHECK-FIXES: {{^}}// UnusedFunc
+// CHECK-FIXES: // UnusedFunc
 using n::operator""_w;
 using n::cout;
 using n::endl;
@@ -120,7 +120,7 @@ template <typename T> void Callee() {
 
 using n::OverloadFunc; // OverloadFunc
 // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: using decl 'OverloadFunc' is unused
-// CHECK-FIXES: {{^}}// OverloadFunc
+// CHECK-FIXES: // OverloadFunc
 
 #define DEFINE_INT(name)        \
   namespace INT {               \
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/raw-string-literal-delimiter.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/raw-string-literal-delimiter.cpp
index 557c99a038182..a47fb849ad9c3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/raw-string-literal-delimiter.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/raw-string-literal-delimiter.cpp
@@ -2,8 +2,8 @@
 
 char const *const ContainsSentinel{"who\\ops)\""};
 // CHECK-MESSAGES: :[[@LINE-1]]:36: warning: {{.*}} can be written as a raw string literal
-// CHECK-FIXES: {{^}}char const *const ContainsSentinel{R"str(who\ops)")str"};{{$}}
+// CHECK-FIXES: char const *const ContainsSentinel{R"str(who\ops)")str"};
 
 //char const *const ContainsDelim{"whoops)\")lit\""};
 // CHECK-XMESSAGES: :[[@LINE-1]]:33: warning: {{.*}} can be written as a raw string literal
-// CHECK-XFIXES: {{^}}char const *const ContainsDelim{R"lit1(whoops)")lit")lit1"};{{$}}
+// CHECK-XFIXES: char const *const ContainsDelim{R"lit1(whoops)")lit")lit1"};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/raw-string-literal-replace-shorter.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/raw-string-literal-replace-shorter.cpp
index 673a8aa5af8a4..a3bc5e21146b1 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/raw-string-literal-replace-shorter.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/raw-string-literal-replace-shorter.cpp
@@ -6,8 +6,8 @@ char const *const NeedDelimiter("\":)\"");
 
 char const *const ManyQuotes("quotes:\'\'\'\'");
 // CHECK-MESSAGES: :[[@LINE-1]]:30: warning: {{.*}} can be written as a raw string literal
-// CHECK-FIXES: {{^}}char const *const ManyQuotes(R"(quotes:'''')");{{$}}
+// CHECK-FIXES: char const *const ManyQuotes(R"(quotes:'''')");
 
 char const *const LongOctal("\042\072\051\042");
 // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: {{.*}} can be written as a raw string literal
-// CHECK-FIXES: {{^}}char const *const LongOctal(R"lit(":)")lit");{{$}}
+// CHECK-FIXES: char const *const LongOctal(R"lit(":)")lit");
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/shrink-to-fit.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/shrink-to-fit.cpp
index 6993d300c25a8..a3754e6822ac8 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/shrink-to-fit.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/shrink-to-fit.cpp
@@ -9,17 +9,17 @@ void f() {
 
   std::vector<int>(v).swap(v);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: the shrink_to_fit method should be used to reduce the capacity of a shrinkable container [modernize-shrink-to-fit]
-  // CHECK-FIXES: {{^  }}v.shrink_to_fit();{{$}}
+  // CHECK-FIXES: v.shrink_to_fit();
 
   std::vector<int> &vref = v;
   std::vector<int>(vref).swap(vref);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: the shrink_to_fit method should
-  // CHECK-FIXES: {{^  }}vref.shrink_to_fit();{{$}}
+  // CHECK-FIXES: vref.shrink_to_fit();
 
   std::vector<int> *vptr = &v;
   std::vector<int>(*vptr).swap(*vptr);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: the shrink_to_fit method should
-  // CHECK-FIXES: {{^  }}vptr->shrink_to_fit();{{$}}
+  // CHECK-FIXES: vptr->shrink_to_fit();
 }
 
 struct X {
@@ -27,12 +27,12 @@ struct X {
   void f() {
     std::vector<int>(v).swap(v);
     // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: the shrink_to_fit method should
-    // CHECK-FIXES: {{^    }}v.shrink_to_fit();{{$}}
+    // CHECK-FIXES: v.shrink_to_fit();
 
     std::vector<int> *vptr = &v;
     std::vector<int>(*vptr).swap(*vptr);
     // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: the shrink_to_fit method should
-    // CHECK-FIXES: {{^    }}vptr->shrink_to_fit();{{$}}
+    // CHECK-FIXES: vptr->shrink_to_fit();
   }
 };
 
@@ -40,22 +40,22 @@ template <typename T> void g() {
   std::vector<int> v;
   std::vector<int>(v).swap(v);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: the shrink_to_fit method should
-  // CHECK-FIXES: {{^  }}v.shrink_to_fit();{{$}}
+  // CHECK-FIXES: v.shrink_to_fit();
 
   std::vector<T> v2;
   std::vector<T>(v2).swap(v2);
-  // CHECK-FIXES: {{^  }}std::vector<T>(v2).swap(v2);{{$}}
+  // CHECK-FIXES: std::vector<T>(v2).swap(v2);
 }
 
 template <typename T> void g2() {
   std::vector<int> v;
   std::vector<int>(v).swap(v);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: the shrink_to_fit method should
-  // CHECK-FIXES: {{^  }}v.shrink_to_fit();{{$}}
+  // CHECK-FIXES: v.shrink_to_fit();
 
   T v3;
   T(v3).swap(v3);
-  // CHECK-FIXES: {{^  }}T(v3).swap(v3);{{$}}
+  // CHECK-FIXES: T(v3).swap(v3);
 }
 
 #define COPY_AND_SWAP_INT_VEC(x) std::vector<int>(x).swap(x)
@@ -69,7 +69,7 @@ void h() {
   std::vector<int> v;
   COPY_AND_SWAP_INT_VEC(v);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: the shrink_to_fit method should
-  // CHECK-FIXES: {{^  }}COPY_AND_SWAP_INT_VEC(v);{{$}}
+  // CHECK-FIXES: COPY_AND_SWAP_INT_VEC(v);
 }
 
 void PR38315() {
@@ -77,11 +77,11 @@ void PR38315() {
   Vector v;
   Vector(v).swap(v);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: the shrink_to_fit method should
-  // CHECK-FIXES: {{^  }}v.shrink_to_fit();{{$}}
+  // CHECK-FIXES: v.shrink_to_fit();
 
   using Vector2 = std::vector<int>;
   Vector2 v2;
   Vector2(v2).swap(v2);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: the shrink_to_fit method should
-  // CHECK-FIXES: {{^  }}v2.shrink_to_fit();{{$}}
+  // CHECK-FIXES: v2.shrink_to_fit();
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-transparent-functors.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-transparent-functors.cpp
index dde427a79201d..d18f5f5a4b780 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-transparent-functors.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-transparent-functors.cpp
@@ -75,18 +75,18 @@ int main() {
   using std::less;
   std::set<int, std::less<int>> s;
   // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: prefer transparent functors 'less<>' [modernize-use-transparent-functors]
-  // CHECK-FIXES: {{^}}  std::set<int, std::less<>> s;{{$}}
+  // CHECK-FIXES: std::set<int, std::less<>> s;
   set<int, std::less<int>> s2;
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: prefer transparent functors
-  // CHECK-FIXES: {{^}}  set<int, std::less<>> s2;{{$}}
+  // CHECK-FIXES: set<int, std::less<>> s2;
   set<int, less<int>> s3;
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: prefer transparent functors
-  // CHECK-FIXES: {{^}}  set<int, less<>> s3;{{$}}
+  // CHECK-FIXES: set<int, less<>> s3;
   std::set<int, std::less<>> s4;
   std::set<char *, std::less<std::string>> s5;
   std::set<set<int, less<int>>, std::less<>> s6;
   // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: prefer transparent functors
-  // CHECK-FIXES: {{^}}  std::set<set<int, less<>>, std::less<>> s6;{{$}}
+  // CHECK-FIXES: std::set<set<int, less<>>, std::less<>> s6;
   std::iterator begin, end;
   sort(begin, end, std::less<int>());
   // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: prefer transparent functors
@@ -96,7 +96,7 @@ int main() {
   std::find_if(begin, end, std::logical_not<>());
   using my_set = std::set<int, std::less<int>>;
   // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: prefer transparent functors
-  // CHECK-FIXES: {{^}}  using my_set = std::set<int, std::less<>>;{{$}}
+  // CHECK-FIXES: using my_set = std::set<int, std::less<>>;
   using my_set2 = std::set<char*, std::less<std::string>>;
   using my_less = std::less<std::string>;
   find_if(begin, end, my_less());
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-using.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-using.cpp
index 2dcac05f0b46c..8288f39126a11 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-using.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-using.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy --match-partial-fixes %s modernize-use-using %t -- -- -fno-delayed-template-parsing -I %S/Inputs/use-using/
+// RUN: %check_clang_tidy %s modernize-use-using %t -- -- -fno-delayed-template-parsing -I %S/Inputs/use-using/
 
 typedef int Type;
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use 'using' instead of 'typedef' [modernize-use-using]
@@ -265,7 +265,7 @@ class Variadic {};
 
 typedef Variadic<Variadic<int, bool, Q<T{0 < 0}.b> >, S<(0 < 0), Variadic<Q<b[0 < 0]> > > > Variadic_t;
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use 'using' instead of 'typedef'
-// CHECK-FIXES: using Variadic_t = Variadic<Variadic<int, bool, Q<T{0 < 0}.b> >, S<(0 < 0), Variadic<Q<b[0 < 0]> > > >
+// CHECK-FIXES: using Variadic_t = Variadic<Variadic<int, bool, Q<T{0 < 0}.b> >, S<(0 < 0), Variadic<Q<b[0 < 0]> > > >;
 
 typedef Variadic<Variadic<int, bool, Q<T{0 < 0}.b> >, S<(0 < 0), Variadic<Q<b[0 < 0]> > > > Variadic_t, *Variadic_p;
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use 'using' instead of 'typedef'
@@ -319,8 +319,8 @@ typedef void (*ISSUE_65055_1)(int);
 typedef bool (*ISSUE_65055_2)(int);
 // CHECK-MESSAGES: :[[@LINE-2]]:1: warning: use 'using' instead of 'typedef'
 // CHECK-MESSAGES: :[[@LINE-2]]:1: warning: use 'using' instead of 'typedef'
-// CHECK-FIXES: {{^}}using ISSUE_65055_1 = void (*)(int);{{$}}
-// CHECK-FIXES: {{^}}using ISSUE_65055_2 = bool (*)(int);{{$}}
+// CHECK-FIXES: using ISSUE_65055_1 = void (*)(int);
+// CHECK-FIXES: using ISSUE_65055_2 = bool (*)(int);
 
 typedef class ISSUE_67529_1 *ISSUE_67529;
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use 'using' instead of 'typedef'

From 09680dc18d460d439a89a21c69c64508457602ec Mon Sep 17 00:00:00 2001
From: Nicolas van Kempen <nvankemp@gmail.com>
Date: Thu, 17 Apr 2025 13:55:34 -0400
Subject: [PATCH 302/710] [NFC][clang-tidy] Remove {{^}} clauses in some tests
 (3/N) (#135826)

`check_clang_tidy` now matches full lines only, so `{{^}}` clauses are
no longer necessary.

I am splitting those changes over multiple PRs to make review easier.
Numbering them but the actual order doesn't matter.
---
 .../abseil/string-find-startswith.cpp         | 32 ++++-----
 .../checkers/bugprone/reserved-identifier-c.c |  2 +-
 .../checkers/bugprone/reserved-identifier.cpp | 72 +++++++++----------
 .../checkers/google/readability-casting.cpp   | 24 +++----
 .../checkers/llvm/qualified-auto.cpp          |  4 +-
 .../checkers/misc/static-assert.cpp           | 68 +++++++++---------
 .../checkers/modernize/make-unique-macros.cpp | 10 +--
 .../modernize/unary-static-assert.cpp         |  6 +-
 .../use-bool-literals-ignore-macros.cpp       | 54 +++++++-------
 .../checkers/modernize/use-override-cxx98.cpp |  6 +-
 10 files changed, 139 insertions(+), 139 deletions(-)

diff --git a/clang-tools-extra/test/clang-tidy/checkers/abseil/string-find-startswith.cpp b/clang-tools-extra/test/clang-tidy/checkers/abseil/string-find-startswith.cpp
index aabb30fe34f78..c2ce0e347e074 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/abseil/string-find-startswith.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/abseil/string-find-startswith.cpp
@@ -29,67 +29,67 @@ std::string bar();
 void tests(std::string s, global_string s2, std::string_view sv) {
   s.find("a") == 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use absl::StartsWith instead of find() == 0 [abseil-string-find-startswith]
-  // CHECK-FIXES: {{^[[:space:]]*}}absl::StartsWith(s, "a");{{$}}
+  // CHECK-FIXES: absl::StartsWith(s, "a");
 
   s.find(s) == 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}absl::StartsWith(s, s);{{$}}
+  // CHECK-FIXES: absl::StartsWith(s, s);
 
   s.find("aaa") != 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StartsWith(s, "aaa");{{$}}
+  // CHECK-FIXES: !absl::StartsWith(s, "aaa");
 
   s.find(foo(foo(bar()))) != 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StartsWith(s, foo(foo(bar())));{{$}}
+  // CHECK-FIXES: !absl::StartsWith(s, foo(foo(bar())));
 
   if (s.find("....") == 0) { /* do something */ }
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}if (absl::StartsWith(s, "....")) { /* do something */ }{{$}}
+  // CHECK-FIXES: if (absl::StartsWith(s, "....")) { /* do something */ }
 
   0 != s.find("a");
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StartsWith(s, "a");{{$}}
+  // CHECK-FIXES: !absl::StartsWith(s, "a");
 
   s2.find("a") == 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}absl::StartsWith(s2, "a");{{$}}
+  // CHECK-FIXES: absl::StartsWith(s2, "a");
 
   s.rfind("a", 0) == 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use absl::StartsWith instead of rfind() == 0 [abseil-string-find-startswith]
-  // CHECK-FIXES: {{^[[:space:]]*}}absl::StartsWith(s, "a");{{$}}
+  // CHECK-FIXES: absl::StartsWith(s, "a");
 
   s.rfind(s, 0) == 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}absl::StartsWith(s, s);{{$}}
+  // CHECK-FIXES: absl::StartsWith(s, s);
 
   s.rfind("aaa", 0) != 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StartsWith(s, "aaa");{{$}}
+  // CHECK-FIXES: !absl::StartsWith(s, "aaa");
 
   s.rfind(foo(foo(bar())), 0) != 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StartsWith(s, foo(foo(bar())));{{$}}
+  // CHECK-FIXES: !absl::StartsWith(s, foo(foo(bar())));
 
   if (s.rfind("....", 0) == 0) { /* do something */ }
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}if (absl::StartsWith(s, "....")) { /* do something */ }{{$}}
+  // CHECK-FIXES: if (absl::StartsWith(s, "....")) { /* do something */ }
 
   0 != s.rfind("a", 0);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StartsWith(s, "a");{{$}}
+  // CHECK-FIXES: !absl::StartsWith(s, "a");
 
   s2.rfind("a", 0) == 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}absl::StartsWith(s2, "a");{{$}}
+  // CHECK-FIXES: absl::StartsWith(s2, "a");
 
   sv.find("a") == 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}absl::StartsWith(sv, "a");{{$}}
+  // CHECK-FIXES: absl::StartsWith(sv, "a");
 
   sv.rfind("a", 0) != 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StartsWith
-  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StartsWith(sv, "a");{{$}}
+  // CHECK-FIXES: !absl::StartsWith(sv, "a");
 
   // expressions that don't trigger the check are here.
   A_MACRO(s.find("a"), 0);
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/reserved-identifier-c.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/reserved-identifier-c.c
index 9d075433ab9eb..30a4423f3d14d 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/reserved-identifier-c.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/reserved-identifier-c.c
@@ -7,4 +7,4 @@ void f__o__o(void);
 void f_________oo(void);
 void __foo(void);
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: declaration uses identifier '__foo', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}void foo(void);{{$}}
+// CHECK-FIXES: void foo(void);
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/reserved-identifier.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/reserved-identifier.cpp
index 1951167b41093..0f36efe656bf9 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/reserved-identifier.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/reserved-identifier.cpp
@@ -8,35 +8,35 @@
 
 #define _MACRO(m) int m = 0
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: declaration uses identifier '_MACRO', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}#define MACRO(m) int m = 0{{$}}
+// CHECK-FIXES: #define MACRO(m) int m = 0
 
 namespace _Ns {
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: declaration uses identifier '_Ns', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}namespace Ns {{{$}}
+// CHECK-FIXES: namespace Ns {
 
 class _Object {
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '_Object', which is a reserved identifier [bugprone-reserved-identifier]
-  // CHECK-FIXES: {{^}}class Object {{{$}}
+  // CHECK-FIXES: class Object {
   int _Member;
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '_Member', which is a reserved identifier [bugprone-reserved-identifier]
-  // CHECK-FIXES: {{^}}  int Member;{{$}}
+  // CHECK-FIXES: int Member;
 };
 
 float _Global;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '_Global', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}float Global;{{$}}
+// CHECK-FIXES: float Global;
 
 void _Function() {}
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: declaration uses identifier '_Function', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}void Function() {}{{$}}
+// CHECK-FIXES: void Function() {}
 
 using _Alias = int;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '_Alias', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}using Alias = int;{{$}}
+// CHECK-FIXES: using Alias = int;
 
 template <typename _TemplateParam>
 // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: declaration uses identifier '_TemplateParam', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}template <typename TemplateParam>{{$}}
+// CHECK-FIXES: template <typename TemplateParam>
 struct S {};
 
 } // namespace _Ns
@@ -45,34 +45,34 @@ struct S {};
 
 #define __macro(m) int m = 0
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: declaration uses identifier '__macro', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}#define _macro(m) int m = 0{{$}}
+// CHECK-FIXES: #define _macro(m) int m = 0
 
 namespace __ns {
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: declaration uses identifier '__ns', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}namespace ns {{{$}}
+// CHECK-FIXES: namespace ns {
 class __object {
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '__object', which is a reserved identifier [bugprone-reserved-identifier]
-  // CHECK-FIXES: {{^}}class _object {{{$}}
+  // CHECK-FIXES: class _object {
   int __member;
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '__member', which is a reserved identifier [bugprone-reserved-identifier]
-  // CHECK-FIXES: {{^}}  int _member;{{$}}
+  // CHECK-FIXES: int _member;
 };
 
 float __global;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '__global', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}float _global;{{$}}
+// CHECK-FIXES: float _global;
 
 void __function() {}
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: declaration uses identifier '__function', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}void _function() {}{{$}}
+// CHECK-FIXES: void _function() {}
 
 using __alias = int;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '__alias', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}using _alias = int;{{$}}
+// CHECK-FIXES: using _alias = int;
 
 template <typename __templateParam>
 // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: declaration uses identifier '__templateParam', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}template <typename _templateParam>{{$}}
+// CHECK-FIXES: template <typename _templateParam>
 struct S {};
 
 } // namespace __ns
@@ -81,34 +81,34 @@ struct S {};
 
 #define macro___m(m) int m = 0
 // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: declaration uses identifier 'macro___m', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}#define macro_m(m) int m = 0{{$}}
+// CHECK-FIXES: #define macro_m(m) int m = 0
 
 namespace ns___n {
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: declaration uses identifier 'ns___n', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}namespace ns_n {{{$}}
+// CHECK-FIXES: namespace ns_n {
 class object___o {
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier 'object___o', which is a reserved identifier [bugprone-reserved-identifier]
-  // CHECK-FIXES: {{^}}class object_o {{{$}}
+  // CHECK-FIXES: class object_o {
   int member___m;
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier 'member___m', which is a reserved identifier [bugprone-reserved-identifier]
-  // CHECK-FIXES: {{^}}  int member_m;{{$}}
+  // CHECK-FIXES: int member_m;
 };
 
 float global___g;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier 'global___g', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}float global_g;{{$}}
+// CHECK-FIXES: float global_g;
 
 void function___f() {}
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: declaration uses identifier 'function___f', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}void function_f() {}{{$}}
+// CHECK-FIXES: void function_f() {}
 
 using alias___a = int;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier 'alias___a', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}using alias_a = int;{{$}}
+// CHECK-FIXES: using alias_a = int;
 
 template <typename templateParam___t>
 // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: declaration uses identifier 'templateParam___t', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}template <typename templateParam_t>{{$}}
+// CHECK-FIXES: template <typename templateParam_t>
 struct S {};
 
 } // namespace ns___n
@@ -121,55 +121,55 @@ struct S {};
 
 namespace _ns {
 // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: declaration uses identifier '_ns', which is reserved in the global namespace [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}namespace ns {{{$}}
+// CHECK-FIXES: namespace ns {
 int _i;
 // no warning
 } // namespace _ns
 class _object {
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '_object', which is reserved in the global namespace [bugprone-reserved-identifier]
-  // CHECK-FIXES: {{^}}class object {{{$}}
+  // CHECK-FIXES: class object {
   int _member;
   // no warning
 };
 float _global;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '_global', which is reserved in the global namespace [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}float global;{{$}}
+// CHECK-FIXES: float global;
 void _function() {}
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: declaration uses identifier '_function', which is reserved in the global namespace [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}void function() {}{{$}}
+// CHECK-FIXES: void function() {}
 using _alias = int;
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: declaration uses identifier '_alias', which is reserved in the global namespace [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}using alias = int;{{$}}
+// CHECK-FIXES: using alias = int;
 template <typename _templateParam> // no warning, template params are not in the global namespace
 struct S {};
 
 void _float() {}
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: declaration uses identifier '_float', which is reserved in the global namespace; cannot be fixed because 'float' would conflict with a keyword [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}void _float() {}{{$}}
+// CHECK-FIXES: void _float() {}
 
 #define SOME_MACRO
 int SOME__MACRO;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: declaration uses identifier 'SOME__MACRO', which is a reserved identifier; cannot be fixed because 'SOME_MACRO' would conflict with a macro definition [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}int SOME__MACRO;{{$}}
+// CHECK-FIXES: int SOME__MACRO;
 
 void _TWO__PROBLEMS() {}
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: declaration uses identifier '_TWO__PROBLEMS', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}void TWO_PROBLEMS() {}{{$}}
+// CHECK-FIXES: void TWO_PROBLEMS() {}
 void _two__problems() {}
 // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: declaration uses identifier '_two__problems', which is a reserved identifier [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}void two_problems() {}{{$}}
+// CHECK-FIXES: void two_problems() {}
 
 int __;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: declaration uses identifier '__', which is a reserved identifier; cannot be fixed automatically [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}int __;{{$}}
+// CHECK-FIXES: int __;
 
 int _________;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: declaration uses identifier '_________', which is a reserved identifier; cannot be fixed automatically [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}int _________;{{$}}
+// CHECK-FIXES: int _________;
 
 int _;
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: declaration uses identifier '_', which is reserved in the global namespace; cannot be fixed automatically [bugprone-reserved-identifier]
-// CHECK-FIXES: {{^}}int _;{{$}}
+// CHECK-FIXES: int _;
 
 // This should not trigger a warning
 // https://github.com/llvm/llvm-project/issues/52895
diff --git a/clang-tools-extra/test/clang-tidy/checkers/google/readability-casting.cpp b/clang-tools-extra/test/clang-tidy/checkers/google/readability-casting.cpp
index bc53b54264af8..7ccdf705e8399 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/google/readability-casting.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/google/readability-casting.cpp
@@ -16,16 +16,16 @@ void f(int a, double b, const char *cpc, const void *cpv, X *pX) {
   Typedef1 t1;
   (Typedef2)t1;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: C-style casts are discouraged; use static_cast (if needed, the cast may be redundant) [google-readability-casting]
-  // CHECK-FIXES: {{^}}  static_cast<Typedef2>(t1);
+  // CHECK-FIXES: static_cast<Typedef2>(t1);
   (const char*)t1;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: {{.*}}; use static_cast (if needed
-  // CHECK-FIXES: {{^}}  static_cast<const char*>(t1);
+  // CHECK-FIXES: static_cast<const char*>(t1);
   (Typedef1)cpc;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: {{.*}}; use static_cast (if needed
-  // CHECK-FIXES: {{^}}  static_cast<Typedef1>(cpc);
+  // CHECK-FIXES: static_cast<Typedef1>(cpc);
   (Typedef1)t1;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: redundant cast to the same type
-  // CHECK-FIXES: {{^}}  t1;
+  // CHECK-FIXES: t1;
 
   char *pc = (char*)cpc;
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: C-style casts are discouraged; use const_cast [google-readability-casting]
@@ -33,15 +33,15 @@ void f(int a, double b, const char *cpc, const void *cpv, X *pX) {
   typedef char Char;
   Char *pChar = (Char*)pc;
   // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: {{.*}}; use static_cast (if needed
-  // CHECK-FIXES: {{^}}  Char *pChar = static_cast<Char*>(pc);
+  // CHECK-FIXES: Char *pChar = static_cast<Char*>(pc);
 
   (Char)*cpc;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: {{.*}}; use static_cast (if needed
-  // CHECK-FIXES: {{^}}  static_cast<Char>(*cpc);
+  // CHECK-FIXES: static_cast<Char>(*cpc);
 
   (char)*pChar;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: {{.*}}; use static_cast (if needed
-  // CHECK-FIXES: {{^}}  static_cast<char>(*pChar);
+  // CHECK-FIXES: static_cast<char>(*pChar);
 
   (const char*)cpv;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: {{.*}}; use static_cast [
@@ -124,24 +124,24 @@ void f(int a, double b, const char *cpc, const void *cpv, X *pX) {
 
   e = (Enum)Enum1;
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: redundant cast to the same type
-  // CHECK-FIXES: {{^}}  e = Enum1;
+  // CHECK-FIXES: e = Enum1;
 
   e = (Enum)e;
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: redundant cast to the same type
-  // CHECK-FIXES: {{^}}  e = e;
+  // CHECK-FIXES: e = e;
 
   e = (Enum)           e;
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: redundant cast to the same type
-  // CHECK-FIXES: {{^}}  e = e;
+  // CHECK-FIXES: e = e;
 
   e = (Enum)           (e);
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: redundant cast to the same type
-  // CHECK-FIXES: {{^}}  e = (e);
+  // CHECK-FIXES: e = (e);
 
   static const int kZero = 0;
   (int)kZero;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: redundant cast to the same type
-  // CHECK-FIXES: {{^}}  kZero;
+  // CHECK-FIXES: kZero;
 
   int b2 = static_cast<double>(b);
   int b3 = b;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/llvm/qualified-auto.cpp b/clang-tools-extra/test/clang-tidy/checkers/llvm/qualified-auto.cpp
index 97473ae41ec64..c39f7184f2794 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/llvm/qualified-auto.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/llvm/qualified-auto.cpp
@@ -10,10 +10,10 @@ const int *getCIntPtr();
 void foo() {
   auto NakedPtr = getIntPtr();
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: 'auto NakedPtr' can be declared as 'auto *NakedPtr'
-  // CHECK-FIXES: {{^}}  auto *NakedPtr = getIntPtr();
+  // CHECK-FIXES: auto *NakedPtr = getIntPtr();
   auto NakedConstPtr = getCIntPtr();
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: 'auto NakedConstPtr' can be declared as 'const auto *NakedConstPtr'
-  // CHECK-FIXES: {{^}}  const auto *NakedConstPtr = getCIntPtr();
+  // CHECK-FIXES: const auto *NakedConstPtr = getCIntPtr();
   auto *Ptr = getIntPtr();
   auto *ConstPtr = getCIntPtr();
   auto &NakedRef = *getIntPtr();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.cpp
index 16c6ba60c1925..897049cbc5388 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.cpp
@@ -40,24 +40,24 @@ constexpr int Y = 1;
 void referenceConstexprVariable() {
   assert(Y > 0);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be replaced by static_assert() [misc-static-assert]
-  // CHECK-FIXES-CXX11: {{^  }}static_assert(Y > 0, "");
-  // CHECK-FIXES-CXX17: {{^  }}static_assert(Y > 0);
+  // CHECK-FIXES-CXX11: static_assert(Y > 0, "");
+  // CHECK-FIXES-CXX17: static_assert(Y > 0);
 }
 
 void useInSizeOf() {
   char a = 0;
   assert(sizeof(a) == 1U);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be replaced by static_assert() [misc-static-assert]
-  // CHECK-FIXES-CXX11: {{^  }}static_assert(sizeof(a) == 1U, "");
-  // CHECK-FIXES-CXX17: {{^  }}static_assert(sizeof(a) == 1U);
+  // CHECK-FIXES-CXX11: static_assert(sizeof(a) == 1U, "");
+  // CHECK-FIXES-CXX17: static_assert(sizeof(a) == 1U);
 }
 
 void useInDecltype() {
   char a = 0;
   assert(static_cast<decltype(a)>(256) == 0);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be replaced by static_assert() [misc-static-assert]
-  // CHECK-FIXES-CXX11: {{^  }}static_assert(static_cast<decltype(a)>(256) == 0, "");
-  // CHECK-FIXES-CXX17: {{^  }}static_assert(static_cast<decltype(a)>(256) == 0);
+  // CHECK-FIXES-CXX11: static_assert(static_cast<decltype(a)>(256) == 0, "");
+  // CHECK-FIXES-CXX17: static_assert(static_cast<decltype(a)>(256) == 0);
 }
 
 }
@@ -80,11 +80,11 @@ class B {
 template <class T> void doSomething(T t) {
   assert(myfunc(1, 2));
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be replaced by static_assert() [misc-static-assert]
-  // CHECK-FIXES-CXX11: {{^  }}static_assert(myfunc(1, 2), "");
-  // CHECK-FIXES-CXX17: {{^  }}static_assert(myfunc(1, 2));
+  // CHECK-FIXES-CXX11: static_assert(myfunc(1, 2), "");
+  // CHECK-FIXES-CXX17: static_assert(myfunc(1, 2));
 
   assert(t.method());
-  // CHECK-FIXES: {{^  }}assert(t.method());
+  // CHECK-FIXES: assert(t.method());
 
   assert(sizeof(T) == 123);
 }
@@ -92,16 +92,16 @@ template <class T> void doSomething(T t) {
 int main() {
   my_macro();
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES: {{^  }}my_macro();
+  // CHECK-FIXES: my_macro();
 
   assert(myfunc(1, 2) && (3 == 4));
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES-CXX11: {{^  }}static_assert(myfunc(1, 2) && (3 == 4), "");
-  // CHECK-FIXES-CXX17: {{^  }}static_assert(myfunc(1, 2) && (3 == 4));
+  // CHECK-FIXES-CXX11: static_assert(myfunc(1, 2) && (3 == 4), "");
+  // CHECK-FIXES-CXX17: static_assert(myfunc(1, 2) && (3 == 4));
 
   int x = 1;
   assert(x == 0);
-  // CHECK-FIXES: {{^  }}assert(x == 0);
+  // CHECK-FIXES: assert(x == 0);
 
   A a;
   B b;
@@ -110,55 +110,55 @@ int main() {
   doSomething<B>(b);
 
   assert(false);
-  // CHECK-FIXES: {{^  }}assert(false);
+  // CHECK-FIXES: assert(false);
 
   assert(False);
-  // CHECK-FIXES: {{^  }}assert(False);
+  // CHECK-FIXES: assert(False);
   assert(FALSE);
-  // CHECK-FIXES: {{^  }}assert(FALSE);
+  // CHECK-FIXES: assert(FALSE);
 
   assert(ZERO_MACRO);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES-CXX11: {{^  }}static_assert(ZERO_MACRO, "");
-  // CHECK-FIXES-CXX17: {{^  }}static_assert(ZERO_MACRO);
+  // CHECK-FIXES-CXX11: static_assert(ZERO_MACRO, "");
+  // CHECK-FIXES-CXX17: static_assert(ZERO_MACRO);
 
   assert(!"Don't report me!");
-  // CHECK-FIXES: {{^  }}assert(!"Don't report me!");
+  // CHECK-FIXES: assert(!"Don't report me!");
 
   assert(0 && "Don't report me!");
-  // CHECK-FIXES: {{^  }}assert(0 && "Don't report me!");
+  // CHECK-FIXES: assert(0 && "Don't report me!");
 
   assert(false && "Don't report me!");
-  // CHECK-FIXES: {{^  }}assert(false && "Don't report me!");
+  // CHECK-FIXES: assert(false && "Don't report me!");
 
 #define NULL ((void*)0)
   assert(NULL && "Don't report me!");
-  // CHECK-FIXES: {{^  }}assert(NULL && "Don't report me!");
+  // CHECK-FIXES: assert(NULL && "Don't report me!");
 
   assert(NULL == "Don't report me!");
-  // CHECK-FIXES: {{^  }}assert(NULL == "Don't report me!");
+  // CHECK-FIXES: assert(NULL == "Don't report me!");
 
   assert("Don't report me!" == NULL);
-  // CHECK-FIXES: {{^  }}assert("Don't report me!" == NULL);
+  // CHECK-FIXES: assert("Don't report me!" == NULL);
 
   assert(0 == "Don't report me!");
-  // CHECK-FIXES: {{^  }}assert(0 == "Don't report me!");
+  // CHECK-FIXES: assert(0 == "Don't report me!");
 
 #define NULL ((unsigned int)0)
   assert(NULL && "Report me!");
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES: {{^  }}static_assert(NULL , "Report me!");
+  // CHECK-FIXES: static_assert(NULL , "Report me!");
 
 #define NULL __null
   assert(__null == "Don't report me!");
-  // CHECK-FIXES: {{^  }}assert(__null == "Don't report me!");
+  // CHECK-FIXES: assert(__null == "Don't report me!");
   assert(NULL == "Don't report me!");
-  // CHECK-FIXES: {{^  }}assert(NULL == "Don't report me!");
+  // CHECK-FIXES: assert(NULL == "Don't report me!");
 #undef NULL
 
   assert(ZERO_MACRO && "Report me!");
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES: {{^  }}static_assert(ZERO_MACRO , "Report me!");
+  // CHECK-FIXES: static_assert(ZERO_MACRO , "Report me!");
 
   assert(0);
 
@@ -171,19 +171,19 @@ int main() {
 
   assert(10==5 && "Report me!");
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES: {{^  }}static_assert(10==5 , "Report me!");
+  // CHECK-FIXES: static_assert(10==5 , "Report me!");
 
   assert(strlen("12345") == 5);
-  // CHECK-FIXES: {{^  }}assert(strlen("12345") == 5);
+  // CHECK-FIXES: assert(strlen("12345") == 5);
 
 #define assert(e) (__builtin_expect(!(e), 0) ? print (#e, __FILE__, __LINE__) : (void)0)
   assert(false);
-  // CHECK-FIXES: {{^  }}assert(false);
+  // CHECK-FIXES: assert(false);
 
   assert(10 == 5 + 5);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be
-  // CHECK-FIXES-CXX11: {{^  }}static_assert(10 == 5 + 5, "");
-  // CHECK-FIXES-CXX17: {{^  }}static_assert(10 == 5 + 5);
+  // CHECK-FIXES-CXX11: static_assert(10 == 5 + 5, "");
+  // CHECK-FIXES-CXX17: static_assert(10 == 5 + 5);
 #undef assert
 
   return 0;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/make-unique-macros.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/make-unique-macros.cpp
index c010d5bae1ea3..78beb911f5a0a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/make-unique-macros.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/make-unique-macros.cpp
@@ -7,19 +7,19 @@
 class Foo {};
 class Bar {};
 #define DEFINE(...) __VA_ARGS__
-// CHECK-FIXES: {{^}}#define DEFINE(...) __VA_ARGS__{{$}}
+// CHECK-FIXES: #define DEFINE(...) __VA_ARGS__
 template<typename T>
 void g2(std::unique_ptr<Foo> *t) {
   DEFINE(
-  // CHECK-FIXES: {{^ *}}DEFINE({{$}}
+  // CHECK-FIXES: DEFINE(
       auto p = std::unique_ptr<Foo>(new Foo);
       // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: use std::make_unique instead
-      // CHECK-FIXES: {{^ *}}auto p = std::unique_ptr<Foo>(new Foo);{{$}}
+      // CHECK-FIXES: auto p = std::unique_ptr<Foo>(new Foo);
       t->reset(new Foo);
       // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: use std::make_unique instead
-      // CHECK-FIXES: {{^ *}}t->reset(new Foo);{{$}}
+      // CHECK-FIXES: t->reset(new Foo);
       );
-      // CHECK-FIXES: {{^ *}});{{$}}
+      // CHECK-FIXES: );
 }
 void macro() {
   std::unique_ptr<Foo> *t;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/unary-static-assert.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/unary-static-assert.cpp
index 74b4209505f97..be368e7f6c1b5 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/unary-static-assert.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/unary-static-assert.cpp
@@ -6,11 +6,11 @@
 void f_textless(int a) {
   static_assert(sizeof(a) <= 10, "");
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use unary 'static_assert' when the string literal is an empty string [modernize-unary-static-assert]
-  // CHECK-FIXES: {{^}}  static_assert(sizeof(a) <= 10 );{{$}}
+  // CHECK-FIXES: static_assert(sizeof(a) <= 10 );
   FOO
-  // CHECK-FIXES: {{^}}  FOO{{$}}
+  // CHECK-FIXES: FOO
   static_assert(sizeof(a) <= 17, MSG);
-  // CHECK-FIXES: {{^}}  static_assert(sizeof(a) <= 17, MSG);{{$}}
+  // CHECK-FIXES: static_assert(sizeof(a) <= 17, MSG);
 }
 
 void f_with_tex(int a) {
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-bool-literals-ignore-macros.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-bool-literals-ignore-macros.cpp
index 392fcfe860f14..8dc01d7316bcb 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-bool-literals-ignore-macros.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-bool-literals-ignore-macros.cpp
@@ -5,41 +5,41 @@
 
 bool IntToTrue = 1;
 // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: converting integer literal to bool, use bool literal instead [modernize-use-bool-literals]
-// CHECK-FIXES: {{^}}bool IntToTrue = true;{{$}}
+// CHECK-FIXES: bool IntToTrue = true;
 
 bool IntToFalse(0);
 // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: converting integer literal to bool
-// CHECK-FIXES: {{^}}bool IntToFalse(false);{{$}}
+// CHECK-FIXES: bool IntToFalse(false);
 
 bool LongLongToTrue{0x1LL};
 // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: converting integer literal to bool
-// CHECK-FIXES: {{^}}bool LongLongToTrue{true};{{$}}
+// CHECK-FIXES: bool LongLongToTrue{true};
 
 bool ExplicitCStyleIntToFalse = (bool)0;
 // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: converting integer literal to bool
-// CHECK-FIXES: {{^}}bool ExplicitCStyleIntToFalse = false;{{$}}
+// CHECK-FIXES: bool ExplicitCStyleIntToFalse = false;
 
 bool ExplicitFunctionalIntToFalse = bool(0);
 // CHECK-MESSAGES: :[[@LINE-1]]:37: warning: converting integer literal to bool
-// CHECK-FIXES: {{^}}bool ExplicitFunctionalIntToFalse = false;{{$}}
+// CHECK-FIXES: bool ExplicitFunctionalIntToFalse = false;
 
 bool ExplicitStaticIntToFalse = static_cast<bool>(0);
 // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: converting integer literal to bool
-// CHECK-FIXES: {{^}}bool ExplicitStaticIntToFalse = false;{{$}}
+// CHECK-FIXES: bool ExplicitStaticIntToFalse = false;
 
 #define TRUE_MACRO 1
-// CHECK-FIXES: {{^}}#define TRUE_MACRO 1{{$}}
+// CHECK-FIXES: #define TRUE_MACRO 1
 
 bool MacroIntToTrue = TRUE_MACRO;
-// CHECK-FIXES: {{^}}bool MacroIntToTrue = TRUE_MACRO;{{$}}
+// CHECK-FIXES: bool MacroIntToTrue = TRUE_MACRO;
 
 #define FALSE_MACRO bool(0)
-// CHECK-FIXES: {{^}}#define FALSE_MACRO bool(0){{$}}
+// CHECK-FIXES: #define FALSE_MACRO bool(0)
 
 bool TrueBool = true; // OK
 
 bool FalseBool = bool(FALSE_MACRO);
-// CHECK-FIXES: {{^}}bool FalseBool = bool(FALSE_MACRO);{{$}}
+// CHECK-FIXES: bool FalseBool = bool(FALSE_MACRO);
 
 void boolFunction(bool bar) {
 
@@ -50,56 +50,56 @@ char Character = 0; // OK
 unsigned long long LongInteger = 1; // OK
 
 #define MACRO_DEPENDENT_CAST(x) static_cast<bool>(x)
-// CHECK-FIXES: {{^}}#define MACRO_DEPENDENT_CAST(x) static_cast<bool>(x){{$}}
+// CHECK-FIXES: #define MACRO_DEPENDENT_CAST(x) static_cast<bool>(x)
 
 bool MacroDependentBool = MACRO_DEPENDENT_CAST(0);
-// CHECK-FIXES: {{^}}bool MacroDependentBool = MACRO_DEPENDENT_CAST(0);{{$}}
+// CHECK-FIXES: bool MacroDependentBool = MACRO_DEPENDENT_CAST(0);
 
 bool ManyMacrosDependent = MACRO_DEPENDENT_CAST(FALSE_MACRO);
-// CHECK-FIXES: {{^}}bool ManyMacrosDependent = MACRO_DEPENDENT_CAST(FALSE_MACRO);{{$}}
+// CHECK-FIXES: bool ManyMacrosDependent = MACRO_DEPENDENT_CAST(FALSE_MACRO);
 
 class FooClass {
   public:
   FooClass() : JustBool(0) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}FooClass() : JustBool(false) {}{{$}}
+  // CHECK-FIXES: FooClass() : JustBool(false) {}
   FooClass(int) : JustBool{0} {}
   // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}FooClass(int) : JustBool{false} {}{{$}}
+  // CHECK-FIXES: FooClass(int) : JustBool{false} {}
   private:
   bool JustBool;
   bool BoolWithBraces{0};
   // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}bool BoolWithBraces{false};{{$}}
+  // CHECK-FIXES: bool BoolWithBraces{false};
   bool BoolFromInt = 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}bool BoolFromInt = false;{{$}}
+  // CHECK-FIXES: bool BoolFromInt = false;
   bool SimpleBool = true; // OK
 };
 
 template<typename type>
 void templateFunction(type) {
   type TemplateType = 0;
-  // CHECK-FIXES: {{^ *}}type TemplateType = 0;{{$}}
+  // CHECK-FIXES: type TemplateType = 0;
 }
 
 template<int c>
 void valueDependentTemplateFunction() {
   bool Boolean = c;
-  // CHECK-FIXES: {{^ *}}bool Boolean = c;{{$}}
+  // CHECK-FIXES: bool Boolean = c;
 }
 
 template<typename type>
 void anotherTemplateFunction(type) {
   bool JustBool = 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:19: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}bool JustBool = false;{{$}}
+  // CHECK-FIXES: bool JustBool = false;
 }
 
 int main() {
   boolFunction(1);
   // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}boolFunction(true);{{$}}
+  // CHECK-FIXES: boolFunction(true);
 
   boolFunction(false);
 
@@ -113,7 +113,7 @@ int main() {
 
   IntToTrue = 1;
   // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}IntToTrue = true;{{$}}
+  // CHECK-FIXES: IntToTrue = true;
 }
 
 static int Value = 1;
@@ -122,7 +122,7 @@ bool Function1() {
   bool Result = Value == 1 ? 1 : 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:30: warning: converting integer literal to bool
   // CHECK-MESSAGES: :[[@LINE-2]]:34: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}bool Result = Value == 1 ? true : false;{{$}}
+  // CHECK-FIXES: bool Result = Value == 1 ? true : false;
   return Result;
 }
 
@@ -130,18 +130,18 @@ bool Function2() {
   return Value == 1 ? 1 : 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: converting integer literal to bool
   // CHECK-MESSAGES: :[[@LINE-2]]:27: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}return Value == 1 ? true : false;{{$}}
+  // CHECK-FIXES: return Value == 1 ? true : false;
 }
 
 void foo() {
   bool Result;
   Result = Value == 1 ? true : 0;
   // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}Result = Value == 1 ? true : false;{{$}}
+  // CHECK-FIXES: Result = Value == 1 ? true : false;
   Result = Value == 1 ? false : bool(0);
   // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}Result = Value == 1 ? false : false;{{$}}
+  // CHECK-FIXES: Result = Value == 1 ? false : false;
   Result = Value == 1 ? (bool)0 : false;
   // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: converting integer literal to bool
-  // CHECK-FIXES: {{^ *}}Result = Value == 1 ? false : false;{{$}}
+  // CHECK-FIXES: Result = Value == 1 ? false : false;
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-override-cxx98.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-override-cxx98.cpp
index aaa9d35cdef57..c0b998eb1616e 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-override-cxx98.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-override-cxx98.cpp
@@ -9,11 +9,11 @@ struct Base {
 struct SimpleCases : public Base {
 public:
   virtual ~SimpleCases();
-  // CHECK-FIXES: {{^}}  virtual ~SimpleCases();
+  // CHECK-FIXES: virtual ~SimpleCases();
 
   void a();
-  // CHECK-FIXES: {{^}}  void a();
+  // CHECK-FIXES: void a();
 
   virtual void b();
-  // CHECK-FIXES: {{^}}  virtual void b();
+  // CHECK-FIXES: virtual void b();
 };

From 8b46b98b916050e739cf3b740526e5c2f6936dad Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Thu, 17 Apr 2025 11:15:49 -0700
Subject: [PATCH 303/710] AMDGPU: Fix the double rounding issue in v2f64 ->
 v2f16 conversion (#135659)

On targets that support v_cvt_pk_f16_f32 instruction, if we make v2f64
-> v2f16 Legal, we will generate the following sequence of instructions:
  v_cvt_f32_f64_e32 v1, s[6:7]
  v_cvt_f32_f64_e32 v2, s[4:5]
  v_cvt_pk_f16_f32 v1, v2, v1
It possibly returns imprecise results due to double rounding. This patch
fixes the issue by not setting the conversion Legal. While we may still
expect the above sequence of code when unsafe fpmath is set, I hope
https://github.com/llvm/llvm-project/pull/134738 can address that
performance concern.

Fixes: SWDEV-523856
---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   3 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |   4 -
 llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll       |  14 +-
 .../CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll    | 213 ++++++++++++++++++
 4 files changed, 222 insertions(+), 12 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 5fcbf810abcbd..beb6432170970 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1054,8 +1054,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
 
   auto &FPTruncActions = getActionDefinitionsBuilder(G_FPTRUNC);
   if (ST.hasCvtPkF16F32Inst())
-    FPTruncActions.legalFor(
-        {{S32, S64}, {S16, S32}, {V2S16, V2S32}, {V2S16, V2S64}});
+    FPTruncActions.legalFor({{S32, S64}, {S16, S32}, {V2S16, V2S32}});
   else
     FPTruncActions.legalFor({{S32, S64}, {S16, S32}});
   FPTruncActions.scalarize(0).lower();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 985af4a86009f..67f2cf2c862f6 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -915,10 +915,6 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal);
   }
 
-  if (Subtarget->hasCvtPkF16F32Inst()) {
-    setOperationAction(ISD::FP_ROUND, MVT::v2f16, Legal);
-  }
-
   setTargetDAGCombine({ISD::ADD,
                        ISD::UADDO_CARRY,
                        ISD::SUB,
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 188b2ada64686..6ab17c91c1439 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -654,7 +654,9 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
 ; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX950-SDAG-NEXT:    v_cvt_f32_f64_e32 v2, v[2:3]
 ; GFX950-SDAG-NEXT:    v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v0, v0, v2
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v1, v2
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-SDAG-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX950-SDAG-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX950-SDAG-NEXT:    s_endpgm
 ;
@@ -666,11 +668,11 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
 ; GFX950-GISEL-NEXT:    s_mov_b32 s2, -1
 ; GFX950-GISEL-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX950-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
-; GFX950-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
-; GFX950-GISEL-NEXT:    v_cvt_f32_f64_e32 v2, v[2:3]
-; GFX950-GISEL-NEXT:    v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v0, v0, v2
+; GFX950-GISEL-NEXT:    v_cvt_f32_f64_e32 v0, s[4:5]
+; GFX950-GISEL-NEXT:    v_cvt_f32_f64_e32 v1, s[6:7]
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX950-GISEL-NEXT:    v_pack_b32_f16 v0, v0, v1
 ; GFX950-GISEL-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX950-GISEL-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll
new file mode 100644
index 0000000000000..9e9b9629931b9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -check-prefixes=GFX950,GFX950-SAFE-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -check-prefixes=GFX950,GFX950-SAFE-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 -enable-unsafe-fp-math < %s | FileCheck -check-prefixes=GFX950,GFX950-UNSAFE %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 -enable-unsafe-fp-math < %s | FileCheck -check-prefixes=GFX950,GFX950-UNSAFE %s
+
+define <2 x half> @v_test_cvt_v2f32_v2f16(<2 x float> %src) {
+; GFX950-LABEL: v_test_cvt_v2f32_v2f16:
+; GFX950:       ; %bb.0:
+; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-NEXT:    s_setpc_b64 s[30:31]
+  %res = fptrunc <2 x float> %src to <2 x half>
+  ret <2 x half> %res
+}
+
+define <2 x half> @v_test_cvt_v2f64_v2f16(<2 x double> %src) {
+; GFX950-SAFE-SDAG-LABEL: v_test_cvt_v2f64_v2f16:
+; GFX950-SAFE-SDAG:       ; %bb.0:
+; GFX950-SAFE-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SAFE-SDAG-NEXT:    s_movk_i32 s0, 0x1ff
+; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v0, v1, s0, v0
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX950-SAFE-SDAG-NEXT:    s_movk_i32 s1, 0xffe
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_bfe_u32 v5, v1, 20, 11
+; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v0, v4, s1, v0
+; GFX950-SAFE-SDAG-NEXT:    v_sub_u32_e32 v6, 0x3f1, v5
+; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v4, 0x1000, v0
+; GFX950-SAFE-SDAG-NEXT:    v_med3_i32 v6, v6, 0, 13
+; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v7, v6, v4
+; GFX950-SAFE-SDAG-NEXT:    v_lshlrev_b32_e32 v6, v6, v7
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, v6, v4
+; GFX950-SAFE-SDAG-NEXT:    v_add_u32_e32 v5, 0xfffffc10, v5
+; GFX950-SAFE-SDAG-NEXT:    v_lshl_or_b32 v6, v5, 12, v0
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v4, v7, v4
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v5
+; GFX950-SAFE-SDAG-NEXT:    s_movk_i32 s2, 0x40f
+; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_and_b32_e32 v6, 7, v4
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v6
+; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 2, v4
+; GFX950-SAFE-SDAG-NEXT:    s_mov_b32 s3, 0x8000
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v6
+; GFX950-SAFE-SDAG-NEXT:    s_nop 1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v6, v6, v7
+; GFX950-SAFE-SDAG-NEXT:    v_add_u32_e32 v4, v4, v6
+; GFX950-SAFE-SDAG-NEXT:    v_mov_b32_e32 v6, 0x7c00
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v5
+; GFX950-SAFE-SDAG-NEXT:    v_mov_b32_e32 v7, 0x7e00
+; GFX950-SAFE-SDAG-NEXT:    s_nop 0
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-SAFE-SDAG-NEXT:    s_nop 1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v5
+; GFX950-SAFE-SDAG-NEXT:    s_nop 1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v0, v1, s3, v0
+; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v1, v3, s0, v2
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
+; GFX950-SAFE-SDAG-NEXT:    v_bfe_u32 v4, v3, 20, 11
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v1, v2, s1, v1
+; GFX950-SAFE-SDAG-NEXT:    v_sub_u32_e32 v5, 0x3f1, v4
+; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v2, 0x1000, v1
+; GFX950-SAFE-SDAG-NEXT:    v_med3_i32 v5, v5, 0, 13
+; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v8, v5, v2
+; GFX950-SAFE-SDAG-NEXT:    v_lshlrev_b32_e32 v5, v5, v8
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v2
+; GFX950-SAFE-SDAG-NEXT:    v_add_u32_e32 v4, 0xfffffc10, v4
+; GFX950-SAFE-SDAG-NEXT:    v_lshl_or_b32 v5, v4, 12, v1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v2, v8, v2
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v4
+; GFX950-SAFE-SDAG-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX950-SAFE-SDAG-NEXT:    s_nop 0
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_and_b32_e32 v5, 7, v2
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v5
+; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 2, v2
+; GFX950-SAFE-SDAG-NEXT:    s_nop 0
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v5
+; GFX950-SAFE-SDAG-NEXT:    s_nop 1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v5, v5, v8
+; GFX950-SAFE-SDAG-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v4
+; GFX950-SAFE-SDAG-NEXT:    s_nop 1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX950-SAFE-SDAG-NEXT:    s_nop 1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v1, v6, v7, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v4
+; GFX950-SAFE-SDAG-NEXT:    s_nop 1
+; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v1, v2, s3, v1
+; GFX950-SAFE-SDAG-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX950-SAFE-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX950-SAFE-GISEL-LABEL: v_test_cvt_v2f64_v2f16:
+; GFX950-SAFE-GISEL:       ; %bb.0:
+; GFX950-SAFE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SAFE-GISEL-NEXT:    s_movk_i32 s0, 0x1ff
+; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v0, v1, s0, v0
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX950-SAFE-GISEL-NEXT:    s_movk_i32 s1, 0xffe
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_bfe_u32 v5, v1, 20, 11
+; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v0, v4, s1, v0
+; GFX950-SAFE-GISEL-NEXT:    v_sub_u32_e32 v6, 0x3f1, v5
+; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v4, 0x1000, v0
+; GFX950-SAFE-GISEL-NEXT:    v_med3_i32 v6, v6, 0, 13
+; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v7, v6, v4
+; GFX950-SAFE-GISEL-NEXT:    v_lshlrev_b32_e32 v6, v6, v7
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, v6, v4
+; GFX950-SAFE-GISEL-NEXT:    v_add_u32_e32 v5, 0xfffffc10, v5
+; GFX950-SAFE-GISEL-NEXT:    v_lshl_or_b32 v6, v5, 12, v0
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v4, v7, v4
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v5
+; GFX950-SAFE-GISEL-NEXT:    s_movk_i32 s2, 0x40f
+; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_and_b32_e32 v6, 7, v4
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v6
+; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 2, v4
+; GFX950-SAFE-GISEL-NEXT:    s_mov_b32 s3, 0x8000
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v6
+; GFX950-SAFE-GISEL-NEXT:    s_nop 1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v6, v6, v7
+; GFX950-SAFE-GISEL-NEXT:    v_add_u32_e32 v4, v4, v6
+; GFX950-SAFE-GISEL-NEXT:    v_mov_b32_e32 v6, 0x7c00
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v5
+; GFX950-SAFE-GISEL-NEXT:    v_mov_b32_e32 v7, 0x7e00
+; GFX950-SAFE-GISEL-NEXT:    s_nop 0
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-SAFE-GISEL-NEXT:    s_nop 1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v5
+; GFX950-SAFE-GISEL-NEXT:    s_nop 1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v0, v1, s3, v0
+; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v1, v3, s0, v2
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
+; GFX950-SAFE-GISEL-NEXT:    v_bfe_u32 v4, v3, 20, 11
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v1, v2, s1, v1
+; GFX950-SAFE-GISEL-NEXT:    v_sub_u32_e32 v5, 0x3f1, v4
+; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v2, 0x1000, v1
+; GFX950-SAFE-GISEL-NEXT:    v_med3_i32 v5, v5, 0, 13
+; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v8, v5, v2
+; GFX950-SAFE-GISEL-NEXT:    v_lshlrev_b32_e32 v5, v5, v8
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v2
+; GFX950-SAFE-GISEL-NEXT:    v_add_u32_e32 v4, 0xfffffc10, v4
+; GFX950-SAFE-GISEL-NEXT:    v_lshl_or_b32 v5, v4, 12, v1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v2, v8, v2
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v4
+; GFX950-SAFE-GISEL-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX950-SAFE-GISEL-NEXT:    s_nop 0
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_and_b32_e32 v5, 7, v2
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v5
+; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 2, v2
+; GFX950-SAFE-GISEL-NEXT:    s_nop 0
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v5
+; GFX950-SAFE-GISEL-NEXT:    s_nop 1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v5, v5, v8
+; GFX950-SAFE-GISEL-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v4
+; GFX950-SAFE-GISEL-NEXT:    s_nop 1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX950-SAFE-GISEL-NEXT:    s_nop 1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v1, v6, v7, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v4
+; GFX950-SAFE-GISEL-NEXT:    s_nop 1
+; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v1, v2, s3, v1
+; GFX950-SAFE-GISEL-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX950-SAFE-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX950-UNSAFE-LABEL: v_test_cvt_v2f64_v2f16:
+; GFX950-UNSAFE:       ; %bb.0:
+; GFX950-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-UNSAFE-NEXT:    v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX950-UNSAFE-NEXT:    v_cvt_f32_f64_e32 v1, v[2:3]
+; GFX950-UNSAFE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-UNSAFE-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX950-UNSAFE-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX950-UNSAFE-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX950-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
+  %res = fptrunc <2 x double> %src to <2 x half>
+  ret <2 x half> %res
+}
+

From e2c382346f3d3e04a784ad69cbe11ec575b26444 Mon Sep 17 00:00:00 2001
From: Daniel Chen <cdchen@ca.ibm.com>
Date: Thu, 17 Apr 2025 14:26:36 -0400
Subject: [PATCH 304/710] [flang] Add 32-bit AIX target specific in order to
 build 32-bit flang-rt (#136051)

---
 flang/lib/Optimizer/CodeGen/Target.cpp    |  34 +++++++
 flang/test/Fir/comdat.fir                 |   3 +-
 flang/test/Fir/target-rewrite-complex.fir | 103 ++++++++++++++++++++++
 3 files changed, 139 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp
index e2f8fb9d239a1..374308fa58947 100644
--- a/flang/lib/Optimizer/CodeGen/Target.cpp
+++ b/flang/lib/Optimizer/CodeGen/Target.cpp
@@ -1020,6 +1020,37 @@ struct TargetAArch64 : public GenericTarget<TargetAArch64> {
 };
 } // namespace
 
+//===----------------------------------------------------------------------===//
+// PPC (AIX 32 bit) target specifics.
+//===----------------------------------------------------------------------===//
+namespace {
+struct TargetPPC : public GenericTarget<TargetPPC> {
+  using GenericTarget::GenericTarget;
+
+  static constexpr int defaultWidth = 32;
+
+  CodeGenSpecifics::Marshalling
+  complexArgumentType(mlir::Location, mlir::Type eleTy) const override {
+    CodeGenSpecifics::Marshalling marshal;
+    // two distinct element type arguments (re, im)
+    marshal.emplace_back(eleTy, AT{});
+    marshal.emplace_back(eleTy, AT{});
+    return marshal;
+  }
+
+  CodeGenSpecifics::Marshalling
+  complexReturnType(mlir::Location, mlir::Type eleTy) const override {
+    CodeGenSpecifics::Marshalling marshal;
+    // Use a type that will be translated into LLVM as:
+    // { t, t }   struct of 2 element type
+    marshal.emplace_back(
+        mlir::TupleType::get(eleTy.getContext(), mlir::TypeRange{eleTy, eleTy}),
+        AT{});
+    return marshal;
+  }
+};
+} // namespace
+
 //===----------------------------------------------------------------------===//
 // PPC64 (AIX 64 bit) target specifics.
 //===----------------------------------------------------------------------===//
@@ -1847,6 +1878,9 @@ fir::CodeGenSpecifics::get(mlir::MLIRContext *ctx, llvm::Triple &&trp,
   case llvm::Triple::ArchType::aarch64:
     return std::make_unique<TargetAArch64>(
         ctx, std::move(trp), std::move(kindMap), targetCPU, targetFeatures, dl);
+  case llvm::Triple::ArchType::ppc:
+    return std::make_unique<TargetPPC>(ctx, std::move(trp), std::move(kindMap),
+                                       targetCPU, targetFeatures, dl);
   case llvm::Triple::ArchType::ppc64:
     return std::make_unique<TargetPPC64>(
         ctx, std::move(trp), std::move(kindMap), targetCPU, targetFeatures, dl);
diff --git a/flang/test/Fir/comdat.fir b/flang/test/Fir/comdat.fir
index 2f5da505e4903..ea777c389ed34 100644
--- a/flang/test/Fir/comdat.fir
+++ b/flang/test/Fir/comdat.fir
@@ -3,6 +3,7 @@
 // RUN: fir-opt %s --fir-to-llvm-ir="target=x86_64-pc-windows-msvc" | FileCheck %s --check-prefixes="CHECK-COMDAT"
 // RUN: fir-opt %s --fir-to-llvm-ir="target=aarch64-apple-darwin" | FileCheck %s --check-prefixes="CHECK-NOCOMDAT"
 // RUN: fir-opt %s --fir-to-llvm-ir="target=powerpc64-ibm-aix" | FileCheck %s --check-prefixes="CHECK-NOCOMDAT"
+// RUN: fir-opt %s --fir-to-llvm-ir="target=powerpc-ibm-aix" | FileCheck %s --check-prefixes="CHECK-NOCOMDAT"
 
 // CHECK-COMDAT: llvm.func linkonce @fun_linkonce(%arg0: i32) -> i32 comdat(@__llvm_comdat::@fun_linkonce)
 // CHECK-NOCOMDAT: llvm.func linkonce @fun_linkonce(%arg0: i32) -> i32 {
@@ -38,4 +39,4 @@ fir.global linkonce @global_linkonce constant : i32 {
 fir.global linkonce_odr @global_linkonce_odr constant : i32 {
   %0 = arith.constant 0 : i32
   fir.has_value %0 : i32
-}
\ No newline at end of file
+}
diff --git a/flang/test/Fir/target-rewrite-complex.fir b/flang/test/Fir/target-rewrite-complex.fir
index 405b14031f0c5..c8cf867713632 100644
--- a/flang/test/Fir/target-rewrite-complex.fir
+++ b/flang/test/Fir/target-rewrite-complex.fir
@@ -8,6 +8,7 @@
 // RUN: fir-opt --target-rewrite="target=sparcv9-sun-solaris2.11" %s | FileCheck %s --check-prefix=SPARCV9
 // RUN: fir-opt --target-rewrite="target=riscv64-unknown-linux-gnu" %s | FileCheck %s --check-prefix=RISCV64
 // RUN: fir-opt --target-rewrite="target=powerpc64-ibm-aix7.2.0.0" %s | FileCheck %s --check-prefix=PPC64
+// RUN: fir-opt --target-rewrite="target=powerpc-ibm-aix7.2.0.0" %s | FileCheck %s --check-prefix=PPC
 // RUN: fir-opt --target-rewrite="target=loongarch64-unknown-linux-gnu" %s | FileCheck %s --check-prefix=LOONGARCH64
 
 // Test that we rewrite the signature and body of a function that returns a
@@ -21,6 +22,7 @@
 // SPARCV9-LABEL: func @returncomplex4() -> tuple<f32, f32>
 // RISCV64-LABEL: func @returncomplex4() -> tuple<f32, f32>
 // PPC64-LABEL: func @returncomplex4() -> tuple<f32, f32>
+// PPC-LABEL: func @returncomplex4() -> tuple<f32, f32>
 // LOONGARCH64-LABEL: func @returncomplex4() -> tuple<f32, f32>
 func.func @returncomplex4() -> complex<f32> {
   // I32: fir.insert_value
@@ -41,6 +43,8 @@ func.func @returncomplex4() -> complex<f32> {
   // RISCV64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value
   // PPC64: fir.insert_value
   // PPC64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value
+  // PPC: fir.insert_value
+  // PPC: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value
   // LOONGARCH64: fir.insert_value
   // LOONGARCH64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value
   %1 = fir.undefined complex<f32>
@@ -94,6 +98,11 @@ func.func @returncomplex4() -> complex<f32> {
   // PPC64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
   // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f32, f32>>
   // PPC64: return [[RES]] : tuple<f32, f32>
+  // PPC: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC: return [[RES]] : tuple<f32, f32>
   // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
   // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
   return %6 : complex<f32>
@@ -113,6 +122,7 @@ func.func @returncomplex4() -> complex<f32> {
 // SPARCV9-LABEL: func @returncomplex8() -> tuple<f64, f64>
 // RISCV64-LABEL: func @returncomplex8() -> tuple<f64, f64>
 // PPC64-LABEL: func @returncomplex8() -> tuple<f64, f64>
+// PPC-LABEL: func @returncomplex8() -> tuple<f64, f64>
 // LOONGARCH64-LABEL: func @returncomplex8() -> tuple<f64, f64>
 func.func @returncomplex8() -> complex<f64> {
   // I32: fir.insert_value
@@ -133,6 +143,8 @@ func.func @returncomplex8() -> complex<f64> {
   // RISCV64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value {{.*}}
   // PPC64: fir.insert_value
   // PPC64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value {{.*}}
+  // PPC: fir.insert_value
+  // PPC: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value {{.*}}
   // LOONGARCH64: fir.insert_value
   // LOONGARCH64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value {{.*}}
   %1 = fir.undefined complex<f64>
@@ -182,6 +194,11 @@ func.func @returncomplex8() -> complex<f64> {
   // PPC64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
   // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
   // PPC64: return [[RES]] : tuple<f64, f64>
+  // PPC: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // PPC: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // PPC: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // PPC: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // PPC: return [[RES]] : tuple<f64, f64>
   // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
   // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
   // LOONGARCH64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
@@ -200,6 +217,7 @@ func.func @returncomplex8() -> complex<f64> {
 // SPARCV9-LABEL: func private @paramcomplex4(f32, f32)
 // RISCV64-LABEL: func private @paramcomplex4(f32, f32)
 // PPC64-LABEL: func private @paramcomplex4(f32, f32)
+// PPC-LABEL: func private @paramcomplex4(f32, f32)
 // LOONGARCH64-LABEL: func private @paramcomplex4(f32, f32)
 func.func private @paramcomplex4(complex<f32>) -> ()
 
@@ -213,6 +231,7 @@ func.func private @paramcomplex4(complex<f32>) -> ()
 // SPARCV9-LABEL: func @callcomplex4
 // RISCV64-LABEL: func @callcomplex4
 // PPC64-LABEL: func @callcomplex4
+// PPC-LABEL: func @callcomplex4
 // LOONGARCH64-LABEL: func @callcomplex4
 func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {
 
@@ -225,6 +244,7 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
   // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
   // RISCV64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
   // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
+  // PPC: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
   // LOONGARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
   %1 = fir.call @returncomplex4() : () -> complex<f32>
 
@@ -308,6 +328,14 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
   // PPC64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
   // PPC64: fir.call @paramcomplex4([[A]], [[B]]) : (f32, f32) -> ()
 
+  // PPC: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // PPC: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // PPC: fir.call @paramcomplex4([[A]], [[B]]) : (f32, f32) -> ()
+
   // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
   // LOONGARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
   // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
@@ -325,6 +353,7 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
   // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<f32, f32> {pass_arg_pos = 0 : i32}
   // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<f32, f32> {pass_arg_pos = 0 : i32}
   // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<f32, f32> {pass_arg_pos = 0 : i32}
+  // PPC: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<f32, f32> {pass_arg_pos = 0 : i32}
   %2 = fir.dispatch "ret_complex"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> complex<f32> {pass_arg_pos = 0 : i32}
 
   // I32: [[ADDRI64:%[0-9A-Za-z]+]] = fir.alloca i64
@@ -400,6 +429,14 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
   // PPC64: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[A]], [[B]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, f32, f32) {pass_arg_pos = 0 : i32}
   fir.dispatch "with_complex"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%arg0, %2 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, complex<f32>) {pass_arg_pos = 0 : i32}
 
+  // PPC: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // PPC: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // PPC: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[A]], [[B]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, f32, f32) {pass_arg_pos = 0 : i32}
+  fir.dispatch "with_complex"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%arg0, %2 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, complex<f32>) {pass_arg_pos = 0 : i32}
 
   // I32: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.ref<tuple<f32, f32>>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
   // I32_MINGW: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.ref<tuple<f32, f32>>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
@@ -410,6 +447,7 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
   // SPARCV9: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
   // RISCV64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
   // PPC64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
+  // PPC: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
   // LOONGARCH64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
   fir.dispatch "with_complex2"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%2, %arg0 : complex<f32>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
 
@@ -426,6 +464,7 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
 // SPARCV9-LABEL: func private @paramcomplex8(f64, f64)
 // RISCV64-LABEL: func private @paramcomplex8(f64, f64)
 // PPC64-LABEL: func private @paramcomplex8(f64, f64)
+// PPC-LABEL: func private @paramcomplex8(f64, f64)
 // LOONGARCH64-LABEL: func private @paramcomplex8(f64, f64)
 func.func private @paramcomplex8(complex<f64>) -> ()
 
@@ -439,6 +478,7 @@ func.func private @paramcomplex8(complex<f64>) -> ()
 // SPARCV9-LABEL: func @callcomplex8()
 // RISCV64-LABEL: func @callcomplex8()
 // PPC64-LABEL: func @callcomplex8()
+// PPC-LABEL: func @callcomplex8()
 // LOONGARCH64-LABEL: func @callcomplex8()
 func.func @callcomplex8() {
   // I32: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
@@ -453,6 +493,7 @@ func.func @callcomplex8() {
   // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
   // RISCV64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
   // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
+  // PPC: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
   // LOONGARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
   %1 = fir.call @returncomplex8() : () -> complex<f64>
 
@@ -527,6 +568,14 @@ func.func @callcomplex8() {
   // PPC64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f64>) -> f64
   // PPC64: fir.call @paramcomplex8([[A]], [[B]]) : (f64, f64) -> ()
 
+  // PPC: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // PPC: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // PPC: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // PPC: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f64>>
+  // PPC: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f64>) -> f64
+  // PPC: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f64>) -> f64
+  // PPC: fir.call @paramcomplex8([[A]], [[B]]) : (f64, f64) -> ()
+
   // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
   // LOONGARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
   // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
@@ -549,6 +598,7 @@ func.func @callcomplex8() {
 // SPARCV9-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
 // RISCV64-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
 // PPC64-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
+// PPC-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
 // LOONGARCH64-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
 func.func private @calleemultipleparamscomplex4(complex<f32>, complex<f32>, complex<f32>) -> ()
 
@@ -570,6 +620,8 @@ func.func private @calleemultipleparamscomplex4(complex<f32>, complex<f32>, comp
 // RISCV64-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
 // PPC64-LABEL: func @multipleparamscomplex4
 // PPC64-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
+// PPC-LABEL: func @multipleparamscomplex4
+// PPC-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
 // LOONGARCH64-LABEL: func @multipleparamscomplex4
 // LOONGARCH64-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
 func.func @multipleparamscomplex4(%z1 : complex<f32>, %z2 : complex<f32>, %z3 : complex<f32>) {
@@ -751,6 +803,7 @@ func.func @multipleparamscomplex4(%z1 : complex<f32>, %z2 : complex<f32>, %z3 :
   // RISCV64-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (complex<f32>) -> f32
 
   // RISCV64: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (f32, f32, f32, f32, f32, f32) -> ()
+
   // PPC64-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
   // PPC64-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
   // PPC64-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
@@ -769,6 +822,26 @@ func.func @multipleparamscomplex4(%z1 : complex<f32>, %z2 : complex<f32>, %z3 :
   // PPC64-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (complex<f32>) -> f32
 
   // PPC64: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (f32, f32, f32, f32, f32, f32) -> ()
+
+  // PPC-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+
+  // PPC-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (complex<f32>) -> f32
+  // PPC-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (complex<f32>) -> f32
+  // PPC-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (complex<f32>) -> f32
+  // PPC-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (complex<f32>) -> f32
+  // PPC-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (complex<f32>) -> f32
+  // PPC-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (complex<f32>) -> f32
+
+  // PPC: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (f32, f32, f32, f32, f32, f32) -> ()
+
   // LOONGARCH64-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
   // LOONGARCH64-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
   // LOONGARCH64-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
@@ -821,6 +894,9 @@ func.func @multipleparamscomplex4(%z1 : complex<f32>, %z2 : complex<f32>, %z3 :
 // PPC64-LABEL: func private @mlircomplexf32
 // PPC64-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32)
 // PPC64-SAME: -> tuple<f32, f32>
+// PPC-LABEL: func private @mlircomplexf32
+// PPC-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32)
+// PPC-SAME: -> tuple<f32, f32>
 // LOONGARCH64-LABEL: func private @mlircomplexf32
 // LOONGARCH64-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32)
 // LOONGARCH64-SAME: -> tuple<f32, f32>
@@ -970,6 +1046,20 @@ func.func private @mlircomplexf32(%z1: complex<f32>, %z2: complex<f32>) -> compl
 
   // PPC64: [[VAL:%[0-9A-Za-z]+]] = fir.call @mlircomplexf32([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]]) : (f32, f32, f32, f32) -> tuple<f32, f32>
 
+  // PPC-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+
+  // PPC-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (complex<f32>) -> f32
+  // PPC-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (complex<f32>) -> f32
+  // PPC-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (complex<f32>) -> f32
+  // PPC-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (complex<f32>) -> f32
+
+  // PPC: [[VAL:%[0-9A-Za-z]+]] = fir.call @mlircomplexf32([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]]) : (f32, f32, f32, f32) -> tuple<f32, f32>
+
   // LOONGARCH64-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
   // LOONGARCH64-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
   // LOONGARCH64-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
@@ -1075,6 +1165,16 @@ func.func private @mlircomplexf32(%z1: complex<f32>, %z2: complex<f32>) -> compl
   // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT_2]] : !fir.ref<tuple<f32, f32>>
   // PPC64: return [[RES]] : tuple<f32, f32>
 
+  // PPC: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC: fir.store [[VAL]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC: [[ADDRT_2:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC: [[ADDRC_2:%[0-9A-Za-z]+]] = fir.convert [[ADDRT_2]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC: fir.store [[V]] to [[ADDRC_2]] : !fir.ref<complex<f32>>
+  // PPC: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT_2]] : !fir.ref<tuple<f32, f32>>
+  // PPC: return [[RES]] : tuple<f32, f32>
+
   // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
   // LOONGARCH64: fir.store [[VAL]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
   // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
@@ -1097,6 +1197,7 @@ func.func private @mlircomplexf32(%z1: complex<f32>, %z2: complex<f32>) -> compl
 // SPARCV9-LABEL: func @addrof()
 // RISCV64-LABEL: func @addrof()
 // PPC64-LABEL: func @addrof()
+// PPC-LABEL: func @addrof()
 // LOONGARCH64-LABEL: func @addrof()
 func.func @addrof() {
   // I32: {{%.*}} = fir.address_of(@returncomplex4) : () -> i64
@@ -1107,6 +1208,7 @@ func.func @addrof() {
   // PPC64le: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
   // RISCV64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
   // PPC64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
+  // PPC: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
   // LOONGARCH64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
   %r = fir.address_of(@returncomplex4) : () -> complex<f32>
 
@@ -1119,6 +1221,7 @@ func.func @addrof() {
   // SPARCV9: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
   // RISCV64: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
   // PPC64: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
+  // PPC: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
   // LOONGARCH64: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
   %p = fir.address_of(@paramcomplex4) : (complex<f32>) -> ()
   return

From 206321b47a877e503dc8bfec7037566da61157fc Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 17 Apr 2025 11:31:31 -0700
Subject: [PATCH 305/710] Reapply "[NFC][CFI] Avoid clang error in CFI tests"
 (#136030) (#136177)

PR #136030 is expected to work after #136032.

This reverts commit 6b0c8c4acd31eba83bf20ab1cf8729b2153e383c.
---
 clang/test/Driver/sanitizer-ld.c | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index 0ace06bac3b31..4239c52a5fdcc 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -840,7 +840,8 @@
 // CHECK-CFI-PREREQ-LINUX: '-fsanitize=cfi' only allowed with '-fvisibility='
 
 // CFI by itself does not link runtime libraries.
-// RUN: not %clang -fsanitize=cfi \
+// RUN: %clang -fsanitize=cfi \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -849,7 +850,8 @@
 // CHECK-CFI-LINUX: "{{.*}}ld{{(.exe)?}}"
 
 // CFI with diagnostics links the UBSan runtime.
-// RUN: not %clang -fsanitize=cfi -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
+// RUN: %clang -fsanitize=cfi -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -859,7 +861,8 @@
 // CHECK-CFI-DIAG-LINUX: "--whole-archive" "{{[^"]*}}libclang_rt.ubsan_standalone.a" "--no-whole-archive"
 
 // Cross-DSO CFI links the CFI runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
@@ -870,7 +873,8 @@
 // CHECK-CFI-CROSS-DSO-LINUX: -export-dynamic
 
 // Cross-DSO CFI with diagnostics links just the CFI runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=x86_64-unknown-linux -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
@@ -882,7 +886,8 @@
 // CHECK-CFI-CROSS-DSO-DIAG-LINUX: -export-dynamic
 
 // Cross-DSO CFI on Android does not link runtime libraries.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     --target=aarch64-linux-android -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree \
@@ -891,7 +896,8 @@
 // CHECK-CFI-CROSS-DSO-ANDROID: "{{.*}}ld{{(.exe)?}}"
 
 // Cross-DSO CFI with diagnostics on Android links just the UBSAN runtime.
-// RUN: not %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN: %clang -fsanitize=cfi -fsanitize-cfi-cross-dso \
+// RUN:     -flto -fvisibility=hidden \
 // RUN:     -fno-sanitize-trap=cfi -fsanitize-recover=cfi \
 // RUN:     --target=aarch64-linux-android -fuse-ld=ld \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \

From 9d5f16308a2a1d778b9022c3a06a3cecc6d5e066 Mon Sep 17 00:00:00 2001
From: Felipe de Azevedo Piovezan <fpiovezan@apple.com>
Date: Thu, 17 Apr 2025 11:33:07 -0700
Subject: [PATCH 306/710] [lldb][nfc] Factor out code from ThreadPlanStepOut
 ctor (#136159)

A future patch will need to create a new constructor for this class, and
extracting code out of its sole existing constructor will make this
easier.

This commit creates a helper function for the code computing the target
frame to step out to.
---
 lldb/source/Target/ThreadPlanStepOut.cpp | 47 +++++++++++++++---------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/lldb/source/Target/ThreadPlanStepOut.cpp b/lldb/source/Target/ThreadPlanStepOut.cpp
index a05c46db6b8ca..26c1abe710319 100644
--- a/lldb/source/Target/ThreadPlanStepOut.cpp
+++ b/lldb/source/Target/ThreadPlanStepOut.cpp
@@ -31,6 +31,32 @@ using namespace lldb_private;
 
 uint32_t ThreadPlanStepOut::s_default_flag_values = 0;
 
+/// Computes the target frame this plan should step out to.
+static StackFrameSP
+ComputeTargetFrame(Thread &thread, uint32_t start_frame_idx,
+                   std::vector<StackFrameSP> &skipped_frames) {
+  uint32_t frame_idx = start_frame_idx + 1;
+  StackFrameSP return_frame_sp = thread.GetStackFrameAtIndex(frame_idx);
+  if (!return_frame_sp)
+    return nullptr;
+
+  while (return_frame_sp->IsArtificial() || return_frame_sp->IsHidden()) {
+    skipped_frames.push_back(return_frame_sp);
+
+    frame_idx++;
+    return_frame_sp = thread.GetStackFrameAtIndex(frame_idx);
+
+    // We never expect to see an artificial frame without a regular ancestor.
+    // Defensively refuse to step out.
+    if (!return_frame_sp) {
+      LLDB_LOG(GetLog(LLDBLog::Step),
+               "Can't step out of frame with artificial ancestors");
+      return nullptr;
+    }
+  }
+  return return_frame_sp;
+}
+
 // ThreadPlanStepOut: Step out of the current frame
 ThreadPlanStepOut::ThreadPlanStepOut(
     Thread &thread, SymbolContext *context, bool first_insn, bool stop_others,
@@ -44,34 +70,18 @@ ThreadPlanStepOut::ThreadPlanStepOut(
       m_return_addr(LLDB_INVALID_ADDRESS), m_stop_others(stop_others),
       m_immediate_step_from_function(nullptr),
       m_calculate_return_value(gather_return_value) {
-  Log *log = GetLog(LLDBLog::Step);
   SetFlagsToDefault();
   SetupAvoidNoDebug(step_out_avoids_code_without_debug_info);
 
   m_step_from_insn = thread.GetRegisterContext()->GetPC(0);
 
-  uint32_t return_frame_index = frame_idx + 1;
-  StackFrameSP return_frame_sp(thread.GetStackFrameAtIndex(return_frame_index));
+  StackFrameSP return_frame_sp =
+      ComputeTargetFrame(thread, frame_idx, m_stepped_past_frames);
   StackFrameSP immediate_return_from_sp(thread.GetStackFrameAtIndex(frame_idx));
 
   if (!return_frame_sp || !immediate_return_from_sp)
     return; // we can't do anything here.  ValidatePlan() will return false.
 
-  // While stepping out, behave as-if artificial frames are not present.
-  while (return_frame_sp->IsArtificial() || return_frame_sp->IsHidden()) {
-    m_stepped_past_frames.push_back(return_frame_sp);
-
-    ++return_frame_index;
-    return_frame_sp = thread.GetStackFrameAtIndex(return_frame_index);
-
-    // We never expect to see an artificial frame without a regular ancestor.
-    // If this happens, log the issue and defensively refuse to step out.
-    if (!return_frame_sp) {
-      LLDB_LOG(log, "Can't step out of frame with artificial ancestors");
-      return;
-    }
-  }
-
   m_step_out_to_id = return_frame_sp->GetStackID();
   m_immediate_step_from_id = immediate_return_from_sp->GetStackID();
 
@@ -125,6 +135,7 @@ ThreadPlanStepOut::ThreadPlanStepOut(
 
     // Perform some additional validation on the return address.
     uint32_t permissions = 0;
+    Log *log = GetLog(LLDBLog::Step);
     if (!m_process.GetLoadAddressPermissions(m_return_addr, permissions)) {
       LLDB_LOGF(log, "ThreadPlanStepOut(%p): Return address (0x%" PRIx64
                 ") permissions not found.", static_cast<void *>(this),

From 09069088ddf8cf0ae28dce7efe181d63cb132f3a Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Thu, 17 Apr 2025 11:44:58 -0700
Subject: [PATCH 307/710] [NVPTX] Add auto-upgrade rules for fabs.{f,d,ftz.f}
 (#136150)

These auto-upgrade rules are required after these intrinsics were
removed in #135644
---
 llvm/lib/IR/AutoUpgrade.cpp                       |  7 +++++++
 .../Assembler/auto_upgrade_nvvm_intrinsics.ll     | 15 +++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 7a91da5f9f7eb..2e17b8ce5eb7b 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1287,6 +1287,9 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
         // nvvm.abs.{i,ii}
         Expand =
             Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
+      else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d")
+        // nvvm.fabs.{f,ftz.f,d}
+        Expand = true;
       else if (Name == "clz.ll" || Name == "popc.ll" || Name == "h2f" ||
                Name == "swap.lo.hi.b64")
         Expand = true;
@@ -2318,6 +2321,10 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
     Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
     Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
     Rep = Builder.CreateBitCast(Abs, CI->getType());
+  } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
+    Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
+                                               : Intrinsic::nvvm_fabs;
+    Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
   } else if (Name.starts_with("atomic.load.add.f32.p") ||
              Name.starts_with("atomic.load.add.f64.p")) {
     Value *Ptr = CI->getArgOperand(0);
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 74b9640df6977..a3ddc62dd34e9 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -13,6 +13,10 @@ declare float @llvm.nvvm.h2f(i16)
 declare i32 @llvm.nvvm.abs.i(i32)
 declare i64 @llvm.nvvm.abs.ll(i64)
 
+declare float @llvm.nvvm.fabs.f(float)
+declare float @llvm.nvvm.fabs.ftz.f(float)
+declare double @llvm.nvvm.fabs.d(double)
+
 declare i16 @llvm.nvvm.max.s(i16, i16)
 declare i32 @llvm.nvvm.max.i(i32, i32)
 declare i64 @llvm.nvvm.max.ll(i64, i64)
@@ -97,6 +101,17 @@ define void @abs(i32 %a, i64 %b) {
   ret void
 }
 
+; CHECK-LABEL: @fabs
+define void @fabs(float %a, double %b) {
+; CHECK: call float @llvm.nvvm.fabs.f32(float %a)
+; CHECK: call float @llvm.nvvm.fabs.ftz.f32(float %a)
+; CHECK: call double @llvm.nvvm.fabs.f64(double %b)
+  %r1 = call float @llvm.nvvm.fabs.f(float %a)
+  %r2 = call float @llvm.nvvm.fabs.ftz.f(float %a)
+  %r3 = call double @llvm.nvvm.fabs.d(double %b)
+  ret void
+}
+
 ; CHECK-LABEL: @min_max
 define void @min_max(i16 %a1, i16 %a2, i32 %b1, i32 %b2, i64 %c1, i64 %c2) {
 ; CHECK: [[maxs:%[a-zA-Z0-9.]+]] = icmp sge i16 %a1, %a2

From da959c92c5a46abd59d38d26d5e0758d48fb7f65 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 17 Apr 2025 11:46:06 -0700
Subject: [PATCH 308/710] [flang] Fixed out-of-bounds access in
 SimplifyIntrinsics. (#136171)

When the mask is scalar, it is incorrect to cast it to
!fir.box<!fir.array<1xlogical<>>>, because the coordinate
operation will try to read the dim-1 stride from the box
to get the address of the first element. Even though
the stride value will be multiplied by 0, and does not matter,
it is still a read past the allocated box object.
Instead, we should just use box_addr to get the address
of the scalar mask.
---
 .../lib/Optimizer/Transforms/SimplifyIntrinsics.cpp  | 10 +---------
 flang/test/Transforms/simplifyintrinsics.fir         | 12 ++++--------
 2 files changed, 5 insertions(+), 17 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
index dd4d1783dac37..4d25a02bf18ba 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
@@ -823,17 +823,9 @@ static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
   if (maskRank == 0) {
     mlir::Type i1Type = builder.getI1Type();
     mlir::Type logical = maskElemType;
-    mlir::IndexType idxTy = builder.getIndexType();
-
-    fir::SequenceType::Shape singleElement(1, 1);
-    mlir::Type arrTy = fir::SequenceType::get(singleElement, logical);
-    mlir::Type boxArrTy = fir::BoxType::get(arrTy);
-    mlir::Value array = builder.create<fir::ConvertOp>(loc, boxArrTy, mask);
-
-    mlir::Value indx = builder.createIntegerConstant(loc, idxTy, 0);
     mlir::Type logicalRefTy = builder.getRefType(logical);
     mlir::Value condAddr =
-        builder.create<fir::CoordinateOp>(loc, logicalRefTy, array, indx);
+        builder.create<fir::BoxAddrOp>(loc, logicalRefTy, mask);
     mlir::Value cond = builder.create<fir::LoadOp>(loc, condAddr);
     mlir::Value condI1 = builder.create<fir::ConvertOp>(loc, i1Type, cond);
 
diff --git a/flang/test/Transforms/simplifyintrinsics.fir b/flang/test/Transforms/simplifyintrinsics.fir
index 1a66e077165b0..1cfda08e39694 100644
--- a/flang/test/Transforms/simplifyintrinsics.fir
+++ b/flang/test/Transforms/simplifyintrinsics.fir
@@ -1996,10 +1996,8 @@ func.func @_QPtestminloc_works1d_scalarmask_f64(%arg0: !fir.ref<!fir.array<10xf6
 // CHECK:           %[[OUTARR_IDX0:.*]] = arith.constant 0 : index
 // CHECK:           %[[OUTARR_ITEM0:.*]] = fir.coordinate_of %[[BOX_OUTARR]], %[[OUTARR_IDX0]] : (!fir.box<!fir.heap<!fir.array<1xi32>>>, index) -> !fir.ref<i32>
 // CHECK:           fir.store %[[INIT_OUT_IDX]] to %[[OUTARR_ITEM0]] : !fir.ref<i32>
-// CHECK:           %[[BOX_MASK:.*]] = fir.convert %[[BOX_MASK_NONE]] : (!fir.box<none>) -> !fir.box<!fir.array<1x!fir.logical<4>>>
-// CHECK:           %[[MASK_IDX0:.*]] = arith.constant 0 : index
-// CHECK:           %[[MASK_ITEM:.*]] = fir.coordinate_of %[[BOX_MASK]], %[[MASK_IDX0]] : (!fir.box<!fir.array<1x!fir.logical<4>>>, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:           %[[MASK_LOGICAL:.*]] = fir.load %[[MASK_ITEM]] : !fir.ref<!fir.logical<4>>
+// CHECK:           %[[MASK_ADDR:.*]] = fir.box_addr %[[BOX_MASK_NONE]] : (!fir.box<none>) -> !fir.ref<!fir.logical<4>>
+// CHECK:           %[[MASK_LOGICAL:.*]] = fir.load %[[MASK_ADDR]] : !fir.ref<!fir.logical<4>>
 // CHECK:           %[[MASK:.*]] = fir.convert %[[MASK_LOGICAL]] : (!fir.logical<4>) -> i1
 // CHECK:           %[[INIT_RES:.*]] = fir.if %[[MASK]] -> (f64) {
 // CHECK:             %[[C_INDEX0:.*]] = arith.constant 0 : index
@@ -2574,10 +2572,8 @@ func.func @_QPtestmaxloc_works1d_scalarmask_f64(%arg0: !fir.ref<!fir.array<10xf6
 // CHECK:           %[[OUTARR_IDX0:.*]] = arith.constant 0 : index
 // CHECK:           %[[OUTARR_ITEM0:.*]] = fir.coordinate_of %[[BOX_OUTARR]], %[[OUTARR_IDX0]] : (!fir.box<!fir.heap<!fir.array<1xi32>>>, index) -> !fir.ref<i32>
 // CHECK:           fir.store %[[INIT_OUT_IDX]] to %[[OUTARR_ITEM0]] : !fir.ref<i32>
-// CHECK:           %[[BOX_MASK:.*]] = fir.convert %[[BOX_MASK_NONE]] : (!fir.box<none>) -> !fir.box<!fir.array<1x!fir.logical<4>>>
-// CHECK:           %[[MASK_IDX0:.*]] = arith.constant 0 : index
-// CHECK:           %[[MASK_ITEM:.*]] = fir.coordinate_of %[[BOX_MASK]], %[[MASK_IDX0]] : (!fir.box<!fir.array<1x!fir.logical<4>>>, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:           %[[MASK_LOGICAL:.*]] = fir.load %[[MASK_ITEM]] : !fir.ref<!fir.logical<4>>
+// CHECK:           %[[MASK_ADDR:.*]] = fir.box_addr %[[BOX_MASK_NONE]] : (!fir.box<none>) -> !fir.ref<!fir.logical<4>>
+// CHECK:           %[[MASK_LOGICAL:.*]] = fir.load %[[MASK_ADDR]] : !fir.ref<!fir.logical<4>>
 // CHECK:           %[[MASK:.*]] = fir.convert %[[MASK_LOGICAL]] : (!fir.logical<4>) -> i1
 // CHECK:           %[[INIT_RES:.*]] = fir.if %[[MASK]] -> (f64) {
 // CHECK:             %[[C_INDEX0:.*]] = arith.constant 0 : index

From 06963fb9c79ae012de5796b73bb810f8da09728e Mon Sep 17 00:00:00 2001
From: Brad Smith <brad@comstyle.com>
Date: Thu, 17 Apr 2025 15:11:33 -0400
Subject: [PATCH 309/710] [OpenMP] Disable load balance on Haiku (#136082)

Haiku does not have a means of retrieving the desired information
and the -1 setting causes the code to fallback anyway.
---
 openmp/runtime/src/kmp_config.h.cmake | 2 +-
 openmp/runtime/src/z_Linux_util.cpp   | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/openmp/runtime/src/kmp_config.h.cmake b/openmp/runtime/src/kmp_config.h.cmake
index b0cd0ed296e7f..d64c9a4b557df 100644
--- a/openmp/runtime/src/kmp_config.h.cmake
+++ b/openmp/runtime/src/kmp_config.h.cmake
@@ -120,7 +120,7 @@
 #define KMP_ASM_INTRINS 1
 #define USE_ITT_BUILD LIBOMP_USE_ITT_NOTIFY
 #define INTEL_ITTNOTIFY_PREFIX __kmp_itt_
-#if ! KMP_MIC
+#if ! (KMP_MIC || KMP_OS_HAIKU)
 # define USE_LOAD_BALANCE 1
 #endif
 #if ! (KMP_OS_WINDOWS || KMP_OS_DARWIN)
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 87f2711186420..3b82b45e24405 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -2522,10 +2522,6 @@ int __kmp_get_load_balance(int max) {
   return running_threads;
 }
 
-#elif KMP_OS_HAIKU
-
-int __kmp_get_load_balance(int max) { return -1; }
-
 #else // Linux* OS
 
 // The function returns number of running (not sleeping) threads, or -1 in case

From 1042d9988764e16e5f6fd984d362042b2fadd0c6 Mon Sep 17 00:00:00 2001
From: Shubham Sandeep Rastogi <srastogi22@apple.com>
Date: Thu, 17 Apr 2025 12:33:30 -0700
Subject: [PATCH 310/710] disable test on older compilers (#136186)

---
 .../TestDataFormatterLibcxxInvalidVectorSimulator.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
index 3f58018b0fbd9..1a23d9a19fe19 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/TestDataFormatterLibcxxInvalidVectorSimulator.py
@@ -14,7 +14,7 @@ class LibcxxInvalidVectorDataFormatterSimulatorTestCase(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
 
-    @skipIf(compiler="clang", compiler_version=['<', '15.0.1'])
+    @skipIf(compiler="clang", compiler_version=['<', '18.0'])
     def test(self):
         self.build()
         lldbutil.run_to_source_breakpoint(self, "return 0", lldb.SBFileSpec("main.cpp"))

From 85110ccee90380a9d2bc930fae8fb0fa395d58d6 Mon Sep 17 00:00:00 2001
From: Alexander Kornienko <alexfh@google.com>
Date: Thu, 17 Apr 2025 21:38:18 +0200
Subject: [PATCH 311/710] [SLP] Replace most uses of for_each with range-for
 loops. NFC (#136146)

This removes a bit of complexity from the code, where it doesn't seem to
be justified.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 95 +++++++++----------
 1 file changed, 47 insertions(+), 48 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d496989cd0581..13c1eb572ef92 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14593,13 +14593,13 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals,
       };
       if (!ValueToExtUses) {
         ValueToExtUses.emplace();
-        for_each(enumerate(ExternalUses), [&](const auto &P) {
+        for (const auto &P : enumerate(ExternalUses)) {
           // Ignore phis in loops.
           if (IsPhiInLoop(P.value()))
-            return;
+            continue;
 
           ValueToExtUses->try_emplace(P.value().Scalar, P.index());
-        });
+        }
       }
       // Can use original instruction, if no operands vectorized or they are
       // marked as externally used already.
@@ -14677,13 +14677,13 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals,
         }
         if (KeepScalar) {
           ExternalUsesAsOriginalScalar.insert(EU.Scalar);
-          for_each(Inst->operands(), [&](Value *V) {
+          for (Value *V : Inst->operands()) {
             auto It = ValueToExtUses->find(V);
             if (It != ValueToExtUses->end()) {
               // Replace all uses to avoid compiler crash.
               ExternalUses[It->second].User = nullptr;
             }
-          });
+          }
           ExtraCost = ScalarCost;
           if (!IsPhiInLoop(EU))
             ExtractsCount[Entry].insert(Inst);
@@ -14692,13 +14692,13 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals,
             // Update the users of the operands of the cast operand to avoid
             // compiler crash.
             if (auto *IOp = dyn_cast<Instruction>(Inst->getOperand(0))) {
-              for_each(IOp->operands(), [&](Value *V) {
+              for (Value *V : IOp->operands()) {
                 auto It = ValueToExtUses->find(V);
                 if (It != ValueToExtUses->end()) {
                   // Replace all uses to avoid compiler crash.
                   ExternalUses[It->second].User = nullptr;
                 }
-              });
+              }
             }
           }
         }
@@ -15334,7 +15334,8 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
     // tree.
     Entries.push_back(FirstEntries.front());
     // Update mapping between values and corresponding tree entries.
-    for_each(UsedValuesEntry, [&](auto &P) { P.second = 0; });
+    for (auto &P : UsedValuesEntry)
+      P.second = 0;
     VF = FirstEntries.front()->getVectorFactor();
   } else {
     // Try to find nodes with the same vector factor.
@@ -15384,13 +15385,13 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
       ValuesToEntries.emplace_back().insert(E->Scalars.begin(),
                                             E->Scalars.end());
     // Update mapping between values and corresponding tree entries.
-    for_each(UsedValuesEntry, [&](auto &P) {
+    for (auto &P : UsedValuesEntry) {
       for (unsigned Idx : seq<unsigned>(ValuesToEntries.size()))
         if (ValuesToEntries[Idx].contains(P.first)) {
           P.second = Idx;
           break;
         }
-    });
+    }
   }
 
   bool IsSplatOrUndefs = isSplat(VL) || all_of(VL, IsaPred<UndefValue>);
@@ -15536,12 +15537,12 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
                                                  (MaxElement % VF) -
                                                      (MinElement % VF) + 1));
     if (NewVF < VF) {
-      for_each(SubMask, [&](int &Idx) {
+      for (int &Idx : SubMask) {
         if (Idx == PoisonMaskElem)
-          return;
+          continue;
         Idx = ((Idx % VF) - (((MinElement % VF) / NewVF) * NewVF)) % NewVF +
               (Idx >= static_cast<int>(VF) ? NewVF : 0);
-      });
+      }
     } else {
       NewVF = VF;
     }
@@ -19306,8 +19307,10 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
     // whole bundle might not be ready.
     ReadyInsts.remove(BundleMember);
     if (ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(V);
-        !Bundles.empty())
-      for_each(Bundles, [&](ScheduleBundle *B) { ReadyInsts.remove(B); });
+        !Bundles.empty()) {
+      for (ScheduleBundle *B : Bundles)
+        ReadyInsts.remove(B);
+    }
 
     if (!BundleMember->isScheduled())
       continue;
@@ -19632,23 +19635,23 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleBundle &Bundle,
       }
       continue;
     }
-    for_each(Bundles, [&](ScheduleBundle *Bundle) {
+    for (ScheduleBundle *Bundle : Bundles) {
       if (!Visited.insert(Bundle).second || Bundle->hasValidDependencies())
-        return;
+        continue;
       assert(isInSchedulingRegion(*Bundle) &&
              "ScheduleData not in scheduling region");
       for_each(Bundle->getBundle(), ProcessNode);
-    });
+    }
     if (InsertInReadyList && SD->isReady()) {
-      for_each(Bundles, [&](ScheduleBundle *Bundle) {
+      for (ScheduleBundle *Bundle : Bundles) {
         assert(isInSchedulingRegion(*Bundle) &&
                "ScheduleData not in scheduling region");
         if (!Bundle->isReady())
-          return;
+          continue;
         ReadyInsts.insert(Bundle);
         LLVM_DEBUG(dbgs() << "SLP:     gets ready on update: " << *Bundle
                           << "\n");
-      });
+      }
     }
   }
 }
@@ -20032,8 +20035,8 @@ bool BoUpSLP::collectValuesToDemote(
         if (Operands.empty()) {
           if (!IsTruncRoot)
             MaxDepthLevel = 1;
-          (void)for_each(E.Scalars, std::bind(IsPotentiallyTruncated, _1,
-                                              std::ref(BitWidth)));
+          for (Value *V : E.Scalars)
+            (void)IsPotentiallyTruncated(V, BitWidth);
         } else {
           // Several vectorized uses? Check if we can truncate it, otherwise -
           // exit.
@@ -21034,17 +21037,16 @@ bool SLPVectorizerPass::vectorizeStores(
       unsigned Sz = 1 + Log2_32(MaxVF) - Log2_32(MinVF);
       SmallVector<unsigned> CandidateVFs(Sz + (NonPowerOf2VF > 0 ? 1 : 0));
       unsigned Size = MinVF;
-      for_each(reverse(CandidateVFs), [&](unsigned &VF) {
+      for (unsigned &VF : reverse(CandidateVFs)) {
         VF = Size > MaxVF ? NonPowerOf2VF : Size;
         Size *= 2;
-      });
+      }
       unsigned End = Operands.size();
       unsigned Repeat = 0;
       constexpr unsigned MaxAttempts = 4;
       OwningArrayRef<std::pair<unsigned, unsigned>> RangeSizes(Operands.size());
-      for_each(RangeSizes, [](std::pair<unsigned, unsigned> &P) {
+      for (std::pair<unsigned, unsigned> &P : RangeSizes)
         P.first = P.second = 1;
-      });
       DenseMap<Value *, std::pair<unsigned, unsigned>> NonSchedulable;
       auto IsNotVectorized = [](bool First,
                                 const std::pair<unsigned, unsigned> &P) {
@@ -21120,22 +21122,19 @@ bool SLPVectorizerPass::vectorizeStores(
                 AnyProfitableGraph = RepeatChanged = Changed = true;
                 // If we vectorized initial block, no need to try to vectorize
                 // it again.
-                for_each(RangeSizes.slice(Cnt, Size),
-                         [](std::pair<unsigned, unsigned> &P) {
-                           P.first = P.second = 0;
-                         });
+                for (std::pair<unsigned, unsigned> &P :
+                     RangeSizes.slice(Cnt, Size))
+                  P.first = P.second = 0;
                 if (Cnt < StartIdx + MinVF) {
-                  for_each(RangeSizes.slice(StartIdx, Cnt - StartIdx),
-                           [](std::pair<unsigned, unsigned> &P) {
-                             P.first = P.second = 0;
-                           });
+                  for (std::pair<unsigned, unsigned> &P :
+                       RangeSizes.slice(StartIdx, Cnt - StartIdx))
+                    P.first = P.second = 0;
                   StartIdx = Cnt + Size;
                 }
                 if (Cnt > Sz - Size - MinVF) {
-                  for_each(RangeSizes.slice(Cnt + Size, Sz - (Cnt + Size)),
-                           [](std::pair<unsigned, unsigned> &P) {
-                             P.first = P.second = 0;
-                           });
+                  for (std::pair<unsigned, unsigned> &P :
+                       RangeSizes.slice(Cnt + Size, Sz - (Cnt + Size)))
+                    P.first = P.second = 0;
                   if (Sz == End)
                     End = Cnt;
                   Sz = Cnt;
@@ -21161,13 +21160,13 @@ bool SLPVectorizerPass::vectorizeStores(
                 continue;
               }
               if (TreeSize > 1)
-                for_each(RangeSizes.slice(Cnt, Size),
-                         [&](std::pair<unsigned, unsigned> &P) {
-                           if (Size >= MaxRegVF)
-                             P.second = std::max(P.second, TreeSize);
-                           else
-                             P.first = std::max(P.first, TreeSize);
-                         });
+                for (std::pair<unsigned, unsigned> &P :
+                     RangeSizes.slice(Cnt, Size)) {
+                  if (Size >= MaxRegVF)
+                    P.second = std::max(P.second, TreeSize);
+                  else
+                    P.first = std::max(P.first, TreeSize);
+                }
               ++Cnt;
               AnyProfitableGraph = true;
             }
@@ -21209,10 +21208,10 @@ bool SLPVectorizerPass::vectorizeStores(
           CandidateVFs.push_back(Limit);
         if (VF > MaxTotalNum || VF >= StoresLimit)
           break;
-        for_each(RangeSizes, [&](std::pair<unsigned, unsigned> &P) {
+        for (std::pair<unsigned, unsigned> &P : RangeSizes) {
           if (P.first != 0)
             P.first = std::max(P.second, P.first);
-        });
+        }
         // Last attempt to vectorize max number of elements, if all previous
         // attempts were unsuccessful because of the cost issues.
         CandidateVFs.push_back(VF);

From b30100b87f24847afd6407b4939a184ebcf16ef9 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 17 Apr 2025 12:39:40 -0700
Subject: [PATCH 312/710] [RISCV] Check that both registers of a CV Reg-Reg
 memory address are GPRs. (#136079)

The assembly parser was checking for any register instead of GPR.

I've removed the custom diagnostic message from the RegReg operand to
give a less confusing error on bad input. The mnemonics are shared with
other encodings that don't use reg-reg memory operand.

I also fixed the parsed operand location, but I'm not sure it matters.
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 15 +++---
 llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td    |  2 -
 llvm/test/MC/RISCV/corev/XCVmem-invalid.s     | 52 +++++++++++--------
 3 files changed, 38 insertions(+), 31 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index f8df7ee4c5019..7e2821404ef95 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -2581,28 +2581,31 @@ ParseStatus RISCVAsmParser::parseRegReg(OperandVector &Operands) {
   if (getLexer().getKind() != AsmToken::Identifier)
     return ParseStatus::NoMatch;
 
+  SMLoc S = getLoc();
   StringRef OffsetRegName = getLexer().getTok().getIdentifier();
   MCRegister OffsetReg = matchRegisterNameHelper(OffsetRegName);
-  if (!OffsetReg)
-    return Error(getLoc(), "invalid register");
+  if (!OffsetReg ||
+      !RISCVMCRegisterClasses[RISCV::GPRRegClassID].contains(OffsetReg))
+    return Error(getLoc(), "expected GPR register");
   getLexer().Lex();
 
   if (parseToken(AsmToken::LParen, "expected '(' or invalid operand"))
     return ParseStatus::Failure;
 
   if (getLexer().getKind() != AsmToken::Identifier)
-    return Error(getLoc(), "expected register");
+    return Error(getLoc(), "expected GPR register");
 
   StringRef BaseRegName = getLexer().getTok().getIdentifier();
   MCRegister BaseReg = matchRegisterNameHelper(BaseRegName);
-  if (!BaseReg)
-    return Error(getLoc(), "invalid register");
+  if (!BaseReg ||
+      !RISCVMCRegisterClasses[RISCV::GPRRegClassID].contains(BaseReg))
+    return Error(getLoc(), "expected GPR register");
   getLexer().Lex();
 
   if (parseToken(AsmToken::RParen, "expected ')'"))
     return ParseStatus::Failure;
 
-  Operands.push_back(RISCVOperand::createRegReg(BaseReg, OffsetReg, getLoc()));
+  Operands.push_back(RISCVOperand::createRegReg(BaseReg, OffsetReg, S));
 
   return ParseStatus::Success;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index 5ce08d64b141d..984460df8a408 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -17,8 +17,6 @@
 def CVrrAsmOperand : AsmOperandClass {
   let Name = "RegReg";
   let ParserMethod = "parseRegReg";
-  let DiagnosticType = "InvalidRegReg";
-  let DiagnosticString = "operands must be register and register";
 }
 
 def CVrr : Operand<i32>,
diff --git a/llvm/test/MC/RISCV/corev/XCVmem-invalid.s b/llvm/test/MC/RISCV/corev/XCVmem-invalid.s
index f21f95310abc4..3f13bd0090d7f 100644
--- a/llvm/test/MC/RISCV/corev/XCVmem-invalid.s
+++ b/llvm/test/MC/RISCV/corev/XCVmem-invalid.s
@@ -2,7 +2,7 @@
 # RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
 
 cv.lb t0, (0), 0
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.lb 0, (t1), 0
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
@@ -17,7 +17,7 @@ cv.lb t0, (t1), 2048
 # CHECK-ERROR: :[[@LINE-1]]:17: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo specifier or an integer in the range [-2048, 2047]
 
 cv.lb t0, (0), t1
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.lb 0, (t1), t1
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
@@ -32,7 +32,7 @@ cv.lb t0, (t1), t2, t3
 # CHECK-ERROR: :[[@LINE-1]]:21: error: invalid operand for instruction
 
 cv.lbu t0, (0), 0
-# CHECK-ERROR: :[[@LINE-1]]:12: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:12: error: invalid operand for instruction
 
 cv.lbu 0, (t1), 0
 # CHECK-ERROR: :[[@LINE-1]]:8: error: invalid operand for instruction
@@ -47,7 +47,7 @@ cv.lbu t0, (t1), 2048
 # CHECK-ERROR: :[[@LINE-1]]:18: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo specifier or an integer in the range [-2048, 2047]
 
 cv.lbu t0, (0), t1
-# CHECK-ERROR: :[[@LINE-1]]:12: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:12: error: invalid operand for instruction
 
 cv.lbu 0, (t1), t1
 # CHECK-ERROR: :[[@LINE-1]]:8: error: invalid operand for instruction
@@ -62,7 +62,7 @@ cv.lbu t0, (t1), t2, t3
 # CHECK-ERROR: :[[@LINE-1]]:22: error: invalid operand for instruction
 
 cv.lh t0, (0), 0
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.lh 0, (t1), 0
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
@@ -77,10 +77,10 @@ cv.lh t0, (t1), 2048
 # CHECK-ERROR: :[[@LINE-1]]:17: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo specifier or an integer in the range [-2048, 2047]
 
 cv.lh t0, (0), t1
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.lh t0, t1(0)
-# CHECK-ERROR: :[[@LINE-1]]:14: error: expected register
+# CHECK-ERROR: :[[@LINE-1]]:14: error: expected GPR register
 
 cv.lh 0, (t1), t1
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
@@ -95,7 +95,7 @@ cv.lh t0, (t1), t2, t3
 # CHECK-ERROR: :[[@LINE-1]]:21: error: invalid operand for instruction
 
 cv.lhu t0, (0), 0
-# CHECK-ERROR: :[[@LINE-1]]:12: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:12: error: invalid operand for instruction
 
 cv.lhu 0, (t1), 0
 # CHECK-ERROR: :[[@LINE-1]]:8: error: invalid operand for instruction
@@ -110,10 +110,10 @@ cv.lhu t0, (t1), 2048
 # CHECK-ERROR: :[[@LINE-1]]:18: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo specifier or an integer in the range [-2048, 2047]
 
 cv.lhu t0, (0), t1
-# CHECK-ERROR: :[[@LINE-1]]:12: error: operands must be register and register 
+# CHECK-ERROR: :[[@LINE-1]]:12: error: invalid operand for instruction
 
 cv.lhu t0, t1(0)
-# CHECK-ERROR: :[[@LINE-1]]:15: error: expected register
+# CHECK-ERROR: :[[@LINE-1]]:15: error: expected GPR register
 
 cv.lhu 0, t0, t1
 # CHECK-ERROR: :[[@LINE-1]]:13: error: expected '(' or invalid operand
@@ -128,7 +128,7 @@ cv.lhu t0, (t1), t2, t3
 # CHECK-ERROR: :[[@LINE-1]]:22: error: invalid operand for instruction
 
 cv.lw t0, (0), 0
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.lw 0, (t1), 0
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
@@ -143,10 +143,10 @@ cv.lw t0, (t1), 2048
 # CHECK-ERROR: :[[@LINE-1]]:17: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo specifier or an integer in the range [-2048, 2047]
 
 cv.lw t0, (0), t1
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.lw t0, t1(0)
-# CHECK-ERROR: :[[@LINE-1]]:14: error: expected register
+# CHECK-ERROR: :[[@LINE-1]]:14: error: expected GPR register
 
 cv.lw 0, (t0), t1
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
@@ -158,22 +158,22 @@ cv.lw t0, (t1)
 # CHECK-ERROR: :[[@LINE-1]]:1: error: too few operands for instruction
 
 cv.lw t0, (t1), t2, t3
-# CHECK-ERROR: :[[@LINE-1]]:21: error: invalid operand for instruction 
+# CHECK-ERROR: :[[@LINE-1]]:21: error: invalid operand for instruction
 
 cv.sb t0, (0), 0
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sb 0, (t0), 0
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
 
 cv.sb t0, 0(t1)
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sb t0, (t1), 2048
 # CHECK-ERROR: :[[@LINE-1]]:17: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo specifier or an integer in the range [-2048, 2047]
 
 cv.sb t0, (0), t1
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sb 0, (t1), t1
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
@@ -182,19 +182,19 @@ cv.sb t0
 # CHECK-ERROR: :[[@LINE-1]]:1: error: too few operands for instruction
 
 cv.sh t0, (0), 0
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sh 0, (t1), 0
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
 
 cv.sh t0, 0(t1)
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sh t0, (t1), 2048
 # CHECK-ERROR: :[[@LINE-1]]:17: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo specifier or an integer in the range [-2048, 2047]
 
 cv.sh t0, (0), t1
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sh 0, (t1), t1
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
@@ -203,22 +203,28 @@ cv.sh t0
 # CHECK-ERROR: :[[@LINE-1]]:1: error: too few operands for instruction
 
 cv.sw t0, (0), 0
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sw 0, (t1), 0
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
 
 cv.sw t0, 0(t1)
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sw t0, (t1), 2048
 # CHECK-ERROR: :[[@LINE-1]]:17: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo specifier or an integer in the range [-2048, 2047]
 
 cv.sw t0, (0), t1
-# CHECK-ERROR: :[[@LINE-1]]:11: error: operands must be register and register
+# CHECK-ERROR: :[[@LINE-1]]:11: error: invalid operand for instruction
 
 cv.sw 0, (t1), t1
 # CHECK-ERROR: :[[@LINE-1]]:7: error: invalid operand for instruction
 
 cv.sw t0
 # CHECK-ERROR: :[[@LINE-1]]:1: error: too few operands for instruction
+
+cv.lb t0, f0(t1)
+# CHECK-ERROR: :[[@LINE-1]]:11: error: expected GPR register
+
+cv.sb t0, t0(f1)
+# CHECK-ERROR: :[[@LINE-1]]:14: error: expected GPR register

From 295e56c0a63a06f35629c8fbabc880e835146ff1 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 17 Apr 2025 12:36:57 -0700
Subject: [PATCH 313/710] [RISCV] Add a couple of cost model tests for shuffles
 requiring legalization

---
 .../CostModel/RISCV/shuffle-permute.ll        | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-permute.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-permute.ll
index c4f6f3f1d874b..973032853c20f 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-permute.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-permute.ll
@@ -222,3 +222,33 @@ define void @general_permute_two_source() {
 
   ret void
 }
+
+
+define void @legalization_examples() {
+; CHECK-LABEL: 'legalization_examples'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %1 = shufflevector <32 x i64> undef, <32 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %2 = shufflevector <16 x i64> undef, <16 x i64> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 0, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 0, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %3 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 32, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %4 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 30, i32 1, i32 22, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 32, i32 23, i32 8, i32 25, i32 7, i32 27, i32 28, i32 15, i32 30, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SIZE-LABEL: 'legalization_examples'
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %1 = shufflevector <32 x i64> undef, <32 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %2 = shufflevector <16 x i64> undef, <16 x i64> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 0, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 0, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %3 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 32, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %4 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 30, i32 1, i32 22, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 32, i32 23, i32 8, i32 25, i32 7, i32 27, i32 28, i32 15, i32 30, i32 31>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; LOG-VRG-LABEL: 'legalization_examples'
+; LOG-VRG-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %1 = shufflevector <32 x i64> undef, <32 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32>
+; LOG-VRG-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %2 = shufflevector <16 x i64> undef, <16 x i64> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 0, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 0, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; LOG-VRG-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %3 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 32, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; LOG-VRG-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %4 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 30, i32 1, i32 22, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 32, i32 23, i32 8, i32 25, i32 7, i32 27, i32 28, i32 15, i32 30, i32 31>
+; LOG-VRG-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  shufflevector <32 x i64> undef, <32 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32>
+  shufflevector <16 x i64> undef, <16 x i64> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 0, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 0, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 32, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 30, i32 1, i32 22, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 32, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 32, i32 23, i32 8, i32 25, i32 7, i32 27, i32 28, i32 15, i32 30, i32 31>
+  ret void
+}

From 1c35f8145a79b5693b1517d9f4517a1ade5a95e4 Mon Sep 17 00:00:00 2001
From: Nick Sarnie <nick.sarnie@intel.com>
Date: Thu, 17 Apr 2025 15:49:06 -0400
Subject: [PATCH 314/710] [clang][SemaARM][NFC] clang-format aarch64 builtin
 check (#136188)

Going to modify this code so needs formatting.

Signed-off-by: Sarnie, Nick <nick.sarnie@intel.com>
---
 clang/lib/Sema/SemaARM.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 5bcbe78e9d633..b5abf244132d8 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -1109,7 +1109,10 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
   default: return false;
   case AArch64::BI__builtin_arm_dmb:
   case AArch64::BI__builtin_arm_dsb:
-  case AArch64::BI__builtin_arm_isb: l = 0; u = 15; break;
+  case AArch64::BI__builtin_arm_isb:
+    l = 0;
+    u = 15;
+    break;
   case AArch64::BI__builtin_arm_tcancel: l = 0; u = 65535; break;
   }
 

From 9f9c1f93819d441c561d18a73a55bcb6d8a692df Mon Sep 17 00:00:00 2001
From: Felipe de Azevedo Piovezan <fpiovezan@apple.com>
Date: Thu, 17 Apr 2025 12:53:25 -0700
Subject: [PATCH 315/710] [lldb][nfc] Split the constructor of
 ThreadPlanStepOut (#136160)

A subsequent commit will create a new constructor for ThreadPlanStepOut,
which needs to reuse much of the same logic of the existing constructor.
This commit places all of that reusable logic into a separate function.
---
 lldb/include/lldb/Target/ThreadPlanStepOut.h |  4 ++++
 lldb/source/Target/ThreadPlanStepOut.cpp     | 11 +++++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/lldb/include/lldb/Target/ThreadPlanStepOut.h b/lldb/include/lldb/Target/ThreadPlanStepOut.h
index 013c675afc33d..e414a6e0a2d49 100644
--- a/lldb/include/lldb/Target/ThreadPlanStepOut.h
+++ b/lldb/include/lldb/Target/ThreadPlanStepOut.h
@@ -82,6 +82,10 @@ class ThreadPlanStepOut : public ThreadPlan, public ThreadPlanShouldStopHere {
       LazyBool step_out_avoids_code_without_debug_info);
 
   void SetupAvoidNoDebug(LazyBool step_out_avoids_code_without_debug_info);
+
+  void SetupReturnAddress(lldb::StackFrameSP return_frame_sp,
+                          lldb::StackFrameSP immediate_return_from_sp,
+                          uint32_t frame_idx, bool continue_to_next_branch);
   // Need an appropriate marker for the current stack so we can tell step out
   // from step in.
 
diff --git a/lldb/source/Target/ThreadPlanStepOut.cpp b/lldb/source/Target/ThreadPlanStepOut.cpp
index 26c1abe710319..f2606403016a6 100644
--- a/lldb/source/Target/ThreadPlanStepOut.cpp
+++ b/lldb/source/Target/ThreadPlanStepOut.cpp
@@ -79,6 +79,13 @@ ThreadPlanStepOut::ThreadPlanStepOut(
       ComputeTargetFrame(thread, frame_idx, m_stepped_past_frames);
   StackFrameSP immediate_return_from_sp(thread.GetStackFrameAtIndex(frame_idx));
 
+  SetupReturnAddress(return_frame_sp, immediate_return_from_sp, frame_idx,
+                     continue_to_next_branch);
+}
+
+void ThreadPlanStepOut::SetupReturnAddress(
+    StackFrameSP return_frame_sp, StackFrameSP immediate_return_from_sp,
+    uint32_t frame_idx, bool continue_to_next_branch) {
   if (!return_frame_sp || !immediate_return_from_sp)
     return; // we can't do anything here.  ValidatePlan() will return false.
 
@@ -94,8 +101,8 @@ ThreadPlanStepOut::ThreadPlanStepOut(
       // First queue a plan that gets us to this inlined frame, and when we get
       // there we'll queue a second plan that walks us out of this frame.
       m_step_out_to_inline_plan_sp = std::make_shared<ThreadPlanStepOut>(
-          thread, nullptr, false, stop_others, eVoteNoOpinion, eVoteNoOpinion,
-          frame_idx - 1, eLazyBoolNo, continue_to_next_branch);
+          GetThread(), nullptr, false, m_stop_others, eVoteNoOpinion,
+          eVoteNoOpinion, frame_idx - 1, eLazyBoolNo, continue_to_next_branch);
       static_cast<ThreadPlanStepOut *>(m_step_out_to_inline_plan_sp.get())
           ->SetShouldStopHereCallbacks(nullptr, nullptr);
       m_step_out_to_inline_plan_sp->SetPrivate(true);

From 91f9f0fa1e3a776006fcbdb9a9c975682f35b10b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Thu, 17 Apr 2025 12:55:08 -0700
Subject: [PATCH 316/710] [flang][cuda] Update cuf.kernel_launch stream and
 conversion (#136179)

Update `cuf.kernel_launch` to take the stream as a reference. Update the
conversion to insert the `cuf.stream_cast` op so the stream can be set
as dependency.
---
 .../flang/Optimizer/Dialect/CUF/CUFOps.td     |  4 +++-
 flang/lib/Lower/ConvertCall.cpp               |  2 +-
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    | 23 +++++++++++++++----
 .../Optimizer/Transforms/CUFOpConversion.cpp  | 10 ++++++--
 flang/test/Fir/CUDA/cuda-launch.fir           |  7 +++---
 flang/test/Lower/CUDA/cuda-kernel-calls.cuf   |  6 ++---
 6 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
index f55f3e8a4466d..ccf9969e73a8e 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
@@ -200,7 +200,7 @@ def cuf_KernelLaunchOp : cuf_Op<"kernel_launch", [CallOpInterface,
 
   let arguments = (ins SymbolRefAttr:$callee, I32:$grid_x, I32:$grid_y,
       I32:$grid_z, I32:$block_x, I32:$block_y, I32:$block_z,
-      Optional<I32>:$bytes, Optional<AnyIntegerType>:$stream,
+      Optional<I32>:$bytes, Optional<fir_ReferenceType>:$stream,
       Variadic<AnyType>:$args, OptionalAttr<DictArrayAttr>:$arg_attrs,
       OptionalAttr<DictArrayAttr>:$res_attrs);
 
@@ -237,6 +237,8 @@ def cuf_KernelLaunchOp : cuf_Op<"kernel_launch", [CallOpInterface,
           *this, getNbNoArgOperand(), getArgs().size() - 1);
     }
   }];
+
+  let hasVerifier = 1;
 }
 
 def cuf_KernelOp : cuf_Op<"kernel", [AttrSizedOperandSegments,
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 31f2650917781..f28778ce6c1c9 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -589,7 +589,7 @@ Fortran::lower::genCallOpAndResult(
 
     mlir::Value stream; // stream is optional.
     if (caller.getCallDescription().chevrons().size() > 3)
-      stream = fir::getBase(converter.genExprValue(
+      stream = fir::getBase(converter.genExprAddr(
           caller.getCallDescription().chevrons()[3], stmtCtx));
 
     builder.create<cuf::KernelLaunchOp>(
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index ce197d48d4860..2c6d22f6f6c7d 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -139,6 +139,24 @@ llvm::LogicalResult cuf::DeallocateOp::verify() {
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// KernelLaunchOp
+//===----------------------------------------------------------------------===//
+
+template <typename OpTy>
+static llvm::LogicalResult checkStreamType(OpTy op) {
+  if (!op.getStream())
+    return mlir::success();
+  auto refTy = mlir::dyn_cast<fir::ReferenceType>(op.getStream().getType());
+  if (!refTy.getEleTy().isInteger(64))
+    return op.emitOpError("stream is expected to be a i64 reference");
+  return mlir::success();
+}
+
+llvm::LogicalResult cuf::KernelLaunchOp::verify() {
+  return checkStreamType(*this);
+}
+
 //===----------------------------------------------------------------------===//
 // KernelOp
 //===----------------------------------------------------------------------===//
@@ -324,10 +342,7 @@ void cuf::SharedMemoryOp::build(
 //===----------------------------------------------------------------------===//
 
 llvm::LogicalResult cuf::StreamCastOp::verify() {
-  auto refTy = mlir::dyn_cast<fir::ReferenceType>(getStream().getType());
-  if (!refTy.getEleTy().isInteger(64))
-    return emitOpError("stream is expected to be a i64 reference");
-  return mlir::success();
+  return checkStreamType(*this);
 }
 
 // Tablegen operators
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index caa59c6c17d0f..77364cb837c3c 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -879,8 +879,13 @@ struct CUFLaunchOpConversion
       gpuLaunchOp.getClusterSizeYMutable().assign(clusterDimY);
       gpuLaunchOp.getClusterSizeZMutable().assign(clusterDimZ);
     }
-    if (op.getStream())
-      gpuLaunchOp.getAsyncObjectMutable().assign(op.getStream());
+    if (op.getStream()) {
+      mlir::OpBuilder::InsertionGuard guard(rewriter);
+      rewriter.setInsertionPoint(gpuLaunchOp);
+      mlir::Value stream =
+          rewriter.create<cuf::StreamCastOp>(loc, op.getStream());
+      gpuLaunchOp.getAsyncDependenciesMutable().append(stream);
+    }
     if (procAttr)
       gpuLaunchOp->setAttr(cuf::getProcAttrName(), procAttr);
     rewriter.replaceOp(op, gpuLaunchOp);
@@ -933,6 +938,7 @@ class CUFOpConversion : public fir::impl::CUFOpConversionBase<CUFOpConversion> {
                                          /*forceUnifiedTBAATree=*/false, *dl);
     target.addLegalDialect<fir::FIROpsDialect, mlir::arith::ArithDialect,
                            mlir::gpu::GPUDialect>();
+    target.addLegalOp<cuf::StreamCastOp>();
     cuf::populateCUFToFIRConversionPatterns(typeConverter, *dl, symtab,
                                             patterns);
     if (mlir::failed(mlir::applyPartialConversion(getOperation(), target,
diff --git a/flang/test/Fir/CUDA/cuda-launch.fir b/flang/test/Fir/CUDA/cuda-launch.fir
index 621772efff415..319991546d3fe 100644
--- a/flang/test/Fir/CUDA/cuda-launch.fir
+++ b/flang/test/Fir/CUDA/cuda-launch.fir
@@ -146,8 +146,7 @@ module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_e
     %1:2 = hlfir.declare %0 {uniq_name = "_QMtest_callFhostEstream"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
     %c1_i32 = arith.constant 1 : i32
     %c0_i32 = arith.constant 0 : i32
-    %2 = fir.load %1#0 : !fir.ref<i64>
-    cuf.kernel_launch @_QMdevptrPtest<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c0_i32, %2 : i64>>>()
+    cuf.kernel_launch @_QMdevptrPtest<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c0_i32, %1#0 : !fir.ref<i64>>>>()
     return
   }
 }
@@ -155,5 +154,5 @@ module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_e
 // CHECK-LABEL: func.func @_QQmain()
 // CHECK: %[[STREAM:.*]] = fir.alloca i64 {bindc_name = "stream", uniq_name = "_QMtest_callFhostEstream"}
 // CHECK: %[[DECL_STREAM:.*]]:2 = hlfir.declare %[[STREAM]] {uniq_name = "_QMtest_callFhostEstream"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-// CHECK: %[[STREAM_LOADED:.*]] = fir.load %[[DECL_STREAM]]#0 : !fir.ref<i64>
-// CHECK: gpu.launch_func <%[[STREAM_LOADED]] : i64> @cuda_device_mod::@_QMdevptrPtest
+// CHECK: %[[TOKEN:.*]] = cuf.stream_cast %[[DECL_STREAM]]#0 : <i64>
+// CHECK: gpu.launch_func [%[[TOKEN]]] @cuda_device_mod::@_QMdevptrPtest
diff --git a/flang/test/Lower/CUDA/cuda-kernel-calls.cuf b/flang/test/Lower/CUDA/cuda-kernel-calls.cuf
index d66d2811f7a8b..71e594e4742ec 100644
--- a/flang/test/Lower/CUDA/cuda-kernel-calls.cuf
+++ b/flang/test/Lower/CUDA/cuda-kernel-calls.cuf
@@ -45,8 +45,8 @@ contains
     call dev_kernel0<<<10, 20, 2>>>()
 ! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}, %c2{{.*}}>>>() 
 
-    call dev_kernel0<<<10, 20, 2, 0>>>()
-! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}, %c2{{.*}}, %c0{{.*}}>>>()
+    call dev_kernel0<<<10, 20, 2, 0_8>>>()
+! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}, %c2{{.*}}, %{{.*}} : !fir.ref<i64>>>>()
 
     call dev_kernel1<<<1, 32>>>(a)
 ! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel1<<<%c1{{.*}}, %c1{{.*}}, %c1{{.*}}, %c32{{.*}}, %c1{{.*}}, %c1{{.*}}>>>(%{{.*}}) : (!fir.ref<f32>)
@@ -55,7 +55,7 @@ contains
 ! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel1<<<%c-1{{.*}}, %c1{{.*}}, %c1{{.*}}, %c32{{.*}}, %c1{{.*}}, %c1{{.*}}>>>(%{{.*}})
 
     call dev_kernel1<<<*,32,0,stream>>>(a)
-! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel1<<<%c-1{{.*}}, %c1{{.*}}, %c1{{.*}}, %c32{{.*}}, %c1{{.*}}, %c1{{.*}}, %c0{{.*}}, %{{.*}} : i64>>>(%{{.*}}) : (!fir.ref<f32>)
+! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel1<<<%c-1{{.*}}, %c1{{.*}}, %c1{{.*}}, %c32{{.*}}, %c1{{.*}}, %c1{{.*}}, %c0{{.*}}, %{{.*}} : !fir.ref<i64>>>>(%{{.*}}) : (!fir.ref<f32>)
 
   end
 

From 91c2607aace5ce910d0514dc5627651eade9c7d0 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <razvan.lupusoru@gmail.com>
Date: Thu, 17 Apr 2025 13:11:52 -0700
Subject: [PATCH 317/710] [flang][acc] Avoid implicitly privatizing IVs already
 privatized (#136181)

When generating `acc.loop`, the IV was always implicitly privatized.
However, if the user explicitly privatized it, the IR generated wasn't
quite right.

For example:
```
  !$acc loop private(i)
  do i = 1, n
    a(i) = b(i)
  end do
```

The IR generated looked like:
```
    %65 = acc.private varPtr(%19#0 : !fir.ref<i32>) -> !fir.ref<i32>
{implicit = true, name = "i"}
    %66:2 = hlfir.declare %65 {uniq_name = "_QFEi"} : (!fir.ref<i32>) ->
(!fir.ref<i32>, !fir.ref<i32>)
    %67 = acc.private varPtr(%66#0 : !fir.ref<i32>) -> !fir.ref<i32>
{name = "i"}
    acc.loop  private(@privatization_ref_i32 -> %65 : !fir.ref<i32>,
@privatization_ref_i32 -> %67 : !fir.ref<i32>) control(%arg0 : i32) =
(%c1_i32_46 : i32) to (%c10_i32_47 : i32)  step (%c1_i32_48 : i32) {
      fir.store %arg0 to %66#0 : !fir.ref<i32>
```

In order to fix this, we first process all of the clauses. Then when
attempting to generate implicit private IV, we look for an already
existing data clause operation.

The result is the following IR:
```
    %65 = acc.private varPtr(%19#0 : !fir.ref<i32>) -> !fir.ref<i32>
{name = "i"}
    %66:2 = hlfir.declare %65 {uniq_name = "_QFEi"} : (!fir.ref<i32>) ->
(!fir.ref<i32>, !fir.ref<i32>)
    acc.loop  private(@privatization_ref_i32 -> %65 : !fir.ref<i32>)
control(%arg0 : i32) = (%c1_i32_46 : i32) to (%c10_i32_47 : i32)  step
(%c1_i32_48 : i32) {
      fir.store %arg0 to %66#0 : !fir.ref<i32>
```
---
 flang/lib/Lower/OpenACC.cpp                   | 212 +++++++++---------
 flang/test/Lower/OpenACC/acc-kernels-loop.f90 |   4 +-
 flang/test/Lower/OpenACC/acc-loop.f90         |  24 +-
 .../test/Lower/OpenACC/acc-parallel-loop.f90  |   6 +-
 .../acc-private-unwrap-defaultbounds.f90      |  14 +-
 flang/test/Lower/OpenACC/acc-private.f90      |  16 +-
 flang/test/Lower/OpenACC/acc-serial-loop.f90  |   6 +-
 7 files changed, 151 insertions(+), 131 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index c83e277b996f3..c4f2e27f69c3b 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -1804,27 +1804,38 @@ static void privatizeIv(Fortran::lower::AbstractConverter &converter,
     builder.restoreInsertionPoint(insPt);
   }
 
-  std::string recipeName =
-      fir::getTypeAsString(ivValue.getType(), converter.getKindMap(),
-                           Fortran::lower::privatizationRecipePrefix);
-  auto recipe = Fortran::lower::createOrGetPrivateRecipe(
-      builder, recipeName, loc, ivValue.getType());
+  mlir::Operation *privateOp = nullptr;
+  for (auto privateVal : privateOperands) {
+    if (mlir::acc::getVar(privateVal.getDefiningOp()) == ivValue) {
+      privateOp = privateVal.getDefiningOp();
+      break;
+    }
+  }
 
-  std::stringstream asFortran;
-  asFortran << Fortran::lower::mangle::demangleName(toStringRef(sym.name()));
-  auto op = createDataEntryOp<mlir::acc::PrivateOp>(
-      builder, loc, ivValue, asFortran, {}, true, /*implicit=*/true,
-      mlir::acc::DataClause::acc_private, ivValue.getType(),
-      /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
+  if (privateOp == nullptr) {
+    std::string recipeName =
+        fir::getTypeAsString(ivValue.getType(), converter.getKindMap(),
+                             Fortran::lower::privatizationRecipePrefix);
+    auto recipe = Fortran::lower::createOrGetPrivateRecipe(
+        builder, recipeName, loc, ivValue.getType());
+
+    std::stringstream asFortran;
+    asFortran << Fortran::lower::mangle::demangleName(toStringRef(sym.name()));
+    auto op = createDataEntryOp<mlir::acc::PrivateOp>(
+        builder, loc, ivValue, asFortran, {}, true, /*implicit=*/true,
+        mlir::acc::DataClause::acc_private, ivValue.getType(),
+        /*async=*/{}, /*asyncDeviceTypes=*/{}, /*asyncOnlyDeviceTypes=*/{});
+    privateOp = op.getOperation();
 
-  privateOperands.push_back(op.getAccVar());
-  privatizations.push_back(mlir::SymbolRefAttr::get(builder.getContext(),
-                                                    recipe.getSymName().str()));
+    privateOperands.push_back(op.getAccVar());
+    privatizations.push_back(mlir::SymbolRefAttr::get(
+        builder.getContext(), recipe.getSymName().str()));
+  }
 
   // Map the new private iv to its symbol for the scope of the loop. bindSymbol
   // might create a hlfir.declare op, if so, we map its result in order to
   // use the sym value in the scope.
-  converter.bindSymbol(sym, op.getAccVar());
+  converter.bindSymbol(sym, mlir::acc::getAccVar(privateOp));
   auto privateValue = converter.getSymbolAddress(sym);
   if (auto declareOp =
           mlir::dyn_cast<hlfir::DeclareOp>(privateValue.getDefiningOp()))
@@ -1863,92 +1874,6 @@ static mlir::acc::LoopOp createLoopOp(
   crtDeviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
       builder.getContext(), mlir::acc::DeviceType::None));
 
-  llvm::SmallVector<mlir::Type> ivTypes;
-  llvm::SmallVector<mlir::Location> ivLocs;
-  llvm::SmallVector<bool> inclusiveBounds;
-
-  llvm::SmallVector<mlir::Location> locs;
-  locs.push_back(currentLocation); // Location of the directive
-  Fortran::lower::pft::Evaluation *crtEval = &eval.getFirstNestedEvaluation();
-  bool isDoConcurrent = outerDoConstruct.IsDoConcurrent();
-  if (isDoConcurrent) {
-    locs.push_back(converter.genLocation(
-        Fortran::parser::FindSourceLocation(outerDoConstruct)));
-    const Fortran::parser::LoopControl *loopControl =
-        &*outerDoConstruct.GetLoopControl();
-    const auto &concurrent =
-        std::get<Fortran::parser::LoopControl::Concurrent>(loopControl->u);
-    if (!std::get<std::list<Fortran::parser::LocalitySpec>>(concurrent.t)
-             .empty())
-      TODO(currentLocation, "DO CONCURRENT with locality spec");
-
-    const auto &concurrentHeader =
-        std::get<Fortran::parser::ConcurrentHeader>(concurrent.t);
-    const auto &controls =
-        std::get<std::list<Fortran::parser::ConcurrentControl>>(
-            concurrentHeader.t);
-    for (const auto &control : controls) {
-      lowerbounds.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(std::get<1>(control.t)), stmtCtx)));
-      upperbounds.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(std::get<2>(control.t)), stmtCtx)));
-      if (const auto &expr =
-              std::get<std::optional<Fortran::parser::ScalarIntExpr>>(
-                  control.t))
-        steps.push_back(fir::getBase(converter.genExprValue(
-            *Fortran::semantics::GetExpr(*expr), stmtCtx)));
-      else // If `step` is not present, assume it is `1`.
-        steps.push_back(builder.createIntegerConstant(
-            currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1));
-
-      const auto &name = std::get<Fortran::parser::Name>(control.t);
-      privatizeIv(converter, *name.symbol, currentLocation, ivTypes, ivLocs,
-                  privateOperands, ivPrivate, privatizations, isDoConcurrent);
-
-      inclusiveBounds.push_back(true);
-    }
-  } else {
-    int64_t collapseValue = Fortran::lower::getCollapseValue(accClauseList);
-    for (unsigned i = 0; i < collapseValue; ++i) {
-      const Fortran::parser::LoopControl *loopControl;
-      if (i == 0) {
-        loopControl = &*outerDoConstruct.GetLoopControl();
-        locs.push_back(converter.genLocation(
-            Fortran::parser::FindSourceLocation(outerDoConstruct)));
-      } else {
-        auto *doCons = crtEval->getIf<Fortran::parser::DoConstruct>();
-        assert(doCons && "expect do construct");
-        loopControl = &*doCons->GetLoopControl();
-        locs.push_back(converter.genLocation(
-            Fortran::parser::FindSourceLocation(*doCons)));
-      }
-
-      const Fortran::parser::LoopControl::Bounds *bounds =
-          std::get_if<Fortran::parser::LoopControl::Bounds>(&loopControl->u);
-      assert(bounds && "Expected bounds on the loop construct");
-      lowerbounds.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(bounds->lower), stmtCtx)));
-      upperbounds.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(bounds->upper), stmtCtx)));
-      if (bounds->step)
-        steps.push_back(fir::getBase(converter.genExprValue(
-            *Fortran::semantics::GetExpr(bounds->step), stmtCtx)));
-      else // If `step` is not present, assume it is `1`.
-        steps.push_back(builder.createIntegerConstant(
-            currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1));
-
-      Fortran::semantics::Symbol &ivSym =
-          bounds->name.thing.symbol->GetUltimate();
-      privatizeIv(converter, ivSym, currentLocation, ivTypes, ivLocs,
-                  privateOperands, ivPrivate, privatizations);
-
-      inclusiveBounds.push_back(true);
-
-      if (i < collapseValue - 1)
-        crtEval = &*std::next(crtEval->getNestedEvaluations().begin());
-    }
-  }
-
   for (const Fortran::parser::AccClause &clause : accClauseList.v) {
     mlir::Location clauseLocation = converter.genLocation(clause.source);
     if (const auto *gangClause =
@@ -2101,6 +2026,91 @@ static mlir::acc::LoopOp createLoopOp(
     }
   }
 
+  llvm::SmallVector<mlir::Type> ivTypes;
+  llvm::SmallVector<mlir::Location> ivLocs;
+  llvm::SmallVector<bool> inclusiveBounds;
+  llvm::SmallVector<mlir::Location> locs;
+  locs.push_back(currentLocation); // Location of the directive
+  Fortran::lower::pft::Evaluation *crtEval = &eval.getFirstNestedEvaluation();
+  bool isDoConcurrent = outerDoConstruct.IsDoConcurrent();
+  if (isDoConcurrent) {
+    locs.push_back(converter.genLocation(
+        Fortran::parser::FindSourceLocation(outerDoConstruct)));
+    const Fortran::parser::LoopControl *loopControl =
+        &*outerDoConstruct.GetLoopControl();
+    const auto &concurrent =
+        std::get<Fortran::parser::LoopControl::Concurrent>(loopControl->u);
+    if (!std::get<std::list<Fortran::parser::LocalitySpec>>(concurrent.t)
+             .empty())
+      TODO(currentLocation, "DO CONCURRENT with locality spec");
+
+    const auto &concurrentHeader =
+        std::get<Fortran::parser::ConcurrentHeader>(concurrent.t);
+    const auto &controls =
+        std::get<std::list<Fortran::parser::ConcurrentControl>>(
+            concurrentHeader.t);
+    for (const auto &control : controls) {
+      lowerbounds.push_back(fir::getBase(converter.genExprValue(
+          *Fortran::semantics::GetExpr(std::get<1>(control.t)), stmtCtx)));
+      upperbounds.push_back(fir::getBase(converter.genExprValue(
+          *Fortran::semantics::GetExpr(std::get<2>(control.t)), stmtCtx)));
+      if (const auto &expr =
+              std::get<std::optional<Fortran::parser::ScalarIntExpr>>(
+                  control.t))
+        steps.push_back(fir::getBase(converter.genExprValue(
+            *Fortran::semantics::GetExpr(*expr), stmtCtx)));
+      else // If `step` is not present, assume it is `1`.
+        steps.push_back(builder.createIntegerConstant(
+            currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1));
+
+      const auto &name = std::get<Fortran::parser::Name>(control.t);
+      privatizeIv(converter, *name.symbol, currentLocation, ivTypes, ivLocs,
+                  privateOperands, ivPrivate, privatizations, isDoConcurrent);
+
+      inclusiveBounds.push_back(true);
+    }
+  } else {
+    int64_t collapseValue = Fortran::lower::getCollapseValue(accClauseList);
+    for (unsigned i = 0; i < collapseValue; ++i) {
+      const Fortran::parser::LoopControl *loopControl;
+      if (i == 0) {
+        loopControl = &*outerDoConstruct.GetLoopControl();
+        locs.push_back(converter.genLocation(
+            Fortran::parser::FindSourceLocation(outerDoConstruct)));
+      } else {
+        auto *doCons = crtEval->getIf<Fortran::parser::DoConstruct>();
+        assert(doCons && "expect do construct");
+        loopControl = &*doCons->GetLoopControl();
+        locs.push_back(converter.genLocation(
+            Fortran::parser::FindSourceLocation(*doCons)));
+      }
+
+      const Fortran::parser::LoopControl::Bounds *bounds =
+          std::get_if<Fortran::parser::LoopControl::Bounds>(&loopControl->u);
+      assert(bounds && "Expected bounds on the loop construct");
+      lowerbounds.push_back(fir::getBase(converter.genExprValue(
+          *Fortran::semantics::GetExpr(bounds->lower), stmtCtx)));
+      upperbounds.push_back(fir::getBase(converter.genExprValue(
+          *Fortran::semantics::GetExpr(bounds->upper), stmtCtx)));
+      if (bounds->step)
+        steps.push_back(fir::getBase(converter.genExprValue(
+            *Fortran::semantics::GetExpr(bounds->step), stmtCtx)));
+      else // If `step` is not present, assume it is `1`.
+        steps.push_back(builder.createIntegerConstant(
+            currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1));
+
+      Fortran::semantics::Symbol &ivSym =
+          bounds->name.thing.symbol->GetUltimate();
+      privatizeIv(converter, ivSym, currentLocation, ivTypes, ivLocs,
+                  privateOperands, ivPrivate, privatizations);
+
+      inclusiveBounds.push_back(true);
+
+      if (i < collapseValue - 1)
+        crtEval = &*std::next(crtEval->getNestedEvaluations().begin());
+    }
+  }
+
   // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value> operands;
   llvm::SmallVector<int32_t> operandSegments;
diff --git a/flang/test/Lower/OpenACC/acc-kernels-loop.f90 b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
index b4ccdfef4213e..0ded708cb1a3b 100644
--- a/flang/test/Lower/OpenACC/acc-kernels-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
@@ -495,7 +495,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -508,7 +508,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
diff --git a/flang/test/Lower/OpenACC/acc-loop.f90 b/flang/test/Lower/OpenACC/acc-loop.f90
index f77aefcc2c314..0246f60705898 100644
--- a/flang/test/Lower/OpenACC/acc-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-loop.f90
@@ -73,7 +73,7 @@ program acc_loop
   END DO
 
 ! CHECK:      [[GANGNUM1:%.*]] = arith.constant 8 : i32
-! CHECK-NEXT: acc.loop gang({num=[[GANGNUM1]] : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      acc.loop gang({num=[[GANGNUM1]] : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
 
@@ -83,7 +83,7 @@ program acc_loop
   END DO
 
 ! CHECK:      [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK-NEXT: acc.loop gang({num=[[GANGNUM2]] : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      acc.loop gang({num=[[GANGNUM2]] : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
 
@@ -145,29 +145,39 @@ program acc_loop
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
 
   !$acc loop private(c)
+  DO i = 1, n
+    c(:,i) = d(:,i)
+  END DO
+
+! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+
+  ! When the induction variable is explicitly private - only a single private entry should be created.
+  !$acc loop private(i)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
 
   !$acc loop private(c, d)
   DO i = 1, n
-    a(i) = b(i)
+    c(:,i) = d(:,i)
   END DO
 
-! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
 
   !$acc loop private(c) private(d)
   DO i = 1, n
-    a(i) = b(i)
+    c(:,i) = d(:,i)
   END DO
 
-! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
+! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
 
diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
index bc3ec617f2bdd..ccd37d87262e3 100644
--- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
@@ -448,7 +448,7 @@ subroutine acc_parallel_loop
 ! CHECK:      %[[ACC_PRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.parallel {{.*}} firstprivate(@firstprivatization_ref_10xf32 -> %[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>{{.*}})
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -510,7 +510,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32})
+! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32})
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -523,7 +523,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32})
+! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32})
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
diff --git a/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
index 044871dc8288c..a43228070bd8f 100644
--- a/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
@@ -191,7 +191,7 @@ program acc_private
   END DO
 
 ! CHECK: %[[C_PRIVATE:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "c"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_i32 -> %[[C_PRIVATE]] : !fir.ref<i32>)
+! CHECK: acc.loop private({{.*}}@privatization_ref_i32 -> %[[C_PRIVATE]] : !fir.ref<i32>{{.*}})
 ! CHECK: acc.yield
 
   !$acc loop private(b)
@@ -205,7 +205,7 @@ program acc_private
 ! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
 ! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_100xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: acc.loop private({{.*}}@privatization_ref_100xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<100xf32>>{{.*}})
 ! CHECK: acc.yield
 
   !$acc loop private(b(1:50))
@@ -219,7 +219,7 @@ program acc_private
 ! CHECK: %[[UB:.*]] = arith.constant 49 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
 ! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(1:50)"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_50xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<50xf32>>)
+! CHECK: acc.loop private({{.*}}@privatization_ref_50xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<50xf32>>{{.*}})
 
   !$acc parallel loop firstprivate(c)
   DO i = 1, n
@@ -275,7 +275,7 @@ subroutine acc_private_assumed_shape(a, n)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>{{.*}})
 
 subroutine acc_private_allocatable_array(a, n)
   integer, allocatable :: a(:)
@@ -298,7 +298,7 @@ subroutine acc_private_allocatable_array(a, n)
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.heap<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>{{.*}})
 ! CHECK: acc.serial private(@privatization_box_heap_Uxi32 -> %{{.*}} : !fir.heap<!fir.array<?xi32>>)
 
 subroutine acc_private_pointer_array(a, n)
@@ -318,7 +318,7 @@ subroutine acc_private_pointer_array(a, n)
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ptr<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>{{.*}})
 
 subroutine acc_private_dynamic_extent(a, n)
   integer :: n, i
@@ -337,7 +337,7 @@ subroutine acc_private_dynamic_extent(a, n)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?x2xi32>>) -> !fir.ref<!fir.array<?x?x2xi32>>
 ! CHECK: %[[PRIV:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>{{.*}})
 
 subroutine acc_firstprivate_assumed_shape(a, n)
   integer :: a(:), i, n
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index 356bb9d825d8e..9827350271e65 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -200,7 +200,7 @@ program acc_private
   END DO
 
 ! CHECK: %[[C_PRIVATE:.*]] = acc.private varPtr(%[[DECLC]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {name = "c"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_i32 -> %[[C_PRIVATE]] : !fir.ref<i32>)
+! CHECK: acc.loop private({{.*}}@privatization_ref_i32 -> %[[C_PRIVATE]] : !fir.ref<i32>{{.*}})
 ! CHECK: acc.yield
 
   !$acc loop private(b)
@@ -210,7 +210,7 @@ program acc_private
   END DO
 
 ! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_100xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: acc.loop private({{.*}}@privatization_ref_100xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<100xf32>>{{.*}})
 ! CHECK: acc.yield
 
   !$acc loop private(b(1:50))
@@ -224,7 +224,7 @@ program acc_private
 ! CHECK: %[[UB:.*]] = arith.constant 49 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
 ! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(1:50)"}
-! CHECK: acc.loop private({{.*}}@privatization_ref_50xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<50xf32>>)
+! CHECK: acc.loop private({{.*}}@privatization_ref_50xf32 -> %[[B_PRIVATE]] : !fir.ref<!fir.array<50xf32>>{{.*}})
 
   !$acc parallel loop firstprivate(c)
   DO i = 1, n
@@ -275,7 +275,7 @@ subroutine acc_private_assumed_shape(a, n)
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[PRIVATE:.*]] = acc.private var(%[[DECL_A]]#0 : !fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.box<!fir.array<?xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.box<!fir.array<?xi32>>{{.*}})
 
 subroutine acc_private_allocatable_array(a, n)
   integer, allocatable :: a(:)
@@ -296,7 +296,7 @@ subroutine acc_private_allocatable_array(a, n)
 ! CHECK: %[[DECLA_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_arrayEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>{{.*}})
 ! CHECK: acc.serial private(@privatization_ref_box_heap_Uxi32 -> %{{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 
 subroutine acc_private_allocatable_scalar(b, a, n)
@@ -319,7 +319,7 @@ subroutine acc_private_allocatable_scalar(b, a, n)
 ! CHECK: %[[DECLA_B:.*]]:2 = hlfir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_scalarEb"} : (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_B]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = "b"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_i32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_heap_i32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.heap<i32>>>{{.*}})
 ! CHECK: acc.serial private(@privatization_ref_box_heap_i32 -> %{{.*}} : !fir.ref<!fir.box<!fir.heap<i32>>>) {
 
 subroutine acc_private_pointer_array(a, n)
@@ -337,7 +337,7 @@ subroutine acc_private_pointer_array(a, n)
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_private_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_ref_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>{{.*}})
 
 subroutine acc_private_dynamic_extent(a, n)
   integer :: n, i
@@ -355,7 +355,7 @@ subroutine acc_private_dynamic_extent(a, n)
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>, !fir.dscope) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[PRIV:.*]] = acc.private var(%[[DECL_A]]#0 : !fir.box<!fir.array<?x?x2xi32>>) -> !fir.box<!fir.array<?x?x2xi32>> {name = "a"}
-! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.box<!fir.array<?x?x2xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.box<!fir.array<?x?x2xi32>>{{.*}})
 
 subroutine acc_firstprivate_assumed_shape(a, n)
   integer :: a(:), i, n
diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90
index e85e037a6f2ec..478dfa0d96c3b 100644
--- a/flang/test/Lower/OpenACC/acc-serial-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90
@@ -389,7 +389,7 @@ subroutine acc_serial_loop
 ! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      acc.serial {{.*}} firstprivate(@firstprivatization_ref_10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>{{.*}})
 ! CHECK-NOT:          fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -451,7 +451,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -464,7 +464,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32}) {{.*}} {
+! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield

From cdad39b7e007aa2a597f6fbeda28a6b269406916 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Thu, 17 Apr 2025 13:17:52 -0700
Subject: [PATCH 318/710] [llvm][docs] Replace `Optional<T>` with
 `std::optional<T>`

The `llvm::Optional` template is no more. Remove it from the error-handling section of LLVM the programmer's manual and the ASTImporter documentation.
---
 clang/docs/LibASTImporter.rst   | 2 +-
 llvm/docs/ProgrammersManual.rst | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang/docs/LibASTImporter.rst b/clang/docs/LibASTImporter.rst
index 515eff7ebe330..f5d40928d01e8 100644
--- a/clang/docs/LibASTImporter.rst
+++ b/clang/docs/LibASTImporter.rst
@@ -468,7 +468,7 @@ Note, there may be several different ASTImporter objects which import into the s
         cxxRecordDecl(hasName("Y"), isDefinition()), ToUnit);
     ToYDef->dump();
     // An error is set for "ToYDef" in the shared state.
-    Optional<ASTImportError> OptErr =
+    std::optional<ASTImportError> OptErr =
         ImporterState->getImportDeclErrorIfAny(ToYDef);
     assert(OptErr);
 
diff --git a/llvm/docs/ProgrammersManual.rst b/llvm/docs/ProgrammersManual.rst
index e2829eb5a8846..bb729597cc5a3 100644
--- a/llvm/docs/ProgrammersManual.rst
+++ b/llvm/docs/ProgrammersManual.rst
@@ -590,14 +590,14 @@ semantics.  For example:
 
 This third form works with any type that can be assigned to from ``T&&``. This
 can be useful if the ``Expected<T>`` value needs to be stored an already-declared
-``Optional<T>``. For example:
+``std::optional<T>``. For example:
 
 .. code-block:: c++
 
   Expected<StringRef> extractClassName(StringRef Definition);
   struct ClassData {
     StringRef Definition;
-    Optional<StringRef> LazyName;
+    std::optional<StringRef> LazyName;
     ...
     Error initialize() {
       if (auto Err = extractClassName(Path).moveInto(LazyName))

From 30013872190ca05eb00333adb989c9f74b1cf3ac Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Thu, 17 Apr 2025 13:19:48 -0700
Subject: [PATCH 319/710] [NVPTX] Basic support for fp128 as a storage type
 (#136006)

While fp128 operations are not natively supported in hardware, emulation
for them is supported by nvcc. This change adds basic support for
fp128 as a storage type allowing for lowering of IR containing these
types.

Fixes: https://github.com/llvm/llvm-project/issues/95471
---
 llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp     | 98 +++++++++----------
 llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h       | 33 +++----
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp   | 28 ++----
 llvm/lib/Target/NVPTX/NVPTXUtilities.cpp      |  4 -
 llvm/lib/Target/NVPTX/NVPTXUtilities.h        |  9 +-
 llvm/test/CodeGen/NVPTX/fp128-storage-type.ll | 56 +++++++++++
 .../test/CodeGen/NVPTX/global-variable-big.ll |  5 +-
 7 files changed, 137 insertions(+), 96 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/fp128-storage-type.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 65cfeadc21a3b..2f4b109e8e9e9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -249,11 +249,6 @@ MCOperand NVPTXAsmPrinter::GetSymbolRef(const MCSymbol *Symbol) {
   return MCOperand::createExpr(Expr);
 }
 
-static bool ShouldPassAsArray(Type *Ty) {
-  return Ty->isAggregateType() || Ty->isVectorTy() || Ty->isIntegerTy(128) ||
-         Ty->isHalfTy() || Ty->isBFloatTy();
-}
-
 void NVPTXAsmPrinter::printReturnValStr(const Function *F, raw_ostream &O) {
   const DataLayout &DL = getDataLayout();
   const NVPTXSubtarget &STI = TM.getSubtarget<NVPTXSubtarget>(*F);
@@ -264,26 +259,21 @@ void NVPTXAsmPrinter::printReturnValStr(const Function *F, raw_ostream &O) {
     return;
   O << " (";
 
-  if ((Ty->isFloatingPointTy() || Ty->isIntegerTy()) &&
-      !ShouldPassAsArray(Ty)) {
-    unsigned size = 0;
-    if (auto *ITy = dyn_cast<IntegerType>(Ty)) {
-      size = ITy->getBitWidth();
-    } else {
-      assert(Ty->isFloatingPointTy() && "Floating point type expected here");
-      size = Ty->getPrimitiveSizeInBits();
-    }
-    size = promoteScalarArgumentSize(size);
-    O << ".param .b" << size << " func_retval0";
-  } else if (isa<PointerType>(Ty)) {
-    O << ".param .b" << TLI->getPointerTy(DL).getSizeInBits()
-      << " func_retval0";
-  } else if (ShouldPassAsArray(Ty)) {
-    unsigned totalsz = DL.getTypeAllocSize(Ty);
-    Align RetAlignment = TLI->getFunctionArgumentAlignment(
+  auto PrintScalarRetVal = [&](unsigned Size) {
+    O << ".param .b" << promoteScalarArgumentSize(Size) << " func_retval0";
+  };
+  if (shouldPassAsArray(Ty)) {
+    const unsigned TotalSize = DL.getTypeAllocSize(Ty);
+    const Align RetAlignment = TLI->getFunctionArgumentAlignment(
         F, Ty, AttributeList::ReturnIndex, DL);
     O << ".param .align " << RetAlignment.value() << " .b8 func_retval0["
-      << totalsz << "]";
+      << TotalSize << "]";
+  } else if (Ty->isFloatingPointTy()) {
+    PrintScalarRetVal(Ty->getPrimitiveSizeInBits());
+  } else if (auto *ITy = dyn_cast<IntegerType>(Ty)) {
+    PrintScalarRetVal(ITy->getBitWidth());
+  } else if (isa<PointerType>(Ty)) {
+    PrintScalarRetVal(TLI->getPointerTy(DL).getSizeInBits());
   } else
     llvm_unreachable("Unknown return type");
   O << ") ";
@@ -975,8 +965,8 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
   O << " .align "
     << GVar->getAlign().value_or(DL.getPrefTypeAlign(ETy)).value();
 
-  if (ETy->isFloatingPointTy() || ETy->isPointerTy() ||
-      (ETy->isIntegerTy() && ETy->getScalarSizeInBits() <= 64)) {
+  if (ETy->isPointerTy() || ((ETy->isIntegerTy() || ETy->isFloatingPointTy()) &&
+                             ETy->getScalarSizeInBits() <= 64)) {
     O << " .";
     // Special case: ABI requires that we use .u8 for predicates
     if (ETy->isIntegerTy(1))
@@ -1016,6 +1006,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
     // and vectors are lowered into arrays of bytes.
     switch (ETy->getTypeID()) {
     case Type::IntegerTyID: // Integers larger than 64 bits
+    case Type::FP128TyID:
     case Type::StructTyID:
     case Type::ArrayTyID:
     case Type::FixedVectorTyID: {
@@ -1266,8 +1257,8 @@ void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
   O << " .align "
     << GVar->getAlign().value_or(DL.getPrefTypeAlign(ETy)).value();
 
-  // Special case for i128
-  if (ETy->isIntegerTy(128)) {
+  // Special case for i128/fp128
+  if (ETy->getScalarSizeInBits() == 128) {
     O << " .b8 ";
     getSymbol(GVar)->print(O, MAI);
     O << "[16]";
@@ -1383,7 +1374,7 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
       continue;
     }
 
-    if (ShouldPassAsArray(Ty)) {
+    if (shouldPassAsArray(Ty)) {
       // Just print .param .align <a> .b8 .param[size];
       // <a>  = optimal alignment for the element type; always multiple of
       //        PAL.getParamAlignment
@@ -1682,48 +1673,49 @@ void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
 void NVPTXAsmPrinter::bufferAggregateConstant(const Constant *CPV,
                                               AggBuffer *aggBuffer) {
   const DataLayout &DL = getDataLayout();
-  int Bytes;
+
+  auto ExtendBuffer = [](APInt Val, AggBuffer *Buffer) {
+    for (unsigned I : llvm::seq(Val.getBitWidth() / 8))
+      Buffer->addByte(Val.extractBitsAsZExtValue(8, I * 8));
+  };
 
   // Integers of arbitrary width
   if (const ConstantInt *CI = dyn_cast<ConstantInt>(CPV)) {
-    APInt Val = CI->getValue();
-    for (unsigned I = 0, E = DL.getTypeAllocSize(CPV->getType()); I < E; ++I) {
-      uint8_t Byte = Val.getLoBits(8).getZExtValue();
-      aggBuffer->addBytes(&Byte, 1, 1);
-      Val.lshrInPlace(8);
-    }
+    ExtendBuffer(CI->getValue(), aggBuffer);
     return;
   }
 
+  // f128
+  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CPV)) {
+    if (CFP->getType()->isFP128Ty()) {
+      ExtendBuffer(CFP->getValueAPF().bitcastToAPInt(), aggBuffer);
+      return;
+    }
+  }
+
   // Old constants
   if (isa<ConstantArray>(CPV) || isa<ConstantVector>(CPV)) {
-    if (CPV->getNumOperands())
-      for (unsigned i = 0, e = CPV->getNumOperands(); i != e; ++i)
-        bufferLEByte(cast<Constant>(CPV->getOperand(i)), 0, aggBuffer);
+    for (const auto &Op : CPV->operands())
+      bufferLEByte(cast<Constant>(Op), 0, aggBuffer);
     return;
   }
 
-  if (const ConstantDataSequential *CDS =
-          dyn_cast<ConstantDataSequential>(CPV)) {
-    if (CDS->getNumElements())
-      for (unsigned i = 0; i < CDS->getNumElements(); ++i)
-        bufferLEByte(cast<Constant>(CDS->getElementAsConstant(i)), 0,
-                     aggBuffer);
+  if (const auto *CDS = dyn_cast<ConstantDataSequential>(CPV)) {
+    for (unsigned I : llvm::seq(CDS->getNumElements()))
+      bufferLEByte(cast<Constant>(CDS->getElementAsConstant(I)), 0, aggBuffer);
     return;
   }
 
   if (isa<ConstantStruct>(CPV)) {
     if (CPV->getNumOperands()) {
       StructType *ST = cast<StructType>(CPV->getType());
-      for (unsigned i = 0, e = CPV->getNumOperands(); i != e; ++i) {
-        if (i == (e - 1))
-          Bytes = DL.getStructLayout(ST)->getElementOffset(0) +
-                  DL.getTypeAllocSize(ST) -
-                  DL.getStructLayout(ST)->getElementOffset(i);
-        else
-          Bytes = DL.getStructLayout(ST)->getElementOffset(i + 1) -
-                  DL.getStructLayout(ST)->getElementOffset(i);
-        bufferLEByte(cast<Constant>(CPV->getOperand(i)), Bytes, aggBuffer);
+      for (unsigned I : llvm::seq(CPV->getNumOperands())) {
+        int EndOffset = (I + 1 == CPV->getNumOperands())
+                            ? DL.getStructLayout(ST)->getElementOffset(0) +
+                                  DL.getTypeAllocSize(ST)
+                            : DL.getStructLayout(ST)->getElementOffset(I + 1);
+        int Bytes = EndOffset - DL.getStructLayout(ST)->getElementOffset(I);
+        bufferLEByte(cast<Constant>(CPV->getOperand(I)), Bytes, aggBuffer);
       }
     }
     return;
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
index 74daaa2fb7134..9ed7e650e7b0c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
@@ -111,27 +111,22 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
 
     // Copy Num bytes from Ptr.
     // if Bytes > Num, zero fill up to Bytes.
-    unsigned addBytes(unsigned char *Ptr, int Num, int Bytes) {
-      assert((curpos + Num) <= size);
-      assert((curpos + Bytes) <= size);
-      for (int i = 0; i < Num; ++i) {
-        buffer[curpos] = Ptr[i];
-        curpos++;
-      }
-      for (int i = Num; i < Bytes; ++i) {
-        buffer[curpos] = 0;
-        curpos++;
-      }
-      return curpos;
+    void addBytes(const unsigned char *Ptr, unsigned Num, unsigned Bytes) {
+      for (unsigned I : llvm::seq(Num))
+        addByte(Ptr[I]);
+      if (Bytes > Num)
+        addZeros(Bytes - Num);
     }
 
-    unsigned addZeros(int Num) {
-      assert((curpos + Num) <= size);
-      for (int i = 0; i < Num; ++i) {
-        buffer[curpos] = 0;
-        curpos++;
-      }
-      return curpos;
+    void addByte(uint8_t Byte) {
+      assert(curpos < size);
+      buffer[curpos] = Byte;
+      curpos++;
+    }
+
+    void addZeros(unsigned Num) {
+      for (unsigned _ : llvm::seq(Num))
+        addByte(0);
     }
 
     void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index abe4c27009698..277a34173e7b8 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -246,14 +246,11 @@ static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
   SmallVector<uint64_t, 16> TempOffsets;
 
   // Special case for i128 - decompose to (i64, i64)
-  if (Ty->isIntegerTy(128)) {
-    ValueVTs.push_back(EVT(MVT::i64));
-    ValueVTs.push_back(EVT(MVT::i64));
+  if (Ty->isIntegerTy(128) || Ty->isFP128Ty()) {
+    ValueVTs.append({MVT::i64, MVT::i64});
 
-    if (Offsets) {
-      Offsets->push_back(StartingOffset + 0);
-      Offsets->push_back(StartingOffset + 8);
-    }
+    if (Offsets)
+      Offsets->append({StartingOffset + 0, StartingOffset + 8});
 
     return;
   }
@@ -1165,11 +1162,6 @@ NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
   return DAG.getNode(NVPTXISD::Wrapper, dl, PtrVT, Op);
 }
 
-static bool IsTypePassedAsArray(const Type *Ty) {
-  return Ty->isAggregateType() || Ty->isVectorTy() || Ty->isIntegerTy(128) ||
-         Ty->isHalfTy() || Ty->isBFloatTy();
-}
-
 std::string NVPTXTargetLowering::getPrototype(
     const DataLayout &DL, Type *retTy, const ArgListTy &Args,
     const SmallVectorImpl<ISD::OutputArg> &Outs, MaybeAlign retAlignment,
@@ -1186,7 +1178,7 @@ std::string NVPTXTargetLowering::getPrototype(
   } else {
     O << "(";
     if ((retTy->isFloatingPointTy() || retTy->isIntegerTy()) &&
-        !IsTypePassedAsArray(retTy)) {
+        !shouldPassAsArray(retTy)) {
       unsigned size = 0;
       if (auto *ITy = dyn_cast<IntegerType>(retTy)) {
         size = ITy->getBitWidth();
@@ -1203,7 +1195,7 @@ std::string NVPTXTargetLowering::getPrototype(
       O << ".param .b" << size << " _";
     } else if (isa<PointerType>(retTy)) {
       O << ".param .b" << PtrVT.getSizeInBits() << " _";
-    } else if (IsTypePassedAsArray(retTy)) {
+    } else if (shouldPassAsArray(retTy)) {
       O << ".param .align " << (retAlignment ? retAlignment->value() : 0)
         << " .b8 _[" << DL.getTypeAllocSize(retTy) << "]";
     } else {
@@ -1224,7 +1216,7 @@ std::string NVPTXTargetLowering::getPrototype(
     first = false;
 
     if (!Outs[OIdx].Flags.isByVal()) {
-      if (IsTypePassedAsArray(Ty)) {
+      if (shouldPassAsArray(Ty)) {
         Align ParamAlign =
             getArgumentAlignment(&CB, Ty, i + AttributeList::FirstArgIndex, DL);
         O << ".param .align " << ParamAlign.value() << " .b8 ";
@@ -1529,7 +1521,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
 
     bool NeedAlign; // Does argument declaration specify alignment?
-    bool PassAsArray = IsByVal || IsTypePassedAsArray(Ty);
+    const bool PassAsArray = IsByVal || shouldPassAsArray(Ty);
     if (IsVAArg) {
       if (ParamCount == FirstVAArg) {
         SDValue DeclareParamOps[] = {
@@ -1718,7 +1710,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     //  .param .align N .b8 retval0[<size-in-bytes>], or
     //  .param .b<size-in-bits> retval0
     unsigned resultsz = DL.getTypeAllocSizeInBits(RetTy);
-    if (!IsTypePassedAsArray(RetTy)) {
+    if (!shouldPassAsArray(RetTy)) {
       resultsz = promoteScalarArgumentSize(resultsz);
       SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
       SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
@@ -3362,7 +3354,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
 
     if (theArgs[i]->use_empty()) {
       // argument is dead
-      if (IsTypePassedAsArray(Ty) && !Ty->isVectorTy()) {
+      if (shouldPassAsArray(Ty) && !Ty->isVectorTy()) {
         SmallVector<EVT, 16> vtparts;
 
         ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts);
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index 3d9d2ae372080..b800445a3b19c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -372,8 +372,4 @@ bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) {
          !isKernelFunction(*F);
 }
 
-bool Isv2x16VT(EVT VT) {
-  return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16);
-}
-
 } // namespace llvm
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
index 9283b398a9c14..2288241ec0178 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
@@ -84,7 +84,14 @@ inline unsigned promoteScalarArgumentSize(unsigned size) {
 
 bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM);
 
-bool Isv2x16VT(EVT VT);
+inline bool Isv2x16VT(EVT VT) {
+  return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16);
+}
+
+inline bool shouldPassAsArray(Type *Ty) {
+  return Ty->isAggregateType() || Ty->isVectorTy() ||
+         Ty->getScalarSizeInBits() == 128 || Ty->isHalfTy() || Ty->isBFloatTy();
+}
 
 namespace NVPTX {
 inline std::string getValidPTXIdentifier(StringRef Name) {
diff --git a/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll b/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll
new file mode 100644
index 0000000000000..5b96f4978a7cb
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_20 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s-mcpu=sm_20 | %ptxas-verify %}
+
+target triple = "nvptx64-unknown-cuda"
+
+define fp128 @identity(fp128 %x) {
+; CHECK-LABEL: identity(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v2.u64 {%rd1, %rd2}, [identity_param_0];
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; CHECK-NEXT:    ret;
+  ret fp128 %x
+}
+
+define void @load_store(ptr %in, ptr %out) {
+; CHECK-LABEL: load_store(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [load_store_param_0];
+; CHECK-NEXT:    ld.u64 %rd2, [%rd1+8];
+; CHECK-NEXT:    ld.u64 %rd3, [%rd1];
+; CHECK-NEXT:    ld.param.u64 %rd4, [load_store_param_1];
+; CHECK-NEXT:    st.u64 [%rd4], %rd3;
+; CHECK-NEXT:    st.u64 [%rd4+8], %rd2;
+; CHECK-NEXT:    ret;
+  %val = load fp128, ptr %in
+  store fp128 %val, ptr %out
+  ret void
+}
+
+define void @call(fp128 %x) {
+; CHECK-LABEL: call(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v2.u64 {%rd1, %rd2}, [call_param_0];
+; CHECK-NEXT:    { // callseq 0, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v2.b64 [param0], {%rd1, %rd2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 0
+; CHECK-NEXT:    ret;
+  call void @call(fp128 %x)
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/global-variable-big.ll b/llvm/test/CodeGen/NVPTX/global-variable-big.ll
index e8d7fb3815b79..09f556e72a2bd 100644
--- a/llvm/test/CodeGen/NVPTX/global-variable-big.ll
+++ b/llvm/test/CodeGen/NVPTX/global-variable-big.ll
@@ -4,12 +4,15 @@
 target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
 target triple = "nvptx64-nvidia-cuda"
 
-; Check that we can handle global variables of large integer type.
+; Check that we can handle global variables of large integer and fp128 type.
 
 ; (lsb) 0x0102'0304'0506...0F10 (msb)
 @gv = addrspace(1) externally_initialized global i128 21345817372864405881847059188222722561, align 16
 ; CHECK: .visible .global .align 16 .b8 gv[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
 
+@gv_fp128 = addrspace(1) externally_initialized global fp128 0xL0807060504030201100F0E0D0C0B0A09, align 16
+; CHECK: .visible .global .align 16 .b8 gv_fp128[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+
 ; Make sure that we do not overflow on large number of elements.
 ; CHECK: .visible .global .align 1 .b8 large_data[4831838208];
 @large_data = global [4831838208 x i8] zeroinitializer

From b3a53cc721807f0cd0e3a1e6ddda03a85c774d4f Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Thu, 17 Apr 2025 13:42:35 -0700
Subject: [PATCH 320/710] [NFC][TableGen] Code cleanup in Wasm disassember
 emitter (#135992)

- Use range for loop to iterate over instructions.
- Emit generated code in anonymous namespace instead of `llvm` and
reduce the scope of this to just the type declarations.
- Emit generated tables as static constexpr
- Replace code to search in operand table with `std::search`.
- Skip the last "null" entry in PrefixTable and use range for loop to
search PrefixTable in the .cpp code.
- Do not generate `WebAssemblyInstructionTableSize` definition as its
already defined in the .cpp file.
- Remove {} for single statement loop/if/else bodies.
---
 .../Disassembler/WebAssemblyDisassembler.cpp  | 10 +--
 .../WebAssemblyDisassemblerEmitter.cpp        | 89 +++++++++----------
 2 files changed, 47 insertions(+), 52 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
index 7bee4af86b52a..0399f9d38e4eb 100644
--- a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
+++ b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
@@ -38,9 +38,9 @@ using DecodeStatus = MCDisassembler::DecodeStatus;
 
 #include "WebAssemblyGenDisassemblerTables.inc"
 
-namespace {
 static constexpr int WebAssemblyInstructionTableSize = 256;
 
+namespace {
 class WebAssemblyDisassembler final : public MCDisassembler {
   std::unique_ptr<const MCInstrInfo> MCII;
 
@@ -171,10 +171,10 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
   // If this is a prefix byte, indirect to another table.
   if (WasmInst->ET == ET_Prefix) {
     WasmInst = nullptr;
-    // Linear search, so far only 2 entries.
-    for (auto PT = PrefixTable; PT->Table; PT++) {
-      if (PT->Prefix == Opc) {
-        WasmInst = PT->Table;
+    // Linear search, so far only 4 entries.
+    for (const auto &[Prefix, Table] : PrefixTable) {
+      if (Prefix == Opc) {
+        WasmInst = Table;
         break;
       }
     }
diff --git a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
index 5aa573ac857dc..1170e35d04702 100644
--- a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
+++ b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
@@ -1,4 +1,4 @@
-//===- WebAssemblyDisassemblerEmitter.cpp - Disassembler tables -*- C++ -*-===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -16,10 +16,11 @@
 #include "WebAssemblyDisassemblerEmitter.h"
 #include "Common/CodeGenInstruction.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/iterator.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Record.h"
 
-static constexpr int WebAssemblyInstructionTableSize = 256;
+constexpr int WebAssemblyInstructionTableSize = 256;
 
 void llvm::emitWebAssemblyDisassemblerTables(
     raw_ostream &OS,
@@ -29,8 +30,8 @@ void llvm::emitWebAssemblyDisassemblerTables(
   std::map<unsigned,
            std::map<unsigned, std::pair<unsigned, const CodeGenInstruction *>>>
       OpcodeTable;
-  for (unsigned I = 0; I != NumberedInstructions.size(); ++I) {
-    const CodeGenInstruction &CGI = *NumberedInstructions[I];
+  for (const auto &[Idx, CGI] :
+       enumerate(make_pointee_range(NumberedInstructions))) {
     const Record &Def = *CGI.TheDef;
     if (!Def.getValue("Inst"))
       continue;
@@ -75,27 +76,31 @@ void llvm::emitWebAssemblyDisassemblerTables(
       }
     }
     // Set this instruction as the one to use.
-    CGIP = {I, &CGI};
+    CGIP = {Idx, &CGI};
   }
-  OS << "#include \"MCTargetDesc/WebAssemblyMCTargetDesc.h\"\n";
-  OS << "\n";
-  OS << "namespace llvm {\n\n";
-  OS << "static constexpr int WebAssemblyInstructionTableSize = ";
-  OS << WebAssemblyInstructionTableSize << ";\n\n";
-  OS << "enum EntryType : uint8_t { ";
-  OS << "ET_Unused, ET_Prefix, ET_Instruction };\n\n";
-  OS << "struct WebAssemblyInstruction {\n";
-  OS << "  uint16_t Opcode;\n";
-  OS << "  EntryType ET;\n";
-  OS << "  uint8_t NumOperands;\n";
-  OS << "  uint16_t OperandStart;\n";
-  OS << "};\n\n";
+
+  OS << R"(
+#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+
+namespace {
+enum EntryType : uint8_t { ET_Unused, ET_Prefix, ET_Instruction };
+
+struct WebAssemblyInstruction {
+  uint16_t Opcode;
+  EntryType ET;
+  uint8_t NumOperands;
+  uint16_t OperandStart;
+};
+} // end anonymous namespace
+
+)";
+
   std::vector<std::string> OperandTable, CurOperandList;
   // Output one table per prefix.
   for (const auto &[Prefix, Table] : OpcodeTable) {
     if (Table.empty())
       continue;
-    OS << "WebAssemblyInstruction InstructionTable" << Prefix;
+    OS << "constexpr WebAssemblyInstruction InstructionTable" << Prefix;
     OS << "[] = {\n";
     for (unsigned I = 0; I < WebAssemblyInstructionTableSize; I++) {
       auto InstIt = Table.find(I);
@@ -116,53 +121,43 @@ void llvm::emitWebAssemblyDisassemblerTables(
         }
         // See if we already have stored this sequence before. This is not
         // strictly necessary but makes the table really small.
-        size_t OperandStart = OperandTable.size();
-        if (CurOperandList.size() <= OperandTable.size()) {
-          for (size_t J = 0; J <= OperandTable.size() - CurOperandList.size();
-               ++J) {
-            size_t K = 0;
-            for (; K < CurOperandList.size(); ++K) {
-              if (OperandTable[J + K] != CurOperandList[K])
-                break;
-            }
-            if (K == CurOperandList.size()) {
-              OperandStart = J;
-              break;
-            }
-          }
-        }
+        auto SearchI =
+            std::search(OperandTable.begin(), OperandTable.end(),
+                        CurOperandList.begin(), CurOperandList.end());
+        OS << std::distance(OperandTable.begin(), SearchI);
         // Store operands if no prior occurrence.
-        if (OperandStart == OperandTable.size()) {
+        if (SearchI == OperandTable.end())
           llvm::append_range(OperandTable, CurOperandList);
-        }
-        OS << OperandStart;
       } else {
         auto PrefixIt = OpcodeTable.find(I);
         // If we have a non-empty table for it that's not 0, this is a prefix.
-        if (PrefixIt != OpcodeTable.end() && I && !Prefix) {
+        if (PrefixIt != OpcodeTable.end() && I && !Prefix)
           OS << "  { 0, ET_Prefix, 0, 0";
-        } else {
+        else
           OS << "  { 0, ET_Unused, 0, 0";
-        }
       }
       OS << "  },\n";
     }
     OS << "};\n\n";
   }
   // Create a table of all operands:
-  OS << "const uint8_t OperandTable[] = {\n";
-  for (auto &Op : OperandTable) {
+  OS << "constexpr uint8_t OperandTable[] = {\n";
+  for (const auto &Op : OperandTable)
     OS << "  " << Op << ",\n";
-  }
   OS << "};\n\n";
+
   // Create a table of all extension tables:
-  OS << "struct { uint8_t Prefix; const WebAssemblyInstruction *Table; }\n";
-  OS << "PrefixTable[] = {\n";
+  OS << R"(
+constexpr struct {
+  uint8_t Prefix;
+  const WebAssemblyInstruction *Table;
+} PrefixTable[] = {
+)";
+
   for (const auto &[Prefix, Table] : OpcodeTable) {
     if (Table.empty() || !Prefix)
       continue;
     OS << "  { " << Prefix << ", InstructionTable" << Prefix << " },\n";
   }
-  OS << "  { 0, nullptr }\n};\n\n";
-  OS << "} // end namespace llvm\n";
+  OS << "};\n";
 }

From 7119b0cfd3678a16664d07e3c7b15399c4b3a321 Mon Sep 17 00:00:00 2001
From: Bangtian Liu <liubangtian@gmail.com>
Date: Thu, 17 Apr 2025 16:52:36 -0400
Subject: [PATCH 321/710] [MLIR][CAPI][python] expose the python binding for
 linalgOp.getIndexingMaps (#136054)

This PR is mainly about exposing the python bindings for
`linalgOp.getIndexingMaps`.

---------

Signed-off-by: Bangtian Liu <liubangtian@gmail.com>
---
 mlir/include/mlir-c/Dialect/Linalg.h       |  3 ++
 mlir/lib/Bindings/Python/DialectLinalg.cpp | 10 +++++
 mlir/lib/CAPI/Dialect/Linalg.cpp           | 10 +++++
 mlir/test/python/dialects/linalg/utils.py  | 49 ++++++++++++++++++++++
 4 files changed, 72 insertions(+)

diff --git a/mlir/include/mlir-c/Dialect/Linalg.h b/mlir/include/mlir-c/Dialect/Linalg.h
index 4f2ee0d434222..339e63d667c5e 100644
--- a/mlir/include/mlir-c/Dialect/Linalg.h
+++ b/mlir/include/mlir-c/Dialect/Linalg.h
@@ -50,6 +50,9 @@ typedef struct MlirLinalgConvolutionDimensions {
 MLIR_CAPI_EXPORTED MlirLinalgConvolutionDimensions
 mlirLinalgInferConvolutionDimensions(MlirOperation op);
 
+MLIR_CAPI_EXPORTED MlirAttribute
+mlirLinalgGetIndexingMapsAttribute(MlirOperation op);
+
 MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Linalg, linalg);
 
 #ifdef __cplusplus
diff --git a/mlir/lib/Bindings/Python/DialectLinalg.cpp b/mlir/lib/Bindings/Python/DialectLinalg.cpp
index ce1102a3b3498..015502371c65b 100644
--- a/mlir/lib/Bindings/Python/DialectLinalg.cpp
+++ b/mlir/lib/Bindings/Python/DialectLinalg.cpp
@@ -120,6 +120,16 @@ static void populateDialectLinalgSubmodule(nb::module_ m) {
 
   m.def("infer_convolution_dimensions", &InferConvolutionDimensions,
         "Infers convolution dimensions", nb::arg("op"));
+
+  m.def(
+      "get_indexing_maps",
+      [](MlirOperation op) -> std::optional<MlirAttribute> {
+        MlirAttribute attr = mlirLinalgGetIndexingMapsAttribute(op);
+        if (mlirAttributeIsNull(attr))
+          return std::nullopt;
+        return attr;
+      },
+      "Returns the indexing_maps attribute for a linalg op.");
 }
 
 NB_MODULE(_mlirDialectsLinalg, m) {
diff --git a/mlir/lib/CAPI/Dialect/Linalg.cpp b/mlir/lib/CAPI/Dialect/Linalg.cpp
index 7c456102a2c0c..0c4f6e88e7078 100644
--- a/mlir/lib/CAPI/Dialect/Linalg.cpp
+++ b/mlir/lib/CAPI/Dialect/Linalg.cpp
@@ -120,4 +120,14 @@ mlirLinalgInferConvolutionDimensions(MlirOperation op) {
   return result;
 }
 
+MLIR_CAPI_EXPORTED MlirAttribute
+mlirLinalgGetIndexingMapsAttribute(MlirOperation op) {
+  auto linalgOp = llvm::dyn_cast<mlir::linalg::LinalgOp>(unwrap(op));
+  if (!linalgOp)
+    return MlirAttribute{nullptr};
+
+  ArrayAttr attr = linalgOp.getIndexingMaps();
+  return wrap(attr);
+}
+
 MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Linalg, linalg, LinalgDialect)
diff --git a/mlir/test/python/dialects/linalg/utils.py b/mlir/test/python/dialects/linalg/utils.py
index 98157b0e443cf..5f7cb6a6c83cb 100644
--- a/mlir/test/python/dialects/linalg/utils.py
+++ b/mlir/test/python/dialects/linalg/utils.py
@@ -159,3 +159,52 @@ def dyn_conv_fn(input, filter, output):
                 assert list(dims.depth) == []
                 assert list(dims.strides) == [1, 1]
                 assert list(dims.dilations) == [1, 1]
+
+
+@run
+def test_get_indexing_maps_attr():
+    with Context(), Location.unknown():
+        module = Module.create()
+        f32 = F32Type.get()
+        with InsertionPoint(module.body):
+            a_type = RankedTensorType.get((4, 8), f32)
+            b_type = RankedTensorType.get((8, 16), f32)
+            c_type = RankedTensorType.get((4, 16), f32)
+
+            dim_m = AffineDimExpr.get(0)
+            dim_n = AffineDimExpr.get(1)
+            dim_k = AffineDimExpr.get(2)
+
+            a_map = AffineMap.get(3, 0, [dim_m, dim_k])
+            b_map = AffineMap.get(3, 0, [dim_k, dim_n])
+            c_map = AffineMap.get(3, 0, [dim_m, dim_n])
+
+            @func.FuncOp.from_py_func(a_type, b_type, c_type)
+            def matmul_func(a, b, c):
+                zero = arith.ConstantOp(value=FloatAttr.get(f32, 0.0), result=f32)
+                assert not linalg.get_indexing_maps(
+                    zero.operation
+                ), "Expected no indexing_maps on non-linalg op"
+
+                init = linalg.fill(zero, outs=[c])
+                fill_op = init.owner
+                fill_maps = linalg.get_indexing_maps(fill_op)
+                assert fill_maps is not None
+                assert len(fill_maps) == 2
+
+                # The fill op should have maps like (d0, d1) -> () and (d0, d1).
+                fill_input_map = fill_maps[0].value
+                fill_output_map = fill_maps[1].value
+                assert fill_input_map == AffineMap.get(2, 0, [])
+                assert fill_output_map == AffineMap.get(2, 0, [dim_m, dim_n])
+
+                result = linalg.matmul(a, b, outs=(init,))
+                matmul_op = result.owner
+                matmul_maps = linalg.get_indexing_maps(matmul_op)
+                assert matmul_maps is not None
+                assert len(matmul_maps) == 3
+
+                maps = [map_attr.value for map_attr in matmul_maps]
+                assert maps[0] == a_map
+                assert maps[1] == b_map
+                assert maps[2] == c_map

From 889dad7f40932ea68c9e287e62441507f4f0f261 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Thu, 17 Apr 2025 21:53:32 +0100
Subject: [PATCH 322/710] [ItaniumDemangle] Add customizable
 printLeft/printRight APIs to OutputBuffer (#133249)

This patch includes the necessary changes for the LLDB feature proposed
in
https://discourse.llvm.org/t/rfc-lldb-highlighting-function-names-in-lldb-backtraces/85309.
The TL;DR is that we want to track where certain parts of a demangled
name begin/end so we can highlight them in backtraces.

We introduce a new `printLeft`/`printRight` API on `OutputBuffer` that a
client (in our case LLDB) can implement to track state while printing
the demangle tree. This requires redirecting all calls to to
`printLeft`/`printRight` to the `OutputBuffer`. One quirk with the new
API is that `Utility.h` would now depend on `ItaniumDemangle.h` and
vice-versa. To keep these files header-only I made the definitions
`inline` and implement the new APIs in `ItaniumDemangle.h` (so the
definition of `Node` is available to them).

Also introduces `notifyInsertion`/`notifyDeletion` APIs that a client can override to respond to cases where the `OutputBuffer` changes arbitrary parts of the name.
---
 libcxxabi/src/demangle/ItaniumDemangle.h      | 91 +++++++++++--------
 libcxxabi/src/demangle/Utility.h              | 26 +++++-
 llvm/include/llvm/Demangle/ItaniumDemangle.h  | 91 +++++++++++--------
 llvm/include/llvm/Demangle/Utility.h          | 26 +++++-
 .../Demangle/ItaniumDemangleTest.cpp          |  2 +-
 llvm/unittests/Demangle/OutputBufferTest.cpp  | 37 ++++++++
 6 files changed, 196 insertions(+), 77 deletions(-)

diff --git a/libcxxabi/src/demangle/ItaniumDemangle.h b/libcxxabi/src/demangle/ItaniumDemangle.h
index eca9ddad66f92..69932e63669bf 100644
--- a/libcxxabi/src/demangle/ItaniumDemangle.h
+++ b/libcxxabi/src/demangle/ItaniumDemangle.h
@@ -283,20 +283,11 @@ class Node {
   }
 
   void print(OutputBuffer &OB) const {
-    printLeft(OB);
+    OB.printLeft(*this);
     if (RHSComponentCache != Cache::No)
-      printRight(OB);
+      OB.printRight(*this);
   }
 
-  // Print the "left" side of this Node into OutputBuffer.
-  virtual void printLeft(OutputBuffer &) const = 0;
-
-  // Print the "right". This distinction is necessary to represent C++ types
-  // that appear on the RHS of their subtype, such as arrays or functions.
-  // Since most types don't have such a component, provide a default
-  // implementation.
-  virtual void printRight(OutputBuffer &) const {}
-
   // Print an initializer list of this type. Returns true if we printed a custom
   // representation, false if nothing has been printed and the default
   // representation should be used.
@@ -312,6 +303,24 @@ class Node {
 #ifndef NDEBUG
   DEMANGLE_DUMP_METHOD void dump() const;
 #endif
+
+private:
+  friend class OutputBuffer;
+
+  // Print the "left" side of this Node into OutputBuffer.
+  //
+  // Note, should only be called from OutputBuffer implementations.
+  // Call \ref OutputBuffer::printLeft instead.
+  virtual void printLeft(OutputBuffer &) const = 0;
+
+  // Print the "right". This distinction is necessary to represent C++ types
+  // that appear on the RHS of their subtype, such as arrays or functions.
+  // Since most types don't have such a component, provide a default
+  // implementation.
+  //
+  // Note, should only be called from OutputBuffer implementations.
+  // Call \ref OutputBuffer::printRight instead.
+  virtual void printRight(OutputBuffer &) const {}
 };
 
 class NodeArray {
@@ -460,11 +469,11 @@ class QualType final : public Node {
   }
 
   void printLeft(OutputBuffer &OB) const override {
-    Child->printLeft(OB);
+    OB.printLeft(*Child);
     printQuals(OB);
   }
 
-  void printRight(OutputBuffer &OB) const override { Child->printRight(OB); }
+  void printRight(OutputBuffer &OB) const override { OB.printRight(*Child); }
 };
 
 class ConversionOperatorType final : public Node {
@@ -493,7 +502,7 @@ class PostfixQualifiedType final : public Node {
   template<typename Fn> void match(Fn F) const { F(Ty, Postfix); }
 
   void printLeft(OutputBuffer &OB) const override {
-    Ty->printLeft(OB);
+    OB.printLeft(*Ty);
     OB += Postfix;
   }
 };
@@ -579,7 +588,7 @@ struct AbiTagAttr : Node {
   std::string_view getBaseName() const override { return Base->getBaseName(); }
 
   void printLeft(OutputBuffer &OB) const override {
-    Base->printLeft(OB);
+    OB.printLeft(*Base);
     OB += "[abi:";
     OB += Tag;
     OB += "]";
@@ -646,7 +655,7 @@ class PointerType final : public Node {
     // We rewrite objc_object<SomeProtocol>* into id<SomeProtocol>.
     if (Pointee->getKind() != KObjCProtoName ||
         !static_cast<const ObjCProtoName *>(Pointee)->isObjCObject()) {
-      Pointee->printLeft(OB);
+      OB.printLeft(*Pointee);
       if (Pointee->hasArray(OB))
         OB += " ";
       if (Pointee->hasArray(OB) || Pointee->hasFunction(OB))
@@ -665,7 +674,7 @@ class PointerType final : public Node {
         !static_cast<const ObjCProtoName *>(Pointee)->isObjCObject()) {
       if (Pointee->hasArray(OB) || Pointee->hasFunction(OB))
         OB += ")";
-      Pointee->printRight(OB);
+      OB.printRight(*Pointee);
     }
   }
 };
@@ -731,7 +740,7 @@ class ReferenceType : public Node {
     std::pair<ReferenceKind, const Node *> Collapsed = collapse(OB);
     if (!Collapsed.second)
       return;
-    Collapsed.second->printLeft(OB);
+    OB.printLeft(*Collapsed.second);
     if (Collapsed.second->hasArray(OB))
       OB += " ";
     if (Collapsed.second->hasArray(OB) || Collapsed.second->hasFunction(OB))
@@ -748,7 +757,7 @@ class ReferenceType : public Node {
       return;
     if (Collapsed.second->hasArray(OB) || Collapsed.second->hasFunction(OB))
       OB += ")";
-    Collapsed.second->printRight(OB);
+    OB.printRight(*Collapsed.second);
   }
 };
 
@@ -768,7 +777,7 @@ class PointerToMemberType final : public Node {
   }
 
   void printLeft(OutputBuffer &OB) const override {
-    MemberType->printLeft(OB);
+    OB.printLeft(*MemberType);
     if (MemberType->hasArray(OB) || MemberType->hasFunction(OB))
       OB += "(";
     else
@@ -780,7 +789,7 @@ class PointerToMemberType final : public Node {
   void printRight(OutputBuffer &OB) const override {
     if (MemberType->hasArray(OB) || MemberType->hasFunction(OB))
       OB += ")";
-    MemberType->printRight(OB);
+    OB.printRight(*MemberType);
   }
 };
 
@@ -800,7 +809,7 @@ class ArrayType final : public Node {
   bool hasRHSComponentSlow(OutputBuffer &) const override { return true; }
   bool hasArraySlow(OutputBuffer &) const override { return true; }
 
-  void printLeft(OutputBuffer &OB) const override { Base->printLeft(OB); }
+  void printLeft(OutputBuffer &OB) const override { OB.printLeft(*Base); }
 
   void printRight(OutputBuffer &OB) const override {
     if (OB.back() != ']')
@@ -809,7 +818,7 @@ class ArrayType final : public Node {
     if (Dimension)
       Dimension->print(OB);
     OB += "]";
-    Base->printRight(OB);
+    OB.printRight(*Base);
   }
 
   bool printInitListAsType(OutputBuffer &OB,
@@ -853,7 +862,7 @@ class FunctionType final : public Node {
   // by printing out the return types's left, then print our parameters, then
   // finally print right of the return type.
   void printLeft(OutputBuffer &OB) const override {
-    Ret->printLeft(OB);
+    OB.printLeft(*Ret);
     OB += " ";
   }
 
@@ -861,7 +870,7 @@ class FunctionType final : public Node {
     OB.printOpen();
     Params.printWithComma(OB);
     OB.printClose();
-    Ret->printRight(OB);
+    OB.printRight(*Ret);
 
     if (CVQuals & QualConst)
       OB += " const";
@@ -966,6 +975,8 @@ class FunctionEncoding final : public Node {
   FunctionRefQual getRefQual() const { return RefQual; }
   NodeArray getParams() const { return Params; }
   const Node *getReturnType() const { return Ret; }
+  const Node *getAttrs() const { return Attrs; }
+  const Node *getRequires() const { return Requires; }
 
   bool hasRHSComponentSlow(OutputBuffer &) const override { return true; }
   bool hasFunctionSlow(OutputBuffer &) const override { return true; }
@@ -974,10 +985,11 @@ class FunctionEncoding final : public Node {
 
   void printLeft(OutputBuffer &OB) const override {
     if (Ret) {
-      Ret->printLeft(OB);
+      OB.printLeft(*Ret);
       if (!Ret->hasRHSComponent(OB))
         OB += " ";
     }
+
     Name->print(OB);
   }
 
@@ -985,8 +997,9 @@ class FunctionEncoding final : public Node {
     OB.printOpen();
     Params.printWithComma(OB);
     OB.printClose();
+
     if (Ret)
-      Ret->printRight(OB);
+      OB.printRight(*Ret);
 
     if (CVQuals & QualConst)
       OB += " const";
@@ -1326,14 +1339,14 @@ class NonTypeTemplateParamDecl final : public Node {
   template<typename Fn> void match(Fn F) const { F(Name, Type); }
 
   void printLeft(OutputBuffer &OB) const override {
-    Type->printLeft(OB);
+    OB.printLeft(*Type);
     if (!Type->hasRHSComponent(OB))
       OB += " ";
   }
 
   void printRight(OutputBuffer &OB) const override {
     Name->print(OB);
-    Type->printRight(OB);
+    OB.printRight(*Type);
   }
 };
 
@@ -1378,11 +1391,11 @@ class TemplateParamPackDecl final : public Node {
   template<typename Fn> void match(Fn F) const { F(Param); }
 
   void printLeft(OutputBuffer &OB) const override {
-    Param->printLeft(OB);
+    OB.printLeft(*Param);
     OB += "...";
   }
 
-  void printRight(OutputBuffer &OB) const override { Param->printRight(OB); }
+  void printRight(OutputBuffer &OB) const override { OB.printRight(*Param); }
 };
 
 /// An unexpanded parameter pack (either in the expression or type context). If
@@ -1447,13 +1460,13 @@ class ParameterPack final : public Node {
     initializePackExpansion(OB);
     size_t Idx = OB.CurrentPackIndex;
     if (Idx < Data.size())
-      Data[Idx]->printLeft(OB);
+      OB.printLeft(*Data[Idx]);
   }
   void printRight(OutputBuffer &OB) const override {
     initializePackExpansion(OB);
     size_t Idx = OB.CurrentPackIndex;
     if (Idx < Data.size())
-      Data[Idx]->printRight(OB);
+      OB.printRight(*Data[Idx]);
   }
 };
 
@@ -1611,13 +1624,13 @@ struct ForwardTemplateReference : Node {
     if (Printing)
       return;
     ScopedOverride<bool> SavePrinting(Printing, true);
-    Ref->printLeft(OB);
+    OB.printLeft(*Ref);
   }
   void printRight(OutputBuffer &OB) const override {
     if (Printing)
       return;
     ScopedOverride<bool> SavePrinting(Printing, true);
-    Ref->printRight(OB);
+    OB.printRight(*Ref);
   }
 };
 
@@ -1769,7 +1782,7 @@ class DtorName : public Node {
 
   void printLeft(OutputBuffer &OB) const override {
     OB += "~";
-    Base->printLeft(OB);
+    OB.printLeft(*Base);
   }
 };
 
@@ -2049,7 +2062,7 @@ class CastExpr : public Node {
     {
       ScopedOverride<unsigned> LT(OB.GtIsGt, 0);
       OB += "<";
-      To->printLeft(OB);
+      OB.printLeft(*To);
       OB += ">";
     }
     OB.printOpen();
@@ -6180,6 +6193,10 @@ struct ManglingParser : AbstractManglingParser<ManglingParser<Alloc>, Alloc> {
                                Alloc>::AbstractManglingParser;
 };
 
+inline void OutputBuffer::printLeft(const Node &N) { N.printLeft(*this); }
+
+inline void OutputBuffer::printRight(const Node &N) { N.printRight(*this); }
+
 DEMANGLE_NAMESPACE_END
 
 #if defined(__clang__)
diff --git a/libcxxabi/src/demangle/Utility.h b/libcxxabi/src/demangle/Utility.h
index f1fad35d60d98..511983ad40f7a 100644
--- a/libcxxabi/src/demangle/Utility.h
+++ b/libcxxabi/src/demangle/Utility.h
@@ -27,6 +27,8 @@
 
 DEMANGLE_NAMESPACE_BEGIN
 
+class Node;
+
 // Stream that AST nodes write their string representation into after the AST
 // has been parsed.
 class OutputBuffer {
@@ -79,10 +81,24 @@ class OutputBuffer {
   OutputBuffer(const OutputBuffer &) = delete;
   OutputBuffer &operator=(const OutputBuffer &) = delete;
 
+  virtual ~OutputBuffer() {}
+
   operator std::string_view() const {
     return std::string_view(Buffer, CurrentPosition);
   }
 
+  /// Called by the demangler when printing the demangle tree. By
+  /// default calls into \c Node::print{Left|Right} but can be overriden
+  /// by clients to track additional state when printing the demangled name.
+  virtual void printLeft(const Node &N);
+  virtual void printRight(const Node &N);
+
+  /// Called when we write to this object anywhere other than the end.
+  virtual void notifyInsertion(size_t /*Position*/, size_t /*Count*/) {}
+
+  /// Called when we make the \c CurrentPosition of this object smaller.
+  virtual void notifyDeletion(size_t /*OldPos*/, size_t /*NewPos*/) {}
+
   /// If a ParameterPackExpansion (or similar type) is encountered, the offset
   /// into the pack that we're currently printing.
   unsigned CurrentPackIndex = std::numeric_limits<unsigned>::max();
@@ -126,6 +142,8 @@ class OutputBuffer {
     std::memcpy(Buffer, &*R.begin(), Size);
     CurrentPosition += Size;
 
+    notifyInsertion(/*Position=*/0, /*Count=*/Size);
+
     return *this;
   }
 
@@ -161,14 +179,20 @@ class OutputBuffer {
     DEMANGLE_ASSERT(Pos <= CurrentPosition, "");
     if (N == 0)
       return;
+
     grow(N);
     std::memmove(Buffer + Pos + N, Buffer + Pos, CurrentPosition - Pos);
     std::memcpy(Buffer + Pos, S, N);
     CurrentPosition += N;
+
+    notifyInsertion(Pos, N);
   }
 
   size_t getCurrentPosition() const { return CurrentPosition; }
-  void setCurrentPosition(size_t NewPos) { CurrentPosition = NewPos; }
+  void setCurrentPosition(size_t NewPos) {
+    notifyDeletion(CurrentPosition, NewPos);
+    CurrentPosition = NewPos;
+  }
 
   char back() const {
     DEMANGLE_ASSERT(CurrentPosition, "");
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index 28fdfc5eff1b6..b4a4c72021fd1 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -283,20 +283,11 @@ class Node {
   }
 
   void print(OutputBuffer &OB) const {
-    printLeft(OB);
+    OB.printLeft(*this);
     if (RHSComponentCache != Cache::No)
-      printRight(OB);
+      OB.printRight(*this);
   }
 
-  // Print the "left" side of this Node into OutputBuffer.
-  virtual void printLeft(OutputBuffer &) const = 0;
-
-  // Print the "right". This distinction is necessary to represent C++ types
-  // that appear on the RHS of their subtype, such as arrays or functions.
-  // Since most types don't have such a component, provide a default
-  // implementation.
-  virtual void printRight(OutputBuffer &) const {}
-
   // Print an initializer list of this type. Returns true if we printed a custom
   // representation, false if nothing has been printed and the default
   // representation should be used.
@@ -312,6 +303,24 @@ class Node {
 #ifndef NDEBUG
   DEMANGLE_DUMP_METHOD void dump() const;
 #endif
+
+private:
+  friend class OutputBuffer;
+
+  // Print the "left" side of this Node into OutputBuffer.
+  //
+  // Note, should only be called from OutputBuffer implementations.
+  // Call \ref OutputBuffer::printLeft instead.
+  virtual void printLeft(OutputBuffer &) const = 0;
+
+  // Print the "right". This distinction is necessary to represent C++ types
+  // that appear on the RHS of their subtype, such as arrays or functions.
+  // Since most types don't have such a component, provide a default
+  // implementation.
+  //
+  // Note, should only be called from OutputBuffer implementations.
+  // Call \ref OutputBuffer::printRight instead.
+  virtual void printRight(OutputBuffer &) const {}
 };
 
 class NodeArray {
@@ -460,11 +469,11 @@ class QualType final : public Node {
   }
 
   void printLeft(OutputBuffer &OB) const override {
-    Child->printLeft(OB);
+    OB.printLeft(*Child);
     printQuals(OB);
   }
 
-  void printRight(OutputBuffer &OB) const override { Child->printRight(OB); }
+  void printRight(OutputBuffer &OB) const override { OB.printRight(*Child); }
 };
 
 class ConversionOperatorType final : public Node {
@@ -493,7 +502,7 @@ class PostfixQualifiedType final : public Node {
   template<typename Fn> void match(Fn F) const { F(Ty, Postfix); }
 
   void printLeft(OutputBuffer &OB) const override {
-    Ty->printLeft(OB);
+    OB.printLeft(*Ty);
     OB += Postfix;
   }
 };
@@ -579,7 +588,7 @@ struct AbiTagAttr : Node {
   std::string_view getBaseName() const override { return Base->getBaseName(); }
 
   void printLeft(OutputBuffer &OB) const override {
-    Base->printLeft(OB);
+    OB.printLeft(*Base);
     OB += "[abi:";
     OB += Tag;
     OB += "]";
@@ -646,7 +655,7 @@ class PointerType final : public Node {
     // We rewrite objc_object<SomeProtocol>* into id<SomeProtocol>.
     if (Pointee->getKind() != KObjCProtoName ||
         !static_cast<const ObjCProtoName *>(Pointee)->isObjCObject()) {
-      Pointee->printLeft(OB);
+      OB.printLeft(*Pointee);
       if (Pointee->hasArray(OB))
         OB += " ";
       if (Pointee->hasArray(OB) || Pointee->hasFunction(OB))
@@ -665,7 +674,7 @@ class PointerType final : public Node {
         !static_cast<const ObjCProtoName *>(Pointee)->isObjCObject()) {
       if (Pointee->hasArray(OB) || Pointee->hasFunction(OB))
         OB += ")";
-      Pointee->printRight(OB);
+      OB.printRight(*Pointee);
     }
   }
 };
@@ -731,7 +740,7 @@ class ReferenceType : public Node {
     std::pair<ReferenceKind, const Node *> Collapsed = collapse(OB);
     if (!Collapsed.second)
       return;
-    Collapsed.second->printLeft(OB);
+    OB.printLeft(*Collapsed.second);
     if (Collapsed.second->hasArray(OB))
       OB += " ";
     if (Collapsed.second->hasArray(OB) || Collapsed.second->hasFunction(OB))
@@ -748,7 +757,7 @@ class ReferenceType : public Node {
       return;
     if (Collapsed.second->hasArray(OB) || Collapsed.second->hasFunction(OB))
       OB += ")";
-    Collapsed.second->printRight(OB);
+    OB.printRight(*Collapsed.second);
   }
 };
 
@@ -768,7 +777,7 @@ class PointerToMemberType final : public Node {
   }
 
   void printLeft(OutputBuffer &OB) const override {
-    MemberType->printLeft(OB);
+    OB.printLeft(*MemberType);
     if (MemberType->hasArray(OB) || MemberType->hasFunction(OB))
       OB += "(";
     else
@@ -780,7 +789,7 @@ class PointerToMemberType final : public Node {
   void printRight(OutputBuffer &OB) const override {
     if (MemberType->hasArray(OB) || MemberType->hasFunction(OB))
       OB += ")";
-    MemberType->printRight(OB);
+    OB.printRight(*MemberType);
   }
 };
 
@@ -800,7 +809,7 @@ class ArrayType final : public Node {
   bool hasRHSComponentSlow(OutputBuffer &) const override { return true; }
   bool hasArraySlow(OutputBuffer &) const override { return true; }
 
-  void printLeft(OutputBuffer &OB) const override { Base->printLeft(OB); }
+  void printLeft(OutputBuffer &OB) const override { OB.printLeft(*Base); }
 
   void printRight(OutputBuffer &OB) const override {
     if (OB.back() != ']')
@@ -809,7 +818,7 @@ class ArrayType final : public Node {
     if (Dimension)
       Dimension->print(OB);
     OB += "]";
-    Base->printRight(OB);
+    OB.printRight(*Base);
   }
 
   bool printInitListAsType(OutputBuffer &OB,
@@ -853,7 +862,7 @@ class FunctionType final : public Node {
   // by printing out the return types's left, then print our parameters, then
   // finally print right of the return type.
   void printLeft(OutputBuffer &OB) const override {
-    Ret->printLeft(OB);
+    OB.printLeft(*Ret);
     OB += " ";
   }
 
@@ -861,7 +870,7 @@ class FunctionType final : public Node {
     OB.printOpen();
     Params.printWithComma(OB);
     OB.printClose();
-    Ret->printRight(OB);
+    OB.printRight(*Ret);
 
     if (CVQuals & QualConst)
       OB += " const";
@@ -966,6 +975,8 @@ class FunctionEncoding final : public Node {
   FunctionRefQual getRefQual() const { return RefQual; }
   NodeArray getParams() const { return Params; }
   const Node *getReturnType() const { return Ret; }
+  const Node *getAttrs() const { return Attrs; }
+  const Node *getRequires() const { return Requires; }
 
   bool hasRHSComponentSlow(OutputBuffer &) const override { return true; }
   bool hasFunctionSlow(OutputBuffer &) const override { return true; }
@@ -974,10 +985,11 @@ class FunctionEncoding final : public Node {
 
   void printLeft(OutputBuffer &OB) const override {
     if (Ret) {
-      Ret->printLeft(OB);
+      OB.printLeft(*Ret);
       if (!Ret->hasRHSComponent(OB))
         OB += " ";
     }
+
     Name->print(OB);
   }
 
@@ -985,8 +997,9 @@ class FunctionEncoding final : public Node {
     OB.printOpen();
     Params.printWithComma(OB);
     OB.printClose();
+
     if (Ret)
-      Ret->printRight(OB);
+      OB.printRight(*Ret);
 
     if (CVQuals & QualConst)
       OB += " const";
@@ -1326,14 +1339,14 @@ class NonTypeTemplateParamDecl final : public Node {
   template<typename Fn> void match(Fn F) const { F(Name, Type); }
 
   void printLeft(OutputBuffer &OB) const override {
-    Type->printLeft(OB);
+    OB.printLeft(*Type);
     if (!Type->hasRHSComponent(OB))
       OB += " ";
   }
 
   void printRight(OutputBuffer &OB) const override {
     Name->print(OB);
-    Type->printRight(OB);
+    OB.printRight(*Type);
   }
 };
 
@@ -1378,11 +1391,11 @@ class TemplateParamPackDecl final : public Node {
   template<typename Fn> void match(Fn F) const { F(Param); }
 
   void printLeft(OutputBuffer &OB) const override {
-    Param->printLeft(OB);
+    OB.printLeft(*Param);
     OB += "...";
   }
 
-  void printRight(OutputBuffer &OB) const override { Param->printRight(OB); }
+  void printRight(OutputBuffer &OB) const override { OB.printRight(*Param); }
 };
 
 /// An unexpanded parameter pack (either in the expression or type context). If
@@ -1447,13 +1460,13 @@ class ParameterPack final : public Node {
     initializePackExpansion(OB);
     size_t Idx = OB.CurrentPackIndex;
     if (Idx < Data.size())
-      Data[Idx]->printLeft(OB);
+      OB.printLeft(*Data[Idx]);
   }
   void printRight(OutputBuffer &OB) const override {
     initializePackExpansion(OB);
     size_t Idx = OB.CurrentPackIndex;
     if (Idx < Data.size())
-      Data[Idx]->printRight(OB);
+      OB.printRight(*Data[Idx]);
   }
 };
 
@@ -1611,13 +1624,13 @@ struct ForwardTemplateReference : Node {
     if (Printing)
       return;
     ScopedOverride<bool> SavePrinting(Printing, true);
-    Ref->printLeft(OB);
+    OB.printLeft(*Ref);
   }
   void printRight(OutputBuffer &OB) const override {
     if (Printing)
       return;
     ScopedOverride<bool> SavePrinting(Printing, true);
-    Ref->printRight(OB);
+    OB.printRight(*Ref);
   }
 };
 
@@ -1769,7 +1782,7 @@ class DtorName : public Node {
 
   void printLeft(OutputBuffer &OB) const override {
     OB += "~";
-    Base->printLeft(OB);
+    OB.printLeft(*Base);
   }
 };
 
@@ -2049,7 +2062,7 @@ class CastExpr : public Node {
     {
       ScopedOverride<unsigned> LT(OB.GtIsGt, 0);
       OB += "<";
-      To->printLeft(OB);
+      OB.printLeft(*To);
       OB += ">";
     }
     OB.printOpen();
@@ -6180,6 +6193,10 @@ struct ManglingParser : AbstractManglingParser<ManglingParser<Alloc>, Alloc> {
                                Alloc>::AbstractManglingParser;
 };
 
+inline void OutputBuffer::printLeft(const Node &N) { N.printLeft(*this); }
+
+inline void OutputBuffer::printRight(const Node &N) { N.printRight(*this); }
+
 DEMANGLE_NAMESPACE_END
 
 #if defined(__clang__)
diff --git a/llvm/include/llvm/Demangle/Utility.h b/llvm/include/llvm/Demangle/Utility.h
index e893cceea2cdc..d59d74511dd4f 100644
--- a/llvm/include/llvm/Demangle/Utility.h
+++ b/llvm/include/llvm/Demangle/Utility.h
@@ -27,6 +27,8 @@
 
 DEMANGLE_NAMESPACE_BEGIN
 
+class Node;
+
 // Stream that AST nodes write their string representation into after the AST
 // has been parsed.
 class OutputBuffer {
@@ -79,10 +81,24 @@ class OutputBuffer {
   OutputBuffer(const OutputBuffer &) = delete;
   OutputBuffer &operator=(const OutputBuffer &) = delete;
 
+  virtual ~OutputBuffer() {}
+
   operator std::string_view() const {
     return std::string_view(Buffer, CurrentPosition);
   }
 
+  /// Called by the demangler when printing the demangle tree. By
+  /// default calls into \c Node::print{Left|Right} but can be overriden
+  /// by clients to track additional state when printing the demangled name.
+  virtual void printLeft(const Node &N);
+  virtual void printRight(const Node &N);
+
+  /// Called when we write to this object anywhere other than the end.
+  virtual void notifyInsertion(size_t /*Position*/, size_t /*Count*/) {}
+
+  /// Called when we make the \c CurrentPosition of this object smaller.
+  virtual void notifyDeletion(size_t /*OldPos*/, size_t /*NewPos*/) {}
+
   /// If a ParameterPackExpansion (or similar type) is encountered, the offset
   /// into the pack that we're currently printing.
   unsigned CurrentPackIndex = std::numeric_limits<unsigned>::max();
@@ -126,6 +142,8 @@ class OutputBuffer {
     std::memcpy(Buffer, &*R.begin(), Size);
     CurrentPosition += Size;
 
+    notifyInsertion(/*Position=*/0, /*Count=*/Size);
+
     return *this;
   }
 
@@ -161,14 +179,20 @@ class OutputBuffer {
     DEMANGLE_ASSERT(Pos <= CurrentPosition, "");
     if (N == 0)
       return;
+
     grow(N);
     std::memmove(Buffer + Pos + N, Buffer + Pos, CurrentPosition - Pos);
     std::memcpy(Buffer + Pos, S, N);
     CurrentPosition += N;
+
+    notifyInsertion(Pos, N);
   }
 
   size_t getCurrentPosition() const { return CurrentPosition; }
-  void setCurrentPosition(size_t NewPos) { CurrentPosition = NewPos; }
+  void setCurrentPosition(size_t NewPos) {
+    notifyDeletion(CurrentPosition, NewPos);
+    CurrentPosition = NewPos;
+  }
 
   char back() const {
     DEMANGLE_ASSERT(CurrentPosition, "");
diff --git a/llvm/unittests/Demangle/ItaniumDemangleTest.cpp b/llvm/unittests/Demangle/ItaniumDemangleTest.cpp
index bc6ccc2e16e65..329f33215817a 100644
--- a/llvm/unittests/Demangle/ItaniumDemangleTest.cpp
+++ b/llvm/unittests/Demangle/ItaniumDemangleTest.cpp
@@ -98,7 +98,7 @@ TEST(ItaniumDemangle, HalfType) {
     Node *parseType() {
       OutputBuffer OB;
       Node *N = AbstractManglingParser<TestParser, TestAllocator>::parseType();
-      N->printLeft(OB);
+      OB.printLeft(*N);
       std::string_view Name = N->getBaseName();
       if (!Name.empty())
         Types.push_back(std::string(Name.begin(), Name.end()));
diff --git a/llvm/unittests/Demangle/OutputBufferTest.cpp b/llvm/unittests/Demangle/OutputBufferTest.cpp
index 76031e523d781..4a30e66eee48e 100644
--- a/llvm/unittests/Demangle/OutputBufferTest.cpp
+++ b/llvm/unittests/Demangle/OutputBufferTest.cpp
@@ -93,3 +93,40 @@ TEST(OutputBufferTest, Extend) {
 
   std::free(OB.getBuffer());
 }
+
+TEST(OutputBufferTest, Notifications) {
+  struct MyOutputBuffer : public OutputBuffer {
+    size_t Inserted = 0;
+    size_t LatestPos = 0;
+
+    void notifyDeletion(size_t OldPos, size_t NewPos) override {
+      LatestPos = NewPos;
+    }
+
+    void notifyInsertion(size_t Position, size_t Count) override {
+      Inserted += Count;
+      LatestPos = Position;
+    }
+  } OB;
+
+  OB.prepend("n");
+  EXPECT_EQ(OB.Inserted, 1U);
+  EXPECT_EQ(OB.LatestPos, 0U);
+
+  OB.prepend("");
+  EXPECT_EQ(OB.Inserted, 1U);
+  EXPECT_EQ(OB.LatestPos, 0U);
+
+  OB.prepend("abc");
+  EXPECT_EQ(OB.Inserted, 4U);
+  EXPECT_EQ(OB.LatestPos, 0U);
+
+  OB.insert(2, "abc", 3U);
+  EXPECT_EQ(OB.Inserted, 7U);
+  EXPECT_EQ(OB.LatestPos, 2U);
+
+  OB.setCurrentPosition(3U);
+  EXPECT_EQ(OB.LatestPos, 3U);
+
+  std::free(OB.getBuffer());
+}

From a0b2a952266a22ed096a7670e33a611c145e3866 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Thu, 17 Apr 2025 14:03:49 -0700
Subject: [PATCH 323/710] [SandboxIR] Implement ConstantDataArray functions
 (#134729)

Mirrors LLVM IR.
---
 llvm/include/llvm/SandboxIR/Constant.h     | 67 +++++++++++++++++++++-
 llvm/include/llvm/SandboxIR/Type.h         |  1 +
 llvm/include/llvm/SandboxIR/Value.h        |  1 +
 llvm/lib/SandboxIR/Type.cpp                |  3 +
 llvm/unittests/SandboxIR/SandboxIRTest.cpp | 51 +++++++++++++++-
 5 files changed, 120 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/Constant.h b/llvm/include/llvm/SandboxIR/Constant.h
index ddd90291c509b..fa00b29dbd803 100644
--- a/llvm/include/llvm/SandboxIR/Constant.h
+++ b/llvm/include/llvm/SandboxIR/Constant.h
@@ -591,7 +591,72 @@ class ConstantDataArray final : public ConstantDataSequential {
   friend class Context;
 
 public:
-  // TODO: Add missing functions.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::ConstantDataArray;
+  }
+  /// get() constructor - Return a constant with array type with an element
+  /// count and element type matching the ArrayRef passed in.  Note that this
+  /// can return a ConstantAggregateZero object.
+  template <typename ElementTy>
+  static Constant *get(Context &Ctx, ArrayRef<ElementTy> Elts) {
+    auto *NewLLVMC = llvm::ConstantDataArray::get(Ctx.LLVMCtx, Elts);
+    return Ctx.getOrCreateConstant(NewLLVMC);
+  }
+
+  /// get() constructor - ArrayTy needs to be compatible with
+  /// ArrayRef<ElementTy>.
+  template <typename ArrayTy>
+  static Constant *get(Context &Ctx, ArrayTy &Elts) {
+    return ConstantDataArray::get(Ctx, ArrayRef(Elts));
+  }
+
+  /// getRaw() constructor - Return a constant with array type with an element
+  /// count and element type matching the NumElements and ElementTy parameters
+  /// passed in. Note that this can return a ConstantAggregateZero object.
+  /// ElementTy must be one of i8/i16/i32/i64/half/bfloat/float/double. Data is
+  /// the buffer containing the elements. Be careful to make sure Data uses the
+  /// right endianness, the buffer will be used as-is.
+  static Constant *getRaw(StringRef Data, uint64_t NumElements,
+                          Type *ElementTy) {
+    auto *LLVMC =
+        llvm::ConstantDataArray::getRaw(Data, NumElements, ElementTy->LLVMTy);
+    return ElementTy->getContext().getOrCreateConstant(LLVMC);
+  }
+  /// getFP() constructors - Return a constant of array type with a float
+  /// element type taken from argument `ElementType', and count taken from
+  /// argument `Elts'.  The amount of bits of the contained type must match the
+  /// number of bits of the type contained in the passed in ArrayRef.
+  /// (i.e. half or bfloat for 16bits, float for 32bits, double for 64bits) Note
+  /// that this can return a ConstantAggregateZero object.
+  static Constant *getFP(Type *ElementType, ArrayRef<uint16_t> Elts) {
+    auto *LLVMC = llvm::ConstantDataArray::getFP(ElementType->LLVMTy, Elts);
+    return ElementType->getContext().getOrCreateConstant(LLVMC);
+  }
+  static Constant *getFP(Type *ElementType, ArrayRef<uint32_t> Elts) {
+    auto *LLVMC = llvm::ConstantDataArray::getFP(ElementType->LLVMTy, Elts);
+    return ElementType->getContext().getOrCreateConstant(LLVMC);
+  }
+  static Constant *getFP(Type *ElementType, ArrayRef<uint64_t> Elts) {
+    auto *LLVMC = llvm::ConstantDataArray::getFP(ElementType->LLVMTy, Elts);
+    return ElementType->getContext().getOrCreateConstant(LLVMC);
+  }
+  /// This method constructs a CDS and initializes it with a text string.
+  /// The default behavior (AddNull==true) causes a null terminator to
+  /// be placed at the end of the array (increasing the length of the string by
+  /// one more than the StringRef would normally indicate.  Pass AddNull=false
+  /// to disable this behavior.
+  static Constant *getString(Context &Ctx, StringRef Initializer,
+                             bool AddNull = true) {
+    auto *LLVMC =
+        llvm::ConstantDataArray::getString(Ctx.LLVMCtx, Initializer, AddNull);
+    return Ctx.getOrCreateConstant(LLVMC);
+  }
+
+  /// Specialize the getType() method to always return an ArrayType,
+  /// which reduces the amount of casting needed in parts of the compiler.
+  inline ArrayType *getType() const {
+    return cast<ArrayType>(Value::getType());
+  }
 };
 
 /// A vector constant whose element type is a simple 1/2/4/8-byte integer or
diff --git a/llvm/include/llvm/SandboxIR/Type.h b/llvm/include/llvm/SandboxIR/Type.h
index 46032ba1084ff..f90ae096443b5 100644
--- a/llvm/include/llvm/SandboxIR/Type.h
+++ b/llvm/include/llvm/SandboxIR/Type.h
@@ -275,6 +275,7 @@ class Type {
   static Type *getInt1Ty(Context &Ctx);
   static Type *getDoubleTy(Context &Ctx);
   static Type *getFloatTy(Context &Ctx);
+  static Type *getHalfTy(Context &Ctx);
   // TODO: missing get*
 
   /// Get the address space of this pointer or pointer vector type.
diff --git a/llvm/include/llvm/SandboxIR/Value.h b/llvm/include/llvm/SandboxIR/Value.h
index 508978709f96c..d45aa4059de69 100644
--- a/llvm/include/llvm/SandboxIR/Value.h
+++ b/llvm/include/llvm/SandboxIR/Value.h
@@ -170,6 +170,7 @@ class Value {
   // expose metadata in sandboxir.
   friend class Region;
   friend class ScoreBoard; // Needs access to `Val` for the instruction cost.
+  friend class ConstantDataArray; // For `Val`
 
   /// All values point to the context.
   Context &Ctx;
diff --git a/llvm/lib/SandboxIR/Type.cpp b/llvm/lib/SandboxIR/Type.cpp
index 51128bf307e9f..dfb54df0c9ce8 100644
--- a/llvm/lib/SandboxIR/Type.cpp
+++ b/llvm/lib/SandboxIR/Type.cpp
@@ -36,6 +36,9 @@ Type *Type::getDoubleTy(Context &Ctx) {
 Type *Type::getFloatTy(Context &Ctx) {
   return Ctx.getType(llvm::Type::getFloatTy(Ctx.LLVMCtx));
 }
+Type *Type::getHalfTy(Context &Ctx) {
+  return Ctx.getType(llvm::Type::getHalfTy(Ctx.LLVMCtx));
+}
 
 #ifndef NDEBUG
 void Type::dumpOS(raw_ostream &OS) { LLVMTy->print(OS); }
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 690e9a521c168..8cce659596a4d 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -298,9 +298,10 @@ define void @foo(i32 %v0) {
 
 TEST_F(SandboxIRTest, ConstantFP) {
   parseIR(C, R"IR(
-define void @foo(float %v0, double %v1) {
+define void @foo(float %v0, double %v1, half %v2) {
   %fadd0 = fadd float %v0, 42.0
   %fadd1 = fadd double %v1, 43.0
+  %fadd2 = fadd half %v2, 44.0
   ret void
 }
 )IR");
@@ -312,12 +313,16 @@ define void @foo(float %v0, double %v1) {
   auto It = BB.begin();
   auto *FAdd0 = cast<sandboxir::BinaryOperator>(&*It++);
   auto *FAdd1 = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *FAdd2 = cast<sandboxir::BinaryOperator>(&*It++);
   auto *FortyTwo = cast<sandboxir::ConstantFP>(FAdd0->getOperand(1));
   [[maybe_unused]] auto *FortyThree =
       cast<sandboxir::ConstantFP>(FAdd1->getOperand(1));
 
   auto *FloatTy = sandboxir::Type::getFloatTy(Ctx);
   auto *DoubleTy = sandboxir::Type::getDoubleTy(Ctx);
+  auto *HalfTy = sandboxir::Type::getHalfTy(Ctx);
+  EXPECT_EQ(HalfTy, Ctx.getType(llvm::Type::getHalfTy(C)));
+  EXPECT_EQ(FAdd2->getType(), HalfTy);
   auto *LLVMFloatTy = Type::getFloatTy(C);
   auto *LLVMDoubleTy = Type::getDoubleTy(C);
   // Check that creating an identical constant gives us the same object.
@@ -616,6 +621,7 @@ define void @foo() {
   %farray = extractvalue [2 x float] [float 0.0, float 1.0], 0
   %fvector = extractelement <2 x double> <double 0.0, double 1.0>, i32 0
   %string = extractvalue [6 x i8] [i8 72, i8 69, i8 76, i8 76, i8 79, i8 0], 0
+  %stringNoNull = extractvalue [5 x i8] [i8 72, i8 69, i8 76, i8 76, i8 79], 0
   ret void
 }
 )IR");
@@ -630,16 +636,19 @@ define void @foo() {
   auto *I2 = &*It++;
   auto *I3 = &*It++;
   auto *I4 = &*It++;
+  auto *I5 = &*It++;
   auto *Array = cast<sandboxir::ConstantDataArray>(I0->getOperand(0));
   EXPECT_TRUE(isa<sandboxir::ConstantDataSequential>(Array));
   auto *Vector = cast<sandboxir::ConstantDataVector>(I1->getOperand(0));
   EXPECT_TRUE(isa<sandboxir::ConstantDataVector>(Vector));
   auto *FArray = cast<sandboxir::ConstantDataArray>(I2->getOperand(0));
   EXPECT_TRUE(isa<sandboxir::ConstantDataSequential>(FArray));
-  auto *FVector = cast<sandboxir::ConstantDataArray>(I3->getOperand(0));
+  auto *FVector = cast<sandboxir::ConstantDataVector>(I3->getOperand(0));
   EXPECT_TRUE(isa<sandboxir::ConstantDataVector>(FVector));
   auto *String = cast<sandboxir::ConstantDataArray>(I4->getOperand(0));
   EXPECT_TRUE(isa<sandboxir::ConstantDataArray>(String));
+  auto *StringNoNull = cast<sandboxir::ConstantDataArray>(I5->getOperand(0));
+  EXPECT_TRUE(isa<sandboxir::ConstantDataArray>(StringNoNull));
 
   auto *Zero8 = sandboxir::ConstantInt::get(sandboxir::Type::getInt8Ty(Ctx), 0);
   auto *One8 = sandboxir::ConstantInt::get(sandboxir::Type::getInt8Ty(Ctx), 1);
@@ -706,6 +715,44 @@ define void @foo() {
   EXPECT_EQ(String->getAsCString(), "HELLO");
   // Check getRawDataValues().
   EXPECT_EQ(String->getRawDataValues(), HelloWithNull);
+
+  // Check ConstantDataArray member functions
+  // ----------------------------------------
+  // Check get<ElementTy>().
+  EXPECT_EQ(sandboxir::ConstantDataArray::get<char>(Ctx, {0, 1}), Array);
+  // Check get<ArrayTy>().
+  SmallVector<char> Elmts({0, 1});
+  EXPECT_EQ(sandboxir::ConstantDataArray::get<SmallVector<char>>(Ctx, Elmts),
+            Array);
+  // Check getRaw().
+  EXPECT_EQ(sandboxir::ConstantDataArray::getRaw(StringRef("HELLO"), 5,
+                                                 Zero8->getType()),
+            StringNoNull);
+  // Check getFP().
+  SmallVector<uint16_t> Elts16({42, 43});
+  SmallVector<uint32_t> Elts32({42, 43});
+  SmallVector<uint64_t> Elts64({42, 43});
+  auto *F16Ty = sandboxir::Type::getHalfTy(Ctx);
+  auto *F32Ty = sandboxir::Type::getFloatTy(Ctx);
+  auto *F64Ty = sandboxir::Type::getDoubleTy(Ctx);
+
+  auto *CDA16 = sandboxir::ConstantDataArray::getFP(F16Ty, Elts16);
+  EXPECT_EQ(CDA16, cast<sandboxir::ConstantDataArray>(
+                       Ctx.getValue(llvm::ConstantDataArray::getFP(
+                           llvm::Type::getHalfTy(C), Elts16))));
+  auto *CDA32 = sandboxir::ConstantDataArray::getFP(F32Ty, Elts32);
+  EXPECT_EQ(CDA32, cast<sandboxir::ConstantDataArray>(
+                       Ctx.getValue(llvm::ConstantDataArray::getFP(
+                           llvm::Type::getFloatTy(C), Elts32))));
+  auto *CDA64 = sandboxir::ConstantDataArray::getFP(F64Ty, Elts64);
+  EXPECT_EQ(CDA64, cast<sandboxir::ConstantDataArray>(
+                       Ctx.getValue(llvm::ConstantDataArray::getFP(
+                           llvm::Type::getDoubleTy(C), Elts64))));
+  // Check getString().
+  EXPECT_EQ(sandboxir::ConstantDataArray::getString(Ctx, "HELLO"), String);
+  EXPECT_EQ(sandboxir::ConstantDataArray::getString(Ctx, "HELLO",
+                                                    /*AddNull=*/false),
+            StringNoNull);
 }
 
 TEST_F(SandboxIRTest, ConstantPointerNull) {

From 12becfff035a33141a0b2fb3ea5d5558738ce7eb Mon Sep 17 00:00:00 2001
From: Felipe de Azevedo Piovezan <fpiovezan@apple.com>
Date: Thu, 17 Apr 2025 14:13:28 -0700
Subject: [PATCH 324/710] [lldb] Create ThreadPlanStepOut ctor that never skips
 frames (#136163)

The function QueueThreadPlanForStepOutNoShouldStop has the semantics of
"go this parent frame"; ThreadPlanStepOut needs to respect that, not
skipping over any frames it finds uninteresting. This commit creates a
constructor that respects such instruction.
---
 lldb/include/lldb/Target/ThreadPlanStepOut.h |  9 ++++++++
 lldb/source/Target/Thread.cpp                |  5 ++---
 lldb/source/Target/ThreadPlanStepOut.cpp     | 22 ++++++++++++++++++++
 3 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/lldb/include/lldb/Target/ThreadPlanStepOut.h b/lldb/include/lldb/Target/ThreadPlanStepOut.h
index e414a6e0a2d49..ad4434be14120 100644
--- a/lldb/include/lldb/Target/ThreadPlanStepOut.h
+++ b/lldb/include/lldb/Target/ThreadPlanStepOut.h
@@ -17,6 +17,9 @@ namespace lldb_private {
 
 class ThreadPlanStepOut : public ThreadPlan, public ThreadPlanShouldStopHere {
 public:
+  /// Creates a thread plan to step out from frame_idx, skipping parent frames
+  /// if they are artificial or hidden frames. Also skips frames without debug
+  /// info based on step_out_avoids_code_without_debug_info.
   ThreadPlanStepOut(Thread &thread, SymbolContext *addr_context,
                     bool first_insn, bool stop_others, Vote report_stop_vote,
                     Vote report_run_vote, uint32_t frame_idx,
@@ -24,6 +27,12 @@ class ThreadPlanStepOut : public ThreadPlan, public ThreadPlanShouldStopHere {
                     bool continue_to_next_branch = false,
                     bool gather_return_value = true);
 
+  /// Creates a thread plan to step out from frame_idx to frame_idx + 1.
+  ThreadPlanStepOut(Thread &thread, bool stop_others, Vote report_stop_vote,
+                    Vote report_run_vote, uint32_t frame_idx,
+                    bool continue_to_next_branch = false,
+                    bool gather_return_value = true);
+
   ~ThreadPlanStepOut() override;
 
   void GetDescription(Stream *s, lldb::DescriptionLevel level) override;
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index accc4708c24e1..b0e0f1e67c060 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -1360,9 +1360,8 @@ ThreadPlanSP Thread::QueueThreadPlanForStepOutNoShouldStop(
   const bool calculate_return_value =
       false; // No need to calculate the return value here.
   ThreadPlanSP thread_plan_sp(new ThreadPlanStepOut(
-      *this, addr_context, first_insn, stop_other_threads, report_stop_vote,
-      report_run_vote, frame_idx, eLazyBoolNo, continue_to_next_branch,
-      calculate_return_value));
+      *this, stop_other_threads, report_stop_vote, report_run_vote, frame_idx,
+      continue_to_next_branch, calculate_return_value));
 
   ThreadPlanStepOut *new_plan =
       static_cast<ThreadPlanStepOut *>(thread_plan_sp.get());
diff --git a/lldb/source/Target/ThreadPlanStepOut.cpp b/lldb/source/Target/ThreadPlanStepOut.cpp
index f2606403016a6..0628451a5abf9 100644
--- a/lldb/source/Target/ThreadPlanStepOut.cpp
+++ b/lldb/source/Target/ThreadPlanStepOut.cpp
@@ -83,6 +83,28 @@ ThreadPlanStepOut::ThreadPlanStepOut(
                      continue_to_next_branch);
 }
 
+ThreadPlanStepOut::ThreadPlanStepOut(Thread &thread, bool stop_others,
+                                     Vote report_stop_vote,
+                                     Vote report_run_vote, uint32_t frame_idx,
+                                     bool continue_to_next_branch,
+                                     bool gather_return_value)
+    : ThreadPlan(ThreadPlan::eKindStepOut, "Step out", thread, report_stop_vote,
+                 report_run_vote),
+      ThreadPlanShouldStopHere(this), m_return_bp_id(LLDB_INVALID_BREAK_ID),
+      m_return_addr(LLDB_INVALID_ADDRESS), m_stop_others(stop_others),
+      m_immediate_step_from_function(nullptr),
+      m_calculate_return_value(gather_return_value) {
+  SetFlagsToDefault();
+  m_step_from_insn = thread.GetRegisterContext()->GetPC(0);
+
+  StackFrameSP return_frame_sp = thread.GetStackFrameAtIndex(frame_idx + 1);
+  StackFrameSP immediate_return_from_sp =
+      thread.GetStackFrameAtIndex(frame_idx);
+
+  SetupReturnAddress(return_frame_sp, immediate_return_from_sp, frame_idx,
+                     continue_to_next_branch);
+}
+
 void ThreadPlanStepOut::SetupReturnAddress(
     StackFrameSP return_frame_sp, StackFrameSP immediate_return_from_sp,
     uint32_t frame_idx, bool continue_to_next_branch) {

From e8b506b2469cfe61efdb3c919aaf2d6c54654782 Mon Sep 17 00:00:00 2001
From: Chao Chen <chao.chen@intel.com>
Date: Thu, 17 Apr 2025 16:15:34 -0500
Subject: [PATCH 325/710] [MLIR][XeGPU] Switch to 1D representation for SIMT
 code (#135116)

This PR switches to using a 1D vector to represent SIMT code for simplification.
---
 .../include/mlir/Dialect/XeGPU/IR/XeGPUOps.td |  19 +-
 .../mlir/Dialect/XeGPU/IR/XeGPUTypes.td       |   3 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp    |  39 ++-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        | 298 +++++++++---------
 mlir/test/Dialect/XeGPU/invalid.mlir          | 102 ++----
 mlir/test/Dialect/XeGPU/ops.mlir              | 162 +++++-----
 6 files changed, 281 insertions(+), 342 deletions(-)

diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
index 16a7f63d60c82..5fa18754305ca 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
@@ -833,16 +833,16 @@ def XeGPU_DpasOp : XeGPU_Op<"dpas", [Pure, AllElementTypesMatch<["lhs", "rhs"]>]
     data type, the matrices are `A: vector<8x16xf16>`, `B: vector<16x16xf16>`,
     and `C/D: vector<8x16xf32>`. Besides the matrix size requirements, DPAS
     also requires A and B to be loaded with the required data layout. Specially,
-
     VNNI layout is required for B operand. It is achieved via adding `packed`
     attribute to the `load_nd` operator.  Due to the VNNI transformation, B operands
     can be represented as a 3D vector, with the last dimension representing the VNNI
     factor, which is computed as `32/bit_width_of_elem_type`. Thus, `B: vector<16x16xf16>`
     can be represented as `B: vector<8x16x2xf16>`.
 
-    In SIMT mode, DpasOp expects layout attributes `a`, `b`, and `c` (only if acc is used)
-    which describe the data fragment owned by each work-item w.r.t. the tensor descriptor
-    these data are loaded from.
+    In SIMT code, each work-item from a subgroup holds a data fragment for A, B, C and the result,
+    which are represented as 1D vectors. Please refer to [OpenCL Intel extentions]
+    (https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_subgroup_matrix_multiply_accumulate.html)
+    for more details about the fragment distribution.
 
     Note: on PVC, the hardware can perform load with VNNI transformation when data
           element type is 16-bit or lower precision, taking 2 or 4 elements from
@@ -850,13 +850,10 @@ def XeGPU_DpasOp : XeGPU_Op<"dpas", [Pure, AllElementTypesMatch<["lhs", "rhs"]>]
   }];
 
   let arguments = (ins
-    XeGPU_DpasOpType : $lhs,
-    XeGPU_DpasOpType : $rhs,
-    Optional<XeGPU_Vector2DType>: $acc,
-    OptionalAttr<XeGPU_LayoutAttr>:$a_layout,
-    OptionalAttr<XeGPU_LayoutAttr>:$b_layout,
-    OptionalAttr<XeGPU_LayoutAttr>:$c_layout);
-  let results = (outs XeGPU_Vector2DType: $result);
+    XeGPU_DpasOprType : $lhs,
+    XeGPU_DpasOprType : $rhs,
+    Optional<XeGPU_DpasResType>: $acc);
+  let results = (outs XeGPU_DpasResType: $result);
 
   let extraClassDeclaration = [{
     VectorType getLhsType() {
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
index 173f1462fdd73..3cb71788a15ef 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
@@ -17,7 +17,8 @@ def XeGPU_IntType: AnyTypeOf<[I1, I8, I16, I32, I64, SI1, SI8, SI16, SI32, SI64,
 def XeGPU_FloatType: AnyTypeOf<[F16, F32, F64, BF16, TF32]>;
 def XeGPU_ScalarType: AnyTypeOf<[XeGPU_IntType, XeGPU_FloatType]>;
 def XeGPU_BaseAddrType: AnyTypeOf<[Non0RankedMemRefOf<[XeGPU_ScalarType]>, UI64, UI32, I64, I32]>;
-def XeGPU_DpasOpType: VectorOfRankAndType<[2, 3], [XeGPU_ScalarType]>;
+def XeGPU_DpasOprType: VectorOfRankAndType<[1, 2, 3], [XeGPU_ScalarType]>;
+def XeGPU_DpasResType: VectorOfRankAndType<[1, 2], [XeGPU_ScalarType]>;
 def XeGPU_OffsetType: VectorOfRankAndType<[1], [Index]>;
 def XeGPU_MaskType: AnyTypeOf<[VectorOfRankAndType<[1], [I1]>, I1]>;
 def XeGPU_ValueType: AnyTypeOf<[VectorOfRankAndType<[1,2,3,4], [XeGPU_ScalarType]>, XeGPU_ScalarType]>;
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
index 171a15ce27b59..b865b80f0075e 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
@@ -10,6 +10,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include <numeric>
 
 namespace mlir {
 namespace xegpu {
@@ -319,36 +320,38 @@ LogicalResult TensorDescType::verify(
 // ---------------------------------------------------------------------
 // Case 1: Regular loads/stores.
 // ---------------------------------------------------------------------
-// Distributed vector shape must be:
-//        [chunk_size / lane_data_size, lane_data_size]
-// If the tensor descriptor shape is 1D, first dimension is ignored (set to 1).
-//        [lane_data_size]
+// The following conditions must be met:
+//        * tensor_desc[0] == lane_layout[0]
+// Distributed vector is a 1D vector with shape:
+//        [chunk_size]
 // ---------------------------------------------------------------------
 // Case 2: Block loads/stores
 // ---------------------------------------------------------------------
 // Additional definitions:
 //        tensor_size = tensor_desc[0] * .. * tensor_desc[r-1] * array_length
 //        n_distribution_units = tensor_size / distribution_unit_size
+//        fragment_size = n_distribution_units * lane_data_size
 // Given above definitions, the following conditions must be met:
 //        * tensor_desc[0] % (lane_layout[0] × lane_data[0]) == 0
 //        * tensor_desc[1] % (lane_layout[1] × lane_data[1]) == 0
-// Distributed vector shape must be:
-//        [n_distribution_units, lane_data_size]
+// Distributed vector is a 1D vector with shape:
+//        [fragment_size]
 FailureOr<VectorType> TensorDescType::getDistributedVectorType() {
   auto layout = llvm::dyn_cast_if_present<LayoutAttr>(getLayout());
-  // If no layout is provided, tensor desc is not used in SIMT mode.
-  if (!layout)
+  // It only works for subgroup level layout, which only has lane_layout
+  // and lane_data, and is to distribute a SIMD code into SIMT code.
+  if (!layout || !layout.isSgLayout())
     return failure();
 
   SmallVector<int64_t> laneData(layout.getLaneData().asArrayRef());
   SmallVector<int64_t> laneLayout(layout.getLaneLayout().asArrayRef());
   auto tdescShape = getShape();
 
-  auto laneDataSize = 1, sgSize = 1;
-  for (auto [laneDim, laneDataDim] : llvm::zip_equal(laneLayout, laneData)) {
-    laneDataSize *= laneDataDim;
-    sgSize *= laneDim;
-  }
+  // compute sgSize by multiply elements of laneLayout
+  // e.g. for 2D layout, sgSize = laneLayout[0] * laneLayout[1]
+  // e.g. for 1D layout, sgSize = laneLayout[0]
+  auto sgSize = std::accumulate(laneLayout.begin(), laneLayout.end(), 1,
+                                std::multiplies<int64_t>());
 
   // Case 1: regular loads/stores
   auto scatterAttr = getEncodingAsScatterTensorDescAttr();
@@ -356,12 +359,9 @@ FailureOr<VectorType> TensorDescType::getDistributedVectorType() {
     auto chunkSize = scatterAttr.getChunkSize().getInt();
     // Verify if the first dimension of the tensor descriptor shape is
     // distributable.
-    assert(tdescShape[0] % (laneLayout[0]) == 0 &&
+    assert(tdescShape[0] == laneLayout[0] &&
            "tensor descriptor shape is not distributable");
-    if (chunkSize > 1)
-      return VectorType::get({chunkSize / laneDataSize, laneDataSize},
-                             getElementType());
-    return VectorType::get({laneDataSize}, getElementType());
+    return VectorType::get({chunkSize}, getElementType());
   }
 
   // Case 2: block loads/stores
@@ -376,8 +376,7 @@ FailureOr<VectorType> TensorDescType::getDistributedVectorType() {
   // tensorSize must be adjusted for array_length.
   tensorSize *= getArrayLength();
 
-  return VectorType::get({tensorSize / (sgSize * laneDataSize), laneDataSize},
-                         getElementType());
+  return VectorType::get({tensorSize / sgSize}, getElementType());
 }
 
 } // namespace xegpu
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 0d67e3d70f945..e0e25365220b5 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -73,38 +73,6 @@ static bool isWriteHintOrNone(const CachePolicyAttr &attr) {
          kind == CachePolicy::WRITE_BACK || kind == CachePolicy::WRITE_THROUGH;
 }
 
-// Helper to validate value shape of LoadNd and StoreNd ops.
-static LogicalResult
-isArgShapesValid(TensorDescType tdescTy, VectorType valueTy,
-                 ArrayRef<int64_t> adjustedTdescShape,
-                 function_ref<InFlightDiagnostic()> emitError) {
-  auto layout = tdescTy.getLayoutAttr();
-  auto valueShape = valueTy.getShape();
-  // layout not present means IR is in SIMD mode. In this case value shape must
-  // match adjusted tensor descriptor shape.
-  if (!layout)
-    return valueShape == adjustedTdescShape
-               ? success()
-               : emitError()
-                     << "Value shape " << makeString(valueShape)
-                     << " is not consistent with tensor descriptor " << tdescTy;
-
-  // layout present means IR is in SIMT mode. In this case layout determines the
-  // value shape.
-  auto expectedValueShapeOrFailure = tdescTy.getDistributedVectorType();
-  assert(succeeded(expectedValueShapeOrFailure) &&
-         "Failed to compute distributed vector shape for "
-         "tensor descriptor ");
-
-  return valueTy == expectedValueShapeOrFailure.value()
-             ? success()
-             : emitError()
-                   << "Result shape " << makeString(valueShape)
-                   << " is not consistent with distributed vector shape "
-                   << makeString(expectedValueShapeOrFailure.value().getShape())
-                   << " for tensor descriptor " << tdescTy;
-}
-
 // Checks if the given shape is evenly distributed based on the layout
 // and data factors provided by the LayoutAttr. The function ensures that
 // each dimension of the shape can be evenly divided by the corresponding
@@ -133,6 +101,53 @@ static bool isEvenDistributed(llvm::ArrayRef<int64_t> shape,
   return true;
 }
 
+static LogicalResult
+isValidGatherScatterParams(Type maskTy, VectorType valueTy,
+                           TensorDescType tdescTy, UnitAttr transposeAttr,
+                           function_ref<InFlightDiagnostic()> emitError) {
+
+  if (!tdescTy.isScattered())
+    return emitError() << "Expects a scattered TensorDesc.";
+
+  if (!valueTy)
+    return emitError() << "Expecting a vector type result.";
+
+  auto maskShape = getShapeOf(maskTy);
+  auto valueShape = getShapeOf(valueTy);
+  auto tdescShape = getShapeOf(tdescTy);
+  auto chunkSize = tdescTy.getChunkSize();
+
+  if (valueTy.getElementType() != tdescTy.getElementType())
+    return emitError()
+           << "Value should have the same element type as TensorDesc.";
+
+  if (tdescShape[0] != maskShape[0])
+    return emitError()
+           << "dim-0 of the Mask and TensorDesc should be the same.";
+
+  // a valid shape for SIMT case
+  if (valueTy.getRank() == 1 && valueTy.getNumElements() == chunkSize) {
+    if (tdescTy.getLayoutAttr())
+      return emitError() << "TensorDesc doesn't need LayoutAttr for SIMT code";
+    if (transposeAttr)
+      return emitError() << "doesn't need TransposeAttr for SIMT code";
+    return success();
+  }
+
+  if (tdescTy.getRank() == 2 && valueTy.getRank() == 2) {
+    if (!transposeAttr)
+      return emitError() << "rank-2 tensor has to be transposed.";
+    transpose({1, 0}, tdescShape);
+  }
+
+  if (tdescShape != valueShape)
+    return emitError() << "Value shape " << makeString(valueShape)
+                       << " is neither a valid distribution for SIMT nor "
+                          "consistent with the tensor descriptor for SIMD "
+                       << tdescTy;
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // XeGPU_CreateNdDescOp
 //===----------------------------------------------------------------------===//
@@ -302,9 +317,32 @@ LogicalResult LoadNdOp::verify() {
   if (!isReadHintOrNone(getL3HintAttr()))
     return emitOpError("invalid l3_hint: ") << getL3HintAttr();
 
-  auto array_len = tdescTy.getArrayLength();
-  // adjusted tensor descriptor shape tracks the expected shape of the result.
-  auto adjustedTdescShape = getShapeOf(tdescTy);
+  int tdescElems = tdescTy.getNumElements() * tdescTy.getArrayLength();
+  int valueElems = valueTy.getNumElements();
+
+  // If the result vector is 1D and has less elements than the tensor
+  // descriptor, it is supposed to be a SIMT op. The layout attribute in
+  // tensor_desc is not needed.
+  if (valueElems < tdescElems && valueTy.getRank() == 1) {
+    // SIMT mode doesn't need LayoutAttr.
+    if (tdescTy.getLayoutAttr())
+      return emitOpError()
+             << "TensorDesc doesn't need LayoutAttr for SIMT code";
+
+    // For SIMT code, the load is evenly distributed across all lanes in a
+    // subgroup. Since subgroup size is arch dependent, we only check even
+    // distribution here.
+    if (tdescElems % valueElems)
+      return emitOpError()
+             << "Result shape " << makeString(getShapeOf(valueTy))
+             << " is not a valid distribution for tensor descriptor "
+             << tdescTy;
+
+    return success();
+  }
+
+  // Check SIMD mode.
+  auto tdescShape = getShapeOf(tdescTy);
   auto valueShape = getShapeOf(valueTy);
 
   if (getTranspose()) {
@@ -316,7 +354,7 @@ LogicalResult LoadNdOp::verify() {
     });
 
     if (valid)
-      transpose(trans, adjustedTdescShape);
+      transpose(trans, tdescShape);
     else
       mlir::emitWarning(getLoc()) << "Invalid transpose attr. It is ignored.";
   }
@@ -325,8 +363,8 @@ LogicalResult LoadNdOp::verify() {
     if (tdescTy.getRank() == 2) {
       const int axis = 0;
       auto vnni_factor = valueShape.back();
-      adjustedTdescShape[axis] /= vnni_factor;
-      adjustedTdescShape.push_back(vnni_factor);
+      tdescShape[axis] /= vnni_factor;
+      tdescShape.push_back(vnni_factor);
     } else {
       mlir::emitWarning(getLoc())
           << "Invalid Packed Attr. It is ignored (available for 2D "
@@ -334,13 +372,18 @@ LogicalResult LoadNdOp::verify() {
     }
   }
 
+  auto array_len = tdescTy.getArrayLength();
   if (array_len > 1) {
-    auto it = adjustedTdescShape.begin();
-    adjustedTdescShape.insert(it, array_len);
+    tdescShape.insert(tdescShape.begin(), array_len);
   }
 
-  return isArgShapesValid(tdescTy, valueTy, adjustedTdescShape,
-                          [&]() { return emitOpError(); });
+  if (tdescShape != valueShape) {
+    return emitOpError() << "Result shape " << makeString(valueShape)
+                         << " is not consistent with tensor descriptor "
+                         << tdescTy;
+  }
+
+  return success();
 }
 
 //===----------------------------------------------------------------------===//
@@ -368,11 +411,40 @@ LogicalResult StoreNdOp::verify() {
   if (!isWriteHintOrNone(getL3HintAttr()))
     return emitOpError("invalid l3_hint: ") << getL3HintAttr();
 
+  auto array_len = dstTy.getArrayLength();
+  if (array_len > 1)
+    return emitOpError("array length is not supported by store_nd.\n");
+
+  auto tdescElems = dstTy.getNumElements();
+  auto valueElems = valTy.getNumElements();
+
+  // Similar to LoadNdOp, if the value vector is 1D and has less elements than
+  // the tensor descriptor, it is supposed to be a SIMT op. The layout attribute
+  // in tensor_desc is not needed.
+  if (valTy.getRank() == 1 && valueElems < tdescElems) {
+    // SIMT mode doesn't need LayoutAttr.
+    if (dstTy.getLayoutAttr())
+      return emitOpError()
+             << "TensorDesc doesn't need LayoutAttr for SIMT code";
+
+    if (tdescElems % valueElems) {
+      return emitOpError()
+             << "Value shape " << makeString(getShapeOf(valTy))
+             << " is not a valid distribution for tensor descriptor " << dstTy;
+    }
+    return success();
+  }
+
+  // SIMD code should have the same shape as the tensor descriptor.
   auto tdescShape = getShapeOf(dstTy);
   auto valueShape = getShapeOf(valTy);
+  if (tdescShape != valueShape) {
+    return emitOpError() << "Value shape " << makeString(valueShape)
+                         << " is not consistent with tensor descriptor "
+                         << dstTy;
+  }
 
-  return isArgShapesValid(dstTy, valTy, tdescShape,
-                          [&]() { return emitOpError(); });
+  return success();
 }
 
 //===----------------------------------------------------------------------===//
@@ -492,12 +564,6 @@ LogicalResult LoadGatherOp::verify() {
   auto maskTy = getMaskType();
   auto valueTy = getValueType();
 
-  if (!valueTy)
-    return emitOpError("Expecting a vector type result.\n");
-
-  if (!tdescTy.isScattered())
-    return emitOpError("Expects a scattered TensorDesc.\n");
-
   if (!isReadHintOrNone(getL1HintAttr()))
     return emitOpError("invalid l1_hint: ") << getL1HintAttr();
 
@@ -507,27 +573,9 @@ LogicalResult LoadGatherOp::verify() {
   if (!isReadHintOrNone(getL3HintAttr()))
     return emitOpError("invalid l3_hint: ") << getL3HintAttr();
 
-  auto tdescElemTy = tdescTy.getElementType();
-  auto valueElemTy = getElementType();
-  if (tdescElemTy != valueElemTy)
-    return emitOpError(
-        "Value should have the same element type as TensorDesc.");
-
-  auto maskShape = getShapeOf(maskTy);
-  auto valueShape = getShapeOf(valueTy);
-  auto tdescShape = getShapeOf(tdescTy);
-
-  if (tdescShape[0] != maskShape[0])
-    return emitOpError("dim-0 of the Mask and TensorDesc should be the same.");
-
-  if (tdescTy.getRank() == 2) {
-    if (!getTransposeAttr())
-      return emitOpError("load of rank-2 tensor has to be transposed.");
-    transpose({1, 0}, tdescShape);
-  }
-
-  return isArgShapesValid(tdescTy, valueTy, tdescShape,
-                          [&]() { return emitOpError(); });
+  return isValidGatherScatterParams(maskTy, valueTy, tdescTy,
+                                    getTransposeAttr(),
+                                    [&]() { return emitOpError(); });
 }
 
 //===----------------------------------------------------------------------===//
@@ -535,8 +583,8 @@ LogicalResult LoadGatherOp::verify() {
 //===----------------------------------------------------------------------===//
 LogicalResult StoreScatterOp::verify() {
   auto tdescTy = getTensorDescType();
-  if (!tdescTy.isScattered())
-    return emitOpError("Expects a scattered TensorDesc.\n");
+  auto maskTy = getMaskType();
+  auto valueTy = getValueType();
 
   if (!isWriteHintOrNone(getL1HintAttr()))
     return emitOpError("invalid l1_hint: ") << getL1HintAttr();
@@ -547,26 +595,9 @@ LogicalResult StoreScatterOp::verify() {
   if (!isWriteHintOrNone(getL3HintAttr()))
     return emitOpError("invalid l3_hint: ") << getL3HintAttr();
 
-  auto maskTy = getMaskType();
-  auto valueTy = getValueType();
-
-  if (!valueTy)
-    return emitOpError("Expecting a vector type for the value.\n");
-
-  auto maskShape = getShapeOf(maskTy);
-  auto tdescShape = getShapeOf(tdescTy);
-  auto valueShape = getShapeOf(valueTy);
-  if (tdescShape[0] != maskShape[0])
-    return emitOpError("dim-0 of the Mask and TensorDesc should be the same.");
-
-  if (tdescTy.getRank() == 2) {
-    if (!getTransposeAttr())
-      return emitOpError("Store of a rank-2 tensor has to be transposed.");
-    transpose({1, 0}, tdescShape);
-  }
-
-  return isArgShapesValid(tdescTy, valueTy, tdescShape,
-                          [&]() { return emitOpError(); });
+  return isValidGatherScatterParams(maskTy, valueTy, tdescTy,
+                                    getTransposeAttr(),
+                                    [&]() { return emitOpError(); });
 }
 
 //===----------------------------------------------------------------------===//
@@ -602,63 +633,34 @@ LogicalResult DpasOp::verify() {
   auto rhsShape = getRhsType().getShape();
   auto resShape = getResultType().getShape();
 
-  auto aLayout = getALayoutAttr();
-  auto bLayout = getBLayoutAttr();
-  auto cLayout = getCLayoutAttr();
-
-  // make sure the layout attribute is either set for every available
-  // operand or simply not set at all. C is special, since ACC is optional.
-  auto hasValidLayoutAttrs = [&]() {
-    bool result = (aLayout != nullptr) ^ (bLayout != nullptr);
-    if (hasAcc()) {
-      result |= (aLayout != nullptr) ^ (cLayout != nullptr);
-    }
-    return !result;
-  };
+  if (getAcc() && getAcc().getType() != getResultType())
+    return emitOpError("Expecting the acc type to be the same as result.");
+
+  // SIMT code: the size of the B operand has to be a multiple of 32 bits.
+  // It skips the semantic check since lack of architecture information.
+  // Users need to ensure the correctness.
+  if (lhsRank == 1 && rhsRank == 1 && resRank == 1) {
+    auto numElems = getRhsType().getNumElements();
+    auto elemTy = getRhsType().getElementType();
+    auto factor = 32 / elemTy.getIntOrFloatBitWidth();
+    if (numElems % factor != 0)
+      return emitOpError("Expecting B operand to be a multiple of 32 bits.");
+    return success();
+  }
 
-  if (!hasValidLayoutAttrs())
+  // SIMD code
+  if (lhsRank != 2 || (rhsRank != 2 && rhsRank != 3) || resRank != 2)
     return emitOpError(
-        "layout attributes should be either set for all operands (for SIMT "
-        "code) or not set at all (for SIMD code).");
-
-  // query the scope from aLayout (a valid setting).
-  if (aLayout) {
-    // In SIMT mode, All data fragments must be 2D
-    if (lhsRank != 2 || rhsRank != 2 || resRank != 2)
-      return emitOpError("expecting lhs, rhs, and result to be a 2D vector.");
-
-    auto laneLayoutA = aLayout.getLaneLayout();
-    auto laneLayoutB = bLayout.getLaneLayout();
-    auto laneLayoutC = cLayout.getLaneLayout();
-    // Obtain the expanded shapes of the operands and result using lane_layout.
-    // NOTE: For B, get rid of the packed dimension for the expanded shape.
-    SmallVector<int64_t> expandedShapeA = {lhsShape[0] * laneLayoutA[0],
-                                           lhsShape[1] * laneLayoutA[1]};
-    SmallVector<int64_t> expandedShapeB = {
-        rhsShape[0] * rhsShape[1] * laneLayoutB[0], 1 * laneLayoutB[1]};
-    SmallVector<int64_t> expandedShapeC = {resShape[0] * laneLayoutC[0],
-                                           resShape[1] * laneLayoutC[1]};
-    auto bK = expandedShapeB[0];
-    if (bK != expandedShapeA[1])
-      return emitOpError("K-dimension mismatch.");
-    if (expandedShapeA[0] != expandedShapeC[0])
-      return emitOpError("M-dimension mismatch.");
-    if (expandedShapeB[1] != expandedShapeC[1])
-      return emitOpError("N-dimension mismatch.");
-  } else { // For other scopes, operands' shape should match the mxkxn
-           // semantics.
-    if (lhsRank != 2 || (rhsRank != 2 && rhsRank != 3) || resRank != 2)
-      return emitOpError(
-          "expecting lhs and result to be a 2D vector, and rhs to be either "
-          "2D or 3D (packed) vector.");
-    auto bK = rhsRank == 3 ? rhsShape[0] * rhsShape[2] : rhsShape[0];
-    if (bK != lhsShape[1])
-      return emitOpError("K-dimension mismatch.");
-    if (lhsShape[0] != resShape[0])
-      return emitOpError("M-dimension mismatch.");
-    if (rhsShape[1] != resShape[1])
-      return emitOpError("N-dimension mismatch.");
-  }
+        "expecting lhs and result to be a 2D vector, and rhs to be either "
+        "2D or 3D (packed) vector.");
+  auto bK = rhsRank == 3 ? rhsShape[0] * rhsShape[2] : rhsShape[0];
+  if (bK != lhsShape[1])
+    return emitOpError("K-dimension mismatch.");
+  if (lhsShape[0] != resShape[0])
+    return emitOpError("M-dimension mismatch.");
+  if (rhsShape[1] != resShape[1])
+    return emitOpError("N-dimension mismatch.");
+
   return success();
 }
 
diff --git a/mlir/test/Dialect/XeGPU/invalid.mlir b/mlir/test/Dialect/XeGPU/invalid.mlir
index 48df33a591908..67ed89e11b4c9 100644
--- a/mlir/test/Dialect/XeGPU/invalid.mlir
+++ b/mlir/test/Dialect/XeGPU/invalid.mlir
@@ -79,25 +79,10 @@ func.func @test_load_nd_vc_3(%src: memref<8x16xf16>) {
 
 // -----
 func.func @test_load_nd_layout(%src: memref<24x32xf32>) {
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
-    !xegpu.tensor_desc<8x16xf32,   #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  // expected-error@+1 {{Result shape [8, 2] is not consistent with distributed vector shape [8, 1]}}
-  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>,
-      l2_hint = #xegpu.cache_hint<uncached>}>
-    : !xegpu.tensor_desc<8x16xf32,   #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-    -> vector<8x2xf32>
-  return
-}
-
-// -----
-func.func @test_load_nd_layout(%src: memref<24x32xf32>) {
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
-    !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  // expected-error@+1 {{Result shape [8] is not consistent with distributed vector shape [1, 1]}}
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<16xf32>
+  // expected-error@+1 {{Result shape [3] is not a valid distribution for tensor descriptor}}
   %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>,
-      l2_hint = #xegpu.cache_hint<uncached>}>
-    : !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-    -> vector<8xf32>
+      l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf32> -> vector<3xf32>
   return
 }
 
@@ -105,7 +90,7 @@ func.func @test_load_nd_layout(%src: memref<24x32xf32>) {
 func.func @test_load_nd_vc_6(%src: memref<24x32xf32>) {
   %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
     !xegpu.tensor_desc<8x16xf32>
-  // expected-error@+1 {{Value shape [8, 1] is not consistent with tensor descriptor}}
+  // expected-error@+1 {{Result shape [8, 1] is not consistent with tensor descriptor}}
   %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>,
       l2_hint = #xegpu.cache_hint<uncached>}>
     : !xegpu.tensor_desc<8x16xf32> -> vector<8x1xf32>
@@ -134,22 +119,19 @@ func.func @test_store_nd_vc_2(%dst: memref<16xf16>) {
 }
 
 // -----
-func.func @test_store_nd_layout(%dst: memref<24x32xf32>, %data: vector<8x2xf32>) {
-  %1 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf32> ->
-    !xegpu.tensor_desc<8x16xf32,   #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  // expected-error@+1 {{Result shape [8, 2] is not consistent with distributed vector shape [8, 1] for tensor descriptor}}
-  xegpu.store_nd %data, %1
-    : vector<8x2xf32>, !xegpu.tensor_desc<8x16xf32,   #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+func.func @test_store_nd_vc_3(%dst: memref<24x32xf16>) {
+  %1 = arith.constant dense<1.0>: vector<2x24x32xf16>
+  %2 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16, #xegpu.block_tdesc_attr<array_length = 2>>
+  // expected-error@+1 {{array length is not supported by store_nd}}
+  xegpu.store_nd %1, %2: vector<2x24x32xf16>, !xegpu.tensor_desc<24x32xf16, #xegpu.block_tdesc_attr<array_length = 2>>
   return
 }
 
 // -----
-func.func @test_store_nd_layout(%dst: memref<24x32xf32>, %data: vector<2xf32>) {
-  %1 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf32> ->
-    !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  // expected-error@+1 {{Result shape [2] is not consistent with distributed vector shape [1, 1] for tensor descriptor}}
-  xegpu.store_nd %data, %1
-    : vector<2xf32>, !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+func.func @test_store_nd_simt(%dst: memref<24x32xf32>, %data: vector<3xf32>) {
+  %1 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<16xf32>
+  // expected-error@+1 {{Value shape [3] is not a valid distribution for tensor descriptor}}
+  xegpu.store_nd %data, %1 : vector<3xf32>, !xegpu.tensor_desc<16xf32>
   return
 }
 
@@ -269,45 +251,23 @@ func.func @test_create_tdesc_layout_3(%src: ui64) {
 }
 
 // -----
-func.func @test_load_gather_layout_1(%src: ui64) {
+func.func @test_load_gather_simt_1(%src: ui64) {
   %0 = arith.constant dense<1>: vector<4xi1>
   %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
-  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>,   #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-  // expected-error@+1 {{Result shape [1, 2] is not consistent with distributed vector shape [2, 1] for tensor descriptor}}
-  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>,  #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xi1> -> vector<1x2xf32>
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  // expected-error@+1 {{Value shape [6] is neither a valid distribution for SIMT nor consistent with the tensor descriptor for SIMD}}
+  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1> -> vector<6xf32>
   return
 }
 
 // -----
-func.func @test_load_gather_layout_2(%src: ui64) {
+func.func @test_store_scatter_simt_1(%src: ui64) {
   %0 = arith.constant dense<1>: vector<4xi1>
   %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
-  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>,  #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-  // expected-error@+1 {{esult shape [2] is not consistent with distributed vector shape [2, 1] for tensor descriptor}}
-  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>,  #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xi1> -> vector<2xf32>
-  return
-}
-
-
-// -----
-func.func @test_store_scatter_layout_1(%src: ui64) {
-  %0 = arith.constant dense<1>: vector<4xi1>
-  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
-  %val = arith.constant dense<2.9>: vector<1x2xf32>
-  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>,  #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-  // expected-error@+1 {{Result shape [1, 2] is not consistent with distributed vector shape [2, 1] for tensor descriptor}}
-  xegpu.store %val, %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : vector<1x2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>,  #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xi1>
-  return
-}
-
-// -----
-func.func @test_store_scatter_layout_2(%src: ui64) {
-  %0 = arith.constant dense<1>: vector<4xi1>
-  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
-  %val = arith.constant dense<2.9>: vector<2xf32>
-  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>,  #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-  // expected-error@+1 {{esult shape [2] is not consistent with distributed vector shape [2, 1] for tensor descriptor}}
-  xegpu.store %val, %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, transpose}> : vector<2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>,  #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xi1>
+  %val = arith.constant dense<2.9>: vector<6xf32>
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  // expected-error@+1 {{Value shape [6] is neither a valid distribution for SIMT nor consistent with the tensor descriptor for SIMD}}
+  xegpu.store %val, %1, %0 <{l1_hint = #xegpu.cache_hint<cached>}> : vector<6xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1>
   return
 }
 
@@ -387,26 +347,16 @@ func.func @test_dpas_4(%a : vector<16x16xf16>, %b: vector<8x16x2xf16>) {
 }
 
 // -----
-func.func @test_dpas_4(%a : vector<8x16xf16>, %b: vector<8x8x2xf16>) {
+func.func @test_dpas_5(%a : vector<8x16xf16>, %b: vector<8x8x2xf16>) {
   // expected-error@+1 {{N-dimension mismatch}}
   %1 = xegpu.dpas %a, %b : vector<8x16xf16>, vector<8x8x2xf16> -> vector<8x16xf32>
   return
 }
 
 // -----
-func.func @test_dpas_layout_1(%a : vector<8x1xf16>, %b: vector<8x2xf16>) {
-  // expected-error@+1 {{layout attributes should be either set for all operands (for SIMT code) or not set at all (for SIMD code)}}
-  %1 = xegpu.dpas %a, %b {a_layout =  #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<8x1xf16>, vector<8x2xf16> -> vector<8x1xf32>
-  return
-}
-
-// -----
-func.func @test_dpas_layout_2(%a : vector<8x1xf16>, %b: vector<4x2xf16>) {
-  // expected-error@+1 {{K-dimension mismatch}}
-  %1 = xegpu.dpas %a, %b {a_layout =  #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>,
-                          b_layout =  #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>,
-                          c_layout =  #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
-                          : vector<8x1xf16>, vector<4x2xf16> -> vector<8x1xf32>
+func.func @test_dpas_simt_1(%a : vector<8xf16>, %b: vector<15xf16>) {
+  // expected-error@+1 {{Expecting B operand to be a multiple of 32 bits}}
+  %1 = xegpu.dpas %a, %b : vector<8xf16>, vector<15xf16> -> vector<8xf32>
   return
 }
 
diff --git a/mlir/test/Dialect/XeGPU/ops.mlir b/mlir/test/Dialect/XeGPU/ops.mlir
index e9895e0d0a71d..71e7e9bdda07d 100644
--- a/mlir/test/Dialect/XeGPU/ops.mlir
+++ b/mlir/test/Dialect/XeGPU/ops.mlir
@@ -125,11 +125,11 @@ gpu.func @test_load_nd_vc(%src: memref<8x16xf16>) {
 
 // CHECK: func @test_load_nd_simt(%[[arg0:.*]]: memref<8x16xf16>) {
 gpu.func @test_load_nd_simt(%src: memref<8x16xf16>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
-  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, packed}> : !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<4x2xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, packed}> : !xegpu.tensor_desc<8x16xf16> -> vector<8xf16>
   %2 = xegpu.load_nd %1 <{packed, l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>
-       : !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<4x2xf16>
+       : !xegpu.tensor_desc<8x16xf16> -> vector<8xf16>
   gpu.return
 }
 
@@ -144,10 +144,10 @@ gpu.func @test_load_nd_vc_2(%src: memref<8x16xf16>) {
 
 // CHECK: func @test_load_nd_simt_2(%[[arg0:.*]]: memref<8x16xf16>) {
 gpu.func @test_load_nd_simt_2(%src: memref<8x16xf16>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>> -> vector<1x1xf16>
-  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>> -> vector<1x1xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<16xf16>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<16xf16>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf16> -> vector<1xf16>
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf16> -> vector<1xf16>
   gpu.return
 }
 
@@ -162,11 +162,10 @@ gpu.func @test_load_nd_vc_3(%src: memref<24x32xf32>) {
 
 // CHECK: func @test_load_nd_simt_3(%[[arg0:.*]]: memref<24x32xf32>) {
 gpu.func @test_load_nd_simt_3(%src: memref<24x32xf32>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
-    !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x1xf32>
-  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x1xf32>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<8x16xf32> -> vector<8xf32>
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<8x16xf32> -> vector<8xf32>
   gpu.return
 }
 
@@ -181,11 +180,10 @@ gpu.func @test_load_nd_vc_4(%src: memref<24x32xf16>) {
 
 // CHECK: func @test_load_nd_simt_4(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @test_load_nd_simt_4(%src: memref<24x32xf16>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> ->
-    !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
-  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<8x2xf16>
-  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<8x2xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
   gpu.return
 }
 
@@ -200,11 +198,10 @@ gpu.func @test_load_nd_vc_5(%src: memref<24x32xf32>) {
 
 // CHECK: func @test_load_nd_simt_5(%[[arg0:.*]]: memref<24x32xf32>) {
 gpu.func @test_load_nd_simt_5(%src: memref<24x32xf32>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<32xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
-    !xegpu.tensor_desc<32xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<32xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>> -> vector<2x1xf32>
-  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<32xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>> -> vector<2x1xf32>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<32xf32>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<32xf32>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<32xf32> -> vector<2xf32>
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<32xf32> -> vector<2xf32>
   gpu.return
 }
 
@@ -219,11 +216,11 @@ gpu.func @test_load_nd_vc_6(%src: memref<24x32xf16>) {
 
 // CHECK: func @test_load_nd_simt_6(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @test_load_nd_simt_6(%src: memref<24x32xf16>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2>, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<32x1xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2>>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>> -> vector<32xf16>
   %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> :
-    !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2>, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<32x1xf16>
+    !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2>> -> vector<32xf16>
   gpu.return
 }
 
@@ -238,11 +235,11 @@ gpu.func @test_load_nd_vc_7(%src: memref<24x32xf16>) {
 
 // CHECK: func @test_load_nd_simt_7(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @test_load_nd_simt_7(%src: memref<24x32xf16>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2>, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
-  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x2xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2>>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>> -> vector<32xf16>
   %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> :
-    !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2>, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x2xf16>
+    !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2>> -> vector<32xf16>
   gpu.return
 }
 
@@ -257,10 +254,10 @@ gpu.func @test_load_nd_vc_8(%src: memref<24x32xf32>) {
 
 // CHECK: func @test_load_nd_simt_8(%[[arg0:.*]]: memref<24x32xf32>) {
 gpu.func @test_load_nd_simt_8(%src: memref<24x32xf32>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>>
-  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>>
-  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x8xf32, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>> -> vector<8x1xf32>
-  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x8xf32, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>> -> vector<8x1xf32>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32>
+  // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x8xf32> -> vector<8xf32>
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x8xf32> -> vector<8xf32>
   gpu.return
 }
 
@@ -277,13 +274,12 @@ gpu.func @test_store_nd_vc(%dst: memref<24x32xf16>) {
 
 // CHECK: func @test_store_nd_simt(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @test_store_nd_simt(%src: memref<24x32xf16>) {
-   // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<48x1xf16>
-  %1 = arith.constant dense<1.0>: vector<48x1xf16>
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  %2 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> ->
-    !xegpu.tensor_desc<24x32xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  // CHECK: xegpu.store_nd %[[C]], %[[R0]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<48x1xf16>, !xegpu.tensor_desc<24x32xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>: vector<48x1xf16>, !xegpu.tensor_desc<24x32xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+  // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<48xf16>
+  %1 = arith.constant dense<1.0>: vector<48xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16>
+  %2 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16>
+  // CHECK: xegpu.store_nd %[[C]], %[[R0]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<48xf16>, !xegpu.tensor_desc<24x32xf16>
+  xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>: vector<48xf16>, !xegpu.tensor_desc<24x32xf16>
   gpu.return
 }
 
@@ -303,13 +299,12 @@ gpu.func @test_store_nd_vc_2(%dst: memref<24x32xf16>) {
 
 // CHECK: func @test_store_nd_simt_2(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @test_store_nd_simt_2(%src: memref<24x32xf16>) {
-   // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<2x1xf16>
-  %1 = arith.constant dense<1.0>: vector<2x1xf16>
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  %2 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> ->
-    !xegpu.tensor_desc<32xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  // CHECK: xegpu.store_nd %[[C]], %[[R0]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<2x1xf16>, !xegpu.tensor_desc<32xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
-  xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>: vector<2x1xf16>, !xegpu.tensor_desc<32xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+  // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<2xf16>
+  %1 = arith.constant dense<1.0>: vector<2xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16>
+  %2 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16>
+  // CHECK: xegpu.store_nd %[[C]], %[[R0]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<2xf16>, !xegpu.tensor_desc<32xf16>
+  xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>: vector<2xf16>, !xegpu.tensor_desc<32xf16>
   gpu.return
 }
 
@@ -425,10 +420,10 @@ gpu.func @test_load_simt(%src: ui64) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
   %1 = arith.constant dense<1>: vector<4xi1>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-  %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xi1> -> vector<2x1xf32>
-  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xi1> -> vector<2x1xf32>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1> -> vector<2xf32>
+  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1> -> vector<2xf32>
   gpu.return
 }
 
@@ -451,10 +446,10 @@ gpu.func @test_load_simt_2(%src: ui64) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
   %1 = arith.constant dense<1>: vector<4xi1>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [4], lane_data = [1]>>
-  %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [4], lane_data = [1]>>
-  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [4], lane_data = [1]>>, vector<4xi1> -> vector<1xf32>
-  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [4], lane_data = [1]>>, vector<4xi1> -> vector<1xf32>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>>
+  %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>>
+  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>>, vector<4xi1> -> vector<1xf32>
+  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>>, vector<4xi1> -> vector<1xf32>
   gpu.return
 }
 
@@ -477,10 +472,10 @@ gpu.func @test_load_simt_3(%src: ui64) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
   %1 = arith.constant dense<1>: vector<4xi1>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 2]>>
-  %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 2]>>
-  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 2]>>, vector<4xi1> -> vector<4x2xf16>
-  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 2]>>, vector<4xi1> -> vector<4x2xf16>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>
+  %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8>>
+  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>, vector<4xi1> -> vector<8xf16>
+  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8>>, vector<4xi1> -> vector<8xf16>
   gpu.return
 }
 
@@ -507,12 +502,12 @@ gpu.func @test_store_simt(%src: ui64) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
   %1 = arith.constant dense<1>: vector<4xi1>
-  //CHECK: %[[cst2:.*]] = arith.constant dense<2.900000e+00> : vector<2x1xf32>
-  %2 = arith.constant dense<2.9>: vector<2x1xf32>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-  %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x1xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xi1>
-  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x1xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xi1>
+  //CHECK: %[[cst2:.*]] = arith.constant dense<2.900000e+00> : vector<2xf32>
+  %2 = arith.constant dense<2.9>: vector<2xf32>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1>
+  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1>
   gpu.return
 }
 
@@ -539,12 +534,12 @@ gpu.func @test_store_simt_2(%src: ui64) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
   %1 = arith.constant dense<1>: vector<4xi1>
-  //CHECK: %[[cst2:.*]] = arith.constant {{.*}} : vector<1x2xf16>
-  %2 = arith.constant dense<2.9>: vector<1x2xf16>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 2]>>
-  %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 2]>>
-  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<1x2xf16>, !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 2]>>, vector<4xi1>
-  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<1x2xf16>, !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 2]>>, vector<4xi1>
+  //CHECK: %[[cst2:.*]] = arith.constant {{.*}} : vector<2xf16>
+  %2 = arith.constant dense<2.9>: vector<2xf16>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<2xf16>, !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1>
+  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<2xf16>, !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1>
   gpu.return
 }
 
@@ -572,10 +567,10 @@ gpu.func @test_store_simt_3(%src: ui64) {
   %1 = arith.constant dense<1>: vector<4xi1>
   //CHECK: %[[cst2:.*]] = arith.constant dense<2.900000e+00> : vector<1xf32>
   %2 = arith.constant dense<2.9>: vector<1xf32>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [4], lane_data = [1]>>
-  %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [4], lane_data = [1]>>
-  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<1xf32>, !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [4], lane_data = [1]>>, vector<4xi1>
-  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<1xf32>, !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [4], lane_data = [1]>>, vector<4xi1>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>>
+  %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>>
+  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<1xf32>, !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>>, vector<4xi1>
+  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<1xf32>, !xegpu.tensor_desc<4xf32, #xegpu.scatter_tdesc_attr<>>, vector<4xi1>
   gpu.return
 }
 
@@ -635,15 +630,10 @@ gpu.func @test_dpas_vc(%a : vector<8x16xf16>, %b: vector<16x16xf16>) {
   gpu.return
 }
 
-// CHECK: gpu.func @test_dpas_simt(%[[arg0:.*]]: vector<8x1xf16>, %[[arg1:.*]]: vector<8x2xf16>)
-gpu.func @test_dpas_simt(%a : vector<8x1xf16>, %b: vector<8x2xf16>) {
-  // CHECK: xegpu.dpas %[[arg0]], %[[arg1]] {a_layout = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>,
-  // CHECK: b_layout = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>,
-  // CHECK: c_layout = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<8x1xf16>, vector<8x2xf16> -> vector<8x1xf32>
-  %1 = xegpu.dpas %a, %b {a_layout = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>,
-                          b_layout = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>,
-                          c_layout = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
-                          : vector<8x1xf16>, vector<8x2xf16> -> vector<8x1xf32>
+// CHECK: gpu.func @test_dpas_simt(%[[arg0:.*]]: vector<8xf16>, %[[arg1:.*]]: vector<16xf16>)
+gpu.func @test_dpas_simt(%a : vector<8xf16>, %b: vector<16xf16>) {
+  // CHECK: xegpu.dpas %[[arg0]], %[[arg1]] : vector<8xf16>, vector<16xf16> -> vector<8xf32>
+  %1 = xegpu.dpas %a, %b : vector<8xf16>, vector<16xf16> -> vector<8xf32>
   gpu.return
 }
 

From 1756fcb8b0192281db641d2038c03b96015e29d4 Mon Sep 17 00:00:00 2001
From: Christopher Ferris <cferris1000@users.noreply.github.com>
Date: Thu, 17 Apr 2025 14:23:42 -0700
Subject: [PATCH 326/710] [scudo] Add primary option to enable/disable cache
 blocks. (#129794)

When configured this way, no primary blocks will be cached except the
batch class. Nothing else changes, no change in the page releasing
strategy.
---
 .../lib/scudo/standalone/CMakeLists.txt       |   2 +-
 .../lib/scudo/standalone/allocator_config.def |   4 +
 compiler-rt/lib/scudo/standalone/combined.h   |  47 ++---
 compiler-rt/lib/scudo/standalone/primary32.h  |  77 +++++----
 compiler-rt/lib/scudo/standalone/primary64.h  | 110 ++++++------
 .../{local_cache.h => size_class_allocator.h} | 161 ++++++++++++++++--
 .../scudo/standalone/tests/combined_test.cpp  |  56 +++++-
 .../scudo/standalone/tests/primary_test.cpp   |  76 +++++----
 .../lib/scudo/standalone/tests/tsd_test.cpp   |  27 +--
 compiler-rt/lib/scudo/standalone/tsd.h        |   9 +-
 compiler-rt/lib/scudo/standalone/tsd_shared.h |   2 +-
 .../lib/scudo/standalone/type_traits.h        |   8 +
 12 files changed, 404 insertions(+), 175 deletions(-)
 rename compiler-rt/lib/scudo/standalone/{local_cache.h => size_class_allocator.h} (53%)

diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
index 3f9ae866a7553..db494a9a74a3f 100644
--- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt
+++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
@@ -74,7 +74,6 @@ set(SCUDO_HEADERS
   internal_defs.h
   linux.h
   list.h
-  local_cache.h
   memtag.h
   mem_map.h
   mem_map_base.h
@@ -90,6 +89,7 @@ set(SCUDO_HEADERS
   report.h
   report_linux.h
   secondary.h
+  size_class_allocator.h
   size_class_map.h
   stack_depot.h
   stats.h
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.def b/compiler-rt/lib/scudo/standalone/allocator_config.def
index 43893e9af3cf3..84fcec0877d40 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_config.def
+++ b/compiler-rt/lib/scudo/standalone/allocator_config.def
@@ -81,6 +81,10 @@ PRIMARY_REQUIRED(const s32, MaxReleaseToOsIntervalMs)
 
 // PRIMARY_OPTIONAL(TYPE, NAME, DEFAULT)
 //
+
+// Enables/disables primary block caching. Batch class still caches.
+PRIMARY_OPTIONAL(const bool, EnableBlockCache, true)
+
 // The scale of a compact pointer. E.g., Ptr = Base + (CompactPtr << Scale).
 PRIMARY_OPTIONAL(const uptr, CompactPtrScale, SCUDO_MIN_ALIGNMENT_LOG)
 
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 5deb8c97f1c86..43655642843cb 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -15,7 +15,6 @@
 #include "common.h"
 #include "flags.h"
 #include "flags_parser.h"
-#include "local_cache.h"
 #include "mem_map.h"
 #include "memtag.h"
 #include "mutex.h"
@@ -23,6 +22,7 @@
 #include "quarantine.h"
 #include "report.h"
 #include "secondary.h"
+#include "size_class_allocator.h"
 #include "stack_depot.h"
 #include "string_utils.h"
 #include "tsd.h"
@@ -54,7 +54,7 @@ class Allocator {
       typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
   using SecondaryT =
       typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
-  using CacheT = typename PrimaryT::CacheT;
+  using SizeClassAllocatorT = typename PrimaryT::SizeClassAllocatorT;
   typedef Allocator<Config, PostInitCallback> ThisT;
   typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
 
@@ -63,8 +63,9 @@ class Allocator {
   }
 
   struct QuarantineCallback {
-    explicit QuarantineCallback(ThisT &Instance, CacheT &LocalCache)
-        : Allocator(Instance), Cache(LocalCache) {}
+    explicit QuarantineCallback(ThisT &Instance,
+                                SizeClassAllocatorT &SizeClassAllocator)
+        : Allocator(Instance), SizeClassAllocator(SizeClassAllocator) {}
 
     // Chunk recycling function, returns a quarantined chunk to the backend,
     // first making sure it hasn't been tampered with.
@@ -80,7 +81,7 @@ class Allocator {
       if (allocatorSupportsMemoryTagging<AllocatorConfig>())
         Ptr = untagPointer(Ptr);
       void *BlockBegin = Allocator::getBlockBegin(Ptr, &Header);
-      Cache.deallocate(Header.ClassId, BlockBegin);
+      SizeClassAllocator.deallocate(Header.ClassId, BlockBegin);
     }
 
     // We take a shortcut when allocating a quarantine batch by working with the
@@ -89,7 +90,7 @@ class Allocator {
     void *allocate(UNUSED uptr Size) {
       const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
           sizeof(QuarantineBatch) + Chunk::getHeaderSize());
-      void *Ptr = Cache.allocate(QuarantineClassId);
+      void *Ptr = SizeClassAllocator.allocate(QuarantineClassId);
       // Quarantine batch allocation failure is fatal.
       if (UNLIKELY(!Ptr))
         reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));
@@ -126,14 +127,15 @@ class Allocator {
 
       Header.State = Chunk::State::Available;
       Chunk::storeHeader(Allocator.Cookie, Ptr, &Header);
-      Cache.deallocate(QuarantineClassId,
-                       reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
-                                                Chunk::getHeaderSize()));
+      SizeClassAllocator.deallocate(
+          QuarantineClassId,
+          reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
+                                   Chunk::getHeaderSize()));
     }
 
   private:
     ThisT &Allocator;
-    CacheT &Cache;
+    SizeClassAllocatorT &SizeClassAllocator;
   };
 
   typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
@@ -263,7 +265,9 @@ class Allocator {
   QuarantineT *getQuarantine() { return &Quarantine; }
 
   // The Cache must be provided zero-initialized.
-  void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); }
+  void initAllocator(SizeClassAllocatorT *SizeClassAllocator) {
+    SizeClassAllocator->init(&Stats, &Primary);
+  }
 
   // Release the resources used by a TSD, which involves:
   // - draining the local quarantine cache to the global quarantine;
@@ -273,15 +277,16 @@ class Allocator {
   void commitBack(TSD<ThisT> *TSD) {
     TSD->assertLocked(/*BypassCheck=*/true);
     Quarantine.drain(&TSD->getQuarantineCache(),
-                     QuarantineCallback(*this, TSD->getCache()));
-    TSD->getCache().destroy(&Stats);
+                     QuarantineCallback(*this, TSD->getSizeClassAllocator()));
+    TSD->getSizeClassAllocator().destroy(&Stats);
   }
 
   void drainCache(TSD<ThisT> *TSD) {
     TSD->assertLocked(/*BypassCheck=*/true);
-    Quarantine.drainAndRecycle(&TSD->getQuarantineCache(),
-                               QuarantineCallback(*this, TSD->getCache()));
-    TSD->getCache().drain();
+    Quarantine.drainAndRecycle(
+        &TSD->getQuarantineCache(),
+        QuarantineCallback(*this, TSD->getSizeClassAllocator()));
+    TSD->getSizeClassAllocator().drain();
   }
   void drainCaches() { TSDRegistry.drainCaches(this); }
 
@@ -390,13 +395,13 @@ class Allocator {
       ClassId = SizeClassMap::getClassIdBySize(NeededSize);
       DCHECK_NE(ClassId, 0U);
       typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
-      Block = TSD->getCache().allocate(ClassId);
+      Block = TSD->getSizeClassAllocator().allocate(ClassId);
       // If the allocation failed, retry in each successively larger class until
       // it fits. If it fails to fit in the largest class, fallback to the
       // Secondary.
       if (UNLIKELY(!Block)) {
         while (ClassId < SizeClassMap::LargestClassId && !Block)
-          Block = TSD->getCache().allocate(++ClassId);
+          Block = TSD->getSizeClassAllocator().allocate(++ClassId);
         if (!Block)
           ClassId = 0;
       }
@@ -1280,7 +1285,8 @@ class Allocator {
         bool CacheDrained;
         {
           typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
-          CacheDrained = TSD->getCache().deallocate(ClassId, BlockBegin);
+          CacheDrained =
+              TSD->getSizeClassAllocator().deallocate(ClassId, BlockBegin);
         }
         // When we have drained some blocks back to the Primary from TSD, that
         // implies that we may have the chance to release some pages as well.
@@ -1296,7 +1302,8 @@ class Allocator {
         retagBlock(Options, TaggedPtr, Ptr, Header, Size, false);
       typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
       Quarantine.put(&TSD->getQuarantineCache(),
-                     QuarantineCallback(*this, TSD->getCache()), Ptr, Size);
+                     QuarantineCallback(*this, TSD->getSizeClassAllocator()),
+                     Ptr, Size);
     }
   }
 
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 596c48f004b22..0932d47ad4563 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -13,10 +13,10 @@
 #include "bytemap.h"
 #include "common.h"
 #include "list.h"
-#include "local_cache.h"
 #include "options.h"
 #include "release.h"
 #include "report.h"
+#include "size_class_allocator.h"
 #include "stats.h"
 #include "string_utils.h"
 #include "thread_annotations.h"
@@ -52,7 +52,10 @@ template <typename Config> class SizeClassAllocator32 {
   static_assert((1UL << Config::getRegionSizeLog()) >= SizeClassMap::MaxSize,
                 "");
   typedef SizeClassAllocator32<Config> ThisT;
-  typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
+  using SizeClassAllocatorT =
+      typename Conditional<Config::getEnableBlockCache(),
+                           SizeClassAllocatorLocalCache<ThisT>,
+                           SizeClassAllocatorNoCache<ThisT>>::type;
   typedef TransferBatch<ThisT> TransferBatchT;
   typedef BatchGroup<ThisT> BatchGroupT;
 
@@ -191,17 +194,19 @@ template <typename Config> class SizeClassAllocator32 {
     return BlockSize > PageSize;
   }
 
-  u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray,
-                const u16 MaxBlockCount) {
+  u16 popBlocks(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                CompactPtrT *ToArray, const u16 MaxBlockCount) {
     DCHECK_LT(ClassId, NumClasses);
     SizeClassInfo *Sci = getSizeClassInfo(ClassId);
     ScopedLock L(Sci->Mutex);
 
-    u16 PopCount = popBlocksImpl(C, ClassId, Sci, ToArray, MaxBlockCount);
+    u16 PopCount =
+        popBlocksImpl(SizeClassAllocator, ClassId, Sci, ToArray, MaxBlockCount);
     if (UNLIKELY(PopCount == 0)) {
-      if (UNLIKELY(!populateFreeList(C, ClassId, Sci)))
+      if (UNLIKELY(!populateFreeList(SizeClassAllocator, ClassId, Sci)))
         return 0U;
-      PopCount = popBlocksImpl(C, ClassId, Sci, ToArray, MaxBlockCount);
+      PopCount = popBlocksImpl(SizeClassAllocator, ClassId, Sci, ToArray,
+                               MaxBlockCount);
       DCHECK_NE(PopCount, 0U);
     }
 
@@ -209,7 +214,8 @@ template <typename Config> class SizeClassAllocator32 {
   }
 
   // Push the array of free blocks to the designated batch group.
-  void pushBlocks(CacheT *C, uptr ClassId, CompactPtrT *Array, u32 Size) {
+  void pushBlocks(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                  CompactPtrT *Array, u32 Size) {
     DCHECK_LT(ClassId, NumClasses);
     DCHECK_GT(Size, 0);
 
@@ -240,7 +246,7 @@ template <typename Config> class SizeClassAllocator32 {
     }
 
     ScopedLock L(Sci->Mutex);
-    pushBlocksImpl(C, ClassId, Sci, Array, Size, SameGroup);
+    pushBlocksImpl(SizeClassAllocator, ClassId, Sci, Array, Size, SameGroup);
   }
 
   void disable() NO_THREAD_SAFETY_ANALYSIS {
@@ -529,8 +535,8 @@ template <typename Config> class SizeClassAllocator32 {
       // memory group here.
       BG->CompactPtrGroupBase = 0;
       BG->BytesInBGAtLastCheckpoint = 0;
-      BG->MaxCachedPerBatch =
-          CacheT::getMaxCached(getSizeByClassId(SizeClassMap::BatchClassId));
+      BG->MaxCachedPerBatch = SizeClassAllocatorT::getMaxCached(
+          getSizeByClassId(SizeClassMap::BatchClassId));
 
       Sci->FreeListInfo.BlockList.push_front(BG);
     }
@@ -597,18 +603,18 @@ template <typename Config> class SizeClassAllocator32 {
   // same group then we will skip checking the group id of each block.
   //
   // The region mutex needs to be held while calling this method.
-  void pushBlocksImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci,
-                      CompactPtrT *Array, u32 Size, bool SameGroup = false)
-      REQUIRES(Sci->Mutex) {
+  void pushBlocksImpl(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                      SizeClassInfo *Sci, CompactPtrT *Array, u32 Size,
+                      bool SameGroup = false) REQUIRES(Sci->Mutex) {
     DCHECK_NE(ClassId, SizeClassMap::BatchClassId);
     DCHECK_GT(Size, 0U);
 
     auto CreateGroup = [&](uptr CompactPtrGroupBase) {
-      BatchGroupT *BG =
-          reinterpret_cast<BatchGroupT *>(C->getBatchClassBlock());
+      BatchGroupT *BG = reinterpret_cast<BatchGroupT *>(
+          SizeClassAllocator->getBatchClassBlock());
       BG->Batches.clear();
-      TransferBatchT *TB =
-          reinterpret_cast<TransferBatchT *>(C->getBatchClassBlock());
+      TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(
+          SizeClassAllocator->getBatchClassBlock());
       TB->clear();
 
       BG->CompactPtrGroupBase = CompactPtrGroupBase;
@@ -629,8 +635,8 @@ template <typename Config> class SizeClassAllocator32 {
         u16 UnusedSlots =
             static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
         if (UnusedSlots == 0) {
-          CurBatch =
-              reinterpret_cast<TransferBatchT *>(C->getBatchClassBlock());
+          CurBatch = reinterpret_cast<TransferBatchT *>(
+              SizeClassAllocator->getBatchClassBlock());
           CurBatch->clear();
           Batches.push_front(CurBatch);
           UnusedSlots = BG->MaxCachedPerBatch;
@@ -704,9 +710,9 @@ template <typename Config> class SizeClassAllocator32 {
     InsertBlocks(Cur, Array + Size - Count, Count);
   }
 
-  u16 popBlocksImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci,
-                    CompactPtrT *ToArray, const u16 MaxBlockCount)
-      REQUIRES(Sci->Mutex) {
+  u16 popBlocksImpl(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                    SizeClassInfo *Sci, CompactPtrT *ToArray,
+                    const u16 MaxBlockCount) REQUIRES(Sci->Mutex) {
     if (Sci->FreeListInfo.BlockList.empty())
       return 0U;
 
@@ -730,11 +736,11 @@ template <typename Config> class SizeClassAllocator32 {
     // So far, instead of always filling the blocks to `MaxBlockCount`, we only
     // examine single `TransferBatch` to minimize the time spent on the primary
     // allocator. Besides, the sizes of `TransferBatch` and
-    // `CacheT::getMaxCached()` may also impact the time spent on accessing the
-    // primary allocator.
+    // `SizeClassAllocatorT::getMaxCached()` may also impact the time spent on
+    // accessing the primary allocator.
     // TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
     // blocks and/or adjust the size of `TransferBatch` according to
-    // `CacheT::getMaxCached()`.
+    // `SizeClassAllocatorT::getMaxCached()`.
     TransferBatchT *B = Batches.front();
     DCHECK_NE(B, nullptr);
     DCHECK_GT(B->getCount(), 0U);
@@ -754,7 +760,7 @@ template <typename Config> class SizeClassAllocator32 {
       // deallocate. Read the comment in `pushBatchClassBlocks()` for more
       // details.
       if (ClassId != SizeClassMap::BatchClassId)
-        C->deallocate(SizeClassMap::BatchClassId, B);
+        SizeClassAllocator->deallocate(SizeClassMap::BatchClassId, B);
 
       if (Batches.empty()) {
         BatchGroupT *BG = Sci->FreeListInfo.BlockList.front();
@@ -766,7 +772,7 @@ template <typename Config> class SizeClassAllocator32 {
         // Which means, once we pop the last TransferBatch, the block is
         // implicitly deallocated.
         if (ClassId != SizeClassMap::BatchClassId)
-          C->deallocate(SizeClassMap::BatchClassId, BG);
+          SizeClassAllocator->deallocate(SizeClassMap::BatchClassId, BG);
       }
     }
 
@@ -774,7 +780,8 @@ template <typename Config> class SizeClassAllocator32 {
     return PopCount;
   }
 
-  NOINLINE bool populateFreeList(CacheT *C, uptr ClassId, SizeClassInfo *Sci)
+  NOINLINE bool populateFreeList(SizeClassAllocatorT *SizeClassAllocator,
+                                 uptr ClassId, SizeClassInfo *Sci)
       REQUIRES(Sci->Mutex) {
     uptr Region;
     uptr Offset;
@@ -791,13 +798,13 @@ template <typename Config> class SizeClassAllocator32 {
       Region = allocateRegion(Sci, ClassId);
       if (UNLIKELY(!Region))
         return false;
-      C->getStats().add(StatMapped, RegionSize);
+      SizeClassAllocator->getStats().add(StatMapped, RegionSize);
       Sci->CurrentRegion = Region;
       Offset = 0;
     }
 
     const uptr Size = getSizeByClassId(ClassId);
-    const u16 MaxCount = CacheT::getMaxCached(Size);
+    const u16 MaxCount = SizeClassAllocatorT::getMaxCached(Size);
     DCHECK_GT(MaxCount, 0U);
     // The maximum number of blocks we should carve in the region is dictated
     // by the maximum number of batches we want to fill, and the amount of
@@ -827,7 +834,8 @@ template <typename Config> class SizeClassAllocator32 {
       for (u32 I = 1; I < NumberOfBlocks; I++) {
         if (UNLIKELY(compactPtrGroupBase(ShuffleArray[I]) != CurGroup)) {
           shuffle(ShuffleArray + I - N, N, &Sci->RandState);
-          pushBlocksImpl(C, ClassId, Sci, ShuffleArray + I - N, N,
+          pushBlocksImpl(SizeClassAllocator, ClassId, Sci, ShuffleArray + I - N,
+                         N,
                          /*SameGroup=*/true);
           N = 1;
           CurGroup = compactPtrGroupBase(ShuffleArray[I]);
@@ -837,7 +845,8 @@ template <typename Config> class SizeClassAllocator32 {
       }
 
       shuffle(ShuffleArray + NumberOfBlocks - N, N, &Sci->RandState);
-      pushBlocksImpl(C, ClassId, Sci, &ShuffleArray[NumberOfBlocks - N], N,
+      pushBlocksImpl(SizeClassAllocator, ClassId, Sci,
+                     &ShuffleArray[NumberOfBlocks - N], N,
                      /*SameGroup=*/true);
     } else {
       pushBatchClassBlocks(Sci, ShuffleArray, NumberOfBlocks);
@@ -850,7 +859,7 @@ template <typename Config> class SizeClassAllocator32 {
     Sci->FreeListInfo.PushedBlocks -= NumberOfBlocks;
 
     const uptr AllocatedUser = Size * NumberOfBlocks;
-    C->getStats().add(StatFree, AllocatedUser);
+    SizeClassAllocator->getStats().add(StatFree, AllocatedUser);
     DCHECK_LE(Sci->CurrentRegionAllocated + AllocatedUser, RegionSize);
     // If there is not enough room in the region currently associated to fit
     // more blocks, we deassociate the region by resetting CurrentRegion and
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index e7da849c339bf..25ee999199114 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -14,11 +14,11 @@
 #include "common.h"
 #include "condition_variable.h"
 #include "list.h"
-#include "local_cache.h"
 #include "mem_map.h"
 #include "memtag.h"
 #include "options.h"
 #include "release.h"
+#include "size_class_allocator.h"
 #include "stats.h"
 #include "string_utils.h"
 #include "thread_annotations.h"
@@ -57,9 +57,12 @@ template <typename Config> class SizeClassAllocator64 {
                 "Group size shouldn't be greater than the region size");
   static const uptr GroupScale = GroupSizeLog - CompactPtrScale;
   typedef SizeClassAllocator64<Config> ThisT;
-  typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
   typedef TransferBatch<ThisT> TransferBatchT;
   typedef BatchGroup<ThisT> BatchGroupT;
+  using SizeClassAllocatorT =
+      typename Conditional<Config::getEnableBlockCache(),
+                           SizeClassAllocatorLocalCache<ThisT>,
+                           SizeClassAllocatorNoCache<ThisT>>::type;
 
   // BachClass is used to store internal metadata so it needs to be at least as
   // large as the largest data structure.
@@ -215,15 +218,16 @@ template <typename Config> class SizeClassAllocator64 {
     DCHECK_EQ(BlocksInUse, BatchClassUsedInFreeLists);
   }
 
-  u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray,
-                const u16 MaxBlockCount) {
+  u16 popBlocks(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                CompactPtrT *ToArray, const u16 MaxBlockCount) {
     DCHECK_LT(ClassId, NumClasses);
     RegionInfo *Region = getRegionInfo(ClassId);
     u16 PopCount = 0;
 
     {
       ScopedLock L(Region->FLLock);
-      PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+      PopCount = popBlocksImpl(SizeClassAllocator, ClassId, Region, ToArray,
+                               MaxBlockCount);
       if (PopCount != 0U)
         return PopCount;
     }
@@ -231,8 +235,8 @@ template <typename Config> class SizeClassAllocator64 {
     bool ReportRegionExhausted = false;
 
     if (conditionVariableEnabled()) {
-      PopCount = popBlocksWithCV(C, ClassId, Region, ToArray, MaxBlockCount,
-                                 ReportRegionExhausted);
+      PopCount = popBlocksWithCV(SizeClassAllocator, ClassId, Region, ToArray,
+                                 MaxBlockCount, ReportRegionExhausted);
     } else {
       while (true) {
         // When two threads compete for `Region->MMLock`, we only want one of
@@ -241,15 +245,16 @@ template <typename Config> class SizeClassAllocator64 {
         ScopedLock ML(Region->MMLock);
         {
           ScopedLock FL(Region->FLLock);
-          PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+          PopCount = popBlocksImpl(SizeClassAllocator, ClassId, Region, ToArray,
+                                   MaxBlockCount);
           if (PopCount != 0U)
             return PopCount;
         }
 
         const bool RegionIsExhausted = Region->Exhausted;
         if (!RegionIsExhausted) {
-          PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
-                                                  MaxBlockCount);
+          PopCount = populateFreeListAndPopBlocks(
+              SizeClassAllocator, ClassId, Region, ToArray, MaxBlockCount);
         }
         ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
         break;
@@ -270,7 +275,8 @@ template <typename Config> class SizeClassAllocator64 {
   }
 
   // Push the array of free blocks to the designated batch group.
-  void pushBlocks(CacheT *C, uptr ClassId, CompactPtrT *Array, u32 Size) {
+  void pushBlocks(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                  CompactPtrT *Array, u32 Size) {
     DCHECK_LT(ClassId, NumClasses);
     DCHECK_GT(Size, 0);
 
@@ -305,7 +311,8 @@ template <typename Config> class SizeClassAllocator64 {
 
     {
       ScopedLock L(Region->FLLock);
-      pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup);
+      pushBlocksImpl(SizeClassAllocator, ClassId, Region, Array, Size,
+                     SameGroup);
       if (conditionVariableEnabled())
         Region->FLLockCV.notifyAll(Region->FLLock);
     }
@@ -697,8 +704,8 @@ template <typename Config> class SizeClassAllocator64 {
       // memory group here.
       BG->CompactPtrGroupBase = 0;
       BG->BytesInBGAtLastCheckpoint = 0;
-      BG->MaxCachedPerBatch =
-          CacheT::getMaxCached(getSizeByClassId(SizeClassMap::BatchClassId));
+      BG->MaxCachedPerBatch = SizeClassAllocatorT::getMaxCached(
+          getSizeByClassId(SizeClassMap::BatchClassId));
 
       Region->FreeListInfo.BlockList.push_front(BG);
     }
@@ -764,18 +771,18 @@ template <typename Config> class SizeClassAllocator64 {
   // that we can get better performance of maintaining sorted property.
   // Use `SameGroup=true` to indicate that all blocks in the array are from the
   // same group then we will skip checking the group id of each block.
-  void pushBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region,
-                      CompactPtrT *Array, u32 Size, bool SameGroup = false)
-      REQUIRES(Region->FLLock) {
+  void pushBlocksImpl(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                      RegionInfo *Region, CompactPtrT *Array, u32 Size,
+                      bool SameGroup = false) REQUIRES(Region->FLLock) {
     DCHECK_NE(ClassId, SizeClassMap::BatchClassId);
     DCHECK_GT(Size, 0U);
 
     auto CreateGroup = [&](uptr CompactPtrGroupBase) {
-      BatchGroupT *BG =
-          reinterpret_cast<BatchGroupT *>(C->getBatchClassBlock());
+      BatchGroupT *BG = reinterpret_cast<BatchGroupT *>(
+          SizeClassAllocator->getBatchClassBlock());
       BG->Batches.clear();
-      TransferBatchT *TB =
-          reinterpret_cast<TransferBatchT *>(C->getBatchClassBlock());
+      TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(
+          SizeClassAllocator->getBatchClassBlock());
       TB->clear();
 
       BG->CompactPtrGroupBase = CompactPtrGroupBase;
@@ -796,8 +803,8 @@ template <typename Config> class SizeClassAllocator64 {
         u16 UnusedSlots =
             static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
         if (UnusedSlots == 0) {
-          CurBatch =
-              reinterpret_cast<TransferBatchT *>(C->getBatchClassBlock());
+          CurBatch = reinterpret_cast<TransferBatchT *>(
+              SizeClassAllocator->getBatchClassBlock());
           CurBatch->clear();
           Batches.push_front(CurBatch);
           UnusedSlots = BG->MaxCachedPerBatch;
@@ -871,9 +878,9 @@ template <typename Config> class SizeClassAllocator64 {
     InsertBlocks(Cur, Array + Size - Count, Count);
   }
 
-  u16 popBlocksWithCV(CacheT *C, uptr ClassId, RegionInfo *Region,
-                      CompactPtrT *ToArray, const u16 MaxBlockCount,
-                      bool &ReportRegionExhausted) {
+  u16 popBlocksWithCV(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                      RegionInfo *Region, CompactPtrT *ToArray,
+                      const u16 MaxBlockCount, bool &ReportRegionExhausted) {
     u16 PopCount = 0;
 
     while (true) {
@@ -895,8 +902,8 @@ template <typename Config> class SizeClassAllocator64 {
 
         const bool RegionIsExhausted = Region->Exhausted;
         if (!RegionIsExhausted) {
-          PopCount = populateFreeListAndPopBlocks(C, ClassId, Region, ToArray,
-                                                  MaxBlockCount);
+          PopCount = populateFreeListAndPopBlocks(
+              SizeClassAllocator, ClassId, Region, ToArray, MaxBlockCount);
         }
         ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
 
@@ -924,7 +931,8 @@ template <typename Config> class SizeClassAllocator64 {
       // blocks were used up right after the refillment. Therefore, we have to
       // check if someone is still populating the freelist.
       ScopedLock FL(Region->FLLock);
-      PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+      PopCount = popBlocksImpl(SizeClassAllocator, ClassId, Region, ToArray,
+                               MaxBlockCount);
       if (PopCount != 0U)
         break;
 
@@ -938,7 +946,8 @@ template <typename Config> class SizeClassAllocator64 {
       // `pushBatchClassBlocks()` and `mergeGroupsToReleaseBack()`.
       Region->FLLockCV.wait(Region->FLLock);
 
-      PopCount = popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+      PopCount = popBlocksImpl(SizeClassAllocator, ClassId, Region, ToArray,
+                               MaxBlockCount);
       if (PopCount != 0U)
         break;
     }
@@ -946,9 +955,9 @@ template <typename Config> class SizeClassAllocator64 {
     return PopCount;
   }
 
-  u16 popBlocksImpl(CacheT *C, uptr ClassId, RegionInfo *Region,
-                    CompactPtrT *ToArray, const u16 MaxBlockCount)
-      REQUIRES(Region->FLLock) {
+  u16 popBlocksImpl(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
+                    RegionInfo *Region, CompactPtrT *ToArray,
+                    const u16 MaxBlockCount) REQUIRES(Region->FLLock) {
     if (Region->FreeListInfo.BlockList.empty())
       return 0U;
 
@@ -972,11 +981,11 @@ template <typename Config> class SizeClassAllocator64 {
     // So far, instead of always filling blocks to `MaxBlockCount`, we only
     // examine single `TransferBatch` to minimize the time spent in the primary
     // allocator. Besides, the sizes of `TransferBatch` and
-    // `CacheT::getMaxCached()` may also impact the time spent on accessing the
-    // primary allocator.
+    // `SizeClassAllocatorT::getMaxCached()` may also impact the time spent on
+    // accessing the primary allocator.
     // TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
     // blocks and/or adjust the size of `TransferBatch` according to
-    // `CacheT::getMaxCached()`.
+    // `SizeClassAllocatorT::getMaxCached()`.
     TransferBatchT *B = Batches.front();
     DCHECK_NE(B, nullptr);
     DCHECK_GT(B->getCount(), 0U);
@@ -996,7 +1005,7 @@ template <typename Config> class SizeClassAllocator64 {
       // deallocate. Read the comment in `pushBatchClassBlocks()` for more
       // details.
       if (ClassId != SizeClassMap::BatchClassId)
-        C->deallocate(SizeClassMap::BatchClassId, B);
+        SizeClassAllocator->deallocate(SizeClassMap::BatchClassId, B);
 
       if (Batches.empty()) {
         BatchGroupT *BG = Region->FreeListInfo.BlockList.front();
@@ -1008,7 +1017,7 @@ template <typename Config> class SizeClassAllocator64 {
         // Which means, once we pop the last TransferBatch, the block is
         // implicitly deallocated.
         if (ClassId != SizeClassMap::BatchClassId)
-          C->deallocate(SizeClassMap::BatchClassId, BG);
+          SizeClassAllocator->deallocate(SizeClassMap::BatchClassId, BG);
       }
     }
 
@@ -1017,11 +1026,10 @@ template <typename Config> class SizeClassAllocator64 {
     return PopCount;
   }
 
-  NOINLINE u16 populateFreeListAndPopBlocks(CacheT *C, uptr ClassId,
-                                            RegionInfo *Region,
-                                            CompactPtrT *ToArray,
-                                            const u16 MaxBlockCount)
-      REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
+  NOINLINE u16 populateFreeListAndPopBlocks(
+      SizeClassAllocatorT *SizeClassAllocator, uptr ClassId, RegionInfo *Region,
+      CompactPtrT *ToArray, const u16 MaxBlockCount) REQUIRES(Region->MMLock)
+      EXCLUDES(Region->FLLock) {
     if (!Config::getEnableContiguousRegions() &&
         !Region->MemMapInfo.MemMap.isAllocated()) {
       ReservedMemoryT ReservedMemory;
@@ -1040,7 +1048,7 @@ template <typename Config> class SizeClassAllocator64 {
 
     DCHECK(Region->MemMapInfo.MemMap.isAllocated());
     const uptr Size = getSizeByClassId(ClassId);
-    const u16 MaxCount = CacheT::getMaxCached(Size);
+    const u16 MaxCount = SizeClassAllocatorT::getMaxCached(Size);
     const uptr RegionBeg = Region->RegionBeg;
     const uptr MappedUser = Region->MemMapInfo.MappedUser;
     const uptr TotalUserBytes =
@@ -1064,7 +1072,7 @@ template <typename Config> class SizeClassAllocator64 {
         return 0U;
       }
       Region->MemMapInfo.MappedUser += MapSize;
-      C->getStats().add(StatMapped, MapSize);
+      SizeClassAllocator->getStats().add(StatMapped, MapSize);
     }
 
     const u32 NumberOfBlocks =
@@ -1092,7 +1100,8 @@ template <typename Config> class SizeClassAllocator64 {
       for (u32 I = 1; I < NumberOfBlocks; I++) {
         if (UNLIKELY(compactPtrGroup(ShuffleArray[I]) != CurGroup)) {
           shuffle(ShuffleArray + I - N, N, &Region->RandState);
-          pushBlocksImpl(C, ClassId, Region, ShuffleArray + I - N, N,
+          pushBlocksImpl(SizeClassAllocator, ClassId, Region,
+                         ShuffleArray + I - N, N,
                          /*SameGroup=*/true);
           N = 1;
           CurGroup = compactPtrGroup(ShuffleArray[I]);
@@ -1102,14 +1111,15 @@ template <typename Config> class SizeClassAllocator64 {
       }
 
       shuffle(ShuffleArray + NumberOfBlocks - N, N, &Region->RandState);
-      pushBlocksImpl(C, ClassId, Region, &ShuffleArray[NumberOfBlocks - N], N,
+      pushBlocksImpl(SizeClassAllocator, ClassId, Region,
+                     &ShuffleArray[NumberOfBlocks - N], N,
                      /*SameGroup=*/true);
     } else {
       pushBatchClassBlocks(Region, ShuffleArray, NumberOfBlocks);
     }
 
-    const u16 PopCount =
-        popBlocksImpl(C, ClassId, Region, ToArray, MaxBlockCount);
+    const u16 PopCount = popBlocksImpl(SizeClassAllocator, ClassId, Region,
+                                       ToArray, MaxBlockCount);
     DCHECK_NE(PopCount, 0U);
 
     // Note that `PushedBlocks` and `PoppedBlocks` are supposed to only record
@@ -1119,7 +1129,7 @@ template <typename Config> class SizeClassAllocator64 {
     Region->FreeListInfo.PushedBlocks -= NumberOfBlocks;
 
     const uptr AllocatedUser = Size * NumberOfBlocks;
-    C->getStats().add(StatFree, AllocatedUser);
+    SizeClassAllocator->getStats().add(StatFree, AllocatedUser);
     Region->MemMapInfo.AllocatedUser += AllocatedUser;
 
     return PopCount;
diff --git a/compiler-rt/lib/scudo/standalone/local_cache.h b/compiler-rt/lib/scudo/standalone/size_class_allocator.h
similarity index 53%
rename from compiler-rt/lib/scudo/standalone/local_cache.h
rename to compiler-rt/lib/scudo/standalone/size_class_allocator.h
index 46d6affdc033b..7c7d6307f8f0a 100644
--- a/compiler-rt/lib/scudo/standalone/local_cache.h
+++ b/compiler-rt/lib/scudo/standalone/size_class_allocator.h
@@ -1,4 +1,4 @@
-//===-- local_cache.h -------------------------------------------*- C++ -*-===//
+//===-- size_class_allocator.h ----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef SCUDO_LOCAL_CACHE_H_
-#define SCUDO_LOCAL_CACHE_H_
+#ifndef SCUDO_SIZE_CLASS_ALLOCATOR_H_
+#define SCUDO_SIZE_CLASS_ALLOCATOR_H_
 
 #include "internal_defs.h"
 #include "list.h"
@@ -28,7 +28,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
     if (LIKELY(S))
       S->link(&Stats);
     Allocator = A;
-    initCache();
+    initAllocator();
   }
 
   void destroy(GlobalStats *S) {
@@ -112,7 +112,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
 
       EmptyCache = false;
       // The size of BatchClass is set to 0 intentionally. See the comment in
-      // initCache() for more details.
+      // initAllocator() for more details.
       const uptr ClassSize = I == BatchClassId
                                  ? SizeClassAllocator::getSizeByClassId(I)
                                  : PerClassArray[I].ClassSize;
@@ -146,7 +146,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
   LocalStats Stats;
   SizeClassAllocator *Allocator = nullptr;
 
-  NOINLINE void initCache() {
+  NOINLINE void initAllocator() {
     for (uptr I = 0; I < NumClasses; I++) {
       PerClass *P = &PerClassArray[I];
       const uptr Size = SizeClassAllocator::getSizeByClassId(I);
@@ -161,11 +161,6 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
     }
   }
 
-  void destroyBatch(uptr ClassId, void *B) {
-    if (ClassId != BatchClassId)
-      deallocate(BatchClassId, B);
-  }
-
   NOINLINE bool refill(PerClass *C, uptr ClassId, u16 MaxRefill) {
     const u16 NumBlocksRefilled =
         Allocator->popBlocks(this, ClassId, C->Chunks, MaxRefill);
@@ -184,6 +179,148 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
   }
 };
 
+template <class SizeClassAllocator> struct SizeClassAllocatorNoCache {
+  typedef typename SizeClassAllocator::SizeClassMap SizeClassMap;
+  typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
+
+  void init(GlobalStats *S, SizeClassAllocator *A) {
+    Stats.init();
+    if (LIKELY(S))
+      S->link(&Stats);
+    Allocator = A;
+    initAllocator();
+  }
+
+  void destroy(GlobalStats *S) {
+    if (LIKELY(S))
+      S->unlink(&Stats);
+  }
+
+  void *allocate(uptr ClassId) {
+    CompactPtrT CompactPtr;
+    uptr NumBlocksPopped = Allocator->popBlocks(this, ClassId, &CompactPtr, 1U);
+    if (NumBlocksPopped == 0)
+      return nullptr;
+    DCHECK_EQ(NumBlocksPopped, 1U);
+    const PerClass *C = &PerClassArray[ClassId];
+    Stats.add(StatAllocated, C->ClassSize);
+    Stats.sub(StatFree, C->ClassSize);
+    return Allocator->decompactPtr(ClassId, CompactPtr);
+  }
+
+  bool deallocate(uptr ClassId, void *P) {
+    CHECK_LT(ClassId, NumClasses);
+
+    if (ClassId == BatchClassId)
+      return deallocateBatchClassBlock(P);
+
+    CompactPtrT CompactPtr =
+        Allocator->compactPtr(ClassId, reinterpret_cast<uptr>(P));
+    Allocator->pushBlocks(this, ClassId, &CompactPtr, 1U);
+    PerClass *C = &PerClassArray[ClassId];
+    Stats.sub(StatAllocated, C->ClassSize);
+    Stats.add(StatFree, C->ClassSize);
+
+    // The following adopts the same strategy of allocator draining as used
+    // in SizeClassAllocatorLocalCache so that use the same hint when doing
+    // a page release.
+    ++C->Count;
+    const bool SuggestDraining = C->Count >= C->MaxCount;
+    if (SuggestDraining)
+      C->Count = 0;
+    return SuggestDraining;
+  }
+
+  void *getBatchClassBlock() {
+    PerClass *C = &PerClassArray[BatchClassId];
+    if (C->Count == 0) {
+      const u16 NumBlocksRefilled = Allocator->popBlocks(
+          this, BatchClassId, BatchClassStorage, C->MaxCount);
+      if (NumBlocksRefilled == 0)
+        reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
+      DCHECK_LE(NumBlocksRefilled, SizeClassMap::MaxNumCachedHint);
+      C->Count = NumBlocksRefilled;
+    }
+
+    const uptr ClassSize = C->ClassSize;
+    CompactPtrT CompactP = BatchClassStorage[--C->Count];
+    Stats.add(StatAllocated, ClassSize);
+    Stats.sub(StatFree, ClassSize);
+
+    return Allocator->decompactPtr(BatchClassId, CompactP);
+  }
+
+  LocalStats &getStats() { return Stats; }
+
+  void getStats(ScopedString *Str) { Str->append("    No block is cached.\n"); }
+
+  bool isEmpty() const {
+    const PerClass *C = &PerClassArray[BatchClassId];
+    return C->Count == 0;
+  }
+  void drain() {
+    PerClass *C = &PerClassArray[BatchClassId];
+    if (C->Count > 0) {
+      Allocator->pushBlocks(this, BatchClassId, BatchClassStorage, C->Count);
+      C->Count = 0;
+    }
+  }
+
+  static u16 getMaxCached(uptr Size) {
+    return Min(SizeClassMap::MaxNumCachedHint,
+               SizeClassMap::getMaxCachedHint(Size));
+  }
+
+private:
+  static const uptr NumClasses = SizeClassMap::NumClasses;
+  static const uptr BatchClassId = SizeClassMap::BatchClassId;
+  struct alignas(SCUDO_CACHE_LINE_SIZE) PerClass {
+    u16 Count = 0;
+    u16 MaxCount;
+    // Note: ClassSize is zero for the transfer batch.
+    uptr ClassSize;
+  };
+  PerClass PerClassArray[NumClasses] = {};
+  // Popping BatchClass blocks requires taking a certain amount of blocks at
+  // once. This restriction comes from how we manage the storing of BatchClass
+  // in the primary allocator. See more details in `popBlocksImpl` in the
+  // primary allocator.
+  CompactPtrT BatchClassStorage[SizeClassMap::MaxNumCachedHint] = {};
+  LocalStats Stats;
+  SizeClassAllocator *Allocator = nullptr;
+
+  bool deallocateBatchClassBlock(void *P) {
+    PerClass *C = &PerClassArray[BatchClassId];
+    // Drain all the blocks.
+    if (C->Count >= C->MaxCount) {
+      Allocator->pushBlocks(this, BatchClassId, BatchClassStorage, C->Count);
+      C->Count = 0;
+    }
+    BatchClassStorage[C->Count++] =
+        Allocator->compactPtr(BatchClassId, reinterpret_cast<uptr>(P));
+
+    // Currently, BatchClass doesn't support page releasing, so we always return
+    // false.
+    return false;
+  }
+
+  NOINLINE void initAllocator() {
+    for (uptr I = 0; I < NumClasses; I++) {
+      PerClass *P = &PerClassArray[I];
+      const uptr Size = SizeClassAllocator::getSizeByClassId(I);
+      if (I != BatchClassId) {
+        P->ClassSize = Size;
+        P->MaxCount = static_cast<u16>(2 * getMaxCached(Size));
+      } else {
+        // ClassSize in this struct is only used for malloc/free stats, which
+        // should only track user allocations, not internal movements.
+        P->ClassSize = 0;
+        P->MaxCount = SizeClassMap::MaxNumCachedHint;
+      }
+    }
+  }
+};
+
 } // namespace scudo
 
-#endif // SCUDO_LOCAL_CACHE_H_
+#endif // SCUDO_SIZE_CLASS_ALLOCATOR_H_
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index 9d665ef88c402..7e8d5b4396d2e 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -212,6 +212,47 @@ struct TestConditionVariableConfig {
   };
   template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
 };
+
+struct TestNoCacheConfig {
+  static const bool MaySupportMemoryTagging = true;
+  template <class A>
+  using TSDRegistryT =
+      scudo::TSDRegistrySharedT<A, 8U, 4U>; // Shared, max 8 TSDs.
+
+  struct Primary {
+    using SizeClassMap = scudo::AndroidSizeClassMap;
+#if SCUDO_CAN_USE_PRIMARY64
+    static const scudo::uptr RegionSizeLog = 28U;
+    typedef scudo::u32 CompactPtrT;
+    static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+    static const scudo::uptr GroupSizeLog = 20U;
+    static const bool EnableRandomOffset = true;
+    static const scudo::uptr MapSizeIncrement = 1UL << 18;
+#else
+    static const scudo::uptr RegionSizeLog = 18U;
+    static const scudo::uptr GroupSizeLog = 18U;
+    typedef scudo::uptr CompactPtrT;
+#endif
+    static const bool EnableBlockCache = false;
+    static const scudo::s32 MinReleaseToOsIntervalMs = 1000;
+    static const scudo::s32 MaxReleaseToOsIntervalMs = 1000;
+  };
+
+#if SCUDO_CAN_USE_PRIMARY64
+  template <typename Config>
+  using PrimaryT = scudo::SizeClassAllocator64<Config>;
+#else
+  template <typename Config>
+  using PrimaryT = scudo::SizeClassAllocator32<Config>;
+#endif
+
+  struct Secondary {
+    template <typename Config>
+    using CacheT = scudo::MapAllocatorNoCache<Config>;
+  };
+  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
+};
+
 } // namespace scudo
 
 #if SCUDO_FUCHSIA
@@ -221,7 +262,8 @@ struct TestConditionVariableConfig {
 #define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME)                              \
   SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, DefaultConfig)                          \
   SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig)                          \
-  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConditionVariableConfig)
+  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConditionVariableConfig)            \
+  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestNoCacheConfig)
 #endif
 
 #define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE)                             \
@@ -597,7 +639,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, Stats) {
   EXPECT_NE(Stats.find("Stats: Quarantine"), std::string::npos);
 }
 
-SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(ScudoCombinedTest, CacheDrain) {
+SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(ScudoCombinedTest, Drain) {
   using AllocatorT = typename BaseT::AllocatorT;
   auto *Allocator = this->Allocator.get();
 
@@ -612,9 +654,9 @@ SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(ScudoCombinedTest, CacheDrain) {
 
   typename AllocatorT::TSDRegistryT::ScopedTSD TSD(
       *Allocator->getTSDRegistry());
-  EXPECT_TRUE(!TSD->getCache().isEmpty());
-  TSD->getCache().drain();
-  EXPECT_TRUE(TSD->getCache().isEmpty());
+  EXPECT_TRUE(!TSD->getSizeClassAllocator().isEmpty());
+  TSD->getSizeClassAllocator().drain();
+  EXPECT_TRUE(TSD->getSizeClassAllocator().isEmpty());
 }
 
 SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(ScudoCombinedTest, ForceCacheDrain) {
@@ -635,7 +677,7 @@ SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(ScudoCombinedTest, ForceCacheDrain) {
 
   typename AllocatorT::TSDRegistryT::ScopedTSD TSD(
       *Allocator->getTSDRegistry());
-  EXPECT_TRUE(TSD->getCache().isEmpty());
+  EXPECT_TRUE(TSD->getSizeClassAllocator().isEmpty());
   EXPECT_EQ(TSD->getQuarantineCache().getSize(), 0U);
   EXPECT_TRUE(Allocator->getQuarantine()->isEmpty());
 }
@@ -1027,7 +1069,7 @@ TEST(ScudoCombinedTest, BasicTrustyConfig) {
   bool UnlockRequired;
   typename AllocatorT::TSDRegistryT::ScopedTSD TSD(
       *Allocator->getTSDRegistry());
-  TSD->getCache().drain();
+  TSD->getSizeClassAllocator().drain();
 
   Allocator->releaseToOS(scudo::ReleaseToOS::Force);
 }
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 0636fe73c0b59..7dc38c2ede8ca 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -211,8 +211,8 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, BasicPrimary) {
   using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
   std::unique_ptr<Primary> Allocator(new Primary);
   Allocator->init(/*ReleaseToOsInterval=*/-1);
-  typename Primary::CacheT Cache;
-  Cache.init(nullptr, Allocator.get());
+  typename Primary::SizeClassAllocatorT SizeClassAllocator;
+  SizeClassAllocator.init(nullptr, Allocator.get());
   const scudo::uptr NumberOfAllocations = 32U;
   for (scudo::uptr I = 0; I <= 16U; I++) {
     const scudo::uptr Size = 1UL << I;
@@ -221,14 +221,14 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, BasicPrimary) {
     const scudo::uptr ClassId = Primary::SizeClassMap::getClassIdBySize(Size);
     void *Pointers[NumberOfAllocations];
     for (scudo::uptr J = 0; J < NumberOfAllocations; J++) {
-      void *P = Cache.allocate(ClassId);
+      void *P = SizeClassAllocator.allocate(ClassId);
       memset(P, 'B', Size);
       Pointers[J] = P;
     }
     for (scudo::uptr J = 0; J < NumberOfAllocations; J++)
-      Cache.deallocate(ClassId, Pointers[J]);
+      SizeClassAllocator.deallocate(ClassId, Pointers[J]);
   }
-  Cache.destroy(nullptr);
+  SizeClassAllocator.destroy(nullptr);
   Allocator->releaseToOS(scudo::ReleaseToOS::Force);
   scudo::ScopedString Str;
   Allocator->getStats(&Str);
@@ -261,19 +261,20 @@ TEST(ScudoPrimaryTest, Primary64OOM) {
       scudo::SizeClassAllocator64<scudo::PrimaryConfig<SmallRegionsConfig>>;
   Primary Allocator;
   Allocator.init(/*ReleaseToOsInterval=*/-1);
-  typename Primary::CacheT Cache;
+  typename Primary::SizeClassAllocatorT SizeClassAllocator;
   scudo::GlobalStats Stats;
   Stats.init();
-  Cache.init(&Stats, &Allocator);
+  SizeClassAllocator.init(&Stats, &Allocator);
   bool AllocationFailed = false;
   std::vector<void *> Blocks;
   const scudo::uptr ClassId = Primary::SizeClassMap::LargestClassId;
   const scudo::uptr Size = Primary::getSizeByClassId(ClassId);
-  const scudo::u16 MaxCachedBlockCount = Primary::CacheT::getMaxCached(Size);
+  const scudo::u16 MaxCachedBlockCount =
+      Primary::SizeClassAllocatorT::getMaxCached(Size);
 
   for (scudo::uptr I = 0; I < 10000U; I++) {
     for (scudo::uptr J = 0; J < MaxCachedBlockCount; ++J) {
-      void *Ptr = Cache.allocate(ClassId);
+      void *Ptr = SizeClassAllocator.allocate(ClassId);
       if (Ptr == nullptr) {
         AllocationFailed = true;
         break;
@@ -284,9 +285,9 @@ TEST(ScudoPrimaryTest, Primary64OOM) {
   }
 
   for (auto *Ptr : Blocks)
-    Cache.deallocate(ClassId, Ptr);
+    SizeClassAllocator.deallocate(ClassId, Ptr);
 
-  Cache.destroy(nullptr);
+  SizeClassAllocator.destroy(nullptr);
   Allocator.releaseToOS(scudo::ReleaseToOS::Force);
   scudo::ScopedString Str;
   Allocator.getStats(&Str);
@@ -299,14 +300,14 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryIterate) {
   using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
   std::unique_ptr<Primary> Allocator(new Primary);
   Allocator->init(/*ReleaseToOsInterval=*/-1);
-  typename Primary::CacheT Cache;
-  Cache.init(nullptr, Allocator.get());
+  typename Primary::SizeClassAllocatorT SizeClassAllocator;
+  SizeClassAllocator.init(nullptr, Allocator.get());
   std::vector<std::pair<scudo::uptr, void *>> V;
   for (scudo::uptr I = 0; I < 64U; I++) {
     const scudo::uptr Size =
         static_cast<scudo::uptr>(std::rand()) % Primary::SizeClassMap::MaxSize;
     const scudo::uptr ClassId = Primary::SizeClassMap::getClassIdBySize(Size);
-    void *P = Cache.allocate(ClassId);
+    void *P = SizeClassAllocator.allocate(ClassId);
     V.push_back(std::make_pair(ClassId, P));
   }
   scudo::uptr Found = 0;
@@ -322,10 +323,10 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryIterate) {
   EXPECT_EQ(Found, V.size());
   while (!V.empty()) {
     auto Pair = V.back();
-    Cache.deallocate(Pair.first, Pair.second);
+    SizeClassAllocator.deallocate(Pair.first, Pair.second);
     V.pop_back();
   }
-  Cache.destroy(nullptr);
+  SizeClassAllocator.destroy(nullptr);
   Allocator->releaseToOS(scudo::ReleaseToOS::Force);
   scudo::ScopedString Str;
   Allocator->getStats(&Str);
@@ -342,8 +343,9 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
   std::thread Threads[32];
   for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++) {
     Threads[I] = std::thread([&]() {
-      static thread_local typename Primary::CacheT Cache;
-      Cache.init(nullptr, Allocator.get());
+      static thread_local
+          typename Primary::SizeClassAllocatorT SizeClassAllocator;
+      SizeClassAllocator.init(nullptr, Allocator.get());
       std::vector<std::pair<scudo::uptr, void *>> V;
       {
         std::unique_lock<std::mutex> Lock(Mutex);
@@ -355,7 +357,7 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
                                  Primary::SizeClassMap::MaxSize / 4;
         const scudo::uptr ClassId =
             Primary::SizeClassMap::getClassIdBySize(Size);
-        void *P = Cache.allocate(ClassId);
+        void *P = SizeClassAllocator.allocate(ClassId);
         if (P)
           V.push_back(std::make_pair(ClassId, P));
       }
@@ -365,14 +367,14 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
 
       while (!V.empty()) {
         auto Pair = V.back();
-        Cache.deallocate(Pair.first, Pair.second);
+        SizeClassAllocator.deallocate(Pair.first, Pair.second);
         V.pop_back();
         // This increases the chance of having non-full TransferBatches and it
         // will jump into the code path of merging TransferBatches.
         if (std::rand() % 8 == 0)
-          Cache.drain();
+          SizeClassAllocator.drain();
       }
-      Cache.destroy(nullptr);
+      SizeClassAllocator.destroy(nullptr);
     });
   }
   {
@@ -397,15 +399,15 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, ReleaseToOS) {
   using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
   std::unique_ptr<Primary> Allocator(new Primary);
   Allocator->init(/*ReleaseToOsInterval=*/-1);
-  typename Primary::CacheT Cache;
-  Cache.init(nullptr, Allocator.get());
+  typename Primary::SizeClassAllocatorT SizeClassAllocator;
+  SizeClassAllocator.init(nullptr, Allocator.get());
   const scudo::uptr Size = scudo::getPageSizeCached() * 2;
   EXPECT_TRUE(Primary::canAllocate(Size));
   const scudo::uptr ClassId = Primary::SizeClassMap::getClassIdBySize(Size);
-  void *P = Cache.allocate(ClassId);
+  void *P = SizeClassAllocator.allocate(ClassId);
   EXPECT_NE(P, nullptr);
-  Cache.deallocate(ClassId, P);
-  Cache.destroy(nullptr);
+  SizeClassAllocator.deallocate(ClassId, P);
+  SizeClassAllocator.destroy(nullptr);
   EXPECT_GT(Allocator->releaseToOS(scudo::ReleaseToOS::ForceAll), 0U);
 }
 
@@ -413,8 +415,8 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, MemoryGroup) {
   using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
   std::unique_ptr<Primary> Allocator(new Primary);
   Allocator->init(/*ReleaseToOsInterval=*/-1);
-  typename Primary::CacheT Cache;
-  Cache.init(nullptr, Allocator.get());
+  typename Primary::SizeClassAllocatorT SizeClassAllocator;
+  SizeClassAllocator.init(nullptr, Allocator.get());
   const scudo::uptr Size = 32U;
   const scudo::uptr ClassId = Primary::SizeClassMap::getClassIdBySize(Size);
 
@@ -434,27 +436,31 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, MemoryGroup) {
   std::mt19937 R;
 
   for (scudo::uptr I = 0; I < PeakNumberOfAllocations; ++I)
-    Blocks.push_back(reinterpret_cast<scudo::uptr>(Cache.allocate(ClassId)));
+    Blocks.push_back(
+        reinterpret_cast<scudo::uptr>(SizeClassAllocator.allocate(ClassId)));
 
   std::shuffle(Blocks.begin(), Blocks.end(), R);
 
   // Release all the allocated blocks, including those held by local cache.
   while (!Blocks.empty()) {
-    Cache.deallocate(ClassId, reinterpret_cast<void *>(Blocks.back()));
+    SizeClassAllocator.deallocate(ClassId,
+                                  reinterpret_cast<void *>(Blocks.back()));
     Blocks.pop_back();
   }
-  Cache.drain();
+  SizeClassAllocator.drain();
 
   for (scudo::uptr I = 0; I < FinalNumberOfAllocations; ++I)
-    Blocks.push_back(reinterpret_cast<scudo::uptr>(Cache.allocate(ClassId)));
+    Blocks.push_back(
+        reinterpret_cast<scudo::uptr>(SizeClassAllocator.allocate(ClassId)));
 
   EXPECT_LE(*std::max_element(Blocks.begin(), Blocks.end()) -
                 *std::min_element(Blocks.begin(), Blocks.end()),
             GroupSizeMem * 2);
 
   while (!Blocks.empty()) {
-    Cache.deallocate(ClassId, reinterpret_cast<void *>(Blocks.back()));
+    SizeClassAllocator.deallocate(ClassId,
+                                  reinterpret_cast<void *>(Blocks.back()));
     Blocks.pop_back();
   }
-  Cache.drain();
+  SizeClassAllocator.drain();
 }
diff --git a/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp b/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp
index 851ac46b9f066..ad50ab3cd75ea 100644
--- a/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp
@@ -26,7 +26,7 @@ template <class Config> class MockAllocator {
 public:
   using ThisT = MockAllocator<Config>;
   using TSDRegistryT = typename Config::template TSDRegistryT<ThisT>;
-  using CacheT = struct MockCache {
+  using SizeClassAllocatorT = struct MockSizeClassAllocator {
     volatile scudo::uptr Canary;
   };
   using QuarantineCacheT = struct MockQuarantine {};
@@ -38,7 +38,9 @@ template <class Config> class MockAllocator {
   }
 
   void unmapTestOnly() { TSDRegistry.unmapTestOnly(this); }
-  void initCache(CacheT *Cache) { *Cache = {}; }
+  void initAllocator(SizeClassAllocatorT *SizeClassAllocator) {
+    *SizeClassAllocator = {};
+  }
   void commitBack(UNUSED scudo::TSD<MockAllocator> *TSD) {}
   TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
   void callPostInitCallback() {}
@@ -103,14 +105,15 @@ static void testRegistry() NO_THREAD_SAFETY_ANALYSIS {
 
   {
     typename AllocatorT::TSDRegistryT::ScopedTSD TSD(*Registry);
-    EXPECT_EQ(TSD->getCache().Canary, 0U);
+    EXPECT_EQ(TSD->getSizeClassAllocator().Canary, 0U);
   }
 
   Registry->initThreadMaybe(Allocator.get(), /*MinimalInit=*/false);
   {
     typename AllocatorT::TSDRegistryT::ScopedTSD TSD(*Registry);
-    EXPECT_EQ(TSD->getCache().Canary, 0U);
-    memset(&TSD->getCache(), 0x42, sizeof(TSD->getCache()));
+    EXPECT_EQ(TSD->getSizeClassAllocator().Canary, 0U);
+    memset(&TSD->getSizeClassAllocator(), 0x42,
+           sizeof(TSD->getSizeClassAllocator()));
   }
 }
 
@@ -126,10 +129,10 @@ static std::mutex Mutex;
 static std::condition_variable Cv;
 static bool Ready;
 
-// Accessing `TSD->getCache()` requires `TSD::Mutex` which isn't easy to test
-// using thread-safety analysis. Alternatively, we verify the thread safety
-// through a runtime check in ScopedTSD and mark the test body with
-// NO_THREAD_SAFETY_ANALYSIS.
+// Accessing `TSD->getSizeClassAllocator()` requires `TSD::Mutex` which isn't
+// easy to test using thread-safety analysis. Alternatively, we verify the
+// thread safety through a runtime check in ScopedTSD and mark the test body
+// with NO_THREAD_SAFETY_ANALYSIS.
 template <typename AllocatorT>
 static void stressCache(AllocatorT *Allocator) NO_THREAD_SAFETY_ANALYSIS {
   auto Registry = Allocator->getTSDRegistry();
@@ -144,15 +147,15 @@ static void stressCache(AllocatorT *Allocator) NO_THREAD_SAFETY_ANALYSIS {
   // same for a shared TSD.
   if (std::is_same<typename AllocatorT::TSDRegistryT,
                    scudo::TSDRegistryExT<AllocatorT>>()) {
-    EXPECT_EQ(TSD->getCache().Canary, 0U);
+    EXPECT_EQ(TSD->getSizeClassAllocator().Canary, 0U);
   }
   // Transform the thread id to a uptr to use it as canary.
   const scudo::uptr Canary = static_cast<scudo::uptr>(
       std::hash<std::thread::id>{}(std::this_thread::get_id()));
-  TSD->getCache().Canary = Canary;
+  TSD->getSizeClassAllocator().Canary = Canary;
   // Loop a few times to make sure that a concurrent thread isn't modifying it.
   for (scudo::uptr I = 0; I < 4096U; I++)
-    EXPECT_EQ(TSD->getCache().Canary, Canary);
+    EXPECT_EQ(TSD->getSizeClassAllocator().Canary, Canary);
 }
 
 template <class AllocatorT> static void testRegistryThreaded() {
diff --git a/compiler-rt/lib/scudo/standalone/tsd.h b/compiler-rt/lib/scudo/standalone/tsd.h
index 72773f2f72b11..9303f57695cda 100644
--- a/compiler-rt/lib/scudo/standalone/tsd.h
+++ b/compiler-rt/lib/scudo/standalone/tsd.h
@@ -31,7 +31,7 @@ template <class Allocator> struct alignas(SCUDO_CACHE_LINE_SIZE) TSD {
   void init(Allocator *Instance) NO_THREAD_SAFETY_ANALYSIS {
     DCHECK_EQ(DestructorIterations, 0U);
     DCHECK(isAligned(reinterpret_cast<uptr>(this), alignof(ThisT)));
-    Instance->initCache(&Cache);
+    Instance->initAllocator(&SizeClassAllocator);
     DestructorIterations = PTHREAD_DESTRUCTOR_ITERATIONS;
   }
 
@@ -72,7 +72,10 @@ template <class Allocator> struct alignas(SCUDO_CACHE_LINE_SIZE) TSD {
   // TODO(chiahungduan): Ideally, we want to do `Mutex.assertHeld` but acquiring
   // TSD doesn't always require holding the lock. Add this assertion while the
   // lock is always acquired.
-  typename Allocator::CacheT &getCache() REQUIRES(Mutex) { return Cache; }
+  typename Allocator::SizeClassAllocatorT &getSizeClassAllocator()
+      REQUIRES(Mutex) {
+    return SizeClassAllocator;
+  }
   typename Allocator::QuarantineCacheT &getQuarantineCache() REQUIRES(Mutex) {
     return QuarantineCache;
   }
@@ -81,7 +84,7 @@ template <class Allocator> struct alignas(SCUDO_CACHE_LINE_SIZE) TSD {
   HybridMutex Mutex;
   atomic_uptr Precedence = {};
 
-  typename Allocator::CacheT Cache GUARDED_BY(Mutex);
+  typename Allocator::SizeClassAllocatorT SizeClassAllocator GUARDED_BY(Mutex);
   typename Allocator::QuarantineCacheT QuarantineCache GUARDED_BY(Mutex);
 };
 
diff --git a/compiler-rt/lib/scudo/standalone/tsd_shared.h b/compiler-rt/lib/scudo/standalone/tsd_shared.h
index dade16dad9f2d..8b570a770b503 100644
--- a/compiler-rt/lib/scudo/standalone/tsd_shared.h
+++ b/compiler-rt/lib/scudo/standalone/tsd_shared.h
@@ -127,7 +127,7 @@ struct TSDRegistrySharedT {
       // analyzer.
       TSDs[I].assertLocked(/*BypassCheck=*/true);
       Str->append("  Shared TSD[%zu]:\n", I);
-      TSDs[I].getCache().getStats(Str);
+      TSDs[I].getSizeClassAllocator().getStats(Str);
       TSDs[I].unlock();
     }
   }
diff --git a/compiler-rt/lib/scudo/standalone/type_traits.h b/compiler-rt/lib/scudo/standalone/type_traits.h
index 16ed5a048f82b..1c36a83ade02f 100644
--- a/compiler-rt/lib/scudo/standalone/type_traits.h
+++ b/compiler-rt/lib/scudo/standalone/type_traits.h
@@ -42,6 +42,14 @@ template <typename T> struct isPointer<T *> {
   static constexpr bool value = true;
 };
 
+template <bool Cond, typename L, typename R> struct Conditional {
+  using type = L;
+};
+
+template <typename L, typename R> struct Conditional<false, L, R> {
+  using type = R;
+};
+
 } // namespace scudo
 
 #endif // SCUDO_TYPE_TRAITS_H_

From 94aa4bfba57cd5df5489ceca5c1b4b59cfcc1288 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 17 Apr 2025 14:38:21 -0700
Subject: [PATCH 327/710] [NVPTX] Fix a warning

This patch fixes:

  llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h:128:21: error: unused
  variable '_' [-Werror,-Wunused-variable]
---
 llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
index 9ed7e650e7b0c..aa953f4596bbd 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
@@ -125,8 +125,10 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
     }
 
     void addZeros(unsigned Num) {
-      for (unsigned _ : llvm::seq(Num))
+      for (unsigned _ : llvm::seq(Num)) {
+        (void)_;
         addByte(0);
+      }
     }
 
     void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) {

From 209d8c8fa4fe16ef41003da17387f7c271002668 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Thu, 17 Apr 2025 18:40:18 -0300
Subject: [PATCH 328/710] [clang] fix unresolved dependent template
 specialization mangling (#136201)

This fixes a regression introduced in #133610 which was reported here
#133610 (comment) and in #136119

This redoes previous attempt in #135111

When mangling a DTST which appears in the prefix,
the template name is not actually relevant, as its prefix is part of the
nested name anyway, and a
substitution is not allowed at that position in any case.

Fixes #136119
---
 clang/lib/AST/ItaniumMangle.cpp           | 13 ++-----------
 clang/test/CodeGenCXX/mangle-template.cpp | 17 +++++++++++++++++
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index d0ab60700cb15..977caa8e55db3 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -1405,16 +1405,6 @@ void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
     //   - a template template parameter with arguments
     // In all of these cases, we should have no prefix.
     if (NestedNameSpecifier *Prefix = qualifier->getPrefix()) {
-      if (const auto *DTST =
-              dyn_cast<DependentTemplateSpecializationType>(type)) {
-        Out << "srN";
-        TemplateName Template = getASTContext().getDependentTemplateName(
-            {Prefix, DTST->getDependentTemplateName().getName(),
-             /*HasTemplateKeyword=*/true});
-        mangleTemplatePrefix(Template);
-        mangleTemplateArgs(Template, DTST->template_arguments());
-        break;
-      }
       mangleUnresolvedPrefix(Prefix,
                              /*recursive=*/true);
     } else {
@@ -2618,7 +2608,8 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
         cast<DependentTemplateSpecializationType>(Ty);
     TemplateName Template = getASTContext().getDependentTemplateName(
         DTST->getDependentTemplateName());
-    mangleTemplatePrefix(Template);
+    const DependentTemplateStorage &S = DTST->getDependentTemplateName();
+    mangleSourceName(S.getName().getIdentifier());
     mangleTemplateArgs(Template, DTST->template_arguments());
     break;
   }
diff --git a/clang/test/CodeGenCXX/mangle-template.cpp b/clang/test/CodeGenCXX/mangle-template.cpp
index 365aba5989baa..867f00f379ead 100644
--- a/clang/test/CodeGenCXX/mangle-template.cpp
+++ b/clang/test/CodeGenCXX/mangle-template.cpp
@@ -416,3 +416,20 @@ namespace unresolved_template_specialization_type {
   template enable_if<true> raw_hash_set<int>::AbslHashValue<HashStateBase>();
   // CHECH: @_ZN39unresolved_template_specialization_type12raw_hash_setIiE13AbslHashValueINS_13HashStateBaseEEENS_9enable_ifIXsrNT_11is_hashableIiEE5valueEEEv
 } // namespace unresolved_template_specialization_type
+
+namespace GH133610 {
+  template <class T> struct A {
+    template <class V> struct B { int MEM; };
+  };
+
+  struct D {};
+  struct C: public A<int>::B<D> {};
+
+  template <class T, class U, class V>
+  auto k(T t, U u, V v) -> decltype (t.U::template B<V>::MEM) { return {}; }
+
+  void t() {
+    k( C(), A<int>(), D() );
+  }
+  // CHECK: @_ZN8GH1336101kINS_1CENS_1AIiEENS_1DEEEDtdtfp_sr1U1BIT1_EE3MEMET_T0_S5_
+} // namespace GH133610

From 69b9ddc76418c6f60ce7751efb5beb1f3b3be3ff Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 17 Apr 2025 15:09:24 -0700
Subject: [PATCH 329/710] [Analysis] Restore the call to reserve (#136215)

commit 47d8fec9b8bd5425af17711317a41743a30a8cef
  Author: Kazu Hirata <kazu@google.com>
  Date:   Wed Apr 16 19:30:01 2025 -0700

has removed the call to reserve here.  This patch restores it as
std::vector::insert, called by llvm::append_range, may not be able to
deduce the size of the range.  For example, std::vector in libc++
distinguishes has two versions of std::vector::insert depending on
whether the iterator is an input iterator or a foward iterator.
---
 llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 57c408968b19a..2237f105e328b 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -1159,6 +1159,7 @@ void BlockFrequencyInfoImpl<BT>::setBlockFreq(const BlockT *BB,
 
 template <class BT> void BlockFrequencyInfoImpl<BT>::initializeRPOT() {
   const BlockT *Entry = &F->front();
+  RPOT.reserve(F->size());
   llvm::append_range(RPOT, post_order(Entry));
   std::reverse(RPOT.begin(), RPOT.end());
 

From 20cd74a1c1555385e65d26ff22de7ef75fcb4090 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Thu, 17 Apr 2025 15:14:52 -0700
Subject: [PATCH 330/710] [sanitizer] Apply AlwaysIn/Out in parseSanitizeArgs
 (#129405)

For backwards compatibility, `parseSanitizeArgs`/`parseSanitizeTrapArgs` had an incomplete refactoring in
https://github.com/llvm/llvm-project/pull/119819, in order to accommodate the special case of vptr in -fsanitize=undefined and its interaction with -fsanitize-trap=undefined. Now that vptr is no longer part of -fsanitize=undefined (https://github.com/llvm/llvm-project/pull/121115), this patch changes parseSanitizeArgs to apply the AlwaysIn/Out invariants in parseSanitizeArgs, which allows simplifying calls to parseSanitizeArgs.

This is not quite NFC: it changes the error message of -fsanitize-trap=vptr.
---
 clang/lib/Driver/SanitizerArgs.cpp | 14 ++++----------
 clang/test/Driver/fsanitize.c      |  2 +-
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index ee151d5c68b85..f27cb813012f2 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -262,11 +262,8 @@ static SanitizerMask setGroupBits(SanitizerMask Kinds) {
 }
 
 // Computes the sanitizer mask as:
-//     Default + Arguments (in or out)
+//     Default + Arguments (in or out) + AlwaysIn - AlwaysOut
 // with arguments parsed from left to right.
-//
-// Error messages are printed if the AlwaysIn or AlwaysOut invariants are
-// violated, but the caller must enforce these invariants themselves.
 static SanitizerMask
 parseSanitizeArgs(const Driver &D, const llvm::opt::ArgList &Args,
                   bool DiagnoseErrors, SanitizerMask Default,
@@ -316,6 +313,9 @@ parseSanitizeArgs(const Driver &D, const llvm::opt::ArgList &Args,
     }
   }
 
+  Output |= AlwaysIn;
+  Output &= ~AlwaysOut;
+
   return Output;
 }
 
@@ -325,10 +325,6 @@ static SanitizerMask parseSanitizeTrapArgs(const Driver &D,
   SanitizerMask AlwaysTrap; // Empty
   SanitizerMask NeverTrap = ~(setGroupBits(TrappingSupported));
 
-  // N.B. We do *not* enforce NeverTrap. This maintains the behavior of
-  // '-fsanitize=undefined -fsanitize-trap=undefined'
-  // (clang/test/Driver/fsanitize.c ), which is that vptr is not enabled at all
-  // (not even in recover mode) in order to avoid the need for a ubsan runtime.
   return parseSanitizeArgs(D, Args, DiagnoseErrors, TrappingDefault, AlwaysTrap,
                            NeverTrap, options::OPT_fsanitize_trap_EQ,
                            options::OPT_fno_sanitize_trap_EQ);
@@ -727,8 +723,6 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC,
       D, Args, DiagnoseErrors, RecoverableByDefault, AlwaysRecoverable,
       Unrecoverable, options::OPT_fsanitize_recover_EQ,
       options::OPT_fno_sanitize_recover_EQ);
-  RecoverableKinds |= AlwaysRecoverable;
-  RecoverableKinds &= ~Unrecoverable;
   RecoverableKinds &= Kinds;
 
   TrappingKinds &= Kinds;
diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c
index 8652f1d875e68..c154e339941f2 100644
--- a/clang/test/Driver/fsanitize.c
+++ b/clang/test/Driver/fsanitize.c
@@ -149,7 +149,7 @@
 // CHECK-FSANITIZE-SHIFT-PARTIAL: "-fsanitize=shift-exponent"
 
 // RUN: not %clang --target=x86_64-linux-gnu -fsanitize=vptr -fsanitize-trap=vptr %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-VPTR-TRAP-UNDEF
-// CHECK-VPTR-TRAP-UNDEF: error: invalid argument '-fsanitize=vptr' not allowed with '-fsanitize-trap=undefined'
+// CHECK-VPTR-TRAP-UNDEF: error: unsupported argument 'vptr' to option '-fsanitize-trap='
 
 // RUN: %clang --target=x86_64-linux-gnu -fsanitize=vptr -fsanitize-undefined-trap-on-error %s -###
 

From 0abf227c13eb80f15659f747094df2db1c34d20d Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin@gmail.com>
Date: Fri, 18 Apr 2025 00:23:49 +0200
Subject: [PATCH 331/710] [mlir][amdgpu] Add `amdgpu.swizzle_bitmode` op
 (#135513)

High level wrapper on top of `rocdl.ds_swizzle`. Also some DPP op
cleanup while I'm at here.

Will do lowering in separate PR.
---
 mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td | 43 +++++++++++++++----
 mlir/test/Dialect/AMDGPU/invalid.mlir         |  8 ++++
 mlir/test/Dialect/AMDGPU/ops.mlir             |  7 +++
 3 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index 108d7237ff703..a92ebf6d8e108 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -35,6 +35,11 @@ def AMDGPU_Dialect : Dialect {
   let useDefaultAttributePrinterParser = 1;
 }
 
+def AnyIntegerOrFloat : AnyTypeOf<[AnySignlessInteger, AnyFloat], "Integer or Float">;
+
+def AnyIntegerOrFloatOr1DVector :
+  AnyTypeOf<[AnyIntegerOrFloat, VectorOfRankAndType<[1], [AnyIntegerOrFloat]>]>;
+
 //===----------------------------------------------------------------------===//
 // AMDGPU general attribute definitions
 //===----------------------------------------------------------------------===//
@@ -533,14 +538,15 @@ def AMDGPU_DPPPerm : I32EnumAttr<"DPPPerm",
 def AMDGPU_DPPPermAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_DPPPerm,
   "dpp_perm">;
 
-def AMDGPU_DPPOp : AMDGPU_Op<"dpp", [SameTypeOperands, AllTypesMatch<["result", "old", "src"]>]>,
+def AMDGPU_DPPOp : AMDGPU_Op<"dpp",
+    [Pure, SameTypeOperands, AllTypesMatch<["result", "old", "src"]>]>,
   Arguments<(ins AnyType:$old,
-                  AnyType:$src,
-                  AMDGPU_DPPPermAttr:$kind,
-                  OptionalAttr<AnyAttrOf<[I32Attr, ArrayAttr, UnitAttr]>>:$permArgument,
-                  DefaultValuedAttr<I32Attr, "0xf">:$row_mask,
-                  DefaultValuedAttr<I32Attr, "0xf">:$bank_mask,
-                  DefaultValuedAttr<BoolAttr, "false">:$bound_ctrl)> {
+                 AnyType:$src,
+                 AMDGPU_DPPPermAttr:$kind,
+                 OptionalAttr<AnyAttrOf<[I32Attr, ArrayAttr, UnitAttr]>>:$permArgument,
+                 DefaultValuedAttr<I32Attr, "0xf">:$row_mask,
+                 DefaultValuedAttr<I32Attr, "0xf">:$bank_mask,
+                 DefaultValuedAttr<BoolAttr, "false">:$bound_ctrl)> {
   let summary = "AMDGPU DPP operation";
   let description = [{
     This operation represents DPP functionality in a GPU program.
@@ -565,6 +571,27 @@ def AMDGPU_DPPOp : AMDGPU_Op<"dpp", [SameTypeOperands, AllTypesMatch<["result",
   let hasVerifier = 1;
 }
 
+def AMDGPU_SwizzleBitModeOp : AMDGPU_Op<"swizzle_bitmode",
+    [Pure, AllTypesMatch<["result", "src"]>]>,
+  Arguments<(ins AnyIntegerOrFloatOr1DVector:$src,
+                 I32Attr:$and_mask,
+                 I32Attr:$or_mask,
+                 I32Attr:$xor_mask
+             )> {
+  let summary = "AMDGPU ds_swizzle op, bitmode variant";
+  let description = [{
+    High-level wrapper on bitmode `rocdl.ds_swizzle` op, masks are represented
+    as separate fields so user won't need to do manual bitpacking.
+
+    Supports arbitrary int/float/vector types, which will be repacked to i32 and
+    one or more `rocdl.ds_swizzle` ops during lowering.
+  }];
+  let results = (outs AnyIntegerOrFloatOr1DVector:$result);
+  let assemblyFormat = [{
+    $src $and_mask $or_mask $xor_mask attr-dict `:` type($result)
+  }];
+}
+
 def AMDGPU_LDSBarrierOp : AMDGPU_Op<"lds_barrier"> {
   let summary = "Barrier that includes a wait for LDS memory operations.";
   let description = [{
@@ -794,7 +821,7 @@ def AMDGPU_GatherToLDSOp :
 
     The `$dst`, along with its indices, points to the memory location the subgroup of this thread
     will write to.
-  
+
     Note: only enabled for gfx942 and later.
   }];
   let assemblyFormat = [{
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index 74a421f6dd50f..40f98ff85688c 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -150,3 +150,11 @@ func.func @fat_raw_buffer_cast_stripping_offset_affine_map(%m: memref<8xi32, aff
   %ret = amdgpu.fat_raw_buffer_cast %m resetOffset : memref<8xi32, affine_map<(d0)[s0] -> (d0 + s0)>> to memref<8xi32, #amdgpu.address_space<fat_raw_buffer>>
   func.return %ret : memref<8xi32, #amdgpu.address_space<fat_raw_buffer>>
 }
+
+// -----
+
+func.func @swizzle_invalid_type(%arg0 : si32) -> si32 {
+  // expected-error@+1 {{amdgpu.swizzle_bitmode' op operand #0 must be Integer or Float or vector of Integer or Float values of ranks 1}}
+  %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : si32
+  func.return %0 : si32
+}
diff --git a/mlir/test/Dialect/AMDGPU/ops.mlir b/mlir/test/Dialect/AMDGPU/ops.mlir
index 665674f2a7873..16b3193d270cb 100644
--- a/mlir/test/Dialect/AMDGPU/ops.mlir
+++ b/mlir/test/Dialect/AMDGPU/ops.mlir
@@ -157,3 +157,10 @@ func.func @wmma(%arg0 : vector<16xf16>, %arg1 : vector<8xf16>) -> vector<8xf16>
   %0 = amdgpu.wmma %arg0 * %arg0 + %arg1 : vector<16xf16>, vector<16xf16>, vector<8xf16>
   func.return %0 : vector<8xf16>
 }
+
+// CHECK-LABEL: func @swizzle_bitmode
+func.func @swizzle_bitmode(%arg0 : f32) -> f32 {
+  // CHECK: amdgpu.swizzle_bitmode
+  %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : f32
+  func.return %0 : f32
+}

From 2af5e01456954f145f9713c14394ef7f2fbcbebc Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 17 Apr 2025 15:27:44 -0700
Subject: [PATCH 332/710] [BOLT][RISCV] Fix MCPlusBuilder instrumentation
 ifaces (#136211)

a) Due to the different capabilities of the functions implemented,
rename the createCmpJE function
b) Refactor the convertIndirectCallToLoad function to override the
interface.

Patch by WangJee, originally posted in #136129
---
 bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index 0e27d29019e95..391c1866c810a 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -555,9 +555,9 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
                .addReg(RegCnt);
   }
 
-  InstructionListType createCmpJE(MCPhysReg RegNo, MCPhysReg RegTmp,
-                                  const MCSymbol *Target,
-                                  MCContext *Ctx) const {
+  InstructionListType createRegCmpJE(MCPhysReg RegNo, MCPhysReg RegTmp,
+                                     const MCSymbol *Target,
+                                     MCContext *Ctx) const {
     InstructionListType Insts;
     Insts.emplace_back(
         MCInstBuilder(RISCV::SUB).addReg(RegTmp).addReg(RegNo).addReg(RegNo));
@@ -718,7 +718,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     Insts.emplace_back();
     loadReg(Insts.back(), RISCV::X10, RISCV::X10, 0);
     InstructionListType cmpJmp =
-        createCmpJE(RISCV::X10, RISCV::X11, IndCallHandler, Ctx);
+        createRegCmpJE(RISCV::X10, RISCV::X11, IndCallHandler, Ctx);
     Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end());
     Insts.emplace_back();
     createStackPointerIncrement(Insts.back(), 16);
@@ -777,14 +777,13 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     return createGetter(Ctx, "__bolt_instr_num_funcs");
   }
 
-  void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg,
-                                 MCPhysReg ZeroReg) const {
+  void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
     bool IsTailCall = isTailCall(Inst);
     if (IsTailCall)
       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
     Inst.setOpcode(RISCV::ADD);
     Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
-    Inst.insert(Inst.begin() + 1, MCOperand::createReg(ZeroReg));
+    Inst.insert(Inst.begin() + 1, MCOperand::createReg(RISCV::X0));
     return;
   }
 
@@ -845,7 +844,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     InstructionListType Insts;
     spillRegs(Insts, {RISCV::X10, RISCV::X11});
     Insts.emplace_back(CallInst);
-    convertIndirectCallToLoad(Insts.back(), RISCV::X10, RISCV::X0);
+    convertIndirectCallToLoad(Insts.back(), RISCV::X10);
     InstructionListType LoadImm = createLoadImmediate(RISCV::X11, CallSiteID);
     Insts.insert(Insts.end(), LoadImm.begin(), LoadImm.end());
     spillRegs(Insts, {RISCV::X10, RISCV::X11});

From 655b9db7b90236342a4085fed1fc910621f835e0 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 17 Apr 2025 15:42:48 -0700
Subject: [PATCH 333/710] [flang] Generalized simplification of HLFIR reduction
 ops. (#136071)

This change generalizes SumAsElemental inlining in
SimplifyHLFIRIntrinsics pass so that it can be applied
to ALL, ANY, COUNT, MAXLOC, MAXVAL, MINLOC, MINVAL, SUM.

This change makes the special handling of the reduction
operations in OptimizedBufferization redundant: once HLFIR
operations are inlined, the hlfir.elemental inlining should
do the rest of the job.
---
 .../flang/Optimizer/Builder/HLFIRTools.h      |    5 +
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |   27 +
 .../Transforms/OptimizedBufferization.cpp     |  465 --------
 .../Transforms/SimplifyHLFIRIntrinsics.cpp    | 1039 +++++++++++++----
 flang/test/HLFIR/all-elemental.fir            |   91 --
 flang/test/HLFIR/any-elemental.fir            |  190 ---
 flang/test/HLFIR/count-elemental.fir          |  314 -----
 flang/test/HLFIR/maxloc-elemental.fir         |  133 ---
 flang/test/HLFIR/maxval-elemental.fir         |  117 --
 flang/test/HLFIR/minloc-elemental.fir         |  397 -------
 flang/test/HLFIR/minval-elemental.fir         |   95 --
 .../HLFIR/simplify-hlfir-intrinsics-all.fir   |  123 ++
 .../HLFIR/simplify-hlfir-intrinsics-any.fir   |  123 ++
 .../HLFIR/simplify-hlfir-intrinsics-count.fir |  127 ++
 .../simplify-hlfir-intrinsics-maxloc.fir      |  343 ++++++
 .../simplify-hlfir-intrinsics-maxval.fir      |  177 +++
 .../simplify-hlfir-intrinsics-minloc.fir      |  343 ++++++
 .../simplify-hlfir-intrinsics-minval.fir      |  177 +++
 18 files changed, 2287 insertions(+), 1999 deletions(-)
 delete mode 100644 flang/test/HLFIR/all-elemental.fir
 delete mode 100644 flang/test/HLFIR/any-elemental.fir
 delete mode 100644 flang/test/HLFIR/count-elemental.fir
 delete mode 100644 flang/test/HLFIR/maxloc-elemental.fir
 delete mode 100644 flang/test/HLFIR/maxval-elemental.fir
 delete mode 100644 flang/test/HLFIR/minloc-elemental.fir
 delete mode 100644 flang/test/HLFIR/minval-elemental.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir

diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index ac80873dc374f..cd259b9dc6071 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -301,6 +301,11 @@ mlir::Value genExtent(mlir::Location loc, fir::FirOpBuilder &builder,
 mlir::Value genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
                       hlfir::Entity entity, unsigned dim);
 
+/// Compute the lower bounds of \p entity, which is an array of known rank.
+llvm::SmallVector<mlir::Value> genLBounds(mlir::Location loc,
+                                          fir::FirOpBuilder &builder,
+                                          hlfir::Entity entity);
+
 /// Generate a vector of extents with index type from a fir.shape
 /// of fir.shape_shift value.
 llvm::SmallVector<mlir::Value> getIndexExtents(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 558ebcb876ddb..2a5e136c57c62 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -659,6 +659,33 @@ mlir::Value hlfir::genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
   return dimInfo.getLowerBound();
 }
 
+llvm::SmallVector<mlir::Value> hlfir::genLBounds(mlir::Location loc,
+                                                 fir::FirOpBuilder &builder,
+                                                 hlfir::Entity entity) {
+  assert(!entity.isAssumedRank() &&
+         "cannot compute all lower bounds for assumed rank");
+  assert(!entity.isScalar() && "expected an array entity");
+  int rank = entity.getRank();
+  mlir::Type idxTy = builder.getIndexType();
+  if (!entity.mayHaveNonDefaultLowerBounds())
+    return {static_cast<std::size_t>(rank),
+            builder.createIntegerConstant(loc, idxTy, 1)};
+
+  if (auto shape = tryRetrievingShapeOrShift(entity)) {
+    auto lbounds = getExplicitLboundsFromShape(shape);
+    if (!lbounds.empty())
+      return lbounds;
+  }
+
+  if (entity.isMutableBox())
+    entity = hlfir::derefPointersAndAllocatables(loc, builder, entity);
+
+  llvm::SmallVector<mlir::Value, Fortran::common::maxRank> lbounds;
+  fir::factory::genDimInfoFromBox(builder, loc, entity, &lbounds,
+                                  /*extents=*/nullptr, /*strides=*/nullptr);
+  return lbounds;
+}
+
 void hlfir::genLengthParameters(mlir::Location loc, fir::FirOpBuilder &builder,
                                 Entity entity,
                                 llvm::SmallVectorImpl<mlir::Value> &result) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index c489450384a35..79aabd2981e1a 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -772,458 +772,6 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   return mlir::success();
 }
 
-using GenBodyFn =
-    std::function<mlir::Value(fir::FirOpBuilder &, mlir::Location, mlir::Value,
-                              const llvm::SmallVectorImpl<mlir::Value> &)>;
-static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
-                                         mlir::Location loc, mlir::Value init,
-                                         mlir::Value shape, GenBodyFn genBody) {
-  auto extents = hlfir::getIndexExtents(loc, builder, shape);
-  mlir::Value reduction = init;
-  mlir::IndexType idxTy = builder.getIndexType();
-  mlir::Value oneIdx = builder.createIntegerConstant(loc, idxTy, 1);
-
-  // Create a reduction loop nest. We use one-based indices so that they can be
-  // passed to the elemental, and reverse the order so that they can be
-  // generated in column-major order for better performance.
-  llvm::SmallVector<mlir::Value> indices(extents.size(), mlir::Value{});
-  for (unsigned i = 0; i < extents.size(); ++i) {
-    auto loop = builder.create<fir::DoLoopOp>(
-        loc, oneIdx, extents[extents.size() - i - 1], oneIdx, false,
-        /*finalCountValue=*/false, reduction);
-    reduction = loop.getRegionIterArgs()[0];
-    indices[extents.size() - i - 1] = loop.getInductionVar();
-    // Set insertion point to the loop body so that the next loop
-    // is inserted inside the current one.
-    builder.setInsertionPointToStart(loop.getBody());
-  }
-
-  // Generate the body
-  reduction = genBody(builder, loc, reduction, indices);
-
-  // Unwind the loop nest.
-  for (unsigned i = 0; i < extents.size(); ++i) {
-    auto result = builder.create<fir::ResultOp>(loc, reduction);
-    auto loop = mlir::cast<fir::DoLoopOp>(result->getParentOp());
-    reduction = loop.getResult(0);
-    // Set insertion point after the loop operation that we have
-    // just processed.
-    builder.setInsertionPointAfter(loop.getOperation());
-  }
-
-  return reduction;
-}
-
-auto makeMinMaxInitValGenerator(bool isMax) {
-  return [isMax](fir::FirOpBuilder builder, mlir::Location loc,
-                 mlir::Type elementType) -> mlir::Value {
-    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
-      const llvm::fltSemantics &sem = ty.getFloatSemantics();
-      llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
-      return builder.createRealConstant(loc, elementType, limit);
-    }
-    unsigned bits = elementType.getIntOrFloatBitWidth();
-    int64_t limitInt =
-        isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
-              : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
-    return builder.createIntegerConstant(loc, elementType, limitInt);
-  };
-}
-
-mlir::Value generateMinMaxComparison(fir::FirOpBuilder builder,
-                                     mlir::Location loc, mlir::Value elem,
-                                     mlir::Value reduction, bool isMax) {
-  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
-    // For FP reductions we want the first smallest value to be used, that
-    // is not NaN. A OGL/OLT condition will usually work for this unless all
-    // the values are Nan or Inf. This follows the same logic as
-    // NumericCompare for Minloc/Maxlox in extrema.cpp.
-    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
-        loc,
-        isMax ? mlir::arith::CmpFPredicate::OGT
-              : mlir::arith::CmpFPredicate::OLT,
-        elem, reduction);
-    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
-  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
-    return builder.create<mlir::arith::CmpIOp>(
-        loc,
-        isMax ? mlir::arith::CmpIPredicate::sgt
-              : mlir::arith::CmpIPredicate::slt,
-        elem, reduction);
-  }
-  llvm_unreachable("unsupported type");
-}
-
-/// Given a reduction operation with an elemental/designate source, attempt to
-/// generate a do-loop to perform the operation inline.
-///   %e = hlfir.elemental %shape unordered
-///   %r = hlfir.count %e
-/// =>
-///   %r = for.do_loop %arg = 1 to bound(%shape) step 1 iter_args(%arg2 = init)
-///     %i = <inline elemental>
-///     %c = <reduce count> %i
-///     fir.result %c
-template <typename Op>
-class ReductionConversion : public mlir::OpRewritePattern<Op> {
-public:
-  using mlir::OpRewritePattern<Op>::OpRewritePattern;
-
-  llvm::LogicalResult
-  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
-    mlir::Location loc = op.getLoc();
-    // Select source and validate its arguments.
-    mlir::Value source;
-    bool valid = false;
-    if constexpr (std::is_same_v<Op, hlfir::AnyOp> ||
-                  std::is_same_v<Op, hlfir::AllOp> ||
-                  std::is_same_v<Op, hlfir::CountOp>) {
-      source = op.getMask();
-      valid = !op.getDim();
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
-                         std::is_same_v<Op, hlfir::MinvalOp>) {
-      source = op.getArray();
-      valid = !op.getDim() && !op.getMask();
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
-                         std::is_same_v<Op, hlfir::MinlocOp>) {
-      source = op.getArray();
-      valid = !op.getDim() && !op.getMask() && !op.getBack();
-    }
-    if (!valid)
-      return rewriter.notifyMatchFailure(
-          op, "Currently does not accept optional arguments");
-
-    hlfir::ElementalOp elemental;
-    hlfir::DesignateOp designate;
-    mlir::Value shape;
-    if ((elemental = source.template getDefiningOp<hlfir::ElementalOp>())) {
-      shape = elemental.getOperand(0);
-    } else if ((designate =
-                    source.template getDefiningOp<hlfir::DesignateOp>())) {
-      shape = designate.getShape();
-    } else {
-      return rewriter.notifyMatchFailure(op, "Did not find valid argument");
-    }
-
-    auto inlineSource =
-        [elemental,
-         &designate](fir::FirOpBuilder builder, mlir::Location loc,
-                     const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-        -> mlir::Value {
-      if (elemental) {
-        // Inline the elemental and get the value from it.
-        auto yield =
-            inlineElementalOp(loc, builder, elemental, oneBasedIndices);
-        auto tmp = yield.getElementValue();
-        yield->erase();
-        return tmp;
-      }
-      if (designate) {
-        // Create a designator over the array designator, then load the
-        // reference.
-        mlir::Value elementAddr = hlfir::getElementAt(
-            loc, builder, hlfir::Entity{designate.getResult()},
-            oneBasedIndices);
-        return builder.create<fir::LoadOp>(loc, elementAddr);
-      }
-      llvm_unreachable("unsupported type");
-    };
-
-    fir::FirOpBuilder builder{rewriter, op.getOperation()};
-
-    mlir::Value init;
-    GenBodyFn genBodyFn;
-    if constexpr (std::is_same_v<Op, hlfir::AnyOp>) {
-      init = builder.createIntegerConstant(loc, builder.getI1Type(), 0);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally set the reduction variable.
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        return builder.create<mlir::arith::OrIOp>(loc, reduction, cond);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::AllOp>) {
-      init = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally set the reduction variable.
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        return builder.create<mlir::arith::AndIOp>(loc, reduction, cond);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
-      init = builder.createIntegerConstant(loc, op.getType(), 0);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally add one to the current value
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        mlir::Value one =
-            builder.createIntegerConstant(loc, reduction.getType(), 1);
-        mlir::Value add1 =
-            builder.create<mlir::arith::AddIOp>(loc, reduction, one);
-        return builder.create<mlir::arith::SelectOp>(loc, cond, add1,
-                                                     reduction);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
-                         std::is_same_v<Op, hlfir::MinlocOp>) {
-      // TODO: implement minloc/maxloc conversion.
-      return rewriter.notifyMatchFailure(
-          op, "Currently minloc/maxloc is not handled");
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
-                         std::is_same_v<Op, hlfir::MinvalOp>) {
-      mlir::Type ty = op.getType();
-      if (!(mlir::isa<mlir::FloatType>(ty) ||
-            mlir::isa<mlir::IntegerType>(ty))) {
-        return rewriter.notifyMatchFailure(
-            op, "Type is not supported for Maxval or Minval yet");
-      }
-
-      bool isMax = std::is_same_v<Op, hlfir::MaxvalOp>;
-      init = makeMinMaxInitValGenerator(isMax)(builder, loc, ty);
-      genBodyFn = [inlineSource, isMax](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        mlir::Value val = inlineSource(builder, loc, oneBasedIndices);
-        mlir::Value cmp =
-            generateMinMaxComparison(builder, loc, val, reduction, isMax);
-        return builder.create<mlir::arith::SelectOp>(loc, cmp, val, reduction);
-      };
-    } else {
-      llvm_unreachable("unsupported type");
-    }
-
-    mlir::Value res =
-        generateReductionLoop(builder, loc, init, shape, genBodyFn);
-    if (res.getType() != op.getType())
-      res = builder.create<fir::ConvertOp>(loc, op.getType(), res);
-
-    // Check if the op was the only user of the source (apart from a destroy),
-    // and remove it if so.
-    mlir::Operation *sourceOp = source.getDefiningOp();
-    mlir::Operation::user_range srcUsers = sourceOp->getUsers();
-    hlfir::DestroyOp srcDestroy;
-    if (std::distance(srcUsers.begin(), srcUsers.end()) == 2) {
-      srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*srcUsers.begin());
-      if (!srcDestroy)
-        srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++srcUsers.begin());
-    }
-
-    rewriter.replaceOp(op, res);
-    if (srcDestroy) {
-      rewriter.eraseOp(srcDestroy);
-      rewriter.eraseOp(sourceOp);
-    }
-    return mlir::success();
-  }
-};
-
-// Look for minloc(mask=elemental) and generate the minloc loop with
-// inlined elemental.
-//  %e = hlfir.elemental %shape ({ ... })
-//  %m = hlfir.minloc %array mask %e
-template <typename Op>
-class ReductionMaskConversion : public mlir::OpRewritePattern<Op> {
-public:
-  using mlir::OpRewritePattern<Op>::OpRewritePattern;
-
-  llvm::LogicalResult
-  matchAndRewrite(Op mloc, mlir::PatternRewriter &rewriter) const override {
-    if (!mloc.getMask() || mloc.getDim() || mloc.getBack())
-      return rewriter.notifyMatchFailure(mloc,
-                                         "Did not find valid minloc/maxloc");
-
-    bool isMax = std::is_same_v<Op, hlfir::MaxlocOp>;
-
-    auto elemental =
-        mloc.getMask().template getDefiningOp<hlfir::ElementalOp>();
-    if (!elemental || hlfir::elementalOpMustProduceTemp(elemental))
-      return rewriter.notifyMatchFailure(mloc, "Did not find elemental");
-
-    mlir::Value array = mloc.getArray();
-
-    unsigned rank = mlir::cast<hlfir::ExprType>(mloc.getType()).getShape()[0];
-    mlir::Type arrayType = array.getType();
-    if (!mlir::isa<fir::BoxType>(arrayType))
-      return rewriter.notifyMatchFailure(
-          mloc, "Currently requires a boxed type input");
-    mlir::Type elementType = hlfir::getFortranElementType(arrayType);
-    if (!fir::isa_trivial(elementType))
-      return rewriter.notifyMatchFailure(
-          mloc, "Character arrays are currently not handled");
-
-    mlir::Location loc = mloc.getLoc();
-    fir::FirOpBuilder builder{rewriter, mloc.getOperation()};
-    mlir::Value resultArr = builder.createTemporary(
-        loc, fir::SequenceType::get(
-                 rank, hlfir::getFortranElementType(mloc.getType())));
-
-    auto init = makeMinMaxInitValGenerator(isMax);
-
-    auto genBodyOp =
-        [&rank, &resultArr, &elemental, isMax](
-            fir::FirOpBuilder builder, mlir::Location loc,
-            mlir::Type elementType, mlir::Value array, mlir::Value flagRef,
-            mlir::Value reduction,
-            const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {
-      // We are in the innermost loop: generate the elemental inline
-      mlir::Value oneIdx =
-          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-      llvm::SmallVector<mlir::Value> oneBasedIndices;
-      llvm::transform(
-          indices, std::back_inserter(oneBasedIndices), [&](mlir::Value V) {
-            return builder.create<mlir::arith::AddIOp>(loc, V, oneIdx);
-          });
-      hlfir::YieldElementOp yield =
-          hlfir::inlineElementalOp(loc, builder, elemental, oneBasedIndices);
-      mlir::Value maskElem = yield.getElementValue();
-      yield->erase();
-
-      mlir::Type ifCompatType = builder.getI1Type();
-      mlir::Value ifCompatElem =
-          builder.create<fir::ConvertOp>(loc, ifCompatType, maskElem);
-
-      llvm::SmallVector<mlir::Type> resultsTy = {elementType, elementType};
-      fir::IfOp maskIfOp =
-          builder.create<fir::IfOp>(loc, elementType, ifCompatElem,
-                                    /*withElseRegion=*/true);
-      builder.setInsertionPointToStart(&maskIfOp.getThenRegion().front());
-
-      // Set flag that mask was true at some point
-      mlir::Value flagSet = builder.createIntegerConstant(
-          loc, mlir::cast<fir::ReferenceType>(flagRef.getType()).getEleTy(), 1);
-      mlir::Value isFirst = builder.create<fir::LoadOp>(loc, flagRef);
-      mlir::Value addr = hlfir::getElementAt(loc, builder, hlfir::Entity{array},
-                                             oneBasedIndices);
-      mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
-
-      // Compare with the max reduction value
-      mlir::Value cmp =
-          generateMinMaxComparison(builder, loc, elem, reduction, isMax);
-
-      // The condition used for the loop is isFirst || <the condition above>.
-      isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
-      isFirst = builder.create<mlir::arith::XOrIOp>(
-          loc, isFirst, builder.createIntegerConstant(loc, cmp.getType(), 1));
-      cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, isFirst);
-
-      // Set the new coordinate to the result
-      fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, cmp,
-                                                 /*withElseRegion*/ true);
-
-      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      builder.create<fir::StoreOp>(loc, flagSet, flagRef);
-      mlir::Type resultElemTy =
-          hlfir::getFortranElementType(resultArr.getType());
-      mlir::Type returnRefTy = builder.getRefType(resultElemTy);
-      mlir::IndexType idxTy = builder.getIndexType();
-
-      for (unsigned int i = 0; i < rank; ++i) {
-        mlir::Value index = builder.createIntegerConstant(loc, idxTy, i + 1);
-        mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
-            loc, returnRefTy, resultArr, index);
-        mlir::Value fortranIndex = builder.create<fir::ConvertOp>(
-            loc, resultElemTy, oneBasedIndices[i]);
-        builder.create<fir::StoreOp>(loc, fortranIndex, resultElemAddr);
-      }
-      builder.create<fir::ResultOp>(loc, elem);
-      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, reduction);
-      builder.setInsertionPointAfter(ifOp);
-
-      // Close the mask if
-      builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
-      builder.setInsertionPointToStart(&maskIfOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, reduction);
-      builder.setInsertionPointAfter(maskIfOp);
-
-      return maskIfOp.getResult(0);
-    };
-    auto getAddrFn = [](fir::FirOpBuilder builder, mlir::Location loc,
-                        const mlir::Type &resultElemType, mlir::Value resultArr,
-                        mlir::Value index) {
-      mlir::Type resultRefTy = builder.getRefType(resultElemType);
-      mlir::Value oneIdx =
-          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-      index = builder.create<mlir::arith::AddIOp>(loc, index, oneIdx);
-      return builder.create<hlfir::DesignateOp>(loc, resultRefTy, resultArr,
-                                                index);
-    };
-
-    // Initialize the result
-    mlir::Type resultElemTy = hlfir::getFortranElementType(resultArr.getType());
-    mlir::Type resultRefTy = builder.getRefType(resultElemTy);
-    mlir::Value returnValue =
-        builder.createIntegerConstant(loc, resultElemTy, 0);
-    for (unsigned int i = 0; i < rank; ++i) {
-      mlir::Value index =
-          builder.createIntegerConstant(loc, builder.getIndexType(), i + 1);
-      mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
-          loc, resultRefTy, resultArr, index);
-      builder.create<fir::StoreOp>(loc, returnValue, resultElemAddr);
-    }
-
-    fir::genMinMaxlocReductionLoop(builder, array, init, genBodyOp, getAddrFn,
-                                   rank, elementType, loc, builder.getI1Type(),
-                                   resultArr, false);
-
-    mlir::Value asExpr = builder.create<hlfir::AsExprOp>(
-        loc, resultArr, builder.createBool(loc, false));
-
-    // Check all the users - the destroy is no longer required, and any assign
-    // can use resultArr directly so that InlineHLFIRAssign pass
-    // can optimize the results. Other operations are replaced with an AsExpr
-    // for the temporary resultArr.
-    llvm::SmallVector<hlfir::DestroyOp> destroys;
-    llvm::SmallVector<hlfir::AssignOp> assigns;
-    for (auto user : mloc->getUsers()) {
-      if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(user))
-        destroys.push_back(destroy);
-      else if (auto assign = mlir::dyn_cast<hlfir::AssignOp>(user))
-        assigns.push_back(assign);
-    }
-
-    // Check if the minloc/maxloc was the only user of the elemental (apart from
-    // a destroy), and remove it if so.
-    mlir::Operation::user_range elemUsers = elemental->getUsers();
-    hlfir::DestroyOp elemDestroy;
-    if (std::distance(elemUsers.begin(), elemUsers.end()) == 2) {
-      elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*elemUsers.begin());
-      if (!elemDestroy)
-        elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++elemUsers.begin());
-    }
-
-    for (auto d : destroys)
-      rewriter.eraseOp(d);
-    for (auto a : assigns)
-      a.setOperand(0, resultArr);
-    rewriter.replaceOp(mloc, asExpr);
-    if (elemDestroy) {
-      rewriter.eraseOp(elemDestroy);
-      rewriter.eraseOp(elemental);
-    }
-    return mlir::success();
-  }
-};
-
 class EvaluateIntoMemoryAssignBufferization
     : public mlir::OpRewritePattern<hlfir::EvaluateInMemoryOp> {
 
@@ -1340,19 +888,6 @@ class OptimizedBufferizationPass
     patterns.insert<ElementalAssignBufferization>(context);
     patterns.insert<BroadcastAssignBufferization>(context);
     patterns.insert<EvaluateIntoMemoryAssignBufferization>(context);
-    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
-    // TODO: implement basic minloc/maxloc conversion.
-    // patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
-    // patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
-    patterns.insert<ReductionMaskConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<ReductionMaskConversion<hlfir::MaxlocOp>>(context);
-    // TODO: implement masked minval/maxval conversion.
-    // patterns.insert<ReductionMaskConversion<hlfir::MaxvalOp>>(context);
-    // patterns.insert<ReductionMaskConversion<hlfir::MinvalOp>>(context);
 
     if (mlir::failed(mlir::applyPatternsGreedily(
             getOperation(), std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index bac10121a881b..d660cf3845f9a 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -173,245 +173,882 @@ class TransposeAsElementalConversion
   }
 };
 
-// Expand the SUM(DIM=CONSTANT) operation into .
-class SumAsElementalConversion : public mlir::OpRewritePattern<hlfir::SumOp> {
+/// Base class for converting reduction-like operations into
+/// a reduction loop[-nest] optionally wrapped into hlfir.elemental.
+/// It is used to handle operations produced for ALL, ANY, COUNT,
+/// MAXLOC, MAXVAL, MINLOC, MINVAL, SUM intrinsics.
+///
+/// All of these operations take an input array, and optional
+/// dim, mask arguments. ALL, ANY, COUNT do not have mask argument.
+class ReductionAsElementalConverter {
 public:
-  using mlir::OpRewritePattern<hlfir::SumOp>::OpRewritePattern;
+  ReductionAsElementalConverter(mlir::Operation *op,
+                                mlir::PatternRewriter &rewriter)
+      : op{op}, rewriter{rewriter}, loc{op->getLoc()}, builder{rewriter, op} {
+    assert(op->getNumResults() == 1);
+  }
+  virtual ~ReductionAsElementalConverter() {}
 
-  llvm::LogicalResult
-  matchAndRewrite(hlfir::SumOp sum,
-                  mlir::PatternRewriter &rewriter) const override {
-    hlfir::Entity array = hlfir::Entity{sum.getArray()};
-    bool isTotalReduction = hlfir::Entity{sum}.getRank() == 0;
-    mlir::Value dim = sum.getDim();
+  /// Do the actual conversion or return mlir::failure(),
+  /// if conversion is not possible.
+  mlir::LogicalResult convert();
+
+private:
+  // Return fir.shape specifying the shape of the result
+  // of a reduction with DIM=dimVal. The second return value
+  // is the extent of the DIM dimension.
+  std::tuple<mlir::Value, mlir::Value>
+  genResultShapeForPartialReduction(hlfir::Entity array, int64_t dimVal);
+
+  /// \p mask is a scalar or array logical mask.
+  /// If \p isPresentPred is not nullptr, it is a dynamic predicate value
+  /// identifying whether the mask's variable is present.
+  /// \p indices is a range of one-based indices to access \p mask
+  /// when it is an array.
+  ///
+  /// The method returns the scalar mask value to guard the access
+  /// to a single element of the input array.
+  mlir::Value genMaskValue(mlir::Value mask, mlir::Value isPresentPred,
+                           mlir::ValueRange indices);
+
+protected:
+  /// Return the input array.
+  virtual mlir::Value getSource() const = 0;
+
+  /// Return DIM or nullptr, if it is not present.
+  virtual mlir::Value getDim() const = 0;
+
+  /// Return MASK or nullptr, if it is not present.
+  virtual mlir::Value getMask() const { return nullptr; }
+
+  /// Return FastMathFlags attached to the operation
+  /// or arith::FastMathFlags::none, if the operation
+  /// does not support FastMathFlags (e.g. ALL, ANY, COUNT).
+  virtual mlir::arith::FastMathFlags getFastMath() const {
+    return mlir::arith::FastMathFlags::none;
+  }
+
+  /// Generates initial values for the reduction values used
+  /// by the reduction loop. In general, there is a single
+  /// loop-carried reduction value (e.g. for SUM), but, for example,
+  /// MAXLOC/MINLOC implementation uses multiple reductions.
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() = 0;
+
+  /// Perform reduction(s) update given a single input array's element
+  /// identified by \p array and \p oneBasedIndices coordinates.
+  /// \p currentValue specifies the current value(s) of the reduction(s)
+  /// inside the reduction loop body.
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) = 0;
+
+  /// Given reduction value(s) in \p reductionResults produced
+  /// by the reduction loop, apply any required updates and return
+  /// new reduction value(s) to be used after the reduction loop
+  /// (e.g. as the result yield of the wrapping hlfir.elemental).
+  /// NOTE: if the reduction loop is wrapped in hlfir.elemental,
+  /// the insertion point of any generated code is inside hlfir.elemental.
+  virtual hlfir::Entity
+  genFinalResult(const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
+    assert(reductionResults.size() == 1 &&
+           "default implementation of genFinalResult expect a single reduction "
+           "value");
+    return hlfir::Entity{reductionResults[0]};
+  }
+
+  /// Return mlir::success(), if the operation can be converted.
+  /// The default implementation always returns mlir::success().
+  /// The derived type may override the default implementation
+  /// with its own definition.
+  virtual mlir::LogicalResult isConvertible() const { return mlir::success(); }
+
+  // Default implementation of isTotalReduction() just checks
+  // if the result of the operation is a scalar.
+  // True result indicates that the reduction has to be done
+  // across all elements, false result indicates that
+  // the result is an array expression produced by an hlfir.elemental
+  // operation with a single reduction loop across the DIM dimension.
+  //
+  // MAXLOC/MINLOC must override this.
+  virtual bool isTotalReduction() const { return getResultRank() == 0; }
+
+  // Return true, if the reduction loop[-nest] may be unordered.
+  // In general, FP reductions may only be unordered when
+  // FastMathFlags::reassoc transformations are allowed.
+  //
+  // Some dervied types may need to override this.
+  virtual bool isUnordered() const {
+    mlir::Type elemType = getSourceElementType();
+    if (mlir::isa<mlir::IntegerType, fir::LogicalType, fir::CharacterType>(
+            elemType))
+      return true;
+    return static_cast<bool>(getFastMath() &
+                             mlir::arith::FastMathFlags::reassoc);
+  }
+
+  /// Return 0, if DIM is not present or its values does not matter
+  /// (for example, a reduction of 1D array does not care about
+  /// the DIM value, assuming that it is a valid program).
+  /// Return mlir::failure(), if DIM is a constant known
+  /// to be invalid for the given array.
+  /// Otherwise, return DIM constant value.
+  mlir::FailureOr<int64_t> getConstDim() const {
     int64_t dimVal = 0;
-    if (!isTotalReduction) {
+    if (!isTotalReduction()) {
       // In case of partial reduction we should ignore the operations
       // with invalid DIM values. They may appear in dead code
       // after constant propagation.
-      auto constDim = fir::getIntIfConstant(dim);
+      auto constDim = fir::getIntIfConstant(getDim());
       if (!constDim)
-        return rewriter.notifyMatchFailure(sum, "Nonconstant DIM for SUM");
+        return rewriter.notifyMatchFailure(op, "Nonconstant DIM");
       dimVal = *constDim;
 
-      if ((dimVal <= 0 || dimVal > array.getRank()))
-        return rewriter.notifyMatchFailure(
-            sum, "Invalid DIM for partial SUM reduction");
+      if ((dimVal <= 0 || dimVal > getSourceRank()))
+        return rewriter.notifyMatchFailure(op,
+                                           "Invalid DIM for partial reduction");
     }
+    return dimVal;
+  }
 
-    mlir::Location loc = sum.getLoc();
-    fir::FirOpBuilder builder{rewriter, sum.getOperation()};
-    mlir::Type elementType = hlfir::getFortranElementType(sum.getType());
-    mlir::Value mask = sum.getMask();
+  /// Return hlfir::Entity of the result.
+  hlfir::Entity getResultEntity() const {
+    return hlfir::Entity{op->getResult(0)};
+  }
 
-    mlir::Value resultShape, dimExtent;
-    llvm::SmallVector<mlir::Value> arrayExtents;
-    if (isTotalReduction)
-      arrayExtents = hlfir::genExtentsVector(loc, builder, array);
-    else
-      std::tie(resultShape, dimExtent) =
-          genResultShapeForPartialReduction(loc, builder, array, dimVal);
-
-    // If the mask is present and is a scalar, then we'd better load its value
-    // outside of the reduction loop making the loop unswitching easier.
-    mlir::Value isPresentPred, maskValue;
-    if (mask) {
-      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-        // MASK represented by a box might be dynamically optional,
-        // so we have to check for its presence before accessing it.
-        isPresentPred =
-            builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
-      }
+  /// Return type of the result (e.g. !hlfir.expr<?xi32>).
+  mlir::Type getResultType() const { return getResultEntity().getType(); }
 
-      if (hlfir::Entity{mask}.isScalar())
-        maskValue = genMaskValue(loc, builder, mask, isPresentPred, {});
-    }
+  /// Return the element type of the result (e.g. i32).
+  mlir::Type getResultElementType() const {
+    return hlfir::getFortranElementType(getResultType());
+  }
 
-    auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                         mlir::ValueRange inputIndices) -> hlfir::Entity {
-      // Loop over all indices in the DIM dimension, and reduce all values.
-      // If DIM is not present, do total reduction.
-
-      // Initial value for the reduction.
-      mlir::Value reductionInitValue =
-          fir::factory::createZeroValue(builder, loc, elementType);
-
-      // The reduction loop may be unordered if FastMathFlags::reassoc
-      // transformations are allowed. The integer reduction is always
-      // unordered.
-      bool isUnordered = mlir::isa<mlir::IntegerType>(elementType) ||
-                         static_cast<bool>(sum.getFastmath() &
-                                           mlir::arith::FastMathFlags::reassoc);
+  /// Return rank of the result.
+  unsigned getResultRank() const { return getResultEntity().getRank(); }
 
-      llvm::SmallVector<mlir::Value> extents;
-      if (isTotalReduction)
-        extents = arrayExtents;
-      else
-        extents.push_back(
-            builder.createConvert(loc, builder.getIndexType(), dimExtent));
+  /// Return the element type of the source.
+  mlir::Type getSourceElementType() const {
+    return hlfir::getFortranElementType(getSource().getType());
+  }
 
-      auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                         mlir::ValueRange oneBasedIndices,
-                         mlir::ValueRange reductionArgs)
-          -> llvm::SmallVector<mlir::Value, 1> {
-        // Generate the reduction loop-nest body.
-        // The initial reduction value in the innermost loop
-        // is passed via reductionArgs[0].
-        llvm::SmallVector<mlir::Value> indices;
-        if (isTotalReduction) {
-          indices = oneBasedIndices;
-        } else {
-          indices = inputIndices;
-          indices.insert(indices.begin() + dimVal - 1, oneBasedIndices[0]);
-        }
+  /// Return rank of the input array.
+  unsigned getSourceRank() const {
+    return hlfir::Entity{getSource()}.getRank();
+  }
 
-        mlir::Value reductionValue = reductionArgs[0];
-        fir::IfOp ifOp;
-        if (mask) {
-          // Make the reduction value update conditional on the value
-          // of the mask.
-          if (!maskValue) {
-            // If the mask is an array, use the elemental and the loop indices
-            // to address the proper mask element.
-            maskValue =
-                genMaskValue(loc, builder, mask, isPresentPred, indices);
-          }
-          mlir::Value isUnmasked = builder.create<fir::ConvertOp>(
-              loc, builder.getI1Type(), maskValue);
-          ifOp = builder.create<fir::IfOp>(loc, elementType, isUnmasked,
-                                           /*withElseRegion=*/true);
-          // In the 'else' block return the current reduction value.
-          builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-          builder.create<fir::ResultOp>(loc, reductionValue);
-
-          // In the 'then' block do the actual addition.
-          builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-        }
+  /// The reduction operation.
+  mlir::Operation *op;
 
-        hlfir::Entity element =
-            hlfir::getElementAt(loc, builder, array, indices);
-        hlfir::Entity elementValue =
-            hlfir::loadTrivialScalar(loc, builder, element);
-        // NOTE: we can use "Kahan summation" same way as the runtime
-        // (e.g. when fast-math is not allowed), but let's start with
-        // the simple version.
-        reductionValue =
-            genScalarAdd(loc, builder, reductionValue, elementValue);
-
-        if (ifOp) {
-          builder.create<fir::ResultOp>(loc, reductionValue);
-          builder.setInsertionPointAfter(ifOp);
-          reductionValue = ifOp.getResult(0);
-        }
+  mlir::PatternRewriter &rewriter;
+  mlir::Location loc;
+  fir::FirOpBuilder builder;
+};
 
-        return {reductionValue};
-      };
+/// Generate initialization value for MIN or MAX reduction
+/// of the given \p type.
+template <bool IS_MAX>
+static mlir::Value genMinMaxInitValue(mlir::Location loc,
+                                      fir::FirOpBuilder &builder,
+                                      mlir::Type type) {
+  if (auto ty = mlir::dyn_cast<mlir::FloatType>(type)) {
+    const llvm::fltSemantics &sem = ty.getFloatSemantics();
+    // We must not use +/-INF here. If the reduction input is empty,
+    // the result of reduction must be +/-LARGEST.
+    llvm::APFloat limit = llvm::APFloat::getLargest(sem, /*Negative=*/IS_MAX);
+    return builder.createRealConstant(loc, type, limit);
+  }
+  unsigned bits = type.getIntOrFloatBitWidth();
+  int64_t limitInt = IS_MAX
+                         ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
+                         : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
+  return builder.createIntegerConstant(loc, type, limitInt);
+}
+
+/// Generate a comparison of an array element value \p elem
+/// and the current reduction value \p reduction for MIN/MAX reduction.
+template <bool IS_MAX>
+static mlir::Value
+genMinMaxComparison(mlir::Location loc, fir::FirOpBuilder &builder,
+                    mlir::Value elem, mlir::Value reduction) {
+  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
+    // For FP reductions we want the first smallest value to be used, that
+    // is not NaN. A OGL/OLT condition will usually work for this unless all
+    // the values are Nan or Inf. This follows the same logic as
+    // NumericCompare for Minloc/Maxloc in extrema.cpp.
+    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
+        loc,
+        IS_MAX ? mlir::arith::CmpFPredicate::OGT
+               : mlir::arith::CmpFPredicate::OLT,
+        elem, reduction);
+    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
+    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
+    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
+    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
+    return builder.create<mlir::arith::CmpIOp>(
+        loc,
+        IS_MAX ? mlir::arith::CmpIPredicate::sgt
+               : mlir::arith::CmpIPredicate::slt,
+        elem, reduction);
+  }
+  llvm_unreachable("unsupported type");
+}
+
+/// Implementation of ReductionAsElementalConverter interface
+/// for MAXLOC/MINLOC.
+template <typename T>
+class MinMaxlocAsElementalConverter : public ReductionAsElementalConverter {
+  static_assert(std::is_same_v<T, hlfir::MaxlocOp> ||
+                std::is_same_v<T, hlfir::MinlocOp>);
+  static constexpr unsigned maxRank = Fortran::common::maxRank;
+  // We have the following reduction values in the reduction loop:
+  //   * N integer coordinates, where N is:
+  //     - RANK(ARRAY) for total reductions.
+  //     - 1 for partial reductions.
+  //   * 1 reduction value holding the current MIN/MAX.
+  //   * 1 boolean indicating whether it is the first time
+  //     the mask is true.
+  static constexpr unsigned maxNumReductions = Fortran::common::maxRank + 2;
+  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxlocOp>;
+  using Base = ReductionAsElementalConverter;
 
-      llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
-          hlfir::genLoopNestWithReductions(loc, builder, extents,
-                                           {reductionInitValue}, genBody,
-                                           isUnordered);
-      return hlfir::Entity{reductionFinalValues[0]};
-    };
+public:
+  MinMaxlocAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
 
-    if (isTotalReduction) {
-      hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
-      rewriter.replaceOp(sum, result);
-      return mlir::success();
+private:
+  virtual mlir::Value getSource() const final { return getOp().getArray(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+  virtual mlir::Value getMask() const final { return getOp().getMask(); }
+  virtual mlir::arith::FastMathFlags getFastMath() const final {
+    return getOp().getFastmath();
+  }
+
+  virtual mlir::LogicalResult isConvertible() const final {
+    if (getOp().getBack())
+      return rewriter.notifyMatchFailure(
+          getOp(), "BACK is not supported for MINLOC/MAXLOC inlining");
+    if (mlir::isa<fir::CharacterType>(getSourceElementType()))
+      return rewriter.notifyMatchFailure(
+          getOp(),
+          "CHARACTER type is not supported for MINLOC/MAXLOC inlining");
+    return mlir::success();
+  }
+
+  // If the result is scalar, then DIM does not matter,
+  // and this is a total reduction.
+  // If DIM is not present, this is a total reduction.
+  virtual bool isTotalReduction() const final {
+    return getResultRank() == 0 || !getDim();
+  }
+
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final;
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) final;
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) final;
+
+private:
+  T getOp() const { return mlir::cast<T>(op); }
+
+  unsigned getNumCoors() const {
+    return isTotalReduction() ? getSourceRank() : 1;
+  }
+
+  void
+  checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    assert(reductions.size() == getNumCoors() + 2 &&
+           "invalid number of reductions for MINLOC/MAXLOC");
+  }
+
+  mlir::Value
+  getCurrentMinMax(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    checkReductions(reductions);
+    return reductions[getNumCoors()];
+  }
+
+  mlir::Value
+  getIsFirst(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    checkReductions(reductions);
+    return reductions[getNumCoors() + 1];
+  }
+};
+
+template <typename T>
+llvm::SmallVector<mlir::Value>
+MinMaxlocAsElementalConverter<T>::genReductionInitValues() {
+  // Initial value for the coordinate(s) is zero.
+  mlir::Value zeroCoor =
+      fir::factory::createZeroValue(builder, loc, getResultElementType());
+  llvm::SmallVector<mlir::Value, maxNumReductions> result(getNumCoors(),
+                                                          zeroCoor);
+
+  // Initial value for the MIN/MAX value.
+  mlir::Value minMaxInit =
+      genMinMaxInitValue<isMax>(loc, builder, getSourceElementType());
+  result.push_back(minMaxInit);
+
+  // Initial value for isFirst predicate. It is switched to false,
+  // when the reduction update dynamically happens inside the reduction
+  // loop.
+  mlir::Value trueVal = builder.createBool(loc, true);
+  result.push_back(trueVal);
+
+  return result;
+}
+
+template <typename T>
+llvm::SmallVector<mlir::Value>
+MinMaxlocAsElementalConverter<T>::reduceOneElement(
+    const llvm::SmallVectorImpl<mlir::Value> &currentValue, hlfir::Entity array,
+    mlir::ValueRange oneBasedIndices) {
+  checkReductions(currentValue);
+  hlfir::Entity elementValue =
+      hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+  mlir::Value cmp = genMinMaxComparison<isMax>(loc, builder, elementValue,
+                                               getCurrentMinMax(currentValue));
+  // If isFirst is true, then do the reduction update regardless
+  // of the FP comparison.
+  cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, getIsFirst(currentValue));
+
+  llvm::SmallVector<mlir::Value, maxNumReductions> newIndices;
+  int64_t dim = 1;
+  if (!isTotalReduction()) {
+    auto dimVal = getConstDim();
+    assert(mlir::succeeded(dimVal) &&
+           "partial MINLOC/MAXLOC reduction with invalid DIM");
+    dim = *dimVal;
+    assert(getNumCoors() == 1 &&
+           "partial MAXLOC/MINLOC reduction must compute one coordinate");
+  }
+
+  for (unsigned coorIdx = 0; coorIdx < getNumCoors(); ++coorIdx) {
+    mlir::Value currentCoor = currentValue[coorIdx];
+    mlir::Value newCoor = builder.createConvert(
+        loc, currentCoor.getType(), oneBasedIndices[coorIdx + dim - 1]);
+    mlir::Value update =
+        builder.create<mlir::arith::SelectOp>(loc, cmp, newCoor, currentCoor);
+    newIndices.push_back(update);
+  }
+
+  mlir::Value newMinMax = builder.create<mlir::arith::SelectOp>(
+      loc, cmp, elementValue, getCurrentMinMax(currentValue));
+  newIndices.push_back(newMinMax);
+
+  mlir::Value newIsFirst = builder.createBool(loc, false);
+  newIndices.push_back(newIsFirst);
+
+  assert(currentValue.size() == newIndices.size() &&
+         "invalid number of updated reductions");
+
+  return newIndices;
+}
+
+template <typename T>
+hlfir::Entity MinMaxlocAsElementalConverter<T>::genFinalResult(
+    const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
+  // Identification of the final result of MINLOC/MAXLOC:
+  //   * If DIM is absent, the result is rank-one array.
+  //   * If DIM is present:
+  //     - The result is scalar for rank-one input.
+  //     - The result is an array of rank RANK(ARRAY)-1.
+  checkReductions(reductionResults);
+
+  // We need to adjust the one-based indices to real array indices.
+  // The adjustment must only be done, if there was an actual update
+  // of the coordinates in the reduction loop. For this check we only
+  // need to compare if any of the reduction results is not zero.
+  mlir::Value zero = fir::factory::createZeroValue(
+      builder, loc, reductionResults[0].getType());
+  mlir::Value doAdjust = builder.create<mlir::arith::CmpIOp>(
+      loc, mlir::arith::CmpIPredicate::ne, reductionResults[0], zero);
+  mlir::Type indexType = builder.getIndexType();
+  mlir::Value one = builder.createIntegerConstant(loc, indexType, 1);
+
+  auto adjustCoor = [&](mlir::Value coor, mlir::Value lbound) {
+    assert(mlir::isa<mlir::IndexType>(lbound.getType()));
+    mlir::Value coorAsIndex = builder.createConvert(loc, indexType, coor);
+    mlir::Value tmp =
+        builder.create<mlir::arith::AddIOp>(loc, coorAsIndex, lbound);
+    tmp = builder.create<mlir::arith::SubIOp>(loc, tmp, one);
+    tmp =
+        builder.create<mlir::arith::SelectOp>(loc, doAdjust, tmp, coorAsIndex);
+    return builder.createConvert(loc, coor.getType(), tmp);
+  };
+
+  // For partial reductions, the final result of the reduction
+  // loop is just a scalar - the coordinate within DIM dimension.
+  if (getResultRank() == 0 || !isTotalReduction()) {
+    // The result is a scalar, so just return the scalar.
+    assert(getNumCoors() == 1 &&
+           "unpexpected number of coordinates for scalar result");
+
+    int64_t dim = 1;
+    if (!isTotalReduction()) {
+      auto dimVal = getConstDim();
+      assert(mlir::succeeded(dimVal) &&
+             "partial MINLOC/MAXLOC reduction with invalid DIM");
+      dim = *dimVal;
     }
+    mlir::Value dimLbound =
+        hlfir::genLBound(loc, builder, hlfir::Entity{getSource()}, dim - 1);
+    return hlfir::Entity{adjustCoor(reductionResults[0], dimLbound)};
+  }
+  // This is a total reduction, and there is no wrapping hlfir.elemental.
+  // We have to pack the reduced coordinates into a rank-one array.
+  unsigned rank = getSourceRank();
+  // TODO: in order to avoid introducing new memory effects
+  // we should not use a temporary in memory.
+  // We can use hlfir.elemental with a switch to pack all the coordinates
+  // into an array expression, or we can have a dedicated HLFIR operation
+  // for this.
+  mlir::Value tempArray = builder.createTemporary(
+      loc, fir::SequenceType::get(rank, getResultElementType()));
+  llvm::SmallVector<mlir::Value, maxRank> arrayLbounds =
+      hlfir::genLBounds(loc, builder, hlfir::Entity(getSource()));
+  for (unsigned i = 0; i < rank; ++i) {
+    mlir::Value coor = adjustCoor(reductionResults[i], arrayLbounds[i]);
+    mlir::Value idx = builder.createIntegerConstant(loc, indexType, i + 1);
+    mlir::Value resultElement =
+        hlfir::getElementAt(loc, builder, hlfir::Entity{tempArray}, {idx});
+    builder.create<hlfir::AssignOp>(loc, coor, resultElement);
+  }
+  mlir::Value tempExpr = builder.create<hlfir::AsExprOp>(
+      loc, tempArray, builder.createBool(loc, false));
+  return hlfir::Entity{tempExpr};
+}
+
+/// Base class for numeric reductions like MAXVAl, MINVAL, SUM.
+template <typename OpT>
+class NumericReductionAsElementalConverterBase
+    : public ReductionAsElementalConverter {
+  using Base = ReductionAsElementalConverter;
+
+protected:
+  NumericReductionAsElementalConverterBase(OpT op,
+                                           mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
+
+  virtual mlir::Value getSource() const final { return getOp().getArray(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+  virtual mlir::Value getMask() const final { return getOp().getMask(); }
+  virtual mlir::arith::FastMathFlags getFastMath() const final {
+    return getOp().getFastmath();
+  }
 
-    hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
-        loc, builder, elementType, resultShape, {}, genKernel,
-        /*isUnordered=*/true, /*polymorphicMold=*/nullptr,
-        sum.getResult().getType());
+  OpT getOp() const { return mlir::cast<OpT>(op); }
 
-    // it wouldn't be safe to replace block arguments with a different
-    // hlfir.expr type. Types can differ due to differing amounts of shape
-    // information
-    assert(elementalOp.getResult().getType() == sum.getResult().getType());
+  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
+    assert(reductions.size() == 1 && "reduction must produce single value");
+  }
+};
+
+/// Reduction converter for MAXMAL/MINVAL.
+template <typename T>
+class MinMaxvalAsElementalConverter
+    : public NumericReductionAsElementalConverterBase<T> {
+  static_assert(std::is_same_v<T, hlfir::MaxvalOp> ||
+                std::is_same_v<T, hlfir::MinvalOp>);
+  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxvalOp>;
+  using Base = NumericReductionAsElementalConverterBase<T>;
+
+public:
+  MinMaxvalAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
 
-    rewriter.replaceOp(sum, elementalOp);
+private:
+  virtual mlir::LogicalResult isConvertible() const final {
+    if (mlir::isa<fir::CharacterType>(this->getSourceElementType()))
+      return this->rewriter.notifyMatchFailure(
+          this->getOp(),
+          "CHARACTER type is not supported for MINVAL/MAXVAL inlining");
     return mlir::success();
   }
 
-private:
-  // Return fir.shape specifying the shape of the result
-  // of a SUM reduction with DIM=dimVal. The second return value
-  // is the extent of the DIM dimension.
-  static std::tuple<mlir::Value, mlir::Value>
-  genResultShapeForPartialReduction(mlir::Location loc,
-                                    fir::FirOpBuilder &builder,
-                                    hlfir::Entity array, int64_t dimVal) {
-    llvm::SmallVector<mlir::Value> inExtents =
-        hlfir::genExtentsVector(loc, builder, array);
-    assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
-           "DIM must be present and a positive constant not exceeding "
-           "the array's rank");
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {genMinMaxInitValue<isMax>(this->loc, this->builder,
+                                      this->getResultElementType())};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    this->checkReductions(currentValue);
+    fir::FirOpBuilder &builder = this->builder;
+    mlir::Location loc = this->loc;
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value cmp =
+        genMinMaxComparison<isMax>(loc, builder, elementValue, currentValue[0]);
+    return {builder.create<mlir::arith::SelectOp>(loc, cmp, elementValue,
+                                                  currentValue[0])};
+  }
+};
+
+/// Reduction converter for SUM.
+class SumAsElementalConverter
+    : public NumericReductionAsElementalConverterBase<hlfir::SumOp> {
+  using Base = NumericReductionAsElementalConverterBase;
 
-    mlir::Value dimExtent = inExtents[dimVal - 1];
-    inExtents.erase(inExtents.begin() + dimVal - 1);
-    return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
+public:
+  SumAsElementalConverter(hlfir::SumOp op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {
+        fir::factory::createZeroValue(builder, loc, getResultElementType())};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    checkReductions(currentValue);
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    // NOTE: we can use "Kahan summation" same way as the runtime
+    // (e.g. when fast-math is not allowed), but let's start with
+    // the simple version.
+    return {genScalarAdd(currentValue[0], elementValue)};
   }
 
   // Generate scalar addition of the two values (of the same data type).
-  static mlir::Value genScalarAdd(mlir::Location loc,
-                                  fir::FirOpBuilder &builder,
-                                  mlir::Value value1, mlir::Value value2) {
-    mlir::Type ty = value1.getType();
-    assert(ty == value2.getType() && "reduction values' types do not match");
-    if (mlir::isa<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
-    else if (mlir::isa<mlir::ComplexType>(ty))
-      return builder.create<fir::AddcOp>(loc, value1, value2);
-    else if (mlir::isa<mlir::IntegerType>(ty))
-      return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
-
-    llvm_unreachable("unsupported SUM reduction type");
+  mlir::Value genScalarAdd(mlir::Value value1, mlir::Value value2);
+};
+
+/// Base class for logical reductions like ALL, ANY, COUNT.
+/// They do not have MASK and FastMathFlags.
+template <typename OpT>
+class LogicalReductionAsElementalConverterBase
+    : public ReductionAsElementalConverter {
+  using Base = ReductionAsElementalConverter;
+
+public:
+  LogicalReductionAsElementalConverterBase(OpT op,
+                                           mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
+
+protected:
+  OpT getOp() const { return mlir::cast<OpT>(op); }
+
+  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
+    assert(reductions.size() == 1 && "reduction must produce single value");
   }
 
-  static mlir::Value genMaskValue(mlir::Location loc,
-                                  fir::FirOpBuilder &builder, mlir::Value mask,
-                                  mlir::Value isPresentPred,
-                                  mlir::ValueRange indices) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    fir::IfOp ifOp;
-    mlir::Type maskType =
-        hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
-    if (isPresentPred) {
-      ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
-                                       /*withElseRegion=*/true);
-
-      // Use 'true', if the mask is not present.
-      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      mlir::Value trueValue = builder.createBool(loc, true);
-      trueValue = builder.createConvert(loc, maskType, trueValue);
-      builder.create<fir::ResultOp>(loc, trueValue);
-
-      // Load the mask value, if the mask is present.
-      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+  virtual mlir::Value getSource() const final { return getOp().getMask(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) override {
+    checkReductions(reductionResults);
+    return hlfir::Entity{reductionResults[0]};
+  }
+};
+
+/// Reduction converter for ALL/ANY.
+template <typename T>
+class AllAnyAsElementalConverter
+    : public LogicalReductionAsElementalConverterBase<T> {
+  static_assert(std::is_same_v<T, hlfir::AllOp> ||
+                std::is_same_v<T, hlfir::AnyOp>);
+  static constexpr bool isAll = std::is_same_v<T, hlfir::AllOp>;
+  using Base = LogicalReductionAsElementalConverterBase<T>;
+
+public:
+  AllAnyAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {this->builder.createBool(this->loc, isAll ? true : false)};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    this->checkReductions(currentValue);
+    fir::FirOpBuilder &builder = this->builder;
+    mlir::Location loc = this->loc;
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value mask =
+        builder.createConvert(loc, builder.getI1Type(), elementValue);
+    if constexpr (isAll)
+      return {builder.create<mlir::arith::AndIOp>(loc, mask, currentValue[0])};
+    else
+      return {builder.create<mlir::arith::OrIOp>(loc, mask, currentValue[0])};
+  }
+
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionValues) final {
+    this->checkReductions(reductionValues);
+    return hlfir::Entity{this->builder.createConvert(
+        this->loc, this->getResultElementType(), reductionValues[0])};
+  }
+};
+
+/// Reduction converter for COUNT.
+class CountAsElementalConverter
+    : public LogicalReductionAsElementalConverterBase<hlfir::CountOp> {
+  using Base = LogicalReductionAsElementalConverterBase<hlfir::CountOp>;
+
+public:
+  CountAsElementalConverter(hlfir::CountOp op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {
+        fir::factory::createZeroValue(builder, loc, getResultElementType())};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    checkReductions(currentValue);
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value cond =
+        builder.createConvert(loc, builder.getI1Type(), elementValue);
+    mlir::Value one =
+        builder.createIntegerConstant(loc, getResultElementType(), 1);
+    mlir::Value add1 =
+        builder.create<mlir::arith::AddIOp>(loc, currentValue[0], one);
+    return {builder.create<mlir::arith::SelectOp>(loc, cond, add1,
+                                                  currentValue[0])};
+  }
+};
+
+mlir::LogicalResult ReductionAsElementalConverter::convert() {
+  mlir::LogicalResult canConvert(isConvertible());
+
+  if (mlir::failed(canConvert))
+    return canConvert;
+
+  hlfir::Entity array = hlfir::Entity{getSource()};
+  bool isTotalReduce = isTotalReduction();
+  auto dimVal = getConstDim();
+  if (mlir::failed(dimVal))
+    return dimVal;
+  mlir::Value mask = getMask();
+  mlir::Value resultShape, dimExtent;
+  llvm::SmallVector<mlir::Value> arrayExtents;
+  if (isTotalReduce)
+    arrayExtents = hlfir::genExtentsVector(loc, builder, array);
+  else
+    std::tie(resultShape, dimExtent) =
+        genResultShapeForPartialReduction(array, *dimVal);
+
+  // If the mask is present and is a scalar, then we'd better load its value
+  // outside of the reduction loop making the loop unswitching easier.
+  mlir::Value isPresentPred, maskValue;
+  if (mask) {
+    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+      // MASK represented by a box might be dynamically optional,
+      // so we have to check for its presence before accessing it.
+      isPresentPred =
+          builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
     }
 
-    hlfir::Entity maskVar{mask};
-    if (maskVar.isScalar()) {
-      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-        // MASK may be a boxed scalar.
-        mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
-        mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
+    if (hlfir::Entity{mask}.isScalar())
+      maskValue = genMaskValue(mask, isPresentPred, {});
+  }
+
+  auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                       mlir::ValueRange inputIndices) -> hlfir::Entity {
+    // Loop over all indices in the DIM dimension, and reduce all values.
+    // If DIM is not present, do total reduction.
+
+    // Initial value for the reduction.
+    llvm::SmallVector<mlir::Value, 1> reductionInitValues =
+        genReductionInitValues();
+
+    llvm::SmallVector<mlir::Value> extents;
+    if (isTotalReduce)
+      extents = arrayExtents;
+    else
+      extents.push_back(
+          builder.createConvert(loc, builder.getIndexType(), dimExtent));
+
+    auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                       mlir::ValueRange oneBasedIndices,
+                       mlir::ValueRange reductionArgs)
+        -> llvm::SmallVector<mlir::Value, 1> {
+      // Generate the reduction loop-nest body.
+      // The initial reduction value in the innermost loop
+      // is passed via reductionArgs[0].
+      llvm::SmallVector<mlir::Value> indices;
+      if (isTotalReduce) {
+        indices = oneBasedIndices;
       } else {
-        mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
+        indices = inputIndices;
+        indices.insert(indices.begin() + *dimVal - 1, oneBasedIndices[0]);
+      }
+
+      llvm::SmallVector<mlir::Value, 1> reductionValues = reductionArgs;
+      llvm::SmallVector<mlir::Type, 1> reductionTypes;
+      llvm::transform(reductionValues, std::back_inserter(reductionTypes),
+                      [](mlir::Value v) { return v.getType(); });
+      fir::IfOp ifOp;
+      if (mask) {
+        // Make the reduction value update conditional on the value
+        // of the mask.
+        if (!maskValue) {
+          // If the mask is an array, use the elemental and the loop indices
+          // to address the proper mask element.
+          maskValue = genMaskValue(mask, isPresentPred, indices);
+        }
+        mlir::Value isUnmasked =
+            builder.create<fir::ConvertOp>(loc, builder.getI1Type(), maskValue);
+        ifOp = builder.create<fir::IfOp>(loc, reductionTypes, isUnmasked,
+                                         /*withElseRegion=*/true);
+        // In the 'else' block return the current reduction value.
+        builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+        builder.create<fir::ResultOp>(loc, reductionValues);
+
+        // In the 'then' block do the actual addition.
+        builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
       }
+      reductionValues = reduceOneElement(reductionValues, array, indices);
+      if (ifOp) {
+        builder.create<fir::ResultOp>(loc, reductionValues);
+        builder.setInsertionPointAfter(ifOp);
+        reductionValues = ifOp.getResults();
+      }
+
+      return reductionValues;
+    };
+
+    llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
+        hlfir::genLoopNestWithReductions(
+            loc, builder, extents, reductionInitValues, genBody, isUnordered());
+    return genFinalResult(reductionFinalValues);
+  };
+
+  if (isTotalReduce) {
+    hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
+    rewriter.replaceOp(op, result);
+    return mlir::success();
+  }
+
+  hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
+      loc, builder, getResultElementType(), resultShape, /*typeParams=*/{},
+      genKernel,
+      /*isUnordered=*/true, /*polymorphicMold=*/nullptr, getResultType());
+
+  // it wouldn't be safe to replace block arguments with a different
+  // hlfir.expr type. Types can differ due to differing amounts of shape
+  // information
+  assert(elementalOp.getResult().getType() == op->getResult(0).getType());
+
+  rewriter.replaceOp(op, elementalOp);
+  return mlir::success();
+}
+
+std::tuple<mlir::Value, mlir::Value>
+ReductionAsElementalConverter::genResultShapeForPartialReduction(
+    hlfir::Entity array, int64_t dimVal) {
+  llvm::SmallVector<mlir::Value> inExtents =
+      hlfir::genExtentsVector(loc, builder, array);
+  assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
+         "DIM must be present and a positive constant not exceeding "
+         "the array's rank");
+
+  mlir::Value dimExtent = inExtents[dimVal - 1];
+  inExtents.erase(inExtents.begin() + dimVal - 1);
+  return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
+}
+
+mlir::Value SumAsElementalConverter::genScalarAdd(mlir::Value value1,
+                                                  mlir::Value value2) {
+  mlir::Type ty = value1.getType();
+  assert(ty == value2.getType() && "reduction values' types do not match");
+  if (mlir::isa<mlir::FloatType>(ty))
+    return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+  else if (mlir::isa<mlir::ComplexType>(ty))
+    return builder.create<fir::AddcOp>(loc, value1, value2);
+  else if (mlir::isa<mlir::IntegerType>(ty))
+    return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+
+  llvm_unreachable("unsupported SUM reduction type");
+}
+
+mlir::Value ReductionAsElementalConverter::genMaskValue(
+    mlir::Value mask, mlir::Value isPresentPred, mlir::ValueRange indices) {
+  mlir::OpBuilder::InsertionGuard guard(builder);
+  fir::IfOp ifOp;
+  mlir::Type maskType =
+      hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
+  if (isPresentPred) {
+    ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
+                                     /*withElseRegion=*/true);
+
+    // Use 'true', if the mask is not present.
+    builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+    mlir::Value trueValue = builder.createBool(loc, true);
+    trueValue = builder.createConvert(loc, maskType, trueValue);
+    builder.create<fir::ResultOp>(loc, trueValue);
+
+    // Load the mask value, if the mask is present.
+    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+  }
+
+  hlfir::Entity maskVar{mask};
+  if (maskVar.isScalar()) {
+    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+      // MASK may be a boxed scalar.
+      mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
+      mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
     } else {
-      // Load from the mask array.
-      assert(!indices.empty() && "no indices for addressing the mask array");
-      maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
       mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
     }
+  } else {
+    // Load from the mask array.
+    assert(!indices.empty() && "no indices for addressing the mask array");
+    maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
+    mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
+  }
+
+  if (!isPresentPred)
+    return mask;
 
-    if (!isPresentPred)
-      return mask;
+  builder.create<fir::ResultOp>(loc, mask);
+  return ifOp.getResult(0);
+}
+
+/// Convert an operation that is a partial or total reduction
+/// over an array of values into a reduction loop[-nest]
+/// optionally wrapped into hlfir.elemental.
+template <typename Op>
+class ReductionConversion : public mlir::OpRewritePattern<Op> {
+public:
+  using mlir::OpRewritePattern<Op>::OpRewritePattern;
 
-    builder.create<fir::ResultOp>(loc, mask);
-    return ifOp.getResult(0);
+  llvm::LogicalResult
+  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
+    if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                  std::is_same_v<Op, hlfir::MinlocOp>) {
+      MinMaxlocAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      MinMaxvalAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
+      CountAsElementalConverter converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::AllOp> ||
+                         std::is_same_v<Op, hlfir::AnyOp>) {
+      AllAnyAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::SumOp>) {
+      SumAsElementalConverter converter{op, rewriter};
+      return converter.convert();
+    }
+    return rewriter.notifyMatchFailure(op, "unexpected reduction operation");
   }
 };
 
@@ -1481,10 +2118,18 @@ class SimplifyHLFIRIntrinsics
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
-    patterns.insert<SumAsElementalConversion>(context);
+    patterns.insert<ReductionConversion<hlfir::SumOp>>(context);
     patterns.insert<CShiftConversion>(context);
     patterns.insert<MatmulConversion<hlfir::MatmulTransposeOp>>(context);
 
+    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
+
     // If forceMatmulAsElemental is false, then hlfir.matmul inlining
     // will introduce hlfir.eval_in_mem operation with new memory side
     // effects. This conflicts with CSE and optimized bufferization, e.g.:
diff --git a/flang/test/HLFIR/all-elemental.fir b/flang/test/HLFIR/all-elemental.fir
deleted file mode 100644
index 1ba8bb1b7a5fb..0000000000000
--- a/flang/test/HLFIR/all-elemental.fir
+++ /dev/null
@@ -1,91 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.all %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
-  return %13 : !fir.logical<4>
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-// CHECK-NEXT:     %true = arith.constant true
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %true) -> (i1) {
-// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.andi %arg4, %[[V16]] : i1
-// CHECK-NEXT:       fir.result %[[V17]] : i1
-// CHECK-NEXT:     }
-// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
-// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
-
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
-  %c2_i32 = arith.constant 2 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.all %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
-  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
-  return %11 : !fir.array<4x!fir.logical<4>>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: %10 = hlfir.all %9 dim %c2_i32
\ No newline at end of file
diff --git a/flang/test/HLFIR/any-elemental.fir b/flang/test/HLFIR/any-elemental.fir
deleted file mode 100644
index a7c559679d965..0000000000000
--- a/flang/test/HLFIR/any-elemental.fir
+++ /dev/null
@@ -1,190 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.any %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
-  return %13 : !fir.logical<4>
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-// CHECK-NEXT:     %false = arith.constant false
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %false) -> (i1) {
-// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.ori %arg4, %[[V16]] : i1
-// CHECK-NEXT:       fir.result %[[V17]] : i1
-// CHECK-NEXT:     }
-// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
-// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
-
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
-  %c2_i32 = arith.constant 2 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.any %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
-  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
-  return %11 : !fir.array<4x!fir.logical<4>>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: {{.*}} = hlfir.any {{.*}} dim %c2_i32
-
-
-func.func @_Qtest_recursive() attributes {fir.bindc_name = "test"} {
-  %c1 = arith.constant 1 : index
-  %true = arith.constant true
-  %false = arith.constant false
-  %c0_i64 = arith.constant 0 : i64
-  %c2_i32 = arith.constant 2 : i32
-  %c0 = arith.constant 0 : index
-  %c1_i32 = arith.constant 1 : i32
-  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  %1:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
-  %2 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
-  %5:2 = hlfir.declare %4 {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "ra", uniq_name = "_QFEra"}
-  %7 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %8:2 = hlfir.declare %6(%7) {uniq_name = "_QFEra"} : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.ref<!fir.array<1x!fir.logical<4>>>)
-  %9 = fir.alloca !fir.logical<4> {bindc_name = "rs", uniq_name = "_QFErs"}
-  %10:2 = hlfir.declare %9 {uniq_name = "_QFErs"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %11 = fir.allocmem !fir.array<?xi32>, %c1 {fir.must_be_heap = true, uniq_name = "_QFEa.alloc"}
-  %12 = fir.embox %11(%7) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
-  fir.store %12 to %1#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  hlfir.assign %c1_i32 to %5#0 : i32, !fir.ref<i32>
-  %13 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  %14:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-  fir.do_loop %arg0 = %c1 to %14#1 step %c1 unordered {
-    %27:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-    %28 = arith.subi %27#0, %c1 : index
-    %29 = arith.addi %arg0, %28 : index
-    %30 = hlfir.designate %13 (%29)  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
-    hlfir.assign %c2_i32 to %30 : i32, !fir.ref<i32>
-  }
-  %15 = fir.load %5#0 : !fir.ref<i32>
-  %16 = fir.convert %15 : (i32) -> i64
-  %17 = arith.cmpi sgt, %16, %c0_i64 : i64
-  %18 = arith.select %17, %16, %c0_i64 : i64
-  %19 = fir.convert %18 : (i64) -> index
-  %20 = fir.shape %19 : (index) -> !fir.shape<1>
-  %21 = hlfir.elemental %20 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg0: index):
-    %27 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    %28:3 = fir.box_dims %27, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-    %29 = arith.addi %28#0, %28#1 : index
-    %30 = arith.subi %29, %c1 : index
-    %31 = arith.subi %30, %28#0 : index
-    %32 = arith.addi %31, %c1 : index
-    %33 = arith.cmpi sgt, %32, %c0 : index
-    %34 = arith.select %33, %32, %c0 : index
-    %35 = fir.shape %34 : (index) -> !fir.shape<1>
-    %36 = hlfir.designate %27 (%28#0:%30:%c1)  shape %35 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-    %37 = hlfir.elemental %35 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-    ^bb0(%arg1: index):
-      %39 = hlfir.designate %36 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-      %40 = fir.load %39 : !fir.ref<i32>
-      %41 = arith.cmpi eq, %40, %c1_i32 : i32
-      %42 = fir.convert %41 : (i1) -> !fir.logical<4>
-      hlfir.yield_element %42 : !fir.logical<4>
-    }
-    %38 = hlfir.any %37 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
-    hlfir.destroy %37 : !hlfir.expr<?x!fir.logical<4>>
-    hlfir.yield_element %38 : !fir.logical<4>
-  }
-  %22 = hlfir.any %21 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %22 to %10#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %21 : !hlfir.expr<?x!fir.logical<4>>
-  %23 = fir.load %10#0 : !fir.ref<!fir.logical<4>>
-  %24 = fir.convert %23 : (!fir.logical<4>) -> i1
-  %25 = arith.xori %24, %true : i1
-  cf.cond_br %25, ^bb1, ^bb2
-^bb1:  // pred: ^bb0
-  fir.call @_FortranAStopStatement(%c2_i32, %false, %false) fastmath<contract> : (i32, i1, i1) -> ()
-  fir.unreachable
-^bb2:  // pred: ^bb0
-  return
-}
-// CHECK-LABEL: func.func @_Qtest_recursive()
-// CHECK:    %[[V20:.*]] = fir.do_loop %arg0 = %c1 to %{{.*}} step %c1 iter_args(%arg1 = %false) -> (i1) {
-// CHECK:      %[[V26:.*]] = fir.load %[[V1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-// CHECK:      %[[V27:.*]]:3 = fir.box_dims %[[V26]], %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-// CHECK:      %[[V28:.*]] = arith.addi %[[V27]]#0, %[[V27]]#1 : index
-// CHECK:      %[[V29:.*]] = arith.subi %[[V28]], %c1 : index
-// CHECK:      %[[V30:.*]] = arith.subi %[[V29]], %[[V27]]#0 : index
-// CHECK:      %[[V31:.*]] = arith.addi %[[V30]], %c1 : index
-// CHECK:      %[[V32:.*]] = arith.cmpi sgt, %[[V31]], %c0 : index
-// CHECK:      %[[V33:.*]] = arith.select %[[V32]], %[[V31]], %c0 : index
-// CHECK:      %[[V34:.*]] = fir.shape %[[V33]] : (index) -> !fir.shape<1>
-// CHECK:      %[[V35:.*]] = hlfir.designate %[[V26]] (%[[V27]]#0:%[[V29]]:%c1)  shape %[[V34]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-// CHECK:      %[[V36:.*]] = fir.do_loop %arg2 = %c1 to %[[V33]] step %c1 iter_args(%arg3 = %false) -> (i1) {
-// CHECK:        %[[V38:.*]] = hlfir.designate %[[V35]] (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:        %[[V39:.*]] = fir.load %[[V38]] : !fir.ref<i32>
-// CHECK:        %[[V40:.*]] = arith.cmpi eq, %[[V39]], %c1_i32 : i32
-// CHECK:        %[[V41:.*]] = arith.ori %arg3, %[[V40]] : i1
-// CHECK:        fir.result %[[V41]] : i1
-// CHECK:      }
-// CHECK:      %[[V37:.*]] = arith.ori %arg1, %[[V36]] : i1
-// CHECK:      fir.result %[[V37]] : i1
-// CHECK:    }
diff --git a/flang/test/HLFIR/count-elemental.fir b/flang/test/HLFIR/count-elemental.fir
deleted file mode 100644
index 0df5cc3c031ea..0000000000000
--- a/flang/test/HLFIR/count-elemental.fir
+++ /dev/null
@@ -1,314 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %12 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<i32>
-  return %13 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
-// CHECK-NEXT:       fir.result %[[V17]] : i32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
-// CHECK-NEXT:     return %[[V12]] : i32
-
-func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i16 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i16
-  hlfir.assign %12 to %4#0 : i16, !fir.ref<i16>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<i16>
-  return %13 : i16
-}
-// CHECK-LABEL:  func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
-// CHECK-NEXT:     %c1_i16 = arith.constant 1 : i16
-// CHECK-NEXT:     %c0_i16 = arith.constant 0 : i16
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i16
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i16) -> (i16) {
-// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i16 : i16
-// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i16
-// CHECK-NEXT:       fir.result %[[V17]] : i16
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i16, !fir.ref<i16>
-// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i16>
-// CHECK-NEXT:     return %[[V12]] : i16
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<7xi32> {
-  %c1_i32 = arith.constant 1 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<7xi32> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<7xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<7xi32>>, !fir.ref<!fir.array<7xi32>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.count %9 dim %c1_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<7xi32>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<7xi32>, !fir.ref<!fir.array<7xi32>>
-  hlfir.destroy %10 : !hlfir.expr<7xi32>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<7xi32>>
-  return %11 : !fir.array<7xi32>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: %{{.*}} = hlfir.count %{{.*}} dim %c1_i32
-
-
-func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %c2 = arith.constant 2 : index
-  %0 = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %0 : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
-  %7 = fir.load %5#0 : !fir.ref<i32>
-  %8 = hlfir.elemental %0 unordered : (!fir.shape<3>) -> !hlfir.expr<4x7x2x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index, %arg5: index):
-    %11 = hlfir.designate %6 (%arg3, %arg4, %arg5)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
-    %12 = fir.load %11 : !fir.ref<i32>
-    %13 = arith.cmpi sge, %12, %7 : i32
-    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %14 : !fir.logical<4>
-  }
-  %9 = hlfir.count %8 : (!hlfir.expr<4x7x2x!fir.logical<4>>) -> i32
-  hlfir.assign %9 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %8 : !hlfir.expr<4x7x2x!fir.logical<4>>
-  %10 = fir.load %4#1 : !fir.ref<i32>
-  return %10 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %c2 = arith.constant 2 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]]) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]] {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %[[V0]] : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
-// CHECK-NEXT:     %[[V7:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V8:.*]] = fir.do_loop %arg3 = %c1 to %c2 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK-NEXT:       %[[V10:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %arg4) -> (i32) {
-// CHECK-NEXT:         %[[V11:.*]] = fir.do_loop %arg7 = %c1 to %c4 step %c1 iter_args(%arg8 = %arg6) -> (i32) {
-// CHECK-NEXT:           %[[V12:.*]] = hlfir.designate %[[V6]] (%arg7, %arg5, %arg3)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
-// CHECK-NEXT:           %[[V13:.*]] = fir.load %[[V12]] : !fir.ref<i32>
-// CHECK-NEXT:           %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V7]] : i32
-// CHECK-NEXT:           %[[V15:.*]] = arith.addi %arg8, %c1_i32 : i32
-// CHECK-NEXT:           %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg8 : i32
-// CHECK-NEXT:           fir.result %[[V16]] : i32
-// CHECK-NEXT:         }
-// CHECK-NEXT:         fir.result %[[V11]] : i32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V10]] : i32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V8]] to %[[V4]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:     %[[V9:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
-// CHECK-NEXT:     return %[[V9]] : i32
-
-
-
-
-
-func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7xi32> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %16 = fir.load %15 : !fir.ref<i32>
-    hlfir.yield_element %16 : i32
-  }
-  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.sum %11 : (!hlfir.expr<7xi32>) -> i32
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7xi32>
-  %14 = fir.load %4#1 : !fir.ref<i32>
-  return %14 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK:    %[[V12:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK:      %[[V14:.*]] = hlfir.sum %[[V11]] : (!hlfir.expr<7xi32>) -> i32
-// CHECK:      %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK:      %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK:      %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
-// CHECK:      fir.result %[[V17]] : i32
-// CHECK:    }
-
-
-
-
-func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %16 = fir.load %15 : !fir.ref<i32>
-    %17 = arith.cmpi sge, %16, %10 : i32
-    %18 = fir.convert %17 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %18 : !fir.logical<4>
-  }
-  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %14 = fir.load %4#1 : !fir.ref<i32>
-  return %14 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK:    %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK:      %[[V13:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %c0_i32) -> (i32) {
-// CHECK:        %[[V17:.*]] = hlfir.designate %[[V9]] (%arg5)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK:        %[[V18:.*]] = fir.load %[[V17]] : !fir.ref<i32>
-// CHECK:        %[[V19:.*]] = arith.cmpi sge, %[[V18]], %[[V10]] : i32
-// CHECK:        %[[V20:.*]] = arith.addi %arg6, %c1_i32 : i32
-// CHECK:        %[[V21:.*]] = arith.select %[[V19]], %[[V20]], %arg6 : i32
-// CHECK:        fir.result %[[V21]] : i32
-// CHECK:      }
-// CHECK:      %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V10]] : i32
-// CHECK:      %[[V15:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK:      %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg4 : i32
-// CHECK:      fir.result %[[V16]] : i32
-// CHECK:    }
diff --git a/flang/test/HLFIR/maxloc-elemental.fir b/flang/test/HLFIR/maxloc-elemental.fir
deleted file mode 100644
index c9210a59f0340..0000000000000
--- a/flang/test/HLFIR/maxloc-elemental.fir
+++ /dev/null
@@ -1,133 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
-// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c-2147483648_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi sgt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3 = fir.load %2#0 : !fir.ref<f32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-    %9 = fir.load %8 : !fir.ref<f32>
-    %10 = arith.cmpf oge, %9, %3 : f32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: _QPtest_float
-// CHECK:        %cst = arith.constant 0xFF800000 : f32
-// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
-// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
-// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
-// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
-// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
-// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf ogt, %[[V20]], %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
-// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
-// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
-// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
-// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:         fir.result %[[V20]] : f32
-// CHECK-NEXT:       } else {
-// CHECK-NEXT:         fir.result %arg4 : f32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V22]] : f32
-// CHECK-NEXT:     } else {
-// CHECK-NEXT:       fir.result %arg4 : f32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     fir.result %[[V18]] : f32
-// CHECK-NEXT:   }
-
diff --git a/flang/test/HLFIR/maxval-elemental.fir b/flang/test/HLFIR/maxval-elemental.fir
deleted file mode 100644
index a21b4858412de..0000000000000
--- a/flang/test/HLFIR/maxval-elemental.fir
+++ /dev/null
@@ -1,117 +0,0 @@
-// Test maxval inlining for both elemental and designate
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-// subroutine test(array)
-//   integer :: array(:), x
-//   x = maxval(abs(array))
-// end subroutine test
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-  %c31_i32 = arith.constant 31 : i32
-  %c0 = arith.constant 0 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg1: index):
-    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.shrsi %9, %c31_i32 : i32
-    %11 = arith.xori %9, %10 : i32
-    %12 = arith.subi %11, %10 : i32
-    hlfir.yield_element %12 : i32
-  }
-  %7 = hlfir.maxval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
-  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
-  hlfir.destroy %6 : !hlfir.expr<?xi32>
-  return
-}
-
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
-// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c-2147483648_i32) -> (i32) {
-// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
-// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi sgt, %[[V10]], %arg2 : i32
-// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
-// CHECK-NEXT:      fir.result %[[V12]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// subroutine test(array)
-//   real :: array(:), x
-//   x = maxval(array(3:6))
-// end subroutine test
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
-  %c4 = arith.constant 4 : index
-  %c1 = arith.constant 1 : index
-  %c6 = arith.constant 6 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-  %6 = hlfir.maxval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
-  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
-  return
-}
-
-// CHECK-LABEL: _QPtest_float
-// CHECK:       %cst = arith.constant 0xFF800000 : f32
-// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
-// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
-// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
-// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
-// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
-// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
-// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
-// CHECK-NEXT:      fir.result %[[V14]] : f32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// Verify that lower bounds of designator are applied in the indexing inside
-// the generated loop (hlfir.designate takes indices relative to the base lower
-// bounds).
-func.func @component_lower_bounds(%arg0: !fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>) -> i32 {
-  %c10 = arith.constant 10 : index
-  %c101 = arith.constant 101 : index
-  %4 = fir.shape_shift %c101, %c10 : (index, index) -> !fir.shapeshift<1>
-  %5 = hlfir.designate %arg0{"i"}   shape %4 : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
-  %6 = hlfir.maxval %5 : (!fir.box<!fir.array<10xi32>>) -> i32
-  return %6 : i32
-}
-// CHECK-LABEL:   func.func @component_lower_bounds(
-// CHECK:  %[[VAL_1:.*]] = arith.constant 100 : index
-// CHECK:  %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:  %[[VAL_4:.*]] = arith.constant 10 : index
-// CHECK:  %[[VAL_5:.*]] = arith.constant 101 : index
-// CHECK:  %[[VAL_6:.*]] = fir.shape_shift %[[VAL_5]], %[[VAL_4]] : (index, index) -> !fir.shapeshift<1>
-// CHECK:  %[[VAL_7:.*]] = hlfir.designate %{{.*}}{"i"}   shape %[[VAL_6]] : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
-// CHECK:  %[[VAL_8:.*]] = fir.do_loop %[[VAL_9:.*]] = %[[VAL_2]] to %[[VAL_4]] {{.*}}
-// CHECK:    %[[VAL_11:.*]] = arith.addi %[[VAL_9]], %[[VAL_1]] : index
-// CHECK:    hlfir.designate %[[VAL_7]] (%[[VAL_11]])  : (!fir.box<!fir.array<10xi32>>, index) -> !fir.ref<i32>
diff --git a/flang/test/HLFIR/minloc-elemental.fir b/flang/test/HLFIR/minloc-elemental.fir
deleted file mode 100644
index 9453a335b4fbf..0000000000000
--- a/flang/test/HLFIR/minloc-elemental.fir
+++ /dev/null
@@ -1,397 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-
-func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi16>, !fir.box<!fir.array<?xi16>>
-  hlfir.destroy %7 : !hlfir.expr<1xi16>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL:  func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c1_i16 = arith.constant 1 : i16
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i16 = arith.constant 0 : i16
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i16
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK-NEXT:    fir.store %c0_i16 to %[[V8]] : !fir.ref<i16>
-// CHECK-NEXT:    fir.store %c0_i16 to %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i16) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i16 to %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i16
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i16>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi16>>, !fir.box<!fir.array<?xi16>>
-// CHECK-NEXT:    return
-
-
-func.func @_QPtest_kind2_convert(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c1 = arith.constant 1 : index
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %10 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %11 = fir.load %10 : !fir.ref<i32>
-    %12 = arith.cmpi sge, %11, %3 : i32
-    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %13 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %9 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg3: index):
-    %10 = hlfir.apply %7, %arg3 : (!hlfir.expr<1xi16>, index) -> i16
-    %11 = fir.convert %10 : (i16) -> i32
-    hlfir.yield_element %11 : i32
-  }
-  hlfir.assign %9 to %1#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %9 : !hlfir.expr<?xi32>
-  hlfir.destroy %7 : !hlfir.expr<1xi16>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL:   func.func @_QPtest_kind2_convert(
-// CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"},
-// CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "val"},
-// CHECK-SAME:                                     %[[VAL_2:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK:           %[[VAL_3:.*]] = arith.constant false
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i16
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_10:.*]] = fir.alloca i16
-// CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
-// CHECK:           %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_16]] : !fir.ref<i16>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<i16>
-// CHECK:           %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_12]]#0, %[[VAL_7]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#1, %[[VAL_9]] : index
-// CHECK:           %[[VAL_19:.*]] = fir.do_loop %[[VAL_20:.*]] = %[[VAL_7]] to %[[VAL_18]] step %[[VAL_9]] iter_args(%[[VAL_21:.*]] = %[[VAL_5]]) -> (i32) {
-// CHECK:             %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_9]] : index
-// CHECK:             %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<i32>
-// CHECK:             %[[VAL_25:.*]] = arith.cmpi sge, %[[VAL_24]], %[[VAL_15]] : i32
-// CHECK:             %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (i32) {
-// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_10]] : !fir.ref<i16>
-// CHECK:               %[[VAL_28:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_28]] : !fir.ref<i32>
-// CHECK:               %[[VAL_30:.*]] = arith.cmpi slt, %[[VAL_29]], %[[VAL_21]] : i32
-// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_27]] : (i16) -> i1
-// CHECK:               %[[VAL_32:.*]] = arith.xori %[[VAL_31]], %[[VAL_4]] : i1
-// CHECK:               %[[VAL_33:.*]] = arith.ori %[[VAL_30]], %[[VAL_32]] : i1
-// CHECK:               %[[VAL_34:.*]] = fir.if %[[VAL_33]] -> (i32) {
-// CHECK:                 fir.store %[[VAL_6]] to %[[VAL_10]] : !fir.ref<i16>
-// CHECK:                 %[[VAL_35:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_22]] : (index) -> i16
-// CHECK:                 fir.store %[[VAL_36]] to %[[VAL_35]] : !fir.ref<i16>
-// CHECK:                 fir.result %[[VAL_29]] : i32
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_21]] : i32
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_34]] : i32
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_21]] : i32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_26]] : i32
-// CHECK:           }
-// CHECK:           %[[VAL_37:.*]] = hlfir.as_expr %[[VAL_11]] move %[[VAL_3]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
-// CHECK:           fir.do_loop %[[VAL_38:.*]] = %[[VAL_9]] to %[[VAL_9]] step %[[VAL_9]] unordered {
-// CHECK:             %[[VAL_39:.*]] = hlfir.apply %[[VAL_37]], %[[VAL_38]] : (!hlfir.expr<1xi16>, index) -> i16
-// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i16) -> i32
-// CHECK:             %[[VAL_41:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_38]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:             hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i32, !fir.ref<i32>
-// CHECK:           }
-// CHECK:           return
-// CHECK:         }
-
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3 = fir.load %2#0 : !fir.ref<f32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-    %9 = fir.load %8 : !fir.ref<f32>
-    %10 = arith.cmpf oge, %9, %3 : f32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: _QPtest_float
-// CHECK:        %cst = arith.constant 0x7F800000 : f32
-// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
-// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
-// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
-// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
-// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
-// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf olt, %[[V20]], %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
-// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
-// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
-// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
-// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:         fir.result %[[V20]] : f32
-// CHECK-NEXT:       } else {
-// CHECK-NEXT:         fir.result %arg4 : f32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V22]] : f32
-// CHECK-NEXT:     } else {
-// CHECK-NEXT:       fir.result %arg4 : f32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     fir.result %[[V18]] : f32
-// CHECK-NEXT:   }
-
-
-func.func @_QPtest_assignshape(%arg0: !fir.ref<!fir.array<3x3xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.ref<!fir.array<3xi32>> {fir.bindc_name = "m"}) {
-  %c2 = arith.constant 2 : index
-  %c1 = arith.constant 1 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.shape %c3, %c3 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFtestEarray"} : (!fir.ref<!fir.array<3x3xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x3xf32>>, !fir.ref<!fir.array<3x3xf32>>)
-  %2 = fir.shape %c3 : (index) -> !fir.shape<1>
-  %3:2 = hlfir.declare %arg2(%2) {uniq_name = "_QFtestEm"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-  %4:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %5 = fir.load %4#0 : !fir.ref<f32>
-  %6 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<3x3x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %10 = hlfir.designate %1#0 (%arg3, %arg4)  : (!fir.ref<!fir.array<3x3xf32>>, index, index) -> !fir.ref<f32>
-    %11 = fir.load %10 : !fir.ref<f32>
-    %12 = arith.cmpf oge, %11, %5 : f32
-    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %13 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %1#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.ref<!fir.array<3x3xf32>>, !hlfir.expr<3x3x!fir.logical<4>>) -> !hlfir.expr<2xi32>
-  %8 = fir.shape %c2 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %3#0 (%c1:%c2:%c1)  shape %8 : (!fir.ref<!fir.array<3xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<2xi32>>
-  hlfir.assign %7 to %9 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
-  hlfir.destroy %7 : !hlfir.expr<2xi32>
-  hlfir.destroy %6 : !hlfir.expr<3x3x!fir.logical<4>>
-  return
-}
-// Not supported as the input is not a box
-// CHECK-LABEL: _QPtest_assignshape
-// CHECK: hlfir.minloc
-
-
-func.func @_QFPtest_character(%arg0: !fir.box<!fir.array<?x!fir.char<1>>> {fir.bindc_name = "b"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c0 = arith.constant 0 : index
-  %c1 = arith.constant 1 : index
-  %0:2 = hlfir.declare %arg0 typeparams %c1 {uniq_name = "_QFFtestEb"} : (!fir.box<!fir.array<?x!fir.char<1>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1>>>, !fir.box<!fir.array<?x!fir.char<1>>>)
-  %1:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestEc"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca !fir.array<1xi32> {bindc_name = "m", uniq_name = "_QFFtestEm"}
-  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %4:2 = hlfir.declare %2(%3) {uniq_name = "_QFFtestEm"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
-  %5 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %6:2 = hlfir.declare %5 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %8 = fir.load %7#0 : !fir.ref<i32>
-  %9:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %10 = fir.shape %9#1 : (index) -> !fir.shape<1>
-  %11 = hlfir.elemental %10 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %16 = hlfir.designate %1#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %17 = fir.load %16 : !fir.ref<i32>
-    %18 = arith.cmpi eq, %17, %8 : i32
-    %19 = fir.convert %18 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %19 : !fir.logical<4>
-  }
-  %12 = hlfir.minloc %0#0 mask %11 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x!fir.char<1>>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %12 to %4#0 : !hlfir.expr<1xi32>, !fir.ref<!fir.array<1xi32>>
-  hlfir.destroy %12 : !hlfir.expr<1xi32>
-  hlfir.destroy %11 : !hlfir.expr<?x!fir.logical<4>>
-  %13 = hlfir.designate %4#0 (%c1)  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-  %14 = fir.load %13 : !fir.ref<i32>
-  hlfir.assign %14 to %6#0 : i32, !fir.ref<i32>
-  %15 = fir.load %6#1 : !fir.ref<i32>
-  return %15 : i32
-}
-// Characters are not supported at the moment
-// CHECK-LABEL: _QFPtest_character
-// CHECK: hlfir.minloc
-
-
-func.func @_QPtest_parts(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "mask"}) -> f32 {
-  %c1 = arith.constant 1 : index
-  %c5 = arith.constant 5 : index
-  %c0 = arith.constant 0 : index
-  %c5_i32 = arith.constant 5 : i32
-  %0:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEmask"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1 = fir.alloca f32 {bindc_name = "test", uniq_name = "_QFtestEtest"}
-  %2:2 = hlfir.declare %1 {uniq_name = "_QFtestEtest"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEx"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg2: index):
-    %11 = hlfir.designate %0#0 (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %12 = fir.load %11 : !fir.ref<i32>
-    %13 = arith.cmpi sge, %12, %c5_i32 : i32
-    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %14 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %3#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %3#0 (%c5:%c5:%c1)  shape %8 : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
-  hlfir.assign %7 to %9 : !hlfir.expr<1xi32>, !fir.box<!fir.array<1xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  %10 = fir.load %2#1 : !fir.ref<f32>
-  return %10 : f32
-}
-// Characters are not supported at the moment
-// CHECK-LABEL: _QPtest_parts
-// CHECK: fir.do_loop %{{.*}} = %c0 to %{{.*}} step %c1 iter_args(%{{.*}} = %c2147483647_i32) -> (i32) {
-
diff --git a/flang/test/HLFIR/minval-elemental.fir b/flang/test/HLFIR/minval-elemental.fir
deleted file mode 100644
index 64cd5403ec558..0000000000000
--- a/flang/test/HLFIR/minval-elemental.fir
+++ /dev/null
@@ -1,95 +0,0 @@
-// Test maxval inlining for both elemental and designate
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-// subroutine test(array)
-//   integer :: array(:), x
-//   x = minval(abs(array))
-// end subroutine test
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-  %c31_i32 = arith.constant 31 : i32
-  %c0 = arith.constant 0 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg1: index):
-    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.shrsi %9, %c31_i32 : i32
-    %11 = arith.xori %9, %10 : i32
-    %12 = arith.subi %11, %10 : i32
-    hlfir.yield_element %12 : i32
-  }
-  %7 = hlfir.minval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
-  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
-  hlfir.destroy %6 : !hlfir.expr<?xi32>
-  return
-}
-
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
-// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi slt, %[[V10]], %arg2 : i32
-// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
-// CHECK-NEXT:      fir.result %[[V12]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// subroutine test(array)
-//   real :: array(:), x
-//   x = minval(array(3:6))
-// end subroutine test
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
-  %c4 = arith.constant 4 : index
-  %c1 = arith.constant 1 : index
-  %c6 = arith.constant 6 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-  %6 = hlfir.minval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
-  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
-  return
-}
-
-// CHECK-LABEL: _QPtest_float
-// CHECK:       %cst = arith.constant 0x7F800000 : f32
-// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
-// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
-// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
-// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
-// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
-// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
-// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
-// CHECK-NEXT:      fir.result %[[V14]] : f32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
new file mode 100644
index 0000000000000..fb0f39811ea74
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
@@ -0,0 +1,123 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+  %0 = hlfir.all %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_14:.*]] = arith.andi %[[VAL_13]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_14]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_15]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.all %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
+// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = arith.andi %[[VAL_15]], %[[VAL_13]] : i1
+// CHECK:               fir.result %[[VAL_16]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
+// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+  %0 = hlfir.all %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant true
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
+// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_20]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_21]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_22]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.all %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant true
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
+// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_27:.*]] = arith.andi %[[VAL_26]], %[[VAL_14]] : i1
+// CHECK:               fir.result %[[VAL_27]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
+// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
new file mode 100644
index 0000000000000..5bd76f3d24927
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
@@ -0,0 +1,123 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+  %0 = hlfir.any %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_14:.*]] = arith.ori %[[VAL_13]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_14]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_15]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.any %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
+// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = arith.ori %[[VAL_15]], %[[VAL_13]] : i1
+// CHECK:               fir.result %[[VAL_16]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
+// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+  %0 = hlfir.any %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
+// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_21:.*]] = arith.ori %[[VAL_20]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_21]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_22]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.any %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
+// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_14]] : i1
+// CHECK:               fir.result %[[VAL_27]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
+// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
new file mode 100644
index 0000000000000..44594c646a368
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
@@ -0,0 +1,127 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
+  %0 = hlfir.count %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (i32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
+// CHECK:               %[[VAL_16:.*]] = arith.select %[[VAL_14]], %[[VAL_15]], %[[VAL_12]] : i32
+// CHECK:               fir.result %[[VAL_16]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : i32
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.count %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?xi16>
+  return %0 : !hlfir.expr<?x?xi16>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i16
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]], %[[VAL_7]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi16> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (i16) {
+// CHECK:               %[[VAL_15:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_13]], %[[VAL_11]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_14]], %[[VAL_1]] : i16
+// CHECK:               %[[VAL_18:.*]] = arith.select %[[VAL_16]], %[[VAL_17]], %[[VAL_14]] : i16
+// CHECK:               fir.result %[[VAL_18]] : i16
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : i16
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?xi16>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
+  %0 = hlfir.count %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (i32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_15:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.addi %[[VAL_11]], %[[VAL_15]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_18:.*]] = arith.addi %[[VAL_8]], %[[VAL_17]] : index
+// CHECK:               %[[VAL_19:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_16]], %[[VAL_18]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_21]], %[[VAL_22]], %[[VAL_12]] : i32
+// CHECK:               fir.result %[[VAL_23]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : i32
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.count %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?xi64>
+  return %0 : !hlfir.expr<?x?xi64>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_6]]#1, %[[VAL_8]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi64> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):
+// CHECK:             %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_15:.*]] = %[[VAL_2]]) -> (i64) {
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_14]], %[[VAL_21]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_24:.*]] = arith.addi %[[VAL_12]], %[[VAL_23]] : index
+// CHECK:               %[[VAL_25:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_28:.*]] = arith.addi %[[VAL_15]], %[[VAL_1]] : i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_15]] : i64
+// CHECK:               fir.result %[[VAL_29]] : i64
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_13]] : i64
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?x?xi64>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
new file mode 100644
index 0000000000000..49e3bb6936338
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
@@ -0,0 +1,343 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_1d_total_expr(
+// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
+// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -2147483648 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
+// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
+// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_26:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
+// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_28]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
+// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
+// CHECK:         }
+
+func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+  %dim = arith.constant 1 : i16
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_1d_dim_expr(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
+// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
+// CHECK:               %[[VAL_18:.*]] = arith.cmpf ogt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
+// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
+// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
+// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
+// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
+// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_29:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (index) -> i32
+// CHECK:           return %[[VAL_30]] : i32
+// CHECK:         }
+
+func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  return %0 : !hlfir.expr<1xi16>
+}
+// CHECK-LABEL:   func.func @test_1d_total_var(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf ogt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
+// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
+// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
+// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
+// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_34:.*]]#0, %[[VAL_6]] : i16
+// CHECK:           %[[VAL_35:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_34]]#0 : (i16) -> index
+// CHECK:           %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_35]]#0 : index
+// CHECK:           %[[VAL_38:.*]] = arith.subi %[[VAL_37]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_33]], %[[VAL_38]], %[[VAL_36]] : index
+// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (index) -> i16
+// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i16, !fir.ref<i16>
+// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
+// CHECK:         }
+
+func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
+  return %0 : i64
+}
+// CHECK-LABEL:   func.func @test_1d_dim_var(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
+// CHECK:               %[[VAL_22:.*]] = arith.cmpf ogt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
+// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
+// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
+// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
+// CHECK:           }
+// CHECK:           %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_33:.*]]#0, %[[VAL_6]] : i64
+// CHECK:           %[[VAL_34:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]]#0 : (i64) -> index
+// CHECK:           %[[VAL_36:.*]] = arith.addi %[[VAL_35]], %[[VAL_34]]#0 : index
+// CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_36]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_32]], %[[VAL_37]], %[[VAL_35]] : index
+// CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
+// CHECK:           return %[[VAL_39]] : i64
+// CHECK:         }
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
+  return %0 : !hlfir.expr<3xi32>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant true
+// CHECK:           %[[VAL_7:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
+// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
+// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
+// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
+// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
+// CHECK:                   %[[VAL_39:.*]] = arith.cmpf ogt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
+// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
+// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
+// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
+// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
+// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
+// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
+// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
+// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
+// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
+// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
+// CHECK:                 } else {
+// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
+// CHECK:                 }
+// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_56:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_57:.*]] = fir.convert %[[VAL_55]] : (index) -> i32
+// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_57]] to %[[VAL_58]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_59:.*]] = fir.convert %[[VAL_56]]#1 : (i32) -> index
+// CHECK:           %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (index) -> i32
+// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_60]] to %[[VAL_61]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_62:.*]] = fir.convert %[[VAL_56]]#2 : (i32) -> index
+// CHECK:           %[[VAL_63:.*]] = fir.convert %[[VAL_62]] : (index) -> i32
+// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_63]] to %[[VAL_64]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
+// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
+  return %0 : !hlfir.expr<?x?xi32>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
+// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
+// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
+// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
+// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
+// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
+// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
+// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
+// CHECK:               }
+// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
+// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
+// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
+// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
+// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_48:.*]] = arith.cmpf ogt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
+// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
+// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
+// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
+// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
+// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
+// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
+// CHECK:             }
+// CHECK:             %[[VAL_58:.*]] = arith.cmpi ne, %[[VAL_59:.*]]#0, %[[VAL_5]] : i32
+// CHECK:             %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_59]]#0 : (i32) -> index
+// CHECK:             %[[VAL_62:.*]] = arith.addi %[[VAL_61]], %[[VAL_60]]#0 : index
+// CHECK:             %[[VAL_63:.*]] = arith.subi %[[VAL_62]], %[[VAL_7]] : index
+// CHECK:             %[[VAL_64:.*]] = arith.select %[[VAL_58]], %[[VAL_63]], %[[VAL_61]] : index
+// CHECK:             %[[VAL_65:.*]] = fir.convert %[[VAL_64]] : (index) -> i32
+// CHECK:             hlfir.yield_element %[[VAL_65]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
+// CHECK:         }
+
+// Character comparisons are not supported yet.
+func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.maxloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_character(
+// CHECK:           hlfir.maxloc
+
+// BACK is not supported yet.
+func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
+  %back = arith.constant true
+  %0 = hlfir.maxloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_back(
+// CHECK:           hlfir.maxloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
new file mode 100644
index 0000000000000..4022b7d3b2b77
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
@@ -0,0 +1,177 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
+  return %0 : f32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (f32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f32) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f32
+// CHECK:                 fir.result %[[VAL_22]] : f32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]] : f32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : f32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : f32
+// CHECK:         }
+
+func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
+  return %0 : !hlfir.expr<?xf64>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant -1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f64) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f64) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f64
+// CHECK:                 fir.result %[[VAL_22]] : f64
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f64
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]] : f64
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_10]] : f64
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
+// CHECK:         }
+
+func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
+  return %0 : f16
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant -6.550400e+04 : f16
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (f16) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f16) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f16) {
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
+// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_25:.*]] = arith.cmpf ogt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
+// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
+// CHECK:                 %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_24]], %[[VAL_12]] : f16
+// CHECK:                 fir.result %[[VAL_30]] : f16
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f16
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_16]] : f16
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : f16
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
+  return %0 : !hlfir.expr<?xf16>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant -6.550400e+04 : f16
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (f16) {
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
+// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
+// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
+// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
+// CHECK:               }
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (f16) {
+// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
+// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_35:.*]] = arith.cmpf ogt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
+// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
+// CHECK:                 %[[VAL_40:.*]] = arith.select %[[VAL_39]], %[[VAL_34]], %[[VAL_14]] : f16
+// CHECK:                 fir.result %[[VAL_40]] : f16
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_14]] : f16
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_26]] : f16
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
new file mode 100644
index 0000000000000..96dcebaf9a2cd
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
@@ -0,0 +1,343 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_1d_total_expr(
+// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
+// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
+// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
+// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_26:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
+// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_28]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
+// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
+// CHECK:         }
+
+func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+  %dim = arith.constant 1 : i16
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_1d_dim_expr(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
+// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
+// CHECK:               %[[VAL_18:.*]] = arith.cmpf olt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
+// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
+// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
+// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
+// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
+// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_29:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (index) -> i32
+// CHECK:           return %[[VAL_30]] : i32
+// CHECK:         }
+
+func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  return %0 : !hlfir.expr<1xi16>
+}
+// CHECK-LABEL:   func.func @test_1d_total_var(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf olt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
+// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
+// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
+// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
+// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_34:.*]]#0, %[[VAL_6]] : i16
+// CHECK:           %[[VAL_35:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_34]]#0 : (i16) -> index
+// CHECK:           %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_35]]#0 : index
+// CHECK:           %[[VAL_38:.*]] = arith.subi %[[VAL_37]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_33]], %[[VAL_38]], %[[VAL_36]] : index
+// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (index) -> i16
+// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i16, !fir.ref<i16>
+// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
+// CHECK:         }
+
+func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
+  return %0 : i64
+}
+// CHECK-LABEL:   func.func @test_1d_dim_var(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
+// CHECK:               %[[VAL_22:.*]] = arith.cmpf olt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
+// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
+// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
+// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
+// CHECK:           }
+// CHECK:           %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_33:.*]]#0, %[[VAL_6]] : i64
+// CHECK:           %[[VAL_34:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]]#0 : (i64) -> index
+// CHECK:           %[[VAL_36:.*]] = arith.addi %[[VAL_35]], %[[VAL_34]]#0 : index
+// CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_36]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_32]], %[[VAL_37]], %[[VAL_35]] : index
+// CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
+// CHECK:           return %[[VAL_39]] : i64
+// CHECK:         }
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
+  return %0 : !hlfir.expr<3xi32>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant true
+// CHECK:           %[[VAL_7:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
+// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
+// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
+// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
+// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
+// CHECK:                   %[[VAL_39:.*]] = arith.cmpf olt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
+// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
+// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
+// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
+// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
+// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
+// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
+// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
+// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
+// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
+// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
+// CHECK:                 } else {
+// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
+// CHECK:                 }
+// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_56:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_57:.*]] = fir.convert %[[VAL_55]] : (index) -> i32
+// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_57]] to %[[VAL_58]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_59:.*]] = fir.convert %[[VAL_56]]#1 : (i32) -> index
+// CHECK:           %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (index) -> i32
+// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_60]] to %[[VAL_61]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_62:.*]] = fir.convert %[[VAL_56]]#2 : (i32) -> index
+// CHECK:           %[[VAL_63:.*]] = fir.convert %[[VAL_62]] : (index) -> i32
+// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_63]] to %[[VAL_64]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
+// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
+  return %0 : !hlfir.expr<?x?xi32>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
+// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
+// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
+// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
+// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
+// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
+// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
+// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
+// CHECK:               }
+// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
+// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
+// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
+// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
+// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_48:.*]] = arith.cmpf olt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
+// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
+// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
+// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
+// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
+// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
+// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
+// CHECK:             }
+// CHECK:             %[[VAL_58:.*]] = arith.cmpi ne, %[[VAL_59:.*]]#0, %[[VAL_5]] : i32
+// CHECK:             %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_59]]#0 : (i32) -> index
+// CHECK:             %[[VAL_62:.*]] = arith.addi %[[VAL_61]], %[[VAL_60]]#0 : index
+// CHECK:             %[[VAL_63:.*]] = arith.subi %[[VAL_62]], %[[VAL_7]] : index
+// CHECK:             %[[VAL_64:.*]] = arith.select %[[VAL_58]], %[[VAL_63]], %[[VAL_61]] : index
+// CHECK:             %[[VAL_65:.*]] = fir.convert %[[VAL_64]] : (index) -> i32
+// CHECK:             hlfir.yield_element %[[VAL_65]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
+// CHECK:         }
+
+// Character comparisons are not supported yet.
+func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.minloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_character(
+// CHECK:           hlfir.minloc
+
+// BACK is not supported yet.
+func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
+  %back = arith.constant true
+  %0 = hlfir.minloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_back(
+// CHECK:           hlfir.minloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
new file mode 100644
index 0000000000000..4278ac7abd457
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
@@ -0,0 +1,177 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
+  return %0 : f32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (f32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f32) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f32
+// CHECK:                 fir.result %[[VAL_22]] : f32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]] : f32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : f32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : f32
+// CHECK:         }
+
+func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
+  return %0 : !hlfir.expr<?xf64>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f64) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f64) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f64
+// CHECK:                 fir.result %[[VAL_22]] : f64
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f64
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]] : f64
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_10]] : f64
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
+// CHECK:         }
+
+func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
+  return %0 : f16
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 6.550400e+04 : f16
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (f16) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f16) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f16) {
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
+// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_25:.*]] = arith.cmpf olt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
+// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
+// CHECK:                 %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_24]], %[[VAL_12]] : f16
+// CHECK:                 fir.result %[[VAL_30]] : f16
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f16
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_16]] : f16
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : f16
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
+  return %0 : !hlfir.expr<?xf16>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant 6.550400e+04 : f16
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (f16) {
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
+// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
+// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
+// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
+// CHECK:               }
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (f16) {
+// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
+// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_35:.*]] = arith.cmpf olt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
+// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
+// CHECK:                 %[[VAL_40:.*]] = arith.select %[[VAL_39]], %[[VAL_34]], %[[VAL_14]] : f16
+// CHECK:                 fir.result %[[VAL_40]] : f16
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_14]] : f16
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_26]] : f16
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
+// CHECK:         }

From 9af61f500c438df9bcbe2eab47c89d25cc5db759 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Thu, 17 Apr 2025 15:43:57 -0700
Subject: [PATCH 334/710] Revert "[Fuchsia] Not building llvm-mt when LIBXML2
 is not enabled." (#136216)

Reverts llvm/llvm-project#135877

This is causing some problems on Fuchsia's windows CI. We'll need a
different solution to triage other builders.


https://ci.chromium.org/ui/p/fuchsia/builders/toolchain.ci/clang-windows-x64/b8717476961063994817/overview
---
 clang/cmake/caches/Fuchsia-stage2.cmake | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index e10855f5ef31b..99890b8246ad7 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -459,6 +459,7 @@ set(LLVM_TOOLCHAIN_TOOLS
   llvm-libtool-darwin
   llvm-lipo
   llvm-ml
+  llvm-mt
   llvm-nm
   llvm-objcopy
   llvm-objdump
@@ -480,10 +481,6 @@ set(LLVM_TOOLCHAIN_TOOLS
   scan-build-py
   CACHE STRING "")
 
-if (LLVM_ENABLE_LIBXML2)
-  list(APPEND LLVM_TOOLCHAIN_TOOLS llvm-mt)
-endif()
-
 set(LLVM_Toolchain_DISTRIBUTION_COMPONENTS
   bolt
   clang

From f0f55453a2df8c984a7a824bb3face2d7ae18a7d Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Thu, 17 Apr 2025 15:45:09 -0700
Subject: [PATCH 335/710] [NVPTX] Address warning and typo in fp128 support
 (#136207)

---
 llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h       | 3 +--
 llvm/test/CodeGen/NVPTX/fp128-storage-type.ll | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
index aa953f4596bbd..65938a9f60cd7 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
@@ -125,8 +125,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
     }
 
     void addZeros(unsigned Num) {
-      for (unsigned _ : llvm::seq(Num)) {
-        (void)_;
+      for ([[maybe_unused]] unsigned _ : llvm::seq(Num)) {
         addByte(0);
       }
     }
diff --git a/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll b/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll
index 5b96f4978a7cb..9a16c11300c70 100644
--- a/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll
+++ b/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mcpu=sm_20 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s-mcpu=sm_20 | %ptxas-verify %}
+; RUN: %if ptxas %{ llc < %s -mcpu=sm_20 | %ptxas-verify %}
 
 target triple = "nvptx64-unknown-cuda"
 

From 767c8c1badc03881aa97ed145ed8f0335c5a1d17 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Thu, 17 Apr 2025 15:46:42 -0700
Subject: [PATCH 336/710] [docs][SandboxIR] Fix cross-reference to sandbox
 vectorizer (#133768)

The cross-reference used to point to a label in the auto-vectorization
document.
---
 llvm/docs/SandboxIR.md    | 2 +-
 llvm/docs/Vectorizers.rst | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/docs/SandboxIR.md b/llvm/docs/SandboxIR.md
index b06c4c89c5b4f..d2b612ba95ef1 100644
--- a/llvm/docs/SandboxIR.md
+++ b/llvm/docs/SandboxIR.md
@@ -106,4 +106,4 @@ Please note that after a call to `revert()` or `accept()` tracking will stop.
 To start tracking again, the user needs to call `save()`.
 
 # Users of Sandbox IR
-- [The Sandbox Vectorizer](sandbox-vectorizer)
+- [The Sandbox Vectorizer](project:SandboxVectorizer.md)
diff --git a/llvm/docs/Vectorizers.rst b/llvm/docs/Vectorizers.rst
index c65b6ac617508..c7bf9133e146b 100644
--- a/llvm/docs/Vectorizers.rst
+++ b/llvm/docs/Vectorizers.rst
@@ -480,8 +480,6 @@ through clang using the command line flag:
 
    $ clang -fno-slp-vectorize file.c
 
-.. _sandbox-vectorizer:
-
 The Sandbox Vectorizer
 ======================
 .. toctree::

From f39242ceed214dfcecd6b33f22f8301ae183c677 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 17 Apr 2025 15:47:46 -0700
Subject: [PATCH 337/710] Revert "[flang] Generalized simplification of HLFIR
 reduction ops." (#136218)

Reverts llvm/llvm-project#136071
---
 .../flang/Optimizer/Builder/HLFIRTools.h      |    5 -
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |   27 -
 .../Transforms/OptimizedBufferization.cpp     |  465 ++++++++
 .../Transforms/SimplifyHLFIRIntrinsics.cpp    | 1039 ++++-------------
 flang/test/HLFIR/all-elemental.fir            |   91 ++
 flang/test/HLFIR/any-elemental.fir            |  190 +++
 flang/test/HLFIR/count-elemental.fir          |  314 +++++
 flang/test/HLFIR/maxloc-elemental.fir         |  133 +++
 flang/test/HLFIR/maxval-elemental.fir         |  117 ++
 flang/test/HLFIR/minloc-elemental.fir         |  397 +++++++
 flang/test/HLFIR/minval-elemental.fir         |   95 ++
 .../HLFIR/simplify-hlfir-intrinsics-all.fir   |  123 --
 .../HLFIR/simplify-hlfir-intrinsics-any.fir   |  123 --
 .../HLFIR/simplify-hlfir-intrinsics-count.fir |  127 --
 .../simplify-hlfir-intrinsics-maxloc.fir      |  343 ------
 .../simplify-hlfir-intrinsics-maxval.fir      |  177 ---
 .../simplify-hlfir-intrinsics-minloc.fir      |  343 ------
 .../simplify-hlfir-intrinsics-minval.fir      |  177 ---
 18 files changed, 1999 insertions(+), 2287 deletions(-)
 create mode 100644 flang/test/HLFIR/all-elemental.fir
 create mode 100644 flang/test/HLFIR/any-elemental.fir
 create mode 100644 flang/test/HLFIR/count-elemental.fir
 create mode 100644 flang/test/HLFIR/maxloc-elemental.fir
 create mode 100644 flang/test/HLFIR/maxval-elemental.fir
 create mode 100644 flang/test/HLFIR/minloc-elemental.fir
 create mode 100644 flang/test/HLFIR/minval-elemental.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir

diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index cd259b9dc6071..ac80873dc374f 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -301,11 +301,6 @@ mlir::Value genExtent(mlir::Location loc, fir::FirOpBuilder &builder,
 mlir::Value genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
                       hlfir::Entity entity, unsigned dim);
 
-/// Compute the lower bounds of \p entity, which is an array of known rank.
-llvm::SmallVector<mlir::Value> genLBounds(mlir::Location loc,
-                                          fir::FirOpBuilder &builder,
-                                          hlfir::Entity entity);
-
 /// Generate a vector of extents with index type from a fir.shape
 /// of fir.shape_shift value.
 llvm::SmallVector<mlir::Value> getIndexExtents(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 2a5e136c57c62..558ebcb876ddb 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -659,33 +659,6 @@ mlir::Value hlfir::genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
   return dimInfo.getLowerBound();
 }
 
-llvm::SmallVector<mlir::Value> hlfir::genLBounds(mlir::Location loc,
-                                                 fir::FirOpBuilder &builder,
-                                                 hlfir::Entity entity) {
-  assert(!entity.isAssumedRank() &&
-         "cannot compute all lower bounds for assumed rank");
-  assert(!entity.isScalar() && "expected an array entity");
-  int rank = entity.getRank();
-  mlir::Type idxTy = builder.getIndexType();
-  if (!entity.mayHaveNonDefaultLowerBounds())
-    return {static_cast<std::size_t>(rank),
-            builder.createIntegerConstant(loc, idxTy, 1)};
-
-  if (auto shape = tryRetrievingShapeOrShift(entity)) {
-    auto lbounds = getExplicitLboundsFromShape(shape);
-    if (!lbounds.empty())
-      return lbounds;
-  }
-
-  if (entity.isMutableBox())
-    entity = hlfir::derefPointersAndAllocatables(loc, builder, entity);
-
-  llvm::SmallVector<mlir::Value, Fortran::common::maxRank> lbounds;
-  fir::factory::genDimInfoFromBox(builder, loc, entity, &lbounds,
-                                  /*extents=*/nullptr, /*strides=*/nullptr);
-  return lbounds;
-}
-
 void hlfir::genLengthParameters(mlir::Location loc, fir::FirOpBuilder &builder,
                                 Entity entity,
                                 llvm::SmallVectorImpl<mlir::Value> &result) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 79aabd2981e1a..c489450384a35 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -772,6 +772,458 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   return mlir::success();
 }
 
+using GenBodyFn =
+    std::function<mlir::Value(fir::FirOpBuilder &, mlir::Location, mlir::Value,
+                              const llvm::SmallVectorImpl<mlir::Value> &)>;
+static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
+                                         mlir::Location loc, mlir::Value init,
+                                         mlir::Value shape, GenBodyFn genBody) {
+  auto extents = hlfir::getIndexExtents(loc, builder, shape);
+  mlir::Value reduction = init;
+  mlir::IndexType idxTy = builder.getIndexType();
+  mlir::Value oneIdx = builder.createIntegerConstant(loc, idxTy, 1);
+
+  // Create a reduction loop nest. We use one-based indices so that they can be
+  // passed to the elemental, and reverse the order so that they can be
+  // generated in column-major order for better performance.
+  llvm::SmallVector<mlir::Value> indices(extents.size(), mlir::Value{});
+  for (unsigned i = 0; i < extents.size(); ++i) {
+    auto loop = builder.create<fir::DoLoopOp>(
+        loc, oneIdx, extents[extents.size() - i - 1], oneIdx, false,
+        /*finalCountValue=*/false, reduction);
+    reduction = loop.getRegionIterArgs()[0];
+    indices[extents.size() - i - 1] = loop.getInductionVar();
+    // Set insertion point to the loop body so that the next loop
+    // is inserted inside the current one.
+    builder.setInsertionPointToStart(loop.getBody());
+  }
+
+  // Generate the body
+  reduction = genBody(builder, loc, reduction, indices);
+
+  // Unwind the loop nest.
+  for (unsigned i = 0; i < extents.size(); ++i) {
+    auto result = builder.create<fir::ResultOp>(loc, reduction);
+    auto loop = mlir::cast<fir::DoLoopOp>(result->getParentOp());
+    reduction = loop.getResult(0);
+    // Set insertion point after the loop operation that we have
+    // just processed.
+    builder.setInsertionPointAfter(loop.getOperation());
+  }
+
+  return reduction;
+}
+
+auto makeMinMaxInitValGenerator(bool isMax) {
+  return [isMax](fir::FirOpBuilder builder, mlir::Location loc,
+                 mlir::Type elementType) -> mlir::Value {
+    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
+      const llvm::fltSemantics &sem = ty.getFloatSemantics();
+      llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
+      return builder.createRealConstant(loc, elementType, limit);
+    }
+    unsigned bits = elementType.getIntOrFloatBitWidth();
+    int64_t limitInt =
+        isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
+              : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
+    return builder.createIntegerConstant(loc, elementType, limitInt);
+  };
+}
+
+mlir::Value generateMinMaxComparison(fir::FirOpBuilder builder,
+                                     mlir::Location loc, mlir::Value elem,
+                                     mlir::Value reduction, bool isMax) {
+  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
+    // For FP reductions we want the first smallest value to be used, that
+    // is not NaN. A OGL/OLT condition will usually work for this unless all
+    // the values are Nan or Inf. This follows the same logic as
+    // NumericCompare for Minloc/Maxlox in extrema.cpp.
+    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
+        loc,
+        isMax ? mlir::arith::CmpFPredicate::OGT
+              : mlir::arith::CmpFPredicate::OLT,
+        elem, reduction);
+    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
+    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
+    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
+    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
+    return builder.create<mlir::arith::CmpIOp>(
+        loc,
+        isMax ? mlir::arith::CmpIPredicate::sgt
+              : mlir::arith::CmpIPredicate::slt,
+        elem, reduction);
+  }
+  llvm_unreachable("unsupported type");
+}
+
+/// Given a reduction operation with an elemental/designate source, attempt to
+/// generate a do-loop to perform the operation inline.
+///   %e = hlfir.elemental %shape unordered
+///   %r = hlfir.count %e
+/// =>
+///   %r = for.do_loop %arg = 1 to bound(%shape) step 1 iter_args(%arg2 = init)
+///     %i = <inline elemental>
+///     %c = <reduce count> %i
+///     fir.result %c
+template <typename Op>
+class ReductionConversion : public mlir::OpRewritePattern<Op> {
+public:
+  using mlir::OpRewritePattern<Op>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
+    mlir::Location loc = op.getLoc();
+    // Select source and validate its arguments.
+    mlir::Value source;
+    bool valid = false;
+    if constexpr (std::is_same_v<Op, hlfir::AnyOp> ||
+                  std::is_same_v<Op, hlfir::AllOp> ||
+                  std::is_same_v<Op, hlfir::CountOp>) {
+      source = op.getMask();
+      valid = !op.getDim();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      source = op.getArray();
+      valid = !op.getDim() && !op.getMask();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                         std::is_same_v<Op, hlfir::MinlocOp>) {
+      source = op.getArray();
+      valid = !op.getDim() && !op.getMask() && !op.getBack();
+    }
+    if (!valid)
+      return rewriter.notifyMatchFailure(
+          op, "Currently does not accept optional arguments");
+
+    hlfir::ElementalOp elemental;
+    hlfir::DesignateOp designate;
+    mlir::Value shape;
+    if ((elemental = source.template getDefiningOp<hlfir::ElementalOp>())) {
+      shape = elemental.getOperand(0);
+    } else if ((designate =
+                    source.template getDefiningOp<hlfir::DesignateOp>())) {
+      shape = designate.getShape();
+    } else {
+      return rewriter.notifyMatchFailure(op, "Did not find valid argument");
+    }
+
+    auto inlineSource =
+        [elemental,
+         &designate](fir::FirOpBuilder builder, mlir::Location loc,
+                     const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+        -> mlir::Value {
+      if (elemental) {
+        // Inline the elemental and get the value from it.
+        auto yield =
+            inlineElementalOp(loc, builder, elemental, oneBasedIndices);
+        auto tmp = yield.getElementValue();
+        yield->erase();
+        return tmp;
+      }
+      if (designate) {
+        // Create a designator over the array designator, then load the
+        // reference.
+        mlir::Value elementAddr = hlfir::getElementAt(
+            loc, builder, hlfir::Entity{designate.getResult()},
+            oneBasedIndices);
+        return builder.create<fir::LoadOp>(loc, elementAddr);
+      }
+      llvm_unreachable("unsupported type");
+    };
+
+    fir::FirOpBuilder builder{rewriter, op.getOperation()};
+
+    mlir::Value init;
+    GenBodyFn genBodyFn;
+    if constexpr (std::is_same_v<Op, hlfir::AnyOp>) {
+      init = builder.createIntegerConstant(loc, builder.getI1Type(), 0);
+      genBodyFn = [inlineSource](
+                      fir::FirOpBuilder builder, mlir::Location loc,
+                      mlir::Value reduction,
+                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+          -> mlir::Value {
+        // Conditionally set the reduction variable.
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(),
+            inlineSource(builder, loc, oneBasedIndices));
+        return builder.create<mlir::arith::OrIOp>(loc, reduction, cond);
+      };
+    } else if constexpr (std::is_same_v<Op, hlfir::AllOp>) {
+      init = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
+      genBodyFn = [inlineSource](
+                      fir::FirOpBuilder builder, mlir::Location loc,
+                      mlir::Value reduction,
+                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+          -> mlir::Value {
+        // Conditionally set the reduction variable.
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(),
+            inlineSource(builder, loc, oneBasedIndices));
+        return builder.create<mlir::arith::AndIOp>(loc, reduction, cond);
+      };
+    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
+      init = builder.createIntegerConstant(loc, op.getType(), 0);
+      genBodyFn = [inlineSource](
+                      fir::FirOpBuilder builder, mlir::Location loc,
+                      mlir::Value reduction,
+                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+          -> mlir::Value {
+        // Conditionally add one to the current value
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(),
+            inlineSource(builder, loc, oneBasedIndices));
+        mlir::Value one =
+            builder.createIntegerConstant(loc, reduction.getType(), 1);
+        mlir::Value add1 =
+            builder.create<mlir::arith::AddIOp>(loc, reduction, one);
+        return builder.create<mlir::arith::SelectOp>(loc, cond, add1,
+                                                     reduction);
+      };
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                         std::is_same_v<Op, hlfir::MinlocOp>) {
+      // TODO: implement minloc/maxloc conversion.
+      return rewriter.notifyMatchFailure(
+          op, "Currently minloc/maxloc is not handled");
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      mlir::Type ty = op.getType();
+      if (!(mlir::isa<mlir::FloatType>(ty) ||
+            mlir::isa<mlir::IntegerType>(ty))) {
+        return rewriter.notifyMatchFailure(
+            op, "Type is not supported for Maxval or Minval yet");
+      }
+
+      bool isMax = std::is_same_v<Op, hlfir::MaxvalOp>;
+      init = makeMinMaxInitValGenerator(isMax)(builder, loc, ty);
+      genBodyFn = [inlineSource, isMax](
+                      fir::FirOpBuilder builder, mlir::Location loc,
+                      mlir::Value reduction,
+                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+          -> mlir::Value {
+        mlir::Value val = inlineSource(builder, loc, oneBasedIndices);
+        mlir::Value cmp =
+            generateMinMaxComparison(builder, loc, val, reduction, isMax);
+        return builder.create<mlir::arith::SelectOp>(loc, cmp, val, reduction);
+      };
+    } else {
+      llvm_unreachable("unsupported type");
+    }
+
+    mlir::Value res =
+        generateReductionLoop(builder, loc, init, shape, genBodyFn);
+    if (res.getType() != op.getType())
+      res = builder.create<fir::ConvertOp>(loc, op.getType(), res);
+
+    // Check if the op was the only user of the source (apart from a destroy),
+    // and remove it if so.
+    mlir::Operation *sourceOp = source.getDefiningOp();
+    mlir::Operation::user_range srcUsers = sourceOp->getUsers();
+    hlfir::DestroyOp srcDestroy;
+    if (std::distance(srcUsers.begin(), srcUsers.end()) == 2) {
+      srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*srcUsers.begin());
+      if (!srcDestroy)
+        srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++srcUsers.begin());
+    }
+
+    rewriter.replaceOp(op, res);
+    if (srcDestroy) {
+      rewriter.eraseOp(srcDestroy);
+      rewriter.eraseOp(sourceOp);
+    }
+    return mlir::success();
+  }
+};
+
+// Look for minloc(mask=elemental) and generate the minloc loop with
+// inlined elemental.
+//  %e = hlfir.elemental %shape ({ ... })
+//  %m = hlfir.minloc %array mask %e
+template <typename Op>
+class ReductionMaskConversion : public mlir::OpRewritePattern<Op> {
+public:
+  using mlir::OpRewritePattern<Op>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(Op mloc, mlir::PatternRewriter &rewriter) const override {
+    if (!mloc.getMask() || mloc.getDim() || mloc.getBack())
+      return rewriter.notifyMatchFailure(mloc,
+                                         "Did not find valid minloc/maxloc");
+
+    bool isMax = std::is_same_v<Op, hlfir::MaxlocOp>;
+
+    auto elemental =
+        mloc.getMask().template getDefiningOp<hlfir::ElementalOp>();
+    if (!elemental || hlfir::elementalOpMustProduceTemp(elemental))
+      return rewriter.notifyMatchFailure(mloc, "Did not find elemental");
+
+    mlir::Value array = mloc.getArray();
+
+    unsigned rank = mlir::cast<hlfir::ExprType>(mloc.getType()).getShape()[0];
+    mlir::Type arrayType = array.getType();
+    if (!mlir::isa<fir::BoxType>(arrayType))
+      return rewriter.notifyMatchFailure(
+          mloc, "Currently requires a boxed type input");
+    mlir::Type elementType = hlfir::getFortranElementType(arrayType);
+    if (!fir::isa_trivial(elementType))
+      return rewriter.notifyMatchFailure(
+          mloc, "Character arrays are currently not handled");
+
+    mlir::Location loc = mloc.getLoc();
+    fir::FirOpBuilder builder{rewriter, mloc.getOperation()};
+    mlir::Value resultArr = builder.createTemporary(
+        loc, fir::SequenceType::get(
+                 rank, hlfir::getFortranElementType(mloc.getType())));
+
+    auto init = makeMinMaxInitValGenerator(isMax);
+
+    auto genBodyOp =
+        [&rank, &resultArr, &elemental, isMax](
+            fir::FirOpBuilder builder, mlir::Location loc,
+            mlir::Type elementType, mlir::Value array, mlir::Value flagRef,
+            mlir::Value reduction,
+            const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {
+      // We are in the innermost loop: generate the elemental inline
+      mlir::Value oneIdx =
+          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
+      llvm::SmallVector<mlir::Value> oneBasedIndices;
+      llvm::transform(
+          indices, std::back_inserter(oneBasedIndices), [&](mlir::Value V) {
+            return builder.create<mlir::arith::AddIOp>(loc, V, oneIdx);
+          });
+      hlfir::YieldElementOp yield =
+          hlfir::inlineElementalOp(loc, builder, elemental, oneBasedIndices);
+      mlir::Value maskElem = yield.getElementValue();
+      yield->erase();
+
+      mlir::Type ifCompatType = builder.getI1Type();
+      mlir::Value ifCompatElem =
+          builder.create<fir::ConvertOp>(loc, ifCompatType, maskElem);
+
+      llvm::SmallVector<mlir::Type> resultsTy = {elementType, elementType};
+      fir::IfOp maskIfOp =
+          builder.create<fir::IfOp>(loc, elementType, ifCompatElem,
+                                    /*withElseRegion=*/true);
+      builder.setInsertionPointToStart(&maskIfOp.getThenRegion().front());
+
+      // Set flag that mask was true at some point
+      mlir::Value flagSet = builder.createIntegerConstant(
+          loc, mlir::cast<fir::ReferenceType>(flagRef.getType()).getEleTy(), 1);
+      mlir::Value isFirst = builder.create<fir::LoadOp>(loc, flagRef);
+      mlir::Value addr = hlfir::getElementAt(loc, builder, hlfir::Entity{array},
+                                             oneBasedIndices);
+      mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
+
+      // Compare with the max reduction value
+      mlir::Value cmp =
+          generateMinMaxComparison(builder, loc, elem, reduction, isMax);
+
+      // The condition used for the loop is isFirst || <the condition above>.
+      isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
+      isFirst = builder.create<mlir::arith::XOrIOp>(
+          loc, isFirst, builder.createIntegerConstant(loc, cmp.getType(), 1));
+      cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, isFirst);
+
+      // Set the new coordinate to the result
+      fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, cmp,
+                                                 /*withElseRegion*/ true);
+
+      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+      builder.create<fir::StoreOp>(loc, flagSet, flagRef);
+      mlir::Type resultElemTy =
+          hlfir::getFortranElementType(resultArr.getType());
+      mlir::Type returnRefTy = builder.getRefType(resultElemTy);
+      mlir::IndexType idxTy = builder.getIndexType();
+
+      for (unsigned int i = 0; i < rank; ++i) {
+        mlir::Value index = builder.createIntegerConstant(loc, idxTy, i + 1);
+        mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
+            loc, returnRefTy, resultArr, index);
+        mlir::Value fortranIndex = builder.create<fir::ConvertOp>(
+            loc, resultElemTy, oneBasedIndices[i]);
+        builder.create<fir::StoreOp>(loc, fortranIndex, resultElemAddr);
+      }
+      builder.create<fir::ResultOp>(loc, elem);
+      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+      builder.create<fir::ResultOp>(loc, reduction);
+      builder.setInsertionPointAfter(ifOp);
+
+      // Close the mask if
+      builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
+      builder.setInsertionPointToStart(&maskIfOp.getElseRegion().front());
+      builder.create<fir::ResultOp>(loc, reduction);
+      builder.setInsertionPointAfter(maskIfOp);
+
+      return maskIfOp.getResult(0);
+    };
+    auto getAddrFn = [](fir::FirOpBuilder builder, mlir::Location loc,
+                        const mlir::Type &resultElemType, mlir::Value resultArr,
+                        mlir::Value index) {
+      mlir::Type resultRefTy = builder.getRefType(resultElemType);
+      mlir::Value oneIdx =
+          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
+      index = builder.create<mlir::arith::AddIOp>(loc, index, oneIdx);
+      return builder.create<hlfir::DesignateOp>(loc, resultRefTy, resultArr,
+                                                index);
+    };
+
+    // Initialize the result
+    mlir::Type resultElemTy = hlfir::getFortranElementType(resultArr.getType());
+    mlir::Type resultRefTy = builder.getRefType(resultElemTy);
+    mlir::Value returnValue =
+        builder.createIntegerConstant(loc, resultElemTy, 0);
+    for (unsigned int i = 0; i < rank; ++i) {
+      mlir::Value index =
+          builder.createIntegerConstant(loc, builder.getIndexType(), i + 1);
+      mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
+          loc, resultRefTy, resultArr, index);
+      builder.create<fir::StoreOp>(loc, returnValue, resultElemAddr);
+    }
+
+    fir::genMinMaxlocReductionLoop(builder, array, init, genBodyOp, getAddrFn,
+                                   rank, elementType, loc, builder.getI1Type(),
+                                   resultArr, false);
+
+    mlir::Value asExpr = builder.create<hlfir::AsExprOp>(
+        loc, resultArr, builder.createBool(loc, false));
+
+    // Check all the users - the destroy is no longer required, and any assign
+    // can use resultArr directly so that InlineHLFIRAssign pass
+    // can optimize the results. Other operations are replaced with an AsExpr
+    // for the temporary resultArr.
+    llvm::SmallVector<hlfir::DestroyOp> destroys;
+    llvm::SmallVector<hlfir::AssignOp> assigns;
+    for (auto user : mloc->getUsers()) {
+      if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(user))
+        destroys.push_back(destroy);
+      else if (auto assign = mlir::dyn_cast<hlfir::AssignOp>(user))
+        assigns.push_back(assign);
+    }
+
+    // Check if the minloc/maxloc was the only user of the elemental (apart from
+    // a destroy), and remove it if so.
+    mlir::Operation::user_range elemUsers = elemental->getUsers();
+    hlfir::DestroyOp elemDestroy;
+    if (std::distance(elemUsers.begin(), elemUsers.end()) == 2) {
+      elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*elemUsers.begin());
+      if (!elemDestroy)
+        elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++elemUsers.begin());
+    }
+
+    for (auto d : destroys)
+      rewriter.eraseOp(d);
+    for (auto a : assigns)
+      a.setOperand(0, resultArr);
+    rewriter.replaceOp(mloc, asExpr);
+    if (elemDestroy) {
+      rewriter.eraseOp(elemDestroy);
+      rewriter.eraseOp(elemental);
+    }
+    return mlir::success();
+  }
+};
+
 class EvaluateIntoMemoryAssignBufferization
     : public mlir::OpRewritePattern<hlfir::EvaluateInMemoryOp> {
 
@@ -888,6 +1340,19 @@ class OptimizedBufferizationPass
     patterns.insert<ElementalAssignBufferization>(context);
     patterns.insert<BroadcastAssignBufferization>(context);
     patterns.insert<EvaluateIntoMemoryAssignBufferization>(context);
+    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
+    // TODO: implement basic minloc/maxloc conversion.
+    // patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
+    // patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
+    patterns.insert<ReductionMaskConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionMaskConversion<hlfir::MaxlocOp>>(context);
+    // TODO: implement masked minval/maxval conversion.
+    // patterns.insert<ReductionMaskConversion<hlfir::MaxvalOp>>(context);
+    // patterns.insert<ReductionMaskConversion<hlfir::MinvalOp>>(context);
 
     if (mlir::failed(mlir::applyPatternsGreedily(
             getOperation(), std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index d660cf3845f9a..bac10121a881b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -173,882 +173,245 @@ class TransposeAsElementalConversion
   }
 };
 
-/// Base class for converting reduction-like operations into
-/// a reduction loop[-nest] optionally wrapped into hlfir.elemental.
-/// It is used to handle operations produced for ALL, ANY, COUNT,
-/// MAXLOC, MAXVAL, MINLOC, MINVAL, SUM intrinsics.
-///
-/// All of these operations take an input array, and optional
-/// dim, mask arguments. ALL, ANY, COUNT do not have mask argument.
-class ReductionAsElementalConverter {
+// Expand the SUM(DIM=CONSTANT) operation into .
+class SumAsElementalConversion : public mlir::OpRewritePattern<hlfir::SumOp> {
 public:
-  ReductionAsElementalConverter(mlir::Operation *op,
-                                mlir::PatternRewriter &rewriter)
-      : op{op}, rewriter{rewriter}, loc{op->getLoc()}, builder{rewriter, op} {
-    assert(op->getNumResults() == 1);
-  }
-  virtual ~ReductionAsElementalConverter() {}
-
-  /// Do the actual conversion or return mlir::failure(),
-  /// if conversion is not possible.
-  mlir::LogicalResult convert();
+  using mlir::OpRewritePattern<hlfir::SumOp>::OpRewritePattern;
 
-private:
-  // Return fir.shape specifying the shape of the result
-  // of a reduction with DIM=dimVal. The second return value
-  // is the extent of the DIM dimension.
-  std::tuple<mlir::Value, mlir::Value>
-  genResultShapeForPartialReduction(hlfir::Entity array, int64_t dimVal);
-
-  /// \p mask is a scalar or array logical mask.
-  /// If \p isPresentPred is not nullptr, it is a dynamic predicate value
-  /// identifying whether the mask's variable is present.
-  /// \p indices is a range of one-based indices to access \p mask
-  /// when it is an array.
-  ///
-  /// The method returns the scalar mask value to guard the access
-  /// to a single element of the input array.
-  mlir::Value genMaskValue(mlir::Value mask, mlir::Value isPresentPred,
-                           mlir::ValueRange indices);
-
-protected:
-  /// Return the input array.
-  virtual mlir::Value getSource() const = 0;
-
-  /// Return DIM or nullptr, if it is not present.
-  virtual mlir::Value getDim() const = 0;
-
-  /// Return MASK or nullptr, if it is not present.
-  virtual mlir::Value getMask() const { return nullptr; }
-
-  /// Return FastMathFlags attached to the operation
-  /// or arith::FastMathFlags::none, if the operation
-  /// does not support FastMathFlags (e.g. ALL, ANY, COUNT).
-  virtual mlir::arith::FastMathFlags getFastMath() const {
-    return mlir::arith::FastMathFlags::none;
-  }
-
-  /// Generates initial values for the reduction values used
-  /// by the reduction loop. In general, there is a single
-  /// loop-carried reduction value (e.g. for SUM), but, for example,
-  /// MAXLOC/MINLOC implementation uses multiple reductions.
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() = 0;
-
-  /// Perform reduction(s) update given a single input array's element
-  /// identified by \p array and \p oneBasedIndices coordinates.
-  /// \p currentValue specifies the current value(s) of the reduction(s)
-  /// inside the reduction loop body.
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) = 0;
-
-  /// Given reduction value(s) in \p reductionResults produced
-  /// by the reduction loop, apply any required updates and return
-  /// new reduction value(s) to be used after the reduction loop
-  /// (e.g. as the result yield of the wrapping hlfir.elemental).
-  /// NOTE: if the reduction loop is wrapped in hlfir.elemental,
-  /// the insertion point of any generated code is inside hlfir.elemental.
-  virtual hlfir::Entity
-  genFinalResult(const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
-    assert(reductionResults.size() == 1 &&
-           "default implementation of genFinalResult expect a single reduction "
-           "value");
-    return hlfir::Entity{reductionResults[0]};
-  }
-
-  /// Return mlir::success(), if the operation can be converted.
-  /// The default implementation always returns mlir::success().
-  /// The derived type may override the default implementation
-  /// with its own definition.
-  virtual mlir::LogicalResult isConvertible() const { return mlir::success(); }
-
-  // Default implementation of isTotalReduction() just checks
-  // if the result of the operation is a scalar.
-  // True result indicates that the reduction has to be done
-  // across all elements, false result indicates that
-  // the result is an array expression produced by an hlfir.elemental
-  // operation with a single reduction loop across the DIM dimension.
-  //
-  // MAXLOC/MINLOC must override this.
-  virtual bool isTotalReduction() const { return getResultRank() == 0; }
-
-  // Return true, if the reduction loop[-nest] may be unordered.
-  // In general, FP reductions may only be unordered when
-  // FastMathFlags::reassoc transformations are allowed.
-  //
-  // Some dervied types may need to override this.
-  virtual bool isUnordered() const {
-    mlir::Type elemType = getSourceElementType();
-    if (mlir::isa<mlir::IntegerType, fir::LogicalType, fir::CharacterType>(
-            elemType))
-      return true;
-    return static_cast<bool>(getFastMath() &
-                             mlir::arith::FastMathFlags::reassoc);
-  }
-
-  /// Return 0, if DIM is not present or its values does not matter
-  /// (for example, a reduction of 1D array does not care about
-  /// the DIM value, assuming that it is a valid program).
-  /// Return mlir::failure(), if DIM is a constant known
-  /// to be invalid for the given array.
-  /// Otherwise, return DIM constant value.
-  mlir::FailureOr<int64_t> getConstDim() const {
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::SumOp sum,
+                  mlir::PatternRewriter &rewriter) const override {
+    hlfir::Entity array = hlfir::Entity{sum.getArray()};
+    bool isTotalReduction = hlfir::Entity{sum}.getRank() == 0;
+    mlir::Value dim = sum.getDim();
     int64_t dimVal = 0;
-    if (!isTotalReduction()) {
+    if (!isTotalReduction) {
       // In case of partial reduction we should ignore the operations
       // with invalid DIM values. They may appear in dead code
       // after constant propagation.
-      auto constDim = fir::getIntIfConstant(getDim());
+      auto constDim = fir::getIntIfConstant(dim);
       if (!constDim)
-        return rewriter.notifyMatchFailure(op, "Nonconstant DIM");
+        return rewriter.notifyMatchFailure(sum, "Nonconstant DIM for SUM");
       dimVal = *constDim;
 
-      if ((dimVal <= 0 || dimVal > getSourceRank()))
-        return rewriter.notifyMatchFailure(op,
-                                           "Invalid DIM for partial reduction");
+      if ((dimVal <= 0 || dimVal > array.getRank()))
+        return rewriter.notifyMatchFailure(
+            sum, "Invalid DIM for partial SUM reduction");
     }
-    return dimVal;
-  }
-
-  /// Return hlfir::Entity of the result.
-  hlfir::Entity getResultEntity() const {
-    return hlfir::Entity{op->getResult(0)};
-  }
-
-  /// Return type of the result (e.g. !hlfir.expr<?xi32>).
-  mlir::Type getResultType() const { return getResultEntity().getType(); }
 
-  /// Return the element type of the result (e.g. i32).
-  mlir::Type getResultElementType() const {
-    return hlfir::getFortranElementType(getResultType());
-  }
-
-  /// Return rank of the result.
-  unsigned getResultRank() const { return getResultEntity().getRank(); }
-
-  /// Return the element type of the source.
-  mlir::Type getSourceElementType() const {
-    return hlfir::getFortranElementType(getSource().getType());
-  }
-
-  /// Return rank of the input array.
-  unsigned getSourceRank() const {
-    return hlfir::Entity{getSource()}.getRank();
-  }
-
-  /// The reduction operation.
-  mlir::Operation *op;
-
-  mlir::PatternRewriter &rewriter;
-  mlir::Location loc;
-  fir::FirOpBuilder builder;
-};
-
-/// Generate initialization value for MIN or MAX reduction
-/// of the given \p type.
-template <bool IS_MAX>
-static mlir::Value genMinMaxInitValue(mlir::Location loc,
-                                      fir::FirOpBuilder &builder,
-                                      mlir::Type type) {
-  if (auto ty = mlir::dyn_cast<mlir::FloatType>(type)) {
-    const llvm::fltSemantics &sem = ty.getFloatSemantics();
-    // We must not use +/-INF here. If the reduction input is empty,
-    // the result of reduction must be +/-LARGEST.
-    llvm::APFloat limit = llvm::APFloat::getLargest(sem, /*Negative=*/IS_MAX);
-    return builder.createRealConstant(loc, type, limit);
-  }
-  unsigned bits = type.getIntOrFloatBitWidth();
-  int64_t limitInt = IS_MAX
-                         ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
-                         : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
-  return builder.createIntegerConstant(loc, type, limitInt);
-}
-
-/// Generate a comparison of an array element value \p elem
-/// and the current reduction value \p reduction for MIN/MAX reduction.
-template <bool IS_MAX>
-static mlir::Value
-genMinMaxComparison(mlir::Location loc, fir::FirOpBuilder &builder,
-                    mlir::Value elem, mlir::Value reduction) {
-  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
-    // For FP reductions we want the first smallest value to be used, that
-    // is not NaN. A OGL/OLT condition will usually work for this unless all
-    // the values are Nan or Inf. This follows the same logic as
-    // NumericCompare for Minloc/Maxloc in extrema.cpp.
-    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
-        loc,
-        IS_MAX ? mlir::arith::CmpFPredicate::OGT
-               : mlir::arith::CmpFPredicate::OLT,
-        elem, reduction);
-    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
-  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
-    return builder.create<mlir::arith::CmpIOp>(
-        loc,
-        IS_MAX ? mlir::arith::CmpIPredicate::sgt
-               : mlir::arith::CmpIPredicate::slt,
-        elem, reduction);
-  }
-  llvm_unreachable("unsupported type");
-}
-
-/// Implementation of ReductionAsElementalConverter interface
-/// for MAXLOC/MINLOC.
-template <typename T>
-class MinMaxlocAsElementalConverter : public ReductionAsElementalConverter {
-  static_assert(std::is_same_v<T, hlfir::MaxlocOp> ||
-                std::is_same_v<T, hlfir::MinlocOp>);
-  static constexpr unsigned maxRank = Fortran::common::maxRank;
-  // We have the following reduction values in the reduction loop:
-  //   * N integer coordinates, where N is:
-  //     - RANK(ARRAY) for total reductions.
-  //     - 1 for partial reductions.
-  //   * 1 reduction value holding the current MIN/MAX.
-  //   * 1 boolean indicating whether it is the first time
-  //     the mask is true.
-  static constexpr unsigned maxNumReductions = Fortran::common::maxRank + 2;
-  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxlocOp>;
-  using Base = ReductionAsElementalConverter;
-
-public:
-  MinMaxlocAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
-      : Base{op.getOperation(), rewriter} {}
-
-private:
-  virtual mlir::Value getSource() const final { return getOp().getArray(); }
-  virtual mlir::Value getDim() const final { return getOp().getDim(); }
-  virtual mlir::Value getMask() const final { return getOp().getMask(); }
-  virtual mlir::arith::FastMathFlags getFastMath() const final {
-    return getOp().getFastmath();
-  }
-
-  virtual mlir::LogicalResult isConvertible() const final {
-    if (getOp().getBack())
-      return rewriter.notifyMatchFailure(
-          getOp(), "BACK is not supported for MINLOC/MAXLOC inlining");
-    if (mlir::isa<fir::CharacterType>(getSourceElementType()))
-      return rewriter.notifyMatchFailure(
-          getOp(),
-          "CHARACTER type is not supported for MINLOC/MAXLOC inlining");
-    return mlir::success();
-  }
+    mlir::Location loc = sum.getLoc();
+    fir::FirOpBuilder builder{rewriter, sum.getOperation()};
+    mlir::Type elementType = hlfir::getFortranElementType(sum.getType());
+    mlir::Value mask = sum.getMask();
 
-  // If the result is scalar, then DIM does not matter,
-  // and this is a total reduction.
-  // If DIM is not present, this is a total reduction.
-  virtual bool isTotalReduction() const final {
-    return getResultRank() == 0 || !getDim();
-  }
+    mlir::Value resultShape, dimExtent;
+    llvm::SmallVector<mlir::Value> arrayExtents;
+    if (isTotalReduction)
+      arrayExtents = hlfir::genExtentsVector(loc, builder, array);
+    else
+      std::tie(resultShape, dimExtent) =
+          genResultShapeForPartialReduction(loc, builder, array, dimVal);
+
+    // If the mask is present and is a scalar, then we'd better load its value
+    // outside of the reduction loop making the loop unswitching easier.
+    mlir::Value isPresentPred, maskValue;
+    if (mask) {
+      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+        // MASK represented by a box might be dynamically optional,
+        // so we have to check for its presence before accessing it.
+        isPresentPred =
+            builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
+      }
 
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final;
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) final;
-  virtual hlfir::Entity genFinalResult(
-      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) final;
+      if (hlfir::Entity{mask}.isScalar())
+        maskValue = genMaskValue(loc, builder, mask, isPresentPred, {});
+    }
 
-private:
-  T getOp() const { return mlir::cast<T>(op); }
+    auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                         mlir::ValueRange inputIndices) -> hlfir::Entity {
+      // Loop over all indices in the DIM dimension, and reduce all values.
+      // If DIM is not present, do total reduction.
+
+      // Initial value for the reduction.
+      mlir::Value reductionInitValue =
+          fir::factory::createZeroValue(builder, loc, elementType);
+
+      // The reduction loop may be unordered if FastMathFlags::reassoc
+      // transformations are allowed. The integer reduction is always
+      // unordered.
+      bool isUnordered = mlir::isa<mlir::IntegerType>(elementType) ||
+                         static_cast<bool>(sum.getFastmath() &
+                                           mlir::arith::FastMathFlags::reassoc);
 
-  unsigned getNumCoors() const {
-    return isTotalReduction() ? getSourceRank() : 1;
-  }
+      llvm::SmallVector<mlir::Value> extents;
+      if (isTotalReduction)
+        extents = arrayExtents;
+      else
+        extents.push_back(
+            builder.createConvert(loc, builder.getIndexType(), dimExtent));
 
-  void
-  checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
-    assert(reductions.size() == getNumCoors() + 2 &&
-           "invalid number of reductions for MINLOC/MAXLOC");
-  }
+      auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                         mlir::ValueRange oneBasedIndices,
+                         mlir::ValueRange reductionArgs)
+          -> llvm::SmallVector<mlir::Value, 1> {
+        // Generate the reduction loop-nest body.
+        // The initial reduction value in the innermost loop
+        // is passed via reductionArgs[0].
+        llvm::SmallVector<mlir::Value> indices;
+        if (isTotalReduction) {
+          indices = oneBasedIndices;
+        } else {
+          indices = inputIndices;
+          indices.insert(indices.begin() + dimVal - 1, oneBasedIndices[0]);
+        }
 
-  mlir::Value
-  getCurrentMinMax(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
-    checkReductions(reductions);
-    return reductions[getNumCoors()];
-  }
+        mlir::Value reductionValue = reductionArgs[0];
+        fir::IfOp ifOp;
+        if (mask) {
+          // Make the reduction value update conditional on the value
+          // of the mask.
+          if (!maskValue) {
+            // If the mask is an array, use the elemental and the loop indices
+            // to address the proper mask element.
+            maskValue =
+                genMaskValue(loc, builder, mask, isPresentPred, indices);
+          }
+          mlir::Value isUnmasked = builder.create<fir::ConvertOp>(
+              loc, builder.getI1Type(), maskValue);
+          ifOp = builder.create<fir::IfOp>(loc, elementType, isUnmasked,
+                                           /*withElseRegion=*/true);
+          // In the 'else' block return the current reduction value.
+          builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+          builder.create<fir::ResultOp>(loc, reductionValue);
+
+          // In the 'then' block do the actual addition.
+          builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+        }
 
-  mlir::Value
-  getIsFirst(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
-    checkReductions(reductions);
-    return reductions[getNumCoors() + 1];
-  }
-};
+        hlfir::Entity element =
+            hlfir::getElementAt(loc, builder, array, indices);
+        hlfir::Entity elementValue =
+            hlfir::loadTrivialScalar(loc, builder, element);
+        // NOTE: we can use "Kahan summation" same way as the runtime
+        // (e.g. when fast-math is not allowed), but let's start with
+        // the simple version.
+        reductionValue =
+            genScalarAdd(loc, builder, reductionValue, elementValue);
+
+        if (ifOp) {
+          builder.create<fir::ResultOp>(loc, reductionValue);
+          builder.setInsertionPointAfter(ifOp);
+          reductionValue = ifOp.getResult(0);
+        }
 
-template <typename T>
-llvm::SmallVector<mlir::Value>
-MinMaxlocAsElementalConverter<T>::genReductionInitValues() {
-  // Initial value for the coordinate(s) is zero.
-  mlir::Value zeroCoor =
-      fir::factory::createZeroValue(builder, loc, getResultElementType());
-  llvm::SmallVector<mlir::Value, maxNumReductions> result(getNumCoors(),
-                                                          zeroCoor);
-
-  // Initial value for the MIN/MAX value.
-  mlir::Value minMaxInit =
-      genMinMaxInitValue<isMax>(loc, builder, getSourceElementType());
-  result.push_back(minMaxInit);
-
-  // Initial value for isFirst predicate. It is switched to false,
-  // when the reduction update dynamically happens inside the reduction
-  // loop.
-  mlir::Value trueVal = builder.createBool(loc, true);
-  result.push_back(trueVal);
-
-  return result;
-}
-
-template <typename T>
-llvm::SmallVector<mlir::Value>
-MinMaxlocAsElementalConverter<T>::reduceOneElement(
-    const llvm::SmallVectorImpl<mlir::Value> &currentValue, hlfir::Entity array,
-    mlir::ValueRange oneBasedIndices) {
-  checkReductions(currentValue);
-  hlfir::Entity elementValue =
-      hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-  mlir::Value cmp = genMinMaxComparison<isMax>(loc, builder, elementValue,
-                                               getCurrentMinMax(currentValue));
-  // If isFirst is true, then do the reduction update regardless
-  // of the FP comparison.
-  cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, getIsFirst(currentValue));
-
-  llvm::SmallVector<mlir::Value, maxNumReductions> newIndices;
-  int64_t dim = 1;
-  if (!isTotalReduction()) {
-    auto dimVal = getConstDim();
-    assert(mlir::succeeded(dimVal) &&
-           "partial MINLOC/MAXLOC reduction with invalid DIM");
-    dim = *dimVal;
-    assert(getNumCoors() == 1 &&
-           "partial MAXLOC/MINLOC reduction must compute one coordinate");
-  }
+        return {reductionValue};
+      };
 
-  for (unsigned coorIdx = 0; coorIdx < getNumCoors(); ++coorIdx) {
-    mlir::Value currentCoor = currentValue[coorIdx];
-    mlir::Value newCoor = builder.createConvert(
-        loc, currentCoor.getType(), oneBasedIndices[coorIdx + dim - 1]);
-    mlir::Value update =
-        builder.create<mlir::arith::SelectOp>(loc, cmp, newCoor, currentCoor);
-    newIndices.push_back(update);
-  }
+      llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
+          hlfir::genLoopNestWithReductions(loc, builder, extents,
+                                           {reductionInitValue}, genBody,
+                                           isUnordered);
+      return hlfir::Entity{reductionFinalValues[0]};
+    };
 
-  mlir::Value newMinMax = builder.create<mlir::arith::SelectOp>(
-      loc, cmp, elementValue, getCurrentMinMax(currentValue));
-  newIndices.push_back(newMinMax);
-
-  mlir::Value newIsFirst = builder.createBool(loc, false);
-  newIndices.push_back(newIsFirst);
-
-  assert(currentValue.size() == newIndices.size() &&
-         "invalid number of updated reductions");
-
-  return newIndices;
-}
-
-template <typename T>
-hlfir::Entity MinMaxlocAsElementalConverter<T>::genFinalResult(
-    const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
-  // Identification of the final result of MINLOC/MAXLOC:
-  //   * If DIM is absent, the result is rank-one array.
-  //   * If DIM is present:
-  //     - The result is scalar for rank-one input.
-  //     - The result is an array of rank RANK(ARRAY)-1.
-  checkReductions(reductionResults);
-
-  // We need to adjust the one-based indices to real array indices.
-  // The adjustment must only be done, if there was an actual update
-  // of the coordinates in the reduction loop. For this check we only
-  // need to compare if any of the reduction results is not zero.
-  mlir::Value zero = fir::factory::createZeroValue(
-      builder, loc, reductionResults[0].getType());
-  mlir::Value doAdjust = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::ne, reductionResults[0], zero);
-  mlir::Type indexType = builder.getIndexType();
-  mlir::Value one = builder.createIntegerConstant(loc, indexType, 1);
-
-  auto adjustCoor = [&](mlir::Value coor, mlir::Value lbound) {
-    assert(mlir::isa<mlir::IndexType>(lbound.getType()));
-    mlir::Value coorAsIndex = builder.createConvert(loc, indexType, coor);
-    mlir::Value tmp =
-        builder.create<mlir::arith::AddIOp>(loc, coorAsIndex, lbound);
-    tmp = builder.create<mlir::arith::SubIOp>(loc, tmp, one);
-    tmp =
-        builder.create<mlir::arith::SelectOp>(loc, doAdjust, tmp, coorAsIndex);
-    return builder.createConvert(loc, coor.getType(), tmp);
-  };
-
-  // For partial reductions, the final result of the reduction
-  // loop is just a scalar - the coordinate within DIM dimension.
-  if (getResultRank() == 0 || !isTotalReduction()) {
-    // The result is a scalar, so just return the scalar.
-    assert(getNumCoors() == 1 &&
-           "unpexpected number of coordinates for scalar result");
-
-    int64_t dim = 1;
-    if (!isTotalReduction()) {
-      auto dimVal = getConstDim();
-      assert(mlir::succeeded(dimVal) &&
-             "partial MINLOC/MAXLOC reduction with invalid DIM");
-      dim = *dimVal;
+    if (isTotalReduction) {
+      hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
+      rewriter.replaceOp(sum, result);
+      return mlir::success();
     }
-    mlir::Value dimLbound =
-        hlfir::genLBound(loc, builder, hlfir::Entity{getSource()}, dim - 1);
-    return hlfir::Entity{adjustCoor(reductionResults[0], dimLbound)};
-  }
-  // This is a total reduction, and there is no wrapping hlfir.elemental.
-  // We have to pack the reduced coordinates into a rank-one array.
-  unsigned rank = getSourceRank();
-  // TODO: in order to avoid introducing new memory effects
-  // we should not use a temporary in memory.
-  // We can use hlfir.elemental with a switch to pack all the coordinates
-  // into an array expression, or we can have a dedicated HLFIR operation
-  // for this.
-  mlir::Value tempArray = builder.createTemporary(
-      loc, fir::SequenceType::get(rank, getResultElementType()));
-  llvm::SmallVector<mlir::Value, maxRank> arrayLbounds =
-      hlfir::genLBounds(loc, builder, hlfir::Entity(getSource()));
-  for (unsigned i = 0; i < rank; ++i) {
-    mlir::Value coor = adjustCoor(reductionResults[i], arrayLbounds[i]);
-    mlir::Value idx = builder.createIntegerConstant(loc, indexType, i + 1);
-    mlir::Value resultElement =
-        hlfir::getElementAt(loc, builder, hlfir::Entity{tempArray}, {idx});
-    builder.create<hlfir::AssignOp>(loc, coor, resultElement);
-  }
-  mlir::Value tempExpr = builder.create<hlfir::AsExprOp>(
-      loc, tempArray, builder.createBool(loc, false));
-  return hlfir::Entity{tempExpr};
-}
-
-/// Base class for numeric reductions like MAXVAl, MINVAL, SUM.
-template <typename OpT>
-class NumericReductionAsElementalConverterBase
-    : public ReductionAsElementalConverter {
-  using Base = ReductionAsElementalConverter;
-
-protected:
-  NumericReductionAsElementalConverterBase(OpT op,
-                                           mlir::PatternRewriter &rewriter)
-      : Base{op.getOperation(), rewriter} {}
-
-  virtual mlir::Value getSource() const final { return getOp().getArray(); }
-  virtual mlir::Value getDim() const final { return getOp().getDim(); }
-  virtual mlir::Value getMask() const final { return getOp().getMask(); }
-  virtual mlir::arith::FastMathFlags getFastMath() const final {
-    return getOp().getFastmath();
-  }
 
-  OpT getOp() const { return mlir::cast<OpT>(op); }
-
-  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
-    assert(reductions.size() == 1 && "reduction must produce single value");
-  }
-};
-
-/// Reduction converter for MAXMAL/MINVAL.
-template <typename T>
-class MinMaxvalAsElementalConverter
-    : public NumericReductionAsElementalConverterBase<T> {
-  static_assert(std::is_same_v<T, hlfir::MaxvalOp> ||
-                std::is_same_v<T, hlfir::MinvalOp>);
-  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxvalOp>;
-  using Base = NumericReductionAsElementalConverterBase<T>;
+    hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
+        loc, builder, elementType, resultShape, {}, genKernel,
+        /*isUnordered=*/true, /*polymorphicMold=*/nullptr,
+        sum.getResult().getType());
 
-public:
-  MinMaxvalAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
-      : Base{op, rewriter} {}
+    // it wouldn't be safe to replace block arguments with a different
+    // hlfir.expr type. Types can differ due to differing amounts of shape
+    // information
+    assert(elementalOp.getResult().getType() == sum.getResult().getType());
 
-private:
-  virtual mlir::LogicalResult isConvertible() const final {
-    if (mlir::isa<fir::CharacterType>(this->getSourceElementType()))
-      return this->rewriter.notifyMatchFailure(
-          this->getOp(),
-          "CHARACTER type is not supported for MINVAL/MAXVAL inlining");
+    rewriter.replaceOp(sum, elementalOp);
     return mlir::success();
   }
 
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
-    return {genMinMaxInitValue<isMax>(this->loc, this->builder,
-                                      this->getResultElementType())};
-  }
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array,
-                   mlir::ValueRange oneBasedIndices) final {
-    this->checkReductions(currentValue);
-    fir::FirOpBuilder &builder = this->builder;
-    mlir::Location loc = this->loc;
-    hlfir::Entity elementValue =
-        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-    mlir::Value cmp =
-        genMinMaxComparison<isMax>(loc, builder, elementValue, currentValue[0]);
-    return {builder.create<mlir::arith::SelectOp>(loc, cmp, elementValue,
-                                                  currentValue[0])};
-  }
-};
-
-/// Reduction converter for SUM.
-class SumAsElementalConverter
-    : public NumericReductionAsElementalConverterBase<hlfir::SumOp> {
-  using Base = NumericReductionAsElementalConverterBase;
-
-public:
-  SumAsElementalConverter(hlfir::SumOp op, mlir::PatternRewriter &rewriter)
-      : Base{op, rewriter} {}
-
-private:
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
-    return {
-        fir::factory::createZeroValue(builder, loc, getResultElementType())};
-  }
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array,
-                   mlir::ValueRange oneBasedIndices) final {
-    checkReductions(currentValue);
-    hlfir::Entity elementValue =
-        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-    // NOTE: we can use "Kahan summation" same way as the runtime
-    // (e.g. when fast-math is not allowed), but let's start with
-    // the simple version.
-    return {genScalarAdd(currentValue[0], elementValue)};
-  }
-
-  // Generate scalar addition of the two values (of the same data type).
-  mlir::Value genScalarAdd(mlir::Value value1, mlir::Value value2);
-};
-
-/// Base class for logical reductions like ALL, ANY, COUNT.
-/// They do not have MASK and FastMathFlags.
-template <typename OpT>
-class LogicalReductionAsElementalConverterBase
-    : public ReductionAsElementalConverter {
-  using Base = ReductionAsElementalConverter;
-
-public:
-  LogicalReductionAsElementalConverterBase(OpT op,
-                                           mlir::PatternRewriter &rewriter)
-      : Base{op.getOperation(), rewriter} {}
-
-protected:
-  OpT getOp() const { return mlir::cast<OpT>(op); }
-
-  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
-    assert(reductions.size() == 1 && "reduction must produce single value");
-  }
-
-  virtual mlir::Value getSource() const final { return getOp().getMask(); }
-  virtual mlir::Value getDim() const final { return getOp().getDim(); }
-
-  virtual hlfir::Entity genFinalResult(
-      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) override {
-    checkReductions(reductionResults);
-    return hlfir::Entity{reductionResults[0]};
-  }
-};
-
-/// Reduction converter for ALL/ANY.
-template <typename T>
-class AllAnyAsElementalConverter
-    : public LogicalReductionAsElementalConverterBase<T> {
-  static_assert(std::is_same_v<T, hlfir::AllOp> ||
-                std::is_same_v<T, hlfir::AnyOp>);
-  static constexpr bool isAll = std::is_same_v<T, hlfir::AllOp>;
-  using Base = LogicalReductionAsElementalConverterBase<T>;
-
-public:
-  AllAnyAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
-      : Base{op, rewriter} {}
-
 private:
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
-    return {this->builder.createBool(this->loc, isAll ? true : false)};
-  }
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array,
-                   mlir::ValueRange oneBasedIndices) final {
-    this->checkReductions(currentValue);
-    fir::FirOpBuilder &builder = this->builder;
-    mlir::Location loc = this->loc;
-    hlfir::Entity elementValue =
-        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-    mlir::Value mask =
-        builder.createConvert(loc, builder.getI1Type(), elementValue);
-    if constexpr (isAll)
-      return {builder.create<mlir::arith::AndIOp>(loc, mask, currentValue[0])};
-    else
-      return {builder.create<mlir::arith::OrIOp>(loc, mask, currentValue[0])};
-  }
+  // Return fir.shape specifying the shape of the result
+  // of a SUM reduction with DIM=dimVal. The second return value
+  // is the extent of the DIM dimension.
+  static std::tuple<mlir::Value, mlir::Value>
+  genResultShapeForPartialReduction(mlir::Location loc,
+                                    fir::FirOpBuilder &builder,
+                                    hlfir::Entity array, int64_t dimVal) {
+    llvm::SmallVector<mlir::Value> inExtents =
+        hlfir::genExtentsVector(loc, builder, array);
+    assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
+           "DIM must be present and a positive constant not exceeding "
+           "the array's rank");
 
-  virtual hlfir::Entity genFinalResult(
-      const llvm::SmallVectorImpl<mlir::Value> &reductionValues) final {
-    this->checkReductions(reductionValues);
-    return hlfir::Entity{this->builder.createConvert(
-        this->loc, this->getResultElementType(), reductionValues[0])};
+    mlir::Value dimExtent = inExtents[dimVal - 1];
+    inExtents.erase(inExtents.begin() + dimVal - 1);
+    return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
   }
-};
 
-/// Reduction converter for COUNT.
-class CountAsElementalConverter
-    : public LogicalReductionAsElementalConverterBase<hlfir::CountOp> {
-  using Base = LogicalReductionAsElementalConverterBase<hlfir::CountOp>;
-
-public:
-  CountAsElementalConverter(hlfir::CountOp op, mlir::PatternRewriter &rewriter)
-      : Base{op, rewriter} {}
-
-private:
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
-    return {
-        fir::factory::createZeroValue(builder, loc, getResultElementType())};
-  }
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array,
-                   mlir::ValueRange oneBasedIndices) final {
-    checkReductions(currentValue);
-    hlfir::Entity elementValue =
-        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-    mlir::Value cond =
-        builder.createConvert(loc, builder.getI1Type(), elementValue);
-    mlir::Value one =
-        builder.createIntegerConstant(loc, getResultElementType(), 1);
-    mlir::Value add1 =
-        builder.create<mlir::arith::AddIOp>(loc, currentValue[0], one);
-    return {builder.create<mlir::arith::SelectOp>(loc, cond, add1,
-                                                  currentValue[0])};
+  // Generate scalar addition of the two values (of the same data type).
+  static mlir::Value genScalarAdd(mlir::Location loc,
+                                  fir::FirOpBuilder &builder,
+                                  mlir::Value value1, mlir::Value value2) {
+    mlir::Type ty = value1.getType();
+    assert(ty == value2.getType() && "reduction values' types do not match");
+    if (mlir::isa<mlir::FloatType>(ty))
+      return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+    else if (mlir::isa<mlir::ComplexType>(ty))
+      return builder.create<fir::AddcOp>(loc, value1, value2);
+    else if (mlir::isa<mlir::IntegerType>(ty))
+      return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+
+    llvm_unreachable("unsupported SUM reduction type");
   }
-};
 
-mlir::LogicalResult ReductionAsElementalConverter::convert() {
-  mlir::LogicalResult canConvert(isConvertible());
-
-  if (mlir::failed(canConvert))
-    return canConvert;
-
-  hlfir::Entity array = hlfir::Entity{getSource()};
-  bool isTotalReduce = isTotalReduction();
-  auto dimVal = getConstDim();
-  if (mlir::failed(dimVal))
-    return dimVal;
-  mlir::Value mask = getMask();
-  mlir::Value resultShape, dimExtent;
-  llvm::SmallVector<mlir::Value> arrayExtents;
-  if (isTotalReduce)
-    arrayExtents = hlfir::genExtentsVector(loc, builder, array);
-  else
-    std::tie(resultShape, dimExtent) =
-        genResultShapeForPartialReduction(array, *dimVal);
-
-  // If the mask is present and is a scalar, then we'd better load its value
-  // outside of the reduction loop making the loop unswitching easier.
-  mlir::Value isPresentPred, maskValue;
-  if (mask) {
-    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-      // MASK represented by a box might be dynamically optional,
-      // so we have to check for its presence before accessing it.
-      isPresentPred =
-          builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
+  static mlir::Value genMaskValue(mlir::Location loc,
+                                  fir::FirOpBuilder &builder, mlir::Value mask,
+                                  mlir::Value isPresentPred,
+                                  mlir::ValueRange indices) {
+    mlir::OpBuilder::InsertionGuard guard(builder);
+    fir::IfOp ifOp;
+    mlir::Type maskType =
+        hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
+    if (isPresentPred) {
+      ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
+                                       /*withElseRegion=*/true);
+
+      // Use 'true', if the mask is not present.
+      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+      mlir::Value trueValue = builder.createBool(loc, true);
+      trueValue = builder.createConvert(loc, maskType, trueValue);
+      builder.create<fir::ResultOp>(loc, trueValue);
+
+      // Load the mask value, if the mask is present.
+      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     }
 
-    if (hlfir::Entity{mask}.isScalar())
-      maskValue = genMaskValue(mask, isPresentPred, {});
-  }
-
-  auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                       mlir::ValueRange inputIndices) -> hlfir::Entity {
-    // Loop over all indices in the DIM dimension, and reduce all values.
-    // If DIM is not present, do total reduction.
-
-    // Initial value for the reduction.
-    llvm::SmallVector<mlir::Value, 1> reductionInitValues =
-        genReductionInitValues();
-
-    llvm::SmallVector<mlir::Value> extents;
-    if (isTotalReduce)
-      extents = arrayExtents;
-    else
-      extents.push_back(
-          builder.createConvert(loc, builder.getIndexType(), dimExtent));
-
-    auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                       mlir::ValueRange oneBasedIndices,
-                       mlir::ValueRange reductionArgs)
-        -> llvm::SmallVector<mlir::Value, 1> {
-      // Generate the reduction loop-nest body.
-      // The initial reduction value in the innermost loop
-      // is passed via reductionArgs[0].
-      llvm::SmallVector<mlir::Value> indices;
-      if (isTotalReduce) {
-        indices = oneBasedIndices;
+    hlfir::Entity maskVar{mask};
+    if (maskVar.isScalar()) {
+      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+        // MASK may be a boxed scalar.
+        mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
+        mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
       } else {
-        indices = inputIndices;
-        indices.insert(indices.begin() + *dimVal - 1, oneBasedIndices[0]);
-      }
-
-      llvm::SmallVector<mlir::Value, 1> reductionValues = reductionArgs;
-      llvm::SmallVector<mlir::Type, 1> reductionTypes;
-      llvm::transform(reductionValues, std::back_inserter(reductionTypes),
-                      [](mlir::Value v) { return v.getType(); });
-      fir::IfOp ifOp;
-      if (mask) {
-        // Make the reduction value update conditional on the value
-        // of the mask.
-        if (!maskValue) {
-          // If the mask is an array, use the elemental and the loop indices
-          // to address the proper mask element.
-          maskValue = genMaskValue(mask, isPresentPred, indices);
-        }
-        mlir::Value isUnmasked =
-            builder.create<fir::ConvertOp>(loc, builder.getI1Type(), maskValue);
-        ifOp = builder.create<fir::IfOp>(loc, reductionTypes, isUnmasked,
-                                         /*withElseRegion=*/true);
-        // In the 'else' block return the current reduction value.
-        builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-        builder.create<fir::ResultOp>(loc, reductionValues);
-
-        // In the 'then' block do the actual addition.
-        builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+        mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
       }
-      reductionValues = reduceOneElement(reductionValues, array, indices);
-      if (ifOp) {
-        builder.create<fir::ResultOp>(loc, reductionValues);
-        builder.setInsertionPointAfter(ifOp);
-        reductionValues = ifOp.getResults();
-      }
-
-      return reductionValues;
-    };
-
-    llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
-        hlfir::genLoopNestWithReductions(
-            loc, builder, extents, reductionInitValues, genBody, isUnordered());
-    return genFinalResult(reductionFinalValues);
-  };
-
-  if (isTotalReduce) {
-    hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
-    rewriter.replaceOp(op, result);
-    return mlir::success();
-  }
-
-  hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
-      loc, builder, getResultElementType(), resultShape, /*typeParams=*/{},
-      genKernel,
-      /*isUnordered=*/true, /*polymorphicMold=*/nullptr, getResultType());
-
-  // it wouldn't be safe to replace block arguments with a different
-  // hlfir.expr type. Types can differ due to differing amounts of shape
-  // information
-  assert(elementalOp.getResult().getType() == op->getResult(0).getType());
-
-  rewriter.replaceOp(op, elementalOp);
-  return mlir::success();
-}
-
-std::tuple<mlir::Value, mlir::Value>
-ReductionAsElementalConverter::genResultShapeForPartialReduction(
-    hlfir::Entity array, int64_t dimVal) {
-  llvm::SmallVector<mlir::Value> inExtents =
-      hlfir::genExtentsVector(loc, builder, array);
-  assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
-         "DIM must be present and a positive constant not exceeding "
-         "the array's rank");
-
-  mlir::Value dimExtent = inExtents[dimVal - 1];
-  inExtents.erase(inExtents.begin() + dimVal - 1);
-  return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
-}
-
-mlir::Value SumAsElementalConverter::genScalarAdd(mlir::Value value1,
-                                                  mlir::Value value2) {
-  mlir::Type ty = value1.getType();
-  assert(ty == value2.getType() && "reduction values' types do not match");
-  if (mlir::isa<mlir::FloatType>(ty))
-    return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
-  else if (mlir::isa<mlir::ComplexType>(ty))
-    return builder.create<fir::AddcOp>(loc, value1, value2);
-  else if (mlir::isa<mlir::IntegerType>(ty))
-    return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
-
-  llvm_unreachable("unsupported SUM reduction type");
-}
-
-mlir::Value ReductionAsElementalConverter::genMaskValue(
-    mlir::Value mask, mlir::Value isPresentPred, mlir::ValueRange indices) {
-  mlir::OpBuilder::InsertionGuard guard(builder);
-  fir::IfOp ifOp;
-  mlir::Type maskType =
-      hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
-  if (isPresentPred) {
-    ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
-                                     /*withElseRegion=*/true);
-
-    // Use 'true', if the mask is not present.
-    builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-    mlir::Value trueValue = builder.createBool(loc, true);
-    trueValue = builder.createConvert(loc, maskType, trueValue);
-    builder.create<fir::ResultOp>(loc, trueValue);
-
-    // Load the mask value, if the mask is present.
-    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  }
-
-  hlfir::Entity maskVar{mask};
-  if (maskVar.isScalar()) {
-    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-      // MASK may be a boxed scalar.
-      mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
-      mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
     } else {
+      // Load from the mask array.
+      assert(!indices.empty() && "no indices for addressing the mask array");
+      maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
       mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
     }
-  } else {
-    // Load from the mask array.
-    assert(!indices.empty() && "no indices for addressing the mask array");
-    maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
-    mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
-  }
-
-  if (!isPresentPred)
-    return mask;
 
-  builder.create<fir::ResultOp>(loc, mask);
-  return ifOp.getResult(0);
-}
-
-/// Convert an operation that is a partial or total reduction
-/// over an array of values into a reduction loop[-nest]
-/// optionally wrapped into hlfir.elemental.
-template <typename Op>
-class ReductionConversion : public mlir::OpRewritePattern<Op> {
-public:
-  using mlir::OpRewritePattern<Op>::OpRewritePattern;
+    if (!isPresentPred)
+      return mask;
 
-  llvm::LogicalResult
-  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
-    if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
-                  std::is_same_v<Op, hlfir::MinlocOp>) {
-      MinMaxlocAsElementalConverter<Op> converter(op, rewriter);
-      return converter.convert();
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
-                         std::is_same_v<Op, hlfir::MinvalOp>) {
-      MinMaxvalAsElementalConverter<Op> converter(op, rewriter);
-      return converter.convert();
-    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
-      CountAsElementalConverter converter(op, rewriter);
-      return converter.convert();
-    } else if constexpr (std::is_same_v<Op, hlfir::AllOp> ||
-                         std::is_same_v<Op, hlfir::AnyOp>) {
-      AllAnyAsElementalConverter<Op> converter(op, rewriter);
-      return converter.convert();
-    } else if constexpr (std::is_same_v<Op, hlfir::SumOp>) {
-      SumAsElementalConverter converter{op, rewriter};
-      return converter.convert();
-    }
-    return rewriter.notifyMatchFailure(op, "unexpected reduction operation");
+    builder.create<fir::ResultOp>(loc, mask);
+    return ifOp.getResult(0);
   }
 };
 
@@ -2118,18 +1481,10 @@ class SimplifyHLFIRIntrinsics
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
-    patterns.insert<ReductionConversion<hlfir::SumOp>>(context);
+    patterns.insert<SumAsElementalConversion>(context);
     patterns.insert<CShiftConversion>(context);
     patterns.insert<MatmulConversion<hlfir::MatmulTransposeOp>>(context);
 
-    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
-
     // If forceMatmulAsElemental is false, then hlfir.matmul inlining
     // will introduce hlfir.eval_in_mem operation with new memory side
     // effects. This conflicts with CSE and optimized bufferization, e.g.:
diff --git a/flang/test/HLFIR/all-elemental.fir b/flang/test/HLFIR/all-elemental.fir
new file mode 100644
index 0000000000000..1ba8bb1b7a5fb
--- /dev/null
+++ b/flang/test/HLFIR/all-elemental.fir
@@ -0,0 +1,91 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %15 = fir.load %14 : !fir.ref<i32>
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %12 = hlfir.all %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
+  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
+  return %13 : !fir.logical<4>
+}
+// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
+// CHECK-NEXT:     %true = arith.constant true
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
+// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
+// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
+// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %true) -> (i1) {
+// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
+// CHECK-NEXT:       %[[V17:.*]] = arith.andi %arg4, %[[V16]] : i1
+// CHECK-NEXT:       fir.result %[[V17]] : i1
+// CHECK-NEXT:     }
+// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
+
+
+func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
+  %c2_i32 = arith.constant 2 : i32
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
+  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
+  %8 = fir.load %6#0 : !fir.ref<i32>
+  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index):
+    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
+    %13 = fir.load %12 : !fir.ref<i32>
+    %14 = arith.cmpi sge, %13, %8 : i32
+    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %15 : !fir.logical<4>
+  }
+  %10 = hlfir.all %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
+  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
+  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
+  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
+  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
+  return %11 : !fir.array<4x!fir.logical<4>>
+}
+// CHECK-LABEL:  func.func @_QFPtest_dim(
+// CHECK: %10 = hlfir.all %9 dim %c2_i32
\ No newline at end of file
diff --git a/flang/test/HLFIR/any-elemental.fir b/flang/test/HLFIR/any-elemental.fir
new file mode 100644
index 0000000000000..a7c559679d965
--- /dev/null
+++ b/flang/test/HLFIR/any-elemental.fir
@@ -0,0 +1,190 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %15 = fir.load %14 : !fir.ref<i32>
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %12 = hlfir.any %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
+  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
+  return %13 : !fir.logical<4>
+}
+// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
+// CHECK-NEXT:     %false = arith.constant false
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
+// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
+// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
+// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %false) -> (i1) {
+// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
+// CHECK-NEXT:       %[[V17:.*]] = arith.ori %arg4, %[[V16]] : i1
+// CHECK-NEXT:       fir.result %[[V17]] : i1
+// CHECK-NEXT:     }
+// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
+
+
+func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
+  %c2_i32 = arith.constant 2 : i32
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
+  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
+  %8 = fir.load %6#0 : !fir.ref<i32>
+  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index):
+    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
+    %13 = fir.load %12 : !fir.ref<i32>
+    %14 = arith.cmpi sge, %13, %8 : i32
+    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %15 : !fir.logical<4>
+  }
+  %10 = hlfir.any %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
+  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
+  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
+  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
+  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
+  return %11 : !fir.array<4x!fir.logical<4>>
+}
+// CHECK-LABEL:  func.func @_QFPtest_dim(
+// CHECK: {{.*}} = hlfir.any {{.*}} dim %c2_i32
+
+
+func.func @_Qtest_recursive() attributes {fir.bindc_name = "test"} {
+  %c1 = arith.constant 1 : index
+  %true = arith.constant true
+  %false = arith.constant false
+  %c0_i64 = arith.constant 0 : i64
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c1_i32 = arith.constant 1 : i32
+  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %1:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+  %2 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
+  %5:2 = hlfir.declare %4 {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "ra", uniq_name = "_QFEra"}
+  %7 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %8:2 = hlfir.declare %6(%7) {uniq_name = "_QFEra"} : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.ref<!fir.array<1x!fir.logical<4>>>)
+  %9 = fir.alloca !fir.logical<4> {bindc_name = "rs", uniq_name = "_QFErs"}
+  %10:2 = hlfir.declare %9 {uniq_name = "_QFErs"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %11 = fir.allocmem !fir.array<?xi32>, %c1 {fir.must_be_heap = true, uniq_name = "_QFEa.alloc"}
+  %12 = fir.embox %11(%7) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+  fir.store %12 to %1#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  hlfir.assign %c1_i32 to %5#0 : i32, !fir.ref<i32>
+  %13 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %14:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+  fir.do_loop %arg0 = %c1 to %14#1 step %c1 unordered {
+    %27:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+    %28 = arith.subi %27#0, %c1 : index
+    %29 = arith.addi %arg0, %28 : index
+    %30 = hlfir.designate %13 (%29)  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+    hlfir.assign %c2_i32 to %30 : i32, !fir.ref<i32>
+  }
+  %15 = fir.load %5#0 : !fir.ref<i32>
+  %16 = fir.convert %15 : (i32) -> i64
+  %17 = arith.cmpi sgt, %16, %c0_i64 : i64
+  %18 = arith.select %17, %16, %c0_i64 : i64
+  %19 = fir.convert %18 : (i64) -> index
+  %20 = fir.shape %19 : (index) -> !fir.shape<1>
+  %21 = hlfir.elemental %20 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg0: index):
+    %27 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+    %28:3 = fir.box_dims %27, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+    %29 = arith.addi %28#0, %28#1 : index
+    %30 = arith.subi %29, %c1 : index
+    %31 = arith.subi %30, %28#0 : index
+    %32 = arith.addi %31, %c1 : index
+    %33 = arith.cmpi sgt, %32, %c0 : index
+    %34 = arith.select %33, %32, %c0 : index
+    %35 = fir.shape %34 : (index) -> !fir.shape<1>
+    %36 = hlfir.designate %27 (%28#0:%30:%c1)  shape %35 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+    %37 = hlfir.elemental %35 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+    ^bb0(%arg1: index):
+      %39 = hlfir.designate %36 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+      %40 = fir.load %39 : !fir.ref<i32>
+      %41 = arith.cmpi eq, %40, %c1_i32 : i32
+      %42 = fir.convert %41 : (i1) -> !fir.logical<4>
+      hlfir.yield_element %42 : !fir.logical<4>
+    }
+    %38 = hlfir.any %37 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
+    hlfir.destroy %37 : !hlfir.expr<?x!fir.logical<4>>
+    hlfir.yield_element %38 : !fir.logical<4>
+  }
+  %22 = hlfir.any %21 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
+  hlfir.assign %22 to %10#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+  hlfir.destroy %21 : !hlfir.expr<?x!fir.logical<4>>
+  %23 = fir.load %10#0 : !fir.ref<!fir.logical<4>>
+  %24 = fir.convert %23 : (!fir.logical<4>) -> i1
+  %25 = arith.xori %24, %true : i1
+  cf.cond_br %25, ^bb1, ^bb2
+^bb1:  // pred: ^bb0
+  fir.call @_FortranAStopStatement(%c2_i32, %false, %false) fastmath<contract> : (i32, i1, i1) -> ()
+  fir.unreachable
+^bb2:  // pred: ^bb0
+  return
+}
+// CHECK-LABEL: func.func @_Qtest_recursive()
+// CHECK:    %[[V20:.*]] = fir.do_loop %arg0 = %c1 to %{{.*}} step %c1 iter_args(%arg1 = %false) -> (i1) {
+// CHECK:      %[[V26:.*]] = fir.load %[[V1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK:      %[[V27:.*]]:3 = fir.box_dims %[[V26]], %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+// CHECK:      %[[V28:.*]] = arith.addi %[[V27]]#0, %[[V27]]#1 : index
+// CHECK:      %[[V29:.*]] = arith.subi %[[V28]], %c1 : index
+// CHECK:      %[[V30:.*]] = arith.subi %[[V29]], %[[V27]]#0 : index
+// CHECK:      %[[V31:.*]] = arith.addi %[[V30]], %c1 : index
+// CHECK:      %[[V32:.*]] = arith.cmpi sgt, %[[V31]], %c0 : index
+// CHECK:      %[[V33:.*]] = arith.select %[[V32]], %[[V31]], %c0 : index
+// CHECK:      %[[V34:.*]] = fir.shape %[[V33]] : (index) -> !fir.shape<1>
+// CHECK:      %[[V35:.*]] = hlfir.designate %[[V26]] (%[[V27]]#0:%[[V29]]:%c1)  shape %[[V34]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:      %[[V36:.*]] = fir.do_loop %arg2 = %c1 to %[[V33]] step %c1 iter_args(%arg3 = %false) -> (i1) {
+// CHECK:        %[[V38:.*]] = hlfir.designate %[[V35]] (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:        %[[V39:.*]] = fir.load %[[V38]] : !fir.ref<i32>
+// CHECK:        %[[V40:.*]] = arith.cmpi eq, %[[V39]], %c1_i32 : i32
+// CHECK:        %[[V41:.*]] = arith.ori %arg3, %[[V40]] : i1
+// CHECK:        fir.result %[[V41]] : i1
+// CHECK:      }
+// CHECK:      %[[V37:.*]] = arith.ori %arg1, %[[V36]] : i1
+// CHECK:      fir.result %[[V37]] : i1
+// CHECK:    }
diff --git a/flang/test/HLFIR/count-elemental.fir b/flang/test/HLFIR/count-elemental.fir
new file mode 100644
index 0000000000000..0df5cc3c031ea
--- /dev/null
+++ b/flang/test/HLFIR/count-elemental.fir
@@ -0,0 +1,314 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %15 = fir.load %14 : !fir.ref<i32>
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
+  hlfir.assign %12 to %4#0 : i32, !fir.ref<i32>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %13 = fir.load %4#1 : !fir.ref<i32>
+  return %13 : i32
+}
+// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
+// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
+// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
+// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
+// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
+// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
+// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
+// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
+// CHECK-NEXT:       fir.result %[[V17]] : i32
+// CHECK-NEXT:     }
+// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
+// CHECK-NEXT:     return %[[V12]] : i32
+
+func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i16 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %15 = fir.load %14 : !fir.ref<i32>
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i16
+  hlfir.assign %12 to %4#0 : i16, !fir.ref<i16>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %13 = fir.load %4#1 : !fir.ref<i16>
+  return %13 : i16
+}
+// CHECK-LABEL:  func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
+// CHECK-NEXT:     %c1_i16 = arith.constant 1 : i16
+// CHECK-NEXT:     %c0_i16 = arith.constant 0 : i16
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i16
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
+// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
+// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
+// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i16) -> (i16) {
+// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
+// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i16 : i16
+// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i16
+// CHECK-NEXT:       fir.result %[[V17]] : i16
+// CHECK-NEXT:     }
+// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i16, !fir.ref<i16>
+// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i16>
+// CHECK-NEXT:     return %[[V12]] : i16
+
+func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<7xi32> {
+  %c1_i32 = arith.constant 1 : i32
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.array<7xi32> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<7xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<7xi32>>, !fir.ref<!fir.array<7xi32>>)
+  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
+  %8 = fir.load %6#0 : !fir.ref<i32>
+  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index):
+    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
+    %13 = fir.load %12 : !fir.ref<i32>
+    %14 = arith.cmpi sge, %13, %8 : i32
+    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %15 : !fir.logical<4>
+  }
+  %10 = hlfir.count %9 dim %c1_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<7xi32>
+  hlfir.assign %10 to %5#0 : !hlfir.expr<7xi32>, !fir.ref<!fir.array<7xi32>>
+  hlfir.destroy %10 : !hlfir.expr<7xi32>
+  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
+  %11 = fir.load %5#1 : !fir.ref<!fir.array<7xi32>>
+  return %11 : !fir.array<7xi32>
+}
+// CHECK-LABEL:  func.func @_QFPtest_dim(
+// CHECK: %{{.*}} = hlfir.count %{{.*}} dim %c1_i32
+
+
+func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %c2 = arith.constant 2 : index
+  %0 = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %0 : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
+  %7 = fir.load %5#0 : !fir.ref<i32>
+  %8 = hlfir.elemental %0 unordered : (!fir.shape<3>) -> !hlfir.expr<4x7x2x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index, %arg5: index):
+    %11 = hlfir.designate %6 (%arg3, %arg4, %arg5)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
+    %12 = fir.load %11 : !fir.ref<i32>
+    %13 = arith.cmpi sge, %12, %7 : i32
+    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %14 : !fir.logical<4>
+  }
+  %9 = hlfir.count %8 : (!hlfir.expr<4x7x2x!fir.logical<4>>) -> i32
+  hlfir.assign %9 to %4#0 : i32, !fir.ref<i32>
+  hlfir.destroy %8 : !hlfir.expr<4x7x2x!fir.logical<4>>
+  %10 = fir.load %4#1 : !fir.ref<i32>
+  return %10 : i32
+}
+// CHECK-LABEL:  func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
+// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %c2 = arith.constant 2 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]]) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]] {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:     %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %[[V0]] : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
+// CHECK-NEXT:     %[[V7:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V8:.*]] = fir.do_loop %arg3 = %c1 to %c2 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
+// CHECK-NEXT:       %[[V10:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %arg4) -> (i32) {
+// CHECK-NEXT:         %[[V11:.*]] = fir.do_loop %arg7 = %c1 to %c4 step %c1 iter_args(%arg8 = %arg6) -> (i32) {
+// CHECK-NEXT:           %[[V12:.*]] = hlfir.designate %[[V6]] (%arg7, %arg5, %arg3)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
+// CHECK-NEXT:           %[[V13:.*]] = fir.load %[[V12]] : !fir.ref<i32>
+// CHECK-NEXT:           %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V7]] : i32
+// CHECK-NEXT:           %[[V15:.*]] = arith.addi %arg8, %c1_i32 : i32
+// CHECK-NEXT:           %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg8 : i32
+// CHECK-NEXT:           fir.result %[[V16]] : i32
+// CHECK-NEXT:         }
+// CHECK-NEXT:         fir.result %[[V11]] : i32
+// CHECK-NEXT:       }
+// CHECK-NEXT:       fir.result %[[V10]] : i32
+// CHECK-NEXT:     }
+// CHECK-NEXT:     hlfir.assign %[[V8]] to %[[V4]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:     %[[V9:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
+// CHECK-NEXT:     return %[[V9]] : i32
+
+
+
+
+
+func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7xi32> {
+  ^bb0(%arg3: index):
+    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %16 = fir.load %15 : !fir.ref<i32>
+    hlfir.yield_element %16 : i32
+  }
+  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %15 = hlfir.sum %11 : (!hlfir.expr<7xi32>) -> i32
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
+  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
+  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
+  hlfir.destroy %11 : !hlfir.expr<7xi32>
+  %14 = fir.load %4#1 : !fir.ref<i32>
+  return %14 : i32
+}
+// CHECK-LABEL:  func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+// CHECK:    %[[V12:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
+// CHECK:      %[[V14:.*]] = hlfir.sum %[[V11]] : (!hlfir.expr<7xi32>) -> i32
+// CHECK:      %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
+// CHECK:      %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
+// CHECK:      %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
+// CHECK:      fir.result %[[V17]] : i32
+// CHECK:    }
+
+
+
+
+func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %16 = fir.load %15 : !fir.ref<i32>
+    %17 = arith.cmpi sge, %16, %10 : i32
+    %18 = fir.convert %17 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %18 : !fir.logical<4>
+  }
+  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %15 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
+  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
+  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %14 = fir.load %4#1 : !fir.ref<i32>
+  return %14 : i32
+}
+// CHECK-LABEL:  func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+// CHECK:    %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
+// CHECK:      %[[V13:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %c0_i32) -> (i32) {
+// CHECK:        %[[V17:.*]] = hlfir.designate %[[V9]] (%arg5)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK:        %[[V18:.*]] = fir.load %[[V17]] : !fir.ref<i32>
+// CHECK:        %[[V19:.*]] = arith.cmpi sge, %[[V18]], %[[V10]] : i32
+// CHECK:        %[[V20:.*]] = arith.addi %arg6, %c1_i32 : i32
+// CHECK:        %[[V21:.*]] = arith.select %[[V19]], %[[V20]], %arg6 : i32
+// CHECK:        fir.result %[[V21]] : i32
+// CHECK:      }
+// CHECK:      %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V10]] : i32
+// CHECK:      %[[V15:.*]] = arith.addi %arg4, %c1_i32 : i32
+// CHECK:      %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg4 : i32
+// CHECK:      fir.result %[[V16]] : i32
+// CHECK:    }
diff --git a/flang/test/HLFIR/maxloc-elemental.fir b/flang/test/HLFIR/maxloc-elemental.fir
new file mode 100644
index 0000000000000..c9210a59f0340
--- /dev/null
+++ b/flang/test/HLFIR/maxloc-elemental.fir
@@ -0,0 +1,133 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.load %2#0 : !fir.ref<i32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.cmpi sge, %9, %3 : i32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+// CHECK-NEXT:    %true = arith.constant true
+// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
+// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
+// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
+// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
+// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
+// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
+// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c-2147483648_i32) -> (i32) {
+// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
+// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
+// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi sgt, %[[V20]], %arg4 : i32
+// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
+// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
+// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
+// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
+// CHECK-NEXT:          fir.result %[[V20]] : i32
+// CHECK-NEXT:        } else {
+// CHECK-NEXT:          fir.result %arg4 : i32
+// CHECK-NEXT:        }
+// CHECK-NEXT:        fir.result %[[V22]] : i32
+// CHECK-NEXT:      } else {
+// CHECK-NEXT:        fir.result %arg4 : i32
+// CHECK-NEXT:      }
+// CHECK-NEXT:      fir.result %[[V18]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %3 = fir.load %2#0 : !fir.ref<f32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+    %9 = fir.load %8 : !fir.ref<f32>
+    %10 = arith.cmpf oge, %9, %3 : f32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL: _QPtest_float
+// CHECK:        %cst = arith.constant 0xFF800000 : f32
+// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
+// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
+// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
+// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
+// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
+// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf ogt, %[[V20]], %arg4 fastmath<contract> : f32
+// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
+// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
+// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
+// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
+// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
+// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
+// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
+// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
+// CHECK-NEXT:         fir.result %[[V20]] : f32
+// CHECK-NEXT:       } else {
+// CHECK-NEXT:         fir.result %arg4 : f32
+// CHECK-NEXT:       }
+// CHECK-NEXT:       fir.result %[[V22]] : f32
+// CHECK-NEXT:     } else {
+// CHECK-NEXT:       fir.result %arg4 : f32
+// CHECK-NEXT:     }
+// CHECK-NEXT:     fir.result %[[V18]] : f32
+// CHECK-NEXT:   }
+
diff --git a/flang/test/HLFIR/maxval-elemental.fir b/flang/test/HLFIR/maxval-elemental.fir
new file mode 100644
index 0000000000000..a21b4858412de
--- /dev/null
+++ b/flang/test/HLFIR/maxval-elemental.fir
@@ -0,0 +1,117 @@
+// Test maxval inlining for both elemental and designate
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+// subroutine test(array)
+//   integer :: array(:), x
+//   x = maxval(abs(array))
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+  %c31_i32 = arith.constant 31 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg1: index):
+    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.shrsi %9, %c31_i32 : i32
+    %11 = arith.xori %9, %10 : i32
+    %12 = arith.subi %11, %10 : i32
+    hlfir.yield_element %12 : i32
+  }
+  %7 = hlfir.maxval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
+  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
+// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c-2147483648_i32) -> (i32) {
+// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
+// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi sgt, %[[V10]], %arg2 : i32
+// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
+// CHECK-NEXT:      fir.result %[[V12]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// subroutine test(array)
+//   real :: array(:), x
+//   x = maxval(array(3:6))
+// end subroutine test
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
+  %c4 = arith.constant 4 : index
+  %c1 = arith.constant 1 : index
+  %c6 = arith.constant 6 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+  %6 = hlfir.maxval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
+  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
+  return
+}
+
+// CHECK-LABEL: _QPtest_float
+// CHECK:       %cst = arith.constant 0xFF800000 : f32
+// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
+// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
+// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
+// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
+// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
+// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
+// CHECK-NEXT:      fir.result %[[V14]] : f32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// Verify that lower bounds of designator are applied in the indexing inside
+// the generated loop (hlfir.designate takes indices relative to the base lower
+// bounds).
+func.func @component_lower_bounds(%arg0: !fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>) -> i32 {
+  %c10 = arith.constant 10 : index
+  %c101 = arith.constant 101 : index
+  %4 = fir.shape_shift %c101, %c10 : (index, index) -> !fir.shapeshift<1>
+  %5 = hlfir.designate %arg0{"i"}   shape %4 : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
+  %6 = hlfir.maxval %5 : (!fir.box<!fir.array<10xi32>>) -> i32
+  return %6 : i32
+}
+// CHECK-LABEL:   func.func @component_lower_bounds(
+// CHECK:  %[[VAL_1:.*]] = arith.constant 100 : index
+// CHECK:  %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:  %[[VAL_4:.*]] = arith.constant 10 : index
+// CHECK:  %[[VAL_5:.*]] = arith.constant 101 : index
+// CHECK:  %[[VAL_6:.*]] = fir.shape_shift %[[VAL_5]], %[[VAL_4]] : (index, index) -> !fir.shapeshift<1>
+// CHECK:  %[[VAL_7:.*]] = hlfir.designate %{{.*}}{"i"}   shape %[[VAL_6]] : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
+// CHECK:  %[[VAL_8:.*]] = fir.do_loop %[[VAL_9:.*]] = %[[VAL_2]] to %[[VAL_4]] {{.*}}
+// CHECK:    %[[VAL_11:.*]] = arith.addi %[[VAL_9]], %[[VAL_1]] : index
+// CHECK:    hlfir.designate %[[VAL_7]] (%[[VAL_11]])  : (!fir.box<!fir.array<10xi32>>, index) -> !fir.ref<i32>
diff --git a/flang/test/HLFIR/minloc-elemental.fir b/flang/test/HLFIR/minloc-elemental.fir
new file mode 100644
index 0000000000000..9453a335b4fbf
--- /dev/null
+++ b/flang/test/HLFIR/minloc-elemental.fir
@@ -0,0 +1,397 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.load %2#0 : !fir.ref<i32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.cmpi sge, %9, %3 : i32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+// CHECK-NEXT:    %true = arith.constant true
+// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
+// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
+// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
+// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
+// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
+// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
+// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
+// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
+// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
+// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
+// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
+// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
+// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
+// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
+// CHECK-NEXT:          fir.result %[[V20]] : i32
+// CHECK-NEXT:        } else {
+// CHECK-NEXT:          fir.result %arg4 : i32
+// CHECK-NEXT:        }
+// CHECK-NEXT:        fir.result %[[V22]] : i32
+// CHECK-NEXT:      } else {
+// CHECK-NEXT:        fir.result %arg4 : i32
+// CHECK-NEXT:      }
+// CHECK-NEXT:      fir.result %[[V18]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+
+func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.load %2#0 : !fir.ref<i32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.cmpi sge, %9, %3 : i32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi16>, !fir.box<!fir.array<?xi16>>
+  hlfir.destroy %7 : !hlfir.expr<1xi16>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL:  func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
+// CHECK-NEXT:    %true = arith.constant true
+// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
+// CHECK-NEXT:    %c1_i16 = arith.constant 1 : i16
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c0_i16 = arith.constant 0 : i16
+// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i16
+// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
+// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK-NEXT:    fir.store %c0_i16 to %[[V8]] : !fir.ref<i16>
+// CHECK-NEXT:    fir.store %c0_i16 to %[[V0]] : !fir.ref<i16>
+// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
+// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
+// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
+// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
+// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i16>
+// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
+// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i16) -> i1
+// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
+// CHECK-NEXT:          fir.store %c1_i16 to %[[V0]] : !fir.ref<i16>
+// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i16
+// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i16>
+// CHECK-NEXT:          fir.result %[[V20]] : i32
+// CHECK-NEXT:        } else {
+// CHECK-NEXT:          fir.result %arg4 : i32
+// CHECK-NEXT:        }
+// CHECK-NEXT:        fir.result %[[V22]] : i32
+// CHECK-NEXT:      } else {
+// CHECK-NEXT:        fir.result %arg4 : i32
+// CHECK-NEXT:      }
+// CHECK-NEXT:      fir.result %[[V18]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi16>>, !fir.box<!fir.array<?xi16>>
+// CHECK-NEXT:    return
+
+
+func.func @_QPtest_kind2_convert(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.load %2#0 : !fir.ref<i32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %10 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %11 = fir.load %10 : !fir.ref<i32>
+    %12 = arith.cmpi sge, %11, %3 : i32
+    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %13 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %9 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg3: index):
+    %10 = hlfir.apply %7, %arg3 : (!hlfir.expr<1xi16>, index) -> i16
+    %11 = fir.convert %10 : (i16) -> i32
+    hlfir.yield_element %11 : i32
+  }
+  hlfir.assign %9 to %1#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %9 : !hlfir.expr<?xi32>
+  hlfir.destroy %7 : !hlfir.expr<1xi16>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPtest_kind2_convert(
+// CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"},
+// CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "val"},
+// CHECK-SAME:                                     %[[VAL_2:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant false
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_10:.*]] = fir.alloca i16
+// CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_16]] : !fir.ref<i16>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<i16>
+// CHECK:           %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_12]]#0, %[[VAL_7]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#1, %[[VAL_9]] : index
+// CHECK:           %[[VAL_19:.*]] = fir.do_loop %[[VAL_20:.*]] = %[[VAL_7]] to %[[VAL_18]] step %[[VAL_9]] iter_args(%[[VAL_21:.*]] = %[[VAL_5]]) -> (i32) {
+// CHECK:             %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_9]] : index
+// CHECK:             %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<i32>
+// CHECK:             %[[VAL_25:.*]] = arith.cmpi sge, %[[VAL_24]], %[[VAL_15]] : i32
+// CHECK:             %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (i32) {
+// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_10]] : !fir.ref<i16>
+// CHECK:               %[[VAL_28:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_28]] : !fir.ref<i32>
+// CHECK:               %[[VAL_30:.*]] = arith.cmpi slt, %[[VAL_29]], %[[VAL_21]] : i32
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_27]] : (i16) -> i1
+// CHECK:               %[[VAL_32:.*]] = arith.xori %[[VAL_31]], %[[VAL_4]] : i1
+// CHECK:               %[[VAL_33:.*]] = arith.ori %[[VAL_30]], %[[VAL_32]] : i1
+// CHECK:               %[[VAL_34:.*]] = fir.if %[[VAL_33]] -> (i32) {
+// CHECK:                 fir.store %[[VAL_6]] to %[[VAL_10]] : !fir.ref<i16>
+// CHECK:                 %[[VAL_35:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_22]] : (index) -> i16
+// CHECK:                 fir.store %[[VAL_36]] to %[[VAL_35]] : !fir.ref<i16>
+// CHECK:                 fir.result %[[VAL_29]] : i32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_21]] : i32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_34]] : i32
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_21]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_26]] : i32
+// CHECK:           }
+// CHECK:           %[[VAL_37:.*]] = hlfir.as_expr %[[VAL_11]] move %[[VAL_3]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           fir.do_loop %[[VAL_38:.*]] = %[[VAL_9]] to %[[VAL_9]] step %[[VAL_9]] unordered {
+// CHECK:             %[[VAL_39:.*]] = hlfir.apply %[[VAL_37]], %[[VAL_38]] : (!hlfir.expr<1xi16>, index) -> i16
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i16) -> i32
+// CHECK:             %[[VAL_41:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_38]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %3 = fir.load %2#0 : !fir.ref<f32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+    %9 = fir.load %8 : !fir.ref<f32>
+    %10 = arith.cmpf oge, %9, %3 : f32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL: _QPtest_float
+// CHECK:        %cst = arith.constant 0x7F800000 : f32
+// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
+// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
+// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
+// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
+// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
+// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf olt, %[[V20]], %arg4 fastmath<contract> : f32
+// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
+// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
+// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
+// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
+// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
+// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
+// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
+// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
+// CHECK-NEXT:         fir.result %[[V20]] : f32
+// CHECK-NEXT:       } else {
+// CHECK-NEXT:         fir.result %arg4 : f32
+// CHECK-NEXT:       }
+// CHECK-NEXT:       fir.result %[[V22]] : f32
+// CHECK-NEXT:     } else {
+// CHECK-NEXT:       fir.result %arg4 : f32
+// CHECK-NEXT:     }
+// CHECK-NEXT:     fir.result %[[V18]] : f32
+// CHECK-NEXT:   }
+
+
+func.func @_QPtest_assignshape(%arg0: !fir.ref<!fir.array<3x3xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.ref<!fir.array<3xi32>> {fir.bindc_name = "m"}) {
+  %c2 = arith.constant 2 : index
+  %c1 = arith.constant 1 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.shape %c3, %c3 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFtestEarray"} : (!fir.ref<!fir.array<3x3xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x3xf32>>, !fir.ref<!fir.array<3x3xf32>>)
+  %2 = fir.shape %c3 : (index) -> !fir.shape<1>
+  %3:2 = hlfir.declare %arg2(%2) {uniq_name = "_QFtestEm"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+  %4:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %5 = fir.load %4#0 : !fir.ref<f32>
+  %6 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<3x3x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index):
+    %10 = hlfir.designate %1#0 (%arg3, %arg4)  : (!fir.ref<!fir.array<3x3xf32>>, index, index) -> !fir.ref<f32>
+    %11 = fir.load %10 : !fir.ref<f32>
+    %12 = arith.cmpf oge, %11, %5 : f32
+    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %13 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %1#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.ref<!fir.array<3x3xf32>>, !hlfir.expr<3x3x!fir.logical<4>>) -> !hlfir.expr<2xi32>
+  %8 = fir.shape %c2 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %3#0 (%c1:%c2:%c1)  shape %8 : (!fir.ref<!fir.array<3xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<2xi32>>
+  hlfir.assign %7 to %9 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
+  hlfir.destroy %7 : !hlfir.expr<2xi32>
+  hlfir.destroy %6 : !hlfir.expr<3x3x!fir.logical<4>>
+  return
+}
+// Not supported as the input is not a box
+// CHECK-LABEL: _QPtest_assignshape
+// CHECK: hlfir.minloc
+
+
+func.func @_QFPtest_character(%arg0: !fir.box<!fir.array<?x!fir.char<1>>> {fir.bindc_name = "b"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0:2 = hlfir.declare %arg0 typeparams %c1 {uniq_name = "_QFFtestEb"} : (!fir.box<!fir.array<?x!fir.char<1>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1>>>, !fir.box<!fir.array<?x!fir.char<1>>>)
+  %1:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestEc"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca !fir.array<1xi32> {bindc_name = "m", uniq_name = "_QFFtestEm"}
+  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %2(%3) {uniq_name = "_QFFtestEm"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %5 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %6:2 = hlfir.declare %5 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %8 = fir.load %7#0 : !fir.ref<i32>
+  %9:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %10 = fir.shape %9#1 : (index) -> !fir.shape<1>
+  %11 = hlfir.elemental %10 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %16 = hlfir.designate %1#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %17 = fir.load %16 : !fir.ref<i32>
+    %18 = arith.cmpi eq, %17, %8 : i32
+    %19 = fir.convert %18 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %19 : !fir.logical<4>
+  }
+  %12 = hlfir.minloc %0#0 mask %11 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x!fir.char<1>>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %12 to %4#0 : !hlfir.expr<1xi32>, !fir.ref<!fir.array<1xi32>>
+  hlfir.destroy %12 : !hlfir.expr<1xi32>
+  hlfir.destroy %11 : !hlfir.expr<?x!fir.logical<4>>
+  %13 = hlfir.designate %4#0 (%c1)  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+  %14 = fir.load %13 : !fir.ref<i32>
+  hlfir.assign %14 to %6#0 : i32, !fir.ref<i32>
+  %15 = fir.load %6#1 : !fir.ref<i32>
+  return %15 : i32
+}
+// Characters are not supported at the moment
+// CHECK-LABEL: _QFPtest_character
+// CHECK: hlfir.minloc
+
+
+func.func @_QPtest_parts(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "mask"}) -> f32 {
+  %c1 = arith.constant 1 : index
+  %c5 = arith.constant 5 : index
+  %c0 = arith.constant 0 : index
+  %c5_i32 = arith.constant 5 : i32
+  %0:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEmask"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1 = fir.alloca f32 {bindc_name = "test", uniq_name = "_QFtestEtest"}
+  %2:2 = hlfir.declare %1 {uniq_name = "_QFtestEtest"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %3:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEx"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg2: index):
+    %11 = hlfir.designate %0#0 (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %12 = fir.load %11 : !fir.ref<i32>
+    %13 = arith.cmpi sge, %12, %c5_i32 : i32
+    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %14 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %3#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %3#0 (%c5:%c5:%c1)  shape %8 : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+  hlfir.assign %7 to %9 : !hlfir.expr<1xi32>, !fir.box<!fir.array<1xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  %10 = fir.load %2#1 : !fir.ref<f32>
+  return %10 : f32
+}
+// Characters are not supported at the moment
+// CHECK-LABEL: _QPtest_parts
+// CHECK: fir.do_loop %{{.*}} = %c0 to %{{.*}} step %c1 iter_args(%{{.*}} = %c2147483647_i32) -> (i32) {
+
diff --git a/flang/test/HLFIR/minval-elemental.fir b/flang/test/HLFIR/minval-elemental.fir
new file mode 100644
index 0000000000000..64cd5403ec558
--- /dev/null
+++ b/flang/test/HLFIR/minval-elemental.fir
@@ -0,0 +1,95 @@
+// Test maxval inlining for both elemental and designate
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+// subroutine test(array)
+//   integer :: array(:), x
+//   x = minval(abs(array))
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+  %c31_i32 = arith.constant 31 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg1: index):
+    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.shrsi %9, %c31_i32 : i32
+    %11 = arith.xori %9, %10 : i32
+    %12 = arith.subi %11, %10 : i32
+    hlfir.yield_element %12 : i32
+  }
+  %7 = hlfir.minval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
+  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
+// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c2147483647_i32) -> (i32) {
+// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
+// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi slt, %[[V10]], %arg2 : i32
+// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
+// CHECK-NEXT:      fir.result %[[V12]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// subroutine test(array)
+//   real :: array(:), x
+//   x = minval(array(3:6))
+// end subroutine test
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
+  %c4 = arith.constant 4 : index
+  %c1 = arith.constant 1 : index
+  %c6 = arith.constant 6 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+  %6 = hlfir.minval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
+  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
+  return
+}
+
+// CHECK-LABEL: _QPtest_float
+// CHECK:       %cst = arith.constant 0x7F800000 : f32
+// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
+// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
+// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
+// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
+// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
+// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
+// CHECK-NEXT:      fir.result %[[V14]] : f32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
deleted file mode 100644
index fb0f39811ea74..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
+++ /dev/null
@@ -1,123 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
-  %0 = hlfir.all %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
-  return %0 : !fir.logical<4>
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant true
-// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
-// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
-// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
-// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_14:.*]] = arith.andi %[[VAL_13]], %[[VAL_11]] : i1
-// CHECK:               fir.result %[[VAL_14]] : i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_9]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
-// CHECK:           return %[[VAL_15]] : !fir.logical<4>
-// CHECK:         }
-
-func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.all %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
-  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant true
-// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
-// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
-// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
-// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
-// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_16:.*]] = arith.andi %[[VAL_15]], %[[VAL_13]] : i1
-// CHECK:               fir.result %[[VAL_16]] : i1
-// CHECK:             }
-// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
-// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
-// CHECK:           }
-// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
-// CHECK:         }
-
-func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
-  %0 = hlfir.all %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
-  return %0 : !fir.logical<4>
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant true
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
-// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
-// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
-// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
-// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
-// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
-// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_20]], %[[VAL_11]] : i1
-// CHECK:               fir.result %[[VAL_21]] : i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_9]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
-// CHECK:           return %[[VAL_22]] : !fir.logical<4>
-// CHECK:         }
-
-func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.all %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
-  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant true
-// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
-// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
-// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
-// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
-// CHECK:               %[[VAL_27:.*]] = arith.andi %[[VAL_26]], %[[VAL_14]] : i1
-// CHECK:               fir.result %[[VAL_27]] : i1
-// CHECK:             }
-// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
-// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
-// CHECK:           }
-// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
-// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
deleted file mode 100644
index 5bd76f3d24927..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
+++ /dev/null
@@ -1,123 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
-  %0 = hlfir.any %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
-  return %0 : !fir.logical<4>
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
-// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
-// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
-// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_14:.*]] = arith.ori %[[VAL_13]], %[[VAL_11]] : i1
-// CHECK:               fir.result %[[VAL_14]] : i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_9]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
-// CHECK:           return %[[VAL_15]] : !fir.logical<4>
-// CHECK:         }
-
-func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.any %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
-  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
-// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
-// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
-// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
-// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_16:.*]] = arith.ori %[[VAL_15]], %[[VAL_13]] : i1
-// CHECK:               fir.result %[[VAL_16]] : i1
-// CHECK:             }
-// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
-// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
-// CHECK:           }
-// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
-// CHECK:         }
-
-func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
-  %0 = hlfir.any %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
-  return %0 : !fir.logical<4>
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant false
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
-// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
-// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
-// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
-// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
-// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
-// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_21:.*]] = arith.ori %[[VAL_20]], %[[VAL_11]] : i1
-// CHECK:               fir.result %[[VAL_21]] : i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_9]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
-// CHECK:           return %[[VAL_22]] : !fir.logical<4>
-// CHECK:         }
-
-func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.any %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
-  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant false
-// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
-// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
-// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
-// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_14]] : i1
-// CHECK:               fir.result %[[VAL_27]] : i1
-// CHECK:             }
-// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
-// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
-// CHECK:           }
-// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
-// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
deleted file mode 100644
index 44594c646a368..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
+++ /dev/null
@@ -1,127 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
-  %0 = hlfir.count %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> i32
-  return %0 : i32
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (i32) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
-// CHECK:               %[[VAL_16:.*]] = arith.select %[[VAL_14]], %[[VAL_15]], %[[VAL_12]] : i32
-// CHECK:               fir.result %[[VAL_16]] : i32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : i32
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : i32
-// CHECK:         }
-
-func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.count %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?xi16>
-  return %0 : !hlfir.expr<?x?xi16>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i16
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]], %[[VAL_7]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi16> {
-// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (i16) {
-// CHECK:               %[[VAL_15:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_13]], %[[VAL_11]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
-// CHECK:               %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_14]], %[[VAL_1]] : i16
-// CHECK:               %[[VAL_18:.*]] = arith.select %[[VAL_16]], %[[VAL_17]], %[[VAL_14]] : i16
-// CHECK:               fir.result %[[VAL_18]] : i16
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_12]] : i16
-// CHECK:           }
-// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?xi16>
-// CHECK:         }
-
-func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
-  %0 = hlfir.count %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32
-  return %0 : i32
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (i32) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
-// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_15:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_16:.*]] = arith.addi %[[VAL_11]], %[[VAL_15]] : index
-// CHECK:               %[[VAL_17:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_18:.*]] = arith.addi %[[VAL_8]], %[[VAL_17]] : index
-// CHECK:               %[[VAL_19:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_16]], %[[VAL_18]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
-// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_21]], %[[VAL_22]], %[[VAL_12]] : i32
-// CHECK:               fir.result %[[VAL_23]] : i32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : i32
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : i32
-// CHECK:         }
-
-func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.count %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?xi64>
-  return %0 : !hlfir.expr<?x?xi64>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i64
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i64
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_6]]#1, %[[VAL_8]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi64> {
-// CHECK:           ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):
-// CHECK:             %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_15:.*]] = %[[VAL_2]]) -> (i64) {
-// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:               %[[VAL_21:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
-// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_14]], %[[VAL_21]] : index
-// CHECK:               %[[VAL_23:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_4]] : index
-// CHECK:               %[[VAL_24:.*]] = arith.addi %[[VAL_12]], %[[VAL_23]] : index
-// CHECK:               %[[VAL_25:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (!fir.logical<2>) -> i1
-// CHECK:               %[[VAL_28:.*]] = arith.addi %[[VAL_15]], %[[VAL_1]] : i64
-// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_15]] : i64
-// CHECK:               fir.result %[[VAL_29]] : i64
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_13]] : i64
-// CHECK:           }
-// CHECK:           return %[[VAL_10]] : !hlfir.expr<?x?xi64>
-// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
deleted file mode 100644
index 49e3bb6936338..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
+++ /dev/null
@@ -1,343 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
-  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_1d_total_expr(
-// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
-// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant -2147483648 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
-// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
-// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
-// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
-// CHECK:               %[[VAL_19:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_13]] : i32
-// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
-// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
-// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
-// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
-// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_26:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
-// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_28]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
-// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
-// CHECK:         }
-
-func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
-  %dim = arith.constant 1 : i16
-  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
-  return %0 : i32
-}
-// CHECK-LABEL:   func.func @test_1d_dim_expr(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
-// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
-// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
-// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
-// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
-// CHECK:               %[[VAL_18:.*]] = arith.cmpf ogt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
-// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
-// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
-// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
-// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
-// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
-// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
-// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_29:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (index) -> i32
-// CHECK:           return %[[VAL_30]] : i32
-// CHECK:         }
-
-func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
-  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  return %0 : !hlfir.expr<1xi16>
-}
-// CHECK-LABEL:   func.func @test_1d_total_var(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
-// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
-// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
-// CHECK:               %[[VAL_23:.*]] = arith.cmpf ogt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
-// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
-// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
-// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
-// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
-// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
-// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
-// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
-// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_34:.*]]#0, %[[VAL_6]] : i16
-// CHECK:           %[[VAL_35:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_34]]#0 : (i16) -> index
-// CHECK:           %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_35]]#0 : index
-// CHECK:           %[[VAL_38:.*]] = arith.subi %[[VAL_37]], %[[VAL_3]] : index
-// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_33]], %[[VAL_38]], %[[VAL_36]] : index
-// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (index) -> i16
-// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i16, !fir.ref<i16>
-// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
-// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
-// CHECK:         }
-
-func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
-  %dim = arith.constant 1 : i32
-  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
-  return %0 : i64
-}
-// CHECK-LABEL:   func.func @test_1d_dim_var(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant -1.7976931348623157E+308 : f64
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
-// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
-// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
-// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
-// CHECK:               %[[VAL_22:.*]] = arith.cmpf ogt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
-// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
-// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
-// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
-// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
-// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
-// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
-// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
-// CHECK:           }
-// CHECK:           %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_33:.*]]#0, %[[VAL_6]] : i64
-// CHECK:           %[[VAL_34:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]]#0 : (i64) -> index
-// CHECK:           %[[VAL_36:.*]] = arith.addi %[[VAL_35]], %[[VAL_34]]#0 : index
-// CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_36]], %[[VAL_3]] : index
-// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_32]], %[[VAL_37]], %[[VAL_35]] : index
-// CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
-// CHECK:           return %[[VAL_39]] : i64
-// CHECK:         }
-
-func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
-  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
-  return %0 : !hlfir.expr<3xi32>
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant false
-// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant true
-// CHECK:           %[[VAL_7:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
-// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
-// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
-// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
-// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
-// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
-// CHECK:                   %[[VAL_39:.*]] = arith.cmpf ogt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
-// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
-// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
-// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
-// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
-// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
-// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
-// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
-// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
-// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
-// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
-// CHECK:                 } else {
-// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
-// CHECK:                 }
-// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_56:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_57:.*]] = fir.convert %[[VAL_55]] : (index) -> i32
-// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_57]] to %[[VAL_58]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_59:.*]] = fir.convert %[[VAL_56]]#1 : (i32) -> index
-// CHECK:           %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (index) -> i32
-// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_60]] to %[[VAL_61]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_62:.*]] = fir.convert %[[VAL_56]]#2 : (i32) -> index
-// CHECK:           %[[VAL_63:.*]] = fir.convert %[[VAL_62]] : (index) -> i32
-// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_63]] to %[[VAL_64]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
-// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
-// CHECK:         }
-
-func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
-  return %0 : !hlfir.expr<?x?xi32>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant true
-// CHECK:           %[[VAL_4:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
-// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
-// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
-// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
-// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
-// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
-// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
-// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
-// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
-// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
-// CHECK:               } else {
-// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
-// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
-// CHECK:               }
-// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
-// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
-// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
-// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
-// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
-// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
-// CHECK:                 %[[VAL_48:.*]] = arith.cmpf ogt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
-// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
-// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
-// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
-// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
-// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
-// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
-// CHECK:             }
-// CHECK:             %[[VAL_58:.*]] = arith.cmpi ne, %[[VAL_59:.*]]#0, %[[VAL_5]] : i32
-// CHECK:             %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_59]]#0 : (i32) -> index
-// CHECK:             %[[VAL_62:.*]] = arith.addi %[[VAL_61]], %[[VAL_60]]#0 : index
-// CHECK:             %[[VAL_63:.*]] = arith.subi %[[VAL_62]], %[[VAL_7]] : index
-// CHECK:             %[[VAL_64:.*]] = arith.select %[[VAL_58]], %[[VAL_63]], %[[VAL_61]] : index
-// CHECK:             %[[VAL_65:.*]] = fir.convert %[[VAL_64]] : (index) -> i32
-// CHECK:             hlfir.yield_element %[[VAL_65]] : i32
-// CHECK:           }
-// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
-// CHECK:         }
-
-// Character comparisons are not supported yet.
-func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
-  %0 = hlfir.maxloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_character(
-// CHECK:           hlfir.maxloc
-
-// BACK is not supported yet.
-func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
-  %back = arith.constant true
-  %0 = hlfir.maxloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_back(
-// CHECK:           hlfir.maxloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
deleted file mode 100644
index 4022b7d3b2b77..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
+++ /dev/null
@@ -1,177 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
-  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
-  return %0 : f32
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (f32) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f32) {
-// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
-// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
-// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
-// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f32
-// CHECK:                 fir.result %[[VAL_22]] : f32
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f32
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_15]] : f32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : f32
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : f32
-// CHECK:         }
-
-func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
-  %dim = arith.constant 1 : i32
-  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
-  return %0 : !hlfir.expr<?xf64>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
-// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant -1.7976931348623157E+308 : f64
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
-// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f64) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f64) {
-// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
-// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
-// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
-// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f64
-// CHECK:                 fir.result %[[VAL_22]] : f64
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f64
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_15]] : f64
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_10]] : f64
-// CHECK:           }
-// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
-// CHECK:         }
-
-func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
-  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
-  return %0 : f16
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
-// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant -6.550400e+04 : f16
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (f16) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f16) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
-// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
-// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f16) {
-// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
-// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
-// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
-// CHECK:                 %[[VAL_25:.*]] = arith.cmpf ogt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
-// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
-// CHECK:                 %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_24]], %[[VAL_12]] : f16
-// CHECK:                 fir.result %[[VAL_30]] : f16
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f16
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_16]] : f16
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : f16
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : f16
-// CHECK:         }
-
-func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
-  return %0 : !hlfir.expr<?xf16>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant true
-// CHECK:           %[[VAL_3:.*]] = arith.constant -6.550400e+04 : f16
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
-// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
-// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (f16) {
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
-// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
-// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
-// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
-// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
-// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
-// CHECK:               } else {
-// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
-// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
-// CHECK:               }
-// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (f16) {
-// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
-// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
-// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
-// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
-// CHECK:                 %[[VAL_35:.*]] = arith.cmpf ogt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
-// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
-// CHECK:                 %[[VAL_40:.*]] = arith.select %[[VAL_39]], %[[VAL_34]], %[[VAL_14]] : f16
-// CHECK:                 fir.result %[[VAL_40]] : f16
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_14]] : f16
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_26]] : f16
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_12]] : f16
-// CHECK:           }
-// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
-// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
deleted file mode 100644
index 96dcebaf9a2cd..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
+++ /dev/null
@@ -1,343 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
-  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_1d_total_expr(
-// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
-// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
-// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
-// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
-// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
-// CHECK:               %[[VAL_19:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_13]] : i32
-// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
-// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
-// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
-// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
-// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_26:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
-// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_28]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
-// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
-// CHECK:         }
-
-func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
-  %dim = arith.constant 1 : i16
-  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
-  return %0 : i32
-}
-// CHECK-LABEL:   func.func @test_1d_dim_expr(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
-// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
-// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
-// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
-// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
-// CHECK:               %[[VAL_18:.*]] = arith.cmpf olt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
-// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
-// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
-// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
-// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
-// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
-// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
-// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_29:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (index) -> i32
-// CHECK:           return %[[VAL_30]] : i32
-// CHECK:         }
-
-func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
-  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  return %0 : !hlfir.expr<1xi16>
-}
-// CHECK-LABEL:   func.func @test_1d_total_var(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
-// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
-// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
-// CHECK:               %[[VAL_23:.*]] = arith.cmpf olt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
-// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
-// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
-// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
-// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
-// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
-// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
-// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
-// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_34:.*]]#0, %[[VAL_6]] : i16
-// CHECK:           %[[VAL_35:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_34]]#0 : (i16) -> index
-// CHECK:           %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_35]]#0 : index
-// CHECK:           %[[VAL_38:.*]] = arith.subi %[[VAL_37]], %[[VAL_3]] : index
-// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_33]], %[[VAL_38]], %[[VAL_36]] : index
-// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (index) -> i16
-// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i16, !fir.ref<i16>
-// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
-// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
-// CHECK:         }
-
-func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
-  %dim = arith.constant 1 : i32
-  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
-  return %0 : i64
-}
-// CHECK-LABEL:   func.func @test_1d_dim_var(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 1.7976931348623157E+308 : f64
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
-// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
-// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
-// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
-// CHECK:               %[[VAL_22:.*]] = arith.cmpf olt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
-// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
-// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
-// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
-// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
-// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
-// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
-// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
-// CHECK:           }
-// CHECK:           %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_33:.*]]#0, %[[VAL_6]] : i64
-// CHECK:           %[[VAL_34:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]]#0 : (i64) -> index
-// CHECK:           %[[VAL_36:.*]] = arith.addi %[[VAL_35]], %[[VAL_34]]#0 : index
-// CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_36]], %[[VAL_3]] : index
-// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_32]], %[[VAL_37]], %[[VAL_35]] : index
-// CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
-// CHECK:           return %[[VAL_39]] : i64
-// CHECK:         }
-
-func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
-  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
-  return %0 : !hlfir.expr<3xi32>
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant false
-// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant true
-// CHECK:           %[[VAL_7:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
-// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
-// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
-// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
-// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
-// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
-// CHECK:                   %[[VAL_39:.*]] = arith.cmpf olt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
-// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
-// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
-// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
-// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
-// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
-// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
-// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
-// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
-// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
-// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
-// CHECK:                 } else {
-// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
-// CHECK:                 }
-// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_56:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_57:.*]] = fir.convert %[[VAL_55]] : (index) -> i32
-// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_57]] to %[[VAL_58]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_59:.*]] = fir.convert %[[VAL_56]]#1 : (i32) -> index
-// CHECK:           %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (index) -> i32
-// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_60]] to %[[VAL_61]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_62:.*]] = fir.convert %[[VAL_56]]#2 : (i32) -> index
-// CHECK:           %[[VAL_63:.*]] = fir.convert %[[VAL_62]] : (index) -> i32
-// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_63]] to %[[VAL_64]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
-// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
-// CHECK:         }
-
-func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
-  return %0 : !hlfir.expr<?x?xi32>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant true
-// CHECK:           %[[VAL_4:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
-// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
-// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
-// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
-// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
-// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
-// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
-// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
-// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
-// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
-// CHECK:               } else {
-// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
-// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
-// CHECK:               }
-// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
-// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
-// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
-// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
-// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
-// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
-// CHECK:                 %[[VAL_48:.*]] = arith.cmpf olt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
-// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
-// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
-// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
-// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
-// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
-// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
-// CHECK:             }
-// CHECK:             %[[VAL_58:.*]] = arith.cmpi ne, %[[VAL_59:.*]]#0, %[[VAL_5]] : i32
-// CHECK:             %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_59]]#0 : (i32) -> index
-// CHECK:             %[[VAL_62:.*]] = arith.addi %[[VAL_61]], %[[VAL_60]]#0 : index
-// CHECK:             %[[VAL_63:.*]] = arith.subi %[[VAL_62]], %[[VAL_7]] : index
-// CHECK:             %[[VAL_64:.*]] = arith.select %[[VAL_58]], %[[VAL_63]], %[[VAL_61]] : index
-// CHECK:             %[[VAL_65:.*]] = fir.convert %[[VAL_64]] : (index) -> i32
-// CHECK:             hlfir.yield_element %[[VAL_65]] : i32
-// CHECK:           }
-// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
-// CHECK:         }
-
-// Character comparisons are not supported yet.
-func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
-  %0 = hlfir.minloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_character(
-// CHECK:           hlfir.minloc
-
-// BACK is not supported yet.
-func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
-  %back = arith.constant true
-  %0 = hlfir.minloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_back(
-// CHECK:           hlfir.minloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
deleted file mode 100644
index 4278ac7abd457..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
+++ /dev/null
@@ -1,177 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
-  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
-  return %0 : f32
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (f32) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f32) {
-// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
-// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
-// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
-// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f32
-// CHECK:                 fir.result %[[VAL_22]] : f32
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f32
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_15]] : f32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : f32
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : f32
-// CHECK:         }
-
-func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
-  %dim = arith.constant 1 : i32
-  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
-  return %0 : !hlfir.expr<?xf64>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
-// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1.7976931348623157E+308 : f64
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
-// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f64) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f64) {
-// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
-// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
-// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
-// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f64
-// CHECK:                 fir.result %[[VAL_22]] : f64
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f64
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_15]] : f64
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_10]] : f64
-// CHECK:           }
-// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
-// CHECK:         }
-
-func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
-  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
-  return %0 : f16
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
-// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 6.550400e+04 : f16
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (f16) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f16) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
-// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
-// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f16) {
-// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
-// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
-// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
-// CHECK:                 %[[VAL_25:.*]] = arith.cmpf olt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
-// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
-// CHECK:                 %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_24]], %[[VAL_12]] : f16
-// CHECK:                 fir.result %[[VAL_30]] : f16
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f16
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_16]] : f16
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : f16
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : f16
-// CHECK:         }
-
-func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
-  return %0 : !hlfir.expr<?xf16>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant true
-// CHECK:           %[[VAL_3:.*]] = arith.constant 6.550400e+04 : f16
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
-// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
-// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (f16) {
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
-// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
-// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
-// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
-// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
-// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
-// CHECK:               } else {
-// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
-// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
-// CHECK:               }
-// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (f16) {
-// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
-// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
-// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
-// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
-// CHECK:                 %[[VAL_35:.*]] = arith.cmpf olt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
-// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
-// CHECK:                 %[[VAL_40:.*]] = arith.select %[[VAL_39]], %[[VAL_34]], %[[VAL_14]] : f16
-// CHECK:                 fir.result %[[VAL_40]] : f16
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_14]] : f16
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_26]] : f16
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_12]] : f16
-// CHECK:           }
-// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
-// CHECK:         }

From 69ade7c090b59373a18c38b2c80c48f97a0afcde Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang@sifive.com>
Date: Fri, 18 Apr 2025 06:51:36 +0800
Subject: [PATCH 338/710] [LV] Check if the VF is scalar by VFRange in
 `handleUncountableEarlyExit`. (#135294)

This patch check if the plan contains scalar VF by VFRange instead of
Plan.
This patch also clamp the range to contains either only scalar or only
vector VFs to prevent mis-compile.

Split from #113903.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp   |  2 +-
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 14 +++++++++++---
 llvm/lib/Transforms/Vectorize/VPlanTransforms.h   |  4 +++-
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4c1ed15ee700f..db2fd300cb8f5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9779,7 +9779,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
           Legal->getUncountableEarlyExitingBlock()) {
     VPlanTransforms::runPass(VPlanTransforms::handleUncountableEarlyExit, *Plan,
                              *PSE.getSE(), OrigLoop, UncountableExitingBlock,
-                             RecipeBuilder);
+                             RecipeBuilder, Range);
   }
   DenseMap<VPValue *, VPValue *> IVEndValues;
   addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9d8f1706cf61b..b80fe18d1bd66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -17,6 +17,7 @@
 #include "VPlanAnalysis.h"
 #include "VPlanCFG.h"
 #include "VPlanDominatorTree.h"
+#include "VPlanHelpers.h"
 #include "VPlanPatternMatch.h"
 #include "VPlanUtils.h"
 #include "VPlanVerifier.h"
@@ -2457,7 +2458,8 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
 
 void VPlanTransforms::handleUncountableEarlyExit(
     VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop,
-    BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder) {
+    BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder,
+    VFRange &Range) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   auto *LatchVPBB = cast<VPBasicBlock>(LoopRegion->getExiting());
   VPBuilder Builder(LatchVPBB->getTerminator());
@@ -2512,8 +2514,14 @@ void VPlanTransforms::handleUncountableEarlyExit(
       ExitIRI->addOperand(IncomingFromLatch);
       ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
     }
-    // Add the incoming value from the early exit.
-    if (!IncomingFromEarlyExit->isLiveIn() && !Plan.hasScalarVFOnly()) {
+
+    auto IsVector = [](ElementCount VF) { return VF.isVector(); };
+    // When the VFs are vectors, need to add `extract` to get the incoming value
+    // from early exit. When the range contains scalar VF, limit the range to
+    // scalar VF to prevent mis-compilation for the range containing both scalar
+    // and vector VFs.
+    if (!IncomingFromEarlyExit->isLiveIn() &&
+        LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) {
       VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
           VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
           "first.active.lane");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index a9461b261ddb6..1200e8eaab0ba 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -28,6 +28,7 @@ class PredicatedScalarEvolution;
 class TargetLibraryInfo;
 class VPBuilder;
 class VPRecipeBuilder;
+class VFRange;
 
 extern cl::opt<bool> VerifyEachVPlan;
 
@@ -173,7 +174,8 @@ struct VPlanTransforms {
   static void handleUncountableEarlyExit(VPlan &Plan, ScalarEvolution &SE,
                                          Loop *OrigLoop,
                                          BasicBlock *UncountableExitingBlock,
-                                         VPRecipeBuilder &RecipeBuilder);
+                                         VPRecipeBuilder &RecipeBuilder,
+                                         VFRange &Range);
 
   /// Lower abstract recipes to concrete ones, that can be codegen'd. Use \p
   /// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis.

From 32311a6b68d3de4642599abe14922c686bdb30fc Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 17 Apr 2025 15:42:48 -0700
Subject: [PATCH 339/710] Reland [flang] Generalized simplification of HLFIR
 reduction ops. (#136071)

This change generalizes SumAsElemental inlining in
SimplifyHLFIRIntrinsics pass so that it can be applied
to ALL, ANY, COUNT, MAXLOC, MAXVAL, MINLOC, MINVAL, SUM.

This change makes the special handling of the reduction
operations in OptimizedBufferization redundant: once HLFIR
operations are inlined, the hlfir.elemental inlining should
do the rest of the job.
---
 .../flang/Optimizer/Builder/HLFIRTools.h      |    5 +
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |   27 +
 .../Transforms/OptimizedBufferization.cpp     |  465 --------
 .../Transforms/SimplifyHLFIRIntrinsics.cpp    | 1038 +++++++++++++----
 flang/test/HLFIR/all-elemental.fir            |   91 --
 flang/test/HLFIR/any-elemental.fir            |  190 ---
 flang/test/HLFIR/count-elemental.fir          |  314 -----
 flang/test/HLFIR/maxloc-elemental.fir         |  133 ---
 flang/test/HLFIR/maxval-elemental.fir         |  117 --
 flang/test/HLFIR/minloc-elemental.fir         |  397 -------
 flang/test/HLFIR/minval-elemental.fir         |   95 --
 .../HLFIR/simplify-hlfir-intrinsics-all.fir   |  123 ++
 .../HLFIR/simplify-hlfir-intrinsics-any.fir   |  123 ++
 .../HLFIR/simplify-hlfir-intrinsics-count.fir |  127 ++
 .../simplify-hlfir-intrinsics-maxloc.fir      |  343 ++++++
 .../simplify-hlfir-intrinsics-maxval.fir      |  177 +++
 .../simplify-hlfir-intrinsics-minloc.fir      |  343 ++++++
 .../simplify-hlfir-intrinsics-minval.fir      |  177 +++
 18 files changed, 2286 insertions(+), 1999 deletions(-)
 delete mode 100644 flang/test/HLFIR/all-elemental.fir
 delete mode 100644 flang/test/HLFIR/any-elemental.fir
 delete mode 100644 flang/test/HLFIR/count-elemental.fir
 delete mode 100644 flang/test/HLFIR/maxloc-elemental.fir
 delete mode 100644 flang/test/HLFIR/maxval-elemental.fir
 delete mode 100644 flang/test/HLFIR/minloc-elemental.fir
 delete mode 100644 flang/test/HLFIR/minval-elemental.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir

diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index ac80873dc374f..cd259b9dc6071 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -301,6 +301,11 @@ mlir::Value genExtent(mlir::Location loc, fir::FirOpBuilder &builder,
 mlir::Value genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
                       hlfir::Entity entity, unsigned dim);
 
+/// Compute the lower bounds of \p entity, which is an array of known rank.
+llvm::SmallVector<mlir::Value> genLBounds(mlir::Location loc,
+                                          fir::FirOpBuilder &builder,
+                                          hlfir::Entity entity);
+
 /// Generate a vector of extents with index type from a fir.shape
 /// of fir.shape_shift value.
 llvm::SmallVector<mlir::Value> getIndexExtents(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 558ebcb876ddb..dccad8095979d 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -659,6 +659,33 @@ mlir::Value hlfir::genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
   return dimInfo.getLowerBound();
 }
 
+llvm::SmallVector<mlir::Value> hlfir::genLBounds(mlir::Location loc,
+                                                 fir::FirOpBuilder &builder,
+                                                 hlfir::Entity entity) {
+  assert(!entity.isAssumedRank() &&
+         "cannot compute all lower bounds for assumed rank");
+  assert(!entity.isScalar() && "expected an array entity");
+  int rank = entity.getRank();
+  mlir::Type idxTy = builder.getIndexType();
+  if (!entity.mayHaveNonDefaultLowerBounds())
+    return {static_cast<std::size_t>(rank),
+            builder.createIntegerConstant(loc, idxTy, 1)};
+
+  if (auto shape = tryRetrievingShapeOrShift(entity)) {
+    auto lbounds = getExplicitLboundsFromShape(shape);
+    if (!lbounds.empty())
+      return lbounds;
+  }
+
+  if (entity.isMutableBox())
+    entity = hlfir::derefPointersAndAllocatables(loc, builder, entity);
+
+  llvm::SmallVector<mlir::Value> lbounds;
+  fir::factory::genDimInfoFromBox(builder, loc, entity, &lbounds,
+                                  /*extents=*/nullptr, /*strides=*/nullptr);
+  return lbounds;
+}
+
 void hlfir::genLengthParameters(mlir::Location loc, fir::FirOpBuilder &builder,
                                 Entity entity,
                                 llvm::SmallVectorImpl<mlir::Value> &result) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index c489450384a35..79aabd2981e1a 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -772,458 +772,6 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   return mlir::success();
 }
 
-using GenBodyFn =
-    std::function<mlir::Value(fir::FirOpBuilder &, mlir::Location, mlir::Value,
-                              const llvm::SmallVectorImpl<mlir::Value> &)>;
-static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
-                                         mlir::Location loc, mlir::Value init,
-                                         mlir::Value shape, GenBodyFn genBody) {
-  auto extents = hlfir::getIndexExtents(loc, builder, shape);
-  mlir::Value reduction = init;
-  mlir::IndexType idxTy = builder.getIndexType();
-  mlir::Value oneIdx = builder.createIntegerConstant(loc, idxTy, 1);
-
-  // Create a reduction loop nest. We use one-based indices so that they can be
-  // passed to the elemental, and reverse the order so that they can be
-  // generated in column-major order for better performance.
-  llvm::SmallVector<mlir::Value> indices(extents.size(), mlir::Value{});
-  for (unsigned i = 0; i < extents.size(); ++i) {
-    auto loop = builder.create<fir::DoLoopOp>(
-        loc, oneIdx, extents[extents.size() - i - 1], oneIdx, false,
-        /*finalCountValue=*/false, reduction);
-    reduction = loop.getRegionIterArgs()[0];
-    indices[extents.size() - i - 1] = loop.getInductionVar();
-    // Set insertion point to the loop body so that the next loop
-    // is inserted inside the current one.
-    builder.setInsertionPointToStart(loop.getBody());
-  }
-
-  // Generate the body
-  reduction = genBody(builder, loc, reduction, indices);
-
-  // Unwind the loop nest.
-  for (unsigned i = 0; i < extents.size(); ++i) {
-    auto result = builder.create<fir::ResultOp>(loc, reduction);
-    auto loop = mlir::cast<fir::DoLoopOp>(result->getParentOp());
-    reduction = loop.getResult(0);
-    // Set insertion point after the loop operation that we have
-    // just processed.
-    builder.setInsertionPointAfter(loop.getOperation());
-  }
-
-  return reduction;
-}
-
-auto makeMinMaxInitValGenerator(bool isMax) {
-  return [isMax](fir::FirOpBuilder builder, mlir::Location loc,
-                 mlir::Type elementType) -> mlir::Value {
-    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
-      const llvm::fltSemantics &sem = ty.getFloatSemantics();
-      llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
-      return builder.createRealConstant(loc, elementType, limit);
-    }
-    unsigned bits = elementType.getIntOrFloatBitWidth();
-    int64_t limitInt =
-        isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
-              : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
-    return builder.createIntegerConstant(loc, elementType, limitInt);
-  };
-}
-
-mlir::Value generateMinMaxComparison(fir::FirOpBuilder builder,
-                                     mlir::Location loc, mlir::Value elem,
-                                     mlir::Value reduction, bool isMax) {
-  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
-    // For FP reductions we want the first smallest value to be used, that
-    // is not NaN. A OGL/OLT condition will usually work for this unless all
-    // the values are Nan or Inf. This follows the same logic as
-    // NumericCompare for Minloc/Maxlox in extrema.cpp.
-    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
-        loc,
-        isMax ? mlir::arith::CmpFPredicate::OGT
-              : mlir::arith::CmpFPredicate::OLT,
-        elem, reduction);
-    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
-  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
-    return builder.create<mlir::arith::CmpIOp>(
-        loc,
-        isMax ? mlir::arith::CmpIPredicate::sgt
-              : mlir::arith::CmpIPredicate::slt,
-        elem, reduction);
-  }
-  llvm_unreachable("unsupported type");
-}
-
-/// Given a reduction operation with an elemental/designate source, attempt to
-/// generate a do-loop to perform the operation inline.
-///   %e = hlfir.elemental %shape unordered
-///   %r = hlfir.count %e
-/// =>
-///   %r = for.do_loop %arg = 1 to bound(%shape) step 1 iter_args(%arg2 = init)
-///     %i = <inline elemental>
-///     %c = <reduce count> %i
-///     fir.result %c
-template <typename Op>
-class ReductionConversion : public mlir::OpRewritePattern<Op> {
-public:
-  using mlir::OpRewritePattern<Op>::OpRewritePattern;
-
-  llvm::LogicalResult
-  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
-    mlir::Location loc = op.getLoc();
-    // Select source and validate its arguments.
-    mlir::Value source;
-    bool valid = false;
-    if constexpr (std::is_same_v<Op, hlfir::AnyOp> ||
-                  std::is_same_v<Op, hlfir::AllOp> ||
-                  std::is_same_v<Op, hlfir::CountOp>) {
-      source = op.getMask();
-      valid = !op.getDim();
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
-                         std::is_same_v<Op, hlfir::MinvalOp>) {
-      source = op.getArray();
-      valid = !op.getDim() && !op.getMask();
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
-                         std::is_same_v<Op, hlfir::MinlocOp>) {
-      source = op.getArray();
-      valid = !op.getDim() && !op.getMask() && !op.getBack();
-    }
-    if (!valid)
-      return rewriter.notifyMatchFailure(
-          op, "Currently does not accept optional arguments");
-
-    hlfir::ElementalOp elemental;
-    hlfir::DesignateOp designate;
-    mlir::Value shape;
-    if ((elemental = source.template getDefiningOp<hlfir::ElementalOp>())) {
-      shape = elemental.getOperand(0);
-    } else if ((designate =
-                    source.template getDefiningOp<hlfir::DesignateOp>())) {
-      shape = designate.getShape();
-    } else {
-      return rewriter.notifyMatchFailure(op, "Did not find valid argument");
-    }
-
-    auto inlineSource =
-        [elemental,
-         &designate](fir::FirOpBuilder builder, mlir::Location loc,
-                     const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-        -> mlir::Value {
-      if (elemental) {
-        // Inline the elemental and get the value from it.
-        auto yield =
-            inlineElementalOp(loc, builder, elemental, oneBasedIndices);
-        auto tmp = yield.getElementValue();
-        yield->erase();
-        return tmp;
-      }
-      if (designate) {
-        // Create a designator over the array designator, then load the
-        // reference.
-        mlir::Value elementAddr = hlfir::getElementAt(
-            loc, builder, hlfir::Entity{designate.getResult()},
-            oneBasedIndices);
-        return builder.create<fir::LoadOp>(loc, elementAddr);
-      }
-      llvm_unreachable("unsupported type");
-    };
-
-    fir::FirOpBuilder builder{rewriter, op.getOperation()};
-
-    mlir::Value init;
-    GenBodyFn genBodyFn;
-    if constexpr (std::is_same_v<Op, hlfir::AnyOp>) {
-      init = builder.createIntegerConstant(loc, builder.getI1Type(), 0);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally set the reduction variable.
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        return builder.create<mlir::arith::OrIOp>(loc, reduction, cond);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::AllOp>) {
-      init = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally set the reduction variable.
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        return builder.create<mlir::arith::AndIOp>(loc, reduction, cond);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
-      init = builder.createIntegerConstant(loc, op.getType(), 0);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally add one to the current value
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        mlir::Value one =
-            builder.createIntegerConstant(loc, reduction.getType(), 1);
-        mlir::Value add1 =
-            builder.create<mlir::arith::AddIOp>(loc, reduction, one);
-        return builder.create<mlir::arith::SelectOp>(loc, cond, add1,
-                                                     reduction);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
-                         std::is_same_v<Op, hlfir::MinlocOp>) {
-      // TODO: implement minloc/maxloc conversion.
-      return rewriter.notifyMatchFailure(
-          op, "Currently minloc/maxloc is not handled");
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
-                         std::is_same_v<Op, hlfir::MinvalOp>) {
-      mlir::Type ty = op.getType();
-      if (!(mlir::isa<mlir::FloatType>(ty) ||
-            mlir::isa<mlir::IntegerType>(ty))) {
-        return rewriter.notifyMatchFailure(
-            op, "Type is not supported for Maxval or Minval yet");
-      }
-
-      bool isMax = std::is_same_v<Op, hlfir::MaxvalOp>;
-      init = makeMinMaxInitValGenerator(isMax)(builder, loc, ty);
-      genBodyFn = [inlineSource, isMax](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        mlir::Value val = inlineSource(builder, loc, oneBasedIndices);
-        mlir::Value cmp =
-            generateMinMaxComparison(builder, loc, val, reduction, isMax);
-        return builder.create<mlir::arith::SelectOp>(loc, cmp, val, reduction);
-      };
-    } else {
-      llvm_unreachable("unsupported type");
-    }
-
-    mlir::Value res =
-        generateReductionLoop(builder, loc, init, shape, genBodyFn);
-    if (res.getType() != op.getType())
-      res = builder.create<fir::ConvertOp>(loc, op.getType(), res);
-
-    // Check if the op was the only user of the source (apart from a destroy),
-    // and remove it if so.
-    mlir::Operation *sourceOp = source.getDefiningOp();
-    mlir::Operation::user_range srcUsers = sourceOp->getUsers();
-    hlfir::DestroyOp srcDestroy;
-    if (std::distance(srcUsers.begin(), srcUsers.end()) == 2) {
-      srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*srcUsers.begin());
-      if (!srcDestroy)
-        srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++srcUsers.begin());
-    }
-
-    rewriter.replaceOp(op, res);
-    if (srcDestroy) {
-      rewriter.eraseOp(srcDestroy);
-      rewriter.eraseOp(sourceOp);
-    }
-    return mlir::success();
-  }
-};
-
-// Look for minloc(mask=elemental) and generate the minloc loop with
-// inlined elemental.
-//  %e = hlfir.elemental %shape ({ ... })
-//  %m = hlfir.minloc %array mask %e
-template <typename Op>
-class ReductionMaskConversion : public mlir::OpRewritePattern<Op> {
-public:
-  using mlir::OpRewritePattern<Op>::OpRewritePattern;
-
-  llvm::LogicalResult
-  matchAndRewrite(Op mloc, mlir::PatternRewriter &rewriter) const override {
-    if (!mloc.getMask() || mloc.getDim() || mloc.getBack())
-      return rewriter.notifyMatchFailure(mloc,
-                                         "Did not find valid minloc/maxloc");
-
-    bool isMax = std::is_same_v<Op, hlfir::MaxlocOp>;
-
-    auto elemental =
-        mloc.getMask().template getDefiningOp<hlfir::ElementalOp>();
-    if (!elemental || hlfir::elementalOpMustProduceTemp(elemental))
-      return rewriter.notifyMatchFailure(mloc, "Did not find elemental");
-
-    mlir::Value array = mloc.getArray();
-
-    unsigned rank = mlir::cast<hlfir::ExprType>(mloc.getType()).getShape()[0];
-    mlir::Type arrayType = array.getType();
-    if (!mlir::isa<fir::BoxType>(arrayType))
-      return rewriter.notifyMatchFailure(
-          mloc, "Currently requires a boxed type input");
-    mlir::Type elementType = hlfir::getFortranElementType(arrayType);
-    if (!fir::isa_trivial(elementType))
-      return rewriter.notifyMatchFailure(
-          mloc, "Character arrays are currently not handled");
-
-    mlir::Location loc = mloc.getLoc();
-    fir::FirOpBuilder builder{rewriter, mloc.getOperation()};
-    mlir::Value resultArr = builder.createTemporary(
-        loc, fir::SequenceType::get(
-                 rank, hlfir::getFortranElementType(mloc.getType())));
-
-    auto init = makeMinMaxInitValGenerator(isMax);
-
-    auto genBodyOp =
-        [&rank, &resultArr, &elemental, isMax](
-            fir::FirOpBuilder builder, mlir::Location loc,
-            mlir::Type elementType, mlir::Value array, mlir::Value flagRef,
-            mlir::Value reduction,
-            const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {
-      // We are in the innermost loop: generate the elemental inline
-      mlir::Value oneIdx =
-          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-      llvm::SmallVector<mlir::Value> oneBasedIndices;
-      llvm::transform(
-          indices, std::back_inserter(oneBasedIndices), [&](mlir::Value V) {
-            return builder.create<mlir::arith::AddIOp>(loc, V, oneIdx);
-          });
-      hlfir::YieldElementOp yield =
-          hlfir::inlineElementalOp(loc, builder, elemental, oneBasedIndices);
-      mlir::Value maskElem = yield.getElementValue();
-      yield->erase();
-
-      mlir::Type ifCompatType = builder.getI1Type();
-      mlir::Value ifCompatElem =
-          builder.create<fir::ConvertOp>(loc, ifCompatType, maskElem);
-
-      llvm::SmallVector<mlir::Type> resultsTy = {elementType, elementType};
-      fir::IfOp maskIfOp =
-          builder.create<fir::IfOp>(loc, elementType, ifCompatElem,
-                                    /*withElseRegion=*/true);
-      builder.setInsertionPointToStart(&maskIfOp.getThenRegion().front());
-
-      // Set flag that mask was true at some point
-      mlir::Value flagSet = builder.createIntegerConstant(
-          loc, mlir::cast<fir::ReferenceType>(flagRef.getType()).getEleTy(), 1);
-      mlir::Value isFirst = builder.create<fir::LoadOp>(loc, flagRef);
-      mlir::Value addr = hlfir::getElementAt(loc, builder, hlfir::Entity{array},
-                                             oneBasedIndices);
-      mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
-
-      // Compare with the max reduction value
-      mlir::Value cmp =
-          generateMinMaxComparison(builder, loc, elem, reduction, isMax);
-
-      // The condition used for the loop is isFirst || <the condition above>.
-      isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
-      isFirst = builder.create<mlir::arith::XOrIOp>(
-          loc, isFirst, builder.createIntegerConstant(loc, cmp.getType(), 1));
-      cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, isFirst);
-
-      // Set the new coordinate to the result
-      fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, cmp,
-                                                 /*withElseRegion*/ true);
-
-      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      builder.create<fir::StoreOp>(loc, flagSet, flagRef);
-      mlir::Type resultElemTy =
-          hlfir::getFortranElementType(resultArr.getType());
-      mlir::Type returnRefTy = builder.getRefType(resultElemTy);
-      mlir::IndexType idxTy = builder.getIndexType();
-
-      for (unsigned int i = 0; i < rank; ++i) {
-        mlir::Value index = builder.createIntegerConstant(loc, idxTy, i + 1);
-        mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
-            loc, returnRefTy, resultArr, index);
-        mlir::Value fortranIndex = builder.create<fir::ConvertOp>(
-            loc, resultElemTy, oneBasedIndices[i]);
-        builder.create<fir::StoreOp>(loc, fortranIndex, resultElemAddr);
-      }
-      builder.create<fir::ResultOp>(loc, elem);
-      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, reduction);
-      builder.setInsertionPointAfter(ifOp);
-
-      // Close the mask if
-      builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
-      builder.setInsertionPointToStart(&maskIfOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, reduction);
-      builder.setInsertionPointAfter(maskIfOp);
-
-      return maskIfOp.getResult(0);
-    };
-    auto getAddrFn = [](fir::FirOpBuilder builder, mlir::Location loc,
-                        const mlir::Type &resultElemType, mlir::Value resultArr,
-                        mlir::Value index) {
-      mlir::Type resultRefTy = builder.getRefType(resultElemType);
-      mlir::Value oneIdx =
-          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-      index = builder.create<mlir::arith::AddIOp>(loc, index, oneIdx);
-      return builder.create<hlfir::DesignateOp>(loc, resultRefTy, resultArr,
-                                                index);
-    };
-
-    // Initialize the result
-    mlir::Type resultElemTy = hlfir::getFortranElementType(resultArr.getType());
-    mlir::Type resultRefTy = builder.getRefType(resultElemTy);
-    mlir::Value returnValue =
-        builder.createIntegerConstant(loc, resultElemTy, 0);
-    for (unsigned int i = 0; i < rank; ++i) {
-      mlir::Value index =
-          builder.createIntegerConstant(loc, builder.getIndexType(), i + 1);
-      mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
-          loc, resultRefTy, resultArr, index);
-      builder.create<fir::StoreOp>(loc, returnValue, resultElemAddr);
-    }
-
-    fir::genMinMaxlocReductionLoop(builder, array, init, genBodyOp, getAddrFn,
-                                   rank, elementType, loc, builder.getI1Type(),
-                                   resultArr, false);
-
-    mlir::Value asExpr = builder.create<hlfir::AsExprOp>(
-        loc, resultArr, builder.createBool(loc, false));
-
-    // Check all the users - the destroy is no longer required, and any assign
-    // can use resultArr directly so that InlineHLFIRAssign pass
-    // can optimize the results. Other operations are replaced with an AsExpr
-    // for the temporary resultArr.
-    llvm::SmallVector<hlfir::DestroyOp> destroys;
-    llvm::SmallVector<hlfir::AssignOp> assigns;
-    for (auto user : mloc->getUsers()) {
-      if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(user))
-        destroys.push_back(destroy);
-      else if (auto assign = mlir::dyn_cast<hlfir::AssignOp>(user))
-        assigns.push_back(assign);
-    }
-
-    // Check if the minloc/maxloc was the only user of the elemental (apart from
-    // a destroy), and remove it if so.
-    mlir::Operation::user_range elemUsers = elemental->getUsers();
-    hlfir::DestroyOp elemDestroy;
-    if (std::distance(elemUsers.begin(), elemUsers.end()) == 2) {
-      elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*elemUsers.begin());
-      if (!elemDestroy)
-        elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++elemUsers.begin());
-    }
-
-    for (auto d : destroys)
-      rewriter.eraseOp(d);
-    for (auto a : assigns)
-      a.setOperand(0, resultArr);
-    rewriter.replaceOp(mloc, asExpr);
-    if (elemDestroy) {
-      rewriter.eraseOp(elemDestroy);
-      rewriter.eraseOp(elemental);
-    }
-    return mlir::success();
-  }
-};
-
 class EvaluateIntoMemoryAssignBufferization
     : public mlir::OpRewritePattern<hlfir::EvaluateInMemoryOp> {
 
@@ -1340,19 +888,6 @@ class OptimizedBufferizationPass
     patterns.insert<ElementalAssignBufferization>(context);
     patterns.insert<BroadcastAssignBufferization>(context);
     patterns.insert<EvaluateIntoMemoryAssignBufferization>(context);
-    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
-    // TODO: implement basic minloc/maxloc conversion.
-    // patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
-    // patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
-    patterns.insert<ReductionMaskConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<ReductionMaskConversion<hlfir::MaxlocOp>>(context);
-    // TODO: implement masked minval/maxval conversion.
-    // patterns.insert<ReductionMaskConversion<hlfir::MaxvalOp>>(context);
-    // patterns.insert<ReductionMaskConversion<hlfir::MinvalOp>>(context);
 
     if (mlir::failed(mlir::applyPatternsGreedily(
             getOperation(), std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index bac10121a881b..6876c1487ac13 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -173,245 +173,881 @@ class TransposeAsElementalConversion
   }
 };
 
-// Expand the SUM(DIM=CONSTANT) operation into .
-class SumAsElementalConversion : public mlir::OpRewritePattern<hlfir::SumOp> {
+/// Base class for converting reduction-like operations into
+/// a reduction loop[-nest] optionally wrapped into hlfir.elemental.
+/// It is used to handle operations produced for ALL, ANY, COUNT,
+/// MAXLOC, MAXVAL, MINLOC, MINVAL, SUM intrinsics.
+///
+/// All of these operations take an input array, and optional
+/// dim, mask arguments. ALL, ANY, COUNT do not have mask argument.
+class ReductionAsElementalConverter {
 public:
-  using mlir::OpRewritePattern<hlfir::SumOp>::OpRewritePattern;
+  ReductionAsElementalConverter(mlir::Operation *op,
+                                mlir::PatternRewriter &rewriter)
+      : op{op}, rewriter{rewriter}, loc{op->getLoc()}, builder{rewriter, op} {
+    assert(op->getNumResults() == 1);
+  }
+  virtual ~ReductionAsElementalConverter() {}
 
-  llvm::LogicalResult
-  matchAndRewrite(hlfir::SumOp sum,
-                  mlir::PatternRewriter &rewriter) const override {
-    hlfir::Entity array = hlfir::Entity{sum.getArray()};
-    bool isTotalReduction = hlfir::Entity{sum}.getRank() == 0;
-    mlir::Value dim = sum.getDim();
+  /// Do the actual conversion or return mlir::failure(),
+  /// if conversion is not possible.
+  mlir::LogicalResult convert();
+
+private:
+  // Return fir.shape specifying the shape of the result
+  // of a reduction with DIM=dimVal. The second return value
+  // is the extent of the DIM dimension.
+  std::tuple<mlir::Value, mlir::Value>
+  genResultShapeForPartialReduction(hlfir::Entity array, int64_t dimVal);
+
+  /// \p mask is a scalar or array logical mask.
+  /// If \p isPresentPred is not nullptr, it is a dynamic predicate value
+  /// identifying whether the mask's variable is present.
+  /// \p indices is a range of one-based indices to access \p mask
+  /// when it is an array.
+  ///
+  /// The method returns the scalar mask value to guard the access
+  /// to a single element of the input array.
+  mlir::Value genMaskValue(mlir::Value mask, mlir::Value isPresentPred,
+                           mlir::ValueRange indices);
+
+protected:
+  /// Return the input array.
+  virtual mlir::Value getSource() const = 0;
+
+  /// Return DIM or nullptr, if it is not present.
+  virtual mlir::Value getDim() const = 0;
+
+  /// Return MASK or nullptr, if it is not present.
+  virtual mlir::Value getMask() const { return nullptr; }
+
+  /// Return FastMathFlags attached to the operation
+  /// or arith::FastMathFlags::none, if the operation
+  /// does not support FastMathFlags (e.g. ALL, ANY, COUNT).
+  virtual mlir::arith::FastMathFlags getFastMath() const {
+    return mlir::arith::FastMathFlags::none;
+  }
+
+  /// Generates initial values for the reduction values used
+  /// by the reduction loop. In general, there is a single
+  /// loop-carried reduction value (e.g. for SUM), but, for example,
+  /// MAXLOC/MINLOC implementation uses multiple reductions.
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() = 0;
+
+  /// Perform reduction(s) update given a single input array's element
+  /// identified by \p array and \p oneBasedIndices coordinates.
+  /// \p currentValue specifies the current value(s) of the reduction(s)
+  /// inside the reduction loop body.
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) = 0;
+
+  /// Given reduction value(s) in \p reductionResults produced
+  /// by the reduction loop, apply any required updates and return
+  /// new reduction value(s) to be used after the reduction loop
+  /// (e.g. as the result yield of the wrapping hlfir.elemental).
+  /// NOTE: if the reduction loop is wrapped in hlfir.elemental,
+  /// the insertion point of any generated code is inside hlfir.elemental.
+  virtual hlfir::Entity
+  genFinalResult(const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
+    assert(reductionResults.size() == 1 &&
+           "default implementation of genFinalResult expect a single reduction "
+           "value");
+    return hlfir::Entity{reductionResults[0]};
+  }
+
+  /// Return mlir::success(), if the operation can be converted.
+  /// The default implementation always returns mlir::success().
+  /// The derived type may override the default implementation
+  /// with its own definition.
+  virtual mlir::LogicalResult isConvertible() const { return mlir::success(); }
+
+  // Default implementation of isTotalReduction() just checks
+  // if the result of the operation is a scalar.
+  // True result indicates that the reduction has to be done
+  // across all elements, false result indicates that
+  // the result is an array expression produced by an hlfir.elemental
+  // operation with a single reduction loop across the DIM dimension.
+  //
+  // MAXLOC/MINLOC must override this.
+  virtual bool isTotalReduction() const { return getResultRank() == 0; }
+
+  // Return true, if the reduction loop[-nest] may be unordered.
+  // In general, FP reductions may only be unordered when
+  // FastMathFlags::reassoc transformations are allowed.
+  //
+  // Some dervied types may need to override this.
+  virtual bool isUnordered() const {
+    mlir::Type elemType = getSourceElementType();
+    if (mlir::isa<mlir::IntegerType, fir::LogicalType, fir::CharacterType>(
+            elemType))
+      return true;
+    return static_cast<bool>(getFastMath() &
+                             mlir::arith::FastMathFlags::reassoc);
+  }
+
+  /// Return 0, if DIM is not present or its values does not matter
+  /// (for example, a reduction of 1D array does not care about
+  /// the DIM value, assuming that it is a valid program).
+  /// Return mlir::failure(), if DIM is a constant known
+  /// to be invalid for the given array.
+  /// Otherwise, return DIM constant value.
+  mlir::FailureOr<int64_t> getConstDim() const {
     int64_t dimVal = 0;
-    if (!isTotalReduction) {
+    if (!isTotalReduction()) {
       // In case of partial reduction we should ignore the operations
       // with invalid DIM values. They may appear in dead code
       // after constant propagation.
-      auto constDim = fir::getIntIfConstant(dim);
+      auto constDim = fir::getIntIfConstant(getDim());
       if (!constDim)
-        return rewriter.notifyMatchFailure(sum, "Nonconstant DIM for SUM");
+        return rewriter.notifyMatchFailure(op, "Nonconstant DIM");
       dimVal = *constDim;
 
-      if ((dimVal <= 0 || dimVal > array.getRank()))
-        return rewriter.notifyMatchFailure(
-            sum, "Invalid DIM for partial SUM reduction");
+      if ((dimVal <= 0 || dimVal > getSourceRank()))
+        return rewriter.notifyMatchFailure(op,
+                                           "Invalid DIM for partial reduction");
     }
+    return dimVal;
+  }
 
-    mlir::Location loc = sum.getLoc();
-    fir::FirOpBuilder builder{rewriter, sum.getOperation()};
-    mlir::Type elementType = hlfir::getFortranElementType(sum.getType());
-    mlir::Value mask = sum.getMask();
+  /// Return hlfir::Entity of the result.
+  hlfir::Entity getResultEntity() const {
+    return hlfir::Entity{op->getResult(0)};
+  }
 
-    mlir::Value resultShape, dimExtent;
-    llvm::SmallVector<mlir::Value> arrayExtents;
-    if (isTotalReduction)
-      arrayExtents = hlfir::genExtentsVector(loc, builder, array);
-    else
-      std::tie(resultShape, dimExtent) =
-          genResultShapeForPartialReduction(loc, builder, array, dimVal);
-
-    // If the mask is present and is a scalar, then we'd better load its value
-    // outside of the reduction loop making the loop unswitching easier.
-    mlir::Value isPresentPred, maskValue;
-    if (mask) {
-      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-        // MASK represented by a box might be dynamically optional,
-        // so we have to check for its presence before accessing it.
-        isPresentPred =
-            builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
-      }
+  /// Return type of the result (e.g. !hlfir.expr<?xi32>).
+  mlir::Type getResultType() const { return getResultEntity().getType(); }
 
-      if (hlfir::Entity{mask}.isScalar())
-        maskValue = genMaskValue(loc, builder, mask, isPresentPred, {});
-    }
+  /// Return the element type of the result (e.g. i32).
+  mlir::Type getResultElementType() const {
+    return hlfir::getFortranElementType(getResultType());
+  }
 
-    auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                         mlir::ValueRange inputIndices) -> hlfir::Entity {
-      // Loop over all indices in the DIM dimension, and reduce all values.
-      // If DIM is not present, do total reduction.
-
-      // Initial value for the reduction.
-      mlir::Value reductionInitValue =
-          fir::factory::createZeroValue(builder, loc, elementType);
-
-      // The reduction loop may be unordered if FastMathFlags::reassoc
-      // transformations are allowed. The integer reduction is always
-      // unordered.
-      bool isUnordered = mlir::isa<mlir::IntegerType>(elementType) ||
-                         static_cast<bool>(sum.getFastmath() &
-                                           mlir::arith::FastMathFlags::reassoc);
+  /// Return rank of the result.
+  unsigned getResultRank() const { return getResultEntity().getRank(); }
 
-      llvm::SmallVector<mlir::Value> extents;
-      if (isTotalReduction)
-        extents = arrayExtents;
-      else
-        extents.push_back(
-            builder.createConvert(loc, builder.getIndexType(), dimExtent));
+  /// Return the element type of the source.
+  mlir::Type getSourceElementType() const {
+    return hlfir::getFortranElementType(getSource().getType());
+  }
 
-      auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                         mlir::ValueRange oneBasedIndices,
-                         mlir::ValueRange reductionArgs)
-          -> llvm::SmallVector<mlir::Value, 1> {
-        // Generate the reduction loop-nest body.
-        // The initial reduction value in the innermost loop
-        // is passed via reductionArgs[0].
-        llvm::SmallVector<mlir::Value> indices;
-        if (isTotalReduction) {
-          indices = oneBasedIndices;
-        } else {
-          indices = inputIndices;
-          indices.insert(indices.begin() + dimVal - 1, oneBasedIndices[0]);
-        }
+  /// Return rank of the input array.
+  unsigned getSourceRank() const {
+    return hlfir::Entity{getSource()}.getRank();
+  }
 
-        mlir::Value reductionValue = reductionArgs[0];
-        fir::IfOp ifOp;
-        if (mask) {
-          // Make the reduction value update conditional on the value
-          // of the mask.
-          if (!maskValue) {
-            // If the mask is an array, use the elemental and the loop indices
-            // to address the proper mask element.
-            maskValue =
-                genMaskValue(loc, builder, mask, isPresentPred, indices);
-          }
-          mlir::Value isUnmasked = builder.create<fir::ConvertOp>(
-              loc, builder.getI1Type(), maskValue);
-          ifOp = builder.create<fir::IfOp>(loc, elementType, isUnmasked,
-                                           /*withElseRegion=*/true);
-          // In the 'else' block return the current reduction value.
-          builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-          builder.create<fir::ResultOp>(loc, reductionValue);
-
-          // In the 'then' block do the actual addition.
-          builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-        }
+  /// The reduction operation.
+  mlir::Operation *op;
 
-        hlfir::Entity element =
-            hlfir::getElementAt(loc, builder, array, indices);
-        hlfir::Entity elementValue =
-            hlfir::loadTrivialScalar(loc, builder, element);
-        // NOTE: we can use "Kahan summation" same way as the runtime
-        // (e.g. when fast-math is not allowed), but let's start with
-        // the simple version.
-        reductionValue =
-            genScalarAdd(loc, builder, reductionValue, elementValue);
-
-        if (ifOp) {
-          builder.create<fir::ResultOp>(loc, reductionValue);
-          builder.setInsertionPointAfter(ifOp);
-          reductionValue = ifOp.getResult(0);
-        }
+  mlir::PatternRewriter &rewriter;
+  mlir::Location loc;
+  fir::FirOpBuilder builder;
+};
 
-        return {reductionValue};
-      };
+/// Generate initialization value for MIN or MAX reduction
+/// of the given \p type.
+template <bool IS_MAX>
+static mlir::Value genMinMaxInitValue(mlir::Location loc,
+                                      fir::FirOpBuilder &builder,
+                                      mlir::Type type) {
+  if (auto ty = mlir::dyn_cast<mlir::FloatType>(type)) {
+    const llvm::fltSemantics &sem = ty.getFloatSemantics();
+    // We must not use +/-INF here. If the reduction input is empty,
+    // the result of reduction must be +/-LARGEST.
+    llvm::APFloat limit = llvm::APFloat::getLargest(sem, /*Negative=*/IS_MAX);
+    return builder.createRealConstant(loc, type, limit);
+  }
+  unsigned bits = type.getIntOrFloatBitWidth();
+  int64_t limitInt = IS_MAX
+                         ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
+                         : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
+  return builder.createIntegerConstant(loc, type, limitInt);
+}
+
+/// Generate a comparison of an array element value \p elem
+/// and the current reduction value \p reduction for MIN/MAX reduction.
+template <bool IS_MAX>
+static mlir::Value
+genMinMaxComparison(mlir::Location loc, fir::FirOpBuilder &builder,
+                    mlir::Value elem, mlir::Value reduction) {
+  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
+    // For FP reductions we want the first smallest value to be used, that
+    // is not NaN. A OGL/OLT condition will usually work for this unless all
+    // the values are Nan or Inf. This follows the same logic as
+    // NumericCompare for Minloc/Maxloc in extrema.cpp.
+    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
+        loc,
+        IS_MAX ? mlir::arith::CmpFPredicate::OGT
+               : mlir::arith::CmpFPredicate::OLT,
+        elem, reduction);
+    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
+    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
+    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
+    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
+    return builder.create<mlir::arith::CmpIOp>(
+        loc,
+        IS_MAX ? mlir::arith::CmpIPredicate::sgt
+               : mlir::arith::CmpIPredicate::slt,
+        elem, reduction);
+  }
+  llvm_unreachable("unsupported type");
+}
+
+/// Implementation of ReductionAsElementalConverter interface
+/// for MAXLOC/MINLOC.
+template <typename T>
+class MinMaxlocAsElementalConverter : public ReductionAsElementalConverter {
+  static_assert(std::is_same_v<T, hlfir::MaxlocOp> ||
+                std::is_same_v<T, hlfir::MinlocOp>);
+  static constexpr unsigned maxRank = Fortran::common::maxRank;
+  // We have the following reduction values in the reduction loop:
+  //   * N integer coordinates, where N is:
+  //     - RANK(ARRAY) for total reductions.
+  //     - 1 for partial reductions.
+  //   * 1 reduction value holding the current MIN/MAX.
+  //   * 1 boolean indicating whether it is the first time
+  //     the mask is true.
+  static constexpr unsigned maxNumReductions = Fortran::common::maxRank + 2;
+  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxlocOp>;
+  using Base = ReductionAsElementalConverter;
 
-      llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
-          hlfir::genLoopNestWithReductions(loc, builder, extents,
-                                           {reductionInitValue}, genBody,
-                                           isUnordered);
-      return hlfir::Entity{reductionFinalValues[0]};
-    };
+public:
+  MinMaxlocAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
 
-    if (isTotalReduction) {
-      hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
-      rewriter.replaceOp(sum, result);
-      return mlir::success();
+private:
+  virtual mlir::Value getSource() const final { return getOp().getArray(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+  virtual mlir::Value getMask() const final { return getOp().getMask(); }
+  virtual mlir::arith::FastMathFlags getFastMath() const final {
+    return getOp().getFastmath();
+  }
+
+  virtual mlir::LogicalResult isConvertible() const final {
+    if (getOp().getBack())
+      return rewriter.notifyMatchFailure(
+          getOp(), "BACK is not supported for MINLOC/MAXLOC inlining");
+    if (mlir::isa<fir::CharacterType>(getSourceElementType()))
+      return rewriter.notifyMatchFailure(
+          getOp(),
+          "CHARACTER type is not supported for MINLOC/MAXLOC inlining");
+    return mlir::success();
+  }
+
+  // If the result is scalar, then DIM does not matter,
+  // and this is a total reduction.
+  // If DIM is not present, this is a total reduction.
+  virtual bool isTotalReduction() const final {
+    return getResultRank() == 0 || !getDim();
+  }
+
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final;
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) final;
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) final;
+
+private:
+  T getOp() const { return mlir::cast<T>(op); }
+
+  unsigned getNumCoors() const {
+    return isTotalReduction() ? getSourceRank() : 1;
+  }
+
+  void
+  checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    assert(reductions.size() == getNumCoors() + 2 &&
+           "invalid number of reductions for MINLOC/MAXLOC");
+  }
+
+  mlir::Value
+  getCurrentMinMax(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    checkReductions(reductions);
+    return reductions[getNumCoors()];
+  }
+
+  mlir::Value
+  getIsFirst(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    checkReductions(reductions);
+    return reductions[getNumCoors() + 1];
+  }
+};
+
+template <typename T>
+llvm::SmallVector<mlir::Value>
+MinMaxlocAsElementalConverter<T>::genReductionInitValues() {
+  // Initial value for the coordinate(s) is zero.
+  mlir::Value zeroCoor =
+      fir::factory::createZeroValue(builder, loc, getResultElementType());
+  llvm::SmallVector<mlir::Value> result(getNumCoors(), zeroCoor);
+
+  // Initial value for the MIN/MAX value.
+  mlir::Value minMaxInit =
+      genMinMaxInitValue<isMax>(loc, builder, getSourceElementType());
+  result.push_back(minMaxInit);
+
+  // Initial value for isFirst predicate. It is switched to false,
+  // when the reduction update dynamically happens inside the reduction
+  // loop.
+  mlir::Value trueVal = builder.createBool(loc, true);
+  result.push_back(trueVal);
+
+  return result;
+}
+
+template <typename T>
+llvm::SmallVector<mlir::Value>
+MinMaxlocAsElementalConverter<T>::reduceOneElement(
+    const llvm::SmallVectorImpl<mlir::Value> &currentValue, hlfir::Entity array,
+    mlir::ValueRange oneBasedIndices) {
+  checkReductions(currentValue);
+  hlfir::Entity elementValue =
+      hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+  mlir::Value cmp = genMinMaxComparison<isMax>(loc, builder, elementValue,
+                                               getCurrentMinMax(currentValue));
+  // If isFirst is true, then do the reduction update regardless
+  // of the FP comparison.
+  cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, getIsFirst(currentValue));
+
+  llvm::SmallVector<mlir::Value> newIndices;
+  int64_t dim = 1;
+  if (!isTotalReduction()) {
+    auto dimVal = getConstDim();
+    assert(mlir::succeeded(dimVal) &&
+           "partial MINLOC/MAXLOC reduction with invalid DIM");
+    dim = *dimVal;
+    assert(getNumCoors() == 1 &&
+           "partial MAXLOC/MINLOC reduction must compute one coordinate");
+  }
+
+  for (unsigned coorIdx = 0; coorIdx < getNumCoors(); ++coorIdx) {
+    mlir::Value currentCoor = currentValue[coorIdx];
+    mlir::Value newCoor = builder.createConvert(
+        loc, currentCoor.getType(), oneBasedIndices[coorIdx + dim - 1]);
+    mlir::Value update =
+        builder.create<mlir::arith::SelectOp>(loc, cmp, newCoor, currentCoor);
+    newIndices.push_back(update);
+  }
+
+  mlir::Value newMinMax = builder.create<mlir::arith::SelectOp>(
+      loc, cmp, elementValue, getCurrentMinMax(currentValue));
+  newIndices.push_back(newMinMax);
+
+  mlir::Value newIsFirst = builder.createBool(loc, false);
+  newIndices.push_back(newIsFirst);
+
+  assert(currentValue.size() == newIndices.size() &&
+         "invalid number of updated reductions");
+
+  return newIndices;
+}
+
+template <typename T>
+hlfir::Entity MinMaxlocAsElementalConverter<T>::genFinalResult(
+    const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
+  // Identification of the final result of MINLOC/MAXLOC:
+  //   * If DIM is absent, the result is rank-one array.
+  //   * If DIM is present:
+  //     - The result is scalar for rank-one input.
+  //     - The result is an array of rank RANK(ARRAY)-1.
+  checkReductions(reductionResults);
+
+  // We need to adjust the one-based indices to real array indices.
+  // The adjustment must only be done, if there was an actual update
+  // of the coordinates in the reduction loop. For this check we only
+  // need to compare if any of the reduction results is not zero.
+  mlir::Value zero = fir::factory::createZeroValue(
+      builder, loc, reductionResults[0].getType());
+  mlir::Value doAdjust = builder.create<mlir::arith::CmpIOp>(
+      loc, mlir::arith::CmpIPredicate::ne, reductionResults[0], zero);
+  mlir::Type indexType = builder.getIndexType();
+  mlir::Value one = builder.createIntegerConstant(loc, indexType, 1);
+
+  auto adjustCoor = [&](mlir::Value coor, mlir::Value lbound) {
+    assert(mlir::isa<mlir::IndexType>(lbound.getType()));
+    mlir::Value coorAsIndex = builder.createConvert(loc, indexType, coor);
+    mlir::Value tmp =
+        builder.create<mlir::arith::AddIOp>(loc, coorAsIndex, lbound);
+    tmp = builder.create<mlir::arith::SubIOp>(loc, tmp, one);
+    tmp =
+        builder.create<mlir::arith::SelectOp>(loc, doAdjust, tmp, coorAsIndex);
+    return builder.createConvert(loc, coor.getType(), tmp);
+  };
+
+  // For partial reductions, the final result of the reduction
+  // loop is just a scalar - the coordinate within DIM dimension.
+  if (getResultRank() == 0 || !isTotalReduction()) {
+    // The result is a scalar, so just return the scalar.
+    assert(getNumCoors() == 1 &&
+           "unpexpected number of coordinates for scalar result");
+
+    int64_t dim = 1;
+    if (!isTotalReduction()) {
+      auto dimVal = getConstDim();
+      assert(mlir::succeeded(dimVal) &&
+             "partial MINLOC/MAXLOC reduction with invalid DIM");
+      dim = *dimVal;
     }
+    mlir::Value dimLbound =
+        hlfir::genLBound(loc, builder, hlfir::Entity{getSource()}, dim - 1);
+    return hlfir::Entity{adjustCoor(reductionResults[0], dimLbound)};
+  }
+  // This is a total reduction, and there is no wrapping hlfir.elemental.
+  // We have to pack the reduced coordinates into a rank-one array.
+  unsigned rank = getSourceRank();
+  // TODO: in order to avoid introducing new memory effects
+  // we should not use a temporary in memory.
+  // We can use hlfir.elemental with a switch to pack all the coordinates
+  // into an array expression, or we can have a dedicated HLFIR operation
+  // for this.
+  mlir::Value tempArray = builder.createTemporary(
+      loc, fir::SequenceType::get(rank, getResultElementType()));
+  llvm::SmallVector<mlir::Value, maxRank> arrayLbounds =
+      hlfir::genLBounds(loc, builder, hlfir::Entity(getSource()));
+  for (unsigned i = 0; i < rank; ++i) {
+    mlir::Value coor = adjustCoor(reductionResults[i], arrayLbounds[i]);
+    mlir::Value idx = builder.createIntegerConstant(loc, indexType, i + 1);
+    mlir::Value resultElement =
+        hlfir::getElementAt(loc, builder, hlfir::Entity{tempArray}, {idx});
+    builder.create<hlfir::AssignOp>(loc, coor, resultElement);
+  }
+  mlir::Value tempExpr = builder.create<hlfir::AsExprOp>(
+      loc, tempArray, builder.createBool(loc, false));
+  return hlfir::Entity{tempExpr};
+}
+
+/// Base class for numeric reductions like MAXVAl, MINVAL, SUM.
+template <typename OpT>
+class NumericReductionAsElementalConverterBase
+    : public ReductionAsElementalConverter {
+  using Base = ReductionAsElementalConverter;
+
+protected:
+  NumericReductionAsElementalConverterBase(OpT op,
+                                           mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
+
+  virtual mlir::Value getSource() const final { return getOp().getArray(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+  virtual mlir::Value getMask() const final { return getOp().getMask(); }
+  virtual mlir::arith::FastMathFlags getFastMath() const final {
+    return getOp().getFastmath();
+  }
 
-    hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
-        loc, builder, elementType, resultShape, {}, genKernel,
-        /*isUnordered=*/true, /*polymorphicMold=*/nullptr,
-        sum.getResult().getType());
+  OpT getOp() const { return mlir::cast<OpT>(op); }
 
-    // it wouldn't be safe to replace block arguments with a different
-    // hlfir.expr type. Types can differ due to differing amounts of shape
-    // information
-    assert(elementalOp.getResult().getType() == sum.getResult().getType());
+  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
+    assert(reductions.size() == 1 && "reduction must produce single value");
+  }
+};
+
+/// Reduction converter for MAXMAL/MINVAL.
+template <typename T>
+class MinMaxvalAsElementalConverter
+    : public NumericReductionAsElementalConverterBase<T> {
+  static_assert(std::is_same_v<T, hlfir::MaxvalOp> ||
+                std::is_same_v<T, hlfir::MinvalOp>);
+  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxvalOp>;
+  using Base = NumericReductionAsElementalConverterBase<T>;
+
+public:
+  MinMaxvalAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
 
-    rewriter.replaceOp(sum, elementalOp);
+private:
+  virtual mlir::LogicalResult isConvertible() const final {
+    if (mlir::isa<fir::CharacterType>(this->getSourceElementType()))
+      return this->rewriter.notifyMatchFailure(
+          this->getOp(),
+          "CHARACTER type is not supported for MINVAL/MAXVAL inlining");
     return mlir::success();
   }
 
-private:
-  // Return fir.shape specifying the shape of the result
-  // of a SUM reduction with DIM=dimVal. The second return value
-  // is the extent of the DIM dimension.
-  static std::tuple<mlir::Value, mlir::Value>
-  genResultShapeForPartialReduction(mlir::Location loc,
-                                    fir::FirOpBuilder &builder,
-                                    hlfir::Entity array, int64_t dimVal) {
-    llvm::SmallVector<mlir::Value> inExtents =
-        hlfir::genExtentsVector(loc, builder, array);
-    assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
-           "DIM must be present and a positive constant not exceeding "
-           "the array's rank");
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {genMinMaxInitValue<isMax>(this->loc, this->builder,
+                                      this->getResultElementType())};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    this->checkReductions(currentValue);
+    fir::FirOpBuilder &builder = this->builder;
+    mlir::Location loc = this->loc;
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value cmp =
+        genMinMaxComparison<isMax>(loc, builder, elementValue, currentValue[0]);
+    return {builder.create<mlir::arith::SelectOp>(loc, cmp, elementValue,
+                                                  currentValue[0])};
+  }
+};
+
+/// Reduction converter for SUM.
+class SumAsElementalConverter
+    : public NumericReductionAsElementalConverterBase<hlfir::SumOp> {
+  using Base = NumericReductionAsElementalConverterBase;
 
-    mlir::Value dimExtent = inExtents[dimVal - 1];
-    inExtents.erase(inExtents.begin() + dimVal - 1);
-    return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
+public:
+  SumAsElementalConverter(hlfir::SumOp op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {
+        fir::factory::createZeroValue(builder, loc, getResultElementType())};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    checkReductions(currentValue);
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    // NOTE: we can use "Kahan summation" same way as the runtime
+    // (e.g. when fast-math is not allowed), but let's start with
+    // the simple version.
+    return {genScalarAdd(currentValue[0], elementValue)};
   }
 
   // Generate scalar addition of the two values (of the same data type).
-  static mlir::Value genScalarAdd(mlir::Location loc,
-                                  fir::FirOpBuilder &builder,
-                                  mlir::Value value1, mlir::Value value2) {
-    mlir::Type ty = value1.getType();
-    assert(ty == value2.getType() && "reduction values' types do not match");
-    if (mlir::isa<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
-    else if (mlir::isa<mlir::ComplexType>(ty))
-      return builder.create<fir::AddcOp>(loc, value1, value2);
-    else if (mlir::isa<mlir::IntegerType>(ty))
-      return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
-
-    llvm_unreachable("unsupported SUM reduction type");
+  mlir::Value genScalarAdd(mlir::Value value1, mlir::Value value2);
+};
+
+/// Base class for logical reductions like ALL, ANY, COUNT.
+/// They do not have MASK and FastMathFlags.
+template <typename OpT>
+class LogicalReductionAsElementalConverterBase
+    : public ReductionAsElementalConverter {
+  using Base = ReductionAsElementalConverter;
+
+public:
+  LogicalReductionAsElementalConverterBase(OpT op,
+                                           mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
+
+protected:
+  OpT getOp() const { return mlir::cast<OpT>(op); }
+
+  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
+    assert(reductions.size() == 1 && "reduction must produce single value");
   }
 
-  static mlir::Value genMaskValue(mlir::Location loc,
-                                  fir::FirOpBuilder &builder, mlir::Value mask,
-                                  mlir::Value isPresentPred,
-                                  mlir::ValueRange indices) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    fir::IfOp ifOp;
-    mlir::Type maskType =
-        hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
-    if (isPresentPred) {
-      ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
-                                       /*withElseRegion=*/true);
-
-      // Use 'true', if the mask is not present.
-      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      mlir::Value trueValue = builder.createBool(loc, true);
-      trueValue = builder.createConvert(loc, maskType, trueValue);
-      builder.create<fir::ResultOp>(loc, trueValue);
-
-      // Load the mask value, if the mask is present.
-      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+  virtual mlir::Value getSource() const final { return getOp().getMask(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) override {
+    checkReductions(reductionResults);
+    return hlfir::Entity{reductionResults[0]};
+  }
+};
+
+/// Reduction converter for ALL/ANY.
+template <typename T>
+class AllAnyAsElementalConverter
+    : public LogicalReductionAsElementalConverterBase<T> {
+  static_assert(std::is_same_v<T, hlfir::AllOp> ||
+                std::is_same_v<T, hlfir::AnyOp>);
+  static constexpr bool isAll = std::is_same_v<T, hlfir::AllOp>;
+  using Base = LogicalReductionAsElementalConverterBase<T>;
+
+public:
+  AllAnyAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {this->builder.createBool(this->loc, isAll ? true : false)};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    this->checkReductions(currentValue);
+    fir::FirOpBuilder &builder = this->builder;
+    mlir::Location loc = this->loc;
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value mask =
+        builder.createConvert(loc, builder.getI1Type(), elementValue);
+    if constexpr (isAll)
+      return {builder.create<mlir::arith::AndIOp>(loc, mask, currentValue[0])};
+    else
+      return {builder.create<mlir::arith::OrIOp>(loc, mask, currentValue[0])};
+  }
+
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionValues) final {
+    this->checkReductions(reductionValues);
+    return hlfir::Entity{this->builder.createConvert(
+        this->loc, this->getResultElementType(), reductionValues[0])};
+  }
+};
+
+/// Reduction converter for COUNT.
+class CountAsElementalConverter
+    : public LogicalReductionAsElementalConverterBase<hlfir::CountOp> {
+  using Base = LogicalReductionAsElementalConverterBase<hlfir::CountOp>;
+
+public:
+  CountAsElementalConverter(hlfir::CountOp op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {
+        fir::factory::createZeroValue(builder, loc, getResultElementType())};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    checkReductions(currentValue);
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value cond =
+        builder.createConvert(loc, builder.getI1Type(), elementValue);
+    mlir::Value one =
+        builder.createIntegerConstant(loc, getResultElementType(), 1);
+    mlir::Value add1 =
+        builder.create<mlir::arith::AddIOp>(loc, currentValue[0], one);
+    return {builder.create<mlir::arith::SelectOp>(loc, cond, add1,
+                                                  currentValue[0])};
+  }
+};
+
+mlir::LogicalResult ReductionAsElementalConverter::convert() {
+  mlir::LogicalResult canConvert(isConvertible());
+
+  if (mlir::failed(canConvert))
+    return canConvert;
+
+  hlfir::Entity array = hlfir::Entity{getSource()};
+  bool isTotalReduce = isTotalReduction();
+  auto dimVal = getConstDim();
+  if (mlir::failed(dimVal))
+    return dimVal;
+  mlir::Value mask = getMask();
+  mlir::Value resultShape, dimExtent;
+  llvm::SmallVector<mlir::Value> arrayExtents;
+  if (isTotalReduce)
+    arrayExtents = hlfir::genExtentsVector(loc, builder, array);
+  else
+    std::tie(resultShape, dimExtent) =
+        genResultShapeForPartialReduction(array, *dimVal);
+
+  // If the mask is present and is a scalar, then we'd better load its value
+  // outside of the reduction loop making the loop unswitching easier.
+  mlir::Value isPresentPred, maskValue;
+  if (mask) {
+    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+      // MASK represented by a box might be dynamically optional,
+      // so we have to check for its presence before accessing it.
+      isPresentPred =
+          builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
     }
 
-    hlfir::Entity maskVar{mask};
-    if (maskVar.isScalar()) {
-      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-        // MASK may be a boxed scalar.
-        mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
-        mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
+    if (hlfir::Entity{mask}.isScalar())
+      maskValue = genMaskValue(mask, isPresentPred, {});
+  }
+
+  auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                       mlir::ValueRange inputIndices) -> hlfir::Entity {
+    // Loop over all indices in the DIM dimension, and reduce all values.
+    // If DIM is not present, do total reduction.
+
+    // Initial value for the reduction.
+    llvm::SmallVector<mlir::Value, 1> reductionInitValues =
+        genReductionInitValues();
+
+    llvm::SmallVector<mlir::Value> extents;
+    if (isTotalReduce)
+      extents = arrayExtents;
+    else
+      extents.push_back(
+          builder.createConvert(loc, builder.getIndexType(), dimExtent));
+
+    auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                       mlir::ValueRange oneBasedIndices,
+                       mlir::ValueRange reductionArgs)
+        -> llvm::SmallVector<mlir::Value, 1> {
+      // Generate the reduction loop-nest body.
+      // The initial reduction value in the innermost loop
+      // is passed via reductionArgs[0].
+      llvm::SmallVector<mlir::Value> indices;
+      if (isTotalReduce) {
+        indices = oneBasedIndices;
       } else {
-        mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
+        indices = inputIndices;
+        indices.insert(indices.begin() + *dimVal - 1, oneBasedIndices[0]);
+      }
+
+      llvm::SmallVector<mlir::Value, 1> reductionValues = reductionArgs;
+      llvm::SmallVector<mlir::Type, 1> reductionTypes;
+      llvm::transform(reductionValues, std::back_inserter(reductionTypes),
+                      [](mlir::Value v) { return v.getType(); });
+      fir::IfOp ifOp;
+      if (mask) {
+        // Make the reduction value update conditional on the value
+        // of the mask.
+        if (!maskValue) {
+          // If the mask is an array, use the elemental and the loop indices
+          // to address the proper mask element.
+          maskValue = genMaskValue(mask, isPresentPred, indices);
+        }
+        mlir::Value isUnmasked =
+            builder.create<fir::ConvertOp>(loc, builder.getI1Type(), maskValue);
+        ifOp = builder.create<fir::IfOp>(loc, reductionTypes, isUnmasked,
+                                         /*withElseRegion=*/true);
+        // In the 'else' block return the current reduction value.
+        builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+        builder.create<fir::ResultOp>(loc, reductionValues);
+
+        // In the 'then' block do the actual addition.
+        builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
       }
+      reductionValues = reduceOneElement(reductionValues, array, indices);
+      if (ifOp) {
+        builder.create<fir::ResultOp>(loc, reductionValues);
+        builder.setInsertionPointAfter(ifOp);
+        reductionValues = ifOp.getResults();
+      }
+
+      return reductionValues;
+    };
+
+    llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
+        hlfir::genLoopNestWithReductions(
+            loc, builder, extents, reductionInitValues, genBody, isUnordered());
+    return genFinalResult(reductionFinalValues);
+  };
+
+  if (isTotalReduce) {
+    hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
+    rewriter.replaceOp(op, result);
+    return mlir::success();
+  }
+
+  hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
+      loc, builder, getResultElementType(), resultShape, /*typeParams=*/{},
+      genKernel,
+      /*isUnordered=*/true, /*polymorphicMold=*/nullptr, getResultType());
+
+  // it wouldn't be safe to replace block arguments with a different
+  // hlfir.expr type. Types can differ due to differing amounts of shape
+  // information
+  assert(elementalOp.getResult().getType() == op->getResult(0).getType());
+
+  rewriter.replaceOp(op, elementalOp);
+  return mlir::success();
+}
+
+std::tuple<mlir::Value, mlir::Value>
+ReductionAsElementalConverter::genResultShapeForPartialReduction(
+    hlfir::Entity array, int64_t dimVal) {
+  llvm::SmallVector<mlir::Value> inExtents =
+      hlfir::genExtentsVector(loc, builder, array);
+  assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
+         "DIM must be present and a positive constant not exceeding "
+         "the array's rank");
+
+  mlir::Value dimExtent = inExtents[dimVal - 1];
+  inExtents.erase(inExtents.begin() + dimVal - 1);
+  return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
+}
+
+mlir::Value SumAsElementalConverter::genScalarAdd(mlir::Value value1,
+                                                  mlir::Value value2) {
+  mlir::Type ty = value1.getType();
+  assert(ty == value2.getType() && "reduction values' types do not match");
+  if (mlir::isa<mlir::FloatType>(ty))
+    return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+  else if (mlir::isa<mlir::ComplexType>(ty))
+    return builder.create<fir::AddcOp>(loc, value1, value2);
+  else if (mlir::isa<mlir::IntegerType>(ty))
+    return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+
+  llvm_unreachable("unsupported SUM reduction type");
+}
+
+mlir::Value ReductionAsElementalConverter::genMaskValue(
+    mlir::Value mask, mlir::Value isPresentPred, mlir::ValueRange indices) {
+  mlir::OpBuilder::InsertionGuard guard(builder);
+  fir::IfOp ifOp;
+  mlir::Type maskType =
+      hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
+  if (isPresentPred) {
+    ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
+                                     /*withElseRegion=*/true);
+
+    // Use 'true', if the mask is not present.
+    builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+    mlir::Value trueValue = builder.createBool(loc, true);
+    trueValue = builder.createConvert(loc, maskType, trueValue);
+    builder.create<fir::ResultOp>(loc, trueValue);
+
+    // Load the mask value, if the mask is present.
+    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+  }
+
+  hlfir::Entity maskVar{mask};
+  if (maskVar.isScalar()) {
+    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+      // MASK may be a boxed scalar.
+      mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
+      mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
     } else {
-      // Load from the mask array.
-      assert(!indices.empty() && "no indices for addressing the mask array");
-      maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
       mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
     }
+  } else {
+    // Load from the mask array.
+    assert(!indices.empty() && "no indices for addressing the mask array");
+    maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
+    mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
+  }
+
+  if (!isPresentPred)
+    return mask;
 
-    if (!isPresentPred)
-      return mask;
+  builder.create<fir::ResultOp>(loc, mask);
+  return ifOp.getResult(0);
+}
+
+/// Convert an operation that is a partial or total reduction
+/// over an array of values into a reduction loop[-nest]
+/// optionally wrapped into hlfir.elemental.
+template <typename Op>
+class ReductionConversion : public mlir::OpRewritePattern<Op> {
+public:
+  using mlir::OpRewritePattern<Op>::OpRewritePattern;
 
-    builder.create<fir::ResultOp>(loc, mask);
-    return ifOp.getResult(0);
+  llvm::LogicalResult
+  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
+    if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                  std::is_same_v<Op, hlfir::MinlocOp>) {
+      MinMaxlocAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      MinMaxvalAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
+      CountAsElementalConverter converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::AllOp> ||
+                         std::is_same_v<Op, hlfir::AnyOp>) {
+      AllAnyAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::SumOp>) {
+      SumAsElementalConverter converter{op, rewriter};
+      return converter.convert();
+    }
+    return rewriter.notifyMatchFailure(op, "unexpected reduction operation");
   }
 };
 
@@ -1481,10 +2117,18 @@ class SimplifyHLFIRIntrinsics
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
-    patterns.insert<SumAsElementalConversion>(context);
+    patterns.insert<ReductionConversion<hlfir::SumOp>>(context);
     patterns.insert<CShiftConversion>(context);
     patterns.insert<MatmulConversion<hlfir::MatmulTransposeOp>>(context);
 
+    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
+
     // If forceMatmulAsElemental is false, then hlfir.matmul inlining
     // will introduce hlfir.eval_in_mem operation with new memory side
     // effects. This conflicts with CSE and optimized bufferization, e.g.:
diff --git a/flang/test/HLFIR/all-elemental.fir b/flang/test/HLFIR/all-elemental.fir
deleted file mode 100644
index 1ba8bb1b7a5fb..0000000000000
--- a/flang/test/HLFIR/all-elemental.fir
+++ /dev/null
@@ -1,91 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.all %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
-  return %13 : !fir.logical<4>
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-// CHECK-NEXT:     %true = arith.constant true
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %true) -> (i1) {
-// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.andi %arg4, %[[V16]] : i1
-// CHECK-NEXT:       fir.result %[[V17]] : i1
-// CHECK-NEXT:     }
-// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
-// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
-
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
-  %c2_i32 = arith.constant 2 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.all %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
-  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
-  return %11 : !fir.array<4x!fir.logical<4>>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: %10 = hlfir.all %9 dim %c2_i32
\ No newline at end of file
diff --git a/flang/test/HLFIR/any-elemental.fir b/flang/test/HLFIR/any-elemental.fir
deleted file mode 100644
index a7c559679d965..0000000000000
--- a/flang/test/HLFIR/any-elemental.fir
+++ /dev/null
@@ -1,190 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.any %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
-  return %13 : !fir.logical<4>
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-// CHECK-NEXT:     %false = arith.constant false
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %false) -> (i1) {
-// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.ori %arg4, %[[V16]] : i1
-// CHECK-NEXT:       fir.result %[[V17]] : i1
-// CHECK-NEXT:     }
-// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
-// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
-
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
-  %c2_i32 = arith.constant 2 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.any %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
-  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
-  return %11 : !fir.array<4x!fir.logical<4>>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: {{.*}} = hlfir.any {{.*}} dim %c2_i32
-
-
-func.func @_Qtest_recursive() attributes {fir.bindc_name = "test"} {
-  %c1 = arith.constant 1 : index
-  %true = arith.constant true
-  %false = arith.constant false
-  %c0_i64 = arith.constant 0 : i64
-  %c2_i32 = arith.constant 2 : i32
-  %c0 = arith.constant 0 : index
-  %c1_i32 = arith.constant 1 : i32
-  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  %1:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
-  %2 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
-  %5:2 = hlfir.declare %4 {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "ra", uniq_name = "_QFEra"}
-  %7 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %8:2 = hlfir.declare %6(%7) {uniq_name = "_QFEra"} : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.ref<!fir.array<1x!fir.logical<4>>>)
-  %9 = fir.alloca !fir.logical<4> {bindc_name = "rs", uniq_name = "_QFErs"}
-  %10:2 = hlfir.declare %9 {uniq_name = "_QFErs"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %11 = fir.allocmem !fir.array<?xi32>, %c1 {fir.must_be_heap = true, uniq_name = "_QFEa.alloc"}
-  %12 = fir.embox %11(%7) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
-  fir.store %12 to %1#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  hlfir.assign %c1_i32 to %5#0 : i32, !fir.ref<i32>
-  %13 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  %14:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-  fir.do_loop %arg0 = %c1 to %14#1 step %c1 unordered {
-    %27:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-    %28 = arith.subi %27#0, %c1 : index
-    %29 = arith.addi %arg0, %28 : index
-    %30 = hlfir.designate %13 (%29)  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
-    hlfir.assign %c2_i32 to %30 : i32, !fir.ref<i32>
-  }
-  %15 = fir.load %5#0 : !fir.ref<i32>
-  %16 = fir.convert %15 : (i32) -> i64
-  %17 = arith.cmpi sgt, %16, %c0_i64 : i64
-  %18 = arith.select %17, %16, %c0_i64 : i64
-  %19 = fir.convert %18 : (i64) -> index
-  %20 = fir.shape %19 : (index) -> !fir.shape<1>
-  %21 = hlfir.elemental %20 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg0: index):
-    %27 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    %28:3 = fir.box_dims %27, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-    %29 = arith.addi %28#0, %28#1 : index
-    %30 = arith.subi %29, %c1 : index
-    %31 = arith.subi %30, %28#0 : index
-    %32 = arith.addi %31, %c1 : index
-    %33 = arith.cmpi sgt, %32, %c0 : index
-    %34 = arith.select %33, %32, %c0 : index
-    %35 = fir.shape %34 : (index) -> !fir.shape<1>
-    %36 = hlfir.designate %27 (%28#0:%30:%c1)  shape %35 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-    %37 = hlfir.elemental %35 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-    ^bb0(%arg1: index):
-      %39 = hlfir.designate %36 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-      %40 = fir.load %39 : !fir.ref<i32>
-      %41 = arith.cmpi eq, %40, %c1_i32 : i32
-      %42 = fir.convert %41 : (i1) -> !fir.logical<4>
-      hlfir.yield_element %42 : !fir.logical<4>
-    }
-    %38 = hlfir.any %37 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
-    hlfir.destroy %37 : !hlfir.expr<?x!fir.logical<4>>
-    hlfir.yield_element %38 : !fir.logical<4>
-  }
-  %22 = hlfir.any %21 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %22 to %10#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %21 : !hlfir.expr<?x!fir.logical<4>>
-  %23 = fir.load %10#0 : !fir.ref<!fir.logical<4>>
-  %24 = fir.convert %23 : (!fir.logical<4>) -> i1
-  %25 = arith.xori %24, %true : i1
-  cf.cond_br %25, ^bb1, ^bb2
-^bb1:  // pred: ^bb0
-  fir.call @_FortranAStopStatement(%c2_i32, %false, %false) fastmath<contract> : (i32, i1, i1) -> ()
-  fir.unreachable
-^bb2:  // pred: ^bb0
-  return
-}
-// CHECK-LABEL: func.func @_Qtest_recursive()
-// CHECK:    %[[V20:.*]] = fir.do_loop %arg0 = %c1 to %{{.*}} step %c1 iter_args(%arg1 = %false) -> (i1) {
-// CHECK:      %[[V26:.*]] = fir.load %[[V1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-// CHECK:      %[[V27:.*]]:3 = fir.box_dims %[[V26]], %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-// CHECK:      %[[V28:.*]] = arith.addi %[[V27]]#0, %[[V27]]#1 : index
-// CHECK:      %[[V29:.*]] = arith.subi %[[V28]], %c1 : index
-// CHECK:      %[[V30:.*]] = arith.subi %[[V29]], %[[V27]]#0 : index
-// CHECK:      %[[V31:.*]] = arith.addi %[[V30]], %c1 : index
-// CHECK:      %[[V32:.*]] = arith.cmpi sgt, %[[V31]], %c0 : index
-// CHECK:      %[[V33:.*]] = arith.select %[[V32]], %[[V31]], %c0 : index
-// CHECK:      %[[V34:.*]] = fir.shape %[[V33]] : (index) -> !fir.shape<1>
-// CHECK:      %[[V35:.*]] = hlfir.designate %[[V26]] (%[[V27]]#0:%[[V29]]:%c1)  shape %[[V34]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-// CHECK:      %[[V36:.*]] = fir.do_loop %arg2 = %c1 to %[[V33]] step %c1 iter_args(%arg3 = %false) -> (i1) {
-// CHECK:        %[[V38:.*]] = hlfir.designate %[[V35]] (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:        %[[V39:.*]] = fir.load %[[V38]] : !fir.ref<i32>
-// CHECK:        %[[V40:.*]] = arith.cmpi eq, %[[V39]], %c1_i32 : i32
-// CHECK:        %[[V41:.*]] = arith.ori %arg3, %[[V40]] : i1
-// CHECK:        fir.result %[[V41]] : i1
-// CHECK:      }
-// CHECK:      %[[V37:.*]] = arith.ori %arg1, %[[V36]] : i1
-// CHECK:      fir.result %[[V37]] : i1
-// CHECK:    }
diff --git a/flang/test/HLFIR/count-elemental.fir b/flang/test/HLFIR/count-elemental.fir
deleted file mode 100644
index 0df5cc3c031ea..0000000000000
--- a/flang/test/HLFIR/count-elemental.fir
+++ /dev/null
@@ -1,314 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %12 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<i32>
-  return %13 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
-// CHECK-NEXT:       fir.result %[[V17]] : i32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
-// CHECK-NEXT:     return %[[V12]] : i32
-
-func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i16 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i16
-  hlfir.assign %12 to %4#0 : i16, !fir.ref<i16>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<i16>
-  return %13 : i16
-}
-// CHECK-LABEL:  func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
-// CHECK-NEXT:     %c1_i16 = arith.constant 1 : i16
-// CHECK-NEXT:     %c0_i16 = arith.constant 0 : i16
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i16
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i16) -> (i16) {
-// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i16 : i16
-// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i16
-// CHECK-NEXT:       fir.result %[[V17]] : i16
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i16, !fir.ref<i16>
-// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i16>
-// CHECK-NEXT:     return %[[V12]] : i16
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<7xi32> {
-  %c1_i32 = arith.constant 1 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<7xi32> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<7xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<7xi32>>, !fir.ref<!fir.array<7xi32>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.count %9 dim %c1_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<7xi32>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<7xi32>, !fir.ref<!fir.array<7xi32>>
-  hlfir.destroy %10 : !hlfir.expr<7xi32>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<7xi32>>
-  return %11 : !fir.array<7xi32>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: %{{.*}} = hlfir.count %{{.*}} dim %c1_i32
-
-
-func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %c2 = arith.constant 2 : index
-  %0 = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %0 : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
-  %7 = fir.load %5#0 : !fir.ref<i32>
-  %8 = hlfir.elemental %0 unordered : (!fir.shape<3>) -> !hlfir.expr<4x7x2x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index, %arg5: index):
-    %11 = hlfir.designate %6 (%arg3, %arg4, %arg5)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
-    %12 = fir.load %11 : !fir.ref<i32>
-    %13 = arith.cmpi sge, %12, %7 : i32
-    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %14 : !fir.logical<4>
-  }
-  %9 = hlfir.count %8 : (!hlfir.expr<4x7x2x!fir.logical<4>>) -> i32
-  hlfir.assign %9 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %8 : !hlfir.expr<4x7x2x!fir.logical<4>>
-  %10 = fir.load %4#1 : !fir.ref<i32>
-  return %10 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %c2 = arith.constant 2 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]]) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]] {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %[[V0]] : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
-// CHECK-NEXT:     %[[V7:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V8:.*]] = fir.do_loop %arg3 = %c1 to %c2 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK-NEXT:       %[[V10:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %arg4) -> (i32) {
-// CHECK-NEXT:         %[[V11:.*]] = fir.do_loop %arg7 = %c1 to %c4 step %c1 iter_args(%arg8 = %arg6) -> (i32) {
-// CHECK-NEXT:           %[[V12:.*]] = hlfir.designate %[[V6]] (%arg7, %arg5, %arg3)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
-// CHECK-NEXT:           %[[V13:.*]] = fir.load %[[V12]] : !fir.ref<i32>
-// CHECK-NEXT:           %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V7]] : i32
-// CHECK-NEXT:           %[[V15:.*]] = arith.addi %arg8, %c1_i32 : i32
-// CHECK-NEXT:           %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg8 : i32
-// CHECK-NEXT:           fir.result %[[V16]] : i32
-// CHECK-NEXT:         }
-// CHECK-NEXT:         fir.result %[[V11]] : i32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V10]] : i32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V8]] to %[[V4]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:     %[[V9:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
-// CHECK-NEXT:     return %[[V9]] : i32
-
-
-
-
-
-func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7xi32> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %16 = fir.load %15 : !fir.ref<i32>
-    hlfir.yield_element %16 : i32
-  }
-  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.sum %11 : (!hlfir.expr<7xi32>) -> i32
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7xi32>
-  %14 = fir.load %4#1 : !fir.ref<i32>
-  return %14 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK:    %[[V12:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK:      %[[V14:.*]] = hlfir.sum %[[V11]] : (!hlfir.expr<7xi32>) -> i32
-// CHECK:      %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK:      %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK:      %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
-// CHECK:      fir.result %[[V17]] : i32
-// CHECK:    }
-
-
-
-
-func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %16 = fir.load %15 : !fir.ref<i32>
-    %17 = arith.cmpi sge, %16, %10 : i32
-    %18 = fir.convert %17 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %18 : !fir.logical<4>
-  }
-  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %14 = fir.load %4#1 : !fir.ref<i32>
-  return %14 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK:    %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK:      %[[V13:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %c0_i32) -> (i32) {
-// CHECK:        %[[V17:.*]] = hlfir.designate %[[V9]] (%arg5)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK:        %[[V18:.*]] = fir.load %[[V17]] : !fir.ref<i32>
-// CHECK:        %[[V19:.*]] = arith.cmpi sge, %[[V18]], %[[V10]] : i32
-// CHECK:        %[[V20:.*]] = arith.addi %arg6, %c1_i32 : i32
-// CHECK:        %[[V21:.*]] = arith.select %[[V19]], %[[V20]], %arg6 : i32
-// CHECK:        fir.result %[[V21]] : i32
-// CHECK:      }
-// CHECK:      %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V10]] : i32
-// CHECK:      %[[V15:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK:      %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg4 : i32
-// CHECK:      fir.result %[[V16]] : i32
-// CHECK:    }
diff --git a/flang/test/HLFIR/maxloc-elemental.fir b/flang/test/HLFIR/maxloc-elemental.fir
deleted file mode 100644
index c9210a59f0340..0000000000000
--- a/flang/test/HLFIR/maxloc-elemental.fir
+++ /dev/null
@@ -1,133 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
-// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c-2147483648_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi sgt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3 = fir.load %2#0 : !fir.ref<f32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-    %9 = fir.load %8 : !fir.ref<f32>
-    %10 = arith.cmpf oge, %9, %3 : f32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: _QPtest_float
-// CHECK:        %cst = arith.constant 0xFF800000 : f32
-// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
-// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
-// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
-// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
-// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
-// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf ogt, %[[V20]], %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
-// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
-// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
-// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
-// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:         fir.result %[[V20]] : f32
-// CHECK-NEXT:       } else {
-// CHECK-NEXT:         fir.result %arg4 : f32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V22]] : f32
-// CHECK-NEXT:     } else {
-// CHECK-NEXT:       fir.result %arg4 : f32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     fir.result %[[V18]] : f32
-// CHECK-NEXT:   }
-
diff --git a/flang/test/HLFIR/maxval-elemental.fir b/flang/test/HLFIR/maxval-elemental.fir
deleted file mode 100644
index a21b4858412de..0000000000000
--- a/flang/test/HLFIR/maxval-elemental.fir
+++ /dev/null
@@ -1,117 +0,0 @@
-// Test maxval inlining for both elemental and designate
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-// subroutine test(array)
-//   integer :: array(:), x
-//   x = maxval(abs(array))
-// end subroutine test
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-  %c31_i32 = arith.constant 31 : i32
-  %c0 = arith.constant 0 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg1: index):
-    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.shrsi %9, %c31_i32 : i32
-    %11 = arith.xori %9, %10 : i32
-    %12 = arith.subi %11, %10 : i32
-    hlfir.yield_element %12 : i32
-  }
-  %7 = hlfir.maxval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
-  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
-  hlfir.destroy %6 : !hlfir.expr<?xi32>
-  return
-}
-
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
-// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c-2147483648_i32) -> (i32) {
-// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
-// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi sgt, %[[V10]], %arg2 : i32
-// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
-// CHECK-NEXT:      fir.result %[[V12]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// subroutine test(array)
-//   real :: array(:), x
-//   x = maxval(array(3:6))
-// end subroutine test
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
-  %c4 = arith.constant 4 : index
-  %c1 = arith.constant 1 : index
-  %c6 = arith.constant 6 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-  %6 = hlfir.maxval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
-  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
-  return
-}
-
-// CHECK-LABEL: _QPtest_float
-// CHECK:       %cst = arith.constant 0xFF800000 : f32
-// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
-// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
-// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
-// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
-// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
-// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
-// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
-// CHECK-NEXT:      fir.result %[[V14]] : f32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// Verify that lower bounds of designator are applied in the indexing inside
-// the generated loop (hlfir.designate takes indices relative to the base lower
-// bounds).
-func.func @component_lower_bounds(%arg0: !fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>) -> i32 {
-  %c10 = arith.constant 10 : index
-  %c101 = arith.constant 101 : index
-  %4 = fir.shape_shift %c101, %c10 : (index, index) -> !fir.shapeshift<1>
-  %5 = hlfir.designate %arg0{"i"}   shape %4 : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
-  %6 = hlfir.maxval %5 : (!fir.box<!fir.array<10xi32>>) -> i32
-  return %6 : i32
-}
-// CHECK-LABEL:   func.func @component_lower_bounds(
-// CHECK:  %[[VAL_1:.*]] = arith.constant 100 : index
-// CHECK:  %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:  %[[VAL_4:.*]] = arith.constant 10 : index
-// CHECK:  %[[VAL_5:.*]] = arith.constant 101 : index
-// CHECK:  %[[VAL_6:.*]] = fir.shape_shift %[[VAL_5]], %[[VAL_4]] : (index, index) -> !fir.shapeshift<1>
-// CHECK:  %[[VAL_7:.*]] = hlfir.designate %{{.*}}{"i"}   shape %[[VAL_6]] : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
-// CHECK:  %[[VAL_8:.*]] = fir.do_loop %[[VAL_9:.*]] = %[[VAL_2]] to %[[VAL_4]] {{.*}}
-// CHECK:    %[[VAL_11:.*]] = arith.addi %[[VAL_9]], %[[VAL_1]] : index
-// CHECK:    hlfir.designate %[[VAL_7]] (%[[VAL_11]])  : (!fir.box<!fir.array<10xi32>>, index) -> !fir.ref<i32>
diff --git a/flang/test/HLFIR/minloc-elemental.fir b/flang/test/HLFIR/minloc-elemental.fir
deleted file mode 100644
index 9453a335b4fbf..0000000000000
--- a/flang/test/HLFIR/minloc-elemental.fir
+++ /dev/null
@@ -1,397 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-
-func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi16>, !fir.box<!fir.array<?xi16>>
-  hlfir.destroy %7 : !hlfir.expr<1xi16>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL:  func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c1_i16 = arith.constant 1 : i16
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i16 = arith.constant 0 : i16
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i16
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK-NEXT:    fir.store %c0_i16 to %[[V8]] : !fir.ref<i16>
-// CHECK-NEXT:    fir.store %c0_i16 to %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i16) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i16 to %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i16
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i16>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi16>>, !fir.box<!fir.array<?xi16>>
-// CHECK-NEXT:    return
-
-
-func.func @_QPtest_kind2_convert(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c1 = arith.constant 1 : index
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %10 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %11 = fir.load %10 : !fir.ref<i32>
-    %12 = arith.cmpi sge, %11, %3 : i32
-    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %13 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %9 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg3: index):
-    %10 = hlfir.apply %7, %arg3 : (!hlfir.expr<1xi16>, index) -> i16
-    %11 = fir.convert %10 : (i16) -> i32
-    hlfir.yield_element %11 : i32
-  }
-  hlfir.assign %9 to %1#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %9 : !hlfir.expr<?xi32>
-  hlfir.destroy %7 : !hlfir.expr<1xi16>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL:   func.func @_QPtest_kind2_convert(
-// CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"},
-// CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "val"},
-// CHECK-SAME:                                     %[[VAL_2:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK:           %[[VAL_3:.*]] = arith.constant false
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i16
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_10:.*]] = fir.alloca i16
-// CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
-// CHECK:           %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_16]] : !fir.ref<i16>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<i16>
-// CHECK:           %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_12]]#0, %[[VAL_7]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#1, %[[VAL_9]] : index
-// CHECK:           %[[VAL_19:.*]] = fir.do_loop %[[VAL_20:.*]] = %[[VAL_7]] to %[[VAL_18]] step %[[VAL_9]] iter_args(%[[VAL_21:.*]] = %[[VAL_5]]) -> (i32) {
-// CHECK:             %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_9]] : index
-// CHECK:             %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<i32>
-// CHECK:             %[[VAL_25:.*]] = arith.cmpi sge, %[[VAL_24]], %[[VAL_15]] : i32
-// CHECK:             %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (i32) {
-// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_10]] : !fir.ref<i16>
-// CHECK:               %[[VAL_28:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_28]] : !fir.ref<i32>
-// CHECK:               %[[VAL_30:.*]] = arith.cmpi slt, %[[VAL_29]], %[[VAL_21]] : i32
-// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_27]] : (i16) -> i1
-// CHECK:               %[[VAL_32:.*]] = arith.xori %[[VAL_31]], %[[VAL_4]] : i1
-// CHECK:               %[[VAL_33:.*]] = arith.ori %[[VAL_30]], %[[VAL_32]] : i1
-// CHECK:               %[[VAL_34:.*]] = fir.if %[[VAL_33]] -> (i32) {
-// CHECK:                 fir.store %[[VAL_6]] to %[[VAL_10]] : !fir.ref<i16>
-// CHECK:                 %[[VAL_35:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_22]] : (index) -> i16
-// CHECK:                 fir.store %[[VAL_36]] to %[[VAL_35]] : !fir.ref<i16>
-// CHECK:                 fir.result %[[VAL_29]] : i32
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_21]] : i32
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_34]] : i32
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_21]] : i32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_26]] : i32
-// CHECK:           }
-// CHECK:           %[[VAL_37:.*]] = hlfir.as_expr %[[VAL_11]] move %[[VAL_3]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
-// CHECK:           fir.do_loop %[[VAL_38:.*]] = %[[VAL_9]] to %[[VAL_9]] step %[[VAL_9]] unordered {
-// CHECK:             %[[VAL_39:.*]] = hlfir.apply %[[VAL_37]], %[[VAL_38]] : (!hlfir.expr<1xi16>, index) -> i16
-// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i16) -> i32
-// CHECK:             %[[VAL_41:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_38]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:             hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i32, !fir.ref<i32>
-// CHECK:           }
-// CHECK:           return
-// CHECK:         }
-
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3 = fir.load %2#0 : !fir.ref<f32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-    %9 = fir.load %8 : !fir.ref<f32>
-    %10 = arith.cmpf oge, %9, %3 : f32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: _QPtest_float
-// CHECK:        %cst = arith.constant 0x7F800000 : f32
-// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
-// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
-// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
-// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
-// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
-// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf olt, %[[V20]], %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
-// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
-// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
-// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
-// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:         fir.result %[[V20]] : f32
-// CHECK-NEXT:       } else {
-// CHECK-NEXT:         fir.result %arg4 : f32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V22]] : f32
-// CHECK-NEXT:     } else {
-// CHECK-NEXT:       fir.result %arg4 : f32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     fir.result %[[V18]] : f32
-// CHECK-NEXT:   }
-
-
-func.func @_QPtest_assignshape(%arg0: !fir.ref<!fir.array<3x3xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.ref<!fir.array<3xi32>> {fir.bindc_name = "m"}) {
-  %c2 = arith.constant 2 : index
-  %c1 = arith.constant 1 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.shape %c3, %c3 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFtestEarray"} : (!fir.ref<!fir.array<3x3xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x3xf32>>, !fir.ref<!fir.array<3x3xf32>>)
-  %2 = fir.shape %c3 : (index) -> !fir.shape<1>
-  %3:2 = hlfir.declare %arg2(%2) {uniq_name = "_QFtestEm"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-  %4:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %5 = fir.load %4#0 : !fir.ref<f32>
-  %6 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<3x3x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %10 = hlfir.designate %1#0 (%arg3, %arg4)  : (!fir.ref<!fir.array<3x3xf32>>, index, index) -> !fir.ref<f32>
-    %11 = fir.load %10 : !fir.ref<f32>
-    %12 = arith.cmpf oge, %11, %5 : f32
-    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %13 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %1#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.ref<!fir.array<3x3xf32>>, !hlfir.expr<3x3x!fir.logical<4>>) -> !hlfir.expr<2xi32>
-  %8 = fir.shape %c2 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %3#0 (%c1:%c2:%c1)  shape %8 : (!fir.ref<!fir.array<3xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<2xi32>>
-  hlfir.assign %7 to %9 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
-  hlfir.destroy %7 : !hlfir.expr<2xi32>
-  hlfir.destroy %6 : !hlfir.expr<3x3x!fir.logical<4>>
-  return
-}
-// Not supported as the input is not a box
-// CHECK-LABEL: _QPtest_assignshape
-// CHECK: hlfir.minloc
-
-
-func.func @_QFPtest_character(%arg0: !fir.box<!fir.array<?x!fir.char<1>>> {fir.bindc_name = "b"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c0 = arith.constant 0 : index
-  %c1 = arith.constant 1 : index
-  %0:2 = hlfir.declare %arg0 typeparams %c1 {uniq_name = "_QFFtestEb"} : (!fir.box<!fir.array<?x!fir.char<1>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1>>>, !fir.box<!fir.array<?x!fir.char<1>>>)
-  %1:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestEc"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca !fir.array<1xi32> {bindc_name = "m", uniq_name = "_QFFtestEm"}
-  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %4:2 = hlfir.declare %2(%3) {uniq_name = "_QFFtestEm"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
-  %5 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %6:2 = hlfir.declare %5 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %8 = fir.load %7#0 : !fir.ref<i32>
-  %9:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %10 = fir.shape %9#1 : (index) -> !fir.shape<1>
-  %11 = hlfir.elemental %10 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %16 = hlfir.designate %1#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %17 = fir.load %16 : !fir.ref<i32>
-    %18 = arith.cmpi eq, %17, %8 : i32
-    %19 = fir.convert %18 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %19 : !fir.logical<4>
-  }
-  %12 = hlfir.minloc %0#0 mask %11 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x!fir.char<1>>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %12 to %4#0 : !hlfir.expr<1xi32>, !fir.ref<!fir.array<1xi32>>
-  hlfir.destroy %12 : !hlfir.expr<1xi32>
-  hlfir.destroy %11 : !hlfir.expr<?x!fir.logical<4>>
-  %13 = hlfir.designate %4#0 (%c1)  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-  %14 = fir.load %13 : !fir.ref<i32>
-  hlfir.assign %14 to %6#0 : i32, !fir.ref<i32>
-  %15 = fir.load %6#1 : !fir.ref<i32>
-  return %15 : i32
-}
-// Characters are not supported at the moment
-// CHECK-LABEL: _QFPtest_character
-// CHECK: hlfir.minloc
-
-
-func.func @_QPtest_parts(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "mask"}) -> f32 {
-  %c1 = arith.constant 1 : index
-  %c5 = arith.constant 5 : index
-  %c0 = arith.constant 0 : index
-  %c5_i32 = arith.constant 5 : i32
-  %0:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEmask"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1 = fir.alloca f32 {bindc_name = "test", uniq_name = "_QFtestEtest"}
-  %2:2 = hlfir.declare %1 {uniq_name = "_QFtestEtest"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEx"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg2: index):
-    %11 = hlfir.designate %0#0 (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %12 = fir.load %11 : !fir.ref<i32>
-    %13 = arith.cmpi sge, %12, %c5_i32 : i32
-    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %14 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %3#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %3#0 (%c5:%c5:%c1)  shape %8 : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
-  hlfir.assign %7 to %9 : !hlfir.expr<1xi32>, !fir.box<!fir.array<1xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  %10 = fir.load %2#1 : !fir.ref<f32>
-  return %10 : f32
-}
-// Characters are not supported at the moment
-// CHECK-LABEL: _QPtest_parts
-// CHECK: fir.do_loop %{{.*}} = %c0 to %{{.*}} step %c1 iter_args(%{{.*}} = %c2147483647_i32) -> (i32) {
-
diff --git a/flang/test/HLFIR/minval-elemental.fir b/flang/test/HLFIR/minval-elemental.fir
deleted file mode 100644
index 64cd5403ec558..0000000000000
--- a/flang/test/HLFIR/minval-elemental.fir
+++ /dev/null
@@ -1,95 +0,0 @@
-// Test maxval inlining for both elemental and designate
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-// subroutine test(array)
-//   integer :: array(:), x
-//   x = minval(abs(array))
-// end subroutine test
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-  %c31_i32 = arith.constant 31 : i32
-  %c0 = arith.constant 0 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg1: index):
-    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.shrsi %9, %c31_i32 : i32
-    %11 = arith.xori %9, %10 : i32
-    %12 = arith.subi %11, %10 : i32
-    hlfir.yield_element %12 : i32
-  }
-  %7 = hlfir.minval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
-  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
-  hlfir.destroy %6 : !hlfir.expr<?xi32>
-  return
-}
-
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
-// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi slt, %[[V10]], %arg2 : i32
-// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
-// CHECK-NEXT:      fir.result %[[V12]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// subroutine test(array)
-//   real :: array(:), x
-//   x = minval(array(3:6))
-// end subroutine test
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
-  %c4 = arith.constant 4 : index
-  %c1 = arith.constant 1 : index
-  %c6 = arith.constant 6 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-  %6 = hlfir.minval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
-  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
-  return
-}
-
-// CHECK-LABEL: _QPtest_float
-// CHECK:       %cst = arith.constant 0x7F800000 : f32
-// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
-// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
-// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
-// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
-// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
-// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
-// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
-// CHECK-NEXT:      fir.result %[[V14]] : f32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
new file mode 100644
index 0000000000000..fb0f39811ea74
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
@@ -0,0 +1,123 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+  %0 = hlfir.all %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_14:.*]] = arith.andi %[[VAL_13]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_14]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_15]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.all %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
+// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = arith.andi %[[VAL_15]], %[[VAL_13]] : i1
+// CHECK:               fir.result %[[VAL_16]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
+// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+  %0 = hlfir.all %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant true
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
+// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_20]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_21]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_22]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.all %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant true
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
+// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_27:.*]] = arith.andi %[[VAL_26]], %[[VAL_14]] : i1
+// CHECK:               fir.result %[[VAL_27]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
+// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
new file mode 100644
index 0000000000000..5bd76f3d24927
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
@@ -0,0 +1,123 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+  %0 = hlfir.any %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_14:.*]] = arith.ori %[[VAL_13]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_14]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_15]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.any %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
+// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = arith.ori %[[VAL_15]], %[[VAL_13]] : i1
+// CHECK:               fir.result %[[VAL_16]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
+// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+  %0 = hlfir.any %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
+// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_21:.*]] = arith.ori %[[VAL_20]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_21]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_22]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.any %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
+// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_14]] : i1
+// CHECK:               fir.result %[[VAL_27]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
+// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
new file mode 100644
index 0000000000000..44594c646a368
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
@@ -0,0 +1,127 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
+  %0 = hlfir.count %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (i32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
+// CHECK:               %[[VAL_16:.*]] = arith.select %[[VAL_14]], %[[VAL_15]], %[[VAL_12]] : i32
+// CHECK:               fir.result %[[VAL_16]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : i32
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.count %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?xi16>
+  return %0 : !hlfir.expr<?x?xi16>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i16
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]], %[[VAL_7]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi16> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (i16) {
+// CHECK:               %[[VAL_15:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_13]], %[[VAL_11]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_14]], %[[VAL_1]] : i16
+// CHECK:               %[[VAL_18:.*]] = arith.select %[[VAL_16]], %[[VAL_17]], %[[VAL_14]] : i16
+// CHECK:               fir.result %[[VAL_18]] : i16
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : i16
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?xi16>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
+  %0 = hlfir.count %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (i32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_15:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.addi %[[VAL_11]], %[[VAL_15]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_18:.*]] = arith.addi %[[VAL_8]], %[[VAL_17]] : index
+// CHECK:               %[[VAL_19:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_16]], %[[VAL_18]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_21]], %[[VAL_22]], %[[VAL_12]] : i32
+// CHECK:               fir.result %[[VAL_23]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : i32
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.count %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?xi64>
+  return %0 : !hlfir.expr<?x?xi64>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_6]]#1, %[[VAL_8]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi64> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):
+// CHECK:             %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_15:.*]] = %[[VAL_2]]) -> (i64) {
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_14]], %[[VAL_21]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_24:.*]] = arith.addi %[[VAL_12]], %[[VAL_23]] : index
+// CHECK:               %[[VAL_25:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_28:.*]] = arith.addi %[[VAL_15]], %[[VAL_1]] : i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_15]] : i64
+// CHECK:               fir.result %[[VAL_29]] : i64
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_13]] : i64
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?x?xi64>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
new file mode 100644
index 0000000000000..49e3bb6936338
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
@@ -0,0 +1,343 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_1d_total_expr(
+// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
+// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -2147483648 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
+// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
+// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_26:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
+// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_28]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
+// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
+// CHECK:         }
+
+func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+  %dim = arith.constant 1 : i16
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_1d_dim_expr(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
+// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
+// CHECK:               %[[VAL_18:.*]] = arith.cmpf ogt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
+// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
+// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
+// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
+// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
+// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_29:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (index) -> i32
+// CHECK:           return %[[VAL_30]] : i32
+// CHECK:         }
+
+func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  return %0 : !hlfir.expr<1xi16>
+}
+// CHECK-LABEL:   func.func @test_1d_total_var(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf ogt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
+// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
+// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
+// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
+// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_34:.*]]#0, %[[VAL_6]] : i16
+// CHECK:           %[[VAL_35:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_34]]#0 : (i16) -> index
+// CHECK:           %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_35]]#0 : index
+// CHECK:           %[[VAL_38:.*]] = arith.subi %[[VAL_37]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_33]], %[[VAL_38]], %[[VAL_36]] : index
+// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (index) -> i16
+// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i16, !fir.ref<i16>
+// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
+// CHECK:         }
+
+func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
+  return %0 : i64
+}
+// CHECK-LABEL:   func.func @test_1d_dim_var(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
+// CHECK:               %[[VAL_22:.*]] = arith.cmpf ogt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
+// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
+// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
+// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
+// CHECK:           }
+// CHECK:           %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_33:.*]]#0, %[[VAL_6]] : i64
+// CHECK:           %[[VAL_34:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]]#0 : (i64) -> index
+// CHECK:           %[[VAL_36:.*]] = arith.addi %[[VAL_35]], %[[VAL_34]]#0 : index
+// CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_36]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_32]], %[[VAL_37]], %[[VAL_35]] : index
+// CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
+// CHECK:           return %[[VAL_39]] : i64
+// CHECK:         }
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
+  return %0 : !hlfir.expr<3xi32>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant true
+// CHECK:           %[[VAL_7:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
+// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
+// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
+// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
+// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
+// CHECK:                   %[[VAL_39:.*]] = arith.cmpf ogt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
+// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
+// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
+// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
+// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
+// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
+// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
+// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
+// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
+// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
+// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
+// CHECK:                 } else {
+// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
+// CHECK:                 }
+// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_56:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_57:.*]] = fir.convert %[[VAL_55]] : (index) -> i32
+// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_57]] to %[[VAL_58]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_59:.*]] = fir.convert %[[VAL_56]]#1 : (i32) -> index
+// CHECK:           %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (index) -> i32
+// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_60]] to %[[VAL_61]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_62:.*]] = fir.convert %[[VAL_56]]#2 : (i32) -> index
+// CHECK:           %[[VAL_63:.*]] = fir.convert %[[VAL_62]] : (index) -> i32
+// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_63]] to %[[VAL_64]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
+// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
+  return %0 : !hlfir.expr<?x?xi32>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
+// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
+// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
+// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
+// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
+// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
+// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
+// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
+// CHECK:               }
+// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
+// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
+// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
+// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
+// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_48:.*]] = arith.cmpf ogt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
+// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
+// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
+// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
+// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
+// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
+// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
+// CHECK:             }
+// CHECK:             %[[VAL_58:.*]] = arith.cmpi ne, %[[VAL_59:.*]]#0, %[[VAL_5]] : i32
+// CHECK:             %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_59]]#0 : (i32) -> index
+// CHECK:             %[[VAL_62:.*]] = arith.addi %[[VAL_61]], %[[VAL_60]]#0 : index
+// CHECK:             %[[VAL_63:.*]] = arith.subi %[[VAL_62]], %[[VAL_7]] : index
+// CHECK:             %[[VAL_64:.*]] = arith.select %[[VAL_58]], %[[VAL_63]], %[[VAL_61]] : index
+// CHECK:             %[[VAL_65:.*]] = fir.convert %[[VAL_64]] : (index) -> i32
+// CHECK:             hlfir.yield_element %[[VAL_65]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
+// CHECK:         }
+
+// Character comparisons are not supported yet.
+func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.maxloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_character(
+// CHECK:           hlfir.maxloc
+
+// BACK is not supported yet.
+func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
+  %back = arith.constant true
+  %0 = hlfir.maxloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_back(
+// CHECK:           hlfir.maxloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
new file mode 100644
index 0000000000000..4022b7d3b2b77
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
@@ -0,0 +1,177 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
+  return %0 : f32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (f32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f32) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f32
+// CHECK:                 fir.result %[[VAL_22]] : f32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]] : f32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : f32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : f32
+// CHECK:         }
+
+func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
+  return %0 : !hlfir.expr<?xf64>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant -1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f64) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f64) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f64
+// CHECK:                 fir.result %[[VAL_22]] : f64
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f64
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]] : f64
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_10]] : f64
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
+// CHECK:         }
+
+func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
+  return %0 : f16
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant -6.550400e+04 : f16
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (f16) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f16) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f16) {
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
+// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_25:.*]] = arith.cmpf ogt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
+// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
+// CHECK:                 %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_24]], %[[VAL_12]] : f16
+// CHECK:                 fir.result %[[VAL_30]] : f16
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f16
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_16]] : f16
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : f16
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
+  return %0 : !hlfir.expr<?xf16>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant -6.550400e+04 : f16
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (f16) {
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
+// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
+// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
+// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
+// CHECK:               }
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (f16) {
+// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
+// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_35:.*]] = arith.cmpf ogt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
+// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
+// CHECK:                 %[[VAL_40:.*]] = arith.select %[[VAL_39]], %[[VAL_34]], %[[VAL_14]] : f16
+// CHECK:                 fir.result %[[VAL_40]] : f16
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_14]] : f16
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_26]] : f16
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
new file mode 100644
index 0000000000000..96dcebaf9a2cd
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
@@ -0,0 +1,343 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_1d_total_expr(
+// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
+// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
+// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
+// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_26:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
+// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_28]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
+// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
+// CHECK:         }
+
+func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+  %dim = arith.constant 1 : i16
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_1d_dim_expr(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
+// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
+// CHECK:               %[[VAL_18:.*]] = arith.cmpf olt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
+// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
+// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
+// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
+// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
+// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_29:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (index) -> i32
+// CHECK:           return %[[VAL_30]] : i32
+// CHECK:         }
+
+func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  return %0 : !hlfir.expr<1xi16>
+}
+// CHECK-LABEL:   func.func @test_1d_total_var(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf olt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
+// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
+// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
+// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
+// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_34:.*]]#0, %[[VAL_6]] : i16
+// CHECK:           %[[VAL_35:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_34]]#0 : (i16) -> index
+// CHECK:           %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_35]]#0 : index
+// CHECK:           %[[VAL_38:.*]] = arith.subi %[[VAL_37]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_33]], %[[VAL_38]], %[[VAL_36]] : index
+// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (index) -> i16
+// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i16, !fir.ref<i16>
+// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
+// CHECK:         }
+
+func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
+  return %0 : i64
+}
+// CHECK-LABEL:   func.func @test_1d_dim_var(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
+// CHECK:               %[[VAL_22:.*]] = arith.cmpf olt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
+// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
+// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
+// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
+// CHECK:           }
+// CHECK:           %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_33:.*]]#0, %[[VAL_6]] : i64
+// CHECK:           %[[VAL_34:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]]#0 : (i64) -> index
+// CHECK:           %[[VAL_36:.*]] = arith.addi %[[VAL_35]], %[[VAL_34]]#0 : index
+// CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_36]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_32]], %[[VAL_37]], %[[VAL_35]] : index
+// CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
+// CHECK:           return %[[VAL_39]] : i64
+// CHECK:         }
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
+  return %0 : !hlfir.expr<3xi32>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant true
+// CHECK:           %[[VAL_7:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
+// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
+// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
+// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
+// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
+// CHECK:                   %[[VAL_39:.*]] = arith.cmpf olt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
+// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
+// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
+// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
+// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
+// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
+// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
+// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
+// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
+// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
+// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
+// CHECK:                 } else {
+// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
+// CHECK:                 }
+// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_56:.*]]#0 : (i32) -> index
+// CHECK:           %[[VAL_57:.*]] = fir.convert %[[VAL_55]] : (index) -> i32
+// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_57]] to %[[VAL_58]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_59:.*]] = fir.convert %[[VAL_56]]#1 : (i32) -> index
+// CHECK:           %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (index) -> i32
+// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_60]] to %[[VAL_61]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_62:.*]] = fir.convert %[[VAL_56]]#2 : (i32) -> index
+// CHECK:           %[[VAL_63:.*]] = fir.convert %[[VAL_62]] : (index) -> i32
+// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_63]] to %[[VAL_64]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
+// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
+  return %0 : !hlfir.expr<?x?xi32>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
+// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
+// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
+// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
+// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
+// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
+// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
+// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
+// CHECK:               }
+// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
+// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
+// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
+// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
+// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_48:.*]] = arith.cmpf olt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
+// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
+// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
+// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
+// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
+// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
+// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
+// CHECK:             }
+// CHECK:             %[[VAL_58:.*]] = arith.cmpi ne, %[[VAL_59:.*]]#0, %[[VAL_5]] : i32
+// CHECK:             %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_59]]#0 : (i32) -> index
+// CHECK:             %[[VAL_62:.*]] = arith.addi %[[VAL_61]], %[[VAL_60]]#0 : index
+// CHECK:             %[[VAL_63:.*]] = arith.subi %[[VAL_62]], %[[VAL_7]] : index
+// CHECK:             %[[VAL_64:.*]] = arith.select %[[VAL_58]], %[[VAL_63]], %[[VAL_61]] : index
+// CHECK:             %[[VAL_65:.*]] = fir.convert %[[VAL_64]] : (index) -> i32
+// CHECK:             hlfir.yield_element %[[VAL_65]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
+// CHECK:         }
+
+// Character comparisons are not supported yet.
+func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.minloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_character(
+// CHECK:           hlfir.minloc
+
+// BACK is not supported yet.
+func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
+  %back = arith.constant true
+  %0 = hlfir.minloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_back(
+// CHECK:           hlfir.minloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
new file mode 100644
index 0000000000000..4278ac7abd457
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
@@ -0,0 +1,177 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
+  return %0 : f32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (f32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f32) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f32
+// CHECK:                 fir.result %[[VAL_22]] : f32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]] : f32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : f32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : f32
+// CHECK:         }
+
+func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
+  return %0 : !hlfir.expr<?xf64>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f64) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f64) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f64
+// CHECK:                 fir.result %[[VAL_22]] : f64
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f64
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]] : f64
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_10]] : f64
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
+// CHECK:         }
+
+func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
+  return %0 : f16
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 6.550400e+04 : f16
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (f16) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f16) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f16) {
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
+// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_25:.*]] = arith.cmpf olt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
+// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
+// CHECK:                 %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_24]], %[[VAL_12]] : f16
+// CHECK:                 fir.result %[[VAL_30]] : f16
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]] : f16
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_16]] : f16
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : f16
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
+  return %0 : !hlfir.expr<?xf16>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant 6.550400e+04 : f16
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (f16) {
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
+// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
+// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
+// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
+// CHECK:               }
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (f16) {
+// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
+// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_35:.*]] = arith.cmpf olt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
+// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
+// CHECK:                 %[[VAL_40:.*]] = arith.select %[[VAL_39]], %[[VAL_34]], %[[VAL_14]] : f16
+// CHECK:                 fir.result %[[VAL_40]] : f16
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_14]] : f16
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_26]] : f16
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
+// CHECK:         }

From f0621b31f8baee3013bec74b2789b9d5d9f9d08f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 17 Apr 2025 16:23:51 -0700
Subject: [PATCH 340/710] [Vectorize] Fix a warning

This patch fixes:

  llvm/lib/Transforms/Vectorize/VPlanTransforms.h:31:1: error: class
  'VFRange' was previously declared as a struct; this is valid, but
  may result in linker errors under the Microsoft C++ ABI
  [-Werror,-Wmismatched-tags]
---
 llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 1200e8eaab0ba..65f3c837f7721 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -28,7 +28,7 @@ class PredicatedScalarEvolution;
 class TargetLibraryInfo;
 class VPBuilder;
 class VPRecipeBuilder;
-class VFRange;
+struct VFRange;
 
 extern cl::opt<bool> VerifyEachVPlan;
 

From bb67de671eeda3b763ed6f1af05bd921b0ad7a29 Mon Sep 17 00:00:00 2001
From: Alexey Samsonov <vonosmas@gmail.com>
Date: Thu, 17 Apr 2025 16:33:38 -0700
Subject: [PATCH 341/710] [libc][bazel] Enforce that libc hand-in-hand libs are
 headers-only. (#136219)

Extend Bazel rule implementation to enforce that all transitive
dependencies of libc_header_library targets (used to implement
hand-in-hand code sharing via headers) indeed only contain header files.

This fixes Bazel portion of PR #133126.
---
 .../libc/libc_build_rules.bzl                 | 28 ++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
index 60add23a46c48..7fe263ab08f71 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
@@ -123,12 +123,15 @@ _get_libc_info_aspect = aspect(
 )
 
 def _libc_srcs_filegroup_impl(ctx):
-    return DefaultInfo(
-        files = depset(transitive = [
-            fn[LibcLibraryInfo].srcs
-            for fn in ctx.attr.libs
-        ]),
-    )
+    srcs = depset(transitive = [
+        fn[LibcLibraryInfo].srcs
+        for fn in ctx.attr.libs
+    ])
+    if ctx.attr.enforce_headers_only:
+        paths = [f.short_path for f in srcs.to_list() if f.extension != "h"]
+        if paths:
+            fail("Unexpected non-header files: {}".format(paths))
+    return DefaultInfo(files = srcs)
 
 _libc_srcs_filegroup = rule(
     doc = "Returns all sources for building the specified libraries.",
@@ -138,6 +141,7 @@ _libc_srcs_filegroup = rule(
             mandatory = True,
             aspects = [_get_libc_info_aspect],
         ),
+        "enforce_headers_only": attr.bool(default = False),
     },
 )
 
@@ -218,6 +222,7 @@ def libc_header_library(name, hdrs, deps = [], **kwargs):
     _libc_srcs_filegroup(
         name = name + "_hdr_deps",
         libs = deps,
+        enforce_headers_only = True,
     )
 
     _libc_textual_hdrs_filegroup(
@@ -231,13 +236,10 @@ def libc_header_library(name, hdrs, deps = [], **kwargs):
 
     native.cc_library(
         name = name,
-        # Technically speaking, we should put _hdr_deps in srcs, as they are
-        # not a part of this cc_library interface. However, we keep it here to
-        # workaround the presence of .cpp files in _hdr_deps - we need to
-        # fix that and enforce their absence, since libc_header_library
-        # should be header-only and not produce any object files.
-        # See PR #133126 which tracks it.
-        hdrs = hdrs + [":" + name + "_hdr_deps"],
+        hdrs = hdrs,
+        # We put _hdr_deps in srcs, as they are not a part of this cc_library
+        # interface, but instead are used to implement shared headers.
+        srcs = [":" + name + "_hdr_deps"],
         deps = [":" + name + "_textual_hdr_library"],
         # copts don't really matter, since it's a header-only library, but we
         # need proper -I flags for header validation, which are specified in

From 62b9cbd8782b2ded15efed67ae10419e75ea0fa7 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 17 Apr 2025 16:30:47 -0700
Subject: [PATCH 342/710] [RISCV] Simplify check lines in
 vector-deinterleave.ll [nfc]

RV32 and RV64 are unused.  Output with zvfh vs zvfhmin are the same,
so just use the weaker predicate.
---
 llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index 883516ea3ff1e..2787bef0c893f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -1,8 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV64
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s
 
 ; Integers
 
@@ -1067,6 +1065,3 @@ define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2
   %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave7.nxv14i64(<vscale x 14 x i64> %vec)
   ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RV32: {{.*}}
-; RV64: {{.*}}

From ebe084f927f14be707d3ca64dab0faaf6c0eee9d Mon Sep 17 00:00:00 2001
From: Volodymyr Sapsai <vsapsai@apple.com>
Date: Thu, 17 Apr 2025 17:09:26 -0700
Subject: [PATCH 343/710] [Modules] Fix the inconsistency of which `Decl`
 should be serialized for an identifier. (#135887)

Fixes the assertion failure
> Assertion failed: (DeclIDs.contains(D) && "Declaration not emitted!"),
function getDeclID, file ASTWriter.cpp, line 6873.

We prepare to serialize a `Decl` by adding it to `DeclIDs` in
`ASTWriter::GetDeclRef`. But the checks before this call aren't the same
as when we are actually serializing a `Decl` in
`ASTIdentifierTableTrait::EmitData` and
`ASTWriter::WriteIdentifierTable`. That's how we can end up serializing
a `Decl` not present in `DeclIDs` and hitting the assertion. With the
assertions disabled clang crashes when trying to use a deserialized null
`Decl`.

Fix by making the code checks before `ASTWriter::GetDeclRef` call
similar to those we have before the serialization.

rdar://139319683
---
 clang/lib/Serialization/ASTWriter.cpp     |   8 +-
 clang/test/Modules/non-modular-decl-use.c | 100 ++++++++++++++++++++++
 2 files changed, 105 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/Modules/non-modular-decl-use.c

diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 5b59087a5a164..e969214e633f3 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5670,14 +5670,16 @@ void ASTWriter::PrepareWritingSpecialDecls(Sema &SemaRef) {
     llvm::SmallVector<const IdentifierInfo*, 256> IIs;
     for (const auto &ID : SemaRef.PP.getIdentifierTable()) {
       const IdentifierInfo *II = ID.second;
-      if (!Chain || !II->isFromAST() || II->hasChangedSinceDeserialization())
+      if (!Chain || !II->isFromAST() || II->hasChangedSinceDeserialization() ||
+          II->hasFETokenInfoChangedSinceDeserialization())
         IIs.push_back(II);
     }
     // Sort the identifiers to visit based on their name.
     llvm::sort(IIs, llvm::deref<std::less<>>());
+    const LangOptions &LangOpts = getLangOpts();
     for (const IdentifierInfo *II : IIs)
-      for (const Decl *D : SemaRef.IdResolver.decls(II))
-        GetDeclRef(D);
+      for (NamedDecl *D : SemaRef.IdResolver.decls(II))
+        GetDeclRef(getDeclForLocalLookup(LangOpts, D));
   }
 
   // Write all of the DeclsToCheckForDeferredDiags.
diff --git a/clang/test/Modules/non-modular-decl-use.c b/clang/test/Modules/non-modular-decl-use.c
new file mode 100644
index 0000000000000..1bd87bf284620
--- /dev/null
+++ b/clang/test/Modules/non-modular-decl-use.c
@@ -0,0 +1,100 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -fsyntax-only -I %t/include %t/test.c \
+// RUN:            -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache
+
+// Test when a decl is present in multiple modules through an inclusion of
+// a non-modular header. Make sure such decl is serialized correctly and can be
+// used after deserialization.
+
+//--- include/non-modular.h
+#ifndef NON_MODULAR_H
+#define NON_MODULAR_H
+
+union TestUnion {
+  int x;
+  float y;
+};
+
+struct ReferenceUnion1 {
+  union TestUnion name;
+  unsigned versionMajor;
+};
+struct ReferenceUnion2 {
+  union TestUnion name;
+  unsigned versionMinor;
+};
+
+// Test another kind of RecordDecl.
+struct TestStruct {
+  int p;
+  float q;
+};
+
+struct ReferenceStruct1 {
+  unsigned fieldA;
+  struct TestStruct fieldB;
+};
+
+struct ReferenceStruct2 {
+  unsigned field1;
+  struct TestStruct field2;
+};
+
+#endif
+
+//--- include/piecewise1-empty.h
+//--- include/piecewise1-initially-hidden.h
+#include <non-modular.h>
+
+//--- include/piecewise2-empty.h
+//--- include/piecewise2-initially-hidden.h
+#include <non-modular.h>
+
+//--- include/with-multiple-decls.h
+#include <piecewise1-empty.h>
+// Include the non-modular header and resolve a name duplication between decl
+// in non-modular.h and in piecewise1-initially-hidden.h, create a
+// redeclaration chain.
+#include <non-modular.h>
+// Include a decl with a duplicate name again to add more to IdentifierResolver.
+#include <piecewise2-empty.h>
+
+//--- include/module.modulemap
+module Piecewise1 {
+  module Empty {
+    header "piecewise1-empty.h"
+  }
+  module InitiallyHidden {
+    header "piecewise1-initially-hidden.h"
+    export *
+  }
+}
+
+module Piecewise2 {
+  module Empty {
+    header "piecewise2-empty.h"
+  }
+  module InitiallyHidden {
+    header "piecewise2-initially-hidden.h"
+    export *
+  }
+}
+
+module WithMultipleDecls {
+  header "with-multiple-decls.h"
+  export *
+}
+
+//--- test.c
+#include <with-multiple-decls.h>
+
+struct Test {
+  int x;
+  union TestUnion name;
+};
+
+struct Test2 {
+  struct TestStruct name;
+  float y;
+};

From 7f107c301963ad1070f2130f89655fd94f2c0714 Mon Sep 17 00:00:00 2001
From: Sirish Pande <sirpande@amd.com>
Date: Thu, 17 Apr 2025 19:21:18 -0500
Subject: [PATCH 344/710] [IndVarsSimplify] sinkUnusedInvariants is skipping
 instructions while sinking. (#135205)

While sinking instructions (that are loop invariant) from preheader to
the exit block, we are skipping instructions due to decrementing
instruction iterator twice.
---
 llvm/lib/Transforms/Scalar/IndVarSimplify.cpp | 43 ++++++------------
 .../IndVarSimplify/ARM/code-size.ll           | 22 +++++-----
 .../ARM/indvar-unroll-imm-cost.ll             |  2 +-
 .../IndVarSimplify/exit-count-select.ll       | 16 +++----
 .../IndVarSimplify/exit_value_test3.ll        |  2 +-
 .../IndVarSimplify/finite-exit-comparisons.ll | 10 ++---
 .../Transforms/IndVarSimplify/pr116483.ll     |  8 ++--
 .../test/Transforms/IndVarSimplify/pr63763.ll |  4 +-
 .../IndVarSimplify/replace-loop-exit-folds.ll | 21 ++++-----
 .../Transforms/IndVarSimplify/sentinel.ll     |  8 ++--
 .../Transforms/LoopUnroll/unroll-cleanup.ll   | 44 +++++++++----------
 .../PhaseOrdering/ARM/arm_mean_q7.ll          | 14 +++---
 12 files changed, 91 insertions(+), 103 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 9619dfdbf4123..3f04f7f57c54f 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -1096,10 +1096,12 @@ bool IndVarSimplify::sinkUnusedInvariants(Loop *L) {
   if (!Preheader) return false;
 
   bool MadeAnyChanges = false;
-  BasicBlock::iterator InsertPt = ExitBlock->getFirstInsertionPt();
-  BasicBlock::iterator I(Preheader->getTerminator());
-  while (I != Preheader->begin()) {
-    --I;
+  for (Instruction &I : llvm::make_early_inc_range(llvm::reverse(*Preheader))) {
+
+    // Skip BB Terminator.
+    if (Preheader->getTerminator() == &I)
+      continue;
+
     // New instructions were inserted at the end of the preheader.
     if (isa<PHINode>(I))
       break;
@@ -1110,28 +1112,28 @@ bool IndVarSimplify::sinkUnusedInvariants(Loop *L) {
     // memory. Note that it's okay if the instruction might have undefined
     // behavior: LoopSimplify guarantees that the preheader dominates the exit
     // block.
-    if (I->mayHaveSideEffects() || I->mayReadFromMemory())
+    if (I.mayHaveSideEffects() || I.mayReadFromMemory())
       continue;
 
-    // Skip debug info intrinsics.
-    if (isa<DbgInfoIntrinsic>(I))
+    // Skip debug or pseudo instructions.
+    if (I.isDebugOrPseudoInst())
       continue;
 
     // Skip eh pad instructions.
-    if (I->isEHPad())
+    if (I.isEHPad())
       continue;
 
     // Don't sink alloca: we never want to sink static alloca's out of the
     // entry block, and correctly sinking dynamic alloca's requires
     // checks for stacksave/stackrestore intrinsics.
     // FIXME: Refactor this check somehow?
-    if (isa<AllocaInst>(I))
+    if (isa<AllocaInst>(&I))
       continue;
 
     // Determine if there is a use in or before the loop (direct or
     // otherwise).
     bool UsedInLoop = false;
-    for (Use &U : I->uses()) {
+    for (Use &U : I.uses()) {
       Instruction *User = cast<Instruction>(U.getUser());
       BasicBlock *UseBB = User->getParent();
       if (PHINode *P = dyn_cast<PHINode>(User)) {
@@ -1150,26 +1152,9 @@ bool IndVarSimplify::sinkUnusedInvariants(Loop *L) {
       continue;
 
     // Otherwise, sink it to the exit block.
-    Instruction *ToMove = &*I;
-    bool Done = false;
-
-    if (I != Preheader->begin()) {
-      // Skip debug info intrinsics.
-      do {
-        --I;
-      } while (I->isDebugOrPseudoInst() && I != Preheader->begin());
-
-      if (I->isDebugOrPseudoInst() && I == Preheader->begin())
-        Done = true;
-    } else {
-      Done = true;
-    }
-
+    I.moveBefore(ExitBlock->getFirstInsertionPt());
+    SE->forgetValue(&I);
     MadeAnyChanges = true;
-    ToMove->moveBefore(*ExitBlock, InsertPt);
-    SE->forgetValue(ToMove);
-    if (Done) break;
-    InsertPt = ToMove->getIterator();
   }
 
   return MadeAnyChanges;
diff --git a/llvm/test/Transforms/IndVarSimplify/ARM/code-size.ll b/llvm/test/Transforms/IndVarSimplify/ARM/code-size.ll
index c7231392229c9..2003b1a72206d 100644
--- a/llvm/test/Transforms/IndVarSimplify/ARM/code-size.ll
+++ b/llvm/test/Transforms/IndVarSimplify/ARM/code-size.ll
@@ -4,30 +4,32 @@
 
 define i32 @remove_loop(i32 %size) #0 {
 ; CHECK-V8M-LABEL: @remove_loop(
+; CHECK-V8M-SAME: i32 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-V8M-NEXT:  entry:
-; CHECK-V8M-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE:%.*]], 31
+; CHECK-V8M-NEXT:    br label %[[WHILE_COND:.*]]
+; CHECK-V8M:       while.cond:
+; CHECK-V8M-NEXT:    br i1 false, label %[[WHILE_COND]], label %[[WHILE_END:.*]]
+; CHECK-V8M:       while.end:
+; CHECK-V8M-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE]], 31
 ; CHECK-V8M-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SIZE]], i32 31)
 ; CHECK-V8M-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[UMIN]]
 ; CHECK-V8M-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 5
 ; CHECK-V8M-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 5
-; CHECK-V8M-NEXT:    br label [[WHILE_COND:%.*]]
-; CHECK-V8M:       while.cond:
-; CHECK-V8M-NEXT:    br i1 false, label [[WHILE_COND]], label [[WHILE_END:%.*]]
-; CHECK-V8M:       while.end:
 ; CHECK-V8M-NEXT:    [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
 ; CHECK-V8M-NEXT:    ret i32 [[TMP4]]
 ;
 ; CHECK-V8A-LABEL: @remove_loop(
+; CHECK-V8A-SAME: i32 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-V8A-NEXT:  entry:
-; CHECK-V8A-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE:%.*]], 31
+; CHECK-V8A-NEXT:    br label %[[WHILE_COND:.*]]
+; CHECK-V8A:       while.cond:
+; CHECK-V8A-NEXT:    br i1 false, label %[[WHILE_COND]], label %[[WHILE_END:.*]]
+; CHECK-V8A:       while.end:
+; CHECK-V8A-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE]], 31
 ; CHECK-V8A-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SIZE]], i32 31)
 ; CHECK-V8A-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[UMIN]]
 ; CHECK-V8A-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 5
 ; CHECK-V8A-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 5
-; CHECK-V8A-NEXT:    br label [[WHILE_COND:%.*]]
-; CHECK-V8A:       while.cond:
-; CHECK-V8A-NEXT:    br i1 false, label [[WHILE_COND]], label [[WHILE_END:%.*]]
-; CHECK-V8A:       while.end:
 ; CHECK-V8A-NEXT:    [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
 ; CHECK-V8A-NEXT:    ret i32 [[TMP4]]
 ;
diff --git a/llvm/test/Transforms/IndVarSimplify/ARM/indvar-unroll-imm-cost.ll b/llvm/test/Transforms/IndVarSimplify/ARM/indvar-unroll-imm-cost.ll
index f907f23e0b520..2261423766792 100644
--- a/llvm/test/Transforms/IndVarSimplify/ARM/indvar-unroll-imm-cost.ll
+++ b/llvm/test/Transforms/IndVarSimplify/ARM/indvar-unroll-imm-cost.ll
@@ -77,7 +77,6 @@ define dso_local arm_aapcscc void @test(ptr nocapture %pDest, ptr nocapture read
 ; CHECK-NEXT:    [[CMP2780:%.*]] = icmp ugt i32 [[ADD25]], [[J_0_LCSSA]]
 ; CHECK-NEXT:    br i1 [[CMP2780]], label [[FOR_BODY29_PREHEADER:%.*]], label [[FOR_END40]]
 ; CHECK:       for.body29.preheader:
-; CHECK-NEXT:    [[TMP10:%.*]] = sub nsw i32 [[ADD25]], [[J_0_LCSSA]]
 ; CHECK-NEXT:    br label [[FOR_BODY29:%.*]]
 ; CHECK:       for.body29:
 ; CHECK-NEXT:    [[J_184:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY29]] ], [ [[J_0_LCSSA]], [[FOR_BODY29_PREHEADER]] ]
@@ -101,6 +100,7 @@ define dso_local arm_aapcscc void @test(ptr nocapture %pDest, ptr nocapture read
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[ADD25]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END40_LOOPEXIT:%.*]], label [[FOR_BODY29]]
 ; CHECK:       for.end40.loopexit:
+; CHECK-NEXT:    [[TMP10:%.*]] = sub nsw i32 [[ADD25]], [[J_0_LCSSA]]
 ; CHECK-NEXT:    [[SCEVGEP93:%.*]] = getelementptr i16, ptr [[PSRCB_ADDR_1_LCSSA]], i32 [[TMP10]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i16, ptr [[PSRCA_ADDR_1_LCSSA]], i32 [[TMP10]]
 ; CHECK-NEXT:    [[SCEVGEP94:%.*]] = getelementptr i32, ptr [[PDEST_ADDR_1_LCSSA]], i32 [[TMP10]]
diff --git a/llvm/test/Transforms/IndVarSimplify/exit-count-select.ll b/llvm/test/Transforms/IndVarSimplify/exit-count-select.ll
index 21806c7f2cdc3..1592b84480e3f 100644
--- a/llvm/test/Transforms/IndVarSimplify/exit-count-select.ll
+++ b/llvm/test/Transforms/IndVarSimplify/exit-count-select.ll
@@ -4,11 +4,11 @@
 define i32 @logical_and_2ops(i32 %n, i32 %m) {
 ; CHECK-LABEL: @logical_and_2ops(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[M:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    br i1 false, label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[M:%.*]]
 ; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP0]], i32 [[N:%.*]])
 ; CHECK-NEXT:    ret i32 [[UMIN]]
 ;
@@ -28,11 +28,11 @@ exit:
 define i32 @logical_or_2ops(i32 %n, i32 %m) {
 ; CHECK-LABEL: @logical_or_2ops(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[M:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[M:%.*]]
 ; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP0]], i32 [[N:%.*]])
 ; CHECK-NEXT:    ret i32 [[UMIN]]
 ;
@@ -52,13 +52,13 @@ exit:
 define i32 @logical_and_3ops(i32 %n, i32 %m, i32 %k) {
 ; CHECK-LABEL: @logical_and_3ops(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[K:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i32 [[M:%.*]]
-; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP0]], i32 [[TMP1]])
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    br i1 false, label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[K:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i32 [[M:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP0]], i32 [[TMP1]])
 ; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN]], i32 [[N:%.*]])
 ; CHECK-NEXT:    ret i32 [[UMIN1]]
 ;
@@ -80,13 +80,13 @@ exit:
 define i32 @logical_or_3ops(i32 %n, i32 %m, i32 %k) {
 ; CHECK-LABEL: @logical_or_3ops(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[K:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i32 [[M:%.*]]
-; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP0]], i32 [[TMP1]])
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[K:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i32 [[M:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP0]], i32 [[TMP1]])
 ; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN]], i32 [[N:%.*]])
 ; CHECK-NEXT:    ret i32 [[UMIN1]]
 ;
diff --git a/llvm/test/Transforms/IndVarSimplify/exit_value_test3.ll b/llvm/test/Transforms/IndVarSimplify/exit_value_test3.ll
index aba7532f5ed92..c03cd95a8c861 100644
--- a/llvm/test/Transforms/IndVarSimplify/exit_value_test3.ll
+++ b/llvm/test/Transforms/IndVarSimplify/exit_value_test3.ll
@@ -4,9 +4,9 @@
 ; is high because the loop can be deleted after the exit value rewrite.
 ;
 ; CHECK-LABEL: @_Z3fooPKcjj(
-; CHECK: udiv
 ; CHECK: [[LABEL:^[a-zA-Z0-9_.]+]]:
 ; CHECK-NOT: br {{.*}} [[LABEL]]
+; CHECK: udiv
 
 define i32 @_Z3fooPKcjj(ptr nocapture readnone %s, i32 %len, i32 %c) #0 {
 entry:
diff --git a/llvm/test/Transforms/IndVarSimplify/finite-exit-comparisons.ll b/llvm/test/Transforms/IndVarSimplify/finite-exit-comparisons.ll
index 3c6b12dac2119..e006d9f6696ca 100644
--- a/llvm/test/Transforms/IndVarSimplify/finite-exit-comparisons.ll
+++ b/llvm/test/Transforms/IndVarSimplify/finite-exit-comparisons.ll
@@ -932,17 +932,17 @@ for.end:                                          ; preds = %for.body, %entry
 define i16 @ult_multiuse_profit(i16 %n.raw, i8 %start) mustprogress {
 ; CHECK-LABEL: @ult_multiuse_profit(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[START:%.*]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i16 254 to i8
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i16 254 to i8
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[START]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i8 [[IV]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[IV_NEXT]], [[TMP2]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[IV_NEXT]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[UMAX:%.*]] = call i16 @llvm.umax.i16(i16 [[TMP1]], i16 254)
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[START:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i16
+; CHECK-NEXT:    [[UMAX:%.*]] = call i16 @llvm.umax.i16(i16 [[TMP2]], i16 254)
 ; CHECK-NEXT:    ret i16 [[UMAX]]
 ;
 entry:
diff --git a/llvm/test/Transforms/IndVarSimplify/pr116483.ll b/llvm/test/Transforms/IndVarSimplify/pr116483.ll
index ae108a525223e..093e25a3caa81 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr116483.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr116483.ll
@@ -4,15 +4,15 @@
 define i32 @test() {
 ; CHECK-LABEL: define i32 @test() {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[LOOP_BODY:.*]]
+; CHECK:       [[LOOP_BODY]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[LOOP_BODY]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 0, 3
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[XOR]], 329
 ; CHECK-NEXT:    [[CONV:%.*]] = trunc i32 [[MUL]] to i16
 ; CHECK-NEXT:    [[SEXT:%.*]] = shl i16 [[CONV]], 8
 ; CHECK-NEXT:    [[CONV1:%.*]] = ashr i16 [[SEXT]], 8
-; CHECK-NEXT:    br label %[[LOOP_BODY:.*]]
-; CHECK:       [[LOOP_BODY]]:
-; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[LOOP_BODY]]
-; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[CONV3:%.*]] = zext i16 [[CONV1]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV3]]
 ;
diff --git a/llvm/test/Transforms/IndVarSimplify/pr63763.ll b/llvm/test/Transforms/IndVarSimplify/pr63763.ll
index 4e62e92ca07ee..427db1e67410a 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr63763.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr63763.ll
@@ -16,12 +16,12 @@ define i32 @test(i1 %c) {
 ; CHECK-NEXT:    [[CONV2:%.*]] = ashr exact i32 [[SEXT]], 24
 ; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = sub nsw i32 7, [[CONV2]]
 ; CHECK-NEXT:    call void @use(i32 [[INVARIANT_OP]])
-; CHECK-NEXT:    [[SEXT_US:%.*]] = shl i32 [[SEL]], 24
-; CHECK-NEXT:    [[CONV2_US:%.*]] = ashr exact i32 [[SEXT_US]], 24
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
+; CHECK-NEXT:    [[SEXT_US:%.*]] = shl i32 [[SEL]], 24
+; CHECK-NEXT:    [[CONV2_US:%.*]] = ashr exact i32 [[SEXT_US]], 24
 ; CHECK-NEXT:    [[INVARIANT_OP_US:%.*]] = sub nsw i32 7, [[CONV2_US]]
 ; CHECK-NEXT:    ret i32 [[INVARIANT_OP_US]]
 ;
diff --git a/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll b/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
index 4692a542053c9..b3162de0f2245 100644
--- a/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
+++ b/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
@@ -4,20 +4,21 @@
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
 define i32 @remove_loop(i32 %size) {
-; CHECK-LABEL: @remove_loop(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE:%.*]], 31
+; CHECK-LABEL: define i32 @remove_loop(
+; CHECK-SAME: i32 [[SIZE:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[WHILE_COND:.*]]
+; CHECK:       [[WHILE_COND]]:
+; CHECK-NEXT:    [[SIZE_ADDR_0:%.*]] = phi i32 [ [[SIZE]], %[[ENTRY]] ], [ [[SUB:%.*]], %[[WHILE_COND]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[SIZE_ADDR_0]], 31
+; CHECK-NEXT:    [[SUB]] = add i32 [[SIZE_ADDR_0]], -32
+; CHECK-NEXT:    br i1 [[CMP]], label %[[WHILE_COND]], label %[[WHILE_END:.*]]
+; CHECK:       [[WHILE_END]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE]], 31
 ; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SIZE]], i32 31)
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[UMIN]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 5
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 5
-; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
-; CHECK:       while.cond:
-; CHECK-NEXT:    [[SIZE_ADDR_0:%.*]] = phi i32 [ [[SIZE]], [[ENTRY:%.*]] ], [ [[SUB:%.*]], [[WHILE_COND]] ]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[SIZE_ADDR_0]], 31
-; CHECK-NEXT:    [[SUB]] = add i32 [[SIZE_ADDR_0]], -32
-; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_COND]], label [[WHILE_END:%.*]]
-; CHECK:       while.end:
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
 ; CHECK-NEXT:    ret i32 [[TMP4]]
 ;
diff --git a/llvm/test/Transforms/IndVarSimplify/sentinel.ll b/llvm/test/Transforms/IndVarSimplify/sentinel.ll
index d1140affb5a4b..523414167956b 100644
--- a/llvm/test/Transforms/IndVarSimplify/sentinel.ll
+++ b/llvm/test/Transforms/IndVarSimplify/sentinel.ll
@@ -10,18 +10,18 @@ define void @test(i1 %arg) personality ptr @snork {
 ; CHECK-NEXT:    br label [[BB4:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add i32 [[INDVARS_IV:%.*]], 1
-; CHECK-NEXT:    [[TMP0:%.*]] = sub i32 [[TMP1:%.*]], [[SMAX:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[TMP6:%.*]], [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMAX:%.*]]
 ; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[BB2:%.*]], label [[BB4]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ [[TMP0]], [[BB1:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ [[TMP1]], [[BB1:%.*]] ]
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[INDVARS_IV]] = phi i32 [ [[INDVARS_IV_NEXT]], [[BB1]] ], [ undef, [[BB:%.*]] ]
 ; CHECK-NEXT:    [[SMAX]] = call i32 @llvm.smax.i32(i32 [[INDVARS_IV]], i32 36)
-; CHECK-NEXT:    [[TMP6:%.*]] = invoke i32 @quux() [ "deopt"(i32 0, i32 0, i32 0, i32 180, i32 0, i32 25, i32 0, i32 7, ptr null, i32 7, ptr null, i32 7, ptr null, i32 3, i32 [[INDVARS_IV]], i32 3, i32 undef, i32 7, ptr null, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 4, double undef, i32 7, ptr null, i32 4, i64 undef, i32 7, ptr null, i32 0, ptr addrspace(1) undef, i32 3, i32 undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 7, ptr null) ]
+; CHECK-NEXT:    [[TMP6]] = invoke i32 @quux() [ "deopt"(i32 0, i32 0, i32 0, i32 180, i32 0, i32 25, i32 0, i32 7, ptr null, i32 7, ptr null, i32 7, ptr null, i32 3, i32 [[INDVARS_IV]], i32 3, i32 undef, i32 7, ptr null, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 4, double undef, i32 7, ptr null, i32 4, i64 undef, i32 7, ptr null, i32 0, ptr addrspace(1) undef, i32 3, i32 undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 0, ptr addrspace(1) undef, i32 7, ptr null) ]
 ; CHECK-NEXT:            to label [[BB7:%.*]] unwind label [[BB15:%.*]]
 ; CHECK:       bb7:
-; CHECK-NEXT:    [[TMP1]] = add i32 [[TMP6]], [[INDVARS_IV]]
 ; CHECK-NEXT:    br label [[BB9:%.*]]
 ; CHECK:       bb9:
 ; CHECK-NEXT:    br i1 true, label [[BB1]], label [[BB9]]
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll b/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
index 49d6bc46dab32..3353a59ea1d12 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
@@ -23,38 +23,38 @@ define void @_Z3fn1v(ptr %r, ptr %a) #0 {
 ; CHECK-LABEL: define void @_Z3fn1v(
 ; CHECK-SAME: ptr writeonly captures(none) [[R:%.*]], ptr readonly captures(none) [[A:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr @b, align 4
-; CHECK-NEXT:    [[TOBOOL20:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[T:%.*]] = load i32, ptr @b, align 4
+; CHECK-NEXT:    [[TOBOOL20:%.*]] = icmp eq i32 [[T]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL20]], label %[[FOR_END6:.*]], label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_COND_LOOPEXIT_LOOPEXIT:.*]]:
 ; CHECK-NEXT:    [[ADD_PTR_LCSSA:%.*]] = phi ptr [ [[ADD_PTR_LCSSA_UNR:%.*]], %[[FOR_BODY3_PROL_LOOPEXIT:.*]] ], [ [[ADD_PTR_1:%.*]], %[[FOR_INC_1:.*]] ]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A_021:%.*]], i64 1
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP1:%.*]]
-; CHECK-NEXT:    [[TMP1_PRE:%.*]] = load i32, ptr @b, align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[T2:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SCEVGEP:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[T1_PRE:%.*]] = load i32, ptr @b, align 4
 ; CHECK-NEXT:    br label %[[FOR_COND_LOOPEXIT:.*]]
 ; CHECK:       [[FOR_COND_LOOPEXIT]]:
-; CHECK-NEXT:    [[T1:%.*]] = phi i32 [ [[T12:%.*]], %[[FOR_BODY]] ], [ [[TMP1_PRE]], %[[FOR_COND_LOOPEXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    [[T1:%.*]] = phi i32 [ [[T12:%.*]], %[[FOR_BODY]] ], [ [[T1_PRE]], %[[FOR_COND_LOOPEXIT_LOOPEXIT]] ]
 ; CHECK-NEXT:    [[R_1_LCSSA:%.*]] = phi ptr [ [[R_022:%.*]], %[[FOR_BODY]] ], [ [[ADD_PTR_LCSSA]], %[[FOR_COND_LOOPEXIT_LOOPEXIT]] ]
-; CHECK-NEXT:    [[A_1_LCSSA:%.*]] = phi ptr [ [[A_021]], %[[FOR_BODY]] ], [ [[SCEVGEP1]], %[[FOR_COND_LOOPEXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    [[A_1_LCSSA:%.*]] = phi ptr [ [[A_021:%.*]], %[[FOR_BODY]] ], [ [[SCEVGEP1]], %[[FOR_COND_LOOPEXIT_LOOPEXIT]] ]
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[T1]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[FOR_END6]], label %[[FOR_BODY]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[T12]] = phi i32 [ [[T1]], %[[FOR_COND_LOOPEXIT]] ], [ [[TMP]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[T12]] = phi i32 [ [[T1]], %[[FOR_COND_LOOPEXIT]] ], [ [[T]], %[[ENTRY]] ]
 ; CHECK-NEXT:    [[R_022]] = phi ptr [ [[R_1_LCSSA]], %[[FOR_COND_LOOPEXIT]] ], [ [[R]], %[[ENTRY]] ]
 ; CHECK-NEXT:    [[A_021]] = phi ptr [ [[A_1_LCSSA]], %[[FOR_COND_LOOPEXIT]] ], [ [[A]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @c, align 4
-; CHECK-NEXT:    [[TOBOOL215:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    [[T2]] = load i32, ptr @c, align 4
+; CHECK-NEXT:    [[TOBOOL215:%.*]] = icmp eq i32 [[T2]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL215]], label %[[FOR_COND_LOOPEXIT]], label %[[FOR_BODY3_PREHEADER:.*]]
 ; CHECK:       [[FOR_BODY3_PREHEADER]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[TMP2]], -1
-; CHECK-NEXT:    [[TMP1]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[XTRAITER:%.*]] = and i32 [[TMP2]], 1
+; CHECK-NEXT:    [[XTRAITER:%.*]] = and i32 [[T2]], 1
 ; CHECK-NEXT:    [[LCMP_MOD_NOT:%.*]] = icmp eq i32 [[XTRAITER]], 0
 ; CHECK-NEXT:    br i1 [[LCMP_MOD_NOT]], label %[[FOR_BODY3_PROL_LOOPEXIT]], label %[[FOR_BODY3_PROL:.*]]
 ; CHECK:       [[FOR_BODY3_PROL]]:
-; CHECK-NEXT:    [[DEC18_PROL:%.*]] = add nsw i32 [[TMP2]], -1
-; CHECK-NEXT:    [[TMP3_PROL:%.*]] = load i8, ptr [[A_021]], align 1
-; CHECK-NEXT:    [[CMP_PROL:%.*]] = icmp eq i8 [[TMP3_PROL]], 0
+; CHECK-NEXT:    [[DEC18_PROL:%.*]] = add nsw i32 [[T2]], -1
+; CHECK-NEXT:    [[T3_PROL:%.*]] = load i8, ptr [[A_021]], align 1
+; CHECK-NEXT:    [[CMP_PROL:%.*]] = icmp eq i8 [[T3_PROL]], 0
 ; CHECK-NEXT:    br i1 [[CMP_PROL]], label %[[IF_THEN_PROL:.*]], label %[[FOR_INC_PROL:.*]]
 ; CHECK:       [[IF_THEN_PROL]]:
 ; CHECK-NEXT:    [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds nuw i8, ptr [[R_022]], i64 2
@@ -69,17 +69,17 @@ define void @_Z3fn1v(ptr %r, ptr %a) #0 {
 ; CHECK-NEXT:    br label %[[FOR_BODY3_PROL_LOOPEXIT]]
 ; CHECK:       [[FOR_BODY3_PROL_LOOPEXIT]]:
 ; CHECK-NEXT:    [[ADD_PTR_LCSSA_UNR]] = phi ptr [ poison, %[[FOR_BODY3_PREHEADER]] ], [ [[ADD_PTR_PROL]], %[[FOR_INC_PROL]] ]
-; CHECK-NEXT:    [[DEC18_IN_UNR:%.*]] = phi i32 [ [[TMP2]], %[[FOR_BODY3_PREHEADER]] ], [ [[DEC18_PROL]], %[[FOR_INC_PROL]] ]
+; CHECK-NEXT:    [[DEC18_IN_UNR:%.*]] = phi i32 [ [[T2]], %[[FOR_BODY3_PREHEADER]] ], [ [[DEC18_PROL]], %[[FOR_INC_PROL]] ]
 ; CHECK-NEXT:    [[R_117_UNR:%.*]] = phi ptr [ [[R_022]], %[[FOR_BODY3_PREHEADER]] ], [ [[ADD_PTR_PROL]], %[[FOR_INC_PROL]] ]
 ; CHECK-NEXT:    [[A_116_UNR:%.*]] = phi ptr [ [[A_021]], %[[FOR_BODY3_PREHEADER]] ], [ [[INCDEC_PTR_PROL]], %[[FOR_INC_PROL]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP0]], 0
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[FOR_COND_LOOPEXIT_LOOPEXIT]], label %[[FOR_BODY3:.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[T2]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[FOR_COND_LOOPEXIT_LOOPEXIT]], label %[[FOR_BODY3:.*]]
 ; CHECK:       [[FOR_BODY3]]:
 ; CHECK-NEXT:    [[DEC18_IN:%.*]] = phi i32 [ [[DEC18_1:%.*]], %[[FOR_INC_1]] ], [ [[DEC18_IN_UNR]], %[[FOR_BODY3_PROL_LOOPEXIT]] ]
 ; CHECK-NEXT:    [[R_117:%.*]] = phi ptr [ [[ADD_PTR_1]], %[[FOR_INC_1]] ], [ [[R_117_UNR]], %[[FOR_BODY3_PROL_LOOPEXIT]] ]
 ; CHECK-NEXT:    [[A_116:%.*]] = phi ptr [ [[INCDEC_PTR_1:%.*]], %[[FOR_INC_1]] ], [ [[A_116_UNR]], %[[FOR_BODY3_PROL_LOOPEXIT]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[A_116]], align 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP3]], 0
+; CHECK-NEXT:    [[T3:%.*]] = load i8, ptr [[A_116]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[T3]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[FOR_INC:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[R_117]], i64 2
@@ -91,8 +91,8 @@ define void @_Z3fn1v(ptr %r, ptr %a) #0 {
 ; CHECK:       [[FOR_INC]]:
 ; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[A_116]], i64 1
 ; CHECK-NEXT:    [[DEC18_1]] = add nsw i32 [[DEC18_IN]], -2
-; CHECK-NEXT:    [[TMP3_1:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
-; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i8 [[TMP3_1]], 0
+; CHECK-NEXT:    [[T3_1:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i8 [[T3_1]], 0
 ; CHECK-NEXT:    br i1 [[CMP_1]], label %[[IF_THEN_1:.*]], label %[[FOR_INC_1]]
 ; CHECK:       [[IF_THEN_1]]:
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[R_117]], i64 6
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
index 778f25f5620f2..a13c36f693bac 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
@@ -13,25 +13,25 @@ define void @arm_mean_q7(ptr noundef %pSrc, i32 noundef %blockSize, ptr noundef
 ; CHECK-NEXT:    br i1 [[CMP_NOT10]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
 ; CHECK:       while.body.preheader:
 ; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[BLOCKSIZE]], 4
-; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[BLOCKSIZE]], -16
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[SUM_013:%.*]] = phi i32 [ [[TMP3:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[SUM_013:%.*]] = phi i32 [ [[TMP2:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[PSRC_ADDR_012:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRC:%.*]], [[WHILE_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[BLKCNT_011:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[SHR]], [[WHILE_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[PSRC_ADDR_012]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> [[TMP1]], i32 0)
-; CHECK-NEXT:    [[TMP3]] = add i32 [[TMP2]], [[SUM_013]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[PSRC_ADDR_012]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> [[TMP0]], i32 0)
+; CHECK-NEXT:    [[TMP2]] = add i32 [[TMP1]], [[SUM_013]]
 ; CHECK-NEXT:    [[DEC]] = add nsw i32 [[BLKCNT_011]], -1
 ; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_012]], i32 16
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0
 ; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
 ; CHECK:       while.end.loopexit:
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[BLOCKSIZE]], -16
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[TMP3]]
 ; CHECK-NEXT:    br label [[WHILE_END]]
 ; CHECK:       while.end:
 ; CHECK-NEXT:    [[PSRC_ADDR_0_LCSSA:%.*]] = phi ptr [ [[PSRC]], [[ENTRY:%.*]] ], [ [[SCEVGEP]], [[WHILE_END_LOOPEXIT]] ]
-; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP3]], [[WHILE_END_LOOPEXIT]] ]
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP2]], [[WHILE_END_LOOPEXIT]] ]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[BLOCKSIZE]], 15
 ; CHECK-NEXT:    [[CMP2_NOT15:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    br i1 [[CMP2_NOT15]], label [[WHILE_END5:%.*]], label [[MIDDLE_BLOCK:%.*]]

From ce0c472791647c61f3af97c16a8c61fb351417ea Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 17 Apr 2025 17:26:55 -0700
Subject: [PATCH 345/710] Revert "Reland [flang] Generalized simplification of
 HLFIR reduction ops. (#136071)"

This reverts commit 32311a6b68d3de4642599abe14922c686bdb30fc.
---
 .../flang/Optimizer/Builder/HLFIRTools.h      |    5 -
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |   27 -
 .../Transforms/OptimizedBufferization.cpp     |  465 ++++++++
 .../Transforms/SimplifyHLFIRIntrinsics.cpp    | 1038 ++++-------------
 flang/test/HLFIR/all-elemental.fir            |   91 ++
 flang/test/HLFIR/any-elemental.fir            |  190 +++
 flang/test/HLFIR/count-elemental.fir          |  314 +++++
 flang/test/HLFIR/maxloc-elemental.fir         |  133 +++
 flang/test/HLFIR/maxval-elemental.fir         |  117 ++
 flang/test/HLFIR/minloc-elemental.fir         |  397 +++++++
 flang/test/HLFIR/minval-elemental.fir         |   95 ++
 .../HLFIR/simplify-hlfir-intrinsics-all.fir   |  123 --
 .../HLFIR/simplify-hlfir-intrinsics-any.fir   |  123 --
 .../HLFIR/simplify-hlfir-intrinsics-count.fir |  127 --
 .../simplify-hlfir-intrinsics-maxloc.fir      |  343 ------
 .../simplify-hlfir-intrinsics-maxval.fir      |  177 ---
 .../simplify-hlfir-intrinsics-minloc.fir      |  343 ------
 .../simplify-hlfir-intrinsics-minval.fir      |  177 ---
 18 files changed, 1999 insertions(+), 2286 deletions(-)
 create mode 100644 flang/test/HLFIR/all-elemental.fir
 create mode 100644 flang/test/HLFIR/any-elemental.fir
 create mode 100644 flang/test/HLFIR/count-elemental.fir
 create mode 100644 flang/test/HLFIR/maxloc-elemental.fir
 create mode 100644 flang/test/HLFIR/maxval-elemental.fir
 create mode 100644 flang/test/HLFIR/minloc-elemental.fir
 create mode 100644 flang/test/HLFIR/minval-elemental.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
 delete mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir

diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index cd259b9dc6071..ac80873dc374f 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -301,11 +301,6 @@ mlir::Value genExtent(mlir::Location loc, fir::FirOpBuilder &builder,
 mlir::Value genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
                       hlfir::Entity entity, unsigned dim);
 
-/// Compute the lower bounds of \p entity, which is an array of known rank.
-llvm::SmallVector<mlir::Value> genLBounds(mlir::Location loc,
-                                          fir::FirOpBuilder &builder,
-                                          hlfir::Entity entity);
-
 /// Generate a vector of extents with index type from a fir.shape
 /// of fir.shape_shift value.
 llvm::SmallVector<mlir::Value> getIndexExtents(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index dccad8095979d..558ebcb876ddb 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -659,33 +659,6 @@ mlir::Value hlfir::genLBound(mlir::Location loc, fir::FirOpBuilder &builder,
   return dimInfo.getLowerBound();
 }
 
-llvm::SmallVector<mlir::Value> hlfir::genLBounds(mlir::Location loc,
-                                                 fir::FirOpBuilder &builder,
-                                                 hlfir::Entity entity) {
-  assert(!entity.isAssumedRank() &&
-         "cannot compute all lower bounds for assumed rank");
-  assert(!entity.isScalar() && "expected an array entity");
-  int rank = entity.getRank();
-  mlir::Type idxTy = builder.getIndexType();
-  if (!entity.mayHaveNonDefaultLowerBounds())
-    return {static_cast<std::size_t>(rank),
-            builder.createIntegerConstant(loc, idxTy, 1)};
-
-  if (auto shape = tryRetrievingShapeOrShift(entity)) {
-    auto lbounds = getExplicitLboundsFromShape(shape);
-    if (!lbounds.empty())
-      return lbounds;
-  }
-
-  if (entity.isMutableBox())
-    entity = hlfir::derefPointersAndAllocatables(loc, builder, entity);
-
-  llvm::SmallVector<mlir::Value> lbounds;
-  fir::factory::genDimInfoFromBox(builder, loc, entity, &lbounds,
-                                  /*extents=*/nullptr, /*strides=*/nullptr);
-  return lbounds;
-}
-
 void hlfir::genLengthParameters(mlir::Location loc, fir::FirOpBuilder &builder,
                                 Entity entity,
                                 llvm::SmallVectorImpl<mlir::Value> &result) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 79aabd2981e1a..c489450384a35 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -772,6 +772,458 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   return mlir::success();
 }
 
+using GenBodyFn =
+    std::function<mlir::Value(fir::FirOpBuilder &, mlir::Location, mlir::Value,
+                              const llvm::SmallVectorImpl<mlir::Value> &)>;
+static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
+                                         mlir::Location loc, mlir::Value init,
+                                         mlir::Value shape, GenBodyFn genBody) {
+  auto extents = hlfir::getIndexExtents(loc, builder, shape);
+  mlir::Value reduction = init;
+  mlir::IndexType idxTy = builder.getIndexType();
+  mlir::Value oneIdx = builder.createIntegerConstant(loc, idxTy, 1);
+
+  // Create a reduction loop nest. We use one-based indices so that they can be
+  // passed to the elemental, and reverse the order so that they can be
+  // generated in column-major order for better performance.
+  llvm::SmallVector<mlir::Value> indices(extents.size(), mlir::Value{});
+  for (unsigned i = 0; i < extents.size(); ++i) {
+    auto loop = builder.create<fir::DoLoopOp>(
+        loc, oneIdx, extents[extents.size() - i - 1], oneIdx, false,
+        /*finalCountValue=*/false, reduction);
+    reduction = loop.getRegionIterArgs()[0];
+    indices[extents.size() - i - 1] = loop.getInductionVar();
+    // Set insertion point to the loop body so that the next loop
+    // is inserted inside the current one.
+    builder.setInsertionPointToStart(loop.getBody());
+  }
+
+  // Generate the body
+  reduction = genBody(builder, loc, reduction, indices);
+
+  // Unwind the loop nest.
+  for (unsigned i = 0; i < extents.size(); ++i) {
+    auto result = builder.create<fir::ResultOp>(loc, reduction);
+    auto loop = mlir::cast<fir::DoLoopOp>(result->getParentOp());
+    reduction = loop.getResult(0);
+    // Set insertion point after the loop operation that we have
+    // just processed.
+    builder.setInsertionPointAfter(loop.getOperation());
+  }
+
+  return reduction;
+}
+
+auto makeMinMaxInitValGenerator(bool isMax) {
+  return [isMax](fir::FirOpBuilder builder, mlir::Location loc,
+                 mlir::Type elementType) -> mlir::Value {
+    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
+      const llvm::fltSemantics &sem = ty.getFloatSemantics();
+      llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
+      return builder.createRealConstant(loc, elementType, limit);
+    }
+    unsigned bits = elementType.getIntOrFloatBitWidth();
+    int64_t limitInt =
+        isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
+              : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
+    return builder.createIntegerConstant(loc, elementType, limitInt);
+  };
+}
+
+mlir::Value generateMinMaxComparison(fir::FirOpBuilder builder,
+                                     mlir::Location loc, mlir::Value elem,
+                                     mlir::Value reduction, bool isMax) {
+  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
+    // For FP reductions we want the first smallest value to be used, that
+    // is not NaN. A OGL/OLT condition will usually work for this unless all
+    // the values are Nan or Inf. This follows the same logic as
+    // NumericCompare for Minloc/Maxlox in extrema.cpp.
+    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
+        loc,
+        isMax ? mlir::arith::CmpFPredicate::OGT
+              : mlir::arith::CmpFPredicate::OLT,
+        elem, reduction);
+    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
+    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
+    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
+    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
+    return builder.create<mlir::arith::CmpIOp>(
+        loc,
+        isMax ? mlir::arith::CmpIPredicate::sgt
+              : mlir::arith::CmpIPredicate::slt,
+        elem, reduction);
+  }
+  llvm_unreachable("unsupported type");
+}
+
+/// Given a reduction operation with an elemental/designate source, attempt to
+/// generate a do-loop to perform the operation inline.
+///   %e = hlfir.elemental %shape unordered
+///   %r = hlfir.count %e
+/// =>
+///   %r = for.do_loop %arg = 1 to bound(%shape) step 1 iter_args(%arg2 = init)
+///     %i = <inline elemental>
+///     %c = <reduce count> %i
+///     fir.result %c
+template <typename Op>
+class ReductionConversion : public mlir::OpRewritePattern<Op> {
+public:
+  using mlir::OpRewritePattern<Op>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
+    mlir::Location loc = op.getLoc();
+    // Select source and validate its arguments.
+    mlir::Value source;
+    bool valid = false;
+    if constexpr (std::is_same_v<Op, hlfir::AnyOp> ||
+                  std::is_same_v<Op, hlfir::AllOp> ||
+                  std::is_same_v<Op, hlfir::CountOp>) {
+      source = op.getMask();
+      valid = !op.getDim();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      source = op.getArray();
+      valid = !op.getDim() && !op.getMask();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                         std::is_same_v<Op, hlfir::MinlocOp>) {
+      source = op.getArray();
+      valid = !op.getDim() && !op.getMask() && !op.getBack();
+    }
+    if (!valid)
+      return rewriter.notifyMatchFailure(
+          op, "Currently does not accept optional arguments");
+
+    hlfir::ElementalOp elemental;
+    hlfir::DesignateOp designate;
+    mlir::Value shape;
+    if ((elemental = source.template getDefiningOp<hlfir::ElementalOp>())) {
+      shape = elemental.getOperand(0);
+    } else if ((designate =
+                    source.template getDefiningOp<hlfir::DesignateOp>())) {
+      shape = designate.getShape();
+    } else {
+      return rewriter.notifyMatchFailure(op, "Did not find valid argument");
+    }
+
+    auto inlineSource =
+        [elemental,
+         &designate](fir::FirOpBuilder builder, mlir::Location loc,
+                     const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+        -> mlir::Value {
+      if (elemental) {
+        // Inline the elemental and get the value from it.
+        auto yield =
+            inlineElementalOp(loc, builder, elemental, oneBasedIndices);
+        auto tmp = yield.getElementValue();
+        yield->erase();
+        return tmp;
+      }
+      if (designate) {
+        // Create a designator over the array designator, then load the
+        // reference.
+        mlir::Value elementAddr = hlfir::getElementAt(
+            loc, builder, hlfir::Entity{designate.getResult()},
+            oneBasedIndices);
+        return builder.create<fir::LoadOp>(loc, elementAddr);
+      }
+      llvm_unreachable("unsupported type");
+    };
+
+    fir::FirOpBuilder builder{rewriter, op.getOperation()};
+
+    mlir::Value init;
+    GenBodyFn genBodyFn;
+    if constexpr (std::is_same_v<Op, hlfir::AnyOp>) {
+      init = builder.createIntegerConstant(loc, builder.getI1Type(), 0);
+      genBodyFn = [inlineSource](
+                      fir::FirOpBuilder builder, mlir::Location loc,
+                      mlir::Value reduction,
+                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+          -> mlir::Value {
+        // Conditionally set the reduction variable.
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(),
+            inlineSource(builder, loc, oneBasedIndices));
+        return builder.create<mlir::arith::OrIOp>(loc, reduction, cond);
+      };
+    } else if constexpr (std::is_same_v<Op, hlfir::AllOp>) {
+      init = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
+      genBodyFn = [inlineSource](
+                      fir::FirOpBuilder builder, mlir::Location loc,
+                      mlir::Value reduction,
+                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+          -> mlir::Value {
+        // Conditionally set the reduction variable.
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(),
+            inlineSource(builder, loc, oneBasedIndices));
+        return builder.create<mlir::arith::AndIOp>(loc, reduction, cond);
+      };
+    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
+      init = builder.createIntegerConstant(loc, op.getType(), 0);
+      genBodyFn = [inlineSource](
+                      fir::FirOpBuilder builder, mlir::Location loc,
+                      mlir::Value reduction,
+                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+          -> mlir::Value {
+        // Conditionally add one to the current value
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(),
+            inlineSource(builder, loc, oneBasedIndices));
+        mlir::Value one =
+            builder.createIntegerConstant(loc, reduction.getType(), 1);
+        mlir::Value add1 =
+            builder.create<mlir::arith::AddIOp>(loc, reduction, one);
+        return builder.create<mlir::arith::SelectOp>(loc, cond, add1,
+                                                     reduction);
+      };
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                         std::is_same_v<Op, hlfir::MinlocOp>) {
+      // TODO: implement minloc/maxloc conversion.
+      return rewriter.notifyMatchFailure(
+          op, "Currently minloc/maxloc is not handled");
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      mlir::Type ty = op.getType();
+      if (!(mlir::isa<mlir::FloatType>(ty) ||
+            mlir::isa<mlir::IntegerType>(ty))) {
+        return rewriter.notifyMatchFailure(
+            op, "Type is not supported for Maxval or Minval yet");
+      }
+
+      bool isMax = std::is_same_v<Op, hlfir::MaxvalOp>;
+      init = makeMinMaxInitValGenerator(isMax)(builder, loc, ty);
+      genBodyFn = [inlineSource, isMax](
+                      fir::FirOpBuilder builder, mlir::Location loc,
+                      mlir::Value reduction,
+                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
+          -> mlir::Value {
+        mlir::Value val = inlineSource(builder, loc, oneBasedIndices);
+        mlir::Value cmp =
+            generateMinMaxComparison(builder, loc, val, reduction, isMax);
+        return builder.create<mlir::arith::SelectOp>(loc, cmp, val, reduction);
+      };
+    } else {
+      llvm_unreachable("unsupported type");
+    }
+
+    mlir::Value res =
+        generateReductionLoop(builder, loc, init, shape, genBodyFn);
+    if (res.getType() != op.getType())
+      res = builder.create<fir::ConvertOp>(loc, op.getType(), res);
+
+    // Check if the op was the only user of the source (apart from a destroy),
+    // and remove it if so.
+    mlir::Operation *sourceOp = source.getDefiningOp();
+    mlir::Operation::user_range srcUsers = sourceOp->getUsers();
+    hlfir::DestroyOp srcDestroy;
+    if (std::distance(srcUsers.begin(), srcUsers.end()) == 2) {
+      srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*srcUsers.begin());
+      if (!srcDestroy)
+        srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++srcUsers.begin());
+    }
+
+    rewriter.replaceOp(op, res);
+    if (srcDestroy) {
+      rewriter.eraseOp(srcDestroy);
+      rewriter.eraseOp(sourceOp);
+    }
+    return mlir::success();
+  }
+};
+
+// Look for minloc(mask=elemental) and generate the minloc loop with
+// inlined elemental.
+//  %e = hlfir.elemental %shape ({ ... })
+//  %m = hlfir.minloc %array mask %e
+template <typename Op>
+class ReductionMaskConversion : public mlir::OpRewritePattern<Op> {
+public:
+  using mlir::OpRewritePattern<Op>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(Op mloc, mlir::PatternRewriter &rewriter) const override {
+    if (!mloc.getMask() || mloc.getDim() || mloc.getBack())
+      return rewriter.notifyMatchFailure(mloc,
+                                         "Did not find valid minloc/maxloc");
+
+    bool isMax = std::is_same_v<Op, hlfir::MaxlocOp>;
+
+    auto elemental =
+        mloc.getMask().template getDefiningOp<hlfir::ElementalOp>();
+    if (!elemental || hlfir::elementalOpMustProduceTemp(elemental))
+      return rewriter.notifyMatchFailure(mloc, "Did not find elemental");
+
+    mlir::Value array = mloc.getArray();
+
+    unsigned rank = mlir::cast<hlfir::ExprType>(mloc.getType()).getShape()[0];
+    mlir::Type arrayType = array.getType();
+    if (!mlir::isa<fir::BoxType>(arrayType))
+      return rewriter.notifyMatchFailure(
+          mloc, "Currently requires a boxed type input");
+    mlir::Type elementType = hlfir::getFortranElementType(arrayType);
+    if (!fir::isa_trivial(elementType))
+      return rewriter.notifyMatchFailure(
+          mloc, "Character arrays are currently not handled");
+
+    mlir::Location loc = mloc.getLoc();
+    fir::FirOpBuilder builder{rewriter, mloc.getOperation()};
+    mlir::Value resultArr = builder.createTemporary(
+        loc, fir::SequenceType::get(
+                 rank, hlfir::getFortranElementType(mloc.getType())));
+
+    auto init = makeMinMaxInitValGenerator(isMax);
+
+    auto genBodyOp =
+        [&rank, &resultArr, &elemental, isMax](
+            fir::FirOpBuilder builder, mlir::Location loc,
+            mlir::Type elementType, mlir::Value array, mlir::Value flagRef,
+            mlir::Value reduction,
+            const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {
+      // We are in the innermost loop: generate the elemental inline
+      mlir::Value oneIdx =
+          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
+      llvm::SmallVector<mlir::Value> oneBasedIndices;
+      llvm::transform(
+          indices, std::back_inserter(oneBasedIndices), [&](mlir::Value V) {
+            return builder.create<mlir::arith::AddIOp>(loc, V, oneIdx);
+          });
+      hlfir::YieldElementOp yield =
+          hlfir::inlineElementalOp(loc, builder, elemental, oneBasedIndices);
+      mlir::Value maskElem = yield.getElementValue();
+      yield->erase();
+
+      mlir::Type ifCompatType = builder.getI1Type();
+      mlir::Value ifCompatElem =
+          builder.create<fir::ConvertOp>(loc, ifCompatType, maskElem);
+
+      llvm::SmallVector<mlir::Type> resultsTy = {elementType, elementType};
+      fir::IfOp maskIfOp =
+          builder.create<fir::IfOp>(loc, elementType, ifCompatElem,
+                                    /*withElseRegion=*/true);
+      builder.setInsertionPointToStart(&maskIfOp.getThenRegion().front());
+
+      // Set flag that mask was true at some point
+      mlir::Value flagSet = builder.createIntegerConstant(
+          loc, mlir::cast<fir::ReferenceType>(flagRef.getType()).getEleTy(), 1);
+      mlir::Value isFirst = builder.create<fir::LoadOp>(loc, flagRef);
+      mlir::Value addr = hlfir::getElementAt(loc, builder, hlfir::Entity{array},
+                                             oneBasedIndices);
+      mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
+
+      // Compare with the max reduction value
+      mlir::Value cmp =
+          generateMinMaxComparison(builder, loc, elem, reduction, isMax);
+
+      // The condition used for the loop is isFirst || <the condition above>.
+      isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
+      isFirst = builder.create<mlir::arith::XOrIOp>(
+          loc, isFirst, builder.createIntegerConstant(loc, cmp.getType(), 1));
+      cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, isFirst);
+
+      // Set the new coordinate to the result
+      fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, cmp,
+                                                 /*withElseRegion*/ true);
+
+      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+      builder.create<fir::StoreOp>(loc, flagSet, flagRef);
+      mlir::Type resultElemTy =
+          hlfir::getFortranElementType(resultArr.getType());
+      mlir::Type returnRefTy = builder.getRefType(resultElemTy);
+      mlir::IndexType idxTy = builder.getIndexType();
+
+      for (unsigned int i = 0; i < rank; ++i) {
+        mlir::Value index = builder.createIntegerConstant(loc, idxTy, i + 1);
+        mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
+            loc, returnRefTy, resultArr, index);
+        mlir::Value fortranIndex = builder.create<fir::ConvertOp>(
+            loc, resultElemTy, oneBasedIndices[i]);
+        builder.create<fir::StoreOp>(loc, fortranIndex, resultElemAddr);
+      }
+      builder.create<fir::ResultOp>(loc, elem);
+      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+      builder.create<fir::ResultOp>(loc, reduction);
+      builder.setInsertionPointAfter(ifOp);
+
+      // Close the mask if
+      builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
+      builder.setInsertionPointToStart(&maskIfOp.getElseRegion().front());
+      builder.create<fir::ResultOp>(loc, reduction);
+      builder.setInsertionPointAfter(maskIfOp);
+
+      return maskIfOp.getResult(0);
+    };
+    auto getAddrFn = [](fir::FirOpBuilder builder, mlir::Location loc,
+                        const mlir::Type &resultElemType, mlir::Value resultArr,
+                        mlir::Value index) {
+      mlir::Type resultRefTy = builder.getRefType(resultElemType);
+      mlir::Value oneIdx =
+          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
+      index = builder.create<mlir::arith::AddIOp>(loc, index, oneIdx);
+      return builder.create<hlfir::DesignateOp>(loc, resultRefTy, resultArr,
+                                                index);
+    };
+
+    // Initialize the result
+    mlir::Type resultElemTy = hlfir::getFortranElementType(resultArr.getType());
+    mlir::Type resultRefTy = builder.getRefType(resultElemTy);
+    mlir::Value returnValue =
+        builder.createIntegerConstant(loc, resultElemTy, 0);
+    for (unsigned int i = 0; i < rank; ++i) {
+      mlir::Value index =
+          builder.createIntegerConstant(loc, builder.getIndexType(), i + 1);
+      mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
+          loc, resultRefTy, resultArr, index);
+      builder.create<fir::StoreOp>(loc, returnValue, resultElemAddr);
+    }
+
+    fir::genMinMaxlocReductionLoop(builder, array, init, genBodyOp, getAddrFn,
+                                   rank, elementType, loc, builder.getI1Type(),
+                                   resultArr, false);
+
+    mlir::Value asExpr = builder.create<hlfir::AsExprOp>(
+        loc, resultArr, builder.createBool(loc, false));
+
+    // Check all the users - the destroy is no longer required, and any assign
+    // can use resultArr directly so that InlineHLFIRAssign pass
+    // can optimize the results. Other operations are replaced with an AsExpr
+    // for the temporary resultArr.
+    llvm::SmallVector<hlfir::DestroyOp> destroys;
+    llvm::SmallVector<hlfir::AssignOp> assigns;
+    for (auto user : mloc->getUsers()) {
+      if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(user))
+        destroys.push_back(destroy);
+      else if (auto assign = mlir::dyn_cast<hlfir::AssignOp>(user))
+        assigns.push_back(assign);
+    }
+
+    // Check if the minloc/maxloc was the only user of the elemental (apart from
+    // a destroy), and remove it if so.
+    mlir::Operation::user_range elemUsers = elemental->getUsers();
+    hlfir::DestroyOp elemDestroy;
+    if (std::distance(elemUsers.begin(), elemUsers.end()) == 2) {
+      elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*elemUsers.begin());
+      if (!elemDestroy)
+        elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++elemUsers.begin());
+    }
+
+    for (auto d : destroys)
+      rewriter.eraseOp(d);
+    for (auto a : assigns)
+      a.setOperand(0, resultArr);
+    rewriter.replaceOp(mloc, asExpr);
+    if (elemDestroy) {
+      rewriter.eraseOp(elemDestroy);
+      rewriter.eraseOp(elemental);
+    }
+    return mlir::success();
+  }
+};
+
 class EvaluateIntoMemoryAssignBufferization
     : public mlir::OpRewritePattern<hlfir::EvaluateInMemoryOp> {
 
@@ -888,6 +1340,19 @@ class OptimizedBufferizationPass
     patterns.insert<ElementalAssignBufferization>(context);
     patterns.insert<BroadcastAssignBufferization>(context);
     patterns.insert<EvaluateIntoMemoryAssignBufferization>(context);
+    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
+    // TODO: implement basic minloc/maxloc conversion.
+    // patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
+    // patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
+    patterns.insert<ReductionMaskConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionMaskConversion<hlfir::MaxlocOp>>(context);
+    // TODO: implement masked minval/maxval conversion.
+    // patterns.insert<ReductionMaskConversion<hlfir::MaxvalOp>>(context);
+    // patterns.insert<ReductionMaskConversion<hlfir::MinvalOp>>(context);
 
     if (mlir::failed(mlir::applyPatternsGreedily(
             getOperation(), std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index 6876c1487ac13..bac10121a881b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -173,881 +173,245 @@ class TransposeAsElementalConversion
   }
 };
 
-/// Base class for converting reduction-like operations into
-/// a reduction loop[-nest] optionally wrapped into hlfir.elemental.
-/// It is used to handle operations produced for ALL, ANY, COUNT,
-/// MAXLOC, MAXVAL, MINLOC, MINVAL, SUM intrinsics.
-///
-/// All of these operations take an input array, and optional
-/// dim, mask arguments. ALL, ANY, COUNT do not have mask argument.
-class ReductionAsElementalConverter {
+// Expand the SUM(DIM=CONSTANT) operation into .
+class SumAsElementalConversion : public mlir::OpRewritePattern<hlfir::SumOp> {
 public:
-  ReductionAsElementalConverter(mlir::Operation *op,
-                                mlir::PatternRewriter &rewriter)
-      : op{op}, rewriter{rewriter}, loc{op->getLoc()}, builder{rewriter, op} {
-    assert(op->getNumResults() == 1);
-  }
-  virtual ~ReductionAsElementalConverter() {}
-
-  /// Do the actual conversion or return mlir::failure(),
-  /// if conversion is not possible.
-  mlir::LogicalResult convert();
+  using mlir::OpRewritePattern<hlfir::SumOp>::OpRewritePattern;
 
-private:
-  // Return fir.shape specifying the shape of the result
-  // of a reduction with DIM=dimVal. The second return value
-  // is the extent of the DIM dimension.
-  std::tuple<mlir::Value, mlir::Value>
-  genResultShapeForPartialReduction(hlfir::Entity array, int64_t dimVal);
-
-  /// \p mask is a scalar or array logical mask.
-  /// If \p isPresentPred is not nullptr, it is a dynamic predicate value
-  /// identifying whether the mask's variable is present.
-  /// \p indices is a range of one-based indices to access \p mask
-  /// when it is an array.
-  ///
-  /// The method returns the scalar mask value to guard the access
-  /// to a single element of the input array.
-  mlir::Value genMaskValue(mlir::Value mask, mlir::Value isPresentPred,
-                           mlir::ValueRange indices);
-
-protected:
-  /// Return the input array.
-  virtual mlir::Value getSource() const = 0;
-
-  /// Return DIM or nullptr, if it is not present.
-  virtual mlir::Value getDim() const = 0;
-
-  /// Return MASK or nullptr, if it is not present.
-  virtual mlir::Value getMask() const { return nullptr; }
-
-  /// Return FastMathFlags attached to the operation
-  /// or arith::FastMathFlags::none, if the operation
-  /// does not support FastMathFlags (e.g. ALL, ANY, COUNT).
-  virtual mlir::arith::FastMathFlags getFastMath() const {
-    return mlir::arith::FastMathFlags::none;
-  }
-
-  /// Generates initial values for the reduction values used
-  /// by the reduction loop. In general, there is a single
-  /// loop-carried reduction value (e.g. for SUM), but, for example,
-  /// MAXLOC/MINLOC implementation uses multiple reductions.
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() = 0;
-
-  /// Perform reduction(s) update given a single input array's element
-  /// identified by \p array and \p oneBasedIndices coordinates.
-  /// \p currentValue specifies the current value(s) of the reduction(s)
-  /// inside the reduction loop body.
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) = 0;
-
-  /// Given reduction value(s) in \p reductionResults produced
-  /// by the reduction loop, apply any required updates and return
-  /// new reduction value(s) to be used after the reduction loop
-  /// (e.g. as the result yield of the wrapping hlfir.elemental).
-  /// NOTE: if the reduction loop is wrapped in hlfir.elemental,
-  /// the insertion point of any generated code is inside hlfir.elemental.
-  virtual hlfir::Entity
-  genFinalResult(const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
-    assert(reductionResults.size() == 1 &&
-           "default implementation of genFinalResult expect a single reduction "
-           "value");
-    return hlfir::Entity{reductionResults[0]};
-  }
-
-  /// Return mlir::success(), if the operation can be converted.
-  /// The default implementation always returns mlir::success().
-  /// The derived type may override the default implementation
-  /// with its own definition.
-  virtual mlir::LogicalResult isConvertible() const { return mlir::success(); }
-
-  // Default implementation of isTotalReduction() just checks
-  // if the result of the operation is a scalar.
-  // True result indicates that the reduction has to be done
-  // across all elements, false result indicates that
-  // the result is an array expression produced by an hlfir.elemental
-  // operation with a single reduction loop across the DIM dimension.
-  //
-  // MAXLOC/MINLOC must override this.
-  virtual bool isTotalReduction() const { return getResultRank() == 0; }
-
-  // Return true, if the reduction loop[-nest] may be unordered.
-  // In general, FP reductions may only be unordered when
-  // FastMathFlags::reassoc transformations are allowed.
-  //
-  // Some dervied types may need to override this.
-  virtual bool isUnordered() const {
-    mlir::Type elemType = getSourceElementType();
-    if (mlir::isa<mlir::IntegerType, fir::LogicalType, fir::CharacterType>(
-            elemType))
-      return true;
-    return static_cast<bool>(getFastMath() &
-                             mlir::arith::FastMathFlags::reassoc);
-  }
-
-  /// Return 0, if DIM is not present or its values does not matter
-  /// (for example, a reduction of 1D array does not care about
-  /// the DIM value, assuming that it is a valid program).
-  /// Return mlir::failure(), if DIM is a constant known
-  /// to be invalid for the given array.
-  /// Otherwise, return DIM constant value.
-  mlir::FailureOr<int64_t> getConstDim() const {
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::SumOp sum,
+                  mlir::PatternRewriter &rewriter) const override {
+    hlfir::Entity array = hlfir::Entity{sum.getArray()};
+    bool isTotalReduction = hlfir::Entity{sum}.getRank() == 0;
+    mlir::Value dim = sum.getDim();
     int64_t dimVal = 0;
-    if (!isTotalReduction()) {
+    if (!isTotalReduction) {
       // In case of partial reduction we should ignore the operations
       // with invalid DIM values. They may appear in dead code
       // after constant propagation.
-      auto constDim = fir::getIntIfConstant(getDim());
+      auto constDim = fir::getIntIfConstant(dim);
       if (!constDim)
-        return rewriter.notifyMatchFailure(op, "Nonconstant DIM");
+        return rewriter.notifyMatchFailure(sum, "Nonconstant DIM for SUM");
       dimVal = *constDim;
 
-      if ((dimVal <= 0 || dimVal > getSourceRank()))
-        return rewriter.notifyMatchFailure(op,
-                                           "Invalid DIM for partial reduction");
+      if ((dimVal <= 0 || dimVal > array.getRank()))
+        return rewriter.notifyMatchFailure(
+            sum, "Invalid DIM for partial SUM reduction");
     }
-    return dimVal;
-  }
-
-  /// Return hlfir::Entity of the result.
-  hlfir::Entity getResultEntity() const {
-    return hlfir::Entity{op->getResult(0)};
-  }
-
-  /// Return type of the result (e.g. !hlfir.expr<?xi32>).
-  mlir::Type getResultType() const { return getResultEntity().getType(); }
 
-  /// Return the element type of the result (e.g. i32).
-  mlir::Type getResultElementType() const {
-    return hlfir::getFortranElementType(getResultType());
-  }
-
-  /// Return rank of the result.
-  unsigned getResultRank() const { return getResultEntity().getRank(); }
-
-  /// Return the element type of the source.
-  mlir::Type getSourceElementType() const {
-    return hlfir::getFortranElementType(getSource().getType());
-  }
-
-  /// Return rank of the input array.
-  unsigned getSourceRank() const {
-    return hlfir::Entity{getSource()}.getRank();
-  }
-
-  /// The reduction operation.
-  mlir::Operation *op;
-
-  mlir::PatternRewriter &rewriter;
-  mlir::Location loc;
-  fir::FirOpBuilder builder;
-};
-
-/// Generate initialization value for MIN or MAX reduction
-/// of the given \p type.
-template <bool IS_MAX>
-static mlir::Value genMinMaxInitValue(mlir::Location loc,
-                                      fir::FirOpBuilder &builder,
-                                      mlir::Type type) {
-  if (auto ty = mlir::dyn_cast<mlir::FloatType>(type)) {
-    const llvm::fltSemantics &sem = ty.getFloatSemantics();
-    // We must not use +/-INF here. If the reduction input is empty,
-    // the result of reduction must be +/-LARGEST.
-    llvm::APFloat limit = llvm::APFloat::getLargest(sem, /*Negative=*/IS_MAX);
-    return builder.createRealConstant(loc, type, limit);
-  }
-  unsigned bits = type.getIntOrFloatBitWidth();
-  int64_t limitInt = IS_MAX
-                         ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
-                         : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
-  return builder.createIntegerConstant(loc, type, limitInt);
-}
-
-/// Generate a comparison of an array element value \p elem
-/// and the current reduction value \p reduction for MIN/MAX reduction.
-template <bool IS_MAX>
-static mlir::Value
-genMinMaxComparison(mlir::Location loc, fir::FirOpBuilder &builder,
-                    mlir::Value elem, mlir::Value reduction) {
-  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
-    // For FP reductions we want the first smallest value to be used, that
-    // is not NaN. A OGL/OLT condition will usually work for this unless all
-    // the values are Nan or Inf. This follows the same logic as
-    // NumericCompare for Minloc/Maxloc in extrema.cpp.
-    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
-        loc,
-        IS_MAX ? mlir::arith::CmpFPredicate::OGT
-               : mlir::arith::CmpFPredicate::OLT,
-        elem, reduction);
-    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
-  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
-    return builder.create<mlir::arith::CmpIOp>(
-        loc,
-        IS_MAX ? mlir::arith::CmpIPredicate::sgt
-               : mlir::arith::CmpIPredicate::slt,
-        elem, reduction);
-  }
-  llvm_unreachable("unsupported type");
-}
-
-/// Implementation of ReductionAsElementalConverter interface
-/// for MAXLOC/MINLOC.
-template <typename T>
-class MinMaxlocAsElementalConverter : public ReductionAsElementalConverter {
-  static_assert(std::is_same_v<T, hlfir::MaxlocOp> ||
-                std::is_same_v<T, hlfir::MinlocOp>);
-  static constexpr unsigned maxRank = Fortran::common::maxRank;
-  // We have the following reduction values in the reduction loop:
-  //   * N integer coordinates, where N is:
-  //     - RANK(ARRAY) for total reductions.
-  //     - 1 for partial reductions.
-  //   * 1 reduction value holding the current MIN/MAX.
-  //   * 1 boolean indicating whether it is the first time
-  //     the mask is true.
-  static constexpr unsigned maxNumReductions = Fortran::common::maxRank + 2;
-  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxlocOp>;
-  using Base = ReductionAsElementalConverter;
-
-public:
-  MinMaxlocAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
-      : Base{op.getOperation(), rewriter} {}
-
-private:
-  virtual mlir::Value getSource() const final { return getOp().getArray(); }
-  virtual mlir::Value getDim() const final { return getOp().getDim(); }
-  virtual mlir::Value getMask() const final { return getOp().getMask(); }
-  virtual mlir::arith::FastMathFlags getFastMath() const final {
-    return getOp().getFastmath();
-  }
-
-  virtual mlir::LogicalResult isConvertible() const final {
-    if (getOp().getBack())
-      return rewriter.notifyMatchFailure(
-          getOp(), "BACK is not supported for MINLOC/MAXLOC inlining");
-    if (mlir::isa<fir::CharacterType>(getSourceElementType()))
-      return rewriter.notifyMatchFailure(
-          getOp(),
-          "CHARACTER type is not supported for MINLOC/MAXLOC inlining");
-    return mlir::success();
-  }
+    mlir::Location loc = sum.getLoc();
+    fir::FirOpBuilder builder{rewriter, sum.getOperation()};
+    mlir::Type elementType = hlfir::getFortranElementType(sum.getType());
+    mlir::Value mask = sum.getMask();
 
-  // If the result is scalar, then DIM does not matter,
-  // and this is a total reduction.
-  // If DIM is not present, this is a total reduction.
-  virtual bool isTotalReduction() const final {
-    return getResultRank() == 0 || !getDim();
-  }
+    mlir::Value resultShape, dimExtent;
+    llvm::SmallVector<mlir::Value> arrayExtents;
+    if (isTotalReduction)
+      arrayExtents = hlfir::genExtentsVector(loc, builder, array);
+    else
+      std::tie(resultShape, dimExtent) =
+          genResultShapeForPartialReduction(loc, builder, array, dimVal);
+
+    // If the mask is present and is a scalar, then we'd better load its value
+    // outside of the reduction loop making the loop unswitching easier.
+    mlir::Value isPresentPred, maskValue;
+    if (mask) {
+      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+        // MASK represented by a box might be dynamically optional,
+        // so we have to check for its presence before accessing it.
+        isPresentPred =
+            builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
+      }
 
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final;
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) final;
-  virtual hlfir::Entity genFinalResult(
-      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) final;
+      if (hlfir::Entity{mask}.isScalar())
+        maskValue = genMaskValue(loc, builder, mask, isPresentPred, {});
+    }
 
-private:
-  T getOp() const { return mlir::cast<T>(op); }
+    auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                         mlir::ValueRange inputIndices) -> hlfir::Entity {
+      // Loop over all indices in the DIM dimension, and reduce all values.
+      // If DIM is not present, do total reduction.
+
+      // Initial value for the reduction.
+      mlir::Value reductionInitValue =
+          fir::factory::createZeroValue(builder, loc, elementType);
+
+      // The reduction loop may be unordered if FastMathFlags::reassoc
+      // transformations are allowed. The integer reduction is always
+      // unordered.
+      bool isUnordered = mlir::isa<mlir::IntegerType>(elementType) ||
+                         static_cast<bool>(sum.getFastmath() &
+                                           mlir::arith::FastMathFlags::reassoc);
 
-  unsigned getNumCoors() const {
-    return isTotalReduction() ? getSourceRank() : 1;
-  }
+      llvm::SmallVector<mlir::Value> extents;
+      if (isTotalReduction)
+        extents = arrayExtents;
+      else
+        extents.push_back(
+            builder.createConvert(loc, builder.getIndexType(), dimExtent));
 
-  void
-  checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
-    assert(reductions.size() == getNumCoors() + 2 &&
-           "invalid number of reductions for MINLOC/MAXLOC");
-  }
+      auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                         mlir::ValueRange oneBasedIndices,
+                         mlir::ValueRange reductionArgs)
+          -> llvm::SmallVector<mlir::Value, 1> {
+        // Generate the reduction loop-nest body.
+        // The initial reduction value in the innermost loop
+        // is passed via reductionArgs[0].
+        llvm::SmallVector<mlir::Value> indices;
+        if (isTotalReduction) {
+          indices = oneBasedIndices;
+        } else {
+          indices = inputIndices;
+          indices.insert(indices.begin() + dimVal - 1, oneBasedIndices[0]);
+        }
 
-  mlir::Value
-  getCurrentMinMax(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
-    checkReductions(reductions);
-    return reductions[getNumCoors()];
-  }
+        mlir::Value reductionValue = reductionArgs[0];
+        fir::IfOp ifOp;
+        if (mask) {
+          // Make the reduction value update conditional on the value
+          // of the mask.
+          if (!maskValue) {
+            // If the mask is an array, use the elemental and the loop indices
+            // to address the proper mask element.
+            maskValue =
+                genMaskValue(loc, builder, mask, isPresentPred, indices);
+          }
+          mlir::Value isUnmasked = builder.create<fir::ConvertOp>(
+              loc, builder.getI1Type(), maskValue);
+          ifOp = builder.create<fir::IfOp>(loc, elementType, isUnmasked,
+                                           /*withElseRegion=*/true);
+          // In the 'else' block return the current reduction value.
+          builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+          builder.create<fir::ResultOp>(loc, reductionValue);
+
+          // In the 'then' block do the actual addition.
+          builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+        }
 
-  mlir::Value
-  getIsFirst(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
-    checkReductions(reductions);
-    return reductions[getNumCoors() + 1];
-  }
-};
+        hlfir::Entity element =
+            hlfir::getElementAt(loc, builder, array, indices);
+        hlfir::Entity elementValue =
+            hlfir::loadTrivialScalar(loc, builder, element);
+        // NOTE: we can use "Kahan summation" same way as the runtime
+        // (e.g. when fast-math is not allowed), but let's start with
+        // the simple version.
+        reductionValue =
+            genScalarAdd(loc, builder, reductionValue, elementValue);
+
+        if (ifOp) {
+          builder.create<fir::ResultOp>(loc, reductionValue);
+          builder.setInsertionPointAfter(ifOp);
+          reductionValue = ifOp.getResult(0);
+        }
 
-template <typename T>
-llvm::SmallVector<mlir::Value>
-MinMaxlocAsElementalConverter<T>::genReductionInitValues() {
-  // Initial value for the coordinate(s) is zero.
-  mlir::Value zeroCoor =
-      fir::factory::createZeroValue(builder, loc, getResultElementType());
-  llvm::SmallVector<mlir::Value> result(getNumCoors(), zeroCoor);
-
-  // Initial value for the MIN/MAX value.
-  mlir::Value minMaxInit =
-      genMinMaxInitValue<isMax>(loc, builder, getSourceElementType());
-  result.push_back(minMaxInit);
-
-  // Initial value for isFirst predicate. It is switched to false,
-  // when the reduction update dynamically happens inside the reduction
-  // loop.
-  mlir::Value trueVal = builder.createBool(loc, true);
-  result.push_back(trueVal);
-
-  return result;
-}
-
-template <typename T>
-llvm::SmallVector<mlir::Value>
-MinMaxlocAsElementalConverter<T>::reduceOneElement(
-    const llvm::SmallVectorImpl<mlir::Value> &currentValue, hlfir::Entity array,
-    mlir::ValueRange oneBasedIndices) {
-  checkReductions(currentValue);
-  hlfir::Entity elementValue =
-      hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-  mlir::Value cmp = genMinMaxComparison<isMax>(loc, builder, elementValue,
-                                               getCurrentMinMax(currentValue));
-  // If isFirst is true, then do the reduction update regardless
-  // of the FP comparison.
-  cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, getIsFirst(currentValue));
-
-  llvm::SmallVector<mlir::Value> newIndices;
-  int64_t dim = 1;
-  if (!isTotalReduction()) {
-    auto dimVal = getConstDim();
-    assert(mlir::succeeded(dimVal) &&
-           "partial MINLOC/MAXLOC reduction with invalid DIM");
-    dim = *dimVal;
-    assert(getNumCoors() == 1 &&
-           "partial MAXLOC/MINLOC reduction must compute one coordinate");
-  }
+        return {reductionValue};
+      };
 
-  for (unsigned coorIdx = 0; coorIdx < getNumCoors(); ++coorIdx) {
-    mlir::Value currentCoor = currentValue[coorIdx];
-    mlir::Value newCoor = builder.createConvert(
-        loc, currentCoor.getType(), oneBasedIndices[coorIdx + dim - 1]);
-    mlir::Value update =
-        builder.create<mlir::arith::SelectOp>(loc, cmp, newCoor, currentCoor);
-    newIndices.push_back(update);
-  }
+      llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
+          hlfir::genLoopNestWithReductions(loc, builder, extents,
+                                           {reductionInitValue}, genBody,
+                                           isUnordered);
+      return hlfir::Entity{reductionFinalValues[0]};
+    };
 
-  mlir::Value newMinMax = builder.create<mlir::arith::SelectOp>(
-      loc, cmp, elementValue, getCurrentMinMax(currentValue));
-  newIndices.push_back(newMinMax);
-
-  mlir::Value newIsFirst = builder.createBool(loc, false);
-  newIndices.push_back(newIsFirst);
-
-  assert(currentValue.size() == newIndices.size() &&
-         "invalid number of updated reductions");
-
-  return newIndices;
-}
-
-template <typename T>
-hlfir::Entity MinMaxlocAsElementalConverter<T>::genFinalResult(
-    const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
-  // Identification of the final result of MINLOC/MAXLOC:
-  //   * If DIM is absent, the result is rank-one array.
-  //   * If DIM is present:
-  //     - The result is scalar for rank-one input.
-  //     - The result is an array of rank RANK(ARRAY)-1.
-  checkReductions(reductionResults);
-
-  // We need to adjust the one-based indices to real array indices.
-  // The adjustment must only be done, if there was an actual update
-  // of the coordinates in the reduction loop. For this check we only
-  // need to compare if any of the reduction results is not zero.
-  mlir::Value zero = fir::factory::createZeroValue(
-      builder, loc, reductionResults[0].getType());
-  mlir::Value doAdjust = builder.create<mlir::arith::CmpIOp>(
-      loc, mlir::arith::CmpIPredicate::ne, reductionResults[0], zero);
-  mlir::Type indexType = builder.getIndexType();
-  mlir::Value one = builder.createIntegerConstant(loc, indexType, 1);
-
-  auto adjustCoor = [&](mlir::Value coor, mlir::Value lbound) {
-    assert(mlir::isa<mlir::IndexType>(lbound.getType()));
-    mlir::Value coorAsIndex = builder.createConvert(loc, indexType, coor);
-    mlir::Value tmp =
-        builder.create<mlir::arith::AddIOp>(loc, coorAsIndex, lbound);
-    tmp = builder.create<mlir::arith::SubIOp>(loc, tmp, one);
-    tmp =
-        builder.create<mlir::arith::SelectOp>(loc, doAdjust, tmp, coorAsIndex);
-    return builder.createConvert(loc, coor.getType(), tmp);
-  };
-
-  // For partial reductions, the final result of the reduction
-  // loop is just a scalar - the coordinate within DIM dimension.
-  if (getResultRank() == 0 || !isTotalReduction()) {
-    // The result is a scalar, so just return the scalar.
-    assert(getNumCoors() == 1 &&
-           "unpexpected number of coordinates for scalar result");
-
-    int64_t dim = 1;
-    if (!isTotalReduction()) {
-      auto dimVal = getConstDim();
-      assert(mlir::succeeded(dimVal) &&
-             "partial MINLOC/MAXLOC reduction with invalid DIM");
-      dim = *dimVal;
+    if (isTotalReduction) {
+      hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
+      rewriter.replaceOp(sum, result);
+      return mlir::success();
     }
-    mlir::Value dimLbound =
-        hlfir::genLBound(loc, builder, hlfir::Entity{getSource()}, dim - 1);
-    return hlfir::Entity{adjustCoor(reductionResults[0], dimLbound)};
-  }
-  // This is a total reduction, and there is no wrapping hlfir.elemental.
-  // We have to pack the reduced coordinates into a rank-one array.
-  unsigned rank = getSourceRank();
-  // TODO: in order to avoid introducing new memory effects
-  // we should not use a temporary in memory.
-  // We can use hlfir.elemental with a switch to pack all the coordinates
-  // into an array expression, or we can have a dedicated HLFIR operation
-  // for this.
-  mlir::Value tempArray = builder.createTemporary(
-      loc, fir::SequenceType::get(rank, getResultElementType()));
-  llvm::SmallVector<mlir::Value, maxRank> arrayLbounds =
-      hlfir::genLBounds(loc, builder, hlfir::Entity(getSource()));
-  for (unsigned i = 0; i < rank; ++i) {
-    mlir::Value coor = adjustCoor(reductionResults[i], arrayLbounds[i]);
-    mlir::Value idx = builder.createIntegerConstant(loc, indexType, i + 1);
-    mlir::Value resultElement =
-        hlfir::getElementAt(loc, builder, hlfir::Entity{tempArray}, {idx});
-    builder.create<hlfir::AssignOp>(loc, coor, resultElement);
-  }
-  mlir::Value tempExpr = builder.create<hlfir::AsExprOp>(
-      loc, tempArray, builder.createBool(loc, false));
-  return hlfir::Entity{tempExpr};
-}
-
-/// Base class for numeric reductions like MAXVAl, MINVAL, SUM.
-template <typename OpT>
-class NumericReductionAsElementalConverterBase
-    : public ReductionAsElementalConverter {
-  using Base = ReductionAsElementalConverter;
-
-protected:
-  NumericReductionAsElementalConverterBase(OpT op,
-                                           mlir::PatternRewriter &rewriter)
-      : Base{op.getOperation(), rewriter} {}
-
-  virtual mlir::Value getSource() const final { return getOp().getArray(); }
-  virtual mlir::Value getDim() const final { return getOp().getDim(); }
-  virtual mlir::Value getMask() const final { return getOp().getMask(); }
-  virtual mlir::arith::FastMathFlags getFastMath() const final {
-    return getOp().getFastmath();
-  }
 
-  OpT getOp() const { return mlir::cast<OpT>(op); }
-
-  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
-    assert(reductions.size() == 1 && "reduction must produce single value");
-  }
-};
-
-/// Reduction converter for MAXMAL/MINVAL.
-template <typename T>
-class MinMaxvalAsElementalConverter
-    : public NumericReductionAsElementalConverterBase<T> {
-  static_assert(std::is_same_v<T, hlfir::MaxvalOp> ||
-                std::is_same_v<T, hlfir::MinvalOp>);
-  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxvalOp>;
-  using Base = NumericReductionAsElementalConverterBase<T>;
+    hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
+        loc, builder, elementType, resultShape, {}, genKernel,
+        /*isUnordered=*/true, /*polymorphicMold=*/nullptr,
+        sum.getResult().getType());
 
-public:
-  MinMaxvalAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
-      : Base{op, rewriter} {}
+    // it wouldn't be safe to replace block arguments with a different
+    // hlfir.expr type. Types can differ due to differing amounts of shape
+    // information
+    assert(elementalOp.getResult().getType() == sum.getResult().getType());
 
-private:
-  virtual mlir::LogicalResult isConvertible() const final {
-    if (mlir::isa<fir::CharacterType>(this->getSourceElementType()))
-      return this->rewriter.notifyMatchFailure(
-          this->getOp(),
-          "CHARACTER type is not supported for MINVAL/MAXVAL inlining");
+    rewriter.replaceOp(sum, elementalOp);
     return mlir::success();
   }
 
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
-    return {genMinMaxInitValue<isMax>(this->loc, this->builder,
-                                      this->getResultElementType())};
-  }
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array,
-                   mlir::ValueRange oneBasedIndices) final {
-    this->checkReductions(currentValue);
-    fir::FirOpBuilder &builder = this->builder;
-    mlir::Location loc = this->loc;
-    hlfir::Entity elementValue =
-        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-    mlir::Value cmp =
-        genMinMaxComparison<isMax>(loc, builder, elementValue, currentValue[0]);
-    return {builder.create<mlir::arith::SelectOp>(loc, cmp, elementValue,
-                                                  currentValue[0])};
-  }
-};
-
-/// Reduction converter for SUM.
-class SumAsElementalConverter
-    : public NumericReductionAsElementalConverterBase<hlfir::SumOp> {
-  using Base = NumericReductionAsElementalConverterBase;
-
-public:
-  SumAsElementalConverter(hlfir::SumOp op, mlir::PatternRewriter &rewriter)
-      : Base{op, rewriter} {}
-
-private:
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
-    return {
-        fir::factory::createZeroValue(builder, loc, getResultElementType())};
-  }
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array,
-                   mlir::ValueRange oneBasedIndices) final {
-    checkReductions(currentValue);
-    hlfir::Entity elementValue =
-        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-    // NOTE: we can use "Kahan summation" same way as the runtime
-    // (e.g. when fast-math is not allowed), but let's start with
-    // the simple version.
-    return {genScalarAdd(currentValue[0], elementValue)};
-  }
-
-  // Generate scalar addition of the two values (of the same data type).
-  mlir::Value genScalarAdd(mlir::Value value1, mlir::Value value2);
-};
-
-/// Base class for logical reductions like ALL, ANY, COUNT.
-/// They do not have MASK and FastMathFlags.
-template <typename OpT>
-class LogicalReductionAsElementalConverterBase
-    : public ReductionAsElementalConverter {
-  using Base = ReductionAsElementalConverter;
-
-public:
-  LogicalReductionAsElementalConverterBase(OpT op,
-                                           mlir::PatternRewriter &rewriter)
-      : Base{op.getOperation(), rewriter} {}
-
-protected:
-  OpT getOp() const { return mlir::cast<OpT>(op); }
-
-  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
-    assert(reductions.size() == 1 && "reduction must produce single value");
-  }
-
-  virtual mlir::Value getSource() const final { return getOp().getMask(); }
-  virtual mlir::Value getDim() const final { return getOp().getDim(); }
-
-  virtual hlfir::Entity genFinalResult(
-      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) override {
-    checkReductions(reductionResults);
-    return hlfir::Entity{reductionResults[0]};
-  }
-};
-
-/// Reduction converter for ALL/ANY.
-template <typename T>
-class AllAnyAsElementalConverter
-    : public LogicalReductionAsElementalConverterBase<T> {
-  static_assert(std::is_same_v<T, hlfir::AllOp> ||
-                std::is_same_v<T, hlfir::AnyOp>);
-  static constexpr bool isAll = std::is_same_v<T, hlfir::AllOp>;
-  using Base = LogicalReductionAsElementalConverterBase<T>;
-
-public:
-  AllAnyAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
-      : Base{op, rewriter} {}
-
 private:
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
-    return {this->builder.createBool(this->loc, isAll ? true : false)};
-  }
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array,
-                   mlir::ValueRange oneBasedIndices) final {
-    this->checkReductions(currentValue);
-    fir::FirOpBuilder &builder = this->builder;
-    mlir::Location loc = this->loc;
-    hlfir::Entity elementValue =
-        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-    mlir::Value mask =
-        builder.createConvert(loc, builder.getI1Type(), elementValue);
-    if constexpr (isAll)
-      return {builder.create<mlir::arith::AndIOp>(loc, mask, currentValue[0])};
-    else
-      return {builder.create<mlir::arith::OrIOp>(loc, mask, currentValue[0])};
-  }
+  // Return fir.shape specifying the shape of the result
+  // of a SUM reduction with DIM=dimVal. The second return value
+  // is the extent of the DIM dimension.
+  static std::tuple<mlir::Value, mlir::Value>
+  genResultShapeForPartialReduction(mlir::Location loc,
+                                    fir::FirOpBuilder &builder,
+                                    hlfir::Entity array, int64_t dimVal) {
+    llvm::SmallVector<mlir::Value> inExtents =
+        hlfir::genExtentsVector(loc, builder, array);
+    assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
+           "DIM must be present and a positive constant not exceeding "
+           "the array's rank");
 
-  virtual hlfir::Entity genFinalResult(
-      const llvm::SmallVectorImpl<mlir::Value> &reductionValues) final {
-    this->checkReductions(reductionValues);
-    return hlfir::Entity{this->builder.createConvert(
-        this->loc, this->getResultElementType(), reductionValues[0])};
+    mlir::Value dimExtent = inExtents[dimVal - 1];
+    inExtents.erase(inExtents.begin() + dimVal - 1);
+    return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
   }
-};
 
-/// Reduction converter for COUNT.
-class CountAsElementalConverter
-    : public LogicalReductionAsElementalConverterBase<hlfir::CountOp> {
-  using Base = LogicalReductionAsElementalConverterBase<hlfir::CountOp>;
-
-public:
-  CountAsElementalConverter(hlfir::CountOp op, mlir::PatternRewriter &rewriter)
-      : Base{op, rewriter} {}
-
-private:
-  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
-    return {
-        fir::factory::createZeroValue(builder, loc, getResultElementType())};
-  }
-  virtual llvm::SmallVector<mlir::Value>
-  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
-                   hlfir::Entity array,
-                   mlir::ValueRange oneBasedIndices) final {
-    checkReductions(currentValue);
-    hlfir::Entity elementValue =
-        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
-    mlir::Value cond =
-        builder.createConvert(loc, builder.getI1Type(), elementValue);
-    mlir::Value one =
-        builder.createIntegerConstant(loc, getResultElementType(), 1);
-    mlir::Value add1 =
-        builder.create<mlir::arith::AddIOp>(loc, currentValue[0], one);
-    return {builder.create<mlir::arith::SelectOp>(loc, cond, add1,
-                                                  currentValue[0])};
+  // Generate scalar addition of the two values (of the same data type).
+  static mlir::Value genScalarAdd(mlir::Location loc,
+                                  fir::FirOpBuilder &builder,
+                                  mlir::Value value1, mlir::Value value2) {
+    mlir::Type ty = value1.getType();
+    assert(ty == value2.getType() && "reduction values' types do not match");
+    if (mlir::isa<mlir::FloatType>(ty))
+      return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+    else if (mlir::isa<mlir::ComplexType>(ty))
+      return builder.create<fir::AddcOp>(loc, value1, value2);
+    else if (mlir::isa<mlir::IntegerType>(ty))
+      return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+
+    llvm_unreachable("unsupported SUM reduction type");
   }
-};
 
-mlir::LogicalResult ReductionAsElementalConverter::convert() {
-  mlir::LogicalResult canConvert(isConvertible());
-
-  if (mlir::failed(canConvert))
-    return canConvert;
-
-  hlfir::Entity array = hlfir::Entity{getSource()};
-  bool isTotalReduce = isTotalReduction();
-  auto dimVal = getConstDim();
-  if (mlir::failed(dimVal))
-    return dimVal;
-  mlir::Value mask = getMask();
-  mlir::Value resultShape, dimExtent;
-  llvm::SmallVector<mlir::Value> arrayExtents;
-  if (isTotalReduce)
-    arrayExtents = hlfir::genExtentsVector(loc, builder, array);
-  else
-    std::tie(resultShape, dimExtent) =
-        genResultShapeForPartialReduction(array, *dimVal);
-
-  // If the mask is present and is a scalar, then we'd better load its value
-  // outside of the reduction loop making the loop unswitching easier.
-  mlir::Value isPresentPred, maskValue;
-  if (mask) {
-    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-      // MASK represented by a box might be dynamically optional,
-      // so we have to check for its presence before accessing it.
-      isPresentPred =
-          builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
+  static mlir::Value genMaskValue(mlir::Location loc,
+                                  fir::FirOpBuilder &builder, mlir::Value mask,
+                                  mlir::Value isPresentPred,
+                                  mlir::ValueRange indices) {
+    mlir::OpBuilder::InsertionGuard guard(builder);
+    fir::IfOp ifOp;
+    mlir::Type maskType =
+        hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
+    if (isPresentPred) {
+      ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
+                                       /*withElseRegion=*/true);
+
+      // Use 'true', if the mask is not present.
+      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+      mlir::Value trueValue = builder.createBool(loc, true);
+      trueValue = builder.createConvert(loc, maskType, trueValue);
+      builder.create<fir::ResultOp>(loc, trueValue);
+
+      // Load the mask value, if the mask is present.
+      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
     }
 
-    if (hlfir::Entity{mask}.isScalar())
-      maskValue = genMaskValue(mask, isPresentPred, {});
-  }
-
-  auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                       mlir::ValueRange inputIndices) -> hlfir::Entity {
-    // Loop over all indices in the DIM dimension, and reduce all values.
-    // If DIM is not present, do total reduction.
-
-    // Initial value for the reduction.
-    llvm::SmallVector<mlir::Value, 1> reductionInitValues =
-        genReductionInitValues();
-
-    llvm::SmallVector<mlir::Value> extents;
-    if (isTotalReduce)
-      extents = arrayExtents;
-    else
-      extents.push_back(
-          builder.createConvert(loc, builder.getIndexType(), dimExtent));
-
-    auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                       mlir::ValueRange oneBasedIndices,
-                       mlir::ValueRange reductionArgs)
-        -> llvm::SmallVector<mlir::Value, 1> {
-      // Generate the reduction loop-nest body.
-      // The initial reduction value in the innermost loop
-      // is passed via reductionArgs[0].
-      llvm::SmallVector<mlir::Value> indices;
-      if (isTotalReduce) {
-        indices = oneBasedIndices;
+    hlfir::Entity maskVar{mask};
+    if (maskVar.isScalar()) {
+      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+        // MASK may be a boxed scalar.
+        mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
+        mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
       } else {
-        indices = inputIndices;
-        indices.insert(indices.begin() + *dimVal - 1, oneBasedIndices[0]);
-      }
-
-      llvm::SmallVector<mlir::Value, 1> reductionValues = reductionArgs;
-      llvm::SmallVector<mlir::Type, 1> reductionTypes;
-      llvm::transform(reductionValues, std::back_inserter(reductionTypes),
-                      [](mlir::Value v) { return v.getType(); });
-      fir::IfOp ifOp;
-      if (mask) {
-        // Make the reduction value update conditional on the value
-        // of the mask.
-        if (!maskValue) {
-          // If the mask is an array, use the elemental and the loop indices
-          // to address the proper mask element.
-          maskValue = genMaskValue(mask, isPresentPred, indices);
-        }
-        mlir::Value isUnmasked =
-            builder.create<fir::ConvertOp>(loc, builder.getI1Type(), maskValue);
-        ifOp = builder.create<fir::IfOp>(loc, reductionTypes, isUnmasked,
-                                         /*withElseRegion=*/true);
-        // In the 'else' block return the current reduction value.
-        builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-        builder.create<fir::ResultOp>(loc, reductionValues);
-
-        // In the 'then' block do the actual addition.
-        builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+        mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
       }
-      reductionValues = reduceOneElement(reductionValues, array, indices);
-      if (ifOp) {
-        builder.create<fir::ResultOp>(loc, reductionValues);
-        builder.setInsertionPointAfter(ifOp);
-        reductionValues = ifOp.getResults();
-      }
-
-      return reductionValues;
-    };
-
-    llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
-        hlfir::genLoopNestWithReductions(
-            loc, builder, extents, reductionInitValues, genBody, isUnordered());
-    return genFinalResult(reductionFinalValues);
-  };
-
-  if (isTotalReduce) {
-    hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
-    rewriter.replaceOp(op, result);
-    return mlir::success();
-  }
-
-  hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
-      loc, builder, getResultElementType(), resultShape, /*typeParams=*/{},
-      genKernel,
-      /*isUnordered=*/true, /*polymorphicMold=*/nullptr, getResultType());
-
-  // it wouldn't be safe to replace block arguments with a different
-  // hlfir.expr type. Types can differ due to differing amounts of shape
-  // information
-  assert(elementalOp.getResult().getType() == op->getResult(0).getType());
-
-  rewriter.replaceOp(op, elementalOp);
-  return mlir::success();
-}
-
-std::tuple<mlir::Value, mlir::Value>
-ReductionAsElementalConverter::genResultShapeForPartialReduction(
-    hlfir::Entity array, int64_t dimVal) {
-  llvm::SmallVector<mlir::Value> inExtents =
-      hlfir::genExtentsVector(loc, builder, array);
-  assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
-         "DIM must be present and a positive constant not exceeding "
-         "the array's rank");
-
-  mlir::Value dimExtent = inExtents[dimVal - 1];
-  inExtents.erase(inExtents.begin() + dimVal - 1);
-  return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
-}
-
-mlir::Value SumAsElementalConverter::genScalarAdd(mlir::Value value1,
-                                                  mlir::Value value2) {
-  mlir::Type ty = value1.getType();
-  assert(ty == value2.getType() && "reduction values' types do not match");
-  if (mlir::isa<mlir::FloatType>(ty))
-    return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
-  else if (mlir::isa<mlir::ComplexType>(ty))
-    return builder.create<fir::AddcOp>(loc, value1, value2);
-  else if (mlir::isa<mlir::IntegerType>(ty))
-    return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
-
-  llvm_unreachable("unsupported SUM reduction type");
-}
-
-mlir::Value ReductionAsElementalConverter::genMaskValue(
-    mlir::Value mask, mlir::Value isPresentPred, mlir::ValueRange indices) {
-  mlir::OpBuilder::InsertionGuard guard(builder);
-  fir::IfOp ifOp;
-  mlir::Type maskType =
-      hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
-  if (isPresentPred) {
-    ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
-                                     /*withElseRegion=*/true);
-
-    // Use 'true', if the mask is not present.
-    builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-    mlir::Value trueValue = builder.createBool(loc, true);
-    trueValue = builder.createConvert(loc, maskType, trueValue);
-    builder.create<fir::ResultOp>(loc, trueValue);
-
-    // Load the mask value, if the mask is present.
-    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  }
-
-  hlfir::Entity maskVar{mask};
-  if (maskVar.isScalar()) {
-    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-      // MASK may be a boxed scalar.
-      mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
-      mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
     } else {
+      // Load from the mask array.
+      assert(!indices.empty() && "no indices for addressing the mask array");
+      maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
       mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
     }
-  } else {
-    // Load from the mask array.
-    assert(!indices.empty() && "no indices for addressing the mask array");
-    maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
-    mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
-  }
-
-  if (!isPresentPred)
-    return mask;
 
-  builder.create<fir::ResultOp>(loc, mask);
-  return ifOp.getResult(0);
-}
-
-/// Convert an operation that is a partial or total reduction
-/// over an array of values into a reduction loop[-nest]
-/// optionally wrapped into hlfir.elemental.
-template <typename Op>
-class ReductionConversion : public mlir::OpRewritePattern<Op> {
-public:
-  using mlir::OpRewritePattern<Op>::OpRewritePattern;
+    if (!isPresentPred)
+      return mask;
 
-  llvm::LogicalResult
-  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
-    if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
-                  std::is_same_v<Op, hlfir::MinlocOp>) {
-      MinMaxlocAsElementalConverter<Op> converter(op, rewriter);
-      return converter.convert();
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
-                         std::is_same_v<Op, hlfir::MinvalOp>) {
-      MinMaxvalAsElementalConverter<Op> converter(op, rewriter);
-      return converter.convert();
-    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
-      CountAsElementalConverter converter(op, rewriter);
-      return converter.convert();
-    } else if constexpr (std::is_same_v<Op, hlfir::AllOp> ||
-                         std::is_same_v<Op, hlfir::AnyOp>) {
-      AllAnyAsElementalConverter<Op> converter(op, rewriter);
-      return converter.convert();
-    } else if constexpr (std::is_same_v<Op, hlfir::SumOp>) {
-      SumAsElementalConverter converter{op, rewriter};
-      return converter.convert();
-    }
-    return rewriter.notifyMatchFailure(op, "unexpected reduction operation");
+    builder.create<fir::ResultOp>(loc, mask);
+    return ifOp.getResult(0);
   }
 };
 
@@ -2117,18 +1481,10 @@ class SimplifyHLFIRIntrinsics
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
-    patterns.insert<ReductionConversion<hlfir::SumOp>>(context);
+    patterns.insert<SumAsElementalConversion>(context);
     patterns.insert<CShiftConversion>(context);
     patterns.insert<MatmulConversion<hlfir::MatmulTransposeOp>>(context);
 
-    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
-
     // If forceMatmulAsElemental is false, then hlfir.matmul inlining
     // will introduce hlfir.eval_in_mem operation with new memory side
     // effects. This conflicts with CSE and optimized bufferization, e.g.:
diff --git a/flang/test/HLFIR/all-elemental.fir b/flang/test/HLFIR/all-elemental.fir
new file mode 100644
index 0000000000000..1ba8bb1b7a5fb
--- /dev/null
+++ b/flang/test/HLFIR/all-elemental.fir
@@ -0,0 +1,91 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %15 = fir.load %14 : !fir.ref<i32>
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %12 = hlfir.all %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
+  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
+  return %13 : !fir.logical<4>
+}
+// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
+// CHECK-NEXT:     %true = arith.constant true
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
+// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
+// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
+// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %true) -> (i1) {
+// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
+// CHECK-NEXT:       %[[V17:.*]] = arith.andi %arg4, %[[V16]] : i1
+// CHECK-NEXT:       fir.result %[[V17]] : i1
+// CHECK-NEXT:     }
+// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
+
+
+func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
+  %c2_i32 = arith.constant 2 : i32
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
+  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
+  %8 = fir.load %6#0 : !fir.ref<i32>
+  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index):
+    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
+    %13 = fir.load %12 : !fir.ref<i32>
+    %14 = arith.cmpi sge, %13, %8 : i32
+    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %15 : !fir.logical<4>
+  }
+  %10 = hlfir.all %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
+  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
+  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
+  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
+  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
+  return %11 : !fir.array<4x!fir.logical<4>>
+}
+// CHECK-LABEL:  func.func @_QFPtest_dim(
+// CHECK: %10 = hlfir.all %9 dim %c2_i32
\ No newline at end of file
diff --git a/flang/test/HLFIR/any-elemental.fir b/flang/test/HLFIR/any-elemental.fir
new file mode 100644
index 0000000000000..a7c559679d965
--- /dev/null
+++ b/flang/test/HLFIR/any-elemental.fir
@@ -0,0 +1,190 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %15 = fir.load %14 : !fir.ref<i32>
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %12 = hlfir.any %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
+  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
+  return %13 : !fir.logical<4>
+}
+// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
+// CHECK-NEXT:     %false = arith.constant false
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
+// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
+// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
+// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %false) -> (i1) {
+// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
+// CHECK-NEXT:       %[[V17:.*]] = arith.ori %arg4, %[[V16]] : i1
+// CHECK-NEXT:       fir.result %[[V17]] : i1
+// CHECK-NEXT:     }
+// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
+
+
+func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
+  %c2_i32 = arith.constant 2 : i32
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
+  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
+  %8 = fir.load %6#0 : !fir.ref<i32>
+  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index):
+    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
+    %13 = fir.load %12 : !fir.ref<i32>
+    %14 = arith.cmpi sge, %13, %8 : i32
+    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %15 : !fir.logical<4>
+  }
+  %10 = hlfir.any %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
+  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
+  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
+  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
+  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
+  return %11 : !fir.array<4x!fir.logical<4>>
+}
+// CHECK-LABEL:  func.func @_QFPtest_dim(
+// CHECK: {{.*}} = hlfir.any {{.*}} dim %c2_i32
+
+
+func.func @_Qtest_recursive() attributes {fir.bindc_name = "test"} {
+  %c1 = arith.constant 1 : index
+  %true = arith.constant true
+  %false = arith.constant false
+  %c0_i64 = arith.constant 0 : i64
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c1_i32 = arith.constant 1 : i32
+  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %1:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+  %2 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
+  %5:2 = hlfir.declare %4 {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "ra", uniq_name = "_QFEra"}
+  %7 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %8:2 = hlfir.declare %6(%7) {uniq_name = "_QFEra"} : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.ref<!fir.array<1x!fir.logical<4>>>)
+  %9 = fir.alloca !fir.logical<4> {bindc_name = "rs", uniq_name = "_QFErs"}
+  %10:2 = hlfir.declare %9 {uniq_name = "_QFErs"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %11 = fir.allocmem !fir.array<?xi32>, %c1 {fir.must_be_heap = true, uniq_name = "_QFEa.alloc"}
+  %12 = fir.embox %11(%7) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+  fir.store %12 to %1#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  hlfir.assign %c1_i32 to %5#0 : i32, !fir.ref<i32>
+  %13 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %14:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+  fir.do_loop %arg0 = %c1 to %14#1 step %c1 unordered {
+    %27:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+    %28 = arith.subi %27#0, %c1 : index
+    %29 = arith.addi %arg0, %28 : index
+    %30 = hlfir.designate %13 (%29)  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+    hlfir.assign %c2_i32 to %30 : i32, !fir.ref<i32>
+  }
+  %15 = fir.load %5#0 : !fir.ref<i32>
+  %16 = fir.convert %15 : (i32) -> i64
+  %17 = arith.cmpi sgt, %16, %c0_i64 : i64
+  %18 = arith.select %17, %16, %c0_i64 : i64
+  %19 = fir.convert %18 : (i64) -> index
+  %20 = fir.shape %19 : (index) -> !fir.shape<1>
+  %21 = hlfir.elemental %20 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg0: index):
+    %27 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+    %28:3 = fir.box_dims %27, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+    %29 = arith.addi %28#0, %28#1 : index
+    %30 = arith.subi %29, %c1 : index
+    %31 = arith.subi %30, %28#0 : index
+    %32 = arith.addi %31, %c1 : index
+    %33 = arith.cmpi sgt, %32, %c0 : index
+    %34 = arith.select %33, %32, %c0 : index
+    %35 = fir.shape %34 : (index) -> !fir.shape<1>
+    %36 = hlfir.designate %27 (%28#0:%30:%c1)  shape %35 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+    %37 = hlfir.elemental %35 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+    ^bb0(%arg1: index):
+      %39 = hlfir.designate %36 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+      %40 = fir.load %39 : !fir.ref<i32>
+      %41 = arith.cmpi eq, %40, %c1_i32 : i32
+      %42 = fir.convert %41 : (i1) -> !fir.logical<4>
+      hlfir.yield_element %42 : !fir.logical<4>
+    }
+    %38 = hlfir.any %37 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
+    hlfir.destroy %37 : !hlfir.expr<?x!fir.logical<4>>
+    hlfir.yield_element %38 : !fir.logical<4>
+  }
+  %22 = hlfir.any %21 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
+  hlfir.assign %22 to %10#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+  hlfir.destroy %21 : !hlfir.expr<?x!fir.logical<4>>
+  %23 = fir.load %10#0 : !fir.ref<!fir.logical<4>>
+  %24 = fir.convert %23 : (!fir.logical<4>) -> i1
+  %25 = arith.xori %24, %true : i1
+  cf.cond_br %25, ^bb1, ^bb2
+^bb1:  // pred: ^bb0
+  fir.call @_FortranAStopStatement(%c2_i32, %false, %false) fastmath<contract> : (i32, i1, i1) -> ()
+  fir.unreachable
+^bb2:  // pred: ^bb0
+  return
+}
+// CHECK-LABEL: func.func @_Qtest_recursive()
+// CHECK:    %[[V20:.*]] = fir.do_loop %arg0 = %c1 to %{{.*}} step %c1 iter_args(%arg1 = %false) -> (i1) {
+// CHECK:      %[[V26:.*]] = fir.load %[[V1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK:      %[[V27:.*]]:3 = fir.box_dims %[[V26]], %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+// CHECK:      %[[V28:.*]] = arith.addi %[[V27]]#0, %[[V27]]#1 : index
+// CHECK:      %[[V29:.*]] = arith.subi %[[V28]], %c1 : index
+// CHECK:      %[[V30:.*]] = arith.subi %[[V29]], %[[V27]]#0 : index
+// CHECK:      %[[V31:.*]] = arith.addi %[[V30]], %c1 : index
+// CHECK:      %[[V32:.*]] = arith.cmpi sgt, %[[V31]], %c0 : index
+// CHECK:      %[[V33:.*]] = arith.select %[[V32]], %[[V31]], %c0 : index
+// CHECK:      %[[V34:.*]] = fir.shape %[[V33]] : (index) -> !fir.shape<1>
+// CHECK:      %[[V35:.*]] = hlfir.designate %[[V26]] (%[[V27]]#0:%[[V29]]:%c1)  shape %[[V34]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:      %[[V36:.*]] = fir.do_loop %arg2 = %c1 to %[[V33]] step %c1 iter_args(%arg3 = %false) -> (i1) {
+// CHECK:        %[[V38:.*]] = hlfir.designate %[[V35]] (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:        %[[V39:.*]] = fir.load %[[V38]] : !fir.ref<i32>
+// CHECK:        %[[V40:.*]] = arith.cmpi eq, %[[V39]], %c1_i32 : i32
+// CHECK:        %[[V41:.*]] = arith.ori %arg3, %[[V40]] : i1
+// CHECK:        fir.result %[[V41]] : i1
+// CHECK:      }
+// CHECK:      %[[V37:.*]] = arith.ori %arg1, %[[V36]] : i1
+// CHECK:      fir.result %[[V37]] : i1
+// CHECK:    }
diff --git a/flang/test/HLFIR/count-elemental.fir b/flang/test/HLFIR/count-elemental.fir
new file mode 100644
index 0000000000000..0df5cc3c031ea
--- /dev/null
+++ b/flang/test/HLFIR/count-elemental.fir
@@ -0,0 +1,314 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %15 = fir.load %14 : !fir.ref<i32>
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
+  hlfir.assign %12 to %4#0 : i32, !fir.ref<i32>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %13 = fir.load %4#1 : !fir.ref<i32>
+  return %13 : i32
+}
+// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
+// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
+// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
+// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
+// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
+// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
+// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
+// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
+// CHECK-NEXT:       fir.result %[[V17]] : i32
+// CHECK-NEXT:     }
+// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
+// CHECK-NEXT:     return %[[V12]] : i32
+
+func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i16 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %15 = fir.load %14 : !fir.ref<i32>
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i16
+  hlfir.assign %12 to %4#0 : i16, !fir.ref<i16>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %13 = fir.load %4#1 : !fir.ref<i16>
+  return %13 : i16
+}
+// CHECK-LABEL:  func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
+// CHECK-NEXT:     %c1_i16 = arith.constant 1 : i16
+// CHECK-NEXT:     %c0_i16 = arith.constant 0 : i16
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i16
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
+// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
+// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
+// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i16) -> (i16) {
+// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
+// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i16 : i16
+// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i16
+// CHECK-NEXT:       fir.result %[[V17]] : i16
+// CHECK-NEXT:     }
+// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i16, !fir.ref<i16>
+// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i16>
+// CHECK-NEXT:     return %[[V12]] : i16
+
+func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<7xi32> {
+  %c1_i32 = arith.constant 1 : i32
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca !fir.array<7xi32> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<7xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<7xi32>>, !fir.ref<!fir.array<7xi32>>)
+  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
+  %8 = fir.load %6#0 : !fir.ref<i32>
+  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index):
+    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
+    %13 = fir.load %12 : !fir.ref<i32>
+    %14 = arith.cmpi sge, %13, %8 : i32
+    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %15 : !fir.logical<4>
+  }
+  %10 = hlfir.count %9 dim %c1_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<7xi32>
+  hlfir.assign %10 to %5#0 : !hlfir.expr<7xi32>, !fir.ref<!fir.array<7xi32>>
+  hlfir.destroy %10 : !hlfir.expr<7xi32>
+  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
+  %11 = fir.load %5#1 : !fir.ref<!fir.array<7xi32>>
+  return %11 : !fir.array<7xi32>
+}
+// CHECK-LABEL:  func.func @_QFPtest_dim(
+// CHECK: %{{.*}} = hlfir.count %{{.*}} dim %c1_i32
+
+
+func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %c2 = arith.constant 2 : index
+  %0 = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %0 : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
+  %7 = fir.load %5#0 : !fir.ref<i32>
+  %8 = hlfir.elemental %0 unordered : (!fir.shape<3>) -> !hlfir.expr<4x7x2x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index, %arg5: index):
+    %11 = hlfir.designate %6 (%arg3, %arg4, %arg5)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
+    %12 = fir.load %11 : !fir.ref<i32>
+    %13 = arith.cmpi sge, %12, %7 : i32
+    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %14 : !fir.logical<4>
+  }
+  %9 = hlfir.count %8 : (!hlfir.expr<4x7x2x!fir.logical<4>>) -> i32
+  hlfir.assign %9 to %4#0 : i32, !fir.ref<i32>
+  hlfir.destroy %8 : !hlfir.expr<4x7x2x!fir.logical<4>>
+  %10 = fir.load %4#1 : !fir.ref<i32>
+  return %10 : i32
+}
+// CHECK-LABEL:  func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
+// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
+// CHECK-NEXT:     %c1 = arith.constant 1 : index
+// CHECK-NEXT:     %c4 = arith.constant 4 : index
+// CHECK-NEXT:     %c7 = arith.constant 7 : index
+// CHECK-NEXT:     %c2 = arith.constant 2 : index
+// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
+// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]]) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
+// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]] {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:     %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %[[V0]] : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
+// CHECK-NEXT:     %[[V7:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
+// CHECK-NEXT:     %[[V8:.*]] = fir.do_loop %arg3 = %c1 to %c2 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
+// CHECK-NEXT:       %[[V10:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %arg4) -> (i32) {
+// CHECK-NEXT:         %[[V11:.*]] = fir.do_loop %arg7 = %c1 to %c4 step %c1 iter_args(%arg8 = %arg6) -> (i32) {
+// CHECK-NEXT:           %[[V12:.*]] = hlfir.designate %[[V6]] (%arg7, %arg5, %arg3)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
+// CHECK-NEXT:           %[[V13:.*]] = fir.load %[[V12]] : !fir.ref<i32>
+// CHECK-NEXT:           %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V7]] : i32
+// CHECK-NEXT:           %[[V15:.*]] = arith.addi %arg8, %c1_i32 : i32
+// CHECK-NEXT:           %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg8 : i32
+// CHECK-NEXT:           fir.result %[[V16]] : i32
+// CHECK-NEXT:         }
+// CHECK-NEXT:         fir.result %[[V11]] : i32
+// CHECK-NEXT:       }
+// CHECK-NEXT:       fir.result %[[V10]] : i32
+// CHECK-NEXT:     }
+// CHECK-NEXT:     hlfir.assign %[[V8]] to %[[V4]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:     %[[V9:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
+// CHECK-NEXT:     return %[[V9]] : i32
+
+
+
+
+
+func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7xi32> {
+  ^bb0(%arg3: index):
+    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %16 = fir.load %15 : !fir.ref<i32>
+    hlfir.yield_element %16 : i32
+  }
+  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %15 = hlfir.sum %11 : (!hlfir.expr<7xi32>) -> i32
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
+  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
+  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
+  hlfir.destroy %11 : !hlfir.expr<7xi32>
+  %14 = fir.load %4#1 : !fir.ref<i32>
+  return %14 : i32
+}
+// CHECK-LABEL:  func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+// CHECK:    %[[V12:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
+// CHECK:      %[[V14:.*]] = hlfir.sum %[[V11]] : (!hlfir.expr<7xi32>) -> i32
+// CHECK:      %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
+// CHECK:      %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
+// CHECK:      %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
+// CHECK:      fir.result %[[V17]] : i32
+// CHECK:    }
+
+
+
+
+func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c7 = arith.constant 7 : index
+  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %6 = fir.load %2#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> i64
+  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
+  %10 = fir.load %5#0 : !fir.ref<i32>
+  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+    %16 = fir.load %15 : !fir.ref<i32>
+    %17 = arith.cmpi sge, %16, %10 : i32
+    %18 = fir.convert %17 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %18 : !fir.logical<4>
+  }
+  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %15 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
+    %16 = arith.cmpi sge, %15, %10 : i32
+    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %17 : !fir.logical<4>
+  }
+  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
+  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
+  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
+  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
+  %14 = fir.load %4#1 : !fir.ref<i32>
+  return %14 : i32
+}
+// CHECK-LABEL:  func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+// CHECK:    %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
+// CHECK:      %[[V13:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %c0_i32) -> (i32) {
+// CHECK:        %[[V17:.*]] = hlfir.designate %[[V9]] (%arg5)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
+// CHECK:        %[[V18:.*]] = fir.load %[[V17]] : !fir.ref<i32>
+// CHECK:        %[[V19:.*]] = arith.cmpi sge, %[[V18]], %[[V10]] : i32
+// CHECK:        %[[V20:.*]] = arith.addi %arg6, %c1_i32 : i32
+// CHECK:        %[[V21:.*]] = arith.select %[[V19]], %[[V20]], %arg6 : i32
+// CHECK:        fir.result %[[V21]] : i32
+// CHECK:      }
+// CHECK:      %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V10]] : i32
+// CHECK:      %[[V15:.*]] = arith.addi %arg4, %c1_i32 : i32
+// CHECK:      %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg4 : i32
+// CHECK:      fir.result %[[V16]] : i32
+// CHECK:    }
diff --git a/flang/test/HLFIR/maxloc-elemental.fir b/flang/test/HLFIR/maxloc-elemental.fir
new file mode 100644
index 0000000000000..c9210a59f0340
--- /dev/null
+++ b/flang/test/HLFIR/maxloc-elemental.fir
@@ -0,0 +1,133 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.load %2#0 : !fir.ref<i32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.cmpi sge, %9, %3 : i32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+// CHECK-NEXT:    %true = arith.constant true
+// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
+// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
+// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
+// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
+// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
+// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
+// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c-2147483648_i32) -> (i32) {
+// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
+// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
+// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi sgt, %[[V20]], %arg4 : i32
+// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
+// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
+// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
+// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
+// CHECK-NEXT:          fir.result %[[V20]] : i32
+// CHECK-NEXT:        } else {
+// CHECK-NEXT:          fir.result %arg4 : i32
+// CHECK-NEXT:        }
+// CHECK-NEXT:        fir.result %[[V22]] : i32
+// CHECK-NEXT:      } else {
+// CHECK-NEXT:        fir.result %arg4 : i32
+// CHECK-NEXT:      }
+// CHECK-NEXT:      fir.result %[[V18]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %3 = fir.load %2#0 : !fir.ref<f32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+    %9 = fir.load %8 : !fir.ref<f32>
+    %10 = arith.cmpf oge, %9, %3 : f32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL: _QPtest_float
+// CHECK:        %cst = arith.constant 0xFF800000 : f32
+// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
+// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
+// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
+// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
+// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
+// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf ogt, %[[V20]], %arg4 fastmath<contract> : f32
+// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
+// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
+// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
+// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
+// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
+// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
+// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
+// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
+// CHECK-NEXT:         fir.result %[[V20]] : f32
+// CHECK-NEXT:       } else {
+// CHECK-NEXT:         fir.result %arg4 : f32
+// CHECK-NEXT:       }
+// CHECK-NEXT:       fir.result %[[V22]] : f32
+// CHECK-NEXT:     } else {
+// CHECK-NEXT:       fir.result %arg4 : f32
+// CHECK-NEXT:     }
+// CHECK-NEXT:     fir.result %[[V18]] : f32
+// CHECK-NEXT:   }
+
diff --git a/flang/test/HLFIR/maxval-elemental.fir b/flang/test/HLFIR/maxval-elemental.fir
new file mode 100644
index 0000000000000..a21b4858412de
--- /dev/null
+++ b/flang/test/HLFIR/maxval-elemental.fir
@@ -0,0 +1,117 @@
+// Test maxval inlining for both elemental and designate
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+// subroutine test(array)
+//   integer :: array(:), x
+//   x = maxval(abs(array))
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+  %c31_i32 = arith.constant 31 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg1: index):
+    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.shrsi %9, %c31_i32 : i32
+    %11 = arith.xori %9, %10 : i32
+    %12 = arith.subi %11, %10 : i32
+    hlfir.yield_element %12 : i32
+  }
+  %7 = hlfir.maxval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
+  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
+// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c-2147483648_i32) -> (i32) {
+// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
+// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi sgt, %[[V10]], %arg2 : i32
+// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
+// CHECK-NEXT:      fir.result %[[V12]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// subroutine test(array)
+//   real :: array(:), x
+//   x = maxval(array(3:6))
+// end subroutine test
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
+  %c4 = arith.constant 4 : index
+  %c1 = arith.constant 1 : index
+  %c6 = arith.constant 6 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+  %6 = hlfir.maxval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
+  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
+  return
+}
+
+// CHECK-LABEL: _QPtest_float
+// CHECK:       %cst = arith.constant 0xFF800000 : f32
+// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
+// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
+// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
+// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
+// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
+// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
+// CHECK-NEXT:      fir.result %[[V14]] : f32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// Verify that lower bounds of designator are applied in the indexing inside
+// the generated loop (hlfir.designate takes indices relative to the base lower
+// bounds).
+func.func @component_lower_bounds(%arg0: !fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>) -> i32 {
+  %c10 = arith.constant 10 : index
+  %c101 = arith.constant 101 : index
+  %4 = fir.shape_shift %c101, %c10 : (index, index) -> !fir.shapeshift<1>
+  %5 = hlfir.designate %arg0{"i"}   shape %4 : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
+  %6 = hlfir.maxval %5 : (!fir.box<!fir.array<10xi32>>) -> i32
+  return %6 : i32
+}
+// CHECK-LABEL:   func.func @component_lower_bounds(
+// CHECK:  %[[VAL_1:.*]] = arith.constant 100 : index
+// CHECK:  %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:  %[[VAL_4:.*]] = arith.constant 10 : index
+// CHECK:  %[[VAL_5:.*]] = arith.constant 101 : index
+// CHECK:  %[[VAL_6:.*]] = fir.shape_shift %[[VAL_5]], %[[VAL_4]] : (index, index) -> !fir.shapeshift<1>
+// CHECK:  %[[VAL_7:.*]] = hlfir.designate %{{.*}}{"i"}   shape %[[VAL_6]] : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
+// CHECK:  %[[VAL_8:.*]] = fir.do_loop %[[VAL_9:.*]] = %[[VAL_2]] to %[[VAL_4]] {{.*}}
+// CHECK:    %[[VAL_11:.*]] = arith.addi %[[VAL_9]], %[[VAL_1]] : index
+// CHECK:    hlfir.designate %[[VAL_7]] (%[[VAL_11]])  : (!fir.box<!fir.array<10xi32>>, index) -> !fir.ref<i32>
diff --git a/flang/test/HLFIR/minloc-elemental.fir b/flang/test/HLFIR/minloc-elemental.fir
new file mode 100644
index 0000000000000..9453a335b4fbf
--- /dev/null
+++ b/flang/test/HLFIR/minloc-elemental.fir
@@ -0,0 +1,397 @@
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.load %2#0 : !fir.ref<i32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.cmpi sge, %9, %3 : i32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+// CHECK-NEXT:    %true = arith.constant true
+// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
+// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
+// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
+// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
+// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
+// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
+// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
+// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
+// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
+// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
+// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
+// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
+// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
+// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
+// CHECK-NEXT:          fir.result %[[V20]] : i32
+// CHECK-NEXT:        } else {
+// CHECK-NEXT:          fir.result %arg4 : i32
+// CHECK-NEXT:        }
+// CHECK-NEXT:        fir.result %[[V22]] : i32
+// CHECK-NEXT:      } else {
+// CHECK-NEXT:        fir.result %arg4 : i32
+// CHECK-NEXT:      }
+// CHECK-NEXT:      fir.result %[[V18]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+
+func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.load %2#0 : !fir.ref<i32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.cmpi sge, %9, %3 : i32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi16>, !fir.box<!fir.array<?xi16>>
+  hlfir.destroy %7 : !hlfir.expr<1xi16>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL:  func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
+// CHECK-NEXT:    %true = arith.constant true
+// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
+// CHECK-NEXT:    %c1_i16 = arith.constant 1 : i16
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c0_i16 = arith.constant 0 : i16
+// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i16
+// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
+// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK-NEXT:    fir.store %c0_i16 to %[[V8]] : !fir.ref<i16>
+// CHECK-NEXT:    fir.store %c0_i16 to %[[V0]] : !fir.ref<i16>
+// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
+// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
+// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
+// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
+// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i16>
+// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
+// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
+// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i16) -> i1
+// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
+// CHECK-NEXT:          fir.store %c1_i16 to %[[V0]] : !fir.ref<i16>
+// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i16
+// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i16>
+// CHECK-NEXT:          fir.result %[[V20]] : i32
+// CHECK-NEXT:        } else {
+// CHECK-NEXT:          fir.result %arg4 : i32
+// CHECK-NEXT:        }
+// CHECK-NEXT:        fir.result %[[V22]] : i32
+// CHECK-NEXT:      } else {
+// CHECK-NEXT:        fir.result %arg4 : i32
+// CHECK-NEXT:      }
+// CHECK-NEXT:      fir.result %[[V18]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi16>>, !fir.box<!fir.array<?xi16>>
+// CHECK-NEXT:    return
+
+
+func.func @_QPtest_kind2_convert(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = fir.load %2#0 : !fir.ref<i32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %10 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %11 = fir.load %10 : !fir.ref<i32>
+    %12 = arith.cmpi sge, %11, %3 : i32
+    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %13 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %9 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg3: index):
+    %10 = hlfir.apply %7, %arg3 : (!hlfir.expr<1xi16>, index) -> i16
+    %11 = fir.convert %10 : (i16) -> i32
+    hlfir.yield_element %11 : i32
+  }
+  hlfir.assign %9 to %1#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %9 : !hlfir.expr<?xi32>
+  hlfir.destroy %7 : !hlfir.expr<1xi16>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPtest_kind2_convert(
+// CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"},
+// CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "val"},
+// CHECK-SAME:                                     %[[VAL_2:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant false
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_10:.*]] = fir.alloca i16
+// CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_16]] : !fir.ref<i16>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<i16>
+// CHECK:           %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_12]]#0, %[[VAL_7]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#1, %[[VAL_9]] : index
+// CHECK:           %[[VAL_19:.*]] = fir.do_loop %[[VAL_20:.*]] = %[[VAL_7]] to %[[VAL_18]] step %[[VAL_9]] iter_args(%[[VAL_21:.*]] = %[[VAL_5]]) -> (i32) {
+// CHECK:             %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_9]] : index
+// CHECK:             %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<i32>
+// CHECK:             %[[VAL_25:.*]] = arith.cmpi sge, %[[VAL_24]], %[[VAL_15]] : i32
+// CHECK:             %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (i32) {
+// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_10]] : !fir.ref<i16>
+// CHECK:               %[[VAL_28:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_28]] : !fir.ref<i32>
+// CHECK:               %[[VAL_30:.*]] = arith.cmpi slt, %[[VAL_29]], %[[VAL_21]] : i32
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_27]] : (i16) -> i1
+// CHECK:               %[[VAL_32:.*]] = arith.xori %[[VAL_31]], %[[VAL_4]] : i1
+// CHECK:               %[[VAL_33:.*]] = arith.ori %[[VAL_30]], %[[VAL_32]] : i1
+// CHECK:               %[[VAL_34:.*]] = fir.if %[[VAL_33]] -> (i32) {
+// CHECK:                 fir.store %[[VAL_6]] to %[[VAL_10]] : !fir.ref<i16>
+// CHECK:                 %[[VAL_35:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_22]] : (index) -> i16
+// CHECK:                 fir.store %[[VAL_36]] to %[[VAL_35]] : !fir.ref<i16>
+// CHECK:                 fir.result %[[VAL_29]] : i32
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_21]] : i32
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_34]] : i32
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_21]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_26]] : i32
+// CHECK:           }
+// CHECK:           %[[VAL_37:.*]] = hlfir.as_expr %[[VAL_11]] move %[[VAL_3]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           fir.do_loop %[[VAL_38:.*]] = %[[VAL_9]] to %[[VAL_9]] step %[[VAL_9]] unordered {
+// CHECK:             %[[VAL_39:.*]] = hlfir.apply %[[VAL_37]], %[[VAL_38]] : (!hlfir.expr<1xi16>, index) -> i16
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i16) -> i32
+// CHECK:             %[[VAL_41:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_38]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
+  %c0 = arith.constant 0 : index
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %3 = fir.load %2#0 : !fir.ref<f32>
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+    %9 = fir.load %8 : !fir.ref<f32>
+    %10 = arith.cmpf oge, %9, %3 : f32
+    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %11 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL: _QPtest_float
+// CHECK:        %cst = arith.constant 0x7F800000 : f32
+// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
+// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
+// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
+// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
+// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
+// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
+// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
+// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf olt, %[[V20]], %arg4 fastmath<contract> : f32
+// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
+// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
+// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
+// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
+// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
+// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
+// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
+// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
+// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
+// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
+// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
+// CHECK-NEXT:         fir.result %[[V20]] : f32
+// CHECK-NEXT:       } else {
+// CHECK-NEXT:         fir.result %arg4 : f32
+// CHECK-NEXT:       }
+// CHECK-NEXT:       fir.result %[[V22]] : f32
+// CHECK-NEXT:     } else {
+// CHECK-NEXT:       fir.result %arg4 : f32
+// CHECK-NEXT:     }
+// CHECK-NEXT:     fir.result %[[V18]] : f32
+// CHECK-NEXT:   }
+
+
+func.func @_QPtest_assignshape(%arg0: !fir.ref<!fir.array<3x3xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.ref<!fir.array<3xi32>> {fir.bindc_name = "m"}) {
+  %c2 = arith.constant 2 : index
+  %c1 = arith.constant 1 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.shape %c3, %c3 : (index, index) -> !fir.shape<2>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFtestEarray"} : (!fir.ref<!fir.array<3x3xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x3xf32>>, !fir.ref<!fir.array<3x3xf32>>)
+  %2 = fir.shape %c3 : (index) -> !fir.shape<1>
+  %3:2 = hlfir.declare %arg2(%2) {uniq_name = "_QFtestEm"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+  %4:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %5 = fir.load %4#0 : !fir.ref<f32>
+  %6 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<3x3x!fir.logical<4>> {
+  ^bb0(%arg3: index, %arg4: index):
+    %10 = hlfir.designate %1#0 (%arg3, %arg4)  : (!fir.ref<!fir.array<3x3xf32>>, index, index) -> !fir.ref<f32>
+    %11 = fir.load %10 : !fir.ref<f32>
+    %12 = arith.cmpf oge, %11, %5 : f32
+    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %13 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %1#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.ref<!fir.array<3x3xf32>>, !hlfir.expr<3x3x!fir.logical<4>>) -> !hlfir.expr<2xi32>
+  %8 = fir.shape %c2 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %3#0 (%c1:%c2:%c1)  shape %8 : (!fir.ref<!fir.array<3xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<2xi32>>
+  hlfir.assign %7 to %9 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
+  hlfir.destroy %7 : !hlfir.expr<2xi32>
+  hlfir.destroy %6 : !hlfir.expr<3x3x!fir.logical<4>>
+  return
+}
+// Not supported as the input is not a box
+// CHECK-LABEL: _QPtest_assignshape
+// CHECK: hlfir.minloc
+
+
+func.func @_QFPtest_character(%arg0: !fir.box<!fir.array<?x!fir.char<1>>> {fir.bindc_name = "b"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0:2 = hlfir.declare %arg0 typeparams %c1 {uniq_name = "_QFFtestEb"} : (!fir.box<!fir.array<?x!fir.char<1>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1>>>, !fir.box<!fir.array<?x!fir.char<1>>>)
+  %1:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestEc"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca !fir.array<1xi32> {bindc_name = "m", uniq_name = "_QFFtestEm"}
+  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %2(%3) {uniq_name = "_QFFtestEm"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
+  %5 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
+  %6:2 = hlfir.declare %5 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %8 = fir.load %7#0 : !fir.ref<i32>
+  %9:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %10 = fir.shape %9#1 : (index) -> !fir.shape<1>
+  %11 = hlfir.elemental %10 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg3: index):
+    %16 = hlfir.designate %1#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %17 = fir.load %16 : !fir.ref<i32>
+    %18 = arith.cmpi eq, %17, %8 : i32
+    %19 = fir.convert %18 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %19 : !fir.logical<4>
+  }
+  %12 = hlfir.minloc %0#0 mask %11 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x!fir.char<1>>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  hlfir.assign %12 to %4#0 : !hlfir.expr<1xi32>, !fir.ref<!fir.array<1xi32>>
+  hlfir.destroy %12 : !hlfir.expr<1xi32>
+  hlfir.destroy %11 : !hlfir.expr<?x!fir.logical<4>>
+  %13 = hlfir.designate %4#0 (%c1)  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+  %14 = fir.load %13 : !fir.ref<i32>
+  hlfir.assign %14 to %6#0 : i32, !fir.ref<i32>
+  %15 = fir.load %6#1 : !fir.ref<i32>
+  return %15 : i32
+}
+// Characters are not supported at the moment
+// CHECK-LABEL: _QFPtest_character
+// CHECK: hlfir.minloc
+
+
+func.func @_QPtest_parts(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "mask"}) -> f32 {
+  %c1 = arith.constant 1 : index
+  %c5 = arith.constant 5 : index
+  %c0 = arith.constant 0 : index
+  %c5_i32 = arith.constant 5 : i32
+  %0:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEmask"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1 = fir.alloca f32 {bindc_name = "test", uniq_name = "_QFtestEtest"}
+  %2:2 = hlfir.declare %1 {uniq_name = "_QFtestEtest"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %3:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEx"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+  ^bb0(%arg2: index):
+    %11 = hlfir.designate %0#0 (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %12 = fir.load %11 : !fir.ref<i32>
+    %13 = arith.cmpi sge, %12, %c5_i32 : i32
+    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
+    hlfir.yield_element %14 : !fir.logical<4>
+  }
+  %7 = hlfir.minloc %3#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
+  %9 = hlfir.designate %3#0 (%c5:%c5:%c1)  shape %8 : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
+  hlfir.assign %7 to %9 : !hlfir.expr<1xi32>, !fir.box<!fir.array<1xi32>>
+  hlfir.destroy %7 : !hlfir.expr<1xi32>
+  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
+  %10 = fir.load %2#1 : !fir.ref<f32>
+  return %10 : f32
+}
+// Characters are not supported at the moment
+// CHECK-LABEL: _QPtest_parts
+// CHECK: fir.do_loop %{{.*}} = %c0 to %{{.*}} step %c1 iter_args(%{{.*}} = %c2147483647_i32) -> (i32) {
+
diff --git a/flang/test/HLFIR/minval-elemental.fir b/flang/test/HLFIR/minval-elemental.fir
new file mode 100644
index 0000000000000..64cd5403ec558
--- /dev/null
+++ b/flang/test/HLFIR/minval-elemental.fir
@@ -0,0 +1,95 @@
+// Test maxval inlining for both elemental and designate
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+// subroutine test(array)
+//   integer :: array(:), x
+//   x = minval(abs(array))
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+  %c31_i32 = arith.constant 31 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg1: index):
+    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.shrsi %9, %c31_i32 : i32
+    %11 = arith.xori %9, %10 : i32
+    %12 = arith.subi %11, %10 : i32
+    hlfir.yield_element %12 : i32
+  }
+  %7 = hlfir.minval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
+  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
+// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c2147483647_i32) -> (i32) {
+// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
+// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi slt, %[[V10]], %arg2 : i32
+// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
+// CHECK-NEXT:      fir.result %[[V12]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// subroutine test(array)
+//   real :: array(:), x
+//   x = minval(array(3:6))
+// end subroutine test
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
+  %c4 = arith.constant 4 : index
+  %c1 = arith.constant 1 : index
+  %c6 = arith.constant 6 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+  %6 = hlfir.minval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
+  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
+  return
+}
+
+// CHECK-LABEL: _QPtest_float
+// CHECK:       %cst = arith.constant 0x7F800000 : f32
+// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
+// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
+// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
+// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
+// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
+// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
+// CHECK-NEXT:      fir.result %[[V14]] : f32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
deleted file mode 100644
index fb0f39811ea74..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
+++ /dev/null
@@ -1,123 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
-  %0 = hlfir.all %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
-  return %0 : !fir.logical<4>
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant true
-// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
-// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
-// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
-// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_14:.*]] = arith.andi %[[VAL_13]], %[[VAL_11]] : i1
-// CHECK:               fir.result %[[VAL_14]] : i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_9]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
-// CHECK:           return %[[VAL_15]] : !fir.logical<4>
-// CHECK:         }
-
-func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.all %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
-  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant true
-// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
-// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
-// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
-// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
-// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_16:.*]] = arith.andi %[[VAL_15]], %[[VAL_13]] : i1
-// CHECK:               fir.result %[[VAL_16]] : i1
-// CHECK:             }
-// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
-// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
-// CHECK:           }
-// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
-// CHECK:         }
-
-func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
-  %0 = hlfir.all %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
-  return %0 : !fir.logical<4>
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant true
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
-// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
-// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
-// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
-// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
-// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
-// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_20]], %[[VAL_11]] : i1
-// CHECK:               fir.result %[[VAL_21]] : i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_9]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
-// CHECK:           return %[[VAL_22]] : !fir.logical<4>
-// CHECK:         }
-
-func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.all %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
-  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant true
-// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
-// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
-// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
-// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
-// CHECK:               %[[VAL_27:.*]] = arith.andi %[[VAL_26]], %[[VAL_14]] : i1
-// CHECK:               fir.result %[[VAL_27]] : i1
-// CHECK:             }
-// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
-// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
-// CHECK:           }
-// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
-// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
deleted file mode 100644
index 5bd76f3d24927..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
+++ /dev/null
@@ -1,123 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
-  %0 = hlfir.any %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
-  return %0 : !fir.logical<4>
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
-// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
-// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
-// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_14:.*]] = arith.ori %[[VAL_13]], %[[VAL_11]] : i1
-// CHECK:               fir.result %[[VAL_14]] : i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_9]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
-// CHECK:           return %[[VAL_15]] : !fir.logical<4>
-// CHECK:         }
-
-func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.any %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
-  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
-// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
-// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
-// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
-// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
-// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_16:.*]] = arith.ori %[[VAL_15]], %[[VAL_13]] : i1
-// CHECK:               fir.result %[[VAL_16]] : i1
-// CHECK:             }
-// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
-// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
-// CHECK:           }
-// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
-// CHECK:         }
-
-func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
-  %0 = hlfir.any %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
-  return %0 : !fir.logical<4>
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant false
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
-// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
-// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
-// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
-// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
-// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
-// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_21:.*]] = arith.ori %[[VAL_20]], %[[VAL_11]] : i1
-// CHECK:               fir.result %[[VAL_21]] : i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_9]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
-// CHECK:           return %[[VAL_22]] : !fir.logical<4>
-// CHECK:         }
-
-func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.any %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
-  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant false
-// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
-// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
-// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
-// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
-// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_14]] : i1
-// CHECK:               fir.result %[[VAL_27]] : i1
-// CHECK:             }
-// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
-// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
-// CHECK:           }
-// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
-// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
deleted file mode 100644
index 44594c646a368..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
+++ /dev/null
@@ -1,127 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
-  %0 = hlfir.count %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> i32
-  return %0 : i32
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (i32) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
-// CHECK:               %[[VAL_16:.*]] = arith.select %[[VAL_14]], %[[VAL_15]], %[[VAL_12]] : i32
-// CHECK:               fir.result %[[VAL_16]] : i32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : i32
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : i32
-// CHECK:         }
-
-func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.count %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?xi16>
-  return %0 : !hlfir.expr<?x?xi16>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i16
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]], %[[VAL_7]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi16> {
-// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (i16) {
-// CHECK:               %[[VAL_15:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_13]], %[[VAL_11]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
-// CHECK:               %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_14]], %[[VAL_1]] : i16
-// CHECK:               %[[VAL_18:.*]] = arith.select %[[VAL_16]], %[[VAL_17]], %[[VAL_14]] : i16
-// CHECK:               fir.result %[[VAL_18]] : i16
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_12]] : i16
-// CHECK:           }
-// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?xi16>
-// CHECK:         }
-
-func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
-  %0 = hlfir.count %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32
-  return %0 : i32
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (i32) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
-// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_15:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_16:.*]] = arith.addi %[[VAL_11]], %[[VAL_15]] : index
-// CHECK:               %[[VAL_17:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_18:.*]] = arith.addi %[[VAL_8]], %[[VAL_17]] : index
-// CHECK:               %[[VAL_19:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_16]], %[[VAL_18]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<!fir.logical<4>>
-// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
-// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_21]], %[[VAL_22]], %[[VAL_12]] : i32
-// CHECK:               fir.result %[[VAL_23]] : i32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : i32
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : i32
-// CHECK:         }
-
-func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.count %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?xi64>
-  return %0 : !hlfir.expr<?x?xi64>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i64
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i64
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_6]]#1, %[[VAL_8]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi64> {
-// CHECK:           ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):
-// CHECK:             %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_15:.*]] = %[[VAL_2]]) -> (i64) {
-// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:               %[[VAL_21:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
-// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_14]], %[[VAL_21]] : index
-// CHECK:               %[[VAL_23:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_4]] : index
-// CHECK:               %[[VAL_24:.*]] = arith.addi %[[VAL_12]], %[[VAL_23]] : index
-// CHECK:               %[[VAL_25:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<!fir.logical<2>>
-// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (!fir.logical<2>) -> i1
-// CHECK:               %[[VAL_28:.*]] = arith.addi %[[VAL_15]], %[[VAL_1]] : i64
-// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_15]] : i64
-// CHECK:               fir.result %[[VAL_29]] : i64
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_13]] : i64
-// CHECK:           }
-// CHECK:           return %[[VAL_10]] : !hlfir.expr<?x?xi64>
-// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
deleted file mode 100644
index 49e3bb6936338..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
+++ /dev/null
@@ -1,343 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
-  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_1d_total_expr(
-// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
-// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant -2147483648 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
-// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
-// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
-// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
-// CHECK:               %[[VAL_19:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_13]] : i32
-// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
-// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
-// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
-// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
-// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_26:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
-// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_28]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
-// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
-// CHECK:         }
-
-func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
-  %dim = arith.constant 1 : i16
-  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
-  return %0 : i32
-}
-// CHECK-LABEL:   func.func @test_1d_dim_expr(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
-// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
-// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
-// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
-// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
-// CHECK:               %[[VAL_18:.*]] = arith.cmpf ogt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
-// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
-// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
-// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
-// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
-// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
-// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
-// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_29:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (index) -> i32
-// CHECK:           return %[[VAL_30]] : i32
-// CHECK:         }
-
-func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
-  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  return %0 : !hlfir.expr<1xi16>
-}
-// CHECK-LABEL:   func.func @test_1d_total_var(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
-// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
-// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
-// CHECK:               %[[VAL_23:.*]] = arith.cmpf ogt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
-// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
-// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
-// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
-// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
-// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
-// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
-// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
-// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_34:.*]]#0, %[[VAL_6]] : i16
-// CHECK:           %[[VAL_35:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_34]]#0 : (i16) -> index
-// CHECK:           %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_35]]#0 : index
-// CHECK:           %[[VAL_38:.*]] = arith.subi %[[VAL_37]], %[[VAL_3]] : index
-// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_33]], %[[VAL_38]], %[[VAL_36]] : index
-// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (index) -> i16
-// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i16, !fir.ref<i16>
-// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
-// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
-// CHECK:         }
-
-func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
-  %dim = arith.constant 1 : i32
-  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
-  return %0 : i64
-}
-// CHECK-LABEL:   func.func @test_1d_dim_var(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant -1.7976931348623157E+308 : f64
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
-// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
-// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
-// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
-// CHECK:               %[[VAL_22:.*]] = arith.cmpf ogt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
-// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
-// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
-// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
-// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
-// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
-// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
-// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
-// CHECK:           }
-// CHECK:           %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_33:.*]]#0, %[[VAL_6]] : i64
-// CHECK:           %[[VAL_34:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]]#0 : (i64) -> index
-// CHECK:           %[[VAL_36:.*]] = arith.addi %[[VAL_35]], %[[VAL_34]]#0 : index
-// CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_36]], %[[VAL_3]] : index
-// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_32]], %[[VAL_37]], %[[VAL_35]] : index
-// CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
-// CHECK:           return %[[VAL_39]] : i64
-// CHECK:         }
-
-func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
-  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
-  return %0 : !hlfir.expr<3xi32>
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant false
-// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant true
-// CHECK:           %[[VAL_7:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
-// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
-// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
-// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
-// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
-// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
-// CHECK:                   %[[VAL_39:.*]] = arith.cmpf ogt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
-// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
-// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
-// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
-// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
-// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
-// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
-// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
-// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
-// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
-// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
-// CHECK:                 } else {
-// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
-// CHECK:                 }
-// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_56:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_57:.*]] = fir.convert %[[VAL_55]] : (index) -> i32
-// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_57]] to %[[VAL_58]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_59:.*]] = fir.convert %[[VAL_56]]#1 : (i32) -> index
-// CHECK:           %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (index) -> i32
-// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_60]] to %[[VAL_61]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_62:.*]] = fir.convert %[[VAL_56]]#2 : (i32) -> index
-// CHECK:           %[[VAL_63:.*]] = fir.convert %[[VAL_62]] : (index) -> i32
-// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_63]] to %[[VAL_64]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
-// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
-// CHECK:         }
-
-func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
-  return %0 : !hlfir.expr<?x?xi32>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant true
-// CHECK:           %[[VAL_4:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
-// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
-// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
-// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
-// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
-// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
-// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
-// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
-// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
-// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
-// CHECK:               } else {
-// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
-// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
-// CHECK:               }
-// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
-// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
-// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
-// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
-// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
-// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
-// CHECK:                 %[[VAL_48:.*]] = arith.cmpf ogt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
-// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
-// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
-// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
-// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
-// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
-// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
-// CHECK:             }
-// CHECK:             %[[VAL_58:.*]] = arith.cmpi ne, %[[VAL_59:.*]]#0, %[[VAL_5]] : i32
-// CHECK:             %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_59]]#0 : (i32) -> index
-// CHECK:             %[[VAL_62:.*]] = arith.addi %[[VAL_61]], %[[VAL_60]]#0 : index
-// CHECK:             %[[VAL_63:.*]] = arith.subi %[[VAL_62]], %[[VAL_7]] : index
-// CHECK:             %[[VAL_64:.*]] = arith.select %[[VAL_58]], %[[VAL_63]], %[[VAL_61]] : index
-// CHECK:             %[[VAL_65:.*]] = fir.convert %[[VAL_64]] : (index) -> i32
-// CHECK:             hlfir.yield_element %[[VAL_65]] : i32
-// CHECK:           }
-// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
-// CHECK:         }
-
-// Character comparisons are not supported yet.
-func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
-  %0 = hlfir.maxloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_character(
-// CHECK:           hlfir.maxloc
-
-// BACK is not supported yet.
-func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
-  %back = arith.constant true
-  %0 = hlfir.maxloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_back(
-// CHECK:           hlfir.maxloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
deleted file mode 100644
index 4022b7d3b2b77..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
+++ /dev/null
@@ -1,177 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
-  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
-  return %0 : f32
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant -3.40282347E+38 : f32
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (f32) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f32) {
-// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
-// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
-// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
-// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f32
-// CHECK:                 fir.result %[[VAL_22]] : f32
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f32
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_15]] : f32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : f32
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : f32
-// CHECK:         }
-
-func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
-  %dim = arith.constant 1 : i32
-  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
-  return %0 : !hlfir.expr<?xf64>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
-// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant -1.7976931348623157E+308 : f64
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
-// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f64) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f64) {
-// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
-// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
-// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
-// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f64
-// CHECK:                 fir.result %[[VAL_22]] : f64
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f64
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_15]] : f64
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_10]] : f64
-// CHECK:           }
-// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
-// CHECK:         }
-
-func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
-  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
-  return %0 : f16
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
-// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant -6.550400e+04 : f16
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (f16) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f16) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
-// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
-// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f16) {
-// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
-// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
-// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
-// CHECK:                 %[[VAL_25:.*]] = arith.cmpf ogt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
-// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
-// CHECK:                 %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_24]], %[[VAL_12]] : f16
-// CHECK:                 fir.result %[[VAL_30]] : f16
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f16
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_16]] : f16
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : f16
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : f16
-// CHECK:         }
-
-func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
-  return %0 : !hlfir.expr<?xf16>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant true
-// CHECK:           %[[VAL_3:.*]] = arith.constant -6.550400e+04 : f16
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
-// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
-// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (f16) {
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
-// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
-// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
-// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
-// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
-// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
-// CHECK:               } else {
-// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
-// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
-// CHECK:               }
-// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (f16) {
-// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
-// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
-// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
-// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
-// CHECK:                 %[[VAL_35:.*]] = arith.cmpf ogt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
-// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
-// CHECK:                 %[[VAL_40:.*]] = arith.select %[[VAL_39]], %[[VAL_34]], %[[VAL_14]] : f16
-// CHECK:                 fir.result %[[VAL_40]] : f16
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_14]] : f16
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_26]] : f16
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_12]] : f16
-// CHECK:           }
-// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
-// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
deleted file mode 100644
index 96dcebaf9a2cd..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
+++ /dev/null
@@ -1,343 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
-  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_1d_total_expr(
-// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
-// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
-// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
-// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
-// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
-// CHECK:               %[[VAL_19:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_13]] : i32
-// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
-// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
-// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
-// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
-// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_26:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
-// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_28]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
-// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
-// CHECK:         }
-
-func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
-  %dim = arith.constant 1 : i16
-  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
-  return %0 : i32
-}
-// CHECK-LABEL:   func.func @test_1d_dim_expr(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
-// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
-// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
-// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
-// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
-// CHECK:               %[[VAL_18:.*]] = arith.cmpf olt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
-// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
-// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
-// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
-// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
-// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
-// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
-// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_29:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (index) -> i32
-// CHECK:           return %[[VAL_30]] : i32
-// CHECK:         }
-
-func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
-  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  return %0 : !hlfir.expr<1xi16>
-}
-// CHECK-LABEL:   func.func @test_1d_total_var(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
-// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
-// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
-// CHECK:               %[[VAL_23:.*]] = arith.cmpf olt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
-// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
-// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
-// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
-// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
-// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
-// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
-// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
-// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_34:.*]]#0, %[[VAL_6]] : i16
-// CHECK:           %[[VAL_35:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_34]]#0 : (i16) -> index
-// CHECK:           %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_35]]#0 : index
-// CHECK:           %[[VAL_38:.*]] = arith.subi %[[VAL_37]], %[[VAL_3]] : index
-// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_33]], %[[VAL_38]], %[[VAL_36]] : index
-// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (index) -> i16
-// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i16, !fir.ref<i16>
-// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
-// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
-// CHECK:         }
-
-func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
-  %dim = arith.constant 1 : i32
-  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
-  return %0 : i64
-}
-// CHECK-LABEL:   func.func @test_1d_dim_var(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 1.7976931348623157E+308 : f64
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
-// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
-// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
-// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
-// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
-// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
-// CHECK:               %[[VAL_22:.*]] = arith.cmpf olt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
-// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
-// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
-// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
-// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
-// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
-// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
-// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
-// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
-// CHECK:           }
-// CHECK:           %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_33:.*]]#0, %[[VAL_6]] : i64
-// CHECK:           %[[VAL_34:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]]#0 : (i64) -> index
-// CHECK:           %[[VAL_36:.*]] = arith.addi %[[VAL_35]], %[[VAL_34]]#0 : index
-// CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_36]], %[[VAL_3]] : index
-// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_32]], %[[VAL_37]], %[[VAL_35]] : index
-// CHECK:           %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
-// CHECK:           return %[[VAL_39]] : i64
-// CHECK:         }
-
-func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
-  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
-  return %0 : !hlfir.expr<3xi32>
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant false
-// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant true
-// CHECK:           %[[VAL_7:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
-// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
-// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
-// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
-// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
-// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
-// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
-// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
-// CHECK:                   %[[VAL_39:.*]] = arith.cmpf olt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
-// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
-// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
-// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
-// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
-// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
-// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
-// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
-// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
-// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
-// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
-// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
-// CHECK:                 } else {
-// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
-// CHECK:                 }
-// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
-// CHECK:           }
-// CHECK:           %[[VAL_55:.*]] = fir.convert %[[VAL_56:.*]]#0 : (i32) -> index
-// CHECK:           %[[VAL_57:.*]] = fir.convert %[[VAL_55]] : (index) -> i32
-// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_57]] to %[[VAL_58]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_59:.*]] = fir.convert %[[VAL_56]]#1 : (i32) -> index
-// CHECK:           %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (index) -> i32
-// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_60]] to %[[VAL_61]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_62:.*]] = fir.convert %[[VAL_56]]#2 : (i32) -> index
-// CHECK:           %[[VAL_63:.*]] = fir.convert %[[VAL_62]] : (index) -> i32
-// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-// CHECK:           hlfir.assign %[[VAL_63]] to %[[VAL_64]] : i32, !fir.ref<i32>
-// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
-// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
-// CHECK:         }
-
-func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
-  return %0 : !hlfir.expr<?x?xi32>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant false
-// CHECK:           %[[VAL_3:.*]] = arith.constant true
-// CHECK:           %[[VAL_4:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
-// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
-// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
-// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
-// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
-// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
-// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
-// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
-// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
-// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
-// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
-// CHECK:               } else {
-// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
-// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
-// CHECK:               }
-// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
-// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
-// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
-// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
-// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
-// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
-// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
-// CHECK:                 %[[VAL_48:.*]] = arith.cmpf olt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
-// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
-// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
-// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
-// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
-// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
-// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
-// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
-// CHECK:             }
-// CHECK:             %[[VAL_58:.*]] = arith.cmpi ne, %[[VAL_59:.*]]#0, %[[VAL_5]] : i32
-// CHECK:             %[[VAL_60:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
-// CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_59]]#0 : (i32) -> index
-// CHECK:             %[[VAL_62:.*]] = arith.addi %[[VAL_61]], %[[VAL_60]]#0 : index
-// CHECK:             %[[VAL_63:.*]] = arith.subi %[[VAL_62]], %[[VAL_7]] : index
-// CHECK:             %[[VAL_64:.*]] = arith.select %[[VAL_58]], %[[VAL_63]], %[[VAL_61]] : index
-// CHECK:             %[[VAL_65:.*]] = fir.convert %[[VAL_64]] : (index) -> i32
-// CHECK:             hlfir.yield_element %[[VAL_65]] : i32
-// CHECK:           }
-// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
-// CHECK:         }
-
-// Character comparisons are not supported yet.
-func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
-  %0 = hlfir.minloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_character(
-// CHECK:           hlfir.minloc
-
-// BACK is not supported yet.
-func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
-  %back = arith.constant true
-  %0 = hlfir.minloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
-  return %0 : !hlfir.expr<1xi32>
-}
-// CHECK-LABEL:   func.func @test_back(
-// CHECK:           hlfir.minloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
deleted file mode 100644
index 4278ac7abd457..0000000000000
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
+++ /dev/null
@@ -1,177 +0,0 @@
-// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
-
-func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
-  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
-  return %0 : f32
-}
-// CHECK-LABEL:   func.func @test_total_expr(
-// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
-// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 3.40282347E+38 : f32
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (f32) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f32) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f32) {
-// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
-// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
-// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
-// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
-// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f32
-// CHECK:                 fir.result %[[VAL_22]] : f32
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f32
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_15]] : f32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : f32
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : f32
-// CHECK:         }
-
-func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
-  %dim = arith.constant 1 : i32
-  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
-  return %0 : !hlfir.expr<?xf64>
-}
-// CHECK-LABEL:   func.func @test_partial_expr(
-// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
-// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1.7976931348623157E+308 : f64
-// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
-// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
-// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]]) -> (f64) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
-// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (f64) {
-// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
-// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
-// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
-// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
-// CHECK:                 %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_16]], %[[VAL_12]] : f64
-// CHECK:                 fir.result %[[VAL_22]] : f64
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f64
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_15]] : f64
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_10]] : f64
-// CHECK:           }
-// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
-// CHECK:         }
-
-func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
-  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
-  return %0 : f16
-}
-// CHECK-LABEL:   func.func @test_total_var(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
-// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 6.550400e+04 : f16
-// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (f16) {
-// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (f16) {
-// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
-// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
-// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (f16) {
-// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
-// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
-// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
-// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
-// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
-// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
-// CHECK:                 %[[VAL_25:.*]] = arith.cmpf olt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
-// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
-// CHECK:                 %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_24]], %[[VAL_12]] : f16
-// CHECK:                 fir.result %[[VAL_30]] : f16
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_12]] : f16
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_16]] : f16
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_10]] : f16
-// CHECK:           }
-// CHECK:           return %[[VAL_7]] : f16
-// CHECK:         }
-
-func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
-  %dim = arith.constant 2 : i32
-  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
-  return %0 : !hlfir.expr<?xf16>
-}
-// CHECK-LABEL:   func.func @test_partial_var(
-// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
-// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant true
-// CHECK:           %[[VAL_3:.*]] = arith.constant 6.550400e+04 : f16
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
-// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
-// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
-// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (f16) {
-// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
-// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
-// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
-// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
-// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
-// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
-// CHECK:               } else {
-// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
-// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
-// CHECK:               }
-// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
-// CHECK:               %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (f16) {
-// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
-// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
-// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
-// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
-// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
-// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
-// CHECK:                 %[[VAL_35:.*]] = arith.cmpf olt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
-// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
-// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
-// CHECK:                 %[[VAL_40:.*]] = arith.select %[[VAL_39]], %[[VAL_34]], %[[VAL_14]] : f16
-// CHECK:                 fir.result %[[VAL_40]] : f16
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_14]] : f16
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_26]] : f16
-// CHECK:             }
-// CHECK:             hlfir.yield_element %[[VAL_12]] : f16
-// CHECK:           }
-// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
-// CHECK:         }

From feb1fb5f0473eb949b35fb25e15c4d32465cd6d7 Mon Sep 17 00:00:00 2001
From: Chandler Carruth <chandlerc@gmail.com>
Date: Thu, 17 Apr 2025 17:27:50 -0700
Subject: [PATCH 346/710] Make the `-disable-free` flag more full featured
 (#136213)

This lets us pass `-no-disable-free` to re-enable freeing memory for
example. This is especially helpful for library users of Clang where it
is important to not slowly leak memory.
---
 clang/include/clang/Driver/Options.td | 10 +++++++---
 clang/test/Driver/clang-translation.c |  4 ++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index e9acb20348654..830d3459a1320 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -7889,9 +7889,13 @@ def code_completion_with_fixits : Flag<["-"], "code-completion-with-fixits">,
 def skip_function_bodies : Flag<["-"], "skip-function-bodies">,
   HelpText<"Skip function bodies when possible">,
   MarshallingInfoFlag<FrontendOpts<"SkipFunctionBodies">>;
-def disable_free : Flag<["-"], "disable-free">,
-  HelpText<"Disable freeing of memory on exit">,
-  MarshallingInfoFlag<FrontendOpts<"DisableFree">>;
+defm disable_free : BoolOption<"",
+  "disable-free",
+  FrontendOpts<"DisableFree">,
+  DefaultFalse,
+  PosFlag<SetTrue, [], [ClangOption], "Disable">,
+  NegFlag<SetFalse, [], [ClangOption], "Don't disable">,
+  BothFlags<[], [ClangOption], " freeing of memory on exit">>;
 defm clear_ast_before_backend : BoolOption<"",
   "clear-ast-before-backend",
   CodeGenOpts<"ClearASTBeforeBackend">,
diff --git a/clang/test/Driver/clang-translation.c b/clang/test/Driver/clang-translation.c
index a7343ea18b213..7fc41ff46ed28 100644
--- a/clang/test/Driver/clang-translation.c
+++ b/clang/test/Driver/clang-translation.c
@@ -10,6 +10,10 @@
 // I386: "-o"
 // I386: clang-translation
 
+// RUN: %clang -target i386-unknown-unknown -### -S %s -o %t.s -Xclang -no-disable-free 2>&1 | FileCheck -check-prefix=FREE %s
+// FREE: "-disable-free"
+// FREE: "-no-disable-free"
+
 // RUN: %clang -target i386-unknown-unknown -### -S %s -fasynchronous-unwind-tables -fno-unwind-tables 2>&1 | FileCheck --check-prefix=UNWIND-TABLES %s --implicit-check-not=warning:
 // UNWIND-TABLES: "-funwind-tables=2"
 

From afc5cc060b8bc44b5fd3c0a4612ebc1a1dc70826 Mon Sep 17 00:00:00 2001
From: Michael Liao <michael.hliao@gmail.com>
Date: Thu, 17 Apr 2025 20:22:40 -0400
Subject: [PATCH 347/710] [yaml2obj] Fix shared build. NFC

---
 llvm/tools/yaml2obj/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/tools/yaml2obj/CMakeLists.txt b/llvm/tools/yaml2obj/CMakeLists.txt
index d1dea734a4e9a..4a10ec851587d 100644
--- a/llvm/tools/yaml2obj/CMakeLists.txt
+++ b/llvm/tools/yaml2obj/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  Object
   ObjectYAML
   Support
   )

From 0f5c66c364a6702edf65214a4f975c3e3f326d9a Mon Sep 17 00:00:00 2001
From: Michael Liao <michael.hliao@gmail.com>
Date: Thu, 17 Apr 2025 20:23:33 -0400
Subject: [PATCH 348/710] [OpenACC][CIR] Fix build error from CIR upstreaming

- 'DeviceTypeArgument' is defined as 'IdentifierLoc' instead of
  'std::pair<IdentifierInfo *, SourceLocation>'.
---
 clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index c14ff9a16841d..30e840cbfa1f7 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -124,7 +124,7 @@ class OpenACCClauseCIREmitter final
 
       for (const DeviceTypeArgument &arg : clause.getArchitectures()) {
         deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
-            builder.getContext(), decodeDeviceType(arg.first)));
+            builder.getContext(), decodeDeviceType(arg.getIdentifierInfo())));
       }
       operation.removeDeviceTypesAttr();
       operation.setDeviceTypesAttr(
@@ -135,7 +135,7 @@ class OpenACCClauseCIREmitter final
 
       if (!clause.getArchitectures().empty())
         operation.setDeviceType(
-            decodeDeviceType(clause.getArchitectures()[0].first));
+            decodeDeviceType(clause.getArchitectures()[0].getIdentifierInfo()));
     } else {
       return clauseNotImplemented(clause);
     }

From 1dbc8ef5538e25f61664d807fda236f7d983cc69 Mon Sep 17 00:00:00 2001
From: Oliver Hunt <oliver@apple.com>
Date: Thu, 17 Apr 2025 17:53:34 -0700
Subject: [PATCH 349/710] [clang][ptrauth] Make ptrauth feature detection tests
 more robust (#136204)

The existing test behavior checked for a warning being emitted under an
#if, but if the feature detection fails the #if fails and the warning is
not expected in the output.

I've made the test more explicit, and added comments to ensure no one
simply adds/moves any expected output around.
---
 clang/test/Sema/ptrauth-qualifier.c     | 8 +++++---
 clang/test/Sema/ptrauth.c               | 8 +++++---
 clang/test/SemaObjC/ptrauth-qualifier.m | 8 +++++---
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/clang/test/Sema/ptrauth-qualifier.c b/clang/test/Sema/ptrauth-qualifier.c
index 99d16b062ca6f..ab12acd8975f4 100644
--- a/clang/test/Sema/ptrauth-qualifier.c
+++ b/clang/test/Sema/ptrauth-qualifier.c
@@ -1,9 +1,11 @@
 // RUN: %clang_cc1 -triple arm64-apple-ios -std=c23 -fsyntax-only -verify -fptrauth-intrinsics %s
 // RUN: %clang_cc1 -triple aarch64-linux-gnu -std=c23 -fsyntax-only -verify -fptrauth-intrinsics %s
 
-#if __has_feature(ptrauth_qualifier)
-#warning __ptrauth qualifier enabled!
-// expected-warning@-1 {{__ptrauth qualifier enabled!}}
+#if !__has_extension(ptrauth_qualifier)
+// This error means that the __ptrauth qualifier availability test says  that it
+// is not available. This error is not expected in the output, if it is seen
+// there is a feature detection regression.
+#error __ptrauth qualifier not enabled
 #endif
 
 #if __aarch64__
diff --git a/clang/test/Sema/ptrauth.c b/clang/test/Sema/ptrauth.c
index fc1ae954fa36b..e3932615c2962 100644
--- a/clang/test/Sema/ptrauth.c
+++ b/clang/test/Sema/ptrauth.c
@@ -1,8 +1,10 @@
 // RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify -fptrauth-intrinsics %s -fexperimental-new-constant-interpreter
 
-#if __has_feature(ptrauth_intrinsics)
-#warning Pointer authentication enabled!
-// expected-warning@-1 {{Pointer authentication enabled!}}
+#if !__has_feature(ptrauth_intrinsics)
+// This error means that the __ptrauth intrinsics availability test says that
+// they are not available. This error is not expected in the output of this
+// test, if it is seen there is a feature detection regression.
+#error __ptrauth intrinsics not enabled
 #endif
 
 #if __aarch64__
diff --git a/clang/test/SemaObjC/ptrauth-qualifier.m b/clang/test/SemaObjC/ptrauth-qualifier.m
index 4836a653dd02f..98d0248f169be 100644
--- a/clang/test/SemaObjC/ptrauth-qualifier.m
+++ b/clang/test/SemaObjC/ptrauth-qualifier.m
@@ -1,9 +1,11 @@
 // RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify -fptrauth-intrinsics %s
 // RUN: %clang_cc1 -triple aarch64-linux-gnu -fsyntax-only -verify -fptrauth-intrinsics %s
 
-#if __has_feature(ptrauth_qualifier)
-#warning __ptrauth qualifier enabled!
-// expected-warning@-1 {{__ptrauth qualifier enabled!}}
+#if !__has_extension(ptrauth_qualifier)
+// This error means that the __ptrauth qualifier availability test says  that it
+// is not available. This error is not expected in the output, if it is seen
+// there is a feature detection regression.
+#error __ptrauth qualifier not enabled
 #endif
 
 @interface Foo

From ca9ec7dfc3a5ebad9e5c25b30511b2ed73287f61 Mon Sep 17 00:00:00 2001
From: Harald van Dijk <harald.vandijk@codeplay.com>
Date: Fri, 18 Apr 2025 02:11:02 +0100
Subject: [PATCH 350/710] [ARM, AArch64] Fix passing of structures with aligned
 base classes (#135564)

RecordLayout::UnadjustedAlignment was documented as "Maximum of the
alignments of the record members in characters", but
RecordLayout::getUnadjustedAlignment(), which just returns
UnadjustedAlignment, was documented as getting "the record alignment in
characters, before alignment adjustement." These are not the same thing:
the former excludes alignment of base classes, the latter takes it into
account. ItaniumRecordLayoutBuilder::LayoutBase was setting it according
to the former, but the AAPCS calling convention handling, currently the
only user, relies on it being set according to the latter.

Fixes #135551.
---
 clang/include/clang/AST/RecordLayout.h      |   7 +-
 clang/lib/AST/RecordLayoutBuilder.cpp       |   1 +
 clang/test/CodeGen/AArch64/args.cpp         | 111 +++++++++++++++++++-
 clang/test/CodeGen/aapcs64-align.cpp        |  95 -----------------
 clang/test/CodeGen/arm-vfp16-arguments2.cpp |   2 +-
 5 files changed, 115 insertions(+), 101 deletions(-)

diff --git a/clang/include/clang/AST/RecordLayout.h b/clang/include/clang/AST/RecordLayout.h
index dd18f9c49f84f..6cf08e76396e2 100644
--- a/clang/include/clang/AST/RecordLayout.h
+++ b/clang/include/clang/AST/RecordLayout.h
@@ -75,8 +75,9 @@ class ASTRecordLayout {
   // performance or backwards compatibility preserving (e.g. AIX-ABI).
   CharUnits PreferredAlignment;
 
-  // UnadjustedAlignment - Maximum of the alignments of the record members in
-  // characters.
+  // UnadjustedAlignment - Alignment of record in characters before alignment
+  // adjustments. Maximum of the alignments of the record members and base
+  // classes in characters.
   CharUnits UnadjustedAlignment;
 
   /// RequiredAlignment - The required alignment of the object.  In the MS-ABI
@@ -186,7 +187,7 @@ class ASTRecordLayout {
   CharUnits getPreferredAlignment() const { return PreferredAlignment; }
 
   /// getUnadjustedAlignment - Get the record alignment in characters, before
-  /// alignment adjustement.
+  /// alignment adjustment.
   CharUnits getUnadjustedAlignment() const { return UnadjustedAlignment; }
 
   /// getSize - Get the record size in characters.
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index ea353f88a8aec..dd932f6179f88 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -1302,6 +1302,7 @@ ItaniumRecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
     setSize(std::max(getSize(), Offset + Layout.getSize()));
 
   // Remember max struct/class alignment.
+  UnadjustedAlignment = std::max(UnadjustedAlignment, BaseAlign);
   UpdateAlignment(BaseAlign, UnpackedAlignTo, PreferredBaseAlign);
 
   return Offset;
diff --git a/clang/test/CodeGen/AArch64/args.cpp b/clang/test/CodeGen/AArch64/args.cpp
index 6f1f3d5e2a062..3cb62d3119ecf 100644
--- a/clang/test/CodeGen/AArch64/args.cpp
+++ b/clang/test/CodeGen/AArch64/args.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -triple arm64-apple-ios7.0 -target-abi darwinpcs -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,DARWIN
-// RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-llvm -o - -x c %s | FileCheck %s --check-prefixes=CHECK,C
-// RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CXX
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-llvm -o - -x c %s | FileCheck %s --check-prefixes=CHECK,C,AAPCS
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,CXX,AAPCS
 
 // Empty structs are ignored for PCS purposes on Darwin and in C mode elsewhere.
 // In C++ mode on ELF they consume a register slot though. Functions are
@@ -110,3 +110,110 @@ EXTERNC struct SortOfEmpty sort_of_empty_arg_variadic(int a, ...) {
   return b;
 }
 
+// Base case, nothing interesting.
+struct S {
+  long x, y;
+};
+
+// CHECK-LABEL: @g_S(
+// CHECK: call void @f_S(i64 noundef 1, [2 x i64] {{.*}})
+// CHECK: call void @fm_S(i64 noundef 1, i64 noundef 2, i64 noundef 3, i64 noundef 4, i64 noundef 5, [2 x i64] {{.*}})
+EXTERNC void f_S(long, struct S);
+EXTERNC void fm_S(long, long, long, long, long, struct S);
+EXTERNC void g_S() {
+  struct S s = {6, 7};
+  f_S(1, s);
+  fm_S(1, 2, 3, 4, 5, s);
+}
+
+// Aligned struct passed according to its natural alignment.
+struct __attribute__((aligned(16))) S16 {
+  long x, y;
+};
+
+// CHECK-LABEL: @g_S16(
+// DARWIN: call void @f_S16(i64 noundef 1, i128 {{.*}})
+// AAPCS: call void @f_S16(i64 noundef 1, [2 x i64] {{.*}})
+// DARWIN: call void @fm_S16(i64 noundef 1, i64 noundef 2, i64 noundef 3, i64 noundef 4, i64 noundef 5, i128 {{.*}})
+// AAPCS: call void @fm_S16(i64 noundef 1, i64 noundef 2, i64 noundef 3, i64 noundef 4, i64 noundef 5, [2 x i64] {{.*}})
+EXTERNC void f_S16(long, struct S16);
+EXTERNC void fm_S16(long, long, long, long, long, struct S16);
+EXTERNC void g_S16() {
+  struct S16 s = {6, 7};
+  f_S16(1, s);
+  fm_S16(1, 2, 3, 4, 5, s);
+}
+
+// Aligned struct with increased natural alignment through an aligned field.
+struct SF16 {
+  __attribute__((aligned(16))) long x;
+  long y;
+};
+
+// CHECK-LABEL: @g_SF16(
+// DARWIN: call void @f_SF16(i64 noundef 1, i128 {{.*}})
+// AAPCS: call void @f_SF16(i64 noundef 1, i128 {{.*}})
+// DARWIN: call void @fm_SF16(i64 noundef 1, i64 noundef 2, i64 noundef 3, i64 noundef 4, i64 noundef 5, i128 {{.*}})
+// AAPCS: call void @fm_SF16(i64 noundef 1, i64 noundef 2, i64 noundef 3, i64 noundef 4, i64 noundef 5, i128 {{.*}})
+EXTERNC void f_SF16(long, struct SF16);
+EXTERNC void fm_SF16(long, long, long, long, long, struct SF16);
+EXTERNC void g_SF16() {
+  struct SF16 s = {6, 7};
+  f_SF16(1, s);
+  fm_SF16(1, 2, 3, 4, 5, s);
+}
+
+#ifdef __cplusplus
+// Aligned struct with increased natural alignment through an aligned base class.
+struct SB16 : S16 {};
+
+// DARWIN-LABEL: @g_SB16(
+// CXX-LABEL: @g_SB16(
+// DARWIN: call void @f_SB16(i64 noundef 1, i128 {{.*}})
+// CXX: call void @f_SB16(i64 noundef 1, i128 {{.*}})
+// DARWIN: call void @fm_SB16(i64 noundef 1, i64 noundef 2, i64 noundef 3, i64 noundef 4, i64 noundef 5, i128 {{.*}})
+// CXX: call void @fm_SB16(i64 noundef 1, i64 noundef 2, i64 noundef 3, i64 noundef 4, i64 noundef 5, i128 {{.*}})
+EXTERNC void f_SB16(long, struct SB16);
+EXTERNC void fm_SB16(long, long, long, long, long, struct SB16);
+EXTERNC void g_SB16() {
+  struct SB16 s = {6, 7};
+  f_SB16(1, s);
+  fm_SB16(1, 2, 3, 4, 5, s);
+}
+#endif
+
+// Packed structure.
+struct  __attribute__((packed)) SP {
+  int x;
+  long y;
+};
+
+// CHECK-LABEL: @g_SP(
+// CHECK: call void @f_SP(i32 noundef 1, [2 x i64] {{.*}})
+// CHECK: call void @fm_SP(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] {{.*}})
+EXTERNC void f_SP(int, struct SP);
+EXTERNC void fm_SP(int, int, int, int, int, struct SP);
+EXTERNC void g_SP() {
+  struct SP s = {6, 7};
+  f_SP(1, s);
+  fm_SP(1, 2, 3, 4, 5, s);
+}
+
+// Packed structure, overaligned, same as above.
+struct  __attribute__((packed, aligned(16))) SP16 {
+  int x;
+  long y;
+};
+
+// CHECK-LABEL: @g_SP16(
+// DARWIN: call void @f_SP16(i32 noundef 1, i128 {{.*}})
+// AAPCS: call void @f_SP16(i32 noundef 1, [2 x i64] {{.*}})
+// DARWIN: call void @fm_SP16(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i128 {{.*}})
+// AAPCS: call void @fm_SP16(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] {{.*}})
+EXTERNC void f_SP16(int, struct SP16);
+EXTERNC void fm_SP16(int, int, int, int, int, struct SP16);
+EXTERNC void g_SP16() {
+  struct SP16 s = {6, 7};
+  f_SP16(1, s);
+  fm_SP16(1, 2, 3, 4, 5, s);
+}
diff --git a/clang/test/CodeGen/aapcs64-align.cpp b/clang/test/CodeGen/aapcs64-align.cpp
index e69faf231936c..1c26d68e434f4 100644
--- a/clang/test/CodeGen/aapcs64-align.cpp
+++ b/clang/test/CodeGen/aapcs64-align.cpp
@@ -12,101 +12,6 @@ extern "C" {
 // CHECK: @sizeof_RidiculouslyOverSizedBitfield ={{.*}} global i32 32
 // CHECK: @alignof_RidiculouslyOverSizedBitfield ={{.*}} global i32 16
 
-// Base case, nothing interesting.
-struct S {
-  long x, y;
-};
-
-void f0(long, S);
-void f0m(long, long, long, long, long, S);
-void g0() {
-  S s = {6, 7};
-  f0(1, s);
-  f0m(1, 2, 3, 4, 5, s);
-}
-// CHECK: define{{.*}} void @g0
-// CHECK: call void @f0(i64 noundef 1, [2 x i64] [i64 6, i64 7]
-// CHECK: call void @f0m{{.*}}[2 x i64] [i64 6, i64 7]
-// CHECK: declare void @f0(i64 noundef, [2 x i64])
-// CHECK: declare void @f0m(i64 noundef, i64 noundef, i64 noundef, i64 noundef, i64 noundef, [2 x i64])
-
-// Aligned struct, passed according to its natural alignment.
-struct __attribute__((aligned(16))) S16 {
-  long x, y;
-} s16;
-
-void f1(long, S16);
-void f1m(long, long, long, long, long, S16);
-void g1() {
-  S16 s = {6, 7};
-  f1(1, s);
-  f1m(1, 2, 3, 4, 5, s);
-}
-// CHECK: define{{.*}} void @g1
-// CHECK: call void @f1{{.*}}[2 x i64] [i64 6, i64 7]
-// CHECK: call void @f1m{{.*}}[2 x i64] [i64 6, i64 7]
-// CHECK: declare void @f1(i64 noundef, [2 x i64])
-// CHECK: declare void @f1m(i64 noundef, i64 noundef, i64 noundef, i64 noundef, i64 noundef, [2 x i64])
-
-// Increased natural alignment.
-struct SF16 {
-  long x __attribute__((aligned(16)));
-  long y;
-};
-
-void f3(long, SF16);
-void f3m(long, long, long, long, long, SF16);
-void g3() {
-  SF16 s = {6, 7};
-  f3(1, s);
-  f3m(1, 2, 3, 4, 5, s);
-}
-// CHECK: define{{.*}} void @g3
-// CHECK: call void @f3(i64 noundef 1, i128 129127208515966861318)
-// CHECK: call void @f3m(i64 noundef 1, i64 noundef 2, i64 noundef 3, i64 noundef 4, i64 noundef 5, i128 129127208515966861318)
-// CHECK: declare void @f3(i64 noundef, i128)
-// CHECK: declare void @f3m(i64 noundef, i64 noundef, i64 noundef, i64 noundef, i64 noundef, i128)
-
-
-// Packed structure.
-struct  __attribute__((packed)) P {
-  int x;
-  long u;
-};
-
-void f4(int, P);
-void f4m(int, int, int, int, int, P);
-void g4() {
-  P s = {6, 7};
-  f4(1, s);
-  f4m(1, 2, 3, 4, 5, s);
-}
-// CHECK: define{{.*}} void @g4()
-// CHECK: call void @f4(i32 noundef 1, [2 x i64] [i64 30064771078, i64 0])
-// CHECK: void @f4m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] [i64 30064771078, i64 0])
-// CHECK: declare void @f4(i32 noundef, [2 x i64])
-// CHECK: declare void @f4m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64])
-
-
-// Packed structure, overaligned, same as above.
-struct  __attribute__((packed, aligned(16))) P16 {
-  int x;
-  long y;
-};
-
-void f5(int, P16);
-void f5m(int, int, int, int, int, P16);
-  void g5() {
-    P16 s = {6, 7};
-    f5(1, s);
-    f5m(1, 2, 3, 4, 5, s);
-}
-// CHECK: define{{.*}} void @g5()
-// CHECK: call void @f5(i32 noundef 1, [2 x i64] [i64 30064771078, i64 0])
-// CHECK: void @f5m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] [i64 30064771078, i64 0])
-// CHECK: declare void @f5(i32 noundef, [2 x i64])
-// CHECK: declare void @f5m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64])
-
 //BitInt alignment
 struct BITINT129 {
     char ch;
diff --git a/clang/test/CodeGen/arm-vfp16-arguments2.cpp b/clang/test/CodeGen/arm-vfp16-arguments2.cpp
index 6e9a24e70c141..7273f64effa92 100644
--- a/clang/test/CodeGen/arm-vfp16-arguments2.cpp
+++ b/clang/test/CodeGen/arm-vfp16-arguments2.cpp
@@ -42,7 +42,7 @@ struct S5 : B1 {
 // CHECK-FULL: define{{.*}} arm_aapcs_vfpcc %struct.S1 @_Z2f12S1(%struct.S1 returned %s1.coerce)
 struct S1 f1(struct S1 s1) { return s1; }
 
-// CHECK-SOFT: define{{.*}} void @_Z2f22S2(ptr dead_on_unwind noalias writable writeonly sret(%struct.S2) align 8 captures(none) initializes((0, 16)) %agg.result, [4 x i32] %s2.coerce)
+// CHECK-SOFT: define{{.*}} void @_Z2f22S2(ptr dead_on_unwind noalias writable writeonly sret(%struct.S2) align 8 captures(none) initializes((0, 16)) %agg.result, [2 x i64] %s2.coerce)
 // CHECK-HARD: define{{.*}} arm_aapcs_vfpcc [2 x <2 x i32>] @_Z2f22S2([2 x <2 x i32>] returned %s2.coerce)
 // CHECK-FULL: define{{.*}} arm_aapcs_vfpcc %struct.S2 @_Z2f22S2(%struct.S2 %s2.coerce)
 struct S2 f2(struct S2 s2) { return s2; }

From 5e430afb84c3cbf285d8fe15c0ae23e0920e946b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 17 Apr 2025 18:55:20 -0700
Subject: [PATCH 351/710] [test] Remove CHECK lines for MCAsmStreamer's fixup
 output

As mentioned in #136088 , the fixup output is a debug aid and should not
be used to test target-specific relocation generation implementation.
The llvm-mc -filetype=obj output is what truly matters.
---
 .../MC/LoongArch/Relocations/relocations.s    | 72 +------------------
 llvm/test/MC/RISCV/fixups-binary-expression.s |  6 --
 llvm/test/MC/RISCV/function-call.s            | 11 ---
 llvm/test/MC/RISCV/linker-relaxation.s        | 40 -----------
 llvm/test/MC/RISCV/option-exact.s             | 12 ----
 llvm/test/MC/RISCV/pseudo-jump.s              |  5 --
 llvm/test/MC/RISCV/relocations.s              | 42 +----------
 llvm/test/MC/RISCV/tail-call.s                | 14 ----
 llvm/test/MC/RISCV/xqcilb-relocations.s       |  9 +--
 llvm/test/MC/RISCV/xqcili-relocations.s       |  9 +--
 10 files changed, 8 insertions(+), 212 deletions(-)

diff --git a/llvm/test/MC/LoongArch/Relocations/relocations.s b/llvm/test/MC/LoongArch/Relocations/relocations.s
index f91a941295d9e..bd8ecd085496b 100644
--- a/llvm/test/MC/LoongArch/Relocations/relocations.s
+++ b/llvm/test/MC/LoongArch/Relocations/relocations.s
@@ -1,11 +1,10 @@
-# RUN: llvm-mc --triple=loongarch64 < %s --show-encoding \
-# RUN:     | FileCheck --check-prefixes=INSTR,FIXUP %s
-# RUN: llvm-mc --filetype=obj --triple=loongarch64 < %s \
+# RUN: llvm-mc --triple=loongarch64 %s --show-encoding \
+# RUN:     | FileCheck --check-prefix=INSTR %s
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 %s \
 # RUN:     | llvm-readobj -r - | FileCheck --check-prefix=RELOC %s
 
 ## Check prefixes:
 ## RELOC - Check the relocation in the object.
-## FIXUP - Check the fixup on the instruction.
 ## INSTR - Check the instruction is handled properly by the ASMPrinter.
 
 .long foo
@@ -17,324 +16,259 @@
 bne $t1, $t2, %b16(foo)
 # RELOC: R_LARCH_B16
 # INSTR: bne $t1, $t2, %b16(foo)
-# FIXUP: fixup A - offset: 0, value: %b16(foo), kind: fixup_loongarch_b16
 
 bnez $t1, %b21(foo)
 # RELOC: R_LARCH_B21
 # INSTR: bnez $t1, %b21(foo)
-# FIXUP: fixup A - offset: 0, value: %b21(foo), kind: fixup_loongarch_b21
 
 bl %plt(foo)
 # RELOC: R_LARCH_B26
 # INSTR: bl %plt(foo)
-# FIXUP: fixup A - offset: 0, value: %plt(foo), kind: fixup_loongarch_b26
 
 bl foo
 # RELOC: R_LARCH_B26
 # INSTR: bl foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_loongarch_b26
 
 lu12i.w $t1, %abs_hi20(foo)
 # RELOC: R_LARCH_ABS_HI20 foo 0x0
 # INSTR: lu12i.w $t1, %abs_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %abs_hi20(foo), kind: fixup_loongarch_abs_hi20
 
 ori $t1, $t1, %abs_lo12(foo)
 # RELOC: R_LARCH_ABS_LO12 foo 0x0
 # INSTR: ori $t1, $t1, %abs_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %abs_lo12(foo), kind: fixup_loongarch_abs_lo12
 
 lu32i.d $t1, %abs64_lo20(foo)
 # RELOC: R_LARCH_ABS64_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %abs64_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %abs64_lo20(foo), kind: fixup_loongarch_abs64_lo20
 
 lu52i.d $t1, $t1, %abs64_hi12(foo)
 # RELOC: R_LARCH_ABS64_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %abs64_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %abs64_hi12(foo), kind: fixup_loongarch_abs64_hi12
 
 pcalau12i $t1, %pc_hi20(foo)
 # RELOC: R_LARCH_PCALA_HI20 foo 0x0
 # INSTR: pcalau12i $t1, %pc_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_hi20(foo), kind: FK_NONE
 
 pcalau12i $t1, %pc_hi20(foo+4)
 # RELOC: R_LARCH_PCALA_HI20 foo 0x4
 # INSTR: pcalau12i $t1, %pc_hi20(foo+4)
-# FIXUP: fixup A - offset: 0, value: %pc_hi20(foo+4), kind: FK_NONE
 
 addi.d $t1, $t1, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: addi.d  $t1, $t1, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
 
 addi.d $t1, $t1, %pc_lo12(foo+4)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x4
 # INSTR: addi.d  $t1, $t1, %pc_lo12(foo+4)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo+4), kind: FK_NONE
 
 jirl $zero, $t1, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: jirl $zero, $t1, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
 
 st.b $t1, $a2, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: st.b  $t1, $a2, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
 
 st.b $t1, $a2, %pc_lo12(foo+4)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x4
 # INSTR: st.b  $t1, $a2, %pc_lo12(foo+4)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo+4), kind: FK_NONE
 
 lu32i.d $t1, %pc64_lo20(foo)
 # RELOC: R_LARCH_PCALA64_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %pc64_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %pc64_lo20(foo), kind: FK_NONE
 
 lu52i.d $t1, $t1, %pc64_hi12(foo)
 # RELOC: R_LARCH_PCALA64_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %pc64_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc64_hi12(foo), kind: FK_NONE
 
 pcalau12i $t1, %got_pc_hi20(foo)
 # RELOC: R_LARCH_GOT_PC_HI20 foo 0x0
 # INSTR: pcalau12i $t1, %got_pc_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %got_pc_hi20(foo), kind: FK_NONE
 
 ld.d $t1, $a2, %got_pc_lo12(foo)
 # RELOC: R_LARCH_GOT_PC_LO12 foo 0x0
 # INSTR: ld.d  $t1, $a2, %got_pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %got_pc_lo12(foo), kind: FK_NONE
 
 lu32i.d $t1, %got64_pc_lo20(foo)
 # RELOC: R_LARCH_GOT64_PC_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %got64_pc_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %got64_pc_lo20(foo), kind: FK_NONE
 
 lu52i.d $t1, $t1, %got64_pc_hi12(foo)
 # RELOC: R_LARCH_GOT64_PC_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %got64_pc_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %got64_pc_hi12(foo), kind: FK_NONE
 
 lu12i.w $t1, %got_hi20(foo)
 # RELOC: R_LARCH_GOT_HI20 foo 0x0
 # INSTR: lu12i.w $t1, %got_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %got_hi20(foo), kind: FK_NONE
 
 ori $t1, $a2, %got_lo12(foo)
 # RELOC: R_LARCH_GOT_LO12 foo 0x0
 # INSTR: ori  $t1, $a2, %got_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %got_lo12(foo), kind: FK_NONE
 
 lu32i.d $t1, %got64_lo20(foo)
 # RELOC: R_LARCH_GOT64_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %got64_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %got64_lo20(foo), kind: FK_NONE
 
 lu52i.d $t1, $t1, %got64_hi12(foo)
 # RELOC: R_LARCH_GOT64_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %got64_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %got64_hi12(foo), kind: FK_NONE
 
 lu12i.w $t1, %le_hi20(foo)
 # RELOC: R_LARCH_TLS_LE_HI20 foo 0x0
 # INSTR: lu12i.w $t1, %le_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %le_hi20(foo), kind: fixup_loongarch_tls_le_hi20
 
 ori $t1, $a2, %le_lo12(foo)
 # RELOC: R_LARCH_TLS_LE_LO12 foo 0x0
 # INSTR: ori  $t1, $a2, %le_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %le_lo12(foo), kind: fixup_loongarch_tls_le_lo12
 
 lu32i.d $t1, %le64_lo20(foo)
 # RELOC: R_LARCH_TLS_LE64_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %le64_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %le64_lo20(foo), kind: fixup_loongarch_tls_le64_lo20
 
 lu52i.d $t1, $t1, %le64_hi12(foo)
 # RELOC: R_LARCH_TLS_LE64_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %le64_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %le64_hi12(foo), kind: fixup_loongarch_tls_le64_hi12
 
 pcalau12i $t1, %ie_pc_hi20(foo)
 # RELOC: R_LARCH_TLS_IE_PC_HI20 foo 0x0
 # INSTR: pcalau12i $t1, %ie_pc_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %ie_pc_hi20(foo), kind: FK_NONE
 
 ld.d $t1, $a2, %ie_pc_lo12(foo)
 # RELOC: R_LARCH_TLS_IE_PC_LO12 foo 0x0
 # INSTR: ld.d  $t1, $a2, %ie_pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %ie_pc_lo12(foo), kind: FK_NONE
 
 lu32i.d $t1, %ie64_pc_lo20(foo)
 # RELOC: R_LARCH_TLS_IE64_PC_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %ie64_pc_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %ie64_pc_lo20(foo), kind: FK_NONE
 
 lu52i.d $t1, $t1, %ie64_pc_hi12(foo)
 # RELOC: R_LARCH_TLS_IE64_PC_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %ie64_pc_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %ie64_pc_hi12(foo), kind: FK_NONE
 
 lu12i.w $t1, %ie_hi20(foo)
 # RELOC: R_LARCH_TLS_IE_HI20 foo 0x0
 # INSTR: lu12i.w $t1, %ie_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %ie_hi20(foo), kind: FK_NONE
 
 ori $t1, $a2, %ie_lo12(foo)
 # RELOC: R_LARCH_TLS_IE_LO12 foo 0x0
 # INSTR: ori  $t1, $a2, %ie_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %ie_lo12(foo), kind: FK_NONE
 
 lu32i.d $t1, %ie64_lo20(foo)
 # RELOC: R_LARCH_TLS_IE64_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %ie64_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %ie64_lo20(foo), kind: FK_NONE
 
 lu52i.d $t1, $t1, %ie64_hi12(foo)
 # RELOC: R_LARCH_TLS_IE64_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %ie64_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %ie64_hi12(foo), kind: FK_NONE
 
 pcalau12i $t1, %ld_pc_hi20(foo)
 # RELOC: R_LARCH_TLS_LD_PC_HI20 foo 0x0
 # INSTR: pcalau12i $t1, %ld_pc_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %ld_pc_hi20(foo), kind: FK_NONE
 
 lu12i.w $t1, %ld_hi20(foo)
 # RELOC: R_LARCH_TLS_LD_HI20 foo 0x0
 # INSTR: lu12i.w $t1, %ld_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %ld_hi20(foo), kind: FK_NONE
 
 pcalau12i $t1, %gd_pc_hi20(foo)
 # RELOC: R_LARCH_TLS_GD_PC_HI20 foo 0x0
 # INSTR: pcalau12i $t1, %gd_pc_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %gd_pc_hi20(foo), kind: FK_NONE
 
 lu12i.w $t1, %gd_hi20(foo)
 # RELOC: R_LARCH_TLS_GD_HI20 foo 0x0
 # INSTR: lu12i.w $t1, %gd_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %gd_hi20(foo), kind: FK_NONE
 
 pcaddu18i $t1, %call36(foo)
 # RELOC: R_LARCH_CALL36 foo 0x0
 # INSTR: pcaddu18i $t1, %call36(foo)
-# FIXUP: fixup A - offset: 0, value: %call36(foo), kind: FK_NONE
 
 pcalau12i $t1, %desc_pc_hi20(foo)
 # RELOC: R_LARCH_TLS_DESC_PC_HI20 foo 0x0
 # INSTR: pcalau12i $t1, %desc_pc_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %desc_pc_hi20(foo), kind: FK_NONE
 
 addi.d $t1, $t1, %desc_pc_lo12(foo)
 # RELOC: R_LARCH_TLS_DESC_PC_LO12 foo 0x0
 # INSTR: addi.d $t1, $t1, %desc_pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %desc_pc_lo12(foo), kind: FK_NONE
 
 lu32i.d $t1, %desc64_pc_lo20(foo)
 # RELOC: R_LARCH_TLS_DESC64_PC_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %desc64_pc_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %desc64_pc_lo20(foo), kind: FK_NONE
 
 lu52i.d $t1, $t1, %desc64_pc_hi12(foo)
 # RELOC: R_LARCH_TLS_DESC64_PC_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %desc64_pc_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %desc64_pc_hi12(foo), kind: FK_NONE
 
 ld.d $ra, $t1, %desc_ld(foo)
 # RELOC: R_LARCH_TLS_DESC_LD foo 0x0
 # INSTR: ld.d $ra, $t1, %desc_ld(foo)
-# FIXUP: fixup A - offset: 0, value: %desc_ld(foo), kind: FK_NONE
 
 jirl $ra, $ra, %desc_call(foo)
 # RELOC: R_LARCH_TLS_DESC_CALL foo 0x0
 # INSTR: jirl $ra, $ra, %desc_call(foo)
-# FIXUP: fixup A - offset: 0, value: %desc_call(foo), kind: FK_NONE
 
 lu12i.w $t1, %desc_hi20(foo)
 # RELOC: R_LARCH_TLS_DESC_HI20 foo 0x0
 # INSTR: lu12i.w $t1, %desc_hi20(foo)
-# FIXUP: fixup A - offset: 0, value: %desc_hi20(foo), kind: FK_NONE
 
 ori $t1, $t1, %desc_lo12(foo)
 # RELOC: R_LARCH_TLS_DESC_LO12 foo 0x0
 # INSTR: ori $t1, $t1, %desc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %desc_lo12(foo), kind: FK_NONE
 
 lu32i.d $t1, %desc64_lo20(foo)
 # RELOC: R_LARCH_TLS_DESC64_LO20 foo 0x0
 # INSTR: lu32i.d $t1, %desc64_lo20(foo)
-# FIXUP: fixup A - offset: 0, value: %desc64_lo20(foo), kind: FK_NONE
 
 lu52i.d $t1, $t1, %desc64_hi12(foo)
 # RELOC: R_LARCH_TLS_DESC64_HI12 foo 0x0
 # INSTR: lu52i.d $t1, $t1, %desc64_hi12(foo)
-# FIXUP: fixup A - offset: 0, value: %desc64_hi12(foo), kind: FK_NONE
 
 lu12i.w $t1, %le_hi20_r(foo)
 # RELOC: R_LARCH_TLS_LE_HI20_R foo 0x0
 # INSTR: lu12i.w $t1, %le_hi20_r(foo)
-# FIXUP: fixup A - offset: 0, value: %le_hi20_r(foo), kind: FK_NONE
 
 add.d $t1, $t2, $tp, %le_add_r(foo)
 # RELOC: R_LARCH_TLS_LE_ADD_R foo 0x0
 # INSTR: add.d $t1, $t2, $tp, %le_add_r(foo)
-# FIXUP: fixup A - offset: 0, value: %le_add_r(foo), kind: FK_NONE
 
 addi.d $t1, $a2, %le_lo12_r(foo)
 # RELOC: R_LARCH_TLS_LE_LO12_R foo 0x0
 # INSTR: addi.d $t1, $a2, %le_lo12_r(foo)
-# FIXUP: fixup A - offset: 0, value: %le_lo12_r(foo), kind: FK_NONE
 
 pcaddi $t1, %pcrel_20(foo)
 # RELOC: R_LARCH_PCREL20_S2 foo 0x0
 # INSTR: pcaddi $t1, %pcrel_20(foo)
-# FIXUP: fixup A - offset: 0, value: %pcrel_20(foo), kind: FK_NONE
 
 pcaddi $t1, %ld_pcrel_20(foo)
 # RELOC: R_LARCH_TLS_LD_PCREL20_S2 foo 0x0
 # INSTR: pcaddi $t1, %ld_pcrel_20(foo)
-# FIXUP: fixup A - offset: 0, value: %ld_pcrel_20(foo), kind: FK_NONE
 
 pcaddi $t1, %gd_pcrel_20(foo)
 # RELOC: R_LARCH_TLS_GD_PCREL20_S2 foo 0x0
 # INSTR: pcaddi $t1, %gd_pcrel_20(foo)
-# FIXUP: fixup A - offset: 0, value: %gd_pcrel_20(foo), kind: FK_NONE
 
 pcaddi $t1, %desc_pcrel_20(foo)
 # RELOC: R_LARCH_TLS_DESC_PCREL20_S2 foo 0x0
 # INSTR: pcaddi $t1, %desc_pcrel_20(foo)
-# FIXUP: fixup A - offset: 0, value: %desc_pcrel_20(foo), kind: FK_NONE
 
 fld.s $ft1, $a0, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: fld.s $ft1, $a0, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
 
 fst.d $ft1, $a0, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: fst.d $ft1, $a0, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
 
 vld $vr9, $a0, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: vld $vr9, $a0, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
 
 vst $vr9, $a0, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: vst $vr9, $a0, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
 
 xvld $xr9, $a0, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: xvld $xr9, $a0, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
 
 xvst $xr9, $a0, %pc_lo12(foo)
 # RELOC: R_LARCH_PCALA_LO12 foo 0x0
 # INSTR: xvst $xr9, $a0, %pc_lo12(foo)
-# FIXUP: fixup A - offset: 0, value: %pc_lo12(foo), kind: FK_NONE
diff --git a/llvm/test/MC/RISCV/fixups-binary-expression.s b/llvm/test/MC/RISCV/fixups-binary-expression.s
index 325a54ceeda9e..457d44968fd6e 100644
--- a/llvm/test/MC/RISCV/fixups-binary-expression.s
+++ b/llvm/test/MC/RISCV/fixups-binary-expression.s
@@ -1,5 +1,3 @@
-# RUN: llvm-mc -triple riscv32 -mattr=+c -M no-aliases < %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=CHECK-FIXUP %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c < %s \
 # RUN:     | llvm-objdump -M no-aliases -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INSTR %s
@@ -11,21 +9,17 @@
 .LBB1:
 
 jal zero, .LBB0+16
-# CHECK-FIXUP: fixup A - offset: 0, value: .LBB0+16, kind: fixup_riscv_jal
 # CHECK-INSTR: jal zero, 0x10
 # CHECK-RELOC:  R_RISCV_JAL
 
 beq a0, a1, .LBB1+32
-# CHECK-FIXUP: fixup A - offset: 0, value: .LBB1+32, kind: fixup_riscv_branch
 # CHECK-INSTR: beq a0, a1, 0x20
 # CHECK-RELOC:  R_RISCV_BRANCH
 
 c.j     .+32
-# CHECK:   fixup A - offset: 0, value: .Ltmp0+32, kind: fixup_riscv_rvc_jump
 # CHECK-INSTR: c.j   0x28
 # CHECK-RELOC:  R_RISCV_RVC_JUMP
 
 c.beqz a0, .-2
-# CHECK-FIXUP: fixup A - offset: 0, value: .Ltmp1-2, kind: fixup_riscv_rvc_branch
 # CHECK-INSTR: c.beqz a0, 0x8
 # CHECK-RELOC:  R_RISCV_RVC_BRANCH
diff --git a/llvm/test/MC/RISCV/function-call.s b/llvm/test/MC/RISCV/function-call.s
index 1521ae7e55e19..7e7a29dbc2d7e 100644
--- a/llvm/test/MC/RISCV/function-call.s
+++ b/llvm/test/MC/RISCV/function-call.s
@@ -2,8 +2,6 @@
 # RUN:     | llvm-objdump -d - | FileCheck --check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
 # RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
-# RUN: llvm-mc -triple riscv32 < %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=FIXUP %s
 
 .long foo
 
@@ -11,12 +9,10 @@ call foo
 # RELOC: R_RISCV_CALL_PLT foo 0x0
 # INSTR: auipc ra, 0
 # INSTR: jalr  ra
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_call
 call bar
 # RELOC: R_RISCV_CALL_PLT bar 0x0
 # INSTR: auipc ra, 0
 # INSTR: jalr  ra
-# FIXUP: fixup A - offset: 0, value: bar, kind: fixup_riscv_call
 
 # Ensure that calls to functions whose names coincide with register names work.
 
@@ -24,25 +20,21 @@ call zero
 # RELOC: R_RISCV_CALL_PLT zero 0x0
 # INSTR: auipc ra, 0
 # INSTR: jalr  ra
-# FIXUP: fixup A - offset: 0, value: zero, kind: fixup_riscv_call
 
 call f1
 # RELOC: R_RISCV_CALL_PLT f1 0x0
 # INSTR: auipc ra, 0
 # INSTR: jalr  ra
-# FIXUP: fixup A - offset: 0, value: f1, kind: fixup_riscv_call
 
 call ra
 # RELOC: R_RISCV_CALL_PLT ra 0x0
 # INSTR: auipc ra, 0
 # INSTR: jalr  ra
-# FIXUP: fixup A - offset: 0, value: ra, kind: fixup_riscv_call
 
 call mstatus
 # RELOC: R_RISCV_CALL_PLT mstatus 0x0
 # INSTR: auipc ra, 0
 # INSTR: jalr  ra
-# FIXUP: fixup A - offset: 0, value: mstatus, kind: fixup_riscv_call
 
 # Ensure that calls to procedure linkage table symbols work.
 
@@ -50,7 +42,6 @@ call foo@plt
 # RELOC: R_RISCV_CALL_PLT foo 0x0
 # INSTR: auipc ra, 0
 # INSTR: jalr  ra
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_call_plt
 
 # Ensure that an explicit register operand can be parsed.
 
@@ -58,10 +49,8 @@ call a0, foo
 # RELOC: R_RISCV_CALL_PLT foo 0x0
 # INSTR: auipc a0, 0
 # INSTR: jalr  a0
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_call
 
 call a0, foo@plt
 # RELOC: R_RISCV_CALL_PLT foo 0x0
 # INSTR: auipc a0, 0
 # INSTR: jalr  a0
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_call_plt
diff --git a/llvm/test/MC/RISCV/linker-relaxation.s b/llvm/test/MC/RISCV/linker-relaxation.s
index 6d0a05fb4267f..7ecc14cc779e4 100644
--- a/llvm/test/MC/RISCV/linker-relaxation.s
+++ b/llvm/test/MC/RISCV/linker-relaxation.s
@@ -2,14 +2,10 @@
 # RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX-RELOC %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=-relax < %s \
 # RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX-RELOC %s
-# RUN: llvm-mc -triple riscv32 -mattr=+relax < %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=RELAX-FIXUP %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+relax < %s \
 # RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX-RELOC %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=-relax < %s \
 # RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX-RELOC %s
-# RUN: llvm-mc -triple riscv64 -mattr=+relax < %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=RELAX-FIXUP %s
 
 .long foo
 
@@ -18,32 +14,24 @@ call foo
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_CALL_PLT foo 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_call
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 lui t1, %hi(foo)
 # NORELAX-RELOC: R_RISCV_HI20 foo 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_HI20 foo 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %hi(foo), kind: fixup_riscv_hi20
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 addi t1, t1, %lo(foo)
 # NORELAX-RELOC: R_RISCV_LO12_I foo 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_LO12_I foo 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %lo(foo), kind: fixup_riscv_lo12_i
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 sb t1, %lo(foo)(a2)
 # NORELAX-RELOC: R_RISCV_LO12_S foo 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_LO12_S foo 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %lo(foo), kind: fixup_riscv_lo12_s
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 1:
 auipc t1, %pcrel_hi(foo)
@@ -51,24 +39,18 @@ auipc t1, %pcrel_hi(foo)
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_PCREL_HI20 foo 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_hi(foo), kind: fixup_riscv_pcrel_hi20
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 addi t1, t1, %pcrel_lo(1b)
 # NORELAX-RELOC: R_RISCV_PCREL_LO12_I .Ltmp0 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_PCREL_LO12_I .Ltmp0 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_lo(.Ltmp0), kind: fixup_riscv_pcrel_lo12_i
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 sb t1, %pcrel_lo(1b)(a2)
 # NORELAX-RELOC: R_RISCV_PCREL_LO12_S .Ltmp0 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_PCREL_LO12_S .Ltmp0 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_lo(.Ltmp0), kind: fixup_riscv_pcrel_lo12_s
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 
 # Check behaviour when a locally defined symbol is referenced.
@@ -77,40 +59,30 @@ bar:
 beq s1, s1, bar
 # NORELAX-RELOC-NOT: R_RISCV_BRANCH
 # RELAX-RELOC: R_RISCV_BRANCH bar 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: bar, kind: fixup_riscv_branch
-# RELAX-FIXUP-NOT: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 call bar
 # NORELAX-RELOC-NOT: R_RISCV_CALL
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_CALL_PLT bar 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: bar, kind: fixup_riscv_call
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 lui t1, %hi(bar)
 # NORELAX-RELOC: R_RISCV_HI20 bar 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_HI20 bar 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %hi(bar), kind: fixup_riscv_hi20
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 addi t1, t1, %lo(bar)
 # NORELAX-RELOC: R_RISCV_LO12_I bar 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_LO12_I bar 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %lo(bar), kind: fixup_riscv_lo12_i
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 sb t1, %lo(bar)(a2)
 # NORELAX-RELOC: R_RISCV_LO12_S bar 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_LO12_S bar 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %lo(bar), kind: fixup_riscv_lo12_s
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 2:
 auipc t1, %pcrel_hi(bar)
@@ -118,47 +90,35 @@ auipc t1, %pcrel_hi(bar)
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_PCREL_HI20 bar 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_hi(bar), kind: fixup_riscv_pcrel_hi20
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 addi t1, t1, %pcrel_lo(2b)
 # NORELAX-RELOC-NOT: R_RISCV_PCREL_LO12_I
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_PCREL_LO12_I .Ltmp1 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_lo(.Ltmp1), kind: fixup_riscv_pcrel_lo12_i
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 sb t1, %pcrel_lo(2b)(a2)
 # NORELAX-RELOC-NOT: R_RISCV_PCREL_LO12_S
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_PCREL_LO12_S .Ltmp1 0x0
 # RELAX-RELOC: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_lo(.Ltmp1), kind: fixup_riscv_pcrel_lo12_s
-# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 ## %hi/%lo on an absolute symbol (not yet defined) leads to relocations when relaxation is enabled.
 lui t2, %hi(abs)
 # NORELAX-RELOC-NOT: R_RISCV_
 # RELAX-RELOC:      R_RISCV_HI20 - 0x12345
 # RELAX-RELOC-NEXT: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP:      fixup A - offset: 0, value: %hi(abs), kind: fixup_riscv_hi20
-# RELAX-FIXUP:      fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 addi t2, t2, %lo(abs)
 # NORELAX-RELOC-NOT: R_RISCV_
 # RELAX-RELOC:      R_RISCV_LO12_I - 0x12345
 # RELAX-RELOC-NEXT: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP:      fixup A - offset: 0, value: %lo(abs), kind: fixup_riscv_lo12_i
-# RELAX-FIXUP:      fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 .set abs, 0x12345
 
 lui t3, %hi(abs)
 # RELAX-RELOC:      R_RISCV_HI20 - 0x12345
 # RELAX-RELOC-NEXT: R_RISCV_RELAX - 0x0
-# RELAX-FIXUP:      fixup A - offset: 0, value: %hi(74565), kind: fixup_riscv_hi20
-# RELAX-FIXUP:      fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 
 # Check that a relocation is not emitted for a symbol difference which has
 # been folded to a fixup with an absolute value. This can happen when a
diff --git a/llvm/test/MC/RISCV/option-exact.s b/llvm/test/MC/RISCV/option-exact.s
index f90bd00621c42..83e25817bfc08 100644
--- a/llvm/test/MC/RISCV/option-exact.s
+++ b/llvm/test/MC/RISCV/option-exact.s
@@ -29,8 +29,6 @@ c.lw a0, 0(a0)
 
 # CHECK-ASM: call undefined
 # CHECK-ASM-SAME: # encoding: [0x97'A',A,A,A,0xe7'A',0x80'A',A,A]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_call_plt
-# CHECK-ASM-NEXT: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 # CHECK-OBJDUMP: auipc ra, 0
 # CHECK-OBJDUMP-NEXT: R_RISCV_CALL_PLT undefined
 # CHECK-OBJDUMP-NEXT: R_RISCV_RELAX *ABS*
@@ -39,7 +37,6 @@ call undefined@plt
 
 # CHECK-ASM: beq a0, a1, undefined
 # CHECK-ASM-SAME: # encoding: [0x63'A',A,0xb5'A',A]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_branch
 # CHECK-OBJDUMP: bne a0, a1, 0x14
 # CHECK-OBJDUMP-NEXT: jal zero, 0x10
 # CHECK-OBJDUMP-NEXT: R_RISCV_JAL undefined
@@ -47,7 +44,6 @@ beq a0, a1, undefined
 
 # CHECK-ASM: c.j undefined
 # CHECK-ASM-SAME: # encoding: [0bAAAAAA01,0b101AAAAA]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_rvc_jump
 # CHECK-OBJDUMP: jal zero, 0x14
 # CHECK-OBJDUMP-NEXT: R_RISCV_JAL undefined
 c.j undefined
@@ -67,8 +63,6 @@ c.lw a0, 0(a0)
 
 # CHECK-ASM: call undefined
 # CHECK-ASM-SAME: # encoding: [0x97'A',A,A,A,0xe7'A',0x80'A',A,A]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_call_plt
-# CHECK-ASM-NOT: fixup_riscv_relax
 # CHECK-OBJDUMP: auipc ra, 0
 # CHECK-OBJDUMP-NEXT: R_RISCV_CALL_PLT undefined
 # CHECK-OBJDUMP-NOT: R_RISCV_RELAX
@@ -77,14 +71,12 @@ call undefined@plt
 
 # CHECK-ASM: beq a0, a1, undefined
 # CHECK-ASM-SAME: # encoding: [0x63'A',A,0xb5'A',A]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_branch
 # CHECK-OBJDUMP: beq a0, a1, 0x26
 # CHECK-OBJDUMP-NEXT: R_RISCV_BRANCH undefined
 beq a0, a1, undefined
 
 # CHECK-ASM: c.j undefined
 # CHECK-ASM-SAME: # encoding: [0bAAAAAA01,0b101AAAAA]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_rvc_jump
 # CHECK-OBJDUMP: c.j 0x2a
 # CHECK-OBJDUMP-NEXT: R_RISCV_RVC_JUMP undefined
 c.j undefined
@@ -104,8 +96,6 @@ c.lw a0, 0(a0)
 
 # CHECK-ASM: call undefined
 # CHECK-ASM-SAME: # encoding: [0x97'A',A,A,A,0xe7'A',0x80'A',A,A]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_call_plt
-# CHECK-ASM-NEXT: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 # CHECK-OBJDUMP: auipc ra, 0
 # CHECK-OBJDUMP-NEXT: R_RISCV_CALL_PLT undefined
 # CHECK-OBJDUMP-NEXT: R_RISCV_RELAX *ABS*
@@ -114,7 +104,6 @@ call undefined@plt
 
 # CHECK-ASM: beq a0, a1, undefined
 # CHECK-ASM-SAME: # encoding: [0x63'A',A,0xb5'A',A]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_branch
 # CHECK-OBJDUMP: bne a0, a1, 0x40
 # CHECK-OBJDUMP-NEXT: jal zero, 0x3c
 # CHECK-OBJDUMP-NEXT: R_RISCV_JAL undefined
@@ -122,7 +111,6 @@ beq a0, a1, undefined
 
 # CHECK-ASM: c.j undefined
 # CHECK-ASM-SAME: # encoding: [0bAAAAAA01,0b101AAAAA]
-# CHECK-ASM-NEXT: fixup A - offset: 0, value: undefined, kind: fixup_riscv_rvc_jump
 # CHECK-OBJDUMP: jal zero, 0x40
 # CHECK-OBJDUMP-NEXT: R_RISCV_JAL undefined
 c.j undefined
diff --git a/llvm/test/MC/RISCV/pseudo-jump.s b/llvm/test/MC/RISCV/pseudo-jump.s
index 8c72547c5bcc2..644dee0945f64 100644
--- a/llvm/test/MC/RISCV/pseudo-jump.s
+++ b/llvm/test/MC/RISCV/pseudo-jump.s
@@ -2,8 +2,6 @@
 # RUN:     | llvm-objdump -d - | FileCheck --check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
 # RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
-# RUN: llvm-mc -triple riscv32 < %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=FIXUP %s
 
 .long foo
 
@@ -11,7 +9,6 @@ jump foo, x31
 # RELOC: R_RISCV_CALL_PLT foo 0x0
 # INSTR: auipc t6, 0
 # INSTR: jr  t6
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_call
 
 # Ensure that jumps to symbols whose names coincide with register names work.
 
@@ -19,10 +16,8 @@ jump zero, x1
 # RELOC: R_RISCV_CALL_PLT zero 0x0
 # INSTR: auipc ra, 0
 # INSTR: ret
-# FIXUP: fixup A - offset: 0, value: zero, kind: fixup_riscv_call
 
 1:
 jump 1b, x31
 # INSTR: auipc t6, 0
 # INSTR: jr  t6
-# FIXUP: fixup A - offset: 0, value: .Ltmp0, kind: fixup_riscv_call
diff --git a/llvm/test/MC/RISCV/relocations.s b/llvm/test/MC/RISCV/relocations.s
index 85a25fee118ed..e1d56885aaf3b 100644
--- a/llvm/test/MC/RISCV/relocations.s
+++ b/llvm/test/MC/RISCV/relocations.s
@@ -1,11 +1,10 @@
-# RUN: llvm-mc -triple riscv32 -M no-aliases < %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
-# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c < %s \
+# RUN: llvm-mc -triple riscv32 -M no-aliases %s -show-encoding \
+# RUN:     | FileCheck -check-prefix=INSTR %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c %s \
 # RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 
 # Check prefixes:
 # RELOC - Check the relocation in the object.
-# FIXUP - Check the fixup on the instruction.
 # INSTR - Check the instruction is handled properly by the ASMPrinter
 
 .long foo
@@ -17,99 +16,80 @@
 lui t1, %hi(foo)
 # RELOC: R_RISCV_HI20 foo 0x0
 # INSTR: lui t1, %hi(foo)
-# FIXUP: fixup A - offset: 0, value: %hi(foo), kind: fixup_riscv_hi20
 
 lui t1, %hi(foo+4)
 # RELOC: R_RISCV_HI20 foo 0x4
 # INSTR: lui t1, %hi(foo+4)
-# FIXUP: fixup A - offset: 0, value: %hi(foo+4), kind: fixup_riscv_hi20
 
 lui t1, %tprel_hi(foo)
 # RELOC: R_RISCV_TPREL_HI20 foo 0x0
 # INSTR: lui t1, %tprel_hi(foo)
-# FIXUP: fixup A - offset: 0, value: %tprel_hi(foo), kind: fixup_riscv_tprel_hi20
 
 lui t1, %tprel_hi(foo+4)
 # RELOC: R_RISCV_TPREL_HI20 foo 0x4
 # INSTR: lui t1, %tprel_hi(foo+4)
-# FIXUP: fixup A - offset: 0, value: %tprel_hi(foo+4), kind: fixup_riscv_tprel_hi20
 
 addi t1, t1, %lo(foo)
 # RELOC: R_RISCV_LO12_I foo 0x0
 # INSTR: addi t1, t1, %lo(foo)
-# FIXUP: fixup A - offset: 0, value: %lo(foo), kind: fixup_riscv_lo12_i
 
 addi t1, t1, %lo(foo+4)
 # RELOC: R_RISCV_LO12_I foo 0x4
 # INSTR: addi t1, t1, %lo(foo+4)
-# FIXUP: fixup A - offset: 0, value: %lo(foo+4), kind: fixup_riscv_lo12_i
 
 addi t1, t1, %tprel_lo(foo)
 # RELOC: R_RISCV_TPREL_LO12_I foo 0x0
 # INSTR: addi t1, t1, %tprel_lo(foo)
-# FIXUP: fixup A - offset: 0, value: %tprel_lo(foo), kind: fixup_riscv_tprel_lo12_i
 
 addi t1, t1, %tprel_lo(foo+4)
 # RELOC: R_RISCV_TPREL_LO12_I foo 0x4
 # INSTR: addi t1, t1, %tprel_lo(foo+4)
-# FIXUP: fixup A - offset: 0, value: %tprel_lo(foo+4), kind: fixup_riscv_tprel_lo12_i
 
 sb t1, %lo(foo)(a2)
 # RELOC: R_RISCV_LO12_S foo 0x0
 # INSTR: sb t1, %lo(foo)(a2)
-# FIXUP: fixup A - offset: 0, value: %lo(foo), kind: fixup_riscv_lo12_s
 
 sb t1, %lo(foo+4)(a2)
 # RELOC: R_RISCV_LO12_S foo 0x4
 # INSTR: sb t1, %lo(foo+4)(a2)
-# FIXUP: fixup A - offset: 0, value: %lo(foo+4), kind: fixup_riscv_lo12_s
 
 sb t1, %tprel_lo(foo)(a2)
 # RELOC: R_RISCV_TPREL_LO12_S foo 0x0
 # INSTR: sb t1, %tprel_lo(foo)(a2)
-# FIXUP: fixup A - offset: 0, value: %tprel_lo(foo), kind: fixup_riscv_tprel_lo12_s
 
 sb t1, %tprel_lo(foo+4)(a2)
 # RELOC: R_RISCV_TPREL_LO12_S foo 0x4
 # INSTR: sb t1, %tprel_lo(foo+4)(a2)
-# FIXUP: fixup A - offset: 0, value: %tprel_lo(foo+4), kind: fixup_riscv_tprel_lo12_s
 
 .L0:
 auipc t1, %pcrel_hi(foo)
 # RELOC: R_RISCV_PCREL_HI20 foo 0x0
 # INSTR: auipc t1, %pcrel_hi(foo)
-# FIXUP: fixup A - offset: 0, value: %pcrel_hi(foo), kind: fixup_riscv_pcrel_hi20
 
 auipc t1, %pcrel_hi(foo+4)
 # RELOC: R_RISCV_PCREL_HI20 foo 0x4
 # INSTR: auipc t1, %pcrel_hi(foo+4)
-# FIXUP: fixup A - offset: 0, value: %pcrel_hi(foo+4), kind: fixup_riscv_pcrel_hi20
 
 addi t1, t1, %pcrel_lo(.L0)
 # RELOC: R_RISCV_PCREL_LO12_I .L0 0x0
 # INSTR: addi t1, t1, %pcrel_lo(.L0)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L0), kind: fixup_riscv_pcrel_lo12_i
 
 sb t1, %pcrel_lo(.L0)(a2)
 # RELOC: R_RISCV_PCREL_LO12_S .L0 0x0
 # INSTR: sb t1, %pcrel_lo(.L0)(a2)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L0), kind: fixup_riscv_pcrel_lo12_s
 
 .L1:
 auipc t1, %got_pcrel_hi(foo)
 # RELOC: R_RISCV_GOT_HI20 foo 0x0
 # INSTR: auipc t1, %got_pcrel_hi(foo)
-# FIXUP: fixup A - offset: 0, value: %got_pcrel_hi(foo), kind: fixup_riscv_got_hi20
 
 addi t1, t1, %pcrel_lo(.L1)
 # RELOC: R_RISCV_PCREL_LO12_I .L1 0x0
 # INSTR: addi t1, t1, %pcrel_lo(.L1)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L1), kind: fixup_riscv_pcrel_lo12_i
 
 sb t1, %pcrel_lo(.L1)(a2)
 # RELOC: R_RISCV_PCREL_LO12_S .L1 0x0
 # INSTR: sb t1, %pcrel_lo(.L1)(a2)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L1), kind: fixup_riscv_pcrel_lo12_s
 
 # Check that GOT relocations aren't evaluated to a constant when the symbol is
 # in the same object file.
@@ -117,83 +97,67 @@ sb t1, %pcrel_lo(.L1)(a2)
 auipc t1, %got_pcrel_hi(.L1)
 # RELOC: R_RISCV_GOT_HI20 .L1 0x0
 # INSTR: auipc t1, %got_pcrel_hi(.L1)
-# FIXUP: fixup A - offset: 0, value: %got_pcrel_hi(.L1), kind: fixup_riscv_got_hi20
 
 addi t1, t1, %pcrel_lo(.L2)
 # RELOC: R_RISCV_PCREL_LO12_I .L2 0x0
 # INSTR: addi t1, t1, %pcrel_lo(.L2)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L2), kind: fixup_riscv_pcrel_lo12_i
 
 sb t1, %pcrel_lo(.L2)(a2)
 # RELOC: R_RISCV_PCREL_LO12_S .L2 0x0
 # INSTR: sb t1, %pcrel_lo(.L2)(a2)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L2), kind: fixup_riscv_pcrel_lo12_s
 
 .L3:
 auipc t1, %tls_ie_pcrel_hi(foo)
 # RELOC: R_RISCV_TLS_GOT_HI20 foo 0x0
 # INSTR: auipc t1, %tls_ie_pcrel_hi(foo)
-# FIXUP: fixup A - offset: 0, value: %tls_ie_pcrel_hi(foo), kind: fixup_riscv_tls_got_hi20
 
 addi t1, t1, %pcrel_lo(.L3)
 # RELOC: R_RISCV_PCREL_LO12_I .L3 0x0
 # INSTR: addi t1, t1, %pcrel_lo(.L3)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L3), kind: fixup_riscv_pcrel_lo12_i
 
 sb t1, %pcrel_lo(.L3)(a2)
 # RELOC: R_RISCV_PCREL_LO12_S .L3 0x0
 # INSTR: sb t1, %pcrel_lo(.L3)(a2)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L3), kind: fixup_riscv_pcrel_lo12_s
 
 .L4:
 auipc t1, %tls_gd_pcrel_hi(foo)
 # RELOC: R_RISCV_TLS_GD_HI20 foo 0x0
 # INSTR: auipc t1, %tls_gd_pcrel_hi(foo)
-# FIXUP: fixup A - offset: 0, value: %tls_gd_pcrel_hi(foo), kind: fixup_riscv_tls_gd_hi20
 
 addi t1, t1, %pcrel_lo(.L4)
 # RELOC: R_RISCV_PCREL_LO12_I .L4 0x0
 # INSTR: addi t1, t1, %pcrel_lo(.L4)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L4), kind: fixup_riscv_pcrel_lo12_i
 
 sb t1, %pcrel_lo(.L4)(a2)
 # RELOC: R_RISCV_PCREL_LO12_S .L4 0x0
 # INSTR: sb t1, %pcrel_lo(.L4)(a2)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L4), kind: fixup_riscv_pcrel_lo12_s
 
 add t1, t1, tp, %tprel_add(foo)
 # RELOC: R_RISCV_TPREL_ADD foo 0x0
 # INSTR: add t1, t1, tp, %tprel_add(foo)
-# FIXUP: fixup A - offset: 0, value: %tprel_add(foo), kind: fixup_riscv_tprel_add
 
 jal zero, foo
 # RELOC: R_RISCV_JAL
 # INSTR: jal zero, foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_jal
 
 # Since foo is undefined, this will be relaxed to (bltu; jal)
 bgeu a0, a1, foo
 # RELOC: R_RISCV_JAL
 # INSTR: bgeu a0, a1, foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_branch
 
 .L5:
 auipc a0, %tlsdesc_hi(a_symbol)
 # RELOC: R_RISCV_TLSDESC_HI20
 # INSTR: auipc a0, %tlsdesc_hi(a_symbol)
-# FIXUP: fixup A - offset: 0, value: %tlsdesc_hi(a_symbol), kind: fixup_riscv_tlsdesc_hi20
 
 lw a1, %tlsdesc_load_lo(.L5)(a0)
 # RELOC: R_RISCV_TLSDESC_LOAD_LO12
 # INSTR: lw a1, %tlsdesc_load_lo(.L5)(a0)
-# FIXUP: fixup A - offset: 0, value: %tlsdesc_load_lo(.L5), kind: fixup_riscv_tlsdesc_load_lo12
 
 addi a0, a0, %tlsdesc_add_lo(.L5)
 # RELOC: R_RISCV_TLSDESC_ADD_LO12
 # INSTR: addi a0, a0, %tlsdesc_add_lo(.L5)
-# FIXUP: fixup A - offset: 0, value: %tlsdesc_add_lo(.L5), kind: fixup_riscv_tlsdesc_add_lo12
 
 jalr t0, 0(a1), %tlsdesc_call(.L5)
 # RELOC: R_RISCV_TLSDESC_CALL
 # INSTR: jalr t0, 0(a1), %tlsdesc_call(.L5)
-# FIXUP: fixup A - offset: 0, value: %tlsdesc_call(.L5), kind: fixup_riscv_tlsdesc_call
diff --git a/llvm/test/MC/RISCV/tail-call.s b/llvm/test/MC/RISCV/tail-call.s
index 7c9f28bdfacda..1c55bf5223caa 100644
--- a/llvm/test/MC/RISCV/tail-call.s
+++ b/llvm/test/MC/RISCV/tail-call.s
@@ -2,29 +2,21 @@
 # RUN:   | llvm-objdump -d - | FileCheck --check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
 # RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
-# RUN: llvm-mc -triple riscv32 < %s -show-encoding \
-# RUN:   | FileCheck -check-prefix=FIXUP %s
 
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
 # RUN:   | llvm-objdump -d - | FileCheck --check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
 # RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
-# RUN: llvm-mc -triple riscv64 < %s -show-encoding \
-# RUN:   | FileCheck -check-prefix=FIXUP %s
 
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-zicfilp < %s \
 # RUN:   | llvm-objdump -d - | FileCheck --check-prefix=INSTR-ZICFILP %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-zicfilp < %s \
 # RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
-# RUN: llvm-mc -triple riscv32 -mattr=+experimental-zicfilp < %s -show-encoding \
-# RUN:   | FileCheck -check-prefix=FIXUP %s
 
 # RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-zicfilp < %s \
 # RUN:   | llvm-objdump -d - | FileCheck --check-prefix=INSTR-ZICFILP %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-zicfilp < %s \
 # RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
-# RUN: llvm-mc -triple riscv64 -mattr=+experimental-zicfilp < %s -show-encoding \
-# RUN:   | FileCheck -check-prefix=FIXUP %s
 
 .long foo
 
@@ -34,7 +26,6 @@ tail foo
 # INSTR: jr  t1
 # INSTR-ZICFILP: auipc t2, 0
 # INSTR-ZICFILP: jr  t2
-# FIXUP: fixup A - offset: 0, value: foo, kind:
 
 tail bar
 # RELOC: R_RISCV_CALL_PLT bar 0x0
@@ -42,7 +33,6 @@ tail bar
 # INSTR: jr  t1
 # INSTR-ZICFILP: auipc t2, 0
 # INSTR-ZICFILP: jr  t2
-# FIXUP: fixup A - offset: 0, value: bar, kind:
 
 # Ensure that tail calls to functions whose names coincide with register names
 # work.
@@ -53,7 +43,6 @@ tail zero
 # INSTR: jr  t1
 # INSTR-ZICFILP: auipc t2, 0
 # INSTR-ZICFILP: jr  t2
-# FIXUP: fixup A - offset: 0, value: zero, kind:
 
 tail f1
 # RELOC: R_RISCV_CALL_PLT f1 0x0
@@ -61,7 +50,6 @@ tail f1
 # INSTR: jr  t1
 # INSTR-ZICFILP: auipc t2, 0
 # INSTR-ZICFILP: jr  t2
-# FIXUP: fixup A - offset: 0, value: f1, kind:
 
 tail ra
 # RELOC: R_RISCV_CALL_PLT ra 0x0
@@ -69,7 +57,6 @@ tail ra
 # INSTR: jr  t1
 # INSTR-ZICFILP: auipc t2, 0
 # INSTR-ZICFILP: jr  t2
-# FIXUP: fixup A - offset: 0, value: ra, kind:
 
 tail foo@plt
 # RELOC: R_RISCV_CALL_PLT foo 0x0
@@ -77,4 +64,3 @@ tail foo@plt
 # INSTR: jr  t1
 # INSTR-ZICFILP: auipc t2, 0
 # INSTR-ZICFILP: jr  t2
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_call_plt
diff --git a/llvm/test/MC/RISCV/xqcilb-relocations.s b/llvm/test/MC/RISCV/xqcilb-relocations.s
index 92e543d94f6b6..66f052283812e 100644
--- a/llvm/test/MC/RISCV/xqcilb-relocations.s
+++ b/llvm/test/MC/RISCV/xqcilb-relocations.s
@@ -1,11 +1,10 @@
 # RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcilb %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+# RUN:     | FileCheck -check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcilb %s -o %t.o
 # RUN: llvm-readobj -r %t.o | FileCheck -check-prefix=RELOC %s
 
 # Check prefixes:
 # RELOC - Check the relocation in the object.
-# FIXUP - Check the fixup on the instruction.
 # INSTR - Check the instruction is handled properly by the ASMPrinter.
 
 .text
@@ -13,32 +12,26 @@
 qc.e.j foo
 # RELOC: R_RISCV_CUSTOM195 foo 0x0
 # INSTR: qc.e.j foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_qc_e_jump_plt
 
 qc.e.jal foo
 # RELOC: R_RISCV_CUSTOM195 foo 0x0
 # INSTR: qc.e.jal foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_qc_e_jump_plt
 
 # Check that a label in a different section is handled similar to an undefined symbol
 qc.e.j .bar
 # RELOC: R_RISCV_CUSTOM195 .bar 0x0
 # INSTR: qc.e.j .bar
-# FIXUP: fixup A - offset: 0, value: .bar, kind: fixup_riscv_qc_e_jump_plt
 
 qc.e.jal .bar
 # RELOC: R_RISCV_CUSTOM195 .bar 0x0
 # INSTR: qc.e.jal .bar
-# FIXUP: fixup A - offset: 0, value: .bar, kind: fixup_riscv_qc_e_jump_plt
 
 # Check that jumps to a defined symbol are handled correctly
 qc.e.j .L1
 # INSTR:qc.e.j .L1
-# FIXUP: fixup A - offset: 0, value: .L1, kind: fixup_riscv_qc_e_jump_plt
 
 qc.e.jal .L1
 # INSTR:qc.e.jal .L1
-# FIXUP: fixup A - offset: 0, value: .L1, kind: fixup_riscv_qc_e_jump_plt
 
 .L1:
   ret
diff --git a/llvm/test/MC/RISCV/xqcili-relocations.s b/llvm/test/MC/RISCV/xqcili-relocations.s
index b0a3f3bae11d5..b7fcb405d27bb 100644
--- a/llvm/test/MC/RISCV/xqcili-relocations.s
+++ b/llvm/test/MC/RISCV/xqcili-relocations.s
@@ -1,11 +1,10 @@
 # RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcili %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+# RUN:     | FileCheck -check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcili %s -o %t.o
 # RUN: llvm-readobj -r %t.o | FileCheck -check-prefix=RELOC %s
 
 # Check prefixes:
 # RELOC - Check the relocation in the object.
-# FIXUP - Check the fixup on the instruction.
 # INSTR - Check the instruction is handled properly by the ASMPrinter.
 
 .text
@@ -13,32 +12,26 @@
 qc.li x4, %qc.abs20(foo)
 # RELOC: R_RISCV_CUSTOM192 foo 0x0
 # INSTR: qc.li tp, %qc.abs20(foo)
-# FIXUP: fixup A - offset: 0, value: %qc.abs20(foo), kind: fixup_riscv_qc_abs20_u
 
 qc.e.li x5, foo
 # RELOC: R_RISCV_CUSTOM194 foo 0x0
 # INSTR: qc.e.li t0, foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_qc_e_32
 
 # Check that a label in a different section is handled similar to an undefined symbol
 qc.li x9, %qc.abs20(.bar)
 # RELOC: R_RISCV_CUSTOM192 .bar 0x0
 # INSTR: qc.li s1, %qc.abs20(.bar)
-# FIXUP: fixup A - offset: 0, value: %qc.abs20(.bar), kind: fixup_riscv_qc_abs20_u
 
 qc.e.li x8, .bar
 # RELOC: R_RISCV_CUSTOM194 .bar 0x0
 # INSTR: qc.e.li s0, .bar
-# FIXUP: fixup A - offset: 0, value: .bar, kind: fixup_riscv_qc_e_32
 
 # Check that branches to a defined symbol are handled correctly
 qc.li x7, %qc.abs20(.L1)
 # INSTR: qc.li t2, %qc.abs20(.L1)
-# FIXUP: fixup A - offset: 0, value: %qc.abs20(.L1), kind: fixup_riscv_qc_abs20_u
 
 qc.e.li x6, .L1
 # INSTR: qc.e.li t1, .L1
-# FIXUP: fixup A - offset: 0, value: .L1, kind: fixup_riscv_qc_e_32
 
 .L1:
   ret

From c99ffe58af23251d83073b8ad002536edc3864ae Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Thu, 17 Apr 2025 19:07:14 -0700
Subject: [PATCH 352/710] [clang-format] Fix a bug in
 FormatToken::isObjCAccessSpecifier() (#136109)

Fix #136092
---
 clang/lib/Format/FormatToken.h                | 5 +++--
 clang/unittests/Format/TokenAnnotatorTest.cpp | 6 ++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 87e16397ad069..946cd7b81587f 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -706,8 +706,9 @@ struct FormatToken {
   [[nodiscard]] bool isTypeOrIdentifier(const LangOptions &LangOpts) const;
 
   bool isObjCAccessSpecifier() const {
-    return Next && Next->isOneOf(tok::objc_public, tok::objc_protected,
-                                 tok::objc_package, tok::objc_private);
+    return is(tok::at) && Next &&
+           Next->isOneOf(tok::objc_public, tok::objc_protected,
+                         tok::objc_package, tok::objc_private);
   }
 
   /// Returns whether \p Tok is ([{ or an opening < of a template or in
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 2c7319ccefec2..043ee2e088ddb 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -3963,6 +3963,12 @@ TEST_F(TokenAnnotatorTest, UTF8StringLiteral) {
   EXPECT_TOKEN(Tokens[1], tok::utf8_string_literal, TT_Unknown);
 }
 
+TEST_F(TokenAnnotatorTest, IdentifierPackage) {
+  auto Tokens = annotate("auto package;");
+  ASSERT_EQ(Tokens.size(), 4u) << Tokens;
+  EXPECT_FALSE(Tokens[0]->isObjCAccessSpecifier());
+}
+
 } // namespace
 } // namespace format
 } // namespace clang

From 77721986d8bd48182ba355bf166a5fa3ffbfe6b6 Mon Sep 17 00:00:00 2001
From: Michael Liao <michael.hliao@gmail.com>
Date: Thu, 17 Apr 2025 22:48:41 -0400
Subject: [PATCH 353/710] [clang-format] Fix shared build. NFC

---
 clang/unittests/Format/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/unittests/Format/CMakeLists.txt b/clang/unittests/Format/CMakeLists.txt
index 5bd6a17182d29..0af571a65dafa 100644
--- a/clang/unittests/Format/CMakeLists.txt
+++ b/clang/unittests/Format/CMakeLists.txt
@@ -35,6 +35,7 @@ add_clang_unittest(FormatTests
   CLANG_LIBS
   clangBasic
   clangFormat
+  clangLex
   clangRewrite
   clangToolingCore
   LLVM_COMPONENTS

From b6dff5660092e4814fa2e2cc129ba0d05ce49a52 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 17 Apr 2025 19:38:59 -0700
Subject: [PATCH 354/710] [RISCV] Add vizip check lines to (de)interleave tests

Reducing churn in advance of a change which makes better use of these.
Note that for very short fixed length shuffles, we already use the
vizip family instructions - but mostly in the form of zipeven/zipodd.
---
 .../fixed-vectors-shuffle-deinterleave2.ll    | 319 +++++++++++++++++
 .../RISCV/rvv/vector-deinterleave-fixed.ll    | 328 +++++++++++++-----
 .../RISCV/rvv/vector-interleave-fixed.ll      | 313 ++++++++++++++++-
 3 files changed, 875 insertions(+), 85 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
index 9279e0a4d3a6c..c65d7c36a2198 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
@@ -5,6 +5,9 @@
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+f,+zve32f,+zvfh,+zvl256b \
 ; RUN:   -lower-interleaved-accesses=false -verify-machineinstrs \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK,ZVE32F
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+zvl256b,+experimental-xrivosvizip \
+; RUN:   -lower-interleaved-accesses=false -verify-machineinstrs \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,ZIP
 
 define void @vnsrl_0_i8(ptr %in, ptr %out) {
 ; CHECK-LABEL: vnsrl_0_i8:
@@ -56,6 +59,15 @@ define void @vnsrl_0_i16(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vnsrl.wi v8, v8, 0
 ; ZVE32F-NEXT:    vse16.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i16:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e16, mf2, ta, ma
+; ZIP-NEXT:    vle16.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vse16.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x i16>, ptr %in, align 2
   %shuffle.i5 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -81,6 +93,15 @@ define void @vnsrl_16_i16(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vnsrl.wi v8, v8, 16
 ; ZVE32F-NEXT:    vse16.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_16_i16:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e16, mf2, ta, ma
+; ZIP-NEXT:    vle16.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 16
+; ZIP-NEXT:    vse16.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x i16>, ptr %in, align 2
   %shuffle.i5 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -106,6 +127,15 @@ define void @vnsrl_0_half(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vnsrl.wi v8, v8, 0
 ; ZVE32F-NEXT:    vse16.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_half:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e16, mf2, ta, ma
+; ZIP-NEXT:    vle16.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vse16.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x half>, ptr %in, align 2
   %shuffle.i5 = shufflevector <8 x half> %0, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -131,6 +161,15 @@ define void @vnsrl_16_half(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vnsrl.wi v8, v8, 16
 ; ZVE32F-NEXT:    vse16.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_16_half:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e16, mf2, ta, ma
+; ZIP-NEXT:    vle16.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 16
+; ZIP-NEXT:    vse16.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x half>, ptr %in, align 2
   %shuffle.i5 = shufflevector <8 x half> %0, <8 x half> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -157,6 +196,15 @@ define void @vnsrl_0_i32(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v8, v9, 1
 ; ZVE32F-NEXT:    vse32.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i32:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e32, mf2, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vse32.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x i32>, ptr %in, align 4
   %shuffle.i5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
@@ -185,6 +233,16 @@ define void @vnsrl_32_i32(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vslidedown.vi v9, v8, 1, v0.t
 ; ZVE32F-NEXT:    vse32.v v9, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_32_i32:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e32, mf2, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    li a0, 32
+; ZIP-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; ZIP-NEXT:    vnsrl.wx v8, v8, a0
+; ZIP-NEXT:    vse32.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x i32>, ptr %in, align 4
   %shuffle.i5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
@@ -211,6 +269,15 @@ define void @vnsrl_0_float(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v8, v9, 1
 ; ZVE32F-NEXT:    vse32.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_float:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e32, mf2, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vse32.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x float>, ptr %in, align 4
   %shuffle.i5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -239,6 +306,16 @@ define void @vnsrl_32_float(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vslidedown.vi v9, v8, 1, v0.t
 ; ZVE32F-NEXT:    vse32.v v9, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_32_float:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e32, mf2, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    li a0, 32
+; ZIP-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; ZIP-NEXT:    vnsrl.wx v8, v8, a0
+; ZIP-NEXT:    vse32.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x float>, ptr %in, align 4
   %shuffle.i5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
@@ -264,6 +341,16 @@ define void @vnsrl_0_i64(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    sd a2, 0(a1)
 ; ZVE32F-NEXT:    sd a0, 8(a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i64:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
+; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vslidedown.vi v9, v8, 2
+; ZIP-NEXT:    vslideup.vi v8, v9, 1
+; ZIP-NEXT:    vse64.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x i64>, ptr %in, align 8
   %shuffle.i5 = shufflevector <4 x i64> %0, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
@@ -290,6 +377,16 @@ define void @vnsrl_64_i64(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    sd a2, 0(a1)
 ; ZVE32F-NEXT:    sd a0, 8(a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_64_i64:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
+; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vslidedown.vi v9, v8, 2
+; ZIP-NEXT:    ri.vzipodd.vv v10, v8, v9
+; ZIP-NEXT:    vse64.v v10, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x i64>, ptr %in, align 8
   %shuffle.i5 = shufflevector <4 x i64> %0, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
@@ -315,6 +412,16 @@ define void @vnsrl_0_double(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    sd a2, 0(a1)
 ; ZVE32F-NEXT:    sd a0, 8(a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_double:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
+; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vslidedown.vi v9, v8, 2
+; ZIP-NEXT:    vslideup.vi v8, v9, 1
+; ZIP-NEXT:    vse64.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x double>, ptr %in, align 8
   %shuffle.i5 = shufflevector <4 x double> %0, <4 x double> poison, <2 x i32> <i32 0, i32 2>
@@ -341,6 +448,16 @@ define void @vnsrl_64_double(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    sd a2, 0(a1)
 ; ZVE32F-NEXT:    sd a0, 8(a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_64_double:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
+; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vslidedown.vi v9, v8, 2
+; ZIP-NEXT:    ri.vzipodd.vv v10, v8, v9
+; ZIP-NEXT:    vse64.v v10, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x double>, ptr %in, align 8
   %shuffle.i5 = shufflevector <4 x double> %0, <4 x double> poison, <2 x i32> <i32 1, i32 3>
@@ -444,6 +561,15 @@ define void @vnsrl_0_i8_single_src(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vnsrl.wi v8, v8, 0
 ; ZVE32F-NEXT:    vse8.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i8_single_src:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT:    vle8.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vse8.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %in, align 1
   %shuffle.i5 = shufflevector <8 x i8> %0, <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -469,6 +595,15 @@ define void @vnsrl_8_i8_single_src(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vnsrl.wi v8, v8, 8
 ; ZVE32F-NEXT:    vse8.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_8_i8_single_src:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT:    vle8.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 8
+; ZIP-NEXT:    vse8.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %in, align 1
   %shuffle.i5 = shufflevector <8 x i8> %0, <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -496,6 +631,16 @@ define void @vnsrl_0_i8_single_wideuse(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
 ; ZVE32F-NEXT:    vse8.v v8, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i8_single_wideuse:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT:    vle8.v v8, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT:    vse8.v v8, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %in, align 1
   %shuffle.i5 = shufflevector <8 x i8> %0, <8 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -531,6 +676,18 @@ define void @vnsrl_0_i32_single_src_m8(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; ZVE32F-NEXT:    vse32.v v16, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i32_single_src_m8:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    li a2, 64
+; ZIP-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    li a0, 32
+; ZIP-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v16, v8, 0
+; ZIP-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; ZIP-NEXT:    vse32.v v16, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <64 x i32>, ptr %in, align 4
   %shuffle.i5 = shufflevector <64 x i32> %0, <64 x i32> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -564,6 +721,18 @@ define void @vnsrl_0_i32_single_src_m8_2(ptr %in, ptr %out) {
 ; ZVE32F-NEXT:    vcompress.vm v16, v8, v24
 ; ZVE32F-NEXT:    vse32.v v16, (a1)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i32_single_src_m8_2:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    li a2, 64
+; ZIP-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    li a0, 32
+; ZIP-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v16, v8, 0
+; ZIP-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; ZIP-NEXT:    vse32.v v16, (a1)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <64 x i32>, ptr %in, align 4
   %shuffle.i5 = shufflevector <64 x i32> %0, <64 x i32> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -597,6 +766,19 @@ define void @vnsrl_0_i8_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v9, v8, 4
 ; ZVE32F-NEXT:    vse8.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i8_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT:    vle8.v v8, (a1)
+; ZIP-NEXT:    vle8.v v9, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vnsrl.wi v9, v9, 0
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT:    vslideup.vi v9, v8, 4
+; ZIP-NEXT:    vse8.v v9, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %in0, align 1
   %1 = load <8 x i8>, ptr %in1, align 1
@@ -631,6 +813,19 @@ define void @vnsrl_8_8_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v9, v8, 4
 ; ZVE32F-NEXT:    vse8.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_8_8_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT:    vle8.v v8, (a1)
+; ZIP-NEXT:    vle8.v v9, (a0)
+; ZIP-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 8
+; ZIP-NEXT:    vnsrl.wi v9, v9, 8
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT:    vslideup.vi v9, v8, 4
+; ZIP-NEXT:    vse8.v v9, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %in0, align 1
   %1 = load <8 x i8>, ptr %in1, align 1
@@ -665,6 +860,19 @@ define void @vnsrl_0_i16_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v9, v8, 2
 ; ZVE32F-NEXT:    vse16.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i16_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vle16.v v8, (a1)
+; ZIP-NEXT:    vle16.v v9, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vnsrl.wi v9, v9, 0
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vslideup.vi v9, v8, 2
+; ZIP-NEXT:    vse16.v v9, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x i16>, ptr %in0, align 2
   %1 = load <4 x i16>, ptr %in1, align 2
@@ -699,6 +907,19 @@ define void @vnsrl_16_i16_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v9, v8, 2
 ; ZVE32F-NEXT:    vse16.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_16_i16_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vle16.v v8, (a1)
+; ZIP-NEXT:    vle16.v v9, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 16
+; ZIP-NEXT:    vnsrl.wi v9, v9, 16
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vslideup.vi v9, v8, 2
+; ZIP-NEXT:    vse16.v v9, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x i16>, ptr %in0, align 2
   %1 = load <4 x i16>, ptr %in1, align 2
@@ -733,6 +954,19 @@ define void @vnsrl_0_half_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v9, v8, 2
 ; ZVE32F-NEXT:    vse16.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_half_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vle16.v v8, (a1)
+; ZIP-NEXT:    vle16.v v9, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 0
+; ZIP-NEXT:    vnsrl.wi v9, v9, 0
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vslideup.vi v9, v8, 2
+; ZIP-NEXT:    vse16.v v9, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x half>, ptr %in0, align 2
   %1 = load <4 x half>, ptr %in1, align 2
@@ -767,6 +1001,19 @@ define void @vnsrl_16_half_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v9, v8, 2
 ; ZVE32F-NEXT:    vse16.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_16_half_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vle16.v v8, (a1)
+; ZIP-NEXT:    vle16.v v9, (a0)
+; ZIP-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZIP-NEXT:    vnsrl.wi v8, v8, 16
+; ZIP-NEXT:    vnsrl.wi v9, v9, 16
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; ZIP-NEXT:    vslideup.vi v9, v8, 2
+; ZIP-NEXT:    vse16.v v9, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x half>, ptr %in0, align 2
   %1 = load <4 x half>, ptr %in1, align 2
@@ -793,6 +1040,15 @@ define void @vnsrl_0_i32_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v8, v9, 1
 ; ZVE32F-NEXT:    vse32.v v8, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i32_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    vle32.v v9, (a1)
+; ZIP-NEXT:    vslideup.vi v8, v9, 1
+; ZIP-NEXT:    vse32.v v8, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <2 x i32>, ptr %in0, align 4
   %1 = load <2 x i32>, ptr %in1, align 4
@@ -821,6 +1077,15 @@ define void @vnsrl_32_i32_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslidedown.vi v9, v8, 1, v0.t
 ; ZVE32F-NEXT:    vse32.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_32_i32_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    vle32.v v9, (a1)
+; ZIP-NEXT:    ri.vzipodd.vv v10, v8, v9
+; ZIP-NEXT:    vse32.v v10, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <2 x i32>, ptr %in0, align 4
   %1 = load <2 x i32>, ptr %in1, align 4
@@ -847,6 +1112,15 @@ define void @vnsrl_0_float_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v8, v9, 1
 ; ZVE32F-NEXT:    vse32.v v8, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_float_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    vle32.v v9, (a1)
+; ZIP-NEXT:    vslideup.vi v8, v9, 1
+; ZIP-NEXT:    vse32.v v8, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <2 x float>, ptr %in0, align 4
   %1 = load <2 x float>, ptr %in1, align 4
@@ -875,6 +1149,15 @@ define void @vnsrl_32_float_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslidedown.vi v9, v8, 1, v0.t
 ; ZVE32F-NEXT:    vse32.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_32_float_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    vle32.v v9, (a1)
+; ZIP-NEXT:    ri.vzipodd.vv v10, v8, v9
+; ZIP-NEXT:    vse32.v v10, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <2 x float>, ptr %in0, align 4
   %1 = load <2 x float>, ptr %in1, align 4
@@ -901,6 +1184,15 @@ define void @vnsrl_0_i64_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslideup.vi v8, v9, 2
 ; ZVE32F-NEXT:    vse32.v v8, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_i64_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    vle64.v v9, (a1)
+; ZIP-NEXT:    vslideup.vi v8, v9, 1
+; ZIP-NEXT:    vse64.v v8, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <2 x i64>, ptr %in0, align 8
   %1 = load <2 x i64>, ptr %in1, align 8
@@ -929,6 +1221,15 @@ define void @vnsrl_64_i64_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    vslidedown.vi v9, v8, 2, v0.t
 ; ZVE32F-NEXT:    vse32.v v9, (a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_64_i64_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    vle64.v v9, (a1)
+; ZIP-NEXT:    ri.vzipodd.vv v10, v8, v9
+; ZIP-NEXT:    vse64.v v10, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <2 x i64>, ptr %in0, align 8
   %1 = load <2 x i64>, ptr %in1, align 8
@@ -954,6 +1255,15 @@ define void @vnsrl_0_double_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    sd a0, 0(a2)
 ; ZVE32F-NEXT:    sd a1, 8(a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_0_double_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    vle64.v v9, (a1)
+; ZIP-NEXT:    vslideup.vi v8, v9, 1
+; ZIP-NEXT:    vse64.v v8, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <2 x double>, ptr %in0, align 8
   %1 = load <2 x double>, ptr %in1, align 8
@@ -980,6 +1290,15 @@ define void @vnsrl_64_double_two_source(ptr %in0, ptr %in1, ptr %out) {
 ; ZVE32F-NEXT:    sd a0, 0(a2)
 ; ZVE32F-NEXT:    sd a1, 8(a2)
 ; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: vnsrl_64_double_two_source:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    vle64.v v9, (a1)
+; ZIP-NEXT:    ri.vzipodd.vv v10, v8, v9
+; ZIP-NEXT:    vse64.v v10, (a2)
+; ZIP-NEXT:    ret
 entry:
   %0 = load <2 x double>, ptr %in0, align 8
   %1 = load <2 x double>, ptr %in1, align 8
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index 6dd6d51862088..f6b5a35aa06d6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck %s --check-prefixes=CHECK,V,RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck %s --check-prefixes=CHECK,V,RV64
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+experimental-xrivosvizip | FileCheck %s --check-prefixes=CHECK,ZIP
 
 ; Integers
 
@@ -71,50 +72,88 @@ ret {<4 x i32>, <4 x i32>} %retval
 }
 
 define {<2 x i64>, <2 x i64>} @vector_deinterleave_v2i64_v4i64(<4 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_v2i64_v4i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 2
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; CHECK-NEXT:    vmv.v.i v0, 1
-; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1, v0.t
-; CHECK-NEXT:    vslideup.vi v8, v10, 1
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_v2i64_v4i64:
+; V:       # %bb.0:
+; V-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
+; V-NEXT:    vslidedown.vi v10, v8, 2
+; V-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT:    vmv.v.i v0, 1
+; V-NEXT:    vmv1r.v v9, v10
+; V-NEXT:    vslidedown.vi v9, v8, 1, v0.t
+; V-NEXT:    vslideup.vi v8, v10, 1
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_v2i64_v4i64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
+; ZIP-NEXT:    vslidedown.vi v10, v8, 2
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    ri.vzipodd.vv v9, v8, v10
+; ZIP-NEXT:    vslideup.vi v8, v10, 1
+; ZIP-NEXT:    ret
 %retval = call {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64> %vec)
 ret {<2 x i64>, <2 x i64>} %retval
 }
 
 define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_v4i64_v8i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.i v0, 8
-; CHECK-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v16, v8, 4
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.i v10, 2
-; CHECK-NEXT:    vmv2r.v v12, v8
-; CHECK-NEXT:    vmv.v.i v11, 12
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; CHECK-NEXT:    vslideup.vi v14, v16, 2
-; CHECK-NEXT:    vslideup.vi v14, v16, 1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vslidedown.vi v12, v8, 1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v11
-; CHECK-NEXT:    vmerge.vvm v12, v12, v14, v0
-; CHECK-NEXT:    vslidedown.vi v14, v8, 1
-; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vslidedown.vi v14, v8, 2, v0.t
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.i v0, 4
-; CHECK-NEXT:    vmv2r.v v8, v16
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; CHECK-NEXT:    vslideup.vi v8, v16, 1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v11
-; CHECK-NEXT:    vmerge.vvm v10, v14, v8, v0
-; CHECK-NEXT:    vmv2r.v v8, v12
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_v4i64_v8i64:
+; V:       # %bb.0:
+; V-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.i v0, 8
+; V-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
+; V-NEXT:    vslidedown.vi v16, v8, 4
+; V-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.i v10, 2
+; V-NEXT:    vmv2r.v v12, v8
+; V-NEXT:    vmv.v.i v11, 12
+; V-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; V-NEXT:    vslideup.vi v14, v16, 2
+; V-NEXT:    vslideup.vi v14, v16, 1, v0.t
+; V-NEXT:    vmv1r.v v0, v10
+; V-NEXT:    vslidedown.vi v12, v8, 1, v0.t
+; V-NEXT:    vmv1r.v v0, v11
+; V-NEXT:    vmerge.vvm v12, v12, v14, v0
+; V-NEXT:    vslidedown.vi v14, v8, 1
+; V-NEXT:    vmv1r.v v0, v10
+; V-NEXT:    vslidedown.vi v14, v8, 2, v0.t
+; V-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.i v0, 4
+; V-NEXT:    vmv2r.v v8, v16
+; V-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; V-NEXT:    vslideup.vi v8, v16, 1, v0.t
+; V-NEXT:    vmv1r.v v0, v11
+; V-NEXT:    vmerge.vvm v10, v14, v8, v0
+; V-NEXT:    vmv2r.v v8, v12
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_v4i64_v8i64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZIP-NEXT:    vslidedown.vi v12, v8, 1
+; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; ZIP-NEXT:    vmv.v.i v0, 2
+; ZIP-NEXT:    vmv.v.i v14, 12
+; ZIP-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
+; ZIP-NEXT:    vslidedown.vi v16, v8, 4
+; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
+; ZIP-NEXT:    vslidedown.vi v10, v8, 2
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; ZIP-NEXT:    vslidedown.vi v12, v8, 2, v0.t
+; ZIP-NEXT:    ri.vzip2a.vv v18, v8, v10
+; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
+; ZIP-NEXT:    vslidedown.vi v8, v16, 2
+; ZIP-NEXT:    vmv1r.v v0, v14
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; ZIP-NEXT:    ri.vzip2a.vv v12, v16, v8, v0.t
+; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; ZIP-NEXT:    vmv.v.i v0, 8
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; ZIP-NEXT:    vslideup.vi v8, v16, 2
+; ZIP-NEXT:    vslideup.vi v8, v16, 1, v0.t
+; ZIP-NEXT:    vmv1r.v v0, v14
+; ZIP-NEXT:    vmerge.vvm v8, v18, v8, v0
+; ZIP-NEXT:    vmv2r.v v10, v12
+; ZIP-NEXT:    ret
   %retval = call {<4 x i64>, <4 x i64>} @llvm.vector.deinterleave2.v8i64(<8 x i64> %vec)
   ret {<4 x i64>, <4 x i64>} %retval
 }
@@ -408,6 +447,97 @@ define {<2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>} @v
 ; RV64-NEXT:    addi sp, sp, 64
 ; RV64-NEXT:    .cfi_def_cfa_offset 0
 ; RV64-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave7_v14i8_v2i8:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    addi sp, sp, -64
+; ZIP-NEXT:    .cfi_def_cfa_offset 64
+; ZIP-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; ZIP-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; ZIP-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; ZIP-NEXT:    .cfi_offset ra, -8
+; ZIP-NEXT:    .cfi_offset s0, -16
+; ZIP-NEXT:    .cfi_offset s1, -24
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 2
+; ZIP-NEXT:    sub sp, sp, a0
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 64 + 4 * vlenb
+; ZIP-NEXT:    addi a0, sp, 32
+; ZIP-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; ZIP-NEXT:    csrr s1, vlenb
+; ZIP-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; ZIP-NEXT:    vslidedown.vi v11, v8, 10
+; ZIP-NEXT:    vslidedown.vi v10, v8, 8
+; ZIP-NEXT:    vslidedown.vi v9, v8, 2
+; ZIP-NEXT:    srli s0, s1, 3
+; ZIP-NEXT:    add a0, s0, s0
+; ZIP-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
+; ZIP-NEXT:    vslideup.vx v10, v11, s0
+; ZIP-NEXT:    vmv1r.v v11, v8
+; ZIP-NEXT:    vslideup.vx v11, v9, s0
+; ZIP-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; ZIP-NEXT:    vslidedown.vi v9, v8, 12
+; ZIP-NEXT:    srli a0, s1, 2
+; ZIP-NEXT:    add a1, a0, s0
+; ZIP-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
+; ZIP-NEXT:    vslideup.vx v10, v9, a0
+; ZIP-NEXT:    csrr a2, vlenb
+; ZIP-NEXT:    slli a2, a2, 1
+; ZIP-NEXT:    add a2, sp, a2
+; ZIP-NEXT:    addi a2, a2, 32
+; ZIP-NEXT:    vs1r.v v10, (a2) # vscale x 8-byte Folded Spill
+; ZIP-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; ZIP-NEXT:    vslidedown.vi v9, v8, 4
+; ZIP-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
+; ZIP-NEXT:    vslideup.vx v11, v9, a0
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    add a0, sp, a0
+; ZIP-NEXT:    addi a0, a0, 32
+; ZIP-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; ZIP-NEXT:    li a1, 3
+; ZIP-NEXT:    mv a0, s0
+; ZIP-NEXT:    call __muldi3
+; ZIP-NEXT:    add s0, a0, s0
+; ZIP-NEXT:    addi a1, sp, 32
+; ZIP-NEXT:    vl1r.v v8, (a1) # vscale x 8-byte Folded Reload
+; ZIP-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; ZIP-NEXT:    vslidedown.vi v8, v8, 6
+; ZIP-NEXT:    srli s1, s1, 1
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    add a1, sp, a1
+; ZIP-NEXT:    addi a1, a1, 32
+; ZIP-NEXT:    vl1r.v v9, (a1) # vscale x 8-byte Folded Reload
+; ZIP-NEXT:    vsetvli zero, s0, e8, mf2, ta, ma
+; ZIP-NEXT:    vslideup.vx v9, v8, a0
+; ZIP-NEXT:    add a0, s1, s1
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    slli a1, a1, 1
+; ZIP-NEXT:    add a1, sp, a1
+; ZIP-NEXT:    addi a1, a1, 32
+; ZIP-NEXT:    vl1r.v v8, (a1) # vscale x 8-byte Folded Reload
+; ZIP-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; ZIP-NEXT:    vslideup.vx v9, v8, s1
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a1, a0, 1
+; ZIP-NEXT:    add a0, a1, a0
+; ZIP-NEXT:    add a0, sp, a0
+; ZIP-NEXT:    addi a0, a0, 32
+; ZIP-NEXT:    vs1r.v v9, (a0)
+; ZIP-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; ZIP-NEXT:    vlseg7e8.v v8, (a0)
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 2
+; ZIP-NEXT:    add sp, sp, a0
+; ZIP-NEXT:    .cfi_def_cfa sp, 64
+; ZIP-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; ZIP-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; ZIP-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; ZIP-NEXT:    .cfi_restore ra
+; ZIP-NEXT:    .cfi_restore s0
+; ZIP-NEXT:    .cfi_restore s1
+; ZIP-NEXT:    addi sp, sp, 64
+; ZIP-NEXT:    .cfi_def_cfa_offset 0
+; ZIP-NEXT:    ret
 	   %res = call {<2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>} @llvm.vector.deinterleave7.v14i8(<14 x i8> %v)
 	   ret {<2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>} %res
 }
@@ -479,50 +609,88 @@ ret  {<4 x float>, <4 x float>} %retval
 }
 
 define {<2 x double>, <2 x double>} @vector_deinterleave_v2f64_v4f64(<4 x double> %vec) {
-; CHECK-LABEL: vector_deinterleave_v2f64_v4f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 2
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; CHECK-NEXT:    vmv.v.i v0, 1
-; CHECK-NEXT:    vmv1r.v v9, v10
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1, v0.t
-; CHECK-NEXT:    vslideup.vi v8, v10, 1
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_v2f64_v4f64:
+; V:       # %bb.0:
+; V-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
+; V-NEXT:    vslidedown.vi v10, v8, 2
+; V-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT:    vmv.v.i v0, 1
+; V-NEXT:    vmv1r.v v9, v10
+; V-NEXT:    vslidedown.vi v9, v8, 1, v0.t
+; V-NEXT:    vslideup.vi v8, v10, 1
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_v2f64_v4f64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
+; ZIP-NEXT:    vslidedown.vi v10, v8, 2
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    ri.vzipodd.vv v9, v8, v10
+; ZIP-NEXT:    vslideup.vi v8, v10, 1
+; ZIP-NEXT:    ret
 %retval = call {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double> %vec)
 ret {<2 x double>, <2 x double>} %retval
 }
 
 define {<4 x double>, <4 x double>} @vector_deinterleave_v4f64_v8f64(<8 x double> %vec) {
-; CHECK-LABEL: vector_deinterleave_v4f64_v8f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.i v0, 8
-; CHECK-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v16, v8, 4
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.i v10, 2
-; CHECK-NEXT:    vmv2r.v v12, v8
-; CHECK-NEXT:    vmv.v.i v11, 12
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; CHECK-NEXT:    vslideup.vi v14, v16, 2
-; CHECK-NEXT:    vslideup.vi v14, v16, 1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vslidedown.vi v12, v8, 1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v11
-; CHECK-NEXT:    vmerge.vvm v12, v12, v14, v0
-; CHECK-NEXT:    vslidedown.vi v14, v8, 1
-; CHECK-NEXT:    vmv1r.v v0, v10
-; CHECK-NEXT:    vslidedown.vi v14, v8, 2, v0.t
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.i v0, 4
-; CHECK-NEXT:    vmv2r.v v8, v16
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; CHECK-NEXT:    vslideup.vi v8, v16, 1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v11
-; CHECK-NEXT:    vmerge.vvm v10, v14, v8, v0
-; CHECK-NEXT:    vmv2r.v v8, v12
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_v4f64_v8f64:
+; V:       # %bb.0:
+; V-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.i v0, 8
+; V-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
+; V-NEXT:    vslidedown.vi v16, v8, 4
+; V-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.i v10, 2
+; V-NEXT:    vmv2r.v v12, v8
+; V-NEXT:    vmv.v.i v11, 12
+; V-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; V-NEXT:    vslideup.vi v14, v16, 2
+; V-NEXT:    vslideup.vi v14, v16, 1, v0.t
+; V-NEXT:    vmv1r.v v0, v10
+; V-NEXT:    vslidedown.vi v12, v8, 1, v0.t
+; V-NEXT:    vmv1r.v v0, v11
+; V-NEXT:    vmerge.vvm v12, v12, v14, v0
+; V-NEXT:    vslidedown.vi v14, v8, 1
+; V-NEXT:    vmv1r.v v0, v10
+; V-NEXT:    vslidedown.vi v14, v8, 2, v0.t
+; V-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.i v0, 4
+; V-NEXT:    vmv2r.v v8, v16
+; V-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; V-NEXT:    vslideup.vi v8, v16, 1, v0.t
+; V-NEXT:    vmv1r.v v0, v11
+; V-NEXT:    vmerge.vvm v10, v14, v8, v0
+; V-NEXT:    vmv2r.v v8, v12
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_v4f64_v8f64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; ZIP-NEXT:    vmv.v.i v0, 8
+; ZIP-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
+; ZIP-NEXT:    vslidedown.vi v16, v8, 4
+; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
+; ZIP-NEXT:    vslidedown.vi v12, v8, 2
+; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; ZIP-NEXT:    vmv.v.i v10, 12
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; ZIP-NEXT:    vslideup.vi v14, v16, 2
+; ZIP-NEXT:    vslideup.vi v14, v16, 1, v0.t
+; ZIP-NEXT:    ri.vzip2a.vv v18, v8, v12
+; ZIP-NEXT:    vmv1r.v v0, v10
+; ZIP-NEXT:    vmerge.vvm v12, v18, v14, v0
+; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
+; ZIP-NEXT:    vslidedown.vi v14, v16, 2
+; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
+; ZIP-NEXT:    vmv.v.i v0, 2
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
+; ZIP-NEXT:    ri.vzip2a.vv v18, v16, v14
+; ZIP-NEXT:    vslidedown.vi v14, v8, 1
+; ZIP-NEXT:    vslidedown.vi v14, v8, 2, v0.t
+; ZIP-NEXT:    vmv1r.v v0, v10
+; ZIP-NEXT:    vmerge.vvm v10, v14, v18, v0
+; ZIP-NEXT:    vmv2r.v v8, v12
+; ZIP-NEXT:    ret
 %retval = call {<4 x double>, <4 x double>} @llvm.vector.deinterleave2.v8f64(<8 x double> %vec)
 ret {<4 x double>, <4 x double>} %retval
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
index e2f81acc9cf99..279779dc49667 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck -check-prefixes=CHECK,RV32 %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck -check-prefixes=CHECK,RV64 %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh | FileCheck -check-prefixes=CHECK %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck -check-prefixes=CHECK %s
 ; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvbb,+zvfh | FileCheck %s --check-prefix=ZVBB
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvbb,+zvfh | FileCheck %s --check-prefix=ZVBB
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvbb,+zvfh,+experimental-xrivosvizip | FileCheck %s --check-prefix=ZIP
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvbb,+zvfh,+experimental-xrivosvizip | FileCheck %s --check-prefix=ZIP
 
 ; Integers
 
@@ -41,6 +43,21 @@ define <32 x i1> @vector_interleave_v32i1_v16i1(<16 x i1> %a, <16 x i1> %b) {
 ; ZVBB-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; ZVBB-NEXT:    vmsne.vi v0, v12, 0
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v32i1_v16i1:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    li a0, 32
+; ZIP-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZIP-NEXT:    vslideup.vi v0, v8, 2
+; ZIP-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; ZIP-NEXT:    vmv.v.i v8, 0
+; ZIP-NEXT:    vmerge.vim v8, v8, 1, v0
+; ZIP-NEXT:    vsetivli zero, 16, e8, m2, ta, ma
+; ZIP-NEXT:    vslidedown.vi v10, v8, 16
+; ZIP-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; ZIP-NEXT:    ri.vzip2a.vv v12, v8, v10
+; ZIP-NEXT:    vmsne.vi v0, v12, 0
+; ZIP-NEXT:    ret
 	   %res = call <32 x i1> @llvm.vector.interleave2.v32i1(<16 x i1> %a, <16 x i1> %b)
 	   ret <32 x i1> %res
 }
@@ -64,6 +81,14 @@ define <16 x i16> @vector_interleave_v16i16_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; ZVBB-NEXT:    vwsll.vi v8, v10, 16
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v11
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v16i16_v8i16:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZIP-NEXT:    vmv1r.v v12, v9
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> %a, <8 x i16> %b)
 	   ret <16 x i16> %res
 }
@@ -88,6 +113,14 @@ define <8 x i32> @vector_interleave_v8i32_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; ZVBB-NEXT:    vwsll.vx v8, v10, a0
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v11
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v8i32_v4i32:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZIP-NEXT:    vmv1r.v v12, v9
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %a, <4 x i32> %b)
 	   ret <8 x i32> %res
 }
@@ -122,6 +155,14 @@ define <4 x i64> @vector_interleave_v4i64_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; ZVBB-NEXT:    vrgatherei16.vv v10, v8, v12
 ; ZVBB-NEXT:    vmv.v.v v8, v10
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v4i64_v2i64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZIP-NEXT:    vmv1r.v v12, v9
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %a, <2 x i64> %b)
 	   ret <4 x i64> %res
 }
@@ -186,6 +227,36 @@ define <6 x i32> @vector_interleave3_v6i32_v2i32(<2 x i32> %a, <2 x i32> %b, <2
 ; ZVBB-NEXT:    addi sp, sp, 16
 ; ZVBB-NEXT:    .cfi_def_cfa_offset 0
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave3_v6i32_v2i32:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    addi sp, sp, -16
+; ZIP-NEXT:    .cfi_def_cfa_offset 16
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    sub sp, sp, a0
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZIP-NEXT:    addi a0, sp, 16
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    srli a1, a1, 1
+; ZIP-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
+; ZIP-NEXT:    vsseg3e32.v v8, (a0)
+; ZIP-NEXT:    add a2, a0, a1
+; ZIP-NEXT:    vle32.v v9, (a2)
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    add a1, a2, a1
+; ZIP-NEXT:    vle32.v v10, (a1)
+; ZIP-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v9, 2
+; ZIP-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v10, 4
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    add sp, sp, a0
+; ZIP-NEXT:    .cfi_def_cfa sp, 16
+; ZIP-NEXT:    addi sp, sp, 16
+; ZIP-NEXT:    .cfi_def_cfa_offset 0
+; ZIP-NEXT:    ret
 	   %res = call <6 x i32> @llvm.vector.interleave3.v6i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c)
 	   ret <6 x i32> %res
 }
@@ -265,6 +336,43 @@ define <10 x i16> @vector_interleave5_v10i16_v2i16(<2 x i16> %a, <2 x i16> %b, <
 ; ZVBB-NEXT:    addi sp, sp, 16
 ; ZVBB-NEXT:    .cfi_def_cfa_offset 0
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave5_v10i16_v2i16:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    addi sp, sp, -16
+; ZIP-NEXT:    .cfi_def_cfa_offset 16
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    sub sp, sp, a0
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZIP-NEXT:    addi a0, sp, 16
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    srli a1, a1, 2
+; ZIP-NEXT:    add a2, a0, a1
+; ZIP-NEXT:    add a3, a2, a1
+; ZIP-NEXT:    vsetvli a4, zero, e16, mf4, ta, ma
+; ZIP-NEXT:    vsseg5e16.v v8, (a0)
+; ZIP-NEXT:    add a4, a3, a1
+; ZIP-NEXT:    vle16.v v9, (a2)
+; ZIP-NEXT:    vle16.v v11, (a4)
+; ZIP-NEXT:    vle16.v v12, (a3)
+; ZIP-NEXT:    vle16.v v8, (a0)
+; ZIP-NEXT:    add a1, a4, a1
+; ZIP-NEXT:    vle16.v v10, (a1)
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZIP-NEXT:    vslideup.vi v12, v11, 2
+; ZIP-NEXT:    vslideup.vi v8, v9, 2
+; ZIP-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v12, 4
+; ZIP-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v10, 8
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    add sp, sp, a0
+; ZIP-NEXT:    .cfi_def_cfa sp, 16
+; ZIP-NEXT:    addi sp, sp, 16
+; ZIP-NEXT:    .cfi_def_cfa_offset 0
+; ZIP-NEXT:    ret
 	   %res = call <10 x i16> @llvm.vector.interleave5.v10i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d, <2 x i16> %e)
 	   ret <10 x i16> %res
 }
@@ -353,6 +461,48 @@ define <14 x i8> @vector_interleave7_v14i8_v2i8(<2 x i8> %a, <2 x i8> %b, <2 x i
 ; ZVBB-NEXT:    addi sp, sp, 16
 ; ZVBB-NEXT:    .cfi_def_cfa_offset 0
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave7_v14i8_v2i8:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    addi sp, sp, -16
+; ZIP-NEXT:    .cfi_def_cfa_offset 16
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    sub sp, sp, a0
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 1 * vlenb
+; ZIP-NEXT:    addi a0, sp, 16
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    srli a1, a1, 3
+; ZIP-NEXT:    add a2, a0, a1
+; ZIP-NEXT:    add a3, a2, a1
+; ZIP-NEXT:    add a4, a3, a1
+; ZIP-NEXT:    vsetvli a5, zero, e8, mf8, ta, ma
+; ZIP-NEXT:    vsseg7e8.v v8, (a0)
+; ZIP-NEXT:    vle8.v v9, (a4)
+; ZIP-NEXT:    add a4, a4, a1
+; ZIP-NEXT:    vle8.v v10, (a2)
+; ZIP-NEXT:    add a2, a4, a1
+; ZIP-NEXT:    add a1, a2, a1
+; ZIP-NEXT:    vle8.v v11, (a2)
+; ZIP-NEXT:    vle8.v v12, (a4)
+; ZIP-NEXT:    vle8.v v8, (a0)
+; ZIP-NEXT:    vle8.v v13, (a1)
+; ZIP-NEXT:    vle8.v v14, (a3)
+; ZIP-NEXT:    vsetivli zero, 4, e8, mf2, tu, ma
+; ZIP-NEXT:    vslideup.vi v12, v11, 2
+; ZIP-NEXT:    vslideup.vi v8, v10, 2
+; ZIP-NEXT:    vsetivli zero, 6, e8, mf2, tu, ma
+; ZIP-NEXT:    vslideup.vi v12, v13, 4
+; ZIP-NEXT:    vslideup.vi v8, v14, 4
+; ZIP-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v9, 6
+; ZIP-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v12, 8
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    add sp, sp, a0
+; ZIP-NEXT:    .cfi_def_cfa sp, 16
+; ZIP-NEXT:    addi sp, sp, 16
+; ZIP-NEXT:    .cfi_def_cfa_offset 0
+; ZIP-NEXT:    ret
 	   %res = call <14 x i8> @llvm.vector.interleave7.v14i8(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x i8> %e, <2 x i8> %f, <2 x i8> %g)
 	   ret <14 x i8> %res
 }
@@ -377,6 +527,13 @@ define <4 x half> @vector_interleave_v4f16_v2f16(<2 x half> %a, <2 x half> %b) {
 ; ZVBB-NEXT:    vwaddu.wv v10, v10, v8
 ; ZVBB-NEXT:    vmv1r.v v8, v10
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v4f16_v2f16:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v9
+; ZIP-NEXT:    vmv1r.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <4 x half> @llvm.vector.interleave2.v4f16(<2 x half> %a, <2 x half> %b)
 	   ret <4 x half> %res
 }
@@ -398,6 +555,13 @@ define <8 x half> @vector_interleave_v8f16_v4f16(<4 x half> %a, <4 x half> %b) {
 ; ZVBB-NEXT:    vwaddu.wv v10, v10, v8
 ; ZVBB-NEXT:    vmv1r.v v8, v10
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v8f16_v4f16:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v9
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <8 x half> @llvm.vector.interleave2.v8f16(<4 x half> %a, <4 x half> %b)
 	   ret <8 x half> %res
 }
@@ -420,6 +584,13 @@ define <4 x float> @vector_interleave_v4f32_v2f32(<2 x float> %a, <2 x float> %b
 ; ZVBB-NEXT:    vwaddu.wv v10, v10, v8
 ; ZVBB-NEXT:    vmv1r.v v8, v10
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v4f32_v2f32:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v9
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %a, <2 x float> %b)
 	   ret <4 x float> %res
 }
@@ -443,6 +614,14 @@ define <16 x half> @vector_interleave_v16f16_v8f16(<8 x half> %a, <8 x half> %b)
 ; ZVBB-NEXT:    vwsll.vi v8, v10, 16
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v11
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v16f16_v8f16:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZIP-NEXT:    vmv1r.v v12, v9
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <16 x half> @llvm.vector.interleave2.v16f16(<8 x half> %a, <8 x half> %b)
 	   ret <16 x half> %res
 }
@@ -467,6 +646,14 @@ define <8 x float> @vector_interleave_v8f32_v4f32(<4 x float> %a, <4 x float> %b
 ; ZVBB-NEXT:    vwsll.vx v8, v10, a0
 ; ZVBB-NEXT:    vwaddu.wv v8, v8, v11
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v8f32_v4f32:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZIP-NEXT:    vmv1r.v v12, v9
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %a, <4 x float> %b)
 	   ret <8 x float> %res
 }
@@ -501,6 +688,14 @@ define <4 x double> @vector_interleave_v4f64_v2f64(<2 x double> %a, <2 x double>
 ; ZVBB-NEXT:    vrgatherei16.vv v10, v8, v12
 ; ZVBB-NEXT:    vmv.v.v v8, v10
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave_v4f64_v2f64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZIP-NEXT:    vmv1r.v v12, v9
+; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
 	   %res = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %a, <2 x double> %b)
 	   ret <4 x double> %res
 }
@@ -565,6 +760,36 @@ define <6 x float> @vector_interleave3_v632_v2f32(<2 x float> %a, <2 x float> %b
 ; ZVBB-NEXT:    addi sp, sp, 16
 ; ZVBB-NEXT:    .cfi_def_cfa_offset 0
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave3_v632_v2f32:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    addi sp, sp, -16
+; ZIP-NEXT:    .cfi_def_cfa_offset 16
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    sub sp, sp, a0
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZIP-NEXT:    addi a0, sp, 16
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    srli a1, a1, 1
+; ZIP-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
+; ZIP-NEXT:    vsseg3e32.v v8, (a0)
+; ZIP-NEXT:    add a2, a0, a1
+; ZIP-NEXT:    vle32.v v9, (a2)
+; ZIP-NEXT:    vle32.v v8, (a0)
+; ZIP-NEXT:    add a1, a2, a1
+; ZIP-NEXT:    vle32.v v10, (a1)
+; ZIP-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v9, 2
+; ZIP-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v10, 4
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    add sp, sp, a0
+; ZIP-NEXT:    .cfi_def_cfa sp, 16
+; ZIP-NEXT:    addi sp, sp, 16
+; ZIP-NEXT:    .cfi_def_cfa_offset 0
+; ZIP-NEXT:    ret
 	   %res = call <6 x float> @llvm.vector.interleave3.v6f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
 	   ret <6 x float> %res
 }
@@ -644,6 +869,43 @@ define <10 x half> @vector_interleave5_v10f16_v2f16(<2 x half> %a, <2 x half> %b
 ; ZVBB-NEXT:    addi sp, sp, 16
 ; ZVBB-NEXT:    .cfi_def_cfa_offset 0
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave5_v10f16_v2f16:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    addi sp, sp, -16
+; ZIP-NEXT:    .cfi_def_cfa_offset 16
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    sub sp, sp, a0
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZIP-NEXT:    addi a0, sp, 16
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    srli a1, a1, 2
+; ZIP-NEXT:    add a2, a0, a1
+; ZIP-NEXT:    add a3, a2, a1
+; ZIP-NEXT:    vsetvli a4, zero, e16, mf4, ta, ma
+; ZIP-NEXT:    vsseg5e16.v v8, (a0)
+; ZIP-NEXT:    add a4, a3, a1
+; ZIP-NEXT:    vle16.v v9, (a2)
+; ZIP-NEXT:    vle16.v v11, (a4)
+; ZIP-NEXT:    vle16.v v12, (a3)
+; ZIP-NEXT:    vle16.v v8, (a0)
+; ZIP-NEXT:    add a1, a4, a1
+; ZIP-NEXT:    vle16.v v10, (a1)
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZIP-NEXT:    vslideup.vi v12, v11, 2
+; ZIP-NEXT:    vslideup.vi v8, v9, 2
+; ZIP-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v12, 4
+; ZIP-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v10, 8
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    add sp, sp, a0
+; ZIP-NEXT:    .cfi_def_cfa sp, 16
+; ZIP-NEXT:    addi sp, sp, 16
+; ZIP-NEXT:    .cfi_def_cfa_offset 0
+; ZIP-NEXT:    ret
 	   %res = call <10 x half> @llvm.vector.interleave5.v10f16(<2 x half> %a, <2 x half> %b, <2 x half> %c, <2 x half> %d, <2 x half> %e)
 	   ret <10 x half> %res
 }
@@ -736,9 +998,50 @@ define <7 x half> @vector_interleave7_v7f16_v1f16(<1 x half> %a, <1 x half> %b,
 ; ZVBB-NEXT:    addi sp, sp, 16
 ; ZVBB-NEXT:    .cfi_def_cfa_offset 0
 ; ZVBB-NEXT:    ret
+;
+; ZIP-LABEL: vector_interleave7_v7f16_v1f16:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    addi sp, sp, -16
+; ZIP-NEXT:    .cfi_def_cfa_offset 16
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    sub sp, sp, a0
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; ZIP-NEXT:    addi a0, sp, 16
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    srli a1, a1, 2
+; ZIP-NEXT:    add a2, a0, a1
+; ZIP-NEXT:    add a3, a2, a1
+; ZIP-NEXT:    add a4, a3, a1
+; ZIP-NEXT:    vsetvli a5, zero, e16, mf4, ta, ma
+; ZIP-NEXT:    vsseg7e16.v v8, (a0)
+; ZIP-NEXT:    vle16.v v9, (a4)
+; ZIP-NEXT:    add a4, a4, a1
+; ZIP-NEXT:    vle16.v v10, (a2)
+; ZIP-NEXT:    add a2, a4, a1
+; ZIP-NEXT:    add a1, a2, a1
+; ZIP-NEXT:    vle16.v v11, (a2)
+; ZIP-NEXT:    vle16.v v12, (a4)
+; ZIP-NEXT:    vle16.v v8, (a0)
+; ZIP-NEXT:    vle16.v v13, (a1)
+; ZIP-NEXT:    vle16.v v14, (a3)
+; ZIP-NEXT:    vsetivli zero, 2, e16, mf2, tu, ma
+; ZIP-NEXT:    vslideup.vi v12, v11, 1
+; ZIP-NEXT:    vslideup.vi v8, v10, 1
+; ZIP-NEXT:    vsetivli zero, 3, e16, mf2, tu, ma
+; ZIP-NEXT:    vslideup.vi v12, v13, 2
+; ZIP-NEXT:    vslideup.vi v8, v14, 2
+; ZIP-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v9, 3
+; ZIP-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v12, 4
+; ZIP-NEXT:    csrr a0, vlenb
+; ZIP-NEXT:    slli a0, a0, 1
+; ZIP-NEXT:    add sp, sp, a0
+; ZIP-NEXT:    .cfi_def_cfa sp, 16
+; ZIP-NEXT:    addi sp, sp, 16
+; ZIP-NEXT:    .cfi_def_cfa_offset 0
+; ZIP-NEXT:    ret
 	   %res = call <7 x half> @llvm.vector.interleave7.v7f16(<1 x half> %a, <1 x half> %b, <1 x half> %c, <1 x half> %d, <1 x half> %e, <1 x half> %f, <1 x half> %g)
 	   ret <7 x half> %res
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RV32: {{.*}}
-; RV64: {{.*}}

From 50f9b34b5340cfb32d14920bb0d41a90f48ffc40 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 17 Apr 2025 19:51:35 -0700
Subject: [PATCH 355/710] [RISCV] Prefer vmv.s.x for build_vector a, undef,
 ..., undef (#136164)

If we have a build vector which could be either a splat or a scalar
insert, prefer the scalar insert. At high LMUL, this reduces vector
register pressure (locally, the use will likely still be aligned), and
the amount of work performed for the splat.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 18 ++++++++-
 .../RISCV/rvv/fixed-vectors-binop-splats.ll   | 40 +++++++++----------
 .../rvv/fixed-vectors-buildvec-of-binop.ll    |  4 +-
 .../rvv/fixed-vectors-shuffle-vslide1up.ll    |  2 +-
 .../RISCV/rvv/fixed-vectors-unaligned.ll      |  4 +-
 .../RISCV/rvv/fold-scalar-load-crash.ll       |  8 ++--
 llvm/test/CodeGen/RISCV/rvv/pr125306.ll       |  2 +-
 7 files changed, 46 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5d4df039a0ed8..98c8bdb4bc114 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -4208,8 +4208,22 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
   if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
     if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
       return Gather;
-    unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
-                                        : RISCVISD::VMV_V_X_VL;
+
+    // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
+    // pressure at high LMUL.
+    if (all_of(Op->ops().drop_front(),
+               [](const SDUse &U) { return U.get().isUndef(); })) {
+      unsigned Opc =
+          VT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
+      if (!VT.isFloatingPoint())
+        Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
+      Splat = DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
+                          Splat, VL);
+      return convertFromScalableVector(VT, Splat, DAG, Subtarget);
+    }
+
+    unsigned Opc =
+        VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
     if (!VT.isFloatingPoint())
       Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
     Splat =
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
index bfc43db2e369e..6b5ca5f391b80 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
@@ -187,7 +187,7 @@ define <2 x i8> @v2i8(i8 %x, i8 %y) {
 ; CHECK-LABEL: v2i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -203,7 +203,7 @@ define <4 x i8> @v4i8(i8 %x, i8 %y) {
 ; CHECK-LABEL: v4i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -219,7 +219,7 @@ define <8 x i8> @v8i8(i8 %x, i8 %y) {
 ; CHECK-LABEL: v8i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -235,7 +235,7 @@ define <16 x i8> @v16i8(i8 %x, i8 %y) {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -252,7 +252,7 @@ define <32 x i8> @v32i8(i8 %x, i8 %y) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a2, 32
 ; CHECK-NEXT:    vsetvli zero, a2, e8, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v10, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v10, 0
 ; CHECK-NEXT:    ret
@@ -269,7 +269,7 @@ define <64 x i8> @v64i8(i8 %x, i8 %y) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a2, 64
 ; CHECK-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v12, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v12, 0
 ; CHECK-NEXT:    ret
@@ -300,7 +300,7 @@ define <2 x i16> @v2i16(i16 %x, i16 %y) {
 ; CHECK-LABEL: v2i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -316,7 +316,7 @@ define <4 x i16> @v4i16(i16 %x, i16 %y) {
 ; CHECK-LABEL: v4i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -332,7 +332,7 @@ define <8 x i16> @v8i16(i16 %x, i16 %y) {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -348,7 +348,7 @@ define <16 x i16> @v16i16(i16 %x, i16 %y) {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v10, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v10, 0
 ; CHECK-NEXT:    ret
@@ -365,7 +365,7 @@ define <32 x i16> @v32i16(i16 %x, i16 %y) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a2, 32
 ; CHECK-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v12, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v12, 0
 ; CHECK-NEXT:    ret
@@ -396,7 +396,7 @@ define <2 x i32> @v2i32(i32 %x, i32 %y) {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -412,7 +412,7 @@ define <4 x i32> @v4i32(i32 %x, i32 %y) {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v9, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -428,7 +428,7 @@ define <8 x i32> @v8i32(i32 %x, i32 %y) {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v10, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v10, 0
 ; CHECK-NEXT:    ret
@@ -444,7 +444,7 @@ define <16 x i32> @v16i32(i32 %x, i32 %y) {
 ; CHECK-LABEL: v16i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    vadd.vx v12, v8, a1
 ; CHECK-NEXT:    vrgather.vi v8, v12, 0
 ; CHECK-NEXT:    ret
@@ -509,7 +509,7 @@ define <2 x i64> @v2i64(i64 %x, i64 %y) {
 ; RV64-LABEL: v2i64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    vmv.s.x v8, a0
 ; RV64-NEXT:    vadd.vx v9, v8, a1
 ; RV64-NEXT:    vrgather.vi v8, v9, 0
 ; RV64-NEXT:    ret
@@ -542,7 +542,7 @@ define <4 x i64> @v4i64(i64 %x, i64 %y) {
 ; RV64-LABEL: v4i64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    vmv.s.x v8, a0
 ; RV64-NEXT:    vadd.vx v10, v8, a1
 ; RV64-NEXT:    vrgather.vi v8, v10, 0
 ; RV64-NEXT:    ret
@@ -575,7 +575,7 @@ define <8 x i64> @v8i64(i64 %x, i64 %y) {
 ; RV64-LABEL: v8i64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    vmv.s.x v8, a0
 ; RV64-NEXT:    vadd.vx v12, v8, a1
 ; RV64-NEXT:    vrgather.vi v8, v12, 0
 ; RV64-NEXT:    ret
@@ -591,7 +591,7 @@ define <4 x half> @v4f16(half %x, half %y) {
 ; CHECK-LABEL: v4f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vfmv.v.f v8, fa0
+; CHECK-NEXT:    vfmv.s.f v8, fa0
 ; CHECK-NEXT:    vfadd.vf v9, v8, fa1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
@@ -607,7 +607,7 @@ define <2 x float> @v2f32(float %x, float %y) {
 ; CHECK-LABEL: v2f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vfmv.v.f v8, fa0
+; CHECK-NEXT:    vfmv.s.f v8, fa0
 ; CHECK-NEXT:    vfadd.vf v9, v8, fa1
 ; CHECK-NEXT:    vrgather.vi v8, v9, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
index dbbb8362144ca..c97545691180e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
@@ -439,7 +439,7 @@ define void @buggy(i32 %0) #0 {
 ; RV32-LABEL: buggy:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vmv.s.x v8, a0
 ; RV32-NEXT:    vadd.vv v8, v8, v8
 ; RV32-NEXT:    vor.vi v8, v8, 1
 ; RV32-NEXT:    vrgather.vi v9, v8, 0
@@ -450,7 +450,7 @@ define void @buggy(i32 %0) #0 {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    slli a0, a0, 1
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    vmv.s.x v8, a0
 ; RV64-NEXT:    vor.vi v8, v8, 1
 ; RV64-NEXT:    vrgather.vi v9, v8, 0
 ; RV64-NEXT:    vse32.v v9, (zero)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
index a171a7f8ac5f1..a29d53b50287b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
@@ -250,7 +250,7 @@ define <4 x double> @vslide1up_4xf64(<4 x double> %v, double %b) {
 ; CHECK-LABEL: vslide1up_4xf64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-NEXT:    vfmv.v.f v10, fa0
+; CHECK-NEXT:    vfmv.s.f v10, fa0
 ; CHECK-NEXT:    vslideup.vi v10, v8, 3
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
index 29fbb8acc3358..b6253c6ea63b2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
@@ -500,7 +500,7 @@ define void @masked_load_v2i32_align1(ptr %a, <2 x i32> %m, ptr %res_ptr) nounwi
 ; RV32-SLOW-NEXT:    or a4, a6, a5
 ; RV32-SLOW-NEXT:    or a3, a4, a3
 ; RV32-SLOW-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; RV32-SLOW-NEXT:    vmv.v.x v8, a3
+; RV32-SLOW-NEXT:    vmv.s.x v8, a3
 ; RV32-SLOW-NEXT:  .LBB8_2: # %else
 ; RV32-SLOW-NEXT:    andi a2, a2, 2
 ; RV32-SLOW-NEXT:    beqz a2, .LBB8_4
@@ -544,7 +544,7 @@ define void @masked_load_v2i32_align1(ptr %a, <2 x i32> %m, ptr %res_ptr) nounwi
 ; RV64-SLOW-NEXT:    or a4, a6, a5
 ; RV64-SLOW-NEXT:    or a3, a4, a3
 ; RV64-SLOW-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; RV64-SLOW-NEXT:    vmv.v.x v8, a3
+; RV64-SLOW-NEXT:    vmv.s.x v8, a3
 ; RV64-SLOW-NEXT:  .LBB8_2: # %else
 ; RV64-SLOW-NEXT:    andi a2, a2, 2
 ; RV64-SLOW-NEXT:    beqz a2, .LBB8_4
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll b/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
index 919c2fd518578..19ea7b7ca481e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
@@ -9,8 +9,8 @@ define i32 @test(i32 %size, ptr %add.ptr, i64 %const) {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi a3, a2, 1
 ; RV32-NEXT:    th.lbib a4, (a1), -1, 0
-; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a4
+; RV32-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
+; RV32-NEXT:    vmv.s.x v8, a4
 ; RV32-NEXT:    vmv.s.x v9, zero
 ; RV32-NEXT:    vsetvli zero, a3, e8, mf2, tu, ma
 ; RV32-NEXT:    vslideup.vx v8, v9, a2
@@ -35,8 +35,8 @@ define i32 @test(i32 %size, ptr %add.ptr, i64 %const) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    addi a3, a2, 1
 ; RV64-NEXT:    th.lbib a4, (a1), -1, 0
-; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a4
+; RV64-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
+; RV64-NEXT:    vmv.s.x v8, a4
 ; RV64-NEXT:    vmv.s.x v9, zero
 ; RV64-NEXT:    vsetvli zero, a3, e8, mf2, tu, ma
 ; RV64-NEXT:    vslideup.vx v8, v9, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr125306.ll b/llvm/test/CodeGen/RISCV/rvv/pr125306.ll
index 111f87de220db..9400c381bc87c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/pr125306.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/pr125306.ll
@@ -60,7 +60,7 @@ define <2 x i32> @main(ptr %0) {
 ; CHECK-NEXT:    vslide1down.vx v8, v8, zero
 ; CHECK-NEXT:    vslide1down.vx v10, v10, zero
 ; CHECK-NEXT:    vmin.vv v8, v10, v8
-; CHECK-NEXT:    vmv.v.x v10, a0
+; CHECK-NEXT:    vmv.s.x v10, a0
 ; CHECK-NEXT:    vslide1down.vx v11, v11, zero
 ; CHECK-NEXT:    vmin.vx v10, v10, a2
 ; CHECK-NEXT:    vmin.vx v10, v10, a1

From a04580f71b98bdb12100da66c9975e9a1001b4d6 Mon Sep 17 00:00:00 2001
From: Shoreshen <372660931@qq.com>
Date: Fri, 18 Apr 2025 10:56:20 +0800
Subject: [PATCH 356/710] [AMDGPU] Implement vop3p complex pattern optmization
 for gisel (#130234)

Seeking opportunities to optimize VOP3P instructions by altering opsel,
opsel_hi, neg, neg_hi bits

Tests differences:
1. fix op_sel_hi bit for inline constant:
   1. `CodeGen/AMDGPU/packed-fp32.ll`
2. use neg bit to remove xor with 0x80008000
   1. `CodeGen/AMDGPU/strict_fsub.f16.ll`
   2. `CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fdot2.ll`
   3. `CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll`
   4. `CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot8.ll`
   5. `CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll`
   6. `CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll`
   7. `CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot8.ll`
3. Remove xor 0x80008000, and use opsel, opsel_hi to remove alignbit
   1. `CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll`
---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      | 604 +++++++++++++++++-
 .../Target/AMDGPU/AMDGPUInstructionSelector.h |   6 +-
 .../CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll   | 163 +++++
 .../AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll    |  12 +-
 .../AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll    |  18 +-
 llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll   |  14 +-
 6 files changed, 753 insertions(+), 64 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 6ef7505ec6f62..87af467ac8f1e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4319,60 +4319,598 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
   }};
 }
 
-std::pair<Register, unsigned>
-AMDGPUInstructionSelector::selectVOP3PModsImpl(
-  Register Src, const MachineRegisterInfo &MRI, bool IsDOT) const {
+enum class SrcStatus {
+  IS_SAME,
+  IS_UPPER_HALF,
+  IS_LOWER_HALF,
+  IS_UPPER_HALF_NEG,
+  // This means current op = [op_upper, op_lower] and src = -op_lower.
+  IS_LOWER_HALF_NEG,
+  IS_HI_NEG,
+  // This means current op = [op_upper, op_lower] and src = [op_upper,
+  // -op_lower].
+  IS_LO_NEG,
+  IS_BOTH_NEG,
+  INVALID,
+  NEG_START = IS_UPPER_HALF_NEG,
+  NEG_END = IS_BOTH_NEG,
+  HALF_START = IS_UPPER_HALF,
+  HALF_END = IS_LOWER_HALF_NEG
+};
+
+static bool isTruncHalf(const MachineInstr *MI,
+                        const MachineRegisterInfo &MRI) {
+  if (MI->getOpcode() != AMDGPU::G_TRUNC)
+    return false;
+
+  unsigned DstSize = MRI.getType(MI->getOperand(0).getReg()).getSizeInBits();
+  unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+  return DstSize * 2 == SrcSize;
+}
+
+static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
+  if (MI->getOpcode() != AMDGPU::G_LSHR)
+    return false;
+
+  Register ShiftSrc;
+  std::optional<ValueAndVReg> ShiftAmt;
+  if (mi_match(MI->getOperand(0).getReg(), MRI,
+               m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
+    unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+    unsigned Shift = ShiftAmt->Value.getZExtValue();
+    return Shift * 2 == SrcSize;
+  }
+  return false;
+}
+
+static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
+  if (MI->getOpcode() != AMDGPU::G_SHL)
+    return false;
+
+  Register ShiftSrc;
+  std::optional<ValueAndVReg> ShiftAmt;
+  if (mi_match(MI->getOperand(0).getReg(), MRI,
+               m_GShl(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
+    unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+    unsigned Shift = ShiftAmt->Value.getZExtValue();
+    return Shift * 2 == SrcSize;
+  }
+  return false;
+}
+
+static std::optional<std::pair<const MachineOperand *, SrcStatus>>
+retOpStat(const MachineOperand *Op, SrcStatus Stat,
+          std::pair<const MachineOperand *, SrcStatus> &Curr) {
+  if (Stat != SrcStatus::INVALID &&
+      ((Op->isReg() && !(Op->getReg().isPhysical())) || Op->isImm() ||
+       Op->isCImm() || Op->isFPImm())) {
+    return std::optional<std::pair<const MachineOperand *, SrcStatus>>(
+        {Op, Stat});
+  }
+
+  return std::nullopt;
+}
+
+enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
+
+static TypeClass isVectorOfTwoOrScalar(const MachineOperand *Op,
+                                       const MachineRegisterInfo &MRI) {
+  if (!Op->isReg() || Op->getReg().isPhysical())
+    return TypeClass::NONE_OF_LISTED;
+  LLT OpTy = MRI.getType(Op->getReg());
+  if (OpTy.isScalar())
+    return TypeClass::SCALAR;
+  if (OpTy.isVector() && OpTy.getNumElements() == 2)
+    return TypeClass::VECTOR_OF_TWO;
+  return TypeClass::NONE_OF_LISTED;
+}
+
+static SrcStatus getNegStatus(const MachineOperand *Op, SrcStatus S,
+                              const MachineRegisterInfo &MRI) {
+  TypeClass NegType = isVectorOfTwoOrScalar(Op, MRI);
+  if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
+    return SrcStatus::INVALID;
+
+  switch (S) {
+  case SrcStatus::IS_SAME:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // [SrcHi, SrcLo]   = [CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
+      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+      // [SrcHi, SrcLo]   = [-OpHi, -OpLo]
+      return SrcStatus::IS_BOTH_NEG;
+    } else if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // [SrcHi, SrcLo]   = [CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
+      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+      // [SrcHi, SrcLo]   = [-OpHi, OpLo]
+      return SrcStatus::IS_HI_NEG;
+    }
+    break;
+  case SrcStatus::IS_HI_NEG:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
+      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+      // [SrcHi, SrcLo]   = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
+      return SrcStatus::IS_LO_NEG;
+    } else if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
+      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+      // [SrcHi, SrcLo]   = [-(-OpHi), OpLo] = [OpHi, OpLo]
+      return SrcStatus::IS_SAME;
+    }
+    break;
+  case SrcStatus::IS_LO_NEG:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // [SrcHi, SrcLo]   = [CurrHi, -CurrLo]
+      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
+      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+      // [SrcHi, SrcLo]   = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
+      return SrcStatus::IS_HI_NEG;
+    } else if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // [SrcHi, SrcLo]   = [CurrHi, -CurrLo]
+      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
+      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+      // [SrcHi, SrcLo]   = [-OpHi, -OpLo]
+      return SrcStatus::IS_BOTH_NEG;
+    }
+    break;
+  case SrcStatus::IS_BOTH_NEG:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // [SrcHi, SrcLo]   = [-CurrHi, -CurrLo]
+      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
+      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+      // [SrcHi, SrcLo]   = [OpHi, OpLo]
+      return SrcStatus::IS_SAME;
+    } else if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // [SrcHi, SrcLo]   = [-CurrHi, -CurrLo]
+      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
+      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+      // [SrcHi, SrcLo]   = [OpHi, -OpLo]
+      return SrcStatus::IS_LO_NEG;
+    }
+    break;
+  case SrcStatus::IS_UPPER_HALF:
+    // Vector of 2:
+    // Src = CurrUpper
+    // Curr = [CurrUpper, CurrLower]
+    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+    // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+    // Src = -OpUpper
+    //
+    // Scalar:
+    // Src = CurrUpper
+    // Curr = [CurrUpper, CurrLower]
+    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+    // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+    // Src = -OpUpper
+    return SrcStatus::IS_UPPER_HALF_NEG;
+  case SrcStatus::IS_LOWER_HALF:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // Src = CurrLower
+      // Curr = [CurrUpper, CurrLower]
+      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+      // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+      // Src = -OpLower
+      return SrcStatus::IS_LOWER_HALF_NEG;
+    } else if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // Src = CurrLower
+      // Curr = [CurrUpper, CurrLower]
+      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+      // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+      // Src = OpLower
+      return SrcStatus::IS_LOWER_HALF;
+    }
+    break;
+  case SrcStatus::IS_UPPER_HALF_NEG:
+    // Vector of 2:
+    // Src = -CurrUpper
+    // Curr = [CurrUpper, CurrLower]
+    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+    // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+    // Src = -(-OpUpper) = OpUpper
+    //
+    // Scalar:
+    // Src = -CurrUpper
+    // Curr = [CurrUpper, CurrLower]
+    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+    // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+    // Src = -(-OpUpper) = OpUpper
+    return SrcStatus::IS_UPPER_HALF;
+  case SrcStatus::IS_LOWER_HALF_NEG:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // Src = -CurrLower
+      // Curr = [CurrUpper, CurrLower]
+      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+      // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+      // Src = -(-OpLower) = OpLower
+      return SrcStatus::IS_LOWER_HALF;
+    } else if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // Src = -CurrLower
+      // Curr = [CurrUpper, CurrLower]
+      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+      // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+      // Src = -OpLower
+      return SrcStatus::IS_LOWER_HALF_NEG;
+    }
+    break;
+  default:
+    llvm_unreachable("unexpected SrcStatus");
+  }
+}
+
+static std::optional<std::pair<const MachineOperand *, SrcStatus>>
+calcNextStatus(std::pair<const MachineOperand *, SrcStatus> Curr,
+               const MachineRegisterInfo &MRI) {
+  if (!Curr.first->isReg())
+    return std::nullopt;
+
+  const MachineInstr *MI = Curr.first->isDef()
+                               ? Curr.first->getParent()
+                               : MRI.getVRegDef(Curr.first->getReg());
+
+  unsigned Opc = MI->getOpcode();
+
+  // Handle general Opc cases.
+  switch (Opc) {
+  case AMDGPU::G_BITCAST:
+  case AMDGPU::G_CONSTANT:
+  case AMDGPU::G_FCONSTANT:
+  case AMDGPU::COPY:
+    return retOpStat(&MI->getOperand(1), Curr.second, Curr);
+  case AMDGPU::G_FNEG:
+    return retOpStat(&MI->getOperand(1),
+                     getNegStatus(Curr.first, Curr.second, MRI), Curr);
+  default:
+    break;
+  }
+
+  // Calc next Stat from current Stat.
+  switch (Curr.second) {
+  case SrcStatus::IS_SAME:
+    if (isTruncHalf(MI, MRI))
+      return retOpStat(&MI->getOperand(1), SrcStatus::IS_LOWER_HALF, Curr);
+    break;
+  case SrcStatus::IS_HI_NEG:
+    if (isTruncHalf(MI, MRI)) {
+      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
+      //                  = [OpLowerHi, OpLowerLo]
+      // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
+      //     = [-OpLowerHi, OpLowerLo]
+      //     = -OpLower
+      return retOpStat(&MI->getOperand(1), SrcStatus::IS_LOWER_HALF_NEG, Curr);
+    }
+    break;
+  case SrcStatus::IS_UPPER_HALF:
+    if (isShlHalf(MI, MRI))
+      return retOpStat(&MI->getOperand(1), SrcStatus::IS_LOWER_HALF, Curr);
+    break;
+  case SrcStatus::IS_LOWER_HALF:
+    if (isLshrHalf(MI, MRI))
+      return retOpStat(&MI->getOperand(1), SrcStatus::IS_UPPER_HALF, Curr);
+    break;
+  case SrcStatus::IS_UPPER_HALF_NEG:
+    if (isShlHalf(MI, MRI))
+      return retOpStat(&MI->getOperand(1), SrcStatus::IS_LOWER_HALF_NEG, Curr);
+    break;
+  case SrcStatus::IS_LOWER_HALF_NEG:
+    if (isLshrHalf(MI, MRI))
+      return retOpStat(&MI->getOperand(1), SrcStatus::IS_UPPER_HALF_NEG, Curr);
+    break;
+  default:
+    break;
+  }
+  return std::nullopt;
+}
+
+class searchOptions {
+private:
+  bool HasNeg = false;
+  // Assume all complex pattern of VOP3P has opsel.
+  bool HasOpsel = true;
+
+public:
+  searchOptions(const MachineOperand *RootOp, const MachineRegisterInfo &MRI) {
+    const MachineInstr *MI = RootOp->getParent();
+    unsigned Opc = MI->getOpcode();
+
+    if (Opc < TargetOpcode::GENERIC_OP_END) {
+      // Keep same for generic op.
+      HasNeg = true;
+    } else if (Opc == TargetOpcode::G_INTRINSIC) {
+      Intrinsic::ID IntrinsicID = cast<GIntrinsic>(*MI).getIntrinsicID();
+      // Only float point intrinsic has neg & neg_hi bits.
+      if (IntrinsicID == Intrinsic::amdgcn_fdot2)
+        HasNeg = true;
+    }
+  }
+  bool checkOptions(SrcStatus Stat) const {
+    if (!HasNeg &&
+        (Stat >= SrcStatus::NEG_START || Stat <= SrcStatus::NEG_END)) {
+      return false;
+    }
+    if (!HasOpsel &&
+        (Stat >= SrcStatus::HALF_START || Stat >= SrcStatus::HALF_END)) {
+      return false;
+    }
+    return true;
+  }
+};
+
+static SmallVector<std::pair<const MachineOperand *, SrcStatus>>
+getSrcStats(const MachineOperand *Op, const MachineRegisterInfo &MRI,
+            searchOptions SearchOptions, int MaxDepth = 6) {
+  int Depth = 0;
+  auto Curr = calcNextStatus({Op, SrcStatus::IS_SAME}, MRI);
+  SmallVector<std::pair<const MachineOperand *, SrcStatus>, 4> Statlist;
+
+  while (Depth <= MaxDepth && Curr.has_value()) {
+    Depth++;
+    if (SearchOptions.checkOptions(Curr.value().second))
+      Statlist.push_back(Curr.value());
+    Curr = calcNextStatus(Curr.value(), MRI);
+  }
+
+  return Statlist;
+}
+
+static std::pair<const MachineOperand *, SrcStatus>
+getLastSameOrNeg(const MachineOperand *Op, const MachineRegisterInfo &MRI,
+                 searchOptions SearchOptions, int MaxDepth = 6) {
+  int Depth = 0;
+  std::pair<const MachineOperand *, SrcStatus> LastSameOrNeg = {
+      Op, SrcStatus::IS_SAME};
+  auto Curr = calcNextStatus(LastSameOrNeg, MRI);
+
+  while (Depth <= MaxDepth && Curr.has_value()) {
+    Depth++;
+    if (SearchOptions.checkOptions(Curr.value().second)) {
+      if (Curr.value().second == SrcStatus::IS_SAME ||
+          Curr.value().second == SrcStatus::IS_HI_NEG ||
+          Curr.value().second == SrcStatus::IS_LO_NEG ||
+          Curr.value().second == SrcStatus::IS_BOTH_NEG)
+        LastSameOrNeg = Curr.value();
+    }
+    Curr = calcNextStatus(Curr.value(), MRI);
+  }
+
+  return LastSameOrNeg;
+}
+
+static bool isInlinableFPConstant(const MachineOperand &Op,
+                                  const SIInstrInfo &TII) {
+  return Op.isFPImm() && TII.isInlineConstant(Op.getFPImm()->getValueAPF());
+}
+
+static bool isSameBitWidth(const MachineOperand *Op1, const MachineOperand *Op2,
+                           const MachineRegisterInfo &MRI) {
+  unsigned Width1 = MRI.getType(Op1->getReg()).getSizeInBits();
+  unsigned Width2 = MRI.getType(Op2->getReg()).getSizeInBits();
+  return Width1 == Width2;
+}
+
+static bool isSameOperand(const MachineOperand *Op1,
+                          const MachineOperand *Op2) {
+  if (Op1->isReg())
+    return Op2->isReg() && Op1->getReg() == Op2->getReg();
+
+  return Op1->isIdenticalTo(*Op2);
+}
+
+static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
+  // SrcStatus::IS_LOWER_HALF remain 0.
+  if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
+    Mods ^= SISrcMods::NEG_HI;
+    Mods |= SISrcMods::OP_SEL_1;
+  } else if (HiStat == SrcStatus::IS_UPPER_HALF)
+    Mods |= SISrcMods::OP_SEL_1;
+  else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
+    Mods ^= SISrcMods::NEG_HI;
+  else if (HiStat == SrcStatus::IS_HI_NEG)
+    Mods ^= SISrcMods::NEG_HI;
+
+  if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
+    Mods ^= SISrcMods::NEG;
+    Mods |= SISrcMods::OP_SEL_0;
+  } else if (LoStat == SrcStatus::IS_UPPER_HALF)
+    Mods |= SISrcMods::OP_SEL_0;
+  else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
+    Mods |= SISrcMods::NEG;
+  else if (LoStat == SrcStatus::IS_HI_NEG)
+    Mods ^= SISrcMods::NEG;
+
+  return Mods;
+}
+
+static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat,
+                          const MachineOperand *NewOp,
+                          const MachineOperand *RootOp, const SIInstrInfo &TII,
+                          const MachineRegisterInfo &MRI) {
+  if (NewOp->isReg()) {
+    auto IsHalfState = [](SrcStatus S) {
+      return S == SrcStatus::IS_UPPER_HALF ||
+             S == SrcStatus::IS_UPPER_HALF_NEG ||
+             S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
+    };
+    return isSameBitWidth(NewOp, RootOp, MRI) && IsHalfState(LoStat) &&
+           IsHalfState(HiStat);
+  } else
+    return ((HiStat == SrcStatus::IS_SAME || HiStat == SrcStatus::IS_HI_NEG) &&
+            (LoStat == SrcStatus::IS_SAME || LoStat == SrcStatus::IS_HI_NEG) &&
+            isInlinableFPConstant(*NewOp, TII));
+
+  return false;
+}
+
+std::pair<const MachineOperand *, unsigned>
+AMDGPUInstructionSelector::selectVOP3PModsImpl(const MachineOperand *RootOp,
+                                               const MachineRegisterInfo &MRI,
+                                               bool IsDOT) const {
   unsigned Mods = 0;
-  MachineInstr *MI = MRI.getVRegDef(Src);
+  const MachineOperand *Op = RootOp;
+  // No modification if Root type is not form of <2 x Type>.
+  if (isVectorOfTwoOrScalar(Op, MRI) != TypeClass::VECTOR_OF_TWO) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {Op, Mods};
+  }
+
+  searchOptions SearchOptions(Op, MRI);
 
-  if (MI->getOpcode() == AMDGPU::G_FNEG &&
-      // It's possible to see an f32 fneg here, but unlikely.
-      // TODO: Treat f32 fneg as only high bit.
-      MRI.getType(Src) == LLT::fixed_vector(2, 16)) {
+  std::pair<const MachineOperand *, SrcStatus> Stat =
+      getLastSameOrNeg(Op, MRI, SearchOptions);
+  if (!Stat.first->isReg()) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {Op, Mods};
+  }
+  if (Stat.second == SrcStatus::IS_BOTH_NEG)
     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
-    Src = MI->getOperand(1).getReg();
-    MI = MRI.getVRegDef(Src);
+  else if (Stat.second == SrcStatus::IS_HI_NEG)
+    Mods ^= SISrcMods::NEG_HI;
+  else if (Stat.second == SrcStatus::IS_LO_NEG)
+    Mods ^= SISrcMods::NEG;
+
+  Op = Stat.first;
+  MachineInstr *MI = MRI.getVRegDef(Op->getReg());
+
+  if (MI->getOpcode() != AMDGPU::G_BUILD_VECTOR || MI->getNumOperands() != 3 ||
+      (IsDOT && Subtarget->hasDOTOpSelHazard())) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {Op, Mods};
   }
 
-  // TODO: Handle G_FSUB 0 as fneg
+  SmallVector<std::pair<const MachineOperand *, SrcStatus>> StatlistHi =
+      getSrcStats(&MI->getOperand(2), MRI, SearchOptions);
 
-  // TODO: Match op_sel through g_build_vector_trunc and g_shuffle_vector.
-  (void)IsDOT; // DOTs do not use OPSEL on gfx942+, check ST.hasDOTOpSelHazard()
+  if (StatlistHi.size() == 0) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {Op, Mods};
+  }
+
+  SmallVector<std::pair<const MachineOperand *, SrcStatus>> StatlistLo =
+      getSrcStats(&MI->getOperand(1), MRI, SearchOptions);
 
+  if (StatlistLo.size() == 0) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {Op, Mods};
+  }
+
+  for (int I = StatlistHi.size() - 1; I >= 0; I--) {
+    for (int J = StatlistLo.size() - 1; J >= 0; J--) {
+      if (isSameOperand(StatlistHi[I].first, StatlistLo[J].first) &&
+          isValidToPack(StatlistHi[I].second, StatlistLo[J].second,
+                        StatlistHi[I].first, RootOp, TII, MRI))
+        return {StatlistHi[I].first,
+                updateMods(StatlistHi[I].second, StatlistLo[J].second, Mods)};
+    }
+  }
   // Packed instructions do not have abs modifiers.
   Mods |= SISrcMods::OP_SEL_1;
 
-  return std::pair(Src, Mods);
+  return {Op, Mods};
 }
 
-InstructionSelector::ComplexRendererFns
-AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI
-    = Root.getParent()->getParent()->getParent()->getRegInfo();
+int64_t getAllKindImm(const MachineOperand *Op) {
+  switch (Op->getType()) {
+  case MachineOperand::MachineOperandType::MO_Immediate:
+    return Op->getImm();
+  case MachineOperand::MachineOperandType::MO_CImmediate:
+    return Op->getCImm()->getSExtValue();
+  case MachineOperand::MachineOperandType::MO_FPImmediate:
+    return Op->getFPImm()->getValueAPF().bitcastToAPInt().getSExtValue();
+  default:
+    llvm_unreachable("not an imm type");
+  }
+}
+
+static bool checkRB(const MachineOperand *Op, unsigned int RBNo,
+                    const AMDGPURegisterBankInfo &RBI,
+                    const MachineRegisterInfo &MRI,
+                    const TargetRegisterInfo &TRI) {
+  const RegisterBank *RB = RBI.getRegBank(Op->getReg(), MRI, TRI);
+  return RB->getID() == RBNo;
+}
+
+// This function is used to get the correct register bank for returned reg.
+// Assume:
+// 1. VOP3P is always legal for VGPR.
+// 2. RootOp's regbank is legal.
+// Thus
+// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
+// 2. If RootOp is VGPR, then NewOp must be VGPR.
+static const MachineOperand *
+getLegalRegBank(const MachineOperand *NewOp, const MachineOperand *RootOp,
+                const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI,
+                const TargetRegisterInfo &TRI, const SIInstrInfo &TII) {
+  // RootOp can only be VGPR or SGPR (some hand written cases such as.
+  // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
+  if (checkRB(RootOp, AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
+      checkRB(NewOp, AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
+    return NewOp;
+
+  MachineInstr *MI = MRI.getVRegDef(RootOp->getReg());
+  if (MI->getOpcode() == AMDGPU::COPY &&
+      isSameOperand(NewOp, &MI->getOperand(1))) {
+    // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
+    return RootOp;
+  }
+
+  MachineBasicBlock *BB = MI->getParent();
+  const TargetRegisterClass *DstRC =
+      TRI.getConstrainedRegClassForOperand(*RootOp, MRI);
+  Register DstReg = MRI.createVirtualRegister(DstRC);
 
-  Register Src;
-  unsigned Mods;
-  std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI);
+  MachineInstrBuilder MIB =
+      BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg)
+          .addReg(NewOp->getReg());
+
+  // only accept VGPR.
+  return &MIB->getOperand(0);
+}
 
+InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
+                                                bool IsDOT) const {
+  MachineRegisterInfo &MRI =
+      Root.getParent()->getParent()->getParent()->getRegInfo();
+  auto [Op, Mods] = selectVOP3PModsImpl(&Root, MRI, IsDOT);
+  if (!(Op->isReg()))
+    return {{
+        [=](MachineInstrBuilder &MIB) { MIB.addImm(getAllKindImm(Op)); },
+        [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
+    }};
+
+  Op = getLegalRegBank(Op, &Root, RBI, MRI, TRI, TII);
   return {{
-      [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }  // src_mods
+      [=](MachineInstrBuilder &MIB) { MIB.addReg(Op->getReg()); },
+      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
   }};
 }
 
 InstructionSelector::ComplexRendererFns
-AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI
-    = Root.getParent()->getParent()->getParent()->getRegInfo();
+AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
 
-  Register Src;
-  unsigned Mods;
-  std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true);
+  return selectVOP3PRetHelper(Root);
+}
 
-  return {{
-      [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }  // src_mods
-  }};
+InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
+
+  return selectVOP3PRetHelper(Root, true);
 }
 
 InstructionSelector::ComplexRendererFns
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 6c3f3026e877a..a224f39f88996 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -187,9 +187,11 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
 
   ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
 
-  std::pair<Register, unsigned>
-  selectVOP3PModsImpl(Register Src, const MachineRegisterInfo &MRI,
+  std::pair<const MachineOperand *, unsigned>
+  selectVOP3PModsImpl(const MachineOperand *Op, const MachineRegisterInfo &MRI,
                       bool IsDOT = false) const;
+  InstructionSelector::ComplexRendererFns
+  selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const;
 
   InstructionSelector::ComplexRendererFns
   selectVOP3PMods(MachineOperand &Root) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
index 543f8e413abd8..534b454775502 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
@@ -106,6 +106,169 @@ define <2 x half> @v_fmul_v2f16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %mul
 }
 
+define <2 x half> @v_fmul_v2f16_partial_neg(<2 x half> %a, <2 x half> %b) {
+; GFX9-LABEL: v_fmul_v2f16_partial_neg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_hi:[1,0]
+; GFX9-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_lo:[1,0]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fmul_v2f16_partial_neg:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80008000, v1
+; GFX8-NEXT:    v_mul_f16_e32 v3, v1, v0
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mul_f16_e32 v1, v2, v3
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fmul_v2f16_partial_neg:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_hi:[1,0]
+; GFX10-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_lo:[1,0]
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %b1 = bitcast <2 x half> %b to float
+  %b2 = fneg float %b1
+  %b3 = bitcast float %b2 to <2 x half>
+  %b4 = fneg <2 x half> %b3
+  %mul1 = fmul <2 x half> %b3, %a
+  %mul2 = fmul <2 x half> %b4, %mul1
+  ret <2 x half> %mul2
+}
+
+define <2 x half> @fmul_v2_half_neg_hi(<2 x half> %a, <2 x half> %b) #0 {
+; GFX9-LABEL: fmul_v2_half_neg_hi:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1 neg_hi:[0,1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: fmul_v2_half_neg_hi:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fmul_v2_half_neg_hi:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1 neg_hi:[0,1]
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %b1 = bitcast <2 x half> %b to float
+  %b2 = fneg float %b1
+  %b3 = bitcast float %b2 to <2 x half>
+  %b4 = extractelement <2 x half> %b3, i64 1
+  %tmp = insertelement <2 x half> poison, half %b4, i64 0
+  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
+  %mul = fmul <2 x half> %a, %k
+  ret <2 x half> %mul
+}
+
+define <2 x half> @fmul_v2_half_neg_hi1(<2 x half> %a, <2 x half> %b) #0 {
+; GFX9-LABEL: fmul_v2_half_neg_hi1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: fmul_v2_half_neg_hi1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fmul_v2_half_neg_hi1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %b1 = bitcast <2 x half> %b to float
+  %b2 = fneg float %b1
+  %b3 = bitcast float %b2 to <2 x half>
+  %b4 = fneg <2 x half> %b3
+  %b5 = extractelement <2 x half> %b4, i64 1
+  %tmp = insertelement <2 x half> poison, half %b5, i64 0
+  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
+  %mul = fmul <2 x half> %a, %k
+  ret <2 x half> %mul
+}
+
+define <2 x half> @fmul_v2_half_neg_lo(<2 x half> %a, <2 x half> %b) #0 {
+; GFX9-LABEL: fmul_v2_half_neg_lo:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0] neg_hi:[0,1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: fmul_v2_half_neg_lo:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fmul_v2_half_neg_lo:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0] neg_hi:[0,1]
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %b1 = bitcast <2 x half> %b to float
+  %b2 = fneg float %b1
+  %b3 = bitcast float %b2 to <2 x half>
+  %b4 = fneg <2 x half> %b3
+  %b5 = extractelement <2 x half> %b4, i64 0
+  %tmp = insertelement <2 x half> poison, half %b5, i64 0
+  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
+  %mul = fmul <2 x half> %a, %k
+  ret <2 x half> %mul
+}
+
+define <2 x half> @fmul_v2_half_neg_lo1(<2 x half> %a, <2 x half> %b) #0 {
+; GFX9-LABEL: fmul_v2_half_neg_lo1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: fmul_v2_half_neg_lo1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fmul_v2_half_neg_lo1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0]
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %b1 = bitcast <2 x half> %b to float
+  %b2 = fneg float %b1
+  %b3 = bitcast float %b2 to <2 x half>
+  %b4 = extractelement <2 x half> %b3, i64 0
+  %tmp = insertelement <2 x half> poison, half %b4, i64 0
+  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
+  %mul = fmul <2 x half> %a, %k
+  ret <2 x half> %mul
+}
+
 define <3 x half> @v_fmul_v3f16(<3 x half> %a, <3 x half> %b) {
 ; GFX9-LABEL: v_fmul_v3f16:
 ; GFX9:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
index 744a5b7feb48d..8f0ae8c47098a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
@@ -304,8 +304,7 @@ define i32 @v_sdot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_sdot2_shuffle10_a:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
+; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_shuffle10_a:
@@ -319,8 +318,7 @@ define i32 @v_sdot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-LABEL: v_sdot2_shuffle10_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
+; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.a = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %shuf.a, <2 x i16> %b, i32 %c, i1 false)
@@ -331,8 +329,7 @@ define i32 @v_sdot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_sdot2_shuffle10_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
+; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_shuffle10_b:
@@ -346,8 +343,7 @@ define i32 @v_sdot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-LABEL: v_sdot2_shuffle10_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
+; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.b = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %shuf.b, i32 %c, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
index 9e623494a5a04..287a009ca1405 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
@@ -289,22 +289,19 @@ define i32 @v_udot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_udot2_shuffle10_a:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_shuffle10_a:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_shuffle10_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.a = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %shuf.a, <2 x i16> %b, i32 %c, i1 false)
@@ -315,22 +312,19 @@ define i32 @v_udot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_udot2_shuffle10_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_shuffle10_b:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_shuffle10_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.b = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %shuf.b, i32 %c, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 2f1dfa11fd34d..141b86a24c1c4 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -742,9 +742,8 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX9-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-GISEL-NEXT:    v_pk_add_f16 v0, s2, v0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT:    v_pk_add_f16 v0, s2, v0 neg_lo:[0,1] neg_hi:[0,1]
 ; GFX9-GISEL-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
@@ -784,8 +783,7 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX10-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
-; GFX10-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
+; GFX10-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
 ; GFX10-GISEL-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-SDAG-TRUE16-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
@@ -810,8 +808,7 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX11-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
-; GFX11-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
+; GFX11-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
 ; GFX11-GISEL-NEXT:    ; return to shader part epilog
 ; GFX10PLUS-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10PLUS-SDAG:       ; %bb.0:
@@ -824,8 +821,7 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ; GFX10PLUS-SDAG-NEXT:    ; return to shader part epilog
 ; GFX10PLUS-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10PLUS-GISEL:       ; %bb.0:
-; GFX10PLUS-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
-; GFX10PLUS-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
+; GFX10PLUS-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
 ; GFX10PLUS-GISEL-NEXT:    ; return to shader part epilog
   %val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret <2 x half> %val

From e5263e3ec83cf662f945a067ffc83e2ed76d9aed Mon Sep 17 00:00:00 2001
From: Shih-Po Hung <shihpo.hung@sifive.com>
Date: Fri, 18 Apr 2025 10:57:11 +0800
Subject: [PATCH 357/710] [LV][NFC] Clean up tail-folding check for early-exit
 loops (#133931)

This patch moves the check for a single latch exit from computeMaxVF()
to LoopVectorizationLegality::canFoldTailByMasking(), as it duplicates
the logic when foldTailByMasking() returns false.

It also updates the NoScalarEpilogueNeeded logic to return false for
loops that are neither single-latch-exit nor early-exit. This avoids
applying tail-folding in unsupported cases and prevents triggering
assertions during analysis.
---
 .../Vectorize/LoopVectorizationLegality.cpp   | 10 ++++++
 .../Transforms/Vectorize/LoopVectorize.cpp    | 31 ++++++-------------
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 3ec6850d6f685..8e09e6f8d4935 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1872,6 +1872,16 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
 }
 
 bool LoopVectorizationLegality::canFoldTailByMasking() const {
+  // The only loops we can vectorize without a scalar epilogue, are loops with
+  // a bottom-test and a single exiting block. We'd have to handle the fact
+  // that not every instruction executes on the last iteration.  This will
+  // require a lane mask which varies through the vector loop body.  (TODO)
+  if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
+    LLVM_DEBUG(
+        dbgs()
+        << "LV: Cannot fold tail by masking. Requires a singe latch exit\n");
+    return false;
+  }
 
   LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index db2fd300cb8f5..87fcf7e5d9742 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3968,22 +3968,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     break;
   }
 
-  // The only loops we can vectorize without a scalar epilogue, are loops with
-  // a bottom-test and a single exiting block. We'd have to handle the fact
-  // that not every instruction executes on the last iteration.  This will
-  // require a lane mask which varies through the vector loop body.  (TODO)
-  if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
-    // If there was a tail-folding hint/switch, but we can't fold the tail by
-    // masking, fallback to a vectorization with a scalar epilogue.
-    if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
-      LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
-                           "scalar epilogue instead.\n");
-      ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
-      return computeFeasibleMaxVF(MaxTC, UserVF, false);
-    }
-    return FixedScalableVFPair::getNone();
-  }
-
   // Now try the tail folding
 
   // Invalidate interleave groups that require an epilogue if we can't mask
@@ -4013,14 +3997,19 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   }
 
   auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
+    // Return false if the loop is neither a single-latch-exit loop nor an
+    // early-exit loop as tail-folding is not supported in that case.
+    if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
+        !Legal->hasUncountableEarlyExit())
+      return false;
     unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
     ScalarEvolution *SE = PSE.getSE();
-    // Currently only loops with countable exits are vectorized, but calling
-    // getSymbolicMaxBackedgeTakenCount allows enablement work for loops with
-    // uncountable exits whilst also ensuring the symbolic maximum and known
-    // back-edge taken count remain identical for loops with countable exits.
+    // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
+    // with uncountable exits. For countable loops, the symbolic maximum must
+    // remain identical to the known back-edge taken count.
     const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
-    assert(BackedgeTakenCount == PSE.getBackedgeTakenCount() &&
+    assert((Legal->hasUncountableEarlyExit() ||
+            BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
            "Invalid loop count");
     const SCEV *ExitCount = SE->getAddExpr(
         BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));

From 0977a7130b291304dae5cfd23575ea3a4b9922ed Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Thu, 17 Apr 2025 19:58:32 -0700
Subject: [PATCH 358/710] [BOLT] Skip FDE emission for patch functions
 (#136224)

Patch functions are used to fix instructions in the original code, i.e.,
they are not functions in a traditional sense, but rather pieces of
emitted code that are embedded into real functions.

We used to emit FDEs for all functions, including patch functions.
However, FDEs for patches are not only unnecessary, but they can lead to
problems with libraries and runtimes that consume FDEs, e.g. C++
exception handling runtime.

Note that we use named patches to fix function entry points and in that
case they behave more like regular functions. Thus we issue FDEs for
those.
---
 bolt/lib/Core/BinaryEmitter.cpp | 6 ++++--
 bolt/test/AArch64/lite-mode.s   | 7 +++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index d3ed9e8088bfa..7b5cd276fee89 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -373,8 +373,10 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
     Streamer.emitLabel(StartSymbol);
   }
 
+  const bool NeedsFDE =
+      Function.hasCFI() && !(Function.isPatch() && Function.isAnonymous());
   // Emit CFI start
-  if (Function.hasCFI()) {
+  if (NeedsFDE) {
     Streamer.emitCFIStartProc(/*IsSimple=*/false);
     if (Function.getPersonalityFunction() != nullptr)
       Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
@@ -421,7 +423,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
     Streamer.emitBytes(BC.MIB->getTrapFillValue());
 
   // Emit CFI end
-  if (Function.hasCFI())
+  if (NeedsFDE)
     Streamer.emitCFIEndProc();
 
   MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());
diff --git a/bolt/test/AArch64/lite-mode.s b/bolt/test/AArch64/lite-mode.s
index d81206089aaef..a71edbe034669 100644
--- a/bolt/test/AArch64/lite-mode.s
+++ b/bolt/test/AArch64/lite-mode.s
@@ -11,6 +11,13 @@
 # RUN: llvm-objdump -d --disassemble-symbols=cold_function %t.bolt \
 # RUN:   | FileCheck %s
 
+
+## Verify that the number of FDEs matches the number of functions in the output
+## binary. There are three original functions and two optimized.
+# RUN: llvm-readelf -u %t.bolt | grep -wc FDE \
+# RUN:   | FileCheck --check-prefix=CHECK-FDE %s
+# CHECK-FDE: 5
+
 ## In lite mode, optimized code will be separated from the original .text by
 ## over 128MB, making it impossible for call/bl instructions in cold functions
 ## to reach optimized functions directly.

From 5b2c743a2eb012d1e0649324ade83f23c6645a5b Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp@bytedance.com>
Date: Fri, 18 Apr 2025 10:59:32 +0800
Subject: [PATCH 359/710] [RISCV][NFC] Add Zicsr to profiles explicitly
 (#136134)

To avoid some misunstandings though Zicsr is implied by F/D.
---
 llvm/lib/Target/RISCV/RISCVProfiles.td | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/Target/RISCV/RISCVProfiles.td b/llvm/lib/Target/RISCV/RISCVProfiles.td
index bcb776e682aea..92389fecd0b1d 100644
--- a/llvm/lib/Target/RISCV/RISCVProfiles.td
+++ b/llvm/lib/Target/RISCV/RISCVProfiles.td
@@ -24,6 +24,7 @@ defvar RVA20U64BaseFeatures = [Feature64Bit,
                                FeatureStdExtF,
                                FeatureStdExtD,
                                FeatureStdExtC,
+                               FeatureStdExtZicsr,
                                FeatureStdExtZicntr,
                                FeatureStdExtZiccif,
                                FeatureStdExtZiccrse,

From a354564a64c6ab3cafa9e5e2b31f7f14d4e27d45 Mon Sep 17 00:00:00 2001
From: Iris <0.0@owo.li>
Date: Fri, 18 Apr 2025 11:16:58 +0800
Subject: [PATCH 360/710] [RISCV] Strengthen register usage validation for
 XTheadMemPair loads (#136241)

Closes #136087


https://github.com/XUANTIE-RV/thead-extension-spec/blob/master/xtheadmempair/lwd.adoc
---
 llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp | 4 ++--
 llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s     | 3 ++-
 llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s     | 3 ++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 7e2821404ef95..4178e29b81976 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -3648,9 +3648,9 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     MCRegister Rd2 = Inst.getOperand(1).getReg();
     MCRegister Rs1 = Inst.getOperand(2).getReg();
     // The encoding with rd1 == rd2 == rs1 is reserved for XTHead load pair.
-    if (Rs1 == Rd1 && Rs1 == Rd2) {
+    if (Rs1 == Rd1 || Rs1 == Rd2 || Rd1 == Rd2) {
       SMLoc Loc = Operands[1]->getStartLoc();
-      return Error(Loc, "rs1, rd1, and rd2 cannot all be the same");
+      return Error(Loc, "rs1, rd1, and rd2 cannot overlap");
     }
   }
 
diff --git a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
index 9124218c1f8f5..6b95b3dc66b43 100644
--- a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
@@ -15,6 +15,7 @@ th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be consta
 th.swd t3, t4, (t5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.swd t3, t4, (t5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
 th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
-th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot all be the same
+th.lwd x6, x7, (x7), 2, 3   # CHECK: [[@LINE]]:8: error: rs1, rd1, and rd2 cannot overlap
+th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot overlap
 th.ldd t0, t1, (t2), 2, 4   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
 th.sdd t0, t1, (t2), 2, 4   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
diff --git a/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
index 342db21bbca62..bbd0118630fbb 100644
--- a/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
@@ -15,4 +15,5 @@ th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be consta
 th.swd t3, t4, (t5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.swd t3, t4, (t5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
 th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
-th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot all be the same
+th.ldd x6, x6, (x7), 2, 3   # CHECK: [[@LINE]]:8: error: rs1, rd1, and rd2 cannot overlap
+th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot overlap

From 594bfadbd683eab5e1cd40a450c25da2834f7768 Mon Sep 17 00:00:00 2001
From: tangaac <tangyan01@loongson.cn>
Date: Fri, 18 Apr 2025 11:17:17 +0800
Subject: [PATCH 361/710] [LoongArch] Pre-commit for broadcast load (#136070)

---
 .../CodeGen/LoongArch/lasx/broadcast-load.ll  | 185 ++++++++++++++++++
 .../CodeGen/LoongArch/lsx/broadcast-load.ll   | 183 +++++++++++++++++
 2 files changed, 368 insertions(+)
 create mode 100644 llvm/test/CodeGen/LoongArch/lasx/broadcast-load.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lsx/broadcast-load.ll

diff --git a/llvm/test/CodeGen/LoongArch/lasx/broadcast-load.ll b/llvm/test/CodeGen/LoongArch/lasx/broadcast-load.ll
new file mode 100644
index 0000000000000..4fcf016376d09
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/broadcast-load.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+lasx < %s | FileCheck %s
+
+; TODO: Load a element and splat it to a vector could be lowerd to xvldrepl
+
+; A load has more than one user shouldn't be lowered to xvldrepl
+define <4 x i64> @should_not_be_optimized(ptr %ptr, ptr %dst) {
+; CHECK-LABEL: should_not_be_optimized:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    xvreplgr2vr.d $xr0, $a0
+; CHECK-NEXT:    st.d $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %tmp = load i64, ptr %ptr
+  store i64 %tmp, ptr %dst
+  %tmp1 = insertelement <4 x i64> zeroinitializer, i64 %tmp, i32 0
+  %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> poison, <4 x i32> zeroinitializer
+  ret <4 x i64> %tmp2
+}
+
+define <4 x i64> @xvldrepl_d_unaligned_offset(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_d_unaligned_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 4
+; CHECK-NEXT:    xvreplgr2vr.d $xr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i32, ptr %ptr, i32 1
+  %tmp = load i64, ptr %p
+  %tmp1 = insertelement <4 x i64> zeroinitializer, i64 %tmp, i32 0
+  %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> poison, <4 x i32> zeroinitializer
+  ret <4 x i64> %tmp2
+}
+
+define <32 x i8> @xvldrepl_b(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.b $a0, $a0, 0
+; CHECK-NEXT:    xvreplgr2vr.b $xr0, $a0
+; CHECK-NEXT:    ret
+  %tmp = load i8, ptr %ptr
+  %tmp1 = insertelement <32 x i8> zeroinitializer, i8 %tmp, i32 0
+  %tmp2 = shufflevector <32 x i8> %tmp1, <32 x i8> poison, <32 x i32> zeroinitializer
+  ret <32 x i8> %tmp2
+}
+
+define <32 x i8> @xvldrepl_b_offset(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_b_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.b $a0, $a0, 33
+; CHECK-NEXT:    xvreplgr2vr.b $xr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i8, ptr %ptr, i64 33
+  %tmp = load i8, ptr %p
+  %tmp1 = insertelement <32 x i8> zeroinitializer, i8 %tmp, i32 0
+  %tmp2 = shufflevector <32 x i8> %tmp1, <32 x i8> poison, <32 x i32> zeroinitializer
+  ret <32 x i8> %tmp2
+}
+
+
+define <16 x i16> @xvldrepl_h(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    xvreplgr2vr.h $xr0, $a0
+; CHECK-NEXT:    ret
+  %tmp = load i16, ptr %ptr
+  %tmp1 = insertelement <16 x i16> zeroinitializer, i16 %tmp, i32 0
+  %tmp2 = shufflevector <16 x i16> %tmp1, <16 x i16> poison, <16 x i32> zeroinitializer
+  ret <16 x i16> %tmp2
+}
+
+define <16 x i16> @xvldrepl_h_offset(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_h_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 66
+; CHECK-NEXT:    xvreplgr2vr.h $xr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i16, ptr %ptr, i64 33
+  %tmp = load i16, ptr %p
+  %tmp1 = insertelement <16 x i16> zeroinitializer, i16 %tmp, i32 0
+  %tmp2 = shufflevector <16 x i16> %tmp1, <16 x i16> poison, <16 x i32> zeroinitializer
+  ret <16 x i16> %tmp2
+}
+
+define <8 x i32> @xvldrepl_w(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_w:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    xvreplgr2vr.w $xr0, $a0
+; CHECK-NEXT:    ret
+  %tmp = load i32, ptr %ptr
+  %tmp1 = insertelement <8 x i32> zeroinitializer, i32 %tmp, i32 0
+  %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> poison, <8 x i32> zeroinitializer
+  ret <8 x i32> %tmp2
+}
+
+define <8 x i32> @xvldrepl_w_offset(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_w_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 132
+; CHECK-NEXT:    xvreplgr2vr.w $xr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i32, ptr %ptr, i64 33
+  %tmp = load i32, ptr %p
+  %tmp1 = insertelement <8 x i32> zeroinitializer, i32 %tmp, i32 0
+  %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> poison, <8 x i32> zeroinitializer
+  ret <8 x i32> %tmp2
+}
+
+
+define <4 x i64> @xvldrepl_d(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    xvreplgr2vr.d $xr0, $a0
+; CHECK-NEXT:    ret
+  %tmp = load i64, ptr %ptr
+  %tmp1 = insertelement <4 x i64> zeroinitializer, i64 %tmp, i32 0
+  %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> poison, <4 x i32> zeroinitializer
+  ret <4 x i64> %tmp2
+}
+
+define <4 x i64> @xvldrepl_d_offset(ptr %ptr) {
+; CHECK-LABEL: xvldrepl_d_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 264
+; CHECK-NEXT:    xvreplgr2vr.d $xr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i64, ptr %ptr, i64 33
+  %tmp = load i64, ptr %p
+  %tmp1 = insertelement <4 x i64> zeroinitializer, i64 %tmp, i32 0
+  %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> poison, <4 x i32> zeroinitializer
+  ret <4 x i64> %tmp2
+}
+
+define <8 x float> @vldrepl_w_flt(ptr %ptr) {
+; CHECK-LABEL: vldrepl_w_flt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.s $fa0, $a0, 0
+; CHECK-NEXT:    xvreplve0.w $xr0, $xr0
+; CHECK-NEXT:    ret
+  %tmp = load float, ptr %ptr
+  %tmp1 = insertelement <8 x float> zeroinitializer, float %tmp, i32 0
+  %tmp2 = shufflevector <8 x float> %tmp1, <8 x float> poison, <8 x i32> zeroinitializer
+  ret <8 x float> %tmp2
+}
+
+define <8 x float> @vldrepl_w_flt_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_w_flt_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.s $fa0, $a0, 264
+; CHECK-NEXT:    xvreplve0.w $xr0, $xr0
+; CHECK-NEXT:    ret
+  %p = getelementptr i64, ptr %ptr, i64 33
+  %tmp = load float, ptr %p
+  %tmp1 = insertelement <8 x float> zeroinitializer, float %tmp, i32 0
+  %tmp2 = shufflevector <8 x float> %tmp1, <8 x float> poison, <8 x i32> zeroinitializer
+  ret <8 x float> %tmp2
+}
+
+define <4 x double> @vldrepl_d_dbl(ptr %ptr) {
+; CHECK-LABEL: vldrepl_d_dbl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.d $fa0, $a0, 0
+; CHECK-NEXT:    xvreplve0.d $xr0, $xr0
+; CHECK-NEXT:    ret
+  %tmp = load double, ptr %ptr
+  %tmp1 = insertelement <4 x double> zeroinitializer, double %tmp, i32 0
+  %tmp2 = shufflevector <4 x double> %tmp1, <4 x double> poison, <4 x i32> zeroinitializer
+  ret <4 x double> %tmp2
+}
+
+define <4 x double> @vldrepl_d_dbl_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_d_dbl_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.d $fa0, $a0, 264
+; CHECK-NEXT:    xvreplve0.d $xr0, $xr0
+; CHECK-NEXT:    ret
+  %p = getelementptr i64, ptr %ptr, i64 33
+  %tmp = load double, ptr %p
+  %tmp1 = insertelement <4 x double> zeroinitializer, double %tmp, i32 0
+  %tmp2 = shufflevector <4 x double> %tmp1, <4 x double> poison, <4 x i32> zeroinitializer
+  ret <4 x double> %tmp2
+}
+
diff --git a/llvm/test/CodeGen/LoongArch/lsx/broadcast-load.ll b/llvm/test/CodeGen/LoongArch/lsx/broadcast-load.ll
new file mode 100644
index 0000000000000..02b68725687dd
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/broadcast-load.ll
@@ -0,0 +1,183 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+lsx < %s | FileCheck %s
+
+; TODO: Load a element and splat it to a vector could be lowerd to vldrepl
+
+; A load has more than one user shouldn't be lowered to vldrepl
+define <2 x i64> @should_not_be_optimized(ptr %ptr, ptr %dst){
+; CHECK-LABEL: should_not_be_optimized:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vreplgr2vr.d $vr0, $a0
+; CHECK-NEXT:    st.d $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %tmp = load i64, ptr %ptr
+  store i64 %tmp, ptr %dst
+  %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
+  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> poison, <2 x i32> zeroinitializer
+  ret <2 x i64> %tmp2
+}
+
+define <2 x i64> @vldrepl_d_unaligned_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_d_unaligned_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 4
+; CHECK-NEXT:    vreplgr2vr.d $vr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i32, ptr %ptr, i32 1
+  %tmp = load i64, ptr %p
+  %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
+  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> poison, <2 x i32> zeroinitializer
+  ret <2 x i64> %tmp2
+}
+
+define <16 x i8> @vldrepl_b(ptr %ptr) {
+; CHECK-LABEL: vldrepl_b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.b $a0, $a0, 0
+; CHECK-NEXT:    vreplgr2vr.b $vr0, $a0
+; CHECK-NEXT:    ret
+  %tmp = load i8, ptr %ptr
+  %tmp1 = insertelement <16 x i8> zeroinitializer, i8 %tmp, i32 0
+  %tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> poison, <16 x i32> zeroinitializer
+  ret <16 x i8> %tmp2
+}
+
+define <16 x i8> @vldrepl_b_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_b_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.b $a0, $a0, 33
+; CHECK-NEXT:    vreplgr2vr.b $vr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i8, ptr %ptr, i64 33
+  %tmp = load i8, ptr %p
+  %tmp1 = insertelement <16 x i8> zeroinitializer, i8 %tmp, i32 0
+  %tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> poison, <16 x i32> zeroinitializer
+  ret <16 x i8> %tmp2
+}
+
+
+define <8 x i16> @vldrepl_h(ptr %ptr) {
+; CHECK-LABEL: vldrepl_h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vreplgr2vr.h $vr0, $a0
+; CHECK-NEXT:    ret
+  %tmp = load i16, ptr %ptr
+  %tmp1 = insertelement <8 x i16> zeroinitializer, i16 %tmp, i32 0
+  %tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> poison, <8 x i32> zeroinitializer
+  ret <8 x i16> %tmp2
+}
+
+define <8 x i16> @vldrepl_h_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_h_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 66
+; CHECK-NEXT:    vreplgr2vr.h $vr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i16, ptr %ptr, i64 33
+  %tmp = load i16, ptr %p
+  %tmp1 = insertelement <8 x i16> zeroinitializer, i16 %tmp, i32 0
+  %tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> poison, <8 x i32> zeroinitializer
+  ret <8 x i16> %tmp2
+}
+
+define <4 x i32> @vldrepl_w(ptr %ptr) {
+; CHECK-LABEL: vldrepl_w:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vreplgr2vr.w $vr0, $a0
+; CHECK-NEXT:    ret
+  %tmp = load i32, ptr %ptr
+  %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
+  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> poison, <4 x i32> zeroinitializer
+  ret <4 x i32> %tmp2
+}
+
+define <4 x i32> @vldrepl_w_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_w_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 132
+; CHECK-NEXT:    vreplgr2vr.w $vr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i32, ptr %ptr, i64 33
+  %tmp = load i32, ptr %p
+  %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
+  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> poison, <4 x i32> zeroinitializer
+  ret <4 x i32> %tmp2
+}
+
+define <2 x i64> @vldrepl_d(ptr %ptr) {
+; CHECK-LABEL: vldrepl_d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vreplgr2vr.d $vr0, $a0
+; CHECK-NEXT:    ret
+  %tmp = load i64, ptr %ptr
+  %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
+  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> poison, <2 x i32> zeroinitializer
+  ret <2 x i64> %tmp2
+}
+
+define <2 x i64> @vldrepl_d_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_d_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 264
+; CHECK-NEXT:    vreplgr2vr.d $vr0, $a0
+; CHECK-NEXT:    ret
+  %p = getelementptr i64, ptr %ptr, i64 33
+  %tmp = load i64, ptr %p
+  %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
+  %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> poison, <2 x i32> zeroinitializer
+  ret <2 x i64> %tmp2
+}
+
+define <4 x float> @vldrepl_w_flt(ptr %ptr) {
+; CHECK-LABEL: vldrepl_w_flt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.s $fa0, $a0, 0
+; CHECK-NEXT:    vreplvei.w $vr0, $vr0, 0
+; CHECK-NEXT:    ret
+  %tmp = load float, ptr %ptr
+  %tmp1 = insertelement <4 x float> zeroinitializer, float %tmp, i32 0
+  %tmp2 = shufflevector <4 x float> %tmp1, <4 x float> poison, <4 x i32> zeroinitializer
+  ret <4 x float> %tmp2
+}
+
+define <4 x float> @vldrepl_w_flt_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_w_flt_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.s $fa0, $a0, 264
+; CHECK-NEXT:    vreplvei.w $vr0, $vr0, 0
+; CHECK-NEXT:    ret
+  %p = getelementptr i64, ptr %ptr, i64 33
+  %tmp = load float, ptr %p
+  %tmp1 = insertelement <4 x float> zeroinitializer, float %tmp, i32 0
+  %tmp2 = shufflevector <4 x float> %tmp1, <4 x float> poison, <4 x i32> zeroinitializer
+  ret <4 x float> %tmp2
+}
+
+define <2 x double> @vldrepl_d_dbl(ptr %ptr) {
+; CHECK-LABEL: vldrepl_d_dbl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.d $fa0, $a0, 0
+; CHECK-NEXT:    vreplvei.d $vr0, $vr0, 0
+; CHECK-NEXT:    ret
+  %tmp = load double, ptr %ptr
+  %tmp1 = insertelement <2 x double> zeroinitializer, double %tmp, i32 0
+  %tmp2 = shufflevector <2 x double> %tmp1, <2 x double> poison, <2 x i32> zeroinitializer
+  ret <2 x double> %tmp2
+}
+
+define <2 x double> @vldrepl_d_dbl_offset(ptr %ptr) {
+; CHECK-LABEL: vldrepl_d_dbl_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.d $fa0, $a0, 264
+; CHECK-NEXT:    vreplvei.d $vr0, $vr0, 0
+; CHECK-NEXT:    ret
+  %p = getelementptr i64, ptr %ptr, i64 33
+  %tmp = load double, ptr %p
+  %tmp1 = insertelement <2 x double> zeroinitializer, double %tmp, i32 0
+  %tmp2 = shufflevector <2 x double> %tmp1, <2 x double> poison, <2 x i32> zeroinitializer
+  ret <2 x double> %tmp2
+}

From a3f38f27cd78e1b2753a3f29489094cfd92198fe Mon Sep 17 00:00:00 2001
From: Shoreshen <372660931@qq.com>
Date: Fri, 18 Apr 2025 11:45:30 +0800
Subject: [PATCH 362/710] Revert "[AMDGPU] Implement vop3p complex pattern
 optmization for gisel" (#136249)

Reverts llvm/llvm-project#130234
---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      | 604 +-----------------
 .../Target/AMDGPU/AMDGPUInstructionSelector.h |   6 +-
 .../CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll   | 163 -----
 .../AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll    |  12 +-
 .../AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll    |  18 +-
 llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll   |  14 +-
 6 files changed, 64 insertions(+), 753 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 87af467ac8f1e..6ef7505ec6f62 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4319,598 +4319,60 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
   }};
 }
 
-enum class SrcStatus {
-  IS_SAME,
-  IS_UPPER_HALF,
-  IS_LOWER_HALF,
-  IS_UPPER_HALF_NEG,
-  // This means current op = [op_upper, op_lower] and src = -op_lower.
-  IS_LOWER_HALF_NEG,
-  IS_HI_NEG,
-  // This means current op = [op_upper, op_lower] and src = [op_upper,
-  // -op_lower].
-  IS_LO_NEG,
-  IS_BOTH_NEG,
-  INVALID,
-  NEG_START = IS_UPPER_HALF_NEG,
-  NEG_END = IS_BOTH_NEG,
-  HALF_START = IS_UPPER_HALF,
-  HALF_END = IS_LOWER_HALF_NEG
-};
-
-static bool isTruncHalf(const MachineInstr *MI,
-                        const MachineRegisterInfo &MRI) {
-  if (MI->getOpcode() != AMDGPU::G_TRUNC)
-    return false;
-
-  unsigned DstSize = MRI.getType(MI->getOperand(0).getReg()).getSizeInBits();
-  unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
-  return DstSize * 2 == SrcSize;
-}
-
-static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
-  if (MI->getOpcode() != AMDGPU::G_LSHR)
-    return false;
-
-  Register ShiftSrc;
-  std::optional<ValueAndVReg> ShiftAmt;
-  if (mi_match(MI->getOperand(0).getReg(), MRI,
-               m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
-    unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
-    unsigned Shift = ShiftAmt->Value.getZExtValue();
-    return Shift * 2 == SrcSize;
-  }
-  return false;
-}
-
-static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
-  if (MI->getOpcode() != AMDGPU::G_SHL)
-    return false;
-
-  Register ShiftSrc;
-  std::optional<ValueAndVReg> ShiftAmt;
-  if (mi_match(MI->getOperand(0).getReg(), MRI,
-               m_GShl(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
-    unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
-    unsigned Shift = ShiftAmt->Value.getZExtValue();
-    return Shift * 2 == SrcSize;
-  }
-  return false;
-}
-
-static std::optional<std::pair<const MachineOperand *, SrcStatus>>
-retOpStat(const MachineOperand *Op, SrcStatus Stat,
-          std::pair<const MachineOperand *, SrcStatus> &Curr) {
-  if (Stat != SrcStatus::INVALID &&
-      ((Op->isReg() && !(Op->getReg().isPhysical())) || Op->isImm() ||
-       Op->isCImm() || Op->isFPImm())) {
-    return std::optional<std::pair<const MachineOperand *, SrcStatus>>(
-        {Op, Stat});
-  }
-
-  return std::nullopt;
-}
-
-enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
-
-static TypeClass isVectorOfTwoOrScalar(const MachineOperand *Op,
-                                       const MachineRegisterInfo &MRI) {
-  if (!Op->isReg() || Op->getReg().isPhysical())
-    return TypeClass::NONE_OF_LISTED;
-  LLT OpTy = MRI.getType(Op->getReg());
-  if (OpTy.isScalar())
-    return TypeClass::SCALAR;
-  if (OpTy.isVector() && OpTy.getNumElements() == 2)
-    return TypeClass::VECTOR_OF_TWO;
-  return TypeClass::NONE_OF_LISTED;
-}
-
-static SrcStatus getNegStatus(const MachineOperand *Op, SrcStatus S,
-                              const MachineRegisterInfo &MRI) {
-  TypeClass NegType = isVectorOfTwoOrScalar(Op, MRI);
-  if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
-    return SrcStatus::INVALID;
-
-  switch (S) {
-  case SrcStatus::IS_SAME:
-    if (NegType == TypeClass::VECTOR_OF_TWO) {
-      // Vector of 2:
-      // [SrcHi, SrcLo]   = [CurrHi, CurrLo]
-      // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
-      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
-      // [SrcHi, SrcLo]   = [-OpHi, -OpLo]
-      return SrcStatus::IS_BOTH_NEG;
-    } else if (NegType == TypeClass::SCALAR) {
-      // Scalar:
-      // [SrcHi, SrcLo]   = [CurrHi, CurrLo]
-      // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
-      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
-      // [SrcHi, SrcLo]   = [-OpHi, OpLo]
-      return SrcStatus::IS_HI_NEG;
-    }
-    break;
-  case SrcStatus::IS_HI_NEG:
-    if (NegType == TypeClass::VECTOR_OF_TWO) {
-      // Vector of 2:
-      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
-      // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
-      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
-      // [SrcHi, SrcLo]   = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
-      return SrcStatus::IS_LO_NEG;
-    } else if (NegType == TypeClass::SCALAR) {
-      // Scalar:
-      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
-      // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
-      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
-      // [SrcHi, SrcLo]   = [-(-OpHi), OpLo] = [OpHi, OpLo]
-      return SrcStatus::IS_SAME;
-    }
-    break;
-  case SrcStatus::IS_LO_NEG:
-    if (NegType == TypeClass::VECTOR_OF_TWO) {
-      // Vector of 2:
-      // [SrcHi, SrcLo]   = [CurrHi, -CurrLo]
-      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
-      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
-      // [SrcHi, SrcLo]   = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
-      return SrcStatus::IS_HI_NEG;
-    } else if (NegType == TypeClass::SCALAR) {
-      // Scalar:
-      // [SrcHi, SrcLo]   = [CurrHi, -CurrLo]
-      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
-      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
-      // [SrcHi, SrcLo]   = [-OpHi, -OpLo]
-      return SrcStatus::IS_BOTH_NEG;
-    }
-    break;
-  case SrcStatus::IS_BOTH_NEG:
-    if (NegType == TypeClass::VECTOR_OF_TWO) {
-      // Vector of 2:
-      // [SrcHi, SrcLo]   = [-CurrHi, -CurrLo]
-      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
-      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
-      // [SrcHi, SrcLo]   = [OpHi, OpLo]
-      return SrcStatus::IS_SAME;
-    } else if (NegType == TypeClass::SCALAR) {
-      // Scalar:
-      // [SrcHi, SrcLo]   = [-CurrHi, -CurrLo]
-      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
-      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
-      // [SrcHi, SrcLo]   = [OpHi, -OpLo]
-      return SrcStatus::IS_LO_NEG;
-    }
-    break;
-  case SrcStatus::IS_UPPER_HALF:
-    // Vector of 2:
-    // Src = CurrUpper
-    // Curr = [CurrUpper, CurrLower]
-    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
-    // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
-    // Src = -OpUpper
-    //
-    // Scalar:
-    // Src = CurrUpper
-    // Curr = [CurrUpper, CurrLower]
-    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
-    // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
-    // Src = -OpUpper
-    return SrcStatus::IS_UPPER_HALF_NEG;
-  case SrcStatus::IS_LOWER_HALF:
-    if (NegType == TypeClass::VECTOR_OF_TWO) {
-      // Vector of 2:
-      // Src = CurrLower
-      // Curr = [CurrUpper, CurrLower]
-      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
-      // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
-      // Src = -OpLower
-      return SrcStatus::IS_LOWER_HALF_NEG;
-    } else if (NegType == TypeClass::SCALAR) {
-      // Scalar:
-      // Src = CurrLower
-      // Curr = [CurrUpper, CurrLower]
-      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
-      // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
-      // Src = OpLower
-      return SrcStatus::IS_LOWER_HALF;
-    }
-    break;
-  case SrcStatus::IS_UPPER_HALF_NEG:
-    // Vector of 2:
-    // Src = -CurrUpper
-    // Curr = [CurrUpper, CurrLower]
-    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
-    // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
-    // Src = -(-OpUpper) = OpUpper
-    //
-    // Scalar:
-    // Src = -CurrUpper
-    // Curr = [CurrUpper, CurrLower]
-    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
-    // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
-    // Src = -(-OpUpper) = OpUpper
-    return SrcStatus::IS_UPPER_HALF;
-  case SrcStatus::IS_LOWER_HALF_NEG:
-    if (NegType == TypeClass::VECTOR_OF_TWO) {
-      // Vector of 2:
-      // Src = -CurrLower
-      // Curr = [CurrUpper, CurrLower]
-      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
-      // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
-      // Src = -(-OpLower) = OpLower
-      return SrcStatus::IS_LOWER_HALF;
-    } else if (NegType == TypeClass::SCALAR) {
-      // Scalar:
-      // Src = -CurrLower
-      // Curr = [CurrUpper, CurrLower]
-      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
-      // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
-      // Src = -OpLower
-      return SrcStatus::IS_LOWER_HALF_NEG;
-    }
-    break;
-  default:
-    llvm_unreachable("unexpected SrcStatus");
-  }
-}
-
-static std::optional<std::pair<const MachineOperand *, SrcStatus>>
-calcNextStatus(std::pair<const MachineOperand *, SrcStatus> Curr,
-               const MachineRegisterInfo &MRI) {
-  if (!Curr.first->isReg())
-    return std::nullopt;
-
-  const MachineInstr *MI = Curr.first->isDef()
-                               ? Curr.first->getParent()
-                               : MRI.getVRegDef(Curr.first->getReg());
-
-  unsigned Opc = MI->getOpcode();
-
-  // Handle general Opc cases.
-  switch (Opc) {
-  case AMDGPU::G_BITCAST:
-  case AMDGPU::G_CONSTANT:
-  case AMDGPU::G_FCONSTANT:
-  case AMDGPU::COPY:
-    return retOpStat(&MI->getOperand(1), Curr.second, Curr);
-  case AMDGPU::G_FNEG:
-    return retOpStat(&MI->getOperand(1),
-                     getNegStatus(Curr.first, Curr.second, MRI), Curr);
-  default:
-    break;
-  }
-
-  // Calc next Stat from current Stat.
-  switch (Curr.second) {
-  case SrcStatus::IS_SAME:
-    if (isTruncHalf(MI, MRI))
-      return retOpStat(&MI->getOperand(1), SrcStatus::IS_LOWER_HALF, Curr);
-    break;
-  case SrcStatus::IS_HI_NEG:
-    if (isTruncHalf(MI, MRI)) {
-      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
-      // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
-      //                  = [OpLowerHi, OpLowerLo]
-      // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
-      //     = [-OpLowerHi, OpLowerLo]
-      //     = -OpLower
-      return retOpStat(&MI->getOperand(1), SrcStatus::IS_LOWER_HALF_NEG, Curr);
-    }
-    break;
-  case SrcStatus::IS_UPPER_HALF:
-    if (isShlHalf(MI, MRI))
-      return retOpStat(&MI->getOperand(1), SrcStatus::IS_LOWER_HALF, Curr);
-    break;
-  case SrcStatus::IS_LOWER_HALF:
-    if (isLshrHalf(MI, MRI))
-      return retOpStat(&MI->getOperand(1), SrcStatus::IS_UPPER_HALF, Curr);
-    break;
-  case SrcStatus::IS_UPPER_HALF_NEG:
-    if (isShlHalf(MI, MRI))
-      return retOpStat(&MI->getOperand(1), SrcStatus::IS_LOWER_HALF_NEG, Curr);
-    break;
-  case SrcStatus::IS_LOWER_HALF_NEG:
-    if (isLshrHalf(MI, MRI))
-      return retOpStat(&MI->getOperand(1), SrcStatus::IS_UPPER_HALF_NEG, Curr);
-    break;
-  default:
-    break;
-  }
-  return std::nullopt;
-}
-
-class searchOptions {
-private:
-  bool HasNeg = false;
-  // Assume all complex pattern of VOP3P has opsel.
-  bool HasOpsel = true;
-
-public:
-  searchOptions(const MachineOperand *RootOp, const MachineRegisterInfo &MRI) {
-    const MachineInstr *MI = RootOp->getParent();
-    unsigned Opc = MI->getOpcode();
-
-    if (Opc < TargetOpcode::GENERIC_OP_END) {
-      // Keep same for generic op.
-      HasNeg = true;
-    } else if (Opc == TargetOpcode::G_INTRINSIC) {
-      Intrinsic::ID IntrinsicID = cast<GIntrinsic>(*MI).getIntrinsicID();
-      // Only float point intrinsic has neg & neg_hi bits.
-      if (IntrinsicID == Intrinsic::amdgcn_fdot2)
-        HasNeg = true;
-    }
-  }
-  bool checkOptions(SrcStatus Stat) const {
-    if (!HasNeg &&
-        (Stat >= SrcStatus::NEG_START || Stat <= SrcStatus::NEG_END)) {
-      return false;
-    }
-    if (!HasOpsel &&
-        (Stat >= SrcStatus::HALF_START || Stat >= SrcStatus::HALF_END)) {
-      return false;
-    }
-    return true;
-  }
-};
-
-static SmallVector<std::pair<const MachineOperand *, SrcStatus>>
-getSrcStats(const MachineOperand *Op, const MachineRegisterInfo &MRI,
-            searchOptions SearchOptions, int MaxDepth = 6) {
-  int Depth = 0;
-  auto Curr = calcNextStatus({Op, SrcStatus::IS_SAME}, MRI);
-  SmallVector<std::pair<const MachineOperand *, SrcStatus>, 4> Statlist;
-
-  while (Depth <= MaxDepth && Curr.has_value()) {
-    Depth++;
-    if (SearchOptions.checkOptions(Curr.value().second))
-      Statlist.push_back(Curr.value());
-    Curr = calcNextStatus(Curr.value(), MRI);
-  }
-
-  return Statlist;
-}
-
-static std::pair<const MachineOperand *, SrcStatus>
-getLastSameOrNeg(const MachineOperand *Op, const MachineRegisterInfo &MRI,
-                 searchOptions SearchOptions, int MaxDepth = 6) {
-  int Depth = 0;
-  std::pair<const MachineOperand *, SrcStatus> LastSameOrNeg = {
-      Op, SrcStatus::IS_SAME};
-  auto Curr = calcNextStatus(LastSameOrNeg, MRI);
-
-  while (Depth <= MaxDepth && Curr.has_value()) {
-    Depth++;
-    if (SearchOptions.checkOptions(Curr.value().second)) {
-      if (Curr.value().second == SrcStatus::IS_SAME ||
-          Curr.value().second == SrcStatus::IS_HI_NEG ||
-          Curr.value().second == SrcStatus::IS_LO_NEG ||
-          Curr.value().second == SrcStatus::IS_BOTH_NEG)
-        LastSameOrNeg = Curr.value();
-    }
-    Curr = calcNextStatus(Curr.value(), MRI);
-  }
-
-  return LastSameOrNeg;
-}
-
-static bool isInlinableFPConstant(const MachineOperand &Op,
-                                  const SIInstrInfo &TII) {
-  return Op.isFPImm() && TII.isInlineConstant(Op.getFPImm()->getValueAPF());
-}
-
-static bool isSameBitWidth(const MachineOperand *Op1, const MachineOperand *Op2,
-                           const MachineRegisterInfo &MRI) {
-  unsigned Width1 = MRI.getType(Op1->getReg()).getSizeInBits();
-  unsigned Width2 = MRI.getType(Op2->getReg()).getSizeInBits();
-  return Width1 == Width2;
-}
-
-static bool isSameOperand(const MachineOperand *Op1,
-                          const MachineOperand *Op2) {
-  if (Op1->isReg())
-    return Op2->isReg() && Op1->getReg() == Op2->getReg();
-
-  return Op1->isIdenticalTo(*Op2);
-}
-
-static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
-  // SrcStatus::IS_LOWER_HALF remain 0.
-  if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
-    Mods ^= SISrcMods::NEG_HI;
-    Mods |= SISrcMods::OP_SEL_1;
-  } else if (HiStat == SrcStatus::IS_UPPER_HALF)
-    Mods |= SISrcMods::OP_SEL_1;
-  else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
-    Mods ^= SISrcMods::NEG_HI;
-  else if (HiStat == SrcStatus::IS_HI_NEG)
-    Mods ^= SISrcMods::NEG_HI;
-
-  if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
-    Mods ^= SISrcMods::NEG;
-    Mods |= SISrcMods::OP_SEL_0;
-  } else if (LoStat == SrcStatus::IS_UPPER_HALF)
-    Mods |= SISrcMods::OP_SEL_0;
-  else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
-    Mods |= SISrcMods::NEG;
-  else if (LoStat == SrcStatus::IS_HI_NEG)
-    Mods ^= SISrcMods::NEG;
-
-  return Mods;
-}
-
-static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat,
-                          const MachineOperand *NewOp,
-                          const MachineOperand *RootOp, const SIInstrInfo &TII,
-                          const MachineRegisterInfo &MRI) {
-  if (NewOp->isReg()) {
-    auto IsHalfState = [](SrcStatus S) {
-      return S == SrcStatus::IS_UPPER_HALF ||
-             S == SrcStatus::IS_UPPER_HALF_NEG ||
-             S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
-    };
-    return isSameBitWidth(NewOp, RootOp, MRI) && IsHalfState(LoStat) &&
-           IsHalfState(HiStat);
-  } else
-    return ((HiStat == SrcStatus::IS_SAME || HiStat == SrcStatus::IS_HI_NEG) &&
-            (LoStat == SrcStatus::IS_SAME || LoStat == SrcStatus::IS_HI_NEG) &&
-            isInlinableFPConstant(*NewOp, TII));
-
-  return false;
-}
-
-std::pair<const MachineOperand *, unsigned>
-AMDGPUInstructionSelector::selectVOP3PModsImpl(const MachineOperand *RootOp,
-                                               const MachineRegisterInfo &MRI,
-                                               bool IsDOT) const {
+std::pair<Register, unsigned>
+AMDGPUInstructionSelector::selectVOP3PModsImpl(
+  Register Src, const MachineRegisterInfo &MRI, bool IsDOT) const {
   unsigned Mods = 0;
-  const MachineOperand *Op = RootOp;
-  // No modification if Root type is not form of <2 x Type>.
-  if (isVectorOfTwoOrScalar(Op, MRI) != TypeClass::VECTOR_OF_TWO) {
-    Mods |= SISrcMods::OP_SEL_1;
-    return {Op, Mods};
-  }
-
-  searchOptions SearchOptions(Op, MRI);
+  MachineInstr *MI = MRI.getVRegDef(Src);
 
-  std::pair<const MachineOperand *, SrcStatus> Stat =
-      getLastSameOrNeg(Op, MRI, SearchOptions);
-  if (!Stat.first->isReg()) {
-    Mods |= SISrcMods::OP_SEL_1;
-    return {Op, Mods};
-  }
-  if (Stat.second == SrcStatus::IS_BOTH_NEG)
+  if (MI->getOpcode() == AMDGPU::G_FNEG &&
+      // It's possible to see an f32 fneg here, but unlikely.
+      // TODO: Treat f32 fneg as only high bit.
+      MRI.getType(Src) == LLT::fixed_vector(2, 16)) {
     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
-  else if (Stat.second == SrcStatus::IS_HI_NEG)
-    Mods ^= SISrcMods::NEG_HI;
-  else if (Stat.second == SrcStatus::IS_LO_NEG)
-    Mods ^= SISrcMods::NEG;
-
-  Op = Stat.first;
-  MachineInstr *MI = MRI.getVRegDef(Op->getReg());
-
-  if (MI->getOpcode() != AMDGPU::G_BUILD_VECTOR || MI->getNumOperands() != 3 ||
-      (IsDOT && Subtarget->hasDOTOpSelHazard())) {
-    Mods |= SISrcMods::OP_SEL_1;
-    return {Op, Mods};
-  }
-
-  SmallVector<std::pair<const MachineOperand *, SrcStatus>> StatlistHi =
-      getSrcStats(&MI->getOperand(2), MRI, SearchOptions);
-
-  if (StatlistHi.size() == 0) {
-    Mods |= SISrcMods::OP_SEL_1;
-    return {Op, Mods};
+    Src = MI->getOperand(1).getReg();
+    MI = MRI.getVRegDef(Src);
   }
 
-  SmallVector<std::pair<const MachineOperand *, SrcStatus>> StatlistLo =
-      getSrcStats(&MI->getOperand(1), MRI, SearchOptions);
+  // TODO: Handle G_FSUB 0 as fneg
 
-  if (StatlistLo.size() == 0) {
-    Mods |= SISrcMods::OP_SEL_1;
-    return {Op, Mods};
-  }
+  // TODO: Match op_sel through g_build_vector_trunc and g_shuffle_vector.
+  (void)IsDOT; // DOTs do not use OPSEL on gfx942+, check ST.hasDOTOpSelHazard()
 
-  for (int I = StatlistHi.size() - 1; I >= 0; I--) {
-    for (int J = StatlistLo.size() - 1; J >= 0; J--) {
-      if (isSameOperand(StatlistHi[I].first, StatlistLo[J].first) &&
-          isValidToPack(StatlistHi[I].second, StatlistLo[J].second,
-                        StatlistHi[I].first, RootOp, TII, MRI))
-        return {StatlistHi[I].first,
-                updateMods(StatlistHi[I].second, StatlistLo[J].second, Mods)};
-    }
-  }
   // Packed instructions do not have abs modifiers.
   Mods |= SISrcMods::OP_SEL_1;
 
-  return {Op, Mods};
-}
-
-int64_t getAllKindImm(const MachineOperand *Op) {
-  switch (Op->getType()) {
-  case MachineOperand::MachineOperandType::MO_Immediate:
-    return Op->getImm();
-  case MachineOperand::MachineOperandType::MO_CImmediate:
-    return Op->getCImm()->getSExtValue();
-  case MachineOperand::MachineOperandType::MO_FPImmediate:
-    return Op->getFPImm()->getValueAPF().bitcastToAPInt().getSExtValue();
-  default:
-    llvm_unreachable("not an imm type");
-  }
-}
-
-static bool checkRB(const MachineOperand *Op, unsigned int RBNo,
-                    const AMDGPURegisterBankInfo &RBI,
-                    const MachineRegisterInfo &MRI,
-                    const TargetRegisterInfo &TRI) {
-  const RegisterBank *RB = RBI.getRegBank(Op->getReg(), MRI, TRI);
-  return RB->getID() == RBNo;
-}
-
-// This function is used to get the correct register bank for returned reg.
-// Assume:
-// 1. VOP3P is always legal for VGPR.
-// 2. RootOp's regbank is legal.
-// Thus
-// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
-// 2. If RootOp is VGPR, then NewOp must be VGPR.
-static const MachineOperand *
-getLegalRegBank(const MachineOperand *NewOp, const MachineOperand *RootOp,
-                const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI,
-                const TargetRegisterInfo &TRI, const SIInstrInfo &TII) {
-  // RootOp can only be VGPR or SGPR (some hand written cases such as.
-  // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
-  if (checkRB(RootOp, AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
-      checkRB(NewOp, AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
-    return NewOp;
-
-  MachineInstr *MI = MRI.getVRegDef(RootOp->getReg());
-  if (MI->getOpcode() == AMDGPU::COPY &&
-      isSameOperand(NewOp, &MI->getOperand(1))) {
-    // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
-    return RootOp;
-  }
-
-  MachineBasicBlock *BB = MI->getParent();
-  const TargetRegisterClass *DstRC =
-      TRI.getConstrainedRegClassForOperand(*RootOp, MRI);
-  Register DstReg = MRI.createVirtualRegister(DstRC);
-
-  MachineInstrBuilder MIB =
-      BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg)
-          .addReg(NewOp->getReg());
-
-  // only accept VGPR.
-  return &MIB->getOperand(0);
+  return std::pair(Src, Mods);
 }
 
 InstructionSelector::ComplexRendererFns
-AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
-                                                bool IsDOT) const {
-  MachineRegisterInfo &MRI =
-      Root.getParent()->getParent()->getParent()->getRegInfo();
-  auto [Op, Mods] = selectVOP3PModsImpl(&Root, MRI, IsDOT);
-  if (!(Op->isReg()))
-    return {{
-        [=](MachineInstrBuilder &MIB) { MIB.addImm(getAllKindImm(Op)); },
-        [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
-    }};
+AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
+  MachineRegisterInfo &MRI
+    = Root.getParent()->getParent()->getParent()->getRegInfo();
+
+  Register Src;
+  unsigned Mods;
+  std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI);
 
-  Op = getLegalRegBank(Op, &Root, RBI, MRI, TRI, TII);
   return {{
-      [=](MachineInstrBuilder &MIB) { MIB.addReg(Op->getReg()); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
+      [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
+      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }  // src_mods
   }};
 }
 
-InstructionSelector::ComplexRendererFns
-AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
-
-  return selectVOP3PRetHelper(Root);
-}
-
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
+  MachineRegisterInfo &MRI
+    = Root.getParent()->getParent()->getParent()->getRegInfo();
+
+  Register Src;
+  unsigned Mods;
+  std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true);
 
-  return selectVOP3PRetHelper(Root, true);
+  return {{
+      [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
+      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }  // src_mods
+  }};
 }
 
 InstructionSelector::ComplexRendererFns
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index a224f39f88996..6c3f3026e877a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -187,11 +187,9 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
 
   ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
 
-  std::pair<const MachineOperand *, unsigned>
-  selectVOP3PModsImpl(const MachineOperand *Op, const MachineRegisterInfo &MRI,
+  std::pair<Register, unsigned>
+  selectVOP3PModsImpl(Register Src, const MachineRegisterInfo &MRI,
                       bool IsDOT = false) const;
-  InstructionSelector::ComplexRendererFns
-  selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const;
 
   InstructionSelector::ComplexRendererFns
   selectVOP3PMods(MachineOperand &Root) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
index 534b454775502..543f8e413abd8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
@@ -106,169 +106,6 @@ define <2 x half> @v_fmul_v2f16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %mul
 }
 
-define <2 x half> @v_fmul_v2f16_partial_neg(<2 x half> %a, <2 x half> %b) {
-; GFX9-LABEL: v_fmul_v2f16_partial_neg:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_hi:[1,0]
-; GFX9-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_lo:[1,0]
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: v_fmul_v2f16_partial_neg:
-; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80008000, v1
-; GFX8-NEXT:    v_mul_f16_e32 v3, v1, v0
-; GFX8-NEXT:    v_mul_f16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_mul_f16_e32 v1, v2, v3
-; GFX8-NEXT:    v_mul_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: v_fmul_v2f16_partial_neg:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_hi:[1,0]
-; GFX10-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_lo:[1,0]
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %b1 = bitcast <2 x half> %b to float
-  %b2 = fneg float %b1
-  %b3 = bitcast float %b2 to <2 x half>
-  %b4 = fneg <2 x half> %b3
-  %mul1 = fmul <2 x half> %b3, %a
-  %mul2 = fmul <2 x half> %b4, %mul1
-  ret <2 x half> %mul2
-}
-
-define <2 x half> @fmul_v2_half_neg_hi(<2 x half> %a, <2 x half> %b) #0 {
-; GFX9-LABEL: fmul_v2_half_neg_hi:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1 neg_hi:[0,1]
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: fmul_v2_half_neg_hi:
-; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
-; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
-; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: fmul_v2_half_neg_hi:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1 neg_hi:[0,1]
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %b1 = bitcast <2 x half> %b to float
-  %b2 = fneg float %b1
-  %b3 = bitcast float %b2 to <2 x half>
-  %b4 = extractelement <2 x half> %b3, i64 1
-  %tmp = insertelement <2 x half> poison, half %b4, i64 0
-  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
-  %mul = fmul <2 x half> %a, %k
-  ret <2 x half> %mul
-}
-
-define <2 x half> @fmul_v2_half_neg_hi1(<2 x half> %a, <2 x half> %b) #0 {
-; GFX9-LABEL: fmul_v2_half_neg_hi1:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: fmul_v2_half_neg_hi1:
-; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80008000, v2
-; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
-; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: fmul_v2_half_neg_hi1:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %b1 = bitcast <2 x half> %b to float
-  %b2 = fneg float %b1
-  %b3 = bitcast float %b2 to <2 x half>
-  %b4 = fneg <2 x half> %b3
-  %b5 = extractelement <2 x half> %b4, i64 1
-  %tmp = insertelement <2 x half> poison, half %b5, i64 0
-  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
-  %mul = fmul <2 x half> %a, %k
-  ret <2 x half> %mul
-}
-
-define <2 x half> @fmul_v2_half_neg_lo(<2 x half> %a, <2 x half> %b) #0 {
-; GFX9-LABEL: fmul_v2_half_neg_lo:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0] neg_hi:[0,1]
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: fmul_v2_half_neg_lo:
-; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80008000, v2
-; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
-; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: fmul_v2_half_neg_lo:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0] neg_hi:[0,1]
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %b1 = bitcast <2 x half> %b to float
-  %b2 = fneg float %b1
-  %b3 = bitcast float %b2 to <2 x half>
-  %b4 = fneg <2 x half> %b3
-  %b5 = extractelement <2 x half> %b4, i64 0
-  %tmp = insertelement <2 x half> poison, half %b5, i64 0
-  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
-  %mul = fmul <2 x half> %a, %k
-  ret <2 x half> %mul
-}
-
-define <2 x half> @fmul_v2_half_neg_lo1(<2 x half> %a, <2 x half> %b) #0 {
-; GFX9-LABEL: fmul_v2_half_neg_lo1:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0]
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: fmul_v2_half_neg_lo1:
-; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
-; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
-; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: fmul_v2_half_neg_lo1:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0]
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %b1 = bitcast <2 x half> %b to float
-  %b2 = fneg float %b1
-  %b3 = bitcast float %b2 to <2 x half>
-  %b4 = extractelement <2 x half> %b3, i64 0
-  %tmp = insertelement <2 x half> poison, half %b4, i64 0
-  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
-  %mul = fmul <2 x half> %a, %k
-  ret <2 x half> %mul
-}
-
 define <3 x half> @v_fmul_v3f16(<3 x half> %a, <3 x half> %b) {
 ; GFX9-LABEL: v_fmul_v3f16:
 ; GFX9:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
index 8f0ae8c47098a..744a5b7feb48d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
@@ -304,7 +304,8 @@ define i32 @v_sdot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_sdot2_shuffle10_a:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
+; GFX906-NEXT:    v_alignbit_b32 v0, v0, v0, 16
+; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_shuffle10_a:
@@ -318,7 +319,8 @@ define i32 @v_sdot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-LABEL: v_sdot2_shuffle10_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
+; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.a = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %shuf.a, <2 x i16> %b, i32 %c, i1 false)
@@ -329,7 +331,8 @@ define i32 @v_sdot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_sdot2_shuffle10_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
+; GFX906-NEXT:    v_alignbit_b32 v1, v1, v1, 16
+; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_shuffle10_b:
@@ -343,7 +346,8 @@ define i32 @v_sdot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-LABEL: v_sdot2_shuffle10_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
+; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
+; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.b = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %shuf.b, i32 %c, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
index 287a009ca1405..9e623494a5a04 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
@@ -289,19 +289,22 @@ define i32 @v_udot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_udot2_shuffle10_a:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
+; GFX906-NEXT:    v_alignbit_b32 v0, v0, v0, 16
+; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_shuffle10_a:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
+; GFX908-NEXT:    v_alignbit_b32 v0, v0, v0, 16
+; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_shuffle10_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
+; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.a = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %shuf.a, <2 x i16> %b, i32 %c, i1 false)
@@ -312,19 +315,22 @@ define i32 @v_udot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_udot2_shuffle10_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
+; GFX906-NEXT:    v_alignbit_b32 v1, v1, v1, 16
+; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_shuffle10_b:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
+; GFX908-NEXT:    v_alignbit_b32 v1, v1, v1, 16
+; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_shuffle10_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
+; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
+; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.b = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %shuf.b, i32 %c, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 141b86a24c1c4..2f1dfa11fd34d 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -742,8 +742,9 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX9-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-GISEL-NEXT:    v_pk_add_f16 v0, s2, v0 neg_lo:[0,1] neg_hi:[0,1]
+; GFX9-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT:    v_pk_add_f16 v0, s2, v0
 ; GFX9-GISEL-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
@@ -783,7 +784,8 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX10-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX10-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
+; GFX10-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
 ; GFX10-GISEL-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-SDAG-TRUE16-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
@@ -808,7 +810,8 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX11-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX11-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
+; GFX11-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
 ; GFX11-GISEL-NEXT:    ; return to shader part epilog
 ; GFX10PLUS-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10PLUS-SDAG:       ; %bb.0:
@@ -821,7 +824,8 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ; GFX10PLUS-SDAG-NEXT:    ; return to shader part epilog
 ; GFX10PLUS-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10PLUS-GISEL:       ; %bb.0:
-; GFX10PLUS-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX10PLUS-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
+; GFX10PLUS-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
 ; GFX10PLUS-GISEL-NEXT:    ; return to shader part epilog
   %val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret <2 x half> %val

From 64de8528dbd89cdafe32d754f37cf3da4fb5403e Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn@gmail.com>
Date: Thu, 17 Apr 2025 20:50:51 -0700
Subject: [PATCH 363/710] [HLSL][RootSignature] Implement initial parsing of
 the descriptor table clause params (#133800)

- Defines `ParseDescriptorTableClauseParams` to establish the pattern of
how we will parse parameters in root signatures. Namely, to use
recursive descent parsing in a way that follows closely to the EBNF
notation definition in the root signature spec.

- Implements parsing of two param types: `UInt32` and `Register` to
demonstrate the parsing implementation and allow for unit testing

- Changes the calling convention to use `std::optional` return values
instead of boolean error returns and parameters by reference

Part two of implementing:
https://github.com/llvm/llvm-project/issues/126569

---------

Co-authored-by: Finn Plummer <finnplummer@microsoft.com>
---
 .../clang/Basic/DiagnosticParseKinds.td       |   5 +-
 .../clang/Parse/ParseHLSLRootSignature.h      |  42 +++--
 clang/lib/Parse/ParseHLSLRootSignature.cpp    | 164 ++++++++++++++++--
 .../Parse/ParseHLSLRootSignatureTest.cpp      | 146 +++++++++++++++-
 .../llvm/Frontend/HLSL/HLSLRootSignature.h    |   9 +
 5 files changed, 336 insertions(+), 30 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 9975520f4f9ff..72e765bcb800d 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1836,8 +1836,11 @@ def err_hlsl_virtual_function
 def err_hlsl_virtual_inheritance
     : Error<"virtual inheritance is unsupported in HLSL">;
 
-// HLSL Root Siganture diagnostic messages
+// HLSL Root Signature Parser Diagnostics
 def err_hlsl_unexpected_end_of_params
     : Error<"expected %0 to denote end of parameters, or, another valid parameter of %1">;
+def err_hlsl_rootsig_repeat_param : Error<"specified the same parameter '%0' multiple times">;
+def err_hlsl_rootsig_missing_param : Error<"did not specify mandatory parameter '%0'">;
+def err_hlsl_number_literal_overflow : Error<"integer literal is too large to be represented as a 32-bit %select{signed |}0 integer type">;
 
 } // end of Parser diagnostics
diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h
index a8dd6b02501ae..e17d6584622ba 100644
--- a/clang/include/clang/Parse/ParseHLSLRootSignature.h
+++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h
@@ -40,26 +40,31 @@ class RootSignatureParser {
 private:
   DiagnosticsEngine &getDiags() { return PP.getDiagnostics(); }
 
-  // All private Parse.* methods follow a similar pattern:
+  // All private parse.* methods follow a similar pattern:
   //   - Each method will start with an assert to denote what the CurToken is
   // expected to be and will parse from that token forward
   //
   //   - Therefore, it is the callers responsibility to ensure that you are
   // at the correct CurToken. This should be done with the pattern of:
   //
-  //  if (TryConsumeExpectedToken(RootSignatureToken::Kind))
-  //    if (Parse.*())
-  //      return true;
+  //  if (tryConsumeExpectedToken(RootSignatureToken::Kind)) {
+  //    auto ParsedObject = parse.*();
+  //    if (!ParsedObject.has_value())
+  //      return std::nullopt;
+  //    ...
+  // }
   //
   // or,
   //
-  //  if (ConsumeExpectedToken(RootSignatureToken::Kind, ...))
-  //    return true;
-  //  if (Parse.*())
-  //    return true;
+  //  if (consumeExpectedToken(RootSignatureToken::Kind, ...))
+  //    return std::nullopt;
+  //  auto ParsedObject = parse.*();
+  //  if (!ParsedObject.has_value())
+  //    return std::nullopt;
+  //  ...
   //
-  //   - All methods return true if a parsing error is encountered. It is the
-  // callers responsibility to propogate this error up, or deal with it
+  //   - All methods return std::nullopt if a parsing error is encountered. It
+  // is the callers responsibility to propogate this error up, or deal with it
   // otherwise
   //
   //   - An error will be raised if the proceeding tokens are not what is
@@ -69,6 +74,23 @@ class RootSignatureParser {
   bool parseDescriptorTable();
   bool parseDescriptorTableClause();
 
+  /// Parameter arguments (eg. `bReg`, `space`, ...) can be specified in any
+  /// order and only exactly once. `ParsedClauseParams` denotes the current
+  /// state of parsed params
+  struct ParsedClauseParams {
+    std::optional<llvm::hlsl::rootsig::Register> Register;
+    std::optional<uint32_t> Space;
+  };
+  std::optional<ParsedClauseParams>
+  parseDescriptorTableClauseParams(RootSignatureToken::Kind RegType);
+
+  std::optional<uint32_t> parseUIntParam();
+  std::optional<llvm::hlsl::rootsig::Register> parseRegister();
+
+  /// Use NumericLiteralParser to convert CurToken.NumSpelling into a unsigned
+  /// 32-bit integer
+  std::optional<uint32_t> handleUIntLiteral();
+
   /// Invoke the Lexer to consume a token and update CurToken with the result
   void consumeNextToken() { CurToken = Lexer.consumeToken(); }
 
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index 3513ef454f750..a13bee9750e2f 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -8,6 +8,8 @@
 
 #include "clang/Parse/ParseHLSLRootSignature.h"
 
+#include "clang/Lex/LiteralSupport.h"
+
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm::hlsl::rootsig;
@@ -41,12 +43,11 @@ bool RootSignatureParser::parse() {
       break;
   }
 
-  if (!tryConsumeExpectedToken(TokenKind::end_of_stream)) {
-    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_unexpected_end_of_params)
-        << /*expected=*/TokenKind::end_of_stream
-        << /*param of=*/TokenKind::kw_RootSignature;
+  if (consumeExpectedToken(TokenKind::end_of_stream,
+                           diag::err_hlsl_unexpected_end_of_params,
+                           /*param of=*/TokenKind::kw_RootSignature))
     return true;
-  }
+
   return false;
 }
 
@@ -72,12 +73,10 @@ bool RootSignatureParser::parseDescriptorTable() {
       break;
   }
 
-  if (!tryConsumeExpectedToken(TokenKind::pu_r_paren)) {
-    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_unexpected_end_of_params)
-        << /*expected=*/TokenKind::pu_r_paren
-        << /*param of=*/TokenKind::kw_DescriptorTable;
+  if (consumeExpectedToken(TokenKind::pu_r_paren,
+                           diag::err_hlsl_unexpected_end_of_params,
+                           /*param of=*/TokenKind::kw_DescriptorTable))
     return true;
-  }
 
   Elements.push_back(Table);
   return false;
@@ -90,36 +89,170 @@ bool RootSignatureParser::parseDescriptorTableClause() {
           CurToken.TokKind == TokenKind::kw_Sampler) &&
          "Expects to only be invoked starting at given keyword");
 
+  TokenKind ParamKind = CurToken.TokKind;
+
+  if (consumeExpectedToken(TokenKind::pu_l_paren, diag::err_expected_after,
+                           CurToken.TokKind))
+    return true;
+
   DescriptorTableClause Clause;
-  switch (CurToken.TokKind) {
+  TokenKind ExpectedReg;
+  switch (ParamKind) {
   default:
     llvm_unreachable("Switch for consumed token was not provided");
   case TokenKind::kw_CBV:
     Clause.Type = ClauseType::CBuffer;
+    ExpectedReg = TokenKind::bReg;
     break;
   case TokenKind::kw_SRV:
     Clause.Type = ClauseType::SRV;
+    ExpectedReg = TokenKind::tReg;
     break;
   case TokenKind::kw_UAV:
     Clause.Type = ClauseType::UAV;
+    ExpectedReg = TokenKind::uReg;
     break;
   case TokenKind::kw_Sampler:
     Clause.Type = ClauseType::Sampler;
+    ExpectedReg = TokenKind::sReg;
     break;
   }
 
-  if (consumeExpectedToken(TokenKind::pu_l_paren, diag::err_expected_after,
-                           CurToken.TokKind))
+  auto Params = parseDescriptorTableClauseParams(ExpectedReg);
+  if (!Params.has_value())
     return true;
 
-  if (consumeExpectedToken(TokenKind::pu_r_paren, diag::err_expected_after,
-                           CurToken.TokKind))
+  // Check mandatory parameters were provided
+  if (!Params->Register.has_value()) {
+    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_missing_param)
+        << ExpectedReg;
+    return true;
+  }
+
+  Clause.Register = Params->Register.value();
+
+  // Fill in optional values
+  if (Params->Space.has_value())
+    Clause.Space = Params->Space.value();
+
+  if (consumeExpectedToken(TokenKind::pu_r_paren,
+                           diag::err_hlsl_unexpected_end_of_params,
+                           /*param of=*/ParamKind))
     return true;
 
   Elements.push_back(Clause);
   return false;
 }
 
+std::optional<RootSignatureParser::ParsedClauseParams>
+RootSignatureParser::parseDescriptorTableClauseParams(TokenKind RegType) {
+  assert(CurToken.TokKind == TokenKind::pu_l_paren &&
+         "Expects to only be invoked starting at given token");
+
+  // Parameter arguments (eg. `bReg`, `space`, ...) can be specified in any
+  // order and only exactly once. Parse through as many arguments as possible
+  // reporting an error if a duplicate is seen.
+  ParsedClauseParams Params;
+  do {
+    // ( `b` | `t` | `u` | `s`) POS_INT
+    if (tryConsumeExpectedToken(RegType)) {
+      if (Params.Register.has_value()) {
+        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
+            << CurToken.TokKind;
+        return std::nullopt;
+      }
+      auto Reg = parseRegister();
+      if (!Reg.has_value())
+        return std::nullopt;
+      Params.Register = Reg;
+    }
+
+    // `space` `=` POS_INT
+    if (tryConsumeExpectedToken(TokenKind::kw_space)) {
+      if (Params.Space.has_value()) {
+        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
+            << CurToken.TokKind;
+        return std::nullopt;
+      }
+
+      if (consumeExpectedToken(TokenKind::pu_equal))
+        return std::nullopt;
+
+      auto Space = parseUIntParam();
+      if (!Space.has_value())
+        return std::nullopt;
+      Params.Space = Space;
+    }
+  } while (tryConsumeExpectedToken(TokenKind::pu_comma));
+
+  return Params;
+}
+
+std::optional<uint32_t> RootSignatureParser::parseUIntParam() {
+  assert(CurToken.TokKind == TokenKind::pu_equal &&
+         "Expects to only be invoked starting at given keyword");
+  tryConsumeExpectedToken(TokenKind::pu_plus);
+  if (consumeExpectedToken(TokenKind::int_literal, diag::err_expected_after,
+                           CurToken.TokKind))
+    return std::nullopt;
+  return handleUIntLiteral();
+}
+
+std::optional<Register> RootSignatureParser::parseRegister() {
+  assert((CurToken.TokKind == TokenKind::bReg ||
+          CurToken.TokKind == TokenKind::tReg ||
+          CurToken.TokKind == TokenKind::uReg ||
+          CurToken.TokKind == TokenKind::sReg) &&
+         "Expects to only be invoked starting at given keyword");
+
+  Register Register;
+  switch (CurToken.TokKind) {
+  default:
+    llvm_unreachable("Switch for consumed token was not provided");
+  case TokenKind::bReg:
+    Register.ViewType = RegisterType::BReg;
+    break;
+  case TokenKind::tReg:
+    Register.ViewType = RegisterType::TReg;
+    break;
+  case TokenKind::uReg:
+    Register.ViewType = RegisterType::UReg;
+    break;
+  case TokenKind::sReg:
+    Register.ViewType = RegisterType::SReg;
+    break;
+  }
+
+  auto Number = handleUIntLiteral();
+  if (!Number.has_value())
+    return std::nullopt; // propogate NumericLiteralParser error
+
+  Register.Number = *Number;
+  return Register;
+}
+
+std::optional<uint32_t> RootSignatureParser::handleUIntLiteral() {
+  // Parse the numeric value and do semantic checks on its specification
+  clang::NumericLiteralParser Literal(CurToken.NumSpelling, CurToken.TokLoc,
+                                      PP.getSourceManager(), PP.getLangOpts(),
+                                      PP.getTargetInfo(), PP.getDiagnostics());
+  if (Literal.hadError)
+    return true; // Error has already been reported so just return
+
+  assert(Literal.isIntegerLiteral() && "IsNumberChar will only support digits");
+
+  llvm::APSInt Val = llvm::APSInt(32, false);
+  if (Literal.GetIntegerValue(Val)) {
+    // Report that the value has overflowed
+    PP.getDiagnostics().Report(CurToken.TokLoc,
+                               diag::err_hlsl_number_literal_overflow)
+        << 0 << CurToken.NumSpelling;
+    return std::nullopt;
+  }
+
+  return Val.getExtValue();
+}
+
 bool RootSignatureParser::peekExpectedToken(TokenKind Expected) {
   return peekExpectedToken(ArrayRef{Expected});
 }
@@ -141,6 +274,7 @@ bool RootSignatureParser::consumeExpectedToken(TokenKind Expected,
   case diag::err_expected:
     DB << Expected;
     break;
+  case diag::err_hlsl_unexpected_end_of_params:
   case diag::err_expected_either:
   case diag::err_expected_after:
     DB << Expected << Context;
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index 19d5b267f310a..56ee5d1183ace 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -129,10 +129,10 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseEmptyTest) {
 TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   const llvm::StringLiteral Source = R"cc(
     DescriptorTable(
-      CBV(),
-      SRV(),
-      Sampler(),
-      UAV()
+      CBV(b0),
+      SRV(space = 3, t42),
+      Sampler(s987, space = +2),
+      UAV(u4294967294)
     ),
     DescriptorTable()
   )cc";
@@ -154,18 +154,34 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   RootElement Elem = Elements[0];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.ViewType,
+            RegisterType::BReg);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.Number, 0u);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 0u);
 
   Elem = Elements[1];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.ViewType,
+            RegisterType::TReg);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.Number, 42u);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 3u);
 
   Elem = Elements[2];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.ViewType,
+            RegisterType::SReg);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.Number, 987u);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 2u);
 
   Elem = Elements[3];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.ViewType,
+            RegisterType::UReg);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.Number, 4294967294u);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 0u);
 
   Elem = Elements[4];
   ASSERT_TRUE(std::holds_alternative<DescriptorTable>(Elem));
@@ -175,6 +191,32 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   Elem = Elements[5];
   ASSERT_TRUE(std::holds_alternative<DescriptorTable>(Elem));
   ASSERT_EQ(std::get<DescriptorTable>(Elem).NumClauses, 0u);
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, ValidTrailingCommaTest) {
+  // This test will checks we can handling trailing commas ','
+  const llvm::StringLiteral Source = R"cc(
+    DescriptorTable(
+      CBV(b0, ),
+      SRV(t42),
+    )
+  )cc";
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+  auto TokLoc = SourceLocation();
+
+  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+
+  // Test no diagnostics produced
+  Consumer->setNoDiag();
+
+  ASSERT_FALSE(Parser.parse());
+
   ASSERT_TRUE(Consumer->isSatisfied());
 }
 
@@ -236,6 +278,102 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseUnexpectedEndOfStreamTest) {
 
   // Test correct diagnostic produced - end of stream
   Consumer->setExpected(diag::err_expected_after);
+
+  ASSERT_TRUE(Parser.parse());
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, InvalidMissingParameterTest) {
+  // This test will check that the parsing fails due a mandatory
+  // parameter (register) not being specified
+  const llvm::StringLiteral Source = R"cc(
+    DescriptorTable(
+      CBV()
+    )
+  )cc";
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+  auto TokLoc = SourceLocation();
+
+  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+
+  // Test correct diagnostic produced
+  Consumer->setExpected(diag::err_hlsl_rootsig_missing_param);
+  ASSERT_TRUE(Parser.parse());
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedMandatoryParameterTest) {
+  // This test will check that the parsing fails due the same mandatory
+  // parameter being specified multiple times
+  const llvm::StringLiteral Source = R"cc(
+    DescriptorTable(
+      CBV(b32, b84)
+    )
+  )cc";
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+  auto TokLoc = SourceLocation();
+
+  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+
+  // Test correct diagnostic produced
+  Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
+  ASSERT_TRUE(Parser.parse());
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedOptionalParameterTest) {
+  // This test will check that the parsing fails due the same optional
+  // parameter being specified multiple times
+  const llvm::StringLiteral Source = R"cc(
+    DescriptorTable(
+      CBV(space = 2, space = 0)
+    )
+  )cc";
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+  auto TokLoc = SourceLocation();
+
+  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+
+  // Test correct diagnostic produced
+  Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
+  ASSERT_TRUE(Parser.parse());
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedNumberTest) {
+  // This test will check that the lexing fails due to an integer overflow
+  const llvm::StringLiteral Source = R"cc(
+    DescriptorTable(
+      CBV(b4294967296)
+    )
+  )cc";
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+  auto TokLoc = SourceLocation();
+
+  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+
+  // Test correct diagnostic produced
+  Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
   ASSERT_TRUE(Parser.parse());
 
   ASSERT_TRUE(Consumer->isSatisfied());
diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
index c1b67844c747f..6a35c114100b9 100644
--- a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
+++ b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
@@ -23,6 +23,13 @@ namespace rootsig {
 
 // Definitions of the in-memory data layout structures
 
+// Models the different registers: bReg | tReg | uReg | sReg
+enum class RegisterType { BReg, TReg, UReg, SReg };
+struct Register {
+  RegisterType ViewType;
+  uint32_t Number;
+};
+
 // Models the end of a descriptor table and stores its visibility
 struct DescriptorTable {
   uint32_t NumClauses = 0; // The number of clauses in the table
@@ -32,6 +39,8 @@ struct DescriptorTable {
 using ClauseType = llvm::dxil::ResourceClass;
 struct DescriptorTableClause {
   ClauseType Type;
+  Register Register;
+  uint32_t Space = 0;
 };
 
 // Models RootElement : DescriptorTable | DescriptorTableClause

From 2721f5af12bdf9af29969a86351fe485af00e648 Mon Sep 17 00:00:00 2001
From: Fehr Mathieu <mathieu.fehr@gmail.com>
Date: Fri, 18 Apr 2025 05:53:26 +0200
Subject: [PATCH 364/710] [mlir][vector] Prevent folding of OOB values in
 insert/extract (#135498)

Out of bound position values should not be folded in vector.extract and
vector.insert operations, as only in bounds constants and -1 are valid.

Fixes #134516
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp   | 15 +++++++--
 mlir/test/Dialect/Vector/canonicalize.mlir | 38 ++++++++++++++++++++++
 2 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index cea31276d10a8..368259b38b153 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -1997,6 +1997,11 @@ static Value extractInsertFoldConstantOp(OpType op, AdaptorType adaptor,
   std::vector<int64_t> staticPosition = op.getStaticPosition().vec();
   OperandRange dynamicPosition = op.getDynamicPosition();
   ArrayRef<Attribute> dynamicPositionAttr = adaptor.getDynamicPosition();
+  ArrayRef<int64_t> vectorShape;
+  if constexpr (std::is_same_v<OpType, ExtractOp>)
+    vectorShape = op.getSourceVectorType().getShape();
+  else
+    vectorShape = op.getDestVectorType().getShape();
 
   // If the dynamic operands is empty, it is returned directly.
   if (!dynamicPosition.size())
@@ -2013,9 +2018,13 @@ static Value extractInsertFoldConstantOp(OpType op, AdaptorType adaptor,
     Attribute positionAttr = dynamicPositionAttr[index];
     Value position = dynamicPosition[index++];
     if (auto attr = mlir::dyn_cast_if_present<IntegerAttr>(positionAttr)) {
-      staticPosition[i] = attr.getInt();
-      opChange = true;
-      continue;
+      int64_t value = attr.getInt();
+      // Do not fold if the value is out of bounds.
+      if (value >= 0 && value < vectorShape[i]) {
+        staticPosition[i] = attr.getInt();
+        opChange = true;
+        continue;
+      }
     }
     operands.push_back(position);
   }
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index f50d92fc9c8b1..2d365ac2b4287 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -3311,3 +3311,41 @@ func.func @fold_insert_constant_indices(%arg : vector<4x1xi32>) -> vector<4x1xi3
   %res = vector.insert %1, %arg[%0, %0] : i32 into vector<4x1xi32>
   return %res : vector<4x1xi32>
 }
+
+// -----
+
+// Check that out of bounds indices are not folded for vector.insert.
+
+// CHECK-LABEL: @fold_insert_oob
+//  CHECK-SAME:   %[[ARG:.*]]: vector<4x1x2xi32>) -> vector<4x1x2xi32> {
+//       CHECK:   %[[OOB1:.*]] = arith.constant -2 : index
+//       CHECK:   %[[OOB2:.*]] = arith.constant 2 : index
+//       CHECK:   %[[VAL:.*]] = arith.constant 1 : i32
+//       CHECK:   %[[RES:.*]] = vector.insert %[[VAL]], %[[ARG]] [0, %[[OOB1]], %[[OOB2]]] : i32 into vector<4x1x2xi32>
+//       CHECK:   return %[[RES]] : vector<4x1x2xi32>
+func.func @fold_insert_oob(%arg : vector<4x1x2xi32>) -> vector<4x1x2xi32> {
+  %c0 = arith.constant 0 : index
+  %c-2 = arith.constant -2 : index
+  %c2 = arith.constant 2 : index
+  %c1 = arith.constant 1 : i32
+  %res = vector.insert %c1, %arg[%c0, %c-2, %c2] : i32 into vector<4x1x2xi32>
+  return %res : vector<4x1x2xi32>
+}
+
+// -----
+
+// Check that out of bounds indices are not folded for vector.extract.
+
+// CHECK-LABEL: @fold_extract_oob
+//  CHECK-SAME:   %[[ARG:.*]]: vector<4x1x2xi32>) -> i32 {
+//       CHECK:   %[[OOB1:.*]] = arith.constant -2 : index
+//       CHECK:   %[[OOB2:.*]] = arith.constant 2 : index
+//       CHECK:   %[[RES:.*]] = vector.extract %[[ARG]][0, %[[OOB1]], %[[OOB2]]] : i32 from vector<4x1x2xi32>
+//       CHECK:   return %[[RES]] : i32
+func.func @fold_extract_oob(%arg : vector<4x1x2xi32>) -> i32 {
+  %c0 = arith.constant 0 : index
+  %c-2 = arith.constant -2 : index
+  %c2 = arith.constant 2 : index
+  %res = vector.extract %arg[%c0, %c-2, %c2] : i32 from vector<4x1x2xi32>
+  return %res : i32
+}

From 6f0a35fbd3296d3f4154e19f189130edcd0980bd Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 17 Apr 2025 20:57:09 -0700
Subject: [PATCH 365/710] [AArch64] Use FirsRelocationKind+i fixup kinds to
 force relocations

For RELA targets, fixup kinds that force relocations (GOT, TLS, ALIGN,
RELAX, etc) can bypass `applyFixup` and be encoded as
`FirstRelocationKind+i`, as seen in LoongArch.
---
 llvm/include/llvm/MC/MCFixup.h                            | 4 ++++
 .../lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp | 8 ++++----
 .../AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp       | 5 +++++
 .../Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp  | 5 ++---
 4 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/MC/MCFixup.h b/llvm/include/llvm/MC/MCFixup.h
index f27ddeae8b173..2f2717d689049 100644
--- a/llvm/include/llvm/MC/MCFixup.h
+++ b/llvm/include/llvm/MC/MCFixup.h
@@ -89,6 +89,10 @@ class MCFixup {
     FI.Loc = Loc;
     return FI;
   }
+  static MCFixup create(uint32_t Offset, const MCExpr *Value, unsigned Kind,
+                        SMLoc Loc = SMLoc()) {
+    return create(Offset, Value, MCFixupKind(Kind), Loc);
+  }
 
   MCFixupKind getKind() const { return Kind; }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index b43de2dba92a2..9600ba508a110 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -67,9 +67,9 @@ class AArch64AsmBackend : public MCAsmBackend {
         {"fixup_aarch64_pcrel_branch26", 0, 26, PCRelFlagVal},
         {"fixup_aarch64_pcrel_call26", 0, 26, PCRelFlagVal}};
 
-    // Fixup kinds from .reloc directive are like R_AARCH64_NONE. They do not
-    // require any extra processing.
-    if (Kind >= FirstLiteralRelocationKind)
+    // Fixup kinds from raw relocation types and .reloc directives force
+    // relocations and do not need these fields.
+    if (Kind >= FirstRelocationKind)
       return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
     if (Kind < FirstTargetFixupKind)
@@ -442,7 +442,7 @@ void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
   if (!Value)
     return; // Doesn't change encoding.
   unsigned Kind = Fixup.getKind();
-  if (Kind >= FirstLiteralRelocationKind)
+  if (Kind >= FirstRelocationKind)
     return;
   unsigned NumBytes = getFixupKindNumBytes(Kind);
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index 44ecc0c2706b1..4457cff30101e 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -127,6 +127,11 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
     break;
   }
 
+  // Extract the relocation type from the fixup kind, after applying STT_TLS as
+  // needed.
+  if (Kind >= FirstRelocationKind)
+    return Kind - FirstRelocationKind;
+
   if (IsPCRel) {
     switch (Kind) {
     case FK_Data_1:
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
index bd43c73396463..9f9b6aba3fe6c 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
@@ -749,9 +749,8 @@ void AArch64MCCodeEmitter::encodeInstruction(const MCInst &MI,
     auto Reloc = STI.getTargetTriple().getEnvironment() == Triple::GNUILP32
                      ? ELF::R_AARCH64_P32_TLSDESC_CALL
                      : ELF::R_AARCH64_TLSDESC_CALL;
-    Fixups.push_back(
-        MCFixup::create(0, MI.getOperand(0).getExpr(),
-                        MCFixupKind(FirstLiteralRelocationKind + Reloc)));
+    Fixups.push_back(MCFixup::create(0, MI.getOperand(0).getExpr(),
+                                     FirstRelocationKind + Reloc));
     return;
   }
 

From c134e99a2adf6ac3f0d7a5f15a2f384f174c80a5 Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn@gmail.com>
Date: Thu, 17 Apr 2025 20:58:16 -0700
Subject: [PATCH 366/710] Revert "[HLSL][RootSignature] Implement initial
 parsing of the descriptor table clause params" (#136252)

Reverts llvm/llvm-project#133800

Reverting to resolve the introduce naming collisions.
---
 .../clang/Basic/DiagnosticParseKinds.td       |   5 +-
 .../clang/Parse/ParseHLSLRootSignature.h      |  42 ++---
 clang/lib/Parse/ParseHLSLRootSignature.cpp    | 164 ++----------------
 .../Parse/ParseHLSLRootSignatureTest.cpp      | 146 +---------------
 .../llvm/Frontend/HLSL/HLSLRootSignature.h    |   9 -
 5 files changed, 30 insertions(+), 336 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 72e765bcb800d..9975520f4f9ff 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1836,11 +1836,8 @@ def err_hlsl_virtual_function
 def err_hlsl_virtual_inheritance
     : Error<"virtual inheritance is unsupported in HLSL">;
 
-// HLSL Root Signature Parser Diagnostics
+// HLSL Root Siganture diagnostic messages
 def err_hlsl_unexpected_end_of_params
     : Error<"expected %0 to denote end of parameters, or, another valid parameter of %1">;
-def err_hlsl_rootsig_repeat_param : Error<"specified the same parameter '%0' multiple times">;
-def err_hlsl_rootsig_missing_param : Error<"did not specify mandatory parameter '%0'">;
-def err_hlsl_number_literal_overflow : Error<"integer literal is too large to be represented as a 32-bit %select{signed |}0 integer type">;
 
 } // end of Parser diagnostics
diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h
index e17d6584622ba..a8dd6b02501ae 100644
--- a/clang/include/clang/Parse/ParseHLSLRootSignature.h
+++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h
@@ -40,31 +40,26 @@ class RootSignatureParser {
 private:
   DiagnosticsEngine &getDiags() { return PP.getDiagnostics(); }
 
-  // All private parse.* methods follow a similar pattern:
+  // All private Parse.* methods follow a similar pattern:
   //   - Each method will start with an assert to denote what the CurToken is
   // expected to be and will parse from that token forward
   //
   //   - Therefore, it is the callers responsibility to ensure that you are
   // at the correct CurToken. This should be done with the pattern of:
   //
-  //  if (tryConsumeExpectedToken(RootSignatureToken::Kind)) {
-  //    auto ParsedObject = parse.*();
-  //    if (!ParsedObject.has_value())
-  //      return std::nullopt;
-  //    ...
-  // }
+  //  if (TryConsumeExpectedToken(RootSignatureToken::Kind))
+  //    if (Parse.*())
+  //      return true;
   //
   // or,
   //
-  //  if (consumeExpectedToken(RootSignatureToken::Kind, ...))
-  //    return std::nullopt;
-  //  auto ParsedObject = parse.*();
-  //  if (!ParsedObject.has_value())
-  //    return std::nullopt;
-  //  ...
+  //  if (ConsumeExpectedToken(RootSignatureToken::Kind, ...))
+  //    return true;
+  //  if (Parse.*())
+  //    return true;
   //
-  //   - All methods return std::nullopt if a parsing error is encountered. It
-  // is the callers responsibility to propogate this error up, or deal with it
+  //   - All methods return true if a parsing error is encountered. It is the
+  // callers responsibility to propogate this error up, or deal with it
   // otherwise
   //
   //   - An error will be raised if the proceeding tokens are not what is
@@ -74,23 +69,6 @@ class RootSignatureParser {
   bool parseDescriptorTable();
   bool parseDescriptorTableClause();
 
-  /// Parameter arguments (eg. `bReg`, `space`, ...) can be specified in any
-  /// order and only exactly once. `ParsedClauseParams` denotes the current
-  /// state of parsed params
-  struct ParsedClauseParams {
-    std::optional<llvm::hlsl::rootsig::Register> Register;
-    std::optional<uint32_t> Space;
-  };
-  std::optional<ParsedClauseParams>
-  parseDescriptorTableClauseParams(RootSignatureToken::Kind RegType);
-
-  std::optional<uint32_t> parseUIntParam();
-  std::optional<llvm::hlsl::rootsig::Register> parseRegister();
-
-  /// Use NumericLiteralParser to convert CurToken.NumSpelling into a unsigned
-  /// 32-bit integer
-  std::optional<uint32_t> handleUIntLiteral();
-
   /// Invoke the Lexer to consume a token and update CurToken with the result
   void consumeNextToken() { CurToken = Lexer.consumeToken(); }
 
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index a13bee9750e2f..3513ef454f750 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -8,8 +8,6 @@
 
 #include "clang/Parse/ParseHLSLRootSignature.h"
 
-#include "clang/Lex/LiteralSupport.h"
-
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm::hlsl::rootsig;
@@ -43,11 +41,12 @@ bool RootSignatureParser::parse() {
       break;
   }
 
-  if (consumeExpectedToken(TokenKind::end_of_stream,
-                           diag::err_hlsl_unexpected_end_of_params,
-                           /*param of=*/TokenKind::kw_RootSignature))
+  if (!tryConsumeExpectedToken(TokenKind::end_of_stream)) {
+    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_unexpected_end_of_params)
+        << /*expected=*/TokenKind::end_of_stream
+        << /*param of=*/TokenKind::kw_RootSignature;
     return true;
-
+  }
   return false;
 }
 
@@ -73,10 +72,12 @@ bool RootSignatureParser::parseDescriptorTable() {
       break;
   }
 
-  if (consumeExpectedToken(TokenKind::pu_r_paren,
-                           diag::err_hlsl_unexpected_end_of_params,
-                           /*param of=*/TokenKind::kw_DescriptorTable))
+  if (!tryConsumeExpectedToken(TokenKind::pu_r_paren)) {
+    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_unexpected_end_of_params)
+        << /*expected=*/TokenKind::pu_r_paren
+        << /*param of=*/TokenKind::kw_DescriptorTable;
     return true;
+  }
 
   Elements.push_back(Table);
   return false;
@@ -89,170 +90,36 @@ bool RootSignatureParser::parseDescriptorTableClause() {
           CurToken.TokKind == TokenKind::kw_Sampler) &&
          "Expects to only be invoked starting at given keyword");
 
-  TokenKind ParamKind = CurToken.TokKind;
-
-  if (consumeExpectedToken(TokenKind::pu_l_paren, diag::err_expected_after,
-                           CurToken.TokKind))
-    return true;
-
   DescriptorTableClause Clause;
-  TokenKind ExpectedReg;
-  switch (ParamKind) {
+  switch (CurToken.TokKind) {
   default:
     llvm_unreachable("Switch for consumed token was not provided");
   case TokenKind::kw_CBV:
     Clause.Type = ClauseType::CBuffer;
-    ExpectedReg = TokenKind::bReg;
     break;
   case TokenKind::kw_SRV:
     Clause.Type = ClauseType::SRV;
-    ExpectedReg = TokenKind::tReg;
     break;
   case TokenKind::kw_UAV:
     Clause.Type = ClauseType::UAV;
-    ExpectedReg = TokenKind::uReg;
     break;
   case TokenKind::kw_Sampler:
     Clause.Type = ClauseType::Sampler;
-    ExpectedReg = TokenKind::sReg;
     break;
   }
 
-  auto Params = parseDescriptorTableClauseParams(ExpectedReg);
-  if (!Params.has_value())
-    return true;
-
-  // Check mandatory parameters were provided
-  if (!Params->Register.has_value()) {
-    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_missing_param)
-        << ExpectedReg;
+  if (consumeExpectedToken(TokenKind::pu_l_paren, diag::err_expected_after,
+                           CurToken.TokKind))
     return true;
-  }
-
-  Clause.Register = Params->Register.value();
-
-  // Fill in optional values
-  if (Params->Space.has_value())
-    Clause.Space = Params->Space.value();
 
-  if (consumeExpectedToken(TokenKind::pu_r_paren,
-                           diag::err_hlsl_unexpected_end_of_params,
-                           /*param of=*/ParamKind))
+  if (consumeExpectedToken(TokenKind::pu_r_paren, diag::err_expected_after,
+                           CurToken.TokKind))
     return true;
 
   Elements.push_back(Clause);
   return false;
 }
 
-std::optional<RootSignatureParser::ParsedClauseParams>
-RootSignatureParser::parseDescriptorTableClauseParams(TokenKind RegType) {
-  assert(CurToken.TokKind == TokenKind::pu_l_paren &&
-         "Expects to only be invoked starting at given token");
-
-  // Parameter arguments (eg. `bReg`, `space`, ...) can be specified in any
-  // order and only exactly once. Parse through as many arguments as possible
-  // reporting an error if a duplicate is seen.
-  ParsedClauseParams Params;
-  do {
-    // ( `b` | `t` | `u` | `s`) POS_INT
-    if (tryConsumeExpectedToken(RegType)) {
-      if (Params.Register.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
-        return std::nullopt;
-      }
-      auto Reg = parseRegister();
-      if (!Reg.has_value())
-        return std::nullopt;
-      Params.Register = Reg;
-    }
-
-    // `space` `=` POS_INT
-    if (tryConsumeExpectedToken(TokenKind::kw_space)) {
-      if (Params.Space.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
-        return std::nullopt;
-      }
-
-      if (consumeExpectedToken(TokenKind::pu_equal))
-        return std::nullopt;
-
-      auto Space = parseUIntParam();
-      if (!Space.has_value())
-        return std::nullopt;
-      Params.Space = Space;
-    }
-  } while (tryConsumeExpectedToken(TokenKind::pu_comma));
-
-  return Params;
-}
-
-std::optional<uint32_t> RootSignatureParser::parseUIntParam() {
-  assert(CurToken.TokKind == TokenKind::pu_equal &&
-         "Expects to only be invoked starting at given keyword");
-  tryConsumeExpectedToken(TokenKind::pu_plus);
-  if (consumeExpectedToken(TokenKind::int_literal, diag::err_expected_after,
-                           CurToken.TokKind))
-    return std::nullopt;
-  return handleUIntLiteral();
-}
-
-std::optional<Register> RootSignatureParser::parseRegister() {
-  assert((CurToken.TokKind == TokenKind::bReg ||
-          CurToken.TokKind == TokenKind::tReg ||
-          CurToken.TokKind == TokenKind::uReg ||
-          CurToken.TokKind == TokenKind::sReg) &&
-         "Expects to only be invoked starting at given keyword");
-
-  Register Register;
-  switch (CurToken.TokKind) {
-  default:
-    llvm_unreachable("Switch for consumed token was not provided");
-  case TokenKind::bReg:
-    Register.ViewType = RegisterType::BReg;
-    break;
-  case TokenKind::tReg:
-    Register.ViewType = RegisterType::TReg;
-    break;
-  case TokenKind::uReg:
-    Register.ViewType = RegisterType::UReg;
-    break;
-  case TokenKind::sReg:
-    Register.ViewType = RegisterType::SReg;
-    break;
-  }
-
-  auto Number = handleUIntLiteral();
-  if (!Number.has_value())
-    return std::nullopt; // propogate NumericLiteralParser error
-
-  Register.Number = *Number;
-  return Register;
-}
-
-std::optional<uint32_t> RootSignatureParser::handleUIntLiteral() {
-  // Parse the numeric value and do semantic checks on its specification
-  clang::NumericLiteralParser Literal(CurToken.NumSpelling, CurToken.TokLoc,
-                                      PP.getSourceManager(), PP.getLangOpts(),
-                                      PP.getTargetInfo(), PP.getDiagnostics());
-  if (Literal.hadError)
-    return true; // Error has already been reported so just return
-
-  assert(Literal.isIntegerLiteral() && "IsNumberChar will only support digits");
-
-  llvm::APSInt Val = llvm::APSInt(32, false);
-  if (Literal.GetIntegerValue(Val)) {
-    // Report that the value has overflowed
-    PP.getDiagnostics().Report(CurToken.TokLoc,
-                               diag::err_hlsl_number_literal_overflow)
-        << 0 << CurToken.NumSpelling;
-    return std::nullopt;
-  }
-
-  return Val.getExtValue();
-}
-
 bool RootSignatureParser::peekExpectedToken(TokenKind Expected) {
   return peekExpectedToken(ArrayRef{Expected});
 }
@@ -274,7 +141,6 @@ bool RootSignatureParser::consumeExpectedToken(TokenKind Expected,
   case diag::err_expected:
     DB << Expected;
     break;
-  case diag::err_hlsl_unexpected_end_of_params:
   case diag::err_expected_either:
   case diag::err_expected_after:
     DB << Expected << Context;
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index 56ee5d1183ace..19d5b267f310a 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -129,10 +129,10 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseEmptyTest) {
 TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   const llvm::StringLiteral Source = R"cc(
     DescriptorTable(
-      CBV(b0),
-      SRV(space = 3, t42),
-      Sampler(s987, space = +2),
-      UAV(u4294967294)
+      CBV(),
+      SRV(),
+      Sampler(),
+      UAV()
     ),
     DescriptorTable()
   )cc";
@@ -154,34 +154,18 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   RootElement Elem = Elements[0];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.ViewType,
-            RegisterType::BReg);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.Number, 0u);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 0u);
 
   Elem = Elements[1];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.ViewType,
-            RegisterType::TReg);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.Number, 42u);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 3u);
 
   Elem = Elements[2];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.ViewType,
-            RegisterType::SReg);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.Number, 987u);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 2u);
 
   Elem = Elements[3];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.ViewType,
-            RegisterType::UReg);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Register.Number, 4294967294u);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 0u);
 
   Elem = Elements[4];
   ASSERT_TRUE(std::holds_alternative<DescriptorTable>(Elem));
@@ -191,32 +175,6 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   Elem = Elements[5];
   ASSERT_TRUE(std::holds_alternative<DescriptorTable>(Elem));
   ASSERT_EQ(std::get<DescriptorTable>(Elem).NumClauses, 0u);
-
-  ASSERT_TRUE(Consumer->isSatisfied());
-}
-
-TEST_F(ParseHLSLRootSignatureTest, ValidTrailingCommaTest) {
-  // This test will checks we can handling trailing commas ','
-  const llvm::StringLiteral Source = R"cc(
-    DescriptorTable(
-      CBV(b0, ),
-      SRV(t42),
-    )
-  )cc";
-
-  TrivialModuleLoader ModLoader;
-  auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
-
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
-  SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
-
-  // Test no diagnostics produced
-  Consumer->setNoDiag();
-
-  ASSERT_FALSE(Parser.parse());
-
   ASSERT_TRUE(Consumer->isSatisfied());
 }
 
@@ -278,102 +236,6 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseUnexpectedEndOfStreamTest) {
 
   // Test correct diagnostic produced - end of stream
   Consumer->setExpected(diag::err_expected_after);
-
-  ASSERT_TRUE(Parser.parse());
-
-  ASSERT_TRUE(Consumer->isSatisfied());
-}
-
-TEST_F(ParseHLSLRootSignatureTest, InvalidMissingParameterTest) {
-  // This test will check that the parsing fails due a mandatory
-  // parameter (register) not being specified
-  const llvm::StringLiteral Source = R"cc(
-    DescriptorTable(
-      CBV()
-    )
-  )cc";
-
-  TrivialModuleLoader ModLoader;
-  auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
-
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
-  SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
-
-  // Test correct diagnostic produced
-  Consumer->setExpected(diag::err_hlsl_rootsig_missing_param);
-  ASSERT_TRUE(Parser.parse());
-
-  ASSERT_TRUE(Consumer->isSatisfied());
-}
-
-TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedMandatoryParameterTest) {
-  // This test will check that the parsing fails due the same mandatory
-  // parameter being specified multiple times
-  const llvm::StringLiteral Source = R"cc(
-    DescriptorTable(
-      CBV(b32, b84)
-    )
-  )cc";
-
-  TrivialModuleLoader ModLoader;
-  auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
-
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
-  SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
-
-  // Test correct diagnostic produced
-  Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
-  ASSERT_TRUE(Parser.parse());
-
-  ASSERT_TRUE(Consumer->isSatisfied());
-}
-
-TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedOptionalParameterTest) {
-  // This test will check that the parsing fails due the same optional
-  // parameter being specified multiple times
-  const llvm::StringLiteral Source = R"cc(
-    DescriptorTable(
-      CBV(space = 2, space = 0)
-    )
-  )cc";
-
-  TrivialModuleLoader ModLoader;
-  auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
-
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
-  SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
-
-  // Test correct diagnostic produced
-  Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
-  ASSERT_TRUE(Parser.parse());
-
-  ASSERT_TRUE(Consumer->isSatisfied());
-}
-
-TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedNumberTest) {
-  // This test will check that the lexing fails due to an integer overflow
-  const llvm::StringLiteral Source = R"cc(
-    DescriptorTable(
-      CBV(b4294967296)
-    )
-  )cc";
-
-  TrivialModuleLoader ModLoader;
-  auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
-
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
-  SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
-
-  // Test correct diagnostic produced
-  Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
   ASSERT_TRUE(Parser.parse());
 
   ASSERT_TRUE(Consumer->isSatisfied());
diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
index 6a35c114100b9..c1b67844c747f 100644
--- a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
+++ b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
@@ -23,13 +23,6 @@ namespace rootsig {
 
 // Definitions of the in-memory data layout structures
 
-// Models the different registers: bReg | tReg | uReg | sReg
-enum class RegisterType { BReg, TReg, UReg, SReg };
-struct Register {
-  RegisterType ViewType;
-  uint32_t Number;
-};
-
 // Models the end of a descriptor table and stores its visibility
 struct DescriptorTable {
   uint32_t NumClauses = 0; // The number of clauses in the table
@@ -39,8 +32,6 @@ struct DescriptorTable {
 using ClauseType = llvm::dxil::ResourceClass;
 struct DescriptorTableClause {
   ClauseType Type;
-  Register Register;
-  uint32_t Space = 0;
 };
 
 // Models RootElement : DescriptorTable | DescriptorTableClause

From 23324b8b109aed1f77cb20cef476b795f33b6835 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 17 Apr 2025 21:03:58 -0700
Subject: [PATCH 367/710] [RISCV] Move checking for constant 3/4 for
 XTHeadMemPair to the instruction matching stage. (#136165)

This removes a special case from processInstruction and removes an
untested range diagnostic we would print if the constant didn't fit in 3
bits.
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 21 ++++------
 .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h |  2 +
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |  6 +++
 llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td | 42 +++++++++++++++----
 .../test/MC/RISCV/rv32xtheadmempair-invalid.s |  4 +-
 5 files changed, 51 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 4178e29b81976..db9ec7ae1df02 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -932,6 +932,14 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     return isUImmPred([](int64_t Imm) { return 0 == Imm; });
   }
 
+  bool isImmThree() const {
+    return isUImmPred([](int64_t Imm) { return 3 == Imm; });
+  }
+
+  bool isImmFour() const {
+    return isUImmPred([](int64_t Imm) { return 4 == Imm; });
+  }
+
   bool isSImm5Plus1() const {
     return isSImmPred(
         [](int64_t Imm) { return Imm != INT64_MIN && isInt<5>(Imm - 1); });
@@ -3663,19 +3671,6 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     }
   }
 
-  bool IsTHeadMemPair32 = (Opcode == RISCV::TH_LWD ||
-                           Opcode == RISCV::TH_LWUD || Opcode == RISCV::TH_SWD);
-  bool IsTHeadMemPair64 = (Opcode == RISCV::TH_LDD || Opcode == RISCV::TH_SDD);
-  // The last operand of XTHeadMemPair instructions must be constant 3 or 4
-  // depending on the data width.
-  if (IsTHeadMemPair32 && Inst.getOperand(4).getImm() != 3) {
-    SMLoc Loc = Operands.back()->getStartLoc();
-    return Error(Loc, "operand must be constant 3");
-  } else if (IsTHeadMemPair64 && Inst.getOperand(4).getImm() != 4) {
-    SMLoc Loc = Operands.back()->getStartLoc();
-    return Error(Loc, "operand must be constant 4");
-  }
-
   const MCInstrDesc &MCID = MII.get(Opcode);
   if (!(MCID.TSFlags & RISCVII::ConstraintMask))
     return false;
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index 84a23de6b1995..6ef94fb5e93da 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -329,6 +329,8 @@ enum OperandType : unsigned {
   OPERAND_UIMM48,
   OPERAND_UIMM64,
   OPERAND_ZERO,
+  OPERAND_THREE,
+  OPERAND_FOUR,
   OPERAND_SIMM5,
   OPERAND_SIMM5_NONZERO,
   OPERAND_SIMM5_PLUS1,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 5d661a3438b1c..04ad56ea83230 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2645,6 +2645,12 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         case RISCVOp::OPERAND_ZERO:
           Ok = Imm == 0;
           break;
+        case RISCVOp::OPERAND_THREE:
+          Ok = Imm == 3;
+          break;
+        case RISCVOp::OPERAND_FOUR:
+          Ok = Imm == 4;
+          break;
           // clang-format off
         CASE_OPERAND_SIMM(5)
         CASE_OPERAND_SIMM(6)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index f69b24ae7d9c5..b1cd2202b9be8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -34,6 +34,30 @@ def th_swd : SDNode<"RISCVISD::TH_SWD", SDT_StorePair,
 def th_sdd : SDNode<"RISCVISD::TH_SDD", SDT_StorePair,
                     [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 
+def ImmThreeAsmOperand : AsmOperandClass {
+  let Name = "ImmThree";
+  let RenderMethod = "addImmOperands";
+  let DiagnosticType = !strconcat("Invalid", Name);
+  let DiagnosticString = "operand must be constant 3";
+}
+
+def immthree : RISCVOp {
+  let ParserMatchClass = ImmThreeAsmOperand;
+  let OperandType = "OPERAND_THREE";
+}
+
+def ImmFourAsmOperand : AsmOperandClass {
+  let Name = "ImmFour";
+  let RenderMethod = "addImmOperands";
+  let DiagnosticType = !strconcat("Invalid", Name);
+  let DiagnosticString = "operand must be constant 4";
+}
+
+def immfour : RISCVOp {
+  let ParserMatchClass = ImmFourAsmOperand;
+  let OperandType = "OPERAND_FOUR";
+}
+
 //===----------------------------------------------------------------------===//
 // Instruction class templates
 //===----------------------------------------------------------------------===//
@@ -131,10 +155,10 @@ class THMulAccumulate_rr<bits<7> funct7, string opcodestr>
 }
 
 let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
-class THLoadPair<bits<5> funct5, string opcodestr>
+class THLoadPair<bits<5> funct5, string opcodestr, Operand consttype>
   : RVInstRBase<0b100, OPC_CUSTOM_0,
                 (outs GPR:$rd, GPR:$rs2),
-                (ins GPR:$rs1, uimm2:$uimm2, uimm7:$const3or4),
+                (ins GPR:$rs1, uimm2:$uimm2, consttype:$const3or4),
                  opcodestr, "$rd, $rs2, (${rs1}), $uimm2, $const3or4"> {
   bits<2> uimm2;
   let Inst{31-27} = funct5;
@@ -144,9 +168,9 @@ class THLoadPair<bits<5> funct5, string opcodestr>
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
-class THStorePair<bits<5> funct5, string opcodestr>
+class THStorePair<bits<5> funct5, string opcodestr, Operand consttype>
   : RVInstRBase<0b101, OPC_CUSTOM_0, (outs),
-              (ins GPR:$rd, GPR:$rs2, GPR:$rs1, uimm2:$uimm2, uimm7:$const3or4),
+              (ins GPR:$rd, GPR:$rs2, GPR:$rs1, uimm2:$uimm2, consttype:$const3or4),
               opcodestr, "$rd, $rs2, (${rs1}), $uimm2, $const3or4"> {
   bits<2> uimm2;
   let Inst{31-27} = funct5;
@@ -290,19 +314,19 @@ def TH_MULSW : THMulAccumulate_rr<0b0010011, "th.mulsw">;
 } // Predicates = [HasVendorXTHeadMac, IsRV64]
 
 let Predicates = [HasVendorXTHeadMemPair] in {
-def TH_LWUD : THLoadPair<0b11110, "th.lwud">,
+def TH_LWUD : THLoadPair<0b11110, "th.lwud", immthree>,
               Sched<[WriteLDW, WriteLDW, ReadMemBase]>;
-def TH_SWD  : THStorePair<0b11100, "th.swd">,
+def TH_SWD  : THStorePair<0b11100, "th.swd", immthree>,
               Sched<[WriteSTW, WriteSTW, ReadStoreData, ReadMemBase]>;
 let IsSignExtendingOpW = 1 in
-def TH_LWD  : THLoadPair<0b11100, "th.lwd">,
+def TH_LWD  : THLoadPair<0b11100, "th.lwd", immthree>,
               Sched<[WriteLDW, WriteLDW, ReadMemBase]>;
 }
 
 let Predicates = [HasVendorXTHeadMemPair, IsRV64] in {
-def TH_LDD : THLoadPair<0b11111, "th.ldd">,
+def TH_LDD : THLoadPair<0b11111, "th.ldd", immfour>,
              Sched<[WriteLDD, WriteLDD, ReadMemBase]>;
-def TH_SDD : THStorePair<0b11111, "th.sdd">,
+def TH_SDD : THStorePair<0b11111, "th.sdd", immfour>,
              Sched<[WriteSTD, WriteSTD, ReadStoreData, ReadMemBase]>;
 }
 
diff --git a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
index 6b95b3dc66b43..8a7202ffd375e 100644
--- a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
@@ -2,10 +2,10 @@
 
 th.ldd t0, t1, (t2), 5, 4   # CHECK: [[@LINE]]:22: error: invalid operand for instruction
 th.ldd t0, t1, (t2)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.ldd t0, t1, (t2), 3, 5   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
+th.ldd t0, t1, (t2), 3, 5   # CHECK: [[@LINE]]:25: error: invalid operand for instruction
 th.sdd a0, a1, (a2), 5, 4   # CHECK: [[@LINE]]:22: error: invalid operand for instruction
 th.sdd a0, a1, (a2)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.sdd a0, a1, (a2), 3, 5   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
+th.sdd a0, a1, (a2), 3, 5   # CHECK: [[@LINE]]:25: error: invalid operand for instruction
 th.lwud t0, t1, (t2), 5, 4  # CHECK: [[@LINE]]:23: error: immediate must be an integer in the range [0, 3]
 th.lwud t0, t1, (t2)        # CHECK: [[@LINE]]:1: error: too few operands for instruction
 th.lwud t0, t1, (t2), 3, 5  # CHECK: [[@LINE]]:26: error: operand must be constant 3

From 8c5a307bd8d406e6167a5cd3ce3c74e2e3bfb2a6 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Fri, 18 Apr 2025 06:10:58 +0200
Subject: [PATCH 368/710] [Clang] Bypass TAD during overload resolution if a
 perfect match exists (#136203)

This implements the same overload resolution behavior as GCC,
as described in https://wg21.link/p3606 (section 1-2, not 3)

If during overload resolution, there is a non-template candidate
that would be always be picked - because each of the argument
is a perfect match (ie the source and target types are the same),
we do not perform deduction for any template candidate
that might exists.

The goal is to be able to merge
https://github.com/llvm/llvm-project/pull/122423 without being too
disruptive.

This change means that the selection of the best viable candidate and
template argument deduction become interleaved.

To avoid rewriting half of Clang we store in OverloadCandidateSet
enough information to be able to deduce template candidates from
OverloadCandidateSet::BestViableFunction. Which means
the lifetime of any object used by template argument must outlive
a call to Add*Template*Candidate.

This two phase resolution is not performed for some initialization
as there are cases where template candidate are better match
in these cases per the standard. It's also bypassed for code completion.

The change has a nice impact on compile times

https://llvm-compile-time-tracker.com/compare.php?from=719b029c16eeb1035da522fd641dfcc4cee6be74&to=bf7041045c9408490c395230047c5461de72fc39&stat=instructions%3Au

Fixes https://github.com/llvm/llvm-project/issues/62096
Fixes https://github.com/llvm/llvm-project/issues/74581

Reapplies https://github.com/llvm/llvm-project/pull/133426
---
 clang/docs/ReleaseNotes.rst                   |   6 +
 clang/include/clang/Sema/Overload.h           | 240 +++++++-
 clang/lib/Sema/SemaCodeComplete.cpp           |   6 +-
 clang/lib/Sema/SemaInit.cpp                   |  18 +-
 clang/lib/Sema/SemaOverload.cpp               | 532 ++++++++++++++----
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   4 +-
 .../constrant-satisfaction-conversions.cpp    |   8 +-
 clang/test/SemaCUDA/function-overload.cu      |   3 -
 .../SemaCXX/implicit-member-functions.cpp     |  21 +-
 ...overload-resolution-deferred-templates.cpp | 206 +++++++
 .../instantiate-function-params.cpp           |   7 +-
 .../Templight/templight-empty-entries-fix.cpp | 126 ++---
 12 files changed, 958 insertions(+), 219 deletions(-)
 create mode 100644 clang/test/SemaCXX/overload-resolution-deferred-templates.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index c75d83a6d1a7a..7b76a2caa694b 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -96,6 +96,12 @@ C++ Language Changes
       asm((std::string_view("nop")) ::: (std::string_view("memory")));
     }
 
+- Clang now implements the changes to overload resolution proposed by section 1 and 2 of
+  `P3606 <https://wg21.link/P3606R0>`_. If a non-template candidate exists in an overload set that is
+  a perfect match (all conversion sequences are identity conversions) template candidates are not instantiated.
+  Diagnostics that would have resulted from the instantiation of these template candidates are no longer
+  produced. This aligns Clang closer to the behavior of GCC, and fixes (#GH62096), (#GH74581), and (#GH74581).
+
 C++2c Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 6e08762dcc6d7..55d714c1222a1 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -407,6 +407,34 @@ class Sema;
              Third == ICK_Identity;
     }
 
+    /// A conversion sequence is perfect if it is an identity conversion and
+    /// the type of the source is the same as the type of the target.
+    bool isPerfect(const ASTContext &C) const {
+      if (!isIdentityConversion())
+        return false;
+      // If we are not performing a reference binding, we can skip comparing
+      // the types, which has a noticeable performance impact.
+      if (!ReferenceBinding) {
+#ifndef NDEBUG
+        auto Decay = [&](QualType T) {
+          return (T->isArrayType() || T->isFunctionType()) ? C.getDecayedType(T)
+                                                           : T;
+        };
+        // The types might differ if there is an array-to-pointer conversion
+        // an function-to-pointer conversion, or lvalue-to-rvalue conversion.
+        // In some cases, this may happen even if First is not set.
+        assert(C.hasSameUnqualifiedType(Decay(getFromType()),
+                                        Decay(getToType(2))));
+#endif
+        return true;
+      }
+      if (!C.hasSameType(getFromType(), getToType(2)))
+        return false;
+      if (BindsToRvalue && IsLvalueReference)
+        return false;
+      return true;
+    }
+
     ImplicitConversionRank getRank() const;
     NarrowingKind
     getNarrowingKind(ASTContext &Context, const Expr *Converted,
@@ -743,6 +771,12 @@ class Sema;
       Standard.setAllToTypes(T);
     }
 
+    /// A conversion sequence is perfect if it is an identity conversion and
+    /// the type of the source is the same as the type of the target.
+    bool isPerfect(const ASTContext &C) const {
+      return isStandard() && Standard.isPerfect(C);
+    }
+
     // True iff this is a conversion sequence from an initializer list to an
     // array or std::initializer.
     bool hasInitializerListContainerType() const {
@@ -939,6 +973,10 @@ class Sema;
     LLVM_PREFERRED_TYPE(CallExpr::ADLCallKind)
     unsigned IsADLCandidate : 1;
 
+    /// Whether FinalConversion has been set.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasFinalConversion : 1;
+
     /// Whether this is a rewritten candidate, and if so, of what kind?
     LLVM_PREFERRED_TYPE(OverloadCandidateRewriteKind)
     unsigned RewriteKind : 2;
@@ -979,6 +1017,20 @@ class Sema;
       return false;
     }
 
+    // An overload is a perfect match if the conversion
+    // sequences for each argument are perfect.
+    bool isPerfectMatch(const ASTContext &Ctx) const {
+      if (!Viable)
+        return false;
+      for (const auto &C : Conversions) {
+        if (!C.isInitialized() || !C.isPerfect(Ctx))
+          return false;
+      }
+      if (HasFinalConversion)
+        return FinalConversion.isPerfect(Ctx);
+      return true;
+    }
+
     bool TryToFixBadConversion(unsigned Idx, Sema &S) {
       bool CanFix = Fix.tryToFixConversion(
                       Conversions[Idx].Bad.FromExpr,
@@ -1012,8 +1064,67 @@ class Sema;
         : IsSurrogate(false), IgnoreObjectArgument(false),
           TookAddressOfOverload(false), StrictPackMatch(false),
           IsADLCandidate(llvm::to_underlying(CallExpr::NotADL)),
-          RewriteKind(CRK_None) {}
+          HasFinalConversion(false), RewriteKind(CRK_None) {}
+  };
+
+  struct DeferredTemplateOverloadCandidate {
+
+    // intrusive linked list support for allocateDeferredCandidate
+    DeferredTemplateOverloadCandidate *Next = nullptr;
+
+    enum Kind { Function, Method, Conversion };
+
+    LLVM_PREFERRED_TYPE(Kind)
+    unsigned Kind : 2;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowObjCConversionOnExplicit : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowResultConversion : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AllowExplicit : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned SuppressUserConversions : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned PartialOverloading : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned AggregateCandidateDeduction : 1;
+  };
+
+  struct DeferredFunctionTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    ArrayRef<Expr *> Args;
+    CallExpr::ADLCallKind IsADLCandidate;
+    OverloadCandidateParamOrder PO;
+  };
+  static_assert(std::is_trivially_destructible_v<
+                DeferredFunctionTemplateOverloadCandidate>);
+
+  struct DeferredMethodTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    ArrayRef<Expr *> Args;
+    CXXRecordDecl *ActingContext;
+    Expr::Classification ObjectClassification;
+    QualType ObjectType;
+    OverloadCandidateParamOrder PO;
   };
+  static_assert(std::is_trivially_destructible_v<
+                DeferredMethodTemplateOverloadCandidate>);
+
+  struct DeferredConversionTemplateOverloadCandidate
+      : public DeferredTemplateOverloadCandidate {
+    FunctionTemplateDecl *FunctionTemplate;
+    DeclAccessPair FoundDecl;
+    CXXRecordDecl *ActingContext;
+    Expr *From;
+    QualType ToType;
+  };
+
+  static_assert(std::is_trivially_destructible_v<
+                DeferredConversionTemplateOverloadCandidate>);
 
   /// OverloadCandidateSet - A set of overload candidates, used in C++
   /// overload resolution (C++ 13.3).
@@ -1043,6 +1154,11 @@ class Sema;
       /// C++ [over.match.call.general]
       /// Resolve a call through the address of an overload set.
       CSK_AddressOfOverloadSet,
+
+      /// When doing overload resolution during code completion,
+      /// we want to show all viable candidates, including otherwise
+      /// deferred template candidates.
+      CSK_CodeCompletion,
     };
 
     /// Information about operator rewrites to consider when adding operator
@@ -1117,7 +1233,15 @@ class Sema;
     SmallVector<OverloadCandidate, 16> Candidates;
     llvm::SmallPtrSet<uintptr_t, 16> Functions;
 
-    // Allocator for ConversionSequenceLists. We store the first few of these
+    DeferredTemplateOverloadCandidate *FirstDeferredCandidate = nullptr;
+    unsigned DeferredCandidatesCount : 8 * sizeof(unsigned) - 2;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasDeferredTemplateConstructors : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned ResolutionByPerfectCandidateIsDisabled : 1;
+
+    // Allocator for ConversionSequenceLists and deferred candidate args.
+    // We store the first few of these
     // inline to avoid allocation for small sets.
     llvm::BumpPtrAllocator SlabAllocator;
 
@@ -1125,8 +1249,11 @@ class Sema;
     CandidateSetKind Kind;
     OperatorRewriteInfo RewriteInfo;
 
+    /// Small storage size for ImplicitConversionSequences
+    /// and the persisted arguments of deferred candidates.
     constexpr static unsigned NumInlineBytes =
-        24 * sizeof(ImplicitConversionSequence);
+        32 * sizeof(ImplicitConversionSequence);
+
     unsigned NumInlineBytesUsed = 0;
     alignas(void *) char InlineSpace[NumInlineBytes];
 
@@ -1137,15 +1264,13 @@ class Sema;
     /// from the slab allocator.
     /// FIXME: It would probably be nice to have a SmallBumpPtrAllocator
     /// instead.
-    /// FIXME: Now that this only allocates ImplicitConversionSequences, do we
-    /// want to un-generalize this?
     template <typename T>
     T *slabAllocate(unsigned N) {
       // It's simpler if this doesn't need to consider alignment.
       static_assert(alignof(T) == alignof(void *),
                     "Only works for pointer-aligned types.");
-      static_assert(std::is_trivial<T>::value ||
-                        std::is_same<ImplicitConversionSequence, T>::value,
+      static_assert(std::is_trivially_destructible_v<T> ||
+                        (std::is_same_v<ImplicitConversionSequence, T>),
                     "Add destruction logic to OverloadCandidateSet::clear().");
 
       unsigned NBytes = sizeof(T) * N;
@@ -1159,12 +1284,34 @@ class Sema;
       return reinterpret_cast<T *>(FreeSpaceStart);
     }
 
+    // Because the size of OverloadCandidateSet has a noticeable impact on
+    // performance, we store each deferred template candidate in the slab
+    // allocator such that deferred candidates are ultimately a singly-linked
+    // intrusive linked list. This ends up being much more efficient than a
+    // SmallVector that is empty in the common case.
+    template <typename T> T *allocateDeferredCandidate() {
+      T *C = slabAllocate<T>(1);
+      if (!FirstDeferredCandidate)
+        FirstDeferredCandidate = C;
+      else {
+        auto *F = FirstDeferredCandidate;
+        while (F->Next)
+          F = F->Next;
+        F->Next = C;
+      }
+      DeferredCandidatesCount++;
+      return C;
+    }
+
     void destroyCandidates();
 
   public:
     OverloadCandidateSet(SourceLocation Loc, CandidateSetKind CSK,
                          OperatorRewriteInfo RewriteInfo = {})
-        : Loc(Loc), Kind(CSK), RewriteInfo(RewriteInfo) {}
+        : FirstDeferredCandidate(nullptr), DeferredCandidatesCount(0),
+          HasDeferredTemplateConstructors(false),
+          ResolutionByPerfectCandidateIsDisabled(false), Loc(Loc), Kind(CSK),
+          RewriteInfo(RewriteInfo) {}
     OverloadCandidateSet(const OverloadCandidateSet &) = delete;
     OverloadCandidateSet &operator=(const OverloadCandidateSet &) = delete;
     ~OverloadCandidateSet() { destroyCandidates(); }
@@ -1176,6 +1323,9 @@ class Sema;
     /// Whether diagnostics should be deferred.
     bool shouldDeferDiags(Sema &S, ArrayRef<Expr *> Args, SourceLocation OpLoc);
 
+    // Whether the resolution of template candidates should be deferred
+    bool shouldDeferTemplateArgumentDeduction(const LangOptions &Opts) const;
+
     /// Determine when this overload candidate will be new to the
     /// overload set.
     bool isNewCandidate(Decl *F, OverloadCandidateParamOrder PO =
@@ -1199,8 +1349,10 @@ class Sema;
     iterator begin() { return Candidates.begin(); }
     iterator end() { return Candidates.end(); }
 
-    size_t size() const { return Candidates.size(); }
-    bool empty() const { return Candidates.empty(); }
+    size_t size() const { return Candidates.size() + DeferredCandidatesCount; }
+    bool empty() const {
+      return Candidates.empty() && DeferredCandidatesCount == 0;
+    }
 
     /// Allocate storage for conversion sequences for NumConversions
     /// conversions.
@@ -1216,6 +1368,24 @@ class Sema;
       return ConversionSequenceList(Conversions, NumConversions);
     }
 
+    /// Provide storage for any Expr* arg that must be preserved
+    /// until deferred template candidates are deduced.
+    /// Typically this should be used for reversed operator arguments
+    /// and any time the argument array is transformed while adding
+    /// a template candidate.
+    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(unsigned N) {
+      Expr **Exprs = slabAllocate<Expr *>(N);
+      return llvm::MutableArrayRef<Expr *>(Exprs, N);
+    }
+
+    template <typename... T>
+    llvm::MutableArrayRef<Expr *> getPersistentArgsArray(T *...Exprs) {
+      llvm::MutableArrayRef<Expr *> Arr =
+          getPersistentArgsArray(sizeof...(Exprs));
+      llvm::copy(std::initializer_list<Expr *>{Exprs...}, Arr.data());
+      return Arr;
+    }
+
     /// Add a new candidate with NumConversions conversion sequence slots
     /// to the overload set.
     OverloadCandidate &addCandidate(unsigned NumConversions = 0,
@@ -1231,6 +1401,32 @@ class Sema;
       return C;
     }
 
+    void AddDeferredTemplateCandidate(
+        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+        ArrayRef<Expr *> Args, bool SuppressUserConversions,
+        bool PartialOverloading, bool AllowExplicit,
+        CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+        bool AggregateCandidateDeduction);
+
+    void AddDeferredMethodTemplateCandidate(
+        FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+        CXXRecordDecl *ActingContext, QualType ObjectType,
+        Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+        bool SuppressUserConversions, bool PartialOverloading,
+        OverloadCandidateParamOrder PO);
+
+    void AddDeferredConversionTemplateCandidate(
+        FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+        CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+        bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+        bool AllowResultConversion);
+
+    void InjectNonDeducedTemplateCandidates(Sema &S);
+
+    void DisableResolutionByPerfectCandidate() {
+      ResolutionByPerfectCandidateIsDisabled = true;
+    }
+
     /// Find the best viable function on this overload set, if it exists.
     OverloadingResult BestViableFunction(Sema &S, SourceLocation Loc,
                                          OverloadCandidateSet::iterator& Best);
@@ -1263,6 +1459,15 @@ class Sema;
       DestAS = AS;
     }
 
+  private:
+    OverloadingResult ResultForBestCandidate(const iterator &Best);
+    void CudaExcludeWrongSideCandidates(
+        Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates);
+    OverloadingResult
+    BestViableFunctionImpl(Sema &S, SourceLocation Loc,
+                           OverloadCandidateSet::iterator &Best);
+    void PerfectViableFunction(Sema &S, SourceLocation Loc,
+                               OverloadCandidateSet::iterator &Best);
   };
 
   bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
@@ -1311,6 +1516,21 @@ class Sema;
   // parameter.
   bool shouldEnforceArgLimit(bool PartialOverloading, FunctionDecl *Function);
 
+  inline bool OverloadCandidateSet::shouldDeferTemplateArgumentDeduction(
+      const LangOptions &Opts) const {
+    return
+        // For user defined conversion we need to check against different
+        // combination of CV qualifiers and look at any explicit specifier, so
+        // always deduce template candidates.
+        Kind != CSK_InitByUserDefinedConversion
+        // When doing code completion, we want to see all the
+        // viable candidates.
+        && Kind != CSK_CodeCompletion
+        // CUDA may prefer template candidates even when a non-candidate
+        // is a perfect match
+        && !Opts.CUDA;
+  }
+
 } // namespace clang
 
 #endif // LLVM_CLANG_SEMA_OVERLOAD_H
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 1e4e6fdc78351..ed10730ef986b 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -6354,7 +6354,8 @@ SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
   Expr *NakedFn = Fn->IgnoreParenCasts();
   // Build an overload candidate set based on the functions we find.
   SourceLocation Loc = Fn->getExprLoc();
-  OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
+  OverloadCandidateSet CandidateSet(Loc,
+                                    OverloadCandidateSet::CSK_CodeCompletion);
 
   if (auto ULE = dyn_cast<UnresolvedLookupExpr>(NakedFn)) {
     SemaRef.AddOverloadedCallCandidates(ULE, ArgsWithoutDependentTypes,
@@ -6557,7 +6558,8 @@ QualType SemaCodeCompletion::ProduceConstructorSignatureHelp(
   // FIXME: Provide support for variadic template constructors.
 
   if (CRD) {
-    OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
+    OverloadCandidateSet CandidateSet(Loc,
+                                      OverloadCandidateSet::CSK_CodeCompletion);
     for (NamedDecl *C : SemaRef.LookupConstructors(CRD)) {
       if (auto *FD = dyn_cast<FunctionDecl>(C)) {
         // FIXME: we can't yet provide correct signature help for initializer
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 77d7f821f2011..9e802a175bb05 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -5231,7 +5231,7 @@ static OverloadingResult TryRefInitWithConversionFunction(
 
   // Add the final conversion sequence, if necessary.
   if (NewRefRelationship == Sema::Ref_Incompatible) {
-    assert(!isa<CXXConstructorDecl>(Function) &&
+    assert(Best->HasFinalConversion && !isa<CXXConstructorDecl>(Function) &&
            "should not have conversion after constructor");
 
     ImplicitConversionSequence ICS;
@@ -6200,6 +6200,7 @@ static void TryUserDefinedConversion(Sema &S,
 
   // If the conversion following the call to the conversion function
   // is interesting, add it as a separate step.
+  assert(Best->HasFinalConversion);
   if (Best->FinalConversion.First || Best->FinalConversion.Second ||
       Best->FinalConversion.Third) {
     ImplicitConversionSequence ICS;
@@ -10029,12 +10030,19 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     //   When [...] the constructor [...] is a candidate by
     //    - [over.match.copy] (in all cases)
     if (TD) {
-      SmallVector<Expr *, 8> TmpInits;
-      for (Expr *E : Inits)
+
+      // As template candidates are not deduced immediately,
+      // persist the array in the overload set.
+      MutableArrayRef<Expr *> TmpInits =
+          Candidates.getPersistentArgsArray(Inits.size());
+
+      for (auto [I, E] : llvm::enumerate(Inits)) {
         if (auto *DI = dyn_cast<DesignatedInitExpr>(E))
-          TmpInits.push_back(DI->getInit());
+          TmpInits[I] = DI->getInit();
         else
-          TmpInits.push_back(E);
+          TmpInits[I] = E;
+      }
+
       AddTemplateOverloadCandidate(
           TD, FoundDecl, /*ExplicitArgs=*/nullptr, TmpInits, Candidates,
           /*SuppressUserConversions=*/false,
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 55634aa75ae25..b7a981e08ead9 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1123,6 +1123,10 @@ void OverloadCandidateSet::clear(CandidateSetKind CSK) {
   Candidates.clear();
   Functions.clear();
   Kind = CSK;
+  FirstDeferredCandidate = nullptr;
+  DeferredCandidatesCount = 0;
+  HasDeferredTemplateConstructors = false;
+  ResolutionByPerfectCandidateIsDisabled = false;
 }
 
 namespace {
@@ -4075,6 +4079,9 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
     }
     if (CXXConversionDecl *Conversion
                  = dyn_cast<CXXConversionDecl>(Best->Function)) {
+
+      assert(Best->HasFinalConversion);
+
       // C++ [over.ics.user]p1:
       //
       //   [...] If the user-defined conversion is specified by a
@@ -5158,6 +5165,9 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
   case OR_Success:
+
+    assert(Best->HasFinalConversion);
+
     // C++ [over.ics.ref]p1:
     //
     //   [...] If the parameter binds directly to the result of
@@ -7795,15 +7805,14 @@ void Sema::AddMethodCandidate(
   }
 }
 
-void Sema::AddMethodTemplateCandidate(
+static void AddMethodTemplateCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
     CXXRecordDecl *ActingContext,
     TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
     Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, OverloadCandidateParamOrder PO) {
-  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
-    return;
+    bool SuppressUserConversions, bool PartialOverloading,
+    OverloadCandidateParamOrder PO) {
 
   // C++ [over.match.funcs]p7:
   //   In each case where a candidate is a function template, candidate
@@ -7817,12 +7826,12 @@ void Sema::AddMethodTemplateCandidate(
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           MethodTmpl, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, /*AggregateDeductionCandidate=*/false,
           /*PartialOrdering=*/false, ObjectType, ObjectClassification,
           [&](ArrayRef<QualType> ParamTypes) {
-            return CheckNonDependentConversions(
+            return S.CheckNonDependentConversions(
                 MethodTmpl, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, ActingContext, ObjectType,
                 ObjectClassification, PO);
@@ -7844,8 +7853,8 @@ void Sema::AddMethodTemplateCandidate(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                            Info);
+      Candidate.DeductionFailure =
+          MakeDeductionFailureInfo(S.Context, Result, Info);
     }
     return;
   }
@@ -7855,10 +7864,34 @@ void Sema::AddMethodTemplateCandidate(
   assert(Specialization && "Missing member function template specialization?");
   assert(isa<CXXMethodDecl>(Specialization) &&
          "Specialization is not a member function?");
-  AddMethodCandidate(cast<CXXMethodDecl>(Specialization), FoundDecl,
-                     ActingContext, ObjectType, ObjectClassification, Args,
-                     CandidateSet, SuppressUserConversions, PartialOverloading,
-                     Conversions, PO, Info.hasStrictPackMatch());
+  S.AddMethodCandidate(
+      cast<CXXMethodDecl>(Specialization), FoundDecl, ActingContext, ObjectType,
+      ObjectClassification, Args, CandidateSet, SuppressUserConversions,
+      PartialOverloading, Conversions, PO, Info.hasStrictPackMatch());
+}
+
+void Sema::AddMethodTemplateCandidate(
+    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext,
+    TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType,
+    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, OverloadCandidateParamOrder PO) {
+  if (!CandidateSet.isNewCandidate(MethodTmpl, PO))
+    return;
+
+  if (ExplicitTemplateArgs ||
+      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts())) {
+    AddMethodTemplateCandidateImmediately(
+        *this, CandidateSet, MethodTmpl, FoundDecl, ActingContext,
+        ExplicitTemplateArgs, ObjectType, ObjectClassification, Args,
+        SuppressUserConversions, PartialOverloading, PO);
+    return;
+  }
+
+  CandidateSet.AddDeferredMethodTemplateCandidate(
+      MethodTmpl, FoundDecl, ActingContext, ObjectType, ObjectClassification,
+      Args, SuppressUserConversions, PartialOverloading, PO);
 }
 
 /// Determine whether a given function template has a simple explicit specifier
@@ -7867,14 +7900,18 @@ static bool isNonDependentlyExplicit(FunctionTemplateDecl *FTD) {
   return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).isExplicit();
 }
 
-void Sema::AddTemplateOverloadCandidate(
+static bool hasDependentExplicit(FunctionTemplateDecl *FTD) {
+  return ExplicitSpecifier::getFromDecl(FTD->getTemplatedDecl()).getKind() ==
+         ExplicitSpecKind::Unresolved;
+}
+
+static void AddTemplateOverloadCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
     TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
-    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
-  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
-    return;
+    bool SuppressUserConversions, bool PartialOverloading, bool AllowExplicit,
+    Sema::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+    bool AggregateCandidateDeduction) {
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -7901,14 +7938,14 @@ void Sema::AddTemplateOverloadCandidate(
                              FunctionTemplate->getTemplateDepth());
   FunctionDecl *Specialization = nullptr;
   ConversionSequenceList Conversions;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           FunctionTemplate, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, AggregateCandidateDeduction,
           /*PartialOrdering=*/false,
           /*ObjectType=*/QualType(),
           /*ObjectClassification=*/Expr::Classification(),
           [&](ArrayRef<QualType> ParamTypes) {
-            return CheckNonDependentConversions(
+            return S.CheckNonDependentConversions(
                 FunctionTemplate, ParamTypes, Args, CandidateSet, Conversions,
                 SuppressUserConversions, nullptr, QualType(), {}, PO);
           });
@@ -7932,8 +7969,8 @@ void Sema::AddTemplateOverloadCandidate(
       Candidate.FailureKind = ovl_fail_bad_conversion;
     else {
       Candidate.FailureKind = ovl_fail_bad_deduction;
-      Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                            Info);
+      Candidate.DeductionFailure =
+          MakeDeductionFailureInfo(S.Context, Result, Info);
     }
     return;
   }
@@ -7941,7 +7978,7 @@ void Sema::AddTemplateOverloadCandidate(
   // Add the function template specialization produced by template argument
   // deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  AddOverloadCandidate(
+  S.AddOverloadCandidate(
       Specialization, FoundDecl, Args, CandidateSet, SuppressUserConversions,
       PartialOverloading, AllowExplicit,
       /*AllowExplicitConversions=*/false, IsADLCandidate, Conversions, PO,
@@ -7949,6 +7986,38 @@ void Sema::AddTemplateOverloadCandidate(
       Info.hasStrictPackMatch());
 }
 
+void Sema::AddTemplateOverloadCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
+    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
+    bool PartialOverloading, bool AllowExplicit, ADLCallKind IsADLCandidate,
+    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
+  if (!CandidateSet.isNewCandidate(FunctionTemplate, PO))
+    return;
+
+  bool DependentExplicitSpecifier = hasDependentExplicit(FunctionTemplate);
+
+  if (ExplicitTemplateArgs ||
+      !CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
+      (isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl()) &&
+       DependentExplicitSpecifier)) {
+
+    AddTemplateOverloadCandidateImmediately(
+        *this, CandidateSet, FunctionTemplate, FoundDecl, ExplicitTemplateArgs,
+        Args, SuppressUserConversions, PartialOverloading, AllowExplicit,
+        IsADLCandidate, PO, AggregateCandidateDeduction);
+
+    if (DependentExplicitSpecifier)
+      CandidateSet.DisableResolutionByPerfectCandidate();
+    return;
+  }
+
+  CandidateSet.AddDeferredTemplateCandidate(
+      FunctionTemplate, FoundDecl, Args, SuppressUserConversions,
+      PartialOverloading, AllowExplicit, IsADLCandidate, PO,
+      AggregateCandidateDeduction);
+}
+
 bool Sema::CheckNonDependentConversions(
     FunctionTemplateDecl *FunctionTemplate, ArrayRef<QualType> ParamTypes,
     ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet,
@@ -8111,6 +8180,7 @@ void Sema::AddConversionCandidate(
   Candidate.FinalConversion.setAsIdentityConversion();
   Candidate.FinalConversion.setFromType(ConvType);
   Candidate.FinalConversion.setAllToTypes(ToType);
+  Candidate.HasFinalConversion = true;
   Candidate.Viable = true;
   Candidate.ExplicitCallArguments = 1;
   Candidate.StrictPackMatch = StrictPackMatch;
@@ -8215,6 +8285,7 @@ void Sema::AddConversionCandidate(
   switch (ICS.getKind()) {
   case ImplicitConversionSequence::StandardConversion:
     Candidate.FinalConversion = ICS.Standard;
+    Candidate.HasFinalConversion = true;
 
     // C++ [over.ics.user]p3:
     //   If the user-defined conversion is specified by a specialization of a
@@ -8264,16 +8335,12 @@ void Sema::AddConversionCandidate(
   }
 }
 
-void Sema::AddTemplateConversionCandidate(
+static void AddTemplateConversionCandidateImmediately(
+    Sema &S, OverloadCandidateSet &CandidateSet,
     FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
-    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
-    bool AllowExplicit, bool AllowResultConversion) {
-  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
-         "Only conversion function templates permitted here");
-
-  if (!CandidateSet.isNewCandidate(FunctionTemplate))
-    return;
+    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+    bool AllowResultConversion) {
 
   // If the function template has a non-dependent explicit specification,
   // exclude it now if appropriate; we are not permitted to perform deduction
@@ -8288,11 +8355,11 @@ void Sema::AddTemplateConversionCandidate(
   }
 
   QualType ObjectType = From->getType();
-  Expr::Classification ObjectClassification = From->Classify(getASTContext());
+  Expr::Classification ObjectClassification = From->Classify(S.Context);
 
   TemplateDeductionInfo Info(CandidateSet.getLocation());
   CXXConversionDecl *Specialization = nullptr;
-  if (TemplateDeductionResult Result = DeduceTemplateArguments(
+  if (TemplateDeductionResult Result = S.DeduceTemplateArguments(
           FunctionTemplate, ObjectType, ObjectClassification, ToType,
           Specialization, Info);
       Result != TemplateDeductionResult::Success) {
@@ -8302,18 +8369,47 @@ void Sema::AddTemplateConversionCandidate(
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_deduction;
     Candidate.ExplicitCallArguments = 1;
-    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
-                                                          Info);
+    Candidate.DeductionFailure =
+        MakeDeductionFailureInfo(S.Context, Result, Info);
     return;
   }
 
   // Add the conversion function template specialization produced by
   // template argument deduction as a candidate.
   assert(Specialization && "Missing function template specialization?");
-  AddConversionCandidate(Specialization, FoundDecl, ActingDC, From, ToType,
-                         CandidateSet, AllowObjCConversionOnExplicit,
-                         AllowExplicit, AllowResultConversion,
-                         Info.hasStrictPackMatch());
+  S.AddConversionCandidate(Specialization, FoundDecl, ActingContext, From,
+                           ToType, CandidateSet, AllowObjCConversionOnExplicit,
+                           AllowExplicit, AllowResultConversion,
+                           Info.hasStrictPackMatch());
+}
+
+void Sema::AddTemplateConversionCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingDC, Expr *From, QualType ToType,
+    OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
+    bool AllowExplicit, bool AllowResultConversion) {
+  assert(isa<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl()) &&
+         "Only conversion function templates permitted here");
+
+  if (!CandidateSet.isNewCandidate(FunctionTemplate))
+    return;
+
+  if (!CandidateSet.shouldDeferTemplateArgumentDeduction(getLangOpts()) ||
+      CandidateSet.getKind() ==
+          OverloadCandidateSet::CSK_InitByUserDefinedConversion ||
+      CandidateSet.getKind() == OverloadCandidateSet::CSK_InitByConstructor) {
+    AddTemplateConversionCandidateImmediately(
+        *this, CandidateSet, FunctionTemplate, FoundDecl, ActingDC, From,
+        ToType, AllowObjCConversionOnExplicit, AllowExplicit,
+        AllowResultConversion);
+
+    CandidateSet.DisableResolutionByPerfectCandidate();
+    return;
+  }
+
+  CandidateSet.AddDeferredConversionTemplateCandidate(
+      FunctionTemplate, FoundDecl, ActingDC, From, ToType,
+      AllowObjCConversionOnExplicit, AllowExplicit, AllowResultConversion);
 }
 
 void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
@@ -8463,11 +8559,17 @@ void Sema::AddNonMemberOperatorCandidates(
     if (FunTmpl) {
       AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
                                    FunctionArgs, CandidateSet);
-      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD))
-        AddTemplateOverloadCandidate(
-            FunTmpl, F.getPair(), ExplicitTemplateArgs,
-            {FunctionArgs[1], FunctionArgs[0]}, CandidateSet, false, false,
-            true, ADLCallKind::NotADL, OverloadCandidateParamOrder::Reversed);
+      if (CandidateSet.getRewriteInfo().shouldAddReversed(*this, Args, FD)) {
+
+        // As template candidates are not deduced immediately,
+        // persist the array in the overload set.
+        ArrayRef<Expr *> Reversed = CandidateSet.getPersistentArgsArray(
+            FunctionArgs[1], FunctionArgs[0]);
+        AddTemplateOverloadCandidate(FunTmpl, F.getPair(), ExplicitTemplateArgs,
+                                     Reversed, CandidateSet, false, false, true,
+                                     ADLCallKind::NotADL,
+                                     OverloadCandidateParamOrder::Reversed);
+      }
     } else {
       if (ExplicitTemplateArgs)
         continue;
@@ -10199,6 +10301,8 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
   // FIXME: Pass in the explicit template arguments?
   ArgumentDependentLookup(Name, Loc, Args, Fns);
 
+  ArrayRef<Expr *> ReversedArgs;
+
   // Erase all of the candidates we already knew about.
   for (OverloadCandidateSet::iterator Cand = CandidateSet.begin(),
                                    CandEnd = CandidateSet.end();
@@ -10238,9 +10342,15 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
           /*AllowExplicit=*/true, ADLCallKind::UsesADL);
       if (CandidateSet.getRewriteInfo().shouldAddReversed(
               *this, Args, FTD->getTemplatedDecl())) {
+
+        // As template candidates are not deduced immediately,
+        // persist the array in the overload set.
+        if (ReversedArgs.empty())
+          ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
+
         AddTemplateOverloadCandidate(
-            FTD, FoundDecl, ExplicitTemplateArgs, {Args[1], Args[0]},
-            CandidateSet, /*SuppressUserConversions=*/false, PartialOverloading,
+            FTD, FoundDecl, ExplicitTemplateArgs, ReversedArgs, CandidateSet,
+            /*SuppressUserConversions=*/false, PartialOverloading,
             /*AllowExplicit=*/true, ADLCallKind::UsesADL,
             OverloadCandidateParamOrder::Reversed);
       }
@@ -10654,6 +10764,8 @@ bool clang::isBetterOverloadCandidate(
       Cand1.Function && Cand2.Function &&
       isa<CXXConversionDecl>(Cand1.Function) &&
       isa<CXXConversionDecl>(Cand2.Function)) {
+
+    assert(Cand1.HasFinalConversion && Cand2.HasFinalConversion);
     // First check whether we prefer one of the conversion functions over the
     // other. This only distinguishes the results in non-standard, extension
     // cases such as the conversion from a lambda closure type to a function
@@ -10913,23 +11025,147 @@ bool OverloadCandidate::NotValidBecauseConstraintExprHasError() const {
              ->Satisfaction.ContainsErrors;
 }
 
-/// Computes the best viable function (C++ 13.3.3)
-/// within an overload candidate set.
-///
-/// \param Loc The location of the function name (or operator symbol) for
-/// which overload resolution occurs.
-///
-/// \param Best If overload resolution was successful or found a deleted
-/// function, \p Best points to the candidate function found.
-///
-/// \returns The result of overload resolution.
+void OverloadCandidateSet::AddDeferredTemplateCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    ArrayRef<Expr *> Args, bool SuppressUserConversions,
+    bool PartialOverloading, bool AllowExplicit,
+    CallExpr::ADLCallKind IsADLCandidate, OverloadCandidateParamOrder PO,
+    bool AggregateCandidateDeduction) {
+
+  auto *C =
+      allocateDeferredCandidate<DeferredFunctionTemplateOverloadCandidate>();
+
+  C = new (C) DeferredFunctionTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Function,
+       /*AllowObjCConversionOnExplicit=*/false,
+       /*AllowResultConversion=*/false, AllowExplicit, SuppressUserConversions,
+       PartialOverloading, AggregateCandidateDeduction},
+      FunctionTemplate,
+      FoundDecl,
+      Args,
+      IsADLCandidate,
+      PO};
+
+  HasDeferredTemplateConstructors |=
+      isa<CXXConstructorDecl>(FunctionTemplate->getTemplatedDecl());
+}
+
+void OverloadCandidateSet::AddDeferredMethodTemplateCandidate(
+    FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext, QualType ObjectType,
+    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
+    bool SuppressUserConversions, bool PartialOverloading,
+    OverloadCandidateParamOrder PO) {
+
+  assert(!isa<CXXConstructorDecl>(MethodTmpl->getTemplatedDecl()));
+
+  auto *C =
+      allocateDeferredCandidate<DeferredMethodTemplateOverloadCandidate>();
+
+  C = new (C) DeferredMethodTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Method,
+       /*AllowObjCConversionOnExplicit=*/false,
+       /*AllowResultConversion=*/false,
+       /*AllowExplicit=*/false, SuppressUserConversions, PartialOverloading,
+       /*AggregateCandidateDeduction=*/false},
+      MethodTmpl,
+      FoundDecl,
+      Args,
+      ActingContext,
+      ObjectClassification,
+      ObjectType,
+      PO};
+}
+
+void OverloadCandidateSet::AddDeferredConversionTemplateCandidate(
+    FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl,
+    CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
+    bool AllowObjCConversionOnExplicit, bool AllowExplicit,
+    bool AllowResultConversion) {
+
+  auto *C =
+      allocateDeferredCandidate<DeferredConversionTemplateOverloadCandidate>();
+
+  C = new (C) DeferredConversionTemplateOverloadCandidate{
+      {nullptr, DeferredFunctionTemplateOverloadCandidate::Conversion,
+       AllowObjCConversionOnExplicit, AllowResultConversion,
+       /*AllowExplicit=*/false,
+       /*SuppressUserConversions=*/false,
+       /*PartialOverloading*/ false,
+       /*AggregateCandidateDeduction=*/false},
+      FunctionTemplate,
+      FoundDecl,
+      ActingContext,
+      From,
+      ToType};
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredMethodTemplateOverloadCandidate &C) {
+
+  AddMethodTemplateCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext,
+      /*ExplicitTemplateArgs=*/nullptr, C.ObjectType, C.ObjectClassification,
+      C.Args, C.SuppressUserConversions, C.PartialOverloading, C.PO);
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredFunctionTemplateOverloadCandidate &C) {
+  AddTemplateOverloadCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl,
+      /*ExplicitTemplateArgs=*/nullptr, C.Args, C.SuppressUserConversions,
+      C.PartialOverloading, C.AllowExplicit, C.IsADLCandidate, C.PO,
+      C.AggregateCandidateDeduction);
+}
+
+static void
+AddTemplateOverloadCandidate(Sema &S, OverloadCandidateSet &CandidateSet,
+                             DeferredConversionTemplateOverloadCandidate &C) {
+  return AddTemplateConversionCandidateImmediately(
+      S, CandidateSet, C.FunctionTemplate, C.FoundDecl, C.ActingContext, C.From,
+      C.ToType, C.AllowObjCConversionOnExplicit, C.AllowExplicit,
+      C.AllowResultConversion);
+}
+
+void OverloadCandidateSet::InjectNonDeducedTemplateCandidates(Sema &S) {
+  Candidates.reserve(Candidates.size() + DeferredCandidatesCount);
+  DeferredTemplateOverloadCandidate *Cand = FirstDeferredCandidate;
+  while (Cand) {
+    switch (Cand->Kind) {
+    case DeferredTemplateOverloadCandidate::Function:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredFunctionTemplateOverloadCandidate *>(Cand));
+      break;
+    case DeferredTemplateOverloadCandidate::Method:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredMethodTemplateOverloadCandidate *>(Cand));
+      break;
+    case DeferredTemplateOverloadCandidate::Conversion:
+      AddTemplateOverloadCandidate(
+          S, *this,
+          *static_cast<DeferredConversionTemplateOverloadCandidate *>(Cand));
+      break;
+    }
+    Cand = Cand->Next;
+  }
+  FirstDeferredCandidate = nullptr;
+  DeferredCandidatesCount = 0;
+}
+
 OverloadingResult
-OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
-                                         iterator &Best) {
-  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
-  std::transform(begin(), end(), std::back_inserter(Candidates),
-                 [](OverloadCandidate &Cand) { return &Cand; });
+OverloadCandidateSet::ResultForBestCandidate(const iterator &Best) {
+  Best->Best = true;
+  if (Best->Function && Best->Function->isDeleted())
+    return OR_Deleted;
+  return OR_Success;
+}
 
+void OverloadCandidateSet::CudaExcludeWrongSideCandidates(
+    Sema &S, SmallVectorImpl<OverloadCandidate *> &Candidates) {
   // [CUDA] HD->H or HD->D calls are technically not allowed by CUDA but
   // are accepted by both clang and NVCC. However, during a particular
   // compilation mode only one call variant is viable. We need to
@@ -10941,27 +11177,112 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   // -fgpu-exclude-wrong-side-overloads is off. When
   // -fgpu-exclude-wrong-side-overloads is on, all candidates are compared
   // uniformly in isBetterOverloadCandidate.
-  if (S.getLangOpts().CUDA && !S.getLangOpts().GPUExcludeWrongSideOverloads) {
-    const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
-    bool ContainsSameSideCandidate =
-        llvm::any_of(Candidates, [&](OverloadCandidate *Cand) {
-          // Check viable function only.
-          return Cand->Viable && Cand->Function &&
-                 S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-                     SemaCUDA::CFP_SameSide;
-        });
-    if (ContainsSameSideCandidate) {
-      auto IsWrongSideCandidate = [&](OverloadCandidate *Cand) {
-        // Check viable function only to avoid unnecessary data copying/moving.
+  if (!S.getLangOpts().CUDA || S.getLangOpts().GPUExcludeWrongSideOverloads)
+    return;
+  const FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
+
+  bool ContainsSameSideCandidate =
+      llvm::any_of(Candidates, [&](const OverloadCandidate *Cand) {
+        // Check viable function only.
         return Cand->Viable && Cand->Function &&
                S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
-                   SemaCUDA::CFP_WrongSide;
-      };
-      llvm::erase_if(Candidates, IsWrongSideCandidate);
+                   SemaCUDA::CFP_SameSide;
+      });
+
+  if (!ContainsSameSideCandidate)
+    return;
+
+  auto IsWrongSideCandidate = [&](const OverloadCandidate *Cand) {
+    // Check viable function only to avoid unnecessary data copying/moving.
+    return Cand->Viable && Cand->Function &&
+           S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
+               SemaCUDA::CFP_WrongSide;
+  };
+  llvm::erase_if(Candidates, IsWrongSideCandidate);
+}
+
+/// Computes the best viable function (C++ 13.3.3)
+/// within an overload candidate set.
+///
+/// \param Loc The location of the function name (or operator symbol) for
+/// which overload resolution occurs.
+///
+/// \param Best If overload resolution was successful or found a deleted
+/// function, \p Best points to the candidate function found.
+///
+/// \returns The result of overload resolution.
+OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
+                                                           SourceLocation Loc,
+                                                           iterator &Best) {
+
+  assert(shouldDeferTemplateArgumentDeduction(S.getLangOpts()) ||
+         DeferredCandidatesCount == 0 &&
+             "Unexpected deferred template candidates");
+
+  bool TwoPhaseResolution =
+      DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
+
+  if (TwoPhaseResolution) {
+
+    PerfectViableFunction(S, Loc, Best);
+    if (Best != end())
+      return ResultForBestCandidate(Best);
+  }
+
+  InjectNonDeducedTemplateCandidates(S);
+  return BestViableFunctionImpl(S, Loc, Best);
+}
+
+void OverloadCandidateSet::PerfectViableFunction(
+    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
+
+  Best = end();
+  for (auto It = begin(); It != end(); ++It) {
+
+    if (!It->isPerfectMatch(S.getASTContext()))
+      continue;
+
+    // We found a suitable conversion function
+    // but if there is a template constructor in the target class
+    // we might prefer that instead.
+    if (HasDeferredTemplateConstructors &&
+        isa_and_nonnull<CXXConversionDecl>(It->Function)) {
+      Best = end();
+      break;
     }
+
+    if (Best == end()) {
+      Best = It;
+      continue;
+    }
+    if (Best->Function && It->Function) {
+      FunctionDecl *D =
+          S.getMoreConstrainedFunction(Best->Function, It->Function);
+      if (D == nullptr) {
+        Best = end();
+        break;
+      }
+      if (D == It->Function)
+        Best = It;
+      continue;
+    }
+    // ambiguous
+    Best = end();
+    break;
   }
+}
+
+OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
+    Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
+
+  llvm::SmallVector<OverloadCandidate *, 16> Candidates;
+  Candidates.reserve(this->Candidates.size());
+  std::transform(begin(), end(), std::back_inserter(Candidates),
+                 [](OverloadCandidate &Cand) { return &Cand; });
+
+  if (S.getLangOpts().CUDA)
+    CudaExcludeWrongSideCandidates(S, Candidates);
 
-  // Find the best viable function.
   Best = end();
   for (auto *Cand : Candidates) {
     Cand->Best = false;
@@ -10983,9 +11304,8 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   if (Best == end())
     return OR_No_Viable_Function;
 
+  llvm::SmallVector<OverloadCandidate *, 4> PendingBest;
   llvm::SmallVector<const NamedDecl *, 4> EquivalentCands;
-
-  llvm::SmallVector<OverloadCandidate*, 4> PendingBest;
   PendingBest.push_back(&*Best);
   Best->Best = true;
 
@@ -11008,25 +11328,15 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
     }
   }
 
-  // If we found more than one best candidate, this is ambiguous.
   if (Best == end())
     return OR_Ambiguous;
 
-  // Best is the best viable function.
-  if (Best->Function && Best->Function->isDeleted())
-    return OR_Deleted;
-
-  if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
-      Kind == CSK_AddressOfOverloadSet && M &&
-      M->isImplicitObjectMemberFunction()) {
-    return OR_No_Viable_Function;
-  }
+  OverloadingResult R = ResultForBestCandidate(Best);
 
   if (!EquivalentCands.empty())
     S.diagnoseEquivalentInternalLinkageDeclarations(Loc, Best->Function,
                                                     EquivalentCands);
-
-  return OR_Success;
+  return R;
 }
 
 namespace {
@@ -12733,6 +13043,9 @@ SmallVector<OverloadCandidate *, 32> OverloadCandidateSet::CompleteCandidates(
     Sema &S, OverloadCandidateDisplayKind OCD, ArrayRef<Expr *> Args,
     SourceLocation OpLoc,
     llvm::function_ref<bool(OverloadCandidate &)> Filter) {
+
+  InjectNonDeducedTemplateCandidates(S);
+
   // Sort the candidates by viability and position.  Sorting directly would
   // be prohibitive, so we make a set of pointers and sort those.
   SmallVector<OverloadCandidate*, 32> Cands;
@@ -14354,10 +14667,12 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
                                          Expr *ExecConfig,
                                          bool AllowTypoCorrection,
                                          bool CalleesAddressIsTaken) {
-  OverloadCandidateSet CandidateSet(
-      Fn->getExprLoc(), CalleesAddressIsTaken
-                            ? OverloadCandidateSet::CSK_AddressOfOverloadSet
-                            : OverloadCandidateSet::CSK_Normal);
+
+  OverloadCandidateSet::CandidateSetKind CSK =
+      CalleesAddressIsTaken ? OverloadCandidateSet::CSK_AddressOfOverloadSet
+                            : OverloadCandidateSet::CSK_Normal;
+
+  OverloadCandidateSet CandidateSet(Fn->getExprLoc(), CSK);
   ExprResult result;
 
   if (buildOverloadedCallSet(S, Fn, ULE, Args, LParenLoc, &CandidateSet,
@@ -14373,6 +14688,17 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
   OverloadingResult OverloadResult =
       CandidateSet.BestViableFunction(*this, Fn->getBeginLoc(), Best);
 
+  // [C++23][over.call.func]
+  // if overload resolution selects a non-static member function,
+  // the call is ill-formed;
+  if (CSK == OverloadCandidateSet::CSK_AddressOfOverloadSet &&
+      Best != CandidateSet.end()) {
+    if (auto *M = dyn_cast_or_null<CXXMethodDecl>(Best->Function);
+        M && M->isImplicitObjectMemberFunction()) {
+      OverloadResult = OR_No_Viable_Function;
+    }
+  }
+
   // Model the case with a call to a templated function whose definition
   // encloses the call and whose return type contains a placeholder type as if
   // the UnresolvedLookupExpr was type-dependent.
@@ -14708,18 +15034,24 @@ void Sema::LookupOverloadedBinOp(OverloadCandidateSet &CandidateSet,
   // rewritten candidates using these functions if necessary.
   AddNonMemberOperatorCandidates(Fns, Args, CandidateSet);
 
+  // As template candidates are not deduced immediately,
+  // persist the array in the overload set.
+  ArrayRef<Expr *> ReversedArgs;
+  if (CandidateSet.getRewriteInfo().allowsReversed(Op) ||
+      CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
+    ReversedArgs = CandidateSet.getPersistentArgsArray(Args[1], Args[0]);
+
   // Add operator candidates that are member functions.
   AddMemberOperatorCandidates(Op, OpLoc, Args, CandidateSet);
   if (CandidateSet.getRewriteInfo().allowsReversed(Op))
-    AddMemberOperatorCandidates(Op, OpLoc, {Args[1], Args[0]}, CandidateSet,
+    AddMemberOperatorCandidates(Op, OpLoc, ReversedArgs, CandidateSet,
                                 OverloadCandidateParamOrder::Reversed);
 
   // In C++20, also add any rewritten member candidates.
   if (ExtraOp) {
     AddMemberOperatorCandidates(ExtraOp, OpLoc, Args, CandidateSet);
     if (CandidateSet.getRewriteInfo().allowsReversed(ExtraOp))
-      AddMemberOperatorCandidates(ExtraOp, OpLoc, {Args[1], Args[0]},
-                                  CandidateSet,
+      AddMemberOperatorCandidates(ExtraOp, OpLoc, ReversedArgs, CandidateSet,
                                   OverloadCandidateParamOrder::Reversed);
   }
 
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 772962ac653f7..0ecdbb3ffb89f 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -6142,9 +6142,9 @@ FunctionDecl *Sema::getMoreConstrainedFunction(FunctionDecl *FD1,
   assert(!FD1->getDescribedTemplate() && !FD2->getDescribedTemplate() &&
          "not for function templates");
   assert(!FD1->isFunctionTemplateSpecialization() ||
-         isa<CXXConversionDecl>(FD1));
+         (isa<CXXConversionDecl, CXXConstructorDecl>(FD1)));
   assert(!FD2->isFunctionTemplateSpecialization() ||
-         isa<CXXConversionDecl>(FD2));
+         (isa<CXXConversionDecl, CXXConstructorDecl>(FD2)));
 
   FunctionDecl *F1 = FD1;
   if (FunctionDecl *P = FD1->getTemplateInstantiationPattern(false))
diff --git a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
index ba8e2dc372e98..083e743818121 100644
--- a/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
+++ b/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
@@ -14,7 +14,7 @@ template<typename T> struct S {
 // expected-note@#FINST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{})
 void f(T);
-void f(int);
+void f(long);
 
 // Ensure this applies to operator && as well.
 // expected-error@+3{{atomic constraint must be of type 'bool' (found 'S<int>')}}
@@ -22,7 +22,7 @@ void f(int);
 // expected-note@#F2INST{{in instantiation of function template specialization}}
 template<typename T> requires (S<T>{} && true)
 void f2(T);
-void f2(int);
+void f2(long);
 
 template<typename T> requires requires {
   requires S<T>{};
@@ -36,12 +36,12 @@ template<typename T> requires requires {
   //
 }
 void f3(T);
-void f3(int);
+void f3(long);
 
 // Doesn't diagnose, since this is no longer a compound requirement.
 template<typename T> requires (bool(1 && 2))
 void f4(T);
-void f4(int);
+void f4(long);
 
 void g() {
   f(0); // #FINST
diff --git a/clang/test/SemaCUDA/function-overload.cu b/clang/test/SemaCUDA/function-overload.cu
index 4710c81763adf..3d05839af7528 100644
--- a/clang/test/SemaCUDA/function-overload.cu
+++ b/clang/test/SemaCUDA/function-overload.cu
@@ -1,6 +1,3 @@
-// REQUIRES: x86-registered-target
-// REQUIRES: nvptx-registered-target
-
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu -fsyntax-only \
 // RUN:   -verify=host,hostdefer,devdefer,expected %s
 // RUN: %clang_cc1 -std=c++14 -triple nvptx64-nvidia-cuda -fsyntax-only \
diff --git a/clang/test/SemaCXX/implicit-member-functions.cpp b/clang/test/SemaCXX/implicit-member-functions.cpp
index 1554b1af5d59a..8350eac5b88a0 100644
--- a/clang/test/SemaCXX/implicit-member-functions.cpp
+++ b/clang/test/SemaCXX/implicit-member-functions.cpp
@@ -54,31 +54,24 @@ namespace PR7594 {
 namespace Recursion {
   template<typename T> struct InvokeCopyConstructor {
     static const T &get();
-    typedef decltype(T(get())) type; // expected-error {{no matching conver}}
+    typedef decltype(T(get())) type;
   };
   struct B;
   struct A {
-    // expected-note@-1 {{while substituting deduced template arguments}}
     typedef B type;
     template<typename T,
              typename = typename InvokeCopyConstructor<typename T::type>::type>
-    // expected-note@-1 {{in instantiation of template class}}
     A(const T &);
-    // expected-note@-1 {{in instantiation of default argument}}
   };
-  struct B { // expected-note {{while declaring the implicit copy constructor for 'B'}}
-    // expected-note@-1 {{candidate constructor (the implicit move }}
-    B(); // expected-note {{candidate constructor not viable}}
+  struct B {
+    B();
     A a;
   };
   // Triggering the declaration of B's copy constructor causes overload
-  // resolution to occur for A's copying constructor, which instantiates
-  // InvokeCopyConstructor<B>, which triggers the declaration of B's copy
-  // constructor. Notionally, this happens when we get to the end of the
-  // definition of 'struct B', so there is no declared copy constructor yet.
-  //
-  // This behavior is g++-compatible, but isn't exactly right; the class is
-  // supposed to be incomplete when we implicitly declare its special members.
+  // resolution to occur for A's copying constructor, which picks
+  // the implicit copy constructor of A.
+  // Because that copy constructor is always a perfect match the template
+  // candidate is not instantiated.
   B b = B();
 
 
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
new file mode 100644
index 0000000000000..d68a942f64050
--- /dev/null
+++ b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -0,0 +1,206 @@
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++11 %s
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++20 %s
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -fsyntax-only -verify -std=c++2c %s
+
+template <typename T>
+struct Invalid { static_assert(false, "instantiated Invalid"); }; // #err-invalid
+
+template <typename T>
+int f(T a, Invalid<T> = {}); // #note-f
+
+// sanity check
+int e1 = f(0);
+//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
+//expected-note@-2 {{in instantiation of default function argument expression for 'f<int>' required here}}
+//expected-note@#note-f {{in instantiation of template class 'Invalid<int>' requested here}}
+//expected-note@#note-f {{passing argument to parameter here}}
+
+int f(int);
+int ok1 = f(0);
+int e4 = f((const int&)(ok1));
+
+int f(int, int = 0);
+int ok2 = f(0, 0);
+
+int e2  = f(0L);
+//expected-error@#err-invalid {{static assertion failed: instantiated Invalid}}
+//expected-note@-2 {{in instantiation of default function argument expression for 'f<long>' required here}}
+//expected-note@#note-f {{in instantiation of template class 'Invalid<long>' requested here}}
+//expected-note@#note-f {{passing argument to parameter here}}
+
+int f(long);
+int ok3 = f(0L);
+
+template <typename T>
+struct Invalid2 { static_assert(false, "instantiated Invalid2"); }; // #err-qualifiers
+
+template <typename T>
+int ref(T a, Invalid2<T> = {}); // expected-note 2{{here}}
+int ref(int&);
+int ref1 = ref(ok3);
+int ref2 = ref((const int&)ok3); // expected-note {{here}}
+//expected-error@#err-qualifiers {{static assertion failed: instantiated Invalid2}}
+
+
+template <typename T>
+int f_alias(T a, Invalid<T> = {});
+using Alias = int;
+int f_alias(Alias);
+int ok4 = f_alias(0);
+
+#if __cplusplus >= 202002
+
+struct Copyable {
+  template <typename T>
+  requires __is_constructible(Copyable, T)
+  explicit Copyable(T op) noexcept; // #1
+  Copyable(const Copyable&) noexcept = default; // #2
+};
+static_assert(__is_constructible(Copyable, const Copyable&));
+
+struct ImplicitlyCopyable {
+  template <typename T>
+  requires __is_constructible(ImplicitlyCopyable, T)
+  explicit ImplicitlyCopyable(T op) = delete; // #1
+};
+static_assert(__is_constructible(ImplicitlyCopyable, const ImplicitlyCopyable&));
+
+
+struct Movable {
+  template <typename T>
+  requires __is_constructible(Movable, T) // #err-self-constraint-1
+  explicit Movable(T op) noexcept; // #1
+  Movable(Movable&&) noexcept = default; // #2
+};
+static_assert(__is_constructible(Movable, Movable&&));
+static_assert(__is_constructible(Movable, const Movable&));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(Movable, const Movable &)'}}
+
+static_assert(__is_constructible(Movable, int));
+// expected-error@-1{{static assertion failed due to requirement '__is_constructible(Movable, int)'}} \
+// expected-note@-1 2{{}}
+// expected-error@#err-self-constraint-1{{satisfaction of constraint '__is_constructible(Movable, T)' depends on itself}}
+// expected-note@#err-self-constraint-1 4{{}}
+
+template <typename T>
+struct Members {
+    constexpr auto f(auto) {
+        static_assert(false, "");
+    }
+    constexpr auto f(int) { return 1; }
+    constexpr auto f(int) requires true { return 2; }
+
+    constexpr auto g(auto) {
+        static_assert(false, "instantiated member"); //#err-qualified-member
+        return 0;
+    }
+    constexpr auto g(int) & { return 1; }
+
+    static constexpr auto s(auto) {
+        static_assert(false, "");
+    }
+    static constexpr auto s(int) {
+        return 1;
+    }
+    static constexpr auto s(int) requires true {
+        return 2;
+    }
+};
+
+static_assert(Members<int[1]>{}.f(0) == 2);
+static_assert(Members<int[2]>{}.g(0) == 0);
+// expected-error@#err-qualified-member {{static assertion failed: instantiated member}} \
+// expected-note@-1{{in instantiation of function template specialization 'Members<int[2]>::g<int>' }}
+Members<int[3]> m1;
+static_assert(m1.g(0) == 1);
+static_assert(Members<int[3]>{}.s(0) == 2);
+
+
+namespace ConstructorInit{
+struct S {
+  template <typename T>
+  S(T&&) {}
+};
+struct Test {
+  operator S() = delete;
+};
+
+static_assert(__is_constructible(S, Test));
+}
+
+namespace RefBinding {
+
+template <typename> struct remove_reference;
+template <typename _Tp> struct remove_reference<_Tp &> {
+  using type = _Tp;
+};
+template <typename _Tp> remove_reference<_Tp>::type move(_Tp &&);
+template <typename _Head> struct _Head_base {
+  _Head_base(_Head &__h) : _M_head_impl(__h) {}
+  template <typename _UHead> _Head_base(_UHead &&);
+  _Head _M_head_impl;
+};
+
+template <typename _Elements> void forward_as_tuple(_Elements &&) {
+  _Head_base<_Elements &&>(_Elements{});
+}
+struct StringRef {
+  void operator[](const StringRef __k) { forward_as_tuple((move)(__k)); }
+};
+
+}
+
+template <class> struct tuple {};
+struct BonkersBananas {
+  template <class T> operator T();
+  template <class = void> explicit operator tuple<int>() = delete;
+};
+static_assert(!__is_constructible(tuple<int>, BonkersBananas));
+
+namespace GH62096 {
+template <typename T>
+struct Oops {
+  static_assert(sizeof(T) == 0); // #GH62096-err
+  static constexpr bool value = true;
+};
+
+template <class OPERATOR>
+concept Operator = Oops<OPERATOR>::value; // #GH62096-note1
+
+template <Operator OP> void f(OP op); // // #GH62096-note2
+void f(int);
+
+void g(int n) { f(n); } // OK
+void h(short n) { f(n); }
+// expected-error@#GH62096-err {{static assertion failed due to requirement 'sizeof(short) == 0'}} \
+// expected-note@-1{{in instantiation of function template specialization}} \
+// expected-note@-1{{while checking constraint satisfaction for template}}
+// expected-note@#GH62096-note1{{in instantiation}}
+// expected-note@#GH62096-note1{{while substituting template arguments into constraint expression here}}
+// expected-note@#GH62096-note2{{while substituting template arguments into constraint expression here}}
+// expected-note@#GH62096-note2{{while checking the satisfaction of concept}}
+// expected-note@#GH62096-err {{expression evaluates}}
+}
+
+#endif
+
+template<typename ...Ts>
+struct t1 {
+};
+struct t6 {
+  template<typename T = int>
+  operator t1<float>() {
+    return {};
+  }
+};
+
+void testT6() {
+  t6 v6;
+  v6.operator t1<float>();
+}
+
+
+using a = void(int &);
+template <typename c> void d(c &);
+void f(a);
+template <class> void f(bool j) { f(&d<int>); }
diff --git a/clang/test/SemaTemplate/instantiate-function-params.cpp b/clang/test/SemaTemplate/instantiate-function-params.cpp
index 7dd5595de58a3..eb2a7c5d4e8d6 100644
--- a/clang/test/SemaTemplate/instantiate-function-params.cpp
+++ b/clang/test/SemaTemplate/instantiate-function-params.cpp
@@ -6,13 +6,12 @@ template<typename T1> struct if_ {
   typedef if_c< static_cast<bool>(T1::value)> almost_type_; // expected-note 7{{in instantiation}}
 };
 template <class Model, void (Model::*)()> struct wrap_constraints { };
-template <class Model> 
+template <class Model>
 inline char has_constraints_(Model* , // expected-note 3{{candidate template ignored}}
-                               wrap_constraints<Model,&Model::constraints>* = 0); // expected-note 4{{in instantiation}}
-
+                               wrap_constraints<Model,&Model::constraints>* = 0);
 template <class Model> struct not_satisfied {
   static const bool value = sizeof( has_constraints_((Model*)0)  == 1); // expected-error 3{{no matching function}} \
-  // expected-note 4{{while substituting deduced template arguments into function template 'has_constraints_' [with }}
+  // expected-note 4{{in instantiation}}
 };
 template <class ModelFn> struct requirement_;
 template <void(*)()> struct instantiate {
diff --git a/clang/test/Templight/templight-empty-entries-fix.cpp b/clang/test/Templight/templight-empty-entries-fix.cpp
index d13b748068efe..7f34b10134929 100644
--- a/clang/test/Templight/templight-empty-entries-fix.cpp
+++ b/clang/test/Templight/templight-empty-entries-fix.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -templight-dump -Wno-unused-value %s 2>&1 | FileCheck %s
 
-void a() {
+void a(long) {
   [] {};
 }
 
@@ -17,14 +17,14 @@ void a() {
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 // CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:4:3'$}}
 
-template <int = 0> void a() { a(); }
+template <int = 0> void a(long) { a(0); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -42,29 +42,29 @@ template <int = 0> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<0>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:31'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:20:35'$}}
 
 template <int> struct b { typedef int c; };
-template <bool d = true, class = typename b<d>::c> void a() { a(); }
+template <bool d = true, class = typename b<d>::c> void a(long) { a(0); }
 
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -130,25 +130,25 @@ template <bool d = true, class = typename b<d>::c> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'a<true, int>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+a$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template non-type parameter 0 of a$}}
 // CHECK: {{^kind:[ ]+DefaultTemplateArgumentInstantiation$}}
@@ -166,34 +166,10 @@ template <bool d = true, class = typename b<d>::c> void a() { a(); }
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:67'$}}
 
 template <bool = true> void d(int = 0) { d(); }
 
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:60:57'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+a$}}
-// CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
-// CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:20:25'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:60:63'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
@@ -249,41 +225,41 @@ void e() {
 }
 
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:248:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:224:5'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:247:3\)'$}}
+// CHECK: {{^name:[ ]+'\(unnamed struct at .*templight-empty-entries-fix.cpp:223:3\)'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:247:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:223:3'$}}
 
 
 template <template<typename> class>
@@ -299,71 +275,71 @@ void foo() {
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
 // CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:265:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:5'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+DeducedTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'d<C>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:290:6'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:266:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:171:29'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:271:3'$}}

From 49dfd72de0131e4c3f1270ef60ea54af43a3c96b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 17 Apr 2025 21:19:01 -0700
Subject: [PATCH 369/710] [test] Remove CHECK lines for MCAsmStreamer's fixup
 output

Similar to 5e430afb84c3cbf285d8fe15c0ae23e0920e946b
---
 .../test/MC/LoongArch/Relocations/relax-tls-le.s | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/llvm/test/MC/LoongArch/Relocations/relax-tls-le.s b/llvm/test/MC/LoongArch/Relocations/relax-tls-le.s
index 899f12f85654d..055f44946cfa6 100644
--- a/llvm/test/MC/LoongArch/Relocations/relax-tls-le.s
+++ b/llvm/test/MC/LoongArch/Relocations/relax-tls-le.s
@@ -2,15 +2,11 @@
 # RUN:     | llvm-readobj -r - | FileCheck --check-prefix=LA32-RELAX-RELOC %s
 # RUN: llvm-mc --filetype=obj --triple=loongarch32 --mattr=-relax < %s \
 # RUN:     | llvm-readobj -r - | FileCheck --check-prefix=LA32-NORELAX-RELOC %s
-# RUN: llvm-mc --triple=loongarch32 --mattr=+relax < %s --show-encoding \
-# RUN:     | FileCheck --check-prefix=LA32-RELAX-FIXUP %s
 
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax --defsym=LA64=1 < %s \
 # RUN:     | llvm-readobj -r - | FileCheck --check-prefix=LA64-RELAX-RELOC %s
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=-relax --defsym=LA64=1 < %s \
 # RUN:     | llvm-readobj -r - | FileCheck --check-prefix=LA64-NORELAX-RELOC %s
-# RUN: llvm-mc --triple=loongarch64 --mattr=+relax --defsym=LA64=1 < %s --show-encoding \
-# RUN:     | FileCheck --check-prefix=LA64-RELAX-FIXUP %s
 
 .long foo
 
@@ -21,24 +17,18 @@ lu12i.w $a0, %le_hi20_r(foo)
 # LA32-NORELAX-RELOC-NOT: R_LARCH_RELAX - 0x0
 # LA32-RELAX-RELOC: R_LARCH_TLS_LE_HI20_R foo 0x0
 # LA32-RELAX-RELOC: R_LARCH_RELAX - 0x0
-# LA32-RELAX-FIXUP: fixup A - offset: 0, value: %le_hi20_r(foo), kind: FK_NONE
-# LA32-RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: FK_NONE
 
 add.w $a0, $a0, $tp, %le_add_r(foo)
 # LA32-NORELAX-RELOC: R_LARCH_TLS_LE_ADD_R foo 0x0
 # LA32-NORELAX-RELOC-NOT: R_LARCH_RELAX - 0x0
 # LA32-RELAX-RELOC: R_LARCH_TLS_LE_ADD_R foo 0x0
 # LA32-RELAX-RELOC: R_LARCH_RELAX - 0x0
-# LA32-RELAX-FIXUP: fixup A - offset: 0, value: %le_add_r(foo), kind: FK_NONE
-# LA32-RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: FK_NONE
 
 addi.w $a0, $a0, %le_lo12_r(foo)
 # LA32-NORELAX-RELOC: R_LARCH_TLS_LE_LO12_R foo 0x0
 # LA32-NORELAX-RELOC-NOT: R_LARCH_RELAX - 0x0
 # LA32-RELAX-RELOC: R_LARCH_TLS_LE_LO12_R foo 0x0
 # LA32-RELAX-RELOC: R_LARCH_RELAX - 0x0
-# LA32-RELAX-FIXUP: fixup A - offset: 0, value: %le_lo12_r(foo), kind: FK_NONE
-# LA32-RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: FK_NONE
 
 .else
 
@@ -47,24 +37,18 @@ lu12i.w $a0, %le_hi20_r(foo)
 # LA64-NORELAX-RELOC-NOT: R_LARCH_RELAX - 0x0
 # LA64-RELAX-RELOC: R_LARCH_TLS_LE_HI20_R foo 0x0
 # LA64-RELAX-RELOC: R_LARCH_RELAX - 0x0
-# LA64-RELAX-FIXUP: fixup A - offset: 0, value: %le_hi20_r(foo), kind: FK_NONE
-# LA64-RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: FK_NONE
 
 add.d $a0, $a0, $tp, %le_add_r(foo)
 # LA64-NORELAX-RELOC: R_LARCH_TLS_LE_ADD_R foo 0x0
 # LA64-NORELAX-RELOC-NOT: R_LARCH_RELAX - 0x0
 # LA64-RELAX-RELOC: R_LARCH_TLS_LE_ADD_R foo 0x0
 # LA64-RELAX-RELOC: R_LARCH_RELAX - 0x0
-# LA64-RELAX-FIXUP: fixup A - offset: 0, value: %le_add_r(foo), kind: FK_NONE
-# LA64-RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: FK_NONE
 
 addi.d $a0, $a0, %le_lo12_r(foo)
 # LA64-NORELAX-RELOC: R_LARCH_TLS_LE_LO12_R foo 0x0
 # LA64-NORELAX-RELOC-NOT: R_LARCH_RELAX - 0x0
 # LA64-RELAX-RELOC: R_LARCH_TLS_LE_LO12_R foo 0x0
 # LA64-RELAX-RELOC: R_LARCH_RELAX - 0x0
-# LA64-RELAX-FIXUP: fixup A - offset: 0, value: %le_lo12_r(foo), kind: FK_NONE
-# LA64-RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: FK_NONE
 
 .endif
 

From 7088910b9f74d49a7e0a107a2261be90c1c8bd1b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 17 Apr 2025 21:25:51 -0700
Subject: [PATCH 370/710] MCAsmStreamer: Print relocation type number if
 applicable

The fixup output is a debug aid and should not be used to test
target-specific relocation generation implementation. The llvm-mc
-filetype=obj output is what truly matters.
---
 llvm/lib/MC/MCAsmStreamer.cpp     | 10 +++++++---
 llvm/test/MC/AArch64/tls-relocs.s |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 9fb79d676f2c7..ff5f13c54b17b 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -2399,12 +2399,16 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
 
   for (unsigned i = 0, e = Fixups.size(); i != e; ++i) {
     MCFixup &F = Fixups[i];
-    const MCFixupKindInfo &Info =
-        getAssembler().getBackend().getFixupKindInfo(F.getKind());
     OS << "  fixup " << char('A' + i) << " - "
        << "offset: " << F.getOffset() << ", value: ";
     F.getValue()->print(OS, MAI);
-    OS << ", kind: " << Info.Name << "\n";
+    auto Kind = F.getKind();
+    if (FirstRelocationKind <= Kind)
+      OS << ", relocation type: " << (Kind - FirstRelocationKind);
+    else
+      OS << ", kind: "
+         << getAssembler().getBackend().getFixupKindInfo(Kind).Name;
+    OS << '\n';
   }
 }
 
diff --git a/llvm/test/MC/AArch64/tls-relocs.s b/llvm/test/MC/AArch64/tls-relocs.s
index 3cdad8e5ca504..619011e8e3cd6 100644
--- a/llvm/test/MC/AArch64/tls-relocs.s
+++ b/llvm/test/MC/AArch64/tls-relocs.s
@@ -389,7 +389,7 @@
 // CHECK: add    x5, x4, :tlsdesc_lo12:var // encoding: [0x85,0bAAAAAA00,0b00AAAAAA,0x91]
 // CHECK:                                  //   fixup A - offset: 0, value: :tlsdesc_lo12:var, kind: fixup_aarch64_add_imm12
 // CHECK: .tlsdesccall var                // encoding: []
-// CHECK-NEXT:                            //   fixup A - offset: 0, value: var, kind: FK_NONE
+// CHECK-NEXT:                            //   fixup A - offset: 0, value: var, relocation type: 569
 // CHECK: blr    x3                      // encoding: [0x60,0x00,0x3f,0xd6]
 
 // CHECK-ELF-NEXT:     0x104 R_AARCH64_TLSDESC_ADR_PAGE21 [[VARSYM]]

From d5f94c3915ace438359524b40cf84f3f54a0afae Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 17 Apr 2025 21:28:28 -0700
Subject: [PATCH 371/710] Revert "[RISCV] Strengthen register usage validation
 for XTheadMemPair loads (#136241)"

This reverts commit a354564a64c6ab3cafa9e5e2b31f7f14d4e27d45.

Broke tests
---
 llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp | 4 ++--
 llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s     | 3 +--
 llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s     | 3 +--
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index db9ec7ae1df02..32751782e08d4 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -3656,9 +3656,9 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     MCRegister Rd2 = Inst.getOperand(1).getReg();
     MCRegister Rs1 = Inst.getOperand(2).getReg();
     // The encoding with rd1 == rd2 == rs1 is reserved for XTHead load pair.
-    if (Rs1 == Rd1 || Rs1 == Rd2 || Rd1 == Rd2) {
+    if (Rs1 == Rd1 && Rs1 == Rd2) {
       SMLoc Loc = Operands[1]->getStartLoc();
-      return Error(Loc, "rs1, rd1, and rd2 cannot overlap");
+      return Error(Loc, "rs1, rd1, and rd2 cannot all be the same");
     }
   }
 
diff --git a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
index 8a7202ffd375e..a9223ae128a40 100644
--- a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
@@ -15,7 +15,6 @@ th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be consta
 th.swd t3, t4, (t5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.swd t3, t4, (t5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
 th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
-th.lwd x6, x7, (x7), 2, 3   # CHECK: [[@LINE]]:8: error: rs1, rd1, and rd2 cannot overlap
-th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot overlap
+th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot all be the same
 th.ldd t0, t1, (t2), 2, 4   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
 th.sdd t0, t1, (t2), 2, 4   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
diff --git a/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
index bbd0118630fbb..342db21bbca62 100644
--- a/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
@@ -15,5 +15,4 @@ th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be consta
 th.swd t3, t4, (t5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.swd t3, t4, (t5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
 th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
-th.ldd x6, x6, (x7), 2, 3   # CHECK: [[@LINE]]:8: error: rs1, rd1, and rd2 cannot overlap
-th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot overlap
+th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot all be the same

From 65d16a8101f98199b5d2505c1e9fdf2df519814b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 17 Apr 2025 21:36:14 -0700
Subject: [PATCH 372/710] [RISCV] Simplify fixup kinds that force relocations

For RELA targets, fixup kinds that force relocations (GOT, TLS, ALIGN,
RELAX, etc) can bypass `applyFixup` and be encoded as
`FirstRelocationKind+i`, as seen in LoongArch. This patch removes
redundant fixup kinds and adopts the `FirstRelocationKind+i` encoding.

The `llvm-mc -show-encoding` output no longer displays descriptive fixup
names, as this information is removed from
`RISCVAsmBackend::getFixupKindInfo`. While a backend hook could be added
to call `llvm::object::getELFRelocationTypeName`, it's unnecessary since
the relocation in `-filetype=obj` output is what truly matters.

Pull Request: https://github.com/llvm/llvm-project/pull/136088
---
 llvm/lib/MC/MCAssembler.cpp                   |  2 +-
 .../RISCV/MCTargetDesc/RISCVAsmBackend.cpp    | 61 ++++++-------------
 .../RISCV/MCTargetDesc/RISCVAsmBackend.h      |  6 +-
 .../MCTargetDesc/RISCVELFObjectWriter.cpp     | 38 ++----------
 .../RISCV/MCTargetDesc/RISCVFixupKinds.h      | 31 ----------
 .../RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp | 35 ++++++-----
 .../Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp | 23 ++++---
 7 files changed, 56 insertions(+), 140 deletions(-)

diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 4e925809d20b0..1b3e7dd4f4ceb 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -203,7 +203,7 @@ bool MCAssembler::evaluateFixup(const MCFixup &Fixup, const MCFragment *DF,
   if (IsResolved) {
     auto TargetVal = Target;
     TargetVal.Cst = Value;
-    if (Fixup.getKind() >= FirstLiteralRelocationKind ||
+    if (Fixup.getKind() >= FirstRelocationKind ||
         getBackend().shouldForceRelocation(*this, Fixup, TargetVal, STI))
       IsResolved = false;
   }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index b36b8bd3fb436..f9799a062c95e 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -34,6 +34,14 @@ static cl::opt<bool> ULEB128Reloc(
     "riscv-uleb128-reloc", cl::init(true), cl::Hidden,
     cl::desc("Emit R_RISCV_SET_ULEB128/E_RISCV_SUB_ULEB128 if appropriate"));
 
+RISCVAsmBackend::RISCVAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI,
+                                 bool Is64Bit, const MCTargetOptions &Options)
+    : MCAsmBackend(llvm::endianness::little,
+                   FirstRelocationKind + ELF::R_RISCV_RELAX),
+      STI(STI), OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {
+  RISCVFeatures::validate(STI.getTargetTriple(), STI.getFeatureBits());
+}
+
 std::optional<MCFixupKind> RISCVAsmBackend::getFixupKind(StringRef Name) const {
   if (STI.getTargetTriple().isOSBinFormatELF()) {
     unsigned Type;
@@ -71,27 +79,13 @@ RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
        MCFixupKindInfo::FKF_IsPCRel | MCFixupKindInfo::FKF_IsTarget},
       {"fixup_riscv_pcrel_lo12_s", 0, 32,
        MCFixupKindInfo::FKF_IsPCRel | MCFixupKindInfo::FKF_IsTarget},
-      {"fixup_riscv_got_hi20", 12, 20, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_tprel_hi20", 12, 20, 0},
-      {"fixup_riscv_tprel_lo12_i", 20, 12, 0},
-      {"fixup_riscv_tprel_lo12_s", 0, 32, 0},
-      {"fixup_riscv_tprel_add", 0, 0, 0},
-      {"fixup_riscv_tls_got_hi20", 12, 20, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_tls_gd_hi20", 12, 20, MCFixupKindInfo::FKF_IsPCRel},
       {"fixup_riscv_jal", 12, 20, MCFixupKindInfo::FKF_IsPCRel},
       {"fixup_riscv_branch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
       {"fixup_riscv_rvc_jump", 2, 11, MCFixupKindInfo::FKF_IsPCRel},
       {"fixup_riscv_rvc_branch", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
       {"fixup_riscv_call", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
       {"fixup_riscv_call_plt", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_relax", 0, 0, 0},
-      {"fixup_riscv_align", 0, 0, 0},
 
-      {"fixup_riscv_tlsdesc_hi20", 12, 20,
-       MCFixupKindInfo::FKF_IsPCRel | MCFixupKindInfo::FKF_IsTarget},
-      {"fixup_riscv_tlsdesc_load_lo12", 20, 12, 0},
-      {"fixup_riscv_tlsdesc_add_lo12", 20, 12, 0},
-      {"fixup_riscv_tlsdesc_call", 0, 0, 0},
       {"fixup_riscv_qc_e_branch", 0, 48, MCFixupKindInfo::FKF_IsPCRel},
       {"fixup_riscv_qc_e_32", 16, 32, 0},
       {"fixup_riscv_qc_abs20_u", 12, 20, 0},
@@ -100,9 +94,9 @@ RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   static_assert((std::size(Infos)) == RISCV::NumTargetFixupKinds,
                 "Not all fixup kinds added to Infos array");
 
-  // Fixup kinds from .reloc directive are like R_RISCV_NONE. They
-  // do not require any extra processing.
-  if (Kind >= FirstLiteralRelocationKind)
+  // Fixup kinds from raw relocation types and .reloc directive are like
+  // R_RISCV_NONE. They do not require any extra processing.
+  if (Kind >= FirstRelocationKind)
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
@@ -131,11 +125,6 @@ bool RISCVAsmBackend::shouldForceRelocation(const MCAssembler &Asm,
     if (Target.isAbsolute())
       return false;
     break;
-  case RISCV::fixup_riscv_got_hi20:
-  case RISCV::fixup_riscv_tls_got_hi20:
-  case RISCV::fixup_riscv_tls_gd_hi20:
-  case RISCV::fixup_riscv_tlsdesc_hi20:
-    return true;
   }
 
   return STI->hasFeature(RISCV::FeatureRelax) || ForceRelocs;
@@ -456,11 +445,6 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
   switch (Fixup.getTargetKind()) {
   default:
     llvm_unreachable("Unknown fixup kind!");
-  case RISCV::fixup_riscv_got_hi20:
-  case RISCV::fixup_riscv_tls_got_hi20:
-  case RISCV::fixup_riscv_tls_gd_hi20:
-  case RISCV::fixup_riscv_tlsdesc_hi20:
-    llvm_unreachable("Relocation should be unconditionally forced\n");
   case FK_Data_1:
   case FK_Data_2:
   case FK_Data_4:
@@ -469,8 +453,6 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     return Value;
   case RISCV::fixup_riscv_lo12_i:
   case RISCV::fixup_riscv_pcrel_lo12_i:
-  case RISCV::fixup_riscv_tprel_lo12_i:
-  case RISCV::fixup_riscv_tlsdesc_load_lo12:
     return Value & 0xfff;
   case RISCV::fixup_riscv_12_i:
     if (!isInt<12>(Value)) {
@@ -480,11 +462,9 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     return Value & 0xfff;
   case RISCV::fixup_riscv_lo12_s:
   case RISCV::fixup_riscv_pcrel_lo12_s:
-  case RISCV::fixup_riscv_tprel_lo12_s:
     return (((Value >> 5) & 0x7f) << 25) | ((Value & 0x1f) << 7);
   case RISCV::fixup_riscv_hi20:
   case RISCV::fixup_riscv_pcrel_hi20:
-  case RISCV::fixup_riscv_tprel_hi20:
     // Add 1 if bit 11 is 1, to compensate for low 12 bits being negative.
     return ((Value + 0x800) >> 12) & 0xfffff;
   case RISCV::fixup_riscv_jal: {
@@ -602,7 +582,6 @@ bool RISCVAsmBackend::evaluateTargetFixup(
   switch (Fixup.getTargetKind()) {
   default:
     llvm_unreachable("Unexpected fixup kind!");
-  case RISCV::fixup_riscv_tlsdesc_hi20:
   case RISCV::fixup_riscv_pcrel_hi20:
     AUIPCFixup = &Fixup;
     AUIPCDF = DF;
@@ -642,7 +621,7 @@ bool RISCVAsmBackend::evaluateTargetFixup(
   Value = Asm.getSymbolOffset(SA) + AUIPCTarget.getConstant();
   Value -= Asm.getFragmentOffset(*AUIPCDF) + AUIPCFixup->getOffset();
 
-  return !shouldForceRelocation(Asm, *AUIPCFixup, AUIPCTarget, STI);
+  return AUIPCFixup->getTargetKind() == RISCV::fixup_riscv_pcrel_hi20;
 }
 
 bool RISCVAsmBackend::handleAddSubRelocations(const MCAssembler &Asm,
@@ -680,12 +659,10 @@ bool RISCVAsmBackend::handleAddSubRelocations(const MCAssembler &Asm,
   }
   MCValue A = MCValue::get(Target.getAddSym(), nullptr, Target.getConstant());
   MCValue B = MCValue::get(Target.getSubSym());
-  auto FA = MCFixup::create(
-      Fixup.getOffset(), nullptr,
-      static_cast<MCFixupKind>(FirstLiteralRelocationKind + TA));
-  auto FB = MCFixup::create(
-      Fixup.getOffset(), nullptr,
-      static_cast<MCFixupKind>(FirstLiteralRelocationKind + TB));
+  auto FA =
+      MCFixup::create(Fixup.getOffset(), nullptr, FirstRelocationKind + TA);
+  auto FB =
+      MCFixup::create(Fixup.getOffset(), nullptr, FirstRelocationKind + TB);
   auto &Assembler = const_cast<MCAssembler &>(Asm);
   Asm.getWriter().recordRelocation(Assembler, &F, FA, A, FixedValueA);
   Asm.getWriter().recordRelocation(Assembler, &F, FB, B, FixedValueB);
@@ -699,7 +676,7 @@ void RISCVAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                  bool IsResolved,
                                  const MCSubtargetInfo *STI) const {
   MCFixupKind Kind = Fixup.getKind();
-  if (Kind >= FirstLiteralRelocationKind)
+  if (Kind >= FirstRelocationKind)
     return;
   MCContext &Ctx = Asm.getContext();
   MCFixupKindInfo Info = getFixupKindInfo(Kind);
@@ -767,8 +744,8 @@ bool RISCVAsmBackend::shouldInsertFixupForCodeAlign(MCAssembler &Asm,
   MCContext &Ctx = Asm.getContext();
   const MCExpr *Dummy = MCConstantExpr::create(0, Ctx);
   // Create fixup_riscv_align fixup.
-  MCFixup Fixup =
-      MCFixup::create(0, Dummy, MCFixupKind(RISCV::fixup_riscv_align), SMLoc());
+  MCFixup Fixup = MCFixup::create(
+      0, Dummy, FirstRelocationKind + ELF::R_RISCV_ALIGN, SMLoc());
 
   uint64_t FixedValue = 0;
   MCValue NopBytes = MCValue::get(Count);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
index 5d585b4efc116..23c8c2ce0b314 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
@@ -30,11 +30,7 @@ class RISCVAsmBackend : public MCAsmBackend {
 
 public:
   RISCVAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI, bool Is64Bit,
-                  const MCTargetOptions &Options)
-      : MCAsmBackend(llvm::endianness::little, RISCV::fixup_riscv_relax),
-        STI(STI), OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {
-    RISCVFeatures::validate(STI.getTargetTriple(), STI.getFeatureBits());
-  }
+                  const MCTargetOptions &Options);
   ~RISCVAsmBackend() override = default;
 
   void setForceRelocs() { ForceRelocs = true; }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
index a0b92b6bb9651..4b32130499f94 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
@@ -51,7 +51,6 @@ unsigned RISCVELFObjectWriter::getRelocType(MCContext &Ctx,
                                             const MCFixup &Fixup,
                                             bool IsPCRel) const {
   const MCExpr *Expr = Fixup.getValue();
-  // Determine the type of the relocation
   unsigned Kind = Fixup.getTargetKind();
   auto Spec = RISCVMCExpr::Specifier(Target.getSpecifier());
   switch (Spec) {
@@ -74,6 +73,11 @@ unsigned RISCVELFObjectWriter::getRelocType(MCContext &Ctx,
     break;
   }
 
+  // Extract the relocation type from the fixup kind, after applying STT_TLS as
+  // needed.
+  if (Kind >= FirstRelocationKind)
+    return Kind - FirstRelocationKind;
+
   if (IsPCRel) {
     switch (Kind) {
     default:
@@ -88,20 +92,6 @@ unsigned RISCVELFObjectWriter::getRelocType(MCContext &Ctx,
       return ELF::R_RISCV_PCREL_LO12_I;
     case RISCV::fixup_riscv_pcrel_lo12_s:
       return ELF::R_RISCV_PCREL_LO12_S;
-    case RISCV::fixup_riscv_got_hi20:
-      return ELF::R_RISCV_GOT_HI20;
-    case RISCV::fixup_riscv_tls_got_hi20:
-      return ELF::R_RISCV_TLS_GOT_HI20;
-    case RISCV::fixup_riscv_tls_gd_hi20:
-      return ELF::R_RISCV_TLS_GD_HI20;
-    case RISCV::fixup_riscv_tlsdesc_hi20:
-      return ELF::R_RISCV_TLSDESC_HI20;
-    case RISCV::fixup_riscv_tlsdesc_load_lo12:
-      return ELF::R_RISCV_TLSDESC_LOAD_LO12;
-    case RISCV::fixup_riscv_tlsdesc_add_lo12:
-      return ELF::R_RISCV_TLSDESC_ADD_LO12;
-    case RISCV::fixup_riscv_tlsdesc_call:
-      return ELF::R_RISCV_TLSDESC_CALL;
     case RISCV::fixup_riscv_jal:
       return ELF::R_RISCV_JAL;
     case RISCV::fixup_riscv_branch:
@@ -125,12 +115,6 @@ unsigned RISCVELFObjectWriter::getRelocType(MCContext &Ctx,
   default:
     Ctx.reportError(Fixup.getLoc(), "unsupported relocation type");
     return ELF::R_RISCV_NONE;
-  case RISCV::fixup_riscv_tlsdesc_load_lo12:
-    return ELF::R_RISCV_TLSDESC_LOAD_LO12;
-  case RISCV::fixup_riscv_tlsdesc_add_lo12:
-    return ELF::R_RISCV_TLSDESC_ADD_LO12;
-  case RISCV::fixup_riscv_tlsdesc_call:
-    return ELF::R_RISCV_TLSDESC_CALL;
 
   case FK_Data_1:
     Ctx.reportError(Fixup.getLoc(), "1-byte data relocations not supported");
@@ -160,18 +144,6 @@ unsigned RISCVELFObjectWriter::getRelocType(MCContext &Ctx,
     return ELF::R_RISCV_LO12_I;
   case RISCV::fixup_riscv_lo12_s:
     return ELF::R_RISCV_LO12_S;
-  case RISCV::fixup_riscv_tprel_hi20:
-    return ELF::R_RISCV_TPREL_HI20;
-  case RISCV::fixup_riscv_tprel_lo12_i:
-    return ELF::R_RISCV_TPREL_LO12_I;
-  case RISCV::fixup_riscv_tprel_lo12_s:
-    return ELF::R_RISCV_TPREL_LO12_S;
-  case RISCV::fixup_riscv_tprel_add:
-    return ELF::R_RISCV_TPREL_ADD;
-  case RISCV::fixup_riscv_relax:
-    return ELF::R_RISCV_RELAX;
-  case RISCV::fixup_riscv_align:
-    return ELF::R_RISCV_ALIGN;
   case RISCV::fixup_riscv_qc_e_32:
     return ELF::R_RISCV_QC_E_32;
   case RISCV::fixup_riscv_qc_abs20_u:
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
index 596c4eb5fffaa..80fbed8d10f99 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
@@ -32,25 +32,6 @@ enum Fixups {
   // 12-bit fixup corresponding to %pcrel_lo(foo) for the S-type store
   // instructions
   fixup_riscv_pcrel_lo12_s,
-  // 20-bit fixup corresponding to %got_pcrel_hi(foo) for instructions like
-  // auipc
-  fixup_riscv_got_hi20,
-  // 20-bit fixup corresponding to %tprel_hi(foo) for instructions like lui
-  fixup_riscv_tprel_hi20,
-  // 12-bit fixup corresponding to %tprel_lo(foo) for instructions like addi
-  fixup_riscv_tprel_lo12_i,
-  // 12-bit fixup corresponding to %tprel_lo(foo) for the S-type store
-  // instructions
-  fixup_riscv_tprel_lo12_s,
-  // Fixup corresponding to %tprel_add(foo) for PseudoAddTPRel, used as a linker
-  // hint
-  fixup_riscv_tprel_add,
-  // 20-bit fixup corresponding to %tls_ie_pcrel_hi(foo) for instructions like
-  // auipc
-  fixup_riscv_tls_got_hi20,
-  // 20-bit fixup corresponding to %tls_gd_pcrel_hi(foo) for instructions like
-  // auipc
-  fixup_riscv_tls_gd_hi20,
   // 20-bit fixup for symbol references in the jal instruction
   fixup_riscv_jal,
   // 12-bit fixup for symbol references in the branch instructions
@@ -65,18 +46,6 @@ enum Fixups {
   // Fixup representing a function call attached to the auipc instruction in a
   // pair composed of adjacent auipc+jalr instructions.
   fixup_riscv_call_plt,
-  // Used to generate an R_RISCV_RELAX relocation, which indicates the linker
-  // may relax the instruction pair.
-  fixup_riscv_relax,
-  // Used to generate an R_RISCV_ALIGN relocation, which indicates the linker
-  // should fixup the alignment after linker relaxation.
-  fixup_riscv_align,
-  // Fixups indicating a TLS descriptor code sequence, corresponding to auipc,
-  // lw/ld, addi, and jalr, respectively.
-  fixup_riscv_tlsdesc_hi20,
-  fixup_riscv_tlsdesc_load_lo12,
-  fixup_riscv_tlsdesc_add_lo12,
-  fixup_riscv_tlsdesc_call,
   // 12-bit fixup for symbol references in the 48-bit Xqcibi branch immediate
   // instructions
   fixup_riscv_qc_e_branch,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index 6283f1d120aaa..8af769d295ae8 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -181,7 +181,7 @@ void RISCVMCCodeEmitter::expandTLSDESCCall(const MCInst &MI,
   MCRegister Dest = MI.getOperand(1).getReg();
   int64_t Imm = MI.getOperand(2).getImm();
   Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(RISCV::fixup_riscv_tlsdesc_call), MI.getLoc()));
+      0, Expr, FirstRelocationKind + ELF::R_RISCV_TLSDESC_CALL, MI.getLoc()));
   MCInst Call =
       MCInstBuilder(RISCV::JALR).addReg(Link).addReg(Dest).addImm(Imm);
 
@@ -210,13 +210,13 @@ void RISCVMCCodeEmitter::expandAddTPRel(const MCInst &MI,
 
   // Emit the correct tprel_add relocation for the symbol.
   Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(RISCV::fixup_riscv_tprel_add), MI.getLoc()));
+      0, Expr, FirstRelocationKind + ELF::R_RISCV_TPREL_ADD, MI.getLoc()));
 
-  // Emit fixup_riscv_relax for tprel_add where the relax feature is enabled.
+  // Emit R_RISCV_RELAX for tprel_add where the relax feature is enabled.
   if (STI.hasFeature(RISCV::FeatureRelax)) {
     const MCConstantExpr *Dummy = MCConstantExpr::create(0, Ctx);
     Fixups.push_back(MCFixup::create(
-        0, Dummy, MCFixupKind(RISCV::fixup_riscv_relax), MI.getLoc()));
+        0, Dummy, FirstRelocationKind + ELF::R_RISCV_RELAX, MI.getLoc()));
   }
 
   // Emit a normal ADD instruction with the given operands.
@@ -567,7 +567,7 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
          "getImmOpValue expects only expressions or immediates");
   const MCExpr *Expr = MO.getExpr();
   MCExpr::ExprKind Kind = Expr->getKind();
-  RISCV::Fixups FixupKind = RISCV::fixup_riscv_invalid;
+  unsigned FixupKind = RISCV::fixup_riscv_invalid;
   bool RelaxCandidate = false;
   if (Kind == MCExpr::Target) {
     const RISCVMCExpr *RVExpr = cast<RISCVMCExpr>(Expr);
@@ -612,26 +612,26 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_GOT_HI:
-      FixupKind = RISCV::fixup_riscv_got_hi20;
+      FixupKind = FirstRelocationKind + ELF::R_RISCV_GOT_HI20;
       break;
     case RISCVMCExpr::VK_TPREL_LO:
       if (MIFrm == RISCVII::InstFormatI)
-        FixupKind = RISCV::fixup_riscv_tprel_lo12_i;
+        FixupKind = FirstRelocationKind + ELF::R_RISCV_TPREL_LO12_I;
       else if (MIFrm == RISCVII::InstFormatS)
-        FixupKind = RISCV::fixup_riscv_tprel_lo12_s;
+        FixupKind = FirstRelocationKind + ELF::R_RISCV_TPREL_LO12_S;
       else
         llvm_unreachable("VK_TPREL_LO used with unexpected instruction format");
       RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_TPREL_HI:
-      FixupKind = RISCV::fixup_riscv_tprel_hi20;
+      FixupKind = FirstRelocationKind + ELF::R_RISCV_TPREL_HI20;
       RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_TLS_GOT_HI:
-      FixupKind = RISCV::fixup_riscv_tls_got_hi20;
+      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLS_GOT_HI20;
       break;
     case RISCVMCExpr::VK_TLS_GD_HI:
-      FixupKind = RISCV::fixup_riscv_tls_gd_hi20;
+      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLS_GD_HI20;
       break;
     case RISCVMCExpr::VK_CALL:
       FixupKind = RISCV::fixup_riscv_call;
@@ -642,16 +642,16 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_TLSDESC_HI:
-      FixupKind = RISCV::fixup_riscv_tlsdesc_hi20;
+      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLSDESC_HI20;
       break;
     case RISCVMCExpr::VK_TLSDESC_LOAD_LO:
-      FixupKind = RISCV::fixup_riscv_tlsdesc_load_lo12;
+      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLSDESC_LOAD_LO12;
       break;
     case RISCVMCExpr::VK_TLSDESC_ADD_LO:
-      FixupKind = RISCV::fixup_riscv_tlsdesc_add_lo12;
+      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLSDESC_ADD_LO12;
       break;
     case RISCVMCExpr::VK_TLSDESC_CALL:
-      FixupKind = RISCV::fixup_riscv_tlsdesc_call;
+      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLSDESC_CALL;
       break;
     case RISCVMCExpr::VK_QC_ABS20:
       FixupKind = RISCV::fixup_riscv_qc_abs20_u;
@@ -689,9 +689,8 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
   // relaxed.
   if (EnableRelax && RelaxCandidate) {
     const MCConstantExpr *Dummy = MCConstantExpr::create(0, Ctx);
-    Fixups.push_back(
-    MCFixup::create(0, Dummy, MCFixupKind(RISCV::fixup_riscv_relax),
-                    MI.getLoc()));
+    Fixups.push_back(MCFixup::create(
+        0, Dummy, FirstRelocationKind + ELF::R_RISCV_RELAX, MI.getLoc()));
     ++MCNumFixups;
   }
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
index d6650e156c8b3..7c953f55a409a 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
@@ -67,17 +67,20 @@ const MCFixup *RISCVMCExpr::getPCRelHiFixup(const MCFragment **DFOut) const {
   for (const MCFixup &F : DF->getFixups()) {
     if (F.getOffset() != Offset)
       continue;
-
-    switch ((unsigned)F.getKind()) {
-    default:
-      continue;
-    case RISCV::fixup_riscv_got_hi20:
-    case RISCV::fixup_riscv_tls_got_hi20:
-    case RISCV::fixup_riscv_tls_gd_hi20:
-    case RISCV::fixup_riscv_pcrel_hi20:
-    case RISCV::fixup_riscv_tlsdesc_hi20:
-      if (DFOut)
+    auto Kind = F.getTargetKind();
+    if (Kind < FirstRelocationKind) {
+      if (Kind == RISCV::fixup_riscv_pcrel_hi20) {
         *DFOut = DF;
+        return &F;
+      }
+      break;
+    }
+    switch (Kind - FirstRelocationKind) {
+    case ELF::R_RISCV_GOT_HI20:
+    case ELF::R_RISCV_TLS_GOT_HI20:
+    case ELF::R_RISCV_TLS_GD_HI20:
+    case ELF::R_RISCV_TLSDESC_HI20:
+      *DFOut = DF;
       return &F;
     }
   }

From 59288761c9a8a3d5ecc70543ba09246497117f9e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 17 Apr 2025 21:46:24 -0700
Subject: [PATCH 373/710] [llvm] Use llvm::binary_search (NFC) (#136228)

---
 llvm/include/llvm/Support/UnicodeCharRanges.h   | 4 +---
 llvm/lib/Target/Mips/Mips16HardFloat.cpp        | 3 +--
 llvm/lib/Target/Mips/Mips16ISelLowering.cpp     | 6 ++----
 llvm/lib/Target/Mips/MipsCCState.cpp            | 3 +--
 llvm/tools/llvm-cov/CodeCoverage.cpp            | 2 +-
 llvm/utils/TableGen/Common/CodeGenRegisters.cpp | 3 +--
 6 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/llvm/include/llvm/Support/UnicodeCharRanges.h b/llvm/include/llvm/Support/UnicodeCharRanges.h
index 73d3603b74df0..7f1a9b3ff0c3b 100644
--- a/llvm/include/llvm/Support/UnicodeCharRanges.h
+++ b/llvm/include/llvm/Support/UnicodeCharRanges.h
@@ -61,9 +61,7 @@ class UnicodeCharSet {
 
   /// Returns true if the character set contains the Unicode code point
   /// \p C.
-  bool contains(uint32_t C) const {
-    return std::binary_search(Ranges.begin(), Ranges.end(), C);
-  }
+  bool contains(uint32_t C) const { return llvm::binary_search(Ranges, C); }
 
 private:
   /// Returns true if each of the ranges is a proper closed range
diff --git a/llvm/lib/Target/Mips/Mips16HardFloat.cpp b/llvm/lib/Target/Mips/Mips16HardFloat.cpp
index f7b623785b70c..97a54f979907a 100644
--- a/llvm/lib/Target/Mips/Mips16HardFloat.cpp
+++ b/llvm/lib/Target/Mips/Mips16HardFloat.cpp
@@ -369,8 +369,7 @@ static const char *const IntrinsicInline[] = {
 };
 
 static bool isIntrinsicInline(Function *F) {
-  return std::binary_search(std::begin(IntrinsicInline),
-                            std::end(IntrinsicInline), F->getName());
+  return llvm::binary_search(IntrinsicInline, F->getName());
 }
 
 // Returns of float, double and complex need to be handled with a helper
diff --git a/llvm/lib/Target/Mips/Mips16ISelLowering.cpp b/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
index 1027bcff84f8c..4fca1f71fdb57 100644
--- a/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
+++ b/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
@@ -427,8 +427,7 @@ getOpndList(SmallVectorImpl<SDValue> &Ops,
     if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(CLI.Callee)) {
       Mips16Libcall Find = { RTLIB::UNKNOWN_LIBCALL, S->getSymbol() };
 
-      if (std::binary_search(std::begin(HardFloatLibCalls),
-                             std::end(HardFloatLibCalls), Find))
+      if (llvm::binary_search(HardFloatLibCalls, Find))
         LookupHelper = false;
       else {
         const char *Symbol = S->getSymbol();
@@ -469,8 +468,7 @@ getOpndList(SmallVectorImpl<SDValue> &Ops,
       Mips16Libcall Find = { RTLIB::UNKNOWN_LIBCALL,
                              G->getGlobal()->getName().data() };
 
-      if (std::binary_search(std::begin(HardFloatLibCalls),
-                             std::end(HardFloatLibCalls), Find))
+      if (llvm::binary_search(HardFloatLibCalls, Find))
         LookupHelper = false;
     }
     if (LookupHelper)
diff --git a/llvm/lib/Target/Mips/MipsCCState.cpp b/llvm/lib/Target/Mips/MipsCCState.cpp
index 781bb7c8c7e6d..9e8cd2ea2fd43 100644
--- a/llvm/lib/Target/Mips/MipsCCState.cpp
+++ b/llvm/lib/Target/Mips/MipsCCState.cpp
@@ -30,8 +30,7 @@ bool MipsCCState::isF128SoftLibCall(const char *CallSym) {
   // Check that LibCalls is sorted alphabetically.
   auto Comp = [](const char *S1, const char *S2) { return strcmp(S1, S2) < 0; };
   assert(llvm::is_sorted(LibCalls, Comp));
-  return std::binary_search(std::begin(LibCalls), std::end(LibCalls), CallSym,
-                            Comp);
+  return llvm::binary_search(LibCalls, CallSym, Comp);
 }
 
 /// This function returns true if Ty is fp128, {f128} or i128 which was
diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp
index a740cdd45b901..c828e25de4b02 100644
--- a/llvm/tools/llvm-cov/CodeCoverage.cpp
+++ b/llvm/tools/llvm-cov/CodeCoverage.cpp
@@ -547,7 +547,7 @@ void CodeCoverageTool::removeUnmappedInputs(const CoverageMapping &Coverage) {
   // The user may have specified source files which aren't in the coverage
   // mapping. Filter these files away.
   llvm::erase_if(SourceFiles, [&](const std::string &SF) {
-    return !std::binary_search(CoveredFiles.begin(), CoveredFiles.end(), SF);
+    return !llvm::binary_search(CoveredFiles, SF);
   });
 }
 
diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
index eb142e66faf2f..3479cad10b66c 100644
--- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
@@ -922,8 +922,7 @@ bool CodeGenRegisterClass::hasType(const ValueTypeByHwMode &VT) const {
 }
 
 bool CodeGenRegisterClass::contains(const CodeGenRegister *Reg) const {
-  return std::binary_search(Members.begin(), Members.end(), Reg,
-                            deref<std::less<>>());
+  return llvm::binary_search(Members, Reg, deref<std::less<>>());
 }
 
 unsigned CodeGenRegisterClass::getWeight(const CodeGenRegBank &RegBank) const {

From b07ee6acffd4c6c19093f8a5e9f8c6c594f3c90a Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <peter@pcc.me.uk>
Date: Thu, 17 Apr 2025 21:41:53 -0700
Subject: [PATCH 374/710] LowerTypeTests: Simplify pointer types.

---
 llvm/lib/Transforms/IPO/LowerTypeTests.cpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 38da7329bbd58..7c01f8b560fea 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -447,10 +447,9 @@ class LowerTypeTestsModule {
 
   IntegerType *Int1Ty = Type::getInt1Ty(M.getContext());
   IntegerType *Int8Ty = Type::getInt8Ty(M.getContext());
-  PointerType *Int8PtrTy = PointerType::getUnqual(M.getContext());
+  PointerType *PtrTy = PointerType::getUnqual(M.getContext());
   ArrayType *Int8Arr0Ty = ArrayType::get(Type::getInt8Ty(M.getContext()), 0);
   IntegerType *Int32Ty = Type::getInt32Ty(M.getContext());
-  PointerType *Int32PtrTy = PointerType::getUnqual(M.getContext());
   IntegerType *Int64Ty = Type::getInt64Ty(M.getContext());
   IntegerType *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext(), 0);
 
@@ -654,7 +653,7 @@ void LowerTypeTestsModule::allocateByteArrays() {
     BAB.allocate(BAI->Bits, BAI->BitSize, ByteArrayOffsets[I], Mask);
 
     BAI->MaskGlobal->replaceAllUsesWith(
-        ConstantExpr::getIntToPtr(ConstantInt::get(Int8Ty, Mask), Int8PtrTy));
+        ConstantExpr::getIntToPtr(ConstantInt::get(Int8Ty, Mask), PtrTy));
     BAI->MaskGlobal->eraseFromParent();
     if (BAI->MaskPtr)
       *BAI->MaskPtr = Mask;
@@ -948,7 +947,7 @@ uint8_t *LowerTypeTestsModule::exportTypeId(StringRef TypeId,
 
   auto ExportConstant = [&](StringRef Name, uint64_t &Storage, Constant *C) {
     if (shouldExportConstantsAsAbsoluteSymbols())
-      ExportGlobal(Name, ConstantExpr::getIntToPtr(C, Int8PtrTy));
+      ExportGlobal(Name, ConstantExpr::getIntToPtr(C, PtrTy));
     else
       Storage = cast<ConstantInt>(C)->getZExtValue();
   };
@@ -1046,7 +1045,7 @@ LowerTypeTestsModule::importTypeId(StringRef TypeId) {
 
   if (TIL.TheKind == TypeTestResolution::ByteArray) {
     TIL.TheByteArray = ImportGlobal("byte_array");
-    TIL.BitMask = ImportConstant("bit_mask", TTRes.BitMask, 8, Int8PtrTy);
+    TIL.BitMask = ImportConstant("bit_mask", TTRes.BitMask, 8, PtrTy);
   }
 
   if (TIL.TheKind == TypeTestResolution::Inline)
@@ -1778,7 +1777,7 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsWASM(
 
   // The indirect function table index space starts at zero, so pass a NULL
   // pointer as the subtracted "jump table" offset.
-  lowerTypeTestCalls(TypeIds, ConstantPointerNull::get(Int32PtrTy),
+  lowerTypeTestCalls(TypeIds, ConstantPointerNull::get(PtrTy),
                      GlobalLayout);
 }
 

From 155fc76f20fec106d59701b3d320b74d46894b37 Mon Sep 17 00:00:00 2001
From: Iris <0.0@owo.li>
Date: Fri, 18 Apr 2025 11:16:58 +0800
Subject: [PATCH 375/710] Recommit "[RISCV] Strengthen register usage
 validation for XTheadMemPair loads (#136241)"

With test fix.

Closes #136087

https://github.com/XUANTIE-RV/thead-extension-spec/blob/master/xtheadmempair/lwd.adoc
---
 llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp | 4 ++--
 llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s     | 3 ++-
 llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s     | 3 ++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 32751782e08d4..db9ec7ae1df02 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -3656,9 +3656,9 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     MCRegister Rd2 = Inst.getOperand(1).getReg();
     MCRegister Rs1 = Inst.getOperand(2).getReg();
     // The encoding with rd1 == rd2 == rs1 is reserved for XTHead load pair.
-    if (Rs1 == Rd1 && Rs1 == Rd2) {
+    if (Rs1 == Rd1 || Rs1 == Rd2 || Rd1 == Rd2) {
       SMLoc Loc = Operands[1]->getStartLoc();
-      return Error(Loc, "rs1, rd1, and rd2 cannot all be the same");
+      return Error(Loc, "rs1, rd1, and rd2 cannot overlap");
     }
   }
 
diff --git a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
index a9223ae128a40..8a7202ffd375e 100644
--- a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
@@ -15,6 +15,7 @@ th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be consta
 th.swd t3, t4, (t5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.swd t3, t4, (t5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
 th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
-th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot all be the same
+th.lwd x6, x7, (x7), 2, 3   # CHECK: [[@LINE]]:8: error: rs1, rd1, and rd2 cannot overlap
+th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot overlap
 th.ldd t0, t1, (t2), 2, 4   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
 th.sdd t0, t1, (t2), 2, 4   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
diff --git a/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
index 342db21bbca62..dffdb0a2cc6f5 100644
--- a/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
@@ -15,4 +15,5 @@ th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be consta
 th.swd t3, t4, (t5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.swd t3, t4, (t5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
 th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
-th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot all be the same
+th.ldd x6, x6, (x7), 2, 4   # CHECK: [[@LINE]]:8: error: rs1, rd1, and rd2 cannot overlap
+th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot overlap

From 2b002d68044e6b482a898c1deff578f76fa9b8a1 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 17 Apr 2025 21:56:41 -0700
Subject: [PATCH 376/710] [gn] port 1756fcb8b019

---
 .../gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
index 0e04549587b2e..9369c240ddbf6 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
@@ -37,7 +37,6 @@ source_set("sources") {
     "linux.cpp",
     "linux.h",
     "list.h",
-    "local_cache.h",
     "mem_map.cpp",
     "mem_map.h",
     "mem_map_base.h",
@@ -59,6 +58,7 @@ source_set("sources") {
     "report_linux.cpp",
     "report_linux.h",
     "secondary.h",
+    "size_class_allocator.h",
     "size_class_map.h",
     "stack_depot.h",
     "stats.h",

From f28408f3af0d4533dd1d8048c5fdcdb14b934d05 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 17 Apr 2025 22:29:42 -0700
Subject: [PATCH 377/710] [test] Remove CHECK lines for MCAsmStreamer's fixup
 output

The fixup output is a debug aid and should not be used to test
target-specific relocation generation implementation. The llvm-mc
-filetype=obj output is what truly matters.
---
 llvm/test/MC/AVR/inst-adiw.s                  |   2 -
 llvm/test/MC/AVR/inst-andi.s                  |   1 -
 llvm/test/MC/AVR/inst-brbc.s                  |   2 -
 llvm/test/MC/AVR/inst-brbs.s                  |   2 -
 llvm/test/MC/AVR/inst-brcc.s                  |   4 -
 llvm/test/MC/AVR/inst-brcs.s                  |   4 -
 llvm/test/MC/AVR/inst-breq.s                  |   4 -
 llvm/test/MC/AVR/inst-brge.s                  |   3 -
 llvm/test/MC/AVR/inst-brhc.s                  |   3 -
 llvm/test/MC/AVR/inst-brhs.s                  |   3 -
 llvm/test/MC/AVR/inst-brid.s                  |   3 -
 llvm/test/MC/AVR/inst-brie.s                  |   3 -
 llvm/test/MC/AVR/inst-brlo.s                  |   3 -
 llvm/test/MC/AVR/inst-brlt.s                  |   3 -
 llvm/test/MC/AVR/inst-brmi.s                  |   3 -
 llvm/test/MC/AVR/inst-brne.s                  |   4 -
 llvm/test/MC/AVR/inst-brpl.s                  |   3 -
 llvm/test/MC/AVR/inst-brsh.s                  |   3 -
 llvm/test/MC/AVR/inst-brtc.s                  |   3 -
 llvm/test/MC/AVR/inst-brts.s                  |   3 -
 llvm/test/MC/AVR/inst-brvc.s                  |   3 -
 llvm/test/MC/AVR/inst-brvs.s                  |   3 -
 llvm/test/MC/AVR/inst-call.s                  |   1 -
 llvm/test/MC/AVR/inst-cbi.s                   |   1 -
 llvm/test/MC/AVR/inst-cpi.s                   |   1 -
 llvm/test/MC/AVR/inst-in.s                    |   1 -
 llvm/test/MC/AVR/inst-jmp.s                   |   1 -
 llvm/test/MC/AVR/inst-ldd.s                   |   1 -
 llvm/test/MC/AVR/inst-ldi.s                   |   1 -
 llvm/test/MC/AVR/inst-lds-tiny.s              |   3 -
 llvm/test/MC/AVR/inst-lds.s                   |   3 -
 llvm/test/MC/AVR/inst-ori.s                   |   1 -
 llvm/test/MC/AVR/inst-out.s                   |   1 -
 llvm/test/MC/AVR/inst-rcall.s                 |   4 -
 llvm/test/MC/AVR/inst-rjmp.s                  |  10 -
 llvm/test/MC/AVR/inst-sbci.s                  |   1 -
 llvm/test/MC/AVR/inst-sbi.s                   |   1 -
 llvm/test/MC/AVR/inst-sbic.s                  |   1 -
 llvm/test/MC/AVR/inst-sbis.s                  |   1 -
 llvm/test/MC/AVR/inst-sbiw.s                  |   2 -
 llvm/test/MC/AVR/inst-sbr.s                   |   1 -
 llvm/test/MC/AVR/inst-std.s                   |   1 -
 llvm/test/MC/AVR/inst-sts-tiny.s              |   3 -
 llvm/test/MC/AVR/inst-sts.s                   |   3 -
 llvm/test/MC/AVR/inst-subi.s                  |   1 -
 llvm/test/MC/AVR/modifiers.s                  |  29 --
 llvm/test/MC/CSKY/basic-16bit.s               |   4 -
 llvm/test/MC/CSKY/basic.s                     |  14 -
 llvm/test/MC/Sparc/sparc-assembly-exprs.s     |   6 -
 llvm/test/MC/Sparc/sparc-ctrl-instructions.s  | 121 ---------
 llvm/test/MC/Sparc/sparc-little-endian.s      |   1 -
 llvm/test/MC/Sparc/sparc-relocations.s        |  18 --
 .../MC/Sparc/sparc-synthetic-instructions.s   |  19 --
 llvm/test/MC/Sparc/sparc-tls-relocations.s    |  18 --
 .../test/MC/Sparc/sparc64-ctrl-instructions.s | 253 ------------------
 .../MC/Sparc/sparcv9-synthetic-instructions.s |  25 --
 56 files changed, 617 deletions(-)

diff --git a/llvm/test/MC/AVR/inst-adiw.s b/llvm/test/MC/AVR/inst-adiw.s
index 6b83027234c11..27df92f48d680 100644
--- a/llvm/test/MC/AVR/inst-adiw.s
+++ b/llvm/test/MC/AVR/inst-adiw.s
@@ -30,9 +30,7 @@ foo:
 ; CHECK: adiw r30,  0                ; encoding: [0x30,0x96]
 
 ; CHECK: adiw r24, SYMBOL            ; encoding: [0b00AAAAAA,0x96]
-                                     ;   fixup A - offset: 0, value: SYMBOL, kind: fixup_6_adiw
 ; CHECK: adiw r24, r25               ; encoding: [0b00AAAAAA,0x96]
-                                     ;   fixup A - offset: 0, value: r25, kind: fixup_6_adiw
 
 ; CHECK-INST: adiw r26,  12
 ; CHECK-INST: adiw r26,  63
diff --git a/llvm/test/MC/AVR/inst-andi.s b/llvm/test/MC/AVR/inst-andi.s
index 14f85bbd66010..1835dcd189eb6 100644
--- a/llvm/test/MC/AVR/inst-andi.s
+++ b/llvm/test/MC/AVR/inst-andi.s
@@ -15,7 +15,6 @@ foo:
 ; CHECK: andi r27, 92                  ; encoding: [0xbc,0x75]
 
 ; CHECK: andi r20, BAR                 ; encoding: [0x40'A',0x70]
-; CHECK:                               ;   fixup A - offset: 0, value: BAR, kind: fixup_ldi
 
 ; CHECK-INST: andi r16, 255
 ; CHECK-INST: andi r29, 190
diff --git a/llvm/test/MC/AVR/inst-brbc.s b/llvm/test/MC/AVR/inst-brbc.s
index 6aa6ed0863e0c..6d96393b42ad1 100644
--- a/llvm/test/MC/AVR/inst-brbc.s
+++ b/llvm/test/MC/AVR/inst-brbc.s
@@ -11,10 +11,8 @@ foo:
   .short 0xf4c7
 
 ; CHECK: brvc .Ltmp0+8+2   ; encoding: [0bAAAAA011,0b111101AA]
-; CHECK-NEXT:                ; fixup A - offset: 0, value: .Ltmp0+8+2, kind: fixup_7_pcrel
 ;
 ; CHECK: brcc .Ltmp1-16+2  ; encoding: [0bAAAAA000,0b111101AA]
-; CHECK-NEXT:                ; fixup A - offset: 0, value: .Ltmp1-16+2, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 23 f4   brvc .+8
diff --git a/llvm/test/MC/AVR/inst-brbs.s b/llvm/test/MC/AVR/inst-brbs.s
index abadd10a134f8..9dde5e16abde3 100644
--- a/llvm/test/MC/AVR/inst-brbs.s
+++ b/llvm/test/MC/AVR/inst-brbs.s
@@ -11,9 +11,7 @@ foo:
   .short 0xf077
 
 ; CHECK: brvs .Ltmp0+8+2   ; encoding: [0bAAAAA011,0b111100AA]
-; CHECK-NEXT:                ; fixup A - offset: 0, value: .Ltmp0+8+2, kind: fixup_7_pcrel
 ; CHECK: brcs .Ltmp1-12+2  ; encoding: [0bAAAAA000,0b111100AA]
-; CHECK-NEXT:                ; fixup A - offset: 0, value: .Ltmp1-12+2, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 23 f0   brvs .+8
diff --git a/llvm/test/MC/AVR/inst-brcc.s b/llvm/test/MC/AVR/inst-brcc.s
index 90d7dc3eca1b9..0edefa167ac44 100644
--- a/llvm/test/MC/AVR/inst-brcc.s
+++ b/llvm/test/MC/AVR/inst-brcc.s
@@ -13,13 +13,9 @@ foo:
 bar:
 
 ; CHECK: brcc .Ltmp0+66+2  ; encoding: [0bAAAAA000,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+66+2, kind: fixup_7_pcrel
 ; CHECK: brcc .Ltmp1-22+2  ; encoding: [0bAAAAA000,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1-22+2, kind: fixup_7_pcrel
 ; CHECK: brcc .Ltmp2+66+2  ; encoding: [0bAAAAA000,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp2+66+2, kind: fixup_7_pcrel
 ; CHECK: brcc bar            ; encoding: [0bAAAAA000,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 08 f5      brsh .+66
diff --git a/llvm/test/MC/AVR/inst-brcs.s b/llvm/test/MC/AVR/inst-brcs.s
index 74cc9a1c97805..ea8a3f527c9e4 100644
--- a/llvm/test/MC/AVR/inst-brcs.s
+++ b/llvm/test/MC/AVR/inst-brcs.s
@@ -13,13 +13,9 @@ foo:
 bar:
 
 ; CHECK: brcs .Ltmp0+8+2  ; encoding: [0bAAAAA000,0b111100AA]
-; CHECK-NEXT:               ;   fixup A - offset: 0, value: .Ltmp0+8+2, kind: fixup_7_pcrel
 ; CHECK: brcs .Ltmp1+4+2  ; encoding: [0bAAAAA000,0b111100AA]
-; CHECK-NEXT:               ;   fixup A - offset: 0, value: .Ltmp1+4+2, kind: fixup_7_pcrel
 ; CHECK: brcs .Ltmp2+8+2  ; encoding: [0bAAAAA000,0b111100AA]
-; CHECK-NEXT:               ;   fixup A - offset: 0, value: .Ltmp2+8+2, kind: fixup_7_pcrel
 ; CHECK: brcs bar           ; encoding: [0bAAAAA000,0b111100AA]
-; CHECK-NEXT:               ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 20 f0      brlo .+8
diff --git a/llvm/test/MC/AVR/inst-breq.s b/llvm/test/MC/AVR/inst-breq.s
index 51bc6192e05ed..d916f6dc18c8f 100644
--- a/llvm/test/MC/AVR/inst-breq.s
+++ b/llvm/test/MC/AVR/inst-breq.s
@@ -13,13 +13,9 @@ foo:
 bar:
 
 ; CHECK: breq    .Ltmp0-18+2     ; encoding: [0bAAAAA001,0b111100AA]
-; CHECK-NEXT:                      ;   fixup A - offset: 0, value: .Ltmp0-18+2, kind: fixup_7_pcrel
 ; CHECK: breq    .Ltmp1-12+2     ; encoding: [0bAAAAA001,0b111100AA]
-; CHECK-NEXT:                      ;   fixup A - offset: 0, value: .Ltmp1-12+2, kind: fixup_7_pcrel
 ; CHECK: brbs    1, .Ltmp2-18+2  ; encoding: [0bAAAAA001,0b111100AA]
-; CHECK-NEXT:                      ;   fixup A - offset: 0, value: .Ltmp2-18+2, kind: fixup_7_pcrel
 ; CHECK: brbs    1, bar            ; encoding: [0bAAAAA001,0b111100AA]
-; CHECK-NEXT:                      ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: b9 f3      breq .-18
diff --git a/llvm/test/MC/AVR/inst-brge.s b/llvm/test/MC/AVR/inst-brge.s
index 904f4a496e777..3a8fd727d773e 100644
--- a/llvm/test/MC/AVR/inst-brge.s
+++ b/llvm/test/MC/AVR/inst-brge.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brge .Ltmp0+50+2  ; encoding: [0bAAAAA100,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+50+2, kind: fixup_7_pcrel
 ; CHECK: brge .Ltmp1+42+2  ; encoding: [0bAAAAA100,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+42+2, kind: fixup_7_pcrel
 ; CHECK: brge bar            ; encoding: [0bAAAAA100,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: cc f4      brge .+50
diff --git a/llvm/test/MC/AVR/inst-brhc.s b/llvm/test/MC/AVR/inst-brhc.s
index 77052e664d389..4fc55b6ab0347 100644
--- a/llvm/test/MC/AVR/inst-brhc.s
+++ b/llvm/test/MC/AVR/inst-brhc.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brhc .Ltmp0+12+2  ; encoding: [0bAAAAA101,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+12+2, kind: fixup_7_pcrel
 ; CHECK: brhc .Ltmp1+14+2  ; encoding: [0bAAAAA101,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+14+2, kind: fixup_7_pcrel
 ; CHECK: brhc bar            ; encoding: [0bAAAAA101,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 35 f4      brhc .+12
diff --git a/llvm/test/MC/AVR/inst-brhs.s b/llvm/test/MC/AVR/inst-brhs.s
index b4c55cafd5de9..d0968753cded2 100644
--- a/llvm/test/MC/AVR/inst-brhs.s
+++ b/llvm/test/MC/AVR/inst-brhs.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brhs .Ltmp0-66+2  ; encoding: [0bAAAAA101,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0-66+2, kind: fixup_7_pcrel
 ; CHECK: brhs .Ltmp1+14+2  ; encoding: [0bAAAAA101,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+14+2, kind: fixup_7_pcrel
 ; CHECK: brhs bar            ; encoding: [0bAAAAA101,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: fd f2      brhs .-66
diff --git a/llvm/test/MC/AVR/inst-brid.s b/llvm/test/MC/AVR/inst-brid.s
index 4cf1869dc1b2c..2a3a30f905cf5 100644
--- a/llvm/test/MC/AVR/inst-brid.s
+++ b/llvm/test/MC/AVR/inst-brid.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brid .Ltmp0+42+2  ; encoding: [0bAAAAA111,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+42+2, kind: fixup_7_pcrel
 ; CHECK: brid .Ltmp1+62+2  ; encoding: [0bAAAAA111,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+62+2, kind: fixup_7_pcrel
 ; CHECK: brid bar            ; encoding: [0bAAAAA111,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: af f4      brid .+42
diff --git a/llvm/test/MC/AVR/inst-brie.s b/llvm/test/MC/AVR/inst-brie.s
index 7c7e97c2b201e..4f867ae99852a 100644
--- a/llvm/test/MC/AVR/inst-brie.s
+++ b/llvm/test/MC/AVR/inst-brie.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brie .Ltmp0+20+2  ; encoding: [0bAAAAA111,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+20+2, kind: fixup_7_pcrel
 ; CHECK: brie .Ltmp1+40+2  ; encoding: [0bAAAAA111,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+40+2, kind: fixup_7_pcrel
 ; CHECK: brie bar            ; encoding: [0bAAAAA111,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 57 f0      brie .+20
diff --git a/llvm/test/MC/AVR/inst-brlo.s b/llvm/test/MC/AVR/inst-brlo.s
index 9523fd5695a99..48499aa69926b 100644
--- a/llvm/test/MC/AVR/inst-brlo.s
+++ b/llvm/test/MC/AVR/inst-brlo.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brlo .Ltmp0+12+2  ; encoding: [0bAAAAA000,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+12+2, kind: fixup_7_pcrel
 ; CHECK: brlo .Ltmp1+28+2  ; encoding: [0bAAAAA000,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+28+2, kind: fixup_7_pcrel
 ; CHECK: brlo bar            ; encoding: [0bAAAAA000,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 30 f0      brlo .+12
diff --git a/llvm/test/MC/AVR/inst-brlt.s b/llvm/test/MC/AVR/inst-brlt.s
index c309310909fa7..e16fd05b3e144 100644
--- a/llvm/test/MC/AVR/inst-brlt.s
+++ b/llvm/test/MC/AVR/inst-brlt.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brlt .Ltmp0+16+2  ; encoding: [0bAAAAA100,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+16+2, kind: fixup_7_pcrel
 ; CHECK: brlt .Ltmp1+2+2   ; encoding: [0bAAAAA100,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+2+2, kind: fixup_7_pcrel
 ; CHECK: brlt bar            ; encoding: [0bAAAAA100,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 44 f0    brlt .+16
diff --git a/llvm/test/MC/AVR/inst-brmi.s b/llvm/test/MC/AVR/inst-brmi.s
index ec60bc4a14f1c..0d46af8b75969 100644
--- a/llvm/test/MC/AVR/inst-brmi.s
+++ b/llvm/test/MC/AVR/inst-brmi.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brmi .Ltmp0+66+2  ; encoding: [0bAAAAA010,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+66+2, kind: fixup_7_pcrel
 ; CHECK: brmi .Ltmp1+58+2  ; encoding: [0bAAAAA010,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+58+2, kind: fixup_7_pcrel
 ; CHECK: brmi bar            ; encoding: [0bAAAAA010,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 0a f1      brmi .+66
diff --git a/llvm/test/MC/AVR/inst-brne.s b/llvm/test/MC/AVR/inst-brne.s
index 2a424a3593247..e87813a60b504 100644
--- a/llvm/test/MC/AVR/inst-brne.s
+++ b/llvm/test/MC/AVR/inst-brne.s
@@ -13,13 +13,9 @@ foo:
 bar:
 
 ; CHECK: brne    .Ltmp0+10+2     ; encoding: [0bAAAAA001,0b111101AA]
-; CHECK-NEXT:                      ;   fixup A - offset: 0, value: .Ltmp0+10+2, kind: fixup_7_pcrel
 ; CHECK: brne    .Ltmp1+2+2      ; encoding: [0bAAAAA001,0b111101AA]
-; CHECK-NEXT:                      ;   fixup A - offset: 0, value: .Ltmp1+2+2, kind: fixup_7_pcrel
 ; CHECK: brbc    1, .Ltmp2+10+2  ; encoding: [0bAAAAA001,0b111101AA]
-; CHECK-NEXT:                      ;   fixup A - offset: 0, value: .Ltmp2+10+2, kind: fixup_7_pcrel
 ; CHECK: brbc    1, bar            ; encoding: [0bAAAAA001,0b111101AA]
-; CHECK-NEXT:                      ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 29 f4      brne .+10
diff --git a/llvm/test/MC/AVR/inst-brpl.s b/llvm/test/MC/AVR/inst-brpl.s
index d752f34ee606b..34877961bf328 100644
--- a/llvm/test/MC/AVR/inst-brpl.s
+++ b/llvm/test/MC/AVR/inst-brpl.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brpl .Ltmp0-12+2  ; encoding: [0bAAAAA010,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0-12+2, kind: fixup_7_pcrel
 ; CHECK: brpl .Ltmp1+18+2  ; encoding: [0bAAAAA010,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+18+2, kind: fixup_7_pcrel
 ; CHECK: brpl bar            ; encoding: [0bAAAAA010,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: d2 f7      brpl .-12
diff --git a/llvm/test/MC/AVR/inst-brsh.s b/llvm/test/MC/AVR/inst-brsh.s
index 95a6a52acb60c..be0a06c445e65 100644
--- a/llvm/test/MC/AVR/inst-brsh.s
+++ b/llvm/test/MC/AVR/inst-brsh.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brsh .Ltmp0+32+2  ; encoding: [0bAAAAA000,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+32+2, kind: fixup_7_pcrel
 ; CHECK: brsh .Ltmp1+70+2  ; encoding: [0bAAAAA000,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+70+2, kind: fixup_7_pcrel
 ; CHECK: brsh bar            ; encoding: [0bAAAAA000,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 80 f4      brsh .+32
diff --git a/llvm/test/MC/AVR/inst-brtc.s b/llvm/test/MC/AVR/inst-brtc.s
index d8704dc6f345d..312c55c3f4729 100644
--- a/llvm/test/MC/AVR/inst-brtc.s
+++ b/llvm/test/MC/AVR/inst-brtc.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brtc .Ltmp0+52+2  ; encoding: [0bAAAAA110,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+52+2, kind: fixup_7_pcrel
 ; CHECK: brtc .Ltmp1+50+2  ; encoding: [0bAAAAA110,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+50+2, kind: fixup_7_pcrel
 ; CHECK: brtc bar            ; encoding: [0bAAAAA110,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: d6 f4      brtc .+52
diff --git a/llvm/test/MC/AVR/inst-brts.s b/llvm/test/MC/AVR/inst-brts.s
index 976f23ff8c208..40ef6af5eef16 100644
--- a/llvm/test/MC/AVR/inst-brts.s
+++ b/llvm/test/MC/AVR/inst-brts.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brts .Ltmp0+18+2  ; encoding: [0bAAAAA110,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+18+2, kind: fixup_7_pcrel
 ; CHECK: brts .Ltmp1+22+2  ; encoding: [0bAAAAA110,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+22+2, kind: fixup_7_pcrel
 ; CHECK: brts bar            ; encoding: [0bAAAAA110,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 4e f0      brts .+18
diff --git a/llvm/test/MC/AVR/inst-brvc.s b/llvm/test/MC/AVR/inst-brvc.s
index 766146cc57aaf..d493ff1fbf544 100644
--- a/llvm/test/MC/AVR/inst-brvc.s
+++ b/llvm/test/MC/AVR/inst-brvc.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brvc .Ltmp0-28+2  ; encoding: [0bAAAAA011,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0-28+2, kind: fixup_7_pcrel
 ; CHECK: brvc .Ltmp1-62+2  ; encoding: [0bAAAAA011,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1-62+2, kind: fixup_7_pcrel
 ; CHECK: brvc bar            ; encoding: [0bAAAAA011,0b111101AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 93 f7      brvc .-28
diff --git a/llvm/test/MC/AVR/inst-brvs.s b/llvm/test/MC/AVR/inst-brvs.s
index 9ebe9c3181ee9..07755d8aea21f 100644
--- a/llvm/test/MC/AVR/inst-brvs.s
+++ b/llvm/test/MC/AVR/inst-brvs.s
@@ -12,11 +12,8 @@ foo:
 bar:
 
 ; CHECK: brvs .Ltmp0+18+2  ; encoding: [0bAAAAA011,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp0+18+2, kind: fixup_7_pcrel
 ; CHECK: brvs .Ltmp1+32+2  ; encoding: [0bAAAAA011,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: .Ltmp1+32+2, kind: fixup_7_pcrel
 ; CHECK: brvs bar            ; encoding: [0bAAAAA011,0b111100AA]
-; CHECK-NEXT:                ;   fixup A - offset: 0, value: bar, kind: fixup_7_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 4b f0      brvs .+18
diff --git a/llvm/test/MC/AVR/inst-call.s b/llvm/test/MC/AVR/inst-call.s
index 491a38346bbe0..af4f95b327cb3 100644
--- a/llvm/test/MC/AVR/inst-call.s
+++ b/llvm/test/MC/AVR/inst-call.s
@@ -15,7 +15,6 @@ r25:
 ; CHECK: call -12                   ; encoding: [0xff,0x95,0xfa,0xff]
 ; CHECK: call  0                    ; encoding: [0x0e,0x94,0x00,0x00]
 ; CHECK: call  r25                  ; encoding: [0x0e'A',0x94'A',0b00AAAAAA,0x00]
-; CHECK:                            ;   fixup A - offset: 0, value: r25, kind: fixup_call
 
 ; CHECK-INST: call 4096
 ; CHECK-INST: call 8388484
diff --git a/llvm/test/MC/AVR/inst-cbi.s b/llvm/test/MC/AVR/inst-cbi.s
index 6fa9c46960fcd..54f4e65c10064 100644
--- a/llvm/test/MC/AVR/inst-cbi.s
+++ b/llvm/test/MC/AVR/inst-cbi.s
@@ -21,7 +21,6 @@ foo:
 ; CHECK: cbi 31, 7                 ; encoding: [0xff,0x98]
 
 ; CHECK: cbi bar-2, 2              ; encoding: [0bAAAAA010,0x98]
-; CHECK:                           ;   fixup A - offset: 0, value: bar-2, kind: fixup_port5
 
 ; CHECK-INST: cbi 3, 5
 ; CHECK-INST: cbi 1, 1
diff --git a/llvm/test/MC/AVR/inst-cpi.s b/llvm/test/MC/AVR/inst-cpi.s
index e17b505e791c5..1c29f6231dd24 100644
--- a/llvm/test/MC/AVR/inst-cpi.s
+++ b/llvm/test/MC/AVR/inst-cpi.s
@@ -15,7 +15,6 @@ foo:
 ; CHECK: cpi r27, 92                   ; encoding: [0xbc,0x35]
 
 ; CHECK: cpi r21, ear                  ; encoding: [0x50'A',0x30]
-; CHECK:                               ;   fixup A - offset: 0, value: ear, kind: fixup_ldi
 
 ; CHECK-INST: cpi r16, 241
 ; CHECK-INST: cpi r29, 190
diff --git a/llvm/test/MC/AVR/inst-in.s b/llvm/test/MC/AVR/inst-in.s
index e17f25b67d03d..87b5874b602a1 100644
--- a/llvm/test/MC/AVR/inst-in.s
+++ b/llvm/test/MC/AVR/inst-in.s
@@ -21,7 +21,6 @@ foo:
 ; CHECK: in r31, 63                 ; encoding: [0xff,0xb7]
 
 ; CHECK: in r20, foo+1              ; encoding: [0x40'A',0xb1'A']
-; CHECK:                            ;   fixup A - offset: 0, value: foo+1, kind: fixup_port6
 
 ; CHECK-INST: in r2, 4
 ; CHECK-INST: in r9, 6
diff --git a/llvm/test/MC/AVR/inst-jmp.s b/llvm/test/MC/AVR/inst-jmp.s
index 0d5b16e02206d..2e992ddb0e91d 100644
--- a/llvm/test/MC/AVR/inst-jmp.s
+++ b/llvm/test/MC/AVR/inst-jmp.s
@@ -19,7 +19,6 @@ foo:
 ; CHECK: jmp  0                    ; encoding: [0x0c,0x94,0x00,0x00]
 
 ; CHECK: jmp foo+1                 ; encoding: [0x0c'A',0x94'A',0b00AAAAAA,0x00]
-; CHECK:                           ;   fixup A - offset: 0, value: foo+1, kind: fixup_call
 
 ; CHECK: jmp 262142                ; encoding: [0x0d,0x94,0xff,0xff]
 ; CHECK: jmp 8126464               ; encoding: [0xfc,0x95,0x00,0x00]
diff --git a/llvm/test/MC/AVR/inst-ldd.s b/llvm/test/MC/AVR/inst-ldd.s
index 70ba44617690e..2c6fdd25cdb00 100644
--- a/llvm/test/MC/AVR/inst-ldd.s
+++ b/llvm/test/MC/AVR/inst-ldd.s
@@ -18,7 +18,6 @@ foo:
 ; CHECK: ldd r7, Z+30                 ; encoding: [0x76,0x8c]
 
 ; CHECK: ldd r9, Z+foo                ; encoding: [0x90'A',0x80'A']
-; CHECK:                              ;   fixup A - offset: 0, value: +foo, kind: fixup_6
 
 ; INST: ldd r2, Y+2
 ; INST: ldd r0, Y+0
diff --git a/llvm/test/MC/AVR/inst-ldi.s b/llvm/test/MC/AVR/inst-ldi.s
index a6e2bc68b59e0..abdac26c4bd33 100644
--- a/llvm/test/MC/AVR/inst-ldi.s
+++ b/llvm/test/MC/AVR/inst-ldi.s
@@ -14,7 +14,6 @@ foo:
 ; CHECK: ldi r27, 92                  ; encoding: [0xbc,0xe5]
 
 ; CHECK: ldi r21, SYMBOL+3            ; encoding: [0x50'A',0xe0]
-; CHECK:                              ;   fixup A - offset: 0, value: SYMBOL+3, kind: fixup_ldi
 
 ; CHECK-INST: ldi r16, 241
 ; CHECK-INST: ldi r29, 190
diff --git a/llvm/test/MC/AVR/inst-lds-tiny.s b/llvm/test/MC/AVR/inst-lds-tiny.s
index 045cbea24b1e9..1cb8ac47a56d2 100644
--- a/llvm/test/MC/AVR/inst-lds-tiny.s
+++ b/llvm/test/MC/AVR/inst-lds-tiny.s
@@ -15,11 +15,8 @@ foo:
 ; CHECK:  lds r22, 44                  ; encoding: [0x6c,0xa2]
 ; CHECK:  lds r27, 92                  ; encoding: [0xbc,0xa5]
 ; CHECK:  lds r20, SYMBOL+12           ; encoding: [0x40'A',0xa0'A']
-; CHECK:                               ; fixup A - offset: 0, value: SYMBOL+12, kind: fixup_lds_sts_16
 ; CHECK:  lds r20, r21                 ; encoding: [0x40'A',0xa0'A']
-; CHECK:                               ; fixup A - offset: 0, value: r21, kind: fixup_lds_sts_16
 ; CHECK:  lds r20, z+6                 ; encoding: [0x40'A',0xa0'A']
-; CHECK:                               ; fixup A - offset: 0, value: z+6, kind: fixup_lds_sts_16
 
 ; CHECK-INST: lds r16, 113
 ; CHECK-INST: lds r29, 62
diff --git a/llvm/test/MC/AVR/inst-lds.s b/llvm/test/MC/AVR/inst-lds.s
index 38f641a07ca79..0d2e20685c702 100644
--- a/llvm/test/MC/AVR/inst-lds.s
+++ b/llvm/test/MC/AVR/inst-lds.s
@@ -15,11 +15,8 @@ foo:
 ; CHECK: lds r22, 172                 ; encoding: [0x60,0x91,0xac,0x00]
 ; CHECK: lds r27, 92                  ; encoding: [0xb0,0x91,0x5c,0x00]
 ; CHECK: lds r4,  SYMBOL+12           ; encoding: [0x40,0x90,A,A]
-; CHECK:                              ;   fixup A - offset: 2, value: SYMBOL+12, kind: fixup_16
 ; CHECK: lds r4,  r25                 ; encoding: [0x40,0x90,A,A]
-; CHECK:                              ;   fixup A - offset: 2, value: r25, kind: fixup_16
 ; CHECK: lds r4,  x+2                 ; encoding: [0x40,0x90,A,A]
-; CHECK:                              ;   fixup A - offset: 2, value: x+2, kind: fixup_16
 
 ; CHECK-INST: lds r16, 241
 ; CHECK-INST: lds r29, 190
diff --git a/llvm/test/MC/AVR/inst-ori.s b/llvm/test/MC/AVR/inst-ori.s
index bdbe510b1504a..8406656a79021 100644
--- a/llvm/test/MC/AVR/inst-ori.s
+++ b/llvm/test/MC/AVR/inst-ori.s
@@ -15,7 +15,6 @@ foo:
 ; CHECK: ori r31, 0             ; encoding: [0xf0,0x60]
 
 ; CHECK: ori r16, FOOBAR        ; encoding: [A,0x60]
-; CHECK:                        ;   fixup A - offset: 0, value: FOOBAR, kind: fixup_ldi
 
 ; CHECK-INST: ori r17, 208
 ; CHECK-INST: ori r24, 190
diff --git a/llvm/test/MC/AVR/inst-out.s b/llvm/test/MC/AVR/inst-out.s
index 76dac1d578be5..8c84918f8bb07 100644
--- a/llvm/test/MC/AVR/inst-out.s
+++ b/llvm/test/MC/AVR/inst-out.s
@@ -21,7 +21,6 @@ foo:
 ; CHECK: out 63, r31                 ; encoding: [0xff,0xbf]
 
 ; CHECK: out bar-8, r29              ; encoding: [0xd0'A',0xb9'A']
-; CHECK:                             ;   fixup A - offset: 0, value: bar-8, kind: fixup_port6
 
 ; CHECK-INST: out 4,  r2
 ; CHECK-INST: out 6,  r9
diff --git a/llvm/test/MC/AVR/inst-rcall.s b/llvm/test/MC/AVR/inst-rcall.s
index d0a9e6b7b0463..1da6e7f5dddf9 100644
--- a/llvm/test/MC/AVR/inst-rcall.s
+++ b/llvm/test/MC/AVR/inst-rcall.s
@@ -12,13 +12,9 @@ foo:
   .short  0xdfea
 
 ; CHECK: rcall .Ltmp0+0+2   ; encoding: [A,0b1101AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp0+0+2, kind: fixup_13_pcrel
 ; CHECK: rcall .Ltmp1-8+2   ; encoding: [A,0b1101AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp1-8+2, kind: fixup_13_pcrel
 ; CHECK: rcall .Ltmp2+12+2  ; encoding: [A,0b1101AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp2+12+2, kind: fixup_13_pcrel
 ; CHECK: rcall .Ltmp3+46+2  ; encoding: [A,0b1101AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp3+46+2, kind: fixup_13_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 00 d0    rcall .+0
diff --git a/llvm/test/MC/AVR/inst-rjmp.s b/llvm/test/MC/AVR/inst-rjmp.s
index 8971ff7ddcd8a..6712319bbc268 100644
--- a/llvm/test/MC/AVR/inst-rjmp.s
+++ b/llvm/test/MC/AVR/inst-rjmp.s
@@ -22,25 +22,15 @@ x:
   rjmp .+4094
 
 ; CHECK: rjmp .Ltmp0+2+2    ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp0+2+2, kind: fixup_13_pcrel
 ; CHECK: rjmp .Ltmp1-2+2    ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp1-2+2, kind: fixup_13_pcrel
 ; CHECK: rjmp foo             ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: foo, kind: fixup_13_pcrel
 ; CHECK: rjmp .Ltmp2+8+2    ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp2+8+2, kind: fixup_13_pcrel
 ; CHECK: rjmp end             ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: end, kind: fixup_13_pcrel
 ; CHECK: rjmp .Ltmp3+0+2    ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp3+0+2, kind: fixup_13_pcrel
 ; CHECK: rjmp .Ltmp4-4+2    ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp4-4+2, kind: fixup_13_pcrel
 ; CHECK: rjmp .Ltmp5-6+2    ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp5-6+2, kind: fixup_13_pcrel
 ; CHECK: rjmp x               ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: x, kind: fixup_13_pcrel
 ; CHECK: rjmp .Ltmp6+4094+2 ; encoding: [A,0b1100AAAA]
-; CHECK-NEXT:                 ;   fixup A - offset: 0, value: .Ltmp6+4094+2, kind: fixup_13_pcrel
 
 ; INST-LABEL: <foo>:
 ; INST-NEXT: 01 c0      rjmp  .+2
diff --git a/llvm/test/MC/AVR/inst-sbci.s b/llvm/test/MC/AVR/inst-sbci.s
index 3330294f768d8..a45cfe0ccd168 100644
--- a/llvm/test/MC/AVR/inst-sbci.s
+++ b/llvm/test/MC/AVR/inst-sbci.s
@@ -14,7 +14,6 @@ foo:
 ; CHECK: sbci r19, 16                 ; encoding: [0x30,0x41]
 
 ; CHECK: sbci r22, FOO                ; encoding: [0x60'A',0x40]
-; CHECK:                              ;   fixup A - offset: 0, value: FOO, kind: fixup_ldi
 
 ; CHECK-INST: sbci r17, 21
 ; CHECK-INST: sbci r23, 196
diff --git a/llvm/test/MC/AVR/inst-sbi.s b/llvm/test/MC/AVR/inst-sbi.s
index ef026efdebc6c..85f4832269df7 100644
--- a/llvm/test/MC/AVR/inst-sbi.s
+++ b/llvm/test/MC/AVR/inst-sbi.s
@@ -21,7 +21,6 @@ foo:
 ; CHECK: sbi 31, 7                 ; encoding: [0xff,0x9a]
 
 ; CHECK: sbi main, 0               ; encoding: [0bAAAAA000,0x9a]
-; CHECK:                           ;   fixup A - offset: 0, value: main, kind: fixup_port5
 
 ; CHECK-INST: sbi 3, 5
 ; CHECK-INST: sbi 1, 1
diff --git a/llvm/test/MC/AVR/inst-sbic.s b/llvm/test/MC/AVR/inst-sbic.s
index 1e8abdb29228d..b0576c9c4029c 100644
--- a/llvm/test/MC/AVR/inst-sbic.s
+++ b/llvm/test/MC/AVR/inst-sbic.s
@@ -21,7 +21,6 @@ foo:
 ; CHECK: sbic 31, 7                  ; encoding: [0xff,0x99]
 
 ; CHECK: sbic foo+1, 1               ; encoding: [0bAAAAA001,0x99]
-; CHECK:                             ;   fixup A - offset: 0, value: foo+1, kind: fixup_port5
 
 ; CHECK-INST: sbic 4,  3
 ; CHECK-INST: sbic 6,  2
diff --git a/llvm/test/MC/AVR/inst-sbis.s b/llvm/test/MC/AVR/inst-sbis.s
index 96996259ed2e5..5ce26120d2790 100644
--- a/llvm/test/MC/AVR/inst-sbis.s
+++ b/llvm/test/MC/AVR/inst-sbis.s
@@ -21,7 +21,6 @@ foo:
 ; CHECK: sbis 31, 7                  ; encoding: [0xff,0x9b]
 
 ; CHECK: sbis FOO+4, 7               ; encoding: [0bAAAAA111,0x9b]
-; CHECK:                             ;   fixup A - offset: 0, value: FOO+4, kind: fixup_port5
 
 ; CHECK-INST: sbis 4,  3
 ; CHECK-INST: sbis 6,  2
diff --git a/llvm/test/MC/AVR/inst-sbiw.s b/llvm/test/MC/AVR/inst-sbiw.s
index 640f503c62aea..aaebe71a5af79 100644
--- a/llvm/test/MC/AVR/inst-sbiw.s
+++ b/llvm/test/MC/AVR/inst-sbiw.s
@@ -30,9 +30,7 @@ foo:
 ; CHECK: sbiw r24,  2                  ; encoding: [0x02,0x97]
 
 ; CHECK: sbiw    r24, SYMBOL-1         ; encoding: [0b00AAAAAA,0x97]
-                                       ;   fixup A - offset: 0, value: SYMBOL-1, kind: fixup_6_adiw
 ; CHECK: sbiw    r24, z+15             ; encoding: [0b00AAAAAA,0x97]
-                                       ;   fixup A - offset: 0, value: z+15, kind: fixup_6_adiw
 
 ; CHECK-INST: sbiw r26, 54
 ; CHECK-INST: sbiw r26, 63
diff --git a/llvm/test/MC/AVR/inst-sbr.s b/llvm/test/MC/AVR/inst-sbr.s
index 9f946e2cc0c86..4e1daa5383f5c 100644
--- a/llvm/test/MC/AVR/inst-sbr.s
+++ b/llvm/test/MC/AVR/inst-sbr.s
@@ -15,7 +15,6 @@ foo:
 ; CHECK: ori r31, 0                    ; encoding: [0xf0,0x60]
 
 ; CHECK: ori     r19, _start           ; encoding: [0x30'A',0x60]
-; CHECK:                               ;   fixup A - offset: 0, value: _start, kind: fixup_ldi
 
 ; CHECK-INST: ori r17, 208
 ; CHECK-INST: ori r24, 190
diff --git a/llvm/test/MC/AVR/inst-std.s b/llvm/test/MC/AVR/inst-std.s
index ca817fa377224..fdbbeb0b17ce3 100644
--- a/llvm/test/MC/AVR/inst-std.s
+++ b/llvm/test/MC/AVR/inst-std.s
@@ -18,7 +18,6 @@ foo:
 ; CHECK: std Z+30, r7                 ; encoding: [0x76,0x8e]
 
 ; CHECK: std Y+foo, r9                ; encoding: [0x98'A',0x82'A']
-; CHECK:                              ;   fixup A - offset: 0, value: +foo, kind: fixup_6
 
 ; INST: std Y+2, r2
 ; INST: std Y+0, r0
diff --git a/llvm/test/MC/AVR/inst-sts-tiny.s b/llvm/test/MC/AVR/inst-sts-tiny.s
index a0d67c7177fbe..95051a23242fe 100644
--- a/llvm/test/MC/AVR/inst-sts-tiny.s
+++ b/llvm/test/MC/AVR/inst-sts-tiny.s
@@ -11,11 +11,8 @@ foo:
 ; CHECK:  sts 3,        r16           ; encoding: [0x03,0xa8]
 ; CHECK:  sts 127,      r17           ; encoding: [0x1f,0xaf]
 ; CHECK:  sts SYMBOL+1, r25           ; encoding: [0x90'A',0xa8'A']
-; CHECK:                              ; fixup A - offset: 0, value: SYMBOL+1, kind: fixup_lds_sts_16
 ; CHECK:  sts x,        r25           ; encoding: [0x90'A',0xa8'A']
-; CHECK:                              ; fixup A - offset: 0, value: x, kind: fixup_lds_sts_16
 ; CHECK:  sts r25+1,    r25           ; encoding: [0x90'A',0xa8'A']
-; CHECK:                              ; fixup A - offset: 0, value: r25+1, kind: fixup_lds_sts_16
 
 ; CHECK-INST: sts 3,   r16
 ; CHECK-INST: sts 127, r17
diff --git a/llvm/test/MC/AVR/inst-sts.s b/llvm/test/MC/AVR/inst-sts.s
index f7bc3e9cd37b0..0b575342baa25 100644
--- a/llvm/test/MC/AVR/inst-sts.s
+++ b/llvm/test/MC/AVR/inst-sts.s
@@ -11,11 +11,8 @@ foo:
 ; CHECK:  sts 3,        r5            ; encoding: [0x50,0x92,0x03,0x00]
 ; CHECK:  sts 255,      r7            ; encoding: [0x70,0x92,0xff,0x00]
 ; CHECK:  sts SYMBOL+1, r25           ; encoding: [0x90,0x93,A,A]
-; CHECK:                              ;   fixup A - offset: 2, value: SYMBOL+1, kind: fixup_16
 ; CHECK:  sts r25,      r25           ; encoding: [0x90,0x93,A,A]
-; CHECK:                              ;   fixup A - offset: 2, value: r25, kind: fixup_16
 ; CHECK:  sts y+3,      r25           ; encoding: [0x90,0x93,A,A]
-; CHECK:                              ;   fixup A - offset: 2, value: y+3, kind: fixup_16
 
 
 ; CHECK-INST: sts 3,   r5
diff --git a/llvm/test/MC/AVR/inst-subi.s b/llvm/test/MC/AVR/inst-subi.s
index 478d229a3e0c9..08aa1187d8f8e 100644
--- a/llvm/test/MC/AVR/inst-subi.s
+++ b/llvm/test/MC/AVR/inst-subi.s
@@ -15,7 +15,6 @@ foo:
 ; CHECK: subi r16, 144                 ; encoding: [0x00,0x59]
 
 ; CHECK: subi    r20, EXTERN_SYMBOL+0  ; encoding: [0x40'A',0x50]
-; CHECK:                               ;   fixup A - offset: 0, value: EXTERN_SYMBOL+0, kind: fixup_ldi
 
 ; CHECK-INST: subi r22, 82
 ; CHECK-INST: subi r27, 39
diff --git a/llvm/test/MC/AVR/modifiers.s b/llvm/test/MC/AVR/modifiers.s
index 51b3ad129f732..58275911cd188 100644
--- a/llvm/test/MC/AVR/modifiers.s
+++ b/llvm/test/MC/AVR/modifiers.s
@@ -21,9 +21,7 @@ bar:
     ldi r24, hi8(bar)
 
 ; CHECK: ldi  r24, lo8(bar)         ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: lo8(bar), kind: fixup_lo8_ldi
 ; CHECK: ldi  r24, hi8(bar)         ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hi8(bar), kind: fixup_hi8_ldi
 
 lo8:
     ldi r24, lo8(0x0815)
@@ -32,9 +30,7 @@ lo8:
 
 ; CHECK: ldi  r24, lo8(2069)        ; encoding: [0x85,0xe1]
 ; CHECK: ldi  r24, lo8(foo)         ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: lo8(foo), kind: fixup_lo8_ldi
 ; CHECK: ldi  r24, lo8(bar+5)       ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: lo8(bar+5), kind: fixup_lo8_ldi
 
 lo8_neg:
     ldi r24, lo8(-(123456))
@@ -42,7 +38,6 @@ lo8_neg:
 
 ; CHECK: ldi  r24, lo8(-(123456))   ; encoding: [0x80,0xec]
 ; CHECK: ldi  r24, lo8(-(foo))      ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: lo8(-(foo)), kind: fixup_lo8_ldi_neg
 
 hi8:
     ldi r24, hi8(0x0815)
@@ -51,9 +46,7 @@ hi8:
 
 ; CHECK: ldi  r24, hi8(2069)        ; encoding: [0x88,0xe0]
 ; CHECK: ldi  r24, hi8(foo)         ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hi8(foo), kind: fixup_hi8_ldi
 ; CHECK: ldi  r24, hi8(bar+5)       ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hi8(bar+5), kind: fixup_hi8_ldi
 
 hi8_neg:
     ldi r24, hi8(-(123456))
@@ -61,7 +54,6 @@ hi8_neg:
 
 ; CHECK: ldi  r24, hi8(-(123456))   ; encoding: [0x8d,0xe1]
 ; CHECK: ldi  r24, hi8(-(foo))      ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hi8(-(foo)), kind: fixup_hi8_ldi_neg
 
 hh8:
     ldi r24, hh8(0x0815)
@@ -70,9 +62,7 @@ hh8:
 
 ; CHECK: ldi  r24, hh8(2069)        ; encoding: [0x80,0xe0]
 ; CHECK: ldi  r24, hh8(foo)         ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hh8(foo), kind: fixup_hh8_ldi
 ; CHECK: ldi  r24, hh8(bar+5)       ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hh8(bar+5), kind: fixup_hh8_ldi
 
 hh8_neg:
     ldi r24, hh8(-(123456))
@@ -80,7 +70,6 @@ hh8_neg:
 
 ; CHECK: ldi  r24, hh8(-(123456))     ; encoding: [0x8e,0xef]
 ; CHECK: ldi  r24, hh8(-(foo))        ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hh8(-(foo)), kind: fixup_hh8_ldi_neg
 
 hlo8: ; synonym with hh8() above, hence the... odd results
     ldi r24, hlo8(0x0815)
@@ -89,9 +78,7 @@ hlo8: ; synonym with hh8() above, hence the... odd results
 
 ; CHECK: ldi  r24, hh8(2069)        ; encoding: [0x80,0xe0]
 ; CHECK: ldi  r24, hh8(foo)         ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hh8(foo), kind: fixup_hh8_ldi
 ; CHECK: ldi  r24, hh8(bar+5)       ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hh8(bar+5), kind: fixup_hh8_ldi
 
 hlo8_neg:
     ldi r24, hlo8(-(123456))
@@ -99,7 +86,6 @@ hlo8_neg:
 
 ; CHECK: ldi  r24, hh8(-(123456))  ; encoding: [0x8e,0xef]
 ; CHECK: ldi  r24, hh8(-(foo))     ; encoding: [0x80'A',0xe0]
-; CHECK:                           ; fixup A - offset: 0, value: hh8(-(foo)), kind: fixup_hh8_ldi_neg
 
 hhi8:
     ldi r24, hhi8(0x0815)
@@ -108,9 +94,7 @@ hhi8:
 
 ; CHECK: ldi  r24, hhi8(2069)       ; encoding: [0x80,0xe0]
 ; CHECK: ldi  r24, hhi8(foo)        ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hhi8(foo), kind: fixup_ms8_ldi
 ; CHECK: ldi  r24, hhi8(bar+5)      ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hhi8(bar+5), kind: fixup_ms8_ldi
 
 hhi8_neg:
     ldi r24, hhi8(-(123456))
@@ -119,7 +103,6 @@ hhi8_neg:
 
 ; CHECK: ldi  r24, hhi8(-(123456))  ; encoding: [0x8f,0xef]
 ; CHECK: ldi  r24, hhi8(-(foo))     ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: hhi8(-(foo)), kind: fixup_ms8_ldi_neg
 
 pm_lo8:
     ldi r24, pm_lo8(0x0815)
@@ -128,9 +111,7 @@ pm_lo8:
 
 ; CHECK: ldi  r24, pm_lo8(2069)     ; encoding: [0x8a,0xe0]
 ; CHECK: ldi  r24, pm_lo8(foo)      ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: pm_lo8(foo), kind: fixup_lo8_ldi_pm
 ; CHECK: ldi  r24, pm_lo8(bar+5)    ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: pm_lo8(bar+5), kind: fixup_lo8_ldi_pm
 
 pm_hi8:
     ldi r24, pm_hi8(0x0815)
@@ -139,9 +120,7 @@ pm_hi8:
 
 ; CHECK: ldi  r24, pm_hi8(2069)     ; encoding: [0x84,0xe0]
 ; CHECK: ldi  r24, pm_hi8(foo)      ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: pm_hi8(foo), kind: fixup_hi8_ldi_pm
 ; CHECK: ldi  r24, pm_hi8(bar+5)    ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: pm_hi8(bar+5), kind: fixup_hi8_ldi_pm
 
 pm_hh8:
     ldi r24, pm_hh8(0x0815)
@@ -150,9 +129,7 @@ pm_hh8:
 
 ; CHECK: ldi  r24, pm_hh8(2069)     ; encoding: [0x80,0xe0]
 ; CHECK: ldi  r24, pm_hh8(foo)      ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: pm_hh8(foo), kind: fixup_hh8_ldi_pm
 ; CHECK: ldi  r24, pm_hh8(bar+5)    ; encoding: [0x80'A',0xe0]
-; CHECK:                            ; fixup A - offset: 0, value: pm_hh8(bar+5), kind: fixup_hh8_ldi_pm
 
 pm_lo8_neg:
     ldi r24, pm_lo8(-(0x0815))
@@ -161,9 +138,7 @@ pm_lo8_neg:
 
 ; CHECK: ldi  r24, pm_lo8(-(2069))   ; encoding: [0x85,0xef]
 ; CHECK: ldi  r24, pm_lo8(-(foo))    ; encoding: [0x80'A',0xe0]
-; CHECK:                             ; fixup A - offset: 0, value: pm_lo8(-(foo)), kind: fixup_lo8_ldi_pm_neg
 ; CHECK: ldi  r24, pm_lo8(-(bar+5))  ; encoding: [0x80'A',0xe0]
-; CHECK:                             ; fixup A - offset: 0, value: pm_lo8(-(bar+5)), kind: fixup_lo8_ldi_pm_neg
 
 pm_hi8_neg:
     ldi r24, pm_hi8(-(0x0815))
@@ -172,9 +147,7 @@ pm_hi8_neg:
 
 ; CHECK: ldi  r24, pm_hi8(-(2069))   ; encoding: [0x8b,0xef]
 ; CHECK: ldi  r24, pm_hi8(-(foo))    ; encoding: [0x80'A',0xe0]
-; CHECK:                             ; fixup A - offset: 0, value: pm_hi8(-(foo)), kind: fixup_hi8_ldi_pm_neg
 ; CHECK: ldi  r24, pm_hi8(-(bar+5))  ; encoding: [0x80'A',0xe0]
-; CHECK:                             ; fixup A - offset: 0, value: pm_hi8(-(bar+5)), kind: fixup_hi8_ldi_pm_neg
 
 pm_hh8_neg:
     ldi r24, pm_hh8(-(0x0815))
@@ -183,6 +156,4 @@ pm_hh8_neg:
 
 ; CHECK: ldi  r24, pm_hh8(-(2069))   ; encoding: [0x8f,0xef]
 ; CHECK: ldi  r24, pm_hh8(-(foo))    ; encoding: [0x80'A',0xe0]
-; CHECK:                             ; fixup A - offset: 0, value: pm_hh8(-(foo)), kind: fixup_hh8_ldi_pm_neg
 ; CHECK: ldi  r24, pm_hh8(-(bar+5))  ; encoding: [0x80'A',0xe0]
-; CHECK:                             ; fixup A - offset: 0, value: pm_hh8(-(bar+5)), kind: fixup_hh8_ldi_pm_neg
diff --git a/llvm/test/MC/CSKY/basic-16bit.s b/llvm/test/MC/CSKY/basic-16bit.s
index e341a1d1c31e0..3799cbac43fa8 100644
--- a/llvm/test/MC/CSKY/basic-16bit.s
+++ b/llvm/test/MC/CSKY/basic-16bit.s
@@ -202,19 +202,16 @@ tstnbz16 a3
 
 # CHECK-ASM: br16 .L.test
 # CHECK-ASM: encoding: [A,0x04'A']
-# CHECK-ASM: fixup A - offset: 0, value: .L.test, kind: fixup_csky_pcrel_imm10_scale2
 .L.test:
 br16 .L.test
 
 # CHECK-ASM: bt16 .L.test2
 # CHECK-ASM: encoding: [A,0x08'A']
-# CHECK-ASM: fixup A - offset: 0, value: .L.test2, kind: fixup_csky_pcrel_imm10_scale2
 .L.test2:
 bt16 .L.test2
 
 # CHECK-ASM: bf16 .L.test3
 # CHECK-ASM: encoding: [A,0x0c'A']
-# CHECK-ASM: fixup A - offset: 0, value: .L.test3, kind: fixup_csky_pcrel_imm10_scale2
 .L.test3:
 bf16 .L.test3
 
@@ -228,7 +225,6 @@ jsr16 a3
 
 # CHECK-ASM: lrw16 a0, [.L.test14]
 # CHECK-ASM: encoding: [A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test14, kind: fixup_csky_pcrel_uimm7_scale4
 .L.test14:
 lrw16 a0, [.L.test14]
 
diff --git a/llvm/test/MC/CSKY/basic.s b/llvm/test/MC/CSKY/basic.s
index 3a8d3868f2957..fd4bb667a0058 100644
--- a/llvm/test/MC/CSKY/basic.s
+++ b/llvm/test/MC/CSKY/basic.s
@@ -476,55 +476,46 @@ doze32
 
 # CHECK-ASM: br32 .L.test
 # CHECK-ASM: encoding: [A,0xe8'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test, kind: fixup_csky_pcrel_imm16_scale2
 .L.test:
 br32 .L.test
 
 # CHECK-ASM: bt32 .L.test2
 # CHECK-ASM: encoding: [0x60'A',0xe8'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test2, kind: fixup_csky_pcrel_imm16_scale2
 .L.test2:
 bt32 .L.test2
 
 # CHECK-ASM: bf32 .L.test3
 # CHECK-ASM: encoding: [0x40'A',0xe8'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test3, kind: fixup_csky_pcrel_imm16_scale2
 .L.test3:
 bf32 .L.test3
 
 # CHECK-ASM: bez32 a0, .L.test4
 # CHECK-ASM: encoding: [A,0xe9'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test4, kind: fixup_csky_pcrel_imm16_scale2
 .L.test4:
 bez32 a0, .L.test4
 
 # CHECK-ASM: bnez32 a0, .L.test5
 # CHECK-ASM: encoding: [0x20'A',0xe9'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test5, kind: fixup_csky_pcrel_imm16_scale2
 .L.test5:
 bnez32 a0, .L.test5
 
 # CHECK-ASM: bhz32 a0, .L.test6
 # CHECK-ASM: encoding: [0x40'A',0xe9'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test6, kind: fixup_csky_pcrel_imm16_scale2
 .L.test6:
 bhz32 a0, .L.test6
 
 # CHECK-ASM: blsz32 a0, .L.test7
 # CHECK-ASM: encoding: [0x60'A',0xe9'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test7, kind: fixup_csky_pcrel_imm16_scale2
 .L.test7:
 blsz32 a0, .L.test7
 
 # CHECK-ASM: blz32 a0, .L.test8
 # CHECK-ASM: encoding: [0x80'A',0xe9'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test8, kind: fixup_csky_pcrel_imm16_scale2
 .L.test8:
 blz32 a0, .L.test8
 
 # CHECK-ASM: bhsz32 a0, .L.test9
 # CHECK-ASM: encoding: [0xa0'A',0xe9'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test9, kind: fixup_csky_pcrel_imm16_scale2
 .L.test9:
 bhsz32 a0, .L.test9
 
@@ -534,13 +525,11 @@ jmp32 a3
 
 # CHECK-ASM: jmpi32 [.L.test10]
 # CHECK-ASM: encoding: [0xc0'A',0xea'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test10, kind: fixup_csky_pcrel_uimm16_scale4
 .L.test10:
 jmpi32 [.L.test10]
 
 # CHECK-ASM: bsr32 .L.test11
 # CHECK-ASM: encoding: [A,0xe0'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test11, kind: fixup_csky_pcrel_imm26_scale2
 .L.test11:
 bsr32 .L.test11
 
@@ -550,19 +539,16 @@ jsr32 a3
 
 # CHECK-ASM: jsri32 [.L.test12]
 # CHECK-ASM: encoding: [0xe0'A',0xea'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test12, kind: fixup_csky_pcrel_uimm16_scale4
 .L.test12:
 jsri32 [.L.test12]
 
 # CHECK-ASM: grs32 a0, .L.test13
 # CHECK-ASM: encoding: [0x0c'A',0xcc'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test13, kind: fixup_csky_pcrel_imm18_scale2
 .L.test13:
 grs32 a0, .L.test13
 
 # CHECK-ASM: lrw32 a0, [.L.test14]
 # CHECK-ASM: encoding: [0x80'A',0xea'A',A,A]
-# CHECK-ASM: fixup A - offset: 0, value: .L.test14, kind: fixup_csky_pcrel_uimm16_scale4
 .L.test14:
 lrw32 a0, [.L.test14]
 
diff --git a/llvm/test/MC/Sparc/sparc-assembly-exprs.s b/llvm/test/MC/Sparc/sparc-assembly-exprs.s
index e950f1635ab3a..e2b877ac93e5b 100644
--- a/llvm/test/MC/Sparc/sparc-assembly-exprs.s
+++ b/llvm/test/MC/Sparc/sparc-assembly-exprs.s
@@ -7,33 +7,27 @@
         mov      ((12+3)<<2), %o2
 
         ! CHECK:   ba      symStart+4           ! encoding: [0x10,0b10AAAAAA,A,A]
-        ! CHECK:   fixup A - offset: 0, value: symStart+4, kind: fixup_sparc_br22
         ! OBJDUMP: ba    1
 symStart:
         b        symStart + 4
 
         ! CHECK:   mov     symEnd-symStart, %g1 ! encoding: [0x82,0x10,0b001AAAAA,A]
-        ! CHECK:   fixup A - offset: 0, value: symEnd-symStart, kind: fixup_sparc_13
         ! OBJDUMP: mov	   24, %g1
         mov      symEnd - symStart, %g1
 
         ! CHECK:   sethi %hi(sym+10), %g2       ! encoding: [0x05,0b00AAAAAA,A,A]
-        ! CHECK:   fixup A - offset: 0, value: %hi(sym+10), kind: fixup_sparc_hi22
         ! OBJDUMP: R_SPARC_HI22	sym+0xa
         sethi    %hi(sym + 10), %g2
 
         ! CHECK:   call foo+40                  ! encoding: [0b01AAAAAA,A,A,A]
-        ! CHECK:   fixup A - offset: 0, value: foo+40, kind: fixup_sparc_call30
         ! OBJDUMP: R_SPARC_WDISP30 foo+0x28
         call     foo + 40
 
         ! CHECK:   add %g1, val+100, %g1        ! encoding: [0x82,0x00,0b011AAAAA,A]
-        ! CHECK:   fixup A - offset: 0, value: val+100, kind: fixup_sparc_13
         ! OBJDUMP: R_SPARC_13 val+0x64
         add      %g1, val + 100, %g1
 
         ! CHECK:   add %g1, 100+val, %g2        ! encoding: [0x84,0x00,0b011AAAAA,A]
-        ! CHECK:   fixup A - offset: 0, value: 100+val, kind: fixup_sparc_13
         ! OBJDUMP: R_SPARC_13	val+0x64
         add      %g1, 100 + val, %g2
 symEnd:
diff --git a/llvm/test/MC/Sparc/sparc-ctrl-instructions.s b/llvm/test/MC/Sparc/sparc-ctrl-instructions.s
index 05a691de4f966..9cb83ecf4456d 100644
--- a/llvm/test/MC/Sparc/sparc-ctrl-instructions.s
+++ b/llvm/test/MC/Sparc/sparc-ctrl-instructions.s
@@ -2,11 +2,9 @@
 ! RUN: llvm-mc %s -triple=sparcv9 -show-encoding | FileCheck %s
 
         ! CHECK: call foo     ! encoding: [0b01AAAAAA,A,A,A]
-        ! CHECK:              !   fixup A - offset: 0, value: foo, kind: fixup_sparc_call30
         call foo
 
         ! CHECK: call foo, 0  ! encoding: [0b01AAAAAA,A,A,A]
-        ! CHECK:              !   fixup A - offset: 0, value: foo, kind: fixup_sparc_call30
         call foo, 0
 
         ! CHECK: call %g1+%i2 ! encoding: [0x9f,0xc0,0x40,0x1a]
@@ -28,11 +26,9 @@
         call %g1, 3
 
         ! CHECK: call %g1+%lo(sym)   ! encoding: [0x9f,0xc0,0b011000AA,A]
-        ! CHECK-NEXT:                ! fixup A - offset: 0, value: %lo(sym), kind: fixup_sparc_lo10
         call %g1+%lo(sym)
 
         ! CHECK: call %g1+%lo(sym), 4   ! encoding: [0x9f,0xc0,0b011000AA,A]
-        ! CHECK-NEXT:                ! fixup A - offset: 0, value: %lo(sym), kind: fixup_sparc_lo10
         call %g1+%lo(sym), 4
 
         ! CHECK-LABEL: .Ltmp0:
@@ -53,7 +49,6 @@
         jmp %g1
 
         ! CHECK: jmp %g1+%lo(sym)   ! encoding: [0x81,0xc0,0b011000AA,A]
-        ! CHECK-NEXT:                ! fixup A - offset: 0, value: %lo(sym), kind: fixup_sparc_lo10
         jmp %g1+%lo(sym)
 
         ! CHECK: jmp sym ! encoding: [0x81,0xc0,0b001AAAAA,A]
@@ -69,469 +64,353 @@
         jmpl %g1, %g2
 
         ! CHECK: jmpl %g1+%lo(sym), %g2   ! encoding: [0x85,0xc0,0b011000AA,A]
-        ! CHECK-NEXT:                     ! fixup A - offset: 0, value: %lo(sym), kind: fixup_sparc_lo10
         jmpl %g1+%lo(sym), %g2
 
         ! CHECK: ba .BB0      ! encoding: [0x10,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         ba .BB0
 
         ! CHECK: bne .BB0     ! encoding: [0x12,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bne .BB0
 
         ! CHECK: bne .BB0     ! encoding: [0x12,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bnz .BB0
 
         ! CHECK: be .BB0      ! encoding: [0x02,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         be .BB0
 
         ! CHECK: be .BB0      ! encoding: [0x02,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bz .BB0
 
         ! CHECK: be .BB0      ! encoding: [0x02,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         beq .BB0
 
         ! CHECK: bg .BB0      ! encoding: [0x14,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bg .BB0
 
         ! CHECK: bg .BB0      ! encoding: [0x14,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bgt .BB0
 
         ! CHECK: ble .BB0      ! encoding: [0x04,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         ble .BB0
 
         ! CHECK: bge .BB0      ! encoding: [0x16,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bge .BB0
 
         ! CHECK: bl .BB0      ! encoding: [0x06,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bl .BB0
 
         ! CHECK: bl .BB0      ! encoding: [0x06,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         blt .BB0
 
         ! CHECK: bgu .BB0      ! encoding: [0x18,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bgu .BB0
 
         ! CHECK: bleu .BB0      ! encoding: [0x08,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bleu .BB0
 
         ! CHECK: bcc .BB0      ! encoding: [0x1a,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bcc .BB0
 
         ! CHECK: bcc .BB0      ! encoding: [0x1a,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bgeu .BB0
 
         ! CHECK: bcs .BB0      ! encoding: [0x0a,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bcs .BB0
 
         ! CHECK: bcs .BB0      ! encoding: [0x0a,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         blu .BB0
 
         ! CHECK: bpos .BB0      ! encoding: [0x1c,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bpos .BB0
 
         ! CHECK: bneg .BB0      ! encoding: [0x0c,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bneg .BB0
 
         ! CHECK: bvc .BB0      ! encoding: [0x1e,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bvc .BB0
 
         ! CHECK: bvs .BB0      ! encoding: [0x0e,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bvs .BB0
 
         ! CHECK:             fba .BB0                        ! encoding: [0x11,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fba .BB0
 
         ! CHECK:             fba .BB0                        ! encoding: [0x11,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fb .BB0
 
         ! CHECK:             fbn .BB0                        ! encoding: [0x01,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbn .BB0
 
         ! CHECK:             fbu .BB0                        ! encoding: [0x0f,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbu .BB0
 
         ! CHECK:             fbg .BB0                        ! encoding: [0x0d,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbg .BB0
 
         ! CHECK:             fbug .BB0                       ! encoding: [0x0b,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbug .BB0
 
         ! CHECK:             fbl .BB0                        ! encoding: [0x09,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbl .BB0
 
         ! CHECK:             fbul .BB0                       ! encoding: [0x07,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbul .BB0
 
         ! CHECK:             fblg .BB0                       ! encoding: [0x05,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fblg .BB0
 
         ! CHECK:             fbne .BB0                       ! encoding: [0x03,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbne .BB0
 
         ! CHECK:             fbne .BB0                       ! encoding: [0x03,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbnz .BB0
 
         ! CHECK:             fbe .BB0                        ! encoding: [0x13,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbe .BB0
 
         ! CHECK:             fbe .BB0                        ! encoding: [0x13,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbz .BB0
 
         ! CHECK:             fbue .BB0                       ! encoding: [0x15,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbue .BB0
 
         ! CHECK:             fbge .BB0                       ! encoding: [0x17,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbge .BB0
 
         ! CHECK:             fbuge .BB0                      ! encoding: [0x19,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbuge .BB0
 
         ! CHECK:             fble .BB0                       ! encoding: [0x1b,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fble .BB0
 
         ! CHECK:             fbule .BB0                      ! encoding: [0x1d,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbule .BB0
 
         ! CHECK:             fbo .BB0                        ! encoding: [0x1f,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbo .BB0
         
         ! CHECK:             cba .BB0                        ! encoding: [0x11,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb .BB0
 
         ! CHECK:             cba .BB0                        ! encoding: [0x11,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cba .BB0
 
         ! CHECK:             cbn .BB0                        ! encoding: [0x01,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cbn .BB0
 
         ! CHECK:             cb3 .BB0                        ! encoding: [0x0f,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb3 .BB0
 
         ! CHECK:             cb2 .BB0                        ! encoding: [0x0d,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb2 .BB0
 
         ! CHECK:             cb23 .BB0                       ! encoding: [0x0b,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb23 .BB0
 
         ! CHECK:             cb1 .BB0                        ! encoding: [0x09,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb1 .BB0
 
         ! CHECK:             cb13 .BB0                       ! encoding: [0x07,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb13 .BB0
 
         ! CHECK:             cb12 .BB0                       ! encoding: [0x05,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb12 .BB0
 
         ! CHECK:             cb123 .BB0                      ! encoding: [0x03,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb123 .BB0
 
         ! CHECK:             cb0 .BB0                        ! encoding: [0x13,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb0 .BB0
 
         ! CHECK:             cb03 .BB0                       ! encoding: [0x15,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb03 .BB0
 
         ! CHECK:             cb02 .BB0                       ! encoding: [0x17,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb02 .BB0
 
         ! CHECK:             cb023 .BB0                      ! encoding: [0x19,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb023 .BB0
 
         ! CHECK:             cb01 .BB0                       ! encoding: [0x1b,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb01 .BB0
 
         ! CHECK:             cb013 .BB0                      ! encoding: [0x1d,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb013 .BB0
 
         ! CHECK:             cb012 .BB0                      ! encoding: [0x1f,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb012 .BB0
 
         ! CHECK: ba,a .BB0    ! encoding: [0x30,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         ba,a .BB0
 
         ! CHECK: bne,a .BB0   ! encoding: [0x32,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bne,a .BB0
 
         ! CHECK: be,a .BB0    ! encoding: [0x22,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         be,a .BB0
 
         ! CHECK: bg,a .BB0    ! encoding: [0x34,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bg,a .BB0
 
         ! CHECK: ble,a .BB0   ! encoding: [0x24,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         ble,a .BB0
 
         ! CHECK: bge,a .BB0   ! encoding: [0x36,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bge,a .BB0
 
         ! CHECK: bl,a .BB0    ! encoding: [0x26,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bl,a .BB0
 
         ! CHECK: bgu,a .BB0   ! encoding: [0x38,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bgu,a .BB0
 
         ! CHECK: bleu,a .BB0  ! encoding: [0x28,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bleu,a .BB0
 
         ! CHECK: bcc,a .BB0   ! encoding: [0x3a,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bcc,a .BB0
 
         ! CHECK: bcs,a .BB0   ! encoding: [0x2a,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bcs,a .BB0
 
         ! CHECK: bpos,a .BB0  ! encoding: [0x3c,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bpos,a .BB0
 
         ! CHECK: bneg,a .BB0  ! encoding: [0x2c,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bneg,a .BB0
 
         ! CHECK: bvc,a .BB0   ! encoding: [0x3e,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bvc,a .BB0
 
         ! CHECK: bvs,a .BB0   ! encoding: [0x2e,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         bvs,a .BB0
 
         ! CHECK:             fbu,a .BB0                      ! encoding: [0x2f,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbu,a .BB0
 
         ! CHECK:             fbg,a .BB0                      ! encoding: [0x2d,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbg,a .BB0
         ! CHECK:             fbug,a .BB0                     ! encoding: [0x2b,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbug,a .BB0
 
         ! CHECK:             fbl,a .BB0                      ! encoding: [0x29,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbl,a .BB0
 
         ! CHECK:             fbul,a .BB0                     ! encoding: [0x27,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbul,a .BB0
 
         ! CHECK:             fblg,a .BB0                     ! encoding: [0x25,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fblg,a .BB0
 
         ! CHECK:             fbne,a .BB0                     ! encoding: [0x23,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbne,a .BB0
 
         ! CHECK:             fbe,a .BB0                      ! encoding: [0x33,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbe,a .BB0
 
         ! CHECK:             fbue,a .BB0                     ! encoding: [0x35,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbue,a .BB0
 
         ! CHECK:             fbge,a .BB0                     ! encoding: [0x37,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbge,a .BB0
 
         ! CHECK:             fbuge,a .BB0                    ! encoding: [0x39,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbuge,a .BB0
 
         ! CHECK:             fble,a .BB0                     ! encoding: [0x3b,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fble,a .BB0
 
         ! CHECK:             fbule,a .BB0                    ! encoding: [0x3d,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbule,a .BB0
 
         ! CHECK:             fbo,a .BB0                      ! encoding: [0x3f,0b10AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         fbo,a .BB0
 
         ! CHECK:  rett %i7+8   ! encoding: [0x81,0xcf,0xe0,0x08]
         rett %i7 + 8
 
         ! CHECK:             cb3,a .BB0                      ! encoding: [0x2f,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb3,a .BB0
 
         ! CHECK:             cb2,a .BB0                      ! encoding: [0x2d,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb2,a .BB0
 
         ! CHECK:             cb23,a .BB0                     ! encoding: [0x2b,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb23,a .BB0
 
         ! CHECK:             cb1,a .BB0                      ! encoding: [0x29,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb1,a .BB0
 
         ! CHECK:             cb13,a .BB0                     ! encoding: [0x27,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb13,a .BB0
 
         ! CHECK:             cb12,a .BB0                     ! encoding: [0x25,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb12,a .BB0
 
         ! CHECK:             cb123,a .BB0                    ! encoding: [0x23,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb123,a .BB0
 
         ! CHECK:             cb0,a .BB0                      ! encoding: [0x33,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb0,a .BB0
 
         ! CHECK:             cb03,a .BB0                     ! encoding: [0x35,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb03,a .BB0
 
         ! CHECK:             cb02,a .BB0                     ! encoding: [0x37,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb02,a .BB0
 
         ! CHECK:             cb023,a .BB0                    ! encoding: [0x39,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb023,a .BB0
 
         ! CHECK:             cb01,a .BB0                     ! encoding: [0x3b,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb01,a .BB0
 
         ! CHECK:             cb013,a .BB0                    ! encoding: [0x3d,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb013,a .BB0
 
         ! CHECK:             cb012,a .BB0                    ! encoding: [0x3f,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb012,a .BB0
 
         ! CHECK:             cb3,a .BB0                      ! encoding: [0x2f,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb3,a .BB0
 
         ! CHECK:             cb2,a .BB0                      ! encoding: [0x2d,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb2,a .BB0
 
         ! CHECK:             cb23,a .BB0                     ! encoding: [0x2b,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb23,a .BB0
 
         ! CHECK:             cb1,a .BB0                      ! encoding: [0x29,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb1,a .BB0
 
         ! CHECK:             cb13,a .BB0                     ! encoding: [0x27,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb13,a .BB0
 
         ! CHECK:             cb12,a .BB0                     ! encoding: [0x25,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb12,a .BB0
 
         ! CHECK:             cb123,a .BB0                    ! encoding: [0x23,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb123,a .BB0
 
         ! CHECK:             cb0,a .BB0                      ! encoding: [0x33,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb0,a .BB0
 
         ! CHECK:             cb03,a .BB0                     ! encoding: [0x35,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb03,a .BB0
 
         ! CHECK:             cb02,a .BB0                     ! encoding: [0x37,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb02,a .BB0
 
         ! CHECK:             cb023,a .BB0                    ! encoding: [0x39,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb023,a .BB0
 
         ! CHECK:             cb01,a .BB0                     ! encoding: [0x3b,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb01,a .BB0
 
         ! CHECK:             cb013,a .BB0                    ! encoding: [0x3d,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb013,a .BB0
 
         ! CHECK:             cb012,a .BB0                    ! encoding: [0x3f,0b11AAAAAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         cb012,a .BB0
 
         ! CHECK:  rett %i7+8                                 ! encoding: [0x81,0xcf,0xe0,0x08]
diff --git a/llvm/test/MC/Sparc/sparc-little-endian.s b/llvm/test/MC/Sparc/sparc-little-endian.s
index e9a56eb7b964d..b0f650bfef9e7 100644
--- a/llvm/test/MC/Sparc/sparc-little-endian.s
+++ b/llvm/test/MC/Sparc/sparc-little-endian.s
@@ -13,6 +13,5 @@
         ! ...and that fixups are applied to the correct bytes.
 
         ! CHECK: ba .BB0      ! encoding: [A,A,0b10AAAAAA,0x10]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br22
         ! CHECK-OBJ: 4: ff ff bf 10  ba 4194303
         ba .BB0
diff --git a/llvm/test/MC/Sparc/sparc-relocations.s b/llvm/test/MC/Sparc/sparc-relocations.s
index 82314e440acf6..027164d77b256 100644
--- a/llvm/test/MC/Sparc/sparc-relocations.s
+++ b/llvm/test/MC/Sparc/sparc-relocations.s
@@ -24,75 +24,57 @@
         ! CHECK-OBJ-NEXT: }
 
         ! CHECK: call foo     ! encoding: [0b01AAAAAA,A,A,A]
-        ! CHECK:              !   fixup A - offset: 0, value: foo, kind: fixup_sparc_call30
         call foo
 
         ! CHECK: or %g1, %lo(sym), %g3 ! encoding: [0x86,0x10,0b011000AA,A]
-        ! CHECK-NEXT:                  !   fixup A - offset: 0, value: %lo(sym), kind: fixup_sparc_lo10
         or %g1, %lo(sym), %g3
 
         ! CHECK: sethi %hi(sym), %l0  ! encoding: [0x21,0b00AAAAAA,A,A]
-        ! CHECK-NEXT:                 !   fixup A - offset: 0, value: %hi(sym), kind: fixup_sparc_hi22
         sethi %hi(sym), %l0
 
         ! CHECK: sethi %h44(sym), %l0  ! encoding: [0x21,0b00AAAAAA,A,A]
-        ! CHECK-NEXT:                  !   fixup A - offset: 0, value: %h44(sym), kind: fixup_sparc_h44
         sethi %h44(sym), %l0
 
         ! CHECK: or %g1, %m44(sym), %g3 ! encoding: [0x86,0x10,0b011000AA,A]
-        ! CHECK-NEXT:                   !   fixup A - offset: 0, value: %m44(sym), kind: fixup_sparc_m44
         or %g1, %m44(sym), %g3
 
         ! CHECK: or %g1, %l44(sym), %g3 ! encoding: [0x86,0x10,0b0110AAAA,A]
-        ! CHECK-NEXT:                   !   fixup A - offset: 0, value: %l44(sym), kind: fixup_sparc_l44
         or %g1, %l44(sym), %g3
 
         ! CHECK: sethi %hh(sym), %l0  ! encoding: [0x21,0b00AAAAAA,A,A]
-        ! CHECK-NEXT:                 !   fixup A - offset: 0, value: %hh(sym), kind: fixup_sparc_hh
         sethi %hh(sym), %l0
 
         ! CHECK: sethi %hh(sym), %l0  ! encoding: [0x21,0b00AAAAAA,A,A]
-        ! CHECK-NEXT:                 !   fixup A - offset: 0, value: %hh(sym), kind: fixup_sparc_hh
         sethi %uhi(sym), %l0
 
         ! CHECK: or %g1, %hm(sym), %g3 ! encoding: [0x86,0x10,0b011000AA,A]
-        ! CHECK-NEXT:                  !   fixup A - offset: 0, value: %hm(sym), kind: fixup_sparc_hm
         or %g1, %hm(sym), %g3
 
         ! CHECK: or %g1, %hm(sym), %g3 ! encoding: [0x86,0x10,0b011000AA,A]
-        ! CHECK-NEXT:                  !   fixup A - offset: 0, value: %hm(sym), kind: fixup_sparc_hm
         or %g1, %ulo(sym), %g3
 
         ! CHECK: sethi %lm(sym), %l0  ! encoding: [0x21,0b00AAAAAA,A,A]
-        ! CHECK-NEXT:                 !   fixup A - offset: 0, value: %lm(sym), kind: fixup_sparc_lm
         sethi %lm(sym), %l0
 
         ! CHECK: or %g1, sym, %g3 ! encoding: [0x86,0x10,0b011AAAAA,A]
-        ! CHECK-NEXT:                  !   fixup A - offset: 0, value: sym, kind: fixup_sparc_13
         or %g1, sym, %g3
 
         ! CHECK: or %g1, sym+4, %g3 ! encoding: [0x86,0x10,0b011AAAAA,A]
-        ! CHECK-NEXT:                  ! fixup A - offset: 0, value: sym+4, kind: fixup_sparc_13
         or %g1, (sym+4), %g3
 
         ! CHECK: sethi %hix(sym), %g1 ! encoding: [0x03,0b00AAAAAA,A,A]
-        ! CHECK-NEXT:                 ! fixup A - offset: 0, value: %hix(sym), kind: fixup_sparc_hix22
         sethi %hix(sym), %g1
 
         ! CHECK: xor %g1, %lox(sym), %g1 ! encoding: [0x82,0x18,0b011AAAAA,A]
-        ! CHECK-NEXT:                    ! fixup A - offset: 0, value: %lox(sym), kind: fixup_sparc_lox10
         xor %g1, %lox(sym), %g1
 
         ! CHECK: sethi %gdop_hix22(sym), %l1 ! encoding: [0x23,0x00,0x00,0x00]
-        ! CHECK-NEXT:                        ! fixup A - offset: 0, value: %gdop_hix22(sym), kind: fixup_sparc_gotdata_hix22
         sethi %gdop_hix22(sym), %l1
 
         ! CHECK: or %l1, %gdop_lox10(sym), %l1 ! encoding: [0xa2,0x14,0x60,0x00]
-        ! CHECK-NEXT:                          ! fixup A - offset: 0, value: %gdop_lox10(sym), kind: fixup_sparc_gotdata_lox10
         or %l1, %gdop_lox10(sym), %l1
 
         ! CHECK: ldx [%l7+%l1], %l2, %gdop(sym) ! encoding: [0xe4,0x5d,0xc0,0x11]
-        ! CHECK-NEXT:                           ! fixup A - offset: 0, value: %gdop(sym), kind: fixup_sparc_gotdata_op
         ldx [%l7 + %l1], %l2, %gdop(sym)
 
         ! This test needs to placed last in the file
diff --git a/llvm/test/MC/Sparc/sparc-synthetic-instructions.s b/llvm/test/MC/Sparc/sparc-synthetic-instructions.s
index 5652f86841950..c5fa6030abb72 100644
--- a/llvm/test/MC/Sparc/sparc-synthetic-instructions.s
+++ b/llvm/test/MC/Sparc/sparc-synthetic-instructions.s
@@ -23,35 +23,27 @@
         save
 
         ! CHECK: sethi %hi(40000), %g1            ! encoding: [0x03,0b00AAAAAA,A,A]
-        ! CHECK:                                  !   fixup A - offset: 0, value: %hi(40000), kind: fixup_sparc_hi22
         ! CHECK: or %g1, %lo(40000), %g1          ! encoding: [0x82,0x10,0b011000AA,A]
-        ! CHECK:                                  !   fixup A - offset: 0, value: %lo(40000), kind: fixup_sparc_lo10
         set 40000, %g1
         ! CHECK: mov 1, %g1 ! encoding: [0x82,0x10,0x20,0x01]
         set 1, %g1
         ! CHECK: sethi %hi(32768), %g1            ! encoding: [0x03,0b00AAAAAA,A,A]
-        ! CHECK:                                  !   fixup A - offset: 0, value: %hi(32768), kind: fixup_sparc_hi22
         set 32768, %g1
 
         ! Expect a 'sethi' without an 'or'.
         ! CHECK: sethi %hi(268431360), %o1       ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! CHECK:                                 !   fixup A - offset: 0, value: %hi(268431360), kind: fixup_sparc_hi22
         set 0x0ffff000, %o1
 
         ! CHECK: sethi %hi(268433408), %o1       ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! CHECK:                                 !   fixup A - offset: 0, value: %hi(268433408), kind: fixup_sparc_hi22
         set 0x0ffff800, %o1
 
         ! This is the boundary case that uses the lowest of the 22 bits in sethi.
         ! CHECK: sethi %hi(268434432), %o1       ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! CHECK:                                 !   fixup A - offset: 0, value: %hi(268434432), kind: fixup_sparc_hi22
         set 0x0ffffc00, %o1
 
         ! Now the synthetic instruction becomes two instructions.
         ! CHECK: sethi %hi(2147483647), %o1      ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! CHECK:                                 !   fixup A - offset: 0, value: %hi(2147483647), kind: fixup_sparc_hi22
         ! CHECK: or %o1, %lo(2147483647), %o1    ! encoding: [0x92,0x12,0b011000AA,A]
-        ! CHECK:                                 !   fixup A - offset: 0, value: %lo(2147483647), kind: fixup_sparc_lo10
         set 2147483647, %o1
 
         ! CHECK: xnor %g1, %g0, %g2               ! encoding: [0x84,0x38,0x40,0x00]
@@ -174,42 +166,31 @@
 
         ! V8: mov        -1, %o1              ! encoding: [0x92,0x10,0x3f,0xff]
         ! V9: sethi %hi(-1), %o1              ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %hi(-1), kind: fixup_sparc_hi22
         ! V9: or %o1, %lo(-1), %o1            ! encoding: [0x92,0x12,0b011000AA,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %lo(-1), kind: fixup_sparc_lo10
         set 0xffffffff, %o1
 
         ! V8: mov        -2, %o1              ! encoding: [0x92,0x10,0x3f,0xfe]
         ! V9: sethi %hi(-2), %o1              ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %hi(-2), kind: fixup_sparc_hi22
         ! V9: or %o1, %lo(-2), %o1            ! encoding: [0x92,0x12,0b011000AA,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %lo(-2), kind: fixup_sparc_lo10
         set 0xfffffffe, %o1
 
         ! V8: mov        -16, %o1             ! encoding: [0x92,0x10,0x3f,0xf0]
         ! V9: sethi %hi(-16), %o1             ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %hi(-16), kind: fixup_sparc_hi22
         ! V9: or %o1, %lo(-16), %o1           ! encoding: [0x92,0x12,0b011000AA,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %lo(-16), kind: fixup_sparc_lo10
         set 0xfffffff0, %o1
 
         ! V8: mov        -256, %o1            ! encoding: [0x92,0x10,0x3f,0x00]
         ! V9: sethi %hi(-256), %o1            ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %hi(-256), kind: fixup_sparc_hi22
         ! V9: or %o1, %lo(-256), %o1          ! encoding: [0x92,0x12,0b011000AA,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %lo(-256), kind: fixup_sparc_lo10
         set 0xffffff00, %o1
 
         ! V8: mov        -4096, %o1           ! encoding: [0x92,0x10,0x30,0x00]
         ! V9: sethi %hi(-4096), %o1           ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %hi(-4096), kind: fixup_sparc_hi22
         set 0xfffff000, %o1
 
         ! These results are the same for V8 and V9, so this test could have
         ! been with the others that weren't segregated by architecture,
         ! but logically it belongs here as a boundary case.
         ! V8: sethi %hi(-8192), %o1           ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! V8:                                 !   fixup A - offset: 0, value: %hi(-8192), kind: fixup_sparc_hi22
         ! V9: sethi %hi(-8192), %o1           ! encoding: [0x13,0b00AAAAAA,A,A]
-        ! V9:                                 !   fixup A - offset: 0, value: %hi(-8192), kind: fixup_sparc_hi22
         set 0xffffe000, %o1
diff --git a/llvm/test/MC/Sparc/sparc-tls-relocations.s b/llvm/test/MC/Sparc/sparc-tls-relocations.s
index 85fe96006a152..8dc6e62baef5e 100644
--- a/llvm/test/MC/Sparc/sparc-tls-relocations.s
+++ b/llvm/test/MC/Sparc/sparc-tls-relocations.s
@@ -31,13 +31,11 @@ foo:
 ! OBJDUMP: {{[0-9,a-f]+}}:  31 00 00 00  sethi 0, %i0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LE_HIX22 Local
 ! ASM: sethi %tle_hix22(Local), %i0 ! encoding: [0x31,0x00,0x00,0x00]
-! ASM:                             !   fixup A - offset: 0, value: %tle_hix22(Local), kind: fixup_sparc_tls_le_hix22
         sethi %tle_hix22(Local), %i0
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  b0 1e 20 00  xor %i0, 0, %i0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LE_LOX10 Local
 ! ASM: xor %i0, %tle_lox10(Local), %i0 ! encoding: [0xb0,0x1e,0x20,0x00]
-! ASM:                                !   fixup A - offset: 0, value: %tle_lox10(Local), kind: fixup_sparc_tls_le_lox10
         xor %i0, %tle_lox10(Local), %i0
 
 
@@ -48,44 +46,37 @@ foo:
 ! OBJDUMP: {{[0-9,a-f]+}}:  33 00 00 00  sethi 0, %i1
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LDO_HIX22 Local
 ! ASM: sethi %tldo_hix22(Local), %i1 ! encoding: [0x33,0b00AAAAAA,A,A]
-! ASM:                              !   fixup A - offset: 0, value: %tldo_hix22(Local), kind: fixup_sparc_tls_ldo_hix22
         sethi %tldo_hix22(Local), %i1
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  35 00 00 00  sethi 0, %i2
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LDM_HI22 Local
 ! ASM: sethi %tldm_hi22(Local), %i2 ! encoding: [0x35,0b00AAAAAA,A,A]
-! ASM:                             !   fixup A - offset: 0, value: %tldm_hi22(Local), kind: fixup_sparc_tls_ldm_hi22
         sethi %tldm_hi22(Local), %i2
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  b4 06 a0 00  add %i2, 0, %i2
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LDM_LO10 Local
 ! ASM: add %i2, %tldm_lo10(Local), %i2 ! encoding: [0xb4,0x06,0b101000AA,A]
-! ASM:                                !   fixup A - offset: 0, value: %tldm_lo10(Local), kind: fixup_sparc_tls_ldm_lo10
         add %i2, %tldm_lo10(Local), %i2
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  90 06 00 1a add %i0, %i2, %o0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LDM_ADD Local
 ! ASM: add %i0, %i2, %o0, %tldm_add(Local) ! encoding: [0x90,0x06,0x00,0x1a]
-! ASM:                                    !   fixup A - offset: 0, value: %tldm_add(Local), kind: fixup_sparc_tls_ldm_add
 	add %i0, %i2, %o0, %tldm_add(Local)
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  b0 1e 60 00  xor %i1, 0, %i0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LDO_LOX10 Local
 ! ASM: xor %i1, %tldo_lox10(Local), %i0 ! encoding: [0xb0,0x1e,0b011000AA,A]
-! ASM:                                 !   fixup A - offset: 0, value: %tldo_lox10(Local), kind: fixup_sparc_tls_ldo_lox10
         xor %i1, %tldo_lox10(Local), %i0
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  40 00 00 00 call 0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LDM_CALL Local
 ! ASM: call __tls_get_addr, %tldm_call(Local) ! encoding: [0x40,0x00,0x00,0x00]
-! ASM:                                       !   fixup A - offset: 0, value: %tldm_call(Local), kind: fixup_sparc_tls_ldm_call
         call __tls_get_addr, %tldm_call(Local)
         nop
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  90 02 00 18 add %o0, %i0, %o0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_LDO_ADD Local
 ! ASM: add %o0, %i0, %o0, %tldo_add(Local) ! encoding: [0x90,0x02,0x00,0x18]
-! ASM:                                    !   fixup A - offset: 0, value: %tldo_add(Local), kind: fixup_sparc_tls_ldo_add
         add %o0, %i0, %o0, %tldo_add(Local)
 
 ! Next two sequences are for extern symbols.
@@ -95,31 +86,26 @@ foo:
 ! OBJDUMP: {{[0-9,a-f]+}}:  33 00 00 00  sethi 0, %i1
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_IE_HI22 Extern
 ! ASM: sethi %tie_hi22(Extern), %i1 ! encoding: [0x33,0b00AAAAAA,A,A]
-! ASM:                                    !   fixup A - offset: 0, value: %tie_hi22(Extern), kind: fixup_sparc_tls_ie_hi22
 	sethi %tie_hi22(Extern), %i1
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  b2 06 60 00  add %i1, 0, %i1
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_IE_LO10 Extern
 ! ASM: add %i1, %tie_lo10(Extern), %i1 ! encoding: [0xb2,0x06,0b011000AA,A]
-! ASM:                                    !   fixup A - offset: 0, value: %tie_lo10(Extern), kind: fixup_sparc_tls_ie_lo10
         add %i1, %tie_lo10(Extern), %i1
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  f0 06 00 19  ld [%i0+%i1], %i0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_IE_LD Extern
 ! ASM: ld [%i0+%i1], %i0, %tie_ld(Extern) ! encoding: [0xf0,0x06,0x00,0x19]
-! ASM:                                    !   fixup A - offset: 0, value: %tie_ld(Extern), kind: fixup_sparc_tls_ie_ld
         ld [%i0+%i1], %i0, %tie_ld(Extern)
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  f0 5e 00 19  ldx [%i0+%i1], %i0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_IE_LDX Extern
 ! ASM: ldx [%i0+%i1], %i0, %tie_ldx(Extern) ! encoding: [0xf0,0x5e,0x00,0x19]
-! ASM:                                      !   fixup A - offset: 0, value: %tie_ldx(Extern), kind: fixup_sparc_tls_ie_ldx
         ldx [%i0+%i1], %i0, %tie_ldx(Extern)
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  90 01 c0 18  add %g7, %i0, %o0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_IE_ADD Extern
 ! ASM: add %g7, %i0, %o0, %tie_add(Extern) ! encoding: [0x90,0x01,0xc0,0x18]
-! ASM:                                    !   fixup A - offset: 0, value: %tie_add(Extern), kind: fixup_sparc_tls_ie_add
         add %g7, %i0, %o0, %tie_add(Extern)
 
 ! General Dynamic model
@@ -128,25 +114,21 @@ foo:
 ! OBJDUMP: {{[0-9,a-f]+}}:  33 00 00 00  sethi 0, %i1
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_GD_HI22 Extern
 ! ASM:  sethi %tgd_hi22(Extern), %i1    ! encoding: [0x33,0b00AAAAAA,A,A]
-! ASM:                                    !   fixup A - offset: 0, value: %tgd_hi22(Extern), kind: fixup_sparc_tls_gd_hi22
         sethi %tgd_hi22(Extern), %i1
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  b2 06 60 00  add %i1, 0, %i1
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_GD_LO10 Extern
 ! ASM: add %i1, %tgd_lo10(Extern), %i1 ! encoding: [0xb2,0x06,0b011000AA,A]
-! ASM:                                 !   fixup A - offset: 0, value: %tgd_lo10(Extern), kind: fixup_sparc_tls_gd_lo10
         add %i1, %tgd_lo10(Extern), %i1
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  90 06 00 19  add %i0, %i1, %o0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_GD_ADD Extern
 ! ASM: add %i0, %i1, %o0, %tgd_add(Extern) ! encoding: [0x90,0x06,0x00,0x19]
-! ASM:                                    !   fixup A - offset: 0, value: %tgd_add(Extern), kind: fixup_sparc_tls_gd_add
         add %i0, %i1, %o0, %tgd_add(Extern)
 
 ! OBJDUMP: {{[0-9,a-f]+}}:  40 00 00 00 call 0
 ! OBJDUMP: {{[0-9,a-f]+}}:     R_SPARC_TLS_GD_CALL Extern
 ! ASM: call __tls_get_addr, %tgd_call(Extern) ! encoding: [0x40,0x00,0x00,0x00]
-! ASM:                                        !   fixup A - offset: 0, value: %tgd_call(Extern), kind: fixup_sparc_tls_gd_call
         call __tls_get_addr, %tgd_call(Extern)
 
         .type  Local,@object
diff --git a/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s b/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
index 188947314f4b7..8681eacd0a7b5 100644
--- a/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
+++ b/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
@@ -2,59 +2,45 @@
 
 
         ! CHECK: bne %xcc, .BB0     ! encoding: [0x12,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne %xcc, .BB0
 
         ! CHECK: be %xcc, .BB0      ! encoding: [0x02,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be %xcc, .BB0
 
         ! CHECK: bg %xcc, .BB0      ! encoding: [0x14,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg %xcc, .BB0
 
         ! CHECK: ble %xcc, .BB0      ! encoding: [0x04,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble %xcc, .BB0
 
         ! CHECK: bge %xcc, .BB0      ! encoding: [0x16,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge %xcc, .BB0
 
         ! CHECK: bl %xcc, .BB0      ! encoding: [0x06,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl %xcc, .BB0
 
         ! CHECK: bgu %xcc, .BB0      ! encoding: [0x18,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu %xcc, .BB0
 
         ! CHECK: bleu %xcc, .BB0      ! encoding: [0x08,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu %xcc, .BB0
 
         ! CHECK: bcc %xcc, .BB0      ! encoding: [0x1a,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc %xcc, .BB0
 
         ! CHECK: bcs %xcc, .BB0      ! encoding: [0x0a,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs %xcc, .BB0
 
         ! CHECK: bpos %xcc, .BB0      ! encoding: [0x1c,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos %xcc, .BB0
 
         ! CHECK: bneg %xcc, .BB0      ! encoding: [0x0c,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg %xcc, .BB0
 
         ! CHECK: bvc %xcc, .BB0      ! encoding: [0x1e,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc %xcc, .BB0
 
         ! CHECK: bvs %xcc, .BB0      ! encoding: [0x0e,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs %xcc, .BB0
 
 
@@ -234,893 +220,670 @@
         fmovso   %fcc0, %f1, %f2
 
         ! CHECK: bne,a %icc, .BB0     ! encoding: [0x32,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,a %icc, .BB0
 
         ! CHECK: be,a %icc, .BB0      ! encoding: [0x22,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,a %icc, .BB0
 
         ! CHECK: bg,a %icc, .BB0      ! encoding: [0x34,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,a %icc, .BB0
 
         ! CHECK: ble,a %icc, .BB0      ! encoding: [0x24,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,a %icc, .BB0
 
         ! CHECK: bge,a %icc, .BB0      ! encoding: [0x36,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,a %icc, .BB0
 
         ! CHECK: bl,a %icc, .BB0      ! encoding: [0x26,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,a %icc, .BB0
 
         ! CHECK: bgu,a %icc, .BB0      ! encoding: [0x38,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,a %icc, .BB0
 
         ! CHECK: bleu,a %icc, .BB0      ! encoding: [0x28,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,a %icc, .BB0
 
         ! CHECK: bcc,a %icc, .BB0      ! encoding: [0x3a,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,a %icc, .BB0
 
         ! CHECK: bcs,a %icc, .BB0      ! encoding: [0x2a,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,a %icc, .BB0
 
         ! CHECK: bpos,a %icc, .BB0      ! encoding: [0x3c,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,a %icc, .BB0
 
         ! CHECK: bneg,a %icc, .BB0      ! encoding: [0x2c,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg,a %icc, .BB0
 
         ! CHECK: bvc,a %icc, .BB0      ! encoding: [0x3e,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc,a %icc, .BB0
 
         ! CHECK: bvs,a %icc, .BB0      ! encoding: [0x2e,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs,a %icc, .BB0
 
         ! CHECK: bne,pn %icc, .BB0     ! encoding: [0x12,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,pn %icc, .BB0
 
         ! CHECK: be,pn %icc, .BB0      ! encoding: [0x02,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,pn %icc, .BB0
 
         ! CHECK: bg,pn %icc, .BB0      ! encoding: [0x14,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,pn %icc, .BB0
 
         ! CHECK: ble,pn %icc, .BB0      ! encoding: [0x04,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,pn %icc, .BB0
 
         ! CHECK: bge,pn %icc, .BB0      ! encoding: [0x16,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,pn %icc, .BB0
 
         ! CHECK: bl,pn %icc, .BB0      ! encoding: [0x06,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,pn %icc, .BB0
 
         ! CHECK: bgu,pn %icc, .BB0      ! encoding: [0x18,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,pn %icc, .BB0
 
         ! CHECK: bleu,pn %icc, .BB0      ! encoding: [0x08,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,pn %icc, .BB0
 
         ! CHECK: bcc,pn %icc, .BB0      ! encoding: [0x1a,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,pn %icc, .BB0
 
         ! CHECK: bcs,pn %icc, .BB0      ! encoding: [0x0a,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,pn %icc, .BB0
 
         ! CHECK: bpos,pn %icc, .BB0      ! encoding: [0x1c,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,pn %icc, .BB0
 
         ! CHECK: bneg,pn %icc, .BB0      ! encoding: [0x0c,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg,pn %icc, .BB0
 
         ! CHECK: bvc,pn %icc, .BB0      ! encoding: [0x1e,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc,pn %icc, .BB0
 
         ! CHECK: bvs,pn %icc, .BB0      ! encoding: [0x0e,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs,pn %icc, .BB0
 
         ! CHECK: bne,a,pn %icc, .BB0     ! encoding: [0x32,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,a,pn %icc, .BB0
 
         ! CHECK: be,a,pn %icc, .BB0      ! encoding: [0x22,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,a,pn %icc, .BB0
 
         ! CHECK: bg,a,pn %icc, .BB0      ! encoding: [0x34,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,a,pn %icc, .BB0
 
         ! CHECK: ble,a,pn %icc, .BB0      ! encoding: [0x24,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,a,pn %icc, .BB0
 
         ! CHECK: bge,a,pn %icc, .BB0      ! encoding: [0x36,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,a,pn %icc, .BB0
 
         ! CHECK: bl,a,pn %icc, .BB0      ! encoding: [0x26,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,a,pn %icc, .BB0
 
         ! CHECK: bgu,a,pn %icc, .BB0      ! encoding: [0x38,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,a,pn %icc, .BB0
 
         ! CHECK: bleu,a,pn %icc, .BB0      ! encoding: [0x28,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,a,pn %icc, .BB0
 
         ! CHECK: bcc,a,pn %icc, .BB0      ! encoding: [0x3a,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,a,pn %icc, .BB0
 
         ! CHECK: bcs,a,pn %icc, .BB0      ! encoding: [0x2a,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,a,pn %icc, .BB0
 
         ! CHECK: bpos,a,pn %icc, .BB0      ! encoding: [0x3c,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,a,pn %icc, .BB0
 
         ! CHECK: bneg,a,pn %icc, .BB0      ! encoding: [0x2c,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg,a,pn %icc, .BB0
 
         ! CHECK: bvc,a,pn %icc, .BB0      ! encoding: [0x3e,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc,a,pn %icc, .BB0
 
         ! CHECK: bvs,a,pn %icc, .BB0      ! encoding: [0x2e,0b01000AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs,a,pn %icc, .BB0
 
         ! CHECK: bne %icc, .BB0     ! encoding: [0x12,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,pt %icc, .BB0
 
         ! CHECK: be %icc, .BB0      ! encoding: [0x02,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,pt %icc, .BB0
 
         ! CHECK: bg %icc, .BB0      ! encoding: [0x14,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,pt %icc, .BB0
 
         ! CHECK: ble %icc, .BB0      ! encoding: [0x04,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,pt %icc, .BB0
 
         ! CHECK: bge %icc, .BB0      ! encoding: [0x16,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,pt %icc, .BB0
 
         ! CHECK: bl %icc, .BB0      ! encoding: [0x06,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,pt %icc, .BB0
 
         ! CHECK: bgu %icc, .BB0      ! encoding: [0x18,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,pt %icc, .BB0
 
         ! CHECK: bleu %icc, .BB0      ! encoding: [0x08,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,pt %icc, .BB0
 
         ! CHECK: bcc %icc, .BB0      ! encoding: [0x1a,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,pt %icc, .BB0
 
         ! CHECK: bcs %icc, .BB0      ! encoding: [0x0a,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,pt %icc, .BB0
 
         ! CHECK: bpos %icc, .BB0      ! encoding: [0x1c,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,pt %icc, .BB0
 
         ! CHECK: bneg %icc, .BB0      ! encoding: [0x0c,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg,pt %icc, .BB0
 
         ! CHECK: bvc %icc, .BB0      ! encoding: [0x1e,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc,pt %icc, .BB0
 
         ! CHECK: bvs %icc, .BB0      ! encoding: [0x0e,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs,pt %icc, .BB0
 
         ! CHECK: bne,a %icc, .BB0     ! encoding: [0x32,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,a,pt %icc, .BB0
 
         ! CHECK: be,a %icc, .BB0      ! encoding: [0x22,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,a,pt %icc, .BB0
 
         ! CHECK: bg,a %icc, .BB0      ! encoding: [0x34,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,a,pt %icc, .BB0
 
         ! CHECK: ble,a %icc, .BB0      ! encoding: [0x24,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,a,pt %icc, .BB0
 
         ! CHECK: bge,a %icc, .BB0      ! encoding: [0x36,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,a,pt %icc, .BB0
 
         ! CHECK: bl,a %icc, .BB0      ! encoding: [0x26,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,a,pt %icc, .BB0
 
         ! CHECK: bgu,a %icc, .BB0      ! encoding: [0x38,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,a,pt %icc, .BB0
 
         ! CHECK: bleu,a %icc, .BB0      ! encoding: [0x28,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,a,pt %icc, .BB0
 
         ! CHECK: bcc,a %icc, .BB0      ! encoding: [0x3a,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,a,pt %icc, .BB0
 
         ! CHECK: bcs,a %icc, .BB0      ! encoding: [0x2a,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,a,pt %icc, .BB0
 
         ! CHECK: bpos,a %icc, .BB0      ! encoding: [0x3c,0b01001AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,a,pt %icc, .BB0
 
 
         ! CHECK: bne,a %xcc, .BB0     ! encoding: [0x32,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,a %xcc, .BB0
 
         ! CHECK: be,a %xcc, .BB0      ! encoding: [0x22,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,a %xcc, .BB0
 
         ! CHECK: bg,a %xcc, .BB0      ! encoding: [0x34,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,a %xcc, .BB0
 
         ! CHECK: ble,a %xcc, .BB0      ! encoding: [0x24,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,a %xcc, .BB0
 
         ! CHECK: bge,a %xcc, .BB0      ! encoding: [0x36,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,a %xcc, .BB0
 
         ! CHECK: bl,a %xcc, .BB0      ! encoding: [0x26,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,a %xcc, .BB0
 
         ! CHECK: bgu,a %xcc, .BB0      ! encoding: [0x38,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,a %xcc, .BB0
 
         ! CHECK: bleu,a %xcc, .BB0      ! encoding: [0x28,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,a %xcc, .BB0
 
         ! CHECK: bcc,a %xcc, .BB0      ! encoding: [0x3a,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,a %xcc, .BB0
 
         ! CHECK: bcs,a %xcc, .BB0      ! encoding: [0x2a,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,a %xcc, .BB0
 
         ! CHECK: bpos,a %xcc, .BB0      ! encoding: [0x3c,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,a %xcc, .BB0
 
         ! CHECK: bneg,a %xcc, .BB0      ! encoding: [0x2c,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg,a %xcc, .BB0
 
         ! CHECK: bvc,a %xcc, .BB0      ! encoding: [0x3e,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc,a %xcc, .BB0
 
         ! CHECK: bvs,a %xcc, .BB0      ! encoding: [0x2e,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs,a %xcc, .BB0
 
         ! CHECK: bne,pn %xcc, .BB0     ! encoding: [0x12,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,pn %xcc, .BB0
 
         ! CHECK: be,pn %xcc, .BB0      ! encoding: [0x02,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,pn %xcc, .BB0
 
         ! CHECK: bg,pn %xcc, .BB0      ! encoding: [0x14,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,pn %xcc, .BB0
 
         ! CHECK: ble,pn %xcc, .BB0      ! encoding: [0x04,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,pn %xcc, .BB0
 
         ! CHECK: bge,pn %xcc, .BB0      ! encoding: [0x16,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,pn %xcc, .BB0
 
         ! CHECK: bl,pn %xcc, .BB0      ! encoding: [0x06,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,pn %xcc, .BB0
 
         ! CHECK: bgu,pn %xcc, .BB0      ! encoding: [0x18,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,pn %xcc, .BB0
 
         ! CHECK: bleu,pn %xcc, .BB0      ! encoding: [0x08,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,pn %xcc, .BB0
 
         ! CHECK: bcc,pn %xcc, .BB0      ! encoding: [0x1a,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,pn %xcc, .BB0
 
         ! CHECK: bcs,pn %xcc, .BB0      ! encoding: [0x0a,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,pn %xcc, .BB0
 
         ! CHECK: bpos,pn %xcc, .BB0      ! encoding: [0x1c,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,pn %xcc, .BB0
 
         ! CHECK: bneg,pn %xcc, .BB0      ! encoding: [0x0c,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg,pn %xcc, .BB0
 
         ! CHECK: bvc,pn %xcc, .BB0      ! encoding: [0x1e,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc,pn %xcc, .BB0
 
         ! CHECK: bvs,pn %xcc, .BB0      ! encoding: [0x0e,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs,pn %xcc, .BB0
 
         ! CHECK: bne,a,pn %xcc, .BB0     ! encoding: [0x32,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,a,pn %xcc, .BB0
 
         ! CHECK: be,a,pn %xcc, .BB0      ! encoding: [0x22,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,a,pn %xcc, .BB0
 
         ! CHECK: bg,a,pn %xcc, .BB0      ! encoding: [0x34,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,a,pn %xcc, .BB0
 
         ! CHECK: ble,a,pn %xcc, .BB0      ! encoding: [0x24,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,a,pn %xcc, .BB0
 
         ! CHECK: bge,a,pn %xcc, .BB0      ! encoding: [0x36,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,a,pn %xcc, .BB0
 
         ! CHECK: bl,a,pn %xcc, .BB0      ! encoding: [0x26,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,a,pn %xcc, .BB0
 
         ! CHECK: bgu,a,pn %xcc, .BB0      ! encoding: [0x38,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,a,pn %xcc, .BB0
 
         ! CHECK: bleu,a,pn %xcc, .BB0      ! encoding: [0x28,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,a,pn %xcc, .BB0
 
         ! CHECK: bcc,a,pn %xcc, .BB0      ! encoding: [0x3a,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,a,pn %xcc, .BB0
 
         ! CHECK: bcs,a,pn %xcc, .BB0      ! encoding: [0x2a,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,a,pn %xcc, .BB0
 
         ! CHECK: bpos,a,pn %xcc, .BB0      ! encoding: [0x3c,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,a,pn %xcc, .BB0
 
         ! CHECK: bneg,a,pn %xcc, .BB0      ! encoding: [0x2c,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg,a,pn %xcc, .BB0
 
         ! CHECK: bvc,a,pn %xcc, .BB0      ! encoding: [0x3e,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc,a,pn %xcc, .BB0
 
         ! CHECK: bvs,a,pn %xcc, .BB0      ! encoding: [0x2e,0b01100AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs,a,pn %xcc, .BB0
 
         ! CHECK: bne %xcc, .BB0     ! encoding: [0x12,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,pt %xcc, .BB0
 
         ! CHECK: be %xcc, .BB0      ! encoding: [0x02,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,pt %xcc, .BB0
 
         ! CHECK: bg %xcc, .BB0      ! encoding: [0x14,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,pt %xcc, .BB0
 
         ! CHECK: ble %xcc, .BB0      ! encoding: [0x04,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,pt %xcc, .BB0
 
         ! CHECK: bge %xcc, .BB0      ! encoding: [0x16,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,pt %xcc, .BB0
 
         ! CHECK: bl %xcc, .BB0      ! encoding: [0x06,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,pt %xcc, .BB0
 
         ! CHECK: bgu %xcc, .BB0      ! encoding: [0x18,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,pt %xcc, .BB0
 
         ! CHECK: bleu %xcc, .BB0      ! encoding: [0x08,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,pt %xcc, .BB0
 
         ! CHECK: bcc %xcc, .BB0      ! encoding: [0x1a,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,pt %xcc, .BB0
 
         ! CHECK: bcs %xcc, .BB0      ! encoding: [0x0a,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,pt %xcc, .BB0
 
         ! CHECK: bpos %xcc, .BB0      ! encoding: [0x1c,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,pt %xcc, .BB0
 
         ! CHECK: bneg %xcc, .BB0      ! encoding: [0x0c,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bneg,pt %xcc, .BB0
 
         ! CHECK: bvc %xcc, .BB0      ! encoding: [0x1e,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvc,pt %xcc, .BB0
 
         ! CHECK: bvs %xcc, .BB0      ! encoding: [0x0e,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bvs,pt %xcc, .BB0
 
         ! CHECK: bne,a %xcc, .BB0     ! encoding: [0x32,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bne,a,pt %xcc, .BB0
 
         ! CHECK: be,a %xcc, .BB0      ! encoding: [0x22,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         be,a,pt %xcc, .BB0
 
         ! CHECK: bg,a %xcc, .BB0      ! encoding: [0x34,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bg,a,pt %xcc, .BB0
 
         ! CHECK: ble,a %xcc, .BB0      ! encoding: [0x24,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         ble,a,pt %xcc, .BB0
 
         ! CHECK: bge,a %xcc, .BB0      ! encoding: [0x36,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bge,a,pt %xcc, .BB0
 
         ! CHECK: bl,a %xcc, .BB0      ! encoding: [0x26,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bl,a,pt %xcc, .BB0
 
         ! CHECK: bgu,a %xcc, .BB0      ! encoding: [0x38,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bgu,a,pt %xcc, .BB0
 
         ! CHECK: bleu,a %xcc, .BB0      ! encoding: [0x28,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bleu,a,pt %xcc, .BB0
 
         ! CHECK: bcc,a %xcc, .BB0      ! encoding: [0x3a,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcc,a,pt %xcc, .BB0
 
         ! CHECK: bcs,a %xcc, .BB0      ! encoding: [0x2a,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bcs,a,pt %xcc, .BB0
 
         ! CHECK: bpos,a %xcc, .BB0      ! encoding: [0x3c,0b01101AAA,A,A]
-        ! CHECK-NEXT:         ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         bpos,a,pt %xcc, .BB0
 
         ! CHECK:             fba %fcc0, .BB0                        ! encoding: [0x11,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fba %fcc0, .BB0
 
         ! CHECK:             fba %fcc0, .BB0                        ! encoding: [0x11,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fb %fcc0, .BB0
 
         ! CHECK:             fbn %fcc0, .BB0                        ! encoding: [0x01,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbn %fcc0, .BB0
 
         ! CHECK:             fbu %fcc0, .BB0                      ! encoding: [0x0f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbu %fcc0, .BB0
 
         ! CHECK:             fbg %fcc0, .BB0                      ! encoding: [0x0d,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbg %fcc0, .BB0
         ! CHECK:             fbug %fcc0, .BB0                     ! encoding: [0x0b,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbug %fcc0, .BB0
 
         ! CHECK:             fbl %fcc0, .BB0                      ! encoding: [0x09,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbl %fcc0, .BB0
 
         ! CHECK:             fbul %fcc0, .BB0                     ! encoding: [0x07,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbul %fcc0, .BB0
 
         ! CHECK:             fblg %fcc0, .BB0                     ! encoding: [0x05,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fblg %fcc0, .BB0
 
         ! CHECK:             fbne %fcc0, .BB0                     ! encoding: [0x03,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbne %fcc0, .BB0
 
         ! CHECK:             fbe %fcc0, .BB0                      ! encoding: [0x13,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbe %fcc0, .BB0
 
         ! CHECK:             fbue %fcc0, .BB0                     ! encoding: [0x15,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbue %fcc0, .BB0
 
         ! CHECK:             fbge %fcc0, .BB0                     ! encoding: [0x17,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbge %fcc0, .BB0
 
         ! CHECK:             fbuge %fcc0, .BB0                    ! encoding: [0x19,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbuge %fcc0, .BB0
 
         ! CHECK:             fble %fcc0, .BB0                     ! encoding: [0x1b,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fble %fcc0, .BB0
 
         ! CHECK:             fbule %fcc0, .BB0                    ! encoding: [0x1d,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbule %fcc0, .BB0
 
         ! CHECK:             fbo %fcc0, .BB0                      ! encoding: [0x1f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbo %fcc0, .BB0
 
         ! CHECK:             fbu %fcc0, .BB0                      ! encoding: [0x0f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbu,pt %fcc0, .BB0
 
         ! CHECK:             fbg %fcc0, .BB0                      ! encoding: [0x0d,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbg,pt %fcc0, .BB0
         ! CHECK:             fbug %fcc0, .BB0                     ! encoding: [0x0b,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbug,pt %fcc0, .BB0
 
         ! CHECK:             fbl %fcc0, .BB0                      ! encoding: [0x09,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbl,pt %fcc0, .BB0
 
         ! CHECK:             fbul %fcc0, .BB0                     ! encoding: [0x07,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbul,pt %fcc0, .BB0
 
         ! CHECK:             fblg %fcc0, .BB0                     ! encoding: [0x05,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fblg,pt %fcc0, .BB0
 
         ! CHECK:             fbne %fcc0, .BB0                     ! encoding: [0x03,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbne,pt %fcc0, .BB0
 
         ! CHECK:             fbe %fcc0, .BB0                      ! encoding: [0x13,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbe,pt %fcc0, .BB0
 
         ! CHECK:             fbue %fcc0, .BB0                     ! encoding: [0x15,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbue,pt %fcc0, .BB0
 
         ! CHECK:             fbge %fcc0, .BB0                     ! encoding: [0x17,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbge,pt %fcc0, .BB0
 
         ! CHECK:             fbuge %fcc0, .BB0                    ! encoding: [0x19,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbuge,pt %fcc0, .BB0
 
         ! CHECK:             fble %fcc0, .BB0                     ! encoding: [0x1b,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fble,pt %fcc0, .BB0
 
         ! CHECK:             fbule %fcc0, .BB0                    ! encoding: [0x1d,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbule,pt %fcc0, .BB0
 
         ! CHECK:             fbo %fcc0, .BB0                      ! encoding: [0x1f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbo,pt %fcc0, .BB0
 
 
         ! CHECK:             fbo,a %fcc0, .BB0                      ! encoding: [0x3f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbo,a %fcc0, .BB0
 
         ! CHECK:             fbu,a %fcc0, .BB0                      ! encoding: [0x2f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbu,a %fcc0, .BB0
 
         ! CHECK:             fbg,a %fcc0, .BB0                      ! encoding: [0x2d,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbg,a %fcc0, .BB0
         ! CHECK:             fbug,a %fcc0, .BB0                     ! encoding: [0x2b,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbug,a %fcc0, .BB0
 
         ! CHECK:             fbl,a %fcc0, .BB0                      ! encoding: [0x29,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbl,a %fcc0, .BB0
 
         ! CHECK:             fbul,a %fcc0, .BB0                     ! encoding: [0x27,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbul,a %fcc0, .BB0
 
         ! CHECK:             fblg,a %fcc0, .BB0                     ! encoding: [0x25,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fblg,a %fcc0, .BB0
 
         ! CHECK:             fbne,a %fcc0, .BB0                     ! encoding: [0x23,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbne,a %fcc0, .BB0
 
         ! CHECK:             fbe,a %fcc0, .BB0                      ! encoding: [0x33,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbe,a %fcc0, .BB0
 
         ! CHECK:             fbue,a %fcc0, .BB0                     ! encoding: [0x35,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbue,a %fcc0, .BB0
 
         ! CHECK:             fbge,a %fcc0, .BB0                     ! encoding: [0x37,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbge,a %fcc0, .BB0
 
         ! CHECK:             fbuge,a %fcc0, .BB0                    ! encoding: [0x39,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbuge,a %fcc0, .BB0
 
         ! CHECK:             fble,a %fcc0, .BB0                     ! encoding: [0x3b,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fble,a %fcc0, .BB0
 
         ! CHECK:             fbule,a %fcc0, .BB0                    ! encoding: [0x3d,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbule,a %fcc0, .BB0
 
         ! CHECK:             fbo,a %fcc0, .BB0                      ! encoding: [0x3f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbo,a %fcc0, .BB0
 
                 ! CHECK:             fbo,a %fcc0, .BB0                      ! encoding: [0x3f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbo,a %fcc0, .BB0
 
         ! CHECK:             fbu,a %fcc0, .BB0                      ! encoding: [0x2f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbu,a,pt %fcc0, .BB0
 
         ! CHECK:             fbg,a %fcc0, .BB0                      ! encoding: [0x2d,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbg,a,pt %fcc0, .BB0
 
         ! CHECK:             fbug,a %fcc0, .BB0                     ! encoding: [0x2b,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbug,a,pt %fcc0, .BB0
 
         ! CHECK:             fbl,a %fcc0, .BB0                      ! encoding: [0x29,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbl,a,pt %fcc0, .BB0
 
         ! CHECK:             fbul,a %fcc0, .BB0                     ! encoding: [0x27,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbul,a,pt %fcc0, .BB0
 
         ! CHECK:             fblg,a %fcc0, .BB0                     ! encoding: [0x25,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fblg,a,pt %fcc0, .BB0
 
         ! CHECK:             fbne,a %fcc0, .BB0                     ! encoding: [0x23,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbne,a,pt %fcc0, .BB0
 
         ! CHECK:             fbe,a %fcc0, .BB0                      ! encoding: [0x33,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbe,a,pt %fcc0, .BB0
 
         ! CHECK:             fbue,a %fcc0, .BB0                     ! encoding: [0x35,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbue,a,pt %fcc0, .BB0
 
         ! CHECK:             fbge,a %fcc0, .BB0                     ! encoding: [0x37,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbge,a,pt %fcc0, .BB0
 
         ! CHECK:             fbuge,a %fcc0, .BB0                    ! encoding: [0x39,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbuge,a,pt %fcc0, .BB0
 
         ! CHECK:             fble,a %fcc0, .BB0                     ! encoding: [0x3b,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fble,a,pt %fcc0, .BB0
 
         ! CHECK:             fbule,a %fcc0, .BB0                    ! encoding: [0x3d,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbule,a,pt %fcc0, .BB0
 
         ! CHECK:             fbo,a %fcc0, .BB0                      ! encoding: [0x3f,0b01001AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbo,a,pt %fcc0, .BB0
 
         ! CHECK:             fbu,pn %fcc0, .BB0                 ! encoding: [0x0f,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbu,pn %fcc0, .BB0
 
         ! CHECK:             fbg,pn %fcc0, .BB0                      ! encoding: [0x0d,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbg,pn %fcc0, .BB0
         ! CHECK:             fbug,pn %fcc0, .BB0                     ! encoding: [0x0b,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbug,pn %fcc0, .BB0
 
         ! CHECK:             fbl,pn %fcc0, .BB0                      ! encoding: [0x09,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbl,pn %fcc0, .BB0
 
         ! CHECK:             fbul,pn %fcc0, .BB0                     ! encoding: [0x07,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbul,pn %fcc0, .BB0
 
         ! CHECK:             fblg,pn %fcc0, .BB0                     ! encoding: [0x05,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fblg,pn %fcc0, .BB0
 
         ! CHECK:             fbne,pn %fcc0, .BB0                     ! encoding: [0x03,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbne,pn %fcc0, .BB0
 
         ! CHECK:             fbe,pn %fcc0, .BB0                      ! encoding: [0x13,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbe,pn %fcc0, .BB0
 
         ! CHECK:             fbue,pn %fcc0, .BB0                     ! encoding: [0x15,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbue,pn %fcc0, .BB0
 
         ! CHECK:             fbge,pn %fcc0, .BB0                     ! encoding: [0x17,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbge,pn %fcc0, .BB0
 
         ! CHECK:             fbuge,pn %fcc0, .BB0                    ! encoding: [0x19,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbuge,pn %fcc0, .BB0
 
         ! CHECK:             fble,pn %fcc0, .BB0                     ! encoding: [0x1b,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fble,pn %fcc0, .BB0
 
         ! CHECK:             fbule,pn %fcc0, .BB0                    ! encoding: [0x1d,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbule,pn %fcc0, .BB0
 
         ! CHECK:             fbo,pn %fcc0, .BB0                      ! encoding: [0x1f,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbo,pn %fcc0, .BB0
 
                 ! CHECK:             fbu,a,pn %fcc0, .BB0                      ! encoding: [0x2f,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbu,a,pn %fcc0, .BB0
 
         ! CHECK:             fbg,a,pn %fcc0, .BB0                      ! encoding: [0x2d,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbg,a,pn %fcc0, .BB0
 
         ! CHECK:             fbug,a,pn %fcc0, .BB0                     ! encoding: [0x2b,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbug,a,pn %fcc0, .BB0
 
         ! CHECK:             fbl,a,pn %fcc0, .BB0                      ! encoding: [0x29,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbl,a,pn %fcc0, .BB0
 
         ! CHECK:             fbul,a,pn %fcc0, .BB0                     ! encoding: [0x27,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbul,a,pn %fcc0, .BB0
 
         ! CHECK:             fblg,a,pn %fcc0, .BB0                     ! encoding: [0x25,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fblg,a,pn %fcc0, .BB0
 
         ! CHECK:             fbne,a,pn %fcc0, .BB0                     ! encoding: [0x23,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbne,a,pn %fcc0, .BB0
 
         ! CHECK:             fbe,a,pn %fcc0, .BB0                      ! encoding: [0x33,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbe,a,pn %fcc0, .BB0
 
         ! CHECK:             fbue,a,pn %fcc0, .BB0                     ! encoding: [0x35,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbue,a,pn %fcc0, .BB0
 
         ! CHECK:             fbge,a,pn %fcc0, .BB0                     ! encoding: [0x37,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbge,a,pn %fcc0, .BB0
 
         ! CHECK:             fbuge,a,pn %fcc0, .BB0                    ! encoding: [0x39,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbuge,a,pn %fcc0, .BB0
 
         ! CHECK:             fble,a,pn %fcc0, .BB0                     ! encoding: [0x3b,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fble,a,pn %fcc0, .BB0
 
         ! CHECK:             fbule,a,pn %fcc0, .BB0                    ! encoding: [0x3d,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbule,a,pn %fcc0, .BB0
 
         ! CHECK:             fbo,a,pn %fcc0, .BB0                      ! encoding: [0x3f,0b01000AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbo,a,pn %fcc0, .BB0
 
         ! CHECK: movu %fcc1, %g1, %g2            ! encoding: [0x85,0x61,0xc8,0x01]
@@ -1130,38 +893,27 @@
         fmovsg %fcc2, %f1, %f2
 
         ! CHECK:             fbug %fcc3, .BB0                ! encoding: [0x0b,0b01111AAA,A,A]
-        ! CHECK-NEXT:                                        ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbug %fcc3, .BB0
 
         ! CHECK:             fbu %fcc3, .BB0                 ! encoding: [0x0f,0b01111AAA,A,A]
-        ! CHECK-NEXT:                                        ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbu,pt %fcc3, .BB0
 
         ! CHECK:             fbl,a %fcc3, .BB0               ! encoding: [0x29,0b01111AAA,A,A]
-        ! CHECK-NEXT:                                        ! fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbl,a %fcc3, .BB0
 
         ! CHECK:             fbue,pn %fcc3, .BB0             ! encoding: [0x15,0b01110AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbue,pn %fcc3, .BB0
 
         ! CHECK:             fbne,a,pn %fcc3, .BB0           ! encoding: [0x23,0b01110AAA,A,A]
-        ! CHECK-NEXT:                                        !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br19
         fbne,a,pn %fcc3, .BB0
 
 
         ! CHECK:                brz %g1, .BB0                   ! encoding: [0x02'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         ! CHECK:                brlez %g1, .BB0                 ! encoding: [0x04'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         ! CHECK:                brlz %g1, .BB0                  ! encoding: [0x06'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         ! CHECK:                brnz %g1, .BB0                  ! encoding: [0x0a'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         ! CHECK:                brgz %g1, .BB0                  ! encoding: [0x0c'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         ! CHECK:                brgez %g1, .BB0                 ! encoding: [0x0e'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
 
         brz   %g1, .BB0
         brlez %g1, .BB0
@@ -1171,23 +923,18 @@
         brgez %g1, .BB0
 
         ! CHECK: brz %g1, .BB0                   ! encoding: [0x02'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,pt   %g1, .BB0
 
         ! CHECK: brz,a %g1, .BB0                 ! encoding: [0x22'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,a   %g1, .BB0
 
         ! CHECK: brz,a %g1, .BB0                 ! encoding: [0x22'A',0xc8'A',0x40'A',A]
-        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,a,pt   %g1, .BB0
 
         ! CHECK:  brz,pn %g1, .BB0               ! encoding: [0x02'A',0xc0'A',0x40'A',A]
-        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,pn   %g1, .BB0
 
         ! CHECK:  brz,a,pn %g1, .BB0              ! encoding: [0x22'A',0xc0'A',0x40'A',A]
-        ! CHECK-NEXT:                             !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,a,pn   %g1, .BB0
 
         ! CHECK: movrz   %g1, %g2, %g3 ! encoding: [0x87,0x78,0x44,0x02]
diff --git a/llvm/test/MC/Sparc/sparcv9-synthetic-instructions.s b/llvm/test/MC/Sparc/sparcv9-synthetic-instructions.s
index 0d14ff558b314..20da7fd1df3fb 100644
--- a/llvm/test/MC/Sparc/sparcv9-synthetic-instructions.s
+++ b/llvm/test/MC/Sparc/sparcv9-synthetic-instructions.s
@@ -24,29 +24,21 @@ setx (0-1), %g1, %o1
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V8-NEXT: setx 0xffffffff, %g1, %o1
 ! V9: sethi %hi(4294967295), %o1      ! encoding: [0x13,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(4294967295), kind: fixup_sparc_hi22
 ! V9: or %o1, %lo(4294967295), %o1    ! encoding: [0x92,0x12,0b011000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %lo(4294967295), kind: fixup_sparc_lo10
 setx 0xffffffff, %g1, %o1
 
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V8-NEXT: setx (0xffff0000+0x0000ffff), %g1, %o1
 ! V9: sethi %hi(4294967295), %o1      ! encoding: [0x13,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(4294967295), kind: fixup_sparc_hi22
 ! V9: or %o1, %lo(4294967295), %o1    ! encoding: [0x92,0x12,0b011000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %lo(4294967295), kind: fixup_sparc_lo10
 setx (0xffff0000+0x0000ffff), %g1, %o1
 
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V8-NEXT: setx 0x0123456789abcdef, %g1, %o0
 ! V9: sethi %hi(81985529216486895), %o0       ! encoding: [0x11,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(81985529216486895), kind: fixup_sparc_hi22
 ! V9: or %o0, %lo(81985529216486895), %o0     ! encoding: [0x90,0x12,0b001000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %lo(81985529216486895), kind: fixup_sparc_lo10
 ! V9: sethi %hh(81985529216486895), %g1       ! encoding: [0x03,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hh(81985529216486895), kind: fixup_sparc_hh
 ! V9: or %g1, %hm(81985529216486895), %g1     ! encoding: [0x82,0x10,0b011000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %hm(81985529216486895), kind: fixup_sparc_hm
 ! V9: sllx %g1, 32, %g1                       ! encoding: [0x83,0x28,0x70,0x20]
 ! V9: or %g1, %o0, %o0                        ! encoding: [0x90,0x10,0x40,0x08]
 setx 0x0123456789abcdef, %g1, %o0
@@ -54,13 +46,9 @@ setx 0x0123456789abcdef, %g1, %o0
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V8-NEXT: setx (0x0123456700000000+0x0000000089abcdef), %g1, %o0
 ! V9: sethi %hi(81985529216486895), %o0       ! encoding: [0x11,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(81985529216486895), kind: fixup_sparc_hi22
 ! V9: or %o0, %lo(81985529216486895), %o0     ! encoding: [0x90,0x12,0b001000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %lo(81985529216486895), kind: fixup_sparc_lo10
 ! V9: sethi %hh(81985529216486895), %g1       ! encoding: [0x03,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hh(81985529216486895), kind: fixup_sparc_hh
 ! V9: or %g1, %hm(81985529216486895), %g1     ! encoding: [0x82,0x10,0b011000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %hm(81985529216486895), kind: fixup_sparc_hm
 ! V9: sllx %g1, 32, %g1                       ! encoding: [0x83,0x28,0x70,0x20]
 ! V9: or %g1, %o0, %o0                        ! encoding: [0x90,0x10,0x40,0x08]
 setx (0x0123456700000000+0x0000000089abcdef), %g1, %o0
@@ -68,13 +56,9 @@ setx (0x0123456700000000+0x0000000089abcdef), %g1, %o0
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V8-NEXT: setx (.BB1-.BB0), %g1, %o0
 ! V9: sethi %hi(.BB1-.BB0), %o0               ! encoding: [0x11,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(.BB1-.BB0), kind: fixup_sparc_hi22
 ! V9: or %o0, %lo(.BB1-.BB0), %o0             ! encoding: [0x90,0x12,0b001000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %lo(.BB1-.BB0), kind: fixup_sparc_lo10
 ! V9: sethi %hh(.BB1-.BB0), %g1               ! encoding: [0x03,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hh(.BB1-.BB0), kind: fixup_sparc_hh
 ! V9: or %g1, %hm(.BB1-.BB0), %g1             ! encoding: [0x82,0x10,0b011000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %hm(.BB1-.BB0), kind: fixup_sparc_hm
 ! V9: sllx %g1, 32, %g1                       ! encoding: [0x83,0x28,0x70,0x20]
 ! V9: or %g1, %o0, %o0                        ! encoding: [0x90,0x10,0x40,0x08]
 setx (.BB1-.BB0), %g1, %o0
@@ -82,7 +66,6 @@ setx (.BB1-.BB0), %g1, %o0
 !! In V9, setuw is a mnemonic alias for set.
 ! V8:      error: invalid instruction mnemonic
 ! V9: sethi %hi(32768), %g1            ! encoding: [0x03,0b00AAAAAA,A,A]
-! V9:                                  !   fixup A - offset: 0, value: %hi(32768), kind: fixup_sparc_hi22
 setuw 32768, %g1
 ! V9: mov 1, %g1 ! encoding: [0x82,0x10,0x20,0x01]
 setuw 1, %g1
@@ -95,30 +78,22 @@ setsw 4095, %g1
 setsw -4096, %g1
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V9: sethi %hi(4096), %g1            ! encoding: [0x03,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(4096), kind: fixup_sparc_hi22
 setsw 4096, %g1
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V9: sethi %hi(-4097), %g1           ! encoding: [0x03,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(-4097), kind: fixup_sparc_hi22
 ! V9: sra %g1, %g0, %g1               ! encoding: [0x83,0x38,0x40,0x00]
 setsw -4097, %g1
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V9: sethi %hi(2147483647), %o1      ! encoding: [0x13,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(2147483647), kind: fixup_sparc_hi22
 ! V9: or %o1, %lo(2147483647), %o1    ! encoding: [0x92,0x12,0b011000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %lo(2147483647), kind: fixup_sparc_lo10
 setsw 2147483647, %o1
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V9: sethi %hi(-2147483647), %o1     ! encoding: [0x13,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(-2147483647), kind: fixup_sparc_hi22
 ! V9: or %o1, %lo(-2147483647), %o1   ! encoding: [0x92,0x12,0b011000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %lo(-2147483647), kind: fixup_sparc_lo10
 ! V9: sra %o1, %g0, %o1               ! encoding: [0x93,0x3a,0x40,0x00]
 setsw -2147483647, %o1
 ! V8:      error: instruction requires a CPU feature not currently enabled
 ! V9: sethi %hi(.Ltmp0), %o1          ! encoding: [0x13,0b00AAAAAA,A,A]
-! V9:                                 !   fixup A - offset: 0, value: %hi(.Ltmp0), kind: fixup_sparc_hi22
 ! V9: or %o1, %lo(.Ltmp0), %o1        ! encoding: [0x92,0x12,0b011000AA,A]
-! V9:                                 !   fixup A - offset: 0, value: %lo(.Ltmp0), kind: fixup_sparc_lo10
 ! V9: sra %o1, %g0, %o1               ! encoding: [0x93,0x3a,0x40,0x00]
 setsw ., %o1

From a42ac55a79db071e7e6da385213c5486d4b14986 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 17 Apr 2025 23:03:25 -0700
Subject: [PATCH 378/710] [IPO] Avoid repeated hash lookups (NFC) (#135750)

---
 llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 561c01cb01f82..257a7de6935d7 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -4099,14 +4099,14 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
         assert(FuncClonesToCallMap.count(FuncClone));
         std::map<CallInfo, CallInfo> &CallMap = FuncClonesToCallMap[FuncClone];
         CallInfo CallClone(Call);
-        if (CallMap.count(Call))
-          CallClone = CallMap[Call];
+        if (auto It = CallMap.find(Call); It != CallMap.end())
+          CallClone = It->second;
         CallsiteClone->setCall(CallClone);
         // Need to do the same for all matching calls.
         for (auto &MatchingCall : Node->MatchingCalls) {
           CallInfo CallClone(MatchingCall);
-          if (CallMap.count(MatchingCall))
-            CallClone = CallMap[MatchingCall];
+          if (auto It = CallMap.find(MatchingCall); It != CallMap.end())
+            CallClone = It->second;
           // Updates the call in the list.
           MatchingCall = CallClone;
         }

From d27175d26e20ebc112b003c877692c06046d195b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 17 Apr 2025 23:03:39 -0700
Subject: [PATCH 379/710] [Scalar] Avoid repeated hash lookups (NFC) (#135751)

---
 llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index 329bd45902242..7eee0283e20e6 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -3332,10 +3332,10 @@ static void computeLiveInValues(DominatorTree &DT, Function &F,
     LiveTmp.set_subtract(Data.KillSet[BB]);
 
     assert(Data.LiveIn.count(BB));
-    const SetVector<Value *> &OldLiveIn = Data.LiveIn[BB];
-    // assert: OldLiveIn is a subset of LiveTmp
-    if (OldLiveIn.size() != LiveTmp.size()) {
-      Data.LiveIn[BB] = LiveTmp;
+    SetVector<Value *> &LiveIn = Data.LiveIn[BB];
+    // assert: LiveIn is a subset of LiveTmp
+    if (LiveIn.size() != LiveTmp.size()) {
+      LiveIn = std::move(LiveTmp);
       Worklist.insert_range(predecessors(BB));
     }
   } // while (!Worklist.empty())

From cc7fc9978fc8c9b8e1da2e283026feaf85807ea5 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Fri, 18 Apr 2025 08:27:15 +0200
Subject: [PATCH 380/710] [clang] Add source range to 'use of undeclared
 identifier' diagnostics (#117671)

---
 clang/lib/Frontend/DiagnosticRenderer.cpp | 75 ++++++++---------------
 clang/lib/Sema/SemaExpr.cpp               | 45 +++++++-------
 2 files changed, 51 insertions(+), 69 deletions(-)

diff --git a/clang/lib/Frontend/DiagnosticRenderer.cpp b/clang/lib/Frontend/DiagnosticRenderer.cpp
index 8b32af1337257..b11806637efda 100644
--- a/clang/lib/Frontend/DiagnosticRenderer.cpp
+++ b/clang/lib/Frontend/DiagnosticRenderer.cpp
@@ -454,62 +454,41 @@ void DiagnosticRenderer::emitSingleMacroExpansion(
                  SpellingRanges, {});
 }
 
-/// Check that the macro argument location of Loc starts with ArgumentLoc.
-/// The starting location of the macro expansions is used to differeniate
-/// different macro expansions.
-static bool checkLocForMacroArgExpansion(SourceLocation Loc,
-                                         const SourceManager &SM,
-                                         SourceLocation ArgumentLoc) {
-  SourceLocation MacroLoc;
-  if (SM.isMacroArgExpansion(Loc, &MacroLoc)) {
-    if (ArgumentLoc == MacroLoc) return true;
-  }
-
-  return false;
-}
-
-/// Check if all the locations in the range have the same macro argument
-/// expansion, and that the expansion starts with ArgumentLoc.
-static bool checkRangeForMacroArgExpansion(CharSourceRange Range,
-                                           const SourceManager &SM,
-                                           SourceLocation ArgumentLoc) {
-  SourceLocation BegLoc = Range.getBegin(), EndLoc = Range.getEnd();
-  while (BegLoc != EndLoc) {
-    if (!checkLocForMacroArgExpansion(BegLoc, SM, ArgumentLoc))
-      return false;
-    BegLoc.getLocWithOffset(1);
-  }
-
-  return checkLocForMacroArgExpansion(BegLoc, SM, ArgumentLoc);
-}
-
 /// A helper function to check if the current ranges are all inside the same
 /// macro argument expansion as Loc.
-static bool checkRangesForMacroArgExpansion(FullSourceLoc Loc,
-                                            ArrayRef<CharSourceRange> Ranges) {
+static bool
+rangesInsideSameMacroArgExpansion(FullSourceLoc Loc,
+                                  ArrayRef<CharSourceRange> Ranges) {
   assert(Loc.isMacroID() && "Must be a macro expansion!");
 
-  SmallVector<CharSourceRange, 4> SpellingRanges;
+  SmallVector<CharSourceRange> SpellingRanges;
   mapDiagnosticRanges(Loc, Ranges, SpellingRanges);
 
-  // Count all valid ranges.
   unsigned ValidCount =
       llvm::count_if(Ranges, [](const auto &R) { return R.isValid(); });
-
   if (ValidCount > SpellingRanges.size())
     return false;
 
-  // To store the source location of the argument location.
-  FullSourceLoc ArgumentLoc;
+  const SourceManager &SM = Loc.getManager();
+  for (const auto &R : Ranges) {
+    // All positions in the range need to point to Loc.
+    SourceLocation Begin = R.getBegin();
+    if (Begin == R.getEnd()) {
+      if (!SM.isMacroArgExpansion(Begin))
+        return false;
+      continue;
+    }
 
-  // Set the ArgumentLoc to the beginning location of the expansion of Loc
-  // so to check if the ranges expands to the same beginning location.
-  if (!Loc.isMacroArgExpansion(&ArgumentLoc))
-    return false;
+    while (Begin != R.getEnd()) {
+      SourceLocation MacroLoc;
+      if (!SM.isMacroArgExpansion(Begin, &MacroLoc))
+        return false;
+      if (MacroLoc != Loc)
+        return false;
 
-  for (const auto &Range : SpellingRanges)
-    if (!checkRangeForMacroArgExpansion(Range, Loc.getManager(), ArgumentLoc))
-      return false;
+      Begin = Begin.getLocWithOffset(1);
+    }
+  }
 
   return true;
 }
@@ -539,13 +518,13 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc,
   while (L.isMacroID()) {
     // If this is the expansion of a macro argument, point the caret at the
     // use of the argument in the definition of the macro, not the expansion.
-    if (SM.isMacroArgExpansion(L))
+    if (SM.isMacroArgExpansion(L)) {
       LocationStack.push_back(SM.getImmediateExpansionRange(L).getBegin());
-    else
-      LocationStack.push_back(L);
 
-    if (checkRangesForMacroArgExpansion(FullSourceLoc(L, SM), Ranges))
-      IgnoredEnd = LocationStack.size();
+      if (rangesInsideSameMacroArgExpansion(FullSourceLoc(L, SM), Ranges))
+        IgnoredEnd = LocationStack.size();
+    } else
+      LocationStack.push_back(L);
 
     L = SM.getImmediateMacroCallerLoc(L);
 
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 01117e7785bc4..9cd1a7e45fb64 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -2376,20 +2376,22 @@ Sema::DecomposeUnqualifiedId(const UnqualifiedId &Id,
   }
 }
 
-static void emitEmptyLookupTypoDiagnostic(
-    const TypoCorrection &TC, Sema &SemaRef, const CXXScopeSpec &SS,
-    DeclarationName Typo, SourceLocation TypoLoc, ArrayRef<Expr *> Args,
-    unsigned DiagnosticID, unsigned DiagnosticSuggestID) {
+static void emitEmptyLookupTypoDiagnostic(const TypoCorrection &TC,
+                                          Sema &SemaRef, const CXXScopeSpec &SS,
+                                          DeclarationName Typo,
+                                          SourceRange TypoRange,
+                                          unsigned DiagnosticID,
+                                          unsigned DiagnosticSuggestID) {
   DeclContext *Ctx =
       SS.isEmpty() ? nullptr : SemaRef.computeDeclContext(SS, false);
   if (!TC) {
     // Emit a special diagnostic for failed member lookups.
     // FIXME: computing the declaration context might fail here (?)
     if (Ctx)
-      SemaRef.Diag(TypoLoc, diag::err_no_member) << Typo << Ctx
-                                                 << SS.getRange();
+      SemaRef.Diag(TypoRange.getBegin(), diag::err_no_member)
+          << Typo << Ctx << TypoRange;
     else
-      SemaRef.Diag(TypoLoc, DiagnosticID) << Typo;
+      SemaRef.Diag(TypoRange.getBegin(), DiagnosticID) << Typo << TypoRange;
     return;
   }
 
@@ -2400,12 +2402,13 @@ static void emitEmptyLookupTypoDiagnostic(
                         ? diag::note_implicit_param_decl
                         : diag::note_previous_decl;
   if (!Ctx)
-    SemaRef.diagnoseTypo(TC, SemaRef.PDiag(DiagnosticSuggestID) << Typo,
-                         SemaRef.PDiag(NoteID));
+    SemaRef.diagnoseTypo(
+        TC, SemaRef.PDiag(DiagnosticSuggestID) << Typo << TypoRange,
+        SemaRef.PDiag(NoteID));
   else
-    SemaRef.diagnoseTypo(TC, SemaRef.PDiag(diag::err_no_member_suggest)
-                                 << Typo << Ctx << DroppedSpecifier
-                                 << SS.getRange(),
+    SemaRef.diagnoseTypo(TC,
+                         SemaRef.PDiag(diag::err_no_member_suggest)
+                             << Typo << Ctx << DroppedSpecifier << TypoRange,
                          SemaRef.PDiag(NoteID));
 }
 
@@ -2474,6 +2477,7 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
                                ArrayRef<Expr *> Args, DeclContext *LookupCtx,
                                TypoExpr **Out) {
   DeclarationName Name = R.getLookupName();
+  SourceRange NameRange = R.getLookupNameInfo().getSourceRange();
 
   unsigned diagnostic = diag::err_undeclared_var_use;
   unsigned diagnostic_suggest = diag::err_undeclared_var_use_suggest;
@@ -2531,13 +2535,12 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
   // We didn't find anything, so try to correct for a typo.
   TypoCorrection Corrected;
   if (S && Out) {
-    SourceLocation TypoLoc = R.getNameLoc();
     assert(!ExplicitTemplateArgs &&
            "Diagnosing an empty lookup with explicit template args!");
     *Out = CorrectTypoDelayed(
         R.getLookupNameInfo(), R.getLookupKind(), S, &SS, CCC,
         [=](const TypoCorrection &TC) {
-          emitEmptyLookupTypoDiagnostic(TC, *this, SS, Name, TypoLoc, Args,
+          emitEmptyLookupTypoDiagnostic(TC, *this, SS, Name, NameRange,
                                         diagnostic, diagnostic_suggest);
         },
         nullptr, CTK_ErrorRecovery, LookupCtx);
@@ -2616,12 +2619,13 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
                             ? diag::note_implicit_param_decl
                             : diag::note_previous_decl;
       if (SS.isEmpty())
-        diagnoseTypo(Corrected, PDiag(diagnostic_suggest) << Name,
+        diagnoseTypo(Corrected, PDiag(diagnostic_suggest) << Name << NameRange,
                      PDiag(NoteID), AcceptableWithRecovery);
       else
-        diagnoseTypo(Corrected, PDiag(diag::err_no_member_suggest)
-                                  << Name << computeDeclContext(SS, false)
-                                  << DroppedSpecifier << SS.getRange(),
+        diagnoseTypo(Corrected,
+                     PDiag(diag::err_no_member_suggest)
+                         << Name << computeDeclContext(SS, false)
+                         << DroppedSpecifier << NameRange,
                      PDiag(NoteID), AcceptableWithRecovery);
 
       // Tell the callee whether to try to recover.
@@ -2634,13 +2638,12 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
   // FIXME: computing the declaration context might fail here (?)
   if (!SS.isEmpty()) {
     Diag(R.getNameLoc(), diag::err_no_member)
-      << Name << computeDeclContext(SS, false)
-      << SS.getRange();
+        << Name << computeDeclContext(SS, false) << NameRange;
     return true;
   }
 
   // Give up, we can't recover.
-  Diag(R.getNameLoc(), diagnostic) << Name;
+  Diag(R.getNameLoc(), diagnostic) << Name << NameRange;
   return true;
 }
 

From e0a6905287050d57ea0413cba7f011803b1f65ef Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Fri, 18 Apr 2025 09:06:21 +0200
Subject: [PATCH 381/710] [libc++] Simplify the generic implementation of
 is_{un}signed (#136095)

---
 libcxx/include/__type_traits/is_signed.h   | 15 ++++-----------
 libcxx/include/__type_traits/is_unsigned.h | 15 ++++-----------
 2 files changed, 8 insertions(+), 22 deletions(-)

diff --git a/libcxx/include/__type_traits/is_signed.h b/libcxx/include/__type_traits/is_signed.h
index 4aae921f293c9..02f51f9cc9b19 100644
--- a/libcxx/include/__type_traits/is_signed.h
+++ b/libcxx/include/__type_traits/is_signed.h
@@ -12,7 +12,6 @@
 #include <__config>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_arithmetic.h>
-#include <__type_traits/is_integral.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -32,24 +31,18 @@ _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_signed_v = __is_signed(_Tp);
 
 #else // __has_builtin(__is_signed)
 
-template <class _Tp, bool = is_integral<_Tp>::value>
-struct __libcpp_is_signed_impl : _BoolConstant<(_Tp(-1) < _Tp(0))> {};
-
-template <class _Tp>
-struct __libcpp_is_signed_impl<_Tp, false> : true_type {}; // floating point
-
 template <class _Tp, bool = is_arithmetic<_Tp>::value>
-struct __libcpp_is_signed : __libcpp_is_signed_impl<_Tp> {};
+inline constexpr bool __is_signed_v = false;
 
 template <class _Tp>
-struct __libcpp_is_signed<_Tp, false> : false_type {};
+inline constexpr bool __is_signed_v<_Tp, true> = _Tp(-1) < _Tp(0);
 
 template <class _Tp>
-struct is_signed : __libcpp_is_signed<_Tp> {};
+struct is_signed : integral_constant<bool, __is_signed_v<_Tp>> {};
 
 #  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
-inline constexpr bool is_signed_v = is_signed<_Tp>::value;
+inline constexpr bool is_signed_v = __is_signed_v<_Tp>;
 #  endif
 
 #endif // __has_builtin(__is_signed)
diff --git a/libcxx/include/__type_traits/is_unsigned.h b/libcxx/include/__type_traits/is_unsigned.h
index 900ff969eae0e..b66027c8d7791 100644
--- a/libcxx/include/__type_traits/is_unsigned.h
+++ b/libcxx/include/__type_traits/is_unsigned.h
@@ -11,7 +11,6 @@
 
 #include <__config>
 #include <__type_traits/integral_constant.h>
-#include <__type_traits/is_arithmetic.h>
 #include <__type_traits/is_integral.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -33,23 +32,17 @@ _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_unsigned_v = __is_unsigned(_
 #else // __has_builtin(__is_unsigned)
 
 template <class _Tp, bool = is_integral<_Tp>::value>
-struct __libcpp_is_unsigned_impl : public _BoolConstant<(_Tp(0) < _Tp(-1))> {};
+inline constexpr bool __is_unsigned_v = false;
 
 template <class _Tp>
-struct __libcpp_is_unsigned_impl<_Tp, false> : public false_type {}; // floating point
-
-template <class _Tp, bool = is_arithmetic<_Tp>::value>
-struct __libcpp_is_unsigned : public __libcpp_is_unsigned_impl<_Tp> {};
-
-template <class _Tp>
-struct __libcpp_is_unsigned<_Tp, false> : public false_type {};
+inline constexpr bool __is_unsigned_v<_Tp, true> = _Tp(0) < _Tp(-1);
 
 template <class _Tp>
-struct is_unsigned : public __libcpp_is_unsigned<_Tp> {};
+struct is_unsigned : integral_constant<bool, __is_unsigned_v<_Tp>> {};
 
 #  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
-inline constexpr bool is_unsigned_v = is_unsigned<_Tp>::value;
+inline constexpr bool is_unsigned_v = __is_unsigned_v<_Tp>;
 #  endif
 
 #endif // __has_builtin(__is_unsigned)

From f4c76bba5909e9c3e542ac723b9a1c0f1c229e79 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 00:15:13 -0700
Subject: [PATCH 382/710] [clang] Use llvm::append_range (NFC) (#136256)

This patch replaces:

  llvm::copy(Src, std::back_inserter(Dst));

with:

  llvm::append_range(Dst, Src);

for breavity.

One side benefit is that llvm::append_range eventually calls
llvm::SmallVector::reserve if Dst is of llvm::SmallVector.
---
 clang/lib/CodeGen/Targets/AArch64.cpp                   | 4 ++--
 clang/lib/Driver/XRayArgs.cpp                           | 4 ++--
 clang/lib/Sema/SemaConcept.cpp                          | 4 ++--
 clang/lib/Sema/SemaSYCL.cpp                             | 2 +-
 clang/tools/clang-installapi/ClangInstallAPI.cpp        | 2 +-
 clang/tools/clang-installapi/Options.cpp                | 4 ++--
 clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp | 2 +-
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index 073ca3cc82690..77e225d197343 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -699,7 +699,7 @@ bool AArch64ABIInfo::passAsPureScalableType(
       return false;
 
     for (uint64_t I = 0; I < NElt; ++I)
-      llvm::copy(EltCoerceToSeq, std::back_inserter(CoerceToSeq));
+      llvm::append_range(CoerceToSeq, EltCoerceToSeq);
 
     NVec += NElt * NV;
     NPred += NElt * NP;
@@ -818,7 +818,7 @@ void AArch64ABIInfo::flattenType(
     flattenType(AT->getElementType(), EltFlattened);
 
     for (uint64_t I = 0; I < NElt; ++I)
-      llvm::copy(EltFlattened, std::back_inserter(Flattened));
+      llvm::append_range(Flattened, EltFlattened);
     return;
   }
 
diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp
index f8c213334a2b4..6649ed14c3982 100644
--- a/clang/lib/Driver/XRayArgs.cpp
+++ b/clang/lib/Driver/XRayArgs.cpp
@@ -157,7 +157,7 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
   // Get the list of modes we want to support.
   auto SpecifiedModes = Args.getAllArgValues(options::OPT_fxray_modes);
   if (SpecifiedModes.empty())
-    llvm::copy(XRaySupportedModes, std::back_inserter(Modes));
+    llvm::append_range(Modes, XRaySupportedModes);
   else
     for (const auto &Arg : SpecifiedModes) {
       // Parse CSV values for -fxray-modes=...
@@ -167,7 +167,7 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
         if (M == "none")
           Modes.clear();
         else if (M == "all")
-          llvm::copy(XRaySupportedModes, std::back_inserter(Modes));
+          llvm::append_range(Modes, XRaySupportedModes);
         else
           Modes.push_back(std::string(M));
     }
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 73bc8731af57d..5424d207d3c77 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -2012,8 +2012,8 @@ FormulaType SubsumptionChecker::Normalize(const NormalizedConstraint &NC) {
     for (const auto &RTransform : Right) {
       Clause Combined;
       Combined.reserve(LTransform.size() + RTransform.size());
-      llvm::copy(LTransform, std::back_inserter(Combined));
-      llvm::copy(RTransform, std::back_inserter(Combined));
+      llvm::append_range(Combined, LTransform);
+      llvm::append_range(Combined, RTransform);
       Add(std::move(Combined));
     }
   }
diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index ddd92782366b5..1969d7b0ba837 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -135,7 +135,7 @@ void SemaSYCL::deepTypeCheckForDevice(SourceLocation UsedAt,
       // When nullptr is discovered, this means we've gone back up a level, so
       // the history should be cleaned.
       StackForRecursion.push_back(nullptr);
-      llvm::copy(RecDecl->fields(), std::back_inserter(StackForRecursion));
+      llvm::append_range(StackForRecursion, RecDecl->fields());
     }
   } while (!StackForRecursion.empty());
 }
diff --git a/clang/tools/clang-installapi/ClangInstallAPI.cpp b/clang/tools/clang-installapi/ClangInstallAPI.cpp
index ce6240b1b56f1..37b69ccf4e00e 100644
--- a/clang/tools/clang-installapi/ClangInstallAPI.cpp
+++ b/clang/tools/clang-installapi/ClangInstallAPI.cpp
@@ -59,7 +59,7 @@ static bool runFrontend(StringRef ProgName, Twine Label, bool Verbose,
   // headers.
   std::vector<std::string> Args = {ProgName.data(), "-target",
                                    Ctx.Slice->getTriple().str().c_str()};
-  llvm::copy(InitialArgs, std::back_inserter(Args));
+  llvm::append_range(Args, InitialArgs);
   Args.push_back(InputFile);
 
   // Create & run invocation.
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index 0dddcfce58ca6..efb36848dbbf1 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -691,7 +691,7 @@ Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
     if (A->getOption().getID() > (unsigned)OPT_UNKNOWN) {
       ClangDriverArgs.push_back(A->getSpelling().data());
     } else
-      llvm::copy(A->getValues(), std::back_inserter(ClangDriverArgs));
+      llvm::append_range(ClangDriverArgs, A->getValues());
   }
   return ClangDriverArgs;
 }
@@ -751,7 +751,7 @@ Options::Options(DiagnosticsEngine &Diag, FileManager *FM,
     if (A->isClaimed())
       continue;
     FrontendArgs.emplace_back(A->getSpelling());
-    llvm::copy(A->getValues(), std::back_inserter(FrontendArgs));
+    llvm::append_range(FrontendArgs, A->getValues());
   }
 }
 
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 4e716e818a232..bdeaa2031d184 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -541,7 +541,7 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args,
       else
         Arg->render(Args, LinkerArgs);
     }
-    llvm::copy(LinkerArgs, std::back_inserter(CmdArgs));
+    llvm::append_range(CmdArgs, LinkerArgs);
   }
 
   // Pass on -mllvm options to the linker invocation.

From a1583522948f966d2393e84befdc2f7fea3bf7b4 Mon Sep 17 00:00:00 2001
From: Yanzuo Liu <zwuis@outlook.com>
Date: Fri, 18 Apr 2025 15:27:23 +0800
Subject: [PATCH 383/710] [Clang][GitHub][NFC] Auto-add clang:bytecode label
 for PR (#136148)

---
 .github/new-prs-labeler.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml
index c0c61748010d0..1ec159bfaa090 100644
--- a/.github/new-prs-labeler.yml
+++ b/.github/new-prs-labeler.yml
@@ -7,6 +7,12 @@ ClangIR:
   - clang/tools/cir-*/**/*
   - clang/test/CIR/**/*
 
+clang:bytecode:
+  - clang/docs/ConstantInterpreter.rst
+  - clang/lib/AST/ByteCode/**/*
+  - clang/test/AST/ByteCode/**/*
+  - clang/unittests/AST/ByteCode/**/*
+
 clang:dataflow:
   - clang/include/clang/Analysis/FlowSensitive/**/*
   - clang/lib/Analysis/FlowSensitive/**/*

From 802e7309c0ab7e9966f8a98eede0231d146c6a97 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Fri, 18 Apr 2025 09:33:20 +0200
Subject: [PATCH 384/710] [lldb] Fix TestExprDiagnostics test (#136269)

Add missing source ranges to the diagnostic output.
---
 .../commands/expression/diagnostics/TestExprDiagnostics.py  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py b/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
index b476a807c6dc0..0cc505aedc4be 100644
--- a/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
+++ b/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
@@ -33,7 +33,7 @@ def test_source_and_caret_printing(self):
         self.assertIn(
             """
     1 | unknown_identifier
-      | ^
+      | ^~~~~~~~~~~~~~~~~~
 """,
             value.GetError().GetCString(),
         )
@@ -45,7 +45,7 @@ def test_source_and_caret_printing(self):
         self.assertIn(
             """
     1 | 1 + unknown_identifier
-      |     ^
+      |     ^~~~~~~~~~~~~~~~~~
 """,
             value.GetError().GetCString(),
         )
@@ -57,7 +57,7 @@ def test_source_and_caret_printing(self):
         self.assertIn(
             """
     2 | foobar +=1;
-      | ^
+      | ^~~~~~
 """,
             value.GetError().GetCString(),
         )

From d1496313d7e2543f1506f71fbfd28a8e8d8a3d6e Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Fri, 18 Apr 2025 10:50:42 +0300
Subject: [PATCH 385/710] [CodeGen] Add another method to CFIInstBuilder
 (#136270)

Mainly for use by downstream targets, but it can find applications in
upstream code as well. Use it in MSP430 so that it doesn't look dead.
---
 llvm/include/llvm/CodeGen/CFIInstBuilder.h     | 4 ++++
 llvm/lib/Target/MSP430/MSP430FrameLowering.cpp | 6 +++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/CFIInstBuilder.h b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
index bf7102eeea497..86e125f11e0c6 100644
--- a/llvm/include/llvm/CodeGen/CFIInstBuilder.h
+++ b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
@@ -71,6 +71,10 @@ class CFIInstBuilder {
     insertCFIInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
   }
 
+  void buildAdjustCFAOffset(int64_t Adjustment) const {
+    insertCFIInst(MCCFIInstruction::createAdjustCfaOffset(nullptr, Adjustment));
+  }
+
   void buildOffset(MCRegister Reg, int64_t Offset) const {
     insertCFIInst(MCCFIInstruction::createOffset(
         nullptr, TRI.getDwarfRegNum(Reg, IsEH), Offset));
diff --git a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
index 39f4324286efd..bc24b3239b43a 100644
--- a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
@@ -14,6 +14,7 @@
 #include "MSP430InstrInfo.h"
 #include "MSP430MachineFunctionInfo.h"
 #include "MSP430Subtarget.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -406,9 +407,8 @@ MachineBasicBlock::iterator MSP430FrameLowering::eliminateCallFramePseudoInstr(
               .addReg(MSP430::SP)
               .addImm(CalleeAmt);
       if (!hasFP(MF)) {
-        DebugLoc DL = I->getDebugLoc();
-        BuildCFI(MBB, I, DL,
-                 MCCFIInstruction::createAdjustCfaOffset(nullptr, CalleeAmt));
+        CFIInstBuilder(MBB, I, MachineInstr::NoFlags)
+            .buildAdjustCFAOffset(CalleeAmt);
       }
       // The SRW implicit def is dead.
       New->getOperand(3).setIsDead();

From a99c978d1b35e30d9a0fe9db68b91e9f2815c8e9 Mon Sep 17 00:00:00 2001
From: Corentin Jabot <corentinjabot@gmail.com>
Date: Fri, 18 Apr 2025 10:00:00 +0200
Subject: [PATCH 386/710] [Clang] Avoid dereferencing an invalid iterator

Fix msan builds after 8c5a307bd8
https://lab.llvm.org/buildbot/#/builders/94/builds/6321
---
 clang/include/clang/Sema/Overload.h |  3 +++
 clang/lib/Sema/SemaOverload.cpp     | 16 ++++++++++------
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 55d714c1222a1..8182ce9c39685 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -1350,6 +1350,9 @@ class Sema;
     iterator end() { return Candidates.end(); }
 
     size_t size() const { return Candidates.size() + DeferredCandidatesCount; }
+
+    size_t nonDeferredCandidatesCount() const { return Candidates.size(); }
+
     bool empty() const {
       return Candidates.empty() && DeferredCandidatesCount == 0;
     }
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index b7a981e08ead9..e4ff8c5489df3 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1108,7 +1108,7 @@ bool OverloadCandidateSet::OperatorRewriteInfo::shouldAddReversed(
 }
 
 void OverloadCandidateSet::destroyCandidates() {
-  for (iterator i = begin(), e = end(); i != e; ++i) {
+  for (iterator i = Candidates.begin(), e = Candidates.end(); i != e; ++i) {
     for (auto &C : i->Conversions)
       C.~ImplicitConversionSequence();
     if (!i->Viable && i->FailureKind == ovl_fail_bad_deduction)
@@ -11237,7 +11237,7 @@ void OverloadCandidateSet::PerfectViableFunction(
     Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
 
   Best = end();
-  for (auto It = begin(); It != end(); ++It) {
+  for (auto It = Candidates.begin(); It != Candidates.end(); ++It) {
 
     if (!It->isPerfectMatch(S.getASTContext()))
       continue;
@@ -11277,7 +11277,8 @@ OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
 
   llvm::SmallVector<OverloadCandidate *, 16> Candidates;
   Candidates.reserve(this->Candidates.size());
-  std::transform(begin(), end(), std::back_inserter(Candidates),
+  std::transform(this->Candidates.begin(), this->Candidates.end(),
+                 std::back_inserter(Candidates),
                  [](OverloadCandidate &Cand) { return &Cand; });
 
   if (S.getLangOpts().CUDA)
@@ -13050,7 +13051,8 @@ SmallVector<OverloadCandidate *, 32> OverloadCandidateSet::CompleteCandidates(
   // be prohibitive, so we make a set of pointers and sort those.
   SmallVector<OverloadCandidate*, 32> Cands;
   if (OCD == OCD_AllCandidates) Cands.reserve(size());
-  for (iterator Cand = begin(), LastCand = end(); Cand != LastCand; ++Cand) {
+  for (iterator Cand = Candidates.begin(), LastCand = Candidates.end();
+       Cand != LastCand; ++Cand) {
     if (!Filter(*Cand))
       continue;
     switch (OCD) {
@@ -13127,7 +13129,8 @@ void OverloadCandidateSet::NoteCandidates(
     NoteCandidates(S, Args, Cands, Opc, OpLoc);
 
   if (OCD == OCD_AmbiguousCandidates)
-    MaybeDiagnoseAmbiguousConstraints(S, {begin(), end()});
+    MaybeDiagnoseAmbiguousConstraints(S,
+                                      {Candidates.begin(), Candidates.end()});
 }
 
 void OverloadCandidateSet::NoteCandidates(Sema &S, ArrayRef<Expr *> Args,
@@ -16255,7 +16258,8 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
   // we filter them out to produce better error diagnostics, ie to avoid
   // showing 2 failed overloads instead of one.
   bool IgnoreSurrogateFunctions = false;
-  if (CandidateSet.size() == 1 && Record->getAsCXXRecordDecl()->isLambda()) {
+  if (CandidateSet.nonDeferredCandidatesCount() == 1 &&
+      Record->getAsCXXRecordDecl()->isLambda()) {
     const OverloadCandidate &Candidate = *CandidateSet.begin();
     if (!Candidate.Viable &&
         Candidate.FailureKind == ovl_fail_constraints_not_satisfied)

From 5db95fd6ca8cae62c6ab6acf17b00c3417d2018d Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 01:07:05 -0700
Subject: [PATCH 387/710] [memprof] Avoid repeated hash lookups (NFC) (#136268)

Note that we don't have to worry about CallstackProfileData[Id]
default-constructing the value side of a new map entry.  If that
happens, AccessHistogramSize > 0 wouldn't be true, and the new map
entry gets deleted right away.
---
 llvm/lib/ProfileData/MemProfReader.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/ProfileData/MemProfReader.cpp b/llvm/lib/ProfileData/MemProfReader.cpp
index c57f9b22273d4..e0f280b9eb2f6 100644
--- a/llvm/lib/ProfileData/MemProfReader.cpp
+++ b/llvm/lib/ProfileData/MemProfReader.cpp
@@ -600,9 +600,12 @@ Error RawMemProfReader::symbolizeAndFilterStackFrames(
   // Drop the entries where the callstack is empty.
   for (const uint64_t Id : EntriesToErase) {
     StackMap.erase(Id);
-    if (CallstackProfileData[Id].AccessHistogramSize > 0)
-      free((void *)CallstackProfileData[Id].AccessHistogram);
-    CallstackProfileData.erase(Id);
+    if (auto It = CallstackProfileData.find(Id);
+        It != CallstackProfileData.end()) {
+      if (It->second.AccessHistogramSize > 0)
+        free((void *)It->second.AccessHistogram);
+      CallstackProfileData.erase(It);
+    }
   }
 
   if (StackMap.empty())

From 64ffecfc433d43a15e3d1d51886ff0e0a876a547 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 18 Apr 2025 09:24:02 +0100
Subject: [PATCH 388/710] [DAG] isKnownNeverNaN - add DemandedElts element mask
 to isKnownNeverNaN calls (#135952)

Matches what we've done for computeKnownBits etc. to improve vector handling
---
 llvm/include/llvm/CodeGen/SelectionDAG.h      | 14 ++++
 llvm/include/llvm/CodeGen/TargetLowering.h    |  1 +
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 82 +++++++++++++------
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |  1 +
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |  7 +-
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h   |  5 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  5 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.h       |  5 +-
 llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll        | 33 ++++----
 9 files changed, 101 insertions(+), 52 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 63423463eeee2..2ab6b4d3027e9 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -2142,11 +2142,25 @@ class SelectionDAG {
   ///     X|Cst == X+Cst iff X&Cst = 0.
   bool isBaseWithConstantOffset(SDValue Op) const;
 
+  /// Test whether the given SDValue (or all elements of it, if it is a
+  /// vector) is known to never be NaN in \p DemandedElts. If \p SNaN is true,
+  /// returns if \p Op is known to never be a signaling NaN (it may still be a
+  /// qNaN).
+  bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN = false,
+                       unsigned Depth = 0) const;
+
   /// Test whether the given SDValue (or all elements of it, if it is a
   /// vector) is known to never be NaN. If \p SNaN is true, returns if \p Op is
   /// known to never be a signaling NaN (it may still be a qNaN).
   bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;
 
+  /// \returns true if \p Op is known to never be a signaling NaN in \p
+  /// DemandedElts.
+  bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts,
+                        unsigned Depth = 0) const {
+    return isKnownNeverNaN(Op, DemandedElts, true, Depth);
+  }
+
   /// \returns true if \p Op is known to never be a signaling NaN.
   bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
     return isKnownNeverNaN(Op, true, Depth);
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 0a36975f4f625..00c36266a069f 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4283,6 +4283,7 @@ class TargetLowering : public TargetLoweringBase {
   /// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling
   /// NaN.
   virtual bool isKnownNeverNaNForTargetNode(SDValue Op,
+                                            const APInt &DemandedElts,
                                             const SelectionDAG &DAG,
                                             bool SNaN = false,
                                             unsigned Depth = 0) const;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index b45369ecf7bfd..f502a536d43b1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5601,7 +5601,24 @@ bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
          (Op.getOpcode() == ISD::ADD || isADDLike(Op));
 }
 
-bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
+bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
+                                   unsigned Depth) const {
+  EVT VT = Op.getValueType();
+
+  // Since the number of lanes in a scalable vector is unknown at compile time,
+  // we track one bit which is implicitly broadcast to all lanes.  This means
+  // that all lanes in a scalable vector are considered demanded.
+  APInt DemandedElts = VT.isFixedLengthVector()
+                           ? APInt::getAllOnes(VT.getVectorNumElements())
+                           : APInt(1, 1);
+
+  return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
+}
+
+bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
+                                   bool SNaN, unsigned Depth) const {
+  assert(!DemandedElts.isZero() && "No demanded elements");
+
   // If we're told that NaNs won't happen, assume they won't.
   if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
     return true;
@@ -5657,21 +5674,21 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
   case ISD::FLDEXP: {
     if (SNaN)
       return true;
-    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
+    return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
   }
   case ISD::FABS:
   case ISD::FNEG:
   case ISD::FCOPYSIGN: {
-    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
+    return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
   }
   case ISD::SELECT:
-    return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
-           isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
+    return isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1) &&
+           isKnownNeverNaN(Op.getOperand(2), DemandedElts, SNaN, Depth + 1);
   case ISD::FP_EXTEND:
   case ISD::FP_ROUND: {
     if (SNaN)
       return true;
-    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
+    return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
   }
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:
@@ -5693,8 +5710,8 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
   case ISD::FMAXIMUMNUM: {
     // Only one needs to be known not-nan, since it will be returned if the
     // other ends up being one.
-    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
-           isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
+    return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) ||
+           isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
   }
   case ISD::FMINNUM_IEEE:
   case ISD::FMAXNUM_IEEE: {
@@ -5702,33 +5719,52 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
       return true;
     // This can return a NaN if either operand is an sNaN, or if both operands
     // are NaN.
-    return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
-            isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
-           (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
-            isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
+    return (isKnownNeverNaN(Op.getOperand(0), DemandedElts, false, Depth + 1) &&
+            isKnownNeverSNaN(Op.getOperand(1), DemandedElts, Depth + 1)) ||
+           (isKnownNeverNaN(Op.getOperand(1), DemandedElts, false, Depth + 1) &&
+            isKnownNeverSNaN(Op.getOperand(0), DemandedElts, Depth + 1));
   }
   case ISD::FMINIMUM:
   case ISD::FMAXIMUM: {
     // TODO: Does this quiet or return the origina NaN as-is?
-    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
-           isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
+    return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) &&
+           isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
+  }
+  case ISD::EXTRACT_VECTOR_ELT: {
+    SDValue Src = Op.getOperand(0);
+    auto *Idx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
+    EVT SrcVT = Src.getValueType();
+    if (SrcVT.isFixedLengthVector() && Idx &&
+        Idx->getAPIntValue().ult(SrcVT.getVectorNumElements())) {
+      APInt DemandedSrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
+                                                  Idx->getZExtValue());
+      return isKnownNeverNaN(Src, DemandedSrcElts, SNaN, Depth + 1);
+    }
+    return isKnownNeverNaN(Src, SNaN, Depth + 1);
   }
-  case ISD::EXTRACT_VECTOR_ELT:
   case ISD::EXTRACT_SUBVECTOR: {
-    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
+    SDValue Src = Op.getOperand(0);
+    if (Src.getValueType().isFixedLengthVector()) {
+      unsigned Idx = Op.getConstantOperandVal(1);
+      unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+      APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
+      return isKnownNeverNaN(Src, DemandedSrcElts, SNaN, Depth + 1);
+    }
+    return isKnownNeverNaN(Src, SNaN, Depth + 1);
   }
   case ISD::BUILD_VECTOR: {
-    for (const SDValue &Opnd : Op->ops())
-      if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1))
+    unsigned NumElts = Op.getNumOperands();
+    for (unsigned I = 0; I != NumElts; ++I)
+      if (DemandedElts[I] &&
+          !isKnownNeverNaN(Op.getOperand(I), SNaN, Depth + 1))
         return false;
     return true;
   }
   default:
-    if (Opcode >= ISD::BUILTIN_OP_END ||
-        Opcode == ISD::INTRINSIC_WO_CHAIN ||
-        Opcode == ISD::INTRINSIC_W_CHAIN ||
-        Opcode == ISD::INTRINSIC_VOID) {
-      return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
+    if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
+        Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
+      return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, *this, SNaN,
+                                               Depth);
     }
 
     return false;
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 5308593b5c988..3995216e3d689 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -3928,6 +3928,7 @@ bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
 }
 
 bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
+                                                  const APInt &DemandedElts,
                                                   const SelectionDAG &DAG,
                                                   bool SNaN,
                                                   unsigned Depth) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 533ad349f7500..2846405a2538c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5971,10 +5971,9 @@ unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
   }
 }
 
-bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
-                                                        const SelectionDAG &DAG,
-                                                        bool SNaN,
-                                                        unsigned Depth) const {
+bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
+    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
+    unsigned Depth) const {
   unsigned Opcode = Op.getOpcode();
   switch (Opcode) {
   case AMDGPUISD::FMIN_LEGACY:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 6705f86e15fc2..fa9d61ec37c24 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -321,9 +321,8 @@ class AMDGPUTargetLowering : public TargetLowering {
                                             const MachineRegisterInfo &MRI,
                                             unsigned Depth = 0) const override;
 
-  bool isKnownNeverNaNForTargetNode(SDValue Op,
-                                    const SelectionDAG &DAG,
-                                    bool SNaN = false,
+  bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
+                                    const SelectionDAG &DAG, bool SNaN = false,
                                     unsigned Depth = 0) const override;
 
   bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 67f2cf2c862f6..724a45062c1f4 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16684,6 +16684,7 @@ bool SITargetLowering::denormalsEnabledForType(
 }
 
 bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
+                                                    const APInt &DemandedElts,
                                                     const SelectionDAG &DAG,
                                                     bool SNaN,
                                                     unsigned Depth) const {
@@ -16696,8 +16697,8 @@ bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
     return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
   }
 
-  return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DAG, SNaN,
-                                                            Depth);
+  return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
+                                                            DAG, SNaN, Depth);
 }
 
 // On older subtargets, global FP atomic instructions have a hardcoded FP mode
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index dc0634331caf9..c42366a1c04c8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -546,9 +546,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
 
   bool isProfitableToHoist(Instruction *I) const override;
 
-  bool isKnownNeverNaNForTargetNode(SDValue Op,
-                                    const SelectionDAG &DAG,
-                                    bool SNaN = false,
+  bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
+                                    const SelectionDAG &DAG, bool SNaN = false,
                                     unsigned Depth = 0) const override;
   AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
   AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index beac41e42e0c6..ef325da272005 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -1057,54 +1057,53 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s
 ; SDAG-GFX1100-TRUE16-LABEL: v_mad_mix_v3f32_clamp_postcvt:
 ; SDAG-GFX1100-TRUE16:       ; %bb.0:
 ; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
 ; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
 ; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
 ; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v6.l
-; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.l
-; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v2, v3, v5, v4 op_sel_hi:[1,1,1]
-; SDAG-GFX1100-TRUE16-NEXT:    v_pack_b32_f16 v1, v0.l, 0
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_pack_b32_f16 v1, v1.l, 0
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v3, v3, v5, v6 op_sel_hi:[1,1,1] clamp
 ; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-TRUE16-NEXT:    v_pk_max_f16 v0, v2, v2 clamp
 ; SDAG-GFX1100-TRUE16-NEXT:    v_pk_max_f16 v1, v1, v1 clamp
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b32_e32 v0, v3
 ; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX1100-FAKE16-LABEL: v_mad_mix_v3f32_clamp_postcvt:
 ; SDAG-GFX1100-FAKE16:       ; %bb.0:
 ; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
 ; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp
 ; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
 ; SDAG-GFX1100-FAKE16-NEXT:    v_pack_b32_f16 v1, v1, 0
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
 ; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX1100-FAKE16-NEXT:    v_pk_max_f16 v0, v6, v6 clamp
 ; SDAG-GFX1100-FAKE16-NEXT:    v_pk_max_f16 v1, v1, v1 clamp
+; SDAG-GFX1100-FAKE16-NEXT:    v_mov_b32_e32 v0, v3
 ; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX900-LABEL: v_mad_mix_v3f32_clamp_postcvt:
 ; SDAG-GFX900:       ; %bb.0:
 ; SDAG-GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX900-NEXT:    v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
 ; SDAG-GFX900-NEXT:    v_mad_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX900-NEXT:    v_mad_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp
 ; SDAG-GFX900-NEXT:    v_pack_b32_f16 v1, v1, 0
-; SDAG-GFX900-NEXT:    v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; SDAG-GFX900-NEXT:    v_pk_max_f16 v0, v6, v6 clamp
+; SDAG-GFX900-NEXT:    v_mad_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
 ; SDAG-GFX900-NEXT:    v_pk_max_f16 v1, v1, v1 clamp
+; SDAG-GFX900-NEXT:    v_mov_b32_e32 v0, v3
 ; SDAG-GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-GFX906-LABEL: v_mad_mix_v3f32_clamp_postcvt:
 ; SDAG-GFX906:       ; %bb.0:
 ; SDAG-GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX906-NEXT:    v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
 ; SDAG-GFX906-NEXT:    v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; SDAG-GFX906-NEXT:    v_fma_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp
 ; SDAG-GFX906-NEXT:    v_pack_b32_f16 v1, v1, 0
-; SDAG-GFX906-NEXT:    v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; SDAG-GFX906-NEXT:    v_pk_max_f16 v0, v6, v6 clamp
+; SDAG-GFX906-NEXT:    v_fma_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
 ; SDAG-GFX906-NEXT:    v_pk_max_f16 v1, v1, v1 clamp
+; SDAG-GFX906-NEXT:    v_mov_b32_e32 v0, v3
 ; SDAG-GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SDAG-VI-LABEL: v_mad_mix_v3f32_clamp_postcvt:

From c7daab259c3281cf8f649583993bad2536febc02 Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Fri, 18 Apr 2025 16:27:27 +0800
Subject: [PATCH 389/710] [Clang] Fix the trailing comma regression (#136273)

925e195 introduced a regression since which we started to accept invalid
trailing commas in many expression lists where they're not allowed by
the grammar. The issue came from the fact that an additional invalid
state - previously handled by ParseExpressionList - was overlooked in
that patch.

Fixes https://github.com/llvm/llvm-project/issues/136254

No release entry because I want to backport it.
---
 clang/lib/Parse/ParseExpr.cpp  |  3 ---
 clang/test/Parser/recovery.cpp | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 1416d52157dca..3e3e7cbcd68b2 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -2239,8 +2239,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
             if (PP.isCodeCompletionReached() && !CalledSignatureHelp)
               RunSignatureHelp();
             LHS = ExprError();
-          } else if (!HasError && HasTrailingComma) {
-            Diag(Tok, diag::err_expected_expression);
           } else if (LHS.isInvalid()) {
             for (auto &E : ArgExprs)
               Actions.CorrectDelayedTyposInExpr(E);
@@ -3750,7 +3748,6 @@ bool Parser::ParseExpressionList(SmallVectorImpl<Expr *> &Exprs,
     if (Tok.is(tok::r_paren)) {
       if (HasTrailingComma)
         *HasTrailingComma = true;
-      break;
     }
   }
   if (SawError) {
diff --git a/clang/test/Parser/recovery.cpp b/clang/test/Parser/recovery.cpp
index 2fce67a52c6b6..261f5dc99bad4 100644
--- a/clang/test/Parser/recovery.cpp
+++ b/clang/test/Parser/recovery.cpp
@@ -222,3 +222,21 @@ void k() {
   func(1, ); // expected-error {{expected expression}}
 }
 }
+
+namespace GH136254 {
+
+void call() {
+  [a(42, )]() {} (); // expected-error {{expected expression}}
+
+  int *b = new int(42, ); // expected-error {{expected expression}}
+
+  struct S {
+    int c;
+
+    S() : c(42, ) {} // expected-error {{expected expression}}
+  };
+
+  int d(42, ); // expected-error {{expected expression}}
+}
+
+}

From 1db03cab70fd29fe13cb60cc102285d0d0f33957 Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg@google.com>
Date: Fri, 18 Apr 2025 10:31:33 +0200
Subject: [PATCH 390/710] [mlir][bazel] Port
 697aa9995c24a977425e672d76a4a434384b16e3

---
 .../llvm-project-overlay/mlir/BUILD.bazel     | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 18e26f7a1b7a1..f8f4a4a26a6f1 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -896,6 +896,26 @@ mlir_c_api_cc_library(
     ],
 )
 
+mlir_c_api_cc_library(
+    name = "CAPISMT",
+    srcs = [
+        "lib/CAPI/Dialect/SMT.cpp",
+        "lib/CAPI/Target/ExportSMTLIB.cpp",
+    ],
+    hdrs = [
+        "include/mlir-c/Dialect/SMT.h",
+        "include/mlir-c/Target/ExportSMTLIB.h",
+    ],
+    capi_deps = [
+        ":CAPIIR",
+    ],
+    includes = ["include"],
+    deps = [
+        ":SMTDialect",
+        ":TargetSMTLIB",
+    ],
+)
+
 mlir_c_api_cc_library(
     name = "CAPIRegisterEverything",
     srcs = ["lib/CAPI/RegisterEverything/RegisterEverything.cpp"],

From 31ddaef8d18d643ff4c343d03ddfe2edae7d22a2 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke@amd.com>
Date: Fri, 18 Apr 2025 15:06:30 +0530
Subject: [PATCH 391/710] [CodeGen][NPM] Port UnreachableMachineBlockElim to
 NPM (#136127)

---
 .../llvm/CodeGen/UnreachableBlockElim.h       |  8 +++
 llvm/include/llvm/InitializePasses.h          |  2 +-
 llvm/include/llvm/Passes/CodeGenPassBuilder.h | 15 +++++
 .../llvm/Passes/MachinePassRegistry.def       |  2 +
 llvm/lib/CodeGen/CodeGen.cpp                  |  2 +-
 llvm/lib/CodeGen/LiveVariables.cpp            |  2 +-
 llvm/lib/CodeGen/UnreachableBlockElim.cpp     | 67 ++++++++++++++-----
 llvm/lib/Passes/PassBuilder.cpp               |  1 +
 .../Hexagon/unreachable-mbb-phi-subreg.mir    |  1 +
 .../MIR/X86/unreachable-block-call-site.mir   |  1 +
 10 files changed, 81 insertions(+), 20 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/UnreachableBlockElim.h b/llvm/include/llvm/CodeGen/UnreachableBlockElim.h
index 7dbae77e34401..3c2b701512df6 100644
--- a/llvm/include/llvm/CodeGen/UnreachableBlockElim.h
+++ b/llvm/include/llvm/CodeGen/UnreachableBlockElim.h
@@ -22,6 +22,7 @@
 #ifndef LLVM_CODEGEN_UNREACHABLEBLOCKELIM_H
 #define LLVM_CODEGEN_UNREACHABLEBLOCKELIM_H
 
+#include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
@@ -31,6 +32,13 @@ class UnreachableBlockElimPass
 public:
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
+
+class UnreachableMachineBlockElimPass
+    : public PassInfoMixin<UnreachableMachineBlockElimPass> {
+public:
+  PreservedAnalyses run(MachineFunction &F, MachineFunctionAnalysisManager &AM);
+};
+
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_UNREACHABLEBLOCKELIM_H
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 9fd8f8228a717..9bef8e496c57e 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -315,7 +315,7 @@ void initializeUniformityInfoWrapperPassPass(PassRegistry &);
 void initializeUnifyLoopExitsLegacyPassPass(PassRegistry &);
 void initializeUnpackMachineBundlesPass(PassRegistry &);
 void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
-void initializeUnreachableMachineBlockElimPass(PassRegistry &);
+void initializeUnreachableMachineBlockElimLegacyPass(PassRegistry &);
 void initializeVerifierLegacyPassPass(PassRegistry &);
 void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 void initializeVirtRegRewriterPass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index d092049022623..61e377de0c424 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -47,6 +47,7 @@
 #include "llvm/CodeGen/JMCInstrumenter.h"
 #include "llvm/CodeGen/LiveDebugValuesPass.h"
 #include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/LocalStackSlotAllocation.h"
 #include "llvm/CodeGen/LowerEmuTLS.h"
 #include "llvm/CodeGen/MIRPrinter.h"
@@ -1172,7 +1173,21 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addOptimizedRegAlloc(
 
   addPass(ProcessImplicitDefsPass());
 
+  // LiveVariables currently requires pure SSA form.
+  //
+  // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
+  // LiveVariables can be removed completely, and LiveIntervals can be directly
+  // computed. (We still either need to regenerate kill flags after regalloc, or
+  // preferably fix the scavenger to not depend on them).
+  // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
+  // When LiveVariables is removed this has to be removed/moved either.
+  // Explicit addition of UnreachableMachineBlockElim allows stopping before or
+  // after it with -stop-before/-stop-after.
+  addPass(UnreachableMachineBlockElimPass());
+  addPass(RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>());
+
   // Edge splitting is smarter with machine loop info.
+  addPass(RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>());
   addPass(PHIEliminationPass());
 
   // Eventually, we want to run LiveIntervals before PHI elimination.
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index fff5083caad1a..8c22a28eba277 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -194,6 +194,8 @@ MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass())
 MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass())
 MACHINE_FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass())
 MACHINE_FUNCTION_PASS("two-address-instruction", TwoAddressInstructionPass())
+MACHINE_FUNCTION_PASS("unreachable-mbb-elimination", 
+                      UnreachableMachineBlockElimPass())
 MACHINE_FUNCTION_PASS("verify", MachineVerifierPass())
 MACHINE_FUNCTION_PASS("verify<machine-trace-metrics>", MachineTraceMetricsVerifierPass())
 MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass())
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 2b0e93803971f..a34adcf4803c5 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -140,7 +140,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeTypePromotionLegacyPass(Registry);
   initializeUnpackMachineBundlesPass(Registry);
   initializeUnreachableBlockElimLegacyPassPass(Registry);
-  initializeUnreachableMachineBlockElimPass(Registry);
+  initializeUnreachableMachineBlockElimLegacyPass(Registry);
   initializeVirtRegMapWrapperLegacyPass(Registry);
   initializeVirtRegRewriterPass(Registry);
   initializeWasmEHPreparePass(Registry);
diff --git a/llvm/lib/CodeGen/LiveVariables.cpp b/llvm/lib/CodeGen/LiveVariables.cpp
index ab95200af00e0..f8e0583839b23 100644
--- a/llvm/lib/CodeGen/LiveVariables.cpp
+++ b/llvm/lib/CodeGen/LiveVariables.cpp
@@ -61,7 +61,7 @@ char LiveVariablesWrapperPass::ID = 0;
 char &llvm::LiveVariablesID = LiveVariablesWrapperPass::ID;
 INITIALIZE_PASS_BEGIN(LiveVariablesWrapperPass, "livevars",
                       "Live Variable Analysis", false, false)
-INITIALIZE_PASS_DEPENDENCY(UnreachableMachineBlockElim)
+INITIALIZE_PASS_DEPENDENCY(UnreachableMachineBlockElimLegacy)
 INITIALIZE_PASS_END(LiveVariablesWrapperPass, "livevars",
                     "Live Variable Analysis", false, false)
 
diff --git a/llvm/lib/CodeGen/UnreachableBlockElim.cpp b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
index aa4ccec3338a9..f9f7b47160b6c 100644
--- a/llvm/lib/CodeGen/UnreachableBlockElim.cpp
+++ b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
@@ -72,32 +72,58 @@ PreservedAnalyses UnreachableBlockElimPass::run(Function &F,
 }
 
 namespace {
-  class UnreachableMachineBlockElim : public MachineFunctionPass {
-    bool runOnMachineFunction(MachineFunction &F) override;
-    void getAnalysisUsage(AnalysisUsage &AU) const override;
-
-  public:
-    static char ID; // Pass identification, replacement for typeid
-    UnreachableMachineBlockElim() : MachineFunctionPass(ID) {}
-  };
-}
-char UnreachableMachineBlockElim::ID = 0;
+class UnreachableMachineBlockElim {
+  MachineDominatorTree *MDT;
+  MachineLoopInfo *MLI;
+
+public:
+  UnreachableMachineBlockElim(MachineDominatorTree *MDT, MachineLoopInfo *MLI)
+      : MDT(MDT), MLI(MLI) {}
+  bool run(MachineFunction &MF);
+};
+
+class UnreachableMachineBlockElimLegacy : public MachineFunctionPass {
+  bool runOnMachineFunction(MachineFunction &F) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+  UnreachableMachineBlockElimLegacy() : MachineFunctionPass(ID) {}
+};
+} // namespace
+
+char UnreachableMachineBlockElimLegacy::ID = 0;
 
-INITIALIZE_PASS(UnreachableMachineBlockElim, "unreachable-mbb-elimination",
-  "Remove unreachable machine basic blocks", false, false)
+INITIALIZE_PASS(UnreachableMachineBlockElimLegacy,
+                "unreachable-mbb-elimination",
+                "Remove unreachable machine basic blocks", false, false)
 
-char &llvm::UnreachableMachineBlockElimID = UnreachableMachineBlockElim::ID;
+char &llvm::UnreachableMachineBlockElimID =
+    UnreachableMachineBlockElimLegacy::ID;
 
-void UnreachableMachineBlockElim::getAnalysisUsage(AnalysisUsage &AU) const {
+void UnreachableMachineBlockElimLegacy::getAnalysisUsage(
+    AnalysisUsage &AU) const {
   AU.addPreserved<MachineLoopInfoWrapperPass>();
   AU.addPreserved<MachineDominatorTreeWrapperPass>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
-bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
-  df_iterator_default_set<MachineBasicBlock*> Reachable;
-  bool ModifiedPHI = false;
+PreservedAnalyses
+UnreachableMachineBlockElimPass::run(MachineFunction &MF,
+                                     MachineFunctionAnalysisManager &AM) {
+  auto *MDT = AM.getCachedResult<MachineDominatorTreeAnalysis>(MF);
+  auto *MLI = AM.getCachedResult<MachineLoopAnalysis>(MF);
+
+  if (!UnreachableMachineBlockElim(MDT, MLI).run(MF))
+    return PreservedAnalyses::all();
+
+  return getMachineFunctionPassPreservedAnalyses()
+      .preserve<MachineLoopAnalysis>()
+      .preserve<MachineDominatorTreeAnalysis>();
+}
 
+bool UnreachableMachineBlockElimLegacy::runOnMachineFunction(
+    MachineFunction &MF) {
   MachineDominatorTreeWrapperPass *MDTWrapper =
       getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();
   MachineDominatorTree *MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr;
@@ -105,6 +131,13 @@ bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
       getAnalysisIfAvailable<MachineLoopInfoWrapperPass>();
   MachineLoopInfo *MLI = MLIWrapper ? &MLIWrapper->getLI() : nullptr;
 
+  return UnreachableMachineBlockElim(MDT, MLI).run(MF);
+}
+
+bool UnreachableMachineBlockElim::run(MachineFunction &F) {
+  df_iterator_default_set<MachineBasicBlock *> Reachable;
+  bool ModifiedPHI = false;
+
   // Mark all reachable blocks.
   for (MachineBasicBlock *BB : depth_first_ext(&F, Reachable))
     (void)BB/* Mark all reachable blocks */;
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index a5c624b96304b..e7057d9a6b625 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -167,6 +167,7 @@
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
 #include "llvm/CodeGen/TypePromotion.h"
+#include "llvm/CodeGen/UnreachableBlockElim.h"
 #include "llvm/CodeGen/VirtRegMap.h"
 #include "llvm/CodeGen/WasmEHPrepare.h"
 #include "llvm/CodeGen/WinEHPrepare.h"
diff --git a/llvm/test/CodeGen/Hexagon/unreachable-mbb-phi-subreg.mir b/llvm/test/CodeGen/Hexagon/unreachable-mbb-phi-subreg.mir
index 79f9d5dfffbf7..697c423d27b3f 100644
--- a/llvm/test/CodeGen/Hexagon/unreachable-mbb-phi-subreg.mir
+++ b/llvm/test/CodeGen/Hexagon/unreachable-mbb-phi-subreg.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=hexagon -run-pass unreachable-mbb-elimination %s -o - | FileCheck %s
+# RUN: llc -mtriple=hexagon -passes unreachable-mbb-elimination %s -o - | FileCheck %s
 
 ---
 name: fred
diff --git a/llvm/test/DebugInfo/MIR/X86/unreachable-block-call-site.mir b/llvm/test/DebugInfo/MIR/X86/unreachable-block-call-site.mir
index e77192db977d7..feb5f1201f6ad 100644
--- a/llvm/test/DebugInfo/MIR/X86/unreachable-block-call-site.mir
+++ b/llvm/test/DebugInfo/MIR/X86/unreachable-block-call-site.mir
@@ -1,4 +1,5 @@
 # RUN: llc -emit-call-site-info -mtriple=x86_64-pc-linux -run-pass=unreachable-mbb-elimination -o - %s | FileCheck %s
+# RUN: llc -emit-call-site-info -mtriple=x86_64-pc-linux -passes=unreachable-mbb-elimination -o - %s | FileCheck %s
 
 # Verify that the call site information for the call residing in the eliminated
 # block is removed. This test case would previously trigger an assertion when

From 9bdd9dc895ade41ec24f1a9918f70b23271ac89b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 12:27:38 +0200
Subject: [PATCH 392/710] AMDGPU: Mark workitem ID intrinsics with range
 attribute (#136196)

This avoids the need to have special handling at every use site.
Unfortunately this means we unnecessarily emit AssertZext in the DAG
(where we already directly understand the range of the intrinsic), andt
we regress in undefined cases as we don't fold out asserts on undef.
---
 clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp   |  25 ----
 clang/test/CodeGenOpenCL/builtins-amdgcn.cl   |  10 +-
 clang/test/CodeGenOpenCL/builtins-r600.cl     |   9 +-
 clang/test/Headers/gpuintrin.c                |   6 +-
 clang/test/Headers/gpuintrin_lang.c           |  12 +-
 llvm/include/llvm/IR/IntrinsicsAMDGPU.td      |  47 ++++---
 llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll     |   2 +-
 llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll     |  33 +++--
 llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll  | 115 ++++++++++--------
 llvm/test/CodeGen/AMDGPU/flat-scratch.ll      |  54 ++++----
 .../CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll    |   6 +-
 .../llvm.amdgcn.sched.group.barrier.gfx11.ll  |  16 +--
 .../llvm.amdgcn.sched.group.barrier.gfx12.ll  |   8 +-
 ...vm.amdgcn.sched.group.barrier.iterative.ll |  20 +--
 .../AMDGPU/llvm.amdgcn.sched.group.barrier.ll |  16 +--
 ...kitem.id-unsupported-calling-convention.ll |  21 +++-
 llvm/test/CodeGen/AMDGPU/memory_clause.ll     |   8 +-
 .../CodeGen/AMDGPU/v_add_u64_pseudo_sdwa.ll   |   8 +-
 .../LoopUnroll/AMDGPU/unroll-for-private.ll   |   4 +-
 19 files changed, 218 insertions(+), 202 deletions(-)

diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index 35c9f8ae48c80..ad012d98635ff 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -171,16 +171,6 @@ static Value *emitFPIntBuiltin(CodeGenFunction &CGF,
   return CGF.Builder.CreateCall(F, {Src0, Src1});
 }
 
-static Value *emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID,
-                                int low, int high) {
-  Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
-  llvm::CallInst *Call = CGF.Builder.CreateCall(F);
-  llvm::ConstantRange CR(APInt(32, low), APInt(32, high));
-  Call->addRangeRetAttr(CR);
-  Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
-  return Call;
-}
-
 // For processing memory ordering and memory scope arguments of various
 // amdgcn builtins.
 // \p Order takes a C++11 comptabile memory-ordering specifier and converts
@@ -934,15 +924,6 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(BuiltinWMMAOp, ArgTypes);
     return Builder.CreateCall(F, Args);
   }
-
-  // amdgcn workitem
-  case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
-    return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_x, 0, 1024);
-  case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
-    return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_y, 0, 1024);
-  case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
-    return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_z, 0, 1024);
-
   // amdgcn workgroup size
   case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
     return EmitAMDGPUWorkGroupSize(*this, 0);
@@ -964,12 +945,6 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
     return emitBuiltinWithOneOverloadedType<1>(*this, E,
                                                Intrinsic::r600_recipsqrt_ieee);
-  case AMDGPU::BI__builtin_r600_read_tidig_x:
-    return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_x, 0, 1024);
-  case AMDGPU::BI__builtin_r600_read_tidig_y:
-    return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_y, 0, 1024);
-  case AMDGPU::BI__builtin_r600_read_tidig_z:
-    return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_z, 0, 1024);
   case AMDGPU::BI__builtin_amdgcn_alignbit: {
     llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
     llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
index ded5f6b5ac4fd..bf022bc6eb446 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
@@ -605,9 +605,9 @@ void test_s_getreg(volatile global uint *out)
 }
 
 // CHECK-LABEL: @test_get_local_id(
-// CHECK: tail call noundef range(i32 0, 1024){{.*}} i32 @llvm.amdgcn.workitem.id.x()
-// CHECK: tail call noundef range(i32 0, 1024){{.*}} i32 @llvm.amdgcn.workitem.id.y()
-// CHECK: tail call noundef range(i32 0, 1024){{.*}} i32 @llvm.amdgcn.workitem.id.z()
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.x()
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.y()
+// CHECK: tail call{{.*}} i32 @llvm.amdgcn.workitem.id.z()
 void test_get_local_id(int d, global int *out)
 {
 	switch (d) {
@@ -618,6 +618,10 @@ void test_get_local_id(int d, global int *out)
 	}
 }
 
+// CHECK: declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+// CHECK: declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+// CHECK: declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.z()
+
 // CHECK-LABEL: @test_get_workgroup_size(
 // CHECK: {{.*}}call align 8 dereferenceable(256){{.*}} ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
 // CHECK: getelementptr inbounds nuw i8, ptr addrspace(4) %{{.*}}, i64 12
diff --git a/clang/test/CodeGenOpenCL/builtins-r600.cl b/clang/test/CodeGenOpenCL/builtins-r600.cl
index a82c4fb90ec50..5fe130f585688 100644
--- a/clang/test/CodeGenOpenCL/builtins-r600.cl
+++ b/clang/test/CodeGenOpenCL/builtins-r600.cl
@@ -39,9 +39,9 @@ void test_get_group_id(int d, global int *out)
 }
 
 // CHECK-LABEL: @test_get_local_id(
-// CHECK: tail call noundef range(i32 0, 1024) i32 @llvm.r600.read.tidig.x()
-// CHECK: tail call noundef range(i32 0, 1024) i32 @llvm.r600.read.tidig.y()
-// CHECK: tail call noundef range(i32 0, 1024) i32 @llvm.r600.read.tidig.z()
+// CHECK: tail call i32 @llvm.r600.read.tidig.x()
+// CHECK: tail call i32 @llvm.r600.read.tidig.y()
+// CHECK: tail call i32 @llvm.r600.read.tidig.z()
 void test_get_local_id(int d, global int *out)
 {
 	switch (d) {
@@ -52,3 +52,6 @@ void test_get_local_id(int d, global int *out)
 	}
 }
 
+// CHECK: declare noundef range(i32 0, 1024) i32 @llvm.r600.read.tidig.x()
+// CHECK: declare noundef range(i32 0, 1024) i32 @llvm.r600.read.tidig.y()
+// CHECK: declare noundef range(i32 0, 1024) i32 @llvm.r600.read.tidig.z()
diff --git a/clang/test/Headers/gpuintrin.c b/clang/test/Headers/gpuintrin.c
index 9a15ce277ba87..f7dfb86ac4652 100644
--- a/clang/test/Headers/gpuintrin.c
+++ b/clang/test/Headers/gpuintrin.c
@@ -291,7 +291,7 @@ __gpu_kernel void foo() {
 // AMDGPU-NEXT:  [[ENTRY:.*:]]
 // AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
 // AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
-// AMDGPU-NEXT:    [[TMP0:%.*]] = call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 // AMDGPU-NEXT:    ret i32 [[TMP0]]
 //
 //
@@ -300,7 +300,7 @@ __gpu_kernel void foo() {
 // AMDGPU-NEXT:  [[ENTRY:.*:]]
 // AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
 // AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
-// AMDGPU-NEXT:    [[TMP0:%.*]] = call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
 // AMDGPU-NEXT:    ret i32 [[TMP0]]
 //
 //
@@ -309,7 +309,7 @@ __gpu_kernel void foo() {
 // AMDGPU-NEXT:  [[ENTRY:.*:]]
 // AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
 // AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
-// AMDGPU-NEXT:    [[TMP0:%.*]] = call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.z()
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
 // AMDGPU-NEXT:    ret i32 [[TMP0]]
 //
 //
diff --git a/clang/test/Headers/gpuintrin_lang.c b/clang/test/Headers/gpuintrin_lang.c
index fa04849f8094d..ab660ac5c8a49 100644
--- a/clang/test/Headers/gpuintrin_lang.c
+++ b/clang/test/Headers/gpuintrin_lang.c
@@ -36,7 +36,7 @@ __device__ int foo() { return __gpu_thread_id_x(); }
 // CUDA-LABEL: define dso_local i32 @foo(
 // CUDA-SAME: ) #[[ATTR0:[0-9]+]] {
 // CUDA-NEXT:  [[ENTRY:.*:]]
-// CUDA-NEXT:    [[TMP0:%.*]] = call {{.*}}i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+// CUDA-NEXT:    [[TMP0:%.*]] = call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
 // CUDA-NEXT:    ret i32 [[TMP0]]
 //
 // HIP-LABEL: define dso_local i32 @foo(
@@ -46,29 +46,29 @@ __device__ int foo() { return __gpu_thread_id_x(); }
 // HIP-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
 // HIP-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
 // HIP-NEXT:    [[RETVAL_ASCAST_I:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL_I]] to ptr
-// HIP-NEXT:    [[TMP0:%.*]] = call noundef {{.*}}i32 @llvm.amdgcn.workitem.id.x()
+// HIP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 // HIP-NEXT:    ret i32 [[TMP0]]
 //
 // OPENCL-LABEL: define dso_local i32 @foo(
 // OPENCL-SAME: ) #[[ATTR0:[0-9]+]] {
 // OPENCL-NEXT:  [[ENTRY:.*:]]
-// OPENCL-NEXT:    [[TMP0:%.*]] = call noundef {{.*}}i32 @llvm.amdgcn.workitem.id.x()
+// OPENCL-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 // OPENCL-NEXT:    ret i32 [[TMP0]]
 //
 // OPENMP-LABEL: define hidden i32 @foo(
 // OPENMP-SAME: ) #[[ATTR0:[0-9]+]] {
 // OPENMP-NEXT:  [[ENTRY:.*:]]
-// OPENMP-NEXT:    [[TMP0:%.*]] = call noundef {{.*}}i32 @llvm.amdgcn.workitem.id.x()
+// OPENMP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 // OPENMP-NEXT:    ret i32 [[TMP0]]
 //
 // C89-LABEL: define dso_local i32 @foo(
-// C89-SAME: ) #[[ATTR2:[0-9]+]] {
+// C89-SAME: ) #[[ATTR0:[0-9]+]] {
 // C89-NEXT:  [[ENTRY:.*:]]
 // C89-NEXT:    [[RETVAL_I:%.*]] = alloca i32, align 4, addrspace(5)
 // C89-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
 // C89-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
 // C89-NEXT:    [[RETVAL_ASCAST_I:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL_I]] to ptr
-// C89-NEXT:    [[TMP0:%.*]] = call noundef {{.*}}i32 @llvm.amdgcn.workitem.id.x()
+// C89-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
 // C89-NEXT:    ret i32 [[TMP0]]
 //
 int foo() { return __gpu_thread_id_x(); }
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 217e43fcce4fd..75068717d9a5f 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -20,11 +20,16 @@ def local_ptr_ty : LLVMQualPointerType<3>;
 // some preloaded register from a function that is known to not need it is a violation
 // of the calling convention and also program-level UB. Outside of such IR-level UB,
 // these preloaded registers are always set to a well-defined value and are thus `noundef`.
-class AMDGPUReadPreloadRegisterIntrinsic
-  : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [NoUndef<RetIndex>, IntrNoMem, IntrSpeculatable]>;
+class AMDGPUReadPreloadRegisterIntrinsic<
+    list<IntrinsicProperty> ExtraAttrs = []>
+    : DefaultAttrsIntrinsic<[llvm_i32_ty], [],
+                            !listconcat([NoUndef<RetIndex>, IntrNoMem,
+                                         IntrSpeculatable],
+                                        ExtraAttrs)>;
 
-class AMDGPUReadPreloadRegisterIntrinsicNamed<string name>
-  : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [NoUndef<RetIndex>, IntrNoMem, IntrSpeculatable]>, ClangBuiltin<name>;
+class AMDGPUReadPreloadRegisterIntrinsicNamed<
+    string name, list<IntrinsicProperty> ExtraAttrs = []>
+    : AMDGPUReadPreloadRegisterIntrinsic<ExtraAttrs>, ClangBuiltin<name>;
 
 // Used to tag image and resource intrinsics with information used to generate
 // mem operands.
@@ -35,17 +40,22 @@ class AMDGPURsrcIntrinsic<int rsrcarg, bit isimage = false> {
 
 let TargetPrefix = "r600" in {
 
-multiclass AMDGPUReadPreloadRegisterIntrinsic_xyz {
-  def _x : AMDGPUReadPreloadRegisterIntrinsic;
-  def _y : AMDGPUReadPreloadRegisterIntrinsic;
-  def _z : AMDGPUReadPreloadRegisterIntrinsic;
-}
+  multiclass AMDGPUReadPreloadRegisterIntrinsic_xyz<
+      list<IntrinsicProperty> ExtraAttrs = []> {
+    def _x : AMDGPUReadPreloadRegisterIntrinsic<ExtraAttrs>;
+    def _y : AMDGPUReadPreloadRegisterIntrinsic<ExtraAttrs>;
+    def _z : AMDGPUReadPreloadRegisterIntrinsic<ExtraAttrs>;
+  }
 
-multiclass AMDGPUReadPreloadRegisterIntrinsic_xyz_named<string prefix> {
-  def _x : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_x")>;
-  def _y : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_y")>;
-  def _z : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_z")>;
-}
+  multiclass AMDGPUReadPreloadRegisterIntrinsic_xyz_named<
+      string prefix, list<IntrinsicProperty> ExtraAttrs = []> {
+    def _x : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_x"),
+                                                     ExtraAttrs>;
+    def _y : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_y"),
+                                                     ExtraAttrs>;
+    def _z : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_z"),
+                                                     ExtraAttrs>;
+  }
 
 defm int_r600_read_global_size : AMDGPUReadPreloadRegisterIntrinsic_xyz_named
                                  <"__builtin_r600_read_global_size">;
@@ -55,7 +65,9 @@ defm int_r600_read_tgid : AMDGPUReadPreloadRegisterIntrinsic_xyz_named
                           <"__builtin_r600_read_tgid">;
 
 defm int_r600_read_local_size : AMDGPUReadPreloadRegisterIntrinsic_xyz;
-defm int_r600_read_tidig : AMDGPUReadPreloadRegisterIntrinsic_xyz;
+defm int_r600_read_tidig
+    : AMDGPUReadPreloadRegisterIntrinsic_xyz_named<
+          "__builtin_r600_read_tidig", [Range<RetIndex, 0, 1024>]>;
 
 def int_r600_group_barrier : ClangBuiltin<"__builtin_r600_group_barrier">,
   Intrinsic<[], [], [IntrConvergent, IntrWillReturn]>;
@@ -146,7 +158,10 @@ let TargetPrefix = "amdgcn" in {
 // ABI Special Intrinsics
 //===----------------------------------------------------------------------===//
 
-defm int_amdgcn_workitem_id : AMDGPUReadPreloadRegisterIntrinsic_xyz;
+defm int_amdgcn_workitem_id
+    : AMDGPUReadPreloadRegisterIntrinsic_xyz_named<
+          "__builtin_amdgcn_workitem_id", [Range<RetIndex, 0, 1024>]>;
+
 defm int_amdgcn_workgroup_id : AMDGPUReadPreloadRegisterIntrinsic_xyz_named
                                <"__builtin_amdgcn_workgroup_id">;
 
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll
index 2ac5c78d8cdb5..b563e03b6080f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-inline.ll
@@ -61,7 +61,7 @@ entry:
 
 ; GCN: define amdgpu_kernel void @test_inliner(
 ; GCN-INL1:     %c1 = tail call coldcc float @foo(
-; GCN-INLDEF:   %cmp.i = fcmp ogt float %tmp2, 0.000000e+00
+; GCN-INLDEF:   %cmp.i = fcmp ogt float %{{.+}}, 0.000000e+00
 ; GCN-MAXBBDEF: %div.i{{[0-9]*}} = fdiv float 1.000000e+00, %c
 ; GCN-MAXBBDEF: %div.i{{[0-9]*}} = fdiv float 2.000000e+00, %tmp1.i
 ; GCN-MAXBB1:   call coldcc void @foo_private_ptr
diff --git a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
index 7819da8b97e55..9cf9d81773037 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
@@ -258,46 +258,41 @@ define amdgpu_kernel void @add_x_shl_max_offset() #1 {
 define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset_alt() #1 {
 ; CI-LABEL: add_x_shl_neg_to_sub_max_offset_alt:
 ; CI:       ; %bb.0:
-; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; CI-NEXT:    v_xor_b32_e32 v0, 0xffff, v0
+; CI-NEXT:    v_mul_i32_i24_e32 v0, -4, v0
 ; CI-NEXT:    v_mov_b32_e32 v1, 13
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    ds_write_b8 v0, v1
+; CI-NEXT:    ds_write_b8 v0, v1 offset:65535
 ; CI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: add_x_shl_neg_to_sub_max_offset_alt:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT:    v_xor_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_mul_i32_i24_e32 v0, -4, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 13
-; GFX9-NEXT:    ds_write_b8 v0, v1
+; GFX9-NEXT:    ds_write_b8 v0, v1 offset:65535
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: add_x_shl_neg_to_sub_max_offset_alt:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT:    v_mul_i32_i24_e32 v0, -4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 13
-; GFX10-NEXT:    v_xor_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    ds_write_b8 v0, v1
+; GFX10-NEXT:    ds_write_b8 v0, v1 offset:65535
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-TRUE16-LABEL: add_x_shl_neg_to_sub_max_offset_alt:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-TRUE16-NEXT:    v_xor_b32_e32 v1, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mul_i32_i24_e32 v1, -4, v0
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 13
-; GFX11-TRUE16-NEXT:    ds_store_b8 v1, v0
+; GFX11-TRUE16-NEXT:    ds_store_b8 v1, v0 offset:65535
 ; GFX11-TRUE16-NEXT:    s_endpgm
 ;
 ; GFX11-FAKE16-LABEL: add_x_shl_neg_to_sub_max_offset_alt:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v1, 13 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-FAKE16-NEXT:    v_xor_b32_e32 v0, 0xffff, v0
-; GFX11-FAKE16-NEXT:    ds_store_b8 v0, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_mul_i32_i24_e32 v0, -4, v0
+; GFX11-FAKE16-NEXT:    ds_store_b8 v0, v1 offset:65535
 ; GFX11-FAKE16-NEXT:    s_endpgm
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x()
   %.neg = mul i32 %x.i, -4
@@ -447,9 +442,9 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use() #1 {
 ;
 ; GFX11-LABEL: add_x_shl_neg_to_sub_multi_use:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_dual_mov_b32 v1, 13 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-NEXT:    v_dual_mov_b32 v1, 13 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-NEXT:    v_sub_nc_u32_e32 v0, 0, v0
 ; GFX11-NEXT:    ds_store_b32 v0, v1 offset:123
 ; GFX11-NEXT:    ds_store_b32 v0, v1 offset:456
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll
index 96d0e383761d1..690e5cc68747f 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll
@@ -142,7 +142,7 @@ define amdgpu_kernel void @soff1_voff2(i32 %soff) {
 ; GFX942-SDAG-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX942-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, s0
-; GFX942-SDAG-NEXT:    v_lshl_add_u32 v0, v0, 1, v2
+; GFX942-SDAG-NEXT:    v_mad_u32_u24 v0, v0, 2, v2
 ; GFX942-SDAG-NEXT:    v_add_u32_e32 v2, 1, v0
 ; GFX942-SDAG-NEXT:    v_add_u32_e32 v3, 2, v0
 ; GFX942-SDAG-NEXT:    scratch_store_byte v2, v1, off sc0 sc1
@@ -160,7 +160,7 @@ define amdgpu_kernel void @soff1_voff2(i32 %soff) {
 ; GFX942-GISEL:       ; %bb.0: ; %bb
 ; GFX942-GISEL-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; GFX942-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX942-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX942-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-GISEL-NEXT:    v_add_u32_e32 v0, s0, v0
@@ -181,13 +181,14 @@ define amdgpu_kernel void @soff1_voff2(i32 %soff) {
 ; GFX11-SDAG:       ; %bb.0: ; %bb
 ; GFX11-SDAG-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 1, v0
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_add3_u32 v0, 0, s0, v0
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_add_nc_u32 v5, 2, v0
 ; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v4, 1, v0
+; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v5, 2, v0
 ; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v0, 4, v0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v4, v1, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -201,8 +202,9 @@ define amdgpu_kernel void @soff1_voff2(i32 %soff) {
 ; GFX11-GISEL:       ; %bb.0: ; %bb
 ; GFX11-GISEL-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 1, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v3, 4
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    v_add_nc_u32_e32 v0, s0, v0
 ; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
@@ -223,7 +225,7 @@ define amdgpu_kernel void @soff1_voff2(i32 %soff) {
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    scratch_store_b8 v0, v1, s0 offset:1 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_storecnt 0x0
@@ -239,7 +241,7 @@ define amdgpu_kernel void @soff1_voff2(i32 %soff) {
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_nc_u32_e32 v0, s0, v0
 ; GFX12-GISEL-NEXT:    scratch_store_b8 v0, v1, off offset:1 scope:SCOPE_SYS
@@ -273,7 +275,7 @@ define amdgpu_kernel void @soff1_voff4(i32 %soff) {
 ; GFX942-SDAG-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX942-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, s0
-; GFX942-SDAG-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
+; GFX942-SDAG-NEXT:    v_mad_u32_u24 v0, v0, 4, v2
 ; GFX942-SDAG-NEXT:    v_add_u32_e32 v2, 1, v0
 ; GFX942-SDAG-NEXT:    v_add_u32_e32 v3, 2, v0
 ; GFX942-SDAG-NEXT:    scratch_store_byte v2, v1, off sc0 sc1
@@ -291,7 +293,7 @@ define amdgpu_kernel void @soff1_voff4(i32 %soff) {
 ; GFX942-GISEL:       ; %bb.0: ; %bb
 ; GFX942-GISEL-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; GFX942-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX942-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX942-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-GISEL-NEXT:    v_add_u32_e32 v0, s0, v0
@@ -312,13 +314,14 @@ define amdgpu_kernel void @soff1_voff4(i32 %soff) {
 ; GFX11-SDAG:       ; %bb.0: ; %bb
 ; GFX11-SDAG-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_add3_u32 v0, 0, s0, v0
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_add_nc_u32 v5, 2, v0
 ; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v4, 1, v0
+; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v5, 2, v0
 ; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v0, 4, v0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v4, v1, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -332,8 +335,9 @@ define amdgpu_kernel void @soff1_voff4(i32 %soff) {
 ; GFX11-GISEL:       ; %bb.0: ; %bb
 ; GFX11-GISEL-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v3, 4
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    v_add_nc_u32_e32 v0, s0, v0
 ; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
@@ -354,7 +358,7 @@ define amdgpu_kernel void @soff1_voff4(i32 %soff) {
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    scratch_store_b8 v0, v1, s0 offset:1 scope:SCOPE_SYS
 ; GFX12-SDAG-NEXT:    s_wait_storecnt 0x0
@@ -370,7 +374,7 @@ define amdgpu_kernel void @soff1_voff4(i32 %soff) {
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    v_add_nc_u32_e32 v0, s0, v0
 ; GFX12-GISEL-NEXT:    scratch_store_b8 v0, v1, off offset:1 scope:SCOPE_SYS
@@ -537,7 +541,7 @@ define amdgpu_kernel void @soff2_voff2(i32 %soff) {
 ; GFX942-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-SDAG-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, s0
-; GFX942-SDAG-NEXT:    v_lshl_add_u32 v0, v0, 1, v2
+; GFX942-SDAG-NEXT:    v_mad_u32_u24 v0, v0, 2, v2
 ; GFX942-SDAG-NEXT:    scratch_store_byte v0, v1, off offset:1 sc0 sc1
 ; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, 2, v0
@@ -554,7 +558,7 @@ define amdgpu_kernel void @soff2_voff2(i32 %soff) {
 ; GFX942-GISEL:       ; %bb.0: ; %bb
 ; GFX942-GISEL-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; GFX942-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX942-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX942-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-GISEL-NEXT:    s_lshl_b32 s0, s0, 1
@@ -576,15 +580,16 @@ define amdgpu_kernel void @soff2_voff2(i32 %soff) {
 ; GFX11-SDAG:       ; %bb.0: ; %bb
 ; GFX11-SDAG-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 1, v0
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
 ; GFX11-SDAG-NEXT:    v_add3_u32 v0, 0, s0, v0
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_add_nc_u32 v5, 4, v0
 ; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v4, 2, v0
+; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v5, 4, v0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v1, off offset:1 dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v4, v2, off dlc
@@ -597,8 +602,9 @@ define amdgpu_kernel void @soff2_voff2(i32 %soff) {
 ; GFX11-GISEL:       ; %bb.0: ; %bb
 ; GFX11-GISEL-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 1, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v3, 4
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -621,7 +627,7 @@ define amdgpu_kernel void @soff2_voff2(i32 %soff) {
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX12-SDAG-NEXT:    scratch_store_b8 v0, v1, s0 offset:1 scope:SCOPE_SYS
@@ -638,7 +644,7 @@ define amdgpu_kernel void @soff2_voff2(i32 %soff) {
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -675,7 +681,7 @@ define amdgpu_kernel void @soff2_voff4(i32 %soff) {
 ; GFX942-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-SDAG-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, s0
-; GFX942-SDAG-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
+; GFX942-SDAG-NEXT:    v_mad_u32_u24 v0, v0, 4, v2
 ; GFX942-SDAG-NEXT:    scratch_store_byte v0, v1, off offset:1 sc0 sc1
 ; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, 2, v0
@@ -692,7 +698,7 @@ define amdgpu_kernel void @soff2_voff4(i32 %soff) {
 ; GFX942-GISEL:       ; %bb.0: ; %bb
 ; GFX942-GISEL-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; GFX942-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX942-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX942-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-GISEL-NEXT:    s_lshl_b32 s0, s0, 1
@@ -714,15 +720,16 @@ define amdgpu_kernel void @soff2_voff4(i32 %soff) {
 ; GFX11-SDAG:       ; %bb.0: ; %bb
 ; GFX11-SDAG-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
 ; GFX11-SDAG-NEXT:    v_add3_u32 v0, 0, s0, v0
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_add_nc_u32 v5, 4, v0
 ; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v4, 2, v0
+; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v5, 4, v0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v1, off offset:1 dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v4, v2, off dlc
@@ -735,8 +742,9 @@ define amdgpu_kernel void @soff2_voff4(i32 %soff) {
 ; GFX11-GISEL:       ; %bb.0: ; %bb
 ; GFX11-GISEL-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v3, 4
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -759,7 +767,7 @@ define amdgpu_kernel void @soff2_voff4(i32 %soff) {
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX12-SDAG-NEXT:    scratch_store_b8 v0, v1, s0 offset:1 scope:SCOPE_SYS
@@ -776,7 +784,7 @@ define amdgpu_kernel void @soff2_voff4(i32 %soff) {
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_lshl_b32 s0, s0, 1
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -945,7 +953,7 @@ define amdgpu_kernel void @soff4_voff2(i32 %soff) {
 ; GFX942-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-SDAG-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, s0
-; GFX942-SDAG-NEXT:    v_lshl_add_u32 v0, v0, 1, v2
+; GFX942-SDAG-NEXT:    v_mad_u32_u24 v0, v0, 2, v2
 ; GFX942-SDAG-NEXT:    scratch_store_byte v0, v1, off offset:1 sc0 sc1
 ; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, 2, v0
@@ -962,7 +970,7 @@ define amdgpu_kernel void @soff4_voff2(i32 %soff) {
 ; GFX942-GISEL:       ; %bb.0: ; %bb
 ; GFX942-GISEL-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; GFX942-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX942-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX942-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-GISEL-NEXT:    s_lshl_b32 s0, s0, 2
@@ -984,15 +992,16 @@ define amdgpu_kernel void @soff4_voff2(i32 %soff) {
 ; GFX11-SDAG:       ; %bb.0: ; %bb
 ; GFX11-SDAG-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 1, v0
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
 ; GFX11-SDAG-NEXT:    v_add3_u32 v0, 0, s0, v0
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_add_nc_u32 v5, 4, v0
 ; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v4, 2, v0
+; GFX11-SDAG-NEXT:    v_add_nc_u32_e32 v5, 4, v0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v1, off offset:1 dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v4, v2, off dlc
@@ -1005,8 +1014,9 @@ define amdgpu_kernel void @soff4_voff2(i32 %soff) {
 ; GFX11-GISEL:       ; %bb.0: ; %bb
 ; GFX11-GISEL-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 1, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v3, 4
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -1029,7 +1039,7 @@ define amdgpu_kernel void @soff4_voff2(i32 %soff) {
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX12-SDAG-NEXT:    scratch_store_b8 v0, v1, s0 offset:1 scope:SCOPE_SYS
@@ -1046,7 +1056,7 @@ define amdgpu_kernel void @soff4_voff2(i32 %soff) {
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 2, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -1084,7 +1094,7 @@ define amdgpu_kernel void @soff4_voff4(i32 %soff) {
 ; GFX942-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-SDAG-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, s0
-; GFX942-SDAG-NEXT:    v_lshl_add_u32 v0, v0, 2, v3
+; GFX942-SDAG-NEXT:    v_mad_u32_u24 v0, v0, 4, v3
 ; GFX942-SDAG-NEXT:    scratch_store_byte v0, v1, off offset:1 sc0 sc1
 ; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-SDAG-NEXT:    scratch_store_byte v0, v2, off offset:2 sc0 sc1
@@ -1099,7 +1109,7 @@ define amdgpu_kernel void @soff4_voff4(i32 %soff) {
 ; GFX942-GISEL:       ; %bb.0: ; %bb
 ; GFX942-GISEL-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; GFX942-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX942-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX942-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-GISEL-NEXT:    s_lshl_b32 s0, s0, 2
@@ -1123,7 +1133,7 @@ define amdgpu_kernel void @soff4_voff4(i32 %soff) {
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 4
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -1142,8 +1152,9 @@ define amdgpu_kernel void @soff4_voff4(i32 %soff) {
 ; GFX11-GISEL:       ; %bb.0: ; %bb
 ; GFX11-GISEL-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT:    v_dual_mov_b32 v3, 4 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v3, 4
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -1166,7 +1177,7 @@ define amdgpu_kernel void @soff4_voff4(i32 %soff) {
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-SDAG-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX12-SDAG-NEXT:    scratch_store_b8 v0, v1, s0 offset:1 scope:SCOPE_SYS
@@ -1183,7 +1194,7 @@ define amdgpu_kernel void @soff4_voff4(i32 %soff) {
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    v_dual_mov_b32 v2, 2 :: v_dual_mov_b32 v3, 4
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-GISEL-NEXT:    v_mul_u32_u24_e32 v0, 4, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
index b25d9b245f5f6..b5e579b78a59c 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
@@ -714,10 +714,10 @@ define amdgpu_kernel void @store_load_vindex_kernel(i32 %n) {
 ; GFX11-LABEL: store_load_vindex_kernel:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_load_b32 s0, s[4:5], 0x24
-; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_lshl_b32 s0, s0, 7
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -732,9 +732,9 @@ define amdgpu_kernel void @store_load_vindex_kernel(i32 %n) {
 ; GFX12-LABEL: store_load_vindex_kernel:
 ; GFX12:       ; %bb.0: ; %bb
 ; GFX12-NEXT:    s_load_b32 s0, s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_lshl_b32 s0, s0, 7
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -769,8 +769,8 @@ define amdgpu_kernel void @store_load_vindex_kernel(i32 %n) {
 ; GFX942-LABEL: store_load_vindex_kernel:
 ; GFX942:       ; %bb.0: ; %bb
 ; GFX942-NEXT:    s_load_dword s0, s[4:5], 0x24
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX942-NEXT:    v_and_b32_e32 v0, 0xffc, v0
 ; GFX942-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_lshl_b32 s0, s0, 7
@@ -809,10 +809,10 @@ define amdgpu_kernel void @store_load_vindex_kernel(i32 %n) {
 ; GFX11-PAL-LABEL: store_load_vindex_kernel:
 ; GFX11-PAL:       ; %bb.0: ; %bb
 ; GFX11-PAL-NEXT:    s_load_b32 s0, s[4:5], 0x0
-; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-PAL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-PAL-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX11-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-PAL-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 7
 ; GFX11-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -827,9 +827,9 @@ define amdgpu_kernel void @store_load_vindex_kernel(i32 %n) {
 ; GFX12-PAL-LABEL: store_load_vindex_kernel:
 ; GFX12-PAL:       ; %bb.0: ; %bb
 ; GFX12-PAL-NEXT:    s_load_b32 s0, s[4:5], 0x0
-; GFX12-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-PAL-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX12-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX12-PAL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-PAL-NEXT:    s_lshl_b32 s0, s0, 7
 ; GFX12-PAL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -1958,10 +1958,10 @@ define amdgpu_kernel void @store_load_vindex_small_offset_kernel(i32 %n) {
 ; GFX11-LABEL: store_load_vindex_small_offset_kernel:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_load_b32 s0, s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-NEXT:    scratch_load_b32 v3, off, off glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-NEXT:    scratch_store_b32 v0, v1, off offset:384 dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1976,10 +1976,10 @@ define amdgpu_kernel void @store_load_vindex_small_offset_kernel(i32 %n) {
 ; GFX12-LABEL: store_load_vindex_small_offset_kernel:
 ; GFX12:       ; %bb.0: ; %bb
 ; GFX12-NEXT:    s_load_b32 s0, s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-NEXT:    scratch_load_b32 v3, off, off scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    scratch_store_b32 v0, v1, off offset:384 scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_storecnt 0x0
@@ -2021,8 +2021,8 @@ define amdgpu_kernel void @store_load_vindex_small_offset_kernel(i32 %n) {
 ; GFX942-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; GFX942-NEXT:    scratch_load_dword v1, off, off sc0 sc1
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX942-NEXT:    v_and_b32_e32 v0, 0xffc, v0
 ; GFX942-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_lshl_b32 s0, s0, 7
@@ -2092,10 +2092,10 @@ define amdgpu_kernel void @store_load_vindex_small_offset_kernel(i32 %n) {
 ; GFX11-PAL-LABEL: store_load_vindex_small_offset_kernel:
 ; GFX11-PAL:       ; %bb.0: ; %bb
 ; GFX11-PAL-NEXT:    s_load_b32 s0, s[4:5], 0x0
-; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, off glc dlc
 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-PAL-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:384 dlc
 ; GFX11-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2110,10 +2110,10 @@ define amdgpu_kernel void @store_load_vindex_small_offset_kernel(i32 %n) {
 ; GFX12-PAL-LABEL: store_load_vindex_small_offset_kernel:
 ; GFX12-PAL:       ; %bb.0: ; %bb
 ; GFX12-PAL-NEXT:    s_load_b32 s0, s[4:5], 0x0
-; GFX12-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-PAL-NEXT:    scratch_load_b32 v3, off, off scope:SCOPE_SYS
 ; GFX12-PAL-NEXT:    s_wait_loadcnt 0x0
-; GFX12-PAL-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX12-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX12-PAL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:384 scope:SCOPE_SYS
 ; GFX12-PAL-NEXT:    s_wait_storecnt 0x0
@@ -3254,10 +3254,10 @@ define amdgpu_kernel void @store_load_vindex_large_offset_kernel(i32 %n) {
 ; GFX11-LABEL: store_load_vindex_large_offset_kernel:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_load_b32 s0, s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-NEXT:    scratch_load_b32 v3, off, off offset:4 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_lshl_b32 s0, s0, 7
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -3274,10 +3274,10 @@ define amdgpu_kernel void @store_load_vindex_large_offset_kernel(i32 %n) {
 ; GFX12-LABEL: store_load_vindex_large_offset_kernel:
 ; GFX12:       ; %bb.0: ; %bb
 ; GFX12-NEXT:    s_load_b32 s0, s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-NEXT:    scratch_load_b32 v3, off, off scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    scratch_store_b32 v0, v1, off offset:16512 scope:SCOPE_SYS
 ; GFX12-NEXT:    s_wait_storecnt 0x0
@@ -3319,8 +3319,8 @@ define amdgpu_kernel void @store_load_vindex_large_offset_kernel(i32 %n) {
 ; GFX942-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; GFX942-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX942-NEXT:    v_and_b32_e32 v0, 0xffc, v0
 ; GFX942-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_lshl_b32 s0, s0, 7
@@ -3391,10 +3391,10 @@ define amdgpu_kernel void @store_load_vindex_large_offset_kernel(i32 %n) {
 ; GFX11-PAL-LABEL: store_load_vindex_large_offset_kernel:
 ; GFX11-PAL:       ; %bb.0: ; %bb
 ; GFX11-PAL-NEXT:    s_load_b32 s0, s[4:5], 0x0
-; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX11-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-PAL-NEXT:    scratch_load_b32 v3, off, off offset:4 glc dlc
 ; GFX11-PAL-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-PAL-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX11-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-PAL-NEXT:    s_lshl_b32 s0, s0, 7
 ; GFX11-PAL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -3411,10 +3411,10 @@ define amdgpu_kernel void @store_load_vindex_large_offset_kernel(i32 %n) {
 ; GFX12-PAL-LABEL: store_load_vindex_large_offset_kernel:
 ; GFX12-PAL:       ; %bb.0: ; %bb
 ; GFX12-PAL-NEXT:    s_load_b32 s0, s[4:5], 0x0
-; GFX12-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-PAL-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-PAL-NEXT:    scratch_load_b32 v3, off, off scope:SCOPE_SYS
 ; GFX12-PAL-NEXT:    s_wait_loadcnt 0x0
-; GFX12-PAL-NEXT:    v_and_b32_e32 v0, 0xffc, v0
+; GFX12-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX12-PAL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-PAL-NEXT:    scratch_store_b32 v0, v1, off offset:16512 scope:SCOPE_SYS
 ; GFX12-PAL-NEXT:    s_wait_storecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
index 08c0d15432915..565ad295ebbb3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
@@ -15,8 +15,8 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in,
 ; GCN-LABEL: test_iglp_opt_mfma_gemm:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
 ; GCN-NEXT:    v_mov_b32_e32 v3, 2.0
 ; GCN-NEXT:    ; iglp_opt mask(0x00000000)
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
@@ -153,8 +153,8 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
 ; GCN-LABEL: test_iglp_opt_rev_mfma_gemm:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
 ; GCN-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GCN-NEXT:    v_mov_b32_e32 v3, 2.0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
@@ -289,8 +289,8 @@ define amdgpu_kernel void @test_iglp_opt_asm_sideeffect(ptr addrspace(3) noalias
 ; GCN-LABEL: test_iglp_opt_asm_sideeffect:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-NEXT:    v_and_b32_e32 v0, 0xffc, v0
 ; GCN-NEXT:    ; iglp_opt mask(0x00000000)
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_add_u32_e32 v1, s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
index 46359f7e99059..6507976872410 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
@@ -6,9 +6,9 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_cluster(ptr ad
 ; GCN-LABEL: test_sched_group_barrier_pipeline_WMMA_cluster:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GCN-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_and_b32_e32 v40, 0x7fe0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v40, 5, v0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_add_nc_u32_e32 v32, s0, v40
 ; GCN-NEXT:    v_dual_mov_b32 v81, s1 :: v_dual_add_nc_u32 v80, s1, v40
@@ -74,9 +74,9 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_cluster(ptr ad
 ; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_WMMA_cluster:
 ; EXACTCUTOFF:       ; %bb.0: ; %entry
 ; EXACTCUTOFF-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; EXACTCUTOFF-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; EXACTCUTOFF-NEXT:    v_and_b32_e32 v40, 0x7fe0, v0
+; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v40, 5, v0
 ; EXACTCUTOFF-NEXT:    s_waitcnt lgkmcnt(0)
 ; EXACTCUTOFF-NEXT:    v_add_nc_u32_e32 v32, s0, v40
 ; EXACTCUTOFF-NEXT:    v_dual_mov_b32 v81, s1 :: v_dual_add_nc_u32 v80, s1, v40
@@ -178,9 +178,9 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_interleave(ptr
 ; GCN-LABEL: test_sched_group_barrier_pipeline_WMMA_interleave:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GCN-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_and_b32_e32 v16, 0x7fe0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v16, 5, v0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_add_nc_u32_e32 v17, s0, v16
 ; GCN-NEXT:    v_add_nc_u32_e32 v16, s1, v16
@@ -260,9 +260,9 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_interleave(ptr
 ; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_WMMA_interleave:
 ; EXACTCUTOFF:       ; %bb.0: ; %entry
 ; EXACTCUTOFF-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; EXACTCUTOFF-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; EXACTCUTOFF-NEXT:    v_and_b32_e32 v16, 0x7fe0, v0
+; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v16, 5, v0
 ; EXACTCUTOFF-NEXT:    s_waitcnt lgkmcnt(0)
 ; EXACTCUTOFF-NEXT:    v_add_nc_u32_e32 v17, s0, v16
 ; EXACTCUTOFF-NEXT:    v_add_nc_u32_e32 v16, s1, v16
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
index dcc3e0df0c744..02e80b62fed6e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
@@ -8,10 +8,10 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_SWMMAC_cluster(ptr
 ; GCN-LABEL: test_sched_group_barrier_pipeline_SWMMAC_cluster:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GCN-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    v_mov_b32_e32 v48, 0
 ; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GCN-NEXT:    v_and_b32_e32 v28, 0x3ff0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v28, 4, v0
 ; GCN-NEXT:    s_wait_kmcnt 0x0
 ; GCN-NEXT:    v_add_nc_u32_e32 v0, s0, v28
 ; GCN-NEXT:    v_dual_mov_b32 v50, s1 :: v_dual_add_nc_u32 v49, s1, v28
@@ -60,10 +60,10 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_SWMMAC_cluster(ptr
 ; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_SWMMAC_cluster:
 ; EXACTCUTOFF:       ; %bb.0: ; %entry
 ; EXACTCUTOFF-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; EXACTCUTOFF-NEXT:    v_mov_b32_e32 v48, 0
 ; EXACTCUTOFF-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; EXACTCUTOFF-NEXT:    v_and_b32_e32 v28, 0x3ff0, v0
+; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v28, 4, v0
 ; EXACTCUTOFF-NEXT:    s_wait_kmcnt 0x0
 ; EXACTCUTOFF-NEXT:    v_add_nc_u32_e32 v0, s0, v28
 ; EXACTCUTOFF-NEXT:    v_dual_mov_b32 v50, s1 :: v_dual_add_nc_u32 v49, s1, v28
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
index 0764cd5d34d75..371b4f070094d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
@@ -7,8 +7,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
 ; GCN-MINREG-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
 ; GCN-MINREG:       ; %bb.0: ; %entry
 ; GCN-MINREG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-MINREG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-MINREG-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-MINREG-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
 ; GCN-MINREG-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GCN-MINREG-NEXT:    v_mov_b32_e32 v1, 2.0
 ; GCN-MINREG-NEXT:    s_waitcnt lgkmcnt(0)
@@ -140,8 +140,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
 ; GCN-MAXOCC-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
 ; GCN-MAXOCC:       ; %bb.0: ; %entry
 ; GCN-MAXOCC-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-MAXOCC-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-MAXOCC-NEXT:    v_and_b32_e32 v1, 0x1ff80, v0
+; GCN-MAXOCC-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GCN-MAXOCC-NEXT:    v_lshlrev_b32_e32 v1, 7, v0
 ; GCN-MAXOCC-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GCN-MAXOCC-NEXT:    v_mov_b32_e32 v3, 2.0
 ; GCN-MAXOCC-NEXT:    s_waitcnt lgkmcnt(0)
@@ -274,8 +274,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
 ; GCN-ILP-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
 ; GCN-ILP:       ; %bb.0: ; %entry
 ; GCN-ILP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-ILP-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-ILP-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-ILP-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
 ; GCN-ILP-NEXT:    v_mov_b32_e32 v1, 1.0
 ; GCN-ILP-NEXT:    v_mov_b32_e32 v2, 2.0
 ; GCN-ILP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -469,8 +469,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_spl
 ; GCN-MINREG-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave_split_region:
 ; GCN-MINREG:       ; %bb.0: ; %entry
 ; GCN-MINREG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-MINREG-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-MINREG-NEXT:    v_and_b32_e32 v2, 0x1ff80, v0
+; GCN-MINREG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GCN-MINREG-NEXT:    v_lshlrev_b32_e32 v2, 7, v0
 ; GCN-MINREG-NEXT:    v_mov_b32_e32 v1, 1.0
 ; GCN-MINREG-NEXT:    v_mov_b32_e32 v0, 2.0
 ; GCN-MINREG-NEXT:    s_waitcnt lgkmcnt(0)
@@ -604,8 +604,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_spl
 ; GCN-MAXOCC-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave_split_region:
 ; GCN-MAXOCC:       ; %bb.0: ; %entry
 ; GCN-MAXOCC-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-MAXOCC-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-MAXOCC-NEXT:    v_and_b32_e32 v3, 0x1ff80, v0
+; GCN-MAXOCC-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GCN-MAXOCC-NEXT:    v_lshlrev_b32_e32 v3, 7, v0
 ; GCN-MAXOCC-NEXT:    v_mov_b32_e32 v1, 1.0
 ; GCN-MAXOCC-NEXT:    v_mov_b32_e32 v2, 2.0
 ; GCN-MAXOCC-NEXT:    s_waitcnt lgkmcnt(0)
@@ -739,8 +739,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_spl
 ; GCN-ILP-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave_split_region:
 ; GCN-ILP:       ; %bb.0: ; %entry
 ; GCN-ILP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-ILP-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-ILP-NEXT:    v_and_b32_e32 v2, 0x1ff80, v0
+; GCN-ILP-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GCN-ILP-NEXT:    v_lshlrev_b32_e32 v2, 7, v0
 ; GCN-ILP-NEXT:    v_mov_b32_e32 v0, 1.0
 ; GCN-ILP-NEXT:    v_mov_b32_e32 v1, 2.0
 ; GCN-ILP-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 37f335561a52c..73586b1243376 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -621,8 +621,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
 ; GCN-LABEL: test_sched_group_barrier_pipeline_MFMA_cluster:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_add_u32_e32 v1, s0, v0
 ; GCN-NEXT:    ds_read_b128 a[156:159], v1 offset:112
@@ -728,8 +728,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
 ; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_MFMA_cluster:
 ; EXACTCUTOFF:       ; %bb.0: ; %entry
 ; EXACTCUTOFF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
 ; EXACTCUTOFF-NEXT:    s_waitcnt lgkmcnt(0)
 ; EXACTCUTOFF-NEXT:    v_add_u32_e32 v1, s0, v0
 ; EXACTCUTOFF-NEXT:    ds_read_b128 a[156:159], v1 offset:112
@@ -871,8 +871,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
 ; GCN-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
 ; GCN-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GCN-NEXT:    v_mov_b32_e32 v3, 2.0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1005,8 +1005,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
 ; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
 ; EXACTCUTOFF:       ; %bb.0: ; %entry
 ; EXACTCUTOFF-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
 ; EXACTCUTOFF-NEXT:    v_mov_b32_e32 v2, 1.0
 ; EXACTCUTOFF-NEXT:    v_mov_b32_e32 v3, 2.0
 ; EXACTCUTOFF-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1202,7 +1202,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
 ; GCN-NEXT:    v_mov_b32_e32 v3, 0x3fb8aa3b
 ; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GCN-NEXT:    v_mov_b32_e32 v7, 0x32a5705f
-; GCN-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mul_f32_e32 v4, s0, v3
 ; GCN-NEXT:    v_rndne_f32_e32 v5, v4
@@ -1212,7 +1212,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
 ; GCN-NEXT:    v_add_f32_e32 v4, v6, v4
 ; GCN-NEXT:    v_exp_f32_e32 v4, v4
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
 ; GCN-NEXT:    v_add_u32_e32 v1, s6, v0
 ; GCN-NEXT:    ds_read_b128 a[124:127], v1 offset:112
 ; GCN-NEXT:    ds_read_b128 a[120:123], v1 offset:96
@@ -1387,7 +1387,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
 ; EXACTCUTOFF-NEXT:    v_mov_b32_e32 v3, 0x3fb8aa3b
 ; EXACTCUTOFF-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; EXACTCUTOFF-NEXT:    v_mov_b32_e32 v7, 0x32a5705f
-; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; EXACTCUTOFF-NEXT:    s_waitcnt lgkmcnt(0)
 ; EXACTCUTOFF-NEXT:    v_mul_f32_e32 v4, s0, v3
 ; EXACTCUTOFF-NEXT:    v_rndne_f32_e32 v5, v4
@@ -1397,7 +1397,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
 ; EXACTCUTOFF-NEXT:    v_add_f32_e32 v4, v6, v4
 ; EXACTCUTOFF-NEXT:    v_exp_f32_e32 v4, v4
 ; EXACTCUTOFF-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; EXACTCUTOFF-NEXT:    v_and_b32_e32 v0, 0x1ff80, v0
+; EXACTCUTOFF-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
 ; EXACTCUTOFF-NEXT:    v_add_u32_e32 v1, s6, v0
 ; EXACTCUTOFF-NEXT:    ds_read_b128 a[124:127], v1 offset:112
 ; EXACTCUTOFF-NEXT:    ds_read_b128 a[120:123], v1 offset:96
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id-unsupported-calling-convention.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id-unsupported-calling-convention.ll
index 6ad98a1cbb3ed..37cfb6cc10180 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id-unsupported-calling-convention.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id-unsupported-calling-convention.ll
@@ -1,15 +1,24 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck -check-prefixes=CHECK,SDAG %s
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck  -check-prefixes=CHECK,GISEL %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.workitem.id.y()
 declare i32 @llvm.amdgcn.workitem.id.z()
 
+; FIXME: It's not worth adding AssertZext to the intrinsic calls, and
+; we don't fold out assertzext undef ->undef
 define amdgpu_ps void @undefined_workitems(ptr addrspace(1) %p, ptr addrspace(1) %q, ptr addrspace(1) %r) {
-; CHECK-LABEL: undefined_workitems:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_endpgm
+; SDAG-LABEL: undefined_workitems:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    global_store_dword v[0:1], v0, off
+; SDAG-NEXT:    global_store_dword v[2:3], v0, off
+; SDAG-NEXT:    global_store_dword v[4:5], v0, off
+; SDAG-NEXT:    s_endpgm
+;
+; GISEL-LABEL: undefined_workitems:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_endpgm
   %id.x = call i32 @llvm.amdgcn.workitem.id.x()
   store i32 %id.x, ptr addrspace(1) %p
   %id.y = call i32 @llvm.amdgcn.workitem.id.y()
@@ -18,3 +27,5 @@ define amdgpu_ps void @undefined_workitems(ptr addrspace(1) %p, ptr addrspace(1)
   store i32 %id.z, ptr addrspace(1) %r
   ret void
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.ll b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
index 4e1d2a754fa61..2bda61ab950f7 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
@@ -146,8 +146,8 @@ define void @mubuf_clause(ptr addrspace(5) noalias nocapture readonly %arg, ptr
 ; GCN-LABEL: mubuf_clause:
 ; GCN:       ; %bb.0: ; %bb
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_lshlrev_b32_e32 v2, 4, v31
-; GCN-NEXT:    v_and_b32_e32 v2, 0x3ff0, v2
+; GCN-NEXT:    v_and_b32_e32 v2, 0x3ff, v31
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
 ; GCN-NEXT:    v_add_u32_e32 v0, v0, v2
 ; GCN-NEXT:    buffer_load_dword v3, v0, s[0:3], 0 offen offset:12
 ; GCN-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen offset:8
@@ -205,8 +205,8 @@ define void @mubuf_clause(ptr addrspace(5) noalias nocapture readonly %arg, ptr
 ; GCN-SCRATCH-LABEL: mubuf_clause:
 ; GCN-SCRATCH:       ; %bb.0: ; %bb
 ; GCN-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-SCRATCH-NEXT:    v_lshlrev_b32_e32 v2, 4, v31
-; GCN-SCRATCH-NEXT:    v_and_b32_e32 v18, 0x3ff0, v2
+; GCN-SCRATCH-NEXT:    v_and_b32_e32 v2, 0x3ff, v31
+; GCN-SCRATCH-NEXT:    v_lshlrev_b32_e32 v18, 4, v2
 ; GCN-SCRATCH-NEXT:    v_add_nc_u32_e32 v0, v0, v18
 ; GCN-SCRATCH-NEXT:    s_clause 0x3
 ; GCN-SCRATCH-NEXT:    scratch_load_dwordx4 v[2:5], v0, off
diff --git a/llvm/test/CodeGen/AMDGPU/v_add_u64_pseudo_sdwa.ll b/llvm/test/CodeGen/AMDGPU/v_add_u64_pseudo_sdwa.ll
index 9ace249e6af4a..2e52e515a2a97 100644
--- a/llvm/test/CodeGen/AMDGPU/v_add_u64_pseudo_sdwa.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_add_u64_pseudo_sdwa.ll
@@ -3,10 +3,10 @@
 define amdgpu_kernel void @sdwa_test() local_unnamed_addr #0 {
 ; GFX9-LABEL: sdwa_test:
 ; GFX9:       ; %bb.0: ; %bb
-; GFX9-NEXT:    v_add_u32_e32 v1, 10, v0
-; GFX9-NEXT:    v_add_u32_e32 v0, 20, v0
-; GFX9-NEXT:    v_add_co_u32_sdwa v0, vcc, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[0:1], 0, 0, vcc
+; GFX9-NEXT:    v_add_u32_e32 v1, 20, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX9-NEXT:    v_add3_u32 v0, v0, v1, 10
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
 ; GFX9-NEXT:    s_endpgm
 bb:
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
index a742d1e59dc72..4f770b91c07fe 100644
--- a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
@@ -34,6 +34,8 @@ for.body:                                         ; preds = %for.body, %entry
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 }
 
+declare i32 @func()
+
 ; Check that we unroll inner loop but not outer
 ; CHECK-LABEL: @invariant_ind
 ; CHECK:       %[[exitcond:[^ ]+]] = icmp eq i32 %{{.*}}, 32
@@ -43,7 +45,7 @@ for.body:                                         ; preds = %for.body, %entry
 define amdgpu_kernel void @invariant_ind(ptr addrspace(1) nocapture %a, i32 %x) {
 entry:
   %arr = alloca [64 x i32], align 4, addrspace(5)
-  %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tmp1 = tail call i32 @func()
   br label %for.cond2.preheader
 
 for.cond2.preheader:                              ; preds = %for.cond.cleanup5, %entry

From 9b8bc53a0bdc270f6d675ba7a2fe9ca9a855610d Mon Sep 17 00:00:00 2001
From: Chengjun <chengjunp@Nvidia.com>
Date: Fri, 18 Apr 2025 03:30:05 -0700
Subject: [PATCH 393/710] [FlattenCFG] Fix an Imprecise Usage of AA (#128117)

In current `FlattenCFG`, using `isNoAlias` for two instructions is
imprecise. For example, when passing a store instruction and a load
instruction directly into `AA->isNoAlias`, it will always return
`NoAlias`. This happens because when checking the types of the two
Values, the store instruction (which has a `void` type) causes the
analysis to return `NoAlias`.

For instructions, we should use `getModRefInfo` instead of `isNoAlias`,
as aliasing is a concept of memory locations.

In this patch, `AAResults::getModRefInfo` is supported to take in two
instructions. It will check whether two instructions may access the same
memory location or not. And in `FlattenCFG`, we use this new helper
function to do the check instead of `isNoAlias`.

Unit tests and lit tests are also included to this patch.
---
 llvm/include/llvm/Analysis/AliasAnalysis.h    |   6 +
 llvm/lib/Analysis/AliasAnalysis.cpp           |  20 +++
 llvm/lib/Transforms/Utils/FlattenCFG.cpp      |   4 +-
 llvm/test/Transforms/Util/flatten-cfg.ll      | 140 ++++++++++++++++++
 llvm/unittests/Analysis/AliasAnalysisTest.cpp |  15 ++
 5 files changed, 183 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index b192a9f5e65e7..b3b44a50ca827 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -521,6 +521,10 @@ class AAResults {
   /// the same memory locations.
   ModRefInfo getModRefInfo(const Instruction *I, const CallBase *Call);
 
+  /// Return information about whether two instructions may refer to the same
+  /// memory locations.
+  ModRefInfo getModRefInfo(const Instruction *I1, const Instruction *I2);
+
   /// Return information about whether a particular call site modifies
   /// or reads the specified memory location \p MemLoc before instruction \p I
   /// in a BasicBlock.
@@ -600,6 +604,8 @@ class AAResults {
   ModRefInfo getModRefInfo(const Instruction *I,
                            const std::optional<MemoryLocation> &OptLoc,
                            AAQueryInfo &AAQIP);
+  ModRefInfo getModRefInfo(const Instruction *I1, const Instruction *I2,
+                           AAQueryInfo &AAQI);
   ModRefInfo callCapturesBefore(const Instruction *I,
                                 const MemoryLocation &MemLoc, DominatorTree *DT,
                                 AAQueryInfo &AAQIP);
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index 9573da93a3755..2a8fca9ac2039 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -336,6 +336,26 @@ ModRefInfo AAResults::getModRefInfo(const CallBase *Call1,
   return Result;
 }
 
+ModRefInfo AAResults::getModRefInfo(const Instruction *I1,
+                                    const Instruction *I2) {
+  SimpleAAQueryInfo AAQIP(*this);
+  return getModRefInfo(I1, I2, AAQIP);
+}
+
+ModRefInfo AAResults::getModRefInfo(const Instruction *I1,
+                                    const Instruction *I2, AAQueryInfo &AAQI) {
+  // Early-exit if either instruction does not read or write memory.
+  if (!I1->mayReadOrWriteMemory() || !I2->mayReadOrWriteMemory())
+    return ModRefInfo::NoModRef;
+
+  if (const auto *Call2 = dyn_cast<CallBase>(I2))
+    return getModRefInfo(I1, Call2, AAQI);
+
+  // FIXME: We can have a more precise result.
+  ModRefInfo MR = getModRefInfo(I1, MemoryLocation::getOrNone(I2), AAQI);
+  return isModOrRefSet(MR) ? ModRefInfo::ModRef : ModRefInfo::NoModRef;
+}
+
 MemoryEffects AAResults::getMemoryEffects(const CallBase *Call,
                                           AAQueryInfo &AAQI) {
   MemoryEffects Result = MemoryEffects::unknown();
diff --git a/llvm/lib/Transforms/Utils/FlattenCFG.cpp b/llvm/lib/Transforms/Utils/FlattenCFG.cpp
index 16b4bb1981d8b..7e6ab4c76a882 100644
--- a/llvm/lib/Transforms/Utils/FlattenCFG.cpp
+++ b/llvm/lib/Transforms/Utils/FlattenCFG.cpp
@@ -356,8 +356,8 @@ bool FlattenCFGOpt::CompareIfRegionBlock(BasicBlock *Block1, BasicBlock *Block2,
     if (iter1->mayWriteToMemory()) {
       for (BasicBlock::iterator BI(PBI2), BE(PTI2); BI != BE; ++BI) {
         if (BI->mayReadFromMemory() || BI->mayWriteToMemory()) {
-          // Check alias with Head2.
-          if (!AA || !AA->isNoAlias(&*iter1, &*BI))
+          // Check whether iter1 and BI may access the same memory location.
+          if (!AA || AA->getModRefInfo(&*iter1, &*BI) != ModRefInfo::NoModRef)
             return false;
         }
       }
diff --git a/llvm/test/Transforms/Util/flatten-cfg.ll b/llvm/test/Transforms/Util/flatten-cfg.ll
index 038dcaa47419a..f6b62dce8c19d 100644
--- a/llvm/test/Transforms/Util/flatten-cfg.ll
+++ b/llvm/test/Transforms/Util/flatten-cfg.ll
@@ -309,3 +309,143 @@ if.then.y:
 exit:
   ret i1 %cmp.y
 }
+
+declare void @foo()
+declare void @bar() #1
+
+; Test that two if-regions are not merged when there's potential aliasing
+; between a store in the first if-region and a load in the second if-region's header
+define i32 @test_alias(i32 %a, i32 %b, ptr %p1, ptr %p2) {
+; CHECK-LABEL: define i32 @test_alias
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 42, ptr [[P1]], align 4
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    br i1 [[COND1]], label [[IF_THEN1:%.*]], label [[IF_END1:%.*]]
+; CHECK:       if.then1:
+; CHECK-NEXT:    store i32 100, ptr [[P2]], align 4
+; CHECK-NEXT:    br label [[IF_END1]]
+; CHECK:       if.end1:
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[P1]], align 4
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[B]], 0
+; CHECK-NEXT:    br i1 [[COND2]], label [[IF_THEN2:%.*]], label [[IF_END2:%.*]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    store i32 100, ptr [[P2]], align 4
+; CHECK-NEXT:    br label [[IF_END2]]
+; CHECK:       if.end2:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  store i32 42, ptr %p1
+  %cond1 = icmp eq i32 %a, 0
+  br i1 %cond1, label %if.then1, label %if.end1
+
+if.then1:
+  store i32 100, ptr %p2  ; May alias with the load below
+  br label %if.end1
+
+if.end1:
+  %val = load i32, ptr %p1  ; This load prevents merging due to potential alias
+  %cond2 = icmp eq i32 %b, 0
+  br i1 %cond2, label %if.then2, label %if.end2
+
+if.then2:
+  store i32 100, ptr %p2
+  br label %if.end2
+
+if.end2:
+  ret i32 0
+}
+
+; Test that two if-regions are not merged when there's potential aliasing
+; between a store in the first if-region and a function call in the second if-region's header
+define i32 @test_alias_2(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @test_alias_2
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 42, ptr [[P]], align 4
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    br i1 [[COND1]], label [[IF_THEN1:%.*]], label [[IF_END1:%.*]]
+; CHECK:       if.then1:
+; CHECK-NEXT:    store i32 100, ptr @g, align 4
+; CHECK-NEXT:    br label [[IF_END1]]
+; CHECK:       if.end1:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[B]], 0
+; CHECK-NEXT:    br i1 [[COND2]], label [[IF_THEN2:%.*]], label [[IF_END2:%.*]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    store i32 100, ptr @g, align 4
+; CHECK-NEXT:    br label [[IF_END2]]
+; CHECK:       if.end2:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %p = alloca i32
+  store i32 42, ptr %p
+  %cond1 = icmp eq i32 %a, 0
+  br i1 %cond1, label %if.then1, label %if.end1
+
+if.then1:
+  store i32 100, ptr @g
+  br label %if.end1
+
+if.end1:
+  call void @foo()
+  %cond2 = icmp eq i32 %b, 0
+  br i1 %cond2, label %if.then2, label %if.end2
+
+if.then2:
+  store i32 100, ptr @g
+  br label %if.end2
+
+if.end2:
+  ret i32 0
+}
+
+; Test that two if-regions are merged when there's no potential aliasing
+; between a store in the first if-region and a load in the second if-region's header
+define i32 @test_no_alias(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @test_no_alias
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 42, ptr [[P]], align 4
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr @g, align 4
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[B]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = or i1 [[COND1]], [[COND2]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN2:%.*]], label [[IF_END2:%.*]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    store i32 100, ptr [[P]], align 4
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[IF_END2]]
+; CHECK:       if.end2:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %p = alloca i32
+  store i32 42, ptr %p
+  %cond1 = icmp eq i32 %a, 0
+  br i1 %cond1, label %if.then1, label %if.end1
+
+if.then1:
+  store i32 100, ptr %p  ; No alias with the load below
+  call void @bar() ; No alias with the load below since it's a pure function
+  br label %if.end1
+
+if.end1:
+  %val = load i32, ptr @g
+  %cond2 = icmp eq i32 %b, 0
+  br i1 %cond2, label %if.then2, label %if.end2
+
+if.then2:
+  store i32 100, ptr %p
+  call void @bar()
+  br label %if.end2
+
+if.end2:
+  ret i32 0
+}
+
+attributes #1 = { readnone willreturn nounwind }
diff --git a/llvm/unittests/Analysis/AliasAnalysisTest.cpp b/llvm/unittests/Analysis/AliasAnalysisTest.cpp
index cb8a412f329a6..5d990521d4839 100644
--- a/llvm/unittests/Analysis/AliasAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/AliasAnalysisTest.cpp
@@ -186,6 +186,14 @@ TEST_F(AliasAnalysisTest, getModRefInfo) {
       AtomicRMWInst::Xchg, Addr, ConstantInt::get(IntType, 1), Alignment,
       AtomicOrdering::Monotonic, SyncScope::System, BB);
 
+  FunctionType *FooBarTy = FunctionType::get(Type::getVoidTy(C), {}, false);
+  Function::Create(FooBarTy, Function::ExternalLinkage, "foo", &M);
+  auto *BarF = Function::Create(FooBarTy, Function::ExternalLinkage, "bar", &M);
+  BarF->setDoesNotAccessMemory();
+
+  const Instruction *Foo = CallInst::Create(M.getFunction("foo"), {}, BB);
+  const Instruction *Bar = CallInst::Create(M.getFunction("bar"), {}, BB);
+
   ReturnInst::Create(C, nullptr, BB);
 
   auto &AA = getAAResults(*F);
@@ -203,6 +211,13 @@ TEST_F(AliasAnalysisTest, getModRefInfo) {
   EXPECT_EQ(AA.getModRefInfo(CmpXChg1, std::nullopt), ModRefInfo::ModRef);
   EXPECT_EQ(AA.getModRefInfo(AtomicRMW, MemoryLocation()), ModRefInfo::ModRef);
   EXPECT_EQ(AA.getModRefInfo(AtomicRMW, std::nullopt), ModRefInfo::ModRef);
+  EXPECT_EQ(AA.getModRefInfo(Store1, Load1), ModRefInfo::ModRef);
+  EXPECT_EQ(AA.getModRefInfo(Store1, Store1), ModRefInfo::ModRef);
+  EXPECT_EQ(AA.getModRefInfo(Store1, Add1), ModRefInfo::NoModRef);
+  EXPECT_EQ(AA.getModRefInfo(Store1, Foo), ModRefInfo::ModRef);
+  EXPECT_EQ(AA.getModRefInfo(Store1, Bar), ModRefInfo::NoModRef);
+  EXPECT_EQ(AA.getModRefInfo(Foo, Bar), ModRefInfo::NoModRef);
+  EXPECT_EQ(AA.getModRefInfo(Foo, Foo), ModRefInfo::ModRef);
 }
 
 static Instruction *getInstructionByName(Function &F, StringRef Name) {

From 1d190065d92a7cc9fbe322edbfa233c2318c8839 Mon Sep 17 00:00:00 2001
From: Zichen Lu <mikaovo2000@gmail.com>
Date: Fri, 18 Apr 2025 18:33:31 +0800
Subject: [PATCH 394/710] [mlir][target] RAII wrap moduleToObject timer to
 ensure call `clear` function (#136142)

As title, we need to call `Timer::clear` to avoid extra log like this:
```
===-------------------------------------------------------------------------===
                           ...
===-------------------------------------------------------------------------===
  Total Execution Time: 0.0000 seconds (0.0000 wall clock)

   ---Wall Time---  --- Name ---
        -----       ....
        -----       Total
```
---
 mlir/lib/Target/LLVM/NVVM/Target.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp
index 586748df8d154..f8420385cbec4 100644
--- a/mlir/lib/Target/LLVM/NVVM/Target.cpp
+++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp
@@ -26,6 +26,7 @@
 #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Export.h"
 
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
@@ -673,6 +674,7 @@ NVPTXSerializer::moduleToObject(llvm::Module &llvmModule) {
   llvm::Timer moduleToObjectTimer(
       "moduleToObjectTimer",
       "Timer for perf llvm-ir -> isa and isa -> binary.");
+  auto clear = llvm::make_scope_exit([&]() { moduleToObjectTimer.clear(); });
   // Return LLVM IR if the compilation target is `offload`.
 #define DEBUG_TYPE "serialize-to-llvm"
   LLVM_DEBUG({

From c5d59723cbf9743f7a0534a6ca87383abb6785a7 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Fri, 18 Apr 2025 12:48:16 +0200
Subject: [PATCH 395/710] [clang][bytecode] Reject constexpr-unknown values in
 CheckStore (#136279)

---
 clang/lib/AST/ByteCode/Interp.cpp | 2 ++
 clang/test/CodeGen/p0963r3.cpp    | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 4625154ac353d..b755a072fec88 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -790,6 +790,8 @@ bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
     return false;
   if (!CheckConst(S, OpPC, Ptr))
     return false;
+  if (!S.inConstantContext() && isConstexprUnknown(Ptr))
+    return false;
   return true;
 }
 
diff --git a/clang/test/CodeGen/p0963r3.cpp b/clang/test/CodeGen/p0963r3.cpp
index 4a5e6c3f5d751..65d33397e6173 100644
--- a/clang/test/CodeGen/p0963r3.cpp
+++ b/clang/test/CodeGen/p0963r3.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2c -verify -emit-llvm -triple=x86_64-pc-linux-gnu %s -o - | FileCheck %s
-// RUN: %clang_cc1 -std=c++2c -verify -fsyntax-only -fexperimental-new-constant-interpreter -triple=x86_64-pc-linux-gnu %s
+// RUN: %clang_cc1 -std=c++2c -verify -emit-llvm -triple=x86_64-pc-linux-gnu %s -o -                                         | FileCheck %s
+// RUN: %clang_cc1 -std=c++2c -verify -emit-llvm -triple=x86_64-pc-linux-gnu %s -o - -fexperimental-new-constant-interpreter | FileCheck %s
 // expected-no-diagnostics
 
 namespace std {

From c609cd2df981d1fcbdfefa1e2601b965b9670630 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Fri, 18 Apr 2025 07:09:27 -0400
Subject: [PATCH 396/710] Give this diagnostic a diagnostic group (#136182)

I put this under -Wunitialized because that's the same group it's under
in GCC.

Fixes #41104
---
 clang/docs/ReleaseNotes.rst                      |  4 ++++
 clang/include/clang/Basic/DiagnosticSemaKinds.td |  3 ++-
 clang/test/Misc/warning-flags.c                  |  3 +--
 clang/test/SemaCXX/uninitialized-no-ctor.cpp     | 13 +++++++++++++
 4 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/SemaCXX/uninitialized-no-ctor.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 7b76a2caa694b..f5cd1fbeabcfe 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -391,6 +391,10 @@ Improvements to Clang's diagnostics
 
   Fixes #GH131127
 
+- ``-Wuninitialized`` now diagnoses when a class does not declare any
+  constructors to initialize their non-modifiable members. The diagnostic is
+  not new; being controlled via a warning group is what's new. Fixes #GH41104
+
 Improvements to Clang's time-trace
 ----------------------------------
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 6cbe8b60fe9bf..c29a3422acd26 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -2266,7 +2266,8 @@ def err_constructor_byvalue_arg : Error<
   "copy constructor must pass its first argument by reference">;
 def warn_no_constructor_for_refconst : Warning<
   "%select{struct|interface|union|class|enum}0 %1 does not declare any "
-  "constructor to initialize its non-modifiable members">;
+  "constructor to initialize its non-modifiable members">,
+  InGroup<Uninitialized>;
 def note_refconst_member_not_initialized : Note<
   "%select{const|reference}0 member %1 will never be initialized">;
 def ext_ms_explicit_constructor_call : ExtWarn<
diff --git a/clang/test/Misc/warning-flags.c b/clang/test/Misc/warning-flags.c
index 7ac77cc91fdc1..3dc4bb55aa69c 100644
--- a/clang/test/Misc/warning-flags.c
+++ b/clang/test/Misc/warning-flags.c
@@ -18,7 +18,7 @@ This test serves two purposes:
 
 The list of warnings below should NEVER grow.  It should gradually shrink to 0.
 
-CHECK: Warnings without flags (57):
+CHECK: Warnings without flags (56):
 
 CHECK-NEXT:   ext_expected_semi_decl_list
 CHECK-NEXT:   ext_missing_whitespace_after_macro_name
@@ -57,7 +57,6 @@ CHECK-NEXT:   warn_method_param_redefinition
 CHECK-NEXT:   warn_missing_case_for_condition
 CHECK-NEXT:   warn_missing_dependent_template_keyword
 CHECK-NEXT:   warn_missing_whitespace_after_macro_name
-CHECK-NEXT:   warn_no_constructor_for_refconst
 CHECK-NEXT:   warn_not_compound_assign
 CHECK-NEXT:   warn_objc_property_copy_missing_on_block
 CHECK-NEXT:   warn_objc_protocol_qualifier_missing_id
diff --git a/clang/test/SemaCXX/uninitialized-no-ctor.cpp b/clang/test/SemaCXX/uninitialized-no-ctor.cpp
new file mode 100644
index 0000000000000..7a08f2f3dc37e
--- /dev/null
+++ b/clang/test/SemaCXX/uninitialized-no-ctor.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -fsyntax-only -Wuninitialized -verify %s
+// RUN: %clang_cc1 -fsyntax-only -Wno-uninitialized  -verify=good %s
+//good-no-diagnostics
+
+template <class T>
+class RefMem { // expected-warning {{class 'RefMem<int &>' does not declare any constructor to initialize its non-modifiable members}}
+  T &M; // expected-note {{reference member 'M' will never be initialized}}
+};
+
+struct RefRef {
+  RefMem<int &> R; // expected-note {{in instantiation of template class 'RefMem<int &>' requested here}}
+};
+

From a8fe21f3f502a49cb05b69b0d6fa74472b93888a Mon Sep 17 00:00:00 2001
From: Andrew Savonichev <andrew.savonichev@gmail.com>
Date: Fri, 18 Apr 2025 20:29:19 +0900
Subject: [PATCH 397/710] [clang] Handle instantiated members to determine
 visibility (#136128)

As reported in issue #103477, visibility of instantiated member
functions used to be ignored when calculating visibility of a
specialization.

This patch modifies `getLVForClassMember` to look up for a source
template for an instantiated member, and changes `mergeTemplateLV` to
apply it.

A similar issue was reported in #31462, but it seems that `extern`
declaration with visibility prevents the function from being emitted as
hidden. This behavior seems correct, even though GCC emits it as with
default visibility instead.

Both tests from #103477 and #31462 are added as LIT tests `test72` and
`test73` respectively.
---
 clang/lib/AST/Decl.cpp               | 13 +++++++---
 clang/test/CodeGenCXX/visibility.cpp | 38 +++++++++++++++++++++++++++-
 2 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index ad1cb01592e9b..b59619892979a 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -400,9 +400,9 @@ void LinkageComputer::mergeTemplateLV(
   FunctionTemplateDecl *temp = specInfo->getTemplate();
   // Merge information from the template declaration.
   LinkageInfo tempLV = getLVForDecl(temp, computation);
-  // The linkage of the specialization should be consistent with the
-  // template declaration.
-  LV.setLinkage(tempLV.getLinkage());
+  // The linkage and visibility of the specialization should be
+  // consistent with the template declaration.
+  LV.mergeMaybeWithVisibility(tempLV, considerVisibility);
 
   // Merge information from the template parameters.
   LinkageInfo paramsLV =
@@ -1051,6 +1051,13 @@ LinkageComputer::getLVForClassMember(const NamedDecl *D,
     if (const auto *redeclTemp = dyn_cast<RedeclarableTemplateDecl>(temp)) {
       if (isExplicitMemberSpecialization(redeclTemp)) {
         explicitSpecSuppressor = temp->getTemplatedDecl();
+      } else if (const RedeclarableTemplateDecl *from =
+                     redeclTemp->getInstantiatedFromMemberTemplate()) {
+        // If no explicit visibility is specified yet, and this is an
+        // instantiated member of a template, look up visibility there
+        // as well.
+        LinkageInfo fromLV = from->getLinkageAndVisibility();
+        LV.mergeMaybeWithVisibility(fromLV, considerVisibility);
       }
     }
   }
diff --git a/clang/test/CodeGenCXX/visibility.cpp b/clang/test/CodeGenCXX/visibility.cpp
index e1061f3dbd18f..b69278a71d48e 100644
--- a/clang/test/CodeGenCXX/visibility.cpp
+++ b/clang/test/CodeGenCXX/visibility.cpp
@@ -1457,9 +1457,45 @@ namespace test71 {
   // CHECK-LABEL: declare hidden noundef i32 @_ZN6test713fooIiE3zedEv(
   // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN6test713fooIiE3barIiEET_v(
   // CHECK-LABEL: define linkonce_odr hidden noundef i64 @_ZN6test713fooIlE3zedEv(
-  // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN6test713fooIlE3barIiEET_v(
+  // CHECK-LABEL: define linkonce_odr hidden noundef i32 @_ZN6test713fooIlE3barIiEET_v(
   // CHECK-HIDDEN-LABEL: declare hidden noundef i32 @_ZN6test713fooIiE3zedEv(
   // CHECK-HIDDEN-LABEL: define linkonce_odr noundef i32 @_ZN6test713fooIiE3barIiEET_v(
   // CHECK-HIDDEN-LABEL: define linkonce_odr hidden noundef i64 @_ZN6test713fooIlE3zedEv(
   // CHECK-HIDDEN-LABEL: define linkonce_odr hidden noundef i32 @_ZN6test713fooIlE3barIiEET_v(
 }
+
+// https://github.com/llvm/llvm-project/issues/103477
+namespace test72 {
+  template <class a>
+  struct t {
+    template <int>
+    static HIDDEN void bar() {}
+  };
+
+  void test() {
+      t<char>::bar<1>();
+  }
+  // CHECK-LABEL: define linkonce_odr hidden void @_ZN6test721tIcE3barILi1EEEvv(
+  // CHECK-HIDDEN-LABEL: define linkonce_odr hidden void @_ZN6test721tIcE3barILi1EEEvv(
+}
+
+// https://github.com/llvm/llvm-project/issues/31462
+namespace test73 {
+  template <class T> struct s {
+    template <class U>
+    __attribute__((__visibility__("hidden"))) U should_not_be_exported();
+  };
+
+  template <class T> template <class U> U s<T>::should_not_be_exported() {
+    return U();
+  }
+
+  extern template struct __attribute__((__visibility__("default"))) s<int>;
+
+  int f() {
+    s<int> o;
+    return o.should_not_be_exported<int>();
+  }
+  // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN6test731sIiE22should_not_be_exportedIiEET_v(
+  // CHECK-HIDDEN-LABEL: define linkonce_odr noundef i32 @_ZN6test731sIiE22should_not_be_exportedIiEET_v(
+}

From 6c4caae4495118d680fd08cadca73b8c583719c7 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Fri, 18 Apr 2025 04:35:05 -0700
Subject: [PATCH 398/710] [LLVM][TableGen] Move DecoderEmitter output to
 anonymous namespace (#136214)

- Move the code generated by DecoderEmitter to anonymous namespace.
- Move AMDGPU's usage of this code from header file to .cpp file.

Note, we get build errors like "call to function 'decodeInstruction'
that is neither visible in the template definition nor found by
argument-dependent lookup" if we do not change AMDGPU.
---
 .../Disassembler/AMDGPUDisassembler.cpp       | 40 +++++++++++++++++++
 .../AMDGPU/Disassembler/AMDGPUDisassembler.h  | 34 +---------------
 llvm/utils/TableGen/DecoderEmitter.cpp        |  4 +-
 3 files changed, 44 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 847121f251361..3fbba17159375 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -487,6 +487,46 @@ static DecodeStatus decodeVersionImm(MCInst &Inst, unsigned Imm,
 //
 //===----------------------------------------------------------------------===//
 
+template <typename InsnType>
+DecodeStatus AMDGPUDisassembler::tryDecodeInst(const uint8_t *Table, MCInst &MI,
+                                               InsnType Inst, uint64_t Address,
+                                               raw_ostream &Comments) const {
+  assert(MI.getOpcode() == 0);
+  assert(MI.getNumOperands() == 0);
+  MCInst TmpInst;
+  HasLiteral = false;
+  const auto SavedBytes = Bytes;
+
+  SmallString<64> LocalComments;
+  raw_svector_ostream LocalCommentStream(LocalComments);
+  CommentStream = &LocalCommentStream;
+
+  DecodeStatus Res =
+      decodeInstruction(Table, TmpInst, Inst, Address, this, STI);
+
+  CommentStream = nullptr;
+
+  if (Res != MCDisassembler::Fail) {
+    MI = TmpInst;
+    Comments << LocalComments;
+    return MCDisassembler::Success;
+  }
+  Bytes = SavedBytes;
+  return MCDisassembler::Fail;
+}
+
+template <typename InsnType>
+DecodeStatus
+AMDGPUDisassembler::tryDecodeInst(const uint8_t *Table1, const uint8_t *Table2,
+                                  MCInst &MI, InsnType Inst, uint64_t Address,
+                                  raw_ostream &Comments) const {
+  for (const uint8_t *T : {Table1, Table2}) {
+    if (DecodeStatus Res = tryDecodeInst(T, MI, Inst, Address, Comments))
+      return Res;
+  }
+  return MCDisassembler::Fail;
+}
+
 template <typename T> static inline T eatBytes(ArrayRef<uint8_t>& Bytes) {
   assert(Bytes.size() >= sizeof(T));
   const auto Res =
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
index 29452166e21a0..498ef655b7f45 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
@@ -130,41 +130,11 @@ class AMDGPUDisassembler : public MCDisassembler {
 
   template <typename InsnType>
   DecodeStatus tryDecodeInst(const uint8_t *Table, MCInst &MI, InsnType Inst,
-                             uint64_t Address, raw_ostream &Comments) const {
-    assert(MI.getOpcode() == 0);
-    assert(MI.getNumOperands() == 0);
-    MCInst TmpInst;
-    HasLiteral = false;
-    const auto SavedBytes = Bytes;
-
-    SmallString<64> LocalComments;
-    raw_svector_ostream LocalCommentStream(LocalComments);
-    CommentStream = &LocalCommentStream;
-
-    DecodeStatus Res =
-        decodeInstruction(Table, TmpInst, Inst, Address, this, STI);
-
-    CommentStream = nullptr;
-
-    if (Res != Fail) {
-      MI = TmpInst;
-      Comments << LocalComments;
-      return MCDisassembler::Success;
-    }
-    Bytes = SavedBytes;
-    return MCDisassembler::Fail;
-  }
-
+                             uint64_t Address, raw_ostream &Comments) const;
   template <typename InsnType>
   DecodeStatus tryDecodeInst(const uint8_t *Table1, const uint8_t *Table2,
                              MCInst &MI, InsnType Inst, uint64_t Address,
-                             raw_ostream &Comments) const {
-    for (const uint8_t *T : {Table1, Table2}) {
-      if (DecodeStatus Res = tryDecodeInst(T, MI, Inst, Address, Comments))
-        return Res;
-    }
-    return MCDisassembler::Fail;
-  }
+                             raw_ostream &Comments) const;
 
   Expected<bool> onSymbolStart(SymbolInfoTy &Symbol, uint64_t &Size,
                                ArrayRef<uint8_t> Bytes,
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 9c6015cc24576..75c8c80aebd6d 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -2417,7 +2417,7 @@ void DecoderEmitter::run(raw_ostream &o) {
 #include "llvm/TargetParser/SubtargetFeature.h"
 #include <assert.h>
 
-namespace llvm {
+namespace {
 )";
 
   emitFieldFromInstruction(OS);
@@ -2561,7 +2561,7 @@ namespace llvm {
   // Emit the main entry point for the decoder, decodeInstruction().
   emitDecodeInstruction(OS, IsVarLenInst);
 
-  OS << "\n} // end namespace llvm\n";
+  OS << "\n} // namespace\n";
 }
 
 void llvm::EmitDecoder(const RecordKeeper &RK, raw_ostream &OS,

From c890b7376fc916c1818dc3c211a080601973aa0b Mon Sep 17 00:00:00 2001
From: Raul Tambre <raul@tambre.ee>
Date: Fri, 18 Apr 2025 14:48:30 +0300
Subject: [PATCH 399/710] [lldb][Telemetry] Fix unit test compile failure with
 LLVM_ENABLE_TELEMETRY=0 (#136115)

It needs to be `TEST_F` to access `received_entries`.
Disabling also works based on the test not the fixture name.

Build failure:
```
lldb/unittests/Core/TelemetryTest.cpp:110:17: error: use of undeclared identifier 'received_entries'
  110 |   ASSERT_EQ(1U, received_entries.size());
      |                 ^
lldb/unittests/Core/TelemetryTest.cpp:112:61: error: use of undeclared identifier 'received_entries'
  112 |             llvm::dyn_cast<lldb_private::FakeTelemetryInfo>(received_entries[0])
      |                                                             ^
```

Fixes: 159b872b37363511a359c800bcc9230bb09f2457
---
 lldb/unittests/Core/TelemetryTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/unittests/Core/TelemetryTest.cpp b/lldb/unittests/Core/TelemetryTest.cpp
index 1e41424bac3ce..910149d865c13 100644
--- a/lldb/unittests/Core/TelemetryTest.cpp
+++ b/lldb/unittests/Core/TelemetryTest.cpp
@@ -96,7 +96,7 @@ class TelemetryTest : public testing::Test {
 #if LLVM_ENABLE_TELEMETRY
 #define TELEMETRY_TEST(suite, test) TEST_F(suite, test)
 #else
-#define TELEMETRY_TEST(suite, test) TEST(DISABLED_##suite, test)
+#define TELEMETRY_TEST(suite, test) TEST_F(suite, DISABLED_##test)
 #endif
 
 TELEMETRY_TEST(TelemetryTest, PluginTest) {

From 6462fad3d04d1cc6ceda303a6525742c8b911e79 Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Fri, 18 Apr 2025 13:57:28 +0200
Subject: [PATCH 400/710] [DebugInfo] getMergedLocation: match scopes based on
 their location (#132286)

getMergedLocation uses a common parent scope of the two input locations
for an output location.
It doesn't consider the case when the common parent scope is from a file
other than L1's and L2's files. In that case, it produces a merged location
with an erroneous scope (https://github.com/llvm/llvm-project/issues/122846).

In some cases, such as https://github.com/llvm/llvm-project/pull/125780#issuecomment-2651657856,
L1, L2 having a common parent scope from another file indicate that
the code at L1 and L2 is included from the same source location.

With this commit, getMergedLocation detects that L1, L2, or their common parent
scope files are different. If so, it assumes that L1 and L2 were included
from some source location, and tries to attach the output location to a scope
with the nearest common source location with regard to L1 and L2.
If the nearest common location is also from another file, getMergedLocation returns it
as a merged location, assuming that L1 and L2 belong to files that were both included
in the nearest common location.

Fixes https://github.com/llvm/llvm-project/issues/122846.
---
 llvm/lib/IR/DebugInfoMetadata.cpp             | 143 ++++++++++--
 .../AArch64/merge-nested-block-loc.ll         |  79 +++++++
 .../AArch64/merge-nested-block-loc2.ll        | 130 +++++++++++
 llvm/unittests/IR/MetadataTest.cpp            | 216 +++++++++++++++++-
 4 files changed, 536 insertions(+), 32 deletions(-)
 create mode 100644 llvm/test/DebugInfo/AArch64/merge-nested-block-loc.ll
 create mode 100644 llvm/test/DebugInfo/AArch64/merge-nested-block-loc2.ll

diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index b8b824aed7178..4f1b9e836120e 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -13,7 +13,7 @@
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "LLVMContextImpl.h"
 #include "MetadataImpl.h"
-#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/IR/DebugProgramInstruction.h"
@@ -125,6 +125,98 @@ DILocation *DILocation::getMergedLocations(ArrayRef<DILocation *> Locs) {
   return Merged;
 }
 
+static DILexicalBlockBase *cloneAndReplaceParentScope(DILexicalBlockBase *LBB,
+                                                      DIScope *NewParent) {
+  TempMDNode ClonedScope = LBB->clone();
+  cast<DILexicalBlockBase>(*ClonedScope).replaceScope(NewParent);
+  return cast<DILexicalBlockBase>(
+      MDNode::replaceWithUniqued(std::move(ClonedScope)));
+}
+
+using LineColumn = std::pair<unsigned /* Line */, unsigned /* Column */>;
+
+/// Returns the location of DILocalScope, if present, or a default value.
+static LineColumn getLocalScopeLocationOr(DIScope *S, LineColumn Default) {
+  assert(isa<DILocalScope>(S) && "Expected DILocalScope.");
+
+  if (isa<DILexicalBlockFile>(S))
+    return Default;
+  if (auto *LB = dyn_cast<DILexicalBlock>(S))
+    return {LB->getLine(), LB->getColumn()};
+  if (auto *SP = dyn_cast<DISubprogram>(S))
+    return {SP->getLine(), 0u};
+
+  llvm_unreachable("Unhandled type of DILocalScope.");
+}
+
+// Returns the nearest matching scope inside a subprogram.
+template <typename MatcherT>
+static std::pair<DIScope *, LineColumn>
+getNearestMatchingScope(const DILocation *L1, const DILocation *L2) {
+  MatcherT Matcher;
+
+  DIScope *S1 = L1->getScope();
+  DIScope *S2 = L2->getScope();
+
+  LineColumn Loc1(L1->getLine(), L1->getColumn());
+  for (; S1; S1 = S1->getScope()) {
+    Loc1 = getLocalScopeLocationOr(S1, Loc1);
+    Matcher.insert(S1, Loc1);
+    if (isa<DISubprogram>(S1))
+      break;
+  }
+
+  LineColumn Loc2(L2->getLine(), L2->getColumn());
+  for (; S2; S2 = S2->getScope()) {
+    Loc2 = getLocalScopeLocationOr(S2, Loc2);
+
+    if (DIScope *S = Matcher.match(S2, Loc2))
+      return std::make_pair(S, Loc2);
+
+    if (isa<DISubprogram>(S2))
+      break;
+  }
+  return std::make_pair(nullptr, LineColumn(L2->getLine(), L2->getColumn()));
+}
+
+// Matches equal scopes.
+struct EqualScopesMatcher {
+  SmallPtrSet<DIScope *, 8> Scopes;
+
+  void insert(DIScope *S, LineColumn Loc) { Scopes.insert(S); }
+
+  DIScope *match(DIScope *S, LineColumn Loc) {
+    return Scopes.contains(S) ? S : nullptr;
+  }
+};
+
+// Matches scopes with the same location.
+struct ScopeLocationsMatcher {
+  SmallMapVector<std::pair<DIFile *, LineColumn>, SmallSetVector<DIScope *, 8>,
+                 8>
+      Scopes;
+
+  void insert(DIScope *S, LineColumn Loc) {
+    Scopes[{S->getFile(), Loc}].insert(S);
+  }
+
+  DIScope *match(DIScope *S, LineColumn Loc) {
+    auto ScopesAtLoc = Scopes.find({S->getFile(), Loc});
+    // No scope found with the given location.
+    if (ScopesAtLoc == Scopes.end())
+      return nullptr;
+
+    // Prefer S over other scopes with the same location.
+    if (ScopesAtLoc->second.contains(S))
+      return S;
+
+    if (!ScopesAtLoc->second.empty())
+      return *ScopesAtLoc->second.begin();
+
+    llvm_unreachable("Scopes must not have empty entries.");
+  }
+};
+
 DILocation *DILocation::getMergedLocation(DILocation *LocA, DILocation *LocB) {
   if (!LocA || !LocB)
     return nullptr;
@@ -208,28 +300,31 @@ DILocation *DILocation::getMergedLocation(DILocation *LocA, DILocation *LocB) {
     if (L1->getScope()->getSubprogram() != L2->getScope()->getSubprogram())
       return nullptr;
 
-    // Return the nearest common scope inside a subprogram.
-    auto GetNearestCommonScope = [](DIScope *S1, DIScope *S2) -> DIScope * {
-      SmallPtrSet<DIScope *, 8> Scopes;
-      for (; S1; S1 = S1->getScope()) {
-        Scopes.insert(S1);
-        if (isa<DISubprogram>(S1))
-          break;
-      }
-
-      for (; S2; S2 = S2->getScope()) {
-        if (Scopes.count(S2))
-          return S2;
-        if (isa<DISubprogram>(S2))
-          break;
-      }
-
-      return nullptr;
-    };
-
-    auto Scope = GetNearestCommonScope(L1->getScope(), L2->getScope());
+    // Find nearest common scope inside subprogram.
+    DIScope *Scope = getNearestMatchingScope<EqualScopesMatcher>(L1, L2).first;
     assert(Scope && "No common scope in the same subprogram?");
 
+    // Try using the nearest scope with common location if files are different.
+    if (Scope->getFile() != L1->getFile() || L1->getFile() != L2->getFile()) {
+      auto [CommonLocScope, CommonLoc] =
+          getNearestMatchingScope<ScopeLocationsMatcher>(L1, L2);
+
+      // If CommonLocScope is a DILexicalBlockBase, clone it and locate
+      // a new scope inside the nearest common scope to preserve
+      // lexical blocks structure.
+      if (auto *LBB = dyn_cast<DILexicalBlockBase>(CommonLocScope);
+          LBB && LBB != Scope)
+        CommonLocScope = cloneAndReplaceParentScope(LBB, Scope);
+
+      Scope = CommonLocScope;
+
+      // If files are still different, assume that L1 and L2 were "included"
+      // from CommonLoc. Use it as merged location.
+      if (Scope->getFile() != L1->getFile() || L1->getFile() != L2->getFile())
+        return DILocation::get(C, CommonLoc.first, CommonLoc.second,
+                               CommonLocScope, InlinedAt);
+    }
+
     bool SameLine = L1->getLine() == L2->getLine();
     bool SameCol = L1->getColumn() == L2->getColumn();
     unsigned Line = SameLine ? L1->getLine() : 0;
@@ -1187,10 +1282,8 @@ DILocalScope *DILocalScope::cloneScopeForSubprogram(
   // cached result).
   DIScope *UpdatedScope = CachedResult ? CachedResult : &NewSP;
   for (DIScope *ScopeToUpdate : reverse(ScopeChain)) {
-    TempMDNode ClonedScope = ScopeToUpdate->clone();
-    cast<DILexicalBlockBase>(*ClonedScope).replaceScope(UpdatedScope);
-    UpdatedScope =
-        cast<DIScope>(MDNode::replaceWithUniqued(std::move(ClonedScope)));
+    UpdatedScope = cloneAndReplaceParentScope(
+        cast<DILexicalBlockBase>(ScopeToUpdate), UpdatedScope);
     Cache[ScopeToUpdate] = UpdatedScope;
   }
 
diff --git a/llvm/test/DebugInfo/AArch64/merge-nested-block-loc.ll b/llvm/test/DebugInfo/AArch64/merge-nested-block-loc.ll
new file mode 100644
index 0000000000000..bff705bfc1ad4
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/merge-nested-block-loc.ll
@@ -0,0 +1,79 @@
+; RUN: opt -mtriple=aarch64-unknown-linux-gnu -S %s -passes=sroa -o - | FileCheck %s
+
+; In this test we want to ensure that the merged location of phi instruction
+; belongs to the correct scope (DILexicalBlockFile), so that line number
+; of that location belongs to the corresponding file.
+
+; Generated with clang from
+; # 1 "1.c" 1
+; # 1 "1.c" 2
+; int foo(int a) {
+;   int i = 0;
+;   if ((a & 1) == 1) {
+;     a -= 1;
+; # 1 "m.c" 1
+; # 40 "m.c"
+; i += a;
+; i -= 10*a;
+; i *= a*a;
+; # 6 "1.c" 2
+;  } else {
+;     a += 3;
+; # 1 "m.c" 1
+; # 40 "m.c"
+; i += a;
+; i -= 10*a;
+; i *= a*a;
+; # 9 "1.c" 2
+;  }
+;   return i;
+; }
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+define i32 @foo() !dbg !3 {
+; CHECK:    phi i32 {{.*}}, !dbg [[PHILOC:![0-9]+]]
+;
+entry:
+  %i = alloca i32, align 4
+  br i1 false, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  store i32 1, ptr %i, align 4, !dbg !7
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  store i32 0, ptr %i, align 4, !dbg !12
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %0 = load i32, ptr %i, align 4
+  ret i32 0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 21.0.0git", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "repro.c", directory: "")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "foo", scope: !4, file: !4, line: 1, type: !5, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !6)
+!4 = !DIFile(filename: "1.c", directory: "")
+!5 = !DISubroutineType(types: !6)
+!6 = !{}
+!7 = !DILocation(line: 42, column: 3, scope: !8)
+!8 = !DILexicalBlockFile(scope: !10, file: !9, discriminator: 0)
+!9 = !DIFile(filename: "m.c", directory: "")
+!10 = distinct !DILexicalBlock(scope: !11, file: !4, line: 3, column: 21)
+!11 = distinct !DILexicalBlock(scope: !3, file: !4, line: 3, column: 7)
+!12 = !DILocation(line: 42, column: 3, scope: !13)
+!13 = !DILexicalBlockFile(scope: !14, file: !9, discriminator: 0)
+!14 = distinct !DILexicalBlock(scope: !11, file: !4, line: 6, column: 9)
+
+; CHECK: [[SP:![0-9]+]] = distinct !DISubprogram(name: "foo", scope: [[FILE1:![0-9]+]], file: [[FILE1]], line: 1
+; CHECK: [[FILE1]] = !DIFile(filename: "1.c", directory: "")
+; CHECK: [[PHILOC]] = !DILocation(line: 42, column: 3, scope: [[LBF:![0-9]+]])
+; CHECK: [[LBF]] = !DILexicalBlockFile(scope: [[LB:![0-9]+]], file: [[FILE2:![0-9]+]], discriminator: 0)
+; CHECK: [[FILE2]] = !DIFile(filename: "m.c", directory: "")
+; CHECK: [[LB]] = distinct !DILexicalBlock(scope: [[SP]], file: [[FILE1]], line: 3, column: 7)
diff --git a/llvm/test/DebugInfo/AArch64/merge-nested-block-loc2.ll b/llvm/test/DebugInfo/AArch64/merge-nested-block-loc2.ll
new file mode 100644
index 0000000000000..e5c9a32318b00
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/merge-nested-block-loc2.ll
@@ -0,0 +1,130 @@
+; RUN: opt -mtriple=aarch64-unknown-linux-gnu -S %s -passes=sroa -o - | FileCheck %s
+
+; In this test we want to ensure that getMergedLocations uses common include
+; location if incoming locations belong to different files.
+; The location of phi instruction merged from locations of %mul3 and %mul10
+; should be the location of do-loop lexical block from y.c.
+
+; Generated with clang from
+;
+; main.c:
+;   int foo(int a) {
+;     int i = 0;
+;     if ((a & 1) == 1) {
+;       a -= 1;
+;   #define A
+;   #include "y.c"
+;    } else {
+;       a += 3;
+;   #undef A
+;   #include "y.c"
+;    }
+;     return i;
+;   }
+;
+; y.c:
+;   # 300 "y.c" 1
+;   do {
+;   #ifdef A
+;   #include "z1.c"
+;   #else
+;   #include "z2.c"
+;   #endif
+;   } while (0);
+;
+; z1.c:
+;   # 100 "z1.c" 1
+;   i += a;
+;   i -= 10*a;
+;   i *= a*a;
+;
+; z2.c:
+;   # 200 "z1.c" 1
+;   i += a;
+;   i -= 10*a;
+;   i *= a*a;
+;
+; Preprocessed source:
+;
+; # 1 "main.c"
+; int foo(int a) {
+;   int i = 0;
+;   if ((a & 1) == 1) {
+;     a -= 1;
+; # 300 "y.c" 1
+; do {
+; # 100 "z1.c" 1
+; i += a;
+; i -= 10*a;
+; i *= a*a;
+; # 303 "y.c" 2
+; } while (0);
+; # 7 "main.c" 2
+;  } else {
+;     a += 3;
+; # 300 "y.c" 1
+; do {
+; # 200 "z2.c" 1
+; i += a;
+; i -= 10*a;
+; i *= a*a;
+; # 305 "y.c" 2
+; } while (0);
+; # 11 "main.c" 2
+;  }
+;   return i;
+; }
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64-apple-macosx15.0.0"
+
+define i32 @foo() !dbg !3 {
+; CHECK:    phi i32 {{.*}}, !dbg [[PHILOC:![0-9]+]]
+;
+entry:
+  %i = alloca i32, align 4
+  br i1 false, label %do.body, label %if.else
+
+do.body:                                          ; preds = %entry
+  store i32 1, ptr %i, align 4, !dbg !6
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  store i32 0, ptr %i, align 4, !dbg !14
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %do.body
+  %0 = load i32, ptr %i, align 4
+  ret i32 0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: Apple, sysroot: "/")
+!1 = !DIFile(filename: "main.c", directory: "")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !4, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !5)
+!4 = !DISubroutineType(types: !5)
+!5 = !{}
+!6 = !DILocation(line: 102, column: 3, scope: !7)
+!7 = !DILexicalBlockFile(scope: !9, file: !8, discriminator: 0)
+!8 = !DIFile(filename: "z1.c", directory: "")
+!9 = distinct !DILexicalBlock(scope: !11, file: !10, line: 300, column: 4)
+!10 = !DIFile(filename: "y.c", directory: "")
+!11 = !DILexicalBlockFile(scope: !12, file: !10, discriminator: 0)
+!12 = distinct !DILexicalBlock(scope: !13, file: !1, line: 3, column: 21)
+!13 = distinct !DILexicalBlock(scope: !3, file: !1, line: 3, column: 7)
+!14 = !DILocation(line: 202, column: 3, scope: !15)
+!15 = !DILexicalBlockFile(scope: !17, file: !16, discriminator: 0)
+!16 = !DIFile(filename: "z2.c", directory: "")
+!17 = distinct !DILexicalBlock(scope: !18, file: !10, line: 300, column: 4)
+!18 = !DILexicalBlockFile(scope: !19, file: !10, discriminator: 0)
+!19 = distinct !DILexicalBlock(scope: !13, file: !1, line: 7, column: 9)
+
+; CHECK: [[FILE_MAIN:![0-9]+]] = !DIFile(filename: "main.c"
+; CHECK: [[SP:![0-9]+]] = distinct !DISubprogram(name: "foo", scope: [[FILE_MAIN]], file: [[FILE_MAIN]], line: 1
+; CHECK: [[PHILOC]] = !DILocation(line: 300, column: 4, scope: [[BLOCK_Y:![0-9]+]])
+; CHECK-NEXT: [[BLOCK_Y]] = !DILexicalBlock(scope: [[BLOCK_MAIN:![0-9]+]], file: [[FILE_Y:![0-9]+]], line: 300, column: 4)
+; CHECK-NEXT: [[FILE_Y]] = !DIFile(filename: "y.c"
+; CHECK: [[BLOCK_MAIN]] = distinct !DILexicalBlock(scope: [[SP]], file: [[FILE_MAIN]], line: 3, column: 7)
diff --git a/llvm/unittests/IR/MetadataTest.cpp b/llvm/unittests/IR/MetadataTest.cpp
index 94cebb0406598..e710f7845df5e 100644
--- a/llvm/unittests/IR/MetadataTest.cpp
+++ b/llvm/unittests/IR/MetadataTest.cpp
@@ -85,10 +85,10 @@ class MetadataTest : public testing::Test {
     return DISubroutineType::getDistinct(Context, DINode::FlagZero, 0,
                                          getNode(nullptr));
   }
-  DISubprogram *getSubprogram() {
-    return DISubprogram::getDistinct(
-        Context, nullptr, "", "", nullptr, 0, nullptr, 0, nullptr, 0, 0,
-        DINode::FlagZero, DISubprogram::SPFlagZero, nullptr);
+  DISubprogram *getSubprogram(DIFile *F = nullptr) {
+    return DISubprogram::getDistinct(Context, nullptr, "", "", F, 0, nullptr, 0,
+                                     nullptr, 0, 0, DINode::FlagZero,
+                                     DISubprogram::SPFlagZero, nullptr);
   }
   DIFile *getFile() {
     return DIFile::getDistinct(Context, "file.c", "/path/to/dir");
@@ -919,8 +919,9 @@ TEST_F(MDNodeTest, deleteTemporaryWithTrackingRef) {
 typedef MetadataTest DILocationTest;
 
 TEST_F(DILocationTest, Merge) {
-  DISubprogram *N = getSubprogram();
-  DIScope *S = DILexicalBlock::get(Context, N, getFile(), 3, 4);
+  DIFile *F = getFile();
+  DISubprogram *N = getSubprogram(F);
+  DIScope *S = DILexicalBlock::get(Context, N, F, 3, 4);
 
   {
     // Identical.
@@ -932,6 +933,18 @@ TEST_F(DILocationTest, Merge) {
     EXPECT_EQ(N, M->getScope());
   }
 
+  {
+    // Identical, inside DILexicalBlockFile.
+    auto *OtherF = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    auto *LBF = DILexicalBlockFile::get(Context, S, OtherF, 0);
+    auto *A = DILocation::get(Context, 2, 7, LBF);
+    auto *B = DILocation::get(Context, 2, 7, LBF);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(2u, M->getLine());
+    EXPECT_EQ(7u, M->getColumn());
+    EXPECT_EQ(LBF, M->getScope());
+  }
+
   {
     // Identical, different scopes.
     auto *A = DILocation::get(Context, 2, 7, N);
@@ -956,7 +969,22 @@ TEST_F(DILocationTest, Merge) {
   }
 
   {
-    // Different lines, same scopes.
+    // Same line, different column, same DILexicalBlockFile scope.
+    auto *OtherF = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    auto *LBF = DILexicalBlockFile::get(Context, S, OtherF, 0);
+    auto *A = DILocation::get(Context, 2, 7, LBF);
+    auto *B = DILocation::get(Context, 2, 10, LBF);
+    auto *M0 = DILocation::getMergedLocation(A, B);
+    auto *M1 = DILocation::getMergedLocation(B, A);
+    for (auto *M : {M0, M1}) {
+      EXPECT_EQ(2u, M->getLine());
+      EXPECT_EQ(0u, M->getColumn());
+      EXPECT_EQ(LBF, M->getScope());
+    }
+  }
+
+  {
+    // Different lines, same DISubprogram scopes.
     auto *A = DILocation::get(Context, 1, 6, N);
     auto *B = DILocation::get(Context, 2, 7, N);
     auto *M = DILocation::getMergedLocation(A, B);
@@ -965,6 +993,28 @@ TEST_F(DILocationTest, Merge) {
     EXPECT_EQ(N, M->getScope());
   }
 
+  {
+    // Different lines, same DILexicalBlockFile scopes.
+    auto *OtherF = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    auto *LBF = DILexicalBlockFile::get(Context, S, OtherF, 0);
+    auto *A = DILocation::get(Context, 1, 6, LBF);
+    auto *B = DILocation::get(Context, 2, 7, LBF);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(0u, M->getLine());
+    EXPECT_EQ(0u, M->getColumn());
+    EXPECT_EQ(LBF, M->getScope());
+  }
+
+  {
+    // Different lines, same DILexicalBlock scopes.
+    auto *A = DILocation::get(Context, 1, 6, S);
+    auto *B = DILocation::get(Context, 2, 7, S);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(0u, M->getLine());
+    EXPECT_EQ(0u, M->getColumn());
+    EXPECT_EQ(S, M->getScope());
+  }
+
   {
     // Twisty locations, all different, same function.
     auto *A = DILocation::get(Context, 1, 6, N);
@@ -975,6 +1025,158 @@ TEST_F(DILocationTest, Merge) {
     EXPECT_EQ(N, M->getScope());
   }
 
+  {
+    // Different files, same line numbers, same subprogram.
+    auto *F1 = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    auto *F2 = DIFile::getDistinct(Context, "file2.c", "/path/to/dir");
+    DISubprogram *N = getSubprogram(F1);
+    auto *LBF = DILexicalBlockFile::get(Context, N, F2, 0);
+    auto *A = DILocation::get(Context, 1, 6, N);
+    auto *B = DILocation::get(Context, 1, 6, LBF);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(0u, M->getLine());
+    EXPECT_EQ(0u, M->getColumn());
+    EXPECT_EQ(N, M->getScope());
+  }
+
+  {
+    // Different files, same line numbers.
+    auto *F1 = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    auto *F2 = DIFile::getDistinct(Context, "file2.c", "/path/to/dir");
+    DISubprogram *N = getSubprogram(F1);
+    auto *LB = DILexicalBlock::getDistinct(Context, N, F1, 4, 9);
+    auto *LBF = DILexicalBlockFile::get(Context, LB, F2, 0);
+    auto *A = DILocation::get(Context, 1, 6, LB);
+    auto *B = DILocation::get(Context, 1, 6, LBF);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(4u, M->getLine());
+    EXPECT_EQ(9u, M->getColumn());
+    EXPECT_EQ(LB, M->getScope());
+  }
+
+  {
+    // Different files, same line numbers,
+    // both locations have DILexicalBlockFile scopes.
+    auto *F1 = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    auto *F2 = DIFile::getDistinct(Context, "file2.c", "/path/to/dir");
+    auto *F3 = DIFile::getDistinct(Context, "file3.c", "/path/to/dir");
+    DISubprogram *N = getSubprogram(F1);
+    auto *LB = DILexicalBlock::getDistinct(Context, N, F1, 4, 9);
+    auto *LBF1 = DILexicalBlockFile::get(Context, LB, F2, 0);
+    auto *LBF2 = DILexicalBlockFile::get(Context, LB, F3, 0);
+    auto *A = DILocation::get(Context, 1, 6, LBF1);
+    auto *B = DILocation::get(Context, 1, 6, LBF2);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(4u, M->getLine());
+    EXPECT_EQ(9u, M->getColumn());
+    EXPECT_EQ(LB, M->getScope());
+  }
+
+  {
+    // Same file, same line numbers, but different LBF objects.
+    // both locations have DILexicalBlockFile scope.
+    auto *F1 = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    DISubprogram *N = getSubprogram(F1);
+    auto *LB1 = DILexicalBlock::getDistinct(Context, N, F1, 4, 9);
+    auto *LB2 = DILexicalBlock::getDistinct(Context, N, F1, 5, 9);
+    auto *F2 = DIFile::getDistinct(Context, "file2.c", "/path/to/dir");
+    auto *LBF1 = DILexicalBlockFile::get(Context, LB1, F2, 0);
+    auto *LBF2 = DILexicalBlockFile::get(Context, LB2, F2, 0);
+    auto *A = DILocation::get(Context, 1, 6, LBF1);
+    auto *B = DILocation::get(Context, 1, 6, LBF2);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(1u, M->getLine());
+    EXPECT_EQ(6u, M->getColumn());
+    EXPECT_EQ(LBF1->getFile(), M->getScope()->getFile());
+    EXPECT_EQ(N, M->getScope()->getScope());
+  }
+
+  {
+    // Merge locations A and B, where B is included in A's file
+    // at the same position as A's position.
+    auto *F1 = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    DISubprogram *N = getSubprogram(F1);
+    auto *LB = DILexicalBlock::getDistinct(Context, N, F1, 4, 9);
+    auto *F2 = DIFile::getDistinct(Context, "file2.c", "/path/to/dir");
+    auto *LBF = DILexicalBlockFile::get(Context, LB, F2, 0);
+    auto *A = DILocation::get(Context, 4, 9, LB);
+    auto *B = DILocation::get(Context, 1, 6, LBF);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(4u, M->getLine());
+    EXPECT_EQ(9u, M->getColumn());
+    EXPECT_EQ(LB, M->getScope());
+  }
+
+  {
+    // Different locations from different files included from the same block.
+    auto *F1 = DIFile::getDistinct(Context, "file1.c", "/path/to/dir");
+    DISubprogram *N = getSubprogram(F1);
+
+    auto *LBCommon = DILexicalBlock::getDistinct(Context, N, F1, 4, 9);
+
+    auto *F2 = DIFile::getDistinct(Context, "file2.c", "/path/to/dir");
+    LBCommon = DILexicalBlock::getDistinct(Context, LBCommon, F2, 5, 9);
+
+    auto *F3 = DIFile::getDistinct(Context, "file3.c", "/path/to/dir");
+    auto *LB1 = DILexicalBlock::getDistinct(Context, LBCommon, F3, 6, 9);
+
+    auto *F4 = DIFile::getDistinct(Context, "file4.c", "/path/to/dir");
+    auto *LB2 = DILexicalBlock::getDistinct(Context, LBCommon, F4, 7, 9);
+
+    auto *F5 = DIFile::getDistinct(Context, "file5.c", "/path/to/dir");
+    auto *LBF1 = DILexicalBlockFile::get(Context, LB1, F5, 0);
+
+    auto *A = DILocation::get(Context, 8, 9, LB2);
+    auto *B = DILocation::get(Context, 9, 6, LBF1);
+    auto *M = DILocation::getMergedLocation(A, B);
+    EXPECT_EQ(5u, M->getLine());
+    EXPECT_EQ(9u, M->getColumn());
+    EXPECT_EQ(LBCommon, M->getScope());
+  }
+
+  {
+    // Different locations from different files having common include parent.
+    auto *LBCommon = DILexicalBlock::getDistinct(Context, N, F, 4, 1);
+
+    // Different scopes.
+    DILexicalBlock *Block[2] = {};
+    Block[0] = DILexicalBlock::get(Context, LBCommon, F, 10, 2);
+    Block[1] = DILexicalBlock::get(Context, LBCommon, F, 20, 3);
+
+    // Includes of the same file.
+    auto *F1 = DIFile::getDistinct(Context, "file1.inc", "/path/to/dir");
+    DILexicalBlock *Block1[2] = {};
+    Block1[0] = DILexicalBlock::get(
+        Context, DILexicalBlockFile::get(Context, Block[0], F1, 0), F1, 30, 4);
+    Block1[1] = DILexicalBlock::get(
+        Context, DILexicalBlockFile::get(Context, Block[1], F1, 0), F1, 30, 4);
+
+    // Different sub-includes.
+    DIFile *F2[2] = {};
+    DILexicalBlock *Block2[2] = {};
+
+    F2[0] = DIFile::getDistinct(Context, "file2_a.inc", "/path/to/dir");
+    Block2[0] = DILexicalBlock::get(
+        Context, DILexicalBlockFile::get(Context, Block1[0], F2[0], 0), F2[0],
+        40, 5);
+
+    F2[1] = DIFile::getDistinct(Context, "file2_b.inc", "/path/to/dir");
+    Block2[1] = DILexicalBlock::get(
+        Context, DILexicalBlockFile::get(Context, Block1[1], F2[1], 0), F2[1],
+        50, 6);
+
+    auto *A = DILocation::get(Context, 41, 7, Block2[0]);
+    auto *B = DILocation::get(Context, 51, 8, Block2[1]);
+    auto *M = DILocation::getMergedLocation(A, B);
+    auto *MScope = dyn_cast<DILexicalBlock>(M->getScope());
+    EXPECT_EQ(30u, M->getLine());
+    EXPECT_EQ(4u, M->getColumn());
+    EXPECT_EQ(Block1[0]->getFile(), MScope->getFile());
+    EXPECT_EQ(Block1[0]->getLine(), MScope->getLine());
+    EXPECT_EQ(Block1[0]->getColumn(), MScope->getColumn());
+    EXPECT_EQ(LBCommon, MScope->getScope());
+  }
+
   {
     // Different function, same inlined-at.
     auto *F = getFile();

From e1b14d4e1c8ecca73e13a0629960d530e336a0d8 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Fri, 18 Apr 2025 05:01:01 -0700
Subject: [PATCH 401/710] [Clang][GPU] Fix unit test for NVPTX tid.x intrinsic
 (#136297)

- llvm.nvvm.read.ptx.sreg.tid.x does not have the result range attribute
yet.
---
 clang/test/Headers/gpuintrin_lang.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/Headers/gpuintrin_lang.c b/clang/test/Headers/gpuintrin_lang.c
index ab660ac5c8a49..b804d46071507 100644
--- a/clang/test/Headers/gpuintrin_lang.c
+++ b/clang/test/Headers/gpuintrin_lang.c
@@ -36,7 +36,7 @@ __device__ int foo() { return __gpu_thread_id_x(); }
 // CUDA-LABEL: define dso_local i32 @foo(
 // CUDA-SAME: ) #[[ATTR0:[0-9]+]] {
 // CUDA-NEXT:  [[ENTRY:.*:]]
-// CUDA-NEXT:    [[TMP0:%.*]] = call range(i32 0, 1024) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+// CUDA-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
 // CUDA-NEXT:    ret i32 [[TMP0]]
 //
 // HIP-LABEL: define dso_local i32 @foo(

From 3ed83630b263f667bacd1ef1a65f26d8665c282e Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Fri, 18 Apr 2025 05:12:35 -0700
Subject: [PATCH 402/710] [NFC][LLVM][TableGen] Use `decodeULEB128` for
 `OPC_SoftFail` emission (#136220)

- Use `decodeULEB128` to decode +ve/-ve mask in OPC_SoftFail case.
- Use current `I`/`E` iterators as inputs to `decodeULEB128`.
---
 llvm/utils/TableGen/DecoderEmitter.cpp | 50 ++++++++++----------------
 1 file changed, 18 insertions(+), 32 deletions(-)

diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 75c8c80aebd6d..a3fe49ef25de7 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -849,8 +849,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
 
       // ULEB128 encoded start value.
       const char *ErrMsg = nullptr;
-      unsigned Start = decodeULEB128(Table.data() + Pos + 1, nullptr,
-                                     Table.data() + Table.size(), &ErrMsg);
+      unsigned Start = decodeULEB128(&*I, nullptr, &*E, &ErrMsg);
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
       emitULEB128(I, OS);
 
@@ -904,8 +903,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       ++I;
       // Decode the Opcode value.
       const char *ErrMsg = nullptr;
-      unsigned Opc = decodeULEB128(Table.data() + Pos + 1, nullptr,
-                                   Table.data() + Table.size(), &ErrMsg);
+      unsigned Opc = decodeULEB128(&*I, nullptr, &*E, &ErrMsg);
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
 
       OS << Indent << "MCD::OPC_" << (IsTry ? "Try" : "") << "Decode, ";
@@ -934,34 +932,22 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     }
     case MCD::OPC_SoftFail: {
       ++I;
-      OS << Indent << "MCD::OPC_SoftFail";
-      // Positive mask
-      uint64_t Value = 0;
-      unsigned Shift = 0;
-      do {
-        OS << ", " << (unsigned)*I;
-        Value += ((uint64_t)(*I & 0x7f)) << Shift;
-        Shift += 7;
-      } while (*I++ >= 128);
-      if (Value > 127) {
-        OS << " /* 0x";
-        OS.write_hex(Value);
-        OS << " */";
-      }
-      // Negative mask
-      Value = 0;
-      Shift = 0;
-      do {
-        OS << ", " << (unsigned)*I;
-        Value += ((uint64_t)(*I & 0x7f)) << Shift;
-        Shift += 7;
-      } while (*I++ >= 128);
-      if (Value > 127) {
-        OS << " /* 0x";
-        OS.write_hex(Value);
-        OS << " */";
-      }
-      OS << ",\n";
+      OS << Indent << "MCD::OPC_SoftFail, ";
+      // Decode the positive mask.
+      const char *ErrMsg = nullptr;
+      uint64_t PositiveMask = decodeULEB128(&*I, nullptr, &*E, &ErrMsg);
+      assert(ErrMsg == nullptr && "ULEB128 value too large!");
+      emitULEB128(I, OS);
+
+      // Decode the negative mask.
+      uint64_t NegativeMask = decodeULEB128(&*I, nullptr, &*E, &ErrMsg);
+      assert(ErrMsg == nullptr && "ULEB128 value too large!");
+      emitULEB128(I, OS);
+      OS << "// +ve mask: 0x";
+      OS.write_hex(PositiveMask);
+      OS << ", -ve mask: 0x";
+      OS.write_hex(NegativeMask);
+      OS << '\n';
       break;
     }
     case MCD::OPC_Fail: {

From db0f754c5af8e6c96770533520bf8b17fc0dc977 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 18 Apr 2025 07:43:31 -0500
Subject: [PATCH 403/710] [OpenMP] Remove 'libomptarget.devicertl.a' fatbinary
 and use static library (#126143)

Summary:
Currently, we build a single `libomptarget.devicertl.a` which is a
fatbinary. It is a host object file that contains the embedded archive
files for both the NVIDIA and AMDGPU targets. This was done primarily as
a convenience due to naming conflicts. Now that the clang driver for the
GPU targets can appropriate link via the per-target runtime-dir, we can
just make two separate static libraries and remove the indirection.

This patch creates two new static libraries that get installed into
```
/lib/amdgcn-amd-amdhsa/libomp.a
/lib/nvptx64-nvidia-cuda/libomp.a
```
for AMDGPU and NVPTX respectively. The link job created by the linker
wrapper now simply needs to do `-lomp` and it will search those
directories and link those static libraries. This requires far less
special handling.

This patch is a precursor to changing the build system entirely to be a
runtimes based one. Soon this target will be a standard `add_library`
and done through the GPU runtime targets.

NOTE that this actually does remove an additional optimization step.
Previously we merged all of the files into a single bitcode object and
forcibly internalized some definitions. This, instead, just treats them
like a normal static library. This may possibly affect performance for
some files, but I think it's better overall to use static library
semantics because it allows us to have an 'include-what-you-use'
relationship with the library.

Performance testing will be required. If we really need the merged blob
then we can simply pack that into a new static library.
---
 clang/lib/Driver/ToolChains/Clang.cpp      |   5 +
 clang/lib/Driver/ToolChains/CommonArgs.cpp |   4 -
 clang/test/Driver/openmp-offload-gpu.c     |   6 +-
 offload/DeviceRTL/CMakeLists.txt           | 160 +++++----------------
 offload/DeviceRTL/src/exports              |  19 ---
 offload/test/lit.cfg                       |  36 ++---
 6 files changed, 65 insertions(+), 165 deletions(-)
 delete mode 100644 offload/DeviceRTL/src/exports

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index b2dd4b3b54869..67a800a643cbe 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -9267,6 +9267,11 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
           A->render(Args, LinkerArgs);
       }
 
+      // If this is OpenMP the device linker will need `-lompdevice`.
+      if (Kind == Action::OFK_OpenMP && !Args.hasArg(OPT_no_offloadlib) &&
+          (TC->getTriple().isAMDGPU() || TC->getTriple().isNVPTX()))
+        LinkerArgs.emplace_back("-lompdevice");
+
       // Forward all of these to the appropriate toolchain.
       for (StringRef Arg : CompilerArgs)
         CmdArgs.push_back(Args.MakeArgString(
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 7aab849abe0c1..8646c55060b17 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1309,10 +1309,6 @@ bool tools::addOpenMPRuntime(const Compilation &C, ArgStringList &CmdArgs,
   if (IsOffloadingHost)
     CmdArgs.push_back("-lomptarget");
 
-  if (IsOffloadingHost &&
-      Args.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib, true))
-    CmdArgs.push_back("-lomptarget.devicertl");
-
   addArchSpecificRPath(TC, Args, CmdArgs);
 
   addOpenMPRuntimeLibraryPath(TC, Args, CmdArgs);
diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c
index 8a7edaef7cc45..5ad038b7dbd37 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -324,18 +324,18 @@
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=sm_52 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
 // RUN:     -foffload-lto %s 2>&1 | FileCheck --check-prefix=CHECK-LTO-LIBRARY %s
 
-// CHECK-LTO-LIBRARY: {{.*}}-lomptarget{{.*}}-lomptarget.devicertl
+// CHECK-LTO-LIBRARY: --device-linker={{.*}}=-lomp{{.*}}-lomptarget{{.*}}
 
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=sm_52 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
 // RUN:     --libomptarget-nvptx-bc-path=%S/Inputs/libomptarget/libomptarget-nvptx-test.bc %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-NO-LTO-LIBRARY %s
 
-// CHECK-NO-LTO-LIBRARY: {{.*}}-lomptarget{{.*}}-lomptarget.devicertl
+// CHECK-NO-LTO-LIBRARY: --device-linker={{.*}}=-lomp{{.*}}-lomptarget{{.*}}
 
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=sm_52 -nogpulib \
 // RUN:     -foffload-lto %s 2>&1 | FileCheck --check-prefix=CHECK-NO-LIBRARY %s
 
-// CHECK-NO-LIBRARY-NOT: {{.*}}-lomptarget{{.*}}-lomptarget.devicertl
+// CHECK-NO-LIBRARY-NOT: --device-linker={{.*}}=-lomp{{.*}}-lomptarget{{.*}}
 
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp --offload-arch=sm_52 -nogpulib \
 // RUN:     -Xoffload-linker a -Xoffload-linker-nvptx64-nvidia-cuda b -Xoffload-linker-nvptx64 c \
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index 8f2a1fd01fabc..2da2ade94f4e4 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -16,23 +16,10 @@ if (LLVM_DIR)
   # Builds that use pre-installed LLVM have LLVM_DIR set.
   # A standalone or LLVM_ENABLE_RUNTIMES=openmp build takes this route
   find_program(CLANG_TOOL clang PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
-  find_program(PACKAGER_TOOL clang-offload-packager PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
-  find_program(LINK_TOOL llvm-link PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
-  find_program(OPT_TOOL opt PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
-  if ((NOT CLANG_TOOL) OR (NOT LINK_TOOL) OR (NOT OPT_TOOL) OR (NOT PACKAGER_TOOL))
-    message(STATUS "Not building DeviceRTL. Missing clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL}, opt: ${OPT_TOOL}, or clang-offload-packager: ${PACKAGER_TOOL}")
-    return()
-  else()
-    message(STATUS "Building DeviceRTL. Using clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL} and opt: ${OPT_TOOL}")
-  endif()
 elseif (LLVM_TOOL_CLANG_BUILD AND NOT CMAKE_CROSSCOMPILING AND NOT OPENMP_STANDALONE_BUILD)
   # LLVM in-tree builds may use CMake target names to discover the tools.
   # A LLVM_ENABLE_PROJECTS=openmp build takes this route
   set(CLANG_TOOL $<TARGET_FILE:clang>)
-  set(PACKAGER_TOOL $<TARGET_FILE:clang-offload-packager>)
-  set(LINK_TOOL $<TARGET_FILE:llvm-link>)
-  set(OPT_TOOL $<TARGET_FILE:opt>)
-  message(STATUS "Building DeviceRTL. Using clang from in-tree build")
 else()
   message(STATUS "Not building DeviceRTL. No appropriate clang found")
   return()
@@ -82,8 +69,6 @@ set(src_files
 # propagation. That said, we will run the vectorizer again after the runtime
 # has been linked into the user program.
 set(clang_opt_flags -O3 -mllvm -openmp-opt-disable -DSHARED_SCRATCHPAD_SIZE=512 -mllvm -vectorize-slp=false )
-set(link_opt_flags  -O3        -openmp-opt-disable -attributor-enable=module -vectorize-slp=false )
-set(link_export_flag -passes=internalize -internalize-public-api-file=${source_directory}/exports)
 
 # If the user built with the GPU C library enabled we will use that instead.
 if(${LIBOMPTARGET_GPU_LIBC_SUPPORT})
@@ -107,15 +92,13 @@ set(bc_flags -c -flto -std=c++17 -fvisibility=hidden
 )
 
 # first create an object target
-add_library(omptarget.devicertl.all_objs OBJECT IMPORTED)
 function(compileDeviceRTLLibrary target_name target_triple)
   set(target_bc_flags ${ARGN})
 
-  set(bc_files "")
   foreach(src ${src_files})
     get_filename_component(infile ${src} ABSOLUTE)
     get_filename_component(outfile ${src} NAME)
-    set(outfile "${outfile}-${target_name}.bc")
+    set(outfile "${outfile}-${target_name}.o")
     set(depfile "${outfile}.d")
 
     # Passing an empty CPU to -march= suppressed target specific metadata.
@@ -142,99 +125,40 @@ function(compileDeviceRTLLibrary target_name target_triple)
     endif()
     set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${outfile})
 
-    list(APPEND bc_files ${outfile})
+    list(APPEND obj_files ${CMAKE_CURRENT_BINARY_DIR}/${outfile})
   endforeach()
-
-  set(bclib_name "libomptarget-${target_name}.bc")
-
-  # Link to a bitcode library.
-  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name}
-      COMMAND ${LINK_TOOL}
-        -o ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} ${bc_files}
-      DEPENDS ${bc_files}
-      COMMENT "Linking LLVM bitcode ${bclib_name}"
-  )
-
-  if(TARGET llvm-link)
-    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name}
-      DEPENDS llvm-link
-      APPEND)
-  endif()
-
-  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
-      COMMAND ${OPT_TOOL} ${link_export_flag} ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name}
-                      -o ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
-      DEPENDS ${source_directory}/exports ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name}
-      COMMENT "Internalizing LLVM bitcode ${bclib_name}"
-  )
-  if(TARGET opt)
-    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
-      DEPENDS opt
-      APPEND)
-  endif()
-
-  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
-      COMMAND ${OPT_TOOL} ${link_opt_flags} ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
-                      -o ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
-      DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name}
-      COMMENT "Optimizing LLVM bitcode ${bclib_name}"
-  )
-  if(TARGET opt)
-    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
-      DEPENDS opt
-      APPEND)
-  endif()
-
-  set(bclib_target_name "omptarget-${target_name}-bc")
-  add_custom_target(${bclib_target_name} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name})
-
-  # Copy library to destination.
-  add_custom_command(TARGET ${bclib_target_name} POST_BUILD
-                    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
-                    ${LIBOMPTARGET_LIBRARY_DIR})
-  add_dependencies(omptarget.devicertl.${target_name} ${bclib_target_name})
-
-  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${bclib_name} ${LIBOMPTARGET_LIBRARY_DIR}/${bclib_name})
-
-  # Install bitcode library under the lib destination folder.
-  install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
-
-  set(target_feature "")
-  if("${target_triple}" STREQUAL "nvptx64-nvidia-cuda")
-    set(target_feature "feature=+ptx63")
-  endif()
-
-  # Package the bitcode in the bitcode and embed it in an ELF for the static library
-  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
-      COMMAND ${PACKAGER_TOOL} -o ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
-        "--image=file=${CMAKE_CURRENT_BINARY_DIR}/${bclib_name},${target_feature},triple=${target_triple},arch=generic,kind=openmp"
-      DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}
-      COMMENT "Packaging LLVM offloading binary ${bclib_name}.out"
-  )
-  if(TARGET clang-offload-packager)
-    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
-      DEPENDS clang-offload-packager
-      APPEND)
-  endif()
-
-  set(output_name "${CMAKE_CURRENT_BINARY_DIR}/devicertl-${target_name}.o")
-  add_custom_command(OUTPUT ${output_name}
-    COMMAND ${CLANG_TOOL} --std=c++17 -c -nostdlib
-            -Xclang -fembed-offload-object=${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name}
-            -o ${output_name}
-            ${source_directory}/Stub.cpp
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name} ${source_directory}/Stub.cpp
-    COMMENT "Embedding LLVM offloading binary in devicertl-${target_name}.o"
-    VERBATIM
+  # Trick to combine these into a bitcode file via the linker's LTO pass. This
+  # is used to provide the legacy `libomptarget-<name>.bc` files. Hack this
+  # through as an executable to get it to use the relocatable link.
+  add_executable(libomptarget-${target_name} ${obj_files})
+  set_target_properties(libomptarget-${target_name} PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY ${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}
+    LINKER_LANGUAGE CXX
+    BUILD_RPATH ""
+    INSTALL_RPATH ""
+    RUNTIME_OUTPUT_NAME libomptarget-${target_name}.bc)
+  target_compile_options(libomptarget-${target_name} PRIVATE "--target=${target_triple}")
+  target_link_options(libomptarget-${target_name} PRIVATE "--target=${target_triple}"
+                      "-r" "-nostdlib" "-flto" "-Wl,--lto-emit-llvm")
+  install(TARGETS libomptarget-${target_name}
+          PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ
+          DESTINATION ${OFFLOAD_INSTALL_LIBDIR})
+
+  add_library(omptarget.${target_name}.all_objs OBJECT IMPORTED)
+  set_property(TARGET omptarget.${target_name}.all_objs APPEND PROPERTY IMPORTED_OBJECTS
+               ${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}/libomptarget-${target_name}.bc)
+
+  # Archive all the object files generated above into a static library
+  add_library(omptarget.${target_name} STATIC)
+  set_target_properties(omptarget.${target_name} PROPERTIES
+    ARCHIVE_OUTPUT_DIRECTORY "${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}/${target_triple}"
+    ARCHIVE_OUTPUT_NAME ompdevice
+    LINKER_LANGUAGE CXX
   )
-  if(TARGET clang)
-    add_custom_command(OUTPUT ${output_name}
-      DEPENDS clang
-      APPEND)
-  endif()
+  target_link_libraries(omptarget.${target_name} PRIVATE omptarget.${target_name}.all_objs)
 
-  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${output_name})
-  set_property(TARGET omptarget.devicertl.all_objs APPEND PROPERTY IMPORTED_OBJECTS ${output_name})
+  install(TARGETS omptarget.${target_name}
+          ARCHIVE DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/${target_triple}")
 
   if (CMAKE_EXPORT_COMPILE_COMMANDS)
     set(ide_target_name omptarget-ide-${target_name})
@@ -254,18 +178,10 @@ function(compileDeviceRTLLibrary target_name target_triple)
   endif()
 endfunction()
 
-add_custom_target(omptarget.devicertl.amdgpu)
-compileDeviceRTLLibrary(amdgpu amdgcn-amd-amdhsa -Xclang -mcode-object-version=none)
-
-add_custom_target(omptarget.devicertl.nvptx)
-compileDeviceRTLLibrary(nvptx nvptx64-nvidia-cuda --cuda-feature=+ptx63)
-
-# Archive all the object files generated above into a static library
-add_library(omptarget.devicertl STATIC)
-set_target_properties(omptarget.devicertl PROPERTIES
-  ARCHIVE_OUTPUT_DIRECTORY "${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}"
-  LINKER_LANGUAGE CXX
-)
-target_link_libraries(omptarget.devicertl PRIVATE omptarget.devicertl.all_objs)
+if(NOT LLVM_TARGETS_TO_BUILD OR "AMDGPU" IN_LIST LLVM_TARGETS_TO_BUILD)
+  compileDeviceRTLLibrary(amdgpu amdgcn-amd-amdhsa -Xclang -mcode-object-version=none)
+endif()
 
-install(TARGETS omptarget.devicertl ARCHIVE DESTINATION ${OFFLOAD_INSTALL_LIBDIR})
+if(NOT LLVM_TARGETS_TO_BUILD OR "NVPTX" IN_LIST LLVM_TARGETS_TO_BUILD)
+  compileDeviceRTLLibrary(nvptx nvptx64-nvidia-cuda --cuda-feature=+ptx63)
+endif()
diff --git a/offload/DeviceRTL/src/exports b/offload/DeviceRTL/src/exports
deleted file mode 100644
index 01667e7aba827..0000000000000
--- a/offload/DeviceRTL/src/exports
+++ /dev/null
@@ -1,19 +0,0 @@
-omp_*
-ompx_*
-*llvm_*
-__kmpc_*
-
-__omp_rtl_debug_kind
-__omp_rtl_assume_no_thread_state
-__omp_rtl_assume_no_nested_parallelism
-
-_ZN4ompx*
-
-IsSPMDMode
-
-malloc
-free
-memcmp
-printf
-__llvm_rpc_client
-__assert_fail
diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index f017bca85dd4f..8cdf2f3f13681 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -164,7 +164,7 @@ else: # Unices
     if config.cuda_libdir:
         config.test_flags += " -Wl,-rpath," + config.cuda_libdir
     if config.libomptarget_current_target.startswith('nvptx'):
-        config.test_flags_clang += " --libomptarget-nvptx-bc-path=" + config.library_dir + '/DeviceRTL'
+        config.test_flags_clang += " --libomptarget-nvptx-bc-path=" + config.llvm_library_intdir
     if config.libomptarget_current_target.endswith('-LTO'):
         config.test_flags += " -foffload-lto"
     if config.libomptarget_current_target.endswith('-JIT-LTO') and evaluate_bool_env(
@@ -173,22 +173,6 @@ else: # Unices
         config.test_flags += " -foffload-lto"
         config.test_flags += " -Wl,--embed-bitcode"
 
-def remove_suffix_if_present(name):
-    if name.endswith('-LTO'):
-        return name[:-4]
-    elif name.endswith('-JIT-LTO'):
-        return name[:-8]
-    else:
-        return name
-
-def add_libraries(source):
-    if config.libomptarget_has_libc:
-        return source + " -Xoffload-linker " + "-lc " + \
-               "-Xoffload-linker " + "-lm " + \
-               config.llvm_library_intdir + "/libomptarget.devicertl.a"
-    else:
-        return source + " " + config.llvm_library_intdir + "/libomptarget.devicertl.a"
-
 # Add platform targets
 host_targets = [
     "aarch64-unknown-linux-gnu",
@@ -207,6 +191,24 @@ if config.libomptarget_current_target.startswith('amdgcn'):
 if config.libomptarget_current_target in host_targets:
     config.available_features.add('host')
 
+def remove_suffix_if_present(name):
+    if name.endswith('-LTO'):
+        return name[:-4]
+    elif name.endswith('-JIT-LTO'):
+        return name[:-8]
+    else:
+        return name
+
+def add_libraries(source):
+    if "gpu" not in config.available_features:
+        return source
+    if config.libomptarget_has_libc:
+        return source + " -Xoffload-linker -lc " + \
+               "-Xoffload-linker -lm " + \
+               "-Xoffload-linker -lompdevice"
+    else:
+        return source + " " + "-Xoffload-lnker -lompdevice"
+
 # substitutions
 # - for targets that exist in the system create the actual command.
 # - for valid targets that do not exist in the system, return false, so that the

From fc1e311966c59368e63b834a42523ce104711a94 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Fri, 18 Apr 2025 14:52:51 +0200
Subject: [PATCH 404/710] [mlir][memref] Fix rollback in test case during
 `convert-to-llvm` (#135958)

This commit is in preparation of the One-Shot Dialect Conversion
refactoring, which removes the rollback from the dialect conversion
framework.

`GenericAtomicRMWOpLowering` (`generic_atomic_rmw`) triggered a rollback
in two test cases. The lowering pattern adds additional basic blocks to
the enclosing operation, which used to be a `func.func` (now
`llvm.func`). Adding a basic block triggers legalization of the op that
owns the basic block. This fails when running
`--convert-to-llvm="filter-dialects=memref"` because no lowering
patterns for the `func` dialect were populated and only `llvm` ops are
considered "legal" by the `convert-to-llvm` pass, causing a rollback of
the entire `GenericAtomicRMWOpLowering` pattern.

Also add extra `CHECK-INTERFACE` to make sure that all test cases are
correctly lowered with `--convert-to-llvm="filter-dialects=memref"`.
---
 .../MemRefToLLVM/memref-to-llvm.mlir          | 92 +++++++++++++++++--
 1 file changed, 85 insertions(+), 7 deletions(-)

diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
index 12e93c96f743d..5538ddf8e4c3c 100644
--- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
@@ -3,12 +3,14 @@
 
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass. This produces slightly different IR
-// because the conversion target is set up differently. Only one test case is
-// checked.
+// because the conversion target is set up differently.
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=memref" --split-input-file %s | FileCheck --check-prefix=CHECK-INTERFACE %s
 
 // CHECK-LABEL: func @view(
 // CHECK: %[[ARG0F:.*]]: index, %[[ARG1F:.*]]: index, %[[ARG2F:.*]]: index
+// CHECK-INTERFACE-LABEL: func @view(
+// CHECK-INTERFACE-NOT: memref.alloc
+// CHECK-INTERFACE-NOT: memref.view
 func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) {
   // CHECK-DAG: %[[ARG2:.*]] = builtin.unrealized_conversion_cast %[[ARG2F]]
   // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0F]]
@@ -141,9 +143,11 @@ func.func @view_empty_memref(%offset: index, %mem: memref<0xi8>) {
 // CHECK32:         %[[ARG0:[a-zA-Z0-9]*]]: index,
 // CHECK32:         %[[ARG1:[a-zA-Z0-9]*]]: index,
 // CHECK32:         %[[ARG2:.*]]: index)
+// CHECK-INTERFACE-LABEL: func @subview(
 func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : index, %arg1 : index, %arg2 : index) {
   // CHECK: memref.subview %[[MEMREF]][%[[ARG0]], %[[ARG1]]] [%[[ARG0]], %[[ARG1]]]
   // CHECK32: memref.subview %[[MEMREF]][%[[ARG0]], %[[ARG1]]] [%[[ARG0]], %[[ARG1]]] [%[[ARG0]], %[[ARG1]]]
+  // CHECK-INTERFACE: memref.subview
   %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] :
     memref<64x4xf32, strided<[4, 1], offset: 0>>
   to memref<?x?xf32, strided<[?, ?], offset: ?>>
@@ -153,11 +157,13 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in
 // -----
 
 // CHECK-LABEL: func @assume_alignment(
+// CHECK-INTERFACE-LABEL: func @assume_alignment(
 func.func @assume_alignment(%0 : memref<4x4xf16>) {
   // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-NEXT: %[[TRUE:.*]] = llvm.mlir.constant(true) : i1
   // CHECK-NEXT: %[[ALIGN:.*]] = llvm.mlir.constant(16 : index) : i64
-    // CHECK-NEXT: llvm.intr.assume %[[TRUE]] ["align"(%[[PTR]], %[[ALIGN]] : !llvm.ptr, i64)] : i1
+  // CHECK-NEXT: llvm.intr.assume %[[TRUE]] ["align"(%[[PTR]], %[[ALIGN]] : !llvm.ptr, i64)] : i1
+  // CHECK-INTERFACE: llvm.intr.assume
   memref.assume_alignment %0, 16 : memref<4x4xf16>
   return
 }
@@ -165,13 +171,15 @@ func.func @assume_alignment(%0 : memref<4x4xf16>) {
 // -----
 
 // CHECK-LABEL: func @assume_alignment_w_offset
+// CHECK-INTERFACE-LABEL: func @assume_alignment_w_offset
 func.func @assume_alignment_w_offset(%0 : memref<4x4xf16, strided<[?, ?], offset: ?>>) {
   // CHECK-DAG: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-DAG: %[[OFFSET:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-DAG: %[[BUFF_ADDR:.*]] =  llvm.getelementptr %[[PTR]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, f16
   // CHECK-DAG: %[[TRUE:.*]] = llvm.mlir.constant(true) : i1
   // CHECK-DAG: %[[ALIGN:.*]] = llvm.mlir.constant(16 : index) : i64
-    // CHECK-NEXT: llvm.intr.assume %[[TRUE]] ["align"(%[[BUFF_ADDR]], %[[ALIGN]] : !llvm.ptr, i64)] : i1
+  // CHECK-NEXT: llvm.intr.assume %[[TRUE]] ["align"(%[[BUFF_ADDR]], %[[ALIGN]] : !llvm.ptr, i64)] : i1
+  // CHECK-INTERFACE: llvm.intr.assume
   memref.assume_alignment %0, 16 : memref<4x4xf16, strided<[?, ?], offset: ?>>
   return
 }
@@ -179,6 +187,7 @@ func.func @assume_alignment_w_offset(%0 : memref<4x4xf16, strided<[?, ?], offset
 
 // CHECK-LABEL: func @dim_of_unranked
 // CHECK32-LABEL: func @dim_of_unranked
+// CHECK-INTERFACE-LABEL: func @dim_of_unranked
 func.func @dim_of_unranked(%unranked: memref<*xi32>) -> index {
   %c0 = arith.constant 0 : index
   %dim = memref.dim %unranked, %c0 : memref<*xi32>
@@ -202,17 +211,23 @@ func.func @dim_of_unranked(%unranked: memref<*xi32>) -> index {
 
 // CHECK32: %[[SIZE:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32
 
+// CHECK-INTERFACE-NOT: memref.dim
+
 // -----
 
 // CHECK-LABEL: func @address_space(
+// CHECK-INTERFACE-LABEL: func @address_space(
 func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) {
   // CHECK: %[[MEMORY:.*]] = llvm.call @malloc(%{{.*}})
   // CHECK: %[[CAST:.*]] = llvm.addrspacecast %[[MEMORY]] : !llvm.ptr to !llvm.ptr<5>
   // CHECK: llvm.insertvalue %[[CAST]], %{{[[:alnum:]]+}}[0]
   // CHECK: llvm.insertvalue %[[CAST]], %{{[[:alnum:]]+}}[1]
+  // CHECK-INTERFACE: llvm.call @malloc
+  // CHECK-INTERFACE: llvm.addrspacecast
   %0 = memref.alloc() : memref<32xf32, affine_map<(d0) -> (d0)>, 5>
   %1 = arith.constant 7 : index
   // CHECK: llvm.load %{{.*}} : !llvm.ptr<5> -> f32
+  // CHECK-INTERFACE: llvm.load
   %2 = memref.load %0[%1] : memref<32xf32, affine_map<(d0) -> (d0)>, 5>
   func.return
 }
@@ -236,6 +251,8 @@ func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) {
 //       CHECK:    llvm.insertvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:   llvm.extractvalue {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.insertvalue {{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>
+// CHECK-INTERFACE-LABEL: func @transpose
+// CHECK-INTERFACE-NOT: memref.transpose
 func.func @transpose(%arg0: memref<?x?x?xf32, strided<[?, ?, 1], offset: ?>>) {
   %0 = memref.transpose %arg0 (i, j, k) -> (k, i, j) : memref<?x?x?xf32, strided<[?, ?, 1], offset: ?>> to memref<?x?x?xf32, strided<[1, ?, ?], offset: ?>>
   return
@@ -247,16 +264,20 @@ func.func @transpose(%arg0: memref<?x?x?xf32, strided<[?, ?, 1], offset: ?>>) {
 // CHECK-NEXT:     %0 = llvm.mlir.undef : !llvm.array<2 x f32>
 // CHECK-NEXT:     llvm.return %0 : !llvm.array<2 x f32>
 // CHECK-NEXT:   }
+// CHECK-INTERFACE: llvm.mlir.global external
 memref.global @gv0 : memref<2xf32> = uninitialized
 
 // CHECK: llvm.mlir.global private @gv1() {addr_space = 0 : i32} : !llvm.array<2 x f32>
+// CHECK-INTERFACE: llvm.mlir.global private
 memref.global "private" @gv1 : memref<2xf32>
 
 // CHECK: llvm.mlir.global external @gv2(dense<{{\[\[}}0.000000e+00, 1.000000e+00, 2.000000e+00], [3.000000e+00, 4.000000e+00, 5.000000e+00]]> : tensor<2x3xf32>) {addr_space = 0 : i32} : !llvm.array<2 x array<3 x f32>>
+// CHECK-INTERFACE: llvm.mlir.global external
 memref.global @gv2 : memref<2x3xf32> = dense<[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]>
 
 // Test 1D memref.
 // CHECK-LABEL: func @get_gv0_memref
+// CHECK-INTERFACE-LABEL: func @get_gv0_memref
 func.func @get_gv0_memref() {
   %0 = memref.get_global @gv0 : memref<2xf32>
   // CHECK: %[[DIM:.*]] = llvm.mlir.constant(2 : index) : i64
@@ -272,11 +293,14 @@ func.func @get_gv0_memref() {
   // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: llvm.insertvalue %[[DIM]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: llvm.insertvalue %[[STRIDE]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK-INTERFACE: llvm.mlir.addressof
+  // CHECK-INTERFACE: llvm.getelementptr
   return
 }
 
 // Test 2D memref.
 // CHECK-LABEL: func @get_gv2_memref
+// CHECK-INTERFACE-LABEL: func @get_gv2_memref
 func.func @get_gv2_memref() {
   // CHECK: %[[DIM0:.*]] = llvm.mlir.constant(2 : index) : i64
   // CHECK: %[[DIM1:.*]] = llvm.mlir.constant(3 : index) : i64
@@ -294,6 +318,8 @@ func.func @get_gv2_memref() {
   // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.insertvalue %[[STRIDE1]], {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK-INTERFACE: llvm.mlir.addressof
+  // CHECK-INTERFACE: llvm.getelementptr
 
   %0 = memref.get_global @gv2 : memref<2x3xf32>
   return
@@ -304,6 +330,7 @@ func.func @get_gv2_memref() {
 memref.global @gv3 : memref<f32> = dense<1.0>
 
 // CHECK-LABEL: func @get_gv3_memref
+// CHECK-INTERFACE-LABEL: func @get_gv3_memref
 func.func @get_gv3_memref() {
   // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv3 : !llvm.ptr
   // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0] : (!llvm.ptr) -> !llvm.ptr, f32
@@ -314,12 +341,15 @@ func.func @get_gv3_memref() {
   // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)>
   // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64
   // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64)>
+  // CHECK-INTERFACE: llvm.mlir.addressof
+  // CHECK-INTERFACE: llvm.getelementptr
   %0 = memref.get_global @gv3 : memref<f32>
   return
 }
 
 // Test scalar memref with an alignment.
 // CHECK: llvm.mlir.global private @gv4(1.000000e+00 : f32) {addr_space = 0 : i32, alignment = 64 : i64} : f32
+// CHECK-INTERFACE: llvm.mlir.global private
 memref.global "private" @gv4 : memref<f32> = dense<1.0> {alignment = 64}
 
 // -----
@@ -327,8 +357,10 @@ memref.global "private" @gv4 : memref<f32> = dense<1.0> {alignment = 64}
 // Expand shapes need to be expanded outside of the memref-to-llvm pass.
 // CHECK-LABEL: func @expand_shape_static(
 // CHECK-SAME:         %[[ARG:.*]]: memref<{{.*}}>)
+// CHECK-INTERFACE-LABEL: func @expand_shape_static(
 func.func @expand_shape_static(%arg0: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32> {
   // CHECK: memref.expand_shape %[[ARG]] {{\[}}[0, 1], [2], [3, 4]] output_shape [1, 3, 4, 1, 5]
+  // CHECK-INTERFACE: memref.expand_shape
   // Reshapes that expand a contiguous tensor with some 1's.
   %0 = memref.expand_shape %arg0 [[0, 1], [2], [3, 4]] output_shape [1, 3, 4, 1, 5]
       : memref<3x4x5xf32> into memref<1x3x4x1x5xf32>
@@ -340,7 +372,10 @@ func.func @expand_shape_static(%arg0: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32
 // Collapse shapes need to be expanded outside of the memref-to-llvm pass.
 // CHECK-LABEL: func @collapse_shape_static
 // CHECK-SAME: %[[ARG:.*]]: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf32> {
+// CHECK-INTERFACE-LABEL: func @collapse_shape_static
 func.func @collapse_shape_static(%arg0: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf32> {
+  // CHECK: memref.collapse_shape %[[ARG]]
+  // CHECK-INTERFACE: memref.collapse_shape
   %0 = memref.collapse_shape %arg0 [[0, 1], [2], [3, 4]] :
     memref<1x3x4x1x5xf32> into memref<3x4x5xf32>
   return %0 : memref<3x4x5xf32>
@@ -350,6 +385,7 @@ func.func @collapse_shape_static(%arg0: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf
 
 // CHECK-LABEL: func @rank_of_unranked
 // CHECK32-LABEL: func @rank_of_unranked
+// CHECK-INTERFACE-LABEL: func @rank_of_unranked
 func.func @rank_of_unranked(%unranked: memref<*xi32>) {
   %rank = memref.rank %unranked : memref<*xi32>
   return
@@ -357,19 +393,25 @@ func.func @rank_of_unranked(%unranked: memref<*xi32>) {
 // CHECK: %[[UNRANKED_DESC:.*]] = builtin.unrealized_conversion_cast
 // CHECK-NEXT: llvm.extractvalue %[[UNRANKED_DESC]][0] : !llvm.struct<(i64, ptr)>
 // CHECK32: llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i32, ptr)>
+// CHECK-INTERFACE-NOT: memref.rank
+
+// -----
 
 // CHECK-LABEL: func @rank_of_ranked
 // CHECK32-LABEL: func @rank_of_ranked
+// CHECK-INTERFACE-LABEL: func @rank_of_ranked
 func.func @rank_of_ranked(%ranked: memref<?xi32>) {
   %rank = memref.rank %ranked : memref<?xi32>
   return
 }
 // CHECK: llvm.mlir.constant(1 : index) : i64
 // CHECK32: llvm.mlir.constant(1 : index) : i32
+// CHECK-INTERFACE: llvm.mlir.constant(1 : index) : i64
 
 // -----
 
 // CHECK-LABEL: func @atomic_rmw
+// CHECK-INTERFACE-LABEL: func @atomic_rmw
 func.func @atomic_rmw(%I : memref<10xi32>, %ival : i32, %F : memref<10xf32>, %fval : f32, %i : index) {
   memref.atomic_rmw assign %fval, %F[%i] : (f32, memref<10xf32>) -> f32
   // CHECK: llvm.atomicrmw xchg %{{.*}}, %{{.*}} acq_rel
@@ -389,6 +431,7 @@ func.func @atomic_rmw(%I : memref<10xi32>, %ival : i32, %F : memref<10xf32>, %fv
   // CHECK: llvm.atomicrmw _or %{{.*}}, %{{.*}} acq_rel
   memref.atomic_rmw andi %ival, %I[%i] : (i32, memref<10xi32>) -> i32
   // CHECK: llvm.atomicrmw _and %{{.*}}, %{{.*}} acq_rel
+  // CHECK-INTERFACE-COUNT-9: llvm.atomicrmw
   return
 }
 
@@ -410,10 +453,16 @@ func.func @atomic_rmw_with_offset(%I : memref<10xi32, strided<[1], offset: 5>>,
 // CHECK:        %[[PTR:.+]] = llvm.getelementptr %[[OFFSET_PTR]][%[[INDEX]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32
 // CHECK:        llvm.atomicrmw _and %[[PTR]], %[[ARG1]] acq_rel
 
+// CHECK-INTERFACE-LABEL:  func @atomic_rmw_with_offset
+// CHECK-INTERFACE: llvm.atomicrmw
+
 // -----
 
 // CHECK-LABEL: func @generic_atomic_rmw
-func.func @generic_atomic_rmw(%I : memref<10xi32>, %i : index) {
+// CHECK-INTERFACE-LABEL: func @generic_atomic_rmw
+llvm.func @generic_atomic_rmw() {
+  %I = "test.foo"() : () -> (memref<10xi32>)
+  %i = "test.foo"() : () -> (index)
   %x = memref.generic_atomic_rmw %I[%i] : memref<10xi32> {
     ^bb0(%old_value : i32):
       memref.atomic_yield %old_value : i32
@@ -429,10 +478,13 @@ func.func @generic_atomic_rmw(%I : memref<10xi32>, %i : index) {
 // CHECK-NEXT:   %[[OK:.*]] = llvm.extractvalue %[[PAIR]][1]
 // CHECK-NEXT:   llvm.cond_br %[[OK]], ^bb2, ^bb1(%[[NEW]] : i32)
 
+// CHECK-INTERFACE: llvm.cmpxchg
+
 // -----
 
 // CHECK-LABEL: func @generic_atomic_rmw_in_alloca_scope
-func.func @generic_atomic_rmw_in_alloca_scope(){
+// CHECK-INTERFACE-LABEL: func @generic_atomic_rmw_in_alloca_scope
+llvm.func @generic_atomic_rmw_in_alloca_scope() {
   %c1 = arith.constant 1 : index
   %alloc = memref.alloc() : memref<2x3xi32>
   memref.alloca_scope  {
@@ -441,7 +493,7 @@ func.func @generic_atomic_rmw_in_alloca_scope(){
       memref.atomic_yield %arg0 : i32
     }
   }
-  return
+  llvm.return
 }
 // CHECK:        %[[STACK_SAVE:.*]] = llvm.intr.stacksave : !llvm.ptr
 // CHECK-NEXT:   llvm.br ^bb1
@@ -460,9 +512,12 @@ func.func @generic_atomic_rmw_in_alloca_scope(){
 // CHECK-NEXT: ^bb4:
 // CHECK-NEXT:   return
 
+// CHECK-INTERFACE: llvm.cmpxchg
+
 // -----
 
 // CHECK-LABEL: func @memref_copy_ranked
+// CHECK-INTERFACE-LABEL: func @memref_copy_ranked
 func.func @memref_copy_ranked() {
   %0 = memref.alloc() : memref<2xf32>
   // CHECK: llvm.mlir.constant(2 : index) : i64
@@ -487,6 +542,7 @@ func.func @memref_copy_ranked() {
   // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
   // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]]) <{isVolatile = false}>
+  // CHECK-INTERFACE: llvm.intr.memcpy
   return
 }
 
@@ -494,6 +550,7 @@ func.func @memref_copy_ranked() {
 // -----
 
 // CHECK-LABEL: func @memref_copy_contiguous
+// CHECK-INTERFACE-LABEL: func @memref_copy_contiguous
 func.func @memref_copy_contiguous(%in: memref<16x4xi32>, %offset: index) {
   %buf = memref.alloc() : memref<1x2xi32>
   %sub = memref.subview %in[%offset, 0] [1, 2] [1, 1] : memref<16x4xi32> to memref<1x2xi32, strided<[4, 1], offset: ?>>
@@ -518,35 +575,41 @@ func.func @memref_copy_contiguous(%in: memref<16x4xi32>, %offset: index) {
   // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32
   // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]]) <{isVolatile = false}>
+  // CHECK-INTERFACE: llvm.intr.memcpy
   return
 }
 
 // -----
 
 // CHECK-LABEL: func @memref_copy_0d_offset
+// CHECK-INTERFACE-LABEL: func @memref_copy_0d_offset
 func.func @memref_copy_0d_offset(%in: memref<2xi32>) {
   %buf = memref.alloc() : memref<i32>
   %sub = memref.subview %in[1] [1] [1] : memref<2xi32> to memref<1xi32, strided<[1], offset: 1>>
   %scalar = memref.collapse_shape %sub [] : memref<1xi32, strided<[1], offset: 1>> into memref<i32, strided<[], offset: 1>>
   memref.copy %scalar, %buf : memref<i32, strided<[], offset: 1>> to memref<i32>
   // CHECK: llvm.intr.memcpy
+  // CHECK-INTERFACE: llvm.intr.memcpy
   return
 }
 
 // -----
 
 // CHECK-LABEL: func @memref_copy_noncontiguous
+// CHECK-INTERFACE-LABEL: func @memref_copy_noncontiguous
 func.func @memref_copy_noncontiguous(%in: memref<16x2xi32>, %offset: index) {
   %buf = memref.alloc() : memref<2x1xi32>
   %sub = memref.subview %in[%offset, 0] [2, 1] [1, 1] : memref<16x2xi32> to memref<2x1xi32, strided<[2, 1], offset: ?>>
   memref.copy %sub, %buf : memref<2x1xi32, strided<[2, 1], offset: ?>> to memref<2x1xi32>
   // CHECK: llvm.call @memrefCopy
+  // CHECK-INTERFACE: llvm.call @memrefCopy
   return
 }
 
 // -----
 
 // CHECK-LABEL: func @memref_copy_unranked
+// CHECK-INTERFACE-LABEL: func @memref_copy_unranked
 func.func @memref_copy_unranked() {
   %0 = memref.alloc() : memref<2xi1>
   // CHECK: llvm.mlir.constant(2 : index) : i64
@@ -574,17 +637,20 @@ func.func @memref_copy_unranked() {
   // CHECK: [[SIZE:%.*]] = llvm.ptrtoint [[SIZEPTR]] : !llvm.ptr to i64
   // CHECK: llvm.call @memrefCopy([[SIZE]], [[ALLOCA2]], [[ALLOCA3]]) : (i64, !llvm.ptr, !llvm.ptr) -> ()
   // CHECK: llvm.intr.stackrestore [[STACKSAVE]]
+  // CHECK-INTERFACE: llvm.call @memrefCopy
   return
 }
 
 // -----
 
 // CHECK-LABEL: func @extract_aligned_pointer_as_index
+// CHECK-INTERFACE-LABEL: func @extract_aligned_pointer_as_index
 func.func @extract_aligned_pointer_as_index(%m: memref<?xf32>) -> index {
   %0 = memref.extract_aligned_pointer_as_index %m: memref<?xf32> -> index
   // CHECK: %[[E:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>
   // CHECK: %[[I64:.*]] = llvm.ptrtoint %[[E]] : !llvm.ptr to i64
   // CHECK: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I64]] : i64 to index
+  // CHECK-INTERFACE-NOT: memref.extract_aligned_pointer_as_index
 
   // CHECK: return %[[R:.*]] : index
   return %0: index
@@ -593,6 +659,7 @@ func.func @extract_aligned_pointer_as_index(%m: memref<?xf32>) -> index {
 // -----
 
 // CHECK-LABEL: func @extract_aligned_pointer_as_index_unranked
+// CHECK-INTERFACE-LABEL: func @extract_aligned_pointer_as_index_unranked
 func.func @extract_aligned_pointer_as_index_unranked(%m: memref<*xf32>) -> index {
   %0 = memref.extract_aligned_pointer_as_index %m: memref<*xf32> -> index
   // CHECK: %[[PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr)>
@@ -600,6 +667,7 @@ func.func @extract_aligned_pointer_as_index_unranked(%m: memref<*xf32>) -> index
   // CHECK: %[[ALIGNED_PTR:.*]] = llvm.load %[[ALIGNED_FIELD]] : !llvm.ptr -> !llvm.ptr
   // CHECK: %[[I64:.*]] = llvm.ptrtoint %[[ALIGNED_PTR]] : !llvm.ptr to i64
   // CHECK: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I64]] : i64 to index
+  // CHECK-INTERFACE-NOT: memref.extract_aligned_pointer_as_index
 
   // CHECK: return %[[R]] : index
   return %0: index
@@ -622,6 +690,10 @@ func.func @extract_aligned_pointer_as_index_unranked(%m: memref<*xf32>) -> index
 // CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+
+// CHECK-INTERFACE-LABEL: func @extract_strided_metadata
+// CHECK-INTERFACE-NOT: memref.extract_strided_metadata
+
 func.func @extract_strided_metadata(
     %ref: memref<?x?xf32, strided<[?,?], offset: ?>>) {
 
@@ -637,9 +709,11 @@ func.func @extract_strided_metadata(
 // -----
 
 // CHECK-LABEL: func @load_non_temporal(
+// CHECK-INTERFACE-LABEL: func @load_non_temporal(
 func.func @load_non_temporal(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>>) {
   %1 = arith.constant 7 : index
   // CHECK: llvm.load %{{.*}} {nontemporal} : !llvm.ptr -> f32
+  // CHECK-INTERFACE: llvm.load
   %2 = memref.load %arg0[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>>
   func.return
 }
@@ -647,10 +721,12 @@ func.func @load_non_temporal(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>>) {
 // -----
 
 // CHECK-LABEL: func @store_non_temporal(
+// CHECK-INTERFACE-LABEL: func @store_non_temporal(
 func.func @store_non_temporal(%input : memref<32xf32, affine_map<(d0) -> (d0)>>, %output : memref<32xf32, affine_map<(d0) -> (d0)>>) {
   %1 = arith.constant 7 : index
   %2 = memref.load %input[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>>
   // CHECK: llvm.store %{{.*}}, %{{.*}}  {nontemporal} : f32, !llvm.ptr
+  // CHECK-INTERFACE: llvm.store
   memref.store %2, %output[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>>
   func.return
 }
@@ -660,8 +736,10 @@ func.func @store_non_temporal(%input : memref<32xf32, affine_map<(d0) -> (d0)>>,
 // Ensure unconvertable memory space not cause a crash
 
 // CHECK-LABEL: @alloca_unconvertable_memory_space
+// CHECK-INTERFACE-LABEL: @alloca_unconvertable_memory_space
 func.func @alloca_unconvertable_memory_space() {
   // CHECK: memref.alloca
+  // CHECK-INTERFACE: memref.alloca
   %alloca = memref.alloca() : memref<1x32x33xi32, #spirv.storage_class<StorageBuffer>>
   func.return
 }

From 622765f976f0a76313aa38d0f549eea9affeebfd Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Fri, 18 Apr 2025 05:52:55 -0700
Subject: [PATCH 405/710] [Clang][GPU] Make NVPTX check more permissive in unit
 test (#136301)

- Seems based on whether NVPTX backend is enabled or not, this call can
have the range() attribute or not. So make this check more permissive.
---
 clang/test/Headers/gpuintrin_lang.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/Headers/gpuintrin_lang.c b/clang/test/Headers/gpuintrin_lang.c
index b804d46071507..653f87aea2ce3 100644
--- a/clang/test/Headers/gpuintrin_lang.c
+++ b/clang/test/Headers/gpuintrin_lang.c
@@ -36,7 +36,7 @@ __device__ int foo() { return __gpu_thread_id_x(); }
 // CUDA-LABEL: define dso_local i32 @foo(
 // CUDA-SAME: ) #[[ATTR0:[0-9]+]] {
 // CUDA-NEXT:  [[ENTRY:.*:]]
-// CUDA-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+// CUDA-NEXT:    [[TMP0:%.*]] = call {{.*}}i32 @llvm.nvvm.read.ptx.sreg.tid.x()
 // CUDA-NEXT:    ret i32 [[TMP0]]
 //
 // HIP-LABEL: define dso_local i32 @foo(

From 860e88411d94b9e83cf9524d05b30a3d2d221b2c Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 18 Apr 2025 22:06:25 +0900
Subject: [PATCH 406/710] [libc++] Make __config_site modular (#134699)

This patch makes the __config_site header modular, which solves various
problems with non-modular headers. This requires going back to
generating the modulemap file, since we only know how to make
__config_site modular when we're not using the per-target runtime dir.

The patch also adds a test that we support
-Wnon-modular-include-in-module, which warns about non-modular includes
from modules.

---------

Co-authored-by: Konstantin Varlamov <varconst@apple.com>
---
 libcxx/docs/Contributing.rst                  |  2 +-
 libcxx/include/CMakeLists.txt                 | 17 +++++++++++++--
 .../{module.modulemap => module.modulemap.in} |  1 +
 ...modular_include_in_module.compile.pass.cpp | 21 +++++++++++++++++++
 libcxx/test/libcxx/headers_in_modulemap.sh.py |  2 +-
 libcxx/test/libcxx/lint/lint_headers.sh.py    |  2 +-
 libcxx/utils/libcxx/header_information.py     |  2 +-
 7 files changed, 41 insertions(+), 6 deletions(-)
 rename libcxx/include/{module.modulemap => module.modulemap.in} (99%)
 create mode 100644 libcxx/test/libcxx/Wnon_modular_include_in_module.compile.pass.cpp

diff --git a/libcxx/docs/Contributing.rst b/libcxx/docs/Contributing.rst
index 5bcd6e3a7f03e..6aaa70764c2fa 100644
--- a/libcxx/docs/Contributing.rst
+++ b/libcxx/docs/Contributing.rst
@@ -116,7 +116,7 @@ sure you don't forget anything:
 - Did you add all new named declarations to the ``std`` module?
 - If you added a header:
 
-  - Did you add it to ``include/module.modulemap``?
+  - Did you add it to ``include/module.modulemap.in``?
   - Did you add it to ``include/CMakeLists.txt``?
   - If it's a public header, did you update ``utils/libcxx/header_information.py``?
 
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 4431582cc33ca..f1bdf684a8549 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -1027,7 +1027,6 @@ set(files
   mdspan
   memory
   memory_resource
-  module.modulemap
   mutex
   new
   numbers
@@ -1667,8 +1666,16 @@ set(files
 configure_file("__config_site.in" "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site" @ONLY)
 configure_file("${LIBCXX_ASSERTION_HANDLER_FILE}" "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler" COPYONLY)
 
+# We generate the modulemap file so that we can include __config_site in it. For now, we don't know how to
+# make __config_site modular when per-target runtime directories are used.
+if (NOT LLVM_ENABLE_PER_TARGET_RUNTIME_DIR)
+  set(LIBCXX_CONFIG_SITE_MODULE_ENTRY "textual header \"__config_site\"")
+endif()
+configure_file("module.modulemap.in" "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap" @ONLY)
+
 set(_all_includes "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site"
-                  "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler")
+                  "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler"
+                  "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap")
 foreach(f ${files})
   set(src "${CMAKE_CURRENT_SOURCE_DIR}/${f}")
   set(dst "${LIBCXX_GENERATED_INCLUDE_DIR}/${f}")
@@ -1726,6 +1733,12 @@ if (LIBCXX_INSTALL_HEADERS)
     PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
     COMPONENT cxx-headers)
 
+  # Install the generated modulemap file to the generic include dir.
+  install(FILES "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap"
+    DESTINATION "${LIBCXX_INSTALL_INCLUDE_DIR}"
+    PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
+    COMPONENT cxx-headers)
+
   if (NOT CMAKE_CONFIGURATION_TYPES)
     add_custom_target(install-cxx-headers
                       DEPENDS cxx-headers
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap.in
similarity index 99%
rename from libcxx/include/module.modulemap
rename to libcxx/include/module.modulemap.in
index 353d9651093ef..af928a63f2315 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap.in
@@ -1,6 +1,7 @@
 // This module contains headers related to the configuration of the library. These headers
 // are free of any dependency on the rest of libc++.
 module std_config [system] {
+  @LIBCXX_CONFIG_SITE_MODULE_ENTRY@ // generated via CMake
   textual header "__config"
   textual header "__configuration/abi.h"
   textual header "__configuration/availability.h"
diff --git a/libcxx/test/libcxx/Wnon_modular_include_in_module.compile.pass.cpp b/libcxx/test/libcxx/Wnon_modular_include_in_module.compile.pass.cpp
new file mode 100644
index 0000000000000..aa7a6d98d7d68
--- /dev/null
+++ b/libcxx/test/libcxx/Wnon_modular_include_in_module.compile.pass.cpp
@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: target={{.*}}-apple-{{.*}}
+// UNSUPPORTED: c++03
+
+// This test ensures that libc++ supports being compiled with modules enabled and with
+// -Wnon-modular-include-in-module. This effectively checks that we don't include any
+// non-modular header from the library.
+//
+// Since most underlying platforms are not modularized properly, this test currently only
+// works on Apple platforms.
+
+// ADDITIONAL_COMPILE_FLAGS: -Wnon-modular-include-in-module -Wsystem-headers-in-module=std -fmodules -fcxx-modules
+
+#include <vector>
diff --git a/libcxx/test/libcxx/headers_in_modulemap.sh.py b/libcxx/test/libcxx/headers_in_modulemap.sh.py
index 51b14377ba89b..d4a18a20c8926 100644
--- a/libcxx/test/libcxx/headers_in_modulemap.sh.py
+++ b/libcxx/test/libcxx/headers_in_modulemap.sh.py
@@ -4,7 +4,7 @@
 sys.path.append(sys.argv[1])
 from libcxx.header_information import all_headers, libcxx_include
 
-with open(libcxx_include / "module.modulemap") as f:
+with open(libcxx_include / "module.modulemap.in") as f:
     modulemap = f.read()
 
 isHeaderMissing = False
diff --git a/libcxx/test/libcxx/lint/lint_headers.sh.py b/libcxx/test/libcxx/lint/lint_headers.sh.py
index c5e582cb0f7cb..ab237c968da7e 100644
--- a/libcxx/test/libcxx/lint/lint_headers.sh.py
+++ b/libcxx/test/libcxx/lint/lint_headers.sh.py
@@ -11,7 +11,7 @@
 def exclude_from_consideration(path):
     return (
         path.endswith(".txt")
-        or path.endswith(".modulemap")
+        or path.endswith(".modulemap.in")
         or os.path.basename(path) == "__config"
         or os.path.basename(path) == "__config_site.in"
         or os.path.basename(path) == "libcxx.imp"
diff --git a/libcxx/utils/libcxx/header_information.py b/libcxx/utils/libcxx/header_information.py
index 9811b42d510ca..a505d37b65b81 100644
--- a/libcxx/utils/libcxx/header_information.py
+++ b/libcxx/utils/libcxx/header_information.py
@@ -15,7 +15,7 @@
 def _is_header_file(file):
     """Returns whether the given file is a header file, i.e. not a directory or the modulemap file."""
     return not file.is_dir() and not file.name in [
-        "module.modulemap",
+        "module.modulemap.in",
         "CMakeLists.txt",
         "libcxx.imp",
         "__config_site.in",

From 63b8f1c9482ed0a964980df4aed89bef922b8078 Mon Sep 17 00:00:00 2001
From: "Oleksandr \"Alex\" Zinenko" <git@ozinenko.com>
Date: Fri, 18 Apr 2025 15:19:57 +0200
Subject: [PATCH 407/710] [mlir] add a fluent API to GreedyRewriterConfig
 (#132253)

This is similar to other configuration objects used across MLIR.
---
 .../Optimizer/CodeGen/LowerRepackArrays.cpp   |  4 +-
 .../HLFIR/Transforms/InlineElementals.cpp     |  4 +-
 .../HLFIR/Transforms/InlineHLFIRAssign.cpp    |  4 +-
 .../HLFIR/Transforms/LowerHLFIRIntrinsics.cpp |  4 +-
 .../Transforms/OptimizedBufferization.cpp     |  4 +-
 .../Transforms/SimplifyHLFIRIntrinsics.cpp    |  4 +-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |  6 +-
 .../Transforms/AssumedRankOpConversion.cpp    |  4 +-
 .../ConstantArgumentGlobalisation.cpp         |  6 +-
 .../Transforms/SimplifyFIROperations.cpp      |  3 +-
 .../lib/Optimizer/Transforms/StackArrays.cpp  |  3 +-
 .../Transforms/GreedyPatternRewriteDriver.h   | 76 +++++++++++++---
 mlir/include/mlir/Transforms/Passes.td        |  2 +-
 .../TransformOps/AffineTransformOps.cpp       | 11 +--
 .../Transforms/AffineDataCopyGeneration.cpp   |  7 +-
 .../Transforms/SimplifyAffineStructures.cpp   |  7 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |  9 +-
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |  8 +-
 .../Transforms/IntRangeOptimizations.cpp      | 16 ++--
 .../BufferDeallocationSimplification.cpp      | 14 +--
 .../TransformOps/LinalgTransformOps.cpp       |  6 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  7 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  8 +-
 .../SPIRV/Transforms/SPIRVConversion.cpp      |  6 +-
 .../lib/Dialect/Transform/IR/TransformOps.cpp | 14 +--
 mlir/lib/Reducer/ReductionTreePass.cpp        |  8 +-
 mlir/lib/Transforms/Canonicalizer.cpp         | 16 ++--
 .../Utils/GreedyPatternRewriteDriver.cpp      | 88 +++++++++----------
 .../lib/Dialect/Affine/TestAffineDataCopy.cpp |  2 +-
 mlir/test/lib/Dialect/Test/TestPatterns.cpp   | 28 +++---
 30 files changed, 217 insertions(+), 162 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
index 7deed3d44ae5b..7fb713ff1a6c7 100644
--- a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
+++ b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
@@ -357,8 +357,8 @@ class LowerRepackArraysPass
     patterns.insert<PackArrayConversion>(context);
     patterns.insert<UnpackArrayConversion>(context);
     mlir::GreedyRewriteConfig config;
-    config.enableRegionSimplification =
-        mlir::GreedySimplifyRegionLevel::Disabled;
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
     (void)applyPatternsGreedily(module, std::move(patterns), config);
   }
 
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
index b68fe6ee0c747..c42b895946d19 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
@@ -119,8 +119,8 @@ class InlineElementalsPass
 
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks.
-    config.enableRegionSimplification =
-        mlir::GreedySimplifyRegionLevel::Disabled;
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<InlineElementalConversion>(context);
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
index 249976d5509b0..6e209cce07ad4 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
@@ -135,8 +135,8 @@ class InlineHLFIRAssignPass
 
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks.
-    config.enableRegionSimplification =
-        mlir::GreedySimplifyRegionLevel::Disabled;
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<InlineHLFIRAssignConversion>(context);
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
index 7c0fcba806869..31e5bc1193e22 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
@@ -557,8 +557,8 @@ class LowerHLFIRIntrinsics
     // Pattern rewriting only requires that the resulting IR is still valid
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks
-    config.enableRegionSimplification =
-        mlir::GreedySimplifyRegionLevel::Disabled;
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
 
     if (mlir::failed(
             mlir::applyPatternsGreedily(module, std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index c489450384a35..c0856082989c1 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -1327,8 +1327,8 @@ class OptimizedBufferizationPass
 
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks
-    config.enableRegionSimplification =
-        mlir::GreedySimplifyRegionLevel::Disabled;
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
 
     mlir::RewritePatternSet patterns(context);
     // TODO: right now the patterns are non-conflicting,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index bac10121a881b..d153db2afff07 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -1476,8 +1476,8 @@ class SimplifyHLFIRIntrinsics
 
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks
-    config.enableRegionSimplification =
-        mlir::GreedySimplifyRegionLevel::Disabled;
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 81ff6bf9b2c6a..7a06a27748ebd 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -35,7 +35,8 @@ void addNestedPassToAllTopLevelOperationsConditionally(
 
 void addCanonicalizerPassWithoutRegionSimplification(mlir::OpPassManager &pm) {
   mlir::GreedyRewriteConfig config;
-  config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
+  config.setRegionSimplificationLevel(
+      mlir::GreedySimplifyRegionLevel::Disabled);
   pm.addPass(mlir::createCanonicalizerPass(config));
 }
 
@@ -163,7 +164,8 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
 
   // simplify the IR
   mlir::GreedyRewriteConfig config;
-  config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
+  config.setRegionSimplificationLevel(
+      mlir::GreedySimplifyRegionLevel::Disabled);
   pm.addPass(mlir::createCSEPass());
   fir::addAVC(pm, pc.OptLevel);
   addNestedPassToAllTopLevelOperations<PassConstructor>(
diff --git a/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp b/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
index d0bd67a236419..6af1cb988a4c1 100644
--- a/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
@@ -152,8 +152,8 @@ class AssumedRankOpConversion
     patterns.insert<ReboxAssumedRankConv>(context, &symbolTable, kindMap);
     patterns.insert<IsAssumedSizeConv>(context, &symbolTable, kindMap);
     mlir::GreedyRewriteConfig config;
-    config.enableRegionSimplification =
-        mlir::GreedySimplifyRegionLevel::Disabled;
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
     (void)applyPatternsGreedily(mod, std::move(patterns), config);
   }
 };
diff --git a/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp b/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
index 562f3058f20f3..239a7cdaa4cf2 100644
--- a/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
+++ b/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
@@ -168,9 +168,9 @@ class ConstantArgumentGlobalisationOpt
     auto *context = &getContext();
     mlir::RewritePatternSet patterns(context);
     mlir::GreedyRewriteConfig config;
-    config.enableRegionSimplification =
-        mlir::GreedySimplifyRegionLevel::Disabled;
-    config.strictMode = mlir::GreedyRewriteStrictness::ExistingOps;
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
+    config.setStrictness(mlir::GreedyRewriteStrictness::ExistingOps);
 
     patterns.insert<CallOpRewriter>(context, *di);
     if (mlir::failed(
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 212de2f2286db..6d106046b70f2 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -205,7 +205,8 @@ void SimplifyFIROperationsPass::runOnOperation() {
   fir::populateSimplifyFIROperationsPatterns(patterns,
                                              preferInlineImplementation);
   mlir::GreedyRewriteConfig config;
-  config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
+  config.setRegionSimplificationLevel(
+      mlir::GreedySimplifyRegionLevel::Disabled);
 
   if (mlir::failed(
           mlir::applyPatternsGreedily(module, std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index 9a6566bef50f1..f9b9b4f4ff385 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -806,7 +806,8 @@ void StackArraysPass::runOnOperation() {
   mlir::RewritePatternSet patterns(&context);
   mlir::GreedyRewriteConfig config;
   // prevent the pattern driver form merging blocks
-  config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
+  config.setRegionSimplificationLevel(
+      mlir::GreedySimplifyRegionLevel::Disabled);
 
   patterns.insert<AllocMemConversion>(&context, *candidateOps);
   if (mlir::failed(mlir::applyOpPatternsGreedily(
diff --git a/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h b/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
index 110b4f64856eb..45e61b68f5db2 100644
--- a/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
+++ b/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
@@ -49,25 +49,43 @@ class GreedyRewriteConfig {
   /// larger patterns when given an ambiguous pattern set.
   ///
   /// Note: Only applicable when simplifying entire regions.
-  bool useTopDownTraversal = false;
+  bool getUseTopDownTraversal() const { return useTopDownTraversal; }
+  GreedyRewriteConfig &setUseTopDownTraversal(bool use = true) {
+    useTopDownTraversal = use;
+    return *this;
+  }
 
   /// Perform control flow optimizations to the region tree after applying all
   /// patterns.
   ///
   /// Note: Only applicable when simplifying entire regions.
-  GreedySimplifyRegionLevel enableRegionSimplification =
-      GreedySimplifyRegionLevel::Aggressive;
+  GreedySimplifyRegionLevel getRegionSimplificationLevel() const {
+    return regionSimplificationLevel;
+  }
+  GreedyRewriteConfig &
+  setRegionSimplificationLevel(GreedySimplifyRegionLevel level) {
+    regionSimplificationLevel = level;
+    return *this;
+  }
 
   /// This specifies the maximum number of times the rewriter will iterate
   /// between applying patterns and simplifying regions. Use `kNoLimit` to
   /// disable this iteration limit.
   ///
   /// Note: Only applicable when simplifying entire regions.
-  int64_t maxIterations = 10;
+  int64_t getMaxIterations() const { return maxIterations; }
+  GreedyRewriteConfig &setMaxIterations(int64_t iterations) {
+    maxIterations = iterations;
+    return *this;
+  }
 
   /// This specifies the maximum number of rewrites within an iteration. Use
   /// `kNoLimit` to disable this limit.
-  int64_t maxNumRewrites = kNoLimit;
+  int64_t getMaxNumRewrites() const { return maxNumRewrites; }
+  GreedyRewriteConfig &setMaxNumRewrites(int64_t limit) {
+    maxNumRewrites = limit;
+    return *this;
+  }
 
   static constexpr int64_t kNoLimit = -1;
 
@@ -75,7 +93,11 @@ class GreedyRewriteConfig {
   /// specified, the closest enclosing region around the initial list of ops
   /// (or the specified region, depending on which greedy rewrite entry point
   /// is used) is used as a scope.
-  Region *scope = nullptr;
+  Region *getScope() const { return scope; }
+  GreedyRewriteConfig &setScope(Region *scope) {
+    this->scope = scope;
+    return *this;
+  }
 
   /// Strict mode can restrict the ops that are added to the worklist during
   /// the rewrite.
@@ -87,16 +109,44 @@ class GreedyRewriteConfig {
   /// * GreedyRewriteStrictness::ExistingOps: Only pre-existing ops (that were
   ///   were on the worklist at the very beginning) enqueued. All other ops are
   ///   excluded.
-  GreedyRewriteStrictness strictMode = GreedyRewriteStrictness::AnyOp;
+  GreedyRewriteStrictness getStrictness() const { return strictness; }
+  GreedyRewriteConfig &setStrictness(GreedyRewriteStrictness mode) {
+    strictness = mode;
+    return *this;
+  }
 
   /// An optional listener that should be notified about IR modifications.
-  RewriterBase::Listener *listener = nullptr;
+  RewriterBase::Listener *getListener() const { return listener; }
+  GreedyRewriteConfig &setListener(RewriterBase::Listener *listener) {
+    this->listener = listener;
+    return *this;
+  }
 
   /// Whether this should fold while greedily rewriting.
-  bool fold = true;
+  bool isFoldingEnabled() const { return fold; }
+  GreedyRewriteConfig &enableFolding(bool enable = true) {
+    fold = enable;
+    return *this;
+  }
 
   /// If set to "true", constants are CSE'd (even across multiple regions that
   /// are in a parent-ancestor relationship).
+  bool isConstantCSEEnabled() const { return cseConstants; }
+  GreedyRewriteConfig &enableConstantCSE(bool enable = true) {
+    cseConstants = enable;
+    return *this;
+  }
+
+private:
+  Region *scope = nullptr;
+  bool useTopDownTraversal = false;
+  GreedySimplifyRegionLevel regionSimplificationLevel =
+      GreedySimplifyRegionLevel::Aggressive;
+  int64_t maxIterations = 10;
+  int64_t maxNumRewrites = kNoLimit;
+  GreedyRewriteStrictness strictness = GreedyRewriteStrictness::AnyOp;
+  RewriterBase::Listener *listener = nullptr;
+  bool fold = true;
   bool cseConstants = true;
 };
 
@@ -128,14 +178,14 @@ applyPatternsGreedily(Region &region, const FrozenRewritePatternSet &patterns,
                       GreedyRewriteConfig config = GreedyRewriteConfig(),
                       bool *changed = nullptr);
 /// Same as `applyPatternsAndGreedily` above with folding.
-/// FIXME: Remove this once transition to above is complieted.
+/// FIXME: Remove this once transition to above is completed.
 LLVM_DEPRECATED("Use applyPatternsGreedily() instead", "applyPatternsGreedily")
 inline LogicalResult
 applyPatternsAndFoldGreedily(Region &region,
                              const FrozenRewritePatternSet &patterns,
                              GreedyRewriteConfig config = GreedyRewriteConfig(),
                              bool *changed = nullptr) {
-  config.fold = true;
+  config.enableFolding();
   return applyPatternsGreedily(region, patterns, config, changed);
 }
 
@@ -187,7 +237,7 @@ applyPatternsAndFoldGreedily(Operation *op,
                              const FrozenRewritePatternSet &patterns,
                              GreedyRewriteConfig config = GreedyRewriteConfig(),
                              bool *changed = nullptr) {
-  config.fold = true;
+  config.enableFolding();
   return applyPatternsGreedily(op, patterns, config, changed);
 }
 
@@ -233,7 +283,7 @@ applyOpPatternsAndFold(ArrayRef<Operation *> ops,
                        const FrozenRewritePatternSet &patterns,
                        GreedyRewriteConfig config = GreedyRewriteConfig(),
                        bool *changed = nullptr, bool *allErased = nullptr) {
-  config.fold = true;
+  config.enableFolding();
   return applyOpPatternsGreedily(ops, patterns, config, changed, allErased);
 }
 
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
index a39ab77fc8fb3..1e89a78912e99 100644
--- a/mlir/include/mlir/Transforms/Passes.td
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -33,7 +33,7 @@ def Canonicalizer : Pass<"canonicalize"> {
     Option<"topDownProcessingEnabled", "top-down", "bool",
            /*default=*/"true",
            "Seed the worklist in general top-down order">,
-    Option<"enableRegionSimplification", "region-simplify", "mlir::GreedySimplifyRegionLevel",
+    Option<"regionSimplifyLevel", "region-simplify", "mlir::GreedySimplifyRegionLevel",
            /*default=*/"mlir::GreedySimplifyRegionLevel::Normal",
            "Perform control flow optimizations to the region tree",
              [{::llvm::cl::values(
diff --git a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
index 9f7df7823d997..43d37ee3332ef 100644
--- a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
+++ b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
@@ -127,12 +127,13 @@ SimplifyBoundedAffineOpsOp::apply(transform::TransformRewriter &rewriter,
   patterns.insert<SimplifyAffineMinMaxOp<AffineMinOp>,
                   SimplifyAffineMinMaxOp<AffineMaxOp>>(getContext(), cstr);
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
-  GreedyRewriteConfig config;
-  config.listener =
-      static_cast<RewriterBase::Listener *>(rewriter.getListener());
-  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
   // Apply the simplification pattern to a fixpoint.
-  if (failed(applyOpPatternsGreedily(targets, frozenPatterns, config))) {
+  if (failed(applyOpPatternsGreedily(
+          targets, frozenPatterns,
+          GreedyRewriteConfig()
+              .setListener(
+                  static_cast<RewriterBase::Listener *>(rewriter.getListener()))
+              .setStrictness(GreedyRewriteStrictness::ExistingAndNewOps)))) {
     auto diag = emitDefiniteFailure()
                 << "affine.min/max simplification did not converge";
     return diag;
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index 4d30213cc6ec2..62c1857e4b1da 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -237,7 +237,8 @@ void AffineDataCopyGeneration::runOnOperation() {
   AffineLoadOp::getCanonicalizationPatterns(patterns, &getContext());
   AffineStoreOp::getCanonicalizationPatterns(patterns, &getContext());
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
-  GreedyRewriteConfig config;
-  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
-  (void)applyOpPatternsGreedily(copyOps, frozenPatterns, config);
+  (void)applyOpPatternsGreedily(
+      copyOps, frozenPatterns,
+      GreedyRewriteConfig().setStrictness(
+          GreedyRewriteStrictness::ExistingAndNewOps));
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
index 31711ade3153b..9e9096c2e3186 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
@@ -109,7 +109,8 @@ void SimplifyAffineStructures::runOnOperation() {
     if (isa<AffineForOp, AffineIfOp, AffineApplyOp>(op))
       opsToSimplify.push_back(op);
   });
-  GreedyRewriteConfig config;
-  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
-  (void)applyOpPatternsGreedily(opsToSimplify, frozenPatterns, config);
+  (void)applyOpPatternsGreedily(
+      opsToSimplify, frozenPatterns,
+      GreedyRewriteConfig().setStrictness(
+          GreedyRewriteStrictness::ExistingAndNewOps));
 }
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index dd539ff685653..0d4ba3940c48e 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -315,11 +315,12 @@ LogicalResult mlir::affine::affineForOpBodySkew(AffineForOp forOp,
         // Simplify/canonicalize the affine.for.
         RewritePatternSet patterns(res.getContext());
         AffineForOp::getCanonicalizationPatterns(patterns, res.getContext());
-        GreedyRewriteConfig config;
-        config.strictMode = GreedyRewriteStrictness::ExistingOps;
         bool erased;
-        (void)applyOpPatternsGreedily(res.getOperation(), std::move(patterns),
-                                      config, /*changed=*/nullptr, &erased);
+        (void)applyOpPatternsGreedily(
+            res.getOperation(), std::move(patterns),
+            GreedyRewriteConfig().setStrictness(
+                GreedyRewriteStrictness::ExistingAndNewOps),
+            /*changed=*/nullptr, &erased);
         if (!erased && !prologue)
           prologue = res;
         if (!erased)
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 2925aa918cb1c..11798b99fa879 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -426,11 +426,11 @@ LogicalResult mlir::affine::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
   RewritePatternSet patterns(ifOp.getContext());
   AffineIfOp::getCanonicalizationPatterns(patterns, ifOp.getContext());
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
-  GreedyRewriteConfig config;
-  config.strictMode = GreedyRewriteStrictness::ExistingOps;
   bool erased;
-  (void)applyOpPatternsGreedily(ifOp.getOperation(), frozenPatterns, config,
-                                /*changed=*/nullptr, &erased);
+  (void)applyOpPatternsGreedily(
+      ifOp.getOperation(), frozenPatterns,
+      GreedyRewriteConfig().setStrictness(GreedyRewriteStrictness::ExistingOps),
+      /*changed=*/nullptr, &erased);
   if (erased) {
     if (folded)
       *folded = true;
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index 602d80a45993e..f2f93883eb2b7 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -494,10 +494,9 @@ struct IntRangeOptimizationsPass final
     RewritePatternSet patterns(ctx);
     populateIntRangeOptimizationsPatterns(patterns, solver);
 
-    GreedyRewriteConfig config;
-    config.listener = &listener;
-
-    if (failed(applyPatternsGreedily(op, std::move(patterns), config)))
+    if (failed(applyPatternsGreedily(
+            op, std::move(patterns),
+            GreedyRewriteConfig().setListener(&listener))))
       signalPassFailure();
   }
 };
@@ -520,13 +519,12 @@ struct IntRangeNarrowingPass final
     RewritePatternSet patterns(ctx);
     populateIntRangeNarrowingPatterns(patterns, solver, bitwidthsSupported);
 
-    GreedyRewriteConfig config;
     // We specifically need bottom-up traversal as cmpi pattern needs range
     // data, attached to its original argument values.
-    config.useTopDownTraversal = false;
-    config.listener = &listener;
-
-    if (failed(applyPatternsGreedily(op, std::move(patterns), config)))
+    if (failed(applyPatternsGreedily(
+            op, std::move(patterns),
+            GreedyRewriteConfig().setUseTopDownTraversal(false).setListener(
+                &listener))))
       signalPassFailure();
   }
 };
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index 35f86a62ae592..c5fab80ecaa08 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -463,15 +463,15 @@ struct BufferDeallocationSimplificationPass
                  SplitDeallocWhenNotAliasingAnyOther,
                  RetainedMemrefAliasingAlwaysDeallocatedMemref>(&getContext(),
                                                                 analysis);
+
+    populateDeallocOpCanonicalizationPatterns(patterns, &getContext());
     // We don't want that the block structure changes invalidating the
-    // `BufferOriginAnalysis` so we apply the rewrites witha `Normal` level of
+    // `BufferOriginAnalysis` so we apply the rewrites with `Normal` level of
     // region simplification
-    GreedyRewriteConfig config;
-    config.enableRegionSimplification = GreedySimplifyRegionLevel::Normal;
-    populateDeallocOpCanonicalizationPatterns(patterns, &getContext());
-
-    if (failed(
-            applyPatternsGreedily(getOperation(), std::move(patterns), config)))
+    if (failed(applyPatternsGreedily(
+            getOperation(), std::move(patterns),
+            GreedyRewriteConfig().setRegionSimplificationLevel(
+                GreedySimplifyRegionLevel::Normal))))
       signalPassFailure();
   }
 };
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index c90ebe4487ca4..b20e6050fb4f8 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -3587,9 +3587,9 @@ transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
   vector::populateVectorStepLoweringPatterns(patterns);
 
   TrackingListener listener(state, *this);
-  GreedyRewriteConfig config;
-  config.listener = &listener;
-  if (failed(applyPatternsGreedily(target, std::move(patterns), config)))
+  if (failed(
+          applyPatternsGreedily(target, std::move(patterns),
+                                GreedyRewriteConfig().setListener(&listener))))
     return emitDefaultDefiniteFailure(target);
 
   results.push_back(target);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index bf70597d5ddfe..62d016b87d627 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -2327,10 +2327,9 @@ struct LinalgElementwiseOpFusionPass
     // Add constant folding patterns.
     populateConstantFoldLinalgOperations(patterns, defaultControlFn);
 
-    // Use TopDownTraversal for compile time reasons
-    GreedyRewriteConfig grc;
-    grc.useTopDownTraversal = true;
-    (void)applyPatternsGreedily(op, std::move(patterns), grc);
+    // Use TopDownTraversal for compile time reasons.
+    (void)applyPatternsGreedily(op, std::move(patterns),
+                                GreedyRewriteConfig().setUseTopDownTraversal());
   }
 };
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index 91862d2e17d71..7edf19689d2e1 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -1438,10 +1438,10 @@ SliceTrackingListener::insertAndApplyPatterns(ArrayRef<Operation *> ops) {
   if (!patterns)
     return success();
 
-  GreedyRewriteConfig config;
-  config.listener = this;
-  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
-  return applyOpPatternsGreedily(ops, patterns.value(), config);
+  return applyOpPatternsGreedily(
+      ops, patterns.value(),
+      GreedyRewriteConfig().setListener(this).setStrictness(
+          GreedyRewriteStrictness::ExistingAndNewOps));
 }
 
 void SliceTrackingListener::notifyOperationInserted(
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index 19b9af146f4a4..811f03abb3461 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -1353,9 +1353,9 @@ LogicalResult mlir::spirv::unrollVectorsInSignatures(Operation *op) {
   // We only want to apply signature conversion once to the existing func ops.
   // Without specifying strictMode, the greedy pattern rewriter will keep
   // looking for newly created func ops.
-  GreedyRewriteConfig config;
-  config.strictMode = GreedyRewriteStrictness::ExistingOps;
-  return applyPatternsGreedily(op, std::move(patterns), config);
+  return applyPatternsGreedily(op, std::move(patterns),
+                               GreedyRewriteConfig().setStrictness(
+                                   GreedyRewriteStrictness::ExistingOps));
 }
 
 LogicalResult mlir::spirv::unrollVectorsInFuncBodies(Operation *op) {
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 798853a75441a..a811fae003584 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -393,16 +393,16 @@ DiagnosedSilenceableFailure transform::ApplyPatternsOp::applyToOne(
 
   // Configure the GreedyPatternRewriteDriver.
   GreedyRewriteConfig config;
-  config.listener =
-      static_cast<RewriterBase::Listener *>(rewriter.getListener());
+  config.setListener(
+      static_cast<RewriterBase::Listener *>(rewriter.getListener()));
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
 
-  config.maxIterations = getMaxIterations() == static_cast<uint64_t>(-1)
-                             ? GreedyRewriteConfig::kNoLimit
-                             : getMaxIterations();
-  config.maxNumRewrites = getMaxNumRewrites() == static_cast<uint64_t>(-1)
+  config.setMaxIterations(getMaxIterations() == static_cast<uint64_t>(-1)
                               ? GreedyRewriteConfig::kNoLimit
-                              : getMaxNumRewrites();
+                              : getMaxIterations());
+  config.setMaxNumRewrites(getMaxNumRewrites() == static_cast<uint64_t>(-1)
+                               ? GreedyRewriteConfig::kNoLimit
+                               : getMaxNumRewrites());
 
   // Apply patterns and CSE repetitively until a fixpoint is reached. If no CSE
   // was requested, apply the greedy pattern rewrite only once. (The greedy
diff --git a/mlir/lib/Reducer/ReductionTreePass.cpp b/mlir/lib/Reducer/ReductionTreePass.cpp
index 7292752c712ae..549e4f2bd813b 100644
--- a/mlir/lib/Reducer/ReductionTreePass.cpp
+++ b/mlir/lib/Reducer/ReductionTreePass.cpp
@@ -62,11 +62,11 @@ static void applyPatterns(Region &region,
   // before that transform.
   for (Operation *op : opsInRange) {
     // `applyOpPatternsGreedily` with folding returns whether the op is
-    // convered. Omit it because we don't have expectation this reduction will
+    // converted. Omit it because we don't have expectation this reduction will
     // be success or not.
-    GreedyRewriteConfig config;
-    config.strictMode = GreedyRewriteStrictness::ExistingOps;
-    (void)applyOpPatternsGreedily(op, patterns, config);
+    (void)applyOpPatternsGreedily(op, patterns,
+                                  GreedyRewriteConfig().setStrictness(
+                                      GreedyRewriteStrictness::ExistingOps));
   }
 
   if (eraseOpNotInRange)
diff --git a/mlir/lib/Transforms/Canonicalizer.cpp b/mlir/lib/Transforms/Canonicalizer.cpp
index 7ccd503fb0288..4b0ac28a03713 100644
--- a/mlir/lib/Transforms/Canonicalizer.cpp
+++ b/mlir/lib/Transforms/Canonicalizer.cpp
@@ -32,10 +32,10 @@ struct Canonicalizer : public impl::CanonicalizerBase<Canonicalizer> {
                 ArrayRef<std::string> disabledPatterns,
                 ArrayRef<std::string> enabledPatterns)
       : config(config) {
-    this->topDownProcessingEnabled = config.useTopDownTraversal;
-    this->enableRegionSimplification = config.enableRegionSimplification;
-    this->maxIterations = config.maxIterations;
-    this->maxNumRewrites = config.maxNumRewrites;
+    this->topDownProcessingEnabled = config.getUseTopDownTraversal();
+    this->regionSimplifyLevel = config.getRegionSimplificationLevel();
+    this->maxIterations = config.getMaxIterations();
+    this->maxNumRewrites = config.getMaxNumRewrites();
     this->disabledPatterns = disabledPatterns;
     this->enabledPatterns = enabledPatterns;
   }
@@ -44,10 +44,10 @@ struct Canonicalizer : public impl::CanonicalizerBase<Canonicalizer> {
   /// execution.
   LogicalResult initialize(MLIRContext *context) override {
     // Set the config from possible pass options set in the meantime.
-    config.useTopDownTraversal = topDownProcessingEnabled;
-    config.enableRegionSimplification = enableRegionSimplification;
-    config.maxIterations = maxIterations;
-    config.maxNumRewrites = maxNumRewrites;
+    config.setUseTopDownTraversal(topDownProcessingEnabled);
+    config.setRegionSimplificationLevel(regionSimplifyLevel);
+    config.setMaxIterations(maxIterations);
+    config.setMaxNumRewrites(maxNumRewrites);
 
     RewritePatternSet owningPatterns(context);
     for (auto *dialect : context->getLoadedDialects())
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index 7c1cfd91f85e6..84d547f03829a 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -455,8 +455,8 @@ bool GreedyPatternRewriteDriver::processWorklist() {
   bool changed = false;
   int64_t numRewrites = 0;
   while (!worklist.empty() &&
-         (numRewrites < config.maxNumRewrites ||
-          config.maxNumRewrites == GreedyRewriteConfig::kNoLimit)) {
+         (numRewrites < config.getMaxNumRewrites() ||
+          config.getMaxNumRewrites() == GreedyRewriteConfig::kNoLimit)) {
     auto *op = worklist.pop();
 
     LLVM_DEBUG({
@@ -488,7 +488,7 @@ bool GreedyPatternRewriteDriver::processWorklist() {
     // infinite folding loop, as every constant op would be folded to an
     // Attribute and then immediately be rematerialized as a constant op, which
     // is then put on the worklist.
-    if (config.fold && !op->hasTrait<OpTrait::ConstantLike>()) {
+    if (config.isFoldingEnabled() && !op->hasTrait<OpTrait::ConstantLike>()) {
       SmallVector<OpFoldResult> foldResults;
       if (succeeded(op->fold(foldResults))) {
         LLVM_DEBUG(logResultWithLine("success", "operation was folded"));
@@ -574,21 +574,21 @@ bool GreedyPatternRewriteDriver::processWorklist() {
         logger.getOStream() << ")' {\n";
         logger.indent();
       });
-      if (config.listener)
-        config.listener->notifyPatternBegin(pattern, op);
+      if (RewriterBase::Listener *listener = config.getListener())
+        listener->notifyPatternBegin(pattern, op);
       return true;
     };
     function_ref<bool(const Pattern &)> canApply = canApplyCallback;
     auto onFailureCallback = [&](const Pattern &pattern) {
       LLVM_DEBUG(logResult("failure", "pattern failed to match"));
-      if (config.listener)
-        config.listener->notifyPatternEnd(pattern, failure());
+      if (RewriterBase::Listener *listener = config.getListener())
+        listener->notifyPatternEnd(pattern, failure());
     };
     function_ref<void(const Pattern &)> onFailure = onFailureCallback;
     auto onSuccessCallback = [&](const Pattern &pattern) {
       LLVM_DEBUG(logResult("success", "pattern applied successfully"));
-      if (config.listener)
-        config.listener->notifyPatternEnd(pattern, success());
+      if (RewriterBase::Listener *listener = config.getListener())
+        listener->notifyPatternEnd(pattern, success());
       return success();
     };
     function_ref<LogicalResult(const Pattern &)> onSuccess = onSuccessCallback;
@@ -640,7 +640,7 @@ void GreedyPatternRewriteDriver::addToWorklist(Operation *op) {
   do {
     ancestors.push_back(op);
     region = op->getParentRegion();
-    if (config.scope == region) {
+    if (config.getScope() == region) {
       // Scope (can be `nullptr`) was reached. Stop traveral and enqueue ops.
       for (Operation *op : ancestors)
         addSingleOpToWorklist(op);
@@ -652,20 +652,20 @@ void GreedyPatternRewriteDriver::addToWorklist(Operation *op) {
 }
 
 void GreedyPatternRewriteDriver::addSingleOpToWorklist(Operation *op) {
-  if (config.strictMode == GreedyRewriteStrictness::AnyOp ||
+  if (config.getStrictness() == GreedyRewriteStrictness::AnyOp ||
       strictModeFilteredOps.contains(op))
     worklist.push(op);
 }
 
 void GreedyPatternRewriteDriver::notifyBlockInserted(
     Block *block, Region *previous, Region::iterator previousIt) {
-  if (config.listener)
-    config.listener->notifyBlockInserted(block, previous, previousIt);
+  if (RewriterBase::Listener *listener = config.getListener())
+    listener->notifyBlockInserted(block, previous, previousIt);
 }
 
 void GreedyPatternRewriteDriver::notifyBlockErased(Block *block) {
-  if (config.listener)
-    config.listener->notifyBlockErased(block);
+  if (RewriterBase::Listener *listener = config.getListener())
+    listener->notifyBlockErased(block);
 }
 
 void GreedyPatternRewriteDriver::notifyOperationInserted(
@@ -674,9 +674,9 @@ void GreedyPatternRewriteDriver::notifyOperationInserted(
     logger.startLine() << "** Insert  : '" << op->getName() << "'(" << op
                        << ")\n";
   });
-  if (config.listener)
-    config.listener->notifyOperationInserted(op, previous);
-  if (config.strictMode == GreedyRewriteStrictness::ExistingAndNewOps)
+  if (RewriterBase::Listener *listener = config.getListener())
+    listener->notifyOperationInserted(op, previous);
+  if (config.getStrictness() == GreedyRewriteStrictness::ExistingAndNewOps)
     strictModeFilteredOps.insert(op);
   addToWorklist(op);
 }
@@ -686,8 +686,8 @@ void GreedyPatternRewriteDriver::notifyOperationModified(Operation *op) {
     logger.startLine() << "** Modified: '" << op->getName() << "'(" << op
                        << ")\n";
   });
-  if (config.listener)
-    config.listener->notifyOperationModified(op);
+  if (RewriterBase::Listener *listener = config.getListener())
+    listener->notifyOperationModified(op);
   addToWorklist(op);
 }
 
@@ -736,18 +736,18 @@ void GreedyPatternRewriteDriver::notifyOperationErased(Operation *op) {
   // the part of the IR that is taken into account for the "expensive checks".
   // A greedy pattern rewrite is not allowed to erase the parent op of the scope
   // region, as that would break the worklist handling and the expensive checks.
-  if (config.scope && config.scope->getParentOp() == op)
+  if (Region *scope = config.getScope(); scope->getParentOp() == op)
     llvm_unreachable(
         "scope region must not be erased during greedy pattern rewrite");
 #endif // NDEBUG
 
-  if (config.listener)
-    config.listener->notifyOperationErased(op);
+  if (RewriterBase::Listener *listener = config.getListener())
+    listener->notifyOperationErased(op);
 
   addOperandsToWorklist(op);
   worklist.remove(op);
 
-  if (config.strictMode != GreedyRewriteStrictness::AnyOp)
+  if (config.getStrictness() != GreedyRewriteStrictness::AnyOp)
     strictModeFilteredOps.erase(op);
 }
 
@@ -757,8 +757,8 @@ void GreedyPatternRewriteDriver::notifyOperationReplaced(
     logger.startLine() << "** Replace : '" << op->getName() << "'(" << op
                        << ")\n";
   });
-  if (config.listener)
-    config.listener->notifyOperationReplaced(op, replacement);
+  if (RewriterBase::Listener *listener = config.getListener())
+    listener->notifyOperationReplaced(op, replacement);
 }
 
 void GreedyPatternRewriteDriver::notifyMatchFailure(
@@ -768,8 +768,8 @@ void GreedyPatternRewriteDriver::notifyMatchFailure(
     reasonCallback(diag);
     logger.startLine() << "** Match Failure : " << diag.str() << "\n";
   });
-  if (config.listener)
-    config.listener->notifyMatchFailure(loc, reasonCallback);
+  if (RewriterBase::Listener *listener = config.getListener())
+    listener->notifyMatchFailure(loc, reasonCallback);
 }
 
 //===----------------------------------------------------------------------===//
@@ -800,7 +800,7 @@ RegionPatternRewriteDriver::RegionPatternRewriteDriver(
     const GreedyRewriteConfig &config, Region &region)
     : GreedyPatternRewriteDriver(ctx, patterns, config), region(region) {
   // Populate strict mode ops.
-  if (config.strictMode != GreedyRewriteStrictness::AnyOp) {
+  if (config.getStrictness() != GreedyRewriteStrictness::AnyOp) {
     region.walk([&](Operation *op) { strictModeFilteredOps.insert(op); });
   }
 }
@@ -829,8 +829,8 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
   MLIRContext *ctx = rewriter.getContext();
   do {
     // Check if the iteration limit was reached.
-    if (++iteration > config.maxIterations &&
-        config.maxIterations != GreedyRewriteConfig::kNoLimit)
+    if (++iteration > config.getMaxIterations() &&
+        config.getMaxIterations() != GreedyRewriteConfig::kNoLimit)
       break;
 
     // New iteration: start with an empty worklist.
@@ -849,16 +849,16 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
       return false;
     };
 
-    if (!config.useTopDownTraversal) {
+    if (!config.getUseTopDownTraversal()) {
       // Add operations to the worklist in postorder.
       region.walk([&](Operation *op) {
-        if (!config.cseConstants || !insertKnownConstant(op))
+        if (!config.isConstantCSEEnabled() || !insertKnownConstant(op))
           addToWorklist(op);
       });
     } else {
       // Add all nested operations to the worklist in preorder.
       region.walk<WalkOrder::PreOrder>([&](Operation *op) {
-        if (!config.cseConstants || !insertKnownConstant(op)) {
+        if (!config.isConstantCSEEnabled() || !insertKnownConstant(op)) {
           addToWorklist(op);
           return WalkResult::advance();
         }
@@ -875,11 +875,11 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
 
           // After applying patterns, make sure that the CFG of each of the
           // regions is kept up to date.
-          if (config.enableRegionSimplification !=
+          if (config.getRegionSimplificationLevel() !=
               GreedySimplifyRegionLevel::Disabled) {
             continueRewrites |= succeeded(simplifyRegions(
                 rewriter, region,
-                /*mergeBlocks=*/config.enableRegionSimplification ==
+                /*mergeBlocks=*/config.getRegionSimplificationLevel() ==
                     GreedySimplifyRegionLevel::Aggressive));
           }
         },
@@ -904,8 +904,8 @@ mlir::applyPatternsGreedily(Region &region,
          "patterns can only be applied to operations IsolatedFromAbove");
 
   // Set scope if not specified.
-  if (!config.scope)
-    config.scope = &region;
+  if (!config.getScope())
+    config.setScope(&region);
 
 #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
   if (failed(verify(config.scope->getParentOp())))
@@ -919,7 +919,7 @@ mlir::applyPatternsGreedily(Region &region,
   LogicalResult converged = std::move(driver).simplify(changed);
   LLVM_DEBUG(if (failed(converged)) {
     llvm::dbgs() << "The pattern rewrite did not converge after scanning "
-                 << config.maxIterations << " times\n";
+                 << config.getMaxIterations() << " times\n";
   });
   return converged;
 }
@@ -960,7 +960,7 @@ MultiOpPatternRewriteDriver::MultiOpPatternRewriteDriver(
     llvm::SmallDenseSet<Operation *, 4> *survivingOps)
     : GreedyPatternRewriteDriver(ctx, patterns, config),
       survivingOps(survivingOps) {
-  if (config.strictMode != GreedyRewriteStrictness::AnyOp)
+  if (config.getStrictness() != GreedyRewriteStrictness::AnyOp)
     strictModeFilteredOps.insert_range(ops);
 
   if (survivingOps) {
@@ -1024,15 +1024,15 @@ LogicalResult mlir::applyOpPatternsGreedily(
   }
 
   // Determine scope of rewrite.
-  if (!config.scope) {
+  if (!config.getScope()) {
     // Compute scope if none was provided. The scope will remain `nullptr` if
     // there is a top-level op among `ops`.
-    config.scope = findCommonAncestor(ops);
+    config.setScope(findCommonAncestor(ops));
   } else {
     // If a scope was provided, make sure that all ops are in scope.
 #ifndef NDEBUG
     bool allOpsInScope = llvm::all_of(ops, [&](Operation *op) {
-      return static_cast<bool>(config.scope->findAncestorOpInRegion(*op));
+      return static_cast<bool>(config.getScope()->findAncestorOpInRegion(*op));
     });
     assert(allOpsInScope && "ops must be within the specified scope");
 #endif // NDEBUG
@@ -1054,7 +1054,7 @@ LogicalResult mlir::applyOpPatternsGreedily(
     *allErased = surviving.empty();
   LLVM_DEBUG(if (failed(converged)) {
     llvm::dbgs() << "The pattern rewrite did not converge after "
-                 << config.maxNumRewrites << " rewrites";
+                 << config.getMaxNumRewrites() << " rewrites";
   });
   return converged;
 }
diff --git a/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp b/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
index d6aaa6faf94cb..2a54e0c28f71f 100644
--- a/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
@@ -144,7 +144,7 @@ void TestAffineDataCopy::runOnOperation() {
     }
   }
   GreedyRewriteConfig config;
-  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
+  config.setStrictness(GreedyRewriteStrictness::ExistingAndNewOps);
   (void)applyOpPatternsGreedily(copyOps, std::move(patterns), config);
 }
 
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index db02a122872d9..d073843484d81 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -386,26 +386,26 @@ struct TestGreedyPatternDriver
     patterns.insert<IncrementIntAttribute<3>>(&getContext());
 
     GreedyRewriteConfig config;
-    config.useTopDownTraversal = this->useTopDownTraversal;
-    config.maxIterations = this->maxIterations;
-    config.fold = this->fold;
-    config.cseConstants = this->cseConstants;
+    config.setUseTopDownTraversal(useTopDownTraversal)
+        .setMaxIterations(this->maxIterations)
+        .enableFolding(this->fold)
+        .enableConstantCSE(this->cseConstants);
     (void)applyPatternsGreedily(getOperation(), std::move(patterns), config);
   }
 
   Option<bool> useTopDownTraversal{
       *this, "top-down",
       llvm::cl::desc("Seed the worklist in general top-down order"),
-      llvm::cl::init(GreedyRewriteConfig().useTopDownTraversal)};
+      llvm::cl::init(GreedyRewriteConfig().getUseTopDownTraversal())};
   Option<int> maxIterations{
       *this, "max-iterations",
       llvm::cl::desc("Max. iterations in the GreedyRewriteConfig"),
-      llvm::cl::init(GreedyRewriteConfig().maxIterations)};
+      llvm::cl::init(GreedyRewriteConfig().getMaxIterations())};
   Option<bool> fold{*this, "fold", llvm::cl::desc("Whether to fold"),
-                    llvm::cl::init(GreedyRewriteConfig().fold)};
-  Option<bool> cseConstants{*this, "cse-constants",
-                            llvm::cl::desc("Whether to CSE constants"),
-                            llvm::cl::init(GreedyRewriteConfig().cseConstants)};
+                    llvm::cl::init(GreedyRewriteConfig().isFoldingEnabled())};
+  Option<bool> cseConstants{
+      *this, "cse-constants", llvm::cl::desc("Whether to CSE constants"),
+      llvm::cl::init(GreedyRewriteConfig().isConstantCSEEnabled())};
 };
 
 struct DumpNotifications : public RewriterBase::Listener {
@@ -501,13 +501,13 @@ struct TestStrictPatternDriver
 
     DumpNotifications dumpNotifications;
     GreedyRewriteConfig config;
-    config.listener = &dumpNotifications;
+    config.setListener(&dumpNotifications);
     if (strictMode == "AnyOp") {
-      config.strictMode = GreedyRewriteStrictness::AnyOp;
+      config.setStrictness(GreedyRewriteStrictness::AnyOp);
     } else if (strictMode == "ExistingAndNewOps") {
-      config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
+      config.setStrictness(GreedyRewriteStrictness::ExistingAndNewOps);
     } else if (strictMode == "ExistingOps") {
-      config.strictMode = GreedyRewriteStrictness::ExistingOps;
+      config.setStrictness(GreedyRewriteStrictness::ExistingOps);
     } else {
       llvm_unreachable("invalid strictness option");
     }

From 730773602f8b039a575e3e48cd838174dd996520 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 15:34:11 +0200
Subject: [PATCH 408/710] llvm-reduce: Avoid using constantdata uselistorder in
 thinlto test (#136288)

This also demonstrates a bug that's a consequence of the two different
paths for the single and multithreaded cases. The parallel path goes
through bitcode serialization and does preserve the uselistorder. It
therefore survives and we can observe a reduced uselistorder with deleted
instructions. In the CloneModule case, nothing is reduced.
---
 .../thinlto-preserve-uselistorder.ll          | 59 ++++++++++++++-----
 1 file changed, 44 insertions(+), 15 deletions(-)

diff --git a/llvm/test/tools/llvm-reduce/thinlto-preserve-uselistorder.ll b/llvm/test/tools/llvm-reduce/thinlto-preserve-uselistorder.ll
index 2332f2d632911..eac3e462aab8a 100644
--- a/llvm/test/tools/llvm-reduce/thinlto-preserve-uselistorder.ll
+++ b/llvm/test/tools/llvm-reduce/thinlto-preserve-uselistorder.ll
@@ -1,19 +1,48 @@
-; RUN: opt --thinlto-bc --thinlto-split-lto-unit %s -o %t.0
-; RUN: llvm-reduce -write-tmp-files-as-bitcode --delta-passes=instructions %t.0 -o %t.1 \
+; RUN: opt --preserve-bc-uselistorder --thinlto-bc --thinlto-split-lto-unit %s -o %t.0
+
+; RUN: llvm-reduce -j=2 --delta-passes=instructions %t.0 -o %t.1 \
 ; RUN:     --test %python --test-arg %p/Inputs/llvm-dis-and-filecheck.py --test-arg llvm-dis --test-arg FileCheck --test-arg --check-prefix=INTERESTING --test-arg %s
 ; RUN: llvm-dis --preserve-ll-uselistorder %t.1 -o %t.2
-; RUN: FileCheck --check-prefix=RESULT %s < %t.2
-
-define i32 @func(i32 %arg0, i32 %arg1) {
-entry:
-  %add0 = add i32 %arg0, 0
-  %add1 = add i32 %add0, 0
-  %add2 = add i32 %add1, 0
-  %add3 = add i32 %arg1, 0
-  %add4 = add i32 %add2, %add3
-  ret i32 %add4
+; RUN: FileCheck --check-prefixes=RESULT,RESULT-PARALLEL %s < %t.2
+
+; FIXME: The single thread path uses CloneModule, which does not
+; preserve uselistorder. Consequently it is incapable of reducing
+; anything a case that depends on uselistorder.
+
+; RUN: llvm-reduce -j=1 --delta-passes=instructions %t.0 -o %t.3 \
+; RUN:     --test %python --test-arg %p/Inputs/llvm-dis-and-filecheck.py --test-arg llvm-dis --test-arg FileCheck --test-arg --check-prefix=INTERESTING --test-arg %s
+; RUN: llvm-dis --preserve-ll-uselistorder %t.3 -o %t.4
+; RUN: FileCheck --check-prefixes=RESULT,RESULT-SINGLE %s < %t.4
+
+@gv0 = external global [0 x i8]
+
+; RESULT-LABEL: @func(
+; RESULT-PARALLEL-NEXT: %gep0 = getelementptr i8, ptr @gv0, i32 %arg0
+; RESULT-PARALLEL-NEXT: %gep1 = getelementptr i8, ptr @gv0, i32 %arg1
+; RESULT-PARALLEL-NEXT: ret void
+
+; RESULT-SINGLE: ptr @gv0
+; RESULT-SINGLE: ptr @gv0
+; RESULT-SINGLE: ptr @gv0
+; RESULT-SINGLE: ptr @gv0
+; RESULT-SINGLE: ptr @gv0
+define void @func(i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3) {
+  %gep0 = getelementptr i8, ptr @gv0, i32 %arg0
+  %gep1 = getelementptr i8, ptr @gv0, i32 %arg1
+  %gep2 = getelementptr i8, ptr @gv0, i32 %arg2
+  %gep3 = getelementptr i8, ptr @gv0, i32 %arg3
+  store i32 0, ptr %gep0
+  store i32 0, ptr %gep1
+  store i32 0, ptr %gep2
+  store i32 0, ptr %gep3
+  store i32 0, ptr @gv0
+  ret void
 }
 
-; INTERESTING: uselistorder i32 0
-; RESULT: uselistorder i32 0, { 0, 2, 1 }
-uselistorder i32 0, { 3, 2, 1, 0 }
+; INTERESTING: uselistorder ptr
+
+; RESULT: uselistorder directives
+; RESULT-PARALLEL: uselistorder ptr @gv0, { 1, 0 }
+; RESULT-SINGLE: uselistorder ptr @gv0, { 3, 4, 2, 1, 0 }
+
+uselistorder ptr @gv0, { 3, 4, 2, 1, 0 }

From 35e6ca47c1b565b59a9e9f90b5239804e64819f7 Mon Sep 17 00:00:00 2001
From: amordo <iammorjj@gmail.com>
Date: Fri, 18 Apr 2025 15:35:02 +0200
Subject: [PATCH 409/710] [docs] Add TOC for InstCombine contributor guide
 (#136293)

---
 llvm/docs/InstCombineContributorGuide.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/docs/InstCombineContributorGuide.md b/llvm/docs/InstCombineContributorGuide.md
index ce5f958058c55..b4041f8a5b93f 100644
--- a/llvm/docs/InstCombineContributorGuide.md
+++ b/llvm/docs/InstCombineContributorGuide.md
@@ -1,5 +1,9 @@
 # InstCombine contributor guide
 
+```{contents}
+:depth: 4
+```
+
 This guide lays out a series of rules that contributions to InstCombine should
 follow. **Following these rules will results in much faster PR approvals.**
 

From 20a104a7d6423784dab04371a5ca728cc27a15a9 Mon Sep 17 00:00:00 2001
From: "Oleksandr \"Alex\" Zinenko" <git@ozinenko.com>
Date: Fri, 18 Apr 2025 16:05:54 +0200
Subject: [PATCH 410/710] [mlir] allow function type cloning to fail (#136300)

`FunctionOpInterface` assumed the fact that the function type (attribute
of the operation) can be cloned with arbirary lists of function
arguments and results to support argument and result list mutation. This
is not always correct, in particular, LLVM dialect functions require
exactly one result making it impossible to erase the result.

Allow function type cloning to fail and propagate this failure through
various APIs that use it. The common assumption is that existing IR has
not been modified.

Fixes #131142.
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td |  3 +-
 .../mlir/Interfaces/FunctionInterfaces.td     | 81 ++++++++++++-------
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |  8 +-
 .../Transforms/BufferResultsToOutParams.cpp   |  3 +-
 .../DropEquivalentBufferResults.cpp           |  3 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp      |  5 +-
 mlir/lib/Query/Query.cpp                      |  7 +-
 mlir/lib/Transforms/RemoveDeadValues.cpp      |  7 +-
 mlir/test/IR/test-func-erase-result.mlir      |  7 +-
 mlir/test/lib/IR/TestFunc.cpp                 | 25 ++++--
 10 files changed, 105 insertions(+), 44 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
index e9f01ba5cd4e4..072fc7be07dd4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
@@ -104,7 +104,8 @@ def LLVMFunctionType : LLVMType<"LLVMFunction", "func"> {
     bool isVarArg() const { return getVarArg(); }
 
     /// Returns a clone of this function type with the given argument
-    /// and result types.
+    /// and result types. Returns null if the resulting function type would
+    /// not verify.
     LLVMFunctionType clone(TypeRange inputs, TypeRange results) const;
 
     /// Returns the result type of the function as an ArrayRef, enabling better
diff --git a/mlir/include/mlir/Interfaces/FunctionInterfaces.td b/mlir/include/mlir/Interfaces/FunctionInterfaces.td
index 1e2ca9a8b4490..cad0e790fdd97 100644
--- a/mlir/include/mlir/Interfaces/FunctionInterfaces.td
+++ b/mlir/include/mlir/Interfaces/FunctionInterfaces.td
@@ -255,79 +255,105 @@ def FunctionOpInterface : OpInterface<"FunctionOpInterface", [
     BlockArgListType getArguments() { return getFunctionBody().getArguments(); }
 
     /// Insert a single argument of type `argType` with attributes `argAttrs` and
-    /// location `argLoc` at `argIndex`.
-    void insertArgument(unsigned argIndex, ::mlir::Type argType, ::mlir::DictionaryAttr argAttrs,
-                        ::mlir::Location argLoc) {
-      insertArguments({argIndex}, {argType}, {argAttrs}, {argLoc});
+    /// location `argLoc` at `argIndex`. Returns failure if the function cannot be
+    /// updated to have the new signature.
+    ::llvm::LogicalResult insertArgument(
+        unsigned argIndex, ::mlir::Type argType, ::mlir::DictionaryAttr argAttrs,
+        ::mlir::Location argLoc) {
+      return insertArguments({argIndex}, {argType}, {argAttrs}, {argLoc});
     }
 
     /// Inserts arguments with the listed types, attributes, and locations at the
     /// listed indices. `argIndices` must be sorted. Arguments are inserted in the
     /// order they are listed, such that arguments with identical index will
-    /// appear in the same order that they were listed here.
-    void insertArguments(::llvm::ArrayRef<unsigned> argIndices, ::mlir::TypeRange argTypes,
-                        ::llvm::ArrayRef<::mlir::DictionaryAttr> argAttrs,
-                        ::llvm::ArrayRef<::mlir::Location> argLocs) {
+    /// appear in the same order that they were listed here. Returns failure if
+    /// the function cannot be updated to have the new signature.
+    ::llvm::LogicalResult insertArguments(
+        ::llvm::ArrayRef<unsigned> argIndices, ::mlir::TypeRange argTypes,
+        ::llvm::ArrayRef<::mlir::DictionaryAttr> argAttrs,
+        ::llvm::ArrayRef<::mlir::Location> argLocs) {
       unsigned originalNumArgs = $_op.getNumArguments();
       ::mlir::Type newType = $_op.getTypeWithArgsAndResults(
           argIndices, argTypes, /*resultIndices=*/{}, /*resultTypes=*/{});
+      if (!newType)
+        return ::llvm::failure();
       ::mlir::function_interface_impl::insertFunctionArguments(
           $_op, argIndices, argTypes, argAttrs, argLocs,
           originalNumArgs, newType);
+      return ::llvm::success();
     }
 
-    /// Insert a single result of type `resultType` at `resultIndex`.
-    void insertResult(unsigned resultIndex, ::mlir::Type resultType,
-                      ::mlir::DictionaryAttr resultAttrs) {
-      insertResults({resultIndex}, {resultType}, {resultAttrs});
+    /// Insert a single result of type `resultType` at `resultIndex`.Returns
+    /// failure if the function cannot be updated to have the new signature.
+    ::llvm::LogicalResult insertResult(
+        unsigned resultIndex, ::mlir::Type resultType,
+        ::mlir::DictionaryAttr resultAttrs) {
+      return insertResults({resultIndex}, {resultType}, {resultAttrs});
     }
 
     /// Inserts results with the listed types at the listed indices.
     /// `resultIndices` must be sorted. Results are inserted in the order they are
     /// listed, such that results with identical index will appear in the same
-    /// order that they were listed here.
-    void insertResults(::llvm::ArrayRef<unsigned> resultIndices, ::mlir::TypeRange resultTypes,
-                       ::llvm::ArrayRef<::mlir::DictionaryAttr> resultAttrs) {
+    /// order that they were listed here. Returns failure if the function
+    /// cannot be updated to have the new signature.
+    ::llvm::LogicalResult insertResults(
+        ::llvm::ArrayRef<unsigned> resultIndices,
+        ::mlir::TypeRange resultTypes,
+        ::llvm::ArrayRef<::mlir::DictionaryAttr> resultAttrs) {
       unsigned originalNumResults = $_op.getNumResults();
       ::mlir::Type newType = $_op.getTypeWithArgsAndResults(
         /*argIndices=*/{}, /*argTypes=*/{}, resultIndices, resultTypes);
+      if (!newType)
+        return ::llvm::failure();
       ::mlir::function_interface_impl::insertFunctionResults(
           $_op, resultIndices, resultTypes, resultAttrs,
           originalNumResults, newType);
+      return ::llvm::success();
     }
 
-    /// Erase a single argument at `argIndex`.
-    void eraseArgument(unsigned argIndex) {
+    /// Erase a single argument at `argIndex`. Returns failure if the function
+    /// cannot be updated to have the new signature.
+    ::llvm::LogicalResult eraseArgument(unsigned argIndex) {
       ::llvm::BitVector argsToErase($_op.getNumArguments());
       argsToErase.set(argIndex);
-      eraseArguments(argsToErase);
+      return eraseArguments(argsToErase);
     }
 
-    /// Erases the arguments listed in `argIndices`.
-    void eraseArguments(const ::llvm::BitVector &argIndices) {
+    /// Erases the arguments listed in `argIndices`. Returns failure if the
+    /// function cannot be updated to have the new signature.
+    ::llvm::LogicalResult eraseArguments(const ::llvm::BitVector &argIndices) {
       ::mlir::Type newType = $_op.getTypeWithoutArgs(argIndices);
+      if (!newType)
+        return ::llvm::failure();
       ::mlir::function_interface_impl::eraseFunctionArguments(
         $_op, argIndices, newType);
+      return ::llvm::success();
     }
 
-    /// Erase a single result at `resultIndex`.
-    void eraseResult(unsigned resultIndex) {
+    /// Erase a single result at `resultIndex`. Returns failure if the function
+    /// cannot be updated to have the new signature.
+    LogicalResult eraseResult(unsigned resultIndex) {
       ::llvm::BitVector resultsToErase($_op.getNumResults());
       resultsToErase.set(resultIndex);
-      eraseResults(resultsToErase);
+      return eraseResults(resultsToErase);
     }
 
-    /// Erases the results listed in `resultIndices`.
-    void eraseResults(const ::llvm::BitVector &resultIndices) {
+    /// Erases the results listed in `resultIndices`.  Returns failure if the
+    /// function cannot be updated to have the new signature.
+    ::llvm::LogicalResult eraseResults(const ::llvm::BitVector &resultIndices) {
       ::mlir::Type newType = $_op.getTypeWithoutResults(resultIndices);
+      if (!newType)
+        return ::llvm::failure();
       ::mlir::function_interface_impl::eraseFunctionResults(
           $_op, resultIndices, newType);
+      return ::llvm::success();
     }
 
     /// Return the type of this function with the specified arguments and
     /// results inserted. This is used to update the function's signature in
     /// the `insertArguments` and `insertResults` methods. The arrays must be
-    /// sorted by increasing index.
+    /// sorted by increasing index. Return nullptr if the updated type would
+    /// not be valid.
     ::mlir::Type getTypeWithArgsAndResults(
       ::llvm::ArrayRef<unsigned> argIndices, ::mlir::TypeRange argTypes,
       ::llvm::ArrayRef<unsigned> resultIndices, ::mlir::TypeRange resultTypes) {
@@ -341,7 +367,8 @@ def FunctionOpInterface : OpInterface<"FunctionOpInterface", [
 
     /// Return the type of this function without the specified arguments and
     /// results. This is used to update the function's signature in the
-    /// `eraseArguments` and `eraseResults` methods.
+    /// `eraseArguments` and `eraseResults` methods. Return nullptr if the
+    /// updated type would not be valid.
     ::mlir::Type getTypeWithoutArgsAndResults(
       const ::llvm::BitVector &argIndices, const ::llvm::BitVector &resultIndices) {
       ::llvm::SmallVector<::mlir::Type> argStorage, resultStorage;
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index f22ad1fd70db2..dea3d99704d9b 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -125,8 +125,12 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
     // Perform signature modification
     rewriter.modifyOpInPlace(
         gpuFuncOp, [gpuFuncOp, &argIndices, &argTypes, &argAttrs, &argLocs]() {
-          static_cast<FunctionOpInterface>(gpuFuncOp).insertArguments(
-              argIndices, argTypes, argAttrs, argLocs);
+          LogicalResult inserted =
+              static_cast<FunctionOpInterface>(gpuFuncOp).insertArguments(
+                  argIndices, argTypes, argAttrs, argLocs);
+          (void)inserted;
+          assert(succeeded(inserted) &&
+                 "expected GPU funcs to support inserting any argument");
         });
   } else {
     workgroupBuffers.reserve(gpuFuncOp.getNumWorkgroupAttributions());
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index 1c95ab77b9f33..acf5f7767d12a 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -92,7 +92,8 @@ updateFuncOp(func::FuncOp func,
   }
 
   // Erase the results.
-  func.eraseResults(erasedResultIndices);
+  if (failed(func.eraseResults(erasedResultIndices)))
+    return failure();
 
   // Add the new arguments to the entry block if the function is not external.
   if (func.isExternal())
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index 9bc75267e70e4..ae011904cb972 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -113,7 +113,8 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
     }
 
     // Update function.
-    funcOp.eraseResults(erasedResultIndices);
+    if (failed(funcOp.eraseResults(erasedResultIndices)))
+      return failure();
     returnOp.getOperandsMutable().assign(newReturnValues);
 
     // Update function calls.
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
index 29cf38c1fefea..e2bccf7f6493e 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
@@ -232,7 +232,10 @@ LLVMFunctionType::getChecked(function_ref<InFlightDiagnostic()> emitError,
 
 LLVMFunctionType LLVMFunctionType::clone(TypeRange inputs,
                                          TypeRange results) const {
-  assert(results.size() == 1 && "expected a single result type");
+  if (results.size() != 1 || !isValidResultType(results[0]))
+    return {};
+  if (!llvm::all_of(inputs, isValidArgumentType))
+    return {};
   return get(results[0], llvm::to_vector(inputs), isVarArg());
 }
 
diff --git a/mlir/lib/Query/Query.cpp b/mlir/lib/Query/Query.cpp
index 869ee8f2ae1dc..73f313cd37fd0 100644
--- a/mlir/lib/Query/Query.cpp
+++ b/mlir/lib/Query/Query.cpp
@@ -88,10 +88,11 @@ static Operation *extractFunction(std::vector<Operation *> &ops,
   // Remove unused function arguments
   size_t currentIndex = 0;
   while (currentIndex < funcOp.getNumArguments()) {
+    // Erase if possible.
     if (funcOp.getArgument(currentIndex).use_empty())
-      funcOp.eraseArgument(currentIndex);
-    else
-      ++currentIndex;
+      if (succeeded(funcOp.eraseArgument(currentIndex)))
+        continue;
+    ++currentIndex;
   }
 
   return funcOp;
diff --git a/mlir/lib/Transforms/RemoveDeadValues.cpp b/mlir/lib/Transforms/RemoveDeadValues.cpp
index 92c05d87a002d..4939dda4a20f9 100644
--- a/mlir/lib/Transforms/RemoveDeadValues.cpp
+++ b/mlir/lib/Transforms/RemoveDeadValues.cpp
@@ -698,8 +698,11 @@ static void cleanUpDeadVals(RDVFinalCleanupList &list) {
 
   // 3. Functions
   for (auto &f : list.functions) {
-    f.funcOp.eraseArguments(f.nonLiveArgs);
-    f.funcOp.eraseResults(f.nonLiveRets);
+    // Some functions may not allow erasing arguments or results. These calls
+    // return failure in such cases without modifying the function, so it's okay
+    // to proceed.
+    (void)f.funcOp.eraseArguments(f.nonLiveArgs);
+    (void)f.funcOp.eraseResults(f.nonLiveRets);
   }
 
   // 4. Operands
diff --git a/mlir/test/IR/test-func-erase-result.mlir b/mlir/test/IR/test-func-erase-result.mlir
index 8fe40b2fdfcbc..a32866227547b 100644
--- a/mlir/test/IR/test-func-erase-result.mlir
+++ b/mlir/test/IR/test-func-erase-result.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-func-erase-result -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -test-func-erase-result -split-input-file -verify-diagnostics | FileCheck %s
 
 // CHECK: func private @f(){{$}}
 // CHECK-NOT: attributes{{.*}}result
@@ -66,3 +66,8 @@ func.func private @f() -> (
   f32 {test.erase_this_result},
   tensor<3xf32>
 )
+
+// -----
+
+// expected-error @below {{failed to erase results}}
+llvm.func @llvm_func(!llvm.ptr, i64)
diff --git a/mlir/test/lib/IR/TestFunc.cpp b/mlir/test/lib/IR/TestFunc.cpp
index 2ade47249863c..94a4610365863 100644
--- a/mlir/test/lib/IR/TestFunc.cpp
+++ b/mlir/test/lib/IR/TestFunc.cpp
@@ -45,8 +45,12 @@ struct TestFuncInsertArg
                                    : unknownLoc);
       }
       func->removeAttr("test.insert_args");
-      func.insertArguments(indicesToInsert, typesToInsert, attrsToInsert,
-                           locsToInsert);
+      if (succeeded(func.insertArguments(indicesToInsert, typesToInsert,
+                                         attrsToInsert, locsToInsert)))
+        continue;
+
+      emitError(func->getLoc()) << "failed to insert arguments";
+      return signalPassFailure();
     }
   }
 };
@@ -79,7 +83,12 @@ struct TestFuncInsertResult
                                     : DictionaryAttr::get(&getContext()));
       }
       func->removeAttr("test.insert_results");
-      func.insertResults(indicesToInsert, typesToInsert, attrsToInsert);
+      if (succeeded(func.insertResults(indicesToInsert, typesToInsert,
+                                       attrsToInsert)))
+        continue;
+
+      emitError(func->getLoc()) << "failed to insert results";
+      return signalPassFailure();
     }
   }
 };
@@ -100,7 +109,10 @@ struct TestFuncEraseArg
       for (auto argIndex : llvm::seq<int>(0, func.getNumArguments()))
         if (func.getArgAttr(argIndex, "test.erase_this_arg"))
           indicesToErase.set(argIndex);
-      func.eraseArguments(indicesToErase);
+      if (succeeded(func.eraseArguments(indicesToErase)))
+        continue;
+      emitError(func->getLoc()) << "failed to erase arguments";
+      return signalPassFailure();
     }
   }
 };
@@ -122,7 +134,10 @@ struct TestFuncEraseResult
       for (auto resultIndex : llvm::seq<int>(0, func.getNumResults()))
         if (func.getResultAttr(resultIndex, "test.erase_this_result"))
           indicesToErase.set(resultIndex);
-      func.eraseResults(indicesToErase);
+      if (succeeded(func.eraseResults(indicesToErase)))
+        continue;
+      emitError(func->getLoc()) << "failed to erase results";
+      return signalPassFailure();
     }
   }
 };

From b1b065f2bf5d626fee277d67b0014577f7e4e499 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Fri, 18 Apr 2025 22:14:06 +0800
Subject: [PATCH 411/710] [ValueTracking] Refactor `isKnownNonEqualFromContext`
 (#127388)

This patch avoids adding RHS for comparisons with two variable operands
(https://github.com/llvm/llvm-project/pull/118493#discussion_r1949397482).
Instead, we iterate over related dominating conditions of both V1 and V2
in `isKnownNonEqualFromContext`, as suggested by goldsteinn
(https://github.com/llvm/llvm-project/pull/117442#discussion_r1944058002).

Compile-time improvement:
https://llvm-compile-time-tracker.com/compare.php?from=c6d95c441a29a45782ff72d6cb82839b86fd0e4a&to=88464baedd7b1731281eaa0ce4438122b4d218a7&stat=instructions:u
---
 llvm/lib/Analysis/ValueTracking.cpp          | 105 +++++++++++--------
 llvm/test/Transforms/InstCombine/icmp-dom.ll |  78 ++++++++++++++
 2 files changed, 139 insertions(+), 44 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index ff8f3b6ae3bc8..cdf7f052943c8 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -3804,6 +3804,63 @@ static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
           (StartOffset.sle(OffsetB) && StepOffset.isNegative()));
 }
 
+static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
+                                       unsigned Depth, const SimplifyQuery &Q) {
+  if (!Q.CxtI)
+    return false;
+
+  // Try to infer NonEqual based on information from dominating conditions.
+  if (Q.DC && Q.DT) {
+    auto IsKnownNonEqualFromDominatingCondition = [&](const Value *V) {
+      for (BranchInst *BI : Q.DC->conditionsFor(V)) {
+        Value *Cond = BI->getCondition();
+        BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
+        if (Q.DT->dominates(Edge0, Q.CxtI->getParent()) &&
+            isImpliedCondition(Cond, ICmpInst::ICMP_NE, V1, V2, Q.DL,
+                               /*LHSIsTrue=*/true, Depth)
+                .value_or(false))
+          return true;
+
+        BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
+        if (Q.DT->dominates(Edge1, Q.CxtI->getParent()) &&
+            isImpliedCondition(Cond, ICmpInst::ICMP_NE, V1, V2, Q.DL,
+                               /*LHSIsTrue=*/false, Depth)
+                .value_or(false))
+          return true;
+      }
+
+      return false;
+    };
+
+    if (IsKnownNonEqualFromDominatingCondition(V1) ||
+        IsKnownNonEqualFromDominatingCondition(V2))
+      return true;
+  }
+
+  if (!Q.AC)
+    return false;
+
+  // Try to infer NonEqual based on information from assumptions.
+  for (auto &AssumeVH : Q.AC->assumptionsFor(V1)) {
+    if (!AssumeVH)
+      continue;
+    CallInst *I = cast<CallInst>(AssumeVH);
+
+    assert(I->getFunction() == Q.CxtI->getFunction() &&
+           "Got assumption for the wrong function!");
+    assert(I->getIntrinsicID() == Intrinsic::assume &&
+           "must be an assume intrinsic");
+
+    if (isImpliedCondition(I->getArgOperand(0), ICmpInst::ICMP_NE, V1, V2, Q.DL,
+                           /*LHSIsTrue=*/true, Depth)
+            .value_or(false) &&
+        isValidAssumeForContext(I, Q.CxtI, Q.DT))
+      return true;
+  }
+
+  return false;
+}
+
 /// Return true if it is known that V1 != V2.
 static bool isKnownNonEqual(const Value *V1, const Value *V2,
                             const APInt &DemandedElts, unsigned Depth,
@@ -3875,49 +3932,8 @@ static bool isKnownNonEqual(const Value *V1, const Value *V2,
       match(V2, m_PtrToIntSameSize(Q.DL, m_Value(B))))
     return isKnownNonEqual(A, B, DemandedElts, Depth + 1, Q);
 
-  if (!Q.CxtI)
-    return false;
-
-  // Try to infer NonEqual based on information from dominating conditions.
-  if (Q.DC && Q.DT) {
-    for (BranchInst *BI : Q.DC->conditionsFor(V1)) {
-      Value *Cond = BI->getCondition();
-      BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
-      if (Q.DT->dominates(Edge0, Q.CxtI->getParent()) &&
-          isImpliedCondition(Cond, ICmpInst::ICMP_NE, V1, V2, Q.DL,
-                             /*LHSIsTrue=*/true, Depth)
-              .value_or(false))
-        return true;
-
-      BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
-      if (Q.DT->dominates(Edge1, Q.CxtI->getParent()) &&
-          isImpliedCondition(Cond, ICmpInst::ICMP_NE, V1, V2, Q.DL,
-                             /*LHSIsTrue=*/false, Depth)
-              .value_or(false))
-        return true;
-    }
-  }
-
-  if (!Q.AC)
-    return false;
-
-  // Try to infer NonEqual based on information from assumptions.
-  for (auto &AssumeVH : Q.AC->assumptionsFor(V1)) {
-    if (!AssumeVH)
-      continue;
-    CallInst *I = cast<CallInst>(AssumeVH);
-
-    assert(I->getFunction() == Q.CxtI->getFunction() &&
-           "Got assumption for the wrong function!");
-    assert(I->getIntrinsicID() == Intrinsic::assume &&
-           "must be an assume intrinsic");
-
-    if (isImpliedCondition(I->getArgOperand(0), ICmpInst::ICMP_NE, V1, V2, Q.DL,
-                           /*LHSIsTrue=*/true, Depth)
-            .value_or(false) &&
-        isValidAssumeForContext(I, Q.CxtI, Q.DT))
-      return true;
-  }
+  if (isKnownNonEqualFromContext(V1, V2, Depth, Q))
+    return true;
 
   return false;
 }
@@ -10348,7 +10364,8 @@ void llvm::findValuesAffectedByCondition(
       bool HasRHSC = match(B, m_ConstantInt());
       if (ICmpInst::isEquality(Pred)) {
         AddAffected(A);
-        AddAffected(B);
+        if (IsAssume)
+          AddAffected(B);
         if (HasRHSC) {
           Value *Y;
           // (X & C) or (X | C).
diff --git a/llvm/test/Transforms/InstCombine/icmp-dom.ll b/llvm/test/Transforms/InstCombine/icmp-dom.ll
index 07793e8951de5..6613bbeb8d6ae 100644
--- a/llvm/test/Transforms/InstCombine/icmp-dom.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-dom.ll
@@ -733,4 +733,82 @@ entry:
   ret i1 %cmp
 }
 
+; TODO: We can prove `%cond2` is always false
+define void @test_nonequal_domcond_loop1(i32 %x0, i1 %x1) {
+; CHECK-LABEL: @test_nonequal_domcond_loop1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X0:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    br i1 [[X1:%.*]], label [[IF_THEN:%.*]], label [[LOOP_HEADER]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X0]], 1
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i32 [[AND]], [[PHI]]
+; CHECK-NEXT:    br i1 [[COND1]], label [[IF_THEN2:%.*]], label [[LATCH]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       indirectbb:
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[PHI]], 31
+; CHECK-NEXT:    br i1 [[COND2]], label [[EXIT:%.*]], label [[LATCH]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %phi = phi i32 [ 0, %entry ], [ %x0, %latch ]
+  br label %latch
+
+latch:
+  br i1 %x1, label %if.then, label %loop.header
+
+if.then:
+  %and = and i32 %x0, 1
+  %cond1 = icmp eq i32 %and, %phi
+  br i1 %cond1, label %if.then2, label %latch
+
+if.then2:
+  br label %indirectbb
+
+indirectbb:
+  %cond2 = icmp eq i32 %phi, 31
+  br i1 %cond2, label %exit, label %latch
+
+exit:
+  ret void
+}
+
+define void @test_nonequal_domcond_loop2(ptr %p) {
+; CHECK-LABEL: @test_nonequal_domcond_loop2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load volatile i8, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load volatile i8, ptr [[P]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[LOAD2]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 [[LOAD2]], [[LOAD1]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[CMP1]], i1 true, i1 [[CMP2]]
+; CHECK-NEXT:    br i1 [[OR]], label [[WHILE_COND]], label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 false, label [[WHILE_COND]], label [[FOR_BODY]]
+;
+entry:
+  %load1 = load volatile i8, ptr %p, align 1
+  br label %while.cond
+
+while.cond:
+  %load2 = load volatile i8, ptr %p, align 1
+  %cmp1 = icmp eq i8 %load2, 0
+  %cmp2 = icmp uge i8 %load2, %load1
+  %or = select i1 %cmp1, i1 true, i1 %cmp2
+  br i1 %or, label %while.cond, label %for.body
+
+for.body:
+  %cond = icmp eq i8 %load1, %load2
+  br i1 %cond, label %while.cond, label %for.body
+}
+
 declare void @side_effect()

From dda4b968e77e1bb2c319bf2d523de3b5c4ccbb23 Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin@gmail.com>
Date: Fri, 18 Apr 2025 16:19:04 +0200
Subject: [PATCH 412/710] [mlir] AMDGPUToROCDL: lower `amdgpu.swizzle_bitmode`
 (#136223)

Repack `amdgpu.swizzle_bitmode` arguments and lower it to
`rocdl.ds_swizzle`.

Repacking logic is follows:
* `sizeof(arg) < sizeof(i32)`: bitcast to integer and zext to i32 and
then trunc and bitcast back.
* `sizeof(arg) == sizeof(i32)`: just bitcast to i32 and back if not i32
* `sizeof(arg) > sizeof(i32)`: bitcast to `vector<Nxi32>`, extract
individual elements and do a series of `rocdl.ds_swizzle` and then
compose vector and bitcast back.

Added repacking logic to LLVM utils so it can be used elsewhere. I'm
planning to use it for `gpu.shuffle` later.
---
 .../mlir/Conversion/LLVMCommon/Pattern.h      | 12 +++
 mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td |  2 +-
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           | 34 +++++++
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    | 93 +++++++++++++++++++
 .../Conversion/AMDGPUToROCDL/swizzle.mlir     | 75 +++++++++++++++
 mlir/test/Dialect/AMDGPU/invalid.mlir         | 10 +-
 6 files changed, 224 insertions(+), 2 deletions(-)
 create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/swizzle.mlir

diff --git a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
index 66c8731ec2bf4..7a58e4fc2f984 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
@@ -31,6 +31,18 @@ LogicalResult oneToOneRewrite(
     IntegerOverflowFlags overflowFlags = IntegerOverflowFlags::none);
 
 } // namespace detail
+
+/// Decomposes a `src` value into a set of values of type `dstType` through
+/// series of bitcasts and vector ops. Src and dst types are expected to be int
+/// or float types or vector types of them.
+SmallVector<Value> decomposeValue(OpBuilder &builder, Location loc, Value src,
+                                  Type dstType);
+
+/// Composes a set of `src` values into a single value of type `dstType` through
+/// series of bitcasts and vector ops. Inversely to `decomposeValue`, this
+/// function is used to combine multiple values into a single value.
+Value composeValue(OpBuilder &builder, Location loc, ValueRange src,
+                   Type dstType);
 } // namespace LLVM
 
 /// Base class for operation conversions targeting the LLVM IR dialect. It
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index a92ebf6d8e108..f14aa5a2e1564 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -38,7 +38,7 @@ def AMDGPU_Dialect : Dialect {
 def AnyIntegerOrFloat : AnyTypeOf<[AnySignlessInteger, AnyFloat], "Integer or Float">;
 
 def AnyIntegerOrFloatOr1DVector :
-  AnyTypeOf<[AnyIntegerOrFloat, VectorOfRankAndType<[1], [AnyIntegerOrFloat]>]>;
+  AnyTypeOf<[AnyIntegerOrFloat, FixedVectorOfRankAndType<[1], [AnyIntegerOrFloat]>]>;
 
 //===----------------------------------------------------------------------===//
 // AMDGPU general attribute definitions
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 5f697bdeef566..91dbc2de65c4e 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -1377,6 +1377,39 @@ struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
   }
 };
 
+struct AMDGPUSwizzleBitModeLowering
+    : public ConvertOpToLLVMPattern<SwizzleBitModeOp> {
+  using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(SwizzleBitModeOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Location loc = op.getLoc();
+    Type i32 = rewriter.getI32Type();
+    Value src = adaptor.getSrc();
+    SmallVector<Value> decomposed =
+        LLVM::decomposeValue(rewriter, loc, src, i32);
+    unsigned andMask = op.getAndMask();
+    unsigned orMask = op.getOrMask();
+    unsigned xorMask = op.getXorMask();
+
+    // bit 15 is 0 for the BitMode swizzle.
+    // https://gpuopen.com/learn/amd-gcn-assembly-cross-lane-operations/
+    unsigned mask = andMask | (orMask << 5) | (xorMask << 10);
+    Value maskValue = createI32Constant(rewriter, loc, mask);
+    SmallVector<Value> swizzled;
+    for (Value v : decomposed) {
+      Value res =
+          rewriter.create<ROCDL::DsSwizzleOp>(loc, v.getType(), v, maskValue);
+      swizzled.emplace_back(res);
+    }
+
+    Value result = LLVM::composeValue(rewriter, loc, swizzled, src.getType());
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
 struct ConvertAMDGPUToROCDLPass
     : public impl::ConvertAMDGPUToROCDLPassBase<ConvertAMDGPUToROCDLPass> {
   using Base::Base;
@@ -1444,4 +1477,5 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
            MFMAOpLowering, WMMAOpLowering, ExtPackedFp8OpLowering,
            PackedTrunc2xFp8OpLowering, PackedStochRoundFp8OpLowering,
            GatherToLDSOpLowering>(converter, chipset);
+  patterns.add<AMDGPUSwizzleBitModeLowering>(converter);
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index 32bfd72475569..1ae99561e9d1b 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -381,3 +381,96 @@ LogicalResult LLVM::detail::oneToOneRewrite(
   rewriter.replaceOp(op, results);
   return success();
 }
+
+static unsigned getBitWidth(Type type) {
+  if (type.isIntOrFloat())
+    return type.getIntOrFloatBitWidth();
+
+  auto vec = cast<VectorType>(type);
+  assert(!vec.isScalable() && "scalable vectors are not supported");
+  return vec.getNumElements() * getBitWidth(vec.getElementType());
+}
+
+static Value createI32Constant(OpBuilder &builder, Location loc,
+                               int32_t value) {
+  Type i32 = builder.getI32Type();
+  return builder.create<LLVM::ConstantOp>(loc, i32, value);
+}
+
+SmallVector<Value> mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
+                                              Value src, Type dstType) {
+  Type srcType = src.getType();
+  if (srcType == dstType)
+    return {src};
+
+  unsigned srcBitWidth = getBitWidth(srcType);
+  unsigned dstBitWidth = getBitWidth(dstType);
+  if (srcBitWidth == dstBitWidth) {
+    Value cast = builder.create<LLVM::BitcastOp>(loc, dstType, src);
+    return {cast};
+  }
+
+  if (dstBitWidth > srcBitWidth) {
+    auto smallerInt = builder.getIntegerType(srcBitWidth);
+    if (srcType != smallerInt)
+      src = builder.create<LLVM::BitcastOp>(loc, smallerInt, src);
+
+    auto largerInt = builder.getIntegerType(dstBitWidth);
+    Value res = builder.create<LLVM::ZExtOp>(loc, largerInt, src);
+    return {res};
+  }
+  assert(srcBitWidth % dstBitWidth == 0 &&
+         "src bit width must be a multiple of dst bit width");
+  int64_t numElements = srcBitWidth / dstBitWidth;
+  auto vecType = VectorType::get(numElements, dstType);
+
+  src = builder.create<LLVM::BitcastOp>(loc, vecType, src);
+
+  SmallVector<Value> res;
+  for (auto i : llvm::seq(numElements)) {
+    Value idx = createI32Constant(builder, loc, i);
+    Value elem = builder.create<LLVM::ExtractElementOp>(loc, src, idx);
+    res.emplace_back(elem);
+  }
+
+  return res;
+}
+
+Value mlir::LLVM::composeValue(OpBuilder &builder, Location loc, ValueRange src,
+                               Type dstType) {
+  assert(!src.empty() && "src range must not be empty");
+  if (src.size() == 1) {
+    Value res = src.front();
+    if (res.getType() == dstType)
+      return res;
+
+    unsigned srcBitWidth = getBitWidth(res.getType());
+    unsigned dstBitWidth = getBitWidth(dstType);
+    if (dstBitWidth < srcBitWidth) {
+      auto largerInt = builder.getIntegerType(srcBitWidth);
+      if (res.getType() != largerInt)
+        res = builder.create<LLVM::BitcastOp>(loc, largerInt, res);
+
+      auto smallerInt = builder.getIntegerType(dstBitWidth);
+      res = builder.create<LLVM::TruncOp>(loc, smallerInt, res);
+    }
+
+    if (res.getType() != dstType)
+      res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+
+    return res;
+  }
+
+  int64_t numElements = src.size();
+  auto srcType = VectorType::get(numElements, src.front().getType());
+  Value res = builder.create<LLVM::PoisonOp>(loc, srcType);
+  for (auto &&[i, elem] : llvm::enumerate(src)) {
+    Value idx = createI32Constant(builder, loc, i);
+    res = builder.create<LLVM::InsertElementOp>(loc, srcType, res, elem, idx);
+  }
+
+  if (res.getType() != dstType)
+    res = builder.create<LLVM::BitcastOp>(loc, dstType, res);
+
+  return res;
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/swizzle.mlir b/mlir/test/Conversion/AMDGPUToROCDL/swizzle.mlir
new file mode 100644
index 0000000000000..ef439efde1bd0
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/swizzle.mlir
@@ -0,0 +1,75 @@
+// RUN: mlir-opt -convert-amdgpu-to-rocdl --canonicalize %s | FileCheck %s
+
+// CHECK-LABEL: func @test_swizzle_i32
+// CHECK-SAME: (%[[ARG0:.*]]: i32)
+func.func @test_swizzle_i32(%arg0 : i32) -> i32 {
+// CHECK:  %[[C:.*]] = llvm.mlir.constant(4161 : i32) : i32
+// CHECK:  %[[RES:.*]] = rocdl.ds_swizzle %[[ARG0]], %[[C]] : (i32, i32) -> i32
+// CHECK:  return %[[RES]] : i32
+  %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : i32
+  return %0 : i32
+}
+
+// CHECK-LABEL: func @test_swizzle_f32
+// CHECK-SAME: (%[[ARG0:.*]]: f32)
+func.func @test_swizzle_f32(%arg0 : f32) -> f32 {
+// CHECK:  %[[C:.*]] = llvm.mlir.constant(4161 : i32) : i32
+// CHECK:  %[[CAST:.*]] = llvm.bitcast %[[ARG0]] : f32 to i32
+// CHECK:  %[[RES:.*]] = rocdl.ds_swizzle %[[CAST]], %[[C]] : (i32, i32) -> i32
+// CHECK:  %[[RES_CAST:.*]] = llvm.bitcast %[[RES]] : i32 to f32
+// CHECK:  return %[[RES_CAST]] : f32
+  %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : f32
+  return %0 : f32
+}
+
+// CHECK-LABEL: func @test_swizzle_f16
+// CHECK-SAME: (%[[ARG0:.*]]: f16)
+func.func @test_swizzle_f16(%arg0 : f16) -> f16 {
+// CHECK:  %[[C:.*]] = llvm.mlir.constant(4161 : i32) : i32
+// CHECK:  %[[CAST:.*]] = llvm.bitcast %[[ARG0]] : f16 to i16
+// CHECK:  %[[ZEXT:.*]] = llvm.zext %[[CAST]] : i16 to i32
+// CHECK:  %[[RES:.*]] = rocdl.ds_swizzle %[[ZEXT]], %[[C]] : (i32, i32) -> i32
+// CHECK:  %[[TRUNC:.*]] = llvm.trunc %[[RES]] : i32 to i16
+// CHECK:  %[[RES_CAST:.*]] = llvm.bitcast %[[TRUNC]] : i16 to f16
+// CHECK:  return %[[RES_CAST]] : f16
+  %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : f16
+  return %0 : f16
+}
+
+// CHECK-LABEL: func @test_swizzle_2xi32
+// CHECK-SAME: (%[[ARG0:.*]]: vector<2xi32>)
+func.func @test_swizzle_2xi32(%arg0 : vector<2xi32>) -> vector<2xi32> {
+// CHECK-DAG:  %[[V1:.*]] = llvm.mlir.poison : vector<2xi32>
+// CHECK-DAG:  %[[C:.*]] = llvm.mlir.constant(4161 : i32) : i32
+// CHECK-DAG:  %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:  %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:  %[[E0:.*]] = llvm.extractelement %[[ARG0]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:  %[[E1:.*]] = llvm.extractelement %[[ARG0]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:  %[[S1:.*]] = rocdl.ds_swizzle %[[E0]], %[[C]] : (i32, i32) -> i32
+// CHECK:  %[[S2:.*]] = rocdl.ds_swizzle %[[E1]], %[[C]] : (i32, i32) -> i32
+// CHECK:  %[[V2:.*]] = llvm.insertelement %[[S1]], %[[V1]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:  %[[V3:.*]] = llvm.insertelement %[[S2]], %[[V2]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:  return %[[V3]] : vector<2xi32>
+  %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : vector<2xi32>
+  return %0 : vector<2xi32>
+}
+
+// CHECK-LABEL: func @test_swizzle_4xf16
+// CHECK-SAME: (%[[ARG0:.*]]: vector<4xf16>)
+func.func @test_swizzle_4xf16(%arg0 : vector<4xf16>) -> vector<4xf16> {
+// CHECK-DAG:  %[[V1:.*]] = llvm.mlir.poison : vector<2xi32>
+// CHECK-DAG:  %[[C:.*]] = llvm.mlir.constant(4161 : i32) : i32
+// CHECK-DAG:  %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:  %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:  %[[CAST1:.*]] = llvm.bitcast %[[ARG0]] : vector<4xf16> to vector<2xi32>
+// CHECK:  %[[E0:.*]] = llvm.extractelement %[[CAST1]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:  %[[E1:.*]] = llvm.extractelement %[[CAST1]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:  %[[S1:.*]] = rocdl.ds_swizzle %[[E0]], %[[C]] : (i32, i32) -> i32
+// CHECK:  %[[S2:.*]] = rocdl.ds_swizzle %[[E1]], %[[C]] : (i32, i32) -> i32
+// CHECK:  %[[V2:.*]] = llvm.insertelement %[[S1]], %[[V1]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:  %[[V3:.*]] = llvm.insertelement %[[S2]], %[[V2]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:  %[[CAST2:.*]] = llvm.bitcast %[[V3]] : vector<2xi32> to vector<4xf16>
+// CHECK:  return %[[CAST2]] : vector<4xf16>
+  %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : vector<4xf16>
+  return %0 : vector<4xf16>
+}
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index 40f98ff85688c..73306ba6b3f93 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -154,7 +154,15 @@ func.func @fat_raw_buffer_cast_stripping_offset_affine_map(%m: memref<8xi32, aff
 // -----
 
 func.func @swizzle_invalid_type(%arg0 : si32) -> si32 {
-  // expected-error@+1 {{amdgpu.swizzle_bitmode' op operand #0 must be Integer or Float or vector of Integer or Float values of ranks 1}}
+  // expected-error@+1 {{'amdgpu.swizzle_bitmode' op operand #0 must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1}}
   %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : si32
   func.return %0 : si32
 }
+
+// -----
+
+func.func @swizzle_scalable_vec(%arg0 : vector<[4]xf32>) -> vector<[4]xf32> {
+  // expected-error@+1 {{'amdgpu.swizzle_bitmode' op operand #0 must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1}}
+  %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : vector<[4]xf32>
+  func.return %0 : vector<[4]xf32>
+}

From f2ecd86e34ed5323f2a8ec2259f11e9f5e9bb078 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Fri, 18 Apr 2025 07:46:31 -0700
Subject: [PATCH 413/710] [Analysis] Remove implicit LocationSize conversion
 from uint64_t (#133342)

This change removes the uint64_t constructor on LocationSize
preventing implicit conversion, and fixes up the using APIs to adapt to
the change. Note that I'm adding a couple of explicit conversion points
on routines where passing in a fixed offset as an integer seems likely
to have well understood semantics.

We had an unfortunate case which arose if you tried to pass a TypeSize
value to a parameter of LocationSize type. We'd find the implicit
conversion path through TypeSize -> uint64_t -> LocationSize which works
just fine for fixed values, but looses information and fails assertions
if the TypeSize was scalable. This change breaks the first link in that
implicit conversion chain since that seemed to be the easier one.
---
 llvm/include/llvm/Analysis/MemoryLocation.h   | 44 ++++++++++---------
 llvm/include/llvm/CodeGen/MachineFunction.h   | 19 ++++++++
 llvm/include/llvm/CodeGen/SelectionDAG.h      |  6 ++-
 llvm/lib/CodeGen/MachineScheduler.cpp         |  2 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  6 +--
 llvm/lib/CodeGen/TargetInstrInfo.cpp          |  2 +-
 .../lib/Target/AMDGPU/SIInsertHardClauses.cpp |  2 +-
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        | 20 +++++----
 llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp  |  6 +--
 llvm/lib/Target/Hexagon/HexagonSubtarget.cpp  |  4 +-
 llvm/lib/Target/Lanai/LanaiInstrInfo.cpp      |  9 ++--
 llvm/lib/Target/PowerPC/PPCInstrInfo.cpp      |  6 ++-
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  6 +--
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |  3 +-
 llvm/lib/Target/X86/X86InstrInfo.cpp          |  4 +-
 .../Target/RISCV/RISCVInstrInfoTest.cpp       |  2 +-
 16 files changed, 85 insertions(+), 56 deletions(-)

diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index ea29e21bd18f2..c046e0e380a36 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -80,11 +80,7 @@ class LocationSize {
 
   uint64_t Value;
 
-  // Hack to support implicit construction. This should disappear when the
-  // public LocationSize ctor goes away.
-  enum DirectConstruction { Direct };
-
-  constexpr LocationSize(uint64_t Raw, DirectConstruction) : Value(Raw) {}
+  constexpr LocationSize(uint64_t Raw) : Value(Raw) {}
   constexpr LocationSize(uint64_t Raw, bool Scalable)
       : Value(Raw > MaxValue ? AfterPointer
                              : Raw | (Scalable ? ScalableBit : uint64_t(0))) {}
@@ -96,14 +92,6 @@ class LocationSize {
   static_assert(~(MaxValue & ScalableBit), "Max value don't have bit 62 set");
 
 public:
-  // FIXME: Migrate all users to construct via either `precise` or `upperBound`,
-  // to make it more obvious at the callsite the kind of size that they're
-  // providing.
-  //
-  // Since the overwhelming majority of users of this provide precise values,
-  // this assumes the provided value is precise.
-  constexpr LocationSize(uint64_t Raw)
-      : Value(Raw > MaxValue ? AfterPointer : Raw) {}
   // Create non-scalable LocationSize
   static LocationSize precise(uint64_t Value) {
     return LocationSize(Value, false /*Scalable*/);
@@ -118,7 +106,7 @@ class LocationSize {
       return precise(0);
     if (LLVM_UNLIKELY(Value > MaxValue))
       return afterPointer();
-    return LocationSize(Value | ImpreciseBit, Direct);
+    return LocationSize(Value | ImpreciseBit);
   }
   static LocationSize upperBound(TypeSize Value) {
     if (Value.isScalable())
@@ -129,21 +117,21 @@ class LocationSize {
   /// Any location after the base pointer (but still within the underlying
   /// object).
   constexpr static LocationSize afterPointer() {
-    return LocationSize(AfterPointer, Direct);
+    return LocationSize(AfterPointer);
   }
 
   /// Any location before or after the base pointer (but still within the
   /// underlying object).
   constexpr static LocationSize beforeOrAfterPointer() {
-    return LocationSize(BeforeOrAfterPointer, Direct);
+    return LocationSize(BeforeOrAfterPointer);
   }
 
   // Sentinel values, generally used for maps.
   constexpr static LocationSize mapTombstone() {
-    return LocationSize(MapTombstone, Direct);
+    return LocationSize(MapTombstone);
   }
   constexpr static LocationSize mapEmpty() {
-    return LocationSize(MapEmpty, Direct);
+    return LocationSize(MapEmpty);
   }
 
   // Returns a LocationSize that can correctly represent either `*this` or
@@ -189,14 +177,16 @@ class LocationSize {
   bool operator==(const LocationSize &Other) const {
     return Value == Other.Value;
   }
-
   bool operator==(const TypeSize &Other) const {
-    return hasValue() && getValue() == Other;
+    return (*this == LocationSize::precise(Other));
+  }
+  bool operator==(uint64_t Other) const {
+    return (*this == LocationSize::precise(Other));
   }
 
   bool operator!=(const LocationSize &Other) const { return !(*this == Other); }
-
   bool operator!=(const TypeSize &Other) const { return !(*this == Other); }
+  bool operator!=(uint64_t Other) const { return !(*this == Other); }
 
   // Ordering operators are not provided, since it's unclear if there's only one
   // reasonable way to compare:
@@ -301,6 +291,12 @@ class MemoryLocation {
   explicit MemoryLocation(const Value *Ptr, LocationSize Size,
                           const AAMDNodes &AATags = AAMDNodes())
       : Ptr(Ptr), Size(Size), AATags(AATags) {}
+  explicit MemoryLocation(const Value *Ptr, TypeSize Size,
+                          const AAMDNodes &AATags = AAMDNodes())
+      : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
+  explicit MemoryLocation(const Value *Ptr, uint64_t Size,
+                          const AAMDNodes &AATags = AAMDNodes())
+      : Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
 
   MemoryLocation getWithNewPtr(const Value *NewPtr) const {
     MemoryLocation Copy(*this);
@@ -313,6 +309,12 @@ class MemoryLocation {
     Copy.Size = NewSize;
     return Copy;
   }
+  MemoryLocation getWithNewSize(uint64_t NewSize) const {
+    return getWithNewSize(LocationSize::precise(NewSize));
+  }
+  MemoryLocation getWithNewSize(TypeSize NewSize) const {
+    return getWithNewSize(LocationSize::precise(NewSize));
+  }
 
   MemoryLocation getWithoutAATags() const {
     MemoryLocation Copy(*this);
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 429dd54de33c2..30d414f0829e5 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1072,6 +1072,16 @@ class LLVM_ABI MachineFunction {
       const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
       AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
       AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+  MachineMemOperand *getMachineMemOperand(
+      MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t Size,
+      Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
+      const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+      AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+      AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
+    return getMachineMemOperand(PtrInfo, F, LocationSize::precise(Size),
+                                BaseAlignment, AAInfo, Ranges, SSID, Ordering,
+                                FailureOrdering);
+  }
   MachineMemOperand *getMachineMemOperand(
       MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize Size,
       Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1098,6 +1108,10 @@ class LLVM_ABI MachineFunction {
             ? LLT::scalable_vector(1, 8 * Size.getValue().getKnownMinValue())
             : LLT::scalar(8 * Size.getValue().getKnownMinValue()));
   }
+  MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+                                          int64_t Offset, uint64_t Size) {
+    return getMachineMemOperand(MMO, Offset, LocationSize::precise(Size));
+  }
   MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
                                           int64_t Offset, TypeSize Size) {
     return getMachineMemOperand(MMO, Offset, LocationSize::precise(Size));
@@ -1113,6 +1127,11 @@ class LLVM_ABI MachineFunction {
   MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
                                           const MachinePointerInfo &PtrInfo,
                                           LLT Ty);
+  MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
+                                          const MachinePointerInfo &PtrInfo,
+                                          uint64_t Size) {
+    return getMachineMemOperand(MMO, PtrInfo, LocationSize::precise(Size));
+  }
   MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
                                           const MachinePointerInfo &PtrInfo,
                                           TypeSize Size) {
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 2ab6b4d3027e9..eefee663c5f73 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1345,7 +1345,8 @@ class SelectionDAG {
       EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
       MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
                                        MachineMemOperand::MOStore,
-      LocationSize Size = 0, const AAMDNodes &AAInfo = AAMDNodes());
+      LocationSize Size = LocationSize::precise(0),
+      const AAMDNodes &AAInfo = AAMDNodes());
 
   inline SDValue getMemIntrinsicNode(
       unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
@@ -1353,7 +1354,8 @@ class SelectionDAG {
       MaybeAlign Alignment = std::nullopt,
       MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
                                        MachineMemOperand::MOStore,
-      LocationSize Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
+      LocationSize Size = LocationSize::precise(0),
+      const AAMDNodes &AAInfo = AAMDNodes()) {
     // Ensure that codegen never sees alignment 0
     return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo,
                                Alignment.value_or(getEVTAlign(MemVT)), Flags,
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 97f27277aface..0c3ffb1bbaa6f 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -2106,7 +2106,7 @@ void BaseMemOpClusterMutation::collectMemOpRecords(
     SmallVector<const MachineOperand *, 4> BaseOps;
     int64_t Offset;
     bool OffsetIsScalable;
-    LocationSize Width = 0;
+    LocationSize Width = LocationSize::precise(0);
     if (TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
                                            OffsetIsScalable, Width, TRI)) {
       if (!Width.hasValue())
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 8cae34d06c8ba..d7a67cca3c197 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5298,9 +5298,9 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
       MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
     else if (Info.fallbackAddressSpace)
       MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
-    Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
-                                     Info.memVT, MPI, Info.align, Info.flags,
-                                     Info.size, I.getAAMetadata());
+    Result = DAG.getMemIntrinsicNode(
+        Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT, MPI, Info.align,
+        Info.flags, LocationSize::precise(Info.size), I.getAAMetadata());
   } else if (!HasChain) {
     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
   } else if (!I.getType()->isVoidTy()) {
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 7e0a1e2a8a06e..6aaeed39bc81d 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -1716,7 +1716,7 @@ bool TargetInstrInfo::getMemOperandWithOffset(
     const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset,
     bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const {
   SmallVector<const MachineOperand *, 4> BaseOps;
-  LocationSize Width = 0;
+  LocationSize Width = LocationSize::precise(0);
   if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable,
                                      Width, TRI) ||
       BaseOps.size() != 1)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
index 71b937f23cc3c..88ff04d55629c 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
@@ -199,7 +199,7 @@ class SIInsertHardClauses {
 
         int64_t Dummy1;
         bool Dummy2;
-        LocationSize Dummy3 = 0;
+        LocationSize Dummy3 = LocationSize::precise(0);
         SmallVector<const MachineOperand *, 4> BaseOps;
         if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
           if (!SII->getMemOperandsWithOffsetWidth(MI, BaseOps, Dummy1, Dummy2,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 61fda0eef6314..ecfffa5067a39 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -382,7 +382,7 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
       DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
       if (DataOpIdx == -1)
         DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
-      Width = getOpSize(LdSt, DataOpIdx);
+      Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
     } else {
       // The 2 offset instructions use offset0 and offset1 instead. We can treat
       // these as a load with a single offset if the 2 offsets are consecutive.
@@ -418,11 +418,12 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
       DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
       if (DataOpIdx == -1) {
         DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
-        Width = getOpSize(LdSt, DataOpIdx);
+        Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
         DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
-        Width = Width.getValue() + getOpSize(LdSt, DataOpIdx);
+        Width = LocationSize::precise(
+            Width.getValue() + TypeSize::getFixed(getOpSize(LdSt, DataOpIdx)));
       } else {
-        Width = getOpSize(LdSt, DataOpIdx);
+        Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
       }
     }
     return true;
@@ -453,7 +454,7 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
       DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
     if (DataOpIdx == -1) // LDS DMA
       return false;
-    Width = getOpSize(LdSt, DataOpIdx);
+    Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
     return true;
   }
 
@@ -475,7 +476,7 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
     DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
     if (DataOpIdx == -1)
       return false; // no return sampler
-    Width = getOpSize(LdSt, DataOpIdx);
+    Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
     return true;
   }
 
@@ -490,7 +491,7 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
     DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);
     if (DataOpIdx == -1)
       return false;
-    Width = getOpSize(LdSt, DataOpIdx);
+    Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
     return true;
   }
 
@@ -509,7 +510,7 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
       DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
     if (DataOpIdx == -1) // LDS DMA
       return false;
-    Width = getOpSize(LdSt, DataOpIdx);
+    Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
     return true;
   }
 
@@ -3798,7 +3799,8 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
                                                const MachineInstr &MIb) const {
   SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
   int64_t Offset0, Offset1;
-  LocationSize Dummy0 = 0, Dummy1 = 0;
+  LocationSize Dummy0 = LocationSize::precise(0);
+  LocationSize Dummy1 = LocationSize::precise(0);
   bool Offset0IsScalable, Offset1IsScalable;
   if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,
                                      Dummy0, &RI) ||
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index b80cd2961f1be..64bc5ca134c86 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -3295,11 +3295,11 @@ HexagonInstrInfo::getBaseAndOffset(const MachineInstr &MI, int64_t &Offset,
                                    LocationSize &AccessSize) const {
   // Return if it is not a base+offset type instruction or a MemOp.
   if (getAddrMode(MI) != HexagonII::BaseImmOffset &&
-      getAddrMode(MI) != HexagonII::BaseLongOffset &&
-      !isMemOp(MI) && !isPostIncrement(MI))
+      getAddrMode(MI) != HexagonII::BaseLongOffset && !isMemOp(MI) &&
+      !isPostIncrement(MI))
     return nullptr;
 
-  AccessSize = getMemAccessSize(MI);
+  AccessSize = LocationSize::precise(getMemAccessSize(MI));
 
   unsigned BasePos = 0, OffsetPos = 0;
   if (!getBaseAndOffsetPosition(MI, BasePos, OffsetPos))
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
index 723a00208ccc0..ecc1b5d2ebe35 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -392,7 +392,7 @@ void HexagonSubtarget::BankConflictMutation::apply(ScheduleDAGInstrs *DAG) {
         HII.getAddrMode(L0) != HexagonII::BaseImmOffset)
       continue;
     int64_t Offset0;
-    LocationSize Size0 = 0;
+    LocationSize Size0 = LocationSize::precise(0);
     MachineOperand *BaseOp0 = HII.getBaseAndOffset(L0, Offset0, Size0);
     // Is the access size is longer than the L1 cache line, skip the check.
     if (BaseOp0 == nullptr || !BaseOp0->isReg() || !Size0.hasValue() ||
@@ -406,7 +406,7 @@ void HexagonSubtarget::BankConflictMutation::apply(ScheduleDAGInstrs *DAG) {
           HII.getAddrMode(L1) != HexagonII::BaseImmOffset)
         continue;
       int64_t Offset1;
-      LocationSize Size1 = 0;
+      LocationSize Size1 = LocationSize::precise(0);
       MachineOperand *BaseOp1 = HII.getBaseAndOffset(L1, Offset1, Size1);
       if (BaseOp1 == nullptr || !BaseOp1->isReg() || !Size0.hasValue() ||
           Size1.getValue() >= 32 || BaseOp0->getReg() != BaseOp1->getReg())
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
index 1aeedd531c4ac..4ca97da16cdeb 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
@@ -102,7 +102,8 @@ bool LanaiInstrInfo::areMemAccessesTriviallyDisjoint(
   const TargetRegisterInfo *TRI = &getRegisterInfo();
   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
   int64_t OffsetA = 0, OffsetB = 0;
-  LocationSize WidthA = 0, WidthB = 0;
+  LocationSize WidthA = LocationSize::precise(0),
+               WidthB = LocationSize::precise(0);
   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
     if (BaseOpA->isIdenticalTo(*BaseOpB)) {
@@ -769,17 +770,17 @@ bool LanaiInstrInfo::getMemOperandWithOffsetWidth(
   case Lanai::LDW_RR:
   case Lanai::SW_RR:
   case Lanai::SW_RI:
-    Width = 4;
+    Width = LocationSize::precise(4);
     break;
   case Lanai::LDHs_RI:
   case Lanai::LDHz_RI:
   case Lanai::STH_RI:
-    Width = 2;
+    Width = LocationSize::precise(2);
     break;
   case Lanai::LDBs_RI:
   case Lanai::LDBz_RI:
   case Lanai::STB_RI:
-    Width = 1;
+    Width = LocationSize::precise(1);
     break;
   }
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 97e9f59328f7e..4d4a3efd1098e 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -2943,7 +2943,8 @@ bool PPCInstrInfo::shouldClusterMemOps(
     return false;
 
   int64_t Offset1 = 0, Offset2 = 0;
-  LocationSize Width1 = 0, Width2 = 0;
+  LocationSize Width1 = LocationSize::precise(0),
+               Width2 = LocationSize::precise(0);
   const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
   if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
       !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
@@ -5798,7 +5799,8 @@ bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
   const TargetRegisterInfo *TRI = &getRegisterInfo();
   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
   int64_t OffsetA = 0, OffsetB = 0;
-  LocationSize WidthA = 0, WidthB = 0;
+  LocationSize WidthA = LocationSize::precise(0),
+               WidthB = LocationSize::precise(0);
   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
     if (BaseOpA->isIdenticalTo(*BaseOpB)) {
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 98c8bdb4bc114..ed723f38ea2c5 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -11582,7 +11582,7 @@ SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
   SDValue Chain = DAG.getMemIntrinsicNode(
       ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), StoreOps,
       ConcatVT.getVectorElementType(), PtrInfo, Alignment,
-      MachineMemOperand::MOStore, MemoryLocation::UnknownSize);
+      MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer());
 
   static const Intrinsic::ID VlsegIntrinsicsIds[] = {
       Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3, Intrinsic::riscv_vlseg4,
@@ -11604,7 +11604,7 @@ SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
   SDValue Load = DAG.getMemIntrinsicNode(
       ISD::INTRINSIC_W_CHAIN, DL, DAG.getVTList({VecTupTy, MVT::Other}),
       LoadOps, ConcatVT.getVectorElementType(), PtrInfo, Alignment,
-      MachineMemOperand::MOLoad, MemoryLocation::UnknownSize);
+      MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer());
 
   SmallVector<SDValue, 8> Res(Factor);
 
@@ -11721,7 +11721,7 @@ SDValue RISCVTargetLowering::lowerVECTOR_INTERLEAVE(SDValue Op,
     SDValue Chain = DAG.getMemIntrinsicNode(
         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
         VecVT.getVectorElementType(), PtrInfo, Alignment,
-        MachineMemOperand::MOStore, MemoryLocation::UnknownSize);
+        MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer());
 
     SmallVector<SDValue, 8> Loads(Factor);
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 04ad56ea83230..c4a2784263af0 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -3077,7 +3077,8 @@ bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
   int64_t OffsetA = 0, OffsetB = 0;
-  LocationSize WidthA = 0, WidthB = 0;
+  LocationSize WidthA = LocationSize::precise(0),
+               WidthB = LocationSize::precise(0);
   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
     if (BaseOpA->isIdenticalTo(*BaseOpB)) {
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index f595642d734e8..48688e4cf8f3c 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4748,8 +4748,8 @@ bool X86InstrInfo::getMemOperandsWithOffsetWidth(
   // FIXME: Relying on memoperands() may not be right thing to do here. Check
   // with X86 maintainers, and fix it accordingly. For now, it is ok, since
   // there is no use of `Width` for X86 back-end at the moment.
-  Width =
-      !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize() : 0;
+  Width = !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize()
+                                     : LocationSize::precise(0);
   BaseOps.push_back(BaseOp);
   return true;
 }
diff --git a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
index 19abac6301ae1..4881f193f34b5 100644
--- a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
+++ b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
@@ -207,7 +207,7 @@ TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) {
   DebugLoc DL;
 
   SmallVector<const MachineOperand *> BaseOps;
-  LocationSize Width = 0;
+  LocationSize Width = LocationSize::precise(0);
   int64_t Offset;
   bool OffsetIsScalable;
 

From d0dd6974b8850dbec0237596fedd58db4552f2ec Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Fri, 18 Apr 2025 11:08:41 -0400
Subject: [PATCH 414/710] [mlir][spirv] Switch to `llvm::interleaved`. NFC.
 (#136240)

Clean up printing code by switching to `llvm::interleaved` from
https://github.com/llvm/llvm-project/pull/135517.
---
 mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp  | 11 ++++----
 mlir/lib/Dialect/SPIRV/IR/SPIRVAttributes.cpp | 18 ++++++-------
 mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp        | 26 ++++++-------------
 3 files changed, 21 insertions(+), 34 deletions(-)

diff --git a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
index ed9a30086deca..577959bbdbeaa 100644
--- a/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/ControlFlowOps.cpp
@@ -15,6 +15,8 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 
+#include "llvm/Support/InterleavedRange.h"
+
 #include "SPIRVOpUtils.h"
 #include "SPIRVParsingUtils.h"
 
@@ -119,12 +121,9 @@ ParseResult BranchConditionalOp::parse(OpAsmParser &parser,
 void BranchConditionalOp::print(OpAsmPrinter &printer) {
   printer << ' ' << getCondition();
 
-  if (auto weights = getBranchWeights()) {
-    printer << " [";
-    llvm::interleaveComma(weights->getValue(), printer, [&](Attribute a) {
-      printer << llvm::cast<IntegerAttr>(a).getInt();
-    });
-    printer << "]";
+  if (std::optional<ArrayAttr> weights = getBranchWeights()) {
+    printer << ' '
+            << llvm::interleaved_array(weights->getAsValueRange<IntegerAttr>());
   }
 
   printer << ", ";
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVAttributes.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVAttributes.cpp
index b71be23fdf47d..2ba6106896c1f 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVAttributes.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVAttributes.cpp
@@ -12,6 +12,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
 using namespace mlir::spirv;
@@ -621,17 +622,14 @@ Attribute SPIRVDialect::parseAttribute(DialectAsmParser &parser,
 //===----------------------------------------------------------------------===//
 
 static void print(spirv::VerCapExtAttr triple, DialectAsmPrinter &printer) {
-  auto &os = printer.getStream();
   printer << spirv::VerCapExtAttr::getKindName() << "<"
-          << spirv::stringifyVersion(triple.getVersion()) << ", [";
-  llvm::interleaveComma(
-      triple.getCapabilities(), os,
-      [&](spirv::Capability cap) { os << spirv::stringifyCapability(cap); });
-  printer << "], [";
-  llvm::interleaveComma(triple.getExtensionsAttr(), os, [&](Attribute attr) {
-    os << llvm::cast<StringAttr>(attr).getValue();
-  });
-  printer << "]>";
+          << spirv::stringifyVersion(triple.getVersion()) << ", "
+          << llvm::interleaved_array(llvm::map_range(
+                 triple.getCapabilities(), spirv::stringifyCapability))
+          << ", "
+          << llvm::interleaved_array(
+                 triple.getExtensionsAttr().getAsValueRange<StringAttr>())
+          << ">";
 }
 
 static void print(spirv::TargetEnvAttr targetEnv, DialectAsmPrinter &printer) {
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 16e91b0cb2cfc..097a18f8a70eb 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -35,6 +35,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/InterleavedRange.h"
 #include <cassert>
 #include <numeric>
 #include <optional>
@@ -807,10 +808,8 @@ void spirv::EntryPointOp::print(OpAsmPrinter &printer) {
   printer << " \"" << stringifyExecutionModel(getExecutionModel()) << "\" ";
   printer.printSymbolName(getFn());
   auto interfaceVars = getInterface().getValue();
-  if (!interfaceVars.empty()) {
-    printer << ", ";
-    llvm::interleaveComma(interfaceVars, printer);
-  }
+  if (!interfaceVars.empty())
+    printer << ", " << llvm::interleaved(interfaceVars);
 }
 
 LogicalResult spirv::EntryPointOp::verify() {
@@ -862,13 +861,9 @@ void spirv::ExecutionModeOp::print(OpAsmPrinter &printer) {
   printer << " ";
   printer.printSymbolName(getFn());
   printer << " \"" << stringifyExecutionMode(getExecutionMode()) << "\"";
-  auto values = this->getValues();
-  if (values.empty())
-    return;
-  printer << ", ";
-  llvm::interleaveComma(values, printer, [&](Attribute a) {
-    printer << llvm::cast<IntegerAttr>(a).getInt();
-  });
+  ArrayAttr values = this->getValues();
+  if (!values.empty())
+    printer << ", " << llvm::interleaved(values.getAsValueRange<IntegerAttr>());
 }
 
 //===----------------------------------------------------------------------===//
@@ -1824,13 +1819,8 @@ ParseResult spirv::SpecConstantCompositeOp::parse(OpAsmParser &parser,
 void spirv::SpecConstantCompositeOp::print(OpAsmPrinter &printer) {
   printer << " ";
   printer.printSymbolName(getSymName());
-  printer << " (";
-  auto constituents = this->getConstituents().getValue();
-
-  if (!constituents.empty())
-    llvm::interleaveComma(constituents, printer);
-
-  printer << ") : " << getType();
+  printer << " (" << llvm::interleaved(this->getConstituents().getValue())
+          << ") : " << getType();
 }
 
 LogicalResult spirv::SpecConstantCompositeOp::verify() {

From 4be84a142e97d25faf7acdd2d279124d0af32b7e Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Fri, 18 Apr 2025 11:10:17 -0400
Subject: [PATCH 415/710] [mlir][gpu] Clean up prints in GPU dialect. NFC.
 (#136250)

Clean up printing code by switching to `llvm::interleaved` from
https://github.com/llvm/llvm-project/pull/135517. Also make some minor
readability & performance fixes.
---
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        | 18 ++++----
 .../GPU/TransformOps/GPUTransformOps.cpp      | 46 +++++++------------
 2 files changed, 26 insertions(+), 38 deletions(-)

diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 976432ea37120..9391d2c4ec840 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -35,6 +35,8 @@
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/StringSaver.h"
 #include <cassert>
 #include <numeric>
@@ -479,10 +481,11 @@ static void printAttributions(OpAsmPrinter &p, StringRef keyword,
   if (values.empty())
     return;
 
-  p << ' ' << keyword << '(';
-  llvm::interleaveComma(
-      values, p, [&p](BlockArgument v) { p << v << " : " << v.getType(); });
-  p << ')';
+  auto printBlockArg = [](BlockArgument v) {
+    return llvm::formatv("{} : {}", v, v.getType());
+  };
+  p << ' ' << keyword << '('
+    << llvm::interleaved(llvm::map_range(values, printBlockArg)) << ')';
 }
 
 /// Verifies a GPU function memory attribution.
@@ -1311,11 +1314,10 @@ static void printLaunchFuncOperands(OpAsmPrinter &printer, Operation *,
   if (operands.empty())
     return;
   printer << "args(";
-  llvm::interleaveComma(llvm::zip(operands, types), printer,
+  llvm::interleaveComma(llvm::zip_equal(operands, types), printer,
                         [&](const auto &pair) {
-                          printer.printOperand(std::get<0>(pair));
-                          printer << " : ";
-                          printer.printType(std::get<1>(pair));
+                          auto [operand, type] = pair;
+                          printer << operand << " : " << type;
                         });
   printer << ")";
 }
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index cce477370a539..3970539db6675 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -40,6 +40,7 @@
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/InterleavedRange.h"
 #include <type_traits>
 
 using namespace mlir;
@@ -450,20 +451,14 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
     // Otherwise, we have a new insertion without a size -> use size 1.
     tmpMappingSizes.push_back(1);
   }
-  LLVM_DEBUG(
-      llvm::interleaveComma(
-          tmpMappingSizes,
-          DBGS() << "----tmpMappingSizes extracted from scf.forall op: ");
-      llvm::dbgs() << "\n");
+  LDBG("----tmpMappingSizes extracted from scf.forall op: "
+       << llvm::interleaved(tmpMappingSizes));
 
   // Step 2. sort the values by the corresponding DeviceMappingAttrInterface.
   SmallVector<int64_t> forallMappingSizes = getValuesSortedByKey(
       forallMappingAttrs.getArrayRef(), tmpMappingSizes, comparator);
-  LLVM_DEBUG(llvm::interleaveComma(forallMappingSizes,
-                                   DBGS() << "----forallMappingSizes: ");
-             llvm::dbgs() << "\n"; llvm::interleaveComma(
-                 forallMappingAttrs, DBGS() << "----forallMappingAttrs: ");
-             llvm::dbgs() << "\n");
+  LDBG("----forallMappingSizes: " << llvm::interleaved(forallMappingSizes));
+  LDBG("----forallMappingAttrs: " << llvm::interleaved(forallMappingAttrs));
 
   // Step 3. Generate the mappingIdOps using the provided generator.
   Location loc = forallOp.getLoc();
@@ -501,17 +496,10 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
     SmallVector<int64_t> availableMappingSizes =
         builderResult.availableMappingSizes;
     SmallVector<Value> activeIdOps = builderResult.activeIdOps;
-    // clang-format off
-    LLVM_DEBUG(
-        llvm::interleaveComma(
-          activeMappingSizes, DBGS() << "----activeMappingSizes: ");
-        llvm::dbgs() << "\n";
-        llvm::interleaveComma(
-          availableMappingSizes, DBGS() << "----availableMappingSizes: ");
-        llvm::dbgs() << "\n";
-        llvm::interleaveComma(activeIdOps, DBGS() << "----activeIdOps: ");
-        llvm::dbgs() << "\n");
-    // clang-format on
+    LDBG("----activeMappingSizes: " << llvm::interleaved(activeMappingSizes));
+    LDBG("----availableMappingSizes: "
+         << llvm::interleaved(availableMappingSizes));
+    LDBG("----activeIdOps: " << llvm::interleaved(activeIdOps));
     for (auto [activeId, activeMappingSize, availableMappingSize] :
          llvm::zip_equal(activeIdOps, activeMappingSizes,
                          availableMappingSizes)) {
@@ -566,11 +554,9 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   // Step 8. Erase old op.
   rewriter.eraseOp(forallOp);
 
-  LLVM_DEBUG(llvm::interleaveComma(forallMappingSizes,
-                                   DBGS() << "----result forallMappingSizes: ");
-             llvm::dbgs() << "\n"; llvm::interleaveComma(
-                 mappingIdOps, DBGS() << "----result mappingIdOps: ");
-             llvm::dbgs() << "\n");
+  LDBG("----result forallMappingSizes: "
+       << llvm::interleaved(forallMappingSizes));
+  LDBG("----result mappingIdOps: " << llvm::interleaved(mappingIdOps));
 
   result = ForallRewriteResult{forallMappingSizes, mappingIdOps};
   return DiagnosedSilenceableFailure::success();
@@ -740,7 +726,7 @@ static DiagnosedSilenceableFailure checkMappingSpec(
     auto diag = definiteFailureHelper(
         transformOp, forallOp,
         Twine("3-D mapping: size of threadIdx.x must be a multiple of ") +
-            std::to_string(factor));
+            Twine(factor));
     return diag;
   }
   if (computeProduct(numParallelIterations) * factor >
@@ -749,9 +735,9 @@ static DiagnosedSilenceableFailure checkMappingSpec(
         transformOp, forallOp,
         Twine("the number of required parallel resources (blocks or "
               "threads) ") +
-            std::to_string(computeProduct(numParallelIterations) * factor) +
-            std::string(" overflows the number of available resources ") +
-            std::to_string(computeProduct(blockOrGridSizes)));
+            Twine(computeProduct(numParallelIterations) * factor) +
+            " overflows the number of available resources " +
+            Twine(computeProduct(blockOrGridSizes)));
     return diag;
   }
   return DiagnosedSilenceableFailure::success();

From c016a65c180c8703d5bdb2a277bb1629df5517b6 Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Fri, 18 Apr 2025 11:22:52 -0400
Subject: [PATCH 416/710] [mlir][vector] Switch to `llvm::interleaved` in debug
 prints. NFC. (#136248)

Clean up printing code by switching to `llvm::interleaved` from
https://github.com/llvm/llvm-project/pull/135517.
---
 mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp | 12 ++++--------
 mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp       |  6 ++----
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 04c38f9f7b2e3..cb348b28c876a 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -13,12 +13,11 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/VectorInterfaces.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Debug.h"
-#include <numeric>
+#include "llvm/Support/InterleavedRange.h"
 #include <optional>
 
 #define DEBUG_TYPE "vector-unroll"
@@ -28,7 +27,7 @@
 using namespace mlir;
 using namespace mlir::vector;
 
-/// Compute the indices of the slice `index` for a tranfer op.
+/// Compute the indices of the slice `index` for a transfer op.
 static SmallVector<Value> sliceTransferIndices(ArrayRef<int64_t> elementOffsets,
                                                ArrayRef<Value> indices,
                                                AffineMap permutationMap,
@@ -88,17 +87,14 @@ getTargetShape(const vector::UnrollVectorOptions &options, Operation *op) {
     LDBG("--could not get shape of op " << *op << " -> BAIL");
     return std::nullopt;
   }
-  LLVM_DEBUG(
-      llvm::interleaveComma(*maybeUnrollShape, DBGS() << "--vector op shape: ");
-      llvm::dbgs() << "\n";);
+  LDBG("--vector op shape: " << llvm::interleaved(*maybeUnrollShape));
 
   std::optional<SmallVector<int64_t>> targetShape = options.nativeShape(op);
   if (!targetShape) {
     LDBG("--no unrolling target shape defined " << *op << "-> SKIP");
     return std::nullopt;
   }
-  LLVM_DEBUG(llvm::interleaveComma(*targetShape, DBGS() << "--target shape: ");
-             llvm::dbgs() << "\n";);
+  LDBG("--target shape: " << llvm::interleaved(*targetShape));
 
   auto maybeShapeRatio = computeShapeRatio(*maybeUnrollShape, *targetShape);
   if (!maybeShapeRatio) {
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 024e8ccb901de..16a2732b53e97 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -27,7 +27,7 @@
 #include "mlir/Support/LLVM.h"
 
 #include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/SetVector.h"
+#include "llvm/Support/InterleavedRange.h"
 
 #define DEBUG_TYPE "vector-utils"
 
@@ -372,9 +372,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
 LogicalResult
 vector::isValidMaskedInputVector(ArrayRef<int64_t> shape,
                                  ArrayRef<int64_t> inputVectorSizes) {
-  LDBG("Iteration space static sizes:");
-  LLVM_DEBUG(llvm::interleaveComma(shape, llvm::dbgs()));
-  LLVM_DEBUG(llvm::dbgs() << "\n");
+  LDBG("Iteration space static sizes:" << llvm::interleaved(shape));
 
   if (inputVectorSizes.size() != shape.size()) {
     LDBG("Input vector sizes don't match the number of loops");

From 257b72758424f56103d38e3d016cfb6baa4da613 Mon Sep 17 00:00:00 2001
From: Nick Sarnie <nick.sarnie@intel.com>
Date: Fri, 18 Apr 2025 11:28:46 -0400
Subject: [PATCH 417/710] [clang][Sema][SYCL] Fix MSVC STL usage on AMDGPU
 (#135979)

The MSVC STL includes specializations of `_Is_memfunptr` for every
function pointer type, including every calling convention.

The problem is the AMDGPU target doesn't support the x86 `vectorcall`
calling convention so clang sets it to the default CC. This ends up
clashing with the already-existing overload for the default CC, so we
get a duplicate definition error when including `type_traits` (which we
heavily use in the SYCL STL) and compiling for AMDGPU on Windows.

This doesn't happen for pure AMDGPU non-SYCL because it doesn't include
the C++ STL, and it doesn't happen for CUDA/HIP because a similar
workaround was done
[here](https://github.com/llvm/llvm-project/commit/fa49c3a888e816969b5ed68cd5c47efc3eb9419f).

I am not an expert in Sema, so I did a kinda of hardcoded fix, please
let me know if there is a better way to fix this.

As far as I can tell we can't do exactly the same fix that was done for
CUDA because we can't differentiate between device and host code so
easily.

---------

Signed-off-by: Sarnie, Nick <nick.sarnie@intel.com>
---
 clang/lib/Sema/SemaDeclAttr.cpp           | 11 ++++++++++-
 clang/test/SemaSYCL/Inputs/vectorcall.hpp | 18 ++++++++++++++++++
 clang/test/SemaSYCL/sycl-cconv-win.cpp    |  5 +++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/SemaSYCL/Inputs/vectorcall.hpp
 create mode 100644 clang/test/SemaSYCL/sycl-cconv-win.cpp

diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 7dd20a8795fc9..cb230774d56fc 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -5495,12 +5495,12 @@ bool Sema::CheckCallingConvAttr(const ParsedAttr &Attrs, CallingConv &CC,
 
   TargetInfo::CallingConvCheckResult A = TargetInfo::CCCR_OK;
   const TargetInfo &TI = Context.getTargetInfo();
+  auto *Aux = Context.getAuxTargetInfo();
   // CUDA functions may have host and/or device attributes which indicate
   // their targeted execution environment, therefore the calling convention
   // of functions in CUDA should be checked against the target deduced based
   // on their host/device attributes.
   if (LangOpts.CUDA) {
-    auto *Aux = Context.getAuxTargetInfo();
     assert(FD || CFT != CUDAFunctionTarget::InvalidTarget);
     auto CudaTarget = FD ? CUDA().IdentifyTarget(FD) : CFT;
     bool CheckHost = false, CheckDevice = false;
@@ -5525,6 +5525,15 @@ bool Sema::CheckCallingConvAttr(const ParsedAttr &Attrs, CallingConv &CC,
       A = HostTI->checkCallingConvention(CC);
     if (A == TargetInfo::CCCR_OK && CheckDevice && DeviceTI)
       A = DeviceTI->checkCallingConvention(CC);
+  } else if (LangOpts.SYCLIsDevice && TI.getTriple().isAMDGPU() &&
+             CC == CC_X86VectorCall) {
+    // Assuming SYCL Device AMDGPU CC_X86VectorCall functions are always to be
+    // emitted on the host. The MSVC STL has CC-based specializations so we
+    // cannot change the CC to be the default as that will cause a clash with
+    // another specialization.
+    A = TI.checkCallingConvention(CC);
+    if (Aux && A != TargetInfo::CCCR_OK)
+      A = Aux->checkCallingConvention(CC);
   } else {
     A = TI.checkCallingConvention(CC);
   }
diff --git a/clang/test/SemaSYCL/Inputs/vectorcall.hpp b/clang/test/SemaSYCL/Inputs/vectorcall.hpp
new file mode 100644
index 0000000000000..566a48dd24c30
--- /dev/null
+++ b/clang/test/SemaSYCL/Inputs/vectorcall.hpp
@@ -0,0 +1,18 @@
+
+template <typename F> struct A{};
+
+template <typename Ret, typename C, typename... Args> struct A<Ret (             C::*)(Args...) noexcept> { static constexpr int value = 0; };
+template <typename Ret, typename C, typename... Args> struct A<Ret (__vectorcall C::*)(Args...) noexcept> { static constexpr int value = 1; };
+
+template <typename F> constexpr int A_v = A<F>::value;
+
+struct B
+{
+    void f() noexcept {}
+    void __vectorcall g() noexcept {}
+};
+
+int main()
+{
+    return A_v<decltype(&B::f)> + A_v<decltype(&B::g)>;
+}
diff --git a/clang/test/SemaSYCL/sycl-cconv-win.cpp b/clang/test/SemaSYCL/sycl-cconv-win.cpp
new file mode 100644
index 0000000000000..f0c22a2ed3921
--- /dev/null
+++ b/clang/test/SemaSYCL/sycl-cconv-win.cpp
@@ -0,0 +1,5 @@
+// RUN: %clang_cc1 -isystem %S/Inputs/ -fsycl-is-device -triple amdgcn-amd-hsa -aux-triple x86_64-pc-windows-msvc -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#include <vectorcall.hpp>

From be9f72cf3716bc1de5912a457284a1973112f9a0 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Fri, 18 Apr 2025 15:37:37 +0000
Subject: [PATCH 418/710] Revert "[ConstraintElim] Simplify cmp after
 uadd.sat/usub.sat (#135603)"

This reverts commit fe54d1afcca055f464840654dd2ec3fd83aea688.

Causes miscompiles, see #135603.
---
 .../Scalar/ConstraintElimination.cpp          | 17 +-------
 .../ConstraintElimination/uadd-usub-sat.ll    | 43 -------------------
 2 files changed, 1 insertion(+), 59 deletions(-)
 delete mode 100644 llvm/test/Transforms/ConstraintElimination/uadd-usub-sat.ll

diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 31e057baee2bf..2604bb9153f17 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1141,8 +1141,6 @@ void State::addInfoFor(BasicBlock &BB) {
         break;
       [[fallthrough]];
     case Intrinsic::abs:
-    case Intrinsic::uadd_sat:
-    case Intrinsic::usub_sat:
       WorkList.push_back(FactOrCheck::getInstFact(DT.getNode(&BB), &I));
       break;
     }
@@ -1893,26 +1891,13 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI,
         AddFact(CmpInst::ICMP_SGE, CB.Inst, X);
         continue;
       }
+
       if (auto *MinMax = dyn_cast<MinMaxIntrinsic>(CB.Inst)) {
         Pred = ICmpInst::getNonStrictPredicate(MinMax->getPredicate());
         AddFact(Pred, MinMax, MinMax->getLHS());
         AddFact(Pred, MinMax, MinMax->getRHS());
         continue;
       }
-      if (auto *USatI = dyn_cast<SaturatingInst>(CB.Inst)) {
-        switch (USatI->getIntrinsicID()) {
-        default:
-          llvm_unreachable("Unexpected intrinsic.");
-        case Intrinsic::uadd_sat:
-          AddFact(ICmpInst::ICMP_UGE, USatI, USatI->getLHS());
-          AddFact(ICmpInst::ICMP_UGE, USatI, USatI->getRHS());
-          break;
-        case Intrinsic::usub_sat:
-          AddFact(ICmpInst::ICMP_ULE, USatI, USatI->getLHS());
-          break;
-        }
-        continue;
-      }
     }
 
     Value *A = nullptr, *B = nullptr;
diff --git a/llvm/test/Transforms/ConstraintElimination/uadd-usub-sat.ll b/llvm/test/Transforms/ConstraintElimination/uadd-usub-sat.ll
deleted file mode 100644
index b0db89dcfdab8..0000000000000
--- a/llvm/test/Transforms/ConstraintElimination/uadd-usub-sat.ll
+++ /dev/null
@@ -1,43 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=constraint-elimination -S %s | FileCheck %s
-
-declare i64 @llvm.uadd.sat.i64(i64, i64)
-declare i64 @llvm.usub.sat.i64(i64, i64)
-
-define i1 @uadd_sat_uge(i64 %a, i64 %b) {
-; CHECK-LABEL: define i1 @uadd_sat_uge(
-; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[ADD_SAT:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A]], i64 [[B]])
-; CHECK-NEXT:    [[CMP:%.*]] = and i1 true, true
-; CHECK-NEXT:    ret i1 [[CMP]]
-;
-  %add.sat = call i64 @llvm.uadd.sat.i64(i64 %a, i64 %b)
-  %cmp1 = icmp uge i64 %add.sat, %a
-  %cmp2 = icmp uge i64 %add.sat, %b
-  %cmp = and i1 %cmp1, %cmp2
-  ret i1 %cmp
-}
-
-define i1 @usub_sat_ule_lhs(i64 %a, i64 %b) {
-; CHECK-LABEL: define i1 @usub_sat_ule_lhs(
-; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[SUB_SAT:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A]], i64 [[B]])
-; CHECK-NEXT:    ret i1 true
-;
-  %sub.sat = call i64 @llvm.usub.sat.i64(i64 %a, i64 %b)
-  %cmp = icmp ule i64 %sub.sat, %a
-  ret i1 %cmp
-}
-
-; Negative test
-define i1 @usub_sat_not_ule_rhs(i64 %a, i64 %b) {
-; CHECK-LABEL: define i1 @usub_sat_not_ule_rhs(
-; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[SUB_SAT:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A]], i64 [[B]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[SUB_SAT]], [[B]]
-; CHECK-NEXT:    ret i1 [[CMP]]
-;
-  %sub.sat = call i64 @llvm.usub.sat.i64(i64 %a, i64 %b)
-  %cmp = icmp ule i64 %sub.sat, %b
-  ret i1 %cmp
-}

From 915de1a5889c4dad1ec7b77ac5c41ebabc20a8ca Mon Sep 17 00:00:00 2001
From: Devon Loehr <DKLoehr@users.noreply.github.com>
Date: Fri, 18 Apr 2025 11:43:00 -0400
Subject: [PATCH 419/710] Generate empty .clang-format-ignore before running
 tests (#136154)

Followup to #136022, this ensures formatting tests are run with an empty
`.clang-format-ignore` in their root directory, to prevent failures if
the file also exists higher in the tree.
---
 clang/test/Format/lit.local.cfg | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/clang/test/Format/lit.local.cfg b/clang/test/Format/lit.local.cfg
index b060c79226cbd..20e217664997b 100644
--- a/clang/test/Format/lit.local.cfg
+++ b/clang/test/Format/lit.local.cfg
@@ -1,3 +1,4 @@
+import os
 import platform
 import lit.formats
 
@@ -27,3 +28,8 @@ config.suffixes = [
 # python implementation does, so use that for cross platform compatibility
 if platform.system() == "AIX":
     config.test_format = lit.formats.ShTest()
+
+# Create an empty .clang-format-ignore file so that tests don't get messed
+# up if one exists higher in the tree
+with open(".clang-format-ignore", 'w'):
+    pass

From 273aecdb2008a58593aa1baf175e5e6e2aec1df5 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Fri, 18 Apr 2025 08:45:52 -0700
Subject: [PATCH 420/710] [flang-rt] Use runtime::memchr instead of
 std::memchr. (#135298)

---
 flang-rt/include/flang-rt/runtime/io-stmt.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang-rt/include/flang-rt/runtime/io-stmt.h b/flang-rt/include/flang-rt/runtime/io-stmt.h
index b6e95d155d01c..73c59b0b5ed29 100644
--- a/flang-rt/include/flang-rt/runtime/io-stmt.h
+++ b/flang-rt/include/flang-rt/runtime/io-stmt.h
@@ -20,6 +20,7 @@
 #include "flang/Common/optional.h"
 #include "flang/Common/reference-wrapper.h"
 #include "flang/Common/visit.h"
+#include "flang/Runtime/freestanding-tools.h"
 #include "flang/Runtime/io-api.h"
 #include <flang/Common/variant.h>
 #include <functional>
@@ -180,8 +181,8 @@ class IoStatementState {
 
   private:
     RT_API_ATTRS void CheckForAsterisk() {
-      hasAsterisk_ =
-          at_ && at_ < limit_ && std::memchr(at_, '*', limit_ - at_) != nullptr;
+      hasAsterisk_ = at_ && at_ < limit_ &&
+          runtime::memchr(at_, '*', limit_ - at_) != nullptr;
     }
 
     ConnectionState &connection_;

From 6c5f50f18694a4d91d7ce53a14188c54ee7c6f3b Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 18 Apr 2025 10:47:32 -0500
Subject: [PATCH 421/710] [Offload] Fix typo on `-Xoffload-linker`

---
 offload/test/lit.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index 8cdf2f3f13681..f7ed287e7aece 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -207,7 +207,7 @@ def add_libraries(source):
                "-Xoffload-linker -lm " + \
                "-Xoffload-linker -lompdevice"
     else:
-        return source + " " + "-Xoffload-lnker -lompdevice"
+        return source + " " + "-Xoffload-linker -lompdevice"
 
 # substitutions
 # - for targets that exist in the system create the actual command.

From b3d2dc321c5c78b7204696afe07fe6ef3375acfd Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 18 Apr 2025 09:12:52 -0700
Subject: [PATCH 422/710] [RISCV] Fix assertion failure when using
 -fstack-clash-protection (#135248)

We can't assume MBBI is still pointing at MBB if we've already expanded
a probe. We need to re-query the MBB from MBBI. Fixes #135206

Co-authored-by: Craig Topper <craig.topper@sifive.com>
---
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp |  8 +-
 llvm/test/CodeGen/RISCV/pr135206.ll          | 84 ++++++++++++++++++++
 2 files changed, 89 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/pr135206.ll

diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index cefe7a732519d..89a9f7d871391 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -2114,11 +2114,13 @@ TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
 }
 
 // Synthesize the probe loop.
-static void emitStackProbeInline(MachineFunction &MF, MachineBasicBlock &MBB,
-                                 MachineBasicBlock::iterator MBBI, DebugLoc DL,
+static void emitStackProbeInline(MachineBasicBlock::iterator MBBI, DebugLoc DL,
                                  Register TargetReg, bool IsRVV) {
   assert(TargetReg != RISCV::X2 && "New top of stack cannot already be in SP");
 
+  MachineBasicBlock &MBB = *MBBI->getParent();
+  MachineFunction &MF = *MBB.getParent();
+
   auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
   const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
   bool IsRV64 = Subtarget.is64Bit();
@@ -2207,7 +2209,7 @@ void RISCVFrameLowering::inlineStackProbe(MachineFunction &MF,
       MachineBasicBlock::iterator MBBI = MI->getIterator();
       DebugLoc DL = MBB.findDebugLoc(MBBI);
       Register TargetReg = MI->getOperand(1).getReg();
-      emitStackProbeInline(MF, MBB, MBBI, DL, TargetReg,
+      emitStackProbeInline(MBBI, DL, TargetReg,
                            (MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV));
       MBBI->eraseFromParent();
     }
diff --git a/llvm/test/CodeGen/RISCV/pr135206.ll b/llvm/test/CodeGen/RISCV/pr135206.ll
new file mode 100644
index 0000000000000..196e78d8ed8b9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/pr135206.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple riscv64 < %s -o - | FileCheck %s
+
+%"buff" = type { [4096 x i64] }
+
+declare void @llvm.memset.p0.i64(ptr, i8, i64, i1)
+declare void @bar()
+
+define i1 @foo() nounwind "probe-stack"="inline-asm" "target-features"="+v" {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -2032
+; CHECK-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 2016(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 2008(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s2, 2000(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s3, 1992(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    lui a0, 7
+; CHECK-NEXT:    sub t1, sp, a0
+; CHECK-NEXT:    lui t2, 1
+; CHECK-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    sub sp, sp, t2
+; CHECK-NEXT:    sd zero, 0(sp)
+; CHECK-NEXT:    bne sp, t1, .LBB0_1
+; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    addi sp, sp, -2048
+; CHECK-NEXT:    addi sp, sp, -96
+; CHECK-NEXT:    csrr t1, vlenb
+; CHECK-NEXT:    lui t2, 1
+; CHECK-NEXT:  .LBB0_3: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    sub sp, sp, t2
+; CHECK-NEXT:    sd zero, 0(sp)
+; CHECK-NEXT:    sub t1, t1, t2
+; CHECK-NEXT:    bge t1, t2, .LBB0_3
+; CHECK-NEXT:  # %bb.4:
+; CHECK-NEXT:    sub sp, sp, t1
+; CHECK-NEXT:    li a0, 86
+; CHECK-NEXT:    addi s0, sp, 48
+; CHECK-NEXT:    addi s1, sp, 32
+; CHECK-NEXT:    addi s2, sp, 16
+; CHECK-NEXT:    lui a1, 353637
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    lui a0, 8
+; CHECK-NEXT:    addiw a0, a0, 32
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-NEXT:    addiw a0, a1, 1622
+; CHECK-NEXT:    vse8.v v8, (s0)
+; CHECK-NEXT:    vse8.v v8, (s1)
+; CHECK-NEXT:    vse8.v v8, (s2)
+; CHECK-NEXT:    slli a1, a0, 32
+; CHECK-NEXT:    add s3, a0, a1
+; CHECK-NEXT:    sd s3, 64(sp)
+; CHECK-NEXT:    call bar
+; CHECK-NEXT:    lui a0, 8
+; CHECK-NEXT:    addiw a0, a0, 32
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vse8.v v8, (s0)
+; CHECK-NEXT:    vse8.v v8, (s1)
+; CHECK-NEXT:    vse8.v v8, (s2)
+; CHECK-NEXT:    sd s3, 64(sp)
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    add sp, sp, a1
+; CHECK-NEXT:    lui a1, 8
+; CHECK-NEXT:    addiw a1, a1, -1952
+; CHECK-NEXT:    add sp, sp, a1
+; CHECK-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s1, 2008(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s2, 2000(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s3, 1992(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 2032
+; CHECK-NEXT:    ret
+  %1 = alloca %"buff", align 8
+  call void @llvm.memset.p0.i64(ptr %1, i8 86, i64 56, i1 false)
+  call void @bar()
+  call void @llvm.memset.p0.i64(ptr %1, i8 86, i64 56, i1 false)
+  ret i1 false
+}
+

From 711301066c2e5a6842866baea50ea9346b837459 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Fri, 18 Apr 2025 09:24:54 -0700
Subject: [PATCH 423/710] [LangRef] Add a description of the semantics of call
 signatures. (#136189)

This doesn't introduce anything new; it's just a reflection of the
semantics we've already had for many years.

Per discussion on #63484.
---
 llvm/docs/LangRef.rst | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 110c30e19220f..a5df82895d839 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -9475,10 +9475,11 @@ This instruction requires several arguments:
    from the :ref:`datalayout string<langref_datalayout>` will be used.
 #. '``ty``': the type of the call instruction itself which is also the
    type of the return value. Functions that return no value are marked
-   ``void``.
+   ``void``. The signature is computed based on the return type and argument
+   types.
 #. '``fnty``': shall be the signature of the function being invoked. The
    argument types must match the types implied by this signature. This
-   type can be omitted if the function is not varargs.
+   is only required if the signature specifies a varargs type.
 #. '``fnptrval``': An LLVM value containing a pointer to a function to
    be invoked. In most cases, this is a direct function invocation, but
    indirect ``invoke``'s are just as possible, calling an arbitrary pointer
@@ -9571,10 +9572,11 @@ This instruction requires several arguments:
    from the :ref:`datalayout string<langref_datalayout>` will be used.
 #. '``ty``': the type of the call instruction itself which is also the
    type of the return value. Functions that return no value are marked
-   ``void``.
+   ``void``.  The signature is computed based on the return type and argument
+   types.
 #. '``fnty``': shall be the signature of the function being called. The
    argument types must match the types implied by this signature. This
-   type can be omitted if the function is not varargs.
+   is only required if the signature specifies a varargs type.
 #. '``fnptrval``': An LLVM value containing a pointer to a function to
    be called. In most cases, this is a direct function call, but
    other ``callbr``'s are just as possible, calling an arbitrary pointer
@@ -13127,10 +13129,11 @@ This instruction requires several arguments:
    from the :ref:`datalayout string<langref_datalayout>` will be used.
 #. '``ty``': the type of the call instruction itself which is also the
    type of the return value. Functions that return no value are marked
-   ``void``.
+   ``void``. The signature is computed based on the return type and argument
+   types.
 #. '``fnty``': shall be the signature of the function being called. The
    argument types must match the types implied by this signature. This
-   type can be omitted if the function is not varargs.
+   is only required if the signature specifies a varargs type.
 #. '``fnptrval``': An LLVM value containing a pointer to a function to
    be called. In most cases, this is a direct function call, but
    indirect ``call``'s are just as possible, calling an arbitrary pointer
@@ -13152,6 +13155,16 @@ values. Upon a '``ret``' instruction in the called function, control
 flow continues with the instruction after the function call, and the
 return value of the function is bound to the result argument.
 
+If the callee refers to an intrinsic function, the signature of the call must
+match the signature of the callee.  Otherwise, if the signature of the call
+does not match the signature of the called function, the behavior is
+target-specific.  For a significant mismatch, this likely results in undefined
+behavior. LLVM interprocedural optimizations generally only optimize calls
+where the signature of the caller matches the signature of the callee.
+
+Note that it is possible for the signatures to mismatch even if a call appears
+to be a "direct" call, like ``call void @f()``.
+
 Example:
 """"""""
 

From f4a47b4003b6cfbdd445615b044f0911d99a0e44 Mon Sep 17 00:00:00 2001
From: Shubham Sandeep Rastogi <srastogi22@apple.com>
Date: Fri, 18 Apr 2025 09:34:48 -0700
Subject: [PATCH 424/710] Disable test on older compilers. (#136322)

---
 .../TestDbgInfoContentDequeFromStdModule.py                     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py
index 24b1e00de67c7..762f06140e27a 100644
--- a/lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py
@@ -10,7 +10,7 @@
 class TestDbgInfoContentDeque(TestBase):
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(compiler="clang", compiler_version=["<", "12.0"])
+    @skipIf(compiler="clang", compiler_version=["<", "18.0"])
     def test(self):
         self.build()
 

From bc48b3f8b812c755c5e9c42d6b970a9c1b496578 Mon Sep 17 00:00:00 2001
From: Daniel Bertalan <dani@danielbertalan.dev>
Date: Fri, 18 Apr 2025 18:36:54 +0200
Subject: [PATCH 425/710] [lld][WebAssembly] Fix spurious signature mismatch
 under LTO (#136197)

When generating C++ vtables, Clang declares virtual functions as
`void(void)` when their signature is not known (e.g.parameter types are
forward-declared). As WASM type checks imports, this would conflict with
the real definition during linking. Commit 59f959ff introduced a
workaround for this by deferring signature assignment until a definition
or direct call is seen.

When performing LTO, LLD first scans the bitcode files and creates
`DefinedFunction` symbol table entries for their contents. After LTO
codegen, they are replaced with `UndefinedFunction`s (so that the
definitions will be pulled in from the native LTO-d files when they are
added). At this point, if a function is only referenced in bitcode, its
signature remains `nullptr`.

From here, it should have behaved like in the non-LTO case: the first
direct call sets the signature. However, as the `isCalledDirectly` flag
was set to true, the missing signature was filled in by the type of the
first reference to the function, which could be a `void(void)` vtable
entry, which would then conflict with the real definition.

This commit sets `isCalledDirectly` to false so that the signature will
only be populated when a direct call is found.

See godotengine/godot#104497 and
emscripten-core/emscripten#10831
---
 .../lto/thinlto-signature-mismatch-unknown.ll | 45 +++++++++++++++++++
 lld/wasm/LTO.cpp                              |  3 +-
 2 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 lld/test/wasm/lto/thinlto-signature-mismatch-unknown.ll

diff --git a/lld/test/wasm/lto/thinlto-signature-mismatch-unknown.ll b/lld/test/wasm/lto/thinlto-signature-mismatch-unknown.ll
new file mode 100644
index 0000000000000..2f216e5853ff8
--- /dev/null
+++ b/lld/test/wasm/lto/thinlto-signature-mismatch-unknown.ll
@@ -0,0 +1,45 @@
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: opt -thinlto-bc a.ll -o a.o
+; RUN: opt -thinlto-bc b.ll -o b.o
+; RUN: llvm-ar rcs b.a b.o
+; RUN: opt -thinlto-bc c.ll -o c.o
+
+;; Taking the address of the incorrectly declared @foo should not generate a warning.
+; RUN: wasm-ld --fatal-warnings --no-entry --export-all a.o b.a -o a.out \
+; RUN:         | FileCheck %s --implicit-check-not 'warning' --allow-empty
+
+;; But we should still warn if we call the function with the wrong signature.
+; RUN: not wasm-ld --fatal-warnings --no-entry --export-all a.o b.a c.o -o b.out 2>&1 \
+; RUN:         | FileCheck %s --check-prefix=INVALID
+
+; INVALID: error: function signature mismatch: foo
+; INVALID: >>> defined as () -> void
+; INVALID: >>> defined as () -> i32
+
+;--- a.ll
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+@ptr = constant ptr @foo
+declare void @foo()
+
+;--- b.ll
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+define i32 @foo() noinline {
+entry:
+  ret i32 42
+}
+
+;--- c.ll
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+declare void @foo()
+
+define void @invalid() {
+entry:
+    call void @foo()
+    ret void
+}
diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp
index b9bd48acd6dc1..ab63281012eae 100644
--- a/lld/wasm/LTO.cpp
+++ b/lld/wasm/LTO.cpp
@@ -108,9 +108,10 @@ BitcodeCompiler::~BitcodeCompiler() = default;
 
 static void undefine(Symbol *s) {
   if (auto f = dyn_cast<DefinedFunction>(s))
+    // If the signature is null, there were no calls from non-bitcode objects.
     replaceSymbol<UndefinedFunction>(f, f->getName(), std::nullopt,
                                      std::nullopt, 0, f->getFile(),
-                                     f->signature);
+                                     f->signature, f->signature != nullptr);
   else if (isa<DefinedData>(s))
     replaceSymbol<UndefinedData>(s, s->getName(), 0, s->getFile());
   else

From ad6c23a7b5ecd5582566a6e51c1aa8105fb3f109 Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Fri, 18 Apr 2025 12:37:16 -0400
Subject: [PATCH 426/710] [Support] Allow `llvm::interleaved` with custom
 ostream types (#136318)

This makes `llvm::interleaved` useable with ostream types that define
custom stream operators that print in a different format from
`raw_ostream`. For example, MLIR's OpAsmPrinter prints values as
operands:
https://github.com/llvm/llvm-project/blob/6c5f50f18694a4d91d7ce53a14188c54ee7c6f3b/mlir/include/mlir/IR/OpImplementation.h#L534-L552
---
 llvm/include/llvm/Support/InterleavedRange.h  |  4 ++--
 .../Support/InterleavedRangeTest.cpp          | 22 +++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Support/InterleavedRange.h b/llvm/include/llvm/Support/InterleavedRange.h
index 4e70028504806..ce7cac98bcdd4 100644
--- a/llvm/include/llvm/Support/InterleavedRange.h
+++ b/llvm/include/llvm/Support/InterleavedRange.h
@@ -42,8 +42,8 @@ template <typename Range> class InterleavedRange {
                    StringRef Suffix)
       : TheRange(R), Separator(Separator), Prefix(Prefix), Suffix(Suffix) {}
 
-  friend raw_ostream &operator<<(raw_ostream &OS,
-                                 const InterleavedRange &Interleaved) {
+  template <typename OStream>
+  friend OStream &operator<<(OStream &OS, const InterleavedRange &Interleaved) {
     if (!Interleaved.Prefix.empty())
       OS << Interleaved.Prefix;
     llvm::interleave(Interleaved.TheRange, OS, Interleaved.Separator);
diff --git a/llvm/unittests/Support/InterleavedRangeTest.cpp b/llvm/unittests/Support/InterleavedRangeTest.cpp
index 8640b81fe8ad8..f258eac04e9c6 100644
--- a/llvm/unittests/Support/InterleavedRangeTest.cpp
+++ b/llvm/unittests/Support/InterleavedRangeTest.cpp
@@ -67,4 +67,26 @@ TEST(InterleavedRangeTest, CustomPrint) {
   EXPECT_EQ("[$$3##, $$4##, $$5##]", interleaved_array(V).str());
 }
 
+struct CustomDoublingOStream : raw_string_ostream {
+  unsigned NumCalled = 0;
+  using raw_string_ostream::raw_string_ostream;
+
+  friend CustomDoublingOStream &operator<<(CustomDoublingOStream &OS, int V) {
+    ++OS.NumCalled;
+    static_cast<raw_string_ostream &>(OS) << (2 * V);
+    return OS;
+  }
+};
+
+TEST(InterleavedRangeTest, CustomOStream) {
+  // Make sure that interleaved calls the stream operator on the derived class,
+  // and that it returns a reference to the same stream type.
+  int V[] = {3, 4, 5};
+  std::string Buf;
+  CustomDoublingOStream OS(Buf);
+  OS << interleaved(V) << 22;
+  EXPECT_EQ("6, 8, 1044", Buf);
+  EXPECT_EQ(OS.NumCalled, 4u);
+}
+
 } // namespace

From 4cb9a3700c31357821e192124baeb3a3a35ff93b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 09:40:27 -0700
Subject: [PATCH 427/710] Revert "[mlir] add a fluent API to
 GreedyRewriterConfig (#132253)"

This reverts commit 63b8f1c9482ed0a964980df4aed89bef922b8078.

Buildbot failure:
https://lab.llvm.org/buildbot/#/builders/172/builds/12083/steps/5/logs/stdio

I've reproduced the error with a release build (-DCMAKE_BUILD_TYPE=Release).
---
 .../Optimizer/CodeGen/LowerRepackArrays.cpp   |  4 +-
 .../HLFIR/Transforms/InlineElementals.cpp     |  4 +-
 .../HLFIR/Transforms/InlineHLFIRAssign.cpp    |  4 +-
 .../HLFIR/Transforms/LowerHLFIRIntrinsics.cpp |  4 +-
 .../Transforms/OptimizedBufferization.cpp     |  4 +-
 .../Transforms/SimplifyHLFIRIntrinsics.cpp    |  4 +-
 flang/lib/Optimizer/Passes/Pipelines.cpp      |  6 +-
 .../Transforms/AssumedRankOpConversion.cpp    |  4 +-
 .../ConstantArgumentGlobalisation.cpp         |  6 +-
 .../Transforms/SimplifyFIROperations.cpp      |  3 +-
 .../lib/Optimizer/Transforms/StackArrays.cpp  |  3 +-
 .../Transforms/GreedyPatternRewriteDriver.h   | 76 +++-------------
 mlir/include/mlir/Transforms/Passes.td        |  2 +-
 .../TransformOps/AffineTransformOps.cpp       | 11 ++-
 .../Transforms/AffineDataCopyGeneration.cpp   |  7 +-
 .../Transforms/SimplifyAffineStructures.cpp   |  7 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |  9 +-
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |  8 +-
 .../Transforms/IntRangeOptimizations.cpp      | 16 ++--
 .../BufferDeallocationSimplification.cpp      | 14 +--
 .../TransformOps/LinalgTransformOps.cpp       |  6 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  7 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  8 +-
 .../SPIRV/Transforms/SPIRVConversion.cpp      |  6 +-
 .../lib/Dialect/Transform/IR/TransformOps.cpp | 14 +--
 mlir/lib/Reducer/ReductionTreePass.cpp        |  8 +-
 mlir/lib/Transforms/Canonicalizer.cpp         | 16 ++--
 .../Utils/GreedyPatternRewriteDriver.cpp      | 88 +++++++++----------
 .../lib/Dialect/Affine/TestAffineDataCopy.cpp |  2 +-
 mlir/test/lib/Dialect/Test/TestPatterns.cpp   | 28 +++---
 30 files changed, 162 insertions(+), 217 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
index 7fb713ff1a6c7..7deed3d44ae5b 100644
--- a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
+++ b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
@@ -357,8 +357,8 @@ class LowerRepackArraysPass
     patterns.insert<PackArrayConversion>(context);
     patterns.insert<UnpackArrayConversion>(context);
     mlir::GreedyRewriteConfig config;
-    config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
+    config.enableRegionSimplification =
+        mlir::GreedySimplifyRegionLevel::Disabled;
     (void)applyPatternsGreedily(module, std::move(patterns), config);
   }
 
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
index c42b895946d19..b68fe6ee0c747 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
@@ -119,8 +119,8 @@ class InlineElementalsPass
 
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks.
-    config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
+    config.enableRegionSimplification =
+        mlir::GreedySimplifyRegionLevel::Disabled;
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<InlineElementalConversion>(context);
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
index 6e209cce07ad4..249976d5509b0 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
@@ -135,8 +135,8 @@ class InlineHLFIRAssignPass
 
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks.
-    config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
+    config.enableRegionSimplification =
+        mlir::GreedySimplifyRegionLevel::Disabled;
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<InlineHLFIRAssignConversion>(context);
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
index 31e5bc1193e22..7c0fcba806869 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
@@ -557,8 +557,8 @@ class LowerHLFIRIntrinsics
     // Pattern rewriting only requires that the resulting IR is still valid
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks
-    config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
+    config.enableRegionSimplification =
+        mlir::GreedySimplifyRegionLevel::Disabled;
 
     if (mlir::failed(
             mlir::applyPatternsGreedily(module, std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index c0856082989c1..c489450384a35 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -1327,8 +1327,8 @@ class OptimizedBufferizationPass
 
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks
-    config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
+    config.enableRegionSimplification =
+        mlir::GreedySimplifyRegionLevel::Disabled;
 
     mlir::RewritePatternSet patterns(context);
     // TODO: right now the patterns are non-conflicting,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index d153db2afff07..bac10121a881b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -1476,8 +1476,8 @@ class SimplifyHLFIRIntrinsics
 
     mlir::GreedyRewriteConfig config;
     // Prevent the pattern driver from merging blocks
-    config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
+    config.enableRegionSimplification =
+        mlir::GreedySimplifyRegionLevel::Disabled;
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 7a06a27748ebd..81ff6bf9b2c6a 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -35,8 +35,7 @@ void addNestedPassToAllTopLevelOperationsConditionally(
 
 void addCanonicalizerPassWithoutRegionSimplification(mlir::OpPassManager &pm) {
   mlir::GreedyRewriteConfig config;
-  config.setRegionSimplificationLevel(
-      mlir::GreedySimplifyRegionLevel::Disabled);
+  config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
   pm.addPass(mlir::createCanonicalizerPass(config));
 }
 
@@ -164,8 +163,7 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
 
   // simplify the IR
   mlir::GreedyRewriteConfig config;
-  config.setRegionSimplificationLevel(
-      mlir::GreedySimplifyRegionLevel::Disabled);
+  config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
   pm.addPass(mlir::createCSEPass());
   fir::addAVC(pm, pc.OptLevel);
   addNestedPassToAllTopLevelOperations<PassConstructor>(
diff --git a/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp b/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
index 6af1cb988a4c1..d0bd67a236419 100644
--- a/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/AssumedRankOpConversion.cpp
@@ -152,8 +152,8 @@ class AssumedRankOpConversion
     patterns.insert<ReboxAssumedRankConv>(context, &symbolTable, kindMap);
     patterns.insert<IsAssumedSizeConv>(context, &symbolTable, kindMap);
     mlir::GreedyRewriteConfig config;
-    config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
+    config.enableRegionSimplification =
+        mlir::GreedySimplifyRegionLevel::Disabled;
     (void)applyPatternsGreedily(mod, std::move(patterns), config);
   }
 };
diff --git a/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp b/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
index 239a7cdaa4cf2..562f3058f20f3 100644
--- a/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
+++ b/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
@@ -168,9 +168,9 @@ class ConstantArgumentGlobalisationOpt
     auto *context = &getContext();
     mlir::RewritePatternSet patterns(context);
     mlir::GreedyRewriteConfig config;
-    config.setRegionSimplificationLevel(
-        mlir::GreedySimplifyRegionLevel::Disabled);
-    config.setStrictness(mlir::GreedyRewriteStrictness::ExistingOps);
+    config.enableRegionSimplification =
+        mlir::GreedySimplifyRegionLevel::Disabled;
+    config.strictMode = mlir::GreedyRewriteStrictness::ExistingOps;
 
     patterns.insert<CallOpRewriter>(context, *di);
     if (mlir::failed(
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 6d106046b70f2..212de2f2286db 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -205,8 +205,7 @@ void SimplifyFIROperationsPass::runOnOperation() {
   fir::populateSimplifyFIROperationsPatterns(patterns,
                                              preferInlineImplementation);
   mlir::GreedyRewriteConfig config;
-  config.setRegionSimplificationLevel(
-      mlir::GreedySimplifyRegionLevel::Disabled);
+  config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
 
   if (mlir::failed(
           mlir::applyPatternsGreedily(module, std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index f9b9b4f4ff385..9a6566bef50f1 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -806,8 +806,7 @@ void StackArraysPass::runOnOperation() {
   mlir::RewritePatternSet patterns(&context);
   mlir::GreedyRewriteConfig config;
   // prevent the pattern driver form merging blocks
-  config.setRegionSimplificationLevel(
-      mlir::GreedySimplifyRegionLevel::Disabled);
+  config.enableRegionSimplification = mlir::GreedySimplifyRegionLevel::Disabled;
 
   patterns.insert<AllocMemConversion>(&context, *candidateOps);
   if (mlir::failed(mlir::applyOpPatternsGreedily(
diff --git a/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h b/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
index 45e61b68f5db2..110b4f64856eb 100644
--- a/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
+++ b/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
@@ -49,43 +49,25 @@ class GreedyRewriteConfig {
   /// larger patterns when given an ambiguous pattern set.
   ///
   /// Note: Only applicable when simplifying entire regions.
-  bool getUseTopDownTraversal() const { return useTopDownTraversal; }
-  GreedyRewriteConfig &setUseTopDownTraversal(bool use = true) {
-    useTopDownTraversal = use;
-    return *this;
-  }
+  bool useTopDownTraversal = false;
 
   /// Perform control flow optimizations to the region tree after applying all
   /// patterns.
   ///
   /// Note: Only applicable when simplifying entire regions.
-  GreedySimplifyRegionLevel getRegionSimplificationLevel() const {
-    return regionSimplificationLevel;
-  }
-  GreedyRewriteConfig &
-  setRegionSimplificationLevel(GreedySimplifyRegionLevel level) {
-    regionSimplificationLevel = level;
-    return *this;
-  }
+  GreedySimplifyRegionLevel enableRegionSimplification =
+      GreedySimplifyRegionLevel::Aggressive;
 
   /// This specifies the maximum number of times the rewriter will iterate
   /// between applying patterns and simplifying regions. Use `kNoLimit` to
   /// disable this iteration limit.
   ///
   /// Note: Only applicable when simplifying entire regions.
-  int64_t getMaxIterations() const { return maxIterations; }
-  GreedyRewriteConfig &setMaxIterations(int64_t iterations) {
-    maxIterations = iterations;
-    return *this;
-  }
+  int64_t maxIterations = 10;
 
   /// This specifies the maximum number of rewrites within an iteration. Use
   /// `kNoLimit` to disable this limit.
-  int64_t getMaxNumRewrites() const { return maxNumRewrites; }
-  GreedyRewriteConfig &setMaxNumRewrites(int64_t limit) {
-    maxNumRewrites = limit;
-    return *this;
-  }
+  int64_t maxNumRewrites = kNoLimit;
 
   static constexpr int64_t kNoLimit = -1;
 
@@ -93,11 +75,7 @@ class GreedyRewriteConfig {
   /// specified, the closest enclosing region around the initial list of ops
   /// (or the specified region, depending on which greedy rewrite entry point
   /// is used) is used as a scope.
-  Region *getScope() const { return scope; }
-  GreedyRewriteConfig &setScope(Region *scope) {
-    this->scope = scope;
-    return *this;
-  }
+  Region *scope = nullptr;
 
   /// Strict mode can restrict the ops that are added to the worklist during
   /// the rewrite.
@@ -109,44 +87,16 @@ class GreedyRewriteConfig {
   /// * GreedyRewriteStrictness::ExistingOps: Only pre-existing ops (that were
   ///   were on the worklist at the very beginning) enqueued. All other ops are
   ///   excluded.
-  GreedyRewriteStrictness getStrictness() const { return strictness; }
-  GreedyRewriteConfig &setStrictness(GreedyRewriteStrictness mode) {
-    strictness = mode;
-    return *this;
-  }
+  GreedyRewriteStrictness strictMode = GreedyRewriteStrictness::AnyOp;
 
   /// An optional listener that should be notified about IR modifications.
-  RewriterBase::Listener *getListener() const { return listener; }
-  GreedyRewriteConfig &setListener(RewriterBase::Listener *listener) {
-    this->listener = listener;
-    return *this;
-  }
+  RewriterBase::Listener *listener = nullptr;
 
   /// Whether this should fold while greedily rewriting.
-  bool isFoldingEnabled() const { return fold; }
-  GreedyRewriteConfig &enableFolding(bool enable = true) {
-    fold = enable;
-    return *this;
-  }
+  bool fold = true;
 
   /// If set to "true", constants are CSE'd (even across multiple regions that
   /// are in a parent-ancestor relationship).
-  bool isConstantCSEEnabled() const { return cseConstants; }
-  GreedyRewriteConfig &enableConstantCSE(bool enable = true) {
-    cseConstants = enable;
-    return *this;
-  }
-
-private:
-  Region *scope = nullptr;
-  bool useTopDownTraversal = false;
-  GreedySimplifyRegionLevel regionSimplificationLevel =
-      GreedySimplifyRegionLevel::Aggressive;
-  int64_t maxIterations = 10;
-  int64_t maxNumRewrites = kNoLimit;
-  GreedyRewriteStrictness strictness = GreedyRewriteStrictness::AnyOp;
-  RewriterBase::Listener *listener = nullptr;
-  bool fold = true;
   bool cseConstants = true;
 };
 
@@ -178,14 +128,14 @@ applyPatternsGreedily(Region &region, const FrozenRewritePatternSet &patterns,
                       GreedyRewriteConfig config = GreedyRewriteConfig(),
                       bool *changed = nullptr);
 /// Same as `applyPatternsAndGreedily` above with folding.
-/// FIXME: Remove this once transition to above is completed.
+/// FIXME: Remove this once transition to above is complieted.
 LLVM_DEPRECATED("Use applyPatternsGreedily() instead", "applyPatternsGreedily")
 inline LogicalResult
 applyPatternsAndFoldGreedily(Region &region,
                              const FrozenRewritePatternSet &patterns,
                              GreedyRewriteConfig config = GreedyRewriteConfig(),
                              bool *changed = nullptr) {
-  config.enableFolding();
+  config.fold = true;
   return applyPatternsGreedily(region, patterns, config, changed);
 }
 
@@ -237,7 +187,7 @@ applyPatternsAndFoldGreedily(Operation *op,
                              const FrozenRewritePatternSet &patterns,
                              GreedyRewriteConfig config = GreedyRewriteConfig(),
                              bool *changed = nullptr) {
-  config.enableFolding();
+  config.fold = true;
   return applyPatternsGreedily(op, patterns, config, changed);
 }
 
@@ -283,7 +233,7 @@ applyOpPatternsAndFold(ArrayRef<Operation *> ops,
                        const FrozenRewritePatternSet &patterns,
                        GreedyRewriteConfig config = GreedyRewriteConfig(),
                        bool *changed = nullptr, bool *allErased = nullptr) {
-  config.enableFolding();
+  config.fold = true;
   return applyOpPatternsGreedily(ops, patterns, config, changed, allErased);
 }
 
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
index 1e89a78912e99..a39ab77fc8fb3 100644
--- a/mlir/include/mlir/Transforms/Passes.td
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -33,7 +33,7 @@ def Canonicalizer : Pass<"canonicalize"> {
     Option<"topDownProcessingEnabled", "top-down", "bool",
            /*default=*/"true",
            "Seed the worklist in general top-down order">,
-    Option<"regionSimplifyLevel", "region-simplify", "mlir::GreedySimplifyRegionLevel",
+    Option<"enableRegionSimplification", "region-simplify", "mlir::GreedySimplifyRegionLevel",
            /*default=*/"mlir::GreedySimplifyRegionLevel::Normal",
            "Perform control flow optimizations to the region tree",
              [{::llvm::cl::values(
diff --git a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
index 43d37ee3332ef..9f7df7823d997 100644
--- a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
+++ b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
@@ -127,13 +127,12 @@ SimplifyBoundedAffineOpsOp::apply(transform::TransformRewriter &rewriter,
   patterns.insert<SimplifyAffineMinMaxOp<AffineMinOp>,
                   SimplifyAffineMinMaxOp<AffineMaxOp>>(getContext(), cstr);
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
+  GreedyRewriteConfig config;
+  config.listener =
+      static_cast<RewriterBase::Listener *>(rewriter.getListener());
+  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
   // Apply the simplification pattern to a fixpoint.
-  if (failed(applyOpPatternsGreedily(
-          targets, frozenPatterns,
-          GreedyRewriteConfig()
-              .setListener(
-                  static_cast<RewriterBase::Listener *>(rewriter.getListener()))
-              .setStrictness(GreedyRewriteStrictness::ExistingAndNewOps)))) {
+  if (failed(applyOpPatternsGreedily(targets, frozenPatterns, config))) {
     auto diag = emitDefiniteFailure()
                 << "affine.min/max simplification did not converge";
     return diag;
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index 62c1857e4b1da..4d30213cc6ec2 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -237,8 +237,7 @@ void AffineDataCopyGeneration::runOnOperation() {
   AffineLoadOp::getCanonicalizationPatterns(patterns, &getContext());
   AffineStoreOp::getCanonicalizationPatterns(patterns, &getContext());
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
-  (void)applyOpPatternsGreedily(
-      copyOps, frozenPatterns,
-      GreedyRewriteConfig().setStrictness(
-          GreedyRewriteStrictness::ExistingAndNewOps));
+  GreedyRewriteConfig config;
+  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
+  (void)applyOpPatternsGreedily(copyOps, frozenPatterns, config);
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
index 9e9096c2e3186..31711ade3153b 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
@@ -109,8 +109,7 @@ void SimplifyAffineStructures::runOnOperation() {
     if (isa<AffineForOp, AffineIfOp, AffineApplyOp>(op))
       opsToSimplify.push_back(op);
   });
-  (void)applyOpPatternsGreedily(
-      opsToSimplify, frozenPatterns,
-      GreedyRewriteConfig().setStrictness(
-          GreedyRewriteStrictness::ExistingAndNewOps));
+  GreedyRewriteConfig config;
+  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
+  (void)applyOpPatternsGreedily(opsToSimplify, frozenPatterns, config);
 }
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 0d4ba3940c48e..dd539ff685653 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -315,12 +315,11 @@ LogicalResult mlir::affine::affineForOpBodySkew(AffineForOp forOp,
         // Simplify/canonicalize the affine.for.
         RewritePatternSet patterns(res.getContext());
         AffineForOp::getCanonicalizationPatterns(patterns, res.getContext());
+        GreedyRewriteConfig config;
+        config.strictMode = GreedyRewriteStrictness::ExistingOps;
         bool erased;
-        (void)applyOpPatternsGreedily(
-            res.getOperation(), std::move(patterns),
-            GreedyRewriteConfig().setStrictness(
-                GreedyRewriteStrictness::ExistingAndNewOps),
-            /*changed=*/nullptr, &erased);
+        (void)applyOpPatternsGreedily(res.getOperation(), std::move(patterns),
+                                      config, /*changed=*/nullptr, &erased);
         if (!erased && !prologue)
           prologue = res;
         if (!erased)
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 11798b99fa879..2925aa918cb1c 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -426,11 +426,11 @@ LogicalResult mlir::affine::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
   RewritePatternSet patterns(ifOp.getContext());
   AffineIfOp::getCanonicalizationPatterns(patterns, ifOp.getContext());
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
+  GreedyRewriteConfig config;
+  config.strictMode = GreedyRewriteStrictness::ExistingOps;
   bool erased;
-  (void)applyOpPatternsGreedily(
-      ifOp.getOperation(), frozenPatterns,
-      GreedyRewriteConfig().setStrictness(GreedyRewriteStrictness::ExistingOps),
-      /*changed=*/nullptr, &erased);
+  (void)applyOpPatternsGreedily(ifOp.getOperation(), frozenPatterns, config,
+                                /*changed=*/nullptr, &erased);
   if (erased) {
     if (folded)
       *folded = true;
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index f2f93883eb2b7..602d80a45993e 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -494,9 +494,10 @@ struct IntRangeOptimizationsPass final
     RewritePatternSet patterns(ctx);
     populateIntRangeOptimizationsPatterns(patterns, solver);
 
-    if (failed(applyPatternsGreedily(
-            op, std::move(patterns),
-            GreedyRewriteConfig().setListener(&listener))))
+    GreedyRewriteConfig config;
+    config.listener = &listener;
+
+    if (failed(applyPatternsGreedily(op, std::move(patterns), config)))
       signalPassFailure();
   }
 };
@@ -519,12 +520,13 @@ struct IntRangeNarrowingPass final
     RewritePatternSet patterns(ctx);
     populateIntRangeNarrowingPatterns(patterns, solver, bitwidthsSupported);
 
+    GreedyRewriteConfig config;
     // We specifically need bottom-up traversal as cmpi pattern needs range
     // data, attached to its original argument values.
-    if (failed(applyPatternsGreedily(
-            op, std::move(patterns),
-            GreedyRewriteConfig().setUseTopDownTraversal(false).setListener(
-                &listener))))
+    config.useTopDownTraversal = false;
+    config.listener = &listener;
+
+    if (failed(applyPatternsGreedily(op, std::move(patterns), config)))
       signalPassFailure();
   }
 };
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index c5fab80ecaa08..35f86a62ae592 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -463,15 +463,15 @@ struct BufferDeallocationSimplificationPass
                  SplitDeallocWhenNotAliasingAnyOther,
                  RetainedMemrefAliasingAlwaysDeallocatedMemref>(&getContext(),
                                                                 analysis);
-
-    populateDeallocOpCanonicalizationPatterns(patterns, &getContext());
     // We don't want that the block structure changes invalidating the
-    // `BufferOriginAnalysis` so we apply the rewrites with `Normal` level of
+    // `BufferOriginAnalysis` so we apply the rewrites witha `Normal` level of
     // region simplification
-    if (failed(applyPatternsGreedily(
-            getOperation(), std::move(patterns),
-            GreedyRewriteConfig().setRegionSimplificationLevel(
-                GreedySimplifyRegionLevel::Normal))))
+    GreedyRewriteConfig config;
+    config.enableRegionSimplification = GreedySimplifyRegionLevel::Normal;
+    populateDeallocOpCanonicalizationPatterns(patterns, &getContext());
+
+    if (failed(
+            applyPatternsGreedily(getOperation(), std::move(patterns), config)))
       signalPassFailure();
   }
 };
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index b20e6050fb4f8..c90ebe4487ca4 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -3587,9 +3587,9 @@ transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
   vector::populateVectorStepLoweringPatterns(patterns);
 
   TrackingListener listener(state, *this);
-  if (failed(
-          applyPatternsGreedily(target, std::move(patterns),
-                                GreedyRewriteConfig().setListener(&listener))))
+  GreedyRewriteConfig config;
+  config.listener = &listener;
+  if (failed(applyPatternsGreedily(target, std::move(patterns), config)))
     return emitDefaultDefiniteFailure(target);
 
   results.push_back(target);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index 62d016b87d627..bf70597d5ddfe 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -2327,9 +2327,10 @@ struct LinalgElementwiseOpFusionPass
     // Add constant folding patterns.
     populateConstantFoldLinalgOperations(patterns, defaultControlFn);
 
-    // Use TopDownTraversal for compile time reasons.
-    (void)applyPatternsGreedily(op, std::move(patterns),
-                                GreedyRewriteConfig().setUseTopDownTraversal());
+    // Use TopDownTraversal for compile time reasons
+    GreedyRewriteConfig grc;
+    grc.useTopDownTraversal = true;
+    (void)applyPatternsGreedily(op, std::move(patterns), grc);
   }
 };
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index 7edf19689d2e1..91862d2e17d71 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -1438,10 +1438,10 @@ SliceTrackingListener::insertAndApplyPatterns(ArrayRef<Operation *> ops) {
   if (!patterns)
     return success();
 
-  return applyOpPatternsGreedily(
-      ops, patterns.value(),
-      GreedyRewriteConfig().setListener(this).setStrictness(
-          GreedyRewriteStrictness::ExistingAndNewOps));
+  GreedyRewriteConfig config;
+  config.listener = this;
+  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
+  return applyOpPatternsGreedily(ops, patterns.value(), config);
 }
 
 void SliceTrackingListener::notifyOperationInserted(
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index 811f03abb3461..19b9af146f4a4 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -1353,9 +1353,9 @@ LogicalResult mlir::spirv::unrollVectorsInSignatures(Operation *op) {
   // We only want to apply signature conversion once to the existing func ops.
   // Without specifying strictMode, the greedy pattern rewriter will keep
   // looking for newly created func ops.
-  return applyPatternsGreedily(op, std::move(patterns),
-                               GreedyRewriteConfig().setStrictness(
-                                   GreedyRewriteStrictness::ExistingOps));
+  GreedyRewriteConfig config;
+  config.strictMode = GreedyRewriteStrictness::ExistingOps;
+  return applyPatternsGreedily(op, std::move(patterns), config);
 }
 
 LogicalResult mlir::spirv::unrollVectorsInFuncBodies(Operation *op) {
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index a811fae003584..798853a75441a 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -393,16 +393,16 @@ DiagnosedSilenceableFailure transform::ApplyPatternsOp::applyToOne(
 
   // Configure the GreedyPatternRewriteDriver.
   GreedyRewriteConfig config;
-  config.setListener(
-      static_cast<RewriterBase::Listener *>(rewriter.getListener()));
+  config.listener =
+      static_cast<RewriterBase::Listener *>(rewriter.getListener());
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
 
-  config.setMaxIterations(getMaxIterations() == static_cast<uint64_t>(-1)
+  config.maxIterations = getMaxIterations() == static_cast<uint64_t>(-1)
+                             ? GreedyRewriteConfig::kNoLimit
+                             : getMaxIterations();
+  config.maxNumRewrites = getMaxNumRewrites() == static_cast<uint64_t>(-1)
                               ? GreedyRewriteConfig::kNoLimit
-                              : getMaxIterations());
-  config.setMaxNumRewrites(getMaxNumRewrites() == static_cast<uint64_t>(-1)
-                               ? GreedyRewriteConfig::kNoLimit
-                               : getMaxNumRewrites());
+                              : getMaxNumRewrites();
 
   // Apply patterns and CSE repetitively until a fixpoint is reached. If no CSE
   // was requested, apply the greedy pattern rewrite only once. (The greedy
diff --git a/mlir/lib/Reducer/ReductionTreePass.cpp b/mlir/lib/Reducer/ReductionTreePass.cpp
index 549e4f2bd813b..7292752c712ae 100644
--- a/mlir/lib/Reducer/ReductionTreePass.cpp
+++ b/mlir/lib/Reducer/ReductionTreePass.cpp
@@ -62,11 +62,11 @@ static void applyPatterns(Region &region,
   // before that transform.
   for (Operation *op : opsInRange) {
     // `applyOpPatternsGreedily` with folding returns whether the op is
-    // converted. Omit it because we don't have expectation this reduction will
+    // convered. Omit it because we don't have expectation this reduction will
     // be success or not.
-    (void)applyOpPatternsGreedily(op, patterns,
-                                  GreedyRewriteConfig().setStrictness(
-                                      GreedyRewriteStrictness::ExistingOps));
+    GreedyRewriteConfig config;
+    config.strictMode = GreedyRewriteStrictness::ExistingOps;
+    (void)applyOpPatternsGreedily(op, patterns, config);
   }
 
   if (eraseOpNotInRange)
diff --git a/mlir/lib/Transforms/Canonicalizer.cpp b/mlir/lib/Transforms/Canonicalizer.cpp
index 4b0ac28a03713..7ccd503fb0288 100644
--- a/mlir/lib/Transforms/Canonicalizer.cpp
+++ b/mlir/lib/Transforms/Canonicalizer.cpp
@@ -32,10 +32,10 @@ struct Canonicalizer : public impl::CanonicalizerBase<Canonicalizer> {
                 ArrayRef<std::string> disabledPatterns,
                 ArrayRef<std::string> enabledPatterns)
       : config(config) {
-    this->topDownProcessingEnabled = config.getUseTopDownTraversal();
-    this->regionSimplifyLevel = config.getRegionSimplificationLevel();
-    this->maxIterations = config.getMaxIterations();
-    this->maxNumRewrites = config.getMaxNumRewrites();
+    this->topDownProcessingEnabled = config.useTopDownTraversal;
+    this->enableRegionSimplification = config.enableRegionSimplification;
+    this->maxIterations = config.maxIterations;
+    this->maxNumRewrites = config.maxNumRewrites;
     this->disabledPatterns = disabledPatterns;
     this->enabledPatterns = enabledPatterns;
   }
@@ -44,10 +44,10 @@ struct Canonicalizer : public impl::CanonicalizerBase<Canonicalizer> {
   /// execution.
   LogicalResult initialize(MLIRContext *context) override {
     // Set the config from possible pass options set in the meantime.
-    config.setUseTopDownTraversal(topDownProcessingEnabled);
-    config.setRegionSimplificationLevel(regionSimplifyLevel);
-    config.setMaxIterations(maxIterations);
-    config.setMaxNumRewrites(maxNumRewrites);
+    config.useTopDownTraversal = topDownProcessingEnabled;
+    config.enableRegionSimplification = enableRegionSimplification;
+    config.maxIterations = maxIterations;
+    config.maxNumRewrites = maxNumRewrites;
 
     RewritePatternSet owningPatterns(context);
     for (auto *dialect : context->getLoadedDialects())
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index 84d547f03829a..7c1cfd91f85e6 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -455,8 +455,8 @@ bool GreedyPatternRewriteDriver::processWorklist() {
   bool changed = false;
   int64_t numRewrites = 0;
   while (!worklist.empty() &&
-         (numRewrites < config.getMaxNumRewrites() ||
-          config.getMaxNumRewrites() == GreedyRewriteConfig::kNoLimit)) {
+         (numRewrites < config.maxNumRewrites ||
+          config.maxNumRewrites == GreedyRewriteConfig::kNoLimit)) {
     auto *op = worklist.pop();
 
     LLVM_DEBUG({
@@ -488,7 +488,7 @@ bool GreedyPatternRewriteDriver::processWorklist() {
     // infinite folding loop, as every constant op would be folded to an
     // Attribute and then immediately be rematerialized as a constant op, which
     // is then put on the worklist.
-    if (config.isFoldingEnabled() && !op->hasTrait<OpTrait::ConstantLike>()) {
+    if (config.fold && !op->hasTrait<OpTrait::ConstantLike>()) {
       SmallVector<OpFoldResult> foldResults;
       if (succeeded(op->fold(foldResults))) {
         LLVM_DEBUG(logResultWithLine("success", "operation was folded"));
@@ -574,21 +574,21 @@ bool GreedyPatternRewriteDriver::processWorklist() {
         logger.getOStream() << ")' {\n";
         logger.indent();
       });
-      if (RewriterBase::Listener *listener = config.getListener())
-        listener->notifyPatternBegin(pattern, op);
+      if (config.listener)
+        config.listener->notifyPatternBegin(pattern, op);
       return true;
     };
     function_ref<bool(const Pattern &)> canApply = canApplyCallback;
     auto onFailureCallback = [&](const Pattern &pattern) {
       LLVM_DEBUG(logResult("failure", "pattern failed to match"));
-      if (RewriterBase::Listener *listener = config.getListener())
-        listener->notifyPatternEnd(pattern, failure());
+      if (config.listener)
+        config.listener->notifyPatternEnd(pattern, failure());
     };
     function_ref<void(const Pattern &)> onFailure = onFailureCallback;
     auto onSuccessCallback = [&](const Pattern &pattern) {
       LLVM_DEBUG(logResult("success", "pattern applied successfully"));
-      if (RewriterBase::Listener *listener = config.getListener())
-        listener->notifyPatternEnd(pattern, success());
+      if (config.listener)
+        config.listener->notifyPatternEnd(pattern, success());
       return success();
     };
     function_ref<LogicalResult(const Pattern &)> onSuccess = onSuccessCallback;
@@ -640,7 +640,7 @@ void GreedyPatternRewriteDriver::addToWorklist(Operation *op) {
   do {
     ancestors.push_back(op);
     region = op->getParentRegion();
-    if (config.getScope() == region) {
+    if (config.scope == region) {
       // Scope (can be `nullptr`) was reached. Stop traveral and enqueue ops.
       for (Operation *op : ancestors)
         addSingleOpToWorklist(op);
@@ -652,20 +652,20 @@ void GreedyPatternRewriteDriver::addToWorklist(Operation *op) {
 }
 
 void GreedyPatternRewriteDriver::addSingleOpToWorklist(Operation *op) {
-  if (config.getStrictness() == GreedyRewriteStrictness::AnyOp ||
+  if (config.strictMode == GreedyRewriteStrictness::AnyOp ||
       strictModeFilteredOps.contains(op))
     worklist.push(op);
 }
 
 void GreedyPatternRewriteDriver::notifyBlockInserted(
     Block *block, Region *previous, Region::iterator previousIt) {
-  if (RewriterBase::Listener *listener = config.getListener())
-    listener->notifyBlockInserted(block, previous, previousIt);
+  if (config.listener)
+    config.listener->notifyBlockInserted(block, previous, previousIt);
 }
 
 void GreedyPatternRewriteDriver::notifyBlockErased(Block *block) {
-  if (RewriterBase::Listener *listener = config.getListener())
-    listener->notifyBlockErased(block);
+  if (config.listener)
+    config.listener->notifyBlockErased(block);
 }
 
 void GreedyPatternRewriteDriver::notifyOperationInserted(
@@ -674,9 +674,9 @@ void GreedyPatternRewriteDriver::notifyOperationInserted(
     logger.startLine() << "** Insert  : '" << op->getName() << "'(" << op
                        << ")\n";
   });
-  if (RewriterBase::Listener *listener = config.getListener())
-    listener->notifyOperationInserted(op, previous);
-  if (config.getStrictness() == GreedyRewriteStrictness::ExistingAndNewOps)
+  if (config.listener)
+    config.listener->notifyOperationInserted(op, previous);
+  if (config.strictMode == GreedyRewriteStrictness::ExistingAndNewOps)
     strictModeFilteredOps.insert(op);
   addToWorklist(op);
 }
@@ -686,8 +686,8 @@ void GreedyPatternRewriteDriver::notifyOperationModified(Operation *op) {
     logger.startLine() << "** Modified: '" << op->getName() << "'(" << op
                        << ")\n";
   });
-  if (RewriterBase::Listener *listener = config.getListener())
-    listener->notifyOperationModified(op);
+  if (config.listener)
+    config.listener->notifyOperationModified(op);
   addToWorklist(op);
 }
 
@@ -736,18 +736,18 @@ void GreedyPatternRewriteDriver::notifyOperationErased(Operation *op) {
   // the part of the IR that is taken into account for the "expensive checks".
   // A greedy pattern rewrite is not allowed to erase the parent op of the scope
   // region, as that would break the worklist handling and the expensive checks.
-  if (Region *scope = config.getScope(); scope->getParentOp() == op)
+  if (config.scope && config.scope->getParentOp() == op)
     llvm_unreachable(
         "scope region must not be erased during greedy pattern rewrite");
 #endif // NDEBUG
 
-  if (RewriterBase::Listener *listener = config.getListener())
-    listener->notifyOperationErased(op);
+  if (config.listener)
+    config.listener->notifyOperationErased(op);
 
   addOperandsToWorklist(op);
   worklist.remove(op);
 
-  if (config.getStrictness() != GreedyRewriteStrictness::AnyOp)
+  if (config.strictMode != GreedyRewriteStrictness::AnyOp)
     strictModeFilteredOps.erase(op);
 }
 
@@ -757,8 +757,8 @@ void GreedyPatternRewriteDriver::notifyOperationReplaced(
     logger.startLine() << "** Replace : '" << op->getName() << "'(" << op
                        << ")\n";
   });
-  if (RewriterBase::Listener *listener = config.getListener())
-    listener->notifyOperationReplaced(op, replacement);
+  if (config.listener)
+    config.listener->notifyOperationReplaced(op, replacement);
 }
 
 void GreedyPatternRewriteDriver::notifyMatchFailure(
@@ -768,8 +768,8 @@ void GreedyPatternRewriteDriver::notifyMatchFailure(
     reasonCallback(diag);
     logger.startLine() << "** Match Failure : " << diag.str() << "\n";
   });
-  if (RewriterBase::Listener *listener = config.getListener())
-    listener->notifyMatchFailure(loc, reasonCallback);
+  if (config.listener)
+    config.listener->notifyMatchFailure(loc, reasonCallback);
 }
 
 //===----------------------------------------------------------------------===//
@@ -800,7 +800,7 @@ RegionPatternRewriteDriver::RegionPatternRewriteDriver(
     const GreedyRewriteConfig &config, Region &region)
     : GreedyPatternRewriteDriver(ctx, patterns, config), region(region) {
   // Populate strict mode ops.
-  if (config.getStrictness() != GreedyRewriteStrictness::AnyOp) {
+  if (config.strictMode != GreedyRewriteStrictness::AnyOp) {
     region.walk([&](Operation *op) { strictModeFilteredOps.insert(op); });
   }
 }
@@ -829,8 +829,8 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
   MLIRContext *ctx = rewriter.getContext();
   do {
     // Check if the iteration limit was reached.
-    if (++iteration > config.getMaxIterations() &&
-        config.getMaxIterations() != GreedyRewriteConfig::kNoLimit)
+    if (++iteration > config.maxIterations &&
+        config.maxIterations != GreedyRewriteConfig::kNoLimit)
       break;
 
     // New iteration: start with an empty worklist.
@@ -849,16 +849,16 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
       return false;
     };
 
-    if (!config.getUseTopDownTraversal()) {
+    if (!config.useTopDownTraversal) {
       // Add operations to the worklist in postorder.
       region.walk([&](Operation *op) {
-        if (!config.isConstantCSEEnabled() || !insertKnownConstant(op))
+        if (!config.cseConstants || !insertKnownConstant(op))
           addToWorklist(op);
       });
     } else {
       // Add all nested operations to the worklist in preorder.
       region.walk<WalkOrder::PreOrder>([&](Operation *op) {
-        if (!config.isConstantCSEEnabled() || !insertKnownConstant(op)) {
+        if (!config.cseConstants || !insertKnownConstant(op)) {
           addToWorklist(op);
           return WalkResult::advance();
         }
@@ -875,11 +875,11 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
 
           // After applying patterns, make sure that the CFG of each of the
           // regions is kept up to date.
-          if (config.getRegionSimplificationLevel() !=
+          if (config.enableRegionSimplification !=
               GreedySimplifyRegionLevel::Disabled) {
             continueRewrites |= succeeded(simplifyRegions(
                 rewriter, region,
-                /*mergeBlocks=*/config.getRegionSimplificationLevel() ==
+                /*mergeBlocks=*/config.enableRegionSimplification ==
                     GreedySimplifyRegionLevel::Aggressive));
           }
         },
@@ -904,8 +904,8 @@ mlir::applyPatternsGreedily(Region &region,
          "patterns can only be applied to operations IsolatedFromAbove");
 
   // Set scope if not specified.
-  if (!config.getScope())
-    config.setScope(&region);
+  if (!config.scope)
+    config.scope = &region;
 
 #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
   if (failed(verify(config.scope->getParentOp())))
@@ -919,7 +919,7 @@ mlir::applyPatternsGreedily(Region &region,
   LogicalResult converged = std::move(driver).simplify(changed);
   LLVM_DEBUG(if (failed(converged)) {
     llvm::dbgs() << "The pattern rewrite did not converge after scanning "
-                 << config.getMaxIterations() << " times\n";
+                 << config.maxIterations << " times\n";
   });
   return converged;
 }
@@ -960,7 +960,7 @@ MultiOpPatternRewriteDriver::MultiOpPatternRewriteDriver(
     llvm::SmallDenseSet<Operation *, 4> *survivingOps)
     : GreedyPatternRewriteDriver(ctx, patterns, config),
       survivingOps(survivingOps) {
-  if (config.getStrictness() != GreedyRewriteStrictness::AnyOp)
+  if (config.strictMode != GreedyRewriteStrictness::AnyOp)
     strictModeFilteredOps.insert_range(ops);
 
   if (survivingOps) {
@@ -1024,15 +1024,15 @@ LogicalResult mlir::applyOpPatternsGreedily(
   }
 
   // Determine scope of rewrite.
-  if (!config.getScope()) {
+  if (!config.scope) {
     // Compute scope if none was provided. The scope will remain `nullptr` if
     // there is a top-level op among `ops`.
-    config.setScope(findCommonAncestor(ops));
+    config.scope = findCommonAncestor(ops);
   } else {
     // If a scope was provided, make sure that all ops are in scope.
 #ifndef NDEBUG
     bool allOpsInScope = llvm::all_of(ops, [&](Operation *op) {
-      return static_cast<bool>(config.getScope()->findAncestorOpInRegion(*op));
+      return static_cast<bool>(config.scope->findAncestorOpInRegion(*op));
     });
     assert(allOpsInScope && "ops must be within the specified scope");
 #endif // NDEBUG
@@ -1054,7 +1054,7 @@ LogicalResult mlir::applyOpPatternsGreedily(
     *allErased = surviving.empty();
   LLVM_DEBUG(if (failed(converged)) {
     llvm::dbgs() << "The pattern rewrite did not converge after "
-                 << config.getMaxNumRewrites() << " rewrites";
+                 << config.maxNumRewrites << " rewrites";
   });
   return converged;
 }
diff --git a/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp b/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
index 2a54e0c28f71f..d6aaa6faf94cb 100644
--- a/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
@@ -144,7 +144,7 @@ void TestAffineDataCopy::runOnOperation() {
     }
   }
   GreedyRewriteConfig config;
-  config.setStrictness(GreedyRewriteStrictness::ExistingAndNewOps);
+  config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
   (void)applyOpPatternsGreedily(copyOps, std::move(patterns), config);
 }
 
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index d073843484d81..db02a122872d9 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -386,26 +386,26 @@ struct TestGreedyPatternDriver
     patterns.insert<IncrementIntAttribute<3>>(&getContext());
 
     GreedyRewriteConfig config;
-    config.setUseTopDownTraversal(useTopDownTraversal)
-        .setMaxIterations(this->maxIterations)
-        .enableFolding(this->fold)
-        .enableConstantCSE(this->cseConstants);
+    config.useTopDownTraversal = this->useTopDownTraversal;
+    config.maxIterations = this->maxIterations;
+    config.fold = this->fold;
+    config.cseConstants = this->cseConstants;
     (void)applyPatternsGreedily(getOperation(), std::move(patterns), config);
   }
 
   Option<bool> useTopDownTraversal{
       *this, "top-down",
       llvm::cl::desc("Seed the worklist in general top-down order"),
-      llvm::cl::init(GreedyRewriteConfig().getUseTopDownTraversal())};
+      llvm::cl::init(GreedyRewriteConfig().useTopDownTraversal)};
   Option<int> maxIterations{
       *this, "max-iterations",
       llvm::cl::desc("Max. iterations in the GreedyRewriteConfig"),
-      llvm::cl::init(GreedyRewriteConfig().getMaxIterations())};
+      llvm::cl::init(GreedyRewriteConfig().maxIterations)};
   Option<bool> fold{*this, "fold", llvm::cl::desc("Whether to fold"),
-                    llvm::cl::init(GreedyRewriteConfig().isFoldingEnabled())};
-  Option<bool> cseConstants{
-      *this, "cse-constants", llvm::cl::desc("Whether to CSE constants"),
-      llvm::cl::init(GreedyRewriteConfig().isConstantCSEEnabled())};
+                    llvm::cl::init(GreedyRewriteConfig().fold)};
+  Option<bool> cseConstants{*this, "cse-constants",
+                            llvm::cl::desc("Whether to CSE constants"),
+                            llvm::cl::init(GreedyRewriteConfig().cseConstants)};
 };
 
 struct DumpNotifications : public RewriterBase::Listener {
@@ -501,13 +501,13 @@ struct TestStrictPatternDriver
 
     DumpNotifications dumpNotifications;
     GreedyRewriteConfig config;
-    config.setListener(&dumpNotifications);
+    config.listener = &dumpNotifications;
     if (strictMode == "AnyOp") {
-      config.setStrictness(GreedyRewriteStrictness::AnyOp);
+      config.strictMode = GreedyRewriteStrictness::AnyOp;
     } else if (strictMode == "ExistingAndNewOps") {
-      config.setStrictness(GreedyRewriteStrictness::ExistingAndNewOps);
+      config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
     } else if (strictMode == "ExistingOps") {
-      config.setStrictness(GreedyRewriteStrictness::ExistingOps);
+      config.strictMode = GreedyRewriteStrictness::ExistingOps;
     } else {
       llvm_unreachable("invalid strictness option");
     }

From 7da385d79729bb7aa1bfc046d3a78e350b7a4f75 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 09:52:28 -0700
Subject: [PATCH 428/710] Revert "[mlir] allow function type cloning to fail
 (#136300)"

This reverts commit 20a104a7d6423784dab04371a5ca728cc27a15a9.

Buildbot failure:
https://lab.llvm.org/buildbot/#/builders/157/builds/25688

I've reproduced the build failure.
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td |  3 +-
 .../mlir/Interfaces/FunctionInterfaces.td     | 81 +++++++------------
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |  8 +-
 .../Transforms/BufferResultsToOutParams.cpp   |  3 +-
 .../DropEquivalentBufferResults.cpp           |  3 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp      |  5 +-
 mlir/lib/Query/Query.cpp                      |  7 +-
 mlir/lib/Transforms/RemoveDeadValues.cpp      |  7 +-
 mlir/test/IR/test-func-erase-result.mlir      |  7 +-
 mlir/test/lib/IR/TestFunc.cpp                 | 25 ++----
 10 files changed, 44 insertions(+), 105 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
index 072fc7be07dd4..e9f01ba5cd4e4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td
@@ -104,8 +104,7 @@ def LLVMFunctionType : LLVMType<"LLVMFunction", "func"> {
     bool isVarArg() const { return getVarArg(); }
 
     /// Returns a clone of this function type with the given argument
-    /// and result types. Returns null if the resulting function type would
-    /// not verify.
+    /// and result types.
     LLVMFunctionType clone(TypeRange inputs, TypeRange results) const;
 
     /// Returns the result type of the function as an ArrayRef, enabling better
diff --git a/mlir/include/mlir/Interfaces/FunctionInterfaces.td b/mlir/include/mlir/Interfaces/FunctionInterfaces.td
index cad0e790fdd97..1e2ca9a8b4490 100644
--- a/mlir/include/mlir/Interfaces/FunctionInterfaces.td
+++ b/mlir/include/mlir/Interfaces/FunctionInterfaces.td
@@ -255,105 +255,79 @@ def FunctionOpInterface : OpInterface<"FunctionOpInterface", [
     BlockArgListType getArguments() { return getFunctionBody().getArguments(); }
 
     /// Insert a single argument of type `argType` with attributes `argAttrs` and
-    /// location `argLoc` at `argIndex`. Returns failure if the function cannot be
-    /// updated to have the new signature.
-    ::llvm::LogicalResult insertArgument(
-        unsigned argIndex, ::mlir::Type argType, ::mlir::DictionaryAttr argAttrs,
-        ::mlir::Location argLoc) {
-      return insertArguments({argIndex}, {argType}, {argAttrs}, {argLoc});
+    /// location `argLoc` at `argIndex`.
+    void insertArgument(unsigned argIndex, ::mlir::Type argType, ::mlir::DictionaryAttr argAttrs,
+                        ::mlir::Location argLoc) {
+      insertArguments({argIndex}, {argType}, {argAttrs}, {argLoc});
     }
 
     /// Inserts arguments with the listed types, attributes, and locations at the
     /// listed indices. `argIndices` must be sorted. Arguments are inserted in the
     /// order they are listed, such that arguments with identical index will
-    /// appear in the same order that they were listed here. Returns failure if
-    /// the function cannot be updated to have the new signature.
-    ::llvm::LogicalResult insertArguments(
-        ::llvm::ArrayRef<unsigned> argIndices, ::mlir::TypeRange argTypes,
-        ::llvm::ArrayRef<::mlir::DictionaryAttr> argAttrs,
-        ::llvm::ArrayRef<::mlir::Location> argLocs) {
+    /// appear in the same order that they were listed here.
+    void insertArguments(::llvm::ArrayRef<unsigned> argIndices, ::mlir::TypeRange argTypes,
+                        ::llvm::ArrayRef<::mlir::DictionaryAttr> argAttrs,
+                        ::llvm::ArrayRef<::mlir::Location> argLocs) {
       unsigned originalNumArgs = $_op.getNumArguments();
       ::mlir::Type newType = $_op.getTypeWithArgsAndResults(
           argIndices, argTypes, /*resultIndices=*/{}, /*resultTypes=*/{});
-      if (!newType)
-        return ::llvm::failure();
       ::mlir::function_interface_impl::insertFunctionArguments(
           $_op, argIndices, argTypes, argAttrs, argLocs,
           originalNumArgs, newType);
-      return ::llvm::success();
     }
 
-    /// Insert a single result of type `resultType` at `resultIndex`.Returns
-    /// failure if the function cannot be updated to have the new signature.
-    ::llvm::LogicalResult insertResult(
-        unsigned resultIndex, ::mlir::Type resultType,
-        ::mlir::DictionaryAttr resultAttrs) {
-      return insertResults({resultIndex}, {resultType}, {resultAttrs});
+    /// Insert a single result of type `resultType` at `resultIndex`.
+    void insertResult(unsigned resultIndex, ::mlir::Type resultType,
+                      ::mlir::DictionaryAttr resultAttrs) {
+      insertResults({resultIndex}, {resultType}, {resultAttrs});
     }
 
     /// Inserts results with the listed types at the listed indices.
     /// `resultIndices` must be sorted. Results are inserted in the order they are
     /// listed, such that results with identical index will appear in the same
-    /// order that they were listed here. Returns failure if the function
-    /// cannot be updated to have the new signature.
-    ::llvm::LogicalResult insertResults(
-        ::llvm::ArrayRef<unsigned> resultIndices,
-        ::mlir::TypeRange resultTypes,
-        ::llvm::ArrayRef<::mlir::DictionaryAttr> resultAttrs) {
+    /// order that they were listed here.
+    void insertResults(::llvm::ArrayRef<unsigned> resultIndices, ::mlir::TypeRange resultTypes,
+                       ::llvm::ArrayRef<::mlir::DictionaryAttr> resultAttrs) {
       unsigned originalNumResults = $_op.getNumResults();
       ::mlir::Type newType = $_op.getTypeWithArgsAndResults(
         /*argIndices=*/{}, /*argTypes=*/{}, resultIndices, resultTypes);
-      if (!newType)
-        return ::llvm::failure();
       ::mlir::function_interface_impl::insertFunctionResults(
           $_op, resultIndices, resultTypes, resultAttrs,
           originalNumResults, newType);
-      return ::llvm::success();
     }
 
-    /// Erase a single argument at `argIndex`. Returns failure if the function
-    /// cannot be updated to have the new signature.
-    ::llvm::LogicalResult eraseArgument(unsigned argIndex) {
+    /// Erase a single argument at `argIndex`.
+    void eraseArgument(unsigned argIndex) {
       ::llvm::BitVector argsToErase($_op.getNumArguments());
       argsToErase.set(argIndex);
-      return eraseArguments(argsToErase);
+      eraseArguments(argsToErase);
     }
 
-    /// Erases the arguments listed in `argIndices`. Returns failure if the
-    /// function cannot be updated to have the new signature.
-    ::llvm::LogicalResult eraseArguments(const ::llvm::BitVector &argIndices) {
+    /// Erases the arguments listed in `argIndices`.
+    void eraseArguments(const ::llvm::BitVector &argIndices) {
       ::mlir::Type newType = $_op.getTypeWithoutArgs(argIndices);
-      if (!newType)
-        return ::llvm::failure();
       ::mlir::function_interface_impl::eraseFunctionArguments(
         $_op, argIndices, newType);
-      return ::llvm::success();
     }
 
-    /// Erase a single result at `resultIndex`. Returns failure if the function
-    /// cannot be updated to have the new signature.
-    LogicalResult eraseResult(unsigned resultIndex) {
+    /// Erase a single result at `resultIndex`.
+    void eraseResult(unsigned resultIndex) {
       ::llvm::BitVector resultsToErase($_op.getNumResults());
       resultsToErase.set(resultIndex);
-      return eraseResults(resultsToErase);
+      eraseResults(resultsToErase);
     }
 
-    /// Erases the results listed in `resultIndices`.  Returns failure if the
-    /// function cannot be updated to have the new signature.
-    ::llvm::LogicalResult eraseResults(const ::llvm::BitVector &resultIndices) {
+    /// Erases the results listed in `resultIndices`.
+    void eraseResults(const ::llvm::BitVector &resultIndices) {
       ::mlir::Type newType = $_op.getTypeWithoutResults(resultIndices);
-      if (!newType)
-        return ::llvm::failure();
       ::mlir::function_interface_impl::eraseFunctionResults(
           $_op, resultIndices, newType);
-      return ::llvm::success();
     }
 
     /// Return the type of this function with the specified arguments and
     /// results inserted. This is used to update the function's signature in
     /// the `insertArguments` and `insertResults` methods. The arrays must be
-    /// sorted by increasing index. Return nullptr if the updated type would
-    /// not be valid.
+    /// sorted by increasing index.
     ::mlir::Type getTypeWithArgsAndResults(
       ::llvm::ArrayRef<unsigned> argIndices, ::mlir::TypeRange argTypes,
       ::llvm::ArrayRef<unsigned> resultIndices, ::mlir::TypeRange resultTypes) {
@@ -367,8 +341,7 @@ def FunctionOpInterface : OpInterface<"FunctionOpInterface", [
 
     /// Return the type of this function without the specified arguments and
     /// results. This is used to update the function's signature in the
-    /// `eraseArguments` and `eraseResults` methods. Return nullptr if the
-    /// updated type would not be valid.
+    /// `eraseArguments` and `eraseResults` methods.
     ::mlir::Type getTypeWithoutArgsAndResults(
       const ::llvm::BitVector &argIndices, const ::llvm::BitVector &resultIndices) {
       ::llvm::SmallVector<::mlir::Type> argStorage, resultStorage;
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index dea3d99704d9b..f22ad1fd70db2 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -125,12 +125,8 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
     // Perform signature modification
     rewriter.modifyOpInPlace(
         gpuFuncOp, [gpuFuncOp, &argIndices, &argTypes, &argAttrs, &argLocs]() {
-          LogicalResult inserted =
-              static_cast<FunctionOpInterface>(gpuFuncOp).insertArguments(
-                  argIndices, argTypes, argAttrs, argLocs);
-          (void)inserted;
-          assert(succeeded(inserted) &&
-                 "expected GPU funcs to support inserting any argument");
+          static_cast<FunctionOpInterface>(gpuFuncOp).insertArguments(
+              argIndices, argTypes, argAttrs, argLocs);
         });
   } else {
     workgroupBuffers.reserve(gpuFuncOp.getNumWorkgroupAttributions());
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index acf5f7767d12a..1c95ab77b9f33 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -92,8 +92,7 @@ updateFuncOp(func::FuncOp func,
   }
 
   // Erase the results.
-  if (failed(func.eraseResults(erasedResultIndices)))
-    return failure();
+  func.eraseResults(erasedResultIndices);
 
   // Add the new arguments to the entry block if the function is not external.
   if (func.isExternal())
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index ae011904cb972..9bc75267e70e4 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -113,8 +113,7 @@ mlir::bufferization::dropEquivalentBufferResults(ModuleOp module) {
     }
 
     // Update function.
-    if (failed(funcOp.eraseResults(erasedResultIndices)))
-      return failure();
+    funcOp.eraseResults(erasedResultIndices);
     returnOp.getOperandsMutable().assign(newReturnValues);
 
     // Update function calls.
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
index e2bccf7f6493e..29cf38c1fefea 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
@@ -232,10 +232,7 @@ LLVMFunctionType::getChecked(function_ref<InFlightDiagnostic()> emitError,
 
 LLVMFunctionType LLVMFunctionType::clone(TypeRange inputs,
                                          TypeRange results) const {
-  if (results.size() != 1 || !isValidResultType(results[0]))
-    return {};
-  if (!llvm::all_of(inputs, isValidArgumentType))
-    return {};
+  assert(results.size() == 1 && "expected a single result type");
   return get(results[0], llvm::to_vector(inputs), isVarArg());
 }
 
diff --git a/mlir/lib/Query/Query.cpp b/mlir/lib/Query/Query.cpp
index 73f313cd37fd0..869ee8f2ae1dc 100644
--- a/mlir/lib/Query/Query.cpp
+++ b/mlir/lib/Query/Query.cpp
@@ -88,11 +88,10 @@ static Operation *extractFunction(std::vector<Operation *> &ops,
   // Remove unused function arguments
   size_t currentIndex = 0;
   while (currentIndex < funcOp.getNumArguments()) {
-    // Erase if possible.
     if (funcOp.getArgument(currentIndex).use_empty())
-      if (succeeded(funcOp.eraseArgument(currentIndex)))
-        continue;
-    ++currentIndex;
+      funcOp.eraseArgument(currentIndex);
+    else
+      ++currentIndex;
   }
 
   return funcOp;
diff --git a/mlir/lib/Transforms/RemoveDeadValues.cpp b/mlir/lib/Transforms/RemoveDeadValues.cpp
index 4939dda4a20f9..92c05d87a002d 100644
--- a/mlir/lib/Transforms/RemoveDeadValues.cpp
+++ b/mlir/lib/Transforms/RemoveDeadValues.cpp
@@ -698,11 +698,8 @@ static void cleanUpDeadVals(RDVFinalCleanupList &list) {
 
   // 3. Functions
   for (auto &f : list.functions) {
-    // Some functions may not allow erasing arguments or results. These calls
-    // return failure in such cases without modifying the function, so it's okay
-    // to proceed.
-    (void)f.funcOp.eraseArguments(f.nonLiveArgs);
-    (void)f.funcOp.eraseResults(f.nonLiveRets);
+    f.funcOp.eraseArguments(f.nonLiveArgs);
+    f.funcOp.eraseResults(f.nonLiveRets);
   }
 
   // 4. Operands
diff --git a/mlir/test/IR/test-func-erase-result.mlir b/mlir/test/IR/test-func-erase-result.mlir
index a32866227547b..8fe40b2fdfcbc 100644
--- a/mlir/test/IR/test-func-erase-result.mlir
+++ b/mlir/test/IR/test-func-erase-result.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-func-erase-result -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt %s -test-func-erase-result -split-input-file | FileCheck %s
 
 // CHECK: func private @f(){{$}}
 // CHECK-NOT: attributes{{.*}}result
@@ -66,8 +66,3 @@ func.func private @f() -> (
   f32 {test.erase_this_result},
   tensor<3xf32>
 )
-
-// -----
-
-// expected-error @below {{failed to erase results}}
-llvm.func @llvm_func(!llvm.ptr, i64)
diff --git a/mlir/test/lib/IR/TestFunc.cpp b/mlir/test/lib/IR/TestFunc.cpp
index 94a4610365863..2ade47249863c 100644
--- a/mlir/test/lib/IR/TestFunc.cpp
+++ b/mlir/test/lib/IR/TestFunc.cpp
@@ -45,12 +45,8 @@ struct TestFuncInsertArg
                                    : unknownLoc);
       }
       func->removeAttr("test.insert_args");
-      if (succeeded(func.insertArguments(indicesToInsert, typesToInsert,
-                                         attrsToInsert, locsToInsert)))
-        continue;
-
-      emitError(func->getLoc()) << "failed to insert arguments";
-      return signalPassFailure();
+      func.insertArguments(indicesToInsert, typesToInsert, attrsToInsert,
+                           locsToInsert);
     }
   }
 };
@@ -83,12 +79,7 @@ struct TestFuncInsertResult
                                     : DictionaryAttr::get(&getContext()));
       }
       func->removeAttr("test.insert_results");
-      if (succeeded(func.insertResults(indicesToInsert, typesToInsert,
-                                       attrsToInsert)))
-        continue;
-
-      emitError(func->getLoc()) << "failed to insert results";
-      return signalPassFailure();
+      func.insertResults(indicesToInsert, typesToInsert, attrsToInsert);
     }
   }
 };
@@ -109,10 +100,7 @@ struct TestFuncEraseArg
       for (auto argIndex : llvm::seq<int>(0, func.getNumArguments()))
         if (func.getArgAttr(argIndex, "test.erase_this_arg"))
           indicesToErase.set(argIndex);
-      if (succeeded(func.eraseArguments(indicesToErase)))
-        continue;
-      emitError(func->getLoc()) << "failed to erase arguments";
-      return signalPassFailure();
+      func.eraseArguments(indicesToErase);
     }
   }
 };
@@ -134,10 +122,7 @@ struct TestFuncEraseResult
       for (auto resultIndex : llvm::seq<int>(0, func.getNumResults()))
         if (func.getResultAttr(resultIndex, "test.erase_this_result"))
           indicesToErase.set(resultIndex);
-      if (succeeded(func.eraseResults(indicesToErase)))
-        continue;
-      emitError(func->getLoc()) << "failed to erase results";
-      return signalPassFailure();
+      func.eraseResults(indicesToErase);
     }
   }
 };

From 46d1cb8335e988e2a9e48b46d80c639a2364a8f9 Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin@gmail.com>
Date: Fri, 18 Apr 2025 18:53:24 +0200
Subject: [PATCH 429/710] [mlir] GPUToROCDL: Add support for non-i32/f32
 shuffle types (#136320)

Use recently added repacking utilities to support other datatypes.

Also, tighten `gpu.shuffle` verification to reject scalable vectors
---
 mlir/include/mlir/Dialect/GPU/IR/GPUOps.td    |  4 +--
 .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp      | 26 +++++++++----------
 .../GPUToROCDL/gpu-to-rocdl-unsupported.mlir  | 13 ----------
 .../Conversion/GPUToROCDL/gpu-to-rocdl.mlir   | 21 +++++++++++++++
 mlir/test/Dialect/GPU/invalid.mlir            | 14 +++++++---
 5 files changed, 46 insertions(+), 32 deletions(-)
 delete mode 100644 mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-unsupported.mlir

diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 3241aff4b683c..68095b7bf5c59 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -840,7 +840,7 @@ def GPU_LaunchOp : GPU_Op<"launch", [
     -   a variadic number of Private memory attributions.
 
     The `kernelFunc` and `kernelModule` attributes are optional and specifies
-    the kernel name and a module in which the kernel should be outlined. 
+    the kernel name and a module in which the kernel should be outlined.
 
     Syntax:
 
@@ -1201,7 +1201,7 @@ def GPU_AllReduceOp : GPU_Op<"all_reduce",
 }
 
 def AnyIntegerOrFloatOr1DVector :
-  AnyTypeOf<[AnyIntegerOrFloat, VectorOfRankAndType<[1], [AnyIntegerOrFloat]>]>;
+  AnyTypeOf<[AnyIntegerOrFloat, FixedVectorOfRankAndType<[1], [AnyIntegerOrFloat]>]>;
 
 def GPU_SubgroupReduceOp : GPU_Op<"subgroup_reduce", [SameOperandsAndResultType]> {
   let summary = "Reduce values among subgroup.";
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index c6c695b442b4f..34129989049d0 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -62,8 +62,8 @@ static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func) {
   return canBeBare;
 }
 
-Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
-                const unsigned indexBitwidth) {
+static Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
+                       const unsigned indexBitwidth) {
   auto int32Type = IntegerType::get(rewriter.getContext(), 32);
   Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 32);
   Value minus1 = rewriter.create<arith::ConstantIntOp>(loc, -1, 32);
@@ -138,10 +138,6 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     Location loc = op->getLoc();
     Value initShflValue = adaptor.getValue();
     Type shflType = initShflValue.getType();
-    // TODO: Add support for non 32-bit shuffle values.
-    if (!shflType.isIntOrFloat() || shflType.getIntOrFloatBitWidth() != 32)
-      return rewriter.notifyMatchFailure(
-          op, "only 32-bit int/float types are supported");
 
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     Value srcLaneId = getLaneId(rewriter, loc, indexBitwidth);
@@ -179,15 +175,17 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     Value two = rewriter.create<LLVM::ConstantOp>(loc, int32Type, 2);
     Value dwordAlignedDstLane =
         rewriter.create<LLVM::ShlOp>(loc, int32Type, selectDstLane, two);
-    if (shflType.isF32()) {
-      initShflValue =
-          rewriter.create<LLVM::BitcastOp>(loc, int32Type, initShflValue);
-    }
-    Value shflValue = rewriter.create<ROCDL::DsBpermuteOp>(
-        loc, int32Type, dwordAlignedDstLane, initShflValue);
-    if (shflType.isF32()) {
-      shflValue = rewriter.create<LLVM::BitcastOp>(loc, shflType, shflValue);
+
+    SmallVector<Value> decomposed =
+        LLVM::decomposeValue(rewriter, loc, initShflValue, int32Type);
+    SmallVector<Value> swizzled;
+    for (Value v : decomposed) {
+      Value res = rewriter.create<ROCDL::DsBpermuteOp>(loc, int32Type,
+                                                       dwordAlignedDstLane, v);
+      swizzled.emplace_back(res);
     }
+    Value shflValue =
+        LLVM::composeValue(rewriter, loc, swizzled, initShflValue.getType());
     rewriter.replaceOp(op, {shflValue, isActiveSrcLane});
     return success();
   }
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-unsupported.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-unsupported.mlir
deleted file mode 100644
index 90f2e5f047cd9..0000000000000
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl-unsupported.mlir
+++ /dev/null
@@ -1,13 +0,0 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl -verify-diagnostics
-
-gpu.module @test_module {
-  // ROCDL lowering only suport shuffles for 32bit ints/floats, but they
-  // shouldn't crash on unsupported types.
-  func.func @gpu_shuffle_unsupported(%arg0 : vector<4xf16>) -> vector<4xf16> {
-    %offset = arith.constant 4 : i32
-    %width = arith.constant 64 : i32
-    // expected-error @+1 {{failed to legalize operation 'gpu.shuffle'}}
-    %shfl, %pred = gpu.shuffle xor %arg0, %offset, %width : vector<4xf16>
-    return %shfl : vector<4xf16>
-  }
-}
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
index e23ab16ccd94b..071cae9d5789f 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
@@ -710,6 +710,27 @@ gpu.module @test_module {
     %shfld, %predd = gpu.shuffle down %arg0, %arg1, %arg2 : f32
     func.return %shfl, %shfli, %shfld : f32, f32, f32
   }
+
+  // CHECK-LABEL: func @gpu_shuffle_vec
+  //  CHECK-SAME: (%[[ARG:.*]]: vector<4xf16>, %{{.*}}: i32, %{{.*}}: i32)
+  func.func @gpu_shuffle_vec(%arg0: vector<4xf16>, %arg1: i32, %arg2: i32) -> vector<4xf16> {
+    // CHECK: %[[CAST1:.*]] = llvm.bitcast %[[ARG]] : vector<4xf16> to vector<2xi32>
+    // CHECK: %[[IDX0:.*]] = llvm.mlir.constant(0 : i32) : i32
+    // CHECK: %[[ELEM0:.*]] = llvm.extractelement %13[%[[IDX0]] : i32] : vector<2xi32>
+    // CHECK: %[[IDX1:.*]] = llvm.mlir.constant(1 : i32) : i32
+    // CHECK: %[[ELEM1:.*]] = llvm.extractelement %13[%[[IDX1]] : i32] : vector<2xi32>
+    // CHECK: %[[PERM0:.*]] = rocdl.ds_bpermute %{{.*}}, %[[ELEM0]] : (i32, i32) -> i32
+    // CHECK: %[[PERM1:.*]] = rocdl.ds_bpermute %{{.*}}, %[[ELEM1]] : (i32, i32) -> i32
+    // CHECK: %[[V0:.*]] = llvm.mlir.poison : vector<2xi32>
+    // CHECK: %[[IDX0:.*]] = llvm.mlir.constant(0 : i32) : i32
+    // CHECK: %[[V1:.*]] = llvm.insertelement %[[PERM0]], %[[V0]][%[[IDX0]] : i32] : vector<2xi32>
+    // CHECK: %[[IDX1:.*]] = llvm.mlir.constant(1 : i32) : i32
+    // CHECK: %[[V2:.*]] = llvm.insertelement %[[PERM1]], %[[V1]][%[[IDX1]] : i32] : vector<2xi32>
+    // CHECK: %[[RES:.*]] = llvm.bitcast %[[V2]] : vector<2xi32> to vector<4xf16>
+    // CHECK: llvm.return %[[RES]] : vector<4xf16>
+    %shfl, %pred = gpu.shuffle xor %arg0, %arg1, %arg2 : vector<4xf16>
+    func.return %shfl : vector<4xf16>
+  }
 }
 
 // -----
diff --git a/mlir/test/Dialect/GPU/invalid.mlir b/mlir/test/Dialect/GPU/invalid.mlir
index 16148a493ce6e..ce1be7b5618fe 100644
--- a/mlir/test/Dialect/GPU/invalid.mlir
+++ b/mlir/test/Dialect/GPU/invalid.mlir
@@ -367,7 +367,7 @@ func.func @subgroup_reduce_cluster_stride_without_size(%arg0 : vector<4xf32>) {
 // -----
 
 func.func @subgroup_reduce_bad_type(%arg0 : vector<2x2xf32>) {
-  // expected-error@+1 {{'gpu.subgroup_reduce' op operand #0 must be Integer or Float or vector of}}
+  // expected-error@+1 {{'gpu.subgroup_reduce' op operand #0 must be Integer or Float or fixed-length vector of}}
   %res = gpu.subgroup_reduce add %arg0 : (vector<2x2xf32>) -> vector<2x2xf32>
   return
 }
@@ -375,7 +375,7 @@ func.func @subgroup_reduce_bad_type(%arg0 : vector<2x2xf32>) {
 // -----
 
 func.func @subgroup_reduce_bad_type_scalable(%arg0 : vector<[2]xf32>) {
-  // expected-error@+1 {{is not compatible with scalable vector types}}
+  // expected-error@+1 {{'gpu.subgroup_reduce' op operand #0 must be Integer or Float or fixed-length vector of}}
   %res = gpu.subgroup_reduce add %arg0 : (vector<[2]xf32>) -> vector<[2]xf32>
   return
 }
@@ -463,13 +463,21 @@ func.func @shuffle_mismatching_type(%arg0 : f32, %arg1 : i32, %arg2 : i32) {
 // -----
 
 func.func @shuffle_unsupported_type(%arg0 : index, %arg1 : i32, %arg2 : i32) {
-  // expected-error@+1 {{op operand #0 must be Integer or Float or vector of Integer or Float values of ranks 1, but got 'index'}}
+  // expected-error@+1 {{op operand #0 must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1, but got 'index'}}
   %shfl, %pred = gpu.shuffle xor %arg0, %arg1, %arg2 : index
   return
 }
 
 // -----
 
+func.func @shuffle_unsupported_type_vec(%arg0 : vector<[4]xf32>, %arg1 : i32, %arg2 : i32) {
+  // expected-error@+1 {{op operand #0 must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1, but got 'vector<[4]xf32>'}}
+  %shfl, %pred = gpu.shuffle xor %arg0, %arg1, %arg2 : vector<[4]xf32>
+  return
+}
+
+// -----
+
 module {
   gpu.module @gpu_funcs {
     // expected-error @+1 {{custom op 'gpu.func' gpu.func requires named arguments}}

From 4c17a5c66346089bf445176c77b36d6ed91d09bd Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 09:59:19 -0700
Subject: [PATCH 430/710] [mlir] Fix a warning

This patch fixes:

  mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp:140:10:
  error: unused variable 'shflType' [-Werror,-Wunused-variable]
---
 mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 34129989049d0..e6dd6f135884e 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -137,7 +137,6 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op->getLoc();
     Value initShflValue = adaptor.getValue();
-    Type shflType = initShflValue.getType();
 
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
     Value srcLaneId = getLaneId(rewriter, loc, indexBitwidth);

From f696508b38d1958aac5992a98a90fe6e773e8709 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Fri, 18 Apr 2025 17:02:39 +0200
Subject: [PATCH 431/710] [BFI] Revert use of append_range (NFC)

This is a partial revert of #136066, which introduced a compile-time
regression. SmallVector::insert() computes the PostOrderIterator
twice in order to call reserve().
---
 llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 2237f105e328b..66ea5ecbc4bfb 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -1160,7 +1160,7 @@ void BlockFrequencyInfoImpl<BT>::setBlockFreq(const BlockT *BB,
 template <class BT> void BlockFrequencyInfoImpl<BT>::initializeRPOT() {
   const BlockT *Entry = &F->front();
   RPOT.reserve(F->size());
-  llvm::append_range(RPOT, post_order(Entry));
+  std::copy(po_begin(Entry), po_end(Entry), std::back_inserter(RPOT));
   std::reverse(RPOT.begin(), RPOT.end());
 
   assert(RPOT.size() - 1 <= BlockNode::getMaxIndex() &&

From 5eabececb0a3c7e761ddda2e49430baa834f2c65 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 18 Apr 2025 12:00:58 -0500
Subject: [PATCH 432/710] [Offload] Fix JIT test

---
 offload/test/jit/empty_kernel.inc    |  6 +--
 offload/test/jit/empty_kernel_lvl2.c | 71 ++--------------------------
 2 files changed, 5 insertions(+), 72 deletions(-)

diff --git a/offload/test/jit/empty_kernel.inc b/offload/test/jit/empty_kernel.inc
index 72de3c140c07b..b4bf83fd2ec6c 100644
--- a/offload/test/jit/empty_kernel.inc
+++ b/offload/test/jit/empty_kernel.inc
@@ -33,9 +33,7 @@ int main(int argc, char** argv) {
 }
 
 // Check for an empty kernel (IR level)
-// FIRST:       define weak_odr {{.*}} void @__omp_offloading_{{.*}}_main{{.*}}
-// FIRST-NEXT:    ret void
+// FIRST:    define weak_odr {{.*}} void @__omp_offloading_{{.*}}_main{{.*}}
 
 // Check for two empty kernels (IR level)
-// SECOND:      define weak_odr {{.*}} void @__omp_offloading_{{.*}}_main{{.*}}
-// SECOND-NEXT:   ret void
+// SECOND:   define weak_odr {{.*}} void @__omp_offloading_{{.*}}_main{{.*}}
diff --git a/offload/test/jit/empty_kernel_lvl2.c b/offload/test/jit/empty_kernel_lvl2.c
index 90e4ce321fe8a..9e2ef17a42fbd 100644
--- a/offload/test/jit/empty_kernel_lvl2.c
+++ b/offload/test/jit/empty_kernel_lvl2.c
@@ -1,12 +1,12 @@
 // clang-format off
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
+// RUN: %libomptarget-compileopt-generic -fopenmp-target-jit \
 // RUN:     -DTGT1_DIRECTIVE="target"                          \
 // RUN:     -DLOOP_DIRECTIVE="for"
 // RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
 // RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
 // RUN:     %libomptarget-run-generic
 // RUN: %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefix=FIRST
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
+// RUN: %libomptarget-compileopt-generic -fopenmp-target-jit \
 // RUN:     -DTGT1_DIRECTIVE="target"                          \
 // RUN:     -DTGT2_DIRECTIVE="target"                          \
 // RUN:     -DLOOP_DIRECTIVE="for"
@@ -15,77 +15,12 @@
 // RUN:     %libomptarget-run-generic
 // RUN: %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefixes=FIRST,SECOND
 //
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
+// RUN: %libomptarget-compileopt-generic -fopenmp-target-jit \
 // RUN:     -DTGT1_DIRECTIVE="target"                          \
 // RUN:     -DLOOP_DIRECTIVE="parallel for"
 // RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
 // RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
 // RUN:     %libomptarget-run-generic
-// TODO:
-// RUN: not %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefix=FIRST
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
-// RUN:     -DTGT1_DIRECTIVE="target"                          \
-// RUN:     -DTGT2_DIRECTIVE="target teams"                    \
-// RUN:     -DLOOP_DIRECTIVE="parallel for"
-// RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
-// RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
-// RUN:     %libomptarget-run-generic
-// TODO:
-// RUN: not %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefixes=FIRST,SECOND
-//
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
-// RUN:     -DTGT1_DIRECTIVE="target teams"                    \
-// RUN:     -DLOOP_DIRECTIVE="distribute"
-// RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
-// RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
-// RUN:     %libomptarget-run-generic
-// TODO:
-// RUN: not %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefix=FIRST
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
-// RUN:     -DTGT1_DIRECTIVE="target teams"                    \
-// RUN:     -DTGT2_DIRECTIVE="target teams"                    \
-// RUN:     -DLOOP_DIRECTIVE="distribute"
-// RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
-// RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
-// RUN:     %libomptarget-run-generic
-// TODO:
-// RUN: not %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefixes=FIRST,SECOND
-//
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
-// RUN:     -DTGT1_DIRECTIVE="target teams"                    \
-// RUN:     -DLOOP_DIRECTIVE="distribute parallel for"
-// RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
-// RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
-// RUN:     %libomptarget-run-generic
-// TODO:
-// RUN: not %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefix=FIRST
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
-// RUN:     -DTGT1_DIRECTIVE="target teams"                    \
-// RUN:     -DTGT2_DIRECTIVE="target teams"                    \
-// RUN:     -DLOOP_DIRECTIVE="distribute parallel for"
-// RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
-// RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
-// RUN:     %libomptarget-run-generic
-// TODO:
-// RUN: not %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefixes=FIRST,SECOND
-//
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
-// RUN:     -DTGT1_DIRECTIVE="target teams"                    \
-// RUN:     -DLOOP_DIRECTIVE="distribute parallel for simd"
-// RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
-// RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
-// RUN:     %libomptarget-run-generic
-// TODO:
-// RUN: not %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefix=FIRST
-// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-jit \
-// RUN:     -DTGT1_DIRECTIVE="target teams"                    \
-// RUN:     -DTGT2_DIRECTIVE="target teams"                    \
-// RUN:     -DLOOP_DIRECTIVE="distribute parallel for simd"
-// RUN: env LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=%t.pre.ll     \
-// RUN:     LIBOMPTARGET_JIT_SKIP_OPT=true                   \
-// RUN:     %libomptarget-run-generic
-// TODO:
-// RUN: not %fcheck-plain-generic --input-file %t.pre.ll %S/empty_kernel.inc --check-prefixes=FIRST,SECOND
 // clang-format on
 
 // REQUIRES: gpu

From d3d9b376c01ac4dff1457a98de81eca5cbb648a1 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 10:04:41 -0700
Subject: [PATCH 433/710] [DebugInfo] Call std::erase directly (NFC) (#136261)

---
 llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
index a8bf33f9ad6b2..b60f9e90b4f1c 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
@@ -50,11 +50,7 @@ namespace logicalview {
     return FAMILY.SET.find(TYPE::FIELD) != FAMILY.SET.end();                   \
   }                                                                            \
   void set##FAMILY##FIELD() { FAMILY.SET.insert(TYPE::FIELD); }                \
-  void reset##FAMILY##FIELD() {                                                \
-    std::set<TYPE>::iterator Iter = FAMILY.SET.find(TYPE::FIELD);              \
-    if (Iter != FAMILY.SET.end())                                              \
-      FAMILY.SET.erase(Iter);                                                  \
-  }
+  void reset##FAMILY##FIELD() { FAMILY.SET.erase(TYPE::FIELD); }
 
 #define STDSET_FUNCTION_5(FAMILY, FIELD, ENTRY, TYPE, SET)                     \
   bool get##FAMILY##FIELD##ENTRY() const {                                     \

From 2a692d265bbd84dae807d470caed7be507e4118d Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 10:05:18 -0700
Subject: [PATCH 434/710] [DebugInfo] Use std::move (NFC) (#136263)

---
 llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
index b60f9e90b4f1c..ac0dfba0f1ede 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
@@ -41,7 +41,9 @@ namespace logicalview {
 // Generate get and set 'std::string' functions.
 #define STD_STRING_FUNCTION(FAMILY, FIELD)                                     \
   std::string get##FAMILY##FIELD() const { return FAMILY.FIELD; }              \
-  void set##FAMILY##FIELD(std::string FIELD) { FAMILY.FIELD = FIELD; }         \
+  void set##FAMILY##FIELD(std::string FIELD) {                                 \
+    FAMILY.FIELD = std::move(FIELD);                                           \
+  }                                                                            \
   void reset##FAMILY##FIELD() { FAMILY.FIELD = ""; }
 
 // Generate get and set 'std::set' functions.

From c244daec1cbcd221c073fc40284ee23399257684 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Fri, 18 Apr 2025 10:05:40 -0700
Subject: [PATCH 435/710] [LLVM][TableGen] Fix Windows failure in
 DecoderEmitter (#136310)

- Avoid dereferencing the end() iterator to get the end pointer, instead
calculate it explicitly
- Fixes a regression introduced in
https://github.com/llvm/llvm-project/pull/136220.
- The windows build failure shows the following call stack:

```
 | Exception Code: 0x80000003
 |  #0 0x00007ff74bc05897 std::_Vector_const_iterator<class std::_Vector_val<struct std::_Simple_types<unsigned char>>>::operator*(void) const C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.37.32822\include\vector:52:0
 |  #1 0x00007ff74bbd3d64 `anonymous namespace'::DecoderEmitter::emitTable D:\buildbot\llvm-worker\clang-cmake-x86_64-avx512-win\llvm\llvm\utils\TableGen\DecoderEmitter.cpp:852:0
```
---
 llvm/utils/TableGen/DecoderEmitter.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index a3fe49ef25de7..485647d4e5910 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -833,6 +833,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
   // appropriate indentation levels.
   DecoderTable::const_iterator I = Table.begin();
   DecoderTable::const_iterator E = Table.end();
+  const uint8_t *const EndPtr = Table.data() + Table.size();
   while (I != E) {
     assert(I < E && "incomplete decode table entry!");
 
@@ -849,7 +850,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
 
       // ULEB128 encoded start value.
       const char *ErrMsg = nullptr;
-      unsigned Start = decodeULEB128(&*I, nullptr, &*E, &ErrMsg);
+      unsigned Start = decodeULEB128(&*I, nullptr, EndPtr, &ErrMsg);
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
       emitULEB128(I, OS);
 
@@ -903,7 +904,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       ++I;
       // Decode the Opcode value.
       const char *ErrMsg = nullptr;
-      unsigned Opc = decodeULEB128(&*I, nullptr, &*E, &ErrMsg);
+      unsigned Opc = decodeULEB128(&*I, nullptr, EndPtr, &ErrMsg);
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
 
       OS << Indent << "MCD::OPC_" << (IsTry ? "Try" : "") << "Decode, ";
@@ -935,12 +936,12 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       OS << Indent << "MCD::OPC_SoftFail, ";
       // Decode the positive mask.
       const char *ErrMsg = nullptr;
-      uint64_t PositiveMask = decodeULEB128(&*I, nullptr, &*E, &ErrMsg);
+      uint64_t PositiveMask = decodeULEB128(&*I, nullptr, EndPtr, &ErrMsg);
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
       emitULEB128(I, OS);
 
       // Decode the negative mask.
-      uint64_t NegativeMask = decodeULEB128(&*I, nullptr, &*E, &ErrMsg);
+      uint64_t NegativeMask = decodeULEB128(&*I, nullptr, EndPtr, &ErrMsg);
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
       emitULEB128(I, OS);
       OS << "// +ve mask: 0x";

From 5e1b0f97735083b6762834b83fdbb35e76002e03 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 10:05:55 -0700
Subject: [PATCH 436/710] [llvm] Use llvm::less_first and llvm::less_second
 (NFC) (#136272)

---
 llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp       | 8 ++------
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 9 ++-------
 llvm/tools/llvm-gpu-loader/nvptx.cpp            | 4 ++--
 3 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 153b14ce60507..815715604bc96 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -588,12 +588,8 @@ void PipelineSolver::populateReadyList(
       ReadyList.push_back(std::pair(*I, -1));
   }
 
-  if (UseCostHeur) {
-    std::sort(ReadyList.begin(), ReadyList.end(),
-              [](std::pair<int, int> A, std::pair<int, int> B) {
-                return A.second < B.second;
-              });
-  }
+  if (UseCostHeur)
+    llvm::sort(ReadyList, llvm::less_second());
 
   assert(ReadyList.size() == CurrSU.second.size());
 }
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 13c1eb572ef92..4301bc7d0d68b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6466,10 +6466,7 @@ static bool clusterSortPtrAccesses(ArrayRef<Value *> VL,
   for (auto &Base : Bases) {
     for (auto &Vec : Base.second) {
       if (Vec.size() > 1) {
-        stable_sort(Vec, [](const std::tuple<Value *, int, unsigned> &X,
-                            const std::tuple<Value *, int, unsigned> &Y) {
-          return std::get<1>(X) < std::get<1>(Y);
-        });
+        stable_sort(Vec, llvm::less_second());
         int InitialOffset = std::get<1>(Vec[0]);
         bool AnyConsecutive =
             all_of(enumerate(Vec), [InitialOffset](const auto &P) {
@@ -7986,9 +7983,7 @@ bool BoUpSLP::canFormVector(ArrayRef<StoreInst *> StoresVec,
   // Check if the stores are consecutive by checking if their difference is 1.
   if (StoreOffsetVec.size() != StoresVec.size())
     return false;
-  sort(StoreOffsetVec,
-       [](const std::pair<int, unsigned> &L,
-          const std::pair<int, unsigned> &R) { return L.first < R.first; });
+  sort(StoreOffsetVec, llvm::less_first());
   unsigned Idx = 0;
   int PrevDist = 0;
   for (const auto &P : StoreOffsetVec) {
diff --git a/llvm/tools/llvm-gpu-loader/nvptx.cpp b/llvm/tools/llvm-gpu-loader/nvptx.cpp
index 13c62d50e6077..781a045e71a94 100644
--- a/llvm/tools/llvm-gpu-loader/nvptx.cpp
+++ b/llvm/tools/llvm-gpu-loader/nvptx.cpp
@@ -84,8 +84,8 @@ Expected<void *> get_ctor_dtor_array(const void *image, const size_t size,
   }
   // Lower priority constructors are run before higher ones. The reverse is true
   // for destructors.
-  llvm::sort(ctors, [](auto x, auto y) { return x.second < y.second; });
-  llvm::sort(dtors, [](auto x, auto y) { return x.second < y.second; });
+  llvm::sort(ctors, llvm::less_second());
+  llvm::sort(dtors, llvm::less_second());
 
   // Allocate host pinned memory to make these arrays visible to the GPU.
   CUdeviceptr *dev_memory = reinterpret_cast<CUdeviceptr *>(allocator(

From f5947ba5a659d6b3ed62761a7afdf1626435abcb Mon Sep 17 00:00:00 2001
From: Erich Keane <ekeane@nvidia.com>
Date: Fri, 18 Apr 2025 10:06:52 -0700
Subject: [PATCH 437/710] Revert "[clang] Handle instantiated members to
 determine visibility" (#136317)

Reverts llvm/llvm-project#136128

See discussion here:
https://github.com/llvm/llvm-project/pull/136128#issuecomment-2815648110
and :
https://github.com/llvm/llvm-project/pull/136128#issuecomment-2815652846

@asavonic can re-submit once the examples provided by @jplehr and/or
@DKLoehr are fixed.
---
 clang/lib/AST/Decl.cpp               | 13 +++-------
 clang/test/CodeGenCXX/visibility.cpp | 38 +---------------------------
 2 files changed, 4 insertions(+), 47 deletions(-)

diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index b59619892979a..ad1cb01592e9b 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -400,9 +400,9 @@ void LinkageComputer::mergeTemplateLV(
   FunctionTemplateDecl *temp = specInfo->getTemplate();
   // Merge information from the template declaration.
   LinkageInfo tempLV = getLVForDecl(temp, computation);
-  // The linkage and visibility of the specialization should be
-  // consistent with the template declaration.
-  LV.mergeMaybeWithVisibility(tempLV, considerVisibility);
+  // The linkage of the specialization should be consistent with the
+  // template declaration.
+  LV.setLinkage(tempLV.getLinkage());
 
   // Merge information from the template parameters.
   LinkageInfo paramsLV =
@@ -1051,13 +1051,6 @@ LinkageComputer::getLVForClassMember(const NamedDecl *D,
     if (const auto *redeclTemp = dyn_cast<RedeclarableTemplateDecl>(temp)) {
       if (isExplicitMemberSpecialization(redeclTemp)) {
         explicitSpecSuppressor = temp->getTemplatedDecl();
-      } else if (const RedeclarableTemplateDecl *from =
-                     redeclTemp->getInstantiatedFromMemberTemplate()) {
-        // If no explicit visibility is specified yet, and this is an
-        // instantiated member of a template, look up visibility there
-        // as well.
-        LinkageInfo fromLV = from->getLinkageAndVisibility();
-        LV.mergeMaybeWithVisibility(fromLV, considerVisibility);
       }
     }
   }
diff --git a/clang/test/CodeGenCXX/visibility.cpp b/clang/test/CodeGenCXX/visibility.cpp
index b69278a71d48e..e1061f3dbd18f 100644
--- a/clang/test/CodeGenCXX/visibility.cpp
+++ b/clang/test/CodeGenCXX/visibility.cpp
@@ -1457,45 +1457,9 @@ namespace test71 {
   // CHECK-LABEL: declare hidden noundef i32 @_ZN6test713fooIiE3zedEv(
   // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN6test713fooIiE3barIiEET_v(
   // CHECK-LABEL: define linkonce_odr hidden noundef i64 @_ZN6test713fooIlE3zedEv(
-  // CHECK-LABEL: define linkonce_odr hidden noundef i32 @_ZN6test713fooIlE3barIiEET_v(
+  // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN6test713fooIlE3barIiEET_v(
   // CHECK-HIDDEN-LABEL: declare hidden noundef i32 @_ZN6test713fooIiE3zedEv(
   // CHECK-HIDDEN-LABEL: define linkonce_odr noundef i32 @_ZN6test713fooIiE3barIiEET_v(
   // CHECK-HIDDEN-LABEL: define linkonce_odr hidden noundef i64 @_ZN6test713fooIlE3zedEv(
   // CHECK-HIDDEN-LABEL: define linkonce_odr hidden noundef i32 @_ZN6test713fooIlE3barIiEET_v(
 }
-
-// https://github.com/llvm/llvm-project/issues/103477
-namespace test72 {
-  template <class a>
-  struct t {
-    template <int>
-    static HIDDEN void bar() {}
-  };
-
-  void test() {
-      t<char>::bar<1>();
-  }
-  // CHECK-LABEL: define linkonce_odr hidden void @_ZN6test721tIcE3barILi1EEEvv(
-  // CHECK-HIDDEN-LABEL: define linkonce_odr hidden void @_ZN6test721tIcE3barILi1EEEvv(
-}
-
-// https://github.com/llvm/llvm-project/issues/31462
-namespace test73 {
-  template <class T> struct s {
-    template <class U>
-    __attribute__((__visibility__("hidden"))) U should_not_be_exported();
-  };
-
-  template <class T> template <class U> U s<T>::should_not_be_exported() {
-    return U();
-  }
-
-  extern template struct __attribute__((__visibility__("default"))) s<int>;
-
-  int f() {
-    s<int> o;
-    return o.should_not_be_exported<int>();
-  }
-  // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN6test731sIiE22should_not_be_exportedIiEET_v(
-  // CHECK-HIDDEN-LABEL: define linkonce_odr noundef i32 @_ZN6test731sIiE22should_not_be_exportedIiEET_v(
-}

From 61820156980ff20ca6e8a7a43a1e9debb950db85 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Fri, 18 Apr 2025 10:08:00 -0700
Subject: [PATCH 438/710] [NFC][LLVM][TableGen] Adjust pointer increments in
 DecoderEmitter (#136230)

- In both `emitTable` and the generated `decodeInstruction` function
increment the pointer to the decoder op as a part of the switch
statement instead of later on in each case.
---
 llvm/utils/TableGen/DecoderEmitter.cpp | 30 +++++++++++---------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 485647d4e5910..7d63126f65ac4 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -841,11 +841,11 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     OS << "/* " << Pos << " */";
     OS.PadToColumn(12);
 
-    switch (*I) {
+    const uint8_t DecoderOp = *I++;
+    switch (DecoderOp) {
     default:
-      PrintFatalError("invalid decode table opcode");
+      PrintFatalError("Invalid decode table opcode: " + Twine(DecoderOp));
     case MCD::OPC_ExtractField: {
-      ++I;
       OS << Indent << "MCD::OPC_ExtractField, ";
 
       // ULEB128 encoded start value.
@@ -862,7 +862,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       break;
     }
     case MCD::OPC_FilterValue: {
-      ++I;
       OS << Indent << "MCD::OPC_FilterValue, ";
       // The filter value is ULEB128 encoded.
       emitULEB128(I, OS);
@@ -873,7 +872,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       break;
     }
     case MCD::OPC_CheckField: {
-      ++I;
       OS << Indent << "MCD::OPC_CheckField, ";
       // ULEB128 encoded start value.
       emitULEB128(I, OS);
@@ -889,7 +887,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       break;
     }
     case MCD::OPC_CheckPredicate: {
-      ++I;
       OS << Indent << "MCD::OPC_CheckPredicate, ";
       emitULEB128(I, OS);
 
@@ -900,8 +897,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     }
     case MCD::OPC_Decode:
     case MCD::OPC_TryDecode: {
-      bool IsTry = *I == MCD::OPC_TryDecode;
-      ++I;
+      bool IsTry = DecoderOp == MCD::OPC_TryDecode;
       // Decode the Opcode value.
       const char *ErrMsg = nullptr;
       unsigned Opc = decodeULEB128(&*I, nullptr, EndPtr, &ErrMsg);
@@ -932,7 +928,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       break;
     }
     case MCD::OPC_SoftFail: {
-      ++I;
       OS << Indent << "MCD::OPC_SoftFail, ";
       // Decode the positive mask.
       const char *ErrMsg = nullptr;
@@ -952,7 +947,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       break;
     }
     case MCD::OPC_Fail: {
-      ++I;
       OS << Indent << "MCD::OPC_Fail,\n";
       break;
     }
@@ -2162,13 +2156,13 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
   DecodeStatus S = MCDisassembler::Success;
   while (true) {
     ptrdiff_t Loc = Ptr - DecodeTable;
-    switch (*Ptr) {
+    switch (*Ptr++) {
     default:
       errs() << Loc << ": Unexpected decode table opcode!\n";
       return MCDisassembler::Fail;
     case MCD::OPC_ExtractField: {
       // Decode the start value.
-      unsigned Start = decodeULEB128AndIncUnsafe(++Ptr);
+      unsigned Start = decodeULEB128AndIncUnsafe(Ptr);
       unsigned Len = *Ptr++;)";
   if (IsVarLenInst)
     OS << "\n      makeUp(insn, Start + Len);";
@@ -2180,7 +2174,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_FilterValue: {
       // Decode the field value.
-      uint64_t Val = decodeULEB128AndIncUnsafe(++Ptr);
+      uint64_t Val = decodeULEB128AndIncUnsafe(Ptr);
       bool Failed = Val != CurFieldValue;
       // NumToSkip is a plain 24-bit integer.
       unsigned NumToSkip = *Ptr++;
@@ -2198,7 +2192,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_CheckField: {
       // Decode the start value.
-      unsigned Start = decodeULEB128AndIncUnsafe(++Ptr);
+      unsigned Start = decodeULEB128AndIncUnsafe(Ptr);
       unsigned Len = *Ptr;)";
   if (IsVarLenInst)
     OS << "\n      makeUp(insn, Start + Len);";
@@ -2226,7 +2220,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_CheckPredicate: {
       // Decode the Predicate Index value.
-      unsigned PIdx = decodeULEB128AndIncUnsafe(++Ptr);
+      unsigned PIdx = decodeULEB128AndIncUnsafe(Ptr);
       // NumToSkip is a plain 24-bit integer.
       unsigned NumToSkip = *Ptr++;
       NumToSkip |= (*Ptr++) << 8;
@@ -2242,7 +2236,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_Decode: {
       // Decode the Opcode value.
-      unsigned Opc = decodeULEB128AndIncUnsafe(++Ptr);
+      unsigned Opc = decodeULEB128AndIncUnsafe(Ptr);
       unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
 
       MI.clear();
@@ -2263,7 +2257,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_TryDecode: {
       // Decode the Opcode value.
-      unsigned Opc = decodeULEB128AndIncUnsafe(++Ptr);
+      unsigned Opc = decodeULEB128AndIncUnsafe(Ptr);
       unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
       // NumToSkip is a plain 24-bit integer.
       unsigned NumToSkip = *Ptr++;
@@ -2296,7 +2290,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_SoftFail: {
       // Decode the mask values.
-      uint64_t PositiveMask = decodeULEB128AndIncUnsafe(++Ptr);
+      uint64_t PositiveMask = decodeULEB128AndIncUnsafe(Ptr);
       uint64_t NegativeMask = decodeULEB128AndIncUnsafe(Ptr);
       bool Fail = (insn & PositiveMask) != 0 || (~insn & NegativeMask) != 0;
       if (Fail)

From 23020a8d01a3f58e4903c42eba4b803d5809653e Mon Sep 17 00:00:00 2001
From: Nirvedh Meshram <96096277+nirvedhmeshram@users.noreply.github.com>
Date: Fri, 18 Apr 2025 12:21:37 -0500
Subject: [PATCH 439/710] [mlir] Add optimization to bubbleUpPadSlice pattern
 for no pad case (#135859)

In cases where there is no padding on a dim, we do not need to compute
new offsets, lengths and padding, for example the new test case added
can just be lowered to
```
    %extracted_slice = tensor.extract_slice %arg0[%arg2, 1, 2] [%arg2, 2, 1] [1, 1, 1] : tensor<3x4x5xf32> to tensor<?x2x1xf32>
 ```
without this PR we will have affine maps like
 ```
#map = affine_map<()[s0] -> (3, s0)>
#map1 = affine_map<()[s0, s1] -> (-s0 + 3, s1)>
%0 = affine.min #map()[%arg2]
 %1 = affine.min #map1()[%0, %arg2]
 %extracted_slice = tensor.extract_slice %arg0[%0, 1, 2] [%1, 2, 1] [1, 1, 1] : tensor<3x4x5xf32> to tensor<?x2x1xf32>
 ```
  which are unnecessary

Signed-off-by: Nirvedh <nirvedh@gmail.com>
---
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp     | 17 +++++++++++++----
 .../Linalg/matmul-shared-memory-padding.mlir    |  6 +++---
 .../Dialect/Linalg/subtensor-of-padtensor.mlir  | 17 +++++++++++++++++
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 138e4be6b18e9..7778a02dbeaf4 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -122,7 +122,7 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   OpFoldResult zero = b.getIndexAttr(0);
 
   // Compute new offsets, lengths, low padding, high padding.
-  SmallVector<OpFoldResult> newOffsets, newLengths, newStrides;
+  SmallVector<OpFoldResult> newOffsets, newLengths;
   SmallVector<OpFoldResult> newLows, newHighs;
   // Set to true if the original data source is not read at all.
   bool hasZeroLen = false;
@@ -131,6 +131,8 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
   Value dynHasZeroLenCond;
 
   int64_t rank = padOp.getSourceType().getRank();
+  // Only unit stride supported.
+  SmallVector<OpFoldResult> newStrides(rank, b.getIndexAttr(1));
   for (unsigned dim = 0; dim < rank; ++dim) {
     auto low = padOp.getMixedLowPad()[dim];
     bool hasLowPad = !isConstantIntValue(low, 0);
@@ -138,6 +140,16 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     bool hasHighPad = !isConstantIntValue(high, 0);
     auto offset = offsets[dim];
     auto length = sizes[dim];
+    // If the dim has no padding, we dont need to calculate new values for that
+    // dim as the exisiting ones are correct even after the pattern.
+    if (!hasLowPad && !hasHighPad) {
+      newOffsets.push_back(offset);
+      newLengths.push_back(length);
+      newLows.push_back(low);
+      newHighs.push_back(high);
+      continue;
+    }
+
     auto srcSize = tensor::getMixedSize(b, loc, padOp.getSource(), dim);
 
     // The new amount of low padding is `low - offset`. Except for the case
@@ -216,9 +228,6 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     OpFoldResult newHigh =
         hasHighPad ? sub(sub(length, newLength), newLow) : zero;
     newHighs.push_back(newHigh);
-
-    // Only unit stride supported.
-    newStrides.push_back(b.getIndexAttr(1));
   }
 
   // The shape of the result can be obtained from the sizes passed in.
diff --git a/mlir/test/Dialect/Linalg/matmul-shared-memory-padding.mlir b/mlir/test/Dialect/Linalg/matmul-shared-memory-padding.mlir
index d6c400dcbf2b9..6cab25b50460d 100644
--- a/mlir/test/Dialect/Linalg/matmul-shared-memory-padding.mlir
+++ b/mlir/test/Dialect/Linalg/matmul-shared-memory-padding.mlir
@@ -7,17 +7,17 @@
 //       CHECK:     scf.for
 //       CHECK:       memref.alloc() : memref<128x16xf32, 3>
 //       CHECK:       scf.forall
-//       CHECK:         vector.create_mask
+//       CHECK:         vector.constant_mask [16, 4] : vector<128x4xi1>
 //       CHECK:         vector.transfer_read
 //       CHECK:         vector.transfer_write
 //       CHECK:       memref.alloc() : memref<16x128xf32, 3>
 //       CHECK:       scf.forall
-//       CHECK:         vector.create_mask
+//       CHECK:         vector.constant_mask [16, 4] : vector<128x4xi1>
 //       CHECK:         vector.transfer_read
 //       CHECK:         vector.transfer_write
 //       CHECK:       memref.alloc() : memref<128x128xf32, 3>
 //       CHECK:       scf.forall
-//       CHECK:         vector.create_mask
+//   CHECK-NOT:         mask
 //       CHECK:         vector.transfer_read
 //       CHECK:         vector.transfer_write
 //       CHECK:       linalg.matmul
diff --git a/mlir/test/Dialect/Linalg/subtensor-of-padtensor.mlir b/mlir/test/Dialect/Linalg/subtensor-of-padtensor.mlir
index b4417641c9f83..d43b9a7ac6c04 100644
--- a/mlir/test/Dialect/Linalg/subtensor-of-padtensor.mlir
+++ b/mlir/test/Dialect/Linalg/subtensor-of-padtensor.mlir
@@ -216,3 +216,20 @@ func.func @dynamic_zero_high_padding(%arg0 : tensor<?x?xf32>, %pad : f32,
   %1 = tensor.extract_slice %0[%o1, %o2] [%s1, %s2] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
   return %1 : tensor<?x?xf32>
 }
+
+// -----
+// CHECK-LABEL: @nopaddim_with_dynamic_extract(
+//  CHECK-SAME:     %[[ARG0:.*]]: tensor<3x4x5xf32>
+//  CHECK-SAME:     %[[ARG1:.*]]: f32
+//  CHECK-SAME:     %[[ARG2:.*]]: index
+//       CHECK:   %[[RESULT:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 1, 2] [%[[ARG2]], 2, 1] [1, 1, 1] : tensor<3x4x5xf32> to tensor<?x2x1xf32>
+//       CHECK:   return %[[RESULT]]
+func.func @nopaddim_with_dynamic_extract(%arg0 : tensor<3x4x5xf32>, %pad : f32, %index : index)
+    -> tensor<?x2x1xf32> {
+  %0 = tensor.pad %arg0 low[0, 0, 0] high[0, 7, 8] {
+    ^bb0(%arg1: index, %arg2: index, %arg3: index):
+      tensor.yield %pad : f32
+    } : tensor<3x4x5xf32> to tensor<3x11x13xf32>
+  %1 = tensor.extract_slice %0[%index, 1, 2] [%index, 2, 1] [1, 1, 1] : tensor<3x11x13xf32> to tensor<?x2x1xf32>
+  return %1 : tensor<?x2x1xf32>
+}

From 5ad32fa6973638a091891942939933a41cbb2162 Mon Sep 17 00:00:00 2001
From: Andrei Safronov <andrei.safronov@espressif.com>
Date: Fri, 18 Apr 2025 20:22:03 +0300
Subject: [PATCH 440/710] [Xtensa] Implement Xtensa Mul and Div Options.
 (#132157)

Implement Xtensa Mul16, Mul32, Mul32High and Div32 Options. Also fix callw test.
---
 llvm/lib/Target/Xtensa/XtensaFeatures.td      |  20 +
 llvm/lib/Target/Xtensa/XtensaISelLowering.cpp |  32 +-
 llvm/lib/Target/Xtensa/XtensaInstrInfo.td     |  30 +
 llvm/lib/Target/Xtensa/XtensaSubtarget.h      |   4 +
 llvm/test/CodeGen/Xtensa/callw.ll             |   4 +-
 llvm/test/CodeGen/Xtensa/div.ll               | 366 ++++++++++++-
 llvm/test/CodeGen/Xtensa/mul.ll               | 515 ++++++++++++++++++
 llvm/test/CodeGen/Xtensa/rem.ll               |  49 ++
 llvm/test/MC/Xtensa/div.s                     |  24 +
 llvm/test/MC/Xtensa/mul.s                     |  29 +
 10 files changed, 1034 insertions(+), 39 deletions(-)
 create mode 100644 llvm/test/CodeGen/Xtensa/rem.ll
 create mode 100644 llvm/test/MC/Xtensa/div.s
 create mode 100644 llvm/test/MC/Xtensa/mul.s

diff --git a/llvm/lib/Target/Xtensa/XtensaFeatures.td b/llvm/lib/Target/Xtensa/XtensaFeatures.td
index 8b2d351dae386..2a47214946401 100644
--- a/llvm/lib/Target/Xtensa/XtensaFeatures.td
+++ b/llvm/lib/Target/Xtensa/XtensaFeatures.td
@@ -47,3 +47,23 @@ def FeatureMINMAX : SubtargetFeature<"minmax", "HasMINMAX", "true",
                                      "Enable Xtensa MINMAX option">;
 def HasMINMAX : Predicate<"Subtarget->hasMINMAX()">,
                 AssemblerPredicate<(all_of FeatureMINMAX)>;
+
+def FeatureMul16 : SubtargetFeature<"mul16", "HasMul16", "true",
+                                    "Enable Xtensa Mul16 option">;
+def HasMul16 : Predicate<"Subtarget->hasMul16()">,
+               AssemblerPredicate<(all_of FeatureMul16)>;
+
+def FeatureMul32 : SubtargetFeature<"mul32", "HasMul32", "true",
+                                    "Enable Xtensa Mul32 option">;
+def HasMul32 : Predicate<"Subtarget->hasMul32()">,
+               AssemblerPredicate<(all_of FeatureMul32)>;
+
+def FeatureMul32High : SubtargetFeature<"mul32high", "HasMul32High", "true",
+                                        "Enable Xtensa Mul32High option">;
+def HasMul32High : Predicate<"Subtarget->hasMul32High()">,
+                   AssemblerPredicate<(all_of FeatureMul32High)>;
+
+def FeatureDiv32 : SubtargetFeature<"div32", "HasDiv32", "true",
+                                    "Enable Xtensa Div32 option">;
+def HasDiv32 : Predicate<"Subtarget->hasDiv32()">,
+               AssemblerPredicate<(all_of FeatureDiv32)>;
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
index 9ba87cf6d5878..b17840aad9b4d 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
@@ -115,16 +115,34 @@ XtensaTargetLowering::XtensaTargetLowering(const TargetMachine &TM,
   setCondCodeAction(ISD::SETUGT, MVT::i32, Expand);
   setCondCodeAction(ISD::SETULE, MVT::i32, Expand);
 
-  setOperationAction(ISD::MUL, MVT::i32, Expand);
-  setOperationAction(ISD::MULHU, MVT::i32, Expand);
-  setOperationAction(ISD::MULHS, MVT::i32, Expand);
+  if (Subtarget.hasMul32())
+    setOperationAction(ISD::MUL, MVT::i32, Legal);
+  else
+    setOperationAction(ISD::MUL, MVT::i32, Expand);
+
+  if (Subtarget.hasMul32High()) {
+    setOperationAction(ISD::MULHU, MVT::i32, Legal);
+    setOperationAction(ISD::MULHS, MVT::i32, Legal);
+  } else {
+    setOperationAction(ISD::MULHU, MVT::i32, Expand);
+    setOperationAction(ISD::MULHS, MVT::i32, Expand);
+  }
+
   setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
   setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
 
-  setOperationAction(ISD::SDIV, MVT::i32, Expand);
-  setOperationAction(ISD::UDIV, MVT::i32, Expand);
-  setOperationAction(ISD::SREM, MVT::i32, Expand);
-  setOperationAction(ISD::UREM, MVT::i32, Expand);
+  if (Subtarget.hasDiv32()) {
+    setOperationAction(ISD::SDIV, MVT::i32, Legal);
+    setOperationAction(ISD::UDIV, MVT::i32, Legal);
+    setOperationAction(ISD::SREM, MVT::i32, Legal);
+    setOperationAction(ISD::UREM, MVT::i32, Legal);
+  } else {
+    setOperationAction(ISD::SDIV, MVT::i32, Expand);
+    setOperationAction(ISD::UDIV, MVT::i32, Expand);
+    setOperationAction(ISD::SREM, MVT::i32, Expand);
+    setOperationAction(ISD::UREM, MVT::i32, Expand);
+  }
+
   setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
   setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
 
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
index 9734e914db335..0bd3ba81340ff 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
@@ -956,6 +956,36 @@ def LOOPNEZ : RRI8_Inst<0x06, (outs), (ins AR:$s, ltarget:$target),
 
 def : InstAlias<"_loopnez\t$s, $target", (LOOPNEZ AR:$s, ltarget:$target)>;
 
+//===----------------------------------------------------------------------===//
+// Mul16 Instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasMul16] in {
+  def MUL16S : RRR_Inst<0x00, 0x01, 0x0D, (outs AR:$r), (ins AR:$s, AR:$t),
+                       "mul16s\t$r, $s, $t", []>;
+  def MUL16U : RRR_Inst<0x00, 0x01, 0x0C, (outs AR:$r), (ins AR:$s, AR:$t),
+                       "mul16u\t$r, $s, $t", []>;
+}
+
+//===----------------------------------------------------------------------===//
+// Mul32 Instructions
+//===----------------------------------------------------------------------===//
+
+def MULL  : ArithLogic_RRR<0x08, 0x02, "mull", mul, 1>, Requires<[HasMul32]>;
+def MULUH : ArithLogic_RRR<0x0A, 0x02, "muluh", mulhu, 1>, Requires<[HasMul32High]>;
+def MULSH : ArithLogic_RRR<0x0B, 0x02, "mulsh", mulhs, 1>, Requires<[HasMul32High]>;
+
+//===----------------------------------------------------------------------===//
+// Div32 Instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasDiv32] in {
+  def QUOS : ArithLogic_RRR<0x0D, 0x02, "quos", sdiv>;
+  def QUOU : ArithLogic_RRR<0x0C, 0x02, "quou", udiv>;
+  def REMS : ArithLogic_RRR<0x0F, 0x02, "rems", srem>;
+  def REMU : ArithLogic_RRR<0x0E, 0x02, "remu", urem>;
+}
+
 //===----------------------------------------------------------------------===//
 // DSP Instructions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Xtensa/XtensaSubtarget.h b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
index ef86df20f9e1f..227ce2134b33b 100644
--- a/llvm/lib/Target/Xtensa/XtensaSubtarget.h
+++ b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
@@ -73,6 +73,10 @@ class XtensaSubtarget : public XtensaGenSubtargetInfo {
   bool hasCLAMPS() const { return HasCLAMPS; }
   bool hasNSA() const { return HasNSA; }
   bool hasMINMAX() const { return HasMINMAX; }
+  bool hasMul16() const { return HasMul16; }
+  bool hasMul32() const { return HasMul32; }
+  bool hasMul32High() const { return HasMul32High; }
+  bool hasDiv32() const { return HasDiv32; }
   bool isWindowedABI() const { return hasWindowed(); }
 
   // Automatically generated by tblgen.
diff --git a/llvm/test/CodeGen/Xtensa/callw.ll b/llvm/test/CodeGen/Xtensa/callw.ll
index 21549bcf22678..7abfcad23f714 100644
--- a/llvm/test/CodeGen/Xtensa/callw.ll
+++ b/llvm/test/CodeGen/Xtensa/callw.ll
@@ -7,8 +7,8 @@ define i32 @test_call_external(i32 %a) nounwind {
 ; CHECK-LABEL: test_call_external:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    entry a1, 32
-; CHECK-NEXT:    l32r a8, .LCPI0_0
 ; CHECK-NEXT:    or a10, a2, a2
+; CHECK-NEXT:    l32r a8, .LCPI0_0
 ; CHECK-NEXT:    callx8 a8
 ; CHECK-NEXT:    or a2, a10, a10
 ; CHECK-NEXT:    retw
@@ -30,8 +30,8 @@ define i32 @test_call_defined(i32 %a) nounwind {
 ; CHECK-LABEL: test_call_defined:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    entry a1, 32
-; CHECK-NEXT:    l32r a8, .LCPI2_0
 ; CHECK-NEXT:    or a10, a2, a2
+; CHECK-NEXT:    l32r a8, .LCPI2_0
 ; CHECK-NEXT:    callx8 a8
 ; CHECK-NEXT:    or a2, a10, a10
 ; CHECK-NEXT:    retw
diff --git a/llvm/test/CodeGen/Xtensa/div.ll b/llvm/test/CodeGen/Xtensa/div.ll
index 8d51c571efb4c..6666ea07822b4 100644
--- a/llvm/test/CodeGen/Xtensa/div.ll
+++ b/llvm/test/CodeGen/Xtensa/div.ll
@@ -1,10 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=xtensa -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=XTENSA %s
+; RUN: llc -mtriple=xtensa -mattr=+div32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=XTENSA-DIV %s
 
 define i32 @udiv(i32 %a, i32 %b) nounwind {
 ; XTENSA-LABEL: udiv:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    l32r a8, .LCPI0_0
@@ -13,13 +16,19 @@ define i32 @udiv(i32 %a, i32 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    quou a2, a2, a3
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i32 %a, %b
   ret i32 %1
 }
 
 define i32 @udiv_constant(i32 %a) nounwind {
 ; XTENSA-LABEL: udiv_constant:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    movi a3, 5
@@ -29,21 +38,34 @@ define i32 @udiv_constant(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv_constant:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    movi a8, 5
+; XTENSA-DIV-NEXT:    quou a2, a2, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i32 %a, 5
   ret i32 %1
 }
 
 define i32 @udiv_pow2(i32 %a) nounwind {
 ; XTENSA-LABEL: udiv_pow2:
-; XTENSA:         srli a2, a2, 3
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    srli a2, a2, 3
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv_pow2:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    srli a2, a2, 3
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i32 %a, 8
   ret i32 %1
 }
 
 define i32 @udiv_constant_lhs(i32 %a) nounwind {
 ; XTENSA-LABEL: udiv_constant_lhs:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    or a3, a2, a2
@@ -54,13 +76,20 @@ define i32 @udiv_constant_lhs(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv_constant_lhs:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    movi a8, 10
+; XTENSA-DIV-NEXT:    quou a2, a8, a2
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i32 10, %a
   ret i32 %1
 }
 
 define i64 @udiv64(i64 %a, i64 %b) nounwind {
 ; XTENSA-LABEL: udiv64:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    l32r a8, .LCPI4_0
@@ -69,13 +98,26 @@ define i64 @udiv64(i64 %a, i64 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv64:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    addi a8, a1, -16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI4_0
+; XTENSA-DIV-NEXT:    callx0 a8
+; XTENSA-DIV-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-DIV-NEXT:    addi a8, a1, 16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i64 %a, %b
   ret i64 %1
 }
 
 define i64 @udiv64_constant(i64 %a) nounwind {
 ; XTENSA-LABEL: udiv64_constant:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    movi a4, 5
@@ -86,13 +128,28 @@ define i64 @udiv64_constant(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv64_constant:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    addi a8, a1, -16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-DIV-NEXT:    movi a4, 5
+; XTENSA-DIV-NEXT:    movi a5, 0
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI5_0
+; XTENSA-DIV-NEXT:    callx0 a8
+; XTENSA-DIV-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-DIV-NEXT:    addi a8, a1, 16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i64 %a, 5
   ret i64 %1
 }
 
 define i64 @udiv64_constant_lhs(i64 %a) nounwind {
 ; XTENSA-LABEL: udiv64_constant_lhs:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    or a5, a3, a3
@@ -105,13 +162,30 @@ define i64 @udiv64_constant_lhs(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv64_constant_lhs:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    addi a8, a1, -16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-DIV-NEXT:    or a5, a3, a3
+; XTENSA-DIV-NEXT:    or a4, a2, a2
+; XTENSA-DIV-NEXT:    movi a2, 10
+; XTENSA-DIV-NEXT:    movi a3, 0
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI6_0
+; XTENSA-DIV-NEXT:    callx0 a8
+; XTENSA-DIV-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-DIV-NEXT:    addi a8, a1, 16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i64 10, %a
   ret i64 %1
 }
 
 define i8 @udiv8(i8 %a, i8 %b) nounwind {
 ; XTENSA-LABEL: udiv8:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    movi a8, 255
@@ -123,13 +197,22 @@ define i8 @udiv8(i8 %a, i8 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv8:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    movi a8, 255
+; XTENSA-DIV-NEXT:    and a9, a3, a8
+; XTENSA-DIV-NEXT:    and a8, a2, a8
+; XTENSA-DIV-NEXT:    quou a2, a8, a9
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i8 %a, %b
   ret i8 %1
 }
 
 define i8 @udiv8_constant(i8 %a) nounwind {
 ; XTENSA-LABEL: udiv8_constant:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    movi a8, 255
@@ -141,23 +224,40 @@ define i8 @udiv8_constant(i8 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv8_constant:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    movi a8, 255
+; XTENSA-DIV-NEXT:    and a8, a2, a8
+; XTENSA-DIV-NEXT:    movi a9, 5
+; XTENSA-DIV-NEXT:    quou a2, a8, a9
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i8 %a, 5
   ret i8 %1
 }
 
 define i8 @udiv8_pow2(i8 %a) nounwind {
 ; XTENSA-LABEL: udiv8_pow2:
-; XTENSA:         movi a8, 248
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    movi a8, 248
 ; XTENSA-NEXT:    and a8, a2, a8
 ; XTENSA-NEXT:    srli a2, a8, 3
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv8_pow2:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    movi a8, 248
+; XTENSA-DIV-NEXT:    and a8, a2, a8
+; XTENSA-DIV-NEXT:    srli a2, a8, 3
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i8 %a, 8
   ret i8 %1
 }
 
 define i8 @udiv8_constant_lhs(i8 %a) nounwind {
 ; XTENSA-LABEL: udiv8_constant_lhs:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    movi a8, 255
@@ -169,13 +269,22 @@ define i8 @udiv8_constant_lhs(i8 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv8_constant_lhs:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    movi a8, 255
+; XTENSA-DIV-NEXT:    and a8, a2, a8
+; XTENSA-DIV-NEXT:    movi a9, 10
+; XTENSA-DIV-NEXT:    quou a2, a9, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i8 10, %a
   ret i8 %1
 }
 
 define i16 @udiv16(i16 %a, i16 %b) nounwind {
 ; XTENSA-LABEL: udiv16:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    l32r a8, .LCPI11_0
@@ -187,13 +296,22 @@ define i16 @udiv16(i16 %a, i16 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv16:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI11_0
+; XTENSA-DIV-NEXT:    and a9, a3, a8
+; XTENSA-DIV-NEXT:    and a8, a2, a8
+; XTENSA-DIV-NEXT:    quou a2, a8, a9
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i16 %a, %b
   ret i16 %1
 }
 
 define i16 @udiv16_constant(i16 %a) nounwind {
 ; XTENSA-LABEL: udiv16_constant:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    l32r a8, .LCPI12_0
@@ -205,23 +323,40 @@ define i16 @udiv16_constant(i16 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv16_constant:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI12_0
+; XTENSA-DIV-NEXT:    and a8, a2, a8
+; XTENSA-DIV-NEXT:    movi a9, 5
+; XTENSA-DIV-NEXT:    quou a2, a8, a9
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i16 %a, 5
   ret i16 %1
 }
 
 define i16 @udiv16_pow2(i16 %a) nounwind {
 ; XTENSA-LABEL: udiv16_pow2:
-; XTENSA:         l32r a8, .LCPI13_0
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    l32r a8, .LCPI13_0
 ; XTENSA-NEXT:    and a8, a2, a8
 ; XTENSA-NEXT:    srli a2, a8, 3
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: udiv16_pow2:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI13_0
+; XTENSA-DIV-NEXT:    and a8, a2, a8
+; XTENSA-DIV-NEXT:    srli a2, a8, 3
+; XTENSA-DIV-NEXT:    ret
   %1 = udiv i16 %a, 8
   ret i16 %1
 }
 
 define i32 @sdiv(i32 %a, i32 %b) nounwind {
 ; XTENSA-LABEL: sdiv:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    l32r a8, .LCPI14_0
@@ -230,13 +365,19 @@ define i32 @sdiv(i32 %a, i32 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    quos a2, a2, a3
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i32 %a, %b
   ret i32 %1
 }
 
 define i32 @sdiv_constant_lhs(i32 %a) nounwind {
 ; XTENSA-LABEL: sdiv_constant_lhs:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    or a3, a2, a2
@@ -247,13 +388,20 @@ define i32 @sdiv_constant_lhs(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv_constant_lhs:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    movi a8, -10
+; XTENSA-DIV-NEXT:    quos a2, a8, a2
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i32 -10, %a
   ret i32 %1
 }
 
 define i64 @sdiv64(i64 %a, i64 %b) nounwind {
 ; XTENSA-LABEL: sdiv64:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    l32r a8, .LCPI16_0
@@ -262,13 +410,26 @@ define i64 @sdiv64(i64 %a, i64 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv64:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    addi a8, a1, -16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI16_0
+; XTENSA-DIV-NEXT:    callx0 a8
+; XTENSA-DIV-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-DIV-NEXT:    addi a8, a1, 16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i64 %a, %b
   ret i64 %1
 }
 
 define i64 @sdiv64_constant(i64 %a) nounwind {
 ; XTENSA-LABEL: sdiv64_constant:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    movi a4, 5
@@ -279,13 +440,28 @@ define i64 @sdiv64_constant(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv64_constant:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    addi a8, a1, -16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-DIV-NEXT:    movi a4, 5
+; XTENSA-DIV-NEXT:    movi a5, 0
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI17_0
+; XTENSA-DIV-NEXT:    callx0 a8
+; XTENSA-DIV-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-DIV-NEXT:    addi a8, a1, 16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i64 %a, 5
   ret i64 %1
 }
 
 define i64 @sdiv64_constant_lhs(i64 %a) nounwind {
 ; XTENSA-LABEL: sdiv64_constant_lhs:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    or a5, a3, a3
@@ -298,6 +474,22 @@ define i64 @sdiv64_constant_lhs(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv64_constant_lhs:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    addi a8, a1, -16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-DIV-NEXT:    or a5, a3, a3
+; XTENSA-DIV-NEXT:    or a4, a2, a2
+; XTENSA-DIV-NEXT:    movi a2, 10
+; XTENSA-DIV-NEXT:    movi a3, 0
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI18_0
+; XTENSA-DIV-NEXT:    callx0 a8
+; XTENSA-DIV-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-DIV-NEXT:    addi a8, a1, 16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i64 10, %a
   ret i64 %1
 }
@@ -305,7 +497,8 @@ define i64 @sdiv64_constant_lhs(i64 %a) nounwind {
 
 define i64 @sdiv64_sext_operands(i32 %a, i32 %b) nounwind {
 ; XTENSA-LABEL: sdiv64_sext_operands:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    or a4, a3, a3
@@ -317,6 +510,21 @@ define i64 @sdiv64_sext_operands(i32 %a, i32 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv64_sext_operands:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    addi a8, a1, -16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-DIV-NEXT:    or a4, a3, a3
+; XTENSA-DIV-NEXT:    srai a3, a2, 31
+; XTENSA-DIV-NEXT:    srai a5, a4, 31
+; XTENSA-DIV-NEXT:    l32r a8, .LCPI19_0
+; XTENSA-DIV-NEXT:    callx0 a8
+; XTENSA-DIV-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-DIV-NEXT:    addi a8, a1, 16
+; XTENSA-DIV-NEXT:    or a1, a8, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = sext i32 %a to i64
   %2 = sext i32 %b to i64
   %3 = sdiv i64 %1, %2
@@ -325,7 +533,8 @@ define i64 @sdiv64_sext_operands(i32 %a, i32 %b) nounwind {
 
 define i8 @sdiv8(i8 %a, i8 %b) nounwind {
 ; XTENSA-LABEL: sdiv8:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    slli a8, a2, 24
@@ -338,13 +547,23 @@ define i8 @sdiv8(i8 %a, i8 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv8:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    slli a8, a3, 24
+; XTENSA-DIV-NEXT:    srai a8, a8, 24
+; XTENSA-DIV-NEXT:    slli a9, a2, 24
+; XTENSA-DIV-NEXT:    srai a9, a9, 24
+; XTENSA-DIV-NEXT:    quos a2, a9, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i8 %a, %b
   ret i8 %1
 }
 
 define i8 @sdiv8_constant(i8 %a) nounwind {
 ; XTENSA-LABEL: sdiv8_constant:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    slli a8, a2, 24
@@ -356,13 +575,22 @@ define i8 @sdiv8_constant(i8 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv8_constant:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    slli a8, a2, 24
+; XTENSA-DIV-NEXT:    srai a8, a8, 24
+; XTENSA-DIV-NEXT:    movi a9, 5
+; XTENSA-DIV-NEXT:    quos a2, a8, a9
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i8 %a, 5
   ret i8 %1
 }
 
 define i8 @sdiv8_pow2(i8 %a) nounwind {
 ; XTENSA-LABEL: sdiv8_pow2:
-; XTENSA:         slli a8, a2, 24
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    slli a8, a2, 24
 ; XTENSA-NEXT:    srai a8, a8, 31
 ; XTENSA-NEXT:    movi a9, 7
 ; XTENSA-NEXT:    and a8, a8, a9
@@ -370,13 +598,25 @@ define i8 @sdiv8_pow2(i8 %a) nounwind {
 ; XTENSA-NEXT:    slli a8, a8, 24
 ; XTENSA-NEXT:    srai a2, a8, 27
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv8_pow2:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    slli a8, a2, 24
+; XTENSA-DIV-NEXT:    srai a8, a8, 31
+; XTENSA-DIV-NEXT:    movi a9, 7
+; XTENSA-DIV-NEXT:    and a8, a8, a9
+; XTENSA-DIV-NEXT:    add a8, a2, a8
+; XTENSA-DIV-NEXT:    slli a8, a8, 24
+; XTENSA-DIV-NEXT:    srai a2, a8, 27
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i8 %a, 8
   ret i8 %1
 }
 
 define i8 @sdiv8_constant_lhs(i8 %a) nounwind {
 ; XTENSA-LABEL: sdiv8_constant_lhs:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    slli a8, a2, 24
@@ -388,13 +628,22 @@ define i8 @sdiv8_constant_lhs(i8 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv8_constant_lhs:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    slli a8, a2, 24
+; XTENSA-DIV-NEXT:    srai a8, a8, 24
+; XTENSA-DIV-NEXT:    movi a9, -10
+; XTENSA-DIV-NEXT:    quos a2, a9, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i8 -10, %a
   ret i8 %1
 }
 
 define i16 @sdiv16(i16 %a, i16 %b) nounwind {
 ; XTENSA-LABEL: sdiv16:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    slli a8, a2, 16
@@ -407,13 +656,23 @@ define i16 @sdiv16(i16 %a, i16 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv16:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    slli a8, a3, 16
+; XTENSA-DIV-NEXT:    srai a8, a8, 16
+; XTENSA-DIV-NEXT:    slli a9, a2, 16
+; XTENSA-DIV-NEXT:    srai a9, a9, 16
+; XTENSA-DIV-NEXT:    quos a2, a9, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i16 %a, %b
   ret i16 %1
 }
 
 define i16 @sdiv16_constant(i16 %a) nounwind {
 ; XTENSA-LABEL: sdiv16_constant:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    slli a8, a2, 16
@@ -425,13 +684,22 @@ define i16 @sdiv16_constant(i16 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv16_constant:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    slli a8, a2, 16
+; XTENSA-DIV-NEXT:    srai a8, a8, 16
+; XTENSA-DIV-NEXT:    movi a9, 5
+; XTENSA-DIV-NEXT:    quos a2, a8, a9
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i16 %a, 5
   ret i16 %1
 }
 
 define i16 @sdiv16_constant_lhs(i16 %a) nounwind {
 ; XTENSA-LABEL: sdiv16_constant_lhs:
-; XTENSA:         addi a8, a1, -16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    addi a8, a1, -16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
 ; XTENSA-NEXT:    slli a8, a2, 16
@@ -443,35 +711,62 @@ define i16 @sdiv16_constant_lhs(i16 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv16_constant_lhs:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    slli a8, a2, 16
+; XTENSA-DIV-NEXT:    srai a8, a8, 16
+; XTENSA-DIV-NEXT:    movi a9, -10
+; XTENSA-DIV-NEXT:    quos a2, a9, a8
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i16 -10, %a
   ret i16 %1
 }
 
 define i32 @sdiv_pow2(i32 %a) nounwind {
 ; XTENSA-LABEL: sdiv_pow2:
-; XTENSA:         srai a8, a2, 31
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    srai a8, a2, 31
 ; XTENSA-NEXT:    extui a8, a8, 29, 3
 ; XTENSA-NEXT:    add a8, a2, a8
 ; XTENSA-NEXT:    srai a2, a8, 3
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv_pow2:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    srai a8, a2, 31
+; XTENSA-DIV-NEXT:    extui a8, a8, 29, 3
+; XTENSA-DIV-NEXT:    add a8, a2, a8
+; XTENSA-DIV-NEXT:    srai a2, a8, 3
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i32 %a, 8
   ret i32 %1
 }
 
 define i32 @sdiv_pow2_2(i32 %a) nounwind {
 ; XTENSA-LABEL: sdiv_pow2_2:
-; XTENSA:         srai a8, a2, 31
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    srai a8, a2, 31
 ; XTENSA-NEXT:    extui a8, a8, 16, 16
 ; XTENSA-NEXT:    add a8, a2, a8
 ; XTENSA-NEXT:    srai a2, a8, 16
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv_pow2_2:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    srai a8, a2, 31
+; XTENSA-DIV-NEXT:    extui a8, a8, 16, 16
+; XTENSA-DIV-NEXT:    add a8, a2, a8
+; XTENSA-DIV-NEXT:    srai a2, a8, 16
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i32 %a, 65536
   ret i32 %1
 }
 
 define i16 @sdiv16_pow2(i16 %a) nounwind {
 ; XTENSA-LABEL: sdiv16_pow2:
-; XTENSA:         slli a8, a2, 16
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    slli a8, a2, 16
 ; XTENSA-NEXT:    srai a8, a8, 31
 ; XTENSA-NEXT:    movi a9, 7
 ; XTENSA-NEXT:    and a8, a8, a9
@@ -479,6 +774,17 @@ define i16 @sdiv16_pow2(i16 %a) nounwind {
 ; XTENSA-NEXT:    slli a8, a8, 16
 ; XTENSA-NEXT:    srai a2, a8, 19
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: sdiv16_pow2:
+; XTENSA-DIV:       # %bb.0:
+; XTENSA-DIV-NEXT:    slli a8, a2, 16
+; XTENSA-DIV-NEXT:    srai a8, a8, 31
+; XTENSA-DIV-NEXT:    movi a9, 7
+; XTENSA-DIV-NEXT:    and a8, a8, a9
+; XTENSA-DIV-NEXT:    add a8, a2, a8
+; XTENSA-DIV-NEXT:    slli a8, a8, 16
+; XTENSA-DIV-NEXT:    srai a2, a8, 19
+; XTENSA-DIV-NEXT:    ret
   %1 = sdiv i16 %a, 8
   ret i16 %1
 }
diff --git a/llvm/test/CodeGen/Xtensa/mul.ll b/llvm/test/CodeGen/Xtensa/mul.ll
index c5995bbc479a6..6901323822280 100644
--- a/llvm/test/CodeGen/Xtensa/mul.ll
+++ b/llvm/test/CodeGen/Xtensa/mul.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=xtensa -verify-machineinstrs < %s \
 ; RUN: | FileCheck -check-prefix=XTENSA %s
+; RUN: llc -mtriple=xtensa -mattr=+mul32high,mul32 -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=XTENSA-MUL %s
 
 define signext i32 @square(i32 %a) nounwind {
 ; XTENSA-LABEL: square:
@@ -15,6 +17,11 @@ define signext i32 @square(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: square:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    mull a2, a2, a2
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, %a
   ret i32 %1
 }
@@ -31,6 +38,11 @@ define signext i32 @mul(i32 %a, i32 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    mull a2, a2, a3
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
 }
@@ -48,6 +60,12 @@ define signext i32 @mul_constant(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul_constant:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 5
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 5
   ret i32 %1
 }
@@ -57,6 +75,11 @@ define i32 @mul_pow2(i32 %a) nounwind {
 ; XTENSA:       # %bb.0:
 ; XTENSA-NEXT:    slli a2, a2, 3
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul_pow2:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    slli a2, a2, 3
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 8
   ret i32 %1
 }
@@ -73,6 +96,16 @@ define i64 @mul64(i64 %a, i64 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul64:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    mull a8, a2, a5
+; XTENSA-MUL-NEXT:    muluh a9, a2, a4
+; XTENSA-MUL-NEXT:    add a8, a9, a8
+; XTENSA-MUL-NEXT:    mull a9, a3, a4
+; XTENSA-MUL-NEXT:    add a3, a8, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a4
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, %b
   ret i64 %1
 }
@@ -91,6 +124,15 @@ define i64 @mul64_constant(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul64_constant:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 5
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, 5
   ret i64 %1
 }
@@ -111,6 +153,11 @@ define i32 @mulhs(i32 %a, i32 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulhs:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    mulsh a2, a2, a3
+; XTENSA-MUL-NEXT:    ret
   %1 = sext i32 %a to i64
   %2 = sext i32 %b to i64
   %3 = mul i64 %1, %2
@@ -135,6 +182,12 @@ define i32 @mulhs_positive_constant(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulhs_positive_constant:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 5
+; XTENSA-MUL-NEXT:    mulsh a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = sext i32 %a to i64
   %2 = mul i64 %1, 5
   %3 = lshr i64 %2, 32
@@ -158,6 +211,12 @@ define i32 @mulhs_negative_constant(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulhs_negative_constant:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, -5
+; XTENSA-MUL-NEXT:    mulsh a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = sext i32 %a to i64
   %2 = mul i64 %1, -5
   %3 = lshr i64 %2, 32
@@ -181,6 +240,11 @@ define zeroext i32 @mulhu(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulhu:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    muluh a2, a2, a3
+; XTENSA-MUL-NEXT:    ret
   %1 = zext i32 %a to i64
   %2 = zext i32 %b to i64
   %3 = mul i64 %1, %2
@@ -205,6 +269,14 @@ define i32 @mulhsu(i32 %a, i32 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulhsu:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    muluh a8, a2, a3
+; XTENSA-MUL-NEXT:    srai a9, a3, 31
+; XTENSA-MUL-NEXT:    mull a9, a2, a9
+; XTENSA-MUL-NEXT:    add a2, a8, a9
+; XTENSA-MUL-NEXT:    ret
   %1 = zext i32 %a to i64
   %2 = sext i32 %b to i64
   %3 = mul i64 %1, %2
@@ -229,6 +301,12 @@ define i32 @mulhu_constant(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulhu_constant:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 5
+; XTENSA-MUL-NEXT:    muluh a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = zext i32 %a to i64
   %2 = mul i64 %1, 5
   %3 = lshr i64 %2, 32
@@ -249,6 +327,12 @@ define i32 @muli32_p65(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_p65:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 65
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 65
   ret i32 %1
 }
@@ -266,6 +350,12 @@ define i32 @muli32_p63(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_p63:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 63
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 63
   ret i32 %1
 }
@@ -284,6 +374,15 @@ define i64 @muli64_p65(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli64_p65:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 65
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, 65
   ret i64 %1
 }
@@ -302,6 +401,15 @@ define i64 @muli64_p63(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli64_p63:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 63
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, 63
   ret i64 %1
 }
@@ -319,6 +427,12 @@ define i32 @muli32_m63(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_m63:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, -63
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, -63
   ret i32 %1
 }
@@ -336,6 +450,12 @@ define i32 @muli32_m65(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_m65:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, -65
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, -65
   ret i32 %1
 }
@@ -354,6 +474,16 @@ define i64 @muli64_m63(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli64_m63:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, -63
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    sub a10, a10, a2
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, -63
   ret i64 %1
 }
@@ -372,6 +502,16 @@ define i64 @muli64_m65(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli64_m65:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, -65
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    sub a10, a10, a2
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, -65
   ret i64 %1
 }
@@ -389,6 +529,12 @@ define i32 @muli32_p384(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_p384:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    movi a8, 384
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 384
   ret i32 %1
 }
@@ -406,6 +552,12 @@ define i32 @muli32_p12288(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_p12288:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI21_0
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 12288
   ret i32 %1
 }
@@ -423,6 +575,12 @@ define i32 @muli32_p4352(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_p4352:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI22_0
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 4352
   ret i32 %1
 }
@@ -440,6 +598,12 @@ define i32 @muli32_p3840(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_p3840:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI23_0
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 3840
   ret i32 %1
 }
@@ -457,6 +621,12 @@ define i32 @muli32_m3840(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_m3840:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI24_0
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, -3840
   ret i32 %1
 }
@@ -474,6 +644,12 @@ define i32 @muli32_m4352(i32 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli32_m4352:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI25_0
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, -4352
   ret i32 %1
 }
@@ -492,6 +668,15 @@ define i64 @muli64_p4352(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli64_p4352:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI26_0
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, 4352
   ret i64 %1
 }
@@ -510,6 +695,15 @@ define i64 @muli64_p3840(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli64_p3840:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI27_0
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, 3840
   ret i64 %1
 }
@@ -528,6 +722,16 @@ define i64 @muli64_m4352(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli64_m4352:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI28_0
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    sub a10, a10, a2
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, -4352
   ret i64 %1
 }
@@ -546,6 +750,16 @@ define i64 @muli64_m3840(i64 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 16
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli64_m3840:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    l32r a8, .LCPI29_0
+; XTENSA-MUL-NEXT:    mull a9, a3, a8
+; XTENSA-MUL-NEXT:    muluh a10, a2, a8
+; XTENSA-MUL-NEXT:    sub a10, a10, a2
+; XTENSA-MUL-NEXT:    add a3, a10, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i64 %a, -3840
   ret i64 %1
 }
@@ -671,6 +885,94 @@ define i128 @muli128_m3840(i128 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 80
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli128_m3840:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    addi a8, a1, -48
+; XTENSA-MUL-NEXT:    or a1, a8, a8
+; XTENSA-MUL-NEXT:    s32i a12, a1, 32 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a13, a1, 28 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a14, a1, 24 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a15, a1, 20 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a5, a1, 16 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    l32r a10, .LCPI30_0
+; XTENSA-MUL-NEXT:    muluh a8, a2, a10
+; XTENSA-MUL-NEXT:    mull a7, a3, a10
+; XTENSA-MUL-NEXT:    add a8, a7, a8
+; XTENSA-MUL-NEXT:    movi a5, 0
+; XTENSA-MUL-NEXT:    movi a11, 1
+; XTENSA-MUL-NEXT:    or a9, a11, a11
+; XTENSA-MUL-NEXT:    bltu a8, a7, .LBB30_2
+; XTENSA-MUL-NEXT:  # %bb.1:
+; XTENSA-MUL-NEXT:    or a9, a5, a5
+; XTENSA-MUL-NEXT:  .LBB30_2:
+; XTENSA-MUL-NEXT:    muluh a7, a3, a10
+; XTENSA-MUL-NEXT:    add a15, a7, a9
+; XTENSA-MUL-NEXT:    sub a8, a8, a2
+; XTENSA-MUL-NEXT:    neg a7, a2
+; XTENSA-MUL-NEXT:    or a9, a11, a11
+; XTENSA-MUL-NEXT:    s32i a8, a1, 12 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a7, a1, 8 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    bltu a8, a7, .LBB30_4
+; XTENSA-MUL-NEXT:  # %bb.3:
+; XTENSA-MUL-NEXT:    or a9, a5, a5
+; XTENSA-MUL-NEXT:  .LBB30_4:
+; XTENSA-MUL-NEXT:    movi a8, -1
+; XTENSA-MUL-NEXT:    muluh a13, a2, a8
+; XTENSA-MUL-NEXT:    add a9, a13, a9
+; XTENSA-MUL-NEXT:    add a12, a15, a9
+; XTENSA-MUL-NEXT:    sub a6, a12, a3
+; XTENSA-MUL-NEXT:    mull a9, a4, a10
+; XTENSA-MUL-NEXT:    sub a14, a9, a2
+; XTENSA-MUL-NEXT:    add a9, a6, a14
+; XTENSA-MUL-NEXT:    or a7, a11, a11
+; XTENSA-MUL-NEXT:    s32i a9, a1, 4 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    bltu a9, a6, .LBB30_6
+; XTENSA-MUL-NEXT:  # %bb.5:
+; XTENSA-MUL-NEXT:    or a7, a5, a5
+; XTENSA-MUL-NEXT:  .LBB30_6:
+; XTENSA-MUL-NEXT:    s32i a7, a1, 0 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    neg a9, a3
+; XTENSA-MUL-NEXT:    or a7, a11, a11
+; XTENSA-MUL-NEXT:    bltu a6, a9, .LBB30_8
+; XTENSA-MUL-NEXT:  # %bb.7:
+; XTENSA-MUL-NEXT:    or a7, a5, a5
+; XTENSA-MUL-NEXT:  .LBB30_8:
+; XTENSA-MUL-NEXT:    or a6, a11, a11
+; XTENSA-MUL-NEXT:    bltu a12, a15, .LBB30_10
+; XTENSA-MUL-NEXT:  # %bb.9:
+; XTENSA-MUL-NEXT:    or a6, a5, a5
+; XTENSA-MUL-NEXT:  .LBB30_10:
+; XTENSA-MUL-NEXT:    muluh a8, a3, a8
+; XTENSA-MUL-NEXT:    add a8, a8, a6
+; XTENSA-MUL-NEXT:    add a8, a8, a7
+; XTENSA-MUL-NEXT:    l32i a9, a1, 16 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    mull a9, a9, a10
+; XTENSA-MUL-NEXT:    muluh a7, a4, a10
+; XTENSA-MUL-NEXT:    sub a7, a7, a4
+; XTENSA-MUL-NEXT:    add a9, a7, a9
+; XTENSA-MUL-NEXT:    sub a7, a13, a2
+; XTENSA-MUL-NEXT:    sub a7, a7, a3
+; XTENSA-MUL-NEXT:    add a7, a7, a9
+; XTENSA-MUL-NEXT:    l32i a9, a1, 8 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    bltu a14, a9, .LBB30_12
+; XTENSA-MUL-NEXT:  # %bb.11:
+; XTENSA-MUL-NEXT:    or a11, a5, a5
+; XTENSA-MUL-NEXT:  .LBB30_12:
+; XTENSA-MUL-NEXT:    add a9, a7, a11
+; XTENSA-MUL-NEXT:    add a8, a8, a9
+; XTENSA-MUL-NEXT:    l32i a9, a1, 0 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    add a5, a8, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a10
+; XTENSA-MUL-NEXT:    l32i a3, a1, 12 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a4, a1, 4 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a15, a1, 20 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a14, a1, 24 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a13, a1, 28 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a12, a1, 32 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    addi a8, a1, 48
+; XTENSA-MUL-NEXT:    or a1, a8, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i128 %a, -3840
   ret i128 %1
 }
@@ -796,6 +1098,94 @@ define i128 @muli128_m63(i128 %a) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 80
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muli128_m63:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    addi a8, a1, -48
+; XTENSA-MUL-NEXT:    or a1, a8, a8
+; XTENSA-MUL-NEXT:    s32i a12, a1, 32 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a13, a1, 28 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a14, a1, 24 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a15, a1, 20 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a5, a1, 16 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    movi a10, -63
+; XTENSA-MUL-NEXT:    muluh a8, a2, a10
+; XTENSA-MUL-NEXT:    mull a7, a3, a10
+; XTENSA-MUL-NEXT:    add a8, a7, a8
+; XTENSA-MUL-NEXT:    movi a5, 0
+; XTENSA-MUL-NEXT:    movi a11, 1
+; XTENSA-MUL-NEXT:    or a9, a11, a11
+; XTENSA-MUL-NEXT:    bltu a8, a7, .LBB31_2
+; XTENSA-MUL-NEXT:  # %bb.1:
+; XTENSA-MUL-NEXT:    or a9, a5, a5
+; XTENSA-MUL-NEXT:  .LBB31_2:
+; XTENSA-MUL-NEXT:    muluh a7, a3, a10
+; XTENSA-MUL-NEXT:    add a15, a7, a9
+; XTENSA-MUL-NEXT:    sub a8, a8, a2
+; XTENSA-MUL-NEXT:    neg a7, a2
+; XTENSA-MUL-NEXT:    or a9, a11, a11
+; XTENSA-MUL-NEXT:    s32i a8, a1, 12 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a7, a1, 8 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    bltu a8, a7, .LBB31_4
+; XTENSA-MUL-NEXT:  # %bb.3:
+; XTENSA-MUL-NEXT:    or a9, a5, a5
+; XTENSA-MUL-NEXT:  .LBB31_4:
+; XTENSA-MUL-NEXT:    movi a8, -1
+; XTENSA-MUL-NEXT:    muluh a13, a2, a8
+; XTENSA-MUL-NEXT:    add a9, a13, a9
+; XTENSA-MUL-NEXT:    add a12, a15, a9
+; XTENSA-MUL-NEXT:    sub a6, a12, a3
+; XTENSA-MUL-NEXT:    mull a9, a4, a10
+; XTENSA-MUL-NEXT:    sub a14, a9, a2
+; XTENSA-MUL-NEXT:    add a9, a6, a14
+; XTENSA-MUL-NEXT:    or a7, a11, a11
+; XTENSA-MUL-NEXT:    s32i a9, a1, 4 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    bltu a9, a6, .LBB31_6
+; XTENSA-MUL-NEXT:  # %bb.5:
+; XTENSA-MUL-NEXT:    or a7, a5, a5
+; XTENSA-MUL-NEXT:  .LBB31_6:
+; XTENSA-MUL-NEXT:    s32i a7, a1, 0 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    neg a9, a3
+; XTENSA-MUL-NEXT:    or a7, a11, a11
+; XTENSA-MUL-NEXT:    bltu a6, a9, .LBB31_8
+; XTENSA-MUL-NEXT:  # %bb.7:
+; XTENSA-MUL-NEXT:    or a7, a5, a5
+; XTENSA-MUL-NEXT:  .LBB31_8:
+; XTENSA-MUL-NEXT:    or a6, a11, a11
+; XTENSA-MUL-NEXT:    bltu a12, a15, .LBB31_10
+; XTENSA-MUL-NEXT:  # %bb.9:
+; XTENSA-MUL-NEXT:    or a6, a5, a5
+; XTENSA-MUL-NEXT:  .LBB31_10:
+; XTENSA-MUL-NEXT:    muluh a8, a3, a8
+; XTENSA-MUL-NEXT:    add a8, a8, a6
+; XTENSA-MUL-NEXT:    add a8, a8, a7
+; XTENSA-MUL-NEXT:    l32i a9, a1, 16 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    mull a9, a9, a10
+; XTENSA-MUL-NEXT:    muluh a7, a4, a10
+; XTENSA-MUL-NEXT:    sub a7, a7, a4
+; XTENSA-MUL-NEXT:    add a9, a7, a9
+; XTENSA-MUL-NEXT:    sub a7, a13, a2
+; XTENSA-MUL-NEXT:    sub a7, a7, a3
+; XTENSA-MUL-NEXT:    add a7, a7, a9
+; XTENSA-MUL-NEXT:    l32i a9, a1, 8 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    bltu a14, a9, .LBB31_12
+; XTENSA-MUL-NEXT:  # %bb.11:
+; XTENSA-MUL-NEXT:    or a11, a5, a5
+; XTENSA-MUL-NEXT:  .LBB31_12:
+; XTENSA-MUL-NEXT:    add a9, a7, a11
+; XTENSA-MUL-NEXT:    add a8, a8, a9
+; XTENSA-MUL-NEXT:    l32i a9, a1, 0 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    add a5, a8, a9
+; XTENSA-MUL-NEXT:    mull a2, a2, a10
+; XTENSA-MUL-NEXT:    l32i a3, a1, 12 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a4, a1, 4 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a15, a1, 20 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a14, a1, 24 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a13, a1, 28 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a12, a1, 32 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    addi a8, a1, 48
+; XTENSA-MUL-NEXT:    or a1, a8, a8
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i128 %a, -63
   ret i128 %1
 }
@@ -917,6 +1307,77 @@ define i64 @mulhsu_i64(i64 %a, i64 %b) nounwind {
 ; XTENSA-NEXT:    addi a8, a1, 64
 ; XTENSA-NEXT:    or a1, a8, a8
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulhsu_i64:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    addi a8, a1, -16
+; XTENSA-MUL-NEXT:    or a1, a8, a8
+; XTENSA-MUL-NEXT:    s32i a12, a1, 12 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a13, a1, 8 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a14, a1, 4 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    s32i a15, a1, 0 # 4-byte Folded Spill
+; XTENSA-MUL-NEXT:    muluh a8, a2, a4
+; XTENSA-MUL-NEXT:    mull a7, a3, a4
+; XTENSA-MUL-NEXT:    add a8, a7, a8
+; XTENSA-MUL-NEXT:    movi a10, 0
+; XTENSA-MUL-NEXT:    movi a9, 1
+; XTENSA-MUL-NEXT:    or a11, a9, a9
+; XTENSA-MUL-NEXT:    bltu a8, a7, .LBB32_2
+; XTENSA-MUL-NEXT:  # %bb.1:
+; XTENSA-MUL-NEXT:    or a11, a10, a10
+; XTENSA-MUL-NEXT:  .LBB32_2:
+; XTENSA-MUL-NEXT:    muluh a7, a3, a4
+; XTENSA-MUL-NEXT:    add a11, a7, a11
+; XTENSA-MUL-NEXT:    mull a7, a2, a5
+; XTENSA-MUL-NEXT:    add a6, a7, a8
+; XTENSA-MUL-NEXT:    or a8, a9, a9
+; XTENSA-MUL-NEXT:    bltu a6, a7, .LBB32_4
+; XTENSA-MUL-NEXT:  # %bb.3:
+; XTENSA-MUL-NEXT:    or a8, a10, a10
+; XTENSA-MUL-NEXT:  .LBB32_4:
+; XTENSA-MUL-NEXT:    muluh a7, a2, a5
+; XTENSA-MUL-NEXT:    add a8, a7, a8
+; XTENSA-MUL-NEXT:    add a12, a11, a8
+; XTENSA-MUL-NEXT:    mull a14, a3, a5
+; XTENSA-MUL-NEXT:    add a15, a14, a12
+; XTENSA-MUL-NEXT:    srai a6, a5, 31
+; XTENSA-MUL-NEXT:    mull a7, a6, a2
+; XTENSA-MUL-NEXT:    add a8, a15, a7
+; XTENSA-MUL-NEXT:    or a4, a9, a9
+; XTENSA-MUL-NEXT:    bgeu a8, a15, .LBB32_9
+; XTENSA-MUL-NEXT:  # %bb.5:
+; XTENSA-MUL-NEXT:    or a13, a9, a9
+; XTENSA-MUL-NEXT:    bgeu a15, a14, .LBB32_10
+; XTENSA-MUL-NEXT:  .LBB32_6:
+; XTENSA-MUL-NEXT:    bltu a12, a11, .LBB32_8
+; XTENSA-MUL-NEXT:  .LBB32_7:
+; XTENSA-MUL-NEXT:    or a9, a10, a10
+; XTENSA-MUL-NEXT:  .LBB32_8:
+; XTENSA-MUL-NEXT:    muluh a10, a3, a5
+; XTENSA-MUL-NEXT:    add a9, a10, a9
+; XTENSA-MUL-NEXT:    add a9, a9, a13
+; XTENSA-MUL-NEXT:    mull a10, a6, a3
+; XTENSA-MUL-NEXT:    muluh a11, a6, a2
+; XTENSA-MUL-NEXT:    add a10, a11, a10
+; XTENSA-MUL-NEXT:    add a10, a10, a7
+; XTENSA-MUL-NEXT:    add a9, a9, a10
+; XTENSA-MUL-NEXT:    add a3, a9, a4
+; XTENSA-MUL-NEXT:    or a2, a8, a8
+; XTENSA-MUL-NEXT:    l32i a15, a1, 0 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a14, a1, 4 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a13, a1, 8 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    l32i a12, a1, 12 # 4-byte Folded Reload
+; XTENSA-MUL-NEXT:    addi a8, a1, 16
+; XTENSA-MUL-NEXT:    or a1, a8, a8
+; XTENSA-MUL-NEXT:    ret
+; XTENSA-MUL-NEXT:  .LBB32_9:
+; XTENSA-MUL-NEXT:    or a4, a10, a10
+; XTENSA-MUL-NEXT:    or a13, a9, a9
+; XTENSA-MUL-NEXT:    bltu a15, a14, .LBB32_6
+; XTENSA-MUL-NEXT:  .LBB32_10:
+; XTENSA-MUL-NEXT:    or a13, a10, a10
+; XTENSA-MUL-NEXT:    bgeu a12, a11, .LBB32_7
+; XTENSA-MUL-NEXT:    j .LBB32_8
   %1 = zext i64 %a to i128
   %2 = sext i64 %b to i128
   %3 = mul i128 %1, %2
@@ -933,6 +1394,14 @@ define i8 @muladd_demand(i8 %x, i8 %y) nounwind {
 ; XTENSA-NEXT:    movi a9, 15
 ; XTENSA-NEXT:    and a2, a8, a9
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muladd_demand:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    slli a8, a2, 1
+; XTENSA-MUL-NEXT:    sub a8, a3, a8
+; XTENSA-MUL-NEXT:    movi a9, 15
+; XTENSA-MUL-NEXT:    and a2, a8, a9
+; XTENSA-MUL-NEXT:    ret
   %m = mul i8 %x, 14
   %a = add i8 %y, %m
   %r = and i8 %a, 15
@@ -946,6 +1415,13 @@ define i8 @mulsub_demand(i8 %x, i8 %y) nounwind {
 ; XTENSA-NEXT:    movi a9, 15
 ; XTENSA-NEXT:    and a2, a8, a9
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulsub_demand:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    addx2 a8, a2, a3
+; XTENSA-MUL-NEXT:    movi a9, 15
+; XTENSA-MUL-NEXT:    and a2, a8, a9
+; XTENSA-MUL-NEXT:    ret
   %m = mul i8 %x, 14
   %a = sub i8 %y, %m
   %r = and i8 %a, 15
@@ -960,6 +1436,14 @@ define i8 @muladd_demand_2(i8 %x, i8 %y) nounwind {
 ; XTENSA-NEXT:    movi a9, -16
 ; XTENSA-NEXT:    or a2, a8, a9
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: muladd_demand_2:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    slli a8, a2, 1
+; XTENSA-MUL-NEXT:    sub a8, a3, a8
+; XTENSA-MUL-NEXT:    movi a9, -16
+; XTENSA-MUL-NEXT:    or a2, a8, a9
+; XTENSA-MUL-NEXT:    ret
   %m = mul i8 %x, 14
   %a = add i8 %y, %m
   %r = or i8 %a, 240
@@ -973,6 +1457,13 @@ define i8 @mulsub_demand_2(i8 %x, i8 %y) nounwind {
 ; XTENSA-NEXT:    movi a9, -16
 ; XTENSA-NEXT:    or a2, a8, a9
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mulsub_demand_2:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    addx2 a8, a2, a3
+; XTENSA-MUL-NEXT:    movi a9, -16
+; XTENSA-MUL-NEXT:    or a2, a8, a9
+; XTENSA-MUL-NEXT:    ret
   %m = mul i8 %x, 14
   %a = sub i8 %y, %m
   %r = or i8 %a, 240
@@ -984,6 +1475,11 @@ define signext i32 @mul_imm_2(i32 %a) nounwind {
 ; XTENSA:       # %bb.0:
 ; XTENSA-NEXT:    slli a2, a2, 1
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul_imm_2:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    slli a2, a2, 1
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 2
   ret i32 %1
 }
@@ -993,6 +1489,11 @@ define signext i32 @mul_imm_1024(i32 %a) nounwind {
 ; XTENSA:       # %bb.0:
 ; XTENSA-NEXT:    slli a2, a2, 10
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul_imm_1024:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    slli a2, a2, 10
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 1024
   ret i32 %1
 }
@@ -1002,6 +1503,11 @@ define signext i32 @mul_imm_16384(i32 %a) nounwind {
 ; XTENSA:       # %bb.0:
 ; XTENSA-NEXT:    slli a2, a2, 14
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul_imm_16384:
+; XTENSA-MUL:       # %bb.0:
+; XTENSA-MUL-NEXT:    slli a2, a2, 14
+; XTENSA-MUL-NEXT:    ret
   %1 = mul i32 %a, 16384
   ret i32 %1
 }
@@ -1015,6 +1521,15 @@ define <4 x i32> @mul_vec_splat_constant(<4 x i32> %a) {
 ; XTENSA-NEXT:    slli a4, a4, 2
 ; XTENSA-NEXT:    slli a5, a5, 2
 ; XTENSA-NEXT:    ret
+;
+; XTENSA-MUL-LABEL: mul_vec_splat_constant:
+; XTENSA-MUL:         .cfi_startproc
+; XTENSA-MUL-NEXT:  # %bb.0:
+; XTENSA-MUL-NEXT:    slli a2, a2, 2
+; XTENSA-MUL-NEXT:    slli a3, a3, 2
+; XTENSA-MUL-NEXT:    slli a4, a4, 2
+; XTENSA-MUL-NEXT:    slli a5, a5, 2
+; XTENSA-MUL-NEXT:    ret
   %mul = mul <4 x i32> %a, <i32 4, i32 4, i32 4, i32 4>
   ret <4 x i32> %mul
 }
diff --git a/llvm/test/CodeGen/Xtensa/rem.ll b/llvm/test/CodeGen/Xtensa/rem.ll
new file mode 100644
index 0000000000000..5246cb4f3cdf7
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/rem.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=xtensa -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=XTENSA %s
+; RUN: llc -mtriple=xtensa -mattr=+div32 -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=XTENSA-DIV %s
+
+define i32 @srem(i32 signext %a0, i32 signext %a1) nounwind readnone {
+; XTENSA-LABEL: srem:
+; XTENSA:       # %bb.0: # %entry
+; XTENSA-NEXT:    addi a8, a1, -16
+; XTENSA-NEXT:    or a1, a8, a8
+; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    l32r a8, .LCPI0_0
+; XTENSA-NEXT:    callx0 a8
+; XTENSA-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    addi a8, a1, 16
+; XTENSA-NEXT:    or a1, a8, a8
+; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: srem:
+; XTENSA-DIV:       # %bb.0: # %entry
+; XTENSA-DIV-NEXT:    rems a2, a2, a3
+; XTENSA-DIV-NEXT:    ret
+entry:
+  %rem = srem i32 %a0, %a1
+  ret i32 %rem
+}
+
+define i32 @urem(i32 signext %a0, i32 signext %a1) nounwind readnone {
+; XTENSA-LABEL: urem:
+; XTENSA:       # %bb.0: # %entry
+; XTENSA-NEXT:    addi a8, a1, -16
+; XTENSA-NEXT:    or a1, a8, a8
+; XTENSA-NEXT:    s32i a0, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    l32r a8, .LCPI1_0
+; XTENSA-NEXT:    callx0 a8
+; XTENSA-NEXT:    l32i a0, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    addi a8, a1, 16
+; XTENSA-NEXT:    or a1, a8, a8
+; XTENSA-NEXT:    ret
+;
+; XTENSA-DIV-LABEL: urem:
+; XTENSA-DIV:       # %bb.0: # %entry
+; XTENSA-DIV-NEXT:    remu a2, a2, a3
+; XTENSA-DIV-NEXT:    ret
+entry:
+  %rem = urem i32 %a0, %a1
+  ret i32 %rem
+}
diff --git a/llvm/test/MC/Xtensa/div.s b/llvm/test/MC/Xtensa/div.s
new file mode 100644
index 0000000000000..05af7f9dbd081
--- /dev/null
+++ b/llvm/test/MC/Xtensa/div.s
@@ -0,0 +1,24 @@
+# RUN: llvm-mc %s -triple=xtensa -show-encoding --mattr=+div32 \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+
+# Instruction format RRR
+# CHECK-INST: quos a3, a4, a5
+# CHECK: encoding: [0x50,0x34,0xd2]
+quos a3, a4, a5
+
+# Instruction format RRR
+# CHECK-INST: quou a3, a4, a5
+# CHECK: encoding: [0x50,0x34,0xc2]
+quou a3, a4, a5
+
+# Instruction format RRR
+# CHECK-INST: rems a3, a4, a5
+# CHECK: encoding:  [0x50,0x34,0xf2]
+rems a3, a4, a5
+
+# Instruction format RRR
+# CHECK-INST: remu a3, a4, a5
+# CHECK: encoding:  [0x50,0x34,0xe2]
+remu a3, a4, a5
diff --git a/llvm/test/MC/Xtensa/mul.s b/llvm/test/MC/Xtensa/mul.s
new file mode 100644
index 0000000000000..d7096a8128cd6
--- /dev/null
+++ b/llvm/test/MC/Xtensa/mul.s
@@ -0,0 +1,29 @@
+# RUN: llvm-mc %s -triple=xtensa -show-encoding --mattr=+mul16,mul32high,mul32 \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+
+# Instruction format RRR
+# CHECK-INST: mul16s a3, a4, a5
+# CHECK: encoding: [0x50,0x34,0xd1]
+mul16s a3, a4, a5
+
+# Instruction format RRR
+# CHECK-INST: mul16u a3, a4, a5
+# CHECK: encoding: [0x50,0x34,0xc1]
+mul16u a3, a4, a5
+
+# Instruction format RRR
+# CHECK-INST: mull a3, a4, a5
+# CHECK: encoding:  [0x50,0x34,0x82]
+mull a3, a4, a5
+
+# Instruction format RRR
+# CHECK-INST: muluh a3, a4, a5
+# CHECK: encoding:  [0x50,0x34,0xa2]
+muluh a3, a4, a5
+
+# Instruction format RRR
+# CHECK-INST: mulsh a3, a4, a5
+# CHECK: encoding:  [0x50,0x34,0xb2]
+mulsh a3, a4, a5

From 58774f1b1f98402c8d62cdde01c1adea067aaf96 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 10:26:48 -0700
Subject: [PATCH 441/710] [CodeGen] Construct SmallVector with iterator ranges
 (NFC) (#136258)

---
 llvm/lib/CodeGen/CodeGenPrepare.cpp            | 5 ++---
 llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 5 ++---
 llvm/lib/CodeGen/MachineInstr.cpp              | 5 ++---
 llvm/lib/CodeGen/RegAllocFast.cpp              | 5 ++---
 llvm/lib/CodeGen/RegisterUsageInfo.cpp         | 6 ++----
 5 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 1bbd1b6f71b14..0cc9fb88b4293 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -834,9 +834,8 @@ bool CodeGenPrepare::eliminateFallThrough(Function &F, DominatorTree *DT) {
   // Scan all of the blocks in the function, except for the entry block.
   // Use a temporary array to avoid iterator being invalidated when
   // deleting blocks.
-  SmallVector<WeakTrackingVH, 16> Blocks;
-  for (auto &Block : llvm::drop_begin(F))
-    Blocks.push_back(&Block);
+  SmallVector<WeakTrackingVH, 16> Blocks(
+      llvm::make_pointer_range(llvm::drop_begin(F)));
 
   SmallSet<WeakTrackingVH, 16> Preds;
   for (auto &Block : Blocks) {
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index dbc838a3782ca..333f0c17bacc5 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -867,9 +867,8 @@ void CombinerHelper::applyCombineExtendingLoads(
 
   // Rewrite all the uses to fix up the types.
   auto &LoadValue = MI.getOperand(0);
-  SmallVector<MachineOperand *, 4> Uses;
-  for (auto &UseMO : MRI.use_operands(LoadValue.getReg()))
-    Uses.push_back(&UseMO);
+  SmallVector<MachineOperand *, 4> Uses(
+      llvm::make_pointer_range(MRI.use_operands(LoadValue.getReg())));
 
   for (auto *UseMO : Uses) {
     MachineInstr *UseMI = UseMO->getParent();
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 7e4a32f67fe6a..33910d0ec6aeb 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -2442,9 +2442,8 @@ static const DIExpression *computeExprForSpill(
 static const DIExpression *computeExprForSpill(const MachineInstr &MI,
                                                Register SpillReg) {
   assert(MI.hasDebugOperandForReg(SpillReg) && "Spill Reg is not used in MI.");
-  SmallVector<const MachineOperand *> SpillOperands;
-  for (const MachineOperand &Op : MI.getDebugOperandsForReg(SpillReg))
-    SpillOperands.push_back(&Op);
+  SmallVector<const MachineOperand *> SpillOperands(
+      llvm::make_pointer_range(MI.getDebugOperandsForReg(SpillReg)));
   return computeExprForSpill(MI, SpillOperands);
 }
 
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index 857cf85a8acbc..bb118dd9e1867 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -1733,9 +1733,8 @@ void RegAllocFastImpl::handleDebugValue(MachineInstr &MI) {
     // See if this virtual register has already been allocated to a physical
     // register or spilled to a stack slot.
     LiveRegMap::iterator LRI = findLiveVirtReg(Reg);
-    SmallVector<MachineOperand *> DbgOps;
-    for (MachineOperand &Op : MI.getDebugOperandsForReg(Reg))
-      DbgOps.push_back(&Op);
+    SmallVector<MachineOperand *> DbgOps(
+        llvm::make_pointer_range(MI.getDebugOperandsForReg(Reg)));
 
     if (LRI != LiveVirtRegs.end() && LRI->PhysReg) {
       // Update every use of Reg within MI.
diff --git a/llvm/lib/CodeGen/RegisterUsageInfo.cpp b/llvm/lib/CodeGen/RegisterUsageInfo.cpp
index 6a3e34be6d391..7a4628a6e91d4 100644
--- a/llvm/lib/CodeGen/RegisterUsageInfo.cpp
+++ b/llvm/lib/CodeGen/RegisterUsageInfo.cpp
@@ -72,11 +72,9 @@ PhysicalRegisterUsageInfo::getRegUsageInfo(const Function &FP) {
 void PhysicalRegisterUsageInfo::print(raw_ostream &OS, const Module *M) const {
   using FuncPtrRegMaskPair = std::pair<const Function *, std::vector<uint32_t>>;
 
-  SmallVector<const FuncPtrRegMaskPair *, 64> FPRMPairVector;
-
   // Create a vector of pointer to RegMasks entries
-  for (const auto &RegMask : RegMasks)
-    FPRMPairVector.push_back(&RegMask);
+  SmallVector<const FuncPtrRegMaskPair *, 64> FPRMPairVector(
+      llvm::make_pointer_range(RegMasks));
 
   // sort the vector to print analysis in alphabatic order of function name.
   llvm::sort(

From b2ba53172e82354e1318ae7e18c637ac2107aeca Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 10:27:05 -0700
Subject: [PATCH 442/710] [Transforms] Construct SmallVector with iterator
 ranges (NFC) (#136259)

---
 llvm/lib/Transforms/IPO/ExpandVariadics.cpp           |  4 +---
 llvm/lib/Transforms/Scalar/ConstraintElimination.cpp  |  4 +---
 llvm/lib/Transforms/Scalar/IndVarSimplify.cpp         |  5 ++---
 llvm/lib/Transforms/Scalar/LoopInterchange.cpp        | 10 ++++------
 llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp        |  5 ++---
 llvm/lib/Transforms/Utils/BasicBlockUtils.cpp         |  4 +---
 llvm/lib/Transforms/Utils/CodeExtractor.cpp           |  5 +----
 llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp |  5 ++---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp       |  5 ++---
 9 files changed, 16 insertions(+), 31 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
index 3121659edadd8..e25f23107966d 100644
--- a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
+++ b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
@@ -604,9 +604,7 @@ ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
   Builder.CreateIntrinsic(Intrinsic::vastart, {DL.getAllocaPtrType(Ctx)},
                           {VaListInstance});
 
-  SmallVector<Value *> Args;
-  for (Argument &A : F.args())
-    Args.push_back(&A);
+  SmallVector<Value *> Args(llvm::make_pointer_range(F.args()));
 
   Type *ParameterType = ABI->vaListParameterType(M);
   if (ABI->vaListPassedInSSARegister())
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 2604bb9153f17..58d705eb6aa96 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1738,9 +1738,7 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI,
                                  OptimizationRemarkEmitter &ORE) {
   bool Changed = false;
   DT.updateDFSNumbers();
-  SmallVector<Value *> FunctionArgs;
-  for (Value &Arg : F.args())
-    FunctionArgs.push_back(&Arg);
+  SmallVector<Value *> FunctionArgs(llvm::make_pointer_range(F.args()));
   ConstraintInfo Info(F.getDataLayout(), FunctionArgs);
   State S(DT, LI, SE);
   std::unique_ptr<Module> ReproducerModule(
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 3f04f7f57c54f..db97f7e1efcfd 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -599,9 +599,8 @@ bool IndVarSimplify::simplifyAndExtend(Loop *L,
       L->getBlocks()[0]->getModule(), Intrinsic::experimental_guard);
   bool HasGuards = GuardDecl && !GuardDecl->use_empty();
 
-  SmallVector<PHINode *, 8> LoopPhis;
-  for (PHINode &PN : L->getHeader()->phis())
-    LoopPhis.push_back(&PN);
+  SmallVector<PHINode *, 8> LoopPhis(
+      llvm::make_pointer_range(L->getHeader()->phis()));
 
   // Each round of simplification iterates through the SimplifyIVUsers worklist
   // for all current phis, then determines whether any IVs can be
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 1dccba4cfa7b8..8c8effe2b013e 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -1574,13 +1574,11 @@ static void moveLCSSAPhis(BasicBlock *InnerExit, BasicBlock *InnerHeader,
     P.eraseFromParent();
   }
 
-  SmallVector<PHINode *, 8> LcssaInnerExit;
-  for (PHINode &P : InnerExit->phis())
-    LcssaInnerExit.push_back(&P);
+  SmallVector<PHINode *, 8> LcssaInnerExit(
+      llvm::make_pointer_range(InnerExit->phis()));
 
-  SmallVector<PHINode *, 8> LcssaInnerLatch;
-  for (PHINode &P : InnerLatch->phis())
-    LcssaInnerLatch.push_back(&P);
+  SmallVector<PHINode *, 8> LcssaInnerLatch(
+      llvm::make_pointer_range(InnerLatch->phis()));
 
   // Lcssa PHIs for values used outside the inner loop are in InnerExit.
   // If a PHI node has users outside of InnerExit, it has a use outside the
diff --git a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
index 4524446e5947f..221094f170ac7 100644
--- a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
@@ -361,9 +361,8 @@ class ConstantTerminatorFoldingImpl {
     for (BasicBlock *BB : DeadExitBlocks) {
       // Eliminate all Phis and LandingPads from dead exits.
       // TODO: Consider removing all instructions in this dead block.
-      SmallVector<Instruction *, 4> DeadInstructions;
-      for (auto &PN : BB->phis())
-        DeadInstructions.push_back(&PN);
+      SmallVector<Instruction *, 4> DeadInstructions(
+          llvm::make_pointer_range(BB->phis()));
 
       if (auto *LandingPad = dyn_cast<LandingPadInst>(BB->getFirstNonPHIIt()))
         DeadInstructions.emplace_back(LandingPad);
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index b78270f6309ff..9ed8b06c22fde 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -165,9 +165,7 @@ bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
                           MemorySSAUpdater *MSSAU) {
   // Recursively deleting a PHI may cause multiple PHIs to be deleted
   // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
-  SmallVector<WeakTrackingVH, 8> PHIs;
-  for (PHINode &PN : BB->phis())
-    PHIs.push_back(&PN);
+  SmallVector<WeakTrackingVH, 8> PHIs(llvm::make_pointer_range(BB->phis()));
 
   bool Changed = false;
   for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 2b055020022be..18af0972bc36d 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -91,10 +91,7 @@ static bool isBlockValidForExtraction(const BasicBlock &BB,
   // don't hoist code that uses another basicblock address, as it's likely to
   // lead to unexpected behavior, like cross-function jumps
   SmallPtrSet<User const *, 16> Visited;
-  SmallVector<User const *, 16> ToVisit;
-
-  for (Instruction const &Inst : BB)
-    ToVisit.push_back(&Inst);
+  SmallVector<User const *, 16> ToVisit(llvm::make_pointer_range(BB));
 
   while (!ToVisit.empty()) {
     User const *Curr = ToVisit.pop_back_val();
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index e25ec6c3b2a58..31d8acb706997 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1715,9 +1715,8 @@ SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
                                   SmallVectorImpl<WeakTrackingVH> &DeadInsts,
                                   const TargetTransformInfo *TTI) {
   // Find integer phis in order of increasing width.
-  SmallVector<PHINode*, 8> Phis;
-  for (PHINode &PN : L->getHeader()->phis())
-    Phis.push_back(&PN);
+  SmallVector<PHINode *, 8> Phis(
+      llvm::make_pointer_range(L->getHeader()->phis()));
 
   if (TTI)
     // Use stable_sort to preserve order of equivalent PHIs, so the order
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 87fcf7e5d9742..bfceee46a237c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8267,9 +8267,8 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
   // The vec.epilog.iter.check block may contain Phi nodes from inductions or
   // reductions which merge control-flow from the latch block and the middle
   // block. Update the incoming values here and move the Phi into the preheader.
-  SmallVector<PHINode *, 4> PhisInBlock;
-  for (PHINode &Phi : VecEpilogueIterationCountCheck->phis())
-    PhisInBlock.push_back(&Phi);
+  SmallVector<PHINode *, 4> PhisInBlock(
+      llvm::make_pointer_range(VecEpilogueIterationCountCheck->phis()));
 
   for (PHINode *Phi : PhisInBlock) {
     Phi->moveBefore(LoopVectorPreHeader->getFirstNonPHIIt());

From e298f16e895f1c3e5571ec1adb84f29646b84637 Mon Sep 17 00:00:00 2001
From: Andy Kaylor <akaylor@nvidia.com>
Date: Fri, 18 Apr 2025 10:30:36 -0700
Subject: [PATCH 443/710] [CIR] Upstream support for record packing and padding
 (#136036)

This change adds support for packing and padding record types in ClangIR
and introduces some infrastructure needed for this computation.

Although union support has not been upstreamed yet, there is no good way
to report unions as NYI in the layout computation, so the code added
here includes layout computation for unions. Unions will be added soon.
---
 .../clang/CIR/Dialect/IR/CIRDataLayout.h      |  33 ++++++
 .../include/clang/CIR/Dialect/IR/CIRTypes.td  |   5 +
 clang/include/clang/CIR/MissingFeatures.h     |   3 -
 .../CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp | 104 ++++++++++++++++--
 clang/lib/CIR/Dialect/IR/CIRTypes.cpp         |  83 +++++++++++++-
 .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp |   5 +-
 clang/test/CIR/CodeGen/struct.c               |  46 +++++++-
 7 files changed, 254 insertions(+), 25 deletions(-)
 create mode 100644 clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h

diff --git a/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h b/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h
new file mode 100644
index 0000000000000..62fc53f2362fb
--- /dev/null
+++ b/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Provides an LLVM-like API wrapper to DLTI and MLIR layout queries. This
+// makes it easier to port some of LLVM codegen layout logic to CIR.
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CIR_DIALECT_IR_CIRDATALAYOUT_H
+#define CLANG_CIR_DIALECT_IR_CIRDATALAYOUT_H
+
+#include "mlir/IR/BuiltinOps.h"
+
+namespace cir {
+
+// TODO(cir): This might be replaced by a CIRDataLayout interface which can
+// provide the same functionalities.
+class CIRDataLayout {
+  // This is starting with the minimum functionality needed for code that is
+  // being upstreamed. Additional methods and members will be added as needed.
+public:
+  mlir::DataLayout layout;
+
+  /// Constructs a DataLayout the module's data layout attribute.
+  CIRDataLayout(mlir::ModuleOp modOp) : layout{modOp} {}
+};
+
+} // namespace cir
+
+#endif // CLANG_CIR_DIALECT_IR_CIRDATALAYOUT_H
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
index 23e20755dca95..b028bc7db4e59 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
@@ -502,6 +502,11 @@ def CIR_RecordType : CIR_Type<"Record", "record",
 
     void complete(llvm::ArrayRef<mlir::Type> members, bool packed,
                   bool isPadded);
+
+  private:
+    unsigned computeStructSize(const mlir::DataLayout &dataLayout) const;
+    uint64_t computeStructAlignment(const mlir::DataLayout &dataLayout) const;
+  public:
   }];
 
   let hasCustomAssemblyFormat = 1;
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 0b66ed31d01f1..0105d1bdaf3fd 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -102,13 +102,10 @@ struct MissingFeatures {
   static bool shouldReverseUnaryCondOnBoolExpr() { return false; }
 
   // RecordType
-  static bool recordTypeLayoutInfo() { return false; }
   static bool recursiveRecordLayout() { return false; }
   static bool skippedLayout() { return false; }
   static bool astRecordDeclAttr() { return false; }
   static bool cxxSupport() { return false; }
-  static bool packedRecords() { return false; }
-  static bool recordPadding() { return false; }
   static bool recordZeroInit() { return false; }
   static bool zeroSizeRecordMembers() { return false; }
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
index 4e3adeaf50187..5e209b5b92503 100644
--- a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/CIR/Dialect/IR/CIRAttrs.h"
+#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
 #include "llvm/Support/Casting.h"
 
 #include <memory>
@@ -57,8 +58,18 @@ struct CIRRecordLowering final {
   CIRRecordLowering(CIRGenTypes &cirGenTypes, const RecordDecl *recordDecl,
                     bool isPacked);
 
+  /// Constructs a MemberInfo instance from an offset and mlir::Type.
+  MemberInfo makeStorageInfo(CharUnits offset, mlir::Type data) {
+    return MemberInfo(offset, MemberInfo::InfoKind::Field, data);
+  }
+
   void lower();
 
+  /// Determines if we need a packed llvm struct.
+  void determinePacked();
+  /// Inserts padding everywhere it's needed.
+  void insertPadding();
+
   void accumulateFields();
 
   CharUnits bitsToCharUnits(uint64_t bitOffset) {
@@ -66,12 +77,33 @@ struct CIRRecordLowering final {
   }
 
   CharUnits getSize(mlir::Type Ty) {
-    assert(!cir::MissingFeatures::recordTypeLayoutInfo());
-    return CharUnits::One();
+    return CharUnits::fromQuantity(dataLayout.layout.getTypeSize(Ty));
   }
   CharUnits getAlignment(mlir::Type Ty) {
-    assert(!cir::MissingFeatures::recordTypeLayoutInfo());
-    return CharUnits::One();
+    return CharUnits::fromQuantity(dataLayout.layout.getTypeABIAlignment(Ty));
+  }
+
+  /// Wraps cir::IntType with some implicit arguments.
+  mlir::Type getUIntNType(uint64_t numBits) {
+    unsigned alignedBits = llvm::PowerOf2Ceil(numBits);
+    alignedBits = std::max(8u, alignedBits);
+    return cir::IntType::get(&cirGenTypes.getMLIRContext(), alignedBits,
+                             /*isSigned=*/false);
+  }
+
+  mlir::Type getCharType() {
+    return cir::IntType::get(&cirGenTypes.getMLIRContext(),
+                             astContext.getCharWidth(),
+                             /*isSigned=*/false);
+  }
+
+  mlir::Type getByteArrayType(CharUnits numberOfChars) {
+    assert(!numberOfChars.isZero() && "Empty byte arrays aren't allowed.");
+    mlir::Type type = getCharType();
+    return numberOfChars == CharUnits::One()
+               ? type
+               : cir::ArrayType::get(type.getContext(), type,
+                                     numberOfChars.getQuantity());
   }
 
   mlir::Type getStorageType(const FieldDecl *fieldDecl) {
@@ -100,6 +132,7 @@ struct CIRRecordLowering final {
   // Output fields, consumed by CIRGenTypes::computeRecordLayout
   llvm::SmallVector<mlir::Type, 16> fieldTypes;
   llvm::DenseMap<const FieldDecl *, unsigned> fields;
+  cir::CIRDataLayout dataLayout;
 
   LLVM_PREFERRED_TYPE(bool)
   unsigned zeroInitializable : 1;
@@ -121,6 +154,7 @@ CIRRecordLowering::CIRRecordLowering(CIRGenTypes &cirGenTypes,
       astContext(cirGenTypes.getASTContext()), recordDecl(recordDecl),
       astRecordLayout(
           cirGenTypes.getASTContext().getASTRecordLayout(recordDecl)),
+      dataLayout(cirGenTypes.getCGModule().getModule()),
       zeroInitializable(true), packed(isPacked), padded(false) {}
 
 void CIRRecordLowering::lower() {
@@ -138,18 +172,20 @@ void CIRRecordLowering::lower() {
 
   assert(!cir::MissingFeatures::cxxSupport());
 
+  CharUnits size = astRecordLayout.getSize();
+
   accumulateFields();
 
   llvm::stable_sort(members);
   // TODO: implement clipTailPadding once bitfields are implemented
   assert(!cir::MissingFeatures::bitfields());
-  // TODO: implemented packed records
-  assert(!cir::MissingFeatures::packedRecords());
-  // TODO: implement padding
-  assert(!cir::MissingFeatures::recordPadding());
-  // TODO: support zeroInit
   assert(!cir::MissingFeatures::recordZeroInit());
 
+  members.push_back(makeStorageInfo(size, getUIntNType(8)));
+  determinePacked();
+  insertPadding();
+  members.pop_back();
+
   fillOutputFields();
 }
 
@@ -186,6 +222,56 @@ void CIRRecordLowering::accumulateFields() {
   }
 }
 
+void CIRRecordLowering::determinePacked() {
+  if (packed)
+    return;
+  CharUnits alignment = CharUnits::One();
+
+  // TODO(cir): handle non-virtual base types
+  assert(!cir::MissingFeatures::cxxSupport());
+
+  for (const MemberInfo &member : members) {
+    if (!member.data)
+      continue;
+    // If any member falls at an offset that it not a multiple of its alignment,
+    // then the entire record must be packed.
+    if (member.offset % getAlignment(member.data))
+      packed = true;
+    alignment = std::max(alignment, getAlignment(member.data));
+  }
+  // If the size of the record (the capstone's offset) is not a multiple of the
+  // record's alignment, it must be packed.
+  if (members.back().offset % alignment)
+    packed = true;
+  // Update the alignment of the sentinel.
+  if (!packed)
+    members.back().data = getUIntNType(astContext.toBits(alignment));
+}
+
+void CIRRecordLowering::insertPadding() {
+  std::vector<std::pair<CharUnits, CharUnits>> padding;
+  CharUnits size = CharUnits::Zero();
+  for (const MemberInfo &member : members) {
+    if (!member.data)
+      continue;
+    CharUnits offset = member.offset;
+    assert(offset >= size);
+    // Insert padding if we need to.
+    if (offset !=
+        size.alignTo(packed ? CharUnits::One() : getAlignment(member.data)))
+      padding.push_back(std::make_pair(size, offset - size));
+    size = offset + getSize(member.data);
+  }
+  if (padding.empty())
+    return;
+  padded = true;
+  // Add the padding to the Members list and sort it.
+  for (const std::pair<CharUnits, CharUnits> &paddingPair : padding)
+    members.push_back(makeStorageInfo(paddingPair.first,
+                                      getByteArrayType(paddingPair.second)));
+  llvm::stable_sort(members);
+}
+
 std::unique_ptr<CIRGenRecordLayout>
 CIRGenTypes::computeRecordLayout(const RecordDecl *rd, cir::RecordType *ty) {
   CIRRecordLowering lowering(*this, rd, /*packed=*/false);
diff --git a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
index 160732d9c3610..8b5646f339bb3 100644
--- a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
@@ -177,6 +177,12 @@ void RecordType::print(mlir::AsmPrinter &printer) const {
   // Type not yet printed: continue printing the entire record.
   printer << ' ';
 
+  if (getPacked())
+    printer << "packed ";
+
+  if (getPadded())
+    printer << "padded ";
+
   if (isIncomplete()) {
     printer << "incomplete";
   } else {
@@ -210,6 +216,10 @@ mlir::StringAttr RecordType::getName() const { return getImpl()->name; }
 
 bool RecordType::getIncomplete() const { return getImpl()->incomplete; }
 
+bool RecordType::getPacked() const { return getImpl()->packed; }
+
+bool RecordType::getPadded() const { return getImpl()->padded; }
+
 cir::RecordType::RecordKind RecordType::getKind() const {
   return getImpl()->kind;
 }
@@ -225,17 +235,78 @@ void RecordType::complete(ArrayRef<Type> members, bool packed, bool padded) {
 //===----------------------------------------------------------------------===//
 
 llvm::TypeSize
-RecordType::getTypeSizeInBits(const ::mlir::DataLayout &dataLayout,
-                              ::mlir::DataLayoutEntryListRef params) const {
-  assert(!cir::MissingFeatures::recordTypeLayoutInfo());
-  return llvm::TypeSize::getFixed(8);
+RecordType::getTypeSizeInBits(const mlir::DataLayout &dataLayout,
+                              mlir::DataLayoutEntryListRef params) const {
+  if (isUnion()) {
+    // TODO(CIR): Implement union layout.
+    return llvm::TypeSize::getFixed(8);
+  }
+
+  unsigned recordSize = computeStructSize(dataLayout);
+  return llvm::TypeSize::getFixed(recordSize * 8);
 }
 
 uint64_t
 RecordType::getABIAlignment(const ::mlir::DataLayout &dataLayout,
                             ::mlir::DataLayoutEntryListRef params) const {
-  assert(!cir::MissingFeatures::recordTypeLayoutInfo());
-  return 4;
+  if (isUnion()) {
+    // TODO(CIR): Implement union layout.
+    return 8;
+  }
+
+  // Packed structures always have an ABI alignment of 1.
+  if (getPacked())
+    return 1;
+  return computeStructAlignment(dataLayout);
+}
+
+unsigned
+RecordType::computeStructSize(const mlir::DataLayout &dataLayout) const {
+  assert(isComplete() && "Cannot get layout of incomplete records");
+
+  // This is a similar algorithm to LLVM's StructLayout.
+  unsigned recordSize = 0;
+  uint64_t recordAlignment = 1;
+
+  // We can't use a range-based for loop here because we might be ignoring the
+  // last element.
+  for (mlir::Type ty : getMembers()) {
+    // This assumes that we're calculating size based on the ABI alignment, not
+    // the preferred alignment for each type.
+    const uint64_t tyAlign =
+        (getPacked() ? 1 : dataLayout.getTypeABIAlignment(ty));
+
+    // Add padding to the struct size to align it to the abi alignment of the
+    // element type before than adding the size of the element.
+    recordSize = llvm::alignTo(recordSize, tyAlign);
+    recordSize += dataLayout.getTypeSize(ty);
+
+    // The alignment requirement of a struct is equal to the strictest alignment
+    // requirement of its elements.
+    recordAlignment = std::max(tyAlign, recordAlignment);
+  }
+
+  // At the end, add padding to the struct to satisfy its own alignment
+  // requirement. Otherwise structs inside of arrays would be misaligned.
+  recordSize = llvm::alignTo(recordSize, recordAlignment);
+  return recordSize;
+}
+
+// We also compute the alignment as part of computeStructSize, but this is more
+// efficient. Ideally, we'd like to compute both at once and cache the result,
+// but that's implemented yet.
+// TODO(CIR): Implement a way to cache the result.
+uint64_t
+RecordType::computeStructAlignment(const mlir::DataLayout &dataLayout) const {
+  assert(isComplete() && "Cannot get layout of incomplete records");
+
+  // This is a similar algorithm to LLVM's StructLayout.
+  uint64_t recordAlignment = 1;
+  for (mlir::Type ty : getMembers())
+    recordAlignment =
+        std::max(dataLayout.getTypeABIAlignment(ty), recordAlignment);
+
+  return recordAlignment;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index cb318c88c09c5..8c4a67258df3f 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -1356,12 +1356,11 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
     if (type.getName()) {
       llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(
           type.getContext(), type.getPrefixedName());
-      assert(!cir::MissingFeatures::packedRecords());
-      if (llvmStruct.setBody(llvmMembers, /*isPacked=*/true).failed())
+      if (llvmStruct.setBody(llvmMembers, type.getPacked()).failed())
         llvm_unreachable("Failed to set body of record");
     } else { // Record has no name: lower as literal record.
       llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(
-          type.getContext(), llvmMembers, /*isPacked=*/true);
+          type.getContext(), llvmMembers, type.getPacked());
     }
 
     return llvmStruct;
diff --git a/clang/test/CIR/CodeGen/struct.c b/clang/test/CIR/CodeGen/struct.c
index e1b01e6a8e86c..55f316f27eeb1 100644
--- a/clang/test/CIR/CodeGen/struct.c
+++ b/clang/test/CIR/CodeGen/struct.c
@@ -5,9 +5,19 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
 // RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
 
+// For LLVM IR checks, the structs are defined before the variables, so these
+// checks are at the top.
+// LLVM: %struct.CompleteS = type { i32, i8 }
+// LLVM: %struct.PackedS = type <{ i32, i8 }>
+// LLVM: %struct.PackedAndPaddedS = type <{ i32, i8, i8 }>
+// OGCG: %struct.CompleteS = type { i32, i8 }
+// OGCG: %struct.PackedS = type <{ i32, i8 }>
+// OGCG: %struct.PackedAndPaddedS = type <{ i32, i8, i8 }>
+
 struct IncompleteS *p;
 
-// CIR: cir.global external @p = #cir.ptr<null> : !cir.ptr<!cir.record<struct "IncompleteS" incomplete>>
+// CIR:      cir.global external @p = #cir.ptr<null> : !cir.ptr<!cir.record<struct
+// CIR-SAME:     "IncompleteS" incomplete>>
 // LLVM: @p = dso_local global ptr null
 // OGCG: @p = global ptr null, align 8
 
@@ -16,17 +26,44 @@ struct CompleteS {
   char b;
 } cs;
 
-// CIR:      cir.global external @cs = #cir.zero : !cir.record<struct "CompleteS" {!s32i, !s8i}>
+// CIR:       cir.global external @cs = #cir.zero : !cir.record<struct
+// CIR-SAME:      "CompleteS" {!s32i, !s8i}>
 // LLVM:      @cs = dso_local global %struct.CompleteS zeroinitializer
 // OGCG:      @cs = global %struct.CompleteS zeroinitializer, align 4
 
+#pragma pack(push)
+#pragma pack(1)
+
+struct PackedS {
+  int  a0;
+  char a1;
+} ps;
+
+// CIR:       cir.global external @ps = #cir.zero : !cir.record<struct "PackedS"
+// CIR-SAME:      packed {!s32i, !s8i}>
+// LLVM:      @ps = dso_local global %struct.PackedS zeroinitializer
+// OGCG:      @ps = global %struct.PackedS zeroinitializer, align 1
+
+struct PackedAndPaddedS {
+  int  b0;
+  char b1;
+} __attribute__((aligned(2))) pps;
+
+// CIR:       cir.global external @pps = #cir.zero : !cir.record<struct
+// CIR-SAME:      "PackedAndPaddedS" packed padded {!s32i, !s8i, !u8i}>
+// LLVM:      @pps = dso_local global %struct.PackedAndPaddedS zeroinitializer
+// OGCG:      @pps = global %struct.PackedAndPaddedS zeroinitializer, align 2
+
+#pragma pack(pop)
+
 void f(void) {
   struct IncompleteS *p;
 }
 
 // CIR:      cir.func @f()
 // CIR-NEXT:   cir.alloca !cir.ptr<!cir.record<struct "IncompleteS" incomplete>>,
-// CIR-SAME:       !cir.ptr<!cir.ptr<!cir.record<struct "IncompleteS" incomplete>>>, ["p"]
+// CIR-SAME:       !cir.ptr<!cir.ptr<!cir.record<struct
+// CIR-SAME:       "IncompleteS" incomplete>>>, ["p"]
 // CIR-NEXT:   cir.return
 
 // LLVM:      define void @f()
@@ -44,7 +81,8 @@ void f2(void) {
 
 // CIR:      cir.func @f2()
 // CIR-NEXT:   cir.alloca !cir.record<struct "CompleteS" {!s32i, !s8i}>,
-// CIR-SAME:       !cir.ptr<!cir.record<struct "CompleteS" {!s32i, !s8i}>>, ["s"]
+// CIR-SAME:       !cir.ptr<!cir.record<struct "CompleteS" {!s32i, !s8i}>>,
+// CIR-SAME:       ["s"] {alignment = 4 : i64}
 // CIR-NEXT:   cir.return
 
 // LLVM:      define void @f2()

From fdcee2dd3687053cf8c69a24178c489d6977e93e Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Fri, 18 Apr 2025 13:37:08 -0400
Subject: [PATCH 444/710] [SLP]Reorder tree, if the reorder indices are non
 empty

Need to consider the ordering for all nodes with the specified ordering,
not only loads/store/extracts.

Reviewers: hiraditya, RKSimon

Reviewed By: hiraditya

Pull Request: https://github.com/llvm/llvm-project/pull/136185
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp |  2 ++
 llvm/test/Transforms/SLPVectorizer/revec.ll     | 14 ++++++--------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4301bc7d0d68b..53bf03c57db65 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6711,6 +6711,8 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
            "BinaryOperator and CastInst.");
     return TE.ReorderIndices;
   }
+  if (!TE.ReorderIndices.empty())
+    return TE.ReorderIndices;
   if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::PHI) {
     if (!TE.ReorderIndices.empty())
       return TE.ReorderIndices;
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 10f52c7c341cb..36dbeed9bbcd5 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -416,16 +416,15 @@ define void @test13(<8 x i32> %0, ptr %out0, ptr %out1, ptr %out2) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <32 x i32> @llvm.vector.insert.v32i32.v8i32(<32 x i32> poison, <8 x i32> [[TMP0:%.*]], i64 0)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i32> [[TMP1]], <32 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i32> [[TMP1]], <32 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i32> [[TMP1]], <32 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT:%.*]]
 ; CHECK:       for.end.loopexit:
-; CHECK-NEXT:    [[TMP5:%.*]] = phi <16 x i32> [ [[TMP4]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.v16i32(<16 x i32> [[TMP5]], i64 12)
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <16 x i32> [ [[TMP3]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.v16i32(<16 x i32> [[TMP4]], i64 4)
 ; CHECK-NEXT:    [[OR0:%.*]] = or <4 x i32> [[TMP6]], zeroinitializer
 ; CHECK-NEXT:    store <4 x i32> [[OR0]], ptr [[OUT0:%.*]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.v16i32(<16 x i32> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.v16i32(<16 x i32> [[TMP3]], i64 0)
 ; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr [[OUT1:%.*]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.v16i32(<16 x i32> [[TMP4]], i64 8)
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.v16i32(<16 x i32> [[TMP3]], i64 12)
 ; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr [[OUT2:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -456,11 +455,10 @@ define void @test14(<8 x i1> %0) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = sext <16 x i1> [[TMP2]] to <16 x i16>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT:%.*]]
 ; CHECK:       for.end.loopexit:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi <16 x i16> [ [[TMP6]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i16> @llvm.vector.extract.v4i16.v16i16(<16 x i16> [[TMP7]], i64 12)
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <16 x i16> [ [[TMP5]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i16> @llvm.vector.extract.v4i16.v16i16(<16 x i16> [[TMP6]], i64 4)
 ; CHECK-NEXT:    [[OR0:%.*]] = or <4 x i16> [[TMP8]], zeroinitializer
 ; CHECK-NEXT:    ret void
 ;

From 722d5890cc0d6e3879ebe43b43405f102f658e30 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Fri, 18 Apr 2025 10:40:15 -0700
Subject: [PATCH 445/710] [RISCV] Lower e64 vector_deinterleave via
 ri.vunzip2{a,b} if available (#136321)

If XRivosVizip is available, the ri.vunzip2a and ri.vunzip2b can be used
to the concatenation and register deinterleave shuffle. This patch only
effects the intrinsic lowering (and thus scalable vectors because the
fixed vectors go through shuffle lowering).

Note that this patch is restricted to e64 for staging purposes only. e64
is obviously profitable (i.e. we remove a vcompress). At e32 and below,
our alternative is a vnsrl instead, and we need a bit more complexity
around lowering with fractional LMUL before the ri.vunzip2a/b versions
becomes always profitable. I'll post the followup change once this
lands.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  22 +-
 llvm/lib/Target/RISCV/RISCVISelLowering.h     |   4 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td |   6 +
 .../CodeGen/RISCV/rvv/vector-deinterleave.ll  | 312 ++++++++++--------
 4 files changed, 209 insertions(+), 135 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index ed723f38ea2c5..98fba9e86e88a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -5018,7 +5018,8 @@ static SDValue lowerVZIP(unsigned Opc, SDValue Op0, SDValue Op1,
                          const SDLoc &DL, SelectionDAG &DAG,
                          const RISCVSubtarget &Subtarget) {
   assert(RISCVISD::RI_VZIPEVEN_VL == Opc || RISCVISD::RI_VZIPODD_VL == Opc ||
-         RISCVISD::RI_VZIP2A_VL == Opc);
+         RISCVISD::RI_VZIP2A_VL == Opc || RISCVISD::RI_VUNZIP2A_VL == Opc ||
+         RISCVISD::RI_VUNZIP2B_VL == Opc);
   assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
 
   MVT VT = Op0.getSimpleValueType();
@@ -6934,7 +6935,7 @@ static bool hasPassthruOp(unsigned Opcode) {
          Opcode <= RISCVISD::LAST_STRICTFP_OPCODE &&
          "not a RISC-V target specific op");
   static_assert(
-      RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP == 130 &&
+      RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP == 132 &&
       RISCVISD::LAST_STRICTFP_OPCODE - RISCVISD::FIRST_STRICTFP_OPCODE == 21 &&
       "adding target specific op should update this function");
   if (Opcode >= RISCVISD::ADD_VL && Opcode <= RISCVISD::VFMAX_VL)
@@ -6958,7 +6959,7 @@ static bool hasMaskOp(unsigned Opcode) {
          Opcode <= RISCVISD::LAST_STRICTFP_OPCODE &&
          "not a RISC-V target specific op");
   static_assert(
-      RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP == 130 &&
+      RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP == 132 &&
       RISCVISD::LAST_STRICTFP_OPCODE - RISCVISD::FIRST_STRICTFP_OPCODE == 21 &&
       "adding target specific op should update this function");
   if (Opcode >= RISCVISD::TRUNCATE_VECTOR_VL && Opcode <= RISCVISD::SETCC_VL)
@@ -11509,6 +11510,19 @@ SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
     return DAG.getMergeValues(Res, DL);
   }
 
+  // TODO: Remove the e64 restriction once the fractional LMUL lowering
+  // is improved to always beat the vnsrl lowering below.
+  if (Subtarget.hasVendorXRivosVizip() && Factor == 2 &&
+      VecVT.getVectorElementType().getSizeInBits() == 64) {
+    SDValue V1 = Op->getOperand(0);
+    SDValue V2 = Op->getOperand(1);
+    SDValue Even =
+        lowerVZIP(RISCVISD::RI_VUNZIP2A_VL, V1, V2, DL, DAG, Subtarget);
+    SDValue Odd =
+        lowerVZIP(RISCVISD::RI_VUNZIP2B_VL, V1, V2, DL, DAG, Subtarget);
+    return DAG.getMergeValues({Even, Odd}, DL);
+  }
+
   SmallVector<SDValue, 8> Ops(Op->op_values());
 
   // Concatenate the vectors as one vector to deinterleave
@@ -22242,6 +22256,8 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(RI_VZIPEVEN_VL)
   NODE_NAME_CASE(RI_VZIPODD_VL)
   NODE_NAME_CASE(RI_VZIP2A_VL)
+  NODE_NAME_CASE(RI_VUNZIP2A_VL)
+  NODE_NAME_CASE(RI_VUNZIP2B_VL)
   NODE_NAME_CASE(READ_CSR)
   NODE_NAME_CASE(WRITE_CSR)
   NODE_NAME_CASE(SWAP_CSR)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index d624ee33d6a63..baf1b2e4d8e6e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -408,8 +408,10 @@ enum NodeType : unsigned {
   RI_VZIPEVEN_VL,
   RI_VZIPODD_VL,
   RI_VZIP2A_VL,
+  RI_VUNZIP2A_VL,
+  RI_VUNZIP2B_VL,
 
-  LAST_VL_VECTOR_OP = RI_VZIP2A_VL,
+  LAST_VL_VECTOR_OP = RI_VUNZIP2B_VL,
 
   // Read VLENB CSR
   READ_VLENB,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td
index 3fe50503f937b..147f89850765a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td
@@ -71,6 +71,8 @@ defm RI_VUNZIP2B_V : VALU_IV_V<"ri.vunzip2b", 0b011000>;
 def ri_vzipeven_vl : SDNode<"RISCVISD::RI_VZIPEVEN_VL", SDT_RISCVIntBinOp_VL>;
 def ri_vzipodd_vl : SDNode<"RISCVISD::RI_VZIPODD_VL", SDT_RISCVIntBinOp_VL>;
 def ri_vzip2a_vl : SDNode<"RISCVISD::RI_VZIP2A_VL", SDT_RISCVIntBinOp_VL>;
+def ri_vunzip2a_vl : SDNode<"RISCVISD::RI_VUNZIP2A_VL", SDT_RISCVIntBinOp_VL>;
+def ri_vunzip2b_vl : SDNode<"RISCVISD::RI_VUNZIP2B_VL", SDT_RISCVIntBinOp_VL>;
 
 multiclass RIVPseudoVALU_VV {
   foreach m = MxList in
@@ -82,6 +84,8 @@ let Predicates = [HasVendorXRivosVizip],
 defm PseudoRI_VZIPEVEN   : RIVPseudoVALU_VV;
 defm PseudoRI_VZIPODD   : RIVPseudoVALU_VV;
 defm PseudoRI_VZIP2A   : RIVPseudoVALU_VV;
+defm PseudoRI_VUNZIP2A   : RIVPseudoVALU_VV;
+defm PseudoRI_VUNZIP2B   : RIVPseudoVALU_VV;
 }
 
 multiclass RIVPatBinaryVL_VV<SDPatternOperator vop, string instruction_name,
@@ -98,6 +102,8 @@ multiclass RIVPatBinaryVL_VV<SDPatternOperator vop, string instruction_name,
 defm : RIVPatBinaryVL_VV<ri_vzipeven_vl, "PseudoRI_VZIPEVEN">;
 defm : RIVPatBinaryVL_VV<ri_vzipodd_vl, "PseudoRI_VZIPODD">;
 defm : RIVPatBinaryVL_VV<ri_vzip2a_vl, "PseudoRI_VZIP2A">;
+defm : RIVPatBinaryVL_VV<ri_vunzip2a_vl, "PseudoRI_VUNZIP2A">;
+defm : RIVPatBinaryVL_VV<ri_vunzip2b_vl, "PseudoRI_VUNZIP2B">;
 
 //===----------------------------------------------------------------------===//
 // XRivosVisni
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index 2787bef0c893f..b0b2390b1de37 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,V
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,V
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvfhmin,+zvfbfmin,+experimental-xrivosvizip | FileCheck %s --check-prefixes=CHECK,ZIP
 
 ; Integers
 
@@ -66,37 +67,55 @@ ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %retval
 }
 
 define {<vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv4i64(<vscale x 4 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv2i64_nxv4i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v16, a0
-; CHECK-NEXT:    li a0, 170
-; CHECK-NEXT:    vmv.v.x v20, a0
-; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vcompress.vm v12, v8, v16
-; CHECK-NEXT:    vcompress.vm v16, v8, v20
-; CHECK-NEXT:    vmv2r.v v8, v12
-; CHECK-NEXT:    vmv2r.v v10, v16
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_nxv2i64_nxv4i64:
+; V:       # %bb.0:
+; V-NEXT:    li a0, 85
+; V-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; V-NEXT:    vmv.v.x v16, a0
+; V-NEXT:    li a0, 170
+; V-NEXT:    vmv.v.x v20, a0
+; V-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; V-NEXT:    vcompress.vm v12, v8, v16
+; V-NEXT:    vcompress.vm v16, v8, v20
+; V-NEXT:    vmv2r.v v8, v12
+; V-NEXT:    vmv2r.v v10, v16
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_nxv2i64_nxv4i64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v12, v8, v10
+; ZIP-NEXT:    ri.vunzip2b.vv v14, v8, v10
+; ZIP-NEXT:    vmv.v.v v8, v12
+; ZIP-NEXT:    vmv.v.v v10, v14
+; ZIP-NEXT:    ret
 %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> %vec)
 ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %retval
 }
 
 define {<vscale x 4 x i64>, <vscale x 4 x i64>} @vector_deinterleave_nxv4i64_nxv8i64(<vscale x 8 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv4i64_nxv8i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.x v24, a0
-; CHECK-NEXT:    li a0, 170
-; CHECK-NEXT:    vmv.v.x v7, a0
-; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vcompress.vm v16, v8, v24
-; CHECK-NEXT:    vcompress.vm v24, v8, v7
-; CHECK-NEXT:    vmv4r.v v8, v16
-; CHECK-NEXT:    vmv4r.v v12, v24
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_nxv4i64_nxv8i64:
+; V:       # %bb.0:
+; V-NEXT:    li a0, 85
+; V-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.x v24, a0
+; V-NEXT:    li a0, 170
+; V-NEXT:    vmv.v.x v7, a0
+; V-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; V-NEXT:    vcompress.vm v16, v8, v24
+; V-NEXT:    vcompress.vm v24, v8, v7
+; V-NEXT:    vmv4r.v v8, v16
+; V-NEXT:    vmv4r.v v12, v24
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_nxv4i64_nxv8i64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v16, v8, v12
+; ZIP-NEXT:    ri.vunzip2b.vv v20, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v16
+; ZIP-NEXT:    vmv.v.v v12, v20
+; ZIP-NEXT:    ret
 %retval = call {<vscale x 4 x i64>, <vscale x 4 x i64>} @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> %vec)
 ret {<vscale x 4 x i64>, <vscale x 4 x i64>} %retval
 }
@@ -171,51 +190,62 @@ ret {<vscale x 16 x i32>, <vscale x 16 x i32>} %retval
 }
 
 define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nxv16i64(<vscale x 16 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv8i64_nxv16i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    sub sp, sp, a0
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.x v7, a0
-; CHECK-NEXT:    li a0, 170
-; CHECK-NEXT:    vmv.v.x v6, a0
-; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vcompress.vm v24, v8, v7
-; CHECK-NEXT:    vmv1r.v v28, v7
-; CHECK-NEXT:    vmv1r.v v29, v6
-; CHECK-NEXT:    vcompress.vm v0, v8, v29
-; CHECK-NEXT:    vcompress.vm v8, v16, v28
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    vcompress.vm v8, v16, v29
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vmv4r.v v28, v8
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vmv4r.v v4, v8
-; CHECK-NEXT:    vmv8r.v v8, v24
-; CHECK-NEXT:    vmv8r.v v16, v0
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    add sp, sp, a0
-; CHECK-NEXT:    .cfi_def_cfa sp, 16
-; CHECK-NEXT:    addi sp, sp, 16
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_nxv8i64_nxv16i64:
+; V:       # %bb.0:
+; V-NEXT:    addi sp, sp, -16
+; V-NEXT:    .cfi_def_cfa_offset 16
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    slli a0, a0, 4
+; V-NEXT:    sub sp, sp, a0
+; V-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; V-NEXT:    li a0, 85
+; V-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.x v7, a0
+; V-NEXT:    li a0, 170
+; V-NEXT:    vmv.v.x v6, a0
+; V-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; V-NEXT:    vcompress.vm v24, v8, v7
+; V-NEXT:    vmv1r.v v28, v7
+; V-NEXT:    vmv1r.v v29, v6
+; V-NEXT:    vcompress.vm v0, v8, v29
+; V-NEXT:    vcompress.vm v8, v16, v28
+; V-NEXT:    addi a0, sp, 16
+; V-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; V-NEXT:    vcompress.vm v8, v16, v29
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    slli a0, a0, 3
+; V-NEXT:    add a0, sp, a0
+; V-NEXT:    addi a0, a0, 16
+; V-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; V-NEXT:    addi a0, sp, 16
+; V-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; V-NEXT:    vmv4r.v v28, v8
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    slli a0, a0, 3
+; V-NEXT:    add a0, sp, a0
+; V-NEXT:    addi a0, a0, 16
+; V-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; V-NEXT:    vmv4r.v v4, v8
+; V-NEXT:    vmv8r.v v8, v24
+; V-NEXT:    vmv8r.v v16, v0
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    slli a0, a0, 4
+; V-NEXT:    add sp, sp, a0
+; V-NEXT:    .cfi_def_cfa sp, 16
+; V-NEXT:    addi sp, sp, 16
+; V-NEXT:    .cfi_def_cfa_offset 0
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_nxv8i64_nxv16i64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v28, v16, v20
+; ZIP-NEXT:    ri.vunzip2b.vv v4, v16, v20
+; ZIP-NEXT:    ri.vunzip2a.vv v24, v8, v12
+; ZIP-NEXT:    ri.vunzip2b.vv v0, v8, v12
+; ZIP-NEXT:    vmv8r.v v8, v24
+; ZIP-NEXT:    vmv8r.v v16, v0
+; ZIP-NEXT:    ret
 %retval = call {<vscale x 8 x i64>, <vscale x 8 x i64>} @llvm.vector.deinterleave2.nxv16i64(<vscale x 16 x i64> %vec)
 ret {<vscale x 8 x i64>, <vscale x 8 x i64>} %retval
 }
@@ -328,19 +358,28 @@ ret  {<vscale x 4 x float>, <vscale x 4 x float>} %retval
 }
 
 define {<vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f64_nxv4f64(<vscale x 4 x double> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv2f64_nxv4f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v16, a0
-; CHECK-NEXT:    li a0, 170
-; CHECK-NEXT:    vmv.v.x v20, a0
-; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vcompress.vm v12, v8, v16
-; CHECK-NEXT:    vcompress.vm v16, v8, v20
-; CHECK-NEXT:    vmv2r.v v8, v12
-; CHECK-NEXT:    vmv2r.v v10, v16
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_nxv2f64_nxv4f64:
+; V:       # %bb.0:
+; V-NEXT:    li a0, 85
+; V-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; V-NEXT:    vmv.v.x v16, a0
+; V-NEXT:    li a0, 170
+; V-NEXT:    vmv.v.x v20, a0
+; V-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; V-NEXT:    vcompress.vm v12, v8, v16
+; V-NEXT:    vcompress.vm v16, v8, v20
+; V-NEXT:    vmv2r.v v8, v12
+; V-NEXT:    vmv2r.v v10, v16
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_nxv2f64_nxv4f64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v12, v8, v10
+; ZIP-NEXT:    ri.vunzip2b.vv v14, v8, v10
+; ZIP-NEXT:    vmv.v.v v8, v12
+; ZIP-NEXT:    vmv.v.v v10, v14
+; ZIP-NEXT:    ret
 %retval = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %vec)
 ret {<vscale x 2 x double>, <vscale x 2 x double>} %retval
 }
@@ -393,51 +432,62 @@ ret  {<vscale x 16 x float>, <vscale x 16 x float>} %retval
 }
 
 define {<vscale x 8 x double>, <vscale x 8 x double>} @vector_deinterleave_nxv8f64_nxv16f64(<vscale x 16 x double> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv8f64_nxv16f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    sub sp, sp, a0
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.x v7, a0
-; CHECK-NEXT:    li a0, 170
-; CHECK-NEXT:    vmv.v.x v6, a0
-; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vcompress.vm v24, v8, v7
-; CHECK-NEXT:    vmv1r.v v28, v7
-; CHECK-NEXT:    vmv1r.v v29, v6
-; CHECK-NEXT:    vcompress.vm v0, v8, v29
-; CHECK-NEXT:    vcompress.vm v8, v16, v28
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    vcompress.vm v8, v16, v29
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vmv4r.v v28, v8
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vmv4r.v v4, v8
-; CHECK-NEXT:    vmv8r.v v8, v24
-; CHECK-NEXT:    vmv8r.v v16, v0
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    add sp, sp, a0
-; CHECK-NEXT:    .cfi_def_cfa sp, 16
-; CHECK-NEXT:    addi sp, sp, 16
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_nxv8f64_nxv16f64:
+; V:       # %bb.0:
+; V-NEXT:    addi sp, sp, -16
+; V-NEXT:    .cfi_def_cfa_offset 16
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    slli a0, a0, 4
+; V-NEXT:    sub sp, sp, a0
+; V-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; V-NEXT:    li a0, 85
+; V-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; V-NEXT:    vmv.v.x v7, a0
+; V-NEXT:    li a0, 170
+; V-NEXT:    vmv.v.x v6, a0
+; V-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; V-NEXT:    vcompress.vm v24, v8, v7
+; V-NEXT:    vmv1r.v v28, v7
+; V-NEXT:    vmv1r.v v29, v6
+; V-NEXT:    vcompress.vm v0, v8, v29
+; V-NEXT:    vcompress.vm v8, v16, v28
+; V-NEXT:    addi a0, sp, 16
+; V-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; V-NEXT:    vcompress.vm v8, v16, v29
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    slli a0, a0, 3
+; V-NEXT:    add a0, sp, a0
+; V-NEXT:    addi a0, a0, 16
+; V-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; V-NEXT:    addi a0, sp, 16
+; V-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; V-NEXT:    vmv4r.v v28, v8
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    slli a0, a0, 3
+; V-NEXT:    add a0, sp, a0
+; V-NEXT:    addi a0, a0, 16
+; V-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; V-NEXT:    vmv4r.v v4, v8
+; V-NEXT:    vmv8r.v v8, v24
+; V-NEXT:    vmv8r.v v16, v0
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    slli a0, a0, 4
+; V-NEXT:    add sp, sp, a0
+; V-NEXT:    .cfi_def_cfa sp, 16
+; V-NEXT:    addi sp, sp, 16
+; V-NEXT:    .cfi_def_cfa_offset 0
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_nxv8f64_nxv16f64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v28, v16, v20
+; ZIP-NEXT:    ri.vunzip2b.vv v4, v16, v20
+; ZIP-NEXT:    ri.vunzip2a.vv v24, v8, v12
+; ZIP-NEXT:    ri.vunzip2b.vv v0, v8, v12
+; ZIP-NEXT:    vmv8r.v v8, v24
+; ZIP-NEXT:    vmv8r.v v16, v0
+; ZIP-NEXT:    ret
 %retval = call {<vscale x 8 x double>, <vscale x 8 x double>} @llvm.vector.deinterleave2.nxv16f64(<vscale x 16 x double> %vec)
 ret {<vscale x 8 x double>, <vscale x 8 x double>} %retval
 }

From ba273be3bd1986f891fba38016bd217b9734b6f8 Mon Sep 17 00:00:00 2001
From: HighW4y2H3ll <zhenghaohuu@gmail.com>
Date: Fri, 18 Apr 2025 10:42:49 -0700
Subject: [PATCH 446/710] [Bazel] Remove redundant gtest_main from
 codegen_tests, codegen_globalisel_tests, mi_tests (#135316)

For codegen_tests, codegen_globalisel_tests and mi_tests, they already
have their own `main` function defined, so there should be no need to
add `gtest_main` dependency for the main function...

- codegen_tests:
https://github.com/llvm/llvm-project/blob/main/llvm/unittests/CodeGen/TargetOptionsTest.cpp#L73
- codegen_globalisel_tests:
https://github.com/llvm/llvm-project/blob/main/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp#L978
- mi_tests:
https://github.com/llvm/llvm-project/blob/main/llvm/unittests/MI/LiveIntervalTest.cpp#L933
---
 utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
index 3b778fc90fe1a..97528392ec07f 100644
--- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
@@ -174,7 +174,6 @@ cc_test(
         "//llvm:Target",
         "//llvm:TestingSupport",
         "//third-party/unittest:gtest",
-        "//third-party/unittest:gtest_main",
     ],
 )
 
@@ -203,7 +202,6 @@ cc_test(
         "//llvm:Support",
         "//llvm:Target",
         "//third-party/unittest:gtest",
-        "//third-party/unittest:gtest_main",
     ],
 )
 
@@ -501,7 +499,6 @@ cc_test(
         "//llvm:Support",
         "//llvm:Target",
         "//third-party/unittest:gtest",
-        "//third-party/unittest:gtest_main",
     ],
 )
 

From d79bb932785d312faa8cb8e82a0afc103436c54c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 18 Apr 2025 10:44:18 -0700
Subject: [PATCH 447/710] [flang][cuda] Carry over the stream information to
 kernel launch (#136217)

In CUDA Fortran the stream is encoded in an INTEGER(cuda_stream_kind)
variable.

This information is carried over the GPU dialect through the
`cuf.stream_cast` and the token in the GPU ops.

When converting the `gpu.launch_func` to runtime call, the
`cuf.stream_cast` becomes a no-op and the reference to the stream is
passed to the runtime.

The runtime is adapted to take integer references instead of value for
stream.
---
 flang-rt/lib/cuda/kernel.cpp                  | 17 +++--
 .../flang/Optimizer/Dialect/CUF/CUFOps.td     |  2 +-
 .../Transforms/CUFGPUToLLVMConversion.h       |  6 +-
 flang/include/flang/Runtime/CUDA/kernel.h     |  6 +-
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    |  6 +-
 .../Transforms/CUFGPUToLLVMConversion.cpp     | 55 ++++++++++++----
 flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir | 65 ++++++++++++++++++-
 flang/test/Fir/CUDA/cuda-launch.fir           |  2 +-
 flang/test/Fir/CUDA/cuda-stream.mlir          |  2 +-
 9 files changed, 125 insertions(+), 36 deletions(-)

diff --git a/flang-rt/lib/cuda/kernel.cpp b/flang-rt/lib/cuda/kernel.cpp
index 73b4e24bf701f..e299a114ed7eb 100644
--- a/flang-rt/lib/cuda/kernel.cpp
+++ b/flang-rt/lib/cuda/kernel.cpp
@@ -17,7 +17,7 @@ extern "C" {
 
 void RTDEF(CUFLaunchKernel)(const void *kernel, intptr_t gridX, intptr_t gridY,
     intptr_t gridZ, intptr_t blockX, intptr_t blockY, intptr_t blockZ,
-    intptr_t stream, int32_t smem, void **params, void **extra) {
+    int64_t *stream, int32_t smem, void **params, void **extra) {
   dim3 gridDim;
   gridDim.x = gridX;
   gridDim.y = gridY;
@@ -77,13 +77,13 @@ void RTDEF(CUFLaunchKernel)(const void *kernel, intptr_t gridX, intptr_t gridY,
   }
   cudaStream_t defaultStream = 0;
   CUDA_REPORT_IF_ERROR(cudaLaunchKernel(kernel, gridDim, blockDim, params, smem,
-      stream != kNoAsyncId ? (cudaStream_t)stream : defaultStream));
+      stream != nullptr ? (cudaStream_t)(*stream) : defaultStream));
 }
 
 void RTDEF(CUFLaunchClusterKernel)(const void *kernel, intptr_t clusterX,
     intptr_t clusterY, intptr_t clusterZ, intptr_t gridX, intptr_t gridY,
     intptr_t gridZ, intptr_t blockX, intptr_t blockY, intptr_t blockZ,
-    intptr_t stream, int32_t smem, void **params, void **extra) {
+    int64_t *stream, int32_t smem, void **params, void **extra) {
   cudaLaunchConfig_t config;
   config.gridDim.x = gridX;
   config.gridDim.y = gridY;
@@ -141,8 +141,8 @@ void RTDEF(CUFLaunchClusterKernel)(const void *kernel, intptr_t clusterX,
     terminator.Crash("Too many invalid grid dimensions");
   }
   config.dynamicSmemBytes = smem;
-  if (stream != kNoAsyncId) {
-    config.stream = (cudaStream_t)stream;
+  if (stream != nullptr) {
+    config.stream = (cudaStream_t)(*stream);
   } else {
     config.stream = 0;
   }
@@ -158,7 +158,7 @@ void RTDEF(CUFLaunchClusterKernel)(const void *kernel, intptr_t clusterX,
 
 void RTDEF(CUFLaunchCooperativeKernel)(const void *kernel, intptr_t gridX,
     intptr_t gridY, intptr_t gridZ, intptr_t blockX, intptr_t blockY,
-    intptr_t blockZ, intptr_t stream, int32_t smem, void **params,
+    intptr_t blockZ, int64_t *stream, int32_t smem, void **params,
     void **extra) {
   dim3 gridDim;
   gridDim.x = gridX;
@@ -218,9 +218,8 @@ void RTDEF(CUFLaunchCooperativeKernel)(const void *kernel, intptr_t gridX,
     terminator.Crash("Too many invalid grid dimensions");
   }
   cudaStream_t defaultStream = 0;
-  CUDA_REPORT_IF_ERROR(
-      cudaLaunchCooperativeKernel(kernel, gridDim, blockDim, params, smem,
-          stream != kNoAsyncId ? (cudaStream_t)stream : defaultStream));
+  CUDA_REPORT_IF_ERROR(cudaLaunchCooperativeKernel(kernel, gridDim, blockDim,
+      params, smem, stream != nullptr ? (cudaStream_t)*stream : defaultStream));
 }
 
 } // extern "C"
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
index ccf9969e73a8e..926983d364ed1 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
@@ -383,7 +383,7 @@ def cuf_StreamCastOp : cuf_Op<"stream_cast", [NoMemoryEffect]> {
     Later in the lowering this will become a no op.
   }];
 
-  let arguments = (ins fir_ReferenceType:$stream);
+  let arguments = (ins AnyTypeOf<[fir_ReferenceType, LLVM_AnyPointer]>:$stream);
 
   let results = (outs GPU_AsyncToken:$token);
 
diff --git a/flang/include/flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h b/flang/include/flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h
index 7d76c1f4e5218..f40f0049e9085 100644
--- a/flang/include/flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h
+++ b/flang/include/flang/Optimizer/Transforms/CUFGPUToLLVMConversion.h
@@ -19,9 +19,9 @@ class LLVMTypeConverter;
 
 namespace cuf {
 
-void populateCUFGPUToLLVMConversionPatterns(
-    const fir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
-    mlir::PatternBenefit benefit = 1);
+void populateCUFGPUToLLVMConversionPatterns(fir::LLVMTypeConverter &converter,
+                                            mlir::RewritePatternSet &patterns,
+                                            mlir::PatternBenefit benefit = 1);
 
 } // namespace cuf
 
diff --git a/flang/include/flang/Runtime/CUDA/kernel.h b/flang/include/flang/Runtime/CUDA/kernel.h
index eb9135868fdee..70eb74bb79554 100644
--- a/flang/include/flang/Runtime/CUDA/kernel.h
+++ b/flang/include/flang/Runtime/CUDA/kernel.h
@@ -21,17 +21,17 @@ extern "C" {
 
 void RTDEF(CUFLaunchKernel)(const void *kernelName, intptr_t gridX,
     intptr_t gridY, intptr_t gridZ, intptr_t blockX, intptr_t blockY,
-    intptr_t blockZ, intptr_t stream, int32_t smem, void **params,
+    intptr_t blockZ, int64_t *stream, int32_t smem, void **params,
     void **extra);
 
 void RTDEF(CUFLaunchClusterKernel)(const void *kernelName, intptr_t clusterX,
     intptr_t clusterY, intptr_t clusterZ, intptr_t gridX, intptr_t gridY,
     intptr_t gridZ, intptr_t blockX, intptr_t blockY, intptr_t blockZ,
-    intptr_t stream, int32_t smem, void **params, void **extra);
+    int64_t *stream, int32_t smem, void **params, void **extra);
 
 void RTDEF(CUFLaunchCooperativeKernel)(const void *kernelName, intptr_t gridX,
     intptr_t gridY, intptr_t gridZ, intptr_t blockX, intptr_t blockY,
-    intptr_t blockZ, intptr_t stream, int32_t smem, void **params,
+    intptr_t blockZ, int64_t *stream, int32_t smem, void **params,
     void **extra);
 
 } // extern "C"
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index 2c6d22f6f6c7d..a86f12c2c4a55 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -147,9 +147,9 @@ template <typename OpTy>
 static llvm::LogicalResult checkStreamType(OpTy op) {
   if (!op.getStream())
     return mlir::success();
-  auto refTy = mlir::dyn_cast<fir::ReferenceType>(op.getStream().getType());
-  if (!refTy.getEleTy().isInteger(64))
-    return op.emitOpError("stream is expected to be a i64 reference");
+  if (auto refTy = mlir::dyn_cast<fir::ReferenceType>(op.getStream().getType()))
+    if (!refTy.getEleTy().isInteger(64))
+      return op.emitOpError("stream is expected to be an i64 reference");
   return mlir::success();
 }
 
diff --git a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
index 205acbfea22b8..02b4e6a5a469c 100644
--- a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
@@ -121,7 +121,7 @@ struct GPULaunchKernelConversion
           voidTy,
           {ptrTy, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
            llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
-           llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, i32Ty, ptrTy, ptrTy},
+           llvmIntPtrType, llvmIntPtrType, ptrTy, i32Ty, ptrTy, ptrTy},
           /*isVarArg=*/false);
       auto cufLaunchClusterKernel = mlir::SymbolRefAttr::get(
           mod.getContext(), RTNAME_STRING(CUFLaunchClusterKernel));
@@ -133,10 +133,15 @@ struct GPULaunchKernelConversion
         launchKernelFuncOp.setVisibility(
             mlir::SymbolTable::Visibility::Private);
       }
-      mlir::Value stream = adaptor.getAsyncObject();
-      if (!stream)
-        stream = rewriter.create<mlir::LLVM::ConstantOp>(
-            loc, llvmIntPtrType, rewriter.getIntegerAttr(llvmIntPtrType, -1));
+
+      mlir::Value stream = nullPtr;
+      if (!adaptor.getAsyncDependencies().empty()) {
+        if (adaptor.getAsyncDependencies().size() != 1)
+          return rewriter.notifyMatchFailure(
+              op, "Can only convert with exactly one stream dependency.");
+        stream = adaptor.getAsyncDependencies().front();
+      }
+
       rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
           op, funcTy, cufLaunchClusterKernel,
           mlir::ValueRange{kernelPtr, adaptor.getClusterSizeX(),
@@ -157,8 +162,8 @@ struct GPULaunchKernelConversion
       auto funcTy = mlir::LLVM::LLVMFunctionType::get(
           voidTy,
           {ptrTy, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
-           llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
-           i32Ty, ptrTy, ptrTy},
+           llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, ptrTy, i32Ty, ptrTy,
+           ptrTy},
           /*isVarArg=*/false);
       auto cufLaunchKernel =
           mlir::SymbolRefAttr::get(mod.getContext(), fctName);
@@ -171,10 +176,13 @@ struct GPULaunchKernelConversion
             mlir::SymbolTable::Visibility::Private);
       }
 
-      mlir::Value stream = adaptor.getAsyncObject();
-      if (!stream)
-        stream = rewriter.create<mlir::LLVM::ConstantOp>(
-            loc, llvmIntPtrType, rewriter.getIntegerAttr(llvmIntPtrType, -1));
+      mlir::Value stream = nullPtr;
+      if (!adaptor.getAsyncDependencies().empty()) {
+        if (adaptor.getAsyncDependencies().size() != 1)
+          return rewriter.notifyMatchFailure(
+              op, "Can only convert with exactly one stream dependency.");
+        stream = adaptor.getAsyncDependencies().front();
+      }
 
       rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
           op, funcTy, cufLaunchKernel,
@@ -251,6 +259,22 @@ struct CUFSharedMemoryOpConversion
   }
 };
 
+struct CUFStreamCastConversion
+    : public mlir::ConvertOpToLLVMPattern<cuf::StreamCastOp> {
+  explicit CUFStreamCastConversion(const fir::LLVMTypeConverter &typeConverter,
+                                   mlir::PatternBenefit benefit)
+      : mlir::ConvertOpToLLVMPattern<cuf::StreamCastOp>(typeConverter,
+                                                        benefit) {}
+  using OpAdaptor = typename cuf::StreamCastOp::Adaptor;
+
+  mlir::LogicalResult
+  matchAndRewrite(cuf::StreamCastOp op, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOp(op, adaptor.getStream());
+    return mlir::success();
+  }
+};
+
 class CUFGPUToLLVMConversion
     : public fir::impl::CUFGPUToLLVMConversionBase<CUFGPUToLLVMConversion> {
 public:
@@ -283,8 +307,11 @@ class CUFGPUToLLVMConversion
 } // namespace
 
 void cuf::populateCUFGPUToLLVMConversionPatterns(
-    const fir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
+    fir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
     mlir::PatternBenefit benefit) {
-  patterns.add<CUFSharedMemoryOpConversion, GPULaunchKernelConversion>(
-      converter, benefit);
+  converter.addConversion([&converter](mlir::gpu::AsyncTokenType) -> Type {
+    return mlir::LLVM::LLVMPointerType::get(&converter.getContext());
+  });
+  patterns.add<CUFSharedMemoryOpConversion, GPULaunchKernelConversion,
+               CUFStreamCastConversion>(converter, benefit);
 }
diff --git a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
index 85266f17bb67a..0319213016e45 100644
--- a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
+++ b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
@@ -113,7 +113,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : ve
 // -----
 
 module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (git@github.com:clementval/llvm-project.git 4116c1370ff76adf1e58eb3c39d0a14721794c70)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
-  llvm.func @_FortranACUFLaunchClusterKernel(!llvm.ptr, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i32, !llvm.ptr, !llvm.ptr) attributes {sym_visibility = "private"}
+  llvm.func @_FortranACUFLaunchClusterKernel(!llvm.ptr, i64, i64, i64, i64, i64, i64, i64, i64, i64, !llvm.ptr, i32, !llvm.ptr, !llvm.ptr) attributes {sym_visibility = "private"}
   llvm.func @_QMmod1Psub1() attributes {cuf.cluster_dims = #cuf.cluster_dims<x = 2 : i64, y = 2 : i64, z = 1 : i64>} {
     llvm.return
   }
@@ -166,3 +166,66 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : ve
 
 // CHECK-LABEL: llvm.func @_QMmod1Phost_sub()
 // CHECK: llvm.call @_FortranACUFLaunchCooperativeKernel
+
+// -----
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (git@github.com:clementval/llvm-project.git 4116c1370ff76adf1e58eb3c39d0a14721794c70)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  llvm.func @_QMmod1Psub1() attributes {cuf.cluster_dims = #cuf.cluster_dims<x = 2 : i64, y = 2 : i64, z = 1 : i64>} {
+    llvm.return
+  }
+  llvm.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %0 = llvm.mlir.constant(1 : index) : i64
+    %stream = llvm.alloca %0 x i64 : (i64) -> !llvm.ptr
+    %1 = llvm.mlir.constant(2 : index) : i64
+    %2 = llvm.mlir.constant(0 : i32) : i32
+    %3 = llvm.mlir.constant(10 : index) : i64
+    %token = cuf.stream_cast %stream : !llvm.ptr
+    gpu.launch_func [%token] @cuda_device_mod::@_QMmod1Psub1 blocks in (%3, %3, %0) threads in (%3, %3, %0) : i64 dynamic_shared_memory_size %2
+    llvm.return
+  }
+  gpu.binary @cuda_device_mod  [#gpu.object<#nvvm.target, "">]
+}
+
+// CHECK-LABEL: llvm.func @_QQmain()
+// CHECK: %[[STREAM:.*]] = llvm.alloca %{{.*}} x i64 : (i64) -> !llvm.ptr
+// CHECK: %[[KERNEL_PTR:.*]] = llvm.mlir.addressof @_QMmod1Psub1
+// CHECK: llvm.call @_FortranACUFLaunchKernel(%[[KERNEL_PTR]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %[[STREAM]], %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i64, i64, i64, i64, i64, i64, !llvm.ptr, i32, !llvm.ptr, !llvm.ptr) -> ()
+
+// -----
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (git@github.com:clementval/llvm-project.git ddcfd4d2dc17bf66cee8c3ef6284118684a2b0e6)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  llvm.func @_QMmod1Phost_sub() {
+    %0 = llvm.mlir.constant(1 : i32) : i32
+    %one = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
+    %stream = llvm.alloca %one x i64 : (i64) -> !llvm.ptr
+    %2 = llvm.mlir.constant(40 : i64) : i64
+    %3 = llvm.mlir.constant(16 : i32) : i32
+    %4 = llvm.mlir.constant(25 : i32) : i32
+    %5 = llvm.mlir.constant(21 : i32) : i32
+    %6 = llvm.mlir.constant(17 : i32) : i32
+    %7 = llvm.mlir.constant(1 : index) : i64
+    %8 = llvm.mlir.constant(27 : i32) : i32
+    %9 = llvm.mlir.constant(6 : i32) : i32
+    %10 = llvm.mlir.constant(1 : i32) : i32
+    %11 = llvm.mlir.constant(0 : i32) : i32
+    %12 = llvm.mlir.constant(10 : index) : i64
+    %13 = llvm.mlir.addressof @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5 : !llvm.ptr
+    %14 = llvm.call @_FortranACUFMemAlloc(%2, %11, %13, %6) : (i64, i32, !llvm.ptr, i32) -> !llvm.ptr
+    %token = cuf.stream_cast %stream : !llvm.ptr
+    gpu.launch_func [%token] @cuda_device_mod::@_QMmod1Psub1 blocks in (%7, %7, %7) threads in (%12, %7, %7) : i64 dynamic_shared_memory_size %11 args(%14 : !llvm.ptr) {cuf.proc_attr = #cuf.cuda_proc<grid_global>}
+    llvm.return
+  }
+  llvm.func @_QMmod1Psub1(!llvm.ptr) -> ()
+  llvm.mlir.global linkonce constant @_QQclX91d13f6e74caa2f03965d7a7c6a8fdd5()  {addr_space = 0 : i32} : !llvm.array<2 x i8> {
+    %0 = llvm.mlir.constant("a\00") : !llvm.array<2 x i8>
+    llvm.return %0 : !llvm.array<2 x i8>
+  }
+  llvm.func @_FortranACUFMemAlloc(i64, i32, !llvm.ptr, i32) -> !llvm.ptr attributes {fir.runtime, sym_visibility = "private"}
+  llvm.func @_FortranACUFMemFree(!llvm.ptr, i32, !llvm.ptr, i32) -> !llvm.struct<()> attributes {fir.runtime, sym_visibility = "private"}
+  gpu.binary @cuda_device_mod  [#gpu.object<#nvvm.target, "">]
+}
+
+// CHECK-LABEL: llvm.func @_QMmod1Phost_sub()
+// CHECK: %[[STREAM:.*]] = llvm.alloca %{{.*}} x i64 : (i64) -> !llvm.ptr
+// CHECK: llvm.call @_FortranACUFLaunchCooperativeKernel(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %[[STREAM]], %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i64, i64, i64, i64, i64, i64, !llvm.ptr, i32, !llvm.ptr, !llvm.ptr) -> ()
diff --git a/flang/test/Fir/CUDA/cuda-launch.fir b/flang/test/Fir/CUDA/cuda-launch.fir
index 319991546d3fe..028279832c703 100644
--- a/flang/test/Fir/CUDA/cuda-launch.fir
+++ b/flang/test/Fir/CUDA/cuda-launch.fir
@@ -154,5 +154,5 @@ module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_e
 // CHECK-LABEL: func.func @_QQmain()
 // CHECK: %[[STREAM:.*]] = fir.alloca i64 {bindc_name = "stream", uniq_name = "_QMtest_callFhostEstream"}
 // CHECK: %[[DECL_STREAM:.*]]:2 = hlfir.declare %[[STREAM]] {uniq_name = "_QMtest_callFhostEstream"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-// CHECK: %[[TOKEN:.*]] = cuf.stream_cast %[[DECL_STREAM]]#0 : <i64>
+// CHECK: %[[TOKEN:.*]] = cuf.stream_cast %[[DECL_STREAM]]#0 : !fir.ref<i64>
 // CHECK: gpu.launch_func [%[[TOKEN]]] @cuda_device_mod::@_QMdevptrPtest
diff --git a/flang/test/Fir/CUDA/cuda-stream.mlir b/flang/test/Fir/CUDA/cuda-stream.mlir
index 50f230467854b..a501603fd35d1 100644
--- a/flang/test/Fir/CUDA/cuda-stream.mlir
+++ b/flang/test/Fir/CUDA/cuda-stream.mlir
@@ -17,5 +17,5 @@ module attributes {gpu.container_module} {
 
 // CHECK-LABEL: func.func @_QMmod1Phost_sub()
 // CHECK: %[[STREAM:.*]] = fir.alloca i64
-// CHECK: %[[TOKEN:.*]] = cuf.stream_cast %[[STREAM]] : <i64>
+// CHECK: %[[TOKEN:.*]] = cuf.stream_cast %[[STREAM]] : !fir.ref<i64>
 // CHECK: gpu.launch_func [%[[TOKEN]]] @cuda_device_mod::@_QMmod1Psub1

From 733c250bd5b39db2fa0cd0e201730661da56cc87 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 18 Apr 2025 10:43:54 -0700
Subject: [PATCH 448/710] [TableGen] Fix typo in comment. NFC

---
 llvm/utils/TableGen/AsmMatcherEmitter.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index c954163cdeb3a..a3a739e6636a5 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -3934,7 +3934,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
           "convert to\n";
     OS << "    // an MCInst, so bail out on this instruction variant now.\n";
     OS << "    if (OperandNearMiss) {\n";
-    OS << "      // If the operand mismatch was the only problem, reprrt it as "
+    OS << "      // If the operand mismatch was the only problem, report it as "
           "a near-miss.\n";
     OS << "      if (NearMisses && !FeaturesNearMiss && "
           "!EarlyPredicateNearMiss) {\n";

From 9ef91677a4a2d4b017d2b96b8c89c2a9b648d6c3 Mon Sep 17 00:00:00 2001
From: Qiongsi Wu <qiongsiwu@gmail.com>
Date: Fri, 18 Apr 2025 10:52:39 -0700
Subject: [PATCH 449/710] [clang][Dependency Scanning] Adding an API to
 Diagnose Invalid Negative Stat Cache Entries (#135703)

We have had numerous situations where the negatively stat cached paths
are invalidated during the build, and such invalidations lead to build
errors.

This PR adds an API to diagnose such cases.
`DependencyScanningFilesystemSharedCache::diagnoseNegativeStatCachedPaths`
allows users of the cache to ask the cache to examine all negatively
stat cached paths, and re-stat the paths using the passed-in file
system. If any re-stat succeeds, the API emits diagnostics.

rdar://149147920
---
 .../DependencyScanningFilesystem.h            |  8 ++++++
 .../DependencyScanningFilesystem.cpp          | 27 +++++++++++++++++++
 .../DependencyScanningFilesystemTest.cpp      | 24 +++++++++++++++++
 3 files changed, 59 insertions(+)

diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index d12814e7c9253..74b40f7452edb 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -220,6 +220,14 @@ class DependencyScanningFilesystemSharedCache {
   CacheShard &getShardForFilename(StringRef Filename) const;
   CacheShard &getShardForUID(llvm::sys::fs::UniqueID UID) const;
 
+  /// Visits all cached entries and re-stat an entry using FS if
+  /// it is negatively stat cached. If re-stat succeeds on a path,
+  /// the path is added to InvalidPaths, indicating that the cache
+  /// may have erroneously negatively cached it. The caller can then
+  /// use InvalidPaths to issue diagnostics.
+  std::vector<StringRef>
+  getInvalidNegativeStatCachedPaths(llvm::vfs::FileSystem &UnderlyingFS) const;
+
 private:
   std::unique_ptr<CacheShard[]> CacheShards;
   unsigned NumShards;
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 4d738e4bea41a..191b6ef439860 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -108,6 +108,33 @@ DependencyScanningFilesystemSharedCache::getShardForUID(
   return CacheShards[Hash % NumShards];
 }
 
+std::vector<StringRef>
+DependencyScanningFilesystemSharedCache::getInvalidNegativeStatCachedPaths(
+    llvm::vfs::FileSystem &UnderlyingFS) const {
+  // Iterate through all shards and look for cached stat errors.
+  std::vector<StringRef> InvalidPaths;
+  for (unsigned i = 0; i < NumShards; i++) {
+    const CacheShard &Shard = CacheShards[i];
+    std::lock_guard<std::mutex> LockGuard(Shard.CacheLock);
+    for (const auto &[Path, CachedPair] : Shard.CacheByFilename) {
+      const CachedFileSystemEntry *Entry = CachedPair.first;
+
+      if (Entry->getError()) {
+        // Only examine cached errors.
+        llvm::ErrorOr<llvm::vfs::Status> Stat = UnderlyingFS.status(Path);
+        if (Stat) {
+          // This is the case where we have cached the non-existence
+          // of the file at Path first, and a a file at the path is created
+          // later. The cache entry is not invalidated (as we have no good
+          // way to do it now), which may lead to missing file build errors.
+          InvalidPaths.push_back(Path);
+        }
+      }
+    }
+  }
+  return InvalidPaths;
+}
+
 const CachedFileSystemEntry *
 DependencyScanningFilesystemSharedCache::CacheShard::findEntryByFilename(
     StringRef Filename) const {
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index 111351fb90cee..5ea8d02353dc3 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -178,3 +178,27 @@ TEST(DependencyScanningFilesystem, CacheStatFailures) {
   DepFS.exists("/cache/a.pcm");
   EXPECT_EQ(InstrumentingFS->NumStatusCalls, 5u);
 }
+
+TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) {
+  auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  InMemoryFS->setCurrentWorkingDirectory("/");
+
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
+
+  bool Path1Exists = DepFS.exists("/path1");
+  EXPECT_EQ(Path1Exists, false);
+
+  // Adding a file that has been stat-ed,
+  InMemoryFS->addFile("/path1", 0, llvm::MemoryBuffer::getMemBuffer(""));
+  Path1Exists = DepFS.exists("/path1");
+  // Due to caching in SharedCache, path1 should not exist in
+  // DepFS's eyes.
+  EXPECT_EQ(Path1Exists, false);
+
+  std::vector<llvm::StringRef> InvalidPaths =
+      SharedCache.getInvalidNegativeStatCachedPaths(*InMemoryFS.get());
+
+  EXPECT_EQ(InvalidPaths.size(), 1u);
+  ASSERT_STREQ("/path1", InvalidPaths[0].str().c_str());
+}

From 89f930a7de5df59c5a4c2bbfce9b67404b33dc63 Mon Sep 17 00:00:00 2001
From: Ernesto Su <ernesto.su@intel.com>
Date: Fri, 18 Apr 2025 10:57:50 -0700
Subject: [PATCH 450/710] [clang][OpenMP] Fix/enforce order-concurrent-nestable
 rules (#135463)

OpenMP has restrictions on directives allowed to be strictly nested
inside a
construct with the order(concurrent) clause specified.
 - OpenMP 5.0, 5.1, and 5.2 allows: 'loop', 'parallel', 'simd', and
     combined directives starting with 'parallel'.
 - OpenMP 6.0 allows: the above directives plus 'atomic' and
     all loop-transformation directives.

Furthermore, a region that corresponds to a construct with
order(concurrent)
specified may not contain calls to the OpenMP runtime API.

This PR fixes the following issues in the current implementation:

With -fopenmp-version=50: none of the nesting restrictions above were
enforced
 With -fopenmp-version=60:
1. Clang did not reject OpenMP runtime APIs encountered in the region.
2. Clang erroneously rejected combined directives starting with
parallel.

---------

Co-authored-by: Zahira Ammarguellat <zahira.ammarguellat@intel.com>
---
 clang/include/clang/Basic/OpenMPKinds.h  |  4 +-
 clang/lib/Basic/OpenMPKinds.cpp          | 17 +++++--
 clang/lib/Sema/SemaOpenMP.cpp            | 57 +++++++++---------------
 clang/test/OpenMP/for_order_messages.cpp | 34 +++++++++-----
 4 files changed, 61 insertions(+), 51 deletions(-)

diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 6ca9f9c550285..9afcce21a499d 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -411,9 +411,11 @@ bool isOpenMPCapturingDirective(OpenMPDirectiveKind DKind);
 /// directive that can be nested within region corresponding to construct
 /// on which order clause was specified with concurrent as ordering argument.
 /// \param DKind Specified directive.
+/// \param LangOpts Used for getting the OpenMP version.
 /// \return true - if the above condition is met for this directive
 /// otherwise - false.
-bool isOpenMPOrderConcurrentNestableDirective(OpenMPDirectiveKind DKind);
+bool isOpenMPOrderConcurrentNestableDirective(OpenMPDirectiveKind DKind,
+                                              const LangOptions &LangOpts);
 }
 
 template <>
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 09921e3b1edfc..7b90861c78de0 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -768,9 +768,20 @@ bool clang::isOpenMPCapturingDirective(OpenMPDirectiveKind DKind) {
 }
 
 bool clang::isOpenMPOrderConcurrentNestableDirective(
-    OpenMPDirectiveKind DKind) {
-  return DKind == OMPD_atomic || DKind == OMPD_loop || DKind == OMPD_simd ||
-         DKind == OMPD_parallel || isOpenMPLoopTransformationDirective(DKind);
+    OpenMPDirectiveKind DKind, const LangOptions &LangOpts) {
+  // Directives strictly nestable in a construct with order(concurrent) are:
+  // OpenMP 5.x: loop, parallel, simd, combined directive starting with parallel
+  // OpenMP 6.0: above plus atomic and all loop-transformation directives
+
+  if (DKind == OMPD_loop || DKind == OMPD_parallel || DKind == OMPD_simd ||
+      isOpenMPCombinedParallelADirective(DKind))
+    return true;
+
+  if (LangOpts.OpenMP >= 60)
+    return DKind == OMPD_atomic ||
+           isOpenMPLoopTransformationDirective(DKind);
+
+  return false;
 }
 
 void clang::getOpenMPCaptureRegions(
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 0e44cfab6f216..274d3b90ff30f 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4789,26 +4789,12 @@ static bool checkNestingOfRegions(Sema &SemaRef, const DSAStackTy *Stack,
       getLeafOrCompositeConstructs(ParentRegion, LeafOrComposite);
   OpenMPDirectiveKind EnclosingConstruct = ParentLOC.back();
 
-  if (Stack->isParentOrderConcurrent()) {
-    bool InvalidOrderNesting = false;
-    if ((SemaRef.LangOpts.OpenMP == 51 || SemaRef.LangOpts.OpenMP == 52) &&
-        CurrentRegion != OMPD_simd && CurrentRegion != OMPD_loop &&
-        CurrentRegion != OMPD_parallel &&
-        !isOpenMPCombinedParallelADirective(CurrentRegion)) {
-      InvalidOrderNesting = true;
-    } else if (SemaRef.LangOpts.OpenMP >= 60 &&
-               !isOpenMPOrderConcurrentNestableDirective(CurrentRegion)) {
-      // OpenMP 6.0 [12.3 order Clause, Restrictions]
-      // Only regions that correspond to order-concurrent-nestable constructs
-      // or order-concurrent-nestable routines may be strictly nested regions
-      // of regions that correspond to constructs on which the order clause is
-      // specified with concurrent as the ordering argument.
-      InvalidOrderNesting = true;
-    }
-    if (InvalidOrderNesting) {
-      SemaRef.Diag(StartLoc, diag::err_omp_prohibited_region_order)
-          << getOpenMPDirectiveName(CurrentRegion);
-    }
+  if (SemaRef.LangOpts.OpenMP >= 50 && Stack->isParentOrderConcurrent() &&
+      !isOpenMPOrderConcurrentNestableDirective(CurrentRegion,
+                                                SemaRef.LangOpts)) {
+    SemaRef.Diag(StartLoc, diag::err_omp_prohibited_region_order)
+        << getOpenMPDirectiveName(CurrentRegion);
+    return true;
   }
   if (isOpenMPSimdDirective(ParentRegion) &&
       ((SemaRef.LangOpts.OpenMP <= 45 && CurrentRegion != OMPD_ordered) ||
@@ -7132,7 +7118,7 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
   if (!CalleeFnDecl)
     return Call;
 
-  if (getLangOpts().OpenMP >= 51 && getLangOpts().OpenMP < 60 &&
+  if (getLangOpts().OpenMP >= 50 && getLangOpts().OpenMP <= 60 &&
       CalleeFnDecl->getIdentifier() &&
       CalleeFnDecl->getName().starts_with_insensitive("omp_")) {
     // checking for any calls inside an Order region
@@ -16373,21 +16359,20 @@ OMPClause *SemaOpenMP::ActOnOpenMPOrderClause(
         << getOpenMPClauseName(OMPC_order);
     return nullptr;
   }
-  if (getLangOpts().OpenMP >= 51) {
-    if (Modifier == OMPC_ORDER_MODIFIER_unknown && MLoc.isValid()) {
-      Diag(MLoc, diag::err_omp_unexpected_clause_value)
-          << getListOfPossibleValues(OMPC_order,
-                                     /*First=*/OMPC_ORDER_MODIFIER_unknown + 1,
-                                     /*Last=*/OMPC_ORDER_MODIFIER_last)
-          << getOpenMPClauseName(OMPC_order);
-    } else {
-      DSAStack->setRegionHasOrderConcurrent(/*HasOrderConcurrent=*/true);
-      if (DSAStack->getCurScope()) {
-        // mark the current scope with 'order' flag
-        unsigned existingFlags = DSAStack->getCurScope()->getFlags();
-        DSAStack->getCurScope()->setFlags(existingFlags |
-                                          Scope::OpenMPOrderClauseScope);
-      }
+  if (getLangOpts().OpenMP >= 51 && Modifier == OMPC_ORDER_MODIFIER_unknown &&
+      MLoc.isValid()) {
+    Diag(MLoc, diag::err_omp_unexpected_clause_value)
+        << getListOfPossibleValues(OMPC_order,
+                                   /*First=*/OMPC_ORDER_MODIFIER_unknown + 1,
+                                   /*Last=*/OMPC_ORDER_MODIFIER_last)
+        << getOpenMPClauseName(OMPC_order);
+  } else if (getLangOpts().OpenMP >= 50) {
+    DSAStack->setRegionHasOrderConcurrent(/*HasOrderConcurrent=*/true);
+    if (DSAStack->getCurScope()) {
+      // mark the current scope with 'order' flag
+      unsigned existingFlags = DSAStack->getCurScope()->getFlags();
+      DSAStack->getCurScope()->setFlags(existingFlags |
+                                        Scope::OpenMPOrderClauseScope);
     }
   }
   return new (getASTContext()) OMPOrderClause(
diff --git a/clang/test/OpenMP/for_order_messages.cpp b/clang/test/OpenMP/for_order_messages.cpp
index 530c051849201..f41d3f7f635ce 100644
--- a/clang/test/OpenMP/for_order_messages.cpp
+++ b/clang/test/OpenMP/for_order_messages.cpp
@@ -8,35 +8,47 @@
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=52 -triple x86_64-unknown-unknown -verify=expected,omp51 %s -Wuninitialized
 // RUN: %clang_cc1 -fsyntax-only -fopenmp-simd -fopenmp-version=60 -triple x86_64-unknown-unknown -verify=expected,omp60 %s -Wuninitialized
 
+// Constructs strictly nestable in a construct with order(concurrent) specified vary by OpenMP version:
+// OMP5.0,5.1,5.2: loop, parallel, simd, and combined constructs with parallel as the first component.
+// OMP6.0: in addition to the ones allowed in OMP5.x, also atomic and all loop-transformation constructs.
+
 extern int omp_get_num_threads  (void);
 
 int main(int argc, char **argv) {
   int A = 0;
 #pragma omp parallel for order(concurrent)
   for (int i = 0; i < 10; ++i)
-    omp_get_num_threads(); // omp51-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}}
+    omp_get_num_threads(); // omp50-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}} omp51-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}} omp60-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}}
 
 #pragma omp parallel for order(reproducible:concurrent) // omp50-error {{expected 'concurrent' in OpenMP clause 'order'}}
   for (int i = 0; i < 10; ++i)
-    omp_get_num_threads(); // omp51-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}}
+    omp_get_num_threads(); // omp51-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}} omp60-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}}
 
 #pragma omp parallel for order(unconstrained:concurrent) // omp50-error {{expected 'concurrent' in OpenMP clause 'order'}}
   for (int i = 0; i < 10; ++i)
-    omp_get_num_threads(); // omp51-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}}
+    omp_get_num_threads(); // omp51-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}} omp60-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}}
 
 #pragma omp parallel for order(concurrent)
   for (int i = 0; i < 10; ++i) {
     for (int j = 0; j < 10; ++j) {
-      omp_get_num_threads(); // omp51-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}}
+      omp_get_num_threads(); // omp50-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}} omp51-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}} omp60-error {{calls to OpenMP runtime API are not allowed within a region that corresponds to a construct with an order clause that specifies concurrent}}
     }
   }
 
+// nested atomic: OK in OMP6.0 but not in OMP5.x
 #pragma omp parallel for order(concurrent)
   for (int i = 0; i < 10; ++i) {
-#pragma omp atomic //omp51-error {{construct 'atomic' not allowed in a region associated with a directive with 'order' clause}}
+#pragma omp atomic //omp50-error {{construct 'atomic' not allowed in a region associated with a directive with 'order' clause}} omp51-error {{construct 'atomic' not allowed in a region associated with a directive with 'order' clause}}
       A++;
   }
 
+// nested loop-transformation construct: OK in OMP6.0 but not in OMP5.x
+#pragma omp parallel for order(concurrent)
+  for (int i = 0; i < 10; ++i) {
+#pragma omp unroll //omp50-error {{construct 'unroll' not allowed in a region associated with a directive with 'order' clause}} omp51-error {{construct 'unroll' not allowed in a region associated with a directive with 'order' clause}}
+    for (int j = 0; j < 10; ++j);
+  }
+
 #pragma omp parallel for order(reproducible: concurrent) // omp50-error {{expected 'concurrent' in OpenMP clause 'order'}}
   for (int i = 0; i < 10; ++i) {
 #pragma omp target //omp51-error {{construct 'target' not allowed in a region associated with a directive with 'order' clause}} omp60-error {{construct 'target' not allowed in a region associated with a directive with 'order' clause}}
@@ -51,7 +63,7 @@ int main(int argc, char **argv) {
 
 #pragma omp loop bind(parallel) order(concurrent)
   for (int i = 0; i < 10; ++i) {
-#pragma omp parallel for //omp60-error {{construct 'parallel for' not allowed in a region associated with a directive with 'order' clause}}
+#pragma omp parallel for
     for (int j = 0; j < 10; ++j) {
       A += j;
     }
@@ -59,7 +71,7 @@ int main(int argc, char **argv) {
 
 #pragma omp distribute order(concurrent)
   for (int i = 0; i < 10; ++i) {
-#pragma omp parallel for simd //omp60-error {{construct 'parallel for simd' not allowed in a region associated with a directive with 'order' clause}}
+#pragma omp parallel for simd
     for (int j = 0; j < 10; ++j) {
       A += j;
     }
@@ -67,7 +79,7 @@ int main(int argc, char **argv) {
 
 #pragma omp for order(concurrent)
   for (int i = 0; i < 10; ++i) {
-#pragma omp parallel master //omp60-error {{construct 'parallel master' not allowed in a region associated with a directive with 'order' clause}}
+#pragma omp parallel master
     for (int j = 0; j < 10; ++j) {
       A += j;
     }
@@ -75,7 +87,7 @@ int main(int argc, char **argv) {
 
 #pragma omp for order(concurrent)
   for (int i = 0; i < 10; ++i) {
-#pragma omp parallel master taskloop //omp60-error {{construct 'parallel master taskloop' not allowed in a region associated with a directive with 'order' clause}}
+#pragma omp parallel master taskloop
     for (int j = 0; j < 10; ++j) {
       A += j;
     }
@@ -83,7 +95,7 @@ int main(int argc, char **argv) {
 
 #pragma omp for order(concurrent)
   for (int i = 0; i < 10; ++i) {
-#pragma omp parallel master taskloop simd //omp60-error {{construct 'parallel master taskloop simd' not allowed in a region associated with a directive with 'order' clause}}
+#pragma omp parallel master taskloop simd
     for (int j = 0; j < 10; ++j) {
       A += j;
     }
@@ -91,7 +103,7 @@ int main(int argc, char **argv) {
 
 #pragma omp for order(concurrent)
   for (int i = 0; i < 10; ++i) {
-    #pragma omp parallel sections //omp60-error {{construct 'parallel sections' not allowed in a region associated with a directive with 'order' clause}}
+    #pragma omp parallel sections
     {
       A++;
     }

From c27a9586267daebc7c586552b380a2f383b8552d Mon Sep 17 00:00:00 2001
From: Bruno Cardoso Lopes <bruno.cardoso@gmail.com>
Date: Fri, 18 Apr 2025 11:03:12 -0700
Subject: [PATCH 451/710] [MLIR][LLVMIR] Extend llrint,lrint,lround for vectors
 of float (#136225)

Matching langref. Note that `llround` is different than the rest.
---
 .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td   | 12 +++--
 mlir/test/Target/LLVMIR/Import/intrinsic.ll   | 46 +++++++++++++------
 .../test/Target/LLVMIR/llvmir-intrinsics.mlir | 29 ++++++++++--
 3 files changed, 66 insertions(+), 21 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index ab928c9e2d0e7..cd8b68e5b1410 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -153,16 +153,18 @@ def LLVM_PowOp : LLVM_BinarySameArgsIntrOpF<"pow">;
 def LLVM_PowIOp : LLVM_PowFI<"powi">;
 def LLVM_RintOp : LLVM_UnaryIntrOpF<"rint">;
 def LLVM_NearbyintOp : LLVM_UnaryIntrOpF<"nearbyint">;
-class LLVM_IntRoundIntrOpBase<string func> :
+class LLVM_IntRoundIntrOpBase<string func, Type element = LLVM_AnyFloat> :
         LLVM_OneResultIntrOp<func, [0], [0], [Pure]> {
-  let arguments = (ins LLVM_AnyFloat:$val);
+  let arguments = (ins element:$val);
   let assemblyFormat = "`(` operands `)` attr-dict `:` "
       "functional-type(operands, results)";
 }
-def LLVM_LroundOp : LLVM_IntRoundIntrOpBase<"lround">;
+class LLVM_ScalarOrVectorIntRoundIntrOpBase<string func> :
+  LLVM_IntRoundIntrOpBase<func, LLVM_ScalarOrVectorOf<LLVM_AnyFloat>>;
+def LLVM_LroundOp : LLVM_ScalarOrVectorIntRoundIntrOpBase<"lround">;
 def LLVM_LlroundOp : LLVM_IntRoundIntrOpBase<"llround">;
-def LLVM_LrintOp : LLVM_IntRoundIntrOpBase<"lrint">;
-def LLVM_LlrintOp : LLVM_IntRoundIntrOpBase<"llrint">;
+def LLVM_LrintOp : LLVM_ScalarOrVectorIntRoundIntrOpBase<"lrint">;
+def LLVM_LlrintOp : LLVM_ScalarOrVectorIntRoundIntrOpBase<"llrint">;
 def LLVM_BitReverseOp : LLVM_UnaryIntrOpI<"bitreverse">;
 def LLVM_ByteSwapOp : LLVM_UnaryIntrOpI<"bswap">;
 def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrOp<"ctlz">;
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index b0a36939d8c48..36afa84a031f4 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -235,15 +235,23 @@ define void @nearbyint_test(float %0, double %1, <8 x float> %2, <8 x double> %3
   ret void
 }
 ; CHECK-LABEL: llvm.func @lround_test
-define void @lround_test(float %0, double %1) {
+define void @lround_test(float %0, double %1, <2 x float> %2, <2 x double> %3) {
   ; CHECK: llvm.intr.lround(%{{.*}}) : (f32) -> i32
-  %3 = call i32 @llvm.lround.i32.f32(float %0)
+  %5 = call i32 @llvm.lround.i32.f32(float %0)
   ; CHECK: llvm.intr.lround(%{{.*}}) : (f32) -> i64
-  %4 = call i64 @llvm.lround.i64.f32(float %0)
+  %6 = call i64 @llvm.lround.i64.f32(float %0)
   ; CHECK: llvm.intr.lround(%{{.*}}) : (f64) -> i32
-  %5 = call i32 @llvm.lround.i32.f64(double %1)
+  %7 = call i32 @llvm.lround.i32.f64(double %1)
   ; CHECK: llvm.intr.lround(%{{.*}}) : (f64) -> i64
-  %6 = call i64 @llvm.lround.i64.f64(double %1)
+  %8 = call i64 @llvm.lround.i64.f64(double %1)
+  ; CHECK: llvm.intr.lround(%{{.*}}) : (vector<2xf32>) -> vector<2xi32>
+  %9 = call <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float> %2)
+  ; CHECK: llvm.intr.lround(%{{.*}}) : (vector<2xf64>) -> vector<2xi32>
+  %10 = call <2 x i32> @llvm.lround.v2i32.v2f64(<2 x double> %3)
+  ; CHECK: llvm.intr.lround(%{{.*}}) : (vector<2xf32>) -> vector<2xi64>
+  %11 = call <2 x i64> @llvm.lround.v2i64.v2f32(<2 x float> %2)
+  ; CHECK: llvm.intr.lround(%{{.*}}) : (vector<2xf64>) -> vector<2xi64>
+  %12 = call <2 x i64> @llvm.lround.v2i64.v2f64(<2 x double> %3)
   ret void
 }
 ; CHECK-LABEL: llvm.func @llround_test
@@ -255,23 +263,35 @@ define void @llround_test(float %0, double %1) {
   ret void
 }
 ; CHECK-LABEL: llvm.func @lrint_test
-define void @lrint_test(float %0, double %1) {
+define void @lrint_test(float %0, double %1, <2 x float> %2, <2 x double> %3) {
   ; CHECK: llvm.intr.lrint(%{{.*}}) : (f32) -> i32
-  %3 = call i32 @llvm.lrint.i32.f32(float %0)
+  %5 = call i32 @llvm.lrint.i32.f32(float %0)
   ; CHECK: llvm.intr.lrint(%{{.*}}) : (f32) -> i64
-  %4 = call i64 @llvm.lrint.i64.f32(float %0)
+  %6 = call i64 @llvm.lrint.i64.f32(float %0)
   ; CHECK: llvm.intr.lrint(%{{.*}}) : (f64) -> i32
-  %5 = call i32 @llvm.lrint.i32.f64(double %1)
+  %7 = call i32 @llvm.lrint.i32.f64(double %1)
   ; CHECK: llvm.intr.lrint(%{{.*}}) : (f64) -> i64
-  %6 = call i64 @llvm.lrint.i64.f64(double %1)
+  %8 = call i64 @llvm.lrint.i64.f64(double %1)
+  ; CHECK: llvm.intr.lrint(%{{.*}}) : (vector<2xf32>) -> vector<2xi32>
+  %9 = call <2 x i32> @llvm.lrint.v2i32.v2f32(<2 x float> %2)
+  ; CHECK: llvm.intr.lrint(%{{.*}}) : (vector<2xf64>) -> vector<2xi32>
+  %10 = call <2 x i32> @llvm.lrint.v2i32.v2f64(<2 x double> %3)
+  ; CHECK: llvm.intr.lrint(%{{.*}}) : (vector<2xf32>) -> vector<2xi64>
+  %11 = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> %2)
+  ; CHECK: llvm.intr.lrint(%{{.*}}) : (vector<2xf64>) -> vector<2xi64>
+  %12 = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> %3)
   ret void
 }
 ; CHECK-LABEL: llvm.func @llrint_test
-define void @llrint_test(float %0, double %1) {
+define void @llrint_test(float %0, double %1, <2 x float> %2, <2 x double> %3) {
   ; CHECK: llvm.intr.llrint(%{{.*}}) : (f32) -> i64
-  %3 = call i64 @llvm.llrint.i64.f32(float %0)
+  %5 = call i64 @llvm.llrint.i64.f32(float %0)
   ; CHECK: llvm.intr.llrint(%{{.*}}) : (f64) -> i64
-  %4 = call i64 @llvm.llrint.i64.f64(double %1)
+  %6 = call i64 @llvm.llrint.i64.f64(double %1)
+  ; CHECK: llvm.intr.llrint(%{{.*}}) : (vector<2xf32>) -> vector<2xi64>
+  %7 = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %2)
+  ; CHECK: llvm.intr.llrint(%{{.*}}) : (vector<2xf64>) -> vector<2xi64>
+  %8 = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %3)
   ret void
 }
 
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 8088dec811e13..ba12140c59b35 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -242,7 +242,8 @@ llvm.func @nearbyint_test(%arg0 : f32, %arg1 : f64, %arg2 : vector<8xf32>, %arg3
 }
 
 // CHECK-LABEL: @lround_test
-llvm.func @lround_test(%arg0 : f32, %arg1 : f64) {
+llvm.func @lround_test(%arg0 : f32, %arg1 : f64,
+                       %arg2 : vector<2xf32>, %arg3 : vector<2xf64>) {
   // CHECK: call i32 @llvm.lround.i32.f32
   "llvm.intr.lround"(%arg0) : (f32) -> i32
   // CHECK: call i64 @llvm.lround.i64.f32
@@ -251,6 +252,14 @@ llvm.func @lround_test(%arg0 : f32, %arg1 : f64) {
   "llvm.intr.lround"(%arg1) : (f64) -> i32
   // CHECK: call i64 @llvm.lround.i64.f64
   "llvm.intr.lround"(%arg1) : (f64) -> i64
+  // CHECK: call <2 x i32> @llvm.lround.v2i32.v2f32
+  "llvm.intr.lround"(%arg2) : (vector<2xf32>) -> vector<2xi32>
+  // CHECK: call <2 x i32> @llvm.lround.v2i32.v2f64
+  "llvm.intr.lround"(%arg3) : (vector<2xf64>) -> vector<2xi32>
+  // CHECK: call <2 x i64> @llvm.lround.v2i64.v2f32
+  "llvm.intr.lround"(%arg2) : (vector<2xf32>) -> vector<2xi64>
+  // CHECK: call <2 x i64> @llvm.lround.v2i64.v2f64
+  "llvm.intr.lround"(%arg3) : (vector<2xf64>) -> vector<2xi64>
   llvm.return
 }
 
@@ -264,7 +273,8 @@ llvm.func @llround_test(%arg0 : f32, %arg1 : f64) {
 }
 
 // CHECK-LABEL: @lrint_test
-llvm.func @lrint_test(%arg0 : f32, %arg1 : f64) {
+llvm.func @lrint_test(%arg0 : f32, %arg1 : f64,
+                      %arg2 : vector<2xf32>, %arg3 : vector<2xf64>) {
   // CHECK: call i32 @llvm.lrint.i32.f32
   "llvm.intr.lrint"(%arg0) : (f32) -> i32
   // CHECK: call i64 @llvm.lrint.i64.f32
@@ -273,15 +283,28 @@ llvm.func @lrint_test(%arg0 : f32, %arg1 : f64) {
   "llvm.intr.lrint"(%arg1) : (f64) -> i32
   // CHECK: call i64 @llvm.lrint.i64.f64
   "llvm.intr.lrint"(%arg1) : (f64) -> i64
+  // CHECK: call <2 x i32> @llvm.lrint.v2i32.v2f32
+  "llvm.intr.lrint"(%arg2) : (vector<2xf32>) -> vector<2xi32>
+  // CHECK: call <2 x i32> @llvm.lrint.v2i32.v2f64
+  "llvm.intr.lrint"(%arg3) : (vector<2xf64>) -> vector<2xi32>
+  // CHECK: call <2 x i64> @llvm.lrint.v2i64.v2f32
+  "llvm.intr.lrint"(%arg2) : (vector<2xf32>) -> vector<2xi64>
+  // CHECK: call <2 x i64> @llvm.lrint.v2i64.v2f64
+  "llvm.intr.lrint"(%arg3) : (vector<2xf64>) -> vector<2xi64>
   llvm.return
 }
 
 // CHECK-LABEL: @llrint_test
-llvm.func @llrint_test(%arg0 : f32, %arg1 : f64) {
+llvm.func @llrint_test(%arg0 : f32, %arg1 : f64,
+                       %arg2 : vector<2xf32>, %arg3 : vector<2xf64>) {
   // CHECK: call i64 @llvm.llrint.i64.f32
   "llvm.intr.llrint"(%arg0) : (f32) -> i64
   // CHECK: call i64 @llvm.llrint.i64.f64
   "llvm.intr.llrint"(%arg1) : (f64) -> i64
+  // CHECK: call <2 x i64> @llvm.llrint.v2i64.v2f32
+  "llvm.intr.llrint"(%arg2) : (vector<2xf32>) -> vector<2xi64>
+  // CHECK: call <2 x i64> @llvm.llrint.v2i64.v2f64
+  "llvm.intr.llrint"(%arg3) : (vector<2xf64>) -> vector<2xi64>
   llvm.return
 }
 

From cab75384af8b5638822475b2dd9020b7e879f1f5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 18 Apr 2025 19:04:17 +0100
Subject: [PATCH 452/710] [VPlan] Only generate exit blocks for unique exit
 blocks.

Make sure we don't generate unnecessary blocks.
---
 llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index eb4394171d3be..fa2d95a44609a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -858,7 +858,7 @@ VPlan::VPlan(Loop *L) {
   ScalarHeader = createVPIRBasicBlock(L->getHeader());
 
   SmallVector<BasicBlock *> IRExitBlocks;
-  L->getExitBlocks(IRExitBlocks);
+  L->getUniqueExitBlocks(IRExitBlocks);
   for (BasicBlock *EB : IRExitBlocks)
     ExitBlocks.push_back(createVPIRBasicBlock(EB));
 }

From 3191cfd824771d75949b1ebf5e5fe9073b7fe746 Mon Sep 17 00:00:00 2001
From: Andy Kaylor <akaylor@nvidia.com>
Date: Fri, 18 Apr 2025 11:25:01 -0700
Subject: [PATCH 453/710] [CIR] Upstream limited support for nested structures
 (#136331)

Previously when we checked to see if it was safe to create CIR for a
structure type, we were conservatively saying no if any structure was in
the process of being converted. That prevented handling nested
structures, even when it would have actually been safe to handle them.
Handling structures which recursively reference structures currently
being processed requires deferring the handling of the recursively
referenced structure, and that still isn't implemented after this
change.

This change adds the less conservative check that allows handling of
non-recursive nested structures.
---
 clang/lib/CIR/CodeGen/CIRGenTypes.cpp | 85 +++++++++++++++++++++++++--
 clang/lib/CIR/CodeGen/CIRGenTypes.h   |  4 ++
 clang/test/CIR/CodeGen/struct.c       | 21 +++++++
 3 files changed, 106 insertions(+), 4 deletions(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index a896b9dce14d6..7bd86cf0c7bcd 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -116,16 +116,93 @@ std::string CIRGenTypes::getRecordTypeName(const clang::RecordDecl *recordDecl,
   return builder.getUniqueRecordName(std::string(typeName));
 }
 
+/// Return true if the specified type is already completely laid out.
+bool CIRGenTypes::isRecordLayoutComplete(const Type *ty) const {
+  const auto it = recordDeclTypes.find(ty);
+  return it != recordDeclTypes.end() && it->second.isComplete();
+}
+
+// We have multiple forms of this function that call each other, so we need to
+// declare one in advance.
+static bool
+isSafeToConvert(QualType qt, CIRGenTypes &cgt,
+                llvm::SmallPtrSetImpl<const RecordDecl *> &alreadyChecked);
+
+/// Return true if it is safe to convert the specified record decl to CIR and
+/// lay it out, false if doing so would cause us to get into a recursive
+/// compilation mess.
+static bool
+isSafeToConvert(const RecordDecl *rd, CIRGenTypes &cgt,
+                llvm::SmallPtrSetImpl<const RecordDecl *> &alreadyChecked) {
+  // If we have already checked this type (maybe the same type is used by-value
+  // multiple times in multiple record fields, don't check again.
+  if (!alreadyChecked.insert(rd).second)
+    return true;
+
+  const Type *key = cgt.getASTContext().getTagDeclType(rd).getTypePtr();
+
+  // If this type is already laid out, converting it is a noop.
+  if (cgt.isRecordLayoutComplete(key))
+    return true;
+
+  // If this type is currently being laid out, we can't recursively compile it.
+  if (cgt.isRecordBeingLaidOut(key))
+    return false;
+
+  // If this type would require laying out bases that are currently being laid
+  // out, don't do it.  This includes virtual base classes which get laid out
+  // when a class is translated, even though they aren't embedded by-value into
+  // the class.
+  if (const CXXRecordDecl *crd = dyn_cast<CXXRecordDecl>(rd)) {
+    assert(!cir::MissingFeatures::cxxSupport());
+    cgt.getCGModule().errorNYI(rd->getSourceRange(),
+                               "isSafeToConvert: CXXRecordDecl");
+    return false;
+  }
+
+  // If this type would require laying out members that are currently being laid
+  // out, don't do it.
+  for (const FieldDecl *field : rd->fields())
+    if (!isSafeToConvert(field->getType(), cgt, alreadyChecked))
+      return false;
+
+  // If there are no problems, lets do it.
+  return true;
+}
+
+/// Return true if it is safe to convert this field type, which requires the
+/// record elements contained by-value to all be recursively safe to convert.
+static bool
+isSafeToConvert(QualType qt, CIRGenTypes &cgt,
+                llvm::SmallPtrSetImpl<const RecordDecl *> &alreadyChecked) {
+  // Strip off atomic type sugar.
+  if (const auto *at = qt->getAs<AtomicType>())
+    qt = at->getValueType();
+
+  // If this is a record, check it.
+  if (const auto *rt = qt->getAs<RecordType>())
+    return isSafeToConvert(rt->getDecl(), cgt, alreadyChecked);
+
+  // If this is an array, check the elements, which are embedded inline.
+  if (const auto *at = cgt.getASTContext().getAsArrayType(qt))
+    return isSafeToConvert(at->getElementType(), cgt, alreadyChecked);
+
+  // Otherwise, there is no concern about transforming this. We only care about
+  // things that are contained by-value in a record that can have another
+  // record as a member.
+  return true;
+}
+
 // Return true if it is safe to convert the specified record decl to CIR and lay
 // it out, false if doing so would cause us to get into a recursive compilation
 // mess.
-static bool isSafeToConvert(const RecordDecl *RD, CIRGenTypes &CGT) {
+static bool isSafeToConvert(const RecordDecl *rd, CIRGenTypes &cgt) {
   // If no records are being laid out, we can certainly do this one.
-  if (CGT.noRecordsBeingLaidOut())
+  if (cgt.noRecordsBeingLaidOut())
     return true;
 
-  assert(!cir::MissingFeatures::recursiveRecordLayout());
-  return false;
+  llvm::SmallPtrSet<const RecordDecl *, 16> alreadyChecked;
+  return isSafeToConvert(rd, cgt, alreadyChecked);
 }
 
 /// Lay out a tagged decl type like struct or union.
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.h b/clang/lib/CIR/CodeGen/CIRGenTypes.h
index fc6e9cf621cb3..2bb78420700f8 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.h
@@ -89,7 +89,11 @@ class CIRGenTypes {
   mlir::MLIRContext &getMLIRContext() const;
   clang::ASTContext &getASTContext() const { return astContext; }
 
+  bool isRecordLayoutComplete(const clang::Type *ty) const;
   bool noRecordsBeingLaidOut() const { return recordsBeingLaidOut.empty(); }
+  bool isRecordBeingLaidOut(const clang::Type *ty) const {
+    return recordsBeingLaidOut.count(ty);
+  }
 
   const ABIInfo &getABIInfo() const { return theABIInfo; }
 
diff --git a/clang/test/CIR/CodeGen/struct.c b/clang/test/CIR/CodeGen/struct.c
index 55f316f27eeb1..3dc1655e15d2c 100644
--- a/clang/test/CIR/CodeGen/struct.c
+++ b/clang/test/CIR/CodeGen/struct.c
@@ -8,9 +8,13 @@
 // For LLVM IR checks, the structs are defined before the variables, so these
 // checks are at the top.
 // LLVM: %struct.CompleteS = type { i32, i8 }
+// LLVM: %struct.OuterS = type { %struct.InnerS, i32 }
+// LLVM: %struct.InnerS = type { i32, i8 }
 // LLVM: %struct.PackedS = type <{ i32, i8 }>
 // LLVM: %struct.PackedAndPaddedS = type <{ i32, i8, i8 }>
 // OGCG: %struct.CompleteS = type { i32, i8 }
+// OGCG: %struct.OuterS = type { %struct.InnerS, i32 }
+// OGCG: %struct.InnerS = type { i32, i8 }
 // OGCG: %struct.PackedS = type <{ i32, i8 }>
 // OGCG: %struct.PackedAndPaddedS = type <{ i32, i8, i8 }>
 
@@ -31,6 +35,23 @@ struct CompleteS {
 // LLVM:      @cs = dso_local global %struct.CompleteS zeroinitializer
 // OGCG:      @cs = global %struct.CompleteS zeroinitializer, align 4
 
+struct InnerS {
+  int a;
+  char b;
+};
+
+struct OuterS {
+  struct InnerS is;
+  int c;
+};
+
+struct OuterS os;
+
+// CIR:       cir.global external @os = #cir.zero : !cir.record<struct
+// CIR-SAME:      "OuterS" {!cir.record<struct "InnerS" {!s32i, !s8i}>, !s32i}>
+// LLVM:      @os = dso_local global %struct.OuterS zeroinitializer
+// OGCG:      @os = global %struct.OuterS zeroinitializer, align 4
+
 #pragma pack(push)
 #pragma pack(1)
 

From 7cc4472037b43971bd3ee373fe75b5043f5abca9 Mon Sep 17 00:00:00 2001
From: thetruestblue <bblueconway@gmail.com>
Date: Fri, 18 Apr 2025 11:25:31 -0700
Subject: [PATCH 454/710] [RTSan][Darwin] Adjust OSSpinLock/_os_nospin_lock
 interceptor and tests (#132867)

These changes align with these lock types and allows builds and tests to
pass with various SDKS.

rdar://147067322
---
 .../lib/rtsan/rtsan_interceptors_posix.cpp    | 37 +++++++----------
 .../tests/rtsan_test_interceptors_posix.cpp   | 40 +++++++++----------
 2 files changed, 32 insertions(+), 45 deletions(-)

diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
index e8cea21ddf9aa..9f9ffd4313810 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
@@ -21,24 +21,6 @@
 #include "rtsan/rtsan.h"
 
 #if SANITIZER_APPLE
-
-#if TARGET_OS_MAC
-// On MacOS OSSpinLockLock is deprecated and no longer present in the headers,
-// but the symbol still exists on the system. Forward declare here so we
-// don't get compilation errors.
-#include <stdint.h>
-extern "C" {
-typedef int32_t OSSpinLock;
-void OSSpinLockLock(volatile OSSpinLock *__lock);
-// A pointer to this type is in the interface for `_os_nospin_lock_lock`, but
-// it's an internal implementation detail of `os/lock.c` on Darwin, and
-// therefore not available in any headers. As a workaround, we forward declare
-// it here, which is enough to facilitate interception of _os_nospin_lock_lock.
-struct _os_nospin_lock_s;
-using _os_nospin_lock_t = _os_nospin_lock_s *;
-}
-#endif // TARGET_OS_MAC
-
 #include <libkern/OSAtomic.h>
 #include <os/lock.h>
 #endif // SANITIZER_APPLE
@@ -717,26 +699,35 @@ INTERCEPTOR(mode_t, umask, mode_t cmask) {
 #pragma clang diagnostic push
 // OSSpinLockLock is deprecated, but still in use in libc++
 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#undef OSSpinLockLock
+
 INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
   __rtsan_notify_intercepted_call("OSSpinLockLock");
   return REAL(OSSpinLockLock)(lock);
 }
-#pragma clang diagnostic pop
+
 #define RTSAN_MAYBE_INTERCEPT_OSSPINLOCKLOCK INTERCEPT_FUNCTION(OSSpinLockLock)
 #else
 #define RTSAN_MAYBE_INTERCEPT_OSSPINLOCKLOCK
 #endif // SANITIZER_APPLE
 
 #if SANITIZER_APPLE
-INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
-  __rtsan_notify_intercepted_call("os_unfair_lock_lock");
-  return REAL(os_unfair_lock_lock)(lock);
-}
+// _os_nospin_lock_lock may replace OSSpinLockLock due to deprecation macro.
+typedef volatile OSSpinLock *_os_nospin_lock_t;
 
 INTERCEPTOR(void, _os_nospin_lock_lock, _os_nospin_lock_t lock) {
   __rtsan_notify_intercepted_call("_os_nospin_lock_lock");
   return REAL(_os_nospin_lock_lock)(lock);
 }
+#pragma clang diagnostic pop // "-Wdeprecated-declarations"
+#endif                       // SANITIZER_APPLE
+
+#if SANITIZER_APPLE
+INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
+  __rtsan_notify_intercepted_call("os_unfair_lock_lock");
+  return REAL(os_unfair_lock_lock)(lock);
+}
+
 #define RTSAN_MAYBE_INTERCEPT_OS_UNFAIR_LOCK_LOCK                              \
   INTERCEPT_FUNCTION(os_unfair_lock_lock)
 #else
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
index 048da5858d665..c2d07400593d5 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
@@ -1126,10 +1126,18 @@ TEST(TestRtsanInterceptors, PthreadJoinDiesWhenRealtime) {
 }
 
 #if SANITIZER_APPLE
-
 #pragma clang diagnostic push
 // OSSpinLockLock is deprecated, but still in use in libc++
 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#undef OSSpinLockLock
+extern "C" {
+typedef int32_t OSSpinLock;
+void OSSpinLockLock(volatile OSSpinLock *__lock);
+// _os_nospin_lock_lock may replace OSSpinLockLock due to deprecation macro.
+typedef volatile OSSpinLock *_os_nospin_lock_t;
+void _os_nospin_lock_lock(_os_nospin_lock_t lock);
+}
+
 TEST(TestRtsanInterceptors, OsSpinLockLockDiesWhenRealtime) {
   auto Func = []() {
     OSSpinLock spin_lock{};
@@ -1138,7 +1146,14 @@ TEST(TestRtsanInterceptors, OsSpinLockLockDiesWhenRealtime) {
   ExpectRealtimeDeath(Func, "OSSpinLockLock");
   ExpectNonRealtimeSurvival(Func);
 }
-#pragma clang diagnostic pop
+
+TEST(TestRtsanInterceptors, OsNoSpinLockLockDiesWhenRealtime) {
+  OSSpinLock lock{};
+  auto Func = [&]() { _os_nospin_lock_lock(&lock); };
+  ExpectRealtimeDeath(Func, "_os_nospin_lock_lock");
+  ExpectNonRealtimeSurvival(Func);
+}
+#pragma clang diagnostic pop //"-Wdeprecated-declarations"
 
 TEST(TestRtsanInterceptors, OsUnfairLockLockDiesWhenRealtime) {
   auto Func = []() {
@@ -1148,26 +1163,7 @@ TEST(TestRtsanInterceptors, OsUnfairLockLockDiesWhenRealtime) {
   ExpectRealtimeDeath(Func, "os_unfair_lock_lock");
   ExpectNonRealtimeSurvival(Func);
 }
-
-// We intercept _os_nospin_lock_lock because it's the internal
-// locking mechanism for MacOS's atomic implementation for data
-// types that are larger than the hardware's maximum lock-free size.
-// However, it's a private implementation detail and not visible in any headers,
-// so we must duplicate the required type definitions to forward declaration
-// what we need here.
-extern "C" {
-struct _os_nospin_lock_s {
-  unsigned int oul_value;
-};
-void _os_nospin_lock_lock(_os_nospin_lock_s *);
-}
-TEST(TestRtsanInterceptors, OsNoSpinLockLockDiesWhenRealtime) {
-  _os_nospin_lock_s lock{};
-  auto Func = [&]() { _os_nospin_lock_lock(&lock); };
-  ExpectRealtimeDeath(Func, "_os_nospin_lock_lock");
-  ExpectNonRealtimeSurvival(Func);
-}
-#endif
+#endif // SANITIZER_APPLE
 
 #if SANITIZER_LINUX
 TEST(TestRtsanInterceptors, SpinLockLockDiesWhenRealtime) {

From 7c51e426159a7751b062e9860e1ec71cd8fc7bb7 Mon Sep 17 00:00:00 2001
From: Andy Kaylor <akaylor@nvidia.com>
Date: Fri, 18 Apr 2025 11:25:41 -0700
Subject: [PATCH 455/710] [CIR] Upstream scalar support for ParenExpr (#136332)

This change adds support for handling ParenExpr in scalar expressions. A
few more places will need to be updated after structure assignment and
complex type support is in place.
---
 clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp |  2 +
 clang/lib/CIR/CodeGen/CIRGenFunction.cpp   |  2 +
 clang/test/CIR/CodeGen/basic.c             | 63 ++++++++++++++++++++++
 3 files changed, 67 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 3dae26dc86f85..1bef1b976a4b5 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -116,6 +116,8 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     return {};
   }
 
+  mlir::Value VisitParenExpr(ParenExpr *pe) { return Visit(pe->getSubExpr()); }
+
   /// Emits the address of the l-value, then loads and returns the result.
   mlir::Value emitLoadOfLValue(const Expr *e) {
     LValue lv = cgf.emitLValue(e);
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index 9dace721e7417..76e9ca4fd61a8 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -515,6 +515,8 @@ LValue CIRGenFunction::emitLValue(const Expr *e) {
     return emitUnaryOpLValue(cast<UnaryOperator>(e));
   case Expr::BinaryOperatorClass:
     return emitBinaryOperatorLValue(cast<BinaryOperator>(e));
+  case Expr::ParenExprClass:
+    return emitLValue(cast<ParenExpr>(e)->getSubExpr());
   case Expr::DeclRefExprClass:
     return emitDeclRefLValue(cast<DeclRefExpr>(e));
   }
diff --git a/clang/test/CIR/CodeGen/basic.c b/clang/test/CIR/CodeGen/basic.c
index cf687e44044c4..8ce8106b0c6cc 100644
--- a/clang/test/CIR/CodeGen/basic.c
+++ b/clang/test/CIR/CodeGen/basic.c
@@ -169,3 +169,66 @@ int f6(void) {
 // OGCG-NEXT: entry:
 // OGCG-NEXT:   %[[GV:.*]] = load i32, ptr @gv, align 4
 // OGCG-NEXT:   ret i32 %[[GV]]
+
+int f7(int a, int b, int c) {
+  return a + (b + c);
+}
+
+// CIR: cir.func @f7
+// CIR:  %[[A_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
+// CIR:  %[[B_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
+// CIR:  %[[C_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["c", init]
+// CIR:  %[[A:.*]] = cir.load %[[A_PTR]] : !cir.ptr<!s32i>, !s32i
+// CIR:  %[[B:.*]] = cir.load %[[B_PTR]] : !cir.ptr<!s32i>, !s32i
+// CIR:  %[[C:.*]] = cir.load %[[C_PTR]] : !cir.ptr<!s32i>, !s32i
+// CIR:  %[[B_PLUS_C:.*]] = cir.binop(add, %[[B]], %[[C]]) nsw : !s32i
+// CIR:  %[[RETVAL:.*]] = cir.binop(add, %[[A]], %[[B_PLUS_C]]) nsw : !s32i
+
+// LLVM: define i32 @f7
+// LLVM:   %[[A_PTR:.*]] = alloca i32, i64 1, align 4
+// LLVM:   %[[B_PTR:.*]] = alloca i32, i64 1, align 4
+// LLVM:   %[[C_PTR:.*]] = alloca i32, i64 1, align 4
+// LLVM:   %[[A:.*]] = load i32, ptr %[[A_PTR]], align 4
+// LLVM:   %[[B:.*]] = load i32, ptr %[[B_PTR]], align 4
+// LLVM:   %[[C:.*]] = load i32, ptr %[[C_PTR]], align 4
+// LLVM:   %[[B_PLUS_C:.*]] = add nsw i32 %[[B]], %[[C]]
+// LLVM:   %[[RETVAL:.*]] = add nsw i32 %[[A]], %[[B_PLUS_C]]
+
+// OGCG: define{{.*}} i32 @f7
+// OGCG: entry:
+// OGCG:   %[[A_PTR:.*]] = alloca i32, align 4
+// OGCG:   %[[B_PTR:.*]] = alloca i32, align 4
+// OGCG:   %[[C_PTR:.*]] = alloca i32, align 4
+// OGCG:   %[[A:.*]] = load i32, ptr %[[A_PTR]], align 4
+// OGCG:   %[[B:.*]] = load i32, ptr %[[B_PTR]], align 4
+// OGCG:   %[[C:.*]] = load i32, ptr %[[C_PTR]], align 4
+// OGCG:   %[[B_PLUS_C:.*]] = add nsw i32 %[[B]], %[[C]]
+// OGCG:   %[[RETVAL:.*]] = add nsw i32 %[[A]], %[[B_PLUS_C]]
+
+int f8(int *p) {
+  (*p) = 2;
+  return (*p);
+}
+
+// CIR: cir.func @f8
+// CIR:    %[[P_PTR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["p", init]
+// CIR:    %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+// CIR:    %[[P:.*]] = cir.load deref %[[P_PTR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR:    cir.store %[[TWO]], %[[P]] : !s32i, !cir.ptr<!s32i>
+// CIR:    %[[P2:.*]] = cir.load deref %[[P_PTR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR:    %[[STAR_P:.*]] = cir.load %[[P2]] : !cir.ptr<!s32i>, !s32i
+
+// LLVM: define i32 @f8
+// LLVM:   %[[P_PTR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[P:.*]] = load ptr, ptr %[[P_PTR]], align 8
+// LLVM:   store i32 2, ptr %[[P]], align 4
+// LLVM:   %[[P2:.*]] = load ptr, ptr %[[P_PTR]], align 8
+// LLVM:   %[[STAR_P:.*]] = load i32, ptr %[[P2]], align 4
+
+// OGCG: define{{.*}} i32 @f8
+// OGCG: entry:
+// OGCG:   %[[P_PTR:.*]] = alloca ptr, align 8
+// OGCG:   %[[P:.*]] = load ptr, ptr %[[P_PTR]], align 8
+// OGCG:   store i32 2, ptr %[[P]], align 4
+// OGCG:   %[[P2:.*]] = load ptr, ptr %[[P_PTR]], align 8
+// OGCG:   %[[STAR_P:.*]] = load i32, ptr %[[P2]], align 4

From 4091f4dd96d1501e007ed044fc93be1bfaafea08 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Fri, 18 Apr 2025 11:56:07 -0700
Subject: [PATCH 456/710] Reland [flang] Generalized simplification of HLFIR
 reduction ops. (#136071) (#136246)

This change generalizes SumAsElemental inlining in
SimplifyHLFIRIntrinsics pass so that it can be applied
to ALL, ANY, COUNT, MAXLOC, MAXVAL, MINLOC, MINVAL, SUM.

This change makes the special handling of the reduction
operations in OptimizedBufferization redundant: once HLFIR
operations are inlined, the hlfir.elemental inlining should
do the rest of the job.
---
 .../Transforms/OptimizedBufferization.cpp     |  465 -------
 .../Transforms/SimplifyHLFIRIntrinsics.cpp    | 1064 +++++++++++++----
 flang/test/HLFIR/all-elemental.fir            |   91 --
 flang/test/HLFIR/any-elemental.fir            |  190 ---
 flang/test/HLFIR/count-elemental.fir          |  314 -----
 flang/test/HLFIR/maxloc-elemental.fir         |  133 ---
 flang/test/HLFIR/maxval-elemental.fir         |  117 --
 flang/test/HLFIR/minloc-elemental.fir         |  397 ------
 flang/test/HLFIR/minval-elemental.fir         |   95 --
 .../HLFIR/simplify-hlfir-intrinsics-all.fir   |  123 ++
 .../HLFIR/simplify-hlfir-intrinsics-any.fir   |  123 ++
 .../HLFIR/simplify-hlfir-intrinsics-count.fir |  127 ++
 .../simplify-hlfir-intrinsics-maxloc.fir      |  312 +++++
 .../simplify-hlfir-intrinsics-maxval.fir      |  186 +++
 .../simplify-hlfir-intrinsics-minloc.fir      |  312 +++++
 .../simplify-hlfir-intrinsics-minval.fir      |  186 +++
 16 files changed, 2233 insertions(+), 2002 deletions(-)
 delete mode 100644 flang/test/HLFIR/all-elemental.fir
 delete mode 100644 flang/test/HLFIR/any-elemental.fir
 delete mode 100644 flang/test/HLFIR/count-elemental.fir
 delete mode 100644 flang/test/HLFIR/maxloc-elemental.fir
 delete mode 100644 flang/test/HLFIR/maxval-elemental.fir
 delete mode 100644 flang/test/HLFIR/minloc-elemental.fir
 delete mode 100644 flang/test/HLFIR/minval-elemental.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
 create mode 100644 flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index c489450384a35..79aabd2981e1a 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -772,458 +772,6 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   return mlir::success();
 }
 
-using GenBodyFn =
-    std::function<mlir::Value(fir::FirOpBuilder &, mlir::Location, mlir::Value,
-                              const llvm::SmallVectorImpl<mlir::Value> &)>;
-static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
-                                         mlir::Location loc, mlir::Value init,
-                                         mlir::Value shape, GenBodyFn genBody) {
-  auto extents = hlfir::getIndexExtents(loc, builder, shape);
-  mlir::Value reduction = init;
-  mlir::IndexType idxTy = builder.getIndexType();
-  mlir::Value oneIdx = builder.createIntegerConstant(loc, idxTy, 1);
-
-  // Create a reduction loop nest. We use one-based indices so that they can be
-  // passed to the elemental, and reverse the order so that they can be
-  // generated in column-major order for better performance.
-  llvm::SmallVector<mlir::Value> indices(extents.size(), mlir::Value{});
-  for (unsigned i = 0; i < extents.size(); ++i) {
-    auto loop = builder.create<fir::DoLoopOp>(
-        loc, oneIdx, extents[extents.size() - i - 1], oneIdx, false,
-        /*finalCountValue=*/false, reduction);
-    reduction = loop.getRegionIterArgs()[0];
-    indices[extents.size() - i - 1] = loop.getInductionVar();
-    // Set insertion point to the loop body so that the next loop
-    // is inserted inside the current one.
-    builder.setInsertionPointToStart(loop.getBody());
-  }
-
-  // Generate the body
-  reduction = genBody(builder, loc, reduction, indices);
-
-  // Unwind the loop nest.
-  for (unsigned i = 0; i < extents.size(); ++i) {
-    auto result = builder.create<fir::ResultOp>(loc, reduction);
-    auto loop = mlir::cast<fir::DoLoopOp>(result->getParentOp());
-    reduction = loop.getResult(0);
-    // Set insertion point after the loop operation that we have
-    // just processed.
-    builder.setInsertionPointAfter(loop.getOperation());
-  }
-
-  return reduction;
-}
-
-auto makeMinMaxInitValGenerator(bool isMax) {
-  return [isMax](fir::FirOpBuilder builder, mlir::Location loc,
-                 mlir::Type elementType) -> mlir::Value {
-    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
-      const llvm::fltSemantics &sem = ty.getFloatSemantics();
-      llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
-      return builder.createRealConstant(loc, elementType, limit);
-    }
-    unsigned bits = elementType.getIntOrFloatBitWidth();
-    int64_t limitInt =
-        isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
-              : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
-    return builder.createIntegerConstant(loc, elementType, limitInt);
-  };
-}
-
-mlir::Value generateMinMaxComparison(fir::FirOpBuilder builder,
-                                     mlir::Location loc, mlir::Value elem,
-                                     mlir::Value reduction, bool isMax) {
-  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
-    // For FP reductions we want the first smallest value to be used, that
-    // is not NaN. A OGL/OLT condition will usually work for this unless all
-    // the values are Nan or Inf. This follows the same logic as
-    // NumericCompare for Minloc/Maxlox in extrema.cpp.
-    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
-        loc,
-        isMax ? mlir::arith::CmpFPredicate::OGT
-              : mlir::arith::CmpFPredicate::OLT,
-        elem, reduction);
-    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
-        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
-  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
-    return builder.create<mlir::arith::CmpIOp>(
-        loc,
-        isMax ? mlir::arith::CmpIPredicate::sgt
-              : mlir::arith::CmpIPredicate::slt,
-        elem, reduction);
-  }
-  llvm_unreachable("unsupported type");
-}
-
-/// Given a reduction operation with an elemental/designate source, attempt to
-/// generate a do-loop to perform the operation inline.
-///   %e = hlfir.elemental %shape unordered
-///   %r = hlfir.count %e
-/// =>
-///   %r = for.do_loop %arg = 1 to bound(%shape) step 1 iter_args(%arg2 = init)
-///     %i = <inline elemental>
-///     %c = <reduce count> %i
-///     fir.result %c
-template <typename Op>
-class ReductionConversion : public mlir::OpRewritePattern<Op> {
-public:
-  using mlir::OpRewritePattern<Op>::OpRewritePattern;
-
-  llvm::LogicalResult
-  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
-    mlir::Location loc = op.getLoc();
-    // Select source and validate its arguments.
-    mlir::Value source;
-    bool valid = false;
-    if constexpr (std::is_same_v<Op, hlfir::AnyOp> ||
-                  std::is_same_v<Op, hlfir::AllOp> ||
-                  std::is_same_v<Op, hlfir::CountOp>) {
-      source = op.getMask();
-      valid = !op.getDim();
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
-                         std::is_same_v<Op, hlfir::MinvalOp>) {
-      source = op.getArray();
-      valid = !op.getDim() && !op.getMask();
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
-                         std::is_same_v<Op, hlfir::MinlocOp>) {
-      source = op.getArray();
-      valid = !op.getDim() && !op.getMask() && !op.getBack();
-    }
-    if (!valid)
-      return rewriter.notifyMatchFailure(
-          op, "Currently does not accept optional arguments");
-
-    hlfir::ElementalOp elemental;
-    hlfir::DesignateOp designate;
-    mlir::Value shape;
-    if ((elemental = source.template getDefiningOp<hlfir::ElementalOp>())) {
-      shape = elemental.getOperand(0);
-    } else if ((designate =
-                    source.template getDefiningOp<hlfir::DesignateOp>())) {
-      shape = designate.getShape();
-    } else {
-      return rewriter.notifyMatchFailure(op, "Did not find valid argument");
-    }
-
-    auto inlineSource =
-        [elemental,
-         &designate](fir::FirOpBuilder builder, mlir::Location loc,
-                     const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-        -> mlir::Value {
-      if (elemental) {
-        // Inline the elemental and get the value from it.
-        auto yield =
-            inlineElementalOp(loc, builder, elemental, oneBasedIndices);
-        auto tmp = yield.getElementValue();
-        yield->erase();
-        return tmp;
-      }
-      if (designate) {
-        // Create a designator over the array designator, then load the
-        // reference.
-        mlir::Value elementAddr = hlfir::getElementAt(
-            loc, builder, hlfir::Entity{designate.getResult()},
-            oneBasedIndices);
-        return builder.create<fir::LoadOp>(loc, elementAddr);
-      }
-      llvm_unreachable("unsupported type");
-    };
-
-    fir::FirOpBuilder builder{rewriter, op.getOperation()};
-
-    mlir::Value init;
-    GenBodyFn genBodyFn;
-    if constexpr (std::is_same_v<Op, hlfir::AnyOp>) {
-      init = builder.createIntegerConstant(loc, builder.getI1Type(), 0);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally set the reduction variable.
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        return builder.create<mlir::arith::OrIOp>(loc, reduction, cond);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::AllOp>) {
-      init = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally set the reduction variable.
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        return builder.create<mlir::arith::AndIOp>(loc, reduction, cond);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
-      init = builder.createIntegerConstant(loc, op.getType(), 0);
-      genBodyFn = [inlineSource](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        // Conditionally add one to the current value
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(),
-            inlineSource(builder, loc, oneBasedIndices));
-        mlir::Value one =
-            builder.createIntegerConstant(loc, reduction.getType(), 1);
-        mlir::Value add1 =
-            builder.create<mlir::arith::AddIOp>(loc, reduction, one);
-        return builder.create<mlir::arith::SelectOp>(loc, cond, add1,
-                                                     reduction);
-      };
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
-                         std::is_same_v<Op, hlfir::MinlocOp>) {
-      // TODO: implement minloc/maxloc conversion.
-      return rewriter.notifyMatchFailure(
-          op, "Currently minloc/maxloc is not handled");
-    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
-                         std::is_same_v<Op, hlfir::MinvalOp>) {
-      mlir::Type ty = op.getType();
-      if (!(mlir::isa<mlir::FloatType>(ty) ||
-            mlir::isa<mlir::IntegerType>(ty))) {
-        return rewriter.notifyMatchFailure(
-            op, "Type is not supported for Maxval or Minval yet");
-      }
-
-      bool isMax = std::is_same_v<Op, hlfir::MaxvalOp>;
-      init = makeMinMaxInitValGenerator(isMax)(builder, loc, ty);
-      genBodyFn = [inlineSource, isMax](
-                      fir::FirOpBuilder builder, mlir::Location loc,
-                      mlir::Value reduction,
-                      const llvm::SmallVectorImpl<mlir::Value> &oneBasedIndices)
-          -> mlir::Value {
-        mlir::Value val = inlineSource(builder, loc, oneBasedIndices);
-        mlir::Value cmp =
-            generateMinMaxComparison(builder, loc, val, reduction, isMax);
-        return builder.create<mlir::arith::SelectOp>(loc, cmp, val, reduction);
-      };
-    } else {
-      llvm_unreachable("unsupported type");
-    }
-
-    mlir::Value res =
-        generateReductionLoop(builder, loc, init, shape, genBodyFn);
-    if (res.getType() != op.getType())
-      res = builder.create<fir::ConvertOp>(loc, op.getType(), res);
-
-    // Check if the op was the only user of the source (apart from a destroy),
-    // and remove it if so.
-    mlir::Operation *sourceOp = source.getDefiningOp();
-    mlir::Operation::user_range srcUsers = sourceOp->getUsers();
-    hlfir::DestroyOp srcDestroy;
-    if (std::distance(srcUsers.begin(), srcUsers.end()) == 2) {
-      srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*srcUsers.begin());
-      if (!srcDestroy)
-        srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++srcUsers.begin());
-    }
-
-    rewriter.replaceOp(op, res);
-    if (srcDestroy) {
-      rewriter.eraseOp(srcDestroy);
-      rewriter.eraseOp(sourceOp);
-    }
-    return mlir::success();
-  }
-};
-
-// Look for minloc(mask=elemental) and generate the minloc loop with
-// inlined elemental.
-//  %e = hlfir.elemental %shape ({ ... })
-//  %m = hlfir.minloc %array mask %e
-template <typename Op>
-class ReductionMaskConversion : public mlir::OpRewritePattern<Op> {
-public:
-  using mlir::OpRewritePattern<Op>::OpRewritePattern;
-
-  llvm::LogicalResult
-  matchAndRewrite(Op mloc, mlir::PatternRewriter &rewriter) const override {
-    if (!mloc.getMask() || mloc.getDim() || mloc.getBack())
-      return rewriter.notifyMatchFailure(mloc,
-                                         "Did not find valid minloc/maxloc");
-
-    bool isMax = std::is_same_v<Op, hlfir::MaxlocOp>;
-
-    auto elemental =
-        mloc.getMask().template getDefiningOp<hlfir::ElementalOp>();
-    if (!elemental || hlfir::elementalOpMustProduceTemp(elemental))
-      return rewriter.notifyMatchFailure(mloc, "Did not find elemental");
-
-    mlir::Value array = mloc.getArray();
-
-    unsigned rank = mlir::cast<hlfir::ExprType>(mloc.getType()).getShape()[0];
-    mlir::Type arrayType = array.getType();
-    if (!mlir::isa<fir::BoxType>(arrayType))
-      return rewriter.notifyMatchFailure(
-          mloc, "Currently requires a boxed type input");
-    mlir::Type elementType = hlfir::getFortranElementType(arrayType);
-    if (!fir::isa_trivial(elementType))
-      return rewriter.notifyMatchFailure(
-          mloc, "Character arrays are currently not handled");
-
-    mlir::Location loc = mloc.getLoc();
-    fir::FirOpBuilder builder{rewriter, mloc.getOperation()};
-    mlir::Value resultArr = builder.createTemporary(
-        loc, fir::SequenceType::get(
-                 rank, hlfir::getFortranElementType(mloc.getType())));
-
-    auto init = makeMinMaxInitValGenerator(isMax);
-
-    auto genBodyOp =
-        [&rank, &resultArr, &elemental, isMax](
-            fir::FirOpBuilder builder, mlir::Location loc,
-            mlir::Type elementType, mlir::Value array, mlir::Value flagRef,
-            mlir::Value reduction,
-            const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {
-      // We are in the innermost loop: generate the elemental inline
-      mlir::Value oneIdx =
-          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-      llvm::SmallVector<mlir::Value> oneBasedIndices;
-      llvm::transform(
-          indices, std::back_inserter(oneBasedIndices), [&](mlir::Value V) {
-            return builder.create<mlir::arith::AddIOp>(loc, V, oneIdx);
-          });
-      hlfir::YieldElementOp yield =
-          hlfir::inlineElementalOp(loc, builder, elemental, oneBasedIndices);
-      mlir::Value maskElem = yield.getElementValue();
-      yield->erase();
-
-      mlir::Type ifCompatType = builder.getI1Type();
-      mlir::Value ifCompatElem =
-          builder.create<fir::ConvertOp>(loc, ifCompatType, maskElem);
-
-      llvm::SmallVector<mlir::Type> resultsTy = {elementType, elementType};
-      fir::IfOp maskIfOp =
-          builder.create<fir::IfOp>(loc, elementType, ifCompatElem,
-                                    /*withElseRegion=*/true);
-      builder.setInsertionPointToStart(&maskIfOp.getThenRegion().front());
-
-      // Set flag that mask was true at some point
-      mlir::Value flagSet = builder.createIntegerConstant(
-          loc, mlir::cast<fir::ReferenceType>(flagRef.getType()).getEleTy(), 1);
-      mlir::Value isFirst = builder.create<fir::LoadOp>(loc, flagRef);
-      mlir::Value addr = hlfir::getElementAt(loc, builder, hlfir::Entity{array},
-                                             oneBasedIndices);
-      mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
-
-      // Compare with the max reduction value
-      mlir::Value cmp =
-          generateMinMaxComparison(builder, loc, elem, reduction, isMax);
-
-      // The condition used for the loop is isFirst || <the condition above>.
-      isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
-      isFirst = builder.create<mlir::arith::XOrIOp>(
-          loc, isFirst, builder.createIntegerConstant(loc, cmp.getType(), 1));
-      cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, isFirst);
-
-      // Set the new coordinate to the result
-      fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, cmp,
-                                                 /*withElseRegion*/ true);
-
-      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-      builder.create<fir::StoreOp>(loc, flagSet, flagRef);
-      mlir::Type resultElemTy =
-          hlfir::getFortranElementType(resultArr.getType());
-      mlir::Type returnRefTy = builder.getRefType(resultElemTy);
-      mlir::IndexType idxTy = builder.getIndexType();
-
-      for (unsigned int i = 0; i < rank; ++i) {
-        mlir::Value index = builder.createIntegerConstant(loc, idxTy, i + 1);
-        mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
-            loc, returnRefTy, resultArr, index);
-        mlir::Value fortranIndex = builder.create<fir::ConvertOp>(
-            loc, resultElemTy, oneBasedIndices[i]);
-        builder.create<fir::StoreOp>(loc, fortranIndex, resultElemAddr);
-      }
-      builder.create<fir::ResultOp>(loc, elem);
-      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, reduction);
-      builder.setInsertionPointAfter(ifOp);
-
-      // Close the mask if
-      builder.create<fir::ResultOp>(loc, ifOp.getResult(0));
-      builder.setInsertionPointToStart(&maskIfOp.getElseRegion().front());
-      builder.create<fir::ResultOp>(loc, reduction);
-      builder.setInsertionPointAfter(maskIfOp);
-
-      return maskIfOp.getResult(0);
-    };
-    auto getAddrFn = [](fir::FirOpBuilder builder, mlir::Location loc,
-                        const mlir::Type &resultElemType, mlir::Value resultArr,
-                        mlir::Value index) {
-      mlir::Type resultRefTy = builder.getRefType(resultElemType);
-      mlir::Value oneIdx =
-          builder.createIntegerConstant(loc, builder.getIndexType(), 1);
-      index = builder.create<mlir::arith::AddIOp>(loc, index, oneIdx);
-      return builder.create<hlfir::DesignateOp>(loc, resultRefTy, resultArr,
-                                                index);
-    };
-
-    // Initialize the result
-    mlir::Type resultElemTy = hlfir::getFortranElementType(resultArr.getType());
-    mlir::Type resultRefTy = builder.getRefType(resultElemTy);
-    mlir::Value returnValue =
-        builder.createIntegerConstant(loc, resultElemTy, 0);
-    for (unsigned int i = 0; i < rank; ++i) {
-      mlir::Value index =
-          builder.createIntegerConstant(loc, builder.getIndexType(), i + 1);
-      mlir::Value resultElemAddr = builder.create<hlfir::DesignateOp>(
-          loc, resultRefTy, resultArr, index);
-      builder.create<fir::StoreOp>(loc, returnValue, resultElemAddr);
-    }
-
-    fir::genMinMaxlocReductionLoop(builder, array, init, genBodyOp, getAddrFn,
-                                   rank, elementType, loc, builder.getI1Type(),
-                                   resultArr, false);
-
-    mlir::Value asExpr = builder.create<hlfir::AsExprOp>(
-        loc, resultArr, builder.createBool(loc, false));
-
-    // Check all the users - the destroy is no longer required, and any assign
-    // can use resultArr directly so that InlineHLFIRAssign pass
-    // can optimize the results. Other operations are replaced with an AsExpr
-    // for the temporary resultArr.
-    llvm::SmallVector<hlfir::DestroyOp> destroys;
-    llvm::SmallVector<hlfir::AssignOp> assigns;
-    for (auto user : mloc->getUsers()) {
-      if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(user))
-        destroys.push_back(destroy);
-      else if (auto assign = mlir::dyn_cast<hlfir::AssignOp>(user))
-        assigns.push_back(assign);
-    }
-
-    // Check if the minloc/maxloc was the only user of the elemental (apart from
-    // a destroy), and remove it if so.
-    mlir::Operation::user_range elemUsers = elemental->getUsers();
-    hlfir::DestroyOp elemDestroy;
-    if (std::distance(elemUsers.begin(), elemUsers.end()) == 2) {
-      elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*elemUsers.begin());
-      if (!elemDestroy)
-        elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++elemUsers.begin());
-    }
-
-    for (auto d : destroys)
-      rewriter.eraseOp(d);
-    for (auto a : assigns)
-      a.setOperand(0, resultArr);
-    rewriter.replaceOp(mloc, asExpr);
-    if (elemDestroy) {
-      rewriter.eraseOp(elemDestroy);
-      rewriter.eraseOp(elemental);
-    }
-    return mlir::success();
-  }
-};
-
 class EvaluateIntoMemoryAssignBufferization
     : public mlir::OpRewritePattern<hlfir::EvaluateInMemoryOp> {
 
@@ -1340,19 +888,6 @@ class OptimizedBufferizationPass
     patterns.insert<ElementalAssignBufferization>(context);
     patterns.insert<BroadcastAssignBufferization>(context);
     patterns.insert<EvaluateIntoMemoryAssignBufferization>(context);
-    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
-    // TODO: implement basic minloc/maxloc conversion.
-    // patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
-    // patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
-    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
-    patterns.insert<ReductionMaskConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<ReductionMaskConversion<hlfir::MaxlocOp>>(context);
-    // TODO: implement masked minval/maxval conversion.
-    // patterns.insert<ReductionMaskConversion<hlfir::MaxvalOp>>(context);
-    // patterns.insert<ReductionMaskConversion<hlfir::MinvalOp>>(context);
 
     if (mlir::failed(mlir::applyPatternsGreedily(
             getOperation(), std::move(patterns), config))) {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index bac10121a881b..e9d820adbd22b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -173,245 +173,901 @@ class TransposeAsElementalConversion
   }
 };
 
-// Expand the SUM(DIM=CONSTANT) operation into .
-class SumAsElementalConversion : public mlir::OpRewritePattern<hlfir::SumOp> {
+/// Base class for converting reduction-like operations into
+/// a reduction loop[-nest] optionally wrapped into hlfir.elemental.
+/// It is used to handle operations produced for ALL, ANY, COUNT,
+/// MAXLOC, MAXVAL, MINLOC, MINVAL, SUM intrinsics.
+///
+/// All of these operations take an input array, and optional
+/// dim, mask arguments. ALL, ANY, COUNT do not have mask argument.
+class ReductionAsElementalConverter {
 public:
-  using mlir::OpRewritePattern<hlfir::SumOp>::OpRewritePattern;
+  ReductionAsElementalConverter(mlir::Operation *op,
+                                mlir::PatternRewriter &rewriter)
+      : op{op}, rewriter{rewriter}, loc{op->getLoc()}, builder{rewriter, op} {
+    assert(op->getNumResults() == 1);
+  }
+  virtual ~ReductionAsElementalConverter() {}
 
-  llvm::LogicalResult
-  matchAndRewrite(hlfir::SumOp sum,
-                  mlir::PatternRewriter &rewriter) const override {
-    hlfir::Entity array = hlfir::Entity{sum.getArray()};
-    bool isTotalReduction = hlfir::Entity{sum}.getRank() == 0;
-    mlir::Value dim = sum.getDim();
+  /// Do the actual conversion or return mlir::failure(),
+  /// if conversion is not possible.
+  mlir::LogicalResult convert();
+
+private:
+  // Return fir.shape specifying the shape of the result
+  // of a reduction with DIM=dimVal. The second return value
+  // is the extent of the DIM dimension.
+  std::tuple<mlir::Value, mlir::Value>
+  genResultShapeForPartialReduction(hlfir::Entity array, int64_t dimVal);
+
+  /// \p mask is a scalar or array logical mask.
+  /// If \p isPresentPred is not nullptr, it is a dynamic predicate value
+  /// identifying whether the mask's variable is present.
+  /// \p indices is a range of one-based indices to access \p mask
+  /// when it is an array.
+  ///
+  /// The method returns the scalar mask value to guard the access
+  /// to a single element of the input array.
+  mlir::Value genMaskValue(mlir::Value mask, mlir::Value isPresentPred,
+                           mlir::ValueRange indices);
+
+protected:
+  /// Return the input array.
+  virtual mlir::Value getSource() const = 0;
+
+  /// Return DIM or nullptr, if it is not present.
+  virtual mlir::Value getDim() const = 0;
+
+  /// Return MASK or nullptr, if it is not present.
+  virtual mlir::Value getMask() const { return nullptr; }
+
+  /// Return FastMathFlags attached to the operation
+  /// or arith::FastMathFlags::none, if the operation
+  /// does not support FastMathFlags (e.g. ALL, ANY, COUNT).
+  virtual mlir::arith::FastMathFlags getFastMath() const {
+    return mlir::arith::FastMathFlags::none;
+  }
+
+  /// Generates initial values for the reduction values used
+  /// by the reduction loop. In general, there is a single
+  /// loop-carried reduction value (e.g. for SUM), but, for example,
+  /// MAXLOC/MINLOC implementation uses multiple reductions.
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() = 0;
+
+  /// Perform reduction(s) update given a single input array's element
+  /// identified by \p array and \p oneBasedIndices coordinates.
+  /// \p currentValue specifies the current value(s) of the reduction(s)
+  /// inside the reduction loop body.
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) = 0;
+
+  /// Given reduction value(s) in \p reductionResults produced
+  /// by the reduction loop, apply any required updates and return
+  /// new reduction value(s) to be used after the reduction loop
+  /// (e.g. as the result yield of the wrapping hlfir.elemental).
+  /// NOTE: if the reduction loop is wrapped in hlfir.elemental,
+  /// the insertion point of any generated code is inside hlfir.elemental.
+  virtual hlfir::Entity
+  genFinalResult(const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
+    assert(reductionResults.size() == 1 &&
+           "default implementation of genFinalResult expect a single reduction "
+           "value");
+    return hlfir::Entity{reductionResults[0]};
+  }
+
+  /// Return mlir::success(), if the operation can be converted.
+  /// The default implementation always returns mlir::success().
+  /// The derived type may override the default implementation
+  /// with its own definition.
+  virtual mlir::LogicalResult isConvertible() const { return mlir::success(); }
+
+  // Default implementation of isTotalReduction() just checks
+  // if the result of the operation is a scalar.
+  // True result indicates that the reduction has to be done
+  // across all elements, false result indicates that
+  // the result is an array expression produced by an hlfir.elemental
+  // operation with a single reduction loop across the DIM dimension.
+  //
+  // MAXLOC/MINLOC must override this.
+  virtual bool isTotalReduction() const { return getResultRank() == 0; }
+
+  // Return true, if the reduction loop[-nest] may be unordered.
+  // In general, FP reductions may only be unordered when
+  // FastMathFlags::reassoc transformations are allowed.
+  //
+  // Some dervied types may need to override this.
+  virtual bool isUnordered() const {
+    mlir::Type elemType = getSourceElementType();
+    if (mlir::isa<mlir::IntegerType, fir::LogicalType, fir::CharacterType>(
+            elemType))
+      return true;
+    return static_cast<bool>(getFastMath() &
+                             mlir::arith::FastMathFlags::reassoc);
+  }
+
+  /// Return 0, if DIM is not present or its values does not matter
+  /// (for example, a reduction of 1D array does not care about
+  /// the DIM value, assuming that it is a valid program).
+  /// Return mlir::failure(), if DIM is a constant known
+  /// to be invalid for the given array.
+  /// Otherwise, return DIM constant value.
+  mlir::FailureOr<int64_t> getConstDim() const {
     int64_t dimVal = 0;
-    if (!isTotalReduction) {
+    if (!isTotalReduction()) {
       // In case of partial reduction we should ignore the operations
       // with invalid DIM values. They may appear in dead code
       // after constant propagation.
-      auto constDim = fir::getIntIfConstant(dim);
+      auto constDim = fir::getIntIfConstant(getDim());
       if (!constDim)
-        return rewriter.notifyMatchFailure(sum, "Nonconstant DIM for SUM");
+        return rewriter.notifyMatchFailure(op, "Nonconstant DIM");
       dimVal = *constDim;
 
-      if ((dimVal <= 0 || dimVal > array.getRank()))
-        return rewriter.notifyMatchFailure(
-            sum, "Invalid DIM for partial SUM reduction");
+      if ((dimVal <= 0 || dimVal > getSourceRank()))
+        return rewriter.notifyMatchFailure(op,
+                                           "Invalid DIM for partial reduction");
     }
+    return dimVal;
+  }
 
-    mlir::Location loc = sum.getLoc();
-    fir::FirOpBuilder builder{rewriter, sum.getOperation()};
-    mlir::Type elementType = hlfir::getFortranElementType(sum.getType());
-    mlir::Value mask = sum.getMask();
+  /// Return hlfir::Entity of the result.
+  hlfir::Entity getResultEntity() const {
+    return hlfir::Entity{op->getResult(0)};
+  }
 
-    mlir::Value resultShape, dimExtent;
-    llvm::SmallVector<mlir::Value> arrayExtents;
-    if (isTotalReduction)
-      arrayExtents = hlfir::genExtentsVector(loc, builder, array);
-    else
-      std::tie(resultShape, dimExtent) =
-          genResultShapeForPartialReduction(loc, builder, array, dimVal);
-
-    // If the mask is present and is a scalar, then we'd better load its value
-    // outside of the reduction loop making the loop unswitching easier.
-    mlir::Value isPresentPred, maskValue;
-    if (mask) {
-      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-        // MASK represented by a box might be dynamically optional,
-        // so we have to check for its presence before accessing it.
-        isPresentPred =
-            builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
-      }
+  /// Return type of the result (e.g. !hlfir.expr<?xi32>).
+  mlir::Type getResultType() const { return getResultEntity().getType(); }
 
-      if (hlfir::Entity{mask}.isScalar())
-        maskValue = genMaskValue(loc, builder, mask, isPresentPred, {});
-    }
+  /// Return the element type of the result (e.g. i32).
+  mlir::Type getResultElementType() const {
+    return hlfir::getFortranElementType(getResultType());
+  }
 
-    auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                         mlir::ValueRange inputIndices) -> hlfir::Entity {
-      // Loop over all indices in the DIM dimension, and reduce all values.
-      // If DIM is not present, do total reduction.
-
-      // Initial value for the reduction.
-      mlir::Value reductionInitValue =
-          fir::factory::createZeroValue(builder, loc, elementType);
-
-      // The reduction loop may be unordered if FastMathFlags::reassoc
-      // transformations are allowed. The integer reduction is always
-      // unordered.
-      bool isUnordered = mlir::isa<mlir::IntegerType>(elementType) ||
-                         static_cast<bool>(sum.getFastmath() &
-                                           mlir::arith::FastMathFlags::reassoc);
+  /// Return rank of the result.
+  unsigned getResultRank() const { return getResultEntity().getRank(); }
 
-      llvm::SmallVector<mlir::Value> extents;
-      if (isTotalReduction)
-        extents = arrayExtents;
-      else
-        extents.push_back(
-            builder.createConvert(loc, builder.getIndexType(), dimExtent));
+  /// Return the element type of the source.
+  mlir::Type getSourceElementType() const {
+    return hlfir::getFortranElementType(getSource().getType());
+  }
 
-      auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                         mlir::ValueRange oneBasedIndices,
-                         mlir::ValueRange reductionArgs)
-          -> llvm::SmallVector<mlir::Value, 1> {
-        // Generate the reduction loop-nest body.
-        // The initial reduction value in the innermost loop
-        // is passed via reductionArgs[0].
-        llvm::SmallVector<mlir::Value> indices;
-        if (isTotalReduction) {
-          indices = oneBasedIndices;
-        } else {
-          indices = inputIndices;
-          indices.insert(indices.begin() + dimVal - 1, oneBasedIndices[0]);
-        }
+  /// Return rank of the input array.
+  unsigned getSourceRank() const {
+    return hlfir::Entity{getSource()}.getRank();
+  }
 
-        mlir::Value reductionValue = reductionArgs[0];
-        fir::IfOp ifOp;
-        if (mask) {
-          // Make the reduction value update conditional on the value
-          // of the mask.
-          if (!maskValue) {
-            // If the mask is an array, use the elemental and the loop indices
-            // to address the proper mask element.
-            maskValue =
-                genMaskValue(loc, builder, mask, isPresentPred, indices);
-          }
-          mlir::Value isUnmasked = builder.create<fir::ConvertOp>(
-              loc, builder.getI1Type(), maskValue);
-          ifOp = builder.create<fir::IfOp>(loc, elementType, isUnmasked,
-                                           /*withElseRegion=*/true);
-          // In the 'else' block return the current reduction value.
-          builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-          builder.create<fir::ResultOp>(loc, reductionValue);
-
-          // In the 'then' block do the actual addition.
-          builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-        }
+  /// The reduction operation.
+  mlir::Operation *op;
 
-        hlfir::Entity element =
-            hlfir::getElementAt(loc, builder, array, indices);
-        hlfir::Entity elementValue =
-            hlfir::loadTrivialScalar(loc, builder, element);
-        // NOTE: we can use "Kahan summation" same way as the runtime
-        // (e.g. when fast-math is not allowed), but let's start with
-        // the simple version.
-        reductionValue =
-            genScalarAdd(loc, builder, reductionValue, elementValue);
-
-        if (ifOp) {
-          builder.create<fir::ResultOp>(loc, reductionValue);
-          builder.setInsertionPointAfter(ifOp);
-          reductionValue = ifOp.getResult(0);
-        }
+  mlir::PatternRewriter &rewriter;
+  mlir::Location loc;
+  fir::FirOpBuilder builder;
+};
 
-        return {reductionValue};
-      };
+/// Generate initialization value for MIN or MAX reduction
+/// of the given \p type.
+template <bool IS_MAX>
+static mlir::Value genMinMaxInitValue(mlir::Location loc,
+                                      fir::FirOpBuilder &builder,
+                                      mlir::Type type) {
+  if (auto ty = mlir::dyn_cast<mlir::FloatType>(type)) {
+    const llvm::fltSemantics &sem = ty.getFloatSemantics();
+    // We must not use +/-INF here. If the reduction input is empty,
+    // the result of reduction must be +/-LARGEST.
+    llvm::APFloat limit = llvm::APFloat::getLargest(sem, /*Negative=*/IS_MAX);
+    return builder.createRealConstant(loc, type, limit);
+  }
+  unsigned bits = type.getIntOrFloatBitWidth();
+  int64_t limitInt = IS_MAX
+                         ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
+                         : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
+  return builder.createIntegerConstant(loc, type, limitInt);
+}
+
+/// Generate a comparison of an array element value \p elem
+/// and the current reduction value \p reduction for MIN/MAX reduction.
+template <bool IS_MAX>
+static mlir::Value
+genMinMaxComparison(mlir::Location loc, fir::FirOpBuilder &builder,
+                    mlir::Value elem, mlir::Value reduction) {
+  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
+    // For FP reductions we want the first smallest value to be used, that
+    // is not NaN. A OGL/OLT condition will usually work for this unless all
+    // the values are Nan or Inf. This follows the same logic as
+    // NumericCompare for Minloc/Maxloc in extrema.cpp.
+    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
+        loc,
+        IS_MAX ? mlir::arith::CmpFPredicate::OGT
+               : mlir::arith::CmpFPredicate::OLT,
+        elem, reduction);
+    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
+    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
+    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
+    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
+    return builder.create<mlir::arith::CmpIOp>(
+        loc,
+        IS_MAX ? mlir::arith::CmpIPredicate::sgt
+               : mlir::arith::CmpIPredicate::slt,
+        elem, reduction);
+  }
+  llvm_unreachable("unsupported type");
+}
+
+/// Implementation of ReductionAsElementalConverter interface
+/// for MAXLOC/MINLOC.
+template <typename T>
+class MinMaxlocAsElementalConverter : public ReductionAsElementalConverter {
+  static_assert(std::is_same_v<T, hlfir::MaxlocOp> ||
+                std::is_same_v<T, hlfir::MinlocOp>);
+  static constexpr unsigned maxRank = Fortran::common::maxRank;
+  // We have the following reduction values in the reduction loop:
+  //   * N integer coordinates, where N is:
+  //     - RANK(ARRAY) for total reductions.
+  //     - 1 for partial reductions.
+  //   * 1 reduction value holding the current MIN/MAX.
+  //   * 1 boolean indicating whether it is the first time
+  //     the mask is true.
+  static constexpr unsigned maxNumReductions = Fortran::common::maxRank + 2;
+  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxlocOp>;
+  using Base = ReductionAsElementalConverter;
 
-      llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
-          hlfir::genLoopNestWithReductions(loc, builder, extents,
-                                           {reductionInitValue}, genBody,
-                                           isUnordered);
-      return hlfir::Entity{reductionFinalValues[0]};
-    };
+public:
+  MinMaxlocAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
 
-    if (isTotalReduction) {
-      hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
-      rewriter.replaceOp(sum, result);
-      return mlir::success();
-    }
+private:
+  virtual mlir::Value getSource() const final { return getOp().getArray(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+  virtual mlir::Value getMask() const final { return getOp().getMask(); }
+  virtual mlir::arith::FastMathFlags getFastMath() const final {
+    return getOp().getFastmath();
+  }
 
-    hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
-        loc, builder, elementType, resultShape, {}, genKernel,
-        /*isUnordered=*/true, /*polymorphicMold=*/nullptr,
-        sum.getResult().getType());
+  virtual mlir::LogicalResult isConvertible() const final {
+    if (getOp().getBack())
+      return rewriter.notifyMatchFailure(
+          getOp(), "BACK is not supported for MINLOC/MAXLOC inlining");
+    if (mlir::isa<fir::CharacterType>(getSourceElementType()))
+      return rewriter.notifyMatchFailure(
+          getOp(),
+          "CHARACTER type is not supported for MINLOC/MAXLOC inlining");
+    return mlir::success();
+  }
 
-    // it wouldn't be safe to replace block arguments with a different
-    // hlfir.expr type. Types can differ due to differing amounts of shape
-    // information
-    assert(elementalOp.getResult().getType() == sum.getResult().getType());
+  // If the result is scalar, then DIM does not matter,
+  // and this is a total reduction.
+  // If DIM is not present, this is a total reduction.
+  virtual bool isTotalReduction() const final {
+    return getResultRank() == 0 || !getDim();
+  }
 
-    rewriter.replaceOp(sum, elementalOp);
-    return mlir::success();
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final;
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array, mlir::ValueRange oneBasedIndices) final;
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) final;
+
+private:
+  T getOp() const { return mlir::cast<T>(op); }
+
+  unsigned getNumCoors() const {
+    return isTotalReduction() ? getSourceRank() : 1;
+  }
+
+  void
+  checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    assert(reductions.size() == getNumCoors() + 2 &&
+           "invalid number of reductions for MINLOC/MAXLOC");
   }
 
+  mlir::Value
+  getCurrentMinMax(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    checkReductions(reductions);
+    return reductions[getNumCoors()];
+  }
+
+  mlir::Value
+  getIsFirst(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    checkReductions(reductions);
+    return reductions[getNumCoors() + 1];
+  }
+};
+
+template <typename T>
+llvm::SmallVector<mlir::Value>
+MinMaxlocAsElementalConverter<T>::genReductionInitValues() {
+  // Initial value for the coordinate(s) is zero.
+  mlir::Value zeroCoor =
+      fir::factory::createZeroValue(builder, loc, getResultElementType());
+  llvm::SmallVector<mlir::Value> result(getNumCoors(), zeroCoor);
+
+  // Initial value for the MIN/MAX value.
+  mlir::Value minMaxInit =
+      genMinMaxInitValue<isMax>(loc, builder, getSourceElementType());
+  result.push_back(minMaxInit);
+
+  // Initial value for isFirst predicate. It is switched to false,
+  // when the reduction update dynamically happens inside the reduction
+  // loop.
+  mlir::Value trueVal = builder.createBool(loc, true);
+  result.push_back(trueVal);
+
+  return result;
+}
+
+template <typename T>
+llvm::SmallVector<mlir::Value>
+MinMaxlocAsElementalConverter<T>::reduceOneElement(
+    const llvm::SmallVectorImpl<mlir::Value> &currentValue, hlfir::Entity array,
+    mlir::ValueRange oneBasedIndices) {
+  checkReductions(currentValue);
+  hlfir::Entity elementValue =
+      hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+  mlir::Value cmp = genMinMaxComparison<isMax>(loc, builder, elementValue,
+                                               getCurrentMinMax(currentValue));
+  // If isFirst is true, then do the reduction update regardless
+  // of the FP comparison.
+  cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, getIsFirst(currentValue));
+
+  llvm::SmallVector<mlir::Value> newIndices;
+  int64_t dim = 1;
+  if (!isTotalReduction()) {
+    auto dimVal = getConstDim();
+    assert(mlir::succeeded(dimVal) &&
+           "partial MINLOC/MAXLOC reduction with invalid DIM");
+    dim = *dimVal;
+    assert(getNumCoors() == 1 &&
+           "partial MAXLOC/MINLOC reduction must compute one coordinate");
+  }
+
+  for (unsigned coorIdx = 0; coorIdx < getNumCoors(); ++coorIdx) {
+    mlir::Value currentCoor = currentValue[coorIdx];
+    mlir::Value newCoor = builder.createConvert(
+        loc, currentCoor.getType(), oneBasedIndices[coorIdx + dim - 1]);
+    mlir::Value update =
+        builder.create<mlir::arith::SelectOp>(loc, cmp, newCoor, currentCoor);
+    newIndices.push_back(update);
+  }
+
+  mlir::Value newMinMax = builder.create<mlir::arith::SelectOp>(
+      loc, cmp, elementValue, getCurrentMinMax(currentValue));
+  newIndices.push_back(newMinMax);
+
+  mlir::Value newIsFirst = builder.createBool(loc, false);
+  newIndices.push_back(newIsFirst);
+
+  assert(currentValue.size() == newIndices.size() &&
+         "invalid number of updated reductions");
+
+  return newIndices;
+}
+
+template <typename T>
+hlfir::Entity MinMaxlocAsElementalConverter<T>::genFinalResult(
+    const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {
+  // Identification of the final result of MINLOC/MAXLOC:
+  //   * If DIM is absent, the result is rank-one array.
+  //   * If DIM is present:
+  //     - The result is scalar for rank-one input.
+  //     - The result is an array of rank RANK(ARRAY)-1.
+  checkReductions(reductionResults);
+
+  // 16.9.137 & 16.9.143:
+  // The subscripts returned by MINLOC/MAXLOC are in the range
+  // 1 to the extent of the corresponding dimension.
+  mlir::Type indexType = builder.getIndexType();
+
+  // For partial reductions, the final result of the reduction
+  // loop is just a scalar - the coordinate within DIM dimension.
+  if (getResultRank() == 0 || !isTotalReduction()) {
+    // The result is a scalar, so just return the scalar.
+    assert(getNumCoors() == 1 &&
+           "unpexpected number of coordinates for scalar result");
+    return hlfir::Entity{reductionResults[0]};
+  }
+  // This is a total reduction, and there is no wrapping hlfir.elemental.
+  // We have to pack the reduced coordinates into a rank-one array.
+  unsigned rank = getSourceRank();
+  // TODO: in order to avoid introducing new memory effects
+  // we should not use a temporary in memory.
+  // We can use hlfir.elemental with a switch to pack all the coordinates
+  // into an array expression, or we can have a dedicated HLFIR operation
+  // for this.
+  mlir::Value tempArray = builder.createTemporary(
+      loc, fir::SequenceType::get(rank, getResultElementType()));
+  for (unsigned i = 0; i < rank; ++i) {
+    mlir::Value coor = reductionResults[i];
+    mlir::Value idx = builder.createIntegerConstant(loc, indexType, i + 1);
+    mlir::Value resultElement =
+        hlfir::getElementAt(loc, builder, hlfir::Entity{tempArray}, {idx});
+    builder.create<hlfir::AssignOp>(loc, coor, resultElement);
+  }
+  mlir::Value tempExpr = builder.create<hlfir::AsExprOp>(
+      loc, tempArray, builder.createBool(loc, false));
+  return hlfir::Entity{tempExpr};
+}
+
+/// Base class for numeric reductions like MAXVAl, MINVAL, SUM.
+template <typename OpT>
+class NumericReductionAsElementalConverterBase
+    : public ReductionAsElementalConverter {
+  using Base = ReductionAsElementalConverter;
+
+protected:
+  NumericReductionAsElementalConverterBase(OpT op,
+                                           mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
+
+  virtual mlir::Value getSource() const final { return getOp().getArray(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+  virtual mlir::Value getMask() const final { return getOp().getMask(); }
+  virtual mlir::arith::FastMathFlags getFastMath() const final {
+    return getOp().getFastmath();
+  }
+
+  OpT getOp() const { return mlir::cast<OpT>(op); }
+
+  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
+    assert(reductions.size() == 1 && "reduction must produce single value");
+  }
+};
+
+/// Reduction converter for MAXMAL/MINVAL.
+template <typename T>
+class MinMaxvalAsElementalConverter
+    : public NumericReductionAsElementalConverterBase<T> {
+  static_assert(std::is_same_v<T, hlfir::MaxvalOp> ||
+                std::is_same_v<T, hlfir::MinvalOp>);
+  // We have two reduction values:
+  //   * The current MIN/MAX value.
+  //   * 1 boolean indicating whether it is the first time
+  //     the mask is true.
+  //
+  // The boolean flag is used to replace the initial value
+  // with the first input element even if it is NaN.
+  static constexpr unsigned numReductions = 2;
+  static constexpr bool isMax = std::is_same_v<T, hlfir::MaxvalOp>;
+  using Base = NumericReductionAsElementalConverterBase<T>;
+
+public:
+  MinMaxvalAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
 private:
-  // Return fir.shape specifying the shape of the result
-  // of a SUM reduction with DIM=dimVal. The second return value
-  // is the extent of the DIM dimension.
-  static std::tuple<mlir::Value, mlir::Value>
-  genResultShapeForPartialReduction(mlir::Location loc,
-                                    fir::FirOpBuilder &builder,
-                                    hlfir::Entity array, int64_t dimVal) {
-    llvm::SmallVector<mlir::Value> inExtents =
-        hlfir::genExtentsVector(loc, builder, array);
-    assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
-           "DIM must be present and a positive constant not exceeding "
-           "the array's rank");
+  virtual mlir::LogicalResult isConvertible() const final {
+    if (mlir::isa<fir::CharacterType>(this->getSourceElementType()))
+      return this->rewriter.notifyMatchFailure(
+          this->getOp(),
+          "CHARACTER type is not supported for MINVAL/MAXVAL inlining");
+    return mlir::success();
+  }
+
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    llvm::SmallVector<mlir::Value> result;
+    fir::FirOpBuilder &builder = this->builder;
+    mlir::Location loc = this->loc;
+    mlir::Value init =
+        genMinMaxInitValue<isMax>(loc, builder, this->getResultElementType());
+    result.push_back(init);
+    // Initial value for isFirst predicate. It is switched to false,
+    // when the reduction update dynamically happens inside the reduction
+    // loop.
+    result.push_back(builder.createBool(loc, true));
+    return result;
+  }
+
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    this->checkReductions(currentValue);
+    llvm::SmallVector<mlir::Value> result;
+    fir::FirOpBuilder &builder = this->builder;
+    mlir::Location loc = this->loc;
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value currentMinMax = getCurrentMinMax(currentValue);
+    mlir::Value cmp =
+        genMinMaxComparison<isMax>(loc, builder, elementValue, currentMinMax);
+    cmp =
+        builder.create<mlir::arith::OrIOp>(loc, cmp, getIsFirst(currentValue));
+    mlir::Value newMinMax = builder.create<mlir::arith::SelectOp>(
+        loc, cmp, elementValue, currentMinMax);
+    result.push_back(newMinMax);
+    result.push_back(builder.createBool(loc, false));
+    return result;
+  }
+
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) final {
+    this->checkReductions(reductionResults);
+    return hlfir::Entity{getCurrentMinMax(reductionResults)};
+  }
+
+  void
+  checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    assert(reductions.size() == numReductions &&
+           "invalid number of reductions for MINVAL/MAXVAL");
+  }
 
-    mlir::Value dimExtent = inExtents[dimVal - 1];
-    inExtents.erase(inExtents.begin() + dimVal - 1);
-    return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
+  mlir::Value
+  getCurrentMinMax(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    this->checkReductions(reductions);
+    return reductions[0];
+  }
+
+  mlir::Value
+  getIsFirst(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {
+    this->checkReductions(reductions);
+    return reductions[1];
+  }
+};
+
+/// Reduction converter for SUM.
+class SumAsElementalConverter
+    : public NumericReductionAsElementalConverterBase<hlfir::SumOp> {
+  using Base = NumericReductionAsElementalConverterBase;
+
+public:
+  SumAsElementalConverter(hlfir::SumOp op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {
+        fir::factory::createZeroValue(builder, loc, getResultElementType())};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    checkReductions(currentValue);
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    // NOTE: we can use "Kahan summation" same way as the runtime
+    // (e.g. when fast-math is not allowed), but let's start with
+    // the simple version.
+    return {genScalarAdd(currentValue[0], elementValue)};
   }
 
   // Generate scalar addition of the two values (of the same data type).
-  static mlir::Value genScalarAdd(mlir::Location loc,
-                                  fir::FirOpBuilder &builder,
-                                  mlir::Value value1, mlir::Value value2) {
-    mlir::Type ty = value1.getType();
-    assert(ty == value2.getType() && "reduction values' types do not match");
-    if (mlir::isa<mlir::FloatType>(ty))
-      return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
-    else if (mlir::isa<mlir::ComplexType>(ty))
-      return builder.create<fir::AddcOp>(loc, value1, value2);
-    else if (mlir::isa<mlir::IntegerType>(ty))
-      return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
-
-    llvm_unreachable("unsupported SUM reduction type");
-  }
-
-  static mlir::Value genMaskValue(mlir::Location loc,
-                                  fir::FirOpBuilder &builder, mlir::Value mask,
-                                  mlir::Value isPresentPred,
-                                  mlir::ValueRange indices) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    fir::IfOp ifOp;
-    mlir::Type maskType =
-        hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
-    if (isPresentPred) {
-      ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
-                                       /*withElseRegion=*/true);
-
-      // Use 'true', if the mask is not present.
-      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-      mlir::Value trueValue = builder.createBool(loc, true);
-      trueValue = builder.createConvert(loc, maskType, trueValue);
-      builder.create<fir::ResultOp>(loc, trueValue);
-
-      // Load the mask value, if the mask is present.
-      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+  mlir::Value genScalarAdd(mlir::Value value1, mlir::Value value2);
+};
+
+/// Base class for logical reductions like ALL, ANY, COUNT.
+/// They do not have MASK and FastMathFlags.
+template <typename OpT>
+class LogicalReductionAsElementalConverterBase
+    : public ReductionAsElementalConverter {
+  using Base = ReductionAsElementalConverter;
+
+public:
+  LogicalReductionAsElementalConverterBase(OpT op,
+                                           mlir::PatternRewriter &rewriter)
+      : Base{op.getOperation(), rewriter} {}
+
+protected:
+  OpT getOp() const { return mlir::cast<OpT>(op); }
+
+  void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {
+    assert(reductions.size() == 1 && "reduction must produce single value");
+  }
+
+  virtual mlir::Value getSource() const final { return getOp().getMask(); }
+  virtual mlir::Value getDim() const final { return getOp().getDim(); }
+
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionResults) override {
+    checkReductions(reductionResults);
+    return hlfir::Entity{reductionResults[0]};
+  }
+};
+
+/// Reduction converter for ALL/ANY.
+template <typename T>
+class AllAnyAsElementalConverter
+    : public LogicalReductionAsElementalConverterBase<T> {
+  static_assert(std::is_same_v<T, hlfir::AllOp> ||
+                std::is_same_v<T, hlfir::AnyOp>);
+  static constexpr bool isAll = std::is_same_v<T, hlfir::AllOp>;
+  using Base = LogicalReductionAsElementalConverterBase<T>;
+
+public:
+  AllAnyAsElementalConverter(T op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {this->builder.createBool(this->loc, isAll ? true : false)};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    this->checkReductions(currentValue);
+    fir::FirOpBuilder &builder = this->builder;
+    mlir::Location loc = this->loc;
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value mask =
+        builder.createConvert(loc, builder.getI1Type(), elementValue);
+    if constexpr (isAll)
+      return {builder.create<mlir::arith::AndIOp>(loc, mask, currentValue[0])};
+    else
+      return {builder.create<mlir::arith::OrIOp>(loc, mask, currentValue[0])};
+  }
+
+  virtual hlfir::Entity genFinalResult(
+      const llvm::SmallVectorImpl<mlir::Value> &reductionValues) final {
+    this->checkReductions(reductionValues);
+    return hlfir::Entity{this->builder.createConvert(
+        this->loc, this->getResultElementType(), reductionValues[0])};
+  }
+};
+
+/// Reduction converter for COUNT.
+class CountAsElementalConverter
+    : public LogicalReductionAsElementalConverterBase<hlfir::CountOp> {
+  using Base = LogicalReductionAsElementalConverterBase<hlfir::CountOp>;
+
+public:
+  CountAsElementalConverter(hlfir::CountOp op, mlir::PatternRewriter &rewriter)
+      : Base{op, rewriter} {}
+
+private:
+  virtual llvm::SmallVector<mlir::Value> genReductionInitValues() final {
+    return {
+        fir::factory::createZeroValue(builder, loc, getResultElementType())};
+  }
+  virtual llvm::SmallVector<mlir::Value>
+  reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> &currentValue,
+                   hlfir::Entity array,
+                   mlir::ValueRange oneBasedIndices) final {
+    checkReductions(currentValue);
+    hlfir::Entity elementValue =
+        hlfir::loadElementAt(loc, builder, array, oneBasedIndices);
+    mlir::Value cond =
+        builder.createConvert(loc, builder.getI1Type(), elementValue);
+    mlir::Value one =
+        builder.createIntegerConstant(loc, getResultElementType(), 1);
+    mlir::Value add1 =
+        builder.create<mlir::arith::AddIOp>(loc, currentValue[0], one);
+    return {builder.create<mlir::arith::SelectOp>(loc, cond, add1,
+                                                  currentValue[0])};
+  }
+};
+
+mlir::LogicalResult ReductionAsElementalConverter::convert() {
+  mlir::LogicalResult canConvert(isConvertible());
+
+  if (mlir::failed(canConvert))
+    return canConvert;
+
+  hlfir::Entity array = hlfir::Entity{getSource()};
+  bool isTotalReduce = isTotalReduction();
+  auto dimVal = getConstDim();
+  if (mlir::failed(dimVal))
+    return dimVal;
+  mlir::Value mask = getMask();
+  mlir::Value resultShape, dimExtent;
+  llvm::SmallVector<mlir::Value> arrayExtents;
+  if (isTotalReduce)
+    arrayExtents = hlfir::genExtentsVector(loc, builder, array);
+  else
+    std::tie(resultShape, dimExtent) =
+        genResultShapeForPartialReduction(array, *dimVal);
+
+  // If the mask is present and is a scalar, then we'd better load its value
+  // outside of the reduction loop making the loop unswitching easier.
+  mlir::Value isPresentPred, maskValue;
+  if (mask) {
+    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+      // MASK represented by a box might be dynamically optional,
+      // so we have to check for its presence before accessing it.
+      isPresentPred =
+          builder.create<fir::IsPresentOp>(loc, builder.getI1Type(), mask);
     }
 
-    hlfir::Entity maskVar{mask};
-    if (maskVar.isScalar()) {
-      if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
-        // MASK may be a boxed scalar.
-        mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
-        mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
+    if (hlfir::Entity{mask}.isScalar())
+      maskValue = genMaskValue(mask, isPresentPred, {});
+  }
+
+  auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                       mlir::ValueRange inputIndices) -> hlfir::Entity {
+    // Loop over all indices in the DIM dimension, and reduce all values.
+    // If DIM is not present, do total reduction.
+
+    // Initial value for the reduction.
+    llvm::SmallVector<mlir::Value, 1> reductionInitValues =
+        genReductionInitValues();
+
+    llvm::SmallVector<mlir::Value> extents;
+    if (isTotalReduce)
+      extents = arrayExtents;
+    else
+      extents.push_back(
+          builder.createConvert(loc, builder.getIndexType(), dimExtent));
+
+    auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                       mlir::ValueRange oneBasedIndices,
+                       mlir::ValueRange reductionArgs)
+        -> llvm::SmallVector<mlir::Value, 1> {
+      // Generate the reduction loop-nest body.
+      // The initial reduction value in the innermost loop
+      // is passed via reductionArgs[0].
+      llvm::SmallVector<mlir::Value> indices;
+      if (isTotalReduce) {
+        indices = oneBasedIndices;
       } else {
-        mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
+        indices = inputIndices;
+        indices.insert(indices.begin() + *dimVal - 1, oneBasedIndices[0]);
       }
+
+      llvm::SmallVector<mlir::Value, 1> reductionValues = reductionArgs;
+      llvm::SmallVector<mlir::Type, 1> reductionTypes;
+      llvm::transform(reductionValues, std::back_inserter(reductionTypes),
+                      [](mlir::Value v) { return v.getType(); });
+      fir::IfOp ifOp;
+      if (mask) {
+        // Make the reduction value update conditional on the value
+        // of the mask.
+        if (!maskValue) {
+          // If the mask is an array, use the elemental and the loop indices
+          // to address the proper mask element.
+          maskValue = genMaskValue(mask, isPresentPred, indices);
+        }
+        mlir::Value isUnmasked =
+            builder.create<fir::ConvertOp>(loc, builder.getI1Type(), maskValue);
+        ifOp = builder.create<fir::IfOp>(loc, reductionTypes, isUnmasked,
+                                         /*withElseRegion=*/true);
+        // In the 'else' block return the current reduction value.
+        builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+        builder.create<fir::ResultOp>(loc, reductionValues);
+
+        // In the 'then' block do the actual addition.
+        builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+      }
+      reductionValues = reduceOneElement(reductionValues, array, indices);
+      if (ifOp) {
+        builder.create<fir::ResultOp>(loc, reductionValues);
+        builder.setInsertionPointAfter(ifOp);
+        reductionValues = ifOp.getResults();
+      }
+
+      return reductionValues;
+    };
+
+    llvm::SmallVector<mlir::Value, 1> reductionFinalValues =
+        hlfir::genLoopNestWithReductions(
+            loc, builder, extents, reductionInitValues, genBody, isUnordered());
+    return genFinalResult(reductionFinalValues);
+  };
+
+  if (isTotalReduce) {
+    hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});
+    rewriter.replaceOp(op, result);
+    return mlir::success();
+  }
+
+  hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
+      loc, builder, getResultElementType(), resultShape, /*typeParams=*/{},
+      genKernel,
+      /*isUnordered=*/true, /*polymorphicMold=*/nullptr, getResultType());
+
+  // it wouldn't be safe to replace block arguments with a different
+  // hlfir.expr type. Types can differ due to differing amounts of shape
+  // information
+  assert(elementalOp.getResult().getType() == op->getResult(0).getType());
+
+  rewriter.replaceOp(op, elementalOp);
+  return mlir::success();
+}
+
+std::tuple<mlir::Value, mlir::Value>
+ReductionAsElementalConverter::genResultShapeForPartialReduction(
+    hlfir::Entity array, int64_t dimVal) {
+  llvm::SmallVector<mlir::Value> inExtents =
+      hlfir::genExtentsVector(loc, builder, array);
+  assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&
+         "DIM must be present and a positive constant not exceeding "
+         "the array's rank");
+
+  mlir::Value dimExtent = inExtents[dimVal - 1];
+  inExtents.erase(inExtents.begin() + dimVal - 1);
+  return {builder.create<fir::ShapeOp>(loc, inExtents), dimExtent};
+}
+
+mlir::Value SumAsElementalConverter::genScalarAdd(mlir::Value value1,
+                                                  mlir::Value value2) {
+  mlir::Type ty = value1.getType();
+  assert(ty == value2.getType() && "reduction values' types do not match");
+  if (mlir::isa<mlir::FloatType>(ty))
+    return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
+  else if (mlir::isa<mlir::ComplexType>(ty))
+    return builder.create<fir::AddcOp>(loc, value1, value2);
+  else if (mlir::isa<mlir::IntegerType>(ty))
+    return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
+
+  llvm_unreachable("unsupported SUM reduction type");
+}
+
+mlir::Value ReductionAsElementalConverter::genMaskValue(
+    mlir::Value mask, mlir::Value isPresentPred, mlir::ValueRange indices) {
+  mlir::OpBuilder::InsertionGuard guard(builder);
+  fir::IfOp ifOp;
+  mlir::Type maskType =
+      hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));
+  if (isPresentPred) {
+    ifOp = builder.create<fir::IfOp>(loc, maskType, isPresentPred,
+                                     /*withElseRegion=*/true);
+
+    // Use 'true', if the mask is not present.
+    builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+    mlir::Value trueValue = builder.createBool(loc, true);
+    trueValue = builder.createConvert(loc, maskType, trueValue);
+    builder.create<fir::ResultOp>(loc, trueValue);
+
+    // Load the mask value, if the mask is present.
+    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+  }
+
+  hlfir::Entity maskVar{mask};
+  if (maskVar.isScalar()) {
+    if (mlir::isa<fir::BaseBoxType>(mask.getType())) {
+      // MASK may be a boxed scalar.
+      mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);
+      mask = builder.create<fir::LoadOp>(loc, hlfir::Entity{addr});
     } else {
-      // Load from the mask array.
-      assert(!indices.empty() && "no indices for addressing the mask array");
-      maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
       mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
     }
+  } else {
+    // Load from the mask array.
+    assert(!indices.empty() && "no indices for addressing the mask array");
+    maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);
+    mask = hlfir::loadTrivialScalar(loc, builder, maskVar);
+  }
 
-    if (!isPresentPred)
-      return mask;
+  if (!isPresentPred)
+    return mask;
 
-    builder.create<fir::ResultOp>(loc, mask);
-    return ifOp.getResult(0);
+  builder.create<fir::ResultOp>(loc, mask);
+  return ifOp.getResult(0);
+}
+
+/// Convert an operation that is a partial or total reduction
+/// over an array of values into a reduction loop[-nest]
+/// optionally wrapped into hlfir.elemental.
+template <typename Op>
+class ReductionConversion : public mlir::OpRewritePattern<Op> {
+public:
+  using mlir::OpRewritePattern<Op>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
+    if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                  std::is_same_v<Op, hlfir::MinlocOp>) {
+      MinMaxlocAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      MinMaxvalAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
+      CountAsElementalConverter converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::AllOp> ||
+                         std::is_same_v<Op, hlfir::AnyOp>) {
+      AllAnyAsElementalConverter<Op> converter(op, rewriter);
+      return converter.convert();
+    } else if constexpr (std::is_same_v<Op, hlfir::SumOp>) {
+      SumAsElementalConverter converter{op, rewriter};
+      return converter.convert();
+    }
+    return rewriter.notifyMatchFailure(op, "unexpected reduction operation");
   }
 };
 
@@ -1481,10 +2137,18 @@ class SimplifyHLFIRIntrinsics
 
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
-    patterns.insert<SumAsElementalConversion>(context);
+    patterns.insert<ReductionConversion<hlfir::SumOp>>(context);
     patterns.insert<CShiftConversion>(context);
     patterns.insert<MatmulConversion<hlfir::MatmulTransposeOp>>(context);
 
+    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
+
     // If forceMatmulAsElemental is false, then hlfir.matmul inlining
     // will introduce hlfir.eval_in_mem operation with new memory side
     // effects. This conflicts with CSE and optimized bufferization, e.g.:
diff --git a/flang/test/HLFIR/all-elemental.fir b/flang/test/HLFIR/all-elemental.fir
deleted file mode 100644
index 1ba8bb1b7a5fb..0000000000000
--- a/flang/test/HLFIR/all-elemental.fir
+++ /dev/null
@@ -1,91 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.all %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
-  return %13 : !fir.logical<4>
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-// CHECK-NEXT:     %true = arith.constant true
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %true) -> (i1) {
-// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.andi %arg4, %[[V16]] : i1
-// CHECK-NEXT:       fir.result %[[V17]] : i1
-// CHECK-NEXT:     }
-// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
-// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
-
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
-  %c2_i32 = arith.constant 2 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.all %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
-  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
-  return %11 : !fir.array<4x!fir.logical<4>>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: %10 = hlfir.all %9 dim %c2_i32
\ No newline at end of file
diff --git a/flang/test/HLFIR/any-elemental.fir b/flang/test/HLFIR/any-elemental.fir
deleted file mode 100644
index a7c559679d965..0000000000000
--- a/flang/test/HLFIR/any-elemental.fir
+++ /dev/null
@@ -1,190 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.logical<4> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.any %11 : (!hlfir.expr<7x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %12 to %4#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<!fir.logical<4>>
-  return %13 : !fir.logical<4>
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.logical<4> {
-// CHECK-NEXT:     %false = arith.constant false
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca !fir.logical<4>
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %false) -> (i1) {
-// CHECK-NEXT:       %[[V14:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = fir.load %[[V14]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V16:.*]] = arith.cmpi sge, %[[V15]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.ori %arg4, %[[V16]] : i1
-// CHECK-NEXT:       fir.result %[[V17]] : i1
-// CHECK-NEXT:     }
-// CHECK-NEXT:     %[[V12:.*]] = fir.convert %[[V11]] : (i1) -> !fir.logical<4>
-// CHECK-NEXT:     hlfir.assign %[[V12]] to %[[V4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     %[[V13:.*]] = fir.load %[[V4]]#1 : !fir.ref<!fir.logical<4>>
-// CHECK-NEXT:     return %[[V13]] : !fir.logical<4>
-
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<4x!fir.logical<4>> {
-  %c2_i32 = arith.constant 2 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<4x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4x!fir.logical<4>>>, !fir.ref<!fir.array<4x!fir.logical<4>>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.any %9 dim %c2_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<4x!fir.logical<4>>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<4x!fir.logical<4>>, !fir.ref<!fir.array<4x!fir.logical<4>>>
-  hlfir.destroy %10 : !hlfir.expr<4x!fir.logical<4>>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<4x!fir.logical<4>>>
-  return %11 : !fir.array<4x!fir.logical<4>>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: {{.*}} = hlfir.any {{.*}} dim %c2_i32
-
-
-func.func @_Qtest_recursive() attributes {fir.bindc_name = "test"} {
-  %c1 = arith.constant 1 : index
-  %true = arith.constant true
-  %false = arith.constant false
-  %c0_i64 = arith.constant 0 : i64
-  %c2_i32 = arith.constant 2 : i32
-  %c0 = arith.constant 0 : index
-  %c1_i32 = arith.constant 1 : i32
-  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  %1:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
-  %2 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
-  %5:2 = hlfir.declare %4 {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "ra", uniq_name = "_QFEra"}
-  %7 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %8:2 = hlfir.declare %6(%7) {uniq_name = "_QFEra"} : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.ref<!fir.array<1x!fir.logical<4>>>)
-  %9 = fir.alloca !fir.logical<4> {bindc_name = "rs", uniq_name = "_QFErs"}
-  %10:2 = hlfir.declare %9 {uniq_name = "_QFErs"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  %11 = fir.allocmem !fir.array<?xi32>, %c1 {fir.must_be_heap = true, uniq_name = "_QFEa.alloc"}
-  %12 = fir.embox %11(%7) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
-  fir.store %12 to %1#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  hlfir.assign %c1_i32 to %5#0 : i32, !fir.ref<i32>
-  %13 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  %14:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-  fir.do_loop %arg0 = %c1 to %14#1 step %c1 unordered {
-    %27:3 = fir.box_dims %13, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-    %28 = arith.subi %27#0, %c1 : index
-    %29 = arith.addi %arg0, %28 : index
-    %30 = hlfir.designate %13 (%29)  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
-    hlfir.assign %c2_i32 to %30 : i32, !fir.ref<i32>
-  }
-  %15 = fir.load %5#0 : !fir.ref<i32>
-  %16 = fir.convert %15 : (i32) -> i64
-  %17 = arith.cmpi sgt, %16, %c0_i64 : i64
-  %18 = arith.select %17, %16, %c0_i64 : i64
-  %19 = fir.convert %18 : (i64) -> index
-  %20 = fir.shape %19 : (index) -> !fir.shape<1>
-  %21 = hlfir.elemental %20 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg0: index):
-    %27 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    %28:3 = fir.box_dims %27, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-    %29 = arith.addi %28#0, %28#1 : index
-    %30 = arith.subi %29, %c1 : index
-    %31 = arith.subi %30, %28#0 : index
-    %32 = arith.addi %31, %c1 : index
-    %33 = arith.cmpi sgt, %32, %c0 : index
-    %34 = arith.select %33, %32, %c0 : index
-    %35 = fir.shape %34 : (index) -> !fir.shape<1>
-    %36 = hlfir.designate %27 (%28#0:%30:%c1)  shape %35 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-    %37 = hlfir.elemental %35 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-    ^bb0(%arg1: index):
-      %39 = hlfir.designate %36 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-      %40 = fir.load %39 : !fir.ref<i32>
-      %41 = arith.cmpi eq, %40, %c1_i32 : i32
-      %42 = fir.convert %41 : (i1) -> !fir.logical<4>
-      hlfir.yield_element %42 : !fir.logical<4>
-    }
-    %38 = hlfir.any %37 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
-    hlfir.destroy %37 : !hlfir.expr<?x!fir.logical<4>>
-    hlfir.yield_element %38 : !fir.logical<4>
-  }
-  %22 = hlfir.any %21 : (!hlfir.expr<?x!fir.logical<4>>) -> !fir.logical<4>
-  hlfir.assign %22 to %10#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-  hlfir.destroy %21 : !hlfir.expr<?x!fir.logical<4>>
-  %23 = fir.load %10#0 : !fir.ref<!fir.logical<4>>
-  %24 = fir.convert %23 : (!fir.logical<4>) -> i1
-  %25 = arith.xori %24, %true : i1
-  cf.cond_br %25, ^bb1, ^bb2
-^bb1:  // pred: ^bb0
-  fir.call @_FortranAStopStatement(%c2_i32, %false, %false) fastmath<contract> : (i32, i1, i1) -> ()
-  fir.unreachable
-^bb2:  // pred: ^bb0
-  return
-}
-// CHECK-LABEL: func.func @_Qtest_recursive()
-// CHECK:    %[[V20:.*]] = fir.do_loop %arg0 = %c1 to %{{.*}} step %c1 iter_args(%arg1 = %false) -> (i1) {
-// CHECK:      %[[V26:.*]] = fir.load %[[V1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-// CHECK:      %[[V27:.*]]:3 = fir.box_dims %[[V26]], %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-// CHECK:      %[[V28:.*]] = arith.addi %[[V27]]#0, %[[V27]]#1 : index
-// CHECK:      %[[V29:.*]] = arith.subi %[[V28]], %c1 : index
-// CHECK:      %[[V30:.*]] = arith.subi %[[V29]], %[[V27]]#0 : index
-// CHECK:      %[[V31:.*]] = arith.addi %[[V30]], %c1 : index
-// CHECK:      %[[V32:.*]] = arith.cmpi sgt, %[[V31]], %c0 : index
-// CHECK:      %[[V33:.*]] = arith.select %[[V32]], %[[V31]], %c0 : index
-// CHECK:      %[[V34:.*]] = fir.shape %[[V33]] : (index) -> !fir.shape<1>
-// CHECK:      %[[V35:.*]] = hlfir.designate %[[V26]] (%[[V27]]#0:%[[V29]]:%c1)  shape %[[V34]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-// CHECK:      %[[V36:.*]] = fir.do_loop %arg2 = %c1 to %[[V33]] step %c1 iter_args(%arg3 = %false) -> (i1) {
-// CHECK:        %[[V38:.*]] = hlfir.designate %[[V35]] (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:        %[[V39:.*]] = fir.load %[[V38]] : !fir.ref<i32>
-// CHECK:        %[[V40:.*]] = arith.cmpi eq, %[[V39]], %c1_i32 : i32
-// CHECK:        %[[V41:.*]] = arith.ori %arg3, %[[V40]] : i1
-// CHECK:        fir.result %[[V41]] : i1
-// CHECK:      }
-// CHECK:      %[[V37:.*]] = arith.ori %arg1, %[[V36]] : i1
-// CHECK:      fir.result %[[V37]] : i1
-// CHECK:    }
diff --git a/flang/test/HLFIR/count-elemental.fir b/flang/test/HLFIR/count-elemental.fir
deleted file mode 100644
index 0df5cc3c031ea..0000000000000
--- a/flang/test/HLFIR/count-elemental.fir
+++ /dev/null
@@ -1,314 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %12 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<i32>
-  return %13 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
-// CHECK-NEXT:       fir.result %[[V17]] : i32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
-// CHECK-NEXT:     return %[[V12]] : i32
-
-func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i16 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %14 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %15 = fir.load %14 : !fir.ref<i32>
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %12 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i16
-  hlfir.assign %12 to %4#0 : i16, !fir.ref<i16>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %13 = fir.load %4#1 : !fir.ref<i16>
-  return %13 : i16
-}
-// CHECK-LABEL:  func.func @_QFPtest_kind2(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i16 {
-// CHECK-NEXT:     %c1_i16 = arith.constant 1 : i16
-// CHECK-NEXT:     %c0_i16 = arith.constant 0 : i16
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]])
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i16
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]]
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2
-// CHECK-NEXT:     %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V7:.*]] = fir.convert %[[V6]] : (i32) -> i64
-// CHECK-NEXT:     %[[V8:.*]] = fir.shape %c7 : (index) -> !fir.shape<1>
-// CHECK-NEXT:     %[[V9:.*]] = hlfir.designate %[[V1]]#0 (%[[V7]], %c1:%c7:%c1)  shape %[[V8]] : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-// CHECK-NEXT:     %[[V10:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i16) -> (i16) {
-// CHECK-NEXT:       %[[V13:.*]] = hlfir.designate %[[V9]] (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:       %[[V14:.*]] = fir.load %[[V13]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK-NEXT:       %[[V16:.*]] = arith.addi %arg4, %c1_i16 : i16
-// CHECK-NEXT:       %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i16
-// CHECK-NEXT:       fir.result %[[V17]] : i16
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V11]] to %[[V4]]#0 : i16, !fir.ref<i16>
-// CHECK-NEXT:     %[[V12:.*]] = fir.load %[[V4]]#1 : !fir.ref<i16>
-// CHECK-NEXT:     return %[[V12]] : i16
-
-func.func @_QFPtest_dim(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> !fir.array<7xi32> {
-  %c1_i32 = arith.constant 1 : i32
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca !fir.array<7xi32> {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %5:2 = hlfir.declare %3(%4) {uniq_name = "_QFFtestEtest"} : (!fir.ref<!fir.array<7xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<7xi32>>, !fir.ref<!fir.array<7xi32>>)
-  %6:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1)  shape %0 : (!fir.ref<!fir.array<4x7xi32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.ref<!fir.array<4x7xi32>>
-  %8 = fir.load %6#0 : !fir.ref<i32>
-  %9 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<4x7x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %12 = hlfir.designate %7 (%arg3, %arg4)  : (!fir.ref<!fir.array<4x7xi32>>, index, index) -> !fir.ref<i32>
-    %13 = fir.load %12 : !fir.ref<i32>
-    %14 = arith.cmpi sge, %13, %8 : i32
-    %15 = fir.convert %14 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %15 : !fir.logical<4>
-  }
-  %10 = hlfir.count %9 dim %c1_i32 : (!hlfir.expr<4x7x!fir.logical<4>>, i32) -> !hlfir.expr<7xi32>
-  hlfir.assign %10 to %5#0 : !hlfir.expr<7xi32>, !fir.ref<!fir.array<7xi32>>
-  hlfir.destroy %10 : !hlfir.expr<7xi32>
-  hlfir.destroy %9 : !hlfir.expr<4x7x!fir.logical<4>>
-  %11 = fir.load %5#1 : !fir.ref<!fir.array<7xi32>>
-  return %11 : !fir.array<7xi32>
-}
-// CHECK-LABEL:  func.func @_QFPtest_dim(
-// CHECK: %{{.*}} = hlfir.count %{{.*}} dim %c1_i32
-
-
-func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %c2 = arith.constant 2 : index
-  %0 = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = hlfir.designate %1#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %0 : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
-  %7 = fir.load %5#0 : !fir.ref<i32>
-  %8 = hlfir.elemental %0 unordered : (!fir.shape<3>) -> !hlfir.expr<4x7x2x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index, %arg5: index):
-    %11 = hlfir.designate %6 (%arg3, %arg4, %arg5)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
-    %12 = fir.load %11 : !fir.ref<i32>
-    %13 = arith.cmpi sge, %12, %7 : i32
-    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %14 : !fir.logical<4>
-  }
-  %9 = hlfir.count %8 : (!hlfir.expr<4x7x2x!fir.logical<4>>) -> i32
-  hlfir.assign %9 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %8 : !hlfir.expr<4x7x2x!fir.logical<4>>
-  %10 = fir.load %4#1 : !fir.ref<i32>
-  return %10 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_multi(%arg0: !fir.ref<!fir.array<4x7x2xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK-NEXT:     %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:     %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:     %c1 = arith.constant 1 : index
-// CHECK-NEXT:     %c4 = arith.constant 4 : index
-// CHECK-NEXT:     %c7 = arith.constant 7 : index
-// CHECK-NEXT:     %c2 = arith.constant 2 : index
-// CHECK-NEXT:     %[[V0:.*]] = fir.shape %c4, %c7, %c2 : (index, index, index) -> !fir.shape<3>
-// CHECK-NEXT:     %[[V1:.*]]:2 = hlfir.declare %arg0(%[[V0]]) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<4x7x2xi32>>, !fir.ref<!fir.array<4x7x2xi32>>)
-// CHECK-NEXT:     %[[V2:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V3:.*]] = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-// CHECK-NEXT:     %[[V4:.*]]:2 = hlfir.declare %[[V3]] {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V5:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:     %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%c1:%c4:%c1, %c1:%c7:%c1, %c1:%c2:%c1)  shape %[[V0]] : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.ref<!fir.array<4x7x2xi32>>
-// CHECK-NEXT:     %[[V7:.*]] = fir.load %[[V5]]#0 : !fir.ref<i32>
-// CHECK-NEXT:     %[[V8:.*]] = fir.do_loop %arg3 = %c1 to %c2 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK-NEXT:       %[[V10:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %arg4) -> (i32) {
-// CHECK-NEXT:         %[[V11:.*]] = fir.do_loop %arg7 = %c1 to %c4 step %c1 iter_args(%arg8 = %arg6) -> (i32) {
-// CHECK-NEXT:           %[[V12:.*]] = hlfir.designate %[[V6]] (%arg7, %arg5, %arg3)  : (!fir.ref<!fir.array<4x7x2xi32>>, index, index, index) -> !fir.ref<i32>
-// CHECK-NEXT:           %[[V13:.*]] = fir.load %[[V12]] : !fir.ref<i32>
-// CHECK-NEXT:           %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V7]] : i32
-// CHECK-NEXT:           %[[V15:.*]] = arith.addi %arg8, %c1_i32 : i32
-// CHECK-NEXT:           %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg8 : i32
-// CHECK-NEXT:           fir.result %[[V16]] : i32
-// CHECK-NEXT:         }
-// CHECK-NEXT:         fir.result %[[V11]] : i32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V10]] : i32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     hlfir.assign %[[V8]] to %[[V4]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:     %[[V9:.*]] = fir.load %[[V4]]#1 : !fir.ref<i32>
-// CHECK-NEXT:     return %[[V9]] : i32
-
-
-
-
-
-func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7xi32> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %16 = fir.load %15 : !fir.ref<i32>
-    hlfir.yield_element %16 : i32
-  }
-  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.sum %11 : (!hlfir.expr<7xi32>) -> i32
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7xi32>
-  %14 = fir.load %4#1 : !fir.ref<i32>
-  return %14 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_rec_sum(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK:    %[[V12:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK:      %[[V14:.*]] = hlfir.sum %[[V11]] : (!hlfir.expr<7xi32>) -> i32
-// CHECK:      %[[V15:.*]] = arith.cmpi sge, %[[V14]], %[[V10]] : i32
-// CHECK:      %[[V16:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK:      %[[V17:.*]] = arith.select %[[V15]], %[[V16]], %arg4 : i32
-// CHECK:      fir.result %[[V17]] : i32
-// CHECK:    }
-
-
-
-
-func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c1 = arith.constant 1 : index
-  %c4 = arith.constant 4 : index
-  %c7 = arith.constant 7 : index
-  %0 = fir.shape %c4, %c7 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFFtestEb"} : (!fir.ref<!fir.array<4x7xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<4x7xi32>>, !fir.ref<!fir.array<4x7xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestErow"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %4:2 = hlfir.declare %3 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %5:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %6 = fir.load %2#0 : !fir.ref<i32>
-  %7 = fir.convert %6 : (i32) -> i64
-  %8 = fir.shape %c7 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %1#0 (%7, %c1:%c7:%c1)  shape %8 : (!fir.ref<!fir.array<4x7xi32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<7xi32>>
-  %10 = fir.load %5#0 : !fir.ref<i32>
-  %11 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.designate %9 (%arg3)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-    %16 = fir.load %15 : !fir.ref<i32>
-    %17 = arith.cmpi sge, %16, %10 : i32
-    %18 = fir.convert %17 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %18 : !fir.logical<4>
-  }
-  %12 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<7x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %15 = hlfir.count %11 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-    %16 = arith.cmpi sge, %15, %10 : i32
-    %17 = fir.convert %16 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %17 : !fir.logical<4>
-  }
-  %13 = hlfir.count %12 : (!hlfir.expr<7x!fir.logical<4>>) -> i32
-  hlfir.assign %13 to %4#0 : i32, !fir.ref<i32>
-  hlfir.destroy %12 : !hlfir.expr<7x!fir.logical<4>>
-  hlfir.destroy %11 : !hlfir.expr<7x!fir.logical<4>>
-  %14 = fir.load %4#1 : !fir.ref<i32>
-  return %14 : i32
-}
-// CHECK-LABEL:  func.func @_QFPtest_rec_count(%arg0: !fir.ref<!fir.array<4x7xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "row"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-// CHECK:    %[[V11:.*]] = fir.do_loop %arg3 = %c1 to %c7 step %c1 iter_args(%arg4 = %c0_i32) -> (i32) {
-// CHECK:      %[[V13:.*]] = fir.do_loop %arg5 = %c1 to %c7 step %c1 iter_args(%arg6 = %c0_i32) -> (i32) {
-// CHECK:        %[[V17:.*]] = hlfir.designate %[[V9]] (%arg5)  : (!fir.box<!fir.array<7xi32>>, index) -> !fir.ref<i32>
-// CHECK:        %[[V18:.*]] = fir.load %[[V17]] : !fir.ref<i32>
-// CHECK:        %[[V19:.*]] = arith.cmpi sge, %[[V18]], %[[V10]] : i32
-// CHECK:        %[[V20:.*]] = arith.addi %arg6, %c1_i32 : i32
-// CHECK:        %[[V21:.*]] = arith.select %[[V19]], %[[V20]], %arg6 : i32
-// CHECK:        fir.result %[[V21]] : i32
-// CHECK:      }
-// CHECK:      %[[V14:.*]] = arith.cmpi sge, %[[V13]], %[[V10]] : i32
-// CHECK:      %[[V15:.*]] = arith.addi %arg4, %c1_i32 : i32
-// CHECK:      %[[V16:.*]] = arith.select %[[V14]], %[[V15]], %arg4 : i32
-// CHECK:      fir.result %[[V16]] : i32
-// CHECK:    }
diff --git a/flang/test/HLFIR/maxloc-elemental.fir b/flang/test/HLFIR/maxloc-elemental.fir
deleted file mode 100644
index c9210a59f0340..0000000000000
--- a/flang/test/HLFIR/maxloc-elemental.fir
+++ /dev/null
@@ -1,133 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
-// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c-2147483648_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi sgt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3 = fir.load %2#0 : !fir.ref<f32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-    %9 = fir.load %8 : !fir.ref<f32>
-    %10 = arith.cmpf oge, %9, %3 : f32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.maxloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: _QPtest_float
-// CHECK:        %cst = arith.constant 0xFF800000 : f32
-// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
-// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
-// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
-// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
-// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
-// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf ogt, %[[V20]], %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
-// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
-// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
-// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
-// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:         fir.result %[[V20]] : f32
-// CHECK-NEXT:       } else {
-// CHECK-NEXT:         fir.result %arg4 : f32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V22]] : f32
-// CHECK-NEXT:     } else {
-// CHECK-NEXT:       fir.result %arg4 : f32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     fir.result %[[V18]] : f32
-// CHECK-NEXT:   }
-
diff --git a/flang/test/HLFIR/maxval-elemental.fir b/flang/test/HLFIR/maxval-elemental.fir
deleted file mode 100644
index a21b4858412de..0000000000000
--- a/flang/test/HLFIR/maxval-elemental.fir
+++ /dev/null
@@ -1,117 +0,0 @@
-// Test maxval inlining for both elemental and designate
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-// subroutine test(array)
-//   integer :: array(:), x
-//   x = maxval(abs(array))
-// end subroutine test
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-  %c31_i32 = arith.constant 31 : i32
-  %c0 = arith.constant 0 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg1: index):
-    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.shrsi %9, %c31_i32 : i32
-    %11 = arith.xori %9, %10 : i32
-    %12 = arith.subi %11, %10 : i32
-    hlfir.yield_element %12 : i32
-  }
-  %7 = hlfir.maxval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
-  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
-  hlfir.destroy %6 : !hlfir.expr<?xi32>
-  return
-}
-
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
-// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c-2147483648_i32) -> (i32) {
-// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
-// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi sgt, %[[V10]], %arg2 : i32
-// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
-// CHECK-NEXT:      fir.result %[[V12]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// subroutine test(array)
-//   real :: array(:), x
-//   x = maxval(array(3:6))
-// end subroutine test
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
-  %c4 = arith.constant 4 : index
-  %c1 = arith.constant 1 : index
-  %c6 = arith.constant 6 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-  %6 = hlfir.maxval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
-  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
-  return
-}
-
-// CHECK-LABEL: _QPtest_float
-// CHECK:       %cst = arith.constant 0xFF800000 : f32
-// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
-// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
-// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
-// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
-// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
-// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
-// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
-// CHECK-NEXT:      fir.result %[[V14]] : f32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// Verify that lower bounds of designator are applied in the indexing inside
-// the generated loop (hlfir.designate takes indices relative to the base lower
-// bounds).
-func.func @component_lower_bounds(%arg0: !fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>) -> i32 {
-  %c10 = arith.constant 10 : index
-  %c101 = arith.constant 101 : index
-  %4 = fir.shape_shift %c101, %c10 : (index, index) -> !fir.shapeshift<1>
-  %5 = hlfir.designate %arg0{"i"}   shape %4 : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
-  %6 = hlfir.maxval %5 : (!fir.box<!fir.array<10xi32>>) -> i32
-  return %6 : i32
-}
-// CHECK-LABEL:   func.func @component_lower_bounds(
-// CHECK:  %[[VAL_1:.*]] = arith.constant 100 : index
-// CHECK:  %[[VAL_2:.*]] = arith.constant 1 : index
-// CHECK:  %[[VAL_4:.*]] = arith.constant 10 : index
-// CHECK:  %[[VAL_5:.*]] = arith.constant 101 : index
-// CHECK:  %[[VAL_6:.*]] = fir.shape_shift %[[VAL_5]], %[[VAL_4]] : (index, index) -> !fir.shapeshift<1>
-// CHECK:  %[[VAL_7:.*]] = hlfir.designate %{{.*}}{"i"}   shape %[[VAL_6]] : (!fir.ref<!fir.type<sometype{i:!fir.array<10xi32>}>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
-// CHECK:  %[[VAL_8:.*]] = fir.do_loop %[[VAL_9:.*]] = %[[VAL_2]] to %[[VAL_4]] {{.*}}
-// CHECK:    %[[VAL_11:.*]] = arith.addi %[[VAL_9]], %[[VAL_1]] : index
-// CHECK:    hlfir.designate %[[VAL_7]] (%[[VAL_11]])  : (!fir.box<!fir.array<10xi32>>, index) -> !fir.ref<i32>
diff --git a/flang/test/HLFIR/minloc-elemental.fir b/flang/test/HLFIR/minloc-elemental.fir
deleted file mode 100644
index 9453a335b4fbf..0000000000000
--- a/flang/test/HLFIR/minloc-elemental.fir
+++ /dev/null
@@ -1,397 +0,0 @@
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c1_i32 = arith.constant 1 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i32 = arith.constant 0 : i32
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i32
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi32>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V8]] : !fir.ref<i32>
-// CHECK-NEXT:    fir.store %c0_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi32>>, !fir.box<!fir.array<?xi32>>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-
-func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.cmpi sge, %9, %3 : i32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi16>, !fir.box<!fir.array<?xi16>>
-  hlfir.destroy %7 : !hlfir.expr<1xi16>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL:  func.func @_QPtest_kind2(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi16>> {fir.bindc_name = "m"}) {
-// CHECK-NEXT:    %true = arith.constant true
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c1_i16 = arith.constant 1 : i16
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c0_i16 = arith.constant 0 : i16
-// CHECK-NEXT:    %[[V0:.*]] = fir.alloca i16
-// CHECK-NEXT:    %[[RES:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi16>>) -> (!fir.box<!fir.array<?xi16>>, !fir.box<!fir.array<?xi16>>)
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]] = fir.load %[[V3]]#0 : !fir.ref<i32>
-// CHECK-NEXT:    %[[V8:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK-NEXT:    fir.store %c0_i16 to %[[V8]] : !fir.ref<i16>
-// CHECK-NEXT:    fir.store %c0_i16 to %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:    %[[V9:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V10:.*]] = arith.subi %[[V9]]#1, %c1 : index
-// CHECK-NEXT:    %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10]] step %c1 iter_args(%arg4 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:      %[[V15:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V17:.*]] = arith.cmpi sge, %[[V16]], %[[V4]] : i32
-// CHECK-NEXT:      %[[V18:.*]] = fir.if %[[V17]] -> (i32) {
-// CHECK-NEXT:        %[[ISFIRST:.*]] = fir.load %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:        %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:        %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<i32>
-// CHECK-NEXT:        %[[V21:.*]] = arith.cmpi slt, %[[V20]], %arg4 : i32
-// CHECK-NEXT:        %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i16) -> i1
-// CHECK-NEXT:        %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:        %[[ORCOND:.*]] = arith.ori %[[V21]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:        %[[V22:.*]] = fir.if %[[ORCOND]] -> (i32) {
-// CHECK-NEXT:          fir.store %c1_i16 to %[[V0]] : !fir.ref<i16>
-// CHECK-NEXT:          %[[V23:.*]] = hlfir.designate %[[RES]] (%c1) : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK-NEXT:          %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i16
-// CHECK-NEXT:          fir.store %[[V24]] to %[[V23]] : !fir.ref<i16>
-// CHECK-NEXT:          fir.result %[[V20]] : i32
-// CHECK-NEXT:        } else {
-// CHECK-NEXT:          fir.result %arg4 : i32
-// CHECK-NEXT:        }
-// CHECK-NEXT:        fir.result %[[V22]] : i32
-// CHECK-NEXT:      } else {
-// CHECK-NEXT:        fir.result %arg4 : i32
-// CHECK-NEXT:      }
-// CHECK-NEXT:      fir.result %[[V18]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[RES]] to %[[V2]]#0 : !fir.ref<!fir.array<1xi16>>, !fir.box<!fir.array<?xi16>>
-// CHECK-NEXT:    return
-
-
-func.func @_QPtest_kind2_convert(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<i32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c1 = arith.constant 1 : index
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %3 = fir.load %2#0 : !fir.ref<i32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %10 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %11 = fir.load %10 : !fir.ref<i32>
-    %12 = arith.cmpi sge, %11, %3 : i32
-    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %13 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
-  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %9 = hlfir.elemental %8 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg3: index):
-    %10 = hlfir.apply %7, %arg3 : (!hlfir.expr<1xi16>, index) -> i16
-    %11 = fir.convert %10 : (i16) -> i32
-    hlfir.yield_element %11 : i32
-  }
-  hlfir.assign %9 to %1#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %9 : !hlfir.expr<?xi32>
-  hlfir.destroy %7 : !hlfir.expr<1xi16>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL:   func.func @_QPtest_kind2_convert(
-// CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"},
-// CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "val"},
-// CHECK-SAME:                                     %[[VAL_2:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-// CHECK:           %[[VAL_3:.*]] = arith.constant false
-// CHECK:           %[[VAL_4:.*]] = arith.constant true
-// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
-// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i16
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i16
-// CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_10:.*]] = fir.alloca i16
-// CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.array<1xi16>
-// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
-// CHECK:           %[[VAL_16:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_16]] : !fir.ref<i16>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<i16>
-// CHECK:           %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_12]]#0, %[[VAL_7]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#1, %[[VAL_9]] : index
-// CHECK:           %[[VAL_19:.*]] = fir.do_loop %[[VAL_20:.*]] = %[[VAL_7]] to %[[VAL_18]] step %[[VAL_9]] iter_args(%[[VAL_21:.*]] = %[[VAL_5]]) -> (i32) {
-// CHECK:             %[[VAL_22:.*]] = arith.addi %[[VAL_20]], %[[VAL_9]] : index
-// CHECK:             %[[VAL_23:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<i32>
-// CHECK:             %[[VAL_25:.*]] = arith.cmpi sge, %[[VAL_24]], %[[VAL_15]] : i32
-// CHECK:             %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (i32) {
-// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_10]] : !fir.ref<i16>
-// CHECK:               %[[VAL_28:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:               %[[VAL_29:.*]] = fir.load %[[VAL_28]] : !fir.ref<i32>
-// CHECK:               %[[VAL_30:.*]] = arith.cmpi slt, %[[VAL_29]], %[[VAL_21]] : i32
-// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_27]] : (i16) -> i1
-// CHECK:               %[[VAL_32:.*]] = arith.xori %[[VAL_31]], %[[VAL_4]] : i1
-// CHECK:               %[[VAL_33:.*]] = arith.ori %[[VAL_30]], %[[VAL_32]] : i1
-// CHECK:               %[[VAL_34:.*]] = fir.if %[[VAL_33]] -> (i32) {
-// CHECK:                 fir.store %[[VAL_6]] to %[[VAL_10]] : !fir.ref<i16>
-// CHECK:                 %[[VAL_35:.*]] = hlfir.designate %[[VAL_11]] (%[[VAL_9]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
-// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_22]] : (index) -> i16
-// CHECK:                 fir.store %[[VAL_36]] to %[[VAL_35]] : !fir.ref<i16>
-// CHECK:                 fir.result %[[VAL_29]] : i32
-// CHECK:               } else {
-// CHECK:                 fir.result %[[VAL_21]] : i32
-// CHECK:               }
-// CHECK:               fir.result %[[VAL_34]] : i32
-// CHECK:             } else {
-// CHECK:               fir.result %[[VAL_21]] : i32
-// CHECK:             }
-// CHECK:             fir.result %[[VAL_26]] : i32
-// CHECK:           }
-// CHECK:           %[[VAL_37:.*]] = hlfir.as_expr %[[VAL_11]] move %[[VAL_3]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
-// CHECK:           fir.do_loop %[[VAL_38:.*]] = %[[VAL_9]] to %[[VAL_9]] step %[[VAL_9]] unordered {
-// CHECK:             %[[VAL_39:.*]] = hlfir.apply %[[VAL_37]], %[[VAL_38]] : (!hlfir.expr<1xi16>, index) -> i16
-// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i16) -> i32
-// CHECK:             %[[VAL_41:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_38]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK:             hlfir.assign %[[VAL_40]] to %[[VAL_41]] : i32, !fir.ref<i32>
-// CHECK:           }
-// CHECK:           return
-// CHECK:         }
-
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "m"}) {
-  %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFtestEm"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3 = fir.load %2#0 : !fir.ref<f32>
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %8 = hlfir.designate %0#0 (%arg3)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-    %9 = fir.load %8 : !fir.ref<f32>
-    %10 = arith.cmpf oge, %9, %3 : f32
-    %11 = fir.convert %10 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %11 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %0#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %7 to %1#0 : !hlfir.expr<1xi32>, !fir.box<!fir.array<?xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  return
-}
-// CHECK-LABEL: _QPtest_float
-// CHECK:        %cst = arith.constant 0x7F800000 : f32
-// CHECK:        %[[V11:.*]] = fir.do_loop %arg3 = %c0 to %[[V10:.*]] step %c1 iter_args(%arg4 = %cst) -> (f32) {
-// CHECK-NEXT:     %[[V14:.*]] = arith.addi %arg3, %c1 : index
-// CHECK-NEXT:     %[[V15:.*]] = hlfir.designate %[[V1:.*]]#0 (%[[V14]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:     %[[V16:.*]] = fir.load %[[V15]] : !fir.ref<f32>
-// CHECK-NEXT:     %[[V17:.*]] = arith.cmpf oge, %[[V16]], %[[V4:.*]] : f32
-// CHECK-NEXT:     %[[V18:.*]] = fir.if %[[V17]] -> (f32) {
-// CHECK-NEXT:       %[[ISFIRST:.*]] = fir.load %[[V0:.*]] : !fir.ref<i32>
-// CHECK-NEXT:       %[[V19:.*]] = hlfir.designate %[[V1]]#0 (%[[V14]]) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:       %[[V20:.*]] = fir.load %[[V19]] : !fir.ref<f32>
-// CHECK-NEXT:       %[[NEW_MIN:.*]] = arith.cmpf olt, %[[V20]], %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDRED:.*]] = arith.cmpf une, %arg4, %arg4 fastmath<contract> : f32
-// CHECK-NEXT:       %[[CONDELEM:.*]] = arith.cmpf oeq, %[[V20]], %[[V20]] fastmath<contract> : f32
-// CHECK-NEXT:       %[[ANDCOND:.*]] = arith.andi %[[CONDRED]], %[[CONDELEM]] : i1
-// CHECK-NEXT:       %[[NEW_MIN2:.*]] = arith.ori %[[NEW_MIN]], %[[ANDCOND]] : i1
-// CHECK-NEXT:       %[[ISFIRSTL:.*]] = fir.convert %[[ISFIRST]] : (i32) -> i1
-// CHECK-NEXT:       %[[ISFIRSTNOT:.*]] = arith.xori %[[ISFIRSTL]], %true : i1
-// CHECK-NEXT:       %[[ORCOND:.*]] = arith.ori %[[NEW_MIN2]], %[[ISFIRSTNOT]] : i1
-// CHECK-NEXT:       %[[V22:.*]] = fir.if %[[ORCOND]] -> (f32) {
-// CHECK-NEXT:         fir.store %c1_i32 to %[[V0]] : !fir.ref<i32>
-// CHECK-NEXT:         %[[V23:.*]] = hlfir.designate %{{.}} (%c1) : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:         %[[V24:.*]] = fir.convert %[[V14]] : (index) -> i32
-// CHECK-NEXT:         fir.store %[[V24]] to %[[V23]] : !fir.ref<i32>
-// CHECK-NEXT:         fir.result %[[V20]] : f32
-// CHECK-NEXT:       } else {
-// CHECK-NEXT:         fir.result %arg4 : f32
-// CHECK-NEXT:       }
-// CHECK-NEXT:       fir.result %[[V22]] : f32
-// CHECK-NEXT:     } else {
-// CHECK-NEXT:       fir.result %arg4 : f32
-// CHECK-NEXT:     }
-// CHECK-NEXT:     fir.result %[[V18]] : f32
-// CHECK-NEXT:   }
-
-
-func.func @_QPtest_assignshape(%arg0: !fir.ref<!fir.array<3x3xf32>> {fir.bindc_name = "array"}, %arg1: !fir.ref<f32> {fir.bindc_name = "val"}, %arg2: !fir.ref<!fir.array<3xi32>> {fir.bindc_name = "m"}) {
-  %c2 = arith.constant 2 : index
-  %c1 = arith.constant 1 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.shape %c3, %c3 : (index, index) -> !fir.shape<2>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFtestEarray"} : (!fir.ref<!fir.array<3x3xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x3xf32>>, !fir.ref<!fir.array<3x3xf32>>)
-  %2 = fir.shape %c3 : (index) -> !fir.shape<1>
-  %3:2 = hlfir.declare %arg2(%2) {uniq_name = "_QFtestEm"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-  %4:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEval"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %5 = fir.load %4#0 : !fir.ref<f32>
-  %6 = hlfir.elemental %0 unordered : (!fir.shape<2>) -> !hlfir.expr<3x3x!fir.logical<4>> {
-  ^bb0(%arg3: index, %arg4: index):
-    %10 = hlfir.designate %1#0 (%arg3, %arg4)  : (!fir.ref<!fir.array<3x3xf32>>, index, index) -> !fir.ref<f32>
-    %11 = fir.load %10 : !fir.ref<f32>
-    %12 = arith.cmpf oge, %11, %5 : f32
-    %13 = fir.convert %12 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %13 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %1#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.ref<!fir.array<3x3xf32>>, !hlfir.expr<3x3x!fir.logical<4>>) -> !hlfir.expr<2xi32>
-  %8 = fir.shape %c2 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %3#0 (%c1:%c2:%c1)  shape %8 : (!fir.ref<!fir.array<3xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<2xi32>>
-  hlfir.assign %7 to %9 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
-  hlfir.destroy %7 : !hlfir.expr<2xi32>
-  hlfir.destroy %6 : !hlfir.expr<3x3x!fir.logical<4>>
-  return
-}
-// Not supported as the input is not a box
-// CHECK-LABEL: _QPtest_assignshape
-// CHECK: hlfir.minloc
-
-
-func.func @_QFPtest_character(%arg0: !fir.box<!fir.array<?x!fir.char<1>>> {fir.bindc_name = "b"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "val"}) -> i32 {
-  %c0 = arith.constant 0 : index
-  %c1 = arith.constant 1 : index
-  %0:2 = hlfir.declare %arg0 typeparams %c1 {uniq_name = "_QFFtestEb"} : (!fir.box<!fir.array<?x!fir.char<1>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1>>>, !fir.box<!fir.array<?x!fir.char<1>>>)
-  %1:2 = hlfir.declare %arg1 {uniq_name = "_QFFtestEc"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca !fir.array<1xi32> {bindc_name = "m", uniq_name = "_QFFtestEm"}
-  %3 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %4:2 = hlfir.declare %2(%3) {uniq_name = "_QFFtestEm"} : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1xi32>>, !fir.ref<!fir.array<1xi32>>)
-  %5 = fir.alloca i32 {bindc_name = "test", uniq_name = "_QFFtestEtest"}
-  %6:2 = hlfir.declare %5 {uniq_name = "_QFFtestEtest"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %7:2 = hlfir.declare %arg2 {uniq_name = "_QFFtestEval"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %8 = fir.load %7#0 : !fir.ref<i32>
-  %9:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %10 = fir.shape %9#1 : (index) -> !fir.shape<1>
-  %11 = hlfir.elemental %10 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg3: index):
-    %16 = hlfir.designate %1#0 (%arg3)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %17 = fir.load %16 : !fir.ref<i32>
-    %18 = arith.cmpi eq, %17, %8 : i32
-    %19 = fir.convert %18 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %19 : !fir.logical<4>
-  }
-  %12 = hlfir.minloc %0#0 mask %11 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x!fir.char<1>>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  hlfir.assign %12 to %4#0 : !hlfir.expr<1xi32>, !fir.ref<!fir.array<1xi32>>
-  hlfir.destroy %12 : !hlfir.expr<1xi32>
-  hlfir.destroy %11 : !hlfir.expr<?x!fir.logical<4>>
-  %13 = hlfir.designate %4#0 (%c1)  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
-  %14 = fir.load %13 : !fir.ref<i32>
-  hlfir.assign %14 to %6#0 : i32, !fir.ref<i32>
-  %15 = fir.load %6#1 : !fir.ref<i32>
-  return %15 : i32
-}
-// Characters are not supported at the moment
-// CHECK-LABEL: _QFPtest_character
-// CHECK: hlfir.minloc
-
-
-func.func @_QPtest_parts(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "mask"}) -> f32 {
-  %c1 = arith.constant 1 : index
-  %c5 = arith.constant 5 : index
-  %c0 = arith.constant 0 : index
-  %c5_i32 = arith.constant 5 : i32
-  %0:2 = hlfir.declare %arg1 {uniq_name = "_QFtestEmask"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %1 = fir.alloca f32 {bindc_name = "test", uniq_name = "_QFtestEtest"}
-  %2:2 = hlfir.declare %1 {uniq_name = "_QFtestEtest"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %3:2 = hlfir.declare %arg0 {uniq_name = "_QFtestEx"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %4:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
-  ^bb0(%arg2: index):
-    %11 = hlfir.designate %0#0 (%arg2)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %12 = fir.load %11 : !fir.ref<i32>
-    %13 = arith.cmpi sge, %12, %c5_i32 : i32
-    %14 = fir.convert %13 : (i1) -> !fir.logical<4>
-    hlfir.yield_element %14 : !fir.logical<4>
-  }
-  %7 = hlfir.minloc %3#0 mask %6 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
-  %8 = fir.shape %c1 : (index) -> !fir.shape<1>
-  %9 = hlfir.designate %3#0 (%c5:%c5:%c1)  shape %8 : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<1xi32>>
-  hlfir.assign %7 to %9 : !hlfir.expr<1xi32>, !fir.box<!fir.array<1xi32>>
-  hlfir.destroy %7 : !hlfir.expr<1xi32>
-  hlfir.destroy %6 : !hlfir.expr<?x!fir.logical<4>>
-  %10 = fir.load %2#1 : !fir.ref<f32>
-  return %10 : f32
-}
-// Characters are not supported at the moment
-// CHECK-LABEL: _QPtest_parts
-// CHECK: fir.do_loop %{{.*}} = %c0 to %{{.*}} step %c1 iter_args(%{{.*}} = %c2147483647_i32) -> (i32) {
-
diff --git a/flang/test/HLFIR/minval-elemental.fir b/flang/test/HLFIR/minval-elemental.fir
deleted file mode 100644
index 64cd5403ec558..0000000000000
--- a/flang/test/HLFIR/minval-elemental.fir
+++ /dev/null
@@ -1,95 +0,0 @@
-// Test maxval inlining for both elemental and designate
-// RUN: fir-opt %s -opt-bufferization | FileCheck %s
-
-// subroutine test(array)
-//   integer :: array(:), x
-//   x = minval(abs(array))
-// end subroutine test
-
-func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-  %c31_i32 = arith.constant 31 : i32
-  %c0 = arith.constant 0 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
-  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
-  ^bb0(%arg1: index):
-    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-    %9 = fir.load %8 : !fir.ref<i32>
-    %10 = arith.shrsi %9, %c31_i32 : i32
-    %11 = arith.xori %9, %10 : i32
-    %12 = arith.subi %11, %10 : i32
-    hlfir.yield_element %12 : i32
-  }
-  %7 = hlfir.minval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
-  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
-  hlfir.destroy %6 : !hlfir.expr<?xi32>
-  return
-}
-
-// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
-// CHECK-NEXT:    %c1 = arith.constant 1 : index
-// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
-// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
-// CHECK-NEXT:    %c0 = arith.constant 0 : index
-// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c2147483647_i32) -> (i32) {
-// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
-// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
-// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi slt, %[[V10]], %arg2 : i32
-// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
-// CHECK-NEXT:      fir.result %[[V12]] : i32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
-
-// subroutine test(array)
-//   real :: array(:), x
-//   x = minval(array(3:6))
-// end subroutine test
-
-func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
-  %c4 = arith.constant 4 : index
-  %c1 = arith.constant 1 : index
-  %c6 = arith.constant 6 : index
-  %c3 = arith.constant 3 : index
-  %0 = fir.dummy_scope : !fir.dscope
-  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
-  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
-  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-  %6 = hlfir.minval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
-  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
-  return
-}
-
-// CHECK-LABEL: _QPtest_float
-// CHECK:       %cst = arith.constant 0x7F800000 : f32
-// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
-// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
-// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
-// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
-// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
-// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 fastmath<contract> : f32
-// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] fastmath<contract> : f32
-// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
-// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
-// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
-// CHECK-NEXT:      fir.result %[[V14]] : f32
-// CHECK-NEXT:    }
-// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
-// CHECK-NEXT:    return
-// CHECK-NEXT:  }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
new file mode 100644
index 0000000000000..fb0f39811ea74
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-all.fir
@@ -0,0 +1,123 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+  %0 = hlfir.all %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_14:.*]] = arith.andi %[[VAL_13]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_14]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_15]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.all %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
+// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = arith.andi %[[VAL_15]], %[[VAL_13]] : i1
+// CHECK:               fir.result %[[VAL_16]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
+// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+  %0 = hlfir.all %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant true
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
+// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_20]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_21]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_22]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.all %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant true
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
+// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_27:.*]] = arith.andi %[[VAL_26]], %[[VAL_14]] : i1
+// CHECK:               fir.result %[[VAL_27]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
+// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
new file mode 100644
index 0000000000000..5bd76f3d24927
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-any.fir
@@ -0,0 +1,123 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+  %0 = hlfir.any %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_1]] to %[[VAL_4]] step %[[VAL_1]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_7]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_14:.*]] = arith.ori %[[VAL_13]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_14]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_15]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.any %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?x!fir.logical<1>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<1>>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_4:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_3]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_4]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<1>> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
+// CHECK:             %[[VAL_11:.*]] = fir.do_loop %[[VAL_12:.*]] = %[[VAL_1]] to %[[VAL_5]] step %[[VAL_1]] unordered iter_args(%[[VAL_13:.*]] = %[[VAL_2]]) -> (i1) {
+// CHECK:               %[[VAL_14:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_9]], %[[VAL_12]], %[[VAL_10]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]] = arith.ori %[[VAL_15]], %[[VAL_13]] : i1
+// CHECK:               fir.result %[[VAL_16]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_17:.*]] = fir.convert %[[VAL_11]] : (i1) -> !fir.logical<1>
+// CHECK:             hlfir.yield_element %[[VAL_17]] : !fir.logical<1>
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?x?x!fir.logical<1>>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+  %0 = hlfir.any %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> !fir.logical<4> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]] = fir.do_loop %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_5]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_8:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:             %[[VAL_9:.*]] = fir.do_loop %[[VAL_10:.*]] = %[[VAL_2]] to %[[VAL_4]]#1 step %[[VAL_2]] unordered iter_args(%[[VAL_11:.*]] = %[[VAL_8]]) -> (i1) {
+// CHECK:               %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]] = arith.subi %[[VAL_12]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_10]], %[[VAL_14]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_2]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_7]], %[[VAL_16]] : index
+// CHECK:               %[[VAL_18:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_15]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_19:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_21:.*]] = arith.ori %[[VAL_20]], %[[VAL_11]] : i1
+// CHECK:               fir.result %[[VAL_21]] : i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_9]] : i1
+// CHECK:           }
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (i1) -> !fir.logical<4>
+// CHECK:           return %[[VAL_22]] : !fir.logical<4>
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.any %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?x!fir.logical<2>>
+  return %0 : !hlfir.expr<?x?x!fir.logical<2>>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]]#1, %[[VAL_7]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?x!fir.logical<2>> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_1]]) -> (i1) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_11]], %[[VAL_22]] : index
+// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]], %[[VAL_21]], %[[VAL_23]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_25:.*]] = fir.load %[[VAL_24]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_14]] : i1
+// CHECK:               fir.result %[[VAL_27]] : i1
+// CHECK:             }
+// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_12]] : (i1) -> !fir.logical<2>
+// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.logical<2>
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?x!fir.logical<2>>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
new file mode 100644
index 0000000000000..44594c646a368
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-count.fir
@@ -0,0 +1,127 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%arg0: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
+  %0 = hlfir.count %arg0 : (!hlfir.expr<?x?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x!fir.logical<4>>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_3]]) -> (i32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
+// CHECK:               %[[VAL_16:.*]] = arith.select %[[VAL_14]], %[[VAL_15]], %[[VAL_12]] : i32
+// CHECK:               fir.result %[[VAL_16]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : i32
+// CHECK:         }
+
+func.func @test_partial_expr(%arg0: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.count %arg0 dim %dim : (!hlfir.expr<?x?x?x!fir.logical<1>>, i32) -> !hlfir.expr<?x?xi16>
+  return %0 : !hlfir.expr<?x?xi16>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<1>>) -> !hlfir.expr<?x?xi16> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i16
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?x!fir.logical<1>>) -> !fir.shape<3>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_7:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_5]], %[[VAL_7]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi16> {
+// CHECK:           ^bb0(%[[VAL_10:.*]]: index, %[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]] = fir.do_loop %[[VAL_13:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]]) -> (i16) {
+// CHECK:               %[[VAL_15:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]], %[[VAL_13]], %[[VAL_11]] : (!hlfir.expr<?x?x?x!fir.logical<1>>, index, index, index) -> !fir.logical<1>
+// CHECK:               %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_14]], %[[VAL_1]] : i16
+// CHECK:               %[[VAL_18:.*]] = arith.select %[[VAL_16]], %[[VAL_17]], %[[VAL_14]] : i16
+// CHECK:               fir.result %[[VAL_18]] : i16
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]] : i16
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : !hlfir.expr<?x?xi16>
+// CHECK:         }
+
+func.func @test_total_var(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
+  %0 = hlfir.count %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32 {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]] = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]]) -> (i32) {
+// CHECK:             %[[VAL_10:.*]] = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]]) -> (i32) {
+// CHECK:               %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_15:.*]] = arith.subi %[[VAL_13]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_16:.*]] = arith.addi %[[VAL_11]], %[[VAL_15]] : index
+// CHECK:               %[[VAL_17:.*]] = arith.subi %[[VAL_14]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_18:.*]] = arith.addi %[[VAL_8]], %[[VAL_17]] : index
+// CHECK:               %[[VAL_19:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_16]], %[[VAL_18]])  : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_21]], %[[VAL_22]], %[[VAL_12]] : i32
+// CHECK:               fir.result %[[VAL_23]] : i32
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]] : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_7]] : i32
+// CHECK:         }
+
+func.func @test_partial_var(%arg0: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.count %arg0 dim %dim : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, i32) -> !hlfir.expr<?x?xi64>
+  return %0 : !hlfir.expr<?x?xi64>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<2>>>) -> !hlfir.expr<?x?xi64> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_6]]#1, %[[VAL_8]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi64> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: index):
+// CHECK:             %[[VAL_13:.*]] = fir.do_loop %[[VAL_14:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_15:.*]] = %[[VAL_2]]) -> (i64) {
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_22:.*]] = arith.addi %[[VAL_14]], %[[VAL_21]] : index
+// CHECK:               %[[VAL_23:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_4]] : index
+// CHECK:               %[[VAL_24:.*]] = arith.addi %[[VAL_12]], %[[VAL_23]] : index
+// CHECK:               %[[VAL_25:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<2>>>, index, index, index) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (!fir.logical<2>) -> i1
+// CHECK:               %[[VAL_28:.*]] = arith.addi %[[VAL_15]], %[[VAL_1]] : i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_15]] : i64
+// CHECK:               fir.result %[[VAL_29]] : i64
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_13]] : i64
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?x?xi64>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
new file mode 100644
index 0000000000000..4e9f5d0ebb08a
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxloc.fir
@@ -0,0 +1,312 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_1d_total_expr(
+// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
+// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -2147483648 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
+// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
+// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_10]]#0 to %[[VAL_28]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
+// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
+// CHECK:         }
+
+func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+  %dim = arith.constant 1 : i16
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_1d_dim_expr(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
+// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
+// CHECK:               %[[VAL_18:.*]] = arith.cmpf ogt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
+// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
+// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
+// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
+// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
+// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_9]]#0 : i32
+// CHECK:         }
+
+func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  return %0 : !hlfir.expr<1xi16>
+}
+// CHECK-LABEL:   func.func @test_1d_total_var(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf ogt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
+// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
+// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
+// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
+// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           hlfir.assign %[[VAL_10]]#0 to %[[VAL_41]] : i16, !fir.ref<i16>
+// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
+// CHECK:         }
+
+func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
+  return %0 : i64
+}
+// CHECK-LABEL:   func.func @test_1d_dim_var(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant -1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
+// CHECK:               %[[VAL_22:.*]] = arith.cmpf ogt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
+// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
+// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
+// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_9]]#0 : i64
+// CHECK:         }
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+  %0 = hlfir.maxloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
+  return %0 : !hlfir.expr<3xi32>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant true
+// CHECK:           %[[VAL_7:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
+// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
+// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
+// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
+// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
+// CHECK:                   %[[VAL_39:.*]] = arith.cmpf ogt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
+// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
+// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
+// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
+// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
+// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
+// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
+// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
+// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
+// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
+// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
+// CHECK:                 } else {
+// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
+// CHECK:                 }
+// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_14]]#0 to %[[VAL_58]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_14]]#1 to %[[VAL_61]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_14]]#2 to %[[VAL_64]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
+// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.maxloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
+  return %0 : !hlfir.expr<?x?xi32>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
+// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
+// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
+// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
+// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
+// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
+// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
+// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
+// CHECK:               }
+// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
+// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
+// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
+// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
+// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_48:.*]] = arith.cmpf ogt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
+// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
+// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
+// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
+// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
+// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
+// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_17]]#0 : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
+// CHECK:         }
+
+// Character comparisons are not supported yet.
+func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.maxloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_character(
+// CHECK:           hlfir.maxloc
+
+// BACK is not supported yet.
+func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
+  %back = arith.constant true
+  %0 = hlfir.maxloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_back(
+// CHECK:           hlfir.maxloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
new file mode 100644
index 0000000000000..8f414e5c4b563
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-maxval.fir
@@ -0,0 +1,186 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
+  return %0 : f32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+// CHECK:           %[[FALSE:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[TRUE:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant -3.40282347E+38 : f32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]]:2 = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]], %[[FIRST1:.*]] = %[[TRUE]]) -> (f32, i1) {
+// CHECK:             %[[VAL_10:.*]]:2 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]], %[[FIRST2:.*]] = %[[FIRST1]]) -> (f32, i1) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]]:2 = fir.if %[[VAL_14]] -> (f32, i1) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[IS_FIRST:.*]] = arith.ori %[[VAL_21]], %[[FIRST2]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[IS_FIRST]], %[[VAL_16]], %[[VAL_12]] : f32
+// CHECK:                 fir.result %[[VAL_22]], %[[FALSE]] : f32, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]], %[[FIRST2]] : f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]]#0, %[[VAL_15]]#1 : f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]]#0, %[[VAL_10]]#1 : f32, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_7]]#0 : f32
+// CHECK:         }
+
+func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
+  return %0 : !hlfir.expr<?xf64>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+// CHECK:           %[[FALSE:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[TRUE:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant -1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
+// CHECK:             %[[VAL_10:.*]]:2 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]], %[[FIRST:.*]] = %[[TRUE]]) -> (f64, i1) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]]:2 = fir.if %[[VAL_14]] -> (f64, i1) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf ogt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[IS_FIRST:.*]] = arith.ori %[[VAL_21]], %[[FIRST]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[IS_FIRST]], %[[VAL_16]], %[[VAL_12]] : f64
+// CHECK:                 fir.result %[[VAL_22]], %[[FALSE]] : f64, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]], %[[FIRST]] : f64, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]]#0, %[[VAL_15]]#1 : f64, i1
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_10]]#0 : f64
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
+// CHECK:         }
+
+func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+  %0 = hlfir.maxval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
+  return %0 : f16
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant -6.550400e+04 : f16
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:2 = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]], %[[FIRST1:.*]] = %[[TRUE]]) -> (f16, i1) {
+// CHECK:             %[[VAL_10:.*]]:2 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]], %[[FIRST2:.*]] = %[[FIRST1]]) -> (f16, i1) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]]:2 = fir.if %[[VAL_15]] -> (f16, i1) {
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
+// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_25:.*]] = arith.cmpf ogt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
+// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
+// CHECK:                 %[[IS_FIRST:.*]] = arith.ori %[[VAL_29]], %[[FIRST2]] : i1
+// CHECK:                 %[[VAL_30:.*]] = arith.select %[[IS_FIRST]], %[[VAL_24]], %[[VAL_12]] : f16
+// CHECK:                 fir.result %[[VAL_30]], %[[FALSE]] : f16, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]], %[[FIRST2]] : f16, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_16]]#0, %[[VAL_16]]#1 : f16, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]]#0, %[[VAL_10]]#1 : f16, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_7]]#0 : f16
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.maxval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
+  return %0 : !hlfir.expr<?xf16>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+// CHECK:           %[[FALSE:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant -6.550400e+04 : f16
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]]:2 = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]], %[[FIRST:.*]] = %[[TRUE]]) -> (f16, i1) {
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
+// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
+// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
+// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
+// CHECK:               }
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_26:.*]]:2 = fir.if %[[VAL_25]] -> (f16, i1) {
+// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
+// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_35:.*]] = arith.cmpf ogt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
+// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
+// CHECK:                 %[[IS_FIRST:.*]] = arith.ori %[[VAL_39]], %[[FIRST]] : i1
+// CHECK:                 %[[VAL_40:.*]] = arith.select %[[IS_FIRST]], %[[VAL_34]], %[[VAL_14]] : f16
+// CHECK:                 fir.result %[[VAL_40]], %[[FALSE]] : f16, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_14]], %[[FIRST]] : f16, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_26]]#0, %[[VAL_26]]#1 : f16, i1
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]]#0 : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
+// CHECK:         }
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
new file mode 100644
index 0000000000000..0c17fd6fea92c
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-minloc.fir
@@ -0,0 +1,312 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_1d_total_expr(%input: !hlfir.expr<?xi32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_1d_total_expr(
+// CHECK-SAME:                                  %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xi32>,
+// CHECK-SAME:                                  %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2147483647 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<1xi32>
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]] step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i32, i32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i32, i32, i1) {
+// CHECK:               %[[VAL_18:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]] : (!hlfir.expr<?xi32>, index) -> i32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               %[[VAL_20:.*]] = arith.ori %[[VAL_19]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_21:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
+// CHECK:               %[[VAL_22:.*]] = arith.select %[[VAL_20]], %[[VAL_21]], %[[VAL_12]] : i32
+// CHECK:               %[[VAL_23:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_13]] : i32
+// CHECK:               fir.result %[[VAL_22]], %[[VAL_23]], %[[VAL_2]] : i32, i32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i32, i32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_24:.*]]#0, %[[VAL_24]]#1, %[[VAL_24]]#2 : i32, i32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_28:.*]] = hlfir.designate %[[VAL_7]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_10]]#0 to %[[VAL_28]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_7]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi32>>, i1) -> !hlfir.expr<1xi32>
+// CHECK:           return %[[VAL_29]] : !hlfir.expr<1xi32>
+// CHECK:         }
+
+func.func @test_1d_dim_expr(%input: !hlfir.expr<?xf32>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+  %dim = arith.constant 1 : i16
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xf32>, i16, !hlfir.expr<?x!fir.logical<4>>) -> i32
+  return %0 : i32
+}
+// CHECK-LABEL:   func.func @test_1d_dim_expr(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?xf32>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i32 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?xf32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i32, f32, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i32, f32, i1) {
+// CHECK:               %[[VAL_17:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_10]] : (!hlfir.expr<?xf32>, index) -> f32
+// CHECK:               %[[VAL_18:.*]] = arith.cmpf olt, %[[VAL_17]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_19:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:               %[[VAL_20:.*]] = arith.cmpf oeq, %[[VAL_17]], %[[VAL_17]] fastmath<contract> : f32
+// CHECK:               %[[VAL_21:.*]] = arith.andi %[[VAL_19]], %[[VAL_20]] : i1
+// CHECK:               %[[VAL_22:.*]] = arith.ori %[[VAL_18]], %[[VAL_21]] : i1
+// CHECK:               %[[VAL_23:.*]] = arith.ori %[[VAL_22]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_10]] : (index) -> i32
+// CHECK:               %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_11]] : i32
+// CHECK:               %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_17]], %[[VAL_12]] : f32
+// CHECK:               fir.result %[[VAL_25]], %[[VAL_26]], %[[VAL_2]] : i32, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_27:.*]]#0, %[[VAL_27]]#1, %[[VAL_27]]#2 : i32, f32, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_9]]#0 : i32
+// CHECK:         }
+
+func.func @test_1d_total_var(%input: !fir.box<!fir.array<?xf32>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf32>>, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16>
+  return %0 : !hlfir.expr<1xi16>
+}
+// CHECK-LABEL:   func.func @test_1d_total_var(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<1xi16> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i16
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<1xi16>
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_9]]#1 step %[[VAL_3]] iter_args(%[[VAL_12:.*]] = %[[VAL_6]], %[[VAL_13:.*]] = %[[VAL_5]], %[[VAL_14:.*]] = %[[VAL_4]]) -> (i16, f32, i1) {
+// CHECK:             %[[VAL_15:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_17:.*]]:3 = fir.if %[[VAL_16]] -> (i16, f32, i1) {
+// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_19:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf olt, %[[VAL_22]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf une, %[[VAL_13]], %[[VAL_13]] fastmath<contract> : f32
+// CHECK:               %[[VAL_25:.*]] = arith.cmpf oeq, %[[VAL_22]], %[[VAL_22]] fastmath<contract> : f32
+// CHECK:               %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_23]], %[[VAL_26]] : i1
+// CHECK:               %[[VAL_28:.*]] = arith.ori %[[VAL_27]], %[[VAL_14]] : i1
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_11]] : (index) -> i16
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_12]] : i16
+// CHECK:               %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_22]], %[[VAL_13]] : f32
+// CHECK:               fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_2]] : i16, f32, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_12]], %[[VAL_13]], %[[VAL_14]] : i16, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_32:.*]]#0, %[[VAL_32]]#1, %[[VAL_32]]#2 : i16, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_41:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_3]])  : (!fir.ref<!fir.array<1xi16>>, index) -> !fir.ref<i16>
+// CHECK:           hlfir.assign %[[VAL_10]]#0 to %[[VAL_41]] : i16, !fir.ref<i16>
+// CHECK:           %[[VAL_42:.*]] = hlfir.as_expr %[[VAL_8]] move %[[VAL_2]] : (!fir.ref<!fir.array<1xi16>>, i1) -> !hlfir.expr<1xi16>
+// CHECK:           return %[[VAL_42]] : !hlfir.expr<1xi16>
+// CHECK:         }
+
+func.func @test_1d_dim_var(%input: !fir.box<!fir.array<?xf64>>, %mask: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xf64>>, i32, !hlfir.expr<?x!fir.logical<4>>) -> i64
+  return %0 : i64
+}
+// CHECK-LABEL:   func.func @test_1d_dim_var(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xf64>>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x!fir.logical<4>>) -> i64 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant true
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_8:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_9:.*]]:3 = fir.do_loop %[[VAL_10:.*]] = %[[VAL_3]] to %[[VAL_8]]#1 step %[[VAL_3]] iter_args(%[[VAL_11:.*]] = %[[VAL_6]], %[[VAL_12:.*]] = %[[VAL_5]], %[[VAL_13:.*]] = %[[VAL_4]]) -> (i64, f64, i1) {
+// CHECK:             %[[VAL_14:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_10]] : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+// CHECK:             %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<4>) -> i1
+// CHECK:             %[[VAL_16:.*]]:3 = fir.if %[[VAL_15]] -> (i64, f64, i1) {
+// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_18:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_10]], %[[VAL_18]] : index
+// CHECK:               %[[VAL_20:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_19]])  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK:               %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<f64>
+// CHECK:               %[[VAL_22:.*]] = arith.cmpf olt, %[[VAL_21]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_23:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f64
+// CHECK:               %[[VAL_24:.*]] = arith.cmpf oeq, %[[VAL_21]], %[[VAL_21]] fastmath<contract> : f64
+// CHECK:               %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
+// CHECK:               %[[VAL_26:.*]] = arith.ori %[[VAL_22]], %[[VAL_25]] : i1
+// CHECK:               %[[VAL_27:.*]] = arith.ori %[[VAL_26]], %[[VAL_13]] : i1
+// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:               %[[VAL_29:.*]] = arith.select %[[VAL_27]], %[[VAL_28]], %[[VAL_11]] : i64
+// CHECK:               %[[VAL_30:.*]] = arith.select %[[VAL_27]], %[[VAL_21]], %[[VAL_12]] : f64
+// CHECK:               fir.result %[[VAL_29]], %[[VAL_30]], %[[VAL_2]] : i64, f64, i1
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_11]], %[[VAL_12]], %[[VAL_13]] : i64, f64, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_31:.*]]#0, %[[VAL_31]]#1, %[[VAL_31]]#2 : i64, f64, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_9]]#0 : i64
+// CHECK:         }
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?x?xf32>, %mask: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+  %0 = hlfir.minloc %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?x?xf32>, !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32>
+  return %0 : !hlfir.expr<3xi32>
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x?x!fir.logical<4>>) -> !hlfir.expr<3xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant true
+// CHECK:           %[[VAL_7:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xi32>
+// CHECK:           %[[VAL_10:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?x?xf32>) -> !fir.shape<3>
+// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 0 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_12:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 1 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_10]] {dim = 2 : index} : (!fir.shape<3>) -> index
+// CHECK:           %[[VAL_14:.*]]:5 = fir.do_loop %[[VAL_15:.*]] = %[[VAL_5]] to %[[VAL_13]] step %[[VAL_5]] unordered iter_args(%[[VAL_16:.*]] = %[[VAL_8]], %[[VAL_17:.*]] = %[[VAL_8]], %[[VAL_18:.*]] = %[[VAL_8]], %[[VAL_19:.*]] = %[[VAL_7]], %[[VAL_20:.*]] = %[[VAL_6]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:             %[[VAL_21:.*]]:5 = fir.do_loop %[[VAL_22:.*]] = %[[VAL_5]] to %[[VAL_12]] step %[[VAL_5]] unordered iter_args(%[[VAL_23:.*]] = %[[VAL_16]], %[[VAL_24:.*]] = %[[VAL_17]], %[[VAL_25:.*]] = %[[VAL_18]], %[[VAL_26:.*]] = %[[VAL_19]], %[[VAL_27:.*]] = %[[VAL_20]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:               %[[VAL_28:.*]]:5 = fir.do_loop %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_5]] unordered iter_args(%[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_24]], %[[VAL_32:.*]] = %[[VAL_25]], %[[VAL_33:.*]] = %[[VAL_26]], %[[VAL_34:.*]] = %[[VAL_27]]) -> (i32, i32, i32, f32, i1) {
+// CHECK:                 %[[VAL_35:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?x!fir.logical<4>>, index, index, index) -> !fir.logical<4>
+// CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.logical<4>) -> i1
+// CHECK:                 %[[VAL_37:.*]]:5 = fir.if %[[VAL_36]] -> (i32, i32, i32, f32, i1) {
+// CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_29]], %[[VAL_22]], %[[VAL_15]] : (!hlfir.expr<?x?x?xf32>, index, index, index) -> f32
+// CHECK:                   %[[VAL_39:.*]] = arith.cmpf olt, %[[VAL_38]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_40:.*]] = arith.cmpf une, %[[VAL_33]], %[[VAL_33]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_41:.*]] = arith.cmpf oeq, %[[VAL_38]], %[[VAL_38]] fastmath<reassoc> : f32
+// CHECK:                   %[[VAL_42:.*]] = arith.andi %[[VAL_40]], %[[VAL_41]] : i1
+// CHECK:                   %[[VAL_43:.*]] = arith.ori %[[VAL_39]], %[[VAL_42]] : i1
+// CHECK:                   %[[VAL_44:.*]] = arith.ori %[[VAL_43]], %[[VAL_34]] : i1
+// CHECK:                   %[[VAL_45:.*]] = fir.convert %[[VAL_29]] : (index) -> i32
+// CHECK:                   %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_30]] : i32
+// CHECK:                   %[[VAL_47:.*]] = fir.convert %[[VAL_22]] : (index) -> i32
+// CHECK:                   %[[VAL_48:.*]] = arith.select %[[VAL_44]], %[[VAL_47]], %[[VAL_31]] : i32
+// CHECK:                   %[[VAL_49:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
+// CHECK:                   %[[VAL_50:.*]] = arith.select %[[VAL_44]], %[[VAL_49]], %[[VAL_32]] : i32
+// CHECK:                   %[[VAL_51:.*]] = arith.select %[[VAL_44]], %[[VAL_38]], %[[VAL_33]] : f32
+// CHECK:                   fir.result %[[VAL_46]], %[[VAL_48]], %[[VAL_50]], %[[VAL_51]], %[[VAL_4]] : i32, i32, i32, f32, i1
+// CHECK:                 } else {
+// CHECK:                   fir.result %[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_33]], %[[VAL_34]] : i32, i32, i32, f32, i1
+// CHECK:                 }
+// CHECK:                 fir.result %[[VAL_52:.*]]#0, %[[VAL_52]]#1, %[[VAL_52]]#2, %[[VAL_52]]#3, %[[VAL_52]]#4 : i32, i32, i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_53:.*]]#0, %[[VAL_53]]#1, %[[VAL_53]]#2, %[[VAL_53]]#3, %[[VAL_53]]#4 : i32, i32, i32, f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_54:.*]]#0, %[[VAL_54]]#1, %[[VAL_54]]#2, %[[VAL_54]]#3, %[[VAL_54]]#4 : i32, i32, i32, f32, i1
+// CHECK:           }
+// CHECK:           %[[VAL_58:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_5]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_14]]#0 to %[[VAL_58]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_61:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_3]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_14]]#1 to %[[VAL_61]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_64:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_2]])  : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+// CHECK:           hlfir.assign %[[VAL_14]]#2 to %[[VAL_64]] : i32, !fir.ref<i32>
+// CHECK:           %[[VAL_65:.*]] = hlfir.as_expr %[[VAL_9]] move %[[VAL_4]] : (!fir.ref<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
+// CHECK:           return %[[VAL_65]] : !hlfir.expr<3xi32>
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?x?xf32>>, %mask: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.minloc %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?x?xf32>>, i32, !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32>
+  return %0 : !hlfir.expr<?x?xi32>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?xf32>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_9]]#1, %[[VAL_11]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_13:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>) -> i1
+// CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_12]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xi32> {
+// CHECK:           ^bb0(%[[VAL_15:.*]]: index, %[[VAL_16:.*]]: index):
+// CHECK:             %[[VAL_17:.*]]:3 = fir.do_loop %[[VAL_18:.*]] = %[[VAL_7]] to %[[VAL_10]]#1 step %[[VAL_7]] unordered iter_args(%[[VAL_19:.*]] = %[[VAL_5]], %[[VAL_20:.*]] = %[[VAL_4]], %[[VAL_21:.*]] = %[[VAL_3]]) -> (i32, f32, i1) {
+// CHECK:               %[[VAL_22:.*]] = fir.if %[[VAL_13]] -> (!fir.logical<4>) {
+// CHECK:                 %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_24:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_26:.*]] = arith.subi %[[VAL_23]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_15]], %[[VAL_26]] : index
+// CHECK:                 %[[VAL_28:.*]] = arith.subi %[[VAL_24]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_29:.*]] = arith.addi %[[VAL_18]], %[[VAL_28]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.subi %[[VAL_25]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.addi %[[VAL_16]], %[[VAL_30]] : index
+// CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_27]], %[[VAL_29]], %[[VAL_31]])  : (!fir.box<!fir.array<?x?x?x!fir.logical<4>>>, index, index, index) -> !fir.ref<!fir.logical<4>>
+// CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_32]] : !fir.ref<!fir.logical<4>>
+// CHECK:                 fir.result %[[VAL_33]] : !fir.logical<4>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_34:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+// CHECK:                 fir.result %[[VAL_34]] : !fir.logical<4>
+// CHECK:               }
+// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_22]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_36:.*]]:3 = fir.if %[[VAL_35]] -> (i32, f32, i1) {
+// CHECK:                 %[[VAL_37:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_8]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_38:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_7]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_6]] : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_37]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_15]], %[[VAL_40]] : index
+// CHECK:                 %[[VAL_42:.*]] = arith.subi %[[VAL_38]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_18]], %[[VAL_42]] : index
+// CHECK:                 %[[VAL_44:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_7]] : index
+// CHECK:                 %[[VAL_45:.*]] = arith.addi %[[VAL_16]], %[[VAL_44]] : index
+// CHECK:                 %[[VAL_46:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_41]], %[[VAL_43]], %[[VAL_45]])  : (!fir.box<!fir.array<?x?x?xf32>>, index, index, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_48:.*]] = arith.cmpf olt, %[[VAL_47]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_49:.*]] = arith.cmpf une, %[[VAL_20]], %[[VAL_20]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_50:.*]] = arith.cmpf oeq, %[[VAL_47]], %[[VAL_47]] fastmath<reassoc> : f32
+// CHECK:                 %[[VAL_51:.*]] = arith.andi %[[VAL_49]], %[[VAL_50]] : i1
+// CHECK:                 %[[VAL_52:.*]] = arith.ori %[[VAL_48]], %[[VAL_51]] : i1
+// CHECK:                 %[[VAL_53:.*]] = arith.ori %[[VAL_52]], %[[VAL_21]] : i1
+// CHECK:                 %[[VAL_54:.*]] = fir.convert %[[VAL_18]] : (index) -> i32
+// CHECK:                 %[[VAL_55:.*]] = arith.select %[[VAL_53]], %[[VAL_54]], %[[VAL_19]] : i32
+// CHECK:                 %[[VAL_56:.*]] = arith.select %[[VAL_53]], %[[VAL_47]], %[[VAL_20]] : f32
+// CHECK:                 fir.result %[[VAL_55]], %[[VAL_56]], %[[VAL_2]] : i32, f32, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_19]], %[[VAL_20]], %[[VAL_21]] : i32, f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_57:.*]]#0, %[[VAL_57]]#1, %[[VAL_57]]#2 : i32, f32, i1
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_17]]#0 : i32
+// CHECK:           }
+// CHECK:           return %[[VAL_14]] : !hlfir.expr<?x?xi32>
+// CHECK:         }
+
+// Character comparisons are not supported yet.
+func.func @test_character(%input: !fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32> {
+  %0 = hlfir.minloc %input : (!fir.box<!fir.array<?x!fir.char<1>>>) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_character(
+// CHECK:           hlfir.minloc
+
+// BACK is not supported yet.
+func.func @test_back(%input: !hlfir.expr<?xi32>) -> !hlfir.expr<1xi32> {
+  %back = arith.constant true
+  %0 = hlfir.minloc %input back %back : (!hlfir.expr<?xi32>, i1) -> !hlfir.expr<1xi32>
+  return %0 : !hlfir.expr<1xi32>
+}
+// CHECK-LABEL:   func.func @test_back(
+// CHECK:           hlfir.minloc
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
new file mode 100644
index 0000000000000..98e4c692fb72f
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-minval.fir
@@ -0,0 +1,186 @@
+// RUN: fir-opt %s --simplify-hlfir-intrinsics | FileCheck %s
+
+func.func @test_total_expr(%input: !hlfir.expr<?x?xf32>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?x!fir.logical<4>>) -> f32
+  return %0 : f32
+}
+// CHECK-LABEL:   func.func @test_total_expr(
+// CHECK-SAME:                               %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf32>,
+// CHECK-SAME:                               %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> f32 {
+// CHECK:           %[[FALSE:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[TRUE:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant 3.40282347E+38 : f32
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]]:2 = fir.do_loop %[[VAL_8:.*]] = %[[VAL_2]] to %[[VAL_6]] step %[[VAL_2]] iter_args(%[[VAL_9:.*]] = %[[VAL_3]], %[[FIRST1:.*]] = %[[TRUE]]) -> (f32, i1) {
+// CHECK:             %[[VAL_10:.*]]:2 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] iter_args(%[[VAL_12:.*]] = %[[VAL_9]], %[[FIRST2:.*]] = %[[FIRST1]]) -> (f32, i1) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]]:2 = fir.if %[[VAL_14]] -> (f32, i1) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_8]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[IS_FIRST:.*]] = arith.ori %[[VAL_21]], %[[FIRST2]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[IS_FIRST]], %[[VAL_16]], %[[VAL_12]] : f32
+// CHECK:                 fir.result %[[VAL_22]], %[[FALSE]] : f32, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]], %[[FIRST2]] : f32, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]]#0, %[[VAL_15]]#1 : f32, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]]#0, %[[VAL_10]]#1 : f32, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_7]]#0 : f32
+// CHECK:         }
+
+func.func @test_partial_expr(%input: !hlfir.expr<?x?xf64>, %mask: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+  %dim = arith.constant 1 : i32
+  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!hlfir.expr<?x?xf64>, i32, !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64>
+  return %0 : !hlfir.expr<?xf64>
+}
+// CHECK-LABEL:   func.func @test_partial_expr(
+// CHECK-SAME:                                 %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?xf64>,
+// CHECK-SAME:                                 %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !hlfir.expr<?x?x!fir.logical<4>>) -> !hlfir.expr<?xf64> {
+// CHECK:           %[[FALSE:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[TRUE:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1.7976931348623157E+308 : f64
+// CHECK:           %[[VAL_4:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xf64>) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_6:.*]] = hlfir.get_extent %[[VAL_4]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf64> {
+// CHECK:           ^bb0(%[[VAL_9:.*]]: index):
+// CHECK:             %[[VAL_10:.*]]:2 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_2]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_3]], %[[FIRST:.*]] = %[[TRUE]]) -> (f64, i1) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.apply %[[VAL_1]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?x!fir.logical<4>>, index, index) -> !fir.logical<4>
+// CHECK:               %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.logical<4>) -> i1
+// CHECK:               %[[VAL_15:.*]]:2 = fir.if %[[VAL_14]] -> (f64, i1) {
+// CHECK:                 %[[VAL_16:.*]] = hlfir.apply %[[VAL_0]], %[[VAL_11]], %[[VAL_9]] : (!hlfir.expr<?x?xf64>, index, index) -> f64
+// CHECK:                 %[[VAL_17:.*]] = arith.cmpf olt, %[[VAL_16]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_18:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_19:.*]] = arith.cmpf oeq, %[[VAL_16]], %[[VAL_16]] fastmath<reassoc> : f64
+// CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i1
+// CHECK:                 %[[VAL_21:.*]] = arith.ori %[[VAL_17]], %[[VAL_20]] : i1
+// CHECK:                 %[[IS_FIRST:.*]] = arith.ori %[[VAL_21]], %[[FIRST]] : i1
+// CHECK:                 %[[VAL_22:.*]] = arith.select %[[IS_FIRST]], %[[VAL_16]], %[[VAL_12]] : f64
+// CHECK:                 fir.result %[[VAL_22]], %[[FALSE]] : f64, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]], %[[FIRST]] : f64, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_15]]#0, %[[VAL_15]]#1 : f64, i1
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_10]]#0 : f64
+// CHECK:           }
+// CHECK:           return %[[VAL_8]] : !hlfir.expr<?xf64>
+// CHECK:         }
+
+func.func @test_total_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+  %0 = hlfir.minval %input mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16
+  return %0 : f16
+}
+// CHECK-LABEL:   func.func @test_total_var(
+// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                              %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.array<2x2x!fir.logical<1>>>) -> f16 {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 6.550400e+04 : f16
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:2 = fir.do_loop %[[VAL_8:.*]] = %[[VAL_3]] to %[[VAL_6]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_9:.*]] = %[[VAL_2]], %[[FIRST1:.*]] = %[[TRUE]]) -> (f16, i1) {
+// CHECK:             %[[VAL_10:.*]]:2 = fir.do_loop %[[VAL_11:.*]] = %[[VAL_3]] to %[[VAL_5]]#1 step %[[VAL_3]] unordered iter_args(%[[VAL_12:.*]] = %[[VAL_9]], %[[FIRST2:.*]] = %[[FIRST1]]) -> (f16, i1) {
+// CHECK:               %[[VAL_13:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_11]], %[[VAL_8]])  : (!fir.ref<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<!fir.logical<1>>
+// CHECK:               %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_16:.*]]:2 = fir.if %[[VAL_15]] -> (f16, i1) {
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_19:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.addi %[[VAL_11]], %[[VAL_19]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.subi %[[VAL_18]]#0, %[[VAL_3]] : index
+// CHECK:                 %[[VAL_22:.*]] = arith.addi %[[VAL_8]], %[[VAL_21]] : index
+// CHECK:                 %[[VAL_23:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_20]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_25:.*]] = arith.cmpf olt, %[[VAL_24]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_12]], %[[VAL_12]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_27:.*]] = arith.cmpf oeq, %[[VAL_24]], %[[VAL_24]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_28:.*]] = arith.andi %[[VAL_26]], %[[VAL_27]] : i1
+// CHECK:                 %[[VAL_29:.*]] = arith.ori %[[VAL_25]], %[[VAL_28]] : i1
+// CHECK:                 %[[IS_FIRST:.*]] = arith.ori %[[VAL_29]], %[[FIRST2]] : i1
+// CHECK:                 %[[VAL_30:.*]] = arith.select %[[IS_FIRST]], %[[VAL_24]], %[[VAL_12]] : f16
+// CHECK:                 fir.result %[[VAL_30]], %[[FALSE]] : f16, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_12]], %[[FIRST2]] : f16, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_16]]#0, %[[VAL_16]]#1 : f16, i1
+// CHECK:             }
+// CHECK:             fir.result %[[VAL_10]]#0, %[[VAL_10]]#1 : f16, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_7]]#0 : f16
+// CHECK:         }
+
+func.func @test_partial_var(%input: !fir.box<!fir.array<?x?xf16>>, %mask: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+  %dim = arith.constant 2 : i32
+  %0 = hlfir.minval %input dim %dim mask %mask {fastmath = #arith.fastmath<reassoc>} : (!fir.box<!fir.array<?x?xf16>>, i32, !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16>
+  return %0 : !hlfir.expr<?xf16>
+}
+// CHECK-LABEL:   func.func @test_partial_var(
+// CHECK-SAME:                                %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?x?xf16>>,
+// CHECK-SAME:                                %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<2x2x!fir.logical<1>>>) -> !hlfir.expr<?xf16> {
+// CHECK:           %[[FALSE:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant 6.550400e+04 : f16
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_1]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>) -> i1
+// CHECK:           %[[VAL_10:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index):
+// CHECK:             %[[VAL_12:.*]]:2 = fir.do_loop %[[VAL_13:.*]] = %[[VAL_4]] to %[[VAL_7]]#1 step %[[VAL_4]] unordered iter_args(%[[VAL_14:.*]] = %[[VAL_3]], %[[FIRST:.*]] = %[[TRUE]]) -> (f16, i1) {
+// CHECK:               %[[VAL_15:.*]] = fir.if %[[VAL_9]] -> (!fir.logical<1>) {
+// CHECK:                 %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_18:.*]] = arith.subi %[[VAL_16]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_19:.*]] = arith.addi %[[VAL_11]], %[[VAL_18]] : index
+// CHECK:                 %[[VAL_20:.*]] = arith.subi %[[VAL_17]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_20]] : index
+// CHECK:                 %[[VAL_22:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_19]], %[[VAL_21]])  : (!fir.box<!fir.array<2x2x!fir.logical<1>>>, index, index) -> !fir.ref<!fir.logical<1>>
+// CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<1>>
+// CHECK:                 fir.result %[[VAL_23]] : !fir.logical<1>
+// CHECK:               } else {
+// CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<1>
+// CHECK:                 fir.result %[[VAL_24]] : !fir.logical<1>
+// CHECK:               }
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_15]] : (!fir.logical<1>) -> i1
+// CHECK:               %[[VAL_26:.*]]:2 = fir.if %[[VAL_25]] -> (f16, i1) {
+// CHECK:                 %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf16>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_29:.*]] = arith.subi %[[VAL_27]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_30:.*]] = arith.addi %[[VAL_11]], %[[VAL_29]] : index
+// CHECK:                 %[[VAL_31:.*]] = arith.subi %[[VAL_28]]#0, %[[VAL_4]] : index
+// CHECK:                 %[[VAL_32:.*]] = arith.addi %[[VAL_13]], %[[VAL_31]] : index
+// CHECK:                 %[[VAL_33:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_30]], %[[VAL_32]])  : (!fir.box<!fir.array<?x?xf16>>, index, index) -> !fir.ref<f16>
+// CHECK:                 %[[VAL_34:.*]] = fir.load %[[VAL_33]] : !fir.ref<f16>
+// CHECK:                 %[[VAL_35:.*]] = arith.cmpf olt, %[[VAL_34]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_14]], %[[VAL_14]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_37:.*]] = arith.cmpf oeq, %[[VAL_34]], %[[VAL_34]] fastmath<reassoc> : f16
+// CHECK:                 %[[VAL_38:.*]] = arith.andi %[[VAL_36]], %[[VAL_37]] : i1
+// CHECK:                 %[[VAL_39:.*]] = arith.ori %[[VAL_35]], %[[VAL_38]] : i1
+// CHECK:                 %[[IS_FIRST:.*]] = arith.ori %[[VAL_39]], %[[FIRST]] : i1
+// CHECK:                 %[[VAL_40:.*]] = arith.select %[[IS_FIRST]], %[[VAL_34]], %[[VAL_14]] : f16
+// CHECK:                 fir.result %[[VAL_40]], %[[FALSE]] : f16, i1
+// CHECK:               } else {
+// CHECK:                 fir.result %[[VAL_14]], %[[FIRST]] : f16, i1
+// CHECK:               }
+// CHECK:               fir.result %[[VAL_26]]#0, %[[VAL_26]]#1 : f16, i1
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_12]]#0 : f16
+// CHECK:           }
+// CHECK:           return %[[VAL_10]] : !hlfir.expr<?xf16>
+// CHECK:         }

From 32503013ea49f74bb63983705b6316b158d74bda Mon Sep 17 00:00:00 2001
From: "Oleksandr \"Alex\" Zinenko" <git@ozinenko.com>
Date: Fri, 18 Apr 2025 20:56:36 +0200
Subject: [PATCH 457/710] [mlir] harden transform.test_emit_remark_and_erase
 (#136307)

Executing this transform op may crash when the payload op is not
erasable. Guard against that. Since this is a test op, no additional
test is defined.

Fixes #128323.
---
 .../lib/Dialect/Transform/TestTransformDialectExtension.cpp  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
index face4de8e27d3..cdf44c2959d50 100644
--- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
+++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
@@ -346,8 +346,11 @@ DiagnosedSilenceableFailure mlir::test::TestEmitRemarkAndEraseOperandOp::apply(
     transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   emitRemark() << getRemark();
-  for (Operation *op : state.getPayloadOps(getTarget()))
+  for (Operation *op : state.getPayloadOps(getTarget())) {
+    if (!op->getUses().empty())
+      return emitSilenceableError() << "cannot erase an op that has uses";
     rewriter.eraseOp(op);
+  }
 
   if (getFailAfterErase())
     return emitSilenceableError() << "silenceable error";

From 28c0d9d37c1f088855119ec4488943d2b4cded8c Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 12:03:49 -0700
Subject: [PATCH 458/710] [mlir] Use llvm::append_range (NFC) (#136257)

This patch replaces:

  llvm::copy(Src, std::back_inserter(Dst));

with:

  llvm::append_range(Dst, Src);

for breavity.

One side benefit is that llvm::append_range eventually calls
llvm::SmallVector::reserve if Dst is of llvm::SmallVector.
---
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp | 4 ++--
 mlir/lib/Dialect/SCF/IR/SCF.cpp     | 6 +++---
 mlir/lib/Dialect/Traits.cpp         | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index b4d7482554fbc..14cd8e4920259 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -1337,7 +1337,7 @@ Block &emitc::SwitchOp::getCaseBlock(unsigned idx) {
 
 void SwitchOp::getSuccessorRegions(
     RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &successors) {
-  llvm::copy(getRegions(), std::back_inserter(successors));
+  llvm::append_range(successors, getRegions());
 }
 
 void SwitchOp::getEntrySuccessorRegions(
@@ -1348,7 +1348,7 @@ void SwitchOp::getEntrySuccessorRegions(
   // If a constant was not provided, all regions are possible successors.
   auto arg = dyn_cast_or_null<IntegerAttr>(adaptor.getArg());
   if (!arg) {
-    llvm::copy(getRegions(), std::back_inserter(successors));
+    llvm::append_range(successors, getRegions());
     return;
   }
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index c454f342b4204..ad65a5fdfe4dc 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -758,7 +758,7 @@ LoopNest mlir::scf::buildLoopNest(
 
   // Return the loops.
   ValueVector nestResults;
-  llvm::copy(loops.front().getResults(), std::back_inserter(nestResults));
+  llvm::append_range(nestResults, loops.front().getResults());
   return LoopNest{std::move(loops), std::move(nestResults)};
 }
 
@@ -4281,7 +4281,7 @@ void IndexSwitchOp::getSuccessorRegions(
     return;
   }
 
-  llvm::copy(getRegions(), std::back_inserter(successors));
+  llvm::append_range(successors, getRegions());
 }
 
 void IndexSwitchOp::getEntrySuccessorRegions(
@@ -4292,7 +4292,7 @@ void IndexSwitchOp::getEntrySuccessorRegions(
   // If a constant was not provided, all regions are possible successors.
   auto arg = dyn_cast_or_null<IntegerAttr>(adaptor.getArg());
   if (!arg) {
-    llvm::copy(getRegions(), std::back_inserter(successors));
+    llvm::append_range(successors, getRegions());
     return;
   }
 
diff --git a/mlir/lib/Dialect/Traits.cpp b/mlir/lib/Dialect/Traits.cpp
index 6e62a33037eb8..2d43a1e52a0b1 100644
--- a/mlir/lib/Dialect/Traits.cpp
+++ b/mlir/lib/Dialect/Traits.cpp
@@ -70,9 +70,9 @@ bool OpTrait::util::getBroadcastedShape(ArrayRef<int64_t> shape1,
 
   resultShape.clear();
   if (shape1.size() > shape2.size()) {
-    std::copy(shape1.begin(), shape1.end(), std::back_inserter(resultShape));
+    llvm::append_range(resultShape, shape1);
   } else {
-    std::copy(shape2.begin(), shape2.end(), std::back_inserter(resultShape));
+    llvm::append_range(resultShape, shape2);
   }
 
   auto i1 = shape1.rbegin(), e1 = shape1.rend();

From c62afbfeda07a2018d167b2724c8dd4cf2b39920 Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Fri, 18 Apr 2025 15:05:27 -0400
Subject: [PATCH 459/710] [mlir][linalg][gpu] Clean up printing. NFC. (#136330)

* Use `llvm::interleaved` from #135517 to simplify printing
* Avoid needless vector allocations
---
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp   |  4 +--
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp | 40 ++++++++----------------
 2 files changed, 14 insertions(+), 30 deletions(-)

diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 9391d2c4ec840..f20126618060a 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -451,9 +451,7 @@ static void printAsyncDependencies(OpAsmPrinter &printer, Operation *op,
     return;
   if (asyncTokenType)
     printer << ' ';
-  printer << '[';
-  llvm::interleaveComma(asyncDependencies, printer);
-  printer << ']';
+  printer << llvm::interleaved_array(asyncDependencies);
 }
 
 // GPU Memory attributions functions shared by LaunchOp and GPUFuncOp.
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 1c160911ce780..6c680498af2ad 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -47,6 +47,7 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
@@ -3660,17 +3661,13 @@ ParseResult MatmulOp::parse(OpAsmParser &parser, OperationState &result) {
 }
 
 void MatmulOp::print(OpAsmPrinter &p) {
-  SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector(
+  SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector<3>(
       MatmulOp::getDefaultIndexingMaps(getContext()),
       [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
-  if (!llvm::equal(getIndexingMaps(), indexingMaps)) {
-    p << " indexing_maps = [";
-    llvm::interleaveComma(getIndexingMaps(), p,
-                          [&](Attribute attr) { p.printAttribute(attr); });
-    p << "]";
-  }
+  if (!llvm::equal(getIndexingMaps(), indexingMaps))
+    p << " indexing_maps = " << llvm::interleaved_array(getIndexingMaps());
 
-  SmallVector<StringRef, 3> elidedAttrs = {
+  std::array<StringRef, 3> elidedAttrs = {
       "operandSegmentSizes", "linalg.memoized_indexing_maps", "indexing_maps"};
   printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(),
                          elidedAttrs);
@@ -3777,10 +3774,7 @@ ParseResult ContractOp::parse(OpAsmParser &parser, OperationState &result) {
 }
 
 void ContractOp::print(OpAsmPrinter &p) {
-  p << " indexing_maps = [";
-  llvm::interleaveComma(getIndexingMaps(), p,
-                        [&](Attribute attr) { p.printAttribute(attr); });
-  p << "]";
+  p << " indexing_maps = " << llvm::interleaved_array(getIndexingMaps());
   printNamedStructuredOp(
       p, getOperation(), getInputs(), getOutputs(),
       /*elidedAttrs=*/{"indexing_maps", "operandSegmentSizes"});
@@ -4013,17 +4007,13 @@ ParseResult BatchMatmulOp::parse(OpAsmParser &parser, OperationState &result) {
 }
 
 void BatchMatmulOp::print(OpAsmPrinter &p) {
-  SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector(
+  SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector<3>(
       BatchMatmulOp::getDefaultIndexingMaps(getContext()),
       [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
-  if (!llvm::equal(getIndexingMaps(), indexingMaps)) {
-    p << " indexing_maps = [";
-    llvm::interleaveComma(getIndexingMaps(), p,
-                          [&](Attribute attr) { p.printAttribute(attr); });
-    p << "]";
-  }
+  if (!llvm::equal(getIndexingMaps(), indexingMaps))
+    p << " indexing_maps = " << llvm::interleaved_array(getIndexingMaps());
 
-  SmallVector<StringRef, 3> elidedAttrs = {
+  std::array<StringRef, 3> elidedAttrs = {
       "operandSegmentSizes", "linalg.memoized_indexing_maps", "indexing_maps"};
   ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(),
                            elidedAttrs);
@@ -4204,17 +4194,13 @@ void ElementwiseOp::print(OpAsmPrinter &p) {
       getArityGroupAsUInt(getArityGroupAndKind(getKind()).arityGroup);
   unsigned numDims = getResultRank();
 
-  SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector(
+  SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector<3>(
       ElementwiseOp::getDefaultIndexingMaps(arity + 1 /*output*/, numDims,
                                             getContext()),
       [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
 
-  if (!llvm::equal(getIndexingMaps(), indexingMaps)) {
-    p << " indexing_maps = [";
-    llvm::interleaveComma(getIndexingMaps(), p,
-                          [&](Attribute attr) { p.printAttribute(attr); });
-    p << "]";
-  }
+  if (!llvm::equal(getIndexingMaps(), indexingMaps))
+    p << " indexing_maps = " << llvm::interleaved_array(getIndexingMaps());
 
   printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(),
                          elidedAttrs);

From 430b0c44341f913552a7badf1860acc4a84033ca Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 21:13:21 +0200
Subject: [PATCH 460/710] CodeGenPrepare: Check use_empty instead of getNumUses
 == 0 (#136334)

---
 llvm/lib/CodeGen/CodeGenPrepare.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 0cc9fb88b4293..12a668507fe65 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -4194,7 +4194,7 @@ class AddressingModeCombiner {
   /// `CommonValue` may be a placeholder inserted by us.
   /// If the placeholder is not used, we should remove this dead instruction.
   void eraseCommonValueIfDead() {
-    if (CommonValue && CommonValue->getNumUses() == 0)
+    if (CommonValue && CommonValue->use_empty())
       if (Instruction *CommonInst = dyn_cast<Instruction>(CommonValue))
         CommonInst->eraseFromParent();
   }

From e2886705f010762d27d090cf98e11ef5169d5cf4 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 21:14:06 +0200
Subject: [PATCH 461/710] SLPVectorizer: Use use_empty instead of getNumUses
 (#136336)

---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 53bf03c57db65..70cd201ab20a6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6751,7 +6751,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
     auto PHICompare = [&](unsigned I1, unsigned I2) {
       Value *V1 = TE.Scalars[I1];
       Value *V2 = TE.Scalars[I2];
-      if (V1 == V2 || (V1->getNumUses() == 0 && V2->getNumUses() == 0))
+      if (V1 == V2 || (V1->use_empty() && V2->use_empty()))
         return false;
       if (isa<PoisonValue>(V1))
         return true;

From cfc035a2b1cc2b79b9b73320f7501dac0328685d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 21:14:13 +0200
Subject: [PATCH 462/710] Attributor: Use use_empty instead of getNumUses == 0
 (#136339)

---
 llvm/lib/Transforms/IPO/Attributor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 4d65354455379..5ef0410ee3616 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -3864,7 +3864,7 @@ static bool runAttributorOnFunctions(InformationCache &InfoCache,
     unsigned FunSize = Functions.size();
     for (unsigned u = 0; u < FunSize; u++) {
       Function *F = Functions[u];
-      if (!F->isDeclaration() && !F->isDefinitionExact() && F->getNumUses() &&
+      if (!F->isDeclaration() && !F->isDefinitionExact() && !F->use_empty() &&
           !GlobalValue::isInterposableLinkage(F->getLinkage())) {
         Function *NewF = Attributor::internalizeFunction(*F);
         assert(NewF && "Could not internalize function.");

From 34f34665ac11a6bbbc641696ffd869cce5b90920 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 21:18:00 +0200
Subject: [PATCH 463/710] LowerGlobalDtors: Use use_empty instead of getNumUses
 == 0 (#136337)

---
 llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp b/llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp
index ff72ba073ad0e..df3a2a94b7ddb 100644
--- a/llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp
+++ b/llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp
@@ -146,7 +146,7 @@ static bool runImpl(Module &M) {
   // the program never exits) we can simply return early and clear out
   // @llvm.global_dtors.
   if (auto F = dyn_cast<Function>(AtExit.getCallee())) {
-    if (F && F->hasExactDefinition() && F->getArg(0)->getNumUses() == 0) {
+    if (F && F->hasExactDefinition() && F->getArg(0)->use_empty()) {
       GV->eraseFromParent();
       return true;
     }

From 21a406c92c6cd79116a03443bccc47c1700d73db Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:48:33 -0700
Subject: [PATCH 464/710] [flang] Improve runtime SAME_TYPE_AS() (#135670)

The present implementation of the intrinsic function SAME_TYPE_AS()
yields false positive .TRUE. results for distinct derived types that
happen to have the same name.

Replace with an implementation that can now depend on derived type
information records being the same type if and only if they are at the
same location, or are PDT instantiations of the same uninstantiated
derived type. And ensure that the derived type information includes
references from instantiated PDTs to their original types. (The derived
type information format supports these references already, but they were
not being set, perhaps because the current faulty SAME_TYPE_AS
implementation didn't need them, and nothing else does.)

Fixes https://github.com/llvm/llvm-project/issues/135580.
---
 flang-rt/include/flang-rt/runtime/type-info.h |  6 ++-
 flang-rt/lib/runtime/derived-api.cpp          | 50 ++++++-------------
 flang/lib/Semantics/runtime-type-info.cpp     | 23 +++++----
 flang/test/Semantics/typeinfo01.f90           |  3 +-
 4 files changed, 35 insertions(+), 47 deletions(-)

diff --git a/flang-rt/include/flang-rt/runtime/type-info.h b/flang-rt/include/flang-rt/runtime/type-info.h
index 9891fcecdcb25..5e79efde164f2 100644
--- a/flang-rt/include/flang-rt/runtime/type-info.h
+++ b/flang-rt/include/flang-rt/runtime/type-info.h
@@ -210,9 +210,13 @@ class DerivedType {
   }
   RT_API_ATTRS const Descriptor &name() const { return name_.descriptor(); }
   RT_API_ATTRS std::uint64_t sizeInBytes() const { return sizeInBytes_; }
-  RT_API_ATTRS const Descriptor &uninstatiated() const {
+  RT_API_ATTRS const Descriptor &uninstantiated() const {
     return uninstantiated_.descriptor();
   }
+  RT_API_ATTRS const DerivedType *uninstantiatedType() const {
+    return reinterpret_cast<const DerivedType *>(
+        uninstantiated().raw().base_addr);
+  }
   RT_API_ATTRS const Descriptor &kindParameter() const {
     return kindParameter_.descriptor();
   }
diff --git a/flang-rt/lib/runtime/derived-api.cpp b/flang-rt/lib/runtime/derived-api.cpp
index 884fa8ee7d095..bb08e0397fe9c 100644
--- a/flang-rt/lib/runtime/derived-api.cpp
+++ b/flang-rt/lib/runtime/derived-api.cpp
@@ -83,27 +83,6 @@ bool RTDEF(ClassIs)(
   return false;
 }
 
-static RT_API_ATTRS bool CompareDerivedTypeNames(
-    const Descriptor &a, const Descriptor &b) {
-  if (a.raw().version == CFI_VERSION &&
-      a.type() == TypeCode{TypeCategory::Character, 1} &&
-      a.ElementBytes() > 0 && a.rank() == 0 && a.OffsetElement() != nullptr &&
-      a.raw().version == CFI_VERSION &&
-      b.type() == TypeCode{TypeCategory::Character, 1} &&
-      b.ElementBytes() > 0 && b.rank() == 0 && b.OffsetElement() != nullptr &&
-      a.ElementBytes() == b.ElementBytes() &&
-      Fortran::runtime::memcmp(
-          a.OffsetElement(), b.OffsetElement(), a.ElementBytes()) == 0) {
-    return true;
-  }
-  return false;
-}
-
-inline RT_API_ATTRS bool CompareDerivedType(
-    const typeInfo::DerivedType *a, const typeInfo::DerivedType *b) {
-  return a == b || CompareDerivedTypeNames(a->name(), b->name());
-}
-
 static RT_API_ATTRS const typeInfo::DerivedType *GetDerivedType(
     const Descriptor &desc) {
   if (const DescriptorAddendum * addendum{desc.Addendum()}) {
@@ -121,22 +100,21 @@ bool RTDEF(SameTypeAs)(const Descriptor &a, const Descriptor &b) {
       (bType != CFI_type_struct && bType != CFI_type_other)) {
     // If either type is intrinsic, they must match.
     return aType == bType;
-  } else {
-    const typeInfo::DerivedType *derivedTypeA{GetDerivedType(a)};
-    const typeInfo::DerivedType *derivedTypeB{GetDerivedType(b)};
-    if (derivedTypeA == nullptr || derivedTypeB == nullptr) {
-      // Unallocated/disassociated CLASS(*) never matches.
-      return false;
-    } else if (derivedTypeA == derivedTypeB) {
-      // Exact match of derived type.
-      return true;
-    } else {
-      // Otherwise compare with the name. Note 16.29 kind type parameters are
-      // not considered in the test.
-      return CompareDerivedTypeNames(
-          derivedTypeA->name(), derivedTypeB->name());
+  } else if (const typeInfo::DerivedType * derivedTypeA{GetDerivedType(a)}) {
+    if (const typeInfo::DerivedType * derivedTypeB{GetDerivedType(b)}) {
+      if (derivedTypeA == derivedTypeB) {
+        return true;
+      } else if (const typeInfo::DerivedType *
+          uninstDerivedTypeA{derivedTypeA->uninstantiatedType()}) {
+        // There are KIND type parameters, are these the same type if those
+        // are ignored?
+        const typeInfo::DerivedType *uninstDerivedTypeB{
+            derivedTypeB->uninstantiatedType()};
+        return uninstDerivedTypeA == uninstDerivedTypeB;
+      }
     }
   }
+  return false;
 }
 
 bool RTDEF(ExtendsTypeOf)(const Descriptor &a, const Descriptor &mold) {
@@ -155,7 +133,7 @@ bool RTDEF(ExtendsTypeOf)(const Descriptor &a, const Descriptor &mold) {
     // dynamic type of MOLD.
     for (const typeInfo::DerivedType *derivedTypeA{GetDerivedType(a)};
          derivedTypeA; derivedTypeA = derivedTypeA->GetParentType()) {
-      if (CompareDerivedType(derivedTypeA, derivedTypeMold)) {
+      if (derivedTypeA == derivedTypeMold) {
         return true;
       }
     }
diff --git a/flang/lib/Semantics/runtime-type-info.cpp b/flang/lib/Semantics/runtime-type-info.cpp
index 9f3eb5fbe11a1..98295f3705a71 100644
--- a/flang/lib/Semantics/runtime-type-info.cpp
+++ b/flang/lib/Semantics/runtime-type-info.cpp
@@ -58,7 +58,7 @@ class RuntimeTableBuilder {
   void DescribeTypes(Scope &scope, bool inSchemata);
 
 private:
-  const Symbol *DescribeType(Scope &);
+  const Symbol *DescribeType(Scope &, bool wantUninstantiatedPDT);
   const Symbol &GetSchemaSymbol(const char *) const;
   const DeclTypeSpec &GetSchema(const char *) const;
   SomeExpr GetEnumValue(const char *) const;
@@ -238,7 +238,7 @@ void RuntimeTableBuilder::DescribeTypes(Scope &scope, bool inSchemata) {
   inSchemata |= ignoreScopes_.find(&scope) != ignoreScopes_.end();
   if (scope.IsDerivedType()) {
     if (!inSchemata) { // don't loop trying to describe a schema
-      DescribeType(scope);
+      DescribeType(scope, /*wantUninstantiatedPDT=*/false);
     }
   } else {
     scope.InstantiateDerivedTypes();
@@ -310,10 +310,10 @@ static SomeExpr StructureExpr(evaluate::StructureConstructor &&x) {
   return SomeExpr{evaluate::Expr<evaluate::SomeDerived>{std::move(x)}};
 }
 
-static int GetIntegerKind(const Symbol &symbol) {
+static int GetIntegerKind(const Symbol &symbol, bool canBeUninstantiated) {
   auto dyType{evaluate::DynamicType::From(symbol)};
   CHECK((dyType && dyType->category() == TypeCategory::Integer) ||
-      symbol.owner().context().HasError(symbol));
+      symbol.owner().context().HasError(symbol) || canBeUninstantiated);
   return dyType && dyType->category() == TypeCategory::Integer
       ? dyType->kind()
       : symbol.owner().context().GetDefaultKind(TypeCategory::Integer);
@@ -395,7 +395,8 @@ static std::optional<std::string> GetSuffixIfTypeKindParameters(
   return std::nullopt;
 }
 
-const Symbol *RuntimeTableBuilder::DescribeType(Scope &dtScope) {
+const Symbol *RuntimeTableBuilder::DescribeType(
+    Scope &dtScope, bool wantUninstantiatedPDT) {
   if (const Symbol * info{dtScope.runtimeDerivedTypeDescription()}) {
     return info;
   }
@@ -449,7 +450,7 @@ const Symbol *RuntimeTableBuilder::DescribeType(Scope &dtScope) {
             GetSuffixIfTypeKindParameters(*derivedTypeSpec, parameters)}) {
       distinctName += *suffix;
     }
-  } else if (isPDTDefinitionWithKindParameters) {
+  } else if (isPDTDefinitionWithKindParameters && !wantUninstantiatedPDT) {
     return nullptr;
   }
   std::string dtDescName{(fir::kTypeDescriptorSeparator + distinctName).str()};
@@ -480,7 +481,8 @@ const Symbol *RuntimeTableBuilder::DescribeType(Scope &dtScope) {
   }
   if (const Symbol *
       uninstDescObject{isPDTInstantiation
-              ? DescribeType(DEREF(const_cast<Scope *>(dtSymbol->scope())))
+              ? DescribeType(DEREF(const_cast<Scope *>(dtSymbol->scope())),
+                    /*wantUninstantiatedPDT=*/true)
               : nullptr}) {
     AddValue(dtValues, derivedTypeSchema_, "uninstantiated"s,
         evaluate::AsGenericExpr(evaluate::Expr<evaluate::SomeDerived>{
@@ -516,7 +518,8 @@ const Symbol *RuntimeTableBuilder::DescribeType(Scope &dtScope) {
           }
           kinds.emplace_back(value);
         } else { // LEN= parameter
-          lenKinds.emplace_back(GetIntegerKind(*inst));
+          lenKinds.emplace_back(
+              GetIntegerKind(*inst, isPDTDefinitionWithKindParameters));
         }
       }
     }
@@ -804,7 +807,9 @@ evaluate::StructureConstructor RuntimeTableBuilder::DescribeComponent(
     const DerivedTypeSpec &spec{dyType.GetDerivedTypeSpec()};
     Scope *derivedScope{const_cast<Scope *>(
         spec.scope() ? spec.scope() : spec.typeSymbol().scope())};
-    if (const Symbol * derivedDescription{DescribeType(DEREF(derivedScope))}) {
+    if (const Symbol *
+        derivedDescription{DescribeType(
+            DEREF(derivedScope), /*wantUninstantiatedPDT=*/false)}) {
       AddValue(values, componentSchema_, "derived"s,
           evaluate::AsGenericExpr(evaluate::Expr<evaluate::SomeDerived>{
               evaluate::Designator<evaluate::SomeDerived>{
diff --git a/flang/test/Semantics/typeinfo01.f90 b/flang/test/Semantics/typeinfo01.f90
index 454d73098408d..c1427f28753cf 100644
--- a/flang/test/Semantics/typeinfo01.f90
+++ b/flang/test/Semantics/typeinfo01.f90
@@ -34,7 +34,8 @@ module m03
   end type
   type(kpdt(4)) :: x
 !CHECK: .c.kpdt.4, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.a,genre=1_1,category=2_1,kind=4_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
-!CHECK: .dt.kpdt.4, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.kpdt,sizeinbytes=4_8,uninstantiated=NULL(),kindparameter=.kp.kpdt.4,lenparameterkind=NULL(),component=.c.kpdt.4,procptr=NULL(),special=NULL(),specialbitset=0_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1)
+!CHECK: .dt.kpdt, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(name=.n.kpdt,uninstantiated=NULL(),kindparameter=.kp.kpdt,lenparameterkind=NULL())
+!CHECK: .dt.kpdt.4, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.kpdt,sizeinbytes=4_8,uninstantiated=.dt.kpdt,kindparameter=.kp.kpdt.4,lenparameterkind=NULL(),component=.c.kpdt.4,procptr=NULL(),special=NULL(),specialbitset=0_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1)
 !CHECK: .kp.kpdt.4, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: INTEGER(8) shape: 0_8:0_8 init:[INTEGER(8)::4_8]
 end module
 

From 46387cd18420e120bbcfa3a646d48e2903a5c28c Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:48:55 -0700
Subject: [PATCH 465/710] [flang] Compile the output of
 -fdebug-unparse-with-modules (#135696)

The output of a compilation with the -fdebug-unparse-with-modules option
comprises its normal unparsed output along with the regenerated contents
of any modules that were required from module files. This is handy for
producing stand-alone test cases.

The modules' contents are generated by the same code that writes module
files, so they can contain some USE associations to private entities in
other modules that are necessary to complete local declarations, usually
initializers. Such USE associations to private entities are not flagged
as fatal errors when modules are read from module files, but they
currently are caught when the output produced by this option is being
read back in to the compiler.

Handle this case by softening the error to a warning when one module
uses a private entity from another with an alias containing the
non-conforming '$' character. (I could have omitted the message
altogether, but there are other valid warnings that will occur due to
undefined function result variables; further, I didn't want to provide a
general hole around the protection of private names.)
---
 flang/docs/ModFiles.md                |  9 ++++++++-
 flang/lib/Semantics/resolve-names.cpp | 16 +++++++++++++---
 flang/test/Semantics/resolve11.f90    | 14 ++++++++++++++
 3 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/flang/docs/ModFiles.md b/flang/docs/ModFiles.md
index cde2e4218d6fd..dd0ade5cebbfc 100644
--- a/flang/docs/ModFiles.md
+++ b/flang/docs/ModFiles.md
@@ -98,7 +98,14 @@ Entities that have been included in a module by means of USE association
 are represented in the module file with `USE` statements.
 Name aliases are sometimes necessary when an entity from another
 module is needed for a declaration and conflicts with another
-entity of the same name.
+entity of the same name, or is `PRIVATE`.
+These aliases have currency symbols (`$`) in them.
+When a module
+is parsed from a module file, no error is emitted for associating
+such an alias with a `PRIVATE` name.
+A module parsed from another source file that is not a module file
+(notably, the output of the `-fdebug-unparse-with-modules` option)
+will emit only warnings.
 
 ## Reading and writing module files
 
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 74367b5229548..7ec6252530d55 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -3325,9 +3325,19 @@ ModuleVisitor::SymbolRename ModuleVisitor::AddUse(
     // Privacy is not enforced in module files so that generic interfaces
     // can be resolved to private specific procedures in specification
     // expressions.
-    Say(useName, "'%s' is PRIVATE in '%s'"_err_en_US, MakeOpName(useName),
-        useModuleScope_->GetName().value());
-    return {};
+    // Local names that contain currency symbols ('$') are created by the
+    // module file writer when a private name in another module is needed to
+    // process a local declaration.  These can show up in the output of
+    // -fdebug-unparse-with-modules, too, so go easy on them.
+    if (currScope().IsModule() &&
+        localName.ToString().find("$") != std::string::npos) {
+      Say(useName, "'%s' is PRIVATE in '%s'"_warn_en_US, MakeOpName(useName),
+          useModuleScope_->GetName().value());
+    } else {
+      Say(useName, "'%s' is PRIVATE in '%s'"_err_en_US, MakeOpName(useName),
+          useModuleScope_->GetName().value());
+      return {};
+    }
   }
   auto &localSymbol{MakeSymbol(localName)};
   DoAddUse(useName, localName, localSymbol, *useSymbol);
diff --git a/flang/test/Semantics/resolve11.f90 b/flang/test/Semantics/resolve11.f90
index db508f062d1d1..39a30b858ebb6 100644
--- a/flang/test/Semantics/resolve11.f90
+++ b/flang/test/Semantics/resolve11.f90
@@ -86,3 +86,17 @@ subroutine s5
   use m5, only: foo, fun
   print *, fun() ! ok
 end
+
+module m6
+  !WARNING: 'foo' is PRIVATE in 'm5'
+  use m5, only: name$with$dollar => foo
+  !ERROR: 'foo' is PRIVATE in 'm5'
+  use m5, only: normal_name => foo
+end
+
+subroutine s6
+  !The special dispensation for USE association of private names to local
+  !aliases with '$' in them only applies to modules.
+  !ERROR: 'foo' is PRIVATE in 'm5'
+  use m5, only: name$with$dollar => foo
+end

From b4ff435b84f0f4e05a932c8bfa0c033c87665d5b Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:50:34 -0700
Subject: [PATCH 466/710] [flang] Fix fixed-form continuations of !$ OpenMP
 conditional lines (#135852)

I broke fixed-form line continuation (without !$) for OpenMP !$
conditional compilation lines. Fix it.
---
 flang/lib/Parser/prescan.cpp                  | 76 ++++++++++++-------
 .../continuation-in-conditional-compilation.f |  6 ++
 flang/test/Parser/unmatched-parens.f90        |  2 +-
 flang/test/Preprocessing/implicit-contin3.F90 |  2 +-
 4 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index b2b3d7fcfe786..2db9fa0a937f3 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -371,7 +371,7 @@ void Prescanner::CheckAndEmitLine(
   // nor before or after conditional source.
   // Applications play shenanigans with line continuation before and
   // after #include'd subprogram argument lists and conditional source.
-  if (!isNestedInIncludeDirective_ && !omitNewline_ &&
+  if (!preprocessingOnly_ && !isNestedInIncludeDirective_ && !omitNewline_ &&
       !afterPreprocessingDirective_ && tokens.BadlyNestedParentheses() &&
       !preprocessor_.InConditional()) {
     if (nextLine_ < limit_ && IsPreprocessorDirectiveLine(nextLine_)) {
@@ -975,17 +975,22 @@ void Prescanner::QuotedCharacterLiteral(
     } else {
       isEscaped = false;
     }
-    EmitQuotedChar(static_cast<unsigned char>(*at_), emit, insert, false,
-        Encoding::LATIN_1);
-    while (PadOutCharacterLiteral(tokens)) {
-    }
     if (*at_ == '\n') {
-      if (!inPreprocessorDirective_) {
+      if (inPreprocessorDirective_) {
+        EmitQuotedChar(static_cast<unsigned char>(*at_), emit, insert, false,
+            Encoding::LATIN_1);
+      } else if (InCompilerDirective() && preprocessingOnly_) {
+        // don't complain about -E output of !$, do it in later compilation
+      } else {
         Say(GetProvenanceRange(start, end),
             "Incomplete character literal"_err_en_US);
       }
       break;
     }
+    EmitQuotedChar(static_cast<unsigned char>(*at_), emit, insert, false,
+        Encoding::LATIN_1);
+    while (PadOutCharacterLiteral(tokens)) {
+    }
     // Here's a weird edge case.  When there's a two or more following
     // continuation lines at this point, and the entire significant part of
     // the next continuation line is the name of a keyword macro, replace
@@ -1314,23 +1319,30 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
   }
   tabInCurrentLine_ = false;
   char col1{*nextLine_};
+  bool canBeNonDirectiveContinuation{
+      (col1 == ' ' ||
+          ((col1 == 'D' || col1 == 'd') &&
+              features_.IsEnabled(LanguageFeature::OldDebugLines))) &&
+      nextLine_[1] == ' ' && nextLine_[2] == ' ' && nextLine_[3] == ' ' &&
+      nextLine_[4] == ' '};
   if (InCompilerDirective()) {
-    if (!IsFixedFormCommentChar(col1)) {
-      return nullptr;
-    } else if (directiveSentinel_[0] == '$' && directiveSentinel_[1] == '\0') {
-      // !$ OpenMP conditional compilation
-      if (preprocessingOnly_) {
-        // in -E mode, don't treat "!$   &" as a continuation
-        return nullptr;
-      } else if (nextLine_[1] == '$') {
-        // accept but do not require a matching sentinel
-        if (!(nextLine_[2] == '&' || IsSpaceOrTab(&nextLine_[2]))) {
-          return nullptr;
+    if (directiveSentinel_[0] == '$' && directiveSentinel_[1] == '\0') {
+      if (IsFixedFormCommentChar(col1)) {
+        if (nextLine_[1] == '$' &&
+            (nextLine_[2] == '&' || IsSpaceOrTab(&nextLine_[2]))) {
+          // Next line is also !$ conditional compilation, might be continuation
+          if (preprocessingOnly_) {
+            return nullptr;
+          }
+        } else {
+          return nullptr; // comment, or distinct directive
         }
-      } else {
-        return nullptr; // distinct directive
+      } else if (!canBeNonDirectiveContinuation) {
+        return nullptr;
       }
-    } else { // all other directives
+    } else if (!IsFixedFormCommentChar(col1)) {
+      return nullptr; // in directive other than !$, but next line is not
+    } else { // in directive other than !$, next line might be continuation
       int j{1};
       for (; j < 5; ++j) {
         char ch{directiveSentinel_[j - 1]};
@@ -1355,6 +1367,22 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
     }
   } else {
     // Normal case: not in a compiler directive.
+    if (IsFixedFormCommentChar(col1)) {
+      if (nextLine_[1] == '$' && nextLine_[2] == ' ' && nextLine_[3] == ' ' &&
+          nextLine_[4] == ' ' &&
+          IsCompilerDirectiveSentinel(&nextLine_[1], 1) &&
+          !preprocessingOnly_) {
+        // !$ conditional compilation line as a continuation
+        const char *col6{nextLine_ + 5};
+        if (*col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {
+          if (mightNeedSpace && !IsSpace(nextLine_ + 6)) {
+            insertASpace_ = true;
+          }
+          return nextLine_ + 6;
+        }
+      }
+      return nullptr;
+    }
     if (col1 == '&' &&
         features_.IsEnabled(
             LanguageFeature::FixedFormContinuationWithColumn1Ampersand)) {
@@ -1370,15 +1398,11 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
       tabInCurrentLine_ = true;
       return nextLine_ + 2; // VAX extension
     }
-    if ((col1 == ' ' ||
-            ((col1 == 'D' || col1 == 'd') &&
-                features_.IsEnabled(LanguageFeature::OldDebugLines))) &&
-        nextLine_[1] == ' ' && nextLine_[2] == ' ' && nextLine_[3] == ' ' &&
-        nextLine_[4] == ' ') {
+    if (canBeNonDirectiveContinuation) {
       const char *col6{nextLine_ + 5};
       if (*col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {
         if ((*col6 == 'i' || *col6 == 'I') && IsIncludeLine(nextLine_)) {
-          // It's An INCLUDE line, not a continuation
+          // It's an INCLUDE line, not a continuation
         } else {
           return nextLine_ + 6;
         }
diff --git a/flang/test/Parser/continuation-in-conditional-compilation.f b/flang/test/Parser/continuation-in-conditional-compilation.f
index 35525b4fda582..57b69de657348 100644
--- a/flang/test/Parser/continuation-in-conditional-compilation.f
+++ b/flang/test/Parser/continuation-in-conditional-compilation.f
@@ -5,6 +5,12 @@ program main
       k01=1+
 !$   &  1
 
+! CHECK: !$    k02=23
+! CHECK: !$   &4
+!$    k02=2
+     +3
+!$   +4
+
 ! CHECK: !$omp parallel private(k01)
 !$omp parallel
 !$omp+ private(k01)
diff --git a/flang/test/Parser/unmatched-parens.f90 b/flang/test/Parser/unmatched-parens.f90
index c0a3e843f4f72..ad8c37cf60af4 100644
--- a/flang/test/Parser/unmatched-parens.f90
+++ b/flang/test/Parser/unmatched-parens.f90
@@ -1,4 +1,4 @@
-! RUN: not %flang_fc1 -E %s 2>&1 | FileCheck %s
+! RUN: not %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
 do i = 1,10
   ! CHECK: Unmatched '('
   if (i != 0) then
diff --git a/flang/test/Preprocessing/implicit-contin3.F90 b/flang/test/Preprocessing/implicit-contin3.F90
index 8c1829d55eee7..4c31d076aa681 100644
--- a/flang/test/Preprocessing/implicit-contin3.F90
+++ b/flang/test/Preprocessing/implicit-contin3.F90
@@ -1,4 +1,4 @@
-! RUN: not %flang -E %s 2>&1 | FileCheck %s
+! RUN: not %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
 ! Test implicit continuation for possible function-like macro calls only
 #define flm(x) x
 call notamacro(3

From 6a7044a7b88a2e5e4ea7f3f9c128b4fbb33233f8 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:51:00 -0700
Subject: [PATCH 467/710] [flang] Improve OpenACC SELF clause parser (#135883)

The current parser can fail on "self(x * 2)" by recognizing just "x" as
a one-element list of object names and then failing at a higher level
because it never reached the right parenthesis. Add lookahead checks and
error recovery.

Fixes https://github.com/llvm/llvm-project/issues/135810.
---
 flang/lib/Parser/openacc-parsers.cpp         |  9 +++++++--
 flang/test/Semantics/OpenACC/bug135810-1.f90 | 14 ++++++++++++++
 flang/test/Semantics/OpenACC/bug135810-2.f90 | 12 ++++++++++++
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Semantics/OpenACC/bug135810-1.f90
 create mode 100644 flang/test/Semantics/OpenACC/bug135810-2.f90

diff --git a/flang/lib/Parser/openacc-parsers.cpp b/flang/lib/Parser/openacc-parsers.cpp
index 072eba99826a1..ad035e6ade321 100644
--- a/flang/lib/Parser/openacc-parsers.cpp
+++ b/flang/lib/Parser/openacc-parsers.cpp
@@ -126,8 +126,13 @@ TYPE_PARSER(construct<AccDefaultClause>(
 // SELF clause is either a simple optional condition for compute construct
 // or a synonym of the HOST clause for the update directive 2.14.4 holding
 // an object list.
-TYPE_PARSER(construct<AccSelfClause>(Parser<AccObjectList>{}) ||
-    construct<AccSelfClause>(scalarLogicalExpr))
+TYPE_PARSER(
+    construct<AccSelfClause>(Parser<AccObjectList>{}) / lookAhead(")"_tok) ||
+    construct<AccSelfClause>(scalarLogicalExpr / lookAhead(")"_tok)) ||
+    construct<AccSelfClause>(
+        recovery(fail<std::optional<ScalarLogicalExpr>>(
+                     "logical expression or object list expected"_err_en_US),
+            SkipTo<')'>{} >> pure<std::optional<ScalarLogicalExpr>>())))
 
 // Modifier for copyin, copyout, cache and create
 TYPE_PARSER(construct<AccDataModifier>(
diff --git a/flang/test/Semantics/OpenACC/bug135810-1.f90 b/flang/test/Semantics/OpenACC/bug135810-1.f90
new file mode 100644
index 0000000000000..a4e1bba174ce7
--- /dev/null
+++ b/flang/test/Semantics/OpenACC/bug135810-1.f90
@@ -0,0 +1,14 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenacc
+integer function square(x, y)
+    implicit none
+    integer, intent(in) :: x, y
+    !$acc parallel self(x * 2 > x) ! ok
+    !$acc end parallel
+!ERROR: Must have LOGICAL type, but is INTEGER(4)
+    !$acc parallel self(x * 2)
+    !$acc end parallel
+!ERROR: SELF clause on the PARALLEL directive only accepts optional scalar logical expression
+    !$acc parallel self(x, y)
+    !$acc end parallel
+    square = x * x
+end function square
diff --git a/flang/test/Semantics/OpenACC/bug135810-2.f90 b/flang/test/Semantics/OpenACC/bug135810-2.f90
new file mode 100644
index 0000000000000..8ed608b0b4cc9
--- /dev/null
+++ b/flang/test/Semantics/OpenACC/bug135810-2.f90
@@ -0,0 +1,12 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenacc
+integer function square(x)
+    implicit none
+    integer, intent(in) :: x
+!ERROR: logical expression or object list expected
+    !$acc parallel self(,)
+    !$acc end parallel
+!ERROR: logical expression or object list expected
+    !$acc parallel self(.true., )
+    !$acc end parallel
+    square = x * x
+end function square

From 32145a5e18f87e4f672bbc051f00b8d57be98eb3 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:51:18 -0700
Subject: [PATCH 468/710] [flang][runtime] Better handling for integer input
 into null address (#135987)

The original descriptor-only path for I/O checks for null data addresses
and crashes with a readable message, but there's no such check on the
new fast path for formatted integer input, and so a READ into (say) a
deallocated allocatable will crash with a segfault. Put a null data
address check on the new fast path.
---
 flang-rt/lib/runtime/edit-input.cpp | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/flang-rt/lib/runtime/edit-input.cpp b/flang-rt/lib/runtime/edit-input.cpp
index 4aa8c70f776cd..68ea34bf63cca 100644
--- a/flang-rt/lib/runtime/edit-input.cpp
+++ b/flang-rt/lib/runtime/edit-input.cpp
@@ -188,7 +188,11 @@ static RT_API_ATTRS char ScanNumericPrefix(IoStatementState &io,
 
 RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
     void *n, int kind, bool isSigned) {
-  RUNTIME_CHECK(io.GetIoErrorHandler(), kind >= 1 && !(kind & (kind - 1)));
+  auto &handler{io.GetIoErrorHandler()};
+  RUNTIME_CHECK(handler, kind >= 1 && !(kind & (kind - 1)));
+  if (!n) {
+    handler.Crash("Null address for integer input item");
+  }
   switch (edit.descriptor) {
   case DataEdit::ListDirected:
     if (IsNamelistNameOrSlash(io)) {
@@ -207,7 +211,7 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
   case 'A': // legacy extension
     return EditCharacterInput(io, edit, reinterpret_cast<char *>(n), kind);
   default:
-    io.GetIoErrorHandler().SignalError(IostatErrorInFormat,
+    handler.SignalError(IostatErrorInFormat,
         "Data edit descriptor '%c' may not be used with an INTEGER data item",
         edit.descriptor);
     return false;
@@ -217,7 +221,7 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
   auto fastField{io.GetUpcomingFastAsciiField()};
   char sign{ScanNumericPrefix(io, edit, next, remaining, &fastField)};
   if (sign == '-' && !isSigned) {
-    io.GetIoErrorHandler().SignalError("Negative sign in UNSIGNED input field");
+    handler.SignalError("Negative sign in UNSIGNED input field");
     return false;
   }
   common::uint128_t value{0};
@@ -254,8 +258,7 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
           break;
         }
       }
-      io.GetIoErrorHandler().SignalError(
-          "Bad character '%lc' in INTEGER input field", ch);
+      handler.SignalError("Bad character '%lc' in INTEGER input field", ch);
       return false;
     }
     static constexpr auto maxu128OverTen{maxu128 / 10};
@@ -268,7 +271,7 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
     any = true;
   }
   if (!any && !remaining) {
-    io.GetIoErrorHandler().SignalError(
+    handler.SignalError(
         "Integer value absent from NAMELIST or list-directed input");
     return false;
   }
@@ -280,14 +283,14 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
     overflow |= value >= maxForKind;
   }
   if (overflow) {
-    io.GetIoErrorHandler().SignalError(IostatIntegerInputOverflow,
+    handler.SignalError(IostatIntegerInputOverflow,
         "Decimal input overflows INTEGER(%d) variable", kind);
     return false;
   }
   if (sign == '-') {
     value = -value;
   }
-  if (any || !io.GetIoErrorHandler().InError()) {
+  if (any || !handler.InError()) {
     // The value is stored in the lower order bits on big endian platform.
     // For memcpy, shift the value to the highest order bits.
 #if USING_NATIVE_INT128_T

From 544940846d5b99831307595b82818b2b4aba6377 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:51:38 -0700
Subject: [PATCH 469/710] =?UTF-8?q?[flang][CUDA]=20Add=20error=20&=20warni?=
 =?UTF-8?q?ng=20for=20device=20argument=20first=20dimension=E2=80=A6=20(#1?=
 =?UTF-8?q?36058)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… discontiguity

For dummy assumed-shape/-rank device arrays, test the associated actual
argument for stride-1 contiguity, and report an error when the actual
argument is known to not be stride-1 contiguous and nonempty, or a
warning when when the actual argument is not known to be empty or
stride-1 contiguous.
---
 .../include/flang/Evaluate/check-expression.h | 26 ++++--
 flang/lib/Evaluate/check-expression.cpp       | 87 +++++++++++--------
 flang/lib/Semantics/check-call.cpp            | 20 ++++-
 flang/test/Semantics/cuf19.cuf                | 30 +++++++
 4 files changed, 118 insertions(+), 45 deletions(-)
 create mode 100644 flang/test/Semantics/cuf19.cuf

diff --git a/flang/include/flang/Evaluate/check-expression.h b/flang/include/flang/Evaluate/check-expression.h
index 7eee1994a888c..0cf12f340ec5c 100644
--- a/flang/include/flang/Evaluate/check-expression.h
+++ b/flang/include/flang/Evaluate/check-expression.h
@@ -113,21 +113,29 @@ extern template void CheckSpecificationExpr(
 // read-only data.
 template <typename A>
 std::optional<bool> IsContiguous(const A &, FoldingContext &,
-    bool namedConstantSectionsAreContiguous = true);
+    bool namedConstantSectionsAreContiguous = true,
+    bool firstDimensionStride1 = false);
 extern template std::optional<bool> IsContiguous(const Expr<SomeType> &,
-    FoldingContext &, bool namedConstantSectionsAreContiguous);
+    FoldingContext &, bool namedConstantSectionsAreContiguous,
+    bool firstDimensionStride1);
 extern template std::optional<bool> IsContiguous(const ArrayRef &,
-    FoldingContext &, bool namedConstantSectionsAreContiguous);
+    FoldingContext &, bool namedConstantSectionsAreContiguous,
+    bool firstDimensionStride1);
 extern template std::optional<bool> IsContiguous(const Substring &,
-    FoldingContext &, bool namedConstantSectionsAreContiguous);
+    FoldingContext &, bool namedConstantSectionsAreContiguous,
+    bool firstDimensionStride1);
 extern template std::optional<bool> IsContiguous(const Component &,
-    FoldingContext &, bool namedConstantSectionsAreContiguous);
+    FoldingContext &, bool namedConstantSectionsAreContiguous,
+    bool firstDimensionStride1);
 extern template std::optional<bool> IsContiguous(const ComplexPart &,
-    FoldingContext &, bool namedConstantSectionsAreContiguous);
+    FoldingContext &, bool namedConstantSectionsAreContiguous,
+    bool firstDimensionStride1);
 extern template std::optional<bool> IsContiguous(const CoarrayRef &,
-    FoldingContext &, bool namedConstantSectionsAreContiguous);
-extern template std::optional<bool> IsContiguous(
-    const Symbol &, FoldingContext &, bool namedConstantSectionsAreContiguous);
+    FoldingContext &, bool namedConstantSectionsAreContiguous,
+    bool firstDimensionStride1);
+extern template std::optional<bool> IsContiguous(const Symbol &,
+    FoldingContext &, bool namedConstantSectionsAreContiguous,
+    bool firstDimensionStride1);
 static inline std::optional<bool> IsContiguous(const SymbolRef &s,
     FoldingContext &c, bool namedConstantSectionsAreContiguous = true) {
   return IsContiguous(s.get(), c, namedConstantSectionsAreContiguous);
diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index 78268cd13377a..d8baaf2e2a7ac 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -881,10 +881,12 @@ class IsContiguousHelper
 public:
   using Result = std::optional<bool>; // tri-state
   using Base = AnyTraverse<IsContiguousHelper, Result>;
-  explicit IsContiguousHelper(
-      FoldingContext &c, bool namedConstantSectionsAreContiguous)
-      : Base{*this}, context_{c}, namedConstantSectionsAreContiguous_{
-                                      namedConstantSectionsAreContiguous} {}
+  explicit IsContiguousHelper(FoldingContext &c,
+      bool namedConstantSectionsAreContiguous,
+      bool firstDimensionStride1 = false)
+      : Base{*this}, context_{c},
+        namedConstantSectionsAreContiguous_{namedConstantSectionsAreContiguous},
+        firstDimensionStride1_{firstDimensionStride1} {}
   using Base::operator();
 
   template <typename T> Result operator()(const Constant<T> &) const {
@@ -956,13 +958,14 @@ class IsContiguousHelper
         if (!*baseIsContiguous) {
           return false;
         }
-        // TODO could be true if base contiguous and this is only component, or
-        // if base has only one element?
+        // TODO: should be true if base is contiguous and this is only
+        // component, or when the base has at most one element
       }
       return std::nullopt;
     }
   }
   Result operator()(const ComplexPart &x) const {
+    // TODO: should be true when base is empty array, too
     return x.complex().Rank() == 0;
   }
   Result operator()(const Substring &x) const {
@@ -1061,6 +1064,9 @@ class IsContiguousHelper
     auto dims{subscript.size()};
     std::vector<bool> knownPartialSlice(dims, false);
     for (auto j{dims}; j-- > 0;) {
+      if (j == 0 && firstDimensionStride1_ && !result.value_or(true)) {
+        result.reset(); // ignore problems on later dimensions
+      }
       std::optional<ConstantSubscript> dimLbound;
       std::optional<ConstantSubscript> dimUbound;
       std::optional<ConstantSubscript> dimExtent;
@@ -1083,18 +1089,20 @@ class IsContiguousHelper
           dimExtent = 0;
         }
       }
-
       if (const auto *triplet{std::get_if<Triplet>(&subscript[j].u)}) {
         ++rank;
+        const Expr<SubscriptInteger> *lowerBound{triplet->GetLower()};
+        const Expr<SubscriptInteger> *upperBound{triplet->GetUpper()};
+        std::optional<ConstantSubscript> lowerVal{lowerBound
+                ? ToInt64(Fold(context_, Expr<SubscriptInteger>{*lowerBound}))
+                : dimLbound};
+        std::optional<ConstantSubscript> upperVal{upperBound
+                ? ToInt64(Fold(context_, Expr<SubscriptInteger>{*upperBound}))
+                : dimUbound};
         if (auto stride{ToInt64(triplet->stride())}) {
-          const Expr<SubscriptInteger> *lowerBound{triplet->GetLower()};
-          const Expr<SubscriptInteger> *upperBound{triplet->GetUpper()};
-          std::optional<ConstantSubscript> lowerVal{lowerBound
-                  ? ToInt64(Fold(context_, Expr<SubscriptInteger>{*lowerBound}))
-                  : dimLbound};
-          std::optional<ConstantSubscript> upperVal{upperBound
-                  ? ToInt64(Fold(context_, Expr<SubscriptInteger>{*upperBound}))
-                  : dimUbound};
+          if (j == 0 && *stride == 1 && firstDimensionStride1_) {
+            result = *stride == 1; // contiguous or empty if so
+          }
           if (lowerVal && upperVal) {
             if (*lowerVal < *upperVal) {
               if (*stride < 0) {
@@ -1110,23 +1118,31 @@ class IsContiguousHelper
                   *lowerVal + *stride >= *upperVal) {
                 result = false; // discontiguous if not empty
               }
-            } else {
-              mayBeEmpty = true;
+            } else { // bounds known and equal
+              if (j == 0 && firstDimensionStride1_) {
+                result = true; // stride doesn't matter
+              }
+            }
+          } else { // bounds not both known
+            mayBeEmpty = true;
+          }
+        } else { // stride not known
+          if (lowerVal && upperVal && *lowerVal == *upperVal) {
+            // stride doesn't matter when bounds are equal
+            if (j == 0 && firstDimensionStride1_) {
+              result = true;
             }
           } else {
             mayBeEmpty = true;
           }
-        } else {
-          mayBeEmpty = true;
         }
-      } else if (subscript[j].Rank() > 0) {
+      } else if (subscript[j].Rank() > 0) { // vector subscript
         ++rank;
         if (!result) {
-          result = false; // vector subscript
+          result = false;
         }
         mayBeEmpty = true;
-      } else {
-        // Scalar subscript.
+      } else { // scalar subscript
         if (dimExtent && *dimExtent > 1) {
           knownPartialSlice[j] = true;
         }
@@ -1138,7 +1154,7 @@ class IsContiguousHelper
     if (result) {
       return result;
     }
-    // Not provably discontiguous at this point.
+    // Not provably contiguous or discontiguous at this point.
     // Return "true" if simply contiguous, otherwise nullopt.
     for (auto j{subscript.size()}; j-- > 0;) {
       if (const auto *triplet{std::get_if<Triplet>(&subscript[j].u)}) {
@@ -1170,33 +1186,36 @@ class IsContiguousHelper
 
   FoldingContext &context_;
   bool namedConstantSectionsAreContiguous_{false};
+  bool firstDimensionStride1_{false};
 };
 
 template <typename A>
 std::optional<bool> IsContiguous(const A &x, FoldingContext &context,
-    bool namedConstantSectionsAreContiguous) {
+    bool namedConstantSectionsAreContiguous, bool firstDimensionStride1) {
   if (!IsVariable(x) &&
       (namedConstantSectionsAreContiguous || !ExtractDataRef(x, true, true))) {
     return true;
   } else {
-    return IsContiguousHelper{context, namedConstantSectionsAreContiguous}(x);
+    return IsContiguousHelper{
+        context, namedConstantSectionsAreContiguous, firstDimensionStride1}(x);
   }
 }
 
 template std::optional<bool> IsContiguous(const Expr<SomeType> &,
-    FoldingContext &, bool namedConstantSectionsAreContiguous);
+    FoldingContext &, bool namedConstantSectionsAreContiguous,
+    bool firstDimensionStride1);
 template std::optional<bool> IsContiguous(const ArrayRef &, FoldingContext &,
-    bool namedConstantSectionsAreContiguous);
+    bool namedConstantSectionsAreContiguous, bool firstDimensionStride1);
 template std::optional<bool> IsContiguous(const Substring &, FoldingContext &,
-    bool namedConstantSectionsAreContiguous);
+    bool namedConstantSectionsAreContiguous, bool firstDimensionStride1);
 template std::optional<bool> IsContiguous(const Component &, FoldingContext &,
-    bool namedConstantSectionsAreContiguous);
+    bool namedConstantSectionsAreContiguous, bool firstDimensionStride1);
 template std::optional<bool> IsContiguous(const ComplexPart &, FoldingContext &,
-    bool namedConstantSectionsAreContiguous);
+    bool namedConstantSectionsAreContiguous, bool firstDimensionStride1);
 template std::optional<bool> IsContiguous(const CoarrayRef &, FoldingContext &,
-    bool namedConstantSectionsAreContiguous);
-template std::optional<bool> IsContiguous(
-    const Symbol &, FoldingContext &, bool namedConstantSectionsAreContiguous);
+    bool namedConstantSectionsAreContiguous, bool firstDimensionStride1);
+template std::optional<bool> IsContiguous(const Symbol &, FoldingContext &,
+    bool namedConstantSectionsAreContiguous, bool firstDimensionStride1);
 
 // IsErrorExpr()
 struct IsErrorExprHelper : public AnyTraverse<IsErrorExprHelper, bool> {
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index ef8282143451c..dfaa0e028d698 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -1015,10 +1015,26 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
         actualDataAttr = common::CUDADataAttr::Device;
       }
     }
+    if (dummyDataAttr == common::CUDADataAttr::Device &&
+        (dummyIsAssumedShape || dummyIsAssumedRank)) {
+      if (auto contig{evaluate::IsContiguous(actual, foldingContext,
+              /*namedConstantSectionsAreContiguous=*/true,
+              /*firstDimensionStride1=*/true)}) {
+        if (!*contig) {
+          messages.Say(
+              "actual argument associated with assumed shape/rank device %s is known to be discontiguous on its first dimension"_err_en_US,
+              dummyName);
+        }
+      } else {
+        messages.Say(
+            "actual argument associated with assumed shape/rank device %s is not known to be contiguous on its first dimension"_warn_en_US,
+            dummyName);
+      }
+    }
     std::optional<std::string> warning;
-    bool isHostDeviceProc = procedure.cudaSubprogramAttrs &&
+    bool isHostDeviceProc{procedure.cudaSubprogramAttrs &&
         *procedure.cudaSubprogramAttrs ==
-            common::CUDASubprogramAttrs::HostDevice;
+            common::CUDASubprogramAttrs::HostDevice};
     if (!common::AreCompatibleCUDADataAttrs(dummyDataAttr, actualDataAttr,
             dummy.ignoreTKR, &warning, /*allowUnifiedMatchingRule=*/true,
             isHostDeviceProc, &context.languageFeatures())) {
diff --git a/flang/test/Semantics/cuf19.cuf b/flang/test/Semantics/cuf19.cuf
new file mode 100644
index 0000000000000..8bec943f99006
--- /dev/null
+++ b/flang/test/Semantics/cuf19.cuf
@@ -0,0 +1,30 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+interface
+  subroutine foo(a)
+    real, device, dimension(:,:) :: a
+  end
+end interface
+
+real, device, allocatable :: a(:,:)
+complex, device, allocatable :: z(:,:)
+integer :: i = 2, j = 3
+allocate(a(10,10))
+allocate(z(10,10))
+call foo(a) ! ok
+call foo(a(:,:)) ! ok
+call foo(a(1:10,1:10)) ! ok
+!ERROR: actual argument associated with assumed shape/rank device dummy argument 'a=' is known to be discontiguous on its first dimension
+call foo(a(1:10:2,1:10))
+call foo(a(1:0:2,1:10)) ! empty dimension is ok
+call foo(a(1:10:2,1:0)) ! any empty dimension is ok
+call foo(a(1:10,1:10:2)) ! discontiguous second dimension is ok
+!WARNING: actual argument associated with assumed shape/rank device dummy argument 'a=' is not known to be contiguous on its first dimension
+call foo(a(1:10:i,1:10))
+!WARNING: actual argument associated with assumed shape/rank device dummy argument 'a=' is not known to be contiguous on its first dimension
+call foo(a(1:i:2,1:10))
+call foo(a(i:j:1,1:10)) ! stride 1, okay, despite unknown bounds
+!WARNING: actual argument associated with assumed shape/rank device dummy argument 'a=' is not known to be contiguous on its first dimension
+call foo(a(i:j:-1,1:10))
+!ERROR: actual argument associated with assumed shape/rank device dummy argument 'a=' is known to be discontiguous on its first dimension
+call foo(z%re)
+end

From aac53ad4d6eafda84ca92196a62e75e96e5fad25 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:52:04 -0700
Subject: [PATCH 470/710] [flang] Don't perform macro replacement on exponents
 (#136176)

See new test. I inadvertently broke this behavior with a recent fix for
another problem, because the effects of the overloaded
TokenSequence::Put() member function on token merging were confusing.
Rename and document the various overloads.
---
 flang/include/flang/Parser/token-sequence.h | 24 +++++++++----
 flang/lib/Parser/preprocessor.cpp           | 38 ++++++++++-----------
 flang/lib/Parser/prescan.cpp                | 13 ++++---
 flang/lib/Parser/token-sequence.cpp         | 13 +++----
 flang/test/Preprocessing/exponent-bug.F90   |  5 +++
 5 files changed, 55 insertions(+), 38 deletions(-)
 create mode 100644 flang/test/Preprocessing/exponent-bug.F90

diff --git a/flang/include/flang/Parser/token-sequence.h b/flang/include/flang/Parser/token-sequence.h
index 047c0bed00762..69291e69526e2 100644
--- a/flang/include/flang/Parser/token-sequence.h
+++ b/flang/include/flang/Parser/token-sequence.h
@@ -35,10 +35,10 @@ class Prescanner;
 class TokenSequence {
 public:
   TokenSequence() {}
-  TokenSequence(const TokenSequence &that) { Put(that); }
+  TokenSequence(const TokenSequence &that) { CopyAll(that); }
   TokenSequence(
       const TokenSequence &that, std::size_t at, std::size_t count = 1) {
-    Put(that, at, count);
+    AppendRange(that, at, count);
   }
   TokenSequence(TokenSequence &&that)
       : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
@@ -48,7 +48,7 @@ class TokenSequence {
 
   TokenSequence &operator=(const TokenSequence &that) {
     clear();
-    Put(that);
+    CopyAll(that);
     return *this;
   }
   TokenSequence &operator=(TokenSequence &&that);
@@ -100,14 +100,26 @@ class TokenSequence {
     start_.pop_back();
   }
 
-  void Put(const TokenSequence &);
-  void Put(const TokenSequence &, ProvenanceRange);
-  void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
+  // These append characters with provenance and then close the token.
+  // When the last token of this sequence remains open beforehand,
+  // the new characters are appended to it.
   void Put(const char *, std::size_t, Provenance);
   void Put(const CharBlock &, Provenance);
   void Put(const std::string &, Provenance);
   void Put(llvm::raw_string_ostream &, Provenance);
 
+  // Appends a full copy of another sequence.  When the last token of this
+  // sequence remains open beforehand, it is closed before the new text
+  // is appended.
+  void CopyAll(const TokenSequence &);
+  // Copies a range of tokens from another sequence.  If the last token of
+  // this sequence remains open, the first token of the copied range will be
+  // appended to it.
+  void AppendRange(
+      const TokenSequence &, std::size_t at, std::size_t tokens = 1);
+  // Copies tokens (via Put above) with new provenance.
+  void CopyWithProvenance(const TokenSequence &, ProvenanceRange);
+
   Provenance GetCharProvenance(std::size_t) const;
   Provenance GetTokenProvenance(
       std::size_t token, std::size_t offset = 0) const;
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index 7e6a1c2ca6977..a47f9c32ad27c 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -101,7 +101,7 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
         continue;
       }
     }
-    result.Put(token, firstToken + j, 1);
+    result.AppendRange(token, firstToken + j, 1);
   }
   return result;
 }
@@ -170,7 +170,7 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
       }
     } else if (pasting && text.TokenAt(j).IsBlank()) {
     } else {
-      result.Put(text, j, 1);
+      result.AppendRange(text, j, 1);
       pasting = false;
     }
   }
@@ -223,7 +223,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
         CHECK(resultSize > 0 &&
             result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
         result.pop_back();
-        result.Put(Stringify(args[index], prescanner.allSources()));
+        result.CopyAll(Stringify(args[index], prescanner.allSources()));
       } else {
         const TokenSequence *arg{&args[index]};
         std::optional<TokenSequence> replaced;
@@ -243,7 +243,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
             }
           }
         }
-        result.Put(DEREF(arg));
+        result.CopyAll(DEREF(arg));
       }
     } else if (bytes == 11 && isVariadic_ &&
         token.ToString() == "__VA_ARGS__") {
@@ -254,7 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
         if (k > argumentCount()) {
           result.Put(","s, commaProvenance);
         }
-        result.Put(args[k]);
+        result.CopyAll(args[k]);
       }
     } else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
         j + 2 < tokens && replacement_.TokenAt(j + 1).OnlyNonBlank() == '(' &&
@@ -274,7 +274,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
           }
         }
       }
-      result.Put(replacement_, j);
+      result.AppendRange(replacement_, j);
     }
   }
   return TokenPasting(std::move(result));
@@ -338,18 +338,18 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
           inIfExpression](std::size_t after, const TokenSequence &replacement,
           std::size_t pFLMOffset) {
         if (after < input.SizeInTokens()) {
-          result.Put(replacement, 0, pFLMOffset);
+          result.AppendRange(replacement, 0, pFLMOffset);
           TokenSequence suffix;
-          suffix.Put(
+          suffix.AppendRange(
               replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);
-          suffix.Put(input, after, input.SizeInTokens() - after);
+          suffix.AppendRange(input, after, input.SizeInTokens() - after);
           auto further{ReplaceMacros(
               suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};
           if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {
             // still not closed
             **partialFunctionLikeMacro += result.SizeInTokens();
           }
-          result.Put(further);
+          result.CopyAll(further);
           return true;
         } else {
           if (partialFunctionLikeMacro) {
@@ -362,7 +362,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
   for (; j < tokens; ++j) {
     CharBlock token{input.TokenAt(j)};
     if (token.IsBlank() || !IsLegalIdentifierStart(token[0])) {
-      result.Put(input, j);
+      result.AppendRange(input, j);
       continue;
     }
     // Process identifier in replacement text.
@@ -388,12 +388,12 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
       }
     }
     if (it == definitions_.end()) {
-      result.Put(input, j);
+      result.AppendRange(input, j);
       continue;
     }
     Definition *def{&it->second};
     if (def->isDisabled()) {
-      result.Put(input, j);
+      result.AppendRange(input, j);
       continue;
     }
     if (!def->isFunctionLike()) {
@@ -444,7 +444,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
         ProvenanceRange use{input.GetTokenProvenanceRange(j)};
         ProvenanceRange newRange{
             allSources_.AddMacroCall(from, use, replaced.ToString())};
-        result.Put(replaced, newRange);
+        result.CopyWithProvenance(replaced, newRange);
       }
     } else {
       // Possible function-like macro call.  Skip spaces and newlines to see
@@ -461,10 +461,10 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
       if (!leftParen) {
         if (partialFunctionLikeMacro) {
           *partialFunctionLikeMacro = result.SizeInTokens();
-          result.Put(input, j, tokens - j);
+          result.AppendRange(input, j, tokens - j);
           return result;
         } else {
-          result.Put(input, j);
+          result.AppendRange(input, j);
           continue;
         }
       }
@@ -491,11 +491,11 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
       }
       if (k >= tokens && partialFunctionLikeMacro) {
         *partialFunctionLikeMacro = result.SizeInTokens();
-        result.Put(input, j, tokens - j);
+        result.AppendRange(input, j, tokens - j);
         return result;
       } else if (k >= tokens || argStart.size() < def->argumentCount() ||
           (argStart.size() > def->argumentCount() && !def->isVariadic())) {
-        result.Put(input, j);
+        result.AppendRange(input, j);
         continue;
       }
       std::vector<TokenSequence> args;
@@ -520,7 +520,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
         ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
         ProvenanceRange newRange{
             allSources_.AddMacroCall(from, use, replaced.ToString())};
-        result.Put(replaced, newRange);
+        result.CopyWithProvenance(replaced, newRange);
       }
       j = k; // advance to the terminal ')'
     }
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 2db9fa0a937f3..46e04c15ade01 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -903,8 +903,7 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
         EmitCharAndAdvance(possible, *at_);
       }
       possible.CloseToken();
-      tokens.CloseToken();
-      tokens.Put(possible);
+      tokens.AppendRange(possible, 0); // appends to current token
       return true;
     }
     // Not an exponent; backtrack
@@ -937,9 +936,9 @@ bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
       preprocessor_.IsNameDefined(separate.TokenAt(1)) &&
       !preprocessor_.IsNameDefined(withUnderscore.ToCharBlock())) {
     // "_foo" is not defined, but "foo" is
-    tokens.Put(separate); // '_' "foo"
+    tokens.CopyAll(separate); // '_' "foo"
   } else {
-    tokens.Put(withUnderscore); // "_foo"
+    tokens.CopyAll(withUnderscore); // "_foo"
   }
   return true;
 }
@@ -1016,7 +1015,7 @@ void Prescanner::QuotedCharacterLiteral(
             ppTokens.Put(id, GetProvenance(idStart));
             if (auto replaced{
                     preprocessor_.MacroReplacement(ppTokens, *this)}) {
-              tokens.Put(*replaced);
+              tokens.CopyAll(*replaced);
               at_ = &idStart[idLen - 1];
               NextLine();
               continue; // try again on the next line
@@ -1836,7 +1835,7 @@ bool Prescanner::CompilerDirectiveContinuation(
   }
   if (ok) {
     tokens.pop_back(); // delete original '&'
-    tokens.Put(followingTokens, startAt, following - startAt);
+    tokens.AppendRange(followingTokens, startAt, following - startAt);
     tokens.RemoveRedundantBlanks();
   } else {
     nextLine_ = origNextLine;
@@ -1866,7 +1865,7 @@ bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {
       }
       followingTokens.RemoveRedundantBlanks();
       tokens.pop_back(); // delete original '&'
-      tokens.Put(followingTokens);
+      tokens.CopyAll(followingTokens);
       return true;
     }
   }
diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index c0655f608b97d..aee76938550f5 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -96,7 +96,7 @@ bool TokenSequence::IsAnythingLeft(std::size_t at) const {
   return false;
 }
 
-void TokenSequence::Put(const TokenSequence &that) {
+void TokenSequence::CopyAll(const TokenSequence &that) {
   if (nextStart_ < char_.size()) {
     start_.push_back(nextStart_);
   }
@@ -109,7 +109,8 @@ void TokenSequence::Put(const TokenSequence &that) {
   provenances_.Put(that.provenances_);
 }
 
-void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
+void TokenSequence::CopyWithProvenance(
+    const TokenSequence &that, ProvenanceRange range) {
   std::size_t offset{0};
   std::size_t tokens{that.SizeInTokens()};
   for (std::size_t j{0}; j < tokens; ++j) {
@@ -120,7 +121,7 @@ void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
   CHECK(offset == range.size());
 }
 
-void TokenSequence::Put(
+void TokenSequence::AppendRange(
     const TokenSequence &that, std::size_t at, std::size_t tokens) {
   ProvenanceRange provenance;
   std::size_t offset{0};
@@ -246,7 +247,7 @@ TokenSequence &TokenSequence::RemoveBlanks(std::size_t firstChar) {
   TokenSequence result;
   for (std::size_t j{0}; j < tokens; ++j) {
     if (!TokenAt(j).IsBlank() || start_[j] < firstChar) {
-      result.Put(*this, j);
+      result.AppendRange(*this, j);
     }
   }
   swap(result);
@@ -260,7 +261,7 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {
   for (std::size_t j{0}; j < tokens; ++j) {
     bool isBlank{TokenAt(j).IsBlank()};
     if (!isBlank || !lastWasBlank || start_[j] < firstChar) {
-      result.Put(*this, j);
+      result.AppendRange(*this, j);
     }
     lastWasBlank = isBlank;
   }
@@ -294,7 +295,7 @@ TokenSequence &TokenSequence::ClipComment(
       } else {
         TokenSequence result;
         if (j > 0) {
-          result.Put(*this, 0, j - 1);
+          result.AppendRange(*this, 0, j - 1);
         }
         swap(result);
         return *this;
diff --git a/flang/test/Preprocessing/exponent-bug.F90 b/flang/test/Preprocessing/exponent-bug.F90
new file mode 100644
index 0000000000000..903b41bb846a4
--- /dev/null
+++ b/flang/test/Preprocessing/exponent-bug.F90
@@ -0,0 +1,5 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+! CHECK: 3.14159e00
+#define e00 e666
+print *, 3.14159e00
+end

From 0dd2ed4ea3b16ee0cd88becf4dc9eb5bacd08ca9 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:52:21 -0700
Subject: [PATCH 471/710] [flang][OpenACC] Fix crash due to truncated scope
 source range (#136206)

A combined construct needs to ensure that the source range of their
initial statement is part of the source range of their scope.

Fixes https://github.com/llvm/llvm-project/issues/136076.
---
 flang/lib/Semantics/resolve-names.cpp | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 7ec6252530d55..f1d2ba4078236 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -1384,6 +1384,20 @@ class AccVisitor : public virtual DeclarationVisitor {
   void Post(const parser::AccEndBlockDirective &) {
     messageHandler().set_currStmtSource(std::nullopt);
   }
+  bool Pre(const parser::AccBeginCombinedDirective &x) {
+    AddAccSourceRange(x.source);
+    return true;
+  }
+  void Post(const parser::AccBeginCombinedDirective &) {
+    messageHandler().set_currStmtSource(std::nullopt);
+  }
+  bool Pre(const parser::AccEndCombinedDirective &x) {
+    AddAccSourceRange(x.source);
+    return true;
+  }
+  void Post(const parser::AccEndCombinedDirective &) {
+    messageHandler().set_currStmtSource(std::nullopt);
+  }
   bool Pre(const parser::AccBeginLoopDirective &x) {
     AddAccSourceRange(x.source);
     return true;

From 03b3620538bcd2514ecf49ab03573014b3346672 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Fri, 18 Apr 2025 12:52:39 -0700
Subject: [PATCH 472/710] [flang] Tweak integer output under width-free I/G
 editing (#136316)

A recent patch fixed Fujitsu test case 0561_0168 by emitting a leading
space for "bare" (no width 'w') I and G output editing of integer
values. This fix has broken another Fujitsu test case (0561_0168), since
the leading space should not be produced at the first column of the
output record. Adjust.
---
 flang-rt/lib/runtime/edit-output.cpp               | 7 +++++--
 flang-rt/unittests/Runtime/NumericalFormatTest.cpp | 4 ++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/flang-rt/lib/runtime/edit-output.cpp b/flang-rt/lib/runtime/edit-output.cpp
index 3699213e1f7b7..f90b6fb10963f 100644
--- a/flang-rt/lib/runtime/edit-output.cpp
+++ b/flang-rt/lib/runtime/edit-output.cpp
@@ -182,8 +182,11 @@ bool RT_API_ATTRS EditIntegerOutput(IoStatementState &io, const DataEdit &edit,
     leadingSpaces = 1;
   } else if (!edit.width) {
     // Bare 'I' and 'G' are interpreted with various default widths in the
-    // compilers that support them, so there's always some leading space.
-    leadingSpaces = std::max(1, leadingSpaces);
+    // compilers that support them, so there's always some leading space
+    // after column 1.
+    if (io.GetConnectionState().positionInRecord > 0) {
+      leadingSpaces = 1;
+    }
   }
   return EmitRepeated(io, ' ', leadingSpaces) &&
       EmitAscii(io, n < 0 ? "-" : "+", signChars) &&
diff --git a/flang-rt/unittests/Runtime/NumericalFormatTest.cpp b/flang-rt/unittests/Runtime/NumericalFormatTest.cpp
index 36170ea1056fd..a752f9d6c723b 100644
--- a/flang-rt/unittests/Runtime/NumericalFormatTest.cpp
+++ b/flang-rt/unittests/Runtime/NumericalFormatTest.cpp
@@ -842,6 +842,10 @@ TEST(IOApiTests, FormatIntegerValues) {
       {"(G0.2)", -1, "-1"},
       {"(G0.2)", 999, "999"},
       {"(G0.4)", 999, "999"},
+      {"(I)", 999, "999"},
+      {"(G)", 999, "999"},
+      {"('x',I)", 999, "x 999"},
+      {"('x',G)", 999, "x 999"},
   };
 
   for (auto const &[fmt, value, expect] : intTestCases) {

From daf3c985f75bae2f4f01594be2fbc02f953e7b06 Mon Sep 17 00:00:00 2001
From: Andy Kaylor <akaylor@nvidia.com>
Date: Fri, 18 Apr 2025 12:52:51 -0700
Subject: [PATCH 473/710] [CIR] Upstream support for typedef and type aliases
 (#136335)

Nothing is actually needed in ClangIR to support typedef and type
aliases, but the Decl kinds need to be explicitly ignored in the
emitDecl handlers to avoid hitting the default NYI errors. This change
does that and adds tests.
---
 clang/lib/CIR/CodeGen/CIRGenDecl.cpp   |  8 ++++++++
 clang/lib/CIR/CodeGen/CIRGenModule.cpp |  2 ++
 clang/test/CIR/CodeGen/basic.c         | 21 ++++++++++++++++++++
 clang/test/CIR/CodeGen/basic.cpp       | 15 ++++++++++++++
 clang/test/CIR/CodeGen/typedef.c       | 27 ++++++++++++++++++++++++++
 5 files changed, 73 insertions(+)
 create mode 100644 clang/test/CIR/CodeGen/typedef.c

diff --git a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
index 3d15a70d1d6ac..d7cbb4f64b2ea 100644
--- a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
@@ -276,6 +276,14 @@ void CIRGenFunction::emitDecl(const Decl &d) {
   case Decl::OpenACCRoutine:
     emitOpenACCRoutine(cast<OpenACCRoutineDecl>(d));
     return;
+  case Decl::Typedef:     // typedef int X;
+  case Decl::TypeAlias: { // using X = int; [C++0x]
+    QualType ty = cast<TypedefNameDecl>(d).getUnderlyingType();
+    assert(!cir::MissingFeatures::generateDebugInfo());
+    if (ty->isVariablyModifiedType())
+      cgm.errorNYI(d.getSourceRange(), "emitDecl: variably modified type");
+    return;
+  }
   default:
     cgm.errorNYI(d.getSourceRange(),
                  std::string("emitDecl: unhandled decl type: ") +
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index 1b504546162c1..3b13d495be5e3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -618,6 +618,8 @@ void CIRGenModule::emitTopLevelDecl(Decl *decl) {
     emitGlobalOpenACCDecl(cast<OpenACCDeclareDecl>(decl));
     break;
 
+  case Decl::Typedef:
+  case Decl::TypeAlias: // using foo = bar; [C++11]
   case Decl::Record:
   case Decl::CXXRecord:
     assert(!cir::MissingFeatures::generateDebugInfo());
diff --git a/clang/test/CIR/CodeGen/basic.c b/clang/test/CIR/CodeGen/basic.c
index 8ce8106b0c6cc..1845d3b64bf68 100644
--- a/clang/test/CIR/CodeGen/basic.c
+++ b/clang/test/CIR/CodeGen/basic.c
@@ -232,3 +232,24 @@ int f8(int *p) {
 // OGCG:   store i32 2, ptr %[[P]], align 4
 // OGCG:   %[[P2:.*]] = load ptr, ptr %[[P_PTR]], align 8
 // OGCG:   %[[STAR_P:.*]] = load i32, ptr %[[P2]], align 4
+
+typedef unsigned long size_type;
+typedef unsigned long _Tp;
+
+size_type max_size(void) {
+  return (size_type)~0 / sizeof(_Tp);
+}
+
+// CIR: cir.func @max_size()
+// CIR:   %0 = cir.alloca !u64i, !cir.ptr<!u64i>, ["__retval"] {alignment = 8 : i64}
+// CIR:   %1 = cir.const #cir.int<0> : !s32i
+// CIR:   %2 = cir.unary(not, %1) : !s32i, !s32i
+// CIR:   %3 = cir.cast(integral, %2 : !s32i), !u64i
+// CIR:   %4 = cir.const #cir.int<8> : !u64i
+// CIR:   %5 = cir.binop(div, %3, %4) : !u64i
+
+// LLVM: define i64 @max_size()
+// LLVM:   store i64 2305843009213693951, ptr
+
+// OGCG: define{{.*}} i64 @max_size()
+// OGCG:   ret i64 2305843009213693951
diff --git a/clang/test/CIR/CodeGen/basic.cpp b/clang/test/CIR/CodeGen/basic.cpp
index 9665b29719004..0f8431325a86f 100644
--- a/clang/test/CIR/CodeGen/basic.cpp
+++ b/clang/test/CIR/CodeGen/basic.cpp
@@ -87,3 +87,18 @@ int *f5() {
 // CHECK-NEXT:   cir.store %[[P]], %[[RET_ADDR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
 // CHECK-NEXT:   %[[RET_VAL:.*]] = cir.load %[[RET_ADDR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
 // CHECK-NEXT:   cir.return %[[RET_VAL]] : !cir.ptr<!s32i>
+
+using size_type = unsigned long;
+using _Tp = unsigned long;
+
+size_type max_size() {
+  return size_type(~0) / sizeof(_Tp);
+}
+
+// CHECK: cir.func @max_size()
+// CHECK:   %0 = cir.alloca !u64i, !cir.ptr<!u64i>, ["__retval"] {alignment = 8 : i64}
+// CHECK:   %1 = cir.const #cir.int<0> : !s32i
+// CHECK:   %2 = cir.unary(not, %1) : !s32i, !s32i
+// CHECK:   %3 = cir.cast(integral, %2 : !s32i), !u64i
+// CHECK:   %4 = cir.const #cir.int<8> : !u64i
+// CHECK:   %5 = cir.binop(div, %3, %4) : !u64i
diff --git a/clang/test/CIR/CodeGen/typedef.c b/clang/test/CIR/CodeGen/typedef.c
new file mode 100644
index 0000000000000..17fce13abf38a
--- /dev/null
+++ b/clang/test/CIR/CodeGen/typedef.c
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+void local_typedef(void) {
+  typedef struct {int a;} Struct;
+  Struct s;
+}
+
+// CIR:      cir.func @local_typedef()
+// CIR:        cir.alloca !cir.record<struct "Struct" {!s32i}>,
+// CIR-SAME:       !cir.ptr<!cir.record<struct "Struct" {!s32i}>>, ["s"]
+// CIR-SAME:       {alignment = 4 : i64}
+// CIR:        cir.return
+
+// LLVM: %struct.Struct = type { i32 }
+// LLVM: define void @local_typedef()
+// LLVM:   alloca %struct.Struct, i64 1, align 4
+// LLVM:   ret void
+
+// OGCG: %struct.Struct = type { i32 }
+// OGCG: define{{.*}} void @local_typedef()
+// OGCG:   alloca %struct.Struct, align 4
+// OGCG:   ret void

From a3f8836ae84dd697e096446d3ca666461da839ed Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 21:58:13 +0200
Subject: [PATCH 474/710] AMDGPU: Regenerate baseline checks

Clean up now unnecessary second check prefix.
---
 .../AMDGPU/addrspacecast-constantexpr.ll      | 106 +++---
 .../AMDGPU/annotate-kernel-features-hsa.ll    | 316 +++++++++---------
 .../AMDGPU/annotate-kernel-features.ll        | 199 ++++++-----
 3 files changed, 308 insertions(+), 313 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
index 3eba47d7d7852..26b3c98e85cde 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=HSA,ATTRIBUTOR_HSA %s
+; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -passes=amdgpu-attributor < %s | FileCheck -check-prefix=HSA %s
 
 declare void @llvm.memcpy.p1.p4.i32(ptr addrspace(1) nocapture, ptr addrspace(4) nocapture, i32, i1) #0
 
@@ -26,30 +26,30 @@ define amdgpu_kernel void @store_cast_0_flat_to_group_addrspacecast() #1 {
 }
 
 define amdgpu_kernel void @store_cast_0_group_to_flat_addrspacecast() #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_cast_0_group_to_flat_addrspacecast
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR2:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) null to ptr addrspace(4)), align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@store_cast_0_group_to_flat_addrspacecast
+; HSA-SAME: () #[[ATTR2:[0-9]+]] {
+; HSA-NEXT:    store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) null to ptr addrspace(4)), align 4
+; HSA-NEXT:    ret void
 ;
   store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) null to ptr addrspace(4))
   ret void
 }
 
 define amdgpu_kernel void @store_constant_cast_group_gv_to_flat() #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_to_flat
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.i32 to ptr addrspace(4)), align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_to_flat
+; HSA-SAME: () #[[ATTR2]] {
+; HSA-NEXT:    store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.i32 to ptr addrspace(4)), align 4
+; HSA-NEXT:    ret void
 ;
   store i32 7, ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.i32 to ptr addrspace(4))
   ret void
 }
 
 define amdgpu_kernel void @store_constant_cast_group_gv_gep_to_flat() #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_gep_to_flat
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    store i32 7, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_gep_to_flat
+; HSA-SAME: () #[[ATTR2]] {
+; HSA-NEXT:    store i32 7, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), align 4
+; HSA-NEXT:    ret void
 ;
   store i32 7, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8)
   ret void
@@ -76,11 +76,11 @@ define amdgpu_kernel void @store_constant_cast_global_gv_gep_to_flat() #1 {
 }
 
 define amdgpu_kernel void @load_constant_cast_group_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@load_constant_cast_group_gv_gep_to_flat
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), align 4
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[OUT]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@load_constant_cast_group_gv_gep_to_flat
+; HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
+; HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), align 4
+; HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[OUT]], align 4
+; HSA-NEXT:    ret void
 ;
   %val = load i32, ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8)
   store i32 %val, ptr addrspace(1) %out
@@ -88,11 +88,11 @@ define amdgpu_kernel void @load_constant_cast_group_gv_gep_to_flat(ptr addrspace
 }
 
 define amdgpu_kernel void @atomicrmw_constant_cast_group_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@atomicrmw_constant_cast_group_gv_gep_to_flat
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = atomicrmw add ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 1 seq_cst, align 4
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[OUT]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@atomicrmw_constant_cast_group_gv_gep_to_flat
+; HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
+; HSA-NEXT:    [[VAL:%.*]] = atomicrmw add ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 1 seq_cst, align 4
+; HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[OUT]], align 4
+; HSA-NEXT:    ret void
 ;
   %val = atomicrmw add ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 1 seq_cst
   store i32 %val, ptr addrspace(1) %out
@@ -100,12 +100,12 @@ define amdgpu_kernel void @atomicrmw_constant_cast_group_gv_gep_to_flat(ptr addr
 }
 
 define amdgpu_kernel void @cmpxchg_constant_cast_group_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@cmpxchg_constant_cast_group_gv_gep_to_flat
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = cmpxchg ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 0, i32 1 seq_cst seq_cst, align 4
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = extractvalue { i32, i1 } [[VAL]], 0
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL0]], ptr addrspace(1) [[OUT]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@cmpxchg_constant_cast_group_gv_gep_to_flat
+; HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
+; HSA-NEXT:    [[VAL:%.*]] = cmpxchg ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 0, i32 1 seq_cst seq_cst, align 4
+; HSA-NEXT:    [[VAL0:%.*]] = extractvalue { i32, i1 } [[VAL]], 0
+; HSA-NEXT:    store i32 [[VAL0]], ptr addrspace(1) [[OUT]], align 4
+; HSA-NEXT:    ret void
 ;
   %val = cmpxchg ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 0, i32 1 seq_cst seq_cst
   %val0 = extractvalue { i32, i1 } %val, 0
@@ -114,10 +114,10 @@ define amdgpu_kernel void @cmpxchg_constant_cast_group_gv_gep_to_flat(ptr addrsp
 }
 
 define amdgpu_kernel void @memcpy_constant_cast_group_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@memcpy_constant_cast_group_gv_gep_to_flat
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    call void @llvm.memcpy.p1.p4.i32(ptr addrspace(1) align 4 [[OUT]], ptr addrspace(4) align 4 getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 32, i1 false)
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@memcpy_constant_cast_group_gv_gep_to_flat
+; HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
+; HSA-NEXT:    call void @llvm.memcpy.p1.p4.i32(ptr addrspace(1) align 4 [[OUT]], ptr addrspace(4) align 4 getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 32, i1 false)
+; HSA-NEXT:    ret void
 ;
   call void @llvm.memcpy.p1.p4.i32(ptr addrspace(1) align 4 %out, ptr addrspace(4) align 4 getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), i32 32, i1 false)
   ret void
@@ -125,10 +125,10 @@ define amdgpu_kernel void @memcpy_constant_cast_group_gv_gep_to_flat(ptr addrspa
 
 ; Can't just search the pointer value
 define amdgpu_kernel void @store_value_constant_cast_lds_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_value_constant_cast_lds_gv_gep_to_flat
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    store ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), ptr addrspace(1) [[OUT]], align 8
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@store_value_constant_cast_lds_gv_gep_to_flat
+; HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
+; HSA-NEXT:    store ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), ptr addrspace(1) [[OUT]], align 8
+; HSA-NEXT:    ret void
 ;
   store ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8), ptr addrspace(1) %out
   ret void
@@ -136,10 +136,10 @@ define amdgpu_kernel void @store_value_constant_cast_lds_gv_gep_to_flat(ptr addr
 
 ; Can't just search pointer types
 define amdgpu_kernel void @store_ptrtoint_value_constant_cast_lds_gv_gep_to_flat(ptr addrspace(1) %out) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_ptrtoint_value_constant_cast_lds_gv_gep_to_flat
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    store i64 ptrtoint (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to i64), ptr addrspace(1) [[OUT]], align 8
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@store_ptrtoint_value_constant_cast_lds_gv_gep_to_flat
+; HSA-SAME: (ptr addrspace(1) [[OUT:%.*]]) #[[ATTR2]] {
+; HSA-NEXT:    store i64 ptrtoint (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to i64), ptr addrspace(1) [[OUT]], align 8
+; HSA-NEXT:    ret void
 ;
   store i64 ptrtoint (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to i64), ptr addrspace(1) %out
   ret void
@@ -147,19 +147,19 @@ define amdgpu_kernel void @store_ptrtoint_value_constant_cast_lds_gv_gep_to_flat
 
 ; Cast group to flat, do GEP, cast back to group
 define amdgpu_kernel void @store_constant_cast_group_gv_gep_to_flat_to_group() #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_gep_to_flat_to_group
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    store i32 7, ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3)), align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@store_constant_cast_group_gv_gep_to_flat_to_group
+; HSA-SAME: () #[[ATTR2]] {
+; HSA-NEXT:    store i32 7, ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3)), align 4
+; HSA-NEXT:    ret void
 ;
   store i32 7, ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3))
   ret void
 }
 
 define ptr addrspace(3) @ret_constant_cast_group_gv_gep_to_flat_to_group() #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@ret_constant_cast_group_gv_gep_to_flat_to_group
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    ret ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3))
+; HSA-LABEL: define {{[^@]+}}@ret_constant_cast_group_gv_gep_to_flat_to_group
+; HSA-SAME: () #[[ATTR2]] {
+; HSA-NEXT:    ret ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3))
 ;
   ret ptr addrspace(3) addrspacecast (ptr addrspace(4) getelementptr ([256 x i32], ptr addrspace(4) addrspacecast (ptr addrspace(3) @lds.arr to ptr addrspace(4)), i64 0, i64 8) to ptr addrspace(3))
 }
@@ -170,11 +170,9 @@ attributes #1 = { nounwind }
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
+; HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
+; HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
 ;.
-; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
-; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-;.
-;.
-; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
+; HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
index 32bb22b699b61..0e387716dda99 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=HSA,ATTRIBUTOR_HSA %s
+; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -passes=amdgpu-attributor < %s | FileCheck -check-prefix=HSA %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
@@ -32,11 +32,11 @@ define amdgpu_kernel void @use_tgid_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_y
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tgid_y
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
+; HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+; HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val = call i32 @llvm.amdgcn.workgroup.id.y()
   store i32 %val, ptr addrspace(1) %ptr
@@ -44,13 +44,13 @@ define amdgpu_kernel void @use_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @multi_use_tgid_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@multi_use_tgid_y
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@multi_use_tgid_y
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
+; HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+; HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+; HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.amdgcn.workgroup.id.y()
   store volatile i32 %val0, ptr addrspace(1) %ptr
@@ -60,13 +60,13 @@ define amdgpu_kernel void @multi_use_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_x_y
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tgid_x_y
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
+; HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+; HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+; HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.amdgcn.workgroup.id.x()
   %val1 = call i32 @llvm.amdgcn.workgroup.id.y()
@@ -76,11 +76,11 @@ define amdgpu_kernel void @use_tgid_x_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_z
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tgid_z
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
+; HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
+; HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val = call i32 @llvm.amdgcn.workgroup.id.z()
   store i32 %val, ptr addrspace(1) %ptr
@@ -88,13 +88,13 @@ define amdgpu_kernel void @use_tgid_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_x_z
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tgid_x_z
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3]] {
+; HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+; HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
+; HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.amdgcn.workgroup.id.x()
   %val1 = call i32 @llvm.amdgcn.workgroup.id.z()
@@ -104,13 +104,13 @@ define amdgpu_kernel void @use_tgid_x_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_y_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_y_z
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tgid_y_z
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
+; HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+; HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
+; HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.amdgcn.workgroup.id.y()
   %val1 = call i32 @llvm.amdgcn.workgroup.id.z()
@@ -120,15 +120,15 @@ define amdgpu_kernel void @use_tgid_y_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_y_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tgid_x_y_z
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tgid_x_y_z
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4]] {
+; HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+; HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+; HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
+; HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.amdgcn.workgroup.id.x()
   %val1 = call i32 @llvm.amdgcn.workgroup.id.y()
@@ -152,11 +152,11 @@ define amdgpu_kernel void @use_tidig_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tidig_y
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tidig_y
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
+; HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
+; HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val = call i32 @llvm.amdgcn.workitem.id.y()
   store i32 %val, ptr addrspace(1) %ptr
@@ -164,11 +164,11 @@ define amdgpu_kernel void @use_tidig_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tidig_z
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tidig_z
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
+; HSA-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
+; HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val = call i32 @llvm.amdgcn.workitem.id.z()
   store i32 %val, ptr addrspace(1) %ptr
@@ -192,13 +192,13 @@ define amdgpu_kernel void @use_tidig_x_tgid_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_y_tgid_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tidig_y_tgid_y
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR7:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tidig_y_tgid_y
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR7:[0-9]+]] {
+; HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
+; HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+; HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.amdgcn.workitem.id.y()
   %val1 = call i32 @llvm.amdgcn.workgroup.id.y()
@@ -208,15 +208,15 @@ define amdgpu_kernel void @use_tidig_y_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_x_y_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_tidig_x_y_z
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR8:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_tidig_x_y_z
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR8:[0-9]+]] {
+; HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
+; HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
+; HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.amdgcn.workitem.id.x()
   %val1 = call i32 @llvm.amdgcn.workitem.id.y()
@@ -228,21 +228,21 @@ define amdgpu_kernel void @use_tidig_x_y_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_all_workitems(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_all_workitems
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR9:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL3:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL4:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL5:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL3]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL4]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 [[VAL5]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_all_workitems
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR9:[0-9]+]] {
+; HSA-NEXT:    [[VAL0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; HSA-NEXT:    [[VAL1:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
+; HSA-NEXT:    [[VAL2:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
+; HSA-NEXT:    [[VAL3:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+; HSA-NEXT:    [[VAL4:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+; HSA-NEXT:    [[VAL5:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
+; HSA-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL3]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL4]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    store volatile i32 [[VAL5]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.amdgcn.workitem.id.x()
   %val1 = call i32 @llvm.amdgcn.workitem.id.y()
@@ -260,12 +260,12 @@ define amdgpu_kernel void @use_all_workitems(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_dispatch_ptr(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_dispatch_ptr
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR10:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) [[DISPATCH_PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_dispatch_ptr
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR10:[0-9]+]] {
+; HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+; HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) [[DISPATCH_PTR]], align 4
+; HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %dispatch.ptr = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
   %val = load i32, ptr addrspace(4) %dispatch.ptr
@@ -274,12 +274,12 @@ define amdgpu_kernel void @use_dispatch_ptr(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_queue_ptr(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_queue_ptr
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR11:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
-; ATTRIBUTOR_HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) [[DISPATCH_PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_queue_ptr
+; HSA-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR11:[0-9]+]] {
+; HSA-NEXT:    [[DISPATCH_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
+; HSA-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(4) [[DISPATCH_PTR]], align 4
+; HSA-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; HSA-NEXT:    ret void
 ;
   %dispatch.ptr = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
   %val = load i32, ptr addrspace(4) %dispatch.ptr
@@ -302,11 +302,11 @@ define amdgpu_kernel void @use_kernarg_segment_ptr(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_group_to_flat_addrspacecast(ptr addrspace(3) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR12:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_group_to_flat_addrspacecast
+; HSA-SAME: (ptr addrspace(3) [[PTR:%.*]]) #[[ATTR12:[0-9]+]] {
+; HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
+; HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; HSA-NEXT:    ret void
 ;
   %stof = addrspacecast ptr addrspace(3) %ptr to ptr
   store volatile i32 0, ptr %stof
@@ -314,11 +314,11 @@ define amdgpu_kernel void @use_group_to_flat_addrspacecast(ptr addrspace(3) %ptr
 }
 
 define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_private_to_flat_addrspacecast
-; ATTRIBUTOR_HSA-SAME: (ptr addrspace(5) [[PTR:%.*]]) #[[ATTR13:[0-9]+]] {
-; ATTRIBUTOR_HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
-; ATTRIBUTOR_HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_private_to_flat_addrspacecast
+; HSA-SAME: (ptr addrspace(5) [[PTR:%.*]]) #[[ATTR13:[0-9]+]] {
+; HSA-NEXT:    [[STOF:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; HSA-NEXT:    store volatile i32 0, ptr [[STOF]], align 4
+; HSA-NEXT:    ret void
 ;
   %stof = addrspacecast ptr addrspace(5) %ptr to ptr
   store volatile i32 0, ptr %stof
@@ -399,12 +399,12 @@ define amdgpu_kernel void @use_flat_to_constant_addrspacecast(ptr %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_is_shared(ptr %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_is_shared
-; ATTRIBUTOR_HSA-SAME: (ptr [[PTR:%.*]]) #[[ATTR12]] {
-; ATTRIBUTOR_HSA-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
-; ATTRIBUTOR_HSA-NEXT:    [[EXT:%.*]] = zext i1 [[IS_SHARED]] to i32
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[EXT]], ptr addrspace(1) poison, align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_is_shared
+; HSA-SAME: (ptr [[PTR:%.*]]) #[[ATTR12]] {
+; HSA-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
+; HSA-NEXT:    [[EXT:%.*]] = zext i1 [[IS_SHARED]] to i32
+; HSA-NEXT:    store i32 [[EXT]], ptr addrspace(1) poison, align 4
+; HSA-NEXT:    ret void
 ;
   %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %ptr)
   %ext = zext i1 %is.shared to i32
@@ -413,12 +413,12 @@ define amdgpu_kernel void @use_is_shared(ptr %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_is_private(ptr %ptr) #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_is_private
-; ATTRIBUTOR_HSA-SAME: (ptr [[PTR:%.*]]) #[[ATTR12]] {
-; ATTRIBUTOR_HSA-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
-; ATTRIBUTOR_HSA-NEXT:    [[EXT:%.*]] = zext i1 [[IS_PRIVATE]] to i32
-; ATTRIBUTOR_HSA-NEXT:    store i32 [[EXT]], ptr addrspace(1) poison, align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_is_private
+; HSA-SAME: (ptr [[PTR:%.*]]) #[[ATTR12]] {
+; HSA-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
+; HSA-NEXT:    [[EXT:%.*]] = zext i1 [[IS_PRIVATE]] to i32
+; HSA-NEXT:    store i32 [[EXT]], ptr addrspace(1) poison, align 4
+; HSA-NEXT:    ret void
 ;
   %is.private = call i1 @llvm.amdgcn.is.private(ptr %ptr)
   %ext = zext i1 %is.private to i32
@@ -427,11 +427,11 @@ define amdgpu_kernel void @use_is_private(ptr %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_alloca() #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_alloca
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR1]] {
-; ATTRIBUTOR_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
-; ATTRIBUTOR_HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_alloca
+; HSA-SAME: () #[[ATTR1]] {
+; HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
+; HSA-NEXT:    ret void
 ;
   %alloca = alloca i32, addrspace(5)
   store i32 0, ptr addrspace(5) %alloca
@@ -439,14 +439,14 @@ define amdgpu_kernel void @use_alloca() #1 {
 }
 
 define amdgpu_kernel void @use_alloca_non_entry_block() #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_alloca_non_entry_block
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR1]] {
-; ATTRIBUTOR_HSA-NEXT:  entry:
-; ATTRIBUTOR_HSA-NEXT:    br label [[BB:%.*]]
-; ATTRIBUTOR_HSA:       bb:
-; ATTRIBUTOR_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
-; ATTRIBUTOR_HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_alloca_non_entry_block
+; HSA-SAME: () #[[ATTR1]] {
+; HSA-NEXT:  entry:
+; HSA-NEXT:    br label [[BB:%.*]]
+; HSA:       bb:
+; HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
+; HSA-NEXT:    ret void
 ;
 entry:
   br label %bb
@@ -458,11 +458,11 @@ bb:
 }
 
 define void @use_alloca_func() #1 {
-; ATTRIBUTOR_HSA-LABEL: define {{[^@]+}}@use_alloca_func
-; ATTRIBUTOR_HSA-SAME: () #[[ATTR1]] {
-; ATTRIBUTOR_HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
-; ATTRIBUTOR_HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
-; ATTRIBUTOR_HSA-NEXT:    ret void
+; HSA-LABEL: define {{[^@]+}}@use_alloca_func
+; HSA-SAME: () #[[ATTR1]] {
+; HSA-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; HSA-NEXT:    store i32 0, ptr addrspace(5) [[ALLOCA]], align 4
+; HSA-NEXT:    ret void
 ;
   %alloca = alloca i32, addrspace(5)
   store i32 0, ptr addrspace(5) %alloca
@@ -475,24 +475,22 @@ attributes #1 = { nounwind }
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 500}
 
-;.
-;.
-; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-;.
 ; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
-; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
+; HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+;.
+; HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
index 15dc1a0529254..81ccf16c4e4bc 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -S -mtriple=amdgcn-unknown-unknown -passes=amdgpu-attributor < %s | FileCheck -check-prefixes=CHECK,ATTRIBUTOR_CHECK %s
+; RUN: opt -S -mtriple=amdgcn-unknown-unknown -passes=amdgpu-attributor < %s | FileCheck %s
 
 declare i32 @llvm.r600.read.tgid.x() #0
 declare i32 @llvm.r600.read.tgid.y() #0
@@ -26,11 +26,11 @@ define amdgpu_kernel void @use_tgid_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_y
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; ATTRIBUTOR_CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tgid_y
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tgid.y()
+; CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val = call i32 @llvm.r600.read.tgid.y()
   store i32 %val, ptr addrspace(1) %ptr
@@ -38,13 +38,13 @@ define amdgpu_kernel void @use_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @multi_use_tgid_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@multi_use_tgid_y
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@multi_use_tgid_y
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.y()
+; CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
+; CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.r600.read.tgid.y()
   store volatile i32 %val0, ptr addrspace(1) %ptr
@@ -54,13 +54,13 @@ define amdgpu_kernel void @multi_use_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_y
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tgid_x_y
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
+; CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
+; CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.r600.read.tgid.x()
   %val1 = call i32 @llvm.r600.read.tgid.y()
@@ -70,11 +70,11 @@ define amdgpu_kernel void @use_tgid_x_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_z
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; ATTRIBUTOR_CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tgid_z
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tgid.z()
+; CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val = call i32 @llvm.r600.read.tgid.z()
   store i32 %val, ptr addrspace(1) %ptr
@@ -82,13 +82,13 @@ define amdgpu_kernel void @use_tgid_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_z
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tgid_x_z
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
+; CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.z()
+; CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.r600.read.tgid.x()
   %val1 = call i32 @llvm.r600.read.tgid.z()
@@ -98,13 +98,13 @@ define amdgpu_kernel void @use_tgid_x_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_y_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_y_z
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tgid_y_z
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.y()
+; CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.z()
+; CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.r600.read.tgid.y()
   %val1 = call i32 @llvm.r600.read.tgid.z()
@@ -114,15 +114,15 @@ define amdgpu_kernel void @use_tgid_y_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tgid_x_y_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tgid_x_y_z
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tgid_x_y_z
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tgid.x()
+; CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
+; CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tgid.z()
+; CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.r600.read.tgid.x()
   %val1 = call i32 @llvm.r600.read.tgid.y()
@@ -146,11 +146,11 @@ define amdgpu_kernel void @use_tidig_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tidig_y
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tidig.y()
-; ATTRIBUTOR_CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tidig_y
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tidig.y()
+; CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val = call i32 @llvm.r600.read.tidig.y()
   store i32 %val, ptr addrspace(1) %ptr
@@ -158,11 +158,11 @@ define amdgpu_kernel void @use_tidig_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tidig_z
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tidig.z()
-; ATTRIBUTOR_CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tidig_z
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR6:[0-9]+]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.r600.read.tidig.z()
+; CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val = call i32 @llvm.r600.read.tidig.z()
   store i32 %val, ptr addrspace(1) %ptr
@@ -186,13 +186,13 @@ define amdgpu_kernel void @use_tidig_x_tgid_x(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_y_tgid_y(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tidig_y_tgid_y
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR7:[0-9]+]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.y()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tidig_y_tgid_y
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR7:[0-9]+]] {
+; CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.y()
+; CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tgid.y()
+; CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.r600.read.tidig.y()
   %val1 = call i32 @llvm.r600.read.tgid.y()
@@ -202,15 +202,15 @@ define amdgpu_kernel void @use_tidig_y_tgid_y(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_tidig_x_y_z(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_tidig_x_y_z
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR8:[0-9]+]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.x()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tidig.y()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tidig.z()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_tidig_x_y_z
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR8:[0-9]+]] {
+; CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.x()
+; CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tidig.y()
+; CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tidig.z()
+; CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.r600.read.tidig.x()
   %val1 = call i32 @llvm.r600.read.tidig.y()
@@ -222,21 +222,21 @@ define amdgpu_kernel void @use_tidig_x_y_z(ptr addrspace(1) %ptr) #1 {
 }
 
 define amdgpu_kernel void @use_all_workitems(ptr addrspace(1) %ptr) #1 {
-; ATTRIBUTOR_CHECK-LABEL: define {{[^@]+}}@use_all_workitems
-; ATTRIBUTOR_CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR9:[0-9]+]] {
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.x()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tidig.y()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tidig.z()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL3:%.*]] = call i32 @llvm.r600.read.tgid.x()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL4:%.*]] = call i32 @llvm.r600.read.tgid.y()
-; ATTRIBUTOR_CHECK-NEXT:    [[VAL5:%.*]] = call i32 @llvm.r600.read.tgid.z()
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL3]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL4]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    store volatile i32 [[VAL5]], ptr addrspace(1) [[PTR]], align 4
-; ATTRIBUTOR_CHECK-NEXT:    ret void
+; CHECK-LABEL: define {{[^@]+}}@use_all_workitems
+; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]]) #[[ATTR9:[0-9]+]] {
+; CHECK-NEXT:    [[VAL0:%.*]] = call i32 @llvm.r600.read.tidig.x()
+; CHECK-NEXT:    [[VAL1:%.*]] = call i32 @llvm.r600.read.tidig.y()
+; CHECK-NEXT:    [[VAL2:%.*]] = call i32 @llvm.r600.read.tidig.z()
+; CHECK-NEXT:    [[VAL3:%.*]] = call i32 @llvm.r600.read.tgid.x()
+; CHECK-NEXT:    [[VAL4:%.*]] = call i32 @llvm.r600.read.tgid.y()
+; CHECK-NEXT:    [[VAL5:%.*]] = call i32 @llvm.r600.read.tgid.z()
+; CHECK-NEXT:    store volatile i32 [[VAL0]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL2]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL3]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL4]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    store volatile i32 [[VAL5]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    ret void
 ;
   %val0 = call i32 @llvm.r600.read.tidig.x()
   %val1 = call i32 @llvm.r600.read.tidig.y()
@@ -293,15 +293,14 @@ attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
 
 ;.
-;.
-; ATTRIBUTOR_CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR3]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR4]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR5]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR6]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR7]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR8]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
-; ATTRIBUTOR_CHECK: attributes #[[ATTR9]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR3]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR4]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR5]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR6]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR7]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR8]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
+; CHECK: attributes #[[ATTR9]] = { nounwind "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" }
 ;.

From 7c9f4f1128ce6fd87aa274089b7c4271185331d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 18 Apr 2025 13:11:37 -0700
Subject: [PATCH 475/710] [flang][openacc] Make num_gangs, num_workers and
 vector_length behavior homogenous with parallel (#136341)

---
 flang/test/Semantics/OpenACC/acc-kernels.f90 | 33 +++++++++++++++++
 llvm/include/llvm/Frontend/OpenACC/ACC.td    | 38 +++++++++-----------
 2 files changed, 50 insertions(+), 21 deletions(-)

diff --git a/flang/test/Semantics/OpenACC/acc-kernels.f90 b/flang/test/Semantics/OpenACC/acc-kernels.f90
index 8a209040a7793..33e253ca8d874 100644
--- a/flang/test/Semantics/OpenACC/acc-kernels.f90
+++ b/flang/test/Semantics/OpenACC/acc-kernels.f90
@@ -61,12 +61,45 @@ program openacc_kernels_validity
   !$acc kernels num_gangs(8)
   !$acc end kernels
 
+  !ERROR: At most one NUM_GANGS clause can appear on the KERNELS directive or in group separated by the DEVICE_TYPE clause
+  !$acc kernels num_gangs(8) num_gangs(10)
+  !$acc end kernels
+
+  !ERROR: At most one NUM_GANGS clause can appear on the KERNELS directive or in group separated by the DEVICE_TYPE clause
+  !$acc kernels device_type(nvidia) num_gangs(8) num_gangs(10)
+  !$acc end kernels
+
+  !$acc kernels device_type(nvidia) num_gangs(8) device_type(radeon) num_gangs(10)
+  !$acc end kernels
+
   !$acc kernels num_workers(8)
   !$acc end kernels
 
+  !ERROR: At most one NUM_WORKERS clause can appear on the KERNELS directive or in group separated by the DEVICE_TYPE clause
+  !$acc kernels num_workers(8) num_workers(4)
+  !$acc end kernels
+
+  !ERROR: At most one NUM_WORKERS clause can appear on the KERNELS directive or in group separated by the DEVICE_TYPE clause
+  !$acc kernels device_type(nvidia) num_workers(8) num_workers(4)
+  !$acc end kernels
+
+  !$acc kernels device_type(nvidia) num_workers(8) device_type(radeon) num_workers(4)
+  !$acc end kernels
+
   !$acc kernels vector_length(128)
   !$acc end kernels
 
+  !ERROR: At most one VECTOR_LENGTH clause can appear on the KERNELS directive or in group separated by the DEVICE_TYPE clause
+  !$acc kernels vector_length(128) vector_length(124)
+  !$acc end kernels
+
+  !ERROR: At most one VECTOR_LENGTH clause can appear on the KERNELS directive or in group separated by the DEVICE_TYPE clause
+  !$acc kernels device_type(nvidia) vector_length(256) vector_length(128)
+  !$acc end kernels
+
+  !$acc kernels device_type(nvidia) vector_length(256) device_type(radeon) vector_length(128)
+  !$acc end kernels
+
   !$acc kernels if(.true.)
   !$acc end kernels
 
diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index e1a4183785d1f..898b7dc9e230f 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -319,27 +319,23 @@ def ACC_Declare : Directive<"declare"> {
 
 // 2.5.3
 def ACC_Kernels : Directive<"kernels"> {
-  let allowedClauses = [
-    VersionedClause<ACCC_Attach>,
-    VersionedClause<ACCC_Copy>,
-    VersionedClause<ACCC_Copyin>,
-    VersionedClause<ACCC_Copyout>,
-    VersionedClause<ACCC_Create>,
-    VersionedClause<ACCC_DeviceType>,
-    VersionedClause<ACCC_NoCreate>,
-    VersionedClause<ACCC_Present>,
-    VersionedClause<ACCC_DevicePtr>,
-    VersionedClause<ACCC_Wait>
-  ];
-  let allowedOnceClauses = [
-    VersionedClause<ACCC_Async>,
-    VersionedClause<ACCC_Default>,
-    VersionedClause<ACCC_If>,
-    VersionedClause<ACCC_NumGangs>,
-    VersionedClause<ACCC_NumWorkers>,
-    VersionedClause<ACCC_Self>,
-    VersionedClause<ACCC_VectorLength>
-  ];
+  let allowedClauses = [VersionedClause<ACCC_Attach>,
+                        VersionedClause<ACCC_Copy>,
+                        VersionedClause<ACCC_Copyin>,
+                        VersionedClause<ACCC_Copyout>,
+                        VersionedClause<ACCC_Create>,
+                        VersionedClause<ACCC_DeviceType>,
+                        VersionedClause<ACCC_NoCreate>,
+                        VersionedClause<ACCC_NumGangs>,
+                        VersionedClause<ACCC_NumWorkers>,
+                        VersionedClause<ACCC_Present>,
+                        VersionedClause<ACCC_DevicePtr>,
+                        VersionedClause<ACCC_VectorLength>,
+                        VersionedClause<ACCC_Wait>];
+  let allowedOnceClauses = [VersionedClause<ACCC_Async>,
+                            VersionedClause<ACCC_Default>,
+                            VersionedClause<ACCC_If>,
+                            VersionedClause<ACCC_Self>];
   let association = AS_Block;
   let category = CA_Executable;
 }

From 15e662bf0646200d1fbe28a651fc8d4247bd13cf Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Fri, 18 Apr 2025 13:12:50 -0700
Subject: [PATCH 476/710] [NFC][flang] Removed literal numerical references
 from some LIT tests. (#136346)

---
 flang/test/Lower/HLFIR/procedure-pointer.f90     | 16 ++++++++--------
 flang/test/Lower/Intrinsics/getpid.f90           |  2 +-
 flang/test/Lower/Intrinsics/time.f90             |  4 ++--
 flang/test/Lower/OpenACC/acc-atomic-capture.f90  |  4 ++--
 .../OpenACC/acc-declare-unwrap-defaultbounds.f90 |  2 +-
 .../OpenACC/acc-private-unwrap-defaultbounds.f90 |  2 +-
 flang/test/Lower/OpenACC/acc-private.f90         |  2 +-
 flang/test/Lower/OpenMP/allocatable-map.f90      |  2 +-
 flang/test/Lower/OpenMP/array-bounds.f90         |  4 ++--
 flang/test/Lower/OpenMP/atomic-read.f90          |  2 +-
 flang/test/Lower/OpenMP/copyin.f90               |  2 +-
 flang/test/Lower/OpenMP/derived-type-map.f90     |  4 ++--
 .../Lower/OpenMP/parallel-private-clause.f90     | 10 +++++-----
 flang/test/Lower/OpenMP/target.f90               |  2 +-
 .../Lower/OpenMP/threadprivate-commonblock.f90   |  6 +++---
 flang/test/Lower/io-derived-type.f90             |  2 +-
 flang/test/Lower/parent-component.f90            |  2 +-
 17 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/flang/test/Lower/HLFIR/procedure-pointer.f90 b/flang/test/Lower/HLFIR/procedure-pointer.f90
index 2ef3545aa6375..053c44f56ed31 100644
--- a/flang/test/Lower/HLFIR/procedure-pointer.f90
+++ b/flang/test/Lower/HLFIR/procedure-pointer.f90
@@ -80,7 +80,7 @@ subroutine  sub2()
 ! CHECK: %[[VAL_1:.*]] = fir.zero_bits (!fir.ref<f32>) -> f32
 ! CHECK: %[[VAL_2:.*]] = fir.emboxproc %[[VAL_1]] : ((!fir.ref<f32>) -> f32) -> !fir.boxproc<(!fir.ref<f32>) -> f32>
 ! CHECK: fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>
-! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub2Ep1"} : (!fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>) -> (!fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>, !fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>)
+! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub2Ep1"} : (!fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>) -> (!fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>, !fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>)
 ! CHECK: %[[VAL_4:.*]] = fir.zero_bits () -> ()
 ! CHECK: %[[VAL_5:.*]] = fir.emboxproc %[[VAL_4]] : (() -> ()) -> !fir.boxproc<() -> ()>
 ! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.boxproc<() -> ()>) -> !fir.boxproc<(!fir.ref<f32>) -> f32>
@@ -97,7 +97,7 @@ subroutine  sub3()
 ! CHECK: %[[VAL_1:.*]] = fir.zero_bits (!fir.ref<f32>) -> f32
 ! CHECK: %[[VAL_2:.*]] = fir.emboxproc %[[VAL_1]] : ((!fir.ref<f32>) -> f32) -> !fir.boxproc<(!fir.ref<f32>) -> f32>
 ! CHECK: fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>
-! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub3Ep1"} : (!fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>) -> (!fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>, !fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>)
+! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub3Ep1"} : (!fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>) -> (!fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>, !fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>)
 ! CHECK: %[[VAL_4:.*]] = fir.address_of(@_QPreal_func) : (!fir.ref<f32>) -> f32
 ! CHECK: %[[VAL_5:.*]] = fir.emboxproc %[[VAL_4]] : ((!fir.ref<f32>) -> f32) -> !fir.boxproc<() -> ()>
 ! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.boxproc<() -> ()>) -> !fir.boxproc<(!fir.ref<f32>) -> f32>
@@ -106,7 +106,7 @@ subroutine  sub3()
   res = p1(r)
 ! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.boxproc<(!fir.ref<f32>) -> f32>>
 ! CHECK: %[[VAL_8:.*]] = fir.box_addr %[[VAL_7]] : (!fir.boxproc<(!fir.ref<f32>) -> f32>) -> ((!fir.ref<f32>) -> f32)
-! CHECK: %[[VAL_9:.*]] = fir.call %[[VAL_8]](%5#0) fastmath<contract> : (!fir.ref<f32>) -> f32
+! CHECK: %[[VAL_9:.*]] = fir.call %[[VAL_8]](%{{.*}}) fastmath<contract> : (!fir.ref<f32>) -> f32
 
   nullify(p1)
 ! CHECK: %[[VAL_10:.*]] = fir.zero_bits () -> ()
@@ -141,7 +141,7 @@ subroutine  sub4()
   res = p2(i)
 ! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> !fir.box<!fir.ptr<!fir.char<1,?>>>>>
 ! CHECK: %[[VAL_13:.*]] = fir.box_addr %[[VAL_12]] : (!fir.boxproc<(!fir.ref<i32>) -> !fir.box<!fir.ptr<!fir.char<1,?>>>>) -> ((!fir.ref<i32>) -> !fir.box<!fir.ptr<!fir.char<1,?>>>)
-! CHECK: %[[VAL_14:.*]] = fir.call %[[VAL_13]](%2#0) fastmath<contract> : (!fir.ref<i32>) -> !fir.box<!fir.ptr<!fir.char<1,?>>>
+! CHECK: %[[VAL_14:.*]] = fir.call %[[VAL_13]](%{{.*}}) fastmath<contract> : (!fir.ref<i32>) -> !fir.box<!fir.ptr<!fir.char<1,?>>>
 end subroutine
 
 subroutine  sub5()
@@ -153,7 +153,7 @@ subroutine  sub5()
 ! CHECK: %[[VAL_1:.*]] = fir.zero_bits () -> f32
 ! CHECK: %[[VAL_2:.*]] = fir.emboxproc %[[VAL_1]] : (() -> f32) -> !fir.boxproc<() -> f32>
 ! CHECK: fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.boxproc<() -> f32>>
-! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub5Ep3"} : (!fir.ref<!fir.boxproc<() -> f32>>) -> (!fir.ref<!fir.boxproc<() -> f32>>, !fir.ref<!fir.boxproc<() -> f32>>)
+! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub5Ep3"} : (!fir.ref<!fir.boxproc<() -> f32>>) -> (!fir.ref<!fir.boxproc<() -> f32>>, !fir.ref<!fir.boxproc<() -> f32>>)
 ! CHECK: %[[VAL_4:.*]] = fir.address_of(@_QPreal_func) : (!fir.ref<f32>) -> f32
 ! CHECK: %[[VAL_5:.*]] = fir.emboxproc %[[VAL_4]] : ((!fir.ref<f32>) -> f32) -> !fir.boxproc<() -> ()>
 ! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.boxproc<() -> ()>) -> !fir.boxproc<() -> f32>
@@ -170,7 +170,7 @@ subroutine  sub6()
 ! CHECK: %[[VAL_1:.*]] = fir.zero_bits () -> ()
 ! CHECK: %[[VAL_2:.*]] = fir.emboxproc %[[VAL_1]] : (() -> ()) -> !fir.boxproc<() -> ()>
 ! CHECK: fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.boxproc<() -> ()>>
-! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub6Ep4"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub6Ep4"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK: %[[VAL_4:.*]] = fir.address_of(@_QPsub) : (!fir.ref<f32>) -> ()
 ! CHECK: %[[VAL_5:.*]] = fir.emboxproc %[[VAL_4]] : ((!fir.ref<f32>) -> ()) -> !fir.boxproc<() -> ()>
 ! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref<!fir.boxproc<() -> ()>>
@@ -178,7 +178,7 @@ subroutine  sub6()
   call p4(r)
 ! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.boxproc<() -> ()>>
 ! CHECK: %[[VAL_7:.*]] = fir.box_addr %[[VAL_6]] : (!fir.boxproc<() -> ()>) -> ((!fir.ref<f32>) -> ())
-! CHECK: fir.call %[[VAL_7]](%5#0) fastmath<contract> : (!fir.ref<f32>) -> ()
+! CHECK: fir.call %[[VAL_7]](%{{.*}}) fastmath<contract> : (!fir.ref<f32>) -> ()
 end subroutine
 
 
@@ -302,7 +302,7 @@ function reffunc(arg) result(pp)
   p1 => reffunc(5)
 ! CHECK: %c5_i32 = arith.constant 5 : i32
 ! CHECK: %[[VAL_4:.*]]:3 = hlfir.associate %c5_i32 {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
-! CHECK: %[[VAL_5:.*]] = fir.call @_QPreffunc(%4#0) fastmath<contract> : (!fir.ref<i32>) -> !fir.boxproc<(!fir.ref<i32>) -> !fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK: %[[VAL_5:.*]] = fir.call @_QPreffunc(%{{.*}}) fastmath<contract> : (!fir.ref<i32>) -> !fir.boxproc<(!fir.ref<i32>) -> !fir.box<!fir.ptr<!fir.char<1,?>>>>
 ! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> !fir.box<!fir.ptr<!fir.char<1,?>>>>>
 ! CHECK: return
 end
diff --git a/flang/test/Lower/Intrinsics/getpid.f90 b/flang/test/Lower/Intrinsics/getpid.f90
index be459ad9e4487..fd03bbf2bafe2 100644
--- a/flang/test/Lower/Intrinsics/getpid.f90
+++ b/flang/test/Lower/Intrinsics/getpid.f90
@@ -2,7 +2,7 @@
 
 ! CHECK-LABEL: func.func @_QPall_args() {
 ! CHECK:         %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "pid", uniq_name = "_QFall_argsEpid"}
-! CHECK:         %[[VAL_1:.*]]:2 = hlfir.declare %0 {uniq_name = "_QFall_argsEpid"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:         %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFall_argsEpid"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:         %[[VAL_2:.*]] = fir.call @_FortranAGetPID() fastmath<contract> : () -> i32
 ! CHECK:         hlfir.assign %[[VAL_2:.*]] to %[[VAL_1:.*]]#0 : i32, !fir.ref<i32>
 ! CHECK:         return
diff --git a/flang/test/Lower/Intrinsics/time.f90 b/flang/test/Lower/Intrinsics/time.f90
index eac492572ef89..67584200547e1 100644
--- a/flang/test/Lower/Intrinsics/time.f90
+++ b/flang/test/Lower/Intrinsics/time.f90
@@ -10,8 +10,8 @@ function time_test()
   !CHECK:     %[[call_result:.*]] = fir.call @_FortranAtime()
   !CHECK-SAME:  -> i64
 
-  !CHECK-DAG:   hlfir.assign %[[call_result]] to %[[func_result_decl]]#[[func_result]] : i64, !fir.ref<i64>
-  !CHECK-DAG:   %[[load_result:.*]] = fir.load %[[func_result_decl]]#[[func_result]] : !fir.ref<i64>
+  !CHECK-DAG:   hlfir.assign %[[call_result]] to %[[func_result_decl]]#0 : i64, !fir.ref<i64>
+  !CHECK-DAG:   %[[load_result:.*]] = fir.load %[[func_result_decl]]#0 : !fir.ref<i64>
   !CHECK:     return %[[load_result]] : i64
   time_test = time()
 end function time_test
diff --git a/flang/test/Lower/OpenACC/acc-atomic-capture.f90 b/flang/test/Lower/OpenACC/acc-atomic-capture.f90
index 6cf39cde6ca0d..82059908bcd0b 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-capture.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-capture.f90
@@ -6,9 +6,9 @@ program acc_atomic_capture_test
     integer :: x, y
 
 !CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
-!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
-!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %2 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[temp:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
 !CHECK: acc.atomic.capture {
 !CHECK: acc.atomic.read %[[X_DECL]]#0 = %[[Y_DECL]]#0 : !fir.ref<i32>, !fir.ref<i32>, i32
diff --git a/flang/test/Lower/OpenACC/acc-declare-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-declare-unwrap-defaultbounds.f90
index 950dc30c96129..5bb1ae3797346 100644
--- a/flang/test/Lower/OpenACC/acc-declare-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-declare-unwrap-defaultbounds.f90
@@ -210,7 +210,7 @@ function acc_declare_in_func()
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>)
 ! CHECK: %[[LOAD:.*]] = fir.load %{{.*}}#0 : !fir.ref<f32>
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>)
-! CHECK: acc.delete accPtr(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>) bounds(%6) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "a"}
+! CHECK: acc.delete accPtr(%[[DEVICE_RESIDENT]] : !fir.ref<!fir.array<1024xf32>>) bounds(%{{.*}}) {dataClause = #acc<data_clause acc_declare_device_resident>, name = "a"}
 ! CHECK: return %[[LOAD]] : f32
 ! CHECK: }
 
diff --git a/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
index a43228070bd8f..b1dc4e79f9f73 100644
--- a/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-private-unwrap-defaultbounds.f90
@@ -394,7 +394,7 @@ subroutine acc_private_use()
 
 ! CHECK-LABEL: func.func @_QPacc_private_use()
 ! CHECK: %[[I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFacc_private_useEi"}
-! CHECK: %[[DECL_I:.*]]:2 = hlfir.declare %0 {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[DECL_I:.*]]:2 = hlfir.declare %[[I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: acc.parallel
 ! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
 ! CHECK: %[[DECL_PRIV_I:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index 9827350271e65..76e23b8915e92 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -412,7 +412,7 @@ subroutine acc_private_use()
 
 ! CHECK-LABEL: func.func @_QPacc_private_use()
 ! CHECK: %[[I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFacc_private_useEi"}
-! CHECK: %[[DECL_I:.*]]:2 = hlfir.declare %0 {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[DECL_I:.*]]:2 = hlfir.declare %[[I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: acc.parallel
 ! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#0 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "i"}
 ! CHECK: %[[DECL_PRIV_I:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/allocatable-map.f90 b/flang/test/Lower/OpenMP/allocatable-map.f90
index 0c9874c5d10a3..ee1c621ad860b 100644
--- a/flang/test/Lower/OpenMP/allocatable-map.f90
+++ b/flang/test/Lower/OpenMP/allocatable-map.f90
@@ -1,6 +1,6 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s --check-prefixes="HLFIRDIALECT"
 
-!HLFIRDIALECT: %[[POINTER:.*]]:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFpointer_routineEpoint"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+!HLFIRDIALECT: %[[POINTER:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFpointer_routineEpoint"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 !HLFIRDIALECT: %[[BOX_OFF:.*]] = fir.box_offset %[[POINTER]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
 !HLFIRDIALECT: %[[POINTER_MAP_MEMBER:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, i32) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
 !HLFIRDIALECT: %[[POINTER_MAP:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to) capture(ByRef) members(%[[POINTER_MAP_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "point"}
diff --git a/flang/test/Lower/OpenMP/array-bounds.f90 b/flang/test/Lower/OpenMP/array-bounds.f90
index a4fd769b1ecd5..8f98d671486ae 100644
--- a/flang/test/Lower/OpenMP/array-bounds.f90
+++ b/flang/test/Lower/OpenMP/array-bounds.f90
@@ -2,7 +2,7 @@
 
 
 !HOST-LABEL:  func.func @_QPread_write_section() {
-!HOST:  %0 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFread_write_sectionEi"}
+!HOST:  %{{.*}} = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFread_write_sectionEi"}
 !HOST:  %[[READ:.*]] = fir.address_of(@_QFread_write_sectionEsp_read) : !fir.ref<!fir.array<10xi32>>
 !HOST:  %[[C10:.*]] = arith.constant 10 : index
 !HOST:  %[[READ_SHAPE:.*]] = fir.shape %[[C10]] : (index) -> !fir.shape<1>
@@ -50,7 +50,7 @@ module assumed_array_routines
 !HOST: %[[C0_1:.*]] = arith.constant 0 : index
 !HOST: %[[DIMS1:.*]]:3 = fir.box_dims %[[ARG0_DECL]]#1, %[[C0_1]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
 !HOST: %[[BOUNDS:.*]] = omp.map.bounds   lower_bound(%[[C3]] : index) upper_bound(%[[C4]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) start_idx(%[[C0]] : index) {stride_in_bytes = true}
-!HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %0 base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+!HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %[[INTERMEDIATE_ALLOCA]] base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
 !HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, i32) map_clauses(tofrom) capture(ByRef) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) bounds(%[[BOUNDS]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>> {name = ""}
 !HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(to) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>> {name = "arr_read_write(2:5)"}
 !HOST: omp.target   map_entries(%[[MAP]] -> %{{.*}}, {{.*}} -> {{.*}}, %[[MAP_INFO_MEMBER]] -> %{{.*}} : !fir.ref<!fir.array<?xi32>>, !fir.ref<i32>, !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) {
diff --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
index e002d5831d855..68dcaac90eef5 100644
--- a/flang/test/Lower/OpenMP/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -60,7 +60,7 @@ end program OmpAtomic
 
 !CHECK-LABEL: func.func @_QPatomic_read_pointer() {
 !CHECK:    %[[X_REF:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "x", uniq_name = "_QFatomic_read_pointerEx"}
-!CHECK:    fir.store %2 to %0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK:    fir.store %{{.*}} to %[[X_REF]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
 !CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFatomic_read_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 !CHECK:    %[[Y_REF:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "y", uniq_name = "_QFatomic_read_pointerEy"}
 !CHECK:    %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFatomic_read_pointerEy"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
diff --git a/flang/test/Lower/OpenMP/copyin.f90 b/flang/test/Lower/OpenMP/copyin.f90
index be70f90ca3e60..9f27b1b20baf8 100644
--- a/flang/test/Lower/OpenMP/copyin.f90
+++ b/flang/test/Lower/OpenMP/copyin.f90
@@ -561,7 +561,7 @@ subroutine common_3()
 ! CHECK:           }
 ! CHECK:          }
 ! CHECK:        omp.barrier
-! CHECK:        %[[VAL_22:.*]] = fir.load %7#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK:        %[[VAL_22:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK:        %[[VAL_23:.*]] = fir.box_addr %[[VAL_22]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
 ! CHECK:       %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.heap<i32>
 ! CHECK:        %[[C1_I32:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/derived-type-map.f90 b/flang/test/Lower/OpenMP/derived-type-map.f90
index e6a3b9ada762b..279cddec51fcf 100644
--- a/flang/test/Lower/OpenMP/derived-type-map.f90
+++ b/flang/test/Lower/OpenMP/derived-type-map.f90
@@ -83,7 +83,7 @@ end subroutine mapType_derived_explicit_multiple_members
 !CHECK: %[[LB:.*]] = arith.constant 1 : index
 !CHECK: %[[UB:.*]] = arith.constant 4 : index
 !CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%[[LB]] : index) upper_bound(%[[UB]] : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index)
-!CHECK: %[[MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[MEMBER]] : !fir.ref<!fir.array<10xi32>>, !fir.array<10xi32>) map_clauses(tofrom) capture(ByRef) bounds(%20) -> !fir.ref<!fir.array<10xi32>> {name = "scalar_arr%array(2:5)"}
+!CHECK: %[[MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[MEMBER]] : !fir.ref<!fir.array<10xi32>>, !fir.array<10xi32>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.ref<!fir.array<10xi32>> {name = "scalar_arr%array(2:5)"}
 !CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[MEMBER_MAP]] : [1] : !fir.ref<!fir.array<10xi32>>) -> !fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr", partial_map = true}
 !CHECK: omp.target map_entries(%[[PARENT_MAP]] -> %[[ARG0:.*]], %[[MEMBER_MAP]] -> %[[ARG1:.*]] : !fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.array<10xi32>>) {
 subroutine mapType_derived_explicit_member_with_bounds
@@ -105,7 +105,7 @@ end subroutine mapType_derived_explicit_member_with_bounds
 !CHECK: %[[NEST_MEMBER:.*]] = hlfir.designate %[[NEST]]{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_single_memberTnested{int:i32,real:f32,array:!fir.array<10xi32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
 !CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%{{.*}} : index) upper_bound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index)
 !CHECK: %[[MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[NEST_MEMBER]] : !fir.ref<!fir.array<10xi32>>, !fir.array<10xi32>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.ref<!fir.array<10xi32>> {name = "scalar_arr%nest%array"}
-!CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : {{.*}}) map_clauses(tofrom) capture(ByRef) members(%35 : [2, 2] : !fir.ref<!fir.array<10xi32>>) -> {{.*}} {name = "scalar_arr", partial_map = true}
+!CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : {{.*}}) map_clauses(tofrom) capture(ByRef) members(%[[MEMBER_MAP]] : [2, 2] : !fir.ref<!fir.array<10xi32>>) -> {{.*}} {name = "scalar_arr", partial_map = true}
 !CHECK: omp.target map_entries(%[[PARENT_MAP]] -> %[[ARG0:.*]], %[[MEMBER_MAP]] -> %[[ARG1:.*]] : {{.*}}, {{.*}}) {
 subroutine mapType_derived_nested_explicit_single_member
   type :: nested
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause.f90 b/flang/test/Lower/OpenMP/parallel-private-clause.f90
index 0f55f791fa4c8..492fb3bb9740d 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause.f90
@@ -166,8 +166,8 @@ subroutine private_clause_allocatable()
 !FIRDialect: func @_QPprivate_clause_real_call_allocatable() {
 !FIRDialect-DAG: %[[X5:.*]] = fir.alloca !fir.box<!fir.heap<f32>> {bindc_name = "x5", uniq_name = "{{.*}}Ex5"}
 !FIRDialect-DAG: {{.*}} = fir.zero_bits !fir.heap<f32>
-!FIRDialect-DAG: {{.*}} = fir.embox %1 : (!fir.heap<f32>) -> !fir.box<!fir.heap<f32>>
-!FIRDialect-DAG: fir.store %2 to %[[X5]] : !fir.ref<!fir.box<!fir.heap<f32>>>
+!FIRDialect-DAG: {{.*}} = fir.embox %{{.*}} : (!fir.heap<f32>) -> !fir.box<!fir.heap<f32>>
+!FIRDialect-DAG: fir.store %{{.*}} to %[[X5]] : !fir.ref<!fir.box<!fir.heap<f32>>>
 !FIRDialect-DAG: %[[X5_DECL:.*]]:2 = hlfir.declare %[[X5]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFprivate_clause_real_call_allocatableEx5"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 !FIRDialect-DAG: omp.parallel private(@{{.*}} %{{.*}}#0 -> %[[X5_PVT:.*]] : {{.*}}) {
 
@@ -223,16 +223,16 @@ end subroutine increment_list_items
 
 !FIRDialect:  func.func @_QPparallel_pointer() {
 !FIRDialect-DAG: %[[Y1:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "y1", uniq_name = "_QFparallel_pointerEy1"}
-!FIRDialect:     fir.store %2 to %[[Y1]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!FIRDialect:     fir.store %{{.*}} to %[[Y1]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
 !FIRDialect-DAG: %[[Y1_DECL:.*]]:2 = hlfir.declare %[[Y1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFparallel_pointerEy1"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 !FIRDialect-DAG: %[[Y2:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xi32>>> {bindc_name = "y2", uniq_name = "_QFparallel_pointerEy2"}
 
-!FIRDialect:     fir.store %7 to %[[Y2]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+!FIRDialect:     fir.store %{{.*}} to %[[Y2]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 !FIRDialect-DAG: %[[Y2_DECL:.*]]:2 = hlfir.declare %[[Y2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFparallel_pointerEy2"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
 !FIRDialect-DAG: %[[Z1:.*]] = fir.alloca i32 {bindc_name = "z1", fir.target, uniq_name = "_QFparallel_pointerEz1"}
 !FIRDialect-DAG: %[[Z1_DECL:.*]]:2 = hlfir.declare %[[Z1]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFparallel_pointerEz1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !FIRDialect-DAG:  %[[Z2:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "z2", fir.target, uniq_name = "_QFparallel_pointerEz2"}
-!FIRDialect-DAG:  %[[Z2_DECL:.*]]:2 = hlfir.declare %[[Z2]](%12) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFparallel_pointerEz2"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+!FIRDialect-DAG:  %[[Z2_DECL:.*]]:2 = hlfir.declare %[[Z2]](%{{.*}}) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFparallel_pointerEz2"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 !FIRDialect:      omp.parallel private(@{{.*}} %{{.*}}#0 -> %[[Y1_PVT:.*]], @{{.*}} %{{.*}}#0 -> %[[Y2_PVT:.*]] : {{.*}}) {
 !FIRDialect-DAG:    %[[Y1_PVT_DECL:.*]]:2 = hlfir.declare %[[Y1_PVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFparallel_pointerEy1"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 !FIRDialect-DAG:    %[[Y2_PVT_DECL:.*]]:2 = hlfir.declare %[[Y2_PVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFparallel_pointerEy2"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
diff --git a/flang/test/Lower/OpenMP/target.f90 b/flang/test/Lower/OpenMP/target.f90
index 36877210c136d..4815e6564fc7e 100644
--- a/flang/test/Lower/OpenMP/target.f90
+++ b/flang/test/Lower/OpenMP/target.f90
@@ -541,7 +541,7 @@ end subroutine omp_target_device_ptr
 subroutine omp_target_device_addr
    integer, pointer :: a
    !CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "a", uniq_name = "_QFomp_target_device_addrEa"}
-   !CHECK: %[[VAL_0_DECL:.*]]:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFomp_target_device_addrEa"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+   !CHECK: %[[VAL_0_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFomp_target_device_addrEa"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
    !CHECK: %[[MAP_MEMBERS:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, i32) map_clauses(tofrom) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
    !CHECK: %[[MAP:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(to) capture(ByRef) members(%[[MAP_MEMBERS]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "a"}
    !CHECK: %[[DEV_ADDR_MEMBERS:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, i32) map_clauses(tofrom) capture(ByRef) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
diff --git a/flang/test/Lower/OpenMP/threadprivate-commonblock.f90 b/flang/test/Lower/OpenMP/threadprivate-commonblock.f90
index 4c9e631713ec7..975c62baa3d71 100644
--- a/flang/test/Lower/OpenMP/threadprivate-commonblock.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-commonblock.f90
@@ -64,7 +64,7 @@ subroutine sub()
     !CHECK-DAG:  %[[TP_PARALLEL_ADDR:.*]] = fir.convert %[[TP_PARALLEL]] : (!fir.ref<!fir.array<103xi8>>) -> !fir.ref<!fir.array<?xi8>>
     !CHECK-DAG:  %[[TP_B_ADDR:.*]] = fir.coordinate_of %[[TP_PARALLEL_ADDR]], {{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
     !CHECK-DAG:  %[[TP_B_ADDR_CVT:.*]] = fir.convert %[[TP_B_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.array<2xf32>>
-    !CHECK-DAG:  %[[TP_B_DECL:.*]]:2 = hlfir.declare %[[TP_B_ADDR_CVT]](%92) {uniq_name = "_QMtestEb"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xf32>>, !fir.ref<!fir.array<2xf32>>)
+    !CHECK-DAG:  %[[TP_B_DECL:.*]]:2 = hlfir.declare %[[TP_B_ADDR_CVT]](%{{.*}}) {uniq_name = "_QMtestEb"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xf32>>, !fir.ref<!fir.array<2xf32>>)
     !CHECK-DAG:  %[[TP_PARALLEL_ADDR:.*]] = fir.convert %[[TP_PARALLEL]] : (!fir.ref<!fir.array<103xi8>>) -> !fir.ref<!fir.array<?xi8>>
     !CHECK-DAG:  %[[TP_C_ADDR:.*]] = fir.coordinate_of %[[TP_PARALLEL_ADDR]], {{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
     !CHECK-DAG:  %[[TP_C_ADDR_CVT:.*]] = fir.convert %[[TP_C_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
@@ -93,11 +93,11 @@ subroutine sub()
     !$omp end parallel
 
   !CHECK-DAG:  %{{.*}} = fir.load %[[A_DECL]]#0 : !fir.ref<i32>
-  !CHECK-DAG:  %{{.*}} = fir.embox %[[B_DECL]]#0(%63) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
+  !CHECK-DAG:  %{{.*}} = fir.embox %[[B_DECL]]#0(%{{.*}}) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
   !CHECK-DAG:  %{{.*}} = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
   !CHECK-DAG:  %{{.*}} = fir.load %[[D_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
   !CHECK-DAG:  %{{.*}} = fir.convert %[[E_DECL]]#0 : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
-  !CHECK-DAG:  %{{.*}} = fir.embox %[[F_DECL]]#0(%79) : (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.char<1,5>>>
+  !CHECK-DAG:  %{{.*}} = fir.embox %[[F_DECL]]#0(%{{.*}}) : (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.char<1,5>>>
     print *, a, b, c, d, e, f
 
   end
diff --git a/flang/test/Lower/io-derived-type.f90 b/flang/test/Lower/io-derived-type.f90
index 316a2cdb5b14f..7d2fef3faa2b7 100644
--- a/flang/test/Lower/io-derived-type.f90
+++ b/flang/test/Lower/io-derived-type.f90
@@ -118,7 +118,7 @@ program p
   print *, 'main, should call wft: ', t(4)
 
   ! CHECK:   %[[V_33:[0-9]+]] = fir.shape %c2{{.*}} : (index) -> !fir.shape<1>
-  ! CHECK:   %[[V_34:[0-9]+]] = hlfir.designate %7#0 (%c2{{.*}}:%c3{{.*}}:%c1{{.*}})  shape %[[V_33]] : (!fir.ref<!fir.array<5x!fir.type<_QMmTt{n:i32}>>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>
+  ! CHECK:   %[[V_34:[0-9]+]] = hlfir.designate %{{.*}}#0 (%c2{{.*}}:%c3{{.*}}:%c1{{.*}})  shape %[[V_33]] : (!fir.ref<!fir.array<5x!fir.type<_QMmTt{n:i32}>>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>
   ! CHECK:   %[[V_35:[0-9]+]] = fir.shape %c2{{.*}} : (index) -> !fir.shape<1>
   ! CHECK:   %[[V_36:[0-9]+]] = fir.embox %[[V_34]](%[[V_35]]) : (!fir.ref<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>
   ! CHECK:   %[[V_37:[0-9]+]] = fir.convert %[[V_36]] : (!fir.box<!fir.array<2x!fir.type<_QMmTt{n:i32}>>>) -> !fir.box<none>
diff --git a/flang/test/Lower/parent-component.f90 b/flang/test/Lower/parent-component.f90
index 7de20ea044905..66d25a0c12c86 100644
--- a/flang/test/Lower/parent-component.f90
+++ b/flang/test/Lower/parent-component.f90
@@ -132,7 +132,7 @@ subroutine init_existing_field()
 
   ! CHECK-LABEL: func.func private @_QFPinit_existing_field
   ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}"_QFFinit_existing_fieldEy"
-  ! CHECK:           %[[VAL_5:.*]] = hlfir.designate %[[VAL_4]]#0{"c"}   shape %[[VAL_3]] : (!fir.ref<!fir.array<2x!fir.type<_QFTz{k:i32,c:!fir.type<_QFTc{p:!fir.type<_QFTp{a:i32}>,b:i32}>}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QFTc{p:!fir.type<_QFTp{a:i32}>,b:i32}>>>
+  ! CHECK:           %[[VAL_5:.*]] = hlfir.designate %[[VAL_4]]#0{"c"}   shape %[[VAL_3:.*]] : (!fir.ref<!fir.array<2x!fir.type<_QFTz{k:i32,c:!fir.type<_QFTc{p:!fir.type<_QFTp{a:i32}>,b:i32}>}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QFTc{p:!fir.type<_QFTp{a:i32}>,b:i32}>>>
   ! CHECK:           %[[VAL_6:.*]] = hlfir.designate %[[VAL_5]]{"p"}   shape %[[VAL_3]] : (!fir.box<!fir.array<2x!fir.type<_QFTc{p:!fir.type<_QFTp{a:i32}>,b:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QFTp{a:i32}>>>
 
   subroutine parent_comp_lhs()

From 058992ea9fc62a9f79ab90eaebecb267469795a7 Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Fri, 18 Apr 2025 22:16:35 +0200
Subject: [PATCH 477/710] [lldb] Fix Python GIL-not-held issue in
 CreateStructuredDataFromScriptObject (#136309)

TestStructuredDataAPI.py fails with Python debug build ver. 3.12+ due to
call to Py_XINCREF while GIL is not held.
---
 .../ScriptInterpreter/Python/ScriptInterpreterPython.cpp        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index a9c81273c1302..553ee7e80b359 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -1569,10 +1569,10 @@ StructuredData::ObjectSP
 ScriptInterpreterPythonImpl::CreateStructuredDataFromScriptObject(
     ScriptObject obj) {
   void *ptr = const_cast<void *>(obj.GetPointer());
+  Locker py_lock(this, Locker::AcquireLock | Locker::NoSTDIN, Locker::FreeLock);
   PythonObject py_obj(PyRefType::Borrowed, static_cast<PyObject *>(ptr));
   if (!py_obj.IsValid() || py_obj.IsNone())
     return {};
-  Locker py_lock(this, Locker::AcquireLock | Locker::NoSTDIN, Locker::FreeLock);
   return py_obj.CreateStructuredObject();
 }
 

From e5f326044fdca0ba8a4ce42cafd0927776bcabff Mon Sep 17 00:00:00 2001
From: Daniel Bertalan <dani@danielbertalan.dev>
Date: Fri, 18 Apr 2025 22:09:40 +0200
Subject: [PATCH 478/710] [gn] Port 860e88411d94

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index aba30f9ce063f..f1d5c86240b4a 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -65,6 +65,13 @@ if (current_toolchain == default_toolchain) {
     values = []
   }
 
+  write_cmake_config("write_modulemap") {
+    input = "module.modulemap.in"
+    output = "$libcxx_generated_include_dir/module.modulemap"
+    values =
+        [ "LIBCXX_CONFIG_SITE_MODULE_ENTRY=textual header \"__config_site\"" ]
+  }
+
   copy("copy_headers") {
     sources = [
       "__algorithm/adjacent_find.h",
@@ -1677,7 +1684,6 @@ if (current_toolchain == default_toolchain) {
       "mdspan",
       "memory",
       "memory_resource",
-      "module.modulemap",
       "mutex",
       "new",
       "numbers",
@@ -1732,6 +1738,7 @@ if (current_toolchain == default_toolchain) {
     deps = [
       ":write_assertion_handler",
       ":write_config_site",
+      ":write_modulemap",
     ]
     if (target_os != "mac" && target_os != "win") {
       # libcxx/cmake/Modules/HandleLibCXXABI.cmake sets

From 30747cfe41f5d4f0b0083750ba9c20cfcccec117 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Fri, 18 Apr 2025 17:42:29 -0300
Subject: [PATCH 479/710] [ci] add all projects as dependencies of ci (#136153)

Add all projects as dependencies of .ci, to make sure everything is
tested when it changes.

Originally split-off from #135499
---
 .ci/compute_projects.py      | 10 ++++++----
 .ci/compute_projects_test.py | 17 +++++++++++++++++
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py
index ff43547c9bbe5..17a2136a270d5 100644
--- a/.ci/compute_projects.py
+++ b/.ci/compute_projects.py
@@ -52,6 +52,9 @@
     "clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
     "clang-tools-extra": {"libc"},
     "mlir": {"flang"},
+    # Test everything if ci scripts are changed.
+    # FIXME: Figure out what is missing and add here.
+    ".ci": {"llvm", "clang", "lld", "lldb"},
 }
 
 DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
@@ -130,12 +133,11 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
 def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     projects_to_test = set()
     for modified_project in modified_projects:
-        # Skip all projects where we cannot run tests.
-        if modified_project not in PROJECT_CHECK_TARGETS:
-            continue
         if modified_project in RUNTIMES:
             continue
-        projects_to_test.add(modified_project)
+        # Skip all projects where we cannot run tests.
+        if modified_project in PROJECT_CHECK_TARGETS:
+            projects_to_test.add(modified_project)
         if modified_project not in DEPENDENTS_TO_TEST:
             continue
         for dependent_project in DEPENDENTS_TO_TEST[modified_project]:
diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py
index e787fd8133c86..f71b56d2525ba 100644
--- a/.ci/compute_projects_test.py
+++ b/.ci/compute_projects_test.py
@@ -188,6 +188,23 @@ def test_exclude_gn(self):
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
 
+    def test_ci(self):
+        env_variables = compute_projects.get_env_variables(
+            [".ci/compute_projects.py"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "clang;lld;llvm;lldb")
+        self.assertEqual(
+            env_variables["project_check_targets"],
+            "check-clang check-lld check-llvm check-lldb",
+        )
+        self.assertEqual(
+            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets"],
+            "check-cxx check-cxxabi check-unwind",
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

From e232d28eff876b1babbb731f1a0e5502c611a0ee Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 18 Apr 2025 21:52:04 +0100
Subject: [PATCH 480/710] [VPlan] Move plain CFG construction to
 VPlanConstruction. (NFC)

Follow-up as discussed in https://github.com/llvm/llvm-project/pull/129402.

After bc03d6cce257, the VPlanHCFGBuilder doesn't actually build a HCFG
any longer. Move what remains directly into VPlanConstruction.cpp.
---
 llvm/lib/Transforms/Vectorize/CMakeLists.txt  |   1 -
 .../Transforms/Vectorize/LoopVectorize.cpp    |  22 +-
 .../Vectorize/VPlanConstruction.cpp           | 345 ++++++++++++++++
 .../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 368 ------------------
 .../Transforms/Vectorize/VPlanHCFGBuilder.h   |  73 ----
 .../Transforms/Vectorize/VPlanTransforms.h    |   4 +
 .../LoopVectorize/vplan_hcfg_stress_test.ll   |   2 +-
 .../Transforms/Vectorize/VPlanSlpTest.cpp     |   1 -
 .../Transforms/Vectorize/VPlanTestBase.h      |   6 +-
 9 files changed, 358 insertions(+), 464 deletions(-)
 delete mode 100644 llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
 delete mode 100644 llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h

diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 7dac6d0059b26..0dc6a7d2f594f 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -24,7 +24,6 @@ add_llvm_component_library(LLVMVectorize
   VPlan.cpp
   VPlanAnalysis.cpp
   VPlanConstruction.cpp
-  VPlanHCFGBuilder.cpp
   VPlanRecipes.cpp
   VPlanSLP.cpp
   VPlanTransforms.cpp
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bfceee46a237c..7a5f618d09e95 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -59,7 +59,6 @@
 #include "VPlan.h"
 #include "VPlanAnalysis.h"
 #include "VPlanCFG.h"
-#include "VPlanHCFGBuilder.h"
 #include "VPlanHelpers.h"
 #include "VPlanPatternMatch.h"
 #include "VPlanTransforms.h"
@@ -9557,13 +9556,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
             return !CM.requiresScalarEpilogue(VF.isVector());
           },
           Range);
-  auto Plan = std::make_unique<VPlan>(OrigLoop);
-  // Build hierarchical CFG.
-  // TODO: Convert to VPlan-transform and consolidate all transforms for VPlan
-  // creation.
-  VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
-  HCFGBuilder.buildPlainCFG();
-
+  DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
+  auto Plan = VPlanTransforms::buildPlainCFG(OrigLoop, *LI, VPB2IRBB);
   VPlanTransforms::createLoopRegions(*Plan, Legal->getWidestInductionType(),
                                      PSE, RequiresScalarEpilogueCheck,
                                      CM.foldTailByMasking(), OrigLoop);
@@ -9642,7 +9636,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
     // Handle VPBBs down to the latch.
     if (VPBB == LoopRegion->getExiting()) {
-      assert(!HCFGBuilder.getIRBBForVPB(VPBB) &&
+      assert(!VPB2IRBB.contains(VPBB) &&
              "the latch block shouldn't have a corresponding IRBB");
       VPBlockUtils::connectBlocks(PrevVPBB, VPBB);
       break;
@@ -9658,7 +9652,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
       // FIXME: At the moment, masks need to be placed at the beginning of the
       // block, as blends introduced for phi nodes need to use it. The created
       // blends should be sunk after the mask recipes.
-      RecipeBuilder.createBlockInMask(HCFGBuilder.getIRBBForVPB(VPBB));
+      RecipeBuilder.createBlockInMask(VPB2IRBB.lookup(VPBB));
     }
 
     // Convert input VPInstructions to widened recipes.
@@ -9862,12 +9856,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
   assert(!OrigLoop->isInnermost());
   assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
 
-  // Create new empty VPlan
-  auto Plan = std::make_unique<VPlan>(OrigLoop);
-  // Build hierarchical CFG
-  VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
-  HCFGBuilder.buildPlainCFG();
-
+  DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
+  auto Plan = VPlanTransforms::buildPlainCFG(OrigLoop, *LI, VPB2IRBB);
   VPlanTransforms::createLoopRegions(*Plan, Legal->getWidestInductionType(),
                                      PSE, true, false, OrigLoop);
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1e687d0879f18..9fcccfcf8117f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -17,10 +17,355 @@
 #include "VPlanDominatorTree.h"
 #include "VPlanTransforms.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopIterator.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 
+#define DEBUG_TYPE "vplan"
+
 using namespace llvm;
 
+namespace {
+// Class that is used to build the plain CFG for the incoming IR.
+class PlainCFGBuilder {
+  // The outermost loop of the input loop nest considered for vectorization.
+  Loop *TheLoop;
+
+  // Loop Info analysis.
+  LoopInfo *LI;
+
+  // Vectorization plan that we are working on.
+  std::unique_ptr<VPlan> Plan;
+
+  // Builder of the VPlan instruction-level representation.
+  VPBuilder VPIRBuilder;
+
+  // NOTE: The following maps are intentionally destroyed after the plain CFG
+  // construction because subsequent VPlan-to-VPlan transformation may
+  // invalidate them.
+  // Map incoming BasicBlocks to their newly-created VPBasicBlocks.
+  DenseMap<BasicBlock *, VPBasicBlock *> BB2VPBB;
+  // Map incoming Value definitions to their newly-created VPValues.
+  DenseMap<Value *, VPValue *> IRDef2VPValue;
+
+  // Hold phi node's that need to be fixed once the plain CFG has been built.
+  SmallVector<PHINode *, 8> PhisToFix;
+
+  // Utility functions.
+  void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
+  void fixHeaderPhis();
+  VPBasicBlock *getOrCreateVPBB(BasicBlock *BB);
+#ifndef NDEBUG
+  bool isExternalDef(Value *Val);
+#endif
+  VPValue *getOrCreateVPOperand(Value *IRVal);
+  void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
+
+public:
+  PlainCFGBuilder(Loop *Lp, LoopInfo *LI)
+      : TheLoop(Lp), LI(LI), Plan(std::make_unique<VPlan>(Lp)) {}
+
+  /// Build plain CFG for TheLoop  and connects it to Plan's entry.
+  std::unique_ptr<VPlan>
+  buildPlainCFG(DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB);
+};
+} // anonymous namespace
+
+// Set predecessors of \p VPBB in the same order as they are in \p BB. \p VPBB
+// must have no predecessors.
+void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) {
+  // Collect VPBB predecessors.
+  SmallVector<VPBlockBase *, 2> VPBBPreds;
+  for (BasicBlock *Pred : predecessors(BB))
+    VPBBPreds.push_back(getOrCreateVPBB(Pred));
+  VPBB->setPredecessors(VPBBPreds);
+}
+
+static bool isHeaderBB(BasicBlock *BB, Loop *L) {
+  return L && BB == L->getHeader();
+}
+
+// Add operands to VPInstructions representing phi nodes from the input IR.
+void PlainCFGBuilder::fixHeaderPhis() {
+  for (auto *Phi : PhisToFix) {
+    assert(IRDef2VPValue.count(Phi) && "Missing VPInstruction for PHINode.");
+    VPValue *VPVal = IRDef2VPValue[Phi];
+    assert(isa<VPWidenPHIRecipe>(VPVal) &&
+           "Expected WidenPHIRecipe for phi node.");
+    auto *VPPhi = cast<VPWidenPHIRecipe>(VPVal);
+    assert(VPPhi->getNumOperands() == 0 &&
+           "Expected VPInstruction with no operands.");
+    assert(isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent())) &&
+           "Expected Phi in header block.");
+    assert(Phi->getNumOperands() == 2 &&
+           "header phi must have exactly 2 operands");
+    for (BasicBlock *Pred : predecessors(Phi->getParent()))
+      VPPhi->addOperand(
+          getOrCreateVPOperand(Phi->getIncomingValueForBlock(Pred)));
+  }
+}
+
+// Create a new empty VPBasicBlock for an incoming BasicBlock or retrieve an
+// existing one if it was already created.
+VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
+  if (auto *VPBB = BB2VPBB.lookup(BB)) {
+    // Retrieve existing VPBB.
+    return VPBB;
+  }
+
+  // Create new VPBB.
+  StringRef Name = BB->getName();
+  LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << Name << "\n");
+  VPBasicBlock *VPBB = Plan->createVPBasicBlock(Name);
+  BB2VPBB[BB] = VPBB;
+  return VPBB;
+}
+
+#ifndef NDEBUG
+// Return true if \p Val is considered an external definition. An external
+// definition is either:
+// 1. A Value that is not an Instruction. This will be refined in the future.
+// 2. An Instruction that is outside of the CFG snippet represented in VPlan,
+// i.e., is not part of: a) the loop nest, b) outermost loop PH and, c)
+// outermost loop exits.
+bool PlainCFGBuilder::isExternalDef(Value *Val) {
+  // All the Values that are not Instructions are considered external
+  // definitions for now.
+  Instruction *Inst = dyn_cast<Instruction>(Val);
+  if (!Inst)
+    return true;
+
+  BasicBlock *InstParent = Inst->getParent();
+  assert(InstParent && "Expected instruction parent.");
+
+  // Check whether Instruction definition is in loop PH.
+  BasicBlock *PH = TheLoop->getLoopPreheader();
+  assert(PH && "Expected loop pre-header.");
+
+  if (InstParent == PH)
+    // Instruction definition is in outermost loop PH.
+    return false;
+
+  // Check whether Instruction definition is in a loop exit.
+  SmallVector<BasicBlock *> ExitBlocks;
+  TheLoop->getExitBlocks(ExitBlocks);
+  if (is_contained(ExitBlocks, InstParent)) {
+    // Instruction definition is in outermost loop exit.
+    return false;
+  }
+
+  // Check whether Instruction definition is in loop body.
+  return !TheLoop->contains(Inst);
+}
+#endif
+
+// Create a new VPValue or retrieve an existing one for the Instruction's
+// operand \p IRVal. This function must only be used to create/retrieve VPValues
+// for *Instruction's operands* and not to create regular VPInstruction's. For
+// the latter, please, look at 'createVPInstructionsForVPBB'.
+VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
+  auto VPValIt = IRDef2VPValue.find(IRVal);
+  if (VPValIt != IRDef2VPValue.end())
+    // Operand has an associated VPInstruction or VPValue that was previously
+    // created.
+    return VPValIt->second;
+
+  // Operand doesn't have a previously created VPInstruction/VPValue. This
+  // means that operand is:
+  //   A) a definition external to VPlan,
+  //   B) any other Value without specific representation in VPlan.
+  // For now, we use VPValue to represent A and B and classify both as external
+  // definitions. We may introduce specific VPValue subclasses for them in the
+  // future.
+  assert(isExternalDef(IRVal) && "Expected external definition as operand.");
+
+  // A and B: Create VPValue and add it to the pool of external definitions and
+  // to the Value->VPValue map.
+  VPValue *NewVPVal = Plan->getOrAddLiveIn(IRVal);
+  IRDef2VPValue[IRVal] = NewVPVal;
+  return NewVPVal;
+}
+
+// Create new VPInstructions in a VPBasicBlock, given its BasicBlock
+// counterpart. This function must be invoked in RPO so that the operands of a
+// VPInstruction in \p BB have been visited before (except for Phi nodes).
+void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
+                                                  BasicBlock *BB) {
+  VPIRBuilder.setInsertPoint(VPBB);
+  // TODO: Model and preserve debug intrinsics in VPlan.
+  for (Instruction &InstRef : BB->instructionsWithoutDebug(false)) {
+    Instruction *Inst = &InstRef;
+
+    // There shouldn't be any VPValue for Inst at this point. Otherwise, we
+    // visited Inst when we shouldn't, breaking the RPO traversal order.
+    assert(!IRDef2VPValue.count(Inst) &&
+           "Instruction shouldn't have been visited.");
+
+    if (auto *Br = dyn_cast<BranchInst>(Inst)) {
+      if (TheLoop->getLoopLatch() == BB ||
+          any_of(successors(BB),
+                 [this](BasicBlock *Succ) { return !TheLoop->contains(Succ); }))
+        continue;
+
+      // Conditional branch instruction are represented using BranchOnCond
+      // recipes.
+      if (Br->isConditional()) {
+        VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
+        VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst);
+      }
+
+      // Skip the rest of the Instruction processing for Branch instructions.
+      continue;
+    }
+
+    if (auto *SI = dyn_cast<SwitchInst>(Inst)) {
+      SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())};
+      for (auto Case : SI->cases())
+        Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
+      VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst);
+      continue;
+    }
+
+    VPSingleDefRecipe *NewR;
+    if (auto *Phi = dyn_cast<PHINode>(Inst)) {
+      // Phi node's operands may have not been visited at this point. We create
+      // an empty VPInstruction that we will fix once the whole plain CFG has
+      // been built.
+      NewR = new VPWidenPHIRecipe(Phi, nullptr, Phi->getDebugLoc(), "vec.phi");
+      VPBB->appendRecipe(NewR);
+      if (isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent()))) {
+        // Header phis need to be fixed after the VPBB for the latch has been
+        // created.
+        PhisToFix.push_back(Phi);
+      } else {
+        // Add operands for VPPhi in the order matching its predecessors in
+        // VPlan.
+        DenseMap<const VPBasicBlock *, VPValue *> VPPredToIncomingValue;
+        for (unsigned I = 0; I != Phi->getNumOperands(); ++I) {
+          VPPredToIncomingValue[BB2VPBB[Phi->getIncomingBlock(I)]] =
+              getOrCreateVPOperand(Phi->getIncomingValue(I));
+        }
+        for (VPBlockBase *Pred : VPBB->getPredecessors())
+          NewR->addOperand(
+              VPPredToIncomingValue.lookup(Pred->getExitingBasicBlock()));
+      }
+    } else {
+      // Translate LLVM-IR operands into VPValue operands and set them in the
+      // new VPInstruction.
+      SmallVector<VPValue *, 4> VPOperands;
+      for (Value *Op : Inst->operands())
+        VPOperands.push_back(getOrCreateVPOperand(Op));
+
+      // Build VPInstruction for any arbitrary Instruction without specific
+      // representation in VPlan.
+      NewR = cast<VPInstruction>(
+          VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
+    }
+
+    IRDef2VPValue[Inst] = NewR;
+  }
+}
+
+// Main interface to build the plain CFG.
+std::unique_ptr<VPlan> PlainCFGBuilder::buildPlainCFG(
+    DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB) {
+  VPIRBasicBlock *Entry = cast<VPIRBasicBlock>(Plan->getEntry());
+  BB2VPBB[Entry->getIRBasicBlock()] = Entry;
+
+  // 1. Scan the body of the loop in a topological order to visit each basic
+  // block after having visited its predecessor basic blocks. Create a VPBB for
+  // each BB and link it to its successor and predecessor VPBBs. Note that
+  // predecessors must be set in the same order as they are in the incomming IR.
+  // Otherwise, there might be problems with existing phi nodes and algorithm
+  // based on predecessors traversal.
+
+  // Loop PH needs to be explicitly visited since it's not taken into account by
+  // LoopBlocksDFS.
+  BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
+  assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
+         "Unexpected loop preheader");
+  for (auto &I : *ThePreheaderBB) {
+    if (I.getType()->isVoidTy())
+      continue;
+    IRDef2VPValue[&I] = Plan->getOrAddLiveIn(&I);
+  }
+
+  LoopBlocksRPO RPO(TheLoop);
+  RPO.perform(LI);
+
+  for (BasicBlock *BB : RPO) {
+    // Create or retrieve the VPBasicBlock for this BB.
+    VPBasicBlock *VPBB = getOrCreateVPBB(BB);
+    Loop *LoopForBB = LI->getLoopFor(BB);
+    // Set VPBB predecessors in the same order as they are in the incoming BB.
+    setVPBBPredsFromBB(VPBB, BB);
+
+    // Create VPInstructions for BB.
+    createVPInstructionsForVPBB(VPBB, BB);
+
+    // Set VPBB successors. We create empty VPBBs for successors if they don't
+    // exist already. Recipes will be created when the successor is visited
+    // during the RPO traversal.
+    if (auto *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
+      SmallVector<VPBlockBase *> Succs = {
+          getOrCreateVPBB(SI->getDefaultDest())};
+      for (auto Case : SI->cases())
+        Succs.push_back(getOrCreateVPBB(Case.getCaseSuccessor()));
+      VPBB->setSuccessors(Succs);
+      continue;
+    }
+    auto *BI = cast<BranchInst>(BB->getTerminator());
+    unsigned NumSuccs = succ_size(BB);
+    if (NumSuccs == 1) {
+      VPBB->setOneSuccessor(getOrCreateVPBB(BB->getSingleSuccessor()));
+      continue;
+    }
+    assert(BI->isConditional() && NumSuccs == 2 && BI->isConditional() &&
+           "block must have conditional branch with 2 successors");
+
+    BasicBlock *IRSucc0 = BI->getSuccessor(0);
+    BasicBlock *IRSucc1 = BI->getSuccessor(1);
+    VPBasicBlock *Successor0 = getOrCreateVPBB(IRSucc0);
+    VPBasicBlock *Successor1 = getOrCreateVPBB(IRSucc1);
+
+    // Don't connect any blocks outside the current loop except the latches for
+    // inner loops.
+    // TODO: Also connect exit blocks during initial VPlan construction.
+    if (LoopForBB == TheLoop || BB != LoopForBB->getLoopLatch()) {
+      if (!LoopForBB->contains(IRSucc0)) {
+        VPBB->setOneSuccessor(Successor1);
+        continue;
+      }
+      if (!LoopForBB->contains(IRSucc1)) {
+        VPBB->setOneSuccessor(Successor0);
+        continue;
+      }
+    }
+
+    VPBB->setTwoSuccessors(Successor0, Successor1);
+  }
+
+  // 2. The whole CFG has been built at this point so all the input Values must
+  // have a VPlan counterpart. Fix VPlan header phi by adding their
+  // corresponding VPlan operands.
+  fixHeaderPhis();
+
+  Plan->getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader()));
+  Plan->getEntry()->setPlan(&*Plan);
+
+  for (const auto &[IRBB, VPB] : BB2VPBB)
+    VPB2IRBB[VPB] = IRBB;
+
+  LLVM_DEBUG(Plan->setName("Plain CFG\n"); dbgs() << *Plan);
+  return std::move(Plan);
+}
+
+std::unique_ptr<VPlan> VPlanTransforms::buildPlainCFG(
+    Loop *TheLoop, LoopInfo &LI,
+    DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB) {
+  PlainCFGBuilder Builder(TheLoop, &LI);
+  return Builder.buildPlainCFG(VPB2IRBB);
+}
+
 /// Checks if \p HeaderVPB is a loop header block in the plain CFG; that is, it
 /// has exactly 2 predecessors (preheader and latch), where the block
 /// dominates the latch and the preheader dominates the block. If it is a
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
deleted file mode 100644
index 5bacd2d4e6d88..0000000000000
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ /dev/null
@@ -1,368 +0,0 @@
-//===-- VPlanHCFGBuilder.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This file implements the construction of a VPlan-based Hierarchical CFG
-/// (H-CFG) for an incoming IR. This construction comprises the following
-/// components and steps:
-//
-/// 1. PlainCFGBuilder class: builds a plain VPBasicBlock-based CFG that
-/// faithfully represents the CFG in the incoming IR.
-/// NOTE: At this point, there is a direct correspondence between all the
-/// VPBasicBlocks created for the initial plain CFG and the incoming
-/// BasicBlocks. However, this might change in the future.
-///
-//===----------------------------------------------------------------------===//
-
-#include "VPlanHCFGBuilder.h"
-#include "LoopVectorizationPlanner.h"
-#include "VPlanCFG.h"
-#include "llvm/Analysis/LoopIterator.h"
-
-#define DEBUG_TYPE "loop-vectorize"
-
-using namespace llvm;
-
-namespace {
-// Class that is used to build the plain CFG for the incoming IR.
-class PlainCFGBuilder {
-private:
-  // The outermost loop of the input loop nest considered for vectorization.
-  Loop *TheLoop;
-
-  // Loop Info analysis.
-  LoopInfo *LI;
-
-  // Vectorization plan that we are working on.
-  VPlan &Plan;
-
-  // Builder of the VPlan instruction-level representation.
-  VPBuilder VPIRBuilder;
-
-  // NOTE: The following maps are intentionally destroyed after the plain CFG
-  // construction because subsequent VPlan-to-VPlan transformation may
-  // invalidate them.
-  // Map incoming BasicBlocks to their newly-created VPBasicBlocks.
-  DenseMap<BasicBlock *, VPBasicBlock *> BB2VPBB;
-  // Map incoming Value definitions to their newly-created VPValues.
-  DenseMap<Value *, VPValue *> IRDef2VPValue;
-
-  // Hold phi node's that need to be fixed once the plain CFG has been built.
-  SmallVector<PHINode *, 8> PhisToFix;
-
-  // Utility functions.
-  void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
-  void fixHeaderPhis();
-  VPBasicBlock *getOrCreateVPBB(BasicBlock *BB);
-#ifndef NDEBUG
-  bool isExternalDef(Value *Val);
-#endif
-  VPValue *getOrCreateVPOperand(Value *IRVal);
-  void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
-
-public:
-  PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
-      : TheLoop(Lp), LI(LI), Plan(P) {}
-
-  /// Build plain CFG for TheLoop  and connects it to Plan's entry.
-  void buildPlainCFG(DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB);
-};
-} // anonymous namespace
-
-// Set predecessors of \p VPBB in the same order as they are in \p BB. \p VPBB
-// must have no predecessors.
-void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) {
-  // Collect VPBB predecessors.
-  SmallVector<VPBlockBase *, 2> VPBBPreds;
-  for (BasicBlock *Pred : predecessors(BB))
-    VPBBPreds.push_back(getOrCreateVPBB(Pred));
-  VPBB->setPredecessors(VPBBPreds);
-}
-
-static bool isHeaderBB(BasicBlock *BB, Loop *L) {
-  return L && BB == L->getHeader();
-}
-
-// Add operands to VPInstructions representing phi nodes from the input IR.
-void PlainCFGBuilder::fixHeaderPhis() {
-  for (auto *Phi : PhisToFix) {
-    assert(IRDef2VPValue.count(Phi) && "Missing VPInstruction for PHINode.");
-    VPValue *VPVal = IRDef2VPValue[Phi];
-    assert(isa<VPWidenPHIRecipe>(VPVal) &&
-           "Expected WidenPHIRecipe for phi node.");
-    auto *VPPhi = cast<VPWidenPHIRecipe>(VPVal);
-    assert(VPPhi->getNumOperands() == 0 &&
-           "Expected VPInstruction with no operands.");
-    assert(isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent())) &&
-           "Expected Phi in header block.");
-    assert(Phi->getNumOperands() == 2 &&
-           "header phi must have exactly 2 operands");
-    for (BasicBlock *Pred : predecessors(Phi->getParent()))
-      VPPhi->addOperand(
-          getOrCreateVPOperand(Phi->getIncomingValueForBlock(Pred)));
-  }
-}
-
-// Create a new empty VPBasicBlock for an incoming BasicBlock or retrieve an
-// existing one if it was already created.
-VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
-  if (auto *VPBB = BB2VPBB.lookup(BB)) {
-    // Retrieve existing VPBB.
-    return VPBB;
-  }
-
-  // Create new VPBB.
-  StringRef Name = BB->getName();
-  LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << Name << "\n");
-  VPBasicBlock *VPBB = Plan.createVPBasicBlock(Name);
-  BB2VPBB[BB] = VPBB;
-  return VPBB;
-}
-
-#ifndef NDEBUG
-// Return true if \p Val is considered an external definition. An external
-// definition is either:
-// 1. A Value that is not an Instruction. This will be refined in the future.
-// 2. An Instruction that is outside of the CFG snippet represented in VPlan,
-// i.e., is not part of: a) the loop nest, b) outermost loop PH and, c)
-// outermost loop exits.
-bool PlainCFGBuilder::isExternalDef(Value *Val) {
-  // All the Values that are not Instructions are considered external
-  // definitions for now.
-  Instruction *Inst = dyn_cast<Instruction>(Val);
-  if (!Inst)
-    return true;
-
-  BasicBlock *InstParent = Inst->getParent();
-  assert(InstParent && "Expected instruction parent.");
-
-  // Check whether Instruction definition is in loop PH.
-  BasicBlock *PH = TheLoop->getLoopPreheader();
-  assert(PH && "Expected loop pre-header.");
-
-  if (InstParent == PH)
-    // Instruction definition is in outermost loop PH.
-    return false;
-
-  // Check whether Instruction definition is in a loop exit.
-  SmallVector<BasicBlock *> ExitBlocks;
-  TheLoop->getExitBlocks(ExitBlocks);
-  if (is_contained(ExitBlocks, InstParent)) {
-    // Instruction definition is in outermost loop exit.
-    return false;
-  }
-
-  // Check whether Instruction definition is in loop body.
-  return !TheLoop->contains(Inst);
-}
-#endif
-
-// Create a new VPValue or retrieve an existing one for the Instruction's
-// operand \p IRVal. This function must only be used to create/retrieve VPValues
-// for *Instruction's operands* and not to create regular VPInstruction's. For
-// the latter, please, look at 'createVPInstructionsForVPBB'.
-VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
-  auto VPValIt = IRDef2VPValue.find(IRVal);
-  if (VPValIt != IRDef2VPValue.end())
-    // Operand has an associated VPInstruction or VPValue that was previously
-    // created.
-    return VPValIt->second;
-
-  // Operand doesn't have a previously created VPInstruction/VPValue. This
-  // means that operand is:
-  //   A) a definition external to VPlan,
-  //   B) any other Value without specific representation in VPlan.
-  // For now, we use VPValue to represent A and B and classify both as external
-  // definitions. We may introduce specific VPValue subclasses for them in the
-  // future.
-  assert(isExternalDef(IRVal) && "Expected external definition as operand.");
-
-  // A and B: Create VPValue and add it to the pool of external definitions and
-  // to the Value->VPValue map.
-  VPValue *NewVPVal = Plan.getOrAddLiveIn(IRVal);
-  IRDef2VPValue[IRVal] = NewVPVal;
-  return NewVPVal;
-}
-
-// Create new VPInstructions in a VPBasicBlock, given its BasicBlock
-// counterpart. This function must be invoked in RPO so that the operands of a
-// VPInstruction in \p BB have been visited before (except for Phi nodes).
-void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
-                                                  BasicBlock *BB) {
-  VPIRBuilder.setInsertPoint(VPBB);
-  // TODO: Model and preserve debug intrinsics in VPlan.
-  for (Instruction &InstRef : BB->instructionsWithoutDebug(false)) {
-    Instruction *Inst = &InstRef;
-
-    // There shouldn't be any VPValue for Inst at this point. Otherwise, we
-    // visited Inst when we shouldn't, breaking the RPO traversal order.
-    assert(!IRDef2VPValue.count(Inst) &&
-           "Instruction shouldn't have been visited.");
-
-    if (auto *Br = dyn_cast<BranchInst>(Inst)) {
-      if (TheLoop->getLoopLatch() == BB ||
-          any_of(successors(BB),
-                 [this](BasicBlock *Succ) { return !TheLoop->contains(Succ); }))
-        continue;
-
-      // Conditional branch instruction are represented using BranchOnCond
-      // recipes.
-      if (Br->isConditional()) {
-        VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
-        VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst);
-      }
-
-      // Skip the rest of the Instruction processing for Branch instructions.
-      continue;
-    }
-
-    if (auto *SI = dyn_cast<SwitchInst>(Inst)) {
-      SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())};
-      for (auto Case : SI->cases())
-        Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
-      VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst);
-      continue;
-    }
-
-    VPSingleDefRecipe *NewR;
-    if (auto *Phi = dyn_cast<PHINode>(Inst)) {
-      // Phi node's operands may have not been visited at this point. We create
-      // an empty VPInstruction that we will fix once the whole plain CFG has
-      // been built.
-      NewR = new VPWidenPHIRecipe(Phi, nullptr, Phi->getDebugLoc(), "vec.phi");
-      VPBB->appendRecipe(NewR);
-      if (isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent()))) {
-        // Header phis need to be fixed after the VPBB for the latch has been
-        // created.
-        PhisToFix.push_back(Phi);
-      } else {
-        // Add operands for VPPhi in the order matching its predecessors in
-        // VPlan.
-        DenseMap<const VPBasicBlock *, VPValue *> VPPredToIncomingValue;
-        for (unsigned I = 0; I != Phi->getNumOperands(); ++I) {
-          VPPredToIncomingValue[BB2VPBB[Phi->getIncomingBlock(I)]] =
-              getOrCreateVPOperand(Phi->getIncomingValue(I));
-        }
-        for (VPBlockBase *Pred : VPBB->getPredecessors())
-          NewR->addOperand(
-              VPPredToIncomingValue.lookup(Pred->getExitingBasicBlock()));
-      }
-    } else {
-      // Translate LLVM-IR operands into VPValue operands and set them in the
-      // new VPInstruction.
-      SmallVector<VPValue *, 4> VPOperands;
-      for (Value *Op : Inst->operands())
-        VPOperands.push_back(getOrCreateVPOperand(Op));
-
-      // Build VPInstruction for any arbitrary Instruction without specific
-      // representation in VPlan.
-      NewR = cast<VPInstruction>(
-          VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
-    }
-
-    IRDef2VPValue[Inst] = NewR;
-  }
-}
-
-// Main interface to build the plain CFG.
-void PlainCFGBuilder::buildPlainCFG(
-    DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB) {
-  VPIRBasicBlock *Entry = cast<VPIRBasicBlock>(Plan.getEntry());
-  BB2VPBB[Entry->getIRBasicBlock()] = Entry;
-
-  // 1. Scan the body of the loop in a topological order to visit each basic
-  // block after having visited its predecessor basic blocks. Create a VPBB for
-  // each BB and link it to its successor and predecessor VPBBs. Note that
-  // predecessors must be set in the same order as they are in the incomming IR.
-  // Otherwise, there might be problems with existing phi nodes and algorithm
-  // based on predecessors traversal.
-
-  // Loop PH needs to be explicitly visited since it's not taken into account by
-  // LoopBlocksDFS.
-  BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
-  assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
-         "Unexpected loop preheader");
-  for (auto &I : *ThePreheaderBB) {
-    if (I.getType()->isVoidTy())
-      continue;
-    IRDef2VPValue[&I] = Plan.getOrAddLiveIn(&I);
-  }
-
-  LoopBlocksRPO RPO(TheLoop);
-  RPO.perform(LI);
-
-  for (BasicBlock *BB : RPO) {
-    // Create or retrieve the VPBasicBlock for this BB.
-    VPBasicBlock *VPBB = getOrCreateVPBB(BB);
-    Loop *LoopForBB = LI->getLoopFor(BB);
-    // Set VPBB predecessors in the same order as they are in the incoming BB.
-    setVPBBPredsFromBB(VPBB, BB);
-
-    // Create VPInstructions for BB.
-    createVPInstructionsForVPBB(VPBB, BB);
-
-    // Set VPBB successors. We create empty VPBBs for successors if they don't
-    // exist already. Recipes will be created when the successor is visited
-    // during the RPO traversal.
-    if (auto *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
-      SmallVector<VPBlockBase *> Succs = {
-          getOrCreateVPBB(SI->getDefaultDest())};
-      for (auto Case : SI->cases())
-        Succs.push_back(getOrCreateVPBB(Case.getCaseSuccessor()));
-      VPBB->setSuccessors(Succs);
-      continue;
-    }
-    auto *BI = cast<BranchInst>(BB->getTerminator());
-    unsigned NumSuccs = succ_size(BB);
-    if (NumSuccs == 1) {
-      VPBB->setOneSuccessor(getOrCreateVPBB(BB->getSingleSuccessor()));
-      continue;
-    }
-    assert(BI->isConditional() && NumSuccs == 2 && BI->isConditional() &&
-           "block must have conditional branch with 2 successors");
-
-    BasicBlock *IRSucc0 = BI->getSuccessor(0);
-    BasicBlock *IRSucc1 = BI->getSuccessor(1);
-    VPBasicBlock *Successor0 = getOrCreateVPBB(IRSucc0);
-    VPBasicBlock *Successor1 = getOrCreateVPBB(IRSucc1);
-
-    // Don't connect any blocks outside the current loop except the latches for
-    // inner loops.
-    // TODO: Also connect exit blocks during initial VPlan construction.
-    if (LoopForBB == TheLoop || BB != LoopForBB->getLoopLatch()) {
-      if (!LoopForBB->contains(IRSucc0)) {
-        VPBB->setOneSuccessor(Successor1);
-        continue;
-      }
-      if (!LoopForBB->contains(IRSucc1)) {
-        VPBB->setOneSuccessor(Successor0);
-        continue;
-      }
-    }
-
-    VPBB->setTwoSuccessors(Successor0, Successor1);
-  }
-
-  // 2. The whole CFG has been built at this point so all the input Values must
-  // have a VPlan counterpart. Fix VPlan header phi by adding their
-  // corresponding VPlan operands.
-  fixHeaderPhis();
-
-  Plan.getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader()));
-  Plan.getEntry()->setPlan(&Plan);
-
-  for (const auto &[IRBB, VPB] : BB2VPBB)
-    VPB2IRBB[VPB] = IRBB;
-
-  LLVM_DEBUG(Plan.setName("Plain CFG\n"); dbgs() << Plan);
-}
-
-void VPlanHCFGBuilder::buildPlainCFG() {
-  PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan);
-  PCFGBuilder.buildPlainCFG(VPB2IRBB);
-}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
deleted file mode 100644
index f2e90d3f4d9b3..0000000000000
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
+++ /dev/null
@@ -1,73 +0,0 @@
-//===-- VPlanHCFGBuilder.h --------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This file defines the VPlanHCFGBuilder class which contains the public
-/// interface (buildHierarchicalCFG) to build a VPlan-based Hierarchical CFG
-/// (H-CFG) for an incoming IR.
-///
-/// A H-CFG in VPlan is a control-flow graph whose nodes are VPBasicBlocks
-/// and/or VPRegionBlocks (i.e., other H-CFGs). The outermost H-CFG of a VPlan
-/// consists of a VPRegionBlock, denoted Top Region, which encloses any other
-/// VPBlockBase in the H-CFG. This guarantees that any VPBlockBase in the H-CFG
-/// other than the Top Region will have a parent VPRegionBlock and allows us
-/// to easily add more nodes before/after the main vector loop (such as the
-/// reduction epilogue).
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
-#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
-
-#include "llvm/ADT/DenseMap.h"
-
-namespace llvm {
-
-class Loop;
-class LoopInfo;
-class VPlan;
-class VPlanTestIRBase;
-class VPBlockBase;
-class BasicBlock;
-
-/// Main class to build the VPlan H-CFG for an incoming IR.
-class VPlanHCFGBuilder {
-  friend VPlanTestIRBase;
-
-private:
-  // The outermost loop of the input loop nest considered for vectorization.
-  Loop *TheLoop;
-
-  // Loop Info analysis.
-  LoopInfo *LI;
-
-  // The VPlan that will contain the H-CFG we are building.
-  VPlan &Plan;
-
-  /// Map of create VP blocks to their input IR basic blocks, if they have been
-  /// created for a input IR basic block.
-  DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
-
-public:
-  VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
-      : TheLoop(Lp), LI(LI), Plan(P) {}
-
-  /// Build plain CFG for TheLoop and connects it to Plan's entry.
-  void buildPlainCFG();
-
-  /// Return the input IR BasicBlock corresponding to \p VPB. Returns nullptr if
-  /// there is no such corresponding block.
-  /// FIXME: This is a temporary workaround to drive the createBlockInMask.
-  /// Remove once mask creation is done on VPlan.
-  BasicBlock *getIRBBForVPB(const VPBlockBase *VPB) const {
-    return VPB2IRBB.lookup(VPB);
-  }
-};
-} // namespace llvm
-
-#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 65f3c837f7721..80c1159a9d5da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -53,6 +53,10 @@ struct VPlanTransforms {
       verifyVPlanIsValid(Plan);
   }
 
+  static std::unique_ptr<VPlan>
+  buildPlainCFG(Loop *TheLoop, LoopInfo &LI,
+                DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB);
+
   /// Replace loops in \p Plan's flat CFG with VPRegionBlocks, turing \p Plan's
   /// flat CFG into a hierarchical CFG. It also creates a VPValue expression for
   /// the original trip count. It will also introduce a dedicated VPBasicBlock
diff --git a/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll b/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
index 29aeb7c4e97f9..de536bf39c2ca 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=vplan -disable-output 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; Verify that the stress testing flag for the VPlan H-CFG builder works as
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index cd8bd4a3565e4..1ffd1a6a7a9b9 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -8,7 +8,6 @@
 
 #include "../lib/Transforms/Vectorize/VPlanSLP.h"
 #include "../lib/Transforms/Vectorize/VPlan.h"
-#include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
 #include "VPlanTestBase.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 486296535996b..d49483f0ebf88 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -13,7 +13,6 @@
 #define LLVM_UNITTESTS_TRANSFORMS_VECTORIZE_VPLANTESTBASE_H
 
 #include "../lib/Transforms/Vectorize/VPlan.h"
-#include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
 #include "../lib/Transforms/Vectorize/VPlanTransforms.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
@@ -71,9 +70,8 @@ class VPlanTestIRBase : public testing::Test {
 
     Loop *L = LI->getLoopFor(LoopHeader);
     PredicatedScalarEvolution PSE(*SE, *L);
-    auto Plan = std::make_unique<VPlan>(L);
-    VPlanHCFGBuilder HCFGBuilder(L, LI.get(), *Plan);
-    HCFGBuilder.buildPlainCFG();
+    DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
+    auto Plan = VPlanTransforms::buildPlainCFG(L, *LI, VPB2IRBB);
     VPlanTransforms::createLoopRegions(*Plan, IntegerType::get(*Ctx, 64), PSE,
                                        true, false, L);
     return Plan;

From 9c3ffa7940c5acd8ec48723748199fe9ed368fde Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 18 Apr 2025 20:52:28 +0000
Subject: [PATCH 481/710] [gn build] Port e232d28eff87

---
 llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index 829bb9e3de617..56dd2dcd170ea 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -31,7 +31,6 @@ static_library("Vectorize") {
     "VPlan.cpp",
     "VPlanAnalysis.cpp",
     "VPlanConstruction.cpp",
-    "VPlanHCFGBuilder.cpp",
     "VPlanRecipes.cpp",
     "VPlanSLP.cpp",
     "VPlanTransforms.cpp",

From 3ad353c91a0e759df5adb2d27c8b1ae214acab4f Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 23:16:42 +0200
Subject: [PATCH 482/710] OMPIRBuilder: Avoid getNumUses (#136350)

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 13b727d226738..439c46645c342 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -798,7 +798,7 @@ void OpenMPIRBuilder::finalize(Function *Fn) {
       ArtificialEntry.eraseFromParent();
     }
     assert(&OutlinedFn->getEntryBlock() == OI.EntryBB);
-    assert(OutlinedFn && OutlinedFn->getNumUses() == 1);
+    assert(OutlinedFn && OutlinedFn->hasNUses(1));
 
     // Run a user callback, e.g. to add attributes.
     if (OI.PostOutlineCB)
@@ -1926,7 +1926,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
                       Mergeable, Priority, EventHandle, TaskAllocaBB,
                       ToBeDeleted](Function &OutlinedFn) mutable {
     // Replace the Stale CI by appropriate RTL function call.
-    assert(OutlinedFn.getNumUses() == 1 &&
+    assert(OutlinedFn.hasOneUse() &&
            "there must be a single user for the outlined function");
     CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
 
@@ -7365,7 +7365,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
 
   OI.PostOutlineCB = [this, ToBeDeleted, Dependencies, HasNoWait,
                       DeviceID](Function &OutlinedFn) mutable {
-    assert(OutlinedFn.getNumUses() == 1 &&
+    assert(OutlinedFn.hasOneUse() &&
            "there must be a single user for the outlined function");
 
     CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
@@ -9269,7 +9269,7 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
     // The stale call instruction will be replaced with a new call instruction
     // for runtime call with the outlined function.
 
-    assert(OutlinedFn.getNumUses() == 1 &&
+    assert(OutlinedFn.hasOneUse() &&
            "there must be a single user for the outlined function");
     CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
     ToBeDeleted.push_back(StaleCI);

From 34e78093975b7c3be0fbd088429d8124702d8336 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 23:17:26 +0200
Subject: [PATCH 483/710] unittests: Avoid using getNumUses (#136352)

---
 .../Analysis/CGSCCPassManagerTest.cpp         |  2 +-
 llvm/unittests/Analysis/LazyCallGraphTest.cpp |  2 +-
 .../Frontend/OpenMPIRBuilderTest.cpp          | 60 +++++++++----------
 llvm/unittests/IR/ValueHandleTest.cpp         |  2 +-
 llvm/unittests/Linker/LinkModulesTest.cpp     |  2 +-
 5 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp b/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp
index cf649776c04fd..17240a1c73bce 100644
--- a/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp
+++ b/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp
@@ -1641,7 +1641,7 @@ TEST_F(CGSCCPassManagerTest, TestUpdateCGAndAnalysisManagerForPasses8) {
         CGU.initialize(CG, C, AM, UR);
         ASSERT_NO_FATAL_FAILURE(CGU.replaceFunctionWith(*FnF, *FnewF));
         ASSERT_TRUE(FnF->isDeclaration());
-        ASSERT_EQ(FnF->getNumUses(), 0U);
+        ASSERT_TRUE(FnF->use_empty());
       }));
 
   ModulePassManager MPM;
diff --git a/llvm/unittests/Analysis/LazyCallGraphTest.cpp b/llvm/unittests/Analysis/LazyCallGraphTest.cpp
index 49677d7301f88..4a4ff3242c4c4 100644
--- a/llvm/unittests/Analysis/LazyCallGraphTest.cpp
+++ b/llvm/unittests/Analysis/LazyCallGraphTest.cpp
@@ -1155,7 +1155,7 @@ TEST(LazyCallGraphTest, InlineAndDeleteFunction) {
   ASSERT_EQ(&D2F, D1Call->getCalledFunction());
   C1Call->setCalledFunction(&D3.getFunction());
   D1Call->setCalledFunction(&D3.getFunction());
-  ASSERT_EQ(0u, D2F.getNumUses());
+  ASSERT_TRUE(D2F.use_empty());
 
   // Insert new edges first.
   CRC.insertTrivialCallEdge(C1, D3);
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 5dd018006ec37..8300c311da604 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -325,7 +325,7 @@ static Value *findStoredValueInAggregateAt(LLVMContext &Ctx, Value *Aggregate,
   }
 
   EXPECT_NE(GEPAtIdx, nullptr);
-  EXPECT_EQ(GEPAtIdx->getNumUses(), 1U);
+  EXPECT_TRUE(GEPAtIdx->hasOneUse());
 
   // Find the value stored to the aggregate.
   StoreInst *StoreToAgg = dyn_cast<StoreInst>(*GEPAtIdx->user_begin());
@@ -443,7 +443,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancel) {
   EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
   EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
   EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
-  EXPECT_EQ(Cancel->getNumUses(), 1U);
+  EXPECT_TRUE(Cancel->hasOneUse());
   Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
   EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
   EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock());
@@ -460,7 +460,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancel) {
   EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
-  EXPECT_EQ(Barrier->getNumUses(), 0U);
+  EXPECT_TRUE(Barrier->use_empty());
   EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
             1U);
   EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
@@ -516,7 +516,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
   EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
   EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
   EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
-  EXPECT_EQ(Cancel->getNumUses(), 1U);
+  EXPECT_TRUE(Cancel->hasOneUse());
   Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
   EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
   EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U);
@@ -535,7 +535,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
   EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
-  EXPECT_EQ(Barrier->getNumUses(), 0U);
+  EXPECT_TRUE(Barrier->use_empty());
   EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
             1U);
   EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
@@ -586,7 +586,7 @@ TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {
   EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
-  EXPECT_EQ(Barrier->getNumUses(), 1U);
+  EXPECT_TRUE(Barrier->hasOneUse());
   Instruction *BarrierBBTI = Barrier->getParent()->getTerminator();
   EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U);
   EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock());
@@ -741,7 +741,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {
   EXPECT_EQ(OutlinedFn->getArg(2)->getType(),
             PointerType::get(M->getContext(), 0));
   EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
-  EXPECT_EQ(OutlinedFn->getNumUses(), 1U);
+  EXPECT_TRUE(OutlinedFn->hasOneUse());
   User *Usr = OutlinedFn->user_back();
   ASSERT_TRUE(isa<CallInst>(Usr));
   CallInst *Parallel51CI = dyn_cast<CallInst>(Usr);
@@ -849,7 +849,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
   EXPECT_EQ(OutlinedFn->arg_size(), 3U);
 
   EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
-  EXPECT_EQ(OutlinedFn->getNumUses(), 1U);
+  EXPECT_TRUE(OutlinedFn->hasOneUse());
   User *Usr = OutlinedFn->user_back();
   ASSERT_TRUE(isa<CallInst>(Usr));
   CallInst *ForkCI = dyn_cast<CallInst>(Usr);
@@ -952,7 +952,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
     EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
     EXPECT_EQ(OutlinedFn.arg_size(), 2U);
 
-    EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
+    EXPECT_TRUE(OutlinedFn.hasOneUse());
     User *Usr = OutlinedFn.user_back();
     ASSERT_TRUE(isa<CallInst>(Usr));
     CallInst *ForkCI = dyn_cast<CallInst>(Usr);
@@ -1071,7 +1071,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
       NumAllocas += isa<AllocaInst>(I);
     EXPECT_EQ(NumAllocas, 1U);
 
-    EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
+    EXPECT_TRUE(OutlinedFn.hasOneUse());
     User *Usr = OutlinedFn.user_back();
     ASSERT_TRUE(isa<CallInst>(Usr));
     CallInst *ForkCI = dyn_cast<CallInst>(Usr);
@@ -1178,7 +1178,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
   EXPECT_EQ(OutlinedFn->arg_size(), 3U);
 
   EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
-  ASSERT_EQ(OutlinedFn->getNumUses(), 1U);
+  ASSERT_TRUE(OutlinedFn->hasOneUse());
 
   CallInst *ForkCI = nullptr;
   for (User *Usr : OutlinedFn->users()) {
@@ -1229,9 +1229,9 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
     BFn = M->getFunction("__kmpc_barrier");
     ASSERT_NE(CBFn, nullptr);
     ASSERT_EQ(BFn, nullptr);
-    ASSERT_EQ(CBFn->getNumUses(), 1U);
+    ASSERT_TRUE(CBFn->hasOneUse());
     ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
-    ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U);
+    ASSERT_TRUE(CBFn->user_back()->hasOneUse());
     CheckedBarrier = cast<CallInst>(CBFn->user_back());
 
     ASSERT_EXPECTED_INIT(
@@ -1242,20 +1242,20 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
     BFn = M->getFunction("__kmpc_barrier");
     ASSERT_NE(CBFn, nullptr);
     ASSERT_NE(BFn, nullptr);
-    ASSERT_EQ(CBFn->getNumUses(), 1U);
-    ASSERT_EQ(BFn->getNumUses(), 1U);
+    ASSERT_TRUE(CBFn->hasOneUse());
+    ASSERT_TRUE(BFn->hasOneUse());
     ASSERT_TRUE(isa<CallInst>(BFn->user_back()));
-    ASSERT_EQ(BFn->user_back()->getNumUses(), 0U);
+    ASSERT_TRUE(BFn->user_back()->use_empty());
 
     ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, BarrierIP3,
                          OMPBuilder.createBarrier(Builder.saveIP(),
                                                   OMPD_parallel, false, false));
     Builder.restoreIP(BarrierIP3);
-    ASSERT_EQ(CBFn->getNumUses(), 2U);
-    ASSERT_EQ(BFn->getNumUses(), 1U);
+    ASSERT_TRUE(CBFn->hasNUses(2));
+    ASSERT_TRUE(BFn->hasOneUse());
     ASSERT_TRUE(CBFn->user_back() != CheckedBarrier);
     ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
-    ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U);
+    ASSERT_TRUE(CBFn->user_back()->use_empty());
   };
 
   auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &,
@@ -1289,7 +1289,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
   EXPECT_EQ(NumBodiesGenerated, 1U);
   EXPECT_EQ(NumPrivatizedVars, 0U);
   EXPECT_EQ(NumFinalizationPoints, 2U);
-  EXPECT_EQ(FakeDestructor->getNumUses(), 2U);
+  EXPECT_TRUE(FakeDestructor->hasNUses(2));
 
   Builder.restoreIP(AfterIP);
   Builder.CreateRetVoid();
@@ -3988,7 +3988,7 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) {
   EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
   EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
 
-  EXPECT_EQ(Sub->getNumUses(), 1U);
+  EXPECT_TRUE(Sub->hasOneUse());
   StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
   AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
 
@@ -4058,7 +4058,7 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) {
   EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
   EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
 
-  EXPECT_EQ(Sub->getNumUses(), 1U);
+  EXPECT_TRUE(Sub->hasOneUse());
   StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
   AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
 
@@ -4127,7 +4127,7 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
   EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
   EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
 
-  EXPECT_EQ(Sub->getNumUses(), 1U);
+  EXPECT_TRUE(Sub->hasOneUse());
   StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
   AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
 
@@ -4942,7 +4942,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) {
   Value *ThreadLimitArg = PushNumTeamsCallInst->getArgOperand(4);
 
   // Get the boolean conversion of if expression
-  ASSERT_EQ(IfExpr->getNumUses(), 1U);
+  ASSERT_TRUE(IfExpr->hasOneUse());
   User *IfExprInst = IfExpr->user_back();
   ICmpInst *IfExprCmpInst = dyn_cast<ICmpInst>(IfExprInst);
   ASSERT_NE(IfExprCmpInst, nullptr);
@@ -5272,7 +5272,7 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
   // Find the GEP instructions preceding stores to the local array.
   Value *FirstArrayElemPtr = nullptr;
   Value *SecondArrayElemPtr = nullptr;
-  EXPECT_EQ(LocalArray->getNumUses(), 3u);
+  EXPECT_TRUE(LocalArray->hasNUses(3));
   ASSERT_TRUE(
       findGEPZeroOne(LocalArray, FirstArrayElemPtr, SecondArrayElemPtr));
 
@@ -5292,7 +5292,7 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
 
   // Check that the result of the runtime reduction call is used for further
   // dispatch.
-  ASSERT_EQ(SwitchArg->getNumUses(), 1u);
+  ASSERT_TRUE(SwitchArg->hasOneUse());
   SwitchInst *Switch = dyn_cast<SwitchInst>(*SwitchArg->user_begin());
   ASSERT_NE(Switch, nullptr);
   EXPECT_EQ(Switch->getNumSuccessors(), 3u);
@@ -6296,7 +6296,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
   // Check num_teams and num_threads kernel arguments (use number 5 starting
   // from the end and counting the call to __tgt_target_kernel as the first use)
   Value *KernelArgs = Call->getArgOperand(Call->arg_size() - 1);
-  EXPECT_TRUE(KernelArgs->getNumUses() >= 4);
+  EXPECT_TRUE(KernelArgs->hasNUsesOrMore(4));
   Value *NumTeamsGetElemPtr = *std::next(KernelArgs->user_begin(), 3);
   EXPECT_TRUE(isa<GetElementPtrInst>(NumTeamsGetElemPtr));
   Value *NumTeamsStore = NumTeamsGetElemPtr->getUniqueUndroppableUser();
@@ -6622,7 +6622,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionSPMD) {
   // Check the trip count kernel argument (use number 5 starting from the end
   // and counting the call to __tgt_target_kernel as the first use)
   Value *KernelArgs = Call->getArgOperand(Call->arg_size() - 1);
-  EXPECT_TRUE(KernelArgs->getNumUses() >= 6);
+  EXPECT_TRUE(KernelArgs->hasNUsesOrMore(6));
   Value *TripCountGetElemPtr = *std::next(KernelArgs->user_begin(), 5);
   EXPECT_TRUE(isa<GetElementPtrInst>(TripCountGetElemPtr));
   Value *TripCountStore = TripCountGetElemPtr->getUniqueUndroppableUser();
@@ -7024,7 +7024,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTask) {
   // Verify that the data argument is used only once, and that too in the load
   // instruction that is then used for accessing shared data.
   Value *DataPtr = OutlinedFn->getArg(1);
-  EXPECT_EQ(DataPtr->getNumUses(), 1U);
+  EXPECT_TRUE(DataPtr->hasOneUse());
   EXPECT_TRUE(isa<LoadInst>(DataPtr->uses().begin()->getUser()));
   Value *Data = DataPtr->uses().begin()->getUser();
   EXPECT_TRUE(all_of(Data->uses(), [](Use &U) {
@@ -7501,7 +7501,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
 
   Function *TaskAllocFn =
       OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);
-  ASSERT_EQ(TaskAllocFn->getNumUses(), 2u);
+  ASSERT_TRUE(TaskAllocFn->hasNUses(2));
 
   CallInst *FirstTaskAllocCall =
       dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin());
diff --git a/llvm/unittests/IR/ValueHandleTest.cpp b/llvm/unittests/IR/ValueHandleTest.cpp
index e11c545520d5a..0d4d0e220b561 100644
--- a/llvm/unittests/IR/ValueHandleTest.cpp
+++ b/llvm/unittests/IR/ValueHandleTest.cpp
@@ -329,7 +329,7 @@ TEST_F(ValueHandle, CallbackVH_DeletionCanRAUW) {
     }
     void allUsesReplacedWith(Value *new_value) override {
       ASSERT_TRUE(nullptr != getValPtr());
-      EXPECT_EQ(1U, getValPtr()->getNumUses());
+      EXPECT_TRUE(getValPtr()->hasOneUse());
       EXPECT_EQ(nullptr, AURWArgument);
       AURWArgument = new_value;
     }
diff --git a/llvm/unittests/Linker/LinkModulesTest.cpp b/llvm/unittests/Linker/LinkModulesTest.cpp
index 758e859e32fbd..c98cdc2b24f79 100644
--- a/llvm/unittests/Linker/LinkModulesTest.cpp
+++ b/llvm/unittests/Linker/LinkModulesTest.cpp
@@ -358,7 +358,7 @@ TEST_F(LinkModuleTest, RemangleIntrinsics) {
   // types, so they must be uniquified by linker. Check that they use the same
   // intrinsic definition.
   Function *F = Foo->getFunction("llvm.ssa.copy.s_struct.rtx_defs");
-  ASSERT_EQ(F->getNumUses(), (unsigned)2);
+  ASSERT_TRUE(F->hasNUses(2));
 }
 
 } // end anonymous namespace

From f2c175f626df67a028e2c56c56db76980135b72f Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 23:17:33 +0200
Subject: [PATCH 484/710] OpenMPOpt: Avoid using getNumUses (#136349)

---
 llvm/lib/Transforms/IPO/OpenMPOpt.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index d055a22f10ebb..594f266cc8650 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -1535,7 +1535,7 @@ struct OpenMPOpt {
     // safely remove it.
     // TODO: This should be somewhere more common in the future.
     if (GlobalVariable *GV = M.getNamedGlobal("__llvm_rpc_client")) {
-      if (GV->getNumUses() >= 1)
+      if (GV->hasNUsesOrMore(1))
         return false;
 
       GV->replaceAllUsesWith(PoisonValue::get(GV->getType()));

From 5dcfa6146f8f44b0ba9937a37c612d54be6f95e7 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 23:19:52 +0200
Subject: [PATCH 485/710] NVPTX: Avoid getNumUses (#136351)

---
 llvm/lib/Target/NVPTX/NVVMReflect.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/NVPTX/NVVMReflect.cpp b/llvm/lib/Target/NVPTX/NVVMReflect.cpp
index 30b522efa4361..208bab52284a3 100644
--- a/llvm/lib/Target/NVPTX/NVVMReflect.cpp
+++ b/llvm/lib/Target/NVPTX/NVVMReflect.cpp
@@ -138,7 +138,7 @@ bool NVVMReflect::handleReflectFunction(Module &M, StringRef ReflectName) {
   assert(F->getReturnType()->isIntegerTy() &&
          "_reflect's return type should be integer");
 
-  const bool Changed = F->getNumUses() > 0;
+  const bool Changed = !F->use_empty();
   for (User *U : make_early_inc_range(F->users())) {
     // Reflect function calls look like:
     // @arch = private unnamed_addr addrspace(1) constant [12 x i8]

From a79e257ff15504fca877601c842b91244e3c0ab0 Mon Sep 17 00:00:00 2001
From: Kaitlin Peng <kaitlinpeng@microsoft.com>
Date: Fri, 18 Apr 2025 14:22:52 -0700
Subject: [PATCH 486/710] [HLSL] XFAIL `smoothstep.ll` and `SV_GroupIndex.ll`
 SPIR-V backend tests (#136343)

When SPIRV-Tools is enabled, tests that specify the target environment
`vulkan1.3` in the validation step fail. This is because SPIRV-Tools
seems to have [grown some capability checks for vulkan
1.3](https://github.com/KhronosGroup/SPIRV-Tools/commit/7e41c7130f8258bf2d62179d8d6dbf3b68a7978b).
The failing tests are:
- `CodeGen/SPIRV/hlsl-intrinsics/SV_GroupIndex.ll`
- `CodeGen/SPIRV/hlsl-intrinsics/smoothstep.ll`

For now, the fix is to XFAIL these tests to unblock the pipeline.

Issue #136344 tracks the long-term solution for this.
---
 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_GroupIndex.ll | 3 +++
 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smoothstep.ll    | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_GroupIndex.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_GroupIndex.ll
index 83caa62150cda..1b89485654727 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_GroupIndex.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_GroupIndex.ll
@@ -1,6 +1,9 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
 
+; TODO(#136344): This test currently fails when --target-env vulkan1.3 is specified.
+; XFAIL: spirv-tools
+
 ; CHECK-DAG:        %[[#int:]] = OpTypeInt 32 0
 ; CHECK-DAG:        %[[#ptr_Input_int:]] = OpTypePointer Input %[[#int]]
 ; CHECK-DAG:        %[[#LocalInvocationIndex:]] = OpVariable %[[#ptr_Input_int]] Input
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smoothstep.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smoothstep.ll
index e0e04dd0faf58..789b0cb15c9e2 100644
--- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smoothstep.ll
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smoothstep.ll
@@ -1,6 +1,9 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
 
+; TODO(#136344): This test currently fails when --target-env vulkan1.3 is specified.
+; XFAIL: spirv-tools
+
 ; Make sure SPIRV operation function calls for smoothstep are lowered correctly.
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"

From d36ce05237ce5e2c1f9d6cccc79e23241d661999 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 23:26:43 +0200
Subject: [PATCH 487/710] ComplexDeinterleaving: Avoid using getNumUses
 (#136354)

---
 llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index a83982de14b28..c14b50120b16e 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -1168,7 +1168,7 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
       // the latter case, we will attempt to separately identify the complex
       // operation from here in order to create a shared
       // ComplexDeinterleavingCompositeNode.
-      if (I != Insn && I->getNumUses() > 1) {
+      if (I != Insn && I->hasNUsesOrMore(2)) {
         LLVM_DEBUG(dbgs() << "Found potential sub-expression: " << *I << "\n");
         Addends.emplace_back(I, IsPositive);
         continue;

From 540ea4ddff75f7bdec41533c1e9059b6a4ec81c1 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 23:29:48 +0200
Subject: [PATCH 488/710] ARM: Avoid using getNumUses (#136355)

---
 llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index 7efd2989aa7fa..b97ad5ad5e9ee 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -808,7 +808,7 @@ Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
   // by a constant, thus we're looking for an add of a phi and a constant
   PHINode *Phi = dyn_cast<PHINode>(Offsets);
   if (Phi == nullptr || Phi->getNumIncomingValues() != 2 ||
-      Phi->getParent() != L->getHeader() || Phi->getNumUses() != 2)
+      Phi->getParent() != L->getHeader() || !Phi->hasNUses(2))
     // No phi means no IV to write back to; if there is a phi, we expect it
     // to have exactly two incoming values; the only phis we are interested in
     // will be loop IV's and have exactly two uses, one in their increment and
@@ -1051,10 +1051,10 @@ bool MVEGatherScatterLowering::optimiseOffsets(Value *Offsets, BasicBlock *BB,
   // If the phi is not used by anything else, we can just adapt it when
   // replacing the instruction; if it is, we'll have to duplicate it
   PHINode *NewPhi;
-  if (Phi->getNumUses() == 2) {
+  if (Phi->hasNUses(2)) {
     // No other users -> reuse existing phi (One user is the instruction
     // we're looking at, the other is the phi increment)
-    if (IncInstruction->getNumUses() != 1) {
+    if (!IncInstruction->hasOneUse()) {
       // If the incrementing instruction does have more users than
       // our phi, we need to copy it
       IncInstruction = BinaryOperator::Create(

From aed4ecc456a00385b8be04c3e3b4680a09c24463 Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Fri, 18 Apr 2025 14:32:20 -0700
Subject: [PATCH 489/710] [DAGCombiner] Fold and/or of NaN SETCC - tests follow
 up (#136168)

Follow up to #135645 to address test cleanup review.
---
 llvm/test/CodeGen/NVPTX/and-or-setcc.ll |   4 +-
 llvm/test/CodeGen/X86/and-or-setcc.ll   | 118 ++++++++++++++++++++++++
 2 files changed, 120 insertions(+), 2 deletions(-)

diff --git a/llvm/test/CodeGen/NVPTX/and-or-setcc.ll b/llvm/test/CodeGen/NVPTX/and-or-setcc.ll
index 21be9df94d553..6c3514c1ad946 100644
--- a/llvm/test/CodeGen/NVPTX/and-or-setcc.ll
+++ b/llvm/test/CodeGen/NVPTX/and-or-setcc.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=nvptx64 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 | %ptxas-verify %}
+; RUN: llc < %s | FileCheck %s
+; RUN: %if ptxas %{ llc < %s | %ptxas-verify %}
 
 target triple = "nvptx64-nvidia-cuda"
 
diff --git a/llvm/test/CodeGen/X86/and-or-setcc.ll b/llvm/test/CodeGen/X86/and-or-setcc.ll
index cb8ecca9348e6..a6a9362908811 100644
--- a/llvm/test/CodeGen/X86/and-or-setcc.ll
+++ b/llvm/test/CodeGen/X86/and-or-setcc.ll
@@ -55,3 +55,121 @@ define i1 @or_uno(float %a, float %b) {
   %e = or i1 %c, %d
   ret i1 %e
 }
+
+define <4 x i1> @and_ord_vec(<4 x float> %a, <4 x float> %b) {
+; X86-LABEL: and_ord_vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fstps (%esp) # 4-byte Folded Spill
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    fucompp
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    fucompp
+; X86-NEXT:    setnp %dh
+; X86-NEXT:    shlb $2, %dh
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    flds (%esp) # 4-byte Folded Reload
+; X86-NEXT:    fxch %st(1)
+; X86-NEXT:    fucompp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    setnp %dl
+; X86-NEXT:    shlb $3, %dl
+; X86-NEXT:    orb %dh, %dl
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %dh
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setnp %al
+; X86-NEXT:    addb %al, %al
+; X86-NEXT:    orb %dh, %al
+; X86-NEXT:    orb %dl, %al
+; X86-NEXT:    movb %al, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %ecx
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: and_ord_vec:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpordps %xmm1, %xmm0
+; X64-NEXT:    retq
+  %c = fcmp ord <4 x float> %a, %a
+  %d = fcmp ord <4 x float> %b, %b
+  %e = and <4 x i1> %c, %d
+  ret <4 x i1> %e
+}
+
+define <4 x i1> @or_uno_vec(<4 x float> %a, <4 x float> %b) {
+; X86-LABEL: or_uno_vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fstps (%esp) # 4-byte Folded Spill
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fucompp
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    fucompp
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    fucompp
+; X86-NEXT:    setp %dh
+; X86-NEXT:    shlb $2, %dh
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    flds (%esp) # 4-byte Folded Reload
+; X86-NEXT:    fxch %st(1)
+; X86-NEXT:    fucompp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    setp %dl
+; X86-NEXT:    shlb $3, %dl
+; X86-NEXT:    orb %dh, %dl
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %dh
+; X86-NEXT:    fnstsw %ax
+; X86-NEXT:    # kill: def $ah killed $ah killed $ax
+; X86-NEXT:    sahf
+; X86-NEXT:    setp %al
+; X86-NEXT:    addb %al, %al
+; X86-NEXT:    orb %dh, %al
+; X86-NEXT:    orb %dl, %al
+; X86-NEXT:    movb %al, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %ecx
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: or_uno_vec:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpunordps %xmm1, %xmm0
+; X64-NEXT:    retq
+  %c = fcmp uno <4 x float> %a, %a
+  %d = fcmp uno <4 x float> %b, %b
+  %e = or <4 x i1> %c, %d
+  ret <4 x i1> %e
+}

From 0baa0b38ae105d3b0ec30000960cf31bfcd057de Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 18 Apr 2025 23:35:27 +0200
Subject: [PATCH 490/710] AArch64: Use use_empty instead of getNumUses == 0
 (#136356)

---
 llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
index 9cb790b99f1de..da9d8d0f8e290 100644
--- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
@@ -328,9 +328,9 @@ bool SVEIntrinsicOpts::optimizePredicateStore(Instruction *I) {
   Builder.CreateStore(BitCast->getOperand(0), Store->getPointerOperand());
 
   Store->eraseFromParent();
-  if (IntrI->getNumUses() == 0)
+  if (IntrI->use_empty())
     IntrI->eraseFromParent();
-  if (BitCast->getNumUses() == 0)
+  if (BitCast->use_empty())
     BitCast->eraseFromParent();
 
   return true;
@@ -386,9 +386,9 @@ bool SVEIntrinsicOpts::optimizePredicateLoad(Instruction *I) {
 
   BitCast->replaceAllUsesWith(LoadPred);
   BitCast->eraseFromParent();
-  if (IntrI->getNumUses() == 0)
+  if (IntrI->use_empty())
     IntrI->eraseFromParent();
-  if (Load->getNumUses() == 0)
+  if (Load->use_empty())
     Load->eraseFromParent();
 
   return true;

From d1cce66469d0739f332b11fdb0fa2a441b1e0483 Mon Sep 17 00:00:00 2001
From: Erich Keane <ekeane@nvidia.com>
Date: Fri, 18 Apr 2025 14:54:21 -0700
Subject: [PATCH 491/710] =?UTF-8?q?[OpenACC]=20Switch=20Clang=20to=20use?=
 =?UTF-8?q?=20the=20Flang=20'appertainment'=20rules=20for=20cla=E2=80=A6?=
 =?UTF-8?q?=20(#135372)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…uses

The Flang implemenation of OpenACC uses a .td file in the llvm/Frontend
directory to determine appertainment in 4 categories:

-Required: If this list has items in it, the directive requires at least
1 of these be present.

-AllowedExclusive: Items on this list are all allowed, but only 1 from
the list may be here (That is, they are exclusive of eachother).

-AllowedOnce: Items on this list are all allowed, but may not be
duplicated.

Allowed: Items on this list are allowed. Note th at the actual list of
'allowed' is all 4 of these lists together.

This is a draft patch to swtich Clang over to use those tables. Surgery
to get this to happen in Clang Sema was somewhat reasonable. However,
some gaps in the implementations are obvious, the existing clang
implementation disagrees with the Flang interpretation of it. SO, we're
keeping a task list here based on what gets discovered.

Changes to Clang:
- [x] Switch 'directive-kind' enum conversions to use tablegen See
ff1a7bddd9435b6ae2890c07eae60bb07898bbf5
- [x] Switch 'clause-kind' enum conversions to use tablegen See
ff1a7bddd9435b6ae2890c07eae60bb07898bbf5
- [x] Investigate 'parse' test differences to see if any new
disagreements arise.
- [x] Clang/Flang disagree as to whether 'collapse' can be multiple
times on a loop. Further research showed no prose to limit this, and the
comment on the clang implementation said "no good reason to allow", so
no standards justification.
- [x] Clang/Flang disagree whether 'num_gangs' can appear >1 on a
compute/combined construct. This ended up being an unjustified
restriction.
- [x] Clang/Flang disagree as to the list of required clauses on a 'set'
construct. My research shows that Clang mistakenly included 'if' in the
list, and that it should be just 'default_async', 'device_num', and
'device_type'.
- [x] Order of 'at least one of' diagnostic has changed. Tests were
updated.
- [x] Ensure we are properly 'de-aliasing' clause names in appertainment
checks?
- [x] What is 'shortloop'? 'shortloop' seems to be an old non-standard
extension that isn't supported by flang, but is parsed for backward
compat reasons. Clang won't parse, but we at least have a spot for it in
the clause list.
- [x] Implemented proposed change for 'routine' gang/worker/vector/seq.
(see issue 539)
- [x] Implement init/shutdown can only have 1 'if' (see issue 540)
- [x] Clang/Flang disagree as to whether 'tile' is permitted more than
once on a 'loop' or combined constructs (Flang prohibits >1). I see no
justification for this in the standard. EDIT: I found a comment in clang
that I did this to make SOMETHING around duplicate checks easier.
Discussion showed we should actually have a better behavior around
'device_type' and duplicates, so I've since implemented that.
- [x] Clang/Flang disagree whether 'gang', 'worker', or 'vector' may
appear on the same construct as a 'seq' on a 'loop' or 'combined'. There
is prose for this in 2022: (a gang, worker, or vector clause may not
appear if a 'seq' clause appears). EDIT: These don't actually disagree,
but aren't in the .td file, so I restored the existing code to do this.
- [x] Clang/Flang disagree on whether 'bind' can appear >1 on a
'routine'. I believe line 3096 (A bind clause may not bind to a routine
name that has a visible bind clause) makes this limitation (Flang
permits >1 bind). we discussed and decided this should have the same
rules as worker/vector/etc, except without the 'exactly 1 of' rule (so
no dupes in individual sections).
- [x] Clang/Flang disagree on whether 'init'/'shutdown' can have
multiple 'device_num' clauses. I believe there is no supporting prose
for this limitation., We decided that `device_num` should only happen
1x.
- [x] Clang/Flang disagree whether 'num_gangs' can appear >1 on a
'kernels' construct. Line 1173 (On a kernels construct, the num_gangs
clause must have a single argument) justifies limiting on a
per-arguement basis, but doesn't do so for multiple num_gangs clauses.
WE decided to do this with the '1-per-device-type' region for num_gangs,
num_workers, and vector_length, see openacc bug here:
https://github.com/OpenACC/openacc-spec/issues/541

Changes to Flang:
- [x] Clang/Flang disgree on whether 'atomic' can take an 'if' clause.
This was added in OpenACC3.3_Next See #135451
- [x] Clang/Flang disagree on whether 'finalize' can be allowed >1 times
on a 'exit_data' construct. see #135415.
- [x] Clang/Flang disagree whether 'if_present' should be allowed >1
times on a 'host_data'/'update' construct. see #135422
- [x] Clang/Flang disagree on whether 'init'/'shutdown' can have
multiple 'device_type' clauses. I believe there is no supporting prose
for this limitation.
- [ ] SEE change for num_gangs/etc above.


Changes that need discussion/research:
---
 .../clang/Basic/DiagnosticSemaKinds.td        |  19 +-
 clang/include/clang/Basic/OpenACCKinds.h      | 170 ++--
 clang/include/clang/Sema/SemaOpenACC.h        |  19 +-
 clang/lib/AST/OpenACCClause.cpp               |  11 +-
 clang/lib/AST/TextNodeDumper.cpp              |   1 +
 clang/lib/Parse/ParseOpenACC.cpp              |   2 +
 clang/lib/Sema/CMakeLists.txt                 |   1 +
 clang/lib/Sema/SemaOpenACC.cpp                | 301 ++++---
 clang/lib/Sema/SemaOpenACCClause.cpp          | 851 ++++--------------
 .../Sema/SemaOpenACCClauseAppertainment.cpp   | 229 +++++
 clang/lib/Sema/TreeTransform.h                |   3 -
 clang/lib/Serialization/ASTReader.cpp         |   1 +
 clang/lib/Serialization/ASTWriter.cpp         |   1 +
 .../AST/ast-print-openacc-set-construct.cpp   |   4 +-
 clang/test/ParserOpenACC/parse-clauses.c      |  42 +-
 clang/test/ParserOpenACC/parse-constructs.c   |  16 +-
 ...d-construct-auto_seq_independent-clauses.c |  12 +-
 .../combined-construct-collapse-clause.cpp    |   4 +-
 .../combined-construct-if-clause.c            |   2 +-
 .../combined-construct-num_gangs-clause.c     |  35 +-
 .../combined-construct-num_workers-clause.c   |  25 +
 .../combined-construct-tile-clause.cpp        |   7 -
 .../combined-construct-vector_length-clause.c |  25 +
 .../compute-construct-device_type-clause.c    |   4 +-
 .../SemaOpenACC/compute-construct-if-clause.c |   2 +-
 .../compute-construct-num_gangs-clause.c      |  15 +
 .../compute-construct-num_workers-clause.c    |  25 +
 .../compute-construct-vector_length-clause.c  |  25 +
 .../SemaOpenACC/data-construct-copy-clause.c  |   4 +-
 .../data-construct-copyin-clause.c            |   2 +-
 .../data-construct-copyout-clause.c           |   2 +-
 .../data-construct-create-clause.c            |   2 +-
 .../data-construct-default-clause.c           |   6 +-
 .../data-construct-delete-clause.c            |   4 +-
 .../data-construct-device_type-clause.c       |   2 +-
 .../data-construct-no_create-clause.c         |   4 +-
 .../data-construct-use_device-clause.c        |   6 +-
 clang/test/SemaOpenACC/data-construct.cpp     |  28 +-
 clang/test/SemaOpenACC/init-construct.cpp     |   8 +-
 ...p-construct-auto_seq_independent-clauses.c |  22 +-
 .../loop-construct-collapse-clause.cpp        |  46 +-
 .../loop-construct-tile-clause.cpp            |  26 +
 clang/test/SemaOpenACC/no-empty-pqr-list.cpp  |   2 +-
 .../SemaOpenACC/routine-construct-clauses.cpp | 553 +++++++++++-
 clang/test/SemaOpenACC/set-construct.cpp      |  15 +-
 clang/test/SemaOpenACC/shutdown-construct.cpp |   8 +-
 clang/test/SemaOpenACC/update-construct.cpp   |  10 +-
 .../llvm/Frontend/Directive/DirectiveBase.td  |   4 +
 llvm/include/llvm/Frontend/OpenACC/ACC.td     |   8 +-
 llvm/include/llvm/TableGen/DirectiveEmitter.h |  48 +
 .../utils/TableGen/Basic/DirectiveEmitter.cpp | 113 ++-
 51 files changed, 1727 insertions(+), 1048 deletions(-)
 create mode 100644 clang/lib/Sema/SemaOpenACCClauseAppertainment.cpp

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index c29a3422acd26..90cf1bd9b2588 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12961,6 +12961,19 @@ def err_acc_clause_after_device_type
 def err_acc_clause_cannot_combine
     : Error<"OpenACC clause '%0' may not appear on the same construct as a "
             "'%1' clause on a '%2' construct">;
+def err_acc_clause_routine_cannot_combine_before_device_type
+    : Error<"OpenACC clause '%0' after 'device_type' clause on a 'routine' "
+              "conflicts with the '%1' clause before the first 'device_type'">;
+def err_acc_clause_routine_cannot_combine_same_device_type
+    : Error<"OpenACC clause '%0' on a 'routine' directive conflicts with the "
+            "'%1' clause applying to the same 'device_type'">;
+def err_acc_clause_routine_one_of_in_region
+    : Error<"OpenACC 'routine' construct must have at least one 'gang', 'seq', "
+            "'vector', or 'worker' clause that applies to each 'device_type'">;
+def err_acc_clause_since_last_device_type
+    : Error<"OpenACC '%0' clause cannot appear more than once%select{| in a "
+            "'device_type' region}2 on a '%1' directive">;
+
 def err_acc_reduction_num_gangs_conflict
     : Error<"OpenACC '%1' clause %select{|with more than 1 argument }0may not "
             "appear on a '%2' construct "
@@ -12978,9 +12991,6 @@ def note_acc_reduction_composite_member_loc : Note<"invalid field is here">;
 def err_acc_loop_not_for_loop
     : Error<"OpenACC '%0' construct can only be applied to a 'for' loop">;
 def note_acc_construct_here : Note<"'%0' construct is here">;
-def err_acc_loop_spec_conflict
-    : Error<"OpenACC clause '%0' on '%1' construct conflicts with previous "
-            "data dependence clause">;
 def err_acc_collapse_loop_count
     : Error<"OpenACC 'collapse' clause loop count must be a %select{constant "
             "expression|positive integer value, evaluated to %1}0">;
@@ -13032,9 +13042,10 @@ def err_acc_gang_reduction_numgangs_conflict
             "'%1' clause %select{inside a compute construct with a|and a}3 "
             "'num_gangs' clause with more than one argument">;
 
-    def err_reduction_op_mismatch
+def err_reduction_op_mismatch
     : Error<"OpenACC 'reduction' variable must have the same operator in all "
             "nested constructs (%0 vs %1)">;
+
 def err_acc_loop_variable_type
     : Error<"loop variable of loop associated with an OpenACC '%0' construct "
             "must be of integer, pointer, or random-access-iterator type (is "
diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h
index 652831e23a758..a798401dfa9f5 100644
--- a/clang/include/clang/Basic/OpenACCKinds.h
+++ b/clang/include/clang/Basic/OpenACCKinds.h
@@ -202,32 +202,21 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &Out,
   return printOpenACCAtomicKind(Out, AK);
 }
 
-/// Represents the kind of an OpenACC clause.
+/// Represents the kind of an OpenACC clause. Sorted alphabetically, since this
+/// order ends up influencing the sorting of the list diagnostic.
 enum class OpenACCClauseKind : uint8_t {
-  /// 'finalize' clause, allowed on 'exit data' directive.
-  Finalize,
-  /// 'if_present' clause, allowed on 'host_data' and 'update' directives.
-  IfPresent,
-  /// 'seq' clause, allowed on 'loop' and 'routine' directives.
-  Seq,
-  /// 'independent' clause, allowed on 'loop' directives.
-  Independent,
+  /// 'async' clause, allowed on Compute, Data, 'update', 'wait', and Combined
+  /// constructs.
+  Async,
+  /// 'attach' clause, allowed on Compute and Combined constructs, plus 'data'
+  /// and 'enter data'.
+  Attach,
   /// 'auto' clause, allowed on 'loop' directives.
   Auto,
-  /// 'worker' clause, allowed on 'loop', Combined, and 'routine' directives.
-  Worker,
-  /// 'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
-  Vector,
-  /// 'nohost' clause, allowed on 'routine' directives.
-  NoHost,
-  /// 'default' clause, allowed on parallel, serial, kernel (and compound)
-  /// constructs.
-  Default,
-  /// 'if' clause, allowed on all the Compute Constructs, Data Constructs,
-  /// Executable Constructs, and Combined Constructs.
-  If,
-  /// 'self' clause, allowed on Compute and Combined Constructs, plus 'update'.
-  Self,
+  /// 'bind' clause, allowed on routine constructs.
+  Bind,
+  /// 'collapse' clause, allowed on 'loop' and Combined constructs.
+  Collapse,
   /// 'copy' clause, allowed on Compute and Combined Constructs, plus 'data' and
   /// 'declare'.
   Copy,
@@ -235,100 +224,118 @@ enum class OpenACCClauseKind : uint8_t {
   PCopy,
   /// 'copy' clause alias 'present_or_copy'.  Preserved for diagnostic purposes.
   PresentOrCopy,
-  /// 'use_device' clause, allowed on 'host_data' construct.
-  UseDevice,
-  /// 'attach' clause, allowed on Compute and Combined constructs, plus 'data'
-  /// and 'enter data'.
-  Attach,
+  /// 'copyin' clause, allowed on Compute and Combined constructs, plus 'data',
+  /// 'enter data', and 'declare'.
+  CopyIn,
+  /// 'copyin' clause alias 'pcopyin'.  Preserved for diagnostic purposes.
+  PCopyIn,
+  /// 'copyin' clause alias 'present_or_copyin'.  Preserved for diagnostic
+  /// purposes.
+  PresentOrCopyIn,
+  /// 'copyout' clause, allowed on Compute and Combined constructs, plus 'data',
+  /// 'exit data', and 'declare'.
+  CopyOut,
+  /// 'copyout' clause alias 'pcopyout'.  Preserved for diagnostic purposes.
+  PCopyOut,
+  /// 'copyout' clause alias 'present_or_copyout'.  Preserved for diagnostic
+  /// purposes.
+  PresentOrCopyOut,
+  /// 'create' clause, allowed on Compute and Combined constructs, plus 'data',
+  /// 'enter data', and 'declare'.
+  Create,
+  /// 'create' clause alias 'pcreate'.  Preserved for diagnostic purposes.
+  PCreate,
+  /// 'create' clause alias 'present_or_create'.  Preserved for diagnostic
+  /// purposes.
+  PresentOrCreate,
+  /// 'default' clause, allowed on parallel, serial, kernel (and compound)
+  /// constructs.
+  Default,
+  /// 'default_async' clause, allowed on 'set' construct.
+  DefaultAsync,
   /// 'delete' clause, allowed on the 'exit data' construct.
   Delete,
   /// 'detach' clause, allowed on the 'exit data' construct.
   Detach,
   /// 'device' clause, allowed on the 'update' construct.
   Device,
+  /// 'device_num' clause, allowed on 'init', 'shutdown', and 'set' constructs.
+  DeviceNum,
   /// 'deviceptr' clause, allowed on Compute and Combined Constructs, plus
   /// 'data' and 'declare'.
   DevicePtr,
   /// 'device_resident' clause, allowed on the 'declare' construct.
   DeviceResident,
+  /// 'device_type' clause, allowed on Compute, 'data', 'init', 'shutdown',
+  /// 'set', update', 'loop', 'routine', and Combined constructs.
+  DeviceType,
+  /// 'dtype' clause, an alias for 'device_type', stored separately for
+  /// diagnostic purposes.
+  DType,
+  /// 'finalize' clause, allowed on 'exit data' directive.
+  Finalize,
   /// 'firstprivate' clause, allowed on 'parallel', 'serial', 'parallel loop',
   /// and 'serial loop' constructs.
   FirstPrivate,
+  /// 'gang' clause, allowed on 'loop' and Combined constructs.
+  Gang,
   /// 'host' clause, allowed on 'update' construct.
   Host,
+  /// 'if' clause, allowed on all the Compute Constructs, Data Constructs,
+  /// Executable Constructs, and Combined Constructs.
+  If,
+  /// 'if_present' clause, allowed on 'host_data' and 'update' directives.
+  IfPresent,
+  /// 'independent' clause, allowed on 'loop' directives.
+  Independent,
   /// 'link' clause, allowed on 'declare' construct.
   Link,
   /// 'no_create' clause, allowed on allowed on Compute and Combined constructs,
   /// plus 'data'.
   NoCreate,
+  /// 'nohost' clause, allowed on 'routine' directives.
+  NoHost,
+  /// 'num_gangs' clause, allowed on 'parallel', 'kernels', parallel loop', and
+  /// 'kernels loop' constructs.
+  NumGangs,
+  /// 'num_workers' clause, allowed on 'parallel', 'kernels', parallel loop',
+  /// and 'kernels loop' constructs.
+  NumWorkers,
   /// 'present' clause, allowed on Compute and Combined constructs, plus 'data'
   /// and 'declare'.
   Present,
   /// 'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel
   /// loop', and 'serial loop' constructs.
   Private,
-  /// 'copyout' clause, allowed on Compute and Combined constructs, plus 'data',
-  /// 'exit data', and 'declare'.
-  CopyOut,
-  /// 'copyout' clause alias 'pcopyout'.  Preserved for diagnostic purposes.
-  PCopyOut,
-  /// 'copyout' clause alias 'present_or_copyout'.  Preserved for diagnostic
-  /// purposes.
-  PresentOrCopyOut,
-  /// 'copyin' clause, allowed on Compute and Combined constructs, plus 'data',
-  /// 'enter data', and 'declare'.
-  CopyIn,
-  /// 'copyin' clause alias 'pcopyin'.  Preserved for diagnostic purposes.
-  PCopyIn,
-  /// 'copyin' clause alias 'present_or_copyin'.  Preserved for diagnostic
-  /// purposes.
-  PresentOrCopyIn,
-  /// 'create' clause, allowed on Compute and Combined constructs, plus 'data',
-  /// 'enter data', and 'declare'.
-  Create,
-  /// 'create' clause alias 'pcreate'.  Preserved for diagnostic purposes.
-  PCreate,
-  /// 'create' clause alias 'present_or_create'.  Preserved for diagnostic
-  /// purposes.
-  PresentOrCreate,
   /// 'reduction' clause, allowed on Parallel, Serial, Loop, and the combined
   /// constructs.
   Reduction,
-  /// 'collapse' clause, allowed on 'loop' and Combined constructs.
-  Collapse,
-  /// 'bind' clause, allowed on routine constructs.
-  Bind,
+  /// 'self' clause, allowed on Compute and Combined Constructs, plus 'update'.
+  Self,
+  /// 'seq' clause, allowed on 'loop' and 'routine' directives.
+  Seq,
+  /// 'tile' clause, allowed on 'loop' and Combined constructs.
+  Tile,
+  /// 'use_device' clause, allowed on 'host_data' construct.
+  UseDevice,
+  /// 'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
+  Vector,
   /// 'vector_length' clause, allowed on 'parallel', 'kernels', 'parallel loop',
   /// and 'kernels loop' constructs.
   VectorLength,
-  /// 'num_gangs' clause, allowed on 'parallel', 'kernels', parallel loop', and
-  /// 'kernels loop' constructs.
-  NumGangs,
-  /// 'num_workers' clause, allowed on 'parallel', 'kernels', parallel loop',
-  /// and 'kernels loop' constructs.
-  NumWorkers,
-  /// 'device_num' clause, allowed on 'init', 'shutdown', and 'set' constructs.
-  DeviceNum,
-  /// 'default_async' clause, allowed on 'set' construct.
-  DefaultAsync,
-  /// 'device_type' clause, allowed on Compute, 'data', 'init', 'shutdown',
-  /// 'set', update', 'loop', 'routine', and Combined constructs.
-  DeviceType,
-  /// 'dtype' clause, an alias for 'device_type', stored separately for
-  /// diagnostic purposes.
-  DType,
-  /// 'async' clause, allowed on Compute, Data, 'update', 'wait', and Combined
-  /// constructs.
-  Async,
-  /// 'tile' clause, allowed on 'loop' and Combined constructs.
-  Tile,
-  /// 'gang' clause, allowed on 'loop' and Combined constructs.
-  Gang,
   /// 'wait' clause, allowed on Compute, Data, 'update', and Combined
   /// constructs.
   Wait,
+  /// 'worker' clause, allowed on 'loop', Combined, and 'routine' directives.
+  Worker,
 
-  /// Represents an invalid clause, for the purposes of parsing.
+  /// 'shortloop' is represented in the ACC.td file, but isn't present in the
+  /// standard. This appears to be an old extension for the nvidia fortran
+  // compiler, but seemingly not elsewhere. Put it here as a placeholder, but it
+  // is never expected to be generated.
+  Shortloop,
+  /// Represents an invalid clause, for the purposes of parsing. Should be
+  /// 'last'.
   Invalid,
 };
 
@@ -485,6 +492,9 @@ inline StreamTy &printOpenACCClauseKind(StreamTy &Out, OpenACCClauseKind K) {
   case OpenACCClauseKind::Wait:
     return Out << "wait";
 
+  case OpenACCClauseKind::Shortloop:
+    llvm_unreachable("Shortloop shouldn't be generated in clang");
+    LLVM_FALLTHROUGH;
   case OpenACCClauseKind::Invalid:
     return Out << "<invalid>";
   }
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 8d31d46444c7e..2c9134b0e6d33 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -78,7 +78,7 @@ class SemaOpenACC : public SemaBase {
   /// Collapse has an 'N' count that makes it apply to a number of loops 'below'
   /// it.
   struct CollapseCheckingInfo {
-    OpenACCCollapseClause *ActiveCollapse = nullptr;
+    const OpenACCCollapseClause *ActiveCollapse = nullptr;
 
     /// This is a value that maintains the current value of the 'N' on the
     /// current collapse, minus the depth that has already been traversed. When
@@ -107,7 +107,7 @@ class SemaOpenACC : public SemaBase {
     /// own counting of elements.
     UnsignedOrNone CurTileCount = std::nullopt;
 
-    /// Records whether we've hit a 'CurTileCount' of '0' on the wya down,
+    /// Records whether we've hit a 'CurTileCount' of '0' on the way down,
     /// which allows us to diagnose if the number of arguments is too large for
     /// the current number of 'for' loops.
     bool TileDepthSatisfied = true;
@@ -177,6 +177,21 @@ class SemaOpenACC : public SemaBase {
 
   void CheckLastRoutineDeclNameConflict(const NamedDecl *ND);
 
+  bool DiagnoseRequiredClauses(OpenACCDirectiveKind DK, SourceLocation DirLoc,
+                               ArrayRef<const OpenACCClause *> Clauses);
+
+  bool DiagnoseAllowedClauses(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
+                              SourceLocation ClauseLoc);
+
+public:
+  // Needed from the visitor, so should be public.
+  bool DiagnoseAllowedOnceClauses(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
+                                  SourceLocation ClauseLoc,
+                                  ArrayRef<const OpenACCClause *> Clauses);
+  bool DiagnoseExclusiveClauses(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
+                                SourceLocation ClauseLoc,
+                                ArrayRef<const OpenACCClause *> Clauses);
+
 public:
   ComputeConstructInfo &getActiveComputeConstructInfo() {
     return ActiveComputeConstructInfo;
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index 2820d7b288658..01e183051934d 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -190,18 +190,15 @@ OpenACCCollapseClause::OpenACCCollapseClause(SourceLocation BeginLoc,
                                              SourceLocation EndLoc)
     : OpenACCClauseWithSingleIntExpr(OpenACCClauseKind::Collapse, BeginLoc,
                                      LParenLoc, LoopCount, EndLoc),
-      HasForce(HasForce) {
-  assert(LoopCount && "LoopCount required");
-}
+      HasForce(HasForce) {}
 
 OpenACCCollapseClause *
 OpenACCCollapseClause::Create(const ASTContext &C, SourceLocation BeginLoc,
                               SourceLocation LParenLoc, bool HasForce,
                               Expr *LoopCount, SourceLocation EndLoc) {
-  assert(
-      LoopCount &&
-      (LoopCount->isInstantiationDependent() || isa<ConstantExpr>(LoopCount)) &&
-      "Loop count not constant expression");
+  assert((!LoopCount || (LoopCount->isInstantiationDependent() ||
+                         isa<ConstantExpr>(LoopCount))) &&
+         "Loop count not constant expression");
   void *Mem =
       C.Allocate(sizeof(OpenACCCollapseClause), alignof(OpenACCCollapseClause));
   return new (Mem)
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 5f8417ac57b8b..51f7d17be4158 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -433,6 +433,7 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
     case OpenACCClauseKind::Vector:
     case OpenACCClauseKind::VectorLength:
     case OpenACCClauseKind::Invalid:
+    case OpenACCClauseKind::Shortloop:
       // The condition expression will be printed as a part of the 'children',
       // but print 'clause' here so it is clear what is happening from the dump.
       OS << " clause";
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 337b3eca49764..3843e1ad51cf5 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -537,6 +537,8 @@ ClauseParensKind getClauseParensKind(OpenACCDirectiveKind DirKind,
   case OpenACCClauseKind::Tile:
     return ClauseParensKind::Required;
 
+  case OpenACCClauseKind::Shortloop:
+    llvm_unreachable("Shortloop shouldn't be generated in clang");
   case OpenACCClauseKind::Auto:
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index cc7921fc32254..d0b0508eb263a 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -75,6 +75,7 @@ add_clang_library(clangSema
   SemaOpenACC.cpp
   SemaOpenACCAtomic.cpp
   SemaOpenACCClause.cpp
+  SemaOpenACCClauseAppertainment.cpp
   SemaOpenCL.cpp
   SemaOpenMP.cpp
   SemaOverload.cpp
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index c80f5f848f60b..74eee9b13ac6e 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -221,6 +221,48 @@ SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(
   }
 }
 
+namespace {
+// Given two collapse clauses, and the uninstanted version of the new one,
+// return the 'best' one for the purposes of setting the collapse checking
+// values.
+const OpenACCCollapseClause *
+getBestCollapseCandidate(const OpenACCCollapseClause *Old,
+                         const OpenACCCollapseClause *New,
+                         const OpenACCCollapseClause *UnInstNew) {
+  // If the loop count is nullptr, it is because instantiation failed, so this
+  // can't be the best one.
+  if (!New->getLoopCount())
+    return Old;
+
+  // If the loop-count had an error, than 'new' isn't a candidate.
+  if (!New->getLoopCount())
+    return Old;
+
+  // Don't consider uninstantiated ones, since we can't really check these.
+  if (New->getLoopCount()->isInstantiationDependent())
+    return Old;
+
+  // If this is an instantiation, and the old version wasn't instantation
+  // dependent, than nothing has changed and we've already done a diagnostic
+  // based on this one, so don't consider it.
+  if (UnInstNew && !UnInstNew->getLoopCount()->isInstantiationDependent())
+    return Old;
+
+  // New is now a valid candidate, so if there isn't an old one at this point,
+  // New is the only valid one.
+  if (!Old)
+    return New;
+
+  // If the 'New' expression has a larger value than 'Old', then it is the new
+  // best candidate.
+  if (cast<ConstantExpr>(Old->getLoopCount())->getResultAsAPSInt() <
+      cast<ConstantExpr>(New->getLoopCount())->getResultAsAPSInt())
+    return New;
+
+  return Old;
+}
+} // namespace
+
 void SemaOpenACC::AssociatedStmtRAII::SetCollapseInfoBeforeAssociatedStmt(
     ArrayRef<const OpenACCClause *> UnInstClauses,
     ArrayRef<OpenACCClause *> Clauses) {
@@ -228,43 +270,57 @@ void SemaOpenACC::AssociatedStmtRAII::SetCollapseInfoBeforeAssociatedStmt(
   // Reset this checking for loops that aren't covered in a RAII object.
   SemaRef.LoopInfo.CurLevelHasLoopAlready = false;
   SemaRef.CollapseInfo.CollapseDepthSatisfied = true;
+  SemaRef.CollapseInfo.CurCollapseCount = 0;
   SemaRef.TileInfo.TileDepthSatisfied = true;
 
   // We make sure to take an optional list of uninstantiated clauses, so that
   // we can check to make sure we don't 'double diagnose' in the event that
-  // the value of 'N' was not dependent in a template. We also ensure during
-  // Sema that there is only 1 collapse on each construct, so we can count on
-  // the fact that if both find a 'collapse', that they are the same one.
-  auto *CollapseClauseItr =
-      llvm::find_if(Clauses, llvm::IsaPred<OpenACCCollapseClause>);
-  auto *UnInstCollapseClauseItr =
+  // the value of 'N' was not dependent in a template. Since we cannot count on
+  // there only being a single collapse clause, we count on the order to make
+  // sure get the matching ones, and we count on TreeTransform not removing
+  // these, even if loop-count instantiation failed. We can check the
+  // non-dependent ones right away, and realize that subsequent instantiation
+  // can only make it more specific.
+
+  auto *UnInstClauseItr =
       llvm::find_if(UnInstClauses, llvm::IsaPred<OpenACCCollapseClause>);
-
-  if (Clauses.end() == CollapseClauseItr)
-    return;
-
-  OpenACCCollapseClause *CollapseClause =
-      cast<OpenACCCollapseClause>(*CollapseClauseItr);
-
-  SemaRef.CollapseInfo.ActiveCollapse = CollapseClause;
-  Expr *LoopCount = CollapseClause->getLoopCount();
-
-  // If the loop count is still instantiation dependent, setting the depth
-  // counter isn't necessary, so return here.
-  if (!LoopCount || LoopCount->isInstantiationDependent())
-    return;
-
-  // Suppress diagnostics if we've done a 'transform' where the previous version
-  // wasn't dependent, meaning we already diagnosed it.
-  if (UnInstCollapseClauseItr != UnInstClauses.end() &&
-      !cast<OpenACCCollapseClause>(*UnInstCollapseClauseItr)
-           ->getLoopCount()
-           ->isInstantiationDependent())
+  auto *ClauseItr =
+      llvm::find_if(Clauses, llvm::IsaPred<OpenACCCollapseClause>);
+  const OpenACCCollapseClause *FoundClause = nullptr;
+
+  // Loop through the list of Collapse clauses and find the one that:
+  // 1- Has a non-dependent, non-null loop count (null means error, likely
+  // during instantiation).
+  // 2- If UnInstClauses isn't empty, its corresponding
+  // loop count was dependent.
+  // 3- Has the largest 'loop count' of all.
+  while (ClauseItr != Clauses.end()) {
+    const OpenACCCollapseClause *CurClause =
+        cast<OpenACCCollapseClause>(*ClauseItr);
+    const OpenACCCollapseClause *UnInstCurClause =
+        UnInstClauseItr == UnInstClauses.end()
+            ? nullptr
+            : cast<OpenACCCollapseClause>(*UnInstClauseItr);
+
+    FoundClause =
+        getBestCollapseCandidate(FoundClause, CurClause, UnInstCurClause);
+
+    UnInstClauseItr =
+        UnInstClauseItr == UnInstClauses.end()
+            ? UnInstClauseItr
+            : std::find_if(std::next(UnInstClauseItr), UnInstClauses.end(),
+                           llvm::IsaPred<OpenACCCollapseClause>);
+    ClauseItr = std::find_if(std::next(ClauseItr), Clauses.end(),
+                             llvm::IsaPred<OpenACCCollapseClause>);
+  }
+
+  if (!FoundClause)
     return;
 
+  SemaRef.CollapseInfo.ActiveCollapse = FoundClause;
   SemaRef.CollapseInfo.CollapseDepthSatisfied = false;
   SemaRef.CollapseInfo.CurCollapseCount =
-      cast<ConstantExpr>(LoopCount)->getResultAsAPSInt();
+      cast<ConstantExpr>(FoundClause->getLoopCount())->getResultAsAPSInt();
   SemaRef.CollapseInfo.DirectiveKind = DirKind;
 }
 
@@ -283,6 +339,17 @@ void SemaOpenACC::AssociatedStmtRAII::SetTileInfoBeforeAssociatedStmt(
     return;
 
   OpenACCTileClause *TileClause = cast<OpenACCTileClause>(*TileClauseItr);
+
+  // Multiple tile clauses are allowed, so ensure that we use the one with the
+  // largest 'tile count'.
+  while (Clauses.end() !=
+         (TileClauseItr = std::find_if(std::next(TileClauseItr), Clauses.end(),
+                                       llvm::IsaPred<OpenACCTileClause>))) {
+    OpenACCTileClause *NewClause = cast<OpenACCTileClause>(*TileClauseItr);
+    if (NewClause->getSizeExprs().size() > TileClause->getSizeExprs().size())
+      TileClause = NewClause;
+  }
+
   SemaRef.TileInfo.ActiveTile = TileClause;
   SemaRef.TileInfo.TileDepthSatisfied = false;
   SemaRef.TileInfo.CurTileCount =
@@ -1426,30 +1493,6 @@ void SemaOpenACC::ActOnForStmtEnd(SourceLocation ForLoc, StmtResult Body) {
 }
 
 namespace {
-// Get a list of clause Kinds for diagnosing a list, joined by a commas and an
-// 'or'.
-std::string GetListOfClauses(llvm::ArrayRef<OpenACCClauseKind> Clauses) {
-  assert(!Clauses.empty() && "empty clause list not supported");
-
-  std::string Output;
-  llvm::raw_string_ostream OS{Output};
-
-  if (Clauses.size() == 1) {
-    OS << '\'' << Clauses[0] << '\'';
-    return Output;
-  }
-
-  llvm::ArrayRef<OpenACCClauseKind> AllButLast{Clauses.begin(),
-                                               Clauses.end() - 1};
-
-  llvm::interleave(
-      AllButLast, [&](OpenACCClauseKind K) { OS << '\'' << K << '\''; },
-      [&] { OS << ", "; });
-
-  OS << " or \'" << Clauses.back() << '\'';
-  return Output;
-}
-
 // Helper that should mirror ActOnRoutineName to get the FunctionDecl out for
 // magic-static checking.
 FunctionDecl *getFunctionFromRoutineName(Expr *RoutineName) {
@@ -1682,86 +1725,8 @@ bool SemaOpenACC::ActOnStartStmtDirective(
         << OpenACCClauseKind::Tile;
   }
 
-  // OpenACC3.3 2.6.5: At least one copy, copyin, copyout, create, no_create,
-  // present, deviceptr, attach, or default clause must appear on a 'data'
-  // construct.
-  if (K == OpenACCDirectiveKind::Data &&
-      llvm::find_if(Clauses,
-                    llvm::IsaPred<OpenACCCopyClause, OpenACCCopyInClause,
-                                  OpenACCCopyOutClause, OpenACCCreateClause,
-                                  OpenACCNoCreateClause, OpenACCPresentClause,
-                                  OpenACCDevicePtrClause, OpenACCAttachClause,
-                                  OpenACCDefaultClause>) == Clauses.end())
-    return Diag(StartLoc, diag::err_acc_construct_one_clause_of)
-           << K
-           << GetListOfClauses(
-                  {OpenACCClauseKind::Copy, OpenACCClauseKind::CopyIn,
-                   OpenACCClauseKind::CopyOut, OpenACCClauseKind::Create,
-                   OpenACCClauseKind::NoCreate, OpenACCClauseKind::Present,
-                   OpenACCClauseKind::DevicePtr, OpenACCClauseKind::Attach,
-                   OpenACCClauseKind::Default});
-
-  // OpenACC3.3 2.6.6: At least one copyin, create, or attach clause must appear
-  // on an enter data directive.
-  if (K == OpenACCDirectiveKind::EnterData &&
-      llvm::find_if(Clauses,
-                    llvm::IsaPred<OpenACCCopyInClause, OpenACCCreateClause,
-                                  OpenACCAttachClause>) == Clauses.end())
-    return Diag(StartLoc, diag::err_acc_construct_one_clause_of)
-           << K
-           << GetListOfClauses({
-                  OpenACCClauseKind::CopyIn,
-                  OpenACCClauseKind::Create,
-                  OpenACCClauseKind::Attach,
-              });
-  // OpenACC3.3 2.6.6: At least one copyout, delete, or detach clause must
-  // appear on an exit data directive.
-  if (K == OpenACCDirectiveKind::ExitData &&
-      llvm::find_if(Clauses,
-                    llvm::IsaPred<OpenACCCopyOutClause, OpenACCDeleteClause,
-                                  OpenACCDetachClause>) == Clauses.end())
-    return Diag(StartLoc, diag::err_acc_construct_one_clause_of)
-           << K
-           << GetListOfClauses({
-                  OpenACCClauseKind::CopyOut,
-                  OpenACCClauseKind::Delete,
-                  OpenACCClauseKind::Detach,
-              });
-
-  // OpenACC3.3 2.8: At least 'one use_device' clause must appear.
-  if (K == OpenACCDirectiveKind::HostData &&
-      llvm::find_if(Clauses, llvm::IsaPred<OpenACCUseDeviceClause>) ==
-          Clauses.end())
-    return Diag(StartLoc, diag::err_acc_construct_one_clause_of)
-           << K << GetListOfClauses({OpenACCClauseKind::UseDevice});
-
-  // OpenACC3.3 2.14.3: At least one default_async, device_num, or device_type
-  // clause must appear.
-  if (K == OpenACCDirectiveKind::Set &&
-      llvm::find_if(
-          Clauses,
-          llvm::IsaPred<OpenACCDefaultAsyncClause, OpenACCDeviceNumClause,
-                        OpenACCDeviceTypeClause, OpenACCIfClause>) ==
-          Clauses.end())
-    return Diag(StartLoc, diag::err_acc_construct_one_clause_of)
-           << K
-           << GetListOfClauses({OpenACCClauseKind::DefaultAsync,
-                                OpenACCClauseKind::DeviceNum,
-                                OpenACCClauseKind::DeviceType,
-                                OpenACCClauseKind::If});
-
-  // OpenACC3.3 2.14.4: At least one self, host, or device clause must appear on
-  // an update directive.
-  if (K == OpenACCDirectiveKind::Update &&
-      llvm::find_if(Clauses, llvm::IsaPred<OpenACCSelfClause, OpenACCHostClause,
-                                           OpenACCDeviceClause>) ==
-          Clauses.end())
-    return Diag(StartLoc, diag::err_acc_construct_one_clause_of)
-           << K
-           << GetListOfClauses({OpenACCClauseKind::Self,
-                                OpenACCClauseKind::Host,
-                                OpenACCClauseKind::Device});
-
+  if (DiagnoseRequiredClauses(K, StartLoc, Clauses))
+    return true;
   return diagnoseConstructAppertainment(*this, K, StartLoc, /*IsStmt=*/true);
 }
 
@@ -1930,6 +1895,62 @@ StmtResult SemaOpenACC::ActOnAssociatedStmt(
   llvm_unreachable("Invalid associated statement application");
 }
 
+namespace {
+
+// Routine has some pretty complicated set of rules for how device_type
+// interacts with 'gang', 'worker', 'vector', and 'seq'. Enforce  part of it
+// here.
+bool CheckValidRoutineGangWorkerVectorSeqClauses(
+    SemaOpenACC &SemaRef, SourceLocation DirectiveLoc,
+    ArrayRef<const OpenACCClause *> Clauses) {
+  auto RequiredPred = llvm::IsaPred<OpenACCGangClause, OpenACCWorkerClause,
+                                    OpenACCVectorClause, OpenACCSeqClause>;
+  // The clause handling has assured us that there is no duplicates.  That is,
+  // if there is 1 before a device_type, there are none after a device_type.
+  // If not, there is at most 1 applying to each device_type.
+
+  // What is left to legalize is that either:
+  // 1- there is 1 before the first device_type.
+  // 2- there is 1 AFTER each device_type.
+  auto *FirstDeviceType =
+      llvm::find_if(Clauses, llvm::IsaPred<OpenACCDeviceTypeClause>);
+
+  // If there is 1 before the first device_type (or at all if no device_type),
+  // we are legal.
+  auto *ClauseItr =
+      std::find_if(Clauses.begin(), FirstDeviceType, RequiredPred);
+
+  if (ClauseItr != FirstDeviceType)
+    return false;
+
+  // If there IS no device_type, and no clause, diagnose.
+  if (FirstDeviceType == Clauses.end())
+    return SemaRef.Diag(DirectiveLoc, diag::err_acc_construct_one_clause_of)
+           << OpenACCDirectiveKind::Routine
+           << "'gang', 'seq', 'vector', or 'worker'";
+
+  // Else, we have to check EACH device_type group. PrevDeviceType is the
+  // device-type before the current group.
+  auto *PrevDeviceType = FirstDeviceType;
+
+  while (PrevDeviceType != Clauses.end()) {
+    auto *NextDeviceType =
+        std::find_if(std::next(PrevDeviceType), Clauses.end(),
+                     llvm::IsaPred<OpenACCDeviceTypeClause>);
+
+    ClauseItr = std::find_if(PrevDeviceType, NextDeviceType, RequiredPred);
+
+    if (ClauseItr == NextDeviceType)
+      return SemaRef.Diag((*PrevDeviceType)->getBeginLoc(),
+                          diag::err_acc_clause_routine_one_of_in_region);
+
+    PrevDeviceType = NextDeviceType;
+  }
+
+  return false;
+}
+} // namespace
+
 bool SemaOpenACC::ActOnStartDeclDirective(
     OpenACCDirectiveKind K, SourceLocation StartLoc,
     ArrayRef<const OpenACCClause *> Clauses) {
@@ -1939,19 +1960,11 @@ bool SemaOpenACC::ActOnStartDeclDirective(
   SemaRef.DiscardCleanupsInEvaluationContext();
   SemaRef.PopExpressionEvaluationContext();
 
+  if (DiagnoseRequiredClauses(K, StartLoc, Clauses))
+    return true;
   if (K == OpenACCDirectiveKind::Routine &&
-      llvm::find_if(Clauses,
-                    llvm::IsaPred<OpenACCGangClause, OpenACCWorkerClause,
-                                  OpenACCVectorClause, OpenACCSeqClause>) ==
-          Clauses.end())
-    return Diag(StartLoc, diag::err_acc_construct_one_clause_of)
-           << K
-           << GetListOfClauses({
-                  OpenACCClauseKind::Gang,
-                  OpenACCClauseKind::Worker,
-                  OpenACCClauseKind::Vector,
-                  OpenACCClauseKind::Seq,
-              });
+      CheckValidRoutineGangWorkerVectorSeqClauses(*this, StartLoc, Clauses))
+    return true;
 
   return diagnoseConstructAppertainment(*this, K, StartLoc, /*IsStmt=*/false);
 }
diff --git a/clang/lib/Sema/SemaOpenACCClause.cpp b/clang/lib/Sema/SemaOpenACCClause.cpp
index 049baead031a1..3694a831b76de 100644
--- a/clang/lib/Sema/SemaOpenACCClause.cpp
+++ b/clang/lib/Sema/SemaOpenACCClause.cpp
@@ -20,502 +20,6 @@
 using namespace clang;
 
 namespace {
-bool doesClauseApplyToDirective(OpenACCDirectiveKind DirectiveKind,
-                                OpenACCClauseKind ClauseKind) {
-  switch (ClauseKind) {
-    // FIXME: For each clause as we implement them, we can add the
-    // 'legalization' list here.
-  case OpenACCClauseKind::Default:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-    case OpenACCDirectiveKind::Data:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::If:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::EnterData:
-    case OpenACCDirectiveKind::ExitData:
-    case OpenACCDirectiveKind::HostData:
-    case OpenACCDirectiveKind::Init:
-    case OpenACCDirectiveKind::Shutdown:
-    case OpenACCDirectiveKind::Set:
-    case OpenACCDirectiveKind::Update:
-    case OpenACCDirectiveKind::Wait:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      // OpenACC 3.4(prerelease) PR #511 adds 'if' to atomic.
-    case OpenACCDirectiveKind::Atomic:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::Self:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Update:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::NumGangs:
-  case OpenACCClauseKind::NumWorkers:
-  case OpenACCClauseKind::VectorLength:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::FirstPrivate:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::Private:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::NoCreate:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::Present:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::Declare:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-
-  case OpenACCClauseKind::Copy:
-  case OpenACCClauseKind::PCopy:
-  case OpenACCClauseKind::PresentOrCopy:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::Declare:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::CopyIn:
-  case OpenACCClauseKind::PCopyIn:
-  case OpenACCClauseKind::PresentOrCopyIn:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::EnterData:
-    case OpenACCDirectiveKind::Declare:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::CopyOut:
-  case OpenACCClauseKind::PCopyOut:
-  case OpenACCClauseKind::PresentOrCopyOut:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::ExitData:
-    case OpenACCDirectiveKind::Declare:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::Create:
-  case OpenACCClauseKind::PCreate:
-  case OpenACCClauseKind::PresentOrCreate:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::EnterData:
-    case OpenACCDirectiveKind::Declare:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-
-  case OpenACCClauseKind::Attach:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::EnterData:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::DevicePtr:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::Declare:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::Async:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::EnterData:
-    case OpenACCDirectiveKind::ExitData:
-    case OpenACCDirectiveKind::Set:
-    case OpenACCDirectiveKind::Update:
-    case OpenACCDirectiveKind::Wait:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::Wait:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::EnterData:
-    case OpenACCDirectiveKind::ExitData:
-    case OpenACCDirectiveKind::Update:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-
-  case OpenACCClauseKind::Seq:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::Routine:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-
-  case OpenACCClauseKind::Independent:
-  case OpenACCClauseKind::Auto:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-
-  case OpenACCClauseKind::Reduction:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-
-  case OpenACCClauseKind::DeviceType:
-  case OpenACCClauseKind::DType:
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Parallel:
-    case OpenACCDirectiveKind::Serial:
-    case OpenACCDirectiveKind::Kernels:
-    case OpenACCDirectiveKind::Data:
-    case OpenACCDirectiveKind::Init:
-    case OpenACCDirectiveKind::Shutdown:
-    case OpenACCDirectiveKind::Set:
-    case OpenACCDirectiveKind::Update:
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::Routine:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-
-  case OpenACCClauseKind::Collapse: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::Tile: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  case OpenACCClauseKind::Gang: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-    case OpenACCDirectiveKind::Routine:
-      return true;
-    default:
-      return false;
-    }
-  case OpenACCClauseKind::Worker: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-    case OpenACCDirectiveKind::Routine:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::Vector: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Loop:
-    case OpenACCDirectiveKind::ParallelLoop:
-    case OpenACCDirectiveKind::SerialLoop:
-    case OpenACCDirectiveKind::KernelsLoop:
-    case OpenACCDirectiveKind::Routine:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::Finalize: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::ExitData:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::IfPresent: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::HostData:
-    case OpenACCDirectiveKind::Update:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::Delete: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::ExitData:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  case OpenACCClauseKind::Detach: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::ExitData:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  case OpenACCClauseKind::DeviceNum: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Init:
-    case OpenACCDirectiveKind::Shutdown:
-    case OpenACCDirectiveKind::Set:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::Link: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Declare:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::DeviceResident: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Declare:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  case OpenACCClauseKind::UseDevice: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::HostData:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::DefaultAsync: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Set:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::Device: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Update:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::Host: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Update:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::NoHost: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Routine:
-      return true;
-    default:
-      return false;
-    }
-  }
-  case OpenACCClauseKind::Bind: {
-    switch (DirectiveKind) {
-    case OpenACCDirectiveKind::Routine:
-      return true;
-    default:
-      return false;
-    }
-  }
-  }
-
-  default:
-    // Do nothing so we can go to the 'unimplemented' diagnostic instead.
-    return true;
-  }
-  llvm_unreachable("Invalid clause kind");
-}
-
-bool checkAlreadyHasClauseOfKind(
-    SemaOpenACC &S, ArrayRef<const OpenACCClause *> ExistingClauses,
-    SemaOpenACC::OpenACCParsedClause &Clause) {
-  const auto *Itr = llvm::find_if(ExistingClauses, [&](const OpenACCClause *C) {
-    return C->getClauseKind() == Clause.getClauseKind();
-  });
-  if (Itr != ExistingClauses.end()) {
-    S.Diag(Clause.getBeginLoc(), diag::err_acc_duplicate_clause_disallowed)
-        << Clause.getDirectiveKind() << Clause.getClauseKind();
-    S.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    return true;
-  }
-  return false;
-}
 bool checkValidAfterDeviceType(
     SemaOpenACC &S, const OpenACCDeviceTypeClause &DeviceTypeClause,
     const SemaOpenACC::OpenACCParsedClause &NewClause) {
@@ -627,14 +131,6 @@ bool checkValidAfterDeviceType(
   return true;
 }
 
-// A temporary function that helps implement the 'not implemented' check at the
-// top of each clause checking function. This should only be used in conjunction
-// with the one being currently implemented/only updated after the entire
-// construct has been implemented.
-bool isDirectiveKindImplemented(OpenACCDirectiveKind DK) {
-  return DK != OpenACCDirectiveKind::Routine;
-}
-
 // GCC looks through linkage specs, but not the other transparent declaration
 // contexts for 'declare' restrictions, so this helper function helps get us
 // through that.
@@ -649,28 +145,20 @@ class SemaOpenACCClauseVisitor {
   SemaOpenACC &SemaRef;
   ASTContext &Ctx;
   ArrayRef<const OpenACCClause *> ExistingClauses;
-  bool NotImplemented = false;
-
-  OpenACCClause *isNotImplemented() {
-    NotImplemented = true;
-    return nullptr;
-  }
 
   // OpenACC 3.3 2.9:
-  // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
-  // appears.
-  // -also-
-  // OpenACC3.3 2.15: (routine)
-  // Exactly one of the 'gang', 'worker', 'vector' or 'seq' clauses must appear.
+  //  A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
+  //  appears.
   bool
   DiagGangWorkerVectorSeqConflict(SemaOpenACC::OpenACCParsedClause &Clause) {
+    if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop &&
+        !isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind()))
+      return false;
+    assert(Clause.getClauseKind() == OpenACCClauseKind::Gang ||
+           Clause.getClauseKind() == OpenACCClauseKind::Worker ||
+           Clause.getClauseKind() == OpenACCClauseKind::Vector);
     const auto *Itr =
-        Clause.getDirectiveKind() == OpenACCDirectiveKind::Routine
-            ? llvm::find_if(
-                  ExistingClauses,
-                  llvm::IsaPred<OpenACCSeqClause, OpenACCGangClause,
-                                OpenACCWorkerClause, OpenACCVectorClause>)
-            : llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSeqClause>);
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSeqClause>);
 
     if (Itr != ExistingClauses.end()) {
       SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
@@ -736,13 +224,165 @@ class SemaOpenACCClauseVisitor {
     llvm_unreachable("didn't return from switch above?");
   }
 
+  // Helper for the 'routine' checks during 'new' clause addition. Precondition
+  // is that we already know the new clause is one of the prohbiited ones.
+  template <typename Pred>
+  bool
+  CheckValidRoutineNewClauseHelper(Pred HasPredicate,
+                                   SemaOpenACC::OpenACCParsedClause &Clause) {
+    if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Routine)
+      return false;
+
+    auto *FirstDeviceType =
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCDeviceTypeClause>);
+
+    if (FirstDeviceType == ExistingClauses.end()) {
+      // If there isn't a device type yet, ANY duplicate is wrong.
+
+      auto *ExistingProhibitedClause =
+          llvm::find_if(ExistingClauses, HasPredicate);
+
+      if (ExistingProhibitedClause == ExistingClauses.end())
+        return false;
+
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
+          << Clause.getClauseKind()
+          << (*ExistingProhibitedClause)->getClauseKind()
+          << Clause.getDirectiveKind();
+      SemaRef.Diag((*ExistingProhibitedClause)->getBeginLoc(),
+                   diag::note_acc_previous_clause_here);
+      return true;
+    }
+
+    // At this point we know that this is 'after' a device type. So this is an
+    // error if: 1- there is one BEFORE the 'device_type' 2- there is one
+    // between this and the previous 'device_type'.
+
+    auto *BeforeDeviceType =
+        std::find_if(ExistingClauses.begin(), FirstDeviceType, HasPredicate);
+    // If there is one before the device_type (and we know we are after a
+    // device_type), than this is ill-formed.
+    if (BeforeDeviceType != FirstDeviceType) {
+      SemaRef.Diag(
+          Clause.getBeginLoc(),
+          diag::err_acc_clause_routine_cannot_combine_before_device_type)
+          << Clause.getClauseKind() << (*BeforeDeviceType)->getClauseKind();
+      SemaRef.Diag((*BeforeDeviceType)->getBeginLoc(),
+                   diag::note_acc_previous_clause_here);
+      SemaRef.Diag((*FirstDeviceType)->getBeginLoc(),
+                   diag::note_acc_previous_clause_here);
+      return true;
+    }
+
+    auto LastDeviceTypeItr =
+        std::find_if(ExistingClauses.rbegin(), ExistingClauses.rend(),
+                     llvm::IsaPred<OpenACCDeviceTypeClause>);
+
+    // We already know there is one in the list, so it is nonsensical to not
+    // have one.
+    assert(LastDeviceTypeItr != ExistingClauses.rend());
+
+    // Get the device-type from-the-front (not reverse) iterator from the
+    // reverse iterator.
+    auto *LastDeviceType = LastDeviceTypeItr.base() - 1;
+
+    auto *ExistingProhibitedSinceLastDevice =
+        std::find_if(LastDeviceType, ExistingClauses.end(), HasPredicate);
+
+    // No prohibited ones since the last device-type.
+    if (ExistingProhibitedSinceLastDevice == ExistingClauses.end())
+      return false;
+
+    SemaRef.Diag(Clause.getBeginLoc(),
+                 diag::err_acc_clause_routine_cannot_combine_same_device_type)
+        << Clause.getClauseKind()
+        << (*ExistingProhibitedSinceLastDevice)->getClauseKind();
+    SemaRef.Diag((*ExistingProhibitedSinceLastDevice)->getBeginLoc(),
+                 diag::note_acc_previous_clause_here);
+    SemaRef.Diag((*LastDeviceType)->getBeginLoc(),
+                 diag::note_acc_previous_clause_here);
+    return true;
+  }
+
+  // Routine has a pretty complicated set of rules for how device_type and the
+  // gang, worker, vector, and seq clauses work.  So diagnose some of it here.
+  bool CheckValidRoutineGangWorkerVectorSeqNewClause(
+      SemaOpenACC::OpenACCParsedClause &Clause) {
+
+    if (Clause.getClauseKind() != OpenACCClauseKind::Gang &&
+        Clause.getClauseKind() != OpenACCClauseKind::Vector &&
+        Clause.getClauseKind() != OpenACCClauseKind::Worker &&
+        Clause.getClauseKind() != OpenACCClauseKind::Seq)
+      return false;
+    auto ProhibitedPred = llvm::IsaPred<OpenACCGangClause, OpenACCWorkerClause,
+                                        OpenACCVectorClause, OpenACCSeqClause>;
+
+    return CheckValidRoutineNewClauseHelper(ProhibitedPred, Clause);
+  }
+
+  // Bind should have similar rules on a routine as gang/worker/vector/seq,
+  // except there is no 'must have 1' rule, so we can get all the checking done
+  // here.
+  bool
+  CheckValidRoutineBindNewClause(SemaOpenACC::OpenACCParsedClause &Clause) {
+
+    if (Clause.getClauseKind() != OpenACCClauseKind::Bind)
+      return false;
+
+    auto HasBindPred = llvm::IsaPred<OpenACCBindClause>;
+    return CheckValidRoutineNewClauseHelper(HasBindPred, Clause);
+  }
+
+  // For 'tile' and 'collapse', only allow 1 per 'device_type'.
+  // Also applies to num_worker, num_gangs, and vector_length.
+  template <typename TheClauseTy>
+  bool DisallowSinceLastDeviceType(SemaOpenACC::OpenACCParsedClause &Clause) {
+    auto LastDeviceTypeItr =
+        std::find_if(ExistingClauses.rbegin(), ExistingClauses.rend(),
+                     llvm::IsaPred<OpenACCDeviceTypeClause>);
+
+    auto Last = std::find_if(ExistingClauses.rbegin(), LastDeviceTypeItr,
+                             llvm::IsaPred<TheClauseTy>);
+
+    if (Last == LastDeviceTypeItr)
+      return false;
+
+    SemaRef.Diag(Clause.getBeginLoc(),
+                 diag::err_acc_clause_since_last_device_type)
+        << Clause.getClauseKind() << Clause.getDirectiveKind()
+        << (LastDeviceTypeItr != ExistingClauses.rend());
+    SemaRef.Diag((*Last)->getBeginLoc(), diag::note_acc_previous_clause_here);
+
+    if (LastDeviceTypeItr != ExistingClauses.rend())
+      SemaRef.Diag((*LastDeviceTypeItr)->getBeginLoc(),
+                   diag::note_acc_previous_clause_here);
+
+    return true;
+  }
+
 public:
   SemaOpenACCClauseVisitor(SemaOpenACC &S,
                            ArrayRef<const OpenACCClause *> ExistingClauses)
       : SemaRef(S), Ctx(S.getASTContext()), ExistingClauses(ExistingClauses) {}
 
   OpenACCClause *Visit(SemaOpenACC::OpenACCParsedClause &Clause) {
+
+    if (SemaRef.DiagnoseAllowedOnceClauses(
+            Clause.getDirectiveKind(), Clause.getClauseKind(),
+            Clause.getBeginLoc(), ExistingClauses) ||
+        SemaRef.DiagnoseExclusiveClauses(Clause.getDirectiveKind(),
+                                         Clause.getClauseKind(),
+                                         Clause.getBeginLoc(), ExistingClauses))
+      return nullptr;
+    if (CheckValidRoutineGangWorkerVectorSeqNewClause(Clause) ||
+        CheckValidRoutineBindNewClause(Clause))
+      return nullptr;
+
     switch (Clause.getClauseKind()) {
+    case OpenACCClauseKind::Shortloop:
+      llvm_unreachable("Shortloop shouldn't be generated in clang");
+    case OpenACCClauseKind::Invalid:
+      return nullptr;
 #define VISIT_CLAUSE(CLAUSE_NAME)                                              \
   case OpenACCClauseKind::CLAUSE_NAME:                                         \
     return Visit##CLAUSE_NAME##Clause(Clause);
@@ -753,8 +393,6 @@ class SemaOpenACCClauseVisitor {
         << Clause.getClauseKind() << OpenACCClauseKind::CLAUSE_NAME;           \
   return Visit##CLAUSE_NAME##Clause(Clause);
 #include "clang/Basic/OpenACCClauses.def"
-    default:
-      return isNotImplemented();
     }
     llvm_unreachable("Invalid clause kind");
   }
@@ -771,13 +409,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDefaultClause(
   if (Clause.getDefaultClauseKind() == OpenACCDefaultClauseKind::Invalid)
     return nullptr;
 
-  // OpenACC 3.3, Section 2.5.4:
-  // At most one 'default' clause may appear, and it must have a value of
-  // either 'none' or 'present'.
-  // Second half of the sentence is diagnosed during parsing.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
-    return nullptr;
-
   return OpenACCDefaultClause::Create(
       Ctx, Clause.getDefaultClauseKind(), Clause.getBeginLoc(),
       Clause.getLParenLoc(), Clause.getEndLoc());
@@ -786,12 +417,7 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDefaultClause(
 OpenACCClause *SemaOpenACCClauseVisitor::VisitTileClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
 
-  // Duplicates here are not really sensible.  We could possible permit
-  // multiples if they all had the same value, but there isn't really a good
-  // reason to do so. Also, this simplifies the suppression of duplicates, in
-  // that we know if we 'find' one after instantiation, that it is the same
-  // clause, which simplifies instantiation/checking/etc.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+  if (DisallowSinceLastDeviceType<OpenACCTileClause>(Clause))
     return nullptr;
 
   llvm::SmallVector<Expr *> NewSizeExprs;
@@ -813,16 +439,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitTileClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitIfClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // There is no prose in the standard that says duplicates aren't allowed,
-  // but this diagnostic is present in other compilers, as well as makes
-  // sense. Prose DOES exist for 'data' and 'host_data', 'set', 'enter data' and
-  // 'exit data' both don't, but other implmementations do this.  OpenACC issue
-  // 519 filed for the latter two. Prose also exists for 'update'.
-  // GCC allows this on init/shutdown, presumably for good reason, so we do too.
-  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Init &&
-      Clause.getDirectiveKind() != OpenACCDirectiveKind::Shutdown &&
-      checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
-    return nullptr;
 
   // The parser has ensured that we have a proper condition expr, so there
   // isn't really much to do here.
@@ -845,11 +461,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitIfClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitSelfClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // There is no prose in the standard that says duplicates aren't allowed,
-  // but this diagnostic is present in other compilers, as well as makes
-  // sense.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
-    return nullptr;
 
   // If the 'if' clause is true, it makes the 'self' clause have no effect,
   // diagnose that here.  This only applies on compute/combined constructs.
@@ -871,10 +482,8 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitSelfClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitNumGangsClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // There is no prose in the standard that says duplicates aren't allowed,
-  // but this diagnostic is present in other compilers, as well as makes
-  // sense.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+
+  if (DisallowSinceLastDeviceType<OpenACCNumGangsClause>(Clause))
     return nullptr;
 
   // num_gangs requires at least 1 int expr in all forms.  Diagnose here, but
@@ -966,10 +575,8 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitNumGangsClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitNumWorkersClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // There is no prose in the standard that says duplicates aren't allowed,
-  // but this diagnostic is present in other compilers, as well as makes
-  // sense.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+
+  if (DisallowSinceLastDeviceType<OpenACCNumWorkersClause>(Clause))
     return nullptr;
 
   // OpenACC 3.3 Section 2.9.2:
@@ -1000,10 +607,8 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitNumWorkersClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitVectorLengthClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // There is no prose in the standard that says duplicates aren't allowed,
-  // but this diagnostic is present in other compilers, as well as makes
-  // sense.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+
+  if (DisallowSinceLastDeviceType<OpenACCVectorLengthClause>(Clause))
     return nullptr;
 
   // OpenACC 3.3 Section 2.9.4:
@@ -1034,12 +639,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitVectorLengthClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitAsyncClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // There is no prose in the standard that says duplicates aren't allowed,
-  // but this diagnostic is present in other compilers, as well as makes
-  // sense.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
-    return nullptr;
-
   assert(Clause.getNumIntExprs() < 2 &&
          "Invalid number of expressions for Async");
   return OpenACCAsyncClause::Create(
@@ -1050,17 +649,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitAsyncClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceNumClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions only properly implemented on certain constructs, so skip/treat
-  // as unimplemented in those cases.
-  if (!isDirectiveKindImplemented(Clause.getDirectiveKind()))
-    return isNotImplemented();
-
-  // OpenACC 3.3 2.14.3: Two instances of the same clause may not appear on the
-  // same directive.
-  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Set &&
-      checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
-    return nullptr;
-
   assert(Clause.getNumIntExprs() == 1 &&
          "Invalid number of expressions for device_num");
   return OpenACCDeviceNumClause::Create(
@@ -1070,11 +658,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceNumClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitDefaultAsyncClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // OpenACC 3.3 2.14.3: Two instances of the same clause may not appear on the
-  // same directive.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
-    return nullptr;
-
   assert(Clause.getNumIntExprs() == 1 &&
          "Invalid number of expressions for default_async");
   return OpenACCDefaultAsyncClause::Create(
@@ -1332,11 +915,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitWaitClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // OpenACC 3.3 2.14.3: Two instances of the same clause may not appear on the
-  // same directive.
-  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Set &&
-      checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
-    return nullptr;
 
   // Based on discussions, having more than 1 'architecture' on a 'set' is
   // nonsensical, so we're going to fix the standard to reflect this.  Implement
@@ -1397,18 +975,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitAutoClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // OpenACC 3.3 2.9:
-  // Only one of the seq, independent, and auto clauses may appear.
-  const auto *Itr =
-      llvm::find_if(ExistingClauses,
-                    llvm::IsaPred<OpenACCAutoClause, OpenACCIndependentClause,
-                                  OpenACCSeqClause>);
-  if (Itr != ExistingClauses.end()) {
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
-        << Clause.getClauseKind() << Clause.getDirectiveKind();
-    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    return nullptr;
-  }
 
   return OpenACCAutoClause::Create(Ctx, Clause.getBeginLoc(),
                                    Clause.getEndLoc());
@@ -1422,17 +988,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitNoHostClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitIndependentClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // OpenACC 3.3 2.9:
-  // Only one of the seq, independent, and auto clauses may appear.
-  const auto *Itr = llvm::find_if(
-      ExistingClauses, llvm::IsaPred<OpenACCIndependentClause,
-                                     OpenACCAutoClause, OpenACCSeqClause>);
-  if (Itr != ExistingClauses.end()) {
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
-        << Clause.getClauseKind() << Clause.getDirectiveKind();
-    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    return nullptr;
-  }
 
   return OpenACCIndependentClause::Create(Ctx, Clause.getBeginLoc(),
                                           Clause.getEndLoc());
@@ -1805,6 +1360,7 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitWorkerClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitGangClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
+
   if (DiagGangWorkerVectorSeqConflict(Clause))
     return nullptr;
 
@@ -1944,45 +1500,23 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitIfPresentClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitSeqClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-
-  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Routine) {
-    // OpenACC 3.3 2.9:
-    // Only one of the seq, independent, and auto clauses may appear.
-    const auto *Itr =
-        llvm::find_if(ExistingClauses,
-                      llvm::IsaPred<OpenACCAutoClause, OpenACCIndependentClause,
-                                    OpenACCSeqClause>);
+  // OpenACC 3.3 2.9:
+  //  A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
+  //  appears.
+  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Loop ||
+      isOpenACCCombinedDirectiveKind(Clause.getDirectiveKind())) {
+    const auto *Itr = llvm::find_if(
+        ExistingClauses, llvm::IsaPred<OpenACCGangClause, OpenACCVectorClause,
+                                       OpenACCWorkerClause>);
     if (Itr != ExistingClauses.end()) {
-      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_loop_spec_conflict)
-          << Clause.getClauseKind() << Clause.getDirectiveKind();
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
+          << Clause.getClauseKind() << (*Itr)->getClauseKind()
+          << Clause.getDirectiveKind();
       SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
       return nullptr;
     }
   }
 
-  // OpenACC 3.3 2.9:
-  // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
-  // appears.
-  // -also-
-  // OpenACC3.3 2.15: (routine)
-  // Exactly one of the 'gang', 'worker', 'vector' or 'seq' clauses must appear.
-  const auto *Itr =
-      Clause.getDirectiveKind() == OpenACCDirectiveKind::Routine
-          ? llvm::find_if(ExistingClauses,
-                          llvm::IsaPred<OpenACCGangClause, OpenACCWorkerClause,
-                                        OpenACCVectorClause, OpenACCSeqClause>)
-          : llvm::find_if(ExistingClauses,
-                          llvm::IsaPred<OpenACCGangClause, OpenACCWorkerClause,
-                                        OpenACCVectorClause>);
-
-  if (Itr != ExistingClauses.end()) {
-    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
-        << Clause.getClauseKind() << (*Itr)->getClauseKind()
-        << Clause.getDirectiveKind();
-    SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
-    return nullptr;
-  }
-
   return OpenACCSeqClause::Create(Ctx, Clause.getBeginLoc(),
                                   Clause.getEndLoc());
 }
@@ -2083,12 +1617,8 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitReductionClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitCollapseClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Duplicates here are not really sensible.  We could possible permit
-  // multiples if they all had the same value, but there isn't really a good
-  // reason to do so. Also, this simplifies the suppression of duplicates, in
-  // that we know if we 'find' one after instantiation, that it is the same
-  // clause, which simplifies instantiation/checking/etc.
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+
+  if (DisallowSinceLastDeviceType<OpenACCCollapseClause>(Clause))
     return nullptr;
 
   ExprResult LoopCount = SemaRef.CheckCollapseLoopCount(Clause.getLoopCount());
@@ -2103,8 +1633,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitCollapseClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitBindClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
-    return nullptr;
 
   if (std::holds_alternative<StringLiteral *>(Clause.getBindDetails()))
     return OpenACCBindClause::Create(
@@ -2182,13 +1710,16 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
   if (Clause.getClauseKind() == OpenACCClauseKind::Invalid)
     return nullptr;
 
-  // Diagnose that we don't support this clause on this directive.
-  if (!doesClauseApplyToDirective(Clause.getDirectiveKind(),
-                                  Clause.getClauseKind())) {
-    Diag(Clause.getBeginLoc(), diag::err_acc_clause_appertainment)
-        << Clause.getDirectiveKind() << Clause.getClauseKind();
+  if (DiagnoseAllowedClauses(Clause.getDirectiveKind(), Clause.getClauseKind(),
+                             Clause.getBeginLoc()))
     return nullptr;
-  }
+  //// Diagnose that we don't support this clause on this directive.
+  // if (!doesClauseApplyToDirective(Clause.getDirectiveKind(),
+  //                                 Clause.getClauseKind())) {
+  //   Diag(Clause.getBeginLoc(), diag::err_acc_clause_appertainment)
+  //       << Clause.getDirectiveKind() << Clause.getClauseKind();
+  //   return nullptr;
+  // }
 
   if (const auto *DevTypeClause = llvm::find_if(
           ExistingClauses, llvm::IsaPred<OpenACCDeviceTypeClause>);
diff --git a/clang/lib/Sema/SemaOpenACCClauseAppertainment.cpp b/clang/lib/Sema/SemaOpenACCClauseAppertainment.cpp
new file mode 100644
index 0000000000000..6069c77d01676
--- /dev/null
+++ b/clang/lib/Sema/SemaOpenACCClauseAppertainment.cpp
@@ -0,0 +1,229 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file implements conversions from the ACC.td from the backend to
+/// determine appertainment, required/etc.
+///
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/DiagnosticSema.h"
+#include "clang/Sema/SemaOpenACC.h"
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/bit.h"
+
+using namespace clang;
+
+namespace {
+// Implements a simple 'enum-set' which stores enum values in a single 64 bit
+// value. Flang has `EnumSet` which is pretty sizable/has a lot of dependencies,
+// so likely not worth bringing in for this use.
+class AccClauseSet {
+  // We're just using a uint64_t as our underlying rep, so if this size ever
+  // gets bigger than 64, we probably need a pair of uint64_ts.
+  static_assert(static_cast<unsigned>(OpenACCClauseKind::Invalid) < 64);
+  uint64_t Data;
+
+  void setBit(OpenACCClauseKind C) {
+    Data |= static_cast<uint64_t>(1) << static_cast<uint64_t>(C);
+  }
+
+public:
+  constexpr AccClauseSet(
+      const std::initializer_list<OpenACCClauseKind> &Clauses)
+      : Data(0) {
+    for (OpenACCClauseKind C : Clauses)
+      setBit(C);
+  }
+
+  constexpr bool isSet(OpenACCClauseKind C) const {
+    return ((Data >> static_cast<uint64_t>(C)) & 1) != 0;
+  }
+
+  void clearBit(OpenACCClauseKind C) {
+    Data &= ~(static_cast<uint64_t>(1) << static_cast<uint64_t>(C));
+  }
+
+  constexpr bool isEmpty() const { return Data == 0; }
+
+  unsigned popcount() const { return llvm::popcount<uint64_t>(Data); }
+};
+
+struct LLVMClauseLists {
+  AccClauseSet Allowed;
+  AccClauseSet AllowedOnce;
+  AccClauseSet AllowedExclusive;
+  AccClauseSet Required;
+};
+struct LLVMDirectiveClauseRelationships {
+  OpenACCDirectiveKind DirKind;
+  LLVMClauseLists Lists;
+};
+
+} // namespace
+
+// This introduces these in a llvm::acc namespace, so make sure this stays in
+// the global namespace.
+#define GEN_CLANG_DIRECTIVE_CLAUSE_SETS
+#include "llvm/Frontend/OpenACC/ACC.inc"
+
+namespace {
+LLVMDirectiveClauseRelationships Relations[] =
+#define GEN_CLANG_DIRECTIVE_CLAUSE_MAP
+#include "llvm/Frontend/OpenACC/ACC.inc"
+    ;
+
+const LLVMClauseLists &getListsForDirective(OpenACCDirectiveKind DK) {
+
+  auto Res = llvm::find_if(Relations,
+                           [=](const LLVMDirectiveClauseRelationships &Rel) {
+                             return Rel.DirKind == DK;
+                           });
+  assert(Res != std::end(Relations) && "Unknown directive kind?");
+
+  return Res->Lists;
+}
+
+std::string getListOfClauses(AccClauseSet Set) {
+  // We could probably come up with a better way to do this smuggling, but this
+  // is good enough for now.
+  std::string Output;
+  llvm::raw_string_ostream OS{Output};
+
+  for (unsigned I = 0; I < static_cast<unsigned>(OpenACCClauseKind::Invalid);
+       ++I) {
+    OpenACCClauseKind CurClause = static_cast<OpenACCClauseKind>(I);
+    if (!Set.isSet(CurClause))
+      continue;
+
+    OS << '\'' << CurClause << '\'';
+
+    Set.clearBit(CurClause);
+
+    if (Set.isEmpty()) {
+      OS.flush();
+      return OS.str();
+    }
+
+    OS << ", ";
+
+    if (Set.popcount() == 1)
+      OS << "or ";
+  }
+  OS.flush();
+  return OS.str();
+}
+
+OpenACCClauseKind dealiasClauseKind(OpenACCClauseKind CK) {
+  switch (CK) {
+  default:
+    return CK;
+#define VISIT_CLAUSE(NAME)
+#define CLAUSE_ALIAS(ALIAS, NAME, DEPRECATED)                                  \
+  case OpenACCClauseKind::ALIAS:                                               \
+    return OpenACCClauseKind::NAME;
+#include "clang/Basic/OpenACCClauses.def"
+  }
+
+  return CK;
+}
+} // namespace
+
+// Diagnoses if `Clauses` list doesn't have at least one of the required
+// clauses.
+bool SemaOpenACC::DiagnoseRequiredClauses(
+    OpenACCDirectiveKind DK, SourceLocation DirectiveLoc,
+    ArrayRef<const OpenACCClause *> Clauses) {
+  if (DK == OpenACCDirectiveKind::Invalid)
+    return false;
+
+  const LLVMClauseLists &Lists = getListsForDirective(DK);
+
+  if (Lists.Required.isEmpty())
+    return false;
+
+  for (auto *C : Clauses) {
+    if (Lists.Required.isSet(dealiasClauseKind(C->getClauseKind())))
+      return false;
+  }
+
+  return Diag(DirectiveLoc, diag::err_acc_construct_one_clause_of)
+         << DK << getListOfClauses(Lists.Required);
+  return true;
+}
+
+// Diagnoses a 'CK' on a 'DK' present more than once in a clause-list when it
+// isn't allowed.
+bool SemaOpenACC::DiagnoseAllowedOnceClauses(
+    OpenACCDirectiveKind DK, OpenACCClauseKind CK, SourceLocation ClauseLoc,
+    ArrayRef<const OpenACCClause *> Clauses) {
+  if (DK == OpenACCDirectiveKind::Invalid || CK == OpenACCClauseKind::Invalid)
+    return false;
+
+  OpenACCClauseKind Dealiased = dealiasClauseKind(CK);
+
+  const LLVMClauseLists &Lists = getListsForDirective(DK);
+  if (!Lists.AllowedOnce.isSet(CK))
+    return false;
+
+  auto Res = llvm::find_if(Clauses, [=](const OpenACCClause *C) {
+    return dealiasClauseKind(C->getClauseKind()) == Dealiased;
+  });
+
+  if (Res == Clauses.end())
+    return false;
+
+  Diag(ClauseLoc, diag::err_acc_duplicate_clause_disallowed) << DK << CK;
+  Diag((*Res)->getBeginLoc(), diag::note_acc_previous_clause_here);
+  return true;
+}
+
+// Diagnoses a 'CK' on a 'DK' being added that isn't allowed to, because another
+// clause in 'Clauses' already exists.
+bool SemaOpenACC::DiagnoseExclusiveClauses(
+    OpenACCDirectiveKind DK, OpenACCClauseKind CK, SourceLocation ClauseLoc,
+    ArrayRef<const OpenACCClause *> Clauses) {
+  if (DK == OpenACCDirectiveKind::Invalid || CK == OpenACCClauseKind::Invalid)
+    return false;
+
+  const LLVMClauseLists &Lists = getListsForDirective(DK);
+  OpenACCClauseKind Dealiased = dealiasClauseKind(CK);
+
+  // If this isn't on the list, this is fine.
+  if (!Lists.AllowedExclusive.isSet(Dealiased))
+    return false;
+
+  for (const OpenACCClause *C : Clauses) {
+    if (Lists.AllowedExclusive.isSet(dealiasClauseKind(C->getClauseKind()))) {
+      Diag(ClauseLoc, diag::err_acc_clause_cannot_combine)
+          << CK << C->getClauseKind() << DK;
+      Diag(C->getBeginLoc(), diag::note_acc_previous_clause_here);
+
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// Diagnoses if 'CK' is not allowed on a directive of 'DK'.
+bool SemaOpenACC::DiagnoseAllowedClauses(OpenACCDirectiveKind DK,
+                                         OpenACCClauseKind CK,
+                                         SourceLocation ClauseLoc) {
+  if (DK == OpenACCDirectiveKind::Invalid || CK == OpenACCClauseKind::Invalid)
+    return false;
+  const LLVMClauseLists &Lists = getListsForDirective(DK);
+  OpenACCClauseKind Dealiased = dealiasClauseKind(CK);
+
+  if (!Lists.Allowed.isSet(Dealiased) && !Lists.AllowedOnce.isSet(Dealiased) &&
+      !Lists.AllowedExclusive.isSet(Dealiased) &&
+      !Lists.Required.isSet(Dealiased))
+    return Diag(ClauseLoc, diag::err_acc_clause_appertainment) << DK << CK;
+
+  return false;
+}
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 2469991bf2ce8..b9a09ade37c12 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -12355,9 +12355,6 @@ void OpenACCClauseTransform<Derived>::VisitCollapseClause(
   NewLoopCount =
       Self.getSema().OpenACC().CheckCollapseLoopCount(NewLoopCount.get());
 
-  if (!NewLoopCount.isUsable())
-    return;
-
   ParsedClause.setCollapseDetails(C.hasForce(), NewLoopCount.get());
   NewClause = OpenACCCollapseClause::Create(
       Self.getSema().getASTContext(), ParsedClause.getBeginLoc(),
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index d683b7930caf5..70b54b7296882 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12906,6 +12906,7 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     return OpenACCBindClause::Create(getContext(), BeginLoc, LParenLoc,
                                      readIdentifier(), EndLoc);
   }
+  case OpenACCClauseKind::Shortloop:
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
   }
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index e969214e633f3..ffbe2c4e4d0f1 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8871,6 +8871,7 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     return;
   }
   case OpenACCClauseKind::Invalid:
+  case OpenACCClauseKind::Shortloop:
     llvm_unreachable("Clause serialization not yet implemented");
   }
   llvm_unreachable("Invalid Clause Kind");
diff --git a/clang/test/AST/ast-print-openacc-set-construct.cpp b/clang/test/AST/ast-print-openacc-set-construct.cpp
index 869a3feb2dd93..1ad6adb17e360 100644
--- a/clang/test/AST/ast-print-openacc-set-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-set-construct.cpp
@@ -15,8 +15,8 @@ void uses() {
 #pragma acc set if (Int == 5) device_type(multicore) device_num(Int)
 // CHECK: #pragma acc set default_async(Int)
 #pragma acc set default_async(Int)
-// CHECK: #pragma acc set if(Int == 5)
-#pragma acc set if (Int == 5)
+// CHECK: #pragma acc set if(Int == 5) device_type(multicore)
+#pragma acc set if (Int == 5) device_type(multicore)
 // CHECK: #pragma acc set device_type(radeon)
 #pragma acc set device_type(radeon)
 // CHECK: #pragma acc set device_num(Int)
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index ef0bccf390297..716d9cffab3f6 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -4,21 +4,21 @@
 
 void func() {
 
-  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
 #pragma acc exit data finalize
 
-  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
 #pragma acc exit data finalize finalize
 
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{invalid OpenACC clause 'invalid'}}
 #pragma acc exit data finalize invalid
 
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{invalid OpenACC clause 'invalid'}}
 #pragma acc exit data finalize invalid invalid finalize
 
-  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
 #pragma acc exit data wait finalize
 
   // expected-error@+1{{OpenACC 'host_data' construct must have at least one 'use_device' clause}}
@@ -27,41 +27,41 @@ void func() {
   // expected-error@+1{{OpenACC 'host_data' construct must have at least one 'use_device' clause}}
 #pragma acc host_data if_present, if_present
 
-  // expected-error@+4{{OpenACC clause 'independent' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+4{{OpenACC clause 'independent' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+3{{previous clause is here}}
-  // expected-error@+2{{OpenACC clause 'auto' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq independent auto
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+4{{OpenACC clause 'independent' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+4{{OpenACC clause 'independent' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+3{{previous clause is here}}
-  // expected-error@+2{{OpenACC clause 'auto' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq, independent auto
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+4{{OpenACC clause 'independent' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+4{{OpenACC clause 'independent' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+3{{previous clause is here}}
-  // expected-error@+2{{OpenACC clause 'auto' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq independent, auto
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+3{{OpenACC clause 'independent' on 'kernels loop' construct conflicts with previous data dependence clause}}
-  // expected-error@+2{{OpenACC clause 'auto' on 'kernels loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+3{{OpenACC clause 'independent' may not appear on the same construct as a 'seq' clause on a 'kernels loop' construct}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'seq' clause on a 'kernels loop' construct}}
   // expected-note@+1 2{{previous clause is here}}
 #pragma acc kernels loop seq independent auto
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+3{{OpenACC clause 'independent' on 'serial loop' construct conflicts with previous data dependence clause}}
-  // expected-error@+2{{OpenACC clause 'auto' on 'serial loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+3{{OpenACC clause 'independent' may not appear on the same construct as a 'seq' clause on a 'serial loop' construct}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'seq' clause on a 'serial loop' construct}}
   // expected-note@+1 2{{previous clause is here}}
 #pragma acc serial loop seq, independent auto
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+3{{OpenACC clause 'independent' on 'parallel loop' construct conflicts with previous data dependence clause}}
-  // expected-error@+2{{OpenACC clause 'auto' on 'parallel loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+3{{OpenACC clause 'independent' may not appear on the same construct as a 'seq' clause on a 'parallel loop' construct}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'seq' clause on a 'parallel loop' construct}}
   // expected-note@+1 2{{previous clause is here}}
 #pragma acc parallel loop seq independent, auto
   for(int i = 0; i < 5;++i) {}
@@ -860,20 +860,20 @@ void IntExprParsing() {
 #pragma acc init device_num(returns_int())
 
   // expected-error@+2{{expected '('}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set default_async
 
   // expected-error@+2{{expected expression}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set default_async()
 
   // expected-error@+2{{use of undeclared identifier 'invalid'}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set default_async(invalid)
 
   // expected-error@+3{{expected ')'}}
   // expected-note@+2{{to match this '('}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set default_async(5, 4)
 
 #pragma acc set default_async(5)
diff --git a/clang/test/ParserOpenACC/parse-constructs.c b/clang/test/ParserOpenACC/parse-constructs.c
index 71e47abd3a825..0365e3ab83d8d 100644
--- a/clang/test/ParserOpenACC/parse-constructs.c
+++ b/clang/test/ParserOpenACC/parse-constructs.c
@@ -54,15 +54,15 @@ void func() {
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
 #pragma acc kernels clause list
   for(;;){}
-    // expected-error@+2{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+2{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
 #pragma acc data clause list
   for(;;){}
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
 #pragma acc enter data clause list
   for(;;){}
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
 #pragma acc exit data clause list
   for(;;){}
@@ -141,10 +141,10 @@ void func() {
 #pragma acc shutdown clause list
   for(;;){}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set clause list
   for(;;){}
-  // expected-error@+2{{OpenACC 'update' construct must have at least one 'self', 'host' or 'device' clause}}
+  // expected-error@+2{{OpenACC 'update' construct must have at least one 'device', 'host', or 'self' clause}}
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
 #pragma acc update clause list
   for(;;){}
@@ -153,7 +153,7 @@ void func() {
 #pragma acc routine seq
 void routine_func();
 // expected-error@+2{{invalid OpenACC clause 'clause'}}
-// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'worker', 'vector' or 'seq' clause}}
+// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'seq', 'vector', or 'worker' clause}}
 #pragma acc routine clause list
 void routine_func();
 
@@ -161,12 +161,12 @@ void routine_func();
 #pragma acc routine (func_name) seq
 // expected-error@+3{{use of undeclared identifier 'func_name'}}
 // expected-error@+2{{invalid OpenACC clause 'clause'}}
-// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'worker', 'vector' or 'seq' clause}}
+// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'seq', 'vector', or 'worker' clause}}
 #pragma acc routine (func_name) clause list
 
 #pragma acc routine (routine_func) seq
 // expected-error@+2{{invalid OpenACC clause 'clause'}}
-// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'worker', 'vector' or 'seq' clause}}
+// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'seq', 'vector', or 'worker' clause}}
 #pragma acc routine (routine_func) clause list
 
 // expected-error@+2{{expected ')'}}
diff --git a/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c b/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
index dbba86a1ffc61..5c64c621b0e7e 100644
--- a/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
+++ b/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
@@ -8,27 +8,27 @@ void uses() {
 #pragma acc parallel loop independent
   for(unsigned i = 0; i < 5; ++i);
 
-  // expected-error@+2{{OpenACC clause 'seq' on 'parallel loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'auto' clause on a 'parallel loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop auto seq
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'independent' on 'parallel loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'independent' may not appear on the same construct as a 'auto' clause on a 'parallel loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop auto independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'auto' on 'parallel loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'seq' clause on a 'parallel loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop seq auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'independent' on 'parallel loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'independent' may not appear on the same construct as a 'seq' clause on a 'parallel loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop seq independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'auto' on 'parallel loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'independent' clause on a 'parallel loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop independent auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'seq' on 'parallel loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'independent' clause on a 'parallel loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc parallel loop independent seq
   for(unsigned i = 0; i < 5; ++i);
diff --git a/clang/test/SemaOpenACC/combined-construct-collapse-clause.cpp b/clang/test/SemaOpenACC/combined-construct-collapse-clause.cpp
index f2be6622007c3..3f8fb4203d0e5 100644
--- a/clang/test/SemaOpenACC/combined-construct-collapse-clause.cpp
+++ b/clang/test/SemaOpenACC/combined-construct-collapse-clause.cpp
@@ -214,7 +214,7 @@ void no_other_directives() {
 #pragma acc serial loop collapse(2)
   for(unsigned i = 0; i < 5; ++i) {
     for(unsigned j = 0; j < 5; ++j) {
-    // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data
       ;
     }
@@ -222,7 +222,7 @@ void no_other_directives() {
   // expected-note@+1{{active 'collapse' clause defined here}}
 #pragma acc kernels loop collapse(2)
   for(unsigned i = 0; i < 5; ++i) {
-    // expected-error@+2{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+2{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
     // expected-error@+1{{OpenACC 'data' construct cannot appear in intervening code of a 'kernels loop' with a 'collapse' clause}}
 #pragma acc data
     for(unsigned j = 0; j < 5; ++j) {
diff --git a/clang/test/SemaOpenACC/combined-construct-if-clause.c b/clang/test/SemaOpenACC/combined-construct-if-clause.c
index 5580fdfe22b7b..d5e42d6105963 100644
--- a/clang/test/SemaOpenACC/combined-construct-if-clause.c
+++ b/clang/test/SemaOpenACC/combined-construct-if-clause.c
@@ -43,7 +43,7 @@ void BoolExpr(int *I, float *F) {
 #pragma acc kernels loop if (*I < *F)
   for (unsigned i = 0; i < 5; ++i);
 
-  // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+  // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data if (*I < *F)
   for (unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop if (*I < *F)
diff --git a/clang/test/SemaOpenACC/combined-construct-num_gangs-clause.c b/clang/test/SemaOpenACC/combined-construct-num_gangs-clause.c
index bd035bd4a51a2..cd06f5e05ee69 100644
--- a/clang/test/SemaOpenACC/combined-construct-num_gangs-clause.c
+++ b/clang/test/SemaOpenACC/combined-construct-num_gangs-clause.c
@@ -13,6 +13,31 @@ void Test() {
 #pragma acc parallel loop num_gangs(1)
   for(int i = 5; i < 10;++i);
 
+  // expected-error@+2{{OpenACC 'num_gangs' clause cannot appear more than once on a 'kernels loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_gangs(1) num_gangs(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{OpenACC 'num_gangs' clause cannot appear more than once on a 'parallel loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop num_gangs(1) num_gangs(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+3{{OpenACC 'num_gangs' clause cannot appear more than once in a 'device_type' region on a 'kernels loop' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_gangs(1) device_type(*) num_gangs(1) num_gangs(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+3{{OpenACC 'num_gangs' clause cannot appear more than once in a 'device_type' region on a 'parallel loop' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop device_type(*) num_gangs(1) num_gangs(2)
+  for(int i = 5; i < 10;++i);
+
+#pragma acc parallel loop num_gangs(1) device_type(*) num_gangs(2)
+  for(int i = 5; i < 10;++i);
+
   // expected-error@+1{{OpenACC clause 'num_gangs' requires expression of integer type}}
 #pragma acc parallel loop num_gangs(getF())
   for(int i = 5; i < 10;++i);
@@ -25,16 +50,6 @@ void Test() {
 #pragma acc parallel loop num_gangs()
   for(int i = 5; i < 10;++i);
 
-  // expected-error@+2{{OpenACC 'num_gangs' clause cannot appear more than once on a 'kernels loop' directive}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels loop num_gangs(1) num_gangs(2)
-  for(int i = 5; i < 10;++i);
-
-  // expected-error@+2{{OpenACC 'num_gangs' clause cannot appear more than once on a 'parallel loop' directive}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc parallel loop num_gangs(1) num_gangs(2)
-  for(int i = 5; i < 10;++i);
-
   // expected-error@+1{{too many integer expression arguments provided to OpenACC 'num_gangs' clause: 'kernels loop' directive expects maximum of 1, 2 were provided}}
 #pragma acc kernels loop num_gangs(1, getS())
   for(int i = 5; i < 10;++i);
diff --git a/clang/test/SemaOpenACC/combined-construct-num_workers-clause.c b/clang/test/SemaOpenACC/combined-construct-num_workers-clause.c
index a5891f071bb03..c3b648e438081 100644
--- a/clang/test/SemaOpenACC/combined-construct-num_workers-clause.c
+++ b/clang/test/SemaOpenACC/combined-construct-num_workers-clause.c
@@ -13,6 +13,31 @@ void Test() {
 #pragma acc parallel loop num_workers(1)
   for(int i = 5; i < 10;++i);
 
+  // expected-error@+2{{OpenACC 'num_workers' clause cannot appear more than once on a 'kernels loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_workers(1) num_workers(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{OpenACC 'num_workers' clause cannot appear more than once on a 'parallel loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop num_workers(1) num_workers(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+3{{OpenACC 'num_workers' clause cannot appear more than once in a 'device_type' region on a 'kernels loop' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop num_workers(1) device_type(*) num_workers(1) num_workers(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+3{{OpenACC 'num_workers' clause cannot appear more than once in a 'device_type' region on a 'parallel loop' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop device_type(*) num_workers(1) num_workers(2)
+  for(int i = 5; i < 10;++i);
+
+#pragma acc parallel loop num_workers(1) device_type(*) num_workers(2)
+  for(int i = 5; i < 10;++i);
+
   // expected-error@+1{{OpenACC clause 'num_workers' requires expression of integer type}}
 #pragma acc parallel loop num_workers(getF())
   for(int i = 5; i < 10;++i);
diff --git a/clang/test/SemaOpenACC/combined-construct-tile-clause.cpp b/clang/test/SemaOpenACC/combined-construct-tile-clause.cpp
index 00c551e163666..deab634c39886 100644
--- a/clang/test/SemaOpenACC/combined-construct-tile-clause.cpp
+++ b/clang/test/SemaOpenACC/combined-construct-tile-clause.cpp
@@ -143,13 +143,6 @@ void only_for_loops() {
     do{}while(true);
 }
 
-void only_one_on_loop() {
-  // expected-error@+2{{OpenACC 'tile' clause cannot appear more than once on a 'serial loop' directive}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc serial loop tile(1) tile(1)
-  for(int i = 0; i < 5; ++i);
-}
-
 template<unsigned Val>
 void depth_too_high_templ() {
   // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
diff --git a/clang/test/SemaOpenACC/combined-construct-vector_length-clause.c b/clang/test/SemaOpenACC/combined-construct-vector_length-clause.c
index 8b6dedd9b83ba..72ab0126b1113 100644
--- a/clang/test/SemaOpenACC/combined-construct-vector_length-clause.c
+++ b/clang/test/SemaOpenACC/combined-construct-vector_length-clause.c
@@ -13,6 +13,31 @@ void Test() {
 #pragma acc parallel loop vector_length(1)
   for(int i = 5; i < 10;++i);
 
+  // expected-error@+2{{OpenACC 'vector_length' clause cannot appear more than once on a 'kernels loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop vector_length(1) vector_length(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+2{{OpenACC 'vector_length' clause cannot appear more than once on a 'parallel loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop vector_length(1) vector_length(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+3{{OpenACC 'vector_length' clause cannot appear more than once in a 'device_type' region on a 'kernels loop' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels loop vector_length(1) device_type(*) vector_length(1) vector_length(2)
+  for(int i = 5; i < 10;++i);
+
+  // expected-error@+3{{OpenACC 'vector_length' clause cannot appear more than once in a 'device_type' region on a 'parallel loop' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel loop device_type(*) vector_length(1) vector_length(2)
+  for(int i = 5; i < 10;++i);
+
+#pragma acc parallel loop vector_length(1) device_type(*) vector_length(2)
+  for(int i = 5; i < 10;++i);
+
   // expected-error@+1{{OpenACC clause 'vector_length' requires expression of integer type}}
 #pragma acc parallel loop vector_length(getF())
   for(int i = 5; i < 10;++i);
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
index 16caa7205721d..2ff6369383714 100644
--- a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
@@ -34,10 +34,10 @@ void uses() {
 #pragma acc kernels dtype(MACRO)
   while(1);
 
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{OpenACC 'device_type' clause is not valid on 'enter data' directive}}
 #pragma acc enter data device_type(I)
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{OpenACC 'dtype' clause is not valid on 'enter data' directive}}
 #pragma acc enter data dtype(I)
 
diff --git a/clang/test/SemaOpenACC/compute-construct-if-clause.c b/clang/test/SemaOpenACC/compute-construct-if-clause.c
index c0ea88f06284d..0064303e1e217 100644
--- a/clang/test/SemaOpenACC/compute-construct-if-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-if-clause.c
@@ -43,7 +43,7 @@ void BoolExpr(int *I, float *F) {
 #pragma acc kernels if (*I < *F)
   while(0);
 
-  // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+  // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data if (*I < *F)
   while(0);
 #pragma acc parallel loop if (*I < *F)
diff --git a/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.c b/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.c
index 40b3b652d21da..9651514ca0fff 100644
--- a/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.c
@@ -22,6 +22,21 @@ void Test() {
 #pragma acc parallel num_gangs(1) num_gangs(2)
   while(1);
 
+  // expected-error@+3{{OpenACC 'num_gangs' clause cannot appear more than once in a 'device_type' region on a 'kernels' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_gangs(1) device_type(*) num_gangs(1) num_gangs(2)
+  while(1);
+
+  // expected-error@+3{{OpenACC 'num_gangs' clause cannot appear more than once in a 'device_type' region on a 'parallel' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel device_type(*) num_gangs(1) num_gangs(2)
+  while(1);
+
+#pragma acc parallel num_gangs(1) device_type(*) num_gangs(2)
+  while(1);
+
   // expected-error@+1{{too many integer expression arguments provided to OpenACC 'num_gangs' clause: 'kernels' directive expects maximum of 1, 2 were provided}}
 #pragma acc kernels num_gangs(1, getS())
   while(1);
diff --git a/clang/test/SemaOpenACC/compute-construct-num_workers-clause.c b/clang/test/SemaOpenACC/compute-construct-num_workers-clause.c
index a78e2c4feadbb..23a73f0bf49bd 100644
--- a/clang/test/SemaOpenACC/compute-construct-num_workers-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-num_workers-clause.c
@@ -8,6 +8,31 @@ void Test() {
 #pragma acc kernels num_workers(1)
   while(1);
 
+  // expected-error@+2{{OpenACC 'num_workers' clause cannot appear more than once on a 'kernels' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_workers(1) num_workers(2)
+  while(1);
+
+  // expected-error@+2{{OpenACC 'num_workers' clause cannot appear more than once on a 'parallel' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel num_workers(1) num_workers(2)
+  while(1);
+
+  // expected-error@+3{{OpenACC 'num_workers' clause cannot appear more than once in a 'device_type' region on a 'kernels' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_workers(1) device_type(*) num_workers(1) num_workers(2)
+  while(1);
+
+  // expected-error@+3{{OpenACC 'num_workers' clause cannot appear more than once in a 'device_type' region on a 'parallel' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel device_type(*) num_workers(1) num_workers(2)
+  while(1);
+
+#pragma acc parallel num_workers(1) device_type(*) num_workers(2)
+  while(1);
+
   // expected-error@+1{{OpenACC 'num_workers' clause is not valid on 'serial' directive}}
 #pragma acc serial num_workers(1)
   while(1);
diff --git a/clang/test/SemaOpenACC/compute-construct-vector_length-clause.c b/clang/test/SemaOpenACC/compute-construct-vector_length-clause.c
index c9083df0df722..c79fc627252f1 100644
--- a/clang/test/SemaOpenACC/compute-construct-vector_length-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-vector_length-clause.c
@@ -8,6 +8,31 @@ void Test() {
 #pragma acc kernels vector_length(1)
   while(1);
 
+  // expected-error@+2{{OpenACC 'vector_length' clause cannot appear more than once on a 'kernels' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels vector_length(1) vector_length(2)
+  while(1);
+
+  // expected-error@+2{{OpenACC 'vector_length' clause cannot appear more than once on a 'parallel' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel vector_length(1) vector_length(2)
+  while(1);
+
+  // expected-error@+3{{OpenACC 'vector_length' clause cannot appear more than once in a 'device_type' region on a 'kernels' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels vector_length(1) device_type(*) vector_length(1) vector_length(2)
+  while(1);
+
+  // expected-error@+3{{OpenACC 'vector_length' clause cannot appear more than once in a 'device_type' region on a 'parallel' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel device_type(*) vector_length(1) vector_length(2)
+  while(1);
+
+#pragma acc parallel vector_length(1) device_type(*) vector_length(2)
+  while(1);
+
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'serial' directive}}
 #pragma acc serial vector_length(1)
   while(1);
diff --git a/clang/test/SemaOpenACC/data-construct-copy-clause.c b/clang/test/SemaOpenACC/data-construct-copy-clause.c
index 0b2b0534073ed..b19f3f465caaf 100644
--- a/clang/test/SemaOpenACC/data-construct-copy-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-copy-clause.c
@@ -56,10 +56,10 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 #pragma acc data copy((float)ArrayParam[2])
   ;
 
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{OpenACC 'copy' clause is not valid on 'enter data' directive}}
 #pragma acc enter data copy(LocalInt)
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{OpenACC 'pcopy' clause is not valid on 'exit data' directive}}
 #pragma acc exit data pcopy(LocalInt)
   // expected-error@+2{{OpenACC 'host_data' construct must have at least one 'use_device' clause}}
diff --git a/clang/test/SemaOpenACC/data-construct-copyin-clause.c b/clang/test/SemaOpenACC/data-construct-copyin-clause.c
index edc3f0a2e91fe..9b51a28d51a3d 100644
--- a/clang/test/SemaOpenACC/data-construct-copyin-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-copyin-clause.c
@@ -63,7 +63,7 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 #pragma acc data copyin(invalid:(float)ArrayParam[2])
   ;
 
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{OpenACC 'copyin' clause is not valid on 'exit data' directive}}
 #pragma acc exit data copyin(LocalInt)
   // expected-error@+2{{OpenACC 'host_data' construct must have at least one 'use_device' clause}}
diff --git a/clang/test/SemaOpenACC/data-construct-copyout-clause.c b/clang/test/SemaOpenACC/data-construct-copyout-clause.c
index 8d137e093db0e..0c2264c56fcb8 100644
--- a/clang/test/SemaOpenACC/data-construct-copyout-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-copyout-clause.c
@@ -63,7 +63,7 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 #pragma acc data copyout(invalid:(float)ArrayParam[2])
   ;
 
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{OpenACC 'copyout' clause is not valid on 'enter data' directive}}
 #pragma acc enter data copyout(LocalInt)
   // expected-error@+2{{OpenACC 'host_data' construct must have at least one 'use_device' clause}}
diff --git a/clang/test/SemaOpenACC/data-construct-create-clause.c b/clang/test/SemaOpenACC/data-construct-create-clause.c
index e49d53b17ee82..560c6b65cc502 100644
--- a/clang/test/SemaOpenACC/data-construct-create-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-create-clause.c
@@ -63,7 +63,7 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 #pragma acc data create(invalid:(float)ArrayParam[2])
   ;
 
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{OpenACC 'create' clause is not valid on 'exit data' directive}}
 #pragma acc exit data create(LocalInt)
   // expected-error@+2{{OpenACC 'host_data' construct must have at least one 'use_device' clause}}
diff --git a/clang/test/SemaOpenACC/data-construct-default-clause.c b/clang/test/SemaOpenACC/data-construct-default-clause.c
index 150d3ec22dcda..9e8ee529e9b8a 100644
--- a/clang/test/SemaOpenACC/data-construct-default-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-default-clause.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 %s -fopenacc -verify
 
 void use() {
-  // expected-error@+2{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+  // expected-error@+2{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
   // expected-error@+1{{invalid value for 'default' clause; expected 'present' or 'none'}}
 #pragma acc data default(garbage)
   ;
@@ -13,11 +13,11 @@ void use() {
   // expected-note@+1{{previous clause is here}}
 #pragma acc data default(none) default(present)
   ;
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{OpenACC 'default' clause is not valid on 'enter data' directive}}
 #pragma acc enter data default(present)
   ;
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{OpenACC 'default' clause is not valid on 'exit data' directive}}
 #pragma acc exit data default(none)
   ;
diff --git a/clang/test/SemaOpenACC/data-construct-delete-clause.c b/clang/test/SemaOpenACC/data-construct-delete-clause.c
index 05093dbc4e01b..d2727de8dba63 100644
--- a/clang/test/SemaOpenACC/data-construct-delete-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-delete-clause.c
@@ -37,11 +37,11 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
   // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, member of a composite variable, or composite variable member}}
 #pragma acc exit data delete((float)ArrayParam[2])
 
-  // expected-error@+2{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+  // expected-error@+2{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
   // expected-error@+1{{OpenACC 'delete' clause is not valid on 'data' directive}}
 #pragma acc data delete(LocalInt)
   ;
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{OpenACC 'delete' clause is not valid on 'enter data' directive}}
 #pragma acc enter data delete(LocalInt)
   // expected-error@+2{{OpenACC 'host_data' construct must have at least one 'use_device' clause}}
diff --git a/clang/test/SemaOpenACC/data-construct-device_type-clause.c b/clang/test/SemaOpenACC/data-construct-device_type-clause.c
index e450e320fa53c..a8234224540ba 100644
--- a/clang/test/SemaOpenACC/data-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-device_type-clause.c
@@ -47,7 +47,7 @@ void uses() {
   // expected-note@+1{{previous clause is here}}
 #pragma acc data default(none) device_type(radeon) attach(Var)
   ;
-  // expected-error@+3{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+  // expected-error@+3{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
   // expected-error@+2{{OpenACC clause 'default' may not follow a 'device_type' clause in a 'data' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc data device_type(radeon) default(none)
diff --git a/clang/test/SemaOpenACC/data-construct-no_create-clause.c b/clang/test/SemaOpenACC/data-construct-no_create-clause.c
index 0eb459eb0009a..1de39955ba8ec 100644
--- a/clang/test/SemaOpenACC/data-construct-no_create-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-no_create-clause.c
@@ -48,10 +48,10 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 #pragma acc data no_create((float)ArrayParam[2])
   ;
 
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{OpenACC 'no_create' clause is not valid on 'exit data' directive}}
 #pragma acc exit data no_create(LocalInt)
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{OpenACC 'no_create' clause is not valid on 'enter data' directive}}
 #pragma acc enter data no_create(LocalInt)
   // expected-error@+2{{OpenACC 'host_data' construct must have at least one 'use_device' clause}}
diff --git a/clang/test/SemaOpenACC/data-construct-use_device-clause.c b/clang/test/SemaOpenACC/data-construct-use_device-clause.c
index c2e7fd17f8c8d..9239757b8e85d 100644
--- a/clang/test/SemaOpenACC/data-construct-use_device-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-use_device-clause.c
@@ -52,14 +52,14 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 #pragma acc host_data use_device((float)ArrayParam[2])
   ;
 
-  // expected-error@+2{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+  // expected-error@+2{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
   // expected-error@+1{{OpenACC 'use_device' clause is not valid on 'data' directive}}
 #pragma acc data use_device(LocalInt)
   ;
-  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+2{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
   // expected-error@+1{{OpenACC 'use_device' clause is not valid on 'enter data' directive}}
 #pragma acc enter data use_device(LocalInt)
-  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+2{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
   // expected-error@+1{{OpenACC 'use_device' clause is not valid on 'exit data' directive}}
 #pragma acc exit data use_device(LocalInt)
 }
diff --git a/clang/test/SemaOpenACC/data-construct.cpp b/clang/test/SemaOpenACC/data-construct.cpp
index e1d0c04d7fdee..ffb037d439c8e 100644
--- a/clang/test/SemaOpenACC/data-construct.cpp
+++ b/clang/test/SemaOpenACC/data-construct.cpp
@@ -43,22 +43,22 @@ void AtLeastOneOf() {
 #pragma acc data default(none)
   ;
 
-    // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data if(Var)
   ;
 
-    // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data async
   ;
 
-    // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data wait
   ;
 
-    // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data device_type(*)
   ;
-    // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data
   ;
 
@@ -67,13 +67,13 @@ void AtLeastOneOf() {
 #pragma acc enter data create(Var)
 #pragma acc enter data attach(VarPtr)
 
-  // expected-error@+1{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+1{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
 #pragma acc enter data if(Var)
-  // expected-error@+1{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+1{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
 #pragma acc enter data async
-  // expected-error@+1{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+1{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
 #pragma acc enter data wait
-  // expected-error@+1{{OpenACC 'enter data' construct must have at least one 'copyin', 'create' or 'attach' clause}}
+  // expected-error@+1{{OpenACC 'enter data' construct must have at least one 'attach', 'copyin', or 'create' clause}}
 #pragma acc enter data
 
   // Exit Data
@@ -81,15 +81,15 @@ void AtLeastOneOf() {
 #pragma acc exit data delete(Var)
 #pragma acc exit data detach(VarPtr)
 
-  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
 #pragma acc exit data if(Var)
-  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
 #pragma acc exit data async
-  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
 #pragma acc exit data wait
-  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
 #pragma acc exit data finalize
-  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete' or 'detach' clause}}
+  // expected-error@+1{{OpenACC 'exit data' construct must have at least one 'copyout', 'delete', or 'detach' clause}}
 #pragma acc exit data
 
   // Host Data
diff --git a/clang/test/SemaOpenACC/init-construct.cpp b/clang/test/SemaOpenACC/init-construct.cpp
index 2cbe5f1ded04a..ad901b3aa211e 100644
--- a/clang/test/SemaOpenACC/init-construct.cpp
+++ b/clang/test/SemaOpenACC/init-construct.cpp
@@ -42,7 +42,13 @@ void TestInst() {
 #pragma acc init
 #pragma acc init if (T::value < T{})
 #pragma acc init device_type(radeon) device_num(getI()) if (getI() < getS())
-#pragma acc init device_type(multicore) device_type(host) device_num(t) if (t < T::value) device_num(getI()) if (getI() < getS())
+  // expected-error@+2{{OpenACC 'device_num' clause cannot appear more than once on a 'init' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc init device_type(multicore) device_type(host) device_num(t) if (t < T::value) device_num(getI()) 
+
+  // expected-error@+2{{OpenACC 'if' clause cannot appear more than once on a 'init' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc init if(t < T::value) if (getI() < getS())
 
   // expected-error@+1{{value of type 'const NotConvertible' is not contextually convertible to 'bool'}}
 #pragma acc init if (T::NCValue)
diff --git a/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c b/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
index 2f2bd15935856..762873f13d96a 100644
--- a/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
+++ b/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
@@ -8,40 +8,40 @@ void uses() {
 #pragma acc loop independent
   for(unsigned i = 0; i < 5; ++i);
 
-  // expected-error@+2{{OpenACC clause 'seq' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'auto' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop auto seq
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'independent' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'independent' may not appear on the same construct as a 'auto' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop auto independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'auto' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'independent' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'independent' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'auto' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'independent' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop independent auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'seq' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'independent' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop independent seq
   for(unsigned i = 0; i < 5; ++i);
 
-  // expected-error@+2{{OpenACC clause 'seq' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq seq
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'independent' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'independent' may not appear on the same construct as a 'independent' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop independent independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'auto' on 'loop' construct conflicts with previous data dependence clause}}
+  // expected-error@+2{{OpenACC clause 'auto' may not appear on the same construct as a 'auto' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop auto auto
   for(unsigned i = 0; i < 5; ++i);
@@ -591,6 +591,10 @@ void uses() {
   // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq gang
+  for(unsigned i = 0; i < 5; ++i);
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'gang' clause on a 'loop' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc loop gang seq
   for(unsigned i = 0; i < 5; ++i);
   // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
   // expected-note@+1{{previous clause is here}}
diff --git a/clang/test/SemaOpenACC/loop-construct-collapse-clause.cpp b/clang/test/SemaOpenACC/loop-construct-collapse-clause.cpp
index 4d45d1107d03b..851742434a1f9 100644
--- a/clang/test/SemaOpenACC/loop-construct-collapse-clause.cpp
+++ b/clang/test/SemaOpenACC/loop-construct-collapse-clause.cpp
@@ -18,7 +18,7 @@ void only_one_on_loop() {
   // expected-error@+2{{OpenACC 'collapse' clause cannot appear more than once on a 'loop' directive}}
   // expected-note@+1{{previous clause is here}}
 #pragma acc loop collapse(1) collapse(1)
-  for(unsigned i = 0; i < 5; ++i);
+  for(int i = 0; i < 5; ++i);
 }
 
 constexpr int three() { return 3; }
@@ -323,7 +323,7 @@ void no_other_directives() {
 #pragma acc loop collapse(2)
   for(unsigned i = 0; i < 5; ++i) {
     for(unsigned j = 0; j < 5; ++j) {
-    // expected-error@+1{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+1{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
 #pragma acc data
       ;
     }
@@ -331,7 +331,7 @@ void no_other_directives() {
   // expected-note@+1{{active 'collapse' clause defined here}}
 #pragma acc loop collapse(2)
   for(unsigned i = 0; i < 5; ++i) {
-    // expected-error@+2{{OpenACC 'data' construct must have at least one 'copy', 'copyin', 'copyout', 'create', 'no_create', 'present', 'deviceptr', 'attach' or 'default' clause}}
+    // expected-error@+2{{OpenACC 'data' construct must have at least one 'attach', 'copy', 'copyin', 'copyout', 'create', 'default', 'deviceptr', 'no_create', or 'present' clause}}
     // expected-error@+1{{OpenACC 'data' construct cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
 #pragma acc data
     for(unsigned j = 0; j < 5; ++j) {
@@ -500,3 +500,43 @@ void intervening_without_force() {
       for(;;);
 }
 
+template<unsigned N>
+void allow_multiple_collapse_templ() {
+    // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+    // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(N) device_type(*) collapse(N+1)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+    // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+    // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(N+1) device_type(*) collapse(N)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+}
+void allow_multiple_collapse() {
+  allow_multiple_collapse_templ<2>(); // expected-note{{in instantiation}}
+
+    // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+    // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(2) device_type(*) collapse(3)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+    // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+    // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3) device_type(*) collapse(2)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+}
+
+void no_dupes_since_last_device_type() {
+  // expected-error@+3{{OpenACC 'collapse' clause cannot appear more than once in a 'device_type' region on a 'loop' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc loop collapse(1) device_type(*) collapse(1) collapse(2)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+ 
+#pragma acc loop collapse(1) device_type(*) collapse(1) device_type(nvidia) collapse(2)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+}
diff --git a/clang/test/SemaOpenACC/loop-construct-tile-clause.cpp b/clang/test/SemaOpenACC/loop-construct-tile-clause.cpp
index 9f6d9c3af9d9f..ebd657791f34d 100644
--- a/clang/test/SemaOpenACC/loop-construct-tile-clause.cpp
+++ b/clang/test/SemaOpenACC/loop-construct-tile-clause.cpp
@@ -384,6 +384,20 @@ void intervening() {
   }
 }
 
+void use_largest_tile() {
+// expected-error@+2{{'tile' clause specifies a loop count greater than the number of available loops}}
+// expected-note@+1{{active 'tile' clause defined here}}
+#pragma acc loop tile(1,2) device_type(*) tile (3,4,5)
+  for(int i = 0; i < 5; ++i)
+    for (int j = 0; j < 5; ++j);
+
+// expected-error@+2{{'tile' clause specifies a loop count greater than the number of available loops}}
+// expected-note@+1{{active 'tile' clause defined here}}
+#pragma acc loop tile (3,4,5) device_type(*) tile(1,2)
+  for(int i = 0; i < 5; ++i)
+    for (int j = 0; j < 5; ++j);
+}
+
 void collapse_tile_depth() {
   // expected-error@+4{{'collapse' clause specifies a loop count greater than the number of available loops}}
   // expected-note@+3{{active 'collapse' clause defined here}}
@@ -394,3 +408,15 @@ void collapse_tile_depth() {
     for(int j = 0; j < 5; ++j);
   }
 }
+void no_dupes_since_last_device_type() {
+  // expected-error@+3{{OpenACC 'tile' clause cannot appear more than once in a 'device_type' region on a 'loop' directive}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc loop tile(1) device_type(*) tile(1) tile(2)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+
+#pragma acc loop tile(1) device_type(*) tile(1) device_type(nvidia) tile(2)
+  for(unsigned i = 0; i < 5; ++i)
+    for(unsigned j = 0; j < 5; ++j);
+}
diff --git a/clang/test/SemaOpenACC/no-empty-pqr-list.cpp b/clang/test/SemaOpenACC/no-empty-pqr-list.cpp
index fdac89646a5f8..eee9c96fe2c4d 100644
--- a/clang/test/SemaOpenACC/no-empty-pqr-list.cpp
+++ b/clang/test/SemaOpenACC/no-empty-pqr-list.cpp
@@ -213,7 +213,7 @@ void Executable(int i) {
   // expected-error@+1{{expected identifier}}
 #pragma acc shutdown device_type()
   // expected-error@+1{{expected identifier}}
-#pragma acc set if(true) device_type()
+#pragma acc set device_num(i) device_type()
   // expected-error@+1{{expected identifier}}
 #pragma acc update self(i) device_type()
 
diff --git a/clang/test/SemaOpenACC/routine-construct-clauses.cpp b/clang/test/SemaOpenACC/routine-construct-clauses.cpp
index b9fc46aeff092..004a0694c7f59 100644
--- a/clang/test/SemaOpenACC/routine-construct-clauses.cpp
+++ b/clang/test/SemaOpenACC/routine-construct-clauses.cpp
@@ -7,11 +7,11 @@ void Func2();
 #pragma acc routine(Func) vector nohost
 #pragma acc routine(Func) nohost seq
 #pragma acc routine(Func) gang
-// expected-error@+2{{OpenACC 'bind' clause cannot appear more than once on a 'routine' directive}}
+// expected-error@+2{{OpenACC clause 'bind' may not appear on the same construct as a 'bind' clause on a 'routine' construct}}
 // expected-note@+1{{previous clause is here}}
 #pragma acc routine(Func) gang bind(a) bind(a)
 
-// expected-error@+2{{OpenACC 'bind' clause cannot appear more than once on a 'routine' directive}}
+// expected-error@+2{{OpenACC clause 'bind' may not appear on the same construct as a 'bind' clause on a 'routine' construct}}
 // expected-note@+1{{previous clause is here}}
 #pragma acc routine gang bind(a) bind(a)
 void DupeImplName();
@@ -65,9 +65,9 @@ void DupeImplName();
 // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
 // expected-note@+1{{previous clause is here}}
 #pragma acc routine(Func) gang gang
-// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'worker', 'vector' or 'seq' clause}}
+// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'seq', 'vector', or 'worker' clause}}
 #pragma acc routine(Func)
-// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'worker', 'vector' or 'seq' clause}}
+// expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'seq', 'vector', or 'worker' clause}}
 #pragma acc routine(Func) nohost
 
 // only the 'dim' syntax for gang is legal.
@@ -227,3 +227,548 @@ void DupeBinds6();
 // expected-note@-3{{previous clause is here}}
 #pragma acc routine bind("asdfDiff") seq
 void DupeBinds6();
+
+// The standard wasn't initially clear, but is being clarified that we require
+// exactly one of (gang, worker, vector, or seq) to apply to each 'device_type'.
+// The ones before any 'device_type' apply to all.
+namespace FigureDupesAllowedAroundDeviceType {
+  // Test conflicts without a device type
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang gang
+  void Func1();
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang worker
+  void Func2();
+  // expected-error@+2{{OpenACC clause 'vector' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang vector
+  void Func3();
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang seq
+  void Func4();
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'worker' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker gang
+  void Func5();
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'worker' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker worker
+  void Func6();
+  // expected-error@+2{{OpenACC clause 'vector' may not appear on the same construct as a 'worker' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker vector
+  void Func7();
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'worker' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker seq
+  void Func8();
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'vector' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector gang
+  void Func9();
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'vector' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector worker
+  void Func10();
+  // expected-error@+2{{OpenACC clause 'vector' may not appear on the same construct as a 'vector' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector vector
+  void Func11();
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'vector' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector seq
+  void Func12();
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'seq' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq gang
+  void Func13();
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'seq' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq worker
+  void Func14();
+  // expected-error@+2{{OpenACC clause 'vector' may not appear on the same construct as a 'seq' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq vector
+  void Func15();
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'seq' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq seq
+  void Func16();
+  // None included for one without a device type
+  // expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'seq', 'vector', or 'worker' clause}}
+#pragma acc routine
+  void Func16();
+
+  // Check same conflicts for 'before' the device_type
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang gang device_type(*)
+  void Func17();
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang worker device_type(*)
+  void Func18();
+  // expected-error@+2{{OpenACC clause 'vector' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang vector device_type(*)
+  void Func19();
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'gang' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang seq device_type(*)
+  void Func20();
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'worker' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker gang device_type(*)
+  void Func21();
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'worker' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker worker device_type(*)
+  void Func22();
+  // expected-error@+2{{OpenACC clause 'vector' may not appear on the same construct as a 'worker' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker vector device_type(*)
+  void Func23();
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'worker' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker seq device_type(*)
+  void Func24();
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'vector' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector gang device_type(*)
+  void Func25();
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'vector' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector worker device_type(*)
+  void Func26();
+  // expected-error@+2{{OpenACC clause 'vector' may not appear on the same construct as a 'vector' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector vector device_type(*)
+  void Func27();
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'vector' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector seq device_type(*)
+  void Func28();
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'seq' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq gang device_type(*)
+  void Func29();
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'seq' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq worker device_type(*)
+  void Func30();
+  // expected-error@+2{{OpenACC clause 'vector' may not appear on the same construct as a 'seq' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq vector device_type(*)
+  void Func31();
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'seq' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq seq device_type(*)
+  void Func32();
+  // None included for the device_type
+  // expected-error@+1{{OpenACC 'routine' construct must have at least one 'gang', 'seq', 'vector', or 'worker' clause that applies to each 'device_type'}}
+#pragma acc routine device_type(*)
+  void Func33();
+
+  // Conflicts between 'global' and 'device_type'
+  // expected-error@+3{{OpenACC clause 'gang' after 'device_type' clause on a 'routine' conflicts with the 'gang' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang device_type(*) gang
+  void Func34();
+  // expected-error@+3{{OpenACC clause 'worker' after 'device_type' clause on a 'routine' conflicts with the 'gang' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang device_type(*) worker
+  void Func35();
+  // expected-error@+3{{OpenACC clause 'vector' after 'device_type' clause on a 'routine' conflicts with the 'gang' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang device_type(*) vector
+  void Func36();
+  // expected-error@+3{{OpenACC clause 'seq' after 'device_type' clause on a 'routine' conflicts with the 'gang' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine gang device_type(*) seq
+  void Func37();
+  // expected-error@+3{{OpenACC clause 'gang' after 'device_type' clause on a 'routine' conflicts with the 'worker' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker device_type(*) gang
+  void Func38();
+  // expected-error@+3{{OpenACC clause 'worker' after 'device_type' clause on a 'routine' conflicts with the 'worker' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker device_type(*) worker
+  void Func39();
+  // expected-error@+3{{OpenACC clause 'vector' after 'device_type' clause on a 'routine' conflicts with the 'worker' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker device_type(*) vector
+  void Func40();
+  // expected-error@+3{{OpenACC clause 'seq' after 'device_type' clause on a 'routine' conflicts with the 'worker' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine worker device_type(*) seq
+  void Func41();
+  // expected-error@+3{{OpenACC clause 'gang' after 'device_type' clause on a 'routine' conflicts with the 'vector' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector device_type(*) gang
+  void Func42();
+  // expected-error@+3{{OpenACC clause 'worker' after 'device_type' clause on a 'routine' conflicts with the 'vector' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector device_type(*) worker
+  void Func43();
+  // expected-error@+3{{OpenACC clause 'vector' after 'device_type' clause on a 'routine' conflicts with the 'vector' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector device_type(*) vector
+  void Func44();
+  // expected-error@+3{{OpenACC clause 'seq' after 'device_type' clause on a 'routine' conflicts with the 'vector' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine vector device_type(*) seq
+  void Func45();
+  // expected-error@+3{{OpenACC clause 'gang' after 'device_type' clause on a 'routine' conflicts with the 'seq' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq device_type(*) gang
+  void Func46();
+  // expected-error@+3{{OpenACC clause 'worker' after 'device_type' clause on a 'routine' conflicts with the 'seq' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq device_type(*) worker
+  void Func47();
+  // expected-error@+3{{OpenACC clause 'vector' after 'device_type' clause on a 'routine' conflicts with the 'seq' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq device_type(*) vector
+  void Func48();
+  // expected-error@+3{{OpenACC clause 'seq' after 'device_type' clause on a 'routine' conflicts with the 'seq' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq device_type(*) seq
+  void Func49();
+
+  // Conflicts within same device_type
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) gang gang
+  void Func50();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) gang worker
+  void Func51();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) gang vector
+  void Func52();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) gang seq
+  void Func53();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) worker gang
+  void Func54();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) worker worker
+  void Func55();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) worker vector
+  void Func56();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) worker seq
+  void Func57();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) vector gang
+  void Func58();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) vector worker
+  void Func59();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) vector vector
+  void Func60();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) vector seq
+  void Func61();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) seq gang
+  void Func62();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) seq worker
+  void Func63();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) seq vector
+  void Func64();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) seq seq
+  void Func65();
+
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) gang gang device_type(nvidia) seq
+  void Func66();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) gang worker device_type(nvidia) seq
+  void Func67();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) gang vector device_type(nvidia) seq
+  void Func68();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) gang seq device_type(nvidia) seq
+  void Func69();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) worker gang device_type(nvidia) seq
+  void Func70();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) worker worker device_type(nvidia) seq
+  void Func71();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) worker vector device_type(nvidia) seq
+  void Func72();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) worker seq device_type(nvidia) seq
+  void Func73();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) vector gang device_type(nvidia) seq
+  void Func74();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) vector worker device_type(nvidia) seq
+  void Func75();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) vector vector device_type(nvidia) seq
+  void Func76();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) vector seq device_type(nvidia) seq
+  void Func77();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) seq gang device_type(nvidia) seq
+  void Func78();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) seq worker device_type(nvidia) seq
+  void Func79();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) seq vector device_type(nvidia) seq
+  void Func80();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(*) seq seq device_type(nvidia) seq
+  void Func81();
+
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) gang gang
+  void Func82();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) gang worker
+  void Func83();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) gang vector
+  void Func84();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'gang' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) gang seq
+  void Func85();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) worker gang
+  void Func86();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) worker worker
+  void Func87();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) worker vector
+  void Func88();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'worker' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) worker seq
+  void Func89();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) vector gang
+  void Func90();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) vector worker
+  void Func91();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) vector vector
+  void Func92();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'vector' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) vector seq
+  void Func93();
+  // expected-error@+3{{OpenACC clause 'gang' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) seq gang
+  void Func94();
+  // expected-error@+3{{OpenACC clause 'worker' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) seq worker
+  void Func95();
+  // expected-error@+3{{OpenACC clause 'vector' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) seq vector
+  void Func96();
+  // expected-error@+3{{OpenACC clause 'seq' on a 'routine' directive conflicts with the 'seq' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine device_type(nvidia) seq device_type(*) seq seq
+  void Func97();
+
+  // All fine?
+#pragma acc routine device_type(*) gang device_type(nvidia) gang
+  void Func98();
+#pragma acc routine device_type(*) gang device_type(nvidia) worker
+  void Func99();
+#pragma acc routine device_type(*) gang device_type(nvidia) vector
+  void Func100();
+#pragma acc routine device_type(*) gang device_type(nvidia) seq
+  void Func101();
+#pragma acc routine device_type(*) worker device_type(nvidia) gang
+  void Func102();
+#pragma acc routine device_type(*) worker device_type(nvidia) worker
+  void Func103();
+#pragma acc routine device_type(*) worker device_type(nvidia) vector
+  void Func104();
+#pragma acc routine device_type(*) worker device_type(nvidia) seq
+  void Func105();
+#pragma acc routine device_type(*) vector device_type(nvidia) gang
+  void Func106();
+#pragma acc routine device_type(*) vector device_type(nvidia) worker
+  void Func107();
+#pragma acc routine device_type(*) vector device_type(nvidia) vector
+  void Func108();
+#pragma acc routine device_type(*) vector device_type(nvidia) seq
+  void Func109();
+#pragma acc routine device_type(*) seq device_type(nvidia) gang
+  void Func110();
+#pragma acc routine device_type(*) seq device_type(nvidia) worker
+  void Func111();
+#pragma acc routine device_type(*) seq device_type(nvidia) vector
+  void Func112();
+#pragma acc routine device_type(*) seq device_type(nvidia) seq
+  void Func113();
+
+}
+
+namespace BindDupes {
+  // expected-error@+2{{OpenACC clause 'bind' may not appear on the same construct as a 'bind' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq bind(asdf) bind(asdf)
+  void Func1();
+  // expected-error@+2{{OpenACC clause 'bind' may not appear on the same construct as a 'bind' clause on a 'routine' construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq bind(asdf) bind(asdf) device_type(*)
+  void Func2();
+  // expected-error@+3{{OpenACC clause 'bind' after 'device_type' clause on a 'routine' conflicts with the 'bind' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq bind(asdf) device_type(*) bind(asdf)
+  void Func3();
+  // expected-error@+3{{OpenACC clause 'bind' after 'device_type' clause on a 'routine' conflicts with the 'bind' clause before the first 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq bind(asdf) device_type(*) bind(asdf) device_type(*)
+  void Func4();
+  // expected-error@+3{{OpenACC clause 'bind' on a 'routine' directive conflicts with the 'bind' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq device_type(*) bind(asdf) bind(asdf)
+  void Func5();
+  // expected-error@+3{{OpenACC clause 'bind' on a 'routine' directive conflicts with the 'bind' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq device_type(*) bind(asdf) bind(asdf) device_type(*)
+  void Func6();
+  // expected-error@+3{{OpenACC clause 'bind' on a 'routine' directive conflicts with the 'bind' clause applying to the same 'device_type'}}
+  // expected-note@+2{{previous clause is here}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc routine seq device_type(*) bind(asdf) device_type(*) bind(asdf) bind(asdf)
+  void Func7();
+
+#pragma acc routine seq device_type(*) bind(asdf) device_type(*) bind(asdf)
+  void Func8();
+}
diff --git a/clang/test/SemaOpenACC/set-construct.cpp b/clang/test/SemaOpenACC/set-construct.cpp
index ac141d097532b..1a152b898ef4a 100644
--- a/clang/test/SemaOpenACC/set-construct.cpp
+++ b/clang/test/SemaOpenACC/set-construct.cpp
@@ -23,10 +23,10 @@ void uses() {
   // expected-error@+1{{value of type 'struct NotConvertible' is not contextually convertible to 'bool'}}
 #pragma acc set if (NC) device_type(radeon)
 
-  // expected-error@+2{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+2{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
   // expected-error@+1{{OpenACC clause 'device_num' requires expression of integer type ('struct NotConvertible' invalid)}}
 #pragma acc set device_num(NC)
-  // expected-error@+4{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+4{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
   // expected-error@+3{{multiple conversions from expression type 'struct AmbiguousConvert' to an integral type}}
   // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
   // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
@@ -36,20 +36,21 @@ void uses() {
 #pragma acc set device_num(Explicit)
 
   // expected-error@+2{{OpenACC clause 'default_async' requires expression of integer type ('struct NotConvertible' invalid)}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set default_async(NC)
   // expected-error@+4{{multiple conversions from expression type 'struct AmbiguousConvert' to an integral type}}
   // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
   // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set default_async(Ambiguous)
   // expected-error@+2{{OpenACC integer expression requires explicit conversion from 'struct ExplicitConvertOnly' to 'int'}}
   // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
 #pragma acc set default_async(Explicit)
 
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set
 
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set if (true)
 
   // expected-error@+2{{'default_async' clause cannot appear more than once on a 'set' directive}}
@@ -68,10 +69,10 @@ void uses() {
 #pragma acc set device_type(acc_device_nvidia) if(true) if (true)
 
   // expected-error@+2{{OpenACC 'device_type' clause on a 'set' construct only permits one architecture}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set device_type(nvidia, radeon)
 
   // expected-error@+2{{OpenACC 'device_type' clause on a 'set' construct only permits one architecture}}
-  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', 'device_type' or 'if' clause}}
+  // expected-error@+1{{OpenACC 'set' construct must have at least one 'default_async', 'device_num', or 'device_type' clause}}
 #pragma acc set device_type(nonsense, nvidia, radeon)
 }
diff --git a/clang/test/SemaOpenACC/shutdown-construct.cpp b/clang/test/SemaOpenACC/shutdown-construct.cpp
index b167ac66c85d9..f13a281edcb0d 100644
--- a/clang/test/SemaOpenACC/shutdown-construct.cpp
+++ b/clang/test/SemaOpenACC/shutdown-construct.cpp
@@ -42,11 +42,17 @@ void TestInst() {
 #pragma acc shutdown
 #pragma acc shutdown if (T::value < T{})
 #pragma acc shutdown device_type(multicore) device_num(getI()) if (getI() < getS())
-#pragma acc shutdown device_type(default) device_type(radeon) device_num(t) if (t < T::value) device_num(getI()) if (getI() < getS())
+  // expected-error@+2{{OpenACC 'device_num' clause cannot appear more than once on a 'shutdown' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc shutdown device_type(default) device_type(radeon) device_num(t) if (t < T::value) device_num(getI()) 
 
   // expected-error@+1{{value of type 'const NotConvertible' is not contextually convertible to 'bool'}}
 #pragma acc shutdown if (T::NCValue)
 
+  // expected-error@+2{{OpenACC 'if' clause cannot appear more than once on a 'shutdown' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc shutdown if(t < T::value) if (getI() < getS())
+
   // expected-error@+1{{OpenACC clause 'device_num' requires expression of integer type ('const NotConvertible' invalid)}}
 #pragma acc shutdown device_num(T::NCValue)
   // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
diff --git a/clang/test/SemaOpenACC/update-construct.cpp b/clang/test/SemaOpenACC/update-construct.cpp
index f0372736452df..a89dc1f20158a 100644
--- a/clang/test/SemaOpenACC/update-construct.cpp
+++ b/clang/test/SemaOpenACC/update-construct.cpp
@@ -36,15 +36,15 @@ void uses() {
 
   // These diagnose because there isn't at least 1 of 'self', 'host', or
   // 'device'.
-  // expected-error@+1{{OpenACC 'update' construct must have at least one 'self', 'host' or 'device' clause}}
+  // expected-error@+1{{OpenACC 'update' construct must have at least one 'device', 'host', or 'self' clause}}
 #pragma acc update async
-  // expected-error@+1{{OpenACC 'update' construct must have at least one 'self', 'host' or 'device' clause}}
+  // expected-error@+1{{OpenACC 'update' construct must have at least one 'device', 'host', or 'self' clause}}
 #pragma acc update wait
-  // expected-error@+1{{OpenACC 'update' construct must have at least one 'self', 'host' or 'device' clause}}
+  // expected-error@+1{{OpenACC 'update' construct must have at least one 'device', 'host', or 'self' clause}}
 #pragma acc update device_type(host)
-  // expected-error@+1{{OpenACC 'update' construct must have at least one 'self', 'host' or 'device' clause}}
+  // expected-error@+1{{OpenACC 'update' construct must have at least one 'device', 'host', or 'self' clause}}
 #pragma acc update if(true)
-  // expected-error@+1{{OpenACC 'update' construct must have at least one 'self', 'host' or 'device' clause}}
+  // expected-error@+1{{OpenACC 'update' construct must have at least one 'device', 'host', or 'self' clause}}
 #pragma acc update if_present
 
   // expected-error@+1{{value of type 'struct NotConvertible' is not contextually convertible to 'bool'}}
diff --git a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
index 707239c7ba364..87ce80846dd93 100644
--- a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
+++ b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
@@ -113,6 +113,10 @@ class Clause<string c> {
   // Set the prefix as optional.
   // `clause([prefix]: value)`
   bit isPrefixOptional = true;
+
+  // When necessary because it isn't covered by rules, the name used in the
+  // clause in the clang::OpenACCClauseKind enum.
+  string clangAccSpelling = "";
 }
 
 // Hold information about clause validity by version.
diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index 898b7dc9e230f..6f6539c1ffebc 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -68,12 +68,14 @@ def ACCC_Copy : Clause<"copy"> {
 // 2.7.7
 def ACCC_Copyin : Clause<"copyin"> {
   let flangClass = "AccObjectListWithModifier";
+  let clangAccSpelling = "CopyIn";
   let aliases = ["present_or_copyin", "pcopyin"];
 }
 
 // 2.7.8
 def ACCC_Copyout : Clause<"copyout"> {
   let flangClass = "AccObjectListWithModifier";
+  let clangAccSpelling = "CopyOut";
   let aliases = ["present_or_copyout", "pcopyout"];
 }
 
@@ -124,6 +126,7 @@ def ACCC_DeviceNum : Clause<"device_num">  {
 // 2.7.4
 def ACCC_DevicePtr : Clause<"deviceptr"> {
   let flangClass = "AccObjectList";
+  let clangAccSpelling = "DevicePtr";
 }
 
 // 2.13.1
@@ -144,6 +147,7 @@ def ACCC_Finalize : Clause<"finalize"> {}
 // 2.5.14
 def ACCC_FirstPrivate : Clause<"firstprivate"> {
   let flangClass = "AccObjectList";
+  let clangAccSpelling = "FirstPrivate";
 }
 
 // 2.9.2
@@ -179,7 +183,9 @@ def ACCC_NoCreate : Clause<"no_create"> {
 }
 
 // 2.15.1
-def ACCC_NoHost : Clause<"nohost"> {}
+def ACCC_NoHost : Clause<"nohost"> { 
+  let clangAccSpelling = "NoHost"; 
+}
 
 // 2.5.10
 def ACCC_NumGangs : Clause<"num_gangs"> {
diff --git a/llvm/include/llvm/TableGen/DirectiveEmitter.h b/llvm/include/llvm/TableGen/DirectiveEmitter.h
index a2c9b2d427cce..28373271ef1e8 100644
--- a/llvm/include/llvm/TableGen/DirectiveEmitter.h
+++ b/llvm/include/llvm/TableGen/DirectiveEmitter.h
@@ -160,6 +160,30 @@ class Directive : public BaseRecord {
   }
 
   const Record *getCategory() const { return Def->getValueAsDef("category"); }
+
+  // Clang uses a different format for names of its directives enum.
+  std::string getClangAccSpelling() const {
+    std::string Name = Def->getValueAsString("name").str();
+
+    // Clang calls the 'unknown' value 'invalid'.
+    if (Name == "unknown")
+      return "Invalid";
+
+    // Clang entries all start with a capital letter, so apply that.
+    Name[0] = std::toupper(Name[0]);
+    // Additionally, spaces/underscores are handled by capitalizing the next
+    // letter of the name and removing the space/underscore.
+    for (unsigned I = 0; I < Name.size(); ++I) {
+      if (Name[I] == ' ' || Name[I] == '_') {
+        Name.erase(I, 1);
+        assert(Name[I] != ' ' && Name[I] != '_' &&
+               "No double spaces/underscores");
+        Name[I] = std::toupper(Name[I]);
+      }
+    }
+
+    return Name;
+  }
 };
 
 // Wrapper class that contains Clause's information defined in DirectiveBase.td
@@ -200,6 +224,30 @@ class Clause : public BaseRecord {
     return N;
   }
 
+  // Clang uses a different format for names of its clause enum, which can be
+  // overwritten with the `clangSpelling` value. So get the proper spelling
+  // here.
+  std::string getClangAccSpelling() const {
+    if (StringRef ClangSpelling = Def->getValueAsString("clangAccSpelling");
+        !ClangSpelling.empty())
+      return ClangSpelling.str();
+
+    std::string Name = Def->getValueAsString("name").str();
+    // Clang entries all start with a capital letter, so apply that.
+    Name[0] = std::toupper(Name[0]);
+    // Additionally, underscores are handled by capitalizing the next letter of
+    // the name and removing the underscore.
+    for (unsigned I = 0; I < Name.size(); ++I) {
+      if (Name[I] == '_') {
+        Name.erase(I, 1);
+        assert(Name[I] != '_' && "No double underscores");
+        Name[I] = std::toupper(Name[I]);
+      }
+    }
+
+    return Name;
+  }
+
   // Optional field.
   StringRef getEnumName() const {
     return Def->getValueAsString("enumClauseValue");
diff --git a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
index ab68e028f1e96..b5276d695f1bc 100644
--- a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
@@ -759,10 +759,25 @@ static void generateGetDirectiveCategory(const DirectiveLanguage &DirLang,
   OS << "}\n";
 }
 
+namespace {
+enum class DirectiveClauseFE { Flang, Clang };
+
+StringRef getFESpelling(DirectiveClauseFE FE) {
+  switch (FE) {
+  case DirectiveClauseFE::Flang:
+    return "flang";
+  case DirectiveClauseFE::Clang:
+    return "clang";
+  }
+  llvm_unreachable("unknown FE kind");
+}
+} // namespace
+
 // Generate a simple enum set with the give clauses.
 static void generateClauseSet(ArrayRef<const Record *> Clauses, raw_ostream &OS,
                               StringRef ClauseSetPrefix, const Directive &Dir,
-                              const DirectiveLanguage &DirLang) {
+                              const DirectiveLanguage &DirLang,
+                              DirectiveClauseFE FE) {
 
   OS << "\n";
   OS << "  static " << DirLang.getClauseEnumSetClass() << " " << ClauseSetPrefix
@@ -770,21 +785,35 @@ static void generateClauseSet(ArrayRef<const Record *> Clauses, raw_ostream &OS,
 
   for (const auto &C : Clauses) {
     VersionedClause VerClause(C);
-    OS << "    llvm::" << DirLang.getCppNamespace()
-       << "::Clause::" << DirLang.getClausePrefix()
-       << VerClause.getClause().getFormattedName() << ",\n";
+    if (FE == DirectiveClauseFE::Flang) {
+      OS << "    llvm::" << DirLang.getCppNamespace()
+         << "::Clause::" << DirLang.getClausePrefix()
+         << VerClause.getClause().getFormattedName() << ",\n";
+    } else {
+      assert(FE == DirectiveClauseFE::Clang);
+      assert(DirLang.getName() == "OpenACC");
+      OS << "   clang::OpenACCClauseKind::"
+         << VerClause.getClause().getClangAccSpelling() << ",\n";
+    }
   }
   OS << "  };\n";
 }
 
 // Generate an enum set for the 4 kinds of clauses linked to a directive.
 static void generateDirectiveClauseSets(const DirectiveLanguage &DirLang,
-                                        raw_ostream &OS) {
+                                        DirectiveClauseFE FE, raw_ostream &OS) {
 
-  IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_SETS", OS);
+  std::string IfDefName{"GEN_"};
+  IfDefName += getFESpelling(FE).upper();
+  IfDefName += "_DIRECTIVE_CLAUSE_SETS";
+  IfDefScope Scope(IfDefName, OS);
 
   OS << "\n";
-  OS << "namespace llvm {\n";
+  // The namespace has to be different for clang vs flang, as 2 structs with the
+  // same name but different layout is UB.  So just put the 'clang' on in the
+  // clang namespace.
+  OS << "namespace " << (FE == DirectiveClauseFE::Flang ? "llvm" : "clang")
+     << " {\n";
 
   // Open namespaces defined in the directive language.
   SmallVector<StringRef, 2> Namespaces;
@@ -797,13 +826,13 @@ static void generateDirectiveClauseSets(const DirectiveLanguage &DirLang,
     OS << "  // Sets for " << Dir.getName() << "\n";
 
     generateClauseSet(Dir.getAllowedClauses(), OS, "allowedClauses_", Dir,
-                      DirLang);
+                      DirLang, FE);
     generateClauseSet(Dir.getAllowedOnceClauses(), OS, "allowedOnceClauses_",
-                      Dir, DirLang);
+                      Dir, DirLang, FE);
     generateClauseSet(Dir.getAllowedExclusiveClauses(), OS,
-                      "allowedExclusiveClauses_", Dir, DirLang);
+                      "allowedExclusiveClauses_", Dir, DirLang, FE);
     generateClauseSet(Dir.getRequiredClauses(), OS, "requiredClauses_", Dir,
-                      DirLang);
+                      DirLang, FE);
   }
 
   // Closing namespaces
@@ -817,27 +846,46 @@ static void generateDirectiveClauseSets(const DirectiveLanguage &DirLang,
 // The struct holds the 4 sets of enumeration for the 4 kinds of clauses
 // allowances (allowed, allowed once, allowed exclusive and required).
 static void generateDirectiveClauseMap(const DirectiveLanguage &DirLang,
-                                       raw_ostream &OS) {
-
-  IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_MAP", OS);
+                                       DirectiveClauseFE FE, raw_ostream &OS) {
+  std::string IfDefName{"GEN_"};
+  IfDefName += getFESpelling(FE).upper();
+  IfDefName += "_DIRECTIVE_CLAUSE_MAP";
+  IfDefScope Scope(IfDefName, OS);
 
   OS << "\n";
   OS << "{\n";
 
+  // The namespace has to be different for clang vs flang, as 2 structs with the
+  // same name but different layout is UB.  So just put the 'clang' on in the
+  // clang namespace.
+  StringRef TopLevelNS = (FE == DirectiveClauseFE::Flang ? "llvm" : "clang");
+
   for (const Directive Dir : DirLang.getDirectives()) {
-    OS << "  {llvm::" << DirLang.getCppNamespace()
-       << "::Directive::" << DirLang.getDirectivePrefix()
-       << Dir.getFormattedName() << ",\n";
+    OS << "  {";
+    if (FE == DirectiveClauseFE::Flang) {
+      OS << TopLevelNS << "::" << DirLang.getCppNamespace()
+         << "::Directive::" << DirLang.getDirectivePrefix()
+         << Dir.getFormattedName() << ",\n";
+    } else {
+      assert(FE == DirectiveClauseFE::Clang);
+      assert(DirLang.getName() == "OpenACC");
+      OS << "clang::OpenACCDirectiveKind::" << Dir.getClangAccSpelling()
+         << ",\n";
+    }
+
     OS << "    {\n";
-    OS << "      llvm::" << DirLang.getCppNamespace() << "::allowedClauses_"
-       << DirLang.getDirectivePrefix() << Dir.getFormattedName() << ",\n";
-    OS << "      llvm::" << DirLang.getCppNamespace() << "::allowedOnceClauses_"
-       << DirLang.getDirectivePrefix() << Dir.getFormattedName() << ",\n";
-    OS << "      llvm::" << DirLang.getCppNamespace()
+    OS << "      " << TopLevelNS << "::" << DirLang.getCppNamespace()
+       << "::allowedClauses_" << DirLang.getDirectivePrefix()
+       << Dir.getFormattedName() << ",\n";
+    OS << "      " << TopLevelNS << "::" << DirLang.getCppNamespace()
+       << "::allowedOnceClauses_" << DirLang.getDirectivePrefix()
+       << Dir.getFormattedName() << ",\n";
+    OS << "      " << TopLevelNS << "::" << DirLang.getCppNamespace()
        << "::allowedExclusiveClauses_" << DirLang.getDirectivePrefix()
        << Dir.getFormattedName() << ",\n";
-    OS << "      llvm::" << DirLang.getCppNamespace() << "::requiredClauses_"
-       << DirLang.getDirectivePrefix() << Dir.getFormattedName() << ",\n";
+    OS << "      " << TopLevelNS << "::" << DirLang.getCppNamespace()
+       << "::requiredClauses_" << DirLang.getDirectivePrefix()
+       << Dir.getFormattedName() << ",\n";
     OS << "    }\n";
     OS << "  },\n";
   }
@@ -1072,13 +1120,24 @@ static void generateFlangClausesParser(const DirectiveLanguage &DirLang,
   OS << ")\n";
 }
 
+// Generate the implementation section for the enumeration in the directive
+// language
+static void emitDirectivesClangImpl(const DirectiveLanguage &DirLang,
+                                    raw_ostream &OS) {
+  // Currently we only have work to do for OpenACC, so skip otherwise.
+  if (DirLang.getName() != "OpenACC")
+    return;
+
+  generateDirectiveClauseSets(DirLang, DirectiveClauseFE::Clang, OS);
+  generateDirectiveClauseMap(DirLang, DirectiveClauseFE::Clang, OS);
+}
 // Generate the implementation section for the enumeration in the directive
 // language
 static void emitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
                                     raw_ostream &OS) {
-  generateDirectiveClauseSets(DirLang, OS);
+  generateDirectiveClauseSets(DirLang, DirectiveClauseFE::Flang, OS);
 
-  generateDirectiveClauseMap(DirLang, OS);
+  generateDirectiveClauseMap(DirLang, DirectiveClauseFE::Flang, OS);
 
   generateFlangClauseParserClass(DirLang, OS);
 
@@ -1211,6 +1270,8 @@ static void emitDirectivesImpl(const RecordKeeper &Records, raw_ostream &OS) {
 
   emitDirectivesFlangImpl(DirLang, OS);
 
+  emitDirectivesClangImpl(DirLang, OS);
+
   generateClauseClassMacro(DirLang, OS);
 
   emitDirectivesBasicImpl(DirLang, OS);

From 9e46830f4ad340ffcbe11174139013c36d5428d9 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 18 Apr 2025 21:54:58 +0000
Subject: [PATCH 492/710] [gn build] Port d1cce66469d0

---
 llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
index 256bab6a9e36a..43b8979776902 100644
--- a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
@@ -99,6 +99,7 @@ static_library("Sema") {
     "SemaOpenACC.cpp",
     "SemaOpenACCAtomic.cpp",
     "SemaOpenACCClause.cpp",
+    "SemaOpenACCClauseAppertainment.cpp",
     "SemaOpenCL.cpp",
     "SemaOpenMP.cpp",
     "SemaOverload.cpp",

From eec1af21c0828544fc49ee1a982a9cdef455b92a Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Fri, 18 Apr 2025 15:21:27 -0700
Subject: [PATCH 493/710] Fix build regression from
 d1cce66469d0739f332b11fdb0fa2a441b1e0483

---
 clang/lib/Sema/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index d0b0508eb263a..4b87004e4b8ea 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -102,6 +102,7 @@ add_clang_library(clangSema
 
   DEPENDS
   ClangOpenCLBuiltinsImpl
+  acc_gen
   omp_gen
   ClangDriverOptions
 

From 59eafd1bdc2881b18639c8798997313c2dab0b80 Mon Sep 17 00:00:00 2001
From: Dan Liew <dan@su-root.co.uk>
Date: Fri, 18 Apr 2025 15:26:04 -0700
Subject: [PATCH 494/710] [BoundsSafety][Sema] Allow counted_by and
 counted_by_or_null on pointers where the pointee type is incomplete but
 potentially completable (#106321)

Previously using the `counted_by` or `counted_by_or_null` attribute on a
pointer with an incomplete pointee type was forbidden. Unfortunately
this prevented a situation like the following from being allowed.

Header file:

```
struct EltTy; // Incomplete type
struct Buffer {
  size_t count;
  struct EltTy* __counted_by(count) buffer; // error before this patch
};
```

Implementation file:

```
struct EltTy {
  // definition
};

void allocBuffer(struct Buffer* b) {
  b->buffer = malloc(sizeof(EltTy)* b->count);
}
```

To allow code like the above but still enforce that the pointee
type size is known in locations where `-fbounds-safety` needs to
emit bounds checks the following scheme is used.

* For incomplete pointee types that can never be completed (e.g. `void`)
  these are treated as error where the attribute is written (just like
  before this patch).
* For incomplete pointee types that might be completable later on
  (struct, union, and enum forward declarations)
  in the translation unit, writing the attribute on the incomplete
  pointee type is allowed on the FieldDecl declaration but "uses" of the
  declared pointer are forbidden if at the point of "use" the pointee
  type is still incomplete.

For this patch a "use" of a FieldDecl covers:

* Explicit and Implicit initialization (note see **Tentative Definition
  Initialization** for an exception to this)
* Assignment
* Conversion to an lvalue (e.g. for use in an expression)

In the swift lang fork of Clang the `counted_by` and
`counted_by_or_null` attribute are allowed in many more contexts. That
isn't the case for upstream Clang so the "use" checks for the attribute
on VarDecl, ParamVarDecl, and function return type have been omitted
from this patch because they can't be tested. However, the
`BoundsSafetyCheckAssignmentToCountAttrPtrWithIncompletePointeeTy` and
`BoundsSafetyCheckUseOfCountAttrPtrWithIncompletePointeeTy` functions
retain the ability to emit diagnostics for these other contexts to avoid
unnecessary divergence between upstream Clang and Apple's internal fork.
Support for checking "uses" will be upstreamed when upstream Clang
allows the `counted_by` and `counted_by_or_null` attribute in additional
contexts.

This patch has a few limitations:

** 1. Tentative Defition Initialization **

This patch currently allows something like:

```
struct IncompleteTy;
struct Buffer {
  int count;
  struct IncompleteTy* __counted_by(count) buf;
};

// Tentative definition
struct Buffer GlobalBuf;
```

The Tentative definition in this example becomes an actual definition
whose initialization **should be checked** but it currently isn't.
Addressing this problem will be done in a subseqent patch.

** 2. When the incomplete pointee type is a typedef diagnostics are slightly misleading **

For this situation:

```

struct IncompleteTy;
typedef struct IncompleteTy Incomplete_t;

struct Buffer {
  int count;
  struct IncompleteTy* __counted_by(count) buf;
};

void use(struct Buffer b) {
  b.buf = 0x0;
}
```

This code emits `note: forward declaration of 'Incomplete_t' (aka
'struct IncompleteTy')` but the location is on the `struct
IncompleteTy;` forward declaration.  This is misleading because
`Incomplete_t` isn't actually forward declared there (instead the
underlying type is). This could be resolved by additional diagnostics
that walk the chain of typedefs and explain each step of the walk.
However, that would be very verbose and didn't seem like a direction
worth pursuing.

rdar://133600117
---
 clang/include/clang/AST/Type.h                |  22 +
 .../clang/Basic/DiagnosticSemaKinds.td        |  24 +
 clang/include/clang/Sema/Sema.h               |  44 ++
 clang/lib/AST/Type.cpp                        |  41 ++
 clang/lib/Sema/SemaBoundsSafety.cpp           | 212 +++++-
 clang/lib/Sema/SemaExpr.cpp                   |  23 +
 clang/lib/Sema/SemaInit.cpp                   |  13 +
 .../attr-counted-by-late-parsed-struct-ptrs.c |   5 +-
 .../Sema/attr-counted-by-or-null-last-field.c |   4 +-
 ...unted-by-or-null-late-parsed-struct-ptrs.c |   5 +-
 ...ruct-ptrs-completable-incomplete-pointee.c | 631 ++++++++++++++++++
 .../attr-counted-by-or-null-struct-ptrs.c     |  12 +-
 ...ruct-ptrs-completable-incomplete-pointee.c | 630 +++++++++++++++++
 clang/test/Sema/attr-counted-by-struct-ptrs.c |  12 +-
 clang/test/Sema/attr-counted-by-vla.c         |   3 +-
 15 files changed, 1654 insertions(+), 27 deletions(-)
 create mode 100644 clang/test/Sema/attr-counted-by-or-null-struct-ptrs-completable-incomplete-pointee.c
 create mode 100644 clang/test/Sema/attr-counted-by-struct-ptrs-completable-incomplete-pointee.c

diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 1ecd64539e2de..20ff529c7e0c6 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -2446,6 +2446,26 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
     return !isFunctionType();
   }
 
+  /// \returns True if the type is incomplete and it is also a type that
+  /// cannot be completed by a later type definition.
+  ///
+  /// E.g. For `void` this is true but for `struct ForwardDecl;` this is false
+  /// because a definition for `ForwardDecl` could be provided later on in the
+  /// translation unit.
+  ///
+  /// Note even for types that this function returns true for it is still
+  /// possible for the declarations that contain this type to later have a
+  /// complete type in a translation unit. E.g.:
+  ///
+  /// \code{.c}
+  /// // This decl has type 'char[]' which is incomplete and cannot be later
+  /// // completed by another by another type declaration.
+  /// extern char foo[];
+  /// // This decl now has complete type 'char[5]'.
+  /// char foo[5]; // foo has a complete type
+  /// \endcode
+  bool isAlwaysIncompleteType() const;
+
   /// Determine whether this type is an object type.
   bool isObjectType() const {
     // C++ [basic.types]p8:
@@ -3376,6 +3396,8 @@ class CountAttributedType final
   static bool classof(const Type *T) {
     return T->getTypeClass() == CountAttributed;
   }
+
+  StringRef getAttributeName(bool WithMacroPrefix) const;
 };
 
 /// Represents a type which was implicitly adjusted by the semantic
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 90cf1bd9b2588..8b8d3d7f6903b 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -6738,6 +6738,30 @@ def err_counted_by_attr_pointee_unknown_size : Error<
     "%select{|. This will be an error in a future compiler version}3"
     ""
   "}2">;
+def err_counted_by_on_incomplete_type_on_assign : Error <
+  "cannot %select{"
+    "assign to %select{object|'%1'}2 with|" // AA_Assigning,
+    "pass argument to %select{parameter|parameter '%1'}2 with|" // AA_Passing,
+    "return|" // AA_Returning,
+    "convert to|" // AA_Converting (UNUSED)
+    "%select{|implicitly }3initialize %select{object|'%1'}2 with|" // AA_Initializing,
+    "pass argument to parameter with|" // AA_Sending (UNUSED)
+    "cast to|" // AA_Casting (UNUSED)
+    "pass argument to parameter with" // AA_Passing_CFAudited (UNUSED)
+  "}0 '%5' attributed type %4 because the pointee type %6 is incomplete">;
+
+def err_counted_by_on_incomplete_type_on_use : Error <
+  "cannot %select{"
+    "use '%1' with '%4' attributed|" // Generic expr
+    "call '%1' with '%4' attributed return" // CallExpr
+  "}0 type %2 because the pointee type %3 is incomplete">;
+
+def note_counted_by_consider_completing_pointee_ty : Note<
+  "consider providing a complete definition for %0">;
+  
+def note_counted_by_consider_using_sized_by : Note<
+  "consider using '__sized_by%select{|_or_null}0' instead of "
+  "'__counted_by%select{|_or_null}0'">;
 
 def warn_counted_by_attr_elt_type_unknown_size :
   Warning<err_counted_by_attr_pointee_unknown_size.Summary>,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index a757f4c6430ae..96d81e618494a 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -2100,6 +2100,50 @@ class Sema final : public SemaBase {
   bool CheckCountedByAttrOnField(FieldDecl *FD, Expr *E, bool CountInBytes,
                                  bool OrNull);
 
+  /// Perform Bounds Safety Semantic checks for assigning to a `__counted_by` or
+  /// `__counted_by_or_null` pointer type \param LHSTy.
+  ///
+  /// \param LHSTy The type being assigned to. Checks will only be performed if
+  ///              the type is a `counted_by` or `counted_by_or_null ` pointer.
+  /// \param RHSExpr The expression being assigned from.
+  /// \param Action The type assignment being performed
+  /// \param Loc The SourceLocation to use for error diagnostics
+  /// \param Assignee The ValueDecl being assigned. This is used to compute
+  ///        the name of the assignee. If the assignee isn't known this can
+  ///        be set to nullptr.
+  /// \param ShowFullyQualifiedAssigneeName If set to true when using \p
+  ///        Assignee to compute the name of the assignee use the fully
+  ///        qualified name, otherwise use the unqualified name.
+  ///
+  /// \returns True iff no diagnostic where emitted, false otherwise.
+  bool BoundsSafetyCheckAssignmentToCountAttrPtr(
+      QualType LHSTy, Expr *RHSExpr, AssignmentAction Action,
+      SourceLocation Loc, const ValueDecl *Assignee,
+      bool ShowFullyQualifiedAssigneeName);
+
+  /// Perform Bounds Safety Semantic checks for initializing a Bounds Safety
+  /// pointer.
+  ///
+  /// \param Entity The entity being initialized
+  /// \param Kind The kind of initialization being performed
+  /// \param Action The type assignment being performed
+  /// \param LHSTy The type being assigned to. Checks will only be performed if
+  ///              the type is a `counted_by` or `counted_by_or_null ` pointer.
+  /// \param RHSExpr The expression being used for initialization.
+  ///
+  /// \returns True iff no diagnostic where emitted, false otherwise.
+  bool BoundsSafetyCheckInitialization(const InitializedEntity &Entity,
+                                       const InitializationKind &Kind,
+                                       AssignmentAction Action,
+                                       QualType LHSType, Expr *RHSExpr);
+
+  /// Perform Bounds Safety semantic checks for uses of invalid uses counted_by
+  /// or counted_by_or_null pointers in \param E.
+  ///
+  /// \param E the expression to check
+  ///
+  /// \returns True iff no diagnostic where emitted, false otherwise.
+  bool BoundsSafetyCheckUseOfCountAttrPtr(const Expr *E);
   ///@}
 
   //
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 42e94d66d1a13..fe1dc7e2fe786 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2499,6 +2499,22 @@ bool Type::isIncompleteType(NamedDecl **Def) const {
   }
 }
 
+bool Type::isAlwaysIncompleteType() const {
+  if (!isIncompleteType())
+    return false;
+
+  // Forward declarations of structs, classes, enums, and unions could be later
+  // completed in a compilation unit by providing a type definition.
+  if (getAsTagDecl())
+    return false;
+
+  // Other types are incompletable.
+  //
+  // E.g. `char[]` and `void`. The type is incomplete and no future
+  // type declarations can make the type complete.
+  return true;
+}
+
 bool Type::isSizelessBuiltinType() const {
   if (isSizelessVectorType())
     return true;
@@ -3940,6 +3956,31 @@ CountAttributedType::CountAttributedType(
     DeclSlot[i] = CoupledDecls[i];
 }
 
+StringRef CountAttributedType::getAttributeName(bool WithMacroPrefix) const {
+// TODO: This method isn't really ideal because it doesn't return the spelling
+// of the attribute that was used in the user's code. This method is used for
+// diagnostics so the fact it doesn't use the spelling of the attribute in
+// the user's code could be confusing (#113585).
+#define ENUMERATE_ATTRS(PREFIX)                                                \
+  do {                                                                         \
+    if (isCountInBytes()) {                                                    \
+      if (isOrNull())                                                          \
+        return PREFIX "sized_by_or_null";                                      \
+      return PREFIX "sized_by";                                                \
+    }                                                                          \
+    if (isOrNull())                                                            \
+      return PREFIX "counted_by_or_null";                                      \
+    return PREFIX "counted_by";                                                \
+  } while (0)
+
+  if (WithMacroPrefix)
+    ENUMERATE_ATTRS("__");
+  else
+    ENUMERATE_ATTRS("");
+
+#undef ENUMERATE_ATTRS
+}
+
 TypedefType::TypedefType(TypeClass tc, const TypedefNameDecl *D,
                          QualType Underlying, QualType can)
     : Type(tc, can, toSemanticDependence(can->getDependence())),
diff --git a/clang/lib/Sema/SemaBoundsSafety.cpp b/clang/lib/Sema/SemaBoundsSafety.cpp
index d63a2389ea11d..1b356eec6b78a 100644
--- a/clang/lib/Sema/SemaBoundsSafety.cpp
+++ b/clang/lib/Sema/SemaBoundsSafety.cpp
@@ -11,6 +11,9 @@
 /// (e.g. `counted_by`)
 ///
 //===----------------------------------------------------------------------===//
+#include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Sema/Initialization.h"
 #include "clang/Sema/Sema.h"
 
 namespace clang {
@@ -102,7 +105,36 @@ bool Sema::CheckCountedByAttrOnField(FieldDecl *FD, Expr *E, bool CountInBytes,
   // only `PointeeTy->isStructureTypeWithFlexibleArrayMember()` is reachable
   // when `FieldTy->isArrayType()`.
   bool ShouldWarn = false;
-  if (PointeeTy->isIncompleteType() && !CountInBytes) {
+  if (PointeeTy->isAlwaysIncompleteType() && !CountInBytes) {
+    // In general using `counted_by` or `counted_by_or_null` on
+    // pointers where the pointee is an incomplete type are problematic. This is
+    // because it isn't possible to compute the pointer's bounds without knowing
+    // the pointee type size. At the same time it is common to forward declare
+    // types in header files.
+    //
+    // E.g.:
+    //
+    // struct Handle;
+    // struct Wrapper {
+    //   size_t size;
+    //   struct Handle* __counted_by(count) handles;
+    // }
+    //
+    // To allow the above code pattern but still prevent the pointee type from
+    // being incomplete in places where bounds checks are needed the following
+    // scheme is used:
+    //
+    // * When the pointee type might not always be an incomplete type (i.e.
+    // a type that is currently incomplete but might be completed later
+    // on in the translation unit) the attribute is allowed by this method
+    // but later uses of the FieldDecl are checked that the pointee type
+    // is complete see `BoundsSafetyCheckAssignmentToCountAttrPtr`,
+    // `BoundsSafetyCheckInitialization`, and
+    // `BoundsSafetyCheckUseOfCountAttrPtr`
+    //
+    // * When the pointee type is always an incomplete type (e.g.
+    // `void`) the attribute is disallowed by this method because we know the
+    // type can never be completed so there's no reason to allow it.
     InvalidTypeKind = CountedByInvalidPointeeTypeKind::INCOMPLETE;
   } else if (PointeeTy->isSizelessType()) {
     InvalidTypeKind = CountedByInvalidPointeeTypeKind::SIZELESS;
@@ -186,4 +218,182 @@ bool Sema::CheckCountedByAttrOnField(FieldDecl *FD, Expr *E, bool CountInBytes,
   return false;
 }
 
+static void EmitIncompleteCountedByPointeeNotes(Sema &S,
+                                                const CountAttributedType *CATy,
+                                                NamedDecl *IncompleteTyDecl) {
+  assert(IncompleteTyDecl == nullptr || isa<TypeDecl>(IncompleteTyDecl));
+
+  if (IncompleteTyDecl) {
+    // Suggest completing the pointee type if its a named typed (i.e.
+    // IncompleteTyDecl isn't nullptr). Suggest this first as it is more likely
+    // to be the correct fix.
+    //
+    // Note the `IncompleteTyDecl` type is the underlying type which might not
+    // be the same as `CATy->getPointeeType()` which could be a typedef.
+    //
+    // The diagnostic printed will be at the location of the underlying type but
+    // the diagnostic text will print the type of `CATy->getPointeeType()` which
+    // could be a typedef name rather than the underlying type. This is ok
+    // though because the diagnostic will print the underlying type name too.
+    S.Diag(IncompleteTyDecl->getBeginLoc(),
+           diag::note_counted_by_consider_completing_pointee_ty)
+        << CATy->getPointeeType();
+  }
+
+  // Suggest using __sized_by(_or_null) instead of __counted_by(_or_null) as
+  // __sized_by(_or_null) doesn't have the complete type restriction.
+  //
+  // We use the source range of the expression on the CountAttributedType as an
+  // approximation for the source range of the attribute. This isn't quite right
+  // but isn't easy to fix right now.
+  //
+  // TODO: Implement logic to find the relevant TypeLoc for the attribute and
+  // get the SourceRange from that (#113582).
+  //
+  // TODO: We should emit a fix-it here.
+  SourceRange AttrSrcRange = CATy->getCountExpr()->getSourceRange();
+  S.Diag(AttrSrcRange.getBegin(), diag::note_counted_by_consider_using_sized_by)
+      << CATy->isOrNull() << AttrSrcRange;
+}
+
+static std::tuple<const CountAttributedType *, QualType>
+GetCountedByAttrOnIncompletePointee(QualType Ty, NamedDecl **ND) {
+  auto *CATy = Ty->getAs<CountAttributedType>();
+  // Incomplete pointee type is only a problem for
+  // counted_by/counted_by_or_null
+  if (!CATy || CATy->isCountInBytes())
+    return {};
+
+  auto PointeeTy = CATy->getPointeeType();
+  if (PointeeTy.isNull()) {
+    // Reachable if `CountAttributedType` wraps an IncompleteArrayType
+    return {};
+  }
+
+  if (!PointeeTy->isIncompleteType(ND))
+    return {};
+
+  return {CATy, PointeeTy};
+}
+
+/// Perform Checks for assigning to a `__counted_by` or
+/// `__counted_by_or_null` pointer type \param LHSTy where the pointee type
+/// is incomplete which is invalid.
+///
+/// \param S The Sema instance.
+/// \param LHSTy The type being assigned to. Checks will only be performed if
+///              the type is a `counted_by` or `counted_by_or_null ` pointer.
+/// \param RHSExpr The expression being assigned from.
+/// \param Action The type assignment being performed
+/// \param Loc The SourceLocation to use for error diagnostics
+/// \param Assignee The ValueDecl being assigned. This is used to compute
+///        the name of the assignee. If the assignee isn't known this can
+///        be set to nullptr.
+/// \param ShowFullyQualifiedAssigneeName If set to true when using \p
+///        Assignee to compute the name of the assignee use the fully
+///        qualified name, otherwise use the unqualified name.
+///
+/// \returns True iff no diagnostic where emitted, false otherwise.
+static bool CheckAssignmentToCountAttrPtrWithIncompletePointeeTy(
+    Sema &S, QualType LHSTy, Expr *RHSExpr, AssignmentAction Action,
+    SourceLocation Loc, const ValueDecl *Assignee,
+    bool ShowFullyQualifiedAssigneeName) {
+  NamedDecl *IncompleteTyDecl = nullptr;
+  auto [CATy, PointeeTy] =
+      GetCountedByAttrOnIncompletePointee(LHSTy, &IncompleteTyDecl);
+  if (!CATy)
+    return true;
+
+  std::string AssigneeStr;
+  if (Assignee) {
+    if (ShowFullyQualifiedAssigneeName) {
+      AssigneeStr = Assignee->getQualifiedNameAsString();
+    } else {
+      AssigneeStr = Assignee->getNameAsString();
+    }
+  }
+
+  S.Diag(Loc, diag::err_counted_by_on_incomplete_type_on_assign)
+      << static_cast<int>(Action) << AssigneeStr << (AssigneeStr.size() > 0)
+      << isa<ImplicitValueInitExpr>(RHSExpr) << LHSTy
+      << CATy->getAttributeName(/*WithMacroPrefix=*/true) << PointeeTy
+      << CATy->isOrNull() << RHSExpr->getSourceRange();
+
+  EmitIncompleteCountedByPointeeNotes(S, CATy, IncompleteTyDecl);
+  return false; // check failed
+}
+
+bool Sema::BoundsSafetyCheckAssignmentToCountAttrPtr(
+    QualType LHSTy, Expr *RHSExpr, AssignmentAction Action, SourceLocation Loc,
+    const ValueDecl *Assignee, bool ShowFullyQualifiedAssigneeName) {
+  return CheckAssignmentToCountAttrPtrWithIncompletePointeeTy(
+      *this, LHSTy, RHSExpr, Action, Loc, Assignee,
+      ShowFullyQualifiedAssigneeName);
+}
+
+bool Sema::BoundsSafetyCheckInitialization(const InitializedEntity &Entity,
+                                           const InitializationKind &Kind,
+                                           AssignmentAction Action,
+                                           QualType LHSType, Expr *RHSExpr) {
+  auto SL = Kind.getLocation();
+
+  // Note: We don't call `BoundsSafetyCheckAssignmentToCountAttrPtr` here
+  // because we need conditionalize what is checked. In downstream
+  // Clang `counted_by` is supported on variable definitions and in that
+  // implementation an error diagnostic will be emitted on the variable
+  // definition if the pointee is an incomplete type. To avoid warning about the
+  // same problem twice (once when the variable is defined, once when Sema
+  // checks the initializer) we skip checking the initializer if it's a
+  // variable.
+  if (Action == AssignmentAction::Initializing &&
+      Entity.getKind() != InitializedEntity::EK_Variable) {
+
+    if (!CheckAssignmentToCountAttrPtrWithIncompletePointeeTy(
+            *this, LHSType, RHSExpr, Action, SL,
+            dyn_cast_or_null<ValueDecl>(Entity.getDecl()),
+            /*ShowFullQualifiedAssigneeName=*/true)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool Sema::BoundsSafetyCheckUseOfCountAttrPtr(const Expr *E) {
+  QualType T = E->getType();
+  if (!T->isPointerType())
+    return true;
+
+  NamedDecl *IncompleteTyDecl = nullptr;
+  auto [CATy, PointeeTy] =
+      GetCountedByAttrOnIncompletePointee(T, &IncompleteTyDecl);
+  if (!CATy)
+    return true;
+
+  // Generate a string for the diagnostic that describes the "use".
+  // The string is specialized for direct calls to produce a better
+  // diagnostic.
+  SmallString<64> UseStr;
+  bool IsDirectCall = false;
+  if (const auto *CE = dyn_cast<CallExpr>(E->IgnoreParens())) {
+    if (const auto *FD = CE->getDirectCallee()) {
+      UseStr = FD->getName();
+      IsDirectCall = true;
+    }
+  }
+
+  if (!IsDirectCall) {
+    llvm::raw_svector_ostream SS(UseStr);
+    E->printPretty(SS, nullptr, getPrintingPolicy());
+  }
+
+  Diag(E->getBeginLoc(), diag::err_counted_by_on_incomplete_type_on_use)
+      << IsDirectCall << UseStr << T << PointeeTy
+      << CATy->getAttributeName(/*WithMacroPrefix=*/true) << CATy->isOrNull()
+      << E->getSourceRange();
+
+  EmitIncompleteCountedByPointeeNotes(*this, CATy, IncompleteTyDecl);
+  return false;
+}
+
 } // namespace clang
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 9cd1a7e45fb64..01a021443c94f 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -732,6 +732,9 @@ ExprResult Sema::DefaultLvalueConversion(Expr *E) {
   if (E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
     Cleanup.setExprNeedsCleanups(true);
 
+  if (!BoundsSafetyCheckUseOfCountAttrPtr(Res.get()))
+    return ExprError();
+
   // C++ [conv.lval]p3:
   //   If T is cv std::nullptr_t, the result is a null pointer constant.
   CastKind CK = T->isNullPtrType() ? CK_NullToPointer : CK_LValueToRValue;
@@ -13882,6 +13885,26 @@ QualType Sema::CheckAssignmentOperands(Expr *LHSExpr, ExprResult &RHS,
   QualType LHSType = LHSExpr->getType();
   QualType RHSType = CompoundType.isNull() ? RHS.get()->getType() :
                                              CompoundType;
+
+  if (RHS.isUsable()) {
+    // Even if this check fails don't return early to allow the best
+    // possible error recovery and to allow any subsequent diagnostics to
+    // work.
+    const ValueDecl *Assignee = nullptr;
+    bool ShowFullyQualifiedAssigneeName = false;
+    // In simple cases describe what is being assigned to
+    if (auto *DR = dyn_cast<DeclRefExpr>(LHSExpr->IgnoreParenCasts())) {
+      Assignee = DR->getDecl();
+    } else if (auto *ME = dyn_cast<MemberExpr>(LHSExpr->IgnoreParenCasts())) {
+      Assignee = ME->getMemberDecl();
+      ShowFullyQualifiedAssigneeName = true;
+    }
+
+    BoundsSafetyCheckAssignmentToCountAttrPtr(
+        LHSType, RHS.get(), AssignmentAction::Assigning, Loc, Assignee,
+        ShowFullyQualifiedAssigneeName);
+  }
+
   // OpenCL v1.2 s6.1.1.1 p2:
   // The half data type can only be used to declare a pointer to a buffer that
   // contains half values
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 9e802a175bb05..0910a820438b0 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -8302,6 +8302,12 @@ ExprResult InitializationSequence::Perform(Sema &S,
             Kind.getRange().getEnd());
       } else {
         CurInit = new (S.Context) ImplicitValueInitExpr(Step->Type);
+        // Note the return value isn't used to return a ExprError() when
+        // initialization fails . For struct initialization allows all field
+        // assignments to be checked rather than bailing on the first error.
+        S.BoundsSafetyCheckInitialization(Entity, Kind,
+                                          AssignmentAction::Initializing,
+                                          Step->Type, CurInit.get());
       }
       break;
     }
@@ -8348,6 +8354,13 @@ ExprResult InitializationSequence::Perform(Sema &S,
         }
       }
 
+      // Note the return value isn't used to return a ExprError() when
+      // initialization fails. For struct initialization this allows all field
+      // assignments to be checked rather than bailing on the first error.
+      S.BoundsSafetyCheckInitialization(Entity, Kind,
+                                        getAssignmentAction(Entity, true),
+                                        Step->Type, InitialCurInit.get());
+
       bool Complained;
       if (S.DiagnoseAssignmentResult(ConvTy, Kind.getLocation(),
                                      Step->Type, SourceType,
diff --git a/clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c b/clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c
index 9ff3b080f6576..8d4e0c510603a 100644
--- a/clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c
+++ b/clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c
@@ -19,13 +19,12 @@ struct on_member_pointer_complete_ty {
 };
 
 struct on_member_pointer_incomplete_ty {
-  struct size_unknown * buf __counted_by(count); // expected-error{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
+  struct size_unknown * buf __counted_by(count); // ok
   int count;
 };
 
 struct on_member_pointer_const_incomplete_ty {
-  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
-  const struct size_unknown * buf __counted_by(count);
+  const struct size_unknown * buf __counted_by(count); // ok
   int count;
 };
 
diff --git a/clang/test/Sema/attr-counted-by-or-null-last-field.c b/clang/test/Sema/attr-counted-by-or-null-last-field.c
index 12c0b6de44f72..60a1f571b19e9 100644
--- a/clang/test/Sema/attr-counted-by-or-null-last-field.c
+++ b/clang/test/Sema/attr-counted-by-or-null-last-field.c
@@ -118,12 +118,12 @@ struct annotated_with_anon_struct {
 struct count_unknown;
 struct on_member_ptr_incomplete_ty_ty_pos {
   int count;
-  struct count_unknown * ptr __counted_by_or_null(count); // expected-error {{'counted_by_or_null' cannot be applied to a pointer with pointee of unknown size because 'struct count_unknown' is an incomplete type}}
+  struct count_unknown * ptr __counted_by_or_null(count); // ok
 };
 
 struct on_member_ptr_incomplete_const_ty_ty_pos {
   int count;
-  const struct count_unknown * ptr __counted_by_or_null(count); // expected-error {{'counted_by_or_null' cannot be applied to a pointer with pointee of unknown size because 'const struct count_unknown' is an incomplete type}}
+  const struct count_unknown * ptr __counted_by_or_null(count); // ok
 };
 
 struct on_member_ptr_void_ty_ty_pos {
diff --git a/clang/test/Sema/attr-counted-by-or-null-late-parsed-struct-ptrs.c b/clang/test/Sema/attr-counted-by-or-null-late-parsed-struct-ptrs.c
index 95f517e3144f7..2150c81f9e9be 100644
--- a/clang/test/Sema/attr-counted-by-or-null-late-parsed-struct-ptrs.c
+++ b/clang/test/Sema/attr-counted-by-or-null-late-parsed-struct-ptrs.c
@@ -20,13 +20,12 @@ struct on_member_pointer_complete_ty {
 };
 
 struct on_member_pointer_incomplete_ty {
-  struct size_unknown * buf __counted_by_or_null(count); // expected-error{{'counted_by_or_null' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
+  struct size_unknown * buf __counted_by_or_null(count); // ok
   int count;
 };
 
 struct on_member_pointer_const_incomplete_ty {
-  // expected-error@+1{{'counted_by_or_null' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
-  const struct size_unknown * buf __counted_by_or_null(count);
+  const struct size_unknown * buf __counted_by_or_null(count); // ok
   int count;
 };
 
diff --git a/clang/test/Sema/attr-counted-by-or-null-struct-ptrs-completable-incomplete-pointee.c b/clang/test/Sema/attr-counted-by-or-null-struct-ptrs-completable-incomplete-pointee.c
new file mode 100644
index 0000000000000..cff5a14c70b99
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-or-null-struct-ptrs-completable-incomplete-pointee.c
@@ -0,0 +1,631 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fexperimental-late-parse-attributes -fsyntax-only -verify %s
+
+#define __counted_by_or_null(f)  __attribute__((counted_by_or_null(f)))
+
+// =============================================================================
+// # Struct incomplete type with attribute in the decl position
+// =============================================================================
+
+// Note: This could be considered misleading. The typedef isn't actually on this
+// line. Also note the discrepancy in diagnostic count (27 vs 51) is due to
+// the pointer arithmetic on incomplete pointee type diagnostic always using
+// diagnostic text that refers to the underlying forward decl, even when the
+// typedef is used.
+// expected-note@+3 27{{consider providing a complete definition for 'Incomplete_t' (aka 'struct IncompleteTy')}}
+// The 'forward declaration' notes come from 'arithmetic on a pointer to an incomplete type' errors
+// expected-note@+1 24{{forward declaration of 'struct IncompleteTy'}}
+struct IncompleteTy; // expected-note 27{{consider providing a complete definition for 'struct IncompleteTy'}}
+
+typedef struct IncompleteTy Incomplete_t; 
+
+struct CBBufDeclPos {
+  int count;
+  struct IncompleteTy* buf __counted_by_or_null(count); // OK expected-note 27{{consider using '__sized_by_or_null' instead of '__counted_by_or_null'}}
+  Incomplete_t* buf_typedef __counted_by_or_null(count); // OK expected-note 27{{consider using '__sized_by_or_null' instead of '__counted_by_or_null'}}
+};
+
+void consume_struct_IncompleteTy(struct IncompleteTy* buf);
+
+int idx(void);
+
+
+
+void test_CBBufDeclPos(struct CBBufDeclPos* ptr) {
+  // ===========================================================================
+  // ## Local variable initialization
+  // ===========================================================================
+  struct CBBufDeclPos explicit_desig_init = {
+    .count = 0,
+    // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+    .buf = 0x0,
+    // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+    .buf_typedef = 0x0
+  };
+  // Variable is not currently marked as invalid so uses of the variable allows
+  // diagnostics to fire.
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  explicit_desig_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot use 'explicit_desig_init.buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  void *tmp = explicit_desig_init.buf;
+  // expected-error@+1{{cannot use 'explicit_desig_init.buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  void *tmp2 = explicit_desig_init.buf_typedef;
+
+  struct CBBufDeclPos partial_explicit_desig_init = {
+    .count = 0,
+    // .buf and .buf_typedef are implicit zero initialized
+    // expected-error@+2{{cannot implicitly initialize 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+    // expected-error@+1{{cannot implicitly initialize 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  };
+
+  struct CBBufDeclPos implicit_full_init = {
+    0
+    // expected-error@+2{{cannot implicitly initialize 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+    // expected-error@+1{{cannot implicitly initialize 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  };
+  // Variable is not currently marked as invalid so uses of the variable allows
+  // diagnostics to fire.
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  implicit_full_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  implicit_full_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot use 'implicit_full_init.buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  void* tmp3 = implicit_full_init.buf;
+  // expected-error@+1{{cannot use 'implicit_full_init.buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  void* tmp4 = implicit_full_init.buf_typedef;
+  
+  struct CBBufDeclPos explicit_non_desig_init = {
+    0,
+    // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+    0x0,
+    // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+    0x0
+  };
+
+
+
+
+  // ===========================================================================
+  // ## Assignment to fields
+  // ===========================================================================
+  struct CBBufDeclPos uninit;
+  uninit.count = 0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  uninit.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  uninit.buf_typedef = 0x0;
+  ptr->count = 0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  ptr->buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  ptr->buf_typedef = 0x0;
+
+
+  // ===========================================================================
+  // ## Make sure modifying the fields through other assignment operators is not
+  //    allowed
+  // ===========================================================================
+  uninit.buf++; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ++uninit.buf; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  uninit.buf += 1; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  uninit.buf_typedef++; // // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ++uninit.buf_typedef; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  uninit.buf_typedef -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  
+  uninit.buf--; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  --uninit.buf; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  uninit.buf -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  uninit.buf_typedef--; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  --uninit.buf_typedef; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  uninit.buf_typedef -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+
+  ptr->buf++; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ++ptr->buf; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ptr->buf += 1; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ptr->buf--; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  --ptr->buf; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ptr->buf -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+
+  ptr->buf_typedef++; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ++ptr->buf_typedef; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ptr->buf_typedef += 1; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ptr->buf_typedef--; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  --ptr->buf_typedef; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ptr->buf_typedef -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+
+  // ===========================================================================
+  // ## Use of fields in expressions
+  // ===========================================================================
+  // expected-error@+2{{cannot use 'uninit.buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  void* addr = 
+    ((char*) uninit.buf ) + 1;
+  // expected-error@+2{{cannot use 'uninit.buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  void* addr_typedef = 
+    ((char*) uninit.buf_typedef ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  void* addr_ptr = 
+    ((char*) ptr->buf ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  void* addr_ptr_typedef = 
+    ((char*) ptr->buf_typedef ) + 1;
+
+
+  // ===========================================================================
+  // ## Take address of fields
+  // ===========================================================================
+  // TODO: This should be forbidden, not because of the incomplete pointee type
+  // but because in the -fbounds-safety language model the address of a
+  // `counted_by` pointer cannot be taken to avoid it being possible to modify
+  // the `counted_by` pointer through another pointer. Whether or not this
+  // should be forbidden when `-fbounds-safety` is off is TBD.
+  //
+  // The incomplete pointee type isn't actually a problem here for
+  // `-fbounds-safety` because taking the address of a pointer returns a pointer
+  // that have the bounds of a single `void*`, so bounds checks on the resulting
+  // pointer don't need to know `sizeof(struct IncompleteTy)` but instead
+  // `sizeof(struct IncompleteTy* buf __counted_by_or_null(count))` which is just the
+  // size of a pointer.
+  void* take_addr = &uninit.buf;
+  void* take_addr_typedef = &uninit.buf_typedef;
+  void* take_addr_ptr = &ptr->buf;
+  void* take_addr_ptr_typedef = &ptr->buf_typedef;
+
+  // expected-error@+1{{cannot use 'uninit.buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  struct IncompleteTy* addr_elt_zero = &uninit.buf[0];
+  // expected-error@+1{{cannot use 'uninit.buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  struct IncompleteTy* addr_elt_idx = &uninit.buf[idx()];
+
+  // expected-error@+1{{cannot use 'uninit.buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  struct IncompleteTy* addr_elt_zero_typedef = &uninit.buf_typedef[0];
+  // expected-error@+1{{cannot use 'uninit.buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  struct IncompleteTy* addr_elt_idx_typedef = &uninit.buf_typedef[idx()];
+
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  struct IncompleteTy* addr_elt_zero_ptr = &ptr->buf[0];
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  struct IncompleteTy* addr_elt_idx_ptr = &ptr->buf[idx()];
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  struct IncompleteTy* addr_elt_zero_ptr_typedef = &ptr->buf_typedef[0];
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  struct IncompleteTy* addr_elt_idx_ptr_typedef = &ptr->buf_typedef[idx()];
+
+
+  // ===========================================================================
+  // ## Use fields as call arguments
+  // ===========================================================================
+  // expected-error@+1{{cannot use 'uninit.buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  consume_struct_IncompleteTy(uninit.buf);
+  // expected-error@+1{{cannot use 'uninit.buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  consume_struct_IncompleteTy(uninit.buf_typedef);
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  consume_struct_IncompleteTy(ptr->buf);
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  consume_struct_IncompleteTy(ptr->buf_typedef);
+
+  // ===========================================================================
+  // ## Use [] operator on fields
+  // ===========================================================================
+  // expected-error@+1 2{{cannot use 'uninit.buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  uninit.buf[0] = uninit.buf[1];
+  // expected-error@+1 2{{cannot use 'uninit.buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  uninit.buf_typedef[0] = uninit.buf_typedef[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  ptr->buf[0] = ptr->buf[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+
+// =============================================================================
+// ## Global initialization
+// =============================================================================
+
+struct CBBufDeclPos global_explicit_desig_init = {
+  .count = 0,
+  // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  .buf = 0x0,
+  // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  .buf_typedef = 0x0
+};
+
+void use_global_explicit_desig_init(void) {
+  // Variable isn't marked as invalid so diagnostics still fire
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  global_explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  global_explicit_desig_init.buf_typedef = 0x0;
+}
+
+struct CBBufDeclPos global_partial_explicit_desig_init = {
+  .count = 0,
+  // .buf and .buf_typedef are implicit zero initialized
+  // expected-error@+2{{cannot implicitly initialize 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  // expected-error@+1{{cannot implicitly initialize 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+};
+
+struct CBBufDeclPos global_implicit_full_init = {
+  0
+  // expected-error@+2{{cannot implicitly initialize 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  // expected-error@+1{{cannot implicitly initialize 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+};
+
+struct CBBufDeclPos global_explicit_non_desig_init = {
+  0,
+  // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  0x0,
+  // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_t * __counted_by_or_null(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  0x0
+};
+
+extern struct CBBufDeclPos global_declaration; // OK
+
+// TODO: These tentative definitions are implicitly empty initialized to zero.
+// This should generate an error diagnostic but currently doesn't. There should
+// be a carve out to allow `__counted_by_or_null(0)` which is the only constant count
+// version of the attribute where it is valid to assign NULL.
+struct CBBufDeclPos global_tentative_defn;
+static struct CBBufDeclPos global_tentative_defn2;
+
+// =============================================================================
+// ## Completing the definition of the type allows use of CBBufDeclPos fields
+// =============================================================================
+struct IncompleteTy {
+  int field;
+};
+
+void test_CBBufDeclPos_completed(struct CBBufDeclPos* ptr) {
+  // Initialization is ok
+  struct CBBufDeclPos explicit_desig_init = {
+    .count = 0,
+    .buf = 0x0,
+    .buf_typedef = 0x0
+  };
+
+  struct CBBufDeclPos partial_explicit_desig_init = {
+    .count = 0,
+    // .buf and .buf_typedef are implicit zero initialized
+  };
+
+  struct CBBufDeclPos implicit_full_init = {0};
+  
+  struct CBBufDeclPos explicit_non_desig_init = {
+    0,
+    0x0,
+    0x0
+  };
+
+  // Assignment to fields is ok
+  ptr->buf = 0x0;
+  ptr->buf_typedef = 0x0;
+
+  // Use of fields in expressions is ok
+  void* tmp = ptr->buf;
+  void* tmp2 = ptr->buf_typedef;
+
+  // Take address of fields is ok
+  void* take_addr_ptr = &ptr->buf;
+  void* take_addr_ptr_typedef = &ptr->buf_typedef;
+
+  struct IncompleteTy* addr_elt_zero_ptr = &ptr->buf[0];
+  struct IncompleteTy* addr_elt_idx_ptr = &ptr->buf[idx()];
+  struct IncompleteTy* addr_elt_zero_ptr_typedef = &ptr->buf_typedef[0];
+  struct IncompleteTy* addr_elt_idx_ptr_typedef = &ptr->buf_typedef[idx()];
+
+  // As call arguments is ok
+  consume_struct_IncompleteTy(ptr->buf);
+  consume_struct_IncompleteTy(ptr->buf_typedef);
+
+  // In [] operator is ok
+  ptr->buf[0] = ptr->buf[1];
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+// Global initialization is ok
+
+struct CBBufDeclPos global_explicit_desig_init_completed = {
+  .count = 0,
+  .buf = 0x0,
+  .buf_typedef = 0x0
+};
+
+struct CBBufDeclPos global_partial_explicit_desig_init_completed = {
+  .count = 0,
+  // .buf and .buf_typedef are implicit zero initialized
+};
+
+struct CBBufDeclPos global_implicit_full_init_completed = {0};
+
+struct CBBufDeclPos global_explicit_non_desig_init_completed = {
+  0,
+  0x0,
+  0x0
+};
+
+extern struct CBBufDeclPos global_declaration;
+struct CBBufDeclPos global_tentative_defn;
+static struct CBBufDeclPos global_tentative_defn2;
+
+// =============================================================================
+// # Struct incomplete type with attribute in the pointer position
+// =============================================================================
+
+// expected-note@+1 8{{consider providing a complete definition for 'Incomplete_ty2' (aka 'struct IncompleteTy2')}}
+struct IncompleteTy2; // expected-note 8{{consider providing a complete definition for 'struct IncompleteTy2'}}
+typedef struct IncompleteTy2 Incomplete_ty2;
+
+void consume_struct_IncompleteTy2(struct IncompleteTy2* buf);
+
+struct CBBufTyPos {
+  int count;
+  struct IncompleteTy2* __counted_by_or_null(count) buf ; // OK expected-note 8{{consider using '__sized_by_or_null' instead of '__counted_by_or_null'}}
+  Incomplete_ty2 *__counted_by_or_null(count) buf_typedef; // OK expected-note 8{{consider using '__sized_by_or_null' instead of '__counted_by_or_null'}}
+};
+
+void use_CBBufTyPos(struct CBBufTyPos* ptr) {
+  struct CBBufTyPos explicit_desig_init = {
+    .count = 0,
+    // expected-error@+1{{cannot initialize 'CBBufTyPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+    .buf = 0x0,
+    // expected-error@+1{{cannot initialize 'CBBufTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_ty2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+    .buf_typedef = 0x0
+  };
+
+  // Assignment
+  // expected-error@+1{{cannot assign to 'CBBufTyPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_ty2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  explicit_desig_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufTyPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  ptr->buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_ty2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  ptr->buf_typedef = 0x0;
+
+  // Use
+  // expected-error@+2{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  void* addr = 
+    ((char*) ptr->buf ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_ty2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  void* addr_typedef = 
+    ((char*) ptr->buf_typedef ) + 1;
+
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  consume_struct_IncompleteTy2(ptr->buf);
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_ty2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  consume_struct_IncompleteTy2(ptr->buf_typedef);
+
+  // expected-error@+1 2{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  ptr->buf[0] = ptr->buf[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_ty2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+struct CBBufTyPos global_explicit_desig_init_struct_type_pos = {
+  .count = 0,
+  // expected-error@+1{{cannot initialize 'CBBufTyPos::buf' with '__counted_by_or_null' attributed type 'struct IncompleteTy2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  .buf = 0x0,
+  // expected-error@+1{{cannot initialize 'CBBufTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'Incomplete_ty2 * __counted_by_or_null(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  .buf_typedef = 0x0
+};
+
+// Defining the type makes `CBBufTyPos` fields usable
+struct IncompleteTy2 {
+  int field;
+};
+
+void use_CBBufTyPos_completed(struct CBBufTyPos* ptr) {
+  ptr->buf = 0x0;
+  ptr->buf_typedef = 0x0;
+  void* addr = ((char*) ptr->buf) + 1;
+  void* addr_typedef = ((char*) ptr->buf_typedef) + 1;
+}
+
+// =============================================================================
+// # union incomplete type
+// =============================================================================
+
+// expected-note@+1 8{{consider providing a complete definition for 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy')}}
+union IncompleteUnionTy; // expected-note 8{{consider providing a complete definition for 'union IncompleteUnionTy'}}
+typedef union IncompleteUnionTy IncompleteUnion_ty;
+
+void consume_struct_IncompleteUnionTy(union IncompleteUnionTy* buf);
+
+struct CBBufUnionTyPos {
+  int count;
+  union IncompleteUnionTy* __counted_by_or_null(count) buf ; // OK expected-note 8{{consider using '__sized_by_or_null' instead of '__counted_by_or_null'}}
+  IncompleteUnion_ty *__counted_by_or_null(count) buf_typedef; // OK expected-note 8{{consider using '__sized_by_or_null' instead of '__counted_by_or_null'}}
+};
+
+void use_CBBufUnionTyPos(struct CBBufUnionTyPos* ptr) {
+  struct CBBufUnionTyPos explicit_desig_init = {
+    .count = 0,
+    // expected-error@+1{{cannot initialize 'CBBufUnionTyPos::buf' with '__counted_by_or_null' attributed type 'union IncompleteUnionTy * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+    .buf = 0x0,
+    // expected-error@+1{{cannot initialize 'CBBufUnionTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteUnion_ty * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+    .buf_typedef = 0x0
+  };
+
+  // Assignment
+  // expected-error@+1{{cannot assign to 'CBBufUnionTyPos::buf' with '__counted_by_or_null' attributed type 'union IncompleteUnionTy * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufUnionTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteUnion_ty * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  explicit_desig_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufUnionTyPos::buf' with '__counted_by_or_null' attributed type 'union IncompleteUnionTy * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  ptr->buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufUnionTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteUnion_ty * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  ptr->buf_typedef = 0x0;
+
+  // Use
+  // expected-error@+2{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'union IncompleteUnionTy * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  void* addr = 
+    ((char*) ptr->buf ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteUnion_ty * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  void* addr_typedef = 
+    ((char*) ptr->buf_typedef ) + 1;
+
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'union IncompleteUnionTy * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  consume_struct_IncompleteUnionTy(ptr->buf);
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteUnion_ty * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  consume_struct_IncompleteUnionTy(ptr->buf_typedef);
+
+  // expected-error@+1 2{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'union IncompleteUnionTy * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  ptr->buf[0] = ptr->buf[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteUnion_ty * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+struct CBBufUnionTyPos global_explicit_desig_init_union_type_pos = {
+  .count = 0,
+  // expected-error@+1{{cannot initialize 'CBBufUnionTyPos::buf' with '__counted_by_or_null' attributed type 'union IncompleteUnionTy * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  .buf = 0x0,
+  // expected-error@+1{{cannot initialize 'CBBufUnionTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteUnion_ty * __counted_by_or_null(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  .buf_typedef = 0x0
+};
+
+// Defining the type makes `CBBufUnionTyPos` fields usable
+union IncompleteUnionTy {
+  int field;
+};
+
+void use_CBBufUnionTyPos_completed(struct CBBufUnionTyPos* ptr) {
+  ptr->buf = 0x0;
+  ptr->buf_typedef = 0x0;
+  void* addr = ((char*) ptr->buf) + 1;
+  void* addr_typedef = ((char*) ptr->buf_typedef) + 1;
+}
+
+// =============================================================================
+// # enum incomplete type
+// =============================================================================
+
+// expected-note@+1 8{{consider providing a complete definition for 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy')}}
+enum IncompleteEnumTy; // expected-note 8{{consider providing a complete definition for 'enum IncompleteEnumTy'}}
+typedef enum IncompleteEnumTy IncompleteEnum_ty;
+
+void consume_struct_IncompleteEnumTy(enum IncompleteEnumTy* buf);
+
+struct CBBufEnumTyPos {
+  int count;
+  enum IncompleteEnumTy* __counted_by_or_null(count) buf ; // OK expected-note 8{{consider using '__sized_by_or_null' instead of '__counted_by_or_null'}}
+  IncompleteEnum_ty *__counted_by_or_null(count) buf_typedef; // OK expected-note 8{{consider using '__sized_by_or_null' instead of '__counted_by_or_null'}}
+};
+
+void use_CBBufEnumTyPos(struct CBBufEnumTyPos* ptr) {
+  struct CBBufEnumTyPos explicit_desig_init = {
+    .count = 0,
+    // expected-error@+1{{cannot initialize 'CBBufEnumTyPos::buf' with '__counted_by_or_null' attributed type 'enum IncompleteEnumTy * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+    .buf = 0x0,
+    // expected-error@+1{{cannot initialize 'CBBufEnumTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteEnum_ty * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+    .buf_typedef = 0x0
+  };
+
+  // Assignment
+  // expected-error@+1{{cannot assign to 'CBBufEnumTyPos::buf' with '__counted_by_or_null' attributed type 'enum IncompleteEnumTy * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufEnumTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteEnum_ty * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  explicit_desig_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufEnumTyPos::buf' with '__counted_by_or_null' attributed type 'enum IncompleteEnumTy * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  ptr->buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufEnumTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteEnum_ty * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  ptr->buf_typedef = 0x0;
+
+  // Use
+  // expected-error@+2{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'enum IncompleteEnumTy * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  void* addr = 
+    ((char*) ptr->buf ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteEnum_ty * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  void* addr_typedef = 
+    ((char*) ptr->buf_typedef ) + 1;
+
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'enum IncompleteEnumTy * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  consume_struct_IncompleteEnumTy(ptr->buf);
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteEnum_ty * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  consume_struct_IncompleteEnumTy(ptr->buf_typedef);
+
+  // expected-error@+1 2{{cannot use 'ptr->buf' with '__counted_by_or_null' attributed type 'enum IncompleteEnumTy * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  ptr->buf[0] = ptr->buf[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteEnum_ty * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+struct CBBufEnumTyPos global_explicit_desig_init_enum_type_pos = {
+  .count = 0,
+  // expected-error@+1{{cannot initialize 'CBBufEnumTyPos::buf' with '__counted_by_or_null' attributed type 'enum IncompleteEnumTy * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  .buf = 0x0,
+  // expected-error@+1{{cannot initialize 'CBBufEnumTyPos::buf_typedef' with '__counted_by_or_null' attributed type 'IncompleteEnum_ty * __counted_by_or_null(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  .buf_typedef = 0x0
+};
+
+// Defining the type makes `CBBufEnumTyPos` fields usable
+enum IncompleteEnumTy {
+  ONE,
+  TWO
+};
+
+void use_CBBufEnumTyPos_completed(struct CBBufEnumTyPos* ptr) {
+  ptr->buf = 0x0;
+  ptr->buf_typedef = 0x0;
+  void* addr = ((char*) ptr->buf) + 1;
+  void* addr_typedef = ((char*) ptr->buf_typedef) + 1;
+}
+
+// Make a complete enum by providing an underlying type
+enum CompleteEnumTy : unsigned;
+typedef enum CompleteEnumTy CompleteEnum_ty;
+struct CBBufEnumTyPos2 {
+  int count;
+  enum CompleteEnumTy* __counted_by_or_null(count) buf;
+  CompleteEnum_ty *__counted_by_or_null(count) buf_typedef;
+};
+
+void use_CBBufEnumTyPos2(struct CBBufEnumTyPos2* ptr) {
+  struct CBBufEnumTyPos2 explicit_desig_init = {
+    .count = 0,
+    .buf = 0x0, // OK
+    .buf_typedef = 0x0 // OK
+  };
+}
+
+// Make a complete enum by providing a concrete declaration
+enum CompleteEnumTy2 {
+  VALUE_ONE,
+  VALUE_TWO
+};
+typedef enum CompleteEnumTy2 CompleteEnum_ty2;
+struct CBBufEnumTyPos3 {
+  int count;
+  enum CompleteEnumTy2* __counted_by_or_null(count) buf;
+  CompleteEnum_ty2 *__counted_by_or_null(count) buf_typedef;
+};
+
+void use_CBBufEnumTyPos3(struct CBBufEnumTyPos3* ptr) {
+  struct CBBufEnumTyPos3 explicit_desig_init = {
+    .count = 0,
+    .buf = 0x0, // OK
+    .buf_typedef = 0x0 // OK
+  };
+}
+
+
+// =============================================================================
+// # Array of __counted_by_or_null pointers
+// =============================================================================
+
+struct IncompleteTy3;
+
+struct CBBufFAMofCountedByPtrs {
+  int size;
+  // TODO: This is misleading. The attribute is written in the type position
+  // but clang currently doesn't treat it like that and it gets treated as
+  // an attribute on the array, rather than on the element type.
+  // expected-error@+1{{'counted_by_or_null' only applies to pointers; did you mean to use 'counted_by'?}}
+  struct IncompleteTy3* __counted_by_or_null(size) arr[];
+};
+
+void arr_of_counted_by_ptr(struct CBBufFAMofCountedByPtrs* ptr) {
+  // TODO: Should be disallowed once parsing attributes in the type position
+  // works.
+  ptr->arr[0] = 0x0;
+  void* addr = ((char*) ptr->arr[0]) + 1;
+}
diff --git a/clang/test/Sema/attr-counted-by-or-null-struct-ptrs.c b/clang/test/Sema/attr-counted-by-or-null-struct-ptrs.c
index 708bb727ce09d..0bb09059c97f9 100644
--- a/clang/test/Sema/attr-counted-by-or-null-struct-ptrs.c
+++ b/clang/test/Sema/attr-counted-by-or-null-struct-ptrs.c
@@ -22,14 +22,12 @@ struct on_member_pointer_complete_ty {
 
 struct on_member_pointer_incomplete_ty {
   int count;
-  // expected-error@+1{{'counted_by_or_null' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
-  struct size_unknown * buf __counted_by_or_null(count);
+  struct size_unknown * buf __counted_by_or_null(count); // ok
 };
 
 struct on_member_pointer_const_incomplete_ty {
   int count;
-  // expected-error@+1{{'counted_by_or_null' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
-  const struct size_unknown * buf __counted_by_or_null(count);
+  const struct size_unknown * buf __counted_by_or_null(count); // ok
 };
 
 struct on_member_pointer_void_ty {
@@ -116,14 +114,12 @@ struct on_member_pointer_complete_ty_ty_pos {
 
 struct on_member_pointer_incomplete_ty_ty_pos {
   int count;
-  // expected-error@+1{{'counted_by_or_null' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
-  struct size_unknown * __counted_by_or_null(count) buf;
+  struct size_unknown * __counted_by_or_null(count) buf; // ok
 };
 
 struct on_member_pointer_const_incomplete_ty_ty_pos {
   int count;
-  // expected-error@+1{{'counted_by_or_null' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
-  const struct size_unknown * __counted_by_or_null(count) buf;
+  const struct size_unknown * __counted_by_or_null(count) buf; // ok
 };
 
 struct on_member_pointer_void_ty_ty_pos {
diff --git a/clang/test/Sema/attr-counted-by-struct-ptrs-completable-incomplete-pointee.c b/clang/test/Sema/attr-counted-by-struct-ptrs-completable-incomplete-pointee.c
new file mode 100644
index 0000000000000..d28a2086b51b8
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-struct-ptrs-completable-incomplete-pointee.c
@@ -0,0 +1,630 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fexperimental-late-parse-attributes -fsyntax-only -verify %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+// =============================================================================
+// # Struct incomplete type with attribute in the decl position
+// =============================================================================
+
+// Note: This could be considered misleading. The typedef isn't actually on this
+// line. Also note the discrepancy in diagnostic count (27 vs 51) is due to
+// the pointer arithmetic on incomplete pointee type diagnostic always using
+// diagnostic text that refers to the underlying forward decl, even when the
+// typedef is used.
+// expected-note@+3 27{{consider providing a complete definition for 'Incomplete_t' (aka 'struct IncompleteTy')}}
+// The 'forward declaration' notes come from 'arithmetic on a pointer to an incomplete type' errors
+// expected-note@+1 24{{forward declaration of 'struct IncompleteTy'}}
+struct IncompleteTy; // expected-note 27{{consider providing a complete definition for 'struct IncompleteTy'}}
+
+typedef struct IncompleteTy Incomplete_t; 
+
+struct CBBufDeclPos {
+  int count;
+  struct IncompleteTy* buf __counted_by(count); // OK expected-note 27{{consider using '__sized_by' instead of '__counted_by'}}
+  Incomplete_t* buf_typedef __counted_by(count); // OK expected-note 27{{consider using '__sized_by' instead of '__counted_by'}}
+};
+
+void consume_struct_IncompleteTy(struct IncompleteTy* buf);
+
+int idx(void);
+
+
+
+void test_CBBufDeclPos(struct CBBufDeclPos* ptr) {
+  // ===========================================================================
+  // ## Local variable initialization
+  // ===========================================================================
+  struct CBBufDeclPos explicit_desig_init = {
+    .count = 0,
+    // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+    .buf = 0x0,
+    // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+    .buf_typedef = 0x0
+  };
+  // Variable is not currently marked as invalid so uses of the variable allows
+  // diagnostics to fire.
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  explicit_desig_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot use 'explicit_desig_init.buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  void *tmp = explicit_desig_init.buf;
+  // expected-error@+1{{cannot use 'explicit_desig_init.buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  void *tmp2 = explicit_desig_init.buf_typedef;
+
+  struct CBBufDeclPos partial_explicit_desig_init = {
+    .count = 0,
+    // .buf and .buf_typedef are implicit zero initialized
+    // expected-error@+2{{cannot implicitly initialize 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+    // expected-error@+1{{cannot implicitly initialize 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  };
+
+  struct CBBufDeclPos implicit_full_init = {
+    0
+    // expected-error@+2{{cannot implicitly initialize 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+    // expected-error@+1{{cannot implicitly initialize 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  };
+  // Variable is not currently marked as invalid so uses of the variable allows
+  // diagnostics to fire.
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  implicit_full_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  implicit_full_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot use 'implicit_full_init.buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  void* tmp3 = implicit_full_init.buf;
+  // expected-error@+1{{cannot use 'implicit_full_init.buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  void* tmp4 = implicit_full_init.buf_typedef;
+  
+  struct CBBufDeclPos explicit_non_desig_init = {
+    0,
+    // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+    0x0,
+    // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+    0x0
+  };
+
+
+
+
+  // ===========================================================================
+  // ## Assignment to fields
+  // ===========================================================================
+  struct CBBufDeclPos uninit;
+  uninit.count = 0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  uninit.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  uninit.buf_typedef = 0x0;
+  ptr->count = 0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  ptr->buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  ptr->buf_typedef = 0x0;
+
+
+  // ===========================================================================
+  // ## Make sure modifying the fields through other assignment operators is not
+  //    allowed
+  // ===========================================================================
+  uninit.buf++; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ++uninit.buf; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  uninit.buf += 1; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  uninit.buf_typedef++; // // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ++uninit.buf_typedef; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  uninit.buf_typedef -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  
+  uninit.buf--; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  --uninit.buf; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  uninit.buf -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  uninit.buf_typedef--; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  --uninit.buf_typedef; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  uninit.buf_typedef -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+
+  ptr->buf++; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ++ptr->buf; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ptr->buf += 1; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ptr->buf--; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  --ptr->buf; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+  ptr->buf -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'struct IncompleteTy'}}
+
+  ptr->buf_typedef++; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ++ptr->buf_typedef; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ptr->buf_typedef += 1; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ptr->buf_typedef--; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  --ptr->buf_typedef; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+  ptr->buf_typedef -= 1; // expected-error{{arithmetic on a pointer to an incomplete type 'Incomplete_t' (aka 'struct IncompleteTy')}}
+
+  // ===========================================================================
+  // ## Use of fields in expressions
+  // ===========================================================================
+  // expected-error@+2{{cannot use 'uninit.buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  void* addr = 
+    ((char*) uninit.buf ) + 1;
+  // expected-error@+2{{cannot use 'uninit.buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  void* addr_typedef = 
+    ((char*) uninit.buf_typedef ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  void* addr_ptr = 
+    ((char*) ptr->buf ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  void* addr_ptr_typedef = 
+    ((char*) ptr->buf_typedef ) + 1;
+
+
+  // ===========================================================================
+  // ## Take address of fields
+  // ===========================================================================
+  // TODO: This should be forbidden, not because of the incomplete pointee type
+  // but because in the -fbounds-safety language model the address of a
+  // `counted_by` pointer cannot be taken to avoid it being possible to modify
+  // the `counted_by` pointer through another pointer. Whether or not this
+  // should be forbidden when `-fbounds-safety` is off is TBD.
+  //
+  // The incomplete pointee type isn't actually a problem here for
+  // `-fbounds-safety` because taking the address of a pointer returns a pointer
+  // that have the bounds of a single `void*`, so bounds checks on the resulting
+  // pointer don't need to know `sizeof(struct IncompleteTy)` but instead
+  // `sizeof(struct IncompleteTy* buf __counted_by(count))` which is just the
+  // size of a pointer.
+  void* take_addr = &uninit.buf;
+  void* take_addr_typedef = &uninit.buf_typedef;
+  void* take_addr_ptr = &ptr->buf;
+  void* take_addr_ptr_typedef = &ptr->buf_typedef;
+
+  // expected-error@+1{{cannot use 'uninit.buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  struct IncompleteTy* addr_elt_zero = &uninit.buf[0];
+  // expected-error@+1{{cannot use 'uninit.buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  struct IncompleteTy* addr_elt_idx = &uninit.buf[idx()];
+
+  // expected-error@+1{{cannot use 'uninit.buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  struct IncompleteTy* addr_elt_zero_typedef = &uninit.buf_typedef[0];
+  // expected-error@+1{{cannot use 'uninit.buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  struct IncompleteTy* addr_elt_idx_typedef = &uninit.buf_typedef[idx()];
+
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  struct IncompleteTy* addr_elt_zero_ptr = &ptr->buf[0];
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  struct IncompleteTy* addr_elt_idx_ptr = &ptr->buf[idx()];
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  struct IncompleteTy* addr_elt_zero_ptr_typedef = &ptr->buf_typedef[0];
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  struct IncompleteTy* addr_elt_idx_ptr_typedef = &ptr->buf_typedef[idx()];
+
+
+  // ===========================================================================
+  // ## Use fields as call arguments
+  // ===========================================================================
+  // expected-error@+1{{cannot use 'uninit.buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  consume_struct_IncompleteTy(uninit.buf);
+  // expected-error@+1{{cannot use 'uninit.buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  consume_struct_IncompleteTy(uninit.buf_typedef);
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  consume_struct_IncompleteTy(ptr->buf);
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  consume_struct_IncompleteTy(ptr->buf_typedef);
+
+  // ===========================================================================
+  // ## Use [] operator on fields
+  // ===========================================================================
+  // expected-error@+1 2{{cannot use 'uninit.buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  uninit.buf[0] = uninit.buf[1];
+  // expected-error@+1 2{{cannot use 'uninit.buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  uninit.buf_typedef[0] = uninit.buf_typedef[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  ptr->buf[0] = ptr->buf[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+
+// =============================================================================
+// ## Global initialization
+// =============================================================================
+
+struct CBBufDeclPos global_explicit_desig_init = {
+  .count = 0,
+  // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  .buf = 0x0,
+  // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  .buf_typedef = 0x0
+};
+
+void use_global_explicit_desig_init(void) {
+  // Variable isn't marked as invalid so diagnostics still fire
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  global_explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  global_explicit_desig_init.buf_typedef = 0x0;
+}
+
+struct CBBufDeclPos global_partial_explicit_desig_init = {
+  .count = 0,
+  // .buf and .buf_typedef are implicit zero initialized
+  // expected-error@+2{{cannot implicitly initialize 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  // expected-error@+1{{cannot implicitly initialize 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+};
+
+struct CBBufDeclPos global_implicit_full_init = {
+  0
+  // expected-error@+2{{cannot implicitly initialize 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  // expected-error@+1{{cannot implicitly initialize 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+};
+
+struct CBBufDeclPos global_explicit_non_desig_init = {
+  0,
+  // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf' with '__counted_by' attributed type 'struct IncompleteTy * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'struct IncompleteTy' is incomplete}}
+  0x0,
+  // expected-error@+1{{cannot initialize 'CBBufDeclPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_t * __counted_by(count)' (aka 'struct IncompleteTy *') because the pointee type 'Incomplete_t' (aka 'struct IncompleteTy') is incomplete}}
+  0x0
+};
+
+extern struct CBBufDeclPos global_declaration; // OK
+
+// TODO: These tentative definitions are implicitly empty initialized to zero.
+// This should generate an error diagnostic but currently doesn't. There should
+// be a carve out to allow `__counted_by(0)` which is the only constant count
+// version of the attribute where it is valid to assign NULL.
+struct CBBufDeclPos global_tentative_defn;
+static struct CBBufDeclPos global_tentative_defn2;
+
+// =============================================================================
+// ## Completing the definition of the type allows use of CBBufDeclPos fields
+// =============================================================================
+struct IncompleteTy {
+  int field;
+};
+
+void test_CBBufDeclPos_completed(struct CBBufDeclPos* ptr) {
+  // Initialization is ok
+  struct CBBufDeclPos explicit_desig_init = {
+    .count = 0,
+    .buf = 0x0,
+    .buf_typedef = 0x0
+  };
+
+  struct CBBufDeclPos partial_explicit_desig_init = {
+    .count = 0,
+    // .buf and .buf_typedef are implicit zero initialized
+  };
+
+  struct CBBufDeclPos implicit_full_init = {0};
+  
+  struct CBBufDeclPos explicit_non_desig_init = {
+    0,
+    0x0,
+    0x0
+  };
+
+  // Assignment to fields is ok
+  ptr->buf = 0x0;
+  ptr->buf_typedef = 0x0;
+
+  // Use of fields in expressions is ok
+  void* tmp = ptr->buf;
+  void* tmp2 = ptr->buf_typedef;
+
+  // Take address of fields is ok
+  void* take_addr_ptr = &ptr->buf;
+  void* take_addr_ptr_typedef = &ptr->buf_typedef;
+
+  struct IncompleteTy* addr_elt_zero_ptr = &ptr->buf[0];
+  struct IncompleteTy* addr_elt_idx_ptr = &ptr->buf[idx()];
+  struct IncompleteTy* addr_elt_zero_ptr_typedef = &ptr->buf_typedef[0];
+  struct IncompleteTy* addr_elt_idx_ptr_typedef = &ptr->buf_typedef[idx()];
+
+  // As call arguments is ok
+  consume_struct_IncompleteTy(ptr->buf);
+  consume_struct_IncompleteTy(ptr->buf_typedef);
+
+  // In [] operator is ok
+  ptr->buf[0] = ptr->buf[1];
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+// Global initialization is ok
+
+struct CBBufDeclPos global_explicit_desig_init_completed = {
+  .count = 0,
+  .buf = 0x0,
+  .buf_typedef = 0x0
+};
+
+struct CBBufDeclPos global_partial_explicit_desig_init_completed = {
+  .count = 0,
+  // .buf and .buf_typedef are implicit zero initialized
+};
+
+struct CBBufDeclPos global_implicit_full_init_completed = {0};
+
+struct CBBufDeclPos global_explicit_non_desig_init_completed = {
+  0,
+  0x0,
+  0x0
+};
+
+extern struct CBBufDeclPos global_declaration;
+struct CBBufDeclPos global_tentative_defn;
+static struct CBBufDeclPos global_tentative_defn2;
+
+// =============================================================================
+// # Struct incomplete type with attribute in the pointer position
+// =============================================================================
+
+// expected-note@+1 8{{consider providing a complete definition for 'Incomplete_ty2' (aka 'struct IncompleteTy2')}}
+struct IncompleteTy2; // expected-note 8{{consider providing a complete definition for 'struct IncompleteTy2'}}
+typedef struct IncompleteTy2 Incomplete_ty2;
+
+void consume_struct_IncompleteTy2(struct IncompleteTy2* buf);
+
+struct CBBufTyPos {
+  int count;
+  struct IncompleteTy2* __counted_by(count) buf ; // OK expected-note 8{{consider using '__sized_by' instead of '__counted_by'}}
+  Incomplete_ty2 *__counted_by(count) buf_typedef; // OK expected-note 8{{consider using '__sized_by' instead of '__counted_by'}}
+};
+
+void use_CBBufTyPos(struct CBBufTyPos* ptr) {
+  struct CBBufTyPos explicit_desig_init = {
+    .count = 0,
+    // expected-error@+1{{cannot initialize 'CBBufTyPos::buf' with '__counted_by' attributed type 'struct IncompleteTy2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+    .buf = 0x0,
+    // expected-error@+1{{cannot initialize 'CBBufTyPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_ty2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+    .buf_typedef = 0x0
+  };
+
+  // Assignment
+  // expected-error@+1{{cannot assign to 'CBBufTyPos::buf' with '__counted_by' attributed type 'struct IncompleteTy2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufTyPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_ty2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  explicit_desig_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufTyPos::buf' with '__counted_by' attributed type 'struct IncompleteTy2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  ptr->buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufTyPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_ty2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  ptr->buf_typedef = 0x0;
+
+  // Use
+  // expected-error@+2{{cannot use 'ptr->buf' with '__counted_by' attributed type 'struct IncompleteTy2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  void* addr = 
+    ((char*) ptr->buf ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'Incomplete_ty2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  void* addr_typedef = 
+    ((char*) ptr->buf_typedef ) + 1;
+
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by' attributed type 'struct IncompleteTy2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  consume_struct_IncompleteTy2(ptr->buf);
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'Incomplete_ty2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  consume_struct_IncompleteTy2(ptr->buf_typedef);
+
+  // expected-error@+1 2{{cannot use 'ptr->buf' with '__counted_by' attributed type 'struct IncompleteTy2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  ptr->buf[0] = ptr->buf[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'Incomplete_ty2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+struct CBBufTyPos global_explicit_desig_init_struct_type_pos = {
+  .count = 0,
+  // expected-error@+1{{cannot initialize 'CBBufTyPos::buf' with '__counted_by' attributed type 'struct IncompleteTy2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'struct IncompleteTy2' is incomplete}}
+  .buf = 0x0,
+  // expected-error@+1{{cannot initialize 'CBBufTyPos::buf_typedef' with '__counted_by' attributed type 'Incomplete_ty2 * __counted_by(count)' (aka 'struct IncompleteTy2 *') because the pointee type 'Incomplete_ty2' (aka 'struct IncompleteTy2') is incomplete}}
+  .buf_typedef = 0x0
+};
+
+// Defining the type makes `CBBufTyPos` fields usable
+struct IncompleteTy2 {
+  int field;
+};
+
+void use_CBBufTyPos_completed(struct CBBufTyPos* ptr) {
+  ptr->buf = 0x0;
+  ptr->buf_typedef = 0x0;
+  void* addr = ((char*) ptr->buf) + 1;
+  void* addr_typedef = ((char*) ptr->buf_typedef) + 1;
+}
+
+// =============================================================================
+// # union incomplete type
+// =============================================================================
+
+// expected-note@+1 8{{consider providing a complete definition for 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy')}}
+union IncompleteUnionTy; // expected-note 8{{consider providing a complete definition for 'union IncompleteUnionTy'}}
+typedef union IncompleteUnionTy IncompleteUnion_ty;
+
+void consume_struct_IncompleteUnionTy(union IncompleteUnionTy* buf);
+
+struct CBBufUnionTyPos {
+  int count;
+  union IncompleteUnionTy* __counted_by(count) buf ; // OK expected-note 8{{consider using '__sized_by' instead of '__counted_by'}}
+  IncompleteUnion_ty *__counted_by(count) buf_typedef; // OK expected-note 8{{consider using '__sized_by' instead of '__counted_by'}}
+};
+
+void use_CBBufUnionTyPos(struct CBBufUnionTyPos* ptr) {
+  struct CBBufUnionTyPos explicit_desig_init = {
+    .count = 0,
+    // expected-error@+1{{cannot initialize 'CBBufUnionTyPos::buf' with '__counted_by' attributed type 'union IncompleteUnionTy * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+    .buf = 0x0,
+    // expected-error@+1{{cannot initialize 'CBBufUnionTyPos::buf_typedef' with '__counted_by' attributed type 'IncompleteUnion_ty * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+    .buf_typedef = 0x0
+  };
+
+  // Assignment
+  // expected-error@+1{{cannot assign to 'CBBufUnionTyPos::buf' with '__counted_by' attributed type 'union IncompleteUnionTy * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufUnionTyPos::buf_typedef' with '__counted_by' attributed type 'IncompleteUnion_ty * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  explicit_desig_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufUnionTyPos::buf' with '__counted_by' attributed type 'union IncompleteUnionTy * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  ptr->buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufUnionTyPos::buf_typedef' with '__counted_by' attributed type 'IncompleteUnion_ty * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  ptr->buf_typedef = 0x0;
+
+  // Use
+  // expected-error@+2{{cannot use 'ptr->buf' with '__counted_by' attributed type 'union IncompleteUnionTy * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  void* addr = 
+    ((char*) ptr->buf ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'IncompleteUnion_ty * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  void* addr_typedef = 
+    ((char*) ptr->buf_typedef ) + 1;
+
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by' attributed type 'union IncompleteUnionTy * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  consume_struct_IncompleteUnionTy(ptr->buf);
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'IncompleteUnion_ty * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  consume_struct_IncompleteUnionTy(ptr->buf_typedef);
+
+  // expected-error@+1 2{{cannot use 'ptr->buf' with '__counted_by' attributed type 'union IncompleteUnionTy * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  ptr->buf[0] = ptr->buf[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'IncompleteUnion_ty * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+struct CBBufUnionTyPos global_explicit_desig_init_union_type_pos = {
+  .count = 0,
+  // expected-error@+1{{cannot initialize 'CBBufUnionTyPos::buf' with '__counted_by' attributed type 'union IncompleteUnionTy * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'union IncompleteUnionTy' is incomplete}}
+  .buf = 0x0,
+  // expected-error@+1{{cannot initialize 'CBBufUnionTyPos::buf_typedef' with '__counted_by' attributed type 'IncompleteUnion_ty * __counted_by(count)' (aka 'union IncompleteUnionTy *') because the pointee type 'IncompleteUnion_ty' (aka 'union IncompleteUnionTy') is incomplete}}
+  .buf_typedef = 0x0
+};
+
+// Defining the type makes `CBBufUnionTyPos` fields usable
+union IncompleteUnionTy {
+  int field;
+};
+
+void use_CBBufUnionTyPos_completed(struct CBBufUnionTyPos* ptr) {
+  ptr->buf = 0x0;
+  ptr->buf_typedef = 0x0;
+  void* addr = ((char*) ptr->buf) + 1;
+  void* addr_typedef = ((char*) ptr->buf_typedef) + 1;
+}
+
+// =============================================================================
+// # enum incomplete type
+// =============================================================================
+
+// expected-note@+1 8{{consider providing a complete definition for 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy')}}
+enum IncompleteEnumTy; // expected-note 8{{consider providing a complete definition for 'enum IncompleteEnumTy'}}
+typedef enum IncompleteEnumTy IncompleteEnum_ty;
+
+void consume_struct_IncompleteEnumTy(enum IncompleteEnumTy* buf);
+
+struct CBBufEnumTyPos {
+  int count;
+  enum IncompleteEnumTy* __counted_by(count) buf ; // OK expected-note 8{{consider using '__sized_by' instead of '__counted_by'}}
+  IncompleteEnum_ty *__counted_by(count) buf_typedef; // OK expected-note 8{{consider using '__sized_by' instead of '__counted_by'}}
+};
+
+void use_CBBufEnumTyPos(struct CBBufEnumTyPos* ptr) {
+  struct CBBufEnumTyPos explicit_desig_init = {
+    .count = 0,
+    // expected-error@+1{{cannot initialize 'CBBufEnumTyPos::buf' with '__counted_by' attributed type 'enum IncompleteEnumTy * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+    .buf = 0x0,
+    // expected-error@+1{{cannot initialize 'CBBufEnumTyPos::buf_typedef' with '__counted_by' attributed type 'IncompleteEnum_ty * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+    .buf_typedef = 0x0
+  };
+
+  // Assignment
+  // expected-error@+1{{cannot assign to 'CBBufEnumTyPos::buf' with '__counted_by' attributed type 'enum IncompleteEnumTy * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  explicit_desig_init.buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufEnumTyPos::buf_typedef' with '__counted_by' attributed type 'IncompleteEnum_ty * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  explicit_desig_init.buf_typedef = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufEnumTyPos::buf' with '__counted_by' attributed type 'enum IncompleteEnumTy * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  ptr->buf = 0x0;
+  // expected-error@+1{{cannot assign to 'CBBufEnumTyPos::buf_typedef' with '__counted_by' attributed type 'IncompleteEnum_ty * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  ptr->buf_typedef = 0x0;
+
+  // Use
+  // expected-error@+2{{cannot use 'ptr->buf' with '__counted_by' attributed type 'enum IncompleteEnumTy * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  void* addr = 
+    ((char*) ptr->buf ) + 1;
+  // expected-error@+2{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'IncompleteEnum_ty * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  void* addr_typedef = 
+    ((char*) ptr->buf_typedef ) + 1;
+
+  // expected-error@+1{{cannot use 'ptr->buf' with '__counted_by' attributed type 'enum IncompleteEnumTy * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  consume_struct_IncompleteEnumTy(ptr->buf);
+  // expected-error@+1{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'IncompleteEnum_ty * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  consume_struct_IncompleteEnumTy(ptr->buf_typedef);
+
+  // expected-error@+1 2{{cannot use 'ptr->buf' with '__counted_by' attributed type 'enum IncompleteEnumTy * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  ptr->buf[0] = ptr->buf[1];
+  // expected-error@+1 2{{cannot use 'ptr->buf_typedef' with '__counted_by' attributed type 'IncompleteEnum_ty * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  ptr->buf_typedef[0] = ptr->buf_typedef[1];
+}
+
+struct CBBufEnumTyPos global_explicit_desig_init_enum_type_pos = {
+  .count = 0,
+  // expected-error@+1{{cannot initialize 'CBBufEnumTyPos::buf' with '__counted_by' attributed type 'enum IncompleteEnumTy * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'enum IncompleteEnumTy' is incomplete}}
+  .buf = 0x0,
+  // expected-error@+1{{cannot initialize 'CBBufEnumTyPos::buf_typedef' with '__counted_by' attributed type 'IncompleteEnum_ty * __counted_by(count)' (aka 'enum IncompleteEnumTy *') because the pointee type 'IncompleteEnum_ty' (aka 'enum IncompleteEnumTy') is incomplete}}
+  .buf_typedef = 0x0
+};
+
+// Defining the type makes `CBBufEnumTyPos` fields usable
+enum IncompleteEnumTy {
+  ONE,
+  TWO
+};
+
+void use_CBBufEnumTyPos_completed(struct CBBufEnumTyPos* ptr) {
+  ptr->buf = 0x0;
+  ptr->buf_typedef = 0x0;
+  void* addr = ((char*) ptr->buf) + 1;
+  void* addr_typedef = ((char*) ptr->buf_typedef) + 1;
+}
+
+// Make a complete enum by providing an underlying type
+enum CompleteEnumTy : unsigned;
+typedef enum CompleteEnumTy CompleteEnum_ty;
+struct CBBufEnumTyPos2 {
+  int count;
+  enum CompleteEnumTy* __counted_by(count) buf;
+  CompleteEnum_ty *__counted_by(count) buf_typedef;
+};
+
+void use_CBBufEnumTyPos2(struct CBBufEnumTyPos2* ptr) {
+  struct CBBufEnumTyPos2 explicit_desig_init = {
+    .count = 0,
+    .buf = 0x0, // OK
+    .buf_typedef = 0x0 // OK
+  };
+}
+
+// Make a complete enum by providing a concrete declaration
+enum CompleteEnumTy2 {
+  VALUE_ONE,
+  VALUE_TWO
+};
+typedef enum CompleteEnumTy2 CompleteEnum_ty2;
+struct CBBufEnumTyPos3 {
+  int count;
+  enum CompleteEnumTy2* __counted_by(count) buf;
+  CompleteEnum_ty2 *__counted_by(count) buf_typedef;
+};
+
+void use_CBBufEnumTyPos3(struct CBBufEnumTyPos3* ptr) {
+  struct CBBufEnumTyPos3 explicit_desig_init = {
+    .count = 0,
+    .buf = 0x0, // OK
+    .buf_typedef = 0x0 // OK
+  };
+}
+
+
+// =============================================================================
+// # Array of __counted_by pointers
+// =============================================================================
+
+struct IncompleteTy3;
+
+struct CBBufFAMofCountedByPtrs {
+  int size;
+  // TODO: This is misleading. The attribute is written in the type position
+  // but clang currently doesn't treat it like that and it gets treated as
+  // an attribute on the array, rather than on the element type.
+  struct IncompleteTy3* __counted_by(size) arr[];
+};
+
+void arr_of_counted_by_ptr(struct CBBufFAMofCountedByPtrs* ptr) {
+  // TODO: Should be disallowed once parsing attributes in the type position
+  // works.
+  ptr->arr[0] = 0x0;
+  void* addr = ((char*) ptr->arr[0]) + 1;
+}
diff --git a/clang/test/Sema/attr-counted-by-struct-ptrs.c b/clang/test/Sema/attr-counted-by-struct-ptrs.c
index 321d6aafbeba2..c05d18262e2b7 100644
--- a/clang/test/Sema/attr-counted-by-struct-ptrs.c
+++ b/clang/test/Sema/attr-counted-by-struct-ptrs.c
@@ -21,14 +21,12 @@ struct on_member_pointer_complete_ty {
 
 struct on_member_pointer_incomplete_ty {
   int count;
-  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
-  struct size_unknown * buf __counted_by(count);
+  struct size_unknown * buf __counted_by(count); // ok
 };
 
 struct on_member_pointer_const_incomplete_ty {
   int count;
-  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
-  const struct size_unknown * buf __counted_by(count);
+  const struct size_unknown * buf __counted_by(count); // ok
 };
 
 struct on_member_pointer_void_ty {
@@ -115,14 +113,12 @@ struct on_member_pointer_complete_ty_ty_pos {
 
 struct on_member_pointer_incomplete_ty_ty_pos {
   int count;
-  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
-  struct size_unknown * __counted_by(count) buf;
+  struct size_unknown * __counted_by(count) buf; // ok
 };
 
 struct on_member_pointer_const_incomplete_ty_ty_pos {
   int count;
-  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
-  const struct size_unknown * __counted_by(count) buf;
+  const struct size_unknown * __counted_by(count) buf; // ok
 };
 
 struct on_member_pointer_void_ty_ty_pos {
diff --git a/clang/test/Sema/attr-counted-by-vla.c b/clang/test/Sema/attr-counted-by-vla.c
index 35737c03f3222..5b1b4833dce7d 100644
--- a/clang/test/Sema/attr-counted-by-vla.c
+++ b/clang/test/Sema/attr-counted-by-vla.c
@@ -98,8 +98,7 @@ struct array_of_ints_count {
 
 struct not_a_fam {
   int count;
-  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct bar' is an incomplete type}}
-  struct bar *non_fam __counted_by(count);
+  struct bar *non_fam __counted_by(count); // ok
 };
 
 struct not_a_c99_fam {

From 15ea45b100af4f97c5e6e7aca3168e4b8fb9c9fa Mon Sep 17 00:00:00 2001
From: Brad Smith <brad@comstyle.com>
Date: Fri, 18 Apr 2025 18:29:06 -0400
Subject: [PATCH 495/710] [sanitizer_common] Updated build fix for newer NetBSD
 (#134742)

Co-authored-by: Thomas Klausner <wiz@gatalith.at>
---
 .../lib/sanitizer_common/sanitizer_linux_libcdep.cpp       | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index c2e2e290b0ecf..3bf9f8b316d36 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -34,11 +34,12 @@
 
 #  if SANITIZER_NETBSD
 #    // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
+#    define _RTLD_SOURCE
+#    include <machine/mcontext.h>
+#    undef _RTLD_SOURCE
 #    include <sys/param.h>
-#    if defined(__NetBSD_Version__) && (__NetBSD_Version__ >= 1099001200)
+#    if __NetBSD_Version__ >= 1099001200
 #      include <machine/lwp_private.h>
-#    else
-#      define _RTLD_SOURCE
 #    endif
 #  endif
 

From 808f63824acec53d60252bef0bd0f6876b3de0c9 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Fri, 18 Apr 2025 19:46:44 -0300
Subject: [PATCH 496/710] [ci] set up llvm-symbolizer environment variable
 (#136156)

Set up llvm-symbolizer environment variable so that its preferred over
any symbolizer just built, as it can be much slower when built for
debugging.
---
 .ci/monolithic-linux.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 6461c9d40ad59..59a6b22ccb06a 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -58,6 +58,11 @@ export PIP_BREAK_SYSTEM_PACKAGES=1
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
+
+# Set the system llvm-symbolizer as preferred.
+export LLVM_SYMBOLIZER_PATH=`which llvm-symbolizer`
+[[ ! -f "${LLVM_SYMBOLIZER_PATH}" ]] && echo "llvm-symbolizer not found!"
+
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
       -G Ninja \

From 2ce97fd43c929404d3420aee7c0f24ef6731f2c5 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Fri, 18 Apr 2025 19:47:06 -0300
Subject: [PATCH 497/710] [ci] upload any generated clang reproducers as
 artifacts (#136157)

Make sure any generated clang reproducers end up as artifacts.
---
 .ci/monolithic-linux.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 59a6b22ccb06a..de399e605cf9b 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -28,10 +28,14 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
   ccache --clear
 fi
 
+mkdir -p artifacts/reproducers
+
+# Make sure any clang reproducers will end up as artifacts.
+export CLANG_CRASH_DIAGNOSTICS_DIR=`realpath artifacts/reproducers`
+
 function at-exit {
   retcode=$?
 
-  mkdir -p artifacts
   ccache --print-stats > artifacts/ccache_stats.txt
   cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
 

From 7f41095db59526f0e83940cdeb2d7f0a117709c2 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Sat, 19 Apr 2025 02:04:41 +0300
Subject: [PATCH 498/710] [AArch64] Use helper class for emitting CFI
 instructions into MIR (NFCI) (#136004)

Pull Request: https://github.com/llvm/llvm-project/pull/136004
---
 llvm/include/llvm/CodeGen/CFIInstBuilder.h    |  12 +-
 .../Target/AArch64/AArch64FrameLowering.cpp   | 199 +++++-------------
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  |  17 +-
 .../lib/Target/AArch64/AArch64PointerAuth.cpp |  28 +--
 4 files changed, 78 insertions(+), 178 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/CFIInstBuilder.h b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
index 86e125f11e0c6..8e22a5bacef6e 100644
--- a/llvm/include/llvm/CodeGen/CFIInstBuilder.h
+++ b/llvm/include/llvm/CodeGen/CFIInstBuilder.h
@@ -67,8 +67,8 @@ class CFIInstBuilder {
         nullptr, TRI.getDwarfRegNum(Reg, IsEH)));
   }
 
-  void buildDefCFAOffset(int64_t Offset) const {
-    insertCFIInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
+  void buildDefCFAOffset(int64_t Offset, MCSymbol *Label = nullptr) const {
+    insertCFIInst(MCCFIInstruction::cfiDefCfaOffset(Label, Offset));
   }
 
   void buildAdjustCFAOffset(int64_t Adjustment) const {
@@ -80,6 +80,14 @@ class CFIInstBuilder {
         nullptr, TRI.getDwarfRegNum(Reg, IsEH), Offset));
   }
 
+  void buildNegateRAState() const {
+    insertCFIInst(MCCFIInstruction::createNegateRAState(nullptr));
+  }
+
+  void buildNegateRAStateWithPC() const {
+    insertCFIInst(MCCFIInstruction::createNegateRAStateWithPC(nullptr));
+  }
+
   void buildRegister(MCRegister Reg1, MCRegister Reg2) const {
     insertCFIInst(MCCFIInstruction::createRegister(
         nullptr, TRI.getDwarfRegNum(Reg1, IsEH),
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 68218e59961c2..78ac57e3e92a6 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -216,6 +216,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -602,33 +603,23 @@ void AArch64FrameLowering::emitCalleeSavedGPRLocations(
   if (CSI.empty())
     return;
 
-  const TargetSubtargetInfo &STI = MF.getSubtarget();
-  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
-  DebugLoc DL = MBB.findDebugLoc(MBBI);
-
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
   for (const auto &Info : CSI) {
     unsigned FrameIdx = Info.getFrameIdx();
     if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector)
       continue;
 
     assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
-    int64_t DwarfReg = TRI.getDwarfRegNum(Info.getReg(), true);
     int64_t Offset = MFI.getObjectOffset(FrameIdx) - getOffsetOfLocalArea();
 
     // The location of VG will be emitted before each streaming-mode change in
     // the function. Only locally-streaming functions require emitting the
     // non-streaming VG location here.
     if ((LocallyStreaming && FrameIdx == AFI->getStreamingVGIdx()) ||
-        (!LocallyStreaming &&
-         DwarfReg == TRI.getDwarfRegNum(AArch64::VG, true)))
+        (!LocallyStreaming && Info.getReg() == AArch64::VG))
       continue;
 
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
-    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIBuilder.buildOffset(Info.getReg(), Offset);
   }
 }
 
@@ -644,9 +635,8 @@ void AArch64FrameLowering::emitCalleeSavedSVELocations(
 
   const TargetSubtargetInfo &STI = MF.getSubtarget();
   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
-  DebugLoc DL = MBB.findDebugLoc(MBBI);
   AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
 
   for (const auto &Info : CSI) {
     if (!(MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
@@ -663,54 +653,32 @@ void AArch64FrameLowering::emitCalleeSavedSVELocations(
         StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) -
         StackOffset::getFixed(AFI.getCalleeSavedStackSize(MFI));
 
-    unsigned CFIIndex = MF.addFrameInst(createCFAOffset(TRI, Reg, Offset));
-    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIBuilder.insertCFIInst(createCFAOffset(TRI, Reg, Offset));
   }
 }
 
-static void insertCFISameValue(const MCInstrDesc &Desc, MachineFunction &MF,
-                               MachineBasicBlock &MBB,
-                               MachineBasicBlock::iterator InsertPt,
-                               unsigned DwarfReg) {
-  unsigned CFIIndex =
-      MF.addFrameInst(MCCFIInstruction::createSameValue(nullptr, DwarfReg));
-  BuildMI(MBB, InsertPt, DebugLoc(), Desc).addCFIIndex(CFIIndex);
-}
-
 void AArch64FrameLowering::resetCFIToInitialState(
     MachineBasicBlock &MBB) const {
 
   MachineFunction &MF = *MBB.getParent();
   const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
-  const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
   const auto &TRI =
       static_cast<const AArch64RegisterInfo &>(*Subtarget.getRegisterInfo());
   const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
 
-  const MCInstrDesc &CFIDesc = TII.get(TargetOpcode::CFI_INSTRUCTION);
-  DebugLoc DL;
+  CFIInstBuilder CFIBuilder(MBB, MBB.begin(), MachineInstr::NoFlags);
 
   // Reset the CFA to `SP + 0`.
-  MachineBasicBlock::iterator InsertPt = MBB.begin();
-  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
-      nullptr, TRI.getDwarfRegNum(AArch64::SP, true), 0));
-  BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
+  CFIBuilder.buildDefCFA(AArch64::SP, 0);
 
   // Flip the RA sign state.
-  if (MFI.shouldSignReturnAddress(MF)) {
-    auto CFIInst = MFI.branchProtectionPAuthLR()
-                       ? MCCFIInstruction::createNegateRAStateWithPC(nullptr)
-                       : MCCFIInstruction::createNegateRAState(nullptr);
-    CFIIndex = MF.addFrameInst(CFIInst);
-    BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
-  }
+  if (MFI.shouldSignReturnAddress(MF))
+    MFI.branchProtectionPAuthLR() ? CFIBuilder.buildNegateRAStateWithPC()
+                                  : CFIBuilder.buildNegateRAState();
 
   // Shadow call stack uses X18, reset it.
   if (MFI.needsShadowCallStackPrologueEpilogue(MF))
-    insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
-                       TRI.getDwarfRegNum(AArch64::X18, true));
+    CFIBuilder.buildSameValue(AArch64::X18);
 
   // Emit .cfi_same_value for callee-saved registers.
   const std::vector<CalleeSavedInfo> &CSI =
@@ -719,8 +687,7 @@ void AArch64FrameLowering::resetCFIToInitialState(
     MCRegister Reg = Info.getReg();
     if (!TRI.regNeedsCFI(Reg, Reg))
       continue;
-    insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
-                       TRI.getDwarfRegNum(Reg, true));
+    CFIBuilder.buildSameValue(Reg);
   }
 }
 
@@ -736,8 +703,7 @@ static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
 
   const TargetSubtargetInfo &STI = MF.getSubtarget();
   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
-  const TargetInstrInfo &TII = *STI.getInstrInfo();
-  DebugLoc DL = MBB.findDebugLoc(MBBI);
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy);
 
   for (const auto &Info : CSI) {
     if (SVE !=
@@ -752,11 +718,7 @@ static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
     if (!Info.isRestored())
       continue;
 
-    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
-        nullptr, TRI.getDwarfRegNum(Info.getReg(), true)));
-    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameDestroy);
+    CFIBuilder.buildRestore(Info.getReg());
   }
 }
 
@@ -905,13 +867,8 @@ void AArch64FrameLowering::allocateStackSpace(
       .addReg(TargetReg);
   if (EmitCFI) {
     // Set the CFA register back to SP.
-    unsigned Reg =
-        Subtarget.getRegisterInfo()->getDwarfRegNum(AArch64::SP, true);
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
-    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+    CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
+        .buildDefCFARegister(AArch64::SP);
   }
   if (RealignmentPadding)
     AFI.setStackRealigned(true);
@@ -1552,13 +1509,9 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
     InsertSEH(*MIB, *TII, FrameFlag);
   }
 
-  if (EmitCFI) {
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset - CSStackSizeInc));
-    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(FrameFlag);
-  }
+  if (EmitCFI)
+    CFIInstBuilder(MBB, MBBI, FrameFlag)
+        .buildDefCFAOffset(CFAOffset - CSStackSizeInc);
 
   return std::prev(MBB.erase(MBBI));
 }
@@ -1673,11 +1626,8 @@ static void emitShadowCallStackPrologue(const TargetInstrInfo &TII,
         static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
         static_cast<char>(-8) & 0x7f, // addend (sleb128)
     };
-    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
-        nullptr, StringRef(CFIInst, sizeof(CFIInst))));
-    BuildMI(MBB, MBBI, DL, TII.get(AArch64::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlag(MachineInstr::FrameSetup);
+    CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
+        .buildEscape(StringRef(CFIInst, sizeof(CFIInst)));
   }
 }
 
@@ -1694,34 +1644,25 @@ static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII,
       .addImm(-8)
       .setMIFlag(MachineInstr::FrameDestroy);
 
-  if (MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF)) {
-    unsigned CFIIndex =
-        MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, 18));
-    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameDestroy);
-  }
+  if (MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF))
+    CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
+        .buildRestore(AArch64::X18);
 }
 
 // Define the current CFA rule to use the provided FP.
 static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator MBBI,
-                                const DebugLoc &DL, unsigned FixedObject) {
+                                unsigned FixedObject) {
   const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
   const AArch64RegisterInfo *TRI = STI.getRegisterInfo();
-  const TargetInstrInfo *TII = STI.getInstrInfo();
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
 
   const int OffsetToFirstCalleeSaveFromFP =
       AFI->getCalleeSaveBaseToFrameRecordOffset() -
       AFI->getCalleeSavedStackSize();
   Register FramePtr = TRI->getFrameRegister(MF);
-  unsigned Reg = TRI->getDwarfRegNum(FramePtr, true);
-  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
-      nullptr, Reg, FixedObject - OffsetToFirstCalleeSaveFromFP));
-  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex)
-      .setMIFlags(MachineInstr::FrameSetup);
+  CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
+      .buildDefCFA(FramePtr, FixedObject - OffsetToFirstCalleeSaveFromFP);
 }
 
 #ifndef NDEBUG
@@ -1916,11 +1857,8 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
         // Label used to tie together the PROLOG_LABEL and the MachineMoves.
         MCSymbol *FrameLabel = MF.getContext().createTempSymbol();
         // Encode the stack size of the leaf function.
-        unsigned CFIIndex = MF.addFrameInst(
-            MCCFIInstruction::cfiDefCfaOffset(FrameLabel, NumBytes));
-        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex)
-            .setMIFlags(MachineInstr::FrameSetup);
+        CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
+            .buildDefCFAOffset(NumBytes, FrameLabel);
       }
     }
 
@@ -2026,7 +1964,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
       }
     }
     if (EmitAsyncCFI)
-      emitDefineCFAWithFP(MF, MBB, MBBI, DL, FixedObject);
+      emitDefineCFAWithFP(MF, MBB, MBBI, FixedObject);
   }
 
   // Now emit the moves for whatever callee saved regs we have (including FP,
@@ -2253,16 +2191,14 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
 
   if (EmitCFI && !EmitAsyncCFI) {
     if (HasFP) {
-      emitDefineCFAWithFP(MF, MBB, MBBI, DL, FixedObject);
+      emitDefineCFAWithFP(MF, MBB, MBBI, FixedObject);
     } else {
       StackOffset TotalSize =
           SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize());
-      unsigned CFIIndex = MF.addFrameInst(createDefCFA(
-          *RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, TotalSize,
-          /*LastAdjustmentWasScalable=*/false));
-      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameSetup);
+      CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
+      CFIBuilder.insertCFIInst(
+          createDefCFA(*RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP,
+                       TotalSize, /*LastAdjustmentWasScalable=*/false));
     }
     emitCalleeSavedGPRLocations(MBB, MBBI);
     emitCalleeSavedSVELocations(MBB, MBBI);
@@ -2460,15 +2396,9 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
     assert(!SVEStackSize && "Cannot combine SP bump with SVE");
 
     // When we are about to restore the CSRs, the CFA register is SP again.
-    if (EmitCFI && hasFP(MF)) {
-      const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
-      unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
-      unsigned CFIIndex =
-          MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, NumBytes));
-      BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameDestroy);
-    }
+    if (EmitCFI && hasFP(MF))
+      CFIInstBuilder(MBB, LastPopI, MachineInstr::FrameDestroy)
+          .buildDefCFA(AArch64::SP, NumBytes);
 
     emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize),
@@ -2587,15 +2517,9 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
                     MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
 
   // When we are about to restore the CSRs, the CFA register is SP again.
-  if (EmitCFI && hasFP(MF)) {
-    const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
-    unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::cfiDefCfa(nullptr, Reg, PrologueSaveSize));
-    BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameDestroy);
-  }
+  if (EmitCFI && hasFP(MF))
+    CFIInstBuilder(MBB, LastPopI, MachineInstr::FrameDestroy)
+        .buildDefCFA(AArch64::SP, PrologueSaveSize);
 
   // This must be placed after the callee-save restore code because that code
   // assumes the SP is at the same location as it was after the callee-save save
@@ -4966,46 +4890,37 @@ MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
 }
 } // namespace
 
-MachineBasicBlock::iterator emitVGSaveRestore(MachineBasicBlock::iterator II,
-                                              const AArch64FrameLowering *TFI) {
+static void emitVGSaveRestore(MachineBasicBlock::iterator II,
+                              const AArch64FrameLowering *TFI) {
   MachineInstr &MI = *II;
   MachineBasicBlock *MBB = MI.getParent();
   MachineFunction *MF = MBB->getParent();
 
   if (MI.getOpcode() != AArch64::VGSavePseudo &&
       MI.getOpcode() != AArch64::VGRestorePseudo)
-    return II;
+    return;
 
   SMEAttrs FuncAttrs(MF->getFunction());
   bool LocallyStreaming =
       FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
   const AArch64FunctionInfo *AFI = MF->getInfo<AArch64FunctionInfo>();
-  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
-  const AArch64InstrInfo *TII =
-      MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
 
   int64_t VGFrameIdx =
       LocallyStreaming ? AFI->getStreamingVGIdx() : AFI->getVGIdx();
   assert(VGFrameIdx != std::numeric_limits<int>::max() &&
          "Expected FrameIdx for VG");
 
-  unsigned CFIIndex;
+  CFIInstBuilder CFIBuilder(*MBB, II, MachineInstr::NoFlags);
   if (MI.getOpcode() == AArch64::VGSavePseudo) {
     const MachineFrameInfo &MFI = MF->getFrameInfo();
     int64_t Offset =
         MFI.getObjectOffset(VGFrameIdx) - TFI->getOffsetOfLocalArea();
-    CFIIndex = MF->addFrameInst(MCCFIInstruction::createOffset(
-        nullptr, TRI->getDwarfRegNum(AArch64::VG, true), Offset));
-  } else
-    CFIIndex = MF->addFrameInst(MCCFIInstruction::createRestore(
-        nullptr, TRI->getDwarfRegNum(AArch64::VG, true)));
-
-  MachineInstr *UnwindInst = BuildMI(*MBB, II, II->getDebugLoc(),
-                                     TII->get(TargetOpcode::CFI_INSTRUCTION))
-                                 .addCFIIndex(CFIIndex);
+    CFIBuilder.buildOffset(AArch64::VG, Offset);
+  } else {
+    CFIBuilder.buildRestore(AArch64::VG);
+  }
 
   MI.eraseFromParent();
-  return UnwindInst->getIterator();
 }
 
 void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
@@ -5013,8 +4928,8 @@ void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
   for (auto &BB : MF)
     for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();) {
       if (requiresSaveVG(MF))
-        II = emitVGSaveRestore(II, this);
-      if (StackTaggingMergeSetTag)
+        emitVGSaveRestore(II++, this);
+      else if (StackTaggingMergeSetTag)
         II = tryMergeAdjacentSTG(II, this, RS);
     }
 }
@@ -5361,14 +5276,8 @@ void AArch64FrameLowering::inlineStackProbeFixed(
     MBB = MBBI->getParent();
     if (EmitAsyncCFI && !HasFP) {
       // Set the CFA register back to SP.
-      const AArch64RegisterInfo &RegInfo =
-          *MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
-      unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
-      unsigned CFIIndex =
-          MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
-      BuildMI(*MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex)
-          .setMIFlags(MachineInstr::FrameSetup);
+      CFIInstBuilder(*MBB, MBBI, MachineInstr::FrameSetup)
+          .buildDefCFARegister(AArch64::SP);
     }
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 74217fad82a7e..8e9ac82b5acd6 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineCombinerPattern.h"
@@ -9879,24 +9880,14 @@ void AArch64InstrInfo::buildOutlinedFrame(
     It = MBB.insert(It, STRXpre);
 
     if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
-      const TargetSubtargetInfo &STI = MF.getSubtarget();
-      const MCRegisterInfo *MRI = STI.getRegisterInfo();
-      unsigned DwarfReg = MRI->getDwarfRegNum(AArch64::LR, true);
+      CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
 
       // Add a CFI saying the stack was moved 16 B down.
-      int64_t StackPosEntry =
-          MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 16));
-      BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
-          .addCFIIndex(StackPosEntry)
-          .setMIFlags(MachineInstr::FrameSetup);
+      CFIBuilder.buildDefCFAOffset(16);
 
       // Add a CFI saying that the LR that we want to find is now 16 B higher
       // than before.
-      int64_t LRPosEntry = MF.addFrameInst(
-          MCCFIInstruction::createOffset(nullptr, DwarfReg, -16));
-      BuildMI(MBB, It, DebugLoc(), get(AArch64::CFI_INSTRUCTION))
-          .addCFIIndex(LRPosEntry)
-          .setMIFlags(MachineInstr::FrameSetup);
+      CFIBuilder.buildOffset(AArch64::LR, -16);
     }
 
     // Insert a restore before the terminator for the function.
diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
index ba03f4e257b69..57e84ebfcf403 100644
--- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
@@ -12,6 +12,7 @@
 #include "AArch64InstrInfo.h"
 #include "AArch64MachineFunctionInfo.h"
 #include "AArch64Subtarget.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
@@ -90,24 +91,17 @@ static void BuildPACM(const AArch64Subtarget &Subtarget, MachineBasicBlock &MBB,
     BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM)).setMIFlag(Flags);
 }
 
-static void emitPACCFI(const AArch64Subtarget &Subtarget,
-                       MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-                       DebugLoc DL, MachineInstr::MIFlag Flags, bool EmitCFI) {
+static void emitPACCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                       MachineInstr::MIFlag Flags, bool EmitCFI) {
   if (!EmitCFI)
     return;
 
-  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
   auto &MF = *MBB.getParent();
   auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
 
-  auto CFIInst = MFnI.branchProtectionPAuthLR()
-                     ? MCCFIInstruction::createNegateRAStateWithPC(nullptr)
-                     : MCCFIInstruction::createNegateRAState(nullptr);
-
-  unsigned CFIIndex = MF.addFrameInst(CFIInst);
-  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex)
-      .setMIFlags(Flags);
+  CFIInstBuilder CFIBuilder(MBB, MBBI, Flags);
+  MFnI.branchProtectionPAuthLR() ? CFIBuilder.buildNegateRAStateWithPC()
+                                 : CFIBuilder.buildNegateRAState();
 }
 
 void AArch64PointerAuth::signLR(MachineFunction &MF,
@@ -137,7 +131,7 @@ void AArch64PointerAuth::signLR(MachineFunction &MF,
   // No SEH opcode for this one; it doesn't materialize into an
   // instruction on Windows.
   if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
-    emitPACCFI(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameSetup, EmitCFI);
+    emitPACCFI(MBB, MBBI, MachineInstr::FrameSetup, EmitCFI);
     BuildMI(MBB, MBBI, DL,
             TII->get(MFnI.shouldSignWithBKey() ? AArch64::PACIBSPPC
                                                : AArch64::PACIASPPC))
@@ -145,7 +139,7 @@ void AArch64PointerAuth::signLR(MachineFunction &MF,
         ->setPreInstrSymbol(MF, MFnI.getSigningInstrLabel());
   } else {
     BuildPACM(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameSetup);
-    emitPACCFI(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameSetup, EmitCFI);
+    emitPACCFI(MBB, MBBI, MachineInstr::FrameSetup, EmitCFI);
     BuildMI(MBB, MBBI, DL,
             TII->get(MFnI.shouldSignWithBKey() ? AArch64::PACIBSP
                                                : AArch64::PACIASP))
@@ -205,16 +199,14 @@ void AArch64PointerAuth::authenticateLR(
     if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
       assert(PACSym && "No PAC instruction to refer to");
       emitPACSymOffsetIntoX16(*TII, MBB, MBBI, DL, PACSym);
-      emitPACCFI(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameDestroy,
-                 EmitAsyncCFI);
+      emitPACCFI(MBB, MBBI, MachineInstr::FrameDestroy, EmitAsyncCFI);
       BuildMI(MBB, MBBI, DL,
               TII->get(UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
           .addSym(PACSym)
           .setMIFlag(MachineInstr::FrameDestroy);
     } else {
       BuildPACM(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameDestroy, PACSym);
-      emitPACCFI(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameDestroy,
-                 EmitAsyncCFI);
+      emitPACCFI(MBB, MBBI, MachineInstr::FrameDestroy, EmitAsyncCFI);
       BuildMI(MBB, MBBI, DL,
               TII->get(UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
           .setMIFlag(MachineInstr::FrameDestroy);

From db97d56c97513d561cfdc3a0b1df6c5b5b909d42 Mon Sep 17 00:00:00 2001
From: Evgenii Kudriashov <evgenii.kudriashov@intel.com>
Date: Sat, 19 Apr 2025 01:48:40 +0200
Subject: [PATCH 499/710] [X86][APX] Handle AND_NF instruction for compare
 peephole (#136233)

---
 llvm/lib/Target/X86/X86InstrInfo.cpp        | 16 +++++-----
 llvm/test/CodeGen/X86/apx/nf-regressions.ll | 33 +++++++++++++++++++++
 2 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 48688e4cf8f3c..459e15031ee16 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -1035,12 +1035,11 @@ inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) {
   return ShAmt < 4 && ShAmt > 0;
 }
 
-static bool findRedundantFlagInstr(MachineInstr &CmpInstr,
-                                   MachineInstr &CmpValDefInstr,
-                                   const MachineRegisterInfo *MRI,
-                                   MachineInstr **AndInstr,
-                                   const TargetRegisterInfo *TRI,
-                                   bool &NoSignFlag, bool &ClearsOverflowFlag) {
+static bool
+findRedundantFlagInstr(MachineInstr &CmpInstr, MachineInstr &CmpValDefInstr,
+                       const MachineRegisterInfo *MRI, MachineInstr **AndInstr,
+                       const TargetRegisterInfo *TRI, const X86Subtarget &ST,
+                       bool &NoSignFlag, bool &ClearsOverflowFlag) {
   if (!(CmpValDefInstr.getOpcode() == X86::SUBREG_TO_REG &&
         CmpInstr.getOpcode() == X86::TEST64rr) &&
       !(CmpValDefInstr.getOpcode() == X86::COPY &&
@@ -1103,7 +1102,8 @@ static bool findRedundantFlagInstr(MachineInstr &CmpInstr,
   if (VregDefInstr->getParent() != CmpValDefInstr.getParent())
     return false;
 
-  if (X86::isAND(VregDefInstr->getOpcode())) {
+  if (X86::isAND(VregDefInstr->getOpcode()) &&
+      (!ST.hasNF() || VregDefInstr->modifiesRegister(X86::EFLAGS, TRI))) {
     // Get a sequence of instructions like
     //   %reg = and* ...                    // Set EFLAGS
     //   ...                                // EFLAGS not changed
@@ -5433,7 +5433,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
         MachineInstr *AndInstr = nullptr;
         if (IsCmpZero &&
             findRedundantFlagInstr(CmpInstr, Inst, MRI, &AndInstr, TRI,
-                                   NoSignFlag, ClearsOverflowFlag)) {
+                                   Subtarget, NoSignFlag, ClearsOverflowFlag)) {
           assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode()));
           MI = AndInstr;
           break;
diff --git a/llvm/test/CodeGen/X86/apx/nf-regressions.ll b/llvm/test/CodeGen/X86/apx/nf-regressions.ll
index 40503d85a1ddb..68bd05a0737b6 100644
--- a/llvm/test/CodeGen/X86/apx/nf-regressions.ll
+++ b/llvm/test/CodeGen/X86/apx/nf-regressions.ll
@@ -73,3 +73,36 @@ bb14:                                             ; preds = %bb12
 bb16:                                             ; preds = %bb14, %bb11, %bb10, %bb
   ret void
 }
+
+; We must not try to replace CMP with AND_NF as it sets no flags
+define void @cmp_peephole_and_nf(i64 %arg0, ptr %ptr1, ptr %ptr2) {
+; CHECK-LABEL: cmp_peephole_and_nf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    negq %rdi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    {nf} andl $1, %eax
+; CHECK-NEXT:    jb .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %true
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    sete (%rsi)
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB1_2: # %false
+; CHECK-NEXT:    movq %rdi, (%rsi)
+; CHECK-NEXT:    movq %rax, (%rdx)
+; CHECK-NEXT:    retq
+entry:
+  %sub_flag = sub i64 0, %arg0
+  %and_nf = and i64 %sub_flag, 1
+  %elim = icmp eq i64 0, %arg0
+  br i1 %elim, label %true, label %false
+
+true:
+  %8 = icmp eq i64 %and_nf, 0
+  store i1 %8, ptr %ptr1
+  ret void
+
+false:
+  store i64 %sub_flag, ptr %ptr1
+  store i64 %and_nf, ptr %ptr2
+  ret void
+}

From 50db7a7d269b42f0cda63eb005aadfdbe25f56cb Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Fri, 18 Apr 2025 17:19:12 -0700
Subject: [PATCH 500/710] [flang] Fixed fir.dummy_scope generation to work for
 TBAA. (#136382)

The nesting of fir.dummy_scope operations defines the roots
of the TBAA forest. If we do not generate fir.dummy_scope
in functions that do not have any dummy arguments, then
the globals accessed in the function and the dummy arguments
accessed by the callee may end up in different sub-trees
of the same root. The added tbaa-with-dummy-scope2.fir
demonstrates the issue.
---
 flang/lib/Lower/Bridge.cpp                    |   5 +-
 flang/test/Driver/emit-mlir.f90               |   1 +
 flang/test/Lower/HLFIR/calls-f77.f90          |   1 +
 flang/test/Lower/HLFIR/dummy-scope.f90        |  52 +++++++++
 flang/test/Lower/OpenMP/real10.f90            |   2 +-
 flang/test/Lower/main_location.f90            |   2 +
 .../Transforms/tbaa-with-dummy-scope2.fir     | 106 ++++++++++++++++++
 7 files changed, 167 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Lower/HLFIR/dummy-scope.f90
 create mode 100644 flang/test/Transforms/tbaa-with-dummy-scope2.fir

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index b4d1197822a43..1652a86ed7e63 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5508,7 +5508,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     for (const Fortran::lower::CalleeInterface::PassedEntity &arg :
          callee.getPassedArguments())
       mapPassedEntity(arg);
-    if (lowerToHighLevelFIR() && !callee.getPassedArguments().empty()) {
+
+    // Always generate fir.dummy_scope even if there are no arguments.
+    // It is currently used to create proper TBAA forest.
+    if (lowerToHighLevelFIR()) {
       mlir::Value scopeOp = builder->create<fir::DummyScopeOp>(toLocation());
       setDummyArgsScope(scopeOp);
     }
diff --git a/flang/test/Driver/emit-mlir.f90 b/flang/test/Driver/emit-mlir.f90
index 2345a7cf53e40..de5a62d6bc7f3 100644
--- a/flang/test/Driver/emit-mlir.f90
+++ b/flang/test/Driver/emit-mlir.f90
@@ -13,6 +13,7 @@
 ! CHECK-SAME: dlti.dl_spec =
 ! CHECK-SAME: llvm.data_layout =
 ! CHECK-LABEL: func @_QQmain() {
+! CHECK-NEXT:  fir.dummy_scope
 ! CHECK-NEXT:  return
 ! CHECK-NEXT: }
 ! CHECK-NEXT: func.func private @_FortranAProgramStart(i32, !llvm.ptr, !llvm.ptr, !llvm.ptr)
diff --git a/flang/test/Lower/HLFIR/calls-f77.f90 b/flang/test/Lower/HLFIR/calls-f77.f90
index 64b90b83b8280..450f8811eb5e0 100644
--- a/flang/test/Lower/HLFIR/calls-f77.f90
+++ b/flang/test/Lower/HLFIR/calls-f77.f90
@@ -9,6 +9,7 @@ subroutine call_no_arg()
   call void()
 end subroutine
 ! CHECK-LABEL: func.func @_QPcall_no_arg() {
+! CHECK-NEXT:  fir.dummy_scope
 ! CHECK-NEXT:  fir.call @_QPvoid() fastmath<contract> : () -> ()
 ! CHECK-NEXT:  return
 
diff --git a/flang/test/Lower/HLFIR/dummy-scope.f90 b/flang/test/Lower/HLFIR/dummy-scope.f90
new file mode 100644
index 0000000000000..4b1a3324c0777
--- /dev/null
+++ b/flang/test/Lower/HLFIR/dummy-scope.f90
@@ -0,0 +1,52 @@
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+
+! Test fir.dummy_scope assignment to argument x
+subroutine sub_arg(x)
+  integer :: x
+end subroutine sub_arg
+! CHECK-LABEL:   func.func @_QPsub_arg(
+! CHECK-SAME:                          %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<i32> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFsub_argEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           return
+! CHECK:         }
+
+! Test fir.dummy_scope is created even when there are no arguments.
+subroutine sub_noarg
+end subroutine sub_noarg
+! CHECK-LABEL:   func.func @_QPsub_noarg() {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           return
+! CHECK:         }
+
+! Test fir.dummy_scope assignment to argument x
+function func_arg(x)
+  integer :: x, func_arg
+  func_arg = x
+end function func_arg
+! CHECK-LABEL:   func.func @_QPfunc_arg(
+! CHECK-SAME:                           %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<i32> {fir.bindc_name = "x"}) -> i32 {
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "func_arg", uniq_name = "_QFfunc_argEfunc_arg"}
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFfunc_argEfunc_arg"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFfunc_argEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
+! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
+! CHECK:           return %[[VAL_6]] : i32
+! CHECK:         }
+
+! Test fir.dummy_scope is created even when there are no arguments.
+function func_noarg
+  integer :: func_noarg
+  func_noarg = 1
+end function func_noarg
+! CHECK-LABEL:   func.func @_QPfunc_noarg() -> i32 {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "func_noarg", uniq_name = "_QFfunc_noargEfunc_noarg"}
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFfunc_noargEfunc_noarg"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]] = arith.constant 1 : i32
+! CHECK:           hlfir.assign %[[VAL_3]] to %[[VAL_2]]#0 : i32, !fir.ref<i32>
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
+! CHECK:           return %[[VAL_4]] : i32
+! CHECK:         }
diff --git a/flang/test/Lower/OpenMP/real10.f90 b/flang/test/Lower/OpenMP/real10.f90
index 0a092ebf6317e..a31d2ace80044 100644
--- a/flang/test/Lower/OpenMP/real10.f90
+++ b/flang/test/Lower/OpenMP/real10.f90
@@ -2,7 +2,7 @@
 
 !RUN: %flang_fc1 -emit-hlfir -fopenmp -triple amdgcn -fopenmp -fopenmp-is-target-device -o - %s | FileCheck %s
 
-!CHECK: hlfir.declare %0 {uniq_name = "_QFEx"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
+!CHECK: hlfir.declare %{{.*}} {uniq_name = "_QFEx"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
 
 program p
   real(10) :: x
diff --git a/flang/test/Lower/main_location.f90 b/flang/test/Lower/main_location.f90
index db63339288f03..73943f845f1ca 100644
--- a/flang/test/Lower/main_location.f90
+++ b/flang/test/Lower/main_location.f90
@@ -12,6 +12,7 @@
 end
 
 ! TEST1: func.func @_QQmain() {
+! TEST1-NEXT: fir.dummy_scope : !fir.dscope loc("{{.*}}test1.f90":1:1)
 ! TEST1-NEXT: return loc("{{.*}}test1.f90":3:1)
 ! TEST1-NEXT: } loc("{{.*}}test1.f90":1:1)
 
@@ -22,5 +23,6 @@
 end program
 
 ! TEST2: func.func @_QQmain() {
+! TEST2-NEXT: fir.dummy_scope : !fir.dscope loc("{{.*}}test2.f90":2:1)
 ! TEST2-NEXT: return loc("{{.*}}test2.f90":4:1)
 ! TEST2-NEXT: } loc("{{.*}}test2.f90":2:1)
diff --git a/flang/test/Transforms/tbaa-with-dummy-scope2.fir b/flang/test/Transforms/tbaa-with-dummy-scope2.fir
new file mode 100644
index 0000000000000..a28b4d56f442c
--- /dev/null
+++ b/flang/test/Transforms/tbaa-with-dummy-scope2.fir
@@ -0,0 +1,106 @@
+// RUN: fir-opt --fir-add-alias-tags --split-input-file %s | FileCheck %s
+
+// This test demonstrates the need for fir.dummy_scope even
+// when a function does not have dummy arguments.
+//
+// The original source is:
+// module m
+//   integer :: glob
+// end module m
+// subroutine test1
+//   use m
+//   call inner(glob)
+//   glob = 2
+// contains
+//   subroutine inner(x)
+//     integer :: x
+//     integer :: y
+//     y = 1
+//     x = y
+//   end subroutine inner
+// end subroutine test1
+//
+// 'inner' function is manually inlined in FIR.
+// When fir.dummy_scope is missing, TBAA tags for glob and x
+// are placed into the same TBAA root. Since glob is a global
+// and x is a dummy argument, TBAA ends up reporting no-alias
+// for them, which is incorrect.
+func.func @_QPtest1() attributes {noinline} {
+  %c1_i32 = arith.constant 1 : i32
+  %c2_i32 = arith.constant 2 : i32
+  %0 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFtest1FinnerEy"}
+  %1 = fir.address_of(@_QMmEglob) : !fir.ref<i32>
+  %2 = fir.declare %1 {uniq_name = "_QMmEglob"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  %3 = fir.dummy_scope : !fir.dscope
+  %4 = fir.declare %2 dummy_scope %3 {uniq_name = "_QFtest1FinnerEx"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %5 = fir.declare %0 {uniq_name = "_QFtest1FinnerEy"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  fir.store %c1_i32 to %5 : !fir.ref<i32>
+  %6 = fir.load %5 : !fir.ref<i32>
+  fir.store %6 to %4 : !fir.ref<i32>
+  fir.store %c2_i32 to %2 : !fir.ref<i32>
+  return
+}
+// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest1">
+// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_0]], 0>}>
+// CHECK: #[[$ATTR_2:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_1]], 0>}>
+// CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_2]], 0>}>
+// CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[$ATTR_2]], 0>}>
+// CHECK: #[[$ATTR_5:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtest1FinnerEx", members = {<#[[$ATTR_3]], 0>}>
+// CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmEglob", members = {<#[[$ATTR_4]], 0>}>
+// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_5]], access_type = #[[$ATTR_5]], offset = 0>
+// CHECK: #[[$ATTR_8:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_6]], access_type = #[[$ATTR_6]], offset = 0>
+// CHECK-LABEL:   func.func @_QPtest1() attributes {noinline} {
+// CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFtest1FinnerEy"}
+// CHECK:           %[[VAL_3:.*]] = fir.address_of(@_QMmEglob) : !fir.ref<i32>
+// CHECK:           %[[VAL_4:.*]] = fir.declare %[[VAL_3]] {uniq_name = "_QMmEglob"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]] = fir.declare %[[VAL_4]] dummy_scope %[[VAL_5]] {uniq_name = "_QFtest1FinnerEx"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+// CHECK:           %[[VAL_7:.*]] = fir.declare %[[VAL_2]] {uniq_name = "_QFtest1FinnerEy"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           fir.store %{{.*}} to %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_6]] {tbaa = [#[[$ATTR_7]]]} : !fir.ref<i32>
+// CHECK:           fir.store %{{.*}} to %[[VAL_4]] {tbaa = [#[[$ATTR_8]]]} : !fir.ref<i32>
+
+// -----
+
+// This test has fir.dummy_scope in place, and TBAA is correct.
+func.func @_QPtest2() attributes {noinline} {
+  %c1_i32 = arith.constant 1 : i32
+  %c2_i32 = arith.constant 2 : i32
+  %0 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFtest2FinnerEy"}
+  %test_dummy_scope = fir.dummy_scope : !fir.dscope
+  %1 = fir.address_of(@_QMmEglob) : !fir.ref<i32>
+  %2 = fir.declare %1 {uniq_name = "_QMmEglob"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  %3 = fir.dummy_scope : !fir.dscope
+  %4 = fir.declare %2 dummy_scope %3 {uniq_name = "_QFtest2FinnerEx"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %5 = fir.declare %0 {uniq_name = "_QFtest2FinnerEy"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  fir.store %c1_i32 to %5 : !fir.ref<i32>
+  %6 = fir.load %5 : !fir.ref<i32>
+  fir.store %6 to %4 : !fir.ref<i32>
+  fir.store %c2_i32 to %2 : !fir.ref<i32>
+  return
+}
+// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2 - Scope 1">
+// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2">
+// CHECK: #[[$ATTR_2:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_0]], 0>}>
+// CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_1]], 0>}>
+// CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_2]], 0>}>
+// CHECK: #[[$ATTR_5:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_3]], 0>}>
+// CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_4]], 0>}>
+// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[$ATTR_5]], 0>}>
+// CHECK: #[[$ATTR_8:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtest2FinnerEx", members = {<#[[$ATTR_6]], 0>}>
+// CHECK: #[[$ATTR_9:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmEglob", members = {<#[[$ATTR_7]], 0>}>
+// CHECK: #[[$ATTR_10:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_8]], access_type = #[[$ATTR_8]], offset = 0>
+// CHECK: #[[$ATTR_11:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_9]], access_type = #[[$ATTR_9]], offset = 0>
+// CHECK-LABEL:   func.func @_QPtest2() attributes {noinline} {
+// CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFtest2FinnerEy"}
+// CHECK:           %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_4:.*]] = fir.address_of(@_QMmEglob) : !fir.ref<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.declare %[[VAL_4]] {uniq_name = "_QMmEglob"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_7:.*]] = fir.declare %[[VAL_5]] dummy_scope %[[VAL_6]] {uniq_name = "_QFtest2FinnerEx"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.declare %[[VAL_2]] {uniq_name = "_QFtest2FinnerEy"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           fir.store %{{.*}} to %[[VAL_8]] : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref<i32>
+// CHECK:           fir.store %[[VAL_9]] to %[[VAL_7]] {tbaa = [#[[$ATTR_10]]]} : !fir.ref<i32>
+// CHECK:           fir.store %{{.*}} to %[[VAL_5]] {tbaa = [#[[$ATTR_11]]]} : !fir.ref<i32>

From 5c4e6c61136f5edbaf779bdcf1bda0ed88c7c3ff Mon Sep 17 00:00:00 2001
From: Rafael Auler <rafaelauler@fb.com>
Date: Fri, 18 Apr 2025 17:29:24 -0700
Subject: [PATCH 501/710] [BOLT] Don't choke on nobits symbols (#136384)

---
 bolt/lib/Rewrite/RewriteInstance.cpp |  5 +++--
 bolt/test/X86/nobits-symbol.s        | 24 ++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 bolt/test/X86/nobits-symbol.s

diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 37dcfa868c211..45bc995ce906f 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -1055,10 +1055,11 @@ void RewriteInstance::discoverFileObjects() {
       continue;
     }
 
-    if (!Section->isText()) {
+    if (!Section->isText() || Section->isVirtual()) {
       assert(SymbolType != SymbolRef::ST_Function &&
              "unexpected function inside non-code section");
-      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code\n");
+      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code or "
+                           "is in nobits section\n");
       registerName(SymbolSize);
       continue;
     }
diff --git a/bolt/test/X86/nobits-symbol.s b/bolt/test/X86/nobits-symbol.s
new file mode 100644
index 0000000000000..4110c79930b29
--- /dev/null
+++ b/bolt/test/X86/nobits-symbol.s
@@ -0,0 +1,24 @@
+## Check that llvm-bolt doesn't choke on symbols defined in nobits sections.
+
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t
+#
+
+  .type            symbol_in_nobits,@object
+  .section        .my.nobits.section,"awx",@nobits
+  .globl           symbol_in_nobits
+  .p2align         4, 0x0
+symbol_in_nobits:
+  .zero  0x100000
+  .size   symbol_in_nobits, 0x100000
+
+	.text
+  .globl main
+  .type main, %function
+main:
+	.cfi_startproc
+.LBB06:
+  retq
+	.cfi_endproc
+.size main, .-main

From 9e3982d9ae8173171cd7247ee505e9c02079c6bf Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245@hotmail.com>
Date: Fri, 18 Apr 2025 20:57:05 -0400
Subject: [PATCH 502/710] [libc++] Replace __libcpp_{ctz, clz} with
 __builtin_{ctzg, clzg} (#133920)

`__libcpp_{ctz, clz}` were previously used as fallbacks for `__builtin_{ctzg, clzg}` to ensure compatibility with older compilers (Clang 18 and earlier), as `__builtin_{ctzg, clzg}` became available in Clang 19. Now that support for Clang 18 has been officially dropped in #130142, we can now safely  replace all instances of `__libcpp_{ctz, clz}` with `__count{l,r}_zero` (which internally call `__builtin_{ctzg, clzg}` and eliminate the fallback logic.

Closes #131179.
---
 libcxx/include/__algorithm/sort.h             | 12 ++--
 libcxx/include/__bit/countl.h                 | 62 -------------------
 libcxx/include/__bit/countr.h                 | 49 ---------------
 libcxx/include/__bit_reference                |  4 +-
 libcxx/include/__charconv/to_chars_integral.h | 15 ++---
 libcxx/include/__charconv/traits.h            | 21 ++-----
 libcxx/include/__hash_table                   |  2 +-
 7 files changed, 21 insertions(+), 144 deletions(-)

diff --git a/libcxx/include/__algorithm/sort.h b/libcxx/include/__algorithm/sort.h
index d7bc1381ba5e1..06cb5b8ce7057 100644
--- a/libcxx/include/__algorithm/sort.h
+++ b/libcxx/include/__algorithm/sort.h
@@ -359,10 +359,10 @@ inline _LIBCPP_HIDE_FROM_ABI void __swap_bitmap_pos(
   // Swap one pair on each iteration as long as both bitsets have at least one
   // element for swapping.
   while (__left_bitset != 0 && __right_bitset != 0) {
-    difference_type __tz_left  = __libcpp_ctz(__left_bitset);
-    __left_bitset              = __libcpp_blsr(__left_bitset);
-    difference_type __tz_right = __libcpp_ctz(__right_bitset);
-    __right_bitset             = __libcpp_blsr(__right_bitset);
+    difference_type __tz_left  = std::__countr_zero(__left_bitset);
+    __left_bitset              = std::__libcpp_blsr(__left_bitset);
+    difference_type __tz_right = std::__countr_zero(__right_bitset);
+    __right_bitset             = std::__libcpp_blsr(__right_bitset);
     _Ops::iter_swap(__first + __tz_left, __last - __tz_right);
   }
 }
@@ -458,7 +458,7 @@ inline _LIBCPP_HIDE_FROM_ABI void __swap_bitmap_pos_within(
     // Swap within the left side.  Need to find set positions in the reverse
     // order.
     while (__left_bitset != 0) {
-      difference_type __tz_left = __detail::__block_size - 1 - __libcpp_clz(__left_bitset);
+      difference_type __tz_left = __detail::__block_size - 1 - std::__countl_zero(__left_bitset);
       __left_bitset &= (static_cast<uint64_t>(1) << __tz_left) - 1;
       _RandomAccessIterator __it = __first + __tz_left;
       if (__it != __lm1) {
@@ -471,7 +471,7 @@ inline _LIBCPP_HIDE_FROM_ABI void __swap_bitmap_pos_within(
     // Swap within the right side.  Need to find set positions in the reverse
     // order.
     while (__right_bitset != 0) {
-      difference_type __tz_right = __detail::__block_size - 1 - __libcpp_clz(__right_bitset);
+      difference_type __tz_right = __detail::__block_size - 1 - std::__countl_zero(__right_bitset);
       __right_bitset &= (static_cast<uint64_t>(1) << __tz_right) - 1;
       _RandomAccessIterator __it = __lm1 - __tz_right;
       if (__it != __first) {
diff --git a/libcxx/include/__bit/countl.h b/libcxx/include/__bit/countl.h
index d4df1d049b294..9c37749f92577 100644
--- a/libcxx/include/__bit/countl.h
+++ b/libcxx/include/__bit/countl.h
@@ -6,9 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-// TODO: __builtin_clzg is available since Clang 19 and GCC 14. When support for older versions is dropped, we can
-//  refactor this code to exclusively use __builtin_clzg.
-
 #ifndef _LIBCPP___BIT_COUNTL_H
 #define _LIBCPP___BIT_COUNTL_H
 
@@ -27,69 +24,10 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned __x) _NOEXCEPT {
-  return __builtin_clz(__x);
-}
-
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned long __x) _NOEXCEPT {
-  return __builtin_clzl(__x);
-}
-
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(unsigned long long __x) _NOEXCEPT {
-  return __builtin_clzll(__x);
-}
-
-#if _LIBCPP_HAS_INT128
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(__uint128_t __x) _NOEXCEPT {
-#  if __has_builtin(__builtin_clzg)
-  return __builtin_clzg(__x);
-#  else
-  // The function is written in this form due to C++ constexpr limitations.
-  // The algorithm:
-  // - Test whether any bit in the high 64-bits is set
-  // - No bits set:
-  //   - The high 64-bits contain 64 leading zeros,
-  //   - Add the result of the low 64-bits.
-  // - Any bits set:
-  //   - The number of leading zeros of the input is the number of leading
-  //     zeros in the high 64-bits.
-  return ((__x >> 64) == 0) ? (64 + __builtin_clzll(static_cast<unsigned long long>(__x)))
-                            : __builtin_clzll(static_cast<unsigned long long>(__x >> 64));
-#  endif
-}
-#endif // _LIBCPP_HAS_INT128
-
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countl_zero(_Tp __t) _NOEXCEPT {
   static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__countl_zero requires an unsigned integer type");
-#if __has_builtin(__builtin_clzg)
   return __builtin_clzg(__t, numeric_limits<_Tp>::digits);
-#else  // __has_builtin(__builtin_clzg)
-  if (__t == 0)
-    return numeric_limits<_Tp>::digits;
-
-  if (sizeof(_Tp) <= sizeof(unsigned int))
-    return std::__libcpp_clz(static_cast<unsigned int>(__t)) -
-           (numeric_limits<unsigned int>::digits - numeric_limits<_Tp>::digits);
-  else if (sizeof(_Tp) <= sizeof(unsigned long))
-    return std::__libcpp_clz(static_cast<unsigned long>(__t)) -
-           (numeric_limits<unsigned long>::digits - numeric_limits<_Tp>::digits);
-  else if (sizeof(_Tp) <= sizeof(unsigned long long))
-    return std::__libcpp_clz(static_cast<unsigned long long>(__t)) -
-           (numeric_limits<unsigned long long>::digits - numeric_limits<_Tp>::digits);
-  else {
-    int __ret                      = 0;
-    int __iter                     = 0;
-    const unsigned int __ulldigits = numeric_limits<unsigned long long>::digits;
-    while (true) {
-      __t = std::__rotl(__t, __ulldigits);
-      if ((__iter = std::__countl_zero(static_cast<unsigned long long>(__t))) != __ulldigits)
-        break;
-      __ret += __iter;
-    }
-    return __ret + __iter;
-  }
-#endif // __has_builtin(__builtin_clzg)
 }
 
 #if _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__bit/countr.h b/libcxx/include/__bit/countr.h
index 46c43921fc60d..f87175971bed3 100644
--- a/libcxx/include/__bit/countr.h
+++ b/libcxx/include/__bit/countr.h
@@ -6,14 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// TODO: __builtin_ctzg is available since Clang 19 and GCC 14. When support for older versions is dropped, we can
-//  refactor this code to exclusively use __builtin_ctzg.
-
 #ifndef _LIBCPP___BIT_COUNTR_H
 #define _LIBCPP___BIT_COUNTR_H
 
 #include <__assert>
-#include <__bit/rotate.h>
 #include <__concepts/arithmetic.h>
 #include <__config>
 #include <__type_traits/is_unsigned.h>
@@ -28,55 +24,10 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned __x) _NOEXCEPT {
-  return __builtin_ctz(__x);
-}
-
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned long __x) _NOEXCEPT {
-  return __builtin_ctzl(__x);
-}
-
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ctz(unsigned long long __x) _NOEXCEPT {
-  return __builtin_ctzll(__x);
-}
-
-// A constexpr implementation for C++11 and later (using clang extensions for constexpr support)
-// Precondition: __t != 0 (the caller __countr_zero handles __t == 0 as a special case)
-template <class _Tp>
-[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __countr_zero_impl(_Tp __t) _NOEXCEPT {
-  _LIBCPP_ASSERT_INTERNAL(__t != 0, "__countr_zero_impl called with zero value");
-  static_assert(is_unsigned<_Tp>::value, "__countr_zero_impl only works with unsigned types");
-  if _LIBCPP_CONSTEXPR (sizeof(_Tp) <= sizeof(unsigned int)) {
-    return std::__libcpp_ctz(static_cast<unsigned int>(__t));
-  } else if _LIBCPP_CONSTEXPR (sizeof(_Tp) <= sizeof(unsigned long)) {
-    return std::__libcpp_ctz(static_cast<unsigned long>(__t));
-  } else if _LIBCPP_CONSTEXPR (sizeof(_Tp) <= sizeof(unsigned long long)) {
-    return std::__libcpp_ctz(static_cast<unsigned long long>(__t));
-  } else {
-#if _LIBCPP_STD_VER == 11
-    unsigned long long __ull       = static_cast<unsigned long long>(__t);
-    const unsigned int __ulldigits = numeric_limits<unsigned long long>::digits;
-    return __ull == 0ull ? __ulldigits + std::__countr_zero_impl<_Tp>(__t >> __ulldigits) : std::__libcpp_ctz(__ull);
-#else
-    int __ret                      = 0;
-    const unsigned int __ulldigits = numeric_limits<unsigned long long>::digits;
-    while (static_cast<unsigned long long>(__t) == 0uLL) {
-      __ret += __ulldigits;
-      __t >>= __ulldigits;
-    }
-    return __ret + std::__libcpp_ctz(static_cast<unsigned long long>(__t));
-#endif
-  }
-}
-
 template <class _Tp>
 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __countr_zero(_Tp __t) _NOEXCEPT {
   static_assert(is_unsigned<_Tp>::value, "__countr_zero only works with unsigned types");
-#if __has_builtin(__builtin_ctzg) // TODO (LLVM 21): This can be dropped once we only support Clang >= 19.
   return __builtin_ctzg(__t, numeric_limits<_Tp>::digits);
-#else
-  return __t != 0 ? std::__countr_zero_impl(__t) : numeric_limits<_Tp>::digits;
-#endif
 }
 
 #if _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__bit_reference b/libcxx/include/__bit_reference
index 552e0f5e4d799..a3e6defd405f8 100644
--- a/libcxx/include/__bit_reference
+++ b/libcxx/include/__bit_reference
@@ -165,7 +165,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void flip() _NOEXCEPT { *__seg_ ^= __mask_; }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false> operator&() const _NOEXCEPT {
-    return __bit_iterator<_Cp, false>(__seg_, static_cast<unsigned>(std::__libcpp_ctz(__mask_)));
+    return __bit_iterator<_Cp, false>(__seg_, static_cast<unsigned>(std::__countr_zero(__mask_)));
   }
 
 private:
@@ -234,7 +234,7 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, true> operator&() const _NOEXCEPT {
-    return __bit_iterator<_Cp, true>(__seg_, static_cast<unsigned>(std::__libcpp_ctz(__mask_)));
+    return __bit_iterator<_Cp, true>(__seg_, static_cast<unsigned>(std::__countr_zero(__mask_)));
   }
 
 private:
diff --git a/libcxx/include/__charconv/to_chars_integral.h b/libcxx/include/__charconv/to_chars_integral.h
index 238c96d7c7a04..f10cc3566875a 100644
--- a/libcxx/include/__charconv/to_chars_integral.h
+++ b/libcxx/include/__charconv/to_chars_integral.h
@@ -114,9 +114,8 @@ struct _LIBCPP_HIDDEN __integral<2> {
   template <typename _Tp>
   _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int __width(_Tp __value) _NOEXCEPT {
     // If value == 0 still need one digit. If the value != this has no
-    // effect since the code scans for the most significant bit set. (Note
-    // that __libcpp_clz doesn't work for 0.)
-    return numeric_limits<_Tp>::digits - std::__libcpp_clz(__value | 1);
+    // effect since the code scans for the most significant bit set.
+    return numeric_limits<_Tp>::digits - std::__countl_zero(__value | 1);
   }
 
   template <typename _Tp>
@@ -150,9 +149,8 @@ struct _LIBCPP_HIDDEN __integral<8> {
   template <typename _Tp>
   _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int __width(_Tp __value) _NOEXCEPT {
     // If value == 0 still need one digit. If the value != this has no
-    // effect since the code scans for the most significat bit set. (Note
-    // that __libcpp_clz doesn't work for 0.)
-    return ((numeric_limits<_Tp>::digits - std::__libcpp_clz(__value | 1)) + 2) / 3;
+    // effect since the code scans for the most significat bit set.
+    return ((numeric_limits<_Tp>::digits - std::__countl_zero(__value | 1)) + 2) / 3;
   }
 
   template <typename _Tp>
@@ -186,9 +184,8 @@ struct _LIBCPP_HIDDEN __integral<16> {
   template <typename _Tp>
   _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int __width(_Tp __value) _NOEXCEPT {
     // If value == 0 still need one digit. If the value != this has no
-    // effect since the code scans for the most significat bit set. (Note
-    // that __libcpp_clz doesn't work for 0.)
-    return (numeric_limits<_Tp>::digits - std::__libcpp_clz(__value | 1) + 3) / 4;
+    // effect since the code scans for the most significat bit set.
+    return (numeric_limits<_Tp>::digits - std::__countl_zero(__value | 1) + 3) / 4;
   }
 
   template <typename _Tp>
diff --git a/libcxx/include/__charconv/traits.h b/libcxx/include/__charconv/traits.h
index 085a3f7886f31..9fd0092ca79c3 100644
--- a/libcxx/include/__charconv/traits.h
+++ b/libcxx/include/__charconv/traits.h
@@ -43,12 +43,9 @@ struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) <= sizeof(uin
   ///
   /// The algorithm is based on
   /// http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10
-  /// Instead of using IntegerLogBase2 it uses __libcpp_clz. Since that
-  /// function requires its input to have at least one bit set the value of
-  /// zero is set to one. This means the first element of the lookup table is
-  /// zero.
+  /// Instead of using IntegerLogBase2 it uses __countl_zero.
   static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __width(_Tp __v) {
-    auto __t = (32 - std::__libcpp_clz(static_cast<type>(__v | 1))) * 1233 >> 12;
+    auto __t = (32 - std::__countl_zero(static_cast<type>(__v | 1))) * 1233 >> 12;
     return __t - (__v < __itoa::__pow10_32[__t]) + 1;
   }
 
@@ -69,12 +66,9 @@ struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) == sizeof(uin
   ///
   /// The algorithm is based on
   /// http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10
-  /// Instead of using IntegerLogBase2 it uses __libcpp_clz. Since that
-  /// function requires its input to have at least one bit set the value of
-  /// zero is set to one. This means the first element of the lookup table is
-  /// zero.
+  /// Instead of using IntegerLogBase2 it uses __countl_zero.
   static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __width(_Tp __v) {
-    auto __t = (64 - std::__libcpp_clz(static_cast<type>(__v | 1))) * 1233 >> 12;
+    auto __t = (64 - std::__countl_zero(static_cast<type>(__v | 1))) * 1233 >> 12;
     return __t - (__v < __itoa::__pow10_64[__t]) + 1;
   }
 
@@ -96,15 +90,12 @@ struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) == sizeof(__u
   ///
   /// The algorithm is based on
   /// http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10
-  /// Instead of using IntegerLogBase2 it uses __libcpp_clz. Since that
-  /// function requires its input to have at least one bit set the value of
-  /// zero is set to one. This means the first element of the lookup table is
-  /// zero.
+  /// Instead of using IntegerLogBase2 it uses __countl_zero.
   static _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __width(_Tp __v) {
     _LIBCPP_ASSERT_INTERNAL(
         __v > numeric_limits<uint64_t>::max(), "The optimizations for this algorithm fail when this isn't true.");
     // There's always a bit set in the upper 64-bits.
-    auto __t = (128 - std::__libcpp_clz(static_cast<uint64_t>(__v >> 64))) * 1233 >> 12;
+    auto __t = (128 - std::__countl_zero(static_cast<uint64_t>(__v >> 64))) * 1233 >> 12;
     _LIBCPP_ASSERT_INTERNAL(__t >= __itoa::__pow10_128_offset, "Index out of bounds");
     // __t is adjusted since the lookup table misses the lower entries.
     return __t - (__v < __itoa::__pow10_128[__t - __itoa::__pow10_128_offset]) + 1;
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index fca58ca296067..aefa8e19c1864 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -147,7 +147,7 @@ inline _LIBCPP_HIDE_FROM_ABI size_t __constrain_hash(size_t __h, size_t __bc) {
 }
 
 inline _LIBCPP_HIDE_FROM_ABI size_t __next_hash_pow2(size_t __n) {
-  return __n < 2 ? __n : (size_t(1) << (numeric_limits<size_t>::digits - __libcpp_clz(__n - 1)));
+  return __n < 2 ? __n : (size_t(1) << (numeric_limits<size_t>::digits - std::__countl_zero(__n - 1)));
 }
 
 template <class _Tp, class _Hash, class _Equal, class _Alloc>

From 062db99353749ec73a19b030b5553beee59cb0c0 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 18 Apr 2025 18:06:57 -0700
Subject: [PATCH 503/710] [clang-doc][NFC] Use destructuring in Mapper.cpp
 (#135515)

Destructuring makes the intent a bit clearer over first/second.
---
 clang-tools-extra/clang-doc/Mapper.cpp | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/clang-tools-extra/clang-doc/Mapper.cpp b/clang-tools-extra/clang-doc/Mapper.cpp
index 022e3b12d82bc..950cdf55d4a63 100644
--- a/clang-tools-extra/clang-doc/Mapper.cpp
+++ b/clang-tools-extra/clang-doc/Mapper.cpp
@@ -59,18 +59,18 @@ bool MapASTVisitor::mapDecl(const T *D, bool IsDefinition) {
   bool IsFileInRootDir;
   llvm::SmallString<128> File =
       getFile(D, D->getASTContext(), CDCtx.SourceRoot, IsFileInRootDir);
-  auto I = serialize::emitInfo(D, getComment(D, D->getASTContext()),
-                               getLine(D, D->getASTContext()), File,
-                               IsFileInRootDir, CDCtx.PublicOnly);
-
-  // A null in place of I indicates that the serializer is skipping this decl
-  // for some reason (e.g. we're only reporting public decls).
-  if (I.first)
-    CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(I.first->USR)),
-                             serialize::serialize(I.first));
-  if (I.second)
-    CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(I.second->USR)),
-                             serialize::serialize(I.second));
+  auto [Child, Parent] = serialize::emitInfo(
+      D, getComment(D, D->getASTContext()), getLine(D, D->getASTContext()),
+      File, IsFileInRootDir, CDCtx.PublicOnly);
+
+  // A null in place of a valid Info indicates that the serializer is skipping
+  // this decl for some reason (e.g. we're only reporting public decls).
+  if (Child)
+    CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Child->USR)),
+                             serialize::serialize(Child));
+  if (Parent)
+    CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Parent->USR)),
+                             serialize::serialize(Parent));
   return true;
 }
 

From 5c3789811fd5b50df1178e7068efb75c5b359383 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 18 Apr 2025 18:08:06 -0700
Subject: [PATCH 504/710] [clang-doc] Handle static members and functions
 (#135457)

clang-doc didn't visit VarDecl, and hence never collected info
from class statics members and functions.

Fixes #59813.
---
 clang-tools-extra/clang-doc/BitcodeReader.cpp |  4 ++
 clang-tools-extra/clang-doc/BitcodeWriter.cpp |  9 +++-
 clang-tools-extra/clang-doc/BitcodeWriter.h   |  2 +
 clang-tools-extra/clang-doc/HTMLGenerator.cpp | 13 ++++--
 clang-tools-extra/clang-doc/MDGenerator.cpp   | 25 +++++------
 clang-tools-extra/clang-doc/Representation.h  | 18 ++++----
 clang-tools-extra/clang-doc/Serialize.cpp     | 42 ++++++++++++++-----
 clang-tools-extra/clang-doc/Serialize.h       |  4 ++
 .../test/clang-doc/basic-project.test         |  8 +++-
 9 files changed, 85 insertions(+), 40 deletions(-)

diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index 1f2fb0a8b2b85..d6c9da81a05c7 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -274,6 +274,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
     return decodeRecord(R, I->Access, Blob);
   case FUNCTION_IS_METHOD:
     return decodeRecord(R, I->IsMethod, Blob);
+  case FUNCTION_IS_STATIC:
+    return decodeRecord(R, I->IsStatic, Blob);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid field for FunctionInfo");
@@ -305,6 +307,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
     return decodeRecord(R, I->Name, Blob);
   case MEMBER_TYPE_ACCESS:
     return decodeRecord(R, I->Access, Blob);
+  case MEMBER_TYPE_IS_STATIC:
+    return decodeRecord(R, I->IsStatic, Blob);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid field for MemberTypeInfo");
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.cpp b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
index 06f30f76e33d8..621af4e51e443 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
@@ -156,6 +156,7 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {FIELD_DEFAULT_VALUE, {"DefaultValue", &StringAbbrev}},
           {MEMBER_TYPE_NAME, {"Name", &StringAbbrev}},
           {MEMBER_TYPE_ACCESS, {"Access", &IntAbbrev}},
+          {MEMBER_TYPE_IS_STATIC, {"IsStatic", &BoolAbbrev}},
           {NAMESPACE_USR, {"USR", &SymbolIDAbbrev}},
           {NAMESPACE_NAME, {"Name", &StringAbbrev}},
           {NAMESPACE_PATH, {"Path", &StringAbbrev}},
@@ -187,6 +188,7 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {FUNCTION_LOCATION, {"Location", &LocationAbbrev}},
           {FUNCTION_ACCESS, {"Access", &IntAbbrev}},
           {FUNCTION_IS_METHOD, {"IsMethod", &BoolAbbrev}},
+          {FUNCTION_IS_STATIC, {"IsStatic", &BoolAbbrev}},
           {REFERENCE_USR, {"USR", &SymbolIDAbbrev}},
           {REFERENCE_NAME, {"Name", &StringAbbrev}},
           {REFERENCE_QUAL_NAME, {"QualName", &StringAbbrev}},
@@ -222,7 +224,8 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
         // FieldType Block
         {BI_FIELD_TYPE_BLOCK_ID, {FIELD_TYPE_NAME, FIELD_DEFAULT_VALUE}},
         // MemberType Block
-        {BI_MEMBER_TYPE_BLOCK_ID, {MEMBER_TYPE_NAME, MEMBER_TYPE_ACCESS}},
+        {BI_MEMBER_TYPE_BLOCK_ID,
+         {MEMBER_TYPE_NAME, MEMBER_TYPE_ACCESS, MEMBER_TYPE_IS_STATIC}},
         // Enum Block
         {BI_ENUM_BLOCK_ID,
          {ENUM_USR, ENUM_NAME, ENUM_DEFLOCATION, ENUM_LOCATION, ENUM_SCOPED}},
@@ -247,7 +250,7 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
         // Function Block
         {BI_FUNCTION_BLOCK_ID,
          {FUNCTION_USR, FUNCTION_NAME, FUNCTION_DEFLOCATION, FUNCTION_LOCATION,
-          FUNCTION_ACCESS, FUNCTION_IS_METHOD}},
+          FUNCTION_ACCESS, FUNCTION_IS_METHOD, FUNCTION_IS_STATIC}},
         // Reference Block
         {BI_REFERENCE_BLOCK_ID,
          {REFERENCE_USR, REFERENCE_NAME, REFERENCE_QUAL_NAME, REFERENCE_TYPE,
@@ -465,6 +468,7 @@ void ClangDocBitcodeWriter::emitBlock(const MemberTypeInfo &T) {
   emitBlock(T.Type, FieldId::F_type);
   emitRecord(T.Name, MEMBER_TYPE_NAME);
   emitRecord(T.Access, MEMBER_TYPE_ACCESS);
+  emitRecord(T.IsStatic, MEMBER_TYPE_IS_STATIC);
   for (const auto &CI : T.Description)
     emitBlock(CI);
 }
@@ -600,6 +604,7 @@ void ClangDocBitcodeWriter::emitBlock(const FunctionInfo &I) {
     emitBlock(CI);
   emitRecord(I.Access, FUNCTION_ACCESS);
   emitRecord(I.IsMethod, FUNCTION_IS_METHOD);
+  emitRecord(I.IsStatic, FUNCTION_IS_STATIC);
   if (I.DefLoc)
     emitRecord(*I.DefLoc, FUNCTION_DEFLOCATION);
   for (const auto &L : I.Loc)
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.h b/clang-tools-extra/clang-doc/BitcodeWriter.h
index 9a572e40e352f..e33a1aece883c 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.h
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.h
@@ -81,6 +81,7 @@ enum RecordId {
   FUNCTION_LOCATION,
   FUNCTION_ACCESS,
   FUNCTION_IS_METHOD,
+  FUNCTION_IS_STATIC,
   COMMENT_KIND,
   COMMENT_TEXT,
   COMMENT_NAME,
@@ -96,6 +97,7 @@ enum RecordId {
   FIELD_DEFAULT_VALUE,
   MEMBER_TYPE_NAME,
   MEMBER_TYPE_ACCESS,
+  MEMBER_TYPE_IS_STATIC,
   NAMESPACE_USR,
   NAMESPACE_NAME,
   NAMESPACE_PATH,
diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index aceb83e8c4c57..ef6e61d34e1bf 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -416,12 +416,14 @@ genRecordMembersBlock(const llvm::SmallVector<MemberTypeInfo, 4> &Members,
   Out.emplace_back(std::make_unique<TagNode>(HTMLTag::TAG_UL));
   auto &ULBody = Out.back();
   for (const auto &M : Members) {
-    std::string Access = getAccessSpelling(M.Access).str();
-    if (Access != "")
-      Access = Access + " ";
+    StringRef Access = getAccessSpelling(M.Access);
     auto LIBody = std::make_unique<TagNode>(HTMLTag::TAG_LI);
     auto MemberDecl = std::make_unique<TagNode>(HTMLTag::TAG_DIV);
-    MemberDecl->Children.emplace_back(std::make_unique<TextNode>(Access));
+    if (!Access.empty())
+      MemberDecl->Children.emplace_back(
+          std::make_unique<TextNode>(Access + " "));
+    if (M.IsStatic)
+      MemberDecl->Children.emplace_back(std::make_unique<TextNode>("static "));
     MemberDecl->Children.emplace_back(genReference(M.Type, ParentInfoDir));
     MemberDecl->Children.emplace_back(std::make_unique<TextNode>(" " + M.Name));
     if (!M.Description.empty())
@@ -739,6 +741,9 @@ genHTML(const FunctionInfo &I, const ClangDocContext &CDCtx,
   if (Access != "")
     FunctionHeader->Children.emplace_back(
         std::make_unique<TextNode>(Access + " "));
+  if (I.IsStatic)
+    FunctionHeader->Children.emplace_back(
+        std::make_unique<TextNode>("static "));
   if (I.ReturnType.Type.Name != "") {
     FunctionHeader->Children.emplace_back(
         genReference(I.ReturnType.Type, ParentInfoDir));
diff --git a/clang-tools-extra/clang-doc/MDGenerator.cpp b/clang-tools-extra/clang-doc/MDGenerator.cpp
index 5c782fcc10da5..43f1d7ba2f117 100644
--- a/clang-tools-extra/clang-doc/MDGenerator.cpp
+++ b/clang-tools-extra/clang-doc/MDGenerator.cpp
@@ -169,15 +169,12 @@ static void genMarkdown(const ClangDocContext &CDCtx, const FunctionInfo &I,
     First = false;
   }
   writeHeader(I.Name, 3, OS);
-  std::string Access = getAccessSpelling(I.Access).str();
-  if (Access != "")
-    writeLine(genItalic(Access + " " + I.ReturnType.Type.QualName + " " +
-                        I.Name + "(" + Stream.str() + ")"),
-              OS);
-  else
-    writeLine(genItalic(I.ReturnType.Type.QualName + " " + I.Name + "(" +
-                        Stream.str() + ")"),
-              OS);
+  StringRef Access = getAccessSpelling(I.Access);
+  writeLine(genItalic(Twine(Access) + (!Access.empty() ? " " : "") +
+                      (I.IsStatic ? "static " : "") +
+                      I.ReturnType.Type.QualName.str() + " " + I.Name.str() +
+                      "(" + Twine(Stream.str()) + ")"),
+            OS);
 
   maybeWriteSourceFileRef(OS, CDCtx, I.DefLoc);
 
@@ -262,11 +259,11 @@ static void genMarkdown(const ClangDocContext &CDCtx, const RecordInfo &I,
   if (!I.Members.empty()) {
     writeHeader("Members", 2, OS);
     for (const auto &Member : I.Members) {
-      std::string Access = getAccessSpelling(Member.Access).str();
-      if (Access != "")
-        writeLine(Access + " " + Member.Type.Name + " " + Member.Name, OS);
-      else
-        writeLine(Member.Type.Name + " " + Member.Name, OS);
+      StringRef Access = getAccessSpelling(Member.Access);
+      writeLine(Twine(Access) + (Access.empty() ? "" : " ") +
+                    (Member.IsStatic ? "static " : "") +
+                    Member.Type.Name.str() + " " + Member.Name.str(),
+                OS);
     }
     writeNewLine(OS);
   }
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index 2153b62864ee3..d7f2453fe11b2 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -220,12 +220,14 @@ struct FieldTypeInfo : public TypeInfo {
 // Info for member types.
 struct MemberTypeInfo : public FieldTypeInfo {
   MemberTypeInfo() = default;
-  MemberTypeInfo(const TypeInfo &TI, StringRef Name, AccessSpecifier Access)
-      : FieldTypeInfo(TI, Name), Access(Access) {}
+  MemberTypeInfo(const TypeInfo &TI, StringRef Name, AccessSpecifier Access,
+                 bool IsStatic = false)
+      : FieldTypeInfo(TI, Name), Access(Access), IsStatic(IsStatic) {}
 
   bool operator==(const MemberTypeInfo &Other) const {
-    return std::tie(Type, Name, Access, Description) ==
-           std::tie(Other.Type, Other.Name, Other.Access, Other.Description);
+    return std::tie(Type, Name, Access, IsStatic, Description) ==
+           std::tie(Other.Type, Other.Name, Other.Access, Other.IsStatic,
+                    Other.Description);
   }
 
   // Access level associated with this info (public, protected, private, none).
@@ -235,6 +237,7 @@ struct MemberTypeInfo : public FieldTypeInfo {
   AccessSpecifier Access = AccessSpecifier::AS_public;
 
   std::vector<CommentInfo> Description; // Comment description of this field.
+  bool IsStatic = false;
 };
 
 struct Location {
@@ -320,9 +323,6 @@ struct SymbolInfo : public Info {
 
   void merge(SymbolInfo &&I);
 
-  std::optional<Location> DefLoc;     // Location where this decl is defined.
-  llvm::SmallVector<Location, 2> Loc; // Locations where this decl is declared.
-
   bool operator<(const SymbolInfo &Other) const {
     // Sort by declaration location since we want the doc to be
     // generated in the order of the source code.
@@ -336,6 +336,10 @@ struct SymbolInfo : public Info {
 
     return extractName() < Other.extractName();
   }
+
+  std::optional<Location> DefLoc;     // Location where this decl is defined.
+  llvm::SmallVector<Location, 2> Loc; // Locations where this decl is declared.
+  bool IsStatic = false;
 };
 
 // TODO: Expand to allow for documenting templating and default args.
diff --git a/clang-tools-extra/clang-doc/Serialize.cpp b/clang-tools-extra/clang-doc/Serialize.cpp
index f737fc75135a1..837ecdd9f1278 100644
--- a/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/clang-tools-extra/clang-doc/Serialize.cpp
@@ -30,7 +30,10 @@ static void
 populateParentNamespaces(llvm::SmallVector<Reference, 4> &Namespaces,
                          const T *D, bool &IsAnonymousNamespace);
 
-static void populateMemberTypeInfo(MemberTypeInfo &I, const FieldDecl *D);
+static void populateMemberTypeInfo(MemberTypeInfo &I, const Decl *D);
+static void populateMemberTypeInfo(RecordInfo &I, AccessSpecifier &Access,
+                                   const DeclaratorDecl *D,
+                                   bool IsStatic = false);
 
 // A function to extract the appropriate relative path for a given info's
 // documentation. The path returned is a composite of the parent namespaces.
@@ -378,15 +381,19 @@ static void parseFields(RecordInfo &I, const RecordDecl *D, bool PublicOnly,
   for (const FieldDecl *F : D->fields()) {
     if (!shouldSerializeInfo(PublicOnly, /*IsInAnonymousNamespace=*/false, F))
       continue;
+    populateMemberTypeInfo(I, Access, F);
+  }
+  const auto *CxxRD = dyn_cast<CXXRecordDecl>(D);
+  if (!CxxRD)
+    return;
+  for (Decl *CxxDecl : CxxRD->decls()) {
+    auto *VD = dyn_cast<VarDecl>(CxxDecl);
+    if (!VD ||
+        !shouldSerializeInfo(PublicOnly, /*IsInAnonymousNamespace=*/false, VD))
+      continue;
 
-    auto &LO = F->getLangOpts();
-    // Use getAccessUnsafe so that we just get the default AS_none if it's not
-    // valid, as opposed to an assert.
-    MemberTypeInfo &NewMember = I.Members.emplace_back(
-        getTypeInfoForType(F->getTypeSourceInfo()->getType(), LO),
-        F->getNameAsString(),
-        getFinalAccessSpecifier(Access, F->getAccessUnsafe()));
-    populateMemberTypeInfo(NewMember, F);
+    if (VD->isStaticDataMember())
+      populateMemberTypeInfo(I, Access, VD, /*IsStatic=*/true);
   }
 }
 
@@ -568,7 +575,7 @@ static void populateFunctionInfo(FunctionInfo &I, const FunctionDecl *D,
   }
 }
 
-static void populateMemberTypeInfo(MemberTypeInfo &I, const FieldDecl *D) {
+static void populateMemberTypeInfo(MemberTypeInfo &I, const Decl *D) {
   assert(D && "Expect non-null FieldDecl in populateMemberTypeInfo");
 
   ASTContext& Context = D->getASTContext();
@@ -585,6 +592,17 @@ static void populateMemberTypeInfo(MemberTypeInfo &I, const FieldDecl *D) {
   }
 }
 
+static void populateMemberTypeInfo(RecordInfo &I, AccessSpecifier &Access,
+                                   const DeclaratorDecl *D, bool IsStatic) {
+  // Use getAccessUnsafe so that we just get the default AS_none if it's not
+  // valid, as opposed to an assert.
+  MemberTypeInfo &NewMember = I.Members.emplace_back(
+      getTypeInfoForType(D->getTypeSourceInfo()->getType(), D->getLangOpts()),
+      D->getNameAsString(),
+      getFinalAccessSpecifier(Access, D->getAccessUnsafe()), IsStatic);
+  populateMemberTypeInfo(NewMember, D);
+}
+
 static void
 parseBases(RecordInfo &I, const CXXRecordDecl *D, bool IsFileInRootDir,
            bool PublicOnly, bool IsParent,
@@ -619,6 +637,7 @@ parseBases(RecordInfo &I, const CXXRecordDecl *D, bool IsFileInRootDir,
               continue;
             FunctionInfo FI;
             FI.IsMethod = true;
+            FI.IsStatic = MD->isStatic();
             // The seventh arg in populateFunctionInfo is a boolean passed by
             // reference, its value is not relevant in here so it's not used
             // anywhere besides the function call.
@@ -702,7 +721,7 @@ emitInfo(const RecordDecl *D, const FullComment *FC, int LineNumber,
                  dyn_cast<ClassTemplatePartialSpecializationDecl *>(SpecOf))
       Specialization.SpecializationOf = getUSRForDecl(CTPSD);
 
-    // Parameters to the specilization. For partial specializations, get the
+    // Parameters to the specialization. For partial specializations, get the
     // parameters "as written" from the ClassTemplatePartialSpecializationDecl
     // because the non-explicit template parameters will have generated internal
     // placeholder names rather than the names the user typed that match the
@@ -755,6 +774,7 @@ emitInfo(const CXXMethodDecl *D, const FullComment *FC, int LineNumber,
     return {};
 
   Func.IsMethod = true;
+  Func.IsStatic = D->isStatic();
 
   const NamedDecl *Parent = nullptr;
   if (const auto *SD =
diff --git a/clang-tools-extra/clang-doc/Serialize.h b/clang-tools-extra/clang-doc/Serialize.h
index 4e203ca7891ac..41946796f39f6 100644
--- a/clang-tools-extra/clang-doc/Serialize.h
+++ b/clang-tools-extra/clang-doc/Serialize.h
@@ -52,6 +52,10 @@ std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
 emitInfo(const FunctionDecl *D, const FullComment *FC, int LineNumber,
          StringRef File, bool IsFileInRootDir, bool PublicOnly);
 
+std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
+emitInfo(const VarDecl *D, const FullComment *FC, int LineNumber,
+         StringRef File, bool IsFileInRootDir, bool PublicOnly);
+
 std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
 emitInfo(const CXXMethodDecl *D, const FullComment *FC, int LineNumber,
          StringRef File, bool IsFileInRootDir, bool PublicOnly);
diff --git a/clang-tools-extra/test/clang-doc/basic-project.test b/clang-tools-extra/test/clang-doc/basic-project.test
index 94484b393df59..9c1ed29973d79 100644
--- a/clang-tools-extra/test/clang-doc/basic-project.test
+++ b/clang-tools-extra/test/clang-doc/basic-project.test
@@ -134,6 +134,9 @@
 //      HTML-CALC: <div>brief</div>
 //      HTML-CALC: <p> Holds a public value.</p>
 //      HTML-CALC: <div>public int public_val</div>
+//      HTML-CALC: <div>brief</div>
+//      HTML-CALC: <p> A static value.</p>
+//      HTML-CALC: <div>public static const int static_val</div>
 
 //      HTML-CALC: <h2 id="Functions">Functions</h2>
 //      HTML-CALC: <h3 id="{{([0-9A-F]{40})}}">add</h3>
@@ -191,7 +194,7 @@
 //      HTML-CALC: <div>throw</div>
 //      HTML-CALC: <p>if b is zero.</p>
 
-//      HTML-CALC: <p>public int mod(int a, int b)</p>
+//      HTML-CALC: <p>public static int mod(int a, int b)</p>
 //   CALC-NO-REPOSITORY: Defined at line 54 of file .{{.}}include{{.}}Calculator.h
 //      CALC-REPOSITORY: Defined at line 
 // CALC-REPOSITORY-NEXT: <a href="https://repository.com/./include/Calculator.h#54">54</a>
@@ -326,6 +329,7 @@
 // MD-CALC:  Provides basic arithmetic operations.
 // MD-CALC: ## Members
 // MD-CALC: public int public_val
+// MD-CALC: public static const int static_val
 // MD-CALC: ## Functions
 // MD-CALC: ### add
 // MD-CALC: *public int add(int a, int b)*
@@ -357,7 +361,7 @@
 // MD-CALC: **return** double The result of a / b.
 // MD-CALC: **throw**if b is zero.
 // MD-CALC: ### mod
-// MD-CALC: *public int mod(int a, int b)*
+// MD-CALC: *public static int mod(int a, int b)*
 // MD-CALC: *Defined at ./include{{[\/]}}Calculator.h#54*
 // MD-CALC: **brief** Performs the mod operation on integers.
 // MD-CALC: **a** First integer.

From 667209e45122d0cb7a4c2ac27018d31165b1be70 Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <peter@pcc.me.uk>
Date: Fri, 18 Apr 2025 18:11:18 -0700
Subject: [PATCH 505/710] Config: Move LLVM_HAS_*_TARGET definitions to a new
 header.

When enabling or disabling a target we typically need to rebuild most
of LLVM because of the change to the values of the LLVM_HAS_*_TARGET
macros in llvm-config.h, which is included by most of the code, but
are unused by LLVM itself. To avoid this, move the LLVM_HAS_*_TARGET
macros to a separate header, Targets.h.

Update the only in-tree user of the macros (MLIR) to refer to the new
header. I expect that out-of-tree users will detect the change either
at compile time if they build with -Wundef, or at runtime. As far as
I can tell, the usage of these macros is rare in out-of-tree projects,
I found no out-of-tree users in projects indexed by Debian code search
[1], and one user [2] in projects indexed by GitHub code search [3]
(excluding forks of LLVM).

[1] https://codesearch.debian.net/search?q=%23.*LLVM_HAS_.*_TARGET&literal=0
[2] https://github.com/AndreyPavlenko/graph-compiler/blob/238706b12b63945dc490f9f5f33a2d20b3c58944/lib/gc/Target/LLVM/XeVM/Target.cpp#L72
[3] https://github.com/search?q=%2F%23.*LLVM_HAS_.*_TARGET%2F&type=code

Reviewers: nico, grypp, mstorsjo, MaskRay

Reviewed By: MaskRay

Pull Request: https://github.com/llvm/llvm-project/pull/136388
---
 llvm/CMakeLists.txt                           |  5 +-
 llvm/include/llvm/Config/Targets.h.cmake      | 90 +++++++++++++++++++
 llvm/include/llvm/Config/llvm-config.h.cmake  | 75 ----------------
 .../llvm/include/llvm/Config/BUILD.gn         | 11 ++-
 mlir/lib/Target/LLVM/NVVM/Target.cpp          |  2 +-
 .../Target/LLVM/SerializeNVVMTarget.cpp       |  2 +-
 6 files changed, 106 insertions(+), 79 deletions(-)
 create mode 100644 llvm/include/llvm/Config/Targets.h.cmake

diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index cfd1a086c0fc2..693cb085b8e2f 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -1163,13 +1163,16 @@ if (NOT TENSORFLOW_AOT_PATH STREQUAL "")
 
 endif()
 
-# Configure the three LLVM configuration header files.
+# Configure the LLVM configuration header files.
 configure_file(
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/config.h.cmake
   ${LLVM_INCLUDE_DIR}/llvm/Config/config.h)
 configure_file(
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/llvm-config.h.cmake
   ${LLVM_INCLUDE_DIR}/llvm/Config/llvm-config.h)
+configure_file(
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/Targets.h.cmake
+  ${LLVM_INCLUDE_DIR}/llvm/Config/Targets.h)
 configure_file(
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/abi-breaking.h.cmake
   ${LLVM_INCLUDE_DIR}/llvm/Config/abi-breaking.h)
diff --git a/llvm/include/llvm/Config/Targets.h.cmake b/llvm/include/llvm/Config/Targets.h.cmake
new file mode 100644
index 0000000000000..5bab635edc542
--- /dev/null
+++ b/llvm/include/llvm/Config/Targets.h.cmake
@@ -0,0 +1,90 @@
+/*===------- llvm/Config/Targets.h - LLVM target checks -----------*- C -*-===*/
+/*                                                                            */
+/* Part of the LLVM Project, under the Apache License v2.0 with LLVM          */
+/* Exceptions.                                                                */
+/* See https://llvm.org/LICENSE.txt for license information.                  */
+/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    */
+/*                                                                            */
+/*===----------------------------------------------------------------------===*/
+
+/* This file enables clients to know whether specific targets are enabled. */
+
+#ifndef LLVM_CONFIG_TARGETS_H
+#define LLVM_CONFIG_TARGETS_H
+
+/* Define if the AArch64 target is built in */
+#cmakedefine01 LLVM_HAS_AARCH64_TARGET
+
+/* Define if the AMDGPU target is built in */
+#cmakedefine01 LLVM_HAS_AMDGPU_TARGET
+
+/* Define if the ARC target is built in */
+#cmakedefine01 LLVM_HAS_ARC_TARGET
+
+/* Define if the ARM target is built in */
+#cmakedefine01 LLVM_HAS_ARM_TARGET
+
+/* Define if the AVR target is built in */
+#cmakedefine01 LLVM_HAS_AVR_TARGET
+
+/* Define if the BPF target is built in */
+#cmakedefine01 LLVM_HAS_BPF_TARGET
+
+/* Define if the CSKY target is built in */
+#cmakedefine01 LLVM_HAS_CSKY_TARGET
+
+/* Define if the DirectX target is built in */
+#cmakedefine01 LLVM_HAS_DIRECTX_TARGET
+
+/* Define if the Hexagon target is built in */
+#cmakedefine01 LLVM_HAS_HEXAGON_TARGET
+
+/* Define if the Lanai target is built in */
+#cmakedefine01 LLVM_HAS_LANAI_TARGET
+
+/* Define if the LoongArch target is built in */
+#cmakedefine01 LLVM_HAS_LOONGARCH_TARGET
+
+/* Define if the M68k target is built in */
+#cmakedefine01 LLVM_HAS_M68K_TARGET
+
+/* Define if the Mips target is built in */
+#cmakedefine01 LLVM_HAS_MIPS_TARGET
+
+/* Define if the MSP430 target is built in */
+#cmakedefine01 LLVM_HAS_MSP430_TARGET
+
+/* Define if the NVPTX target is built in */
+#cmakedefine01 LLVM_HAS_NVPTX_TARGET
+
+/* Define if the PowerPC target is built in */
+#cmakedefine01 LLVM_HAS_POWERPC_TARGET
+
+/* Define if the RISCV target is built in */
+#cmakedefine01 LLVM_HAS_RISCV_TARGET
+
+/* Define if the Sparc target is built in */
+#cmakedefine01 LLVM_HAS_SPARC_TARGET
+
+/* Define if the SPIRV target is built in */
+#cmakedefine01 LLVM_HAS_SPIRV_TARGET
+
+/* Define if the SystemZ target is built in */
+#cmakedefine01 LLVM_HAS_SYSTEMZ_TARGET
+
+/* Define if the VE target is built in */
+#cmakedefine01 LLVM_HAS_VE_TARGET
+
+/* Define if the WebAssembly target is built in */
+#cmakedefine01 LLVM_HAS_WEBASSEMBLY_TARGET
+
+/* Define if the X86 target is built in */
+#cmakedefine01 LLVM_HAS_X86_TARGET
+
+/* Define if the XCore target is built in */
+#cmakedefine01 LLVM_HAS_XCORE_TARGET
+
+/* Define if the Xtensa target is built in */
+#cmakedefine01 LLVM_HAS_XTENSA_TARGET
+
+#endif
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 4230eb850fa4b..97f996338bc0c 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -54,81 +54,6 @@
 /* LLVM name for the native target MCA init function, if available */
 #cmakedefine LLVM_NATIVE_TARGETMCA LLVMInitialize${LLVM_NATIVE_ARCH}TargetMCA
 
-/* Define if the AArch64 target is built in */
-#cmakedefine01 LLVM_HAS_AARCH64_TARGET
-
-/* Define if the AMDGPU target is built in */
-#cmakedefine01 LLVM_HAS_AMDGPU_TARGET
-
-/* Define if the ARC target is built in */
-#cmakedefine01 LLVM_HAS_ARC_TARGET
-
-/* Define if the ARM target is built in */
-#cmakedefine01 LLVM_HAS_ARM_TARGET
-
-/* Define if the AVR target is built in */
-#cmakedefine01 LLVM_HAS_AVR_TARGET
-
-/* Define if the BPF target is built in */
-#cmakedefine01 LLVM_HAS_BPF_TARGET
-
-/* Define if the CSKY target is built in */
-#cmakedefine01 LLVM_HAS_CSKY_TARGET
-
-/* Define if the DirectX target is built in */
-#cmakedefine01 LLVM_HAS_DIRECTX_TARGET
-
-/* Define if the Hexagon target is built in */
-#cmakedefine01 LLVM_HAS_HEXAGON_TARGET
-
-/* Define if the Lanai target is built in */
-#cmakedefine01 LLVM_HAS_LANAI_TARGET
-
-/* Define if the LoongArch target is built in */
-#cmakedefine01 LLVM_HAS_LOONGARCH_TARGET
-
-/* Define if the M68k target is built in */
-#cmakedefine01 LLVM_HAS_M68K_TARGET
-
-/* Define if the Mips target is built in */
-#cmakedefine01 LLVM_HAS_MIPS_TARGET
-
-/* Define if the MSP430 target is built in */
-#cmakedefine01 LLVM_HAS_MSP430_TARGET
-
-/* Define if the NVPTX target is built in */
-#cmakedefine01 LLVM_HAS_NVPTX_TARGET
-
-/* Define if the PowerPC target is built in */
-#cmakedefine01 LLVM_HAS_POWERPC_TARGET
-
-/* Define if the RISCV target is built in */
-#cmakedefine01 LLVM_HAS_RISCV_TARGET
-
-/* Define if the Sparc target is built in */
-#cmakedefine01 LLVM_HAS_SPARC_TARGET
-
-/* Define if the SPIRV target is built in */
-#cmakedefine01 LLVM_HAS_SPIRV_TARGET
-
-/* Define if the SystemZ target is built in */
-#cmakedefine01 LLVM_HAS_SYSTEMZ_TARGET
-
-/* Define if the VE target is built in */
-#cmakedefine01 LLVM_HAS_VE_TARGET
-
-/* Define if the WebAssembly target is built in */
-#cmakedefine01 LLVM_HAS_WEBASSEMBLY_TARGET
-
-/* Define if the X86 target is built in */
-#cmakedefine01 LLVM_HAS_X86_TARGET
-
-/* Define if the XCore target is built in */
-#cmakedefine01 LLVM_HAS_XCORE_TARGET
-
-/* Define if the Xtensa target is built in */
-#cmakedefine01 LLVM_HAS_XTENSA_TARGET
-
 /* Define if this is Unixish platform */
 #cmakedefine LLVM_ON_UNIX ${LLVM_ON_UNIX}
 
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 4a7cbe4af9e9b..36ec4a2c6faec 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -74,7 +74,10 @@ write_cmake_config("abi-breaking") {
 }
 
 write_cmake_config("config") {
-  public_deps = [ ":llvm-config" ]
+  public_deps = [
+    ":Targets",
+    ":llvm-config",
+  ]
 
   input = "config.h.cmake"
   output = "$target_gen_dir/config.h"
@@ -384,6 +387,12 @@ write_cmake_config("llvm-config") {
   } else {
     values += [ "LLVM_FORCE_ENABLE_STATS=" ]
   }
+}
+
+write_cmake_config("Targets") {
+  input = "Targets.h.cmake"
+  output = "$target_gen_dir/Targets.h"
+  values = []
 
   if (llvm_build_AArch64) {
     values += [ "LLVM_HAS_AARCH64_TARGET=1" ]
diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp
index f8420385cbec4..914a349696617 100644
--- a/mlir/lib/Target/LLVM/NVVM/Target.cpp
+++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp
@@ -27,7 +27,7 @@
 #include "mlir/Target/LLVMIR/Export.h"
 
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Config/llvm-config.h"
+#include "llvm/Config/Targets.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/FormatVariadic.h"
diff --git a/mlir/unittests/Target/LLVM/SerializeNVVMTarget.cpp b/mlir/unittests/Target/LLVM/SerializeNVVMTarget.cpp
index cae713a1ce1d2..af0af89c7d07e 100644
--- a/mlir/unittests/Target/LLVM/SerializeNVVMTarget.cpp
+++ b/mlir/unittests/Target/LLVM/SerializeNVVMTarget.cpp
@@ -19,7 +19,7 @@
 #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h"
 
 #include "llvm/Bitcode/BitcodeWriter.h"
-#include "llvm/Config/llvm-config.h" // for LLVM_HAS_NVPTX_TARGET
+#include "llvm/Config/Targets.h" // for LLVM_HAS_NVPTX_TARGET
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Support/Process.h"

From c64f670255d3fa279f026994da2cd80f3394e0c8 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Fri, 18 Apr 2025 18:11:56 -0700
Subject: [PATCH 506/710] [clang-format] Fix a crash in EnumTrailingComma
 (#135903)

Fix #135819
---
 clang/lib/Format/Format.cpp           | 17 +++++++++++------
 clang/unittests/Format/FormatTest.cpp | 27 +++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index c601967a8715c..0667813110454 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -2427,19 +2427,23 @@ class EnumTrailingCommaEditor : public TokenAnalyzer {
 private:
   void editEnumTrailingComma(SmallVectorImpl<AnnotatedLine *> &Lines,
                              tooling::Replacements &Result) {
+    bool InEnumBraces = false;
+    const FormatToken *BeforeRBrace = nullptr;
     const auto &SourceMgr = Env.getSourceManager();
     for (auto *Line : Lines) {
       if (!Line->Children.empty())
         editEnumTrailingComma(Line->Children, Result);
-      if (!Line->Affected)
-        continue;
       for (const auto *Token = Line->First; Token && !Token->Finalized;
            Token = Token->Next) {
-        if (Token->isNot(TT_EnumRBrace))
+        if (Token->isNot(TT_EnumRBrace)) {
+          if (Token->is(TT_EnumLBrace))
+            InEnumBraces = true;
+          else if (InEnumBraces && Token->isNot(tok::comment))
+            BeforeRBrace = Line->Affected ? Token : nullptr;
           continue;
-        const auto *BeforeRBrace = Token->getPreviousNonComment();
-        assert(BeforeRBrace);
-        if (BeforeRBrace->is(TT_EnumLBrace)) // Empty braces.
+        }
+        InEnumBraces = false;
+        if (!BeforeRBrace) // Empty braces or Line not affected.
           continue;
         if (BeforeRBrace->is(tok::comma)) {
           if (Style.EnumTrailingComma == FormatStyle::ETC_Remove)
@@ -2448,6 +2452,7 @@ class EnumTrailingCommaEditor : public TokenAnalyzer {
           cantFail(Result.add(tooling::Replacement(
               SourceMgr, BeforeRBrace->Tok.getEndLoc(), 0, ",")));
         }
+        BeforeRBrace = nullptr;
       }
     }
   }
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 49284c7f51e27..d5a6206847d68 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -27992,6 +27992,33 @@ TEST_F(FormatTest, EnumTrailingComma) {
                "};\n"
                "enum Color { red, green, blue /**/ };",
                Code, Style);
+
+  EXPECT_TRUE(Style.AllowShortEnumsOnASingleLine);
+  Style.AllowShortEnumsOnASingleLine = false;
+
+  constexpr StringRef Input("enum {\n"
+                            "  //\n"
+                            "  a,\n"
+                            "  /**/\n"
+                            "  b,\n"
+                            "};");
+  verifyFormat(Input, Input, Style, {tooling::Range(12, 3)}); // line 3
+  verifyFormat("enum {\n"
+               "  //\n"
+               "  a,\n"
+               "  /**/\n"
+               "  b\n"
+               "};",
+               Input, Style, {tooling::Range(24, 3)}); // line 5
+
+  Style.EnumTrailingComma = FormatStyle::ETC_Insert;
+  verifyFormat("enum class MyEnum_E {\n"
+               "  MY_ENUM = 0U,\n"
+               "};",
+               "enum class MyEnum_E {\n"
+               "  MY_ENUM = 0U\n"
+               "};",
+               Style);
 }
 
 TEST_F(FormatTest, BreakAfterAttributes) {

From ab4c939579fe6ca85d9a1022511635c4e6f83f55 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Fri, 18 Apr 2025 18:12:27 -0700
Subject: [PATCH 507/710] [clang-format] Fix a bug in annotating
 TT_PointerOrReference (#136073)

Fix #135972
---
 clang/lib/Format/TokenAnnotator.cpp           | 2 +-
 clang/unittests/Format/TokenAnnotatorTest.cpp | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 144983f675828..b4f303e281c1d 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -3026,7 +3026,7 @@ class AnnotatingParser {
 
     if (!NextToken ||
         NextToken->isOneOf(tok::arrow, tok::equal, tok::comma, tok::r_paren,
-                           TT_RequiresClause) ||
+                           TT_RequiresClause, TT_FunctionDeclarationLParen) ||
         (NextToken->is(tok::kw_noexcept) && !IsExpression) ||
         NextToken->canBePointerOrReferenceQualifier() ||
         (NextToken->is(tok::l_brace) && !NextToken->getNextNonComment())) {
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 043ee2e088ddb..a6410f3c8fe53 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -389,6 +389,12 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
   EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_RequiresExpressionLBrace);
   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_CompoundRequirementLBrace);
   EXPECT_TOKEN(Tokens[22], tok::star, TT_BinaryOperator);
+
+  Tokens = annotate("return s.operator int *();");
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_FunctionDeclarationName);
+  EXPECT_TOKEN(Tokens[5], tok::star, TT_PointerOrReference);
+  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {

From 58a5c469d97302bb33a54338cf827a4cbe4e416a Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Fri, 18 Apr 2025 18:13:02 -0700
Subject: [PATCH 508/710] [clang-format] Fix a bug in BWACS_MultiLine (#136281)

Fixes #136266
---
 clang/lib/Format/UnwrappedLineFormatter.cpp |  3 ++-
 clang/unittests/Format/FormatTest.cpp       | 19 +++++++++++++------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 6806ab18312ea..1298e3e7bab38 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -907,7 +907,8 @@ class LineJoiner {
         // { <-- current Line
         //   baz();
         // }
-        if (Line.First == Line.Last && Line.First->isNot(TT_FunctionLBrace) &&
+        if (Line.First == Line.Last &&
+            Line.First->is(TT_ControlStatementLBrace) &&
             Style.BraceWrapping.AfterControlStatement ==
                 FormatStyle::BWACS_MultiLine) {
           return 0;
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index d5a6206847d68..f1b3b7dd8c0c3 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -2866,19 +2866,21 @@ TEST_F(FormatTest, ShortEnums) {
 }
 
 TEST_F(FormatTest, ShortCompoundRequirement) {
+  constexpr StringRef Code("template <typename T>\n"
+                           "concept c = requires(T x) {\n"
+                           "  { x + 1 } -> std::same_as<int>;\n"
+                           "};");
+
   FormatStyle Style = getLLVMStyle();
   EXPECT_TRUE(Style.AllowShortCompoundRequirementOnASingleLine);
-  verifyFormat("template <typename T>\n"
-               "concept c = requires(T x) {\n"
-               "  { x + 1 } -> std::same_as<int>;\n"
-               "};",
-               Style);
+  verifyFormat(Code, Style);
   verifyFormat("template <typename T>\n"
                "concept c = requires(T x) {\n"
                "  { x + 1 } -> std::same_as<int>;\n"
                "  { x + 2 } -> std::same_as<int>;\n"
                "};",
                Style);
+
   Style.AllowShortCompoundRequirementOnASingleLine = false;
   verifyFormat("template <typename T>\n"
                "concept c = requires(T x) {\n"
@@ -2886,7 +2888,7 @@ TEST_F(FormatTest, ShortCompoundRequirement) {
                "    x + 1\n"
                "  } -> std::same_as<int>;\n"
                "};",
-               Style);
+               Code, Style);
   verifyFormat("template <typename T>\n"
                "concept c = requires(T x) {\n"
                "  {\n"
@@ -2897,6 +2899,11 @@ TEST_F(FormatTest, ShortCompoundRequirement) {
                "  } -> std::same_as<int>;\n"
                "};",
                Style);
+
+  Style.AllowShortCompoundRequirementOnASingleLine = true;
+  Style.BreakBeforeBraces = FormatStyle::BS_Custom;
+  Style.BraceWrapping.AfterControlStatement = FormatStyle::BWACS_MultiLine;
+  verifyFormat(Code, Style);
 }
 
 TEST_F(FormatTest, ShortCaseLabels) {

From f9bd89b7ac35920241740969b2b83c45a6a6ddb3 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 18:18:30 -0700
Subject: [PATCH 509/710] MCFixup: Add isRelocation/isRelocRelocation helpers

Add two helper functions to simplify checks for relocation types,
replacing direct comparisons with FirstRelocationKind and
FirstLiteralRelocationKind. Note: Some targets haven't utilized
isRelocation yet.

Also, update RelaxFixupKind to use 0 as the sentinel value.
---
 llvm/include/llvm/MC/MCAsmBackend.h               |  4 ++--
 llvm/include/llvm/MC/MCFixup.h                    | 15 +++++++++++++++
 llvm/lib/MC/ELFObjectWriter.cpp                   |  4 ++--
 llvm/lib/MC/MCAsmStreamer.cpp                     |  2 +-
 .../AArch64/MCTargetDesc/AArch64AsmBackend.cpp    |  2 +-
 .../AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp      |  4 ++--
 .../lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp |  6 +++---
 .../lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp |  4 ++--
 .../Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp   |  2 +-
 .../Target/Mips/MCTargetDesc/MipsAsmBackend.cpp   |  2 +-
 .../Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp |  4 ++--
 .../Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp |  6 +++---
 .../RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp   |  2 +-
 .../Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp |  4 ++--
 .../SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp  |  4 ++--
 .../lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp |  6 +++---
 16 files changed, 43 insertions(+), 28 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index 10eabd41e80fc..61e3b2e59e3ab 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -41,7 +41,7 @@ class raw_ostream;
 /// Generic interface to target specific assembler backends.
 class MCAsmBackend {
 protected: // Can only create subclasses.
-  MCAsmBackend(llvm::endianness Endian, unsigned RelaxFixupKind = MaxFixupKind);
+  MCAsmBackend(llvm::endianness Endian, unsigned RelaxFixupKind = 0);
 
 public:
   MCAsmBackend(const MCAsmBackend &) = delete;
@@ -53,7 +53,7 @@ class MCAsmBackend {
   /// Fixup kind used for linker relaxation. Currently only used by RISC-V
   /// and LoongArch.
   const unsigned RelaxFixupKind;
-  bool allowLinkerRelaxation() const { return RelaxFixupKind != MaxFixupKind; }
+  bool allowLinkerRelaxation() const { return RelaxFixupKind != 0; }
 
   /// Return true if this target might automatically pad instructions and thus
   /// need to emit padding enable/disable directives around sensative code.
diff --git a/llvm/include/llvm/MC/MCFixup.h b/llvm/include/llvm/MC/MCFixup.h
index 2f2717d689049..c23594d1f6df9 100644
--- a/llvm/include/llvm/MC/MCFixup.h
+++ b/llvm/include/llvm/MC/MCFixup.h
@@ -122,6 +122,21 @@ class MCFixup {
   SMLoc getLoc() const { return Loc; }
 };
 
+namespace mc {
+// Check if the fixup kind is a relocation type. Return false if the fixup can
+// be resolved without a relocation.
+inline bool isRelocation(MCFixupKind FixupKind) {
+  return FixupKind >= FirstRelocationKind;
+}
+
+// Check if the fixup kind represents a relocation type from a .reloc directive.
+// In ELF, this skips STT_SECTION adjustment and STT_TLS symbol type setting for
+// TLS relocations.
+inline bool isRelocRelocation(MCFixupKind FixupKind) {
+  return FixupKind >= FirstLiteralRelocationKind;
+}
+} // namespace mc
+
 } // End llvm namespace
 
 #endif
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 2ae9c21271624..5c39daef9c98d 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1385,7 +1385,7 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
 
   auto EMachine = TargetObjectWriter->getEMachine();
   unsigned Type;
-  if (Fixup.getKind() >= FirstLiteralRelocationKind)
+  if (mc::isRelocRelocation(Fixup.getKind()))
     Type = Fixup.getKind() - FirstLiteralRelocationKind;
   else
     Type = TargetObjectWriter->getRelocType(Ctx, Target, Fixup, IsPCRel);
@@ -1396,7 +1396,7 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
     UseSectionSym = useSectionSymbol(Asm, Target, SymA, C, Type);
 
     // Disable STT_SECTION adjustment for .reloc directives.
-    UseSectionSym &= Fixup.getKind() < FirstLiteralRelocationKind;
+    UseSectionSym &= !mc::isRelocRelocation(Fixup.getKind());
   }
 
   uint64_t Addend = UseSectionSym ? C + Asm.getSymbolOffset(*SymA) : C;
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index ff5f13c54b17b..b4beaaf1def1f 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -2403,7 +2403,7 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
        << "offset: " << F.getOffset() << ", value: ";
     F.getValue()->print(OS, MAI);
     auto Kind = F.getKind();
-    if (FirstRelocationKind <= Kind)
+    if (mc::isRelocation(Kind))
       OS << ", relocation type: " << (Kind - FirstRelocationKind);
     else
       OS << ", kind: "
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 9600ba508a110..8a042e789295f 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -69,7 +69,7 @@ class AArch64AsmBackend : public MCAsmBackend {
 
     // Fixup kinds from raw relocation types and .reloc directives force
     // relocations and do not need these fields.
-    if (Kind >= FirstRelocationKind)
+    if (mc::isRelocation(Kind))
       return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
     if (Kind < FirstTargetFixupKind)
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
index 248cd2033aa4e..7873d35855b64 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
@@ -135,7 +135,7 @@ void AMDGPUAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                   MutableArrayRef<char> Data, uint64_t Value,
                                   bool IsResolved,
                                   const MCSubtargetInfo *STI) const {
-  if (Fixup.getKind() >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Fixup.getKind()))
     return;
 
   Value = adjustFixupValue(Fixup, Value, &Asm.getContext());
@@ -179,7 +179,7 @@ const MCFixupKindInfo &AMDGPUAsmBackend::getFixupKindInfo(
     { "fixup_si_sopp_br",     0,     16,   MCFixupKindInfo::FKF_IsPCRel },
   };
 
-  if (Kind >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index 88bcf4cc9c6c6..5b1016e7c5c91 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -193,7 +193,7 @@ const MCFixupKindInfo &ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 
   // Fixup kinds from .reloc directive are like R_ARM_NONE. They do not require
   // any extra processing.
-  if (Kind >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
@@ -1146,8 +1146,8 @@ void ARMAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                MutableArrayRef<char> Data, uint64_t Value,
                                bool IsResolved,
                                const MCSubtargetInfo* STI) const {
-  unsigned Kind = Fixup.getKind();
-  if (Kind >= FirstLiteralRelocationKind)
+  auto Kind = Fixup.getKind();
+  if (mc::isRelocation(Kind))
     return;
   MCContext &Ctx = Asm.getContext();
   Value = adjustFixupValue(Asm, Fixup, Target, Value, IsResolved, Ctx, STI);
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
index 9abe11b39e6d6..6340e589e31f1 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
@@ -373,7 +373,7 @@ void AVRAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                MutableArrayRef<char> Data, uint64_t Value,
                                bool IsResolved,
                                const MCSubtargetInfo *STI) const {
-  if (Fixup.getKind() >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Fixup.getKind()))
     return;
   adjustFixupValue(Fixup, Target, Value, &Asm.getContext());
   if (Value == 0)
@@ -475,7 +475,7 @@ MCFixupKindInfo const &AVRAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 
   // Fixup kinds from .reloc directive are like R_AVR_NONE. They do not require
   // any extra processing.
-  if (Kind >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
index 792a55555a0d0..9000d8db389fd 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
@@ -201,7 +201,7 @@ void CSKYAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                 bool IsResolved,
                                 const MCSubtargetInfo *STI) const {
   MCFixupKind Kind = Fixup.getKind();
-  if (Kind >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Kind))
     return;
   MCContext &Ctx = Asm.getContext();
   MCFixupKindInfo Info = getFixupKindInfo(Kind);
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index 0c295997ab526..8bd2ad8c86a73 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -527,7 +527,7 @@ getFixupKindInfo(MCFixupKind Kind) const {
   static_assert(std::size(BigEndianInfos) == Mips::NumTargetFixupKinds,
                 "Not all MIPS big endian fixup kinds added!");
 
-  if (Kind >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 34a1424f08486..61c274638f347 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -120,7 +120,7 @@ class PPCAsmBackend : public MCAsmBackend {
 
     // Fixup kinds from .reloc directive are like R_PPC_NONE/R_PPC64_NONE. They
     // do not require any extra processing.
-    if (Kind >= FirstLiteralRelocationKind)
+    if (mc::isRelocation(Kind))
       return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
     if (Kind < FirstTargetFixupKind)
@@ -138,7 +138,7 @@ class PPCAsmBackend : public MCAsmBackend {
                   uint64_t Value, bool IsResolved,
                   const MCSubtargetInfo *STI) const override {
     MCFixupKind Kind = Fixup.getKind();
-    if (Kind >= FirstLiteralRelocationKind)
+    if (mc::isRelocation(Kind))
       return;
     Value = adjustFixupValue(Kind, Value);
     if (!Value) return;           // Doesn't change encoding.
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index f9799a062c95e..5ee39c3afa76f 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -94,9 +94,9 @@ RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   static_assert((std::size(Infos)) == RISCV::NumTargetFixupKinds,
                 "Not all fixup kinds added to Infos array");
 
-  // Fixup kinds from raw relocation types and .reloc directive are like
-  // R_RISCV_NONE. They do not require any extra processing.
-  if (Kind >= FirstRelocationKind)
+  // Fixup kinds from raw relocation types and .reloc directives force
+  // relocations and do not use these fields.
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
index 4b32130499f94..427ea935e5241 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
@@ -75,7 +75,7 @@ unsigned RISCVELFObjectWriter::getRelocType(MCContext &Ctx,
 
   // Extract the relocation type from the fixup kind, after applying STT_TLS as
   // needed.
-  if (Kind >= FirstRelocationKind)
+  if (mc::isRelocation(Fixup.getKind()))
     return Kind - FirstRelocationKind;
 
   if (IsPCRel) {
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index e689c534b7058..d8d673b08f741 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -255,7 +255,7 @@ namespace {
 
       // Fixup kinds from .reloc directive are like R_SPARC_NONE. They do
       // not require any extra processing.
-      if (Kind >= FirstLiteralRelocationKind)
+      if (mc::isRelocation(Kind))
         return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
       if (Kind < FirstTargetFixupKind)
@@ -335,7 +335,7 @@ namespace {
                     uint64_t Value, bool IsResolved,
                     const MCSubtargetInfo *STI) const override {
 
-      if (Fixup.getKind() >= FirstLiteralRelocationKind)
+      if (mc::isRelocation(Fixup.getKind()))
         return;
       Value = adjustFixupValue(Fixup.getKind(), Value);
       if (!Value) return;           // Doesn't change encoding.
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index 4f488d7a3bd2e..22ba533fa5845 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -142,7 +142,7 @@ const MCFixupKindInfo &
 SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from .reloc directive are like R_390_NONE. They
   // do not require any extra processing.
-  if (Kind >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
@@ -160,7 +160,7 @@ void SystemZMCAsmBackend::applyFixup(const MCAssembler &Asm,
                                      bool IsResolved,
                                      const MCSubtargetInfo *STI) const {
   MCFixupKind Kind = Fixup.getKind();
-  if (Kind >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Kind))
     return;
   unsigned Offset = Fixup.getOffset();
   unsigned BitSize = getFixupKindInfo(Kind).TargetSize;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index cb23487e6fbe0..6e02c2e4c5433 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -639,7 +639,7 @@ const MCFixupKindInfo &X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 
   // Fixup kinds from .reloc directive are like R_386_NONE/R_X86_64_NONE. They
   // do not require any extra processing.
-  if (Kind >= FirstLiteralRelocationKind)
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
@@ -691,8 +691,8 @@ void X86AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                const MCValue &, MutableArrayRef<char> Data,
                                uint64_t Value, bool IsResolved,
                                const MCSubtargetInfo *STI) const {
-  unsigned Kind = Fixup.getKind();
-  if (Kind >= FirstLiteralRelocationKind)
+  auto Kind = Fixup.getKind();
+  if (mc::isRelocation(Kind))
     return;
   unsigned Size = getFixupKindSize(Kind);
 

From b9e11eade7f1c1e2464f9f891d3769d8e48c6509 Mon Sep 17 00:00:00 2001
From: YLChenZ <chentongyongcz@gmail.com>
Date: Sat, 19 Apr 2025 09:32:40 +0800
Subject: [PATCH 510/710] [llvm][ir]: fix llc crashes on function definitions
 with label parameters (#136285)

Closes #136144.

After the patch:
```llvm
; label-crash.ll
define void @invalid_arg_type(i32 %0) {
1:
  call void @foo(label %1)
  ret void
}

declare void @foo(label)
```
```
lambda@ubuntu22:~/test$ llc -o out.s label-crash.ll
Function argument cannot be of label type!
label %0
ptr @foo
llc: error: 'label-crash.ll': input module cannot be verified
```
---
 llvm/lib/IR/Verifier.cpp                  |  2 ++
 llvm/test/Verifier/invalid-label-param.ll | 14 ++++++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 llvm/test/Verifier/invalid-label-param.ll

diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 35c4d60cf325e..8afe360d088bc 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2931,6 +2931,8 @@ void Verifier::visitFunction(const Function &F) {
           FT->getParamType(i));
     Check(Arg.getType()->isFirstClassType(),
           "Function arguments must have first-class types!", &Arg);
+    Check(!Arg.getType()->isLabelTy(),
+          "Function argument cannot be of label type!", &Arg, &F);
     if (!IsIntrinsic) {
       Check(!Arg.getType()->isMetadataTy(),
             "Function takes metadata but isn't an intrinsic", &Arg, &F);
diff --git a/llvm/test/Verifier/invalid-label-param.ll b/llvm/test/Verifier/invalid-label-param.ll
new file mode 100644
index 0000000000000..c89ce99ce1e9f
--- /dev/null
+++ b/llvm/test/Verifier/invalid-label-param.ll
@@ -0,0 +1,14 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+define void @invalid_arg_type(i32 %0) {
+1:
+  call void @foo(label %1)
+  ret void
+}
+
+declare void @foo(label)
+; CHECK: Function argument cannot be of label type!
+; CHECK-NEXT: label %0
+; CHECK-NEXT: ptr @foo
+
+

From 7962820d4dc23852456e1891f785225c4fbc7b36 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 18:38:53 -0700
Subject: [PATCH 511/710] MC: Simplify code with isRelocation

---
 llvm/lib/MC/MCAssembler.cpp                         |  2 +-
 .../lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp | 13 ++++---------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 1b3e7dd4f4ceb..b1fd253d1cf60 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -203,7 +203,7 @@ bool MCAssembler::evaluateFixup(const MCFixup &Fixup, const MCFragment *DF,
   if (IsResolved) {
     auto TargetVal = Target;
     TargetVal.Cst = Value;
-    if (Fixup.getKind() >= FirstRelocationKind ||
+    if (mc::isRelocation(Fixup.getKind()) ||
         getBackend().shouldForceRelocation(*this, Fixup, TargetVal, STI))
       IsResolved = false;
   }
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
index 9000d8db389fd..1c305ac8bebfd 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
@@ -71,16 +71,11 @@ CSKYAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   assert(Infos.size() == CSKY::NumTargetFixupKinds &&
          "Not all fixup kinds added to Infos array");
 
-  if (FirstTargetFixupKind <= Kind && Kind < FirstLiteralRelocationKind) {
-    assert(unsigned(Kind - FirstTargetFixupKind) < CSKY::NumTargetFixupKinds &&
-           "Invalid kind!");
-
-    return Infos[Kind];
-  } else if (Kind < FirstTargetFixupKind) {
-    return MCAsmBackend::getFixupKindInfo(Kind);
-  } else {
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
-  }
+  if (Kind < FirstTargetFixupKind)
+    return MCAsmBackend::getFixupKindInfo(Kind);
+  return Infos[Kind];
 }
 
 static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,

From 06de4d52d5eff6ee083e901d24362a77e102fb16 Mon Sep 17 00:00:00 2001
From: "A. Jiang" <de34@live.cn>
Date: Sat, 19 Apr 2025 10:01:48 +0800
Subject: [PATCH 512/710] [libc++] Properly decay functions in CTAD for `pair`
 (#134544)

This patch makes instantiation of `pair` in CTAD a bit lazier to avoid
instantiating invalid `pair` specialization before the decaying explicit
deduction guide works.
---
 libcxx/include/__utility/pair.h               | 104 ++++++++++--------
 .../explicit_deduction_guides.pass.cpp        |  60 ++++++++++
 2 files changed, 119 insertions(+), 45 deletions(-)
 create mode 100644 libcxx/test/std/utilities/utility/pairs/pairs.pair/explicit_deduction_guides.pass.cpp

diff --git a/libcxx/include/__utility/pair.h b/libcxx/include/__utility/pair.h
index d921a779db7f4..9beff24b149bb 100644
--- a/libcxx/include/__utility/pair.h
+++ b/libcxx/include/__utility/pair.h
@@ -49,6 +49,33 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _T1, class _T2>
+struct __check_pair_construction {
+  template <int&...>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_implicit_default() {
+    return __is_implicitly_default_constructible<_T1>::value && __is_implicitly_default_constructible<_T2>::value;
+  }
+
+  template <int&...>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_default() {
+    return is_default_constructible<_T1>::value && is_default_constructible<_T2>::value;
+  }
+
+  template <class _U1, class _U2>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __is_pair_constructible() {
+    return is_constructible<_T1, _U1>::value && is_constructible<_T2, _U2>::value;
+  }
+
+  template <class _U1, class _U2>
+  static _LIBCPP_HIDE_FROM_ABI constexpr bool __is_implicit() {
+    return is_convertible<_U1, _T1>::value && is_convertible<_U2, _T2>::value;
+  }
+};
+
+#endif
+
 template <class, class>
 struct __non_trivially_copyable_base {
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __non_trivially_copyable_base() _NOEXCEPT {}
@@ -104,40 +131,16 @@ struct pair
     return *this;
   }
 #else
-  struct _CheckArgs {
-    template <int&...>
-    static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_implicit_default() {
-      return __is_implicitly_default_constructible<_T1>::value && __is_implicitly_default_constructible<_T2>::value;
-    }
-
-    template <int&...>
-    static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_default() {
-      return is_default_constructible<_T1>::value && is_default_constructible<_T2>::value;
-    }
-
-    template <class _U1, class _U2>
-    static _LIBCPP_HIDE_FROM_ABI constexpr bool __is_pair_constructible() {
-      return is_constructible<first_type, _U1>::value && is_constructible<second_type, _U2>::value;
-    }
-
-    template <class _U1, class _U2>
-    static _LIBCPP_HIDE_FROM_ABI constexpr bool __is_implicit() {
-      return is_convertible<_U1, first_type>::value && is_convertible<_U2, second_type>::value;
-    }
-  };
-
-  template <bool _MaybeEnable>
-  using _CheckArgsDep _LIBCPP_NODEBUG = __conditional_t<_MaybeEnable, _CheckArgs, void>;
-
-  template <bool _Dummy = true, __enable_if_t<_CheckArgsDep<_Dummy>::__enable_default(), int> = 0>
-  explicit(!_CheckArgsDep<_Dummy>::__enable_implicit_default()) _LIBCPP_HIDE_FROM_ABI constexpr pair() noexcept(
+  template <class _CheckArgsDep                                   = __check_pair_construction<_T1, _T2>,
+            __enable_if_t<_CheckArgsDep::__enable_default(), int> = 0>
+  explicit(!_CheckArgsDep::__enable_implicit_default()) _LIBCPP_HIDE_FROM_ABI constexpr pair() noexcept(
       is_nothrow_default_constructible<first_type>::value && is_nothrow_default_constructible<second_type>::value)
       : first(), second() {}
 
-  template <bool _Dummy = true,
-            __enable_if_t<_CheckArgsDep<_Dummy>::template __is_pair_constructible<_T1 const&, _T2 const&>(), int> = 0>
+  template <class _CheckArgsDep = __check_pair_construction<_T1, _T2>,
+            __enable_if_t<_CheckArgsDep::template __is_pair_constructible<_T1 const&, _T2 const&>(), int> = 0>
   _LIBCPP_HIDE_FROM_ABI
-  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgsDep<_Dummy>::template __is_implicit<_T1 const&, _T2 const&>())
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgsDep::template __is_implicit<_T1 const&, _T2 const&>())
       pair(_T1 const& __t1, _T2 const& __t2) noexcept(is_nothrow_copy_constructible<first_type>::value &&
                                                       is_nothrow_copy_constructible<second_type>::value)
       : first(__t1), second(__t2) {}
@@ -150,41 +153,52 @@ struct pair
       class _U1,
       class _U2,
 #  endif
-      __enable_if_t<_CheckArgs::template __is_pair_constructible<_U1, _U2>(), int> = 0 >
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgs::template __is_implicit<_U1, _U2>())
+      __enable_if_t<__check_pair_construction<_T1, _T2>::template __is_pair_constructible<_U1, _U2>(), int> = 0 >
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!__check_pair_construction<_T1, _T2>::template __is_implicit<_U1, _U2>())
       pair(_U1&& __u1, _U2&& __u2) noexcept(is_nothrow_constructible<first_type, _U1>::value &&
                                             is_nothrow_constructible<second_type, _U2>::value)
       : first(std::forward<_U1>(__u1)), second(std::forward<_U2>(__u2)) {
   }
 
 #  if _LIBCPP_STD_VER >= 23
-  template <class _U1, class _U2, __enable_if_t<_CheckArgs::template __is_pair_constructible<_U1&, _U2&>(), int> = 0>
-  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_CheckArgs::template __is_implicit<_U1&, _U2&>())
+  template <class _U1,
+            class _U2,
+            __enable_if_t<__check_pair_construction<_T1, _T2>::template __is_pair_constructible<_U1&, _U2&>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!__check_pair_construction<_T1, _T2>::template __is_implicit<_U1&, _U2&>())
       pair(pair<_U1, _U2>& __p) noexcept((is_nothrow_constructible<first_type, _U1&>::value &&
                                           is_nothrow_constructible<second_type, _U2&>::value))
       : first(__p.first), second(__p.second) {}
 #  endif
 
-  template <class _U1,
-            class _U2,
-            __enable_if_t<_CheckArgs::template __is_pair_constructible<_U1 const&, _U2 const&>(), int> = 0>
-  _LIBCPP_HIDE_FROM_ABI
-  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgs::template __is_implicit<_U1 const&, _U2 const&>())
+  template <
+      class _U1,
+      class _U2,
+      __enable_if_t<__check_pair_construction<_T1, _T2>::template __is_pair_constructible<_U1 const&, _U2 const&>(),
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(
+      !__check_pair_construction<_T1, _T2>::template __is_implicit<_U1 const&, _U2 const&>())
       pair(pair<_U1, _U2> const& __p) noexcept(is_nothrow_constructible<first_type, _U1 const&>::value &&
                                                is_nothrow_constructible<second_type, _U2 const&>::value)
       : first(__p.first), second(__p.second) {}
 
-  template <class _U1, class _U2, __enable_if_t<_CheckArgs::template __is_pair_constructible<_U1, _U2>(), int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!_CheckArgs::template __is_implicit<_U1, _U2>())
+  template <class _U1,
+            class _U2,
+            __enable_if_t<__check_pair_construction<_T1, _T2>::template __is_pair_constructible<_U1, _U2>(), int> = 0>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(!__check_pair_construction<_T1, _T2>::template __is_implicit<_U1, _U2>())
       pair(pair<_U1, _U2>&& __p) noexcept(is_nothrow_constructible<first_type, _U1&&>::value &&
                                           is_nothrow_constructible<second_type, _U2&&>::value)
       : first(std::forward<_U1>(__p.first)), second(std::forward<_U2>(__p.second)) {}
 
 #  if _LIBCPP_STD_VER >= 23
-  template <class _U1,
-            class _U2,
-            __enable_if_t<_CheckArgs::template __is_pair_constructible<const _U1&&, const _U2&&>(), int> = 0>
-  _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_CheckArgs::template __is_implicit<const _U1&&, const _U2&&>())
+  template <
+      class _U1,
+      class _U2,
+      __enable_if_t<__check_pair_construction<_T1, _T2>::template __is_pair_constructible<const _U1&&, const _U2&&>(),
+                    int> = 0>
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(
+      !__check_pair_construction<_T1, _T2>::template __is_implicit<const _U1&&, const _U2&&>())
       pair(const pair<_U1, _U2>&& __p) noexcept(is_nothrow_constructible<first_type, const _U1&&>::value &&
                                                 is_nothrow_constructible<second_type, const _U2&&>::value)
       : first(std::move(__p.first)), second(std::move(__p.second)) {}
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/explicit_deduction_guides.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/explicit_deduction_guides.pass.cpp
new file mode 100644
index 0000000000000..d580e04012706
--- /dev/null
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/explicit_deduction_guides.pass.cpp
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++17
+
+// <utility>
+
+// template<class T1, class T2>
+//   pair(T1, T2) -> pair<T1, T2>;
+
+// Test that the explicit deduction guide for std::pair correctly decays function lvalues and
+// behaves different from std::make_pair.
+
+#include <cassert>
+#include <functional>
+#include <type_traits>
+#include <utility>
+
+#include "test_macros.h"
+
+void dummy() {}
+
+constexpr void test_decay() {
+  char arr[1]{};
+  std::pair pr(arr, dummy);
+
+  ASSERT_SAME_TYPE(decltype(pr), std::pair<char*, void (*)()>);
+
+  assert(pr == std::make_pair(arr, dummy));
+}
+
+TEST_CONSTEXPR_CXX20 void test_unwrap() {
+  int n = 0;
+  std::pair pr(std::ref(n), dummy);
+
+  ASSERT_SAME_TYPE(decltype(pr), std::pair<std::reference_wrapper<int>, void (*)()>);
+  static_assert(!std::is_same_v<decltype(pr), decltype(std::make_pair(std::ref(n), dummy))>);
+
+  assert(&(pr.first.get()) == &n);
+  assert(pr.second == dummy);
+}
+
+constexpr bool test() {
+  test_decay();
+  if (TEST_STD_AT_LEAST_20_OR_RUNTIME_EVALUATED)
+    test_unwrap();
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}

From 7b0409ad0e5a7e06fd36a3565b7c24f9586344e6 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 19:09:16 -0700
Subject: [PATCH 513/710] MC: Simplify code with isRelocation

---
 llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp | 7 ++++---
 .../Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp | 2 +-
 .../LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp    | 2 +-
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp     | 2 +-
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp         | 2 +-
 5 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 8a042e789295f..1517fb22a19f5 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -421,6 +421,10 @@ void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                    MutableArrayRef<char> Data, uint64_t Value,
                                    bool IsResolved,
                                    const MCSubtargetInfo *STI) const {
+  MCFixupKind Kind = Fixup.getKind();
+  if (mc::isRelocation(Kind))
+    return;
+
   if (Fixup.getTargetKind() == FK_Data_8 && TheTriple.isOSBinFormatELF()) {
     auto RefKind = static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
     AArch64MCExpr::Specifier SymLoc = AArch64MCExpr::getSymbolLoc(RefKind);
@@ -441,9 +445,6 @@ void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
 
   if (!Value)
     return; // Doesn't change encoding.
-  unsigned Kind = Fixup.getKind();
-  if (Kind >= FirstRelocationKind)
-    return;
   unsigned NumBytes = getFixupKindNumBytes(Kind);
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
   MCContext &Ctx = Asm.getContext();
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index 4457cff30101e..8e0567ba7307d 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -129,7 +129,7 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
 
   // Extract the relocation type from the fixup kind, after applying STT_TLS as
   // needed.
-  if (Kind >= FirstRelocationKind)
+  if (mc::isRelocation(Fixup.getKind()))
     return Kind - FirstRelocationKind;
 
   if (IsPCRel) {
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
index d129e0e2497e4..923e35d1cdf1b 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
@@ -72,7 +72,7 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx,
   }
 
   unsigned Kind = Fixup.getTargetKind();
-  if (Kind >= FirstRelocationKind)
+  if (mc::isRelocation(Fixup.getKind()))
     return Kind - FirstRelocationKind;
   switch (Kind) {
   default:
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index 5ee39c3afa76f..acdad71757573 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -676,7 +676,7 @@ void RISCVAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                  bool IsResolved,
                                  const MCSubtargetInfo *STI) const {
   MCFixupKind Kind = Fixup.getKind();
-  if (Kind >= FirstRelocationKind)
+  if (mc::isRelocation(Kind))
     return;
   MCContext &Ctx = Asm.getContext();
   MCFixupKindInfo Info = getFixupKindInfo(Kind);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
index 7c953f55a409a..af318b4d3ea28 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
@@ -68,7 +68,7 @@ const MCFixup *RISCVMCExpr::getPCRelHiFixup(const MCFragment **DFOut) const {
     if (F.getOffset() != Offset)
       continue;
     auto Kind = F.getTargetKind();
-    if (Kind < FirstRelocationKind) {
+    if (!mc::isRelocation(F.getKind())) {
       if (Kind == RISCV::fixup_riscv_pcrel_hi20) {
         *DFOut = DF;
         return &F;

From 6311b11595f152a8f4aa02cfa544e72ede66688a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= <alex@alexrp.com>
Date: Sat, 19 Apr 2025 04:09:58 +0200
Subject: [PATCH 514/710] [clang][Driver] Support passing arbitrary args to
 `-cc1as` with `-Xclangas`. (#100714)

Unlike the `-Xassembler` idea mentioned in #97517, this one comes with
no GCC compatibility concerns, and simply mirrors what `-Xclang` does
for `-cc1`.

This is useful for mostly the same reasons that `-Xclang` is. The
motivating use case is `zig cc`, however, where we use `-Xclang
-target-feature` to pass the exhaustive list of target features to Clang
for C-family files. Before this commit, there was no way to do the same
for assembly files.

For context, Zig bases all of its target feature info directly on the
info in LLVM's backends, including all the dependency relationships
therein. So it just makes more sense for Zig to be able to directly pass
all this info to the assembler, rather than having to manually
reconstruct the corresponding frontend command line flags for every
target.

Closes #97517.
---
 clang/docs/UsersManual.rst            | 1 +
 clang/include/clang/Driver/Options.td | 8 ++++++++
 clang/lib/Driver/ToolChains/Clang.cpp | 6 ++++++
 clang/test/Driver/Xclangas.s          | 4 ++++
 4 files changed, 19 insertions(+)
 create mode 100644 clang/test/Driver/Xclangas.s

diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 69256527f40c9..6c2053e01da8b 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -5146,6 +5146,7 @@ Execute ``clang-cl /?`` to see a list of supported options:
       -v                      Show commands to run and use verbose output
       -W<warning>             Enable the specified warning
       -Xclang <arg>           Pass <arg> to the clang compiler
+      -Xclangas <arg>         Pass <arg> to the clang assembler
 
 The /clang: Option
 ^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 830d3459a1320..919c1c643d080 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -958,10 +958,18 @@ def Xclang : Separate<["-"], "Xclang">,
   HelpText<"Pass <arg> to clang -cc1">, MetaVarName<"<arg>">,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CLOption, DXCOption]>,
   Group<CompileOnly_Group>;
+def Xclangas : Separate<["-"], "Xclangas">,
+  HelpText<"Pass <arg> to clang -cc1as">, MetaVarName<"<arg>">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CLOption, DXCOption]>,
+  Group<CompileOnly_Group>;
 def : Joined<["-"], "Xclang=">, Group<CompileOnly_Group>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CLOption, DXCOption]>,
   Alias<Xclang>,
   HelpText<"Alias for -Xclang">, MetaVarName<"<arg>">;
+def : Joined<["-"], "Xclangas=">, Group<CompileOnly_Group>,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CLOption, DXCOption]>,
+  Alias<Xclangas>,
+  HelpText<"Alias for -Xclangas">, MetaVarName<"<arg>">;
 def Xcuda_fatbinary : Separate<["-"], "Xcuda-fatbinary">,
   HelpText<"Pass <arg> to fatbinary invocation">, MetaVarName<"<arg>">;
 def Xcuda_ptxas : Separate<["-"], "Xcuda-ptxas">,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 67a800a643cbe..f2f5231933c88 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -8895,6 +8895,12 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
   CollectArgsForIntegratedAssembler(C, Args, CmdArgs,
                                     getToolChain().getDriver());
 
+  // Forward -Xclangas arguments to -cc1as
+  for (auto Arg : Args.filtered(options::OPT_Xclangas)) {
+    Arg->claim();
+    CmdArgs.push_back(Arg->getValue());
+  }
+
   Args.AddAllArgs(CmdArgs, options::OPT_mllvm);
 
   if (DebugInfoKind > llvm::codegenoptions::NoDebugInfo && Output.isFilename())
diff --git a/clang/test/Driver/Xclangas.s b/clang/test/Driver/Xclangas.s
new file mode 100644
index 0000000000000..0d3911170eee1
--- /dev/null
+++ b/clang/test/Driver/Xclangas.s
@@ -0,0 +1,4 @@
+/// Check that -Xclangas/-Xclangas= are passed to -cc1as.
+// RUN: %clang -### -Werror -Xclangas -target-feature -Xclangas=+v5t %s 2>&1 | FileCheck %s
+// CHECK: -cc1as
+// CHECK-SAME: "-target-feature" "+v5t"

From f336c908cdae0135d05763e4bb05913d27076fc8 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 19:37:52 -0700
Subject: [PATCH 515/710] LoongArch: Use mc::isRelocation. NFC

---
 .../Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index 78a54a9385b08..75cb0b587e0a6 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -70,7 +70,7 @@ LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 
   // Fixup kinds from .reloc directive are like R_LARCH_NONE. They
   // do not require any extra processing.
-  if (unsigned(Kind) >= FirstRelocationKind)
+  if (mc::isRelocation(Kind))
     return MCAsmBackend::getFixupKindInfo(FK_NONE);
 
   if (Kind < FirstTargetFixupKind)
@@ -152,10 +152,10 @@ void LoongArchAsmBackend::applyFixup(const MCAssembler &Asm,
   if (!Value)
     return; // Doesn't change encoding.
 
-  auto Kind = Fixup.getTargetKind();
-  if (Kind >= FirstRelocationKind)
+  auto Kind = Fixup.getKind();
+  if (mc::isRelocation(Kind))
     return;
-  MCFixupKindInfo Info = getFixupKindInfo(MCFixupKind(Kind));
+  MCFixupKindInfo Info = getFixupKindInfo(Kind);
   MCContext &Ctx = Asm.getContext();
 
   // Fixup leb128 separately.

From 0c0b5abff7ebedde53ccb84a04dadb00b9b91f2b Mon Sep 17 00:00:00 2001
From: Selim Keles <mehmetselimkeles@outlook.com>
Date: Sat, 19 Apr 2025 05:42:22 +0300
Subject: [PATCH 516/710] [clang-format] Handle spaces in filename paths in
 clang-format-diff.py (#135779)

This PR resolves an issue in clang-format-diff.py where
filenames containing spaces were not correctly extracted from Git diffs.
Due to the previous regex implementation, filenames were being
truncated, causing the script to fail when processing diffs with such
filenames.

Fixes #135619.
---
 clang/tools/clang-format/clang-format-diff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/tools/clang-format/clang-format-diff.py b/clang/tools/clang-format/clang-format-diff.py
index c82b41e8bd031..3059982ba231f 100755
--- a/clang/tools/clang-format/clang-format-diff.py
+++ b/clang/tools/clang-format/clang-format-diff.py
@@ -102,7 +102,7 @@ def main():
     filename = None
     lines_by_file = {}
     for line in sys.stdin:
-        match = re.search(r"^\+\+\+\ (.*?/){%s}(\S*)" % args.p, line)
+        match = re.search(r"^\+\+\+\ (.*?/){%s}(.+)" % args.p, line.rstrip())
         if match:
             filename = match.group(2)
         if filename is None:

From 40789ce7f1b7cff6de82b7f93db25a8c54194d46 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 19:51:56 -0700
Subject: [PATCH 517/710] MCFixup: Move relocation values before FK_NONE

Simplify the process of encoding a raw relocation type using MCFixupKind.

Currently, FirstRelocationkind is utilized by AArch64, LoongArch, and
RISCV.
---
 llvm/include/llvm/MC/MCFixup.h                | 34 ++++++++----------
 llvm/lib/MC/MCAsmBackend.cpp                  |  4 +--
 llvm/lib/MC/MCAsmStreamer.cpp                 |  2 +-
 .../MCTargetDesc/AArch64ELFObjectWriter.cpp   |  2 +-
 .../MCTargetDesc/AArch64MCCodeEmitter.cpp     |  3 +-
 .../MCTargetDesc/LoongArchAsmBackend.cpp      | 28 +++++++--------
 .../MCTargetDesc/LoongArchELFObjectWriter.cpp |  2 +-
 .../MCTargetDesc/LoongArchFixupKinds.h        | 14 ++++----
 .../RISCV/MCTargetDesc/RISCVAsmBackend.cpp    | 14 +++-----
 .../MCTargetDesc/RISCVELFObjectWriter.cpp     |  2 +-
 .../RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp | 36 +++++++++----------
 .../Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp |  2 +-
 12 files changed, 63 insertions(+), 80 deletions(-)

diff --git a/llvm/include/llvm/MC/MCFixup.h b/llvm/include/llvm/MC/MCFixup.h
index c23594d1f6df9..f6e4051dbbd22 100644
--- a/llvm/include/llvm/MC/MCFixup.h
+++ b/llvm/include/llvm/MC/MCFixup.h
@@ -18,8 +18,17 @@ namespace llvm {
 class MCExpr;
 
 /// Extensible enumeration to represent the type of a fixup.
-enum MCFixupKind {
-  FK_NONE = 0,    ///< A no-op fixup.
+enum MCFixupKind : uint16_t {
+  // [0, FirstLiteralRelocationKind) encodes raw relocation types.
+
+  // [FirstLiteralRelocationKind, FK_NONE) encodes raw relocation types coming
+  // from .reloc directives. Fixup kind
+  // FirstLiteralRelocationKind+t encodes relocation type t.
+  FirstLiteralRelocationKind = 2000,
+
+  // Other kinds indicate the fixup may resolve to a constant, allowing the
+  // assembler to update the instruction or data directly without a relocation.
+  FK_NONE = 4000, ///< A no-op fixup.
   FK_Data_1,      ///< A one-byte fixup.
   FK_Data_2,      ///< A two-byte fixup.
   FK_Data_4,      ///< A four-byte fixup.
@@ -34,19 +43,7 @@ enum MCFixupKind {
   FK_SecRel_4,    ///< A four-byte section relative fixup.
   FK_SecRel_8,    ///< A eight-byte section relative fixup.
 
-  FirstTargetFixupKind = 128,
-
-  /// Targets can use FirstRelocationKind+t to encode relocation type t.
-  FirstRelocationKind = 256,
-
-  /// The range [FirstLiteralRelocationKind, MaxTargetFixupKind) is used for
-  /// relocations coming from .reloc directive. Fixup kind
-  /// FirstLiteralRelocationKind+V represents the relocation type with number V.
-  FirstLiteralRelocationKind = 256 + 1032 + 32,
-
-  /// Set limit to accommodate the highest reloc type in use for all Targets,
-  /// currently R_AARCH64_IRELATIVE at 1032, including room for expansion.
-  MaxFixupKind = FirstLiteralRelocationKind + 1032 + 32,
+  FirstTargetFixupKind,
 };
 
 /// Encode information on a single operation to perform on a byte
@@ -81,7 +78,6 @@ class MCFixup {
 public:
   static MCFixup create(uint32_t Offset, const MCExpr *Value,
                         MCFixupKind Kind, SMLoc Loc = SMLoc()) {
-    assert(Kind <= MaxFixupKind && "Kind out of range!");
     MCFixup FI;
     FI.Value = Value;
     FI.Offset = Offset;
@@ -125,15 +121,13 @@ class MCFixup {
 namespace mc {
 // Check if the fixup kind is a relocation type. Return false if the fixup can
 // be resolved without a relocation.
-inline bool isRelocation(MCFixupKind FixupKind) {
-  return FixupKind >= FirstRelocationKind;
-}
+inline bool isRelocation(MCFixupKind FixupKind) { return FixupKind < FK_NONE; }
 
 // Check if the fixup kind represents a relocation type from a .reloc directive.
 // In ELF, this skips STT_SECTION adjustment and STT_TLS symbol type setting for
 // TLS relocations.
 inline bool isRelocRelocation(MCFixupKind FixupKind) {
-  return FixupKind >= FirstLiteralRelocationKind;
+  return FirstLiteralRelocationKind <= FixupKind && FixupKind < FK_NONE;
 }
 } // namespace mc
 
diff --git a/llvm/lib/MC/MCAsmBackend.cpp b/llvm/lib/MC/MCAsmBackend.cpp
index 85a208ffbce0a..03a39bd6a4cff 100644
--- a/llvm/lib/MC/MCAsmBackend.cpp
+++ b/llvm/lib/MC/MCAsmBackend.cpp
@@ -105,8 +105,8 @@ const MCFixupKindInfo &MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"FK_SecRel_8", 0, 64, 0},
   };
 
-  assert((size_t)Kind <= std::size(Builtins) && "Unknown fixup kind");
-  return Builtins[Kind];
+  assert(size_t(Kind - FK_NONE) < std::size(Builtins) && "Unknown fixup kind");
+  return Builtins[Kind - FK_NONE];
 }
 
 bool MCAsmBackend::shouldForceRelocation(const MCAssembler &, const MCFixup &,
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index b4beaaf1def1f..361f9492d60b4 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -2404,7 +2404,7 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
     F.getValue()->print(OS, MAI);
     auto Kind = F.getKind();
     if (mc::isRelocation(Kind))
-      OS << ", relocation type: " << (Kind - FirstRelocationKind);
+      OS << ", relocation type: " << Kind;
     else
       OS << ", kind: "
          << getAssembler().getBackend().getFixupKindInfo(Kind).Name;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index 8e0567ba7307d..53acf7dd61054 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -130,7 +130,7 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
   // Extract the relocation type from the fixup kind, after applying STT_TLS as
   // needed.
   if (mc::isRelocation(Fixup.getKind()))
-    return Kind - FirstRelocationKind;
+    return Kind;
 
   if (IsPCRel) {
     switch (Kind) {
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
index 9f9b6aba3fe6c..4dc30b48c902f 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
@@ -749,8 +749,7 @@ void AArch64MCCodeEmitter::encodeInstruction(const MCInst &MI,
     auto Reloc = STI.getTargetTriple().getEnvironment() == Triple::GNUILP32
                      ? ELF::R_AARCH64_P32_TLSDESC_CALL
                      : ELF::R_AARCH64_TLSDESC_CALL;
-    Fixups.push_back(MCFixup::create(0, MI.getOperand(0).getExpr(),
-                                     FirstRelocationKind + Reloc));
+    Fixups.push_back(MCFixup::create(0, MI.getOperand(0).getExpr(), Reloc));
     return;
   }
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index 75cb0b587e0a6..ac2127c96019c 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -271,27 +271,23 @@ getRelocPairForSize(unsigned Size) {
   default:
     llvm_unreachable("unsupported fixup size");
   case 6:
-    return std::make_pair(MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD6),
-                          MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB6));
+    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD6),
+                          MCFixupKind(ELF::R_LARCH_SUB6));
   case 8:
-    return std::make_pair(MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD8),
-                          MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB8));
+    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD8),
+                          MCFixupKind(ELF::R_LARCH_SUB8));
   case 16:
-    return std::make_pair(
-        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD16),
-        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB16));
+    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD16),
+                          MCFixupKind(ELF::R_LARCH_SUB16));
   case 32:
-    return std::make_pair(
-        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD32),
-        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB32));
+    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD32),
+                          MCFixupKind(ELF::R_LARCH_SUB32));
   case 64:
-    return std::make_pair(
-        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD64),
-        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB64));
+    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD64),
+                          MCFixupKind(ELF::R_LARCH_SUB64));
   case 128:
-    return std::make_pair(
-        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_ADD_ULEB128),
-        MCFixupKind(FirstRelocationKind + ELF::R_LARCH_SUB_ULEB128));
+    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD_ULEB128),
+                          MCFixupKind(ELF::R_LARCH_SUB_ULEB128));
   }
 }
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
index 923e35d1cdf1b..0c0caead731b5 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
@@ -73,7 +73,7 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx,
 
   unsigned Kind = Fixup.getTargetKind();
   if (mc::isRelocation(Fixup.getKind()))
-    return Kind - FirstRelocationKind;
+    return Kind;
   switch (Kind) {
   default:
     Ctx.reportError(Fixup.getLoc(), "Unsupported relocation type");
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
index aac295c35db69..4bd64be7e94f9 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
@@ -55,7 +55,7 @@ enum Fixups {
   // Define fixups for force relocation as FirstRelocationKind+V
   // represents the relocation type with number V.
   // 20-bit fixup corresponding to %pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_pcala_hi20 = FirstRelocationKind + ELF::R_LARCH_PCALA_HI20,
+  fixup_loongarch_pcala_hi20 = ELF::R_LARCH_PCALA_HI20,
   // 12-bit fixup corresponding to %pc_lo12(foo) for instructions like addi.w/d.
   fixup_loongarch_pcala_lo12,
   // 20-bit fixup corresponding to %pc64_lo20(foo) for instruction lu32i.d.
@@ -81,8 +81,7 @@ enum Fixups {
   fixup_loongarch_got64_hi12,
   // Skip R_LARCH_TLS_LE_*.
   // 20-bit fixup corresponding to %ie_pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_tls_ie_pc_hi20 =
-      FirstRelocationKind + ELF::R_LARCH_TLS_IE_PC_HI20,
+  fixup_loongarch_tls_ie_pc_hi20 = ELF::R_LARCH_TLS_IE_PC_HI20,
   // 12-bit fixup corresponding to %ie_pc_lo12(foo) for instructions
   // ld.w/ld.d/add.d.
   fixup_loongarch_tls_ie_pc_lo12,
@@ -107,17 +106,16 @@ enum Fixups {
   // 20-bit fixup corresponding to %gd_hi20(foo) for instruction lu12i.w.
   fixup_loongarch_tls_gd_hi20,
   // Generate an R_LARCH_RELAX which indicates the linker may relax here.
-  fixup_loongarch_relax = FirstRelocationKind + ELF::R_LARCH_RELAX,
+  fixup_loongarch_relax = ELF::R_LARCH_RELAX,
   // Generate an R_LARCH_ALIGN which indicates the linker may fixup align here.
-  fixup_loongarch_align = FirstRelocationKind + ELF::R_LARCH_ALIGN,
+  fixup_loongarch_align = ELF::R_LARCH_ALIGN,
   // 20-bit fixup corresponding to %pcrel_20(foo) for instruction pcaddi.
   fixup_loongarch_pcrel20_s2,
   // 36-bit fixup corresponding to %call36(foo) for a pair instructions:
   // pcaddu18i+jirl.
-  fixup_loongarch_call36 = FirstRelocationKind + ELF::R_LARCH_CALL36,
+  fixup_loongarch_call36 = ELF::R_LARCH_CALL36,
   // 20-bit fixup corresponding to %desc_pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_tls_desc_pc_hi20 =
-      FirstRelocationKind + ELF::R_LARCH_TLS_DESC_PC_HI20,
+  fixup_loongarch_tls_desc_pc_hi20 = ELF::R_LARCH_TLS_DESC_PC_HI20,
   // 12-bit fixup corresponding to %desc_pc_lo12(foo) for instructions like
   // addi.w/d.
   fixup_loongarch_tls_desc_pc_lo12,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index acdad71757573..48b1145f58b41 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -36,9 +36,8 @@ static cl::opt<bool> ULEB128Reloc(
 
 RISCVAsmBackend::RISCVAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI,
                                  bool Is64Bit, const MCTargetOptions &Options)
-    : MCAsmBackend(llvm::endianness::little,
-                   FirstRelocationKind + ELF::R_RISCV_RELAX),
-      STI(STI), OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {
+    : MCAsmBackend(llvm::endianness::little, ELF::R_RISCV_RELAX), STI(STI),
+      OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {
   RISCVFeatures::validate(STI.getTargetTriple(), STI.getFeatureBits());
 }
 
@@ -659,10 +658,8 @@ bool RISCVAsmBackend::handleAddSubRelocations(const MCAssembler &Asm,
   }
   MCValue A = MCValue::get(Target.getAddSym(), nullptr, Target.getConstant());
   MCValue B = MCValue::get(Target.getSubSym());
-  auto FA =
-      MCFixup::create(Fixup.getOffset(), nullptr, FirstRelocationKind + TA);
-  auto FB =
-      MCFixup::create(Fixup.getOffset(), nullptr, FirstRelocationKind + TB);
+  auto FA = MCFixup::create(Fixup.getOffset(), nullptr, TA);
+  auto FB = MCFixup::create(Fixup.getOffset(), nullptr, TB);
   auto &Assembler = const_cast<MCAssembler &>(Asm);
   Asm.getWriter().recordRelocation(Assembler, &F, FA, A, FixedValueA);
   Asm.getWriter().recordRelocation(Assembler, &F, FB, B, FixedValueB);
@@ -744,8 +741,7 @@ bool RISCVAsmBackend::shouldInsertFixupForCodeAlign(MCAssembler &Asm,
   MCContext &Ctx = Asm.getContext();
   const MCExpr *Dummy = MCConstantExpr::create(0, Ctx);
   // Create fixup_riscv_align fixup.
-  MCFixup Fixup = MCFixup::create(
-      0, Dummy, FirstRelocationKind + ELF::R_RISCV_ALIGN, SMLoc());
+  MCFixup Fixup = MCFixup::create(0, Dummy, ELF::R_RISCV_ALIGN, SMLoc());
 
   uint64_t FixedValue = 0;
   MCValue NopBytes = MCValue::get(Count);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
index 427ea935e5241..35e75489794f7 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
@@ -76,7 +76,7 @@ unsigned RISCVELFObjectWriter::getRelocType(MCContext &Ctx,
   // Extract the relocation type from the fixup kind, after applying STT_TLS as
   // needed.
   if (mc::isRelocation(Fixup.getKind()))
-    return Kind - FirstRelocationKind;
+    return Kind;
 
   if (IsPCRel) {
     switch (Kind) {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index 8af769d295ae8..0f03bf2f68ece 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -180,8 +180,8 @@ void RISCVMCCodeEmitter::expandTLSDESCCall(const MCInst &MI,
   MCRegister Link = MI.getOperand(0).getReg();
   MCRegister Dest = MI.getOperand(1).getReg();
   int64_t Imm = MI.getOperand(2).getImm();
-  Fixups.push_back(MCFixup::create(
-      0, Expr, FirstRelocationKind + ELF::R_RISCV_TLSDESC_CALL, MI.getLoc()));
+  Fixups.push_back(
+      MCFixup::create(0, Expr, ELF::R_RISCV_TLSDESC_CALL, MI.getLoc()));
   MCInst Call =
       MCInstBuilder(RISCV::JALR).addReg(Link).addReg(Dest).addImm(Imm);
 
@@ -209,14 +209,14 @@ void RISCVMCCodeEmitter::expandAddTPRel(const MCInst &MI,
          "Expected tprel_add relocation on TP-relative symbol");
 
   // Emit the correct tprel_add relocation for the symbol.
-  Fixups.push_back(MCFixup::create(
-      0, Expr, FirstRelocationKind + ELF::R_RISCV_TPREL_ADD, MI.getLoc()));
+  Fixups.push_back(
+      MCFixup::create(0, Expr, ELF::R_RISCV_TPREL_ADD, MI.getLoc()));
 
   // Emit R_RISCV_RELAX for tprel_add where the relax feature is enabled.
   if (STI.hasFeature(RISCV::FeatureRelax)) {
     const MCConstantExpr *Dummy = MCConstantExpr::create(0, Ctx);
-    Fixups.push_back(MCFixup::create(
-        0, Dummy, FirstRelocationKind + ELF::R_RISCV_RELAX, MI.getLoc()));
+    Fixups.push_back(
+        MCFixup::create(0, Dummy, ELF::R_RISCV_RELAX, MI.getLoc()));
   }
 
   // Emit a normal ADD instruction with the given operands.
@@ -612,26 +612,26 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_GOT_HI:
-      FixupKind = FirstRelocationKind + ELF::R_RISCV_GOT_HI20;
+      FixupKind = ELF::R_RISCV_GOT_HI20;
       break;
     case RISCVMCExpr::VK_TPREL_LO:
       if (MIFrm == RISCVII::InstFormatI)
-        FixupKind = FirstRelocationKind + ELF::R_RISCV_TPREL_LO12_I;
+        FixupKind = ELF::R_RISCV_TPREL_LO12_I;
       else if (MIFrm == RISCVII::InstFormatS)
-        FixupKind = FirstRelocationKind + ELF::R_RISCV_TPREL_LO12_S;
+        FixupKind = ELF::R_RISCV_TPREL_LO12_S;
       else
         llvm_unreachable("VK_TPREL_LO used with unexpected instruction format");
       RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_TPREL_HI:
-      FixupKind = FirstRelocationKind + ELF::R_RISCV_TPREL_HI20;
+      FixupKind = ELF::R_RISCV_TPREL_HI20;
       RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_TLS_GOT_HI:
-      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLS_GOT_HI20;
+      FixupKind = ELF::R_RISCV_TLS_GOT_HI20;
       break;
     case RISCVMCExpr::VK_TLS_GD_HI:
-      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLS_GD_HI20;
+      FixupKind = ELF::R_RISCV_TLS_GD_HI20;
       break;
     case RISCVMCExpr::VK_CALL:
       FixupKind = RISCV::fixup_riscv_call;
@@ -642,16 +642,16 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_TLSDESC_HI:
-      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLSDESC_HI20;
+      FixupKind = ELF::R_RISCV_TLSDESC_HI20;
       break;
     case RISCVMCExpr::VK_TLSDESC_LOAD_LO:
-      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLSDESC_LOAD_LO12;
+      FixupKind = ELF::R_RISCV_TLSDESC_LOAD_LO12;
       break;
     case RISCVMCExpr::VK_TLSDESC_ADD_LO:
-      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLSDESC_ADD_LO12;
+      FixupKind = ELF::R_RISCV_TLSDESC_ADD_LO12;
       break;
     case RISCVMCExpr::VK_TLSDESC_CALL:
-      FixupKind = FirstRelocationKind + ELF::R_RISCV_TLSDESC_CALL;
+      FixupKind = ELF::R_RISCV_TLSDESC_CALL;
       break;
     case RISCVMCExpr::VK_QC_ABS20:
       FixupKind = RISCV::fixup_riscv_qc_abs20_u;
@@ -689,8 +689,8 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
   // relaxed.
   if (EnableRelax && RelaxCandidate) {
     const MCConstantExpr *Dummy = MCConstantExpr::create(0, Ctx);
-    Fixups.push_back(MCFixup::create(
-        0, Dummy, FirstRelocationKind + ELF::R_RISCV_RELAX, MI.getLoc()));
+    Fixups.push_back(
+        MCFixup::create(0, Dummy, ELF::R_RISCV_RELAX, MI.getLoc()));
     ++MCNumFixups;
   }
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
index af318b4d3ea28..31a3ace2c0477 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
@@ -75,7 +75,7 @@ const MCFixup *RISCVMCExpr::getPCRelHiFixup(const MCFragment **DFOut) const {
       }
       break;
     }
-    switch (Kind - FirstRelocationKind) {
+    switch (Kind) {
     case ELF::R_RISCV_GOT_HI20:
     case ELF::R_RISCV_TLS_GOT_HI20:
     case ELF::R_RISCV_TLS_GD_HI20:

From dca0ccff4f2e6e5df6345352a0392f7230c6bd57 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 20:02:57 -0700
Subject: [PATCH 518/710] LoongArch: Remove unreachable TLS fixup from
 adjustFixupValue

---
 .../lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index ac2127c96019c..2d531b244af38 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -120,16 +120,12 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     return ((Value & 0x3fffc) << 8) | ((Value >> 18) & 0x3ff);
   }
   case LoongArch::fixup_loongarch_abs_hi20:
-  case LoongArch::fixup_loongarch_tls_le_hi20:
     return (Value >> 12) & 0xfffff;
   case LoongArch::fixup_loongarch_abs_lo12:
-  case LoongArch::fixup_loongarch_tls_le_lo12:
     return Value & 0xfff;
   case LoongArch::fixup_loongarch_abs64_lo20:
-  case LoongArch::fixup_loongarch_tls_le64_lo20:
     return (Value >> 32) & 0xfffff;
   case LoongArch::fixup_loongarch_abs64_hi12:
-  case LoongArch::fixup_loongarch_tls_le64_hi12:
     return (Value >> 52) & 0xfff;
   }
 }

From 609586f7f61abf170425883fd8ae390b4a69cc0c Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 20:16:50 -0700
Subject: [PATCH 519/710] LoongArch: Remove fixup kinds that map to relocation
 types

Follow-up to 40789ce7f1b7cff6de82b7f93db25a8c54194d46
("MCFixup: Move relocation values before FK_NONE")
---
 .../MCTargetDesc/LoongArchAsmBackend.cpp      |   4 +-
 .../MCTargetDesc/LoongArchAsmBackend.h        |   5 +-
 .../MCTargetDesc/LoongArchELFObjectWriter.cpp |   3 -
 .../MCTargetDesc/LoongArchFixupKinds.h        |  96 ----------------
 .../MCTargetDesc/LoongArchMCCodeEmitter.cpp   | 105 +++++++++---------
 5 files changed, 55 insertions(+), 158 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index 2d531b244af38..2b65901f1f8a1 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -217,9 +217,7 @@ bool LoongArchAsmBackend::shouldInsertFixupForCodeAlign(MCAssembler &Asm,
   MCSection *Sec = AF.getParent();
   MCContext &Ctx = Asm.getContext();
   const MCExpr *Dummy = MCConstantExpr::create(0, Ctx);
-  // Create fixup_loongarch_align fixup.
-  MCFixup Fixup =
-      MCFixup::create(0, Dummy, MCFixupKind(LoongArch::fixup_loongarch_align));
+  MCFixup Fixup = MCFixup::create(0, Dummy, ELF::R_LARCH_ALIGN);
   unsigned MaxBytesToEmit = AF.getMaxBytesToEmit();
 
   auto createExtendedValue = [&]() {
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
index 099bdabce199f..e7c09a0e97c6e 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
@@ -34,9 +34,8 @@ class LoongArchAsmBackend : public MCAsmBackend {
 public:
   LoongArchAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI, bool Is64Bit,
                       const MCTargetOptions &Options)
-      : MCAsmBackend(llvm::endianness::little,
-                     LoongArch::fixup_loongarch_relax),
-        STI(STI), OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {}
+      : MCAsmBackend(llvm::endianness::little, ELF::R_LARCH_RELAX), STI(STI),
+        OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {}
   ~LoongArchAsmBackend() override {}
 
   bool handleAddSubRelocations(const MCAssembler &Asm, const MCFragment &F,
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
index 0c0caead731b5..bfbf9a4253d0c 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
@@ -110,9 +110,6 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx,
     return ELF::R_LARCH_TLS_LE64_LO20;
   case LoongArch::fixup_loongarch_tls_le64_hi12:
     return ELF::R_LARCH_TLS_LE64_HI12;
-  case LoongArch::fixup_loongarch_call36:
-    return ELF::R_LARCH_CALL36;
-    // TODO: Handle more fixup-kinds.
   }
 }
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
index 4bd64be7e94f9..e2a2cd79e1d8b 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
@@ -51,102 +51,6 @@ enum Fixups {
   // LoongArchAsmBackend::applyFixup.
   fixup_loongarch_invalid,
   NumTargetFixupKinds = fixup_loongarch_invalid - FirstTargetFixupKind,
-
-  // Define fixups for force relocation as FirstRelocationKind+V
-  // represents the relocation type with number V.
-  // 20-bit fixup corresponding to %pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_pcala_hi20 = ELF::R_LARCH_PCALA_HI20,
-  // 12-bit fixup corresponding to %pc_lo12(foo) for instructions like addi.w/d.
-  fixup_loongarch_pcala_lo12,
-  // 20-bit fixup corresponding to %pc64_lo20(foo) for instruction lu32i.d.
-  fixup_loongarch_pcala64_lo20,
-  // 12-bit fixup corresponding to %pc64_hi12(foo) for instruction lu52i.d.
-  fixup_loongarch_pcala64_hi12,
-  // 20-bit fixup corresponding to %got_pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_got_pc_hi20,
-  // 12-bit fixup corresponding to %got_pc_lo12(foo) for instructions
-  // ld.w/ld.d/add.d.
-  fixup_loongarch_got_pc_lo12,
-  // 20-bit fixup corresponding to %got64_pc_lo20(foo) for instruction lu32i.d.
-  fixup_loongarch_got64_pc_lo20,
-  // 12-bit fixup corresponding to %got64_pc_hi12(foo) for instruction lu52i.d.
-  fixup_loongarch_got64_pc_hi12,
-  // 20-bit fixup corresponding to %got_hi20(foo) for instruction lu12i.w.
-  fixup_loongarch_got_hi20,
-  // 12-bit fixup corresponding to %got_lo12(foo) for instruction ori.
-  fixup_loongarch_got_lo12,
-  // 20-bit fixup corresponding to %got64_lo20(foo) for instruction lu32i.d.
-  fixup_loongarch_got64_lo20,
-  // 12-bit fixup corresponding to %got64_hi12(foo) for instruction lu52i.d.
-  fixup_loongarch_got64_hi12,
-  // Skip R_LARCH_TLS_LE_*.
-  // 20-bit fixup corresponding to %ie_pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_tls_ie_pc_hi20 = ELF::R_LARCH_TLS_IE_PC_HI20,
-  // 12-bit fixup corresponding to %ie_pc_lo12(foo) for instructions
-  // ld.w/ld.d/add.d.
-  fixup_loongarch_tls_ie_pc_lo12,
-  // 20-bit fixup corresponding to %ie64_pc_lo20(foo) for instruction lu32i.d.
-  fixup_loongarch_tls_ie64_pc_lo20,
-  // 12-bit fixup corresponding to %ie64_pc_hi12(foo) for instruction lu52i.d.
-  fixup_loongarch_tls_ie64_pc_hi12,
-  // 20-bit fixup corresponding to %ie_hi20(foo) for instruction lu12i.w.
-  fixup_loongarch_tls_ie_hi20,
-  // 12-bit fixup corresponding to %ie_lo12(foo) for instruction ori.
-  fixup_loongarch_tls_ie_lo12,
-  // 20-bit fixup corresponding to %ie64_lo20(foo) for instruction lu32i.d.
-  fixup_loongarch_tls_ie64_lo20,
-  // 12-bit fixup corresponding to %ie64_hi12(foo) for instruction lu52i.d.
-  fixup_loongarch_tls_ie64_hi12,
-  // 20-bit fixup corresponding to %ld_pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_tls_ld_pc_hi20,
-  // 20-bit fixup corresponding to %ld_hi20(foo) for instruction lu12i.w.
-  fixup_loongarch_tls_ld_hi20,
-  // 20-bit fixup corresponding to %gd_pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_tls_gd_pc_hi20,
-  // 20-bit fixup corresponding to %gd_hi20(foo) for instruction lu12i.w.
-  fixup_loongarch_tls_gd_hi20,
-  // Generate an R_LARCH_RELAX which indicates the linker may relax here.
-  fixup_loongarch_relax = ELF::R_LARCH_RELAX,
-  // Generate an R_LARCH_ALIGN which indicates the linker may fixup align here.
-  fixup_loongarch_align = ELF::R_LARCH_ALIGN,
-  // 20-bit fixup corresponding to %pcrel_20(foo) for instruction pcaddi.
-  fixup_loongarch_pcrel20_s2,
-  // 36-bit fixup corresponding to %call36(foo) for a pair instructions:
-  // pcaddu18i+jirl.
-  fixup_loongarch_call36 = ELF::R_LARCH_CALL36,
-  // 20-bit fixup corresponding to %desc_pc_hi20(foo) for instruction pcalau12i.
-  fixup_loongarch_tls_desc_pc_hi20 = ELF::R_LARCH_TLS_DESC_PC_HI20,
-  // 12-bit fixup corresponding to %desc_pc_lo12(foo) for instructions like
-  // addi.w/d.
-  fixup_loongarch_tls_desc_pc_lo12,
-  // 20-bit fixup corresponding to %desc64_pc_lo20(foo) for instruction lu32i.d.
-  fixup_loongarch_tls_desc64_pc_lo20,
-  // 12-bit fixup corresponding to %desc64_pc_hi12(foo) for instruction lu52i.d.
-  fixup_loongarch_tls_desc64_pc_hi12,
-  // 20-bit fixup corresponding to %desc_hi20(foo) for instruction lu12i.w.
-  fixup_loongarch_tls_desc_hi20,
-  // 12-bit fixup corresponding to %desc_lo12(foo) for instruction ori.
-  fixup_loongarch_tls_desc_lo12,
-  // 20-bit fixup corresponding to %desc64_lo20(foo) for instruction lu32i.d.
-  fixup_loongarch_tls_desc64_lo20,
-  // 12-bit fixup corresponding to %desc64_hi12(foo) for instruction lu52i.d.
-  fixup_loongarch_tls_desc64_hi12,
-  // 12-bit fixup corresponding to %desc_ld(foo) for instruction ld.w/d.
-  fixup_loongarch_tls_desc_ld,
-  // 12-bit fixup corresponding to %desc_call(foo) for instruction jirl.
-  fixup_loongarch_tls_desc_call,
-  // 20-bit fixup corresponding to %le_hi20_r(foo) for instruction lu12i.w.
-  fixup_loongarch_tls_le_hi20_r,
-  // Fixup corresponding to %le_add_r(foo) for instruction PseudoAddTPRel_W/D.
-  fixup_loongarch_tls_le_add_r,
-  // 12-bit fixup corresponding to %le_lo12_r(foo) for instruction addi.w/d.
-  fixup_loongarch_tls_le_lo12_r,
-  // 20-bit fixup corresponding to %ld_pcrel_20(foo) for instruction pcaddi.
-  fixup_loongarch_tls_ld_pcrel20_s2,
-  // 20-bit fixup corresponding to %gd_pcrel_20(foo) for instruction pcaddi.
-  fixup_loongarch_tls_gd_pcrel20_s2,
-  // 20-bit fixup corresponding to %desc_pcrel_20(foo) for instruction pcaddi.
-  fixup_loongarch_tls_desc_pcrel20_s2,
 };
 } // end namespace LoongArch
 } // end namespace llvm
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
index b0785e34862d0..b72d84e19e9eb 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
@@ -128,7 +128,7 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
   bool EnableRelax = STI.hasFeature(LoongArch::FeatureRelax);
   const MCExpr *Expr = MO.getExpr();
   MCExpr::ExprKind Kind = Expr->getKind();
-  LoongArch::Fixups FixupKind = LoongArch::fixup_loongarch_invalid;
+  unsigned FixupKind = LoongArch::fixup_loongarch_invalid;
   if (Kind == MCExpr::Target) {
     const LoongArchMCExpr *LAExpr = cast<LoongArchMCExpr>(Expr);
 
@@ -163,142 +163,142 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
       FixupKind = LoongArch::fixup_loongarch_abs64_hi12;
       break;
     case LoongArchMCExpr::VK_PCALA_HI20:
-      FixupKind = LoongArch::fixup_loongarch_pcala_hi20;
+      FixupKind = ELF::R_LARCH_PCALA_HI20;
       break;
     case LoongArchMCExpr::VK_PCALA_LO12:
-      FixupKind = LoongArch::fixup_loongarch_pcala_lo12;
+      FixupKind = ELF::R_LARCH_PCALA_LO12;
       break;
     case LoongArchMCExpr::VK_PCALA64_LO20:
-      FixupKind = LoongArch::fixup_loongarch_pcala64_lo20;
+      FixupKind = ELF::R_LARCH_PCALA64_LO20;
       break;
     case LoongArchMCExpr::VK_PCALA64_HI12:
-      FixupKind = LoongArch::fixup_loongarch_pcala64_hi12;
+      FixupKind = ELF::R_LARCH_PCALA64_HI12;
       break;
     case LoongArchMCExpr::VK_GOT_PC_HI20:
-      FixupKind = LoongArch::fixup_loongarch_got_pc_hi20;
+      FixupKind = ELF::R_LARCH_GOT_PC_HI20;
       break;
     case LoongArchMCExpr::VK_GOT_PC_LO12:
-      FixupKind = LoongArch::fixup_loongarch_got_pc_lo12;
+      FixupKind = ELF::R_LARCH_GOT_PC_LO12;
       break;
     case LoongArchMCExpr::VK_GOT64_PC_LO20:
-      FixupKind = LoongArch::fixup_loongarch_got64_pc_lo20;
+      FixupKind = ELF::R_LARCH_GOT64_PC_LO20;
       break;
     case LoongArchMCExpr::VK_GOT64_PC_HI12:
-      FixupKind = LoongArch::fixup_loongarch_got64_pc_hi12;
+      FixupKind = ELF::R_LARCH_GOT64_PC_HI12;
       break;
     case LoongArchMCExpr::VK_GOT_HI20:
-      FixupKind = LoongArch::fixup_loongarch_got_hi20;
+      FixupKind = ELF::R_LARCH_GOT_HI20;
       break;
     case LoongArchMCExpr::VK_GOT_LO12:
-      FixupKind = LoongArch::fixup_loongarch_got_lo12;
+      FixupKind = ELF::R_LARCH_GOT_LO12;
       break;
     case LoongArchMCExpr::VK_GOT64_LO20:
-      FixupKind = LoongArch::fixup_loongarch_got64_lo20;
+      FixupKind = ELF::R_LARCH_GOT64_LO20;
       break;
     case LoongArchMCExpr::VK_GOT64_HI12:
-      FixupKind = LoongArch::fixup_loongarch_got64_hi12;
+      FixupKind = ELF::R_LARCH_GOT64_HI12;
       break;
     case LoongArchMCExpr::VK_TLS_LE_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_le_hi20;
+      FixupKind = ELF::R_LARCH_TLS_LE_HI20;
       break;
     case LoongArchMCExpr::VK_TLS_LE_LO12:
-      FixupKind = LoongArch::fixup_loongarch_tls_le_lo12;
+      FixupKind = ELF::R_LARCH_TLS_LE_LO12;
       break;
     case LoongArchMCExpr::VK_TLS_LE64_LO20:
-      FixupKind = LoongArch::fixup_loongarch_tls_le64_lo20;
+      FixupKind = ELF::R_LARCH_TLS_LE64_LO20;
       break;
     case LoongArchMCExpr::VK_TLS_LE64_HI12:
-      FixupKind = LoongArch::fixup_loongarch_tls_le64_hi12;
+      FixupKind = ELF::R_LARCH_TLS_LE64_HI12;
       break;
     case LoongArchMCExpr::VK_TLS_IE_PC_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_ie_pc_hi20;
+      FixupKind = ELF::R_LARCH_TLS_IE_PC_HI20;
       break;
     case LoongArchMCExpr::VK_TLS_IE_PC_LO12:
-      FixupKind = LoongArch::fixup_loongarch_tls_ie_pc_lo12;
+      FixupKind = ELF::R_LARCH_TLS_IE_PC_LO12;
       break;
     case LoongArchMCExpr::VK_TLS_IE64_PC_LO20:
-      FixupKind = LoongArch::fixup_loongarch_tls_ie64_pc_lo20;
+      FixupKind = ELF::R_LARCH_TLS_IE64_PC_LO20;
       break;
     case LoongArchMCExpr::VK_TLS_IE64_PC_HI12:
-      FixupKind = LoongArch::fixup_loongarch_tls_ie64_pc_hi12;
+      FixupKind = ELF::R_LARCH_TLS_IE64_PC_HI12;
       break;
     case LoongArchMCExpr::VK_TLS_IE_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_ie_hi20;
+      FixupKind = ELF::R_LARCH_TLS_IE_HI20;
       break;
     case LoongArchMCExpr::VK_TLS_IE_LO12:
-      FixupKind = LoongArch::fixup_loongarch_tls_ie_lo12;
+      FixupKind = ELF::R_LARCH_TLS_IE_LO12;
       break;
     case LoongArchMCExpr::VK_TLS_IE64_LO20:
-      FixupKind = LoongArch::fixup_loongarch_tls_ie64_lo20;
+      FixupKind = ELF::R_LARCH_TLS_IE64_LO20;
       break;
     case LoongArchMCExpr::VK_TLS_IE64_HI12:
-      FixupKind = LoongArch::fixup_loongarch_tls_ie64_hi12;
+      FixupKind = ELF::R_LARCH_TLS_IE64_HI12;
       break;
     case LoongArchMCExpr::VK_TLS_LD_PC_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_ld_pc_hi20;
+      FixupKind = ELF::R_LARCH_TLS_LD_PC_HI20;
       break;
     case LoongArchMCExpr::VK_TLS_LD_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_ld_hi20;
+      FixupKind = ELF::R_LARCH_TLS_LD_HI20;
       break;
     case LoongArchMCExpr::VK_TLS_GD_PC_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_gd_pc_hi20;
+      FixupKind = ELF::R_LARCH_TLS_GD_PC_HI20;
       break;
     case LoongArchMCExpr::VK_TLS_GD_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_gd_hi20;
+      FixupKind = ELF::R_LARCH_TLS_GD_HI20;
       break;
     case LoongArchMCExpr::VK_CALL36:
-      FixupKind = LoongArch::fixup_loongarch_call36;
+      FixupKind = ELF::R_LARCH_CALL36;
       RelaxCandidate = true;
       break;
     case LoongArchMCExpr::VK_TLS_DESC_PC_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc_pc_hi20;
+      FixupKind = ELF::R_LARCH_TLS_DESC_PC_HI20;
       break;
     case LoongArchMCExpr::VK_TLS_DESC_PC_LO12:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc_pc_lo12;
+      FixupKind = ELF::R_LARCH_TLS_DESC_PC_LO12;
       break;
     case LoongArchMCExpr::VK_TLS_DESC64_PC_LO20:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc64_pc_lo20;
+      FixupKind = ELF::R_LARCH_TLS_DESC64_PC_LO20;
       break;
     case LoongArchMCExpr::VK_TLS_DESC64_PC_HI12:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc64_pc_hi12;
+      FixupKind = ELF::R_LARCH_TLS_DESC64_PC_HI12;
       break;
     case LoongArchMCExpr::VK_TLS_DESC_HI20:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc_hi20;
+      FixupKind = ELF::R_LARCH_TLS_DESC_HI20;
       break;
     case LoongArchMCExpr::VK_TLS_DESC_LO12:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc_lo12;
+      FixupKind = ELF::R_LARCH_TLS_DESC_LO12;
       break;
     case LoongArchMCExpr::VK_TLS_DESC64_LO20:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc64_lo20;
+      FixupKind = ELF::R_LARCH_TLS_DESC64_LO20;
       break;
     case LoongArchMCExpr::VK_TLS_DESC64_HI12:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc64_hi12;
+      FixupKind = ELF::R_LARCH_TLS_DESC64_HI12;
       break;
     case LoongArchMCExpr::VK_TLS_DESC_LD:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc_ld;
+      FixupKind = ELF::R_LARCH_TLS_DESC_LD;
       break;
     case LoongArchMCExpr::VK_TLS_DESC_CALL:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc_call;
+      FixupKind = ELF::R_LARCH_TLS_DESC_CALL;
       break;
     case LoongArchMCExpr::VK_TLS_LE_HI20_R:
-      FixupKind = LoongArch::fixup_loongarch_tls_le_hi20_r;
+      FixupKind = ELF::R_LARCH_TLS_LE_HI20_R;
       RelaxCandidate = true;
       break;
     case LoongArchMCExpr::VK_TLS_LE_LO12_R:
-      FixupKind = LoongArch::fixup_loongarch_tls_le_lo12_r;
+      FixupKind = ELF::R_LARCH_TLS_LE_LO12_R;
       RelaxCandidate = true;
       break;
     case LoongArchMCExpr::VK_PCREL20_S2:
-      FixupKind = LoongArch::fixup_loongarch_pcrel20_s2;
+      FixupKind = ELF::R_LARCH_PCREL20_S2;
       break;
     case LoongArchMCExpr::VK_TLS_LD_PCREL20_S2:
-      FixupKind = LoongArch::fixup_loongarch_tls_ld_pcrel20_s2;
+      FixupKind = ELF::R_LARCH_TLS_LD_PCREL20_S2;
       break;
     case LoongArchMCExpr::VK_TLS_GD_PCREL20_S2:
-      FixupKind = LoongArch::fixup_loongarch_tls_gd_pcrel20_s2;
+      FixupKind = ELF::R_LARCH_TLS_GD_PCREL20_S2;
       break;
     case LoongArchMCExpr::VK_TLS_DESC_PCREL20_S2:
-      FixupKind = LoongArch::fixup_loongarch_tls_desc_pcrel20_s2;
+      FixupKind = ELF::R_LARCH_TLS_DESC_PCREL20_S2;
       break;
     }
   } else if (Kind == MCExpr::SymbolRef &&
@@ -338,8 +338,8 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
   // hint.
   if (EnableRelax && RelaxCandidate) {
     const MCConstantExpr *Dummy = MCConstantExpr::create(0, Ctx);
-    Fixups.push_back(MCFixup::create(
-        0, Dummy, MCFixupKind(LoongArch::fixup_loongarch_relax), MI.getLoc()));
+    Fixups.push_back(
+        MCFixup::create(0, Dummy, ELF::R_LARCH_RELAX, MI.getLoc()));
   }
 
   return 0;
@@ -388,15 +388,14 @@ void LoongArchMCCodeEmitter::expandAddTPRel(const MCInst &MI,
          "Expected %le_add_r relocation on TP-relative symbol");
 
   // Emit the correct %le_add_r relocation for the symbol.
-  Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(LoongArch::fixup_loongarch_tls_le_add_r),
-      MI.getLoc()));
+  Fixups.push_back(
+      MCFixup::create(0, Expr, ELF::R_LARCH_TLS_LE_ADD_R, MI.getLoc()));
 
   // Emit R_LARCH_RELAX for %le_add_r when the relax feature is enabled.
   if (STI.hasFeature(LoongArch::FeatureRelax)) {
     const MCConstantExpr *Dummy = MCConstantExpr::create(0, Ctx);
-    Fixups.push_back(MCFixup::create(
-        0, Dummy, MCFixupKind(LoongArch::fixup_loongarch_relax), MI.getLoc()));
+    Fixups.push_back(
+        MCFixup::create(0, Dummy, ELF::R_LARCH_RELAX, MI.getLoc()));
   }
 
   // Emit a normal ADD instruction with the given operands.

From 89687e6f383b742a3c6542dc673a84d9f82d02de Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 20:19:41 -0700
Subject: [PATCH 520/710] LoongArch: Remove TLS fixup kinds that map to
 relocationtypes

Follow-up to 40789ce7f1b7cff6de82b7f93db25a8c54194d46
("MCFixup: Move relocation values before FK_NONE")
---
 .../LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp       | 5 -----
 .../LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp  | 8 --------
 .../Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h  | 9 ---------
 3 files changed, 22 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index 2b65901f1f8a1..bab85235c622e 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -58,11 +58,6 @@ LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_loongarch_abs_lo12", 10, 12, 0},
       {"fixup_loongarch_abs64_lo20", 5, 20, 0},
       {"fixup_loongarch_abs64_hi12", 10, 12, 0},
-      {"fixup_loongarch_tls_le_hi20", 5, 20, 0},
-      {"fixup_loongarch_tls_le_lo12", 10, 12, 0},
-      {"fixup_loongarch_tls_le64_lo20", 5, 20, 0},
-      {"fixup_loongarch_tls_le64_hi12", 10, 12, 0},
-      // TODO: Add more fixup kinds.
   };
 
   static_assert((std::size(Infos)) == LoongArch::NumTargetFixupKinds,
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
index bfbf9a4253d0c..5b6ecc1e8bc1f 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
@@ -102,14 +102,6 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx,
     return ELF::R_LARCH_ABS64_LO20;
   case LoongArch::fixup_loongarch_abs64_hi12:
     return ELF::R_LARCH_ABS64_HI12;
-  case LoongArch::fixup_loongarch_tls_le_hi20:
-    return ELF::R_LARCH_TLS_LE_HI20;
-  case LoongArch::fixup_loongarch_tls_le_lo12:
-    return ELF::R_LARCH_TLS_LE_LO12;
-  case LoongArch::fixup_loongarch_tls_le64_lo20:
-    return ELF::R_LARCH_TLS_LE64_LO20;
-  case LoongArch::fixup_loongarch_tls_le64_hi12:
-    return ELF::R_LARCH_TLS_LE64_HI12;
   }
 }
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
index e2a2cd79e1d8b..515eab0480f2f 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h
@@ -37,15 +37,6 @@ enum Fixups {
   fixup_loongarch_abs64_lo20,
   // 12-bit fixup corresponding to %abs_hi12(foo) for instruction lu52i.d.
   fixup_loongarch_abs64_hi12,
-  // 20-bit fixup corresponding to %le_hi20(foo) for instruction lu12i.w.
-  fixup_loongarch_tls_le_hi20,
-  // 12-bit fixup corresponding to %le_lo12(foo) for instruction ori.
-  fixup_loongarch_tls_le_lo12,
-  // 20-bit fixup corresponding to %le64_lo20(foo) for instruction lu32i.d.
-  fixup_loongarch_tls_le64_lo20,
-  // 12-bit fixup corresponding to %le64_hi12(foo) for instruction lu52i.d.
-  fixup_loongarch_tls_le64_hi12,
-  // TODO: Add more fixup kind.
 
   // Used as a sentinel, must be the last of the fixup which can be handled by
   // LoongArchAsmBackend::applyFixup.

From 68cef491ebdcaffa0585cb2b22e2be88bd6e36ef Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 20:33:40 -0700
Subject: [PATCH 521/710] [RISCV,test] Remove CHECK lines for MCAsmStreamer's
 fixup output

The fixup output is a debug aid and should not be used to test
target-specific relocation generation implementation. The llvm-mc
-filetype=obj output is what truly matters.
---
 llvm/test/MC/RISCV/compressed-relocations.s |  9 +++------
 llvm/test/MC/RISCV/fixups-compressed.s      |  6 ------
 llvm/test/MC/RISCV/fixups.s                 | 16 ----------------
 llvm/test/MC/RISCV/xqcibi-relocations.s     |  7 +------
 4 files changed, 4 insertions(+), 34 deletions(-)

diff --git a/llvm/test/MC/RISCV/compressed-relocations.s b/llvm/test/MC/RISCV/compressed-relocations.s
index 196d9873f40e6..82cb1a12c389b 100644
--- a/llvm/test/MC/RISCV/compressed-relocations.s
+++ b/llvm/test/MC/RISCV/compressed-relocations.s
@@ -1,5 +1,5 @@
-# RUN: llvm-mc -triple riscv32 -mattr=+c -M no-aliases < %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+# RUN: llvm-mc -triple riscv32 -mattr=+c -M no-aliases %s -show-encoding \
+# RUN:     | FileCheck -check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c < %s \
 # RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,+relax < %s \
@@ -7,17 +7,14 @@
 
 # Check prefixes:
 # RELOC - Check the relocation in the object.
-# FIXUP - Check the fixup on the instruction.
 # INSTR - Check the instruction is handled properly by the ASMPrinter
 c.jal foo
 # A compressed jump (c.j) to an unresolved symbol will be relaxed to a (jal).
 # RELOC: R_RISCV_JAL
 # INSTR: c.jal foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_rvc_jump
 
 c.bnez a0, foo
 # A compressed branch (c.bnez) to an unresolved symbol will be relaxed to a (bnez).
 # The (bnez) to an unresolved symbol will in turn be relaxed to (beqz; jal)
-# RELOC: R_RISCV_JAL
+# RELOC-NEXT: R_RISCV_JAL
 # INSTR: c.bnez a0, foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_rvc_branch
diff --git a/llvm/test/MC/RISCV/fixups-compressed.s b/llvm/test/MC/RISCV/fixups-compressed.s
index e856fd8a17948..6e21e664e705b 100644
--- a/llvm/test/MC/RISCV/fixups-compressed.s
+++ b/llvm/test/MC/RISCV/fixups-compressed.s
@@ -1,21 +1,15 @@
-# RUN: llvm-mc %s -triple riscv32 -mattr=+c -show-encoding \
-# RUN:     | FileCheck -check-prefix=CHECK-FIXUP %s
 # RUN: llvm-mc -triple riscv32 -filetype=obj -mattr=+c < %s \
 # RUN:     | llvm-objdump -d -M no-aliases - | FileCheck --check-prefix=CHECK-INSTR %s
 # RUN: llvm-mc -filetype=obj -mattr=+c -triple=riscv32 %s \
 # RUN:     | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-REL
 
 .LBB0_2:
-# CHECK-FIXUP:   fixup A - offset: 0, value: .LBB0_2, kind: fixup_riscv_rvc_jump
 # CHECK-INSTR: c.j     0
 c.j     .LBB0_2
-# CHECK:   fixup A - offset: 0, value: func1, kind: fixup_riscv_rvc_jump
 # CHECK-INSTR: c.jal   0x8
 c.jal   func1
-# CHECK-FIXUP:   fixup A - offset: 0, value: .LBB0_2, kind: fixup_riscv_rvc_branch
 # CHECK-INSTR: c.beqz  a3, 0x0
 c.beqz  a3, .LBB0_2
-# CHECK-FIXUP:   fixup A - offset: 0, value: .LBB0_2, kind: fixup_riscv_rvc_branch
 # CHECK-INSTR: c.bnez  a5, 0x0
 c.bnez  a5, .LBB0_2
 
diff --git a/llvm/test/MC/RISCV/fixups.s b/llvm/test/MC/RISCV/fixups.s
index 5145dbe8db888..9ebf512609124 100644
--- a/llvm/test/MC/RISCV/fixups.s
+++ b/llvm/test/MC/RISCV/fixups.s
@@ -1,5 +1,3 @@
-# RUN: llvm-mc -triple riscv32 -M no-aliases < %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=CHECK-FIXUP %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
 # RUN:     | llvm-objdump --no-print-imm-hex -M no-aliases -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INSTR %s
@@ -11,41 +9,30 @@
 
 .LBB0:
 lui t1, %hi(val)
-# CHECK-FIXUP: fixup A - offset: 0, value: %hi(val), kind: fixup_riscv_hi20
 # CHECK-INSTR: lui t1, 74565
 
 lw a0, %lo(val)(t1)
-# CHECK-FIXUP: fixup A - offset: 0, value: %lo(val), kind: fixup_riscv_lo12_i
 # CHECK-INSTR: lw a0, 1656(t1)
 addi a1, t1, %lo(val)
-# CHECK-FIXUP: fixup A - offset: 0, value: %lo(val), kind: fixup_riscv_lo12_i
 # CHECK-INSTR: addi a1, t1, 1656
 sw a0, %lo(val)(t1)
-# CHECK-FIXUP: fixup A - offset: 0, value: %lo(val), kind: fixup_riscv_lo12_s
 # CHECK-INSTR: sw a0, 1656(t1)
 
 1:
 auipc t1, %pcrel_hi(.LBB0)
-# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_hi(.LBB0), kind: fixup_riscv_pcrel_hi20
 # CHECK-INSTR: auipc t1, 0
 addi t1, t1, %pcrel_lo(1b)
-# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_lo(.Ltmp0), kind: fixup_riscv_pcrel_lo12_i
 # CHECK-INSTR: addi t1, t1, -16
 sw t1, %pcrel_lo(1b)(t1)
-# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_lo(.Ltmp0), kind: fixup_riscv_pcrel_lo12_s
 # CHECK-INSTR: sw t1, -16(t1)
 
 jal zero, .LBB0
-# CHECK-FIXUP: fixup A - offset: 0, value: .LBB0, kind: fixup_riscv_jal
 # CHECK-INSTR: jal zero, 0x0
 jal zero, .LBB2
-# CHECK-FIXUP: fixup A - offset: 0, value: .LBB2, kind: fixup_riscv_jal
 # CHECK-INSTR: jal zero, 0x50d14
 beq a0, a1, .LBB0
-# CHECK-FIXUP: fixup A - offset: 0, value: .LBB0, kind: fixup_riscv_branch
 # CHECK-INSTR: beq a0, a1, 0x0
 blt a0, a1, .LBB1
-# CHECK-FIXUP: fixup A - offset: 0, value: .LBB1, kind: fixup_riscv_branch
 # CHECK-INSTR: blt a0, a1, 0x47c
 
 .fill 1104
@@ -66,18 +53,15 @@ addi zero, zero, 0
 func:
 .fill 100
 call func
-# CHECK-FIXUP: fixup A - offset: 0, value: func, kind: fixup_riscv_call
 # CHECK-INSTR: auipc   ra, 0
 # CHECK-INSTR: jalr    ra, -100(ra)
 
 .fill 10000
 call func
-# CHECK-FIXUP: fixup A - offset: 0, value: func, kind: fixup_riscv_call
 # CHECK-INSTR: auipc   ra, 1048574
 # CHECK-INSTR: jalr    ra, -1916(ra)
 
 .fill 20888
 call func
-# CHECK-FIXUP: fixup A - offset: 0, value: func, kind: fixup_riscv_call
 # CHECK-INSTR: auipc   ra, 1048568
 # CHECK-INSTR: jalr    ra, 1764(ra)
diff --git a/llvm/test/MC/RISCV/xqcibi-relocations.s b/llvm/test/MC/RISCV/xqcibi-relocations.s
index 7028e8a737c86..054a890a4b5e5 100644
--- a/llvm/test/MC/RISCV/xqcibi-relocations.s
+++ b/llvm/test/MC/RISCV/xqcibi-relocations.s
@@ -1,11 +1,10 @@
 # RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcibi %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+# RUN:     | FileCheck -check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcibi %s -o %t.o
 # RUN: llvm-readobj -r %t.o | FileCheck -check-prefix=RELOC %s
 
 # Check prefixes:
 # RELOC - Check the relocation in the object.
-# FIXUP - Check the fixup on the instruction.
 # INSTR - Check the instruction is handled properly by the ASMPrinter.
 
 .text
@@ -14,25 +13,21 @@
 qc.bnei x6, 10, foo
 # RELOC: R_RISCV_JAL foo 0x0
 # INSTR: qc.bnei t1, 10, foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_branch
 
 # Since foo is undefined, this will be relaxed to (qc.e.bltui + jal)
 qc.e.bgeui x8, 12, foo
 # RELOC: R_RISCV_JAL foo 0x0
 # INSTR: qc.e.bgeui s0, 12, foo
-# FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_qc_e_branch
 
 # Check that a label in a different section is handled similar to an undefined
 # symbol and gets relaxed to (qc.e.bgeui + jal)
 qc.e.bltui x4, 9, .bar
 # RELOC: R_RISCV_JAL .bar 0x0
 # INSTR: qc.e.bltui tp, 9, .bar
-# FIXUP: fixup A - offset: 0, value: .bar, kind: fixup_riscv_qc_e_branch
 
 # Check that branches to a defined symbol are handled correctly
 qc.e.beqi x7, 8, .L1
 # INSTR: qc.e.beqi t2, 8, .L1
-# FIXUP: fixup A - offset: 0, value: .L1, kind: fixup_riscv_qc_e_branch
 
 .L1:
   ret

From e00693659b9b18647565e380c27da039507673f4 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 20:34:01 -0700
Subject: [PATCH 522/710] [clang-tidy] Use llvm::less_second (NFC) (#136396)

---
 clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
index 38ef7712aa4ef..ff5a302dfa75d 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
@@ -415,9 +415,7 @@ void UseStdNumbersCheck::check(const MatchFinder::MatchResult &Result) {
     return;
   }
 
-  llvm::sort(MatchedLiterals, [](const auto &LHS, const auto &RHS) {
-    return std::get<1>(LHS) < std::get<1>(RHS);
-  });
+  llvm::sort(MatchedLiterals, llvm::less_second());
 
   const auto &[Constant, Diff, Node] = MatchedLiterals.front();
 

From bb2e222407c199e330976aebfccb0e05bdf03957 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 20:34:20 -0700
Subject: [PATCH 523/710] [lld] Use llvm::less_first (NFC) (#136397)

---
 lld/MachO/MapFile.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lld/MachO/MapFile.cpp b/lld/MachO/MapFile.cpp
index 8919c8d2f9b9c..f3e221a700b14 100644
--- a/lld/MachO/MapFile.cpp
+++ b/lld/MachO/MapFile.cpp
@@ -116,9 +116,7 @@ static MapInfo gatherMapInfo() {
   // cstrings are not stored in sorted order in their OutputSections, so we sort
   // them here.
   for (auto &liveCStrings : info.liveCStringsForSection)
-    parallelSort(liveCStrings.second, [](const auto &p1, const auto &p2) {
-      return p1.first < p2.first;
-    });
+    parallelSort(liveCStrings.second, llvm::less_first());
   return info;
 }
 

From 356ff75e077ce0684b439869f79c72c6df7db687 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 18 Apr 2025 20:34:42 -0700
Subject: [PATCH 524/710] [mlir] Use llvm::less_first (NFC) (#136398)

---
 mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
index ea5533dfc6bac..6e816dc0661a1 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
@@ -352,8 +352,7 @@ void LoopEmitter::initSubSectIterator(OpBuilder &builder, Location loc) {
     if (depRedOrder.empty())
       continue;
 
-    std::sort(depRedOrder.begin(), depRedOrder.end(),
-              [](auto &l, auto &r) { return std::get<0>(l) < std::get<0>(r); });
+    llvm::sort(depRedOrder, llvm::less_first());
 
     SmallVector<SparseIterator *> lastIter(tensors.size(), nullptr);
     for (auto [loop, t, lvl] : depRedOrder) {

From c239acb5b6b80ddb3c6407ffc9e051f636ae41c1 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 18 Apr 2025 20:55:42 -0700
Subject: [PATCH 525/710] MCFixup: Make FixupKindInfo smaller and change
 getFixupKindInfo to return value

We will increase the use of raw relocation types and eliminate fixup
kinds that correspond to relocation types. The getFixupKindInfo
functions will return an rvalue instead. Let's update the return type
from a const reference to a value type.
---
 bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp           | 2 +-
 bolt/lib/Target/X86/X86MCPlusBuilder.cpp                   | 2 +-
 llvm/include/llvm/MC/MCAsmBackend.h                        | 2 +-
 llvm/include/llvm/MC/MCFixupKindInfo.h                     | 6 ++++--
 llvm/lib/MC/MCAsmBackend.cpp                               | 2 +-
 llvm/lib/MC/MCAsmStreamer.cpp                              | 2 +-
 llvm/lib/MC/MachObjectWriter.cpp                           | 3 +--
 llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp | 2 +-
 .../AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp    | 2 +-
 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp   | 5 ++---
 llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp         | 2 +-
 llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h           | 2 +-
 llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp         | 4 ++--
 llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h           | 3 +--
 llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp         | 5 ++---
 llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp       | 3 +--
 llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h         | 2 +-
 llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp | 3 +--
 llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp     | 6 ++----
 .../Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp  | 3 +--
 .../Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h    | 2 +-
 llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp   | 3 +--
 llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp       | 3 +--
 llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h         | 3 +--
 llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp     | 2 +-
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp     | 3 +--
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h       | 2 +-
 llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp     | 2 +-
 .../Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp    | 5 ++---
 llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp           | 3 +--
 .../WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp     | 7 +++----
 llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp         | 4 ++--
 llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp   | 5 ++---
 33 files changed, 45 insertions(+), 60 deletions(-)

diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 106f0a880d780..e00d6a18b0f6c 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -2336,7 +2336,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
   std::optional<Relocation>
   createRelocation(const MCFixup &Fixup,
                    const MCAsmBackend &MAB) const override {
-    const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind());
+    MCFixupKindInfo FKI = MAB.getFixupKindInfo(Fixup.getKind());
 
     assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
     const uint64_t RelOffset = Fixup.getOffset();
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index 8e459e10244fd..b909d7fb6bf28 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -2441,7 +2441,7 @@ class X86MCPlusBuilder : public MCPlusBuilder {
   std::optional<Relocation>
   createRelocation(const MCFixup &Fixup,
                    const MCAsmBackend &MAB) const override {
-    const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind());
+    MCFixupKindInfo FKI = MAB.getFixupKindInfo(Fixup.getKind());
 
     assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
     const uint64_t RelOffset = Fixup.getOffset();
diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index 61e3b2e59e3ab..691988ae1f1df 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -87,7 +87,7 @@ class MCAsmBackend {
   virtual std::optional<MCFixupKind> getFixupKind(StringRef Name) const;
 
   /// Get information on a fixup kind.
-  virtual const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const;
+  virtual MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const;
 
   // Hook to check if a relocation is needed. The default implementation tests
   // whether the MCValue has a relocation specifier.
diff --git a/llvm/include/llvm/MC/MCFixupKindInfo.h b/llvm/include/llvm/MC/MCFixupKindInfo.h
index ecf85fa569318..4fae148f38628 100644
--- a/llvm/include/llvm/MC/MCFixupKindInfo.h
+++ b/llvm/include/llvm/MC/MCFixupKindInfo.h
@@ -9,6 +9,8 @@
 #ifndef LLVM_MC_MCFIXUPKINDINFO_H
 #define LLVM_MC_MCFIXUPKINDINFO_H
 
+#include <cstdint>
+
 namespace llvm {
 
 /// Target independent information on a fixup kind.
@@ -35,11 +37,11 @@ struct MCFixupKindInfo {
   const char *Name;
 
   /// The bit offset to write the relocation into.
-  unsigned TargetOffset;
+  uint8_t TargetOffset;
 
   /// The number of bits written by this fixup. The bits are assumed to be
   /// contiguous.
-  unsigned TargetSize;
+  uint8_t TargetSize;
 
   /// Flags describing additional information on this fixup kind.
   unsigned Flags;
diff --git a/llvm/lib/MC/MCAsmBackend.cpp b/llvm/lib/MC/MCAsmBackend.cpp
index 03a39bd6a4cff..3c51755c8820b 100644
--- a/llvm/lib/MC/MCAsmBackend.cpp
+++ b/llvm/lib/MC/MCAsmBackend.cpp
@@ -87,7 +87,7 @@ std::optional<MCFixupKind> MCAsmBackend::getFixupKind(StringRef Name) const {
   return std::nullopt;
 }
 
-const MCFixupKindInfo &MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   static const MCFixupKindInfo Builtins[] = {
       {"FK_NONE", 0, 0, 0},
       {"FK_Data_1", 0, 8, 0},
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 361f9492d60b4..6d39b92ab25cc 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -2338,7 +2338,7 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
 
   for (unsigned i = 0, e = Fixups.size(); i != e; ++i) {
     MCFixup &F = Fixups[i];
-    const MCFixupKindInfo &Info =
+    MCFixupKindInfo Info =
         getAssembler().getBackend().getFixupKindInfo(F.getKind());
     for (unsigned j = 0; j != Info.TargetSize; ++j) {
       unsigned Index = F.getOffset() * 8 + Info.TargetOffset + j;
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 7fac6c78bc1a8..940327c296d85 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -82,8 +82,7 @@ MachSymbolData::operator<(const MachSymbolData &RHS) const {
 }
 
 bool MachObjectWriter::isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind) {
-  const MCFixupKindInfo &FKI = Asm.getBackend().getFixupKindInfo(
-    (MCFixupKind) Kind);
+  MCFixupKindInfo FKI = Asm.getBackend().getFixupKindInfo((MCFixupKind)Kind);
 
   return FKI.Flags & MCFixupKindInfo::FKF_IsPCRel;
 }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 1517fb22a19f5..c1cfa9520508b 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -44,7 +44,7 @@ class AArch64AsmBackend : public MCAsmBackend {
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override {
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
     const static MCFixupKindInfo Infos[AArch64::NumTargetFixupKinds] = {
         // This table *must* be in the order that the fixup_* kinds are defined
         // in AArch64FixupKinds.h.
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
index 1e01fe25ede80..3009bd2ca2758 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
@@ -86,7 +86,7 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
                                           A64E->getSpecifierName() +
                                           " unsupported on COFF targets");
     } else {
-      const MCFixupKindInfo &Info = MAB.getFixupKindInfo(Fixup.getKind());
+      MCFixupKindInfo Info = MAB.getFixupKindInfo(Fixup.getKind());
       Ctx.reportError(Fixup.getLoc(), Twine("relocation type ") + Info.Name +
                                           " unsupported on COFF targets");
     }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
index 7873d35855b64..19e394e5bc38c 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
@@ -50,7 +50,7 @@ class AMDGPUAsmBackend : public MCAsmBackend {
                     const MCSubtargetInfo *STI) const override;
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 };
 
 } //End anonymous namespace
@@ -172,8 +172,7 @@ AMDGPUAsmBackend::getFixupKind(StringRef Name) const {
   return std::nullopt;
 }
 
-const MCFixupKindInfo &AMDGPUAsmBackend::getFixupKindInfo(
-                                                       MCFixupKind Kind) const {
+MCFixupKindInfo AMDGPUAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[AMDGPU::NumTargetFixupKinds] = {
     // name                   offset bits  flags
     { "fixup_si_sopp_br",     0,     16,   MCFixupKindInfo::FKF_IsPCRel },
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index 5b1016e7c5c91..266ae1ff84e64 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -65,7 +65,7 @@ ARMAsmBackendELF::getFixupKind(StringRef Name) const {
   return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
 }
 
-const MCFixupKindInfo &ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo InfosLE[ARM::NumTargetFixupKinds] = {
       // This table *must* be in the order that the fixup_* kinds are defined in
       // ARMFixupKinds.h.
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
index 57588d989d545..bcf044b340da7 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
@@ -30,7 +30,7 @@ class ARMAsmBackend : public MCAsmBackend {
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup,
                              const MCValue &Target,
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
index 6340e589e31f1..fa3f1fe6081ee 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
@@ -382,7 +382,7 @@ void AVRAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
 
   // The number of bits in the fixup mask
-  auto NumBits = Info.TargetSize + Info.TargetOffset;
+  unsigned NumBits = Info.TargetSize + Info.TargetOffset;
   auto NumBytes = (NumBits / 8) + ((NumBits % 8) == 0 ? 0 : 1);
 
   // Shift the value into position.
@@ -414,7 +414,7 @@ std::optional<MCFixupKind> AVRAsmBackend::getFixupKind(StringRef Name) const {
   return std::nullopt;
 }
 
-MCFixupKindInfo const &AVRAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo AVRAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // NOTE: Many AVR fixups work on sets of non-contignous bits. We work around
   // this by saying that the fixup is the size of the entire instruction.
   const static MCFixupKindInfo Infos[AVR::NumTargetFixupKinds] = {
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
index c96f3e541eb6f..c98828bdc50b4 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
@@ -43,8 +43,7 @@ class AVRAsmBackend : public MCAsmBackend {
                   const MCSubtargetInfo *STI) const override;
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
-
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
index b15e5be1a37a4..d2610e8a74457 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
@@ -35,7 +35,7 @@ class BPFAsmBackend : public MCAsmBackend {
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override;
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
@@ -43,8 +43,7 @@ class BPFAsmBackend : public MCAsmBackend {
 
 } // end anonymous namespace
 
-const MCFixupKindInfo &
-BPFAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo BPFAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[BPF::NumTargetFixupKinds] = {
     { "FK_BPF_PCRel_4",  0, 32, MCFixupKindInfo::FKF_IsPCRel },
   };
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
index 1c305ac8bebfd..62763b3a57b3d 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
@@ -25,8 +25,7 @@ CSKYAsmBackend::createObjectTargetWriter() const {
   return createCSKYELFObjectWriter();
 }
 
-const MCFixupKindInfo &
-CSKYAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo CSKYAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 
   static llvm::DenseMap<unsigned, MCFixupKindInfo> Infos = {
       {CSKY::Fixups::fixup_csky_addr32, {"fixup_csky_addr32", 0, 32, 0}},
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
index 3ce2f37212dca..b0421d1017499 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
@@ -28,7 +28,7 @@ class CSKYAsmBackend : public MCAsmBackend {
                   uint64_t Value, bool IsResolved,
                   const MCSubtargetInfo *STI) const override;
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   bool fixupNeedsRelaxation(const MCFixup &Fixup,
                             uint64_t Value) const override;
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index ae1dac57d9294..9f4a2fd539fa9 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -83,8 +83,7 @@ class HexagonAsmBackend : public MCAsmBackend {
     return Result;
   }
 
-
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override {
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
     const static MCFixupKindInfo Infos[Hexagon::NumTargetFixupKinds] = {
       // This table *must* be in same the order of fixup_* kinds in
       // HexagonFixupKinds.h.
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
index 0401489e0041e..0a5f37f31296c 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
@@ -55,8 +55,7 @@ class LanaiAsmBackend : public MCAsmBackend {
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override;
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
-
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
@@ -116,8 +115,7 @@ LanaiAsmBackend::createObjectTargetWriter() const {
   return createLanaiELFObjectWriter(MCELFObjectTargetWriter::getOSABI(OSType));
 }
 
-const MCFixupKindInfo &
-LanaiAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo LanaiAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   static const MCFixupKindInfo Infos[Lanai::NumTargetFixupKinds] = {
       // This table *must* be in same the order of fixup_* kinds in
       // LanaiFixupKinds.h.
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index bab85235c622e..eaf56f6698f4a 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -44,8 +44,7 @@ LoongArchAsmBackend::getFixupKind(StringRef Name) const {
   return std::nullopt;
 }
 
-const MCFixupKindInfo &
-LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[] = {
       // This table *must* be in the order that the fixup_* kinds are defined in
       // LoongArchFixupKinds.h.
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
index e7c09a0e97c6e..ec78f345bb390 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
@@ -62,7 +62,7 @@ class LoongArchAsmBackend : public MCAsmBackend {
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   void relaxInstruction(MCInst &Inst,
                         const MCSubtargetInfo &STI) const override {}
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
index b3a6c717f7368..cf237a9ba59c2 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
@@ -46,8 +46,7 @@ class MSP430AsmBackend : public MCAsmBackend {
     return createMSP430ELFObjectWriter(OSABI);
   }
 
-
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override {
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
     const static MCFixupKindInfo Infos[MSP430::NumTargetFixupKinds] = {
       // This table must be in the same order of enum in MSP430FixupKinds.h.
       //
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index 8bd2ad8c86a73..4f7b2c6d25252 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -353,8 +353,7 @@ std::optional<MCFixupKind> MipsAsmBackend::getFixupKind(StringRef Name) const {
       .Default(MCAsmBackend::getFixupKind(Name));
 }
 
-const MCFixupKindInfo &MipsAsmBackend::
-getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo LittleEndianInfos[] = {
       // This table *must* be in same the order of fixup_* kinds in
       // MipsFixupKinds.h.
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
index 1e8504aaf2aa2..971611b6d8db6 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
@@ -45,8 +45,7 @@ class MipsAsmBackend : public MCAsmBackend {
                   const MCSubtargetInfo *STI) const override;
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
-
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 61c274638f347..136bb4817706f 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -90,7 +90,7 @@ class PPCAsmBackend : public MCAsmBackend {
                                          : llvm::endianness::big),
         TT(TT) {}
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override {
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
     const static MCFixupKindInfo InfosBE[PPC::NumTargetFixupKinds] = {
       // name                    offset  bits  flags
       { "fixup_ppc_br24",        6,      24,   MCFixupKindInfo::FKF_IsPCRel },
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index 48b1145f58b41..d8cfeb07e52b6 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -61,8 +61,7 @@ std::optional<MCFixupKind> RISCVAsmBackend::getFixupKind(StringRef Name) const {
   return std::nullopt;
 }
 
-const MCFixupKindInfo &
-RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[] = {
       // This table *must* be in the order that the fixup_* kinds are defined in
       // RISCVFixupKinds.h.
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
index 23c8c2ce0b314..5db55ad0b8567 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
@@ -70,7 +70,7 @@ class RISCVAsmBackend : public MCAsmBackend {
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   bool mayNeedRelaxation(const MCInst &Inst,
                          const MCSubtargetInfo &STI) const override;
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index d8d673b08f741..cc8b86e6135b5 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -160,7 +160,7 @@ namespace {
       return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
     }
 
-    const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override {
+    MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
       const static MCFixupKindInfo InfosBE[Sparc::NumTargetFixupKinds] = {
         // name                    offset bits  flags
         { "fixup_sparc_call30",     2,     30,  MCFixupKindInfo::FKF_IsPCRel },
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index 22ba533fa5845..58bb7baea1d47 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -111,7 +111,7 @@ class SystemZMCAsmBackend : public MCAsmBackend {
 
   // Override MCAsmBackend
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
                   uint64_t Value, bool IsResolved,
@@ -138,8 +138,7 @@ SystemZMCAsmBackend::getFixupKind(StringRef Name) const {
   return std::nullopt;
 }
 
-const MCFixupKindInfo &
-SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from .reloc directive are like R_390_NONE. They
   // do not require any extra processing.
   if (mc::isRelocation(Kind))
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
index aa4b1207d56c0..6be168b2c52a6 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
@@ -100,8 +100,7 @@ class VEAsmBackend : public MCAsmBackend {
   VEAsmBackend(const Target &T)
       : MCAsmBackend(llvm::endianness::little), TheTarget(T) {}
 
-
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override {
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
     const static MCFixupKindInfo Infos[VE::NumTargetFixupKinds] = {
         // name, offset, bits, flags
         {"fixup_ve_reflong", 0, 32, 0},
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
index 36b049deec20b..91a1db80deb3c 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
@@ -36,8 +36,7 @@ class WebAssemblyAsmBackend final : public MCAsmBackend {
       : MCAsmBackend(llvm::endianness::little), Is64Bit(Is64Bit),
         IsEmscripten(IsEmscripten) {}
 
-
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
@@ -51,7 +50,7 @@ class WebAssemblyAsmBackend final : public MCAsmBackend {
                     const MCSubtargetInfo *STI) const override;
 };
 
-const MCFixupKindInfo &
+MCFixupKindInfo
 WebAssemblyAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[WebAssembly::NumTargetFixupKinds] = {
       // This table *must* be in the order that the fixup_* kinds are defined in
@@ -87,7 +86,7 @@ void WebAssemblyAsmBackend::applyFixup(const MCAssembler &Asm,
                                        MutableArrayRef<char> Data,
                                        uint64_t Value, bool IsPCRel,
                                        const MCSubtargetInfo *STI) const {
-  const MCFixupKindInfo &Info = getFixupKindInfo(Fixup.getKind());
+  MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
   assert(Info.Flags == 0 && "WebAssembly does not use MCFixupKindInfo flags");
 
   unsigned NumBytes = alignTo(Info.TargetSize, 8) / 8;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 6e02c2e4c5433..3cee53a886ebf 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -167,7 +167,7 @@ class X86AsmBackend : public MCAsmBackend {
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
@@ -620,7 +620,7 @@ std::optional<MCFixupKind> X86AsmBackend::getFixupKind(StringRef Name) const {
   return MCAsmBackend::getFixupKind(Name);
 }
 
-const MCFixupKindInfo &X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[X86::NumTargetFixupKinds] = {
       // clang-format off
       {"reloc_riprel_4byte", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
index ba2744d5facb5..53df3c7c7faae 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
@@ -33,7 +33,7 @@ class XtensaMCAsmBackend : public MCAsmBackend {
       : MCAsmBackend(llvm::endianness::little), OSABI(osABI),
         IsLittleEndian(isLE) {}
 
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
                   uint64_t Value, bool IsResolved,
@@ -51,8 +51,7 @@ class XtensaMCAsmBackend : public MCAsmBackend {
 };
 } // namespace llvm
 
-const MCFixupKindInfo &
-XtensaMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+MCFixupKindInfo XtensaMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[Xtensa::NumTargetFixupKinds] = {
       // name                     offset bits  flags
       {"fixup_xtensa_branch_6", 0, 16, MCFixupKindInfo::FKF_IsPCRel},

From 3bcb724903d82414a0952fe0349d5181ff2f0f26 Mon Sep 17 00:00:00 2001
From: Rafael Auler <rafaelauler@meta.com>
Date: Fri, 18 Apr 2025 21:02:09 -0700
Subject: [PATCH 526/710] [BOLT] Add --custom-allocation-vma flag (#136385)

Add an advanced-user flag so we are able to rewrite binaries when we fail
to identify a suitable location to put new code. User then can supply a
custom location via --custom-allocation-vma. This happens more obviously if the
binary has segments mapped to very high addresses.
---
 bolt/lib/Rewrite/RewriteInstance.cpp | 25 +++++++++++++++
 bolt/test/X86/high-segments.s        | 46 ++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 bolt/test/X86/high-segments.s

diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 45bc995ce906f..69fb736d7bde0 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -237,6 +237,12 @@ UseGnuStack("use-gnu-stack",
   cl::ZeroOrMore,
   cl::cat(BoltCategory));
 
+static cl::opt<uint64_t> CustomAllocationVMA(
+    "custom-allocation-vma",
+    cl::desc("use a custom address at which new code will be put, "
+             "bypassing BOLT's logic to detect where to put code"),
+    cl::Hidden, cl::cat(BoltCategory));
+
 static cl::opt<bool>
 SequentialDisassembly("sequential-disassembly",
   cl::desc("performs disassembly sequentially"),
@@ -592,6 +598,25 @@ Error RewriteInstance::discoverStorage() {
 
   FirstNonAllocatableOffset = NextAvailableOffset;
 
+  if (opts::CustomAllocationVMA) {
+    // If user specified a custom address where we should start writing new
+    // data, honor that.
+    NextAvailableAddress = opts::CustomAllocationVMA;
+    // Sanity check the user-supplied address and emit warnings if something
+    // seems off.
+    for (const ELF64LE::Phdr &Phdr : PHs) {
+      switch (Phdr.p_type) {
+      case ELF::PT_LOAD:
+        if (NextAvailableAddress >= Phdr.p_vaddr &&
+            NextAvailableAddress < Phdr.p_vaddr + Phdr.p_memsz) {
+          BC->errs() << "BOLT-WARNING: user-supplied allocation vma 0x"
+                     << Twine::utohexstr(NextAvailableAddress)
+                     << " conflicts with ELF segment at 0x"
+                     << Twine::utohexstr(Phdr.p_vaddr) << "\n";
+        }
+      }
+    }
+  }
   NextAvailableAddress = alignTo(NextAvailableAddress, BC->PageAlign);
   NextAvailableOffset = alignTo(NextAvailableOffset, BC->PageAlign);
 
diff --git a/bolt/test/X86/high-segments.s b/bolt/test/X86/high-segments.s
new file mode 100644
index 0000000000000..dfddf18004c2a
--- /dev/null
+++ b/bolt/test/X86/high-segments.s
@@ -0,0 +1,46 @@
+// Check that we are able to rewrite binaries when we fail to identify a
+// suitable location to put new code and user supplies a custom one via
+// --custom-allocation-vma. This happens more obviously if the binary has
+// segments mapped to very high addresses.
+
+// In this example, my.reserved.section is mapped to a segment to be loaded
+// at address 0x10000000000, while regular text should be at 0x200000. We
+// pick a vma in the middle at 0x700000 to carve space for BOLT to put data,
+// since BOLT's usual route of allocating after the last segment will put
+// code far away and that will blow up relocations from main.
+
+// RUN: split-file %s %t
+// RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %t/main.s -o %t.o
+// RUN: %clang %cflags -no-pie %t.o -o %t.exe -Wl,-T %t/main.ls
+// RUN: llvm-bolt %t.exe -o %t.bolt --custom-allocation-vma=0x700000
+
+//--- main.s
+  .type            reserved_space,@object
+  .section        .my.reserved.section,"awx",@nobits
+  .globl           reserved_space
+  .p2align         4, 0x0
+reserved_space:
+  .zero  0x80000000
+  .size   reserved_space, 0x80000000
+
+	.text
+  .globl main
+  .globl _start
+  .type main, %function
+_start:
+main:
+	.cfi_startproc
+  nop
+  nop
+  nop
+  retq
+	.cfi_endproc
+.size main, .-main
+
+//--- main.ls
+SECTIONS
+{
+    .my.reserved.section 1<<40 : {
+      *(.my.reserved.section);
+    }
+} INSERT BEFORE .comment;

From c3000333cda22ec7d5ae2597c6aadcdc5d6ea7f8 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 18 Apr 2025 22:14:03 -0700
Subject: [PATCH 527/710] Revert "[Reland][Clang][CodeGen][UBSan] Add more
 precise attributes to recoverable ubsan handlers" (#136402)

Reverts llvm/llvm-project#135135

Breaks several bots, details in #135135.
---
 clang/lib/CodeGen/CGExpr.cpp                  |  31 +--
 clang/lib/CodeGen/CodeGenFunction.h           |   4 +-
 .../AArch64/ubsan-handler-pass-by-ref.c       |  22 --
 clang/test/CodeGen/allow-ubsan-check.c        |  28 +--
 clang/test/CodeGen/attr-counted-by.c          | 232 +++++++++---------
 clang/test/CodeGen/ubsan-attr.cpp             | 100 --------
 6 files changed, 137 insertions(+), 280 deletions(-)
 delete mode 100644 clang/test/CodeGen/AArch64/ubsan-handler-pass-by-ref.c
 delete mode 100644 clang/test/CodeGen/ubsan-attr.cpp

diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 1efc626a8f5a6..abb88477062fc 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3486,8 +3486,7 @@ llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
   return GV;
 }
 
-llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V,
-                                             bool &MayReadFromPtrToInt) {
+llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
   llvm::Type *TargetTy = IntPtrTy;
 
   if (V->getType() == TargetTy)
@@ -3513,7 +3512,6 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V,
     Builder.CreateStore(V, Ptr);
     V = Ptr.getPointer();
   }
-  MayReadFromPtrToInt = true;
   return Builder.CreatePtrToInt(V, TargetTy);
 }
 
@@ -3619,8 +3617,7 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF,
                                  ArrayRef<llvm::Value *> FnArgs,
                                  SanitizerHandler CheckHandler,
                                  CheckRecoverableKind RecoverKind, bool IsFatal,
-                                 llvm::BasicBlock *ContBB, bool NoMerge,
-                                 bool MayReadFromPtrToInt) {
+                                 llvm::BasicBlock *ContBB, bool NoMerge) {
   assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
   std::optional<ApplyDebugLocation> DL;
   if (!CGF.Builder.getCurrentDebugLocation()) {
@@ -3648,20 +3645,6 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF,
         .addAttribute(llvm::Attribute::NoUnwind);
   }
   B.addUWTableAttr(llvm::UWTableKind::Default);
-  // Add more precise attributes to recoverable ubsan handlers for better
-  // optimizations.
-  if (CGF.CGM.getCodeGenOpts().OptimizationLevel > 0 && MayReturn) {
-    // __ubsan_handle_dynamic_type_cache_miss reads the vtable, which is also
-    // accessible by the current module.
-    if (CheckHandler != SanitizerHandler::DynamicTypeCacheMiss) {
-      llvm::MemoryEffects ME =
-          llvm::MemoryEffects::argMemOnly(llvm::ModRefInfo::Ref) |
-          llvm::MemoryEffects::inaccessibleMemOnly();
-      if (MayReadFromPtrToInt)
-        ME |= llvm::MemoryEffects::readOnly();
-      B.addMemoryAttr(ME);
-    }
-  }
 
   llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
       FnType, FnName,
@@ -3758,7 +3741,6 @@ void CodeGenFunction::EmitCheck(
   // representing operand values.
   SmallVector<llvm::Value *, 4> Args;
   SmallVector<llvm::Type *, 4> ArgTypes;
-  bool MayReadFromPtrToInt = false;
   if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
     Args.reserve(DynamicArgs.size() + 1);
     ArgTypes.reserve(DynamicArgs.size() + 1);
@@ -3778,7 +3760,7 @@ void CodeGenFunction::EmitCheck(
     }
 
     for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
-      Args.push_back(EmitCheckValue(DynamicArgs[i], MayReadFromPtrToInt));
+      Args.push_back(EmitCheckValue(DynamicArgs[i]));
       ArgTypes.push_back(IntPtrTy);
     }
   }
@@ -3790,8 +3772,7 @@ void CodeGenFunction::EmitCheck(
     // Simple case: we need to generate a single handler call, either
     // fatal, or non-fatal.
     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,
-                         (FatalCond != nullptr), Cont, NoMerge,
-                         MayReadFromPtrToInt);
+                         (FatalCond != nullptr), Cont, NoMerge);
   } else {
     // Emit two handler calls: first one for set of unrecoverable checks,
     // another one for recoverable.
@@ -3801,10 +3782,10 @@ void CodeGenFunction::EmitCheck(
     Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
     EmitBlock(FatalHandlerBB);
     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,
-                         NonFatalHandlerBB, NoMerge, MayReadFromPtrToInt);
+                         NonFatalHandlerBB, NoMerge);
     EmitBlock(NonFatalHandlerBB);
     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,
-                         Cont, NoMerge, MayReadFromPtrToInt);
+                         Cont, NoMerge);
   }
 
   EmitBlock(Cont);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 9254c7077237f..4c5e8a8a44926 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5244,9 +5244,7 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// Convert a value into a format suitable for passing to a runtime
   /// sanitizer handler.
-  /// If the check value is a pointer or passed by reference, set \p
-  /// MayReadFromPtrToInt to true.
-  llvm::Value *EmitCheckValue(llvm::Value *V, bool &MayReadFromPtrToInt);
+  llvm::Value *EmitCheckValue(llvm::Value *V);
 
   /// Emit a description of a source location in a format suitable for
   /// passing to a runtime sanitizer handler.
diff --git a/clang/test/CodeGen/AArch64/ubsan-handler-pass-by-ref.c b/clang/test/CodeGen/AArch64/ubsan-handler-pass-by-ref.c
deleted file mode 100644
index a8ff941a124a4..0000000000000
--- a/clang/test/CodeGen/AArch64/ubsan-handler-pass-by-ref.c
+++ /dev/null
@@ -1,22 +0,0 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
-// RUN: %clang_cc1 -triple armv7-linux-androideabi24 -emit-llvm -fsanitize=signed-integer-overflow -O3 %s -o - -fsanitize-recover=signed-integer-overflow -w | FileCheck %s --check-prefixes=RECOVER
-// REQUIRES: aarch64-registered-target
-
-#define LLONG_MIN  (-__LONG_LONG_MAX__-1LL)
-
-// RECOVER-LABEL: define dso_local noundef i32 @main(
-// RECOVER-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// RECOVER-NEXT:  [[ENTRY:.*:]]
-// RECOVER-NEXT:    [[TMP:%.*]] = alloca i64, align 8
-// RECOVER-NEXT:    store i64 -9223372036854775808, ptr [[TMP]], align 8, !nosanitize [[META3:![0-9]+]]
-// RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[TMP]] to i32, !nosanitize [[META3]]
-// RECOVER-NEXT:    call void @__ubsan_handle_negate_overflow(ptr nonnull @[[GLOB1:[0-9]+]], i32 [[TMP0]]) #[[ATTR2:[0-9]+]], !nosanitize [[META3]]
-// RECOVER-NEXT:    ret i32 0
-//
-int main() {
-  __builtin_llabs(LLONG_MIN);
-  return 0;
-}
-//.
-// RECOVER: [[META3]] = !{}
-//.
diff --git a/clang/test/CodeGen/allow-ubsan-check.c b/clang/test/CodeGen/allow-ubsan-check.c
index 6358425aa40be..e225fb63f08eb 100644
--- a/clang/test/CodeGen/allow-ubsan-check.c
+++ b/clang/test/CodeGen/allow-ubsan-check.c
@@ -29,7 +29,7 @@
 // CHECK:       [[HANDLER_DIVREM_OVERFLOW]]:
 // CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// CHECK-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR8:[0-9]+]], !nosanitize [[META2]]
+// CHECK-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR6:[0-9]+]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 // CHECK:       [[CONT]]:
 // CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[X]], [[Y]]
@@ -75,7 +75,7 @@
 // REC:       [[HANDLER_DIVREM_OVERFLOW]]:
 // REC-NEXT:    [[TMP10:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // REC-NEXT:    [[TMP11:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// REC-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR8:[0-9]+]], !nosanitize [[META2]]
+// REC-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP10]], i64 [[TMP11]]) #[[ATTR6:[0-9]+]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[CONT]], !nosanitize [[META2]]
 // REC:       [[CONT]]:
 // REC-NEXT:    [[DIV:%.*]] = sdiv i32 [[X]], [[Y]]
@@ -86,7 +86,7 @@ int div(int x, int y) {
 }
 
 // CHECK-LABEL: define dso_local i32 @null(
-// CHECK-SAME: ptr noundef readonly captures(address_is_null) [[X:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: ptr noundef readonly captures(address_is_null) [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = icmp eq ptr [[X]], null, !nosanitize [[META2]]
 //
@@ -95,7 +95,7 @@ int div(int x, int y) {
 // CHECK-NEXT:    [[DOTNOT1:%.*]] = and i1 [[TMP0]], [[TMP1]]
 // CHECK-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_TYPE_MISMATCH:.*]], label %[[CONT:.*]], !prof [[PROF4:![0-9]+]], !nosanitize [[META2]]
 // CHECK:       [[HANDLER_TYPE_MISMATCH]]:
-// CHECK-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
+// CHECK-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR6]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 // CHECK:       [[CONT]]:
 // CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA5:![0-9]+]]
@@ -116,14 +116,14 @@ int div(int x, int y) {
 // TR-NEXT:    ret i32 [[TMP2]]
 //
 // REC-LABEL: define dso_local i32 @null(
-// REC-SAME: ptr noundef readonly captures(address_is_null) [[X:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// REC-SAME: ptr noundef readonly captures(address_is_null) [[X:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // REC-NEXT:  [[ENTRY:.*:]]
 // REC-NEXT:    [[TMP0:%.*]] = icmp eq ptr [[X]], null, !nosanitize [[META2]]
 // REC-NEXT:    [[TMP1:%.*]] = tail call i1 @llvm.allow.ubsan.check(i8 29), !nosanitize [[META2]]
 // REC-NEXT:    [[DOTNOT1:%.*]] = and i1 [[TMP0]], [[TMP1]]
 // REC-NEXT:    br i1 [[DOTNOT1]], label %[[HANDLER_TYPE_MISMATCH:.*]], label %[[CONT:.*]], !prof [[PROF4:![0-9]+]], !nosanitize [[META2]]
 // REC:       [[HANDLER_TYPE_MISMATCH]]:
-// REC-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
+// REC-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1(ptr nonnull @[[GLOB2:[0-9]+]], i64 0) #[[ATTR6]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[CONT]], !nosanitize [[META2]]
 // REC:       [[CONT]]:
 // REC-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X]], align 4, !tbaa [[TBAA5:![0-9]+]]
@@ -146,7 +146,7 @@ int null(int* x) {
 // CHECK:       [[HANDLER_ADD_OVERFLOW]]:
 // CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// CHECK-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP3]], i64 [[TMP4]]) #[[ATTR8]], !nosanitize [[META2]]
+// CHECK-NEXT:    tail call void @__ubsan_handle_add_overflow_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP3]], i64 [[TMP4]]) #[[ATTR6]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 // CHECK:       [[CONT]]:
 // CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
@@ -178,7 +178,7 @@ int null(int* x) {
 // REC:       [[HANDLER_ADD_OVERFLOW]]:
 // REC-NEXT:    [[TMP3:%.*]] = zext i32 [[X]] to i64, !nosanitize [[META2]]
 // REC-NEXT:    [[TMP4:%.*]] = zext i32 [[Y]] to i64, !nosanitize [[META2]]
-// REC-NEXT:    tail call void @__ubsan_handle_add_overflow(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP3]], i64 [[TMP4]]) #[[ATTR8]], !nosanitize [[META2]]
+// REC-NEXT:    tail call void @__ubsan_handle_add_overflow(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[TMP3]], i64 [[TMP4]]) #[[ATTR6]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[CONT]], !nosanitize [[META2]]
 // REC:       [[CONT]]:
 // REC-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0, !nosanitize [[META2]]
@@ -191,11 +191,11 @@ int overflow(int x, int y) {
 void use(double*);
 
 // CHECK-LABEL: define dso_local double @lbounds(
-// CHECK-SAME: i32 noundef [[B:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
+// CHECK-SAME: i32 noundef [[B:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[B]] to i64
 // CHECK-NEXT:    [[VLA:%.*]] = alloca double, i64 [[TMP0]], align 16
-// CHECK-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR9:[0-9]+]]
+// CHECK-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR7:[0-9]+]]
 // CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[TMP0]], [[IDXPROM]]
 //
@@ -208,7 +208,7 @@ void use(double*);
 // CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA9:![0-9]+]]
 // CHECK-NEXT:    ret double [[TMP5]]
 // CHECK:       [[TRAP]]:
-// CHECK-NEXT:    call void @__ubsan_handle_local_out_of_bounds_abort() #[[ATTR8]], !nosanitize [[META2]]
+// CHECK-NEXT:    call void @__ubsan_handle_local_out_of_bounds_abort() #[[ATTR6]], !nosanitize [[META2]]
 // CHECK-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // TR-LABEL: define dso_local double @lbounds(
@@ -231,11 +231,11 @@ void use(double*);
 // TR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // REC-LABEL: define dso_local double @lbounds(
-// REC-SAME: i32 noundef [[B:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
+// REC-SAME: i32 noundef [[B:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // REC-NEXT:  [[ENTRY:.*:]]
 // REC-NEXT:    [[TMP0:%.*]] = zext i32 [[B]] to i64
 // REC-NEXT:    [[VLA:%.*]] = alloca double, i64 [[TMP0]], align 16
-// REC-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR7:[0-9]+]]
+// REC-NEXT:    call void @use(ptr noundef nonnull [[VLA]]) #[[ATTR5:[0-9]+]]
 // REC-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // REC-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[TMP0]], [[IDXPROM]]
 // REC-NEXT:    [[TMP2:%.*]] = call i1 @llvm.allow.ubsan.check(i8 71), !nosanitize [[META2]]
@@ -246,7 +246,7 @@ void use(double*);
 // REC-NEXT:    [[TMP5:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA9:![0-9]+]]
 // REC-NEXT:    ret double [[TMP5]]
 // REC:       [[TRAP]]:
-// REC-NEXT:    call void @__ubsan_handle_local_out_of_bounds() #[[ATTR8]], !nosanitize [[META2]]
+// REC-NEXT:    call void @__ubsan_handle_local_out_of_bounds() #[[ATTR6]], !nosanitize [[META2]]
 // REC-NEXT:    br label %[[BB4]], !nosanitize [[META2]]
 //
 double lbounds(int b, int i) {
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
index 8e062629464dd..dfdf06587f0e2 100644
--- a/clang/test/CodeGen/attr-counted-by.c
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -68,7 +68,7 @@ struct anon_struct {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12:[0-9]+]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8:[0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -116,7 +116,7 @@ void test1(struct annotated *p, int index, int val) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB3:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -159,7 +159,7 @@ void test2(struct annotated *p, size_t index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934592, 8589934589) i64 @test2_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
@@ -181,7 +181,7 @@ void test2(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test2_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -203,7 +203,7 @@ size_t test2_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -242,7 +242,7 @@ void test3(struct annotated *p, size_t index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test3_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -252,7 +252,7 @@ void test3(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test3_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -275,7 +275,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 2
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 3) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 3) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl i32 [[DOTCOUNTED_BY_LOAD]], 2
@@ -283,7 +283,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT12:%.*]], label [[HANDLER_OUT_OF_BOUNDS8:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds8:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont12:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[DOTCOUNTED_BY_LOAD]], 2
@@ -295,7 +295,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT81:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 3
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT81]], label [[HANDLER_OUT_OF_BOUNDS18:%.*]], label [[CONT19:%.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds18:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 4) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 4) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont19:
 // SANITIZE-WITH-ATTR-NEXT:    [[ADD:%.*]] = add nsw i32 [[INDEX]], 1
@@ -303,7 +303,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[IDXPROM31]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP5]], label [[CONT38:%.*]], label [[HANDLER_OUT_OF_BOUNDS34:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds34:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM31]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM31]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont38:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[DOTCOUNTED_BY_LOAD]], 3
@@ -318,7 +318,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp ugt i64 [[IDXPROM42]], [[TMP8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS45:%.*]], label [[CONT46:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds45:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM42]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM42]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont46:
 // SANITIZE-WITH-ATTR-NEXT:    [[ADD59:%.*]] = add nsw i32 [[INDEX]], 2
@@ -326,7 +326,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = icmp ult i64 [[IDXPROM60]], [[TMP8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP9]], label [[CONT67:%.*]], label [[HANDLER_OUT_OF_BOUNDS63:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds63:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[IDXPROM60]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[IDXPROM60]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont67:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX65:%.*]] = getelementptr inbounds nuw [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM60]]
@@ -422,7 +422,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 8589934589) i64 @test4_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
@@ -431,7 +431,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp ugt i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], label [[CONT1:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = sext i32 [[DOTCOUNTED_BY_LOAD]] to i64
@@ -456,7 +456,7 @@ void test4(struct annotated *p, int index, int fam_idx) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP4]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test4_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -478,7 +478,7 @@ size_t test4_bdos(struct annotated *p, int index) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[DOTCOUNTED_BY_LOAD]], [[IDXPROM]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB12:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB12:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
@@ -518,7 +518,7 @@ void test5(struct anon_struct *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test5_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -528,7 +528,7 @@ void test5(struct anon_struct *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test5_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -550,7 +550,7 @@ size_t test5_bdos(struct anon_struct *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[COUNTED_BY_LOAD]], [[IDXPROM]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
@@ -598,7 +598,7 @@ void test6(struct anon_struct *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, -9223372036854775808) i64 @test6_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i64, ptr [[COUNTED_BY_GEP]], align 4
@@ -616,7 +616,7 @@ void test6(struct anon_struct *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test6_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -639,7 +639,7 @@ size_t test6_bdos(struct anon_struct *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
 // SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
@@ -679,7 +679,7 @@ void test7(struct union_of_fams *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test7_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -689,7 +689,7 @@ void test7(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test7_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -712,7 +712,7 @@ size_t test7_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT14:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont14:
 // SANITIZE-WITH-ATTR-NEXT:    [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9
@@ -754,7 +754,7 @@ void test8(struct union_of_fams *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 256) i64 @test8_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4
@@ -770,7 +770,7 @@ void test8(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[COUNT]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test8_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -793,7 +793,7 @@ size_t test8_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont7:
 // SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -833,7 +833,7 @@ void test9(struct union_of_fams *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test9_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -843,7 +843,7 @@ void test9(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test9_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -866,7 +866,7 @@ size_t test9_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT14:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB19:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB19:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont14:
 // SANITIZE-WITH-ATTR-NEXT:    [[BYTES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -912,7 +912,7 @@ void test10(struct union_of_fams *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 2147483648) i64 @test10_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -930,7 +930,7 @@ void test10(struct union_of_fams *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test10_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -953,7 +953,7 @@ size_t test10_bdos(struct union_of_fams *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT6:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont6:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -1003,7 +1003,7 @@ void test11(struct annotated *p, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 -8589934584, 8589934597) i64 @test11_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_LOAD:%.*]] = load i32, ptr [[COUNTED_BY_GEP]], align 4
@@ -1027,7 +1027,7 @@ void test11(struct annotated *p, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test11_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1056,16 +1056,16 @@ struct hang {
 int test12_a, test12_b;
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test12(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR13:[0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR9:[0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT10:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB22:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB22:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
@@ -1075,10 +1075,10 @@ int test12_a, test12_b;
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 0
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds4:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB24:[0-9]+]], i64 0) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB24:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.type_mismatch6:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB25:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB25:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test12(
@@ -1098,16 +1098,16 @@ int test12_a, test12_b;
 // NO-SANITIZE-WITH-ATTR-NEXT:    br label [[FOR_COND]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test12(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR11:[0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR7:[0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META9:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP1]]) #[[ATTR12:[0-9]+]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP1]]) #[[ATTR8:[0-9]+]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [6 x i32], ptr [[BAZ]], i64 0, i64 [[TMP1]]
@@ -1117,10 +1117,10 @@ int test12_a, test12_b;
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 0
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS4:%.*]], label [[HANDLER_TYPE_MISMATCH6:%.*]], !prof [[PROF10:![0-9]+]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds4:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 0) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 0) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.type_mismatch6:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_type_mismatch_v1_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @test12_foo, i64 4) to i64)) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test12(
@@ -1159,7 +1159,7 @@ struct test13_bar {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i32 @test13(
-// SANITIZE-WITH-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr @test13_f, align 8, !tbaa [[TBAA11:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
@@ -1168,7 +1168,7 @@ struct test13_bar {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[INDEX]], [[TMP1]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB28:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB28:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
@@ -1186,7 +1186,7 @@ struct test13_bar {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 0
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i32 @test13(
-// SANITIZE-WITHOUT-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR7:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr @test13_f, align 8, !tbaa [[TBAA11:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
@@ -1195,7 +1195,7 @@ struct test13_bar {
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[INDEX]], [[TMP1]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont5:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[REVMAP:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
@@ -1223,13 +1223,13 @@ struct test14_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test14(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB29:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB29:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 undef
@@ -1248,13 +1248,13 @@ struct test14_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test14(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont3:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 undef
@@ -1277,13 +1277,13 @@ int test14(int idx) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test15(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR8]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB31:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB31:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 undef
@@ -1297,13 +1297,13 @@ int test14(int idx) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IDX]], 0
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB11:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont1:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 undef
@@ -1326,14 +1326,14 @@ int test15(int idx) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test19(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 680
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 1
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB32:[0-9]+]], i64 2) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB32:[0-9]+]], i64 2) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
@@ -1344,7 +1344,7 @@ int test15(int idx) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test19(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1359,7 +1359,7 @@ size_t test19(struct annotated *p) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test20(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1369,7 +1369,7 @@ size_t test19(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test20(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1384,7 +1384,7 @@ size_t test20(struct annotated *p) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test21(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1394,7 +1394,7 @@ size_t test20(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test21(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1409,7 +1409,7 @@ size_t test21(struct annotated *p) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test22(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1419,7 +1419,7 @@ size_t test21(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test22(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1434,7 +1434,7 @@ size_t test22(struct annotated *p) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test23(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -1444,7 +1444,7 @@ size_t test22(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test23(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -1464,14 +1464,14 @@ struct tests_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test24(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 40
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 10
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT4:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB33:[0-9]+]], i64 10) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB33:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont4:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
@@ -1486,7 +1486,7 @@ struct tests_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test24(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR8:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[VAR]], i64 84
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA2]]
@@ -1505,14 +1505,14 @@ int test24(int c, struct tests_foo *var) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test25(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA17:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[TMP0]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[DOTCOUNTED_BY_LOAD]], 10
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB34:[0-9]+]], i64 10) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB34:[0-9]+]], i64 10) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
@@ -1528,7 +1528,7 @@ int test24(int c, struct tests_foo *var) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP1]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test25(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR8]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[VAR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !tbaa [[TBAA17:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 44
@@ -1555,7 +1555,7 @@ struct test26_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test26(
-// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[S:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 4
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[C]] to i64
@@ -1564,7 +1564,7 @@ struct test26_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT5:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB35:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB35:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont5:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
@@ -1582,7 +1582,7 @@ struct test26_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP0]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test26(
-// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR8]] {
+// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[C:%.*]], ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[C]] to i64
@@ -1626,7 +1626,7 @@ struct test27_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local ptr @test27(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -1635,7 +1635,7 @@ struct test27_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB37:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB37:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
@@ -1657,7 +1657,7 @@ struct test27_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret ptr [[ARRAYIDX2]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local ptr @test27(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR8]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]], i32 noundef [[J:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ENTRIES:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 24
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
@@ -1689,7 +1689,7 @@ struct test28_foo {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test28(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA21:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA21]]
@@ -1701,7 +1701,7 @@ struct test28_foo {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT17:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB39:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont17:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARR:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
@@ -1722,7 +1722,7 @@ struct test28_foo {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 [[TMP3]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test28(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR8]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[I:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA21:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8, !tbaa [[TBAA21]]
@@ -1757,13 +1757,13 @@ struct annotated_struct_array {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test29(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR7]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 10
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB41:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB41:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
@@ -1775,7 +1775,7 @@ struct annotated_struct_array {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[IDXPROM27]], [[TMP3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT32:%.*]], label [[HANDLER_OUT_OF_BOUNDS28:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds28:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB42:[0-9]+]], i64 [[IDXPROM27]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB42:[0-9]+]], i64 [[IDXPROM27]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont32:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 12
@@ -1802,13 +1802,13 @@ struct annotated_struct_array {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test29(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR7]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[ANN:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 10
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT21:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB13:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont21:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [10 x ptr], ptr [[ANN]], i64 0, i64 [[TMP1]]
@@ -1847,10 +1847,10 @@ struct test30_struct {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR9:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB44:[0-9]+]], i64 [[TMP0]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB44:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test30(
@@ -1863,10 +1863,10 @@ struct test30_struct {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test30(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR9:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[IDX]] to i64, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP0]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 //
 // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test30(
@@ -1891,7 +1891,7 @@ struct test31_struct {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test31(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i32 -1
 //
@@ -1901,7 +1901,7 @@ struct test31_struct {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i32 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test31(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i32 -1
 //
@@ -1927,7 +1927,7 @@ struct annotated_with_array {
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX2]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB46:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB46:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
@@ -1937,7 +1937,7 @@ struct annotated_with_array {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[IDXPROM4]], [[TMP2]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP3]], label [[CONT9:%.*]], label [[HANDLER_OUT_OF_BOUNDS5:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds5:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB48:[0-9]+]], i64 [[IDXPROM4]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB48:[0-9]+]], i64 [[IDXPROM4]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont9:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 344
@@ -1979,7 +1979,7 @@ struct annotated_with_array {
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX2]] to i64, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB17:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB17:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont7:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 344
@@ -2002,13 +2002,13 @@ void test32(struct annotated_with_array *ptr, int idx1, int idx2) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local range(i64 0, 17179869521) i64 @test32_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 43
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB49:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB49:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 336
@@ -2039,13 +2039,13 @@ void test32(struct annotated_with_array *ptr, int idx1, int idx2) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 [[TMP4]]
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test32_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 43
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB18:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont1:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
@@ -2060,7 +2060,7 @@ size_t test32_bdos(struct annotated_with_array *ptr, int index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test33(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR8]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -2070,7 +2070,7 @@ size_t test32_bdos(struct annotated_with_array *ptr, int index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test33(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
@@ -2091,20 +2091,20 @@ struct multi_subscripts {
 };
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test34(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR8]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 42
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB51:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB51:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont1:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[IDX2]], 43
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS2:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds2:
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[IDX2]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB52:[0-9]+]], i64 [[TMP3]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB52:[0-9]+]], i64 [[TMP3]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
@@ -2115,20 +2115,20 @@ struct multi_subscripts {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test34(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[IDX1:%.*]], i32 noundef [[IDX2:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[IDX1]], 42
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP0]], label [[CONT1:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[IDX1]] to i64, !nosanitize [[META9]]
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[TMP1]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB20:[0-9]+]], i64 [[TMP1]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont1:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[IDX2]], 43
 // SANITIZE-WITHOUT-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS2:%.*]], !prof [[PROF8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       handler.out_of_bounds2:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP3:%.*]] = zext i32 [[IDX2]] to i64
-// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB21:[0-9]+]], i64 [[TMP3]]) #[[ATTR12]], !nosanitize [[META9]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB21:[0-9]+]], i64 [[TMP3]]) #[[ATTR8]], !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    unreachable, !nosanitize [[META9]]
 // SANITIZE-WITHOUT-ATTR:       cont3:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
@@ -2151,7 +2151,7 @@ size_t test34(struct multi_subscripts *ptr, int idx1, int idx2) {
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[INDEX]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT3:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       handler.out_of_bounds:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB53:[0-9]+]], i64 [[INDEX]]) #[[ATTR12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB53:[0-9]+]], i64 [[INDEX]]) #[[ATTR8]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR:       cont3:
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
@@ -2188,7 +2188,7 @@ void test35(struct annotated *p, size_t index) {
 }
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local noundef i64 @test35_bdos(
-// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITH-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 0
 //
@@ -2198,7 +2198,7 @@ void test35(struct annotated *p, size_t index) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 0
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local noundef i64 @test35_bdos(
-// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR3]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef readnone captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 0
 //
@@ -2225,7 +2225,7 @@ struct {
 } x;
 
 // SANITIZE-WITH-ATTR-LABEL: define dso_local i64 @test36(
-// SANITIZE-WITH-ATTR-SAME: ) local_unnamed_addr #[[ATTR10:[0-9]+]] {
+// SANITIZE-WITH-ATTR-SAME: ) local_unnamed_addr #[[ATTR6:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
@@ -2235,7 +2235,7 @@ struct {
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret i64 -1
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i64 @test36(
-// SANITIZE-WITHOUT-ATTR-SAME: ) local_unnamed_addr #[[ATTR10:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-SAME: ) local_unnamed_addr #[[ATTR6:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    ret i64 -1
 //
diff --git a/clang/test/CodeGen/ubsan-attr.cpp b/clang/test/CodeGen/ubsan-attr.cpp
deleted file mode 100644
index 48f1b83307660..0000000000000
--- a/clang/test/CodeGen/ubsan-attr.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 5
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -fsanitize=integer-divide-by-zero -O3 %s -o - -fsanitize-recover=integer-divide-by-zero | FileCheck %s --check-prefixes=RECOVER
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -fsanitize=integer-divide-by-zero -O3 %s -o - | FileCheck %s --check-prefixes=ABORT
-
-// RECOVER: Function Attrs: mustprogress nounwind memory(read, argmem: readwrite, inaccessiblemem: readwrite)
-// RECOVER-LABEL: define dso_local noundef range(i32 -32768, 32768) i32 @_Z4testRiRs(
-// RECOVER-SAME: ptr noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr noundef nonnull readonly align 2 captures(none) dereferenceable(2) [[C:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// RECOVER-NEXT:  [[ENTRY:.*:]]
-// RECOVER-NEXT:    [[TMP0:%.*]] = load i16, ptr [[C]], align 2, !tbaa [[TBAA2:![0-9]+]]
-// RECOVER-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4, !tbaa [[TBAA6:![0-9]+]]
-// RECOVER-NEXT:    [[DOTNOT:%.*]] = icmp eq i16 [[TMP0]], 0
-// RECOVER-NEXT:    br i1 [[DOTNOT]], label %[[HANDLER_DIVREM_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META9:![0-9]+]]
-// RECOVER:       [[HANDLER_DIVREM_OVERFLOW]]:
-// RECOVER-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64, !nosanitize [[META9]]
-// RECOVER-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @{{.+}}, i64 [[TMP2]], i64 0) #[[ATTR3:[0-9]+]], !nosanitize [[META9]]
-// RECOVER-NEXT:    br label %[[CONT]], !nosanitize [[META9]]
-// RECOVER:       [[CONT]]:
-// RECOVER-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
-// RECOVER-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP1]], [[CONV]]
-// RECOVER-NEXT:    store i32 [[DIV]], ptr [[A]], align 4, !tbaa [[TBAA6]]
-// RECOVER-NEXT:    ret i32 [[CONV]]
-//
-// ABORT: Function Attrs: mustprogress nounwind
-// ABORT-LABEL: define dso_local noundef range(i32 -32768, 32768) i32 @_Z4testRiRs(
-// ABORT-SAME: ptr noundef nonnull align 4 captures(none) dereferenceable(4) [[A:%.*]], ptr noundef nonnull readonly align 2 captures(none) dereferenceable(2) [[C:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// ABORT-NEXT:  [[ENTRY:.*:]]
-// ABORT-NEXT:    [[TMP0:%.*]] = load i16, ptr [[C]], align 2, !tbaa [[TBAA2:![0-9]+]]
-// ABORT-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4, !tbaa [[TBAA6:![0-9]+]]
-// ABORT-NEXT:    [[DOTNOT:%.*]] = icmp eq i16 [[TMP0]], 0
-// ABORT-NEXT:    br i1 [[DOTNOT]], label %[[HANDLER_DIVREM_OVERFLOW:.*]], label %[[CONT:.*]], !prof [[PROF8:![0-9]+]], !nosanitize [[META9:![0-9]+]]
-// ABORT:       [[HANDLER_DIVREM_OVERFLOW]]:
-// ABORT-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64, !nosanitize [[META9]]
-// ABORT-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @{{.+}}, i64 [[TMP2]], i64 0) #[[ATTR3:[0-9]+]], !nosanitize [[META9]]
-// ABORT-NEXT:    unreachable, !nosanitize [[META9]]
-// ABORT:       [[CONT]]:
-// ABORT-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
-// ABORT-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP1]], [[CONV]]
-// ABORT-NEXT:    store i32 [[DIV]], ptr [[A]], align 4, !tbaa [[TBAA6]]
-// ABORT-NEXT:    ret i32 [[CONV]]
-//
-int test(int &a, short &c) {
-  a /= c;
-  return c;
-}
-
-// Make sure that we don't eliminate the call to ubsan handler.
-// RECOVER: Function Attrs: mustprogress noreturn nounwind memory(read, argmem: none, inaccessiblemem: readwrite)
-// RECOVER-LABEL: define dso_local noundef i32 @_Z16test_unreachablev(
-// RECOVER-SAME: ) local_unnamed_addr #[[ATTR2:[0-9]+]] {
-// RECOVER-NEXT:  [[ENTRY:.*:]]
-// RECOVER-NEXT:    tail call void @__ubsan_handle_divrem_overflow(ptr nonnull @{{.+}}, i64 1, i64 0) #[[ATTR3]], !nosanitize [[META9]]
-// RECOVER-NEXT:    unreachable
-//
-// ABORT: Function Attrs: mustprogress noreturn nounwind
-// ABORT-LABEL: define dso_local noundef i32 @_Z16test_unreachablev(
-// ABORT-SAME: ) local_unnamed_addr #[[ATTR2:[0-9]+]] {
-// ABORT-NEXT:  [[ENTRY:.*:]]
-// ABORT-NEXT:    tail call void @__ubsan_handle_divrem_overflow_abort(ptr nonnull @{{.+}}, i64 1, i64 0) #[[ATTR3]], !nosanitize [[META9]]
-// ABORT-NEXT:    unreachable, !nosanitize [[META9]]
-//
-int test_unreachable() {
-  int a = 1;
-  int b = 0;
-  if (a / b) return 1;
-  return 0;
-}
-
-//.
-// RECOVER: attributes #[[ATTR0]] = { mustprogress nounwind memory(read, argmem: readwrite, inaccessiblemem: readwrite) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-// RECOVER: attributes #[[ATTR1:[0-9]+]] = { memory(argmem: read, inaccessiblemem: readwrite) uwtable }
-// RECOVER: attributes #[[ATTR2]] = { mustprogress noreturn nounwind memory(read, argmem: none, inaccessiblemem: readwrite) "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-// RECOVER: attributes #[[ATTR3]] = { nomerge nounwind }
-//.
-// ABORT: attributes #[[ATTR0]] = { mustprogress nounwind "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-// ABORT: attributes #[[ATTR1:[0-9]+]] = { noreturn nounwind uwtable }
-// ABORT: attributes #[[ATTR2]] = { mustprogress noreturn nounwind "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
-// ABORT: attributes #[[ATTR3]] = { nomerge noreturn nounwind }
-//.
-// RECOVER: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-// RECOVER: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-// RECOVER: [[TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0}
-// RECOVER: [[META3]] = !{!"short", [[META4:![0-9]+]], i64 0}
-// RECOVER: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
-// RECOVER: [[META5]] = !{!"Simple C++ TBAA"}
-// RECOVER: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
-// RECOVER: [[META7]] = !{!"int", [[META4]], i64 0}
-// RECOVER: [[PROF8]] = !{!"branch_weights", i32 1, i32 1048575}
-// RECOVER: [[META9]] = !{}
-//.
-// ABORT: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-// ABORT: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
-// ABORT: [[TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0}
-// ABORT: [[META3]] = !{!"short", [[META4:![0-9]+]], i64 0}
-// ABORT: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
-// ABORT: [[META5]] = !{!"Simple C++ TBAA"}
-// ABORT: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
-// ABORT: [[META7]] = !{!"int", [[META4]], i64 0}
-// ABORT: [[PROF8]] = !{!"branch_weights", i32 1, i32 1048575}
-// ABORT: [[META9]] = !{}
-//.

From ee4c8b556c5cf42c55ce9540bbb0e29c11894a71 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Sat, 19 Apr 2025 08:21:59 +0300
Subject: [PATCH 528/710] [TableGen] Use llvm::enumerate in one place (NFC)
 (#136399)

---
 .../TableGen/Common/CodeGenInstAlias.cpp      | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp b/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp
index 5537a2fa8b980..94c0d51faf2e9 100644
--- a/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp
@@ -200,17 +200,15 @@ CodeGenInstAlias::CodeGenInstAlias(const Record *R, const CodeGenTarget &T)
 
   // Decode and validate the arguments of the result.
   unsigned AliasOpNo = 0;
-  for (unsigned i = 0, e = ResultInst->Operands.size(); i != e; ++i) {
-
+  for (auto [OpIdx, OpInfo] : enumerate(ResultInst->Operands)) {
     // Tied registers don't have an entry in the result dag unless they're part
     // of a complex operand, in which case we include them anyways, as we
     // don't have any other way to specify the whole operand.
-    if (ResultInst->Operands[i].MINumOperands == 1 &&
-        ResultInst->Operands[i].getTiedRegister() != -1) {
+    if (OpInfo.MINumOperands == 1 && OpInfo.getTiedRegister() != -1) {
       // Tied operands of different RegisterClass should be explicit within an
       // instruction's syntax and so cannot be skipped.
-      int TiedOpNum = ResultInst->Operands[i].getTiedRegister();
-      if (ResultInst->Operands[i].Rec->getName() ==
+      int TiedOpNum = OpInfo.getTiedRegister();
+      if (OpInfo.Rec->getName() ==
           ResultInst->Operands[TiedOpNum].Rec->getName())
         continue;
     }
@@ -218,8 +216,8 @@ CodeGenInstAlias::CodeGenInstAlias(const Record *R, const CodeGenTarget &T)
     if (AliasOpNo >= Result->getNumArgs())
       PrintFatalError(R->getLoc(), "not enough arguments for instruction!");
 
-    const Record *InstOpRec = ResultInst->Operands[i].Rec;
-    unsigned NumSubOps = ResultInst->Operands[i].MINumOperands;
+    const Record *InstOpRec = OpInfo.Rec;
+    unsigned NumSubOps = OpInfo.MINumOperands;
     ResultOperand ResOp(static_cast<int64_t>(0));
     if (tryAliasOpMatch(Result, AliasOpNo, InstOpRec, (NumSubOps > 1),
                         R->getLoc(), T, ResOp)) {
@@ -229,12 +227,12 @@ CodeGenInstAlias::CodeGenInstAlias(const Record *R, const CodeGenTarget &T)
                              InstOpRec->getValueAsDef("ParserMatchClass")
                                      ->getValueAsString("Name") != "Imm")) {
         ResultOperands.push_back(std::move(ResOp));
-        ResultInstOperandIndex.emplace_back(i, -1);
+        ResultInstOperandIndex.emplace_back(OpIdx, -1);
         ++AliasOpNo;
 
         // Otherwise, we need to match each of the suboperands individually.
       } else {
-        const DagInit *MIOI = ResultInst->Operands[i].MIOperandInfo;
+        const DagInit *MIOI = OpInfo.MIOperandInfo;
         for (unsigned SubOp = 0; SubOp != NumSubOps; ++SubOp) {
           const Record *SubRec = cast<DefInit>(MIOI->getArg(SubOp))->getDef();
 
@@ -244,7 +242,7 @@ CodeGenInstAlias::CodeGenInstAlias(const Record *R, const CodeGenTarget &T)
               Result->getArgName(AliasOpNo)->getAsUnquotedString() + "." +
                   MIOI->getArgName(SubOp)->getAsUnquotedString(),
               SubRec);
-          ResultInstOperandIndex.emplace_back(i, SubOp);
+          ResultInstOperandIndex.emplace_back(OpIdx, SubOp);
         }
         ++AliasOpNo;
       }
@@ -254,7 +252,7 @@ CodeGenInstAlias::CodeGenInstAlias(const Record *R, const CodeGenTarget &T)
     // If the argument did not match the instruction operand, and the operand
     // is composed of multiple suboperands, try matching the suboperands.
     if (NumSubOps > 1) {
-      const DagInit *MIOI = ResultInst->Operands[i].MIOperandInfo;
+      const DagInit *MIOI = OpInfo.MIOperandInfo;
       for (unsigned SubOp = 0; SubOp != NumSubOps; ++SubOp) {
         if (AliasOpNo >= Result->getNumArgs())
           PrintFatalError(R->getLoc(), "not enough arguments for instruction!");
@@ -262,7 +260,7 @@ CodeGenInstAlias::CodeGenInstAlias(const Record *R, const CodeGenTarget &T)
         if (tryAliasOpMatch(Result, AliasOpNo, SubRec, false, R->getLoc(), T,
                             ResOp)) {
           ResultOperands.push_back(ResOp);
-          ResultInstOperandIndex.emplace_back(i, SubOp);
+          ResultInstOperandIndex.emplace_back(OpIdx, SubOp);
           ++AliasOpNo;
         } else {
           PrintFatalError(

From a71a2542a1a6d66425a7f078cf5391e95ff3d374 Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg@google.com>
Date: Sat, 19 Apr 2025 07:57:43 +0200
Subject: [PATCH 529/710] [clang][bazel] Port
 d1cce66469d0739f332b11fdb0fa2a441b1e0483

---
 utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index c1bd75c6f71aa..6ac7889b026d3 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -962,7 +962,9 @@ cc_library(
     srcs = glob([
         "lib/Sema/*.cpp",
         "lib/Sema/*.h",
-    ]),
+    ]) + [
+        "//llvm:include/llvm/Frontend/OpenACC/ACC.inc",
+    ],
     hdrs = glob([
         "include/clang/Sema/*.h",
         "include/clang-c/*.h",

From 5607a3b09c6fb6026adfcfb42710306a63523c57 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 18 Apr 2025 23:25:39 -0700
Subject: [PATCH 530/710] [clang-doc][NFC] Use LLVM style naming in
 YAMLGenerator.cpp (#136393)

---
 clang-tools-extra/clang-doc/YAMLGenerator.cpp | 44 +++++++++----------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/clang-tools-extra/clang-doc/YAMLGenerator.cpp b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
index ffabd2fd82229..1b741f52ea5e3 100644
--- a/clang-tools-extra/clang-doc/YAMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
@@ -92,11 +92,11 @@ template <> struct ScalarTraits<std::array<unsigned char, 20>> {
                          std::array<unsigned char, 20> &Value) {
     if (Scalar.size() != 40)
       return "Error: Incorrect scalar size for USR.";
-    Value = StringToSymbol(Scalar);
+    Value = stringToSymbol(Scalar);
     return StringRef();
   }
 
-  static SymbolID StringToSymbol(llvm::StringRef Value) {
+  static SymbolID stringToSymbol(llvm::StringRef Value) {
     SymbolID USR;
     std::string HexString = fromHex(Value);
     std::copy(HexString.begin(), HexString.end(), USR.begin());
@@ -108,17 +108,17 @@ template <> struct ScalarTraits<std::array<unsigned char, 20>> {
 
 // Helper functions to map infos to YAML.
 
-static void TypeInfoMapping(IO &IO, TypeInfo &I) {
+static void typeInfoMapping(IO &IO, TypeInfo &I) {
   IO.mapOptional("Type", I.Type, Reference());
 }
 
-static void FieldTypeInfoMapping(IO &IO, FieldTypeInfo &I) {
-  TypeInfoMapping(IO, I);
+static void fieldTypeInfoMapping(IO &IO, FieldTypeInfo &I) {
+  typeInfoMapping(IO, I);
   IO.mapOptional("Name", I.Name, SmallString<16>());
   IO.mapOptional("DefaultValue", I.DefaultValue, SmallString<16>());
 }
 
-static void InfoMapping(IO &IO, Info &I) {
+static void infoMapping(IO &IO, Info &I) {
   IO.mapRequired("USR", I.USR);
   IO.mapOptional("Name", I.Name, SmallString<16>());
   IO.mapOptional("Path", I.Path, SmallString<128>());
@@ -126,14 +126,14 @@ static void InfoMapping(IO &IO, Info &I) {
   IO.mapOptional("Description", I.Description);
 }
 
-static void SymbolInfoMapping(IO &IO, SymbolInfo &I) {
-  InfoMapping(IO, I);
+static void symbolInfoMapping(IO &IO, SymbolInfo &I) {
+  infoMapping(IO, I);
   IO.mapOptional("DefLocation", I.DefLoc, std::optional<Location>());
   IO.mapOptional("Location", I.Loc, llvm::SmallVector<Location, 2>());
 }
 
-static void RecordInfoMapping(IO &IO, RecordInfo &I) {
-  SymbolInfoMapping(IO, I);
+static void recordInfoMapping(IO &IO, RecordInfo &I) {
+  symbolInfoMapping(IO, I);
   IO.mapOptional("TagType", I.TagType);
   IO.mapOptional("IsTypeDef", I.IsTypeDef, false);
   IO.mapOptional("Members", I.Members);
@@ -148,7 +148,7 @@ static void RecordInfoMapping(IO &IO, RecordInfo &I) {
   IO.mapOptional("Template", I.Template);
 }
 
-static void CommentInfoMapping(IO &IO, CommentInfo &I) {
+static void commentInfoMapping(IO &IO, CommentInfo &I) {
   IO.mapOptional("Kind", I.Kind, SmallString<16>());
   IO.mapOptional("Text", I.Text, SmallString<64>());
   IO.mapOptional("Name", I.Name, SmallString<16>());
@@ -185,12 +185,12 @@ template <> struct MappingTraits<Reference> {
 };
 
 template <> struct MappingTraits<TypeInfo> {
-  static void mapping(IO &IO, TypeInfo &I) { TypeInfoMapping(IO, I); }
+  static void mapping(IO &IO, TypeInfo &I) { typeInfoMapping(IO, I); }
 };
 
 template <> struct MappingTraits<FieldTypeInfo> {
   static void mapping(IO &IO, FieldTypeInfo &I) {
-    TypeInfoMapping(IO, I);
+    typeInfoMapping(IO, I);
     IO.mapOptional("Name", I.Name, SmallString<16>());
     IO.mapOptional("DefaultValue", I.DefaultValue, SmallString<16>());
   }
@@ -198,7 +198,7 @@ template <> struct MappingTraits<FieldTypeInfo> {
 
 template <> struct MappingTraits<MemberTypeInfo> {
   static void mapping(IO &IO, MemberTypeInfo &I) {
-    FieldTypeInfoMapping(IO, I);
+    fieldTypeInfoMapping(IO, I);
     // clang::AccessSpecifier::AS_none is used as the default here because it's
     // the AS that shouldn't be part of the output. Even though AS_public is the
     // default in the struct, it should be displayed in the YAML output.
@@ -209,7 +209,7 @@ template <> struct MappingTraits<MemberTypeInfo> {
 
 template <> struct MappingTraits<NamespaceInfo> {
   static void mapping(IO &IO, NamespaceInfo &I) {
-    InfoMapping(IO, I);
+    infoMapping(IO, I);
     IO.mapOptional("ChildNamespaces", I.Children.Namespaces,
                    std::vector<Reference>());
     IO.mapOptional("ChildRecords", I.Children.Records,
@@ -221,12 +221,12 @@ template <> struct MappingTraits<NamespaceInfo> {
 };
 
 template <> struct MappingTraits<RecordInfo> {
-  static void mapping(IO &IO, RecordInfo &I) { RecordInfoMapping(IO, I); }
+  static void mapping(IO &IO, RecordInfo &I) { recordInfoMapping(IO, I); }
 };
 
 template <> struct MappingTraits<BaseRecordInfo> {
   static void mapping(IO &IO, BaseRecordInfo &I) {
-    RecordInfoMapping(IO, I);
+    recordInfoMapping(IO, I);
     IO.mapOptional("IsVirtual", I.IsVirtual, false);
     // clang::AccessSpecifier::AS_none is used as the default here because it's
     // the AS that shouldn't be part of the output. Even though AS_public is the
@@ -246,7 +246,7 @@ template <> struct MappingTraits<EnumValueInfo> {
 
 template <> struct MappingTraits<EnumInfo> {
   static void mapping(IO &IO, EnumInfo &I) {
-    SymbolInfoMapping(IO, I);
+    symbolInfoMapping(IO, I);
     IO.mapOptional("Scoped", I.Scoped, false);
     IO.mapOptional("BaseType", I.BaseType);
     IO.mapOptional("Members", I.Members);
@@ -255,7 +255,7 @@ template <> struct MappingTraits<EnumInfo> {
 
 template <> struct MappingTraits<TypedefInfo> {
   static void mapping(IO &IO, TypedefInfo &I) {
-    SymbolInfoMapping(IO, I);
+    symbolInfoMapping(IO, I);
     IO.mapOptional("Underlying", I.Underlying.Type);
     IO.mapOptional("IsUsing", I.IsUsing, false);
   }
@@ -263,7 +263,7 @@ template <> struct MappingTraits<TypedefInfo> {
 
 template <> struct MappingTraits<FunctionInfo> {
   static void mapping(IO &IO, FunctionInfo &I) {
-    SymbolInfoMapping(IO, I);
+    symbolInfoMapping(IO, I);
     IO.mapOptional("IsMethod", I.IsMethod, false);
     IO.mapOptional("Parent", I.Parent, Reference());
     IO.mapOptional("Params", I.Params);
@@ -298,13 +298,13 @@ template <> struct MappingTraits<TemplateInfo> {
 };
 
 template <> struct MappingTraits<CommentInfo> {
-  static void mapping(IO &IO, CommentInfo &I) { CommentInfoMapping(IO, I); }
+  static void mapping(IO &IO, CommentInfo &I) { commentInfoMapping(IO, I); }
 };
 
 template <> struct MappingTraits<std::unique_ptr<CommentInfo>> {
   static void mapping(IO &IO, std::unique_ptr<CommentInfo> &I) {
     if (I)
-      CommentInfoMapping(IO, *I);
+      commentInfoMapping(IO, *I);
   }
 };
 

From e27aa12d47468e4717622a230bb560490b66c3e7 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 18 Apr 2025 23:34:03 -0700
Subject: [PATCH 531/710] [clang-doc][NFC] Use qualified auto (#136394)

---
 clang-tools-extra/clang-doc/Representation.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 0947ecff72272..54d2cb58ea2d9 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -215,7 +215,7 @@ void SymbolInfo::merge(SymbolInfo &&Other) {
   // Unconditionally extend the list of locations, since we want all of them.
   std::move(Other.Loc.begin(), Other.Loc.end(), std::back_inserter(Loc));
   llvm::sort(Loc);
-  auto Last = std::unique(Loc.begin(), Loc.end());
+  auto *Last = std::unique(Loc.begin(), Loc.end());
   Loc.erase(Last, Loc.end());
   mergeBase(std::move(Other));
 }

From 8376edb143c4ea0d4d166b81c9667c2341a9533d Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 18 Apr 2025 23:34:32 -0700
Subject: [PATCH 532/710] [clang-doc][NFC] Avoid C-style casts (#136390)

---
 clang-tools-extra/clang-doc/BitcodeReader.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index d6c9da81a05c7..019960eae39f6 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -46,7 +46,7 @@ llvm::Error decodeRecord(const Record &R, int &Field, llvm::StringRef Blob) {
   if (R[0] > INT_MAX)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "integer too large to parse");
-  Field = (int)R[0];
+  Field = static_cast<int>(R[0]);
   return llvm::Error::success();
 }
 
@@ -85,7 +85,7 @@ llvm::Error decodeRecord(const Record &R, std::optional<Location> &Field,
   if (R[0] > INT_MAX)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "integer too large to parse");
-  Field.emplace((int)R[0], Blob, (bool)R[1]);
+  Field.emplace(static_cast<int>(R[0]), Blob, static_cast<bool>(R[1]));
   return llvm::Error::success();
 }
 
@@ -135,7 +135,7 @@ llvm::Error decodeRecord(const Record &R,
   if (R[0] > INT_MAX)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "integer too large to parse");
-  Field.emplace_back((int)R[0], Blob, (bool)R[1]);
+  Field.emplace_back(static_cast<int>(R[0]), Blob, static_cast<bool>(R[1]));
   return llvm::Error::success();
 }
 

From c5c285178ccb1769e6d4dfeeb8df7663ec068a55 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 18 Apr 2025 23:34:44 -0700
Subject: [PATCH 533/710] [clang-doc][NFC] Avoid else after return (#136389)

---
 clang-tools-extra/clang-doc/BitcodeReader.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index 019960eae39f6..e9e2bf07b59fc 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -878,7 +878,7 @@ llvm::Error ClangDocBitcodeReader::validateStream() {
     Expected<llvm::SimpleBitstreamCursor::word_t> MaybeRead = Stream.Read(8);
     if (!MaybeRead)
       return MaybeRead.takeError();
-    else if (MaybeRead.get() != BitCodeConstants::Signature[Idx])
+    if (MaybeRead.get() != BitCodeConstants::Signature[Idx])
       return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                      "invalid bitcode signature");
   }
@@ -890,8 +890,7 @@ llvm::Error ClangDocBitcodeReader::readBlockInfoBlock() {
       Stream.ReadBlockInfoBlock();
   if (!MaybeBlockInfo)
     return MaybeBlockInfo.takeError();
-  else
-    BlockInfo = MaybeBlockInfo.get();
+  BlockInfo = MaybeBlockInfo.get();
   if (!BlockInfo)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "unable to parse BlockInfoBlock");

From c274bbed6aa5254ec3b97b25471dd08c67224350 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 18 Apr 2025 23:34:57 -0700
Subject: [PATCH 534/710] [clang-doc][NFC] Prefer static functions for internal
 APIs (#136391)

Additionally moving the declarations from Generator.cpp
prevents misc-use-internal-linkage warnings from clang-tidy.
We fix those here too, since the static functions are part
of the same diagnostic
---
 clang-tools-extra/clang-doc/BitcodeReader.cpp | 128 +++++++++---------
 clang-tools-extra/clang-doc/Generators.cpp    |   3 -
 clang-tools-extra/clang-doc/Generators.h      |   6 +
 clang-tools-extra/clang-doc/Mapper.cpp        |   2 +-
 clang-tools-extra/clang-doc/Serialize.cpp     |  20 +--
 .../clang-doc/tool/ClangDocMain.cpp           |  17 +--
 6 files changed, 94 insertions(+), 82 deletions(-)

diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index e9e2bf07b59fc..2e064acb5dd76 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -18,14 +18,15 @@ namespace doc {
 using Record = llvm::SmallVector<uint64_t, 1024>;
 
 // This implements decode for SmallString.
-llvm::Error decodeRecord(const Record &R, llvm::SmallVectorImpl<char> &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R,
+                                llvm::SmallVectorImpl<char> &Field,
+                                llvm::StringRef Blob) {
   Field.assign(Blob.begin(), Blob.end());
   return llvm::Error::success();
 }
 
-llvm::Error decodeRecord(const Record &R, SymbolID &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R, SymbolID &Field,
+                                llvm::StringRef Blob) {
   if (R[0] != BitCodeConstants::USRHashSize)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "incorrect USR size");
@@ -37,12 +38,14 @@ llvm::Error decodeRecord(const Record &R, SymbolID &Field,
   return llvm::Error::success();
 }
 
-llvm::Error decodeRecord(const Record &R, bool &Field, llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R, bool &Field,
+                                llvm::StringRef Blob) {
   Field = R[0] != 0;
   return llvm::Error::success();
 }
 
-llvm::Error decodeRecord(const Record &R, int &Field, llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R, int &Field,
+                                llvm::StringRef Blob) {
   if (R[0] > INT_MAX)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "integer too large to parse");
@@ -50,8 +53,8 @@ llvm::Error decodeRecord(const Record &R, int &Field, llvm::StringRef Blob) {
   return llvm::Error::success();
 }
 
-llvm::Error decodeRecord(const Record &R, AccessSpecifier &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R, AccessSpecifier &Field,
+                                llvm::StringRef Blob) {
   switch (R[0]) {
   case AS_public:
   case AS_private:
@@ -65,8 +68,8 @@ llvm::Error decodeRecord(const Record &R, AccessSpecifier &Field,
   }
 }
 
-llvm::Error decodeRecord(const Record &R, TagTypeKind &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R, TagTypeKind &Field,
+                                llvm::StringRef Blob) {
   switch (static_cast<TagTypeKind>(R[0])) {
   case TagTypeKind::Struct:
   case TagTypeKind::Interface:
@@ -80,8 +83,8 @@ llvm::Error decodeRecord(const Record &R, TagTypeKind &Field,
                                  "invalid value for TagTypeKind");
 }
 
-llvm::Error decodeRecord(const Record &R, std::optional<Location> &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R, std::optional<Location> &Field,
+                                llvm::StringRef Blob) {
   if (R[0] > INT_MAX)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "integer too large to parse");
@@ -89,8 +92,8 @@ llvm::Error decodeRecord(const Record &R, std::optional<Location> &Field,
   return llvm::Error::success();
 }
 
-llvm::Error decodeRecord(const Record &R, InfoType &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R, InfoType &Field,
+                                llvm::StringRef Blob) {
   switch (auto IT = static_cast<InfoType>(R[0])) {
   case InfoType::IT_namespace:
   case InfoType::IT_record:
@@ -105,8 +108,8 @@ llvm::Error decodeRecord(const Record &R, InfoType &Field,
                                  "invalid value for InfoType");
 }
 
-llvm::Error decodeRecord(const Record &R, FieldId &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R, FieldId &Field,
+                                llvm::StringRef Blob) {
   switch (auto F = static_cast<FieldId>(R[0])) {
   case FieldId::F_namespace:
   case FieldId::F_parent:
@@ -122,16 +125,17 @@ llvm::Error decodeRecord(const Record &R, FieldId &Field,
                                  "invalid value for FieldId");
 }
 
-llvm::Error decodeRecord(const Record &R,
-                         llvm::SmallVectorImpl<llvm::SmallString<16>> &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error
+decodeRecord(const Record &R,
+             llvm::SmallVectorImpl<llvm::SmallString<16>> &Field,
+             llvm::StringRef Blob) {
   Field.push_back(Blob);
   return llvm::Error::success();
 }
 
-llvm::Error decodeRecord(const Record &R,
-                         llvm::SmallVectorImpl<Location> &Field,
-                         llvm::StringRef Blob) {
+static llvm::Error decodeRecord(const Record &R,
+                                llvm::SmallVectorImpl<Location> &Field,
+                                llvm::StringRef Blob) {
   if (R[0] > INT_MAX)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "integer too large to parse");
@@ -139,16 +143,16 @@ llvm::Error decodeRecord(const Record &R,
   return llvm::Error::success();
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        const unsigned VersionNo) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, const unsigned VersionNo) {
   if (ID == VERSION && R[0] == VersionNo)
     return llvm::Error::success();
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "mismatched bitcode version number");
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        NamespaceInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, NamespaceInfo *I) {
   switch (ID) {
   case NAMESPACE_USR:
     return decodeRecord(R, I->USR, Blob);
@@ -162,8 +166,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        RecordInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, RecordInfo *I) {
   switch (ID) {
   case RECORD_USR:
     return decodeRecord(R, I->USR, Blob);
@@ -185,8 +189,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        BaseRecordInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, BaseRecordInfo *I) {
   switch (ID) {
   case BASE_RECORD_USR:
     return decodeRecord(R, I->USR, Blob);
@@ -208,8 +212,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        EnumInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, EnumInfo *I) {
   switch (ID) {
   case ENUM_USR:
     return decodeRecord(R, I->USR, Blob);
@@ -227,8 +231,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        TypedefInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, TypedefInfo *I) {
   switch (ID) {
   case TYPEDEF_USR:
     return decodeRecord(R, I->USR, Blob);
@@ -244,8 +248,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        EnumValueInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, EnumValueInfo *I) {
   switch (ID) {
   case ENUM_VALUE_NAME:
     return decodeRecord(R, I->Name, Blob);
@@ -259,8 +263,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        FunctionInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, FunctionInfo *I) {
   switch (ID) {
   case FUNCTION_USR:
     return decodeRecord(R, I->USR, Blob);
@@ -282,13 +286,13 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        TypeInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, TypeInfo *I) {
   return llvm::Error::success();
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        FieldTypeInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, FieldTypeInfo *I) {
   switch (ID) {
   case FIELD_TYPE_NAME:
     return decodeRecord(R, I->Name, Blob);
@@ -300,8 +304,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        MemberTypeInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, MemberTypeInfo *I) {
   switch (ID) {
   case MEMBER_TYPE_NAME:
     return decodeRecord(R, I->Name, Blob);
@@ -315,8 +319,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        CommentInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, CommentInfo *I) {
   switch (ID) {
   case COMMENT_KIND:
     return decodeRecord(R, I->Kind, Blob);
@@ -346,8 +350,8 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        Reference *I, FieldId &F) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, Reference *I, FieldId &F) {
   switch (ID) {
   case REFERENCE_USR:
     return decodeRecord(R, I->USR, Blob);
@@ -367,30 +371,31 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        TemplateInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, TemplateInfo *I) {
   // Currently there are no child records of TemplateInfo (only child blocks).
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid field for TemplateParamInfo");
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        TemplateSpecializationInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob,
+                               TemplateSpecializationInfo *I) {
   if (ID == TEMPLATE_SPECIALIZATION_OF)
     return decodeRecord(R, I->SpecializationOf, Blob);
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid field for TemplateParamInfo");
 }
 
-llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
-                        TemplateParamInfo *I) {
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, TemplateParamInfo *I) {
   if (ID == TEMPLATE_PARAM_CONTENTS)
     return decodeRecord(R, I->Contents, Blob);
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid field for TemplateParamInfo");
 }
 
-template <typename T> llvm::Expected<CommentInfo *> getCommentInfo(T I) {
+template <typename T> static llvm::Expected<CommentInfo *> getCommentInfo(T I) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid type cannot contain CommentInfo");
 }
@@ -437,7 +442,7 @@ llvm::Expected<CommentInfo *> getCommentInfo(std::unique_ptr<CommentInfo> &I) {
 // the parent block to set it. The template specializations define what to do
 // for each supported parent block.
 template <typename T, typename TTypeInfo>
-llvm::Error addTypeInfo(T I, TTypeInfo &&TI) {
+static llvm::Error addTypeInfo(T I, TTypeInfo &&TI) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid type cannot contain TypeInfo");
 }
@@ -472,7 +477,8 @@ template <> llvm::Error addTypeInfo(TypedefInfo *I, TypeInfo &&T) {
   return llvm::Error::success();
 }
 
-template <typename T> llvm::Error addReference(T I, Reference &&R, FieldId F) {
+template <typename T>
+static llvm::Error addReference(T I, Reference &&R, FieldId F) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid type cannot contain Reference");
 }
@@ -588,7 +594,7 @@ template <> llvm::Error addReference(RecordInfo *I, Reference &&R, FieldId F) {
 }
 
 template <typename T, typename ChildInfoType>
-void addChild(T I, ChildInfoType &&R) {
+static void addChild(T I, ChildInfoType &&R) {
   llvm::errs() << "invalid child type for info";
   exit(1);
 }
@@ -629,7 +635,7 @@ template <> void addChild(BaseRecordInfo *I, FunctionInfo &&R) {
 // TemplateParam children. These go into either a TemplateInfo (for template
 // parameters) or TemplateSpecializationInfo (for the specialization's
 // parameters).
-template <typename T> void addTemplateParam(T I, TemplateParamInfo &&P) {
+template <typename T> static void addTemplateParam(T I, TemplateParamInfo &&P) {
   llvm::errs() << "invalid container for template parameter";
   exit(1);
 }
@@ -642,7 +648,7 @@ void addTemplateParam(TemplateSpecializationInfo *I, TemplateParamInfo &&P) {
 }
 
 // Template info. These apply to either records or functions.
-template <typename T> void addTemplate(T I, TemplateInfo &&P) {
+template <typename T> static void addTemplate(T I, TemplateInfo &&P) {
   llvm::errs() << "invalid container for template info";
   exit(1);
 }
@@ -655,7 +661,7 @@ template <> void addTemplate(FunctionInfo *I, TemplateInfo &&P) {
 
 // Template specializations go only into template records.
 template <typename T>
-void addTemplateSpecialization(T I, TemplateSpecializationInfo &&TSI) {
+static void addTemplateSpecialization(T I, TemplateSpecializationInfo &&TSI) {
   llvm::errs() << "invalid container for template specialization info";
   exit(1);
 }
diff --git a/clang-tools-extra/clang-doc/Generators.cpp b/clang-tools-extra/clang-doc/Generators.cpp
index a3986b66f3c74..4bb5366800f7f 100644
--- a/clang-tools-extra/clang-doc/Generators.cpp
+++ b/clang-tools-extra/clang-doc/Generators.cpp
@@ -97,9 +97,6 @@ void Generator::addInfoToIndex(Index &Idx, const doc::Info *Info) {
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the generators.
-extern volatile int YAMLGeneratorAnchorSource;
-extern volatile int MDGeneratorAnchorSource;
-extern volatile int HTMLGeneratorAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED YAMLGeneratorAnchorDest =
     YAMLGeneratorAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED MDGeneratorAnchorDest =
diff --git a/clang-tools-extra/clang-doc/Generators.h b/clang-tools-extra/clang-doc/Generators.h
index d62d7faa9a69f..2e71890389256 100644
--- a/clang-tools-extra/clang-doc/Generators.h
+++ b/clang-tools-extra/clang-doc/Generators.h
@@ -52,6 +52,12 @@ findGeneratorByName(llvm::StringRef Format);
 
 std::string getTagType(TagTypeKind AS);
 
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the generators.
+extern volatile int YAMLGeneratorAnchorSource;
+extern volatile int MDGeneratorAnchorSource;
+extern volatile int HTMLGeneratorAnchorSource;
+
 } // namespace doc
 } // namespace clang
 
diff --git a/clang-tools-extra/clang-doc/Mapper.cpp b/clang-tools-extra/clang-doc/Mapper.cpp
index 950cdf55d4a63..3ccf026958c00 100644
--- a/clang-tools-extra/clang-doc/Mapper.cpp
+++ b/clang-tools-extra/clang-doc/Mapper.cpp
@@ -21,7 +21,7 @@ namespace doc {
 static llvm::StringSet<> USRVisited;
 static llvm::sys::SmartMutex<true> USRVisitedGuard;
 
-template <typename T> bool isTypedefAnonRecord(const T *D) {
+template <typename T> static bool isTypedefAnonRecord(const T *D) {
   if (const auto *C = dyn_cast<CXXRecordDecl>(D)) {
     return C->getTypedefNameForAnonDecl();
   }
diff --git a/clang-tools-extra/clang-doc/Serialize.cpp b/clang-tools-extra/clang-doc/Serialize.cpp
index 837ecdd9f1278..1dcccf3c3c2ab 100644
--- a/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/clang-tools-extra/clang-doc/Serialize.cpp
@@ -48,7 +48,7 @@ static void populateMemberTypeInfo(RecordInfo &I, AccessSpecifier &Access,
 //
 // }
 // }
-llvm::SmallString<128>
+static llvm::SmallString<128>
 getInfoRelativePath(const llvm::SmallVectorImpl<doc::Reference> &Namespaces) {
   llvm::SmallString<128> Path;
   for (auto R = Namespaces.rbegin(), E = Namespaces.rend(); R != E; ++R)
@@ -56,7 +56,7 @@ getInfoRelativePath(const llvm::SmallVectorImpl<doc::Reference> &Namespaces) {
   return Path;
 }
 
-llvm::SmallString<128> getInfoRelativePath(const Decl *D) {
+static llvm::SmallString<128> getInfoRelativePath(const Decl *D) {
   llvm::SmallVector<Reference, 4> Namespaces;
   // The third arg in populateParentNamespaces is a boolean passed by reference,
   // its value is not relevant in here so it's not used anywhere besides the
@@ -185,7 +185,7 @@ std::string ClangDocCommentVisitor::getCommandName(unsigned CommandID) const {
 
 // Serializing functions.
 
-std::string getSourceCode(const Decl *D, const SourceRange &R) {
+static std::string getSourceCode(const Decl *D, const SourceRange &R) {
   return Lexer::getSourceText(CharSourceRange::getTokenRange(R),
                               D->getASTContext().getSourceManager(),
                               D->getASTContext().getLangOpts())
@@ -239,7 +239,8 @@ static RecordDecl *getRecordDeclForType(const QualType &T) {
   return nullptr;
 }
 
-TypeInfo getTypeInfoForType(const QualType &T, const PrintingPolicy &Policy) {
+static TypeInfo getTypeInfoForType(const QualType &T,
+                                   const PrintingPolicy &Policy) {
   const TagDecl *TD = getTagDeclForType(T);
   if (!TD)
     return TypeInfo(Reference(SymbolID(), T.getAsString(Policy)));
@@ -319,7 +320,7 @@ static void InsertChild(ScopeChildren &Scope, TypedefInfo Info) {
 // parameter. Since each variant is used once, it's not worth having a more
 // elaborate system to automatically deduce this information.
 template <typename ChildType>
-std::unique_ptr<Info> MakeAndInsertIntoParent(ChildType Child) {
+static std::unique_ptr<Info> MakeAndInsertIntoParent(ChildType Child) {
   if (Child.Namespace.empty()) {
     // Insert into unnamed parent namespace.
     auto ParentNS = std::make_unique<NamespaceInfo>();
@@ -496,8 +497,9 @@ populateParentNamespaces(llvm::SmallVector<Reference, 4> &Namespaces,
                             InfoType::IT_namespace);
 }
 
-void PopulateTemplateParameters(std::optional<TemplateInfo> &TemplateInfo,
-                                const clang::Decl *D) {
+static void
+PopulateTemplateParameters(std::optional<TemplateInfo> &TemplateInfo,
+                           const clang::Decl *D) {
   if (const TemplateParameterList *ParamList =
           D->getDescribedTemplateParams()) {
     if (!TemplateInfo) {
@@ -510,8 +512,8 @@ void PopulateTemplateParameters(std::optional<TemplateInfo> &TemplateInfo,
   }
 }
 
-TemplateParamInfo TemplateArgumentToInfo(const clang::Decl *D,
-                                         const TemplateArgument &Arg) {
+static TemplateParamInfo TemplateArgumentToInfo(const clang::Decl *D,
+                                                const TemplateArgument &Arg) {
   // The TemplateArgument's pretty printing handles all the normal cases
   // well enough for our requirements.
   std::string Str;
diff --git a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
index c58d2060bc1e7..8e8f7053a8f87 100644
--- a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -127,7 +127,7 @@ static llvm::cl::opt<OutputFormatTy>
                llvm::cl::init(OutputFormatTy::yaml),
                llvm::cl::cat(ClangDocCategory));
 
-std::string getFormatString() {
+static std::string getFormatString() {
   switch (FormatEnum) {
   case OutputFormatTy::yaml:
     return "yaml";
@@ -144,11 +144,11 @@ std::string getFormatString() {
 // GetMainExecutable (since some platforms don't support taking the
 // address of main, and some platforms can't implement GetMainExecutable
 // without being given the address of a function in the main executable).
-std::string getExecutablePath(const char *Argv0, void *MainAddr) {
+static std::string getExecutablePath(const char *Argv0, void *MainAddr) {
   return llvm::sys::fs::getMainExecutable(Argv0, MainAddr);
 }
 
-llvm::Error getAssetFiles(clang::doc::ClangDocContext &CDCtx) {
+static llvm::Error getAssetFiles(clang::doc::ClangDocContext &CDCtx) {
   using DirIt = llvm::sys::fs::directory_iterator;
   std::error_code FileErr;
   llvm::SmallString<128> FilePath(UserAssetPath);
@@ -168,8 +168,8 @@ llvm::Error getAssetFiles(clang::doc::ClangDocContext &CDCtx) {
   return llvm::Error::success();
 }
 
-llvm::Error getDefaultAssetFiles(const char *Argv0,
-                                 clang::doc::ClangDocContext &CDCtx) {
+static llvm::Error getDefaultAssetFiles(const char *Argv0,
+                                        clang::doc::ClangDocContext &CDCtx) {
   void *MainAddr = (void *)(intptr_t)getExecutablePath;
   std::string ClangDocPath = getExecutablePath(Argv0, MainAddr);
   llvm::SmallString<128> NativeClangDocPath;
@@ -204,8 +204,8 @@ llvm::Error getDefaultAssetFiles(const char *Argv0,
   return llvm::Error::success();
 }
 
-llvm::Error getHtmlAssetFiles(const char *Argv0,
-                              clang::doc::ClangDocContext &CDCtx) {
+static llvm::Error getHtmlAssetFiles(const char *Argv0,
+                                     clang::doc::ClangDocContext &CDCtx) {
   if (!UserAssetPath.empty() &&
       !llvm::sys::fs::is_directory(std::string(UserAssetPath)))
     llvm::outs() << "Asset path supply is not a directory: " << UserAssetPath
@@ -217,7 +217,8 @@ llvm::Error getHtmlAssetFiles(const char *Argv0,
 
 /// Make the output of clang-doc deterministic by sorting the children of
 /// namespaces and records.
-void sortUsrToInfo(llvm::StringMap<std::unique_ptr<doc::Info>> &USRToInfo) {
+static void
+sortUsrToInfo(llvm::StringMap<std::unique_ptr<doc::Info>> &USRToInfo) {
   for (auto &I : USRToInfo) {
     auto &Info = I.second;
     if (Info->IT == doc::InfoType::IT_namespace) {

From e1df0defa8a16b9769f6bde83ee4e4f79426d53f Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Fri, 18 Apr 2025 23:38:36 -0700
Subject: [PATCH 535/710] [CMake] Disable LTO library when PIC is disabled
 (#136372)

Building shared libraries requires PIC. This matches what we do for
Remarks which is another tool library.
---
 llvm/tools/lto/CMakeLists.txt | 91 ++++++++++++++++++-----------------
 1 file changed, 48 insertions(+), 43 deletions(-)

diff --git a/llvm/tools/lto/CMakeLists.txt b/llvm/tools/lto/CMakeLists.txt
index c2f7d81063af8..8c7d7904e16ea 100644
--- a/llvm/tools/lto/CMakeLists.txt
+++ b/llvm/tools/lto/CMakeLists.txt
@@ -1,48 +1,53 @@
-set(LLVM_LINK_COMPONENTS
-  AllTargetsAsmParsers
-  AllTargetsCodeGens
-  AllTargetsDescs
-  AllTargetsDisassemblers
-  AllTargetsInfos
-  BitReader
-  Core
-  CodeGen
-  LTO
-  MC
-  MCDisassembler
-  Support
-  Target
-  TargetParser
-  )
-
-set(SOURCES
-  LTODisassembler.cpp
-  lto.cpp
-  )
-
-set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/lto.exports)
-
-if(CMAKE_SYSTEM_NAME STREQUAL AIX)
-    set(LTO_LIBRARY_TYPE MODULE)
-    set(LTO_LIBRARY_NAME libLTO)
-  else()
-    set(LTO_LIBRARY_TYPE SHARED)
-    set(LTO_LIBRARY_NAME LTO)
-endif()
+# Building shared libraries requires PIC objects.
+if(LLVM_ENABLE_PIC)
+
+  set(LLVM_LINK_COMPONENTS
+    AllTargetsAsmParsers
+    AllTargetsCodeGens
+    AllTargetsDescs
+    AllTargetsDisassemblers
+    AllTargetsInfos
+    BitReader
+    Core
+    CodeGen
+    LTO
+    MC
+    MCDisassembler
+    Support
+    Target
+    TargetParser
+    )
 
-add_llvm_library(${LTO_LIBRARY_NAME} ${LTO_LIBRARY_TYPE} INSTALL_WITH_TOOLCHAIN
-    ${SOURCES} DEPENDS intrinsics_gen)
+  set(SOURCES
+    LTODisassembler.cpp
+    lto.cpp
+    )
 
-install(FILES ${LLVM_MAIN_INCLUDE_DIR}/llvm-c/lto.h
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/llvm-c"
-  COMPONENT LTO)
+  set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/lto.exports)
 
-if (APPLE)
-  set(LTO_VERSION ${LLVM_VERSION_MAJOR})
-  if(LLVM_LTO_VERSION_OFFSET)
-    math(EXPR LTO_VERSION "${LLVM_VERSION_MAJOR} + ${LLVM_LTO_VERSION_OFFSET}")
+  if(CMAKE_SYSTEM_NAME STREQUAL AIX)
+      set(LTO_LIBRARY_TYPE MODULE)
+      set(LTO_LIBRARY_NAME libLTO)
+    else()
+      set(LTO_LIBRARY_TYPE SHARED)
+      set(LTO_LIBRARY_NAME LTO)
   endif()
-  set_property(TARGET LTO APPEND_STRING PROPERTY
-              LINK_FLAGS
-              " -compatibility_version 1 -current_version ${LTO_VERSION}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}")
+
+  add_llvm_library(${LTO_LIBRARY_NAME} ${LTO_LIBRARY_TYPE} INSTALL_WITH_TOOLCHAIN
+      ${SOURCES} DEPENDS intrinsics_gen)
+
+  install(FILES ${LLVM_MAIN_INCLUDE_DIR}/llvm-c/lto.h
+    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/llvm-c"
+    COMPONENT LTO)
+
+  if (APPLE)
+    set(LTO_VERSION ${LLVM_VERSION_MAJOR})
+    if(LLVM_LTO_VERSION_OFFSET)
+      math(EXPR LTO_VERSION "${LLVM_VERSION_MAJOR} + ${LLVM_LTO_VERSION_OFFSET}")
+    endif()
+    set_property(TARGET LTO APPEND_STRING PROPERTY
+                LINK_FLAGS
+                " -compatibility_version 1 -current_version ${LTO_VERSION}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}")
+  endif()
+
 endif()

From 9b6fbc06ad532e455960bbe6e4bffa5e72eb9e8c Mon Sep 17 00:00:00 2001
From: Henrich Lauko <xlauko@mail.muni.cz>
Date: Sat, 19 Apr 2025 08:40:11 +0200
Subject: [PATCH 536/710] [CIR] Infer MLIR context in type builders when
 possible (#136362)

This mirrors incubator changes from
https://github.com/llvm/clangir/pull/1570
---
 clang/include/clang/CIR/Dialect/IR/CIRTypes.td      | 8 ++++++++
 clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp        | 3 +--
 clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp | 3 +--
 clang/lib/CIR/CodeGen/CIRGenTypes.cpp               | 3 +--
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
index b028bc7db4e59..a552b6081f5dc 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
@@ -294,6 +294,14 @@ def CIR_ArrayType : CIR_Type<"Array", "array",
 
   let parameters = (ins "mlir::Type":$eltType, "uint64_t":$size);
 
+  let builders = [
+    TypeBuilderWithInferredContext<(ins
+      "mlir::Type":$eltType, "uint64_t":$size
+    ), [{
+        return $_get(eltType.getContext(), eltType, size);
+    }]>,
+  ];
+
   let assemblyFormat = [{
     `<` $eltType `x` $size `>`
   }];
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index 50fa029851f33..0caa8961ed0a6 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -206,8 +206,7 @@ emitArrayConstant(CIRGenModule &cgm, mlir::Type desiredType,
       eles.push_back(element);
 
     return cir::ConstArrayAttr::get(
-        cir::ArrayType::get(builder.getContext(), commonElementType,
-                            arrayBound),
+        cir::ArrayType::get(commonElementType, arrayBound),
         mlir::ArrayAttr::get(builder.getContext(), eles));
   }
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
index 5e209b5b92503..83aba256cd48e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
@@ -102,8 +102,7 @@ struct CIRRecordLowering final {
     mlir::Type type = getCharType();
     return numberOfChars == CharUnits::One()
                ? type
-               : cir::ArrayType::get(type.getContext(), type,
-                                     numberOfChars.getQuantity());
+               : cir::ArrayType::get(type, numberOfChars.getQuantity());
   }
 
   mlir::Type getStorageType(const FieldDecl *fieldDecl) {
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index 7bd86cf0c7bcd..c286aef360b01 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -395,8 +395,7 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
   case Type::ConstantArray: {
     const ConstantArrayType *arrTy = cast<ConstantArrayType>(ty);
     mlir::Type elemTy = convertTypeForMem(arrTy->getElementType());
-    resultType = cir::ArrayType::get(builder.getContext(), elemTy,
-                                     arrTy->getSize().getZExtValue());
+    resultType = cir::ArrayType::get(elemTy, arrTy->getSize().getZExtValue());
     break;
   }
 

From e2944b5897e70f17edb366c1ce10a2479ddc4bda Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 18 Apr 2025 23:05:25 -0700
Subject: [PATCH 537/710] [RISCV] Rename
 InvalidRegClassGPRX0->InvalidRegClassGPRNoX0. NFC

---
 llvm/lib/Target/RISCV/RISCVRegisterInfo.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index 8d09caf1da2d5..eb3d5e553f1ef 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -257,7 +257,7 @@ def GPRX5 : GPRRegisterClass<(add X5)> {
 }
 
 def GPRNoX0 : GPRRegisterClass<(sub GPR, X0)> {
-  let DiagnosticType = "InvalidRegClassGPRX0";
+  let DiagnosticType = "InvalidRegClassGPRNoX0";
   let DiagnosticString = "register must be a GPR excluding zero (x0)";
 }
 

From 1c4ff5128a03772572dbaf021e93d6dadb17f898 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Sat, 19 Apr 2025 03:50:30 -0300
Subject: [PATCH 538/710] [ci] add dependencies for lldb python binding tests
 (#136158)

---
 .ci/monolithic-linux.sh    | 7 +++++++
 lldb/test/requirements.txt | 2 ++
 2 files changed, 9 insertions(+)

diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index de399e605cf9b..a56a8bcbffecb 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -67,9 +67,13 @@ pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
 export LLVM_SYMBOLIZER_PATH=`which llvm-symbolizer`
 [[ ! -f "${LLVM_SYMBOLIZER_PATH}" ]] && echo "llvm-symbolizer not found!"
 
+# Set up all runtimes either way. libcxx is a dependency of LLDB.
+# It will not be built unless it is used.
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
+      -D LLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
       -G Ninja \
+      -D CMAKE_PREFIX_PATH="${HOME}/.local" \
       -D CMAKE_BUILD_TYPE=Release \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
@@ -78,7 +82,10 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_LLD=ON \
       -D CMAKE_CXX_FLAGS=-gmlt \
       -D LLVM_CCACHE_BUILD=ON \
+      -D LIBCXX_CXX_ABI=libcxxabi \
       -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
+      -D LLDB_ENABLE_PYTHON=ON \
+      -D LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=ON \
       -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
 
 echo "--- ninja"
diff --git a/lldb/test/requirements.txt b/lldb/test/requirements.txt
index 708f2327a6a04..e2438bf70b6e5 100644
--- a/lldb/test/requirements.txt
+++ b/lldb/test/requirements.txt
@@ -5,3 +5,5 @@ psutil>=5.9.4
 # See llvm.org/pr22274.
 pexpect>=4.9.0; sys_platform != 'win32'
 packaging
+# Required for python tests
+swig>=4.0.0

From d4340978d153fc7ea6445da793320b4104062b12 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 19 Apr 2025 09:26:15 +0200
Subject: [PATCH 539/710] X86: Avoid using getNumUses (#136348)

---
 llvm/lib/Target/X86/X86LowerAMXType.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index 54f5977fe76eb..77253835b0590 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -1084,7 +1084,7 @@ bool X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
   assert(Tile->getType()->isX86_AMXTy() && "Not Tile Operand!");
 
   // TODO: Specially handle the multi-use case.
-  if (Tile->getNumUses() != 1)
+  if (!Tile->hasOneUse())
     return false;
 
   // We don't fetch shape from tilestore, we only get shape from tiledef,

From bbd1a153f521e7faf73e04c95322d7ca2b790609 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 19 Apr 2025 09:27:21 +0200
Subject: [PATCH 540/710] SPIRV: Use use_empty instead of getNumUses == 0
 (#136338)

---
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 0067d2400529a..a7e4487f6f5d5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -1829,7 +1829,7 @@ void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
                                        {GV.getType(), Ty}, {&GV, Const});
     InitInst->setArgOperand(1, Init);
   }
-  if (!Init && GV.getNumUses() == 0)
+  if (!Init && GV.use_empty())
     B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV);
 }
 

From 90de46c51b52226f41354da6d6c488500632f566 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 00:39:40 -0700
Subject: [PATCH 541/710] [clang-doc] Fix warnings

This patch fixes:

  clang-tools-extra/clang-doc/BitcodeReader.cpp:47:20: error: unused
  function 'decodeRecord' [-Werror,-Wunused-function]

  clang-tools-extra/clang-doc/BitcodeReader.cpp:437:31: error: unused
  function 'getCommentInfo<std::unique_ptr<clang::doc::CommentInfo>
  &>' [-Werror,-Wunused-function]
---
 clang-tools-extra/clang-doc/BitcodeReader.cpp | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index 2e064acb5dd76..92674c5507e74 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -44,15 +44,6 @@ static llvm::Error decodeRecord(const Record &R, bool &Field,
   return llvm::Error::success();
 }
 
-static llvm::Error decodeRecord(const Record &R, int &Field,
-                                llvm::StringRef Blob) {
-  if (R[0] > INT_MAX)
-    return llvm::createStringError(llvm::inconvertibleErrorCode(),
-                                   "integer too large to parse");
-  Field = static_cast<int>(R[0]);
-  return llvm::Error::success();
-}
-
 static llvm::Error decodeRecord(const Record &R, AccessSpecifier &Field,
                                 llvm::StringRef Blob) {
   switch (R[0]) {
@@ -433,11 +424,6 @@ template <> llvm::Expected<CommentInfo *> getCommentInfo(CommentInfo *I) {
   return I->Children.back().get();
 }
 
-template <>
-llvm::Expected<CommentInfo *> getCommentInfo(std::unique_ptr<CommentInfo> &I) {
-  return getCommentInfo(I.get());
-}
-
 // When readSubBlock encounters a TypeInfo sub-block, it calls addTypeInfo on
 // the parent block to set it. The template specializations define what to do
 // for each supported parent block.

From c47042c5b3f5e87761715a9277712721512cd48d Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Sat, 19 Apr 2025 10:01:50 +0200
Subject: [PATCH 542/710] [mlir][SPRIV][NFC] Avoid rollback in
 `TypeCastingOpPattern` (#136284)

This pattern used to create an op and then attached the converted
rounding mode attribute. When the latter failed, the pattern aborted and
a rollback was triggered.

This commit inverses the logic: the converted rounding mode is computed
first, so that no changes have to be rolled back.

Note: This is in preparation of the One-Shot Dialect Conversion
refactoring.
---
 .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp  | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index 9c4dfa27b1447..434d7df853a5e 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -847,24 +847,28 @@ struct TypeCastingOpPattern final : public OpConversionPattern<Op> {
       // Then we can just erase this operation by forwarding its operand.
       rewriter.replaceOp(op, adaptor.getOperands().front());
     } else {
-      auto newOp = rewriter.template replaceOpWithNewOp<SPIRVOp>(
-          op, dstType, adaptor.getOperands());
+      // Compute new rounding mode (if any).
+      std::optional<spirv::FPRoundingMode> rm = std::nullopt;
       if (auto roundingModeOp =
               dyn_cast<arith::ArithRoundingModeInterface>(*op)) {
         if (arith::RoundingModeAttr roundingMode =
                 roundingModeOp.getRoundingModeAttr()) {
-          if (auto rm =
-                  convertArithRoundingModeToSPIRV(roundingMode.getValue())) {
-            newOp->setAttr(
-                getDecorationString(spirv::Decoration::FPRoundingMode),
-                spirv::FPRoundingModeAttr::get(rewriter.getContext(), *rm));
-          } else {
+          if (!(rm =
+                    convertArithRoundingModeToSPIRV(roundingMode.getValue()))) {
             return rewriter.notifyMatchFailure(
                 op->getLoc(),
                 llvm::formatv("unsupported rounding mode '{0}'", roundingMode));
           }
         }
       }
+      // Create replacement op and attach rounding mode attribute (if any).
+      auto newOp = rewriter.template replaceOpWithNewOp<SPIRVOp>(
+          op, dstType, adaptor.getOperands());
+      if (rm) {
+        newOp->setAttr(
+            getDecorationString(spirv::Decoration::FPRoundingMode),
+            spirv::FPRoundingModeAttr::get(rewriter.getContext(), *rm));
+      }
     }
     return success();
   }

From f91df0d58df4fcb5edcb89f204f0d22ade9a9262 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sat, 19 Apr 2025 11:34:25 +0200
Subject: [PATCH 543/710] [clang][bytecode] Diagnose failed MemberPtrPtr casts
 differently (#136407)

Return Invalid() here instead of just false to match the diagnostic
output of the current interpreter.
---
 clang/lib/AST/ByteCode/Interp.h            |  2 +-
 clang/test/AST/ByteCode/memberpointers.cpp | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 49e2326186bf8..866d8e4bf2251 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -1932,7 +1932,7 @@ inline bool CastMemberPtrPtr(InterpState &S, CodePtr OpPC) {
     S.Stk.push<Pointer>(*Ptr);
     return true;
   }
-  return false;
+  return Invalid(S, OpPC);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/test/AST/ByteCode/memberpointers.cpp b/clang/test/AST/ByteCode/memberpointers.cpp
index db4a86ae03aff..5d622187e97f2 100644
--- a/clang/test/AST/ByteCode/memberpointers.cpp
+++ b/clang/test/AST/ByteCode/memberpointers.cpp
@@ -249,3 +249,21 @@ namespace CallExprTypeMismatch {
   }
   static_assert(test(), "");
 }
+
+namespace CastMemberPtrPtrFailed{
+  struct S {
+    constexpr S() {}
+    constexpr int f() const;
+    constexpr int g() const;
+  };
+  struct T : S {
+    constexpr T(int n) : S(), n(n) {}
+    int n;
+  };
+
+  constexpr int S::g() const {
+    return this->*(int(S::*))&T::n; // both-note {{subexpression}}
+  }
+  static_assert(S().g(), ""); // both-error {{constant expression}} \
+                              // both-note {{in call to 'S().g()'}}
+}

From e9280a1d39af88468ffea9a14fad5bf96d51d6e6 Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245@hotmail.com>
Date: Sat, 19 Apr 2025 07:12:43 -0400
Subject: [PATCH 544/710] [libc++] Backport segmented iterator optimization for
 std::for_each to C++11 (#134960)

Previously, the segmented iterator optimization for `std::for_each` was restricted to C++23 and later due to its dependency on `__movable_box`, which is not available in earlier standards. This patch eliminates that restriction, enabling consistent optimizations starting from C++11.

By backporting this enhancement, we improve performance across older standards and create opportunities to extend similar optimizations to other algorithms by forwarding their calls to `std::for_each`.
---
 libcxx/docs/ReleaseNotes/21.rst               |   3 +
 libcxx/include/__algorithm/for_each.h         |  44 +-
 libcxx/include/algorithm                      |   1 +
 libcxx/include/array                          |   1 +
 libcxx/include/bitset                         |   1 +
 libcxx/include/codecvt                        |   1 +
 libcxx/include/condition_variable             |   1 +
 libcxx/include/ios                            |   1 +
 libcxx/include/locale                         |   1 +
 libcxx/include/streambuf                      |   1 +
 libcxx/include/string                         |   1 +
 libcxx/include/string_view                    |   1 +
 libcxx/include/system_error                   |   1 +
 libcxx/include/vector                         |   1 +
 ...obust_against_copying_comparators.pass.cpp | 381 +++++++++++-------
 15 files changed, 277 insertions(+), 163 deletions(-)

diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst
index 2091a713ea200..a7382c5222d08 100644
--- a/libcxx/docs/ReleaseNotes/21.rst
+++ b/libcxx/docs/ReleaseNotes/21.rst
@@ -67,6 +67,9 @@ Improvements and New Features
 - The ``std::stable_sort`` algorithm uses radix sort for floating-point types now, which can improve the performance
   up to 10x, depending on type of sorted elements and the initial state of the sorted array.
 
+- The segmented iterator optimization for ``std::for_each`` has been backported to C++11. Previously it was only available
+  in C++23 and later.
+
 Deprecations and Removals
 -------------------------
 
diff --git a/libcxx/include/__algorithm/for_each.h b/libcxx/include/__algorithm/for_each.h
index e08f583504c01..b6c2c7c056edd 100644
--- a/libcxx/include/__algorithm/for_each.h
+++ b/libcxx/include/__algorithm/for_each.h
@@ -13,44 +13,40 @@
 #include <__algorithm/for_each_segment.h>
 #include <__config>
 #include <__iterator/segmented_iterator.h>
-#include <__ranges/movable_box.h>
-#include <__utility/in_place.h>
-#include <__utility/move.h>
+#include <__type_traits/enable_if.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
-_LIBCPP_PUSH_MACROS
-#include <__undef_macros>
-
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _InputIterator, class _Function>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Function
-for_each(_InputIterator __first, _InputIterator __last, _Function __f) {
+template <class _InputIterator, class _Sent, class _Func>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __for_each(_InputIterator __first, _Sent __last, _Func& __f) {
   for (; __first != __last; ++__first)
     __f(*__first);
-  return __f;
 }
 
-// __movable_box is available in C++20, but is actually a copyable-box, so optimization is only correct in C++23
-#if _LIBCPP_STD_VER >= 23
-template <class _SegmentedIterator, class _Function>
-  requires __is_segmented_iterator<_SegmentedIterator>::value
-_LIBCPP_HIDE_FROM_ABI constexpr _Function
-for_each(_SegmentedIterator __first, _SegmentedIterator __last, _Function __func) {
-  ranges::__movable_box<_Function> __wrapped_func(in_place, std::move(__func));
-  std::__for_each_segment(__first, __last, [&](auto __lfirst, auto __llast) {
-    __wrapped_func =
-        ranges::__movable_box<_Function>(in_place, std::for_each(__lfirst, __llast, std::move(*__wrapped_func)));
+#ifndef _LIBCPP_CXX03_LANG
+template <class _SegmentedIterator,
+          class _Function,
+          __enable_if_t<__is_segmented_iterator<_SegmentedIterator>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
+__for_each(_SegmentedIterator __first, _SegmentedIterator __last, _Function& __func) {
+  using __local_iterator_t = typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator;
+  std::__for_each_segment(__first, __last, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
+    std::__for_each(__lfirst, __llast, __func);
   });
-  return std::move(*__wrapped_func);
 }
-#endif // _LIBCPP_STD_VER >= 23
+#endif // !_LIBCPP_CXX03_LANG
 
-_LIBCPP_END_NAMESPACE_STD
+template <class _InputIterator, class _Function>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Function
+for_each(_InputIterator __first, _InputIterator __last, _Function __f) {
+  std::__for_each(__first, __last, __f);
+  return __f;
+}
 
-_LIBCPP_POP_MACROS
+_LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___ALGORITHM_FOR_EACH_H
diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index bf67d3363a595..64e9fa6306145 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -2061,6 +2061,7 @@ template <class BidirectionalIterator, class Compare>
 #    include <cstring>
 #    include <iterator>
 #    include <memory>
+#    include <optional>
 #    include <stdexcept>
 #    include <type_traits>
 #    include <utility>
diff --git a/libcxx/include/array b/libcxx/include/array
index 6e3a1d82abb1b..ff46838e2e8e2 100644
--- a/libcxx/include/array
+++ b/libcxx/include/array
@@ -566,6 +566,7 @@ _LIBCPP_POP_MACROS
 #    include <cstdlib>
 #    include <iterator>
 #    include <new>
+#    include <optional>
 #    include <type_traits>
 #    include <utility>
 #  endif
diff --git a/libcxx/include/bitset b/libcxx/include/bitset
index c403c533db7e9..9273ccabbb4e3 100644
--- a/libcxx/include/bitset
+++ b/libcxx/include/bitset
@@ -973,6 +973,7 @@ _LIBCPP_POP_MACROS
 #  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #    include <concepts>
 #    include <cstdlib>
+#    include <optional>
 #    include <type_traits>
 #  endif
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
diff --git a/libcxx/include/codecvt b/libcxx/include/codecvt
index 9f241b734d694..33ade1d298a7e 100644
--- a/libcxx/include/codecvt
+++ b/libcxx/include/codecvt
@@ -596,6 +596,7 @@ _LIBCPP_END_NAMESPACE_STD
 #    include <limits>
 #    include <mutex>
 #    include <new>
+#    include <optional>
 #    include <stdexcept>
 #    include <type_traits>
 #    include <typeinfo>
diff --git a/libcxx/include/condition_variable b/libcxx/include/condition_variable
index 7f44990547f55..99c74b02807ae 100644
--- a/libcxx/include/condition_variable
+++ b/libcxx/include/condition_variable
@@ -357,6 +357,7 @@ _LIBCPP_POP_MACROS
 #    include <initializer_list>
 #    include <iosfwd>
 #    include <new>
+#    include <optional>
 #    include <stdexcept>
 #    include <system_error>
 #    include <type_traits>
diff --git a/libcxx/include/ios b/libcxx/include/ios
index 9d2968753c507..9e48ec88ce59d 100644
--- a/libcxx/include/ios
+++ b/libcxx/include/ios
@@ -887,6 +887,7 @@ _LIBCPP_POP_MACROS
 #    include <limits>
 #    include <mutex>
 #    include <new>
+#    include <optional>
 #    include <stdexcept>
 #    include <system_error>
 #    include <type_traits>
diff --git a/libcxx/include/locale b/libcxx/include/locale
index 26495497d660f..fa2620d883598 100644
--- a/libcxx/include/locale
+++ b/libcxx/include/locale
@@ -3692,6 +3692,7 @@ _LIBCPP_POP_MACROS
 #    include <cstdarg>
 #    include <iterator>
 #    include <mutex>
+#    include <optional>
 #    include <stdexcept>
 #    include <type_traits>
 #    include <typeinfo>
diff --git a/libcxx/include/streambuf b/libcxx/include/streambuf
index eed7eb4d75ecb..85f3af1b88ae7 100644
--- a/libcxx/include/streambuf
+++ b/libcxx/include/streambuf
@@ -386,6 +386,7 @@ _LIBCPP_POP_MACROS
 
 #  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #    include <cstdint>
+#    include <optional>
 #  endif
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
diff --git a/libcxx/include/string b/libcxx/include/string
index e7e541e31432d..f3f97655d79ea 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -4027,6 +4027,7 @@ _LIBCPP_POP_MACROS
 #    include <cstdlib>
 #    include <iterator>
 #    include <new>
+#    include <optional>
 #    include <type_traits>
 #    include <typeinfo>
 #    include <utility>
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index 5054b14efd2d5..861187c0640e1 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -952,6 +952,7 @@ _LIBCPP_POP_MACROS
 #    include <concepts>
 #    include <cstdlib>
 #    include <iterator>
+#    include <optional>
 #    include <type_traits>
 #  endif
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
diff --git a/libcxx/include/system_error b/libcxx/include/system_error
index 4dadc0a6ab483..2b668e5f8f1bc 100644
--- a/libcxx/include/system_error
+++ b/libcxx/include/system_error
@@ -168,6 +168,7 @@ template <> struct hash<std::error_condition>;
 #    include <cstdint>
 #    include <cstring>
 #    include <limits>
+#    include <optional>
 #    include <type_traits>
 #  endif
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
diff --git a/libcxx/include/vector b/libcxx/include/vector
index 9fa81dcb7e76e..d2d5fcf4a3199 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -362,6 +362,7 @@ template<class T, class charT> requires is-vector-bool-reference<T> // Since C++
 #    if _LIBCPP_HAS_LOCALIZATION
 #      include <locale>
 #    endif
+#    include <optional>
 #    include <string>
 #    include <string_view>
 #    include <tuple>
diff --git a/libcxx/test/libcxx/algorithms/robust_against_copying_comparators.pass.cpp b/libcxx/test/libcxx/algorithms/robust_against_copying_comparators.pass.cpp
index df656f296bd05..256251686bb3e 100644
--- a/libcxx/test/libcxx/algorithms/robust_against_copying_comparators.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/robust_against_copying_comparators.pass.cpp
@@ -12,209 +12,312 @@
 #include <cassert>
 #include <compare>
 #include <cstddef>
+#include <deque>
+#include <ranges>
 #include <type_traits>
+#include <vector>
 
 #include "test_macros.h"
 
 template <class T>
 struct Less {
-    int *copies_;
-    TEST_CONSTEXPR explicit Less(int *copies) : copies_(copies) {}
-    TEST_CONSTEXPR_CXX14 Less(const Less& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
-    TEST_CONSTEXPR_CXX14 Less& operator=(const Less&) = default;
-    TEST_CONSTEXPR bool operator()(T, T) const { return false; }
+  int* copies_;
+  TEST_CONSTEXPR explicit Less(int* copies) : copies_(copies) {}
+  TEST_CONSTEXPR_CXX14 Less(const Less& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
+  TEST_CONSTEXPR_CXX14 Less& operator=(const Less&) = default;
+  TEST_CONSTEXPR bool operator()(T, T) const { return false; }
 };
 
 template <class T>
 struct Equal {
-    int *copies_;
-    TEST_CONSTEXPR explicit Equal(int *copies) : copies_(copies) {}
-    TEST_CONSTEXPR_CXX14 Equal(const Equal& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
-    TEST_CONSTEXPR_CXX14 Equal& operator=(const Equal&) = default;
-    TEST_CONSTEXPR bool operator()(T, T) const { return true; }
+  int* copies_;
+  TEST_CONSTEXPR explicit Equal(int* copies) : copies_(copies) {}
+  TEST_CONSTEXPR_CXX14 Equal(const Equal& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
+  TEST_CONSTEXPR_CXX14 Equal& operator=(const Equal&) = default;
+  TEST_CONSTEXPR bool operator()(T, T) const { return true; }
 };
 
 template <class T>
 struct UnaryVoid {
-    int *copies_;
-    TEST_CONSTEXPR explicit UnaryVoid(int *copies) : copies_(copies) {}
-    TEST_CONSTEXPR_CXX14 UnaryVoid(const UnaryVoid& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
-    TEST_CONSTEXPR_CXX14 UnaryVoid& operator=(const UnaryVoid&) = default;
-    TEST_CONSTEXPR_CXX14 void operator()(T) const {}
+  int* copies_;
+  TEST_CONSTEXPR explicit UnaryVoid(int* copies) : copies_(copies) {}
+  TEST_CONSTEXPR_CXX14 UnaryVoid(const UnaryVoid& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
+  TEST_CONSTEXPR_CXX14 UnaryVoid& operator=(const UnaryVoid&) = default;
+  TEST_CONSTEXPR_CXX14 void operator()(T) const {}
 };
 
 template <class T>
 struct UnaryTrue {
-    int *copies_;
-    TEST_CONSTEXPR explicit UnaryTrue(int *copies) : copies_(copies) {}
-    TEST_CONSTEXPR_CXX14 UnaryTrue(const UnaryTrue& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
-    TEST_CONSTEXPR_CXX14 UnaryTrue& operator=(const UnaryTrue&) = default;
-    TEST_CONSTEXPR bool operator()(T) const { return true; }
+  int* copies_;
+  TEST_CONSTEXPR explicit UnaryTrue(int* copies) : copies_(copies) {}
+  TEST_CONSTEXPR_CXX14 UnaryTrue(const UnaryTrue& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
+  TEST_CONSTEXPR_CXX14 UnaryTrue& operator=(const UnaryTrue&) = default;
+  TEST_CONSTEXPR bool operator()(T) const { return true; }
 };
 
 template <class T>
 struct NullaryValue {
-    int *copies_;
-    TEST_CONSTEXPR explicit NullaryValue(int *copies) : copies_(copies) {}
-    TEST_CONSTEXPR_CXX14 NullaryValue(const NullaryValue& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
-    TEST_CONSTEXPR_CXX14 NullaryValue& operator=(const NullaryValue&) = default;
-    TEST_CONSTEXPR T operator()() const { return 0; }
+  int* copies_;
+  TEST_CONSTEXPR explicit NullaryValue(int* copies) : copies_(copies) {}
+  TEST_CONSTEXPR_CXX14 NullaryValue(const NullaryValue& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
+  TEST_CONSTEXPR_CXX14 NullaryValue& operator=(const NullaryValue&) = default;
+  TEST_CONSTEXPR T operator()() const { return 0; }
 };
 
 template <class T>
 struct UnaryTransform {
-    int *copies_;
-    TEST_CONSTEXPR explicit UnaryTransform(int *copies) : copies_(copies) {}
-    TEST_CONSTEXPR_CXX14 UnaryTransform(const UnaryTransform& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
-    TEST_CONSTEXPR_CXX14 UnaryTransform& operator=(const UnaryTransform&) = default;
-    TEST_CONSTEXPR T operator()(T) const { return 0; }
+  int* copies_;
+  TEST_CONSTEXPR explicit UnaryTransform(int* copies) : copies_(copies) {}
+  TEST_CONSTEXPR_CXX14 UnaryTransform(const UnaryTransform& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
+  TEST_CONSTEXPR_CXX14 UnaryTransform& operator=(const UnaryTransform&) = default;
+  TEST_CONSTEXPR T operator()(T) const { return 0; }
 };
 
 template <class T>
 struct BinaryTransform {
-    int *copies_;
-    TEST_CONSTEXPR explicit BinaryTransform(int *copies) : copies_(copies) {}
-    TEST_CONSTEXPR_CXX14 BinaryTransform(const BinaryTransform& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
-    TEST_CONSTEXPR_CXX14 BinaryTransform& operator=(const BinaryTransform&) = default;
-    TEST_CONSTEXPR T operator()(T, T) const { return 0; }
+  int* copies_;
+  TEST_CONSTEXPR explicit BinaryTransform(int* copies) : copies_(copies) {}
+  TEST_CONSTEXPR_CXX14 BinaryTransform(const BinaryTransform& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
+  TEST_CONSTEXPR_CXX14 BinaryTransform& operator=(const BinaryTransform&) = default;
+  TEST_CONSTEXPR T operator()(T, T) const { return 0; }
 };
 
 #if TEST_STD_VER > 17
 template <class T>
 struct ThreeWay {
-    int *copies_;
-    constexpr explicit ThreeWay(int *copies) : copies_(copies) {}
-    constexpr ThreeWay(const ThreeWay& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
-    constexpr ThreeWay& operator=(const ThreeWay&) = default;
-    constexpr std::strong_ordering operator()(T, T) const { return std::strong_ordering::equal; }
+  int* copies_;
+  constexpr explicit ThreeWay(int* copies) : copies_(copies) {}
+  constexpr ThreeWay(const ThreeWay& rhs) : copies_(rhs.copies_) { *copies_ += 1; }
+  constexpr ThreeWay& operator=(const ThreeWay&) = default;
+  constexpr std::strong_ordering operator()(T, T) const { return std::strong_ordering::equal; }
 };
 #endif
 
 template <class T>
-TEST_CONSTEXPR_CXX20 bool all_the_algorithms()
-{
-    T a[10] = {};
-    T b[10] = {};
-    T *first = a;
-    T *mid = a+5;
-    T *last = a+10;
-    T *first2 = b;
-    T *mid2 = b+5;
-    T *last2 = b+10;
-    T value = 0;
-    int count = 1;
-
-    int copies = 0;
-    (void)std::adjacent_find(first, last, Equal<T>(&copies)); assert(copies == 0);
+TEST_CONSTEXPR_CXX20 bool all_the_algorithms() {
+  T a[10]   = {};
+  T b[10]   = {};
+  T* first  = a;
+  T* mid    = a + 5;
+  T* last   = a + 10;
+  T* first2 = b;
+  T* mid2   = b + 5;
+  T* last2  = b + 10;
+  T value   = 0;
+  int count = 1;
+
+  int copies = 0;
+  (void)std::adjacent_find(first, last, Equal<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER >= 11
-    (void)std::all_of(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::any_of(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
+  (void)std::all_of(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::any_of(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::binary_search(first, last, value, Less<T>(&copies)); assert(copies == 0);
+  (void)std::binary_search(first, last, value, Less<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER > 17
-    (void)std::clamp(value, value, value, Less<T>(&copies)); assert(copies == 0);
+  (void)std::clamp(value, value, value, Less<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::count_if(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::copy_if(first, last, first2, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::equal(first, last, first2, Equal<T>(&copies)); assert(copies == 0);
+  (void)std::count_if(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::copy_if(first, last, first2, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::equal(first, last, first2, Equal<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER > 11
-    (void)std::equal(first, last, first2, last2, Equal<T>(&copies)); assert(copies == 0);
+  (void)std::equal(first, last, first2, last2, Equal<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::equal_range(first, last, value, Less<T>(&copies)); assert(copies == 0);
-    (void)std::find_end(first, last, first2, mid2, Equal<T>(&copies)); assert(copies == 0);
-    (void)std::find_first_of(first, last, first2, last2, Equal<T>(&copies)); assert(copies == 0);
-    (void)std::find_if(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::find_if_not(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::for_each(first, last, UnaryVoid<T>(&copies)); assert(copies == 1); copies = 0;
+  (void)std::equal_range(first, last, value, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::find_end(first, last, first2, mid2, Equal<T>(&copies));
+  assert(copies == 0);
+  (void)std::find_first_of(first, last, first2, last2, Equal<T>(&copies));
+  assert(copies == 0);
+  (void)std::find_if(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::find_if_not(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::for_each(first, last, UnaryVoid<T>(&copies));
+  assert(copies == 1);
+  copies = 0;
 #if TEST_STD_VER > 14
-    (void)std::for_each_n(first, count, UnaryVoid<T>(&copies)); assert(copies == 0);
+  (void)std::for_each_n(first, count, UnaryVoid<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::generate(first, last, NullaryValue<T>(&copies)); assert(copies == 0);
-    (void)std::generate_n(first, count, NullaryValue<T>(&copies)); assert(copies == 0);
-    (void)std::includes(first, last, first2, last2, Less<T>(&copies)); assert(copies == 0);
-    (void)std::is_heap(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::is_heap_until(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::is_partitioned(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::is_permutation(first, last, first2, Equal<T>(&copies)); assert(copies == 0);
+  (void)std::generate(first, last, NullaryValue<T>(&copies));
+  assert(copies == 0);
+  (void)std::generate_n(first, count, NullaryValue<T>(&copies));
+  assert(copies == 0);
+  (void)std::includes(first, last, first2, last2, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::is_heap(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::is_heap_until(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::is_partitioned(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::is_permutation(first, last, first2, Equal<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER > 11
-    (void)std::is_permutation(first, last, first2, last2, Equal<T>(&copies)); assert(copies == 0);
+  (void)std::is_permutation(first, last, first2, last2, Equal<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::is_sorted(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::is_sorted_until(first, last, Less<T>(&copies)); assert(copies == 0);
-    if (!TEST_IS_CONSTANT_EVALUATED) { (void)std::inplace_merge(first, mid, last, Less<T>(&copies)); assert(copies == 0); }
-    (void)std::lexicographical_compare(first, last, first2, last2, Less<T>(&copies)); assert(copies == 0);
+  (void)std::is_sorted(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::is_sorted_until(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    (void)std::inplace_merge(first, mid, last, Less<T>(&copies));
+    assert(copies == 0);
+  }
+  (void)std::lexicographical_compare(first, last, first2, last2, Less<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER > 17
-    (void)std::lexicographical_compare_three_way(first, last, first2, last2, ThreeWay<T>(&copies)); assert(copies == 0);
+  (void)std::lexicographical_compare_three_way(first, last, first2, last2, ThreeWay<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::lower_bound(first, last, value, Less<T>(&copies)); assert(copies == 0);
-    (void)std::make_heap(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::max(value, value, Less<T>(&copies)); assert(copies == 0);
+  (void)std::lower_bound(first, last, value, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::make_heap(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::max(value, value, Less<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER >= 11
-    (void)std::max({ value, value }, Less<T>(&copies)); assert(copies == 0);
+  (void)std::max({value, value}, Less<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::max_element(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::merge(first, mid, mid, last, first2, Less<T>(&copies)); assert(copies == 0);
-    (void)std::min(value, value, Less<T>(&copies)); assert(copies == 0);
+  (void)std::max_element(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::merge(first, mid, mid, last, first2, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::min(value, value, Less<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER >= 11
-    (void)std::min({ value, value }, Less<T>(&copies)); assert(copies == 0);
+  (void)std::min({value, value}, Less<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::min_element(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::minmax(value, value, Less<T>(&copies)); assert(copies == 0);
+  (void)std::min_element(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::minmax(value, value, Less<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER >= 11
-    (void)std::minmax({ value, value }, Less<T>(&copies)); assert(copies == 0);
+  (void)std::minmax({value, value}, Less<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::minmax_element(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::mismatch(first, last, first2, Equal<T>(&copies)); assert(copies == 0);
+  (void)std::minmax_element(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::mismatch(first, last, first2, Equal<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER > 11
-    (void)std::mismatch(first, last, first2, last2, Equal<T>(&copies)); assert(copies == 0);
+  (void)std::mismatch(first, last, first2, last2, Equal<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::next_permutation(first, last, Less<T>(&copies)); assert(copies == 0);
+  (void)std::next_permutation(first, last, Less<T>(&copies));
+  assert(copies == 0);
 #if TEST_STD_VER >= 11
-    (void)std::none_of(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
+  (void)std::none_of(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
 #endif
-    (void)std::nth_element(first, mid, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::partial_sort(first, mid, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::partial_sort_copy(first, last, first2, mid2, Less<T>(&copies)); assert(copies == 0);
-    (void)std::partition(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::partition_copy(first, last, first2, last2, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::partition_point(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::pop_heap(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::prev_permutation(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::push_heap(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::remove_copy_if(first, last, first2, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::remove_if(first, last, UnaryTrue<T>(&copies)); assert(copies == 0);
-    (void)std::replace_copy_if(first, last, first2, UnaryTrue<T>(&copies), value); assert(copies == 0);
-    (void)std::replace_if(first, last, UnaryTrue<T>(&copies), value); assert(copies == 0);
-    (void)std::search(first, last, first2, mid2, Equal<T>(&copies)); assert(copies == 0);
-    (void)std::search_n(first, last, count, value, Equal<T>(&copies)); assert(copies == 0);
-    (void)std::set_difference(first, mid, mid, last, first2, Less<T>(&copies)); assert(copies == 0);
-    (void)std::set_intersection(first, mid, mid, last, first2, Less<T>(&copies)); assert(copies == 0);
-    (void)std::set_symmetric_difference(first, mid, mid, last, first2, Less<T>(&copies)); assert(copies == 0);
-    (void)std::set_union(first, mid, mid, last, first2, Less<T>(&copies)); assert(copies == 0);
-    (void)std::sort(first, first+3, Less<T>(&copies)); assert(copies == 0);
-    (void)std::sort(first, first+4, Less<T>(&copies)); assert(copies == 0);
-    (void)std::sort(first, first+5, Less<T>(&copies)); assert(copies == 0);
-    (void)std::sort(first, last, Less<T>(&copies)); assert(copies == 0);
-    (void)std::sort_heap(first, last, Less<T>(&copies)); assert(copies == 0);
-    if (!TEST_IS_CONSTANT_EVALUATED) { (void)std::stable_partition(first, last, UnaryTrue<T>(&copies)); assert(copies == 0); }
-    if (!TEST_IS_CONSTANT_EVALUATED) { (void)std::stable_sort(first, last, Less<T>(&copies)); assert(copies == 0); }
-    (void)std::transform(first, last, first2, UnaryTransform<T>(&copies)); assert(copies == 0);
-    (void)std::transform(first, mid, mid, first2, BinaryTransform<T>(&copies)); assert(copies == 0);
-    (void)std::unique(first, last, Equal<T>(&copies)); assert(copies == 0);
-    (void)std::unique_copy(first, last, first2, Equal<T>(&copies)); assert(copies == 0);
-    (void)std::upper_bound(first, last, value, Less<T>(&copies)); assert(copies == 0);
-
-    return true;
+  (void)std::nth_element(first, mid, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::partial_sort(first, mid, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::partial_sort_copy(first, last, first2, mid2, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::partition(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::partition_copy(first, last, first2, last2, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::partition_point(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::pop_heap(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::prev_permutation(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::push_heap(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::remove_copy_if(first, last, first2, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::remove_if(first, last, UnaryTrue<T>(&copies));
+  assert(copies == 0);
+  (void)std::replace_copy_if(first, last, first2, UnaryTrue<T>(&copies), value);
+  assert(copies == 0);
+  (void)std::replace_if(first, last, UnaryTrue<T>(&copies), value);
+  assert(copies == 0);
+  (void)std::search(first, last, first2, mid2, Equal<T>(&copies));
+  assert(copies == 0);
+  (void)std::search_n(first, last, count, value, Equal<T>(&copies));
+  assert(copies == 0);
+  (void)std::set_difference(first, mid, mid, last, first2, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::set_intersection(first, mid, mid, last, first2, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::set_symmetric_difference(first, mid, mid, last, first2, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::set_union(first, mid, mid, last, first2, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::sort(first, first + 3, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::sort(first, first + 4, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::sort(first, first + 5, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::sort(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  (void)std::sort_heap(first, last, Less<T>(&copies));
+  assert(copies == 0);
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    (void)std::stable_partition(first, last, UnaryTrue<T>(&copies));
+    assert(copies == 0);
+  }
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    (void)std::stable_sort(first, last, Less<T>(&copies));
+    assert(copies == 0);
+  }
+  (void)std::transform(first, last, first2, UnaryTransform<T>(&copies));
+  assert(copies == 0);
+  (void)std::transform(first, mid, mid, first2, BinaryTransform<T>(&copies));
+  assert(copies == 0);
+  (void)std::unique(first, last, Equal<T>(&copies));
+  assert(copies == 0);
+  (void)std::unique_copy(first, last, first2, Equal<T>(&copies));
+  assert(copies == 0);
+  (void)std::upper_bound(first, last, value, Less<T>(&copies));
+  assert(copies == 0);
+
+  return true;
+}
+
+bool test_segmented_iterator() {
+  int copies = 0;
+  std::deque<int> dq(10);
+  (void)std::for_each(dq.begin(), dq.end(), UnaryVoid<int>(&copies));
+  assert(copies == 1);
+  copies = 0;
+
+#if TEST_STD_VER >= 20
+  std::vector<std::vector<int>> vecs(3, std::vector<int>(10));
+  auto v = std::views::join(vecs);
+  (void)std::for_each(v.begin(), v.end(), UnaryVoid<int>(&copies));
+  assert(copies == 1);
+  copies = 0;
+#endif
+
+  return true;
 }
 
-int main(int, char**)
-{
-    all_the_algorithms<void*>();
-    all_the_algorithms<int>();
+int main(int, char**) {
+  all_the_algorithms<void*>();
+  all_the_algorithms<int>();
+  assert(test_segmented_iterator());
 #if TEST_STD_VER > 17
-    static_assert(all_the_algorithms<void*>());
-    static_assert(all_the_algorithms<int>());
+  static_assert(all_the_algorithms<void*>());
+  static_assert(all_the_algorithms<int>());
 #endif
 
-    return 0;
+  return 0;
 }

From 19c708c18963ac24822564824cb5401e71a49943 Mon Sep 17 00:00:00 2001
From: Robert Dazi <14996868+v01dXYZ@users.noreply.github.com>
Date: Sat, 19 Apr 2025 13:39:18 +0200
Subject: [PATCH 545/710] FunctionDecl::getFunctionTypeLoc: ignore function
 type attributes (#118420)

Related to #118290.

---------

Co-authored-by: v01dxyz <v01dxyz@v01d.xyz>
Co-authored-by: Owen Anderson <resistor@mac.com>
---
 .../clangd/unittests/InlayHintTests.cpp       | 14 ++--
 clang/lib/AST/Decl.cpp                        | 21 ++++-
 clang/unittests/AST/AttrTest.cpp              | 78 +++++++++++++++++++
 3 files changed, 105 insertions(+), 8 deletions(-)

diff --git a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
index 77d78b8777fe3..030e499577706 100644
--- a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
+++ b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
@@ -1577,19 +1577,21 @@ TEST(TypeHints, Aliased) {
 }
 
 TEST(TypeHints, CallingConvention) {
-  // Check that we don't crash for lambdas without a FunctionTypeLoc
+  // Check that we don't crash for lambdas with an annotation
   // https://github.com/clangd/clangd/issues/2223
-  std::string Code = R"cpp(
+  Annotations Source(R"cpp(
     void test() {
-      []() __cdecl {};
+      []($lambda[[)]]__cdecl {};
     }
-  )cpp";
-  TestTU TU = TestTU::withCode(Code);
+  )cpp");
+  TestTU TU = TestTU::withCode(Source.code());
   TU.ExtraArgs.push_back("--target=x86_64-w64-mingw32");
   TU.PredefineMacros = true; // for the __cdecl
   auto AST = TU.build();
 
-  EXPECT_THAT(hintsOfKind(AST, InlayHintKind::Type), IsEmpty());
+  EXPECT_THAT(
+      hintsOfKind(AST, InlayHintKind::Type),
+      ElementsAre(HintMatcher(ExpectedHint{"-> void", "lambda"}, Source)));
 }
 
 TEST(TypeHints, Decltype) {
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index ad1cb01592e9b..1d9208f0e1c72 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -3910,8 +3910,25 @@ bool FunctionDecl::doesDeclarationForceExternallyVisibleDefinition() const {
 
 FunctionTypeLoc FunctionDecl::getFunctionTypeLoc() const {
   const TypeSourceInfo *TSI = getTypeSourceInfo();
-  return TSI ? TSI->getTypeLoc().IgnoreParens().getAs<FunctionTypeLoc>()
-             : FunctionTypeLoc();
+
+  if (!TSI)
+    return FunctionTypeLoc();
+
+  TypeLoc TL = TSI->getTypeLoc();
+  FunctionTypeLoc FTL;
+
+  while (!(FTL = TL.getAs<FunctionTypeLoc>())) {
+    if (const auto PTL = TL.getAs<ParenTypeLoc>())
+      TL = PTL.getInnerLoc();
+    else if (const auto ATL = TL.getAs<AttributedTypeLoc>())
+      TL = ATL.getEquivalentTypeLoc();
+    else if (const auto MQTL = TL.getAs<MacroQualifiedTypeLoc>())
+      TL = MQTL.getInnerLoc();
+    else
+      break;
+  }
+
+  return FTL;
 }
 
 SourceRange FunctionDecl::getReturnTypeSourceRange() const {
diff --git a/clang/unittests/AST/AttrTest.cpp b/clang/unittests/AST/AttrTest.cpp
index 46c3f5729021e..3be362895b77e 100644
--- a/clang/unittests/AST/AttrTest.cpp
+++ b/clang/unittests/AST/AttrTest.cpp
@@ -86,6 +86,23 @@ TEST(Attr, AnnotateType) {
     struct S { int mem; };
     int [[clang::annotate_type("int")]]
     S::* [[clang::annotate_type("ptr_to_mem")]] ptr_to_member = &S::mem;
+
+    // Function Type Attributes
+    __attribute__((noreturn)) int f_noreturn();
+    __attribute__((preserve_most)) int f_cc_preserve_most();
+
+    #define PRESERVE_MOST __attribute__((preserve_most))
+    PRESERVE_MOST int f_macro_attribue();
+
+    int (__attribute__((preserve_most)) f_paren_attribute)();
+
+    int (
+      PRESERVE_MOST
+      (
+        __attribute__((warn_unused_result))
+        (f_w_paren_and_attr)
+      )
+    ) ();
   )cpp");
 
   {
@@ -153,6 +170,67 @@ TEST(Attr, AnnotateType) {
     EXPECT_EQ(IntTL.getType(), AST->getASTContext().IntTy);
   }
 
+  {
+    const FunctionDecl *Func = getFunctionNode(AST.get(), "f_noreturn");
+    const FunctionTypeLoc FTL = Func->getFunctionTypeLoc();
+    const FunctionType *FT = FTL.getTypePtr();
+
+    EXPECT_TRUE(FT->getNoReturnAttr());
+  }
+
+  {
+    const FunctionDecl *Func = getFunctionNode(AST.get(), "f_cc_preserve_most");
+    const FunctionTypeLoc FTL = Func->getFunctionTypeLoc();
+    const FunctionType *FT = FTL.getTypePtr();
+
+    EXPECT_TRUE(FT->getCallConv() == CC_PreserveMost);
+  }
+
+  {
+    for (auto should_have_func_type_loc : {
+             "f_macro_attribue",
+             "f_paren_attribute",
+             "f_w_paren_and_attr",
+         }) {
+      llvm::errs() << "O: " << should_have_func_type_loc << "\n";
+      const FunctionDecl *Func =
+          getFunctionNode(AST.get(), should_have_func_type_loc);
+
+      EXPECT_TRUE(Func->getFunctionTypeLoc());
+    }
+  }
+
+  // The following test verifies getFunctionTypeLoc returns a type
+  // which takes into account the attribute (instead of only the nake
+  // type).
+  //
+  // This is hard to do with C/C++ because it seems using a function
+  // type attribute with a C/C++ function declaration only results
+  // with either:
+  //
+  // 1. It does NOT produce any AttributedType (for example it only
+  //   sets one flag of the FunctionType's ExtInfo, e.g. NoReturn).
+  // 2. It produces an AttributedType with modified type and
+  //   equivalent type that are equal (for example, that's what
+  //   happens with Calling Convention attributes).
+  //
+  // Fortunately, ObjC has one specific function type attribute that
+  // creates an AttributedType with different modified type and
+  // equivalent type.
+  auto AST_ObjC = buildASTFromCodeWithArgs(
+      R"objc(
+    __attribute__((ns_returns_retained)) id f();
+  )objc",
+      {"-fobjc-arc", "-fsyntax-only", "-fobjc-runtime=macosx-10.7"},
+      "input.mm");
+  {
+    const FunctionDecl *f = getFunctionNode(AST_ObjC.get(), "f");
+    const FunctionTypeLoc FTL = f->getFunctionTypeLoc();
+
+    const FunctionType *FT = FTL.getTypePtr();
+    EXPECT_TRUE(FT->getExtInfo().getProducesResult());
+  }
+
   // Test type annotation on an `__auto_type` type in C mode.
   AST = buildASTFromCodeWithArgs(R"c(
     __auto_type [[clang::annotate_type("auto")]] auto_var = 1;

From a15ef95de47620d580df21bdf35afeeb324e452d Mon Sep 17 00:00:00 2001
From: Owen Anderson <resistor@mac.com>
Date: Sun, 20 Apr 2025 00:01:53 +1200
Subject: [PATCH 546/710] Revert "FunctionDecl::getFunctionTypeLoc: ignore
 function type attributes (#118420)"

This reverts commit 19c708c18963ac24822564824cb5401e71a49943 because it caused test failures on non-x86 targets.
---
 .../clangd/unittests/InlayHintTests.cpp       | 14 ++--
 clang/lib/AST/Decl.cpp                        | 21 +----
 clang/unittests/AST/AttrTest.cpp              | 78 -------------------
 3 files changed, 8 insertions(+), 105 deletions(-)

diff --git a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
index 030e499577706..77d78b8777fe3 100644
--- a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
+++ b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
@@ -1577,21 +1577,19 @@ TEST(TypeHints, Aliased) {
 }
 
 TEST(TypeHints, CallingConvention) {
-  // Check that we don't crash for lambdas with an annotation
+  // Check that we don't crash for lambdas without a FunctionTypeLoc
   // https://github.com/clangd/clangd/issues/2223
-  Annotations Source(R"cpp(
+  std::string Code = R"cpp(
     void test() {
-      []($lambda[[)]]__cdecl {};
+      []() __cdecl {};
     }
-  )cpp");
-  TestTU TU = TestTU::withCode(Source.code());
+  )cpp";
+  TestTU TU = TestTU::withCode(Code);
   TU.ExtraArgs.push_back("--target=x86_64-w64-mingw32");
   TU.PredefineMacros = true; // for the __cdecl
   auto AST = TU.build();
 
-  EXPECT_THAT(
-      hintsOfKind(AST, InlayHintKind::Type),
-      ElementsAre(HintMatcher(ExpectedHint{"-> void", "lambda"}, Source)));
+  EXPECT_THAT(hintsOfKind(AST, InlayHintKind::Type), IsEmpty());
 }
 
 TEST(TypeHints, Decltype) {
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 1d9208f0e1c72..ad1cb01592e9b 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -3910,25 +3910,8 @@ bool FunctionDecl::doesDeclarationForceExternallyVisibleDefinition() const {
 
 FunctionTypeLoc FunctionDecl::getFunctionTypeLoc() const {
   const TypeSourceInfo *TSI = getTypeSourceInfo();
-
-  if (!TSI)
-    return FunctionTypeLoc();
-
-  TypeLoc TL = TSI->getTypeLoc();
-  FunctionTypeLoc FTL;
-
-  while (!(FTL = TL.getAs<FunctionTypeLoc>())) {
-    if (const auto PTL = TL.getAs<ParenTypeLoc>())
-      TL = PTL.getInnerLoc();
-    else if (const auto ATL = TL.getAs<AttributedTypeLoc>())
-      TL = ATL.getEquivalentTypeLoc();
-    else if (const auto MQTL = TL.getAs<MacroQualifiedTypeLoc>())
-      TL = MQTL.getInnerLoc();
-    else
-      break;
-  }
-
-  return FTL;
+  return TSI ? TSI->getTypeLoc().IgnoreParens().getAs<FunctionTypeLoc>()
+             : FunctionTypeLoc();
 }
 
 SourceRange FunctionDecl::getReturnTypeSourceRange() const {
diff --git a/clang/unittests/AST/AttrTest.cpp b/clang/unittests/AST/AttrTest.cpp
index 3be362895b77e..46c3f5729021e 100644
--- a/clang/unittests/AST/AttrTest.cpp
+++ b/clang/unittests/AST/AttrTest.cpp
@@ -86,23 +86,6 @@ TEST(Attr, AnnotateType) {
     struct S { int mem; };
     int [[clang::annotate_type("int")]]
     S::* [[clang::annotate_type("ptr_to_mem")]] ptr_to_member = &S::mem;
-
-    // Function Type Attributes
-    __attribute__((noreturn)) int f_noreturn();
-    __attribute__((preserve_most)) int f_cc_preserve_most();
-
-    #define PRESERVE_MOST __attribute__((preserve_most))
-    PRESERVE_MOST int f_macro_attribue();
-
-    int (__attribute__((preserve_most)) f_paren_attribute)();
-
-    int (
-      PRESERVE_MOST
-      (
-        __attribute__((warn_unused_result))
-        (f_w_paren_and_attr)
-      )
-    ) ();
   )cpp");
 
   {
@@ -170,67 +153,6 @@ TEST(Attr, AnnotateType) {
     EXPECT_EQ(IntTL.getType(), AST->getASTContext().IntTy);
   }
 
-  {
-    const FunctionDecl *Func = getFunctionNode(AST.get(), "f_noreturn");
-    const FunctionTypeLoc FTL = Func->getFunctionTypeLoc();
-    const FunctionType *FT = FTL.getTypePtr();
-
-    EXPECT_TRUE(FT->getNoReturnAttr());
-  }
-
-  {
-    const FunctionDecl *Func = getFunctionNode(AST.get(), "f_cc_preserve_most");
-    const FunctionTypeLoc FTL = Func->getFunctionTypeLoc();
-    const FunctionType *FT = FTL.getTypePtr();
-
-    EXPECT_TRUE(FT->getCallConv() == CC_PreserveMost);
-  }
-
-  {
-    for (auto should_have_func_type_loc : {
-             "f_macro_attribue",
-             "f_paren_attribute",
-             "f_w_paren_and_attr",
-         }) {
-      llvm::errs() << "O: " << should_have_func_type_loc << "\n";
-      const FunctionDecl *Func =
-          getFunctionNode(AST.get(), should_have_func_type_loc);
-
-      EXPECT_TRUE(Func->getFunctionTypeLoc());
-    }
-  }
-
-  // The following test verifies getFunctionTypeLoc returns a type
-  // which takes into account the attribute (instead of only the nake
-  // type).
-  //
-  // This is hard to do with C/C++ because it seems using a function
-  // type attribute with a C/C++ function declaration only results
-  // with either:
-  //
-  // 1. It does NOT produce any AttributedType (for example it only
-  //   sets one flag of the FunctionType's ExtInfo, e.g. NoReturn).
-  // 2. It produces an AttributedType with modified type and
-  //   equivalent type that are equal (for example, that's what
-  //   happens with Calling Convention attributes).
-  //
-  // Fortunately, ObjC has one specific function type attribute that
-  // creates an AttributedType with different modified type and
-  // equivalent type.
-  auto AST_ObjC = buildASTFromCodeWithArgs(
-      R"objc(
-    __attribute__((ns_returns_retained)) id f();
-  )objc",
-      {"-fobjc-arc", "-fsyntax-only", "-fobjc-runtime=macosx-10.7"},
-      "input.mm");
-  {
-    const FunctionDecl *f = getFunctionNode(AST_ObjC.get(), "f");
-    const FunctionTypeLoc FTL = f->getFunctionTypeLoc();
-
-    const FunctionType *FT = FTL.getTypePtr();
-    EXPECT_TRUE(FT->getExtInfo().getProducesResult());
-  }
-
   // Test type annotation on an `__auto_type` type in C mode.
   AST = buildASTFromCodeWithArgs(R"c(
     __auto_type [[clang::annotate_type("auto")]] auto_var = 1;

From fc55ad4ceb7dda17d4d7eb3d44ac3833ecda1a84 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Sat, 19 Apr 2025 08:04:48 -0400
Subject: [PATCH 547/710] Revert "[StructurizeCFG] Refactor insertConditions.
 NFC. (#115476)" (#136370)

---
 llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 31 ++++++++++---------
 .../simple-structurizecfg-crash.ll            | 16 +++++++---
 2 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 00c4fcc76e791..eb22b50532695 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -612,25 +612,28 @@ void StructurizeCFG::insertConditions(bool Loops) {
     BasicBlock *SuccTrue = Term->getSuccessor(0);
     BasicBlock *SuccFalse = Term->getSuccessor(1);
 
-    BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue];
+    PhiInserter.Initialize(Boolean, "");
+    PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default);
 
-    if (Preds.size() == 1 && Preds.begin()->first == Parent) {
-      auto &PI = Preds.begin()->second;
-      Term->setCondition(PI.Pred);
-      CondBranchWeights::setMetadata(*Term, PI.Weights);
-    } else {
-      PhiInserter.Initialize(Boolean, "");
-      PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default);
+    BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue];
 
-      NearestCommonDominator Dominator(DT);
-      Dominator.addBlock(Parent);
+    NearestCommonDominator Dominator(DT);
+    Dominator.addBlock(Parent);
 
-      for (auto [BB, PI] : Preds) {
-        assert(BB != Parent);
-        PhiInserter.AddAvailableValue(BB, PI.Pred);
-        Dominator.addAndRememberBlock(BB);
+    PredInfo ParentInfo{nullptr, std::nullopt};
+    for (auto [BB, PI] : Preds) {
+      if (BB == Parent) {
+        ParentInfo = PI;
+        break;
       }
+      PhiInserter.AddAvailableValue(BB, PI.Pred);
+      Dominator.addAndRememberBlock(BB);
+    }
 
+    if (ParentInfo.Pred) {
+      Term->setCondition(ParentInfo.Pred);
+      CondBranchWeights::setMetadata(*Term, ParentInfo.Weights);
+    } else {
       if (!Dominator.resultIsRememberedBlock())
         PhiInserter.AddAvailableValue(Dominator.result(), Default);
 
diff --git a/llvm/test/Transforms/StructurizeCFG/simple-structurizecfg-crash.ll b/llvm/test/Transforms/StructurizeCFG/simple-structurizecfg-crash.ll
index 6edc0b883f51e..691f43bdcf948 100644
--- a/llvm/test/Transforms/StructurizeCFG/simple-structurizecfg-crash.ll
+++ b/llvm/test/Transforms/StructurizeCFG/simple-structurizecfg-crash.ll
@@ -1,11 +1,19 @@
-; RUN: opt -S -passes=structurizecfg %s -o -
-; REQUIRES: asserts
-; XFAIL: *
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=structurizecfg %s -o - | FileCheck %s
 
 ; Issue tracking: https://github.com/llvm/llvm-project/issues/126534.
-; FIXME: This test is expected to crash. Generate checklines after the crash is fixed.
 
 define void @foo() {
+; CHECK-LABEL: define void @foo() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_TRUE:.*]]:
+; CHECK-NEXT:    br label %[[COND_END:.*]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    br i1 false, label %[[COND_TRUE]], label %[[COND_END]]
+; CHECK:       [[COND_END]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 false, label %cond.true, label %cond.false
 

From 8fdebff69d92fd3c6023bf2d0646ca909bb4ec05 Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Sat, 19 Apr 2025 17:40:50 +0300
Subject: [PATCH 548/710] [PAC][ThinLTO] Fix auth key for GOT entries of
 function symbols (#131467)

Symtab is first filled with the data from the bitcode file, and all
undefined symbols except TLS ones are `STT_NOTYPE`. Since auth key for a
signed GOT entry depends on the symbol type being `STT_FUNC` or not, we
need to update the symtab after the bitcode is compiled to an ELF object
and update symbol types for function symbols. This patch implements the
described behavior.
---
 lld/ELF/Driver.cpp                       | 12 +++++
 lld/test/ELF/lto/aarch64-pac-got-func.ll | 66 ++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 lld/test/ELF/lto/aarch64-pac-got-func.ll

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index b3c5518b42877..9d36071e1532f 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -2638,6 +2638,18 @@ void LinkerDriver::compileBitcodeFiles(bool skipLinkedOutput) {
     auto *obj = cast<ObjFile<ELFT>>(file.get());
     obj->parse(/*ignoreComdats=*/true);
 
+    // This is only needed for AArch64 PAuth to set correct key in AUTH GOT
+    // entry based on symbol type (STT_FUNC or not).
+    // TODO: check if PAuth is actually used.
+    if (ctx.arg.emachine == EM_AARCH64) {
+      for (typename ELFT::Sym elfSym : obj->template getGlobalELFSyms<ELFT>()) {
+        StringRef elfSymName = check(elfSym.getName(obj->getStringTable()));
+        if (Symbol *sym = ctx.symtab->find(elfSymName))
+          if (sym->type == STT_NOTYPE)
+            sym->type = elfSym.getType();
+      }
+    }
+
     // For defined symbols in non-relocatable output,
     // compute isExported and parse '@'.
     if (!ctx.arg.relocatable)
diff --git a/lld/test/ELF/lto/aarch64-pac-got-func.ll b/lld/test/ELF/lto/aarch64-pac-got-func.ll
new file mode 100644
index 0000000000000..a37c67a2f3ba8
--- /dev/null
+++ b/lld/test/ELF/lto/aarch64-pac-got-func.ll
@@ -0,0 +1,66 @@
+; REQUIRES: aarch64
+
+; RUN: llvm-as %s -o %t.o
+; RUN: ld.lld %t.o -shared -o %t
+; RUN: llvm-readelf -r -x.got %t | FileCheck %s
+
+; CHECK:      Relocation section '.rela.dyn' at offset 0x3d0 contains 8 entries:
+; CHECK-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+; CHECK-NEXT: 00000000000206a0  0000000100000412 R_AARCH64_AUTH_GLOB_DAT 0000000000000000 func_undef + 0
+; CHECK-NEXT: 00000000000206a8  0000000200000412 R_AARCH64_AUTH_GLOB_DAT 0000000000000000 g1 + 0
+; CHECK-NEXT: 00000000000206b0  0000000300000412 R_AARCH64_AUTH_GLOB_DAT 0000000000000000 g2 + 0
+; CHECK-NEXT: 00000000000206b8  0000000400000412 R_AARCH64_AUTH_GLOB_DAT 0000000000000000 g3 + 0
+; CHECK-NEXT: 00000000000206c0  0000000500000412 R_AARCH64_AUTH_GLOB_DAT 0000000000000000 g4 + 0
+; CHECK-NEXT: 00000000000206c8  0000000600000412 R_AARCH64_AUTH_GLOB_DAT 0000000000000000 var_undef + 0
+; CHECK-NEXT: 0000000000020690  0000000700000412 R_AARCH64_AUTH_GLOB_DAT 0000000000010490 func + 0
+; CHECK-NEXT: 0000000000020698  0000000a00000412 R_AARCH64_AUTH_GLOB_DAT 00000000000306d0 var + 0
+
+; CHECK:      Hex dump of section '.got':
+; CHECK-NEXT: 0x00020690 00000000 00000080 00000000 000000a0
+;;                                      ^^ func: 0b10000000 bit 63 address diversity = true, bits 61..60 key = IA
+;;                                                        ^^ var: 0b10100000 bit 63 address diversity = true, bits 61..60 key = DA
+; CHECK-NEXT: 0x000206a0 00000000 00000080 00000000 000000a0
+;;                                      ^^ func_undef: 0b10000000 bit 63 address diversity = true, bits 61..60 key = IA
+;;                                                        ^^ g1: 0b10100000 bit 63 address diversity = true, bits 61..60 key = DA
+; CHECK-NEXT: 0x000206b0 00000000 000000a0 00000000 000000a0
+;;                                      ^^ g2: 0b10100000 bit 63 address diversity = true, bits 61..60 key = DA
+;;                                                        ^^ g3: 0b10100000 bit 63 address diversity = true, bits 61..60 key = DA
+; CHECK-NEXT: 0x000206c0 00000000 000000a0 00000000 000000a0
+;;                                      ^^ g4: 0b10100000 bit 63 address diversity = true, bits 61..60 key = DA
+;;                                                        ^^ var_undef: 0b10100000 bit 63 address diversity = true, bits 61..60 key = DA
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+@g1 = external global ptr
+@g2 = external global ptr
+@g3 = external global ptr
+@g4 = external global ptr
+
+define void @func() {
+entry:
+  ret void
+}
+declare void @func_undef()
+
+@var = global i32 42
+@var_undef = external global i32
+
+define void @bar() #0 {
+entry:
+  store ptr ptrauth (ptr @func, i32 0), ptr @g1
+  store ptr ptrauth (ptr @func_undef, i32 0), ptr @g2
+  store ptr ptrauth (ptr @var, i32 0), ptr @g3
+  store ptr ptrauth (ptr @var_undef, i32 0), ptr @g4
+  ret void
+}
+
+define void @_start() {
+entry:
+  ret void
+}
+
+attributes #0 = {"target-features"="+pauth"}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 8, !"ptrauth-elf-got", i32 1}

From 6196b4ee8c47dc4a2267b7c704d1811e68c7f1c1 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sat, 19 Apr 2025 16:48:49 +0200
Subject: [PATCH 549/710] [clang][bytecode] Don't set OnePastEnd bit for array
 elements (#136422)

If we refer to arr[N], don't set the OnePastEnd bit of the APValue,
since that is already encoded in the array index.
---
 clang/lib/AST/ByteCode/Pointer.cpp |  9 +++++----
 clang/test/AST/ByteCode/arrays.cpp | 10 ++++++++++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index 686ec381d232a..059503cae3561 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -223,6 +223,7 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
     UsePath = false;
 
   // Build the path into the object.
+  bool OnePastEnd = isOnePastEnd();
   Pointer Ptr = *this;
   while (Ptr.isField() || Ptr.isArrayElement()) {
 
@@ -251,9 +252,10 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
       Ptr = Ptr.expand();
       const Descriptor *Desc = Ptr.getFieldDesc();
       unsigned Index;
-      if (Ptr.isOnePastEnd())
+      if (Ptr.isOnePastEnd()) {
         Index = Ptr.getArray().getNumElems();
-      else
+        OnePastEnd = false;
+      } else
         Index = Ptr.getIndex();
 
       QualType ElemType = Desc->getElemQualType();
@@ -304,8 +306,7 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   std::reverse(Path.begin(), Path.end());
 
   if (UsePath)
-    return APValue(Base, Offset, Path,
-                   /*IsOnePastEnd=*/!isElementPastEnd() && isOnePastEnd());
+    return APValue(Base, Offset, Path, OnePastEnd);
 
   return APValue(Base, Offset, APValue::NoLValuePath());
 }
diff --git a/clang/test/AST/ByteCode/arrays.cpp b/clang/test/AST/ByteCode/arrays.cpp
index e50839e0f0877..934c3a3e7aff1 100644
--- a/clang/test/AST/ByteCode/arrays.cpp
+++ b/clang/test/AST/ByteCode/arrays.cpp
@@ -761,3 +761,13 @@ namespace PointerSubscript {
   struct S{};
   static_assert((foo<S>(), true));
 }
+
+namespace OnePastEndDiag {
+
+  constexpr int a(const int *b) {
+    return *b; // both-note {{read of dereferenced one-past-the-end pointer}}
+  }
+  constexpr int foo[] = {1,2};
+  constexpr int k = a(foo + 2); // both-error {{must be initialized by a constant expression}} \
+                                // both-note {{in call to 'a(&foo[2])'}}
+}

From 5e80487dd63c53545560726583ef25a160cd8a40 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 09:09:20 -0700
Subject: [PATCH 550/710] [Utils] Avoid repeated hash lookups (NFC) (#136414)

---
 .../Transforms/Utils/ScalarEvolutionExpander.cpp   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 31d8acb706997..fb5e73727b2ef 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1633,13 +1633,13 @@ void SCEVExpander::replaceCongruentIVInc(
   // If this phi has the same width but is more canonical, replace the
   // original with it. As part of the "more canonical" determination,
   // respect a prior decision to use an IV chain.
-  if (OrigPhi->getType() == Phi->getType() &&
-      !(ChainedPhis.count(Phi) ||
-        isExpandedAddRecExprPHI(OrigPhi, OrigInc, L)) &&
-      (ChainedPhis.count(Phi) ||
-       isExpandedAddRecExprPHI(Phi, IsomorphicInc, L))) {
-    std::swap(OrigPhi, Phi);
-    std::swap(OrigInc, IsomorphicInc);
+  if (OrigPhi->getType() == Phi->getType()) {
+    bool Chained = ChainedPhis.contains(Phi);
+    if (!(Chained || isExpandedAddRecExprPHI(OrigPhi, OrigInc, L)) &&
+        (Chained || isExpandedAddRecExprPHI(Phi, IsomorphicInc, L))) {
+      std::swap(OrigPhi, Phi);
+      std::swap(OrigInc, IsomorphicInc);
+    }
   }
 
   // Replacing the congruent phi is sufficient because acyclic

From e7c07a021034c7469c3c4a6f65299152291091ed Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 09:09:41 -0700
Subject: [PATCH 551/710] [AMDGPU] Construct SmallVector with iterator ranges
 (NFC) (#136415)

---
 .../AMDGPU/AMDGPULowerBufferFatPointers.cpp       |  5 ++---
 .../Target/AMDGPU/AMDGPURewriteOutArguments.cpp   |  4 +---
 llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp          |  7 ++-----
 llvm/lib/Target/AMDGPU/SIFoldOperands.cpp         | 15 ++++++---------
 4 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 766a4ea250942..a62fc60461193 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -2261,11 +2261,10 @@ PtrParts SplitPtrStructs::visitIntrinsicInst(IntrinsicInst &I) {
 
 void SplitPtrStructs::processFunction(Function &F) {
   ST = &TM->getSubtarget<GCNSubtarget>(F);
-  SmallVector<Instruction *, 0> Originals;
+  SmallVector<Instruction *, 0> Originals(
+      llvm::make_pointer_range(instructions(F)));
   LLVM_DEBUG(dbgs() << "Splitting pointer structs in function: " << F.getName()
                     << "\n");
-  for (Instruction &I : instructions(F))
-    Originals.push_back(&I);
   for (Instruction *I : Originals) {
     auto [Rsrc, Off] = visit(I);
     assert(((Rsrc && Off) || (!Rsrc && !Off)) &&
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
index 6347636b60e58..bb68f4a0e70de 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -114,9 +114,7 @@ Type *AMDGPURewriteOutArguments::getStoredType(Value &Arg) const {
   const int MaxUses = 10;
   int UseCount = 0;
 
-  SmallVector<Use *> Worklist;
-  for (Use &U : Arg.uses())
-    Worklist.push_back(&U);
+  SmallVector<Use *> Worklist(llvm::make_pointer_range(Arg.uses()));
 
   Type *StoredType = nullptr;
   while (!Worklist.empty()) {
diff --git a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
index 5439ea2f59111..0d8113da275bd 100644
--- a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
@@ -629,11 +629,8 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
 
   OrigMIs.push_back(&MovMI);
   bool Rollback = true;
-  SmallVector<MachineOperand*, 16> Uses;
-
-  for (auto &Use : MRI->use_nodbg_operands(DPPMovReg)) {
-    Uses.push_back(&Use);
-  }
+  SmallVector<MachineOperand *, 16> Uses(
+      llvm::make_pointer_range(MRI->use_nodbg_operands(DPPMovReg)));
 
   while (!Uses.empty()) {
     MachineOperand *Use = Uses.pop_back_val();
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index d6acf9e081b9f..5e8273a968e82 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -972,9 +972,8 @@ void SIFoldOperandsImpl::foldOperand(
     unsigned RegSeqDstSubReg = UseMI->getOperand(UseOpIdx + 1).getImm();
 
     // Grab the use operands first
-    SmallVector<MachineOperand *, 4> UsesToProcess;
-    for (auto &Use : MRI->use_nodbg_operands(RegSeqDstReg))
-      UsesToProcess.push_back(&Use);
+    SmallVector<MachineOperand *, 4> UsesToProcess(
+        llvm::make_pointer_range(MRI->use_nodbg_operands(RegSeqDstReg)));
     for (auto *RSUse : UsesToProcess) {
       MachineInstr *RSUseMI = RSUse->getParent();
 
@@ -1552,9 +1551,8 @@ bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
     }
   }
 
-  SmallVector<MachineOperand *, 4> UsesToProcess;
-  for (auto &Use : MRI->use_nodbg_operands(Dst.getReg()))
-    UsesToProcess.push_back(&Use);
+  SmallVector<MachineOperand *, 4> UsesToProcess(
+      llvm::make_pointer_range(MRI->use_nodbg_operands(Dst.getReg())));
   for (auto *U : UsesToProcess) {
     MachineInstr *UseMI = U->getParent();
     foldOperand(OpToFold, UseMI, UseMI->getOperandNo(U), FoldList,
@@ -2380,10 +2378,9 @@ bool SIFoldOperandsImpl::tryFoldLoad(MachineInstr &MI) {
   if (DefReg.isPhysical() || !TRI->isVGPR(*MRI, DefReg))
     return false;
 
-  SmallVector<const MachineInstr*, 8> Users;
+  SmallVector<const MachineInstr *, 8> Users(
+      llvm::make_pointer_range(MRI->use_nodbg_instructions(DefReg)));
   SmallVector<Register, 8> MoveRegs;
-  for (const MachineInstr &I : MRI->use_nodbg_instructions(DefReg))
-    Users.push_back(&I);
 
   if (Users.empty())
     return false;

From 6b2fd90bb8e9567e5f200ae343d6d0eaedc03782 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 09:10:08 -0700
Subject: [PATCH 552/710] [Hexagon] Construct SmallVector with an iterator
 range (NFC) (#136416)

This patch switches to SmallVector so that we can construct an
instance with an iterator range.
---
 llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index 79169a0c20e58..9604f252dd3df 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -1079,9 +1079,7 @@ bool PolynomialMultiplyRecognize::promoteTypes(BasicBlock *LoopB,
       return false;
 
   // Perform the promotion.
-  std::vector<Instruction*> LoopIns;
-  std::transform(LoopB->begin(), LoopB->end(), std::back_inserter(LoopIns),
-                 [](Instruction &In) { return &In; });
+  SmallVector<Instruction *> LoopIns(llvm::make_pointer_range(*LoopB));
   for (Instruction *In : LoopIns)
     if (!In->isTerminator())
       promoteTo(In, DestTy, LoopB);

From 584aefbc9be371fceb5f8f494a7e49a0f24e98a3 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 09:10:43 -0700
Subject: [PATCH 553/710] [llvm] Use range-based for loops with
 llvm::drop_begin (NFC) (#136417)

---
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp                   | 6 +++---
 llvm/lib/IR/DebugInfo.cpp                                   | 6 +++---
 llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp             | 3 +--
 llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp                 | 5 ++---
 .../Vectorize/SandboxVectorizer/SeedCollector.cpp           | 2 +-
 5 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 5c62ef4ad8e4e..b152adfbd93d8 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -8079,9 +8079,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
     case bitc::FS_PERMODULE_CALLSITE_INFO: {
       unsigned ValueID = Record[0];
       SmallVector<unsigned> StackIdList;
-      for (auto R = Record.begin() + 1; R != Record.end(); R++) {
-        assert(*R < StackIds.size());
-        StackIdList.push_back(TheIndex.addOrGetStackIdIndex(StackIds[*R]));
+      for (uint64_t R : drop_begin(Record)) {
+        assert(R < StackIds.size());
+        StackIdList.push_back(TheIndex.addOrGetStackIdIndex(StackIds[R]));
       }
       ValueInfo VI = std::get<0>(getValueInfoFromValueId(ValueID));
       PendingCallsites.push_back(CallsiteInfo({VI, std::move(StackIdList)}));
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index cefd6fe14a3ac..7a133b209459b 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -976,9 +976,9 @@ void Instruction::mergeDIAssignID(
     return; // No DIAssignID tags to process.
 
   DIAssignID *MergeID = IDs[0];
-  for (auto It = std::next(IDs.begin()), End = IDs.end(); It != End; ++It) {
-    if (*It != MergeID)
-      at::RAUW(*It, MergeID);
+  for (DIAssignID *AssignID : drop_begin(IDs)) {
+    if (AssignID != MergeID)
+      at::RAUW(AssignID, MergeID);
   }
   setMetadata(LLVMContext::MD_DIAssignID, MergeID);
 }
diff --git a/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp b/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
index 2b890ddccf628..c21a1d03630fa 100644
--- a/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
+++ b/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
@@ -345,8 +345,7 @@ static bool foldGEPChainAsStructAccess(SmallVector<GetElementPtrInst *> &GEPs,
   Info.Indices.append(First->idx_begin(), First->idx_end());
   Info.Members.push_back(First);
 
-  for (auto *Iter = GEPs.begin() + 1; Iter != GEPs.end(); ++Iter) {
-    GetElementPtrInst *GEP = *Iter;
+  for (GetElementPtrInst *GEP : drop_begin(GEPs)) {
     if (!isZero(*GEP->idx_begin())) {
       Info.reset();
       return false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
index 324b80fe2efc6..68448acf45c3c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVStructurizer.cpp
@@ -509,9 +509,8 @@ class SPIRVStructurizer : public FunctionPass {
       }
 
       SwitchInst *Sw = ExitBuilder.CreateSwitch(Load, Dsts[0], Dsts.size() - 1);
-      for (auto It = Dsts.begin() + 1; It != Dsts.end(); ++It) {
-        Sw->addCase(DstToIndex[*It], *It);
-      }
+      for (BasicBlock *BB : drop_begin(Dsts))
+        Sw->addCase(DstToIndex[BB], BB);
       return NewExit;
     }
   };
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
index 427c7d9895d88..ee7063437a8fc 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
@@ -46,7 +46,7 @@ ArrayRef<Instruction *> SeedBundle::getSlice(unsigned StartIdx,
   uint32_t BitCountPowerOfTwo = 0;
   // Can't start a slice with a used instruction.
   assert(!isUsed(StartIdx) && "Expected unused at StartIdx");
-  for (auto S : make_range(Seeds.begin() + StartIdx, Seeds.end())) {
+  for (Instruction *S : drop_begin(Seeds, StartIdx)) {
     // Stop if this instruction is used. This needs to be done before
     // getNumBits() because a "used" instruction may have been erased.
     if (isUsed(StartIdx + NumElements))

From 198c5dac37dbe9c6a5f10e2b5113afc39b6eb93d Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Sat, 19 Apr 2025 12:11:06 -0400
Subject: [PATCH 554/710] [mlir][transform] Clean up prints. NFC. (#136401)

Use `llvm::interleaved` from #135517 to simplify printing.
---
 mlir/lib/Dialect/GPU/TransformOps/Utils.cpp   | 48 ++++++++-----------
 .../DebugExtension/DebugExtensionOps.cpp      |  3 +-
 .../lib/Dialect/Transform/IR/TransformOps.cpp |  8 ++--
 .../Transform/Interfaces/MatchInterfaces.cpp  |  5 +-
 .../Interfaces/TransformInterfaces.cpp        | 12 +++--
 .../Transforms/XeGPUSubgroupDistribute.cpp    |  5 +-
 6 files changed, 37 insertions(+), 44 deletions(-)

diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index f5a6d0893d9a9..9853e80828390 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -34,6 +34,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
 using namespace mlir::gpu;
@@ -50,10 +51,8 @@ using namespace mlir::transform::gpu;
 template <typename ThreadOrBlockIdOp>
 static Value buildLinearId(RewriterBase &rewriter, Location loc,
                            ArrayRef<OpFoldResult> originalBasisOfr) {
-  LLVM_DEBUG(llvm::interleaveComma(
-                 originalBasisOfr,
-                 DBGS() << "----buildLinearId with originalBasisOfr:  ");
-             llvm::dbgs() << "\n");
+  LLVM_DEBUG(DBGS() << "----buildLinearId with originalBasisOfr:  "
+                    << llvm::interleaved(originalBasisOfr) << "\n");
   assert(originalBasisOfr.size() == 3 && "expected 3 sizes");
   IndexType indexType = rewriter.getIndexType();
   AffineExpr tx, ty, tz, bdx, bdy;
@@ -99,32 +98,25 @@ static GpuIdBuilderFnType commonLinearIdBuilderFn(int64_t multiplicity = 1) {
           affine::makeComposedAffineApply(rewriter, loc, e, {scaledLinearId}));
     }
 
-    // clang-format off
-      LLVM_DEBUG(llvm::interleaveComma(reverseBasisSizes,
-                                       DBGS() << "--delinearization basis: ");
-                 llvm::dbgs() << "\n";
-                 llvm::interleaveComma(strides,
-                                       DBGS() << "--delinearization strides: ");
-                 llvm::dbgs() << "\n";
-                 llvm::interleaveComma(delinearizingExprs,
-                                       DBGS() << "--delinearization exprs: ");
-                 llvm::dbgs() << "\n";
-                 llvm::interleaveComma(ids, DBGS() << "--ids: ");
-                 llvm::dbgs() << "\n";);
-    // clang-format on
+    LLVM_DEBUG(DBGS() << "--delinearization basis: "
+                      << llvm::interleaved(reverseBasisSizes) << "\n";
+               DBGS() << "--delinearization strides: "
+                      << llvm::interleaved(strides) << "\n";
+               DBGS() << "--delinearization exprs: "
+                      << llvm::interleaved(delinearizingExprs) << "\n";
+               DBGS() << "--ids: " << llvm::interleaved(ids) << "\n");
 
     // Return n-D ids for indexing and 1-D size + id for predicate generation.
-      return IdBuilderResult{
-          /*mappingIdOps=*/ids,
-          /*availableMappingSizes=*/
-          SmallVector<int64_t>{computeProduct(originalBasis)},
-          // `forallMappingSizes` iterate in the scaled basis, they need to be
-          // scaled back into the original basis to provide tight
-          // activeMappingSizes quantities for predication.
-          /*activeMappingSizes=*/
-          SmallVector<int64_t>{computeProduct(forallMappingSizes) *
-                               multiplicity},
-          /*activeIdOps=*/SmallVector<Value>{cast<Value>(linearId)}};
+    return IdBuilderResult{
+        /*mappingIdOps=*/ids,
+        /*availableMappingSizes=*/
+        SmallVector<int64_t>{computeProduct(originalBasis)},
+        // `forallMappingSizes` iterate in the scaled basis, they need to be
+        // scaled back into the original basis to provide tight
+        // activeMappingSizes quantities for predication.
+        /*activeMappingSizes=*/
+        SmallVector<int64_t>{computeProduct(forallMappingSizes) * multiplicity},
+        /*activeIdOps=*/SmallVector<Value>{cast<Value>(linearId)}};
   };
 
   return res;
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
index 7ad017af485f9..7a9f8f4b1b528 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/IR/OpImplementation.h"
+#include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
 
@@ -58,7 +59,7 @@ DiagnosedSilenceableFailure transform::DebugEmitParamAsRemarkOp::apply(
   llvm::raw_string_ostream os(str);
   if (getMessage())
     os << *getMessage() << " ";
-  llvm::interleaveComma(state.getParams(getParam()), os);
+  os << llvm::interleaved(state.getParams(getParam()));
   if (!getAnchor()) {
     emitRemark() << str;
     return DiagnosedSilenceableFailure::success();
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 798853a75441a..4fe89f3f7fb9e 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -41,6 +41,7 @@
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/InterleavedRange.h"
 #include <optional>
 
 #define DEBUG_TYPE "transform-dialect"
@@ -2630,11 +2631,8 @@ static void printSequenceOpOperands(OpAsmPrinter &printer, Operation *op,
     printer << "(";
 
   printer << rootType;
-  if (hasExtras) {
-    printer << ", ";
-    llvm::interleaveComma(extraBindingTypes, printer.getStream());
-    printer << ")";
-  }
+  if (hasExtras)
+    printer << ", " << llvm::interleaved(extraBindingTypes) << ')';
 }
 
 /// Returns `true` if the given op operand may be consuming the handle value in
diff --git a/mlir/lib/Dialect/Transform/Interfaces/MatchInterfaces.cpp b/mlir/lib/Dialect/Transform/Interfaces/MatchInterfaces.cpp
index 783eb137b9aff..38f0dcfa189ff 100644
--- a/mlir/lib/Dialect/Transform/Interfaces/MatchInterfaces.cpp
+++ b/mlir/lib/Dialect/Transform/Interfaces/MatchInterfaces.cpp
@@ -8,6 +8,8 @@
 
 #include "mlir/Dialect/Transform/Interfaces/MatchInterfaces.h"
 
+#include "llvm/Support/InterleavedRange.h"
+
 using namespace mlir;
 
 //===----------------------------------------------------------------------===//
@@ -68,8 +70,7 @@ void transform::printTransformMatchDims(OpAsmPrinter &printer, Operation *op,
   if (isInverted) {
     printer << kDimExceptKeyword << "(";
   }
-  llvm::interleaveComma(rawDimList.asArrayRef(), printer.getStream(),
-                        [&](int64_t value) { printer << value; });
+  printer << llvm::interleaved(rawDimList.asArrayRef());
   if (isInverted) {
     printer << ")";
   }
diff --git a/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp b/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
index e0a5df0c758b3..c0230e0d11e64 100644
--- a/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
+++ b/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
@@ -15,8 +15,10 @@
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/iterator.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/InterleavedRange.h"
 
 #define DEBUG_TYPE "transform-dialect"
 #define DEBUG_TYPE_FULL "transform-dialect-full"
@@ -486,11 +488,11 @@ void transform::TransformState::recordOpHandleInvalidationOne(
     return;
 
   FULL_LDBG("--recordOpHandleInvalidationOne\n");
-  DEBUG_WITH_TYPE(
-      DEBUG_TYPE_FULL,
-      llvm::interleaveComma(potentialAncestors, DBGS() << "--ancestors: ",
-                            [](Operation *op) { llvm::dbgs() << *op; });
-      llvm::dbgs() << "\n");
+  DEBUG_WITH_TYPE(DEBUG_TYPE_FULL, {
+    (DBGS() << "--ancestors: "
+            << llvm::interleaved(llvm::make_pointee_range(potentialAncestors))
+            << "\n");
+  });
 
   Operation *owner = consumingHandle.getOwner();
   unsigned operandNo = consumingHandle.getOperandNumber();
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index 9531837625878..721a815cf76b9 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
@@ -58,9 +59,7 @@ struct Layout {
 };
 
 void Layout::print(llvm::raw_ostream &os) const {
-  os << "[";
-  llvm::interleaveComma(layout, os);
-  os << "]";
+  os << llvm::interleaved_array(layout);
 }
 
 int64_t Layout::operator[](size_t idx) const {

From 4d756917617c3fa2990fd41a5efc3c97bd3506d8 Mon Sep 17 00:00:00 2001
From: Pedro Lobo <pedro.lobo@tecnico.ulisboa.pt>
Date: Sat, 19 Apr 2025 17:15:07 +0100
Subject: [PATCH 555/710] [Hexagon] Change placeholder from `undef` to `poison`
 (#135555)

A `poison` vector should be used when performing an identity shuffle.

Co-authored-by: Nuno Lopes <nuno.lopes@tecnico.ulisboa.pt>
---
 llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index f800446592c06..d0f14d8ad7286 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -2318,9 +2318,9 @@ auto HexagonVectorCombine::insertb(IRBuilderBase &Builder, Value *Dst,
   assert(0 <= Where && Where + Length <= DstLen);
 
   int P2Len = PowerOf2Ceil(SrcLen | DstLen);
-  auto *Undef = UndefValue::get(getByteTy());
-  Value *P2Src = vresize(Builder, Src, P2Len, Undef);
-  Value *P2Dst = vresize(Builder, Dst, P2Len, Undef);
+  auto *Poison = PoisonValue::get(getByteTy());
+  Value *P2Src = vresize(Builder, Src, P2Len, Poison);
+  Value *P2Dst = vresize(Builder, Dst, P2Len, Poison);
 
   SmallVector<int, 256> SMask(P2Len);
   for (int i = 0; i != P2Len; ++i) {
@@ -2331,7 +2331,7 @@ auto HexagonVectorCombine::insertb(IRBuilderBase &Builder, Value *Dst,
   }
 
   Value *P2Insert = Builder.CreateShuffleVector(P2Dst, P2Src, SMask, "shf");
-  return vresize(Builder, P2Insert, DstLen, Undef);
+  return vresize(Builder, P2Insert, DstLen, Poison);
 }
 
 auto HexagonVectorCombine::vlalignb(IRBuilderBase &Builder, Value *Lo,

From d46c7334e86040a71b4002786fe021ebf6946694 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 09:39:39 -0700
Subject: [PATCH 556/710] [Scalar] Avoid repeated hash lookups (NFC) (#136264)

---
 llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index 7eee0283e20e6..8005be8d4ca05 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -3311,7 +3311,7 @@ static void computeLiveInValues(DominatorTree &DT, Function &F,
 
     // Compute our new liveout set, then exit early if it hasn't changed despite
     // the contribution of our successor.
-    SetVector<Value *> LiveOut = Data.LiveOut[BB];
+    SetVector<Value *> &LiveOut = Data.LiveOut[BB];
     const auto OldLiveOutSize = LiveOut.size();
     for (BasicBlock *Succ : successors(BB)) {
       assert(Data.LiveIn.count(Succ));
@@ -3324,7 +3324,6 @@ static void computeLiveInValues(DominatorTree &DT, Function &F,
       // hasn't changed.
       continue;
     }
-    Data.LiveOut[BB] = LiveOut;
 
     // Apply the effects of this basic block
     SetVector<Value *> LiveTmp = LiveOut;

From 0ed1c9862df408d3df171762bd8fa0dafbe1b7e2 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 10:37:25 -0700
Subject: [PATCH 557/710] [clang] llvm::append_range (NFC) (#136440)

---
 clang/include/clang/Parse/Parser.h                          | 2 +-
 clang/lib/AST/DeclBase.cpp                                  | 2 +-
 clang/lib/AST/Randstruct.cpp                                | 2 +-
 clang/lib/Analysis/FlowSensitive/CNFFormula.cpp             | 2 +-
 clang/lib/Basic/Targets/RISCV.cpp                           | 4 ++--
 clang/lib/CodeGen/CGObjCMac.cpp                             | 2 +-
 clang/lib/Interpreter/Interpreter.cpp                       | 4 ++--
 clang/lib/Parse/ParseDecl.cpp                               | 2 +-
 clang/lib/Parse/ParseStmtAsm.cpp                            | 2 +-
 clang/lib/Sema/SemaChecking.cpp                             | 2 +-
 clang/lib/Sema/SemaExprCXX.cpp                              | 4 ++--
 clang/lib/Sema/SemaHLSL.cpp                                 | 6 +++---
 clang/lib/Sema/SemaLookup.cpp                               | 2 +-
 clang/lib/Sema/TreeTransform.h                              | 2 +-
 clang/lib/Serialization/ASTWriter.cpp                       | 4 ++--
 clang/lib/StaticAnalyzer/Core/BugReporter.cpp               | 2 +-
 clang/lib/StaticAnalyzer/Core/RegionStore.cpp               | 2 +-
 clang/lib/Tooling/Tooling.cpp                               | 2 +-
 .../Interpreter/ExceptionTests/InterpreterExceptionTest.cpp | 2 +-
 clang/unittests/Interpreter/InterpreterTest.cpp             | 2 +-
 .../Tooling/DependencyScanning/DependencyScannerTest.cpp    | 2 +-
 clang/utils/TableGen/ClangAttrEmitter.cpp                   | 2 +-
 clang/utils/TableGen/MveEmitter.cpp                         | 2 +-
 23 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 662f54d0e8d8a..7ffe23c73eda9 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3739,7 +3739,7 @@ class Parser : public CodeCompletionHandler {
     SmallVector<Expr *> getAllExprs() {
       SmallVector<Expr *> Out;
       Out.push_back(DevNumExpr);
-      Out.insert(Out.end(), QueueIdExprs.begin(), QueueIdExprs.end());
+      llvm::append_range(Out, QueueIdExprs);
       return Out;
     }
   };
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 6260b92733ab7..1afda9aac9e18 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -1981,7 +1981,7 @@ void DeclContext::localUncachedLookup(DeclarationName Name,
   // the results.
   if (!hasExternalVisibleStorage() && !hasExternalLexicalStorage() && Name) {
     lookup_result LookupResults = lookup(Name);
-    Results.insert(Results.end(), LookupResults.begin(), LookupResults.end());
+    llvm::append_range(Results, LookupResults);
     if (!Results.empty())
       return;
   }
diff --git a/clang/lib/AST/Randstruct.cpp b/clang/lib/AST/Randstruct.cpp
index 4537ba5309e0b..5cba2f0a3c154 100644
--- a/clang/lib/AST/Randstruct.cpp
+++ b/clang/lib/AST/Randstruct.cpp
@@ -159,7 +159,7 @@ void randomizeStructureLayoutImpl(const ASTContext &Context,
     if (!B->isBitfieldRun())
       std::shuffle(std::begin(RandFields), std::end(RandFields), RNG);
 
-    FinalOrder.insert(FinalOrder.end(), RandFields.begin(), RandFields.end());
+    llvm::append_range(FinalOrder, RandFields);
   }
 
   FieldsOut = FinalOrder;
diff --git a/clang/lib/Analysis/FlowSensitive/CNFFormula.cpp b/clang/lib/Analysis/FlowSensitive/CNFFormula.cpp
index 2410ce1e7bd60..39c46d73e9dda 100644
--- a/clang/lib/Analysis/FlowSensitive/CNFFormula.cpp
+++ b/clang/lib/Analysis/FlowSensitive/CNFFormula.cpp
@@ -110,7 +110,7 @@ void CNFFormula::addClause(ArrayRef<Literal> lits) {
 
   const size_t S = Clauses.size();
   ClauseStarts.push_back(S);
-  Clauses.insert(Clauses.end(), lits.begin(), lits.end());
+  llvm::append_range(Clauses, lits);
 }
 
 CNFFormula buildCNF(const llvm::ArrayRef<const Formula *> &Formulas,
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index fad698d985af7..390ef0f3ac884 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -438,7 +438,7 @@ static void populateNegativeRISCVFeatures(std::vector<std::string> &Features) {
 
   std::vector<std::string> FeatStrings =
       (*RII)->toFeatures(/* AddAllExtensions */ true);
-  Features.insert(Features.end(), FeatStrings.begin(), FeatStrings.end());
+  llvm::append_range(Features, FeatStrings);
 }
 
 static void handleFullArchString(StringRef FullArchStr,
@@ -454,7 +454,7 @@ static void handleFullArchString(StringRef FullArchStr,
     populateNegativeRISCVFeatures(Features);
     std::vector<std::string> FeatStrings =
         (*RII)->toFeatures(/* AddAllExtensions */ true);
-    Features.insert(Features.end(), FeatStrings.begin(), FeatStrings.end());
+    llvm::append_range(Features, FeatStrings);
   }
 }
 
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 1f11347b81411..1c23a8b4db918 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -7120,7 +7120,7 @@ RValue CGObjCNonFragileABIMac::EmitVTableMessageSend(
   // the actual argument value blank for now.
   args.add(RValue::get(nullptr), ObjCTypes.MessageRefCPtrTy);
 
-  args.insert(args.end(), formalArgs.begin(), formalArgs.end());
+  llvm::append_range(args, formalArgs);
 
   MessageSendInfo MSI = getMessageSendInfo(method, resultType, args);
 
diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp
index f8c8d0a425659..45fa583650f03 100644
--- a/clang/lib/Interpreter/Interpreter.cpp
+++ b/clang/lib/Interpreter/Interpreter.cpp
@@ -203,7 +203,7 @@ IncrementalCompilerBuilder::CreateCpp() {
   Argv.push_back("wasm32-unknown-emscripten");
   Argv.push_back("-fvisibility=default");
 #endif
-  Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end());
+  llvm::append_range(Argv, UserArgs);
 
   std::string TT = TargetTriple ? *TargetTriple : llvm::sys::getProcessTriple();
   return IncrementalCompilerBuilder::create(TT, Argv);
@@ -232,7 +232,7 @@ IncrementalCompilerBuilder::createCuda(bool device) {
     Argv.push_back(ArchArg.c_str());
   }
 
-  Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end());
+  llvm::append_range(Argv, UserArgs);
 
   std::string TT = TargetTriple ? *TargetTriple : llvm::sys::getProcessTriple();
   return IncrementalCompilerBuilder::create(TT, Argv);
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 8444ff3332e08..69d40baaf4298 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -673,7 +673,7 @@ unsigned Parser::ParseAttributeArgsCommon(
         }
       }
 
-      ArgExprs.insert(ArgExprs.end(), ParsedExprs.begin(), ParsedExprs.end());
+      llvm::append_range(ArgExprs, ParsedExprs);
     }
   }
 
diff --git a/clang/lib/Parse/ParseStmtAsm.cpp b/clang/lib/Parse/ParseStmtAsm.cpp
index e32094bdd99b8..f3415c30dde7a 100644
--- a/clang/lib/Parse/ParseStmtAsm.cpp
+++ b/clang/lib/Parse/ParseStmtAsm.cpp
@@ -645,7 +645,7 @@ StmtResult Parser::ParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
   });
 
   // Build the vector of clobber StringRefs.
-  ClobberRefs.insert(ClobberRefs.end(), Clobbers.begin(), Clobbers.end());
+  llvm::append_range(ClobberRefs, Clobbers);
 
   // Recast the void pointers and build the vector of constraint StringRefs.
   unsigned NumExprs = NumOutputs + NumInputs;
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 26c2dc655a931..c48e009fd7242 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -511,7 +511,7 @@ struct BuiltinDumpStructGenerator {
     Args.reserve((TheCall->getNumArgs() - 2) + /*Format*/ 1 + Exprs.size());
     Args.assign(TheCall->arg_begin() + 2, TheCall->arg_end());
     Args.push_back(getStringLiteral(Format));
-    Args.insert(Args.end(), Exprs.begin(), Exprs.end());
+    llvm::append_range(Args, Exprs);
 
     // Register a note to explain why we're performing the call.
     Sema::CodeSynthesisContext Ctx;
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 394e56c465c24..d46322d75721e 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -2532,7 +2532,7 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
                               : &OpaqueAllocationSize);
     if (isAlignedAllocation(IAP.PassAlignment))
       CallArgs.emplace_back(&DesiredAlignment);
-    CallArgs.insert(CallArgs.end(), PlacementArgs.begin(), PlacementArgs.end());
+    llvm::append_range(CallArgs, PlacementArgs);
 
     DiagnoseSentinelCalls(OperatorNew, PlacementLParen, CallArgs);
 
@@ -2969,7 +2969,7 @@ bool Sema::FindAllocationFunctions(
   if (IncludeAlignParam)
     AllocArgs.push_back(&Align);
 
-  AllocArgs.insert(AllocArgs.end(), PlaceArgs.begin(), PlaceArgs.end());
+  llvm::append_range(AllocArgs, PlaceArgs);
 
   // Find the allocation function.
   {
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 10c22817b082c..65f3275b3469f 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -2749,7 +2749,7 @@ static void BuildFlattenedTypeList(QualType BaseTy,
       BuildFlattenedTypeList(AT->getElementType(), ElementFields);
       // Repeat the element's field list n times.
       for (uint64_t Ct = 0; Ct < AT->getZExtSize(); ++Ct)
-        List.insert(List.end(), ElementFields.begin(), ElementFields.end());
+        llvm::append_range(List, ElementFields);
       continue;
     }
     // Vectors can only have element types that are builtin types, so this can
@@ -2777,7 +2777,7 @@ static void BuildFlattenedTypeList(QualType BaseTy,
         FieldTypes.push_back(FD->getType());
       // Reverse the newly added sub-range.
       std::reverse(FieldTypes.begin(), FieldTypes.end());
-      WorkList.insert(WorkList.end(), FieldTypes.begin(), FieldTypes.end());
+      llvm::append_range(WorkList, FieldTypes);
 
       // If this wasn't a standard layout type we may also have some base
       // classes to deal with.
@@ -2786,7 +2786,7 @@ static void BuildFlattenedTypeList(QualType BaseTy,
         for (const auto &Base : RD->bases())
           FieldTypes.push_back(Base.getType());
         std::reverse(FieldTypes.begin(), FieldTypes.end());
-        WorkList.insert(WorkList.end(), FieldTypes.begin(), FieldTypes.end());
+        llvm::append_range(WorkList, FieldTypes);
       }
       continue;
     }
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 58134603424a6..53f64e91588bf 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -5611,7 +5611,7 @@ void Sema::diagnoseMissingImport(SourceLocation Loc, const NamedDecl *Decl,
   llvm::SmallVector<Module*, 8> OwningModules;
   OwningModules.push_back(Owner);
   auto Merged = Context.getModulesWithMergedDefinition(Def);
-  OwningModules.insert(OwningModules.end(), Merged.begin(), Merged.end());
+  llvm::append_range(OwningModules, Merged);
 
   diagnoseMissingImport(Loc, Def, Def->getLocation(), OwningModules, MIK,
                         Recover);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index b9a09ade37c12..071389543ab68 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -4205,7 +4205,7 @@ class TreeTransform {
       ArrayRef<OpenACCClause *> Clauses) {
     llvm::SmallVector<Expr *> Exprs;
     Exprs.push_back(DevNumExpr);
-    Exprs.insert(Exprs.end(), QueueIdExprs.begin(), QueueIdExprs.end());
+    llvm::append_range(Exprs, QueueIdExprs);
     return getSema().OpenACC().ActOnEndStmtDirective(
         OpenACCDirectiveKind::Wait, BeginLoc, DirLoc, LParenLoc, QueuesLoc,
         Exprs, OpenACCAtomicKind::None, RParenLoc, EndLoc, Clauses, {});
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index ffbe2c4e4d0f1..04e73b17af3bc 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5242,13 +5242,13 @@ void ASTWriter::AddToken(const Token &Tok, RecordDataImpl &Record) {
 
 void ASTWriter::AddString(StringRef Str, RecordDataImpl &Record) {
   Record.push_back(Str.size());
-  Record.insert(Record.end(), Str.begin(), Str.end());
+  llvm::append_range(Record, Str);
 }
 
 void ASTWriter::AddStringBlob(StringRef Str, RecordDataImpl &Record,
                               SmallVectorImpl<char> &Blob) {
   Record.push_back(Str.size());
-  Blob.insert(Blob.end(), Str.begin(), Str.end());
+  llvm::append_range(Blob, Str);
 }
 
 bool ASTWriter::PreparePathForOutput(SmallVectorImpl<char> &Path) {
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index 5f78fc433275d..f8db8d0bc8343 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -2827,7 +2827,7 @@ static void CompactMacroExpandedPieces(PathPieces &path,
   // Now take the pieces and construct a new PathDiagnostic.
   path.clear();
 
-  path.insert(path.end(), Pieces.begin(), Pieces.end());
+  llvm::append_range(path, Pieces);
 }
 
 /// Generate notes from all visitors.
diff --git a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
index 79cb5a07701fd..1cc9cb84cbfa4 100644
--- a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
@@ -2434,7 +2434,7 @@ RegionStoreManager::getInterestingValues(nonloc::LazyCompoundVal LCV) {
 
     if (auto InnerLCV = V.getAs<nonloc::LazyCompoundVal>()) {
       const SValListTy &InnerList = getInterestingValues(*InnerLCV);
-      List.insert(List.end(), InnerList.begin(), InnerList.end());
+      llvm::append_range(List, InnerList);
     }
 
     List.push_back(V);
diff --git a/clang/lib/Tooling/Tooling.cpp b/clang/lib/Tooling/Tooling.cpp
index 03523c3f17eda..f0955ef4a0829 100644
--- a/clang/lib/Tooling/Tooling.cpp
+++ b/clang/lib/Tooling/Tooling.cpp
@@ -198,7 +198,7 @@ getSyntaxOnlyToolArgs(const Twine &ToolName,
   std::vector<std::string> Args;
   Args.push_back(ToolName.str());
   Args.push_back("-fsyntax-only");
-  Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end());
+  llvm::append_range(Args, ExtraArgs);
   Args.push_back(FileName.str());
   return Args;
 }
diff --git a/clang/unittests/Interpreter/ExceptionTests/InterpreterExceptionTest.cpp b/clang/unittests/Interpreter/ExceptionTests/InterpreterExceptionTest.cpp
index bc2bd11df6aef..c8522e9160276 100644
--- a/clang/unittests/Interpreter/ExceptionTests/InterpreterExceptionTest.cpp
+++ b/clang/unittests/Interpreter/ExceptionTests/InterpreterExceptionTest.cpp
@@ -49,7 +49,7 @@ static std::unique_ptr<Interpreter>
 createInterpreter(const Args &ExtraArgs = {},
                   DiagnosticConsumer *Client = nullptr) {
   Args ClangArgs = {"-Xclang", "-emit-llvm-only"};
-  ClangArgs.insert(ClangArgs.end(), ExtraArgs.begin(), ExtraArgs.end());
+  llvm::append_range(ClangArgs, ExtraArgs);
   auto CB = clang::IncrementalCompilerBuilder();
   CB.SetCompilerArgs(ClangArgs);
   auto CI = cantFail(CB.CreateCpp());
diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp
index 578f1d4c0eac6..03b9c33232080 100644
--- a/clang/unittests/Interpreter/InterpreterTest.cpp
+++ b/clang/unittests/Interpreter/InterpreterTest.cpp
@@ -43,7 +43,7 @@ static std::unique_ptr<Interpreter>
 createInterpreter(const Args &ExtraArgs = {},
                   DiagnosticConsumer *Client = nullptr) {
   Args ClangArgs = {"-Xclang", "-emit-llvm-only"};
-  ClangArgs.insert(ClangArgs.end(), ExtraArgs.begin(), ExtraArgs.end());
+  llvm::append_range(ClangArgs, ExtraArgs);
   auto CB = clang::IncrementalCompilerBuilder();
   CB.SetCompilerArgs(ClangArgs);
   auto CI = cantFail(CB.CreateCpp());
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
index 49eb40ec5cda5..db24874ed2498 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
@@ -42,7 +42,7 @@ class TestFileCollector : public DependencyFileGenerator {
 
   void finishedMainFile(DiagnosticsEngine &Diags) override {
     auto NewDeps = getDependencies();
-    Deps.insert(Deps.end(), NewDeps.begin(), NewDeps.end());
+    llvm::append_range(Deps, NewDeps);
   }
 
 private:
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index 07844c8f0967b..bc5033ed3c89a 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -4560,7 +4560,7 @@ static void GenerateTargetRequirements(const Record &Attr,
         std::vector<StringRef> DA =
             I.second->getValueAsDef("Target")->getValueAsListOfStrings(
                 "Arches");
-        Arches.insert(Arches.end(), DA.begin(), DA.end());
+        llvm::append_range(Arches, DA);
       }
     }
   }
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index 7bee2996382c1..95fd658b3bde2 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -753,7 +753,7 @@ class IRIntrinsicResult : public Result {
     OS << "})";
   }
   void morePrerequisites(std::vector<Ptr> &output) const override {
-    output.insert(output.end(), Args.begin(), Args.end());
+    llvm::append_range(output, Args);
   }
 };
 

From 0c4309bcc4154562ae91c3705a9e6c398833119b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 19 Apr 2025 20:45:04 +0200
Subject: [PATCH 558/710] IR: Avoid repeating assert condition in Value
 destructor (#136340)

---
 llvm/lib/IR/Value.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 6c52ced5f73b2..0eaf75b7bfaec 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -98,9 +98,10 @@ Value::~Value() {
     dbgs() << "While deleting: " << *VTy << " %" << getName() << "\n";
     for (auto *U : users())
       dbgs() << "Use still stuck around after Def is destroyed:" << *U << "\n";
+
+    llvm_unreachable("Uses remain when a value is destroyed!");
   }
 #endif
-  assert(materialized_use_empty() && "Uses remain when a value is destroyed!");
 
   // If this value is named, destroy the name.  This should not be in a symtab
   // at this point.

From 031475594abd8a66b88a1687706651579e840421 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 11:49:19 -0700
Subject: [PATCH 559/710] [llvm] Use llvm::SmallVector::pop_back_val (NFC)
 (#136441)

---
 llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp | 6 ++----
 llvm/lib/CodeGen/TwoAddressInstructionPass.cpp | 3 +--
 llvm/lib/Support/SuffixTree.cpp                | 3 +--
 llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp | 3 +--
 llvm/lib/Transforms/Utils/CodeMoverUtils.cpp   | 3 +--
 5 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index c14b50120b16e..a04b6e4c061a7 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -1793,8 +1793,7 @@ bool ComplexDeinterleavingGraph::checkNodes() {
   // Extract all instructions that are used by all XCMLA/XCADD/ADD/SUB/NEG
   // chains
   while (!Worklist.empty()) {
-    auto *I = Worklist.back();
-    Worklist.pop_back();
+    auto *I = Worklist.pop_back_val();
 
     if (!AllInstructions.insert(I).second)
       continue;
@@ -1828,8 +1827,7 @@ bool ComplexDeinterleavingGraph::checkNodes() {
   // that somehow connect to those instructions.
   SmallPtrSet<Instruction *, 16> Visited;
   while (!Worklist.empty()) {
-    auto *I = Worklist.back();
-    Worklist.pop_back();
+    auto *I = Worklist.pop_back_val();
     if (!Visited.insert(I).second)
       continue;
 
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index 2ea99e332cbda..36e2156165f49 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -854,8 +854,7 @@ void TwoAddressInstructionImpl::scanUses(Register DstReg) {
   }
 
   if (!VirtRegPairs.empty()) {
-    Register ToReg = VirtRegPairs.back();
-    VirtRegPairs.pop_back();
+    Register ToReg = VirtRegPairs.pop_back_val();
     while (!VirtRegPairs.empty()) {
       Register FromReg = VirtRegPairs.pop_back_val();
       bool isNew = DstRegMap.insert(std::make_pair(FromReg, ToReg)).second;
diff --git a/llvm/lib/Support/SuffixTree.cpp b/llvm/lib/Support/SuffixTree.cpp
index b2e606c86dd57..daf66108657b3 100644
--- a/llvm/lib/Support/SuffixTree.cpp
+++ b/llvm/lib/Support/SuffixTree.cpp
@@ -306,8 +306,7 @@ void SuffixTree::RepeatedSubstringIterator::advance() {
   // Continue visiting nodes until we find one which repeats more than once.
   while (!InternalNodesToVisit.empty()) {
     RepeatedSubstringStarts.clear();
-    auto *Curr = InternalNodesToVisit.back();
-    InternalNodesToVisit.pop_back();
+    auto *Curr = InternalNodesToVisit.pop_back_val();
 
     // Keep track of the length of the string associated with the node. If
     // it's too short, we'll quit.
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 17a35c66c5966..6407f48dc2c05 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -1528,8 +1528,7 @@ bool MemCpyOptPass::performStackMoveOptzn(Instruction *Load, Instruction *Store,
     Worklist.reserve(MaxUsesToExplore);
     SmallSet<const Use *, 20> Visited;
     while (!Worklist.empty()) {
-      Instruction *I = Worklist.back();
-      Worklist.pop_back();
+      Instruction *I = Worklist.pop_back_val();
       for (const Use &U : I->uses()) {
         auto *UI = cast<Instruction>(U.getUser());
         // If any use that isn't dominated by SrcAlloca exists, we move src
diff --git a/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp b/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
index ecf71b6056f2a..0e076c60d6085 100644
--- a/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
+++ b/llvm/lib/Transforms/Utils/CodeMoverUtils.cpp
@@ -464,8 +464,7 @@ bool llvm::nonStrictlyPostDominate(const BasicBlock *ThisBlock,
   SmallPtrSet<const BasicBlock *, 8> Visited;
   WorkList.push_back(ThisBlock);
   while (!WorkList.empty()) {
-    const BasicBlock *CurBlock = WorkList.back();
-    WorkList.pop_back();
+    const BasicBlock *CurBlock = WorkList.pop_back_val();
     Visited.insert(CurBlock);
     if (PDT->dominates(CurBlock, OtherBlock))
       return true;

From c2d6c7cea7d857f7a9268703e30a7d1ffed7d64b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 12:21:14 -0700
Subject: [PATCH 560/710] [clang] Use llvm::append_range (NFC) (#136448)

---
 clang/lib/AST/ByteCode/State.cpp               |  6 ++----
 clang/lib/AST/ExprConstant.cpp                 |  6 ++----
 clang/lib/AST/ItaniumMangle.cpp                | 18 ++++++------------
 clang/lib/AST/Randstruct.cpp                   |  6 ++----
 clang/lib/Analysis/IntervalPartition.cpp       |  2 +-
 clang/lib/CodeGen/CGCall.h                     |  9 +++------
 clang/lib/CodeGen/CGLoopInfo.cpp               |  3 +--
 clang/lib/Driver/MultilibBuilder.cpp           |  7 +++----
 clang/lib/Driver/ToolChains/HIPUtility.cpp     |  3 +--
 clang/lib/Format/SortJavaScriptImports.cpp     |  3 +--
 clang/lib/Frontend/CompilerInstance.cpp        |  3 +--
 clang/lib/Frontend/CompilerInvocation.cpp      |  2 +-
 clang/lib/Sema/ParsedAttr.cpp                  |  4 ++--
 clang/lib/Sema/SemaConcept.cpp                 |  3 +--
 clang/lib/Serialization/MultiOnDiskHashTable.h |  3 +--
 .../DependencyScanningWorker.cpp               |  3 +--
 clang/tools/clang-installapi/Options.cpp       |  3 +--
 .../Format/MacroCallReconstructorTest.cpp      |  3 +--
 18 files changed, 31 insertions(+), 56 deletions(-)

diff --git a/clang/lib/AST/ByteCode/State.cpp b/clang/lib/AST/ByteCode/State.cpp
index b4db86e8d22c7..3204d1a193a74 100644
--- a/clang/lib/AST/ByteCode/State.cpp
+++ b/clang/lib/AST/ByteCode/State.cpp
@@ -67,10 +67,8 @@ OptionalDiagnostic State::Note(SourceLocation Loc, diag::kind DiagId) {
 }
 
 void State::addNotes(ArrayRef<PartialDiagnosticAt> Diags) {
-  if (hasActiveDiagnostic()) {
-    getEvalStatus().Diag->insert(getEvalStatus().Diag->end(), Diags.begin(),
-                                 Diags.end());
-  }
+  if (hasActiveDiagnostic())
+    llvm::append_range(*getEvalStatus().Diag, Diags);
 }
 
 DiagnosticBuilder State::report(SourceLocation Loc, diag::kind DiagId) {
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index b14ff21a8ebc2..f598ef5929aa4 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -298,8 +298,7 @@ namespace {
       assert(V.isLValue() && "Non-LValue used to make an LValue designator?");
       if (!Invalid) {
         IsOnePastTheEnd = V.isLValueOnePastTheEnd();
-        ArrayRef<PathEntry> VEntries = V.getLValuePath();
-        Entries.insert(Entries.end(), VEntries.begin(), VEntries.end());
+        llvm::append_range(Entries, V.getLValuePath());
         if (V.getLValueBase()) {
           bool IsArray = false;
           bool FirstIsUnsizedArray = false;
@@ -1832,8 +1831,7 @@ namespace {
       DeclAndIsDerivedMember.setPointer(V.getMemberPointerDecl());
       DeclAndIsDerivedMember.setInt(V.isMemberPointerToDerivedMember());
       Path.clear();
-      ArrayRef<const CXXRecordDecl*> P = V.getMemberPointerPath();
-      Path.insert(Path.end(), P.begin(), P.end());
+      llvm::append_range(Path, V.getMemberPointerPath());
     }
 
     /// DeclAndIsDerivedMember - The member declaration, and a flag indicating
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 977caa8e55db3..97ef1c0d46960 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -309,10 +309,8 @@ class CXXNameMangler {
             !AdditionalAbiTags &&
             "only function and variables need a list of additional abi tags");
         if (const auto *NS = dyn_cast<NamespaceDecl>(ND)) {
-          if (const auto *AbiTag = NS->getAttr<AbiTagAttr>()) {
-            UsedAbiTags.insert(UsedAbiTags.end(), AbiTag->tags().begin(),
-                               AbiTag->tags().end());
-          }
+          if (const auto *AbiTag = NS->getAttr<AbiTagAttr>())
+            llvm::append_range(UsedAbiTags, AbiTag->tags());
           // Don't emit abi tags for namespaces.
           return;
         }
@@ -320,17 +318,13 @@ class CXXNameMangler {
 
       AbiTagList TagList;
       if (const auto *AbiTag = ND->getAttr<AbiTagAttr>()) {
-        UsedAbiTags.insert(UsedAbiTags.end(), AbiTag->tags().begin(),
-                           AbiTag->tags().end());
-        TagList.insert(TagList.end(), AbiTag->tags().begin(),
-                       AbiTag->tags().end());
+        llvm::append_range(UsedAbiTags, AbiTag->tags());
+        llvm::append_range(TagList, AbiTag->tags());
       }
 
       if (AdditionalAbiTags) {
-        UsedAbiTags.insert(UsedAbiTags.end(), AdditionalAbiTags->begin(),
-                           AdditionalAbiTags->end());
-        TagList.insert(TagList.end(), AdditionalAbiTags->begin(),
-                       AdditionalAbiTags->end());
+        llvm::append_range(UsedAbiTags, *AdditionalAbiTags);
+        llvm::append_range(TagList, *AdditionalAbiTags);
       }
 
       llvm::sort(TagList);
diff --git a/clang/lib/AST/Randstruct.cpp b/clang/lib/AST/Randstruct.cpp
index 5cba2f0a3c154..c1bd76a2d0962 100644
--- a/clang/lib/AST/Randstruct.cpp
+++ b/clang/lib/AST/Randstruct.cpp
@@ -208,12 +208,10 @@ bool randomizeStructureLayout(const ASTContext &Context, RecordDecl *RD,
   randomizeStructureLayoutImpl(Context, RandomizedFields, RNG);
 
   // Plorp the randomized decls into the final ordering.
-  FinalOrdering.insert(FinalOrdering.end(), RandomizedFields.begin(),
-                       RandomizedFields.end());
+  llvm::append_range(FinalOrdering, RandomizedFields);
 
   // Add fields that belong towards the end of the RecordDecl.
-  FinalOrdering.insert(FinalOrdering.end(), PostRandomizedFields.begin(),
-                       PostRandomizedFields.end());
+  llvm::append_range(FinalOrdering, PostRandomizedFields);
 
   // Add back the flexible array.
   if (FlexibleArray)
diff --git a/clang/lib/Analysis/IntervalPartition.cpp b/clang/lib/Analysis/IntervalPartition.cpp
index 41199f358c5b9..0733239925747 100644
--- a/clang/lib/Analysis/IntervalPartition.cpp
+++ b/clang/lib/Analysis/IntervalPartition.cpp
@@ -132,7 +132,7 @@ static void fillIntervalNode(CFGIntervalGraph &Graph,
       Count += N->Nodes.size();
     Nodes.reserve(Count);
     for (auto &N : Result.Nodes)
-      Nodes.insert(Nodes.end(), N->Nodes.begin(), N->Nodes.end());
+      llvm::append_range(Nodes, N->Nodes);
     Interval.Nodes = std::move(Nodes);
   }
 }
diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h
index 92e0cc43919ca..0b4e3f9cb0365 100644
--- a/clang/lib/CodeGen/CGCall.h
+++ b/clang/lib/CodeGen/CGCall.h
@@ -312,12 +312,9 @@ class CallArgList : public SmallVector<CallArg, 8> {
   /// this, the old CallArgList retains its list of arguments, but must not
   /// be used to emit a call.
   void addFrom(const CallArgList &other) {
-    insert(end(), other.begin(), other.end());
-    Writebacks.insert(Writebacks.end(), other.Writebacks.begin(),
-                      other.Writebacks.end());
-    CleanupsToDeactivate.insert(CleanupsToDeactivate.end(),
-                                other.CleanupsToDeactivate.begin(),
-                                other.CleanupsToDeactivate.end());
+    llvm::append_range(*this, other);
+    llvm::append_range(Writebacks, other.Writebacks);
+    llvm::append_range(CleanupsToDeactivate, other.CleanupsToDeactivate);
     assert(!(StackBase && other.StackBase) && "can't merge stackbases");
     if (!StackBase)
       StackBase = other.StackBase;
diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp
index 2b7d7881ab990..4a9092842858b 100644
--- a/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -424,8 +424,7 @@ SmallVector<Metadata *, 4> LoopInfo::createMetadata(
     LoopProperties.push_back(MDNode::get(Ctx, Vals));
   }
 
-  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
-                        AdditionalLoopProperties.end());
+  llvm::append_range(LoopProperties, AdditionalLoopProperties);
   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
 }
 
diff --git a/clang/lib/Driver/MultilibBuilder.cpp b/clang/lib/Driver/MultilibBuilder.cpp
index 4fbe9d9047bde..8440a82f78bef 100644
--- a/clang/lib/Driver/MultilibBuilder.cpp
+++ b/clang/lib/Driver/MultilibBuilder.cpp
@@ -142,8 +142,8 @@ static MultilibBuilder compose(const MultilibBuilder &Base,
 
   MultilibBuilder::flags_list &Flags = Composed.flags();
 
-  Flags.insert(Flags.end(), Base.flags().begin(), Base.flags().end());
-  Flags.insert(Flags.end(), New.flags().begin(), New.flags().end());
+  llvm::append_range(Flags, Base.flags());
+  llvm::append_range(Flags, New.flags());
 
   return Composed;
 }
@@ -153,8 +153,7 @@ MultilibSetBuilder::Either(ArrayRef<MultilibBuilder> MultilibSegments) {
   multilib_list Composed;
 
   if (Multilibs.empty())
-    Multilibs.insert(Multilibs.end(), MultilibSegments.begin(),
-                     MultilibSegments.end());
+    llvm::append_range(Multilibs, MultilibSegments);
   else {
     for (const auto &New : MultilibSegments) {
       for (const auto &Base : Multilibs) {
diff --git a/clang/lib/Driver/ToolChains/HIPUtility.cpp b/clang/lib/Driver/ToolChains/HIPUtility.cpp
index b4469c05cea3f..1fc9118419684 100644
--- a/clang/lib/Driver/ToolChains/HIPUtility.cpp
+++ b/clang/lib/Driver/ToolChains/HIPUtility.cpp
@@ -190,8 +190,7 @@ class HIPUndefinedFatBinSymbols {
 
         processInput(BufferOrErr.get()->getMemBufferRef());
       } else
-        WorkList.insert(WorkList.end(), CurrentAction->getInputs().begin(),
-                        CurrentAction->getInputs().end());
+        llvm::append_range(WorkList, CurrentAction->getInputs());
     }
   }
 
diff --git a/clang/lib/Format/SortJavaScriptImports.cpp b/clang/lib/Format/SortJavaScriptImports.cpp
index 1acce26ff2795..f55993fb4c11c 100644
--- a/clang/lib/Format/SortJavaScriptImports.cpp
+++ b/clang/lib/Format/SortJavaScriptImports.cpp
@@ -276,8 +276,7 @@ class JavaScriptImportSorter : public TokenAnalyzer {
       }
       stable_sort(SortChunk);
       mergeModuleReferences(SortChunk);
-      ReferencesSorted.insert(ReferencesSorted.end(), SortChunk.begin(),
-                              SortChunk.end());
+      llvm::append_range(ReferencesSorted, SortChunk);
     }
     return ReferencesSorted;
   }
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 93e4e31c2891d..533cef35e776b 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -598,8 +598,7 @@ struct ReadModuleNames : ASTReaderListener {
           if (Current->IsUnimportable) continue;
           Current->IsAvailable = true;
           auto SubmodulesRange = Current->submodules();
-          Stack.insert(Stack.end(), SubmodulesRange.begin(),
-                       SubmodulesRange.end());
+          llvm::append_range(Stack, SubmodulesRange);
         }
       }
     }
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index cfc5c069b0849..9e9eed4efc577 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -2078,7 +2078,7 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
     A->render(Args, ASL);
     for (const auto &arg : ASL) {
       StringRef ArgStr(arg);
-      Opts.CmdArgs.insert(Opts.CmdArgs.end(), ArgStr.begin(), ArgStr.end());
+      llvm::append_range(Opts.CmdArgs, ArgStr);
       // using \00 to separate each commandline options.
       Opts.CmdArgs.push_back('\0');
     }
diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp
index c149cef478539..ba12a5aae5773 100644
--- a/clang/lib/Sema/ParsedAttr.cpp
+++ b/clang/lib/Sema/ParsedAttr.cpp
@@ -86,14 +86,14 @@ void AttributeFactory::reclaimPool(AttributePool &cur) {
 }
 
 void AttributePool::takePool(AttributePool &pool) {
-  Attrs.insert(Attrs.end(), pool.Attrs.begin(), pool.Attrs.end());
+  llvm::append_range(Attrs, pool.Attrs);
   pool.Attrs.clear();
 }
 
 void AttributePool::takeFrom(ParsedAttributesView &List, AttributePool &Pool) {
   assert(&Pool != this && "AttributePool can't take attributes from itself");
   llvm::for_each(List.AttrList, [&Pool](ParsedAttr *A) { Pool.remove(A); });
-  Attrs.insert(Attrs.end(), List.AttrList.begin(), List.AttrList.end());
+  llvm::append_range(Attrs, List.AttrList);
 }
 
 namespace {
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 5424d207d3c77..15b9c97489e7f 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -639,8 +639,7 @@ bool Sema::CheckConstraintSatisfaction(
   // here.
   llvm::SmallVector<TemplateArgument, 4> FlattenedArgs;
   for (auto List : TemplateArgsLists)
-    FlattenedArgs.insert(FlattenedArgs.end(), List.Args.begin(),
-                         List.Args.end());
+    llvm::append_range(FlattenedArgs, List.Args);
 
   llvm::FoldingSetNodeID ID;
   ConstraintSatisfaction::Profile(ID, Context, Template, FlattenedArgs);
diff --git a/clang/lib/Serialization/MultiOnDiskHashTable.h b/clang/lib/Serialization/MultiOnDiskHashTable.h
index 996e9b94287f4..a179016d47419 100644
--- a/clang/lib/Serialization/MultiOnDiskHashTable.h
+++ b/clang/lib/Serialization/MultiOnDiskHashTable.h
@@ -209,8 +209,7 @@ template<typename Info> class MultiOnDiskHashTable {
     OverriddenFiles.reserve(NumFiles);
     for (/**/; NumFiles != 0; --NumFiles)
       OverriddenFiles.push_back(InfoObj.ReadFileRef(Ptr));
-    PendingOverrides.insert(PendingOverrides.end(), OverriddenFiles.begin(),
-                            OverriddenFiles.end());
+    llvm::append_range(PendingOverrides, OverriddenFiles);
 
     // Read the OnDiskChainedHashTable header.
     storage_type Buckets = Data + BucketOffset;
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index bae436afe0897..b88a7cb2dca21 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -752,8 +752,7 @@ bool DependencyScanningWorker::scanDependencies(
           // Insert -cc1 comand line options into Argv
           std::vector<std::string> Argv;
           Argv.push_back(Cmd.getExecutable());
-          Argv.insert(Argv.end(), Cmd.getArguments().begin(),
-                      Cmd.getArguments().end());
+          llvm::append_range(Argv, Cmd.getArguments());
 
           // Create an invocation that uses the underlying file
           // system to ensure that any file system requests that
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index efb36848dbbf1..9bc168c8cd4f8 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -801,8 +801,7 @@ std::pair<LibAttrs, ReexportedInterfaces> Options::getReexportedLibraries() {
   PathSeq FwkSearchPaths(FEOpts.FwkPaths.begin(), FEOpts.FwkPaths.end());
   for (const PlatformType P : Platforms) {
     PathSeq PlatformSearchPaths = getPathsForPlatform(FEOpts.SystemFwkPaths, P);
-    FwkSearchPaths.insert(FwkSearchPaths.end(), PlatformSearchPaths.begin(),
-                          PlatformSearchPaths.end());
+    llvm::append_range(FwkSearchPaths, PlatformSearchPaths);
     for (const StringMapEntry<ArchitectureSet> &Lib :
          LinkerOpts.ReexportedFrameworks) {
       std::string Name = (Lib.getKey() + ".framework/" + Lib.getKey()).str();
diff --git a/clang/unittests/Format/MacroCallReconstructorTest.cpp b/clang/unittests/Format/MacroCallReconstructorTest.cpp
index b58241fd8c4e8..86997524fd49b 100644
--- a/clang/unittests/Format/MacroCallReconstructorTest.cpp
+++ b/clang/unittests/Format/MacroCallReconstructorTest.cpp
@@ -165,8 +165,7 @@ class MacroCallReconstructorTest : public testing::Test {
     UnwrappedLine Result;
     Result.Level = Level;
     for (const Chunk &Chunk : Chunks) {
-      Result.Tokens.insert(Result.Tokens.end(), Chunk.Tokens.begin(),
-                           Chunk.Tokens.end());
+      llvm::append_range(Result.Tokens, Chunk.Tokens);
       assert(!Result.Tokens.empty());
       Result.Tokens.back().Children.append(Chunk.Children.begin(),
                                            Chunk.Children.end());

From 1cf188a1bca6d1ce135eab4cce5dead4f8f55026 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 13:21:29 -0700
Subject: [PATCH 561/710] [mlir] Use llvm::SmallVector::pop_back_val() (NFC)
 (#136452)

---
 mlir/lib/Analysis/Liveness.cpp | 3 +--
 mlir/lib/Rewrite/ByteCode.cpp  | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Analysis/Liveness.cpp b/mlir/lib/Analysis/Liveness.cpp
index 3e39d974c0e21..3fcc0b5a6ffef 100644
--- a/mlir/lib/Analysis/Liveness.cpp
+++ b/mlir/lib/Analysis/Liveness.cpp
@@ -198,8 +198,7 @@ Liveness::OperationListT Liveness::resolveLiveness(Value value) const {
 
   while (!toProcess.empty()) {
     // Get block and block liveness information.
-    Block *block = toProcess.back();
-    toProcess.pop_back();
+    Block *block = toProcess.pop_back_val();
     const LivenessBlockInfo *blockInfo = getLiveness(block);
 
     // Note that start and end will be in the same block.
diff --git a/mlir/lib/Rewrite/ByteCode.cpp b/mlir/lib/Rewrite/ByteCode.cpp
index 26510f976efc9..17cb3a74184f1 100644
--- a/mlir/lib/Rewrite/ByteCode.cpp
+++ b/mlir/lib/Rewrite/ByteCode.cpp
@@ -1198,8 +1198,7 @@ class ByteCodeExecutor {
   /// Pops a code iterator from the stack, returning true on success.
   void popCodeIt() {
     assert(!resumeCodeIt.empty() && "attempt to pop code off empty stack");
-    curCodeIt = resumeCodeIt.back();
-    resumeCodeIt.pop_back();
+    curCodeIt = resumeCodeIt.pop_back_val();
   }
 
   /// Return the bytecode iterator at the start of the current op code.

From 94733492b720c2e65a8fe9bd2179a35a7b7d9916 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 13:35:18 -0700
Subject: [PATCH 562/710] [clang] Use llvm::SmallVector::pop_back_val (NFC)
 (#136451)

---
 clang/lib/Analysis/CFG.cpp               | 3 +--
 clang/lib/Basic/Module.cpp               | 3 +--
 clang/lib/Driver/Driver.cpp              | 3 +--
 clang/lib/Sema/AnalysisBasedWarnings.cpp | 3 +--
 clang/lib/Sema/ParsedAttr.cpp            | 3 +--
 clang/lib/Sema/SemaHLSL.cpp              | 6 ++----
 6 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index 9af1e915482da..d03a0a544b016 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -6258,8 +6258,7 @@ bool CFGBlock::isInevitablySinking() const {
 
   DFSWorkList.push_back(StartBlk);
   while (!DFSWorkList.empty()) {
-    const CFGBlock *Blk = DFSWorkList.back();
-    DFSWorkList.pop_back();
+    const CFGBlock *Blk = DFSWorkList.pop_back_val();
     Visited.insert(Blk);
 
     // If at least one path reaches the CFG exit, it means that control is
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index bb85a4010931f..6330bfe9418e8 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -334,8 +334,7 @@ void Module::markUnavailable(bool Unimportable) {
   SmallVector<Module *, 2> Stack;
   Stack.push_back(this);
   while (!Stack.empty()) {
-    Module *Current = Stack.back();
-    Stack.pop_back();
+    Module *Current = Stack.pop_back_val();
 
     if (!needUpdate(Current))
       continue;
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 90d8e823d1d02..2a51ac2e9f0b9 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -2812,8 +2812,7 @@ void Driver::BuildUniversalActions(Compilation &C, const ToolChain &TC,
 
       // Verify the debug info output.
       if (Args.hasArg(options::OPT_verify_debug_info)) {
-        Action* LastAction = Actions.back();
-        Actions.pop_back();
+        Action *LastAction = Actions.pop_back_val();
         Actions.push_back(C.MakeAction<VerifyDebugInfoJobAction>(
             LastAction, types::TY_Nothing));
       }
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index 34045a7274021..32c7ee92466ad 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -313,8 +313,7 @@ static bool throwEscapes(Sema &S, const CXXThrowExpr *E, CFGBlock &ThrowBlock,
   Queued[ThrowBlock.getBlockID()] = true;
 
   while (!Stack.empty()) {
-    CFGBlock &UnwindBlock = *Stack.back();
-    Stack.pop_back();
+    CFGBlock &UnwindBlock = *Stack.pop_back_val();
 
     for (auto &Succ : UnwindBlock.succs()) {
       if (!Succ.isReachable() || Queued[Succ->getBlockID()])
diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp
index ba12a5aae5773..4e4859d538cd6 100644
--- a/clang/lib/Sema/ParsedAttr.cpp
+++ b/clang/lib/Sema/ParsedAttr.cpp
@@ -54,8 +54,7 @@ void *AttributeFactory::allocate(size_t size) {
   // Check for a previously reclaimed attribute.
   size_t index = getFreeListIndexForSize(size);
   if (index < FreeLists.size() && !FreeLists[index].empty()) {
-    ParsedAttr *attr = FreeLists[index].back();
-    FreeLists[index].pop_back();
+    ParsedAttr *attr = FreeLists[index].pop_back_val();
     return attr;
   }
 
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 65f3275b3469f..d725eb4b2fe6c 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -3375,8 +3375,7 @@ static bool BuildInitializerList(Sema &S, ASTContext &Ctx, Expr *E,
       RecordTypes.push_back(D->bases_begin()->getType()->getAs<RecordType>());
     }
     while (!RecordTypes.empty()) {
-      const RecordType *RT = RecordTypes.back();
-      RecordTypes.pop_back();
+      const RecordType *RT = RecordTypes.pop_back_val();
       for (auto *FD : RT->getDecl()->fields()) {
         DeclAccessPair Found = DeclAccessPair::make(FD, FD->getAccess());
         DeclarationNameInfo NameInfo(FD->getDeclName(), E->getBeginLoc());
@@ -3424,8 +3423,7 @@ static Expr *GenerateInitLists(ASTContext &Ctx, QualType Ty,
       RecordTypes.push_back(D->bases_begin()->getType()->getAs<RecordType>());
     }
     while (!RecordTypes.empty()) {
-      const RecordType *RT = RecordTypes.back();
-      RecordTypes.pop_back();
+      const RecordType *RT = RecordTypes.pop_back_val();
       for (auto *FD : RT->getDecl()->fields()) {
         Inits.push_back(GenerateInitLists(Ctx, FD->getType(), It));
       }

From f347a0659132705f32551e3e599fd051d1b5e58e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 13:35:51 -0700
Subject: [PATCH 563/710] [lld] Use llvm::unique (NFC) (#136453)

---
 lld/COFF/MapFile.cpp                            |  7 +++----
 lld/ELF/AArch64ErrataFix.cpp                    | 12 ++++++------
 lld/ELF/ARMErrataFix.cpp                        | 10 +++++-----
 lld/ELF/BPSectionOrderer.cpp                    |  2 +-
 lld/ELF/SyntheticSections.cpp                   |  2 +-
 lld/MachO/BPSectionOrderer.cpp                  |  2 +-
 lld/include/lld/Common/BPSectionOrdererBase.inc |  2 +-
 7 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/lld/COFF/MapFile.cpp b/lld/COFF/MapFile.cpp
index eb98bb484f9f4..9b7c05b9e68b9 100644
--- a/lld/COFF/MapFile.cpp
+++ b/lld/COFF/MapFile.cpp
@@ -75,10 +75,9 @@ static void sortUniqueSymbols(std::vector<Defined *> &syms,
 
   // Remove duplicate symbol pointers
   parallelSort(v, std::less<SortEntry>());
-  auto end = std::unique(v.begin(), v.end(),
-                         [](const SortEntry &a, const SortEntry &b) {
-                           return a.first == b.first;
-                         });
+  auto end = llvm::unique(v, [](const SortEntry &a, const SortEntry &b) {
+    return a.first == b.first;
+  });
   v.erase(end, v.end());
 
   // Sort by RVA then original order
diff --git a/lld/ELF/AArch64ErrataFix.cpp b/lld/ELF/AArch64ErrataFix.cpp
index b5641e5d9ce55..e2b1cf14daa72 100644
--- a/lld/ELF/AArch64ErrataFix.cpp
+++ b/lld/ELF/AArch64ErrataFix.cpp
@@ -461,12 +461,12 @@ void AArch64Err843419Patcher::init() {
     llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) {
       return a->value < b->value;
     });
-    mapSyms.erase(
-        std::unique(mapSyms.begin(), mapSyms.end(),
-                    [=](const Defined *a, const Defined *b) {
-                      return isCodeMapSymbol(a) == isCodeMapSymbol(b);
-                    }),
-        mapSyms.end());
+    mapSyms.erase(llvm::unique(mapSyms,
+                               [=](const Defined *a, const Defined *b) {
+                                 return isCodeMapSymbol(a) ==
+                                        isCodeMapSymbol(b);
+                               }),
+                  mapSyms.end());
     // Always start with a Code Mapping Symbol.
     if (!mapSyms.empty() && !isCodeMapSymbol(mapSyms.front()))
       mapSyms.erase(mapSyms.begin());
diff --git a/lld/ELF/ARMErrataFix.cpp b/lld/ELF/ARMErrataFix.cpp
index a7120c43e51d3..d14d28ee43a2f 100644
--- a/lld/ELF/ARMErrataFix.cpp
+++ b/lld/ELF/ARMErrataFix.cpp
@@ -354,11 +354,11 @@ void ARMErr657417Patcher::init() {
     llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) {
       return a->value < b->value;
     });
-    mapSyms.erase(std::unique(mapSyms.begin(), mapSyms.end(),
-                              [=](const Defined *a, const Defined *b) {
-                                return (isThumbMapSymbol(a) ==
-                                        isThumbMapSymbol(b));
-                              }),
+    mapSyms.erase(llvm::unique(mapSyms,
+                               [=](const Defined *a, const Defined *b) {
+                                 return (isThumbMapSymbol(a) ==
+                                         isThumbMapSymbol(b));
+                               }),
                   mapSyms.end());
     // Always start with a Thumb Mapping Symbol
     if (!mapSyms.empty() && !isThumbMapSymbol(mapSyms.front()))
diff --git a/lld/ELF/BPSectionOrderer.cpp b/lld/ELF/BPSectionOrderer.cpp
index 4adb42ef4ff93..793176c7725a3 100644
--- a/lld/ELF/BPSectionOrderer.cpp
+++ b/lld/ELF/BPSectionOrderer.cpp
@@ -53,7 +53,7 @@ struct BPOrdererELF : lld::BPOrderer<BPOrdererELF> {
       hashes.push_back(byte);
 
     llvm::sort(hashes);
-    hashes.erase(std::unique(hashes.begin(), hashes.end()), hashes.end());
+    hashes.erase(llvm::unique(hashes), hashes.end());
   }
 
   static StringRef getSymName(const Defined &sym) { return sym.getName(); }
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 196ecc4fee8c8..2531227cb99b7 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -590,7 +590,7 @@ SmallVector<EhFrameSection::FdeData, 0> EhFrameSection::getFdeData() const {
   auto eq = [](const FdeData &a, const FdeData &b) {
     return a.pcRel == b.pcRel;
   };
-  ret.erase(std::unique(ret.begin(), ret.end(), eq), ret.end());
+  ret.erase(llvm::unique(ret, eq), ret.end());
 
   return ret;
 }
diff --git a/lld/MachO/BPSectionOrderer.cpp b/lld/MachO/BPSectionOrderer.cpp
index 950afd0421f06..1295d21cad8a1 100644
--- a/lld/MachO/BPSectionOrderer.cpp
+++ b/lld/MachO/BPSectionOrderer.cpp
@@ -73,7 +73,7 @@ struct BPOrdererMachO : lld::BPOrderer<BPOrdererMachO> {
     }
 
     llvm::sort(hashes);
-    hashes.erase(std::unique(hashes.begin(), hashes.end()), hashes.end());
+    hashes.erase(llvm::unique(hashes), hashes.end());
   }
 
   static llvm::StringRef getSymName(const Defined &sym) {
diff --git a/lld/include/lld/Common/BPSectionOrdererBase.inc b/lld/include/lld/Common/BPSectionOrdererBase.inc
index b19d3670d34cc..51dfb6471644a 100644
--- a/lld/include/lld/Common/BPSectionOrdererBase.inc
+++ b/lld/include/lld/Common/BPSectionOrdererBase.inc
@@ -260,7 +260,7 @@ auto BPOrderer<D>::computeOrder(
       auto &uns = startupSectionIdxUNs[sectionIdx];
       uns.append(compressionUns);
       llvm::sort(uns);
-      uns.erase(std::unique(uns.begin(), uns.end()), uns.end());
+      uns.erase(llvm::unique(uns), uns.end());
     }
   }
 

From ec6828c1ecad7fe2f799f7b5af3361885a5d5bb9 Mon Sep 17 00:00:00 2001
From: John Harrison <harjohn@google.com>
Date: Sat, 19 Apr 2025 13:45:59 -0700
Subject: [PATCH 564/710] [lldb-dap] Remove an incorrect assumption on reverse
 requests. (#136210)

Reverse requests do have a 'seq' set still from VSCode. I incorrectly
interpreted
https://github.com/microsoft/vscode/blob/dede7bb4b7e9c9ec69155a243bb84037a40588fe/src/vs/workbench/contrib/debug/common/abstractDebugAdapter.ts#L65
to mean they have a 'seq' of '0', however the 'seq' is set in
'internalSend' here
https://github.com/microsoft/vscode/blob/dede7bb4b7e9c9ec69155a243bb84037a40588fe/src/vs/workbench/contrib/debug/common/abstractDebugAdapter.ts#L178.

Removing the check that 'seq=0' on reverse requests and updating the
dap_server.py impl to also set the seq.
---
 .../Python/lldbsuite/test/tools/lldb-dap/dap_server.py       | 4 ----
 lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp                | 5 -----
 2 files changed, 9 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index 61d7fa94479b8..a9915ba2f6de6 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -343,25 +343,21 @@ def send_recv(self, command):
                     self.send_packet(
                         {
                             "type": "response",
-                            "seq": 0,
                             "request_seq": response_or_request["seq"],
                             "success": True,
                             "command": "runInTerminal",
                             "body": {},
                         },
-                        set_sequence=False,
                     )
                 elif response_or_request["command"] == "startDebugging":
                     self.send_packet(
                         {
                             "type": "response",
-                            "seq": 0,
                             "request_seq": response_or_request["seq"],
                             "success": True,
                             "command": "startDebugging",
                             "body": {},
                         },
-                        set_sequence=False,
                     )
                 else:
                     desc = 'unknown reverse request "%s"' % (
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
index bfd68448fb483..bc4fee4aa8b8d 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
@@ -163,11 +163,6 @@ bool fromJSON(json::Value const &Params, Response &R, json::Path P) {
     return false;
   }
 
-  if (seq != 0) {
-    P.field("seq").report("expected to be '0'");
-    return false;
-  }
-
   if (R.command.empty()) {
     P.field("command").report("expected to not be ''");
     return false;

From e8245d53247f155d234e1de647aac2678c49ac28 Mon Sep 17 00:00:00 2001
From: Evgenii Kudriashov <evgenii.kudriashov@intel.com>
Date: Sat, 19 Apr 2025 22:59:18 +0200
Subject: [PATCH 565/710] [X86][GlobalISel] Support addr matching in SDAG
 patterns (#130445)

addr matching was the only gatekeeper for starting selecting G_LOAD
and G_STORE using SDAG patterns.

* Introduce a complex renderer gi_addr for addr. In this patch only
the existing functionality has been implemented. The renderer's name is
the same as in SDAG: selectAddr. Apparently the type of
GIComplexOperandMatcher doesn't matter as RISCV also uses s32 for
both 64 and 32 bit pointers.
* X86SelectAddress now is used for both: pattern matching and manual
selection. As a result it accumulates all the code that previously was
distributed among different selection functions.
* Replace getLoadStoreOp with getPtrLoadStoreOp in Load/Store
selector as GlobalISel matcher or emitter can't map the pointer type
into i32/i64 types used in SDAG patterns for pointers. So the load and
store selection of pointers is still manual. getLoadStoreOp is still
present because it is used in G_FCONSTANT lowering that requires extra
efforts to select it using SDAG patterns.
* Since truncating stores are not supported, we custom legalize them by
matching types of store and MMO.
* Introduce a constant pool flag in X86AddressMode because otherwise
we need to introduce a GlobalISel copy for X86ISelAddressMode.
* Also please notice in the tests that GlobalISel prefers to fold memory
operands immediately comparing to SDAG. The reason is that GlobalISel
doesn't have target hooks in GIM_CheckIsSafeToFold. Or maybe another
check on profitability is required along with safety check that is
currently not present.
---
 .../CodeGen/GlobalISel/GenericMachineInstrs.h |   3 +
 .../X86/GISel/X86InstructionSelector.cpp      | 185 ++++++++++++------
 .../lib/Target/X86/GISel/X86LegalizerInfo.cpp |  47 ++++-
 llvm/lib/Target/X86/GISel/X86LegalizerInfo.h  |   3 +
 llvm/lib/Target/X86/X86InstrBuilder.h         |   1 +
 llvm/lib/Target/X86/X86InstrFragments.td      |  34 +++-
 llvm/lib/Target/X86/X86InstrFragmentsSIMD.td  |  14 +-
 llvm/test/CodeGen/X86/GlobalISel/GV.ll        |  12 +-
 .../test/CodeGen/X86/GlobalISel/add-scalar.ll |   8 +-
 .../CodeGen/X86/GlobalISel/callingconv.ll     |   7 +-
 .../CodeGen/X86/GlobalISel/isel-fcmp-i686.mir |  56 +++---
 .../GlobalISel/legalize-memop-scalar-32.mir   |   2 +-
 .../GlobalISel/legalize-memop-scalar-64.mir   |   2 +-
 .../X86/GlobalISel/legalize-mul-scalar.mir    |   2 +-
 .../CodeGen/X86/GlobalISel/legalize-trunc.mir |   4 +-
 .../CodeGen/X86/GlobalISel/legalize-undef.mir |   4 +-
 .../X86/GlobalISel/memop-scalar-x32.ll        |   6 +-
 .../test/CodeGen/X86/GlobalISel/mul-scalar.ll |  12 +-
 .../CodeGen/X86/GlobalISel/select-GV-32.mir   |   8 +-
 .../CodeGen/X86/GlobalISel/select-GV-64.mir   |   8 +-
 .../X86/GlobalISel/select-memop-v128.mir      |  18 +-
 .../X86/GlobalISel/select-memop-v256.mir      |  21 +-
 llvm/test/CodeGen/X86/GlobalISel/sqrt.mir     |   4 +-
 .../test/CodeGen/X86/GlobalISel/sub-scalar.ll |   8 +-
 llvm/test/CodeGen/X86/isel-and.ll             |  80 ++++----
 llvm/test/CodeGen/X86/isel-buildvector-sse.ll |  48 ++---
 .../test/CodeGen/X86/isel-buildvector-sse2.ll |  21 +-
 llvm/test/CodeGen/X86/isel-icmp.ll            |   8 +-
 llvm/test/CodeGen/X86/isel-or.ll              |  80 ++++----
 llvm/test/CodeGen/X86/isel-phi.ll             |   6 +-
 llvm/test/CodeGen/X86/isel-sdiv.ll            |   8 +-
 llvm/test/CodeGen/X86/isel-select-cmov.ll     |  40 ++--
 llvm/test/CodeGen/X86/isel-srem.ll            |   8 +-
 llvm/test/CodeGen/X86/isel-traps.ll           |   3 +-
 llvm/test/CodeGen/X86/isel-udiv.ll            |   8 +-
 llvm/test/CodeGen/X86/isel-urem.ll            |   8 +-
 llvm/test/CodeGen/X86/isel-x87.ll             |   6 +-
 llvm/test/CodeGen/X86/isel-xor.ll             |  80 ++++----
 38 files changed, 509 insertions(+), 364 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 9e5d4d34f24d2..571ec6dd7e9ba 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -66,6 +66,9 @@ class GMemOperation : public GenericMachineInstr {
   /// memory operations can't be reordered.
   bool isUnordered() const { return getMMO().isUnordered(); }
 
+  /// Return the minimum known alignment in bytes of the actual memory
+  /// reference.
+  Align getAlign() const { return getMMO().getAlign(); }
   /// Returns the size in bytes of the memory access.
   LocationSize getMemSize() const { return getMMO().getSize(); }
   /// Returns the size in bits of the memory access.
diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
index 64a1fa1780a77..ea08f71b8af4a 100644
--- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
+++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
@@ -73,6 +73,9 @@ class X86InstructionSelector : public InstructionSelector {
   // TODO: remove after supported by Tablegen-erated instruction selection.
   unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
                           Align Alignment) const;
+  // TODO: remove once p0<->i32/i64 matching is available
+  unsigned getPtrLoadStoreOp(const LLT &Ty, const RegisterBank &RB,
+                             unsigned Opc) const;
 
   bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
                          MachineFunction &MF) const;
@@ -119,6 +122,8 @@ class X86InstructionSelector : public InstructionSelector {
   bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
                     MachineFunction &MF) const;
 
+  ComplexRendererFns selectAddr(MachineOperand &Root) const;
+
   // emit insert subreg instruction and insert it before MachineInstr &I
   bool emitInsertSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
                         MachineRegisterInfo &MRI, MachineFunction &MF) const;
@@ -445,6 +450,25 @@ bool X86InstructionSelector::select(MachineInstr &I) {
   return false;
 }
 
+unsigned X86InstructionSelector::getPtrLoadStoreOp(const LLT &Ty,
+                                                   const RegisterBank &RB,
+                                                   unsigned Opc) const {
+  assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
+         "Only G_STORE and G_LOAD are expected for selection");
+  if (Ty.isPointer() && X86::GPRRegBankID == RB.getID()) {
+    bool IsLoad = (Opc == TargetOpcode::G_LOAD);
+    switch (Ty.getSizeInBits()) {
+    default:
+      break;
+    case 32:
+      return IsLoad ? X86::MOV32rm : X86::MOV32mr;
+    case 64:
+      return IsLoad ? X86::MOV64rm : X86::MOV64mr;
+    }
+  }
+  return Opc;
+}
+
 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
                                                 const RegisterBank &RB,
                                                 unsigned Opc,
@@ -460,7 +484,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
   } else if (Ty == LLT::scalar(16)) {
     if (X86::GPRRegBankID == RB.getID())
       return Isload ? X86::MOV16rm : X86::MOV16mr;
-  } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
+  } else if (Ty == LLT::scalar(32)) {
     if (X86::GPRRegBankID == RB.getID())
       return Isload ? X86::MOV32rm : X86::MOV32mr;
     if (X86::VECRRegBankID == RB.getID())
@@ -472,7 +496,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
                                    X86::MOVSSmr);
     if (X86::PSRRegBankID == RB.getID())
       return Isload ? X86::LD_Fp32m : X86::ST_Fp32m;
-  } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
+  } else if (Ty == LLT::scalar(64)) {
     if (X86::GPRRegBankID == RB.getID())
       return Isload ? X86::MOV64rm : X86::MOV64mr;
     if (X86::VECRRegBankID == RB.getID())
@@ -530,30 +554,76 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
 }
 
 // Fill in an address from the given instruction.
-static void X86SelectAddress(const MachineInstr &I,
+static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM,
                              const MachineRegisterInfo &MRI,
-                             X86AddressMode &AM) {
-  assert(I.getOperand(0).isReg() && "unsupported opperand.");
+                             const X86Subtarget &STI, X86AddressMode &AM) {
+  assert(I.getOperand(0).isReg() && "unsupported operand.");
   assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
          "unsupported type.");
 
-  if (I.getOpcode() == TargetOpcode::G_PTR_ADD) {
+  switch (I.getOpcode()) {
+  default:
+    break;
+  case TargetOpcode::G_FRAME_INDEX:
+    AM.Base.FrameIndex = I.getOperand(1).getIndex();
+    AM.BaseType = X86AddressMode::FrameIndexBase;
+    return true;
+  case TargetOpcode::G_PTR_ADD: {
     if (auto COff = getIConstantVRegSExtVal(I.getOperand(2).getReg(), MRI)) {
       int64_t Imm = *COff;
       if (isInt<32>(Imm)) { // Check for displacement overflow.
         AM.Disp = static_cast<int32_t>(Imm);
         AM.Base.Reg = I.getOperand(1).getReg();
-        return;
+        return true;
       }
     }
-  } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
-    AM.Base.FrameIndex = I.getOperand(1).getIndex();
-    AM.BaseType = X86AddressMode::FrameIndexBase;
-    return;
+    break;
   }
+  case TargetOpcode::G_GLOBAL_VALUE: {
+    auto GV = I.getOperand(1).getGlobal();
+    if (GV->isThreadLocal()) {
+      return false; // TODO: we don't support TLS yet.
+    }
+    // Can't handle alternate code models yet.
+    if (TM.getCodeModel() != CodeModel::Small)
+      return false;
+    AM.GV = GV;
+    AM.GVOpFlags = STI.classifyGlobalReference(GV);
+
+    // TODO: The ABI requires an extra load. not supported yet.
+    if (isGlobalStubReference(AM.GVOpFlags))
+      return false;
+
+    // TODO: This reference is relative to the pic base. not supported yet.
+    if (isGlobalRelativeToPICBase(AM.GVOpFlags))
+      return false;
+
+    if (STI.isPICStyleRIPRel()) {
+      // Use rip-relative addressing.
+      assert(AM.Base.Reg == 0 && AM.IndexReg == 0 &&
+             "RIP-relative addresses can't have additional register operands");
+      AM.Base.Reg = X86::RIP;
+    }
+    return true;
+  }
+  case TargetOpcode::G_CONSTANT_POOL: {
+    // TODO: Need a separate move for Large model
+    if (TM.getCodeModel() == CodeModel::Large)
+      return false;
 
+    AM.GVOpFlags = STI.classifyLocalReference(nullptr);
+    if (AM.GVOpFlags == X86II::MO_GOTOFF)
+      AM.Base.Reg = STI.getInstrInfo()->getGlobalBaseReg(I.getMF());
+    else if (STI.is64Bit())
+      AM.Base.Reg = X86::RIP;
+    AM.CP = true;
+    AM.Disp = I.getOperand(1).getIndex();
+    return true;
+  }
+  }
   // Default behavior.
   AM.Base.Reg = I.getOperand(0).getReg();
+  return true;
 }
 
 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
@@ -586,36 +656,18 @@ bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
     }
   }
 
-  unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign());
+  unsigned NewOpc = getPtrLoadStoreOp(Ty, RB, Opc);
   if (NewOpc == Opc)
     return false;
 
   I.setDesc(TII.get(NewOpc));
   MachineInstrBuilder MIB(MF, I);
-  const MachineInstr *Ptr = MRI.getVRegDef(I.getOperand(1).getReg());
-
-  if (Ptr->getOpcode() == TargetOpcode::G_CONSTANT_POOL) {
-    assert(Opc == TargetOpcode::G_LOAD &&
-           "Only G_LOAD from constant pool is expected");
-    // TODO: Need a separate move for Large model
-    if (TM.getCodeModel() == CodeModel::Large)
-      return false;
-
-    unsigned char OpFlag = STI.classifyLocalReference(nullptr);
-    Register PICBase;
-    if (OpFlag == X86II::MO_GOTOFF)
-      PICBase = TII.getGlobalBaseReg(&MF);
-    else if (STI.is64Bit())
-      PICBase = X86::RIP;
-
-    I.removeOperand(1);
-    addConstantPoolReference(MIB, Ptr->getOperand(1).getIndex(), PICBase,
-                             OpFlag);
-    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
-  }
+  MachineInstr *Ptr = MRI.getVRegDef(I.getOperand(1).getReg());
 
   X86AddressMode AM;
-  X86SelectAddress(*Ptr, MRI, AM);
+  if (!X86SelectAddress(*Ptr, TM, MRI, STI, AM))
+    return false;
+
   if (Opc == TargetOpcode::G_LOAD) {
     I.removeOperand(1);
     addFullAddress(MIB, AM);
@@ -673,33 +725,10 @@ bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
   assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
          "unexpected instruction");
 
-  auto GV = I.getOperand(1).getGlobal();
-  if (GV->isThreadLocal()) {
-    return false; // TODO: we don't support TLS yet.
-  }
-
-  // Can't handle alternate code models yet.
-  if (TM.getCodeModel() != CodeModel::Small)
-    return false;
-
   X86AddressMode AM;
-  AM.GV = GV;
-  AM.GVOpFlags = STI.classifyGlobalReference(GV);
-
-  // TODO: The ABI requires an extra load. not supported yet.
-  if (isGlobalStubReference(AM.GVOpFlags))
+  if (!X86SelectAddress(I, TM, MRI, STI, AM))
     return false;
 
-  // TODO: This reference is relative to the pic base. not supported yet.
-  if (isGlobalRelativeToPICBase(AM.GVOpFlags))
-    return false;
-
-  if (STI.isPICStyleRIPRel()) {
-    // Use rip-relative addressing.
-    assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
-    AM.Base.Reg = X86::RIP;
-  }
-
   const Register DefReg = I.getOperand(0).getReg();
   LLT Ty = MRI.getType(DefReg);
   unsigned NewOpc = getLeaOP(Ty, STI);
@@ -1880,6 +1909,46 @@ bool X86InstructionSelector::selectSelect(MachineInstr &I,
   return true;
 }
 
+InstructionSelector::ComplexRendererFns
+X86InstructionSelector::selectAddr(MachineOperand &Root) const {
+  MachineInstr *MI = Root.getParent();
+  MachineIRBuilder MIRBuilder(*MI);
+
+  MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
+  MachineInstr *Ptr = MRI.getVRegDef(Root.getReg());
+  X86AddressMode AM;
+  X86SelectAddress(*Ptr, TM, MRI, STI, AM);
+
+  if (AM.IndexReg)
+    return std::nullopt;
+
+  return {// Base
+          {[=](MachineInstrBuilder &MIB) {
+             if (AM.BaseType == X86AddressMode::RegBase)
+               MIB.addUse(AM.Base.Reg);
+             else {
+               assert(AM.BaseType == X86AddressMode::FrameIndexBase &&
+                      "Unknown type of address base");
+               MIB.addFrameIndex(AM.Base.FrameIndex);
+             }
+           },
+           // Scale
+           [=](MachineInstrBuilder &MIB) { MIB.addImm(AM.Scale); },
+           // Index
+           [=](MachineInstrBuilder &MIB) { MIB.addUse(0); },
+           // Disp
+           [=](MachineInstrBuilder &MIB) {
+             if (AM.GV)
+               MIB.addGlobalAddress(AM.GV, AM.Disp, AM.GVOpFlags);
+             else if (AM.CP)
+               MIB.addConstantPoolIndex(AM.Disp, 0, AM.GVOpFlags);
+             else
+               MIB.addImm(AM.Disp);
+           },
+           // Segment
+           [=](MachineInstrBuilder &MIB) { MIB.addUse(0); }}};
+}
+
 InstructionSelector *
 llvm::createX86InstructionSelector(const X86TargetMachine &TM,
                                    const X86Subtarget &Subtarget,
diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
index ba9fa254a477a..f008cb1bea839 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
@@ -373,22 +373,16 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
   // load/store: add more corner cases
   for (unsigned Op : {G_LOAD, G_STORE}) {
     auto &Action = getActionDefinitionsBuilder(Op);
-    Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},
-                                     {s8, p0, s8, 1},
-                                     {s16, p0, s8, 1},
+    Action.legalForTypesWithMemDesc({{s8, p0, s8, 1},
                                      {s16, p0, s16, 1},
-                                     {s32, p0, s8, 1},
-                                     {s32, p0, s16, 1},
                                      {s32, p0, s32, 1},
                                      {s80, p0, s80, 1},
                                      {p0, p0, p0, 1},
                                      {v4s8, p0, v4s8, 1}});
     if (Is64Bit)
-      Action.legalForTypesWithMemDesc({{s64, p0, s8, 1},
-                                       {s64, p0, s16, 1},
-                                       {s64, p0, s32, 1},
-                                       {s64, p0, s64, 1},
-                                       {v2s32, p0, v2s32, 1}});
+      Action.legalForTypesWithMemDesc(
+          {{s64, p0, s64, 1}, {v2s32, p0, v2s32, 1}});
+
     if (HasSSE1)
       Action.legalForTypesWithMemDesc({{v4s32, p0, v4s32, 1}});
     if (HasSSE2)
@@ -407,6 +401,21 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
                                        {v32s16, p0, v32s16, 1},
                                        {v16s32, p0, v16s32, 1},
                                        {v8s64, p0, v8s64, 1}});
+
+    // X86 supports extending loads but not stores for GPRs
+    if (Op == G_LOAD) {
+      Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},
+                                       {s16, p0, s8, 1},
+                                       {s32, p0, s8, 1},
+                                       {s32, p0, s16, 1}});
+      if (Is64Bit)
+        Action.legalForTypesWithMemDesc(
+            {{s64, p0, s8, 1}, {s64, p0, s16, 1}, {s64, p0, s32, 1}});
+    } else {
+      Action.customIf([=](const LegalityQuery &Query) {
+        return Query.Types[0] != Query.MMODescrs[0].MemoryTy;
+      });
+    }
     Action.widenScalarToNextPow2(0, /*Min=*/8)
         .clampScalar(0, s8, sMaxScalar)
         .scalarize(0);
@@ -660,6 +669,8 @@ bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
     return legalizeFPTOUI(MI, MRI, Helper);
   case TargetOpcode::G_UITOFP:
     return legalizeUITOFP(MI, MRI, Helper);
+  case TargetOpcode::G_STORE:
+    return legalizeNarrowingStore(MI, MRI, Helper);
   }
   llvm_unreachable("expected switch to return");
 }
@@ -754,6 +765,22 @@ bool X86LegalizerInfo::legalizeUITOFP(MachineInstr &MI,
   return false;
 }
 
+bool X86LegalizerInfo::legalizeNarrowingStore(MachineInstr &MI,
+                                              MachineRegisterInfo &MRI,
+                                              LegalizerHelper &Helper) const {
+  auto &Store = cast<GStore>(MI);
+  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
+  MachineMemOperand &MMO = **Store.memoperands_begin();
+  MachineFunction &MF = MIRBuilder.getMF();
+  LLT ValTy = MRI.getType(Store.getValueReg());
+  auto *NewMMO = MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), ValTy);
+
+  Helper.Observer.changingInstr(Store);
+  Store.setMemRefs(MF, {NewMMO});
+  Helper.Observer.changedInstr(Store);
+  return true;
+}
+
 bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
                                          MachineInstr &MI) const {
   return true;
diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
index 39bd9892e2f16..54f776456397b 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
@@ -45,6 +45,9 @@ class X86LegalizerInfo : public LegalizerInfo {
 
   bool legalizeUITOFP(MachineInstr &MI, MachineRegisterInfo &MRI,
                       LegalizerHelper &Helper) const;
+
+  bool legalizeNarrowingStore(MachineInstr &MI, MachineRegisterInfo &MRI,
+                              LegalizerHelper &Helper) const;
 };
 } // namespace llvm
 #endif
diff --git a/llvm/lib/Target/X86/X86InstrBuilder.h b/llvm/lib/Target/X86/X86InstrBuilder.h
index 98f1018d3938a..7a47c75a2f841 100644
--- a/llvm/lib/Target/X86/X86InstrBuilder.h
+++ b/llvm/lib/Target/X86/X86InstrBuilder.h
@@ -54,6 +54,7 @@ struct X86AddressMode {
   int Disp = 0;
   const GlobalValue *GV = nullptr;
   unsigned GVOpFlags = 0;
+  bool CP = false;
 
   void getFullAddress(SmallVectorImpl<MachineOperand> &MO) {
     assert(Scale == 1 || Scale == 2 || Scale == 4 || Scale == 8);
diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td
index a7ef72be9316f..f9d70d1bb5d85 100644
--- a/llvm/lib/Target/X86/X86InstrFragments.td
+++ b/llvm/lib/Target/X86/X86InstrFragments.td
@@ -354,6 +354,8 @@ def X86cmpccxadd : SDNode<"X86ISD::CMPCCXADD", SDTX86Cmpccxadd,
 // Define X86-specific addressing mode.
 let WantsParent = true in
 def addr      : ComplexPattern<iPTR, 5, "selectAddr">;
+def gi_addr   : GIComplexOperandMatcher<s32, "selectAddr">,
+                GIComplexPatternEquiv<addr>;
 def lea32addr : ComplexPattern<i32, 5, "selectLEAAddr",
                                [add, sub, mul, X86mul_imm, shl, or, xor, frameindex],
                                []>;
@@ -444,7 +446,11 @@ def i64relocImmSExt32 : PatLeaf<(i64 relocImm), [{
 //
 def imm_su : PatLeaf<(imm), [{
     return !shouldAvoidImmediateInstFormsForSize(N);
-}]>;
+}]> {
+  // TODO : introduce the same check as in SDAG
+  let GISelPredicateCode = [{ return true; }];
+}
+
 def i64immSExt32_su : PatLeaf<(i64immSExt32), [{
     return !shouldAvoidImmediateInstFormsForSize(N);
 }]>;
@@ -500,7 +506,9 @@ def loadi8 : PatFrag<(ops node:$ptr), (i8 (unindexedload node:$ptr)), [{
   ISD::LoadExtType ExtType = LD->getExtensionType();
   return ExtType == ISD::NON_EXTLOAD || ExtType == ISD::EXTLOAD ||
          ExtType == ISD::ZEXTLOAD;
-}]>;
+}]> {
+  let GISelPredicateCode = [{ return isa<GLoad>(MI); }];
+}
 
 // It's always safe to treat a anyext i16 load as a i32 load if the i16 is
 // known to be 32-bit aligned or better. Ditto for i8 to i16.
@@ -512,7 +520,16 @@ def loadi16 : PatFrag<(ops node:$ptr), (i16 (unindexedload node:$ptr)), [{
   if (ExtType == ISD::EXTLOAD && EnablePromoteAnyextLoad)
     return LD->getAlign() >= 2 && LD->isSimple();
   return false;
-}]>;
+}]> {
+  let GISelPredicateCode = [{
+    auto &Load = cast<GLoad>(MI);
+    LLT Ty = MRI.getType(Load.getDstReg());
+    // Non extending load has MMO and destination types of the same size
+    if (Load.getMemSizeInBits() == Ty.getSizeInBits())
+       return true;
+    return Load.getAlign() >= 2 && Load.isSimple();
+  }];
+}
 
 def loadi32 : PatFrag<(ops node:$ptr), (i32 (unindexedload node:$ptr)), [{
   LoadSDNode *LD = cast<LoadSDNode>(N);
@@ -522,7 +539,16 @@ def loadi32 : PatFrag<(ops node:$ptr), (i32 (unindexedload node:$ptr)), [{
   if (ExtType == ISD::EXTLOAD && EnablePromoteAnyextLoad)
     return LD->getAlign() >= 4 && LD->isSimple();
   return false;
-}]>;
+}]> {
+  let GISelPredicateCode = [{
+    auto &Load = cast<GLoad>(MI);
+    LLT Ty = MRI.getType(Load.getDstReg());
+    // Non extending load has MMO and destination types of the same size
+    if (Load.getMemSizeInBits() == Ty.getSizeInBits())
+       return true;
+    return Load.getAlign() >= 4 && Load.isSimple();
+  }];
+}
 
 def loadi64  : PatFrag<(ops node:$ptr), (i64 (load node:$ptr))>;
 def loadf16  : PatFrag<(ops node:$ptr), (f16 (load node:$ptr))>;
diff --git a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
index de70570481fc2..0c20ffed77e77 100644
--- a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -1008,13 +1008,23 @@ def alignedstore : PatFrag<(ops node:$val, node:$ptr),
                            (store node:$val, node:$ptr), [{
   auto *St = cast<StoreSDNode>(N);
   return St->getAlign() >= St->getMemoryVT().getStoreSize();
-}]>;
+}]> {
+  let GISelPredicateCode = [{
+    auto &LdSt = cast<GLoadStore>(MI);
+    return LdSt.getAlign() >= LdSt.getMemSize().getValue();
+  }];
+}
 
 // Like 'load', but always requires vector size alignment.
 def alignedload : PatFrag<(ops node:$ptr), (load node:$ptr), [{
   auto *Ld = cast<LoadSDNode>(N);
   return Ld->getAlign() >= Ld->getMemoryVT().getStoreSize();
-}]>;
+}]> {
+  let GISelPredicateCode = [{
+    auto &LdSt = cast<GLoadStore>(MI);
+    return LdSt.getAlign() >= LdSt.getMemSize().getValue();
+  }];
+}
 
 // 128-bit aligned load pattern fragments
 // NOTE: all 128-bit integer vector loads are promoted to v2i64
diff --git a/llvm/test/CodeGen/X86/GlobalISel/GV.ll b/llvm/test/CodeGen/X86/GlobalISel/GV.ll
index 8c14b54864752..c161a32fd6b95 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/GV.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/GV.ll
@@ -36,26 +36,22 @@ entry:
 define dso_local i32 @test_global_valv() #3 {
 ; X64-LABEL: test_global_valv:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    leaq g_int, %rax
-; X64-NEXT:    movl (%rax), %eax
+; X64-NEXT:    movl g_int, %eax
 ; X64-NEXT:    retq
 ;
 ; X64_DARWIN_PIC-LABEL: test_global_valv:
 ; X64_DARWIN_PIC:       ## %bb.0: ## %entry
-; X64_DARWIN_PIC-NEXT:    leaq _g_int(%rip), %rax
-; X64_DARWIN_PIC-NEXT:    movl (%rax), %eax
+; X64_DARWIN_PIC-NEXT:    movl _g_int(%rip), %eax
 ; X64_DARWIN_PIC-NEXT:    retq
 ;
 ; X32-LABEL: test_global_valv:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    leal g_int, %eax
-; X32-NEXT:    movl (%eax), %eax
+; X32-NEXT:    movl g_int, %eax
 ; X32-NEXT:    retl
 ;
 ; X32ABI-LABEL: test_global_valv:
 ; X32ABI:       # %bb.0: # %entry
-; X32ABI-NEXT:    leal g_int, %eax
-; X32ABI-NEXT:    movl (%eax), %eax
+; X32ABI-NEXT:    movl g_int, %eax
 ; X32ABI-NEXT:    retq
 entry:
   %0 = load i32, ptr @g_int, align 4
diff --git a/llvm/test/CodeGen/X86/GlobalISel/add-scalar.ll b/llvm/test/CodeGen/X86/GlobalISel/add-scalar.ll
index bf7a327257fcc..7bde1b7a7a8be 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/add-scalar.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/add-scalar.ll
@@ -80,8 +80,8 @@ define i16 @test_add_i16(i16 %arg1, i16 %arg2) {
 ;
 ; X86-LABEL: test_add_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    addw %cx, %ax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
@@ -100,8 +100,8 @@ define i8 @test_add_i8(i8 %arg1, i8 %arg2) {
 ;
 ; X86-LABEL: test_add_i8:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    addb %cl, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/GlobalISel/callingconv.ll b/llvm/test/CodeGen/X86/GlobalISel/callingconv.ll
index 33d4de16c9772..ab8880734afe0 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/callingconv.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/callingconv.ll
@@ -32,7 +32,7 @@ define i64 @test_ret_i64() {
 define i8 @test_arg_i8(i8 %a) {
 ; X32-LABEL: test_arg_i8:
 ; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    # kill: def $al killed $al killed $eax
 ; X32-NEXT:    retl
 ;
@@ -47,7 +47,7 @@ define i8 @test_arg_i8(i8 %a) {
 define i16 @test_arg_i16(i16 %a) {
 ; X32-LABEL: test_arg_i16:
 ; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X32-NEXT:    retl
 ;
@@ -230,8 +230,7 @@ define i32 @test_simple_return_callee() {
 ; X32:       # %bb.0:
 ; X32-NEXT:    subl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 16
-; X32-NEXT:    movl $5, %eax
-; X32-NEXT:    movl %eax, (%esp)
+; X32-NEXT:    movl $5, (%esp)
 ; X32-NEXT:    calll simple_return_callee
 ; X32-NEXT:    addl %eax, %eax
 ; X32-NEXT:    addl $12, %esp
diff --git a/llvm/test/CodeGen/X86/GlobalISel/isel-fcmp-i686.mir b/llvm/test/CodeGen/X86/GlobalISel/isel-fcmp-i686.mir
index fb4e25f2a5ced..8273677592607 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/isel-fcmp-i686.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/isel-fcmp-i686.mir
@@ -17,8 +17,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_oeq
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
     ; GISEL-X86-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 11, implicit $eflags
@@ -47,8 +47,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_ogt
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 7, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -75,8 +75,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_oge
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 3, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -103,8 +103,8 @@ fixedStack:
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_olt
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m1]], [[LD_Fp64m]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 7, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -131,8 +131,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_ole
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m1]], [[LD_Fp64m]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 3, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -158,8 +158,8 @@ fixedStack:
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_one
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 5, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -186,8 +186,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_ord
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 11, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -213,8 +213,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_uno
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 10, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -241,8 +241,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_ueq
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -270,8 +270,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_ugt
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m1]], [[LD_Fp64m]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 2, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -298,8 +298,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_uge
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m1]], [[LD_Fp64m]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -327,8 +327,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_ult
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 2, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -355,8 +355,8 @@ stack:           []
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_ule
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
     ; GISEL-X86-NEXT: $al = COPY [[SETCCr]]
@@ -382,8 +382,8 @@ fixedStack:
 body:             |
   bb.1.entry:
     ; GISEL-X86-LABEL: name: fcmp_double_une
-    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
-    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
+    ; GISEL-X86: [[LD_Fp64m:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0)
+    ; GISEL-X86-NEXT: [[LD_Fp64m1:%[0-9]+]]:rfp64 = nofpexcept LD_Fp64m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s64) from %fixed-stack.0 + 8)
     ; GISEL-X86-NEXT: UCOM_FpIr64 [[LD_Fp64m]], [[LD_Fp64m1]], implicit-def $eflags, implicit-def $fpsw, implicit $fpcw
     ; GISEL-X86-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 5, implicit $eflags
     ; GISEL-X86-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 10, implicit $eflags
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-32.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-32.mir
index 2f5af5e0951b4..ba72c4f2f332b 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-32.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-32.mir
@@ -17,7 +17,7 @@ body:             |
     ; X32: [[LOAD4:%[0-9]+]]:_(p0) = G_LOAD [[DEF]](p0) :: (load (p0))
     ; X32: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
     ; X32: [[AND:%[0-9]+]]:_(s8) = G_AND [[LOAD]], [[C]]
-    ; X32: G_STORE [[AND]](s8), [[DEF]](p0) :: (store (s1))
+    ; X32: G_STORE [[AND]](s8), [[DEF]](p0) :: (store (s8))
     ; X32: G_STORE [[LOAD1]](s8), [[DEF]](p0) :: (store (s8))
     ; X32: G_STORE [[LOAD2]](s16), [[DEF]](p0) :: (store (s16))
     ; X32: G_STORE [[LOAD3]](s32), [[DEF]](p0) :: (store (s32))
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-64.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-64.mir
index 1eb082a074514..fe511c65efd88 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-64.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-64.mir
@@ -17,7 +17,7 @@ body:             |
     ; X64: [[LOAD4:%[0-9]+]]:_(p0) = G_LOAD [[DEF]](p0) :: (load (p0))
     ; X64: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
     ; X64: [[AND:%[0-9]+]]:_(s8) = G_AND [[LOAD]], [[C]]
-    ; X64: G_STORE [[AND]](s8), [[DEF]](p0) :: (store (s1))
+    ; X64: G_STORE [[AND]](s8), [[DEF]](p0) :: (store (s8))
     ; X64: G_STORE [[LOAD1]](s8), [[DEF]](p0) :: (store (s8))
     ; X64: G_STORE [[LOAD2]](s16), [[DEF]](p0) :: (store (s16))
     ; X64: G_STORE [[LOAD3]](s32), [[DEF]](p0) :: (store (s32))
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir
index deff2ba439a47..3cd47e7923a51 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir
@@ -55,7 +55,7 @@ body:             |
     ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s8) = G_AND [[MUL]], [[C]]
-    ; CHECK-NEXT: G_STORE [[AND]](s8), [[DEF]](p0) :: (store (s1))
+    ; CHECK-NEXT: G_STORE [[AND]](s8), [[DEF]](p0) :: (store (s8))
     ; CHECK-NEXT: RET 0
     %0(s32) = COPY $edx
     %1(s1) = G_TRUNC %0(s32)
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-trunc.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-trunc.mir
index 9a2414a4eb755..e1390f229871f 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/legalize-trunc.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-trunc.mir
@@ -22,7 +22,7 @@ body:             |
     ; X32-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[DEF]](s32)
     ; X32-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
     ; X32-NEXT: [[AND:%[0-9]+]]:_(s8) = G_AND [[TRUNC]], [[C]]
-    ; X32-NEXT: G_STORE [[AND]](s8), [[DEF1]](p0) :: (store (s1))
+    ; X32-NEXT: G_STORE [[AND]](s8), [[DEF1]](p0) :: (store (s8))
     ; X32-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[DEF]](s32)
     ; X32-NEXT: G_STORE [[TRUNC1]](s8), [[DEF1]](p0) :: (store (s8))
     ; X32-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
@@ -35,7 +35,7 @@ body:             |
     ; X64-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[DEF]](s32)
     ; X64-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
     ; X64-NEXT: [[AND:%[0-9]+]]:_(s8) = G_AND [[TRUNC]], [[C]]
-    ; X64-NEXT: G_STORE [[AND]](s8), [[DEF1]](p0) :: (store (s1))
+    ; X64-NEXT: G_STORE [[AND]](s8), [[DEF1]](p0) :: (store (s8))
     ; X64-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[DEF]](s32)
     ; X64-NEXT: G_STORE [[TRUNC1]](s8), [[DEF1]](p0) :: (store (s8))
     ; X64-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-undef.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-undef.mir
index 37028db243d15..8711d847530a1 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/legalize-undef.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-undef.mir
@@ -12,7 +12,7 @@ body: |
     ; X64-LABEL: name: test_implicit_def
     ; X64: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
     ; X64-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 0
-    ; X64-NEXT: G_STORE [[C]](s8), [[DEF]](p0) :: (store (s1))
+    ; X64-NEXT: G_STORE [[C]](s8), [[DEF]](p0) :: (store (s8))
     ; X64-NEXT: [[DEF1:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
     ; X64-NEXT: G_STORE [[DEF1]](s8), [[DEF]](p0) :: (store (s8))
     ; X64-NEXT: [[DEF2:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
@@ -24,7 +24,7 @@ body: |
     ; X32-LABEL: name: test_implicit_def
     ; X32: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
     ; X32-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 0
-    ; X32-NEXT: G_STORE [[C]](s8), [[DEF]](p0) :: (store (s1))
+    ; X32-NEXT: G_STORE [[C]](s8), [[DEF]](p0) :: (store (s8))
     ; X32-NEXT: [[DEF1:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
     ; X32-NEXT: G_STORE [[DEF1]](s8), [[DEF]](p0) :: (store (s8))
     ; X32-NEXT: [[DEF2:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/X86/GlobalISel/memop-scalar-x32.ll b/llvm/test/CodeGen/X86/GlobalISel/memop-scalar-x32.ll
index f92537ad170ff..86d2a74bbab83 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/memop-scalar-x32.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/memop-scalar-x32.ll
@@ -47,7 +47,7 @@ define i32 @test_load_i32(ptr %p1) {
 define ptr @test_store_i1(i1 %val, ptr %p1) {
 ; CHECK-LABEL: test_store_i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl 4(%esp), %ecx
+; CHECK-NEXT:    movzbl 4(%esp), %ecx
 ; CHECK-NEXT:    movl 8(%esp), %eax
 ; CHECK-NEXT:    andb $1, %cl
 ; CHECK-NEXT:    movb %cl, (%eax)
@@ -59,7 +59,7 @@ define ptr @test_store_i1(i1 %val, ptr %p1) {
 define ptr @test_store_i8(i8 %val, ptr %p1) {
 ; CHECK-LABEL: test_store_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl 4(%esp), %ecx
+; CHECK-NEXT:    movzbl 4(%esp), %ecx
 ; CHECK-NEXT:    movl 8(%esp), %eax
 ; CHECK-NEXT:    movb %cl, (%eax)
 ; CHECK-NEXT:    retl
@@ -70,7 +70,7 @@ define ptr @test_store_i8(i8 %val, ptr %p1) {
 define ptr @test_store_i16(i16 %val, ptr %p1) {
 ; CHECK-LABEL: test_store_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl 4(%esp), %ecx
+; CHECK-NEXT:    movzwl 4(%esp), %ecx
 ; CHECK-NEXT:    movl 8(%esp), %eax
 ; CHECK-NEXT:    movw %cx, (%eax)
 ; CHECK-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/GlobalISel/mul-scalar.ll b/llvm/test/CodeGen/X86/GlobalISel/mul-scalar.ll
index f401f45a06f6a..a98035157aab6 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/mul-scalar.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/mul-scalar.ll
@@ -11,8 +11,8 @@ define i8 @test_mul_i8(i8 %arg1, i8 %arg2) nounwind {
 ;
 ; X86-LABEL: test_mul_i8:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    cbtw
 ; X86-NEXT:    imulb %cl
 ; X86-NEXT:    retl
@@ -30,8 +30,8 @@ define i16 @test_mul_i16(i16 %arg1, i16 %arg2) nounwind {
 ;
 ; X86-LABEL: test_mul_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    imulw %cx, %ax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
@@ -69,11 +69,11 @@ define i64 @test_mul_i64(i64 %arg1, i64 %arg2) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    imull %eax, %esi
-; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    imull %edx, %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    imull %edx, %edi
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    addl %edi, %esi
 ; X86-NEXT:    mull %edx
 ; X86-NEXT:    addl %esi, %edx
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir b/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir
index 43c4105c883e1..58a9277a35cad 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir
@@ -61,17 +61,15 @@ legalized:       true
 regBankSelected: true
 # X32ALL:              registers:
 # X32ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
-# X32ALL-NEXT:           - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
+# X32ALL-NEXT:           - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
-# X32:                     %1:gr32 = LEA32r $noreg, 1, $noreg, @g_int, $noreg
-# X32-NEXT:                %0:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load (s32) from @g_int)
+# X32:                     %0:gr32 = MOV32rm $noreg, 1, $noreg, @g_int, $noreg :: (load (s32) from @g_int)
 # X32-NEXT:                $eax = COPY %0
 # X32-NEXT:                RET 0, implicit $eax
 #
-# X32ABI:                  %1:gr32 = LEA64_32r $noreg, 1, $noreg, @g_int, $noreg
-# X32ABI-NEXT:             %0:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load (s32) from @g_int)
+# X32ABI:                  %0:gr32 = MOV32rm $noreg, 1, $noreg, @g_int, $noreg :: (load (s32) from @g_int)
 # X32ABI-NEXT:             $eax = COPY %0
 # X32ABI-NEXT:             RET 0, implicit $eax
 body:             |
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir b/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir
index d292bbfcaa98b..6938569ce5583 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir
@@ -59,18 +59,16 @@ legalized:       true
 regBankSelected: true
 # X64ALL:              registers:
 # X64ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
-# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
+# X64ALL-NEXT:           - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
 #
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
-# X64:                     %1:gr64 = LEA64r $noreg, 1, $noreg, @g_int, $noreg
-# X64-NEXT:                %0:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load (s32) from @g_int)
+# X64:                     %0:gr32 = MOV32rm $noreg, 1, $noreg, @g_int, $noreg :: (load (s32) from @g_int)
 # X64-NEXT:                $eax = COPY %0
 # X64-NEXT:                RET 0, implicit $eax
 #
-# X64_DARWIN_PIC:          %1:gr64 = LEA64r $rip, 1, $noreg, @g_int, $noreg
-# X64_DARWIN_PIC-NEXT:     %0:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load (s32) from @g_int)
+# X64_DARWIN_PIC:          %0:gr32 = MOV32rm $rip, 1, $noreg, @g_int, $noreg :: (load (s32) from @g_int)
 # X64_DARWIN_PIC-NEXT:     $eax = COPY %0
 # X64_DARWIN_PIC-NEXT:     RET 0, implicit $eax
 #
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-memop-v128.mir b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v128.mir
index f1530940998ed..0aab64ee52c50 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-memop-v128.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v128.mir
@@ -1,7 +1,7 @@
 # RUN: llc -mtriple=x86_64-linux-gnu                                  -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=ALL,NO_AVX512F,SSE
 # RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx                      -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=ALL,NO_AVX512F,AVX
-# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f                  -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=ALL,AVX512ALL,AVX512F
-# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f -mattr=+avx512vl -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=ALL,AVX512ALL,AVX512VL
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f                  -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=ALL,AVX512F
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f -mattr=+avx512vl -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=ALL,AVX512VL
 
 --- |
   define <4 x i32> @test_load_v4i32_noalign(ptr %p1) {
@@ -37,7 +37,7 @@ registers:
 # ALL:      %0:gr64 = COPY $rdi
 # SSE:      %1:vr128 = MOVUPSrm %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1, align 1)
 # AVX:      %1:vr128 = VMOVUPSrm %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1, align 1)
-# AVX512F:  %1:vr128x = VMOVUPSZ128rm_NOVLX %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1, align 1)
+# AVX512F:  %1:vr128 = VMOVUPSrm %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1, align 1)
 # AVX512VL: %1:vr128x = VMOVUPSZ128rm %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1, align 1)
 # ALL: $xmm0 = COPY %1
 body:             |
@@ -62,7 +62,7 @@ registers:
 # ALL:      %0:gr64 = COPY $rdi
 # SSE:      %1:vr128 = MOVAPSrm %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1)
 # AVX:      %1:vr128 = VMOVAPSrm %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1)
-# AVX512F:  %1:vr128x = VMOVAPSZ128rm_NOVLX %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1)
+# AVX512F:  %1:vr128 = VMOVAPSrm %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1)
 # AVX512VL: %1:vr128x = VMOVAPSZ128rm %0, 1, $noreg, 0, $noreg :: (load (<4 x s32>) from %ir.p1)
 # ALL: $xmm0 = COPY %1
 body:             |
@@ -85,11 +85,12 @@ registers:
   - { id: 0, class: vecr }
   - { id: 1, class: gpr }
 # NO_AVX512F: %0:vr128 = COPY $xmm0
-# AVX512ALL:  %0:vr128x = COPY $xmm0
+# AVX512F:   %0:vr128 = COPY $xmm0
+# AVX512VL:  %0:vr128x = COPY $xmm0
 # ALL:       %1:gr64 = COPY $rdi
 # SSE:       MOVAPSmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1)
 # AVX:       VMOVAPSmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1)
-# AVX512F:   VMOVAPSZ128mr_NOVLX %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1)
+# AVX512F:   VMOVAPSmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1)
 # AVX512VL:  VMOVAPSZ128mr %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1)
 # ALL:       $rax = COPY %1
 body:             |
@@ -113,11 +114,12 @@ registers:
   - { id: 0, class: vecr }
   - { id: 1, class: gpr }
 # NO_AVX512F: %0:vr128 = COPY $xmm0
-# AVX512ALL:  %0:vr128x = COPY $xmm0
+# AVX512F:   %0:vr128 = COPY $xmm0
+# AVX512VL:  %0:vr128x = COPY $xmm0
 # ALL:       %1:gr64 = COPY $rdi
 # SSE:       MOVUPSmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1, align 1)
 # AVX:       VMOVUPSmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1, align 1)
-# AVX512F:   VMOVUPSZ128mr_NOVLX %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1, align 1)
+# AVX512F:   VMOVUPSmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1, align 1)
 # AVX512VL:  VMOVUPSZ128mr %1, 1, $noreg, 0, $noreg, %0 :: (store (<4 x s32>) into %ir.p1, align 1)
 # ALL:       $rax = COPY %1
 body:             |
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir
index 86a83c417d63e..eb7328181f256 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir
@@ -38,7 +38,8 @@ regBankSelected: true
 #
 # AVX512ALL:        registers:
 # AVX512ALL-NEXT:     - { id: 0, class: gr64, preferred-register: '', flags: [   ] }
-# AVX512ALL-NEXT:     - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512F-NEXT:       - { id: 1, class: vr256, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:      - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr }
   - { id: 1, class: vecr }
@@ -48,7 +49,7 @@ registers:
 # NO_AVX512F-NEXT:      RET 0, implicit $ymm0
 #
 # AVX512F:              %0:gr64 = COPY $rdi
-# AVX512F-NEXT:         %1:vr256x = VMOVUPSZ256rm_NOVLX %0, 1, $noreg, 0, $noreg :: (load (<8 x s32>) from %ir.p1, align 1)
+# AVX512F-NEXT:         %1:vr256 = VMOVUPSYrm %0, 1, $noreg, 0, $noreg :: (load (<8 x s32>) from %ir.p1, align 1)
 # AVX512F-NEXT:         $ymm0 = COPY %1
 # AVX512F-NEXT:         RET 0, implicit $ymm0
 #
@@ -81,7 +82,7 @@ registers:
 # NO_AVX512F-NEXT:      RET 0, implicit $ymm0
 #
 # AVX512F:              %0:gr64 = COPY $rdi
-# AVX512F-NEXT:         %1:vr256x = VMOVAPSZ256rm_NOVLX %0, 1, $noreg, 0, $noreg :: (load (<8 x s32>) from %ir.p1)
+# AVX512F-NEXT:         %1:vr256 = VMOVAPSYrm %0, 1, $noreg, 0, $noreg :: (load (<8 x s32>) from %ir.p1)
 # AVX512F-NEXT:         $ymm0 = COPY %1
 # AVX512F-NEXT:         RET 0, implicit $ymm0
 #
@@ -110,7 +111,8 @@ regBankSelected: true
 # NO_AVX512F-NEXT:    - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 #
 # AVX512ALL:        registers:
-# AVX512ALL-NEXT:     - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512F-NEXT:       - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:      - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
 # AVX512ALL-NEXT:     - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
@@ -120,9 +122,9 @@ registers:
 # NO_AVX512F-NEXT:      VMOVUPSYmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<8 x s32>) into %ir.p1, align 1)
 # NO_AVX512F-NEXT:      RET 0
 #
-# AVX512F:              %0:vr256x = COPY $ymm0
+# AVX512F:              %0:vr256 = COPY $ymm0
 # AVX512F-NEXT:         %1:gr64 = COPY $rdi
-# AVX512F-NEXT:         VMOVUPSZ256mr_NOVLX %1, 1, $noreg, 0, $noreg, %0 :: (store (<8 x s32>) into %ir.p1, align 1)
+# AVX512F-NEXT:         VMOVUPSYmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<8 x s32>) into %ir.p1, align 1)
 # AVX512F-NEXT:         RET 0
 #
 # AVX512VL:             %0:vr256x = COPY $ymm0
@@ -150,7 +152,8 @@ regBankSelected: true
 # NO_AVX512F-NEXT:    - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 #
 # AVX512ALL:        registers:
-# AVX512ALL-NEXT:     - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512F-NEXT:       - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:      - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
 # AVX512ALL-NEXT:     - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
@@ -160,9 +163,9 @@ registers:
 # NO_AVX512F-NEXT:      VMOVAPSYmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<8 x s32>) into %ir.p1)
 # NO_AVX512F-NEXT:      RET 0
 #
-# AVX512F:              %0:vr256x = COPY $ymm0
+# AVX512F:              %0:vr256 = COPY $ymm0
 # AVX512F-NEXT:         %1:gr64 = COPY $rdi
-# AVX512F-NEXT:         VMOVAPSZ256mr_NOVLX %1, 1, $noreg, 0, $noreg, %0 :: (store (<8 x s32>) into %ir.p1)
+# AVX512F-NEXT:         VMOVAPSYmr %1, 1, $noreg, 0, $noreg, %0 :: (store (<8 x s32>) into %ir.p1)
 # AVX512F-NEXT:         RET 0
 #
 # AVX512VL:             %0:vr256x = COPY $ymm0
diff --git a/llvm/test/CodeGen/X86/GlobalISel/sqrt.mir b/llvm/test/CodeGen/X86/GlobalISel/sqrt.mir
index 6c6786e449a1e..5af5120af51be 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/sqrt.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/sqrt.mir
@@ -12,7 +12,7 @@ fixedStack:
 body:             |
   bb.1:
     ; GISEL-I686-LABEL: name: test_sqrt_f32
-    ; GISEL-I686: [[LD_Fp32m:%[0-9]+]]:rfp32 = LD_Fp32m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s32) from %fixed-stack.0, align 16)
+    ; GISEL-I686: [[LD_Fp32m:%[0-9]+]]:rfp32 = nofpexcept LD_Fp32m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s32) from %fixed-stack.0, align 16)
     ; GISEL-I686-NEXT: [[SQRT_Fp32_:%[0-9]+]]:rfp32 = nofpexcept SQRT_Fp32 [[LD_Fp32m]], implicit-def dead $fpsw, implicit $fpcw
     ; GISEL-I686-NEXT: $fp0 = COPY [[SQRT_Fp32_]]
     ; GISEL-I686-NEXT: RET 0, implicit $fp0
@@ -55,7 +55,7 @@ fixedStack:
 body:             |
   bb.1:
     ; GISEL-I686-LABEL: name: test_sqrt_f80
-    ; GISEL-I686: [[LD_Fp80m:%[0-9]+]]:rfp80 = LD_Fp80m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def $fpsw, implicit $fpcw :: (invariant load (s80) from %fixed-stack.0, align 16)
+    ; GISEL-I686: [[LD_Fp80m:%[0-9]+]]:rfp80 = nofpexcept LD_Fp80m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s80) from %fixed-stack.0, align 16)
     ; GISEL-I686-NEXT: [[SQRT_Fp80_:%[0-9]+]]:rfp80 = nofpexcept SQRT_Fp80 [[LD_Fp80m]], implicit-def dead $fpsw, implicit $fpcw
     ; GISEL-I686-NEXT: $fp0 = COPY [[SQRT_Fp80_]]
     ; GISEL-I686-NEXT: RET 0, implicit $fp0
diff --git a/llvm/test/CodeGen/X86/GlobalISel/sub-scalar.ll b/llvm/test/CodeGen/X86/GlobalISel/sub-scalar.ll
index 94ef4d201d4a1..7a035f5e4ad4d 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/sub-scalar.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/sub-scalar.ll
@@ -79,8 +79,8 @@ define i16 @test_sub_i16(i16 %arg1, i16 %arg2) {
 ;
 ; X86-LABEL: test_sub_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    subw %cx, %ax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
@@ -98,8 +98,8 @@ define i8 @test_sub_i8(i8 %arg1, i8 %arg2) {
 ;
 ; X86-LABEL: test_sub_i8:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    subb %cl, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/isel-and.ll b/llvm/test/CodeGen/X86/isel-and.ll
index 8db8060a0b9c7..3fda0e1d86391 100644
--- a/llvm/test/CodeGen/X86/isel-and.ll
+++ b/llvm/test/CodeGen/X86/isel-and.ll
@@ -21,8 +21,8 @@ define i1 @and_i1(i1 %a, i1 %b) {
 ;
 ; GISEL-X86-LABEL: and_i1:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    andb %cl, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -65,8 +65,8 @@ define i8 @and_i8(i8 %a, i8 %b) {
 ;
 ; GISEL-X86-LABEL: and_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    andb %cl, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -110,8 +110,8 @@ define i16 @and_i16(i16 %a, i16 %b) {
 ;
 ; GISEL-X86-LABEL: and_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    andw %cx, %ax
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -229,7 +229,7 @@ define i8 @and_imm8_i8(i8 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm8_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    andb $1, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -260,7 +260,7 @@ define i16 @and_imm8_i16(i16 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm8_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    andw $6, %ax
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -290,11 +290,23 @@ define i16 @and_imm8_i16(i16 %a) {
 }
 
 define i32 @and_imm8_i32(i32 %a) {
-; X86-LABEL: and_imm8_i32:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    andl $-5, %eax
-; X86-NEXT:    retl
+; SDAG-X86-LABEL: and_imm8_i32:
+; SDAG-X86:       # %bb.0:
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    andl $-5, %eax
+; SDAG-X86-NEXT:    retl
+;
+; FASTISEL-X86-LABEL: and_imm8_i32:
+; FASTISEL-X86:       # %bb.0:
+; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; FASTISEL-X86-NEXT:    andl $-5, %eax
+; FASTISEL-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: and_imm8_i32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    movl $-5, %eax
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: and_imm8_i32:
 ; X64:       # %bb.0:
@@ -322,10 +334,10 @@ define i64 @and_imm8_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm8_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    andl $1, %eax
-; GISEL-X86-NEXT:    andl $0, %edx
+; GISEL-X86-NEXT:    movl $1, %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; SDAG-X64-LABEL: and_imm8_i64:
@@ -365,7 +377,7 @@ define i16 @and_imm16_i16(i16 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm16_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    andw $1023, %ax # imm = 0x3FF
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -409,8 +421,8 @@ define i32 @and_imm16_i32(i32 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm16_i32:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    andl $2044, %eax # imm = 0x7FC
+; GISEL-X86-NEXT:    movl $2044, %eax # imm = 0x7FC
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: and_imm16_i32:
@@ -439,10 +451,10 @@ define i64 @and_imm16_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm16_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    andl $-5022, %eax # imm = 0xEC62
-; GISEL-X86-NEXT:    andl $-1, %edx
+; GISEL-X86-NEXT:    movl $-5022, %eax # imm = 0xEC62
+; GISEL-X86-NEXT:    movl $-1, %edx
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: and_imm16_i64:
@@ -469,8 +481,8 @@ define i32 @and_imm32_i32(i32 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm32_i32:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    andl $85538, %eax # imm = 0x14E22
+; GISEL-X86-NEXT:    movl $85538, %eax # imm = 0x14E22
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: and_imm32_i32:
@@ -499,10 +511,10 @@ define i64 @and_imm32_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm32_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    andl $-125778, %eax # imm = 0xFFFE14AE
-; GISEL-X86-NEXT:    andl $-1, %edx
+; GISEL-X86-NEXT:    movl $-125778, %eax # imm = 0xFFFE14AE
+; GISEL-X86-NEXT:    movl $-1, %edx
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: and_imm32_i64:
@@ -533,10 +545,10 @@ define i64 @and_imm64_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: and_imm64_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    andl $-1850691612, %eax # imm = 0x91B0AFE4
-; GISEL-X86-NEXT:    andl $-2, %edx
+; GISEL-X86-NEXT:    movl $-1850691612, %eax # imm = 0x91B0AFE4
+; GISEL-X86-NEXT:    movl $-2, %edx
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: and_imm64_i64:
diff --git a/llvm/test/CodeGen/X86/isel-buildvector-sse.ll b/llvm/test/CodeGen/X86/isel-buildvector-sse.ll
index 7f580aad78764..7f53e36188ec3 100644
--- a/llvm/test/CodeGen/X86/isel-buildvector-sse.ll
+++ b/llvm/test/CodeGen/X86/isel-buildvector-sse.ll
@@ -23,22 +23,14 @@ define <8 x i32> @test_vector_v8i32() {
 ; SSE-X64-GISEL-LABEL: test_vector_v8i32:
 ; SSE-X64-GISEL:       # %bb.0:
 ; SSE-X64-GISEL-NEXT:    movq %rdi, %rax
-; SSE-X64-GISEL-NEXT:    movl $128100944, %ecx # imm = 0x7A2AA50
-; SSE-X64-GISEL-NEXT:    movl $-632258670, %edx # imm = 0xDA507F92
-; SSE-X64-GISEL-NEXT:    movl $-408980432, %esi # imm = 0xE79F7430
-; SSE-X64-GISEL-NEXT:    movl $708630551, %edi # imm = 0x2A3CD817
-; SSE-X64-GISEL-NEXT:    movl $-871899055, %r8d # imm = 0xCC07E051
-; SSE-X64-GISEL-NEXT:    movl $-633489957, %r9d # imm = 0xDA3DB5DB
-; SSE-X64-GISEL-NEXT:    movl $591019567, %r10d # imm = 0x233A3E2F
-; SSE-X64-GISEL-NEXT:    movl $708632899, %r11d # imm = 0x2A3CE143
-; SSE-X64-GISEL-NEXT:    movl %ecx, (%rax)
-; SSE-X64-GISEL-NEXT:    movl %edx, 4(%rax)
-; SSE-X64-GISEL-NEXT:    movl %esi, 8(%rax)
-; SSE-X64-GISEL-NEXT:    movl %edi, 12(%rax)
-; SSE-X64-GISEL-NEXT:    movl %r8d, 16(%rax)
-; SSE-X64-GISEL-NEXT:    movl %r9d, 20(%rax)
-; SSE-X64-GISEL-NEXT:    movl %r10d, 24(%rax)
-; SSE-X64-GISEL-NEXT:    movl %r11d, 28(%rax)
+; SSE-X64-GISEL-NEXT:    movl $128100944, (%rdi) # imm = 0x7A2AA50
+; SSE-X64-GISEL-NEXT:    movl $-632258670, 4(%rdi) # imm = 0xDA507F92
+; SSE-X64-GISEL-NEXT:    movl $-408980432, 8(%rdi) # imm = 0xE79F7430
+; SSE-X64-GISEL-NEXT:    movl $708630551, 12(%rdi) # imm = 0x2A3CD817
+; SSE-X64-GISEL-NEXT:    movl $-871899055, 16(%rdi) # imm = 0xCC07E051
+; SSE-X64-GISEL-NEXT:    movl $-633489957, 20(%rdi) # imm = 0xDA3DB5DB
+; SSE-X64-GISEL-NEXT:    movl $591019567, 24(%rdi) # imm = 0x233A3E2F
+; SSE-X64-GISEL-NEXT:    movl $708632899, 28(%rdi) # imm = 0x2A3CE143
 ; SSE-X64-GISEL-NEXT:    retq
 ;
 ; SSE-X86-LABEL: test_vector_v8i32:
@@ -57,22 +49,14 @@ define <8 x i32> @test_vector_v8i32() {
 ; SSE-X86-GISEL-LABEL: test_vector_v8i32:
 ; SSE-X86-GISEL:       # %bb.0:
 ; SSE-X86-GISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE-X86-GISEL-NEXT:    movl $128100944, %ecx # imm = 0x7A2AA50
-; SSE-X86-GISEL-NEXT:    movl %ecx, (%eax)
-; SSE-X86-GISEL-NEXT:    movl $-632258670, %ecx # imm = 0xDA507F92
-; SSE-X86-GISEL-NEXT:    movl %ecx, 4(%eax)
-; SSE-X86-GISEL-NEXT:    movl $-408980432, %ecx # imm = 0xE79F7430
-; SSE-X86-GISEL-NEXT:    movl %ecx, 8(%eax)
-; SSE-X86-GISEL-NEXT:    movl $708630551, %ecx # imm = 0x2A3CD817
-; SSE-X86-GISEL-NEXT:    movl %ecx, 12(%eax)
-; SSE-X86-GISEL-NEXT:    movl $-871899055, %ecx # imm = 0xCC07E051
-; SSE-X86-GISEL-NEXT:    movl %ecx, 16(%eax)
-; SSE-X86-GISEL-NEXT:    movl $-633489957, %ecx # imm = 0xDA3DB5DB
-; SSE-X86-GISEL-NEXT:    movl %ecx, 20(%eax)
-; SSE-X86-GISEL-NEXT:    movl $591019567, %ecx # imm = 0x233A3E2F
-; SSE-X86-GISEL-NEXT:    movl %ecx, 24(%eax)
-; SSE-X86-GISEL-NEXT:    movl $708632899, %ecx # imm = 0x2A3CE143
-; SSE-X86-GISEL-NEXT:    movl %ecx, 28(%eax)
+; SSE-X86-GISEL-NEXT:    movl $128100944, (%eax) # imm = 0x7A2AA50
+; SSE-X86-GISEL-NEXT:    movl $-632258670, 4(%eax) # imm = 0xDA507F92
+; SSE-X86-GISEL-NEXT:    movl $-408980432, 8(%eax) # imm = 0xE79F7430
+; SSE-X86-GISEL-NEXT:    movl $708630551, 12(%eax) # imm = 0x2A3CD817
+; SSE-X86-GISEL-NEXT:    movl $-871899055, 16(%eax) # imm = 0xCC07E051
+; SSE-X86-GISEL-NEXT:    movl $-633489957, 20(%eax) # imm = 0xDA3DB5DB
+; SSE-X86-GISEL-NEXT:    movl $591019567, 24(%eax) # imm = 0x233A3E2F
+; SSE-X86-GISEL-NEXT:    movl $708632899, 28(%eax) # imm = 0x2A3CE143
 ; SSE-X86-GISEL-NEXT:    retl
   ret <8 x i32> <i32 128100944, i32 3662708626, i32 3885986864, i32 708630551, i32 -871899055, i32 3661477339, i32 4885986863, i32 708632899>
 }
diff --git a/llvm/test/CodeGen/X86/isel-buildvector-sse2.ll b/llvm/test/CodeGen/X86/isel-buildvector-sse2.ll
index da089dda6d03d..aba37e039eae6 100644
--- a/llvm/test/CodeGen/X86/isel-buildvector-sse2.ll
+++ b/llvm/test/CodeGen/X86/isel-buildvector-sse2.ll
@@ -20,20 +20,13 @@ define <7 x i8> @test_vector_v7i8() {
 ; SSE2-GISEL-LABEL: test_vector_v7i8:
 ; SSE2-GISEL:       # %bb.0:
 ; SSE2-GISEL-NEXT:    movq %rdi, %rax
-; SSE2-GISEL-NEXT:    movb $4, %cl
-; SSE2-GISEL-NEXT:    movb $8, %dl
-; SSE2-GISEL-NEXT:    movb $15, %sil
-; SSE2-GISEL-NEXT:    movb $16, %dil
-; SSE2-GISEL-NEXT:    movb $23, %r8b
-; SSE2-GISEL-NEXT:    movb $42, %r9b
-; SSE2-GISEL-NEXT:    movb $63, %r10b
-; SSE2-GISEL-NEXT:    movb %cl, (%rax)
-; SSE2-GISEL-NEXT:    movb %dl, 1(%rax)
-; SSE2-GISEL-NEXT:    movb %sil, 2(%rax)
-; SSE2-GISEL-NEXT:    movb %dil, 3(%rax)
-; SSE2-GISEL-NEXT:    movb %r8b, 4(%rax)
-; SSE2-GISEL-NEXT:    movb %r9b, 5(%rax)
-; SSE2-GISEL-NEXT:    movb %r10b, 6(%rax)
+; SSE2-GISEL-NEXT:    movb $4, (%rdi)
+; SSE2-GISEL-NEXT:    movb $8, 1(%rdi)
+; SSE2-GISEL-NEXT:    movb $15, 2(%rdi)
+; SSE2-GISEL-NEXT:    movb $16, 3(%rdi)
+; SSE2-GISEL-NEXT:    movb $23, 4(%rdi)
+; SSE2-GISEL-NEXT:    movb $42, 5(%rdi)
+; SSE2-GISEL-NEXT:    movb $63, 6(%rdi)
 ; SSE2-GISEL-NEXT:    retq
   ret <7 x i8> <i8 4, i8 8, i8 15, i8 16, i8 23, i8 42, i8 63>
 }
diff --git a/llvm/test/CodeGen/X86/isel-icmp.ll b/llvm/test/CodeGen/X86/isel-icmp.ll
index d6f8fba81ce4b..8a4d035086112 100644
--- a/llvm/test/CodeGen/X86/isel-icmp.ll
+++ b/llvm/test/CodeGen/X86/isel-icmp.ll
@@ -50,8 +50,8 @@ define i32 @test_icmp_eq_i8(i8 %a, i8 %b) {
 ;
 ; GISEL-X86-LABEL: test_icmp_eq_i8:
 ; GISEL-X86:       ## %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    xorl %eax, %eax
 ; GISEL-X86-NEXT:    cmpb %dl, %cl
 ; GISEL-X86-NEXT:    sete %al
@@ -105,8 +105,8 @@ define i32 @test_icmp_eq_i16(i16 %a, i16 %b) {
 ;
 ; GISEL-X86-LABEL: test_icmp_eq_i16:
 ; GISEL-X86:       ## %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    xorl %eax, %eax
 ; GISEL-X86-NEXT:    cmpw %dx, %cx
 ; GISEL-X86-NEXT:    sete %al
diff --git a/llvm/test/CodeGen/X86/isel-or.ll b/llvm/test/CodeGen/X86/isel-or.ll
index ad11e4e5bd3b1..449f29a027743 100644
--- a/llvm/test/CodeGen/X86/isel-or.ll
+++ b/llvm/test/CodeGen/X86/isel-or.ll
@@ -21,8 +21,8 @@ define i1 @or_i1(i1 %a, i1 %b) {
 ;
 ; GISEL-X86-LABEL: or_i1:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    orb %cl, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -66,8 +66,8 @@ define i8 @or_i8(i8 %a, i8 %b) {
 ;
 ; GISEL-X86-LABEL: or_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    orb %cl, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -111,8 +111,8 @@ define i16 @or_i16(i16 %a, i16 %b) {
 ;
 ; GISEL-X86-LABEL: or_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    orw %cx, %ax
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -230,7 +230,7 @@ define i8 @or_imm8_i8(i8 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm8_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    orb $1, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -261,7 +261,7 @@ define i16 @or_imm8_i16(i16 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm8_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    orw $6, %ax
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -291,11 +291,23 @@ define i16 @or_imm8_i16(i16 %a) {
 }
 
 define i32 @or_imm8_i32(i32 %a) {
-; X86-LABEL: or_imm8_i32:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    orl $-5, %eax
-; X86-NEXT:    retl
+; SDAG-X86-LABEL: or_imm8_i32:
+; SDAG-X86:       # %bb.0:
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    orl $-5, %eax
+; SDAG-X86-NEXT:    retl
+;
+; FASTISEL-X86-LABEL: or_imm8_i32:
+; FASTISEL-X86:       # %bb.0:
+; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; FASTISEL-X86-NEXT:    orl $-5, %eax
+; FASTISEL-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: or_imm8_i32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    movl $-5, %eax
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: or_imm8_i32:
 ; X64:       # %bb.0:
@@ -323,10 +335,10 @@ define i64 @or_imm8_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm8_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    orl $1, %eax
-; GISEL-X86-NEXT:    orl $0, %edx
+; GISEL-X86-NEXT:    movl $1, %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: or_imm8_i64:
@@ -354,7 +366,7 @@ define i16 @or_imm16_i16(i16 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm16_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    orw $1023, %ax # imm = 0x3FF
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -398,8 +410,8 @@ define i32 @or_imm16_i32(i32 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm16_i32:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    orl $2044, %eax # imm = 0x7FC
+; GISEL-X86-NEXT:    movl $2044, %eax # imm = 0x7FC
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: or_imm16_i32:
@@ -428,10 +440,10 @@ define i64 @or_imm16_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm16_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    orl $-5022, %eax # imm = 0xEC62
-; GISEL-X86-NEXT:    orl $-1, %edx
+; GISEL-X86-NEXT:    movl $-5022, %eax # imm = 0xEC62
+; GISEL-X86-NEXT:    movl $-1, %edx
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: or_imm16_i64:
@@ -458,8 +470,8 @@ define i32 @or_imm32_i32(i32 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm32_i32:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    orl $85538, %eax # imm = 0x14E22
+; GISEL-X86-NEXT:    movl $85538, %eax # imm = 0x14E22
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: or_imm32_i32:
@@ -488,10 +500,10 @@ define i64 @or_imm32_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm32_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    orl $-125778, %eax # imm = 0xFFFE14AE
-; GISEL-X86-NEXT:    orl $-1, %edx
+; GISEL-X86-NEXT:    movl $-125778, %eax # imm = 0xFFFE14AE
+; GISEL-X86-NEXT:    movl $-1, %edx
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: or_imm32_i64:
@@ -522,10 +534,10 @@ define i64 @or_imm64_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: or_imm64_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    orl $-1850691612, %eax # imm = 0x91B0AFE4
-; GISEL-X86-NEXT:    orl $-2, %edx
+; GISEL-X86-NEXT:    movl $-1850691612, %eax # imm = 0x91B0AFE4
+; GISEL-X86-NEXT:    movl $-2, %edx
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    orl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: or_imm64_i64:
diff --git a/llvm/test/CodeGen/X86/isel-phi.ll b/llvm/test/CodeGen/X86/isel-phi.ll
index 412aeac5eed6a..ee2039492abfd 100644
--- a/llvm/test/CodeGen/X86/isel-phi.ll
+++ b/llvm/test/CodeGen/X86/isel-phi.ll
@@ -364,9 +364,9 @@ define ptr @test_ptr(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, ptr %g, i1
 ; GLOBAL-X64-LABEL: test_ptr:
 ; GLOBAL-X64:       # %bb.0: # %entry
 ; GLOBAL-X64-NEXT:    movq %rdi, %rax
-; GLOBAL-X64-NEXT:    movl {{[0-9]+}}(%rsp), %r11d
-; GLOBAL-X64-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
-; GLOBAL-X64-NEXT:    movl {{[0-9]+}}(%rsp), %edi
+; GLOBAL-X64-NEXT:    movzbl {{[0-9]+}}(%rsp), %r11d
+; GLOBAL-X64-NEXT:    movzbl {{[0-9]+}}(%rsp), %r10d
+; GLOBAL-X64-NEXT:    movzbl {{[0-9]+}}(%rsp), %edi
 ; GLOBAL-X64-NEXT:    testb $1, %r11b
 ; GLOBAL-X64-NEXT:    je .LBB7_4
 ; GLOBAL-X64-NEXT:  # %bb.1: # %cond.true
diff --git a/llvm/test/CodeGen/X86/isel-sdiv.ll b/llvm/test/CodeGen/X86/isel-sdiv.ll
index 6a6b2da8dc2f8..1aca8d1035664 100644
--- a/llvm/test/CodeGen/X86/isel-sdiv.ll
+++ b/llvm/test/CodeGen/X86/isel-sdiv.ll
@@ -21,9 +21,9 @@ define i8 @test_sdiv_i8(i8 %arg1, i8 %arg2) nounwind {
 ;
 ; GISEL-X86-LABEL: test_sdiv_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    cbtw
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    idivb %cl
 ; GISEL-X86-NEXT:    retl
   %ret = sdiv i8 %arg1, %arg2
@@ -48,8 +48,8 @@ define i16 @test_sdiv_i16(i16 %arg1, i16 %arg2) nounwind {
 ;
 ; GISEL-X86-LABEL: test_sdiv_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    cwtd
 ; GISEL-X86-NEXT:    idivw %cx
diff --git a/llvm/test/CodeGen/X86/isel-select-cmov.ll b/llvm/test/CodeGen/X86/isel-select-cmov.ll
index fbd9bd073155e..d013ad2c7fbff 100644
--- a/llvm/test/CodeGen/X86/isel-select-cmov.ll
+++ b/llvm/test/CodeGen/X86/isel-select-cmov.ll
@@ -95,23 +95,23 @@ define zeroext i8 @select_cmov_i8(i1 zeroext %cond, i8 zeroext %a, i8 zeroext %b
 ;
 ; GISEL-X86-LABEL: select_cmov_i8:
 ; GISEL-X86:       ## %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    testl %eax, %eax
 ; GISEL-X86-NEXT:    je LBB0_1
 ; GISEL-X86-NEXT:  ## %bb.2:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    ## kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
 ; GISEL-X86-NEXT:  LBB0_1:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    ## kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
 ;
 ; GISEL-X86-CMOV-LABEL: select_cmov_i8:
 ; GISEL-X86-CMOV:       ## %bb.0:
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-CMOV-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-CMOV-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
+; GISEL-X86-CMOV-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-CMOV-NEXT:    testl %ecx, %ecx
 ; GISEL-X86-CMOV-NEXT:    cmovnew %dx, %ax
 ; GISEL-X86-CMOV-NEXT:    ## kill: def $al killed $al killed $eax
@@ -198,23 +198,23 @@ define zeroext i16 @select_cmov_i16(i1 zeroext %cond, i16 zeroext %a, i16 zeroex
 ;
 ; GISEL-X86-LABEL: select_cmov_i16:
 ; GISEL-X86:       ## %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    testl %eax, %eax
 ; GISEL-X86-NEXT:    je LBB1_1
 ; GISEL-X86-NEXT:  ## %bb.2:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
 ; GISEL-X86-NEXT:  LBB1_1:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
 ;
 ; GISEL-X86-CMOV-LABEL: select_cmov_i16:
 ; GISEL-X86-CMOV:       ## %bb.0:
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-CMOV-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-CMOV-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; GISEL-X86-CMOV-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-CMOV-NEXT:    testl %ecx, %ecx
 ; GISEL-X86-CMOV-NEXT:    cmovnew %dx, %ax
 ; GISEL-X86-CMOV-NEXT:    ## kill: def $ax killed $ax killed $eax
@@ -300,8 +300,8 @@ define zeroext i16 @select_cmp_cmov_i16(i16 zeroext %a, i16 zeroext %b) {
 ;
 ; GISEL-X86-LABEL: select_cmp_cmov_i16:
 ; GISEL-X86:       ## %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    xorl %edx, %edx
 ; GISEL-X86-NEXT:    cmpw %ax, %cx
 ; GISEL-X86-NEXT:    setb %dl
@@ -315,8 +315,8 @@ define zeroext i16 @select_cmp_cmov_i16(i16 zeroext %a, i16 zeroext %b) {
 ;
 ; GISEL-X86-CMOV-LABEL: select_cmp_cmov_i16:
 ; GISEL-X86-CMOV:       ## %bb.0:
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-CMOV-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-CMOV-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-CMOV-NEXT:    xorl %edx, %edx
 ; GISEL-X86-CMOV-NEXT:    cmpw %cx, %ax
 ; GISEL-X86-CMOV-NEXT:    setb %dl
@@ -400,7 +400,7 @@ define i32 @select_cmov_i32(i1 zeroext %cond, i32 %a, i32 %b) {
 ;
 ; GISEL-X86-LABEL: select_cmov_i32:
 ; GISEL-X86:       ## %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    testl %eax, %eax
 ; GISEL-X86-NEXT:    je LBB3_1
 ; GISEL-X86-NEXT:  ## %bb.2:
@@ -412,7 +412,7 @@ define i32 @select_cmov_i32(i1 zeroext %cond, i32 %a, i32 %b) {
 ;
 ; GISEL-X86-CMOV-LABEL: select_cmov_i32:
 ; GISEL-X86-CMOV:       ## %bb.0:
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-CMOV-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-CMOV-NEXT:    testl %ecx, %ecx
 ; GISEL-X86-CMOV-NEXT:    cmovnel {{[0-9]+}}(%esp), %eax
@@ -595,7 +595,7 @@ define i64 @select_cmov_i64(i1 zeroext %cond, i64 %a, i64 %b) {
 ;
 ; GISEL-X86-LABEL: select_cmov_i64:
 ; GISEL-X86:       ## %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    testl %ecx, %ecx
 ; GISEL-X86-NEXT:    je LBB5_1
 ; GISEL-X86-NEXT:  ## %bb.2:
@@ -615,7 +615,7 @@ define i64 @select_cmov_i64(i1 zeroext %cond, i64 %a, i64 %b) {
 ;
 ; GISEL-X86-CMOV-LABEL: select_cmov_i64:
 ; GISEL-X86-CMOV:       ## %bb.0:
-; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-CMOV-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-CMOV-NEXT:    testl %ecx, %ecx
diff --git a/llvm/test/CodeGen/X86/isel-srem.ll b/llvm/test/CodeGen/X86/isel-srem.ll
index 56716e10a9d99..1dabf4175c852 100644
--- a/llvm/test/CodeGen/X86/isel-srem.ll
+++ b/llvm/test/CodeGen/X86/isel-srem.ll
@@ -48,9 +48,9 @@ define i8 @test_srem_i8(i8 %arg1, i8 %arg2) nounwind {
 ;
 ; GISEL-X86-LABEL: test_srem_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    cbtw
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    idivb %cl
 ; GISEL-X86-NEXT:    movb %ah, %al
 ; GISEL-X86-NEXT:    retl
@@ -78,8 +78,8 @@ define i16 @test_srem_i16(i16 %arg1, i16 %arg2) nounwind {
 ;
 ; GISEL-X86-LABEL: test_srem_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    cwtd
 ; GISEL-X86-NEXT:    idivw %cx
diff --git a/llvm/test/CodeGen/X86/isel-traps.ll b/llvm/test/CodeGen/X86/isel-traps.ll
index c207387166a6b..de02779046a5e 100644
--- a/llvm/test/CodeGen/X86/isel-traps.ll
+++ b/llvm/test/CodeGen/X86/isel-traps.ll
@@ -63,8 +63,7 @@ define void @test_ubsantrap_custom() nounwind {
 ; GISEL-X86-LABEL: test_ubsantrap_custom:
 ; GISEL-X86:       # %bb.0:
 ; GISEL-X86-NEXT:    subl $12, %esp
-; GISEL-X86-NEXT:    movl $42, %eax
-; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    movl $42, (%esp)
 ; GISEL-X86-NEXT:    calll guide
 ; GISEL-X86-NEXT:    addl $12, %esp
 ; GISEL-X86-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/isel-udiv.ll b/llvm/test/CodeGen/X86/isel-udiv.ll
index b56b8b112fe47..b123b3c7780fa 100644
--- a/llvm/test/CodeGen/X86/isel-udiv.ll
+++ b/llvm/test/CodeGen/X86/isel-udiv.ll
@@ -21,9 +21,9 @@ define i8 @test_udiv_i8(i8 %arg1, i8 %arg2) nounwind {
 ;
 ; GISEL-X86-LABEL: test_udiv_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    movzbl %al, %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    divb %cl
 ; GISEL-X86-NEXT:    retl
   %ret = udiv i8 %arg1, %arg2
@@ -48,8 +48,8 @@ define i16 @test_udiv_i16(i16 %arg1, i16 %arg2) nounwind {
 ;
 ; GISEL-X86-LABEL: test_udiv_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    xorl %edx, %edx
 ; GISEL-X86-NEXT:    divw %cx
diff --git a/llvm/test/CodeGen/X86/isel-urem.ll b/llvm/test/CodeGen/X86/isel-urem.ll
index 50b9c1250ff87..386f08151ad9c 100644
--- a/llvm/test/CodeGen/X86/isel-urem.ll
+++ b/llvm/test/CodeGen/X86/isel-urem.ll
@@ -48,9 +48,9 @@ define i8 @test_urem_i8(i8 %arg1, i8 %arg2) nounwind {
 ;
 ; GISEL-X86-LABEL: test_urem_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    movzbl %al, %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    divb %cl
 ; GISEL-X86-NEXT:    movb %ah, %al
 ; GISEL-X86-NEXT:    retl
@@ -78,8 +78,8 @@ define i16 @test_urem_i16(i16 %arg1, i16 %arg2) nounwind {
 ;
 ; GISEL-X86-LABEL: test_urem_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    xorl %edx, %edx
 ; GISEL-X86-NEXT:    divw %cx
diff --git a/llvm/test/CodeGen/X86/isel-x87.ll b/llvm/test/CodeGen/X86/isel-x87.ll
index 690c1f6ea968c..492faaa19cd66 100644
--- a/llvm/test/CodeGen/X86/isel-x87.ll
+++ b/llvm/test/CodeGen/X86/isel-x87.ll
@@ -186,8 +186,7 @@ define void @f6(ptr %a, ptr %b) nounwind {
 ; GISEL_X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; GISEL_X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; GISEL_X86-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
-; GISEL_X86-NEXT:    flds (%eax)
-; GISEL_X86-NEXT:    faddp %st, %st(1)
+; GISEL_X86-NEXT:    fadds (%eax)
 ; GISEL_X86-NEXT:    fstps (%ecx)
 ; GISEL_X86-NEXT:    retl
 ;
@@ -203,8 +202,7 @@ define void @f6(ptr %a, ptr %b) nounwind {
 ; GISEL_X64-LABEL: f6:
 ; GISEL_X64:       # %bb.0:
 ; GISEL_X64-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}(%rip)
-; GISEL_X64-NEXT:    flds (%rdi)
-; GISEL_X64-NEXT:    faddp %st, %st(1)
+; GISEL_X64-NEXT:    fadds (%rdi)
 ; GISEL_X64-NEXT:    fstps (%rsi)
 ; GISEL_X64-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/isel-xor.ll b/llvm/test/CodeGen/X86/isel-xor.ll
index 979993cb7bdcf..a31ad78524ee1 100644
--- a/llvm/test/CodeGen/X86/isel-xor.ll
+++ b/llvm/test/CodeGen/X86/isel-xor.ll
@@ -21,8 +21,8 @@ define i1 @xor_i1(i1 %a, i1 %b) {
 ;
 ; GISEL-X86-LABEL: xor_i1:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    xorb %cl, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -66,8 +66,8 @@ define i8 @xor_i8(i8 %a, i8 %b) {
 ;
 ; GISEL-X86-LABEL: xor_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    xorb %cl, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -111,8 +111,8 @@ define i16 @xor_i16(i16 %a, i16 %b) {
 ;
 ; GISEL-X86-LABEL: xor_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    xorw %cx, %ax
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -230,7 +230,7 @@ define i8 @xor_imm8_i8(i8 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm8_i8:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    xorb $1, %al
 ; GISEL-X86-NEXT:    # kill: def $al killed $al killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -261,7 +261,7 @@ define i16 @xor_imm8_i16(i16 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm8_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    xorw $6, %ax
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -291,11 +291,23 @@ define i16 @xor_imm8_i16(i16 %a) {
 }
 
 define i32 @xor_imm8_i32(i32 %a) {
-; X86-LABEL: xor_imm8_i32:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    xorl $-5, %eax
-; X86-NEXT:    retl
+; SDAG-X86-LABEL: xor_imm8_i32:
+; SDAG-X86:       # %bb.0:
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    xorl $-5, %eax
+; SDAG-X86-NEXT:    retl
+;
+; FASTISEL-X86-LABEL: xor_imm8_i32:
+; FASTISEL-X86:       # %bb.0:
+; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; FASTISEL-X86-NEXT:    xorl $-5, %eax
+; FASTISEL-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: xor_imm8_i32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    movl $-5, %eax
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_imm8_i32:
 ; X64:       # %bb.0:
@@ -323,10 +335,10 @@ define i64 @xor_imm8_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm8_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    xorl $1, %eax
-; GISEL-X86-NEXT:    xorl $0, %edx
+; GISEL-X86-NEXT:    movl $1, %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_imm8_i64:
@@ -354,7 +366,7 @@ define i16 @xor_imm16_i16(i16 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm16_i16:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    xorw $1023, %ax # imm = 0x3FF
 ; GISEL-X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GISEL-X86-NEXT:    retl
@@ -398,8 +410,8 @@ define i32 @xor_imm16_i32(i32 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm16_i32:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    xorl $2044, %eax # imm = 0x7FC
+; GISEL-X86-NEXT:    movl $2044, %eax # imm = 0x7FC
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_imm16_i32:
@@ -430,10 +442,10 @@ define i64 @xor_imm16_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm16_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    xorl $-5022, %eax # imm = 0xEC62
-; GISEL-X86-NEXT:    notl %edx
+; GISEL-X86-NEXT:    movl $-5022, %eax # imm = 0xEC62
+; GISEL-X86-NEXT:    movl $-1, %edx
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_imm16_i64:
@@ -460,8 +472,8 @@ define i32 @xor_imm32_i32(i32 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm32_i32:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    xorl $85538, %eax # imm = 0x14E22
+; GISEL-X86-NEXT:    movl $85538, %eax # imm = 0x14E22
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_imm32_i32:
@@ -492,10 +504,10 @@ define i64 @xor_imm32_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm32_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    xorl $-125778, %eax # imm = 0xFFFE14AE
-; GISEL-X86-NEXT:    notl %edx
+; GISEL-X86-NEXT:    movl $-125778, %eax # imm = 0xFFFE14AE
+; GISEL-X86-NEXT:    movl $-1, %edx
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_imm32_i64:
@@ -526,10 +538,10 @@ define i64 @xor_imm64_i64(i64 %a) {
 ;
 ; GISEL-X86-LABEL: xor_imm64_i64:
 ; GISEL-X86:       # %bb.0:
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GISEL-X86-NEXT:    xorl $-1850691612, %eax # imm = 0x91B0AFE4
-; GISEL-X86-NEXT:    xorl $-2, %edx
+; GISEL-X86-NEXT:    movl $-1850691612, %eax # imm = 0x91B0AFE4
+; GISEL-X86-NEXT:    movl $-2, %edx
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; GISEL-X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_imm64_i64:

From 3215fd70294369b73b0e944238097a28550e7435 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 16:14:39 -0700
Subject: [PATCH 566/710] [PBQP] Add begin and end to Vector (NFC) (#136454)

This patch adds begin and end to Vector.

The new functions will allow us to use llvm::interleaved in operator<<
for example.
---
 llvm/include/llvm/CodeGen/PBQP/Math.h | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/PBQP/Math.h b/llvm/include/llvm/CodeGen/PBQP/Math.h
index 099ba788e9a2d..6a8942e08b101 100644
--- a/llvm/include/llvm/CodeGen/PBQP/Math.h
+++ b/llvm/include/llvm/CodeGen/PBQP/Math.h
@@ -33,13 +33,13 @@ class Vector {
   /// Construct a PBQP vector with initializer.
   Vector(unsigned Length, PBQPNum InitVal)
     : Length(Length), Data(std::make_unique<PBQPNum []>(Length)) {
-    std::fill(Data.get(), Data.get() + Length, InitVal);
+    std::fill(begin(), end(), InitVal);
   }
 
   /// Copy construct a PBQP vector.
   Vector(const Vector &V)
     : Length(V.Length), Data(std::make_unique<PBQPNum []>(Length)) {
-    std::copy(V.Data.get(), V.Data.get() + Length, Data.get());
+    llvm::copy(V, begin());
   }
 
   /// Move construct a PBQP vector.
@@ -48,12 +48,16 @@ class Vector {
     V.Length = 0;
   }
 
+  // Iterator-based access.
+  const PBQPNum *begin() const { return Data.get(); }
+  const PBQPNum *end() const { return Data.get() + Length; }
+  PBQPNum *begin() { return Data.get(); }
+  PBQPNum *end() { return Data.get() + Length; }
+
   /// Comparison operator.
   bool operator==(const Vector &V) const {
     assert(Length != 0 && Data && "Invalid vector");
-    if (Length != V.Length)
-      return false;
-    return std::equal(Data.get(), Data.get() + Length, V.Data.get());
+    return llvm::equal(*this, V);
   }
 
   /// Return the length of the vector
@@ -80,15 +84,14 @@ class Vector {
   Vector& operator+=(const Vector &V) {
     assert(Length != 0 && Data && "Invalid vector");
     assert(Length == V.Length && "Vector length mismatch.");
-    std::transform(Data.get(), Data.get() + Length, V.Data.get(), Data.get(),
-                   std::plus<PBQPNum>());
+    std::transform(begin(), end(), V.begin(), begin(), std::plus<PBQPNum>());
     return *this;
   }
 
   /// Returns the index of the minimum value in this vector
   unsigned minIndex() const {
     assert(Length != 0 && Data && "Invalid vector");
-    return std::min_element(Data.get(), Data.get() + Length) - Data.get();
+    return llvm::min_element(*this) - begin();
   }
 
 private:
@@ -98,8 +101,8 @@ class Vector {
 
 /// Return a hash_value for the given vector.
 inline hash_code hash_value(const Vector &V) {
-  unsigned *VBegin = reinterpret_cast<unsigned*>(V.Data.get());
-  unsigned *VEnd = reinterpret_cast<unsigned*>(V.Data.get() + V.Length);
+  const unsigned *VBegin = reinterpret_cast<const unsigned *>(V.begin());
+  const unsigned *VEnd = reinterpret_cast<const unsigned *>(V.end());
   return hash_combine(V.Length, hash_combine_range(VBegin, VEnd));
 }
 

From e5aae97f577b64a5d3285ccec8795d417a330dab Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 18:39:01 -0700
Subject: [PATCH 567/710] [PBQP] Use llvm::interleaved (NFC) (#136462)

---
 llvm/include/llvm/CodeGen/PBQP/Math.h | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/PBQP/Math.h b/llvm/include/llvm/CodeGen/PBQP/Math.h
index 6a8942e08b101..2b2b498e54fce 100644
--- a/llvm/include/llvm/CodeGen/PBQP/Math.h
+++ b/llvm/include/llvm/CodeGen/PBQP/Math.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/InterleavedRange.h"
 #include <algorithm>
 #include <cassert>
 #include <functional>
@@ -111,12 +112,7 @@ inline hash_code hash_value(const Vector &V) {
 template <typename OStream>
 OStream& operator<<(OStream &OS, const Vector &V) {
   assert((V.getLength() != 0) && "Zero-length vector badness.");
-
-  OS << "[ " << V[0];
-  for (unsigned i = 1; i < V.getLength(); ++i)
-    OS << ", " << V[i];
-  OS << " ]";
-
+  OS << "[ " << llvm::interleaved(V) << " ]";
   return OS;
 }
 

From 6aab2b14e6e5f80d6a10cd56c16ab33cde5b71d3 Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Sat, 19 Apr 2025 21:51:19 -0400
Subject: [PATCH 568/710] [mlir][linalg] Clean up debug prints. NFC. (#136438)

Use `llvm::interleaved` from #135517 to simplify printing.
---
 .../Dialect/Linalg/Transforms/Transforms.cpp  | 115 +++++++++---------
 1 file changed, 58 insertions(+), 57 deletions(-)

diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 4c2a7c36d8b5e..6700b4e0c2cb6 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -33,6 +33,7 @@
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/raw_ostream.h"
 #include <type_traits>
 #include <utility>
@@ -95,6 +96,10 @@ static bool hasAtMostOneResultFunctionOfDim(AffineMap map, int64_t dim) {
   }
   return true;
 }
+
+static std::string stringifyReassocIndices(ReassociationIndicesRef ri) {
+  return llvm::interleaved(ri, ", ", /*Prefix=*/"|", /*Suffix=*/"");
+}
 #endif // NDEBUG
 
 /// Return the index of the first result of `map` that is a function of
@@ -278,22 +283,21 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
                                      highs, paddingValue, /*nofold=*/false);
 
   LLVM_DEBUG(
-      DBGSNL(); DBGSNL(); llvm::interleaveComma(packingMetadata.insertPositions,
-                                                DBGS() << "insertPositions: ");
-      DBGSNL(); llvm::interleaveComma(packingMetadata.outerPositions,
-                                      DBGS() << "outerPositions: ");
-      DBGSNL(); llvm::interleaveComma(packedTensorType.getShape(),
-                                      DBGS() << "packedShape: ");
+      DBGSNL(); DBGSNL();
+      DBGS() << "insertPositions: "
+             << llvm::interleaved(packingMetadata.insertPositions);
+      DBGSNL(); DBGS() << "outerPositions: "
+                       << llvm::interleaved(packingMetadata.outerPositions);
+      DBGSNL(); DBGS() << "packedShape: "
+                       << llvm::interleaved(packedTensorType.getShape());
+      DBGSNL(); DBGS() << "packedToStripMinedShapePerm: "
+                       << llvm::interleaved(packedToStripMinedShapePerm);
       DBGSNL();
-      llvm::interleaveComma(packedToStripMinedShapePerm,
-                            DBGS() << "packedToStripMinedShapePerm: ");
-      DBGSNL(); llvm::interleaveComma(
-          packingMetadata.reassociations, DBGS() << "reassociations: ",
-          [&](ReassociationIndices ri) {
-            llvm::interleaveComma(ri, llvm::dbgs() << "|");
-          });
+      DBGS() << "reassociations: "
+             << llvm::interleaved(llvm::map_range(
+                    packingMetadata.reassociations, stringifyReassocIndices));
       DBGSNL();
-      llvm::interleaveComma(stripMinedShape, DBGS() << "stripMinedShape: ");
+      DBGS() << "stripMinedShape: " << llvm::interleaved(stripMinedShape);
       DBGSNL(); DBGS() << "collapsed type: " << collapsed; DBGSNL(););
 
   if (lowerPadLikeWithInsertSlice && packOp.isLikePad()) {
@@ -343,7 +347,7 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
 
   LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
              DBGS() << "reshape op: " << reshapeOp; DBGSNL();
-             llvm::interleaveComma(transpPerm, DBGS() << "transpPerm: ");
+             DBGS() << "transpPerm: " << llvm::interleaved(transpPerm);
              DBGSNL(); DBGS() << "transpose op: " << transposeOp; DBGSNL(););
 
   // 7. Replace packOp by transposeOp.
@@ -412,20 +416,19 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
       loc, unPackOp.getSource(), emptyOp, packedToStripMinedShapePerm);
 
   LLVM_DEBUG(
-      DBGSNL(); DBGSNL(); llvm::interleaveComma(packingMetadata.insertPositions,
-                                                DBGS() << "insertPositions: ");
-      DBGSNL(); llvm::interleaveComma(packedTensorType.getShape(),
-                                      DBGS() << "packedShape: ");
+      DBGSNL(); DBGSNL();
+      DBGS() << "insertPositions: "
+             << llvm::interleaved(packingMetadata.insertPositions);
+      DBGSNL(); DBGS() << "packedShape: "
+                       << llvm::interleaved(packedTensorType.getShape());
+      DBGSNL(); DBGS() << "packedToStripMinedShapePerm: "
+                       << llvm::interleaved(packedToStripMinedShapePerm);
       DBGSNL();
-      llvm::interleaveComma(packedToStripMinedShapePerm,
-                            DBGS() << "packedToStripMinedShapePerm: ");
-      DBGSNL(); llvm::interleaveComma(
-          packingMetadata.reassociations, DBGS() << "reassociations: ",
-          [&](ReassociationIndices ri) {
-            llvm::interleaveComma(ri, llvm::dbgs() << "|");
-          });
+      DBGS() << "reassociations: "
+             << llvm::interleaved(llvm::map_range(
+                    packingMetadata.reassociations, stringifyReassocIndices));
       DBGSNL();
-      llvm::interleaveComma(stripMinedShape, DBGS() << "stripMinedShape: ");
+      DBGS() << "stripMinedShape: " << llvm::interleaved(stripMinedShape);
       DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
 
   // 4. Collapse from the stripMinedShape to the padded result.
@@ -488,10 +491,10 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
   SmallVector<AffineMap> indexingMaps = linalgOp.getIndexingMapsArray();
   SmallVector<utils::IteratorType> iteratorTypes =
       linalgOp.getIteratorTypesArray();
-  LLVM_DEBUG(DBGS() << "Start packing: " << linalgOp << "\n";
-             llvm::interleaveComma(indexingMaps, DBGS() << "maps: "); DBGSNL();
-             llvm::interleaveComma(iteratorTypes, DBGS() << "iterators: ");
-             DBGSNL(););
+  LLVM_DEBUG(DBGS() << "Start packing: " << linalgOp << "\n"
+                    << "maps: " << llvm::interleaved(indexingMaps) << "\n"
+                    << "iterators: " << llvm::interleaved(iteratorTypes)
+                    << "\n");
 
   SmallVector<linalg::PackOp> packOps;
   SmallVector<linalg::UnPackOp> unPackOps;
@@ -515,18 +518,18 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
 
     LLVM_DEBUG(
         DBGS() << "++++ After pack size #" << i << ": " << packedSizes[i]
-               << "\n";
-        llvm::interleaveComma(indexingMaps, DBGS() << "maps: "); DBGSNL();
-        llvm::interleaveComma(iteratorTypes, DBGS() << "iterators: "); DBGSNL();
-        llvm::interleaveComma(packedOperandsDims.packedDimForEachOperand,
-                              DBGS() << "packedDimForEachOperand: ");
-        DBGSNL(););
+               << "\n"
+               << "maps: " << llvm::interleaved(indexingMaps) << "\n"
+               << "iterators: " << llvm::interleaved(iteratorTypes) << "\n"
+               << "packedDimForEachOperand: "
+               << llvm::interleaved(packedOperandsDims.packedDimForEachOperand)
+               << "\n");
   }
 
   // Step 2. Propagate packing to all LinalgOp operands.
   SmallVector<Value> inputsAndInits, results;
-  SmallVector<OpOperand *> initOperands = llvm::to_vector(llvm::map_range(
-      linalgOp.getDpsInitsMutable(), [](OpOperand &o) { return &o; }));
+  SmallVector<OpOperand *> initOperands =
+      llvm::to_vector(llvm::make_pointer_range(linalgOp.getDpsInitsMutable()));
   SmallVector<OpOperand *> inputOperands = linalgOp.getDpsInputOperands();
   for (const auto &operandsList : {inputOperands, initOperands}) {
     for (OpOperand *opOperand : operandsList) {
@@ -536,11 +539,10 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
           listOfPackedOperandsDim.extractPackedDimsForOperand(pos);
       SmallVector<OpFoldResult> innerPackSizes =
           listOfPackedOperandsDim.extractPackSizesForOperand(pos);
-      LLVM_DEBUG(
-          DBGS() << "operand: " << operand << "\n";
-          llvm::interleaveComma(innerPos, DBGS() << "innerPos: "); DBGSNL();
-          llvm::interleaveComma(innerPackSizes, DBGS() << "innerPackSizes: ");
-          DBGSNL(););
+      LLVM_DEBUG(DBGS() << "operand: " << operand << "\n"
+                        << "innerPos: " << llvm::interleaved(innerPos) << "\n"
+                        << "innerPackSizes: "
+                        << llvm::interleaved(innerPackSizes) << "\n");
       if (innerPackSizes.empty()) {
         inputsAndInits.push_back(operand);
         continue;
@@ -835,7 +837,7 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
   // not change the indexings of any operand.
   SmallVector<int64_t> permutation =
       computePermutationVector(numLoops, {mPos, nPos, kPos}, mmnnkkPos);
-  LLVM_DEBUG(llvm::interleaveComma(permutation, DBGS() << "perm: "); DBGSNL(););
+  LLVM_DEBUG(DBGS() << "perm: " << llvm::interleaved(permutation) << "\n");
   // Sign .. unsigned pollution.
   SmallVector<unsigned> unsignedPerm(permutation.begin(), permutation.end());
   FailureOr<GenericOp> interchangeResult =
@@ -864,12 +866,12 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
 
   // Add leading zeros to match numLoops, we only pack the last 3 dimensions
   // post interchange.
-  LLVM_DEBUG(llvm::interleaveComma(paddedSizesNextMultipleOf,
-                                   DBGS() << "paddedSizesNextMultipleOf: ");
-             DBGSNL(););
-  LLVM_DEBUG(llvm::interleaveComma(loopRanges, DBGS() << "loopRanges: ",
-                                   [](Range r) { llvm::dbgs() << r.size; });
-             DBGSNL(););
+  LLVM_DEBUG(DBGS() << "paddedSizesNextMultipleOf: "
+                    << llvm::interleaved(paddedSizesNextMultipleOf) << "\n"
+                    << "loopRanges: "
+                    << llvm::interleaved(llvm::map_range(
+                           loopRanges, [](Range r) { return r.size; }))
+                    << "\n");
   SmallVector<OpFoldResult> adjustedPackedSizes(numLoops - packedSizes.size(),
                                                 rewriter.getIndexAttr(0));
   for (int64_t i = 0, e = numPackedDims; i < e; ++i) {
@@ -885,9 +887,8 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
         {loopRanges[adjustedPackedSizes.size()].size,
          rewriter.getIndexAttr(paddedSizesNextMultipleOf[i])}));
   }
-  LLVM_DEBUG(llvm::interleaveComma(adjustedPackedSizes,
-                                   DBGS() << "adjustedPackedSizes: ");
-             DBGSNL(););
+  LLVM_DEBUG(DBGS() << "adjustedPackedSizes: "
+                    << llvm::interleaved(adjustedPackedSizes) << "\n");
 
   // TODO: If we wanted to give the genericOp a name after packing, after
   // calling `pack` would be a good time. One would still need to check that
@@ -1202,9 +1203,9 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
 
   srcPermForTranspose.append(SmallVector<int64_t>(packOp.getInnerDimsPos()));
 
-  LLVM_DEBUG(DBGS() << "Pack permutation: " << packOp << "\n";
-             llvm::interleaveComma(srcPermForTranspose, DBGS() << "perm: ");
-             DBGSNL(););
+  LLVM_DEBUG(DBGS() << "Pack permutation: " << packOp << "\n"
+                    << "perm: " << llvm::interleaved(srcPermForTranspose)
+                    << "\n");
 
   // 2.1 Create tensor.empty (init value for TransposeOp)
   SmallVector<OpFoldResult> transShapeForEmptyOp(srcRank - numTiles,

From e9487fed29f386bd67e468b6c7a9c86684a6247d Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 19:07:10 -0700
Subject: [PATCH 569/710] [llvm] Construct SmallVector with iterator ranges
 (NFC) (#136460)

---
 llvm/lib/Analysis/CGSCCPassManager.cpp        |  4 +--
 .../CodeGen/AsmPrinter/DwarfStringPool.cpp    |  7 ++----
 .../JITLink/EHFrameSupport.cpp                |  4 +--
 llvm/lib/FuzzMutate/IRMutator.cpp             | 25 ++++++++-----------
 llvm/lib/FuzzMutate/RandomIRBuilder.cpp       | 12 +++------
 .../AArch64/AArch64Arm64ECCallLowering.cpp    |  8 ++----
 llvm/lib/Target/ARM/A15SDOptimizer.cpp        |  5 ++--
 llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp      |  4 +--
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp |  5 ++--
 .../SPIRV/SPIRVPreLegalizerCombiner.cpp       |  5 ++--
 llvm/lib/Target/X86/X86CmovConversion.cpp     |  4 +--
 llvm/lib/Transforms/Scalar/IndVarSimplify.cpp |  4 +--
 .../ExecutionEngine/MCJIT/MCJITTestBase.h     |  5 +---
 .../FuzzMutate/RandomIRBuilderTest.cpp        |  5 +---
 llvm/unittests/IR/BasicBlockDbgInfoTest.cpp   | 10 +++-----
 llvm/unittests/IR/FunctionTest.cpp            |  4 +--
 .../SandboxVectorizer/IntervalTest.cpp        |  4 +--
 17 files changed, 37 insertions(+), 78 deletions(-)

diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp
index 1f0cda7e3f91f..30b6c1613d3ad 100644
--- a/llvm/lib/Analysis/CGSCCPassManager.cpp
+++ b/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -515,9 +515,7 @@ PreservedAnalyses CGSCCToFunctionPassAdaptor::run(LazyCallGraph::SCC &C,
   FunctionAnalysisManager &FAM =
       AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
 
-  SmallVector<LazyCallGraph::Node *, 4> Nodes;
-  for (LazyCallGraph::Node &N : C)
-    Nodes.push_back(&N);
+  SmallVector<LazyCallGraph::Node *, 4> Nodes(llvm::make_pointer_range(C));
 
   // The SCC may get split while we are optimizing functions due to deleting
   // edges. If this happens, the current SCC can shift, so keep track of
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
index 2292590b135ea..743101c228200 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
@@ -81,11 +81,8 @@ void DwarfStringPool::emit(AsmPrinter &Asm, MCSection *StrSection,
   Asm.OutStreamer->switchSection(StrSection);
 
   // Get all of the string pool entries and sort them by their offset.
-  SmallVector<const StringMapEntry<EntryTy> *, 64> Entries;
-  Entries.reserve(Pool.size());
-
-  for (const auto &E : Pool)
-    Entries.push_back(&E);
+  SmallVector<const StringMapEntry<EntryTy> *, 64> Entries(
+      llvm::make_pointer_range(Pool));
 
   llvm::sort(Entries, [](const StringMapEntry<EntryTy> *A,
                          const StringMapEntry<EntryTy> *B) {
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
index abe9ae7a3da16..44c9ec028aadb 100644
--- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
@@ -643,9 +643,7 @@ EHFrameCFIBlockInspector EHFrameCFIBlockInspector::FromEdgeScan(Block &B) {
     return EHFrameCFIBlockInspector(nullptr);
   if (B.edges_size() == 1)
     return EHFrameCFIBlockInspector(&*B.edges().begin());
-  SmallVector<Edge *, 3> Es;
-  for (auto &E : B.edges())
-    Es.push_back(&E);
+  SmallVector<Edge *, 3> Es(llvm::make_pointer_range(B.edges()));
   assert(Es.size() >= 2 && Es.size() <= 3 && "Unexpected number of edges");
   llvm::sort(Es, [](const Edge *LHS, const Edge *RHS) {
     return LHS->getOffset() < RHS->getOffset();
diff --git a/llvm/lib/FuzzMutate/IRMutator.cpp b/llvm/lib/FuzzMutate/IRMutator.cpp
index 06f240301322c..7e28f58046b1f 100644
--- a/llvm/lib/FuzzMutate/IRMutator.cpp
+++ b/llvm/lib/FuzzMutate/IRMutator.cpp
@@ -122,9 +122,8 @@ getInsertionRange(BasicBlock &BB) {
 }
 
 void InjectorIRStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
-  SmallVector<Instruction *, 32> Insts;
-  for (Instruction &I : getInsertionRange(BB))
-    Insts.push_back(&I);
+  SmallVector<Instruction *, 32> Insts(
+      llvm::make_pointer_range(getInsertionRange(BB)));
   if (Insts.size() < 1)
     return;
 
@@ -395,9 +394,8 @@ void InsertFunctionStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
     return isRetVoid ? nullptr : Call;
   };
 
-  SmallVector<Instruction *, 32> Insts;
-  for (Instruction &I : getInsertionRange(BB))
-    Insts.push_back(&I);
+  SmallVector<Instruction *, 32> Insts(
+      llvm::make_pointer_range(getInsertionRange(BB)));
   if (Insts.size() < 1)
     return;
 
@@ -421,9 +419,8 @@ void InsertFunctionStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
 }
 
 void InsertCFGStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
-  SmallVector<Instruction *, 32> Insts;
-  for (Instruction &I : getInsertionRange(BB))
-    Insts.push_back(&I);
+  SmallVector<Instruction *, 32> Insts(
+      llvm::make_pointer_range(getInsertionRange(BB)));
   if (Insts.size() < 1)
     return;
 
@@ -561,9 +558,8 @@ void InsertPHIStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
     }
     PHI->addIncoming(Src, Pred);
   }
-  SmallVector<Instruction *, 32> InstsAfter;
-  for (Instruction &I : getInsertionRange(BB))
-    InstsAfter.push_back(&I);
+  SmallVector<Instruction *, 32> InstsAfter(
+      llvm::make_pointer_range(getInsertionRange(BB)));
   IB.connectToSink(BB, InstsAfter, PHI);
 }
 
@@ -573,9 +569,8 @@ void SinkInstructionStrategy::mutate(Function &F, RandomIRBuilder &IB) {
   }
 }
 void SinkInstructionStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
-  SmallVector<Instruction *, 32> Insts;
-  for (Instruction &I : getInsertionRange(BB))
-    Insts.push_back(&I);
+  SmallVector<Instruction *, 32> Insts(
+      llvm::make_pointer_range(getInsertionRange(BB)));
   if (Insts.size() < 1)
     return;
   // Choose an Instruction to mutate.
diff --git a/llvm/lib/FuzzMutate/RandomIRBuilder.cpp b/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
index 8aea3d6f7e059..22ce5200e132d 100644
--- a/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
+++ b/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
@@ -84,10 +84,8 @@ RandomIRBuilder::findOrCreateGlobalVariable(Module *M, ArrayRef<Value *> Srcs,
     return Pred.matches(Srcs, PoisonValue::get(GV->getValueType()));
   };
   bool DidCreate = false;
-  SmallVector<GlobalVariable *, 4> GlobalVars;
-  for (GlobalVariable &GV : M->globals()) {
-    GlobalVars.push_back(&GV);
-  }
+  SmallVector<GlobalVariable *, 4> GlobalVars(
+      llvm::make_pointer_range(M->globals()));
   auto RS = makeSampler(Rand, make_filter_range(GlobalVars, MatchesPred));
   RS.sample(nullptr, 1);
   GlobalVariable *GV = RS.getSelection();
@@ -146,10 +144,8 @@ Value *RandomIRBuilder::findOrCreateSource(BasicBlock &BB,
       auto Dominators = getDominators(&BB);
       std::shuffle(Dominators.begin(), Dominators.end(), Rand);
       for (BasicBlock *Dom : Dominators) {
-        SmallVector<Instruction *, 16> Instructions;
-        for (Instruction &I : *Dom) {
-          Instructions.push_back(&I);
-        }
+        SmallVector<Instruction *, 16> Instructions(
+            llvm::make_pointer_range(*Dom));
         auto RS =
             makeSampler(Rand, make_filter_range(Instructions, MatchesPred));
         // Also consider choosing no source, meaning we want a new one.
diff --git a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
index 11e2c940548e2..acc22da393ee0 100644
--- a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
@@ -656,9 +656,7 @@ Function *AArch64Arm64ECCallLowering::buildGuestExitThunk(Function *F) {
   GuardCheck->setCallingConv(CallingConv::CFGuard_Check);
 
   Value *GuardRetVal = B.CreateBitCast(GuardCheck, PtrTy);
-  SmallVector<Value *> Args;
-  for (Argument &Arg : GuestExit->args())
-    Args.push_back(&Arg);
+  SmallVector<Value *> Args(llvm::make_pointer_range(GuestExit->args()));
   CallInst *Call = B.CreateCall(Arm64Ty, GuardRetVal, Args);
   Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
 
@@ -715,9 +713,7 @@ AArch64Arm64ECCallLowering::buildPatchableThunk(GlobalAlias *UnmangledAlias,
   Dispatch->setCallingConv(CallingConv::CFGuard_Check);
 
   Value *DispatchRetVal = B.CreateBitCast(Dispatch, PtrTy);
-  SmallVector<Value *> Args;
-  for (Argument &Arg : GuestExit->args())
-    Args.push_back(&Arg);
+  SmallVector<Value *> Args(llvm::make_pointer_range(GuestExit->args()));
   CallInst *Call = B.CreateCall(Arm64Ty, DispatchRetVal, Args);
   Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
 
diff --git a/llvm/lib/Target/ARM/A15SDOptimizer.cpp b/llvm/lib/Target/ARM/A15SDOptimizer.cpp
index bb9a0a2bdf98b..f6705669714da 100644
--- a/llvm/lib/Target/ARM/A15SDOptimizer.cpp
+++ b/llvm/lib/Target/ARM/A15SDOptimizer.cpp
@@ -617,10 +617,9 @@ bool A15SDOptimizer::runOnInstruction(MachineInstr *MI) {
         continue;
 
       // Collect all the uses of this MI's DPR def for updating later.
-      SmallVector<MachineOperand*, 8> Uses;
       Register DPRDefReg = MI->getOperand(0).getReg();
-      for (MachineOperand &MO : MRI->use_operands(DPRDefReg))
-        Uses.push_back(&MO);
+      SmallVector<MachineOperand *, 8> Uses(
+          llvm::make_pointer_range(MRI->use_operands(DPRDefReg)));
 
       // We can optimize this.
       unsigned NewReg = optimizeSDPattern(MI);
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index 6452fa05947dd..ee835dba9b020 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -554,9 +554,7 @@ static void handleByValParam(const NVPTXTargetMachine &TM, Argument *Arg) {
   if (ArgUseIsReadOnly && AUC.Conditionals.empty()) {
     // Convert all loads and intermediate operations to use parameter AS and
     // skip creation of a local copy of the argument.
-    SmallVector<Use *, 16> UsesToUpdate;
-    for (Use &U : Arg->uses())
-      UsesToUpdate.push_back(&U);
+    SmallVector<Use *, 16> UsesToUpdate(llvm::make_pointer_range(Arg->uses()));
 
     Value *ArgInParamAS = new AddrSpaceCastInst(
         Arg, PointerType::get(StructType->getContext(), ADDRESS_SPACE_PARAM),
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index a7e4487f6f5d5..6205dfedb79fb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2399,9 +2399,8 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
 
   preprocessUndefs(B);
   preprocessCompositeConstants(B);
-  SmallVector<Instruction *> Worklist;
-  for (auto &I : instructions(Func))
-    Worklist.push_back(&I);
+  SmallVector<Instruction *> Worklist(
+      llvm::make_pointer_range(instructions(Func)));
 
   applyDemangledPtrArgTypes(B);
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp
index d2de71ce10c10..c96ee6b02491a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp
@@ -99,9 +99,8 @@ void applySPIRVDistance(MachineInstr &MI, MachineRegisterInfo &MRI,
   SPIRVGlobalRegistry *GR =
       MI.getMF()->getSubtarget<SPIRVSubtarget>().getSPIRVGlobalRegistry();
   auto RemoveAllUses = [&](Register Reg) {
-    SmallVector<MachineInstr *, 4> UsesToErase;
-    for (auto &UseMI : MRI.use_instructions(Reg))
-      UsesToErase.push_back(&UseMI);
+    SmallVector<MachineInstr *, 4> UsesToErase(
+        llvm::make_pointer_range(MRI.use_instructions(Reg)));
 
     // calling eraseFromParent to early invalidates the iterator.
     for (auto *MIToErase : UsesToErase) {
diff --git a/llvm/lib/Target/X86/X86CmovConversion.cpp b/llvm/lib/Target/X86/X86CmovConversion.cpp
index 488b3126b8609..e389ce9a296b1 100644
--- a/llvm/lib/Target/X86/X86CmovConversion.cpp
+++ b/llvm/lib/Target/X86/X86CmovConversion.cpp
@@ -190,9 +190,7 @@ bool X86CmovConverterPass::runOnMachineFunction(MachineFunction &MF) {
   // execution behind a branch is better suited to handle on modern x86 chips.
   if (ForceMemOperand || ForceAll) {
     CmovGroups AllCmovGroups;
-    SmallVector<MachineBasicBlock *, 4> Blocks;
-    for (auto &MBB : MF)
-      Blocks.push_back(&MBB);
+    SmallVector<MachineBasicBlock *, 4> Blocks(llvm::make_pointer_range(MF));
     if (collectCmovCandidates(Blocks, AllCmovGroups, /*IncludeLoads*/ true)) {
       for (auto &Group : AllCmovGroups) {
         // Skip any group that doesn't do at least one memory operand cmov.
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index db97f7e1efcfd..70610f7c020c8 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -406,9 +406,7 @@ bool IndVarSimplify::rewriteNonIntegerIVs(Loop *L) {
   // the SCEV routines.
   BasicBlock *Header = L->getHeader();
 
-  SmallVector<WeakTrackingVH, 8> PHIs;
-  for (PHINode &PN : Header->phis())
-    PHIs.push_back(&PN);
+  SmallVector<WeakTrackingVH, 8> PHIs(llvm::make_pointer_range(Header->phis()));
 
   bool Changed = false;
   for (WeakTrackingVH &PHI : PHIs)
diff --git a/llvm/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h b/llvm/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
index a7cd47325845a..1928adc2d7e72 100644
--- a/llvm/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
+++ b/llvm/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
@@ -63,10 +63,7 @@ class TrivialModuleBuilder {
   Function *insertSimpleCallFunction(Module *M, Function *Callee) {
     Function *Result = startFunction(M, Callee->getFunctionType(), "caller");
 
-    SmallVector<Value*, 1> CallArgs;
-
-    for (Argument &A : Result->args())
-      CallArgs.push_back(&A);
+    SmallVector<Value *, 1> CallArgs(llvm::make_pointer_range(Result->args()));
 
     Value *ReturnCode = Builder.CreateCall(Callee, CallArgs);
     Builder.CreateRet(ReturnCode);
diff --git a/llvm/unittests/FuzzMutate/RandomIRBuilderTest.cpp b/llvm/unittests/FuzzMutate/RandomIRBuilderTest.cpp
index 02d87251a428c..b7f5234ffcda1 100644
--- a/llvm/unittests/FuzzMutate/RandomIRBuilderTest.cpp
+++ b/llvm/unittests/FuzzMutate/RandomIRBuilderTest.cpp
@@ -597,10 +597,7 @@ TEST(RandomIRBuilderTest, SrcAndSinkWOrphanBlock) {
     std::unique_ptr<Module> M = parseAssembly(Source, Ctx);
     Function &F = *M->getFunction("test");
     for (BasicBlock &BB : F) {
-      SmallVector<Instruction *, 4> Insts;
-      for (Instruction &I : BB) {
-        Insts.push_back(&I);
-      }
+      SmallVector<Instruction *, 4> Insts(llvm::make_pointer_range(BB));
       for (int j = 0; j < 10; j++) {
         IB.findOrCreateSource(BB, Insts);
       }
diff --git a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
index 00d45e2957dde..5607c633b7a88 100644
--- a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
+++ b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
@@ -224,9 +224,8 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) {
   EXPECT_EQ(BB.size(), 1u);
   EXPECT_EQ(Marker2->StoredDbgRecords.size(), 2u);
   // They should also be in the correct order.
-  SmallVector<DbgRecord *, 2> DVRs;
-  for (DbgRecord &DVR : Marker2->getDbgRecordRange())
-    DVRs.push_back(&DVR);
+  SmallVector<DbgRecord *, 2> DVRs(
+      llvm::make_pointer_range(Marker2->getDbgRecordRange()));
   EXPECT_EQ(DVRs[0], DVR1);
   EXPECT_EQ(DVRs[1], DVR2);
 
@@ -577,9 +576,8 @@ class DbgSpliceTest : public ::testing::Test {
 
   bool CheckDVROrder(Instruction *I,
                      SmallVector<DbgVariableRecord *> CheckVals) {
-    SmallVector<DbgRecord *> Vals;
-    for (DbgRecord &D : I->getDbgRecordRange())
-      Vals.push_back(&D);
+    SmallVector<DbgRecord *> Vals(
+        llvm::make_pointer_range(I->getDbgRecordRange()));
 
     EXPECT_EQ(Vals.size(), CheckVals.size());
     if (Vals.size() != CheckVals.size())
diff --git a/llvm/unittests/IR/FunctionTest.cpp b/llvm/unittests/IR/FunctionTest.cpp
index f6735fb55a6d5..7ba7584e22976 100644
--- a/llvm/unittests/IR/FunctionTest.cpp
+++ b/llvm/unittests/IR/FunctionTest.cpp
@@ -83,9 +83,7 @@ TEST(FunctionTest, stealArgumentListFrom) {
 
   // Save arguments from F1 for later assertions.  F1 won't have lazy arguments
   // anymore.
-  SmallVector<Argument *, 4> Args;
-  for (Argument &A : F1->args())
-    Args.push_back(&A);
+  SmallVector<Argument *, 4> Args(llvm::make_pointer_range(F1->args()));
   EXPECT_EQ(2u, Args.size());
   EXPECT_FALSE(F1->hasLazyArguments());
 
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
index 59498371b4d73..ea09fba85ba6a 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
@@ -157,9 +157,7 @@ define void @foo(i8 %v0) {
 // of references.
 template <typename RangeT>
 static SmallVector<sandboxir::Instruction *> getPtrVec(RangeT Range) {
-  SmallVector<sandboxir::Instruction *> PtrVec;
-  for (sandboxir::Instruction &I : Range)
-    PtrVec.push_back(&I);
+  SmallVector<sandboxir::Instruction *> PtrVec(llvm::make_pointer_range(Range));
   return PtrVec;
 }
 

From eb78b9b42f56c59a7fec7fae8f14b570e0a94646 Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Sat, 19 Apr 2025 22:59:57 -0400
Subject: [PATCH 570/710] [mlir] Clean up prints with `llvm::interleaved`. NFC.
 (#136468)

---
 .../Analysis/FlatLinearValueConstraints.cpp   |  7 +--
 mlir/lib/Debug/Observers/ActionLogging.cpp    |  9 ++--
 .../Linalg/TransformOps/GPUHeuristics.cpp     | 46 ++++++++-----------
 .../Linalg/TransformOps/LinalgMatchOps.cpp    | 14 +++---
 .../Dialect/Linalg/TransformOps/Syntax.cpp    |  3 +-
 mlir/lib/IR/ExtensibleDialect.cpp             |  5 +-
 mlir/lib/IR/PDL/PDLPatternMatch.cpp           |  5 +-
 mlir/lib/Interfaces/InferTypeOpInterface.cpp  | 19 ++++----
 8 files changed, 48 insertions(+), 60 deletions(-)

diff --git a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
index 6ad39a3a91293..6cc5ec639f448 100644
--- a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
+++ b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -1106,9 +1107,9 @@ IntegerSet FlatLinearConstraints::getAsIntegerSet(MLIRContext *context) const {
     }
     if (!noLocalRepVars.empty()) {
       LLVM_DEBUG({
-        llvm::dbgs() << "local variables at position(s) ";
-        llvm::interleaveComma(noLocalRepVars, llvm::dbgs());
-        llvm::dbgs() << " do not have an explicit representation in:\n";
+        llvm::dbgs() << "local variables at position(s) "
+                     << llvm::interleaved(noLocalRepVars)
+                     << " do not have an explicit representation in:\n";
         this->dump();
       });
       return IntegerSet();
diff --git a/mlir/lib/Debug/Observers/ActionLogging.cpp b/mlir/lib/Debug/Observers/ActionLogging.cpp
index add16e84653e5..9c0d041b0d36f 100644
--- a/mlir/lib/Debug/Observers/ActionLogging.cpp
+++ b/mlir/lib/Debug/Observers/ActionLogging.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Debug/Observers/ActionLogging.h"
 #include "mlir/Debug/BreakpointManager.h"
 #include "mlir/IR/Action.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -55,11 +56,9 @@ void ActionLogger::beforeExecute(const ActionActiveStack *action,
     action->getAction().print(os);
   else
     os << action->getAction().getTag();
-  if (printIRUnits) {
-    os << " (";
-    interleaveComma(action->getAction().getContextIRUnits(), os);
-    os << ")";
-  }
+  if (printIRUnits)
+    os << " (" << llvm::interleaved(action->getAction().getContextIRUnits())
+       << ")";
   os << "`\n";
 }
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/GPUHeuristics.cpp b/mlir/lib/Dialect/Linalg/TransformOps/GPUHeuristics.cpp
index 0657a87d1d1ac..5b569e11388b0 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/GPUHeuristics.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/GPUHeuristics.cpp
@@ -13,6 +13,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cmath>
@@ -44,8 +45,8 @@ transform::gpu::CopyMappingInfo::CopyMappingInfo(MLIRContext *ctx,
          "only 1,2,3-D copies are supported for now");
 
   LDBG("START CopyMappingInfo, favorPredication: " << favorPredication);
-  LLVM_DEBUG(llvm::interleaveComma(copySizes, DBGS() << "--copy shape: ");
-             llvm::dbgs() << "\n";);
+  LLVM_DEBUG(DBGS() << "--copy shape: " << llvm::interleaved(copySizes)
+                    << "\n");
 
   // Greedily find the largest vector size that can be used to copy the most
   // minor dimension: we are in the business of filling kMaxVectorLoadBitWidth
@@ -63,11 +64,10 @@ transform::gpu::CopyMappingInfo::CopyMappingInfo(MLIRContext *ctx,
   if (status == Status::Invalid)
     return;
 
-  LLVM_DEBUG(llvm::interleaveComma(copySizes, DBGS() << "--copy: ");
-             llvm::dbgs() << "\n"; llvm::interleaveComma(
-                 this->numThreads, DBGS() << "--numThreads: ");
-             llvm::dbgs() << "\n";);
-  LDBG("--vectorSize: " << this->vectorSize);
+  LLVM_DEBUG(DBGS() << "--copy: " << llvm::interleaved(copySizes) << "\n"
+                    << "--numThreads: " << llvm::interleaved(this->numThreads)
+                    << "\n"
+                    << "--vectorSize: " << this->vectorSize << "\n");
   assert(this->numThreads.size() == copySizes.size() &&
          "compute copy mapping expected same number of threads and copy sizes");
 
@@ -158,10 +158,8 @@ static SmallVector<int64_t> maximizeNumThreads(ArrayRef<int64_t> sizes,
     int64_t localBest = factor * product(nestedThreadsPerDim);
     if (localBest > best && localBest <= maxNumThreads) {
       LDBG(indent << "new localBest: " << localBest);
-      LLVM_DEBUG(
-          llvm::interleaveComma(nestedThreadsPerDim,
-                                DBGS() << indent << "nestedThreadsPerDim: ");
-          llvm::dbgs() << "\n";);
+      LDBG(indent << "nestedThreadsPerDim: "
+                  << llvm::interleaved(nestedThreadsPerDim));
       localThreadsPerDim.clear();
       localThreadsPerDim.push_back(factor);
       llvm::append_range(localThreadsPerDim, nestedThreadsPerDim);
@@ -170,10 +168,7 @@ static SmallVector<int64_t> maximizeNumThreads(ArrayRef<int64_t> sizes,
   }
 
   LDBG(indent << "found globalBest: " << best);
-  LLVM_DEBUG(llvm::interleaveComma(localThreadsPerDim,
-                                   DBGS() << indent << "numThreads: ");
-             llvm::dbgs() << "\n";);
-
+  LDBG(indent << "numThreads: " << llvm::interleaved(localThreadsPerDim));
   return localThreadsPerDim;
 }
 
@@ -231,10 +226,8 @@ transform::gpu::CopyMappingInfo::inferNumThreadsImpl(
   SmallVector<int64_t> inferredNumThreads =
       maximizeNumThreads(scaledSizes, 0, totalNumThreads);
 
-  LLVM_DEBUG(llvm::interleaveComma(inferredNumThreads,
-                                   DBGS() << "inferred numThreads: ");
-             llvm::dbgs() << "\n";
-             LDBG("computed actualVectorSize: " << desiredVectorSize););
+  LDBG("inferred numThreads: " << llvm::interleaved(inferredNumThreads));
+  LDBG("computed actualVectorSize: " << desiredVectorSize);
 
   // Corner case: we cannot use more threads than available. If the dimension of
   // the copy is so bad it is because higher-level tiling did not do its job, we
@@ -255,13 +248,10 @@ transform::gpu::CopyMappingInfo::inferNumThreadsImpl(
 }
 
 void transform::gpu::CopyMappingInfo::print(llvm::raw_ostream &os) const {
-  os << "MappingInfo{";
-  os << "CopyMappingInfo: ";
-  os << "valid: " << (status != Status::Invalid) << ", ";
-  os << "vectorSize: " << vectorSize << ", ";
-  llvm::interleaveComma(numThreads, os << ", numThreads: {");
-  llvm::interleaveComma(smallestBoundingTileSizes,
-                        os << "}, smallestBoundingTileSizes: {");
-  llvm::interleaveComma(threadMapping, os << "}, threadMapping: {");
-  os << "}}";
+  os << "MappingInfo{"
+     << "CopyMappingInfo: " << "valid: " << (status != Status::Invalid) << ", "
+     << "vectorSize: " << vectorSize << ", numThreads: {"
+     << llvm::interleaved(numThreads) << "}, smallestBoundingTileSizes: {"
+     << llvm::interleaved(smallestBoundingTileSizes) << "}, threadMapping: {"
+     << llvm::interleaved(threadMapping) << "}}";
 }
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
index 5460f50328a45..67bb1de59dd43 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Interfaces/FunctionImplementation.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
 
@@ -219,14 +220,11 @@ LogicalResult transform::MatchStructuredBodyOp::verify() {
                        getElementwise() + getContraction().has_value();
 
   if (numOptions > 1) {
-    std::string attributeNames;
-    llvm::raw_string_ostream os(attributeNames);
-    llvm::interleaveComma(ArrayRef<StringAttr>{getReductionPositionAttrName(),
-                                               getPassthroughAttrName(),
-                                               getElementwiseAttrName(),
-                                               getContractionAttrName()},
-                          os);
-    return emitOpError() << "only one of {" << attributeNames << "} is allowed";
+    StringAttr attributeNames[] = {
+        getReductionPositionAttrName(), getPassthroughAttrName(),
+        getElementwiseAttrName(), getContractionAttrName()};
+    return emitOpError() << "only one of {" << llvm::interleaved(attributeNames)
+                         << "} is allowed";
   }
 
   if (std::optional<ArrayAttr> contractionAttr = getContraction()) {
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/Syntax.cpp b/mlir/lib/Dialect/Linalg/TransformOps/Syntax.cpp
index 266c9ad3314a3..e968c73d8e075 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/Syntax.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/Syntax.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/Linalg/TransformOps/Syntax.h"
 #include "mlir/IR/OpImplementation.h"
+#include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
 
@@ -67,7 +68,7 @@ void mlir::printSemiFunctionType(OpAsmPrinter &printer, Operation *op,
 
   if (resultType.size() > 1)
     printer << "(";
-  llvm::interleaveComma(resultType, printer.getStream());
+  printer << llvm::interleaved(resultType);
   if (resultType.size() > 1)
     printer << ")";
 }
diff --git a/mlir/lib/IR/ExtensibleDialect.cpp b/mlir/lib/IR/ExtensibleDialect.cpp
index 8a7d74700006c..3312549ddfcd4 100644
--- a/mlir/lib/IR/ExtensibleDialect.cpp
+++ b/mlir/lib/IR/ExtensibleDialect.cpp
@@ -11,6 +11,7 @@
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/StorageUniquerSupport.h"
+#include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
 
@@ -47,9 +48,7 @@ static void typeOrAttrPrinter(AsmPrinter &printer, ArrayRef<Attribute> params) {
   if (params.empty())
     return;
 
-  printer << "<";
-  interleaveComma(params, printer.getStream());
-  printer << ">";
+  printer << "<" << llvm::interleaved(params) << ">";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/PDL/PDLPatternMatch.cpp b/mlir/lib/IR/PDL/PDLPatternMatch.cpp
index 0d05153bf7dab..28b39dd75e26c 100644
--- a/mlir/lib/IR/PDL/PDLPatternMatch.cpp
+++ b/mlir/lib/IR/PDL/PDLPatternMatch.cpp
@@ -11,6 +11,7 @@
 #include "mlir/IR/Iterators.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/RegionKindInterface.h"
+#include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
 
@@ -34,13 +35,13 @@ void PDLValue::print(raw_ostream &os) const {
     os << cast<Type>();
     break;
   case Kind::TypeRange:
-    llvm::interleaveComma(cast<TypeRange>(), os);
+    os << llvm::interleaved(cast<TypeRange>());
     break;
   case Kind::Value:
     os << cast<Value>();
     break;
   case Kind::ValueRange:
-    llvm::interleaveComma(cast<ValueRange>(), os);
+    os << llvm::interleaved(cast<ValueRange>());
     break;
   }
 }
diff --git a/mlir/lib/Interfaces/InferTypeOpInterface.cpp b/mlir/lib/Interfaces/InferTypeOpInterface.cpp
index 6b5e103cd36c2..9f4f672fb9f4d 100644
--- a/mlir/lib/Interfaces/InferTypeOpInterface.cpp
+++ b/mlir/lib/Interfaces/InferTypeOpInterface.cpp
@@ -15,6 +15,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Matchers.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/InterleavedRange.h"
 
 using namespace mlir;
 
@@ -176,9 +177,8 @@ void ShapeAdaptor::dump() const {
       return "?";
     return llvm::formatv("{0}", dim).str();
   });
-  llvm::errs() << "rank = " << getRank() << " dims = [";
-  llvm::interleave(mapped, llvm::errs(), "x");
-  llvm::errs() << "]\n";
+  llvm::errs() << "rank = " << getRank()
+               << " dims = " << llvm::interleaved_array(mapped, "x") << "\n";
 }
 
 ShapeAdaptor ValueShapeRange::getValueAsShape(int index) {
@@ -243,13 +243,12 @@ LogicalResult mlir::detail::verifyInferredResultTypes(Operation *op) {
 void mlir::detail::reportFatalInferReturnTypesError(OperationState &state) {
   std::string buffer;
   llvm::raw_string_ostream os(buffer);
-  os << "Failed to infer result type(s):\n";
-  os << "\"" << state.name << "\"(...) ";
-  os << state.attributes.getDictionary(state.location.getContext());
-  os << " : (";
-  llvm::interleaveComma(state.operands, os,
-                        [&](Value val) { os << val.getType(); });
-  os << ") -> ( ??? )";
+  os << "Failed to infer result type(s):\n"
+     << "\"" << state.name << "\"(...) "
+     << state.attributes.getDictionary(state.location.getContext()) << " : ("
+     << llvm::interleaved(llvm::map_range(
+            state.operands, [](Value val) { return val.getType(); }))
+     << ") -> ( ??? )";
   emitRemark(state.location, "location of op");
   llvm::report_fatal_error(llvm::StringRef(buffer));
 }

From f2ec5e40d96d6ba8f99e419009b6547965905cfe Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 20:33:53 -0700
Subject: [PATCH 571/710] [clang] Use llvm::unique (NFC) (#136469)

---
 clang/lib/AST/ASTContext.cpp          |  2 +-
 clang/lib/AST/ItaniumMangle.cpp       |  5 ++---
 clang/lib/Driver/ToolChain.cpp        |  2 +-
 clang/lib/Driver/XRayArgs.cpp         |  2 +-
 clang/lib/Format/Format.cpp           | 18 +++++++++---------
 clang/lib/Sema/SemaDeclAttr.cpp       |  4 ++--
 clang/lib/Sema/SemaStmt.cpp           |  6 ++----
 clang/lib/Serialization/ASTWriter.cpp |  3 +--
 clang/utils/TableGen/NeonEmitter.cpp  |  3 +--
 clang/utils/TableGen/SveEmitter.cpp   |  3 +--
 10 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 860e6ec0fb47e..2836d68b05ff6 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -5975,7 +5975,7 @@ SortAndUniqueProtocols(SmallVectorImpl<ObjCProtocolDecl *> &Protocols) {
     P = P->getCanonicalDecl();
 
   // Remove duplicates.
-  auto ProtocolsEnd = std::unique(Protocols.begin(), Protocols.end());
+  auto ProtocolsEnd = llvm::unique(Protocols);
   Protocols.erase(ProtocolsEnd, Protocols.end());
 }
 
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 97ef1c0d46960..33a8728728574 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -328,7 +328,7 @@ class CXXNameMangler {
       }
 
       llvm::sort(TagList);
-      TagList.erase(std::unique(TagList.begin(), TagList.end()), TagList.end());
+      TagList.erase(llvm::unique(TagList), TagList.end());
 
       writeSortedUniqueAbiTags(Out, TagList);
     }
@@ -344,8 +344,7 @@ class CXXNameMangler {
 
     const AbiTagList &getSortedUniqueUsedAbiTags() {
       llvm::sort(UsedAbiTags);
-      UsedAbiTags.erase(std::unique(UsedAbiTags.begin(), UsedAbiTags.end()),
-                        UsedAbiTags.end());
+      UsedAbiTags.erase(llvm::unique(UsedAbiTags), UsedAbiTags.end());
       return UsedAbiTags;
     }
 
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 97317579c8a50..5cd5755e01587 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -378,7 +378,7 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
 
   // Sort and remove duplicates.
   std::sort(Result.begin(), Result.end());
-  Result.erase(std::unique(Result.begin(), Result.end()), Result.end());
+  Result.erase(llvm::unique(Result), Result.end());
   return Result;
 }
 
diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp
index 6649ed14c3982..701dd2906dccb 100644
--- a/clang/lib/Driver/XRayArgs.cpp
+++ b/clang/lib/Driver/XRayArgs.cpp
@@ -174,7 +174,7 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
 
   // Then we want to sort and unique the modes we've collected.
   llvm::sort(Modes);
-  Modes.erase(std::unique(Modes.begin(), Modes.end()), Modes.end());
+  Modes.erase(llvm::unique(Modes), Modes.end());
 }
 
 void XRayArgs::addArgs(const ToolChain &TC, const ArgList &Args,
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 0667813110454..5a1c3f556b331 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3244,11 +3244,11 @@ static void sortCppIncludes(const FormatStyle &Style,
   }
 
   // Deduplicate #includes.
-  Indices.erase(std::unique(Indices.begin(), Indices.end(),
-                            [&](unsigned LHSI, unsigned RHSI) {
-                              return Includes[LHSI].Text.trim() ==
-                                     Includes[RHSI].Text.trim();
-                            }),
+  Indices.erase(llvm::unique(Indices,
+                             [&](unsigned LHSI, unsigned RHSI) {
+                               return Includes[LHSI].Text.trim() ==
+                                      Includes[RHSI].Text.trim();
+                             }),
                 Indices.end());
 
   int CurrentCategory = Includes.front().Category;
@@ -3476,10 +3476,10 @@ static void sortJavaImports(const FormatStyle &Style,
   });
 
   // Deduplicate imports.
-  Indices.erase(std::unique(Indices.begin(), Indices.end(),
-                            [&](unsigned LHSI, unsigned RHSI) {
-                              return Imports[LHSI].Text == Imports[RHSI].Text;
-                            }),
+  Indices.erase(llvm::unique(Indices,
+                             [&](unsigned LHSI, unsigned RHSI) {
+                               return Imports[LHSI].Text == Imports[RHSI].Text;
+                             }),
                 Indices.end());
 
   bool CurrentIsStatic = Imports[Indices.front()].IsStatic;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index cb230774d56fc..0cadbff13bdbf 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -909,7 +909,7 @@ static void handleNoBuiltinAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   // Repeating the same attribute is fine.
   llvm::sort(Names);
-  Names.erase(std::unique(Names.begin(), Names.end()), Names.end());
+  Names.erase(llvm::unique(Names), Names.end());
 
   // Empty no_builtin must be on its own.
   if (HasWildcard && Names.size() > 1)
@@ -6037,7 +6037,7 @@ static void handleAbiTagAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   // Store tags sorted and without duplicates.
   llvm::sort(Tags);
-  Tags.erase(std::unique(Tags.begin(), Tags.end()), Tags.end());
+  Tags.erase(llvm::unique(Tags), Tags.end());
 
   D->addAttr(::new (S.Context)
                  AbiTagAttr(S.Context, AL, Tags.data(), Tags.size()));
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index 39c2e157591df..87a400e8a6291 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -1623,8 +1623,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
         EnumVals.push_back(std::make_pair(Val, EDI));
       }
       llvm::stable_sort(EnumVals, CmpEnumVals);
-      auto EI = EnumVals.begin(), EIEnd =
-        std::unique(EnumVals.begin(), EnumVals.end(), EqEnumVals);
+      auto EI = EnumVals.begin(), EIEnd = llvm::unique(EnumVals, EqEnumVals);
 
       // See which case values aren't in enum.
       for (CaseValsTy::const_iterator CI = CaseVals.begin();
@@ -1777,8 +1776,7 @@ Sema::DiagnoseAssignmentEnum(QualType DstType, QualType SrcType,
           if (EnumVals.empty())
             return;
           llvm::stable_sort(EnumVals, CmpEnumVals);
-          EnumValsTy::iterator EIend =
-              std::unique(EnumVals.begin(), EnumVals.end(), EqEnumVals);
+          EnumValsTy::iterator EIend = llvm::unique(EnumVals, EqEnumVals);
 
           // See which values aren't in the enum.
           EnumValsTy::const_iterator EI = EnumVals.begin();
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 04e73b17af3bc..bea8fd5055358 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6094,8 +6094,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema *SemaPtr, StringRef isysroot,
 
       // Sort and deduplicate module IDs.
       llvm::sort(Imports, Cmp);
-      Imports.erase(std::unique(Imports.begin(), Imports.end(), Eq),
-                    Imports.end());
+      Imports.erase(llvm::unique(Imports, Eq), Imports.end());
 
       RecordData ImportedModules;
       for (const auto &Import : Imports) {
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 51bfe212464cf..c8cd214417efe 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -2052,8 +2052,7 @@ void NeonEmitter::createIntrinsic(const Record *R,
   }
 
   sort(NewTypeSpecs);
-  NewTypeSpecs.erase(std::unique(NewTypeSpecs.begin(), NewTypeSpecs.end()),
-		     NewTypeSpecs.end());
+  NewTypeSpecs.erase(llvm::unique(NewTypeSpecs), NewTypeSpecs.end());
   auto &Entry = IntrinsicMap[Name];
 
   for (auto &I : NewTypeSpecs) {
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 200f57960fff8..c48210633516e 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -1231,8 +1231,7 @@ void SVEEmitter::createIntrinsic(
 
   // Remove duplicate type specs.
   sort(TypeSpecs);
-  TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
-                  TypeSpecs.end());
+  TypeSpecs.erase(llvm::unique(TypeSpecs), TypeSpecs.end());
 
   // Create an Intrinsic for each type spec.
   for (auto TS : TypeSpecs) {

From 9bcb18db89c5a3b317c0de2a5e94f18044390c3f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 19 Apr 2025 20:34:03 -0700
Subject: [PATCH 572/710] [clangd] Use llvm::unique (NFC) (#136470)

---
 clang-tools-extra/clangd/IncludeFixer.cpp     |  2 +-
 clang-tools-extra/clangd/InlayHints.cpp       |  2 +-
 .../clangd/SemanticHighlighting.cpp           |  2 +-
 clang-tools-extra/clangd/SourceCode.cpp       |  2 +-
 clang-tools-extra/clangd/XRefs.cpp            | 26 +++++++++----------
 clang-tools-extra/clangd/index/FileIndex.cpp  |  3 +--
 clang-tools-extra/clangd/index/Relation.cpp   |  3 +--
 .../clangd/index/dex/Trigram.cpp              |  2 +-
 clang-tools-extra/clangd/refactor/Rename.cpp  |  5 ++--
 .../refactor/tweaks/RemoveUsingNamespace.cpp  |  4 +--
 10 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/clang-tools-extra/clangd/IncludeFixer.cpp b/clang-tools-extra/clangd/IncludeFixer.cpp
index 8b74c761d23ca..4ff021c4c390a 100644
--- a/clang-tools-extra/clangd/IncludeFixer.cpp
+++ b/clang-tools-extra/clangd/IncludeFixer.cpp
@@ -484,7 +484,7 @@ collectAccessibleScopes(Sema &Sem, const DeclarationNameInfo &Typo, Scope *S,
                          /*IncludeGlobalScope=*/false,
                          /*LoadExternal=*/false);
   llvm::sort(Scopes);
-  Scopes.erase(std::unique(Scopes.begin(), Scopes.end()), Scopes.end());
+  Scopes.erase(llvm::unique(Scopes), Scopes.end());
   return Scopes;
 }
 
diff --git a/clang-tools-extra/clangd/InlayHints.cpp b/clang-tools-extra/clangd/InlayHints.cpp
index 1b1bcf78c9855..40a824618f782 100644
--- a/clang-tools-extra/clangd/InlayHints.cpp
+++ b/clang-tools-extra/clangd/InlayHints.cpp
@@ -1194,7 +1194,7 @@ std::vector<InlayHint> inlayHints(ParsedAST &AST,
   // De-duplicate hints. Duplicates can sometimes occur due to e.g. explicit
   // template instantiations.
   llvm::sort(Results);
-  Results.erase(std::unique(Results.begin(), Results.end()), Results.end());
+  Results.erase(llvm::unique(Results), Results.end());
 
   return Results;
 }
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index 86ca05644c703..dc574dcd11703 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -489,7 +489,7 @@ class HighlightingsBuilder {
     // Initializer lists can give duplicates of tokens, therefore all tokens
     // must be deduplicated.
     llvm::sort(Tokens);
-    auto Last = std::unique(Tokens.begin(), Tokens.end());
+    auto Last = llvm::unique(Tokens);
     Tokens.erase(Last, Tokens.end());
 
     // Macros can give tokens that have the same source range but conflicting
diff --git a/clang-tools-extra/clangd/SourceCode.cpp b/clang-tools-extra/clangd/SourceCode.cpp
index 780aaa471dc8b..1e1f7c68350ea 100644
--- a/clang-tools-extra/clangd/SourceCode.cpp
+++ b/clang-tools-extra/clangd/SourceCode.cpp
@@ -864,7 +864,7 @@ std::vector<std::string> visibleNamespaces(llvm::StringRef Code,
       return true;
     return LHS < RHS;
   });
-  Found.erase(std::unique(Found.begin(), Found.end()), Found.end());
+  Found.erase(llvm::unique(Found), Found.end());
   return Found;
 }
 
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index 8b9fffa3f64cd..053e2c044c774 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -909,13 +909,13 @@ class ReferenceFinder : public index::IndexDataConsumer {
       return std::tie(LTok, L.Role) < std::tie(RTok, R.Role);
     });
     // We sometimes see duplicates when parts of the AST get traversed twice.
-    References.erase(std::unique(References.begin(), References.end(),
-                                 [](const Reference &L, const Reference &R) {
-                                   auto LTok = L.SpelledTok.location();
-                                   auto RTok = R.SpelledTok.location();
-                                   return std::tie(LTok, L.Role) ==
-                                          std::tie(RTok, R.Role);
-                                 }),
+    References.erase(llvm::unique(References,
+                                  [](const Reference &L, const Reference &R) {
+                                    auto LTok = L.SpelledTok.location();
+                                    auto RTok = R.SpelledTok.location();
+                                    return std::tie(LTok, L.Role) ==
+                                           std::tie(RTok, R.Role);
+                                  }),
                      References.end());
     return std::move(References);
   }
@@ -1502,12 +1502,12 @@ ReferencesResult findReferences(ParsedAST &AST, Position Pos, uint32_t Limit,
     // We may get multiple refs with the same location and different Roles, as
     // cross-reference is only interested in locations, we deduplicate them
     // by the location to avoid emitting duplicated locations.
-    MainFileRefs.erase(std::unique(MainFileRefs.begin(), MainFileRefs.end(),
-                                   [](const ReferenceFinder::Reference &L,
-                                      const ReferenceFinder::Reference &R) {
-                                     return L.SpelledTok.location() ==
-                                            R.SpelledTok.location();
-                                   }),
+    MainFileRefs.erase(llvm::unique(MainFileRefs,
+                                    [](const ReferenceFinder::Reference &L,
+                                       const ReferenceFinder::Reference &R) {
+                                      return L.SpelledTok.location() ==
+                                             R.SpelledTok.location();
+                                    }),
                        MainFileRefs.end());
     for (const auto &Ref : MainFileRefs) {
       ReferencesResult::Reference Result;
diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp
index aa573e312a756..c8f354de3c00f 100644
--- a/clang-tools-extra/clangd/index/FileIndex.cpp
+++ b/clang-tools-extra/clangd/index/FileIndex.cpp
@@ -370,8 +370,7 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
   // relations being stored in both the shards containing their
   // subject and object.
   llvm::sort(AllRelations);
-  AllRelations.erase(std::unique(AllRelations.begin(), AllRelations.end()),
-                     AllRelations.end());
+  AllRelations.erase(llvm::unique(AllRelations), AllRelations.end());
 
   size_t StorageSize =
       RefsStorage.size() * sizeof(Ref) + SymsStorage.size() * sizeof(Symbol);
diff --git a/clang-tools-extra/clangd/index/Relation.cpp b/clang-tools-extra/clangd/index/Relation.cpp
index 2e5ae33939b0e..3b3e0ddc6d375 100644
--- a/clang-tools-extra/clangd/index/Relation.cpp
+++ b/clang-tools-extra/clangd/index/Relation.cpp
@@ -43,8 +43,7 @@ RelationSlab RelationSlab::Builder::build() && {
   llvm::sort(Relations);
 
   // Remove duplicates.
-  Relations.erase(std::unique(Relations.begin(), Relations.end()),
-                  Relations.end());
+  Relations.erase(llvm::unique(Relations), Relations.end());
 
   return RelationSlab{std::move(Relations)};
 }
diff --git a/clang-tools-extra/clangd/index/dex/Trigram.cpp b/clang-tools-extra/clangd/index/dex/Trigram.cpp
index c52af4e275a42..8a07086ff23e7 100644
--- a/clang-tools-extra/clangd/index/dex/Trigram.cpp
+++ b/clang-tools-extra/clangd/index/dex/Trigram.cpp
@@ -116,7 +116,7 @@ void generateIdentifierTrigrams(llvm::StringRef Identifier,
   } else {
     identifierTrigrams(Identifier, [&](Trigram T) { Result.push_back(T); });
     llvm::sort(Result);
-    Result.erase(std::unique(Result.begin(), Result.end()), Result.end());
+    Result.erase(llvm::unique(Result), Result.end());
   }
 }
 
diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp
index 26059167208aa..e464f1ad45c52 100644
--- a/clang-tools-extra/clangd/refactor/Rename.cpp
+++ b/clang-tools-extra/clangd/refactor/Rename.cpp
@@ -906,7 +906,7 @@ findOccurrencesOutsideFile(const NamedDecl &RenameDecl,
   for (auto &FileAndOccurrences : AffectedFiles) {
     auto &Ranges = FileAndOccurrences.getValue();
     llvm::sort(Ranges);
-    Ranges.erase(std::unique(Ranges.begin(), Ranges.end()), Ranges.end());
+    Ranges.erase(llvm::unique(Ranges), Ranges.end());
 
     SPAN_ATTACH(Tracer, FileAndOccurrences.first(),
                 static_cast<int64_t>(Ranges.size()));
@@ -1210,8 +1210,7 @@ llvm::Expected<Edit> buildRenameEdit(llvm::StringRef AbsFilePath,
               static_cast<int64_t>(Occurrences.size()));
 
   assert(llvm::is_sorted(Occurrences));
-  assert(std::unique(Occurrences.begin(), Occurrences.end()) ==
-             Occurrences.end() &&
+  assert(llvm::unique(Occurrences) == Occurrences.end() &&
          "Occurrences must be unique");
 
   // These two always correspond to the same position.
diff --git a/clang-tools-extra/clangd/refactor/tweaks/RemoveUsingNamespace.cpp b/clang-tools-extra/clangd/refactor/tweaks/RemoveUsingNamespace.cpp
index da32e00a0ee06..5baa970bcb0c5 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/RemoveUsingNamespace.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/RemoveUsingNamespace.cpp
@@ -196,9 +196,7 @@ Expected<Tweak::Effect> RemoveUsingNamespace::apply(const Selection &Inputs) {
   }
   // Remove duplicates.
   llvm::sort(IdentsToQualify);
-  IdentsToQualify.erase(
-      std::unique(IdentsToQualify.begin(), IdentsToQualify.end()),
-      IdentsToQualify.end());
+  IdentsToQualify.erase(llvm::unique(IdentsToQualify), IdentsToQualify.end());
 
   // Produce replacements to remove the using directives.
   tooling::Replacements R;

From ac8fc09688e10e983b99224b5dc5cbbeeedb1879 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgorny@gentoo.org>
Date: Sun, 20 Apr 2025 06:16:19 +0200
Subject: [PATCH 573/710] [offload] Unset `-march` when building GPU libraries
 (#136442)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Unset `-march` when invoking the compiler and linker to build the GPU
libraries. These libraries use GPU targets rather than the CPU targets,
and an incidental `-march=native` causes Clang to be able to determine
the GPU used — which causes the build to fail when there is no GPU
available. Resetting `-march=` should suffice to revert to building
generic code for the time being.

See the discussion in:
https://github.com/llvm/llvm-project/pull/126143#issuecomment-2816718492
---
 offload/DeviceRTL/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index 2da2ade94f4e4..346b65c887bba 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -137,9 +137,9 @@ function(compileDeviceRTLLibrary target_name target_triple)
     BUILD_RPATH ""
     INSTALL_RPATH ""
     RUNTIME_OUTPUT_NAME libomptarget-${target_name}.bc)
-  target_compile_options(libomptarget-${target_name} PRIVATE "--target=${target_triple}")
+  target_compile_options(libomptarget-${target_name} PRIVATE "--target=${target_triple}" "-march=")
   target_link_options(libomptarget-${target_name} PRIVATE "--target=${target_triple}"
-                      "-r" "-nostdlib" "-flto" "-Wl,--lto-emit-llvm")
+                      "-r" "-nostdlib" "-flto" "-Wl,--lto-emit-llvm" "-march=")
   install(TARGETS libomptarget-${target_name}
           PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ
           DESTINATION ${OFFLOAD_INSTALL_LIBDIR})

From cd84ba2e4d554242e5ddea1d130018bbd701f8af Mon Sep 17 00:00:00 2001
From: Prabhu Rajasekaran <prabhukr@google.com>
Date: Sat, 19 Apr 2025 22:11:06 -0700
Subject: [PATCH 574/710] [llvm] Use masm for UEFI (#132461)

For UEFI targets use Microsoft masm.
---
 llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index 3b5869c19f2d3..ceea417dc32e8 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -444,7 +444,7 @@ static MCAsmInfo *createX86MCAsmInfo(const MCRegisterInfo &MRI,
     // Force the use of an ELF container.
     MAI = new X86ELFMCAsmInfo(TheTriple);
   } else if (TheTriple.isWindowsMSVCEnvironment() ||
-             TheTriple.isWindowsCoreCLREnvironment()) {
+             TheTriple.isWindowsCoreCLREnvironment() || TheTriple.isUEFI()) {
     if (Options.getAssemblyLanguage().equals_insensitive("masm"))
       MAI = new X86MCAsmInfoMicrosoftMASM(TheTriple);
     else

From f88fd89909deaf7a174544b02c973a3c2ada02d6 Mon Sep 17 00:00:00 2001
From: Jim Lin <jim@andestech.com>
Date: Sat, 19 Apr 2025 15:25:39 +0800
Subject: [PATCH 575/710] [RISCV] Extract tryUnsignedBitfieldExtract as a
 member function of RISCVDAGToDAGISel. NFC.

---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 39 ++++++++++-----------
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h   |  2 ++
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 2d07a66ff275e..093e8b5966fee 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -663,6 +663,20 @@ bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
   return false;
 }
 
+bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node, SDLoc DL,
+                                                   MVT VT, SDValue X,
+                                                   unsigned Msb, unsigned Lsb) {
+  // Only supported with XTHeadBb at the moment.
+  if (!Subtarget->hasVendorXTHeadBb())
+    return false;
+
+  SDNode *TH_EXTU = CurDAG->getMachineNode(
+      RISCV::TH_EXTU, DL, VT, X, CurDAG->getTargetConstant(Msb, DL, VT),
+      CurDAG->getTargetConstant(Lsb, DL, VT));
+  ReplaceNode(Node, TH_EXTU);
+  return true;
+}
+
 bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
   // Target does not support indexed loads.
   if (!Subtarget->hasVendorXTHeadMemIdx())
@@ -1122,16 +1136,12 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       return;
     }
 
-    unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
-    if (Subtarget->hasVendorXTHeadBb()) {
-      SDNode *THEXTU = CurDAG->getMachineNode(
-          RISCV::TH_EXTU, DL, VT, N0->getOperand(0),
-          CurDAG->getTargetConstant(TrailingOnes - 1, DL, VT),
-          CurDAG->getTargetConstant(ShAmt, DL, VT));
-      ReplaceNode(Node, THEXTU);
+    const unsigned Msb = TrailingOnes - 1;
+    const unsigned Lsb = ShAmt;
+    if (tryUnsignedBitfieldExtract(Node, DL, VT, N0->getOperand(0), Msb, Lsb))
       return;
-    }
 
+    unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
     SDNode *SLLI =
         CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
                                CurDAG->getTargetConstant(LShAmt, DL, VT));
@@ -1188,19 +1198,6 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
 
     SDValue N0 = Node->getOperand(0);
 
-    auto tryUnsignedBitfieldExtract = [&](SDNode *Node, SDLoc DL, MVT VT,
-                                          SDValue X, unsigned Msb,
-                                          unsigned Lsb) {
-      if (!Subtarget->hasVendorXTHeadBb())
-        return false;
-
-      SDNode *TH_EXTU = CurDAG->getMachineNode(
-          RISCV::TH_EXTU, DL, VT, X, CurDAG->getTargetConstant(Msb, DL, VT),
-          CurDAG->getTargetConstant(Lsb, DL, VT));
-      ReplaceNode(Node, TH_EXTU);
-      return true;
-    };
-
     bool LeftShift = N0.getOpcode() == ISD::SHL;
     if (LeftShift || N0.getOpcode() == ISD::SRL) {
       auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index db09ad146b655..82a47a9a52501 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -81,6 +81,8 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
 
   bool tryShrinkShlLogicImm(SDNode *Node);
   bool trySignedBitfieldExtract(SDNode *Node);
+  bool tryUnsignedBitfieldExtract(SDNode *Node, SDLoc DL, MVT VT, SDValue X,
+                                  unsigned Msb, unsigned Lsb);
   bool tryIndexedLoad(SDNode *Node);
 
   bool selectShiftMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt);

From 71037ee9decd05ec07617e5ee77a05e0114b9b67 Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg@google.com>
Date: Sun, 20 Apr 2025 07:55:07 +0200
Subject: [PATCH 576/710] [llvm][bazel] Port
 667209e45122d0cb7a4c2ac27018d31165b1be70.

---
 .../llvm-project-overlay/llvm/BUILD.bazel     |  1 +
 .../llvm/include/llvm/Config/Targets.h        | 11 +++
 utils/bazel/llvm_configs/llvm-config.h.cmake  | 75 -------------------
 3 files changed, 12 insertions(+), 75 deletions(-)
 create mode 100644 utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/Targets.h

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index b77ddf634eec6..d2cb3413e88b9 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -174,6 +174,7 @@ llvm_config_target_defines = [
 cc_library(
     name = "config",
     hdrs = [
+        "include/llvm/Config/Targets.h",
         "include/llvm/Config/abi-breaking.h",
         "include/llvm/Config/llvm-config.h",
     ],
diff --git a/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/Targets.h b/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/Targets.h
new file mode 100644
index 0000000000000..185c12f969d6d
--- /dev/null
+++ b/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/Targets.h
@@ -0,0 +1,11 @@
+/*===----- llvm/Config/Targets.h - llvm targets configuration -----*- C -*-===*/
+/*                                                                            */
+/* Part of the LLVM Project, under the Apache License v2.0 with LLVM          */
+/* Exceptions.                                                                */
+/* See https://llvm.org/LICENSE.txt for license information.                  */
+/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    */
+/*                                                                            */
+/*===----------------------------------------------------------------------===*/
+
+// Bazel builds provides target #defines through @llvm-project//llvm:config.
+// This file is therefore empty, but it exists so it can be #included.
diff --git a/utils/bazel/llvm_configs/llvm-config.h.cmake b/utils/bazel/llvm_configs/llvm-config.h.cmake
index 4230eb850fa4b..97f996338bc0c 100644
--- a/utils/bazel/llvm_configs/llvm-config.h.cmake
+++ b/utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -54,81 +54,6 @@
 /* LLVM name for the native target MCA init function, if available */
 #cmakedefine LLVM_NATIVE_TARGETMCA LLVMInitialize${LLVM_NATIVE_ARCH}TargetMCA
 
-/* Define if the AArch64 target is built in */
-#cmakedefine01 LLVM_HAS_AARCH64_TARGET
-
-/* Define if the AMDGPU target is built in */
-#cmakedefine01 LLVM_HAS_AMDGPU_TARGET
-
-/* Define if the ARC target is built in */
-#cmakedefine01 LLVM_HAS_ARC_TARGET
-
-/* Define if the ARM target is built in */
-#cmakedefine01 LLVM_HAS_ARM_TARGET
-
-/* Define if the AVR target is built in */
-#cmakedefine01 LLVM_HAS_AVR_TARGET
-
-/* Define if the BPF target is built in */
-#cmakedefine01 LLVM_HAS_BPF_TARGET
-
-/* Define if the CSKY target is built in */
-#cmakedefine01 LLVM_HAS_CSKY_TARGET
-
-/* Define if the DirectX target is built in */
-#cmakedefine01 LLVM_HAS_DIRECTX_TARGET
-
-/* Define if the Hexagon target is built in */
-#cmakedefine01 LLVM_HAS_HEXAGON_TARGET
-
-/* Define if the Lanai target is built in */
-#cmakedefine01 LLVM_HAS_LANAI_TARGET
-
-/* Define if the LoongArch target is built in */
-#cmakedefine01 LLVM_HAS_LOONGARCH_TARGET
-
-/* Define if the M68k target is built in */
-#cmakedefine01 LLVM_HAS_M68K_TARGET
-
-/* Define if the Mips target is built in */
-#cmakedefine01 LLVM_HAS_MIPS_TARGET
-
-/* Define if the MSP430 target is built in */
-#cmakedefine01 LLVM_HAS_MSP430_TARGET
-
-/* Define if the NVPTX target is built in */
-#cmakedefine01 LLVM_HAS_NVPTX_TARGET
-
-/* Define if the PowerPC target is built in */
-#cmakedefine01 LLVM_HAS_POWERPC_TARGET
-
-/* Define if the RISCV target is built in */
-#cmakedefine01 LLVM_HAS_RISCV_TARGET
-
-/* Define if the Sparc target is built in */
-#cmakedefine01 LLVM_HAS_SPARC_TARGET
-
-/* Define if the SPIRV target is built in */
-#cmakedefine01 LLVM_HAS_SPIRV_TARGET
-
-/* Define if the SystemZ target is built in */
-#cmakedefine01 LLVM_HAS_SYSTEMZ_TARGET
-
-/* Define if the VE target is built in */
-#cmakedefine01 LLVM_HAS_VE_TARGET
-
-/* Define if the WebAssembly target is built in */
-#cmakedefine01 LLVM_HAS_WEBASSEMBLY_TARGET
-
-/* Define if the X86 target is built in */
-#cmakedefine01 LLVM_HAS_X86_TARGET
-
-/* Define if the XCore target is built in */
-#cmakedefine01 LLVM_HAS_XCORE_TARGET
-
-/* Define if the Xtensa target is built in */
-#cmakedefine01 LLVM_HAS_XTENSA_TARGET
-
 /* Define if this is Unixish platform */
 #cmakedefine LLVM_ON_UNIX ${LLVM_ON_UNIX}
 

From 8553efd2e98575fbee2c81a334ec28b12e055406 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Sun, 20 Apr 2025 10:06:22 +0200
Subject: [PATCH 577/710] [mlir][LLVM] Add `OpBuilder &` to `lookupOrCreateFn`
 functions (#136421)

These functions are called from lowering patterns. All IR modifications
in a pattern must be performed through the provided rewriter, but these
functions used to instantiate a new `OpBuilder`, bypassing the provided
rewriter.
---
 .../mlir/Dialect/LLVMIR/FunctionCallUtils.h   | 57 +++++++-----
 .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp    |  4 +-
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    |  4 +-
 .../Conversion/LLVMCommon/PrintCallHelper.cpp |  2 +-
 .../MemRefToLLVM/AllocLikeConversion.cpp      | 24 ++---
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  | 14 +--
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      | 22 ++---
 .../Dialect/LLVMIR/IR/FunctionCallUtils.cpp   | 91 ++++++++++---------
 8 files changed, 120 insertions(+), 98 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
index 05e9fe9d58859..4a7ec6f2efe64 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h
@@ -33,40 +33,53 @@ class LLVMFuncOp;
 /// implemented separately (e.g. as part of a support runtime library or as part
 /// of the libc).
 /// Failure if an unexpected version of function is found.
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintI64Fn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintU64Fn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF16Fn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintBF16Fn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF32Fn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF64Fn(Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintI64Fn(OpBuilder &b,
+                                                     Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintU64Fn(OpBuilder &b,
+                                                     Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF16Fn(OpBuilder &b,
+                                                     Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintBF16Fn(OpBuilder &b,
+                                                      Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF32Fn(OpBuilder &b,
+                                                     Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintF64Fn(OpBuilder &b,
+                                                     Operation *moduleOp);
 /// Declares a function to print a C-string.
 /// If a custom runtime function is defined via `runtimeFunctionName`, it must
 /// have the signature void(char const*). The default function is `printString`.
 FailureOr<LLVM::LLVMFuncOp>
-lookupOrCreatePrintStringFn(Operation *moduleOp,
+lookupOrCreatePrintStringFn(OpBuilder &b, Operation *moduleOp,
                             std::optional<StringRef> runtimeFunctionName = {});
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintOpenFn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintCloseFn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintCommaFn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintNewlineFn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreateMallocFn(Operation *moduleOp,
-                                                   Type indexType);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreateAlignedAllocFn(Operation *moduleOp,
-                                                         Type indexType);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreateFreeFn(Operation *moduleOp);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreateGenericAllocFn(Operation *moduleOp,
-                                                         Type indexType);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintOpenFn(OpBuilder &b,
+                                                      Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintCloseFn(OpBuilder &b,
+                                                       Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintCommaFn(OpBuilder &b,
+                                                       Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreatePrintNewlineFn(OpBuilder &b,
+                                                         Operation *moduleOp);
 FailureOr<LLVM::LLVMFuncOp>
-lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp, Type indexType);
-FailureOr<LLVM::LLVMFuncOp> lookupOrCreateGenericFreeFn(Operation *moduleOp);
+lookupOrCreateMallocFn(OpBuilder &b, Operation *moduleOp, Type indexType);
 FailureOr<LLVM::LLVMFuncOp>
-lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType,
+lookupOrCreateAlignedAllocFn(OpBuilder &b, Operation *moduleOp, Type indexType);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreateFreeFn(OpBuilder &b,
+                                                 Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp>
+lookupOrCreateGenericAllocFn(OpBuilder &b, Operation *moduleOp, Type indexType);
+FailureOr<LLVM::LLVMFuncOp>
+lookupOrCreateGenericAlignedAllocFn(OpBuilder &b, Operation *moduleOp,
+                                    Type indexType);
+FailureOr<LLVM::LLVMFuncOp> lookupOrCreateGenericFreeFn(OpBuilder &b,
+                                                        Operation *moduleOp);
+FailureOr<LLVM::LLVMFuncOp>
+lookupOrCreateMemRefCopyFn(OpBuilder &b, Operation *moduleOp, Type indexType,
                            Type unrankedDescriptorType);
 
 /// Create a FuncOp with signature `resultType`(`paramTypes`)` and name `name`.
 /// Return a failure if the FuncOp found has unexpected signature.
 FailureOr<LLVM::LLVMFuncOp>
-lookupOrCreateFn(Operation *moduleOp, StringRef name,
+lookupOrCreateFn(OpBuilder &b, Operation *moduleOp, StringRef name,
                  ArrayRef<Type> paramTypes = {}, Type resultType = {},
                  bool isVarArg = false, bool isReserved = false);
 
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index 47d4474a5c28d..c95e375ce9afe 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -395,7 +395,7 @@ class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
 
     // Allocate memory for the coroutine frame.
     auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn(
-        op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
+        rewriter, op->getParentOfType<ModuleOp>(), rewriter.getI64Type());
     if (failed(allocFuncOp))
       return failure();
     auto coroAlloc = rewriter.create<LLVM::CallOp>(
@@ -432,7 +432,7 @@ class CoroFreeOpConversion : public AsyncOpConversionPattern<CoroFreeOp> {
 
     // Free the memory.
     auto freeFuncOp =
-        LLVM::lookupOrCreateFreeFn(op->getParentOfType<ModuleOp>());
+        LLVM::lookupOrCreateFreeFn(rewriter, op->getParentOfType<ModuleOp>());
     if (failed(freeFuncOp))
       return failure();
     rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, freeFuncOp.value(),
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index 1ae99561e9d1b..0505214de2015 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -278,12 +278,12 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
   auto module = builder.getInsertionPoint()->getParentOfType<ModuleOp>();
   FailureOr<LLVM::LLVMFuncOp> freeFunc, mallocFunc;
   if (toDynamic) {
-    mallocFunc = LLVM::lookupOrCreateMallocFn(module, indexType);
+    mallocFunc = LLVM::lookupOrCreateMallocFn(builder, module, indexType);
     if (failed(mallocFunc))
       return failure();
   }
   if (!toDynamic) {
-    freeFunc = LLVM::lookupOrCreateFreeFn(module);
+    freeFunc = LLVM::lookupOrCreateFreeFn(builder, module);
     if (failed(freeFunc))
       return failure();
   }
diff --git a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
index 337c01f01a7cc..2815e05b3e11b 100644
--- a/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/PrintCallHelper.cpp
@@ -60,7 +60,7 @@ LogicalResult mlir::LLVM::createPrintStrCall(
   Value gep =
       builder.create<LLVM::GEPOp>(loc, ptrTy, arrayTy, msgAddr, indices);
   FailureOr<LLVM::LLVMFuncOp> printer =
-      LLVM::lookupOrCreatePrintStringFn(moduleOp, runtimeFunctionName);
+      LLVM::lookupOrCreatePrintStringFn(builder, moduleOp, runtimeFunctionName);
   if (failed(printer))
     return failure();
   builder.create<LLVM::CallOp>(loc, TypeRange(),
diff --git a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
index bad209a4ddecf..e9b79983696aa 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
@@ -15,24 +15,24 @@
 using namespace mlir;
 
 static FailureOr<LLVM::LLVMFuncOp>
-getNotalignedAllocFn(const LLVMTypeConverter *typeConverter, Operation *module,
-                     Type indexType) {
+getNotalignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
+                     Operation *module, Type indexType) {
   bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
   if (useGenericFn)
-    return LLVM::lookupOrCreateGenericAllocFn(module, indexType);
+    return LLVM::lookupOrCreateGenericAllocFn(b, module, indexType);
 
-  return LLVM::lookupOrCreateMallocFn(module, indexType);
+  return LLVM::lookupOrCreateMallocFn(b, module, indexType);
 }
 
 static FailureOr<LLVM::LLVMFuncOp>
-getAlignedAllocFn(const LLVMTypeConverter *typeConverter, Operation *module,
-                  Type indexType) {
+getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
+                  Operation *module, Type indexType) {
   bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
 
   if (useGenericFn)
-    return LLVM::lookupOrCreateGenericAlignedAllocFn(module, indexType);
+    return LLVM::lookupOrCreateGenericAlignedAllocFn(b, module, indexType);
 
-  return LLVM::lookupOrCreateAlignedAllocFn(module, indexType);
+  return LLVM::lookupOrCreateAlignedAllocFn(b, module, indexType);
 }
 
 Value AllocationOpLLVMLowering::createAligned(
@@ -75,8 +75,8 @@ std::tuple<Value, Value> AllocationOpLLVMLowering::allocateBufferManuallyAlign(
   Type elementPtrType = this->getElementPtrType(memRefType);
   assert(elementPtrType && "could not compute element ptr type");
   FailureOr<LLVM::LLVMFuncOp> allocFuncOp = getNotalignedAllocFn(
-      getTypeConverter(), op->getParentWithTrait<OpTrait::SymbolTable>(),
-      getIndexType());
+      rewriter, getTypeConverter(),
+      op->getParentWithTrait<OpTrait::SymbolTable>(), getIndexType());
   if (failed(allocFuncOp))
     return std::make_tuple(Value(), Value());
   auto results =
@@ -144,8 +144,8 @@ Value AllocationOpLLVMLowering::allocateBufferAutoAlign(
 
   Type elementPtrType = this->getElementPtrType(memRefType);
   FailureOr<LLVM::LLVMFuncOp> allocFuncOp = getAlignedAllocFn(
-      getTypeConverter(), op->getParentWithTrait<OpTrait::SymbolTable>(),
-      getIndexType());
+      rewriter, getTypeConverter(),
+      op->getParentWithTrait<OpTrait::SymbolTable>(), getIndexType());
   if (failed(allocFuncOp))
     return Value();
   auto results = rewriter.create<LLVM::CallOp>(
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index cb4317ef1bcec..9c219d8a3d8cb 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -43,13 +43,14 @@ static bool isStaticStrideOrOffset(int64_t strideOrOffset) {
 }
 
 static FailureOr<LLVM::LLVMFuncOp>
-getFreeFn(const LLVMTypeConverter *typeConverter, ModuleOp module) {
+getFreeFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
+          ModuleOp module) {
   bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
 
   if (useGenericFn)
-    return LLVM::lookupOrCreateGenericFreeFn(module);
+    return LLVM::lookupOrCreateGenericFreeFn(b, module);
 
-  return LLVM::lookupOrCreateFreeFn(module);
+  return LLVM::lookupOrCreateFreeFn(b, module);
 }
 
 struct AllocOpLowering : public AllocLikeOpLLVMLowering {
@@ -223,8 +224,8 @@ struct DeallocOpLowering : public ConvertOpToLLVMPattern<memref::DeallocOp> {
   matchAndRewrite(memref::DeallocOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Insert the `free` declaration if it is not already present.
-    FailureOr<LLVM::LLVMFuncOp> freeFunc =
-        getFreeFn(getTypeConverter(), op->getParentOfType<ModuleOp>());
+    FailureOr<LLVM::LLVMFuncOp> freeFunc = getFreeFn(
+        rewriter, getTypeConverter(), op->getParentOfType<ModuleOp>());
     if (failed(freeFunc))
       return failure();
     Value allocatedPtr;
@@ -834,7 +835,8 @@ struct MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     // potential alignment
     auto elemSize = getSizeInBytes(loc, srcType.getElementType(), rewriter);
     auto copyFn = LLVM::lookupOrCreateMemRefCopyFn(
-        op->getParentOfType<ModuleOp>(), getIndexType(), sourcePtr.getType());
+        rewriter, op->getParentOfType<ModuleOp>(), getIndexType(),
+        sourcePtr.getType());
     if (failed(copyFn))
       return failure();
     rewriter.create<LLVM::CallOp>(loc, copyFn.value(),
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 1a35d08196459..076e5512f375b 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -1570,13 +1570,13 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
       FailureOr<LLVM::LLVMFuncOp> op = [&]() {
         switch (punct) {
         case PrintPunctuation::Close:
-          return LLVM::lookupOrCreatePrintCloseFn(parent);
+          return LLVM::lookupOrCreatePrintCloseFn(rewriter, parent);
         case PrintPunctuation::Open:
-          return LLVM::lookupOrCreatePrintOpenFn(parent);
+          return LLVM::lookupOrCreatePrintOpenFn(rewriter, parent);
         case PrintPunctuation::Comma:
-          return LLVM::lookupOrCreatePrintCommaFn(parent);
+          return LLVM::lookupOrCreatePrintCommaFn(rewriter, parent);
         case PrintPunctuation::NewLine:
-          return LLVM::lookupOrCreatePrintNewlineFn(parent);
+          return LLVM::lookupOrCreatePrintNewlineFn(rewriter, parent);
         default:
           llvm_unreachable("unexpected punctuation");
         }
@@ -1610,17 +1610,17 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
     PrintConversion conversion = PrintConversion::None;
     FailureOr<Operation *> printer;
     if (printType.isF32()) {
-      printer = LLVM::lookupOrCreatePrintF32Fn(parent);
+      printer = LLVM::lookupOrCreatePrintF32Fn(rewriter, parent);
     } else if (printType.isF64()) {
-      printer = LLVM::lookupOrCreatePrintF64Fn(parent);
+      printer = LLVM::lookupOrCreatePrintF64Fn(rewriter, parent);
     } else if (printType.isF16()) {
       conversion = PrintConversion::Bitcast16; // bits!
-      printer = LLVM::lookupOrCreatePrintF16Fn(parent);
+      printer = LLVM::lookupOrCreatePrintF16Fn(rewriter, parent);
     } else if (printType.isBF16()) {
       conversion = PrintConversion::Bitcast16; // bits!
-      printer = LLVM::lookupOrCreatePrintBF16Fn(parent);
+      printer = LLVM::lookupOrCreatePrintBF16Fn(rewriter, parent);
     } else if (printType.isIndex()) {
-      printer = LLVM::lookupOrCreatePrintU64Fn(parent);
+      printer = LLVM::lookupOrCreatePrintU64Fn(rewriter, parent);
     } else if (auto intTy = dyn_cast<IntegerType>(printType)) {
       // Integers need a zero or sign extension on the operand
       // (depending on the source type) as well as a signed or
@@ -1630,7 +1630,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
         if (width <= 64) {
           if (width < 64)
             conversion = PrintConversion::ZeroExt64;
-          printer = LLVM::lookupOrCreatePrintU64Fn(parent);
+          printer = LLVM::lookupOrCreatePrintU64Fn(rewriter, parent);
         } else {
           return failure();
         }
@@ -1643,7 +1643,7 @@ class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
             conversion = PrintConversion::ZeroExt64;
           else if (width < 64)
             conversion = PrintConversion::SignExt64;
-          printer = LLVM::lookupOrCreatePrintI64Fn(parent);
+          printer = LLVM::lookupOrCreatePrintI64Fn(rewriter, parent);
         } else {
           return failure();
         }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
index 68d4426e65301..1b4a8f496d3d0 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp
@@ -46,7 +46,7 @@ static constexpr llvm::StringRef kMemRefCopy = "memrefCopy";
 
 /// Generic print function lookupOrCreate helper.
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreateFn(Operation *moduleOp, StringRef name,
+mlir::LLVM::lookupOrCreateFn(OpBuilder &b, Operation *moduleOp, StringRef name,
                              ArrayRef<Type> paramTypes, Type resultType,
                              bool isVarArg, bool isReserved) {
   assert(moduleOp->hasTrait<OpTrait::SymbolTable>() &&
@@ -69,60 +69,63 @@ mlir::LLVM::lookupOrCreateFn(Operation *moduleOp, StringRef name,
     }
     return func;
   }
-  OpBuilder b(moduleOp->getRegion(0));
+
+  OpBuilder::InsertionGuard g(b);
+  assert(!moduleOp->getRegion(0).empty() && "expected non-empty region");
+  b.setInsertionPointToStart(&moduleOp->getRegion(0).front());
   return b.create<LLVM::LLVMFuncOp>(
       moduleOp->getLoc(), name,
       LLVM::LLVMFunctionType::get(resultType, paramTypes, isVarArg));
 }
 
 static FailureOr<LLVM::LLVMFuncOp>
-lookupOrCreateReservedFn(Operation *moduleOp, StringRef name,
+lookupOrCreateReservedFn(OpBuilder &b, Operation *moduleOp, StringRef name,
                          ArrayRef<Type> paramTypes, Type resultType) {
-  return lookupOrCreateFn(moduleOp, name, paramTypes, resultType,
+  return lookupOrCreateFn(b, moduleOp, name, paramTypes, resultType,
                           /*isVarArg=*/false, /*isReserved=*/true);
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintI64Fn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintI64Fn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintI64, IntegerType::get(moduleOp->getContext(), 64),
+      b, moduleOp, kPrintI64, IntegerType::get(moduleOp->getContext(), 64),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintU64Fn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintU64Fn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintU64, IntegerType::get(moduleOp->getContext(), 64),
+      b, moduleOp, kPrintU64, IntegerType::get(moduleOp->getContext(), 64),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintF16Fn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintF16Fn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintF16,
+      b, moduleOp, kPrintF16,
       IntegerType::get(moduleOp->getContext(), 16), // bits!
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintBF16Fn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintBF16Fn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintBF16,
+      b, moduleOp, kPrintBF16,
       IntegerType::get(moduleOp->getContext(), 16), // bits!
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintF32Fn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintF32Fn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintF32, Float32Type::get(moduleOp->getContext()),
+      b, moduleOp, kPrintF32, Float32Type::get(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintF64Fn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintF64Fn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintF64, Float64Type::get(moduleOp->getContext()),
+      b, moduleOp, kPrintF64, Float64Type::get(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
@@ -136,87 +139,91 @@ static LLVM::LLVMPointerType getVoidPtr(MLIRContext *context) {
 }
 
 FailureOr<LLVM::LLVMFuncOp> mlir::LLVM::lookupOrCreatePrintStringFn(
-    Operation *moduleOp, std::optional<StringRef> runtimeFunctionName) {
+    OpBuilder &b, Operation *moduleOp,
+    std::optional<StringRef> runtimeFunctionName) {
   return lookupOrCreateReservedFn(
-      moduleOp, runtimeFunctionName.value_or(kPrintString),
+      b, moduleOp, runtimeFunctionName.value_or(kPrintString),
       getCharPtr(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintOpenFn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintOpenFn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintOpen, {},
+      b, moduleOp, kPrintOpen, {},
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintCloseFn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintCloseFn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintClose, {},
+      b, moduleOp, kPrintClose, {},
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintCommaFn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintCommaFn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintComma, {},
+      b, moduleOp, kPrintComma, {},
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreatePrintNewlineFn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreatePrintNewlineFn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kPrintNewline, {},
+      b, moduleOp, kPrintNewline, {},
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreateMallocFn(Operation *moduleOp, Type indexType) {
-  return lookupOrCreateReservedFn(moduleOp, kMalloc, indexType,
+mlir::LLVM::lookupOrCreateMallocFn(OpBuilder &b, Operation *moduleOp,
+                                   Type indexType) {
+  return lookupOrCreateReservedFn(b, moduleOp, kMalloc, indexType,
                                   getVoidPtr(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreateAlignedAllocFn(Operation *moduleOp, Type indexType) {
-  return lookupOrCreateReservedFn(moduleOp, kAlignedAlloc,
+mlir::LLVM::lookupOrCreateAlignedAllocFn(OpBuilder &b, Operation *moduleOp,
+                                         Type indexType) {
+  return lookupOrCreateReservedFn(b, moduleOp, kAlignedAlloc,
                                   {indexType, indexType},
                                   getVoidPtr(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreateFreeFn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreateFreeFn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kFree, getVoidPtr(moduleOp->getContext()),
+      b, moduleOp, kFree, getVoidPtr(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreateGenericAllocFn(Operation *moduleOp, Type indexType) {
-  return lookupOrCreateReservedFn(moduleOp, kGenericAlloc, indexType,
+mlir::LLVM::lookupOrCreateGenericAllocFn(OpBuilder &b, Operation *moduleOp,
+                                         Type indexType) {
+  return lookupOrCreateReservedFn(b, moduleOp, kGenericAlloc, indexType,
                                   getVoidPtr(moduleOp->getContext()));
 }
 
-FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(Operation *moduleOp,
-                                                Type indexType) {
-  return lookupOrCreateReservedFn(moduleOp, kGenericAlignedAlloc,
+FailureOr<LLVM::LLVMFuncOp> mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(
+    OpBuilder &b, Operation *moduleOp, Type indexType) {
+  return lookupOrCreateReservedFn(b, moduleOp, kGenericAlignedAlloc,
                                   {indexType, indexType},
                                   getVoidPtr(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreateGenericFreeFn(Operation *moduleOp) {
+mlir::LLVM::lookupOrCreateGenericFreeFn(OpBuilder &b, Operation *moduleOp) {
   return lookupOrCreateReservedFn(
-      moduleOp, kGenericFree, getVoidPtr(moduleOp->getContext()),
+      b, moduleOp, kGenericFree, getVoidPtr(moduleOp->getContext()),
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }
 
 FailureOr<LLVM::LLVMFuncOp>
-mlir::LLVM::lookupOrCreateMemRefCopyFn(Operation *moduleOp, Type indexType,
+mlir::LLVM::lookupOrCreateMemRefCopyFn(OpBuilder &b, Operation *moduleOp,
+                                       Type indexType,
                                        Type unrankedDescriptorType) {
   return lookupOrCreateReservedFn(
-      moduleOp, kMemRefCopy,
+      b, moduleOp, kMemRefCopy,
       ArrayRef<Type>{indexType, unrankedDescriptorType, unrankedDescriptorType},
       LLVM::LLVMVoidType::get(moduleOp->getContext()));
 }

From 366ff3a89880139a132fe2738f36b39c89f5333e Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 20 Apr 2025 10:36:55 +0100
Subject: [PATCH 578/710] [VPlan] Get address space from inferred pointer type
 (NFC)

Remove a use of the underlying ingredient by getting the address space
from the inferred pointer type.
---
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2cc558f49ccce..ff1ad29e75e2d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2684,8 +2684,8 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
   Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
   const Align Alignment =
       getLoadStoreAlignment(const_cast<Instruction *>(&Ingredient));
-  unsigned AS =
-      getLoadStoreAddressSpace(const_cast<Instruction *>(&Ingredient));
+  unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
+                    ->getAddressSpace();
 
   if (!Consecutive) {
     // TODO: Using the original IR may not be accurate.

From 90c845fb3babac387688dfa6d560d3ba8ed8e340 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sun, 20 Apr 2025 12:20:21 +0200
Subject: [PATCH 579/710] [clang][bytecode] Start implementing
 __builtin_{,dynamic}_object_size (#136478)

---
 clang/lib/AST/ByteCode/Descriptor.h           |  2 +-
 clang/lib/AST/ByteCode/InterpBuiltin.cpp      | 52 +++++++++++++++++++
 .../test/AST/ByteCode/builtin-object-size.cpp | 14 +++++
 3 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/AST/ByteCode/builtin-object-size.cpp

diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index a0705cc8c377f..532b407c2c788 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -265,7 +265,7 @@ struct Descriptor final {
   bool isUnknownSizeArray() const { return Size == UnknownSizeMark; }
 
   /// Checks if the descriptor is of a primitive.
-  bool isPrimitive() const { return !IsArray && !ElemRecord && !IsDummy; }
+  bool isPrimitive() const { return !IsArray && !ElemRecord && PrimT; }
 
   /// Checks if the descriptor is of an array.
   bool isArray() const { return IsArray; }
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 34553301ef630..aaf0f3f019b94 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -2179,6 +2179,52 @@ static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static unsigned computeFullDescSize(const ASTContext &ASTCtx,
+                                    const Descriptor *Desc) {
+
+  if (Desc->isPrimitive())
+    return ASTCtx.getTypeSizeInChars(Desc->getType()).getQuantity();
+
+  if (Desc->isArray())
+    return ASTCtx.getTypeSizeInChars(Desc->getElemQualType()).getQuantity() *
+           Desc->getNumElems();
+
+  if (Desc->isRecord())
+    return ASTCtx.getTypeSizeInChars(Desc->getType()).getQuantity();
+
+  llvm_unreachable("Unhandled descriptor type");
+  return 0;
+}
+
+static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
+                                        const InterpFrame *Frame,
+                                        const Function *Func,
+                                        const CallExpr *Call) {
+  PrimType KindT = *S.getContext().classify(Call->getArg(1));
+  [[maybe_unused]] unsigned Kind = peekToAPSInt(S.Stk, KindT).getZExtValue();
+
+  assert(Kind <= 3 && "unexpected kind");
+
+  const Pointer &Ptr =
+      S.Stk.peek<Pointer>(align(primSize(KindT)) + align(primSize(PT_Ptr)));
+
+  if (Ptr.isZero())
+    return false;
+
+  const Descriptor *DeclDesc = Ptr.getDeclDesc();
+  if (!DeclDesc)
+    return false;
+
+  const ASTContext &ASTCtx = S.getASTContext();
+
+  unsigned ByteOffset = 0;
+  unsigned FullSize = computeFullDescSize(ASTCtx, DeclDesc);
+
+  pushInteger(S, FullSize - ByteOffset, Call->getType());
+
+  return true;
+}
+
 bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
                       const CallExpr *Call, uint32_t BuiltinID) {
   if (!S.getASTContext().BuiltinInfo.isConstantEvaluated(BuiltinID))
@@ -2681,6 +2727,12 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
       return false;
     break;
 
+  case Builtin::BI__builtin_object_size:
+  case Builtin::BI__builtin_dynamic_object_size:
+    if (!interp__builtin_object_size(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+
   default:
     S.FFDiag(S.Current->getLocation(OpPC),
              diag::note_invalid_subexpr_in_const_expr)
diff --git a/clang/test/AST/ByteCode/builtin-object-size.cpp b/clang/test/AST/ByteCode/builtin-object-size.cpp
new file mode 100644
index 0000000000000..62bb1642c5301
--- /dev/null
+++ b/clang/test/AST/ByteCode/builtin-object-size.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=both,expected %s
+// RUN: %clang_cc1                                         -verify=both,ref      %s
+
+// both-no-diagnostics
+
+int a;
+static_assert(__builtin_object_size(&a, 0) == sizeof(int), "");
+float f;
+static_assert(__builtin_object_size(&f, 0) == sizeof(float), "");
+int arr[2];
+static_assert(__builtin_object_size(&arr, 0) == (sizeof(int)*2), "");
+
+float arrf[2];
+static_assert(__builtin_object_size(&arrf, 0) == (sizeof(float)*2), "");

From bb21a6819b3fb9d689de776f7ee768030dfbacea Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Sun, 20 Apr 2025 12:35:47 +0200
Subject: [PATCH 580/710] [mlir][memref] Move `AllocLikeConversion.h` helpers
 into `MemRefToLLVM.cpp` (#136424)

This commit moves code around: The helper functions/classes are moved
into `MemRefToLLVM.cpp`. This simplifies the code a bit: fewer
templatized functions, fewer function calls, fewer lines of code.

This commit also moves checks in `matchAndRewrite` to the beginning of
the functions, such that patterns bail out (`return failure()`) before
starting to modify any IR. (Apart from that, this change is NFC.) This
is in preparation of the One-Shot Dialect Conversion refactoring, which
will disallow pattern rollbacks.
---
 .../MemRefToLLVM/AllocLikeConversion.h        | 153 --------
 .../MemRefToLLVM/AllocLikeConversion.cpp      | 195 ----------
 .../Conversion/MemRefToLLVM/CMakeLists.txt    |   1 -
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  | 357 +++++++++++++++---
 4 files changed, 301 insertions(+), 405 deletions(-)
 delete mode 100644 mlir/include/mlir/Conversion/MemRefToLLVM/AllocLikeConversion.h
 delete mode 100644 mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp

diff --git a/mlir/include/mlir/Conversion/MemRefToLLVM/AllocLikeConversion.h b/mlir/include/mlir/Conversion/MemRefToLLVM/AllocLikeConversion.h
deleted file mode 100644
index 8bf04219c759a..0000000000000
--- a/mlir/include/mlir/Conversion/MemRefToLLVM/AllocLikeConversion.h
+++ /dev/null
@@ -1,153 +0,0 @@
-//===- AllocLikeConversion.h - Convert allocation ops to LLVM ---*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_CONVERSION_MEMREFTOLLVM_ALLOCLIKECONVERSION_H
-#define MLIR_CONVERSION_MEMREFTOLLVM_ALLOCLIKECONVERSION_H
-
-#include "mlir/Conversion/LLVMCommon/Pattern.h"
-
-namespace mlir {
-
-/// Lowering for memory allocation ops.
-struct AllocationOpLLVMLowering : public ConvertToLLVMPattern {
-  using ConvertToLLVMPattern::createIndexAttrConstant;
-  using ConvertToLLVMPattern::getIndexType;
-  using ConvertToLLVMPattern::getVoidPtrType;
-
-  explicit AllocationOpLLVMLowering(StringRef opName,
-                                    const LLVMTypeConverter &converter,
-                                    PatternBenefit benefit = 1)
-      : ConvertToLLVMPattern(opName, &converter.getContext(), converter,
-                             benefit) {}
-
-protected:
-  /// Computes the aligned value for 'input' as follows:
-  ///   bumped = input + alignement - 1
-  ///   aligned = bumped - bumped % alignment
-  static Value createAligned(ConversionPatternRewriter &rewriter, Location loc,
-                             Value input, Value alignment);
-
-  static MemRefType getMemRefResultType(Operation *op) {
-    return cast<MemRefType>(op->getResult(0).getType());
-  }
-
-  /// Computes the alignment for the given memory allocation op.
-  template <typename OpType>
-  Value getAlignment(ConversionPatternRewriter &rewriter, Location loc,
-                     OpType op) const {
-    MemRefType memRefType = op.getType();
-    Value alignment;
-    if (auto alignmentAttr = op.getAlignment()) {
-      Type indexType = getIndexType();
-      alignment =
-          createIndexAttrConstant(rewriter, loc, indexType, *alignmentAttr);
-    } else if (!memRefType.getElementType().isSignlessIntOrIndexOrFloat()) {
-      // In the case where no alignment is specified, we may want to override
-      // `malloc's` behavior. `malloc` typically aligns at the size of the
-      // biggest scalar on a target HW. For non-scalars, use the natural
-      // alignment of the LLVM type given by the LLVM DataLayout.
-      alignment = getSizeInBytes(loc, memRefType.getElementType(), rewriter);
-    }
-    return alignment;
-  }
-
-  /// Computes the alignment for aligned_alloc used to allocate the buffer for
-  /// the memory allocation op.
-  ///
-  /// Aligned_alloc requires the allocation size to be a power of two, and the
-  /// allocation size to be a multiple of the alignment.
-  template <typename OpType>
-  int64_t alignedAllocationGetAlignment(ConversionPatternRewriter &rewriter,
-                                        Location loc, OpType op,
-                                        const DataLayout *defaultLayout) const {
-    if (std::optional<uint64_t> alignment = op.getAlignment())
-      return *alignment;
-
-    // Whenever we don't have alignment set, we will use an alignment
-    // consistent with the element type; since the allocation size has to be a
-    // power of two, we will bump to the next power of two if it isn't.
-    unsigned eltSizeBytes =
-        getMemRefEltSizeInBytes(op.getType(), op, defaultLayout);
-    return std::max(kMinAlignedAllocAlignment,
-                    llvm::PowerOf2Ceil(eltSizeBytes));
-  }
-
-  /// Allocates a memory buffer using an allocation method that doesn't
-  /// guarantee alignment. Returns the pointer and its aligned value.
-  std::tuple<Value, Value>
-  allocateBufferManuallyAlign(ConversionPatternRewriter &rewriter, Location loc,
-                              Value sizeBytes, Operation *op,
-                              Value alignment) const;
-
-  /// Allocates a memory buffer using an aligned allocation method.
-  Value allocateBufferAutoAlign(ConversionPatternRewriter &rewriter,
-                                Location loc, Value sizeBytes, Operation *op,
-                                const DataLayout *defaultLayout,
-                                int64_t alignment) const;
-
-private:
-  /// Computes the byte size for the MemRef element type.
-  unsigned getMemRefEltSizeInBytes(MemRefType memRefType, Operation *op,
-                                   const DataLayout *defaultLayout) const;
-
-  /// Returns true if the memref size in bytes is known to be a multiple of
-  /// factor.
-  bool isMemRefSizeMultipleOf(MemRefType type, uint64_t factor, Operation *op,
-                              const DataLayout *defaultLayout) const;
-
-  /// The minimum alignment to use with aligned_alloc (has to be a power of 2).
-  static constexpr uint64_t kMinAlignedAllocAlignment = 16UL;
-};
-
-/// Lowering for AllocOp and AllocaOp.
-struct AllocLikeOpLLVMLowering : public AllocationOpLLVMLowering {
-  explicit AllocLikeOpLLVMLowering(StringRef opName,
-                                   const LLVMTypeConverter &converter,
-                                   PatternBenefit benefit = 1)
-      : AllocationOpLLVMLowering(opName, converter, benefit) {}
-
-protected:
-  /// Allocates the underlying buffer. Returns the allocated pointer and the
-  /// aligned pointer.
-  virtual std::tuple<Value, Value>
-  allocateBuffer(ConversionPatternRewriter &rewriter, Location loc, Value size,
-                 Operation *op) const = 0;
-
-  /// Sets the flag 'requiresNumElements', specifying the Op requires the number
-  /// of elements instead of the size in bytes.
-  void setRequiresNumElements();
-
-private:
-  // An `alloc` is converted into a definition of a memref descriptor value and
-  // a call to `malloc` to allocate the underlying data buffer.  The memref
-  // descriptor is of the LLVM structure type where:
-  //   1. the first element is a pointer to the allocated (typed) data buffer,
-  //   2. the second element is a pointer to the (typed) payload, aligned to the
-  //      specified alignment,
-  //   3. the remaining elements serve to store all the sizes and strides of the
-  //      memref using LLVM-converted `index` type.
-  //
-  // Alignment is performed by allocating `alignment` more bytes than
-  // requested and shifting the aligned pointer relative to the allocated
-  // memory. Note: `alignment - <minimum malloc alignment>` would actually be
-  // sufficient. If alignment is unspecified, the two pointers are equal.
-
-  // An `alloca` is converted into a definition of a memref descriptor value and
-  // an llvm.alloca to allocate the underlying data buffer.
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override;
-
-  // Flag for specifying the Op requires the number of elements instead of the
-  // size in bytes.
-  bool requiresNumElements = false;
-};
-
-} // namespace mlir
-
-#endif // MLIR_CONVERSION_MEMREFTOLLVM_ALLOCLIKECONVERSION_H
diff --git a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
deleted file mode 100644
index e9b79983696aa..0000000000000
--- a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp
+++ /dev/null
@@ -1,195 +0,0 @@
-//===- AllocLikeConversion.cpp - LLVM conversion for alloc operations -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Conversion/MemRefToLLVM/AllocLikeConversion.h"
-#include "mlir/Analysis/DataLayoutAnalysis.h"
-#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/IR/SymbolTable.h"
-
-using namespace mlir;
-
-static FailureOr<LLVM::LLVMFuncOp>
-getNotalignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
-                     Operation *module, Type indexType) {
-  bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
-  if (useGenericFn)
-    return LLVM::lookupOrCreateGenericAllocFn(b, module, indexType);
-
-  return LLVM::lookupOrCreateMallocFn(b, module, indexType);
-}
-
-static FailureOr<LLVM::LLVMFuncOp>
-getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
-                  Operation *module, Type indexType) {
-  bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
-
-  if (useGenericFn)
-    return LLVM::lookupOrCreateGenericAlignedAllocFn(b, module, indexType);
-
-  return LLVM::lookupOrCreateAlignedAllocFn(b, module, indexType);
-}
-
-Value AllocationOpLLVMLowering::createAligned(
-    ConversionPatternRewriter &rewriter, Location loc, Value input,
-    Value alignment) {
-  Value one = createIndexAttrConstant(rewriter, loc, alignment.getType(), 1);
-  Value bump = rewriter.create<LLVM::SubOp>(loc, alignment, one);
-  Value bumped = rewriter.create<LLVM::AddOp>(loc, input, bump);
-  Value mod = rewriter.create<LLVM::URemOp>(loc, bumped, alignment);
-  return rewriter.create<LLVM::SubOp>(loc, bumped, mod);
-}
-
-static Value castAllocFuncResult(ConversionPatternRewriter &rewriter,
-                                 Location loc, Value allocatedPtr,
-                                 MemRefType memRefType, Type elementPtrType,
-                                 const LLVMTypeConverter &typeConverter) {
-  auto allocatedPtrTy = cast<LLVM::LLVMPointerType>(allocatedPtr.getType());
-  FailureOr<unsigned> maybeMemrefAddrSpace =
-      typeConverter.getMemRefAddressSpace(memRefType);
-  if (failed(maybeMemrefAddrSpace))
-    return Value();
-  unsigned memrefAddrSpace = *maybeMemrefAddrSpace;
-  if (allocatedPtrTy.getAddressSpace() != memrefAddrSpace)
-    allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
-        loc, LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
-        allocatedPtr);
-  return allocatedPtr;
-}
-
-std::tuple<Value, Value> AllocationOpLLVMLowering::allocateBufferManuallyAlign(
-    ConversionPatternRewriter &rewriter, Location loc, Value sizeBytes,
-    Operation *op, Value alignment) const {
-  if (alignment) {
-    // Adjust the allocation size to consider alignment.
-    sizeBytes = rewriter.create<LLVM::AddOp>(loc, sizeBytes, alignment);
-  }
-
-  MemRefType memRefType = getMemRefResultType(op);
-  // Allocate the underlying buffer.
-  Type elementPtrType = this->getElementPtrType(memRefType);
-  assert(elementPtrType && "could not compute element ptr type");
-  FailureOr<LLVM::LLVMFuncOp> allocFuncOp = getNotalignedAllocFn(
-      rewriter, getTypeConverter(),
-      op->getParentWithTrait<OpTrait::SymbolTable>(), getIndexType());
-  if (failed(allocFuncOp))
-    return std::make_tuple(Value(), Value());
-  auto results =
-      rewriter.create<LLVM::CallOp>(loc, allocFuncOp.value(), sizeBytes);
-
-  Value allocatedPtr =
-      castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
-                          elementPtrType, *getTypeConverter());
-  if (!allocatedPtr)
-    return std::make_tuple(Value(), Value());
-  Value alignedPtr = allocatedPtr;
-  if (alignment) {
-    // Compute the aligned pointer.
-    Value allocatedInt =
-        rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), allocatedPtr);
-    Value alignmentInt = createAligned(rewriter, loc, allocatedInt, alignment);
-    alignedPtr =
-        rewriter.create<LLVM::IntToPtrOp>(loc, elementPtrType, alignmentInt);
-  }
-
-  return std::make_tuple(allocatedPtr, alignedPtr);
-}
-
-unsigned AllocationOpLLVMLowering::getMemRefEltSizeInBytes(
-    MemRefType memRefType, Operation *op,
-    const DataLayout *defaultLayout) const {
-  const DataLayout *layout = defaultLayout;
-  if (const DataLayoutAnalysis *analysis =
-          getTypeConverter()->getDataLayoutAnalysis()) {
-    layout = &analysis->getAbove(op);
-  }
-  Type elementType = memRefType.getElementType();
-  if (auto memRefElementType = dyn_cast<MemRefType>(elementType))
-    return getTypeConverter()->getMemRefDescriptorSize(memRefElementType,
-                                                       *layout);
-  if (auto memRefElementType = dyn_cast<UnrankedMemRefType>(elementType))
-    return getTypeConverter()->getUnrankedMemRefDescriptorSize(
-        memRefElementType, *layout);
-  return layout->getTypeSize(elementType);
-}
-
-bool AllocationOpLLVMLowering::isMemRefSizeMultipleOf(
-    MemRefType type, uint64_t factor, Operation *op,
-    const DataLayout *defaultLayout) const {
-  uint64_t sizeDivisor = getMemRefEltSizeInBytes(type, op, defaultLayout);
-  for (unsigned i = 0, e = type.getRank(); i < e; i++) {
-    if (type.isDynamicDim(i))
-      continue;
-    sizeDivisor = sizeDivisor * type.getDimSize(i);
-  }
-  return sizeDivisor % factor == 0;
-}
-
-Value AllocationOpLLVMLowering::allocateBufferAutoAlign(
-    ConversionPatternRewriter &rewriter, Location loc, Value sizeBytes,
-    Operation *op, const DataLayout *defaultLayout, int64_t alignment) const {
-  Value allocAlignment =
-      createIndexAttrConstant(rewriter, loc, getIndexType(), alignment);
-
-  MemRefType memRefType = getMemRefResultType(op);
-  // Function aligned_alloc requires size to be a multiple of alignment; we pad
-  // the size to the next multiple if necessary.
-  if (!isMemRefSizeMultipleOf(memRefType, alignment, op, defaultLayout))
-    sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
-
-  Type elementPtrType = this->getElementPtrType(memRefType);
-  FailureOr<LLVM::LLVMFuncOp> allocFuncOp = getAlignedAllocFn(
-      rewriter, getTypeConverter(),
-      op->getParentWithTrait<OpTrait::SymbolTable>(), getIndexType());
-  if (failed(allocFuncOp))
-    return Value();
-  auto results = rewriter.create<LLVM::CallOp>(
-      loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
-
-  return castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
-                             elementPtrType, *getTypeConverter());
-}
-
-void AllocLikeOpLLVMLowering::setRequiresNumElements() {
-  requiresNumElements = true;
-}
-
-LogicalResult AllocLikeOpLLVMLowering::matchAndRewrite(
-    Operation *op, ArrayRef<Value> operands,
-    ConversionPatternRewriter &rewriter) const {
-  MemRefType memRefType = getMemRefResultType(op);
-  if (!isConvertibleAndHasIdentityMaps(memRefType))
-    return rewriter.notifyMatchFailure(op, "incompatible memref type");
-  auto loc = op->getLoc();
-
-  // Get actual sizes of the memref as values: static sizes are constant
-  // values and dynamic sizes are passed to 'alloc' as operands.  In case of
-  // zero-dimensional memref, assume a scalar (size 1).
-  SmallVector<Value, 4> sizes;
-  SmallVector<Value, 4> strides;
-  Value size;
-
-  this->getMemRefDescriptorSizes(loc, memRefType, operands, rewriter, sizes,
-                                 strides, size, !requiresNumElements);
-
-  // Allocate the underlying buffer.
-  auto [allocatedPtr, alignedPtr] =
-      this->allocateBuffer(rewriter, loc, size, op);
-
-  if (!allocatedPtr || !alignedPtr)
-    return rewriter.notifyMatchFailure(loc,
-                                       "underlying buffer allocation failed");
-
-  // Create the MemRef descriptor.
-  auto memRefDescriptor = this->createMemRefDescriptor(
-      loc, memRefType, allocatedPtr, alignedPtr, sizes, strides, rewriter);
-
-  // Return the final value of the descriptor.
-  rewriter.replaceOp(op, {memRefDescriptor});
-  return success();
-}
diff --git a/mlir/lib/Conversion/MemRefToLLVM/CMakeLists.txt b/mlir/lib/Conversion/MemRefToLLVM/CMakeLists.txt
index f0d95f5ada290..9da4b23d42f41 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/MemRefToLLVM/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_mlir_conversion_library(MLIRMemRefToLLVM
-  AllocLikeConversion.cpp
   MemRefToLLVM.cpp
 
   ADDITIONAL_HEADER_DIRS
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 9c219d8a3d8cb..c8b2c0bdc6c20 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
-#include "mlir/Conversion/MemRefToLLVM/AllocLikeConversion.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
@@ -53,33 +52,247 @@ getFreeFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
   return LLVM::lookupOrCreateFreeFn(b, module);
 }
 
-struct AllocOpLowering : public AllocLikeOpLLVMLowering {
-  AllocOpLowering(const LLVMTypeConverter &converter)
-      : AllocLikeOpLLVMLowering(memref::AllocOp::getOperationName(),
-                                converter) {}
-  std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,
-                                          Location loc, Value sizeBytes,
-                                          Operation *op) const override {
-    return allocateBufferManuallyAlign(
-        rewriter, loc, sizeBytes, op,
-        getAlignment(rewriter, loc, cast<memref::AllocOp>(op)));
+static FailureOr<LLVM::LLVMFuncOp>
+getNotalignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
+                     Operation *module, Type indexType) {
+  bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
+  if (useGenericFn)
+    return LLVM::lookupOrCreateGenericAllocFn(b, module, indexType);
+
+  return LLVM::lookupOrCreateMallocFn(b, module, indexType);
+}
+
+static FailureOr<LLVM::LLVMFuncOp>
+getAlignedAllocFn(OpBuilder &b, const LLVMTypeConverter *typeConverter,
+                  Operation *module, Type indexType) {
+  bool useGenericFn = typeConverter->getOptions().useGenericFunctions;
+
+  if (useGenericFn)
+    return LLVM::lookupOrCreateGenericAlignedAllocFn(b, module, indexType);
+
+  return LLVM::lookupOrCreateAlignedAllocFn(b, module, indexType);
+}
+
+/// Computes the aligned value for 'input' as follows:
+///   bumped = input + alignement - 1
+///   aligned = bumped - bumped % alignment
+static Value createAligned(ConversionPatternRewriter &rewriter, Location loc,
+                           Value input, Value alignment) {
+  Value one = rewriter.create<LLVM::ConstantOp>(loc, alignment.getType(),
+                                                rewriter.getIndexAttr(1));
+  Value bump = rewriter.create<LLVM::SubOp>(loc, alignment, one);
+  Value bumped = rewriter.create<LLVM::AddOp>(loc, input, bump);
+  Value mod = rewriter.create<LLVM::URemOp>(loc, bumped, alignment);
+  return rewriter.create<LLVM::SubOp>(loc, bumped, mod);
+}
+
+/// Computes the byte size for the MemRef element type.
+static unsigned getMemRefEltSizeInBytes(const LLVMTypeConverter *typeConverter,
+                                        MemRefType memRefType, Operation *op,
+                                        const DataLayout *defaultLayout) {
+  const DataLayout *layout = defaultLayout;
+  if (const DataLayoutAnalysis *analysis =
+          typeConverter->getDataLayoutAnalysis()) {
+    layout = &analysis->getAbove(op);
+  }
+  Type elementType = memRefType.getElementType();
+  if (auto memRefElementType = dyn_cast<MemRefType>(elementType))
+    return typeConverter->getMemRefDescriptorSize(memRefElementType, *layout);
+  if (auto memRefElementType = dyn_cast<UnrankedMemRefType>(elementType))
+    return typeConverter->getUnrankedMemRefDescriptorSize(memRefElementType,
+                                                          *layout);
+  return layout->getTypeSize(elementType);
+}
+
+static Value castAllocFuncResult(ConversionPatternRewriter &rewriter,
+                                 Location loc, Value allocatedPtr,
+                                 MemRefType memRefType, Type elementPtrType,
+                                 const LLVMTypeConverter &typeConverter) {
+  auto allocatedPtrTy = cast<LLVM::LLVMPointerType>(allocatedPtr.getType());
+  FailureOr<unsigned> maybeMemrefAddrSpace =
+      typeConverter.getMemRefAddressSpace(memRefType);
+  assert(succeeded(maybeMemrefAddrSpace) && "unsupported address space");
+  unsigned memrefAddrSpace = *maybeMemrefAddrSpace;
+  if (allocatedPtrTy.getAddressSpace() != memrefAddrSpace)
+    allocatedPtr = rewriter.create<LLVM::AddrSpaceCastOp>(
+        loc, LLVM::LLVMPointerType::get(rewriter.getContext(), memrefAddrSpace),
+        allocatedPtr);
+  return allocatedPtr;
+}
+
+struct AllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
+  using ConvertOpToLLVMPattern<memref::AllocOp>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(memref::AllocOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op.getLoc();
+    MemRefType memRefType = op.getType();
+    if (!isConvertibleAndHasIdentityMaps(memRefType))
+      return rewriter.notifyMatchFailure(op, "incompatible memref type");
+
+    // Get or insert alloc function into the module.
+    FailureOr<LLVM::LLVMFuncOp> allocFuncOp = getNotalignedAllocFn(
+        rewriter, getTypeConverter(),
+        op->getParentWithTrait<OpTrait::SymbolTable>(), getIndexType());
+    if (failed(allocFuncOp))
+      return failure();
+
+    // Get actual sizes of the memref as values: static sizes are constant
+    // values and dynamic sizes are passed to 'alloc' as operands.  In case of
+    // zero-dimensional memref, assume a scalar (size 1).
+    SmallVector<Value, 4> sizes;
+    SmallVector<Value, 4> strides;
+    Value sizeBytes;
+
+    this->getMemRefDescriptorSizes(loc, memRefType, adaptor.getOperands(),
+                                   rewriter, sizes, strides, sizeBytes, true);
+
+    Value alignment = getAlignment(rewriter, loc, op);
+    if (alignment) {
+      // Adjust the allocation size to consider alignment.
+      sizeBytes = rewriter.create<LLVM::AddOp>(loc, sizeBytes, alignment);
+    }
+
+    // Allocate the underlying buffer.
+    Type elementPtrType = this->getElementPtrType(memRefType);
+    assert(elementPtrType && "could not compute element ptr type");
+    auto results =
+        rewriter.create<LLVM::CallOp>(loc, allocFuncOp.value(), sizeBytes);
+
+    Value allocatedPtr =
+        castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
+                            elementPtrType, *getTypeConverter());
+    Value alignedPtr = allocatedPtr;
+    if (alignment) {
+      // Compute the aligned pointer.
+      Value allocatedInt =
+          rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), allocatedPtr);
+      Value alignmentInt =
+          createAligned(rewriter, loc, allocatedInt, alignment);
+      alignedPtr =
+          rewriter.create<LLVM::IntToPtrOp>(loc, elementPtrType, alignmentInt);
+    }
+
+    // Create the MemRef descriptor.
+    auto memRefDescriptor = this->createMemRefDescriptor(
+        loc, memRefType, allocatedPtr, alignedPtr, sizes, strides, rewriter);
+
+    // Return the final value of the descriptor.
+    rewriter.replaceOp(op, {memRefDescriptor});
+    return success();
+  }
+
+  /// Computes the alignment for the given memory allocation op.
+  template <typename OpType>
+  Value getAlignment(ConversionPatternRewriter &rewriter, Location loc,
+                     OpType op) const {
+    MemRefType memRefType = op.getType();
+    Value alignment;
+    if (auto alignmentAttr = op.getAlignment()) {
+      Type indexType = getIndexType();
+      alignment =
+          createIndexAttrConstant(rewriter, loc, indexType, *alignmentAttr);
+    } else if (!memRefType.getElementType().isSignlessIntOrIndexOrFloat()) {
+      // In the case where no alignment is specified, we may want to override
+      // `malloc's` behavior. `malloc` typically aligns at the size of the
+      // biggest scalar on a target HW. For non-scalars, use the natural
+      // alignment of the LLVM type given by the LLVM DataLayout.
+      alignment = getSizeInBytes(loc, memRefType.getElementType(), rewriter);
+    }
+    return alignment;
   }
 };
 
-struct AlignedAllocOpLowering : public AllocLikeOpLLVMLowering {
-  AlignedAllocOpLowering(const LLVMTypeConverter &converter)
-      : AllocLikeOpLLVMLowering(memref::AllocOp::getOperationName(),
-                                converter) {}
-  std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,
-                                          Location loc, Value sizeBytes,
-                                          Operation *op) const override {
-    Value ptr = allocateBufferAutoAlign(
-        rewriter, loc, sizeBytes, op, &defaultLayout,
-        alignedAllocationGetAlignment(rewriter, loc, cast<memref::AllocOp>(op),
-                                      &defaultLayout));
-    if (!ptr)
-      return std::make_tuple(Value(), Value());
-    return std::make_tuple(ptr, ptr);
+struct AlignedAllocOpLowering : public ConvertOpToLLVMPattern<memref::AllocOp> {
+  using ConvertOpToLLVMPattern<memref::AllocOp>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(memref::AllocOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op.getLoc();
+    MemRefType memRefType = op.getType();
+    if (!isConvertibleAndHasIdentityMaps(memRefType))
+      return rewriter.notifyMatchFailure(op, "incompatible memref type");
+
+    // Get or insert alloc function into module.
+    FailureOr<LLVM::LLVMFuncOp> allocFuncOp = getAlignedAllocFn(
+        rewriter, getTypeConverter(),
+        op->getParentWithTrait<OpTrait::SymbolTable>(), getIndexType());
+    if (failed(allocFuncOp))
+      return failure();
+
+    // Get actual sizes of the memref as values: static sizes are constant
+    // values and dynamic sizes are passed to 'alloc' as operands.  In case of
+    // zero-dimensional memref, assume a scalar (size 1).
+    SmallVector<Value, 4> sizes;
+    SmallVector<Value, 4> strides;
+    Value sizeBytes;
+
+    this->getMemRefDescriptorSizes(loc, memRefType, adaptor.getOperands(),
+                                   rewriter, sizes, strides, sizeBytes, !false);
+
+    int64_t alignment = alignedAllocationGetAlignment(op, &defaultLayout);
+
+    Value allocAlignment =
+        createIndexAttrConstant(rewriter, loc, getIndexType(), alignment);
+
+    // Function aligned_alloc requires size to be a multiple of alignment; we
+    // pad the size to the next multiple if necessary.
+    if (!isMemRefSizeMultipleOf(memRefType, alignment, op, &defaultLayout))
+      sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);
+
+    Type elementPtrType = this->getElementPtrType(memRefType);
+    auto results = rewriter.create<LLVM::CallOp>(
+        loc, allocFuncOp.value(), ValueRange({allocAlignment, sizeBytes}));
+
+    Value ptr =
+        castAllocFuncResult(rewriter, loc, results.getResult(), memRefType,
+                            elementPtrType, *getTypeConverter());
+
+    // Create the MemRef descriptor.
+    auto memRefDescriptor = this->createMemRefDescriptor(
+        loc, memRefType, ptr, ptr, sizes, strides, rewriter);
+
+    // Return the final value of the descriptor.
+    rewriter.replaceOp(op, {memRefDescriptor});
+    return success();
+  }
+
+  /// The minimum alignment to use with aligned_alloc (has to be a power of 2).
+  static constexpr uint64_t kMinAlignedAllocAlignment = 16UL;
+
+  /// Computes the alignment for aligned_alloc used to allocate the buffer for
+  /// the memory allocation op.
+  ///
+  /// Aligned_alloc requires the allocation size to be a power of two, and the
+  /// allocation size to be a multiple of the alignment.
+  int64_t alignedAllocationGetAlignment(memref::AllocOp op,
+                                        const DataLayout *defaultLayout) const {
+    if (std::optional<uint64_t> alignment = op.getAlignment())
+      return *alignment;
+
+    // Whenever we don't have alignment set, we will use an alignment
+    // consistent with the element type; since the allocation size has to be a
+    // power of two, we will bump to the next power of two if it isn't.
+    unsigned eltSizeBytes = getMemRefEltSizeInBytes(
+        getTypeConverter(), op.getType(), op, defaultLayout);
+    return std::max(kMinAlignedAllocAlignment,
+                    llvm::PowerOf2Ceil(eltSizeBytes));
+  }
+
+  /// Returns true if the memref size in bytes is known to be a multiple of
+  /// factor.
+  bool isMemRefSizeMultipleOf(MemRefType type, uint64_t factor, Operation *op,
+                              const DataLayout *defaultLayout) const {
+    uint64_t sizeDivisor =
+        getMemRefEltSizeInBytes(getTypeConverter(), type, op, defaultLayout);
+    for (unsigned i = 0, e = type.getRank(); i < e; i++) {
+      if (type.isDynamicDim(i))
+        continue;
+      sizeDivisor = sizeDivisor * type.getDimSize(i);
+    }
+    return sizeDivisor % factor == 0;
   }
 
 private:
@@ -87,38 +300,52 @@ struct AlignedAllocOpLowering : public AllocLikeOpLLVMLowering {
   DataLayout defaultLayout;
 };
 
-struct AllocaOpLowering : public AllocLikeOpLLVMLowering {
-  AllocaOpLowering(const LLVMTypeConverter &converter)
-      : AllocLikeOpLLVMLowering(memref::AllocaOp::getOperationName(),
-                                converter) {
-    setRequiresNumElements();
-  }
+struct AllocaOpLowering : public ConvertOpToLLVMPattern<memref::AllocaOp> {
+  using ConvertOpToLLVMPattern<memref::AllocaOp>::ConvertOpToLLVMPattern;
 
   /// Allocates the underlying buffer using the right call. `allocatedBytePtr`
   /// is set to null for stack allocations. `accessAlignment` is set if
   /// alignment is needed post allocation (for eg. in conjunction with malloc).
-  std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,
-                                          Location loc, Value size,
-                                          Operation *op) const override {
+  LogicalResult
+  matchAndRewrite(memref::AllocaOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op.getLoc();
+    MemRefType memRefType = op.getType();
+    if (!isConvertibleAndHasIdentityMaps(memRefType))
+      return rewriter.notifyMatchFailure(op, "incompatible memref type");
+
+    // Get actual sizes of the memref as values: static sizes are constant
+    // values and dynamic sizes are passed to 'alloc' as operands.  In case of
+    // zero-dimensional memref, assume a scalar (size 1).
+    SmallVector<Value, 4> sizes;
+    SmallVector<Value, 4> strides;
+    Value size;
+
+    this->getMemRefDescriptorSizes(loc, memRefType, adaptor.getOperands(),
+                                   rewriter, sizes, strides, size, !true);
 
     // With alloca, one gets a pointer to the element type right away.
     // For stack allocations.
-    auto allocaOp = cast<memref::AllocaOp>(op);
     auto elementType =
-        typeConverter->convertType(allocaOp.getType().getElementType());
+        typeConverter->convertType(op.getType().getElementType());
     FailureOr<unsigned> maybeAddressSpace =
-        getTypeConverter()->getMemRefAddressSpace(allocaOp.getType());
-    if (failed(maybeAddressSpace))
-      return std::make_tuple(Value(), Value());
+        getTypeConverter()->getMemRefAddressSpace(op.getType());
+    assert(succeeded(maybeAddressSpace) && "unsupported address space");
     unsigned addrSpace = *maybeAddressSpace;
     auto elementPtrType =
         LLVM::LLVMPointerType::get(rewriter.getContext(), addrSpace);
 
-    auto allocatedElementPtr =
-        rewriter.create<LLVM::AllocaOp>(loc, elementPtrType, elementType, size,
-                                        allocaOp.getAlignment().value_or(0));
+    auto allocatedElementPtr = rewriter.create<LLVM::AllocaOp>(
+        loc, elementPtrType, elementType, size, op.getAlignment().value_or(0));
 
-    return std::make_tuple(allocatedElementPtr, allocatedElementPtr);
+    // Create the MemRef descriptor.
+    auto memRefDescriptor = this->createMemRefDescriptor(
+        loc, memRefType, allocatedElementPtr, allocatedElementPtr, sizes,
+        strides, rewriter);
+
+    // Return the final value of the descriptor.
+    rewriter.replaceOp(op, {memRefDescriptor});
+    return success();
   }
 };
 
@@ -527,31 +754,43 @@ struct GlobalMemrefOpLowering
 /// GetGlobalMemrefOp is lowered into a Memref descriptor with the pointer to
 /// the first element stashed into the descriptor. This reuses
 /// `AllocLikeOpLowering` to reuse the Memref descriptor construction.
-struct GetGlobalMemrefOpLowering : public AllocLikeOpLLVMLowering {
-  GetGlobalMemrefOpLowering(const LLVMTypeConverter &converter)
-      : AllocLikeOpLLVMLowering(memref::GetGlobalOp::getOperationName(),
-                                converter) {}
+struct GetGlobalMemrefOpLowering
+    : public ConvertOpToLLVMPattern<memref::GetGlobalOp> {
+  using ConvertOpToLLVMPattern<memref::GetGlobalOp>::ConvertOpToLLVMPattern;
 
   /// Buffer "allocation" for memref.get_global op is getting the address of
   /// the global variable referenced.
-  std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,
-                                          Location loc, Value sizeBytes,
-                                          Operation *op) const override {
-    auto getGlobalOp = cast<memref::GetGlobalOp>(op);
-    MemRefType type = cast<MemRefType>(getGlobalOp.getResult().getType());
+  LogicalResult
+  matchAndRewrite(memref::GetGlobalOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op.getLoc();
+    MemRefType memRefType = op.getType();
+    if (!isConvertibleAndHasIdentityMaps(memRefType))
+      return rewriter.notifyMatchFailure(op, "incompatible memref type");
+
+    // Get actual sizes of the memref as values: static sizes are constant
+    // values and dynamic sizes are passed to 'alloc' as operands.  In case of
+    // zero-dimensional memref, assume a scalar (size 1).
+    SmallVector<Value, 4> sizes;
+    SmallVector<Value, 4> strides;
+    Value sizeBytes;
+
+    this->getMemRefDescriptorSizes(loc, memRefType, adaptor.getOperands(),
+                                   rewriter, sizes, strides, sizeBytes, !false);
+
+    MemRefType type = cast<MemRefType>(op.getResult().getType());
 
     // This is called after a type conversion, which would have failed if this
     // call fails.
     FailureOr<unsigned> maybeAddressSpace =
         getTypeConverter()->getMemRefAddressSpace(type);
-    if (failed(maybeAddressSpace))
-      return std::make_tuple(Value(), Value());
+    assert(succeeded(maybeAddressSpace) && "unsupported address space");
     unsigned memSpace = *maybeAddressSpace;
 
     Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter());
     auto ptrTy = LLVM::LLVMPointerType::get(rewriter.getContext(), memSpace);
     auto addressOf =
-        rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, getGlobalOp.getName());
+        rewriter.create<LLVM::AddressOfOp>(loc, ptrTy, op.getName());
 
     // Get the address of the first element in the array by creating a GEP with
     // the address of the GV as the base, and (rank + 1) number of 0 indices.
@@ -570,7 +809,13 @@ struct GetGlobalMemrefOpLowering : public AllocLikeOpLLVMLowering {
 
     // Both allocated and aligned pointers are same. We could potentially stash
     // a nullptr for the allocated pointer since we do not expect any dealloc.
-    return std::make_tuple(deadBeefPtr, gep);
+    // Create the MemRef descriptor.
+    auto memRefDescriptor = this->createMemRefDescriptor(
+        loc, memRefType, deadBeefPtr, gep, sizes, strides, rewriter);
+
+    // Return the final value of the descriptor.
+    rewriter.replaceOp(op, {memRefDescriptor});
+    return success();
   }
 };
 

From fc71b2c8db5d0a648ca5417e4c502755bfbe75b6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 20 Apr 2025 14:25:32 +0100
Subject: [PATCH 581/710] [VPlan] Get opcode from recipe in
 VPWidenMemRecipe::computeCost (NFC).

Remove some uses of the underlying ingredient by getting the opcode
directly via the recipe ID.
---
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ff1ad29e75e2d..c2cfd93d23ec0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2686,6 +2686,9 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
       getLoadStoreAlignment(const_cast<Instruction *>(&Ingredient));
   unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
                     ->getAddressSpace();
+  unsigned Opcode = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(this)
+                        ? Instruction::Load
+                        : Instruction::Store;
 
   if (!Consecutive) {
     // TODO: Using the original IR may not be accurate.
@@ -2695,20 +2698,19 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
     assert(!Reverse &&
            "Inconsecutive memory access should not have the order.");
     return Ctx.TTI.getAddressComputationCost(Ty) +
-           Ctx.TTI.getGatherScatterOpCost(Ingredient.getOpcode(), Ty, Ptr,
-                                          IsMasked, Alignment, Ctx.CostKind,
-                                          &Ingredient);
+           Ctx.TTI.getGatherScatterOpCost(Opcode, Ty, Ptr, IsMasked, Alignment,
+                                          Ctx.CostKind, &Ingredient);
   }
 
   InstructionCost Cost = 0;
   if (IsMasked) {
-    Cost += Ctx.TTI.getMaskedMemoryOpCost(Ingredient.getOpcode(), Ty, Alignment,
-                                          AS, Ctx.CostKind);
+    Cost +=
+        Ctx.TTI.getMaskedMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind);
   } else {
     TTI::OperandValueInfo OpInfo =
         Ctx.TTI.getOperandInfo(Ingredient.getOperand(0));
-    Cost += Ctx.TTI.getMemoryOpCost(Ingredient.getOpcode(), Ty, Alignment, AS,
-                                    Ctx.CostKind, OpInfo, &Ingredient);
+    Cost += Ctx.TTI.getMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind,
+                                    OpInfo, &Ingredient);
   }
   if (!Reverse)
     return Cost;

From 060f9556a2f6ef4669f1c2cd8c4a4d76748a440f Mon Sep 17 00:00:00 2001
From: Pavel Skripkin <paskripkin@gmail.com>
Date: Sun, 20 Apr 2025 10:14:41 -0400
Subject: [PATCH 582/710] [clang][analyzer] Fix error path of builtin overflow
 (#136345)

According to
https://clang.llvm.org/docs/LanguageExtensions.html#checked-arithmetic-builtins,
result of builtin_*_overflow functions will be initialized even in case
of overflow. Align analyzer logic to docs and always initialize 3rd
argument of such builtins.

Closes #136292
---
 .../Checkers/BuiltinFunctionChecker.cpp       | 86 +++++++++++--------
 clang/test/Analysis/builtin_overflow.c        |  6 +-
 clang/test/Analysis/builtin_overflow_notes.c  | 10 ++-
 3 files changed, 58 insertions(+), 44 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index b1a11442dec53..8eb68b1fa638e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -96,10 +96,14 @@ class BuiltinFunctionChecker : public Checker<eval::Call> {
   void handleOverflowBuiltin(const CallEvent &Call, CheckerContext &C,
                              BinaryOperator::Opcode Op,
                              QualType ResultType) const;
-  const NoteTag *createBuiltinNoOverflowNoteTag(CheckerContext &C,
-                                                bool BothFeasible, SVal Arg1,
-                                                SVal Arg2, SVal Result) const;
-  const NoteTag *createBuiltinOverflowNoteTag(CheckerContext &C) const;
+  const NoteTag *createBuiltinOverflowNoteTag(CheckerContext &C,
+                                              bool BothFeasible, SVal Arg1,
+                                              SVal Arg2, SVal Result) const;
+  ProgramStateRef initStateAftetBuiltinOverflow(CheckerContext &C,
+                                                ProgramStateRef State,
+                                                const CallEvent &Call,
+                                                SVal RetCal,
+                                                bool IsOverflow) const;
   std::pair<bool, bool> checkOverflow(CheckerContext &C, SVal RetVal,
                                       QualType Res) const;
 
@@ -121,30 +125,24 @@ class BuiltinFunctionChecker : public Checker<eval::Call> {
 
 } // namespace
 
-const NoteTag *BuiltinFunctionChecker::createBuiltinNoOverflowNoteTag(
-    CheckerContext &C, bool BothFeasible, SVal Arg1, SVal Arg2,
-    SVal Result) const {
-  return C.getNoteTag([Result, Arg1, Arg2, BothFeasible](
-                          PathSensitiveBugReport &BR, llvm::raw_ostream &OS) {
+const NoteTag *BuiltinFunctionChecker::createBuiltinOverflowNoteTag(
+    CheckerContext &C, bool overflow, SVal Arg1, SVal Arg2, SVal Result) const {
+  return C.getNoteTag([Result, Arg1, Arg2, overflow](PathSensitiveBugReport &BR,
+                                                     llvm::raw_ostream &OS) {
     if (!BR.isInteresting(Result))
       return;
 
-    // Propagate interestingness to input argumets if result is interesting.
+    // Propagate interestingness to input arguments if result is interesting.
     BR.markInteresting(Arg1);
     BR.markInteresting(Arg2);
 
-    if (BothFeasible)
+    if (overflow)
+      OS << "Assuming overflow";
+    else
       OS << "Assuming no overflow";
   });
 }
 
-const NoteTag *
-BuiltinFunctionChecker::createBuiltinOverflowNoteTag(CheckerContext &C) const {
-  return C.getNoteTag([](PathSensitiveBugReport &BR,
-                         llvm::raw_ostream &OS) { OS << "Assuming overflow"; },
-                      /*isPrunable=*/true);
-}
-
 std::pair<bool, bool>
 BuiltinFunctionChecker::checkOverflow(CheckerContext &C, SVal RetVal,
                                       QualType Res) const {
@@ -174,6 +172,29 @@ BuiltinFunctionChecker::checkOverflow(CheckerContext &C, SVal RetVal,
   return {MayOverflow || MayUnderflow, MayNotOverflow && MayNotUnderflow};
 }
 
+ProgramStateRef BuiltinFunctionChecker::initStateAftetBuiltinOverflow(
+    CheckerContext &C, ProgramStateRef State, const CallEvent &Call,
+    SVal RetVal, bool IsOverflow) const {
+  SValBuilder &SVB = C.getSValBuilder();
+  SVal Arg1 = Call.getArgSVal(0);
+  SVal Arg2 = Call.getArgSVal(1);
+  auto BoolTy = C.getASTContext().BoolTy;
+
+  ProgramStateRef NewState =
+      State->BindExpr(Call.getOriginExpr(), C.getLocationContext(),
+                      SVB.makeTruthVal(IsOverflow, BoolTy));
+
+  if (auto L = Call.getArgSVal(2).getAs<Loc>()) {
+    NewState = NewState->bindLoc(*L, RetVal, C.getLocationContext());
+
+    // Propagate taint if any of the arguments were tainted
+    if (isTainted(State, Arg1) || isTainted(State, Arg2))
+      NewState = addTaint(NewState, *L);
+  }
+
+  return NewState;
+}
+
 void BuiltinFunctionChecker::handleOverflowBuiltin(const CallEvent &Call,
                                                    CheckerContext &C,
                                                    BinaryOperator::Opcode Op,
@@ -183,8 +204,6 @@ void BuiltinFunctionChecker::handleOverflowBuiltin(const CallEvent &Call,
 
   ProgramStateRef State = C.getState();
   SValBuilder &SVB = C.getSValBuilder();
-  const Expr *CE = Call.getOriginExpr();
-  auto BoolTy = C.getASTContext().BoolTy;
 
   SVal Arg1 = Call.getArgSVal(0);
   SVal Arg2 = Call.getArgSVal(1);
@@ -194,29 +213,20 @@ void BuiltinFunctionChecker::handleOverflowBuiltin(const CallEvent &Call,
   SVal RetVal = SVB.evalBinOp(State, Op, Arg1, Arg2, ResultType);
 
   auto [Overflow, NotOverflow] = checkOverflow(C, RetValMax, ResultType);
-  if (NotOverflow) {
-    ProgramStateRef StateNoOverflow = State->BindExpr(
-        CE, C.getLocationContext(), SVB.makeTruthVal(false, BoolTy));
-
-    if (auto L = Call.getArgSVal(2).getAs<Loc>()) {
-      StateNoOverflow =
-          StateNoOverflow->bindLoc(*L, RetVal, C.getLocationContext());
 
-      // Propagate taint if any of the argumets were tainted
-      if (isTainted(State, Arg1) || isTainted(State, Arg2))
-        StateNoOverflow = addTaint(StateNoOverflow, *L);
-    }
+  if (NotOverflow) {
+    auto NewState =
+        initStateAftetBuiltinOverflow(C, State, Call, RetVal, false);
 
-    C.addTransition(
-        StateNoOverflow,
-        createBuiltinNoOverflowNoteTag(
-            C, /*BothFeasible=*/NotOverflow && Overflow, Arg1, Arg2, RetVal));
+    C.addTransition(NewState, createBuiltinOverflowNoteTag(
+                                  C, /*overflow=*/false, Arg1, Arg2, RetVal));
   }
 
   if (Overflow) {
-    C.addTransition(State->BindExpr(CE, C.getLocationContext(),
-                                    SVB.makeTruthVal(true, BoolTy)),
-                    createBuiltinOverflowNoteTag(C));
+    auto NewState = initStateAftetBuiltinOverflow(C, State, Call, RetVal, true);
+
+    C.addTransition(NewState, createBuiltinOverflowNoteTag(C, /*overflow=*/true,
+                                                           Arg1, Arg2, RetVal));
   }
 }
 
diff --git a/clang/test/Analysis/builtin_overflow.c b/clang/test/Analysis/builtin_overflow.c
index 9d98ce7a1af45..d290333071dc9 100644
--- a/clang/test/Analysis/builtin_overflow.c
+++ b/clang/test/Analysis/builtin_overflow.c
@@ -26,7 +26,7 @@ void test_add_overflow(void)
    int res;
 
    if (__builtin_add_overflow(__INT_MAX__, 1, &res)) {
-     clang_analyzer_dump_int(res); //expected-warning{{1st function call argument is an uninitialized value}}
+     clang_analyzer_dump_int(res); //expected-warning{{-2147483648 S32b}}
      return;
    }
 
@@ -38,7 +38,7 @@ void test_add_underoverflow(void)
    int res;
 
    if (__builtin_add_overflow(__INT_MIN__, -1, &res)) {
-     clang_analyzer_dump_int(res); //expected-warning{{1st function call argument is an uninitialized value}}
+     clang_analyzer_dump_int(res); //expected-warning{{2147483647 S32b}}
      return;
    }
 
@@ -160,7 +160,7 @@ void test_bool_assign(void)
 {
     int res;
 
-    // Reproduce issue from GH#111147. __builtin_*_overflow funcions
+    // Reproduce issue from GH#111147. __builtin_*_overflow functions
     // should return _Bool, but not int.
     _Bool ret = __builtin_mul_overflow(10, 20, &res); // no crash
 }
diff --git a/clang/test/Analysis/builtin_overflow_notes.c b/clang/test/Analysis/builtin_overflow_notes.c
index 5fa2156df925c..94c79b5ed334a 100644
--- a/clang/test/Analysis/builtin_overflow_notes.c
+++ b/clang/test/Analysis/builtin_overflow_notes.c
@@ -19,12 +19,16 @@ void test_no_overflow_note(int a, int b)
 
 void test_overflow_note(int a, int b)
 {
-   int res; // expected-note{{'res' declared without an initial value}}
+   int res;
 
    if (__builtin_add_overflow(a, b, &res)) { // expected-note {{Assuming overflow}}
                                              // expected-note@-1 {{Taking true branch}}
-     int var = res; // expected-warning{{Assigned value is uninitialized}}
-                    // expected-note@-1 {{Assigned value is uninitialized}}
+     if (res) { // expected-note {{Assuming 'res' is not equal to 0}}
+                // expected-note@-1 {{Taking true branch}}
+        int *ptr = 0; // expected-note {{'ptr' initialized to a null pointer value}}
+        int var = *(int *) ptr; //expected-warning {{Dereference of null pointer}}
+                                //expected-note@-1 {{Dereference of null pointer}}
+     }
      return;
    }
 }

From 461168a3d3bb1e75ff41d351a5cc506e103da5d6 Mon Sep 17 00:00:00 2001
From: Fangyi Zhou <me@fangyi.io>
Date: Sun, 20 Apr 2025 16:32:42 +0100
Subject: [PATCH 583/710] [clang][analyzer] Handle CXXParenInitListExpr
 alongside InitListExpr (#136041)

As reported in #135665, C++20 parenthesis initializer list expressions
are not handled correctly and were causing crashes. This commit attempts
to fix the issue by handing parenthesis initializer lists along side
existing initializer lists.

Fixes #135665.
---
 clang/docs/ReleaseNotes.rst                   |  2 ++
 .../Checkers/DynamicTypePropagation.cpp       |  6 +++---
 .../lib/StaticAnalyzer/Core/ExprEngineCXX.cpp | 10 ++++++----
 clang/test/Analysis/PR135665.cpp              | 19 +++++++++++++++++++
 4 files changed, 30 insertions(+), 7 deletions(-)
 create mode 100644 clang/test/Analysis/PR135665.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index f5cd1fbeabcfe..25302764778d1 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -677,6 +677,8 @@ Code Completion
 
 Static Analyzer
 ---------------
+- Fixed a crash when C++20 parenthesized initializer lists are used. This issue
+  was causing a crash in clang-tidy. (#GH136041)
 
 New features
 ^^^^^^^^^^^^
diff --git a/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp b/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
index a0bf776b11f53..e58329817d7cd 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
@@ -379,9 +379,9 @@ void DynamicTypePropagation::checkPostCall(const CallEvent &Call,
         // aggregates, and in such case no top-frame constructor will be called.
         // Figure out if we need to do anything in this case.
         // FIXME: Instead of relying on the ParentMap, we should have the
-        // trigger-statement (InitListExpr in this case) available in this
-        // callback, ideally as part of CallEvent.
-        if (isa_and_nonnull<InitListExpr>(
+        // trigger-statement (InitListExpr or CXXParenListInitExpr in this case)
+        // available in this callback, ideally as part of CallEvent.
+        if (isa_and_nonnull<InitListExpr, CXXParenListInitExpr>(
                 LCtx->getParentMap().getParent(Ctor->getOriginExpr())))
           return;
 
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
index 7e878f922a939..92ce3fa2225c8 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
@@ -644,9 +644,10 @@ void ExprEngine::handleConstructor(const Expr *E,
     // FIXME: For now this code essentially bails out. We need to find the
     // correct target region and set it.
     // FIXME: Instead of relying on the ParentMap, we should have the
-    // trigger-statement (InitListExpr in this case) passed down from CFG or
-    // otherwise always available during construction.
-    if (isa_and_nonnull<InitListExpr>(LCtx->getParentMap().getParent(E))) {
+    // trigger-statement (InitListExpr or CXXParenListInitExpr in this case)
+    // passed down from CFG or otherwise always available during construction.
+    if (isa_and_nonnull<InitListExpr, CXXParenListInitExpr>(
+            LCtx->getParentMap().getParent(E))) {
       MemRegionManager &MRMgr = getSValBuilder().getRegionManager();
       Target = loc::MemRegionVal(MRMgr.getCXXTempObjectRegion(E, LCtx));
       CallOpts.IsCtorOrDtorWithImproperlyModeledTargetRegion = true;
@@ -1017,7 +1018,8 @@ void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
       // values are properly placed inside the required region, however if an
       // initializer list is used, this doesn't happen automatically.
       auto *Init = CNE->getInitializer();
-      bool isInitList = isa_and_nonnull<InitListExpr>(Init);
+      bool isInitList =
+          isa_and_nonnull<InitListExpr, CXXParenListInitExpr>(Init);
 
       QualType ObjTy =
           isInitList ? Init->getType() : CNE->getType()->getPointeeType();
diff --git a/clang/test/Analysis/PR135665.cpp b/clang/test/Analysis/PR135665.cpp
new file mode 100644
index 0000000000000..c03ba551b3ae2
--- /dev/null
+++ b/clang/test/Analysis/PR135665.cpp
@@ -0,0 +1,19 @@
+// RUN: %clang_analyze_cc1 -std=c++20 -analyzer-checker=core -verify %s
+
+// expected-no-diagnostics
+
+template<typename... F>
+struct overload : public F...
+{
+  using F::operator()...;
+};
+
+template<typename... F>
+overload(F&&...) -> overload<F...>;
+
+int main()
+{
+  const auto l = overload([](const int* i) {}); // no-crash
+
+  return 0;
+}

From 839f52147c0cdbe7d4590cfdda089f250e5ec119 Mon Sep 17 00:00:00 2001
From: Ely Ronnen <elyronnen@gmail.com>
Date: Sun, 20 Apr 2025 17:33:28 +0200
Subject: [PATCH 584/710] [lldb-dap] fix inconsistent debugAdapterHostname
 argument name (#135544)

the argument is written as `debugAdapterHostname` in package.json but
used as `debugAdapterHost`
---
 lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts        | 2 +-
 lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts b/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
index e23d717a70101..3a86d1f3f418f 100644
--- a/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
+++ b/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
@@ -211,7 +211,7 @@ export class LLDBDapDescriptorFactory
     if (session.configuration.debugAdapterPort) {
       return new vscode.DebugAdapterServer(
         session.configuration.debugAdapterPort,
-        session.configuration.debugAdapterHost,
+        session.configuration.debugAdapterHostname,
       );
     }
 
diff --git a/lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts b/lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts
index 0272509ee55f7..8d92139c02a00 100644
--- a/lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts
+++ b/lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts
@@ -32,11 +32,11 @@ export class LLDBDapConfigurationProvider
   ): Promise<vscode.DebugConfiguration | null | undefined> {
     try {
       if (
-        "debugAdapterHost" in debugConfiguration &&
+        "debugAdapterHostname" in debugConfiguration &&
         !("debugAdapterPort" in debugConfiguration)
       ) {
         throw new ErrorWithNotification(
-          "A debugAdapterPort must be provided when debugAdapterHost is set. Please update your launch configuration.",
+          "A debugAdapterPort must be provided when debugAdapterHostname is set. Please update your launch configuration.",
           new ConfigureButton(),
         );
       }
@@ -83,7 +83,7 @@ export class LLDBDapConfigurationProvider
           // and list of arguments.
           delete debugConfiguration.debugAdapterExecutable;
           delete debugConfiguration.debugAdapterArgs;
-          debugConfiguration.debugAdapterHost = serverInfo.host;
+          debugConfiguration.debugAdapterHostname = serverInfo.host;
           debugConfiguration.debugAdapterPort = serverInfo.port;
         }
       }

From ea3eb8d6258a018f118b5d41057ca333d1c8d4a0 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sun, 20 Apr 2025 20:12:47 +0200
Subject: [PATCH 585/710] [clang][bytecode] Fix bos/bdos with non-zero offset
 applied (#136482)

Compute the offset from the record layout.
Unfortunately, not all the test cases from the current interpreter work.
---
 clang/lib/AST/ByteCode/InterpBuiltin.cpp      | 49 ++++++++++++++++++-
 .../test/AST/ByteCode/builtin-object-size.cpp | 44 ++++++++++++++++-
 2 files changed, 91 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index aaf0f3f019b94..523e471d3c82c 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -2196,6 +2196,53 @@ static unsigned computeFullDescSize(const ASTContext &ASTCtx,
   return 0;
 }
 
+static unsigned computePointerOffset(const ASTContext &ASTCtx,
+                                     const Pointer &Ptr) {
+  unsigned Result = 0;
+
+  Pointer P = Ptr;
+  while (P.isArrayElement() || P.isField()) {
+    P = P.expand();
+    const Descriptor *D = P.getFieldDesc();
+
+    if (P.isArrayElement()) {
+      unsigned ElemSize =
+          ASTCtx.getTypeSizeInChars(D->getElemQualType()).getQuantity();
+      if (P.isOnePastEnd())
+        Result += ElemSize * P.getNumElems();
+      else
+        Result += ElemSize * P.getIndex();
+      P = P.expand().getArray();
+    } else if (P.isBaseClass()) {
+
+      const auto *RD = cast<CXXRecordDecl>(D->asDecl());
+      bool IsVirtual = Ptr.isVirtualBaseClass();
+      P = P.getBase();
+      const Record *BaseRecord = P.getRecord();
+
+      const ASTRecordLayout &Layout =
+          ASTCtx.getASTRecordLayout(cast<CXXRecordDecl>(BaseRecord->getDecl()));
+      if (IsVirtual)
+        Result += Layout.getVBaseClassOffset(RD).getQuantity();
+      else
+        Result += Layout.getBaseClassOffset(RD).getQuantity();
+    } else if (P.isField()) {
+      const FieldDecl *FD = P.getField();
+      const ASTRecordLayout &Layout =
+          ASTCtx.getASTRecordLayout(FD->getParent());
+      unsigned FieldIndex = FD->getFieldIndex();
+      uint64_t FieldOffset =
+          ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex))
+              .getQuantity();
+      Result += FieldOffset;
+      P = P.getBase();
+    } else
+      llvm_unreachable("Unhandled descriptor type");
+  }
+
+  return Result;
+}
+
 static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
                                         const InterpFrame *Frame,
                                         const Function *Func,
@@ -2217,7 +2264,7 @@ static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
 
   const ASTContext &ASTCtx = S.getASTContext();
 
-  unsigned ByteOffset = 0;
+  unsigned ByteOffset = computePointerOffset(ASTCtx, Ptr);
   unsigned FullSize = computeFullDescSize(ASTCtx, DeclDesc);
 
   pushInteger(S, FullSize - ByteOffset, Call->getType());
diff --git a/clang/test/AST/ByteCode/builtin-object-size.cpp b/clang/test/AST/ByteCode/builtin-object-size.cpp
index 62bb1642c5301..6f4ef54bcbafa 100644
--- a/clang/test/AST/ByteCode/builtin-object-size.cpp
+++ b/clang/test/AST/ByteCode/builtin-object-size.cpp
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=both,expected %s
 // RUN: %clang_cc1                                         -verify=both,ref      %s
 
-// both-no-diagnostics
+// ref-no-diagnostics
+
+typedef __SIZE_TYPE__ size_t;
 
 int a;
 static_assert(__builtin_object_size(&a, 0) == sizeof(int), "");
@@ -12,3 +14,43 @@ static_assert(__builtin_object_size(&arr, 0) == (sizeof(int)*2), "");
 
 float arrf[2];
 static_assert(__builtin_object_size(&arrf, 0) == (sizeof(float)*2), "");
+static_assert(__builtin_object_size(&arrf[1], 0) == sizeof(float), "");
+static_assert(__builtin_object_size(&arrf[2], 0) == 0, "");
+
+
+
+struct S {
+  int a;
+  char c;
+};
+
+S s;
+static_assert(__builtin_object_size(&s, 0) == sizeof(s), "");
+
+S ss[2];
+static_assert(__builtin_object_size(&ss[1], 0) == sizeof(s), "");
+static_assert(__builtin_object_size(&ss[1].c, 0) == sizeof(int), "");
+
+struct A { char buf[16]; };
+struct B : A {};
+struct C { int i; B bs[1]; } c;
+static_assert(__builtin_object_size(&c.bs[0], 3) == 16);
+static_assert(__builtin_object_size(&c.bs[1], 3) == 0);
+
+/// These are from test/SemaCXX/builtin-object-size-cxx14.
+/// They all don't work since they are rejected when evaluating the first
+/// parameter of the __builtin_object_size call.
+///
+/// GCC rejects them as well.
+namespace InvalidBase {
+  // Ensure this doesn't crash.
+  struct S { const char *name; };
+  S invalid_base(); // expected-note {{declared here}}
+  constexpr size_t bos_name = __builtin_object_size(invalid_base().name, 1); // expected-error {{must be initialized by a constant expression}} \
+                                                                             // expected-note {{non-constexpr function 'invalid_base'}}
+
+  struct T { ~T(); };
+  T invalid_base_2();
+  constexpr size_t bos_dtor = __builtin_object_size(&(T&)(T&&)invalid_base_2(), 0); // expected-error {{must be initialized by a constant expression}} \
+                                                                                    // expected-note {{non-literal type 'T'}}
+}

From f0cc50cdc969d141015cd132cf56a2ab8447f763 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 11:20:32 -0700
Subject: [PATCH 586/710] [ADT] Add hash_combine_range that takes a range (NFC)
 (#136459)

The new function will allow us to replace:

  hash_combine_range(Ops.begin(), Ops.end())

with:

  hash_combine_range(Ops)
---
 llvm/include/llvm/ADT/Hashing.h    | 5 +++++
 llvm/unittests/ADT/HashingTest.cpp | 4 ++++
 2 files changed, 9 insertions(+)

diff --git a/llvm/include/llvm/ADT/Hashing.h b/llvm/include/llvm/ADT/Hashing.h
index 17dcf31c616aa..9b8643b7765cf 100644
--- a/llvm/include/llvm/ADT/Hashing.h
+++ b/llvm/include/llvm/ADT/Hashing.h
@@ -44,6 +44,7 @@
 #ifndef LLVM_ADT_HASHING_H
 #define LLVM_ADT_HASHING_H
 
+#include "llvm/ADT/ADL.h"
 #include "llvm/Config/abi-breaking.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -469,6 +470,10 @@ hash_code hash_combine_range(InputIteratorT first, InputIteratorT last) {
   return ::llvm::hashing::detail::hash_combine_range_impl(first, last);
 }
 
+// A wrapper for hash_combine_range above.
+template <typename RangeT> hash_code hash_combine_range(RangeT &&R) {
+  return hash_combine_range(adl_begin(R), adl_end(R));
+}
 
 // Implementation details for hash_combine.
 namespace hashing {
diff --git a/llvm/unittests/ADT/HashingTest.cpp b/llvm/unittests/ADT/HashingTest.cpp
index c28356e229e66..e116ee934a0a4 100644
--- a/llvm/unittests/ADT/HashingTest.cpp
+++ b/llvm/unittests/ADT/HashingTest.cpp
@@ -166,15 +166,19 @@ TEST(HashingTest, HashCombineRangeBasicTest) {
   hash_code arr1_hash = hash_combine_range(begin(arr1), end(arr1));
   EXPECT_NE(dummy_hash, arr1_hash);
   EXPECT_EQ(arr1_hash, hash_combine_range(begin(arr1), end(arr1)));
+  EXPECT_EQ(arr1_hash, hash_combine_range(arr1));
 
   const std::vector<int> vec(begin(arr1), end(arr1));
   EXPECT_EQ(arr1_hash, hash_combine_range(vec.begin(), vec.end()));
+  EXPECT_EQ(arr1_hash, hash_combine_range(vec));
 
   const std::list<int> list(begin(arr1), end(arr1));
   EXPECT_EQ(arr1_hash, hash_combine_range(list.begin(), list.end()));
+  EXPECT_EQ(arr1_hash, hash_combine_range(list));
 
   const std::deque<int> deque(begin(arr1), end(arr1));
   EXPECT_EQ(arr1_hash, hash_combine_range(deque.begin(), deque.end()));
+  EXPECT_EQ(arr1_hash, hash_combine_range(deque));
 
   const int arr2[] = { 3, 2, 1 };
   hash_code arr2_hash = hash_combine_range(begin(arr2), end(arr2));

From 842e5915778a820c63cf38b75bec932a6ea8c18b Mon Sep 17 00:00:00 2001
From: Baranov Victor <bar.victor.2002@gmail.com>
Date: Sun, 20 Apr 2025 21:41:13 +0300
Subject: [PATCH 587/710] [clang-tidy][NFC] fix `clang-tidy` warnings in
 `clang-tools-extra/clang-tidy` directory (#136097)

Mostly stylistic changes to `clang-tidy` source code.

Command run:
`python3 clang-tools-extra/clang-tidy/tool/run-clang-tidy.py -p build/
-j $(nproc) clang-tools-extra/clang-tidy`
---
 .../clang-tidy/abseil/CleanupCtadCheck.cpp    | 10 +-
 .../clang-tidy/bugprone/BranchCloneCheck.cpp  | 10 +-
 .../bugprone/SignalHandlerCheck.cpp           |  2 +-
 .../bugprone/StandaloneEmptyCheck.cpp         |  4 +-
 .../bugprone/StringviewNullptrCheck.cpp       | 97 +++++++++----------
 .../bugprone/TaggedUnionMemberCountCheck.cpp  | 11 +--
 .../clang-tidy/hicpp/NoAssemblerCheck.cpp     |  2 +-
 .../misc/ConfusableIdentifierCheck.cpp        |  6 +-
 .../ConfusableTable/BuildConfusableTable.cpp  | 24 ++---
 .../clang-tidy/modernize/MacroToEnumCheck.cpp |  6 +-
 .../clang-tidy/objc/AssertEquals.cpp          | 36 +++----
 .../performance/MoveConstArgCheck.cpp         |  4 +-
 .../portability/StdAllocatorConstCheck.cpp    | 14 +--
 .../clang-tidy/tool/ClangTidyMain.cpp         |  2 +-
 .../clang-tidy/utils/ExceptionAnalyzer.cpp    | 16 +--
 .../clang-tidy/utils/ExprSequence.cpp         |  8 +-
 16 files changed, 126 insertions(+), 126 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp b/clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp
index 06f98c76269b5..664ec59997b59 100644
--- a/clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp
@@ -21,9 +21,9 @@ using namespace ::clang::transformer;
 
 namespace clang::tidy::abseil {
 
-RewriteRuleWith<std::string> CleanupCtadCheckImpl() {
-  auto warning_message = cat("prefer absl::Cleanup's class template argument "
-                             "deduction pattern in C++17 and higher");
+RewriteRuleWith<std::string> cleanupCtadCheckImpl() {
+  auto WarningMessage = cat("prefer absl::Cleanup's class template argument "
+                            "deduction pattern in C++17 and higher");
 
   return makeRule(
       declStmt(hasSingleDecl(varDecl(
@@ -34,10 +34,10 @@ RewriteRuleWith<std::string> CleanupCtadCheckImpl() {
                   .bind("make_cleanup_call")))))),
       {changeTo(node("auto_type_loc"), cat("absl::Cleanup")),
        changeTo(node("make_cleanup_call"), cat(callArgs("make_cleanup_call")))},
-      warning_message);
+      WarningMessage);
 }
 
 CleanupCtadCheck::CleanupCtadCheck(StringRef Name, ClangTidyContext *Context)
-    : utils::TransformerClangTidyCheck(CleanupCtadCheckImpl(), Name, Context) {}
+    : utils::TransformerClangTidyCheck(cleanupCtadCheckImpl(), Name, Context) {}
 
 } // namespace clang::tidy::abseil
diff --git a/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
index a7e25141b3fe2..a544ef0d9dd04 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
@@ -250,10 +250,10 @@ static bool isIdenticalStmt(const ASTContext &Ctx, const Stmt *Stmt1,
 
     if (!llvm::all_of(llvm::zip(CompStmt1->body(), CompStmt2->body()),
                       [&Ctx, IgnoreSideEffects](
-                          std::tuple<const Stmt *, const Stmt *> stmtPair) {
-                        const Stmt *stmt0 = std::get<0>(stmtPair);
-                        const Stmt *stmt1 = std::get<1>(stmtPair);
-                        return isIdenticalStmt(Ctx, stmt0, stmt1,
+                          std::tuple<const Stmt *, const Stmt *> StmtPair) {
+                        const Stmt *Stmt0 = std::get<0>(StmtPair);
+                        const Stmt *Stmt1 = std::get<1>(StmtPair);
+                        return isIdenticalStmt(Ctx, Stmt0, Stmt1,
                                                IgnoreSideEffects);
                       })) {
       return false;
@@ -477,7 +477,7 @@ void BranchCloneCheck::check(const MatchFinder::MatchResult &Result) {
 
   if (const auto *IS = Result.Nodes.getNodeAs<IfStmt>("ifWithDescendantIf")) {
     const Stmt *Then = IS->getThen();
-    auto CS = dyn_cast<CompoundStmt>(Then);
+    const auto *CS = dyn_cast<CompoundStmt>(Then);
     if (CS && (!CS->body_empty())) {
       const auto *InnerIf = dyn_cast<IfStmt>(*CS->body_begin());
       if (InnerIf && isIdenticalStmt(Context, IS->getCond(), InnerIf->getCond(),
diff --git a/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
index 27045816a80d3..c066b3e7b19a5 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
@@ -301,7 +301,7 @@ bool isCXXOnlyStmt(const Stmt *S) {
 /// It is unspecified which call is found if multiple calls exist, but the order
 /// should be deterministic (depend only on the AST).
 Expr *findCallExpr(const CallGraphNode *Caller, const CallGraphNode *Callee) {
-  auto FoundCallee = llvm::find_if(
+  const auto *FoundCallee = llvm::find_if(
       Caller->callees(), [Callee](const CallGraphNode::CallRecord &Call) {
         return Call.Callee == Callee;
       });
diff --git a/clang-tools-extra/clang-tidy/bugprone/StandaloneEmptyCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/StandaloneEmptyCheck.cpp
index a1d7b9931e419..7114adfb311bc 100644
--- a/clang-tools-extra/clang-tidy/bugprone/StandaloneEmptyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/StandaloneEmptyCheck.cpp
@@ -99,8 +99,8 @@ void StandaloneEmptyCheck::check(const MatchFinder::MatchResult &Result) {
   if (Result.Nodes.getNodeAs<Expr>("parent"))
     return;
 
-  const auto PParentStmtExpr = Result.Nodes.getNodeAs<Expr>("stexpr");
-  const auto ParentCompStmt = Result.Nodes.getNodeAs<CompoundStmt>("parent");
+  const auto *PParentStmtExpr = Result.Nodes.getNodeAs<Expr>("stexpr");
+  const auto *ParentCompStmt = Result.Nodes.getNodeAs<CompoundStmt>("parent");
   const auto *ParentCond = getCondition(Result.Nodes, "parent");
   const auto *ParentReturnStmt = Result.Nodes.getNodeAs<ReturnStmt>("parent");
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp
index ea50250f829f0..2489de17b740b 100644
--- a/clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp
@@ -35,23 +35,23 @@ AST_MATCHER(clang::VarDecl, isDirectInitialization) {
 
 } // namespace
 
-RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
-  auto construction_warning =
+RewriteRuleWith<std::string> stringviewNullptrCheckImpl() {
+  auto ConstructionWarning =
       cat("constructing basic_string_view from null is undefined; replace with "
           "the default constructor");
-  auto static_cast_warning =
+  auto StaticCastWarning =
       cat("casting to basic_string_view from null is undefined; replace with "
           "the empty string");
-  auto argument_construction_warning =
+  auto ArgumentConstructionWarning =
       cat("passing null as basic_string_view is undefined; replace with the "
           "empty string");
-  auto assignment_warning =
+  auto AssignmentWarning =
       cat("assignment to basic_string_view from null is undefined; replace "
           "with the default constructor");
-  auto relative_comparison_warning =
+  auto RelativeComparisonWarning =
       cat("comparing basic_string_view to null is undefined; replace with the "
           "empty string");
-  auto equality_comparison_warning =
+  auto EqualityComparisonWarning =
       cat("comparing basic_string_view to null is undefined; replace with the "
           "emptiness query");
 
@@ -84,7 +84,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
   auto HandleTemporaryCXXFunctionalCastExpr =
       makeRule(cxxFunctionalCastExpr(hasSourceExpression(
                    BasicStringViewConstructingFromNullExpr)),
-               remove(node("null_arg_expr")), construction_warning);
+               remove(node("null_arg_expr")), ConstructionWarning);
 
   // `std::string_view{null_arg_expr}` and `(std::string_view){null_arg_expr}`
   auto HandleTemporaryCXXTemporaryObjectExprAndCompoundLiteralExpr = makeRule(
@@ -93,28 +93,28 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
           hasAnyArgument(/* `hasArgument` would skip over parens */ anyOf(
               NullLiteral, NullInitList, EmptyInitList)),
           has(expr().bind("null_arg_expr")))),
-      remove(node("null_arg_expr")), construction_warning);
+      remove(node("null_arg_expr")), ConstructionWarning);
 
   // `(std::string_view) null_arg_expr`
-  auto HandleTemporaryCStyleCastExpr = makeRule(
-      cStyleCastExpr(
-          hasSourceExpression(BasicStringViewConstructingFromNullExpr)),
-      changeTo(node("null_arg_expr"), cat("{}")), construction_warning);
+  auto HandleTemporaryCStyleCastExpr =
+      makeRule(cStyleCastExpr(hasSourceExpression(
+                   BasicStringViewConstructingFromNullExpr)),
+               changeTo(node("null_arg_expr"), cat("{}")), ConstructionWarning);
 
   // `static_cast<std::string_view>(null_arg_expr)`
-  auto HandleTemporaryCXXStaticCastExpr = makeRule(
-      cxxStaticCastExpr(
-          hasSourceExpression(BasicStringViewConstructingFromNullExpr)),
-      changeTo(node("null_arg_expr"), cat("\"\"")), static_cast_warning);
+  auto HandleTemporaryCXXStaticCastExpr =
+      makeRule(cxxStaticCastExpr(hasSourceExpression(
+                   BasicStringViewConstructingFromNullExpr)),
+               changeTo(node("null_arg_expr"), cat("\"\"")), StaticCastWarning);
 
   // `std::string_view sv = null_arg_expr;`
-  auto HandleStackCopyInitialization = makeRule(
-      varDecl(HasBasicStringViewType,
-              hasInitializer(ignoringImpCasts(
-                  cxxConstructExpr(BasicStringViewConstructingFromNullExpr,
-                                   unless(isListInitialization())))),
-              unless(isDirectInitialization())),
-      changeTo(node("null_arg_expr"), cat("{}")), construction_warning);
+  auto HandleStackCopyInitialization =
+      makeRule(varDecl(HasBasicStringViewType,
+                       hasInitializer(ignoringImpCasts(cxxConstructExpr(
+                           BasicStringViewConstructingFromNullExpr,
+                           unless(isListInitialization())))),
+                       unless(isDirectInitialization())),
+               changeTo(node("null_arg_expr"), cat("{}")), ConstructionWarning);
 
   // `std::string_view sv = {null_arg_expr};`
   auto HandleStackCopyListInitialization =
@@ -123,7 +123,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                            BasicStringViewConstructingFromNullExpr,
                            isListInitialization())),
                        unless(isDirectInitialization())),
-               remove(node("null_arg_expr")), construction_warning);
+               remove(node("null_arg_expr")), ConstructionWarning);
 
   // `std::string_view sv(null_arg_expr);`
   auto HandleStackDirectInitialization =
@@ -134,7 +134,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                        isDirectInitialization())
                    .bind("var_decl"),
                changeTo(node("construct_expr"), cat(name("var_decl"))),
-               construction_warning);
+               ConstructionWarning);
 
   // `std::string_view sv{null_arg_expr};`
   auto HandleStackDirectListInitialization =
@@ -143,7 +143,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                            BasicStringViewConstructingFromNullExpr,
                            isListInitialization())),
                        isDirectInitialization()),
-               remove(node("null_arg_expr")), construction_warning);
+               remove(node("null_arg_expr")), ConstructionWarning);
 
   // `struct S { std::string_view sv = null_arg_expr; };`
   auto HandleFieldInClassCopyInitialization = makeRule(
@@ -151,7 +151,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                 hasInClassInitializer(ignoringImpCasts(
                     cxxConstructExpr(BasicStringViewConstructingFromNullExpr,
                                      unless(isListInitialization()))))),
-      changeTo(node("null_arg_expr"), cat("{}")), construction_warning);
+      changeTo(node("null_arg_expr"), cat("{}")), ConstructionWarning);
 
   // `struct S { std::string_view sv = {null_arg_expr}; };` and
   // `struct S { std::string_view sv{null_arg_expr}; };`
@@ -160,7 +160,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                 hasInClassInitializer(ignoringImpCasts(
                     cxxConstructExpr(BasicStringViewConstructingFromNullExpr,
                                      isListInitialization())))),
-      remove(node("null_arg_expr")), construction_warning);
+      remove(node("null_arg_expr")), ConstructionWarning);
 
   // `class C { std::string_view sv; C() : sv(null_arg_expr) {} };`
   auto HandleConstructorDirectInitialization =
@@ -168,7 +168,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                                   withInitializer(cxxConstructExpr(
                                       BasicStringViewConstructingFromNullExpr,
                                       unless(isListInitialization())))),
-               remove(node("null_arg_expr")), construction_warning);
+               remove(node("null_arg_expr")), ConstructionWarning);
 
   // `class C { std::string_view sv; C() : sv{null_arg_expr} {} };`
   auto HandleConstructorDirectListInitialization =
@@ -176,15 +176,15 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                                   withInitializer(cxxConstructExpr(
                                       BasicStringViewConstructingFromNullExpr,
                                       isListInitialization()))),
-               remove(node("null_arg_expr")), construction_warning);
+               remove(node("null_arg_expr")), ConstructionWarning);
 
   // `void f(std::string_view sv = null_arg_expr);`
-  auto HandleDefaultArgumentCopyInitialization = makeRule(
-      parmVarDecl(HasBasicStringViewType,
-                  hasInitializer(ignoringImpCasts(
-                      cxxConstructExpr(BasicStringViewConstructingFromNullExpr,
-                                       unless(isListInitialization()))))),
-      changeTo(node("null_arg_expr"), cat("{}")), construction_warning);
+  auto HandleDefaultArgumentCopyInitialization =
+      makeRule(parmVarDecl(HasBasicStringViewType,
+                           hasInitializer(ignoringImpCasts(cxxConstructExpr(
+                               BasicStringViewConstructingFromNullExpr,
+                               unless(isListInitialization()))))),
+               changeTo(node("null_arg_expr"), cat("{}")), ConstructionWarning);
 
   // `void f(std::string_view sv = {null_arg_expr});`
   auto HandleDefaultArgumentCopyListInitialization =
@@ -192,21 +192,21 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                            hasInitializer(cxxConstructExpr(
                                BasicStringViewConstructingFromNullExpr,
                                isListInitialization()))),
-               remove(node("null_arg_expr")), construction_warning);
+               remove(node("null_arg_expr")), ConstructionWarning);
 
   // `new std::string_view(null_arg_expr)`
   auto HandleHeapDirectInitialization = makeRule(
       cxxNewExpr(has(cxxConstructExpr(BasicStringViewConstructingFromNullExpr,
                                       unless(isListInitialization()))),
                  unless(isArray()), unless(hasAnyPlacementArg(anything()))),
-      remove(node("null_arg_expr")), construction_warning);
+      remove(node("null_arg_expr")), ConstructionWarning);
 
   // `new std::string_view{null_arg_expr}`
   auto HandleHeapDirectListInitialization = makeRule(
       cxxNewExpr(has(cxxConstructExpr(BasicStringViewConstructingFromNullExpr,
                                       isListInitialization())),
                  unless(isArray()), unless(hasAnyPlacementArg(anything()))),
-      remove(node("null_arg_expr")), construction_warning);
+      remove(node("null_arg_expr")), ConstructionWarning);
 
   // `function(null_arg_expr)`
   auto HandleFunctionArgumentInitialization =
@@ -214,22 +214,21 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                             BasicStringViewConstructingFromNullExpr)),
                         unless(cxxOperatorCallExpr())),
                changeTo(node("construct_expr"), cat("\"\"")),
-               argument_construction_warning);
+               ArgumentConstructionWarning);
 
   // `sv = null_arg_expr`
   auto HandleAssignment = makeRule(
       cxxOperatorCallExpr(hasOverloadedOperatorName("="),
                           hasRHS(materializeTemporaryExpr(
                               has(BasicStringViewConstructingFromNullExpr)))),
-      changeTo(node("construct_expr"), cat("{}")), assignment_warning);
+      changeTo(node("construct_expr"), cat("{}")), AssignmentWarning);
 
   // `sv < null_arg_expr`
   auto HandleRelativeComparison = makeRule(
       cxxOperatorCallExpr(hasAnyOverloadedOperatorName("<", "<=", ">", ">="),
                           hasEitherOperand(ignoringImpCasts(
                               BasicStringViewConstructingFromNullExpr))),
-      changeTo(node("construct_expr"), cat("\"\"")),
-      relative_comparison_warning);
+      changeTo(node("construct_expr"), cat("\"\"")), RelativeComparisonWarning);
 
   // `sv == null_arg_expr`
   auto HandleEmptyEqualityComparison = makeRule(
@@ -240,7 +239,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                                expr().bind("instance"))))
           .bind("root"),
       changeTo(node("root"), cat(access("instance", cat("empty")), "()")),
-      equality_comparison_warning);
+      EqualityComparisonWarning);
 
   // `sv != null_arg_expr`
   auto HandleNonEmptyEqualityComparison = makeRule(
@@ -251,13 +250,13 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                                expr().bind("instance"))))
           .bind("root"),
       changeTo(node("root"), cat("!", access("instance", cat("empty")), "()")),
-      equality_comparison_warning);
+      EqualityComparisonWarning);
 
   // `return null_arg_expr;`
   auto HandleReturnStatement = makeRule(
       returnStmt(hasReturnValue(
           ignoringImpCasts(BasicStringViewConstructingFromNullExpr))),
-      changeTo(node("construct_expr"), cat("{}")), construction_warning);
+      changeTo(node("construct_expr"), cat("{}")), ConstructionWarning);
 
   // `T(null_arg_expr)`
   auto HandleConstructorInvocation =
@@ -267,7 +266,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
                                       BasicStringViewConstructingFromNullExpr)),
                    unless(HasBasicStringViewType)),
                changeTo(node("construct_expr"), cat("\"\"")),
-               argument_construction_warning);
+               ArgumentConstructionWarning);
 
   return applyFirst(
       {HandleTemporaryCXXFunctionalCastExpr,
@@ -297,7 +296,7 @@ RewriteRuleWith<std::string> StringviewNullptrCheckImpl() {
 
 StringviewNullptrCheck::StringviewNullptrCheck(StringRef Name,
                                                ClangTidyContext *Context)
-    : utils::TransformerClangTidyCheck(StringviewNullptrCheckImpl(), Name,
+    : utils::TransformerClangTidyCheck(stringviewNullptrCheckImpl(), Name,
                                        Context) {}
 
 } // namespace clang::tidy::bugprone
diff --git a/clang-tools-extra/clang-tidy/bugprone/TaggedUnionMemberCountCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/TaggedUnionMemberCountCheck.cpp
index db99ef3786e5f..db0ac281ddfcf 100644
--- a/clang-tools-extra/clang-tidy/bugprone/TaggedUnionMemberCountCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/TaggedUnionMemberCountCheck.cpp
@@ -52,9 +52,8 @@ AST_MATCHER_P2(RecordDecl, fieldCountOfKindIsOne,
     if (InnerMatcher.matches(*Field, Finder, &TempBuilder)) {
       if (FirstMatch) {
         return false;
-      } else {
-        FirstMatch = Field;
       }
+      FirstMatch = Field;
     }
   }
 
@@ -112,11 +111,11 @@ void TaggedUnionMemberCountCheck::registerMatchers(MatchFinder *Finder) {
   auto EnumField = fieldDecl(hasType(
       qualType(hasCanonicalType(enumType(hasDeclaration(enumDecl()))))));
 
-  auto hasOneUnionField = fieldCountOfKindIsOne(UnionField, UnionMatchBindName);
-  auto hasOneEnumField = fieldCountOfKindIsOne(EnumField, TagMatchBindName);
+  auto HasOneUnionField = fieldCountOfKindIsOne(UnionField, UnionMatchBindName);
+  auto HasOneEnumField = fieldCountOfKindIsOne(EnumField, TagMatchBindName);
 
-  Finder->addMatcher(recordDecl(anyOf(isStruct(), isClass()), hasOneUnionField,
-                                hasOneEnumField, unless(isImplicit()))
+  Finder->addMatcher(recordDecl(anyOf(isStruct(), isClass()), HasOneUnionField,
+                                HasOneEnumField, unless(isImplicit()))
                          .bind(RootMatchBindName),
                      this);
 }
diff --git a/clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.cpp b/clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.cpp
index 1dc2463d0fc0a..9f31227d9753b 100644
--- a/clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.cpp
+++ b/clang-tools-extra/clang-tidy/hicpp/NoAssemblerCheck.cpp
@@ -18,7 +18,7 @@ namespace {
 AST_MATCHER(VarDecl, isAsm) { return Node.hasAttr<clang::AsmLabelAttr>(); }
 const ast_matchers::internal::VariadicDynCastAllOfMatcher<Decl,
                                                           FileScopeAsmDecl>
-    fileScopeAsmDecl;
+    fileScopeAsmDecl; // NOLINT(readability-identifier-*) preserve clang style
 } // namespace
 
 void NoAssemblerCheck::registerMatchers(MatchFinder *Finder) {
diff --git a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
index 6df565c9a9d69..b45a3ebcf6c4b 100644
--- a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
@@ -66,9 +66,9 @@ static llvm::SmallString<64U> skeleton(StringRef Name) {
     }
 
     StringRef Key(Prev, Curr - Prev);
-    auto Where = llvm::lower_bound(ConfusableEntries, CodePoint,
-                                   [](decltype(ConfusableEntries[0]) x,
-                                      UTF32 y) { return x.codepoint < y; });
+    auto *Where = llvm::lower_bound(ConfusableEntries, CodePoint,
+                                    [](decltype(ConfusableEntries[0]) X,
+                                       UTF32 Y) { return X.codepoint < Y; });
     if (Where == std::end(ConfusableEntries) || CodePoint != Where->codepoint) {
       Skeleton.append(Prev, Curr);
     } else {
diff --git a/clang-tools-extra/clang-tidy/misc/ConfusableTable/BuildConfusableTable.cpp b/clang-tools-extra/clang-tidy/misc/ConfusableTable/BuildConfusableTable.cpp
index e269ab3983f36..57190942a1756 100644
--- a/clang-tools-extra/clang-tidy/misc/ConfusableTable/BuildConfusableTable.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConfusableTable/BuildConfusableTable.cpp
@@ -20,7 +20,7 @@ int main(int argc, char *argv[]) {
     return 1;
   std::unique_ptr<MemoryBuffer> Buffer = std::move(ErrorOrBuffer.get());
   StringRef Content = Buffer->getBuffer();
-  Content = Content.drop_until([](char c) { return c == '#'; });
+  Content = Content.drop_until([](char C) { return C == '#'; });
   SmallVector<StringRef> Lines;
   SplitString(Content, Lines, "\r\n");
 
@@ -44,9 +44,9 @@ int main(int argc, char *argv[]) {
     SmallVector<llvm::UTF32> To;
     SmallVector<StringRef> ToN;
     Values[1].split(ToN, ' ', -1, false);
-    for (StringRef To_ : ToN) {
+    for (StringRef ToI : ToN) {
       llvm::UTF32 ToCodePoint = 0;
-      To_.trim().getAsInteger(16, ToCodePoint);
+      ToI.trim().getAsInteger(16, ToCodePoint);
       To.push_back(ToCodePoint);
     }
     // Sentinel
@@ -63,23 +63,23 @@ int main(int argc, char *argv[]) {
                        })
           ->second.size();
 
-  std::error_code ec;
-  llvm::raw_fd_ostream os(argv[2], ec);
+  std::error_code Ec;
+  llvm::raw_fd_ostream Os(argv[2], Ec);
 
   // FIXME: If memory consumption and/or lookup time becomes a constraint, it
   // maybe worth using a more elaborate data structure.
-  os << "struct {llvm::UTF32 codepoint; llvm::UTF32 values[" << LargestValue
+  Os << "struct {llvm::UTF32 codepoint; llvm::UTF32 values[" << LargestValue
      << "];} "
         "ConfusableEntries[] = {\n";
   for (const auto &Values : Entries) {
-    os << "  { ";
-    os << Values.first;
-    os << ", {";
+    Os << "  { ";
+    Os << Values.first;
+    Os << ", {";
     for (auto CP : Values.second)
-      os << CP << ", ";
+      Os << CP << ", ";
 
-    os << "}},\n";
+    Os << "}},\n";
   }
-  os << "};\n";
+  Os << "};\n";
   return 0;
 }
diff --git a/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
index 0b47ed316ca27..c2bbd2e563d8f 100644
--- a/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
@@ -321,8 +321,8 @@ bool MacroToEnumCallbacks::isInitializer(ArrayRef<Token> MacroTokens)
 {
   IntegralLiteralExpressionMatcher Matcher(MacroTokens, LangOpts.C99 == 0);
   bool Matched = Matcher.match();
-  bool isC = !LangOpts.CPlusPlus;
-  if (isC && (Matcher.largestLiteralSize() != LiteralSize::Int &&
+  bool IsC = !LangOpts.CPlusPlus;
+  if (IsC && (Matcher.largestLiteralSize() != LiteralSize::Int &&
               Matcher.largestLiteralSize() != LiteralSize::UnsignedInt))
     return false;
 
@@ -375,7 +375,7 @@ void MacroToEnumCallbacks::MacroUndefined(const Token &MacroNameTok,
     return getTokenName(Macro.Name) == getTokenName(MacroNameTok);
   };
 
-  auto It = llvm::find_if(Enums, [MatchesToken](const MacroList &MacroList) {
+  auto *It = llvm::find_if(Enums, [MatchesToken](const MacroList &MacroList) {
     return llvm::any_of(MacroList, MatchesToken);
   });
   if (It != Enums.end())
diff --git a/clang-tools-extra/clang-tidy/objc/AssertEquals.cpp b/clang-tools-extra/clang-tidy/objc/AssertEquals.cpp
index da98e19c67412..3d9b9fa401910 100644
--- a/clang-tools-extra/clang-tidy/objc/AssertEquals.cpp
+++ b/clang-tools-extra/clang-tidy/objc/AssertEquals.cpp
@@ -16,44 +16,44 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::objc {
 
 // Mapping from `XCTAssert*Equal` to `XCTAssert*EqualObjects` name.
-static const std::map<std::string, std::string> &NameMap() {
-  static std::map<std::string, std::string> map{
+static const std::map<std::string, std::string> &nameMap() {
+  static std::map<std::string, std::string> Map{
       {"XCTAssertEqual", "XCTAssertEqualObjects"},
       {"XCTAssertNotEqual", "XCTAssertNotEqualObjects"},
 
   };
-  return map;
+  return Map;
 }
 
-void AssertEquals::registerMatchers(MatchFinder *finder) {
-  for (const auto &pair : NameMap()) {
-    finder->addMatcher(
+void AssertEquals::registerMatchers(MatchFinder *Finder) {
+  for (const auto &Pair : nameMap()) {
+    Finder->addMatcher(
         binaryOperator(anyOf(hasOperatorName("!="), hasOperatorName("==")),
-                       isExpandedFromMacro(pair.first),
+                       isExpandedFromMacro(Pair.first),
                        anyOf(hasLHS(hasType(qualType(
                                  hasCanonicalType(asString("NSString *"))))),
                              hasRHS(hasType(qualType(
                                  hasCanonicalType(asString("NSString *"))))))
 
                            )
-            .bind(pair.first),
+            .bind(Pair.first),
         this);
   }
 }
 
-void AssertEquals::check(const ast_matchers::MatchFinder::MatchResult &result) {
-  for (const auto &pair : NameMap()) {
-    if (const auto *root = result.Nodes.getNodeAs<BinaryOperator>(pair.first)) {
-      SourceManager *sm = result.SourceManager;
+void AssertEquals::check(const ast_matchers::MatchFinder::MatchResult &Result) {
+  for (const auto &Pair : nameMap()) {
+    if (const auto *Root = Result.Nodes.getNodeAs<BinaryOperator>(Pair.first)) {
+      SourceManager *Sm = Result.SourceManager;
       // The macros are nested two levels, so going up twice.
-      auto macro_callsite = sm->getImmediateMacroCallerLoc(
-          sm->getImmediateMacroCallerLoc(root->getBeginLoc()));
-      diag(macro_callsite, "use " + pair.second + " for comparing objects")
+      auto MacroCallsite = Sm->getImmediateMacroCallerLoc(
+          Sm->getImmediateMacroCallerLoc(Root->getBeginLoc()));
+      diag(MacroCallsite, "use " + Pair.second + " for comparing objects")
           << FixItHint::CreateReplacement(
                  clang::CharSourceRange::getCharRange(
-                     macro_callsite,
-                     macro_callsite.getLocWithOffset(pair.first.length())),
-                 pair.second);
+                     MacroCallsite,
+                     MacroCallsite.getLocWithOffset(Pair.first.length())),
+                 Pair.second);
     }
   }
 }
diff --git a/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp b/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
index 703ad162f99cf..75ef7a697031e 100644
--- a/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
@@ -78,7 +78,7 @@ void MoveConstArgCheck::registerMatchers(MatchFinder *Finder) {
       this);
 }
 
-bool IsRValueReferenceParam(const Expr *Invocation,
+bool isRValueReferenceParam(const Expr *Invocation,
                             const QualType *InvocationParmType,
                             const Expr *Arg) {
   if (Invocation && (*InvocationParmType)->isRValueReferenceType() &&
@@ -147,7 +147,7 @@ void MoveConstArgCheck::check(const MatchFinder::MatchResult &Result) {
     // std::move shouldn't be removed when an lvalue wrapped by std::move is
     // passed to the function with an rvalue reference parameter.
     bool IsRVRefParam =
-        IsRValueReferenceParam(ReceivingExpr, InvocationParmType, Arg);
+        isRValueReferenceParam(ReceivingExpr, InvocationParmType, Arg);
     const auto *Var =
         IsVariable ? dyn_cast<DeclRefExpr>(Arg)->getDecl() : nullptr;
 
diff --git a/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.cpp b/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.cpp
index b34cb0153785d..3b4d65be7dfa1 100644
--- a/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.cpp
+++ b/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.cpp
@@ -15,12 +15,12 @@ namespace clang::tidy::portability {
 
 void StdAllocatorConstCheck::registerMatchers(MatchFinder *Finder) {
   // Match std::allocator<const T>.
-  auto allocatorConst =
+  auto AllocatorConst =
       recordType(hasDeclaration(classTemplateSpecializationDecl(
           hasName("::std::allocator"),
           hasTemplateArgument(0, refersToType(qualType(isConstQualified()))))));
 
-  auto hasContainerName =
+  auto HasContainerName =
       hasAnyName("::std::vector", "::std::deque", "::std::list",
                  "::std::multiset", "::std::set", "::std::unordered_multiset",
                  "::std::unordered_set", "::absl::flat_hash_set");
@@ -34,17 +34,17 @@ void StdAllocatorConstCheck::registerMatchers(MatchFinder *Finder) {
           templateSpecializationTypeLoc(),
           loc(hasUnqualifiedDesugaredType(anyOf(
               recordType(hasDeclaration(classTemplateSpecializationDecl(
-                  hasContainerName,
+                  HasContainerName,
                   anyOf(
-                      hasTemplateArgument(1, refersToType(allocatorConst)),
-                      hasTemplateArgument(2, refersToType(allocatorConst)),
-                      hasTemplateArgument(3, refersToType(allocatorConst)))))),
+                      hasTemplateArgument(1, refersToType(AllocatorConst)),
+                      hasTemplateArgument(2, refersToType(AllocatorConst)),
+                      hasTemplateArgument(3, refersToType(AllocatorConst)))))),
               // Match std::vector<const dependent>
               templateSpecializationType(
                   templateArgumentCountIs(1),
                   hasTemplateArgument(
                       0, refersToType(qualType(isConstQualified()))),
-                  hasDeclaration(namedDecl(hasContainerName)))))))
+                  hasDeclaration(namedDecl(HasContainerName)))))))
           .bind("type_loc"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
index fa8887e4639b4..7c3aee29c146d 100644
--- a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
+++ b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
@@ -33,7 +33,7 @@
 using namespace clang::tooling;
 using namespace llvm;
 
-static cl::desc desc(StringRef description) { return {description.ltrim()}; }
+static cl::desc desc(StringRef Description) { return {Description.ltrim()}; }
 
 static cl::OptionCategory ClangTidyCategory("clang-tidy options");
 
diff --git a/clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.cpp b/clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.cpp
index e28ee7d9c70f7..6ccc05845220f 100644
--- a/clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.cpp
+++ b/clang-tools-extra/clang-tidy/utils/ExceptionAnalyzer.cpp
@@ -209,12 +209,14 @@ bool isQualificationConvertiblePointer(QualType From, QualType To,
   // cv-decomposition of T, that is, cv_1, cv_2, ... , cv_n, is called the
   // cv-qualification signature of T.
 
-  auto isValidP_i = [](QualType P) {
+  // NOLINTNEXTLINE (readability-identifier-naming): Preserve original notation
+  auto IsValidP_i = [](QualType P) {
     return P->isPointerType() || P->isMemberPointerType() ||
            P->isConstantArrayType() || P->isIncompleteArrayType();
   };
 
-  auto isSameP_i = [](QualType P1, QualType P2) {
+  // NOLINTNEXTLINE (readability-identifier-naming): Preserve original notation
+  auto IsSameP_i = [](QualType P1, QualType P2) {
     if (P1->isPointerType())
       return P2->isPointerType();
 
@@ -273,7 +275,7 @@ bool isQualificationConvertiblePointer(QualType From, QualType To,
     return true;
   };
 
-  while (isValidP_i(From) && isValidP_i(To)) {
+  while (IsValidP_i(From) && IsValidP_i(To)) {
     // Remove every sugar.
     From = From.getCanonicalType();
     To = To.getCanonicalType();
@@ -281,7 +283,7 @@ bool isQualificationConvertiblePointer(QualType From, QualType To,
     if (!SatisfiesCVRules(From, To))
       return false;
 
-    if (!isSameP_i(From, To)) {
+    if (!IsSameP_i(From, To)) {
       if (LangOpts.CPlusPlus20) {
         if (From->isConstantArrayType() && !To->isIncompleteArrayType())
           return false;
@@ -309,7 +311,7 @@ bool isQualificationConvertiblePointer(QualType From, QualType To,
   }
 
   // In this case the length (n) of From and To are not the same.
-  if (isValidP_i(From) || isValidP_i(To))
+  if (IsValidP_i(From) || IsValidP_i(To))
     return false;
 
   // We hit U.
@@ -337,7 +339,7 @@ static bool canThrow(const FunctionDecl *Func) {
   case CT_Dependent: {
     const Expr *NoexceptExpr = FunProto->getNoexceptExpr();
     if (!NoexceptExpr)
-      return true; // no noexept - can throw
+      return true; // no noexcept - can throw
 
     if (NoexceptExpr->isValueDependent())
       return true; // depend on template - some instance can throw
@@ -569,7 +571,7 @@ ExceptionAnalyzer::ExceptionInfo ExceptionAnalyzer::throwsException(
     Results.merge(throwsException(Coro->getExceptionHandler(),
                                   Excs.getExceptionTypes(), CallStack));
     for (const Type *Throwable : Excs.getExceptionTypes()) {
-      if (const auto ThrowableRec = Throwable->getAsCXXRecordDecl()) {
+      if (const auto *ThrowableRec = Throwable->getAsCXXRecordDecl()) {
         ExceptionInfo DestructorExcs =
             throwsException(ThrowableRec->getDestructor(), Caught, CallStack);
         Results.merge(DestructorExcs);
diff --git a/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp b/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
index 685277d8bfbca..91736cf5e2f9a 100644
--- a/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
+++ b/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
@@ -71,12 +71,12 @@ bool isDescendantOfArgs(const Stmt *Descendant, const CallExpr *Call,
 
 llvm::SmallVector<const InitListExpr *>
 getAllInitListForms(const InitListExpr *InitList) {
-  llvm::SmallVector<const InitListExpr *> result = {InitList};
+  llvm::SmallVector<const InitListExpr *> Result = {InitList};
   if (const InitListExpr *AltForm = InitList->getSyntacticForm())
-    result.push_back(AltForm);
+    Result.push_back(AltForm);
   if (const InitListExpr *AltForm = InitList->getSemanticForm())
-    result.push_back(AltForm);
-  return result;
+    Result.push_back(AltForm);
+  return Result;
 }
 
 } // namespace

From 3ac1aa4c88d4fe40166209cad616f4ae867c20a2 Mon Sep 17 00:00:00 2001
From: Oliver Hunt <oliver@apple.com>
Date: Sun, 20 Apr 2025 14:16:51 -0700
Subject: [PATCH 588/710] [Clang] Consider preferred_type in bitfield warnings
 (#116760) (#116785)

Very simply extends the bitfield sema checks for assignment to fields
with a preferred type specified to consider the preferred type if the
decl storage type is not explicitly an enum type.

This does mean that if the preferred and explicit types have different
storage requirements we may not warn in all possible cases, but that's a
scenario for which the warnings are much more complex and confusing.
---
 clang/docs/ReleaseNotes.rst                   |   8 +
 clang/include/clang/Basic/DiagnosticGroups.td |   3 +
 .../clang/Basic/DiagnosticSemaKinds.td        |  17 +
 clang/lib/Sema/SemaChecking.cpp               |  36 +-
 .../bitfield-preferred-type-sizing.cpp        | 411 ++++++++++++++++++
 5 files changed, 468 insertions(+), 7 deletions(-)
 create mode 100644 clang/test/SemaCXX/bitfield-preferred-type-sizing.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 25302764778d1..7417fdd71a392 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -395,6 +395,14 @@ Improvements to Clang's diagnostics
   constructors to initialize their non-modifiable members. The diagnostic is
   not new; being controlled via a warning group is what's new. Fixes #GH41104
 
+- Improved bit-field diagnostics to consider the type specified by the
+  ``preferred_type`` attribute. These diagnostics are controlled by the flags
+  ``-Wpreferred-type-bitfield-enum-conversion`` and
+  ``-Wpreferred-type-bitfield-width``. These warnings are on by default as they
+  they're only triggered if the authors are already making the choice to use
+  ``preferred_type`` attribute.
+
+
 Improvements to Clang's time-trace
 ----------------------------------
 
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index b234d60fee8fc..b29fe40b05c6f 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -49,6 +49,9 @@ def SingleBitBitFieldConstantConversion :
   DiagGroup<"single-bit-bitfield-constant-conversion">;
 def BitFieldConstantConversion : DiagGroup<"bitfield-constant-conversion",
                                            [SingleBitBitFieldConstantConversion]>;
+def PreferredTypeBitFieldEnumConversion
+    : DiagGroup<"preferred-type-bitfield-enum-conversion">;
+def PreferredTypeBitFieldWidth : DiagGroup<"preferred-type-bitfield-width">;
 def BitFieldEnumConversion : DiagGroup<"bitfield-enum-conversion">;
 def BitFieldWidth : DiagGroup<"bitfield-width">;
 def CompoundTokenSplitByMacro : DiagGroup<"compound-token-split-by-macro">;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 8b8d3d7f6903b..45b6e1dc29980 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -6492,8 +6492,25 @@ def warn_signed_bitfield_enum_conversion : Warning<
   "signed bit-field %0 needs an extra bit to represent the largest positive "
   "enumerators of %1">,
   InGroup<BitFieldEnumConversion>, DefaultIgnore;
+def warn_preferred_type_bitfield_too_small_for_enum
+    : Warning<"bit-field %0 is not wide enough to store all enumerators of "
+              "preferred type %1">,
+      InGroup<PreferredTypeBitFieldEnumConversion>;
+def warn_preferred_type_unsigned_bitfield_assigned_signed_enum
+    : Warning<"assigning value of preferred signed enum type %1 to unsigned "
+              "bit-field %0; "
+              "negative enumerators of enum %1 will be converted to positive "
+              "values">,
+      InGroup<PreferredTypeBitFieldEnumConversion>;
+def warn_preferred_type_signed_bitfield_enum_conversion
+    : Warning<"signed bit-field %0 needs an extra bit to represent the largest "
+              "positive "
+              "enumerators of preferred type %1">,
+      InGroup<PreferredTypeBitFieldEnumConversion>;
 def note_change_bitfield_sign : Note<
   "consider making the bit-field type %select{unsigned|signed}0">;
+def note_bitfield_preferred_type
+    : Note<"preferred type for bit-field %0 specified here">;
 
 def warn_missing_braces : Warning<
   "suggest braces around initialization of subobject">,
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index c48e009fd7242..b2b26a2c39cf2 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -11293,9 +11293,16 @@ static bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
     // The RHS is not constant.  If the RHS has an enum type, make sure the
     // bitfield is wide enough to hold all the values of the enum without
     // truncation.
-    if (const auto *EnumTy = OriginalInit->getType()->getAs<EnumType>()) {
+    const auto *EnumTy = OriginalInit->getType()->getAs<EnumType>();
+    const PreferredTypeAttr *PTAttr = nullptr;
+    if (!EnumTy) {
+      PTAttr = Bitfield->getAttr<PreferredTypeAttr>();
+      if (PTAttr)
+        EnumTy = PTAttr->getType()->getAs<EnumType>();
+    }
+    if (EnumTy) {
       EnumDecl *ED = EnumTy->getDecl();
-      bool SignedBitfield = BitfieldType->isSignedIntegerType();
+      bool SignedBitfield = BitfieldType->isSignedIntegerOrEnumerationType();
 
       // Enum types are implicitly signed on Windows, so check if there are any
       // negative enumerators to see if the enum was intended to be signed or
@@ -11309,12 +11316,18 @@ static bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
       // on Windows where unfixed enums always use an underlying type of 'int'.
       unsigned DiagID = 0;
       if (SignedEnum && !SignedBitfield) {
-        DiagID = diag::warn_unsigned_bitfield_assigned_signed_enum;
+        DiagID =
+            PTAttr == nullptr
+                ? diag::warn_unsigned_bitfield_assigned_signed_enum
+                : diag::
+                      warn_preferred_type_unsigned_bitfield_assigned_signed_enum;
       } else if (SignedBitfield && !SignedEnum &&
                  ED->getNumPositiveBits() == FieldWidth) {
-        DiagID = diag::warn_signed_bitfield_enum_conversion;
+        DiagID =
+            PTAttr == nullptr
+                ? diag::warn_signed_bitfield_enum_conversion
+                : diag::warn_preferred_type_signed_bitfield_enum_conversion;
       }
-
       if (DiagID) {
         S.Diag(InitLoc, DiagID) << Bitfield << ED;
         TypeSourceInfo *TSI = Bitfield->getTypeSourceInfo();
@@ -11322,6 +11335,9 @@ static bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
             TSI ? TSI->getTypeLoc().getSourceRange() : SourceRange();
         S.Diag(Bitfield->getTypeSpecStartLoc(), diag::note_change_bitfield_sign)
             << SignedEnum << TypeRange;
+        if (PTAttr)
+          S.Diag(PTAttr->getLocation(), diag::note_bitfield_preferred_type)
+              << ED;
       }
 
       // Compute the required bitwidth. If the enum has negative values, we need
@@ -11334,10 +11350,16 @@ static bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
       // Check the bitwidth.
       if (BitsNeeded > FieldWidth) {
         Expr *WidthExpr = Bitfield->getBitWidth();
-        S.Diag(InitLoc, diag::warn_bitfield_too_small_for_enum)
-            << Bitfield << ED;
+        auto DiagID =
+            PTAttr == nullptr
+                ? diag::warn_bitfield_too_small_for_enum
+                : diag::warn_preferred_type_bitfield_too_small_for_enum;
+        S.Diag(InitLoc, DiagID) << Bitfield << ED;
         S.Diag(WidthExpr->getExprLoc(), diag::note_widen_bitfield)
             << BitsNeeded << ED << WidthExpr->getSourceRange();
+        if (PTAttr)
+          S.Diag(PTAttr->getLocation(), diag::note_bitfield_preferred_type)
+              << ED;
       }
     }
 
diff --git a/clang/test/SemaCXX/bitfield-preferred-type-sizing.cpp b/clang/test/SemaCXX/bitfield-preferred-type-sizing.cpp
new file mode 100644
index 0000000000000..5eaa3eb2d85e3
--- /dev/null
+++ b/clang/test/SemaCXX/bitfield-preferred-type-sizing.cpp
@@ -0,0 +1,411 @@
+// RUN: %clang_cc1 %s      -std=c++23 -triple=x86_64-apple-darwin10 -fsyntax-only -verify=expected,bitfieldwarnings,cpp -Wno-unused-value -Wno-unused-but-set-variable -Wbitfield-width -Wbitfield-enum-conversion
+// RUN: %clang_cc1 %s      -std=c++23 -triple=x86_64-apple-darwin10 -fsyntax-only -verify=expected,cpp -Wno-unused-value -Wno-unused-but-set-variable
+
+// RUN: %clang_cc1 %s -x c -std=c23   -triple=x86_64-apple-darwin10 -fsyntax-only -verify=expected,c -Wno-unused-value -Wno-unused-but-set-variable
+// RUN: %clang_cc1 %s -x c -std=c23   -triple=x86_64-apple-darwin10 -fsyntax-only -verify=expected,bitfieldwarnings,c -Wno-unused-value -Wno-unused-but-set-variable -Wbitfield-width -Wbitfield-enum-conversion
+
+
+typedef enum A {
+  A_a,
+  A_b,
+  A_c,
+  A_d
+} A;
+
+#ifdef __cplusplus
+#define DEFINE_ENUM(_Name, _Type, ...) enum class _Name : _Type { __VA_ARGS__ } ;
+#define ENUM_CLASS_REF(_Name, _Enum) _Name::_Enum
+#else
+#define DEFINE_ENUM(_Name, _Type, ...) typedef enum _Name : _Type { __VA_ARGS__ } _Name;
+#define ENUM_CLASS_REF(_Name, _Enum) _Enum
+#endif
+
+DEFINE_ENUM(B, int, B_a, B_b, B_c, B_d );
+DEFINE_ENUM(C, unsigned, C_a, C_b, C_c, C_d );
+DEFINE_ENUM(D, unsigned, D_a, D_b);
+
+// Not using templates here so we can more easily distinguish the responsible
+// party for each warning
+
+typedef struct S_A {
+  A field1 : 1; // #S_A_field1
+  A field2 : 2; // #S_A_field2
+  A field3 : 8; // #S_A_field3
+  __attribute__((preferred_type(A))) // #preferred_S_A_field4
+  unsigned field4 : 1; // #S_A_field4
+  __attribute__((preferred_type(A)))
+  unsigned field5 : 2; // #S_A_field5
+  __attribute__((preferred_type(A)))
+  unsigned field6 : 8; // #S_A_field6
+  __attribute__((preferred_type(A))) // #preferred_S_A_field7
+  int field7 : 1; // #S_A_field7
+  __attribute__((preferred_type(A))) // #preferred_S_A_field8
+  int field8 : 2; // #S_A_field8
+  __attribute__((preferred_type(A)))
+  int field9 : 8; // #S_A_field9
+  __attribute__((preferred_type(A)))
+  D field10 : 1; // #S_A_field10
+  __attribute__((preferred_type(A))) // #preferred_S_A_field11
+  D field11 : 2; // #S_A_field11
+  __attribute__((preferred_type(A)))
+  D field12 : 8; // #S_A_field12
+} S_A;
+
+typedef struct S_B {
+  B field1 : 1; // #S_B_field1
+  B field2 : 2; // #S_B_field2
+  B field3 : 8; // #S_B_field3
+  __attribute__((preferred_type(B))) // #preferred_S_B_field4
+  unsigned field4 : 1; // #S_B_field4
+  __attribute__((preferred_type(B)))
+  unsigned field5 : 2; // #S_B_field5
+  __attribute__((preferred_type(B)))
+  unsigned field6 : 8; // #S_B_field6
+  __attribute__((preferred_type(B))) // #preferred_S_B_field7
+  int field7 : 1; // #S_B_field7
+  __attribute__((preferred_type(B))) // #preferred_S_B_field8
+  int field8 : 2; // #S_B_field8
+  __attribute__((preferred_type(B)))
+  int field9 : 8; // #S_B_field9
+  __attribute__((preferred_type(B)))
+  D field10 : 1; // #S_B_field10
+  __attribute__((preferred_type(B))) // #preferred_S_B_field11
+  D field11 : 2; // #S_B_field11
+  __attribute__((preferred_type(B)))
+  D field12 : 8; // #S_B_field12
+} S_B;
+
+typedef struct S_C {
+  C field1 : 1; // #S_C_field1
+  C field2 : 2; // #S_C_field2
+  C field3 : 8; // #S_C_field3
+  __attribute__((preferred_type(C))) // #preferred_S_C_field4
+  unsigned field4 : 1; // #S_C_field4
+  __attribute__((preferred_type(C)))
+  unsigned field5 : 2; // #S_C_field5
+  __attribute__((preferred_type(C)))
+  unsigned field6 : 8; // #S_C_field6
+  __attribute__((preferred_type(C))) // #preferred_S_C_field7
+  int field7 : 1; // #S_C_field7
+  __attribute__((preferred_type(C))) // #preferred_S_C_field8
+  int field8 : 2; // #S_C_field8
+  __attribute__((preferred_type(C)))
+  int field9 : 8; // #S_C_field9
+  __attribute__((preferred_type(C)))
+  D field10 : 1; // #S_C_field10
+  __attribute__((preferred_type(C))) // #preferred_S_C_field11
+  D field11 : 2; // #S_C_field11
+  __attribute__((preferred_type(C)))
+  D field12 : 8; // #S_C_field12
+} S_C;
+
+void read_enumA(S_A *s) {
+  A x;
+  x = s->field1;
+  x = s->field2;
+  x = s->field3;
+  x = (A)s->field4;
+  x = (A)s->field5;
+  x = (A)s->field6;
+  x = (A)s->field7;
+  x = (A)s->field8;
+  x = (A)s->field9;
+  x = (A)s->field10;
+  x = (A)s->field11;
+  x = (A)s->field12;
+}
+
+void read_enumB(S_B *s) {
+  B x;
+  x = s->field1;
+  x = s->field2;
+  x = s->field3;
+  x = (B)s->field4;
+  x = (B)s->field5;
+  x = (B)s->field6;
+  x = (B)s->field7;
+  x = (B)s->field8;
+  x = (B)s->field9;
+  x = (B)s->field10;
+  x = (B)s->field11;
+  x = (B)s->field12;
+}
+
+void read_enumC(S_C *s) {
+  C x;
+  x = s->field1;
+  x = s->field2;
+  x = s->field3;
+  x = (C)s->field4;
+  x = (C)s->field5;
+  x = (C)s->field6;
+  x = (C)s->field7;
+  x = (C)s->field8;
+  x = (C)s->field9;
+  x = (C)s->field10;
+  x = (C)s->field11;
+  x = (C)s->field12;
+}
+
+void write_enumA(S_A *s, A x) {
+  s->field1 = x;
+  // bitfieldwarnings-warning@-1 {{bit-field 'field1' is not wide enough to store all enumerators of 'A'}}
+  // bitfieldwarnings-note@#S_A_field1 {{widen this field to 2 bits to store all values of 'A'}}
+  s->field2 = x;
+  s->field3 = x;
+  s->field4 = x;
+  // bitfieldwarnings-warning@-1 {{bit-field 'field4' is not wide enough to store all enumerators of 'A'}}
+  // bitfieldwarnings-note@#S_A_field4 {{widen this field to 2 bits to store all values of 'A'}}
+  s->field5 = x;
+  s->field6 = x;
+  s->field7 = x;
+  // bitfieldwarnings-warning@-1 {{bit-field 'field7' is not wide enough to store all enumerators of 'A'}}
+  // bitfieldwarnings-note@#S_A_field7 {{widen this field to 2 bits to store all values of 'A'}}
+  s->field8 = x;
+  // bitfieldwarnings-warning@-1 {{signed bit-field 'field8' needs an extra bit to represent the largest positive enumerators of 'A'}}
+  // bitfieldwarnings-note@#S_A_field8 {{consider making the bit-field type unsigned}}
+  s->field9 = x;
+  s->field10 = (D)x;
+  s->field11 = (D)x;
+  s->field12 = (D)x;
+}
+
+void write_enumB(S_B *s, B x) {
+  s->field1 = x;
+  // bitfieldwarnings-warning@-1 {{bit-field 'field1' is not wide enough to store all enumerators of 'B'}}
+  // bitfieldwarnings-note@#S_B_field1 {{widen this field to 2 bits to store all values of 'B'}}
+  s->field2 = x;
+  // bitfieldwarnings-warning@-1 {{signed bit-field 'field2' needs an extra bit to represent the largest positive enumerators of 'B'}}
+  // bitfieldwarnings-note@#S_B_field2 {{consider making the bit-field type unsigned}}
+  s->field3 = x;
+  s->field4 = (unsigned)x;
+  // expected-warning@-1 {{bit-field 'field4' is not wide enough to store all enumerators of preferred type 'B'}}
+  // expected-note@#S_B_field4 {{widen this field to 2 bits to store all values of 'B'}}
+  // expected-note@#preferred_S_B_field4 {{preferred type for bit-field 'B' specified here}}
+  s->field5 = (unsigned)x;
+  s->field6 = (unsigned)x;
+  s->field7 = (int)x;
+  // expected-warning@-1 {{bit-field 'field7' is not wide enough to store all enumerators of preferred type 'B'}}
+  // expected-note@#S_B_field7 {{widen this field to 2 bits to store all values of 'B'}}
+  // expected-note@#preferred_S_B_field7 {{preferred type for bit-field 'B' specified here}}
+  s->field8 = (int)x;
+  // expected-warning@-1 {{signed bit-field 'field8' needs an extra bit to represent the largest positive enumerators of preferred type 'B'}}
+  // expected-note@#S_B_field8 {{consider making the bit-field type unsigned}}
+  // expected-note@#preferred_S_B_field8 {{preferred type for bit-field 'B' specified here}}
+  s->field9 = (int)x;
+  s->field10 = (D)x;
+  s->field11 = (D)x;
+  s->field12 = (D)x;
+}
+
+void write_enumC(S_C *s, C x) {
+  s->field1 = x;
+  // bitfieldwarnings-warning@-1 {{bit-field 'field1' is not wide enough to store all enumerators of 'C'}}
+  // bitfieldwarnings-note@#S_C_field1 {{widen this field to 2 bits to store all values of 'C'}}
+  s->field2 = x;
+  s->field3 = x;
+  s->field4 = (unsigned)x;
+  // expected-warning@-1 {{bit-field 'field4' is not wide enough to store all enumerators of preferred type 'C'}}
+  // expected-note@#S_C_field4 {{widen this field to 2 bits to store all values of 'C'}}
+  // expected-note@#preferred_S_C_field4 {{preferred type for bit-field 'C' specified here}}
+  s->field5 = (unsigned)x;
+  s->field6 = (unsigned)x;
+  s->field7 = (int)x;
+  // expected-warning@-1 {{bit-field 'field7' is not wide enough to store all enumerators of preferred type 'C'}}
+  // expected-note@#S_C_field7 {{widen this field to 2 bits to store all values of 'C'}}
+  // expected-note@#preferred_S_C_field7 {{preferred type for bit-field 'C' specified here}}
+  s->field8 = (int)x;
+  // expected-warning@-1 {{signed bit-field 'field8' needs an extra bit to represent the largest positive enumerators of preferred type 'C'}}
+  // expected-note@#S_C_field8 {{consider making the bit-field type unsigned}}
+  // expected-note@#preferred_S_C_field8 {{preferred type for bit-field 'C' specified here}}
+  s->field9 = (int)x;
+  s->field10 = (D)x;
+  s->field11 = (D)x;
+  s->field12 = (D)x;
+}
+
+void write_enum_intA(struct S_A *s, int x) {
+  s->field1 = (A)x;
+  // bitfieldwarnings-warning@-1 {{bit-field 'field1' is not wide enough to store all enumerators of 'A'}}
+  // bitfieldwarnings-note@#S_A_field1 {{widen this field to 2 bits to store all values of 'A'}}
+  s->field2 = (A)x;
+  s->field3 = (A)x;
+  s->field4 = x;
+  // expected-warning@-1 {{bit-field 'field4' is not wide enough to store all enumerators of preferred type 'A'}}
+  // expected-note@#S_A_field4 {{widen this field to 2 bits to store all values of 'A'}}
+  // expected-note@#preferred_S_A_field4 {{preferred type for bit-field 'A' specified here}}
+  s->field5 = x;
+  s->field6 = x;
+  s->field7 = x;
+  // expected-warning@-1 {{bit-field 'field7' is not wide enough to store all enumerators of preferred type 'A'}}
+  // expected-note@#S_A_field7 {{widen this field to 2 bits to store all values of 'A'}}
+  // expected-note@#preferred_S_A_field7 {{preferred type for bit-field 'A' specified here}}
+  s->field8 = x;
+  // expected-warning@-1 {{signed bit-field 'field8' needs an extra bit to represent the largest positive enumerators of preferred type 'A'}}
+  // expected-note@#S_A_field8 {{consider making the bit-field type unsigned}}
+  // expected-note@#preferred_S_A_field8 {{preferred type for bit-field 'A' specified here}}
+  s->field9 = x;
+  s->field10 = (D)x;
+  s->field11 = (D)x;
+  s->field12 = (D)x;
+}
+
+void write_enum_intB(struct S_B *s, int x) {
+  s->field1 = (B)x;
+  // bitfieldwarnings-warning@-1 {{bit-field 'field1' is not wide enough to store all enumerators of 'B'}}
+  // bitfieldwarnings-note@#S_B_field1 {{widen this field to 2 bits to store all values of 'B'}}
+  s->field2 = (B)x;
+  // bitfieldwarnings-warning@-1 {{signed bit-field 'field2' needs an extra bit to represent the largest positive enumerators of 'B'}}
+  // bitfieldwarnings-note@#S_B_field2 {{consider making the bit-field type unsigned}}
+  s->field3 = (B)x;
+  s->field4 = x;
+  // expected-warning@-1 {{bit-field 'field4' is not wide enough to store all enumerators of preferred type 'B'}}
+  // expected-note@#S_B_field4 {{widen this field to 2 bits to store all values of 'B'}}
+  // expected-note@#preferred_S_B_field4 {{preferred type for bit-field 'B' specified here}}
+  s->field5 = x;
+  s->field6 = x;
+  s->field7 = x;
+  // expected-warning@-1 {{bit-field 'field7' is not wide enough to store all enumerators of preferred type 'B'}}
+  // expected-note@#S_B_field7 {{widen this field to 2 bits to store all values of 'B'}}
+  // expected-note@#preferred_S_B_field7 {{preferred type for bit-field 'B' specified here}}
+  s->field8 = x;
+  // expected-warning@-1 {{signed bit-field 'field8' needs an extra bit to represent the largest positive enumerators of preferred type 'B'}}
+  // expected-note@#S_B_field8 {{consider making the bit-field type unsigned}}
+  // expected-note@#preferred_S_B_field8 {{preferred type for bit-field 'B' specified here}}
+  s->field9 = x;
+  s->field10 = (D)x;
+  s->field11 = (D)x;
+  s->field12 = (D)x;
+}
+
+void write_enum_intC(struct S_C *s, int x) {
+  s->field1 = (C)x;
+  // bitfieldwarnings-warning@-1 {{bit-field 'field1' is not wide enough to store all enumerators of 'C'}}
+  // bitfieldwarnings-note@#S_C_field1 {{widen this field to 2 bits to store all values of 'C'}}
+  s->field2 = (C)x;
+  s->field3 = (C)x;
+  s->field4 = x;
+  // expected-warning@-1 {{bit-field 'field4' is not wide enough to store all enumerators of preferred type 'C'}}
+  // expected-note@#S_C_field4 {{widen this field to 2 bits to store all values of 'C'}}
+  // expected-note@#preferred_S_C_field4 {{preferred type for bit-field 'C' specified here}}
+  s->field5 = x;
+  s->field6 = x;
+  s->field7 = x;
+  // expected-warning@-1 {{bit-field 'field7' is not wide enough to store all enumerators of preferred type 'C'}}
+  // expected-note@#S_C_field7 {{widen this field to 2 bits to store all values of 'C'}}
+  // expected-note@#preferred_S_C_field7 {{preferred type for bit-field 'C' specified here}}
+  s->field8 = x;
+  // expected-warning@-1 {{signed bit-field 'field8' needs an extra bit to represent the largest positive enumerators of preferred type 'C'}}
+  // expected-note@#S_C_field8 {{consider making the bit-field type unsigned}}
+  // expected-note@#preferred_S_C_field8 {{preferred type for bit-field 'C' specified here}}
+  s->field9 = x;
+  s->field10 = (D)x;
+  s->field11 = (D)x;
+  s->field12 = (D)x;
+}
+
+void write_low_constantA(S_A *s) {
+  s->field1 = A_a;
+  s->field2 = A_a;
+  s->field3 = A_a;
+  s->field4 = A_a;
+  s->field5 = A_a;
+  s->field6 = A_a;
+  s->field7 = A_a;
+  s->field8 = A_a;
+  s->field9 = A_a;
+  s->field10 = (D)A_a;
+  s->field11 = (D)A_a;
+  s->field12 = (D)A_a;
+};
+
+void write_low_constantB(S_B *s) {
+  s->field1 = ENUM_CLASS_REF(B, B_a);
+  s->field2 = ENUM_CLASS_REF(B, B_a);
+  s->field3 = ENUM_CLASS_REF(B, B_a);
+  s->field4 = (unsigned)ENUM_CLASS_REF(B, B_a);
+  s->field5 = (unsigned)ENUM_CLASS_REF(B, B_a);
+  s->field6 = (unsigned)ENUM_CLASS_REF(B, B_a);
+  s->field7 = (int)ENUM_CLASS_REF(B, B_a);
+  s->field8 = (int)ENUM_CLASS_REF(B, B_a);
+  s->field9 = (int)ENUM_CLASS_REF(B, B_a);
+  s->field10 = (D)ENUM_CLASS_REF(B, B_a);
+  s->field11 = (D)ENUM_CLASS_REF(B, B_a);
+  s->field12 = (D)ENUM_CLASS_REF(B, B_a);
+};
+
+void write_low_constantC(S_C *s) {
+  s->field1 = ENUM_CLASS_REF(C, C_a);
+  s->field2 = ENUM_CLASS_REF(C, C_a);
+  s->field3 = ENUM_CLASS_REF(C, C_a);
+  s->field4 = (unsigned)ENUM_CLASS_REF(C, C_a);
+  s->field5 = (unsigned)ENUM_CLASS_REF(C, C_a);
+  s->field6 = (unsigned)ENUM_CLASS_REF(C, C_a);
+  s->field7 = (int)ENUM_CLASS_REF(C, C_a);
+  s->field8 = (int)ENUM_CLASS_REF(C, C_a);
+  s->field9 = (int)ENUM_CLASS_REF(C, C_a);
+  s->field10 = (D)ENUM_CLASS_REF(C, C_a);
+  s->field11 = (D)ENUM_CLASS_REF(C, C_a);
+  s->field12 = (D)ENUM_CLASS_REF(C, C_a);
+};
+
+void write_high_constantA(S_A *s) {
+  s->field1 = A_d;
+  // cpp-warning@-1 {{implicit truncation from 'A' to bit-field changes value from 3 to 1}}
+  // c-warning@-2 {{implicit truncation from 'int' to bit-field changes value from 3 to 1}}
+  s->field2 = A_d;
+  s->field3 = A_d;
+  s->field4 = A_d;
+  // cpp-warning@-1 {{implicit truncation from 'A' to bit-field changes value from 3 to 1}}
+  // c-warning@-2 {{implicit truncation from 'int' to bit-field changes value from 3 to 1}}
+  s->field5 = A_d;
+  s->field6 = A_d;
+  s->field7 = A_d;
+  // cpp-warning@-1 {{implicit truncation from 'A' to bit-field changes value from 3 to -1}}
+  // c-warning@-2 {{implicit truncation from 'int' to bit-field changes value from 3 to -1}}
+  s->field8 = A_d;
+  // cpp-warning@-1 {{implicit truncation from 'A' to bit-field changes value from 3 to -1}}
+  // c-warning@-2 {{implicit truncation from 'int' to bit-field changes value from 3 to -1}}
+  s->field9 = A_d;
+  s->field10 = (D)A_d;
+  // cpp-warning@-1 {{implicit truncation from 'D' to bit-field changes value from 3 to 1}}
+  // c-warning@-2 {{implicit truncation from 'D' (aka 'enum D') to bit-field changes value from 3 to 1}}
+  s->field11 = (D)A_d;
+  s->field12 = (D)A_d;
+};
+
+void write_high_constantB(S_B *s) {
+  s->field1 = ENUM_CLASS_REF(B, B_d);
+  // cpp-warning@-1 {{implicit truncation from 'B' to bit-field changes value from 3 to 1}}
+  // c-warning@-2 {{implicit truncation from 'int' to bit-field changes value from 3 to -1}}
+  s->field2 = ENUM_CLASS_REF(B, B_d);
+  // c-warning@-1 {{implicit truncation from 'int' to bit-field changes value from 3 to -1}}
+  s->field3 = ENUM_CLASS_REF(B, B_d);
+  s->field4 = (unsigned)ENUM_CLASS_REF(B, B_d);
+  // expected-warning@-1 {{implicit truncation from 'unsigned int' to bit-field changes value from 3 to 1}}
+  s->field5 = (unsigned)ENUM_CLASS_REF(B, B_d);
+  s->field6 = (unsigned)ENUM_CLASS_REF(B, B_d);
+  s->field7 = (int)ENUM_CLASS_REF(B, B_d);
+  // expected-warning@-1 {{implicit truncation from 'int' to bit-field changes value from 3 to -1}}
+  s->field8 = (int)ENUM_CLASS_REF(B, B_d);
+  // expected-warning@-1 {{implicit truncation from 'int' to bit-field changes value from 3 to -1}}
+  s->field9 = (int)ENUM_CLASS_REF(B, B_d);
+};
+
+
+void write_high_constantC(S_C *s) {
+  s->field1 = ENUM_CLASS_REF(C, C_d);
+  // cpp-warning@-1 {{implicit truncation from 'C' to bit-field changes value from 3 to 1}}
+  // c-warning@-2 {{implicit truncation from 'unsigned int' to bit-field changes value from 3 to 1}}
+  s->field2 = ENUM_CLASS_REF(C, C_d);
+  s->field3 = ENUM_CLASS_REF(C, C_d);
+  s->field4 = (unsigned)ENUM_CLASS_REF(C, C_d);
+  // expected-warning@-1 {{implicit truncation from 'unsigned int' to bit-field changes value from 3 to 1}}
+  s->field5 = (unsigned)ENUM_CLASS_REF(C, C_d);
+  s->field6 = (unsigned)ENUM_CLASS_REF(C, C_d);
+  s->field7 = (int)ENUM_CLASS_REF(C, C_d);
+  // expected-warning@-1 {{implicit truncation from 'int' to bit-field changes value from 3 to -1}}
+  s->field8 = (int)ENUM_CLASS_REF(C, C_d);
+  // expected-warning@-1 {{implicit truncation from 'int' to bit-field changes value from 3 to -1}}
+  s->field9 = (int)ENUM_CLASS_REF(C, C_d);
+};

From f87109f018faad5f3f1bf8a4668754c24e84e886 Mon Sep 17 00:00:00 2001
From: Tristan Ross <tristan.ross@midstall.com>
Date: Sun, 20 Apr 2025 14:23:53 -0700
Subject: [PATCH 589/710] [compiler-rt] allow building with uefi (#131499)

I'm trying to put together an LLVM built toolchain (including LLVM libc)
targeting UEFI, currently I get an error saying "Unknown target". This
PR enables compiling compiler-rt for UEFI.
---
 compiler-rt/lib/builtins/int_lib.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/builtins/int_lib.h b/compiler-rt/lib/builtins/int_lib.h
index f6c1b7cff4b99..943430de259d8 100644
--- a/compiler-rt/lib/builtins/int_lib.h
+++ b/compiler-rt/lib/builtins/int_lib.h
@@ -49,7 +49,7 @@
 #define SYMBOL_NAME(name) XSTR(__USER_LABEL_PREFIX__) #name
 
 #if defined(__ELF__) || defined(__MINGW32__) || defined(__wasm__) ||           \
-    defined(_AIX)    || defined(__CYGWIN__)
+    defined(_AIX) || defined(__CYGWIN__)
 #define COMPILER_RT_ALIAS(name, aliasname) \
   COMPILER_RT_ABI __typeof(name) aliasname __attribute__((__alias__(#name)));
 #elif defined(__APPLE__)
@@ -64,7 +64,7 @@
   COMPILER_RT_ALIAS_VISIBILITY(aliasname) \
   __asm__(SYMBOL_NAME(aliasname) " = " SYMBOL_NAME(name)); \
   COMPILER_RT_ABI __typeof(name) aliasname;
-#elif defined(_WIN32)
+#elif defined(_WIN32) || defined(__UEFI__)
 #define COMPILER_RT_ALIAS(name, aliasname)
 #else
 #error Unsupported target

From b01e25debaf3fde3c76c9763e78e118a7da3d592 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 16:36:03 -0700
Subject: [PATCH 590/710] [llvm] Call hash_combine_range with ranges (NFC)
 (#136511)

---
 llvm/include/llvm/ADT/ArrayRef.h              |  2 +-
 llvm/include/llvm/ADT/Hashing.h               |  2 +-
 .../llvm/Analysis/IRSimilarityIdentifier.h    | 36 +++++++++----------
 llvm/include/llvm/ProfileData/SampleProf.h    |  2 +-
 .../Transforms/IPO/FunctionSpecialization.h   |  3 +-
 llvm/lib/CodeGen/MIRVRegNamerUtils.cpp        |  2 +-
 llvm/lib/CodeGen/MachineInstr.cpp             |  2 +-
 llvm/lib/CodeGen/RegisterBankInfo.cpp         |  2 +-
 .../ExecutionEngine/Orc/IRPartitionLayer.cpp  |  2 +-
 llvm/lib/IR/ConstantsContext.h                | 14 +++-----
 llvm/lib/IR/LLVMContextImpl.cpp               |  2 +-
 llvm/lib/IR/LLVMContextImpl.h                 | 22 ++++--------
 llvm/lib/Linker/IRMover.cpp                   |  3 +-
 llvm/lib/Support/StringRef.cpp                |  4 +--
 llvm/lib/Transforms/IPO/IROutliner.cpp        |  6 ++--
 llvm/lib/Transforms/Scalar/GVN.cpp            |  5 ++-
 llvm/lib/Transforms/Scalar/GVNSink.cpp        |  2 +-
 .../Transforms/Scalar/LoopStrengthReduce.cpp  |  2 +-
 .../Transforms/Utils/LoopRotationUtils.cpp    |  3 +-
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp     |  3 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  2 +-
 llvm/lib/Transforms/Vectorize/VPlanSLP.h      |  2 +-
 llvm/tools/llvm-profgen/PerfReader.h          |  4 +--
 23 files changed, 51 insertions(+), 76 deletions(-)

diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h
index 1139fd81cbd07..a1317423cdd1a 100644
--- a/llvm/include/llvm/ADT/ArrayRef.h
+++ b/llvm/include/llvm/ADT/ArrayRef.h
@@ -562,7 +562,7 @@ namespace llvm {
   /// @}
 
   template <typename T> hash_code hash_value(ArrayRef<T> S) {
-    return hash_combine_range(S.begin(), S.end());
+    return hash_combine_range(S);
   }
 
   // Provide DenseMapInfo for ArrayRefs.
diff --git a/llvm/include/llvm/ADT/Hashing.h b/llvm/include/llvm/ADT/Hashing.h
index 9b8643b7765cf..0093c281aac8a 100644
--- a/llvm/include/llvm/ADT/Hashing.h
+++ b/llvm/include/llvm/ADT/Hashing.h
@@ -649,7 +649,7 @@ template <typename... Ts> hash_code hash_value(const std::tuple<Ts...> &arg) {
 // infrastructure is available.
 template <typename T>
 hash_code hash_value(const std::basic_string<T> &arg) {
-  return hash_combine_range(arg.begin(), arg.end());
+  return hash_combine_range(arg);
 }
 
 template <typename T> hash_code hash_value(const std::optional<T> &arg) {
diff --git a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
index 9da465ddf75f9..eb1827850567d 100644
--- a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
+++ b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
@@ -263,37 +263,35 @@ struct IRInstructionData
       OperTypes.push_back(V->getType());
 
     if (isa<CmpInst>(ID.Inst))
-      return llvm::hash_combine(
-          llvm::hash_value(ID.Inst->getOpcode()),
-          llvm::hash_value(ID.Inst->getType()),
-          llvm::hash_value(ID.getPredicate()),
-          llvm::hash_combine_range(OperTypes.begin(), OperTypes.end()));
+      return llvm::hash_combine(llvm::hash_value(ID.Inst->getOpcode()),
+                                llvm::hash_value(ID.Inst->getType()),
+                                llvm::hash_value(ID.getPredicate()),
+                                llvm::hash_combine_range(OperTypes));
 
     if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(ID.Inst)) {
       // To hash intrinsics, we use the opcode, and types like the other
       // instructions, but also, the Intrinsic ID, and the Name of the
       // intrinsic.
       Intrinsic::ID IntrinsicID = II->getIntrinsicID();
-      return llvm::hash_combine(
-          llvm::hash_value(ID.Inst->getOpcode()),
-          llvm::hash_value(ID.Inst->getType()), llvm::hash_value(IntrinsicID),
-          llvm::hash_value(*ID.CalleeName),
-          llvm::hash_combine_range(OperTypes.begin(), OperTypes.end()));
+      return llvm::hash_combine(llvm::hash_value(ID.Inst->getOpcode()),
+                                llvm::hash_value(ID.Inst->getType()),
+                                llvm::hash_value(IntrinsicID),
+                                llvm::hash_value(*ID.CalleeName),
+                                llvm::hash_combine_range(OperTypes));
     }
 
     if (isa<CallInst>(ID.Inst)) {
       std::string FunctionName = *ID.CalleeName;
-      return llvm::hash_combine(
-          llvm::hash_value(ID.Inst->getOpcode()),
-          llvm::hash_value(ID.Inst->getType()),
-          llvm::hash_value(ID.Inst->getType()), llvm::hash_value(FunctionName),
-          llvm::hash_combine_range(OperTypes.begin(), OperTypes.end()));
+      return llvm::hash_combine(llvm::hash_value(ID.Inst->getOpcode()),
+                                llvm::hash_value(ID.Inst->getType()),
+                                llvm::hash_value(ID.Inst->getType()),
+                                llvm::hash_value(FunctionName),
+                                llvm::hash_combine_range(OperTypes));
     }
 
-    return llvm::hash_combine(
-        llvm::hash_value(ID.Inst->getOpcode()),
-        llvm::hash_value(ID.Inst->getType()),
-        llvm::hash_combine_range(OperTypes.begin(), OperTypes.end()));
+    return llvm::hash_combine(llvm::hash_value(ID.Inst->getOpcode()),
+                              llvm::hash_value(ID.Inst->getType()),
+                              llvm::hash_combine_range(OperTypes));
   }
 
   IRInstructionDataList *IDL = nullptr;
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index e7b154dff0697..66fe0da21deb7 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -506,7 +506,7 @@ using SampleContextFrames = ArrayRef<SampleContextFrame>;
 
 struct SampleContextFrameHash {
   uint64_t operator()(const SampleContextFrameVector &S) const {
-    return hash_combine_range(S.begin(), S.end());
+    return hash_combine_range(S);
   }
 };
 
diff --git a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
index e4972cb1cef84..d15a3d777ab0e 100644
--- a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
+++ b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
@@ -119,8 +119,7 @@ struct SpecSig {
   }
 
   friend hash_code hash_value(const SpecSig &S) {
-    return hash_combine(hash_value(S.Key),
-                        hash_combine_range(S.Args.begin(), S.Args.end()));
+    return hash_combine(hash_value(S.Key), hash_combine_range(S.Args));
   }
 };
 
diff --git a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
index b888b81e92b92..a22cc91b90542 100644
--- a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
+++ b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
@@ -131,7 +131,7 @@ std::string VRegRenamer::getInstructionOpcodeHash(MachineInstr &MI) {
     MIOperands.push_back((unsigned)Op->getFailureOrdering());
   }
 
-  auto HashMI = hash_combine_range(MIOperands.begin(), MIOperands.end());
+  auto HashMI = hash_combine_range(MIOperands);
   OS << format_hex_no_prefix(HashMI, 16, true);
   return OS.str();
 }
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 33910d0ec6aeb..2bc18001081ea 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -2306,7 +2306,7 @@ MachineInstrExpressionTrait::getHashValue(const MachineInstr* const &MI) {
 
     HashComponents.push_back(hash_value(MO));
   }
-  return hash_combine_range(HashComponents.begin(), HashComponents.end());
+  return hash_combine_range(HashComponents);
 }
 
 const MDNode *MachineInstr::getLocCookieMD() const {
diff --git a/llvm/lib/CodeGen/RegisterBankInfo.cpp b/llvm/lib/CodeGen/RegisterBankInfo.cpp
index eaef13969a7a1..79b3c73ea34f1 100644
--- a/llvm/lib/CodeGen/RegisterBankInfo.cpp
+++ b/llvm/lib/CodeGen/RegisterBankInfo.cpp
@@ -307,7 +307,7 @@ hashValueMapping(const RegisterBankInfo::PartialMapping *BreakDown,
   SmallVector<size_t, 8> Hashes(NumBreakDowns);
   for (unsigned Idx = 0; Idx != NumBreakDowns; ++Idx)
     Hashes.push_back(hash_value(BreakDown[Idx]));
-  return hash_combine_range(Hashes.begin(), Hashes.end());
+  return hash_combine_range(Hashes);
 }
 
 const RegisterBankInfo::ValueMapping &
diff --git a/llvm/lib/ExecutionEngine/Orc/IRPartitionLayer.cpp b/llvm/lib/ExecutionEngine/Orc/IRPartitionLayer.cpp
index 1a37469c35d2e..7a105c3a7137a 100644
--- a/llvm/lib/ExecutionEngine/Orc/IRPartitionLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/IRPartitionLayer.cpp
@@ -268,7 +268,7 @@ void IRPartitionLayer::emitPartition(
       for (const auto *GV : HashGVs) {
         assert(GV->hasName() && "All GVs to extract should be named by now");
         auto GVName = GV->getName();
-        HC = hash_combine(HC, hash_combine_range(GVName.begin(), GVName.end()));
+        HC = hash_combine(HC, hash_combine_range(GVName));
       }
       raw_string_ostream(SubModuleName)
           << ".submodule."
diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h
index e5c9622e09927..51fb40bad201d 100644
--- a/llvm/lib/IR/ConstantsContext.h
+++ b/llvm/lib/IR/ConstantsContext.h
@@ -322,9 +322,7 @@ template <class ConstantClass> struct ConstantAggrKeyType {
     return true;
   }
 
-  unsigned getHash() const {
-    return hash_combine_range(Operands.begin(), Operands.end());
-  }
+  unsigned getHash() const { return hash_combine_range(Operands); }
 
   using TypeClass = typename ConstantInfo<ConstantClass>::TypeClass;
 
@@ -478,10 +476,8 @@ struct ConstantExprKeyType {
   }
 
   unsigned getHash() const {
-    return hash_combine(
-        Opcode, SubclassOptionalData,
-        hash_combine_range(Ops.begin(), Ops.end()),
-        hash_combine_range(ShuffleMask.begin(), ShuffleMask.end()), ExplicitTy);
+    return hash_combine(Opcode, SubclassOptionalData, hash_combine_range(Ops),
+                        hash_combine_range(ShuffleMask), ExplicitTy);
   }
 
   using TypeClass = ConstantInfo<ConstantExpr>::TypeClass;
@@ -537,9 +533,7 @@ struct ConstantPtrAuthKeyType {
     return true;
   }
 
-  unsigned getHash() const {
-    return hash_combine_range(Operands.begin(), Operands.end());
-  }
+  unsigned getHash() const { return hash_combine_range(Operands); }
 
   using TypeClass = typename ConstantInfo<ConstantPtrAuth>::TypeClass;
 
diff --git a/llvm/lib/IR/LLVMContextImpl.cpp b/llvm/lib/IR/LLVMContextImpl.cpp
index 5e51b3ca4e87d..d8cdbf8370984 100644
--- a/llvm/lib/IR/LLVMContextImpl.cpp
+++ b/llvm/lib/IR/LLVMContextImpl.cpp
@@ -207,7 +207,7 @@ unsigned MDNodeOpsKey::calculateHash(MDNode *N, unsigned Offset) {
 }
 
 unsigned MDNodeOpsKey::calculateHash(ArrayRef<Metadata *> Ops) {
-  return hash_combine_range(Ops.begin(), Ops.end());
+  return hash_combine_range(Ops);
 }
 
 StringMapEntry<uint32_t> *LLVMContextImpl::getOrInsertBundleTag(StringRef Tag) {
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index efabe40fab192..9b60b57e64130 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -119,8 +119,7 @@ struct AnonStructTypeKeyInfo {
   }
 
   static unsigned getHashValue(const KeyTy &Key) {
-    return hash_combine(
-        hash_combine_range(Key.ETypes.begin(), Key.ETypes.end()), Key.isPacked);
+    return hash_combine(hash_combine_range(Key.ETypes), Key.isPacked);
   }
 
   static unsigned getHashValue(const StructType *ST) {
@@ -171,9 +170,8 @@ struct FunctionTypeKeyInfo {
   }
 
   static unsigned getHashValue(const KeyTy &Key) {
-    return hash_combine(
-        Key.ReturnType,
-        hash_combine_range(Key.Params.begin(), Key.Params.end()), Key.isVarArg);
+    return hash_combine(Key.ReturnType, hash_combine_range(Key.Params),
+                        Key.isVarArg);
   }
 
   static unsigned getHashValue(const FunctionType *FT) {
@@ -219,10 +217,8 @@ struct TargetExtTypeKeyInfo {
   }
 
   static unsigned getHashValue(const KeyTy &Key) {
-    return hash_combine(
-        Key.Name,
-        hash_combine_range(Key.TypeParams.begin(), Key.TypeParams.end()),
-        hash_combine_range(Key.IntParams.begin(), Key.IntParams.end()));
+    return hash_combine(Key.Name, hash_combine_range(Key.TypeParams),
+                        hash_combine_range(Key.IntParams));
   }
 
   static unsigned getHashValue(const TargetExtType *FT) {
@@ -1324,9 +1320,7 @@ template <> struct MDNodeKeyImpl<DIExpression> {
     return Elements == RHS->getElements();
   }
 
-  unsigned getHashValue() const {
-    return hash_combine_range(Elements.begin(), Elements.end());
-  }
+  unsigned getHashValue() const { return hash_combine_range(Elements); }
 };
 
 template <> struct MDNodeKeyImpl<DIGlobalVariableExpression> {
@@ -1463,9 +1457,7 @@ struct DIArgListKeyInfo {
 
   bool isKeyOf(const DIArgList *RHS) const { return Args == RHS->getArgs(); }
 
-  unsigned getHashValue() const {
-    return hash_combine_range(Args.begin(), Args.end());
-  }
+  unsigned getHashValue() const { return hash_combine_range(Args); }
 };
 
 /// DenseMapInfo for DIArgList.
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index e732367b5127e..0f32e9e1a05a5 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -1689,8 +1689,7 @@ StructType *IRMover::StructTypeKeyInfo::getTombstoneKey() {
 }
 
 unsigned IRMover::StructTypeKeyInfo::getHashValue(const KeyTy &Key) {
-  return hash_combine(hash_combine_range(Key.ETypes.begin(), Key.ETypes.end()),
-                      Key.IsPacked);
+  return hash_combine(hash_combine_range(Key.ETypes), Key.IsPacked);
 }
 
 unsigned IRMover::StructTypeKeyInfo::getHashValue(const StructType *ST) {
diff --git a/llvm/lib/Support/StringRef.cpp b/llvm/lib/Support/StringRef.cpp
index 4f5fcb4857e80..96829bd062a78 100644
--- a/llvm/lib/Support/StringRef.cpp
+++ b/llvm/lib/Support/StringRef.cpp
@@ -613,9 +613,7 @@ bool StringRef::getAsDouble(double &Result, bool AllowInexact) const {
 }
 
 // Implementation of StringRef hashing.
-hash_code llvm::hash_value(StringRef S) {
-  return hash_combine_range(S.begin(), S.end());
-}
+hash_code llvm::hash_value(StringRef S) { return hash_combine_range(S); }
 
 unsigned DenseMapInfo<StringRef, void>::getHashValue(StringRef Val) {
   assert(Val.data() != getEmptyKey().data() &&
diff --git a/llvm/lib/Transforms/IPO/IROutliner.cpp b/llvm/lib/Transforms/IPO/IROutliner.cpp
index eec869d57a6db..6c9d499b0e7ea 100644
--- a/llvm/lib/Transforms/IPO/IROutliner.cpp
+++ b/llvm/lib/Transforms/IPO/IROutliner.cpp
@@ -1154,9 +1154,9 @@ using PHINodeData = std::pair<ArgLocWithBBCanon, CanonList>;
 /// \param PND - The data to hash.
 /// \returns The hash code of \p PND.
 static hash_code encodePHINodeData(PHINodeData &PND) {
-  return llvm::hash_combine(
-      llvm::hash_value(PND.first.first), llvm::hash_value(PND.first.second),
-      llvm::hash_combine_range(PND.second.begin(), PND.second.end()));
+  return llvm::hash_combine(llvm::hash_value(PND.first.first),
+                            llvm::hash_value(PND.first.second),
+                            llvm::hash_combine_range(PND.second));
 }
 
 /// Create a special GVN for PHINodes that will be used outside of
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 6233e8e2ee681..c0ae944ee82f2 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -165,9 +165,8 @@ struct llvm::GVNPass::Expression {
   }
 
   friend hash_code hash_value(const Expression &Value) {
-    return hash_combine(
-        Value.Opcode, Value.Ty,
-        hash_combine_range(Value.VarArgs.begin(), Value.VarArgs.end()));
+    return hash_combine(Value.Opcode, Value.Ty,
+                        hash_combine_range(Value.VarArgs));
   }
 };
 
diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index 6f88408abfdbc..2058df33ea331 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -249,7 +249,7 @@ class ModelledPHI {
   // Hash functor
   unsigned hash() const {
     // Is deterministic because Values are saved in a specific order.
-    return (unsigned)hash_combine_range(Values.begin(), Values.end());
+    return (unsigned)hash_combine_range(Values);
   }
 
   bool operator==(const ModelledPHI &Other) const {
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index ed3709b74d60f..fd16593b2e874 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -1291,7 +1291,7 @@ struct UniquifierDenseMapInfo {
   }
 
   static unsigned getHashValue(const SmallVector<const SCEV *, 4> &V) {
-    return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
+    return static_cast<unsigned>(hash_combine_range(V));
   }
 
   static bool isEqual(const SmallVector<const SCEV *, 4> &LHS,
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index b07f3451420c7..c5c4968ef4dcc 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -553,8 +553,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
         std::pair<std::pair<hash_code, DILocalVariable *>, DIExpression *>;
     auto makeHash = [](auto *D) -> DbgIntrinsicHash {
       auto VarLocOps = D->location_ops();
-      return {{hash_combine_range(VarLocOps.begin(), VarLocOps.end()),
-               D->getVariable()},
+      return {{hash_combine_range(VarLocOps), D->getVariable()},
               D->getExpression()};
     };
 
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 7f53aa7d4f73d..9e6ee58cafc79 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -7469,8 +7469,7 @@ template <> struct DenseMapInfo<const SwitchSuccWrapper *> {
     for (PHINode &Phi : BB->phis())
       PhiValsForBB.emplace_back((*SSW->PhiPredIVs)[&Phi][BB]);
 
-    return hash_combine(
-        BB, hash_combine_range(PhiValsForBB.begin(), PhiValsForBB.end()));
+    return hash_combine(BB, hash_combine_range(PhiValsForBB));
   }
   static bool isEqual(const SwitchSuccWrapper *LHS,
                       const SwitchSuccWrapper *RHS) {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 70cd201ab20a6..a56728226c039 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5085,7 +5085,7 @@ class BoUpSLP {
     }
 
     static unsigned getHashValue(const OrdersType &V) {
-      return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
+      return static_cast<unsigned>(hash_combine_range(V));
     }
 
     static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.h b/llvm/lib/Transforms/Vectorize/VPlanSLP.h
index 7f123689170ad..2b927b93e24cf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.h
@@ -85,7 +85,7 @@ class VPlanSlp {
     }
 
     static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
-      return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
+      return static_cast<unsigned>(hash_combine_range(V));
     }
 
     static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
diff --git a/llvm/tools/llvm-profgen/PerfReader.h b/llvm/tools/llvm-profgen/PerfReader.h
index a3bd7a0a6493e..4b3ac8f569755 100644
--- a/llvm/tools/llvm-profgen/PerfReader.h
+++ b/llvm/tools/llvm-profgen/PerfReader.h
@@ -386,9 +386,7 @@ struct AddrBasedCtxKey : public ContextKey {
     return Context == Other->Context;
   }
 
-  void genHashCode() override {
-    HashCode = hash_combine_range(Context.begin(), Context.end());
-  }
+  void genHashCode() override { HashCode = hash_combine_range(Context); }
 };
 
 // The counter of branch samples for one function indexed by the branch,

From 5e834b9ec7fc85072ec12de780d3c793c03b7d00 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 16:36:35 -0700
Subject: [PATCH 591/710] [mlir] Call hash_combine_range with ranges (NFC)
 (#136512)

---
 .../mlir/Dialect/Polynomial/IR/Polynomial.h     |  2 +-
 mlir/include/mlir/IR/BlockSupport.h             |  2 +-
 mlir/include/mlir/IR/TypeRange.h                |  2 +-
 mlir/lib/Dialect/Quant/IR/TypeDetail.h          | 17 +++++++----------
 .../Linking/ModuleCombiner/ModuleCombiner.cpp   |  5 ++---
 .../Dialect/Vector/Transforms/VectorUnroll.cpp  |  2 +-
 mlir/lib/Transforms/Utils/DialectConversion.cpp |  2 +-
 mlir/unittests/IR/OpPropertiesTest.cpp          |  7 +++----
 8 files changed, 17 insertions(+), 22 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
index 3f206cd1e545a..8d7f1436fdc60 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
@@ -260,7 +260,7 @@ class FloatPolynomial : public PolynomialBase<FloatPolynomial, FloatMonomial> {
 // Make Polynomials hashable.
 template <class D, typename T>
 inline ::llvm::hash_code hash_value(const PolynomialBase<D, T> &arg) {
-  return ::llvm::hash_combine_range(arg.terms.begin(), arg.terms.end());
+  return ::llvm::hash_combine_range(arg.terms);
 }
 
 template <class D, typename T>
diff --git a/mlir/include/mlir/IR/BlockSupport.h b/mlir/include/mlir/IR/BlockSupport.h
index 292938e46c27e..41434269d555b 100644
--- a/mlir/include/mlir/IR/BlockSupport.h
+++ b/mlir/include/mlir/IR/BlockSupport.h
@@ -185,7 +185,7 @@ struct DenseMapInfo<mlir::SuccessorRange> {
     return mlir::SuccessorRange(pointer, 0);
   }
   static unsigned getHashValue(mlir::SuccessorRange value) {
-    return llvm::hash_combine_range(value.begin(), value.end());
+    return llvm::hash_combine_range(value);
   }
   static bool isEqual(mlir::SuccessorRange lhs, mlir::SuccessorRange rhs) {
     if (rhs.getBase() == getEmptyKey().getBase())
diff --git a/mlir/include/mlir/IR/TypeRange.h b/mlir/include/mlir/IR/TypeRange.h
index 3fb58d78617c0..e098370ae6e58 100644
--- a/mlir/include/mlir/IR/TypeRange.h
+++ b/mlir/include/mlir/IR/TypeRange.h
@@ -72,7 +72,7 @@ class TypeRange : public llvm::detail::indexed_accessor_range_base<
 
 /// Make TypeRange hashable.
 inline ::llvm::hash_code hash_value(TypeRange arg) {
-  return ::llvm::hash_combine_range(arg.begin(), arg.end());
+  return ::llvm::hash_combine_range(arg);
 }
 
 /// Emit a type range to the given output stream.
diff --git a/mlir/lib/Dialect/Quant/IR/TypeDetail.h b/mlir/lib/Dialect/Quant/IR/TypeDetail.h
index bb38b1a2a91e2..a43bce354c324 100644
--- a/mlir/lib/Dialect/Quant/IR/TypeDetail.h
+++ b/mlir/lib/Dialect/Quant/IR/TypeDetail.h
@@ -207,11 +207,10 @@ struct UniformQuantizedPerAxisTypeStorage : public QuantizedTypeStorage {
     unsigned getHashValue() const {
       int64_t *scalesCast = llvm::bit_cast<int64_t *>(scales.data());
       ArrayRef<int64_t> scalesBits(scalesCast, scales.size());
-      return llvm::hash_combine(
-          flags, storageType, expressedType,
-          llvm::hash_combine_range(scalesBits.begin(), scalesBits.end()),
-          llvm::hash_combine_range(zeroPoints.begin(), zeroPoints.end()),
-          storageTypeMin, storageTypeMax);
+      return llvm::hash_combine(flags, storageType, expressedType,
+                                llvm::hash_combine_range(scalesBits),
+                                llvm::hash_combine_range(zeroPoints),
+                                storageTypeMin, storageTypeMax);
     }
   };
 
@@ -318,11 +317,9 @@ struct UniformQuantizedSubChannelTypeStorage : public QuantizedTypeStorage {
       }
 
       // Hash the quantized dimensions and block sizes.
-      hash = llvm::hash_combine(
-          hash,
-          llvm::hash_combine_range(quantizedDimensions.begin(),
-                                   quantizedDimensions.end()),
-          llvm::hash_combine_range(blockSizes.begin(), blockSizes.end()));
+      hash = llvm::hash_combine(hash,
+                                llvm::hash_combine_range(quantizedDimensions),
+                                llvm::hash_combine_range(blockSizes));
 
       return hash;
     }
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 1f708c2ae37ad..71122f8e20512 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -79,9 +79,8 @@ static llvm::hash_code computeHash(SymbolOpInterface symbolOp) {
         return attr.getName() != SymbolTable::getSymbolAttrName();
       });
 
-  return llvm::hash_combine(
-      symbolOp->getName(),
-      llvm::hash_combine_range(range.begin(), range.end()));
+  return llvm::hash_combine(symbolOp->getName(),
+                            llvm::hash_combine_range(range));
 }
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index cb348b28c876a..dffb13c3a7923 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -238,7 +238,7 @@ struct OffsetMapInfo {
   static SmallVector<int64_t> getTombstoneKey() { return {int64_t(-2)}; }
 
   static unsigned getHashValue(const SmallVector<int64_t> &v) {
-    return static_cast<unsigned>(llvm::hash_combine_range(v.begin(), v.end()));
+    return static_cast<unsigned>(llvm::hash_combine_range(v));
   }
 
   static bool isEqual(const SmallVector<int64_t> &lhs,
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index a56fca25e1697..962207059c8aa 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -106,7 +106,7 @@ struct ValueVectorMapInfo {
   static ValueVector getEmptyKey() { return ValueVector{Value()}; }
   static ValueVector getTombstoneKey() { return ValueVector{Value(), Value()}; }
   static ::llvm::hash_code getHashValue(const ValueVector &val) {
-    return ::llvm::hash_combine_range(val.begin(), val.end());
+    return ::llvm::hash_combine_range(val);
   }
   static bool isEqual(const ValueVector &LHS, const ValueVector &RHS) {
     return LHS == RHS;
diff --git a/mlir/unittests/IR/OpPropertiesTest.cpp b/mlir/unittests/IR/OpPropertiesTest.cpp
index b4a633a2c62e6..4759735d99605 100644
--- a/mlir/unittests/IR/OpPropertiesTest.cpp
+++ b/mlir/unittests/IR/OpPropertiesTest.cpp
@@ -96,10 +96,9 @@ inline llvm::hash_code computeHash(const TestProperties &prop) {
   // We hash `b` which is a float using its underlying array of char:
   unsigned char const *p = reinterpret_cast<unsigned char const *>(&prop.b);
   ArrayRef<unsigned char> bBytes{p, sizeof(prop.b)};
-  return llvm::hash_combine(
-      prop.a, llvm::hash_combine_range(bBytes.begin(), bBytes.end()),
-      llvm::hash_combine_range(prop.array.begin(), prop.array.end()),
-      StringRef(*prop.label));
+  return llvm::hash_combine(prop.a, llvm::hash_combine_range(bBytes),
+                            llvm::hash_combine_range(prop.array),
+                            StringRef(*prop.label));
 }
 
 /// A custom operation for the purpose of showcasing how to use "properties".

From b07d2e62c63414bbce5c1d61bf72153071a8c05d Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Mon, 21 Apr 2025 04:21:37 +0300
Subject: [PATCH 592/710] [Mips] Use helper class for emitting CFI instructions
 (NFCI) (#136242)

---
 llvm/lib/Target/Mips/Mips16FrameLowering.cpp | 27 ++------
 llvm/lib/Target/Mips/MipsSEFrameLowering.cpp | 71 ++++++--------------
 2 files changed, 25 insertions(+), 73 deletions(-)

diff --git a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp
index 576b93701f356..44fc720e6ce14 100644
--- a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp
+++ b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp
@@ -16,6 +16,7 @@
 #include "MipsRegisterInfo.h"
 #include "MipsSubtarget.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -24,9 +25,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/IR/DebugLoc.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCDwarf.h"
-#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/MathExtras.h"
 #include <cstdint>
 #include <vector>
@@ -52,32 +50,19 @@ void Mips16FrameLowering::emitPrologue(MachineFunction &MF,
   // No need to allocate space on the stack.
   if (StackSize == 0 && !MFI.adjustsStack()) return;
 
-  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
-
   // Adjust stack.
   TII.makeFrame(Mips::SP, StackSize, MBB, MBBI);
 
-  // emit ".cfi_def_cfa_offset StackSize"
-  unsigned CFIIndex =
-      MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, StackSize));
-  BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex);
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::NoFlags);
+  CFIBuilder.buildDefCFAOffset(StackSize);
 
   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
 
   if (!CSI.empty()) {
-    const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
-
-    for (const CalleeSavedInfo &I : CSI) {
-      int64_t Offset = MFI.getObjectOffset(I.getFrameIdx());
-      MCRegister Reg = I.getReg();
-      unsigned DReg = MRI->getDwarfRegNum(Reg, true);
-      unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::createOffset(nullptr, DReg, Offset));
-      BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex);
-    }
+    for (const CalleeSavedInfo &I : CSI)
+      CFIBuilder.buildOffset(I.getReg(), MFI.getObjectOffset(I.getFrameIdx()));
   }
+
   if (hasFP(MF))
     BuildMI(MBB, MBBI, dl, TII.get(Mips::MoveR3216), Mips::S0)
       .addReg(Mips::SP).setMIFlag(MachineInstr::FrameSetup);
diff --git a/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp b/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
index e9ac94183066f..d775f5a16bcd0 100644
--- a/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -33,8 +34,6 @@
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Function.h"
-#include "llvm/MC/MCDwarf.h"
-#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
@@ -426,28 +425,23 @@ void MipsSEFrameLowering::emitPrologue(MachineFunction &MF,
   // No need to allocate space on the stack.
   if (StackSize == 0 && !MFI.adjustsStack()) return;
 
-  const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
+  CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::NoFlags);
 
   // Adjust stack.
   TII.adjustStackPtr(SP, -StackSize, MBB, MBBI);
-
-  // emit ".cfi_def_cfa_offset StackSize"
-  unsigned CFIIndex =
-      MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, StackSize));
-  BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex);
+  CFIBuilder.buildDefCFAOffset(StackSize);
 
   if (MF.getFunction().hasFnAttribute("interrupt"))
     emitInterruptPrologueStub(MF, MBB);
 
   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
 
-  if (!CSI.empty()) {
-    // Find the instruction past the last instruction that saves a callee-saved
-    // register to the stack.
-    for (unsigned i = 0; i < CSI.size(); ++i)
-      ++MBBI;
+  // Find the instruction past the last instruction that saves a callee-saved
+  // register to the stack.
+  std::advance(MBBI, CSI.size());
+  CFIBuilder.setInsertPoint(MBBI);
 
+  if (!CSI.empty()) {
     // Iterate over list of callee-saved registers and emit .cfi_offset
     // directives.
     for (const CalleeSavedInfo &I : CSI) {
@@ -457,45 +451,26 @@ void MipsSEFrameLowering::emitPrologue(MachineFunction &MF,
       // If Reg is a double precision register, emit two cfa_offsets,
       // one for each of the paired single precision registers.
       if (Mips::AFGR64RegClass.contains(Reg)) {
-        unsigned Reg0 =
-            MRI->getDwarfRegNum(RegInfo.getSubReg(Reg, Mips::sub_lo), true);
-        unsigned Reg1 =
-            MRI->getDwarfRegNum(RegInfo.getSubReg(Reg, Mips::sub_hi), true);
+        MCRegister Reg0 = RegInfo.getSubReg(Reg, Mips::sub_lo);
+        MCRegister Reg1 = RegInfo.getSubReg(Reg, Mips::sub_hi);
 
         if (!STI.isLittle())
           std::swap(Reg0, Reg1);
 
-        unsigned CFIIndex = MF.addFrameInst(
-            MCCFIInstruction::createOffset(nullptr, Reg0, Offset));
-        BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex);
-
-        CFIIndex = MF.addFrameInst(
-            MCCFIInstruction::createOffset(nullptr, Reg1, Offset + 4));
-        BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex);
+        CFIBuilder.buildOffset(Reg0, Offset);
+        CFIBuilder.buildOffset(Reg1, Offset + 4);
       } else if (Mips::FGR64RegClass.contains(Reg)) {
-        unsigned Reg0 = MRI->getDwarfRegNum(Reg, true);
-        unsigned Reg1 = MRI->getDwarfRegNum(Reg, true) + 1;
+        MCRegister Reg0 = Reg;
+        MCRegister Reg1 = Reg + 1;
 
         if (!STI.isLittle())
           std::swap(Reg0, Reg1);
 
-        unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::createOffset(nullptr, Reg0, Offset));
-        BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex);
-
-        CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::createOffset(nullptr, Reg1, Offset + 4));
-        BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex);
+        CFIBuilder.buildOffset(Reg0, Offset);
+        CFIBuilder.buildOffset(Reg1, Offset + 4);
       } else {
         // Reg is either in GPR32 or FGR32.
-        unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
-            nullptr, MRI->getDwarfRegNum(Reg, true), Offset));
-        BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-            .addCFIIndex(CFIIndex);
+        CFIBuilder.buildOffset(Reg, Offset);
       }
     }
   }
@@ -513,11 +488,7 @@ void MipsSEFrameLowering::emitPrologue(MachineFunction &MF,
     // Emit .cfi_offset directives for eh data registers.
     for (int I = 0; I < 4; ++I) {
       int64_t Offset = MFI.getObjectOffset(MipsFI->getEhDataRegFI(I));
-      unsigned Reg = MRI->getDwarfRegNum(ABI.GetEhDataReg(I), true);
-      unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::createOffset(nullptr, Reg, Offset));
-      BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-          .addCFIIndex(CFIIndex);
+      CFIBuilder.buildOffset(ABI.GetEhDataReg(I), Offset);
     }
   }
 
@@ -527,11 +498,7 @@ void MipsSEFrameLowering::emitPrologue(MachineFunction &MF,
     BuildMI(MBB, MBBI, dl, TII.get(MOVE), FP).addReg(SP).addReg(ZERO)
       .setMIFlag(MachineInstr::FrameSetup);
 
-    // emit ".cfi_def_cfa_register $fp"
-    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(
-        nullptr, MRI->getDwarfRegNum(FP, true)));
-    BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
-        .addCFIIndex(CFIIndex);
+    CFIBuilder.buildDefCFARegister(FP);
 
     if (RegInfo.hasStackRealignment(MF)) {
       // addiu $Reg, $zero, -MaxAlignment

From c6e7bb19f70f6509e1f54e6010a6de50f6a4804f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 18:29:51 -0700
Subject: [PATCH 593/710] [BOLT] Use llvm::unique (NFC) (#136513)

---
 bolt/lib/Core/BinaryBasicBlock.cpp | 3 +--
 bolt/lib/Core/BinaryFunction.cpp   | 2 +-
 bolt/lib/Core/DebugNames.cpp       | 3 +--
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/bolt/lib/Core/BinaryBasicBlock.cpp b/bolt/lib/Core/BinaryBasicBlock.cpp
index 2a2192b79bb4b..311d5c15b8dca 100644
--- a/bolt/lib/Core/BinaryBasicBlock.cpp
+++ b/bolt/lib/Core/BinaryBasicBlock.cpp
@@ -372,8 +372,7 @@ void BinaryBasicBlock::updateJumpTableSuccessors() {
              [](const BinaryBasicBlock *BB1, const BinaryBasicBlock *BB2) {
                return BB1->getInputOffset() < BB2->getInputOffset();
              });
-  SuccessorBBs.erase(std::unique(SuccessorBBs.begin(), SuccessorBBs.end()),
-                     SuccessorBBs.end());
+  SuccessorBBs.erase(llvm::unique(SuccessorBBs), SuccessorBBs.end());
 
   for (BinaryBasicBlock *BB : SuccessorBBs)
     addSuccessor(BB);
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 184a4462b356a..4624abadc701a 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -2376,7 +2376,7 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
   // Without doing jump table value profiling we don't have a use for extra
   // (duplicate) branches.
   llvm::sort(TakenBranches);
-  auto NewEnd = std::unique(TakenBranches.begin(), TakenBranches.end());
+  auto NewEnd = llvm::unique(TakenBranches);
   TakenBranches.erase(NewEnd, TakenBranches.end());
 
   for (std::pair<uint32_t, uint32_t> &Branch : TakenBranches) {
diff --git a/bolt/lib/Core/DebugNames.cpp b/bolt/lib/Core/DebugNames.cpp
index 366c22c38e616..82cabe689eb14 100644
--- a/bolt/lib/Core/DebugNames.cpp
+++ b/bolt/lib/Core/DebugNames.cpp
@@ -440,8 +440,7 @@ void DWARF5AcceleratorTable::computeBucketCount() {
   for (const auto &E : Entries)
     Uniques.push_back(E.second.HashValue);
   array_pod_sort(Uniques.begin(), Uniques.end());
-  std::vector<uint32_t>::iterator P =
-      std::unique(Uniques.begin(), Uniques.end());
+  std::vector<uint32_t>::iterator P = llvm::unique(Uniques);
 
   UniqueHashCount = std::distance(Uniques.begin(), P);
 

From be48727b95bf9075e4290cc8938ab87db8b7410c Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 18:30:05 -0700
Subject: [PATCH 594/710] [clang-tools-extra] Use llvm::unique (NFC) (#136514)

---
 clang-tools-extra/clang-doc/Representation.cpp |  4 ++--
 .../IncludeFixerContext.cpp                    | 18 +++++++++---------
 .../clang-tidy/ClangTidyDiagnosticConsumer.cpp |  3 +--
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 54d2cb58ea2d9..9ab2f342d969a 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -200,7 +200,7 @@ void Info::mergeBase(Info &&Other) {
   std::move(Other.Description.begin(), Other.Description.end(),
             std::back_inserter(Description));
   llvm::sort(Description);
-  auto Last = std::unique(Description.begin(), Description.end());
+  auto Last = llvm::unique(Description);
   Description.erase(Last, Description.end());
 }
 
@@ -215,7 +215,7 @@ void SymbolInfo::merge(SymbolInfo &&Other) {
   // Unconditionally extend the list of locations, since we want all of them.
   std::move(Other.Loc.begin(), Other.Loc.end(), std::back_inserter(Loc));
   llvm::sort(Loc);
-  auto *Last = std::unique(Loc.begin(), Loc.end());
+  auto *Last = llvm::unique(Loc);
   Loc.erase(Last, Loc.end());
   mergeBase(std::move(Other));
 }
diff --git a/clang-tools-extra/clang-include-fixer/IncludeFixerContext.cpp b/clang-tools-extra/clang-include-fixer/IncludeFixerContext.cpp
index d7369b162dc10..4eac0617ed4a9 100644
--- a/clang-tools-extra/clang-include-fixer/IncludeFixerContext.cpp
+++ b/clang-tools-extra/clang-include-fixer/IncludeFixerContext.cpp
@@ -90,10 +90,10 @@ IncludeFixerContext::IncludeFixerContext(
                       std::make_pair(B.Range.getOffset(), B.Range.getLength());
              });
   QuerySymbolInfos.erase(
-      std::unique(QuerySymbolInfos.begin(), QuerySymbolInfos.end(),
-                  [](const QuerySymbolInfo &A, const QuerySymbolInfo &B) {
-                    return A.Range == B.Range;
-                  }),
+      llvm::unique(QuerySymbolInfos,
+                   [](const QuerySymbolInfo &A, const QuerySymbolInfo &B) {
+                     return A.Range == B.Range;
+                   }),
       QuerySymbolInfos.end());
   for (const auto &Symbol : MatchedSymbols) {
     HeaderInfos.push_back(
@@ -103,11 +103,11 @@ IncludeFixerContext::IncludeFixerContext(
              QuerySymbolInfos.front().ScopedQualifiers, Symbol)});
   }
   // Deduplicate header infos.
-  HeaderInfos.erase(std::unique(HeaderInfos.begin(), HeaderInfos.end(),
-                                [](const HeaderInfo &A, const HeaderInfo &B) {
-                                  return A.Header == B.Header &&
-                                         A.QualifiedName == B.QualifiedName;
-                                }),
+  HeaderInfos.erase(llvm::unique(HeaderInfos,
+                                 [](const HeaderInfo &A, const HeaderInfo &B) {
+                                   return A.Header == B.Header &&
+                                          A.QualifiedName == B.QualifiedName;
+                                 }),
                     HeaderInfos.end());
 }
 
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index 731141a545a48..b216970bfbd8c 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -754,8 +754,7 @@ std::vector<ClangTidyError> ClangTidyDiagnosticConsumer::take() {
   finalizeLastError();
 
   llvm::stable_sort(Errors, LessClangTidyError());
-  Errors.erase(std::unique(Errors.begin(), Errors.end(), EqualClangTidyError()),
-               Errors.end());
+  Errors.erase(llvm::unique(Errors, EqualClangTidyError()), Errors.end());
   if (RemoveIncompatibleErrors)
     removeIncompatibleErrors();
   return std::move(Errors);

From 27653bdc49161b6d0e785185384a5c96a55e9e24 Mon Sep 17 00:00:00 2001
From: Oliver Hunt <oliver@apple.com>
Date: Sun, 20 Apr 2025 18:42:09 -0700
Subject: [PATCH 595/710] Ensure bit-fields storing FPEvalMethodKind are wide
 enough to do so (#136515)

After landing #116760 we hit build failures due to existing fields
storing FPEvalMethodKind not being wide enough.
---
 clang/include/clang/Basic/FPOptions.def   | 2 +-
 clang/include/clang/Basic/LangOptions.def | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/FPOptions.def b/clang/include/clang/Basic/FPOptions.def
index 90428c3c73c8b..85986b4ff0b9c 100644
--- a/clang/include/clang/Basic/FPOptions.def
+++ b/clang/include/clang/Basic/FPOptions.def
@@ -24,7 +24,7 @@ OPTION(NoHonorInfs, bool, 1, NoHonorNaNs)
 OPTION(NoSignedZero, bool, 1, NoHonorInfs)
 OPTION(AllowReciprocal, bool, 1, NoSignedZero)
 OPTION(AllowApproxFunc, bool, 1, AllowReciprocal)
-OPTION(FPEvalMethod, LangOptions::FPEvalMethodKind, 2, AllowApproxFunc)
+OPTION(FPEvalMethod, LangOptions::FPEvalMethodKind, 3, AllowApproxFunc)
 OPTION(Float16ExcessPrecision, LangOptions::ExcessPrecisionKind, 2, FPEvalMethod)
 OPTION(BFloat16ExcessPrecision, LangOptions::ExcessPrecisionKind, 2, Float16ExcessPrecision)
 OPTION(MathErrno, bool, 1, BFloat16ExcessPrecision)
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 930c1c06d1a76..85ca523c44157 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -347,7 +347,7 @@ BENIGN_ENUM_LANGOPT(DefaultFPContractMode, FPModeKind, 2, FPM_Off, "FP contracti
 COMPATIBLE_LANGOPT(ExpStrictFP, 1, false, "Enable experimental strict floating point")
 BENIGN_LANGOPT(RoundingMath, 1, false, "Do not assume default floating-point rounding behavior")
 BENIGN_ENUM_LANGOPT(FPExceptionMode, FPExceptionModeKind, 2, FPE_Default, "FP Exception Behavior Mode type")
-BENIGN_ENUM_LANGOPT(FPEvalMethod, FPEvalMethodKind, 2, FEM_UnsetOnCommandLine, "FP type used for floating point arithmetic")
+BENIGN_ENUM_LANGOPT(FPEvalMethod, FPEvalMethodKind, 3, FEM_UnsetOnCommandLine, "FP type used for floating point arithmetic")
 ENUM_LANGOPT(Float16ExcessPrecision, ExcessPrecisionKind, 2, FPP_Standard, "Intermediate truncation behavior for Float16 arithmetic")
 ENUM_LANGOPT(BFloat16ExcessPrecision, ExcessPrecisionKind, 2, FPP_Standard, "Intermediate truncation behavior for BFloat16 arithmetic")
 LANGOPT(NoBitFieldTypeAlign , 1, 0, "bit-field type alignment")

From 0e3e0bf42c25b280d8caa455c6ae7e4a04d3667a Mon Sep 17 00:00:00 2001
From: Chyaka <lilium23187@gmail.com>
Date: Mon, 21 Apr 2025 10:06:43 +0800
Subject: [PATCH 596/710] [RISCV] Add processor definition for
 XiangShan-KunMingHu-V2R2 (#123193)

XiangShan-KunMingHu is the third generation of Open-source
high-performance RISC-V processor developed by Beijing Institute of Open
Source Chip (BOSC) , and its latest version is V2R2.

The KunMingHu manual is now available at
https://github.com/OpenXiangShan/XiangShan-User-Guide/releases.
It will be updated on the official XiangShan documentation site:
https://docs.xiangshan.cc/zh-cn/latest

You can find the corresponding ISA extension from the XiangShan Github
repository:
https://github.com/OpenXiangShan/XiangShan/blob/master/src/main/scala/xiangshan/Parameters.scala

If you want to track the latest performance data of KunMingHu, please
check XiangShan Biweekly: https://docs.xiangshan.cc/zh-cn/latest/blog

This PR adds the processor definition for KunMingHu V2R2, developed by
the XSCC team https://github.com/orgs/OpenXiangShan/teams/xscc.

The scheduling model for XiangShan-KunMingHu V2R2 will be submitted in a
subsequent PR.

---------

Co-authored-by: Shenglin Tang <tangshenglin@ict.ac.cn>
Co-authored-by: Xu, Zefan <ceba_robot@outlook.com>
Co-authored-by: Tang Haojin <tanghaojin@outlook.com>
---
 clang/test/Driver/riscv-cpus.c                | 72 +++++++++++++++++++
 .../test/Misc/target-invalid-cpu-note/riscv.c |  2 +
 llvm/docs/ReleaseNotes.md                     |  1 +
 llvm/lib/Target/RISCV/RISCVProcessors.td      | 27 +++++++
 4 files changed, 102 insertions(+)

diff --git a/clang/test/Driver/riscv-cpus.c b/clang/test/Driver/riscv-cpus.c
index e97b6940662d9..c2314efd34aa6 100644
--- a/clang/test/Driver/riscv-cpus.c
+++ b/clang/test/Driver/riscv-cpus.c
@@ -31,6 +31,78 @@
 // MCPU-XIANGSHAN-NANHU-SAME: "-target-feature" "+zks" "-target-feature" "+zksed" "-target-feature" "+zksh" "-target-feature" "+svinval"
 // MCPU-XIANGSHAN-NANHU-SAME: "-target-abi" "lp64d"
 
+// RUN: %clang --target=riscv64 -### -c %s 2>&1 -mcpu=xiangshan-kunminghu | FileCheck -check-prefix=MCPU-XIANGSHAN-KUNMINGHU %s
+// MCPU-XIANGSHAN-KUNMINGHU: "-nostdsysteminc" "-target-cpu" "xiangshan-kunminghu"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+m"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+a"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+f"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+d"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+c"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+b"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+v"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+h"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zic64b" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zicbom" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zicbop" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zicboz" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+ziccamoa" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+ziccif" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zicclsm" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+ziccrse" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zicntr" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zicond" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zicsr" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zihintntl" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zacas" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zawrs" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zfa" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zfh" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zca" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zcb" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zcmop" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zba" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zbb" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zbc"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zbkb" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zbkc" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zbkx" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zbs"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zkn" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zks" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zvbb" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zve64d" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zve64f" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zve64x" 
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zvfh"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zvkb"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zvkt"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+zvl128b"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+sha"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+shcounterenw"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+shgatpa"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+shtvala"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+shvsatpa"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+shvstvala"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+shvstvecd"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+smcsrind"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+smdbltrp"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+smmpm"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+smnpm"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+smrnmi"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+smstateen"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+sscofpmf"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+sscsrind"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+ssdbltrp"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+ssnpm"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+sspm"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+ssstateen"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+ssstrict"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+sstc"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+ssu64xl"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+supm"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-feature" "+svnapot"
+// MCPU-XIANGSHAN-KUNMINGHU-SAME: "-target-abi" "lp64d"
+
 // RUN: %clang --target=riscv64 -### -c %s 2>&1 -mcpu=spacemit-x60 | FileCheck -check-prefix=MCPU-SPACEMIT-X60 %s
 // MCPU-SPACEMIT-X60: "-nostdsysteminc" "-target-cpu" "spacemit-x60"
 // MCPU-SPACEMIT-X60-SAME: "-target-feature" "+m"
diff --git a/clang/test/Misc/target-invalid-cpu-note/riscv.c b/clang/test/Misc/target-invalid-cpu-note/riscv.c
index 6e4323958957a..199916f70c14f 100644
--- a/clang/test/Misc/target-invalid-cpu-note/riscv.c
+++ b/clang/test/Misc/target-invalid-cpu-note/riscv.c
@@ -45,6 +45,7 @@
 // RISCV64-SAME: {{^}}, syntacore-scr7
 // RISCV64-SAME: {{^}}, tt-ascalon-d8
 // RISCV64-SAME: {{^}}, veyron-v1
+// RISCV64-SAME: {{^}}, xiangshan-kunminghu
 // RISCV64-SAME: {{^}}, xiangshan-nanhu
 // RISCV64-SAME: {{$}}
 
@@ -95,6 +96,7 @@
 // TUNE-RISCV64-SAME: {{^}}, syntacore-scr7
 // TUNE-RISCV64-SAME: {{^}}, tt-ascalon-d8
 // TUNE-RISCV64-SAME: {{^}}, veyron-v1
+// TUNE-RISCV64-SAME: {{^}}, xiangshan-kunminghu
 // TUNE-RISCV64-SAME: {{^}}, xiangshan-nanhu
 // TUNE-RISCV64-SAME: {{^}}, generic
 // TUNE-RISCV64-SAME: {{^}}, generic-ooo
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 48c4663d63410..cc4fd38944ff6 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -167,6 +167,7 @@ Changes to the RISC-V Backend
 * Adds assembler support for ``.option exact``, which disables automatic compression,
   and branch and linker relaxation. This can be disabled with ``.option noexact``,
   which is also the default.
+* `-mcpu=xiangshan-kunminghu` was added.
 
 Changes to the WebAssembly Backend
 ----------------------------------
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index ece12c48b1cd9..4b288a9cfcb49 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -558,6 +558,33 @@ def XIANGSHAN_NANHU : RISCVProcessorModel<"xiangshan-nanhu",
                                             TuneZExtWFusion,
                                             TuneShiftedZExtWFusion]>;
 
+def XIANGSHAN_KUNMINGHU : RISCVProcessorModel<"xiangshan-kunminghu",
+                                              NoSchedModel,
+                                              !listconcat(RVA23S64Features,
+                                              [FeatureStdExtZacas,
+                                               FeatureStdExtZbc,
+                                               FeatureStdExtZfh,
+                                               FeatureStdExtZkn,
+                                               FeatureStdExtZks,
+                                               FeatureStdExtZvfh,
+                                               FeatureStdExtSmaia,
+                                               FeatureStdExtSmcsrind,
+                                               FeatureStdExtSmdbltrp,
+                                               FeatureStdExtSmmpm,
+                                               FeatureStdExtSmnpm,
+                                               FeatureStdExtSmrnmi,
+                                               FeatureStdExtSmstateen,
+                                               FeatureStdExtSsaia,
+                                               FeatureStdExtSscsrind,
+                                               FeatureStdExtSsdbltrp,
+                                               FeatureStdExtSspm,
+                                               FeatureStdExtSsstrict,
+                                               FeatureStdExtZvl128b]),
+                                              [TuneNoDefaultUnroll,
+                                               TuneZExtHFusion,
+                                               TuneZExtWFusion,
+                                               TuneShiftedZExtWFusion]>;
+
 def SPACEMIT_X60 : RISCVProcessorModel<"spacemit-x60",
                                        NoSchedModel,
                                        !listconcat(RVA22S64Features,

From ddee2d8559fd18519f712989dc59d299c3334fc1 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Mon, 21 Apr 2025 10:07:45 +0800
Subject: [PATCH 597/710] [X86][APX] Combine (X86Sub 0, AND(X, Y)) to (X86And
 X, Y) for CLOAD/CSTORE (#136429)

https://godbolt.org/z/TsWochrbf
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 32 +++++++++++++++-------
 llvm/test/CodeGen/X86/apx/cf.ll         | 36 +++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 024b653f78cb5..993118c52564e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -57808,10 +57808,6 @@ static SDValue combineSubSetcc(SDNode *N, SelectionDAG &DAG) {
 }
 
 static SDValue combineX86CloadCstore(SDNode *N, SelectionDAG &DAG) {
-  // res, flags2 = sub 0, (setcc cc, flag)
-  // cload/cstore ..., cond_ne, flag2
-  // ->
-  // cload/cstore cc, flag
   if (N->getConstantOperandVal(3) != X86::COND_NE)
     return SDValue();
 
@@ -57819,16 +57815,34 @@ static SDValue combineX86CloadCstore(SDNode *N, SelectionDAG &DAG) {
   if (Sub.getOpcode() != X86ISD::SUB)
     return SDValue();
 
-  SDValue SetCC = Sub.getOperand(1);
+  SDValue Op1 = Sub.getOperand(1);
 
-  if (!X86::isZeroNode(Sub.getOperand(0)) || SetCC.getOpcode() != X86ISD::SETCC)
+  if (!X86::isZeroNode(Sub.getOperand(0)))
     return SDValue();
 
+  SDLoc DL(N);
   SmallVector<SDValue, 5> Ops(N->op_values());
-  Ops[3] = SetCC.getOperand(0);
-  Ops[4] = SetCC.getOperand(1);
+  if (Op1.getOpcode() == X86ISD::SETCC) {
+    // res, flags2 = sub 0, (setcc cc, flag)
+    // cload/cstore ..., cond_ne, flag2
+    // ->
+    // cload/cstore cc, flag
+    Ops[3] = Op1.getOperand(0);
+    Ops[4] = Op1.getOperand(1);
+  } else if (Op1.getOpcode() == ISD::AND && Sub.getValue(0).use_empty()) {
+    // res, flags2 = sub 0, (and X, Y)
+    // cload/cstore ..., cond_ne, flag2
+    // ->
+    // res, flags2 = and X, Y
+    // cload/cstore ..., cond_ne, flag2
+    Ops[4] = DAG.getNode(X86ISD::AND, DL, Sub->getVTList(), Op1.getOperand(0),
+                         Op1.getOperand(1))
+                 .getValue(1);
+  } else {
+    return SDValue();
+  }
 
-  return DAG.getMemIntrinsicNode(N->getOpcode(), SDLoc(N), N->getVTList(), Ops,
+  return DAG.getMemIntrinsicNode(N->getOpcode(), DL, N->getVTList(), Ops,
                                  cast<MemSDNode>(N)->getMemoryVT(),
                                  cast<MemSDNode>(N)->getMemOperand());
 }
diff --git a/llvm/test/CodeGen/X86/apx/cf.ll b/llvm/test/CodeGen/X86/apx/cf.ll
index 8d104e5f3ced2..1e4ac3f419314 100644
--- a/llvm/test/CodeGen/X86/apx/cf.ll
+++ b/llvm/test/CodeGen/X86/apx/cf.ll
@@ -158,3 +158,39 @@ entry:
   tail call void @llvm.masked.store.v1i16.p0(<1 x i16> %5, ptr %p, i32 2, <1 x i1> %1)
   ret void
 }
+
+define void @load_zext(i1 %cond, ptr %b, ptr %p) {
+; CHECK-LABEL: load_zext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andb $1, %dil
+; CHECK-NEXT:    cfcmovnew (%rsi), %ax
+; CHECK-NEXT:    movzwl %ax, %eax
+; CHECK-NEXT:    cfcmovnel %eax, (%rdx)
+; CHECK-NEXT:    retq
+entry:
+  %0 = bitcast i1 %cond to <1 x i1>
+  %1 = call <1 x i16> @llvm.masked.load.v1i16.p0(ptr %b, i32 2, <1 x i1> %0, <1 x i16> poison)
+  %2 = bitcast <1 x i16> %1 to i16
+  %zext = zext i16 %2 to i32
+  %3 = bitcast i32 %zext to <1 x i32>
+  call void @llvm.masked.store.v1i32.p0(<1 x i32> %3, ptr %p, i32 4, <1 x i1> %0)
+  ret void
+}
+
+define void @load_sext(i1 %cond, ptr %b, ptr %p) {
+; CHECK-LABEL: load_sext:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andb $1, %dil
+; CHECK-NEXT:    cfcmovnel (%rsi), %eax
+; CHECK-NEXT:    cltq
+; CHECK-NEXT:    cfcmovneq %rax, (%rdx)
+; CHECK-NEXT:    retq
+entry:
+  %0 = bitcast i1 %cond to <1 x i1>
+  %1 = call <1 x i32> @llvm.masked.load.v1i32.p0(ptr %b, i32 2, <1 x i1> %0, <1 x i32> poison)
+  %2 = bitcast <1 x i32> %1 to i32
+  %zext = sext i32 %2 to i64
+  %3 = bitcast i64 %zext to <1 x i64>
+  call void @llvm.masked.store.v1i64.p0(<1 x i64> %3, ptr %p, i32 4, <1 x i1> %0)
+  ret void
+}

From f86953c733c084fd410ca9906f6a2c53641de519 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Sun, 20 Apr 2025 19:35:56 -0700
Subject: [PATCH 598/710] [docs] Correct my email address

---
 llvm/docs/CodeOfConduct.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/docs/CodeOfConduct.rst b/llvm/docs/CodeOfConduct.rst
index 4febc8a61986f..9f7c3b68ebf26 100644
--- a/llvm/docs/CodeOfConduct.rst
+++ b/llvm/docs/CodeOfConduct.rst
@@ -162,7 +162,7 @@ The current committee members are:
 * Aaron Ballman (aaron.ballman\@llvm.org)
 * Kristof Beyls (kristof.beyls\@llvm.org)
 * David Blaikie (dblaikie\@llvm.org)
-* Jonas Devlieghere (jonas.devlieghere\@llvm.org)
+* Jonas Devlieghere (jdevlieghere\@llvm.org)
 * Cyndy Ishida (cishida\@llvm.org)
 * Tanya Lattner (tanyalattner\@llvm.org)
 * Stella Stamenova (sstamenova\@llvm.org)

From a8644b3d881fabaac217ba8d8b4afa446e29b453 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 19:41:26 -0700
Subject: [PATCH 599/710] [BOLT] Call hash_combine_range with ranges (NFC)
 (#136524)

---
 bolt/lib/Profile/YAMLProfileWriter.cpp | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp
index e394858163560..2bdc7b65b4804 100644
--- a/bolt/lib/Profile/YAMLProfileWriter.cpp
+++ b/bolt/lib/Profile/YAMLProfileWriter.cpp
@@ -133,12 +133,9 @@ std::vector<yaml::bolt::PseudoProbeInfo>
 YAMLProfileWriter::convertNodeProbes(NodeIdToProbes &NodeProbes) {
   struct BlockProbeInfoHasher {
     size_t operator()(const yaml::bolt::PseudoProbeInfo &BPI) const {
-      auto HashCombine = [](auto &Range) {
-        return llvm::hash_combine_range(Range.begin(), Range.end());
-      };
-      return llvm::hash_combine(HashCombine(BPI.BlockProbes),
-                                HashCombine(BPI.CallProbes),
-                                HashCombine(BPI.IndCallProbes));
+      return llvm::hash_combine(llvm::hash_combine_range(BPI.BlockProbes),
+                                llvm::hash_combine_range(BPI.CallProbes),
+                                llvm::hash_combine_range(BPI.IndCallProbes));
     }
   };
 

From 02b55d2b78d318dad331a66ddac81c7318b0b87a Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 19:59:19 -0700
Subject: [PATCH 600/710] [AST] Call hash_combine_range with a range (NFC)
 (#136525)

---
 clang/lib/AST/ItaniumCXXABI.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/AST/ItaniumCXXABI.cpp b/clang/lib/AST/ItaniumCXXABI.cpp
index a1b2551419f5e..6ceedd657fe7e 100644
--- a/clang/lib/AST/ItaniumCXXABI.cpp
+++ b/clang/lib/AST/ItaniumCXXABI.cpp
@@ -110,7 +110,7 @@ struct DenseMapInfo<DecompositionDeclName> {
   }
   static unsigned getHashValue(DecompositionDeclName Key) {
     assert(!isEqual(Key, getEmptyKey()) && !isEqual(Key, getTombstoneKey()));
-    return llvm::hash_combine_range(Key.begin(), Key.end());
+    return llvm::hash_combine_range(Key);
   }
   static bool isEqual(DecompositionDeclName LHS, DecompositionDeclName RHS) {
     if (std::optional<bool> Result =

From 6274442f8c657597233b9691298df7b5cd743e66 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 19:59:28 -0700
Subject: [PATCH 601/710] [clangd] Call hash_combine_range with a range (NFC)
 (#136526)

---
 clang-tools-extra/clangd/SystemIncludeExtractor.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang-tools-extra/clangd/SystemIncludeExtractor.cpp b/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
index 9399b910025b6..d9642f1115a6d 100644
--- a/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
+++ b/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
@@ -239,8 +239,7 @@ template <> struct DenseMapInfo<DriverArgs> {
         Val.Stdlib,
     });
 
-    unsigned SpecsHash =
-        llvm::hash_combine_range(Val.Specs.begin(), Val.Specs.end());
+    unsigned SpecsHash = llvm::hash_combine_range(Val.Specs);
 
     return llvm::hash_combine(FixedFieldsHash, SpecsHash);
   }

From 2ba20c52e43ae881dc54037fff94d7e2f217c99d Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 20 Apr 2025 19:59:37 -0700
Subject: [PATCH 602/710] [lldb] Use llvm::unique (NFC) (#136527)

---
 lldb/source/Symbol/Symtab.cpp                              | 6 ++----
 lldb/source/Target/Target.cpp                              | 5 ++---
 lldb/tools/lldb-dap/Handler/BreakpointLocationsHandler.cpp | 3 +--
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/lldb/source/Symbol/Symtab.cpp b/lldb/source/Symbol/Symtab.cpp
index 3c5075d9bb18b..9aee5d3e813d8 100644
--- a/lldb/source/Symbol/Symtab.cpp
+++ b/lldb/source/Symbol/Symtab.cpp
@@ -642,7 +642,7 @@ void Symtab::SortSymbolIndexesByValue(std::vector<uint32_t> &indexes,
 
   // Remove any duplicates if requested
   if (remove_duplicates) {
-    auto last = std::unique(indexes.begin(), indexes.end());
+    auto last = llvm::unique(indexes);
     indexes.erase(last, indexes.end());
   }
 }
@@ -1151,9 +1151,7 @@ void Symtab::FindFunctionSymbols(ConstString name, uint32_t name_type_mask,
 
   if (!symbol_indexes.empty()) {
     llvm::sort(symbol_indexes);
-    symbol_indexes.erase(
-        std::unique(symbol_indexes.begin(), symbol_indexes.end()),
-        symbol_indexes.end());
+    symbol_indexes.erase(llvm::unique(symbol_indexes), symbol_indexes.end());
     SymbolIndicesToSymbolContextList(symbol_indexes, sc_list);
   }
 }
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index b6186b76d6236..0fa61b20e19b9 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -2641,9 +2641,8 @@ Target::GetScratchTypeSystems(bool create_on_demand) {
   }
 
   std::sort(scratch_type_systems.begin(), scratch_type_systems.end());
-  scratch_type_systems.erase(
-      std::unique(scratch_type_systems.begin(), scratch_type_systems.end()),
-      scratch_type_systems.end());
+  scratch_type_systems.erase(llvm::unique(scratch_type_systems),
+                             scratch_type_systems.end());
   return scratch_type_systems;
 }
 
diff --git a/lldb/tools/lldb-dap/Handler/BreakpointLocationsHandler.cpp b/lldb/tools/lldb-dap/Handler/BreakpointLocationsHandler.cpp
index 022baf77458d0..7a477f3e97875 100644
--- a/lldb/tools/lldb-dap/Handler/BreakpointLocationsHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/BreakpointLocationsHandler.cpp
@@ -189,8 +189,7 @@ void BreakpointLocationsRequestHandler::operator()(
   // The line entries are sorted by addresses, but we must return the list
   // ordered by line / column position.
   std::sort(locations.begin(), locations.end());
-  locations.erase(std::unique(locations.begin(), locations.end()),
-                  locations.end());
+  locations.erase(llvm::unique(locations), locations.end());
 
   llvm::json::Array locations_json;
   for (auto &l : locations) {

From f024aab844ab89ab6455ddf1e2232a957c72328e Mon Sep 17 00:00:00 2001
From: Jim Lin <jim@andestech.com>
Date: Mon, 21 Apr 2025 11:02:04 +0800
Subject: [PATCH 603/710] [RISCV] Remove the TODO that lbu and lhu should be
 selected to avoid the unnecessary masking. NFC.

RISCVTargetLowering::isZExtFree has been implemented in
https://github.com/llvm/llvm-project/commit/15e894baeeb96612ae471fa83d1729a2d3388fc8.
---
 llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
index 771a72f8d55f0..abd49d6bf7575 100644
--- a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
+++ b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
@@ -2,8 +2,6 @@
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV32I
 
-; TODO: lbu and lhu should be selected to avoid the unnecessary masking.
-
 @bytes = dso_local global [5 x i8] zeroinitializer, align 1
 
 define dso_local i32 @test_zext_i8() nounwind {

From e183459b8b341ec787b2856be7e86a9e6200e2f8 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 21 Apr 2025 05:33:30 +0100
Subject: [PATCH 604/710] [CostModel] Make sure getCmpSelInstrCost is passed a
 CondTy (#135535)

It is already required along certain code paths that the CondTy is
valid. Fix some of the uses to make sure it is passed.
---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h        | 8 +++-----
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp       | 3 ++-
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 ++++----
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp  | 8 ++++----
 4 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index be9ee027a31f1..f6ec21caa4d72 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1384,11 +1384,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
         return InstructionCost::getInvalid();
 
       unsigned Num = cast<FixedVectorType>(ValVTy)->getNumElements();
-      if (CondTy)
-        CondTy = CondTy->getScalarType();
-      InstructionCost Cost =
-          thisT()->getCmpSelInstrCost(Opcode, ValVTy->getScalarType(), CondTy,
-                                      VecPred, CostKind, Op1Info, Op2Info, I);
+      InstructionCost Cost = thisT()->getCmpSelInstrCost(
+          Opcode, ValVTy->getScalarType(), CondTy->getScalarType(), VecPred,
+          CostKind, Op1Info, Op2Info, I);
 
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 9e6ee58cafc79..6bf4ea37e2f96 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -3094,7 +3094,8 @@ static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB,
     if (ThenV == OrigV)
       continue;
 
-    Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr,
+    Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(),
+                                   CmpInst::makeCmpResultType(PN.getType()),
                                    CmpInst::BAD_ICMP_PREDICATE, CostKind);
 
     // Don't convert to selects if we could remove undefined behavior instead.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7a5f618d09e95..137f32ded7248 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6966,10 +6966,10 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
     }
 
     VectorTy = toVectorTy(ValTy, VF);
-    return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, nullptr,
-                                  cast<CmpInst>(I)->getPredicate(), CostKind,
-                                  {TTI::OK_AnyValue, TTI::OP_None},
-                                  {TTI::OK_AnyValue, TTI::OP_None}, I);
+    return TTI.getCmpSelInstrCost(
+        I->getOpcode(), VectorTy, CmpInst::makeCmpResultType(VectorTy),
+        cast<CmpInst>(I)->getPredicate(), CostKind,
+        {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, I);
   }
   case Instruction::Store:
   case Instruction::Load: {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c2cfd93d23ec0..ff3b2aee7916c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1765,10 +1765,10 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
   case Instruction::FCmp: {
     Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
     Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
-    return Ctx.TTI.getCmpSelInstrCost(Opcode, VectorTy, nullptr, getPredicate(),
-                                      Ctx.CostKind,
-                                      {TTI::OK_AnyValue, TTI::OP_None},
-                                      {TTI::OK_AnyValue, TTI::OP_None}, CtxI);
+    return Ctx.TTI.getCmpSelInstrCost(
+        Opcode, VectorTy, CmpInst::makeCmpResultType(VectorTy), getPredicate(),
+        Ctx.CostKind, {TTI::OK_AnyValue, TTI::OP_None},
+        {TTI::OK_AnyValue, TTI::OP_None}, CtxI);
   }
   default:
     llvm_unreachable("Unsupported opcode for instruction");

From 2ba455ff3dd192ff814514647d485491510a6342 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 21 Apr 2025 06:31:03 +0100
Subject: [PATCH 605/710] [AArch64] Add CostKind to getSpliceCost (#135537)

This likely does not alter much yet with how the costs are used. Like
other cost functions the CostKind should be passed into and through the
function.
---
 .../AArch64/AArch64TargetTransformInfo.cpp    |  6 ++--
 .../AArch64/AArch64TargetTransformInfo.h      |  3 +-
 .../CostModel/AArch64/sve-intrinsics.ll       | 32 +++++++++----------
 3 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 2b9d32f9208fe..372b70a4b2d64 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5207,7 +5207,8 @@ AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
   return BaseT::getMulAccReductionCost(IsUnsigned, ResTy, VecTy, CostKind);
 }
 
-InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index) {
+InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index,
+                                              TTI::TargetCostKind CostKind) {
   static const CostTblEntry ShuffleTbl[] = {
       { TTI::SK_Splice, MVT::nxv16i8,  1 },
       { TTI::SK_Splice, MVT::nxv8i16,  1 },
@@ -5233,7 +5234,6 @@ InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index) {
 
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
   Type *LegalVTy = EVT(LT.second).getTypeForEVT(Tp->getContext());
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   EVT PromotedVT = LT.second.getScalarType() == MVT::i1
                        ? TLI->getPromotedVTForPredicate(EVT(LT.second))
                        : LT.second;
@@ -5616,7 +5616,7 @@ InstructionCost AArch64TTIImpl::getShuffleCost(
   }
 
   if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(Tp))
-    return getSpliceCost(Tp, Index);
+    return getSpliceCost(Tp, Index, CostKind);
 
   // Inserting a subvector can often be done with either a D, S or H register
   // move, so long as the inserted vector is "aligned".
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index ae0df6b895ec8..372ec22bd548f 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -219,7 +219,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
                                                 VectorType *ValTy,
                                                 TTI::TargetCostKind CostKind);
 
-  InstructionCost getSpliceCost(VectorType *Tp, int Index);
+  InstructionCost getSpliceCost(VectorType *Tp, int Index,
+                                TTI::TargetCostKind CostKind);
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index 1483d476bef0d..1716f48f4b042 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -638,10 +638,10 @@ define void @vector_splice() #0 {
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:3 SizeLat:3 for: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:3 SizeLat:3 for: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:3 SizeLat:3 for: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:3 SizeLat:3 for: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
@@ -671,10 +671,10 @@ define void @vector_splice() #0 {
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
@@ -702,10 +702,10 @@ define void @vector_splice() #0 {
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:3 SizeLat:3 for: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:3 SizeLat:3 for: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:3 SizeLat:3 for: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:3 SizeLat:3 for: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
@@ -735,10 +735,10 @@ define void @vector_splice() #0 {
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;

From 8435de0916d9df5a3a9dd9eeb44d5bf4aba87ba3 Mon Sep 17 00:00:00 2001
From: Tongsheng Wu <tongshengwu0@gmail.com>
Date: Mon, 21 Apr 2025 02:09:51 -0400
Subject: [PATCH 606/710] [clangd] Print include spelling in IncludeCleaner
 error message (#136237)

---
 clang-tools-extra/clangd/IncludeCleaner.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp
index e34706172f0bf..dc4c8fc498b1f 100644
--- a/clang-tools-extra/clangd/IncludeCleaner.cpp
+++ b/clang-tools-extra/clangd/IncludeCleaner.cpp
@@ -345,8 +345,9 @@ include_cleaner::Includes convertIncludes(const ParsedAST &AST) {
     // which is based on FileManager::getCanonicalName(ParentDir).
     auto FE = SM.getFileManager().getFileRef(Inc.Resolved);
     if (!FE) {
-      elog("IncludeCleaner: Failed to get an entry for resolved path {0}: {1}",
-           Inc.Resolved, FE.takeError());
+      elog("IncludeCleaner: Failed to get an entry for resolved path '{0}' "
+           "from include {1} : {2}",
+           Inc.Resolved, Inc.Written, FE.takeError());
       continue;
     }
     TransformedInc.Resolved = *FE;

From 050ca5e27082055f41ad7e3dd2dea2f39364b3ab Mon Sep 17 00:00:00 2001
From: Henrich Lauko <xlauko@mail.muni.cz>
Date: Mon, 21 Apr 2025 09:07:57 +0200
Subject: [PATCH 607/710] [CIR][NFC] Simplify BoolAttr builders (#136366)

This mirrors incubator changes from https://github.com/llvm/clangir/pull/1572
---
 .../clang/CIR/Dialect/Builder/CIRBaseBuilder.h      | 10 +++++++---
 clang/include/clang/CIR/Dialect/IR/CIRAttrs.td      |  6 ++++++
 clang/include/clang/CIR/Dialect/IR/CIROps.td        |  8 +++++++-
 clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp          | 13 +++----------
 clang/lib/CIR/CodeGen/CIRGenStmt.cpp                |  4 +---
 clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp |  4 +---
 6 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index d2a241964f34f..ee8af62ede0da 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -57,6 +57,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 public:
   CIRBaseBuilderTy(mlir::MLIRContext &mlirContext)
       : mlir::OpBuilder(&mlirContext) {}
+  CIRBaseBuilderTy(mlir::OpBuilder &builder) : mlir::OpBuilder(builder) {}
 
   mlir::Value getConstAPInt(mlir::Location loc, mlir::Type typ,
                             const llvm::APInt &val) {
@@ -98,13 +99,13 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     if (auto recordTy = mlir::dyn_cast<cir::RecordType>(ty))
       return getZeroAttr(recordTy);
     if (mlir::isa<cir::BoolType>(ty)) {
-      return getCIRBoolAttr(false);
+      return getFalseAttr();
     }
     llvm_unreachable("Zero initializer for given type is NYI");
   }
 
   cir::ConstantOp getBool(bool state, mlir::Location loc) {
-    return create<cir::ConstantOp>(loc, getBoolTy(), getCIRBoolAttr(state));
+    return create<cir::ConstantOp>(loc, getCIRBoolAttr(state));
   }
   cir::ConstantOp getFalse(mlir::Location loc) { return getBool(false, loc); }
   cir::ConstantOp getTrue(mlir::Location loc) { return getBool(true, loc); }
@@ -120,9 +121,12 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   }
 
   cir::BoolAttr getCIRBoolAttr(bool state) {
-    return cir::BoolAttr::get(getContext(), getBoolTy(), state);
+    return cir::BoolAttr::get(getContext(), state);
   }
 
+  cir::BoolAttr getTrueAttr() { return getCIRBoolAttr(true); }
+  cir::BoolAttr getFalseAttr() { return getCIRBoolAttr(false); }
+
   mlir::Value createNot(mlir::Value value) {
     return create<cir::UnaryOp>(value.getLoc(), value.getType(),
                                 cir::UnaryOpKind::Not, value);
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index 3680ded4afafe..25ceded7e8a5b 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -49,6 +49,12 @@ def CIR_BoolAttr : CIR_Attr<"Bool", "bool", [TypedAttrInterface]> {
                         "", "cir::BoolType">:$type,
                     "bool":$value);
 
+  let builders = [
+    AttrBuilder<(ins "bool":$value), [{
+      return $_get($_ctxt, cir::BoolType::get($_ctxt), value);
+    }]>,
+  ];
+
   let assemblyFormat = [{
     `<` $value `>`
   }];
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 5ba4b33dc1a12..b526d077a910c 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -294,6 +294,12 @@ def ConstantOp : CIR_Op<"const",
   // The constant operation returns a single value of CIR_AnyType.
   let results = (outs CIR_AnyType:$res);
 
+  let builders = [
+    OpBuilder<(ins "cir::BoolAttr":$value), [{
+      build($_builder, $_state, value.getType(), value);
+    }]>
+  ];
+
   let assemblyFormat = "attr-dict $value";
 
   let hasVerifier = 1;
@@ -844,7 +850,7 @@ def UnaryOp : CIR_Op<"unary", [Pure, SameOperandsAndResultType]> {
   let assemblyFormat = [{
       `(` $kind `,` $input `)`
       (`nsw` $no_signed_wrap^)?
-      `:` type($input) `,` type($result) attr-dict 
+      `:` type($input) `,` type($result) attr-dict
   }];
 
   let hasVerifier = 1;
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 1bef1b976a4b5..f1561d1b26fc0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -152,10 +152,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
   }
 
   mlir::Value VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *e) {
-    mlir::Type type = cgf.convertType(e->getType());
-    return builder.create<cir::ConstantOp>(
-        cgf.getLoc(e->getExprLoc()), type,
-        builder.getCIRBoolAttr(e->getValue()));
+    return builder.getBool(e->getValue(), cgf.getLoc(e->getExprLoc()));
   }
 
   mlir::Value VisitCastExpr(CastExpr *e);
@@ -215,9 +212,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
 
     if (llvm::isa<MemberPointerType>(srcType)) {
       cgf.getCIRGenModule().errorNYI(loc, "member pointer to bool conversion");
-      mlir::Type boolType = builder.getBoolTy();
-      return builder.create<cir::ConstantOp>(loc, boolType,
-                                             builder.getCIRBoolAttr(false));
+      return builder.getFalse(loc);
     }
 
     if (srcType->isIntegerType())
@@ -354,9 +349,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     // An interesting aspect of this is that increment is always true.
     // Decrement does not have this property.
     if (isInc && type->isBooleanType()) {
-      value = builder.create<cir::ConstantOp>(cgf.getLoc(e->getExprLoc()),
-                                              cgf.convertType(type),
-                                              builder.getCIRBoolAttr(true));
+      value = builder.getTrue(cgf.getLoc(e->getExprLoc()));
     } else if (type->isIntegerType()) {
       QualType promotedType;
       bool canPerformLossyDemotionCheck = false;
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index f503e2344f952..82ac53706b7f9 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -456,9 +456,7 @@ mlir::LogicalResult CIRGenFunction::emitForStmt(const ForStmt &s) {
             // scalar type.
             condVal = evaluateExprAsBool(s.getCond());
           } else {
-            cir::BoolType boolTy = cir::BoolType::get(b.getContext());
-            condVal = b.create<cir::ConstantOp>(
-                loc, boolTy, cir::BoolAttr::get(b.getContext(), boolTy, true));
+            condVal = b.create<cir::ConstantOp>(loc, builder.getTrueAttr());
           }
           builder.createCondition(condVal);
         },
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 8c4a67258df3f..945e415d0dada 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -692,9 +692,7 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
     // during a pass as long as they don't live past the end of the pass.
     attr = op.getValue();
   } else if (mlir::isa<cir::BoolType>(op.getType())) {
-    int value = (op.getValue() ==
-                 cir::BoolAttr::get(getContext(),
-                                    cir::BoolType::get(getContext()), true));
+    int value = mlir::cast<cir::BoolAttr>(op.getValue()).getValue();
     attr = rewriter.getIntegerAttr(typeConverter->convertType(op.getType()),
                                    value);
   } else if (mlir::isa<cir::IntType>(op.getType())) {

From b6820c35c59a4da3e59c11f657093ffbd79ae1db Mon Sep 17 00:00:00 2001
From: Ryotaro Kasuga <kasuga.ryotaro@fujitsu.com>
Date: Mon, 21 Apr 2025 16:12:25 +0900
Subject: [PATCH 608/710] [MachinePipeliner] Remove UB from tests (NFC)
 (#123169)

This patch removes UB from some tests for MachinePipeliner. This patch
fixes following cases.

- Branching on an `undef` value.
- Using `undef`/`null` as a pointer operand of a load/store.

There are other tests of pipeliner that contain the same UB, but for
now, this patch fixes particularly unstable cases when I developed
pipeliner.
---
 llvm/test/CodeGen/Hexagon/swp-epilog-phi11.ll |  6 +++---
 llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll | 13 +++++++------
 llvm/test/CodeGen/Hexagon/swp-epilog-phi13.ll |  9 +++++----
 llvm/test/CodeGen/Hexagon/swp-epilog-phi6.ll  |  6 +++---
 llvm/test/CodeGen/Hexagon/swp-epilog-phi9.ll  | 13 +++++++------
 5 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi11.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi11.ll
index f98677570d93d..d1b9c51c45a2d 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi11.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi11.ll
@@ -9,7 +9,7 @@
 ; CHECK: r{{[0-9]+}} = sfsub([[REG0]],[[REG1]])
 ; CHECK: r{{[0-9]+}} = sfsub([[REG0]],r{{[0-9]+}})
 
-define dso_local void @test(i32 %m) local_unnamed_addr #0 {
+define dso_local void @test(i32 %m, ptr noalias %p0, ptr noalias %p1) local_unnamed_addr #0 {
 entry:
   %div = sdiv i32 %m, 2
   %sub = add nsw i32 %div, -1
@@ -20,9 +20,9 @@ for.body.prol:
   %sr.prol = phi float [ %0, %for.body.prol ], [ undef, %entry ]
   %sr109.prol = phi float [ %sr.prol, %for.body.prol ], [ undef, %entry ]
   %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ undef, %entry ]
-  %0 = load float, ptr undef, align 4
+  %0 = load float, ptr %p0, align 4
   %sub7.prol = fsub contract float %sr109.prol, %0
-  store float %sub7.prol, ptr null, align 4
+  store float %sub7.prol, ptr %p1, align 4
   %prol.iter.sub = add i32 %prol.iter, -1
   %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
   br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll
index 0a2d1f16bcbfd..ba479b696f16c 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi12.ll
@@ -10,21 +10,22 @@
 ; CHECK: = add([[REG1]],#8)
 
 ; Function Attrs: nounwind
-define ptr @f0(ptr nocapture readonly %a0, i32 %a1) #0 {
+define ptr @f0(ptr nocapture readonly %a0, i32 %a1, ptr noalias %p0, ptr noalias %p1, ptr noalias %p2) #0 {
 b0:
   %v0 = alloca [129 x i32], align 8
-  br i1 undef, label %b1, label %b3
+  %cond = freeze i1 poison
+  br i1 %cond, label %b1, label %b3
 
 b1:                                               ; preds = %b0
   br label %b2
 
 b2:                                               ; preds = %b2, %b1
   %v1 = phi ptr [ %a0, %b1 ], [ %v2, %b2 ]
-  %v2 = phi ptr [ undef, %b1 ], [ %v15, %b2 ]
-  %v3 = phi ptr [ null, %b1 ], [ %v4, %b2 ]
-  %v4 = phi ptr [ null, %b1 ], [ %v14, %b2 ]
+  %v2 = phi ptr [ %p0, %b1 ], [ %v15, %b2 ]
+  %v3 = phi ptr [ %p1, %b1 ], [ %v4, %b2 ]
+  %v4 = phi ptr [ %p1, %b1 ], [ %v14, %b2 ]
   %v5 = phi i32 [ 0, %b1 ], [ %v13, %b2 ]
-  %v6 = phi ptr [ undef, %b1 ], [ %v12, %b2 ]
+  %v6 = phi ptr [ %p2, %b1 ], [ %v12, %b2 ]
   %v7 = load i16, ptr %v2, align 2
   %v8 = sext i16 %v7 to i32
   %v9 = call i32 @llvm.hexagon.M2.mpy.ll.s0(i32 %v8, i32 %v8) #2
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi13.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi13.ll
index 82a1067becede..fa0f4f946e183 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi13.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi13.ll
@@ -10,9 +10,10 @@
 ; CHECK: endloop0
 
 ; Function Attrs: nounwind
-define ptr @f0(ptr nocapture readonly %a0, i32 %a1, i32 %a2, i32 %a3,  ptr %b) #0 {
+define ptr @f0(ptr nocapture readonly %a0, i32 %a1, i32 %a2, i32 %a3, ptr %b, ptr %c) #0 {
 b0:
-  br i1 undef, label %b1, label %b3
+  %cond = freeze i1 poison
+  br i1 %cond, label %b1, label %b3
 
 b1:                                               ; preds = %b0
   br label %b2
@@ -20,8 +21,8 @@ b1:                                               ; preds = %b0
 b2:                                               ; preds = %b2, %b1
   %v1 = phi ptr [ %a0, %b1 ], [ %v2, %b2 ]
   %v2 = phi ptr [ undef, %b1 ], [ %v15, %b2 ]
-  %v3 = phi ptr [ null, %b1 ], [ %v4, %b2 ]
-  %v4 = phi ptr [ null, %b1 ], [ %v14, %b2 ]
+  %v3 = phi ptr [ %c, %b1 ], [ %v4, %b2 ]
+  %v4 = phi ptr [ %c, %b1 ], [ %v14, %b2 ]
   %v5 = phi i32 [ 0, %b1 ], [ %v13, %b2 ]
   %v6 = phi ptr [ undef, %b1 ], [ %v12, %b2 ]
   %a = mul i32 %v5, %a2
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi6.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi6.ll
index 424c6e448e764..3b9ee75b9ffaf 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi6.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi6.ll
@@ -13,7 +13,7 @@
 ; CHECK: epilog:
 ; CHECK: %{{[0-9]+}}:intregs = PHI %{{.*}}, %[[REG]]
 
-define void @f0(i32 %a0, i32 %a1) #0 {
+define void @f0(i32 %a0, i32 %a1, ptr noalias %p0, ptr noalias %p1) #0 {
 b0:
   %v0 = icmp sgt i32 %a0, 64
   br i1 %v0, label %b1, label %b3
@@ -23,7 +23,7 @@ b1:                                               ; preds = %b0
 
 b2:                                               ; preds = %b2, %b1
   %v1 = phi i32 [ %a0, %b1 ], [ %v13, %b2 ]
-  %v2 = phi ptr [ null, %b1 ], [ %v3, %b2 ]
+  %v2 = phi ptr [ %p0, %b1 ], [ %v3, %b2 ]
   %v3 = getelementptr inbounds <16 x i32>, ptr %v2, i32 1
   %v4 = load <16 x i32>, ptr %v2, align 64
   %v5 = load <16 x i32>, ptr undef, align 64
@@ -34,7 +34,7 @@ b2:                                               ; preds = %b2, %b1
   %v10 = tail call <32 x i32> @llvm.hexagon.V6.vmpybus.acc(<32 x i32> %v9, <16 x i32> zeroinitializer, i32 undef)
   %v11 = tail call <16 x i32> @llvm.hexagon.V6.lo(<32 x i32> %v10)
   %v12 = tail call <16 x i32> @llvm.hexagon.V6.vasrhubsat(<16 x i32> undef, <16 x i32> %v11, i32 %a1)
-  store <16 x i32> %v12, ptr null, align 64
+  store <16 x i32> %v12, ptr %p1, align 64
   %v13 = add nsw i32 %v1, -64
   %v14 = icmp sgt i32 %v13, 64
   br i1 %v14, label %b2, label %b3
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi9.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi9.ll
index af1b848a8cf2d..08e3e604b5b05 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi9.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi9.ll
@@ -12,21 +12,22 @@
 ; CHECK: [[REG0]] = add(r{{[0-9]+}},#8)
 
 ; Function Attrs: nounwind
-define void @f0(ptr nocapture readonly %a0, i32 %a1) #0 {
+define void @f0(ptr noalias nocapture readonly %a0, i32 %a1, ptr noalias %a2, ptr %a3) #0 {
 b0:
   %v0 = alloca [129 x i32], align 8
-  br i1 undef, label %b1, label %b3
+  %cond = freeze i1 poison
+  br i1 %cond, label %b1, label %b3
 
 b1:                                               ; preds = %b0
   br label %b2
 
 b2:                                               ; preds = %b2, %b1
   %v1 = phi ptr [ %a0, %b1 ], [ %v2, %b2 ]
-  %v2 = phi ptr [ undef, %b1 ], [ %v15, %b2 ]
-  %v3 = phi ptr [ null, %b1 ], [ %v4, %b2 ]
-  %v4 = phi ptr [ null, %b1 ], [ %v14, %b2 ]
+  %v2 = phi ptr [ %a0, %b1 ], [ %v15, %b2 ]
+  %v3 = phi ptr [ %a2, %b1 ], [ %v4, %b2 ]
+  %v4 = phi ptr [ %a2, %b1 ], [ %v14, %b2 ]
   %v5 = phi i32 [ 0, %b1 ], [ %v13, %b2 ]
-  %v6 = phi ptr [ undef, %b1 ], [ %v12, %b2 ]
+  %v6 = phi ptr [ %a3, %b1 ], [ %v12, %b2 ]
   %v7 = load i16, ptr %v2, align 2
   %v8 = sext i16 %v7 to i32
   %v9 = call i32 @llvm.hexagon.M2.mpy.ll.s0(i32 %v8, i32 %v8) #2

From 4853bf0956058f4433fb81e880a2087da4487f26 Mon Sep 17 00:00:00 2001
From: tangaac <tangyan01@loongson.cn>
Date: Mon, 21 Apr 2025 16:04:28 +0800
Subject: [PATCH 609/710] [LoongArch] Lower build_vector to broadcast load if
 possible (#135896)

---
 .../LoongArch/LoongArchISelLowering.cpp       | 49 +++++++++++++++++++
 .../Target/LoongArch/LoongArchISelLowering.h  |  5 +-
 .../Target/LoongArch/LoongArchInstrInfo.td    |  9 ++--
 .../LoongArch/LoongArchLASXInstrInfo.td       |  8 +++
 .../Target/LoongArch/LoongArchLSXInstrInfo.td | 21 ++++++++
 .../CodeGen/LoongArch/lasx/broadcast-load.ll  | 40 ++++++---------
 .../CodeGen/LoongArch/lsx/broadcast-load.ll   | 40 ++++++---------
 7 files changed, 116 insertions(+), 56 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 7e37bbd652114..d4e1d9c6f3ca6 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -1876,6 +1876,51 @@ static bool isConstantOrUndefBUILD_VECTOR(const BuildVectorSDNode *Op) {
   return false;
 }
 
+// Lower BUILD_VECTOR as broadcast load (if possible).
+// For example:
+//   %a = load i8, ptr %ptr
+//   %b = build_vector %a, %a, %a, %a
+// is lowered to :
+//   (VLDREPL_B $a0, 0)
+static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp,
+                                                const SDLoc &DL,
+                                                SelectionDAG &DAG) {
+  MVT VT = BVOp->getSimpleValueType(0);
+  int NumOps = BVOp->getNumOperands();
+
+  assert((VT.is128BitVector() || VT.is256BitVector()) &&
+         "Unsupported vector type for broadcast.");
+
+  SDValue IdentitySrc;
+  bool IsIdeneity = true;
+
+  for (int i = 0; i != NumOps; i++) {
+    SDValue Op = BVOp->getOperand(i);
+    if (Op.getOpcode() != ISD::LOAD || (IdentitySrc && Op != IdentitySrc)) {
+      IsIdeneity = false;
+      break;
+    }
+    IdentitySrc = BVOp->getOperand(0);
+  }
+
+  // make sure that this load is valid and only has one user.
+  if (!IdentitySrc || !BVOp->isOnlyUserOf(IdentitySrc.getNode()))
+    return SDValue();
+
+  if (IsIdeneity) {
+    auto *LN = cast<LoadSDNode>(IdentitySrc);
+    SDVTList Tys =
+        LN->isIndexed()
+            ? DAG.getVTList(VT, LN->getBasePtr().getValueType(), MVT::Other)
+            : DAG.getVTList(VT, MVT::Other);
+    SDValue Ops[] = {LN->getChain(), LN->getBasePtr(), LN->getOffset()};
+    SDValue BCast = DAG.getNode(LoongArchISD::VLDREPL, DL, Tys, Ops);
+    DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
+    return BCast;
+  }
+  return SDValue();
+}
+
 SDValue LoongArchTargetLowering::lowerBUILD_VECTOR(SDValue Op,
                                                    SelectionDAG &DAG) const {
   BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Op);
@@ -1891,6 +1936,9 @@ SDValue LoongArchTargetLowering::lowerBUILD_VECTOR(SDValue Op,
       (!Subtarget.hasExtLASX() || !Is256Vec))
     return SDValue();
 
+  if (SDValue Result = lowerBUILD_VECTORAsBroadCastLoad(Node, DL, DAG))
+    return Result;
+
   if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
                             /*MinSplatBits=*/8) &&
       SplatBitSize <= 64) {
@@ -5326,6 +5374,7 @@ const char *LoongArchTargetLowering::getTargetNodeName(unsigned Opcode) const {
     NODE_NAME_CASE(VSRLI)
     NODE_NAME_CASE(VBSLL)
     NODE_NAME_CASE(VBSRL)
+    NODE_NAME_CASE(VLDREPL)
   }
 #undef NODE_NAME_CASE
   return nullptr;
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 8c2d2597a26ec..40a36b653b1b3 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -155,7 +155,10 @@ enum NodeType : unsigned {
 
   // Vector byte logicial left / right shift
   VBSLL,
-  VBSRL
+  VBSRL,
+
+  // Scalar load broadcast to vector
+  VLDREPL
 
   // Intrinsic operations end =============================================
 };
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index e4feaa600c57d..775d9289af7c4 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -307,7 +307,8 @@ def simm8_lsl # I : Operand<GRLenVT> {
 }
 }
 
-def simm9_lsl3 : Operand<GRLenVT> {
+def simm9_lsl3 : Operand<GRLenVT>,
+                 ImmLeaf<GRLenVT, [{return isShiftedInt<9,3>(Imm);}]> {
   let ParserMatchClass = SImmAsmOperand<9, "lsl3">;
   let EncoderMethod = "getImmOpValueAsr<3>";
   let DecoderMethod = "decodeSImmOperand<9, 3>";
@@ -317,13 +318,15 @@ def simm10 : Operand<GRLenVT> {
   let ParserMatchClass = SImmAsmOperand<10>;
 }
 
-def simm10_lsl2 : Operand<GRLenVT> {
+def simm10_lsl2 : Operand<GRLenVT>,
+                  ImmLeaf<GRLenVT, [{return isShiftedInt<10,2>(Imm);}]> {
   let ParserMatchClass = SImmAsmOperand<10, "lsl2">;
   let EncoderMethod = "getImmOpValueAsr<2>";
   let DecoderMethod = "decodeSImmOperand<10, 2>";
 }
 
-def simm11_lsl1 : Operand<GRLenVT> {
+def simm11_lsl1 : Operand<GRLenVT>,
+                  ImmLeaf<GRLenVT, [{return isShiftedInt<11,1>(Imm);}]> {
   let ParserMatchClass = SImmAsmOperand<11, "lsl1">;
   let EncoderMethod = "getImmOpValueAsr<1>";
   let DecoderMethod = "decodeSImmOperand<11, 1>";
diff --git a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
index aaa4b94b6e994..e4268920e0b27 100644
--- a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
@@ -2165,6 +2165,7 @@ def : Pat<(int_loongarch_lasx_xvld GPR:$rj, timm:$imm),
 def : Pat<(int_loongarch_lasx_xvldx GPR:$rj, GPR:$rk),
           (XVLDX GPR:$rj, GPR:$rk)>;
 
+// xvldrepl
 def : Pat<(int_loongarch_lasx_xvldrepl_b GPR:$rj, timm:$imm),
           (XVLDREPL_B GPR:$rj, (to_valid_timm timm:$imm))>;
 def : Pat<(int_loongarch_lasx_xvldrepl_h GPR:$rj, timm:$imm),
@@ -2174,6 +2175,13 @@ def : Pat<(int_loongarch_lasx_xvldrepl_w GPR:$rj, timm:$imm),
 def : Pat<(int_loongarch_lasx_xvldrepl_d GPR:$rj, timm:$imm),
           (XVLDREPL_D GPR:$rj, (to_valid_timm timm:$imm))>;
 
+defm : VldreplPat<v32i8, XVLDREPL_B, simm12_addlike>;
+defm : VldreplPat<v16i16, XVLDREPL_H, simm11_lsl1>;
+defm : VldreplPat<v8i32, XVLDREPL_W, simm10_lsl2>;
+defm : VldreplPat<v4i64, XVLDREPL_D, simm9_lsl3>;
+defm : VldreplPat<v8f32, XVLDREPL_W, simm10_lsl2>;
+defm : VldreplPat<v4f64, XVLDREPL_D, simm9_lsl3>;
+
 // store
 def : Pat<(int_loongarch_lasx_xvst LASX256:$xd, GPR:$rj, timm:$imm),
           (XVST LASX256:$xd, GPR:$rj, (to_valid_timm timm:$imm))>;
diff --git a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
index e7327ce7461f7..1ffc5f8056b96 100644
--- a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
@@ -26,6 +26,7 @@ def SDT_LoongArchV1RUimm: SDTypeProfile<1, 2, [SDTCisVec<0>,
 def SDT_LoongArchVreplgr2vr : SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisVec<0>, SDTCisInt<1>]>;
 def SDT_LoongArchVFRECIPE : SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisVec<0>, SDTCisSameAs<0, 1>]>;
 def SDT_LoongArchVFRSQRTE : SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisVec<0>, SDTCisSameAs<0, 1>]>;
+def SDT_LoongArchVLDREPL : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisPtrTy<1>]>;
 
 // Target nodes.
 def loongarch_vreplve : SDNode<"LoongArchISD::VREPLVE", SDT_LoongArchVreplve>;
@@ -64,6 +65,10 @@ def loongarch_vsrli : SDNode<"LoongArchISD::VSRLI", SDT_LoongArchV1RUimm>;
 def loongarch_vbsll : SDNode<"LoongArchISD::VBSLL", SDT_LoongArchV1RUimm>;
 def loongarch_vbsrl : SDNode<"LoongArchISD::VBSRL", SDT_LoongArchV1RUimm>;
 
+def loongarch_vldrepl
+    : SDNode<"LoongArchISD::VLDREPL",
+             SDT_LoongArchVLDREPL, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+
 def immZExt1 : ImmLeaf<i64, [{return isUInt<1>(Imm);}]>;
 def immZExt2 : ImmLeaf<i64, [{return isUInt<2>(Imm);}]>;
 def immZExt3 : ImmLeaf<i64, [{return isUInt<3>(Imm);}]>;
@@ -1433,6 +1438,14 @@ multiclass PatCCVrVrF<CondCode CC, string Inst> {
             (!cast<LAInst>(Inst#"_D") LSX128:$vj, LSX128:$vk)>;
 }
 
+multiclass VldreplPat<ValueType vt, LAInst Inst, Operand ImmOpnd> {
+  def : Pat<(vt(loongarch_vldrepl BaseAddr:$rj)), (Inst BaseAddr:$rj, 0)>;
+  def : Pat<(vt(loongarch_vldrepl(AddrConstant GPR:$rj, ImmOpnd:$imm))),
+            (Inst GPR:$rj, ImmOpnd:$imm)>;
+  def : Pat<(vt(loongarch_vldrepl(AddLike BaseAddr:$rj, ImmOpnd:$imm))),
+            (Inst BaseAddr:$rj, ImmOpnd:$imm)>;
+}
+
 let Predicates = [HasExtLSX] in {
 
 // VADD_{B/H/W/D}
@@ -2342,6 +2355,7 @@ def : Pat<(int_loongarch_lsx_vld GPR:$rj, timm:$imm),
 def : Pat<(int_loongarch_lsx_vldx GPR:$rj, GPR:$rk),
           (VLDX GPR:$rj, GPR:$rk)>;
 
+// vldrepl
 def : Pat<(int_loongarch_lsx_vldrepl_b GPR:$rj, timm:$imm),
           (VLDREPL_B GPR:$rj, (to_valid_timm timm:$imm))>;
 def : Pat<(int_loongarch_lsx_vldrepl_h GPR:$rj, timm:$imm),
@@ -2351,6 +2365,13 @@ def : Pat<(int_loongarch_lsx_vldrepl_w GPR:$rj, timm:$imm),
 def : Pat<(int_loongarch_lsx_vldrepl_d GPR:$rj, timm:$imm),
           (VLDREPL_D GPR:$rj, (to_valid_timm timm:$imm))>;
 
+defm : VldreplPat<v16i8, VLDREPL_B, simm12_addlike>;
+defm : VldreplPat<v8i16, VLDREPL_H, simm11_lsl1>;
+defm : VldreplPat<v4i32, VLDREPL_W, simm10_lsl2>;
+defm : VldreplPat<v2i64, VLDREPL_D, simm9_lsl3>;
+defm : VldreplPat<v4f32, VLDREPL_W, simm10_lsl2>;
+defm : VldreplPat<v2f64, VLDREPL_D, simm9_lsl3>;
+
 // store
 def : Pat<(int_loongarch_lsx_vst LSX128:$vd, GPR:$rj, timm:$imm),
           (VST LSX128:$vd, GPR:$rj, (to_valid_timm timm:$imm))>;
diff --git a/llvm/test/CodeGen/LoongArch/lasx/broadcast-load.ll b/llvm/test/CodeGen/LoongArch/lasx/broadcast-load.ll
index 4fcf016376d09..976924bdca686 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/broadcast-load.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/broadcast-load.ll
@@ -21,8 +21,8 @@ define <4 x i64> @should_not_be_optimized(ptr %ptr, ptr %dst) {
 define <4 x i64> @xvldrepl_d_unaligned_offset(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_d_unaligned_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.d $a0, $a0, 4
-; CHECK-NEXT:    xvreplgr2vr.d $xr0, $a0
+; CHECK-NEXT:    addi.d $a0, $a0, 4
+; CHECK-NEXT:    xvldrepl.d $xr0, $a0, 0
 ; CHECK-NEXT:    ret
   %p = getelementptr i32, ptr %ptr, i32 1
   %tmp = load i64, ptr %p
@@ -34,8 +34,7 @@ define <4 x i64> @xvldrepl_d_unaligned_offset(ptr %ptr) {
 define <32 x i8> @xvldrepl_b(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_b:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.b $a0, $a0, 0
-; CHECK-NEXT:    xvreplgr2vr.b $xr0, $a0
+; CHECK-NEXT:    xvldrepl.b $xr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load i8, ptr %ptr
   %tmp1 = insertelement <32 x i8> zeroinitializer, i8 %tmp, i32 0
@@ -46,8 +45,7 @@ define <32 x i8> @xvldrepl_b(ptr %ptr) {
 define <32 x i8> @xvldrepl_b_offset(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_b_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.b $a0, $a0, 33
-; CHECK-NEXT:    xvreplgr2vr.b $xr0, $a0
+; CHECK-NEXT:    xvldrepl.b $xr0, $a0, 33
 ; CHECK-NEXT:    ret
   %p = getelementptr i8, ptr %ptr, i64 33
   %tmp = load i8, ptr %p
@@ -60,8 +58,7 @@ define <32 x i8> @xvldrepl_b_offset(ptr %ptr) {
 define <16 x i16> @xvldrepl_h(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_h:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.h $a0, $a0, 0
-; CHECK-NEXT:    xvreplgr2vr.h $xr0, $a0
+; CHECK-NEXT:    xvldrepl.h $xr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load i16, ptr %ptr
   %tmp1 = insertelement <16 x i16> zeroinitializer, i16 %tmp, i32 0
@@ -72,8 +69,7 @@ define <16 x i16> @xvldrepl_h(ptr %ptr) {
 define <16 x i16> @xvldrepl_h_offset(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_h_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.h $a0, $a0, 66
-; CHECK-NEXT:    xvreplgr2vr.h $xr0, $a0
+; CHECK-NEXT:    xvldrepl.h $xr0, $a0, 66
 ; CHECK-NEXT:    ret
   %p = getelementptr i16, ptr %ptr, i64 33
   %tmp = load i16, ptr %p
@@ -85,8 +81,7 @@ define <16 x i16> @xvldrepl_h_offset(ptr %ptr) {
 define <8 x i32> @xvldrepl_w(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_w:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.w $a0, $a0, 0
-; CHECK-NEXT:    xvreplgr2vr.w $xr0, $a0
+; CHECK-NEXT:    xvldrepl.w $xr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load i32, ptr %ptr
   %tmp1 = insertelement <8 x i32> zeroinitializer, i32 %tmp, i32 0
@@ -97,8 +92,7 @@ define <8 x i32> @xvldrepl_w(ptr %ptr) {
 define <8 x i32> @xvldrepl_w_offset(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_w_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.w $a0, $a0, 132
-; CHECK-NEXT:    xvreplgr2vr.w $xr0, $a0
+; CHECK-NEXT:    xvldrepl.w $xr0, $a0, 132
 ; CHECK-NEXT:    ret
   %p = getelementptr i32, ptr %ptr, i64 33
   %tmp = load i32, ptr %p
@@ -111,8 +105,7 @@ define <8 x i32> @xvldrepl_w_offset(ptr %ptr) {
 define <4 x i64> @xvldrepl_d(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_d:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.d $a0, $a0, 0
-; CHECK-NEXT:    xvreplgr2vr.d $xr0, $a0
+; CHECK-NEXT:    xvldrepl.d $xr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load i64, ptr %ptr
   %tmp1 = insertelement <4 x i64> zeroinitializer, i64 %tmp, i32 0
@@ -123,8 +116,7 @@ define <4 x i64> @xvldrepl_d(ptr %ptr) {
 define <4 x i64> @xvldrepl_d_offset(ptr %ptr) {
 ; CHECK-LABEL: xvldrepl_d_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.d $a0, $a0, 264
-; CHECK-NEXT:    xvreplgr2vr.d $xr0, $a0
+; CHECK-NEXT:    xvldrepl.d $xr0, $a0, 264
 ; CHECK-NEXT:    ret
   %p = getelementptr i64, ptr %ptr, i64 33
   %tmp = load i64, ptr %p
@@ -136,8 +128,7 @@ define <4 x i64> @xvldrepl_d_offset(ptr %ptr) {
 define <8 x float> @vldrepl_w_flt(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_w_flt:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fld.s $fa0, $a0, 0
-; CHECK-NEXT:    xvreplve0.w $xr0, $xr0
+; CHECK-NEXT:    xvldrepl.w $xr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load float, ptr %ptr
   %tmp1 = insertelement <8 x float> zeroinitializer, float %tmp, i32 0
@@ -148,8 +139,7 @@ define <8 x float> @vldrepl_w_flt(ptr %ptr) {
 define <8 x float> @vldrepl_w_flt_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_w_flt_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fld.s $fa0, $a0, 264
-; CHECK-NEXT:    xvreplve0.w $xr0, $xr0
+; CHECK-NEXT:    xvldrepl.w $xr0, $a0, 264
 ; CHECK-NEXT:    ret
   %p = getelementptr i64, ptr %ptr, i64 33
   %tmp = load float, ptr %p
@@ -161,8 +151,7 @@ define <8 x float> @vldrepl_w_flt_offset(ptr %ptr) {
 define <4 x double> @vldrepl_d_dbl(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_d_dbl:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fld.d $fa0, $a0, 0
-; CHECK-NEXT:    xvreplve0.d $xr0, $xr0
+; CHECK-NEXT:    xvldrepl.d $xr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load double, ptr %ptr
   %tmp1 = insertelement <4 x double> zeroinitializer, double %tmp, i32 0
@@ -173,8 +162,7 @@ define <4 x double> @vldrepl_d_dbl(ptr %ptr) {
 define <4 x double> @vldrepl_d_dbl_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_d_dbl_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fld.d $fa0, $a0, 264
-; CHECK-NEXT:    xvreplve0.d $xr0, $xr0
+; CHECK-NEXT:    xvldrepl.d $xr0, $a0, 264
 ; CHECK-NEXT:    ret
   %p = getelementptr i64, ptr %ptr, i64 33
   %tmp = load double, ptr %p
diff --git a/llvm/test/CodeGen/LoongArch/lsx/broadcast-load.ll b/llvm/test/CodeGen/LoongArch/lsx/broadcast-load.ll
index 02b68725687dd..c46747ef30509 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/broadcast-load.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/broadcast-load.ll
@@ -21,8 +21,8 @@ define <2 x i64> @should_not_be_optimized(ptr %ptr, ptr %dst){
 define <2 x i64> @vldrepl_d_unaligned_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_d_unaligned_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.d $a0, $a0, 4
-; CHECK-NEXT:    vreplgr2vr.d $vr0, $a0
+; CHECK-NEXT:    addi.d $a0, $a0, 4
+; CHECK-NEXT:    vldrepl.d $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %p = getelementptr i32, ptr %ptr, i32 1
   %tmp = load i64, ptr %p
@@ -34,8 +34,7 @@ define <2 x i64> @vldrepl_d_unaligned_offset(ptr %ptr) {
 define <16 x i8> @vldrepl_b(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_b:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.b $a0, $a0, 0
-; CHECK-NEXT:    vreplgr2vr.b $vr0, $a0
+; CHECK-NEXT:    vldrepl.b $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load i8, ptr %ptr
   %tmp1 = insertelement <16 x i8> zeroinitializer, i8 %tmp, i32 0
@@ -46,8 +45,7 @@ define <16 x i8> @vldrepl_b(ptr %ptr) {
 define <16 x i8> @vldrepl_b_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_b_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.b $a0, $a0, 33
-; CHECK-NEXT:    vreplgr2vr.b $vr0, $a0
+; CHECK-NEXT:    vldrepl.b $vr0, $a0, 33
 ; CHECK-NEXT:    ret
   %p = getelementptr i8, ptr %ptr, i64 33
   %tmp = load i8, ptr %p
@@ -60,8 +58,7 @@ define <16 x i8> @vldrepl_b_offset(ptr %ptr) {
 define <8 x i16> @vldrepl_h(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_h:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.h $a0, $a0, 0
-; CHECK-NEXT:    vreplgr2vr.h $vr0, $a0
+; CHECK-NEXT:    vldrepl.h $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load i16, ptr %ptr
   %tmp1 = insertelement <8 x i16> zeroinitializer, i16 %tmp, i32 0
@@ -72,8 +69,7 @@ define <8 x i16> @vldrepl_h(ptr %ptr) {
 define <8 x i16> @vldrepl_h_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_h_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.h $a0, $a0, 66
-; CHECK-NEXT:    vreplgr2vr.h $vr0, $a0
+; CHECK-NEXT:    vldrepl.h $vr0, $a0, 66
 ; CHECK-NEXT:    ret
   %p = getelementptr i16, ptr %ptr, i64 33
   %tmp = load i16, ptr %p
@@ -85,8 +81,7 @@ define <8 x i16> @vldrepl_h_offset(ptr %ptr) {
 define <4 x i32> @vldrepl_w(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_w:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.w $a0, $a0, 0
-; CHECK-NEXT:    vreplgr2vr.w $vr0, $a0
+; CHECK-NEXT:    vldrepl.w $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load i32, ptr %ptr
   %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0
@@ -97,8 +92,7 @@ define <4 x i32> @vldrepl_w(ptr %ptr) {
 define <4 x i32> @vldrepl_w_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_w_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.w $a0, $a0, 132
-; CHECK-NEXT:    vreplgr2vr.w $vr0, $a0
+; CHECK-NEXT:    vldrepl.w $vr0, $a0, 132
 ; CHECK-NEXT:    ret
   %p = getelementptr i32, ptr %ptr, i64 33
   %tmp = load i32, ptr %p
@@ -110,8 +104,7 @@ define <4 x i32> @vldrepl_w_offset(ptr %ptr) {
 define <2 x i64> @vldrepl_d(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_d:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.d $a0, $a0, 0
-; CHECK-NEXT:    vreplgr2vr.d $vr0, $a0
+; CHECK-NEXT:    vldrepl.d $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load i64, ptr %ptr
   %tmp1 = insertelement <2 x i64> zeroinitializer, i64 %tmp, i32 0
@@ -122,8 +115,7 @@ define <2 x i64> @vldrepl_d(ptr %ptr) {
 define <2 x i64> @vldrepl_d_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_d_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld.d $a0, $a0, 264
-; CHECK-NEXT:    vreplgr2vr.d $vr0, $a0
+; CHECK-NEXT:    vldrepl.d $vr0, $a0, 264
 ; CHECK-NEXT:    ret
   %p = getelementptr i64, ptr %ptr, i64 33
   %tmp = load i64, ptr %p
@@ -135,8 +127,7 @@ define <2 x i64> @vldrepl_d_offset(ptr %ptr) {
 define <4 x float> @vldrepl_w_flt(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_w_flt:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fld.s $fa0, $a0, 0
-; CHECK-NEXT:    vreplvei.w $vr0, $vr0, 0
+; CHECK-NEXT:    vldrepl.w $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load float, ptr %ptr
   %tmp1 = insertelement <4 x float> zeroinitializer, float %tmp, i32 0
@@ -147,8 +138,7 @@ define <4 x float> @vldrepl_w_flt(ptr %ptr) {
 define <4 x float> @vldrepl_w_flt_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_w_flt_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fld.s $fa0, $a0, 264
-; CHECK-NEXT:    vreplvei.w $vr0, $vr0, 0
+; CHECK-NEXT:    vldrepl.w $vr0, $a0, 264
 ; CHECK-NEXT:    ret
   %p = getelementptr i64, ptr %ptr, i64 33
   %tmp = load float, ptr %p
@@ -160,8 +150,7 @@ define <4 x float> @vldrepl_w_flt_offset(ptr %ptr) {
 define <2 x double> @vldrepl_d_dbl(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_d_dbl:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fld.d $fa0, $a0, 0
-; CHECK-NEXT:    vreplvei.d $vr0, $vr0, 0
+; CHECK-NEXT:    vldrepl.d $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %tmp = load double, ptr %ptr
   %tmp1 = insertelement <2 x double> zeroinitializer, double %tmp, i32 0
@@ -172,8 +161,7 @@ define <2 x double> @vldrepl_d_dbl(ptr %ptr) {
 define <2 x double> @vldrepl_d_dbl_offset(ptr %ptr) {
 ; CHECK-LABEL: vldrepl_d_dbl_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fld.d $fa0, $a0, 264
-; CHECK-NEXT:    vreplvei.d $vr0, $vr0, 0
+; CHECK-NEXT:    vldrepl.d $vr0, $a0, 264
 ; CHECK-NEXT:    ret
   %p = getelementptr i64, ptr %ptr, i64 33
   %tmp = load double, ptr %p

From 053451cb3502144564b4d0b30a9046045d1820d4 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Mon, 21 Apr 2025 16:04:25 +0800
Subject: [PATCH 610/710] [RISCV] Handle scalarized reductions in
 getArithmeticReductionCost

This fixes a crash reported at
https://github.com/llvm/llvm-project/pull/114250#issuecomment-2813686061

If the vector type isn't legal at all, e.g. bfloat with +zvfbfmin,
then the legalized type will be scalarized. So use getScalarType()
instead of getVectorElement() when checking for f16/bf16.
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |   5 +-
 .../Analysis/CostModel/RISCV/reduce-fadd.ll   | 167 ++++++++++++++----
 2 files changed, 137 insertions(+), 35 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 9b91de36a688a..6bd60c20c8626 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1874,9 +1874,8 @@ RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
     break;
   case ISD::FADD:
     // We can't promote f16/bf16 fadd reductions.
-    if ((LT.second.getVectorElementType() == MVT::f16 &&
-         !ST->hasVInstructionsF16()) ||
-        LT.second.getVectorElementType() == MVT::bf16)
+    if ((LT.second.getScalarType() == MVT::f16 && !ST->hasVInstructionsF16()) ||
+        LT.second.getScalarType() == MVT::bf16)
       return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
     if (TTI::requiresOrderedReduction(FMF)) {
       Opcodes.push_back(RISCV::VFMV_S_F);
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
index 1762f701a9b2d..71685b4acc822 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
@@ -1,25 +1,60 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+zfbfmin,+zvfbfmin -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefixes=FP-REDUCE,FP-REDUCE-ZVFH
 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin,+zfbfmin,+zvfbfmin -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefixes=FP-REDUCE,FP-REDUCE-ZVFHMIN
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefixes=FP-REDUCE,FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN
 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+zfbfmin,+zvfbfmin -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s  --check-prefix=SIZE
 
 define void @reduce_fadd_bfloat() {
-; FP-REDUCE-LABEL: 'reduce_fadd_bfloat'
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call fast bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call fast bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR0000, <8 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16 = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 211 for instruction: %v32 = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 541 for instruction: %V64 = call fast bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 573 for instruction: %V128 = call fast bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; FP-REDUCE-ZVFH-LABEL: 'reduce_fadd_bfloat'
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call fast bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call fast bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR0000, <8 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16 = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 211 for instruction: %v32 = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 541 for instruction: %V64 = call fast bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 573 for instruction: %V128 = call fast bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FP-REDUCE-ZVFHMIN-LABEL: 'reduce_fadd_bfloat'
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call fast bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call fast bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR0000, <8 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16 = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 211 for instruction: %v32 = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 541 for instruction: %V64 = call fast bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 573 for instruction: %V128 = call fast bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-LABEL: 'reduce_fadd_bfloat'
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V1 = call fast bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V2 = call fast bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V4 = call fast bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V8 = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR0000, <8 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V16 = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %v32 = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V64 = call fast bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V128 = call fast bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fadd_bfloat'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
@@ -90,6 +125,23 @@ define void @reduce_fadd_half() {
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-LABEL: 'reduce_fadd_half'
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V1 = call fast half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V2 = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V4 = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V8 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %v32 = call fast half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V64 = call fast half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V128 = call fast half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call fast half @llvm.vector.reduce.fadd.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call fast half @llvm.vector.reduce.fadd.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call fast half @llvm.vector.reduce.fadd.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call fast half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call fast half @llvm.vector.reduce.fadd.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call fast half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
 ; SIZE-LABEL: 'reduce_fadd_half'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call fast half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
@@ -220,22 +272,56 @@ define void @reduce_fadd_double() {
 }
 
 define void @reduce_ordered_fadd_bfloat() {
-; FP-REDUCE-LABEL: 'reduce_ordered_fadd_bfloat'
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8 = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR0000, <8 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V16 = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32 = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %V64 = call bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %V128 = call bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; FP-REDUCE-ZVFH-LABEL: 'reduce_ordered_fadd_bfloat'
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8 = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR0000, <8 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V16 = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32 = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %V64 = call bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %V128 = call bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FP-REDUCE-ZVFHMIN-LABEL: 'reduce_ordered_fadd_bfloat'
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8 = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR0000, <8 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V16 = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32 = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %V64 = call bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %V128 = call bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-LABEL: 'reduce_ordered_fadd_bfloat'
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V1 = call bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V2 = call bfloat @llvm.vector.reduce.fadd.v2bf16(bfloat 0xR0000, <2 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V4 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V8 = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR0000, <8 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V16 = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %v32 = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0xR0000, <32 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V64 = call bfloat @llvm.vector.reduce.fadd.v64bf16(bfloat 0xR0000, <64 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V128 = call bfloat @llvm.vector.reduce.fadd.v128bf16(bfloat 0xR0000, <128 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call bfloat @llvm.vector.reduce.fadd.nxv1bf16(bfloat 0xR0000, <vscale x 1 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat 0xR0000, <vscale x 2 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat 0xR0000, <vscale x 4 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat 0xR0000, <vscale x 8 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call bfloat @llvm.vector.reduce.fadd.nxv16bf16(bfloat 0xR0000, <vscale x 16 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call bfloat @llvm.vector.reduce.fadd.nxv32bf16(bfloat 0xR0000, <vscale x 32 x bfloat> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_ordered_fadd_bfloat'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call bfloat @llvm.vector.reduce.fadd.v1bf16(bfloat 0xR0000, <1 x bfloat> undef)
@@ -306,6 +392,23 @@ define void @reduce_ordered_fadd_half() {
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
 ; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-LABEL: 'reduce_ordered_fadd_half'
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V1 = call half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V2 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V4 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V8 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %v32 = call half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V64 = call half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %V128 = call half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV1 = call half @llvm.vector.reduce.fadd.nxv1f16(half 0xH0000, <vscale x 1 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV2 = call half @llvm.vector.reduce.fadd.nxv2f16(half 0xH0000, <vscale x 2 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV4 = call half @llvm.vector.reduce.fadd.nxv4f16(half 0xH0000, <vscale x 4 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV8 = call half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV16 = call half @llvm.vector.reduce.fadd.nxv16f16(half 0xH0000, <vscale x 16 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Invalid cost for instruction: %NXV32 = call half @llvm.vector.reduce.fadd.nxv32f16(half 0xH0000, <vscale x 32 x half> undef)
+; FP-REDUCE-NO-ZFHMIN-NO-ZFBFMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
 ; SIZE-LABEL: 'reduce_ordered_fadd_half'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)

From 6423c901fd5a5a8370a5326c5fbe16eb5a837374 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Mon, 21 Apr 2025 10:38:49 +0200
Subject: [PATCH 611/710] [mlir][LLVM][NFC] Avoid rollback in FuncOp --> LLVM
 lowering (#136477)

This pattern used to create an `llvm.func` op, then check additional
requirements and return "failure". This commit moves the checks before
the creation of the replacement op, so that no rollback is necessary
when one of the checks fails.

Note: This is in preparation of the One-Shot Dialect Conversion
refactoring, which removes the rollback functionality.
---
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp | 41 +++++++++++--------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 55f0a9ac3bbb2..328c605add65c 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -320,13 +320,22 @@ mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
   // overriden with an LLVM pointer type for later processing.
   SmallVector<std::optional<NamedAttribute>> byValRefNonPtrAttrs;
   TypeConverter::SignatureConversion result(funcOp.getNumArguments());
-  auto llvmType = converter.convertFunctionSignature(
-      funcOp, varargsAttr && varargsAttr.getValue(),
-      shouldUseBarePtrCallConv(funcOp, &converter), result,
-      byValRefNonPtrAttrs);
+  auto llvmType = dyn_cast_or_null<LLVM::LLVMFunctionType>(
+      converter.convertFunctionSignature(
+          funcOp, varargsAttr && varargsAttr.getValue(),
+          shouldUseBarePtrCallConv(funcOp, &converter), result,
+          byValRefNonPtrAttrs));
   if (!llvmType)
     return rewriter.notifyMatchFailure(funcOp, "signature conversion failed");
 
+  // Check for unsupported variadic functions.
+  if (!shouldUseBarePtrCallConv(funcOp, &converter))
+    if (funcOp->getAttrOfType<UnitAttr>(
+            LLVM::LLVMDialect::getEmitCWrapperAttrName()))
+      if (llvmType.isVarArg())
+        return funcOp.emitError("C interface for variadic functions is not "
+                                "supported yet.");
+
   // Create an LLVM function, use external linkage by default until MLIR
   // functions have linkage.
   LLVM::Linkage linkage = LLVM::Linkage::External;
@@ -342,6 +351,18 @@ mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
     linkage = attr.getLinkage();
   }
 
+  // Check for invalid attributes.
+  StringRef readnoneAttrName = LLVM::LLVMDialect::getReadnoneAttrName();
+  if (funcOp->hasAttr(readnoneAttrName)) {
+    auto attr = funcOp->getAttrOfType<UnitAttr>(readnoneAttrName);
+    if (!attr) {
+      funcOp->emitError() << "Contains " << readnoneAttrName
+                          << " attribute not of type UnitAttr";
+      return rewriter.notifyMatchFailure(
+          funcOp, "Contains readnone attribute not of type UnitAttr");
+    }
+  }
+
   SmallVector<NamedAttribute, 4> attributes;
   filterFuncAttributes(funcOp, attributes);
   auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
@@ -352,15 +373,7 @@ mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
       .setVisibility(funcOp.getVisibility());
 
   // Create a memory effect attribute corresponding to readnone.
-  StringRef readnoneAttrName = LLVM::LLVMDialect::getReadnoneAttrName();
   if (funcOp->hasAttr(readnoneAttrName)) {
-    auto attr = funcOp->getAttrOfType<UnitAttr>(readnoneAttrName);
-    if (!attr) {
-      funcOp->emitError() << "Contains " << readnoneAttrName
-                          << " attribute not of type UnitAttr";
-      return rewriter.notifyMatchFailure(
-          funcOp, "Contains readnone attribute not of type UnitAttr");
-    }
     auto memoryAttr = LLVM::MemoryEffectsAttr::get(
         rewriter.getContext(),
         {LLVM::ModRefInfo::NoModRef, LLVM::ModRefInfo::NoModRef,
@@ -447,10 +460,6 @@ mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
   if (!shouldUseBarePtrCallConv(funcOp, &converter)) {
     if (funcOp->getAttrOfType<UnitAttr>(
             LLVM::LLVMDialect::getEmitCWrapperAttrName())) {
-      if (newFuncOp.isVarArg())
-        return funcOp.emitError("C interface for variadic functions is not "
-                                "supported yet.");
-
       if (newFuncOp.isExternal())
         wrapExternalFunction(rewriter, funcOp->getLoc(), converter, funcOp,
                              newFuncOp);

From 47ca7f1cce56332ee8ac0d525a0172d3bce8508e Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234@gmail.com>
Date: Mon, 21 Apr 2025 04:47:01 -0400
Subject: [PATCH 612/710] [X86] Remove unused BitVector TILERegs (NFC)
 (#136508)

It is not used.
---
 llvm/lib/Target/X86/X86LowerTileCopy.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index fe13ebc6c7675..a4845c0b67efa 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -79,8 +79,6 @@ bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) {
   const TargetRegisterInfo *TRI = ST.getRegisterInfo();
   BitVector GR64Regs =
       TRI->getAllocatableSet(MF, TRI->getRegClass(X86::GR64RegClassID));
-  BitVector TILERegs =
-      TRI->getAllocatableSet(MF, TRI->getRegClass(X86::TILERegClassID));
   bool Changed = false;
 
   for (MachineBasicBlock &MBB : MF) {

From c347ad21ab55fc28cbc1366e86de3ad073dd693a Mon Sep 17 00:00:00 2001
From: Liao Chunyu <chunyu@iscas.ac.cn>
Date: Mon, 21 Apr 2025 17:51:57 +0800
Subject: [PATCH 613/710] [RISCV][NFC] Delete
 RISCVAsmParser::parsePseudoQCJumpSymbol (#136552)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fix, warning:
RISCVAsmParser::parsePseudoQCJumpSymbol(llvm::OperandVector&)’ defined
but not used [-Wunused-function]
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp  | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index db9ec7ae1df02..6cec11a48d042 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -202,7 +202,6 @@ class RISCVAsmParser : public MCTargetAsmParser {
   ParseStatus parseOperandWithSpecifier(OperandVector &Operands);
   ParseStatus parseBareSymbol(OperandVector &Operands);
   ParseStatus parseCallSymbol(OperandVector &Operands);
-  ParseStatus parsePseudoQCJumpSymbol(OperandVector &Operands);
   ParseStatus parsePseudoJumpSymbol(OperandVector &Operands);
   ParseStatus parseJALOffset(OperandVector &Operands);
   ParseStatus parseVTypeI(OperandVector &Operands);
@@ -2139,23 +2138,6 @@ ParseStatus RISCVAsmParser::parseCallSymbol(OperandVector &Operands) {
   return ParseStatus::Success;
 }
 
-ParseStatus RISCVAsmParser::parsePseudoQCJumpSymbol(OperandVector &Operands) {
-  SMLoc S = getLoc();
-  const MCExpr *Res;
-
-  if (getLexer().getKind() != AsmToken::Identifier)
-    return ParseStatus::NoMatch;
-
-  std::string Identifier(getTok().getIdentifier());
-  SMLoc E = getTok().getEndLoc();
-  Lex();
-
-  MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
-  Res = MCSymbolRefExpr::create(Sym, getContext());
-  Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64()));
-  return ParseStatus::Success;
-}
-
 ParseStatus RISCVAsmParser::parsePseudoJumpSymbol(OperandVector &Operands) {
   SMLoc S = getLoc();
   SMLoc E;

From 940108b24d1d7cb25b06dd66c6529bddc7daefab Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <christudasan.devadasan@amd.com>
Date: Mon, 21 Apr 2025 15:34:15 +0530
Subject: [PATCH 614/710] [AMDGPU][NewPM] Make the pass flow consistent with
 the legacy pipeline. (#136551)

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 90e3489ced923..b6cc5137d711a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -2081,13 +2081,11 @@ void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
 
 void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
 
-  if (TM.getOptLevel() > CodeGenOptLevel::None)
+  if (TM.getOptLevel() > CodeGenOptLevel::None) {
     addPass(FlattenCFGPass());
-
-  if (TM.getOptLevel() > CodeGenOptLevel::None)
     addPass(SinkingPass());
-
-  addPass(AMDGPULateCodeGenPreparePass(TM));
+    addPass(AMDGPULateCodeGenPreparePass(TM));
+  }
 
   // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
   // regions formed by them.

From 788b50a4384985f2c221cfd8d290cabc6f59e646 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 21 Apr 2025 13:24:20 +0100
Subject: [PATCH 615/710] [X86] Add test coverage for #136368

---
 llvm/test/CodeGen/X86/combine-fcopysign.ll | 35 ++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/llvm/test/CodeGen/X86/combine-fcopysign.ll b/llvm/test/CodeGen/X86/combine-fcopysign.ll
index d38fd817f6242..d7031be3addd9 100644
--- a/llvm/test/CodeGen/X86/combine-fcopysign.ll
+++ b/llvm/test/CodeGen/X86/combine-fcopysign.ll
@@ -248,6 +248,41 @@ define <4 x float> @combine_vec_fcopysign_fptrunc_sgn(<4 x float> %x, <4 x doubl
   ret <4 x float> %2
 }
 
+define double @PR136368(double %x) {
+; SSE-LABEL: PR136368:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd {{.*#+}} xmm1 = [NaN,NaN]
+; SSE-NEXT:    movapd %xmm0, %xmm2
+; SSE-NEXT:    andpd %xmm1, %xmm2
+; SSE-NEXT:    movsd {{.*#+}} xmm3 = [1.5707963267948966E+0,0.0E+0]
+; SSE-NEXT:    movapd %xmm3, %xmm4
+; SSE-NEXT:    cmpltsd %xmm2, %xmm4
+; SSE-NEXT:    andpd %xmm3, %xmm4
+; SSE-NEXT:    andpd %xmm1, %xmm4
+; SSE-NEXT:    andnpd %xmm0, %xmm1
+; SSE-NEXT:    orpd %xmm4, %xmm1
+; SSE-NEXT:    movapd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: PR136368:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovddup {{.*#+}} xmm1 = [NaN,NaN]
+; AVX-NEXT:    # xmm1 = mem[0,0]
+; AVX-NEXT:    vandpd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovsd {{.*#+}} xmm3 = [1.5707963267948966E+0,0.0E+0]
+; AVX-NEXT:    vcmpltsd %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vandpd %xmm3, %xmm2, %xmm2
+; AVX-NEXT:    vandnpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vandpd %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vorpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %fabs = tail call double @llvm.fabs.f64(double %x)
+  %cmp = fcmp ogt double %fabs, 0x3FF921FB54442D18
+  %cond = select i1 %cmp, double 0x3FF921FB54442D18, double 0.000000e+00
+  %res = tail call double @llvm.copysign.f64(double %cond, double %x)
+  ret double %res
+}
+
 declare <4 x float> @llvm.fabs.v4f32(<4 x float> %p)
 declare <4 x float> @llvm.copysign.v4f32(<4 x float> %Mag, <4 x float> %Sgn)
 declare <4 x double> @llvm.copysign.v4f64(<4 x double> %Mag, <4 x double> %Sgn)

From f12078e72601e7c03e5d66afab034313caf8f791 Mon Sep 17 00:00:00 2001
From: zhijian lin <zhijian@ca.ibm.com>
Date: Mon, 21 Apr 2025 10:02:21 -0400
Subject: [PATCH 616/710] [SelectionDAG] Folding  ZERO-EXTEND/SIGN_EXTEND
 poison to Poison value in getNode (#122741)

The PR will fix the issue
https://github.com/llvm/llvm-project/issues/122728

This patch addresses the signed/zero extension of poison by using a
poison value of the extended type instead of a constant zero of the
extended type.
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  8 ++++
 .../CodeGen/AArch64/arm64-bitfield-extract.ll |  2 +-
 .../CodeGen/AArch64/optimize-cond-branch.ll   |  2 +-
 .../CodeGen/AArch64/sve-extract-element.ll    |  1 -
 .../CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll    | 23 +++--------
 .../atomic_optimizations_global_pointer.ll    | 14 +++----
 llvm/test/CodeGen/AMDGPU/ctpop16.ll           | 41 +++++++++----------
 .../dead-machine-elim-after-dead-lane.ll      |  9 ++--
 .../CodeGen/AMDGPU/mdt-preserving-crash.ll    | 12 ++----
 llvm/test/CodeGen/PowerPC/undef-args.ll       | 17 ++++----
 llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll |  8 ----
 11 files changed, 60 insertions(+), 77 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f502a536d43b1..61509cdb4a372 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6282,6 +6282,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
         Flags.setNonNeg(N1->getFlags().hasNonNeg());
       return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
     }
+
+    if (OpOpcode == ISD::POISON)
+      return getPOISON(VT);
+
     if (N1.isUndef())
       // sext(undef) = 0, because the top bits will all be the same.
       return getConstant(0, DL, VT);
@@ -6302,6 +6306,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
       Flags.setNonNeg(N1->getFlags().hasNonNeg());
       return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0), Flags);
     }
+
+    if (OpOpcode == ISD::POISON)
+      return getPOISON(VT);
+
     if (N1.isUndef())
       // zext(undef) = 0, because the top bits will be zero.
       return getConstant(0, DL, VT);
diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
index 62ff4bbcc9c27..4b1fff642e5f5 100644
--- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
@@ -1019,7 +1019,7 @@ define i16 @test_ignored_rightbits(i32 %dst, i32 %in) {
 define void @sameOperandBFI(i64 %src, i64 %src2, ptr %ptr) {
 ; LLC-LABEL: sameOperandBFI:
 ; LLC:       // %bb.0: // %entry
-; LLC-NEXT:    cbnz wzr, .LBB30_2
+; LLC-NEXT:    cbnz w8, .LBB30_2
 ; LLC-NEXT:  // %bb.1: // %if.else
 ; LLC-NEXT:    lsr x8, x0, #47
 ; LLC-NEXT:    and w9, w1, #0x3
diff --git a/llvm/test/CodeGen/AArch64/optimize-cond-branch.ll b/llvm/test/CodeGen/AArch64/optimize-cond-branch.ll
index fdf972990e745..8a56360935717 100644
--- a/llvm/test/CodeGen/AArch64/optimize-cond-branch.ll
+++ b/llvm/test/CodeGen/AArch64/optimize-cond-branch.ll
@@ -16,7 +16,7 @@ define void @func() uwtable {
 ; CHECK-NEXT:    mov w8, #1 // =0x1
 ; CHECK-NEXT:    cbnz w8, .LBB0_3
 ; CHECK-NEXT:  // %bb.1: // %b1
-; CHECK-NEXT:    cbz wzr, .LBB0_4
+; CHECK-NEXT:    cbz w8, .LBB0_4
 ; CHECK-NEXT:  // %bb.2: // %b3
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    and w0, w8, #0x100
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-element.ll b/llvm/test/CodeGen/AArch64/sve-extract-element.ll
index 9ebeb098c60c0..c340df1385124 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-element.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-element.ll
@@ -523,7 +523,6 @@ define double @test_lanex_2xf64(<vscale x 2 x double> %a, i32 %x) #0 {
 define i32 @test_undef_lane_4xi32(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: test_undef_lane_4xi32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
   %b = extractelement <vscale x 4 x i32> %a, i32 poison
   ret i32 %b
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
index b3d9e61b65b6f..b0627c3d4e77d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
@@ -92,27 +92,17 @@ define i16 @bitcast_f16_to_i16(half %a, i32 %b) {
 ; GCN-LABEL: bitcast_f16_to_i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v2, v0
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v2
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB1_3
-; GCN-NEXT:  ; %bb.1: ; %Flow
-; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB1_4
-; GCN-NEXT:  .LBB1_2: ; %end
-; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_setpc_b64 s[30:31]
-; GCN-NEXT:  .LBB1_3: ; %cmp.false
-; GCN-NEXT:    v_mov_b32_e32 v0, v1
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_2
-; GCN-NEXT:  .LBB1_4: ; %cmp.true
-; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
+; GCN-NEXT:  ; %bb.1:
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
 ; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:  .LBB1_2:
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -249,10 +239,9 @@ define i16 @bitcast_bf16_to_i16(bfloat %a, i32 %b) {
 ; GCN-LABEL: bitcast_bf16_to_i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v2, v0
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GCN-NEXT:    v_mul_f32_e32 v1, 1.0, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, 1.0, v0
+; GCN-NEXT:    ; implicit-def: $vgpr0
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execnz .LBB3_3
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index e2dfcf55b7856..c019c83da5ef8 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -8383,10 +8383,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_load_dword s6, s[4:5], 0xd
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:                       ; implicit-def: $vgpr0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB15_2
 ; GFX7LESS-NEXT:  ; %bb.1:
@@ -8731,10 +8731,10 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_load_dword s10, s[4:5], 0xd
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s6, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v4, s7, v1
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v4, s7, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX7LESS-NEXT:                               ; implicit-def: $vgpr0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_4
 ; GFX7LESS-NEXT:  ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
index 6bf126af5ade2..fb418afb8b039 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
@@ -1292,7 +1292,7 @@ define amdgpu_kernel void @ctpop_i16_in_br(ptr addrspace(1) %out, ptr addrspace(
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ; SI-NEXT:  .LBB14_4:
-; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:                    ; implicit-def: $vgpr0
 ; SI-NEXT:    s_branch .LBB14_2
 ;
 ; VI-LABEL: ctpop_i16_in_br:
@@ -1329,48 +1329,47 @@ define amdgpu_kernel void @ctpop_i16_in_br(ptr addrspace(1) %out, ptr addrspace(
 ; EG:       ; %bb.0: ; %entry
 ; EG-NEXT:    ALU 0, @20, KC0[], KC1[]
 ; EG-NEXT:    TEX 0 @14
-; EG-NEXT:    ALU_PUSH_BEFORE 4, @21, KC0[], KC1[]
+; EG-NEXT:    ALU_PUSH_BEFORE 3, @21, KC0[], KC1[]
 ; EG-NEXT:    JUMP @7 POP:1
-; EG-NEXT:    ALU 0, @26, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 0, @25, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @16
-; EG-NEXT:    ALU_POP_AFTER 1, @27, KC0[], KC1[]
-; EG-NEXT:    ALU_PUSH_BEFORE 2, @29, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU_POP_AFTER 1, @26, KC0[], KC1[]
+; EG-NEXT:    ALU_PUSH_BEFORE 2, @28, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    JUMP @11 POP:1
 ; EG-NEXT:    TEX 0 @18
-; EG-NEXT:    ALU_POP_AFTER 0, @32, KC0[], KC1[]
-; EG-NEXT:    ALU 11, @33, KC0[], KC1[]
+; EG-NEXT:    ALU_POP_AFTER 0, @31, KC0[], KC1[]
+; EG-NEXT:    ALU 11, @32, KC0[], KC1[]
 ; EG-NEXT:    MEM_RAT MSKOR T1.XW, T0.X
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    Fetch clause starting at 14:
-; EG-NEXT:     VTX_READ_16 T2.X, T1.X, 46, #3
+; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 46, #3
 ; EG-NEXT:    Fetch clause starting at 16:
-; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 2, #1
+; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 2, #1
 ; EG-NEXT:    Fetch clause starting at 18:
-; EG-NEXT:     VTX_READ_16 T0.X, T1.X, 44, #3
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
 ; EG-NEXT:    ALU clause starting at 20:
-; EG-NEXT:     MOV * T1.X, 0.0,
+; EG-NEXT:     MOV * T0.X, 0.0,
 ; EG-NEXT:    ALU clause starting at 21:
-; EG-NEXT:     MOV T0.X, literal.x,
-; EG-NEXT:     MOV T1.W, literal.y,
-; EG-NEXT:     SETNE_INT * T0.W, T2.X, 0.0,
-; EG-NEXT:    0(0.000000e+00), 1(1.401298e-45)
+; EG-NEXT:     MOV T1.W, literal.x,
+; EG-NEXT:     SETNE_INT * T0.W, T1.X, 0.0,
+; EG-NEXT:    1(1.401298e-45), 0(0.000000e+00)
 ; EG-NEXT:     PRED_SETNE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
+; EG-NEXT:    ALU clause starting at 25:
+; EG-NEXT:     MOV * T1.X, KC0[2].Z,
 ; EG-NEXT:    ALU clause starting at 26:
-; EG-NEXT:     MOV * T0.X, KC0[2].Z,
-; EG-NEXT:    ALU clause starting at 27:
 ; EG-NEXT:     MOV * T1.W, literal.x,
 ; EG-NEXT:    0(0.000000e+00), 0(0.000000e+00)
-; EG-NEXT:    ALU clause starting at 29:
+; EG-NEXT:    ALU clause starting at 28:
 ; EG-NEXT:     MOV T0.W, KC0[2].Y,
 ; EG-NEXT:     SETE_INT * T1.W, T1.W, 0.0,
 ; EG-NEXT:     PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
+; EG-NEXT:    ALU clause starting at 31:
+; EG-NEXT:     BCNT_INT * T1.X, T0.X,
 ; EG-NEXT:    ALU clause starting at 32:
-; EG-NEXT:     BCNT_INT * T0.X, T0.X,
-; EG-NEXT:    ALU clause starting at 33:
 ; EG-NEXT:     LSHL * T1.W, T0.W, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
 ; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T0.X, literal.y,
+; EG-NEXT:     AND_INT * T2.W, T1.X, literal.y,
 ; EG-NEXT:    24(3.363116e-44), 65535(9.183409e-41)
 ; EG-NEXT:     LSHL T1.X, PS, PV.W,
 ; EG-NEXT:     LSHL * T1.W, literal.x, PV.W,
diff --git a/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll b/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
index d616fecfdc1ff..f68b035334fd5 100644
--- a/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
+++ b/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
@@ -6,15 +6,16 @@
 define amdgpu_kernel void @foo() {
 ; CHECK-LABEL: foo:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_2
-; CHECK-NEXT:  ; %bb.1: ; %LeafBlock1
-; CHECK-NEXT:  .LBB0_2: ; %foo.exit
+; CHECK-NEXT:   ; %bb.1:                                ; %LeafBlock1
+; CHECK-NEXT:    s_cmp_eq_u32 s0, 10
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_3
+; CHECK-NEXT:  ; %bb.2:
 ; CHECK-NEXT:    s_mov_b32 s3, 0xf000
 ; CHECK-NEXT:    s_mov_b32 s2, -1
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; CHECK-NEXT:    s_endpgm
-; CHECK-NEXT:  ; %bb.3: ; %sw.bb10
+; CHECK-NEXT: .LBB0_3:
 entry:
   switch i8 poison, label %foo.exit [
     i8 4, label %sw.bb4
diff --git a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
index 3bb840eb51690..4b8ef2c9613a5 100644
--- a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
@@ -29,17 +29,13 @@ define protected amdgpu_kernel void @_RSENC_PRInit______________________________
 ; CHECK-NEXT:    s_cmp_eq_u32 s4, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB0_2
 ; CHECK-NEXT:  ; %bb.3: ; %if.end60
-; CHECK-NEXT:    s_mov_b64 vcc, exec
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_11
 ; CHECK-NEXT:  ; %bb.4: ; %if.end5.i
-; CHECK-NEXT:    s_mov_b64 vcc, vcc
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_11
+; CHECK-NEXT:    s_cbranch_scc0 .LBB0_11
 ; CHECK-NEXT:  ; %bb.5: ; %if.end5.i314
-; CHECK-NEXT:    s_mov_b64 vcc, exec
-; CHECK-NEXT:    s_cbranch_execz .LBB0_11
+; CHECK-NEXT:    s_cbranch_scc0 .LBB0_11
 ; CHECK-NEXT:  ; %bb.6: ; %if.end5.i338
-; CHECK-NEXT:    s_mov_b64 vcc, vcc
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_11
+; CHECK-NEXT:    s_cbranch_scc0 .LBB0_11
 ; CHECK-NEXT:  ; %bb.7: ; %if.end5.i362
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
@@ -50,7 +46,7 @@ define protected amdgpu_kernel void @_RSENC_PRInit______________________________
 ; CHECK-NEXT:    buffer_store_byte v0, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
 ; CHECK-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:257
-; CHECK-NEXT:    s_cbranch_execz .LBB0_11
+; CHECK-NEXT:    s_cbranch_scc0 .LBB0_11
 ; CHECK-NEXT:  ; %bb.8: ; %if.end5.i400
 ; CHECK-NEXT:    flat_load_ubyte v0, v[0:1]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/PowerPC/undef-args.ll b/llvm/test/CodeGen/PowerPC/undef-args.ll
index 6b698ef478b51..cdaaec2add3ee 100644
--- a/llvm/test/CodeGen/PowerPC/undef-args.ll
+++ b/llvm/test/CodeGen/PowerPC/undef-args.ll
@@ -1,5 +1,7 @@
-;; Tests that extending poison results in undef.
-;; Also tests that there are redundant instructions loading 0 into argument registers for unused arguments.
+;; Tests that extending poison results in poison.
+;; Also tests that there are no redundant instructions loading 0 into argument registers for unused arguments.
+
+; REQUIRES: asserts
 
 ; REQUIRES: asserts
 
@@ -62,7 +64,7 @@ entry:
 ; CHECKISEL64-NEXT:   t7: i64 = Register $x1
 ; CHECKISEL64-NEXT:       t0: ch,glue = EntryToken
 ; CHECKISEL64-NEXT:     t6: ch,glue = callseq_start t0, TargetConstant:i64<112>, TargetConstant:i64<0>
-; CHECKISEL64-NEXT:   t11: ch,glue = CopyToReg t6, Register:i64 $x3, Constant:i64<0> 
+; CHECKISEL64-NEXT:   t11: ch,glue = CopyToReg t6, Register:i64 $x3, poison:i64
 ; CHECKISEL64-NEXT:   t13: ch,glue = CopyToReg t11, Register:i64 $x4, Constant:i64<255>, t11:1
 ; CHECKISEL64-NEXT:   t17: ch,glue = PPCISD::CALL_NOP t13, MCSymbol:i64, Register:i64 $x3, Register:i64 $x4, Register:i64 $x2, RegisterMask:Untyped, t13:1
 ; CHECKISEL64-NEXT:     t18: ch,glue = callseq_end t17, TargetConstant:i64<112>, TargetConstant:i64<0>, t17:1
@@ -72,7 +74,6 @@ entry:
 ; CHECKASM64-NEXT: # %bb.0:                                # %entry
 ; CHECKASM64-NEXT:         mflr 0
 ; CHECKASM64-NEXT:         stdu 1, -112(1)
-; CHECKASM64-NEXT:         li 3, 0
 ; CHECKASM64-NEXT:         li 4, 255
 ; CHECKASM64-NEXT:         std 0, 128(1)
 ; CHECKASM64-NEXT:         bl .bar32
@@ -104,7 +105,7 @@ entry:
 ; CHECKISEL32-NEXT:   t9: i32 = Register $r1
 ; CHECKISEL32-NEXT:       t0: ch,glue = EntryToken
 ; CHECKISEL32-NEXT:     t8: ch,glue = callseq_start t0, TargetConstant:i32<56>, TargetConstant:i32<0>
-; CHECKISEL32-NEXT:   t11: ch,glue = CopyToReg t8, Register:i32 $r3, Constant:i32<0> 
+; CHECKISEL32-NEXT:   t11: ch,glue = CopyToReg t8, Register:i32 $r3, poison:i32
 ; CHECKISEL32-NEXT:   t13: ch,glue = CopyToReg t11, Register:i32 $r4, Constant:i32<255>, t11:1
 ; CHECKISEL32-NEXT:   t17: ch,glue = PPCISD::CALL_NOP t13, MCSymbol:i32, Register:i32 $r3, Register:i32 $r4, Register:i32 $r2, RegisterMask:Untyped, t13:1
 ; CHECKISEL32-NEXT:     t18: ch,glue = callseq_end t17, TargetConstant:i32<56>, TargetConstant:i32<0>, t17:1
@@ -114,7 +115,6 @@ entry:
 ; CHECKASM32-NEXT: # %bb.0:                                # %entry
 ; CHECKASM32-NEXT:         mflr 0
 ; CHECKASM32-NEXT:         stwu 1, -64(1)
-; CHECKASM32-NEXT:         li 3, 0
 ; CHECKASM32-NEXT:         li 4, 255
 ; CHECKASM32-NEXT:         stw 0, 72(1)
 ; CHECKASM32-NEXT:         bl .bar8
@@ -128,7 +128,6 @@ entry:
 ; CHECKASM64-NEXT: # %bb.0:                                # %entry
 ; CHECKASM64-NEXT:         mflr 0
 ; CHECKASM64-NEXT:         stdu 1, -112(1)
-; CHECKASM64-NEXT:         li 3, 0
 ; CHECKASM64-NEXT:         li 4, 255
 ; CHECKASM64-NEXT:         std 0, 128(1)
 ; CHECKASM64-NEXT:         bl .bar8
@@ -143,12 +142,12 @@ entry:
 ; CHECKISEL64-NEXT:   t1: i64 = GlobalAddress<ptr @bar8> 0
 ; CHECKISEL64-NEXT:   t2: i8 = poison
 ; CHECKISEL64-NEXT:   t3: i8 = Constant<-1>
-; CHECKISEL64-NEXT:   t4: i32 = Constant<0> 
+; CHECKISEL64-NEXT:   t4: i32 = poison
 ; CHECKISEL64-NEXT:   t5: i32 = Constant<255>
 ; CHECKISEL64-NEXT:   t9: i64 = Register $x1
 ; CHECKISEL64-NEXT:       t0: ch,glue = EntryToken
 ; CHECKISEL64-NEXT:     t8: ch,glue = callseq_start t0, TargetConstant:i64<112>, TargetConstant:i64<0>
-; CHECKISEL64-NEXT:   t13: ch,glue = CopyToReg t8, Register:i64 $x3, Constant:i64<0> 
+; CHECKISEL64-NEXT:   t13: ch,glue = CopyToReg t8, Register:i64 $x3, poison:i64
 ; CHECKISEL64-NEXT:   t15: ch,glue = CopyToReg t13, Register:i64 $x4, Constant:i64<255>, t13:1
 ; CHECKISEL64-NEXT:   t19: ch,glue = PPCISD::CALL_NOP t15, MCSymbol:i64, Register:i64 $x3, Register:i64 $x4, Register:i64 $x2, RegisterMask:Untyped, t15:1
 ; CHECKISEL64-NEXT:     t20: ch,glue = callseq_end t19, TargetConstant:i64<112>, TargetConstant:i64<0>, t19:1
diff --git a/llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll b/llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll
index a5847365159a8..395fc99ea0536 100644
--- a/llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll
+++ b/llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll
@@ -27,14 +27,6 @@ define signext i32 @foo() #1 personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    li a0, 0
-; CHECK-NEXT:    li a1, 0
-; CHECK-NEXT:    li a2, 0
-; CHECK-NEXT:    li a3, 0
-; CHECK-NEXT:    li a4, 0
-; CHECK-NEXT:    li a5, 0
-; CHECK-NEXT:    li a6, 0
-; CHECK-NEXT:    li a7, 0
 ; CHECK-NEXT:    call _Z3fooiiiiiiiiiiPi
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:  .Ltmp1:

From 111af76bd74708929a048f94ef5c52af646f1c64 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi@microsoft.com>
Date: Mon, 21 Apr 2025 10:26:27 -0400
Subject: [PATCH 617/710] [DirectX] add Function name to
 DiagnosticInfoUnsupported Msg in DXILOpLowering (#136234)

fixes #135654

In #128613 we added safe guards to prevent the lowering of just any
intrinsic in the backend. We used `DiagnosticInfoUnsupported` to do
this.

What we found was when using `opt` the diagnostic print function was
called but when using clang the diagnostic message was used.

Printing message in the clang version means we miss valuable debugging
information like function name and function type when LLVMContext was
only needed to call `getBestLocationFromDebugLoc`.

There are a few potential fixes

1. Write a custom DiagnosticInfoUnsupported so we can change the Message
just for DirectX. Too heavy handed so rejected.

2. Add the function name to the Message in DirectX code. Very simple one
line change. Downside is when using opt you see the function name twice.
But makes the clang-dxc bugs more actionable.

3. change CodeGenAction.cpp to always use the print function and not the
message directly. Downside is a bunch of innacurate information shows up
in the message if you don't specify `-debug-info-kind=standalone`.

4. add some book keeping to know which function called the intrinsic
keep a map of these so we can pass the calling function to
`DiagnosticInfoUnsupported` instead of the intrinsic. This would only be
useful if we had debug info so we could distinguish different uses of
the intrinsic by line\col number. We would also need to change from
iterating on every function to doing something like a LazyCallGraph
which is a nonstarter.

5. pick a different means of doing a Diagnostic error, because other
uses of `DiagnosticInfoUnsupported` error when we are in the body of a
function not when we see one being used like in the intrinsic case.

This PR went with a combo of option 2 & 5. Its low code change that also
only impacts the DirectX backend.
---
 clang/test/CodeGenDirectX/unsupported_intrinsic.hlsl | 8 ++++++++
 llvm/lib/Target/DirectX/DXILOpLowering.cpp           | 8 ++++----
 llvm/test/CodeGen/DirectX/unsupported_intrinsic.ll   | 2 +-
 3 files changed, 13 insertions(+), 5 deletions(-)
 create mode 100644 clang/test/CodeGenDirectX/unsupported_intrinsic.hlsl

diff --git a/clang/test/CodeGenDirectX/unsupported_intrinsic.hlsl b/clang/test/CodeGenDirectX/unsupported_intrinsic.hlsl
new file mode 100644
index 0000000000000..db91cb8fb789e
--- /dev/null
+++ b/clang/test/CodeGenDirectX/unsupported_intrinsic.hlsl
@@ -0,0 +1,8 @@
+// REQUIRES: directx-registered-target
+// RUN: not %clang_dxc -T lib_6_3 %s 2>&1 | FileCheck %s
+
+// CHECK: error: Unsupported intrinsic llvm.vector.reduce.and.v4i32 for DXIL lowering
+
+export int vecReduceAndTest(int4 vec) {
+    return __builtin_reduce_and(vec);
+}
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 4574e5f7bbd96..f28171b6d4908 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -8,7 +8,6 @@
 
 #include "DXILOpLowering.h"
 #include "DXILConstants.h"
-#include "DXILIntrinsicExpansion.h"
 #include "DXILOpBuilder.h"
 #include "DXILShaderFlags.h"
 #include "DirectX.h"
@@ -27,6 +26,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "dxil-op-lower"
 
@@ -756,9 +756,9 @@ class OpLowerer {
       case Intrinsic::not_intrinsic:
         continue;
       default: {
-        DiagnosticInfoUnsupported Diag(
-            F, "Unsupported intrinsic for DXIL lowering");
-        M.getContext().diagnose(Diag);
+        SmallString<128> Msg =
+            formatv("Unsupported intrinsic {0} for DXIL lowering", F.getName());
+        M.getContext().emitError(Msg);
         HasErrors |= true;
         break;
       }
diff --git a/llvm/test/CodeGen/DirectX/unsupported_intrinsic.ll b/llvm/test/CodeGen/DirectX/unsupported_intrinsic.ll
index d703f2f04c494..aaf17d901aef7 100644
--- a/llvm/test/CodeGen/DirectX/unsupported_intrinsic.ll
+++ b/llvm/test/CodeGen/DirectX/unsupported_intrinsic.ll
@@ -1,6 +1,6 @@
 ; RUN: not opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
 
-; CHECK: error: <unknown>:0:0: in function llvm.vector.reduce.and.v4i32 i32 (<4 x i32>): Unsupported intrinsic for DXIL lowering
+; CHECK: error: Unsupported intrinsic llvm.vector.reduce.and.v4i32 for DXIL lowering
 define i32 @fn_and(<4 x i32> %0) local_unnamed_addr #0 {
   %2 = tail call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %0)
   ret i32 %2

From e18a77cfbe7549720d1a2584ab160edeb60963a6 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 21 Apr 2025 10:51:03 -0400
Subject: [PATCH 618/710] Revert "[SelectionDAG] Folding 
 ZERO-EXTEND/SIGN_EXTEND poison to Poison value in getNode (#122741)"

This reverts commit f12078e72601e7c03e5d66afab034313caf8f791.

Breaks `check-llvm`, see comments on https://github.com/llvm/llvm-project/pull/122741
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  8 ----
 .../CodeGen/AArch64/arm64-bitfield-extract.ll |  2 +-
 .../CodeGen/AArch64/optimize-cond-branch.ll   |  2 +-
 .../CodeGen/AArch64/sve-extract-element.ll    |  1 +
 .../CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll    | 23 ++++++++---
 .../atomic_optimizations_global_pointer.ll    | 14 +++----
 llvm/test/CodeGen/AMDGPU/ctpop16.ll           | 41 ++++++++++---------
 .../dead-machine-elim-after-dead-lane.ll      |  9 ++--
 .../CodeGen/AMDGPU/mdt-preserving-crash.ll    | 12 ++++--
 llvm/test/CodeGen/PowerPC/undef-args.ll       | 17 ++++----
 llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll |  8 ++++
 11 files changed, 77 insertions(+), 60 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 61509cdb4a372..f502a536d43b1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6282,10 +6282,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
         Flags.setNonNeg(N1->getFlags().hasNonNeg());
       return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
     }
-
-    if (OpOpcode == ISD::POISON)
-      return getPOISON(VT);
-
     if (N1.isUndef())
       // sext(undef) = 0, because the top bits will all be the same.
       return getConstant(0, DL, VT);
@@ -6306,10 +6302,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
       Flags.setNonNeg(N1->getFlags().hasNonNeg());
       return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0), Flags);
     }
-
-    if (OpOpcode == ISD::POISON)
-      return getPOISON(VT);
-
     if (N1.isUndef())
       // zext(undef) = 0, because the top bits will be zero.
       return getConstant(0, DL, VT);
diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
index 4b1fff642e5f5..62ff4bbcc9c27 100644
--- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
@@ -1019,7 +1019,7 @@ define i16 @test_ignored_rightbits(i32 %dst, i32 %in) {
 define void @sameOperandBFI(i64 %src, i64 %src2, ptr %ptr) {
 ; LLC-LABEL: sameOperandBFI:
 ; LLC:       // %bb.0: // %entry
-; LLC-NEXT:    cbnz w8, .LBB30_2
+; LLC-NEXT:    cbnz wzr, .LBB30_2
 ; LLC-NEXT:  // %bb.1: // %if.else
 ; LLC-NEXT:    lsr x8, x0, #47
 ; LLC-NEXT:    and w9, w1, #0x3
diff --git a/llvm/test/CodeGen/AArch64/optimize-cond-branch.ll b/llvm/test/CodeGen/AArch64/optimize-cond-branch.ll
index 8a56360935717..fdf972990e745 100644
--- a/llvm/test/CodeGen/AArch64/optimize-cond-branch.ll
+++ b/llvm/test/CodeGen/AArch64/optimize-cond-branch.ll
@@ -16,7 +16,7 @@ define void @func() uwtable {
 ; CHECK-NEXT:    mov w8, #1 // =0x1
 ; CHECK-NEXT:    cbnz w8, .LBB0_3
 ; CHECK-NEXT:  // %bb.1: // %b1
-; CHECK-NEXT:    cbz w8, .LBB0_4
+; CHECK-NEXT:    cbz wzr, .LBB0_4
 ; CHECK-NEXT:  // %bb.2: // %b3
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    and w0, w8, #0x100
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-element.ll b/llvm/test/CodeGen/AArch64/sve-extract-element.ll
index c340df1385124..9ebeb098c60c0 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-element.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-element.ll
@@ -523,6 +523,7 @@ define double @test_lanex_2xf64(<vscale x 2 x double> %a, i32 %x) #0 {
 define i32 @test_undef_lane_4xi32(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: test_undef_lane_4xi32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
   %b = extractelement <vscale x 4 x i32> %a, i32 poison
   ret i32 %b
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
index b0627c3d4e77d..b3d9e61b65b6f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
@@ -92,17 +92,27 @@ define i16 @bitcast_f16_to_i16(half %a, i32 %b) {
 ; GCN-LABEL: bitcast_f16_to_i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v2
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB1_3
+; GCN-NEXT:  ; %bb.1: ; %Flow
+; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GCN-NEXT:    s_cbranch_execnz .LBB1_4
+; GCN-NEXT:  .LBB1_2: ; %end
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB1_3: ; %cmp.false
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_2
-; GCN-NEXT:  ; %bb.1:
-; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:  .LBB1_4: ; %cmp.true
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v1
 ; GCN-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
 ; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GCN-NEXT:  .LBB1_2:
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -239,9 +249,10 @@ define i16 @bitcast_bf16_to_i16(bfloat %a, i32 %b) {
 ; GCN-LABEL: bitcast_bf16_to_i16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GCN-NEXT:    v_mul_f32_e32 v1, 1.0, v0
-; GCN-NEXT:    ; implicit-def: $vgpr0
+; GCN-NEXT:    v_mul_f32_e32 v1, 1.0, v2
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execnz .LBB3_3
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index c019c83da5ef8..e2dfcf55b7856 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -8383,10 +8383,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_load_dword s6, s[4:5], 0xd
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT:                       ; implicit-def: $vgpr0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB15_2
 ; GFX7LESS-NEXT:  ; %bb.1:
@@ -8731,10 +8731,10 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_load_dword s10, s[4:5], 0xd
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v4, s7, v1
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX7LESS-NEXT:                               ; implicit-def: $vgpr0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_4
 ; GFX7LESS-NEXT:  ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
index fb418afb8b039..6bf126af5ade2 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
@@ -1292,7 +1292,7 @@ define amdgpu_kernel void @ctpop_i16_in_br(ptr addrspace(1) %out, ptr addrspace(
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ; SI-NEXT:  .LBB14_4:
-; SI-NEXT:                    ; implicit-def: $vgpr0
+; SI-NEXT:    v_mov_b32_e32 v0, 0
 ; SI-NEXT:    s_branch .LBB14_2
 ;
 ; VI-LABEL: ctpop_i16_in_br:
@@ -1329,47 +1329,48 @@ define amdgpu_kernel void @ctpop_i16_in_br(ptr addrspace(1) %out, ptr addrspace(
 ; EG:       ; %bb.0: ; %entry
 ; EG-NEXT:    ALU 0, @20, KC0[], KC1[]
 ; EG-NEXT:    TEX 0 @14
-; EG-NEXT:    ALU_PUSH_BEFORE 3, @21, KC0[], KC1[]
+; EG-NEXT:    ALU_PUSH_BEFORE 4, @21, KC0[], KC1[]
 ; EG-NEXT:    JUMP @7 POP:1
-; EG-NEXT:    ALU 0, @25, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 0, @26, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    TEX 0 @16
-; EG-NEXT:    ALU_POP_AFTER 1, @26, KC0[], KC1[]
-; EG-NEXT:    ALU_PUSH_BEFORE 2, @28, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU_POP_AFTER 1, @27, KC0[], KC1[]
+; EG-NEXT:    ALU_PUSH_BEFORE 2, @29, KC0[CB0:0-32], KC1[]
 ; EG-NEXT:    JUMP @11 POP:1
 ; EG-NEXT:    TEX 0 @18
-; EG-NEXT:    ALU_POP_AFTER 0, @31, KC0[], KC1[]
-; EG-NEXT:    ALU 11, @32, KC0[], KC1[]
+; EG-NEXT:    ALU_POP_AFTER 0, @32, KC0[], KC1[]
+; EG-NEXT:    ALU 11, @33, KC0[], KC1[]
 ; EG-NEXT:    MEM_RAT MSKOR T1.XW, T0.X
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    Fetch clause starting at 14:
-; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 46, #3
+; EG-NEXT:     VTX_READ_16 T2.X, T1.X, 46, #3
 ; EG-NEXT:    Fetch clause starting at 16:
-; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 2, #1
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 2, #1
 ; EG-NEXT:    Fetch clause starting at 18:
-; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
+; EG-NEXT:     VTX_READ_16 T0.X, T1.X, 44, #3
 ; EG-NEXT:    ALU clause starting at 20:
-; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:     MOV * T1.X, 0.0,
 ; EG-NEXT:    ALU clause starting at 21:
-; EG-NEXT:     MOV T1.W, literal.x,
-; EG-NEXT:     SETNE_INT * T0.W, T1.X, 0.0,
-; EG-NEXT:    1(1.401298e-45), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.X, literal.x,
+; EG-NEXT:     MOV T1.W, literal.y,
+; EG-NEXT:     SETNE_INT * T0.W, T2.X, 0.0,
+; EG-NEXT:    0(0.000000e+00), 1(1.401298e-45)
 ; EG-NEXT:     PRED_SETNE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
-; EG-NEXT:    ALU clause starting at 25:
-; EG-NEXT:     MOV * T1.X, KC0[2].Z,
 ; EG-NEXT:    ALU clause starting at 26:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 27:
 ; EG-NEXT:     MOV * T1.W, literal.x,
 ; EG-NEXT:    0(0.000000e+00), 0(0.000000e+00)
-; EG-NEXT:    ALU clause starting at 28:
+; EG-NEXT:    ALU clause starting at 29:
 ; EG-NEXT:     MOV T0.W, KC0[2].Y,
 ; EG-NEXT:     SETE_INT * T1.W, T1.W, 0.0,
 ; EG-NEXT:     PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
-; EG-NEXT:    ALU clause starting at 31:
-; EG-NEXT:     BCNT_INT * T1.X, T0.X,
 ; EG-NEXT:    ALU clause starting at 32:
+; EG-NEXT:     BCNT_INT * T0.X, T0.X,
+; EG-NEXT:    ALU clause starting at 33:
 ; EG-NEXT:     LSHL * T1.W, T0.W, literal.x,
 ; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
 ; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
-; EG-NEXT:     AND_INT * T2.W, T1.X, literal.y,
+; EG-NEXT:     AND_INT * T2.W, T0.X, literal.y,
 ; EG-NEXT:    24(3.363116e-44), 65535(9.183409e-41)
 ; EG-NEXT:     LSHL T1.X, PS, PV.W,
 ; EG-NEXT:     LSHL * T1.W, literal.x, PV.W,
diff --git a/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll b/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
index f68b035334fd5..d616fecfdc1ff 100644
--- a/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
+++ b/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
@@ -6,16 +6,15 @@
 define amdgpu_kernel void @foo() {
 ; CHECK-LABEL: foo:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:   ; %bb.1:                                ; %LeafBlock1
-; CHECK-NEXT:    s_cmp_eq_u32 s0, 10
-; CHECK-NEXT:    s_cbranch_scc1 .LBB0_3
-; CHECK-NEXT:  ; %bb.2:
+; CHECK-NEXT:    s_cbranch_execnz .LBB0_2
+; CHECK-NEXT:  ; %bb.1: ; %LeafBlock1
+; CHECK-NEXT:  .LBB0_2: ; %foo.exit
 ; CHECK-NEXT:    s_mov_b32 s3, 0xf000
 ; CHECK-NEXT:    s_mov_b32 s2, -1
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; CHECK-NEXT:    s_endpgm
-; CHECK-NEXT: .LBB0_3:
+; CHECK-NEXT:  ; %bb.3: ; %sw.bb10
 entry:
   switch i8 poison, label %foo.exit [
     i8 4, label %sw.bb4
diff --git a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
index 4b8ef2c9613a5..3bb840eb51690 100644
--- a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
@@ -29,13 +29,17 @@ define protected amdgpu_kernel void @_RSENC_PRInit______________________________
 ; CHECK-NEXT:    s_cmp_eq_u32 s4, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB0_2
 ; CHECK-NEXT:  ; %bb.3: ; %if.end60
+; CHECK-NEXT:    s_mov_b64 vcc, exec
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_11
 ; CHECK-NEXT:  ; %bb.4: ; %if.end5.i
-; CHECK-NEXT:    s_cbranch_scc0 .LBB0_11
+; CHECK-NEXT:    s_mov_b64 vcc, vcc
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_11
 ; CHECK-NEXT:  ; %bb.5: ; %if.end5.i314
-; CHECK-NEXT:    s_cbranch_scc0 .LBB0_11
+; CHECK-NEXT:    s_mov_b64 vcc, exec
+; CHECK-NEXT:    s_cbranch_execz .LBB0_11
 ; CHECK-NEXT:  ; %bb.6: ; %if.end5.i338
-; CHECK-NEXT:    s_cbranch_scc0 .LBB0_11
+; CHECK-NEXT:    s_mov_b64 vcc, vcc
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_11
 ; CHECK-NEXT:  ; %bb.7: ; %if.end5.i362
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
@@ -46,7 +50,7 @@ define protected amdgpu_kernel void @_RSENC_PRInit______________________________
 ; CHECK-NEXT:    buffer_store_byte v0, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
 ; CHECK-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:257
-; CHECK-NEXT:    s_cbranch_scc0 .LBB0_11
+; CHECK-NEXT:    s_cbranch_execz .LBB0_11
 ; CHECK-NEXT:  ; %bb.8: ; %if.end5.i400
 ; CHECK-NEXT:    flat_load_ubyte v0, v[0:1]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/PowerPC/undef-args.ll b/llvm/test/CodeGen/PowerPC/undef-args.ll
index cdaaec2add3ee..6b698ef478b51 100644
--- a/llvm/test/CodeGen/PowerPC/undef-args.ll
+++ b/llvm/test/CodeGen/PowerPC/undef-args.ll
@@ -1,7 +1,5 @@
-;; Tests that extending poison results in poison.
-;; Also tests that there are no redundant instructions loading 0 into argument registers for unused arguments.
-
-; REQUIRES: asserts
+;; Tests that extending poison results in undef.
+;; Also tests that there are redundant instructions loading 0 into argument registers for unused arguments.
 
 ; REQUIRES: asserts
 
@@ -64,7 +62,7 @@ entry:
 ; CHECKISEL64-NEXT:   t7: i64 = Register $x1
 ; CHECKISEL64-NEXT:       t0: ch,glue = EntryToken
 ; CHECKISEL64-NEXT:     t6: ch,glue = callseq_start t0, TargetConstant:i64<112>, TargetConstant:i64<0>
-; CHECKISEL64-NEXT:   t11: ch,glue = CopyToReg t6, Register:i64 $x3, poison:i64
+; CHECKISEL64-NEXT:   t11: ch,glue = CopyToReg t6, Register:i64 $x3, Constant:i64<0> 
 ; CHECKISEL64-NEXT:   t13: ch,glue = CopyToReg t11, Register:i64 $x4, Constant:i64<255>, t11:1
 ; CHECKISEL64-NEXT:   t17: ch,glue = PPCISD::CALL_NOP t13, MCSymbol:i64, Register:i64 $x3, Register:i64 $x4, Register:i64 $x2, RegisterMask:Untyped, t13:1
 ; CHECKISEL64-NEXT:     t18: ch,glue = callseq_end t17, TargetConstant:i64<112>, TargetConstant:i64<0>, t17:1
@@ -74,6 +72,7 @@ entry:
 ; CHECKASM64-NEXT: # %bb.0:                                # %entry
 ; CHECKASM64-NEXT:         mflr 0
 ; CHECKASM64-NEXT:         stdu 1, -112(1)
+; CHECKASM64-NEXT:         li 3, 0
 ; CHECKASM64-NEXT:         li 4, 255
 ; CHECKASM64-NEXT:         std 0, 128(1)
 ; CHECKASM64-NEXT:         bl .bar32
@@ -105,7 +104,7 @@ entry:
 ; CHECKISEL32-NEXT:   t9: i32 = Register $r1
 ; CHECKISEL32-NEXT:       t0: ch,glue = EntryToken
 ; CHECKISEL32-NEXT:     t8: ch,glue = callseq_start t0, TargetConstant:i32<56>, TargetConstant:i32<0>
-; CHECKISEL32-NEXT:   t11: ch,glue = CopyToReg t8, Register:i32 $r3, poison:i32
+; CHECKISEL32-NEXT:   t11: ch,glue = CopyToReg t8, Register:i32 $r3, Constant:i32<0> 
 ; CHECKISEL32-NEXT:   t13: ch,glue = CopyToReg t11, Register:i32 $r4, Constant:i32<255>, t11:1
 ; CHECKISEL32-NEXT:   t17: ch,glue = PPCISD::CALL_NOP t13, MCSymbol:i32, Register:i32 $r3, Register:i32 $r4, Register:i32 $r2, RegisterMask:Untyped, t13:1
 ; CHECKISEL32-NEXT:     t18: ch,glue = callseq_end t17, TargetConstant:i32<56>, TargetConstant:i32<0>, t17:1
@@ -115,6 +114,7 @@ entry:
 ; CHECKASM32-NEXT: # %bb.0:                                # %entry
 ; CHECKASM32-NEXT:         mflr 0
 ; CHECKASM32-NEXT:         stwu 1, -64(1)
+; CHECKASM32-NEXT:         li 3, 0
 ; CHECKASM32-NEXT:         li 4, 255
 ; CHECKASM32-NEXT:         stw 0, 72(1)
 ; CHECKASM32-NEXT:         bl .bar8
@@ -128,6 +128,7 @@ entry:
 ; CHECKASM64-NEXT: # %bb.0:                                # %entry
 ; CHECKASM64-NEXT:         mflr 0
 ; CHECKASM64-NEXT:         stdu 1, -112(1)
+; CHECKASM64-NEXT:         li 3, 0
 ; CHECKASM64-NEXT:         li 4, 255
 ; CHECKASM64-NEXT:         std 0, 128(1)
 ; CHECKASM64-NEXT:         bl .bar8
@@ -142,12 +143,12 @@ entry:
 ; CHECKISEL64-NEXT:   t1: i64 = GlobalAddress<ptr @bar8> 0
 ; CHECKISEL64-NEXT:   t2: i8 = poison
 ; CHECKISEL64-NEXT:   t3: i8 = Constant<-1>
-; CHECKISEL64-NEXT:   t4: i32 = poison
+; CHECKISEL64-NEXT:   t4: i32 = Constant<0> 
 ; CHECKISEL64-NEXT:   t5: i32 = Constant<255>
 ; CHECKISEL64-NEXT:   t9: i64 = Register $x1
 ; CHECKISEL64-NEXT:       t0: ch,glue = EntryToken
 ; CHECKISEL64-NEXT:     t8: ch,glue = callseq_start t0, TargetConstant:i64<112>, TargetConstant:i64<0>
-; CHECKISEL64-NEXT:   t13: ch,glue = CopyToReg t8, Register:i64 $x3, poison:i64
+; CHECKISEL64-NEXT:   t13: ch,glue = CopyToReg t8, Register:i64 $x3, Constant:i64<0> 
 ; CHECKISEL64-NEXT:   t15: ch,glue = CopyToReg t13, Register:i64 $x4, Constant:i64<255>, t13:1
 ; CHECKISEL64-NEXT:   t19: ch,glue = PPCISD::CALL_NOP t15, MCSymbol:i64, Register:i64 $x3, Register:i64 $x4, Register:i64 $x2, RegisterMask:Untyped, t15:1
 ; CHECKISEL64-NEXT:     t20: ch,glue = callseq_end t19, TargetConstant:i64<112>, TargetConstant:i64<0>, t19:1
diff --git a/llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll b/llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll
index 395fc99ea0536..a5847365159a8 100644
--- a/llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll
+++ b/llvm/test/CodeGen/RISCV/miss-sp-restore-eh.ll
@@ -27,6 +27,14 @@ define signext i32 @foo() #1 personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    addi sp, sp, -32
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    li a1, 0
+; CHECK-NEXT:    li a2, 0
+; CHECK-NEXT:    li a3, 0
+; CHECK-NEXT:    li a4, 0
+; CHECK-NEXT:    li a5, 0
+; CHECK-NEXT:    li a6, 0
+; CHECK-NEXT:    li a7, 0
 ; CHECK-NEXT:    call _Z3fooiiiiiiiiiiPi
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:  .Ltmp1:

From 1cb82ff9cd95c216f6d8da18637ee47a15bd41f6 Mon Sep 17 00:00:00 2001
From: Michael Liao <michael.hliao@gmail.com>
Date: Mon, 21 Apr 2025 10:12:18 -0400
Subject: [PATCH 619/710] [clang][CIR] Fix various warnings. NFC

- Fix various warnings including '-Wparentheses', '-Wunused-variable',
  and '-Wreturn-type'.
---
 clang/include/clang/CIR/Dialect/IR/CIRTypesDetails.h | 2 +-
 clang/lib/CIR/CodeGen/CIRGenBuilder.h                | 1 +
 clang/lib/CIR/CodeGen/CIRGenTypes.cpp                | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypesDetails.h b/clang/include/clang/CIR/Dialect/IR/CIRTypesDetails.h
index 72ead2381ee47..de8c17cab8642 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypesDetails.h
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypesDetails.h
@@ -54,7 +54,7 @@ struct RecordTypeStorage : public mlir::TypeStorage {
                     RecordType::RecordKind kind)
       : members(members), name(name), incomplete(incomplete), packed(packed),
         padded(padded), kind(kind) {
-    assert(name || !incomplete && "Incomplete records must have a name");
+    assert((name || !incomplete) && "Incomplete records must have a name");
   }
 
   KeyTy getAsKey() const {
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
index d98416734b56e..4f7ff5128d914 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
@@ -63,6 +63,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     case clang::TagTypeKind::Enum:
       llvm_unreachable("enums are not records");
     }
+    llvm_unreachable("Unsupported record kind");
   }
 
   /// Get an incomplete CIR struct type. If we have a complete record
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index c286aef360b01..1e1a2c4fee585 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -153,7 +153,7 @@ isSafeToConvert(const RecordDecl *rd, CIRGenTypes &cgt,
   // out, don't do it.  This includes virtual base classes which get laid out
   // when a class is translated, even though they aren't embedded by-value into
   // the class.
-  if (const CXXRecordDecl *crd = dyn_cast<CXXRecordDecl>(rd)) {
+  if (isa<CXXRecordDecl>(rd)) {
     assert(!cir::MissingFeatures::cxxSupport());
     cgt.getCGModule().errorNYI(rd->getSourceRange(),
                                "isSafeToConvert: CXXRecordDecl");
@@ -237,7 +237,7 @@ mlir::Type CIRGenTypes::convertRecordDeclType(const clang::RecordDecl *rd) {
   assert(insertResult && "isSafeToCovert() should have caught this.");
 
   // Force conversion of non-virtual base classes recursively.
-  if (const auto *cxxRecordDecl = dyn_cast<CXXRecordDecl>(rd)) {
+  if (isa<CXXRecordDecl>(rd)) {
     cgm.errorNYI(rd->getSourceRange(), "CXXRecordDecl");
   }
 

From 08b0c2517eb859667d59873bf04b6aa78a7583a7 Mon Sep 17 00:00:00 2001
From: "Oleksandr T." <oleksandr.tarasiuk@outlook.com>
Date: Mon, 21 Apr 2025 18:12:07 +0300
Subject: [PATCH 620/710] [Clang] disallow use of attributes before extern
 template declarations (#136328)

Fixes #79893


---

This PR addresses the issue of _attributes_ being incorrectly allowed on
`extern template` declarations

```cpp
[[deprecated]] extern template struct S<int>;
```
---
 clang/docs/ReleaseNotes.rst                      | 3 +++
 clang/lib/Parse/Parser.cpp                       | 2 ++
 clang/test/Parser/extern-template-attributes.cpp | 8 ++++++++
 3 files changed, 13 insertions(+)
 create mode 100644 clang/test/Parser/extern-template-attributes.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 7417fdd71a392..36b528d9e20f8 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -301,6 +301,9 @@ related warnings within the method body.
   particularly relevant for AMDGPU targets, where they map to corresponding IR
   metadata.
 
+- Clang now disallows the use of attributes applied before an
+  ``extern template`` declaration (#GH79893).
+
 Improvements to Clang's diagnostics
 -----------------------------------
 
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index d528664bca352..a82aea162716a 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1049,6 +1049,8 @@ Parser::ParseExternalDeclaration(ParsedAttributes &Attrs,
 
   case tok::kw_extern:
     if (getLangOpts().CPlusPlus && NextToken().is(tok::kw_template)) {
+      ProhibitAttributes(Attrs);
+      ProhibitAttributes(DeclSpecAttrs);
       // Extern templates
       SourceLocation ExternLoc = ConsumeToken();
       SourceLocation TemplateLoc = ConsumeToken();
diff --git a/clang/test/Parser/extern-template-attributes.cpp b/clang/test/Parser/extern-template-attributes.cpp
new file mode 100644
index 0000000000000..1301f2f680ccb
--- /dev/null
+++ b/clang/test/Parser/extern-template-attributes.cpp
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -std=c++17 -fms-extensions -verify %s
+
+template <class>
+struct S {};
+
+[[deprecated]] extern template struct S<int>;              // expected-error {{an attribute list cannot appear here}}
+__attribute__((deprecated)) extern template struct S<int>; // expected-error {{an attribute list cannot appear here}}
+__declspec(deprecated) extern template struct S<int>;      // expected-error {{expected unqualified-id}}

From cfc2b0d094f705aa0995eff0dc5f1faf1440a769 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 21 Apr 2025 08:13:16 -0700
Subject: [PATCH 621/710] [llvm] Use llvm::SmallVector::pop_back_val (NFC)
 (#136533)

---
 llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp | 3 +--
 llvm/lib/CodeGen/SplitKit.cpp                  | 3 +--
 llvm/lib/Support/SuffixTree.cpp                | 3 +--
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index a04b6e4c061a7..790edf12b46be 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -1151,8 +1151,7 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
     SmallVector<PointerIntPair<Value *, 1, bool>> Worklist = {{Insn, true}};
     SmallPtrSet<Value *, 8> Visited;
     while (!Worklist.empty()) {
-      auto [V, IsPositive] = Worklist.back();
-      Worklist.pop_back();
+      auto [V, IsPositive] = Worklist.pop_back_val();
       if (!Visited.insert(V).second)
         continue;
 
diff --git a/llvm/lib/CodeGen/SplitKit.cpp b/llvm/lib/CodeGen/SplitKit.cpp
index e55dfaebd028e..f118ee521925f 100644
--- a/llvm/lib/CodeGen/SplitKit.cpp
+++ b/llvm/lib/CodeGen/SplitKit.cpp
@@ -1519,8 +1519,7 @@ void SplitEditor::forceRecomputeVNI(const VNInfo &ParentVNI) {
   const LiveInterval &ParentLI = Edit->getParent();
   const SlotIndexes &Indexes = *LIS.getSlotIndexes();
   do {
-    const VNInfo &VNI = *WorkList.back();
-    WorkList.pop_back();
+    const VNInfo &VNI = *WorkList.pop_back_val();
     for (unsigned I = 0, E = Edit->size(); I != E; ++I)
       forceRecompute(I, VNI);
     if (!VNI.isPHIDef())
diff --git a/llvm/lib/Support/SuffixTree.cpp b/llvm/lib/Support/SuffixTree.cpp
index daf66108657b3..a31287836e860 100644
--- a/llvm/lib/Support/SuffixTree.cpp
+++ b/llvm/lib/Support/SuffixTree.cpp
@@ -95,8 +95,7 @@ void SuffixTree::setSuffixIndices() {
   unsigned CurrNodeLen = 0;
   ToVisit.push_back({CurrNode, CurrNodeLen});
   while (!ToVisit.empty()) {
-    std::tie(CurrNode, CurrNodeLen) = ToVisit.back();
-    ToVisit.pop_back();
+    std::tie(CurrNode, CurrNodeLen) = ToVisit.pop_back_val();
     // Length of the current node from the root down to here.
     CurrNode->setConcatLen(CurrNodeLen);
     if (auto *InternalNode = dyn_cast<SuffixTreeInternalNode>(CurrNode))

From e1bb7f6ddec37567230d3e46719aee5bcd268d5a Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 21 Apr 2025 08:15:08 -0700
Subject: [PATCH 622/710] [LLVM][TableGen] Parameterize NumToSkip in
 DecoderEmitter  (#136456)

- Add command line option `num-to-skip-size` to parameterize the size of
`NumToSkip` bytes in the decoder table. Default value will be 2, and
targets that need larger size can use 3.
- Keep all existing targets, except AArch64, to use size 2, and change
AArch64 to use size 3 since it run into the "disassembler decoding table
too large" error with size 2.
- Additional fixes on top of earlier revert: mark `decodeNumToSkip` as
static (not necessary anymore as the generated code is now in anonymous
namespace, but doing it for consistency) and incorporate Bazel build
changes from https://github.com/llvm/llvm-project/pull/136212
- Following is a rough reduction in size for the decoder tables by
switching to size 2.

```
Target         Old Size   New Size   % Reduction
================================================
AArch64           153254     153254        0.00
AMDGPU            471566     412805       12.46
ARC                 5724       5061       11.58
ARM                84936      73831       13.07
AVR                 1497       1306       12.76
BPF                 2172       1927       11.28
CSKY               10064       8692       13.63
Hexagon            47967      41965       12.51
Lanai               1108        982       11.37
LoongArch          24446      21621       11.56
MSP430              4200       3716       11.52
Mips               36330      31415       13.53
PPC                31897      28098       11.91
RISCV              37979      32790       13.66
Sparc               8331       7252       12.95
SystemZ            36722      32248       12.18
VE                 48296      42873       11.23
XCore               2590       2316       10.58
Xtensa              3827       3316       13.35
```
---
 llvm/lib/Target/AArch64/CMakeLists.txt        |   3 +-
 llvm/test/TableGen/VarLenDecoder.td           |  22 ++--
 llvm/test/TableGen/trydecode-emission.td      |  29 ++++-
 llvm/test/TableGen/trydecode-emission2.td     |  30 +++--
 llvm/test/TableGen/trydecode-emission3.td     |  22 +++-
 llvm/test/TableGen/trydecode-emission4.td     |  24 ++--
 llvm/utils/TableGen/DecoderEmitter.cpp        | 107 ++++++++++--------
 .../llvm-project-overlay/llvm/BUILD.bazel     |   2 +-
 8 files changed, 154 insertions(+), 85 deletions(-)

diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 2300e479bc110..583003f2f46e6 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -7,7 +7,8 @@ tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
 tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
 tablegen(LLVM AArch64GenCallingConv.inc -gen-callingconv)
 tablegen(LLVM AArch64GenDAGISel.inc -gen-dag-isel)
-tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler)
+tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler
+              --large-decoder-table)
 tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel)
 tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel)
 tablegen(LLVM AArch64GenO0PreLegalizeGICombiner.inc -gen-global-isel-combiner
diff --git a/llvm/test/TableGen/VarLenDecoder.td b/llvm/test/TableGen/VarLenDecoder.td
index 5cf0bf8911859..0ca5c6ce53506 100644
--- a/llvm/test/TableGen/VarLenDecoder.td
+++ b/llvm/test/TableGen/VarLenDecoder.td
@@ -1,4 +1,5 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s --check-prefixes=CHECK,CHECK-SMALL
+// RUN: llvm-tblgen -gen-disassembler --large-decoder-table -I %p/../../include %s | FileCheck %s --check-prefixes=CHECK,CHECK-LARGE
 
 include "llvm/Target/Target.td"
 
@@ -46,12 +47,19 @@ def FOO32 : MyVarInst<MemOp32> {
   );
 }
 
-// CHECK:      MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
-// CHECK-NEXT: MCD::OPC_FilterValue, 8, 4, 0, 0, // Skip to: 12
-// CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
-// CHECK-NEXT: MCD::OPC_FilterValue, 9, 4, 0, 0, // Skip to: 21
-// CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
-// CHECK-NEXT: MCD::OPC_Fail,
+// CHECK-SMALL:      MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
+// CHECK-SMALL-NEXT: MCD::OPC_FilterValue, 8, 4, 0, // Skip to: 11
+// CHECK-SMALL-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
+// CHECK-SMALL-NEXT: MCD::OPC_FilterValue, 9, 4, 0, // Skip to: 19
+// CHECK-SMALL-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
+// CHECK-SMALL-NEXT: MCD::OPC_Fail,
+
+// CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
+// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 8, 4, 0, 0, // Skip to: 12
+// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_Decode, 178, 2, 0, // Opcode: FOO16
+// CHECK-LARGE-NEXT: /* 12 */      MCD::OPC_FilterValue, 9, 4, 0, 0, // Skip to: 21
+// CHECK-LARGE-NEXT: /* 17 */      MCD::OPC_Decode, 179, 2, 1, // Opcode: FOO32
+// CHECK-LARGE-NEXT: /* 21 */      MCD::OPC_Fail,
 
 // Instruction length table
 // CHECK: 27,
diff --git a/llvm/test/TableGen/trydecode-emission.td b/llvm/test/TableGen/trydecode-emission.td
index 20d2446eeac7f..ebd7169b96f3d 100644
--- a/llvm/test/TableGen/trydecode-emission.td
+++ b/llvm/test/TableGen/trydecode-emission.td
@@ -1,4 +1,5 @@
 // RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler --large-decoder-table -I %p/../../include %s | FileCheck %s --check-prefix=CHECK-LARGE
 
 // Check that if decoding of an instruction fails and the instruction does not
 // have a complete decoder method that can determine if the bitpattern is valid
@@ -34,10 +35,28 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
-// CHECK-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
-// CHECK-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
-// CHECK-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 26 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23
+// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19
+// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 19
+// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
+// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
+
+// CHECK:       unsigned NumToSkip = *Ptr++;
+// CHECK-NEXT:  NumToSkip |= (*Ptr++) << 8;
+// CHECK-NEXT:  return NumToSkip;
+
+// CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
+// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
+// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
+// CHECK-LARGE-NEXT: /* 15 */      MCD::OPC_TryDecode, 179, 2, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
+// CHECK-LARGE-NEXT: /* 22 */      MCD::OPC_Decode, 178, 2, 1, // Opcode: InstA
+// CHECK-LARGE-NEXT: /* 26 */      MCD::OPC_Fail,
+
+// CHECK-LARGE: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
+
+// CHECK-LARGE:       unsigned NumToSkip = *Ptr++;
+// CHECK-LARGE-NEXT:  NumToSkip |= (*Ptr++) << 8;
+// CHECK-LARGE-NEXT:  NumToSkip |= (*Ptr++) << 16;
+// CHECK-LARGE-NEXT:  return NumToSkip;
diff --git a/llvm/test/TableGen/trydecode-emission2.td b/llvm/test/TableGen/trydecode-emission2.td
index 0584034e41233..f455c6345f40c 100644
--- a/llvm/test/TableGen/trydecode-emission2.td
+++ b/llvm/test/TableGen/trydecode-emission2.td
@@ -1,4 +1,5 @@
 // RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler --large-decoder-table -I %p/../../include %s | FileCheck %s --check-prefix=CHECK-LARGE
 
 include "llvm/Target/Target.td"
 
@@ -31,14 +32,27 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 36, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 8 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
-// CHECK-NEXT: /* 11 */      MCD::OPC_FilterValue, 0, 28, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 16 */      MCD::OPC_CheckField, 0, 2, 3, 7, 0, 0, // Skip to: 30
-// CHECK-NEXT: /* 23 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 30
-// CHECK-NEXT: /* 30 */      MCD::OPC_CheckField, 3, 2, 0, 7, 0, 0, // Skip to: 44
-// CHECK-NEXT: /* 37 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstA, skip to: 44
-// CHECK-NEXT: /* 44 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 31, 0, // Skip to: 38
+// CHECK-NEXT: /* 7 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-NEXT: /* 10 */      MCD::OPC_FilterValue, 0, 24, 0, // Skip to: 38
+// CHECK-NEXT: /* 14 */      MCD::OPC_CheckField, 0, 2, 3, 6, 0, // Skip to: 26
+// CHECK-NEXT: /* 20 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 26
+// CHECK-NEXT: /* 26 */      MCD::OPC_CheckField, 3, 2, 0, 6, 0, // Skip to: 38
+// CHECK-NEXT: /* 32 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, // Opcode: InstA, skip to: 38
+// CHECK-NEXT: /* 38 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 // CHECK: if (!Check(S, DecodeInstA(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
+
+// CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
+// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 36, 0, 0, // Skip to: 44
+// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-LARGE-NEXT: /* 11 */      MCD::OPC_FilterValue, 0, 28, 0, 0, // Skip to: 44
+// CHECK-LARGE-NEXT: /* 16 */      MCD::OPC_CheckField, 0, 2, 3, 7, 0, 0, // Skip to: 30
+// CHECK-LARGE-NEXT: /* 23 */      MCD::OPC_TryDecode, 179, 2, 0, 0, 0, 0, // Opcode: InstB, skip to: 30
+// CHECK-LARGE-NEXT: /* 30 */      MCD::OPC_CheckField, 3, 2, 0, 7, 0, 0, // Skip to: 44
+// CHECK-LARGE-NEXT: /* 37 */      MCD::OPC_TryDecode, 178, 2, 1, 0, 0, 0, // Opcode: InstA, skip to: 44
+// CHECK-LARGE-NEXT: /* 44 */      MCD::OPC_Fail,
+
+// CHECK-LARGE: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
+// CHECK-LARGE: if (!Check(S, DecodeInstA(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission3.td b/llvm/test/TableGen/trydecode-emission3.td
index 4c5be7e1af229..acd61a14b89c5 100644
--- a/llvm/test/TableGen/trydecode-emission3.td
+++ b/llvm/test/TableGen/trydecode-emission3.td
@@ -1,4 +1,5 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler  -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler --large-decoder-table -I %p/../../include %s | FileCheck %s --check-prefix=CHECK-LARGE
 
 include "llvm/Target/Target.td"
 
@@ -35,10 +36,19 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
-// CHECK-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
-// CHECK-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
-// CHECK-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 26 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23
+// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19
+// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, 179, 2, 0, 0, 0, // Opcode: InstB, skip to: 19
+// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, 178, 2, 1, // Opcode: InstA
+// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstBOp(MI, tmp, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
+
+// CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
+// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
+// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
+// CHECK-LARGE-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
+// CHECK-LARGE-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
+// CHECK-LARGE-NEXT: /* 26 */      MCD::OPC_Fail,
+
+// CHECK-LARGE: if (!Check(S, DecodeInstBOp(MI, tmp, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/llvm/test/TableGen/trydecode-emission4.td b/llvm/test/TableGen/trydecode-emission4.td
index 1e51ba5e40768..57eec0fa139d5 100644
--- a/llvm/test/TableGen/trydecode-emission4.td
+++ b/llvm/test/TableGen/trydecode-emission4.td
@@ -1,4 +1,5 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler  -I %p/../../include %s | FileCheck %s
+// RUN: llvm-tblgen -gen-disassembler --large-decoder-table -I %p/../../include %s | FileCheck %s --check-prefix=CHECK-LARGE
 
 // Test for OPC_ExtractField/OPC_CheckField with start bit > 255.
 // These large start values may arise for architectures with long instruction
@@ -32,13 +33,22 @@ def InstB : TestInstruction {
   let hasCompleteDecoder = 0;
 }
 
-
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 250, 3, 4,  // Inst{509-506} ...
-// CHECK-NEXT: /* 4 */       MCD::OPC_FilterValue, 0, 19, 0, 0, // Skip to: 28
-// CHECK-NEXT: /* 9 */       MCD::OPC_CheckField, 248, 3, 2, 0, 7, 0, 0, // Skip to: 24
-// CHECK-NEXT: /* 17 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 24
-// CHECK-NEXT: /* 24 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 28 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 4 */       MCD::OPC_FilterValue, 0, 17, 0, // Skip to: 25
+// CHECK-NEXT: /* 8 */       MCD::OPC_CheckField, 248, 3, 2, 0, 6, 0, // Skip to: 21
+// CHECK-NEXT: /* 15 */      MCD::OPC_TryDecode, 179, 2, 0, 0, 0, // Opcode: InstB, skip to: 21
+// CHECK-NEXT: /* 21 */      MCD::OPC_Decode, 178, 2, 1, // Opcode: InstA
+// CHECK-NEXT: /* 25 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 
+
+// CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 250, 3, 4,  // Inst{509-506} ...
+// CHECK-LARGE-NEXT: /* 4 */       MCD::OPC_FilterValue, 0, 19, 0, 0, // Skip to: 28
+// CHECK-LARGE-NEXT: /* 9 */       MCD::OPC_CheckField, 248, 3, 2, 0, 7, 0, 0, // Skip to: 24
+// CHECK-LARGE-NEXT: /* 17 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 24
+// CHECK-LARGE-NEXT: /* 24 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
+// CHECK-LARGE-NEXT: /* 28 */      MCD::OPC_Fail,
+
+// CHECK-LARGE: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
+
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 7d63126f65ac4..ba360333f86e5 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -32,8 +32,10 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/LEB128.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -76,6 +78,12 @@ static cl::opt<SuppressLevel> DecoderEmitterSuppressDuplicates(
             "significantly reducing Table Duplications")),
     cl::init(SUPPRESSION_DISABLE), cl::cat(DisassemblerEmitterCat));
 
+static cl::opt<bool> LargeTable(
+    "large-decoder-table",
+    cl::desc("Use large decoder table format. This uses 24 bits for offset\n"
+             "in the table instead of the default 16 bits."),
+    cl::init(false), cl::cat(DisassemblerEmitterCat));
+
 STATISTIC(NumEncodings, "Number of encodings considered");
 STATISTIC(NumEncodingsLackingDisasm,
           "Number of encodings without disassembler info");
@@ -85,6 +93,8 @@ STATISTIC(NumEncodingsOmitted, "Number of encodings omitted");
 
 namespace {
 
+unsigned getNumToSkipInBytes() { return LargeTable ? 3 : 2; }
+
 struct EncodingField {
   unsigned Base, Width, Offset;
   EncodingField(unsigned B, unsigned W, unsigned O)
@@ -130,10 +140,29 @@ struct DecoderTable : public std::vector<uint8_t> {
   // in the table for patching.
   size_t insertNumToSkip() {
     size_t Size = size();
-    insert(end(), 3, 0);
+    insert(end(), getNumToSkipInBytes(), 0);
     return Size;
   }
+
+  void patchNumToSkip(size_t FixupIdx, uint32_t DestIdx) {
+    // Calculate the distance from the byte following the fixup entry byte
+    // to the destination. The Target is calculated from after the
+    // `getNumToSkipInBytes()`-byte NumToSkip entry itself, so subtract
+    // `getNumToSkipInBytes()` from the displacement here to account for that.
+    assert(DestIdx >= FixupIdx + getNumToSkipInBytes() &&
+           "Expecting a forward jump in the decoding table");
+    uint32_t Delta = DestIdx - FixupIdx - getNumToSkipInBytes();
+    if (!isUIntN(8 * getNumToSkipInBytes(), Delta))
+      PrintFatalError(
+          "disassembler decoding table too large, try --large-decoder-table");
+
+    (*this)[FixupIdx] = static_cast<uint8_t>(Delta);
+    (*this)[FixupIdx + 1] = static_cast<uint8_t>(Delta >> 8);
+    if (getNumToSkipInBytes() == 3)
+      (*this)[FixupIdx + 2] = static_cast<uint8_t>(Delta >> 16);
+  }
 };
+
 struct DecoderTableInfo {
   DecoderTable Table;
   FixupScopeList FixupStack;
@@ -690,19 +719,8 @@ static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups,
                                uint32_t DestIdx) {
   // Any NumToSkip fixups in the current scope can resolve to the
   // current location.
-  for (uint32_t FixupIdx : reverse(Fixups)) {
-    // Calculate the distance from the byte following the fixup entry byte
-    // to the destination. The Target is calculated from after the 24-bit
-    // NumToSkip entry itself, so subtract three from the displacement here
-    // to account for that.
-    uint32_t Delta = DestIdx - FixupIdx - 3;
-    // Our NumToSkip entries are 24-bits. Make sure our table isn't too
-    // big.
-    assert(isUInt<24>(Delta));
-    Table[FixupIdx] = (uint8_t)Delta;
-    Table[FixupIdx + 1] = (uint8_t)(Delta >> 8);
-    Table[FixupIdx + 2] = (uint8_t)(Delta >> 16);
-  }
+  for (uint32_t FixupIdx : Fixups)
+    Table.patchNumToSkip(FixupIdx, DestIdx);
 }
 
 // Emit table entries to decode instructions given a segment or segments
@@ -759,15 +777,9 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
     Delegate->emitTableEntries(TableInfo);
 
     // Now that we've emitted the body of the handler, update the NumToSkip
-    // of the filter itself to be able to skip forward when false. Subtract
-    // three as to account for the width of the NumToSkip field itself.
-    if (PrevFilter) {
-      uint32_t NumToSkip = Table.size() - PrevFilter - 3;
-      assert(isUInt<24>(NumToSkip) && "disassembler decoding table too large!");
-      Table[PrevFilter] = (uint8_t)NumToSkip;
-      Table[PrevFilter + 1] = (uint8_t)(NumToSkip >> 8);
-      Table[PrevFilter + 2] = (uint8_t)(NumToSkip >> 16);
-    }
+    // of the filter itself to be able to skip forward when false.
+    if (PrevFilter)
+      Table.patchNumToSkip(PrevFilter, Table.size());
   }
 
   // If there is no fallthrough, then the final filter should get fixed
@@ -814,7 +826,8 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     OS << (unsigned)*I++ << ", ";
   };
 
-  // Emit 24-bit numtoskip value to OS, returning the NumToSkip value.
+  // Emit `getNumToSkipInBytes()`-byte numtoskip value to OS, returning the
+  // NumToSkip value.
   auto emitNumToSkip = [](DecoderTable::const_iterator &I,
                           formatted_raw_ostream &OS) {
     uint8_t Byte = *I++;
@@ -823,9 +836,11 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     Byte = *I++;
     OS << (unsigned)Byte << ", ";
     NumToSkip |= Byte << 8;
-    Byte = *I++;
-    OS << (unsigned)(Byte) << ", ";
-    NumToSkip |= Byte << 16;
+    if (getNumToSkipInBytes() == 3) {
+      Byte = *I++;
+      OS << (unsigned)(Byte) << ", ";
+      NumToSkip |= Byte << 16;
+    }
     return NumToSkip;
   };
 
@@ -866,7 +881,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // The filter value is ULEB128 encoded.
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -881,7 +895,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // ULEB128 encoded field value.
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -890,7 +903,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       OS << Indent << "MCD::OPC_CheckPredicate, ";
       emitULEB128(I, OS);
 
-      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
@@ -920,7 +932,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
 
       // Fallthrough for OPC_TryDecode.
 
-      // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
 
       OS << "// Opcode: " << NumberedEncodings[EncodingID]
@@ -1392,9 +1403,8 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
     TableInfo.Table.push_back(NumBits);
     TableInfo.Table.insertULEB128(Ilnd.FieldVal);
 
-    // The fixup is always 24-bits, so go ahead and allocate the space
-    // in the table so all our relative position calculations work OK even
-    // before we fully resolve the real value here.
+    // Allocate space in the table for fixup so all our relative position
+    // calculations work OK even before we fully resolve the real value here.
 
     // Push location for NumToSkip backpatching.
     TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
@@ -2139,6 +2149,15 @@ insertBits(InsnType &field, uint64_t bits, unsigned startBit, unsigned numBits)
 static void emitDecodeInstruction(formatted_raw_ostream &OS,
                                   bool IsVarLenInst) {
   OS << R"(
+static unsigned decodeNumToSkip(const uint8_t *&Ptr) {
+  unsigned NumToSkip = *Ptr++;
+  NumToSkip |= (*Ptr++) << 8;
+)";
+  if (getNumToSkipInBytes() == 3)
+    OS << "  NumToSkip |= (*Ptr++) << 16;\n";
+  OS << R"(  return NumToSkip;
+}
+
 template <typename InsnType>
 static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
                                       InsnType insn, uint64_t Address,
@@ -2176,10 +2195,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the field value.
       uint64_t Val = decodeULEB128AndIncUnsafe(Ptr);
       bool Failed = Val != CurFieldValue;
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // Perform the filter operation.
       if (Failed)
@@ -2203,10 +2219,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       uint64_t ExpectedValue = decodeULEB128(++Ptr, &PtrLen);
       Ptr += PtrLen;
       bool Failed = ExpectedValue != FieldValue;
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // If the actual and expected values don't match, skip.
       if (Failed)
@@ -2221,10 +2234,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     case MCD::OPC_CheckPredicate: {
       // Decode the Predicate Index value.
       unsigned PIdx = decodeULEB128AndIncUnsafe(Ptr);
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
       // Check the predicate.
       bool Failed = !checkDecoderPredicate(PIdx, Bits);
       if (Failed)
@@ -2259,10 +2269,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       // Decode the Opcode value.
       unsigned Opc = decodeULEB128AndIncUnsafe(Ptr);
       unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
-      // NumToSkip is a plain 24-bit integer.
-      unsigned NumToSkip = *Ptr++;
-      NumToSkip |= (*Ptr++) << 8;
-      NumToSkip |= (*Ptr++) << 16;
+      unsigned NumToSkip = decodeNumToSkip(Ptr);
 
       // Perform the decode operation.
       MCInst TmpMI;
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index d2cb3413e88b9..978f9e6b17222 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -2149,7 +2149,7 @@ llvm_target_lib_list = [lib for lib in [
                 "lib/Target/AArch64/AArch64GenSubtargetInfo.inc",
             ),
             (
-                ["-gen-disassembler"],
+                ["-gen-disassembler", "--large-decoder-table"],
                 "lib/Target/AArch64/AArch64GenDisassemblerTables.inc",
             ),
             (

From 52a5332a710d4db22327c3777ebf6873ff20732f Mon Sep 17 00:00:00 2001
From: Sarah Spall <sarahspall@microsoft.com>
Date: Mon, 21 Apr 2025 08:18:56 -0700
Subject: [PATCH 623/710] [HLSL] Make it possible to assign an array from a
 cbuffer (#134174)

Update Sema Checking to always do an HLSL Array RValue cast in the case
we are dealing with hlsl constant array types
Instead of comparing canonical types, compare canonical unqualified
types
Add a test to show it is possible to assign an array from a cbuffer.
Closes #133767
---
 clang/lib/Sema/SemaExprCXX.cpp              | 14 ++---
 clang/lib/Sema/SemaOverload.cpp             | 11 ++--
 clang/test/CodeGenHLSL/ArrayAssignable.hlsl | 63 ++++++++++++++++++++-
 3 files changed, 71 insertions(+), 17 deletions(-)

diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index d46322d75721e..f5a10e0db85ad 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -4745,19 +4745,13 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
   case ICK_HLSL_Array_RValue:
     if (ToType->isArrayParameterType()) {
       FromType = Context.getArrayParameterType(FromType);
-      From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
-                               /*BasePath=*/nullptr, CCK)
-                 .get();
-    } else { // FromType must be ArrayParameterType
-      assert(FromType->isArrayParameterType() &&
-             "FromType must be ArrayParameterType in ICK_HLSL_Array_RValue \
-              if it is not ToType");
+    } else if (FromType->isArrayParameterType()) {
       const ArrayParameterType *APT = cast<ArrayParameterType>(FromType);
       FromType = APT->getConstantArrayType(Context);
-      From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
-                               /*BasePath=*/nullptr, CCK)
-                 .get();
     }
+    From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
+                             /*BasePath=*/nullptr, CCK)
+               .get();
     break;
 
   case ICK_Function_To_Pointer:
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index e4ff8c5489df3..5b224b6c08fef 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -2275,17 +2275,16 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
     // handling here.
     if (ToType->isArrayParameterType()) {
       FromType = S.Context.getArrayParameterType(FromType);
-      SCS.First = ICK_HLSL_Array_RValue;
     } else if (FromType->isArrayParameterType()) {
       const ArrayParameterType *APT = cast<ArrayParameterType>(FromType);
       FromType = APT->getConstantArrayType(S.Context);
-      SCS.First = ICK_HLSL_Array_RValue;
-    } else {
-      SCS.First = ICK_Identity;
     }
 
-    if (S.Context.getCanonicalType(FromType) !=
-        S.Context.getCanonicalType(ToType))
+    SCS.First = ICK_HLSL_Array_RValue;
+
+    // Don't consider qualifiers, which include things like address spaces
+    if (FromType.getCanonicalType().getUnqualifiedType() !=
+        ToType.getCanonicalType().getUnqualifiedType())
       return false;
 
     SCS.setAllToTypes(ToType);
diff --git a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
index e2ff2de68ed99..723d521596baf 100644
--- a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
+++ b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
@@ -1,4 +1,23 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --enable-var-scope
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -finclude-default-header -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+struct S {
+  int x;
+  float f;
+};
+
+// CHECK: [[CBLayout:%.*]] = type <{ [2 x float], [2 x <4 x i32>], [2 x [2 x i32]], [1 x target("dx.Layout", %S, 8, 0, 4)] }>
+// CHECK: @CBArrays.cb = global target("dx.CBuffer", target("dx.Layout", [[CBLayout]], 136, 0, 32, 64, 128))
+// CHECK: @c1 = external addrspace(2) global [2 x float], align 4
+// CHECK: @c2 = external addrspace(2) global [2 x <4 x i32>], align 16
+// CHECK: @c3 = external addrspace(2) global [2 x [2 x i32]], align 4
+// CHECK: @c4 = external addrspace(2) global [1 x target("dx.Layout", %S, 8, 0, 4)], align 4
+
+cbuffer CBArrays {
+  float c1[2];
+  int4 c2[2];
+  int c3[2][2];
+  S c4[1];
+}
 
 // CHECK-LABEL: define void {{.*}}arr_assign1
 // CHECK: [[Arr:%.*]] = alloca [2 x i32], align 4
@@ -116,3 +135,45 @@ void arr_assign7() {
   int Arr2[2][2] = {{0, 0}, {1, 1}};
   (Arr = Arr2)[0] = {6, 6};
 }
+
+// Verify you can assign from a cbuffer array
+
+// CHECK-LABEL: define void {{.*}}arr_assign8
+// CHECK: [[C:%.*]] = alloca [2 x float], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[C]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c1, i32 8, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign8() {
+  float C[2] = {1.0, 2.0};
+  C = c1;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign9
+// CHECK: [[C:%.*]] = alloca [2 x <4 x i32>], align 16
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[C]], ptr align 16 {{.*}}, i32 32, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 16 [[C]], ptr addrspace(2) align 16 @c2, i32 32, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign9() {
+  int4 C[2] = {1,2,3,4,5,6,7,8};
+  C = c2;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign10
+// CHECK: [[C:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[C]], ptr align 4 {{.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c3, i32 16, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign10() {
+  int C[2][2] = {1,2,3,4};
+  C = c3;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign11
+// CHECK: [[C:%.*]] = alloca [1 x %struct.S], align 4
+// CHECK: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c4, i32 8, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign11() {
+  S s = {1, 2.0};
+  S C[1] = {s};
+  C = c4;
+}

From 93b74f7178e510a12f0eae700a25f7d3eb226265 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Mon, 21 Apr 2025 08:43:21 -0700
Subject: [PATCH 624/710] [ctxprof] Scale up everything under a root by its
 `TotalRootEntryCount` (#136015)

`TotalRootEntryCount` captures how many times that root was entered - regardless if a profile was also collected or not (profile collection for a given root happens on only one thread at a time).

We don't do this in compiler_rt because the goal there is to flush out the data as fast as possible, so traversing and multiplying vectors is punted to the profile user.

We really just need to do this when flattening the profile so that the values across roots and flat profiles match. We could do it earlier, too - like when loading the profile - but it seems beneficial (at least for debugging) to keep the counter values the same as the loaded ones. We can revisit this later.
---
 llvm/lib/Analysis/CtxProfAnalysis.cpp         | 59 +++++++++++--------
 .../CtxProfAnalysis/flatten-and-annotate.ll   | 26 ++++----
 .../CtxProfAnalysis/flatten-check-path.ll     |  4 +-
 .../flatten-insert-icp-mdprof.ll              |  2 +-
 .../CtxProfAnalysis/flatten-zero-path.ll      |  2 +-
 .../Analysis/CtxProfAnalysis/full-cycle.ll    |  8 +--
 llvm/test/Analysis/CtxProfAnalysis/inline.ll  |  8 ++-
 llvm/test/Analysis/CtxProfAnalysis/load.ll    |  8 +--
 8 files changed, 66 insertions(+), 51 deletions(-)

diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp
index 391631e15aa89..0f0999cae49e5 100644
--- a/llvm/lib/Analysis/CtxProfAnalysis.cpp
+++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp
@@ -621,17 +621,23 @@ CtxProfAnalysis::getSelectInstrumentation(SelectInst &SI) {
   return nullptr;
 }
 
-template <class ProfilesTy, class ProfTy>
-static void preorderVisit(ProfilesTy &Profiles,
-                          function_ref<void(ProfTy &)> Visitor) {
+template <class ProfTy>
+static void preorderVisitOneRoot(ProfTy &Profile,
+                                 function_ref<void(ProfTy &)> Visitor) {
   std::function<void(ProfTy &)> Traverser = [&](auto &Ctx) {
     Visitor(Ctx);
     for (auto &[_, SubCtxSet] : Ctx.callsites())
       for (auto &[__, Subctx] : SubCtxSet)
         Traverser(Subctx);
   };
+  Traverser(Profile);
+}
+
+template <class ProfilesTy, class ProfTy>
+static void preorderVisit(ProfilesTy &Profiles,
+                          function_ref<void(ProfTy &)> Visitor) {
   for (auto &[_, P] : Profiles)
-    Traverser(P);
+    preorderVisitOneRoot<ProfTy>(P, Visitor);
 }
 
 void PGOContextualProfile::initIndex() {
@@ -683,40 +689,47 @@ void PGOContextualProfile::visit(ConstVisitor V, const Function *F) const {
 const CtxProfFlatProfile PGOContextualProfile::flatten() const {
   CtxProfFlatProfile Flat;
   auto Accummulate = [](SmallVectorImpl<uint64_t> &Into,
-                        const SmallVectorImpl<uint64_t> &From) {
+                        const SmallVectorImpl<uint64_t> &From,
+                        uint64_t SamplingRate) {
     if (Into.empty())
       Into.resize(From.size());
     assert(Into.size() == From.size() &&
            "All contexts corresponding to a function should have the exact "
            "same number of counters.");
     for (size_t I = 0, E = Into.size(); I < E; ++I)
-      Into[I] += From[I];
+      Into[I] += From[I] * SamplingRate;
   };
 
-  preorderVisit<const PGOCtxProfContext::CallTargetMapTy,
-                const PGOCtxProfContext>(
-      Profiles.Contexts, [&](const PGOCtxProfContext &Ctx) {
-        Accummulate(Flat[Ctx.guid()], Ctx.counters());
-      });
-  for (const auto &[_, RC] : Profiles.Contexts)
-    for (const auto &[G, Unh] : RC.getUnhandled())
-      Accummulate(Flat[G], Unh);
+  for (const auto &[_, CtxRoot] : Profiles.Contexts) {
+    const uint64_t SamplingFactor = CtxRoot.getTotalRootEntryCount();
+    preorderVisitOneRoot<const PGOCtxProfContext>(
+        CtxRoot, [&](const PGOCtxProfContext &Ctx) {
+          Accummulate(Flat[Ctx.guid()], Ctx.counters(), SamplingFactor);
+        });
+
+    for (const auto &[G, Unh] : CtxRoot.getUnhandled())
+      Accummulate(Flat[G], Unh, SamplingFactor);
+  }
+  // We don't sample "Flat" currently, so sampling rate is 1.
   for (const auto &[G, FC] : Profiles.FlatProfiles)
-    Accummulate(Flat[G], FC);
+    Accummulate(Flat[G], FC, /*SamplingRate=*/1);
   return Flat;
 }
 
 const CtxProfFlatIndirectCallProfile
 PGOContextualProfile::flattenVirtCalls() const {
   CtxProfFlatIndirectCallProfile Ret;
-  preorderVisit<const PGOCtxProfContext::CallTargetMapTy,
-                const PGOCtxProfContext>(
-      Profiles.Contexts, [&](const PGOCtxProfContext &Ctx) {
-        auto &Targets = Ret[Ctx.guid()];
-        for (const auto &[ID, SubctxSet] : Ctx.callsites())
-          for (const auto &Subctx : SubctxSet)
-            Targets[ID][Subctx.first] += Subctx.second.getEntrycount();
-      });
+  for (const auto &[_, CtxRoot] : Profiles.Contexts) {
+    const uint64_t TotalRootEntryCount = CtxRoot.getTotalRootEntryCount();
+    preorderVisitOneRoot<const PGOCtxProfContext>(
+        CtxRoot, [&](const PGOCtxProfContext &Ctx) {
+          auto &Targets = Ret[Ctx.guid()];
+          for (const auto &[ID, SubctxSet] : Ctx.callsites())
+            for (const auto &Subctx : SubctxSet)
+              Targets[ID][Subctx.first] +=
+                  Subctx.second.getEntrycount() * TotalRootEntryCount;
+        });
+  }
   return Ret;
 }
 
diff --git a/llvm/test/Analysis/CtxProfAnalysis/flatten-and-annotate.ll b/llvm/test/Analysis/CtxProfAnalysis/flatten-and-annotate.ll
index 6daf4f5020043..d91f44047e739 100644
--- a/llvm/test/Analysis/CtxProfAnalysis/flatten-and-annotate.ll
+++ b/llvm/test/Analysis/CtxProfAnalysis/flatten-and-annotate.ll
@@ -23,13 +23,13 @@
 ; PRELINK-LABEL: yes:
 ; PRELINK-NEXT:    call void @llvm.instrprof.increment(ptr @an_entrypoint, i64 [[#]], i32 2, i32 1)
 ; PRELINK:      ![[#]] = !{i32 1, !"ProfileSummary", !1}
-; PRELINK:      ![[#]] = !{!"TotalCount", i64 3595}
-; PRELINK:      ![[#]] = !{!"MaxCount", i64 3000}
-; PRELINK:      ![[#]] = !{!"MaxInternalCount", i64 3000}
-; PRELINK:      ![[#]] = !{!"MaxFunctionCount", i64 300}
+; PRELINK:      ![[#]] = !{!"TotalCount", i64 151600}
+; PRELINK:      ![[#]] = !{!"MaxCount", i64 102000}
+; PRELINK:      ![[#]] = !{!"MaxInternalCount", i64 102000}
+; PRELINK:      ![[#]] = !{!"MaxFunctionCount", i64 20100}
 ; PRELINK:      ![[#]] = !{!"NumCounts", i64 6}
 ; PRELINK:      ![[#]] = !{!"NumFunctions", i64 3}
-; PRELINK:       ![[PREPROF]] = !{!"branch_weights", i32 40, i32 60}
+; PRELINK:       ![[PREPROF]] = !{!"branch_weights", i32 4000, i32 6000}
 
 ; Check that the output has:
 ;  - no instrumentation
@@ -49,10 +49,10 @@
 ; The postlink summary is restricted to the stuff under the root - including the
 ; "unhandled" data.
 ; POSTLINK:      ![[#]] = !{i32 1, !"ProfileSummary", !1}
-; POSTLINK:      ![[#]] = !{!"TotalCount", i64 1495}
-; POSTLINK:      ![[#]] = !{!"MaxCount", i64 1000}
-; POSTLINK:      ![[#]] = !{!"MaxInternalCount", i64 1000}
-; POSTLINK:      ![[#]] = !{!"MaxFunctionCount", i64 200}
+; POSTLINK:      ![[#]] = !{!"TotalCount", i64 149500}
+; POSTLINK:      ![[#]] = !{!"MaxCount", i64 100000}
+; POSTLINK:      ![[#]] = !{!"MaxInternalCount", i64 100000}
+; POSTLINK:      ![[#]] = !{!"MaxFunctionCount", i64 20000}
 ; POSTLINK:      ![[#]] = !{!"NumCounts", i64 6}
 ; POSTLINK:      ![[#]] = !{!"NumFunctions", i64 3}
 
@@ -60,14 +60,14 @@
 ; @foo will be called both unconditionally and conditionally, on the "yes" branch
 ; which has a count of 40. So 140 times.
 
-; POSTLINK:       ![[FOO_EP]] = !{!"function_entry_count", i64 140} 
+; POSTLINK:       ![[FOO_EP]] = !{!"function_entry_count", i64 14000} 
 
 ; foo's "no" branch is taken 10+5 times (from the 2 contexts belonging to foo).
 ; Which means its "yes" branch is taken 140 - 15 times.
 
-; POSTLINK:       ![[FOO_BW]] = !{!"branch_weights", i32 125, i32 15} 
-; POSTLINK:       ![[AN_ENTRYPOINT_EP]] = !{!"function_entry_count", i64 100}
-; POSTLINK:       ![[AN_ENTRYPOINT_BW]] = !{!"branch_weights", i32 40, i32 60} 
+; POSTLINK:       ![[FOO_BW]] = !{!"branch_weights", i32 12500, i32 1500} 
+; POSTLINK:       ![[AN_ENTRYPOINT_EP]] = !{!"function_entry_count", i64 10000}
+; POSTLINK:       ![[AN_ENTRYPOINT_BW]] = !{!"branch_weights", i32 4000, i32 6000} 
 
 ;--- profile.yaml
 Contexts:
diff --git a/llvm/test/Analysis/CtxProfAnalysis/flatten-check-path.ll b/llvm/test/Analysis/CtxProfAnalysis/flatten-check-path.ll
index bf672998c1e39..d69768d295907 100644
--- a/llvm/test/Analysis/CtxProfAnalysis/flatten-check-path.ll
+++ b/llvm/test/Analysis/CtxProfAnalysis/flatten-check-path.ll
@@ -13,8 +13,8 @@
 
 ; CHECK: br i1 %x, label %b1, label %exit, !prof ![[PROF1:[0-9]+]]
 ; CHECK: br i1 %y, label %blk, label %exit, !prof ![[PROF2:[0-9]+]]
-; CHECK: ![[PROF1]] = !{!"branch_weights", i32 1, i32 1}
-; CHECK: ![[PROF2]] = !{!"branch_weights", i32 0, i32 1}
+; CHECK: ![[PROF1]] = !{!"branch_weights", i32 2, i32 2}
+; CHECK: ![[PROF2]] = !{!"branch_weights", i32 0, i32 2}
 ; ASSERTION: Assertion `allTakenPathsExit()
 
 ; b1->exit is the only way out from b1, but the exit block would have been
diff --git a/llvm/test/Analysis/CtxProfAnalysis/flatten-insert-icp-mdprof.ll b/llvm/test/Analysis/CtxProfAnalysis/flatten-insert-icp-mdprof.ll
index 13beddc05c7a2..2943dce43e01f 100644
--- a/llvm/test/Analysis/CtxProfAnalysis/flatten-insert-icp-mdprof.ll
+++ b/llvm/test/Analysis/CtxProfAnalysis/flatten-insert-icp-mdprof.ll
@@ -11,7 +11,7 @@
 ; PRELINK-NEXT: call void %p(), !prof ![[VPPROF:[0-9]+]]
 ; PRELINK-NEXT: call void @llvm.instrprof.callsite(ptr @foo, i64 1234, i32 2, i32 1, ptr @bar)
 ; PRELINK-NEXT: call void @bar(){{$}}
-; PRELINK:      ![[VPPROF]] = !{!"VP", i32 0, i64 5, i64 5678, i64 4, i64 5555, i64 1}
+; PRELINK:      ![[VPPROF]] = !{!"VP", i32 0, i64 25, i64 5678, i64 20, i64 5555, i64 5}
 
 ; RUN: cp %t/example.ll %t/1234.ll
 ; RUN: opt -passes=ctx-prof-flatten %t/1234.ll -use-ctx-profile=%t/profile.ctxprofdata \
diff --git a/llvm/test/Analysis/CtxProfAnalysis/flatten-zero-path.ll b/llvm/test/Analysis/CtxProfAnalysis/flatten-zero-path.ll
index 558f385b4bb9d..3d349119cfd0c 100644
--- a/llvm/test/Analysis/CtxProfAnalysis/flatten-zero-path.ll
+++ b/llvm/test/Analysis/CtxProfAnalysis/flatten-zero-path.ll
@@ -14,7 +14,7 @@
 ; CHECK-LABEL: yes:
 ; CHECK:          br i1 %t3, label %yes1, label %yes2, !prof ![[C1]]
 ; CHECK-NOT:   !prof
-; CHECK: ![[C1]] = !{!"branch_weights", i32 6, i32 0}
+; CHECK: ![[C1]] = !{!"branch_weights", i32 72, i32 0}
 
 ;--- 1234.ll
 define void @f1(i32 %cond) !guid !0 {
diff --git a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll
index 63abdd892bffb..8cadcae1654c9 100644
--- a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll
+++ b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll
@@ -108,7 +108,7 @@ Contexts:
                 Counters:        [ 1, 2 ]
 
 Flat Profile:
-2072045998141807037 : 7 
-3087265239403591524 : 11 9 
-4197650231481825559 : 2 
-10507721908651011566 : 1 
+2072045998141807037 : 70 
+3087265239403591524 : 110 90 
+4197650231481825559 : 20 
+10507721908651011566 : 10 
diff --git a/llvm/test/Analysis/CtxProfAnalysis/inline.ll b/llvm/test/Analysis/CtxProfAnalysis/inline.ll
index a069acee1c943..77880dbce1ab8 100644
--- a/llvm/test/Analysis/CtxProfAnalysis/inline.ll
+++ b/llvm/test/Analysis/CtxProfAnalysis/inline.ll
@@ -44,11 +44,13 @@
 ; PIPELINE-LABEL: loop:
 ; PIPELINE:         br i1 %cond, label %loop, label %exit, !prof ![[LOOP_BW_ORIG:[0-9]+]]
 
-; PIPELINE: ![[ENTRYPOINT_COUNT]] = !{!"function_entry_count", i64 10}
+; *Note* that all values are multiplied by the TotalRootEntryCount, which is 24
+;
+; PIPELINE: ![[ENTRYPOINT_COUNT]] = !{!"function_entry_count", i64 240}
 ; These are the weights of the inlined @a, where the counters were 2, 100 (2 for entry, 100 for loop)
-; PIPELINE: ![[LOOP_BW_INL]] = !{!"branch_weights", i32 98, i32 2}
+; PIPELINE: ![[LOOP_BW_INL]] = !{!"branch_weights", i32 2352, i32 48}
 ; These are the weights of the un-inlined @a, where the counters were 8, 500 (8 for entry, 500 for loop)
-; PIPELINE: ![[LOOP_BW_ORIG]] = !{!"branch_weights", i32 492, i32 8}
+; PIPELINE: ![[LOOP_BW_ORIG]] = !{!"branch_weights", i32 11808, i32 192}
 
 ;--- 1000.ll
 define i32 @entrypoint(i32 %x) !guid !0 {
diff --git a/llvm/test/Analysis/CtxProfAnalysis/load.ll b/llvm/test/Analysis/CtxProfAnalysis/load.ll
index bd21a4b710630..92e7fdc40e229 100644
--- a/llvm/test/Analysis/CtxProfAnalysis/load.ll
+++ b/llvm/test/Analysis/CtxProfAnalysis/load.ll
@@ -60,10 +60,10 @@ Contexts:
     Counters:        [ 5 ]
 
 Flat Profile:
-12341 : 9 
-728453322856651412 : 6 7 
-11872291593386833696 : 1 
-12074870348631550642 : 5 
+12341 : 810 
+728453322856651412 : 24 28 
+11872291593386833696 : 4 
+12074870348631550642 : 120 
 ;--- example.ll
 declare void @bar()
 

From 86a03367bf62375cf0cc299a56f09bed5f0c4875 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Mon, 21 Apr 2025 08:59:41 -0700
Subject: [PATCH 625/710] [flang] Support fir.pack_array in FIR alias analysis.
 (#131946)

`fir.pack_array` is just a pass-through op for the process
of finding the source in FIR alias analysis (as defined in #127147).
---
 .../lib/Optimizer/Analysis/AliasAnalysis.cpp  |  8 ++++
 .../alias-analysis-pack-array.fir             | 43 +++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 flang/test/Analysis/AliasAnalysis/alias-analysis-pack-array.fir

diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
index 2ff8ec05782c1..bda98871a1c7a 100644
--- a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
+++ b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -542,6 +542,14 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
           v = op->getOperand(0);
           defOp = v.getDefiningOp();
         })
+        .Case<fir::PackArrayOp>([&](auto op) {
+          // The packed array is not distinguishable from the original
+          // array, so skip PackArrayOp and track further through
+          // the array operand.
+          v = op.getArray();
+          defOp = v.getDefiningOp();
+          approximateSource = true;
+        })
         .Case<fir::BoxAddrOp>([&](auto op) {
           v = op->getOperand(0);
           defOp = v.getDefiningOp();
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-pack-array.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-pack-array.fir
new file mode 100644
index 0000000000000..06d090407a26b
--- /dev/null
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-pack-array.fir
@@ -0,0 +1,43 @@
+// Test alias analysis queries for dummy arguments wired
+// through fir.pack_array.
+// fir.pack_array is a pass-through operation for FIR alias analysis.
+// RUN: fir-opt %s --test-fir-alias-analysis -split-input-file --mlir-disable-threading 2>&1 | FileCheck %s
+
+// The two pointers referencing two different maybe repacked
+// versions of the original dummy arguments do not alias:
+// CHECK-DAG: test1_y_repack(1)#0 <-> test1_x_repack(1)#0: NoAlias
+// CHECK-DAG: test1_x_orig(1)#0 <-> test1_y_orig(1)#0: NoAlias
+
+// Repacked dummy does not alias with another original dummy:
+// CHECK-DAG: test1_y_repack(1)#0 <-> test1_x_orig(1)#0: NoAlias
+// CHECK-DAG: test1_x_repack(1)#0 <-> test1_y_orig(1)#0: NoAlias
+
+// Repacked dummy may alias with its original:
+// CHECK-DAG: test1_x_repack(1)#0 <-> test1_x_orig(1)#0: MayAlias
+// CHECK-DAG: test1_y_repack(1)#0 <-> test1_y_orig(1)#0: MayAlias
+
+// Ideally, these should report MustAlias, but MayAlias
+// may work as well:
+// CHECK-DAG: test1_y_repack(1)#0 <-> test1_y_repack2(1)#0: MayAlias
+// CHECK-DAG: test1_x_repack(1)#0 <-> test1_x_repack2(1)#0: MayAlias
+
+
+func.func @_QFtest1(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "y"}) {
+  %c1 = arith.constant 1 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.pack_array %arg0 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %2:2 = hlfir.declare %1 dummy_scope %0 {uniq_name = "_QFtest1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %3 = fir.pack_array %arg1 heap whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  %4:2 = hlfir.declare %3 dummy_scope %0 {uniq_name = "_QFtest1Ey"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %5 = fir.box_addr %4#0 {test.ptr = "test1_y_repack(1)"} : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<f32>
+  %52 = fir.box_addr %4#0 {test.ptr = "test1_y_repack2(1)"} : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<f32>
+  %6 = fir.load %5 : !fir.ref<f32>
+  %7 = fir.box_addr %2#0 {test.ptr = "test1_x_repack(1)"} : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<f32>
+  %72 = fir.box_addr %2#0 {test.ptr = "test1_x_repack2(1)"} : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<f32>
+  hlfir.assign %6 to %7 : f32, !fir.ref<f32>
+  fir.unpack_array %3 to %arg1 heap : !fir.box<!fir.array<?xf32>>
+  fir.unpack_array %1 to %arg0 heap : !fir.box<!fir.array<?xf32>>
+  %8 = fir.box_addr %arg0 {test.ptr = "test1_x_orig(1)"} : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<f32>
+  %9 = fir.box_addr %arg1 {test.ptr = "test1_y_orig(1)"} : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<f32>
+  return
+}

From 5133b432bc89f66547f6aab9a935f0a29f3b4572 Mon Sep 17 00:00:00 2001
From: Daniel Chen <cdchen@ca.ibm.com>
Date: Mon, 21 Apr 2025 12:05:48 -0400
Subject: [PATCH 626/710] Enable `-m32`, `-maix32` and `-maix64` for Flang on
 AIX. (#136202)

This PR enables `-m32`, -`maix32` and `-maix64` for AIX only. For other
platforms, the driver will issue an error that `-m32` is not supported.
---
 clang/include/clang/Driver/Options.td |  8 +++++---
 clang/lib/Driver/Driver.cpp           | 15 ++++++++++-----
 flang/test/Driver/m32-option.f90      | 14 ++++++++++++++
 3 files changed, 29 insertions(+), 8 deletions(-)
 create mode 100644 flang/test/Driver/m32-option.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 919c1c643d080..8224e734a5a5c 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -4702,15 +4702,17 @@ def EB : Flag<["-"], "EB">, Alias<mbig_endian>;
 def m16 : Flag<["-"], "m16">, Group<m_Group>, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CLOption, DXCOption]>;
 def m32 : Flag<["-"], "m32">, Group<m_Group>, Flags<[NoXarchOption]>,
-  Visibility<[ClangOption, CLOption, DXCOption]>;
-def maix32 : Flag<["-"], "maix32">, Group<m_Group>, Flags<[NoXarchOption]>;
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>;
+def maix32 : Flag<["-"], "maix32">, Group<m_Group>, Flags<[NoXarchOption]>,
+  Visibility<[FlangOption]>;
 def mqdsp6_compat : Flag<["-"], "mqdsp6-compat">, Group<m_Group>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable hexagon-qdsp6 backward compatibility">,
   MarshallingInfoFlag<LangOpts<"HexagonQdsp6Compat">>;
 def m64 : Flag<["-"], "m64">, Group<m_Group>, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>;
-def maix64 : Flag<["-"], "maix64">, Group<m_Group>, Flags<[NoXarchOption]>;
+def maix64 : Flag<["-"], "maix64">, Group<m_Group>, Flags<[NoXarchOption]>,
+  Visibility<[FlangOption]>;
 def mx32 : Flag<["-"], "mx32">, Group<m_Group>, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CLOption, DXCOption]>;
 def miamcu : Flag<["-"], "miamcu">, Group<m_Group>, Flags<[NoXarchOption]>,
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 2a51ac2e9f0b9..2b8c6e35263b1 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -733,11 +733,16 @@ static llvm::Triple computeTargetTriple(const Driver &D,
         Target.setEnvironment(llvm::Triple::GNUX32);
     } else if (A->getOption().matches(options::OPT_m32) ||
                A->getOption().matches(options::OPT_maix32)) {
-      AT = Target.get32BitArchVariant().getArch();
-      if (Target.getEnvironment() == llvm::Triple::GNUX32)
-        Target.setEnvironment(llvm::Triple::GNU);
-      else if (Target.getEnvironment() == llvm::Triple::MuslX32)
-        Target.setEnvironment(llvm::Triple::Musl);
+      if (D.IsFlangMode() && !Target.isOSAIX()) {
+        D.Diag(diag::err_drv_unsupported_opt_for_target)
+            << A->getAsString(Args) << Target.str();
+      } else {
+        AT = Target.get32BitArchVariant().getArch();
+        if (Target.getEnvironment() == llvm::Triple::GNUX32)
+          Target.setEnvironment(llvm::Triple::GNU);
+        else if (Target.getEnvironment() == llvm::Triple::MuslX32)
+          Target.setEnvironment(llvm::Triple::Musl);
+      }
     } else if (A->getOption().matches(options::OPT_m16) &&
                Target.get32BitArchVariant().getArch() == llvm::Triple::x86) {
       AT = llvm::Triple::x86;
diff --git a/flang/test/Driver/m32-option.f90 b/flang/test/Driver/m32-option.f90
new file mode 100644
index 0000000000000..722bddfa43739
--- /dev/null
+++ b/flang/test/Driver/m32-option.f90
@@ -0,0 +1,14 @@
+! Check support of -m32.
+! RUN: %flang -target powerpc-ibm-aix -m32 -### - %s 2>&1 | FileCheck -check-prefix=M32 %s
+! RUN: %flang -target powerpc64-ibm-aix -m32 -### - %s 2>&1 | FileCheck -check-prefix=M32 %s
+! RUN: %flang -target powerpc-ibm-aix -maix32 -### - %s 2>&1 | FileCheck -check-prefix=M32 %s
+! RUN: %flang -target powerpc64-ibm-aix -maix32 -### - %s 2>&1 | FileCheck -check-prefix=M32 %s
+! RUN: %flang -target powerpc-ibm-aix -maix64 -### - %s 2>&1 | FileCheck -check-prefix=M64 %s
+! RUN: %flang -target powerpc64-ibm-aix -maix64 -### - %s 2>&1 | FileCheck -check-prefix=M64 %s
+! RUN: not %flang -target powerpc64le-unknown-linux-gnu -m32 -### - %s 2>&1 | FileCheck -check-prefix=M32-ERROR %s
+! RUN: not %flang -target powerpc64le-unknown-linux-gnu -maix32 -### - %s 2>&1 | FileCheck -check-prefix=MAIX32-ERROR %s
+
+! M32: "-triple" "powerpc-ibm-aix{{.*}}"
+! M64: "-triple" "powerpc64-ibm-aix{{.*}}"
+! M32-ERROR: error: unsupported option '-m32' for target 'powerpc64le-unknown-linux-gnu'
+! MAIX32-ERROR: error: unsupported option '-maix32' for target 'powerpc64le-unknown-linux-gnu'

From 76ced7fa782f0d7db9efea871fa6de74706dd9cc Mon Sep 17 00:00:00 2001
From: Pankaj Dwivedi <pankajkumar.divedi@amd.com>
Date: Mon, 21 Apr 2025 21:57:16 +0530
Subject: [PATCH 627/710] [AMDGPU] Insert readfirstlane in the function returns
 in sgpr. (#135326)

insert `readfirstlane` in the function returns in sgpr.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |   7 +-
 .../AMDGPU/add64-low-32-bits-known-zero.ll    |   4 +-
 .../CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll  |  48 +-
 llvm/test/CodeGen/AMDGPU/constrained-shift.ll |  30 +-
 .../AMDGPU/dag-preserve-disjoint-flag.ll      |  44 +-
 .../CodeGen/AMDGPU/flat-atomic-fadd.f64.ll    |  18 +-
 .../CodeGen/AMDGPU/global-atomic-fadd.f64.ll  |  18 +-
 llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll        | 836 +++++++++---------
 .../AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll    |  64 +-
 llvm/test/CodeGen/AMDGPU/ptrmask.ll           |   4 +-
 .../CodeGen/AMDGPU/sdag-print-divergence.ll   |  15 +-
 .../AMDGPU/sub64-low-32-bits-known-zero.ll    |   4 +-
 .../AMDGPU/uniform-vgpr-to-sgpr-return.ll     | 119 +++
 .../Inputs/amdgpu_isel.ll.expected            |  60 +-
 14 files changed, 733 insertions(+), 538 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/uniform-vgpr-to-sgpr-return.ll

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 724a45062c1f4..2e3cd5ca6692d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3221,6 +3221,7 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                               const SDLoc &DL, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
+  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
   if (AMDGPU::isKernel(CallConv)) {
     return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
@@ -3247,6 +3248,8 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   SmallVector<SDValue, 48> RetOps;
   RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
 
+  SDValue ReadFirstLane =
+      DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
   // Copy the result values into the output registers.
   for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
        ++I, ++RealRVLocIdx) {
@@ -3274,7 +3277,9 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     default:
       llvm_unreachable("Unknown loc info!");
     }
-
+    if (TRI->isSGPRPhysReg(VA.getLocReg()))
+      Arg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Arg.getValueType(),
+                        ReadFirstLane, Arg);
     Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Glue);
     Glue = Chain.getValue(1);
     RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
diff --git a/llvm/test/CodeGen/AMDGPU/add64-low-32-bits-known-zero.ll b/llvm/test/CodeGen/AMDGPU/add64-low-32-bits-known-zero.ll
index 52259c4c2e6e1..1d51b8a077566 100644
--- a/llvm/test/CodeGen/AMDGPU/add64-low-32-bits-known-zero.ll
+++ b/llvm/test/CodeGen/AMDGPU/add64-low-32-bits-known-zero.ll
@@ -148,8 +148,8 @@ define <2 x i64> @v_add_v2i64_nonsplat_const_low_bits_known0_0(<2 x i64> %reg) {
 define amdgpu_ps <2 x i64> @s_add_v2i64_splat_const_low_bits_known0_0(<2 x i64> inreg %reg) {
 ; GFX9-LABEL: s_add_v2i64_splat_const_low_bits_known0_0:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_add_i32 s1, s1, 1
 ; GFX9-NEXT:    s_add_i32 s3, s3, 1
+; GFX9-NEXT:    s_add_i32 s1, s1, 1
 ; GFX9-NEXT:    ; return to shader part epilog
   %add = add <2 x i64> %reg, <i64 4294967296, i64 4294967296> ; (1 << 32)
   ret <2 x i64> %add
@@ -158,8 +158,8 @@ define amdgpu_ps <2 x i64> @s_add_v2i64_splat_const_low_bits_known0_0(<2 x i64>
 define amdgpu_ps <2 x i64> @s_add_v2i64_nonsplat_const_low_bits_known0_0(<2 x i64> inreg %reg) {
 ; GFX9-LABEL: s_add_v2i64_nonsplat_const_low_bits_known0_0:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_add_i32 s1, s1, 1
 ; GFX9-NEXT:    s_add_i32 s3, s3, 2
+; GFX9-NEXT:    s_add_i32 s1, s1, 1
 ; GFX9-NEXT:    ; return to shader part epilog
   %add = add <2 x i64> %reg, <i64 4294967296, i64 8589934592> ; (1 << 32), (1 << 33)
   ret <2 x i64> %add
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
index 6885657bbfa36..37928a78622a6 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
@@ -110,9 +110,11 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_offset_rtn(double %val, <4 x i32
   ; GFX90A_GFX942-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFSET_RTN [[COPY7]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY8]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY8]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY9]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY9]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret double %ret
@@ -136,9 +138,11 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_offen_rtn(double %val, <4 x i32>
   ; GFX90A_GFX942-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFEN_RTN [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY9]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY9]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY10]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret double %ret
@@ -162,9 +166,11 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_idxen_rtn(double %val, <4 x i32>
   ; GFX90A_GFX942-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_IDXEN_RTN [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY9]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY9]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY10]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret double %ret
@@ -190,9 +196,11 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_bothen_rtn(double %val, <4 x i32
   ; GFX90A_GFX942-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN [[COPY9]], killed [[REG_SEQUENCE2]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY10]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY11]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY11]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret double %ret
@@ -334,9 +342,11 @@ define amdgpu_ps double @buffer_ptr_atomic_fadd_f64_offset_rtn(double %val, ptr
   ; GFX90A_GFX942-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFSET_RTN [[COPY11]], killed [[REG_SEQUENCE2]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64) on %ir.rsrc, align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY12]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY13]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %val, ptr addrspace(8) %rsrc, i32 0, i32 %soffset, i32 0)
   ret double %ret
@@ -366,9 +376,11 @@ define amdgpu_ps double @buffer_ptr_atomic_fadd_f64_offen_rtn(double %val, ptr a
   ; GFX90A_GFX942-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFEN_RTN [[COPY12]], [[COPY1]], killed [[REG_SEQUENCE2]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64) on %ir.rsrc, align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY13]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY14]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY14]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret double %ret
@@ -398,9 +410,11 @@ define amdgpu_ps double @buffer_ptr_atomic_fadd_f64_idxen_rtn(double %val, ptr a
   ; GFX90A_GFX942-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_IDXEN_RTN [[COPY12]], [[COPY1]], killed [[REG_SEQUENCE2]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64) on %ir.rsrc, align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY13]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY14]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY14]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret double %ret
@@ -432,9 +446,11 @@ define amdgpu_ps double @buffer_ptr_atomic_fadd_f64_bothen_rtn(double %val, ptr
   ; GFX90A_GFX942-NEXT:   [[COPY13:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN [[COPY13]], killed [[REG_SEQUENCE4]], killed [[REG_SEQUENCE2]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64) on %ir.rsrc, align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY14]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY14]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY15]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY15]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret double %ret
diff --git a/llvm/test/CodeGen/AMDGPU/constrained-shift.ll b/llvm/test/CodeGen/AMDGPU/constrained-shift.ll
index 661af021e8a84..af4ca2ad7120a 100644
--- a/llvm/test/CodeGen/AMDGPU/constrained-shift.ll
+++ b/llvm/test/CodeGen/AMDGPU/constrained-shift.ll
@@ -168,26 +168,26 @@ define <4 x i32> @csh_v4i32(<4 x i32> %a, <4 x i32> %b) {
 define amdgpu_ps <4 x i32> @s_csh_v4i32(<4 x i32> inreg %a, <4 x i32> inreg %b) {
 ; CHECK-LABEL: s_csh_v4i32:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_lshl_b32 s8, s0, s4
-; CHECK-NEXT:    s_lshl_b32 s9, s1, s5
-; CHECK-NEXT:    s_lshl_b32 s10, s2, s6
-; CHECK-NEXT:    s_lshl_b32 s11, s3, s7
-; CHECK-NEXT:    s_lshr_b32 s12, s0, s4
-; CHECK-NEXT:    s_lshr_b32 s13, s1, s5
-; CHECK-NEXT:    s_lshr_b32 s14, s2, s6
-; CHECK-NEXT:    s_lshr_b32 s15, s3, s7
-; CHECK-NEXT:    s_ashr_i32 s3, s3, s7
-; CHECK-NEXT:    s_ashr_i32 s2, s2, s6
-; CHECK-NEXT:    s_ashr_i32 s1, s1, s5
+; CHECK-NEXT:    s_lshl_b32 s8, s3, s7
+; CHECK-NEXT:    s_lshl_b32 s9, s2, s6
+; CHECK-NEXT:    s_lshl_b32 s10, s1, s5
+; CHECK-NEXT:    s_lshl_b32 s11, s0, s4
+; CHECK-NEXT:    s_lshr_b32 s12, s3, s7
+; CHECK-NEXT:    s_lshr_b32 s13, s2, s6
+; CHECK-NEXT:    s_lshr_b32 s14, s1, s5
+; CHECK-NEXT:    s_lshr_b32 s15, s0, s4
 ; CHECK-NEXT:    s_ashr_i32 s0, s0, s4
+; CHECK-NEXT:    s_ashr_i32 s1, s1, s5
+; CHECK-NEXT:    s_ashr_i32 s2, s2, s6
+; CHECK-NEXT:    s_ashr_i32 s3, s3, s7
 ; CHECK-NEXT:    s_add_i32 s4, s11, s15
 ; CHECK-NEXT:    s_add_i32 s5, s10, s14
 ; CHECK-NEXT:    s_add_i32 s6, s9, s13
 ; CHECK-NEXT:    s_add_i32 s7, s8, s12
-; CHECK-NEXT:    s_add_i32 s0, s7, s0
-; CHECK-NEXT:    s_add_i32 s1, s6, s1
-; CHECK-NEXT:    s_add_i32 s2, s5, s2
-; CHECK-NEXT:    s_add_i32 s3, s4, s3
+; CHECK-NEXT:    s_add_i32 s3, s7, s3
+; CHECK-NEXT:    s_add_i32 s2, s6, s2
+; CHECK-NEXT:    s_add_i32 s1, s5, s1
+; CHECK-NEXT:    s_add_i32 s0, s4, s0
 ; CHECK-NEXT:    ; return to shader part epilog
 ;
 ; GISEL-LABEL: s_csh_v4i32:
diff --git a/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll b/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll
index 4b4718a2acb80..d63a36c4b2958 100644
--- a/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll
+++ b/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll
@@ -10,8 +10,10 @@ define amdgpu_ps i32 @s_or_i32_disjoint(i32 inreg %a, i32 inreg %b) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; CHECK-NEXT:   %3:sreg_32 = disjoint S_OR_B32 [[COPY1]], [[COPY]], implicit-def dead $scc
-  ; CHECK-NEXT:   $sgpr0 = COPY %3
+  ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = disjoint S_OR_B32 [[COPY1]], [[COPY]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY2]], implicit $exec
+  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0
   %result = or disjoint i32 %a, %b
   ret i32 %result
@@ -26,10 +28,14 @@ define amdgpu_ps <2 x i32> @s_or_v2i32_disjoint(<2 x i32> inreg %a, <2 x i32> in
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr2
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; CHECK-NEXT:   %5:sreg_32 = disjoint S_OR_B32 [[COPY3]], [[COPY1]], implicit-def dead $scc
-  ; CHECK-NEXT:   %6:sreg_32 = disjoint S_OR_B32 [[COPY2]], [[COPY]], implicit-def dead $scc
-  ; CHECK-NEXT:   $sgpr0 = COPY %5
-  ; CHECK-NEXT:   $sgpr1 = COPY %6
+  ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = disjoint S_OR_B32 [[COPY2]], [[COPY]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32 = disjoint S_OR_B32 [[COPY3]], [[COPY1]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_1]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY4]], implicit $exec
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
+  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %result = or disjoint <2 x i32> %a, %b
   ret <2 x i32> %result
@@ -42,8 +48,8 @@ define i32 @v_or_i32_disjoint(i32 %a, i32 %b) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; CHECK-NEXT:   %10:vgpr_32 = disjoint V_OR_B32_e64 [[COPY1]], [[COPY]], implicit $exec
-  ; CHECK-NEXT:   $vgpr0 = COPY %10
+  ; CHECK-NEXT:   [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = disjoint V_OR_B32_e64 [[COPY1]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY [[V_OR_B32_e64_]]
   ; CHECK-NEXT:   SI_RETURN implicit $vgpr0
   %result = or disjoint i32 %a, %b
   ret i32 %result
@@ -58,10 +64,10 @@ define <2 x i32> @v_or_v2i32_disjoint(<2 x i32> %a, <2 x i32> %b) {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; CHECK-NEXT:   %12:vgpr_32 = disjoint V_OR_B32_e64 [[COPY3]], [[COPY1]], implicit $exec
-  ; CHECK-NEXT:   %13:vgpr_32 = disjoint V_OR_B32_e64 [[COPY2]], [[COPY]], implicit $exec
-  ; CHECK-NEXT:   $vgpr0 = COPY %12
-  ; CHECK-NEXT:   $vgpr1 = COPY %13
+  ; CHECK-NEXT:   [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = disjoint V_OR_B32_e64 [[COPY3]], [[COPY1]], implicit $exec
+  ; CHECK-NEXT:   [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = disjoint V_OR_B32_e64 [[COPY2]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY [[V_OR_B32_e64_]]
+  ; CHECK-NEXT:   $vgpr1 = COPY [[V_OR_B32_e64_1]]
   ; CHECK-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
   %result = or disjoint <2 x i32> %a, %b
   ret <2 x i32> %result
@@ -78,11 +84,15 @@ define amdgpu_ps i64 @s_or_i64_disjoint(i64 inreg %a, i64 inreg %b) {
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
-  ; CHECK-NEXT:   %7:sreg_64 = disjoint S_OR_B64 killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY %7.sub1
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY %7.sub0
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY5]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[COPY4]]
+  ; CHECK-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = disjoint S_OR_B64 killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub1
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
+  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %result = or disjoint i64 %a, %b
   ret i64 %result
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/flat-atomic-fadd.f64.ll
index 36714b386e7e5..f2f8c0a5cfa8f 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomic-fadd.f64.ll
@@ -44,9 +44,11 @@ define amdgpu_ps double @flat_atomic_fadd_f64_rtn_intrinsic(ptr %ptr, double %da
   ; GFX90A_GFX942-NEXT:   [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
   ; GFX90A_GFX942-NEXT:   [[FLAT_ATOMIC_ADD_F64_RTN:%[0-9]+]]:vreg_64_align2 = FLAT_ATOMIC_ADD_F64_RTN killed [[COPY4]], killed [[COPY5]], 0, 1, implicit $exec, implicit $flat_scr :: (load store syncscope("agent") seq_cst (s64) on %ir.ptr)
   ; GFX90A_GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY7]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.flat.atomic.fadd.f64.p1.f64(ptr %ptr, double %data)
   ret double %ret
@@ -117,9 +119,11 @@ define amdgpu_ps double @flat_atomic_fadd_f64_rtn_atomicrmw(ptr %ptr, double %da
   ; GFX90A_GFX942-NEXT:   [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
   ; GFX90A_GFX942-NEXT:   [[FLAT_ATOMIC_ADD_F64_RTN:%[0-9]+]]:vreg_64_align2 = FLAT_ATOMIC_ADD_F64_RTN killed [[COPY4]], killed [[COPY5]], 0, 1, implicit $exec, implicit $flat_scr :: (load store syncscope("wavefront") monotonic (s64) on %ir.ptr)
   ; GFX90A_GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY7]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = atomicrmw fadd ptr %ptr, double %data syncscope("wavefront") monotonic, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
   ret double %ret
@@ -144,9 +148,11 @@ define amdgpu_ps double @flat_atomic_fadd_f64_rtn_atomicrmw__noprivate(ptr %ptr,
   ; GFX90A_GFX942-NEXT:   [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
   ; GFX90A_GFX942-NEXT:   [[FLAT_ATOMIC_ADD_F64_RTN:%[0-9]+]]:vreg_64_align2 = FLAT_ATOMIC_ADD_F64_RTN killed [[COPY4]], killed [[COPY5]], 0, 1, implicit $exec, implicit $flat_scr :: (load store syncscope("wavefront") monotonic (s64) on %ir.ptr)
   ; GFX90A_GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY7]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = atomicrmw fadd ptr %ptr, double %data syncscope("wavefront") monotonic, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
   ret double %ret
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f64.ll
index fa4e7f87853dd..682c1cd8060aa 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f64.ll
@@ -62,9 +62,11 @@ define amdgpu_ps double @global_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(1) %
   ; GFX90A-NEXT:   [[PHI3:%[0-9]+]]:sreg_64 = PHI [[SI_IF_BREAK]], %bb.1
   ; GFX90A-NEXT:   SI_END_CF [[PHI3]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
   ; GFX90A-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[PHI2]].sub0
+  ; GFX90A-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
   ; GFX90A-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[PHI2]].sub1
-  ; GFX90A-NEXT:   $sgpr0 = COPY [[COPY12]]
-  ; GFX90A-NEXT:   $sgpr1 = COPY [[COPY13]]
+  ; GFX90A-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
+  ; GFX90A-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   ;
   ; GFX942-LABEL: name: global_atomic_fadd_f64_rtn_atomicrmw
@@ -81,9 +83,11 @@ define amdgpu_ps double @global_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(1) %
   ; GFX942-NEXT:   [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
   ; GFX942-NEXT:   [[GLOBAL_ATOMIC_ADD_F64_RTN:%[0-9]+]]:vreg_64_align2 = GLOBAL_ATOMIC_ADD_F64_RTN killed [[COPY4]], killed [[COPY5]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s64) on %ir.ptr, addrspace 1)
   ; GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_ADD_F64_RTN]].sub0
+  ; GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
   ; GFX942-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_ADD_F64_RTN]].sub1
-  ; GFX942-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; GFX942-NEXT:   $sgpr1 = COPY [[COPY7]]
+  ; GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
+  ; GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double %data syncscope("wavefront") monotonic
   ret double %ret
@@ -123,9 +127,11 @@ define amdgpu_ps double @global_atomic_fadd_f64_saddr_rtn_atomicrmw(ptr addrspac
   ; GFX90A_GFX942-NEXT:   [[COPY4:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[GLOBAL_ATOMIC_ADD_F64_SADDR_RTN:%[0-9]+]]:vreg_64_align2 = GLOBAL_ATOMIC_ADD_F64_SADDR_RTN killed [[V_MOV_B32_e32_]], killed [[COPY4]], killed [[REG_SEQUENCE]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s64) on %ir.ptr, addrspace 1)
   ; GFX90A_GFX942-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_ADD_F64_SADDR_RTN]].sub0
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_ADD_F64_SADDR_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY5]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY6]]
+  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
   ret double %ret
diff --git a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
index c88113d62a887..e82801eadc936 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
@@ -358,8 +358,8 @@ define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
 define amdgpu_ps <2 x i32> @s_uitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX7-LABEL: s_uitofp_v2i1_to_v2bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s2, 1, s0
-; GFX7-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
@@ -367,14 +367,14 @@ define amdgpu_ps <2 x i32> @s_uitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uitofp_v2i1_to_v2bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s2, 1, s0
-; GFX9-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
@@ -396,77 +396,75 @@ define amdgpu_ps <2 x i32> @s_uitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_uitofp_v2i1_to_v2bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s0, 1, s0
-; GFX11-NEXT:    s_bitcmp1_b32 s1, 0
-; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
 ; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
-; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v1
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_uitofp_v2i1_to_v2bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s0, 1, s0
-; GFX12-NEXT:    s_bitcmp1_b32 s1, 0
-; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
 ; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v1
 ; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
   %op = uitofp <2 x i1> %num to <2 x bfloat>
@@ -706,11 +704,11 @@ define <3 x bfloat> @v_uitofp_v3i1_to_v3bf16(<3 x i1> %num) {
 define amdgpu_ps <3 x i32> @s_uitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX7-LABEL: s_uitofp_v3i1_to_v3bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s4, 1, s0
-; GFX7-NEXT:    s_and_b32 s3, 1, s1
-; GFX7-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX7-NEXT:    s_and_b32 s4, 1, s2
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX7-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s4, 1
 ; GFX7-NEXT:    s_cselect_b64 s[4:5], -1, 0
@@ -720,18 +718,18 @@ define amdgpu_ps <3 x i32> @s_uitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uitofp_v3i1_to_v3bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s4, 1, s0
-; GFX9-NEXT:    s_and_b32 s3, 1, s1
-; GFX9-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX9-NEXT:    s_and_b32 s4, 1, s2
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s4, 1
 ; GFX9-NEXT:    s_cselect_b64 s[4:5], -1, 0
@@ -759,99 +757,96 @@ define amdgpu_ps <3 x i32> @s_uitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
 ; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_uitofp_v3i1_to_v3bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s0, 1, s0
+; GFX11-NEXT:    s_and_b32 s2, 1, s2
 ; GFX11-NEXT:    s_and_b32 s1, 1, s1
-; GFX11-NEXT:    s_bitcmp1_b32 s2, 0
-; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
 ; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
 ; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
-; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s0
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v1
 ; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
 ; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v2
-; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
 ; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_uitofp_v3i1_to_v3bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s0, 1, s0
+; GFX12-NEXT:    s_and_b32 s2, 1, s2
 ; GFX12-NEXT:    s_and_b32 s1, 1, s1
-; GFX12-NEXT:    s_bitcmp1_b32 s2, 0
-; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s2
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
 ; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
 ; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
-; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s0
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v1
 ; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v2
-; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
 ; GFX12-NEXT:    v_add_nc_u32_e32 v4, v4, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX12-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -1154,10 +1149,10 @@ define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
 define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX7-LABEL: s_uitofp_v4i1_to_v4bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s6, 1, s0
-; GFX7-NEXT:    s_and_b32 s4, 1, s1
-; GFX7-NEXT:    s_and_b32 s2, 1, s2
-; GFX7-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX7-NEXT:    s_and_b32 s6, 1, s3
+; GFX7-NEXT:    s_and_b32 s4, 1, s2
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -1173,18 +1168,18 @@ define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uitofp_v4i1_to_v4bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s6, 1, s0
-; GFX9-NEXT:    s_and_b32 s4, 1, s1
-; GFX9-NEXT:    s_and_b32 s2, 1, s2
-; GFX9-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX9-NEXT:    s_and_b32 s6, 1, s3
+; GFX9-NEXT:    s_and_b32 s4, 1, s2
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -1216,7 +1211,7 @@ define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s[0:1]
 ; GFX9-NEXT:    v_bfe_u32 v5, v3, 16, 1
@@ -1225,133 +1220,126 @@ define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v5, 0x7fff, v5
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_uitofp_v4i1_to_v4bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s0, 1, s0
-; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_and_b32 s3, 1, s3
 ; GFX11-NEXT:    s_and_b32 s2, 1, s2
-; GFX11-NEXT:    s_bitcmp1_b32 s3, 0
-; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s3
-; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s0
 ; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s1
-; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s2
+; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
 ; GFX11-NEXT:    v_bfe_u32 v7, v4, 16, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s3
 ; GFX11-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v10, 0x400000, v4
 ; GFX11-NEXT:    v_add_nc_u32_e32 v7, v7, v4
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v0
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v6, v6, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_or_b32_e32 v11, 0x400000, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX11-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
-; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v4
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v2
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s2, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_uitofp_v4i1_to_v4bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s0, 1, s0
-; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_and_b32 s3, 1, s3
 ; GFX12-NEXT:    s_and_b32 s2, 1, s2
-; GFX12-NEXT:    s_bitcmp1_b32 s3, 0
-; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s3
-; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s2
+; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s0
 ; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s1
-; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s2
+; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
 ; GFX12-NEXT:    v_bfe_u32 v7, v4, 16, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s3
 ; GFX12-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v10, 0x400000, v4
 ; GFX12-NEXT:    v_add_nc_u32_e32 v7, v7, v4
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
 ; GFX12-NEXT:    v_add_nc_u32_e32 v6, v6, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_or_b32_e32 v11, 0x400000, v3
+; GFX12-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX12-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
-; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_readfirstlane_b32 s1, v2
-; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_readfirstlane_b32 s2, v3
-; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX12-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
   %op = uitofp <4 x i1> %num to <4 x bfloat>
@@ -1712,8 +1700,8 @@ define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
 define amdgpu_ps <2 x i32> @s_sitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX7-LABEL: s_sitofp_v2i1_to_v2bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s2, 1, s1
-; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_and_b32 s2, 1, s0
+; GFX7-NEXT:    s_bitcmp1_b32 s1, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[0:1]
@@ -1721,14 +1709,14 @@ define amdgpu_ps <2 x i32> @s_sitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s[0:1]
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_sitofp_v2i1_to_v2bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s2, 1, s1
-; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_and_b32 s2, 1, s0
+; GFX9-NEXT:    s_bitcmp1_b32 s1, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[0:1]
@@ -1750,75 +1738,77 @@ define amdgpu_ps <2 x i32> @s_sitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_sitofp_v2i1_to_v2bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s1, 1, s1
-; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
+; GFX11-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
 ; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
-; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v1
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_sitofp_v2i1_to_v2bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s1, 1, s1
-; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
+; GFX12-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
 ; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
   %op = sitofp <2 x i1> %num to <2 x bfloat>
@@ -2058,11 +2048,11 @@ define <3 x bfloat> @v_sitofp_v3i1_to_v3bf16(<3 x i1> %num) {
 define amdgpu_ps <3 x i32> @s_sitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX7-LABEL: s_sitofp_v3i1_to_v3bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s4, 1, s2
-; GFX7-NEXT:    s_and_b32 s2, 1, s1
-; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_and_b32 s4, 1, s0
+; GFX7-NEXT:    s_and_b32 s3, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s2, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX7-NEXT:    s_cmp_eq_u32 s3, 1
 ; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s4, 1
 ; GFX7-NEXT:    s_cselect_b64 s[4:5], -1, 0
@@ -2072,18 +2062,18 @@ define amdgpu_ps <3 x i32> @s_sitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_sitofp_v3i1_to_v3bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s4, 1, s2
-; GFX9-NEXT:    s_and_b32 s2, 1, s1
-; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_and_b32 s4, 1, s0
+; GFX9-NEXT:    s_and_b32 s3, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s2, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX9-NEXT:    s_cmp_eq_u32 s3, 1
 ; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s4, 1
 ; GFX9-NEXT:    s_cselect_b64 s[4:5], -1, 0
@@ -2111,96 +2101,99 @@ define amdgpu_ps <3 x i32> @s_sitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
 ; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_sitofp_v3i1_to_v3bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
 ; GFX11-NEXT:    s_and_b32 s1, 1, s1
-; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
 ; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s2
 ; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
 ; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
 ; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s0
-; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v1
+; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
 ; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
-; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v2
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v2
+; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_sitofp_v3i1_to_v3bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
 ; GFX12-NEXT:    s_and_b32 s1, 1, s1
-; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
 ; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s2
 ; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
 ; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s0
-; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX12-NEXT:    v_add_nc_u32_e32 v4, v4, v1
-; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v2
+; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX12-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
+; GFX12-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
@@ -2503,10 +2496,10 @@ define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
 define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX7-LABEL: s_sitofp_v4i1_to_v4bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s6, 1, s3
-; GFX7-NEXT:    s_and_b32 s4, 1, s2
-; GFX7-NEXT:    s_and_b32 s2, 1, s1
-; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_and_b32 s6, 1, s0
+; GFX7-NEXT:    s_and_b32 s4, 1, s1
+; GFX7-NEXT:    s_and_b32 s2, 1, s2
+; GFX7-NEXT:    s_bitcmp1_b32 s3, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -2522,18 +2515,18 @@ define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v3
-; GFX7-NEXT:    v_readfirstlane_b32 s1, v2
-; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_sitofp_v4i1_to_v4bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s6, 1, s3
-; GFX9-NEXT:    s_and_b32 s4, 1, s2
-; GFX9-NEXT:    s_and_b32 s2, 1, s1
-; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_and_b32 s6, 1, s0
+; GFX9-NEXT:    s_and_b32 s4, 1, s1
+; GFX9-NEXT:    s_and_b32 s2, 1, s2
+; GFX9-NEXT:    s_bitcmp1_b32 s3, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -2565,7 +2558,7 @@ define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s[0:1]
 ; GFX9-NEXT:    v_bfe_u32 v5, v3, 16, 1
@@ -2574,126 +2567,133 @@ define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v5, 0x7fff, v5
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_sitofp_v4i1_to_v4bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s3, 1, s3
-; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
 ; GFX11-NEXT:    s_and_b32 s1, 1, s1
-; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s0
-; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
 ; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s3
 ; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s3, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s2
-; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s2
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    v_bfe_u32 v7, v4, 16, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s3
-; GFX11-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
 ; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v10, 0x400000, v4
-; GFX11-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX11-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v0
-; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v6, v6, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-NEXT:    v_or_b32_e32 v11, 0x400000, v3
-; GFX11-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
 ; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v4
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
-; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 16, v3
-; GFX11-NEXT:    v_ashrrev_i32_e32 v3, 16, v4
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v3
-; GFX11-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX11-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_sitofp_v4i1_to_v4bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s3, 1, s3
-; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
 ; GFX12-NEXT:    s_and_b32 s1, 1, s1
-; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s0
-; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
 ; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s3
 ; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s3, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s2
-; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s2
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    v_bfe_u32 v7, v4, 16, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s3
-; GFX12-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
 ; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v10, 0x400000, v4
-; GFX12-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX12-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v0
-; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
 ; GFX12-NEXT:    v_add_nc_u32_e32 v6, v6, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    v_or_b32_e32 v11, 0x400000, v3
-; GFX12-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
 ; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX12-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
-; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 16, v3
-; GFX12-NEXT:    v_ashrrev_i32_e32 v3, 16, v4
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX12-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_readfirstlane_b32 s1, v3
-; GFX12-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX12-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
   %op = sitofp <4 x i1> %num to <4 x bfloat>
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
index 3aa5ea995559f..dfde10329fe80 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
@@ -11,12 +11,18 @@ define amdgpu_ps ptr addrspace(8) @basic_raw_buffer(ptr inreg %p) {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
   ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], killed [[S_MOV_B32_]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_AND_B32_]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY2]], implicit $exec
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
   ; CHECK-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1234
-  ; CHECK-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 5678
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[S_AND_B32_]]
-  ; CHECK-NEXT:   $sgpr2 = COPY [[S_MOV_B32_1]]
-  ; CHECK-NEXT:   $sgpr3 = COPY [[S_MOV_B32_2]]
+  ; CHECK-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[S_MOV_B32_1]]
+  ; CHECK-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 5678
+  ; CHECK-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[S_MOV_B32_3]]
+  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_]]
+  ; CHECK-NEXT:   $sgpr2 = COPY [[S_MOV_B32_2]]
+  ; CHECK-NEXT:   $sgpr3 = COPY [[S_MOV_B32_4]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3
   %rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 0, i32 1234, i32 5678)
   ret ptr addrspace(8) %rsrc
@@ -52,12 +58,18 @@ define amdgpu_ps ptr addrspace(8) @basic_struct_buffer(ptr inreg %p) {
   ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], killed [[S_MOV_B32_]], implicit-def dead $scc
   ; CHECK-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 262144
   ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 killed [[S_AND_B32_]], killed [[S_MOV_B32_1]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY2]], implicit $exec
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
   ; CHECK-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 1234
-  ; CHECK-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 5678
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[S_OR_B32_]]
-  ; CHECK-NEXT:   $sgpr2 = COPY [[S_MOV_B32_2]]
-  ; CHECK-NEXT:   $sgpr3 = COPY [[S_MOV_B32_3]]
+  ; CHECK-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[S_MOV_B32_2]]
+  ; CHECK-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 5678
+  ; CHECK-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[S_MOV_B32_4]]
+  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_]]
+  ; CHECK-NEXT:   $sgpr2 = COPY [[S_MOV_B32_3]]
+  ; CHECK-NEXT:   $sgpr3 = COPY [[S_MOV_B32_5]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3
   %rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 4, i32 1234, i32 5678)
   ret ptr addrspace(8) %rsrc
@@ -76,10 +88,18 @@ define amdgpu_ps ptr addrspace(8) @variable_top_half(ptr inreg %p, i32 inreg %nu
   ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY2]], killed [[S_MOV_B32_]], implicit-def dead $scc
   ; CHECK-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 262144
   ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 killed [[S_AND_B32_]], killed [[S_MOV_B32_1]], implicit-def dead $scc
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY3]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[S_OR_B32_]]
-  ; CHECK-NEXT:   $sgpr2 = COPY [[COPY1]]
-  ; CHECK-NEXT:   $sgpr3 = COPY [[COPY]]
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY4]], implicit $exec
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_]]
+  ; CHECK-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; CHECK-NEXT:   $sgpr3 = COPY [[V_READFIRSTLANE_B32_3]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3
   %rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 4, i32 %numVals, i32 %flags)
   ret ptr addrspace(8) %rsrc
@@ -99,10 +119,18 @@ define amdgpu_ps ptr addrspace(8) @general_case(ptr inreg %p, i16 inreg %stride,
   ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY3]], killed [[S_MOV_B32_]], implicit-def dead $scc
   ; CHECK-NEXT:   [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], 16, implicit-def dead $scc
   ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 killed [[S_AND_B32_]], killed [[S_LSHL_B32_]], implicit-def dead $scc
-  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY4]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[S_OR_B32_]]
-  ; CHECK-NEXT:   $sgpr2 = COPY [[COPY1]]
-  ; CHECK-NEXT:   $sgpr3 = COPY [[COPY]]
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_]]
+  ; CHECK-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; CHECK-NEXT:   $sgpr3 = COPY [[V_READFIRSTLANE_B32_3]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3
   %rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 %stride, i32 %numVals, i32 %flags)
   ret ptr addrspace(8) %rsrc
diff --git a/llvm/test/CodeGen/AMDGPU/ptrmask.ll b/llvm/test/CodeGen/AMDGPU/ptrmask.ll
index 8594549318dda..9ad9c80d82ff3 100644
--- a/llvm/test/CodeGen/AMDGPU/ptrmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptrmask.ll
@@ -186,8 +186,10 @@ define ptr addrspace(8) @v_ptrmask_buffer_resource_variable_i128_neg8(ptr addrsp
 define amdgpu_ps ptr addrspace(8) @s_ptrmask_buffer_resource_variable_i128(ptr addrspace(8) inreg %ptr, i128 inreg %mask) {
 ; GCN-LABEL: s_ptrmask_buffer_resource_variable_i128:
 ; GCN:       ; %bb.0:
+; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], s[8:9]
 ; GCN-NEXT:    s_and_b64 s[0:1], s[2:3], s[6:7]
-; GCN-NEXT:    s_and_b64 s[2:3], s[4:5], s[8:9]
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX10PLUS-LABEL: s_ptrmask_buffer_resource_variable_i128:
diff --git a/llvm/test/CodeGen/AMDGPU/sdag-print-divergence.ll b/llvm/test/CodeGen/AMDGPU/sdag-print-divergence.ll
index 695d5225421de..482f78889ff4e 100644
--- a/llvm/test/CodeGen/AMDGPU/sdag-print-divergence.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdag-print-divergence.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -O0 -verify-machineinstrs < %s -debug-only=isel 2>&1 | FileCheck --check-prefixes=GCN,GCN-DEFAULT %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -O0 -verify-machineinstrs < %s -debug-only=isel -dag-dump-verbose 2>&1 | FileCheck --check-prefixes=GCN,GCN-VERBOSE %s
 
@@ -5,23 +6,25 @@
 
 ; GCN-LABEL: === test_sdag_dump
 ; GCN: Initial selection DAG: %bb.0 'test_sdag_dump:entry'
-; GCN: SelectionDAG has 10 nodes:
+; GCN: SelectionDAG has 11 nodes:
 
 ; GCN-DEFAULT:  t0: ch,glue = EntryToken
 ; GCN-DEFAULT:  t2: f32,ch = CopyFromReg t0, Register:f32 %0
+; GCN-DEFAULT:  t7: i32 = TargetConstant<3222>
 ; GCN-DEFAULT:      t5: f32 = fadd t2, t2
 ; GCN-DEFAULT:      t4: f32,ch = CopyFromReg # D:1 t0, Register:f32 %1
-; GCN-DEFAULT:    t6: f32 = fadd # D:1 t5, t4
-; GCN-DEFAULT:  t8: ch,glue = CopyToReg # D:1 t0, Register:f32 $vgpr0, t6
-; GCN-DEFAULT:  t9: ch = RETURN_TO_EPILOG t8, Register:f32 $vgpr0, t8:1
+; GCN-DEFAULT:   t6: f32 = fadd # D:1 t5, t4
+; GCN-DEFAULT:  t9: ch,glue = CopyToReg # D:1 t0, Register:f32 $vgpr0, t6
+; GCN-DEFAULT:  t10: ch = RETURN_TO_EPILOG t9, Register:f32 $vgpr0, t9:1
 
 ; GCN-VERBOSE:  t0: ch,glue = EntryToken # D:0
 ; GCN-VERBOSE:  t2: f32,ch = CopyFromReg [ORD=1] # D:0 t0, Register:f32 %0 # D:0
+; GCN-VERBOSE:  t7: i32 = TargetConstant<3222>
 ; GCN-VERBOSE:      t5: f32 = fadd [ORD=2] # D:0 t2, t2
 ; GCN-VERBOSE:      t4: f32,ch = CopyFromReg [ORD=1] # D:1 t0, Register:f32 %1 # D:0
 ; GCN-VERBOSE:    t6: f32 = fadd [ORD=3] # D:1 t5, t4
-; GCN-VERBOSE:  t8: ch,glue = CopyToReg [ORD=4] # D:1 t0, Register:f32 $vgpr0 # D:0, t6
-; GCN-VERBOSE:  t9: ch = RETURN_TO_EPILOG [ORD=4] # D:0 t8, Register:f32 $vgpr0 # D:0, t8:1
+; GCN-VERBOSE:  t9: ch,glue = CopyToReg [ORD=4] # D:1 t0, Register:f32 $vgpr0 # D:0, t6
+; GCN-VERBOSE:  t10: ch = RETURN_TO_EPILOG [ORD=4] # D:0 t9, Register:f32 $vgpr0 # D:0, t9:1
 
 define amdgpu_ps float @test_sdag_dump(float inreg %scalar, float %vector)  {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/sub64-low-32-bits-known-zero.ll b/llvm/test/CodeGen/AMDGPU/sub64-low-32-bits-known-zero.ll
index f52f1164f2ba2..6e4391a5ecaab 100644
--- a/llvm/test/CodeGen/AMDGPU/sub64-low-32-bits-known-zero.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub64-low-32-bits-known-zero.ll
@@ -148,8 +148,8 @@ define <2 x i64> @v_sub_v2i64_nonsplat_const_low_bits_known0_0(<2 x i64> %reg) {
 define amdgpu_ps <2 x i64> @s_sub_v2i64_splat_const_low_bits_known0_0(<2 x i64> inreg %reg) {
 ; GFX9-LABEL: s_sub_v2i64_splat_const_low_bits_known0_0:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_add_i32 s1, s1, -1
 ; GFX9-NEXT:    s_add_i32 s3, s3, -1
+; GFX9-NEXT:    s_add_i32 s1, s1, -1
 ; GFX9-NEXT:    ; return to shader part epilog
   %sub = sub <2 x i64> %reg, <i64 4294967296, i64 4294967296> ; (1 << 32)
   ret <2 x i64> %sub
@@ -158,8 +158,8 @@ define amdgpu_ps <2 x i64> @s_sub_v2i64_splat_const_low_bits_known0_0(<2 x i64>
 define amdgpu_ps <2 x i64> @s_sub_v2i64_nonsplat_const_low_bits_known0_0(<2 x i64> inreg %reg) {
 ; GFX9-LABEL: s_sub_v2i64_nonsplat_const_low_bits_known0_0:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_add_i32 s1, s1, -1
 ; GFX9-NEXT:    s_add_i32 s3, s3, -2
+; GFX9-NEXT:    s_add_i32 s1, s1, -1
 ; GFX9-NEXT:    ; return to shader part epilog
   %sub = sub <2 x i64> %reg, <i64 4294967296, i64 8589934592> ; (1 << 32), (1 << 33)
   ret <2 x i64> %sub
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-vgpr-to-sgpr-return.ll b/llvm/test/CodeGen/AMDGPU/uniform-vgpr-to-sgpr-return.ll
new file mode 100644
index 0000000000000..5476c26e39ba9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/uniform-vgpr-to-sgpr-return.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefixes=GFX11
+
+define amdgpu_ps i32 @uniform_v_to_s_i32(float inreg %a, float inreg %b) {
+; GFX11-LABEL: uniform_v_to_s_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_max_f32_e64 v0, s0, s1
+; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %max0 = call float @llvm.maximum.f32(float %a, float %b)
+  %cast = bitcast float %max0 to i32
+  ret i32 %cast
+}
+
+define amdgpu_ps i64 @uniform_v_to_s_i64(double inreg %a, double inreg %b) {
+; GFX11-LABEL: uniform_v_to_s_i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_max_f64 v[0:1], s[0:1], s[2:3]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, s[0:1], s[2:3]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %max0 = call double @llvm.maximum.f64(double %a, double %b)
+  %cast = bitcast double %max0 to i64
+  ret i64 %cast
+}
+
+define amdgpu_ps <2 x i32> @uniform_v_to_s_2_i32(<2 x float> inreg %a, <2 x float> inreg %b) {
+; GFX11-LABEL: uniform_v_to_s_2_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_max_f32_e64 v0, s0, s2
+; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s2
+; GFX11-NEXT:    v_max_f32_e64 v1, s1, s3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s1, s3
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v1, vcc_lo
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    ; return to shader part epilog
+  %max0 = call <2 x float> @llvm.maximum.f32(<2 x float> %a, <2 x float> %b)
+  %cast = bitcast <2 x float> %max0 to <2 x i32>
+  ret <2 x i32> %cast
+}
+
+define amdgpu_ps ptr @uniform_v_to_s_ptr(ptr inreg %x) {
+; GFX11-LABEL: uniform_v_to_s_ptr:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_mov_b32 s1, 0
+; GFX11-NEXT:    flat_load_b32 v0, v[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_max_f32_e32 v1, 1.0, v0
+; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v1, vcc_lo
+; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %val = load float, ptr %x, align 4
+  %max = call float @llvm.maximum.f32(float %val, float 1.0)
+  %int = fptoui float %max to i32
+  %ptr = inttoptr i32 %int to ptr
+  ret ptr %ptr
+}
+
+define amdgpu_ps double @uniform_v_to_s_double(double inreg %a, double inreg %b) {
+; GFX11-LABEL: uniform_v_to_s_double:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_max_f64 v[0:1], s[0:1], s[2:3]
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, s[0:1], s[2:3]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %max0 = call double @llvm.maximum.f64(double %a, double %b)
+  ret double %max0
+}
+
+define amdgpu_ps <2 x i16> @uniform_v_to_s_2_i16(float inreg %a, float inreg %b) {
+; GFX11-LABEL: uniform_v_to_s_2_i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_max_f32_e64 v0, s0, s1
+; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %max0 = call float @llvm.maximum.f32(float %a, float %b)
+  %cast = bitcast float %max0 to <2 x i16>
+  ret <2 x i16> %cast
+}
+
+define amdgpu_ps i16 @uniform_v_to_s_i16(half inreg %a, half inreg %b) {
+; GFX11-LABEL: uniform_v_to_s_i16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_max_f16_e64 v0, s0, s1
+; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s0, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %max = call half @llvm.maximum.f16(half %a, half %b)
+  %cast = bitcast half %max to i16
+  ret i16 %cast
+}
+
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
index 2202b6446fd15..f590324f1120d 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
@@ -7,16 +7,16 @@ define i64 @i64_test(i64 %i) nounwind readnone {
 ; CHECK-NEXT:    t0: ch,glue = EntryToken
 ; CHECK-NEXT:    t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
 ; CHECK-NEXT:    t4: i32,ch = CopyFromReg # D:1 t0, Register:i32 %9
-; CHECK-NEXT:    t49: i64 = REG_SEQUENCE # D:1 TargetConstant:i32<60>, t2, TargetConstant:i32<3>, t4, TargetConstant:i32<11>
-; CHECK-NEXT:    t26: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc, align 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT:    t29: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc + 4, basealign 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<4>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT:    t32: v2i32 = REG_SEQUENCE # D:1 TargetConstant:i32<60>, t26, TargetConstant:i32<3>, t29, TargetConstant:i32<11>
-; CHECK-NEXT:    t10: i64 = V_ADD_U64_PSEUDO # D:1 t49, t32
-; CHECK-NEXT:    t23: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<3>
-; CHECK-NEXT:    t16: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t23
-; CHECK-NEXT:    t38: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<11>
-; CHECK-NEXT:    t18: ch,glue = CopyToReg # D:1 t16, Register:i32 $vgpr1, t38, t16:1
-; CHECK-NEXT:    t19: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t18, t18:1
+; CHECK-NEXT:    t50: i64 = REG_SEQUENCE # D:1 TargetConstant:i32<60>, t2, TargetConstant:i32<3>, t4, TargetConstant:i32<11>
+; CHECK-NEXT:    t27: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc, align 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
+; CHECK-NEXT:    t30: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc + 4, basealign 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<4>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
+; CHECK-NEXT:    t33: v2i32 = REG_SEQUENCE # D:1 TargetConstant:i32<60>, t27, TargetConstant:i32<3>, t30, TargetConstant:i32<11>
+; CHECK-NEXT:    t10: i64 = V_ADD_U64_PSEUDO # D:1 t50, t33
+; CHECK-NEXT:    t24: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<3>
+; CHECK-NEXT:    t17: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t24
+; CHECK-NEXT:    t39: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<11>
+; CHECK-NEXT:    t19: ch,glue = CopyToReg # D:1 t17, Register:i32 $vgpr1, t39, t17:1
+; CHECK-NEXT:    t20: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t19, t19:1
 ; CHECK-EMPTY:
   %loc = alloca i64, addrspace(5)
   %j = load i64, ptr addrspace(5) %loc
@@ -31,10 +31,10 @@ define i64 @i32_test(i32 %i) nounwind readnone {
 ; CHECK-NEXT:    t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
 ; CHECK-NEXT:    t6: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
 ; CHECK-NEXT:    t7: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t6, TargetConstant:i1<0>
-; CHECK-NEXT:    t14: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t7
-; CHECK-NEXT:    t22: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
-; CHECK-NEXT:    t16: ch,glue = CopyToReg t14, Register:i32 $vgpr1, t22, t14:1
-; CHECK-NEXT:    t17: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t16, t16:1
+; CHECK-NEXT:    t15: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t7
+; CHECK-NEXT:    t23: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
+; CHECK-NEXT:    t17: ch,glue = CopyToReg t15, Register:i32 $vgpr1, t23, t15:1
+; CHECK-NEXT:    t18: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t17, t17:1
 ; CHECK-EMPTY:
   %loc = alloca i32, addrspace(5)
   %j = load i32, ptr addrspace(5) %loc
@@ -48,14 +48,14 @@ define i64 @i16_test(i16 %i) nounwind readnone {
 ; CHECK:       SelectionDAG has 18 nodes:
 ; CHECK-NEXT:    t0: ch,glue = EntryToken
 ; CHECK-NEXT:    t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
-; CHECK-NEXT:    t19: i32,ch = BUFFER_LOAD_USHORT_OFFEN<Mem:(dereferenceable load (s16) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT:    t20: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t19, TargetConstant:i1<0>
-; CHECK-NEXT:    t24: i32 = S_MOV_B32 TargetConstant:i32<65535>
-; CHECK-NEXT:    t25: i32 = V_AND_B32_e64 # D:1 t20, t24
-; CHECK-NEXT:    t15: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t25
-; CHECK-NEXT:    t31: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
-; CHECK-NEXT:    t17: ch,glue = CopyToReg t15, Register:i32 $vgpr1, t31, t15:1
-; CHECK-NEXT:    t18: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t17, t17:1
+; CHECK-NEXT:    t20: i32,ch = BUFFER_LOAD_USHORT_OFFEN<Mem:(dereferenceable load (s16) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
+; CHECK-NEXT:    t21: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t20, TargetConstant:i1<0>
+; CHECK-NEXT:    t25: i32 = S_MOV_B32 TargetConstant:i32<65535>
+; CHECK-NEXT:    t26: i32 = V_AND_B32_e64 # D:1 t21, t25
+; CHECK-NEXT:    t16: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t26
+; CHECK-NEXT:    t32: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
+; CHECK-NEXT:    t18: ch,glue = CopyToReg t16, Register:i32 $vgpr1, t32, t16:1
+; CHECK-NEXT:    t19: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t18, t18:1
 ; CHECK-EMPTY:
   %loc = alloca i16, addrspace(5)
   %j = load i16, ptr addrspace(5) %loc
@@ -69,14 +69,14 @@ define i64 @i8_test(i8 %i) nounwind readnone {
 ; CHECK:       SelectionDAG has 18 nodes:
 ; CHECK-NEXT:    t0: ch,glue = EntryToken
 ; CHECK-NEXT:    t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
-; CHECK-NEXT:    t19: i32,ch = BUFFER_LOAD_UBYTE_OFFEN<Mem:(dereferenceable load (s8) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT:    t20: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t19, TargetConstant:i1<0>
-; CHECK-NEXT:    t24: i32 = S_MOV_B32 TargetConstant:i32<255>
-; CHECK-NEXT:    t25: i32 = V_AND_B32_e64 # D:1 t20, t24
-; CHECK-NEXT:    t15: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t25
-; CHECK-NEXT:    t31: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
-; CHECK-NEXT:    t17: ch,glue = CopyToReg t15, Register:i32 $vgpr1, t31, t15:1
-; CHECK-NEXT:    t18: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t17, t17:1
+; CHECK-NEXT:    t20: i32,ch = BUFFER_LOAD_UBYTE_OFFEN<Mem:(dereferenceable load (s8) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
+; CHECK-NEXT:    t21: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t20, TargetConstant:i1<0>
+; CHECK-NEXT:    t25: i32 = S_MOV_B32 TargetConstant:i32<255>
+; CHECK-NEXT:    t26: i32 = V_AND_B32_e64 # D:1 t21, t25
+; CHECK-NEXT:    t16: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t26
+; CHECK-NEXT:    t32: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
+; CHECK-NEXT:    t18: ch,glue = CopyToReg t16, Register:i32 $vgpr1, t32, t16:1
+; CHECK-NEXT:    t19: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t18, t18:1
 ; CHECK-EMPTY:
   %loc = alloca i8, addrspace(5)
   %j = load i8, ptr addrspace(5) %loc

From f0a59c49aaa951a25bea57cf753a8eb1735fd810 Mon Sep 17 00:00:00 2001
From: Sarah Spall <sarahspall@microsoft.com>
Date: Mon, 21 Apr 2025 09:29:22 -0700
Subject: [PATCH 628/710] Revert "[HLSL] Make it possible to assign an array
 from a cbuffer" (#136576)

Reverts llvm/llvm-project#134174
Revert due to test error in hlsl/ArrayAssignable.hlsl breaking a lot of bots.
---
 clang/lib/Sema/SemaExprCXX.cpp              | 14 +++--
 clang/lib/Sema/SemaOverload.cpp             | 11 ++--
 clang/test/CodeGenHLSL/ArrayAssignable.hlsl | 63 +--------------------
 3 files changed, 17 insertions(+), 71 deletions(-)

diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index f5a10e0db85ad..d46322d75721e 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -4745,13 +4745,19 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
   case ICK_HLSL_Array_RValue:
     if (ToType->isArrayParameterType()) {
       FromType = Context.getArrayParameterType(FromType);
-    } else if (FromType->isArrayParameterType()) {
+      From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
+                               /*BasePath=*/nullptr, CCK)
+                 .get();
+    } else { // FromType must be ArrayParameterType
+      assert(FromType->isArrayParameterType() &&
+             "FromType must be ArrayParameterType in ICK_HLSL_Array_RValue \
+              if it is not ToType");
       const ArrayParameterType *APT = cast<ArrayParameterType>(FromType);
       FromType = APT->getConstantArrayType(Context);
+      From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
+                               /*BasePath=*/nullptr, CCK)
+                 .get();
     }
-    From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
-                             /*BasePath=*/nullptr, CCK)
-               .get();
     break;
 
   case ICK_Function_To_Pointer:
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 5b224b6c08fef..e4ff8c5489df3 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -2275,16 +2275,17 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
     // handling here.
     if (ToType->isArrayParameterType()) {
       FromType = S.Context.getArrayParameterType(FromType);
+      SCS.First = ICK_HLSL_Array_RValue;
     } else if (FromType->isArrayParameterType()) {
       const ArrayParameterType *APT = cast<ArrayParameterType>(FromType);
       FromType = APT->getConstantArrayType(S.Context);
+      SCS.First = ICK_HLSL_Array_RValue;
+    } else {
+      SCS.First = ICK_Identity;
     }
 
-    SCS.First = ICK_HLSL_Array_RValue;
-
-    // Don't consider qualifiers, which include things like address spaces
-    if (FromType.getCanonicalType().getUnqualifiedType() !=
-        ToType.getCanonicalType().getUnqualifiedType())
+    if (S.Context.getCanonicalType(FromType) !=
+        S.Context.getCanonicalType(ToType))
       return false;
 
     SCS.setAllToTypes(ToType);
diff --git a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
index 723d521596baf..e2ff2de68ed99 100644
--- a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
+++ b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
@@ -1,23 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -finclude-default-header -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
-
-struct S {
-  int x;
-  float f;
-};
-
-// CHECK: [[CBLayout:%.*]] = type <{ [2 x float], [2 x <4 x i32>], [2 x [2 x i32]], [1 x target("dx.Layout", %S, 8, 0, 4)] }>
-// CHECK: @CBArrays.cb = global target("dx.CBuffer", target("dx.Layout", [[CBLayout]], 136, 0, 32, 64, 128))
-// CHECK: @c1 = external addrspace(2) global [2 x float], align 4
-// CHECK: @c2 = external addrspace(2) global [2 x <4 x i32>], align 16
-// CHECK: @c3 = external addrspace(2) global [2 x [2 x i32]], align 4
-// CHECK: @c4 = external addrspace(2) global [1 x target("dx.Layout", %S, 8, 0, 4)], align 4
-
-cbuffer CBArrays {
-  float c1[2];
-  int4 c2[2];
-  int c3[2][2];
-  S c4[1];
-}
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --enable-var-scope
 
 // CHECK-LABEL: define void {{.*}}arr_assign1
 // CHECK: [[Arr:%.*]] = alloca [2 x i32], align 4
@@ -135,45 +116,3 @@ void arr_assign7() {
   int Arr2[2][2] = {{0, 0}, {1, 1}};
   (Arr = Arr2)[0] = {6, 6};
 }
-
-// Verify you can assign from a cbuffer array
-
-// CHECK-LABEL: define void {{.*}}arr_assign8
-// CHECK: [[C:%.*]] = alloca [2 x float], align 4
-// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[C]], ptr align 4 {{.*}}, i32 8, i1 false)
-// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c1, i32 8, i1 false)
-// CHECK-NEXT: ret void
-void arr_assign8() {
-  float C[2] = {1.0, 2.0};
-  C = c1;
-}
-
-// CHECK-LABEL: define void {{.*}}arr_assign9
-// CHECK: [[C:%.*]] = alloca [2 x <4 x i32>], align 16
-// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[C]], ptr align 16 {{.*}}, i32 32, i1 false)
-// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 16 [[C]], ptr addrspace(2) align 16 @c2, i32 32, i1 false)
-// CHECK-NEXT: ret void
-void arr_assign9() {
-  int4 C[2] = {1,2,3,4,5,6,7,8};
-  C = c2;
-}
-
-// CHECK-LABEL: define void {{.*}}arr_assign10
-// CHECK: [[C:%.*]] = alloca [2 x [2 x i32]], align 4
-// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[C]], ptr align 4 {{.*}}, i32 16, i1 false)
-// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c3, i32 16, i1 false)
-// CHECK-NEXT: ret void
-void arr_assign10() {
-  int C[2][2] = {1,2,3,4};
-  C = c3;
-}
-
-// CHECK-LABEL: define void {{.*}}arr_assign11
-// CHECK: [[C:%.*]] = alloca [1 x %struct.S], align 4
-// CHECK: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c4, i32 8, i1 false)
-// CHECK-NEXT: ret void
-void arr_assign11() {
-  S s = {1, 2.0};
-  S C[1] = {s};
-  C = c4;
-}

From b95ec24ff08c6ffc300bd85c8b0e53dfda4bceef Mon Sep 17 00:00:00 2001
From: Jim Lin <jim@andestech.com>
Date: Tue, 22 Apr 2025 01:19:56 +0800
Subject: [PATCH 629/710] [SDAG] Handle insert_subvector in isKnownNeverNaN
 (#131989)

Propagate nnan across insert_subvector.
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 28 +++++++++++
 .../RISCV/rvv/fixed-vectors-fmaximum.ll       | 49 +++++++++++++++++++
 .../RISCV/rvv/fixed-vectors-fminimum.ll       | 49 +++++++++++++++++++
 3 files changed, 126 insertions(+)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f502a536d43b1..09d73633462b6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5752,6 +5752,34 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
     }
     return isKnownNeverNaN(Src, SNaN, Depth + 1);
   }
+  case ISD::INSERT_SUBVECTOR: {
+    SDValue BaseVector = Op.getOperand(0);
+    SDValue SubVector = Op.getOperand(1);
+    EVT BaseVectorVT = BaseVector.getValueType();
+    if (BaseVectorVT.isFixedLengthVector()) {
+      unsigned Idx = Op.getConstantOperandVal(2);
+      unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
+      unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
+
+      // Clear/Extract the bits at the position where the subvector will be
+      // inserted.
+      APInt DemandedMask =
+          APInt::getBitsSet(NumBaseElts, Idx, Idx + NumSubElts);
+      APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
+      APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
+
+      bool NeverNaN = true;
+      if (!DemandedSrcElts.isZero())
+        NeverNaN &=
+            isKnownNeverNaN(BaseVector, DemandedSrcElts, SNaN, Depth + 1);
+      if (NeverNaN && !DemandedSubElts.isZero())
+        NeverNaN &=
+            isKnownNeverNaN(SubVector, DemandedSubElts, SNaN, Depth + 1);
+      return NeverNaN;
+    }
+    return isKnownNeverNaN(BaseVector, SNaN, Depth + 1) &&
+           isKnownNeverNaN(SubVector, SNaN, Depth + 1);
+  }
   case ISD::BUILD_VECTOR: {
     unsigned NumElts = Op.getNumOperands();
     for (unsigned I = 0; I != NumElts; ++I)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
index 04e73ac1ea956..c6cd366497218 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmaximum.ll
@@ -357,3 +357,52 @@ define <2 x half> @vfmax_v2f16_vv_nnanb(<2 x half> %a, <2 x half> %b) {
   %v = call <2 x half> @llvm.maximum.v2f16(<2 x half> %a, <2 x half> %c)
   ret <2 x half> %v
 }
+
+declare <4 x half> @llvm.vector.insert.v2f32.v4f32(<4 x half>, <2 x half>, i64)
+
+define <4 x half> @vfmax_v2f16_vv_nnan_insert_subvector(<2 x half> %a, <2 x half> %b, <4 x half> %c) {
+; ZVFH-LABEL: vfmax_v2f16_vv_nnan_insert_subvector:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfadd.vv v8, v8, v8
+; ZVFH-NEXT:    vfadd.vv v9, v9, v9
+; ZVFH-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFH-NEXT:    vslideup.vi v8, v9, 2
+; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
+; ZVFH-NEXT:    vfmax.vv v8, v8, v10
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: vfmax_v2f16_vv_nnan_insert_subvector:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfadd.vv v9, v11, v11
+; ZVFHMIN-NEXT:    vfadd.vv v8, v8, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v11, v9
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v9, v8
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslideup.vi v11, v9, 2
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v11
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v9, v8, v0
+; ZVFHMIN-NEXT:    vfmax.vv v9, v8, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
+; ZVFHMIN-NEXT:    ret
+  %d = fadd nnan <2 x half> %a, %a
+  %e = fadd nnan <2 x half> %b, %b
+  %f = call <4 x half> @llvm.vector.insert.v2f32.v4f32(<4 x half> undef, <2 x half> %d, i64 0)
+  %g = call <4 x half> @llvm.vector.insert.v2f32.v4f32(<4 x half> %f, <2 x half> %e, i64 2)
+  %v = call <4 x half> @llvm.maximum.v4f16(<4 x half> %g, <4 x half> %c)
+  ret <4 x half> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
index a0334a9a5d20a..568923db83591 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fminimum.ll
@@ -357,3 +357,52 @@ define <2 x half> @vfmin_v2f16_vv_nnanb(<2 x half> %a, <2 x half> %b) {
   %v = call <2 x half> @llvm.minimum.v2f16(<2 x half> %a, <2 x half> %c)
   ret <2 x half> %v
 }
+
+declare <4 x half> @llvm.vector.insert.v2f32.v4f32(<4 x half>, <2 x half>, i64)
+
+define <4 x half> @vfmin_v2f16_vv_nnan_insert_subvector(<2 x half> %a, <2 x half> %b, <4 x half> %c) {
+; ZVFH-LABEL: vfmin_v2f16_vv_nnan_insert_subvector:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfadd.vv v8, v8, v8
+; ZVFH-NEXT:    vfadd.vv v9, v9, v9
+; ZVFH-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFH-NEXT:    vmfeq.vv v0, v10, v10
+; ZVFH-NEXT:    vslideup.vi v8, v9, 2
+; ZVFH-NEXT:    vmerge.vvm v8, v10, v8, v0
+; ZVFH-NEXT:    vfmin.vv v8, v8, v10
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: vfmin_v2f16_vv_nnan_insert_subvector:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v11, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfadd.vv v9, v11, v11
+; ZVFHMIN-NEXT:    vfadd.vv v8, v8, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v11, v9
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v9, v8
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslideup.vi v11, v9, 2
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v11
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v8, v8
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFHMIN-NEXT:    vmerge.vvm v10, v8, v9, v0
+; ZVFHMIN-NEXT:    vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT:    vmerge.vvm v8, v9, v8, v0
+; ZVFHMIN-NEXT:    vfmin.vv v9, v8, v10
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
+; ZVFHMIN-NEXT:    ret
+  %d = fadd nnan <2 x half> %a, %a
+  %e = fadd nnan <2 x half> %b, %b
+  %f = call <4 x half> @llvm.vector.insert.v2f32.v4f32(<4 x half> undef, <2 x half> %d, i64 0)
+  %g = call <4 x half> @llvm.vector.insert.v2f32.v4f32(<4 x half> %f, <2 x half> %e, i64 2)
+  %v = call <4 x half> @llvm.minimum.v4f16(<4 x half> %g, <4 x half> %c)
+  ret <4 x half> %v
+}

From c17984711343c6cd16454431000b55627fc338eb Mon Sep 17 00:00:00 2001
From: Bruno Cardoso Lopes <bruno.cardoso@gmail.com>
Date: Mon, 21 Apr 2025 10:25:56 -0700
Subject: [PATCH 630/710] Reapply [MLIR][LLVM] Support for indirectbr (#136378)

Fix msan issue that caused revert in
https://github.com/llvm/llvm-project/pull/135695

### Original message

Now that LLVM dialect has `blockaddress` support, introduce
import/translation for `indirectbr` instruction.
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   |  63 ++++++++++-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    | 103 +++++++++++++++---
 .../LLVMIR/LLVMToLLVMIRTranslation.cpp        |   9 ++
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |  31 +++++-
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  |   8 ++
 .../LLVMIR/blockaddress-canonicalize.mlir     |   2 +-
 mlir/test/Dialect/LLVMIR/indirectbr.mlir      |  83 ++++++++++++++
 .../Target/LLVMIR/Import/import-failure.ll    |  12 --
 mlir/test/Target/LLVMIR/Import/indirectbr.ll  |  86 +++++++++++++++
 mlir/test/Target/LLVMIR/indirectbr.mlir       |  40 +++++++
 .../test/mlir-translate/import-diagnostics.ll |  14 ++-
 11 files changed, 411 insertions(+), 40 deletions(-)
 create mode 100644 mlir/test/Dialect/LLVMIR/indirectbr.mlir
 create mode 100644 mlir/test/Target/LLVMIR/Import/indirectbr.ll
 create mode 100644 mlir/test/Target/LLVMIR/indirectbr.mlir

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 76a2ec47b3a22..5745d370f7268 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1690,8 +1690,7 @@ def LLVM_BlockTagOp : LLVM_Op<"blocktag"> {
     the target address.
 
     A given function should have at most one `llvm.blocktag` operation with a
-    given `tag`. This operation cannot be used as a terminator but can be
-    placed everywhere else in a block.
+    given `tag`. This operation cannot be used as a terminator.
 
     Example:
 
@@ -1711,6 +1710,66 @@ def LLVM_BlockTagOp : LLVM_Op<"blocktag"> {
   let hasVerifier = 0;
 }
 
+//===----------------------------------------------------------------------===//
+// IndirectBrOp
+//===----------------------------------------------------------------------===//
+
+def LLVM_IndirectBrOp : LLVM_TerminatorOp<"indirectbr",
+    [SameVariadicOperandSize, DeclareOpInterfaceMethods<BranchOpInterface>,
+     Pure]> {
+  let description = [{
+    Transfer control flow to address in `$addr`. A list of possible target
+    blocks in `$successors` can be provided and maybe used as a hint in LLVM:
+
+    ```mlir
+    ...
+    llvm.func @g(...
+      %dest = llvm.blockaddress <function = @g, tag = <id = 0>> : !llvm.ptr
+      llvm.indirectbr %dest : !llvm.ptr, [
+        ^head
+      ]
+    ^head:
+      llvm.blocktag <id = 0>
+      llvm.return %arg0 : i32
+      ...
+    ```
+
+    It also supports a list of operands that can be passed to a target block:
+
+    ```mlir
+      llvm.indirectbr %dest : !llvm.ptr, [
+        ^head(%arg0 : i32),
+        ^tail(%arg1, %arg0 : i32, i32)
+      ]
+    ^head(%r0 : i32):
+      llvm.return %r0 : i32
+    ^tail(%r1 : i32, %r2 : i32):
+      ...
+    ```
+  }];
+  let arguments = (ins LLVM_AnyPointer:$addr,
+    VariadicOfVariadic<AnyType, "indbr_operand_segments">:$succOperands,
+    DenseI32ArrayAttr:$indbr_operand_segments
+  );
+  let successors = (successor VariadicSuccessor<AnySuccessor>:$successors);
+  let assemblyFormat = [{
+    $addr `:` type($addr) `,`
+      custom<IndirectBrOpSucessors>(ref(type($addr)),
+                                    $successors,
+                                    $succOperands,
+                                    type($succOperands))
+    attr-dict
+  }];
+
+  let skipDefaultBuilders = 1;
+  let builders = [
+    OpBuilder<(ins "Value":$addr,
+      CArg<"ArrayRef<ValueRange>", "{}">:$succOperands,
+      CArg<"BlockRange", "{}">:$successors
+      )>
+  ];
+}
+
 def LLVM_ComdatSelectorOp : LLVM_Op<"comdat_selector", [Symbol]> {
   let arguments = (ins
     SymbolNameAttr:$sym_name,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index d1d00ca9681ec..0022be84c212e 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -2240,24 +2240,21 @@ static LogicalResult verifyComdat(Operation *op,
 
 static LogicalResult verifyBlockTags(LLVMFuncOp funcOp) {
   llvm::DenseSet<BlockTagAttr> blockTags;
-  BlockTagOp badBlockTagOp;
-  if (funcOp
-          .walk([&](BlockTagOp blockTagOp) {
-            if (blockTags.contains(blockTagOp.getTag())) {
-              badBlockTagOp = blockTagOp;
-              return WalkResult::interrupt();
-            }
-            blockTags.insert(blockTagOp.getTag());
-            return WalkResult::advance();
-          })
-          .wasInterrupted()) {
-    badBlockTagOp.emitError()
-        << "duplicate block tag '" << badBlockTagOp.getTag().getId()
-        << "' in the same function: ";
-    return failure();
-  }
+  // Note that presence of `BlockTagOp`s currently can't prevent an unrecheable
+  // block to be removed by canonicalizer's region simplify pass, which needs to
+  // be dialect aware to allow extra constraints to be described.
+  WalkResult res = funcOp.walk([&](BlockTagOp blockTagOp) {
+    if (blockTags.contains(blockTagOp.getTag())) {
+      blockTagOp.emitError()
+          << "duplicate block tag '" << blockTagOp.getTag().getId()
+          << "' in the same function: ";
+      return WalkResult::interrupt();
+    }
+    blockTags.insert(blockTagOp.getTag());
+    return WalkResult::advance();
+  });
 
-  return success();
+  return failure(res.wasInterrupted());
 }
 
 /// Parse common attributes that might show up in the same order in both
@@ -3818,6 +3815,78 @@ LogicalResult BlockAddressOp::verify() {
 /// attribute.
 OpFoldResult BlockAddressOp::fold(FoldAdaptor) { return getBlockAddr(); }
 
+//===----------------------------------------------------------------------===//
+// LLVM::IndirectBrOp
+//===----------------------------------------------------------------------===//
+
+SuccessorOperands IndirectBrOp::getSuccessorOperands(unsigned index) {
+  assert(index < getNumSuccessors() && "invalid successor index");
+  return SuccessorOperands(getSuccOperandsMutable()[index]);
+}
+
+void IndirectBrOp::build(OpBuilder &odsBuilder, OperationState &odsState,
+                         Value addr, ArrayRef<ValueRange> succOperands,
+                         BlockRange successors) {
+  odsState.addOperands(addr);
+  for (ValueRange range : succOperands)
+    odsState.addOperands(range);
+  SmallVector<int32_t> rangeSegments;
+  for (ValueRange range : succOperands)
+    rangeSegments.push_back(range.size());
+  odsState.getOrAddProperties<Properties>().indbr_operand_segments =
+      odsBuilder.getDenseI32ArrayAttr(rangeSegments);
+  odsState.addSuccessors(successors);
+}
+
+static ParseResult parseIndirectBrOpSucessors(
+    OpAsmParser &parser, Type &flagType,
+    SmallVectorImpl<Block *> &succOperandBlocks,
+    SmallVectorImpl<SmallVector<OpAsmParser::UnresolvedOperand>> &succOperands,
+    SmallVectorImpl<SmallVector<Type>> &succOperandsTypes) {
+  if (failed(parser.parseCommaSeparatedList(
+          OpAsmParser::Delimiter::Square,
+          [&]() {
+            Block *destination = nullptr;
+            SmallVector<OpAsmParser::UnresolvedOperand> operands;
+            SmallVector<Type> operandTypes;
+
+            if (parser.parseSuccessor(destination).failed())
+              return failure();
+
+            if (succeeded(parser.parseOptionalLParen())) {
+              if (failed(parser.parseOperandList(
+                      operands, OpAsmParser::Delimiter::None)) ||
+                  failed(parser.parseColonTypeList(operandTypes)) ||
+                  failed(parser.parseRParen()))
+                return failure();
+            }
+            succOperandBlocks.push_back(destination);
+            succOperands.emplace_back(operands);
+            succOperandsTypes.emplace_back(operandTypes);
+            return success();
+          },
+          "successor blocks")))
+    return failure();
+  return success();
+}
+
+static void
+printIndirectBrOpSucessors(OpAsmPrinter &p, IndirectBrOp op, Type flagType,
+                           SuccessorRange succs, OperandRangeRange succOperands,
+                           const TypeRangeRange &succOperandsTypes) {
+  p << "[";
+  llvm::interleave(
+      llvm::zip(succs, succOperands),
+      [&](auto i) {
+        p.printNewline();
+        p.printSuccessorAndUseList(std::get<0>(i), std::get<1>(i));
+      },
+      [&] { p << ','; });
+  if (!succOperands.empty())
+    p.printNewline();
+  p << "]";
+}
+
 //===----------------------------------------------------------------------===//
 // AssumeOp (intrinsic)
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
index ad48ca2ce8ad3..7038b5d73d266 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
@@ -505,6 +505,15 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
     moduleTranslation.mapBranch(&opInst, switchInst);
     return success();
   }
+  if (auto indBrOp = dyn_cast<LLVM::IndirectBrOp>(opInst)) {
+    llvm::IndirectBrInst *indBr = builder.CreateIndirectBr(
+        moduleTranslation.lookupValue(indBrOp.getAddr()),
+        indBrOp->getNumSuccessors());
+    for (auto *succ : indBrOp.getSuccessors())
+      indBr->addDestination(moduleTranslation.lookupBlock(succ));
+    moduleTranslation.mapBranch(&opInst, indBr);
+    return success();
+  }
 
   // Emit addressof.  We need to look up the global value referenced by the
   // operation and store it in the MLIR-to-LLVM value mapping.  This does not
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 481d719787397..df7c8d6ea3579 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -1988,6 +1988,33 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     return success();
   }
 
+  if (inst->getOpcode() == llvm::Instruction::IndirectBr) {
+    auto *indBrInst = cast<llvm::IndirectBrInst>(inst);
+
+    FailureOr<Value> basePtr = convertValue(indBrInst->getAddress());
+    if (failed(basePtr))
+      return failure();
+
+    SmallVector<Block *> succBlocks;
+    SmallVector<SmallVector<Value>> succBlockArgs;
+    for (auto i : llvm::seq<unsigned>(0, indBrInst->getNumSuccessors())) {
+      llvm::BasicBlock *succ = indBrInst->getSuccessor(i);
+      SmallVector<Value> blockArgs;
+      if (failed(convertBranchArgs(indBrInst, succ, blockArgs)))
+        return failure();
+      succBlocks.push_back(lookupBlock(succ));
+      succBlockArgs.push_back(blockArgs);
+    }
+    SmallVector<ValueRange> succBlockArgsRange =
+        llvm::to_vector_of<ValueRange>(succBlockArgs);
+    Location loc = translateLoc(inst->getDebugLoc());
+    auto indBrOp = builder.create<LLVM::IndirectBrOp>(
+        loc, *basePtr, succBlockArgsRange, succBlocks);
+
+    mapNoResultOp(inst, indBrOp);
+    return success();
+  }
+
   // Convert all instructions that have an mlirBuilder.
   if (succeeded(convertInstructionImpl(builder, inst, *this, iface)))
     return success();
@@ -1998,8 +2025,8 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
 LogicalResult ModuleImport::processInstruction(llvm::Instruction *inst) {
   // FIXME: Support uses of SubtargetData.
   // FIXME: Add support for call / operand attributes.
-  // FIXME: Add support for the indirectbr, cleanupret, catchret, catchswitch,
-  // callbr, vaarg, catchpad, cleanuppad instructions.
+  // FIXME: Add support for the cleanupret, catchret, catchswitch, callbr,
+  // vaarg, catchpad, cleanuppad instructions.
 
   // Convert LLVM intrinsics calls to MLIR intrinsics.
   if (auto *intrinsic = dyn_cast<llvm::IntrinsicInst>(inst))
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index d39ec0e801437..44899c4576a20 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -805,6 +805,14 @@ static Value getPHISourceValue(Block *current, Block *pred,
         return switchOp.getCaseOperands(i.index())[index];
   }
 
+  if (auto indBrOp = dyn_cast<LLVM::IndirectBrOp>(terminator)) {
+    // For indirect branches we take operands for each successor.
+    for (const auto &i : llvm::enumerate(indBrOp->getSuccessors())) {
+      if (indBrOp->getSuccessor(i.index()) == current)
+        return indBrOp.getSuccessorOperands(i.index())[index];
+    }
+  }
+
   if (auto invokeOp = dyn_cast<LLVM::InvokeOp>(terminator)) {
     return invokeOp.getNormalDest() == current
                ? invokeOp.getNormalDestOperands()[index]
diff --git a/mlir/test/Dialect/LLVMIR/blockaddress-canonicalize.mlir b/mlir/test/Dialect/LLVMIR/blockaddress-canonicalize.mlir
index 11dd6f0d97f78..241a2ea2edc66 100644
--- a/mlir/test/Dialect/LLVMIR/blockaddress-canonicalize.mlir
+++ b/mlir/test/Dialect/LLVMIR/blockaddress-canonicalize.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -pass-pipeline='builtin.module(llvm.func(canonicalize{region-simplify=aggressive}))' -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -pass-pipeline='builtin.module(llvm.func(canonicalize{region-simplify=aggressive}))' -verify-diagnostics -split-input-file | FileCheck %s
 
 llvm.mlir.global private @x() {addr_space = 0 : i32, dso_local} : !llvm.ptr {
   %0 = llvm.blockaddress <function = @ba, tag = <id = 2>> : !llvm.ptr
diff --git a/mlir/test/Dialect/LLVMIR/indirectbr.mlir b/mlir/test/Dialect/LLVMIR/indirectbr.mlir
new file mode 100644
index 0000000000000..63bdfea0c3095
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/indirectbr.mlir
@@ -0,0 +1,83 @@
+// RUN: mlir-opt -split-input-file --verify-roundtrip %s | FileCheck %s
+
+llvm.func @ib0(%dest : !llvm.ptr, %arg0 : i32, %arg1 : i32) -> i32 {
+  llvm.indirectbr %dest : !llvm.ptr, [
+  ^head(%arg0 : i32),
+  ^tail(%arg1, %arg0 : i32, i32)
+  ]
+^head(%r0 : i32):
+  llvm.return %r0 : i32
+^tail(%r1 : i32, %r2 : i32):
+  %r = llvm.add %r1, %r2 : i32
+  llvm.return %r : i32
+}
+
+// CHECK: llvm.func @ib0(%[[Addr:.*]]: !llvm.ptr, %[[A0:.*]]: i32, %[[A1:.*]]: i32) -> i32 {
+// CHECK:   llvm.indirectbr %[[Addr]] : !llvm.ptr, [
+// CHECK:   ^bb1(%[[A0:.*]] : i32)
+// CHECK:   ^bb2(%[[A1:.*]], %[[A0]] : i32, i32)
+// CHECK:   ]
+// CHECK: ^bb1(%[[Op0:.*]]: i32):
+// CHECK:   llvm.return %[[Op0]] : i32
+// CHECK: ^bb2(%[[Op1:.*]]: i32, %[[Op2:.*]]: i32):
+// CHECK:   %[[Op3:.*]] = llvm.add %[[Op1]], %[[Op2]] : i32
+// CHECK:   llvm.return %[[Op3]] : i32
+// CHECK: }
+
+// -----
+
+llvm.func @ib1(%dest : !llvm.ptr) {
+  llvm.indirectbr %dest : !llvm.ptr, []
+}
+
+// CHECK: llvm.func @ib1(%[[Addr:.*]]: !llvm.ptr) {
+// CHECK:   llvm.indirectbr %[[Addr]] : !llvm.ptr, []
+// CHECK: }
+
+// -----
+
+// CHECK: llvm.func @test_indirectbr_phi(
+// CHECK-SAME: %arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr, %arg3: i32) -> i32 {
+llvm.func @callee(!llvm.ptr, i32, i32) -> i32
+llvm.func @test_indirectbr_phi(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr, %arg3: i32) -> i32 {
+  %0 = llvm.mlir.undef : i1
+  %1 = llvm.mlir.addressof @test_indirectbr_phi : !llvm.ptr
+  %2 = llvm.blockaddress <function = @test_indirectbr_phi, tag = <id = 1>> : !llvm.ptr
+  // CHECK:   %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : i32
+  %3 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK:   %[[TWO:.*]] = llvm.mlir.constant(2 : i32) : i32
+  %4 = llvm.mlir.constant(2 : i32) : i32
+  %5 = llvm.select %0, %2, %arg0 : i1, !llvm.ptr
+  // CHECK:   llvm.indirectbr {{.*}} : !llvm.ptr, [
+  // CHECK:   ^[[HEAD_BB:.*]],
+  // CHECK:   ^[[TAIL_BB:.*]](%[[ONE]] : i32)
+  // CHECK:   ]
+  llvm.indirectbr %5 : !llvm.ptr, [
+  ^bb1,
+  ^bb2(%3 : i32)
+  ]
+^bb1:
+  // CHECK: ^[[HEAD_BB]]:
+  // CHECK:   llvm.indirectbr {{.*}} : !llvm.ptr, [
+  // CHECK:   ^[[TAIL_BB]](%[[TWO]] : i32),
+  // CHECK:   ^[[END_BB:.*]]
+  // CHECK:   ]
+  %6 = llvm.select %0, %2, %arg0 : i1, !llvm.ptr
+  llvm.indirectbr %6 : !llvm.ptr, [
+  ^bb2(%4 : i32),
+  ^bb3
+  ]
+^bb2(%7: i32):
+  // CHECK: ^[[TAIL_BB]](%[[BLOCK_ARG:.*]]: i32):
+  // CHECK:   {{.*}} = llvm.call @callee({{.*}}, %[[BLOCK_ARG]])
+  // CHECK:   llvm.return
+  %8 = llvm.call @callee(%arg1, %arg3, %7) : (!llvm.ptr, i32, i32) -> i32
+  llvm.return %8 : i32
+^bb3:
+  // CHECK: ^[[END_BB]]:
+  // CHECK:   llvm.blocktag
+  // CHECK:   llvm.return
+  // CHECK: }
+  llvm.blocktag <id = 1>
+  llvm.return %arg3 : i32
+}
diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll
index 4fbf187659a7b..782925a0a938e 100644
--- a/mlir/test/Target/LLVMIR/Import/import-failure.ll
+++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll
@@ -1,17 +1,5 @@
 ; RUN: not mlir-translate -import-llvm -emit-expensive-warnings -split-input-file %s 2>&1 -o /dev/null | FileCheck %s
 
-; CHECK:      <unknown>
-; CHECK-SAME: error: unhandled instruction: indirectbr ptr %dst, [label %bb1, label %bb2]
-define i32 @unhandled_instruction(ptr %dst) {
-  indirectbr ptr %dst, [label %bb1, label %bb2]
-bb1:
-  ret i32 0
-bb2:
-  ret i32 1
-}
-
-; // -----
-
 ; Check that debug intrinsics with an unsupported argument are dropped.
 
 declare void @llvm.dbg.value(metadata, metadata, metadata)
diff --git a/mlir/test/Target/LLVMIR/Import/indirectbr.ll b/mlir/test/Target/LLVMIR/Import/indirectbr.ll
new file mode 100644
index 0000000000000..7c197a5281ddd
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/indirectbr.ll
@@ -0,0 +1,86 @@
+; RUN: mlir-translate --import-llvm %s -split-input-file | FileCheck %s
+
+; CHECK: llvm.func @basic(%[[arg0:.*]]: !llvm.ptr)
+define i32 @basic(ptr %dst) {
+  ; CHECK:   llvm.indirectbr %[[arg0]] : !llvm.ptr, [
+  ; CHECK:   ^[[bb1:.*]],
+  ; CHECK:   ^[[bb2:.*]]
+  ; CHECK:   ]
+  indirectbr ptr %dst, [label %bb1, label %bb2]
+bb1:
+  ; CHECK: ^[[bb1]]:
+  ; CHECK:   llvm.return
+  ret i32 0
+bb2:
+  ; CHECK: ^[[bb2]]:
+  ; CHECK:   llvm.return
+  ret i32 1
+}
+
+; // -----
+
+; CHECK: llvm.mlir.global external @addr()
+@addr = global ptr null
+
+; CHECK-LABEL:  llvm.func @test_indirectbr() {
+define void @test_indirectbr() {
+  ; CHECK:    %[[BA:.*]] = llvm.blockaddress <function = @test_indirectbr, tag = <id = 1>> : !llvm.ptr
+  ; CHECK:    {{.*}} = llvm.mlir.addressof @addr : !llvm.ptr
+  ; CHECK:    llvm.store %[[BA]], {{.*}} : !llvm.ptr, !llvm.ptr
+  store ptr blockaddress(@test_indirectbr, %block), ptr @addr
+  ; CHECK:    %[[TARGET_ADDR:.*]] = llvm.load {{.*}} : !llvm.ptr -> !llvm.ptr
+  %val = load ptr, ptr @addr
+  ; CHECK:    llvm.indirectbr %[[TARGET_ADDR]] : !llvm.ptr, [
+  ; CHECK:    ^[[TARGET_BB:.*]]
+  ; CHECK:    ]
+  indirectbr ptr %val, [label %block]
+  ; CHECK:  ^[[TARGET_BB]]:
+  ; CHECK:    llvm.blocktag <id = 1>
+  ; CHECK:    llvm.return
+  ; CHECK:  }
+block:
+  ret void
+}
+
+; // -----
+
+; CHECK: llvm.func @callee(!llvm.ptr, i32, i32) -> i32
+declare i32 @callee(ptr %a, i32 %v, i32 %p)
+
+; CHECK: llvm.func @test_indirectbr_phi(
+; CHECK-SAME: %arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr, %arg3: i32) -> i32 {
+define i32 @test_indirectbr_phi(ptr %address, ptr %a, ptr %b, i32 %v) {
+entry:
+  ; CHECK:   %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : i32
+  ; CHECK:   %[[TWO:.*]] = llvm.mlir.constant(2 : i32) : i32
+  %dest = select i1 poison, ptr blockaddress(@test_indirectbr_phi, %end), ptr %address
+  ; CHECK:   llvm.indirectbr {{.*}} : !llvm.ptr, [
+  ; CHECK:   ^[[HEAD_BB:.*]],
+  ; CHECK:   ^[[TAIL_BB:.*]](%[[ONE]] : i32)
+  ; CHECK:   ]
+  indirectbr ptr %dest, [label %head, label %tail]
+
+head:
+  ; CHECK: ^[[HEAD_BB]]:
+  ; CHECK:   llvm.indirectbr {{.*}} : !llvm.ptr, [
+  ; CHECK:   ^[[TAIL_BB]](%[[TWO]] : i32),
+  ; CHECK:   ^[[END_BB:.*]]
+  ; CHECK:   ]
+  %dest2 = select i1 poison, ptr blockaddress(@test_indirectbr_phi, %end), ptr %address
+  indirectbr ptr %dest2, [label %tail, label %end]
+
+tail:
+  ; CHECK: ^[[TAIL_BB]](%[[BLOCK_ARG:.*]]: i32):
+  ; CHECK:   {{.*}} = llvm.call @callee({{.*}}, %[[BLOCK_ARG]])
+  ; CHECK:   llvm.return
+  %p = phi i32 [1, %entry], [2, %head]
+  %r = call i32 @callee(ptr %a, i32 %v, i32 %p)
+  ret i32 %r
+
+end:
+  ; CHECK: ^[[END_BB]]:
+  ; CHECK:   llvm.blocktag
+  ; CHECK:   llvm.return
+  ; CHECK: }
+  ret i32 %v
+}
diff --git a/mlir/test/Target/LLVMIR/indirectbr.mlir b/mlir/test/Target/LLVMIR/indirectbr.mlir
new file mode 100644
index 0000000000000..538220721e9eb
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/indirectbr.mlir
@@ -0,0 +1,40 @@
+// RUN: mlir-translate -mlir-to-llvmir %s -split-input-file | FileCheck %s
+
+llvm.func @callee(!llvm.ptr, i32, i32) -> i32
+
+// CHECK: define i32 @test_indirectbr_phi(ptr %[[IN_PTR:.*]], ptr %[[ARG1:.*]], i32 %[[ARG2:.*]]) {
+llvm.func @test_indirectbr_phi(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg3: i32) -> i32 {
+  %0 = llvm.mlir.undef : i1
+  %2 = llvm.blockaddress <function = @test_indirectbr_phi, tag = <id = 1>> : !llvm.ptr
+  %3 = llvm.mlir.constant(1 : i32) : i32
+  %4 = llvm.mlir.constant(2 : i32) : i32
+  %5 = llvm.select %0, %2, %arg0 : i1, !llvm.ptr
+  // CHECK:   %[[BA0:.*]] = select i1 undef, ptr blockaddress(@test_indirectbr_phi, %[[RET_BB:.*]]), ptr %[[IN_PTR]]
+  // CHECK:   indirectbr ptr %[[BA0]], [label %[[BB1:.*]], label %[[BB2:.*]]]
+  llvm.indirectbr %5 : !llvm.ptr, [
+  ^bb1,
+  ^bb2(%3 : i32)
+  ]
+^bb1:
+  // CHECK: [[BB1]]:
+  // CHECK:   %[[BA1:.*]] = select i1 undef, ptr blockaddress(@test_indirectbr_phi, %[[RET_BB]]), ptr %[[IN_PTR]]
+  // CHECK:   indirectbr ptr %[[BA1]], [label %[[BB2]], label %[[RET_BB]]]
+  %6 = llvm.select %0, %2, %arg0 : i1, !llvm.ptr
+  llvm.indirectbr %6 : !llvm.ptr, [
+  ^bb2(%4 : i32),
+  ^bb3
+  ]
+^bb2(%7: i32):
+  // CHECK: [[BB2]]:
+  // CHECK:   %[[PHI:.*]] = phi i32 [ 2, %[[BB1]] ], [ 1, {{.*}} ]
+  // CHECK:   %[[CALL:.*]] = call i32 @callee(ptr %[[ARG1]], i32 %[[ARG2]], i32 %[[PHI]])
+  // CHECK:   ret i32 %[[CALL]]
+  %8 = llvm.call @callee(%arg1, %arg3, %7) : (!llvm.ptr, i32, i32) -> i32
+  llvm.return %8 : i32
+^bb3:
+  // CHECK: [[RET_BB]]:
+  // CHECK:   ret i32 %[[ARG2]]
+  // CHECK: }
+  llvm.blocktag <id = 1>
+  llvm.return %arg3 : i32
+}
diff --git a/mlir/test/mlir-translate/import-diagnostics.ll b/mlir/test/mlir-translate/import-diagnostics.ll
index acf45eb1c7c5d..6767d948e3bbd 100644
--- a/mlir/test/mlir-translate/import-diagnostics.ll
+++ b/mlir/test/mlir-translate/import-diagnostics.ll
@@ -21,14 +21,16 @@ end:
 ; ERROR:     error:
 ; DEFAULT:   error:
 ; EXPENSIVE: error:
-define i32 @error(ptr %dst) {
-  indirectbr ptr %dst, [label %bb1, label %bb2]
-bb1:
-  ret i32 0
-bb2:
-  ret i32 1
+define dso_local void @tbaa(ptr %0) {
+  store i32 1, ptr %0, align 4, !tbaa !2
+  ret void
 }
 
+!2 = !{!3, !3, i64 0, i64 4}
+!3 = !{!4, i64 4, !"int"}
+!4 = !{!5, i64 1, !"omnipotent char"}
+!5 = !{!"Simple C++ TBAA"}
+
 ; // -----
 
 declare void @llvm.dbg.value(metadata, metadata, metadata)

From f02b285dbd284057d5f1dd5fb599d1c2155670f5 Mon Sep 17 00:00:00 2001
From: Sarah Spall <sarahspall@microsoft.com>
Date: Mon, 21 Apr 2025 10:33:06 -0700
Subject: [PATCH 631/710] [HLSL] Reland; Make it possible to assign an array
 from a cbuffer (#136580)

Reland #134174
Update Sema Checking to always do an HLSL Array RValue cast in the case
we are dealing with hlsl constant array types
Instead of comparing canonical types, compare canonical unqualified
types
Add a test to show it is possible to assign an array from a cbuffer.
Closes https://github.com/llvm/llvm-project/issues/133767
---
 clang/lib/Sema/SemaExprCXX.cpp              | 14 ++---
 clang/lib/Sema/SemaOverload.cpp             | 11 ++--
 clang/test/CodeGenHLSL/ArrayAssignable.hlsl | 63 ++++++++++++++++++++-
 3 files changed, 71 insertions(+), 17 deletions(-)

diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index d46322d75721e..f5a10e0db85ad 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -4745,19 +4745,13 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
   case ICK_HLSL_Array_RValue:
     if (ToType->isArrayParameterType()) {
       FromType = Context.getArrayParameterType(FromType);
-      From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
-                               /*BasePath=*/nullptr, CCK)
-                 .get();
-    } else { // FromType must be ArrayParameterType
-      assert(FromType->isArrayParameterType() &&
-             "FromType must be ArrayParameterType in ICK_HLSL_Array_RValue \
-              if it is not ToType");
+    } else if (FromType->isArrayParameterType()) {
       const ArrayParameterType *APT = cast<ArrayParameterType>(FromType);
       FromType = APT->getConstantArrayType(Context);
-      From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
-                               /*BasePath=*/nullptr, CCK)
-                 .get();
     }
+    From = ImpCastExprToType(From, FromType, CK_HLSLArrayRValue, VK_PRValue,
+                             /*BasePath=*/nullptr, CCK)
+               .get();
     break;
 
   case ICK_Function_To_Pointer:
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index e4ff8c5489df3..5b224b6c08fef 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -2275,17 +2275,16 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
     // handling here.
     if (ToType->isArrayParameterType()) {
       FromType = S.Context.getArrayParameterType(FromType);
-      SCS.First = ICK_HLSL_Array_RValue;
     } else if (FromType->isArrayParameterType()) {
       const ArrayParameterType *APT = cast<ArrayParameterType>(FromType);
       FromType = APT->getConstantArrayType(S.Context);
-      SCS.First = ICK_HLSL_Array_RValue;
-    } else {
-      SCS.First = ICK_Identity;
     }
 
-    if (S.Context.getCanonicalType(FromType) !=
-        S.Context.getCanonicalType(ToType))
+    SCS.First = ICK_HLSL_Array_RValue;
+
+    // Don't consider qualifiers, which include things like address spaces
+    if (FromType.getCanonicalType().getUnqualifiedType() !=
+        ToType.getCanonicalType().getUnqualifiedType())
       return false;
 
     SCS.setAllToTypes(ToType);
diff --git a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
index e2ff2de68ed99..6374f91230546 100644
--- a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
+++ b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
@@ -1,4 +1,23 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --enable-var-scope
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -finclude-default-header -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+struct S {
+  int x;
+  float f;
+};
+
+// CHECK: [[CBLayout:%.*]] = type <{ [2 x float], [2 x <4 x i32>], [2 x [2 x i32]], [1 x target("dx.Layout", %S, 8, 0, 4)] }>
+// CHECK: @CBArrays.cb = global target("dx.CBuffer", target("dx.Layout", [[CBLayout]], 136, 0, 32, 64, 128))
+// CHECK: @c1 = external addrspace(2) global [2 x float], align 4
+// CHECK: @c2 = external addrspace(2) global [2 x <4 x i32>], align 16
+// CHECK: @c3 = external addrspace(2) global [2 x [2 x i32]], align 4
+// CHECK: @c4 = external addrspace(2) global [1 x target("dx.Layout", %S, 8, 0, 4)], align 4
+
+cbuffer CBArrays : register(b0) {
+  float c1[2];
+  int4 c2[2];
+  int c3[2][2];
+  S c4[1];
+}
 
 // CHECK-LABEL: define void {{.*}}arr_assign1
 // CHECK: [[Arr:%.*]] = alloca [2 x i32], align 4
@@ -116,3 +135,45 @@ void arr_assign7() {
   int Arr2[2][2] = {{0, 0}, {1, 1}};
   (Arr = Arr2)[0] = {6, 6};
 }
+
+// Verify you can assign from a cbuffer array
+
+// CHECK-LABEL: define void {{.*}}arr_assign8
+// CHECK: [[C:%.*]] = alloca [2 x float], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[C]], ptr align 4 {{.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c1, i32 8, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign8() {
+  float C[2] = {1.0, 2.0};
+  C = c1;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign9
+// CHECK: [[C:%.*]] = alloca [2 x <4 x i32>], align 16
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[C]], ptr align 16 {{.*}}, i32 32, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 16 [[C]], ptr addrspace(2) align 16 @c2, i32 32, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign9() {
+  int4 C[2] = {1,2,3,4,5,6,7,8};
+  C = c2;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign10
+// CHECK: [[C:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[C]], ptr align 4 {{.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c3, i32 16, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign10() {
+  int C[2][2] = {1,2,3,4};
+  C = c3;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign11
+// CHECK: [[C:%.*]] = alloca [1 x %struct.S], align 4
+// CHECK: call void @llvm.memcpy.p0.p2.i32(ptr align 4 [[C]], ptr addrspace(2) align 4 @c4, i32 8, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign11() {
+  S s = {1, 2.0};
+  S C[1] = {s};
+  C = c4;
+}

From 5739a22fbb593eb44490d23e0c8cc12386e4b5b6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 21 Apr 2025 18:36:43 +0100
Subject: [PATCH 632/710] [VPlan] Also duplicated scalar-steps when it enables
 sinking scalars. (#136021)

Extend sinking logic to duplicate scalar steps recipe if it enables
sinking, that is if all users in a destination block require all lanes.

This should be the last step before removing legacy sinkScalarOperands.

PR: https://github.com/llvm/llvm-project/pull/136021
---
 .../Transforms/Vectorize/VPlanTransforms.cpp  |  19 ++-
 .../LoopVectorize/AArch64/blend-costs.ll      |  38 +++---
 .../LoopVectorize/RISCV/uniform-load-store.ll |  12 +-
 .../LoopVectorize/X86/cost-model.ll           |  17 +--
 .../test/Transforms/LoopVectorize/debugloc.ll |   6 +-
 ...able-info-from-assumption-constant-size.ll |  30 ++---
 .../LoopVectorize/float-induction.ll          |  18 +--
 .../LoopVectorize/if-pred-stores.ll           | 100 ++++++++--------
 .../Transforms/LoopVectorize/induction.ll     | 112 +++++++++---------
 .../LoopVectorize/load-deref-pred-align.ll    |  92 +++++++-------
 .../load-of-struct-deref-pred.ll              |  88 +++++++-------
 .../Transforms/LoopVectorize/loop-form.ll     |   4 +-
 .../LoopVectorize/select-cmp-multiuse.ll      |   8 +-
 .../LoopVectorize/select-cmp-predicated.ll    |  10 +-
 .../Transforms/LoopVectorize/struct-return.ll |   2 +-
 .../trip-count-expansion-may-introduce-ub.ll  |  12 +-
 .../LoopVectorize/vplan-printing.ll           |   3 +-
 .../vplan-sink-scalars-and-merge.ll           |   3 +-
 18 files changed, 295 insertions(+), 279 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b80fe18d1bd66..1d3e88037ea43 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -164,8 +164,10 @@ static bool sinkScalarOperands(VPlan &Plan) {
       if (UI->getParent() == SinkTo)
         return true;
       NeedsDuplicating = UI->onlyFirstLaneUsed(SinkCandidate);
-      // We only know how to duplicate VPRecipeRecipes for now.
-      return NeedsDuplicating && isa<VPReplicateRecipe>(SinkCandidate);
+      // We only know how to duplicate VPReplicateRecipes and
+      // VPScalarIVStepsRecipes for now.
+      return NeedsDuplicating &&
+             isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(SinkCandidate);
     };
     if (!all_of(SinkCandidate->users(), CanSinkWithUser))
       continue;
@@ -173,9 +175,16 @@ static bool sinkScalarOperands(VPlan &Plan) {
     if (NeedsDuplicating) {
       if (ScalarVFOnly)
         continue;
-      Instruction *I = SinkCandidate->getUnderlyingInstr();
-      auto *Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true);
-      // TODO: add ".cloned" suffix to name of Clone's VPValue.
+      VPSingleDefRecipe *Clone;
+      if (isa<VPReplicateRecipe>(SinkCandidate)) {
+        // TODO: Handle converting to uniform recipes as separate transform,
+        // then cloning should be sufficient here.
+        Instruction *I = SinkCandidate->getUnderlyingInstr();
+        Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true);
+        // TODO: add ".cloned" suffix to name of Clone's VPValue.
+      } else {
+        Clone = SinkCandidate->clone();
+      }
 
       Clone->insertBefore(SinkCandidate);
       SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
index e302bf195ef8e..3c8bbaa46f275 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
@@ -211,8 +211,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor <16 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE30:.*]] ]
-; CHECK-NEXT:    [[IV:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE30:.*]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[IV]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[GEP_SRC]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
@@ -224,7 +223,8 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x i1> [[TMP7]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT:    [[TMP72:%.*]] = add i32 [[IV]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP72]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 0
 ; CHECK-NEXT:    store i8 [[TMP10]], ptr [[TMP9]], align 1
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
@@ -232,7 +232,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x i1> [[TMP7]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
 ; CHECK:       [[PRED_STORE_IF1]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 1
 ; CHECK-NEXT:    store i8 [[TMP14]], ptr [[TMP13]], align 1
@@ -241,7 +241,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i1> [[TMP7]], i32 2
 ; CHECK-NEXT:    br i1 [[TMP15]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
 ; CHECK:       [[PRED_STORE_IF3]]:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[IV]], 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 2
 ; CHECK-NEXT:    store i8 [[TMP18]], ptr [[TMP17]], align 1
@@ -250,7 +250,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i1> [[TMP7]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP19]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
 ; CHECK:       [[PRED_STORE_IF5]]:
-; CHECK-NEXT:    [[TMP20:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = add i32 [[IV]], 3
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP20]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 3
 ; CHECK-NEXT:    store i8 [[TMP22]], ptr [[TMP21]], align 1
@@ -259,7 +259,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i1> [[TMP7]], i32 4
 ; CHECK-NEXT:    br i1 [[TMP23]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
 ; CHECK:       [[PRED_STORE_IF7]]:
-; CHECK-NEXT:    [[TMP24:%.*]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP24:%.*]] = add i32 [[IV]], 4
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP24]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 4
 ; CHECK-NEXT:    store i8 [[TMP26]], ptr [[TMP25]], align 1
@@ -268,7 +268,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i1> [[TMP7]], i32 5
 ; CHECK-NEXT:    br i1 [[TMP27]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
 ; CHECK:       [[PRED_STORE_IF9]]:
-; CHECK-NEXT:    [[TMP28:%.*]] = add i32 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP28:%.*]] = add i32 [[IV]], 5
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP28]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 5
 ; CHECK-NEXT:    store i8 [[TMP30]], ptr [[TMP29]], align 1
@@ -277,7 +277,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i1> [[TMP7]], i32 6
 ; CHECK-NEXT:    br i1 [[TMP31]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
 ; CHECK:       [[PRED_STORE_IF11]]:
-; CHECK-NEXT:    [[TMP32:%.*]] = add i32 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP32:%.*]] = add i32 [[IV]], 6
 ; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP32]]
 ; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 6
 ; CHECK-NEXT:    store i8 [[TMP34]], ptr [[TMP33]], align 1
@@ -286,7 +286,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i1> [[TMP7]], i32 7
 ; CHECK-NEXT:    br i1 [[TMP35]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
 ; CHECK:       [[PRED_STORE_IF13]]:
-; CHECK-NEXT:    [[TMP36:%.*]] = add i32 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = add i32 [[IV]], 7
 ; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 7
 ; CHECK-NEXT:    store i8 [[TMP38]], ptr [[TMP37]], align 1
@@ -295,7 +295,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i1> [[TMP7]], i32 8
 ; CHECK-NEXT:    br i1 [[TMP39]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
 ; CHECK:       [[PRED_STORE_IF15]]:
-; CHECK-NEXT:    [[TMP40:%.*]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP40:%.*]] = add i32 [[IV]], 8
 ; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP40]]
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 8
 ; CHECK-NEXT:    store i8 [[TMP42]], ptr [[TMP41]], align 1
@@ -304,7 +304,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <16 x i1> [[TMP7]], i32 9
 ; CHECK-NEXT:    br i1 [[TMP43]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
 ; CHECK:       [[PRED_STORE_IF17]]:
-; CHECK-NEXT:    [[TMP44:%.*]] = add i32 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP44:%.*]] = add i32 [[IV]], 9
 ; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP44]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 9
 ; CHECK-NEXT:    store i8 [[TMP46]], ptr [[TMP45]], align 1
@@ -313,7 +313,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <16 x i1> [[TMP7]], i32 10
 ; CHECK-NEXT:    br i1 [[TMP47]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
 ; CHECK:       [[PRED_STORE_IF19]]:
-; CHECK-NEXT:    [[TMP48:%.*]] = add i32 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP48:%.*]] = add i32 [[IV]], 10
 ; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP48]]
 ; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 10
 ; CHECK-NEXT:    store i8 [[TMP50]], ptr [[TMP49]], align 1
@@ -322,7 +322,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <16 x i1> [[TMP7]], i32 11
 ; CHECK-NEXT:    br i1 [[TMP51]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
 ; CHECK:       [[PRED_STORE_IF21]]:
-; CHECK-NEXT:    [[TMP52:%.*]] = add i32 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP52:%.*]] = add i32 [[IV]], 11
 ; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP52]]
 ; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 11
 ; CHECK-NEXT:    store i8 [[TMP54]], ptr [[TMP53]], align 1
@@ -331,7 +331,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <16 x i1> [[TMP7]], i32 12
 ; CHECK-NEXT:    br i1 [[TMP55]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
 ; CHECK:       [[PRED_STORE_IF23]]:
-; CHECK-NEXT:    [[TMP56:%.*]] = add i32 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP56:%.*]] = add i32 [[IV]], 12
 ; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP56]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 12
 ; CHECK-NEXT:    store i8 [[TMP58]], ptr [[TMP57]], align 1
@@ -340,7 +340,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <16 x i1> [[TMP7]], i32 13
 ; CHECK-NEXT:    br i1 [[TMP59]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
 ; CHECK:       [[PRED_STORE_IF25]]:
-; CHECK-NEXT:    [[TMP60:%.*]] = add i32 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP60:%.*]] = add i32 [[IV]], 13
 ; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP60]]
 ; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 13
 ; CHECK-NEXT:    store i8 [[TMP62]], ptr [[TMP61]], align 1
@@ -349,7 +349,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP63:%.*]] = extractelement <16 x i1> [[TMP7]], i32 14
 ; CHECK-NEXT:    br i1 [[TMP63]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
 ; CHECK:       [[PRED_STORE_IF27]]:
-; CHECK-NEXT:    [[TMP64:%.*]] = add i32 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP64:%.*]] = add i32 [[IV]], 14
 ; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP64]]
 ; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 14
 ; CHECK-NEXT:    store i8 [[TMP66]], ptr [[TMP65]], align 1
@@ -358,13 +358,13 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
 ; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <16 x i1> [[TMP7]], i32 15
 ; CHECK-NEXT:    br i1 [[TMP67]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30]]
 ; CHECK:       [[PRED_STORE_IF29]]:
-; CHECK-NEXT:    [[TMP68:%.*]] = add i32 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP68:%.*]] = add i32 [[IV]], 15
 ; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP68]]
 ; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 15
 ; CHECK-NEXT:    store i8 [[TMP70]], ptr [[TMP69]], align 1
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE30]]
 ; CHECK:       [[PRED_STORE_CONTINUE30]]:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[IV]], 16
 ; CHECK-NEXT:    [[TMP71:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
 ; CHECK-NEXT:    br i1 [[TMP71]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index d8713bdda689a..827612cfe36d5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -1045,12 +1045,12 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; TF-FIXEDLEN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TF-FIXEDLEN:       [[VECTOR_BODY]]:
 ; TF-FIXEDLEN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
-; TF-FIXEDLEN-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; TF-FIXEDLEN-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[TMP0]], i64 1025)
-; TF-FIXEDLEN-NEXT:    [[TMP1:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 0
-; TF-FIXEDLEN-NEXT:    br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TF-FIXEDLEN-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 1025)
+; TF-FIXEDLEN-NEXT:    [[TMP0:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 0
+; TF-FIXEDLEN-NEXT:    br i1 [[TMP0]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; TF-FIXEDLEN:       [[PRED_STORE_IF]]:
-; TF-FIXEDLEN-NEXT:    store i64 [[TMP0]], ptr [[B]], align 8
+; TF-FIXEDLEN-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
+; TF-FIXEDLEN-NEXT:    store i64 [[TMP1]], ptr [[B]], align 8
 ; TF-FIXEDLEN-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; TF-FIXEDLEN:       [[PRED_STORE_CONTINUE]]:
 ; TF-FIXEDLEN-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 1
@@ -1074,7 +1074,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; TF-FIXEDLEN-NEXT:    store i64 [[TMP7]], ptr [[B]], align 8
 ; TF-FIXEDLEN-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
 ; TF-FIXEDLEN:       [[PRED_STORE_CONTINUE6]]:
-; TF-FIXEDLEN-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; TF-FIXEDLEN-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TF-FIXEDLEN-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
 ; TF-FIXEDLEN-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP9]], i32 8, <4 x i1> [[ACTIVE_LANE_MASK]])
 ; TF-FIXEDLEN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index a1c727f62ba7a..ceedb787054a0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -603,11 +603,10 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE37:%.*]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 12
-; CHECK-NEXT:    [[TMP7:%.*]] = shl nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = shl nsw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl nsw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = shl nsw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = shl nsw i64 [[TMP6]], 2
@@ -630,6 +629,7 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i1> [[TMP19]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP24:%.*]] = shl nsw i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP24]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP25]], align 8
@@ -665,7 +665,8 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <4 x i1> [[TMP20]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP38]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]]
 ; CHECK:       pred.store.if14:
-; CHECK-NEXT:    [[TMP39:%.*]] = shl nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP88:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP39:%.*]] = shl nsw i64 [[TMP88]], 2
 ; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP39]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP40]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE15]]
@@ -700,7 +701,8 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <4 x i1> [[TMP21]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP53]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]]
 ; CHECK:       pred.store.if22:
-; CHECK-NEXT:    [[TMP54:%.*]] = shl nsw i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP107:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP54:%.*]] = shl nsw i64 [[TMP107]], 2
 ; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP54]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP55]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE23]]
@@ -735,7 +737,8 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <4 x i1> [[TMP22]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP68]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31:%.*]]
 ; CHECK:       pred.store.if30:
-; CHECK-NEXT:    [[TMP69:%.*]] = shl nsw i64 [[TMP6]], 2
+; CHECK-NEXT:    [[TMP108:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP69:%.*]] = shl nsw i64 [[TMP108]], 2
 ; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP69]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP70]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE31]]
@@ -785,8 +788,7 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX40:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL1]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT51:%.*]], [[PRED_STORE_CONTINUE50:%.*]] ]
-; CHECK-NEXT:    [[TMP86:%.*]] = add i64 [[INDEX40]], 0
-; CHECK-NEXT:    [[TMP87:%.*]] = shl nsw i64 [[TMP86]], 2
+; CHECK-NEXT:    [[TMP87:%.*]] = shl nsw i64 [[INDEX40]], 2
 ; CHECK-NEXT:    [[TMP89:%.*]] = getelementptr nusw double, ptr [[A]], i64 [[TMP87]]
 ; CHECK-NEXT:    [[WIDE_VEC41:%.*]] = load <16 x double>, ptr [[TMP89]], align 8
 ; CHECK-NEXT:    [[STRIDED_VEC42:%.*]] = shufflevector <16 x double> [[WIDE_VEC41]], <16 x double> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
@@ -794,6 +796,7 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
 ; CHECK-NEXT:    [[TMP91:%.*]] = extractelement <4 x i1> [[TMP90]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP91]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]]
 ; CHECK:       pred.store.if43:
+; CHECK-NEXT:    [[TMP86:%.*]] = add i64 [[INDEX40]], 0
 ; CHECK-NEXT:    [[TMP92:%.*]] = shl nsw i64 [[TMP86]], 2
 ; CHECK-NEXT:    [[TMP93:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP92]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP93]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/debugloc.ll b/llvm/test/Transforms/LoopVectorize/debugloc.ll
index 8fe355c6d567d..f3add22d701d0 100644
--- a/llvm/test/Transforms/LoopVectorize/debugloc.ll
+++ b/llvm/test/Transforms/LoopVectorize/debugloc.ll
@@ -67,7 +67,8 @@ define i32 @test_debug_loc_on_branch_in_loop(ptr noalias %src, ptr noalias %dst)
 ; CHECK-NEXT:   br i1 [[EXT]], label %pred.store.if, label %pred.store.continue, !dbg [[LOC3]]
 ; CHECK-EMPTY:
 ; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT:   [[GEP:%.+]] = getelementptr inbounds i32, ptr %dst, i64 {{.+}}
+; CHECK-NEXT:   [[IDX:%.+]] = add i64 %index, 0
+; CHECK-NEXT:   [[GEP:%.+]] = getelementptr inbounds i32, ptr %dst, i64 [[IDX]]
 ; CHECK-NEXT:   store i32 0, ptr [[GEP]], align 4
 ; CHECK-NEXT:   br label %pred.store.continue, !dbg [[LOC3]]
 ; CHECK-EMPTY:
@@ -106,7 +107,8 @@ define i32 @test_different_debug_loc_on_replicate_recipe(ptr noalias %src, ptr n
 ; CHECK-NEXT:   br i1 [[EXT]], label %pred.store.if, label %pred.store.continue, !dbg [[LOC4]]
 ; CHECK-EMPTY:
 ; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT:   [[GEP:%.+]] = getelementptr inbounds i32, ptr %dst, i64 {{.+}}, !dbg [[LOC5:!.+]]
+; CHECK-NEXT:   [[IDX:%.+]] = add i64 %index, 0
+; CHECK-NEXT:   [[GEP:%.+]] = getelementptr inbounds i32, ptr %dst, i64 [[IDX]], !dbg [[LOC5:!.+]]
 ; CHECK-NEXT:   store i32 0, ptr [[GEP]], align 4
 ; CHECK-NEXT:   br label %pred.store.continue, !dbg [[LOC4]]
 ; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
index eb688f86d84ee..dfae2d3f41d48 100644
--- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
@@ -967,8 +967,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
@@ -977,7 +976,8 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
@@ -986,7 +986,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 1
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
@@ -997,7 +997,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP17]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -1217,8 +1217,7 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
@@ -1227,7 +1226,8 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
@@ -1236,7 +1236,7 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
@@ -1247,7 +1247,7 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP17]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -1312,8 +1312,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
@@ -1322,7 +1321,8 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
@@ -1331,7 +1331,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
@@ -1342,7 +1342,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP17]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/float-induction.ll b/llvm/test/Transforms/LoopVectorize/float-induction.ll
index 1c6f5de7b05df..fb6e6be7ab22f 100644
--- a/llvm/test/Transforms/LoopVectorize/float-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/float-induction.ll
@@ -1483,11 +1483,11 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC4_INTERL2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; VEC4_INTERL2:       scalar.ph:
 ; VEC4_INTERL2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC4_INTERL2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi float [ [[DOTCAST]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; VEC4_INTERL2-NEXT:    [[BC_RESUME_VAL17:%.*]] = phi float [ [[DOTCAST]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; VEC4_INTERL2-NEXT:    br label [[FOR_BODY:%.*]]
 ; VEC4_INTERL2:       for.body:
 ; VEC4_INTERL2-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC4_INTERL2-NEXT:    [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC4_INTERL2-NEXT:    [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL17]], [[SCALAR_PH]] ]
 ; VEC4_INTERL2-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[I]]
 ; VEC4_INTERL2-NEXT:    [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
 ; VEC4_INTERL2-NEXT:    [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
@@ -1514,22 +1514,22 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC1_INTERL2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC1_INTERL2:       vector.body:
 ; VEC1_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
-; VEC1_INTERL2-NEXT:    [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
-; VEC1_INTERL2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT:    [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; VEC1_INTERL2-NEXT:    [[TMP1:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
 ; VEC1_INTERL2-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 4
-; VEC1_INTERL2-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP9]], align 4
+; VEC1_INTERL2-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP0]], align 4
 ; VEC1_INTERL2-NEXT:    [[TMP4:%.*]] = load float, ptr [[TMP2]], align 4
 ; VEC1_INTERL2-NEXT:    [[TMP5:%.*]] = fcmp fast oeq float [[TMP3]], 0.000000e+00
 ; VEC1_INTERL2-NEXT:    [[TMP6:%.*]] = fcmp fast oeq float [[TMP4]], 0.000000e+00
 ; VEC1_INTERL2-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; VEC1_INTERL2:       pred.store.if:
-; VEC1_INTERL2-NEXT:    store float [[DOTCAST2]], ptr [[TMP9]], align 4
+; VEC1_INTERL2-NEXT:    store float [[DOTCAST1]], ptr [[TMP0]], align 4
 ; VEC1_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VEC1_INTERL2:       pred.store.continue:
 ; VEC1_INTERL2-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
 ; VEC1_INTERL2:       pred.store.if2:
-; VEC1_INTERL2-NEXT:    [[TMP7:%.*]] = fadd fast float [[DOTCAST2]], 1.000000e+00
+; VEC1_INTERL2-NEXT:    [[TMP7:%.*]] = fadd fast float [[DOTCAST1]], 1.000000e+00
 ; VEC1_INTERL2-NEXT:    store float [[TMP7]], ptr [[TMP2]], align 4
 ; VEC1_INTERL2-NEXT:    br label [[PRED_STORE_CONTINUE3]]
 ; VEC1_INTERL2:       pred.store.continue3:
@@ -1541,11 +1541,11 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) {
 ; VEC1_INTERL2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; VEC1_INTERL2:       scalar.ph:
 ; VEC1_INTERL2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC1_INTERL2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi float [ [[DOTCAST]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; VEC1_INTERL2-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi float [ [[DOTCAST]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; VEC1_INTERL2-NEXT:    br label [[FOR_BODY:%.*]]
 ; VEC1_INTERL2:       for.body:
 ; VEC1_INTERL2-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; VEC1_INTERL2-NEXT:    [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; VEC1_INTERL2-NEXT:    [[J:%.*]] = phi float [ [[J_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
 ; VEC1_INTERL2-NEXT:    [[VAR0:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[I]]
 ; VEC1_INTERL2-NEXT:    [[VAR1:%.*]] = load float, ptr [[VAR0]], align 4
 ; VEC1_INTERL2-NEXT:    [[VAR2:%.*]] = fcmp fast oeq float [[VAR1]], 0.000000e+00
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 7ac0eb038cb7a..94e4d83ac82d6 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -93,33 +93,33 @@ define i32 @test(ptr nocapture %f) #0 {
 ; VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC:       vector.body:
 ; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; VEC-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[F:%.*]], i64 [[TMP0]]
+; VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[F:%.*]], i64 [[INDEX]]
 ; VEC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
 ; VEC-NEXT:    [[TMP3:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], splat (i32 100)
 ; VEC-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
 ; VEC-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; VEC:       pred.store.if:
-; VEC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP0]]
-; VEC-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
-; VEC-NEXT:    [[TMP7:%.*]] = add nsw i32 [[TMP6]], 20
-; VEC-NEXT:    store i32 [[TMP7]], ptr [[TMP5]], align 4
+; VEC-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
+; VEC-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP5]]
+; VEC-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
+; VEC-NEXT:    [[TMP8:%.*]] = add nsw i32 [[TMP7]], 20
+; VEC-NEXT:    store i32 [[TMP8]], ptr [[TMP6]], align 4
 ; VEC-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VEC:       pred.store.continue:
-; VEC-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
-; VEC-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
+; VEC-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
+; VEC-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.if1:
-; VEC-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP9]]
-; VEC-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
-; VEC-NEXT:    [[TMP12:%.*]] = add nsw i32 [[TMP11]], 20
-; VEC-NEXT:    store i32 [[TMP12]], ptr [[TMP10]], align 4
+; VEC-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[TMP10]]
+; VEC-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
+; VEC-NEXT:    [[TMP13:%.*]] = add nsw i32 [[TMP12]], 20
+; VEC-NEXT:    store i32 [[TMP13]], ptr [[TMP11]], align 4
 ; VEC-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.continue2:
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; VEC-NEXT:    br i1 [[TMP13]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VEC-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; VEC-NEXT:    br i1 [[TMP14]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; VEC:       for.end:
 ; VEC-NEXT:    ret i32 0
 ;
@@ -321,50 +321,50 @@ define void @bug18724(i1 %cond, ptr %ptr, i1 %cond.2, i64 %v.1, i32 %v.2) {
 ; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
 ; VEC-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[PRED_STORE_CONTINUE2]] ]
 ; VEC-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[V_1]], [[INDEX]]
-; VEC-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 0
-; VEC-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR:%.*]], i64 0, i64 [[TMP6]]
+; VEC-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR:%.*]], i64 0, i64 [[OFFSET_IDX]]
 ; VEC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0
 ; VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4
 ; VEC-NEXT:    br i1 [[COND_2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.if:
-; VEC-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP6]]
+; VEC-NEXT:    [[INDVARS_IV3:%.*]] = add i64 [[OFFSET_IDX]], 0
+; VEC-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[INDVARS_IV3]]
 ; VEC-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
-; VEC-NEXT:    store i32 [[TMP11]], ptr [[TMP10]], align 4
-; VEC-NEXT:    [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 1
-; VEC-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP13]]
-; VEC-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
-; VEC-NEXT:    store i32 [[TMP15]], ptr [[TMP14]], align 4
+; VEC-NEXT:    store i32 [[TMP11]], ptr [[ARRAYIDX16]], align 4
+; VEC-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 1
+; VEC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP12]]
+; VEC-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
+; VEC-NEXT:    store i32 [[TMP14]], ptr [[TMP13]], align 4
 ; VEC-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.continue2:
-; VEC-NEXT:    [[TMP16:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 1)
-; VEC-NEXT:    [[PREDPHI]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP16]], <2 x i32> [[VEC_PHI]]
+; VEC-NEXT:    [[TMP15:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 1)
+; VEC-NEXT:    [[PREDPHI]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP15]], <2 x i32> [[VEC_PHI]]
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VEC-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VEC-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; VEC:       middle.block:
-; VEC-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PREDPHI]])
+; VEC-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PREDPHI]])
 ; VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
-; VEC-NEXT:    [[TMP20:%.*]] = xor i1 [[CMP_N]], true
-; VEC-NEXT:    call void @llvm.assume(i1 [[TMP20]])
+; VEC-NEXT:    [[TMP18:%.*]] = xor i1 [[CMP_N]], true
+; VEC-NEXT:    call void @llvm.assume(i1 [[TMP18]])
 ; VEC-NEXT:    br label [[SCALAR_PH]]
 ; VEC:       scalar.ph:
 ; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[V_1]], [[ENTRY:%.*]] ]
-; VEC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[V_2]], [[ENTRY]] ]
+; VEC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ [[V_2]], [[ENTRY]] ]
 ; VEC-NEXT:    br label [[FOR_BODY14:%.*]]
 ; VEC:       for.body14:
-; VEC-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], [[FOR_INC23:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; VEC-NEXT:    [[INDVARS_IV4:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], [[FOR_INC23:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; VEC-NEXT:    [[INEWCHUNKS_120:%.*]] = phi i32 [ [[INEWCHUNKS_2:%.*]], [[FOR_INC23]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; VEC-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[INDVARS_IV3]]
-; VEC-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARRAYIDX16]], align 4
+; VEC-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[INDVARS_IV4]]
+; VEC-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARRAYIDX17]], align 4
 ; VEC-NEXT:    br i1 [[COND_2]], label [[IF_THEN18:%.*]], label [[FOR_INC23]]
 ; VEC:       if.then18:
-; VEC-NEXT:    store i32 [[TMP]], ptr [[ARRAYIDX16]], align 4
+; VEC-NEXT:    store i32 [[TMP]], ptr [[ARRAYIDX17]], align 4
 ; VEC-NEXT:    [[INC21:%.*]] = add nsw i32 [[INEWCHUNKS_120]], 1
 ; VEC-NEXT:    br label [[FOR_INC23]]
 ; VEC:       for.inc23:
 ; VEC-NEXT:    [[INEWCHUNKS_2]] = phi i32 [ [[INC21]], [[IF_THEN18]] ], [ [[INEWCHUNKS_120]], [[FOR_BODY14]] ]
-; VEC-NEXT:    [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV3]], 1
-; VEC-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV3]] to i32
+; VEC-NEXT:    [[INDVARS_IV_NEXT4]] = add nsw i64 [[INDVARS_IV4]], 1
+; VEC-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV4]] to i32
 ; VEC-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[TMP1]], 0
 ; VEC-NEXT:    call void @llvm.assume(i1 [[CMP13]])
 ; VEC-NEXT:    br label [[FOR_BODY14]]
@@ -481,24 +481,24 @@ define void @minimal_bit_widths(i1 %c) {
 ; VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC:       vector.body:
 ; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; VEC-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; VEC-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
+; VEC-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr undef, i64 [[INDEX]]
 ; VEC-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
 ; VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
 ; VEC-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.if:
-; VEC-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
-; VEC-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 0
-; VEC-NEXT:    store i8 [[TMP5]], ptr [[TMP4]], align 1
-; VEC-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr undef, i64 [[TMP7]]
-; VEC-NEXT:    [[TMP9:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 1
-; VEC-NEXT:    store i8 [[TMP9]], ptr [[TMP8]], align 1
+; VEC-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
+; VEC-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr undef, i64 [[TMP8]]
+; VEC-NEXT:    [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 0
+; VEC-NEXT:    store i8 [[TMP4]], ptr [[TMP3]], align 1
+; VEC-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr undef, i64 [[TMP5]]
+; VEC-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 1
+; VEC-NEXT:    store i8 [[TMP7]], ptr [[TMP6]], align 1
 ; VEC-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.continue2:
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VEC-NEXT:    br i1 [[TMP10]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VEC-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VEC-NEXT:    br i1 [[TMP11]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; VEC:       for.end:
 ; VEC-NEXT:    ret void
 ;
@@ -611,13 +611,13 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
 ; VEC-NEXT:    br label [[FOR_BODY:%.*]]
 ; VEC:       vector.body:
 ; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; VEC-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; VEC-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 [[TMP0]]
+; VEC-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 [[INDEX]]
 ; VEC-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i32 0
 ; VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP3]], align 1
 ; VEC-NEXT:    store <2 x i8> zeroinitializer, ptr [[TMP3]], align 1
 ; VEC-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE2]]
 ; VEC:       pred.store.if:
+; VEC-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; VEC-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
 ; VEC-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 0
 ; VEC-NEXT:    store i8 [[TMP5]], ptr [[TMP4]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 9c3ba8fbcf036..1517ec84e9e01 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -1959,40 +1959,40 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE2:%.*]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_UDIV_CONTINUE2]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP0]]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[PRED_UDIV_CONTINUE2]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
 ; CHECK:       pred.udiv.if:
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = udiv i32 [[TMP4]], [[TMP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = udiv i32 [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
 ; CHECK-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; CHECK:       pred.udiv.continue:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ]
 ; CHECK-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
 ; CHECK:       pred.udiv.if1:
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
-; CHECK-NEXT:    [[TMP11:%.*]] = udiv i32 [[TMP10]], [[TMP9]]
-; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP14]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[TMP9]], i32 1
 ; CHECK-NEXT:    br label [[PRED_UDIV_CONTINUE2]]
 ; CHECK:       pred.udiv.continue2:
-; CHECK-NEXT:    [[TMP13:%.*]] = phi <2 x i32> [ [[TMP7]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], [[PRED_UDIV_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP13]], <2 x i32> [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP15]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP11:%.*]] = phi <2 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP11]], <2 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP16]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP15]])
+; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP16]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[SMAX]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ]
@@ -2010,7 +2010,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[VAR5]]
 ;
 ; IND-LABEL: @scalarize_induction_variable_05(
@@ -2182,64 +2182,64 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE8:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[PRED_UDIV_CONTINUE8]] ]
-; UNROLL-NO-IC-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[PRED_UDIV_CONTINUE8]] ]
-; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
-; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP0]]
-; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
-; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 2
-; UNROLL-NO-IC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
-; UNROLL-NO-IC-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4
+; UNROLL-NO-IC-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP32:%.*]], [[PRED_UDIV_CONTINUE8]] ]
+; UNROLL-NO-IC-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[PRED_UDIV_CONTINUE8]] ]
+; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]
+; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0
+; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 2
+; UNROLL-NO-IC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
+; UNROLL-NO-IC-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
 ; UNROLL-NO-IC-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
 ; UNROLL-NO-IC:       pred.udiv.if:
-; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
-; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = udiv i32 [[TMP5]], [[TMP0]]
-; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
+; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = add i32 [[INDEX]], 0
+; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
+; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
+; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i32 0
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; UNROLL-NO-IC:       pred.udiv.continue:
-; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ]
+; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP10]], [[PRED_UDIV_IF]] ]
 ; UNROLL-NO-IC-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
 ; UNROLL-NO-IC:       pred.udiv.if3:
-; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = add i32 [[INDEX]], 1
-; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
-; UNROLL-NO-IC-NEXT:    [[TMP12:%.*]] = udiv i32 [[TMP11]], [[TMP10]]
-; UNROLL-NO-IC-NEXT:    [[TMP13:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP12]], i32 1
+; UNROLL-NO-IC-NEXT:    [[TMP13:%.*]] = add i32 [[INDEX]], 1
+; UNROLL-NO-IC-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
+; UNROLL-NO-IC-NEXT:    [[TMP15:%.*]] = udiv i32 [[TMP14]], [[TMP13]]
+; UNROLL-NO-IC-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP15]], i32 1
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
 ; UNROLL-NO-IC:       pred.udiv.continue4:
-; UNROLL-NO-IC-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_UDIV_IF3]] ]
+; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = phi <2 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP16]], [[PRED_UDIV_IF3]] ]
 ; UNROLL-NO-IC-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
 ; UNROLL-NO-IC:       pred.udiv.if5:
-; UNROLL-NO-IC-NEXT:    [[TMP16:%.*]] = add i32 [[INDEX]], 2
-; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i32 0
-; UNROLL-NO-IC-NEXT:    [[TMP18:%.*]] = udiv i32 [[TMP17]], [[TMP16]]
-; UNROLL-NO-IC-NEXT:    [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0
+; UNROLL-NO-IC-NEXT:    [[TMP19:%.*]] = add i32 [[INDEX]], 2
+; UNROLL-NO-IC-NEXT:    [[TMP20:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i32 0
+; UNROLL-NO-IC-NEXT:    [[TMP21:%.*]] = udiv i32 [[TMP20]], [[TMP19]]
+; UNROLL-NO-IC-NEXT:    [[TMP22:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i32 0
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE6]]
 ; UNROLL-NO-IC:       pred.udiv.continue6:
-; UNROLL-NO-IC-NEXT:    [[TMP20:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP19]], [[PRED_UDIV_IF5]] ]
+; UNROLL-NO-IC-NEXT:    [[TMP23:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP22]], [[PRED_UDIV_IF5]] ]
 ; UNROLL-NO-IC-NEXT:    br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]]
 ; UNROLL-NO-IC:       pred.udiv.if7:
-; UNROLL-NO-IC-NEXT:    [[TMP22:%.*]] = add i32 [[INDEX]], 3
-; UNROLL-NO-IC-NEXT:    [[TMP23:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i32 1
-; UNROLL-NO-IC-NEXT:    [[TMP24:%.*]] = udiv i32 [[TMP23]], [[TMP22]]
-; UNROLL-NO-IC-NEXT:    [[TMP25:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP24]], i32 1
+; UNROLL-NO-IC-NEXT:    [[TMP25:%.*]] = add i32 [[INDEX]], 3
+; UNROLL-NO-IC-NEXT:    [[TMP26:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i32 1
+; UNROLL-NO-IC-NEXT:    [[TMP27:%.*]] = udiv i32 [[TMP26]], [[TMP25]]
+; UNROLL-NO-IC-NEXT:    [[TMP28:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP27]], i32 1
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE8]]
 ; UNROLL-NO-IC:       pred.udiv.continue8:
-; UNROLL-NO-IC-NEXT:    [[TMP26:%.*]] = phi <2 x i32> [ [[TMP20]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP25]], [[PRED_UDIV_IF7]] ]
-; UNROLL-NO-IC-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP14]], <2 x i32> [[WIDE_LOAD]]
-; UNROLL-NO-IC-NEXT:    [[PREDPHI9:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP26]], <2 x i32> [[WIDE_LOAD2]]
-; UNROLL-NO-IC-NEXT:    [[TMP29]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
-; UNROLL-NO-IC-NEXT:    [[TMP30]] = add <2 x i32> [[PREDPHI9]], [[VEC_PHI1]]
+; UNROLL-NO-IC-NEXT:    [[TMP29:%.*]] = phi <2 x i32> [ [[TMP23]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP28]], [[PRED_UDIV_IF7]] ]
+; UNROLL-NO-IC-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP17]], <2 x i32> [[WIDE_LOAD]]
+; UNROLL-NO-IC-NEXT:    [[PREDPHI9:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> [[TMP29]], <2 x i32> [[WIDE_LOAD2]]
+; UNROLL-NO-IC-NEXT:    [[TMP32]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
+; UNROLL-NO-IC-NEXT:    [[TMP33]] = add <2 x i32> [[PREDPHI9]], [[VEC_PHI1]]
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; UNROLL-NO-IC-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; UNROLL-NO-IC-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP34]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; UNROLL-NO-IC:       middle.block:
-; UNROLL-NO-IC-NEXT:    [[BIN_RDX:%.*]] = add <2 x i32> [[TMP30]], [[TMP29]]
-; UNROLL-NO-IC-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]])
+; UNROLL-NO-IC-NEXT:    [[BIN_RDX:%.*]] = add <2 x i32> [[TMP33]], [[TMP32]]
+; UNROLL-NO-IC-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]])
 ; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[SMAX]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
 ; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP32]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP30]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-IC:       for.body:
 ; UNROLL-NO-IC-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ]
@@ -2257,7 +2257,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    [[COND:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP25:![0-9]+]]
 ; UNROLL-NO-IC:       for.end:
-; UNROLL-NO-IC-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP32]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP30]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-IC-NEXT:    ret i32 [[VAR5]]
 ;
 ; INTERLEAVE-LABEL: @scalarize_induction_variable_05(
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
index 402632b51698a..0c1a4901a56da 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
@@ -18,31 +18,31 @@ define i16 @test_access_size_not_multiple_of_align(i64 %len, ptr %test_base) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_LOAD_CONTINUE2]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i8> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK:       pred.load.if:
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = load i16, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i16, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i16> poison, i16 [[TMP7]], i32 0
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.if1:
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = load i16, ptr [[TMP11]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP12]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i16> [[TMP9]], i16 [[TMP13]], i32 1
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.continue2:
-; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i16> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[TMP14]], <2 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = phi <2 x i16> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[TMP18]], <2 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP15]] = add <2 x i16> [[VEC_PHI]], [[PREDPHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
@@ -115,31 +115,31 @@ define i32 @test_access_size_multiple_of_align_but_offset_by_1(i64 %len, ptr %te
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_LOAD_CONTINUE2]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i8> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK:       pred.load.if:
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[START]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[START]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.if1:
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[START]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP12]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[START]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.continue2:
-; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP14]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = phi <2 x i32> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP18]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP15]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
@@ -229,17 +229,17 @@ define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[P2]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <2 x i32>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[DEST:%.*]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0
 ; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i32> [[WIDE_LOAD3]], i32 0
@@ -260,7 +260,7 @@ define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
 ; CHECK:       pred.store.continue4:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -287,7 +287,7 @@ define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[IND_NEXT]] to i32
 ; CHECK-NEXT:    [[GEP_IND_NEXT]] = add i64 [[GEP_IND]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[CONV]], [[END_CLAMPED]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -343,15 +343,14 @@ define void @test_rev_loops_deref_loads(ptr nocapture noundef writeonly %dest) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 -1
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <2 x i32> [[REVERSE]], splat (i32 3)
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 -1
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4
@@ -359,6 +358,7 @@ define void @test_rev_loops_deref_loads(ptr nocapture noundef writeonly %dest) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[REVERSE2]], i32 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = shl nsw i32 [[TMP11]], 2
@@ -377,7 +377,7 @@ define void @test_rev_loops_deref_loads(ptr nocapture noundef writeonly %dest) {
 ; CHECK:       pred.store.continue4:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -399,7 +399,7 @@ define void @test_rev_loops_deref_loads(ptr nocapture noundef writeonly %dest) {
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
 ; CHECK-NEXT:    [[CMP2_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)
 ; CHECK-NEXT:    ret void
@@ -490,7 +490,7 @@ define void @test_rev_loops_non_deref_loads(ptr nocapture noundef writeonly %des
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 -2)
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -513,7 +513,7 @@ define void @test_rev_loops_non_deref_loads(ptr nocapture noundef writeonly %des
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
 ; CHECK-NEXT:    [[CMP2_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)
 ; CHECK-NEXT:    ret void
@@ -584,7 +584,7 @@ define i16 @test_strided_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP13]])
 ; CHECK-NEXT:    br i1 true, label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
@@ -609,7 +609,7 @@ define i16 @test_strided_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i16 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i16 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp eq i64 [[IV]], 4095
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i16 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i16 [[ACCUM_NEXT_LCSSA]]
@@ -658,8 +658,7 @@ define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 511, i64 510>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 511, [[INDEX]]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 -1
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4
@@ -676,6 +675,7 @@ define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = shl nsw i32 [[TMP11]], 2
 ; CHECK-NEXT:    store i32 [[TMP15]], ptr [[TMP14]], align 4
@@ -693,7 +693,7 @@ define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 -2)
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
-; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -716,7 +716,7 @@ define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
 ; CHECK-NEXT:    [[CMP2_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP2_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/LoopVectorize/load-of-struct-deref-pred.ll b/llvm/test/Transforms/LoopVectorize/load-of-struct-deref-pred.ll
index 7e00cb74a69ed..b418fa715dee2 100644
--- a/llvm/test/Transforms/LoopVectorize/load-of-struct-deref-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-of-struct-deref-pred.ll
@@ -16,21 +16,21 @@ define void @accesses_to_struct_dereferenceable(ptr noalias %dst) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[TMP4]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> [[WIDE_LOAD1]]
-; CHECK-NEXT:    store <4 x i32> [[PREDPHI]], ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT:    store <4 x i32> [[PREDPHI]], ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -99,22 +99,22 @@ define void @accesses_to_struct_may_not_be_dereferenceable_due_to_loop_bound(ptr
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
-; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK:       pred.load.if:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i32 0
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; CHECK:       pred.load.if1:
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
@@ -124,7 +124,7 @@ define void @accesses_to_struct_may_not_be_dereferenceable_due_to_loop_bound(ptr
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.continue2:
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <4 x i32> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; CHECK:       pred.load.if3:
 ; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 2
@@ -134,7 +134,7 @@ define void @accesses_to_struct_may_not_be_dereferenceable_due_to_loop_bound(ptr
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
 ; CHECK:       pred.load.continue4:
 ; CHECK-NEXT:    [[TMP21:%.*]] = phi <4 x i32> [ [[TMP15]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP20]], [[PRED_LOAD_IF3]] ]
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP22]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
 ; CHECK:       pred.load.if5:
 ; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[INDEX]], 3
@@ -144,11 +144,11 @@ define void @accesses_to_struct_may_not_be_dereferenceable_due_to_loop_bound(ptr
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
 ; CHECK:       pred.load.continue6:
 ; CHECK-NEXT:    [[TMP27:%.*]] = phi <4 x i32> [ [[TMP21]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP26]], [[PRED_LOAD_IF5]] ]
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i32, ptr [[TMP28]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> [[TMP27]]
-; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> [[TMP27]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0
 ; CHECK-NEXT:    store <4 x i32> [[PREDPHI]], ptr [[TMP30]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
@@ -221,22 +221,22 @@ define void @accesses_to_struct_may_not_be_dereferenceable_access_size(ptr noali
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
-; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK:       pred.load.if:
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i64> poison, i64 [[TMP7]], i32 0
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi <4 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; CHECK:       pred.load.if1:
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
@@ -246,7 +246,7 @@ define void @accesses_to_struct_may_not_be_dereferenceable_access_size(ptr noali
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.continue2:
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <4 x i64> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; CHECK:       pred.load.if3:
 ; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 2
@@ -256,7 +256,7 @@ define void @accesses_to_struct_may_not_be_dereferenceable_access_size(ptr noali
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
 ; CHECK:       pred.load.continue4:
 ; CHECK-NEXT:    [[TMP21:%.*]] = phi <4 x i64> [ [[TMP15]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP20]], [[PRED_LOAD_IF3]] ]
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP22]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
 ; CHECK:       pred.load.if5:
 ; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[INDEX]], 3
@@ -267,11 +267,11 @@ define void @accesses_to_struct_may_not_be_dereferenceable_access_size(ptr noali
 ; CHECK:       pred.load.continue6:
 ; CHECK-NEXT:    [[TMP27:%.*]] = phi <4 x i64> [ [[TMP21]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP26]], [[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc <4 x i64> [[TMP27]] to <4 x i32>
-; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i32, ptr [[TMP29]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP30]], align 4
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> [[TMP28]]
-; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0
 ; CHECK-NEXT:    store <4 x i32> [[PREDPHI]], ptr [[TMP31]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
diff --git a/llvm/test/Transforms/LoopVectorize/loop-form.ll b/llvm/test/Transforms/LoopVectorize/loop-form.ll
index 914b9ad4a9e5a..78146bee69945 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-form.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-form.ll
@@ -1076,8 +1076,7 @@ define void @scalar_predication(ptr %addr) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr float, ptr [[ADDR:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr float, ptr [[ADDR:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = fcmp oeq <2 x float> [[WIDE_LOAD]], zeroinitializer
@@ -1085,6 +1084,7 @@ define void @scalar_predication(ptr %addr) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr float, ptr [[ADDR]], i64 [[TMP0]]
 ; CHECK-NEXT:    store float 1.000000e+01, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
index 4c9b4a4df0e37..a4b2f0cff2c43 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
@@ -483,8 +483,7 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE8:%.*]] ]
 ; CHECK-VF4-IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[PRED_STORE_CONTINUE8]] ]
 ; CHECK-VF4-IC1-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[PRED_STORE_CONTINUE8]] ]
-; CHECK-VF4-IC1-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
-; CHECK-VF4-IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP1]]
+; CHECK-VF4-IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-VF4-IC1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 0
 ; CHECK-VF4-IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP3]], align 4, !alias.scope [[META6:![0-9]+]]
 ; CHECK-VF4-IC1-NEXT:    [[TMP4:%.*]] = fcmp olt <4 x float> [[WIDE_LOAD]], zeroinitializer
@@ -494,6 +493,7 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC1-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
 ; CHECK-VF4-IC1-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK-VF4-IC1:       pred.store.if:
+; CHECK-VF4-IC1-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-VF4-IC1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP1]]
 ; CHECK-VF4-IC1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
 ; CHECK-VF4-IC1-NEXT:    [[TMP11:%.*]] = add nsw i32 [[TMP10]], 1
@@ -597,8 +597,7 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[PRED_STORE_CONTINUE19]] ]
 ; CHECK-VF4-IC2-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[PRED_STORE_CONTINUE19]] ]
 ; CHECK-VF4-IC2-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[PRED_STORE_CONTINUE19]] ]
-; CHECK-VF4-IC2-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
-; CHECK-VF4-IC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP1]]
+; CHECK-VF4-IC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-VF4-IC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i32 0
 ; CHECK-VF4-IC2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i32 4
 ; CHECK-VF4-IC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP5]], align 4, !alias.scope [[META6:![0-9]+]]
@@ -614,6 +613,7 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
 ; CHECK-VF4-IC2-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP7]], i32 0
 ; CHECK-VF4-IC2-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK-VF4-IC2:       pred.store.if:
+; CHECK-VF4-IC2-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-VF4-IC2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP1]]
 ; CHECK-VF4-IC2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
 ; CHECK-VF4-IC2-NEXT:    [[TMP18:%.*]] = add nsw i32 [[TMP17]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-predicated.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-predicated.ll
index 11294b8fef3ee..c17985dc56c4d 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-predicated.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-predicated.ll
@@ -13,9 +13,8 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1
 ; CHECK-VF2IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-VF2IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-VF2IC1:       [[VECTOR_BODY]]:
-; CHECK-VF2IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-VF2IC1-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-VF2IC1-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_LOAD_CONTINUE2]] ]
-; CHECK-VF2IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-VF2IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[TMP0]]
 ; CHECK-VF2IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-VF2IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
@@ -23,7 +22,8 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1
 ; CHECK-VF2IC1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-VF2IC1-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK-VF2IC1:       [[PRED_LOAD_IF]]:
-; CHECK-VF2IC1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP0]]
+; CHECK-VF2IC1-NEXT:    [[TMP19:%.*]] = add i64 [[TMP0]], 0
+; CHECK-VF2IC1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP19]]
 ; CHECK-VF2IC1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-VF2IC1-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
 ; CHECK-VF2IC1-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
@@ -32,7 +32,7 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1
 ; CHECK-VF2IC1-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
 ; CHECK-VF2IC1-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK-VF2IC1:       [[PRED_LOAD_IF1]]:
-; CHECK-VF2IC1-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF2IC1-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 1
 ; CHECK-VF2IC1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP11]]
 ; CHECK-VF2IC1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
 ; CHECK-VF2IC1-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
@@ -42,7 +42,7 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1
 ; CHECK-VF2IC1-NEXT:    [[TMP16:%.*]] = icmp eq <2 x i32> [[TMP15]], splat (i32 2)
 ; CHECK-VF2IC1-NEXT:    [[TMP17:%.*]] = or <2 x i1> [[VEC_PHI]], [[TMP16]]
 ; CHECK-VF2IC1-NEXT:    [[PREDPHI]] = select <2 x i1> [[TMP4]], <2 x i1> [[TMP17]], <2 x i1> [[VEC_PHI]]
-; CHECK-VF2IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-VF2IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
 ; CHECK-VF2IC1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF2IC1-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-VF2IC1:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return.ll b/llvm/test/Transforms/LoopVectorize/struct-return.ll
index 57d0aa3787c45..50b9ba12af82d 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return.ll
@@ -215,7 +215,7 @@ define void @scalarized_predicated_struct_return(ptr %a) {
 ; CHECK-LABEL: define void @scalarized_predicated_struct_return
 ; CHECK:  vector.body:
 ; CHECK:  pred.store.if:
-; CHECK:     tail call { i64, i64 } @bar_i64(i64 %5)
+; CHECK:     tail call { i64, i64 } @bar_i64(i64 {{.+}})
 entry:
   br label %for.body
 
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index f31885219f53f..514e858d6a272 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -152,15 +152,15 @@ define i64 @multi_exit_2_exit_count_with_udiv_by_value_in_block_executed_uncondi
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[IV]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 10)
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[IV]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP20]]
 ; CHECK-NEXT:    store i32 1, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
@@ -264,15 +264,15 @@ define i64 @multi_exit_2_exit_count_with_udiv_by_constant_in_block_executed_unco
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[IV]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 10)
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[IV]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP20]]
 ; CHECK-NEXT:    store i32 1, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 37b318ed6c55d..fc80d348042a0 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -176,7 +176,8 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
 ; CHECK-NEXT:   Successor(s): pred.udiv.if, pred.udiv.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.udiv.if:
-; CHECK-NEXT:     REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V)
+; CHECK-NEXT:     vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
+; CHECK-NEXT:     REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS2]]> (S->V)
 ; CHECK-NEXT:   Successor(s): pred.udiv.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.udiv.continue:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 43172cc2f170a..f704f8a274cda 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -992,7 +992,8 @@ define void @sinking_requires_duplication(ptr %addr) {
 ; CHECK-NEXT:   Successor(s): pred.store.if, pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.if:
-; CHECK-NEXT:     REPLICATE ir<%gep>.1 = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT:     vp<[[STEPS_SUNK:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
+; CHECK-NEXT:     REPLICATE ir<%gep>.1 = getelementptr ir<%addr>, vp<[[STEPS_SUNK]]>
 ; CHECK-NEXT:     REPLICATE store ir<1.000000e+01>, ir<%gep>.1
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:

From d08e9800650b78fdd9cbed1375faa0f55e4ac09c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Mon, 21 Apr 2025 10:51:48 -0700
Subject: [PATCH 633/710] [flang][cuda] Only convert launch from CUDA Fortran
 kernels (#136221)

Make sure `gpu.launch_func` has a CUDA proc attribute and update the
conversion pattern to only convert those with the attribute.
---
 .../Transforms/CUFGPUToLLVMConversion.cpp         | 15 ++++++++++++++-
 .../lib/Optimizer/Transforms/CUFOpConversion.cpp  |  5 +++++
 flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir     |  6 +++---
 flang/test/Fir/CUDA/cuda-launch.fir               | 10 +++++-----
 4 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
index 02b4e6a5a469c..ef6b80b5739c6 100644
--- a/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFGPUToLLVMConversion.cpp
@@ -82,6 +82,11 @@ struct GPULaunchKernelConversion
   mlir::LogicalResult
   matchAndRewrite(mlir::gpu::LaunchFuncOp op, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
+    // Only convert gpu.launch_func for CUDA Fortran.
+    if (!op.getOperation()->getAttrOfType<cuf::ProcAttributeAttr>(
+            cuf::getProcAttrName()))
+      return mlir::failure();
+
     mlir::Location loc = op.getLoc();
     auto *ctx = rewriter.getContext();
     mlir::ModuleOp mod = op->getParentOfType<mlir::ModuleOp>();
@@ -293,7 +298,15 @@ class CUFGPUToLLVMConversion
     fir::LLVMTypeConverter typeConverter(module, /*applyTBAA=*/false,
                                          /*forceUnifiedTBAATree=*/false, *dl);
     cuf::populateCUFGPUToLLVMConversionPatterns(typeConverter, patterns);
-    target.addIllegalOp<mlir::gpu::LaunchFuncOp>();
+
+    target.addDynamicallyLegalOp<mlir::gpu::LaunchFuncOp>(
+        [&](mlir::gpu::LaunchFuncOp op) {
+          if (op.getOperation()->getAttrOfType<cuf::ProcAttributeAttr>(
+                  cuf::getProcAttrName()))
+            return false;
+          return true;
+        });
+
     target.addIllegalOp<cuf::SharedMemoryOp>();
     target.addLegalDialect<mlir::LLVM::LLVMDialect>();
     if (mlir::failed(mlir::applyPartialConversion(getOperation(), target,
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 77364cb837c3c..e70ceb3a67d98 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -888,6 +888,11 @@ struct CUFLaunchOpConversion
     }
     if (procAttr)
       gpuLaunchOp->setAttr(cuf::getProcAttrName(), procAttr);
+    else
+      // Set default global attribute of the original was not found.
+      gpuLaunchOp->setAttr(cuf::getProcAttrName(),
+                           cuf::ProcAttributeAttr::get(
+                               op.getContext(), cuf::ProcAttribute::Global));
     rewriter.replaceOp(op, gpuLaunchOp);
     return mlir::success();
   }
diff --git a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
index 0319213016e45..a2f89d822c455 100644
--- a/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
+++ b/flang/test/Fir/CUDA/cuda-gpu-launch-func.mlir
@@ -54,7 +54,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i1, dense<8> : ve
     llvm.br ^bb1(%44 : i64)
   ^bb3:  // pred: ^bb1
     %45 = llvm.call @_FortranACUFDataTransferPtrPtr(%14, %25, %2, %11, %13, %5) : (!llvm.ptr, !llvm.ptr, i64, i32, !llvm.ptr, i32) -> !llvm.struct<()>
-    gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 blocks in (%7, %7, %7) threads in (%12, %7, %7) : i64 dynamic_shared_memory_size %11 args(%14 : !llvm.ptr)
+    gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 blocks in (%7, %7, %7) threads in (%12, %7, %7) : i64 dynamic_shared_memory_size %11 args(%14 : !llvm.ptr) {cuf.proc_attr = #cuf.cuda_proc<global>}
     %46 = llvm.call @_FortranACUFDataTransferPtrPtr(%25, %14, %2, %10, %13, %4) : (!llvm.ptr, !llvm.ptr, i64, i32, !llvm.ptr, i32) -> !llvm.struct<()>
     %47 = llvm.call @_FortranAioBeginExternalListOutput(%9, %13, %8) {fastmathFlags = #llvm.fastmath<contract>} : (i32, !llvm.ptr, i32) -> !llvm.ptr
     %48 = llvm.mlir.constant(9 : i32) : i32
@@ -122,7 +122,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr<272>, d
     %1 = llvm.mlir.constant(2 : index) : i64
     %2 = llvm.mlir.constant(0 : i32) : i32
     %3 = llvm.mlir.constant(10 : index) : i64
-    gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 clusters in (%1, %1, %0) blocks in (%3, %3, %0) threads in (%3, %3, %0) : i64 dynamic_shared_memory_size %2
+    gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 clusters in (%1, %1, %0) blocks in (%3, %3, %0) threads in (%3, %3, %0) : i64 dynamic_shared_memory_size %2  {cuf.proc_attr = #cuf.cuda_proc<global>}
     llvm.return
   }
   gpu.binary @cuda_device_mod  [#gpu.object<#nvvm.target, "">]
@@ -180,7 +180,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr<272>, d
     %2 = llvm.mlir.constant(0 : i32) : i32
     %3 = llvm.mlir.constant(10 : index) : i64
     %token = cuf.stream_cast %stream : !llvm.ptr
-    gpu.launch_func [%token] @cuda_device_mod::@_QMmod1Psub1 blocks in (%3, %3, %0) threads in (%3, %3, %0) : i64 dynamic_shared_memory_size %2
+    gpu.launch_func [%token] @cuda_device_mod::@_QMmod1Psub1 blocks in (%3, %3, %0) threads in (%3, %3, %0) : i64 dynamic_shared_memory_size %2 {cuf.proc_attr = #cuf.cuda_proc<global>}
     llvm.return
   }
   gpu.binary @cuda_device_mod  [#gpu.object<#nvvm.target, "">]
diff --git a/flang/test/Fir/CUDA/cuda-launch.fir b/flang/test/Fir/CUDA/cuda-launch.fir
index 028279832c703..1e8036e628650 100644
--- a/flang/test/Fir/CUDA/cuda-launch.fir
+++ b/flang/test/Fir/CUDA/cuda-launch.fir
@@ -26,13 +26,13 @@ module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_e
     %c1024_i32 = arith.constant 1024 : i32
     %c6_i32 = arith.constant 6 : i32
     %c1_i32 = arith.constant 1 : i32
-    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device1 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c0{{.*}}
+    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device1 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c0{{.*}} {cuf.proc_attr = #cuf.cuda_proc<global>}
     cuf.kernel_launch @cuda_device_mod::@_QPsub_device1<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32>>>()
 
-    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device1 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c1024{{.*}}
+    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device1 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c1024{{.*}} {cuf.proc_attr = #cuf.cuda_proc<global>}
     cuf.kernel_launch @cuda_device_mod::@_QPsub_device1<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1024_i32>>>()
 
-    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device2 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c0{{.*}} args(%[[ALLOCA]] : !fir.ref<f32>)
+    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device2 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c0{{.*}} args(%[[ALLOCA]] : !fir.ref<f32>) {cuf.proc_attr = #cuf.cuda_proc<global>}
     cuf.kernel_launch @cuda_device_mod::@_QPsub_device2<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32>>>(%0) : (!fir.ref<f32>)
     return
   }
@@ -64,7 +64,7 @@ module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_e
 }
 
 // CHECK-LABEL: func.func @_QMmod1Phost_sub()
-// CHECK: gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 clusters in (%c2{{.*}}, %c2{{.*}}, %c1{{.*}})
+// CHECK: gpu.launch_func  @cuda_device_mod::@_QMmod1Psub1 clusters in (%c2{{.*}}, %c2{{.*}}, %c1{{.*}}) {cuf.proc_attr = #cuf.cuda_proc<global>}
 
 // -----
 
@@ -107,7 +107,7 @@ module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_e
 // CHECK: %[[CONV_ADDR:.*]] = fir.convert %[[ADDROF]] : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> !fir.llvm_ptr<i8>
 // CHECK: %[[DEVADDR:.*]] = fir.call @_FortranACUFGetDeviceAddress(%[[CONV_ADDR]], %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> !fir.llvm_ptr<i8>
 // CHECK: %[[CONV_DEVADDR:.*]] = fir.convert %[[DEVADDR]] : (!fir.llvm_ptr<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
-// CHECK: gpu.launch_func  @cuda_device_mod::@_QMdevptrPtest blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %{{.*}} args(%[[CONV_DEVADDR]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+// CHECK: gpu.launch_func  @cuda_device_mod::@_QMdevptrPtest blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %{{.*}} args(%[[CONV_DEVADDR]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) {cuf.proc_attr = #cuf.cuda_proc<global>}
 
 // -----
 

From 3e5a9d9aa04e2c99601ffb6f65032a299a56069c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 21 Apr 2025 18:57:55 +0100
Subject: [PATCH 634/710] [VPlan] Rename setFlags -> applyFlags (NFC).

Update name to apply flags to instructions, as suggested in
https://github.com/llvm/llvm-project/pull/135272.

Also changes the arg to a reference.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  2 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         | 30 +++++++++----------
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 ++++----
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 137f32ded7248..36e14bb27a029 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2357,7 +2357,7 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
 #endif
   }
 
-  RepRecipe->setFlags(Cloned);
+  RepRecipe->applyFlags(*Cloned);
 
   if (auto DL = RepRecipe->getDebugLoc())
     State.setDebugLocFrom(DL);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 7084676af6d5b..fdcee2bd28997 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -764,33 +764,33 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
     }
   }
 
-  /// Set the IR flags for \p I.
-  void setFlags(Instruction *I) const {
+  /// Apply the IR flags to \p I.
+  void applyFlags(Instruction &I) const {
     switch (OpType) {
     case OperationType::OverflowingBinOp:
-      I->setHasNoUnsignedWrap(WrapFlags.HasNUW);
-      I->setHasNoSignedWrap(WrapFlags.HasNSW);
+      I.setHasNoUnsignedWrap(WrapFlags.HasNUW);
+      I.setHasNoSignedWrap(WrapFlags.HasNSW);
       break;
     case OperationType::DisjointOp:
-      cast<PossiblyDisjointInst>(I)->setIsDisjoint(DisjointFlags.IsDisjoint);
+      cast<PossiblyDisjointInst>(&I)->setIsDisjoint(DisjointFlags.IsDisjoint);
       break;
     case OperationType::PossiblyExactOp:
-      I->setIsExact(ExactFlags.IsExact);
+      I.setIsExact(ExactFlags.IsExact);
       break;
     case OperationType::GEPOp:
-      cast<GetElementPtrInst>(I)->setNoWrapFlags(GEPFlags);
+      cast<GetElementPtrInst>(&I)->setNoWrapFlags(GEPFlags);
       break;
     case OperationType::FPMathOp:
-      I->setHasAllowReassoc(FMFs.AllowReassoc);
-      I->setHasNoNaNs(FMFs.NoNaNs);
-      I->setHasNoInfs(FMFs.NoInfs);
-      I->setHasNoSignedZeros(FMFs.NoSignedZeros);
-      I->setHasAllowReciprocal(FMFs.AllowReciprocal);
-      I->setHasAllowContract(FMFs.AllowContract);
-      I->setHasApproxFunc(FMFs.ApproxFunc);
+      I.setHasAllowReassoc(FMFs.AllowReassoc);
+      I.setHasNoNaNs(FMFs.NoNaNs);
+      I.setHasNoInfs(FMFs.NoInfs);
+      I.setHasNoSignedZeros(FMFs.NoSignedZeros);
+      I.setHasAllowReciprocal(FMFs.AllowReciprocal);
+      I.setHasAllowContract(FMFs.AllowContract);
+      I.setHasApproxFunc(FMFs.ApproxFunc);
       break;
     case OperationType::NonNegOp:
-      I->setNonNeg(NonNegFlags.NonNeg);
+      I.setNonNeg(NonNegFlags.NonNeg);
       break;
     case OperationType::Cmp:
     case OperationType::Other:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ff3b2aee7916c..dac31518997d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -463,7 +463,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     auto *Res =
         Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
     if (auto *I = dyn_cast<Instruction>(Res))
-      setFlags(I);
+      applyFlags(*I);
     return Res;
   }
 
@@ -1230,7 +1230,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
     CI->getOperandBundlesAsDefs(OpBundles);
 
   CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);
-  setFlags(V);
+  applyFlags(*V);
 
   if (!V->getType()->isVoidTy())
     State.set(this, V);
@@ -1309,7 +1309,7 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
 
   CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
 
-  setFlags(V);
+  applyFlags(*V);
 
   if (!V->getType()->isVoidTy())
     State.set(this, V);
@@ -1509,7 +1509,7 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
   Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
   State.set(this, Sel);
   if (isa<FPMathOperator>(Sel))
-    setFlags(cast<Instruction>(Sel));
+    applyFlags(*cast<Instruction>(Sel));
   State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
 }
 
@@ -1642,7 +1642,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
     Value *V = Builder.CreateNAryOp(Opcode, Ops);
 
     if (auto *VecOp = dyn_cast<Instruction>(V))
-      setFlags(VecOp);
+      applyFlags(*VecOp);
 
     // Use this vector value for all users of the original instruction.
     State.set(this, V);
@@ -1797,7 +1797,7 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
   State.set(this, Cast);
   State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue()));
   if (auto *CastOp = dyn_cast<Instruction>(Cast))
-    setFlags(CastOp);
+    applyFlags(*CastOp);
 }
 
 InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF,

From 1ab9e53e4910557f2a4c0f00c7a7f8b5ac493b48 Mon Sep 17 00:00:00 2001
From: Jason Molenda <jmolenda@apple.com>
Date: Mon, 21 Apr 2025 10:58:58 -0700
Subject: [PATCH 635/710] [lldb][Mach-O corefiles] Don't init Target arch to
 corefile (#136065)

This patch is making three changes, when loading a Mach-O corefile:

1. At the start of `DoLoadCore`, if a binary was provided in addition to
the corefile, initialize the Target's ArchSpec.

2. Before ProcessMachCore does its "exhaustive search" fallback, looking
through the corefile contents for a userland dyld or mach kernel, we
must make sure the Target has an ArchSpec, or methods that check the
address word size, or initialize a DataExtractor based on the Target
arch will not succeed.

3. Add logging when setting the Target's arch listing exactly what that
setting was based on -- the corefile itself, or the main binary.

Jonas landed a change last August (started with a patch from me) which
removed the Target ArchSpec initialization at the start of DoLoadCore,
in a scenario where the corefile had arch armv7 and the main binary had
arch armv7em (Cortex-M), and there was python code in the main binary's
dSYM which sets the operating system threads provider based on the
Target arch. It did different things for armv7 or armv7em, and so it
would fail.

Jonas' patch removed any ArchSpec setting at the start of DoLoadCore, so
we wouldn't have an incorrect arch value, but that broke the exhaustive
search for kernel binaries, because we didn't have an address word size
or endianness.

This patch should navigate the needs of both use cases.

I spent a good bit of time trying to construct a test to capture all of
these requirements -- but it turns out to be a good bit difficult,
encompassing both a genuine kernel corefiles and a microcontroller
firmware corefiles.

rdar://146821929
---
 .../Process/mach-core/ProcessMachCore.cpp     | 48 ++++++++++++++++++-
 1 file changed, 46 insertions(+), 2 deletions(-)

diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp
index 281f3a0db8f69..39584bad2d745 100644
--- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp
+++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp
@@ -426,6 +426,22 @@ void ProcessMachCore::LoadBinariesViaExhaustiveSearch() {
   std::vector<addr_t> dylds_found;
   std::vector<addr_t> kernels_found;
 
+  // To do an exhaustive search, we'll need to create data extractors
+  // to get correctly sized/endianness fields.  If we had a main binary
+  // already, we would have set the Target to that - so here we'll use
+  // the corefile's cputype/cpusubtype as the best guess.
+  if (!GetTarget().GetArchitecture().IsValid()) {
+    // The corefile's architecture is our best starting point.
+    ArchSpec arch(m_core_module_sp->GetArchitecture());
+    if (arch.IsValid()) {
+      LLDB_LOGF(log,
+                "ProcessMachCore::%s: Setting target ArchSpec based on "
+                "corefile mach-o cputype/cpusubtype",
+                __FUNCTION__);
+      GetTarget().SetArchitecture(arch);
+    }
+  }
+
   const size_t num_core_aranges = m_core_aranges.GetSize();
   for (size_t i = 0; i < num_core_aranges; ++i) {
     const VMRangeToFileOffset::Entry *entry = m_core_aranges.GetEntryAtIndex(i);
@@ -569,6 +585,7 @@ Status ProcessMachCore::DoLoadCore() {
     error = Status::FromErrorString("invalid core module");
     return error;
   }
+  Log *log(GetLog(LLDBLog::DynamicLoader | LLDBLog::Target));
 
   ObjectFile *core_objfile = m_core_module_sp->GetObjectFile();
   if (core_objfile == nullptr) {
@@ -578,20 +595,47 @@ Status ProcessMachCore::DoLoadCore() {
 
   SetCanJIT(false);
 
+  // If we have an executable binary in the Target already,
+  // use that to set the Target's ArchSpec.
+  //
+  // Don't initialize the ArchSpec based on the corefile's cputype/cpusubtype
+  // here, the corefile creator may not know the correct subtype of the code
+  // that is executing, initialize the Target to that, and if the
+  // main binary has Python code which initializes based on the Target arch,
+  // get the wrong subtype value.
+  ModuleSP exe_module_sp = GetTarget().GetExecutableModule();
+  if (exe_module_sp && exe_module_sp->GetArchitecture().IsValid()) {
+    LLDB_LOGF(log,
+              "ProcessMachCore::%s: Was given binary + corefile, setting "
+              "target ArchSpec to binary to start",
+              __FUNCTION__);
+    GetTarget().SetArchitecture(exe_module_sp->GetArchitecture());
+  }
+
   CreateMemoryRegions();
 
   LoadBinariesAndSetDYLD();
 
   CleanupMemoryRegionPermissions();
 
-  ModuleSP exe_module_sp = GetTarget().GetExecutableModule();
+  exe_module_sp = GetTarget().GetExecutableModule();
   if (exe_module_sp && exe_module_sp->GetArchitecture().IsValid()) {
+    LLDB_LOGF(log,
+              "ProcessMachCore::%s: have executable binary in the Target "
+              "after metadata/scan.  Setting Target's ArchSpec based on "
+              "that.",
+              __FUNCTION__);
     GetTarget().SetArchitecture(exe_module_sp->GetArchitecture());
   } else {
     // The corefile's architecture is our best starting point.
     ArchSpec arch(m_core_module_sp->GetArchitecture());
-    if (arch.IsValid())
+    if (arch.IsValid()) {
+      LLDB_LOGF(log,
+                "ProcessMachCore::%s: Setting target ArchSpec based on "
+                "corefile mach-o cputype/cpusubtype",
+                __FUNCTION__);
       GetTarget().SetArchitecture(arch);
+    }
   }
 
   AddressableBits addressable_bits = core_objfile->GetAddressableBits();

From 846986f35b5a7baa498db7c61ae15dd9b2dee797 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Mon, 21 Apr 2025 11:04:47 -0700
Subject: [PATCH 636/710] [Fuchsia] Disable PIC (#136376)

We aren't distributing any shared libraries so we don't need PIC and
using PIE instead of PIC can improve performance.
---
 clang/cmake/caches/Fuchsia-stage2.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 99890b8246ad7..2b29b43f44936 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -18,6 +18,7 @@ set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
 set(LLVM_ENABLE_LLD ON CACHE BOOL "")
 set(LLVM_ENABLE_LTO ON CACHE BOOL "")
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "")
+set(LLVM_ENABLE_PIC OFF CACHE BOOL "")
 set(LLVM_ENABLE_PLUGINS OFF CACHE BOOL "")
 set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "")
 set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")

From 3d14596ccfa3aa6b0735a630e3efb575a2085a06 Mon Sep 17 00:00:00 2001
From: Amr Hesham <amr96@programmer.net>
Date: Mon, 21 Apr 2025 20:07:01 +0200
Subject: [PATCH 637/710] [CIR][NFC] Fix an unused variable warning (#136431)

This fixes a warning where a variable assigned in 'if' statement wasn't
referenced again.

From 09006611151c7f85862a9da8da34872c456c2c37 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Mon, 21 Apr 2025 11:09:23 -0700
Subject: [PATCH 638/710] [clang] Enable making `CompilerInstance` VFS
 thread-safe (#135737)

The `llvm::vfs::FileSystem` interface makes no promises around
thread-safety. To enable making `CompilerInstance` thread-safe, this PR
allows passing an explicit VFS to `cloneForModuleCompile()`. This will
be used from the dependency scanner.
---
 .../include/clang/Frontend/CompilerInstance.h | 15 +++++++---
 clang/lib/Frontend/CompilerInstance.cpp       | 28 +++++++++++--------
 2 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index 41dc7f1fef461..d70f5c45b3d38 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -845,18 +845,25 @@ class CompilerInstance : public ModuleLoader {
   /// Creates a \c CompilerInstance for compiling a module.
   ///
   /// This expects a properly initialized \c FrontendInputFile.
+  ///
+  /// Explicitly-specified \c VFS takes precedence over the VFS of this instance
+  /// when creating the clone and also prevents \c FileManager sharing.
   std::unique_ptr<CompilerInstance> cloneForModuleCompileImpl(
       SourceLocation ImportLoc, StringRef ModuleName, FrontendInputFile Input,
-      StringRef OriginalModuleMapFile, StringRef ModuleFileName);
+      StringRef OriginalModuleMapFile, StringRef ModuleFileName,
+      IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
 
 public:
   /// Creates a new \c CompilerInstance for compiling a module.
   ///
   /// This takes care of creating appropriate \c FrontendInputFile for
   /// public/private frameworks, inferred modules and such.
-  std::unique_ptr<CompilerInstance>
-  cloneForModuleCompile(SourceLocation ImportLoc, Module *Module,
-                        StringRef ModuleFileName);
+  ///
+  /// Explicitly-specified \c VFS takes precedence over the VFS of this instance
+  /// when creating the clone and also prevents \c FileManager sharing.
+  std::unique_ptr<CompilerInstance> cloneForModuleCompile(
+      SourceLocation ImportLoc, Module *Module, StringRef ModuleFileName,
+      IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
 
   /// Compile a module file for the given module, using the options
   /// provided by the importing compiler instance. Returns true if the module
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 533cef35e776b..bc663acb034e7 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1152,7 +1152,8 @@ static Language getLanguageFromOptions(const LangOptions &LangOpts) {
 
 std::unique_ptr<CompilerInstance> CompilerInstance::cloneForModuleCompileImpl(
     SourceLocation ImportLoc, StringRef ModuleName, FrontendInputFile Input,
-    StringRef OriginalModuleMapFile, StringRef ModuleFileName) {
+    StringRef OriginalModuleMapFile, StringRef ModuleFileName,
+    IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
   // Construct a compiler invocation for creating this module.
   auto Invocation = std::make_shared<CompilerInvocation>(getInvocation());
 
@@ -1212,19 +1213,21 @@ std::unique_ptr<CompilerInstance> CompilerInstance::cloneForModuleCompileImpl(
   auto &Inv = *Invocation;
   Instance.setInvocation(std::move(Invocation));
 
+  if (VFS) {
+    Instance.createFileManager(std::move(VFS));
+  } else if (FrontendOpts.ModulesShareFileManager) {
+    Instance.setFileManager(&getFileManager());
+  } else {
+    Instance.createFileManager(&getVirtualFileSystem());
+  }
+
   Instance.createDiagnostics(
-      getVirtualFileSystem(),
+      Instance.getVirtualFileSystem(),
       new ForwardingDiagnosticConsumer(getDiagnosticClient()),
       /*ShouldOwnClient=*/true);
-
   if (llvm::is_contained(DiagOpts.SystemHeaderWarningsModules, ModuleName))
     Instance.getDiagnostics().setSuppressSystemWarnings(false);
 
-  if (FrontendOpts.ModulesShareFileManager) {
-    Instance.setFileManager(&getFileManager());
-  } else {
-    Instance.createFileManager(&getVirtualFileSystem());
-  }
   Instance.createSourceManager(Instance.getFileManager());
   SourceManager &SourceMgr = Instance.getSourceManager();
 
@@ -1318,7 +1321,8 @@ static OptionalFileEntryRef getPublicModuleMap(FileEntryRef File,
 }
 
 std::unique_ptr<CompilerInstance> CompilerInstance::cloneForModuleCompile(
-    SourceLocation ImportLoc, Module *Module, StringRef ModuleFileName) {
+    SourceLocation ImportLoc, Module *Module, StringRef ModuleFileName,
+    IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
   StringRef ModuleName = Module->getTopLevelModuleName();
 
   InputKind IK(getLanguageFromOptions(getLangOpts()), InputKind::ModuleMap);
@@ -1363,7 +1367,8 @@ std::unique_ptr<CompilerInstance> CompilerInstance::cloneForModuleCompile(
     return cloneForModuleCompileImpl(
         ImportLoc, ModuleName,
         FrontendInputFile(ModuleMapFilePath, IK, IsSystem),
-        ModMap.getModuleMapFileForUniquing(Module)->getName(), ModuleFileName);
+        ModMap.getModuleMapFileForUniquing(Module)->getName(), ModuleFileName,
+        std::move(VFS));
   }
 
   // FIXME: We only need to fake up an input file here as a way of
@@ -1380,7 +1385,8 @@ std::unique_ptr<CompilerInstance> CompilerInstance::cloneForModuleCompile(
   auto Instance = cloneForModuleCompileImpl(
       ImportLoc, ModuleName,
       FrontendInputFile(FakeModuleMapFile, IK, +Module->IsSystem),
-      ModMap.getModuleMapFileForUniquing(Module)->getName(), ModuleFileName);
+      ModMap.getModuleMapFileForUniquing(Module)->getName(), ModuleFileName,
+      std::move(VFS));
 
   std::unique_ptr<llvm::MemoryBuffer> ModuleMapBuffer =
       llvm::MemoryBuffer::getMemBufferCopy(InferredModuleMapContent);

From 4c0ea476c40f7e426c84c37aaf5787aa80fb5e3b Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 21 Apr 2025 11:27:24 -0700
Subject: [PATCH 639/710] [RISCV] Report error if Zilsd is used on RV64.
 (#136577)

Fixes #136564.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp           |  3 +++
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp | 10 ++++++++++
 2 files changed, 13 insertions(+)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 524d6dc01e8aa..1e7144ce6d22b 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -789,6 +789,9 @@ Error RISCVISAInfo::checkDependency() {
       return getIncompatibleError("zclsd", "zcf");
   }
 
+  if (XLen != 32 && Exts.count("zilsd") != 0)
+    return getError("'zilsd' is only supported for 'rv32'");
+
   for (auto Ext : XqciExts)
     if (Exts.count(Ext.str()) && (XLen != 32))
       return getError("'" + Twine(Ext) + "'" + " is only supported for 'rv32'");
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index ff0a5e64ab3e1..6e190ad3e7969 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -658,6 +658,16 @@ TEST(ParseArchString, RejectsConflictingExtensions) {
               "'xwchc' and 'zcb' extensions are incompatible");
   }
 
+  for (StringRef Input : {"rv64i_zilsd"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zilsd' is only supported for 'rv32'");
+  }
+
+  for (StringRef Input : {"rv64i_zclsd"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zclsd' is only supported for 'rv32'");
+  }
+
   for (StringRef Input : {"rv32i_zcf_zclsd"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
               "'zclsd' and 'zcf' extensions are incompatible");

From 9968ba8652b8a2936858a1a307207a9c8690ee5c Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Mon, 21 Apr 2025 14:31:10 -0400
Subject: [PATCH 640/710] Revert "[AMDGPU] Insert readfirstlane in the function
 returns in sgpr. (#135326)"

This reverts commit 76ced7fa782f0d7db9efea871fa6de74706dd9cc since it breaks a
lot of bots.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |   7 +-
 .../AMDGPU/add64-low-32-bits-known-zero.ll    |   4 +-
 .../CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll  |  48 +-
 llvm/test/CodeGen/AMDGPU/constrained-shift.ll |  30 +-
 .../AMDGPU/dag-preserve-disjoint-flag.ll      |  44 +-
 .../CodeGen/AMDGPU/flat-atomic-fadd.f64.ll    |  18 +-
 .../CodeGen/AMDGPU/global-atomic-fadd.f64.ll  |  18 +-
 llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll        | 836 +++++++++---------
 .../AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll    |  64 +-
 llvm/test/CodeGen/AMDGPU/ptrmask.ll           |   4 +-
 .../CodeGen/AMDGPU/sdag-print-divergence.ll   |  15 +-
 .../AMDGPU/sub64-low-32-bits-known-zero.ll    |   4 +-
 .../AMDGPU/uniform-vgpr-to-sgpr-return.ll     | 119 ---
 .../Inputs/amdgpu_isel.ll.expected            |  60 +-
 14 files changed, 538 insertions(+), 733 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AMDGPU/uniform-vgpr-to-sgpr-return.ll

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 2e3cd5ca6692d..724a45062c1f4 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3221,7 +3221,6 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                               const SDLoc &DL, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
-  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
   if (AMDGPU::isKernel(CallConv)) {
     return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
@@ -3248,8 +3247,6 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   SmallVector<SDValue, 48> RetOps;
   RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
 
-  SDValue ReadFirstLane =
-      DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
   // Copy the result values into the output registers.
   for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
        ++I, ++RealRVLocIdx) {
@@ -3277,9 +3274,7 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     default:
       llvm_unreachable("Unknown loc info!");
     }
-    if (TRI->isSGPRPhysReg(VA.getLocReg()))
-      Arg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Arg.getValueType(),
-                        ReadFirstLane, Arg);
+
     Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Glue);
     Glue = Chain.getValue(1);
     RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
diff --git a/llvm/test/CodeGen/AMDGPU/add64-low-32-bits-known-zero.ll b/llvm/test/CodeGen/AMDGPU/add64-low-32-bits-known-zero.ll
index 1d51b8a077566..52259c4c2e6e1 100644
--- a/llvm/test/CodeGen/AMDGPU/add64-low-32-bits-known-zero.ll
+++ b/llvm/test/CodeGen/AMDGPU/add64-low-32-bits-known-zero.ll
@@ -148,8 +148,8 @@ define <2 x i64> @v_add_v2i64_nonsplat_const_low_bits_known0_0(<2 x i64> %reg) {
 define amdgpu_ps <2 x i64> @s_add_v2i64_splat_const_low_bits_known0_0(<2 x i64> inreg %reg) {
 ; GFX9-LABEL: s_add_v2i64_splat_const_low_bits_known0_0:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_add_i32 s3, s3, 1
 ; GFX9-NEXT:    s_add_i32 s1, s1, 1
+; GFX9-NEXT:    s_add_i32 s3, s3, 1
 ; GFX9-NEXT:    ; return to shader part epilog
   %add = add <2 x i64> %reg, <i64 4294967296, i64 4294967296> ; (1 << 32)
   ret <2 x i64> %add
@@ -158,8 +158,8 @@ define amdgpu_ps <2 x i64> @s_add_v2i64_splat_const_low_bits_known0_0(<2 x i64>
 define amdgpu_ps <2 x i64> @s_add_v2i64_nonsplat_const_low_bits_known0_0(<2 x i64> inreg %reg) {
 ; GFX9-LABEL: s_add_v2i64_nonsplat_const_low_bits_known0_0:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_add_i32 s3, s3, 2
 ; GFX9-NEXT:    s_add_i32 s1, s1, 1
+; GFX9-NEXT:    s_add_i32 s3, s3, 2
 ; GFX9-NEXT:    ; return to shader part epilog
   %add = add <2 x i64> %reg, <i64 4294967296, i64 8589934592> ; (1 << 32), (1 << 33)
   ret <2 x i64> %add
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
index 37928a78622a6..6885657bbfa36 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
@@ -110,11 +110,9 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_offset_rtn(double %val, <4 x i32
   ; GFX90A_GFX942-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFSET_RTN [[COPY7]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY8]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY9]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY8]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY9]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret double %ret
@@ -138,11 +136,9 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_offen_rtn(double %val, <4 x i32>
   ; GFX90A_GFX942-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFEN_RTN [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY9]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY9]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY10]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret double %ret
@@ -166,11 +162,9 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_idxen_rtn(double %val, <4 x i32>
   ; GFX90A_GFX942-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_IDXEN_RTN [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY9]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY9]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY10]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret double %ret
@@ -196,11 +190,9 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_bothen_rtn(double %val, <4 x i32
   ; GFX90A_GFX942-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN [[COPY9]], killed [[REG_SEQUENCE2]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY11]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY10]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY11]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret double %ret
@@ -342,11 +334,9 @@ define amdgpu_ps double @buffer_ptr_atomic_fadd_f64_offset_rtn(double %val, ptr
   ; GFX90A_GFX942-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFSET_RTN [[COPY11]], killed [[REG_SEQUENCE2]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64) on %ir.rsrc, align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY12]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY13]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %val, ptr addrspace(8) %rsrc, i32 0, i32 %soffset, i32 0)
   ret double %ret
@@ -376,11 +366,9 @@ define amdgpu_ps double @buffer_ptr_atomic_fadd_f64_offen_rtn(double %val, ptr a
   ; GFX90A_GFX942-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFEN_RTN [[COPY12]], [[COPY1]], killed [[REG_SEQUENCE2]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64) on %ir.rsrc, align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY14]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY13]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY14]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret double %ret
@@ -410,11 +398,9 @@ define amdgpu_ps double @buffer_ptr_atomic_fadd_f64_idxen_rtn(double %val, ptr a
   ; GFX90A_GFX942-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_IDXEN_RTN [[COPY12]], [[COPY1]], killed [[REG_SEQUENCE2]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64) on %ir.rsrc, align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY14]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY13]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY14]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret double %ret
@@ -446,11 +432,9 @@ define amdgpu_ps double @buffer_ptr_atomic_fadd_f64_bothen_rtn(double %val, ptr
   ; GFX90A_GFX942-NEXT:   [[COPY13:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
   ; GFX90A_GFX942-NEXT:   [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN [[COPY13]], killed [[REG_SEQUENCE4]], killed [[REG_SEQUENCE2]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64) on %ir.rsrc, align 1, addrspace 8)
   ; GFX90A_GFX942-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY14]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY15]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY14]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY15]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret double %ret
diff --git a/llvm/test/CodeGen/AMDGPU/constrained-shift.ll b/llvm/test/CodeGen/AMDGPU/constrained-shift.ll
index af4ca2ad7120a..661af021e8a84 100644
--- a/llvm/test/CodeGen/AMDGPU/constrained-shift.ll
+++ b/llvm/test/CodeGen/AMDGPU/constrained-shift.ll
@@ -168,26 +168,26 @@ define <4 x i32> @csh_v4i32(<4 x i32> %a, <4 x i32> %b) {
 define amdgpu_ps <4 x i32> @s_csh_v4i32(<4 x i32> inreg %a, <4 x i32> inreg %b) {
 ; CHECK-LABEL: s_csh_v4i32:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_lshl_b32 s8, s3, s7
-; CHECK-NEXT:    s_lshl_b32 s9, s2, s6
-; CHECK-NEXT:    s_lshl_b32 s10, s1, s5
-; CHECK-NEXT:    s_lshl_b32 s11, s0, s4
-; CHECK-NEXT:    s_lshr_b32 s12, s3, s7
-; CHECK-NEXT:    s_lshr_b32 s13, s2, s6
-; CHECK-NEXT:    s_lshr_b32 s14, s1, s5
-; CHECK-NEXT:    s_lshr_b32 s15, s0, s4
-; CHECK-NEXT:    s_ashr_i32 s0, s0, s4
-; CHECK-NEXT:    s_ashr_i32 s1, s1, s5
-; CHECK-NEXT:    s_ashr_i32 s2, s2, s6
+; CHECK-NEXT:    s_lshl_b32 s8, s0, s4
+; CHECK-NEXT:    s_lshl_b32 s9, s1, s5
+; CHECK-NEXT:    s_lshl_b32 s10, s2, s6
+; CHECK-NEXT:    s_lshl_b32 s11, s3, s7
+; CHECK-NEXT:    s_lshr_b32 s12, s0, s4
+; CHECK-NEXT:    s_lshr_b32 s13, s1, s5
+; CHECK-NEXT:    s_lshr_b32 s14, s2, s6
+; CHECK-NEXT:    s_lshr_b32 s15, s3, s7
 ; CHECK-NEXT:    s_ashr_i32 s3, s3, s7
+; CHECK-NEXT:    s_ashr_i32 s2, s2, s6
+; CHECK-NEXT:    s_ashr_i32 s1, s1, s5
+; CHECK-NEXT:    s_ashr_i32 s0, s0, s4
 ; CHECK-NEXT:    s_add_i32 s4, s11, s15
 ; CHECK-NEXT:    s_add_i32 s5, s10, s14
 ; CHECK-NEXT:    s_add_i32 s6, s9, s13
 ; CHECK-NEXT:    s_add_i32 s7, s8, s12
-; CHECK-NEXT:    s_add_i32 s3, s7, s3
-; CHECK-NEXT:    s_add_i32 s2, s6, s2
-; CHECK-NEXT:    s_add_i32 s1, s5, s1
-; CHECK-NEXT:    s_add_i32 s0, s4, s0
+; CHECK-NEXT:    s_add_i32 s0, s7, s0
+; CHECK-NEXT:    s_add_i32 s1, s6, s1
+; CHECK-NEXT:    s_add_i32 s2, s5, s2
+; CHECK-NEXT:    s_add_i32 s3, s4, s3
 ; CHECK-NEXT:    ; return to shader part epilog
 ;
 ; GISEL-LABEL: s_csh_v4i32:
diff --git a/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll b/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll
index d63a36c4b2958..4b4718a2acb80 100644
--- a/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll
+++ b/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll
@@ -10,10 +10,8 @@ define amdgpu_ps i32 @s_or_i32_disjoint(i32 inreg %a, i32 inreg %b) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = disjoint S_OR_B32 [[COPY1]], [[COPY]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY2]], implicit $exec
-  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; CHECK-NEXT:   %3:sreg_32 = disjoint S_OR_B32 [[COPY1]], [[COPY]], implicit-def dead $scc
+  ; CHECK-NEXT:   $sgpr0 = COPY %3
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0
   %result = or disjoint i32 %a, %b
   ret i32 %result
@@ -28,14 +26,10 @@ define amdgpu_ps <2 x i32> @s_or_v2i32_disjoint(<2 x i32> inreg %a, <2 x i32> in
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr2
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = disjoint S_OR_B32 [[COPY2]], [[COPY]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32 = disjoint S_OR_B32 [[COPY3]], [[COPY1]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_1]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY4]], implicit $exec
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
-  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; CHECK-NEXT:   %5:sreg_32 = disjoint S_OR_B32 [[COPY3]], [[COPY1]], implicit-def dead $scc
+  ; CHECK-NEXT:   %6:sreg_32 = disjoint S_OR_B32 [[COPY2]], [[COPY]], implicit-def dead $scc
+  ; CHECK-NEXT:   $sgpr0 = COPY %5
+  ; CHECK-NEXT:   $sgpr1 = COPY %6
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %result = or disjoint <2 x i32> %a, %b
   ret <2 x i32> %result
@@ -48,8 +42,8 @@ define i32 @v_or_i32_disjoint(i32 %a, i32 %b) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; CHECK-NEXT:   [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = disjoint V_OR_B32_e64 [[COPY1]], [[COPY]], implicit $exec
-  ; CHECK-NEXT:   $vgpr0 = COPY [[V_OR_B32_e64_]]
+  ; CHECK-NEXT:   %10:vgpr_32 = disjoint V_OR_B32_e64 [[COPY1]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY %10
   ; CHECK-NEXT:   SI_RETURN implicit $vgpr0
   %result = or disjoint i32 %a, %b
   ret i32 %result
@@ -64,10 +58,10 @@ define <2 x i32> @v_or_v2i32_disjoint(<2 x i32> %a, <2 x i32> %b) {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; CHECK-NEXT:   [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = disjoint V_OR_B32_e64 [[COPY3]], [[COPY1]], implicit $exec
-  ; CHECK-NEXT:   [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = disjoint V_OR_B32_e64 [[COPY2]], [[COPY]], implicit $exec
-  ; CHECK-NEXT:   $vgpr0 = COPY [[V_OR_B32_e64_]]
-  ; CHECK-NEXT:   $vgpr1 = COPY [[V_OR_B32_e64_1]]
+  ; CHECK-NEXT:   %12:vgpr_32 = disjoint V_OR_B32_e64 [[COPY3]], [[COPY1]], implicit $exec
+  ; CHECK-NEXT:   %13:vgpr_32 = disjoint V_OR_B32_e64 [[COPY2]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY %12
+  ; CHECK-NEXT:   $vgpr1 = COPY %13
   ; CHECK-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
   %result = or disjoint <2 x i32> %a, %b
   ret <2 x i32> %result
@@ -84,15 +78,11 @@ define amdgpu_ps i64 @s_or_i64_disjoint(i64 inreg %a, i64 inreg %b) {
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
-  ; CHECK-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = disjoint S_OR_B64 killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub1
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
-  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
-  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
-  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; CHECK-NEXT:   %7:sreg_64 = disjoint S_OR_B64 killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY %7.sub1
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY %7.sub0
+  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY5]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[COPY4]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %result = or disjoint i64 %a, %b
   ret i64 %result
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/flat-atomic-fadd.f64.ll
index f2f8c0a5cfa8f..36714b386e7e5 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomic-fadd.f64.ll
@@ -44,11 +44,9 @@ define amdgpu_ps double @flat_atomic_fadd_f64_rtn_intrinsic(ptr %ptr, double %da
   ; GFX90A_GFX942-NEXT:   [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
   ; GFX90A_GFX942-NEXT:   [[FLAT_ATOMIC_ADD_F64_RTN:%[0-9]+]]:vreg_64_align2 = FLAT_ATOMIC_ADD_F64_RTN killed [[COPY4]], killed [[COPY5]], 0, 1, implicit $exec, implicit $flat_scr :: (load store syncscope("agent") seq_cst (s64) on %ir.ptr)
   ; GFX90A_GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY6]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY7]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = call double @llvm.amdgcn.flat.atomic.fadd.f64.p1.f64(ptr %ptr, double %data)
   ret double %ret
@@ -119,11 +117,9 @@ define amdgpu_ps double @flat_atomic_fadd_f64_rtn_atomicrmw(ptr %ptr, double %da
   ; GFX90A_GFX942-NEXT:   [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
   ; GFX90A_GFX942-NEXT:   [[FLAT_ATOMIC_ADD_F64_RTN:%[0-9]+]]:vreg_64_align2 = FLAT_ATOMIC_ADD_F64_RTN killed [[COPY4]], killed [[COPY5]], 0, 1, implicit $exec, implicit $flat_scr :: (load store syncscope("wavefront") monotonic (s64) on %ir.ptr)
   ; GFX90A_GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY6]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY7]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = atomicrmw fadd ptr %ptr, double %data syncscope("wavefront") monotonic, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
   ret double %ret
@@ -148,11 +144,9 @@ define amdgpu_ps double @flat_atomic_fadd_f64_rtn_atomicrmw__noprivate(ptr %ptr,
   ; GFX90A_GFX942-NEXT:   [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
   ; GFX90A_GFX942-NEXT:   [[FLAT_ATOMIC_ADD_F64_RTN:%[0-9]+]]:vreg_64_align2 = FLAT_ATOMIC_ADD_F64_RTN killed [[COPY4]], killed [[COPY5]], 0, 1, implicit $exec, implicit $flat_scr :: (load store syncscope("wavefront") monotonic (s64) on %ir.ptr)
   ; GFX90A_GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[FLAT_ATOMIC_ADD_F64_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY6]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY7]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = atomicrmw fadd ptr %ptr, double %data syncscope("wavefront") monotonic, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
   ret double %ret
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f64.ll
index 682c1cd8060aa..fa4e7f87853dd 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f64.ll
@@ -62,11 +62,9 @@ define amdgpu_ps double @global_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(1) %
   ; GFX90A-NEXT:   [[PHI3:%[0-9]+]]:sreg_64 = PHI [[SI_IF_BREAK]], %bb.1
   ; GFX90A-NEXT:   SI_END_CF [[PHI3]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
   ; GFX90A-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[PHI2]].sub0
-  ; GFX90A-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
   ; GFX90A-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[PHI2]].sub1
-  ; GFX90A-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
-  ; GFX90A-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A-NEXT:   $sgpr0 = COPY [[COPY12]]
+  ; GFX90A-NEXT:   $sgpr1 = COPY [[COPY13]]
   ; GFX90A-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   ;
   ; GFX942-LABEL: name: global_atomic_fadd_f64_rtn_atomicrmw
@@ -83,11 +81,9 @@ define amdgpu_ps double @global_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(1) %
   ; GFX942-NEXT:   [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
   ; GFX942-NEXT:   [[GLOBAL_ATOMIC_ADD_F64_RTN:%[0-9]+]]:vreg_64_align2 = GLOBAL_ATOMIC_ADD_F64_RTN killed [[COPY4]], killed [[COPY5]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s64) on %ir.ptr, addrspace 1)
   ; GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_ADD_F64_RTN]].sub0
-  ; GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
   ; GFX942-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_ADD_F64_RTN]].sub1
-  ; GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY7]], implicit $exec
-  ; GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX942-NEXT:   $sgpr0 = COPY [[COPY6]]
+  ; GFX942-NEXT:   $sgpr1 = COPY [[COPY7]]
   ; GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double %data syncscope("wavefront") monotonic
   ret double %ret
@@ -127,11 +123,9 @@ define amdgpu_ps double @global_atomic_fadd_f64_saddr_rtn_atomicrmw(ptr addrspac
   ; GFX90A_GFX942-NEXT:   [[COPY4:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
   ; GFX90A_GFX942-NEXT:   [[GLOBAL_ATOMIC_ADD_F64_SADDR_RTN:%[0-9]+]]:vreg_64_align2 = GLOBAL_ATOMIC_ADD_F64_SADDR_RTN killed [[V_MOV_B32_e32_]], killed [[COPY4]], killed [[REG_SEQUENCE]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s64) on %ir.ptr, addrspace 1)
   ; GFX90A_GFX942-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_ADD_F64_SADDR_RTN]].sub0
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
   ; GFX90A_GFX942-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_ATOMIC_ADD_F64_SADDR_RTN]].sub1
-  ; GFX90A_GFX942-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY6]], implicit $exec
-  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX90A_GFX942-NEXT:   $sgpr0 = COPY [[COPY5]]
+  ; GFX90A_GFX942-NEXT:   $sgpr1 = COPY [[COPY6]]
   ; GFX90A_GFX942-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
   ret double %ret
diff --git a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
index e82801eadc936..c88113d62a887 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
@@ -358,8 +358,8 @@ define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
 define amdgpu_ps <2 x i32> @s_uitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX7-LABEL: s_uitofp_v2i1_to_v2bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s2, 1, s1
-; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_and_b32 s2, 1, s0
+; GFX7-NEXT:    s_bitcmp1_b32 s1, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
@@ -367,14 +367,14 @@ define amdgpu_ps <2 x i32> @s_uitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uitofp_v2i1_to_v2bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s2, 1, s1
-; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_and_b32 s2, 1, s0
+; GFX9-NEXT:    s_bitcmp1_b32 s1, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
@@ -396,75 +396,77 @@ define amdgpu_ps <2 x i32> @s_uitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_uitofp_v2i1_to_v2bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s1, 1, s1
-; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
+; GFX11-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
 ; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
-; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v1
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_uitofp_v2i1_to_v2bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s1, 1, s1
-; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
+; GFX12-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
 ; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
   %op = uitofp <2 x i1> %num to <2 x bfloat>
@@ -704,11 +706,11 @@ define <3 x bfloat> @v_uitofp_v3i1_to_v3bf16(<3 x i1> %num) {
 define amdgpu_ps <3 x i32> @s_uitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX7-LABEL: s_uitofp_v3i1_to_v3bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s4, 1, s2
-; GFX7-NEXT:    s_and_b32 s2, 1, s1
-; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_and_b32 s4, 1, s0
+; GFX7-NEXT:    s_and_b32 s3, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s2, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX7-NEXT:    s_cmp_eq_u32 s3, 1
 ; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s4, 1
 ; GFX7-NEXT:    s_cselect_b64 s[4:5], -1, 0
@@ -718,18 +720,18 @@ define amdgpu_ps <3 x i32> @s_uitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uitofp_v3i1_to_v3bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s4, 1, s2
-; GFX9-NEXT:    s_and_b32 s2, 1, s1
-; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_and_b32 s4, 1, s0
+; GFX9-NEXT:    s_and_b32 s3, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s2, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX9-NEXT:    s_cmp_eq_u32 s3, 1
 ; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s4, 1
 ; GFX9-NEXT:    s_cselect_b64 s[4:5], -1, 0
@@ -757,96 +759,99 @@ define amdgpu_ps <3 x i32> @s_uitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
 ; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_uitofp_v3i1_to_v3bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
 ; GFX11-NEXT:    s_and_b32 s1, 1, s1
-; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
 ; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s2
 ; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
 ; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
 ; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s0
-; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v1
+; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
 ; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
-; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v2
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v2
+; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_uitofp_v3i1_to_v3bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
 ; GFX12-NEXT:    s_and_b32 s1, 1, s1
-; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
 ; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s2
 ; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
 ; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s0
-; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
 ; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX12-NEXT:    v_add_nc_u32_e32 v4, v4, v1
-; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v2
+; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX12-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
+; GFX12-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
@@ -1149,10 +1154,10 @@ define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
 define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX7-LABEL: s_uitofp_v4i1_to_v4bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s6, 1, s3
-; GFX7-NEXT:    s_and_b32 s4, 1, s2
-; GFX7-NEXT:    s_and_b32 s2, 1, s1
-; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_and_b32 s6, 1, s0
+; GFX7-NEXT:    s_and_b32 s4, 1, s1
+; GFX7-NEXT:    s_and_b32 s2, 1, s2
+; GFX7-NEXT:    s_bitcmp1_b32 s3, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -1168,18 +1173,18 @@ define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v3
-; GFX7-NEXT:    v_readfirstlane_b32 s1, v2
-; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uitofp_v4i1_to_v4bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s6, 1, s3
-; GFX9-NEXT:    s_and_b32 s4, 1, s2
-; GFX9-NEXT:    s_and_b32 s2, 1, s1
-; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_and_b32 s6, 1, s0
+; GFX9-NEXT:    s_and_b32 s4, 1, s1
+; GFX9-NEXT:    s_and_b32 s2, 1, s2
+; GFX9-NEXT:    s_bitcmp1_b32 s3, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -1211,7 +1216,7 @@ define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s[0:1]
 ; GFX9-NEXT:    v_bfe_u32 v5, v3, 16, 1
@@ -1220,126 +1225,133 @@ define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v5, 0x7fff, v5
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_uitofp_v4i1_to_v4bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s3, 1, s3
-; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
 ; GFX11-NEXT:    s_and_b32 s1, 1, s1
-; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s0
-; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
 ; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s3
 ; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s3, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s2
-; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s2
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    v_bfe_u32 v7, v4, 16, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s3
-; GFX11-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
 ; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v10, 0x400000, v4
-; GFX11-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX11-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v0
-; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v6, v6, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-NEXT:    v_or_b32_e32 v11, 0x400000, v3
-; GFX11-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
 ; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v4
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v4
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v3
-; GFX11-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_uitofp_v4i1_to_v4bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s3, 1, s3
-; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
 ; GFX12-NEXT:    s_and_b32 s1, 1, s1
-; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s0
-; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
 ; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s3
 ; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s3, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s2
-; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s2
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    v_bfe_u32 v7, v4, 16, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s3
-; GFX12-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
 ; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v10, 0x400000, v4
-; GFX12-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX12-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v0
-; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
 ; GFX12-NEXT:    v_add_nc_u32_e32 v6, v6, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    v_or_b32_e32 v11, 0x400000, v3
-; GFX12-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
 ; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX12-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
-; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v4
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_readfirstlane_b32 s1, v3
-; GFX12-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
   %op = uitofp <4 x i1> %num to <4 x bfloat>
@@ -1700,8 +1712,8 @@ define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
 define amdgpu_ps <2 x i32> @s_sitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX7-LABEL: s_sitofp_v2i1_to_v2bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s2, 1, s0
-; GFX7-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[0:1]
@@ -1709,14 +1721,14 @@ define amdgpu_ps <2 x i32> @s_sitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s[0:1]
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_sitofp_v2i1_to_v2bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s2, 1, s0
-; GFX9-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[0:1]
@@ -1738,77 +1750,75 @@ define amdgpu_ps <2 x i32> @s_sitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_sitofp_v2i1_to_v2bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s0, 1, s0
-; GFX11-NEXT:    s_bitcmp1_b32 s1, 0
-; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
 ; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
-; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v1
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
 ; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_sitofp_v2i1_to_v2bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s0, 1, s0
-; GFX12-NEXT:    s_bitcmp1_b32 s1, 0
-; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
 ; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v1
 ; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
   %op = sitofp <2 x i1> %num to <2 x bfloat>
@@ -2048,11 +2058,11 @@ define <3 x bfloat> @v_sitofp_v3i1_to_v3bf16(<3 x i1> %num) {
 define amdgpu_ps <3 x i32> @s_sitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX7-LABEL: s_sitofp_v3i1_to_v3bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s4, 1, s0
-; GFX7-NEXT:    s_and_b32 s3, 1, s1
-; GFX7-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX7-NEXT:    s_and_b32 s4, 1, s2
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX7-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s4, 1
 ; GFX7-NEXT:    s_cselect_b64 s[4:5], -1, 0
@@ -2062,18 +2072,18 @@ define amdgpu_ps <3 x i32> @s_sitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_sitofp_v3i1_to_v3bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s4, 1, s0
-; GFX9-NEXT:    s_and_b32 s3, 1, s1
-; GFX9-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX9-NEXT:    s_and_b32 s4, 1, s2
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s4, 1
 ; GFX9-NEXT:    s_cselect_b64 s[4:5], -1, 0
@@ -2101,99 +2111,96 @@ define amdgpu_ps <3 x i32> @s_sitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
 ; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_sitofp_v3i1_to_v3bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s0, 1, s0
+; GFX11-NEXT:    s_and_b32 s2, 1, s2
 ; GFX11-NEXT:    s_and_b32 s1, 1, s1
-; GFX11-NEXT:    s_bitcmp1_b32 s2, 0
-; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
 ; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
 ; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
-; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s0
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v1
 ; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
 ; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v2
-; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
 ; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_sitofp_v3i1_to_v3bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s0, 1, s0
+; GFX12-NEXT:    s_and_b32 s2, 1, s2
 ; GFX12-NEXT:    s_and_b32 s1, 1, s1
-; GFX12-NEXT:    s_bitcmp1_b32 s2, 0
-; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s2
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
 ; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
 ; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
-; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
-; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s0
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v1
 ; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
 ; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v2
-; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
 ; GFX12-NEXT:    v_add_nc_u32_e32 v4, v4, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX12-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
@@ -2496,10 +2503,10 @@ define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
 define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX7-LABEL: s_sitofp_v4i1_to_v4bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s6, 1, s0
-; GFX7-NEXT:    s_and_b32 s4, 1, s1
-; GFX7-NEXT:    s_and_b32 s2, 1, s2
-; GFX7-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX7-NEXT:    s_and_b32 s6, 1, s3
+; GFX7-NEXT:    s_and_b32 s4, 1, s2
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -2515,18 +2522,18 @@ define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_sitofp_v4i1_to_v4bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_and_b32 s6, 1, s0
-; GFX9-NEXT:    s_and_b32 s4, 1, s1
-; GFX9-NEXT:    s_and_b32 s2, 1, s2
-; GFX9-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX9-NEXT:    s_and_b32 s6, 1, s3
+; GFX9-NEXT:    s_and_b32 s4, 1, s2
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
 ; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -2558,7 +2565,7 @@ define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s[0:1]
 ; GFX9-NEXT:    v_bfe_u32 v5, v3, 16, 1
@@ -2567,133 +2574,126 @@ define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
 ; GFX9-NEXT:    v_add_u32_e32 v5, 0x7fff, v5
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
-; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_sitofp_v4i1_to_v4bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_and_b32 s0, 1, s0
-; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_and_b32 s3, 1, s3
 ; GFX11-NEXT:    s_and_b32 s2, 1, s2
-; GFX11-NEXT:    s_bitcmp1_b32 s3, 0
-; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s3
-; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s2
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s0
 ; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s1
-; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s2
+; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
 ; GFX11-NEXT:    v_bfe_u32 v7, v4, 16, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
-; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s3
 ; GFX11-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v0
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v10, 0x400000, v4
 ; GFX11-NEXT:    v_add_nc_u32_e32 v7, v7, v4
-; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v0
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v6, v6, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_or_b32_e32 v11, 0x400000, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX11-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
-; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
 ; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 16, v3
+; GFX11-NEXT:    v_ashrrev_i32_e32 v3, 16, v4
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v2
-; GFX11-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s2, v3
-; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: s_sitofp_v4i1_to_v4bf16:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_and_b32 s0, 1, s0
-; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_and_b32 s3, 1, s3
 ; GFX12-NEXT:    s_and_b32 s2, 1, s2
-; GFX12-NEXT:    s_bitcmp1_b32 s3, 0
-; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s3
-; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s2
+; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s0
 ; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
-; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
-; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s1
-; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s2
+; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
 ; GFX12-NEXT:    v_bfe_u32 v7, v4, 16, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
-; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s3
 ; GFX12-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v0
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v10, 0x400000, v4
 ; GFX12-NEXT:    v_add_nc_u32_e32 v7, v7, v4
-; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
 ; GFX12-NEXT:    v_add_nc_u32_e32 v6, v6, v3
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_or_b32_e32 v11, 0x400000, v3
+; GFX12-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
-; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX12-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
-; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
 ; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
-; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_readfirstlane_b32 s1, v2
-; GFX12-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX12-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_readfirstlane_b32 s2, v3
-; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 16, v3
+; GFX12-NEXT:    v_ashrrev_i32_e32 v3, 16, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX12-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    ; return to shader part epilog
   %op = sitofp <4 x i1> %num to <4 x bfloat>
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
index dfde10329fe80..3aa5ea995559f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
@@ -11,18 +11,12 @@ define amdgpu_ps ptr addrspace(8) @basic_raw_buffer(ptr inreg %p) {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
   ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], killed [[S_MOV_B32_]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_AND_B32_]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY2]], implicit $exec
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
   ; CHECK-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1234
-  ; CHECK-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[S_MOV_B32_1]]
-  ; CHECK-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 5678
-  ; CHECK-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[S_MOV_B32_3]]
-  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_]]
-  ; CHECK-NEXT:   $sgpr2 = COPY [[S_MOV_B32_2]]
-  ; CHECK-NEXT:   $sgpr3 = COPY [[S_MOV_B32_4]]
+  ; CHECK-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 5678
+  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY1]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[S_AND_B32_]]
+  ; CHECK-NEXT:   $sgpr2 = COPY [[S_MOV_B32_1]]
+  ; CHECK-NEXT:   $sgpr3 = COPY [[S_MOV_B32_2]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3
   %rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 0, i32 1234, i32 5678)
   ret ptr addrspace(8) %rsrc
@@ -58,18 +52,12 @@ define amdgpu_ps ptr addrspace(8) @basic_struct_buffer(ptr inreg %p) {
   ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], killed [[S_MOV_B32_]], implicit-def dead $scc
   ; CHECK-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 262144
   ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 killed [[S_AND_B32_]], killed [[S_MOV_B32_1]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY2]], implicit $exec
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
   ; CHECK-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 1234
-  ; CHECK-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[S_MOV_B32_2]]
-  ; CHECK-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 5678
-  ; CHECK-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[S_MOV_B32_4]]
-  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_]]
-  ; CHECK-NEXT:   $sgpr2 = COPY [[S_MOV_B32_3]]
-  ; CHECK-NEXT:   $sgpr3 = COPY [[S_MOV_B32_5]]
+  ; CHECK-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 5678
+  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY1]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   $sgpr2 = COPY [[S_MOV_B32_2]]
+  ; CHECK-NEXT:   $sgpr3 = COPY [[S_MOV_B32_3]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3
   %rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 4, i32 1234, i32 5678)
   ret ptr addrspace(8) %rsrc
@@ -88,18 +76,10 @@ define amdgpu_ps ptr addrspace(8) @variable_top_half(ptr inreg %p, i32 inreg %nu
   ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY2]], killed [[S_MOV_B32_]], implicit-def dead $scc
   ; CHECK-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 262144
   ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 killed [[S_AND_B32_]], killed [[S_MOV_B32_1]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY4]], implicit $exec
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_]]
-  ; CHECK-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
-  ; CHECK-NEXT:   $sgpr3 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   $sgpr2 = COPY [[COPY1]]
+  ; CHECK-NEXT:   $sgpr3 = COPY [[COPY]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3
   %rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 4, i32 %numVals, i32 %flags)
   ret ptr addrspace(8) %rsrc
@@ -119,18 +99,10 @@ define amdgpu_ps ptr addrspace(8) @general_case(ptr inreg %p, i16 inreg %stride,
   ; CHECK-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY3]], killed [[S_MOV_B32_]], implicit-def dead $scc
   ; CHECK-NEXT:   [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], 16, implicit-def dead $scc
   ; CHECK-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 killed [[S_AND_B32_]], killed [[S_LSHL_B32_]], implicit-def dead $scc
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 killed [[COPY5]], implicit $exec
-  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
-  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_]]
-  ; CHECK-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
-  ; CHECK-NEXT:   $sgpr3 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY4]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[S_OR_B32_]]
+  ; CHECK-NEXT:   $sgpr2 = COPY [[COPY1]]
+  ; CHECK-NEXT:   $sgpr3 = COPY [[COPY]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3
   %rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 %stride, i32 %numVals, i32 %flags)
   ret ptr addrspace(8) %rsrc
diff --git a/llvm/test/CodeGen/AMDGPU/ptrmask.ll b/llvm/test/CodeGen/AMDGPU/ptrmask.ll
index 9ad9c80d82ff3..8594549318dda 100644
--- a/llvm/test/CodeGen/AMDGPU/ptrmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptrmask.ll
@@ -186,10 +186,8 @@ define ptr addrspace(8) @v_ptrmask_buffer_resource_variable_i128_neg8(ptr addrsp
 define amdgpu_ps ptr addrspace(8) @s_ptrmask_buffer_resource_variable_i128(ptr addrspace(8) inreg %ptr, i128 inreg %mask) {
 ; GCN-LABEL: s_ptrmask_buffer_resource_variable_i128:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], s[8:9]
 ; GCN-NEXT:    s_and_b64 s[0:1], s[2:3], s[6:7]
-; GCN-NEXT:    s_mov_b32 s2, s4
-; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_and_b64 s[2:3], s[4:5], s[8:9]
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX10PLUS-LABEL: s_ptrmask_buffer_resource_variable_i128:
diff --git a/llvm/test/CodeGen/AMDGPU/sdag-print-divergence.ll b/llvm/test/CodeGen/AMDGPU/sdag-print-divergence.ll
index 482f78889ff4e..695d5225421de 100644
--- a/llvm/test/CodeGen/AMDGPU/sdag-print-divergence.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdag-print-divergence.ll
@@ -1,4 +1,3 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -O0 -verify-machineinstrs < %s -debug-only=isel 2>&1 | FileCheck --check-prefixes=GCN,GCN-DEFAULT %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -O0 -verify-machineinstrs < %s -debug-only=isel -dag-dump-verbose 2>&1 | FileCheck --check-prefixes=GCN,GCN-VERBOSE %s
 
@@ -6,25 +5,23 @@
 
 ; GCN-LABEL: === test_sdag_dump
 ; GCN: Initial selection DAG: %bb.0 'test_sdag_dump:entry'
-; GCN: SelectionDAG has 11 nodes:
+; GCN: SelectionDAG has 10 nodes:
 
 ; GCN-DEFAULT:  t0: ch,glue = EntryToken
 ; GCN-DEFAULT:  t2: f32,ch = CopyFromReg t0, Register:f32 %0
-; GCN-DEFAULT:  t7: i32 = TargetConstant<3222>
 ; GCN-DEFAULT:      t5: f32 = fadd t2, t2
 ; GCN-DEFAULT:      t4: f32,ch = CopyFromReg # D:1 t0, Register:f32 %1
-; GCN-DEFAULT:   t6: f32 = fadd # D:1 t5, t4
-; GCN-DEFAULT:  t9: ch,glue = CopyToReg # D:1 t0, Register:f32 $vgpr0, t6
-; GCN-DEFAULT:  t10: ch = RETURN_TO_EPILOG t9, Register:f32 $vgpr0, t9:1
+; GCN-DEFAULT:    t6: f32 = fadd # D:1 t5, t4
+; GCN-DEFAULT:  t8: ch,glue = CopyToReg # D:1 t0, Register:f32 $vgpr0, t6
+; GCN-DEFAULT:  t9: ch = RETURN_TO_EPILOG t8, Register:f32 $vgpr0, t8:1
 
 ; GCN-VERBOSE:  t0: ch,glue = EntryToken # D:0
 ; GCN-VERBOSE:  t2: f32,ch = CopyFromReg [ORD=1] # D:0 t0, Register:f32 %0 # D:0
-; GCN-VERBOSE:  t7: i32 = TargetConstant<3222>
 ; GCN-VERBOSE:      t5: f32 = fadd [ORD=2] # D:0 t2, t2
 ; GCN-VERBOSE:      t4: f32,ch = CopyFromReg [ORD=1] # D:1 t0, Register:f32 %1 # D:0
 ; GCN-VERBOSE:    t6: f32 = fadd [ORD=3] # D:1 t5, t4
-; GCN-VERBOSE:  t9: ch,glue = CopyToReg [ORD=4] # D:1 t0, Register:f32 $vgpr0 # D:0, t6
-; GCN-VERBOSE:  t10: ch = RETURN_TO_EPILOG [ORD=4] # D:0 t9, Register:f32 $vgpr0 # D:0, t9:1
+; GCN-VERBOSE:  t8: ch,glue = CopyToReg [ORD=4] # D:1 t0, Register:f32 $vgpr0 # D:0, t6
+; GCN-VERBOSE:  t9: ch = RETURN_TO_EPILOG [ORD=4] # D:0 t8, Register:f32 $vgpr0 # D:0, t8:1
 
 define amdgpu_ps float @test_sdag_dump(float inreg %scalar, float %vector)  {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/sub64-low-32-bits-known-zero.ll b/llvm/test/CodeGen/AMDGPU/sub64-low-32-bits-known-zero.ll
index 6e4391a5ecaab..f52f1164f2ba2 100644
--- a/llvm/test/CodeGen/AMDGPU/sub64-low-32-bits-known-zero.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub64-low-32-bits-known-zero.ll
@@ -148,8 +148,8 @@ define <2 x i64> @v_sub_v2i64_nonsplat_const_low_bits_known0_0(<2 x i64> %reg) {
 define amdgpu_ps <2 x i64> @s_sub_v2i64_splat_const_low_bits_known0_0(<2 x i64> inreg %reg) {
 ; GFX9-LABEL: s_sub_v2i64_splat_const_low_bits_known0_0:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_add_i32 s3, s3, -1
 ; GFX9-NEXT:    s_add_i32 s1, s1, -1
+; GFX9-NEXT:    s_add_i32 s3, s3, -1
 ; GFX9-NEXT:    ; return to shader part epilog
   %sub = sub <2 x i64> %reg, <i64 4294967296, i64 4294967296> ; (1 << 32)
   ret <2 x i64> %sub
@@ -158,8 +158,8 @@ define amdgpu_ps <2 x i64> @s_sub_v2i64_splat_const_low_bits_known0_0(<2 x i64>
 define amdgpu_ps <2 x i64> @s_sub_v2i64_nonsplat_const_low_bits_known0_0(<2 x i64> inreg %reg) {
 ; GFX9-LABEL: s_sub_v2i64_nonsplat_const_low_bits_known0_0:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_add_i32 s3, s3, -2
 ; GFX9-NEXT:    s_add_i32 s1, s1, -1
+; GFX9-NEXT:    s_add_i32 s3, s3, -2
 ; GFX9-NEXT:    ; return to shader part epilog
   %sub = sub <2 x i64> %reg, <i64 4294967296, i64 8589934592> ; (1 << 32), (1 << 33)
   ret <2 x i64> %sub
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-vgpr-to-sgpr-return.ll b/llvm/test/CodeGen/AMDGPU/uniform-vgpr-to-sgpr-return.ll
deleted file mode 100644
index 5476c26e39ba9..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/uniform-vgpr-to-sgpr-return.ll
+++ /dev/null
@@ -1,119 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefixes=GFX11
-
-define amdgpu_ps i32 @uniform_v_to_s_i32(float inreg %a, float inreg %b) {
-; GFX11-LABEL: uniform_v_to_s_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_max_f32_e64 v0, s0, s1
-; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    ; return to shader part epilog
-  %max0 = call float @llvm.maximum.f32(float %a, float %b)
-  %cast = bitcast float %max0 to i32
-  ret i32 %cast
-}
-
-define amdgpu_ps i64 @uniform_v_to_s_i64(double inreg %a, double inreg %b) {
-; GFX11-LABEL: uniform_v_to_s_i64:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_max_f64 v[0:1], s[0:1], s[2:3]
-; GFX11-NEXT:    v_cmp_u_f64_e64 s0, s[0:1], s[2:3]
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x7ff80000, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    ; return to shader part epilog
-  %max0 = call double @llvm.maximum.f64(double %a, double %b)
-  %cast = bitcast double %max0 to i64
-  ret i64 %cast
-}
-
-define amdgpu_ps <2 x i32> @uniform_v_to_s_2_i32(<2 x float> inreg %a, <2 x float> inreg %b) {
-; GFX11-LABEL: uniform_v_to_s_2_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_max_f32_e64 v0, s0, s2
-; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s2
-; GFX11-NEXT:    v_max_f32_e64 v1, s1, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s1, s3
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7fc00000, v1, vcc_lo
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX11-NEXT:    ; return to shader part epilog
-  %max0 = call <2 x float> @llvm.maximum.f32(<2 x float> %a, <2 x float> %b)
-  %cast = bitcast <2 x float> %max0 to <2 x i32>
-  ret <2 x i32> %cast
-}
-
-define amdgpu_ps ptr @uniform_v_to_s_ptr(ptr inreg %x) {
-; GFX11-LABEL: uniform_v_to_s_ptr:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT:    s_mov_b32 s1, 0
-; GFX11-NEXT:    flat_load_b32 v0, v[0:1]
-; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_max_f32_e32 v1, 1.0, v0
-; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v1, vcc_lo
-; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    ; return to shader part epilog
-  %val = load float, ptr %x, align 4
-  %max = call float @llvm.maximum.f32(float %val, float 1.0)
-  %int = fptoui float %max to i32
-  %ptr = inttoptr i32 %int to ptr
-  ret ptr %ptr
-}
-
-define amdgpu_ps double @uniform_v_to_s_double(double inreg %a, double inreg %b) {
-; GFX11-LABEL: uniform_v_to_s_double:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_max_f64 v[0:1], s[0:1], s[2:3]
-; GFX11-NEXT:    v_cmp_u_f64_e64 s0, s[0:1], s[2:3]
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x7ff80000, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    ; return to shader part epilog
-  %max0 = call double @llvm.maximum.f64(double %a, double %b)
-  ret double %max0
-}
-
-define amdgpu_ps <2 x i16> @uniform_v_to_s_2_i16(float inreg %a, float inreg %b) {
-; GFX11-LABEL: uniform_v_to_s_2_i16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_max_f32_e64 v0, s0, s1
-; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    ; return to shader part epilog
-  %max0 = call float @llvm.maximum.f32(float %a, float %b)
-  %cast = bitcast float %max0 to <2 x i16>
-  ret <2 x i16> %cast
-}
-
-define amdgpu_ps i16 @uniform_v_to_s_i16(half inreg %a, half inreg %b) {
-; GFX11-LABEL: uniform_v_to_s_i16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_max_f16_e64 v0, s0, s1
-; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s0, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX11-NEXT:    ; return to shader part epilog
-  %max = call half @llvm.maximum.f16(half %a, half %b)
-  %cast = bitcast half %max to i16
-  ret i16 %cast
-}
-
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
index f590324f1120d..2202b6446fd15 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
@@ -7,16 +7,16 @@ define i64 @i64_test(i64 %i) nounwind readnone {
 ; CHECK-NEXT:    t0: ch,glue = EntryToken
 ; CHECK-NEXT:    t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
 ; CHECK-NEXT:    t4: i32,ch = CopyFromReg # D:1 t0, Register:i32 %9
-; CHECK-NEXT:    t50: i64 = REG_SEQUENCE # D:1 TargetConstant:i32<60>, t2, TargetConstant:i32<3>, t4, TargetConstant:i32<11>
-; CHECK-NEXT:    t27: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc, align 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT:    t30: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc + 4, basealign 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<4>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT:    t33: v2i32 = REG_SEQUENCE # D:1 TargetConstant:i32<60>, t27, TargetConstant:i32<3>, t30, TargetConstant:i32<11>
-; CHECK-NEXT:    t10: i64 = V_ADD_U64_PSEUDO # D:1 t50, t33
-; CHECK-NEXT:    t24: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<3>
-; CHECK-NEXT:    t17: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t24
-; CHECK-NEXT:    t39: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<11>
-; CHECK-NEXT:    t19: ch,glue = CopyToReg # D:1 t17, Register:i32 $vgpr1, t39, t17:1
-; CHECK-NEXT:    t20: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t19, t19:1
+; CHECK-NEXT:    t49: i64 = REG_SEQUENCE # D:1 TargetConstant:i32<60>, t2, TargetConstant:i32<3>, t4, TargetConstant:i32<11>
+; CHECK-NEXT:    t26: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc, align 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
+; CHECK-NEXT:    t29: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc + 4, basealign 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<4>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
+; CHECK-NEXT:    t32: v2i32 = REG_SEQUENCE # D:1 TargetConstant:i32<60>, t26, TargetConstant:i32<3>, t29, TargetConstant:i32<11>
+; CHECK-NEXT:    t10: i64 = V_ADD_U64_PSEUDO # D:1 t49, t32
+; CHECK-NEXT:    t23: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<3>
+; CHECK-NEXT:    t16: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t23
+; CHECK-NEXT:    t38: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<11>
+; CHECK-NEXT:    t18: ch,glue = CopyToReg # D:1 t16, Register:i32 $vgpr1, t38, t16:1
+; CHECK-NEXT:    t19: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t18, t18:1
 ; CHECK-EMPTY:
   %loc = alloca i64, addrspace(5)
   %j = load i64, ptr addrspace(5) %loc
@@ -31,10 +31,10 @@ define i64 @i32_test(i32 %i) nounwind readnone {
 ; CHECK-NEXT:    t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
 ; CHECK-NEXT:    t6: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
 ; CHECK-NEXT:    t7: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t6, TargetConstant:i1<0>
-; CHECK-NEXT:    t15: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t7
-; CHECK-NEXT:    t23: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
-; CHECK-NEXT:    t17: ch,glue = CopyToReg t15, Register:i32 $vgpr1, t23, t15:1
-; CHECK-NEXT:    t18: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t17, t17:1
+; CHECK-NEXT:    t14: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t7
+; CHECK-NEXT:    t22: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
+; CHECK-NEXT:    t16: ch,glue = CopyToReg t14, Register:i32 $vgpr1, t22, t14:1
+; CHECK-NEXT:    t17: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t16, t16:1
 ; CHECK-EMPTY:
   %loc = alloca i32, addrspace(5)
   %j = load i32, ptr addrspace(5) %loc
@@ -48,14 +48,14 @@ define i64 @i16_test(i16 %i) nounwind readnone {
 ; CHECK:       SelectionDAG has 18 nodes:
 ; CHECK-NEXT:    t0: ch,glue = EntryToken
 ; CHECK-NEXT:    t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
-; CHECK-NEXT:    t20: i32,ch = BUFFER_LOAD_USHORT_OFFEN<Mem:(dereferenceable load (s16) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT:    t21: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t20, TargetConstant:i1<0>
-; CHECK-NEXT:    t25: i32 = S_MOV_B32 TargetConstant:i32<65535>
-; CHECK-NEXT:    t26: i32 = V_AND_B32_e64 # D:1 t21, t25
-; CHECK-NEXT:    t16: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t26
-; CHECK-NEXT:    t32: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
-; CHECK-NEXT:    t18: ch,glue = CopyToReg t16, Register:i32 $vgpr1, t32, t16:1
-; CHECK-NEXT:    t19: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t18, t18:1
+; CHECK-NEXT:    t19: i32,ch = BUFFER_LOAD_USHORT_OFFEN<Mem:(dereferenceable load (s16) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
+; CHECK-NEXT:    t20: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t19, TargetConstant:i1<0>
+; CHECK-NEXT:    t24: i32 = S_MOV_B32 TargetConstant:i32<65535>
+; CHECK-NEXT:    t25: i32 = V_AND_B32_e64 # D:1 t20, t24
+; CHECK-NEXT:    t15: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t25
+; CHECK-NEXT:    t31: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
+; CHECK-NEXT:    t17: ch,glue = CopyToReg t15, Register:i32 $vgpr1, t31, t15:1
+; CHECK-NEXT:    t18: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t17, t17:1
 ; CHECK-EMPTY:
   %loc = alloca i16, addrspace(5)
   %j = load i16, ptr addrspace(5) %loc
@@ -69,14 +69,14 @@ define i64 @i8_test(i8 %i) nounwind readnone {
 ; CHECK:       SelectionDAG has 18 nodes:
 ; CHECK-NEXT:    t0: ch,glue = EntryToken
 ; CHECK-NEXT:    t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
-; CHECK-NEXT:    t20: i32,ch = BUFFER_LOAD_UBYTE_OFFEN<Mem:(dereferenceable load (s8) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT:    t21: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t20, TargetConstant:i1<0>
-; CHECK-NEXT:    t25: i32 = S_MOV_B32 TargetConstant:i32<255>
-; CHECK-NEXT:    t26: i32 = V_AND_B32_e64 # D:1 t21, t25
-; CHECK-NEXT:    t16: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t26
-; CHECK-NEXT:    t32: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
-; CHECK-NEXT:    t18: ch,glue = CopyToReg t16, Register:i32 $vgpr1, t32, t16:1
-; CHECK-NEXT:    t19: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t18, t18:1
+; CHECK-NEXT:    t19: i32,ch = BUFFER_LOAD_UBYTE_OFFEN<Mem:(dereferenceable load (s8) from %ir.loc, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
+; CHECK-NEXT:    t20: i32,i1 = V_ADD_CO_U32_e64 # D:1 t2, t19, TargetConstant:i1<0>
+; CHECK-NEXT:    t24: i32 = S_MOV_B32 TargetConstant:i32<255>
+; CHECK-NEXT:    t25: i32 = V_AND_B32_e64 # D:1 t20, t24
+; CHECK-NEXT:    t15: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t25
+; CHECK-NEXT:    t31: i32 = V_MOV_B32_e32 TargetConstant:i32<0>
+; CHECK-NEXT:    t17: ch,glue = CopyToReg t15, Register:i32 $vgpr1, t31, t15:1
+; CHECK-NEXT:    t18: ch = SI_RETURN Register:i32 $vgpr0, Register:i32 $vgpr1, t17, t17:1
 ; CHECK-EMPTY:
   %loc = alloca i8, addrspace(5)
   %j = load i8, ptr addrspace(5) %loc

From c873ca25976d81f54421d9f4e24c5f700824d0d1 Mon Sep 17 00:00:00 2001
From: royitaqi <royitaqi@users.noreply.github.com>
Date: Mon, 21 Apr 2025 11:36:26 -0700
Subject: [PATCH 641/710] [lldb] Add symbol/table count into statistics
 (#136226)

# New stats

The following stats are added and are available in both "statistics
dump" command and in python API.

1. In summary:
1. Add `totalSymbolsLoaded`. The total number of symbols loaded in all
modules.
2. Add `totalSymbolTablesLoaded `. The total number symbol tables loaded
in all modules.
2. In each module's stats:
1. Add `symbolsLoaded`. The number of symbols loaded in the current
module.


# Example

Example `statistics dump` output:
```
(lldb) statistics dump
{
  ...,
  "modules": [
    {
      "path": "/Users/<username>/demo/simple/a.out",
      "symbolsLoaded": 6,
      ...
    },
    ...
  ],
  ...
  "totalSymbolTablesLoaded": 42,
  "totalSymbolsLoaded": 32198
}
```

# Tests

**Manual test**: Built and ran lldb on a helloworld program. Ran
`statistics dump`. Verified the above stats.

**Unit test**: Ran the following tests:
```
$ bin/lldb-dotest -p TestStats.py ~/llvm-sand/external/llvm-project/lldb/test/API/commands/statistics/basic/
...
Ran 18 tests in 192.676s

OK (skipped=3)
```
---
 lldb/include/lldb/Target/Statistics.h         |  1 +
 lldb/source/Target/Statistics.cpp             | 18 +++++++---
 .../commands/statistics/basic/TestStats.py    | 33 ++++++++++++++++---
 3 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/lldb/include/lldb/Target/Statistics.h b/lldb/include/lldb/Target/Statistics.h
index ee365357fcf31..b87a12a8ab9cd 100644
--- a/lldb/include/lldb/Target/Statistics.h
+++ b/lldb/include/lldb/Target/Statistics.h
@@ -120,6 +120,7 @@ struct ModuleStats {
   llvm::StringMap<llvm::json::Value> type_system_stats;
   double symtab_parse_time = 0.0;
   double symtab_index_time = 0.0;
+  uint32_t num_symbols_loaded = 0;
   double debug_parse_time = 0.0;
   double debug_index_time = 0.0;
   uint64_t debug_info_size = 0;
diff --git a/lldb/source/Target/Statistics.cpp b/lldb/source/Target/Statistics.cpp
index b5d2e7bda1edf..2bb93dfffa5aa 100644
--- a/lldb/source/Target/Statistics.cpp
+++ b/lldb/source/Target/Statistics.cpp
@@ -71,6 +71,7 @@ json::Value ModuleStats::ToJSON() const {
   module.try_emplace("debugInfoHadIncompleteTypes",
                      debug_info_had_incomplete_types);
   module.try_emplace("symbolTableStripped", symtab_stripped);
+  module.try_emplace("symbolsLoaded", num_symbols_loaded);
   if (!symfile_path.empty())
     module.try_emplace("symbolFilePath", symfile_path);
 
@@ -293,7 +294,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
   double debug_parse_time = 0.0;
   double debug_index_time = 0.0;
   uint32_t symtabs_loaded = 0;
-  uint32_t symtabs_saved = 0;
+  uint32_t symtabs_loaded_from_cache = 0;
+  uint32_t symtabs_saved_to_cache = 0;
   uint32_t debug_index_loaded = 0;
   uint32_t debug_index_saved = 0;
   uint64_t debug_info_size = 0;
@@ -309,6 +311,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
   uint32_t num_modules_with_variable_errors = 0;
   uint32_t num_modules_with_incomplete_types = 0;
   uint32_t num_stripped_modules = 0;
+  uint32_t num_symbols_loaded = 0;
   for (size_t image_idx = 0; image_idx < num_modules; ++image_idx) {
     Module *module = target != nullptr
                          ? target->GetImages().GetModuleAtIndex(image_idx).get()
@@ -318,12 +321,15 @@ llvm::json::Value DebuggerStats::ReportStatistics(
     module_stat.symtab_index_time = module->GetSymtabIndexTime().get().count();
     Symtab *symtab = module->GetSymtab(/*can_create=*/false);
     if (symtab) {
+      module_stat.num_symbols_loaded = symtab->GetNumSymbols();
+      num_symbols_loaded += module_stat.num_symbols_loaded;
+      ++symtabs_loaded;
       module_stat.symtab_loaded_from_cache = symtab->GetWasLoadedFromCache();
       if (module_stat.symtab_loaded_from_cache)
-        ++symtabs_loaded;
+        ++symtabs_loaded_from_cache;
       module_stat.symtab_saved_to_cache = symtab->GetWasSavedToCache();
       if (module_stat.symtab_saved_to_cache)
-        ++symtabs_saved;
+        ++symtabs_saved_to_cache;
     }
     SymbolFile *sym_file = module->GetSymbolFile(/*can_create=*/false);
     if (sym_file) {
@@ -393,8 +399,9 @@ llvm::json::Value DebuggerStats::ReportStatistics(
   json::Object global_stats{
       {"totalSymbolTableParseTime", symtab_parse_time},
       {"totalSymbolTableIndexTime", symtab_index_time},
-      {"totalSymbolTablesLoadedFromCache", symtabs_loaded},
-      {"totalSymbolTablesSavedToCache", symtabs_saved},
+      {"totalSymbolTablesLoaded", symtabs_loaded},
+      {"totalSymbolTablesLoadedFromCache", symtabs_loaded_from_cache},
+      {"totalSymbolTablesSavedToCache", symtabs_saved_to_cache},
       {"totalDebugInfoParseTime", debug_parse_time},
       {"totalDebugInfoIndexTime", debug_index_time},
       {"totalDebugInfoIndexLoadedFromCache", debug_index_loaded},
@@ -407,6 +414,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
        num_modules_with_incomplete_types},
       {"totalDebugInfoEnabled", num_debug_info_enabled_modules},
       {"totalSymbolTableStripped", num_stripped_modules},
+      {"totalSymbolsLoaded", num_symbols_loaded},
   };
 
   if (include_targets) {
diff --git a/lldb/test/API/commands/statistics/basic/TestStats.py b/lldb/test/API/commands/statistics/basic/TestStats.py
index 54881c13bcb68..0265a0d7c9948 100644
--- a/lldb/test/API/commands/statistics/basic/TestStats.py
+++ b/lldb/test/API/commands/statistics/basic/TestStats.py
@@ -159,6 +159,8 @@ def test_default_no_run(self):
         """
         self.build()
         target = self.createTestTarget()
+
+        # Verify top-level keys.
         debug_stats = self.get_stats()
         debug_stat_keys = [
             "memory",
@@ -168,6 +170,8 @@ def test_default_no_run(self):
             "totalSymbolTableIndexTime",
             "totalSymbolTablesLoadedFromCache",
             "totalSymbolTablesSavedToCache",
+            "totalSymbolsLoaded",
+            "totalSymbolTablesLoaded",
             "totalDebugInfoByteSize",
             "totalDebugInfoIndexTime",
             "totalDebugInfoIndexLoadedFromCache",
@@ -175,16 +179,35 @@ def test_default_no_run(self):
             "totalDebugInfoParseTime",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
-        stats = debug_stats["targets"][0]
-        keys_exist = [
+        self.assertGreater(debug_stats["totalSymbolsLoaded"], 0)
+        self.assertGreater(debug_stats["totalSymbolTablesLoaded"], 0)
+
+        # Verify target stats keys.
+        target_stats = debug_stats["targets"][0]
+        target_stat_keys_exist = [
             "expressionEvaluation",
             "frameVariable",
             "moduleIdentifiers",
             "targetCreateTime",
         ]
-        keys_missing = ["firstStopTime", "launchOrAttachTime"]
-        self.verify_keys(stats, '"stats"', keys_exist, keys_missing)
-        self.assertGreater(stats["targetCreateTime"], 0.0)
+        target_stat_keys_missing = ["firstStopTime", "launchOrAttachTime"]
+        self.verify_keys(
+            target_stats,
+            '"target_stats"',
+            target_stat_keys_exist,
+            target_stat_keys_missing,
+        )
+        self.assertGreater(target_stats["targetCreateTime"], 0.0)
+
+        # Verify module stats keys.
+        for module_stats in debug_stats["modules"]:
+            module_stat_keys_exist = [
+                "symbolsLoaded",
+            ]
+            self.verify_keys(
+                module_stats, '"module_stats"', module_stat_keys_exist, None
+            )
+            self.assertGreater(module_stats["symbolsLoaded"], 0)
 
     def test_default_with_run(self):
         """Test "statistics dump" when running the target to a breakpoint.

From e0c1e23b99e9719d0a01ab7dfc0807d891004bd4 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Mon, 21 Apr 2025 21:42:40 +0300
Subject: [PATCH 642/710] [TTI] Constify BasicTTIImplBase::thisT() (NFCI)
 (#136575)

The main change is making `thisT` method `const`, the rest of the
changes is fixing compilation errors (*).

(*) There are two tricky methods, `getVectorInstrCost()` and
`getIntImmCost()`.
They have several overloads; some of these overloads are typically
pulled in to derived classes using the `using` directive, and then
hidden by methods in the derived class.
The compiler does not complain if the hiding methods are not marked as
`const`, which means that clients will use the methods from the base
class. If after this change your target fails cost model tests, this
must be the reason. To resolve the issue you need  to make all hiding
overloads `const`. See the second commit in this PR.

Pull Request: https://github.com/llvm/llvm-project/pull/136575
---
 .../llvm/Analysis/TargetTransformInfoImpl.h   |  2 +
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      | 94 ++++++++++---------
 .../AArch64/AArch64TargetTransformInfo.cpp    | 47 +++++-----
 .../AArch64/AArch64TargetTransformInfo.h      | 60 ++++++------
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      | 15 ++-
 .../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 23 +++--
 .../Target/AMDGPU/R600TargetTransformInfo.cpp |  4 +-
 .../Target/AMDGPU/R600TargetTransformInfo.h   |  5 +-
 .../lib/Target/ARM/ARMTargetTransformInfo.cpp | 33 ++++---
 llvm/lib/Target/ARM/ARMTargetTransformInfo.h  | 50 +++++-----
 llvm/lib/Target/BPF/BPFTargetTransformInfo.h  |  5 +-
 .../Hexagon/HexagonTargetTransformInfo.cpp    | 25 +++--
 .../Hexagon/HexagonTargetTransformInfo.h      | 32 ++++---
 .../Target/Lanai/LanaiTargetTransformInfo.h   |  3 +-
 .../Target/NVPTX/NVPTXTargetTransformInfo.cpp |  3 +-
 .../Target/NVPTX/NVPTXTargetTransformInfo.h   |  5 +-
 .../Target/PowerPC/PPCTargetTransformInfo.cpp | 22 ++---
 .../Target/PowerPC/PPCTargetTransformInfo.h   | 30 +++---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp | 55 +++++------
 .../Target/RISCV/RISCVTargetTransformInfo.h   | 76 +++++++--------
 .../SystemZ/SystemZTargetTransformInfo.cpp    | 38 ++++----
 .../SystemZ/SystemZTargetTransformInfo.h      | 48 +++++-----
 .../WebAssemblyTargetTransformInfo.cpp        | 13 ++-
 .../WebAssemblyTargetTransformInfo.h          | 10 +-
 .../lib/Target/X86/X86TargetTransformInfo.cpp | 47 +++++-----
 llvm/lib/Target/X86/X86TargetTransformInfo.h  | 61 ++++++------
 26 files changed, 421 insertions(+), 385 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 3fe0a9101fdee..ea481baddc5c3 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -47,6 +47,8 @@ class TargetTransformInfoImplBase {
 
   const DataLayout &getDataLayout() const { return DL; }
 
+  // FIXME: It looks like this implementation is dead. All clients appear to
+  //  use the (non-const) version from `TargetTransformInfoImplCRTPBase`.
   InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr,
                              ArrayRef<const Value *> Operands, Type *AccessType,
                              TTI::TargetCostKind CostKind) const {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index f6ec21caa4d72..b42223eda9922 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -84,12 +84,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   using TTI = TargetTransformInfo;
 
   /// Helper function to access this as a T.
-  T *thisT() { return static_cast<T *>(this); }
+  const T *thisT() const { return static_cast<const T *>(this); }
 
   /// Estimate a cost of Broadcast as an extract and sequence of insert
   /// operations.
-  InstructionCost getBroadcastShuffleOverhead(FixedVectorType *VTy,
-                                              TTI::TargetCostKind CostKind) {
+  InstructionCost
+  getBroadcastShuffleOverhead(FixedVectorType *VTy,
+                              TTI::TargetCostKind CostKind) const {
     InstructionCost Cost = 0;
     // Broadcast cost is equal to the cost of extracting the zero'th element
     // plus the cost of inserting it into every element of the result vector.
@@ -105,8 +106,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   /// Estimate a cost of shuffle as a sequence of extract and insert
   /// operations.
-  InstructionCost getPermuteShuffleOverhead(FixedVectorType *VTy,
-                                            TTI::TargetCostKind CostKind) {
+  InstructionCost
+  getPermuteShuffleOverhead(FixedVectorType *VTy,
+                            TTI::TargetCostKind CostKind) const {
     InstructionCost Cost = 0;
     // Shuffle cost is equal to the cost of extracting element from its argument
     // plus the cost of inserting them onto the result vector.
@@ -129,7 +131,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost getExtractSubvectorOverhead(VectorType *VTy,
                                               TTI::TargetCostKind CostKind,
                                               int Index,
-                                              FixedVectorType *SubVTy) {
+                                              FixedVectorType *SubVTy) const {
     assert(VTy && SubVTy &&
            "Can only extract subvectors from vectors");
     int NumSubElts = SubVTy->getNumElements();
@@ -157,7 +159,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost getInsertSubvectorOverhead(VectorType *VTy,
                                              TTI::TargetCostKind CostKind,
                                              int Index,
-                                             FixedVectorType *SubVTy) {
+                                             FixedVectorType *SubVTy) const {
     assert(VTy && SubVTy &&
            "Can only insert subvectors into vectors");
     int NumSubElts = SubVTy->getNumElements();
@@ -211,7 +213,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                               bool VariableMask,
                                               bool IsGatherScatter,
                                               TTI::TargetCostKind CostKind,
-                                              unsigned AddressSpace = 0) {
+                                              unsigned AddressSpace = 0) const {
     // We cannot scalarize scalable vectors, so return Invalid.
     if (isa<ScalableVectorType>(DataTy))
       return InstructionCost::getInvalid();
@@ -299,7 +301,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// (e.g. scalarization).
   std::optional<InstructionCost> getMultipleResultIntrinsicVectorLibCallCost(
       const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind,
-      RTLIB::Libcall LC, std::optional<unsigned> CallRetElementIndex = {}) {
+      RTLIB::Libcall LC,
+      std::optional<unsigned> CallRetElementIndex = {}) const {
     Type *RetTy = ICA.getReturnType();
     // Vector variants of the intrinsic can be mapped to a vector library call.
     auto const *LibInfo = ICA.getLibInfo();
@@ -866,7 +869,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
                                            TTI::TargetCostKind CostKind,
-                                           ArrayRef<Value *> VL = {}) {
+                                           ArrayRef<Value *> VL = {}) const {
     /// FIXME: a bitfield is not a reasonable abstraction for talking about
     /// which elements are needed from a scalable vector
     if (isa<ScalableVectorType>(InTy))
@@ -917,7 +920,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// Helper wrapper for the DemandedElts variant of getScalarizationOverhead.
   InstructionCost getScalarizationOverhead(VectorType *InTy, bool Insert,
                                            bool Extract,
-                                           TTI::TargetCostKind CostKind) {
+                                           TTI::TargetCostKind CostKind) const {
     if (isa<ScalableVectorType>(InTy))
       return InstructionCost::getInvalid();
     auto *Ty = cast<FixedVectorType>(InTy);
@@ -933,7 +936,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost
   getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
                                    ArrayRef<Type *> Tys,
-                                   TTI::TargetCostKind CostKind) {
+                                   TTI::TargetCostKind CostKind) const {
     assert(Args.size() == Tys.size() && "Expected matching Args and Tys");
 
     InstructionCost Cost = 0;
@@ -963,7 +966,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost getScalarizationOverhead(VectorType *RetTy,
                                            ArrayRef<const Value *> Args,
                                            ArrayRef<Type *> Tys,
-                                           TTI::TargetCostKind CostKind) {
+                                           TTI::TargetCostKind CostKind) const {
     InstructionCost Cost = getScalarizationOverhead(
         RetTy, /*Insert*/ true, /*Extract*/ false, CostKind);
     if (!Args.empty())
@@ -1018,7 +1021,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr) {
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const {
     // Check if any of the operands are vector operands.
     const TargetLoweringBase *TLI = getTLI();
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -1147,7 +1151,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr) {
+                                 const Instruction *CxtI = nullptr) const {
     switch (improveShuffleKindFromMask(Kind, Mask, Tp, Index, SubTp)) {
     case TTI::SK_Broadcast:
       if (auto *FVT = dyn_cast<FixedVectorType>(Tp))
@@ -1175,7 +1179,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr) {
+                                   const Instruction *I = nullptr) const {
     if (BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I) == 0)
       return 0;
 
@@ -1289,7 +1293,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
           DstVTy->getElementCount().isVector()) {
         Type *SplitDstTy = VectorType::getHalfElementsVectorType(DstVTy);
         Type *SplitSrcTy = VectorType::getHalfElementsVectorType(SrcVTy);
-        T *TTI = static_cast<T *>(this);
+        const T *TTI = thisT();
         // If both types need to be split then the split is free.
         InstructionCost SplitCost =
             (!SplitSrc || !SplitDst) ? TTI->getVectorSplitCost() : 0;
@@ -1342,7 +1346,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr) {
+                                 const Instruction *I = nullptr) const {
     return BaseT::getCFInstrCost(Opcode, CostKind, I);
   }
 
@@ -1351,7 +1355,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr) {
+      const Instruction *I = nullptr) const {
     const TargetLoweringBase *TLI = getTLI();
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     assert(ISD && "Invalid opcode");
@@ -1401,7 +1405,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1) {
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const {
     return getRegUsageForType(Val->getScalarType());
   }
 
@@ -1430,10 +1435,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                        Op1);
   }
 
-  InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
-                                            int VF,
-                                            const APInt &DemandedDstElts,
-                                            TTI::TargetCostKind CostKind) {
+  InstructionCost
+  getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
+                            const APInt &DemandedDstElts,
+                            TTI::TargetCostKind CostKind) const {
     assert(DemandedDstElts.getBitWidth() == (unsigned)VF * ReplicationFactor &&
            "Unexpected size of DemandedDstElts.");
 
@@ -1463,11 +1468,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return Cost;
   }
 
-  InstructionCost
-  getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
-                  unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
-                  const Instruction *I = nullptr) {
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const {
     assert(!Src->isVoidTy() && "Invalid type");
     // Assume types, such as structs, are expensive.
     if (getTLI()->getValueType(DL, Src,  true) == MVT::Other)
@@ -1510,7 +1515,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *DataTy,
                                         Align Alignment, unsigned AddressSpace,
-                                        TTI::TargetCostKind CostKind) {
+                                        TTI::TargetCostKind CostKind) const {
     // TODO: Pass on AddressSpace when we have test coverage.
     return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
                                        CostKind);
@@ -1520,14 +1525,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
-                                         const Instruction *I = nullptr) {
+                                         const Instruction *I = nullptr) const {
     return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
                                        true, CostKind);
   }
 
   InstructionCost getExpandCompressMemoryOpCost(
       unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
-      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) {
+      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) const {
     // Treat expand load/compress store as gather/scatter operation.
     // TODO: implement more precise cost estimation for these intrinsics.
     return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
@@ -1538,7 +1543,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
-                                         const Instruction *I) {
+                                         const Instruction *I) const {
     // For a target without strided memory operations (or for an illegal
     // operation type on one which does), assume we lower to a gather/scatter
     // operation.  (Which may in turn be scalarized.)
@@ -1691,7 +1696,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   /// Get intrinsic cost based on arguments.
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind) {
+                                        TTI::TargetCostKind CostKind) const {
     // Check for generically free intrinsics.
     if (BaseT::getIntrinsicInstrCost(ICA, CostKind) == 0)
       return 0;
@@ -2095,7 +2100,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// based on types.
   InstructionCost
   getTypeBasedIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                 TTI::TargetCostKind CostKind) {
+                                 TTI::TargetCostKind CostKind) const {
     Intrinsic::ID IID = ICA.getID();
     Type *RetTy = ICA.getReturnType();
     const SmallVectorImpl<Type *> &Tys = ICA.getArgTypes();
@@ -2859,11 +2864,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// \returns The cost of Call instruction.
   InstructionCost getCallInstrCost(Function *F, Type *RetTy,
                                    ArrayRef<Type *> Tys,
-                                   TTI::TargetCostKind CostKind) {
+                                   TTI::TargetCostKind CostKind) const {
     return 10;
   }
 
-  unsigned getNumberOfParts(Type *Tp) {
+  unsigned getNumberOfParts(Type *Tp) const {
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
     if (!LT.first.isValid())
       return 0;
@@ -2907,7 +2912,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// The cost model should take into account that the actual length of the
   /// vector is reduced on each iteration.
   InstructionCost getTreeReductionCost(unsigned Opcode, VectorType *Ty,
-                                       TTI::TargetCostKind CostKind) {
+                                       TTI::TargetCostKind CostKind) const {
     // Targets must implement a default value for the scalable case, since
     // we don't know how many lanes the vector has.
     if (isa<ScalableVectorType>(Ty))
@@ -2983,7 +2988,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// fixed-width vectors here because for scalable vectors we do not know the
   /// runtime number of operations.
   InstructionCost getOrderedReductionCost(unsigned Opcode, VectorType *Ty,
-                                          TTI::TargetCostKind CostKind) {
+                                          TTI::TargetCostKind CostKind) const {
     // Targets must implement a default value for the scalable case, since
     // we don't know how many lanes the vector has.
     if (isa<ScalableVectorType>(Ty))
@@ -2999,9 +3004,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return ExtractCost + ArithCost;
   }
 
-  InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
-                                             std::optional<FastMathFlags> FMF,
-                                             TTI::TargetCostKind CostKind) {
+  InstructionCost
+  getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+                             std::optional<FastMathFlags> FMF,
+                             TTI::TargetCostKind CostKind) const {
     assert(Ty && "Unknown reduction vector type");
     if (TTI::requiresOrderedReduction(FMF))
       return getOrderedReductionCost(Opcode, Ty, CostKind);
@@ -3012,7 +3018,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   /// \param CondTy Conditional type for the Select instruction.
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
-                                         TTI::TargetCostKind CostKind) {
+                                         TTI::TargetCostKind CostKind) const {
     // Targets must implement a default value for the scalable case, since
     // we don't know how many lanes the vector has.
     if (isa<ScalableVectorType>(Ty))
@@ -3106,7 +3112,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return RedCost + MulCost + 2 * ExtCost;
   }
 
-  InstructionCost getVectorSplitCost() { return 1; }
+  InstructionCost getVectorSplitCost() const { return 1; }
 
   /// @}
 };
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 372b70a4b2d64..51fa5237fcc50 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -371,7 +371,7 @@ bool AArch64TTIImpl::shouldMaximizeVectorBandwidth(
 /// Calculate the cost of materializing a 64-bit value. This helper
 /// method might only calculate a fraction of a larger immediate. Therefore it
 /// is valid to return a cost of ZERO.
-InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) {
+InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) const {
   // Check if the immediate can be encoded within an instruction.
   if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, 64))
     return 0;
@@ -386,8 +386,9 @@ InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) {
 }
 
 /// Calculate the cost of materializing the given constant.
-InstructionCost AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
-                                              TTI::TargetCostKind CostKind) {
+InstructionCost
+AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
+                              TTI::TargetCostKind CostKind) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -577,7 +578,7 @@ static InstructionCost getHistogramCost(const IntrinsicCostAttributes &ICA) {
 
 InstructionCost
 AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                      TTI::TargetCostKind CostKind) {
+                                      TTI::TargetCostKind CostKind) const {
   // The code-generator is currently not able to handle scalable vectors
   // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
   // it. This change will be removed when code-generation for these types is
@@ -2806,7 +2807,7 @@ AArch64TTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
 
 bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
                                            ArrayRef<const Value *> Args,
-                                           Type *SrcOverrideTy) {
+                                           Type *SrcOverrideTy) const {
   // A helper that returns a vector type from the given type. The number of
   // elements in type Ty determines the vector width.
   auto toVectorTy = [&](Type *ArgTy) {
@@ -2903,7 +2904,7 @@ bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
 //   trunc i16 (lshr (add %x, %y), 1) -> i8
 //
 bool AArch64TTIImpl::isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
-                                        Type *Src) {
+                                        Type *Src) const {
   // The source should be a legal vector type.
   if (!Src->isVectorTy() || !TLI->isTypeLegal(TLI->getValueType(DL, Src)) ||
       (Src->isScalableTy() && !ST->hasSVE2()))
@@ -2948,7 +2949,7 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                                  Type *Src,
                                                  TTI::CastContextHint CCH,
                                                  TTI::TargetCostKind CostKind,
-                                                 const Instruction *I) {
+                                                 const Instruction *I) const {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
   // If the cast is observable, and it is used by a widening instruction (e.g.,
@@ -3619,7 +3620,7 @@ InstructionCost AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode,
 
 InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
                                                TTI::TargetCostKind CostKind,
-                                               const Instruction *I) {
+                                               const Instruction *I) const {
   if (CostKind != TTI::TCK_RecipThroughput)
     return Opcode == Instruction::PHI ? 0 : 1;
   assert(CostKind == TTI::TCK_RecipThroughput && "unexpected CostKind");
@@ -3630,7 +3631,7 @@ InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
 InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
     unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
     bool HasRealUse, const Instruction *I, Value *Scalar,
-    ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) {
+    ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {
   assert(Val->isVectorTy() && "This must be a vector type");
 
   if (Index != -1U) {
@@ -3802,7 +3803,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
 InstructionCost AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                                    TTI::TargetCostKind CostKind,
                                                    unsigned Index, Value *Op0,
-                                                   Value *Op1) {
+                                                   Value *Op1) const {
   bool HasRealUse =
       Opcode == Instruction::InsertElement && Op0 && !isa<UndefValue>(Op0);
   return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, HasRealUse);
@@ -3826,7 +3827,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I,
 
 InstructionCost AArch64TTIImpl::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
-    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) const {
   if (isa<ScalableVectorType>(Ty))
     return InstructionCost::getInvalid();
   if (Ty->getElementType()->isFloatingPointTy())
@@ -3840,8 +3841,7 @@ InstructionCost AArch64TTIImpl::getScalarizationOverhead(
 InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
 
   // The code-generator is currently not able to handle scalable vectors
   // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
@@ -4171,7 +4171,7 @@ InstructionCost AArch64TTIImpl::getAddressComputationCost(Type *Ty,
 InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
     unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
-    TTI::OperandValueInfo Op2Info, const Instruction *I) {
+    TTI::OperandValueInfo Op2Info, const Instruction *I) const {
   // TODO: Handle other cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
@@ -4284,7 +4284,7 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const {
 InstructionCost
 AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                       Align Alignment, unsigned AddressSpace,
-                                      TTI::TargetCostKind CostKind) {
+                                      TTI::TargetCostKind CostKind) const {
   if (useNeonVector(Src))
     return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                         CostKind);
@@ -4331,7 +4331,7 @@ static unsigned getSVEGatherScatterOverhead(unsigned Opcode,
 
 InstructionCost AArch64TTIImpl::getGatherScatterOpCost(
     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
-    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
+    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
   if (useNeonVector(DataTy) || !isLegalMaskedGatherScatter(DataTy))
     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                          Alignment, CostKind, I);
@@ -4371,7 +4371,7 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
                                                 unsigned AddressSpace,
                                                 TTI::TargetCostKind CostKind,
                                                 TTI::OperandValueInfo OpInfo,
-                                                const Instruction *I) {
+                                                const Instruction *I) const {
   EVT VT = TLI->getValueType(DL, Ty, true);
   // Type legalization can't handle structs
   if (VT == MVT::Other)
@@ -4980,7 +4980,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
 InstructionCost
 AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                        FastMathFlags FMF,
-                                       TTI::TargetCostKind CostKind) {
+                                       TTI::TargetCostKind CostKind) const {
   // The code-generator is currently not able to handle scalable vectors
   // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
   // it. This change will be removed when code-generation for these types is
@@ -5005,7 +5005,7 @@ AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
 }
 
 InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE(
-    unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) {
+    unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) const {
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
   InstructionCost LegalizationCost = 0;
   if (LT.first > 1) {
@@ -5032,7 +5032,7 @@ InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE(
 InstructionCost
 AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
                                            std::optional<FastMathFlags> FMF,
-                                           TTI::TargetCostKind CostKind) {
+                                           TTI::TargetCostKind CostKind) const {
   // The code-generator is currently not able to handle scalable vectors
   // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
   // it. This change will be removed when code-generation for these types is
@@ -5207,8 +5207,9 @@ AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
   return BaseT::getMulAccReductionCost(IsUnsigned, ResTy, VecTy, CostKind);
 }
 
-InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index,
-                                              TTI::TargetCostKind CostKind) {
+InstructionCost
+AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index,
+                              TTI::TargetCostKind CostKind) const {
   static const CostTblEntry ShuffleTbl[] = {
       { TTI::SK_Splice, MVT::nxv16i8,  1 },
       { TTI::SK_Splice, MVT::nxv8i16,  1 },
@@ -5340,7 +5341,7 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
 InstructionCost AArch64TTIImpl::getShuffleCost(
     TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
     TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
-    ArrayRef<const Value *> Args, const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
 
   // If we have a Mask, and the LT is being legalized somehow, split the Mask
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 372ec22bd548f..46d2dc57b012d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -61,7 +61,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
 
   bool isWideningInstruction(Type *DstTy, unsigned Opcode,
                              ArrayRef<const Value *> Args,
-                             Type *SrcOverrideTy = nullptr);
+                             Type *SrcOverrideTy = nullptr) const;
 
   // A helper function called by 'getVectorInstrCost'.
   //
@@ -75,7 +75,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   InstructionCost getVectorInstrCostHelper(
       unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
       bool HasRealUse, const Instruction *I = nullptr, Value *Scalar = nullptr,
-      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx = {});
+      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx = {}) const;
 
 public:
   explicit AArch64TTIImpl(const AArch64TargetMachine *TM, const Function &F)
@@ -99,9 +99,9 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   /// @{
 
   using BaseT::getIntImmCost;
-  InstructionCost getIntImmCost(int64_t Val);
+  InstructionCost getIntImmCost(int64_t Val) const;
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                TTI::TargetCostKind CostKind);
+                                TTI::TargetCostKind CostKind) const;
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,
@@ -131,7 +131,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   }
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
 
   std::optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
                                                     IntrinsicInst &II) const;
@@ -173,30 +173,32 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
 
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
 
   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
-                                         const Instruction *I = nullptr);
+                                         const Instruction *I = nullptr) const;
 
-  bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst, Type *Src);
+  bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
+                          Type *Src) const;
 
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr);
+                                   const Instruction *I = nullptr) const;
 
   InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst,
                                            VectorType *VecTy, unsigned Index);
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr);
+                                 const Instruction *I = nullptr) const;
 
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
 
   /// \param ScalarUserAndIdx encodes the information about extracts from a
   /// vector with 'Scalar' being the value being extracted,'User' being the user
@@ -213,20 +215,21 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
 
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
-                                         TTI::TargetCostKind CostKind);
+                                         TTI::TargetCostKind CostKind) const;
 
-  InstructionCost getArithmeticReductionCostSVE(unsigned Opcode,
-                                                VectorType *ValTy,
-                                                TTI::TargetCostKind CostKind);
+  InstructionCost
+  getArithmeticReductionCostSVE(unsigned Opcode, VectorType *ValTy,
+                                TTI::TargetCostKind CostKind) const;
 
   InstructionCost getSpliceCost(VectorType *Tp, int Index,
-                                TTI::TargetCostKind CostKind);
+                                TTI::TargetCostKind CostKind) const;
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
 
   InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
                                             const SCEV *Ptr);
@@ -236,17 +239,17 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr);
+      const Instruction *I = nullptr) const;
 
   TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize,
                                                     bool IsZeroCmp) const;
   bool useNeonVector(const Type *Ty) const;
 
-  InstructionCost
-  getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
-                  unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
-                  const Instruction *I = nullptr);
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const;
 
   InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys);
 
@@ -423,9 +426,10 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     return ST->hasSVE();
   }
 
-  InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
-                                             std::optional<FastMathFlags> FMF,
-                                             TTI::TargetCostKind CostKind);
+  InstructionCost
+  getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+                             std::optional<FastMathFlags> FMF,
+                             TTI::TargetCostKind CostKind) const;
 
   InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned,
                                            Type *ResTy, VectorType *ValTy,
@@ -441,13 +445,13 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr);
+                                 const Instruction *CxtI = nullptr) const;
 
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
                                            TTI::TargetCostKind CostKind,
-                                           ArrayRef<Value *> VL = {});
+                                           ArrayRef<Value *> VL = {}) const;
 
   /// Return the cost of the scaling factor used in the addressing
   /// mode represented by AM for this target, for a load/store
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 09f7877b13b3a..3f2e13f8cfc5b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -523,8 +523,7 @@ bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
 InstructionCost GCNTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
 
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
@@ -702,7 +701,7 @@ static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) {
 
 InstructionCost
 GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                  TTI::TargetCostKind CostKind) {
+                                  TTI::TargetCostKind CostKind) const {
   if (ICA.getID() == Intrinsic::fabs)
     return 0;
 
@@ -772,7 +771,7 @@ GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
 
 InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
                                            TTI::TargetCostKind CostKind,
-                                           const Instruction *I) {
+                                           const Instruction *I) const {
   assert((I == nullptr || I->getOpcode() == Opcode) &&
          "Opcode should reflect passed instruction.");
   const bool SCost =
@@ -803,7 +802,7 @@ InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
 InstructionCost
 GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                                        std::optional<FastMathFlags> FMF,
-                                       TTI::TargetCostKind CostKind) {
+                                       TTI::TargetCostKind CostKind) const {
   if (TTI::requiresOrderedReduction(FMF))
     return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
 
@@ -821,7 +820,7 @@ GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
 InstructionCost
 GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                    FastMathFlags FMF,
-                                   TTI::TargetCostKind CostKind) {
+                                   TTI::TargetCostKind CostKind) const {
   EVT OrigTy = TLI->getValueType(DL, Ty);
 
   // Computes cost on targets that have packed math instructions(which support
@@ -836,7 +835,7 @@ GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
 InstructionCost GCNTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
                                                TTI::TargetCostKind CostKind,
                                                unsigned Index, Value *Op0,
-                                               Value *Op1) {
+                                               Value *Op1) const {
   switch (Opcode) {
   case Instruction::ExtractElement:
   case Instruction::InsertElement: {
@@ -1125,7 +1124,7 @@ InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
                                            TTI::TargetCostKind CostKind,
                                            int Index, VectorType *SubTp,
                                            ArrayRef<const Value *> Args,
-                                           const Instruction *CxtI) {
+                                           const Instruction *CxtI) const {
   if (!isa<FixedVectorType>(VT))
     return BaseT::getShuffleCost(Kind, VT, Mask, CostKind, Index, SubTp);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index f5062070ac6f4..165ed3d59c35d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -157,10 +157,11 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr);
+                                 const Instruction *I = nullptr) const;
 
   bool isInlineAsmSourceOfDivergence(const CallInst *CI,
                                      ArrayRef<unsigned> Indices = {}) const;
@@ -168,7 +169,8 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
 
   bool isReadRegisterSourceOfDivergence(const IntrinsicInst *ReadReg) const;
   bool isSourceOfDivergence(const Value *V) const;
@@ -238,14 +240,14 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
       std::function<void(Instruction *, unsigned, APInt, APInt &)>
           SimplifyAndSetOp) const;
 
-  InstructionCost getVectorSplitCost() { return 0; }
+  InstructionCost getVectorSplitCost() const { return 0; }
 
   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr);
+                                 const Instruction *CxtI = nullptr) const;
 
   bool isProfitableToSinkOperands(Instruction *I,
                                   SmallVectorImpl<Use *> &Ops) const;
@@ -260,15 +262,16 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
 
   int getInlinerVectorBonusPercent() const { return InlinerVectorBonusPercent; }
 
-  InstructionCost getArithmeticReductionCost(
-      unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF,
-      TTI::TargetCostKind CostKind);
+  InstructionCost
+  getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+                             std::optional<FastMathFlags> FMF,
+                             TTI::TargetCostKind CostKind) const;
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
-                                         TTI::TargetCostKind CostKind);
+                                         TTI::TargetCostKind CostKind) const;
 
   /// Data cache line size for LoopDataPrefetch pass. Has no use before GFX12.
   unsigned getCacheLineSize() const override { return 128; }
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
index ad4aaa8fdef84..3f7274bad374c 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
@@ -93,7 +93,7 @@ unsigned R600TTIImpl::getMaxInterleaveFactor(ElementCount VF) {
 
 InstructionCost R600TTIImpl::getCFInstrCost(unsigned Opcode,
                                             TTI::TargetCostKind CostKind,
-                                            const Instruction *I) {
+                                            const Instruction *I) const {
   if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
     return Opcode == Instruction::PHI ? 0 : 1;
 
@@ -110,7 +110,7 @@ InstructionCost R600TTIImpl::getCFInstrCost(unsigned Opcode,
 InstructionCost R600TTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
                                                 TTI::TargetCostKind CostKind,
                                                 unsigned Index, Value *Op0,
-                                                Value *Op1) {
+                                                Value *Op1) const {
   switch (Opcode) {
   case Instruction::ExtractElement:
   case Instruction::InsertElement: {
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
index 2934b0151f4df..4adece65a2863 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
@@ -59,11 +59,12 @@ class R600TTIImpl final : public BasicTTIImplBase<R600TTIImpl> {
                                     unsigned AddrSpace) const;
   unsigned getMaxInterleaveFactor(ElementCount VF);
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr);
+                                 const Instruction *I = nullptr) const;
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 2f9c262511ae4..9c12e4bf62b67 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -318,7 +318,7 @@ std::optional<Value *> ARMTTIImpl::simplifyDemandedVectorEltsIntrinsic(
 }
 
 InstructionCost ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
-                                          TTI::TargetCostKind CostKind) {
+                                          TTI::TargetCostKind CostKind) const {
   assert(Ty->isIntegerTy());
 
  unsigned Bits = Ty->getPrimitiveSizeInBits();
@@ -482,7 +482,7 @@ InstructionCost ARMTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
 
 InstructionCost ARMTTIImpl::getCFInstrCost(unsigned Opcode,
                                            TTI::TargetCostKind CostKind,
-                                           const Instruction *I) {
+                                           const Instruction *I) const {
   if (CostKind == TTI::TCK_RecipThroughput &&
       (ST->hasNEON() || ST->hasMVEIntegerOps())) {
     // FIXME: The vectorizer is highly sensistive to the cost of these
@@ -498,7 +498,7 @@ InstructionCost ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                              Type *Src,
                                              TTI::CastContextHint CCH,
                                              TTI::TargetCostKind CostKind,
-                                             const Instruction *I) {
+                                             const Instruction *I) const {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
@@ -901,7 +901,7 @@ InstructionCost ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
 InstructionCost ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
                                                TTI::TargetCostKind CostKind,
                                                unsigned Index, Value *Op0,
-                                               Value *Op1) {
+                                               Value *Op1) const {
   // Penalize inserting into an D-subregister. We end up with a three times
   // lower estimated throughput on swift.
   if (ST->hasSlowLoadDSubregister() && Opcode == Instruction::InsertElement &&
@@ -940,7 +940,7 @@ InstructionCost ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
 InstructionCost ARMTTIImpl::getCmpSelInstrCost(
     unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
-    TTI::OperandValueInfo Op2Info, const Instruction *I) {
+    TTI::OperandValueInfo Op2Info, const Instruction *I) const {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
 
   // Thumb scalar code size cost for select.
@@ -1123,7 +1123,7 @@ bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) {
 }
 
 bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
-                                   unsigned /*AddressSpace*/) {
+                                   unsigned /*AddressSpace*/) const {
   if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
     return false;
 
@@ -1221,7 +1221,7 @@ int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const {
   return -1;
 }
 
-InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) {
+InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const {
   int NumOps = getNumMemOps(cast<IntrinsicInst>(I));
 
   // To model the cost of a library call, we assume 1 for the call, and
@@ -1236,7 +1236,7 @@ InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
                                            TTI::TargetCostKind CostKind,
                                            int Index, VectorType *SubTp,
                                            ArrayRef<const Value *> Args,
-                                           const Instruction *CxtI) {
+                                           const Instruction *CxtI) const {
   Kind = improveShuffleKindFromMask(Kind, Mask, Tp, Index, SubTp);
   // Treat extractsubvector as single op permutation.
   bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector;
@@ -1349,8 +1349,7 @@ InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
 InstructionCost ARMTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
   int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
   if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(1)) {
     // Make operations on i1 relatively expensive as this often involves
@@ -1549,7 +1548,7 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                             unsigned AddressSpace,
                                             TTI::TargetCostKind CostKind,
                                             TTI::OperandValueInfo OpInfo,
-                                            const Instruction *I) {
+                                            const Instruction *I) const {
   // TODO: Handle other cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
     return 1;
@@ -1594,7 +1593,7 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
 InstructionCost
 ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                   unsigned AddressSpace,
-                                  TTI::TargetCostKind CostKind) {
+                                  TTI::TargetCostKind CostKind) const {
   if (ST->hasMVEIntegerOps()) {
     if (Opcode == Instruction::Load &&
         isLegalMaskedLoad(Src, Alignment, AddressSpace))
@@ -1654,7 +1653,7 @@ InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost(
 
 InstructionCost ARMTTIImpl::getGatherScatterOpCost(
     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
-    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
+    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
   using namespace PatternMatch;
   if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
@@ -1759,7 +1758,7 @@ InstructionCost ARMTTIImpl::getGatherScatterOpCost(
 InstructionCost
 ARMTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
                                        std::optional<FastMathFlags> FMF,
-                                       TTI::TargetCostKind CostKind) {
+                                       TTI::TargetCostKind CostKind) const {
 
   EVT ValVT = TLI->getValueType(DL, ValTy);
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -1906,7 +1905,7 @@ ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
 InstructionCost
 ARMTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                    FastMathFlags FMF,
-                                   TTI::TargetCostKind CostKind) {
+                                   TTI::TargetCostKind CostKind) const {
   EVT ValVT = TLI->getValueType(DL, Ty);
 
   // In general floating point reductions are a series of elementwise
@@ -1965,7 +1964,7 @@ ARMTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
 
 InstructionCost
 ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                  TTI::TargetCostKind CostKind) {
+                                  TTI::TargetCostKind CostKind) const {
   unsigned Opc = ICA.getID();
   switch (Opc) {
   case Intrinsic::get_active_lane_mask:
@@ -2109,7 +2108,7 @@ ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
   return BaseT::getIntrinsicInstrCost(ICA, CostKind);
 }
 
-bool ARMTTIImpl::isLoweredToCall(const Function *F) {
+bool ARMTTIImpl::isLoweredToCall(const Function *F) const {
   if (!F->isIntrinsic())
     return BaseT::isLoweredToCall(F);
 
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 2b144f1628038..307a0270fc0a9 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -135,7 +135,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
 
   using BaseT::getIntImmCost;
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                TTI::TargetCostKind CostKind);
+                                TTI::TargetCostKind CostKind) const;
 
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
@@ -184,10 +184,11 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
 
   bool isProfitableLSRChainElement(Instruction *I);
 
-  bool isLegalMaskedLoad(Type *DataTy, Align Alignment, unsigned AddressSpace);
+  bool isLegalMaskedLoad(Type *DataTy, Align Alignment,
+                         unsigned AddressSpace) const;
 
   bool isLegalMaskedStore(Type *DataTy, Align Alignment,
-                          unsigned AddressSpace) {
+                          unsigned AddressSpace) const {
     return isLegalMaskedLoad(DataTy, Alignment, AddressSpace);
   }
 
@@ -209,7 +210,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
     return isLegalMaskedGather(Ty, Alignment);
   }
 
-  InstructionCost getMemcpyCost(const Instruction *I);
+  InstructionCost getMemcpyCost(const Instruction *I) const;
 
   uint64_t getMaxMemIntrinsicInlineSizeThreshold() const {
     return ST->getMaxInlineSizeThreshold();
@@ -222,7 +223,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr);
+                                 const Instruction *CxtI = nullptr) const;
 
   bool preferInLoopReduction(RecurKind Kind, Type *Ty) const;
 
@@ -231,24 +232,25 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   bool shouldExpandReduction(const IntrinsicInst *II) const { return false; }
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr);
+                                 const Instruction *I = nullptr) const;
 
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr);
+                                   const Instruction *I = nullptr) const;
 
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr);
+      const Instruction *I = nullptr) const;
 
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
 
   InstructionCost getAddressComputationCost(Type *Val, ScalarEvolution *SE,
                                             const SCEV *Ptr);
@@ -257,17 +259,18 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
 
-  InstructionCost
-  getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
-                  unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
-                  const Instruction *I = nullptr);
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const;
 
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
 
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
@@ -278,11 +281,12 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
-                                         const Instruction *I = nullptr);
+                                         const Instruction *I = nullptr) const;
 
-  InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
-                                             std::optional<FastMathFlags> FMF,
-                                             TTI::TargetCostKind CostKind);
+  InstructionCost
+  getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
+                             std::optional<FastMathFlags> FMF,
+                             TTI::TargetCostKind CostKind) const;
   InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned,
                                            Type *ResTy, VectorType *ValTy,
                                            std::optional<FastMathFlags> FMF,
@@ -293,10 +297,10 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
 
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
-                                         TTI::TargetCostKind CostKind);
+                                         TTI::TargetCostKind CostKind) const;
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
 
   /// getScalingFactorCost - Return the cost of the scaling used in
   /// addressing mode represented by AM.
@@ -307,7 +311,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
                                        int64_t Scale, unsigned AddrSpace) const;
 
   bool maybeLoweredToCall(Instruction &I);
-  bool isLoweredToCall(const Function *F);
+  bool isLoweredToCall(const Function *F) const;
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                 AssumptionCache &AC,
                                 TargetLibraryInfo *LibInfo,
diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index bf0bef3a2b2f9..47d11bf6805c9 100644
--- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -49,7 +49,7 @@ class BPFTTIImpl : public BasicTTIImplBase<BPFTTIImpl> {
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const llvm::Instruction *I = nullptr) {
+      const llvm::Instruction *I = nullptr) const {
     if (Opcode == Instruction::Select)
       return SCEVCheapExpansionBudget.getValue();
 
@@ -61,7 +61,8 @@ class BPFTTIImpl : public BasicTTIImplBase<BPFTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr) {
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const {
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     if (ISD == ISD::ADD && CostKind == TTI::TCK_RecipThroughput)
       return SCEVCheapExpansionBudget.getValue() + 1;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index c3c77b514882b..0c2deec0a0dab 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -138,15 +138,15 @@ ElementCount HexagonTTIImpl::getMinimumVF(unsigned ElemWidth,
   return ElementCount::getFixed((8 * ST.getVectorLength()) / ElemWidth);
 }
 
-InstructionCost HexagonTTIImpl::getCallInstrCost(Function *F, Type *RetTy,
-                                                 ArrayRef<Type *> Tys,
-                                                 TTI::TargetCostKind CostKind) {
+InstructionCost
+HexagonTTIImpl::getCallInstrCost(Function *F, Type *RetTy, ArrayRef<Type *> Tys,
+                                 TTI::TargetCostKind CostKind) const {
   return BaseT::getCallInstrCost(F, RetTy, Tys, CostKind);
 }
 
 InstructionCost
 HexagonTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                      TTI::TargetCostKind CostKind) {
+                                      TTI::TargetCostKind CostKind) const {
   if (ICA.getID() == Intrinsic::bswap) {
     std::pair<InstructionCost, MVT> LT =
         getTypeLegalizationCost(ICA.getReturnType());
@@ -166,7 +166,7 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                                 unsigned AddressSpace,
                                                 TTI::TargetCostKind CostKind,
                                                 TTI::OperandValueInfo OpInfo,
-                                                const Instruction *I) {
+                                                const Instruction *I) const {
   assert(Opcode == Instruction::Load || Opcode == Instruction::Store);
   // TODO: Handle other cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
@@ -221,7 +221,7 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
 InstructionCost
 HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                       Align Alignment, unsigned AddressSpace,
-                                      TTI::TargetCostKind CostKind) {
+                                      TTI::TargetCostKind CostKind) const {
   return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                       CostKind);
 }
@@ -231,13 +231,13 @@ InstructionCost HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
                                                TTI::TargetCostKind CostKind,
                                                int Index, Type *SubTp,
                                                ArrayRef<const Value *> Args,
-                                               const Instruction *CxtI) {
+                                               const Instruction *CxtI) const {
   return 1;
 }
 
 InstructionCost HexagonTTIImpl::getGatherScatterOpCost(
     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
-    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
+    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
   return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                        Alignment, CostKind, I);
 }
@@ -258,7 +258,7 @@ InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost(
 InstructionCost HexagonTTIImpl::getCmpSelInstrCost(
     unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
-    TTI::OperandValueInfo Op2Info, const Instruction *I) {
+    TTI::OperandValueInfo Op2Info, const Instruction *I) const {
   if (ValTy->isVectorTy() && CostKind == TTI::TCK_RecipThroughput) {
     if (!isHVXVectorType(ValTy) && ValTy->isFPOrFPVectorTy())
       return InstructionCost::getMax();
@@ -273,8 +273,7 @@ InstructionCost HexagonTTIImpl::getCmpSelInstrCost(
 InstructionCost HexagonTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
@@ -295,7 +294,7 @@ InstructionCost HexagonTTIImpl::getCastInstrCost(unsigned Opcode, Type *DstTy,
                                                  Type *SrcTy,
                                                  TTI::CastContextHint CCH,
                                                  TTI::TargetCostKind CostKind,
-                                                 const Instruction *I) {
+                                                 const Instruction *I) const {
   auto isNonHVXFP = [this] (Type *Ty) {
     return Ty->isVectorTy() && !isHVXVectorType(Ty) && Ty->isFPOrFPVectorTy();
   };
@@ -321,7 +320,7 @@ InstructionCost HexagonTTIImpl::getCastInstrCost(unsigned Opcode, Type *DstTy,
 InstructionCost HexagonTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                                    TTI::TargetCostKind CostKind,
                                                    unsigned Index, Value *Op0,
-                                                   Value *Op1) {
+                                                   Value *Op1) const {
   Type *ElemTy = Val->isVectorTy() ? cast<VectorType>(Val)->getElementType()
                                    : Val;
   if (Opcode == Instruction::InsertElement) {
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index b23369ac054b9..a3820b71b1e4c 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -105,29 +105,29 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
 
   InstructionCost getCallInstrCost(Function *F, Type *RetTy,
                                    ArrayRef<Type *> Tys,
-                                   TTI::TargetCostKind CostKind);
+                                   TTI::TargetCostKind CostKind) const;
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
   InstructionCost getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
                                             const SCEV *S);
-  InstructionCost
-  getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
-                  unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
-                  const Instruction *I = nullptr);
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const;
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
                                  Type *SubTp, ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr);
+                                 const Instruction *CxtI = nullptr) const;
   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
-                                         const Instruction *I);
+                                         const Instruction *I) const;
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -137,23 +137,25 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr);
+      const Instruction *I = nullptr) const;
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr);
+                                   const Instruction *I = nullptr) const;
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr) {
+                                 const Instruction *I = nullptr) const {
     return 1;
   }
 
diff --git a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
index 5fe63e4a2e031..14759e7ca8f4a 100644
--- a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
@@ -94,7 +94,8 @@ class LanaiTTIImpl : public BasicTTIImplBase<LanaiTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr) {
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const {
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
 
     switch (ISD) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 72245fe83491d..b5f7d90cd29d9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -512,8 +512,7 @@ NVPTXTTIImpl::getInstructionCost(const User *U,
 InstructionCost NVPTXTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index 9e77f628da7a7..9fd5c17f58959 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -103,13 +103,14 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
 
   InstructionCost getScalarizationOverhead(VectorType *InTy,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
                                            TTI::TargetCostKind CostKind,
-                                           ArrayRef<Value *> VL = {}) {
+                                           ArrayRef<Value *> VL = {}) const {
     if (!InTy->getElementCount().isFixed())
       return InstructionCost::getInvalid();
 
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 3a4c2fcad8c83..77149045d6b72 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -161,7 +161,7 @@ PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 }
 
 InstructionCost PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
-                                          TTI::TargetCostKind CostKind) {
+                                          TTI::TargetCostKind CostKind) const {
   if (DisablePPCConstHoist)
     return BaseT::getIntImmCost(Imm, Ty, CostKind);
 
@@ -548,7 +548,8 @@ unsigned PPCTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
 // thereby reducing the overall throughput of vector code wrt. scalar code.
 // An invalid instruction cost is returned if the type is an MMA vector type.
 InstructionCost PPCTTIImpl::vectorCostAdjustmentFactor(unsigned Opcode,
-                                                       Type *Ty1, Type *Ty2) {
+                                                       Type *Ty1,
+                                                       Type *Ty2) const {
   // If the vector type is of an MMA type (v256i1, v512i1), an invalid
   // instruction cost is returned. This is to signify to other cost computing
   // functions to return the maximum instruction cost in order to prevent any
@@ -581,8 +582,7 @@ InstructionCost PPCTTIImpl::vectorCostAdjustmentFactor(unsigned Opcode,
 InstructionCost PPCTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
 
   InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty, nullptr);
@@ -605,7 +605,7 @@ InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
                                            TTI::TargetCostKind CostKind,
                                            int Index, Type *SubTp,
                                            ArrayRef<const Value *> Args,
-                                           const Instruction *CxtI) {
+                                           const Instruction *CxtI) const {
 
   InstructionCost CostFactor =
       vectorCostAdjustmentFactor(Instruction::ShuffleVector, Tp, nullptr);
@@ -625,7 +625,7 @@ InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
 
 InstructionCost PPCTTIImpl::getCFInstrCost(unsigned Opcode,
                                            TTI::TargetCostKind CostKind,
-                                           const Instruction *I) {
+                                           const Instruction *I) const {
   if (CostKind != TTI::TCK_RecipThroughput)
     return Opcode == Instruction::PHI ? 0 : 1;
   // Branches are assumed to be predicted.
@@ -636,7 +636,7 @@ InstructionCost PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                              Type *Src,
                                              TTI::CastContextHint CCH,
                                              TTI::TargetCostKind CostKind,
-                                             const Instruction *I) {
+                                             const Instruction *I) const {
   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
 
   InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Dst, Src);
@@ -655,7 +655,7 @@ InstructionCost PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
 InstructionCost PPCTTIImpl::getCmpSelInstrCost(
     unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
-    TTI::OperandValueInfo Op2Info, const Instruction *I) {
+    TTI::OperandValueInfo Op2Info, const Instruction *I) const {
   InstructionCost CostFactor =
       vectorCostAdjustmentFactor(Opcode, ValTy, nullptr);
   if (!CostFactor.isValid())
@@ -672,7 +672,7 @@ InstructionCost PPCTTIImpl::getCmpSelInstrCost(
 InstructionCost PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                                TTI::TargetCostKind CostKind,
                                                unsigned Index, Value *Op0,
-                                               Value *Op1) {
+                                               Value *Op1) const {
   assert(Val->isVectorTy() && "This must be a vector type");
 
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -764,7 +764,7 @@ InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                             unsigned AddressSpace,
                                             TTI::TargetCostKind CostKind,
                                             TTI::OperandValueInfo OpInfo,
-                                            const Instruction *I) {
+                                            const Instruction *I) const {
 
   InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Src, nullptr);
   if (!CostFactor.isValid())
@@ -890,7 +890,7 @@ InstructionCost PPCTTIImpl::getInterleavedMemoryOpCost(
 
 InstructionCost
 PPCTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                  TTI::TargetCostKind CostKind) {
+                                  TTI::TargetCostKind CostKind) const {
   return BaseT::getIntrinsicInstrCost(ICA, CostKind);
 }
 
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index bf3ddad134e14..f81082d2d7029 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -48,7 +48,7 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
 
   using BaseT::getIntImmCost;
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                TTI::TargetCostKind CostKind);
+                                TTI::TargetCostKind CostKind) const;
 
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
@@ -101,44 +101,46 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
   unsigned getPrefetchDistance() const override;
   unsigned getMaxInterleaveFactor(ElementCount VF);
   InstructionCost vectorCostAdjustmentFactor(unsigned Opcode, Type *Ty1,
-                                             Type *Ty2);
+                                             Type *Ty2) const;
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
                                  Type *SubTp, ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr);
+                                 const Instruction *CxtI = nullptr) const;
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr);
+                                   const Instruction *I = nullptr) const;
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr);
+                                 const Instruction *I = nullptr) const;
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr);
+      const Instruction *I = nullptr) const;
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
-  InstructionCost
-  getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
-                  unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
-                  const Instruction *I = nullptr);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const;
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
       bool UseMaskForCond = false, bool UseMaskForGaps = false);
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
   bool areTypesABICompatible(const Function *Caller, const Function *Callee,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 6bd60c20c8626..6bedc43c92c0f 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -45,7 +45,7 @@ static cl::opt<unsigned>
 
 InstructionCost
 RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
-                                      TTI::TargetCostKind CostKind) {
+                                      TTI::TargetCostKind CostKind) const {
   // Check if the type is valid for all CostKind
   if (!VT.isVector())
     return InstructionCost::getInvalid();
@@ -343,7 +343,8 @@ RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
 }
 
 InstructionCost
-RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,  TTI::TargetCostKind CostKind) {
+RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,
+                                      TTI::TargetCostKind CostKind) const {
   // Add a cost of address generation + the cost of the load. The address
   // is expected to be a PC relative offset to a constant pool entry
   // using auipc/addi.
@@ -397,7 +398,7 @@ static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,
 /// TTI::TCC_Basic). If the source register is just reused, the cost for
 /// this operation is 0.
 static InstructionCost
-costShuffleViaVRegSplitting(RISCVTTIImpl &TTI, MVT LegalVT,
+costShuffleViaVRegSplitting(const RISCVTTIImpl &TTI, MVT LegalVT,
                             std::optional<unsigned> VLen, VectorType *Tp,
                             ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {
   assert(LegalVT.isFixedLengthVector());
@@ -466,7 +467,7 @@ costShuffleViaVRegSplitting(RISCVTTIImpl &TTI, MVT LegalVT,
 
 InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
                                            ArrayRef<int> Mask,
-                                           TTI::TargetCostKind CostKind) {
+                                           TTI::TargetCostKind CostKind) const {
   // Avoid missing masks and length changing shuffles
   if (Mask.size() <= 2 || Mask.size() != Tp->getNumElements())
     return InstructionCost::getInvalid();
@@ -527,7 +528,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
                                              TTI::TargetCostKind CostKind,
                                              int Index, VectorType *SubTp,
                                              ArrayRef<const Value *> Args,
-                                             const Instruction *CxtI) {
+                                             const Instruction *CxtI) const {
   Kind = improveShuffleKindFromMask(Kind, Mask, Tp, Index, SubTp);
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
 
@@ -837,7 +838,7 @@ static unsigned isM1OrSmaller(MVT VT) {
 
 InstructionCost RISCVTTIImpl::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
-    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) const {
   if (isa<ScalableVectorType>(Ty))
     return InstructionCost::getInvalid();
 
@@ -875,7 +876,7 @@ InstructionCost RISCVTTIImpl::getScalarizationOverhead(
 InstructionCost
 RISCVTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                     unsigned AddressSpace,
-                                    TTI::TargetCostKind CostKind) {
+                                    TTI::TargetCostKind CostKind) const {
   if (!isLegalMaskedLoadStore(Src, Alignment) ||
       CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
@@ -983,7 +984,7 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
 
 InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
-    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
+    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                          Alignment, CostKind, I);
@@ -1008,7 +1009,7 @@ InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
 
 InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
     unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
-    TTI::TargetCostKind CostKind, const Instruction *I) {
+    TTI::TargetCostKind CostKind, const Instruction *I) const {
   bool IsLegal = (Opcode == Instruction::Store &&
                   isLegalMaskedCompressStore(DataTy, Alignment)) ||
                  (Opcode == Instruction::Load &&
@@ -1046,7 +1047,7 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
 
 InstructionCost RISCVTTIImpl::getStridedMemoryOpCost(
     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
-    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
+    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
   if (((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
        !isLegalStridedLoadStore(DataTy, Alignment)) ||
       (Opcode != Instruction::Load && Opcode != Instruction::Store))
@@ -1167,7 +1168,7 @@ static unsigned getISDForVPIntrinsicID(Intrinsic::ID ID) {
 
 InstructionCost
 RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                    TTI::TargetCostKind CostKind) {
+                                    TTI::TargetCostKind CostKind) const {
   auto *RetTy = ICA.getReturnType();
   switch (ICA.getID()) {
   case Intrinsic::lrint:
@@ -1470,7 +1471,7 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                                Type *Src,
                                                TTI::CastContextHint CCH,
                                                TTI::TargetCostKind CostKind,
-                                               const Instruction *I) {
+                                               const Instruction *I) const {
   bool IsVectorType = isa<VectorType>(Dst) && isa<VectorType>(Src);
   if (!IsVectorType)
     return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
@@ -1661,7 +1662,7 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 }
 
-unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) {
+unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) const {
   if (isa<ScalableVectorType>(Ty)) {
     const unsigned EltSize = DL.getTypeSizeInBits(Ty->getElementType());
     const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue();
@@ -1674,7 +1675,7 @@ unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) {
 InstructionCost
 RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                      FastMathFlags FMF,
-                                     TTI::TargetCostKind CostKind) {
+                                     TTI::TargetCostKind CostKind) const {
   if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())
     return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
 
@@ -1780,7 +1781,7 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
 InstructionCost
 RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                                          std::optional<FastMathFlags> FMF,
-                                         TTI::TargetCostKind CostKind) {
+                                         TTI::TargetCostKind CostKind) const {
   if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())
     return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
 
@@ -1930,9 +1931,9 @@ InstructionCost RISCVTTIImpl::getExtendedReductionCost(
          getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);
 }
 
-InstructionCost RISCVTTIImpl::getStoreImmCost(Type *Ty,
-                                              TTI::OperandValueInfo OpInfo,
-                                              TTI::TargetCostKind CostKind) {
+InstructionCost
+RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,
+                              TTI::TargetCostKind CostKind) const {
   assert(OpInfo.isConstant() && "non constant operand?");
   if (!isa<VectorType>(Ty))
     // FIXME: We need to account for immediate materialization here, but doing
@@ -1949,13 +1950,12 @@ InstructionCost RISCVTTIImpl::getStoreImmCost(Type *Ty,
   return getConstantPoolLoadCost(Ty, CostKind);
 }
 
-
 InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                               MaybeAlign Alignment,
                                               unsigned AddressSpace,
                                               TTI::TargetCostKind CostKind,
                                               TTI::OperandValueInfo OpInfo,
-                                              const Instruction *I) {
+                                              const Instruction *I) const {
   EVT VT = TLI->getValueType(DL, Src, true);
   // Type legalization can't handle structs
   if (VT == MVT::Other)
@@ -1993,13 +1993,12 @@ InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   if (LT.second.isVector() && CostKind != TTI::TCK_CodeSize)
     BaseCost *= TLI->getLMULCost(LT.second);
   return Cost + BaseCost;
-
 }
 
 InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
     unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
-    TTI::OperandValueInfo Op2Info, const Instruction *I) {
+    TTI::OperandValueInfo Op2Info, const Instruction *I) const {
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
                                      Op1Info, Op2Info, I);
@@ -2158,7 +2157,7 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
 
 InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
                                              TTI::TargetCostKind CostKind,
-                                             const Instruction *I) {
+                                             const Instruction *I) const {
   if (CostKind != TTI::TCK_RecipThroughput)
     return Opcode == Instruction::PHI ? 0 : 1;
   // Branches are assumed to be predicted.
@@ -2168,7 +2167,7 @@ InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
 InstructionCost RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                                  TTI::TargetCostKind CostKind,
                                                  unsigned Index, Value *Op0,
-                                                 Value *Op1) {
+                                                 Value *Op1) const {
   assert(Val->isVectorTy() && "This must be a vector type");
 
   if (Opcode != Instruction::ExtractElement &&
@@ -2315,7 +2314,7 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
 
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
@@ -2635,7 +2634,8 @@ bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
                   C2.ScaleCost, C2.ImmCost, C2.SetupCost);
 }
 
-bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) {
+bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
+                                           Align Alignment) const {
   auto *VTy = dyn_cast<VectorType>(DataTy);
   if (!VTy || VTy->isScalableTy())
     return false;
@@ -2652,7 +2652,8 @@ bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) {
   return true;
 }
 
-bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy, Align Alignment) {
+bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,
+                                              Align Alignment) const {
   auto *VTy = dyn_cast<VectorType>(DataTy);
   if (!VTy || VTy->isScalableTy())
     return false;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index c61dd1507f168..b72ffc2448313 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -45,7 +45,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   /// This does unfortunately mean that we can both undershoot and overshot
   /// the true cost significantly if getVScaleForTuning is wildly off for the
   /// actual target hardware.
-  unsigned getEstimatedVLFor(VectorType *Ty);
+  unsigned getEstimatedVLFor(VectorType *Ty) const;
 
   /// This function calculates the costs for one or more RVV opcodes based
   /// on the vtype and the cost kind.
@@ -56,17 +56,17 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   /// refers to the result or source type.
   /// \param CostKind The type of cost to compute.
   InstructionCost getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
-                                          TTI::TargetCostKind CostKind);
+                                          TTI::TargetCostKind CostKind) const;
 
   /// Return the cost of accessing a constant pool entry of the specified
   /// type.
   InstructionCost getConstantPoolLoadCost(Type *Ty,
-                                          TTI::TargetCostKind CostKind);
+                                          TTI::TargetCostKind CostKind) const;
 
   /// If this shuffle can be lowered as a masked slide pair (at worst),
   /// return a cost for it.
   InstructionCost getSlideCost(FixedVectorType *Tp, ArrayRef<int> Mask,
-                               TTI::TargetCostKind CostKind);
+                               TTI::TargetCostKind CostKind) const;
 
 public:
   explicit RISCVTTIImpl(const RISCVTargetMachine *TM, const Function &F)
@@ -76,7 +76,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   /// Return the cost of materializing an immediate for a value operand of
   /// a store instruction.
   InstructionCost getStoreImmCost(Type *VecTy, TTI::OperandValueInfo OpInfo,
-                                  TTI::TargetCostKind CostKind);
+                                  TTI::TargetCostKind CostKind) const;
 
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
                                 TTI::TargetCostKind CostKind);
@@ -135,7 +135,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
 
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
 
   InstructionCost getPointersChainCost(ArrayRef<const Value *> Ptrs,
                                        const Value *Base,
@@ -159,16 +159,16 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr);
+                                 const Instruction *CxtI = nullptr) const;
 
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
                                            TTI::TargetCostKind CostKind,
-                                           ArrayRef<Value *> VL = {});
+                                           ArrayRef<Value *> VL = {}) const;
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
 
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
@@ -179,72 +179,74 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
-                                         const Instruction *I);
+                                         const Instruction *I) const;
 
-  InstructionCost getExpandCompressMemoryOpCost(unsigned Opcode, Type *Src,
-                                                bool VariableMask,
-                                                Align Alignment,
-                                                TTI::TargetCostKind CostKind,
-                                                const Instruction *I = nullptr);
+  InstructionCost
+  getExpandCompressMemoryOpCost(unsigned Opcode, Type *Src, bool VariableMask,
+                                Align Alignment, TTI::TargetCostKind CostKind,
+                                const Instruction *I = nullptr) const;
 
   InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
-                                         const Instruction *I);
+                                         const Instruction *I) const;
 
   InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys);
 
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr);
+                                   const Instruction *I = nullptr) const;
 
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
-                                         TTI::TargetCostKind CostKind);
+                                         TTI::TargetCostKind CostKind) const;
 
-  InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
-                                             std::optional<FastMathFlags> FMF,
-                                             TTI::TargetCostKind CostKind);
+  InstructionCost
+  getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+                             std::optional<FastMathFlags> FMF,
+                             TTI::TargetCostKind CostKind) const;
 
   InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned,
                                            Type *ResTy, VectorType *ValTy,
                                            std::optional<FastMathFlags> FMF,
                                            TTI::TargetCostKind CostKind);
 
-  InstructionCost
-  getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
-                  unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  TTI::OperandValueInfo OpdInfo = {TTI::OK_AnyValue, TTI::OP_None},
-                  const Instruction *I = nullptr);
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpdInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const;
 
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr);
+      const Instruction *I = nullptr) const;
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr);
+                                 const Instruction *I = nullptr) const;
 
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
 
   bool isElementTypeLegalForScalableVector(Type *Ty) const {
     return TLI->isLegalElementTypeForRVV(TLI->getValueType(DL, Ty));
   }
 
-  bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) {
+  bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) const {
     if (!ST->hasVInstructions())
       return false;
 
@@ -270,7 +272,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
     return isLegalMaskedLoadStore(DataType, Alignment);
   }
 
-  bool isLegalMaskedGatherScatter(Type *DataType, Align Alignment) {
+  bool isLegalMaskedGatherScatter(Type *DataType, Align Alignment) const {
     if (!ST->hasVInstructions())
       return false;
 
@@ -293,10 +295,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
     return TLI->isLegalElementTypeForRVV(ElemType);
   }
 
-  bool isLegalMaskedGather(Type *DataType, Align Alignment) {
+  bool isLegalMaskedGather(Type *DataType, Align Alignment) const {
     return isLegalMaskedGatherScatter(DataType, Alignment);
   }
-  bool isLegalMaskedScatter(Type *DataType, Align Alignment) {
+  bool isLegalMaskedScatter(Type *DataType, Align Alignment) const {
     return isLegalMaskedGatherScatter(DataType, Alignment);
   }
 
@@ -310,7 +312,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
     return ST->is64Bit() && !ST->hasVInstructionsI64();
   }
 
-  bool isLegalStridedLoadStore(Type *DataType, Align Alignment) {
+  bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const {
     EVT DataTypeVT = TLI->getValueType(DL, DataType);
     return TLI->isLegalStridedLoadStore(DataTypeVT, Alignment);
   }
@@ -321,9 +323,9 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
                                              DL);
   }
 
-  bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment);
+  bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) const;
 
-  bool isLegalMaskedCompressStore(Type *DataTy, Align Alignment);
+  bool isLegalMaskedCompressStore(Type *DataTy, Align Alignment) const;
 
   bool isVScaleKnownToBeAPowerOfTwo() const {
     return TLI->isVScaleKnownToBeAPowerOfTwo();
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index e0b0099466c52..3811657aed83e 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -493,7 +493,7 @@ static bool isFreeEltLoad(Value *Op) {
 
 InstructionCost SystemZTTIImpl::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
-    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) const {
   unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
   InstructionCost Cost = 0;
 
@@ -541,8 +541,7 @@ static unsigned getNumVectorRegs(Type *Ty) {
 InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
 
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
@@ -727,7 +726,7 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
 InstructionCost SystemZTTIImpl::getShuffleCost(
     TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
     TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
-    ArrayRef<const Value *> Args, const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
   Kind = improveShuffleKindFromMask(Kind, Mask, Tp, Index, SubTp);
   if (ST->hasVector()) {
     unsigned NumVectors = getNumVectorRegs(Tp);
@@ -776,8 +775,7 @@ static unsigned getElSizeLog2Diff(Type *Ty0, Type *Ty1) {
 }
 
 // Return the number of instructions needed to truncate SrcTy to DstTy.
-unsigned SystemZTTIImpl::
-getVectorTruncCost(Type *SrcTy, Type *DstTy) {
+unsigned SystemZTTIImpl::getVectorTruncCost(Type *SrcTy, Type *DstTy) const {
   assert (SrcTy->isVectorTy() && DstTy->isVectorTy());
   assert(SrcTy->getPrimitiveSizeInBits().getFixedValue() >
              DstTy->getPrimitiveSizeInBits().getFixedValue() &&
@@ -818,8 +816,8 @@ getVectorTruncCost(Type *SrcTy, Type *DstTy) {
 
 // Return the cost of converting a vector bitmask produced by a compare
 // (SrcTy), to the type of the select or extend instruction (DstTy).
-unsigned SystemZTTIImpl::
-getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) {
+unsigned SystemZTTIImpl::getVectorBitmaskConversionCost(Type *SrcTy,
+                                                        Type *DstTy) const {
   assert (SrcTy->isVectorTy() && DstTy->isVectorTy() &&
           "Should only be called with vector types.");
 
@@ -869,9 +867,9 @@ static Type *getCmpOpsType(const Instruction *I, unsigned VF = 1) {
 
 // Get the cost of converting a boolean vector to a vector with same width
 // and element size as Dst, plus the cost of zero extending if needed.
-unsigned SystemZTTIImpl::
-getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
-                              const Instruction *I) {
+unsigned
+SystemZTTIImpl::getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
+                                              const Instruction *I) const {
   auto *DstVTy = cast<FixedVectorType>(Dst);
   unsigned VF = DstVTy->getNumElements();
   unsigned Cost = 0;
@@ -890,7 +888,7 @@ InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                                  Type *Src,
                                                  TTI::CastContextHint CCH,
                                                  TTI::TargetCostKind CostKind,
-                                                 const Instruction *I) {
+                                                 const Instruction *I) const {
   // FIXME: Can the logic below also be used for these cost kinds?
   if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency) {
     auto BaseCost = BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
@@ -1087,7 +1085,7 @@ static unsigned getOperandsExtensionCost(const Instruction *I) {
 InstructionCost SystemZTTIImpl::getCmpSelInstrCost(
     unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
-    TTI::OperandValueInfo Op2Info, const Instruction *I) {
+    TTI::OperandValueInfo Op2Info, const Instruction *I) const {
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
                                      Op1Info, Op2Info);
@@ -1182,7 +1180,7 @@ InstructionCost SystemZTTIImpl::getCmpSelInstrCost(
 InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                                    TTI::TargetCostKind CostKind,
                                                    unsigned Index, Value *Op0,
-                                                   Value *Op1) {
+                                                   Value *Op1) const {
   if (Opcode == Instruction::InsertElement) {
     // Vector Element Load.
     if (Op1 != nullptr && isFreeEltLoad(Op1))
@@ -1209,8 +1207,8 @@ InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 }
 
 // Check if a load may be folded as a memory operand in its user.
-bool SystemZTTIImpl::
-isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue) {
+bool SystemZTTIImpl::isFoldableLoad(const LoadInst *Ld,
+                                    const Instruction *&FoldedValue) const {
   if (!Ld->hasOneUse())
     return false;
   FoldedValue = Ld;
@@ -1302,7 +1300,7 @@ InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                                 unsigned AddressSpace,
                                                 TTI::TargetCostKind CostKind,
                                                 TTI::OperandValueInfo OpInfo,
-                                                const Instruction *I) {
+                                                const Instruction *I) const {
   assert(!Src->isVoidTy() && "Invalid type");
 
   // TODO: Handle other cost kinds.
@@ -1457,7 +1455,7 @@ inline bool customCostReductions(unsigned Opcode) {
 InstructionCost
 SystemZTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                                            std::optional<FastMathFlags> FMF,
-                                           TTI::TargetCostKind CostKind) {
+                                           TTI::TargetCostKind CostKind) const {
   unsigned ScalarBits = Ty->getScalarSizeInBits();
   // The following is only for subtargets with vector math, non-ordered
   // reductions, and reasonable scalar sizes for int and fp add/mul.
@@ -1484,7 +1482,7 @@ SystemZTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
 InstructionCost
 SystemZTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                        FastMathFlags FMF,
-                                       TTI::TargetCostKind CostKind) {
+                                       TTI::TargetCostKind CostKind) const {
   // Return custom costs only on subtargets with vector enhancements.
   if (ST->hasVectorEnhancements1()) {
     unsigned NumVectors = getNumVectorRegs(Ty);
@@ -1513,7 +1511,7 @@ getVectorIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
 
 InstructionCost
 SystemZTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                      TTI::TargetCostKind CostKind) {
+                                      TTI::TargetCostKind CostKind) const {
   InstructionCost Cost = getVectorIntrinsicInstrCost(
       ICA.getID(), ICA.getReturnType(), ICA.getArgTypes());
   if (Cost != -1)
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index e64b1f1ccbd93..16a750d561f56 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -28,7 +28,9 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
 
   unsigned const LIBCALL_COST = 30;
 
-  bool isInt128InVR(Type *Ty) { return Ty->isIntegerTy(128) && ST->hasVector(); }
+  bool isInt128InVR(Type *Ty) const {
+    return Ty->isIntegerTy(128) && ST->hasVector();
+  }
 
 public:
   explicit SystemZTTIImpl(const SystemZTargetMachine *TM, const Function &F)
@@ -89,7 +91,7 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
                                            TTI::TargetCostKind CostKind,
-                                           ArrayRef<Value *> VL = {});
+                                           ArrayRef<Value *> VL = {}) const;
   bool supportsEfficientVectorElementLoadStore() { return true; }
   bool enableInterleavedAccessVectorization() { return true; }
 
@@ -97,52 +99,56 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr);
-  unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy);
-  unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy);
+                                 const Instruction *CxtI = nullptr) const;
+  unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy) const;
+  unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) const;
   unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
-                                         const Instruction *I);
+                                         const Instruction *I) const;
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr);
+                                   const Instruction *I = nullptr) const;
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr);
+      const Instruction *I = nullptr) const;
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
-  bool isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue);
-  InstructionCost
-  getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
-                  unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
-                  const Instruction *I = nullptr);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
+  bool isFoldableLoad(const LoadInst *Ld,
+                      const Instruction *&FoldedValue) const;
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const;
 
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
       bool UseMaskForCond = false, bool UseMaskForGaps = false);
 
-  InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
-                                             std::optional<FastMathFlags> FMF,
-                                             TTI::TargetCostKind CostKind);
+  InstructionCost
+  getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+                             std::optional<FastMathFlags> FMF,
+                             TTI::TargetCostKind CostKind) const;
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
-                                         TTI::TargetCostKind CostKind);
+                                         TTI::TargetCostKind CostKind) const;
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
 
   bool shouldExpandReduction(const IntrinsicInst *II) const;
   /// @}
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index e4dc38686a445..406daa8a6f9b8 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -53,7 +53,7 @@ TypeSize WebAssemblyTTIImpl::getRegisterBitWidth(
 InstructionCost WebAssemblyTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
 
   InstructionCost Cost =
       BasicTTIImplBase<WebAssemblyTTIImpl>::getArithmeticInstrCost(
@@ -81,7 +81,7 @@ InstructionCost WebAssemblyTTIImpl::getArithmeticInstrCost(
 
 InstructionCost WebAssemblyTTIImpl::getCastInstrCost(
     unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH,
-    TTI::TargetCostKind CostKind, const Instruction *I) {
+    TTI::TargetCostKind CostKind, const Instruction *I) const {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   auto SrcTy = TLI->getValueType(DL, Src);
   auto DstTy = TLI->getValueType(DL, Dst);
@@ -144,7 +144,7 @@ InstructionCost WebAssemblyTTIImpl::getCastInstrCost(
 InstructionCost WebAssemblyTTIImpl::getMemoryOpCost(
     unsigned Opcode, Type *Ty, MaybeAlign Alignment, unsigned AddressSpace,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,
-    const Instruction *I) {
+    const Instruction *I) const {
   if (!ST->hasSIMD128() || !isa<FixedVectorType>(Ty)) {
     return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
                                   CostKind);
@@ -182,10 +182,9 @@ InstructionCost WebAssemblyTTIImpl::getMemoryOpCost(
   return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace, CostKind);
 }
 
-InstructionCost
-WebAssemblyTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
-                                       TTI::TargetCostKind CostKind,
-                                       unsigned Index, Value *Op0, Value *Op1) {
+InstructionCost WebAssemblyTTIImpl::getVectorInstrCost(
+    unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+    Value *Op0, Value *Op1) const {
   InstructionCost Cost = BasicTTIImplBase::getVectorInstrCost(
       Opcode, Val, CostKind, Index, Op0, Op1);
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index ba66306374c6c..dd0e672a2a21c 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -65,21 +65,23 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
 
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr);
+                                   const Instruction *I = nullptr) const;
   InstructionCost getMemoryOpCost(
       unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr);
+      const Instruction *I = nullptr) const;
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
   InstructionCost
   getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB,
                           Type *AccumType, ElementCount VF,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 41e437a47ba29..59bcd3b921799 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -246,8 +246,7 @@ unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF) {
 InstructionCost X86TTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args,
-    const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
 
   // vXi8 multiplications are always promoted to vXi16.
   // Sub-128-bit types can be extended/packed more efficiently.
@@ -1526,7 +1525,7 @@ X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
 InstructionCost X86TTIImpl::getShuffleCost(
     TTI::ShuffleKind Kind, VectorType *BaseTp, ArrayRef<int> Mask,
     TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
-    ArrayRef<const Value *> Args, const Instruction *CxtI) {
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
   // 64-bit packed float vectors (v2f32) are widened to type v4f32.
   // 64-bit packed integer vectors (v2i32) are widened to type v4i32.
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(BaseTp);
@@ -2271,7 +2270,7 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                              Type *Src,
                                              TTI::CastContextHint CCH,
                                              TTI::TargetCostKind CostKind,
-                                             const Instruction *I) {
+                                             const Instruction *I) const {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
@@ -3296,7 +3295,7 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
 InstructionCost X86TTIImpl::getCmpSelInstrCost(
     unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
-    TTI::OperandValueInfo Op2Info, const Instruction *I) {
+    TTI::OperandValueInfo Op2Info, const Instruction *I) const {
   // Early out if this type isn't scalar/vector integer/float.
   if (!(ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy()))
     return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
@@ -3596,7 +3595,7 @@ unsigned X86TTIImpl::getAtomicMemIntrinsicMaxElementSize() const { return 16; }
 
 InstructionCost
 X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                  TTI::TargetCostKind CostKind) {
+                                  TTI::TargetCostKind CostKind) const {
   // Costs should match the codegen from:
   // BITREVERSE: llvm\test\CodeGen\X86\vector-bitreverse.ll
   // BSWAP: llvm\test\CodeGen\X86\bswap-vector.ll
@@ -4769,7 +4768,7 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
 InstructionCost X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                                TTI::TargetCostKind CostKind,
                                                unsigned Index, Value *Op0,
-                                               Value *Op1) {
+                                               Value *Op1) const {
   static const CostTblEntry SLMCostTbl[] = {
      { ISD::EXTRACT_VECTOR_ELT,       MVT::i8,      4 },
      { ISD::EXTRACT_VECTOR_ELT,       MVT::i16,     4 },
@@ -4917,7 +4916,7 @@ InstructionCost X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 
 InstructionCost X86TTIImpl::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
-    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) const {
   assert(DemandedElts.getBitWidth() ==
              cast<FixedVectorType>(Ty)->getNumElements() &&
          "Vector size mismatch");
@@ -5081,7 +5080,7 @@ InstructionCost X86TTIImpl::getScalarizationOverhead(
 InstructionCost
 X86TTIImpl::getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
                                       int VF, const APInt &DemandedDstElts,
-                                      TTI::TargetCostKind CostKind) {
+                                      TTI::TargetCostKind CostKind) const {
   const unsigned EltTyBits = DL.getTypeSizeInBits(EltTy);
   // We don't differentiate element types here, only element bit width.
   EltTy = IntegerType::getIntNTy(EltTy->getContext(), EltTyBits);
@@ -5189,7 +5188,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                             unsigned AddressSpace,
                                             TTI::TargetCostKind CostKind,
                                             TTI::OperandValueInfo OpInfo,
-                                            const Instruction *I) {
+                                            const Instruction *I) const {
   // TODO: Handle other cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput) {
     if (auto *SI = dyn_cast_or_null<StoreInst>(I)) {
@@ -5357,7 +5356,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
 InstructionCost
 X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy, Align Alignment,
                                   unsigned AddressSpace,
-                                  TTI::TargetCostKind CostKind) {
+                                  TTI::TargetCostKind CostKind) const {
   bool IsLoad = (Instruction::Load == Opcode);
   bool IsStore = (Instruction::Store == Opcode);
 
@@ -5472,7 +5471,7 @@ InstructionCost X86TTIImpl::getAddressComputationCost(Type *Ty,
 InstructionCost
 X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
                                        std::optional<FastMathFlags> FMF,
-                                       TTI::TargetCostKind CostKind) {
+                                       TTI::TargetCostKind CostKind) const {
   if (TTI::requiresOrderedReduction(FMF))
     return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);
 
@@ -5721,7 +5720,7 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
 
 InstructionCost X86TTIImpl::getMinMaxCost(Intrinsic::ID IID, Type *Ty,
                                           TTI::TargetCostKind CostKind,
-                                          FastMathFlags FMF) {
+                                          FastMathFlags FMF) const {
   IntrinsicCostAttributes ICA(IID, Ty, {Ty, Ty}, FMF);
   return getIntrinsicInstrCost(ICA, CostKind);
 }
@@ -5729,7 +5728,7 @@ InstructionCost X86TTIImpl::getMinMaxCost(Intrinsic::ID IID, Type *Ty,
 InstructionCost
 X86TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *ValTy,
                                    FastMathFlags FMF,
-                                   TTI::TargetCostKind CostKind) {
+                                   TTI::TargetCostKind CostKind) const {
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
 
   MVT MTy = LT.second;
@@ -6089,7 +6088,7 @@ InstructionCost X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
 
 InstructionCost X86TTIImpl::getCFInstrCost(unsigned Opcode,
                                            TTI::TargetCostKind CostKind,
-                                           const Instruction *I) {
+                                           const Instruction *I) const {
   if (CostKind != TTI::TCK_RecipThroughput)
     return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic;
   // Branches are assumed to be predicted.
@@ -6121,7 +6120,7 @@ InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
                                             TTI::TargetCostKind CostKind,
                                             Type *SrcVTy, const Value *Ptr,
                                             Align Alignment,
-                                            unsigned AddressSpace) {
+                                            unsigned AddressSpace) const {
 
   assert(isa<VectorType>(SrcVTy) && "Unexpected type in getGSVectorCost");
   unsigned VF = cast<FixedVectorType>(SrcVTy)->getNumElements();
@@ -6192,7 +6191,7 @@ InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
 InstructionCost X86TTIImpl::getGatherScatterOpCost(
     unsigned Opcode, Type *SrcVTy, const Value *Ptr, bool VariableMask,
     Align Alignment, TTI::TargetCostKind CostKind,
-    const Instruction *I = nullptr) {
+    const Instruction *I = nullptr) const {
   if ((Opcode == Instruction::Load &&
        (!isLegalMaskedGather(SrcVTy, Align(Alignment)) ||
         forceScalarizeMaskedGather(cast<VectorType>(SrcVTy),
@@ -6255,7 +6254,7 @@ static bool isLegalMaskedLoadStore(Type *ScalarTy, const X86Subtarget *ST) {
 }
 
 bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
-                                   unsigned AddressSpace) {
+                                   unsigned AddressSpace) const {
   Type *ScalarTy = DataTy->getScalarType();
 
   // The backend can't handle a single element vector w/o CFCMOV.
@@ -6268,7 +6267,7 @@ bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
 }
 
 bool X86TTIImpl::isLegalMaskedStore(Type *DataTy, Align Alignment,
-                                    unsigned AddressSpace) {
+                                    unsigned AddressSpace) const {
   Type *ScalarTy = DataTy->getScalarType();
 
   // The backend can't handle a single element vector w/o CFCMOV.
@@ -6358,7 +6357,8 @@ bool X86TTIImpl::supportsGather() const {
   return ST->hasAVX512() || (ST->hasFastGather() && ST->hasAVX2());
 }
 
-bool X86TTIImpl::forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) {
+bool X86TTIImpl::forceScalarizeMaskedGather(VectorType *VTy,
+                                            Align Alignment) const {
   // Gather / Scatter for vector 2 is not profitable on KNL / SKX
   // Vector-4 of gather/scatter instruction does not exist on KNL. We can extend
   // it to 8 elements, but zeroing upper bits of the mask vector will add more
@@ -6370,7 +6370,8 @@ bool X86TTIImpl::forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) {
          (ST->hasAVX512() && (NumElts == 2 || (NumElts == 4 && !ST->hasVLX())));
 }
 
-bool X86TTIImpl::isLegalMaskedGatherScatter(Type *DataTy, Align Alignment) {
+bool X86TTIImpl::isLegalMaskedGatherScatter(Type *DataTy,
+                                            Align Alignment) const {
   Type *ScalarTy = DataTy->getScalarType();
   if (ScalarTy->isPointerTy())
     return true;
@@ -6385,7 +6386,7 @@ bool X86TTIImpl::isLegalMaskedGatherScatter(Type *DataTy, Align Alignment) {
   return IntWidth == 32 || IntWidth == 64;
 }
 
-bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, Align Alignment) {
+bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, Align Alignment) const {
   if (!supportsGather() || !ST->preferGather())
     return false;
   return isLegalMaskedGatherScatter(DataTy, Alignment);
@@ -6424,7 +6425,7 @@ bool X86TTIImpl::isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
   return false;
 }
 
-bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) {
+bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) const {
   // AVX2 doesn't support scatter
   if (!ST->hasAVX512() || !ST->preferScatter())
     return false;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 5b6204d665206..f93b97e94550f 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -140,7 +140,8 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
+      ArrayRef<const Value *> Args = {},
+      const Instruction *CxtI = nullptr) const;
   InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
                                   unsigned Opcode1,
                                   const SmallBitVector &OpcodeMask,
@@ -151,43 +152,44 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args = {},
-                                 const Instruction *CxtI = nullptr);
+                                 const Instruction *CxtI = nullptr) const;
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I = nullptr);
+                                   const Instruction *I = nullptr) const;
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      const Instruction *I = nullptr);
+      const Instruction *I = nullptr) const;
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index, Value *Op0, Value *Op1);
+                                     unsigned Index, Value *Op0,
+                                     Value *Op1) const;
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
                                            TTI::TargetCostKind CostKind,
-                                           ArrayRef<Value *> VL = {});
+                                           ArrayRef<Value *> VL = {}) const;
   InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
                                             int VF,
                                             const APInt &DemandedDstElts,
-                                            TTI::TargetCostKind CostKind);
-  InstructionCost
-  getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
-                  unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
-                  const Instruction *I = nullptr);
+                                            TTI::TargetCostKind CostKind) const;
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const;
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
                                          Align Alignment,
                                          TTI::TargetCostKind CostKind,
-                                         const Instruction *I);
+                                         const Instruction *I) const;
   InstructionCost getPointersChainCost(ArrayRef<const Value *> Ptrs,
                                        const Value *Base,
                                        const TTI::PointersChainInfo &Info,
@@ -211,19 +213,20 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   unsigned getAtomicMemIntrinsicMaxElementSize() const;
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind);
+                                        TTI::TargetCostKind CostKind) const;
 
-  InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
-                                             std::optional<FastMathFlags> FMF,
-                                             TTI::TargetCostKind CostKind);
+  InstructionCost
+  getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
+                             std::optional<FastMathFlags> FMF,
+                             TTI::TargetCostKind CostKind) const;
 
   InstructionCost getMinMaxCost(Intrinsic::ID IID, Type *Ty,
                                 TTI::TargetCostKind CostKind,
-                                FastMathFlags FMF);
+                                FastMathFlags FMF) const;
 
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
-                                         TTI::TargetCostKind CostKind);
+                                         TTI::TargetCostKind CostKind) const;
 
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
@@ -241,7 +244,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                                 TTI::TargetCostKind CostKind);
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr);
+                                 const Instruction *I = nullptr) const;
 
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
@@ -263,19 +266,19 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                      const TargetTransformInfo::LSRCost &C2);
   bool canMacroFuseCmp();
   bool isLegalMaskedLoad(Type *DataType, Align Alignment,
-                         unsigned AddressSpace);
+                         unsigned AddressSpace) const;
   bool isLegalMaskedStore(Type *DataType, Align Alignment,
-                          unsigned AddressSpace);
+                          unsigned AddressSpace) const;
   bool isLegalNTLoad(Type *DataType, Align Alignment);
   bool isLegalNTStore(Type *DataType, Align Alignment);
   bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const;
-  bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment);
-  bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) {
+  bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) const;
+  bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) const {
     return forceScalarizeMaskedGather(VTy, Alignment);
   }
-  bool isLegalMaskedGatherScatter(Type *DataType, Align Alignment);
-  bool isLegalMaskedGather(Type *DataType, Align Alignment);
-  bool isLegalMaskedScatter(Type *DataType, Align Alignment);
+  bool isLegalMaskedGatherScatter(Type *DataType, Align Alignment) const;
+  bool isLegalMaskedGather(Type *DataType, Align Alignment) const;
+  bool isLegalMaskedScatter(Type *DataType, Align Alignment) const;
   bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment);
   bool isLegalMaskedCompressStore(Type *DataType, Align Alignment);
   bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
@@ -313,7 +316,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   bool supportsGather() const;
   InstructionCost getGSVectorCost(unsigned Opcode, TTI::TargetCostKind CostKind,
                                   Type *DataTy, const Value *Ptr,
-                                  Align Alignment, unsigned AddressSpace);
+                                  Align Alignment, unsigned AddressSpace) const;
 
   int getGatherOverhead() const;
   int getScatterOverhead() const;

From 0ae9dac262afccea1e1a2e02520f737ab38b286c Mon Sep 17 00:00:00 2001
From: Erich Keane <ekeane@nvidia.com>
Date: Mon, 21 Apr 2025 11:51:26 -0700
Subject: [PATCH 643/710] [OpenACC][CIR] Lower 'num_workers' for
 parallel/kernels (#136578)

This patch also includes the first one to handle 'device_type' properly,
which is where most of the 'challenge' here comes from.

From the best I can tell: we must keep two lists of the same size, 1 of
all of the 'num_workers' items, and 1 of the 'device_type' value for
that 'num_workers'. Additionally, the 'device_type' list can only handle
single 'device_type' values, so we have to duplicate the 'num_workers'
items in cases where there are multiple applicable 'device_type' values.

This patch accomplishes this by keeping the two in sync, and saving the
current 'device_type' in the visitor.
---
 clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp | 65 ++++++++++++++++++++-
 clang/test/CIR/CodeGenOpenACC/kernels.c     | 52 +++++++++++++++++
 clang/test/CIR/CodeGenOpenACC/parallel.c    | 52 +++++++++++++++++
 3 files changed, 168 insertions(+), 1 deletion(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index 30e840cbfa1f7..604fdf369860e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -46,10 +46,27 @@ class OpenACCClauseCIREmitter final
   // diagnostics are gone.
   SourceLocation dirLoc;
 
+  const OpenACCDeviceTypeClause *lastDeviceTypeClause = nullptr;
+
   void clauseNotImplemented(const OpenACCClause &c) {
     cgf.cgm.errorNYI(c.getSourceRange(), "OpenACC Clause", c.getClauseKind());
   }
 
+  mlir::Value createIntExpr(const Expr *intExpr) {
+    mlir::Value expr = cgf.emitScalarExpr(intExpr);
+    mlir::Location exprLoc = cgf.cgm.getLoc(intExpr->getBeginLoc());
+
+    mlir::IntegerType targetType = mlir::IntegerType::get(
+        &cgf.getMLIRContext(), cgf.getContext().getIntWidth(intExpr->getType()),
+        intExpr->getType()->isSignedIntegerOrEnumerationType()
+            ? mlir::IntegerType::SignednessSemantics::Signed
+            : mlir::IntegerType::SignednessSemantics::Unsigned);
+
+    auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>(
+        exprLoc, targetType, expr);
+    return conversionOp.getResult(0);
+  }
+
   // 'condition' as an OpenACC grammar production is used for 'if' and (some
   // variants of) 'self'.  It needs to be emitted as a signless-1-bit value, so
   // this function emits the expression, then sets the unrealized conversion
@@ -109,6 +126,7 @@ class OpenACCClauseCIREmitter final
   }
 
   void VisitDeviceTypeClause(const OpenACCDeviceTypeClause &clause) {
+    lastDeviceTypeClause = &clause;
     if constexpr (isOneOfTypes<OpTy, InitOp, ShutdownOp>) {
       llvm::SmallVector<mlir::Attribute> deviceTypes;
       std::optional<mlir::ArrayAttr> existingDeviceTypes =
@@ -116,7 +134,7 @@ class OpenACCClauseCIREmitter final
 
       // Ensure we keep the existing ones, and in the correct 'new' order.
       if (existingDeviceTypes) {
-        for (const mlir::Attribute &Attr : *existingDeviceTypes)
+        for (mlir::Attribute Attr : *existingDeviceTypes)
           deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
               builder.getContext(),
               cast<mlir::acc::DeviceTypeAttr>(Attr).getValue()));
@@ -136,6 +154,51 @@ class OpenACCClauseCIREmitter final
       if (!clause.getArchitectures().empty())
         operation.setDeviceType(
             decodeDeviceType(clause.getArchitectures()[0].getIdentifierInfo()));
+    } else if constexpr (isOneOfTypes<OpTy, ParallelOp, SerialOp, KernelsOp>) {
+      // Nothing to do here, these constructs don't have any IR for these, as
+      // they just modify the other clauses IR.  So setting of `lastDeviceType`
+      // (done above) is all we need.
+    } else {
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitNumWorkersClause(const OpenACCNumWorkersClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, ParallelOp, KernelsOp>) {
+      // Collect the 'existing' device-type attributes so we can re-create them
+      // and insert them.
+      llvm::SmallVector<mlir::Attribute> deviceTypes;
+      mlir::ArrayAttr existingDeviceTypes =
+          operation.getNumWorkersDeviceTypeAttr();
+
+      if (existingDeviceTypes) {
+        for (mlir::Attribute Attr : existingDeviceTypes)
+          deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+              builder.getContext(),
+              cast<mlir::acc::DeviceTypeAttr>(Attr).getValue()));
+      }
+
+      // Insert 1 version of the 'int-expr' to the NumWorkers list per-current
+      // device type.
+      mlir::Value intExpr = createIntExpr(clause.getIntExpr());
+      if (lastDeviceTypeClause) {
+        for (const DeviceTypeArgument &arg :
+             lastDeviceTypeClause->getArchitectures()) {
+          deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+              builder.getContext(), decodeDeviceType(arg.getIdentifierInfo())));
+          operation.getNumWorkersMutable().append(intExpr);
+        }
+      } else {
+        // Else, we just add a single for 'none'.
+        deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+            builder.getContext(), mlir::acc::DeviceType::None));
+        operation.getNumWorkersMutable().append(intExpr);
+      }
+
+      operation.setNumWorkersDeviceTypeAttr(
+          mlir::ArrayAttr::get(builder.getContext(), deviceTypes));
+    } else if constexpr (isOneOfTypes<OpTy, SerialOp>) {
+      llvm_unreachable("num_workers not valid on serial");
     } else {
       return clauseNotImplemented(clause);
     }
diff --git a/clang/test/CIR/CodeGenOpenACC/kernels.c b/clang/test/CIR/CodeGenOpenACC/kernels.c
index ca5bfebcb4ff3..6459b310546cd 100644
--- a/clang/test/CIR/CodeGenOpenACC/kernels.c
+++ b/clang/test/CIR/CodeGenOpenACC/kernels.c
@@ -106,5 +106,57 @@ void acc_kernels(int cond) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
 
+#pragma acc kernels num_workers(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: acc.kernels num_workers(%[[CONV_CAST]] : si32) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels num_workers(cond) device_type(nvidia) num_workers(2u)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !u32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !u32i to ui32
+  // CHECK-NEXT: acc.kernels num_workers(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : ui32 [#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels num_workers(cond) device_type(nvidia, host) num_workers(2) device_type(radeon) num_workers(3)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.kernels num_workers(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[TWO_CAST]] : si32 [#acc.device_type<host>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels num_workers(cond) device_type(nvidia) num_workers(2) device_type(radeon, multicore) num_workers(3)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.kernels num_workers(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>], %[[THREE_CAST]] : si32 [#acc.device_type<multicore>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels device_type(nvidia) num_workers(2) device_type(radeon) num_workers(3)
+  {}
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.kernels num_workers(%[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }
diff --git a/clang/test/CIR/CodeGenOpenACC/parallel.c b/clang/test/CIR/CodeGenOpenACC/parallel.c
index 3fb0b987409db..bdb506ee7e1d2 100644
--- a/clang/test/CIR/CodeGenOpenACC/parallel.c
+++ b/clang/test/CIR/CodeGenOpenACC/parallel.c
@@ -105,5 +105,57 @@ void acc_parallel(int cond) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
 
+#pragma acc parallel num_workers(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: acc.parallel num_workers(%[[CONV_CAST]] : si32) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel num_workers(cond) device_type(nvidia) num_workers(2u)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !u32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !u32i to ui32
+  // CHECK-NEXT: acc.parallel num_workers(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : ui32 [#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel num_workers(cond) device_type(nvidia, host) num_workers(2) device_type(radeon) num_workers(3)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.parallel num_workers(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[TWO_CAST]] : si32 [#acc.device_type<host>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel num_workers(cond) device_type(nvidia) num_workers(2) device_type(radeon, multicore) num_workers(4)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[FOUR_LITERAL:.*]] = cir.const #cir.int<4> : !s32i
+  // CHECK-NEXT: %[[FOUR_CAST:.*]] = builtin.unrealized_conversion_cast %[[FOUR_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.parallel num_workers(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[FOUR_CAST]] : si32 [#acc.device_type<radeon>], %[[FOUR_CAST]] : si32 [#acc.device_type<multicore>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel device_type(nvidia) num_workers(2) device_type(radeon) num_workers(3)
+  {}
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.parallel num_workers(%[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }

From 4b98955680e0aad33cad0e810f1daaa8bc7c0a23 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Mon, 21 Apr 2025 11:58:53 -0700
Subject: [PATCH 644/710] Fix capitalization re review from 0ae9dac

---
 clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index 604fdf369860e..b79baa96a3fc3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -134,10 +134,10 @@ class OpenACCClauseCIREmitter final
 
       // Ensure we keep the existing ones, and in the correct 'new' order.
       if (existingDeviceTypes) {
-        for (mlir::Attribute Attr : *existingDeviceTypes)
+        for (mlir::Attribute attr : *existingDeviceTypes)
           deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
               builder.getContext(),
-              cast<mlir::acc::DeviceTypeAttr>(Attr).getValue()));
+              cast<mlir::acc::DeviceTypeAttr>(attr).getValue()));
       }
 
       for (const DeviceTypeArgument &arg : clause.getArchitectures()) {

From 99e4b3927c5b8700b818ca2436f3c8394245e97c Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 21 Apr 2025 12:36:34 -0700
Subject: [PATCH 645/710] [LLVM] Cleanup pass initialization for Analysis
 passes (#135858)

- Do not call pass initialization from pass constructors.
- Instead, pass initialization should happen in the `initializeAnalysis`
function.
- https://github.com/llvm/llvm-project/issues/111767
---
 .../llvm/Analysis/LazyBlockFrequencyInfo.h    |  2 --
 llvm/include/llvm/InitializePasses.h          |  1 +
 llvm/lib/Analysis/AliasAnalysis.cpp           | 12 ++-----
 llvm/lib/Analysis/Analysis.cpp                |  9 +++++
 llvm/lib/Analysis/AssumptionCache.cpp         |  4 +--
 llvm/lib/Analysis/BasicAliasAnalysis.cpp      |  4 +--
 llvm/lib/Analysis/BlockFrequencyInfo.cpp      |  4 +--
 llvm/lib/Analysis/BranchProbabilityInfo.cpp   |  5 +--
 llvm/lib/Analysis/CallGraph.cpp               |  4 +--
 llvm/lib/Analysis/CycleAnalysis.cpp           |  4 +--
 llvm/lib/Analysis/DXILMetadataAnalysis.cpp    |  5 +--
 llvm/lib/Analysis/DXILResource.cpp            |  9 ++---
 llvm/lib/Analysis/DependenceAnalysis.cpp      |  4 +--
 llvm/lib/Analysis/DomPrinter.cpp              | 34 +++++--------------
 llvm/lib/Analysis/DominanceFrontier.cpp       |  4 +--
 llvm/lib/Analysis/GlobalsModRef.cpp           |  4 +--
 llvm/lib/Analysis/IRSimilarityIdentifier.cpp  |  5 +--
 llvm/lib/Analysis/IVUsers.cpp                 |  4 +--
 llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp  |  4 +--
 llvm/lib/Analysis/LazyValueInfo.cpp           |  4 +--
 llvm/lib/Analysis/LoopInfo.cpp                |  4 +--
 llvm/lib/Analysis/LoopPass.cpp                |  4 +--
 .../lib/Analysis/MemoryDependenceAnalysis.cpp |  4 +--
 llvm/lib/Analysis/MemorySSA.cpp               |  4 +--
 llvm/lib/Analysis/ModuleSummaryAnalysis.cpp   |  9 ++---
 .../Analysis/OptimizationRemarkEmitter.cpp    |  5 +--
 llvm/lib/Analysis/PhiValues.cpp               |  4 +--
 llvm/lib/Analysis/PostDominators.cpp          |  4 +--
 llvm/lib/Analysis/ProfileSummaryInfo.cpp      |  4 +--
 llvm/lib/Analysis/RegionInfo.cpp              |  4 +--
 llvm/lib/Analysis/RegionPrinter.cpp           | 10 ++----
 llvm/lib/Analysis/ScalarEvolution.cpp         |  4 +--
 .../Analysis/ScalarEvolutionAliasAnalysis.cpp |  4 +--
 llvm/lib/Analysis/ScopedNoAliasAA.cpp         |  4 +--
 llvm/lib/Analysis/StackSafetyAnalysis.cpp     |  9 ++---
 llvm/lib/Analysis/StaticDataProfileInfo.cpp   |  5 +--
 llvm/lib/Analysis/TargetLibraryInfo.cpp       | 12 ++-----
 llvm/lib/Analysis/TargetTransformInfo.cpp     | 10 ++----
 llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp  |  4 +--
 llvm/lib/Analysis/UniformityAnalysis.cpp      |  4 +--
 llvm/lib/CodeGen/TargetPassConfig.cpp         | 11 +++---
 41 files changed, 70 insertions(+), 179 deletions(-)

diff --git a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
index 8ad3c6bafc26d..0ce882fcd7a9d 100644
--- a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
@@ -122,7 +122,5 @@ class LazyBlockFrequencyInfoPass : public FunctionPass {
   void print(raw_ostream &OS, const Module *M) const override;
 };
 
-/// Helper for client passes to initialize dependent passes for LBFI.
-void initializeLazyBFIPassPass(PassRegistry &Registry);
 } // namespace llvm
 #endif
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 9bef8e496c57e..1ce36a95317b4 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -145,6 +145,7 @@ void initializeJMCInstrumenterPass(PassRegistry &);
 void initializeKCFIPass(PassRegistry &);
 void initializeLCSSAVerificationPassPass(PassRegistry &);
 void initializeLCSSAWrapperPassPass(PassRegistry &);
+void initializeLazyBFIPassPass(PassRegistry &);
 void initializeLazyBlockFrequencyInfoPassPass(PassRegistry &);
 void initializeLazyBranchProbabilityInfoPassPass(PassRegistry &);
 void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry &);
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index 2a8fca9ac2039..efabf69b06047 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -688,14 +688,10 @@ AAResults::Concept::~Concept() = default;
 // Provide a definition for the static object used to identify passes.
 AnalysisKey AAManager::Key;
 
-ExternalAAWrapperPass::ExternalAAWrapperPass() : ImmutablePass(ID) {
-  initializeExternalAAWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+ExternalAAWrapperPass::ExternalAAWrapperPass() : ImmutablePass(ID) {}
 
 ExternalAAWrapperPass::ExternalAAWrapperPass(CallbackT CB)
-    : ImmutablePass(ID), CB(std::move(CB)) {
-  initializeExternalAAWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID), CB(std::move(CB)) {}
 
 char ExternalAAWrapperPass::ID = 0;
 
@@ -707,9 +703,7 @@ llvm::createExternalAAWrapperPass(ExternalAAWrapperPass::CallbackT Callback) {
   return new ExternalAAWrapperPass(std::move(Callback));
 }
 
-AAResultsWrapperPass::AAResultsWrapperPass() : FunctionPass(ID) {
-  initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+AAResultsWrapperPass::AAResultsWrapperPass() : FunctionPass(ID) {}
 
 char AAResultsWrapperPass::ID = 0;
 
diff --git a/llvm/lib/Analysis/Analysis.cpp b/llvm/lib/Analysis/Analysis.cpp
index 5cd5115bc4cd4..43ea30f3a9a81 100644
--- a/llvm/lib/Analysis/Analysis.cpp
+++ b/llvm/lib/Analysis/Analysis.cpp
@@ -18,6 +18,7 @@ using namespace llvm;
 
 /// initializeAnalysis - Initialize all passes linked into the Analysis library.
 void llvm::initializeAnalysis(PassRegistry &Registry) {
+  initializeAssumptionCacheTrackerPass(Registry);
   initializeBasicAAWrapperPassPass(Registry);
   initializeBlockFrequencyInfoWrapperPassPass(Registry);
   initializeBranchProbabilityInfoWrapperPassPass(Registry);
@@ -25,6 +26,7 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializeCallGraphDOTPrinterPass(Registry);
   initializeCallGraphViewerPass(Registry);
   initializeCycleInfoWrapperPassPass(Registry);
+  initializeDXILMetadataAnalysisWrapperPassPass(Registry);
   initializeDXILResourceWrapperPassPass(Registry);
   initializeDXILResourceTypeWrapperPassPass(Registry);
   initializeDependenceAnalysisWrapperPassPass(Registry);
@@ -39,9 +41,12 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializePostDomOnlyPrinterWrapperPassPass(Registry);
   initializeAAResultsWrapperPassPass(Registry);
   initializeGlobalsAAWrapperPassPass(Registry);
+  initializeExternalAAWrapperPassPass(Registry);
+  initializeImmutableModuleSummaryIndexWrapperPassPass(Registry);
   initializeIVUsersWrapperPassPass(Registry);
   initializeIRSimilarityIdentifierWrapperPassPass(Registry);
   initializeLazyBranchProbabilityInfoPassPass(Registry);
+  initializeLazyBFIPassPass(Registry);
   initializeLazyBlockFrequencyInfoPassPass(Registry);
   initializeLazyValueInfoWrapperPassPass(Registry);
   initializeLoopInfoWrapperPassPass(Registry);
@@ -50,6 +55,7 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializeOptimizationRemarkEmitterWrapperPassPass(Registry);
   initializePhiValuesWrapperPassPass(Registry);
   initializePostDominatorTreeWrapperPassPass(Registry);
+  initializeProfileSummaryInfoWrapperPassPass(Registry);
   initializeRegionInfoPassPass(Registry);
   initializeRegionViewerPass(Registry);
   initializeRegionPrinterPass(Registry);
@@ -59,11 +65,14 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializeScalarEvolutionWrapperPassPass(Registry);
   initializeStackSafetyGlobalInfoWrapperPassPass(Registry);
   initializeStackSafetyInfoWrapperPassPass(Registry);
+  initializeTargetLibraryInfoWrapperPassPass(Registry);
   initializeTargetTransformInfoWrapperPassPass(Registry);
   initializeTypeBasedAAWrapperPassPass(Registry);
   initializeScopedNoAliasAAWrapperPassPass(Registry);
+  initializeStaticDataProfileInfoWrapperPassPass(Registry);
   initializeLCSSAVerificationPassPass(Registry);
   initializeMemorySSAWrapperPassPass(Registry);
+  initializeUniformityInfoWrapperPassPass(Registry);
 }
 
 LLVMBool LLVMVerifyModule(LLVMModuleRef M, LLVMVerifierFailureAction Action,
diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp
index a0e57ab741dfa..980a891266e50 100644
--- a/llvm/lib/Analysis/AssumptionCache.cpp
+++ b/llvm/lib/Analysis/AssumptionCache.cpp
@@ -292,9 +292,7 @@ void AssumptionCacheTracker::verifyAnalysis() const {
   }
 }
 
-AssumptionCacheTracker::AssumptionCacheTracker() : ImmutablePass(ID) {
-  initializeAssumptionCacheTrackerPass(*PassRegistry::getPassRegistry());
-}
+AssumptionCacheTracker::AssumptionCacheTracker() : ImmutablePass(ID) {}
 
 AssumptionCacheTracker::~AssumptionCacheTracker() = default;
 
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 12d9c8706a8e1..cbec13c7808be 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -1968,9 +1968,7 @@ BasicAAResult BasicAA::run(Function &F, FunctionAnalysisManager &AM) {
   return BasicAAResult(F.getDataLayout(), F, TLI, AC, DT);
 }
 
-BasicAAWrapperPass::BasicAAWrapperPass() : FunctionPass(ID) {
-  initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+BasicAAWrapperPass::BasicAAWrapperPass() : FunctionPass(ID) {}
 
 char BasicAAWrapperPass::ID = 0;
 
diff --git a/llvm/lib/Analysis/BlockFrequencyInfo.cpp b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
index d1b21e8c83f2c..df99ec299a409 100644
--- a/llvm/lib/Analysis/BlockFrequencyInfo.cpp
+++ b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
@@ -302,9 +302,7 @@ INITIALIZE_PASS_END(BlockFrequencyInfoWrapperPass, "block-freq",
 char BlockFrequencyInfoWrapperPass::ID = 0;
 
 BlockFrequencyInfoWrapperPass::BlockFrequencyInfoWrapperPass()
-    : FunctionPass(ID) {
-  initializeBlockFrequencyInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : FunctionPass(ID) {}
 
 BlockFrequencyInfoWrapperPass::~BlockFrequencyInfoWrapperPass() = default;
 
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index 62c2b361f1b81..71c7319d98473 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -69,10 +69,7 @@ INITIALIZE_PASS_END(BranchProbabilityInfoWrapperPass, "branch-prob",
                     "Branch Probability Analysis", false, true)
 
 BranchProbabilityInfoWrapperPass::BranchProbabilityInfoWrapperPass()
-    : FunctionPass(ID) {
-  initializeBranchProbabilityInfoWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : FunctionPass(ID) {}
 
 char BranchProbabilityInfoWrapperPass::ID = 0;
 
diff --git a/llvm/lib/Analysis/CallGraph.cpp b/llvm/lib/Analysis/CallGraph.cpp
index 01344ee154fa5..5d1af52e8ab58 100644
--- a/llvm/lib/Analysis/CallGraph.cpp
+++ b/llvm/lib/Analysis/CallGraph.cpp
@@ -300,9 +300,7 @@ PreservedAnalyses CallGraphSCCsPrinterPass::run(Module &M,
 // Implementations of the CallGraphWrapperPass class methods.
 //
 
-CallGraphWrapperPass::CallGraphWrapperPass() : ModulePass(ID) {
-  initializeCallGraphWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+CallGraphWrapperPass::CallGraphWrapperPass() : ModulePass(ID) {}
 
 CallGraphWrapperPass::~CallGraphWrapperPass() = default;
 
diff --git a/llvm/lib/Analysis/CycleAnalysis.cpp b/llvm/lib/Analysis/CycleAnalysis.cpp
index bdba3ab2acfc9..1e7c5e966babb 100644
--- a/llvm/lib/Analysis/CycleAnalysis.cpp
+++ b/llvm/lib/Analysis/CycleAnalysis.cpp
@@ -52,9 +52,7 @@ PreservedAnalyses CycleInfoVerifierPass::run(Function &F,
 
 char CycleInfoWrapperPass::ID = 0;
 
-CycleInfoWrapperPass::CycleInfoWrapperPass() : FunctionPass(ID) {
-  initializeCycleInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+CycleInfoWrapperPass::CycleInfoWrapperPass() : FunctionPass(ID) {}
 
 INITIALIZE_PASS_BEGIN(CycleInfoWrapperPass, "cycles", "Cycle Info Analysis",
                       true, true)
diff --git a/llvm/lib/Analysis/DXILMetadataAnalysis.cpp b/llvm/lib/Analysis/DXILMetadataAnalysis.cpp
index 87aa14e9c701f..23f1aa82ae8a3 100644
--- a/llvm/lib/Analysis/DXILMetadataAnalysis.cpp
+++ b/llvm/lib/Analysis/DXILMetadataAnalysis.cpp
@@ -109,10 +109,7 @@ DXILMetadataAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &AM) {
 // DXILMetadataAnalysisWrapperPass
 
 DXILMetadataAnalysisWrapperPass::DXILMetadataAnalysisWrapperPass()
-    : ModulePass(ID) {
-  initializeDXILMetadataAnalysisWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : ModulePass(ID) {}
 
 DXILMetadataAnalysisWrapperPass::~DXILMetadataAnalysisWrapperPass() = default;
 
diff --git a/llvm/lib/Analysis/DXILResource.cpp b/llvm/lib/Analysis/DXILResource.cpp
index 1c4348321c1d0..6334f5d82213d 100644
--- a/llvm/lib/Analysis/DXILResource.cpp
+++ b/llvm/lib/Analysis/DXILResource.cpp
@@ -856,9 +856,8 @@ PreservedAnalyses DXILResourcePrinterPass::run(Module &M,
 
 void DXILResourceTypeWrapperPass::anchor() {}
 
-DXILResourceTypeWrapperPass::DXILResourceTypeWrapperPass() : ImmutablePass(ID) {
-  initializeDXILResourceTypeWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+DXILResourceTypeWrapperPass::DXILResourceTypeWrapperPass()
+    : ImmutablePass(ID) {}
 
 INITIALIZE_PASS(DXILResourceTypeWrapperPass, "dxil-resource-type",
                 "DXIL Resource Type Analysis", false, true)
@@ -868,9 +867,7 @@ ModulePass *llvm::createDXILResourceTypeWrapperPassPass() {
   return new DXILResourceTypeWrapperPass();
 }
 
-DXILResourceWrapperPass::DXILResourceWrapperPass() : ModulePass(ID) {
-  initializeDXILResourceWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+DXILResourceWrapperPass::DXILResourceWrapperPass() : ModulePass(ID) {}
 
 DXILResourceWrapperPass::~DXILResourceWrapperPass() = default;
 
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index dc0ed22dbcc0b..580cf43be3cb9 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -145,9 +145,7 @@ INITIALIZE_PASS_END(DependenceAnalysisWrapperPass, "da", "Dependence Analysis",
 char DependenceAnalysisWrapperPass::ID = 0;
 
 DependenceAnalysisWrapperPass::DependenceAnalysisWrapperPass()
-    : FunctionPass(ID) {
-  initializeDependenceAnalysisWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : FunctionPass(ID) {}
 
 FunctionPass *llvm::createDependenceAnalysisWrapperPass() {
   return new DependenceAnalysisWrapperPass();
diff --git a/llvm/lib/Analysis/DomPrinter.cpp b/llvm/lib/Analysis/DomPrinter.cpp
index e9f5103e12761..892f664a4e096 100644
--- a/llvm/lib/Analysis/DomPrinter.cpp
+++ b/llvm/lib/Analysis/DomPrinter.cpp
@@ -56,9 +56,7 @@ struct DomViewerWrapperPass
   DomViewerWrapperPass()
       : DOTGraphTraitsViewerWrapperPass<
             DominatorTreeWrapperPass, false, DominatorTree *,
-            LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("dom", ID) {
-    initializeDomViewerWrapperPassPass(*PassRegistry::getPassRegistry());
-  }
+            LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("dom", ID) {}
 };
 
 struct DomOnlyViewerWrapperPass
@@ -69,9 +67,7 @@ struct DomOnlyViewerWrapperPass
   DomOnlyViewerWrapperPass()
       : DOTGraphTraitsViewerWrapperPass<
             DominatorTreeWrapperPass, true, DominatorTree *,
-            LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("domonly", ID) {
-    initializeDomOnlyViewerWrapperPassPass(*PassRegistry::getPassRegistry());
-  }
+            LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("domonly", ID) {}
 };
 
 struct LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits {
@@ -89,9 +85,7 @@ struct PostDomViewerWrapperPass
       : DOTGraphTraitsViewerWrapperPass<
             PostDominatorTreeWrapperPass, false, PostDominatorTree *,
             LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits>("postdom",
-                                                                   ID) {
-    initializePostDomViewerWrapperPassPass(*PassRegistry::getPassRegistry());
-  }
+                                                                   ID) {}
 };
 
 struct PostDomOnlyViewerWrapperPass
@@ -103,10 +97,7 @@ struct PostDomOnlyViewerWrapperPass
       : DOTGraphTraitsViewerWrapperPass<
             PostDominatorTreeWrapperPass, true, PostDominatorTree *,
             LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits>(
-            "postdomonly", ID) {
-    initializePostDomOnlyViewerWrapperPassPass(
-        *PassRegistry::getPassRegistry());
-  }
+            "postdomonly", ID) {}
 };
 } // end anonymous namespace
 
@@ -138,9 +129,7 @@ struct DomPrinterWrapperPass
   DomPrinterWrapperPass()
       : DOTGraphTraitsPrinterWrapperPass<
             DominatorTreeWrapperPass, false, DominatorTree *,
-            LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("dom", ID) {
-    initializeDomPrinterWrapperPassPass(*PassRegistry::getPassRegistry());
-  }
+            LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("dom", ID) {}
 };
 
 struct DomOnlyPrinterWrapperPass
@@ -151,9 +140,7 @@ struct DomOnlyPrinterWrapperPass
   DomOnlyPrinterWrapperPass()
       : DOTGraphTraitsPrinterWrapperPass<
             DominatorTreeWrapperPass, true, DominatorTree *,
-            LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("domonly", ID) {
-    initializeDomOnlyPrinterWrapperPassPass(*PassRegistry::getPassRegistry());
-  }
+            LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("domonly", ID) {}
 };
 
 struct PostDomPrinterWrapperPass
@@ -165,9 +152,7 @@ struct PostDomPrinterWrapperPass
       : DOTGraphTraitsPrinterWrapperPass<
             PostDominatorTreeWrapperPass, false, PostDominatorTree *,
             LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits>("postdom",
-                                                                   ID) {
-    initializePostDomPrinterWrapperPassPass(*PassRegistry::getPassRegistry());
-  }
+                                                                   ID) {}
 };
 
 struct PostDomOnlyPrinterWrapperPass
@@ -179,10 +164,7 @@ struct PostDomOnlyPrinterWrapperPass
       : DOTGraphTraitsPrinterWrapperPass<
             PostDominatorTreeWrapperPass, true, PostDominatorTree *,
             LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits>(
-            "postdomonly", ID) {
-    initializePostDomOnlyPrinterWrapperPassPass(
-        *PassRegistry::getPassRegistry());
-  }
+            "postdomonly", ID) {}
 };
 } // end anonymous namespace
 
diff --git a/llvm/lib/Analysis/DominanceFrontier.cpp b/llvm/lib/Analysis/DominanceFrontier.cpp
index ccba913ccfe5e..b7396a17f35d2 100644
--- a/llvm/lib/Analysis/DominanceFrontier.cpp
+++ b/llvm/lib/Analysis/DominanceFrontier.cpp
@@ -36,9 +36,7 @@ INITIALIZE_PASS_END(DominanceFrontierWrapperPass, "domfrontier",
                 "Dominance Frontier Construction", true, true)
 
 DominanceFrontierWrapperPass::DominanceFrontierWrapperPass()
-    : FunctionPass(ID) {
-  initializeDominanceFrontierWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : FunctionPass(ID) {}
 
 void DominanceFrontierWrapperPass::releaseMemory() {
   DF.releaseMemory();
diff --git a/llvm/lib/Analysis/GlobalsModRef.cpp b/llvm/lib/Analysis/GlobalsModRef.cpp
index 644969dd25212..295e267848b23 100644
--- a/llvm/lib/Analysis/GlobalsModRef.cpp
+++ b/llvm/lib/Analysis/GlobalsModRef.cpp
@@ -1048,9 +1048,7 @@ ModulePass *llvm::createGlobalsAAWrapperPass() {
   return new GlobalsAAWrapperPass();
 }
 
-GlobalsAAWrapperPass::GlobalsAAWrapperPass() : ModulePass(ID) {
-  initializeGlobalsAAWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+GlobalsAAWrapperPass::GlobalsAAWrapperPass() : ModulePass(ID) {}
 
 bool GlobalsAAWrapperPass::runOnModule(Module &M) {
   auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
diff --git a/llvm/lib/Analysis/IRSimilarityIdentifier.cpp b/llvm/lib/Analysis/IRSimilarityIdentifier.cpp
index a6af7304b1c7e..ffb684d284a94 100644
--- a/llvm/lib/Analysis/IRSimilarityIdentifier.cpp
+++ b/llvm/lib/Analysis/IRSimilarityIdentifier.cpp
@@ -1472,10 +1472,7 @@ INITIALIZE_PASS(IRSimilarityIdentifierWrapperPass, "ir-similarity-identifier",
                 "ir-similarity-identifier", false, true)
 
 IRSimilarityIdentifierWrapperPass::IRSimilarityIdentifierWrapperPass()
-    : ModulePass(ID) {
-  initializeIRSimilarityIdentifierWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : ModulePass(ID) {}
 
 bool IRSimilarityIdentifierWrapperPass::doInitialization(Module &M) {
   IRSI.reset(new IRSimilarityIdentifier(!DisableBranches, !DisableIndirectCalls,
diff --git a/llvm/lib/Analysis/IVUsers.cpp b/llvm/lib/Analysis/IVUsers.cpp
index 0880701d83082..fb7778e9ef153 100644
--- a/llvm/lib/Analysis/IVUsers.cpp
+++ b/llvm/lib/Analysis/IVUsers.cpp
@@ -297,9 +297,7 @@ void IVUsers::releaseMemory() {
   IVUses.clear();
 }
 
-IVUsersWrapperPass::IVUsersWrapperPass() : LoopPass(ID) {
-  initializeIVUsersWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+IVUsersWrapperPass::IVUsersWrapperPass() : LoopPass(ID) {}
 
 void IVUsersWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<AssumptionCacheTracker>();
diff --git a/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp b/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
index 636baf82eedff..ac1072843fd7f 100644
--- a/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
+++ b/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
@@ -32,9 +32,7 @@ INITIALIZE_PASS_END(LazyBlockFrequencyInfoPass, DEBUG_TYPE,
 
 char LazyBlockFrequencyInfoPass::ID = 0;
 
-LazyBlockFrequencyInfoPass::LazyBlockFrequencyInfoPass() : FunctionPass(ID) {
-  initializeLazyBlockFrequencyInfoPassPass(*PassRegistry::getPassRegistry());
-}
+LazyBlockFrequencyInfoPass::LazyBlockFrequencyInfoPass() : FunctionPass(ID) {}
 
 void LazyBlockFrequencyInfoPass::print(raw_ostream &OS, const Module *) const {
   LBFI.getCalculated().print(OS);
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 581358a645780..e49e004a48a51 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -51,9 +51,7 @@ using namespace PatternMatch;
 static const unsigned MaxProcessedPerValue = 500;
 
 char LazyValueInfoWrapperPass::ID = 0;
-LazyValueInfoWrapperPass::LazyValueInfoWrapperPass() : FunctionPass(ID) {
-  initializeLazyValueInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+LazyValueInfoWrapperPass::LazyValueInfoWrapperPass() : FunctionPass(ID) {}
 INITIALIZE_PASS_BEGIN(LazyValueInfoWrapperPass, "lazy-value-info",
                 "Lazy Value Information Analysis", false, true)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp
index 0eaf1dca59675..3ef9fb282afdb 100644
--- a/llvm/lib/Analysis/LoopInfo.cpp
+++ b/llvm/lib/Analysis/LoopInfo.cpp
@@ -1210,9 +1210,7 @@ MDNode *llvm::makePostTransformationMetadata(LLVMContext &Context,
 // LoopInfo implementation
 //
 
-LoopInfoWrapperPass::LoopInfoWrapperPass() : FunctionPass(ID) {
-  initializeLoopInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+LoopInfoWrapperPass::LoopInfoWrapperPass() : FunctionPass(ID) {}
 
 char LoopInfoWrapperPass::ID = 0;
 INITIALIZE_PASS_BEGIN(LoopInfoWrapperPass, "loops", "Natural Loop Information",
diff --git a/llvm/lib/Analysis/LoopPass.cpp b/llvm/lib/Analysis/LoopPass.cpp
index 3dc9e75b6168f..5d0f9abbb5ce7 100644
--- a/llvm/lib/Analysis/LoopPass.cpp
+++ b/llvm/lib/Analysis/LoopPass.cpp
@@ -388,9 +388,7 @@ bool LoopPass::skipLoop(const Loop *L) const {
   return false;
 }
 
-LCSSAVerificationPass::LCSSAVerificationPass() : FunctionPass(ID) {
-  initializeLCSSAVerificationPassPass(*PassRegistry::getPassRegistry());
-}
+LCSSAVerificationPass::LCSSAVerificationPass() : FunctionPass(ID) {}
 
 char LCSSAVerificationPass::ID = 0;
 INITIALIZE_PASS(LCSSAVerificationPass, "lcssa-verification", "LCSSA Verifier",
diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index 8550aaacb4fc9..f062189bac6a0 100644
--- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -1729,9 +1729,7 @@ INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_END(MemoryDependenceWrapperPass, "memdep",
                     "Memory Dependence Analysis", false, true)
 
-MemoryDependenceWrapperPass::MemoryDependenceWrapperPass() : FunctionPass(ID) {
-  initializeMemoryDependenceWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+MemoryDependenceWrapperPass::MemoryDependenceWrapperPass() : FunctionPass(ID) {}
 
 MemoryDependenceWrapperPass::~MemoryDependenceWrapperPass() = default;
 
diff --git a/llvm/lib/Analysis/MemorySSA.cpp b/llvm/lib/Analysis/MemorySSA.cpp
index fe3aea3039a49..9f8979a9043dc 100644
--- a/llvm/lib/Analysis/MemorySSA.cpp
+++ b/llvm/lib/Analysis/MemorySSA.cpp
@@ -2414,9 +2414,7 @@ PreservedAnalyses MemorySSAVerifierPass::run(Function &F,
 
 char MemorySSAWrapperPass::ID = 0;
 
-MemorySSAWrapperPass::MemorySSAWrapperPass() : FunctionPass(ID) {
-  initializeMemorySSAWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+MemorySSAWrapperPass::MemorySSAWrapperPass() : FunctionPass(ID) {}
 
 void MemorySSAWrapperPass::releaseMemory() { MSSA.reset(); }
 
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 5693e235836bf..4b50f1e31277b 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -1156,9 +1156,7 @@ ModulePass *llvm::createModuleSummaryIndexWrapperPass() {
 }
 
 ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass()
-    : ModulePass(ID) {
-  initializeModuleSummaryIndexWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : ModulePass(ID) {}
 
 bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) {
   auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
@@ -1196,10 +1194,7 @@ char ImmutableModuleSummaryIndexWrapperPass::ID = 0;
 
 ImmutableModuleSummaryIndexWrapperPass::ImmutableModuleSummaryIndexWrapperPass(
     const ModuleSummaryIndex *Index)
-    : ImmutablePass(ID), Index(Index) {
-  initializeImmutableModuleSummaryIndexWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID), Index(Index) {}
 
 void ImmutableModuleSummaryIndexWrapperPass::getAnalysisUsage(
     AnalysisUsage &AU) const {
diff --git a/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp b/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp
index 95d55ea44dbfd..fbb5971e10750 100644
--- a/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp
+++ b/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp
@@ -91,10 +91,7 @@ void OptimizationRemarkEmitter::emit(
 }
 
 OptimizationRemarkEmitterWrapperPass::OptimizationRemarkEmitterWrapperPass()
-    : FunctionPass(ID) {
-  initializeOptimizationRemarkEmitterWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : FunctionPass(ID) {}
 
 bool OptimizationRemarkEmitterWrapperPass::runOnFunction(Function &Fn) {
   BlockFrequencyInfo *BFI;
diff --git a/llvm/lib/Analysis/PhiValues.cpp b/llvm/lib/Analysis/PhiValues.cpp
index dad91a91202c5..ee1363f485f6e 100644
--- a/llvm/lib/Analysis/PhiValues.cpp
+++ b/llvm/lib/Analysis/PhiValues.cpp
@@ -205,9 +205,7 @@ PreservedAnalyses PhiValuesPrinterPass::run(Function &F,
   return PreservedAnalyses::all();
 }
 
-PhiValuesWrapperPass::PhiValuesWrapperPass() : FunctionPass(ID) {
-  initializePhiValuesWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+PhiValuesWrapperPass::PhiValuesWrapperPass() : FunctionPass(ID) {}
 
 bool PhiValuesWrapperPass::runOnFunction(Function &F) {
   Result.reset(new PhiValues(F));
diff --git a/llvm/lib/Analysis/PostDominators.cpp b/llvm/lib/Analysis/PostDominators.cpp
index f01d51504d7cd..c71eb418c0353 100644
--- a/llvm/lib/Analysis/PostDominators.cpp
+++ b/llvm/lib/Analysis/PostDominators.cpp
@@ -35,9 +35,7 @@ static constexpr bool ExpensiveChecksEnabled = false;
 char PostDominatorTreeWrapperPass::ID = 0;
 
 PostDominatorTreeWrapperPass::PostDominatorTreeWrapperPass()
-    : FunctionPass(ID) {
-  initializePostDominatorTreeWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : FunctionPass(ID) {}
 
 INITIALIZE_PASS(PostDominatorTreeWrapperPass, "postdomtree",
                 "Post-Dominator Tree Construction", true, true)
diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
index d3d7fb997b9d3..e098c7a448ab0 100644
--- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp
+++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
@@ -227,9 +227,7 @@ INITIALIZE_PASS(ProfileSummaryInfoWrapperPass, "profile-summary-info",
                 "Profile summary info", false, true)
 
 ProfileSummaryInfoWrapperPass::ProfileSummaryInfoWrapperPass()
-    : ImmutablePass(ID) {
-  initializeProfileSummaryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID) {}
 
 bool ProfileSummaryInfoWrapperPass::doInitialization(Module &M) {
   PSI.reset(new ProfileSummaryInfo(M));
diff --git a/llvm/lib/Analysis/RegionInfo.cpp b/llvm/lib/Analysis/RegionInfo.cpp
index 15257b4a9a926..a0177b0c8810c 100644
--- a/llvm/lib/Analysis/RegionInfo.cpp
+++ b/llvm/lib/Analysis/RegionInfo.cpp
@@ -116,9 +116,7 @@ void RegionInfo::viewOnly() { viewRegionOnly(this); }
 // RegionInfoPass implementation
 //
 
-RegionInfoPass::RegionInfoPass() : FunctionPass(ID) {
-  initializeRegionInfoPassPass(*PassRegistry::getPassRegistry());
-}
+RegionInfoPass::RegionInfoPass() : FunctionPass(ID) {}
 
 RegionInfoPass::~RegionInfoPass() = default;
 
diff --git a/llvm/lib/Analysis/RegionPrinter.cpp b/llvm/lib/Analysis/RegionPrinter.cpp
index ad59c79f4075d..a83af4ebb430e 100644
--- a/llvm/lib/Analysis/RegionPrinter.cpp
+++ b/llvm/lib/Analysis/RegionPrinter.cpp
@@ -142,7 +142,6 @@ struct RegionPrinter
   RegionPrinter()
       : DOTGraphTraitsPrinterWrapperPass<RegionInfoPass, false, RegionInfo *,
                                          RegionInfoPassGraphTraits>("reg", ID) {
-    initializeRegionPrinterPass(*PassRegistry::getPassRegistry());
   }
 };
 char RegionPrinter::ID = 0;
@@ -154,7 +153,6 @@ struct RegionOnlyPrinter
   RegionOnlyPrinter()
       : DOTGraphTraitsPrinterWrapperPass<RegionInfoPass, true, RegionInfo *,
                                          RegionInfoPassGraphTraits>("reg", ID) {
-    initializeRegionOnlyPrinterPass(*PassRegistry::getPassRegistry());
   }
 };
 char RegionOnlyPrinter::ID = 0;
@@ -165,9 +163,7 @@ struct RegionViewer
   static char ID;
   RegionViewer()
       : DOTGraphTraitsViewerWrapperPass<RegionInfoPass, false, RegionInfo *,
-                                        RegionInfoPassGraphTraits>("reg", ID) {
-    initializeRegionViewerPass(*PassRegistry::getPassRegistry());
-  }
+                                        RegionInfoPassGraphTraits>("reg", ID) {}
 };
 char RegionViewer::ID = 0;
 
@@ -178,9 +174,7 @@ struct RegionOnlyViewer
   RegionOnlyViewer()
       : DOTGraphTraitsViewerWrapperPass<RegionInfoPass, true, RegionInfo *,
                                         RegionInfoPassGraphTraits>("regonly",
-                                                                   ID) {
-    initializeRegionOnlyViewerPass(*PassRegistry::getPassRegistry());
-  }
+                                                                   ID) {}
 };
 char RegionOnlyViewer::ID = 0;
 
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 5132ee13a9632..471c71f75eb23 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -14685,9 +14685,7 @@ INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",
 
 char ScalarEvolutionWrapperPass::ID = 0;
 
-ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {
-  initializeScalarEvolutionWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {}
 
 bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
   SE.reset(new ScalarEvolution(
diff --git a/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp b/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
index 7bcec7931b219..ee5480c460836 100644
--- a/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
@@ -166,9 +166,7 @@ FunctionPass *llvm::createSCEVAAWrapperPass() {
   return new SCEVAAWrapperPass();
 }
 
-SCEVAAWrapperPass::SCEVAAWrapperPass() : FunctionPass(ID) {
-  initializeSCEVAAWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+SCEVAAWrapperPass::SCEVAAWrapperPass() : FunctionPass(ID) {}
 
 bool SCEVAAWrapperPass::runOnFunction(Function &F) {
   Result.reset(
diff --git a/llvm/lib/Analysis/ScopedNoAliasAA.cpp b/llvm/lib/Analysis/ScopedNoAliasAA.cpp
index 3815bdf49d59c..a951ecb27a5af 100644
--- a/llvm/lib/Analysis/ScopedNoAliasAA.cpp
+++ b/llvm/lib/Analysis/ScopedNoAliasAA.cpp
@@ -161,9 +161,7 @@ ImmutablePass *llvm::createScopedNoAliasAAWrapperPass() {
   return new ScopedNoAliasAAWrapperPass();
 }
 
-ScopedNoAliasAAWrapperPass::ScopedNoAliasAAWrapperPass() : ImmutablePass(ID) {
-  initializeScopedNoAliasAAWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+ScopedNoAliasAAWrapperPass::ScopedNoAliasAAWrapperPass() : ImmutablePass(ID) {}
 
 bool ScopedNoAliasAAWrapperPass::doInitialization(Module &M) {
   Result.reset(new ScopedNoAliasAAResult());
diff --git a/llvm/lib/Analysis/StackSafetyAnalysis.cpp b/llvm/lib/Analysis/StackSafetyAnalysis.cpp
index fde5345229f55..5e94e0bfe6738 100644
--- a/llvm/lib/Analysis/StackSafetyAnalysis.cpp
+++ b/llvm/lib/Analysis/StackSafetyAnalysis.cpp
@@ -1048,9 +1048,7 @@ PreservedAnalyses StackSafetyPrinterPass::run(Function &F,
 
 char StackSafetyInfoWrapperPass::ID = 0;
 
-StackSafetyInfoWrapperPass::StackSafetyInfoWrapperPass() : FunctionPass(ID) {
-  initializeStackSafetyInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+StackSafetyInfoWrapperPass::StackSafetyInfoWrapperPass() : FunctionPass(ID) {}
 
 void StackSafetyInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequiredTransitive<ScalarEvolutionWrapperPass>();
@@ -1091,10 +1089,7 @@ PreservedAnalyses StackSafetyGlobalPrinterPass::run(Module &M,
 char StackSafetyGlobalInfoWrapperPass::ID = 0;
 
 StackSafetyGlobalInfoWrapperPass::StackSafetyGlobalInfoWrapperPass()
-    : ModulePass(ID) {
-  initializeStackSafetyGlobalInfoWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : ModulePass(ID) {}
 
 StackSafetyGlobalInfoWrapperPass::~StackSafetyGlobalInfoWrapperPass() = default;
 
diff --git a/llvm/lib/Analysis/StaticDataProfileInfo.cpp b/llvm/lib/Analysis/StaticDataProfileInfo.cpp
index a435aa00c2e9d..b89065147195f 100644
--- a/llvm/lib/Analysis/StaticDataProfileInfo.cpp
+++ b/llvm/lib/Analysis/StaticDataProfileInfo.cpp
@@ -64,9 +64,6 @@ INITIALIZE_PASS(StaticDataProfileInfoWrapperPass, "static-data-profile-info",
                 "Static Data Profile Info", false, true)
 
 StaticDataProfileInfoWrapperPass::StaticDataProfileInfoWrapperPass()
-    : ImmutablePass(ID) {
-  initializeStaticDataProfileInfoWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID) {}
 
 char StaticDataProfileInfoWrapperPass::ID = 0;
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index df4720c66f723..e2fd2aa13cce7 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -1471,20 +1471,14 @@ unsigned TargetLibraryInfoImpl::getSizeTSize(const Module &M) const {
 }
 
 TargetLibraryInfoWrapperPass::TargetLibraryInfoWrapperPass()
-    : ImmutablePass(ID), TLA(TargetLibraryInfoImpl()) {
-  initializeTargetLibraryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID), TLA(TargetLibraryInfoImpl()) {}
 
 TargetLibraryInfoWrapperPass::TargetLibraryInfoWrapperPass(const Triple &T)
-    : ImmutablePass(ID), TLA(TargetLibraryInfoImpl(T)) {
-  initializeTargetLibraryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID), TLA(TargetLibraryInfoImpl(T)) {}
 
 TargetLibraryInfoWrapperPass::TargetLibraryInfoWrapperPass(
     const TargetLibraryInfoImpl &TLIImpl)
-    : ImmutablePass(ID), TLA(TLIImpl) {
-  initializeTargetLibraryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID), TLA(TLIImpl) {}
 
 TargetLibraryInfoWrapperPass::TargetLibraryInfoWrapperPass(
     const TargetLibraryInfo &TLIOther)
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 4fea4e5711f5a..3f97484fb2fa3 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1499,17 +1499,11 @@ char TargetTransformInfoWrapperPass::ID = 0;
 void TargetTransformInfoWrapperPass::anchor() {}
 
 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
-    : ImmutablePass(ID) {
-  initializeTargetTransformInfoWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID) {}
 
 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
     TargetIRAnalysis TIRA)
-    : ImmutablePass(ID), TIRA(std::move(TIRA)) {
-  initializeTargetTransformInfoWrapperPassPass(
-      *PassRegistry::getPassRegistry());
-}
+    : ImmutablePass(ID), TIRA(std::move(TIRA)) {}
 
 TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
   FunctionAnalysisManager DummyFAM;
diff --git a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
index 3f44f746eb173..c871070eb037e 100644
--- a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
@@ -723,9 +723,7 @@ ImmutablePass *llvm::createTypeBasedAAWrapperPass() {
   return new TypeBasedAAWrapperPass();
 }
 
-TypeBasedAAWrapperPass::TypeBasedAAWrapperPass() : ImmutablePass(ID) {
-  initializeTypeBasedAAWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+TypeBasedAAWrapperPass::TypeBasedAAWrapperPass() : ImmutablePass(ID) {}
 
 bool TypeBasedAAWrapperPass::doInitialization(Module &M) {
   Result.reset(new TypeBasedAAResult(/*UsingTypeSanitizer=*/false));
diff --git a/llvm/lib/Analysis/UniformityAnalysis.cpp b/llvm/lib/Analysis/UniformityAnalysis.cpp
index 71e72ef678d35..2101fdfacfc8f 100644
--- a/llvm/lib/Analysis/UniformityAnalysis.cpp
+++ b/llvm/lib/Analysis/UniformityAnalysis.cpp
@@ -143,9 +143,7 @@ PreservedAnalyses UniformityInfoPrinterPass::run(Function &F,
 
 char UniformityInfoWrapperPass::ID = 0;
 
-UniformityInfoWrapperPass::UniformityInfoWrapperPass() : FunctionPass(ID) {
-  initializeUniformityInfoWrapperPassPass(*PassRegistry::getPassRegistry());
-}
+UniformityInfoWrapperPass::UniformityInfoWrapperPass() : FunctionPass(ID) {}
 
 INITIALIZE_PASS_BEGIN(UniformityInfoWrapperPass, "uniformity",
                       "Uniformity Analysis", true, true)
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 1bf6621cdf791..0095ce3d96277 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -584,17 +584,18 @@ TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM)
     : ImmutablePass(ID), PM(&PM), TM(&TM) {
   Impl = new PassConfigImpl();
 
+  PassRegistry &PR = *PassRegistry::getPassRegistry();
   // Register all target independent codegen passes to activate their PassIDs,
   // including this pass itself.
-  initializeCodeGen(*PassRegistry::getPassRegistry());
+  initializeCodeGen(PR);
 
   // Also register alias analysis passes required by codegen passes.
-  initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
-  initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
+  initializeBasicAAWrapperPassPass(PR);
+  initializeAAResultsWrapperPassPass(PR);
 
-  if (EnableIPRA.getNumOccurrences())
+  if (EnableIPRA.getNumOccurrences()) {
     TM.Options.EnableIPRA = EnableIPRA;
-  else {
+  } else {
     // If not explicitly specified, use target default.
     TM.Options.EnableIPRA |= TM.useIPRA();
   }

From b7c521b922f8b81544ecb0ccff2847644cac3107 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Mon, 21 Apr 2025 10:44:56 -0700
Subject: [PATCH 646/710] [OpenACC][CIR] Lowering for 'vector_length' on
 compute constructs

This is the same as the 'num_workers', with slightly different names in
places, so we just do the same exact implementation.  This extracts the
implementation as well, which should make it easier to reuse.
---
 clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp | 112 ++++++++++++--------
 clang/test/CIR/CodeGenOpenACC/kernels.c     |  52 +++++++++
 clang/test/CIR/CodeGenOpenACC/parallel.c    |  52 +++++++++
 3 files changed, 171 insertions(+), 45 deletions(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index b79baa96a3fc3..e7dd2e74b0864 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -82,6 +82,56 @@ class OpenACCClauseCIREmitter final
     return conversionOp.getResult(0);
   }
 
+  mlir::acc::DeviceType decodeDeviceType(const IdentifierInfo *ii) {
+    // '*' case leaves no identifier-info, just a nullptr.
+    if (!ii)
+      return mlir::acc::DeviceType::Star;
+    return llvm::StringSwitch<mlir::acc::DeviceType>(ii->getName())
+        .CaseLower("default", mlir::acc::DeviceType::Default)
+        .CaseLower("host", mlir::acc::DeviceType::Host)
+        .CaseLower("multicore", mlir::acc::DeviceType::Multicore)
+        .CasesLower("nvidia", "acc_device_nvidia",
+                    mlir::acc::DeviceType::Nvidia)
+        .CaseLower("radeon", mlir::acc::DeviceType::Radeon);
+  }
+
+  // Handle a clause affected by the 'device-type' to the point that they need
+  // to have the attributes added in the correct/corresponding order, such as
+  // 'num_workers' or 'vector_length' on a compute construct.
+  mlir::ArrayAttr
+  handleDeviceTypeAffectedClause(mlir::ArrayAttr existingDeviceTypes,
+                                 mlir::Value argument,
+                                 mlir::MutableOperandRange &argCollection) {
+    llvm::SmallVector<mlir::Attribute> deviceTypes;
+
+    // Collect the 'existing' device-type attributes so we can re-create them
+    // and insert them.
+    if (existingDeviceTypes) {
+      for (const mlir::Attribute &Attr : existingDeviceTypes)
+        deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+            builder.getContext(),
+            cast<mlir::acc::DeviceTypeAttr>(Attr).getValue()));
+    }
+
+    // Insert 1 version of the 'expr' to the NumWorkers list per-current
+    // device type.
+    if (lastDeviceTypeClause) {
+      for (const DeviceTypeArgument &arch :
+           lastDeviceTypeClause->getArchitectures()) {
+        deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+            builder.getContext(), decodeDeviceType(arch.getIdentifierInfo())));
+        argCollection.append(argument);
+      }
+    } else {
+      // Else, we just add a single for 'none'.
+      deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+          builder.getContext(), mlir::acc::DeviceType::None));
+      argCollection.append(argument);
+    }
+
+    return mlir::ArrayAttr::get(builder.getContext(), deviceTypes);
+  }
+
 public:
   OpenACCClauseCIREmitter(OpTy &operation, CIRGenFunction &cgf,
                           CIRGenBuilderTy &builder,
@@ -112,19 +162,6 @@ class OpenACCClauseCIREmitter final
     }
   }
 
-  mlir::acc::DeviceType decodeDeviceType(const IdentifierInfo *ii) {
-    // '*' case leaves no identifier-info, just a nullptr.
-    if (!ii)
-      return mlir::acc::DeviceType::Star;
-    return llvm::StringSwitch<mlir::acc::DeviceType>(ii->getName())
-        .CaseLower("default", mlir::acc::DeviceType::Default)
-        .CaseLower("host", mlir::acc::DeviceType::Host)
-        .CaseLower("multicore", mlir::acc::DeviceType::Multicore)
-        .CasesLower("nvidia", "acc_device_nvidia",
-                    mlir::acc::DeviceType::Nvidia)
-        .CaseLower("radeon", mlir::acc::DeviceType::Radeon);
-  }
-
   void VisitDeviceTypeClause(const OpenACCDeviceTypeClause &clause) {
     lastDeviceTypeClause = &clause;
     if constexpr (isOneOfTypes<OpTy, InitOp, ShutdownOp>) {
@@ -165,38 +202,10 @@ class OpenACCClauseCIREmitter final
 
   void VisitNumWorkersClause(const OpenACCNumWorkersClause &clause) {
     if constexpr (isOneOfTypes<OpTy, ParallelOp, KernelsOp>) {
-      // Collect the 'existing' device-type attributes so we can re-create them
-      // and insert them.
-      llvm::SmallVector<mlir::Attribute> deviceTypes;
-      mlir::ArrayAttr existingDeviceTypes =
-          operation.getNumWorkersDeviceTypeAttr();
-
-      if (existingDeviceTypes) {
-        for (mlir::Attribute Attr : existingDeviceTypes)
-          deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
-              builder.getContext(),
-              cast<mlir::acc::DeviceTypeAttr>(Attr).getValue()));
-      }
-
-      // Insert 1 version of the 'int-expr' to the NumWorkers list per-current
-      // device type.
-      mlir::Value intExpr = createIntExpr(clause.getIntExpr());
-      if (lastDeviceTypeClause) {
-        for (const DeviceTypeArgument &arg :
-             lastDeviceTypeClause->getArchitectures()) {
-          deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
-              builder.getContext(), decodeDeviceType(arg.getIdentifierInfo())));
-          operation.getNumWorkersMutable().append(intExpr);
-        }
-      } else {
-        // Else, we just add a single for 'none'.
-        deviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
-            builder.getContext(), mlir::acc::DeviceType::None));
-        operation.getNumWorkersMutable().append(intExpr);
-      }
-
-      operation.setNumWorkersDeviceTypeAttr(
-          mlir::ArrayAttr::get(builder.getContext(), deviceTypes));
+      mlir::MutableOperandRange range = operation.getNumWorkersMutable();
+      operation.setNumWorkersDeviceTypeAttr(handleDeviceTypeAffectedClause(
+          operation.getNumWorkersDeviceTypeAttr(),
+          createIntExpr(clause.getIntExpr()), range));
     } else if constexpr (isOneOfTypes<OpTy, SerialOp>) {
       llvm_unreachable("num_workers not valid on serial");
     } else {
@@ -204,6 +213,19 @@ class OpenACCClauseCIREmitter final
     }
   }
 
+  void VisitVectorLengthClause(const OpenACCVectorLengthClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, ParallelOp, KernelsOp>) {
+      mlir::MutableOperandRange range = operation.getVectorLengthMutable();
+      operation.setVectorLengthDeviceTypeAttr(handleDeviceTypeAffectedClause(
+          operation.getVectorLengthDeviceTypeAttr(),
+          createIntExpr(clause.getIntExpr()), range));
+    } else if constexpr (isOneOfTypes<OpTy, SerialOp>) {
+      llvm_unreachable("vector_length not valid on serial");
+    } else {
+      return clauseNotImplemented(clause);
+    }
+  }
+
   void VisitSelfClause(const OpenACCSelfClause &clause) {
     if constexpr (isOneOfTypes<OpTy, ParallelOp, SerialOp, KernelsOp>) {
       if (clause.isEmptySelfClause()) {
diff --git a/clang/test/CIR/CodeGenOpenACC/kernels.c b/clang/test/CIR/CodeGenOpenACC/kernels.c
index 6459b310546cd..d2da1d18f1534 100644
--- a/clang/test/CIR/CodeGenOpenACC/kernels.c
+++ b/clang/test/CIR/CodeGenOpenACC/kernels.c
@@ -158,5 +158,57 @@ void acc_kernels(int cond) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
 
+#pragma acc kernels vector_length(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: acc.kernels vector_length(%[[CONV_CAST]] : si32) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels vector_length(cond) device_type(nvidia) vector_length(2u)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !u32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !u32i to ui32
+  // CHECK-NEXT: acc.kernels vector_length(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : ui32 [#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels vector_length(cond) device_type(nvidia, host) vector_length(2) device_type(radeon) vector_length(3)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.kernels vector_length(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[TWO_CAST]] : si32 [#acc.device_type<host>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels vector_length(cond) device_type(nvidia) vector_length(2) device_type(radeon, multicore) vector_length(3)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.kernels vector_length(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>], %[[THREE_CAST]] : si32 [#acc.device_type<multicore>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels device_type(nvidia) vector_length(2) device_type(radeon) vector_length(3)
+  {}
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.kernels vector_length(%[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }
diff --git a/clang/test/CIR/CodeGenOpenACC/parallel.c b/clang/test/CIR/CodeGenOpenACC/parallel.c
index bdb506ee7e1d2..61dccc591c252 100644
--- a/clang/test/CIR/CodeGenOpenACC/parallel.c
+++ b/clang/test/CIR/CodeGenOpenACC/parallel.c
@@ -157,5 +157,57 @@ void acc_parallel(int cond) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
 
+#pragma acc parallel vector_length(cond)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: acc.parallel vector_length(%[[CONV_CAST]] : si32) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel vector_length(cond) device_type(nvidia) vector_length(2u)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !u32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !u32i to ui32
+  // CHECK-NEXT: acc.parallel vector_length(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : ui32 [#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel vector_length(cond) device_type(nvidia, host) vector_length(2) device_type(radeon) vector_length(3)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.parallel vector_length(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[TWO_CAST]] : si32 [#acc.device_type<host>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel vector_length(cond) device_type(nvidia) vector_length(2) device_type(radeon, multicore) vector_length(4)
+  {}
+  // CHECK-NEXT: %[[COND_LOAD:.*]] = cir.load %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[CONV_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_LOAD]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[FOUR_LITERAL:.*]] = cir.const #cir.int<4> : !s32i
+  // CHECK-NEXT: %[[FOUR_CAST:.*]] = builtin.unrealized_conversion_cast %[[FOUR_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.parallel vector_length(%[[CONV_CAST]] : si32, %[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[FOUR_CAST]] : si32 [#acc.device_type<radeon>], %[[FOUR_CAST]] : si32 [#acc.device_type<multicore>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel device_type(nvidia) vector_length(2) device_type(radeon) vector_length(3)
+  {}
+  // CHECK-NEXT: %[[TWO_LITERAL:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE_LITERAL:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE_LITERAL]] : !s32i to si32
+  // CHECK-NEXT: acc.parallel vector_length(%[[TWO_CAST]] : si32 [#acc.device_type<nvidia>], %[[THREE_CAST]] : si32 [#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
   // CHECK-NEXT: cir.return
 }

From 1b4919d0824411ab1a4fb055b22532fd466541f7 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 21 Apr 2025 15:59:45 -0400
Subject: [PATCH 647/710] [gn] port e1bb7f6ddec375

---
 .../secondary/llvm/lib/Target/AArch64/Disassembler/BUILD.gn  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/Disassembler/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/Disassembler/BUILD.gn
index eda8422db3afa..43fccb1dd58e8 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/Disassembler/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/Disassembler/BUILD.gn
@@ -2,7 +2,10 @@ import("//llvm/utils/TableGen/tablegen.gni")
 
 tablegen("AArch64GenDisassemblerTables") {
   visibility = [ ":Disassembler" ]
-  args = [ "-gen-disassembler" ]
+  args = [
+    "-gen-disassembler",
+    "--large-decoder-table",
+  ]
   td_file = "../AArch64.td"
 }
 

From 8c83355d5baa6ecbb5884240df195826380ed952 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 21 Apr 2025 21:03:39 +0100
Subject: [PATCH 648/710] [VPlan] Handle VPIRPhi in VPRecipeBase::isPhi (NFC).

Also handle VPIRPhi in VPRecipeBase::isPhi, to simplify existing code
dealing with VPIRPhis.

Suggested as part of https://github.com/llvm/llvm-project/pull/136455.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 19 +++++++------------
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  3 ++-
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 13 ++++---------
 3 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 36e14bb27a029..32c3435ccb38d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9324,10 +9324,8 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
   VPBuilder ScalarPHBuilder(ScalarPH);
   VPValue *OneVPV = Plan.getOrAddLiveIn(
       ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
-  for (VPRecipeBase &ScalarPhiR : *Plan.getScalarHeader()) {
-    auto *ScalarPhiIRI = dyn_cast<VPIRPhi>(&ScalarPhiR);
-    if (!ScalarPhiIRI)
-      break;
+  for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) {
+    auto *ScalarPhiIRI = cast<VPIRPhi>(&ScalarPhiR);
 
     // TODO: Extract final value from induction recipe initially, optimize to
     // pre-computed end value together in optimizeInductionExitUsers.
@@ -9381,10 +9379,8 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
     if (ExitVPBB->getNumPredecessors() == 0)
       continue;
 
-    for (VPRecipeBase &R : *ExitVPBB) {
-      auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
-      if (!ExitIRI)
-        break;
+    for (VPRecipeBase &R : ExitVPBB->phis()) {
+      auto *ExitIRI = cast<VPIRPhi>(&R);
       if (ExitVPBB->getSinglePredecessor() != Plan.getMiddleBlock()) {
         assert(ExitIRI->getNumOperands() ==
                    ExitVPBB->getPredecessors().size() &&
@@ -10579,10 +10575,9 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
     EpiWidenedPhis.insert(
         cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
   }
-  for (VPRecipeBase &R : make_early_inc_range(*MainPlan.getScalarHeader())) {
-    auto *VPIRInst = dyn_cast<VPIRPhi>(&R);
-    if (!VPIRInst)
-      break;
+  for (VPRecipeBase &R :
+       make_early_inc_range(MainPlan.getScalarHeader()->phis())) {
+    auto *VPIRInst = cast<VPIRPhi>(&R);
     if (EpiWidenedPhis.contains(&VPIRInst->getIRPhi()))
       continue;
     // There is no corresponding wide induction in the epilogue plan that would
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index dac31518997d8..ce64ba7cd2714 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -274,7 +274,8 @@ InstructionCost VPRecipeBase::computeCost(ElementCount VF,
 bool VPRecipeBase::isPhi() const {
   return (getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC) ||
          (isa<VPInstruction>(this) &&
-          cast<VPInstruction>(this)->getOpcode() == Instruction::PHI);
+          cast<VPInstruction>(this)->getOpcode() == Instruction::PHI) ||
+         isa<VPIRPhi>(this);
 }
 
 bool VPRecipeBase::isScalarCast() const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1d3e88037ea43..f2dc68b2ea8b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -874,10 +874,8 @@ void VPlanTransforms::optimizeInductionExitUsers(
   VPBlockBase *MiddleVPBB = Plan.getMiddleBlock();
   VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
   for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
-    for (VPRecipeBase &R : *ExitVPBB) {
-      auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
-      if (!ExitIRI)
-        break;
+    for (VPRecipeBase &R : ExitVPBB->phis()) {
+      auto *ExitIRI = cast<VPIRPhi>(&R);
 
       for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
         VPValue *Escape = nullptr;
@@ -2505,11 +2503,8 @@ void VPlanTransforms::handleUncountableEarlyExit(
   // Update the exit phis in the early exit block.
   VPBuilder MiddleBuilder(NewMiddle);
   VPBuilder EarlyExitB(VectorEarlyExitVPBB);
-  for (VPRecipeBase &R : *VPEarlyExitBlock) {
-    auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
-    if (!ExitIRI)
-      break;
-
+  for (VPRecipeBase &R : VPEarlyExitBlock->phis()) {
+    auto *ExitIRI = cast<VPIRPhi>(&R);
     PHINode &ExitPhi = ExitIRI->getIRPhi();
     VPValue *IncomingFromEarlyExit = RecipeBuilder.getVPValueOrAddLiveIn(
         ExitPhi.getIncomingValueForBlock(UncountableExitingBlock));

From c1049e42a1beab9b240cec80418b3ce32c9da03e Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Mon, 21 Apr 2025 13:16:05 -0700
Subject: [PATCH 649/710] [AMDGPU] Correct VOP3P encoding. NFC. (#136005)

VOP3P encoding field was defined as 9 bit in GFX8/GFX9 and opcode
was 7 bit, while it is 8 and 8 since GFX10. There are no changes
to instructions, but it fixes difference with SP3 in terms of
fields breakdown.
---
 llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 40 ++++++++++-----------
 llvm/lib/Target/AMDGPU/VOPInstructions.td   | 30 +++++++++-------
 2 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index d8088b8c638fd..a3be49bf02648 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1686,7 +1686,7 @@ let WaveSizePredicate = isWave64, SubtargetPredicate = isGFX12Plus in {
 // Begin Real Encodings
 //===----------------------------------------------------------------------===//
 
-class VOP3P_DPP16<bits<7> op, VOP_DPP_Pseudo ps, int subtarget,
+class VOP3P_DPP16<bits<8> op, VOP_DPP_Pseudo ps, int subtarget,
                   string opName = ps.OpName>
     : VOP3P_DPP<op, opName, ps.Pfl, 1>, SIMCInstr<ps.PseudoInstr, subtarget> {
   let hasSideEffects = ps.hasSideEffects;
@@ -1699,7 +1699,7 @@ class VOP3P_DPP16<bits<7> op, VOP_DPP_Pseudo ps, int subtarget,
   let IsPacked = ps.IsPacked;
 }
 
-class VOP3P_DPP8_Base<bits<7> op, VOP_Pseudo ps, string opName = ps.OpName>
+class VOP3P_DPP8_Base<bits<8> op, VOP_Pseudo ps, string opName = ps.OpName>
     : VOP3P_DPP8<op, opName, ps.Pfl> {
   let hasSideEffects = ps.hasSideEffects;
   let Defs = ps.Defs;
@@ -1714,14 +1714,14 @@ class VOP3P_DPP8_Base<bits<7> op, VOP_Pseudo ps, string opName = ps.OpName>
 // GFX11, GFX12
 //===----------------------------------------------------------------------===//
 
-multiclass VOP3P_Real_Base<GFXGen Gen, bits<7> op, string backing_ps_name = NAME,
+multiclass VOP3P_Real_Base<GFXGen Gen, bits<8> op, string backing_ps_name = NAME,
                       string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> {
   def Gen.Suffix :
     VOP3P_Real_Gen<!cast<VOP3P_Pseudo>(backing_ps_name), Gen, asmName>,
     VOP3Pe_gfx11_gfx12<op, !cast<VOP3P_Pseudo>(backing_ps_name).Pfl>;
 }
 
-class VOP3PeWmma<bits<7> op, VOPProfile P, VOP3PWMMA_Profile WMMAP>
+class VOP3PeWmma<bits<8> op, VOPProfile P, VOP3PWMMA_Profile WMMAP>
     : VOP3Pe_gfx11_gfx12<op, P>{
   // opsel
   let Inst{11} = !cond(!eq(WMMAP.IndexType, 0)  : 0,
@@ -1745,7 +1745,7 @@ class VOP3PeWmma<bits<7> op, VOPProfile P, VOP3PWMMA_Profile WMMAP>
   let Inst{15} = !if(WMMAP.IsIU, clamp{0}, 0);
 }
 
-multiclass VOP3P_WMMA_Real_Base<GFXGen Gen, bits<7> op, VOP3PWMMA_Profile WMMAP,
+multiclass VOP3P_WMMA_Real_Base<GFXGen Gen, bits<8> op, VOP3PWMMA_Profile WMMAP,
                                 string backing_ps_name = NAME,
                                 string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> {
   def Gen.Suffix :
@@ -1753,13 +1753,13 @@ multiclass VOP3P_WMMA_Real_Base<GFXGen Gen, bits<7> op, VOP3PWMMA_Profile WMMAP,
     VOP3PeWmma<op, !cast<VOP3P_Pseudo>(backing_ps_name).Pfl, WMMAP>;
 }
 
-multiclass VOP3P_Real_WMMA_gfx12 <bits<7> op, VOP3PWMMA_Profile WMMAP> {
+multiclass VOP3P_Real_WMMA_gfx12 <bits<8> op, VOP3PWMMA_Profile WMMAP> {
   let WaveSizePredicate = isWave32, DecoderNamespace = "GFX12" in {
     defm _twoaddr : VOP3P_WMMA_Real_Base <GFX12Gen, op, WMMAP>;
   }
 }
 
-multiclass VOP3P_Real_WMMA_gfx12w64 <bits<7> op, VOP3PWMMA_Profile WMMAP> {
+multiclass VOP3P_Real_WMMA_gfx12w64 <bits<8> op, VOP3PWMMA_Profile WMMAP> {
   let WaveSizePredicate = isWave64, DecoderNamespace = "GFX12W64" in {
     defm _twoaddr : VOP3P_WMMA_Real_Base <GFX12Gen, op, WMMAP>;
   }
@@ -1814,7 +1814,7 @@ defm V_SWMMAC_F32_16X16X32_FP8_BF8_w64 : VOP3P_Real_WMMA_gfx12w64 <0x058, F32_FP
 defm V_SWMMAC_F32_16X16X32_BF8_FP8_w64 : VOP3P_Real_WMMA_gfx12w64 <0x059, F32_FP8BF8_SWMMAC_w64>;
 defm V_SWMMAC_F32_16X16X32_BF8_BF8_w64 : VOP3P_Real_WMMA_gfx12w64 <0x05a, F32_FP8BF8_SWMMAC_w64>;
 
-multiclass VOP3P_Real_with_name<GFXGen Gen, bits<7> op,
+multiclass VOP3P_Real_with_name<GFXGen Gen, bits<8> op,
                           string backing_ps_name = NAME,
                           string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> {
   defvar ps = !cast<VOP3P_Pseudo>(backing_ps_name);
@@ -1828,7 +1828,7 @@ multiclass VOP3P_Real_with_name<GFXGen Gen, bits<7> op,
   }
 }
 
-multiclass VOP3P_Real_dpp<GFXGen Gen, bits<7> op, string backing_ps_name = NAME,
+multiclass VOP3P_Real_dpp<GFXGen Gen, bits<8> op, string backing_ps_name = NAME,
                           string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> {
   defvar ps = !cast<VOP3P_Pseudo>(backing_ps_name);
   def _dpp#Gen.Suffix
@@ -1840,7 +1840,7 @@ multiclass VOP3P_Real_dpp<GFXGen Gen, bits<7> op, string backing_ps_name = NAME,
   }
 }
 
-multiclass VOP3P_Real_dpp8<GFXGen Gen, bits<7> op, string backing_ps_name = NAME,
+multiclass VOP3P_Real_dpp8<GFXGen Gen, bits<8> op, string backing_ps_name = NAME,
                            string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> {
   defvar ps = !cast<VOP3P_Pseudo>(backing_ps_name);
   def _dpp8#Gen.Suffix : VOP3P_DPP8_Base<op, ps> {
@@ -1850,7 +1850,7 @@ multiclass VOP3P_Real_dpp8<GFXGen Gen, bits<7> op, string backing_ps_name = NAME
   }
 }
 
-multiclass VOP3P_Realtriple<GFXGen Gen, bits<7> op, string backing_ps_name = NAME,
+multiclass VOP3P_Realtriple<GFXGen Gen, bits<8> op, string backing_ps_name = NAME,
                             string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic>
     : VOP3P_Real_Base<Gen, op, backing_ps_name, asmName>,
       VOP3P_Real_dpp<Gen, op, backing_ps_name, asmName>,
@@ -1860,9 +1860,9 @@ multiclass VOP3P_Realtriple<GFXGen Gen, bits<7> op, string backing_ps_name = NAM
 // GFX12
 //===----------------------------------------------------------------------===//
 
-multiclass VOP3P_Real_gfx12<bits<7> op> : VOP3P_Real_Base<GFX12Gen, op>;
+multiclass VOP3P_Real_gfx12<bits<8> op> : VOP3P_Real_Base<GFX12Gen, op>;
 
-multiclass VOP3P_Real_with_name_gfx12<bits<7> op,
+multiclass VOP3P_Real_with_name_gfx12<bits<8> op,
                           string backing_ps_name = NAME,
                           string asmName = !cast<VOP3P_Pseudo>(NAME).Mnemonic> :
   VOP3P_Real_with_name<GFX12Gen, op, backing_ps_name, asmName>;
@@ -1882,7 +1882,7 @@ defm V_DOT4_F32_BF8_BF8 : VOP3P_Realtriple<GFX12Gen, 0x27>;
 // GFX11
 //===----------------------------------------------------------------------===//
 
-multiclass VOP3P_Real_gfx11_gfx12<bits<7> op> :
+multiclass VOP3P_Real_gfx11_gfx12<bits<8> op> :
    VOP3P_Real_Base<GFX11Gen, op>, VOP3P_Real_Base<GFX12Gen, op>;
 
 defm V_DOT4_I32_IU8  : VOP3P_Real_gfx11_gfx12<0x16>;
@@ -1894,7 +1894,7 @@ let AssemblerPredicate = isGFX11Plus in {
   def : AMDGPUMnemonicAlias<"v_dot8_i32_i4", "v_dot8_i32_iu4">;
 }
 
-multiclass VOP3P_Real_WMMA <bits<7> op> {
+multiclass VOP3P_Real_WMMA <bits<8> op> {
   let WaveSizePredicate = isWave32, DecoderNamespace = "GFX11" in {
     defm _twoaddr_w32 : VOP3P_Real_Base <GFX11Gen, op>;
   }
@@ -1916,7 +1916,7 @@ defm V_WMMA_I32_16X16X16_IU4   : VOP3P_Real_WMMA <0x045>;
 
 multiclass VOP3P_Real_vi<bits<7> op> {
   def _vi : VOP3P_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.VI>,
-            VOP3Pe <op, !cast<VOP3_Pseudo>(NAME).Pfl> {
+            VOP3Pe_vi <op, !cast<VOP3_Pseudo>(NAME).Pfl> {
     let AssemblerPredicate = HasVOP3PInsts;
     let DecoderNamespace = "GFX8";
     let VOP3P = 1;
@@ -2257,19 +2257,19 @@ defm V_PK_MOV_B32 : VOP3P_Real_vi <0x33>;
 //===----------------------------------------------------------------------===//
 
 let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10", VOP3P = 1 in {
-  multiclass VOP3P_Real_gfx10<bits<7> op> {
+  multiclass VOP3P_Real_gfx10<bits<8> op> {
     def _gfx10 : VOP3P_Real<!cast<VOP3P_Pseudo>(NAME), SIEncodingFamily.GFX10>,
                  VOP3Pe_gfx10 <op, !cast<VOP3P_Pseudo>(NAME).Pfl>;
   }
 } // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10", VOP3P = 1
 
-multiclass VOP3P_Real_gfx10_gfx11<bits<7> op> :
+multiclass VOP3P_Real_gfx10_gfx11<bits<8> op> :
   VOP3P_Real_gfx10<op>, VOP3P_Real_Base<GFX11Gen, op>;
 
-multiclass VOP3P_Real_gfx10_gfx11_gfx12<bits<7> op> :
+multiclass VOP3P_Real_gfx10_gfx11_gfx12<bits<8> op> :
   VOP3P_Real_gfx10_gfx11<op>, VOP3P_Real_Base<GFX12Gen, op>;
 
-multiclass VOP3P_Real_gfx10_gfx11_gfx12_Triple<bits<7> op> :
+multiclass VOP3P_Real_gfx10_gfx11_gfx12_Triple<bits<8> op> :
   VOP3P_Real_gfx10<op>, VOP3P_Realtriple<GFX11Gen, op>,
   VOP3P_Realtriple<GFX12Gen, op>;
 
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index d6ad01c8f9b35..f9fa83c3f5ae7 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -427,7 +427,7 @@ class VOP3be <VOPProfile P> : Enc64 {
   let Inst{63}    = !if(P.HasSrc2Mods, src2_modifiers{0}, 0);
 }
 
-class VOP3Pe <bits<7> op, VOPProfile P> : Enc64 {
+class VOP3Pe <VOPProfile P> : Enc64 {
   bits<8> vdst;
   bits<4> src0_modifiers;
   bits<9> src0;
@@ -452,8 +452,6 @@ class VOP3Pe <bits<7> op, VOPProfile P> : Enc64 {
 
   let Inst{15} = !if(P.HasClamp, clamp{0}, 0);
 
-  let Inst{22-16} = op;
-  let Inst{31-23} = 0x1a7; //encoding
   let Inst{40-32} = !if(P.HasSrc0, src0, 0);
   let Inst{49-41} = !if(P.HasSrc1, src1, 0);
   let Inst{58-50} = !if(P.HasSrc2, src2, 0);
@@ -577,11 +575,17 @@ class VOP3PXe <bits<7> op, VOPProfile MFMAPfl, bit acc_cd = 0> : Enc128, VOP3Pe_
   let Inst{127-125} = !if(MFMAPfl.HasSrc1, blgp, 0);
 }
 
-class VOP3Pe_gfx10 <bits<7> op, VOPProfile P> : VOP3Pe<op, P> {
-  let Inst{31-23} = 0x198; //encoding
+class VOP3Pe_vi <bits<7> op, VOPProfile P> : VOP3Pe<P> {
+  let Inst{22-16} = op;
+  let Inst{31-23} = 0x1a7; //encoding
 }
 
-class VOP3Pe_gfx11_gfx12<bits<7> op, VOPProfile P> : VOP3Pe_gfx10<op, P>;
+class VOP3Pe_gfx10 <bits<8> op, VOPProfile P> : VOP3Pe<P> {
+  let Inst{23-16} = op;
+  let Inst{31-24} = 0xcc; //encoding
+}
+
+class VOP3Pe_gfx11_gfx12<bits<8> op, VOPProfile P> : VOP3Pe_gfx10<op, P>;
 
 class VOP3be_gfx6_gfx7<bits<9> op, VOPProfile p> : VOP3be<p> {
   let Inst{25-17} = op;
@@ -902,7 +906,7 @@ class VOP3_DPPe_Common_t16<bits<10> op, VOPProfile P> : VOP3_DPPe_Common_Base<op
   let Inst{58-50} = !if(P.HasSrc2, src2{8-0}, 0);
 }
 
-class VOP3P_DPPe_Common_Base<bits<7> op, VOPProfile P> : Enc96 {
+class VOP3P_DPPe_Common_Base<bits<8> op, VOPProfile P> : Enc96 {
   bits<4> src0_modifiers;
   bits<4> src1_modifiers;
   bits<4> src2_modifiers;
@@ -917,8 +921,8 @@ class VOP3P_DPPe_Common_Base<bits<7> op, VOPProfile P> : Enc96 {
   let Inst{13} = !if(!and(P.HasSrc2, P.HasOpSel), src2_modifiers{2}, 0); // op_sel(2)
   let Inst{14} = !if(!and(P.HasSrc2, P.HasOpSel), src2_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(2)
   let Inst{15} = !if(P.HasClamp, clamp{0}, 0);
-  let Inst{22-16} = op;
-  let Inst{31-23} = 0x198; // encoding
+  let Inst{23-16} = op;
+  let Inst{31-24} = 0xcc; // encoding
   let Inst{59}    = !if(!and(P.HasSrc0, P.HasOpSel), src0_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(0)
   let Inst{60}    = !if(!and(P.HasSrc1, P.HasOpSel), src1_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(1)
   let Inst{61}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0); // neg (lo)
@@ -926,7 +930,7 @@ class VOP3P_DPPe_Common_Base<bits<7> op, VOPProfile P> : Enc96 {
   let Inst{63}    = !if(P.HasSrc2Mods, src2_modifiers{0}, 0); // neg (lo)
 }
 
-class VOP3P_DPPe_Common<bits<7> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P> {
+class VOP3P_DPPe_Common<bits<8> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P> {
   bits<8> vdst;
   bits<9> src1;
   bits<9> src2;
@@ -936,7 +940,7 @@ class VOP3P_DPPe_Common<bits<7> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P
   let Inst{58-50} = !if(P.HasSrc2, src2, 0);
 }
 
-class VOP3P_DPPe_Common_t16<bits<7> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P> {
+class VOP3P_DPPe_Common_t16<bits<8> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P> {
   bits<11> vdst;
   bits<11> src1;
   bits<11> src2;
@@ -1107,7 +1111,7 @@ class VOP3_DPP_t16<bits<10> op, string OpName, VOPProfile P, bit IsDPP16,
       VOP3_DPP_Enc_t16<op, P, IsDPP16> {
 }
 
-class VOP3P_DPP <bits<7> op, string OpName, VOPProfile P, bit IsDPP16,
+class VOP3P_DPP <bits<8> op, string OpName, VOPProfile P, bit IsDPP16,
                dag InsDPP = !if(IsDPP16, P.InsVOP3DPP16, P.InsVOP3DPP),
                string AsmDPP = !if(IsDPP16, P.AsmVOP3DPP16, P.AsmVOP3DPP)> :
   VOP3_DPP_Base<OpName, P, IsDPP16, InsDPP, AsmDPP>, VOP3P_DPPe_Common<op, P>,
@@ -1201,7 +1205,7 @@ class VOP3_DPP8_Enc_t16 <bits<10> op, VOPProfile P> :
 class VOP3_DPP8_t16<bits<10> op, string OpName, VOPProfile P> :
   VOP3_DPP8_Base<OpName, P>, VOP3_DPP8_Enc_t16<op, P>;
 
-class VOP3P_DPP8<bits<7> op, string OpName, VOPProfile P> :
+class VOP3P_DPP8<bits<8> op, string OpName, VOPProfile P> :
   VOP3_DPP8_Base<OpName, P>, VOP3P_DPPe_Common<op, P>,
   VOP3_DPP8e_Fields {
 

From 56bf0e720288ae6ba6e8635c7ec12f1e7d6f783b Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 21 Apr 2025 15:21:47 -0500
Subject: [PATCH 650/710] [OpenMP] Remove dependency on LLVM include directory
 from DeviceRTL (#136359)

Summary:
Currently we depend on a single LLVM include directory. This is actually
only required to define one enum, which is highly unlikely to change.
THis patch makes the `Environment.h` include directory more hermetic so
we no long depend on other libraries. In exchange, we get a simpler
dependency list for the price of hard-coding `1` somewhere. I think it's
a valid trade considering that this flag is highly unlikely to change at
this point.

@ronlieb AMD version
https://gist.github.com/jhuber6/3313e6f957be14dc79fe85e5126d2cb3
---
 offload/DeviceRTL/CMakeLists.txt     |  6 ------
 offload/DeviceRTL/src/Kernel.cpp     | 14 ++++++++++----
 offload/DeviceRTL/src/Mapping.cpp    |  2 --
 offload/include/Shared/Environment.h | 12 ++----------
 4 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index 346b65c887bba..12f53a30761f3 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -75,10 +75,6 @@ if(${LIBOMPTARGET_GPU_LIBC_SUPPORT})
   list(APPEND clang_opt_flags -DOMPTARGET_HAS_LIBC)
 endif()
 
-# Prepend -I to each list element
-set (LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL "${LIBOMPTARGET_LLVM_INCLUDE_DIRS}")
-list(TRANSFORM LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL PREPEND "-I")
-
 # Set flags for LLVM Bitcode compilation.
 set(bc_flags -c -flto -std=c++17 -fvisibility=hidden
              ${clang_opt_flags} -nogpulib -nostdlibinc
@@ -88,7 +84,6 @@ set(bc_flags -c -flto -std=c++17 -fvisibility=hidden
              -I${include_directory}
              -I${devicertl_base_directory}/../include
              -I${devicertl_base_directory}/../../libc
-             ${LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL}
 )
 
 # first create an object target
@@ -172,7 +167,6 @@ function(compileDeviceRTLLibrary target_name target_triple)
       ${include_directory}
       ${devicertl_base_directory}/../../libc
       ${devicertl_base_directory}/../include
-      ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
     )
     install(TARGETS ${ide_target_name} EXCLUDE_FROM_ALL)
   endif()
diff --git a/offload/DeviceRTL/src/Kernel.cpp b/offload/DeviceRTL/src/Kernel.cpp
index 9bb89573dc0cb..eac68a15538c4 100644
--- a/offload/DeviceRTL/src/Kernel.cpp
+++ b/offload/DeviceRTL/src/Kernel.cpp
@@ -21,10 +21,17 @@
 #include "Synchronization.h"
 #include "Workshare.h"
 
-#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
-
 using namespace ompx;
 
+// These flags are copied from "llvm/Frontend/OpenMP/OMPDeviceConstants.h" and
+// must be kept in-sync.
+enum OMPTgtExecModeFlags : unsigned char {
+  OMP_TGT_EXEC_MODE_GENERIC = 1 << 0,
+  OMP_TGT_EXEC_MODE_SPMD = 1 << 1,
+  OMP_TGT_EXEC_MODE_GENERIC_SPMD =
+      OMP_TGT_EXEC_MODE_GENERIC | OMP_TGT_EXEC_MODE_SPMD
+};
+
 static void
 inititializeRuntime(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment,
                     KernelLaunchEnvironmentTy &KernelLaunchEnvironment) {
@@ -74,8 +81,7 @@ extern "C" {
 int32_t __kmpc_target_init(KernelEnvironmentTy &KernelEnvironment,
                            KernelLaunchEnvironmentTy &KernelLaunchEnvironment) {
   ConfigurationEnvironmentTy &Configuration = KernelEnvironment.Configuration;
-  bool IsSPMD = Configuration.ExecMode &
-                llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_SPMD;
+  bool IsSPMD = Configuration.ExecMode & OMP_TGT_EXEC_MODE_SPMD;
   bool UseGenericStateMachine = Configuration.UseGenericStateMachine;
   if (IsSPMD) {
     inititializeRuntime(/*IsSPMD=*/true, KernelEnvironment,
diff --git a/offload/DeviceRTL/src/Mapping.cpp b/offload/DeviceRTL/src/Mapping.cpp
index e951556c2ad4e..b145892d1ece0 100644
--- a/offload/DeviceRTL/src/Mapping.cpp
+++ b/offload/DeviceRTL/src/Mapping.cpp
@@ -16,8 +16,6 @@
 #include "State.h"
 #include "gpuintrin.h"
 
-#include "llvm/Frontend/OpenMP/OMPGridValues.h"
-
 using namespace ompx;
 
 // FIXME: This resolves the handling for the AMDGPU workgroup size when the ABI
diff --git a/offload/include/Shared/Environment.h b/offload/include/Shared/Environment.h
index db8443a7be933..2a283bd6fa4ed 100644
--- a/offload/include/Shared/Environment.h
+++ b/offload/include/Shared/Environment.h
@@ -15,15 +15,7 @@
 
 #include <stdint.h>
 
-#ifdef OMPTARGET_DEVICE_RUNTIME
-#include "DeviceTypes.h"
-#else
-#include "SourceInfo.h"
-
-using IdentTy = ident_t;
-#endif
-
-#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
+struct IdentTy;
 
 enum class DeviceDebugKind : uint32_t {
   Assertion = 1U << 0,
@@ -80,7 +72,7 @@ struct DynamicEnvironmentTy {
 struct ConfigurationEnvironmentTy {
   uint8_t UseGenericStateMachine = 2;
   uint8_t MayUseNestedParallelism = 2;
-  llvm::omp::OMPTgtExecModeFlags ExecMode = llvm::omp::OMP_TGT_EXEC_MODE_SPMD;
+  uint8_t ExecMode = 0;
   // Information about (legal) launch configurations.
   //{
   int32_t MinThreads = -1;

From 863ead29e92de800fe50e56958b6eba6ede023e9 Mon Sep 17 00:00:00 2001
From: Alexey Samsonov <vonosmas@gmail.com>
Date: Mon, 21 Apr 2025 13:43:31 -0700
Subject: [PATCH 651/710] [libc][bazel] Add a helper library with all deps of
 generated headers. (#136582)

Create "public_header_deps" that is a convenient way to express
dependencies of a generated headers as a single (and same) target. It's
also convenient to use it in unit tests - which is also demonstrated in
this PR by adding the BUILD.bazel placeholder for test/include unit
tests, and creating a libc_test target for one of these tests.

See issue #134780.
---
 .../llvm-project-overlay/libc/BUILD.bazel     | 14 ++++++++++++-
 .../libc/test/include/BUILD.bazel             | 20 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 utils/bazel/llvm-project-overlay/libc/test/include/BUILD.bazel

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 4412f2beaffcd..fea2cf54ab1ed 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -90,6 +90,18 @@ py_binary(
     main = "utils/hdrgen/hdrgen/main.py",
 )
 
+# Library containing all headers that can be transitively included by generated llvm-libc
+# public headers (or by the unit tests).
+libc_support_library(
+    name = "public_headers_deps",
+    textual_hdrs = [
+        "include/__llvm-libc-common.h",
+    ] + glob([
+        "include/llvm-libc-types/**/*.h",
+        "include/llvm-libc-macros/**/*.h",
+    ]),
+)
+
 ################################## Base Config #################################
 
 libc_support_library(
@@ -1898,11 +1910,11 @@ libc_support_library(
     name = "atan_utils",
     hdrs = ["src/math/generic/atan_utils.h"],
     deps = [
-        ":__support_integer_literals",
         ":__support_fputil_double_double",
         ":__support_fputil_dyadic_float",
         ":__support_fputil_multiply_add",
         ":__support_fputil_polyeval",
+        ":__support_integer_literals",
         ":__support_macros_optimization",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/include/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/include/BUILD.bazel
new file mode 100644
index 0000000000000..361c7c8db6542
--- /dev/null
+++ b/utils/bazel/llvm-project-overlay/libc/test/include/BUILD.bazel
@@ -0,0 +1,20 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Tests for LLVM libc public headers.
+
+load("//libc/test:libc_test_rules.bzl", "libc_test")
+
+licenses(["notice"])
+
+libc_test(
+    name = "stdbit_test",
+    srcs = [
+        "stdbit_stub.h",
+        "stdbit_test.cpp",
+    ],
+    deps = [
+        "//libc:public_headers_deps",
+    ],
+)

From 704fc6542cf171c603bd387663a2de3a3fa9c4ff Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 21 Apr 2025 13:49:49 -0700
Subject: [PATCH 652/710] [SelectionDAG] Prefer to use ATOMIC_LOAD extension
 type over getExtendForAtomicOps() in computeKnownBits/ComputeNumSignBits.
 (#136600)

If an ATOMIC_LOAD has ZEXTLOAD/SEXTLOAD extension type we should trust
that over getExtendForAtomicOps().

SystemZ is the only target that uses setAtomicLoadExtAction and they
return ANY_EXTEND from getExtendForAtomicOps(). So I'm not sure there's
a way to get a contradiction currently.

Note, type legalization uses getExtendForAtomicOps() when promoting
ATOMIC_LOAD so we may not need to check getExtendForAtomicOps() for
ATOMIC_LOAD. I have not done much investigating of this.
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 40 ++++++++++++-------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 09d73633462b6..ba6c5d884d381 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4406,14 +4406,19 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
   case ISD::ATOMIC_LOAD_UMIN:
   case ISD::ATOMIC_LOAD_UMAX:
   case ISD::ATOMIC_LOAD: {
-    unsigned MemBits =
-        cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
     // If we are looking at the loaded value.
     if (Op.getResNo() == 0) {
-      if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
-        Known.Zero.setBitsFrom(MemBits);
-      else if (Op->getOpcode() == ISD::ATOMIC_LOAD &&
-               cast<AtomicSDNode>(Op)->getExtensionType() == ISD::ZEXTLOAD)
+      auto *AT = cast<AtomicSDNode>(Op);
+      unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
+
+      // For atomic_load, prefer to use the extension type.
+      if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
+        if (AT->getExtensionType() == ISD::ZEXTLOAD)
+          Known.Zero.setBitsFrom(MemBits);
+        else if (AT->getExtensionType() != ISD::SEXTLOAD &&
+                 TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
+          Known.Zero.setBitsFrom(MemBits);
+      } else if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
         Known.Zero.setBitsFrom(MemBits);
     }
     break;
@@ -5252,22 +5257,29 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
   case ISD::ATOMIC_LOAD_UMIN:
   case ISD::ATOMIC_LOAD_UMAX:
   case ISD::ATOMIC_LOAD: {
-    Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
+    auto *AT = cast<AtomicSDNode>(Op);
     // If we are looking at the loaded value.
     if (Op.getResNo() == 0) {
+      Tmp = AT->getMemoryVT().getScalarSizeInBits();
       if (Tmp == VTBits)
         return 1; // early-out
-      if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
-        return VTBits - Tmp + 1;
-      if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
-        return VTBits - Tmp;
+
+      // For atomic_load, prefer to use the extension type.
       if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
-        ISD::LoadExtType ETy = cast<AtomicSDNode>(Op)->getExtensionType();
-        if (ETy == ISD::SEXTLOAD)
+        switch (AT->getExtensionType()) {
+        default:
+          break;
+        case ISD::SEXTLOAD:
           return VTBits - Tmp + 1;
-        if (ETy == ISD::ZEXTLOAD)
+        case ISD::ZEXTLOAD:
           return VTBits - Tmp;
+        }
       }
+
+      if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
+        return VTBits - Tmp + 1;
+      if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
+        return VTBits - Tmp;
     }
     break;
   }

From aa5cdc0600ac8157a80f54fa2555ae7dfd5fc520 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Mon, 21 Apr 2025 14:11:10 -0700
Subject: [PATCH 653/710] [msan][NFCI] Add avx512fp16-{arith-intrinsics,
 arith-vl-intrinsics, intrinsics} tests (#136260)

Forked from llvm/test/CodeGen/X86/avx512fp16-{arith-intrinsics,
arith-vl-intrinsics, intrinsics}.ll

The tests show that many instructions are handled suboptimally or
heuristically, but this commit message is too narrow to contain them.
---
 .../X86/avx512fp16-arith-intrinsics.ll        | 1822 +++++++++
 .../X86/avx512fp16-arith-vl-intrinsics.ll     | 1898 ++++++++++
 .../X86/avx512fp16-intrinsics.ll              | 3248 +++++++++++++++++
 3 files changed, 6968 insertions(+)
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-vl-intrinsics.ll
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-intrinsics.ll

diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll
new file mode 100644
index 0000000000000..121a17894fe9e
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll
@@ -0,0 +1,1822 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt %s -S -mattr=+avx512f -passes=msan 2>&1 | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/avx512fp16-arith-intrinsics.ll
+;
+; Handled suboptimally (visitInstruction):
+; - llvm.x86.avx512fp16.add.ph.512
+; - llvm.x86.avx512fp16.div.ph.512
+; - llvm.x86.avx512fp16.mask.vcvtpd2ph.512
+; - llvm.x86.avx512fp16.mask.vcvtph2pd.512
+; - llvm.x86.avx512fp16.mask.vcvtph2psx.512
+; - llvm.x86.avx512fp16.mask.vcvtps2phx.512
+; - llvm.x86.avx512fp16.mask.vcvtsd2sh.round
+; - llvm.x86.avx512fp16.mask.vcvtsh2sd.round
+; - llvm.x86.avx512fp16.mask.vcvtsh2ss.round
+; - llvm.x86.avx512fp16.mask.vcvtss2sh.round
+; - llvm.x86.avx512fp16.max.ph.512
+; - llvm.x86.avx512fp16.min.ph.512
+; - llvm.x86.avx512fp16.mul.ph.512
+; - llvm.x86.avx512fp16.sub.ph.512
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half>, <32 x half>, i32)
+
+define <32 x half> @test_int_x86_avx512fp16_add_ph_512(<32 x half> %x1, <32 x half> %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_add_ph_512(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  ret <32 x half> %res
+}
+
+define <32 x half> @test_int_x86_avx512fp16_mask_add_ph_512(<32 x half> %src, <32 x half> %x1, <32 x half> %x2, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_mask_add_ph_512(
+; CHECK-SAME: <32 x half> [[SRC:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> [[SRC]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  %res = select <32 x i1> %mask, <32 x half> %res0, <32 x half> %src
+  ret <32 x half> %res
+}
+
+define <32 x half> @test_int_x86_avx512fp16_maskz_add_ph_512(<32 x half> %src, <32 x half> %x1, <32 x half> %x2, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_maskz_add_ph_512(
+; CHECK-SAME: <32 x half> [[SRC:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <32 x half>, ptr [[PTR]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP19]], <32 x i16> [[TMP15]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP21]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[T2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[VAL]], i32 4)
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x half> [[T2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP28]], <32 x i16> [[TMP24]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T2]], <32 x half> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT1]]
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <32 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES3]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %val = load <32 x half>, ptr %ptr
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  %res1 = select <32 x i1> %mask, <32 x half> %res0, <32 x half> zeroinitializer
+  %t2 = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> %x1, <32 x half> %val, i32 4)
+  %res2 = select <32 x i1> %mask, <32 x half> %t2, <32 x half> zeroinitializer
+  %res3  =  fadd <32 x half> %res1, %res2
+  ret <32 x half> %res3
+}
+
+define <32 x half> @test_int_x86_avx512fp16_add_ph_512_round(<32 x half> %x1, <32 x half> %x2, <32 x half> %src, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_add_ph_512_round(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], <32 x half> [[SRC:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[T1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[T1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T1]], <32 x half> [[SRC]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %t1 = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> %x1, <32 x half> %x2, i32 10)
+  %res = select <32 x i1> %mask, <32 x half> %t1, <32 x half> %src
+  ret <32 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half>, <32 x half>, i32)
+
+define <32 x half> @test_int_x86_avx512fp16_sub_ph_512(<32 x half> %x1, <32 x half> %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_sub_ph_512(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  ret <32 x half> %res
+}
+
+define <32 x half> @test_int_x86_avx512fp16_mask_sub_ph_512(<32 x half> %src, <32 x half> %x1, <32 x half> %x2, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_mask_sub_ph_512(
+; CHECK-SAME: <32 x half> [[SRC:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> [[SRC]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  %res = select <32 x i1> %mask, <32 x half> %res0, <32 x half> %src
+  ret <32 x half> %res
+}
+
+define <32 x half> @test_int_x86_avx512fp16_maskz_sub_ph_512(<32 x half> %src, <32 x half> %x1, <32 x half> %x2, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_maskz_sub_ph_512(
+; CHECK-SAME: <32 x half> [[SRC:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <32 x half>, ptr [[PTR]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP19]], <32 x i16> [[TMP15]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP21]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[T2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[VAL]], i32 4)
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x half> [[T2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP28]], <32 x i16> [[TMP24]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T2]], <32 x half> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT1]]
+; CHECK-NEXT:    [[RES3:%.*]] = fsub <32 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES3]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %val = load <32 x half>, ptr %ptr
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  %res1 = select <32 x i1> %mask, <32 x half> %res0, <32 x half> zeroinitializer
+  %t2 = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> %x1, <32 x half> %val, i32 4)
+  %res2 = select <32 x i1> %mask, <32 x half> %t2, <32 x half> zeroinitializer
+  %res3  =  fsub <32 x half> %res1, %res2
+  ret <32 x half> %res3
+}
+
+define <32 x half> @test_int_x86_avx512fp16_sub_ph_512_round(<32 x half> %x1, <32 x half> %x2, <32 x half> %src, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_sub_ph_512_round(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], <32 x half> [[SRC:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[T1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[T1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T1]], <32 x half> [[SRC]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %t1 = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> %x1, <32 x half> %x2, i32 10)
+  %res = select <32 x i1> %mask, <32 x half> %t1, <32 x half> %src
+  ret <32 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half>, <32 x half>, i32)
+
+define <32 x half> @test_int_x86_avx512fp16_mul_ph_512(<32 x half> %x1, <32 x half> %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_mul_ph_512(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  ret <32 x half> %res
+}
+
+define <32 x half> @test_int_x86_avx512fp16_mask_mul_ph_512(<32 x half> %src, <32 x half> %x1, <32 x half> %x2, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_mask_mul_ph_512(
+; CHECK-SAME: <32 x half> [[SRC:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> [[SRC]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  %res = select <32 x i1> %mask, <32 x half> %res0, <32 x half> %src
+  ret <32 x half> %res
+}
+
+define <32 x half> @test_int_x86_avx512fp16_maskz_mul_ph_512(<32 x half> %src, <32 x half> %x1, <32 x half> %x2, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_maskz_mul_ph_512(
+; CHECK-SAME: <32 x half> [[SRC:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <32 x half>, ptr [[PTR]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP19]], <32 x i16> [[TMP15]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP21]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[T2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[VAL]], i32 4)
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x half> [[T2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP28]], <32 x i16> [[TMP24]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T2]], <32 x half> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT1]]
+; CHECK-NEXT:    [[RES3:%.*]] = fmul <32 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES3]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %val = load <32 x half>, ptr %ptr
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  %res1 = select <32 x i1> %mask, <32 x half> %res0, <32 x half> zeroinitializer
+  %t2 = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> %x1, <32 x half> %val, i32 4)
+  %res2 = select <32 x i1> %mask, <32 x half> %t2, <32 x half> zeroinitializer
+  %res3  =  fmul <32 x half> %res1, %res2
+  ret <32 x half> %res3
+}
+
+define <32 x half> @test_int_x86_avx512fp16_mul_ph_512_round(<32 x half> %x1, <32 x half> %x2, <32 x half> %src, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_mul_ph_512_round(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], <32 x half> [[SRC:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[T1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[T1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T1]], <32 x half> [[SRC]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %t1 = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> %x1, <32 x half> %x2, i32 10)
+  %res = select <32 x i1> %mask, <32 x half> %t1, <32 x half> %src
+  ret <32 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half>, <32 x half>, i32)
+
+define <32 x half> @test_int_x86_avx512fp16_div_ph_512(<32 x half> %x1, <32 x half> %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_div_ph_512(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  ret <32 x half> %res
+}
+
+define <32 x half> @test_int_x86_avx512fp16_mask_div_ph_512(<32 x half> %src, <32 x half> %x1, <32 x half> %x2, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_mask_div_ph_512(
+; CHECK-SAME: <32 x half> [[SRC:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> [[SRC]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  %res = select <32 x i1> %mask, <32 x half> %res0, <32 x half> %src
+  ret <32 x half> %res
+}
+
+define <32 x half> @test_int_x86_avx512fp16_maskz_div_ph_512(<32 x half> %src, <32 x half> %x1, <32 x half> %x2, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_maskz_div_ph_512(
+; CHECK-SAME: <32 x half> [[SRC:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <32 x half>, ptr [[PTR]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP19]], <32 x i16> [[TMP15]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP21]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[T2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[VAL]], i32 4)
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x half> [[T2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP28]], <32 x i16> [[TMP24]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T2]], <32 x half> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT1]]
+; CHECK-NEXT:    [[RES3:%.*]] = fdiv <32 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES3]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %val = load <32 x half>, ptr %ptr
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
+  %res1 = select <32 x i1> %mask, <32 x half> %res0, <32 x half> zeroinitializer
+  %t2 = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> %x1, <32 x half> %val, i32 4)
+  %res2 = select <32 x i1> %mask, <32 x half> %t2, <32 x half> zeroinitializer
+  %res3  =  fdiv <32 x half> %res1, %res2
+  ret <32 x half> %res3
+}
+
+define <32 x half> @test_int_x86_avx512fp16_div_ph_512_round(<32 x half> %x1, <32 x half> %x2, <32 x half> %src, i32 %msk, ptr %ptr) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_div_ph_512_round(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], <32 x half> [[SRC:%.*]], i32 [[MSK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[T1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[T1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T1]], <32 x half> [[SRC]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %t1 = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> %x1, <32 x half> %x2, i32 10)
+  %res = select <32 x i1> %mask, <32 x half> %t1, <32 x half> %src
+  ret <32 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.min.ph.512(<32 x half>, <32 x half>, i32)
+
+define <32 x half> @test_min_ph(<32 x half> %x1, <32 x half> %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_min_ph(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <32 x i16> [[_MSPROP]] to <32 x i1>
+; CHECK-NEXT:    [[RES0:%.*]] = fcmp olt <32 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <32 x i1> [[RES0]], <32 x i16> [[TMP1]], <32 x i16> [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x half> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x half> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP3]], <32 x i16> [[TMP9]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[RES0]], <32 x half> [[X1]], <32 x half> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES1]]
+;
+  %res0 = fcmp olt <32 x half> %x1, %x2
+  %res1 = select <32 x i1> %res0, <32 x half> %x1, <32 x half> %x2
+  ret  <32 x half> %res1
+}
+
+define <32 x half> @test_int_x86_avx512fp16_min_ph_512_sae(<32 x half> %x1, <32 x half> %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_min_ph_512_sae(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.min.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 8)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES0]]
+;
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.min.ph.512(<32 x half> %x1, <32 x half> %x2, i32 8)
+  ret <32 x half> %res0
+}
+
+define <32 x half> @test_int_x86_avx512fp16_maskz_min_ph_512_sae(<32 x half> %x1, <32 x half> %x2, i32 %msk) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_maskz_min_ph_512_sae(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.min.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <32 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i16> [[TMP13]], <32 x i16> [[TMP9]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES1]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.min.ph.512(<32 x half> %x1, <32 x half> %x2, i32 8)
+  %res1 = select <32 x i1> %mask, <32 x half> %res0, <32 x half> zeroinitializer
+  ret <32 x half> %res1
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.max.ph.512(<32 x half>, <32 x half>, i32)
+
+define <32 x half> @test_max_ph(<32 x half> %x1, <32 x half> %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_max_ph(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <32 x i16> [[_MSPROP]] to <32 x i1>
+; CHECK-NEXT:    [[RES0:%.*]] = fcmp ogt <32 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <32 x i1> [[RES0]], <32 x i16> [[TMP1]], <32 x i16> [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x half> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x half> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP3]], <32 x i16> [[TMP9]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[RES0]], <32 x half> [[X1]], <32 x half> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES1]]
+;
+  %res0 = fcmp ogt <32 x half> %x1, %x2
+  %res1 = select <32 x i1> %res0, <32 x half> %x1, <32 x half> %x2
+  ret  <32 x half> %res1
+}
+
+define <32 x half> @test_int_x86_avx512fp16_max_ph_512_sae(<32 x half> %x1, <32 x half> %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_max_ph_512_sae(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.max.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 8)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES0]]
+;
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.max.ph.512(<32 x half> %x1, <32 x half> %x2, i32 8)
+  ret <32 x half> %res0
+}
+
+define <32 x half> @test_int_x86_avx512fp16_maskz_max_ph_512_sae(<32 x half> %x1, <32 x half> %x2, i32 %msk) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512fp16_maskz_max_ph_512_sae(
+; CHECK-SAME: <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[MSK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.max.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <32 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i16> [[TMP13]], <32 x i16> [[TMP9]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES1]]
+;
+  %mask = bitcast i32 %msk to <32 x i1>
+  %res0 = call <32 x half> @llvm.x86.avx512fp16.max.ph.512(<32 x half> %x1, <32 x half> %x2, i32 8)
+  %res1 = select <32 x i1> %mask, <32 x half> %res0, <32 x half> zeroinitializer
+  ret <32 x half> %res1
+}
+
+declare <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half>, <8 x double>, i8, i32)
+
+define <8 x double> @test_int_x86_avx512_mask_vcvt_ph2pd(<8 x half> %x0, <8 x double> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x double> @test_int_x86_avx512_mask_vcvt_ph2pd(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x double> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half> [[X0]], <8 x double> [[X1]], i8 [[X2]], i32 4)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %res = call <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half> %x0, <8 x double> %x1, i8 %x2, i32 4)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_sae(<8 x half> %x0, <8 x double> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_sae(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x double> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half> [[X0]], <8 x double> [[X1]], i8 [[X2]], i32 8)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %res = call <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half> %x0, <8 x double> %x1, i8 %x2, i32 8)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_nomask(<8 x half> %x0, <8 x double> %x1) #0 {
+; CHECK-LABEL: define <8 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x double> [[X1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half> [[X0]], <8 x double> [[X1]], i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %res = call <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half> %x0, <8 x double> %x1, i8 -1, i32 4)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_load(ptr %px0, <8 x double> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_load(
+; CHECK-SAME: ptr [[PX0:%.*]], <8 x double> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[X0:%.*]] = load <8 x half>, ptr [[PX0]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PX0]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB11:.*]], label %[[BB12:.*]], !prof [[PROF1]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half> [[X0]], <8 x double> [[X1]], i8 [[X2]], i32 4)
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES]]
+;
+  %x0 = load <8 x half>, ptr %px0, align 16
+  %res = call <8 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.512(<8 x half> %x0, <8 x double> %x1, i8 %x2, i32 4)
+  ret <8 x double> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.512(<8 x double>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph(<8 x double> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph(
+; CHECK-SAME: <8 x double> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.512(<8 x double> [[X0]], <8 x half> [[X1]], i8 [[X2]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.512(<8 x double> %x0, <8 x half> %x1, i8 %x2, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_r(<8 x double> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_r(
+; CHECK-SAME: <8 x double> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.512(<8 x double> [[X0]], <8 x half> [[X1]], i8 [[X2]], i32 11)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.512(<8 x double> %x0, <8 x half> %x1, i8 %x2, i32 11)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_load(ptr %px0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_load(
+; CHECK-SAME: ptr [[PX0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[X0:%.*]] = load <8 x double>, ptr [[PX0]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PX0]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB11:.*]], label %[[BB12:.*]], !prof [[PROF1]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.512(<8 x double> [[X0]], <8 x half> [[X1]], i8 [[X2]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %x0 = load <8 x double>, ptr %px0, align 64
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.512(<8 x double> %x0, <8 x half> %x1, i8 %x2, i32 4)
+  ret <8 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half>, <4 x float>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512fp16_mask_cvt_ss2sh_round(<8 x half> %x0, <4 x float> %x1, <8 x half> %x2, i8 %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_cvt_ss2sh_round(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x float> [[X1:%.*]], <8 x half> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half> [[X0]], <4 x float> [[X1]], <8 x half> [[X2]], i8 [[X3]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half> %x0, <4 x float> %x1, <8 x half> %x2, i8 %x3, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_cvt_ss2sh_round_r(<8 x half> %x0, <4 x float> %x1, <8 x half> %x2, i8 %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_cvt_ss2sh_round_r(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x float> [[X1:%.*]], <8 x half> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half> [[X0]], <4 x float> [[X1]], <8 x half> [[X2]], i8 [[X3]], i32 11)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half> %x0, <4 x float> %x1, <8 x half> %x2, i8 %x3, i32 11)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_cvt_ss2sh_round_nomask(<8 x half> %x0, <4 x float> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_cvt_ss2sh_round_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x float> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half> [[X0]], <4 x float> [[X1]], <8 x half> [[X2]], i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half> %x0, <4 x float> %x1, <8 x half> %x2, i8 -1, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_cvt_ss2sh_round_z(<8 x half> %x0, <4 x float> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_cvt_ss2sh_round_z(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x float> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half> [[X0]], <4 x float> [[X1]], <8 x half> zeroinitializer, i8 [[X2]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtss2sh.round(<8 x half> %x0, <4 x float> %x1, <8 x half> zeroinitializer, i8 %x2, i32 4)
+  ret <8 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half>, <2 x double>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512fp16_mask_cvt_sd2sh_round(<8 x half> %x0, <2 x double> %x1, <8 x half> %x2, i8 %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_cvt_sd2sh_round(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <2 x double> [[X1:%.*]], <8 x half> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half> [[X0]], <2 x double> [[X1]], <8 x half> [[X2]], i8 [[X3]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half> %x0, <2 x double> %x1, <8 x half> %x2, i8 %x3, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_cvt_sd2sh_round_r(<8 x half> %x0, <2 x double> %x1, <8 x half> %x2, i8 %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_cvt_sd2sh_round_r(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <2 x double> [[X1:%.*]], <8 x half> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half> [[X0]], <2 x double> [[X1]], <8 x half> [[X2]], i8 [[X3]], i32 11)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half> %x0, <2 x double> %x1, <8 x half> %x2, i8 %x3, i32 11)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_cvt_sd2sh_round_nomask(<8 x half> %x0, <2 x double> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_cvt_sd2sh_round_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <2 x double> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half> [[X0]], <2 x double> [[X1]], <8 x half> [[X2]], i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half> %x0, <2 x double> %x1, <8 x half> %x2, i8 -1, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_cvt_sd2sh_round_z(<8 x half> %x0, <2 x double> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_cvt_sd2sh_round_z(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <2 x double> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half> [[X0]], <2 x double> [[X1]], <8 x half> zeroinitializer, i8 [[X2]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtsd2sh.round(<8 x half> %x0, <2 x double> %x1, <8 x half> zeroinitializer, i8 %x2, i32 4)
+  ret <8 x half> %res
+}
+
+declare <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float>, <8 x half>, <4 x float>, i8, i32)
+
+define <4 x float> @test_int_x86_avx512fp16_mask_cvt_sh2ss_round(<4 x float> %x0, <8 x half> %x1, <4 x float> %x2, i8 %x3) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512fp16_mask_cvt_sh2ss_round(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <8 x half> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float> [[X0]], <8 x half> [[X1]], <4 x float> [[X2]], i8 [[X3]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float> %x0, <8 x half> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_int_x86_avx512fp16_mask_cvt_sh2ss_round_r(<4 x float> %x0, <8 x half> %x1, <4 x float> %x2, i8 %x3) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512fp16_mask_cvt_sh2ss_round_r(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <8 x half> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float> [[X0]], <8 x half> [[X1]], <4 x float> [[X2]], i8 [[X3]], i32 8)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float> %x0, <8 x half> %x1, <4 x float> %x2, i8 %x3, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_int_x86_avx512fp16_mask_cvt_sh2ss_round_nomask(<4 x float> %x0, <8 x half> %x1, <4 x float> %x2) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512fp16_mask_cvt_sh2ss_round_nomask(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <8 x half> [[X1:%.*]], <4 x float> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float> [[X0]], <8 x half> [[X1]], <4 x float> [[X2]], i8 -1, i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float> %x0, <8 x half> %x1, <4 x float> %x2, i8 -1, i32 4)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_int_x86_avx512fp16_mask_cvt_sh2ss_round_z(<4 x float> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512fp16_mask_cvt_sh2ss_round_z(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float> [[X0]], <8 x half> [[X1]], <4 x float> zeroinitializer, i8 [[X2]], i32 4)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtsh2ss.round(<4 x float> %x0, <8 x half> %x1, <4 x float> zeroinitializer, i8 %x2, i32 4)
+  ret <4 x float> %res
+}
+
+declare <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double>, <8 x half>, <2 x double>, i8, i32)
+
+define <2 x double> @test_int_x86_avx512fp16_mask_cvt_sh2sd_round(<2 x double> %x0, <8 x half> %x1, <2 x double> %x2, i8 %x3) #0 {
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512fp16_mask_cvt_sh2sd_round(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <8 x half> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double> [[X0]], <8 x half> [[X1]], <2 x double> [[X2]], i8 [[X3]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double> %x0, <8 x half> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_int_x86_avx512fp16_mask_cvt_sh2sd_round_r(<2 x double> %x0, <8 x half> %x1, <2 x double> %x2, i8 %x3) #0 {
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512fp16_mask_cvt_sh2sd_round_r(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <8 x half> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double> [[X0]], <8 x half> [[X1]], <2 x double> [[X2]], i8 [[X3]], i32 8)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double> %x0, <8 x half> %x1, <2 x double> %x2, i8 %x3, i32 8)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_int_x86_avx512fp16_mask_cvt_sh2sd_round_nomask(<2 x double> %x0, <8 x half> %x1, <2 x double> %x2) #0 {
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512fp16_mask_cvt_sh2sd_round_nomask(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <8 x half> [[X1:%.*]], <2 x double> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double> [[X0]], <8 x half> [[X1]], <2 x double> [[X2]], i8 -1, i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double> %x0, <8 x half> %x1, <2 x double> %x2, i8 -1, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_int_x86_avx512fp16_mask_cvt_sh2sd_round_z(<2 x double> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512fp16_mask_cvt_sh2sd_round_z(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double> [[X0]], <8 x half> [[X1]], <2 x double> zeroinitializer, i8 [[X2]], i32 4)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtsh2sd.round(<2 x double> %x0, <8 x half> %x1, <2 x double> zeroinitializer, i8 %x2, i32 4)
+  ret <2 x double> %res
+}
+
+declare <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half>, <16 x float>, i16, i32)
+
+define <16 x float> @test_int_x86_avx512_cvt_ph2psx_512(<16 x half> %x0) #0 {
+; CHECK-LABEL: define <16 x float> @test_int_x86_avx512_cvt_ph2psx_512(
+; CHECK-SAME: <16 x half> [[X0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> [[X0]], <16 x float> poison, i16 -1, i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> %x0, <16 x float> poison, i16 -1, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_int_x86_avx512_mask_cvt_ph2psx_512(<16 x half> %x0, <16 x float> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x float> @test_int_x86_avx512_mask_cvt_ph2psx_512(
+; CHECK-SAME: <16 x half> [[X0:%.*]], <16 x float> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> [[X0]], <16 x float> [[X1]], i16 [[X2]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> %x0, <16 x float> %x1, i16 %x2, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_int_x86_avx512_maskz_cvt_ph2psx_512(<16 x half> %x0, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x float> @test_int_x86_avx512_maskz_cvt_ph2psx_512(
+; CHECK-SAME: <16 x half> [[X0:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> [[X0]], <16 x float> zeroinitializer, i16 [[X2]], i32 4)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> %x0, <16 x float> zeroinitializer, i16 %x2, i32 4)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_int_x86_avx512_cvt_ph2psx_512r(<16 x half> %x0) #0 {
+; CHECK-LABEL: define <16 x float> @test_int_x86_avx512_cvt_ph2psx_512r(
+; CHECK-SAME: <16 x half> [[X0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> [[X0]], <16 x float> poison, i16 -1, i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> %x0, <16 x float> poison, i16 -1, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_int_x86_avx512_mask_cvt_ph2psx_512r(<16 x half> %x0, <16 x float> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x float> @test_int_x86_avx512_mask_cvt_ph2psx_512r(
+; CHECK-SAME: <16 x half> [[X0:%.*]], <16 x float> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> [[X0]], <16 x float> [[X1]], i16 [[X2]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> %x0, <16 x float> %x1, i16 %x2, i32 8)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_int_x86_avx512_maskz_cvt_ph2psx_512r(<16 x half> %x0, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x float> @test_int_x86_avx512_maskz_cvt_ph2psx_512r(
+; CHECK-SAME: <16 x half> [[X0:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> [[X0]], <16 x float> zeroinitializer, i16 [[X2]], i32 8)
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = call <16 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.512(<16 x half> %x0, <16 x float> zeroinitializer, i16 %x2, i32 8)
+  ret <16 x float> %res
+}
+
+declare <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float>, <16 x half>, i16, i32)
+
+define <16 x half> @test_int_x86_avx512_cvt_ps2phx_512(<16 x float> %x0) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_cvt_ps2phx_512(
+; CHECK-SAME: <16 x float> [[X0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> [[X0]], <16 x half> poison, i16 -1, i32 4)
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> %x0, <16 x half> poison, i16 -1, i32 4)
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_ps2phx_512(<16 x float> %x0, <16 x half> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_ps2phx_512(
+; CHECK-SAME: <16 x float> [[X0:%.*]], <16 x half> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> [[X0]], <16 x half> [[X1]], i16 [[X2]], i32 4)
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> %x0, <16 x half> %x1, i16 %x2, i32 4)
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512_maskz_cvt_ps2phx_512(<16 x float> %x0, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_maskz_cvt_ps2phx_512(
+; CHECK-SAME: <16 x float> [[X0:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i16 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> [[X0]], <16 x half> zeroinitializer, i16 [[X2]], i32 4)
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> %x0, <16 x half> zeroinitializer, i16 %x2, i32 4)
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_ps2phx_512r(<16 x float> %x0, <16 x half> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_ps2phx_512r(
+; CHECK-SAME: <16 x float> [[X0:%.*]], <16 x half> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> [[X0]], <16 x half> [[X1]], i16 [[X2]], i32 9)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> [[X0]], <16 x half> [[X1]], i16 -1, i32 10)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <16 x half> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES2]]
+;
+  %res = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> %x0, <16 x half> %x1, i16 %x2, i32 9)
+  %res1 = call <16 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.512(<16 x float> %x0, <16 x half> %x1, i16 -1, i32 10)
+  %res2 = fadd <16 x half> %res, %res1
+  ret <16 x half> %res2
+}
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-vl-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-vl-intrinsics.ll
new file mode 100644
index 0000000000000..e67e5e73134e9
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-vl-intrinsics.ll
@@ -0,0 +1,1898 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt %s -S -mattr=+avx512f -passes=msan 2>&1 | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/avx512fp16-arith-vl-intrinsics.ll
+;
+; Handled suboptimally (visitInstruction):
+; - llvm.x86.avx512fp16.mask.vcvtpd2ph.128
+; - llvm.x86.avx512fp16.mask.vcvtpd2ph.256
+; - llvm.x86.avx512fp16.mask.vcvtph2pd.128
+; - llvm.x86.avx512fp16.mask.vcvtph2pd.256
+; - llvm.x86.avx512fp16.mask.vcvtph2psx.128
+; - llvm.x86.avx512fp16.mask.vcvtph2psx.256
+; - llvm.x86.avx512fp16.mask.vcvtph2udq.128
+; - llvm.x86.avx512fp16.mask.vcvtph2udq.256
+; - llvm.x86.avx512fp16.mask.vcvtps2phx.128
+; - llvm.x86.avx512fp16.mask.vcvtps2phx.256
+; - llvm.x86.avx512fp16.mask.vcvttph2dq.128
+; - llvm.x86.avx512fp16.mask.vcvttph2dq.256
+; - llvm.x86.avx512fp16.mask.vcvttph2udq.128
+; - llvm.x86.avx512fp16.mask.vcvttph2udq.256
+;
+; Handled heuristically:
+; - llvm.x86.avx512fp16.max.ph.128
+; - llvm.x86.avx512fp16.max.ph.256
+; - llvm.x86.avx512fp16.min.ph.128
+; - llvm.x86.avx512fp16.min.ph.256
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <16 x half> @test_int_x86_avx512fp16_add_ph_256(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_add_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fadd <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = fadd <16 x half> %x1, %x2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_mask_add_ph_256(<16 x half> %x1, <16 x half> %x2, <16 x half> %src, i16 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_mask_add_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]], <16 x half> [[SRC:%.*]], i16 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <16 x half>, ptr [[PTR]], align 32
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i16>, ptr [[TMP11]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES0:%.*]] = fadd <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x half> [[SRC]] to <16 x i16>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i16> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i16> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i16> [[TMP16]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i16> [[TMP17]], <16 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[RES0]], <16 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd <16 x half> [[X1]], [[VAL]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP1]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x half> [[T3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x half> [[SRC]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <16 x i16> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i16> [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i16> [[TMP22]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT2:%.*]] = select <16 x i1> [[TMP6]], <16 x i16> [[TMP23]], <16 x i16> [[TMP18]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[T3]], <16 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT2]]
+; CHECK-NEXT:    [[RES:%.*]] = fadd <16 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %msk = bitcast i16 %mask to <16 x i1>
+  %val = load <16 x half>, ptr %ptr
+  %res0 = fadd <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %msk, <16 x half> %res0, <16 x half> %src
+  %t3 = fadd <16 x half> %x1, %val
+  %res2 = select <16 x i1> %msk, <16 x half> %t3, <16 x half> %src
+  %res   =  fadd <16 x half> %res1 , %res2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_maskz_add_ph_256(<16 x half> %x1, <16 x half> %x2, i16 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_maskz_add_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]], i16 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES0:%.*]] = fadd <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP]], <16 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i16> [[TMP9]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[RES0]], <16 x half> zeroinitializer
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES1]]
+;
+  %msk = bitcast i16 %mask to <16 x i1>
+  %res0 = fadd <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %msk, <16 x half> %res0, <16 x half> zeroinitializer
+  ret <16 x half> %res1
+}
+
+define <8 x half> @test_int_x86_avx512fp16_add_ph_128(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_add_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fadd <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = fadd <8 x half> %x1, %x2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_add_ph_128(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_add_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <8 x half>, ptr [[PTR]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP11]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES0:%.*]] = fadd <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x half> [[SRC]] to <8 x i16>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i16> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i16> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i16> [[TMP16]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP17]], <8 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[RES0]], <8 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd <8 x half> [[X1]], [[VAL]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP1]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x half> [[T3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x half> [[SRC]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <8 x i16> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i16> [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i16> [[TMP22]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT2:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP23]], <8 x i16> [[TMP18]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[T3]], <8 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT2]]
+; CHECK-NEXT:    [[RES:%.*]] = fadd <8 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %msk = bitcast i8 %mask to <8 x i1>
+  %val = load <8 x half>, ptr %ptr
+  %res0 = fadd <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %msk, <8 x half> %res0, <8 x half> %src
+  %t3 = fadd <8 x half> %x1, %val
+  %res2 = select <8 x i1> %msk, <8 x half> %t3, <8 x half> %src
+  %res   =  fadd <8 x half> %res1 , %res2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_maskz_add_ph_128(<8 x half> %x1, <8 x half> %x2, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_maskz_add_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES0:%.*]] = fadd <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP]], <8 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> [[TMP9]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[RES0]], <8 x half> zeroinitializer
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES1]]
+;
+  %msk = bitcast i8 %mask to <8 x i1>
+  %res0 = fadd <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %msk, <8 x half> %res0, <8 x half> zeroinitializer
+  ret <8 x half> %res1
+}
+
+define <16 x half> @test_int_x86_avx512fp16_sub_ph_256(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_sub_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fsub <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = fsub <16 x half> %x1, %x2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_mask_sub_ph_256(<16 x half> %x1, <16 x half> %x2, <16 x half> %src, i16 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_mask_sub_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]], <16 x half> [[SRC:%.*]], i16 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <16 x half>, ptr [[PTR]], align 32
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i16>, ptr [[TMP11]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES0:%.*]] = fsub <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x half> [[SRC]] to <16 x i16>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i16> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i16> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i16> [[TMP16]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i16> [[TMP17]], <16 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[RES0]], <16 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[T3:%.*]] = fsub <16 x half> [[X1]], [[VAL]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP1]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x half> [[T3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x half> [[SRC]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <16 x i16> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i16> [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i16> [[TMP22]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT2:%.*]] = select <16 x i1> [[TMP6]], <16 x i16> [[TMP23]], <16 x i16> [[TMP18]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[T3]], <16 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT2]]
+; CHECK-NEXT:    [[RES:%.*]] = fsub <16 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %msk = bitcast i16 %mask to <16 x i1>
+  %val = load <16 x half>, ptr %ptr
+  %res0 = fsub <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %msk, <16 x half> %res0, <16 x half> %src
+  %t3 = fsub <16 x half> %x1, %val
+  %res2 = select <16 x i1> %msk, <16 x half> %t3, <16 x half> %src
+  %res   =  fsub <16 x half> %res1 , %res2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_maskz_sub_ph_256(<16 x half> %x1, <16 x half> %x2, i16 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_maskz_sub_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]], i16 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES0:%.*]] = fsub <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP]], <16 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i16> [[TMP9]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[RES0]], <16 x half> zeroinitializer
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES1]]
+;
+  %msk = bitcast i16 %mask to <16 x i1>
+  %res0 = fsub <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %msk, <16 x half> %res0, <16 x half> zeroinitializer
+  ret <16 x half> %res1
+}
+
+define <8 x half> @test_int_x86_avx512fp16_sub_ph_128(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_sub_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fsub <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = fsub <8 x half> %x1, %x2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_sub_ph_128(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_sub_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <8 x half>, ptr [[PTR]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP11]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES0:%.*]] = fsub <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x half> [[SRC]] to <8 x i16>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i16> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i16> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i16> [[TMP16]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP17]], <8 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[RES0]], <8 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[T3:%.*]] = fsub <8 x half> [[X1]], [[VAL]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP1]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x half> [[T3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x half> [[SRC]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <8 x i16> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i16> [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i16> [[TMP22]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT2:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP23]], <8 x i16> [[TMP18]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[T3]], <8 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT2]]
+; CHECK-NEXT:    [[RES:%.*]] = fsub <8 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %msk = bitcast i8 %mask to <8 x i1>
+  %val = load <8 x half>, ptr %ptr
+  %res0 = fsub <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %msk, <8 x half> %res0, <8 x half> %src
+  %t3 = fsub <8 x half> %x1, %val
+  %res2 = select <8 x i1> %msk, <8 x half> %t3, <8 x half> %src
+  %res   =  fsub <8 x half> %res1 , %res2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_maskz_sub_ph_128(<8 x half> %x1, <8 x half> %x2, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_maskz_sub_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES0:%.*]] = fsub <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP]], <8 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> [[TMP9]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[RES0]], <8 x half> zeroinitializer
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES1]]
+;
+  %msk = bitcast i8 %mask to <8 x i1>
+  %res0 = fsub <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %msk, <8 x half> %res0, <8 x half> zeroinitializer
+  ret <8 x half> %res1
+}
+
+define <16 x half> @test_int_x86_avx512fp16_mul_ph_256(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_mul_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fmul <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = fmul <16 x half> %x1, %x2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_mask_mul_ph_256(<16 x half> %x1, <16 x half> %x2, <16 x half> %src, i16 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_mask_mul_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]], <16 x half> [[SRC:%.*]], i16 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <16 x half>, ptr [[PTR]], align 32
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i16>, ptr [[TMP11]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES0:%.*]] = fmul <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x half> [[SRC]] to <16 x i16>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i16> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i16> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i16> [[TMP16]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i16> [[TMP17]], <16 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[RES0]], <16 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[T3:%.*]] = fmul <16 x half> [[X1]], [[VAL]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP1]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x half> [[T3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x half> [[SRC]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <16 x i16> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i16> [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i16> [[TMP22]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT2:%.*]] = select <16 x i1> [[TMP6]], <16 x i16> [[TMP23]], <16 x i16> [[TMP18]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[T3]], <16 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT2]]
+; CHECK-NEXT:    [[RES:%.*]] = fmul <16 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %msk = bitcast i16 %mask to <16 x i1>
+  %val = load <16 x half>, ptr %ptr
+  %res0 = fmul <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %msk, <16 x half> %res0, <16 x half> %src
+  %t3 = fmul <16 x half> %x1, %val
+  %res2 = select <16 x i1> %msk, <16 x half> %t3, <16 x half> %src
+  %res   =  fmul <16 x half> %res1 , %res2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_maskz_mul_ph_256(<16 x half> %x1, <16 x half> %x2, i16 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_maskz_mul_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]], i16 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES0:%.*]] = fmul <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP]], <16 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i16> [[TMP9]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[RES0]], <16 x half> zeroinitializer
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES1]]
+;
+  %msk = bitcast i16 %mask to <16 x i1>
+  %res0 = fmul <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %msk, <16 x half> %res0, <16 x half> zeroinitializer
+  ret <16 x half> %res1
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mul_ph_128(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mul_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fmul <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = fmul <8 x half> %x1, %x2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_mul_ph_128(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_mul_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <8 x half>, ptr [[PTR]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP11]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES0:%.*]] = fmul <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x half> [[SRC]] to <8 x i16>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i16> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i16> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i16> [[TMP16]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP17]], <8 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[RES0]], <8 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[T3:%.*]] = fmul <8 x half> [[X1]], [[VAL]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP1]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x half> [[T3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x half> [[SRC]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <8 x i16> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i16> [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i16> [[TMP22]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT2:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP23]], <8 x i16> [[TMP18]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[T3]], <8 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT2]]
+; CHECK-NEXT:    [[RES:%.*]] = fmul <8 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %msk = bitcast i8 %mask to <8 x i1>
+  %val = load <8 x half>, ptr %ptr
+  %res0 = fmul <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %msk, <8 x half> %res0, <8 x half> %src
+  %t3 = fmul <8 x half> %x1, %val
+  %res2 = select <8 x i1> %msk, <8 x half> %t3, <8 x half> %src
+  %res   =  fmul <8 x half> %res1 , %res2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_maskz_mul_ph_128(<8 x half> %x1, <8 x half> %x2, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_maskz_mul_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES0:%.*]] = fmul <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP]], <8 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> [[TMP9]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[RES0]], <8 x half> zeroinitializer
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES1]]
+;
+  %msk = bitcast i8 %mask to <8 x i1>
+  %res0 = fmul <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %msk, <8 x half> %res0, <8 x half> zeroinitializer
+  ret <8 x half> %res1
+}
+
+define <16 x half> @test_int_x86_avx512fp16_div_ph_256(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_div_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fdiv <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = fdiv <16 x half> %x1, %x2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_div_ph_256_fast(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_div_ph_256_fast(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fdiv fast <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = fdiv fast <16 x half> %x1, %x2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_mask_div_ph_256(<16 x half> %x1, <16 x half> %x2, <16 x half> %src, i16 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_mask_div_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]], <16 x half> [[SRC:%.*]], i16 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <16 x half>, ptr [[PTR]], align 32
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i16>, ptr [[TMP11]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES0:%.*]] = fdiv <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x half> [[SRC]] to <16 x i16>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i16> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i16> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i16> [[TMP16]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i16> [[TMP17]], <16 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[RES0]], <16 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[T3:%.*]] = fdiv <16 x half> [[X1]], [[VAL]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP1]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x half> [[T3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x half> [[SRC]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <16 x i16> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i16> [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <16 x i16> [[TMP22]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT2:%.*]] = select <16 x i1> [[TMP6]], <16 x i16> [[TMP23]], <16 x i16> [[TMP18]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[T3]], <16 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT2]]
+; CHECK-NEXT:    [[RES:%.*]] = fdiv <16 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %msk = bitcast i16 %mask to <16 x i1>
+  %val = load <16 x half>, ptr %ptr
+  %res0 = fdiv <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %msk, <16 x half> %res0, <16 x half> %src
+  %t3 = fdiv <16 x half> %x1, %val
+  %res2 = select <16 x i1> %msk, <16 x half> %t3, <16 x half> %src
+  %res   =  fdiv <16 x half> %res1 , %res2
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512fp16_maskz_div_ph_256(<16 x half> %x1, <16 x half> %x2, i16 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512fp16_maskz_div_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]], i16 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES0:%.*]] = fdiv <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[_MSPROP]], <16 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i16> [[TMP9]], <16 x i16> [[TMP5]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[MSK]], <16 x half> [[RES0]], <16 x half> zeroinitializer
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES1]]
+;
+  %msk = bitcast i16 %mask to <16 x i1>
+  %res0 = fdiv <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %msk, <16 x half> %res0, <16 x half> zeroinitializer
+  ret <16 x half> %res1
+}
+
+define <8 x half> @test_int_x86_avx512fp16_div_ph_128(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_div_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fdiv <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = fdiv <8 x half> %x1, %x2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_div_ph_128_fast(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_div_ph_128_fast(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = fdiv fast <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = fdiv fast <8 x half> %x1, %x2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_mask_div_ph_128(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_div_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[VAL:%.*]] = load <8 x half>, ptr [[PTR]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP11]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES0:%.*]] = fdiv <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x half> [[SRC]] to <8 x i16>
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i16> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i16> [[TMP15]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i16> [[TMP16]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP17]], <8 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[RES0]], <8 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[T3:%.*]] = fdiv <8 x half> [[X1]], [[VAL]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP1]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x half> [[T3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x half> [[SRC]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <8 x i16> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i16> [[TMP21]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <8 x i16> [[TMP22]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT2:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP23]], <8 x i16> [[TMP18]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[T3]], <8 x half> [[SRC]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i16> [[_MSPROP_SELECT]], [[_MSPROP_SELECT2]]
+; CHECK-NEXT:    [[RES:%.*]] = fdiv <8 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %msk = bitcast i8 %mask to <8 x i1>
+  %val = load <8 x half>, ptr %ptr
+  %res0 = fdiv <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %msk, <8 x half> %res0, <8 x half> %src
+  %t3 = fdiv <8 x half> %x1, %val
+  %res2 = select <8 x i1> %msk, <8 x half> %t3, <8 x half> %src
+  %res   =  fdiv <8 x half> %res1 , %res2
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512fp16_maskz_div_ph_128(<8 x half> %x1, <8 x half> %x2, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_maskz_div_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES0:%.*]] = fdiv <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[_MSPROP]], <8 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> [[TMP9]], <8 x i16> [[TMP5]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[MSK]], <8 x half> [[RES0]], <8 x half> zeroinitializer
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES1]]
+;
+  %msk = bitcast i8 %mask to <8 x i1>
+  %res0 = fdiv <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %msk, <8 x half> %res0, <8 x half> zeroinitializer
+  ret <8 x half> %res1
+}
+
+define <16 x half> @test_min_ph_256(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_min_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <16 x i16> [[_MSPROP]] to <16 x i1>
+; CHECK-NEXT:    [[RES0:%.*]] = fcmp olt <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <16 x i1> [[RES0]], <16 x i16> [[TMP1]], <16 x i16> [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x half> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x half> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i16> [[TMP7]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP3]], <16 x i16> [[TMP9]], <16 x i16> [[TMP4]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[RES0]], <16 x half> [[X1]], <16 x half> [[X2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES1]]
+;
+  %res0 = fcmp olt <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %res0, <16 x half> %x1, <16 x half> %x2
+  ret  <16 x half> %res1
+}
+
+define <16 x half> @test_max_ph_256(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_max_ph_256(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <16 x i16> [[_MSPROP]] to <16 x i1>
+; CHECK-NEXT:    [[RES0:%.*]] = fcmp ogt <16 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <16 x i1> [[RES0]], <16 x i16> [[TMP1]], <16 x i16> [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x half> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x half> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i16> [[TMP7]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP3]], <16 x i16> [[TMP9]], <16 x i16> [[TMP4]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <16 x i1> [[RES0]], <16 x half> [[X1]], <16 x half> [[X2]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES1]]
+;
+  %res0 = fcmp ogt <16 x half> %x1, %x2
+  %res1 = select <16 x i1> %res0, <16 x half> %x1, <16 x half> %x2
+  ret  <16 x half> %res1
+}
+
+define <8 x half> @test_min_ph_128(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_min_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <8 x i16> [[_MSPROP]] to <8 x i1>
+; CHECK-NEXT:    [[RES0:%.*]] = fcmp olt <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <8 x i1> [[RES0]], <8 x i16> [[TMP1]], <8 x i16> [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x half> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x half> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i16> [[TMP7]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP3]], <8 x i16> [[TMP9]], <8 x i16> [[TMP4]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[RES0]], <8 x half> [[X1]], <8 x half> [[X2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES1]]
+;
+  %res0 = fcmp olt <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %res0, <8 x half> %x1, <8 x half> %x2
+  ret  <8 x half> %res1
+}
+
+define <8 x half> @test_max_ph_128(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_max_ph_128(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <8 x i16> [[_MSPROP]] to <8 x i1>
+; CHECK-NEXT:    [[RES0:%.*]] = fcmp ogt <8 x half> [[X1]], [[X2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <8 x i1> [[RES0]], <8 x i16> [[TMP1]], <8 x i16> [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x half> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x half> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i16> [[TMP7]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP3]], <8 x i16> [[TMP9]], <8 x i16> [[TMP4]]
+; CHECK-NEXT:    [[RES1:%.*]] = select <8 x i1> [[RES0]], <8 x half> [[X1]], <8 x half> [[X2]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES1]]
+;
+  %res0 = fcmp ogt <8 x half> %x1, %x2
+  %res1 = select <8 x i1> %res0, <8 x half> %x1, <8 x half> %x2
+  ret  <8 x half> %res1
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.max.ph.128(<8 x half>, <8 x half>)
+declare <16 x half> @llvm.x86.avx512fp16.max.ph.256(<16 x half>, <16 x half>)
+
+define <8 x half> @test_max_ph_128_2(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_max_ph_128_2(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512fp16.max.ph.128(<8 x half> [[X1]], <8 x half> [[X2]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES0]]
+;
+  %res0 = call <8 x half> @llvm.x86.avx512fp16.max.ph.128(<8 x half> %x1, <8 x half> %x2)
+  ret  <8 x half> %res0
+}
+
+define <16 x half> @test_max_ph_256_2(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_max_ph_256_2(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x half> @llvm.x86.avx512fp16.max.ph.256(<16 x half> [[X1]], <16 x half> [[X2]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES0]]
+;
+  %res0 = call <16 x half> @llvm.x86.avx512fp16.max.ph.256(<16 x half> %x1, <16 x half> %x2)
+  ret  <16 x half> %res0
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.min.ph.128(<8 x half>, <8 x half>)
+declare <16 x half> @llvm.x86.avx512fp16.min.ph.256(<16 x half>, <16 x half>)
+
+define <8 x half> @test_min_ph_128_2(<8 x half> %x1, <8 x half> %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_min_ph_128_2(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512fp16.min.ph.128(<8 x half> [[X1]], <8 x half> [[X2]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES0]]
+;
+  %res0 = call <8 x half> @llvm.x86.avx512fp16.min.ph.128(<8 x half> %x1, <8 x half> %x2)
+  ret  <8 x half> %res0
+}
+
+define <16 x half> @test_min_ph_256_2(<16 x half> %x1, <16 x half> %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_min_ph_256_2(
+; CHECK-SAME: <16 x half> [[X1:%.*]], <16 x half> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x half> @llvm.x86.avx512fp16.min.ph.256(<16 x half> [[X1]], <16 x half> [[X2]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES0]]
+;
+  %res0 = call <16 x half> @llvm.x86.avx512fp16.min.ph.256(<16 x half> %x1, <16 x half> %x2)
+  ret  <16 x half> %res0
+}
+
+declare <4 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.256(<8 x half>, <4 x double>, i8)
+
+define <4 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_256(<8 x half> %x0, <4 x double> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x double> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.256(<8 x half> [[X0]], <4 x double> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.256(<8 x half> %x0, <4 x double> %x1, i8 %x2)
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_256_nomask(<8 x half> %x0, <4 x double> %x1) #0 {
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_256_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x double> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.256(<8 x half> [[X0]], <4 x double> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.256(<8 x half> %x0, <4 x double> %x1, i8 -1)
+  ret <4 x double> %res
+}
+
+declare <2 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.128(<8 x half>, <2 x double>, i8)
+
+define <2 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_128(<8 x half> %x0, <2 x double> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <2 x double> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.128(<8 x half> [[X0]], <2 x double> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.128(<8 x half> %x0, <2 x double> %x1, i8 %x2)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_128_nomask(<8 x half> %x0, <2 x double> %x1) #0 {
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask_vcvt_ph2pd_128_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <2 x double> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.128(<8 x half> [[X0]], <2 x double> [[X1]], i8 -1)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512fp16.mask.vcvtph2pd.128(<8 x half> %x0, <2 x double> %x1, i8 -1)
+  ret <2 x double> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.256(<4 x double>, <8 x half>, i8)
+
+define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_256(<4 x double> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.256(<4 x double> [[X0]], <8 x half> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.256(<4 x double> %x0, <8 x half> %x1, i8 %x2)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_256_load(ptr %px0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_256_load(
+; CHECK-SAME: ptr [[PX0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[X0:%.*]] = load <4 x double>, ptr [[PX0]], align 32
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PX0]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 32
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i64> [[_MSLD]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB11:.*]], label %[[BB12:.*]], !prof [[PROF1]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.256(<4 x double> [[X0]], <8 x half> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %x0 = load <4 x double>, ptr %px0, align 32
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.256(<4 x double> %x0, <8 x half> %x1, i8 %x2)
+  ret <8 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.128(<2 x double>, <8 x half>, i8)
+
+define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_128(<2 x double> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.128(<2 x double> [[X0]], <8 x half> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.128(<2 x double> %x0, <8 x half> %x1, i8 %x2)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_128_load(ptr %px0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_vcvt_pd2ph_128_load(
+; CHECK-SAME: ptr [[PX0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[X0:%.*]] = load <2 x double>, ptr [[PX0]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PX0]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSOR]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB11:.*]], label %[[BB12:.*]], !prof [[PROF1]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.128(<2 x double> [[X0]], <8 x half> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %x0 = load <2 x double>, ptr %px0, align 16
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtpd2ph.128(<2 x double> %x0, <8 x half> %x1, i8 %x2)
+  ret <8 x half> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.128(<8 x half>, <4 x i32>, i8)
+
+define <4 x i32> @test_int_x86_avx512_cvt_ph2udq_128(<8 x half> %x0) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvt_ph2udq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.128(<8 x half> [[X0]], <4 x i32> poison, i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.128(<8 x half> %x0, <4 x i32> poison, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_int_x86_avx512_mask_cvt_ph2udq_128(<8 x half> %x0, <4 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvt_ph2udq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.128(<8 x half> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.128(<8 x half> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_int_x86_avx512_maskz_cvt_ph2udq_128(<8 x half> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_cvt_ph2udq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.128(<8 x half> [[X0]], <4 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.128(<8 x half> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.256(<8 x half>, <8 x i32>, i8)
+
+define <8 x i32> @test_int_x86_avx512_cvt_ph2udq_256(<8 x half> %x0) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_cvt_ph2udq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.256(<8 x half> [[X0]], <8 x i32> poison, i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.256(<8 x half> %x0, <8 x i32> poison, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_int_x86_avx512_mask_cvt_ph2udq_256(<8 x half> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_cvt_ph2udq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.256(<8 x half> [[X0]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.256(<8 x half> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_int_x86_avx512_maskz_cvt_ph2udq_256(<8 x half> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_maskz_cvt_ph2udq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.256(<8 x half> [[X0]], <8 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.256(<8 x half> %x0, <8 x i32> zeroinitializer, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.128(<8 x half>, <4 x i32>, i8)
+
+define <4 x i32> @test_int_x86_avx512_cvtt_ph2dq_128(<8 x half> %x0) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvtt_ph2dq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.128(<8 x half> [[X0]], <4 x i32> poison, i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.128(<8 x half> %x0, <4 x i32> poison, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_int_x86_avx512_mask_cvtt_ph2dq_128(<8 x half> %x0, <4 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvtt_ph2dq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.128(<8 x half> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.128(<8 x half> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_int_x86_avx512_maskz_cvtt_ph2dq_128(<8 x half> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_cvtt_ph2dq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.128(<8 x half> [[X0]], <4 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.128(<8 x half> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.256(<8 x half>, <8 x i32>, i8)
+
+define <8 x i32> @test_int_x86_avx512_cvtt_ph2dq_256(<8 x half> %x0) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_cvtt_ph2dq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.256(<8 x half> [[X0]], <8 x i32> poison, i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.256(<8 x half> %x0, <8 x i32> poison, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_int_x86_avx512_mask_cvtt_ph2dq_256(<8 x half> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_cvtt_ph2dq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.256(<8 x half> [[X0]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.256(<8 x half> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_int_x86_avx512_maskz_cvtt_ph2dq_256(<8 x half> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_maskz_cvtt_ph2dq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.256(<8 x half> [[X0]], <8 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.256(<8 x half> %x0, <8 x i32> zeroinitializer, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.128(<8 x half>, <4 x i32>, i8)
+
+define <4 x i32> @test_int_x86_avx512_cvtt_ph2udq_128(<8 x half> %x0) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvtt_ph2udq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.128(<8 x half> [[X0]], <4 x i32> poison, i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.128(<8 x half> %x0, <4 x i32> poison, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_int_x86_avx512_mask_cvtt_ph2udq_128(<8 x half> %x0, <4 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvtt_ph2udq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.128(<8 x half> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.128(<8 x half> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_int_x86_avx512_maskz_cvtt_ph2udq_128(<8 x half> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_cvtt_ph2udq_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.128(<8 x half> [[X0]], <4 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.128(<8 x half> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.256(<8 x half>, <8 x i32>, i8)
+
+define <8 x i32> @test_int_x86_avx512_cvtt_ph2udq_256(<8 x half> %x0) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_cvtt_ph2udq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.256(<8 x half> [[X0]], <8 x i32> poison, i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.256(<8 x half> %x0, <8 x i32> poison, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_int_x86_avx512_mask_cvtt_ph2udq_256(<8 x half> %x0, <8 x i32> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_cvtt_ph2udq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.256(<8 x half> [[X0]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.256(<8 x half> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_int_x86_avx512_maskz_cvtt_ph2udq_256(<8 x half> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_maskz_cvtt_ph2udq_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.256(<8 x half> [[X0]], <8 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.256(<8 x half> %x0, <8 x i32> zeroinitializer, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.128(<8 x half>, <4 x float>, i8)
+
+define <4 x float> @test_int_x86_avx512_cvt_ph2psx_128(<8 x half> %x0) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_cvt_ph2psx_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.128(<8 x half> [[X0]], <4 x float> poison, i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.128(<8 x half> %x0, <4 x float> poison, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_int_x86_avx512_mask_cvt_ph2psx_128(<8 x half> %x0, <4 x float> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_cvt_ph2psx_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <4 x float> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.128(<8 x half> [[X0]], <4 x float> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.128(<8 x half> %x0, <4 x float> %x1, i8 %x2)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_int_x86_avx512_maskz_cvt_ph2psx_128(<8 x half> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_maskz_cvt_ph2psx_128(
+; CHECK-SAME: <8 x half> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.128(<8 x half> [[X0]], <4 x float> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.128(<8 x half> %x0, <4 x float> zeroinitializer, i8 %x2)
+  ret <4 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.256(<8 x half>, <8 x float>, i8)
+
+define <8 x float> @test_int_x86_avx512_cvt_ph2psx_256(<8 x half> %x0) #0 {
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_cvt_ph2psx_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.256(<8 x half> [[X0]], <8 x float> poison, i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.256(<8 x half> %x0, <8 x float> poison, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_int_x86_avx512_mask_cvt_ph2psx_256(<8 x half> %x0, <8 x float> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask_cvt_ph2psx_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x float> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.256(<8 x half> [[X0]], <8 x float> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.256(<8 x half> %x0, <8 x float> %x1, i8 %x2)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_int_x86_avx512_maskz_cvt_ph2psx_256(<8 x half> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_maskz_cvt_ph2psx_256(
+; CHECK-SAME: <8 x half> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.256(<8 x half> [[X0]], <8 x float> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512fp16.mask.vcvtph2psx.256(<8 x half> %x0, <8 x float> zeroinitializer, i8 %x2)
+  ret <8 x float> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.128(<4 x float>, <8 x half>, i8)
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_ps2phx_128(<4 x float> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_ps2phx_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.128(<4 x float> [[X0]], <8 x half> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.128(<4 x float> [[X0]], <8 x half> [[X1]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <8 x half> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES2]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.128(<4 x float> %x0, <8 x half> %x1, i8 %x2)
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.128(<4 x float> %x0, <8 x half> %x1, i8 -1)
+  %res2 = fadd <8 x half> %res, %res1
+  ret <8 x half> %res2
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.256(<8 x float>, <8 x half>, i8)
+
+define <8 x half> @test_int_x86_avx512_cvt_ps2phx_256(<8 x float> %x0) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_cvt_ps2phx_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.256(<8 x float> [[X0]], <8 x half> poison, i8 -1)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.256(<8 x float> %x0, <8 x half> poison, i8 -1)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_ps2phx_256(<8 x float> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_ps2phx_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.256(<8 x float> [[X0]], <8 x half> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.256(<8 x float> %x0, <8 x half> %x1, i8 %x2)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_maskz_cvt_ps2phx_256(<8 x float> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_maskz_cvt_ps2phx_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.256(<8 x float> [[X0]], <8 x half> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.vcvtps2phx.256(<8 x float> %x0, <8 x half> zeroinitializer, i8 %x2)
+  ret <8 x half> %res
+}
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-intrinsics.ll
new file mode 100644
index 0000000000000..61a32e5e2042e
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-intrinsics.ll
@@ -0,0 +1,3248 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt %s -S -mattr=+avx512f -passes=msan 2>&1 | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
+;
+; Handled suboptimally (visitInstruction):
+; - llvm.x86.avx512fp16.fpclass.ph.512
+; - llvm.x86.avx512fp16.mask.add.sh.round
+; - llvm.x86.avx512fp16.mask.cmp.sh
+; - llvm.x86.avx512fp16.mask.div.sh.round
+; - llvm.x86.avx512fp16.mask.fpclass.sh
+; - llvm.x86.avx512fp16.mask.getexp.ph.512
+; - llvm.x86.avx512fp16.mask.getexp.sh
+; - llvm.x86.avx512fp16.mask.getmant.ph.512
+; - llvm.x86.avx512fp16.mask.getmant.sh
+; - llvm.x86.avx512fp16.mask.max.sh.round
+; - llvm.x86.avx512fp16.mask.min.sh.round
+; - llvm.x86.avx512fp16.mask.mul.sh.round
+; - llvm.x86.avx512fp16.mask.rcp.ph.512
+; - llvm.x86.avx512fp16.mask.rcp.sh
+; - llvm.x86.avx512fp16.mask.reduce.ph.512
+; - llvm.x86.avx512fp16.mask.reduce.sh
+; - llvm.x86.avx512fp16.mask.rndscale.ph.512
+; - llvm.x86.avx512fp16.mask.rndscale.sh
+; - llvm.x86.avx512fp16.mask.rsqrt.ph.512
+; - llvm.x86.avx512fp16.mask.rsqrt.sh
+; - llvm.x86.avx512fp16.mask.scalef.ph.512
+; - llvm.x86.avx512fp16.mask.scalef.sh
+; - llvm.x86.avx512fp16.mask.sqrt.sh
+; - llvm.x86.avx512fp16.mask.sub.sh.round
+; - llvm.x86.avx512fp16.mask.vcvtph2dq.512
+; - llvm.x86.avx512fp16.mask.vcvtph2qq.512
+; - llvm.x86.avx512fp16.mask.vcvtph2udq.512
+; - llvm.x86.avx512fp16.mask.vcvtph2uqq.512
+; - llvm.x86.avx512fp16.mask.vcvttph2dq.512
+; - llvm.x86.avx512fp16.mask.vcvttph2udq.512
+; - llvm.x86.avx512fp16.mask.vcvttph2uqq.512
+; - llvm.x86.avx512fp16.sqrt.ph.512
+; - llvm.x86.avx512fp16.vcomi.sh
+; - llvm.x86.avx512fp16.vcvtsh2si32
+; - llvm.x86.avx512fp16.vcvtsh2si64
+; - llvm.x86.avx512fp16.vcvtsh2usi32
+; - llvm.x86.avx512fp16.vcvtsh2usi64
+; - llvm.x86.avx512fp16.vcvtsi2sh
+; - llvm.x86.avx512fp16.vcvtsi642sh
+; - llvm.x86.avx512fp16.vcvttsh2si32
+; - llvm.x86.avx512fp16.vcvttsh2si64
+; - llvm.x86.avx512fp16.vcvttsh2usi32
+; - llvm.x86.avx512fp16.vcvttsh2usi64
+; - llvm.x86.avx512fp16.vcvtusi2sh
+; - llvm.x86.avx512fp16.vcvtusi642sh
+; - llvm.x86.avx512.sitofp.round.v16f16.v16i32
+; - llvm.x86.avx512.sitofp.round.v8f16.v8i64
+; - llvm.x86.avx512.uitofp.round.v16f16.v16i32
+; - llvm.x86.avx512.uitofp.round.v8f16.v8i64
+;
+; Handled heuristically:
+; - llvm.sqrt.f16
+; - llvm.sqrt.v32f16
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half>, <8 x half>, i32, i32)
+
+define i32 @test_x86_avx512fp16_ucomi_sh_lt(<8 x half> %a0, <8 x half> %a1) #0 {
+; CHECK-LABEL: define i32 @test_x86_avx512fp16_ucomi_sh_lt(
+; CHECK-SAME: <8 x half> [[A0:%.*]], <8 x half> [[A1:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> [[A0]], <8 x half> [[A1]], i32 9, i32 4)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %a0, <8 x half> %a1, i32 9, i32 4)
+  ret i32 %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half>, i32) nounwind readnone
+
+define <32 x half> @test_sqrt_ph_512(<32 x half> %a0) #0 {
+; CHECK-LABEL: define <32 x half> @test_sqrt_ph_512(
+; CHECK-SAME: <32 x half> [[A0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x half> @llvm.sqrt.v32f16(<32 x half> [[A0]])
+; CHECK-NEXT:    store <32 x i16> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP2]]
+;
+  %1 = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
+  ret <32 x half> %1
+}
+
+define <32 x half> @test_sqrt_ph_512_fast(<32 x half> %a0, <32 x half> %a1) #0 {
+; CHECK-LABEL: define <32 x half> @test_sqrt_ph_512_fast(
+; CHECK-SAME: <32 x half> [[A0:%.*]], <32 x half> [[A1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call fast <32 x half> @llvm.sqrt.v32f16(<32 x half> [[A0]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fdiv fast <32 x half> [[A1]], [[TMP3]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP4]]
+;
+  %1 = call fast <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
+  %2 = fdiv fast <32 x half> %a1, %1
+  ret <32 x half> %2
+}
+
+define <32 x half> @test_sqrt_ph_512_fast_estimate_attribute(<32 x half> %a0, <32 x half> %a1) "reciprocal-estimates"="vec-sqrt" {
+; CHECK-LABEL: define <32 x half> @test_sqrt_ph_512_fast_estimate_attribute(
+; CHECK-SAME: <32 x half> [[A0:%.*]], <32 x half> [[A1:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <32 x half> @llvm.sqrt.v32f16(<32 x half> [[A0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = fdiv fast <32 x half> [[A1]], [[TMP1]]
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP2]]
+;
+  %1 = call fast <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
+  %2 = fdiv fast <32 x half> %a1, %1
+  ret <32 x half> %2
+}
+
+define <32 x half> @test_sqrt_ph_512_fast_estimate_attribute_2(<32 x half> %a0, <32 x half> %a1) "reciprocal-estimates"="vec-sqrth:1" {
+; CHECK-LABEL: define <32 x half> @test_sqrt_ph_512_fast_estimate_attribute_2(
+; CHECK-SAME: <32 x half> [[A0:%.*]], <32 x half> [[A1:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <32 x half> @llvm.sqrt.v32f16(<32 x half> [[A0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = fdiv fast <32 x half> [[A1]], [[TMP1]]
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP2]]
+;
+  %1 = call fast <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
+  %2 = fdiv fast <32 x half> %a1, %1
+  ret <32 x half> %2
+}
+
+define <32 x half> @test_mask_sqrt_ph_512(<32 x half> %a0, <32 x half> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: define <32 x half> @test_mask_sqrt_ph_512(
+; CHECK-SAME: <32 x half> [[A0:%.*]], <32 x half> [[PASSTHRU:%.*]], i32 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x half> @llvm.sqrt.v32f16(<32 x half> [[A0]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP1]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x half> [[TMP4]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x half> [[PASSTHRU]] to <32 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <32 x i16> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP12]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP6]], <32 x half> [[TMP4]], <32 x half> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP13]]
+;
+  %1 = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x half> %1, <32 x half> %passthru
+  ret <32 x half> %3
+}
+
+define <32 x half> @test_maskz_sqrt_ph_512(<32 x half> %a0, i32 %mask) #0 {
+; CHECK-LABEL: define <32 x half> @test_maskz_sqrt_ph_512(
+; CHECK-SAME: <32 x half> [[A0:%.*]], i32 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x half> @llvm.sqrt.v32f16(<32 x half> [[A0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP1]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x half> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i16> [[TMP10]], <32 x i16> [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP5]], <32 x half> [[TMP3]], <32 x half> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP11]]
+;
+  %1 = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x half> %1, <32 x half> zeroinitializer
+  ret <32 x half> %3
+}
+
+declare <32 x half> @llvm.sqrt.v32f16(<32 x half>)
+
+define <32 x half> @test_sqrt_round_ph_512(<32 x half> %a0) #0 {
+; CHECK-LABEL: define <32 x half> @test_sqrt_round_ph_512(
+; CHECK-SAME: <32 x half> [[A0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> [[A0]], i32 11)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP5]]
+;
+  %1 = call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %a0, i32 11)
+  ret <32 x half> %1
+}
+
+define <32 x half> @test_mask_sqrt_round_ph_512(<32 x half> %a0, <32 x half> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: define <32 x half> @test_mask_sqrt_round_ph_512(
+; CHECK-SAME: <32 x half> [[A0:%.*]], <32 x half> [[PASSTHRU:%.*]], i32 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> [[A0]], i32 11)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[TMP9]], <32 x i16> zeroinitializer, <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[TMP7]] to <32 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[PASSTHRU]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP8]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP9]], <32 x half> [[TMP7]], <32 x half> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP16]]
+;
+  %1 = call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %a0, i32 11)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x half> %1, <32 x half> %passthru
+  ret <32 x half> %3
+}
+
+define <32 x half> @test_maskz_sqrt_round_ph_512(<32 x half> %a0, i32 %mask) #0 {
+; CHECK-LABEL: define <32 x half> @test_maskz_sqrt_round_ph_512(
+; CHECK-SAME: <32 x half> [[A0:%.*]], i32 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> [[A0]], i32 11)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[TMP8]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x half> [[TMP6]] to <32 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <32 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP13]], <32 x i16> [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <32 x i1> [[TMP8]], <32 x half> [[TMP6]], <32 x half> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[TMP14]]
+;
+  %1 = call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %a0, i32 11)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x half> %1, <32 x half> zeroinitializer
+  ret <32 x half> %3
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half>, <8 x half>, <8 x half>, i8, i32) nounwind readnone
+
+define <8 x half> @test_sqrt_sh(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask) #0 {
+; CHECK-LABEL: define <8 x half> @test_sqrt_sh(
+; CHECK-SAME: <8 x half> [[A0:%.*]], <8 x half> [[A1:%.*]], <8 x half> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> [[A0]], <8 x half> [[A1]], <8 x half> [[A2]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask, i32 4)
+  ret <8 x half> %res
+}
+
+define half @test_sqrt_sh2(half %a0, half %a1) #0 {
+; CHECK-LABEL: define half @test_sqrt_sh2(
+; CHECK-SAME: half [[A0:%.*]], half [[A1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call fast half @llvm.sqrt.f16(half [[A0]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i16 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fdiv fast half [[A1]], [[TMP3]]
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret half [[TMP4]]
+;
+  %1 = call fast half @llvm.sqrt.f16(half %a0)
+  %2 = fdiv fast half %a1, %1
+  ret half %2
+}
+
+define half @test_sqrt_sh3(half %a0, half %a1) #0 {
+; CHECK-LABEL: define half @test_sqrt_sh3(
+; CHECK-SAME: half [[A0:%.*]], half [[A1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast half @llvm.sqrt.f16(half [[A0]])
+; CHECK-NEXT:    store i16 [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret half [[TMP2]]
+;
+  %1 = call fast half @llvm.sqrt.f16(half %a0)
+  ret half %1
+}
+
+declare half @llvm.sqrt.f16(half)
+
+define <8 x half> @test_sqrt_sh_r(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask) #0 {
+; CHECK-LABEL: define <8 x half> @test_sqrt_sh_r(
+; CHECK-SAME: <8 x half> [[A0:%.*]], <8 x half> [[A1:%.*]], <8 x half> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> [[A0]], <8 x half> [[A1]], <8 x half> [[A2]], i8 [[MASK]], i32 10)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask, i32 10)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_sqrt_sh_nomask(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2) #0 {
+; CHECK-LABEL: define <8 x half> @test_sqrt_sh_nomask(
+; CHECK-SAME: <8 x half> [[A0:%.*]], <8 x half> [[A1:%.*]], <8 x half> [[A2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> [[A0]], <8 x half> [[A1]], <8 x half> [[A2]], i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 -1, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_sqrt_sh_z(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask) #0 {
+; CHECK-LABEL: define <8 x half> @test_sqrt_sh_z(
+; CHECK-SAME: <8 x half> [[A0:%.*]], <8 x half> [[A1:%.*]], <8 x half> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> [[A0]], <8 x half> [[A1]], <8 x half> zeroinitializer, i8 [[MASK]], i32 10)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %a0, <8 x half> %a1, <8 x half> zeroinitializer, i8 %mask, i32 10)
+  ret <8 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.512(<32 x half>, <32 x half>, i32)
+declare <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half>, <8 x half>, <8 x half>, i8)
+
+define <32 x half> @test_rsqrt_ph_512(<32 x half> %a0) #0 {
+; CHECK-LABEL: define <32 x half> @test_rsqrt_ph_512(
+; CHECK-SAME: <32 x half> [[A0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.512(<32 x half> [[A0]], <32 x half> zeroinitializer, i32 -1)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.512(<32 x half> %a0, <32 x half> zeroinitializer, i32 -1)
+  ret <32 x half> %res
+}
+
+define <8 x half> @test_rsqrt_sh(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2) #0 {
+; CHECK-LABEL: define <8 x half> @test_rsqrt_sh(
+; CHECK-SAME: <8 x half> [[A0:%.*]], <8 x half> [[A1:%.*]], <8 x half> [[A2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half> [[A0]], <8 x half> [[A0]], <8 x half> [[A2]], i8 -1)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half> %a0, <8 x half> %a0, <8 x half> %a2, i8 -1)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_rsqrt_sh_load(<8 x half> %a0, ptr %a1ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_rsqrt_sh_load(
+; CHECK-SAME: <8 x half> [[A0:%.*]], ptr [[A1PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[A1:%.*]] = load <8 x half>, ptr [[A1PTR]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A1PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half> [[A0]], <8 x half> [[A1]], <8 x half> poison, i8 -1)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %a1 = load <8 x half>, ptr %a1ptr
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half> %a0, <8 x half> %a1, <8 x half> poison, i8 -1)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_rsqrt_sh_maskz(<8 x half> %a0, i8 %mask) #0 {
+; CHECK-LABEL: define <8 x half> @test_rsqrt_sh_maskz(
+; CHECK-SAME: <8 x half> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half> [[A0]], <8 x half> [[A0]], <8 x half> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half> %a0, <8 x half> %a0, <8 x half> zeroinitializer, i8 %mask)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_rsqrt_sh_mask(<8 x half> %a0, <8 x half> %b0, <8 x half> %c0, i8 %mask) #0 {
+; CHECK-LABEL: define <8 x half> @test_rsqrt_sh_mask(
+; CHECK-SAME: <8 x half> [[A0:%.*]], <8 x half> [[B0:%.*]], <8 x half> [[C0:%.*]], i8 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half> [[A0]], <8 x half> [[B0]], <8 x half> [[C0]], i8 [[MASK]])
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh(<8 x half> %a0, <8 x half> %b0, <8 x half> %c0, i8 %mask)
+  ret <8 x half> %res
+}
+
+declare <32 x i1> @llvm.x86.avx512fp16.fpclass.ph.512(<32 x half>, i32)
+
+define i32 @test_int_x86_avx512_fpclass_ph_512(<32 x half> %x0) #0 {
+; CHECK-LABEL: define i32 @test_int_x86_avx512_fpclass_ph_512(
+; CHECK-SAME: <32 x half> [[X0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i1> @llvm.x86.avx512fp16.fpclass.ph.512(<32 x half> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <32 x i1> @llvm.x86.avx512fp16.fpclass.ph.512(<32 x half> [[X0]], i32 2)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <32 x i1> [[RES1]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = and <32 x i1> zeroinitializer, [[RES]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i1> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[RES1]], [[RES]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <32 x i1> [[TMP11]] to i32
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <32 x i1> [[TMP12]] to i32
+; CHECK-NEXT:    store i32 [[TMP13]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP14]]
+;
+  %res = call <32 x i1> @llvm.x86.avx512fp16.fpclass.ph.512(<32 x half> %x0, i32 4)
+  %res1 = call <32 x i1> @llvm.x86.avx512fp16.fpclass.ph.512(<32 x half> %x0, i32 2)
+  %1 = and <32 x i1> %res1, %res
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+declare i8 @llvm.x86.avx512fp16.mask.fpclass.sh(<8 x half>, i32, i8)
+
+define i8 @test_int_x86_avx512_mask_fpclass_sh(<8 x half> %x0) #0 {
+; CHECK-LABEL: define i8 @test_int_x86_avx512_mask_fpclass_sh(
+; CHECK-SAME: <8 x half> [[X0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.x86.avx512fp16.mask.fpclass.sh(<8 x half> [[X0]], i32 2, i8 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i8 @llvm.x86.avx512fp16.mask.fpclass.sh(<8 x half> [[X0]], i32 4, i8 [[RES]])
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %res = call i8 @llvm.x86.avx512fp16.mask.fpclass.sh(<8 x half> %x0, i32 2, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512fp16.mask.fpclass.sh(<8 x half> %x0, i32 4, i8 %res)
+  ret i8 %res1
+}
+
+define i8 @test_int_x86_avx512_mask_fpclass_sh_load(ptr %x0ptr) #0 {
+; CHECK-LABEL: define i8 @test_int_x86_avx512_mask_fpclass_sh_load(
+; CHECK-SAME: ptr [[X0PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[X0:%.*]] = load <8 x half>, ptr [[X0PTR]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X0PTR]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.x86.avx512fp16.mask.fpclass.sh(<8 x half> [[X0]], i32 4, i8 -1)
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %x0 = load <8 x half>, ptr %x0ptr
+  %res = call i8 @llvm.x86.avx512fp16.mask.fpclass.sh(<8 x half> %x0, i32 4, i8 -1)
+  ret i8 %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.mask.rcp.ph.512(<32 x half>, <32 x half>, i32)
+
+define <32 x half> @test_rcp_ph_512(<32 x half> %a0, <32 x half> %a1, i32 %mask) #0 {
+; CHECK-LABEL: define <32 x half> @test_rcp_ph_512(
+; CHECK-SAME: <32 x half> [[A0:%.*]], <32 x half> [[A1:%.*]], i32 [[MASK:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.rcp.ph.512(<32 x half> [[A0]], <32 x half> [[A1]], i32 [[MASK]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.mask.rcp.ph.512(<32 x half> %a0, <32 x half> %a1, i32 %mask)
+  ret <32 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh(<8 x half>, <8 x half>, <8 x half>, i8)
+
+define <8 x half> @test_rcp_sh(<8 x half> %a0) #0 {
+; CHECK-LABEL: define <8 x half> @test_rcp_sh(
+; CHECK-SAME: <8 x half> [[A0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh(<8 x half> [[A0]], <8 x half> [[A0]], <8 x half> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh(<8 x half> %a0, <8 x half> %a0, <8 x half> zeroinitializer, i8 -1)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_rcp_sh_load(<8 x half> %a0, ptr %a1ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_rcp_sh_load(
+; CHECK-SAME: <8 x half> [[A0:%.*]], ptr [[A1PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[A1:%.*]] = load <8 x half>, ptr [[A1PTR]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A1PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh(<8 x half> [[A0]], <8 x half> [[A1]], <8 x half> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %a1 = load <8 x half>, ptr %a1ptr
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh(<8 x half> %a0, <8 x half> %a1, <8 x half> zeroinitializer, i8 -1)
+  ret <8 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.mask.reduce.ph.512(<32 x half>, i32, <32 x half>, i32, i32)
+
+define <32 x half>@test_int_x86_avx512_mask_reduce_ph_512(<32 x half> %x0, <32 x half> %x2, i32 %x3) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512_mask_reduce_ph_512(
+; CHECK-SAME: <32 x half> [[X0:%.*]], <32 x half> [[X2:%.*]], i32 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.reduce.ph.512(<32 x half> [[X0]], i32 8, <32 x half> [[X2]], i32 [[X3]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.reduce.ph.512(<32 x half> [[X0]], i32 4, <32 x half> [[X2]], i32 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <32 x half> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES2]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.mask.reduce.ph.512(<32 x half> %x0, i32 8, <32 x half> %x2, i32 %x3, i32 4)
+  %res1 = call <32 x half> @llvm.x86.avx512fp16.mask.reduce.ph.512(<32 x half> %x0, i32 4, <32 x half> %x2, i32 -1, i32 8)
+  %res2 = fadd <32 x half> %res, %res1
+  ret <32 x half> %res2
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.reduce.sh(<8 x half>, <8 x half>,<8 x half>, i8, i32, i32)
+
+define <8 x half>@test_int_x86_avx512_mask_reduce_sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_reduce_sh(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]], i8 [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.reduce.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> [[X3]], i8 [[X4]], i32 4, i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.reduce.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4, i32 4, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half>@test_int_x86_avx512_mask_reduce_sh_nomask(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_reduce_sh_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.reduce.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> [[X3]], i8 -1, i32 4, i32 8)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.reduce.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 -1, i32 4, i32 8)
+  ret <8 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512(<32 x half>, i32, <32 x half>, i32, i32)
+
+define <32 x half>@test_int_x86_avx512_mask_rndscale_ph_512(<32 x half> %x0, <32 x half> %x2, i32 %x3) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512_mask_rndscale_ph_512(
+; CHECK-SAME: <32 x half> [[X0:%.*]], <32 x half> [[X2:%.*]], i32 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512(<32 x half> [[X0]], i32 8, <32 x half> [[X2]], i32 [[X3]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512(<32 x half> [[X0]], i32 4, <32 x half> [[X2]], i32 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <32 x half> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES2]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512(<32 x half> %x0, i32 8, <32 x half> %x2, i32 %x3, i32 4)
+  %res1 = call <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512(<32 x half> %x0, i32 4, <32 x half> %x2, i32 -1, i32 8)
+  %res2 = fadd <32 x half> %res, %res1
+  ret <32 x half> %res2
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh(<8 x half>, <8 x half>,<8 x half>, i8, i32, i32)
+
+define <8 x half>@test_int_x86_avx512_mask_rndscale_sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_rndscale_sh(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]], i8 [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> [[X3]], i8 [[X4]], i32 4, i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4, i32 4, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half>@test_int_x86_avx512_mask_rndscale_sh_nomask(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_rndscale_sh_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> [[X3]], i8 -1, i32 4, i32 8)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 -1, i32 4, i32 8)
+  ret <8 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.mask.getexp.ph.512(<32 x half>, <32 x half>, i32, i32)
+
+define <32 x half>@test_int_x86_avx512_mask_getexp_ph_512(<32 x half> %x0, <32 x half> %x1, i32 %x2) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512_mask_getexp_ph_512(
+; CHECK-SAME: <32 x half> [[X0:%.*]], <32 x half> [[X1:%.*]], i32 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.getexp.ph.512(<32 x half> [[X0]], <32 x half> [[X1]], i32 [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.getexp.ph.512(<32 x half> [[X0]], <32 x half> zeroinitializer, i32 -1, i32 8)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <32 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES3]]
+;
+  %res1 = call <32 x half> @llvm.x86.avx512fp16.mask.getexp.ph.512(<32 x half> %x0, <32 x half> %x1, i32 %x2, i32 4)
+  %res2 = call <32 x half> @llvm.x86.avx512fp16.mask.getexp.ph.512(<32 x half> %x0, <32 x half> zeroinitializer, i32 -1, i32 8)
+  %res3 = fadd <32 x half> %res1, %res2
+  ret <32 x half> %res3
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.getexp.sh(<8 x half>, <8 x half>,<8 x half>, i8, i32)
+
+define <8 x half>@test_int_x86_avx512_mask_getexp_sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_getexp_sh(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]], i8 [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.getexp.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> [[X3]], i8 [[X4]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.getexp.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half>@test_int_x86_avx512_mask_getexp_sh_nomask(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_getexp_sh_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.getexp.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> [[X3]], i8 -1, i32 8)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.getexp.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 -1, i32 8)
+  ret <8 x half> %res
+}
+
+define <8 x half>@test_int_x86_avx512_mask_getexp_sh_load(<8 x half> %x0, ptr %x1ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_getexp_sh_load(
+; CHECK-SAME: <8 x half> [[X0:%.*]], ptr [[X1PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[X1:%.*]] = load <8 x half>, ptr [[X1PTR]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[X1PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.getexp.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> poison, i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %x1 = load <8 x half>, ptr %x1ptr
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.getexp.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> poison, i8 -1, i32 4)
+  ret <8 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.mask.getmant.ph.512(<32 x half>, i32, <32 x half>, i32, i32)
+
+define <32 x half>@test_int_x86_avx512_mask_getmant_ph_512(<32 x half> %x0, <32 x half> %x2, i32 %x3) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512_mask_getmant_ph_512(
+; CHECK-SAME: <32 x half> [[X0:%.*]], <32 x half> [[X2:%.*]], i32 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.getmant.ph.512(<32 x half> [[X0]], i32 8, <32 x half> [[X2]], i32 [[X3]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.getmant.ph.512(<32 x half> [[X0]], i32 4, <32 x half> [[X2]], i32 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <32 x half> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES2]]
+;
+  %res = call <32 x half> @llvm.x86.avx512fp16.mask.getmant.ph.512(<32 x half> %x0, i32 8, <32 x half> %x2, i32 %x3, i32 4)
+  %res1 = call <32 x half> @llvm.x86.avx512fp16.mask.getmant.ph.512(<32 x half> %x0, i32 4, <32 x half> %x2, i32 -1, i32 8)
+  %res2 = fadd <32 x half> %res, %res1
+  ret <32 x half> %res2
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.getmant.sh(<8 x half>, <8 x half>, i32, <8 x half>, i8, i32)
+
+define <8 x half>@test_int_x86_avx512_mask_getmant_sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_getmant_sh(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]], i8 [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.getmant.sh(<8 x half> [[X0]], <8 x half> [[X1]], i32 11, <8 x half> [[X3]], i8 [[X4]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.getmant.sh(<8 x half> %x0, <8 x half> %x1, i32 11, <8 x half> %x3, i8 %x4, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half>@test_int_x86_avx512_mask_getmant_sh_nomask(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_getmant_sh_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.getmant.sh(<8 x half> [[X0]], <8 x half> [[X1]], i32 11, <8 x half> [[X3]], i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.getmant.sh(<8 x half> %x0, <8 x half> %x1, i32 11, <8 x half> %x3, i8 -1, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half>@test_int_x86_avx512_mask_getmant_sh_z(<8 x half> %x0, <8 x half> %x1, i8 %x4) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_getmant_sh_z(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.getmant.sh(<8 x half> [[X0]], <8 x half> [[X1]], i32 11, <8 x half> zeroinitializer, i8 [[X4]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.getmant.sh(<8 x half> %x0, <8 x half> %x1, i32 11, <8 x half> zeroinitializer, i8 %x4, i32 4)
+  ret <8 x half> %res
+}
+
+declare <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512(<32 x half>, <32 x half>, <32 x half>, i32, i32)
+
+define <32 x half>@test_int_x86_avx512_mask_scalef_ph_512(<32 x half> %x0, <32 x half> %x1, <32 x half> %x2, i32 %x3) #0 {
+; CHECK-LABEL: define <32 x half> @test_int_x86_avx512_mask_scalef_ph_512(
+; CHECK-SAME: <32 x half> [[X0:%.*]], <32 x half> [[X1:%.*]], <32 x half> [[X2:%.*]], i32 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[X3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512(<32 x half> [[X0]], <32 x half> [[X1]], <32 x half> [[X2]], i32 [[X3]], i32 11)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    br i1 [[_MSOR8]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512(<32 x half> [[X0]], <32 x half> [[X1]], <32 x half> zeroinitializer, i32 -1, i32 8)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <32 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES3]]
+;
+  %mask = bitcast i32 %x3 to <32 x i1>
+  %res1 = call <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512(<32 x half> %x0, <32 x half> %x1, <32 x half> %x2, i32 %x3, i32 11)
+  %res2 = call <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512(<32 x half> %x0, <32 x half> %x1, <32 x half> zeroinitializer, i32 -1, i32 8)
+  %res3 = fadd <32 x half> %res1, %res2
+  ret <32 x half> %res3
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half>, <8 x half>,<8 x half>, i8, i32)
+
+define <8 x half>@test_int_x86_avx512_mask_scalef_sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_scalef_sh(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]], i8 [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> [[X3]], i8 [[X4]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 %x4, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half>@test_int_x86_avx512_mask_scalef_sh_nomask(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_scalef_sh_nomask(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], <8 x half> [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> [[X3]], i8 -1, i32 8)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> %x3, i8 -1, i32 8)
+  ret <8 x half> %res
+}
+
+define <8 x half>@test_int_x86_avx512_mask_scalef_sh_load(<8 x half> %x0, ptr %x1ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_scalef_sh_load(
+; CHECK-SAME: <8 x half> [[X0:%.*]], ptr [[X1PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[X1:%.*]] = load <8 x half>, ptr [[X1PTR]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[X1PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> [[X0]], <8 x half> [[X1]], <8 x half> poison, i8 -1, i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %x1 = load <8 x half>, ptr %x1ptr
+  %res = call <8 x half> @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> %x0, <8 x half> %x1, <8 x half> poison, i8 -1, i32 4)
+  ret <8 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half>, <8 x half>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512fp16_mask_add_sh(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_add_sh(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL_HALF:%.*]] = load half, ptr [[PTR]], align 2
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> splat (i16 -1), i16 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VAL:%.*]] = insertelement <8 x half> poison, half [[VAL_HALF]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half> [[X1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half> [[RES0]], <8 x half> [[X2]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSCMP8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half> [[RES1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i16> [[_MSPROP]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB24:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half> [[RES2]], <8 x half> [[VAL]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES3]]
+;
+  %val.half = load half,ptr %ptr
+  %val = insertelement <8 x half> poison, half %val.half, i32 0
+  %res0 = call <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half> %x1, <8 x half> %x2, <8 x half> zeroinitializer, i8 -1, i32 4)
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half> %res0, <8 x half> %x2, <8 x half> %src , i8 %mask, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half> %res1, <8 x half> %x2, <8 x half> zeroinitializer , i8 %mask, i32 4)
+  %res3 = call <8 x half> @llvm.x86.avx512fp16.mask.add.sh.round(<8 x half> %res2, <8 x half> %val, <8 x half> %src , i8 %mask, i32 4)
+  ret <8 x half> %res3
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half>, <8 x half>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512fp16_mask_sub_sh(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_sub_sh(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL_HALF:%.*]] = load half, ptr [[PTR]], align 2
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> splat (i16 -1), i16 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VAL:%.*]] = insertelement <8 x half> poison, half [[VAL_HALF]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half> [[X1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half> [[RES0]], <8 x half> [[X2]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSCMP8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half> [[RES1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i16> [[_MSPROP]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB24:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half> [[RES2]], <8 x half> [[VAL]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES3]]
+;
+  %val.half = load half,ptr %ptr
+  %val = insertelement <8 x half> poison, half %val.half, i32 0
+  %res0 = call <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half> %x1, <8 x half> %x2, <8 x half> zeroinitializer, i8 -1, i32 4)
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half> %res0, <8 x half> %x2, <8 x half> %src , i8 %mask, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half> %res1, <8 x half> %x2, <8 x half> zeroinitializer , i8 %mask, i32 4)
+  %res3 = call <8 x half> @llvm.x86.avx512fp16.mask.sub.sh.round(<8 x half> %res2, <8 x half> %val, <8 x half> %src , i8 %mask, i32 4)
+  ret <8 x half> %res3
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half>, <8 x half>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512fp16_mask_mul_sh(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_mul_sh(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL_HALF:%.*]] = load half, ptr [[PTR]], align 2
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> splat (i16 -1), i16 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VAL:%.*]] = insertelement <8 x half> poison, half [[VAL_HALF]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half> [[X1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half> [[RES0]], <8 x half> [[X2]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSCMP8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half> [[RES1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i16> [[_MSPROP]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB24:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half> [[RES2]], <8 x half> [[VAL]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES3]]
+;
+  %val.half = load half,ptr %ptr
+  %val = insertelement <8 x half> poison, half %val.half, i32 0
+  %res0 = call <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half> %x1, <8 x half> %x2, <8 x half> zeroinitializer, i8 -1, i32 4)
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half> %res0, <8 x half> %x2, <8 x half> %src , i8 %mask, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half> %res1, <8 x half> %x2, <8 x half> zeroinitializer , i8 %mask, i32 4)
+  %res3 = call <8 x half> @llvm.x86.avx512fp16.mask.mul.sh.round(<8 x half> %res2, <8 x half> %val, <8 x half> %src , i8 %mask, i32 4)
+  ret <8 x half> %res3
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half>, <8 x half>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512fp16_mask_div_sh(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_div_sh(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL_HALF:%.*]] = load half, ptr [[PTR]], align 2
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> splat (i16 -1), i16 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VAL:%.*]] = insertelement <8 x half> poison, half [[VAL_HALF]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half> [[X1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half> [[RES0]], <8 x half> [[X2]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSCMP8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half> [[RES1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i16> [[_MSPROP]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB24:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half> [[RES2]], <8 x half> [[VAL]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES3]]
+;
+  %val.half = load half,ptr %ptr
+  %val = insertelement <8 x half> poison, half %val.half, i32 0
+  %res0 = call <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half> %x1, <8 x half> %x2, <8 x half> zeroinitializer, i8 -1, i32 4)
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half> %res0, <8 x half> %x2, <8 x half> %src , i8 %mask, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half> %res1, <8 x half> %x2, <8 x half> zeroinitializer , i8 %mask, i32 4)
+  %res3 = call <8 x half> @llvm.x86.avx512fp16.mask.div.sh.round(<8 x half> %res2, <8 x half> %val, <8 x half> %src , i8 %mask, i32 4)
+  ret <8 x half> %res3
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half>, <8 x half>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512fp16_mask_min_sh(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_min_sh(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL_HALF:%.*]] = load half, ptr [[PTR]], align 2
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> splat (i16 -1), i16 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VAL:%.*]] = insertelement <8 x half> poison, half [[VAL_HALF]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half> [[X1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half> [[RES0]], <8 x half> [[X2]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSCMP8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half> [[RES1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i16> [[_MSPROP]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB24:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half> [[RES2]], <8 x half> [[VAL]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES3]]
+;
+  %val.half = load half,ptr %ptr
+  %val = insertelement <8 x half> poison, half %val.half, i32 0
+  %res0 = call <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half> %x1, <8 x half> %x2, <8 x half> zeroinitializer, i8 -1, i32 4)
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half> %res0, <8 x half> %x2, <8 x half> %src , i8 %mask, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half> %res1, <8 x half> %x2, <8 x half> zeroinitializer , i8 %mask, i32 4)
+  %res3 = call <8 x half> @llvm.x86.avx512fp16.mask.min.sh.round(<8 x half> %res2, <8 x half> %val, <8 x half> %src , i8 %mask, i32 4)
+  ret <8 x half> %res3
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half>, <8 x half>, <8 x half>, i8, i32)
+
+define <8 x half> @test_int_x86_avx512fp16_mask_max_sh(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, ptr %ptr) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512fp16_mask_max_sh(
+; CHECK-SAME: <8 x half> [[X1:%.*]], <8 x half> [[X2:%.*]], <8 x half> [[SRC:%.*]], i8 [[MASK:%.*]], ptr [[PTR:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[VAL_HALF:%.*]] = load half, ptr [[PTR]], align 2
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> splat (i16 -1), i16 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VAL:%.*]] = insertelement <8 x half> poison, half [[VAL_HALF]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half> [[X1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half> [[RES0]], <8 x half> [[X2]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSCMP8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half> [[RES1]], <8 x half> [[X2]], <8 x half> zeroinitializer, i8 [[MASK]], i32 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i16> [[_MSPROP]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP22]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i16> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i8 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB24:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK:       [[BB24]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half> [[RES2]], <8 x half> [[VAL]], <8 x half> [[SRC]], i8 [[MASK]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES3]]
+;
+  %val.half = load half,ptr %ptr
+  %val = insertelement <8 x half> poison, half %val.half, i32 0
+  %res0 = call <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half> %x1, <8 x half> %x2, <8 x half> zeroinitializer, i8 -1, i32 4)
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half> %res0, <8 x half> %x2, <8 x half> %src , i8 %mask, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half> %res1, <8 x half> %x2, <8 x half> zeroinitializer , i8 %mask, i32 4)
+  %res3 = call <8 x half> @llvm.x86.avx512fp16.mask.max.sh.round(<8 x half> %res2, <8 x half> %val, <8 x half> %src , i8 %mask, i32 4)
+  ret <8 x half> %res3
+}
+
+declare i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half>, <8 x half>, i32, i8, i32)
+
+define i8 @test_int_x86_avx512_mask_cmp_sh(<8 x half> %x0, <8 x half> %x1, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: define i8 @test_int_x86_avx512_mask_cmp_sh(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X3:%.*]], i32 [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> [[X0]], <8 x half> [[X1]], i32 3, i8 [[X3]], i32 4)
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES2]]
+;
+  %res2 = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> %x0, <8 x half> %x1, i32 3, i8 %x3, i32 4)
+  ret i8 %res2
+}
+
+
+define i8 @test_int_x86_avx512_mask_cmp_sh_all(<8 x half> %x0, <8 x half> %x1, i8 %x3, i32 %x4) #0 {
+; CHECK-LABEL: define i8 @test_int_x86_avx512_mask_cmp_sh_all(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X3:%.*]], i32 [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> [[X0]], <8 x half> [[X1]], i32 2, i8 -1, i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> [[X0]], <8 x half> [[X1]], i32 3, i8 -1, i32 8)
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSCMP5]], [[_MSCMP6]]
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSOR7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> [[X0]], <8 x half> [[X1]], i32 4, i8 [[X3]], i32 4)
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP10:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP17]], 0
+; CHECK-NEXT:    [[_MSOR12:%.*]] = or i1 [[_MSCMP10]], [[_MSCMP11]]
+; CHECK-NEXT:    [[_MSCMP13:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR14:%.*]] = or i1 [[_MSOR12]], [[_MSCMP13]]
+; CHECK-NEXT:    br i1 [[_MSOR14]], label %[[BB18:.*]], label %[[BB19:.*]], !prof [[PROF1]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB19]]:
+; CHECK-NEXT:    [[RES4:%.*]] = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> [[X0]], <8 x half> [[X1]], i32 5, i8 [[X3]], i32 8)
+; CHECK-NEXT:    [[TMP20:%.*]] = and i8 [[RES1]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = and i8 0, [[RES2]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or i8 0, [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or i8 [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[RES11:%.*]] = and i8 [[RES1]], [[RES2]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and i8 [[RES3]], 0
+; CHECK-NEXT:    [[TMP25:%.*]] = and i8 0, [[RES4]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or i8 0, [[TMP24]]
+; CHECK-NEXT:    [[TMP27:%.*]] = or i8 [[TMP26]], [[TMP25]]
+; CHECK-NEXT:    [[RES12:%.*]] = and i8 [[RES3]], [[RES4]]
+; CHECK-NEXT:    [[TMP28:%.*]] = and i8 [[TMP23]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = and i8 [[RES11]], [[TMP27]]
+; CHECK-NEXT:    [[TMP30:%.*]] = and i8 [[TMP23]], [[RES12]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or i8 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = or i8 [[TMP31]], [[TMP30]]
+; CHECK-NEXT:    [[RES13:%.*]] = and i8 [[RES11]], [[RES12]]
+; CHECK-NEXT:    store i8 [[TMP32]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[RES13]]
+;
+  %res1 = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> %x0, <8 x half> %x1, i32 2, i8 -1, i32 4)
+  %res2 = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> %x0, <8 x half> %x1, i32 3, i8 -1, i32 8)
+  %res3 = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> %x0, <8 x half> %x1, i32 4, i8 %x3, i32 4)
+  %res4 = call i8 @llvm.x86.avx512fp16.mask.cmp.sh(<8 x half> %x0, <8 x half> %x1, i32 5, i8 %x3, i32 8)
+
+  %res11 = and i8 %res1, %res2
+  %res12 = and i8 %res3, %res4
+  %res13 = and i8 %res11, %res12
+  ret i8 %res13
+}
+
+declare <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32>, i32)
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_dq2ph_512(<16 x i32> %x0, <16 x half> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_dq2ph_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x half> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i16 [[X2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[MASK]], <16 x i16> zeroinitializer, <16 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x half> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i16> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i16> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i16> [[TMP13]], <16 x i16> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[MASK]], <16 x half> [[RES0]], <16 x half> [[X1]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %mask = bitcast i16 %x2 to <16 x i1>
+  %res0 = call <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32> %x0, i32 4)
+  %res = select <16 x i1> %mask, <16 x half> %res0, <16 x half> %x1
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_dq2ph_512_r(<16 x i32> %x0, <16 x half> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_dq2ph_512_r(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x half> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i16 [[X2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32> [[X0]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[MASK]], <16 x i16> zeroinitializer, <16 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x half> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i16> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i16> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i16> [[TMP13]], <16 x i16> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[MASK]], <16 x half> [[RES0]], <16 x half> [[X1]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %mask = bitcast i16 %x2 to <16 x i1>
+  %res0 = call <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32> %x0, i32 10)
+  %res = select <16 x i1> %mask, <16 x half> %res0, <16 x half> %x1
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_dq2ph_512_nomask(<16 x i32> %x0, <16 x half> %x1) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_dq2ph_512_nomask(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x half> [[X1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32> [[X0]], i32 4)
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = call <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32> %x0, i32 4)
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_dq2ph_512_z(<16 x i32> %x0, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_dq2ph_512_z(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i16 [[X2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[MASK]], <16 x i16> zeroinitializer, <16 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP3]], <16 x i16> [[TMP11]], <16 x i16> [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[MASK]], <16 x half> [[RES0]], <16 x half> zeroinitializer
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %mask = bitcast i16 %x2 to <16 x i1>
+  %res0 = call <16 x half> @llvm.x86.avx512.sitofp.round.v16f16.v16i32(<16 x i32> %x0, i32 4)
+  %res = select <16 x i1> %mask, <16 x half> %res0, <16 x half> zeroinitializer
+  ret <16 x half> %res
+}
+
+define <16 x half> @sint_to_fp_16i32_to_16f16(<16 x i32> %x) #0 {
+; CHECK-LABEL: define <16 x half> @sint_to_fp_16i32_to_16f16(
+; CHECK-SAME: <16 x i32> [[X:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <16 x i32> [[TMP1]] to <16 x i16>
+; CHECK-NEXT:    [[RES:%.*]] = sitofp <16 x i32> [[X]] to <16 x half>
+; CHECK-NEXT:    store <16 x i16> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = sitofp <16 x i32> %x to <16 x half>
+  ret <16 x half> %res
+}
+
+declare <16 x half> @llvm.x86.avx512.uitofp.round.v16f16.v16i32(<16 x i32>, i32)
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_udq2ph_512_r(<16 x i32> %x0, <16 x half> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_udq2ph_512_r(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x half> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i16 [[X2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x half> @llvm.x86.avx512.uitofp.round.v16f16.v16i32(<16 x i32> [[X0]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[MASK]], <16 x i16> zeroinitializer, <16 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x half> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i16> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i16> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP4]], <16 x i16> [[TMP13]], <16 x i16> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[MASK]], <16 x half> [[RES0]], <16 x half> [[X1]]
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %mask = bitcast i16 %x2 to <16 x i1>
+  %res0 = call <16 x half> @llvm.x86.avx512.uitofp.round.v16f16.v16i32(<16 x i32> %x0, i32 10)
+  %res = select <16 x i1> %mask, <16 x half> %res0, <16 x half> %x1
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_udq2ph_512_nomask(<16 x i32> %x0, <16 x half> %x1) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_udq2ph_512_nomask(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x half> [[X1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x half> @llvm.x86.avx512.uitofp.round.v16f16.v16i32(<16 x i32> [[X0]], i32 4)
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = call <16 x half> @llvm.x86.avx512.uitofp.round.v16f16.v16i32(<16 x i32> %x0, i32 4)
+  ret <16 x half> %res
+}
+
+define <16 x half> @test_int_x86_avx512_mask_cvt_udq2ph_512_z(<16 x i32> %x0, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x half> @test_int_x86_avx512_mask_cvt_udq2ph_512_z(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i16 [[X2]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x half> @llvm.x86.avx512.uitofp.round.v16f16.v16i32(<16 x i32> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[MASK]], <16 x i16> zeroinitializer, <16 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x half> [[RES0]] to <16 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP3]], <16 x i16> [[TMP11]], <16 x i16> [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[MASK]], <16 x half> [[RES0]], <16 x half> zeroinitializer
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %mask = bitcast i16 %x2 to <16 x i1>
+  %res0 = call <16 x half> @llvm.x86.avx512.uitofp.round.v16f16.v16i32(<16 x i32> %x0, i32 4)
+  %res = select <16 x i1> %mask, <16 x half> %res0, <16 x half> zeroinitializer
+  ret <16 x half> %res
+}
+
+define <16 x half> @uint_to_fp_16i32_to_16f16(<16 x i32> %x) #0 {
+; CHECK-LABEL: define <16 x half> @uint_to_fp_16i32_to_16f16(
+; CHECK-SAME: <16 x i32> [[X:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <16 x i32> [[TMP1]] to <16 x i16>
+; CHECK-NEXT:    [[RES:%.*]] = uitofp <16 x i32> [[X]] to <16 x half>
+; CHECK-NEXT:    store <16 x i16> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %res = uitofp <16 x i32> %x to <16 x half>
+  ret <16 x half> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2dq.512(<16 x half>, <16 x i32>, i16, i32)
+
+define <16 x i32> @test_int_x86_avx512_mask_cvt_ph2dq_512(<16 x half> %x0, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_mask_cvt_ph2dq_512(
+; CHECK-SAME: <16 x half> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2dq.512(<16 x half> [[X0]], <16 x i32> [[X1]], i16 [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2dq.512(<16 x half> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2dq.512(<16 x half> %x0, <16 x i32> %x1, i16 %x2, i32 10)
+  %res1 = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2dq.512(<16 x half> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.512(<16 x half>, <16 x i32>, i16, i32)
+
+define <16 x i32> @test_int_x86_avx512_mask_cvt_ph2udq_512(<16 x half> %x0, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_mask_cvt_ph2udq_512(
+; CHECK-SAME: <16 x half> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.512(<16 x half> [[X0]], <16 x i32> [[X1]], i16 [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.512(<16 x half> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.512(<16 x half> %x0, <16 x i32> %x1, i16 %x2, i32 10)
+  %res1 = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvtph2udq.512(<16 x half> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.512(<16 x half>, <16 x i32>, i16, i32)
+
+define <16 x i32> @test_int_x86_avx512_mask_cvtt_ph2dq_512(<16 x half> %x0, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_mask_cvtt_ph2dq_512(
+; CHECK-SAME: <16 x half> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.512(<16 x half> [[X0]], <16 x i32> [[X1]], i16 [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.512(<16 x half> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.512(<16 x half> %x0, <16 x i32> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2dq.512(<16 x half> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.512(<16 x half>, <16 x i32>, i16, i32)
+
+define <16 x i32> @test_int_x86_avx512_mask_cvtt_ph2udq_512(<16 x half> %x0, <16 x i32> %x1, i16 %x2) #0 {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_mask_cvtt_ph2udq_512(
+; CHECK-SAME: <16 x half> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.512(<16 x half> [[X0]], <16 x i32> [[X1]], i16 [[X2]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i16> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.512(<16 x half> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES2]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.512(<16 x half> %x0, <16 x i32> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x i32> @llvm.x86.avx512fp16.mask.vcvttph2udq.512(<16 x half> %x0, <16 x i32> %x1, i16 -1, i32 8)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
+}
+
+declare <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64>, i32)
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_qq2ph_512(<8 x i64> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_qq2ph_512(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i8 [[X2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[MASK]], <8 x i16> zeroinitializer, <8 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x half> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i16> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i16> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> [[TMP13]], <8 x i16> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK]], <8 x half> [[RES0]], <8 x half> [[X1]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %mask = bitcast i8 %x2 to <8 x i1>
+  %res0 = call <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64> %x0, i32 4)
+  %res = select <8 x i1> %mask, <8 x half> %res0, <8 x half> %x1
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_qq2ph_512_r(<8 x i64> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_qq2ph_512_r(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i8 [[X2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64> [[X0]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[MASK]], <8 x i16> zeroinitializer, <8 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x half> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i16> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i16> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> [[TMP13]], <8 x i16> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK]], <8 x half> [[RES0]], <8 x half> [[X1]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %mask = bitcast i8 %x2 to <8 x i1>
+  %res0 = call <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64> %x0, i32 10)
+  %res = select <8 x i1> %mask, <8 x half> %res0, <8 x half> %x1
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_qq2ph_512_nomask(<8 x i64> %x0, <8 x half> %x1) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_qq2ph_512_nomask(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], <8 x half> [[X1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64> [[X0]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64> %x0, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_qq2ph_512_z(<8 x i64> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_qq2ph_512_z(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i8 [[X2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[MASK]], <8 x i16> zeroinitializer, <8 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP3]], <8 x i16> [[TMP11]], <8 x i16> [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK]], <8 x half> [[RES0]], <8 x half> zeroinitializer
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %mask = bitcast i8 %x2 to <8 x i1>
+  %res0 = call <8 x half> @llvm.x86.avx512.sitofp.round.v8f16.v8i64(<8 x i64> %x0, i32 4)
+  %res = select <8 x i1> %mask, <8 x half> %res0, <8 x half> zeroinitializer
+  ret <8 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64>, i32)
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_uqq2ph_512(<8 x i64> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_uqq2ph_512(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i8 [[X2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[MASK]], <8 x i16> zeroinitializer, <8 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x half> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i16> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i16> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> [[TMP13]], <8 x i16> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK]], <8 x half> [[RES0]], <8 x half> [[X1]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %mask = bitcast i8 %x2 to <8 x i1>
+  %res0 = call <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64> %x0, i32 4)
+  %res = select <8 x i1> %mask, <8 x half> %res0, <8 x half> %x1
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_uqq2ph_512_r(<8 x i64> %x0, <8 x half> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_uqq2ph_512_r(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], <8 x half> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i8 [[X2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64> [[X0]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[MASK]], <8 x i16> zeroinitializer, <8 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x half> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i16> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i16> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP4]], <8 x i16> [[TMP13]], <8 x i16> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK]], <8 x half> [[RES0]], <8 x half> [[X1]]
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %mask = bitcast i8 %x2 to <8 x i1>
+  %res0 = call <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64> %x0, i32 10)
+  %res = select <8 x i1> %mask, <8 x half> %res0, <8 x half> %x1
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_uqq2ph_512_nomask(<8 x i64> %x0, <8 x half> %x1) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_uqq2ph_512_nomask(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], <8 x half> [[X1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64> [[X0]], i32 4)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res = call <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64> %x0, i32 4)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_int_x86_avx512_mask_cvt_uqq2ph_512_z(<8 x i64> %x0, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x half> @test_int_x86_avx512_mask_cvt_uqq2ph_512_z(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MASK:%.*]] = bitcast i8 [[X2]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[MASK]], <8 x i16> zeroinitializer, <8 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x half> [[RES0]] to <8 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP3]], <8 x i16> [[TMP11]], <8 x i16> [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[MASK]], <8 x half> [[RES0]], <8 x half> zeroinitializer
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %mask = bitcast i8 %x2 to <8 x i1>
+  %res0 = call <8 x half> @llvm.x86.avx512.uitofp.round.v8f16.v8i64(<8 x i64> %x0, i32 4)
+  %res = select <8 x i1> %mask, <8 x half> %res0, <8 x half> zeroinitializer
+  ret <8 x half> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2qq.512(<8 x half>, <8 x i64>, i8, i32)
+
+define <8 x i64> @test_int_x86_avx512_mask_cvt_ph2qq_512(<8 x half> %x0, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i64> @test_int_x86_avx512_mask_cvt_ph2qq_512(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2qq.512(<8 x half> [[X0]], <8 x i64> [[X1]], i8 [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2qq.512(<8 x half> [[X0]], <8 x i64> [[X1]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i64> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2qq.512(<8 x half> %x0, <8 x i64> %x1, i8 %x2, i32 10)
+  %res1 = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2qq.512(<8 x half> %x0, <8 x i64> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2uqq.512(<8 x half>, <8 x i64>, i8, i32)
+
+define <8 x i64> @test_int_x86_avx512_mask_cvt_ph2uqq_512(<8 x half> %x0, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i64> @test_int_x86_avx512_mask_cvt_ph2uqq_512(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2uqq.512(<8 x half> [[X0]], <8 x i64> [[X1]], i8 [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2uqq.512(<8 x half> [[X0]], <8 x i64> [[X1]], i8 -1, i32 8)
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i64> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2uqq.512(<8 x half> %x0, <8 x i64> %x1, i8 %x2, i32 10)
+  %res1 = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvtph2uqq.512(<8 x half> %x0, <8 x i64> %x1, i8 -1, i32 8)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare <8 x i64> @llvm.x86.avx512fp16.mask.vcvttph2uqq.512(<8 x half>, <8 x i64>, i8, i32)
+
+define <8 x i64> @test_int_x86_avx512_mask_cvtt_ph2uqq_512(<8 x half> %x0, <8 x i64> %x1, i8 %x2) #0 {
+; CHECK-LABEL: define <8 x i64> @test_int_x86_avx512_mask_cvtt_ph2uqq_512(
+; CHECK-SAME: <8 x half> [[X0:%.*]], <8 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvttph2uqq.512(<8 x half> [[X0]], <8 x i64> [[X1]], i8 [[X2]], i32 8)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvttph2uqq.512(<8 x half> [[X0]], <8 x i64> [[X1]], i8 -1, i32 4)
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i64> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[RES2]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvttph2uqq.512(<8 x half> %x0, <8 x i64> %x1, i8 %x2, i32 8)
+  %res1 = call <8 x i64> @llvm.x86.avx512fp16.mask.vcvttph2uqq.512(<8 x half> %x0, <8 x i64> %x1, i8 -1, i32 4)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
+}
+
+declare i32 @llvm.x86.avx512fp16.vcvtsh2si32(<8 x half>, i32)
+
+define i32 @test_x86_avx512fp16_vcvtsh2si32(<8 x half> %arg0) #0 {
+; CHECK-LABEL: define i32 @test_x86_avx512fp16_vcvtsh2si32(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512fp16.vcvtsh2si32(<8 x half> [[ARG0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @llvm.x86.avx512fp16.vcvtsh2si32(<8 x half> [[ARG0]], i32 11)
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[RES1]], [[RES2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res1 = call i32 @llvm.x86.avx512fp16.vcvtsh2si32(<8 x half> %arg0, i32 4)
+  %res2 = call i32 @llvm.x86.avx512fp16.vcvtsh2si32(<8 x half> %arg0, i32 11)
+  %res = add i32 %res1, %res2
+  ret i32 %res
+}
+
+declare i64 @llvm.x86.avx512fp16.vcvtsh2si64(<8 x half>, i32)
+
+define i64 @test_x86_avx512fp16_vcvtsh2si64(<8 x half> %arg0) #0 {
+; CHECK-LABEL: define i64 @test_x86_avx512fp16_vcvtsh2si64(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i64 @llvm.x86.avx512fp16.vcvtsh2si64(<8 x half> [[ARG0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i64 @llvm.x86.avx512fp16.vcvtsh2si64(<8 x half> [[ARG0]], i32 10)
+; CHECK-NEXT:    [[RES:%.*]] = add i64 [[RES1]], [[RES2]]
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %res1 = call i64 @llvm.x86.avx512fp16.vcvtsh2si64(<8 x half> %arg0, i32 4)
+  %res2 = call i64 @llvm.x86.avx512fp16.vcvtsh2si64(<8 x half> %arg0, i32 10)
+  %res = add i64 %res1, %res2
+  ret i64 %res
+}
+
+declare i32 @llvm.x86.avx512fp16.vcvttsh2si32(<8 x half>, i32)
+
+define i32 @test_x86_avx512fp16_vcvttsh2si32(<8 x half> %arg0) #0 {
+; CHECK-LABEL: define i32 @test_x86_avx512fp16_vcvttsh2si32(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512fp16.vcvttsh2si32(<8 x half> [[ARG0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @llvm.x86.avx512fp16.vcvttsh2si32(<8 x half> [[ARG0]], i32 8)
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[RES1]], [[RES2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res1 = call i32 @llvm.x86.avx512fp16.vcvttsh2si32(<8 x half> %arg0, i32 4)
+  %res2 = call i32 @llvm.x86.avx512fp16.vcvttsh2si32(<8 x half> %arg0, i32 8)
+  %res = add i32 %res1, %res2
+  ret i32 %res
+}
+
+declare i64 @llvm.x86.avx512fp16.vcvttsh2si64(<8 x half>, i32)
+
+define i64 @test_x86_avx512fp16_vcvttsh2si64(<8 x half> %arg0) #0 {
+; CHECK-LABEL: define i64 @test_x86_avx512fp16_vcvttsh2si64(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i64 @llvm.x86.avx512fp16.vcvttsh2si64(<8 x half> [[ARG0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i64 @llvm.x86.avx512fp16.vcvttsh2si64(<8 x half> [[ARG0]], i32 8)
+; CHECK-NEXT:    [[RES:%.*]] = add i64 [[RES1]], [[RES2]]
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %res1 = call i64 @llvm.x86.avx512fp16.vcvttsh2si64(<8 x half> %arg0, i32 4)
+  %res2 = call i64 @llvm.x86.avx512fp16.vcvttsh2si64(<8 x half> %arg0, i32 8)
+  %res = add i64 %res1, %res2
+  ret i64 %res
+}
+
+
+declare i32 @llvm.x86.avx512fp16.vcvtsh2usi32(<8 x half>, i32)
+
+define i32 @test_x86_avx512fp16_vcvtsh2usi32(<8 x half> %arg0) #0 {
+; CHECK-LABEL: define i32 @test_x86_avx512fp16_vcvtsh2usi32(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512fp16.vcvtsh2usi32(<8 x half> [[ARG0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @llvm.x86.avx512fp16.vcvtsh2usi32(<8 x half> [[ARG0]], i32 9)
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[RES1]], [[RES2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res1 = call i32 @llvm.x86.avx512fp16.vcvtsh2usi32(<8 x half> %arg0, i32 4)
+  %res2 = call i32 @llvm.x86.avx512fp16.vcvtsh2usi32(<8 x half> %arg0, i32 9)
+  %res = add i32 %res1, %res2
+  ret i32 %res
+}
+
+
+declare i64 @llvm.x86.avx512fp16.vcvtsh2usi64(<8 x half>, i32)
+
+define i64 @test_x86_avx512fp16_vcvtsh2usi64(<8 x half> %arg0) #0 {
+; CHECK-LABEL: define i64 @test_x86_avx512fp16_vcvtsh2usi64(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i64 @llvm.x86.avx512fp16.vcvtsh2usi64(<8 x half> [[ARG0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i64 @llvm.x86.avx512fp16.vcvtsh2usi64(<8 x half> [[ARG0]], i32 10)
+; CHECK-NEXT:    [[RES:%.*]] = add i64 [[RES1]], [[RES2]]
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %res1 = call i64 @llvm.x86.avx512fp16.vcvtsh2usi64(<8 x half> %arg0, i32 4)
+  %res2 = call i64 @llvm.x86.avx512fp16.vcvtsh2usi64(<8 x half> %arg0, i32 10)
+  %res = add i64 %res1, %res2
+  ret i64 %res
+}
+
+declare i32 @llvm.x86.avx512fp16.vcvttsh2usi32(<8 x half>, i32)
+
+define i32 @test_x86_avx512fp16_vcvttsh2usi32(<8 x half> %arg0) #0 {
+; CHECK-LABEL: define i32 @test_x86_avx512fp16_vcvttsh2usi32(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @llvm.x86.avx512fp16.vcvttsh2usi32(<8 x half> [[ARG0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @llvm.x86.avx512fp16.vcvttsh2usi32(<8 x half> [[ARG0]], i32 8)
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[RES1]], [[RES2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res1 = call i32 @llvm.x86.avx512fp16.vcvttsh2usi32(<8 x half> %arg0, i32 4)
+  %res2 = call i32 @llvm.x86.avx512fp16.vcvttsh2usi32(<8 x half> %arg0, i32 8)
+  %res = add i32 %res1, %res2
+  ret i32 %res
+}
+
+declare i64 @llvm.x86.avx512fp16.vcvttsh2usi64(<8 x half>, i32)
+
+define i64 @test_x86_avx512fp16_vcvttsh2usi64(<8 x half> %arg0) #0 {
+; CHECK-LABEL: define i64 @test_x86_avx512fp16_vcvttsh2usi64(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call i64 @llvm.x86.avx512fp16.vcvttsh2usi64(<8 x half> [[ARG0]], i32 4)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call i64 @llvm.x86.avx512fp16.vcvttsh2usi64(<8 x half> [[ARG0]], i32 8)
+; CHECK-NEXT:    [[RES:%.*]] = add i64 [[RES1]], [[RES2]]
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %res1 = call i64 @llvm.x86.avx512fp16.vcvttsh2usi64(<8 x half> %arg0, i32 4)
+  %res2 = call i64 @llvm.x86.avx512fp16.vcvttsh2usi64(<8 x half> %arg0, i32 8)
+  %res = add i64 %res1, %res2
+  ret i64 %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.vcvtsi2sh(<8 x half>, i32, i32)
+
+define <8 x half> @test_x86_avx512fp16_vcvtsi2sh(<8 x half> %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define <8 x half> @test_x86_avx512fp16_vcvtsi2sh(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.vcvtsi2sh(<8 x half> [[ARG0]], i32 [[ARG1]], i32 4)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.vcvtsi2sh(<8 x half> [[ARG0]], i32 [[ARG1]], i32 9)
+; CHECK-NEXT:    [[RES:%.*]] = fadd <8 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.vcvtsi2sh(<8 x half> %arg0, i32 %arg1, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.vcvtsi2sh(<8 x half> %arg0, i32 %arg1, i32 9)
+  %res = fadd <8 x half> %res1, %res2
+  ret <8 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.vcvtsi642sh(<8 x half>, i64, i32)
+
+define <8 x half> @test_x86_avx512fp16_vcvtsi642sh(<8 x half> %arg0, i64 %arg1) #0 {
+; CHECK-LABEL: define <8 x half> @test_x86_avx512fp16_vcvtsi642sh(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]], i64 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.vcvtsi642sh(<8 x half> [[ARG0]], i64 [[ARG1]], i32 4)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.vcvtsi642sh(<8 x half> [[ARG0]], i64 [[ARG1]], i32 8)
+; CHECK-NEXT:    [[RES:%.*]] = fadd <8 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.vcvtsi642sh(<8 x half> %arg0, i64 %arg1, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.vcvtsi642sh(<8 x half> %arg0, i64 %arg1, i32 8)
+  %res = fadd <8 x half> %res1, %res2
+  ret <8 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.vcvtusi2sh(<8 x half>, i32, i32)
+
+define <8 x half> @test_x86_avx512fp16_vcvtusi2sh(<8 x half> %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define <8 x half> @test_x86_avx512fp16_vcvtusi2sh(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.vcvtusi2sh(<8 x half> [[ARG0]], i32 [[ARG1]], i32 4)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.vcvtusi2sh(<8 x half> [[ARG0]], i32 [[ARG1]], i32 9)
+; CHECK-NEXT:    [[RES:%.*]] = fadd <8 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.vcvtusi2sh(<8 x half> %arg0, i32 %arg1, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.vcvtusi2sh(<8 x half> %arg0, i32 %arg1, i32 9)
+  %res = fadd <8 x half> %res1, %res2
+  ret <8 x half> %res
+}
+
+declare <8 x half> @llvm.x86.avx512fp16.vcvtusi642sh(<8 x half>, i64, i32)
+
+define <8 x half> @test_x86_avx512fp16_vcvtusi642sh(<8 x half> %arg0, i64 %arg1) #0 {
+; CHECK-LABEL: define <8 x half> @test_x86_avx512fp16_vcvtusi642sh(
+; CHECK-SAME: <8 x half> [[ARG0:%.*]], i64 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x half> @llvm.x86.avx512fp16.vcvtusi642sh(<8 x half> [[ARG0]], i64 [[ARG1]], i32 4)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR4]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x half> @llvm.x86.avx512fp16.vcvtusi642sh(<8 x half> [[ARG0]], i64 [[ARG1]], i32 9)
+; CHECK-NEXT:    [[RES:%.*]] = fadd <8 x half> [[RES1]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x half> [[RES]]
+;
+  %res1 = call <8 x half> @llvm.x86.avx512fp16.vcvtusi642sh(<8 x half> %arg0, i64 %arg1, i32 4)
+  %res2 = call <8 x half> @llvm.x86.avx512fp16.vcvtusi642sh(<8 x half> %arg0, i64 %arg1, i32 9)
+  %res = fadd <8 x half> %res1, %res2
+  ret <8 x half> %res
+}
+
+
+define <16 x half> @test_mm256_castph128_ph256_freeze(<8 x half> %a0) nounwind {
+; CHECK-LABEL: define <16 x half> @test_mm256_castph128_ph256_freeze(
+; CHECK-SAME: <8 x half> [[A0:%.*]]) #[[ATTR5:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <8 x half> poison
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x half> [[A0]], <8 x half> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x half> [[RES]]
+;
+  %a1 = freeze <8 x half> poison
+; The cold never bothered me anyway
+  %res = shufflevector <8 x half> %a0, <8 x half> %a1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x half> %res
+}
+
+
+define <32 x half> @test_mm512_castph128_ph512_freeze(<8 x half> %a0) nounwind {
+; CHECK-LABEL: define <32 x half> @test_mm512_castph128_ph512_freeze(
+; CHECK-SAME: <8 x half> [[A0:%.*]]) #[[ATTR5]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <8 x half> poison
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x half> [[A0]], <8 x half> [[A1]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %a1 = freeze <8 x half> poison
+  %res = shufflevector <8 x half> %a0, <8 x half> %a1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <32 x half> %res
+}
+
+
+define <32 x half> @test_mm512_castph256_ph512_freeze(<16 x half> %a0) nounwind {
+; CHECK-LABEL: define <32 x half> @test_mm512_castph256_ph512_freeze(
+; CHECK-SAME: <16 x half> [[A0:%.*]]) #[[ATTR5]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[A1:%.*]] = freeze <16 x half> poison
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <16 x half> [[A0]], <16 x half> [[A1]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x half> [[RES]]
+;
+  %a1 = freeze <16 x half> poison
+  %res = shufflevector <16 x half> %a0, <16 x half> %a1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x half> %res
+}
+
+attributes #0 = { sanitize_memory }

From 1e803770a5a0d7e5a12511f6d505b779a1931724 Mon Sep 17 00:00:00 2001
From: lexi-nadia <me@lexibromfield.com>
Date: Mon, 21 Apr 2025 17:34:34 -0400
Subject: [PATCH 654/710] [mlir][SMT] update Bazel configuration to match
 697aa99 (#136616)

Update Bazel configuration to match
697aa9995c24a977425e672d76a4a434384b16e3.
---
 .../mlir/python/BUILD.bazel                   | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
index b4fc353500bb8..3904da8c90c30 100644
--- a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
@@ -1112,6 +1112,53 @@ filegroup(
     ],
 )
 
+##---------------------------------------------------------------------------##
+# SMT dialect.
+##---------------------------------------------------------------------------##
+
+td_library(
+    name = "SMTOpsPyTdFiles",
+    srcs = [],
+    includes = ["../include"],
+    deps = [
+        "//mlir:SMTTdFiles",
+        "//mlir:OpBaseTdFiles",
+    ],
+)
+
+gentbl_filegroup(
+    name = "SMTOpsPyGen",
+    tbl_outs = [
+        (
+            [
+                "-gen-python-enum-bindings",
+                "-bind-dialect=smt",
+            ],
+            "mlir/dialects/_smt_enum_gen.py",
+        ),
+        (
+            [
+                "-gen-python-op-bindings",
+                "-bind-dialect=smt",
+            ],
+            "mlir/dialects/_smt_ops_gen.py",
+        ),
+    ],
+    tblgen = "//mlir:mlir-tblgen",
+    td_file = "mlir/dialects/SMTOps.td",
+    deps = [
+        ":SMTOpsPyTdFiles",
+    ],
+)
+
+filegroup(
+    name = "SMTOpsPyFiles",
+    srcs = [
+        "mlir/dialects/smt.py",
+        ":SMTOpsPyGen",
+    ],
+)
+
 ##---------------------------------------------------------------------------##
 # SparseTensor dialect.
 ##---------------------------------------------------------------------------##

From a27f7ac5c4e44b495f52e8d77b68a164dd833fda Mon Sep 17 00:00:00 2001
From: gulfemsavrun <gulfem@google.com>
Date: Mon, 21 Apr 2025 14:34:43 -0700
Subject: [PATCH 655/710] Revert "[Fuchsia] Disable PIC" (#136617)

Reverts llvm/llvm-project#136376 because it broke Fuchsia Clang
toolchain builders.
---
 clang/cmake/caches/Fuchsia-stage2.cmake | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 2b29b43f44936..99890b8246ad7 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -18,7 +18,6 @@ set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
 set(LLVM_ENABLE_LLD ON CACHE BOOL "")
 set(LLVM_ENABLE_LTO ON CACHE BOOL "")
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "")
-set(LLVM_ENABLE_PIC OFF CACHE BOOL "")
 set(LLVM_ENABLE_PLUGINS OFF CACHE BOOL "")
 set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "")
 set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")

From a945f5917cb01350487999b24b068e18111bfbfe Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Mon, 21 Apr 2025 14:38:02 -0700
Subject: [PATCH 656/710] AMDGPU: Add global-isel checks and rename
 fptrunc.v2f16.fpmath.ll (#136609)

Also remove the checks with -enable-unsafe-fp-math (already in fptrunc.f16.ll)
---
 .../CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll    | 213 ------------------
 .../AMDGPU/fptrunc.v2f16.no.fast.path.ll      | 193 ++++++++++++++++
 2 files changed, 193 insertions(+), 213 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.path.ll

diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll
deleted file mode 100644
index 9e9b9629931b9..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.fpmath.ll
+++ /dev/null
@@ -1,213 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -check-prefixes=GFX950,GFX950-SAFE-SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -check-prefixes=GFX950,GFX950-SAFE-GISEL %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 -enable-unsafe-fp-math < %s | FileCheck -check-prefixes=GFX950,GFX950-UNSAFE %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 -enable-unsafe-fp-math < %s | FileCheck -check-prefixes=GFX950,GFX950-UNSAFE %s
-
-define <2 x half> @v_test_cvt_v2f32_v2f16(<2 x float> %src) {
-; GFX950-LABEL: v_test_cvt_v2f32_v2f16:
-; GFX950:       ; %bb.0:
-; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
-; GFX950-NEXT:    s_setpc_b64 s[30:31]
-  %res = fptrunc <2 x float> %src to <2 x half>
-  ret <2 x half> %res
-}
-
-define <2 x half> @v_test_cvt_v2f64_v2f16(<2 x double> %src) {
-; GFX950-SAFE-SDAG-LABEL: v_test_cvt_v2f64_v2f16:
-; GFX950-SAFE-SDAG:       ; %bb.0:
-; GFX950-SAFE-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-SAFE-SDAG-NEXT:    s_movk_i32 s0, 0x1ff
-; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v0, v1, s0, v0
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
-; GFX950-SAFE-SDAG-NEXT:    s_movk_i32 s1, 0xffe
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_bfe_u32 v5, v1, 20, 11
-; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v0, v4, s1, v0
-; GFX950-SAFE-SDAG-NEXT:    v_sub_u32_e32 v6, 0x3f1, v5
-; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v4, 0x1000, v0
-; GFX950-SAFE-SDAG-NEXT:    v_med3_i32 v6, v6, 0, 13
-; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v7, v6, v4
-; GFX950-SAFE-SDAG-NEXT:    v_lshlrev_b32_e32 v6, v6, v7
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, v6, v4
-; GFX950-SAFE-SDAG-NEXT:    v_add_u32_e32 v5, 0xfffffc10, v5
-; GFX950-SAFE-SDAG-NEXT:    v_lshl_or_b32 v6, v5, 12, v0
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v4, v7, v4
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v5
-; GFX950-SAFE-SDAG-NEXT:    s_movk_i32 s2, 0x40f
-; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_and_b32_e32 v6, 7, v4
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v6
-; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 2, v4
-; GFX950-SAFE-SDAG-NEXT:    s_mov_b32 s3, 0x8000
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v6
-; GFX950-SAFE-SDAG-NEXT:    s_nop 1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v6, v6, v7
-; GFX950-SAFE-SDAG-NEXT:    v_add_u32_e32 v4, v4, v6
-; GFX950-SAFE-SDAG-NEXT:    v_mov_b32_e32 v6, 0x7c00
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v5
-; GFX950-SAFE-SDAG-NEXT:    v_mov_b32_e32 v7, 0x7e00
-; GFX950-SAFE-SDAG-NEXT:    s_nop 0
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX950-SAFE-SDAG-NEXT:    s_nop 1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v5
-; GFX950-SAFE-SDAG-NEXT:    s_nop 1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v0, v1, s3, v0
-; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v1, v3, s0, v2
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
-; GFX950-SAFE-SDAG-NEXT:    v_bfe_u32 v4, v3, 20, 11
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v1, v2, s1, v1
-; GFX950-SAFE-SDAG-NEXT:    v_sub_u32_e32 v5, 0x3f1, v4
-; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v2, 0x1000, v1
-; GFX950-SAFE-SDAG-NEXT:    v_med3_i32 v5, v5, 0, 13
-; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v8, v5, v2
-; GFX950-SAFE-SDAG-NEXT:    v_lshlrev_b32_e32 v5, v5, v8
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v2
-; GFX950-SAFE-SDAG-NEXT:    v_add_u32_e32 v4, 0xfffffc10, v4
-; GFX950-SAFE-SDAG-NEXT:    v_lshl_or_b32 v5, v4, 12, v1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v2, v8, v2
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v4
-; GFX950-SAFE-SDAG-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-SAFE-SDAG-NEXT:    s_nop 0
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_and_b32_e32 v5, 7, v2
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 2, v2
-; GFX950-SAFE-SDAG-NEXT:    s_nop 0
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX950-SAFE-SDAG-NEXT:    s_nop 1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_or_b32_e32 v5, v5, v8
-; GFX950-SAFE-SDAG-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v4
-; GFX950-SAFE-SDAG-NEXT:    s_nop 1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX950-SAFE-SDAG-NEXT:    s_nop 1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v1, v6, v7, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v4
-; GFX950-SAFE-SDAG-NEXT:    s_nop 1
-; GFX950-SAFE-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX950-SAFE-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; GFX950-SAFE-SDAG-NEXT:    v_and_or_b32 v1, v2, s3, v1
-; GFX950-SAFE-SDAG-NEXT:    v_perm_b32 v0, v1, v0, s0
-; GFX950-SAFE-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX950-SAFE-GISEL-LABEL: v_test_cvt_v2f64_v2f16:
-; GFX950-SAFE-GISEL:       ; %bb.0:
-; GFX950-SAFE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-SAFE-GISEL-NEXT:    s_movk_i32 s0, 0x1ff
-; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v0, v1, s0, v0
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
-; GFX950-SAFE-GISEL-NEXT:    s_movk_i32 s1, 0xffe
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_bfe_u32 v5, v1, 20, 11
-; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v0, v4, s1, v0
-; GFX950-SAFE-GISEL-NEXT:    v_sub_u32_e32 v6, 0x3f1, v5
-; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v4, 0x1000, v0
-; GFX950-SAFE-GISEL-NEXT:    v_med3_i32 v6, v6, 0, 13
-; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v7, v6, v4
-; GFX950-SAFE-GISEL-NEXT:    v_lshlrev_b32_e32 v6, v6, v7
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, v6, v4
-; GFX950-SAFE-GISEL-NEXT:    v_add_u32_e32 v5, 0xfffffc10, v5
-; GFX950-SAFE-GISEL-NEXT:    v_lshl_or_b32 v6, v5, 12, v0
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v4, v7, v4
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v5
-; GFX950-SAFE-GISEL-NEXT:    s_movk_i32 s2, 0x40f
-; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_and_b32_e32 v6, 7, v4
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v6
-; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 2, v4
-; GFX950-SAFE-GISEL-NEXT:    s_mov_b32 s3, 0x8000
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v6
-; GFX950-SAFE-GISEL-NEXT:    s_nop 1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v6, v6, v7
-; GFX950-SAFE-GISEL-NEXT:    v_add_u32_e32 v4, v4, v6
-; GFX950-SAFE-GISEL-NEXT:    v_mov_b32_e32 v6, 0x7c00
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v5
-; GFX950-SAFE-GISEL-NEXT:    v_mov_b32_e32 v7, 0x7e00
-; GFX950-SAFE-GISEL-NEXT:    s_nop 0
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX950-SAFE-GISEL-NEXT:    s_nop 1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v5
-; GFX950-SAFE-GISEL-NEXT:    s_nop 1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v0, v1, s3, v0
-; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v1, v3, s0, v2
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
-; GFX950-SAFE-GISEL-NEXT:    v_bfe_u32 v4, v3, 20, 11
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v1, v2, s1, v1
-; GFX950-SAFE-GISEL-NEXT:    v_sub_u32_e32 v5, 0x3f1, v4
-; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v2, 0x1000, v1
-; GFX950-SAFE-GISEL-NEXT:    v_med3_i32 v5, v5, 0, 13
-; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v8, v5, v2
-; GFX950-SAFE-GISEL-NEXT:    v_lshlrev_b32_e32 v5, v5, v8
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v2
-; GFX950-SAFE-GISEL-NEXT:    v_add_u32_e32 v4, 0xfffffc10, v4
-; GFX950-SAFE-GISEL-NEXT:    v_lshl_or_b32 v5, v4, 12, v1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v2, v8, v2
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v4
-; GFX950-SAFE-GISEL-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-SAFE-GISEL-NEXT:    s_nop 0
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_and_b32_e32 v5, 7, v2
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 2, v2
-; GFX950-SAFE-GISEL-NEXT:    s_nop 0
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX950-SAFE-GISEL-NEXT:    s_nop 1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_or_b32_e32 v5, v5, v8
-; GFX950-SAFE-GISEL-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v4
-; GFX950-SAFE-GISEL-NEXT:    s_nop 1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX950-SAFE-GISEL-NEXT:    s_nop 1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v1, v6, v7, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v4
-; GFX950-SAFE-GISEL-NEXT:    s_nop 1
-; GFX950-SAFE-GISEL-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX950-SAFE-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; GFX950-SAFE-GISEL-NEXT:    v_and_or_b32 v1, v2, s3, v1
-; GFX950-SAFE-GISEL-NEXT:    v_perm_b32 v0, v1, v0, s0
-; GFX950-SAFE-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX950-UNSAFE-LABEL: v_test_cvt_v2f64_v2f16:
-; GFX950-UNSAFE:       ; %bb.0:
-; GFX950-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-UNSAFE-NEXT:    v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX950-UNSAFE-NEXT:    v_cvt_f32_f64_e32 v1, v[2:3]
-; GFX950-UNSAFE-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX950-UNSAFE-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX950-UNSAFE-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-UNSAFE-NEXT:    v_perm_b32 v0, v1, v0, s0
-; GFX950-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-  %res = fptrunc <2 x double> %src to <2 x half>
-  ret <2 x half> %res
-}
-
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.path.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.path.ll
new file mode 100644
index 0000000000000..5a5e39489d888
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.path.ll
@@ -0,0 +1,193 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -check-prefixes=GFX950,GFX950-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 < %s | FileCheck -check-prefixes=GFX950,GFX950-GISEL %s
+
+define <2 x half> @v_test_cvt_v2f32_v2f16(<2 x float> %src) {
+; GFX950-LABEL: v_test_cvt_v2f32_v2f16:
+; GFX950:       ; %bb.0:
+; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-NEXT:    s_setpc_b64 s[30:31]
+  %res = fptrunc <2 x float> %src to <2 x half>
+  ret <2 x half> %res
+}
+
+define <2 x half> @v_test_cvt_v2f64_v2f16(<2 x double> %src) {
+; GFX950-SDAG-LABEL: v_test_cvt_v2f64_v2f16:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT:    s_movk_i32 s0, 0x1ff
+; GFX950-SDAG-NEXT:    v_and_or_b32 v0, v1, s0, v0
+; GFX950-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX950-SDAG-NEXT:    s_movk_i32 s1, 0xffe
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-SDAG-NEXT:    v_bfe_u32 v5, v1, 20, 11
+; GFX950-SDAG-NEXT:    v_and_or_b32 v0, v4, s1, v0
+; GFX950-SDAG-NEXT:    v_sub_u32_e32 v6, 0x3f1, v5
+; GFX950-SDAG-NEXT:    v_or_b32_e32 v4, 0x1000, v0
+; GFX950-SDAG-NEXT:    v_med3_i32 v6, v6, 0, 13
+; GFX950-SDAG-NEXT:    v_lshrrev_b32_e32 v7, v6, v4
+; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v6, v6, v7
+; GFX950-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, v6, v4
+; GFX950-SDAG-NEXT:    v_add_u32_e32 v5, 0xfffffc10, v5
+; GFX950-SDAG-NEXT:    v_lshl_or_b32 v6, v5, 12, v0
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX950-SDAG-NEXT:    v_or_b32_e32 v4, v7, v4
+; GFX950-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v5
+; GFX950-SDAG-NEXT:    s_movk_i32 s2, 0x40f
+; GFX950-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX950-SDAG-NEXT:    v_and_b32_e32 v6, 7, v4
+; GFX950-SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v6
+; GFX950-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 2, v4
+; GFX950-SDAG-NEXT:    s_mov_b32 s3, 0x8000
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX950-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v6
+; GFX950-SDAG-NEXT:    s_nop 1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX950-SDAG-NEXT:    v_or_b32_e32 v6, v6, v7
+; GFX950-SDAG-NEXT:    v_add_u32_e32 v4, v4, v6
+; GFX950-SDAG-NEXT:    v_mov_b32_e32 v6, 0x7c00
+; GFX950-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v5
+; GFX950-SDAG-NEXT:    v_mov_b32_e32 v7, 0x7e00
+; GFX950-SDAG-NEXT:    s_nop 0
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX950-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-SDAG-NEXT:    s_nop 1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX950-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v5
+; GFX950-SDAG-NEXT:    s_nop 1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX950-SDAG-NEXT:    v_and_or_b32 v0, v1, s3, v0
+; GFX950-SDAG-NEXT:    v_and_or_b32 v1, v3, s0, v2
+; GFX950-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX950-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
+; GFX950-SDAG-NEXT:    v_bfe_u32 v4, v3, 20, 11
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX950-SDAG-NEXT:    v_and_or_b32 v1, v2, s1, v1
+; GFX950-SDAG-NEXT:    v_sub_u32_e32 v5, 0x3f1, v4
+; GFX950-SDAG-NEXT:    v_or_b32_e32 v2, 0x1000, v1
+; GFX950-SDAG-NEXT:    v_med3_i32 v5, v5, 0, 13
+; GFX950-SDAG-NEXT:    v_lshrrev_b32_e32 v8, v5, v2
+; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v5, v5, v8
+; GFX950-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, v5, v2
+; GFX950-SDAG-NEXT:    v_add_u32_e32 v4, 0xfffffc10, v4
+; GFX950-SDAG-NEXT:    v_lshl_or_b32 v5, v4, 12, v1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-SDAG-NEXT:    v_or_b32_e32 v2, v8, v2
+; GFX950-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v4
+; GFX950-SDAG-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX950-SDAG-NEXT:    s_nop 0
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX950-SDAG-NEXT:    v_and_b32_e32 v5, 7, v2
+; GFX950-SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 5, v5
+; GFX950-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 2, v2
+; GFX950-SDAG-NEXT:    s_nop 0
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX950-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v5
+; GFX950-SDAG-NEXT:    s_nop 1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX950-SDAG-NEXT:    v_or_b32_e32 v5, v5, v8
+; GFX950-SDAG-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX950-SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 31, v4
+; GFX950-SDAG-NEXT:    s_nop 1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX950-SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX950-SDAG-NEXT:    s_nop 1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v1, v6, v7, vcc
+; GFX950-SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v4
+; GFX950-SDAG-NEXT:    s_nop 1
+; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX950-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX950-SDAG-NEXT:    v_and_or_b32 v1, v2, s3, v1
+; GFX950-SDAG-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: v_test_cvt_v2f64_v2f16:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v7, 0x1ff
+; GFX950-GISEL-NEXT:    v_and_or_b32 v0, v1, v7, v0
+; GFX950-GISEL-NEXT:    v_bfe_u32 v4, v1, 20, 11
+; GFX950-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-GISEL-NEXT:    v_add_u32_e32 v4, 0xfffffc10, v4
+; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v6, 0xffe
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-GISEL-NEXT:    v_and_or_b32 v0, v5, v6, v0
+; GFX950-GISEL-NEXT:    v_sub_u32_e32 v10, 1, v4
+; GFX950-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-GISEL-NEXT:    v_lshl_or_b32 v9, v4, 12, v0
+; GFX950-GISEL-NEXT:    v_med3_i32 v10, v10, 0, 13
+; GFX950-GISEL-NEXT:    v_or_b32_e32 v0, 0x1000, v0
+; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v11, v10, v0
+; GFX950-GISEL-NEXT:    v_lshlrev_b32_e32 v10, v10, v11
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX950-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, v10, v0
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v8, 0x7c00
+; GFX950-GISEL-NEXT:    v_lshl_or_b32 v5, v5, 9, v8
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-GISEL-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX950-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v4
+; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX950-GISEL-NEXT:    v_and_or_b32 v2, v3, v7, v2
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v9, 7, v0
+; GFX950-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v9
+; GFX950-GISEL-NEXT:    v_cmp_lt_i32_e64 s[0:1], 5, v9
+; GFX950-GISEL-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v0, 2, v0
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[0:1]
+; GFX950-GISEL-NEXT:    v_add_u32_e32 v0, v0, v9
+; GFX950-GISEL-NEXT:    v_cmp_lt_i32_e32 vcc, 30, v4
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v9, 0x40f
+; GFX950-GISEL-NEXT:    s_nop 0
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX950-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v9
+; GFX950-GISEL-NEXT:    v_mov_b32_e32 v4, 0x8000
+; GFX950-GISEL-NEXT:    s_nop 0
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX950-GISEL-NEXT:    v_and_or_b32 v0, v1, v4, v0
+; GFX950-GISEL-NEXT:    v_bfe_u32 v1, v3, 20, 11
+; GFX950-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GFX950-GISEL-NEXT:    v_add_u32_e32 v1, 0xfffffc10, v1
+; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-GISEL-NEXT:    v_and_or_b32 v2, v5, v6, v2
+; GFX950-GISEL-NEXT:    v_sub_u32_e32 v7, 1, v1
+; GFX950-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GFX950-GISEL-NEXT:    v_lshl_or_b32 v6, v1, 12, v2
+; GFX950-GISEL-NEXT:    v_med3_i32 v7, v7, 0, 13
+; GFX950-GISEL-NEXT:    v_or_b32_e32 v2, 0x1000, v2
+; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v10, v7, v2
+; GFX950-GISEL-NEXT:    v_lshlrev_b32_e32 v7, v7, v10
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX950-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, v7, v2
+; GFX950-GISEL-NEXT:    v_lshl_or_b32 v5, v5, 9, v8
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-GISEL-NEXT:    v_or_b32_e32 v2, v10, v2
+; GFX950-GISEL-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v1
+; GFX950-GISEL-NEXT:    s_nop 1
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX950-GISEL-NEXT:    v_and_b32_e32 v6, 7, v2
+; GFX950-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v6
+; GFX950-GISEL-NEXT:    v_cmp_lt_i32_e64 s[0:1], 5, v6
+; GFX950-GISEL-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 2, v2
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[0:1]
+; GFX950-GISEL-NEXT:    v_add_u32_e32 v2, v2, v6
+; GFX950-GISEL-NEXT:    v_cmp_lt_i32_e32 vcc, 30, v1
+; GFX950-GISEL-NEXT:    s_nop 1
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX950-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v9
+; GFX950-GISEL-NEXT:    s_nop 1
+; GFX950-GISEL-NEXT:    v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX950-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX950-GISEL-NEXT:    v_and_or_b32 v1, v2, v4, v1
+; GFX950-GISEL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %res = fptrunc <2 x double> %src to <2 x half>
+  ret <2 x half> %res
+}

From 3ceb3d96ff166305f015f1f00b2740e148347532 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Mon, 21 Apr 2025 14:48:47 -0700
Subject: [PATCH 657/710] [flang][openacc] Make async clause behavior
 homogenous (#136610)

The `async` clause was not handed in a similar way on `serial`,
`parallel` and `kernels` directive. This patches updates the `ACC.td`
file and the flang semantic to make it homogenous.
---
 flang/lib/Semantics/check-acc-structure.cpp   |  7 ++-
 flang/test/Semantics/OpenACC/acc-kernels.f90  | 11 +++++
 flang/test/Semantics/OpenACC/acc-parallel.f90 | 11 +++++
 flang/test/Semantics/OpenACC/acc-serial.f90   |  9 +++-
 llvm/include/llvm/Frontend/OpenACC/ACC.td     | 44 +++++++++----------
 5 files changed, 56 insertions(+), 26 deletions(-)

diff --git a/flang/lib/Semantics/check-acc-structure.cpp b/flang/lib/Semantics/check-acc-structure.cpp
index 7afc285e7b9ae..9cbea9742a6a2 100644
--- a/flang/lib/Semantics/check-acc-structure.cpp
+++ b/flang/lib/Semantics/check-acc-structure.cpp
@@ -374,7 +374,6 @@ void AccStructureChecker::Leave(const parser::OpenACCCacheConstruct &x) {
 
 // Clause checkers
 CHECK_SIMPLE_CLAUSE(Auto, ACCC_auto)
-CHECK_SIMPLE_CLAUSE(Async, ACCC_async)
 CHECK_SIMPLE_CLAUSE(Attach, ACCC_attach)
 CHECK_SIMPLE_CLAUSE(Bind, ACCC_bind)
 CHECK_SIMPLE_CLAUSE(Capture, ACCC_capture)
@@ -444,6 +443,12 @@ void AccStructureChecker::CheckMultipleOccurrenceInDeclare(
   CheckMultipleOccurrenceInDeclare(objectList, clause);
 }
 
+void AccStructureChecker::Enter(const parser::AccClause::Async &c) {
+  llvm::acc::Clause crtClause = llvm::acc::Clause::ACCC_async;
+  CheckAllowed(crtClause);
+  CheckAllowedOncePerGroup(crtClause, llvm::acc::Clause::ACCC_device_type);
+}
+
 void AccStructureChecker::Enter(const parser::AccClause::Create &c) {
   CheckAllowed(llvm::acc::Clause::ACCC_create);
   const auto &modifierClause{c.v};
diff --git a/flang/test/Semantics/OpenACC/acc-kernels.f90 b/flang/test/Semantics/OpenACC/acc-kernels.f90
index 33e253ca8d874..44e532a9012bf 100644
--- a/flang/test/Semantics/OpenACC/acc-kernels.f90
+++ b/flang/test/Semantics/OpenACC/acc-kernels.f90
@@ -37,6 +37,17 @@ program openacc_kernels_validity
   !$acc kernels async(async1)
   !$acc end kernels
 
+  !ERROR: At most one ASYNC clause can appear on the KERNELS directive or in group separated by the DEVICE_TYPE clause
+  !$acc kernels async(async1) async(2)
+  !$acc end kernels
+
+  !$acc kernels async(async1) device_type(multicore) async(2) ! ok
+  !$acc end kernels
+
+  !ERROR: At most one ASYNC clause can appear on the KERNELS directive or in group separated by the DEVICE_TYPE clause
+  !$acc kernels async(async1) device_type(multicore) async(2) async(3)
+  !$acc end kernels
+
   !$acc kernels wait(wait1)
   !$acc end kernels
 
diff --git a/flang/test/Semantics/OpenACC/acc-parallel.f90 b/flang/test/Semantics/OpenACC/acc-parallel.f90
index 3f17d8fc862a6..b9d989ec6fced 100644
--- a/flang/test/Semantics/OpenACC/acc-parallel.f90
+++ b/flang/test/Semantics/OpenACC/acc-parallel.f90
@@ -34,6 +34,17 @@ program openacc_parallel_validity
   !$acc parallel async(1)
   !$acc end parallel
 
+  !ERROR: At most one ASYNC clause can appear on the PARALLEL directive or in group separated by the DEVICE_TYPE clause
+  !$acc parallel async(1) async(2)
+  !$acc end parallel
+
+  !$acc parallel async(1) device_type(nvidia) async(3)
+  !$acc end parallel
+
+  !ERROR: At most one ASYNC clause can appear on the PARALLEL directive or in group separated by the DEVICE_TYPE clause
+  !$acc parallel async(1) device_type(nvidia) async(2) async(3)
+  !$acc end parallel
+
   !$acc parallel async(async1)
   !$acc end parallel
 
diff --git a/flang/test/Semantics/OpenACC/acc-serial.f90 b/flang/test/Semantics/OpenACC/acc-serial.f90
index a23daecce8dd3..1f22003ed6b11 100644
--- a/flang/test/Semantics/OpenACC/acc-serial.f90
+++ b/flang/test/Semantics/OpenACC/acc-serial.f90
@@ -52,10 +52,17 @@ program openacc_serial_validity
   !$acc serial async(1)
   !$acc end serial
 
-  !ERROR: At most one ASYNC clause can appear on the SERIAL directive
+  !ERROR: At most one ASYNC clause can appear on the SERIAL directive or in group separated by the DEVICE_TYPE clause
   !$acc serial async(1) async(2)
   !$acc end serial
 
+  !ERROR: At most one ASYNC clause can appear on the SERIAL directive or in group separated by the DEVICE_TYPE clause
+  !$acc serial async(1) device_type(nvidia) async(2) async(4)
+  !$acc end serial
+
+  !$acc serial async(1) device_type(nvidia) async(2)
+  !$acc end serial
+
   !$acc serial async(async1)
   !$acc end serial
 
diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index 6f6539c1ffebc..7dc97e6c89bee 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -325,7 +325,8 @@ def ACC_Declare : Directive<"declare"> {
 
 // 2.5.3
 def ACC_Kernels : Directive<"kernels"> {
-  let allowedClauses = [VersionedClause<ACCC_Attach>,
+  let allowedClauses = [VersionedClause<ACCC_Async>,
+                        VersionedClause<ACCC_Attach>,
                         VersionedClause<ACCC_Copy>,
                         VersionedClause<ACCC_Copyin>,
                         VersionedClause<ACCC_Copyout>,
@@ -338,8 +339,7 @@ def ACC_Kernels : Directive<"kernels"> {
                         VersionedClause<ACCC_DevicePtr>,
                         VersionedClause<ACCC_VectorLength>,
                         VersionedClause<ACCC_Wait>];
-  let allowedOnceClauses = [VersionedClause<ACCC_Async>,
-                            VersionedClause<ACCC_Default>,
+  let allowedOnceClauses = [VersionedClause<ACCC_Default>,
                             VersionedClause<ACCC_If>,
                             VersionedClause<ACCC_Self>];
   let association = AS_Block;
@@ -380,27 +380,23 @@ def ACC_Parallel : Directive<"parallel"> {
 def ACC_Serial : Directive<"serial"> {
   // Spec line 950-951: clause is as for the parallel construct except that the
   // num_gangs, num_workers, and vector_length clauses are not permitted.
-  let allowedClauses = [
-    VersionedClause<ACCC_Attach>,
-    VersionedClause<ACCC_Copy>,
-    VersionedClause<ACCC_Copyin>,
-    VersionedClause<ACCC_Copyout>,
-    VersionedClause<ACCC_Create>,
-    VersionedClause<ACCC_DevicePtr>,
-    VersionedClause<ACCC_DeviceType>,
-    VersionedClause<ACCC_NoCreate>,
-    VersionedClause<ACCC_Present>,
-    VersionedClause<ACCC_Private>,
-    VersionedClause<ACCC_FirstPrivate>,
-    VersionedClause<ACCC_Reduction>,
-    VersionedClause<ACCC_Wait>
-  ];
-  let allowedOnceClauses = [
-    VersionedClause<ACCC_Async>,
-    VersionedClause<ACCC_Default>,
-    VersionedClause<ACCC_If>,
-    VersionedClause<ACCC_Self>
-  ];
+  let allowedClauses = [VersionedClause<ACCC_Async>,
+                        VersionedClause<ACCC_Attach>,
+                        VersionedClause<ACCC_Copy>,
+                        VersionedClause<ACCC_Copyin>,
+                        VersionedClause<ACCC_Copyout>,
+                        VersionedClause<ACCC_Create>,
+                        VersionedClause<ACCC_DevicePtr>,
+                        VersionedClause<ACCC_DeviceType>,
+                        VersionedClause<ACCC_NoCreate>,
+                        VersionedClause<ACCC_Present>,
+                        VersionedClause<ACCC_Private>,
+                        VersionedClause<ACCC_FirstPrivate>,
+                        VersionedClause<ACCC_Reduction>,
+                        VersionedClause<ACCC_Wait>];
+  let allowedOnceClauses = [VersionedClause<ACCC_Default>,
+                            VersionedClause<ACCC_If>,
+                            VersionedClause<ACCC_Self>];
   let association = AS_Block;
   let category = CA_Executable;
 }

From 515564aa6ebeadc06eea03ff92aeda64727f1da5 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 21 Apr 2025 14:55:08 -0700
Subject: [PATCH 658/710] [AMDGPU] Partially revert my llvm::less_second patch
 (#136615)

This patch partially reverts:

  commit 5e1b0f97735083b6762834b83fdbb35e76002e03
  Author: Kazu Hirata <kazu@google.com>
  Date:   Fri Apr 18 10:05:55 2025 -0700

to fix:

  LLVM :: CodeGen/AMDGPU/sched-group-barrier-pipeline-solver.mir
  LLVM :: CodeGen/AMDGPU/sched-group-barrier-pre-RA.mir

under LLVM_ENABLE_EXPENSIVE_CHECKS.
---
 llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 815715604bc96..87c1d2586cce5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -589,7 +589,7 @@ void PipelineSolver::populateReadyList(
   }
 
   if (UseCostHeur)
-    llvm::sort(ReadyList, llvm::less_second());
+    std::sort(ReadyList.begin(), ReadyList.end(), llvm::less_second());
 
   assert(ReadyList.size() == CurrSU.second.size());
 }

From de0153da325b21340747d8a064906a8c1c2f95ea Mon Sep 17 00:00:00 2001
From: Nick Sarnie <nick.sarnie@intel.com>
Date: Mon, 21 Apr 2025 18:09:49 -0400
Subject: [PATCH 659/710] [clang][ARM][AArch64] Define intrinsics guarded by
 __has_builtin on all platforms  (#128222)

Instead of defining ARM ACLE intrinsics only on MSVC and guarding
wrapper functions in headers with `__has_builtin`, universally define
the intrinsics as target header builtins.

---------

Signed-off-by: Sarnie, Nick <nick.sarnie@intel.com>
---
 clang/include/clang/Basic/BuiltinHeaders.def  |  1 +
 clang/include/clang/Basic/BuiltinsAArch64.def | 18 +++---
 clang/include/clang/Basic/BuiltinsARM.def     | 17 +++---
 clang/lib/Headers/arm_acle.h                  | 47 +++------------
 clang/lib/Sema/SemaARM.cpp                    |  6 ++
 ...rmer-microsoft-intrinsics-header-warning.c | 50 ++++++++++++++++
 .../CodeGen/arm-former-microsoft-intrinsics.c | 53 +++++++++++++++++
 clang/test/CodeGen/arm-microsoft-intrinsics.c | 21 -------
 ...rmer-microsoft-intrinsics-header-warning.c | 53 +++++++++++++++++
 .../arm64-former-microsoft-intrinsics.c       | 59 +++++++++++++++++++
 .../test/CodeGen/arm64-microsoft-intrinsics.c | 56 ------------------
 clang/test/CodeGen/builtins-arm-microsoft.c   | 12 ++--
 clang/test/Headers/Inputs/include/stdint.h    | 12 ++++
 clang/test/Headers/ms-intrin.cpp              |  4 ++
 14 files changed, 269 insertions(+), 140 deletions(-)
 create mode 100644 clang/test/CodeGen/arm-former-microsoft-intrinsics-header-warning.c
 create mode 100644 clang/test/CodeGen/arm-former-microsoft-intrinsics.c
 create mode 100644 clang/test/CodeGen/arm64-former-microsoft-intrinsics-header-warning.c
 create mode 100644 clang/test/CodeGen/arm64-former-microsoft-intrinsics.c

diff --git a/clang/include/clang/Basic/BuiltinHeaders.def b/clang/include/clang/Basic/BuiltinHeaders.def
index 8e4a2f9bee9aa..22668ec7a3396 100644
--- a/clang/include/clang/Basic/BuiltinHeaders.def
+++ b/clang/include/clang/Basic/BuiltinHeaders.def
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 HEADER(NO_HEADER, nullptr)
+HEADER(ARMACLE_H, "arm_acle.h")
 HEADER(BLOCKS_H, "Blocks.h")
 HEADER(COMPLEX_H, "complex.h")
 HEADER(CTYPE_H, "ctype.h")
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 7933686ffe84e..19fa7215b5000 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -50,6 +50,11 @@ BUILTIN(__builtin_arm_wfi, "v", "")
 BUILTIN(__builtin_arm_sev, "v", "")
 BUILTIN(__builtin_arm_sevl, "v", "")
 BUILTIN(__builtin_arm_chkfeat, "WUiWUi", "")
+TARGET_HEADER_BUILTIN(__yield, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfe,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfi,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sev,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sevl,  "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Like __builtin_trap but provide an 16-bit immediate reason code (which goes into `brk #N`).
 BUILTIN(__builtin_arm_trap, "vUIs", "nr")
@@ -82,6 +87,9 @@ TARGET_BUILTIN(__builtin_arm_mops_memset_tag, "v*v*iz", "", "mte,mops")
 BUILTIN(__builtin_arm_dmb, "vUi", "nc")
 BUILTIN(__builtin_arm_dsb, "vUi", "nc")
 BUILTIN(__builtin_arm_isb, "vUi", "nc")
+TARGET_HEADER_BUILTIN(__dmb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__dsb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__isb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
 
 TARGET_BUILTIN(__builtin_arm_jcvt, "Zid", "nc", "v8.3a")
 
@@ -98,16 +106,6 @@ BUILTIN(__builtin_arm_wsr64, "vcC*WUi", "nc")
 TARGET_BUILTIN(__builtin_arm_wsr128, "vcC*LLLUi", "nc", "d128")
 BUILTIN(__builtin_arm_wsrp, "vcC*vC*", "nc")
 
-// MSVC
-LANGBUILTIN(__dmb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__dsb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__isb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__yield, "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfe,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfi,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__sev,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__sevl,  "v", "",   ALL_MS_LANGUAGES)
-
 // Misc
 BUILTIN(__builtin_sponentry, "v*", "c")
 
diff --git a/clang/include/clang/Basic/BuiltinsARM.def b/clang/include/clang/Basic/BuiltinsARM.def
index cbab87cecbc7d..2592e25e95c37 100644
--- a/clang/include/clang/Basic/BuiltinsARM.def
+++ b/clang/include/clang/Basic/BuiltinsARM.def
@@ -186,11 +186,19 @@ BUILTIN(__builtin_arm_wfi, "v", "")
 BUILTIN(__builtin_arm_sev, "v", "")
 BUILTIN(__builtin_arm_sevl, "v", "")
 BUILTIN(__builtin_arm_dbg, "vUi", "")
+TARGET_HEADER_BUILTIN(__yield, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfe, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfi, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sev, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sevl, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Data barrier
 BUILTIN(__builtin_arm_dmb, "vUi", "nc")
 BUILTIN(__builtin_arm_dsb, "vUi", "nc")
 BUILTIN(__builtin_arm_isb, "vUi", "nc")
+TARGET_HEADER_BUILTIN(__dmb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__dsb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__isb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Prefetch
 BUILTIN(__builtin_arm_prefetch, "vvC*UiUi", "nc")
@@ -209,15 +217,6 @@ BUILTIN(__builtin_sponentry, "v*", "c")
 // MSVC
 LANGBUILTIN(__emit, "vIUiC", "", ALL_MS_LANGUAGES)
 
-LANGBUILTIN(__yield, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfe, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfi, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__sev, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__sevl, "v", "", ALL_MS_LANGUAGES)
-
-LANGBUILTIN(__dmb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__dsb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__isb, "vUi", "nc", ALL_MS_LANGUAGES)
 LANGBUILTIN(__ldrexd, "WiWiCD*", "", ALL_MS_LANGUAGES)
 LANGBUILTIN(_MoveFromCoprocessor, "UiIUiIUiIUiIUiIUi", "", ALL_MS_LANGUAGES)
 LANGBUILTIN(_MoveFromCoprocessor2, "UiIUiIUiIUiIUiIUi", "", ALL_MS_LANGUAGES)
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index b1dc90f84ad36..89db5dc2d7aaa 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -29,47 +29,16 @@ extern "C" {
 
 /* 7 SYNCHRONIZATION, BARRIER AND HINT INTRINSICS */
 /* 7.3 Memory barriers */
-#if !__has_builtin(__dmb)
-#define __dmb(i) __builtin_arm_dmb(i)
-#endif
-#if !__has_builtin(__dsb)
-#define __dsb(i) __builtin_arm_dsb(i)
-#endif
-#if !__has_builtin(__isb)
-#define __isb(i) __builtin_arm_isb(i)
-#endif
+void __dmb(unsigned int);
+void __dsb(unsigned int);
+void __isb(unsigned int);
 
 /* 7.4 Hints */
-
-#if !__has_builtin(__wfi)
-static __inline__ void __attribute__((__always_inline__, __nodebug__)) __wfi(void) {
-  __builtin_arm_wfi();
-}
-#endif
-
-#if !__has_builtin(__wfe)
-static __inline__ void __attribute__((__always_inline__, __nodebug__)) __wfe(void) {
-  __builtin_arm_wfe();
-}
-#endif
-
-#if !__has_builtin(__sev)
-static __inline__ void __attribute__((__always_inline__, __nodebug__)) __sev(void) {
-  __builtin_arm_sev();
-}
-#endif
-
-#if !__has_builtin(__sevl)
-static __inline__ void __attribute__((__always_inline__, __nodebug__)) __sevl(void) {
-  __builtin_arm_sevl();
-}
-#endif
-
-#if !__has_builtin(__yield)
-static __inline__ void __attribute__((__always_inline__, __nodebug__)) __yield(void) {
-  __builtin_arm_yield();
-}
-#endif
+void __wfi(void);
+void __wfe(void);
+void __sev(void);
+void __sevl(void);
+void __yield(void);
 
 #if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
 #define __dbg(t) __builtin_arm_dbg(t)
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index b5abf244132d8..136a3e277dea8 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -1010,8 +1010,11 @@ bool SemaARM::CheckARMBuiltinFunctionCall(const TargetInfo &TI,
   case ARM::BI__builtin_arm_vcvtr_d:
     return SemaRef.BuiltinConstantArgRange(TheCall, 1, 0, 1);
   case ARM::BI__builtin_arm_dmb:
+  case ARM::BI__dmb:
   case ARM::BI__builtin_arm_dsb:
+  case ARM::BI__dsb:
   case ARM::BI__builtin_arm_isb:
+  case ARM::BI__isb:
   case ARM::BI__builtin_arm_dbg:
     return SemaRef.BuiltinConstantArgRange(TheCall, 0, 0, 15);
   case ARM::BI__builtin_arm_cdp:
@@ -1108,8 +1111,11 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
   switch (BuiltinID) {
   default: return false;
   case AArch64::BI__builtin_arm_dmb:
+  case AArch64::BI__dmb:
   case AArch64::BI__builtin_arm_dsb:
+  case AArch64::BI__dsb:
   case AArch64::BI__builtin_arm_isb:
+  case AArch64::BI__isb:
     l = 0;
     u = 15;
     break;
diff --git a/clang/test/CodeGen/arm-former-microsoft-intrinsics-header-warning.c b/clang/test/CodeGen/arm-former-microsoft-intrinsics-header-warning.c
new file mode 100644
index 0000000000000..8edcbbeb0375d
--- /dev/null
+++ b/clang/test/CodeGen/arm-former-microsoft-intrinsics-header-warning.c
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -triple thumbv7-windows -fms-compatibility -Wno-everything -Wimplicit-function-declaration -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple armv7-eabi -Wno-everything -Wimplicit-function-declaration -fsyntax-only -verify %s
+
+void check__dmb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __dmb(0);
+}
+
+void check__dsb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __dsb(0);
+}
+
+void check__isb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __isb(0);
+}
+
+void check__yield(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __yield();
+}
+
+void check__wfe(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __wfe();
+}
+
+void check__wfi(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __wfi();
+}
+
+void check__sev(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __sev();
+}
+
+void check__sevl(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __sevl();
+}
diff --git a/clang/test/CodeGen/arm-former-microsoft-intrinsics.c b/clang/test/CodeGen/arm-former-microsoft-intrinsics.c
new file mode 100644
index 0000000000000..8a02602dc9876
--- /dev/null
+++ b/clang/test/CodeGen/arm-former-microsoft-intrinsics.c
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -triple thumbv7-windows -fms-compatibility -emit-llvm -o - %s \
+// RUN:    | FileCheck %s
+
+// RUN: %clang_cc1 -triple armv7-eabi -Werror -emit-llvm -o - %s \
+// RUN:    | FileCheck %s
+#include <arm_acle.h>
+void check__dmb(void) {
+  __dmb(0);
+}
+
+// CHECK: @llvm.arm.dmb(i32 0)
+
+void check__dsb(void) {
+  __dsb(0);
+}
+
+// CHECK: @llvm.arm.dsb(i32 0)
+
+void check__isb(void) {
+  __isb(0);
+}
+
+// CHECK: @llvm.arm.isb(i32 0)
+
+void check__yield(void) {
+  __yield();
+}
+
+// CHECK: @llvm.arm.hint(i32 1)
+
+void check__wfe(void) {
+  __wfe();
+}
+
+// CHECK: @llvm.arm.hint(i32 2)
+
+void check__wfi(void) {
+  __wfi();
+}
+
+// CHECK: @llvm.arm.hint(i32 3)
+
+void check__sev(void) {
+  __sev();
+}
+
+// CHECK: @llvm.arm.hint(i32 4)
+
+void check__sevl(void) {
+  __sevl();
+}
+
+// CHECK: @llvm.arm.hint(i32 5)
diff --git a/clang/test/CodeGen/arm-microsoft-intrinsics.c b/clang/test/CodeGen/arm-microsoft-intrinsics.c
index 6793b3879eede..a5f3e2c719227 100644
--- a/clang/test/CodeGen/arm-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm-microsoft-intrinsics.c
@@ -4,27 +4,6 @@
 // RUN: not %clang_cc1 -triple armv7-eabi -Werror -S -o /dev/null %s 2>&1 \
 // RUN:    | FileCheck %s -check-prefix CHECK-EABI
 
-void check__dmb(void) {
-  __dmb(0);
-}
-
-// CHECK-MSVC: @llvm.arm.dmb(i32 0)
-// CHECK-EABI: error: call to undeclared function '__dmb'
-
-void check__dsb(void) {
-  __dsb(0);
-}
-
-// CHECK-MSVC: @llvm.arm.dsb(i32 0)
-// CHECK-EABI: error: call to undeclared function '__dsb'
-
-void check__isb(void) {
-  __isb(0);
-}
-
-// CHECK-MSVC: @llvm.arm.isb(i32 0)
-// CHECK-EABI: error: call to undeclared function '__isb'
-
 __INT64_TYPE__ check__ldrexd(void) {
   __INT64_TYPE__ i64;
   return __ldrexd(&i64);
diff --git a/clang/test/CodeGen/arm64-former-microsoft-intrinsics-header-warning.c b/clang/test/CodeGen/arm64-former-microsoft-intrinsics-header-warning.c
new file mode 100644
index 0000000000000..52fed49db4dd2
--- /dev/null
+++ b/clang/test/CodeGen/arm64-former-microsoft-intrinsics-header-warning.c
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -triple arm64-windows -Wno-everything -Wimplicit-function-declaration -fms-compatibility -fsyntax-only -verify %s
+
+// RUN: %clang_cc1 -triple arm64-linux -Wno-everything -Wimplicit-function-declaration -fsyntax-only -verify %s
+
+// RUN: %clang_cc1 -triple arm64-darwin -Wno-everything -Wimplicit-function-declaration -fms-compatibility -fsyntax-only -verify %s
+
+void check__dmb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __dmb(0);
+}
+
+void check__dsb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __dsb(0);
+}
+
+void check__isb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __isb(0);
+}
+
+void check__yield(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __yield();
+}
+
+void check__wfe(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __wfe();
+}
+
+void check__wfi(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __wfi();
+}
+
+void check__sev(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __sev();
+}
+
+void check__sevl(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __sevl();
+}
diff --git a/clang/test/CodeGen/arm64-former-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-former-microsoft-intrinsics.c
new file mode 100644
index 0000000000000..f1d9af2c6d25e
--- /dev/null
+++ b/clang/test/CodeGen/arm64-former-microsoft-intrinsics.c
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -triple arm64-windows -Wno-implicit-function-declaration -fms-compatibility -emit-llvm -o - %s \
+// RUN:    | FileCheck %s
+
+// RUN: %clang_cc1 -triple arm64-linux -Werror -emit-llvm -o - %s 2>&1 \
+// RUN:    | FileCheck %s
+
+// RUN: %clang_cc1 -triple arm64-darwin -Wno-implicit-function-declaration -fms-compatibility -emit-llvm -o - %s \
+// RUN:    | FileCheck %s
+
+#include <arm_acle.h>
+
+void check__dmb(void) {
+  __dmb(0);
+}
+
+// CHECK: @llvm.aarch64.dmb(i32 0)
+
+void check__dsb(void) {
+  __dsb(0);
+}
+
+// CHECK: @llvm.aarch64.dsb(i32 0)
+
+void check__isb(void) {
+  __isb(0);
+}
+
+// CHECK: @llvm.aarch64.isb(i32 0)
+
+void check__yield(void) {
+  __yield();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 1)
+
+void check__wfe(void) {
+  __wfe();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 2)
+
+void check__wfi(void) {
+  __wfi();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 3)
+
+void check__sev(void) {
+  __sev();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 4)
+
+void check__sevl(void) {
+  __sevl();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 5)
+
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 74c6f2f79e632..e18977a4559b1 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -35,62 +35,6 @@ __int64 test_InterlockedAdd64_constant(__int64 volatile *Addend) {
 // CHECK-MSVC: ret i64 %[[NEWVAL:[0-9]+]]
 // CHECK-LINUX: error: call to undeclared function '_InterlockedAdd64'
 
-void check__dmb(void) {
-  __dmb(0);
-}
-
-// CHECK-MSVC: @llvm.aarch64.dmb(i32 0)
-// CHECK-LINUX: error: call to undeclared function '__dmb'
-
-void check__dsb(void) {
-  __dsb(0);
-}
-
-// CHECK-MSVC: @llvm.aarch64.dsb(i32 0)
-// CHECK-LINUX: error: call to undeclared function '__dsb'
-
-void check__isb(void) {
-  __isb(0);
-}
-
-// CHECK-MSVC: @llvm.aarch64.isb(i32 0)
-// CHECK-LINUX: error: call to undeclared function '__isb'
-
-void check__yield(void) {
-  __yield();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 1)
-// CHECK-LINUX: error: call to undeclared function '__yield'
-
-void check__wfe(void) {
-  __wfe();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 2)
-// CHECK-LINUX: error: call to undeclared function '__wfe'
-
-void check__wfi(void) {
-  __wfi();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 3)
-// CHECK-LINUX: error: call to undeclared function '__wfi'
-
-void check__sev(void) {
-  __sev();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 4)
-// CHECK-LINUX: error: call to undeclared function '__sev'
-
-void check__sevl(void) {
-  __sevl();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 5)
-// CHECK-LINUX: error: call to undeclared function '__sevl'
-
 void check_ReadWriteBarrier(void) {
   _ReadWriteBarrier();
 }
diff --git a/clang/test/CodeGen/builtins-arm-microsoft.c b/clang/test/CodeGen/builtins-arm-microsoft.c
index 841a8a500d533..b2681197e33e0 100644
--- a/clang/test/CodeGen/builtins-arm-microsoft.c
+++ b/clang/test/CodeGen/builtins-arm-microsoft.c
@@ -4,38 +4,40 @@
 // RUN:     | FileCheck %s -check-prefix CHECK-EABI
 // REQUIRES: arm-registered-target
 
+#include <arm_acle.h>
+
 void test_yield_intrinsic() {
   __yield();
 }
 
 // CHECK-MSVC: call void @llvm.arm.hint(i32 1)
-// CHECK-EABI-NOT: call void @llvm.arm.hint(i32 1)
+// CHECK-EABI: call void @llvm.arm.hint(i32 1)
 
 void wfe() {
   __wfe();
 }
 
 // CHECK-MSVC: call {{.*}} @llvm.arm.hint(i32 2)
-// CHECK-EABI-NOT: call {{.*}} @llvm.arm.hint(i32 2)
+// CHECK-EABI: call {{.*}} @llvm.arm.hint(i32 2)
 
 void wfi() {
   __wfi();
 }
 
 // CHECK-MSVC: call {{.*}} @llvm.arm.hint(i32 3)
-// CHECK-EABI-NOT: call {{.*}} @llvm.arm.hint(i32 3)
+// CHECK-EABI: call {{.*}} @llvm.arm.hint(i32 3)
 
 void sev() {
   __sev();
 }
 
 // CHECK-MSVC: call {{.*}} @llvm.arm.hint(i32 4)
-// CHECK-EABI-NOT: call {{.*}} @llvm.arm.hint(i32 4)
+// CHECK-EABI: call {{.*}} @llvm.arm.hint(i32 4)
 
 void sevl() {
   __sevl();
 }
 
 // CHECK-MSVC: call {{.*}} @llvm.arm.hint(i32 5)
-// CHECK-EABI-NOT: call {{.*}} @llvm.arm.hint(i32 5)
+// CHECK-EABI: call {{.*}} @llvm.arm.hint(i32 5)
 
diff --git a/clang/test/Headers/Inputs/include/stdint.h b/clang/test/Headers/Inputs/include/stdint.h
index 67b27b8dfc7b9..eb09272b40c33 100644
--- a/clang/test/Headers/Inputs/include/stdint.h
+++ b/clang/test/Headers/Inputs/include/stdint.h
@@ -1,11 +1,23 @@
 #ifndef STDINT_H
 #define STDINT_H
 
+#ifdef __INT8_TYPE__
+typedef __INT8_TYPE__ int8_t;
+#endif
+typedef unsigned char uint8_t;
+
+#ifdef __INT16_TYPE__
+typedef __INT16_TYPE__ int16_t;
+typedef unsigned __INT16_TYPE__ uint16_t;
+#endif
+
 #ifdef __INT32_TYPE__
+typedef __INT32_TYPE__ int32_t;
 typedef unsigned __INT32_TYPE__ uint32_t;
 #endif
 
 #ifdef __INT64_TYPE__
+typedef __INT64_TYPE__ int64_t;
 typedef unsigned __INT64_TYPE__ uint64_t;
 #endif
 
diff --git a/clang/test/Headers/ms-intrin.cpp b/clang/test/Headers/ms-intrin.cpp
index d630883e79d6a..985f8577c46bf 100644
--- a/clang/test/Headers/ms-intrin.cpp
+++ b/clang/test/Headers/ms-intrin.cpp
@@ -36,6 +36,10 @@ typedef __SIZE_TYPE__ size_t;
 
 #include <intrin.h>
 
+#ifdef __ARM_ACLE
+#include <arm_acle.h>
+#endif
+
 // Use some C++ to make sure we closed the extern "C" brackets.
 template <typename T>
 void foo(T V) {}

From 7810d8484413ae64029c93f8dfc43f77b0ba98c3 Mon Sep 17 00:00:00 2001
From: Sarah Spall <sarahspall@microsoft.com>
Date: Mon, 21 Apr 2025 15:11:39 -0700
Subject: [PATCH 660/710] [HLSL] Boolean in a RawBuffer should be i32 and
 Boolean vector in a RawBuffer should be <N x i32> (#135848)

Instead of converting the type in a RawBuffer to its HLSL type using
'ConvertType', use 'ConvertTypeForMem'.
ConvertTypeForMem handles booleans being i32 and boolean vectors being <
N x i32 >.
Add tests to show booleans and boolean vectors in RawBuffers now have
the correct type of i32, and <N x i32> respectively.
Closes #135635
---
 clang/lib/CodeGen/Targets/DirectX.cpp                       | 2 +-
 .../builtins/AppendStructuredBuffer-elementtype.hlsl        | 4 ++++
 .../builtins/ConsumeStructuredBuffer-elementtype.hlsl       | 4 ++++
 .../builtins/RWStructuredBuffer-elementtype.hlsl            | 6 ++++++
 .../CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl  | 6 ++++++
 5 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/Targets/DirectX.cpp b/clang/lib/CodeGen/Targets/DirectX.cpp
index 0c5fc6e0d9fb1..96a1284661394 100644
--- a/clang/lib/CodeGen/Targets/DirectX.cpp
+++ b/clang/lib/CodeGen/Targets/DirectX.cpp
@@ -52,7 +52,7 @@ llvm::Type *DirectXTargetCodeGenInfo::getHLSLType(
       return nullptr;
 
     // convert element type
-    llvm::Type *ElemType = CGM.getTypes().ConvertType(ContainedTy);
+    llvm::Type *ElemType = CGM.getTypes().ConvertTypeForMem(ContainedTy);
 
     llvm::StringRef TypeName =
         ResAttrs.RawBuffer ? "dx.RawBuffer" : "dx.TypedBuffer";
diff --git a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
index 7fc01e59d3f8c..eea148c46f3a0 100644
--- a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
@@ -20,6 +20,8 @@ struct MyStruct {
 // DXIL: %"class.hlsl::AppendStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 0)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct, 1, 0)
 // DXIL: %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }
+// DXIL: %"class.hlsl::AppendStructuredBuffer.13" = type { target("dx.RawBuffer", i32, 1, 0)
+// DXIL: %"class.hlsl::AppendStructuredBuffer.14" = type { target("dx.RawBuffer", <4 x i32>, 1, 0)
 
 AppendStructuredBuffer<int16_t> BufI16;
 AppendStructuredBuffer<uint16_t> BufU16;
@@ -41,6 +43,8 @@ AppendStructuredBuffer<float3> BufF32x3;
 // TODO: AppendStructuredBuffer<snorm double> BufSNormF64;
 // TODO: AppendStructuredBuffer<unorm double> BufUNormF64;
 AppendStructuredBuffer<MyStruct> BufMyStruct;
+AppendStructuredBuffer<bool> BufBool;
+AppendStructuredBuffer<bool4> BufBoolVec;
 
 [numthreads(1,1,1)]
 void main(int GI : SV_GroupIndex) {
diff --git a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
index 338d73e9a3890..c303d48f7bd87 100644
--- a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
@@ -20,6 +20,8 @@ struct MyStruct {
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 0)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct, 1, 0)
 // DXIL: %struct.MyStruct = type { <4 x float>, <2 x i32>, [8 x i8] }
+// DXIL: %"class.hlsl::ConsumeStructuredBuffer.13" = type { target("dx.RawBuffer", i32, 1, 0)
+// DXIL: %"class.hlsl::ConsumeStructuredBuffer.14" = type { target("dx.RawBuffer", <4 x i32>, 1, 0)
 
 ConsumeStructuredBuffer<int16_t> BufI16;
 ConsumeStructuredBuffer<uint16_t> BufU16;
@@ -41,6 +43,8 @@ ConsumeStructuredBuffer<float3> BufF32x3;
 // TODO: ConsumeStructuredBuffer<snorm double> BufSNormF64;
 // TODO: ConsumeStructuredBuffer<unorm double> BufUNormF64;
 ConsumeStructuredBuffer<MyStruct> BufMyStruct;
+ConsumeStructuredBuffer<bool> BufBool;
+ConsumeStructuredBuffer<bool4> BufBoolVec;
 
 [numthreads(1,1,1)]
 void main(int GI : SV_GroupIndex) {
diff --git a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
index 2fdeca2dda6fb..1d68736d1197c 100644
--- a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
@@ -13,6 +13,8 @@
 // CHECK: %"class.hlsl::RWStructuredBuffer.9" = type { target("dx.RawBuffer", <3 x i32>, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.12" = type { target("dx.RawBuffer", i32, 1, 0) }
+// CHECK: %"class.hlsl::RWStructuredBuffer.13" = type { target("dx.RawBuffer", <4 x i32>, 1, 0) }
 
 RWStructuredBuffer<int16_t> BufI16;
 RWStructuredBuffer<uint16_t> BufU16;
@@ -27,6 +29,8 @@ RWStructuredBuffer< vector<int16_t, 4> > BufI16x4;
 RWStructuredBuffer< vector<uint, 3> > BufU32x3;
 RWStructuredBuffer<half2> BufF16x2;
 RWStructuredBuffer<float3> BufF32x3;
+RWStructuredBuffer<bool> BufBool;
+RWStructuredBuffer<bool4> BufBoolVec;
 // TODO: RWStructuredBuffer<snorm half> BufSNormF16;
 // TODO: RWStructuredBuffer<unorm half> BufUNormF16;
 // TODO: RWStructuredBuffer<snorm float> BufSNormF32;
@@ -49,4 +53,6 @@ void main(int GI : SV_GroupIndex) {
   BufU32x3[GI] = 0;
   BufF16x2[GI] = 0;
   BufF32x3[GI] = 0;
+  BufBool[GI] = false;
+  BufBool[GI] = false;
 }
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
index 0986c9b3330f0..f7e6434b6967e 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
@@ -13,6 +13,8 @@
 // CHECK: %"class.hlsl::StructuredBuffer.9" = type { target("dx.RawBuffer", <3 x i32>, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.12" = type { target("dx.RawBuffer", i32, 0, 0) }
+// CHECK: %"class.hlsl::StructuredBuffer.13" = type { target("dx.RawBuffer", <4 x i32>, 0, 0) }
 
 StructuredBuffer<int16_t> BufI16;
 StructuredBuffer<uint16_t> BufU16;
@@ -27,6 +29,8 @@ StructuredBuffer< vector<int16_t, 4> > BufI16x4;
 StructuredBuffer< vector<uint, 3> > BufU32x3;
 StructuredBuffer<half2> BufF16x2;
 StructuredBuffer<float3> BufF32x3;
+StructuredBuffer<bool> BufBool;
+StructuredBuffer<bool4> BufBoolVec;
 // TODO: StructuredBuffer<snorm half> BufSNormF16;
 // TODO: StructuredBuffer<unorm half> BufUNormF16;
 // TODO: StructuredBuffer<snorm float> BufSNormF32;
@@ -49,4 +53,6 @@ void main(int GI : SV_GroupIndex) {
   vector<int, 3> v11 = BufU32x3[GI];
   half2 v12 = BufF16x2[GI];
   float3 v13 = BufF32x3[GI];
+  bool v14 = BufBool[GI];
+  bool4 v15 = BufBoolVec[GI];
 }

From 8389d6fad76bd880f02bddce7f0f2612ff0afc40 Mon Sep 17 00:00:00 2001
From: Pranav Kant <prka@google.com>
Date: Mon, 21 Apr 2025 15:17:03 -0700
Subject: [PATCH 661/710] [lld] Merge equivalent symbols found during ICF
 (#134342)

Fixes a correctness issue for AArch64 when ADRP and LDR instructions are
outlined in separate sections and sections are fed to ICF for
deduplication.

See test case (based on
https://github.com/llvm/llvm-project/issues/129122) for details. All
rodata.* sections are folded into a single section with ICF. This leads
to all f2_* function sections getting folded into one (as their
relocation target symbols g* belong to .rodata.g* sections that have
already been folded into one). Since relocations still refer original g*
symbols, we end up creating duplicate GOT entry for all such symbols.
This PR addresses that by tracking such folded symbols and create one
GOT entry for all such symbols.

Fixes https://github.com/llvm/llvm-project/issues/129122

Co-authored by: @jyknight
---
 lld/ELF/ICF.cpp                        | 54 ++++++++++++++-
 lld/ELF/SymbolTable.cpp                |  6 ++
 lld/ELF/SymbolTable.h                  |  1 +
 lld/test/ELF/aarch64-got-merging-icf.s | 95 ++++++++++++++++++++++++++
 lld/test/ELF/icf-preemptible.s         |  6 ++
 5 files changed, 161 insertions(+), 1 deletion(-)
 create mode 100644 lld/test/ELF/aarch64-got-merging-icf.s

diff --git a/lld/ELF/ICF.cpp b/lld/ELF/ICF.cpp
index 1cdcf6be9d8a9..849f6bdd445f9 100644
--- a/lld/ELF/ICF.cpp
+++ b/lld/ELF/ICF.cpp
@@ -80,6 +80,7 @@
 #include "SymbolTable.h"
 #include "Symbols.h"
 #include "SyntheticSections.h"
+#include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Object/ELF.h"
 #include "llvm/Support/Parallel.h"
@@ -333,6 +334,28 @@ bool ICF<ELFT>::equalsConstant(const InputSection *a, const InputSection *b) {
              : constantEq(a, ra.relas, b, rb.relas);
 }
 
+template <class RelTy>
+static SmallVector<Symbol *> getReloc(const InputSection *sec,
+                                      Relocs<RelTy> relocs) {
+  SmallVector<Symbol *> syms;
+  for (auto ri = relocs.begin(), re = relocs.end(); ri != re; ++ri) {
+    Symbol &sym = sec->file->getRelocTargetSym(*ri);
+    syms.push_back(&sym);
+  }
+  return syms;
+}
+
+template <class ELFT>
+static SmallVector<Symbol *> getRelocTargetSyms(const InputSection *sec) {
+  const RelsOrRelas<ELFT> rel = sec->template relsOrRelas<ELFT>();
+  if (rel.areRelocsCrel())
+    return getReloc(sec, rel.crels);
+  if (rel.areRelocsRel())
+    return getReloc(sec, rel.rels);
+
+  return getReloc(sec, rel.relas);
+}
+
 // Compare two lists of relocations. Returns true if all pairs of
 // relocations point to the same section in terms of ICF.
 template <class ELFT>
@@ -535,14 +558,28 @@ template <class ELFT> void ICF<ELFT>::run() {
   auto print = [&ctx = ctx]() -> ELFSyncStream {
     return {ctx, ctx.arg.printIcfSections ? DiagLevel::Msg : DiagLevel::None};
   };
+
+  EquivalenceClasses<Symbol *> symbolEquivalence;
   // Merge sections by the equivalence class.
+  // Merge symbols identified as equivalent during ICF.
   forEachClassRange(0, sections.size(), [&](size_t begin, size_t end) {
     if (end - begin == 1)
       return;
     print() << "selected section " << sections[begin];
+    SmallVector<Symbol *> syms = getRelocTargetSyms<ELFT>(sections[begin]);
     for (size_t i = begin + 1; i < end; ++i) {
       print() << "  removing identical section " << sections[i];
       sections[begin]->replace(sections[i]);
+      SmallVector<Symbol *> replacedSyms =
+          getRelocTargetSyms<ELFT>(sections[i]);
+      assert(syms.size() == replacedSyms.size() &&
+             "Should have same number of syms!");
+      for (size_t i = 0; i < syms.size(); i++) {
+        if (syms[i] == replacedSyms[i] || !syms[i]->isGlobal() ||
+            !replacedSyms[i]->isGlobal())
+          continue;
+        symbolEquivalence.unionSets(syms[i], replacedSyms[i]);
+      }
 
       // At this point we know sections merged are fully identical and hence
       // we want to remove duplicate implicit dependencies such as link order
@@ -561,11 +598,26 @@ template <class ELFT> void ICF<ELFT>::run() {
           d->folded = true;
         }
   };
-  for (Symbol *sym : ctx.symtab->getSymbols())
+  for (Symbol *sym : ctx.symtab->getSymbols()) {
     fold(sym);
+    auto it = symbolEquivalence.findLeader(sym);
+    if (it != symbolEquivalence.member_end() && *it != sym) {
+      print() << "redirecting '" << sym->getName() << "' in symtab to '"
+              << (*it)->getName() << "'";
+      ctx.symtab->redirect(sym, *it);
+    }
+  }
   parallelForEach(ctx.objectFiles, [&](ELFFileBase *file) {
     for (Symbol *sym : file->getLocalSymbols())
       fold(sym);
+    for (Symbol *&sym : file->getMutableGlobalSymbols()) {
+      auto it = symbolEquivalence.findLeader(sym);
+      if (it != symbolEquivalence.member_end() && *it != sym) {
+        print() << "redirecting '" << sym->getName() << "' to '"
+                << (*it)->getName() << "'";
+        sym = *it;
+      }
+    }
   });
 
   // InputSectionDescription::sections is populated by processSectionCommands().
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index b8a70d4e898fc..91e47e15b01a4 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -29,6 +29,12 @@ using namespace llvm::ELF;
 using namespace lld;
 using namespace lld::elf;
 
+void SymbolTable::redirect(Symbol *from, Symbol *to) {
+  int &fromIdx = symMap[CachedHashStringRef(from->getName())];
+  const int toIdx = symMap[CachedHashStringRef(to->getName())];
+  fromIdx = toIdx;
+}
+
 void SymbolTable::wrap(Symbol *sym, Symbol *real, Symbol *wrap) {
   // Redirect __real_foo to the original foo and foo to the original __wrap_foo.
   int &idx1 = symMap[CachedHashStringRef(sym->getName())];
diff --git a/lld/ELF/SymbolTable.h b/lld/ELF/SymbolTable.h
index d6443742f7baa..e3a39bac85f97 100644
--- a/lld/ELF/SymbolTable.h
+++ b/lld/ELF/SymbolTable.h
@@ -41,6 +41,7 @@ class SymbolTable {
   SymbolTable(Ctx &ctx) : ctx(ctx) {}
   ArrayRef<Symbol *> getSymbols() const { return symVector; }
 
+  void redirect(Symbol *from, Symbol *to);
   void wrap(Symbol *sym, Symbol *real, Symbol *wrap);
 
   Symbol *insert(StringRef name);
diff --git a/lld/test/ELF/aarch64-got-merging-icf.s b/lld/test/ELF/aarch64-got-merging-icf.s
new file mode 100644
index 0000000000000..01d7bdfd75185
--- /dev/null
+++ b/lld/test/ELF/aarch64-got-merging-icf.s
@@ -0,0 +1,95 @@
+## REQUIRES: aarch64
+## Check that symbols that ICF assumes to be the same get a single GOT entry
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t
+# RUN: llvm-mc -filetype=obj -crel -triple=aarch64 %s -o %tcrel
+# RUN: ld.lld %t -o %t2 --icf=all
+# RUN: ld.lld %tcrel -o %tcrel2 --icf=all
+
+# RUN: llvm-objdump --section-headers %t2 | FileCheck %s --check-prefix=EXE
+# RUN: llvm-objdump --section-headers %tcrel2 | FileCheck %s --check-prefix=EXE
+
+# RUN: ld.lld -shared %t -o %t3 --icf=all
+# RUN: ld.lld -shared %tcrel -o %tcrel3 --icf=all
+
+# RUN: llvm-objdump --section-headers %t3 | FileCheck %s --check-prefix=DSO
+# RUN: llvm-objdump --section-headers %tcrel3 | FileCheck %s --check-prefix=DSO
+
+## All global g* symbols should merge into a single GOT entry while non-global
+## gets its own GOT entry.
+# EXE: {{.*}}.got 00000010{{.*}}
+
+## When symbols are preemptible in DSO mode, GOT entries wouldn't be merged
+# DSO: {{.*}}.got 00000028{{.*}}
+
+.addrsig
+
+callee:
+ret
+
+.macro f, index, isglobal
+
+# (Kept unique) first instruction of the GOT code sequence
+.section .text.f1_\index,"ax",@progbits
+f1_\index:
+adrp x0, :got:g\index
+mov x1, #\index
+b f2_\index
+
+# Folded, second instruction of the GOT code sequence
+.section .text.f2_\index,"ax",@progbits
+f2_\index:
+ldr x0, [x0, :got_lo12:g\index] 
+b callee
+
+# Folded
+.ifnb \isglobal
+.globl g\index
+.endif
+.section .rodata.g\index,"a",@progbits
+g_\index:
+.long 111
+.long 122
+
+g\index:
+.byte 123
+
+.section .text._start,"ax",@progbits
+bl f1_\index
+
+.endm
+
+## Another set of sections merging: g1 <- g2. Linker should be able to
+## resolve both g1 and g2 to g0 based on ICF on previous sections.
+
+.section .text.t1_0,"ax",@progbits
+t1_0:
+adrp x2, :got:g1
+mov x3, #1
+b t2_0
+
+.section .text.t2_0,"ax",@progbits
+t2_0:
+ldr x2, [x2, :got_lo12:g1]
+b callee
+
+.section .text.t1_1,"ax",@progbits
+t1_1:
+adrp x2, :got:g2
+mov x3, #2
+b t2_1
+
+.section .text.t2_1,"ax",@progbits
+t2_1:
+ldr x2, [x2, :got_lo12:g2]
+b callee
+
+.section .text._start,"ax",@progbits
+.globl _start
+_start:
+
+f 0 1
+f 1 1
+f 2 1
+f 3 1
+f 4
diff --git a/lld/test/ELF/icf-preemptible.s b/lld/test/ELF/icf-preemptible.s
index 4bd1eca438b19..9352493600695 100644
--- a/lld/test/ELF/icf-preemptible.s
+++ b/lld/test/ELF/icf-preemptible.s
@@ -17,6 +17,12 @@
 # EXE-NEXT: selected section {{.*}}:(.text.h1)
 # EXE-NEXT:   removing identical section {{.*}}:(.text.h2)
 # EXE-NEXT:   removing identical section {{.*}}:(.text.h3)
+# EXE-NEXT: redirecting 'f2' in symtab to 'f1'
+# EXE-NEXT: redirecting 'g2' in symtab to 'g1'
+# EXE-NEXT: redirecting 'g3' in symtab to 'g1'
+# EXE-NEXT: redirecting 'f2' to 'f1'
+# EXE-NEXT: redirecting 'g2' to 'g1'
+# EXE-NEXT: redirecting 'g3' to 'g1'
 # EXE-NOT:  {{.}}
 
 ## Definitions are preemptible in a DSO. Only leaf functions can be folded.

From 92c0b425ad4ca7f4c9b8f7f663e6e64d1e070353 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro@nvidia.com>
Date: Mon, 21 Apr 2025 15:21:39 -0700
Subject: [PATCH 662/710] [NVPTX] add test case for vector reduction intrinsics
 (#136381)

Test how these intrinsics are handled by the NVPTX backend. Currently, these intrinsics are lowered to sequential reductions by the ExpandReductions pass.
---
 .../CodeGen/NVPTX/reduction-intrinsics.ll     | 1958 +++++++++++++++++
 1 file changed, 1958 insertions(+)
 create mode 100644 llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll

diff --git a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
new file mode 100644
index 0000000000000..2a12e9b364a54
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
@@ -0,0 +1,1958 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_80 -mattr=+ptx70 -O0 \
+; RUN:      -disable-post-ra -verify-machineinstrs \
+; RUN: | FileCheck -check-prefixes CHECK,CHECK-SM80 %s
+; RUN: %if ptxas-12.8 %{ llc < %s -mcpu=sm_80 -mattr=+ptx70 -O0 \
+; RUN:      -disable-post-ra -verify-machineinstrs \
+; RUN: | %ptxas-verify -arch=sm_80 %}
+; RUN: llc < %s -mcpu=sm_100 -mattr=+ptx87 -O0 \
+; RUN:      -disable-post-ra -verify-machineinstrs \
+; RUN: | FileCheck -check-prefixes CHECK,CHECK-SM100 %s
+; RUN: %if ptxas-12.8 %{ llc < %s -mcpu=sm_100 -mattr=+ptx87 -O0 \
+; RUN:      -disable-post-ra -verify-machineinstrs \
+; RUN: | %ptxas-verify -arch=sm_100 %}
+target triple = "nvptx64-nvidia-cuda"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+; Check straight line reduction.
+define half @reduce_fadd_half(<8 x half> %in) {
+; CHECK-LABEL: reduce_fadd_half(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<18>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fadd_half_param_0];
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    mov.b16 %rs3, 0x0000;
+; CHECK-NEXT:    add.rn.f16 %rs4, %rs1, %rs3;
+; CHECK-NEXT:    add.rn.f16 %rs5, %rs4, %rs2;
+; CHECK-NEXT:    mov.b32 {%rs6, %rs7}, %r2;
+; CHECK-NEXT:    add.rn.f16 %rs8, %rs5, %rs6;
+; CHECK-NEXT:    add.rn.f16 %rs9, %rs8, %rs7;
+; CHECK-NEXT:    mov.b32 {%rs10, %rs11}, %r3;
+; CHECK-NEXT:    add.rn.f16 %rs12, %rs9, %rs10;
+; CHECK-NEXT:    add.rn.f16 %rs13, %rs12, %rs11;
+; CHECK-NEXT:    mov.b32 {%rs14, %rs15}, %r4;
+; CHECK-NEXT:    add.rn.f16 %rs16, %rs13, %rs14;
+; CHECK-NEXT:    add.rn.f16 %rs17, %rs16, %rs15;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs17;
+; CHECK-NEXT:    ret;
+  %res = call half @llvm.vector.reduce.fadd(half 0.0, <8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fadd_half_reassoc(<8 x half> %in) {
+; CHECK-SM80-LABEL: reduce_fadd_half_reassoc(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<6>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<10>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fadd_half_reassoc_param_0];
+; CHECK-SM80-NEXT:    add.rn.f16x2 %r5, %r2, %r4;
+; CHECK-SM80-NEXT:    add.rn.f16x2 %r6, %r1, %r3;
+; CHECK-SM80-NEXT:    add.rn.f16x2 %r7, %r6, %r5;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
+; CHECK-SM80-NEXT:    // implicit-def: %rs2
+; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM80-NEXT:    add.rn.f16x2 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
+; CHECK-SM80-NEXT:    mov.b16 %rs4, 0x0000;
+; CHECK-SM80-NEXT:    add.rn.f16 %rs5, %rs3, %rs4;
+; CHECK-SM80-NEXT:    st.param.b16 [func_retval0], %rs5;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fadd_half_reassoc(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<6>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<10>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fadd_half_reassoc_param_0];
+; CHECK-SM100-NEXT:    add.rn.f16x2 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    add.rn.f16x2 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    add.rn.f16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    add.rn.f16x2 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    mov.b16 %rs4, 0x0000;
+; CHECK-SM100-NEXT:    add.rn.f16 %rs5, %rs3, %rs4;
+; CHECK-SM100-NEXT:    st.param.b16 [func_retval0], %rs5;
+; CHECK-SM100-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fadd(half 0.0, <8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fadd_half_reassoc_nonpow2(<7 x half> %in) {
+; CHECK-LABEL: reduce_fadd_half_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<16>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fadd_half_reassoc_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fadd_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fadd_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b16 %rs8, 0x0000;
+; CHECK-NEXT:    add.rn.f16 %rs9, %rs1, %rs8;
+; CHECK-NEXT:    add.rn.f16 %rs10, %rs9, %rs2;
+; CHECK-NEXT:    add.rn.f16 %rs11, %rs10, %rs3;
+; CHECK-NEXT:    add.rn.f16 %rs12, %rs11, %rs4;
+; CHECK-NEXT:    add.rn.f16 %rs13, %rs12, %rs5;
+; CHECK-NEXT:    add.rn.f16 %rs14, %rs13, %rs6;
+; CHECK-NEXT:    add.rn.f16 %rs15, %rs14, %rs7;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs15;
+; CHECK-NEXT:    ret;
+  %res = call half @llvm.vector.reduce.fadd(half 0.0, <7 x half> %in)
+  ret half %res
+}
+
+; Check straight-line reduction.
+define float @reduce_fadd_float(<8 x float> %in) {
+; CHECK-LABEL: reduce_fadd_float(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<17>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fadd_float_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fadd_float_param_0];
+; CHECK-NEXT:    add.rn.f32 %f9, %f1, 0f00000000;
+; CHECK-NEXT:    add.rn.f32 %f10, %f9, %f2;
+; CHECK-NEXT:    add.rn.f32 %f11, %f10, %f3;
+; CHECK-NEXT:    add.rn.f32 %f12, %f11, %f4;
+; CHECK-NEXT:    add.rn.f32 %f13, %f12, %f5;
+; CHECK-NEXT:    add.rn.f32 %f14, %f13, %f6;
+; CHECK-NEXT:    add.rn.f32 %f15, %f14, %f7;
+; CHECK-NEXT:    add.rn.f32 %f16, %f15, %f8;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f16;
+; CHECK-NEXT:    ret;
+  %res = call float @llvm.vector.reduce.fadd(float 0.0, <8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fadd_float_reassoc(<8 x float> %in) {
+; CHECK-LABEL: reduce_fadd_float_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<17>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fadd_float_reassoc_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fadd_float_reassoc_param_0];
+; CHECK-NEXT:    add.rn.f32 %f9, %f3, %f7;
+; CHECK-NEXT:    add.rn.f32 %f10, %f1, %f5;
+; CHECK-NEXT:    add.rn.f32 %f11, %f4, %f8;
+; CHECK-NEXT:    add.rn.f32 %f12, %f2, %f6;
+; CHECK-NEXT:    add.rn.f32 %f13, %f12, %f11;
+; CHECK-NEXT:    add.rn.f32 %f14, %f10, %f9;
+; CHECK-NEXT:    add.rn.f32 %f15, %f14, %f13;
+; CHECK-NEXT:    add.rn.f32 %f16, %f15, 0f00000000;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f16;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fadd(float 0.0, <8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fadd_float_reassoc_nonpow2(<7 x float> %in) {
+; CHECK-LABEL: reduce_fadd_float_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<15>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f7, [reduce_fadd_float_reassoc_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.f32 {%f5, %f6}, [reduce_fadd_float_reassoc_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fadd_float_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    add.rn.f32 %f8, %f3, %f7;
+; CHECK-NEXT:    add.rn.f32 %f9, %f1, %f5;
+; CHECK-NEXT:    add.rn.f32 %f10, %f9, %f8;
+; CHECK-NEXT:    add.rn.f32 %f11, %f2, %f6;
+; CHECK-NEXT:    add.rn.f32 %f12, %f11, %f4;
+; CHECK-NEXT:    add.rn.f32 %f13, %f10, %f12;
+; CHECK-NEXT:    add.rn.f32 %f14, %f13, 0f00000000;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f14;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fadd(float 0.0, <7 x float> %in)
+  ret float %res
+}
+
+; Check straight line reduction.
+define half @reduce_fmul_half(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmul_half(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<16>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmul_half_param_0];
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    mul.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-NEXT:    mul.rn.f16 %rs6, %rs3, %rs4;
+; CHECK-NEXT:    mul.rn.f16 %rs7, %rs6, %rs5;
+; CHECK-NEXT:    mov.b32 {%rs8, %rs9}, %r3;
+; CHECK-NEXT:    mul.rn.f16 %rs10, %rs7, %rs8;
+; CHECK-NEXT:    mul.rn.f16 %rs11, %rs10, %rs9;
+; CHECK-NEXT:    mov.b32 {%rs12, %rs13}, %r4;
+; CHECK-NEXT:    mul.rn.f16 %rs14, %rs11, %rs12;
+; CHECK-NEXT:    mul.rn.f16 %rs15, %rs14, %rs13;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs15;
+; CHECK-NEXT:    ret;
+  %res = call half @llvm.vector.reduce.fmul(half 1.0, <8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fmul_half_reassoc(<8 x half> %in) {
+; CHECK-SM80-LABEL: reduce_fmul_half_reassoc(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<10>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmul_half_reassoc_param_0];
+; CHECK-SM80-NEXT:    mul.rn.f16x2 %r5, %r2, %r4;
+; CHECK-SM80-NEXT:    mul.rn.f16x2 %r6, %r1, %r3;
+; CHECK-SM80-NEXT:    mul.rn.f16x2 %r7, %r6, %r5;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
+; CHECK-SM80-NEXT:    // implicit-def: %rs2
+; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM80-NEXT:    mul.rn.f16x2 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
+; CHECK-SM80-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmul_half_reassoc(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<10>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmul_half_reassoc_param_0];
+; CHECK-SM100-NEXT:    mul.rn.f16x2 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    mul.rn.f16x2 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    mul.rn.f16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    mul.rn.f16x2 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-SM100-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fmul(half 1.0, <8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fmul_half_reassoc_nonpow2(<7 x half> %in) {
+; CHECK-LABEL: reduce_fmul_half_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmul_half_reassoc_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmul_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmul_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    mov.b16 %rs8, 0x3C00;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    mul.rn.f16x2 %r5, %r3, %r4;
+; CHECK-NEXT:    mul.rn.f16x2 %r6, %r2, %r1;
+; CHECK-NEXT:    mul.rn.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    mul.rn.f16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fmul(half 1.0, <7 x half> %in)
+  ret half %res
+}
+
+; Check straight-line reduction.
+define float @reduce_fmul_float(<8 x float> %in) {
+; CHECK-LABEL: reduce_fmul_float(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fmul_float_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmul_float_param_0];
+; CHECK-NEXT:    mul.rn.f32 %f9, %f1, %f2;
+; CHECK-NEXT:    mul.rn.f32 %f10, %f9, %f3;
+; CHECK-NEXT:    mul.rn.f32 %f11, %f10, %f4;
+; CHECK-NEXT:    mul.rn.f32 %f12, %f11, %f5;
+; CHECK-NEXT:    mul.rn.f32 %f13, %f12, %f6;
+; CHECK-NEXT:    mul.rn.f32 %f14, %f13, %f7;
+; CHECK-NEXT:    mul.rn.f32 %f15, %f14, %f8;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call float @llvm.vector.reduce.fmul(float 1.0, <8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmul_float_reassoc(<8 x float> %in) {
+; CHECK-LABEL: reduce_fmul_float_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fmul_float_reassoc_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmul_float_reassoc_param_0];
+; CHECK-NEXT:    mul.rn.f32 %f9, %f3, %f7;
+; CHECK-NEXT:    mul.rn.f32 %f10, %f1, %f5;
+; CHECK-NEXT:    mul.rn.f32 %f11, %f4, %f8;
+; CHECK-NEXT:    mul.rn.f32 %f12, %f2, %f6;
+; CHECK-NEXT:    mul.rn.f32 %f13, %f12, %f11;
+; CHECK-NEXT:    mul.rn.f32 %f14, %f10, %f9;
+; CHECK-NEXT:    mul.rn.f32 %f15, %f14, %f13;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fmul(float 1.0, <8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmul_float_reassoc_nonpow2(<7 x float> %in) {
+; CHECK-LABEL: reduce_fmul_float_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f7, [reduce_fmul_float_reassoc_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.f32 {%f5, %f6}, [reduce_fmul_float_reassoc_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmul_float_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mul.rn.f32 %f8, %f3, %f7;
+; CHECK-NEXT:    mul.rn.f32 %f9, %f1, %f5;
+; CHECK-NEXT:    mul.rn.f32 %f10, %f9, %f8;
+; CHECK-NEXT:    mul.rn.f32 %f11, %f2, %f6;
+; CHECK-NEXT:    mul.rn.f32 %f12, %f11, %f4;
+; CHECK-NEXT:    mul.rn.f32 %f13, %f10, %f12;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f13;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fmul(float 1.0, <7 x float> %in)
+  ret float %res
+}
+
+; Check straight line reduction.
+define half @reduce_fmax_half(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmax_half(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmax_half_param_0];
+; CHECK-NEXT:    max.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    max.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    max.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    max.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %res = call half @llvm.vector.reduce.fmax(<8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fmax_half_reassoc(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmax_half_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmax_half_reassoc_param_0];
+; CHECK-NEXT:    max.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    max.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    max.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    max.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fmax(<8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fmax_half_reassoc_nonpow2(<7 x half> %in) {
+; CHECK-LABEL: reduce_fmax_half_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmax_half_reassoc_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmax_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmax_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    mov.b16 %rs8, 0xFE00;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    max.f16x2 %r5, %r3, %r4;
+; CHECK-NEXT:    max.f16x2 %r6, %r2, %r1;
+; CHECK-NEXT:    max.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    max.f16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fmax(<7 x half> %in)
+  ret half %res
+}
+
+; Check straight-line reduction.
+define float @reduce_fmax_float(<8 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmax_float(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fmax_float_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmax_float_param_0];
+; CHECK-NEXT:    max.f32 %f9, %f4, %f8;
+; CHECK-NEXT:    max.f32 %f10, %f2, %f6;
+; CHECK-NEXT:    max.f32 %f11, %f10, %f9;
+; CHECK-NEXT:    max.f32 %f12, %f3, %f7;
+; CHECK-NEXT:    max.f32 %f13, %f1, %f5;
+; CHECK-NEXT:    max.f32 %f14, %f13, %f12;
+; CHECK-NEXT:    max.f32 %f15, %f14, %f11;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call float @llvm.vector.reduce.fmax(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmax_float_reassoc(<8 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmax_float_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fmax_float_reassoc_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmax_float_reassoc_param_0];
+; CHECK-NEXT:    max.f32 %f9, %f4, %f8;
+; CHECK-NEXT:    max.f32 %f10, %f2, %f6;
+; CHECK-NEXT:    max.f32 %f11, %f10, %f9;
+; CHECK-NEXT:    max.f32 %f12, %f3, %f7;
+; CHECK-NEXT:    max.f32 %f13, %f1, %f5;
+; CHECK-NEXT:    max.f32 %f14, %f13, %f12;
+; CHECK-NEXT:    max.f32 %f15, %f14, %f11;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fmax(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmax_float_reassoc_nonpow2(<7 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmax_float_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f7, [reduce_fmax_float_reassoc_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.f32 {%f5, %f6}, [reduce_fmax_float_reassoc_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmax_float_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    max.f32 %f8, %f3, %f7;
+; CHECK-NEXT:    max.f32 %f9, %f1, %f5;
+; CHECK-NEXT:    max.f32 %f10, %f9, %f8;
+; CHECK-NEXT:    max.f32 %f11, %f2, %f6;
+; CHECK-NEXT:    max.f32 %f12, %f11, %f4;
+; CHECK-NEXT:    max.f32 %f13, %f10, %f12;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f13;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fmax(<7 x float> %in)
+  ret float %res
+}
+
+; Check straight line reduction.
+define half @reduce_fmin_half(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmin_half(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmin_half_param_0];
+; CHECK-NEXT:    min.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    min.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    min.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    min.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %res = call half @llvm.vector.reduce.fmin(<8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fmin_half_reassoc(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmin_half_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmin_half_reassoc_param_0];
+; CHECK-NEXT:    min.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    min.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    min.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    min.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fmin(<8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fmin_half_reassoc_nonpow2(<7 x half> %in) {
+; CHECK-LABEL: reduce_fmin_half_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmin_half_reassoc_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmin_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmin_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    mov.b16 %rs8, 0x7E00;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    min.f16x2 %r5, %r3, %r4;
+; CHECK-NEXT:    min.f16x2 %r6, %r2, %r1;
+; CHECK-NEXT:    min.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    min.f16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fmin(<7 x half> %in)
+  ret half %res
+}
+
+; Check straight-line reduction.
+define float @reduce_fmin_float(<8 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmin_float(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fmin_float_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmin_float_param_0];
+; CHECK-NEXT:    min.f32 %f9, %f4, %f8;
+; CHECK-NEXT:    min.f32 %f10, %f2, %f6;
+; CHECK-NEXT:    min.f32 %f11, %f10, %f9;
+; CHECK-NEXT:    min.f32 %f12, %f3, %f7;
+; CHECK-NEXT:    min.f32 %f13, %f1, %f5;
+; CHECK-NEXT:    min.f32 %f14, %f13, %f12;
+; CHECK-NEXT:    min.f32 %f15, %f14, %f11;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call float @llvm.vector.reduce.fmin(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmin_float_reassoc(<8 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmin_float_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fmin_float_reassoc_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmin_float_reassoc_param_0];
+; CHECK-NEXT:    min.f32 %f9, %f4, %f8;
+; CHECK-NEXT:    min.f32 %f10, %f2, %f6;
+; CHECK-NEXT:    min.f32 %f11, %f10, %f9;
+; CHECK-NEXT:    min.f32 %f12, %f3, %f7;
+; CHECK-NEXT:    min.f32 %f13, %f1, %f5;
+; CHECK-NEXT:    min.f32 %f14, %f13, %f12;
+; CHECK-NEXT:    min.f32 %f15, %f14, %f11;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fmin(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmin_float_reassoc_nonpow2(<7 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmin_float_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f7, [reduce_fmin_float_reassoc_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.f32 {%f5, %f6}, [reduce_fmin_float_reassoc_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmin_float_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    min.f32 %f8, %f3, %f7;
+; CHECK-NEXT:    min.f32 %f9, %f1, %f5;
+; CHECK-NEXT:    min.f32 %f10, %f9, %f8;
+; CHECK-NEXT:    min.f32 %f11, %f2, %f6;
+; CHECK-NEXT:    min.f32 %f12, %f11, %f4;
+; CHECK-NEXT:    min.f32 %f13, %f10, %f12;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f13;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fmin(<7 x float> %in)
+  ret float %res
+}
+
+; Check straight-line reduction.
+define half @reduce_fmaximum_half(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmaximum_half(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmaximum_half_param_0];
+; CHECK-NEXT:    max.NaN.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    max.NaN.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    max.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    max.NaN.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %res = call half @llvm.vector.reduce.fmaximum(<8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fmaximum_half_reassoc(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmaximum_half_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fmaximum_half_reassoc_param_0];
+; CHECK-NEXT:    max.NaN.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    max.NaN.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    max.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    max.NaN.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fmaximum(<8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fmaximum_half_reassoc_nonpow2(<7 x half> %in) {
+; CHECK-LABEL: reduce_fmaximum_half_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmaximum_half_reassoc_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmaximum_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmaximum_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    mov.b16 %rs8, 0xFC00;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    max.NaN.f16x2 %r5, %r3, %r4;
+; CHECK-NEXT:    max.NaN.f16x2 %r6, %r2, %r1;
+; CHECK-NEXT:    max.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    max.NaN.f16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fmaximum(<7 x half> %in)
+  ret half %res
+}
+
+; Check straight-line reduction.
+define float @reduce_fmaximum_float(<8 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmaximum_float(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fmaximum_float_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmaximum_float_param_0];
+; CHECK-NEXT:    max.NaN.f32 %f9, %f4, %f8;
+; CHECK-NEXT:    max.NaN.f32 %f10, %f2, %f6;
+; CHECK-NEXT:    max.NaN.f32 %f11, %f10, %f9;
+; CHECK-NEXT:    max.NaN.f32 %f12, %f3, %f7;
+; CHECK-NEXT:    max.NaN.f32 %f13, %f1, %f5;
+; CHECK-NEXT:    max.NaN.f32 %f14, %f13, %f12;
+; CHECK-NEXT:    max.NaN.f32 %f15, %f14, %f11;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call float @llvm.vector.reduce.fmaximum(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmaximum_float_reassoc(<8 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmaximum_float_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fmaximum_float_reassoc_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmaximum_float_reassoc_param_0];
+; CHECK-NEXT:    max.NaN.f32 %f9, %f4, %f8;
+; CHECK-NEXT:    max.NaN.f32 %f10, %f2, %f6;
+; CHECK-NEXT:    max.NaN.f32 %f11, %f10, %f9;
+; CHECK-NEXT:    max.NaN.f32 %f12, %f3, %f7;
+; CHECK-NEXT:    max.NaN.f32 %f13, %f1, %f5;
+; CHECK-NEXT:    max.NaN.f32 %f14, %f13, %f12;
+; CHECK-NEXT:    max.NaN.f32 %f15, %f14, %f11;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fmaximum(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmaximum_float_reassoc_nonpow2(<7 x float> %in) {
+;
+; CHECK-LABEL: reduce_fmaximum_float_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f7, [reduce_fmaximum_float_reassoc_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.f32 {%f5, %f6}, [reduce_fmaximum_float_reassoc_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fmaximum_float_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    max.NaN.f32 %f8, %f3, %f7;
+; CHECK-NEXT:    max.NaN.f32 %f9, %f1, %f5;
+; CHECK-NEXT:    max.NaN.f32 %f10, %f9, %f8;
+; CHECK-NEXT:    max.NaN.f32 %f11, %f2, %f6;
+; CHECK-NEXT:    max.NaN.f32 %f12, %f11, %f4;
+; CHECK-NEXT:    max.NaN.f32 %f13, %f10, %f12;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f13;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fmaximum(<7 x float> %in)
+  ret float %res
+}
+
+; Check straight-line reduction.
+define half @reduce_fminimum_half(<8 x half> %in) {
+; CHECK-LABEL: reduce_fminimum_half(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fminimum_half_param_0];
+; CHECK-NEXT:    min.NaN.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    min.NaN.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    min.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    min.NaN.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %res = call half @llvm.vector.reduce.fminimum(<8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fminimum_half_reassoc(<8 x half> %in) {
+; CHECK-LABEL: reduce_fminimum_half_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_fminimum_half_reassoc_param_0];
+; CHECK-NEXT:    min.NaN.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    min.NaN.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    min.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    min.NaN.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fminimum(<8 x half> %in)
+  ret half %res
+}
+
+define half @reduce_fminimum_half_reassoc_nonpow2(<7 x half> %in) {
+; CHECK-LABEL: reduce_fminimum_half_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fminimum_half_reassoc_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fminimum_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fminimum_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    mov.b16 %rs8, 0x7C00;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    min.NaN.f16x2 %r5, %r3, %r4;
+; CHECK-NEXT:    min.NaN.f16x2 %r6, %r2, %r1;
+; CHECK-NEXT:    min.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    min.NaN.f16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
+; CHECK-NEXT:    ret;
+  %res = call reassoc half @llvm.vector.reduce.fminimum(<7 x half> %in)
+  ret half %res
+}
+
+; Check straight-line reduction.
+define float @reduce_fminimum_float(<8 x float> %in) {
+;
+; CHECK-LABEL: reduce_fminimum_float(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fminimum_float_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fminimum_float_param_0];
+; CHECK-NEXT:    min.NaN.f32 %f9, %f4, %f8;
+; CHECK-NEXT:    min.NaN.f32 %f10, %f2, %f6;
+; CHECK-NEXT:    min.NaN.f32 %f11, %f10, %f9;
+; CHECK-NEXT:    min.NaN.f32 %f12, %f3, %f7;
+; CHECK-NEXT:    min.NaN.f32 %f13, %f1, %f5;
+; CHECK-NEXT:    min.NaN.f32 %f14, %f13, %f12;
+; CHECK-NEXT:    min.NaN.f32 %f15, %f14, %f11;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call float @llvm.vector.reduce.fminimum(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fminimum_float_reassoc(<8 x float> %in) {
+;
+; CHECK-LABEL: reduce_fminimum_float_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.f32 {%f5, %f6, %f7, %f8}, [reduce_fminimum_float_reassoc_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fminimum_float_reassoc_param_0];
+; CHECK-NEXT:    min.NaN.f32 %f9, %f4, %f8;
+; CHECK-NEXT:    min.NaN.f32 %f10, %f2, %f6;
+; CHECK-NEXT:    min.NaN.f32 %f11, %f10, %f9;
+; CHECK-NEXT:    min.NaN.f32 %f12, %f3, %f7;
+; CHECK-NEXT:    min.NaN.f32 %f13, %f1, %f5;
+; CHECK-NEXT:    min.NaN.f32 %f14, %f13, %f12;
+; CHECK-NEXT:    min.NaN.f32 %f15, %f14, %f11;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f15;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fminimum(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fminimum_float_reassoc_nonpow2(<7 x float> %in) {
+;
+; CHECK-LABEL: reduce_fminimum_float_reassoc_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f7, [reduce_fminimum_float_reassoc_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.f32 {%f5, %f6}, [reduce_fminimum_float_reassoc_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.f32 {%f1, %f2, %f3, %f4}, [reduce_fminimum_float_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    min.NaN.f32 %f8, %f3, %f7;
+; CHECK-NEXT:    min.NaN.f32 %f9, %f1, %f5;
+; CHECK-NEXT:    min.NaN.f32 %f10, %f9, %f8;
+; CHECK-NEXT:    min.NaN.f32 %f11, %f2, %f6;
+; CHECK-NEXT:    min.NaN.f32 %f12, %f11, %f4;
+; CHECK-NEXT:    min.NaN.f32 %f13, %f10, %f12;
+; CHECK-NEXT:    st.param.f32 [func_retval0], %f13;
+; CHECK-NEXT:    ret;
+  %res = call reassoc float @llvm.vector.reduce.fminimum(<7 x float> %in)
+  ret float %res
+}
+
+define i16 @reduce_add_i16(<8 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_add_i16(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<16>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<6>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_add_i16_param_0];
+; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; CHECK-SM80-NEXT:    add.s16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
+; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
+; CHECK-SM80-NEXT:    add.s16 %rs10, %rs8, %rs6;
+; CHECK-SM80-NEXT:    add.s16 %rs11, %rs4, %rs2;
+; CHECK-SM80-NEXT:    add.s16 %rs12, %rs9, %rs7;
+; CHECK-SM80-NEXT:    add.s16 %rs13, %rs12, %rs11;
+; CHECK-SM80-NEXT:    add.s16 %rs14, %rs10, %rs5;
+; CHECK-SM80-NEXT:    add.s16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_add_i16(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_add_i16_param_0];
+; CHECK-SM100-NEXT:    add.s16x2 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    add.s16x2 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    add.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    add.s16x2 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.add(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_add_i16_nonpow2(<7 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_add_i16_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.u32 %r1, [reduce_add_i16_nonpow2_param_0+8];
+; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM80-NEXT:    ld.param.u16 %rs7, [reduce_add_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_add_i16_nonpow2_param_0];
+; CHECK-SM80-NEXT:    add.s16 %rs8, %rs3, %rs7;
+; CHECK-SM80-NEXT:    add.s16 %rs9, %rs1, %rs5;
+; CHECK-SM80-NEXT:    add.s16 %rs10, %rs9, %rs8;
+; CHECK-SM80-NEXT:    add.s16 %rs11, %rs2, %rs6;
+; CHECK-SM80-NEXT:    add.s16 %rs12, %rs11, %rs4;
+; CHECK-SM80-NEXT:    add.s16 %rs13, %rs10, %rs12;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_add_i16_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<12>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.u32 %r1, [reduce_add_i16_nonpow2_param_0+8];
+; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM100-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_add_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.u16 %rs7, [reduce_add_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    mov.b16 %rs8, 0;
+; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    add.s16x2 %r5, %r3, %r4;
+; CHECK-SM100-NEXT:    add.s16x2 %r6, %r2, %r1;
+; CHECK-SM100-NEXT:    add.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-SM100-NEXT:    add.s16 %rs11, %rs9, %rs10;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.add(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_add_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_add_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_add_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_add_i32_param_0];
+; CHECK-NEXT:    add.s32 %r9, %r3, %r7;
+; CHECK-NEXT:    add.s32 %r10, %r1, %r5;
+; CHECK-NEXT:    add.s32 %r11, %r4, %r8;
+; CHECK-NEXT:    add.s32 %r12, %r2, %r6;
+; CHECK-NEXT:    add.s32 %r13, %r12, %r11;
+; CHECK-NEXT:    add.s32 %r14, %r10, %r9;
+; CHECK-NEXT:    add.s32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.add(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_add_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_add_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_add_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_add_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_add_i32_nonpow2_param_0];
+; CHECK-NEXT:    add.s32 %r8, %r3, %r7;
+; CHECK-NEXT:    add.s32 %r9, %r1, %r5;
+; CHECK-NEXT:    add.s32 %r10, %r9, %r8;
+; CHECK-NEXT:    add.s32 %r11, %r2, %r6;
+; CHECK-NEXT:    add.s32 %r12, %r11, %r4;
+; CHECK-NEXT:    add.s32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.add(<7 x i32> %in)
+  ret i32 %res
+}
+
+define i16 @reduce_mul_i16(<8 x i16> %in) {
+; CHECK-LABEL: reduce_mul_i16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<16>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_mul_i16_param_0];
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; CHECK-NEXT:    mul.lo.s16 %rs5, %rs3, %rs1;
+; CHECK-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
+; CHECK-NEXT:    mul.lo.s16 %rs10, %rs8, %rs6;
+; CHECK-NEXT:    mul.lo.s16 %rs11, %rs4, %rs2;
+; CHECK-NEXT:    mul.lo.s16 %rs12, %rs9, %rs7;
+; CHECK-NEXT:    mul.lo.s16 %rs13, %rs12, %rs11;
+; CHECK-NEXT:    mul.lo.s16 %rs14, %rs10, %rs5;
+; CHECK-NEXT:    mul.lo.s16 %rs15, %rs14, %rs13;
+; CHECK-NEXT:    cvt.u32.u16 %r5, %rs15;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.mul(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_mul_i16_nonpow2(<7 x i16> %in) {
+; CHECK-LABEL: reduce_mul_i16_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<14>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [reduce_mul_i16_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.u16 %rs7, [reduce_mul_i16_nonpow2_param_0+12];
+; CHECK-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_mul_i16_nonpow2_param_0];
+; CHECK-NEXT:    mul.lo.s16 %rs8, %rs3, %rs7;
+; CHECK-NEXT:    mul.lo.s16 %rs9, %rs1, %rs5;
+; CHECK-NEXT:    mul.lo.s16 %rs10, %rs9, %rs8;
+; CHECK-NEXT:    mul.lo.s16 %rs11, %rs2, %rs6;
+; CHECK-NEXT:    mul.lo.s16 %rs12, %rs4, %rs11;
+; CHECK-NEXT:    mul.lo.s16 %rs13, %rs10, %rs12;
+; CHECK-NEXT:    cvt.u32.u16 %r2, %rs13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.mul(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_mul_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_mul_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_mul_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_mul_i32_param_0];
+; CHECK-NEXT:    mul.lo.s32 %r9, %r3, %r7;
+; CHECK-NEXT:    mul.lo.s32 %r10, %r1, %r5;
+; CHECK-NEXT:    mul.lo.s32 %r11, %r4, %r8;
+; CHECK-NEXT:    mul.lo.s32 %r12, %r2, %r6;
+; CHECK-NEXT:    mul.lo.s32 %r13, %r12, %r11;
+; CHECK-NEXT:    mul.lo.s32 %r14, %r10, %r9;
+; CHECK-NEXT:    mul.lo.s32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.mul(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_mul_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_mul_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_mul_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_mul_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_mul_i32_nonpow2_param_0];
+; CHECK-NEXT:    mul.lo.s32 %r8, %r3, %r7;
+; CHECK-NEXT:    mul.lo.s32 %r9, %r1, %r5;
+; CHECK-NEXT:    mul.lo.s32 %r10, %r9, %r8;
+; CHECK-NEXT:    mul.lo.s32 %r11, %r2, %r6;
+; CHECK-NEXT:    mul.lo.s32 %r12, %r4, %r11;
+; CHECK-NEXT:    mul.lo.s32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.mul(<7 x i32> %in)
+  ret i32 %res
+}
+
+define i16 @reduce_umax_i16(<8 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_umax_i16(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<16>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<6>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_umax_i16_param_0];
+; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; CHECK-SM80-NEXT:    max.u16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
+; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
+; CHECK-SM80-NEXT:    max.u16 %rs10, %rs8, %rs6;
+; CHECK-SM80-NEXT:    max.u16 %rs11, %rs4, %rs2;
+; CHECK-SM80-NEXT:    max.u16 %rs12, %rs9, %rs7;
+; CHECK-SM80-NEXT:    max.u16 %rs13, %rs12, %rs11;
+; CHECK-SM80-NEXT:    max.u16 %rs14, %rs10, %rs5;
+; CHECK-SM80-NEXT:    max.u16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_umax_i16(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_umax_i16_param_0];
+; CHECK-SM100-NEXT:    max.u16x2 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    max.u16x2 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    max.u16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    max.u16x2 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.umax(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_umax_i16_nonpow2(<7 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_umax_i16_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.u32 %r1, [reduce_umax_i16_nonpow2_param_0+8];
+; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM80-NEXT:    ld.param.u16 %rs7, [reduce_umax_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umax_i16_nonpow2_param_0];
+; CHECK-SM80-NEXT:    max.u16 %rs8, %rs3, %rs7;
+; CHECK-SM80-NEXT:    max.u16 %rs9, %rs1, %rs5;
+; CHECK-SM80-NEXT:    max.u16 %rs10, %rs9, %rs8;
+; CHECK-SM80-NEXT:    max.u16 %rs11, %rs2, %rs6;
+; CHECK-SM80-NEXT:    max.u16 %rs12, %rs4, %rs11;
+; CHECK-SM80-NEXT:    max.u16 %rs13, %rs10, %rs12;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_umax_i16_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<12>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.u32 %r1, [reduce_umax_i16_nonpow2_param_0+8];
+; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM100-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umax_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.u16 %rs7, [reduce_umax_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    mov.b16 %rs8, 0;
+; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    max.u16x2 %r5, %r3, %r4;
+; CHECK-SM100-NEXT:    max.u16x2 %r6, %r2, %r1;
+; CHECK-SM100-NEXT:    max.u16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-SM100-NEXT:    max.u16 %rs11, %rs9, %rs10;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.umax(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_umax_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_umax_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_umax_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_umax_i32_param_0];
+; CHECK-NEXT:    max.u32 %r9, %r3, %r7;
+; CHECK-NEXT:    max.u32 %r10, %r1, %r5;
+; CHECK-NEXT:    max.u32 %r11, %r4, %r8;
+; CHECK-NEXT:    max.u32 %r12, %r2, %r6;
+; CHECK-NEXT:    max.u32 %r13, %r12, %r11;
+; CHECK-NEXT:    max.u32 %r14, %r10, %r9;
+; CHECK-NEXT:    max.u32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.umax(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_umax_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_umax_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_umax_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_umax_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_umax_i32_nonpow2_param_0];
+; CHECK-NEXT:    max.u32 %r8, %r3, %r7;
+; CHECK-NEXT:    max.u32 %r9, %r1, %r5;
+; CHECK-NEXT:    max.u32 %r10, %r9, %r8;
+; CHECK-NEXT:    max.u32 %r11, %r2, %r6;
+; CHECK-NEXT:    max.u32 %r12, %r4, %r11;
+; CHECK-NEXT:    max.u32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.umax(<7 x i32> %in)
+  ret i32 %res
+}
+
+define i16 @reduce_umin_i16(<8 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_umin_i16(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<16>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<6>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_umin_i16_param_0];
+; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; CHECK-SM80-NEXT:    min.u16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
+; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
+; CHECK-SM80-NEXT:    min.u16 %rs10, %rs8, %rs6;
+; CHECK-SM80-NEXT:    min.u16 %rs11, %rs4, %rs2;
+; CHECK-SM80-NEXT:    min.u16 %rs12, %rs9, %rs7;
+; CHECK-SM80-NEXT:    min.u16 %rs13, %rs12, %rs11;
+; CHECK-SM80-NEXT:    min.u16 %rs14, %rs10, %rs5;
+; CHECK-SM80-NEXT:    min.u16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_umin_i16(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_umin_i16_param_0];
+; CHECK-SM100-NEXT:    min.u16x2 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    min.u16x2 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    min.u16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    min.u16x2 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.umin(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_umin_i16_nonpow2(<7 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_umin_i16_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.u32 %r1, [reduce_umin_i16_nonpow2_param_0+8];
+; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM80-NEXT:    ld.param.u16 %rs7, [reduce_umin_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umin_i16_nonpow2_param_0];
+; CHECK-SM80-NEXT:    min.u16 %rs8, %rs3, %rs7;
+; CHECK-SM80-NEXT:    min.u16 %rs9, %rs1, %rs5;
+; CHECK-SM80-NEXT:    min.u16 %rs10, %rs9, %rs8;
+; CHECK-SM80-NEXT:    min.u16 %rs11, %rs2, %rs6;
+; CHECK-SM80-NEXT:    min.u16 %rs12, %rs4, %rs11;
+; CHECK-SM80-NEXT:    min.u16 %rs13, %rs10, %rs12;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_umin_i16_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<12>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.u32 %r1, [reduce_umin_i16_nonpow2_param_0+8];
+; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM100-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umin_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.u16 %rs7, [reduce_umin_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    mov.b16 %rs8, -1;
+; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    min.u16x2 %r5, %r3, %r4;
+; CHECK-SM100-NEXT:    min.u16x2 %r6, %r2, %r1;
+; CHECK-SM100-NEXT:    min.u16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-SM100-NEXT:    min.u16 %rs11, %rs9, %rs10;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.umin(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_umin_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_umin_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_umin_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_umin_i32_param_0];
+; CHECK-NEXT:    min.u32 %r9, %r3, %r7;
+; CHECK-NEXT:    min.u32 %r10, %r1, %r5;
+; CHECK-NEXT:    min.u32 %r11, %r4, %r8;
+; CHECK-NEXT:    min.u32 %r12, %r2, %r6;
+; CHECK-NEXT:    min.u32 %r13, %r12, %r11;
+; CHECK-NEXT:    min.u32 %r14, %r10, %r9;
+; CHECK-NEXT:    min.u32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.umin(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_umin_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_umin_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_umin_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_umin_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_umin_i32_nonpow2_param_0];
+; CHECK-NEXT:    min.u32 %r8, %r3, %r7;
+; CHECK-NEXT:    min.u32 %r9, %r1, %r5;
+; CHECK-NEXT:    min.u32 %r10, %r9, %r8;
+; CHECK-NEXT:    min.u32 %r11, %r2, %r6;
+; CHECK-NEXT:    min.u32 %r12, %r4, %r11;
+; CHECK-NEXT:    min.u32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.umin(<7 x i32> %in)
+  ret i32 %res
+}
+
+define i16 @reduce_smax_i16(<8 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_smax_i16(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<16>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<6>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_smax_i16_param_0];
+; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; CHECK-SM80-NEXT:    max.s16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
+; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
+; CHECK-SM80-NEXT:    max.s16 %rs10, %rs8, %rs6;
+; CHECK-SM80-NEXT:    max.s16 %rs11, %rs4, %rs2;
+; CHECK-SM80-NEXT:    max.s16 %rs12, %rs9, %rs7;
+; CHECK-SM80-NEXT:    max.s16 %rs13, %rs12, %rs11;
+; CHECK-SM80-NEXT:    max.s16 %rs14, %rs10, %rs5;
+; CHECK-SM80-NEXT:    max.s16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_smax_i16(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_smax_i16_param_0];
+; CHECK-SM100-NEXT:    max.s16x2 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    max.s16x2 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    max.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    max.s16x2 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.smax(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_smax_i16_nonpow2(<7 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_smax_i16_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.u32 %r1, [reduce_smax_i16_nonpow2_param_0+8];
+; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM80-NEXT:    ld.param.u16 %rs7, [reduce_smax_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smax_i16_nonpow2_param_0];
+; CHECK-SM80-NEXT:    max.s16 %rs8, %rs3, %rs7;
+; CHECK-SM80-NEXT:    max.s16 %rs9, %rs1, %rs5;
+; CHECK-SM80-NEXT:    max.s16 %rs10, %rs9, %rs8;
+; CHECK-SM80-NEXT:    max.s16 %rs11, %rs2, %rs6;
+; CHECK-SM80-NEXT:    max.s16 %rs12, %rs4, %rs11;
+; CHECK-SM80-NEXT:    max.s16 %rs13, %rs10, %rs12;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_smax_i16_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<12>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.u32 %r1, [reduce_smax_i16_nonpow2_param_0+8];
+; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM100-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smax_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.u16 %rs7, [reduce_smax_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    mov.b16 %rs8, -32768;
+; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    max.s16x2 %r5, %r3, %r4;
+; CHECK-SM100-NEXT:    max.s16x2 %r6, %r2, %r1;
+; CHECK-SM100-NEXT:    max.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-SM100-NEXT:    max.s16 %rs11, %rs9, %rs10;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.smax(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_smax_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_smax_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_smax_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_smax_i32_param_0];
+; CHECK-NEXT:    max.s32 %r9, %r3, %r7;
+; CHECK-NEXT:    max.s32 %r10, %r1, %r5;
+; CHECK-NEXT:    max.s32 %r11, %r4, %r8;
+; CHECK-NEXT:    max.s32 %r12, %r2, %r6;
+; CHECK-NEXT:    max.s32 %r13, %r12, %r11;
+; CHECK-NEXT:    max.s32 %r14, %r10, %r9;
+; CHECK-NEXT:    max.s32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.smax(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_smax_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_smax_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_smax_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_smax_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_smax_i32_nonpow2_param_0];
+; CHECK-NEXT:    max.s32 %r8, %r3, %r7;
+; CHECK-NEXT:    max.s32 %r9, %r1, %r5;
+; CHECK-NEXT:    max.s32 %r10, %r9, %r8;
+; CHECK-NEXT:    max.s32 %r11, %r2, %r6;
+; CHECK-NEXT:    max.s32 %r12, %r4, %r11;
+; CHECK-NEXT:    max.s32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.smax(<7 x i32> %in)
+  ret i32 %res
+}
+
+define i16 @reduce_smin_i16(<8 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_smin_i16(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<16>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<6>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_smin_i16_param_0];
+; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; CHECK-SM80-NEXT:    min.s16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
+; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
+; CHECK-SM80-NEXT:    min.s16 %rs10, %rs8, %rs6;
+; CHECK-SM80-NEXT:    min.s16 %rs11, %rs4, %rs2;
+; CHECK-SM80-NEXT:    min.s16 %rs12, %rs9, %rs7;
+; CHECK-SM80-NEXT:    min.s16 %rs13, %rs12, %rs11;
+; CHECK-SM80-NEXT:    min.s16 %rs14, %rs10, %rs5;
+; CHECK-SM80-NEXT:    min.s16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_smin_i16(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_smin_i16_param_0];
+; CHECK-SM100-NEXT:    min.s16x2 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    min.s16x2 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    min.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    min.s16x2 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.smin(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_smin_i16_nonpow2(<7 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_smin_i16_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.u32 %r1, [reduce_smin_i16_nonpow2_param_0+8];
+; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM80-NEXT:    ld.param.u16 %rs7, [reduce_smin_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smin_i16_nonpow2_param_0];
+; CHECK-SM80-NEXT:    min.s16 %rs8, %rs3, %rs7;
+; CHECK-SM80-NEXT:    min.s16 %rs9, %rs1, %rs5;
+; CHECK-SM80-NEXT:    min.s16 %rs10, %rs9, %rs8;
+; CHECK-SM80-NEXT:    min.s16 %rs11, %rs2, %rs6;
+; CHECK-SM80-NEXT:    min.s16 %rs12, %rs4, %rs11;
+; CHECK-SM80-NEXT:    min.s16 %rs13, %rs10, %rs12;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_smin_i16_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<12>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.u32 %r1, [reduce_smin_i16_nonpow2_param_0+8];
+; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-SM100-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smin_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.u16 %rs7, [reduce_smin_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    mov.b16 %rs8, 32767;
+; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    min.s16x2 %r5, %r3, %r4;
+; CHECK-SM100-NEXT:    min.s16x2 %r6, %r2, %r1;
+; CHECK-SM100-NEXT:    min.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-SM100-NEXT:    min.s16 %rs11, %rs9, %rs10;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.smin(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_smin_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_smin_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_smin_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_smin_i32_param_0];
+; CHECK-NEXT:    min.s32 %r9, %r3, %r7;
+; CHECK-NEXT:    min.s32 %r10, %r1, %r5;
+; CHECK-NEXT:    min.s32 %r11, %r4, %r8;
+; CHECK-NEXT:    min.s32 %r12, %r2, %r6;
+; CHECK-NEXT:    min.s32 %r13, %r12, %r11;
+; CHECK-NEXT:    min.s32 %r14, %r10, %r9;
+; CHECK-NEXT:    min.s32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.smin(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_smin_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_smin_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_smin_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_smin_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_smin_i32_nonpow2_param_0];
+; CHECK-NEXT:    min.s32 %r8, %r3, %r7;
+; CHECK-NEXT:    min.s32 %r9, %r1, %r5;
+; CHECK-NEXT:    min.s32 %r10, %r9, %r8;
+; CHECK-NEXT:    min.s32 %r11, %r2, %r6;
+; CHECK-NEXT:    min.s32 %r12, %r4, %r11;
+; CHECK-NEXT:    min.s32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.smin(<7 x i32> %in)
+  ret i32 %res
+}
+
+define i16 @reduce_and_i16(<8 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_and_i16(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_and_i16_param_0];
+; CHECK-SM80-NEXT:    and.b32 %r5, %r2, %r4;
+; CHECK-SM80-NEXT:    and.b32 %r6, %r1, %r3;
+; CHECK-SM80-NEXT:    and.b32 %r7, %r6, %r5;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
+; CHECK-SM80-NEXT:    // implicit-def: %rs2
+; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM80-NEXT:    and.b32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_and_i16(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_and_i16_param_0];
+; CHECK-SM100-NEXT:    and.b32 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    and.b32 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    and.b32 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    and.b32 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.and(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_and_i16_nonpow2(<7 x i16> %in) {
+; CHECK-LABEL: reduce_and_i16_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [reduce_and_i16_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_and_i16_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.u16 %rs7, [reduce_and_i16_nonpow2_param_0+12];
+; CHECK-NEXT:    mov.b16 %rs8, -1;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    and.b32 %r5, %r3, %r4;
+; CHECK-NEXT:    and.b32 %r6, %r2, %r1;
+; CHECK-NEXT:    and.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    and.b16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    cvt.u32.u16 %r8, %rs11;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.and(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_and_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_and_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_and_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_and_i32_param_0];
+; CHECK-NEXT:    and.b32 %r9, %r3, %r7;
+; CHECK-NEXT:    and.b32 %r10, %r1, %r5;
+; CHECK-NEXT:    and.b32 %r11, %r4, %r8;
+; CHECK-NEXT:    and.b32 %r12, %r2, %r6;
+; CHECK-NEXT:    and.b32 %r13, %r12, %r11;
+; CHECK-NEXT:    and.b32 %r14, %r10, %r9;
+; CHECK-NEXT:    and.b32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.and(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_and_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_and_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_and_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_and_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_and_i32_nonpow2_param_0];
+; CHECK-NEXT:    and.b32 %r8, %r3, %r7;
+; CHECK-NEXT:    and.b32 %r9, %r1, %r5;
+; CHECK-NEXT:    and.b32 %r10, %r9, %r8;
+; CHECK-NEXT:    and.b32 %r11, %r2, %r6;
+; CHECK-NEXT:    and.b32 %r12, %r11, %r4;
+; CHECK-NEXT:    and.b32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.and(<7 x i32> %in)
+  ret i32 %res
+}
+
+define i16 @reduce_or_i16(<8 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_or_i16(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_or_i16_param_0];
+; CHECK-SM80-NEXT:    or.b32 %r5, %r2, %r4;
+; CHECK-SM80-NEXT:    or.b32 %r6, %r1, %r3;
+; CHECK-SM80-NEXT:    or.b32 %r7, %r6, %r5;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
+; CHECK-SM80-NEXT:    // implicit-def: %rs2
+; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM80-NEXT:    or.b32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_or_i16(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_or_i16_param_0];
+; CHECK-SM100-NEXT:    or.b32 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    or.b32 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    or.b32 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    or.b32 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.or(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_or_i16_nonpow2(<7 x i16> %in) {
+; CHECK-LABEL: reduce_or_i16_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [reduce_or_i16_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_or_i16_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.u16 %rs7, [reduce_or_i16_nonpow2_param_0+12];
+; CHECK-NEXT:    mov.b16 %rs8, 0;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    or.b32 %r5, %r3, %r4;
+; CHECK-NEXT:    or.b32 %r6, %r2, %r1;
+; CHECK-NEXT:    or.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    or.b16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    cvt.u32.u16 %r8, %rs11;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.or(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_or_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_or_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_or_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_or_i32_param_0];
+; CHECK-NEXT:    or.b32 %r9, %r3, %r7;
+; CHECK-NEXT:    or.b32 %r10, %r1, %r5;
+; CHECK-NEXT:    or.b32 %r11, %r4, %r8;
+; CHECK-NEXT:    or.b32 %r12, %r2, %r6;
+; CHECK-NEXT:    or.b32 %r13, %r12, %r11;
+; CHECK-NEXT:    or.b32 %r14, %r10, %r9;
+; CHECK-NEXT:    or.b32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.or(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_or_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_or_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_or_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_or_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_or_i32_nonpow2_param_0];
+; CHECK-NEXT:    or.b32 %r8, %r3, %r7;
+; CHECK-NEXT:    or.b32 %r9, %r1, %r5;
+; CHECK-NEXT:    or.b32 %r10, %r9, %r8;
+; CHECK-NEXT:    or.b32 %r11, %r2, %r6;
+; CHECK-NEXT:    or.b32 %r12, %r11, %r4;
+; CHECK-NEXT:    or.b32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.or(<7 x i32> %in)
+  ret i32 %res
+}
+
+define i16 @reduce_xor_i16(<8 x i16> %in) {
+; CHECK-SM80-LABEL: reduce_xor_i16(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_xor_i16_param_0];
+; CHECK-SM80-NEXT:    xor.b32 %r5, %r2, %r4;
+; CHECK-SM80-NEXT:    xor.b32 %r6, %r1, %r3;
+; CHECK-SM80-NEXT:    xor.b32 %r7, %r6, %r5;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
+; CHECK-SM80-NEXT:    // implicit-def: %rs2
+; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM80-NEXT:    xor.b32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_xor_i16(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_xor_i16_param_0];
+; CHECK-SM100-NEXT:    xor.b32 %r5, %r2, %r4;
+; CHECK-SM100-NEXT:    xor.b32 %r6, %r1, %r3;
+; CHECK-SM100-NEXT:    xor.b32 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
+; CHECK-SM100-NEXT:    // implicit-def: %rs2
+; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
+; CHECK-SM100-NEXT:    xor.b32 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.xor(<8 x i16> %in)
+  ret i16 %res
+}
+
+define i16 @reduce_xor_i16_nonpow2(<7 x i16> %in) {
+; CHECK-LABEL: reduce_xor_i16_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [reduce_xor_i16_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v4.u16 {%rs1, %rs2, %rs3, %rs4}, [reduce_xor_i16_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.u16 %rs7, [reduce_xor_i16_nonpow2_param_0+12];
+; CHECK-NEXT:    mov.b16 %rs8, 0;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    xor.b32 %r5, %r3, %r4;
+; CHECK-NEXT:    xor.b32 %r6, %r2, %r1;
+; CHECK-NEXT:    xor.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    xor.b16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    cvt.u32.u16 %r8, %rs11;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
+  %res = call i16 @llvm.vector.reduce.xor(<7 x i16> %in)
+  ret i16 %res
+}
+
+define i32 @reduce_xor_i32(<8 x i32> %in) {
+; CHECK-LABEL: reduce_xor_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.u32 {%r5, %r6, %r7, %r8}, [reduce_xor_i32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_xor_i32_param_0];
+; CHECK-NEXT:    xor.b32 %r9, %r3, %r7;
+; CHECK-NEXT:    xor.b32 %r10, %r1, %r5;
+; CHECK-NEXT:    xor.b32 %r11, %r4, %r8;
+; CHECK-NEXT:    xor.b32 %r12, %r2, %r6;
+; CHECK-NEXT:    xor.b32 %r13, %r12, %r11;
+; CHECK-NEXT:    xor.b32 %r14, %r10, %r9;
+; CHECK-NEXT:    xor.b32 %r15, %r14, %r13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.xor(<8 x i32> %in)
+  ret i32 %res
+}
+
+define i32 @reduce_xor_i32_nonpow2(<7 x i32> %in) {
+; CHECK-LABEL: reduce_xor_i32_nonpow2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r7, [reduce_xor_i32_nonpow2_param_0+24];
+; CHECK-NEXT:    ld.param.v2.u32 {%r5, %r6}, [reduce_xor_i32_nonpow2_param_0+16];
+; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [reduce_xor_i32_nonpow2_param_0];
+; CHECK-NEXT:    xor.b32 %r8, %r3, %r7;
+; CHECK-NEXT:    xor.b32 %r9, %r1, %r5;
+; CHECK-NEXT:    xor.b32 %r10, %r9, %r8;
+; CHECK-NEXT:    xor.b32 %r11, %r2, %r6;
+; CHECK-NEXT:    xor.b32 %r12, %r11, %r4;
+; CHECK-NEXT:    xor.b32 %r13, %r10, %r12;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT:    ret;
+  %res = call i32 @llvm.vector.reduce.xor(<7 x i32> %in)
+  ret i32 %res
+}

From 6d765e1c916aba7a038f5ace963001fca4fc9314 Mon Sep 17 00:00:00 2001
From: Owen Anderson <resistor@mac.com>
Date: Tue, 22 Apr 2025 10:23:27 +1200
Subject: [PATCH 663/710] Reapply 19c708c "FunctionDecl::getFunctionTypeLoc:
 ignore function type attributes (#118420)" (#136484)

Avoid using PreservesMost in the testcase as it is not supported on all
targets.

Original PR #118290.

Co-authored-by: Robert Dazi <14996868+v01dXYZ@users.noreply.github.com>
Co-authored-by: v01dxyz <v01dxyz@v01d.xyz>
---
 .../clangd/unittests/InlayHintTests.cpp       | 14 ++--
 clang/lib/AST/Decl.cpp                        | 21 +++++-
 clang/unittests/AST/AttrTest.cpp              | 69 +++++++++++++++++++
 3 files changed, 96 insertions(+), 8 deletions(-)

diff --git a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
index 77d78b8777fe3..030e499577706 100644
--- a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
+++ b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
@@ -1577,19 +1577,21 @@ TEST(TypeHints, Aliased) {
 }
 
 TEST(TypeHints, CallingConvention) {
-  // Check that we don't crash for lambdas without a FunctionTypeLoc
+  // Check that we don't crash for lambdas with an annotation
   // https://github.com/clangd/clangd/issues/2223
-  std::string Code = R"cpp(
+  Annotations Source(R"cpp(
     void test() {
-      []() __cdecl {};
+      []($lambda[[)]]__cdecl {};
     }
-  )cpp";
-  TestTU TU = TestTU::withCode(Code);
+  )cpp");
+  TestTU TU = TestTU::withCode(Source.code());
   TU.ExtraArgs.push_back("--target=x86_64-w64-mingw32");
   TU.PredefineMacros = true; // for the __cdecl
   auto AST = TU.build();
 
-  EXPECT_THAT(hintsOfKind(AST, InlayHintKind::Type), IsEmpty());
+  EXPECT_THAT(
+      hintsOfKind(AST, InlayHintKind::Type),
+      ElementsAre(HintMatcher(ExpectedHint{"-> void", "lambda"}, Source)));
 }
 
 TEST(TypeHints, Decltype) {
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index ad1cb01592e9b..1d9208f0e1c72 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -3910,8 +3910,25 @@ bool FunctionDecl::doesDeclarationForceExternallyVisibleDefinition() const {
 
 FunctionTypeLoc FunctionDecl::getFunctionTypeLoc() const {
   const TypeSourceInfo *TSI = getTypeSourceInfo();
-  return TSI ? TSI->getTypeLoc().IgnoreParens().getAs<FunctionTypeLoc>()
-             : FunctionTypeLoc();
+
+  if (!TSI)
+    return FunctionTypeLoc();
+
+  TypeLoc TL = TSI->getTypeLoc();
+  FunctionTypeLoc FTL;
+
+  while (!(FTL = TL.getAs<FunctionTypeLoc>())) {
+    if (const auto PTL = TL.getAs<ParenTypeLoc>())
+      TL = PTL.getInnerLoc();
+    else if (const auto ATL = TL.getAs<AttributedTypeLoc>())
+      TL = ATL.getEquivalentTypeLoc();
+    else if (const auto MQTL = TL.getAs<MacroQualifiedTypeLoc>())
+      TL = MQTL.getInnerLoc();
+    else
+      break;
+  }
+
+  return FTL;
 }
 
 SourceRange FunctionDecl::getReturnTypeSourceRange() const {
diff --git a/clang/unittests/AST/AttrTest.cpp b/clang/unittests/AST/AttrTest.cpp
index 46c3f5729021e..bbf1c0f9a382b 100644
--- a/clang/unittests/AST/AttrTest.cpp
+++ b/clang/unittests/AST/AttrTest.cpp
@@ -86,6 +86,22 @@ TEST(Attr, AnnotateType) {
     struct S { int mem; };
     int [[clang::annotate_type("int")]]
     S::* [[clang::annotate_type("ptr_to_mem")]] ptr_to_member = &S::mem;
+
+    // Function Type Attributes
+    __attribute__((noreturn)) int f_noreturn();
+
+    #define NO_RETURN __attribute__((noreturn))
+    NO_RETURN int f_macro_attribue();
+
+    int (__attribute__((noreturn)) f_paren_attribute)();
+
+    int (
+      NO_RETURN
+      (
+        __attribute__((warn_unused_result))
+        (f_w_paren_and_attr)
+      )
+    ) ();
   )cpp");
 
   {
@@ -153,6 +169,59 @@ TEST(Attr, AnnotateType) {
     EXPECT_EQ(IntTL.getType(), AST->getASTContext().IntTy);
   }
 
+  {
+    const FunctionDecl *Func = getFunctionNode(AST.get(), "f_noreturn");
+    const FunctionTypeLoc FTL = Func->getFunctionTypeLoc();
+    const FunctionType *FT = FTL.getTypePtr();
+
+    EXPECT_TRUE(FT->getNoReturnAttr());
+  }
+
+  {
+    for (auto should_have_func_type_loc : {
+             "f_macro_attribue",
+             "f_paren_attribute",
+             "f_w_paren_and_attr",
+         }) {
+      llvm::errs() << "O: " << should_have_func_type_loc << "\n";
+      const FunctionDecl *Func =
+          getFunctionNode(AST.get(), should_have_func_type_loc);
+
+      EXPECT_TRUE(Func->getFunctionTypeLoc());
+    }
+  }
+
+  // The following test verifies getFunctionTypeLoc returns a type
+  // which takes into account the attribute (instead of only the nake
+  // type).
+  //
+  // This is hard to do with C/C++ because it seems using a function
+  // type attribute with a C/C++ function declaration only results
+  // with either:
+  //
+  // 1. It does NOT produce any AttributedType (for example it only
+  //   sets one flag of the FunctionType's ExtInfo, e.g. NoReturn).
+  // 2. It produces an AttributedType with modified type and
+  //   equivalent type that are equal (for example, that's what
+  //   happens with Calling Convention attributes).
+  //
+  // Fortunately, ObjC has one specific function type attribute that
+  // creates an AttributedType with different modified type and
+  // equivalent type.
+  auto AST_ObjC = buildASTFromCodeWithArgs(
+      R"objc(
+    __attribute__((ns_returns_retained)) id f();
+  )objc",
+      {"-fobjc-arc", "-fsyntax-only", "-fobjc-runtime=macosx-10.7"},
+      "input.mm");
+  {
+    const FunctionDecl *f = getFunctionNode(AST_ObjC.get(), "f");
+    const FunctionTypeLoc FTL = f->getFunctionTypeLoc();
+
+    const FunctionType *FT = FTL.getTypePtr();
+    EXPECT_TRUE(FT->getExtInfo().getProducesResult());
+  }
+
   // Test type annotation on an `__auto_type` type in C mode.
   AST = buildASTFromCodeWithArgs(R"c(
     __auto_type [[clang::annotate_type("auto")]] auto_var = 1;

From 3ca2fa704e149ca22bd9739101ee7d5c498eb1a6 Mon Sep 17 00:00:00 2001
From: Theo de Magalhaes <theodemagalhaes@icloud.com>
Date: Tue, 22 Apr 2025 00:25:14 +0200
Subject: [PATCH 664/710] [CLANG][MS-STRUCT] bitfield padding warning presents
 padding to exact bit count (#136062)

Aims to fix #131647.
---
 clang/lib/AST/RecordLayoutBuilder.cpp           |  4 +++-
 clang/test/SemaCXX/windows-Wpadded-bitfield.cpp | 15 +++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index dd932f6179f88..aacc079f2521d 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -1539,6 +1539,7 @@ void ItaniumRecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
   uint64_t StorageUnitSize = FieldInfo.Width;
   unsigned FieldAlign = FieldInfo.Align;
   bool AlignIsRequired = FieldInfo.isAlignRequired();
+  unsigned char PaddingInLastUnit = 0;
 
   // UnfilledBitsInLastUnit is the difference between the end of the
   // last allocated bitfield (i.e. the first bit offset available for
@@ -1611,6 +1612,7 @@ void ItaniumRecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
       if (!LastBitfieldStorageUnitSize && !FieldSize)
         FieldAlign = 1;
 
+      PaddingInLastUnit = UnfilledBitsInLastUnit;
       UnfilledBitsInLastUnit = 0;
       LastBitfieldStorageUnitSize = 0;
     }
@@ -1707,7 +1709,7 @@ void ItaniumRecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
   // For purposes of diagnostics, we're going to simultaneously
   // compute the field offsets that we would have used if we weren't
   // adding any alignment padding or if the field weren't packed.
-  uint64_t UnpaddedFieldOffset = FieldOffset;
+  uint64_t UnpaddedFieldOffset = FieldOffset - PaddingInLastUnit;
   uint64_t UnpackedFieldOffset = FieldOffset;
 
   // Check if we need to add padding to fit the bitfield within an
diff --git a/clang/test/SemaCXX/windows-Wpadded-bitfield.cpp b/clang/test/SemaCXX/windows-Wpadded-bitfield.cpp
index ee5a57124eca5..28917ffaadb6f 100644
--- a/clang/test/SemaCXX/windows-Wpadded-bitfield.cpp
+++ b/clang/test/SemaCXX/windows-Wpadded-bitfield.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-windows-msvc -fsyntax-only -verify -Wpadded %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsyntax-only -verify -Wpadded %s
 
 struct __attribute__((ms_struct)) BitfieldStruct { // expected-warning {{padding size of 'BitfieldStruct' with 3 bytes to alignment boundary}}
   char c : 1;
@@ -24,9 +25,23 @@ struct __attribute__((ms_struct)) DifferentUnitSizeBitfield { // expected-warnin
   char i; // expected-warning {{padding struct 'DifferentUnitSizeBitfield' with 31 bits to align 'i'}}
 };
 
+struct __attribute__((ms_struct)) BitfieldBigPadding { // expected-warning {{padding size of 'BitfieldBigPadding' with 63 bits to alignment boundary}}
+  long long x;
+  char a : 1;
+  long long b : 1; // expected-warning {{padding struct 'BitfieldBigPadding' with 63 bits to align 'b'}}
+};
+
+struct __attribute__((ms_struct)) SameUnitSizeMultiple { // expected-warning {{padding size of 'SameUnitSizeMultiple' with 2 bits to alignment boundary}}
+  char c : 1;
+  char cc : 2;
+  char ccc : 3;
+};
+
 int main() {
   BitfieldStruct b;
   SevenBitfieldStruct s;
   SameUnitSizeBitfield su;
   DifferentUnitSizeBitfield du;
+  BitfieldBigPadding bbp;
+  SameUnitSizeMultiple susm;
 }

From c40d3a411ce3f0a35ae4f3d2f0e1e7df45673856 Mon Sep 17 00:00:00 2001
From: Maryam Moghadas <maryammo@ca.ibm.com>
Date: Mon, 21 Apr 2025 18:39:44 -0400
Subject: [PATCH 665/710] [PowerPC] Add dense math bfloat16 floating-point
 outer-product accumulate to DMR instructions (#133109)

This patch adds the following Dense Math Facility bfloat16
floating-point calculation instructions: dmxvbf16gerx2,
dmxvbf16gerx2pp,dmxvbf16gerx2pn, dmxvbf16gerx2np, dmxvbf16gerx2nn,
pmdmxvbf16gerx2, pmdmxvbf16gerx2pp, pmdmxvbf16gerx2pn,
pmdmxvbf16gerx2np, pmdmxvbf16gerx2nn, along with their corresponding
intrinsics and tests.
---
 llvm/include/llvm/IR/IntrinsicsPowerPC.td     |  23 +
 llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td  | 162 +++++-
 .../test/CodeGen/PowerPC/dmf-outer-product.ll | 484 ++++++++++++++++++
 .../PowerPC/ppc-encoding-ISAFuture.txt        |  32 +-
 .../PowerPC/ppc64le-encoding-ISAFuture.txt    |  30 ++
 llvm/test/MC/PowerPC/ppc-encoding-ISAFuture.s |  50 ++
 6 files changed, 779 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index e4d39134a4a25..b57102ef68f09 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -280,6 +280,22 @@ multiclass PowerPC_MMA_ACC_PP_Intrinsic<list<LLVMType> args> {
                                  [IntrNoMem]>;
 }
 
+multiclass PowerPC_MMA_DMR_Intrinsic<list<LLVMType> args> {
+  def NAME: DefaultAttrsIntrinsic<[llvm_v1024i1_ty], args, [IntrNoMem]>;
+  def pp : DefaultAttrsIntrinsic<[llvm_v1024i1_ty],
+                                 !listconcat([llvm_v1024i1_ty], args),
+                                 [IntrNoMem]>;
+  def pn : DefaultAttrsIntrinsic<[llvm_v1024i1_ty],
+                                 !listconcat([llvm_v1024i1_ty], args),
+                                 [IntrNoMem]>;
+  def np : DefaultAttrsIntrinsic<[llvm_v1024i1_ty],
+                                 !listconcat([llvm_v1024i1_ty], args),
+                                 [IntrNoMem]>;
+  def nn : DefaultAttrsIntrinsic<[llvm_v1024i1_ty],
+                                 !listconcat([llvm_v1024i1_ty], args),
+                                 [IntrNoMem]>;
+}
+
 multiclass PowerPC_MMA_DMR_PP_Intrinsic<list<LLVMType> args> {
   def NAME: DefaultAttrsIntrinsic<[llvm_v1024i1_ty], args, [IntrNoMem]>;
   def pp : DefaultAttrsIntrinsic<[llvm_v1024i1_ty],
@@ -1732,6 +1748,13 @@ let TargetPrefix = "ppc" in {
                             [llvm_v1024i1_ty, llvm_v256i1_ty, llvm_v16i8_ty,
                              llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
                             [IntrNoMem]>;
+
+  // MMA+ Reduced-Precision: bfloat16 Outer Product Intrinsic Definitions.
+  defm int_ppc_mma_dmxvbf16gerx2 :
+       PowerPC_MMA_DMR_Intrinsic<[llvm_v256i1_ty, llvm_v16i8_ty]>;
+  defm int_ppc_mma_pmdmxvbf16gerx2 :
+       PowerPC_MMA_DMR_Intrinsic<[llvm_v256i1_ty, llvm_v16i8_ty, llvm_i32_ty,
+                                     llvm_i32_ty, llvm_i32_ty]>;
 }
 
 // XL Compat intrinsics.
diff --git a/llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td b/llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td
index 15215f7dc5faa..e9c69f7c16858 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td
@@ -95,7 +95,7 @@ class MMIRR_XX3Form_X8YP4_XAp5B6<bits<6> opcode, bits<8> xo, dag OOL, dag IOL,
                                  list<dag> pattern>
   : PI<1, opcode, OOL, IOL, asmstr, itin> {
   bits<3> AT;
-  bits<6> XAp;
+  bits<5> XAp;
   bits<6> XB;
   bits<8> XMSK;
   bits<4> YMSK;
@@ -123,6 +123,40 @@ class MMIRR_XX3Form_X8YP4_XAp5B6<bits<6> opcode, bits<8> xo, dag OOL, dag IOL,
   let Inst{63} = 0;
 }
 
+class MMIRR_XX3Form_X8Y4P2_XAp5B6<bits<6> opcode, bits<8> xo, dag OOL, dag IOL,
+                                 string asmstr, InstrItinClass itin,
+                                 list<dag> pattern>
+  : PI<1, opcode, OOL, IOL, asmstr, itin> {
+  bits<3> AT;
+  bits<5> XAp;
+  bits<6> XB;
+  bits<8> XMSK;
+  bits<4> YMSK;
+  bits<2> PMSK;
+
+  let Pattern = pattern;
+
+  // The prefix.
+  let Inst{6-7} = 3;
+  let Inst{8-11} = 9;
+  let Inst{12-15} = 0;
+  let Inst{16-17} = PMSK;
+  let Inst{18-19} = 0;
+  let Inst{20-27} = XMSK;
+  let Inst{28-31} = YMSK;
+
+  // The instruction.
+  let Inst{38-40} = AT;
+  let Inst{41-42} = 0;
+  let Inst{43-46} = XAp{3-0};
+  let Inst{47} = 0;
+  let Inst{48-52} = XB{4-0};
+  let Inst{53-60} = xo;
+  let Inst{61} = XAp{4};
+  let Inst{62} = XB{5};
+  let Inst{63} = 0;
+}
+
 multiclass DMR_UM_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
                        string asmstr> {
   let Predicates = [MMA, IsISAFuture] in {
@@ -159,6 +193,83 @@ multiclass DMR_UM_M448_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
   }
 }
 
+multiclass DMR_BF16_UM_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
+                       string asmstr> {
+  let Predicates = [MMA, IsISAFuture] in {
+  def NAME :
+    XX3Form_AT3_XAp5B6<opcode, !or(xo, 0x11), (outs dmr:$AT), IOL,
+                     !strconcat(asmbase#" ", asmstr), IIC_VecFP, []>,
+    RegConstraint<"@earlyclobber $AT">;
+  def PP :
+    XX3Form_AT3_XAp5B6<opcode, xo, (outs dmr:$AT), !con((ins dmr:$ATi), IOL),
+                     !strconcat(asmbase#"pp ", asmstr), IIC_VecFP, []>,
+    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
+  }
+}
+
+multiclass DMR_UM_M284_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
+                            string asmstr> {
+  defm NAME : DMR_BF16_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
+  let Predicates = [MMA, PrefixInstrs, IsISAFuture] in {
+  def PM#NAME :
+    MMIRR_XX3Form_X8Y4P2_XAp5B6<
+      opcode, !or(xo, 0x11), (outs dmr:$AT),
+      !con(IOL, (ins u8imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK)),
+      !strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
+      IIC_VecFP, []>,
+    RegConstraint<"@earlyclobber $AT">;
+  def PM#NAME#PP :
+    MMIRR_XX3Form_X8Y4P2_XAp5B6<
+      opcode, xo, (outs dmr:$AT),
+      !con((ins dmr:$ATi), !con(IOL, (ins u8imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
+      !strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
+      IIC_VecFP, []>,
+    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
+  }
+}
+
+multiclass DMR_NEG_UM_M284_XOXORf939a0<bits<6> opcode, bits<8> xo, dag IOL,
+                                  string asmbase, string asmstr> {
+  defm NAME : DMR_UM_M284_XOEO<opcode, xo, IOL, asmbase, asmstr>;
+  let Predicates = [MMA, IsISAFuture] in {
+  def PN : XX3Form_AT3_XAp5B6<
+             opcode, !xor(xo, 0xF9), (outs dmr:$AT), !con((ins dmr:$ATi), IOL),
+             !strconcat(asmbase#"pn ", asmstr), IIC_VecFP, []>,
+           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
+  def NP : XX3Form_AT3_XAp5B6<
+             opcode, !xor(xo, 0x39), (outs dmr:$AT), !con((ins dmr:$ATi), IOL),
+             !strconcat(asmbase#"np ", asmstr), IIC_VecFP, []>,
+           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
+  def NN : XX3Form_AT3_XAp5B6<
+             opcode, !xor(xo, 0xA0), (outs dmr:$AT), !con((ins dmr:$ATi), IOL),
+             !strconcat(asmbase#"nn ", asmstr), IIC_VecFP, []>,
+           RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
+  }
+  let Predicates = [MMA, PrefixInstrs, IsISAFuture] in {
+   def PM#NAME#PN :
+    MMIRR_XX3Form_X8Y4P2_XAp5B6<
+      opcode, !xor(xo, 0xF9), (outs dmr:$AT),
+      !con((ins dmr:$ATi), !con(IOL, (ins u8imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
+      !strconcat("pm"#asmbase#"pn ", asmstr#", $XMSK, $YMSK, $PMSK"),
+      IIC_VecFP, []>,
+    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
+  def PM#NAME#NP :
+    MMIRR_XX3Form_X8Y4P2_XAp5B6<
+      opcode, !xor(xo, 0x39), (outs dmr:$AT),
+      !con((ins dmr:$ATi), !con(IOL, (ins u8imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
+      !strconcat("pm"#asmbase#"np ", asmstr#", $XMSK, $YMSK, $PMSK"),
+      IIC_VecFP, []>,
+    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
+  def PM#NAME#NN :
+    MMIRR_XX3Form_X8Y4P2_XAp5B6<
+      opcode, !xor(xo, 0xA0), (outs dmr:$AT),
+      !con((ins dmr:$ATi), !con(IOL, (ins u8imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
+      !strconcat("pm"#asmbase#"nn ", asmstr#", $XMSK, $YMSK, $PMSK"),
+      IIC_VecFP, []>,
+    RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
+  }
+}
+
 let Predicates = [IsISAFuture] in {
   def DMXXEXTFDMR512 : XX3Form_AT3_XABp5_P1<60, 226,
                                             (outs vsrprc:$XAp, vsrprc:$XBp),
@@ -231,6 +342,11 @@ let Predicates = [MMA, PrefixInstrs, IsISAFuture] in {
     RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
 }
 
+// DMXVBF16GERX2, DMXVBF16GERX2PP, DMXVBF16GERX2PN, dMXVBF16GERX2NP, DMXVBF16GERX2NN
+// PMDMXVBF16GERX2, PMDMXVBF16GERX2PP, PMDMXVBF16GERX2PN, PMDMXVBF16GERX2NP, PMDMXVBF16GERX2NN
+defm DMXVBF16GERX2 : DMR_NEG_UM_M284_XOXORf939a0<59, 74, (ins vsrprc:$XAp, vsrc:$XB),
+                                         "dmxvbf16gerx2", "$AT, $XAp, $XB">;
+
 // MMA+ Intrinsics
 let Predicates = [MMA, IsISAFuture] in {
   def : Pat<(v1024i1 (int_ppc_mma_dmxvi8gerx4 v256i1:$XAp, v16i8:$XB)),
@@ -240,6 +356,21 @@ let Predicates = [MMA, IsISAFuture] in {
 
   def : Pat<(v1024i1 (int_ppc_mma_dmxvi8gerx4spp v1024i1:$ATi, v256i1:$XAp, v16i8:$XB)),
             (DMXVI8GERX4SPP $ATi, $XAp, RCCp.BToVSRC)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_dmxvbf16gerx2 v256i1:$XAp, v16i8:$XB)),
+            (DMXVBF16GERX2 $XAp, RCCp.BToVSRC)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_dmxvbf16gerx2pp v1024i1:$ATi, v256i1:$XAp, v16i8:$XB)),
+            (DMXVBF16GERX2PP $ATi, $XAp, RCCp.BToVSRC)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_dmxvbf16gerx2pn v1024i1:$ATi, v256i1:$XAp, v16i8:$XB)),
+            (DMXVBF16GERX2PN $ATi, $XAp, RCCp.BToVSRC)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_dmxvbf16gerx2np v1024i1:$ATi, v256i1:$XAp, v16i8:$XB)),
+            (DMXVBF16GERX2NP $ATi, $XAp, RCCp.BToVSRC)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_dmxvbf16gerx2nn v1024i1:$ATi, v256i1:$XAp, v16i8:$XB)),
+            (DMXVBF16GERX2NN $ATi, $XAp, RCCp.BToVSRC)>;
 }
 
 let Predicates = [MMA, PrefixInstrs, IsISAFuture] in {
@@ -259,4 +390,33 @@ let Predicates = [MMA, PrefixInstrs, IsISAFuture] in {
                                                Msk4Imm:$PMSK)),
             (PMDMXVI8GERX4SPP $ATi, $XAp, RCCp.BToVSRC, Msk8Imm:$XMSK,
                            Msk4Imm:$YMSK, Msk4Imm:$PMSK)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_pmdmxvbf16gerx2 v256i1:$XAp, v16i8:$XB, Msk8Imm:$XMSK,
+                                            Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
+            (PMDMXVBF16GERX2 $XAp, RCCp.BToVSRC, Msk8Imm:$XMSK,
+                        Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_pmdmxvbf16gerx2pp v1024i1:$ATi, v256i1:$XAp, v16i8:$XB,
+                                              Msk8Imm:$XMSK, Msk4Imm:$YMSK,
+                                              Msk2Imm:$PMSK)),
+            (PMDMXVBF16GERX2PP $ATi, $XAp, RCCp.BToVSRC, Msk8Imm:$XMSK,
+                          Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_pmdmxvbf16gerx2pn v1024i1:$ATi, v256i1:$XAp, v16i8:$XB,
+                                               Msk8Imm:$XMSK, Msk4Imm:$YMSK,
+                                               Msk2Imm:$PMSK)),
+            (PMDMXVBF16GERX2PN $ATi, $XAp, RCCp.BToVSRC, Msk8Imm:$XMSK,
+                           Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_pmdmxvbf16gerx2np v1024i1:$ATi, v256i1:$XAp, v16i8:$XB,
+                                              Msk8Imm:$XMSK, Msk4Imm:$YMSK,
+                                              Msk2Imm:$PMSK)),
+            (PMDMXVBF16GERX2NP $ATi, $XAp, RCCp.BToVSRC, Msk8Imm:$XMSK,
+                          Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
+
+  def : Pat<(v1024i1 (int_ppc_mma_pmdmxvbf16gerx2nn v1024i1:$ATi, v256i1:$XAp, v16i8:$XB,
+                                               Msk8Imm:$XMSK, Msk4Imm:$YMSK,
+                                               Msk2Imm:$PMSK)),
+            (PMDMXVBF16GERX2NN $ATi, $XAp, RCCp.BToVSRC, Msk8Imm:$XMSK,
+                           Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
 }
diff --git a/llvm/test/CodeGen/PowerPC/dmf-outer-product.ll b/llvm/test/CodeGen/PowerPC/dmf-outer-product.ll
index ec43776509b44..c120b0d94f738 100644
--- a/llvm/test/CodeGen/PowerPC/dmf-outer-product.ll
+++ b/llvm/test/CodeGen/PowerPC/dmf-outer-product.ll
@@ -285,3 +285,487 @@ entry:
   store <1024 x i1> %call, ptr %resp, align 64
   ret void
 }
+
+declare <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2(<256 x i1>, <16 x i8>)
+
+define void @test_dmxvbf16gerx2(ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_dmxvbf16gerx2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv v2, 16(r3)
+; CHECK-NEXT:    lxv vs0, 0(r4)
+; CHECK-NEXT:    lxv v3, 0(r3)
+; CHECK-NEXT:    dmxvbf16gerx2 dmr0, vsp34, vs0
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r5)
+; CHECK-NEXT:    stxvp vsp36, 64(r5)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r5)
+; CHECK-NEXT:    stxvp vsp36, 0(r5)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_dmxvbf16gerx2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxv v2, 0(r3)
+; CHECK-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-BE-NEXT:    lxv v3, 16(r3)
+; CHECK-BE-NEXT:    dmxvbf16gerx2 dmr0, vsp34, vs0
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r5)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r5)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r5)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r5)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2(<256 x i1> %v1, <16 x i8> %v2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2pp(<1024 x i1>, <256 x i1>, <16 x i8>)
+
+define void @test_dmxvbf16gerx2pp(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_dmxvbf16gerx2pp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    dmxvbf16gerx2pp dmr0, vsp34, vs0
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r6)
+; CHECK-NEXT:    stxvp vsp36, 64(r6)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r6)
+; CHECK-NEXT:    stxvp vsp36, 0(r6)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_dmxvbf16gerx2pp:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvp vsp34, 96(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 64(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-BE-NEXT:    lxvp vsp34, 32(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 0(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r5)
+; CHECK-BE-NEXT:    lxv v3, 16(r4)
+; CHECK-BE-NEXT:    dmxvbf16gerx2pp dmr0, vsp34, vs0
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r6)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r6)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2pp(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2pn(<1024 x i1>, <256 x i1>, <16 x i8>)
+
+define void @test_dmxvbf16gerx2pn(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_dmxvbf16gerx2pn:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    dmxvbf16gerx2pn dmr0, vsp34, vs0
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r6)
+; CHECK-NEXT:    stxvp vsp36, 64(r6)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r6)
+; CHECK-NEXT:    stxvp vsp36, 0(r6)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_dmxvbf16gerx2pn:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvp vsp34, 96(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 64(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-BE-NEXT:    lxvp vsp34, 32(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 0(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r5)
+; CHECK-BE-NEXT:    lxv v3, 16(r4)
+; CHECK-BE-NEXT:    dmxvbf16gerx2pn dmr0, vsp34, vs0
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r6)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r6)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2pn(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2np(<1024 x i1>, <256 x i1>, <16 x i8>)
+
+define void @test_dmxvbf16gerx2np(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_dmxvbf16gerx2np:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    dmxvbf16gerx2np dmr0, vsp34, vs0
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r6)
+; CHECK-NEXT:    stxvp vsp36, 64(r6)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r6)
+; CHECK-NEXT:    stxvp vsp36, 0(r6)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_dmxvbf16gerx2np:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvp vsp34, 96(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 64(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-BE-NEXT:    lxvp vsp34, 32(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 0(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r5)
+; CHECK-BE-NEXT:    lxv v3, 16(r4)
+; CHECK-BE-NEXT:    dmxvbf16gerx2np dmr0, vsp34, vs0
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r6)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r6)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2np(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2nn(<1024 x i1>, <256 x i1>, <16 x i8>)
+
+define void @test_dmxvbf16gerx2nn(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_dmxvbf16gerx2nn:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    dmxvbf16gerx2nn dmr0, vsp34, vs0
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r6)
+; CHECK-NEXT:    stxvp vsp36, 64(r6)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r6)
+; CHECK-NEXT:    stxvp vsp36, 0(r6)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_dmxvbf16gerx2nn:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvp vsp34, 96(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 64(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-BE-NEXT:    lxvp vsp34, 32(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 0(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r5)
+; CHECK-BE-NEXT:    lxv v3, 16(r4)
+; CHECK-BE-NEXT:    dmxvbf16gerx2nn dmr0, vsp34, vs0
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r6)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r6)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2nn(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2(<256 x i1>, <16 x i8>, i32, i32, i32)
+
+define void @test_pmdmxvbf16gerx2(ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_pmdmxvbf16gerx2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv v2, 16(r3)
+; CHECK-NEXT:    lxv vs0, 0(r4)
+; CHECK-NEXT:    lxv v3, 0(r3)
+; CHECK-NEXT:    pmdmxvbf16gerx2 dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r5)
+; CHECK-NEXT:    stxvp vsp36, 64(r5)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r5)
+; CHECK-NEXT:    stxvp vsp36, 0(r5)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_pmdmxvbf16gerx2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxv v2, 0(r3)
+; CHECK-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-BE-NEXT:    lxv v3, 16(r3)
+; CHECK-BE-NEXT:    pmdmxvbf16gerx2 dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r5)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r5)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r5)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r5)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2(<256 x i1> %v1, <16 x i8> %v2, i32 33, i32 5, i32 2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2pp(<1024 x i1>, <256 x i1>, <16 x i8>, i32, i32, i32)
+
+define void @test_pmdmxvbf16gerx2pp(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_pmdmxvbf16gerx2pp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    pmdmxvbf16gerx2pp dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r6)
+; CHECK-NEXT:    stxvp vsp36, 64(r6)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r6)
+; CHECK-NEXT:    stxvp vsp36, 0(r6)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_pmdmxvbf16gerx2pp:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvp vsp34, 96(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 64(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-BE-NEXT:    lxvp vsp34, 32(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 0(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r5)
+; CHECK-BE-NEXT:    lxv v3, 16(r4)
+; CHECK-BE-NEXT:    pmdmxvbf16gerx2pp dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r6)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r6)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2pp(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2, i32 33, i32 5, i32 2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2pn(<1024 x i1>, <256 x i1>, <16 x i8>, i32, i32, i32)
+
+define void @test_pmdmxvbf16gerx2pn(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_pmdmxvbf16gerx2pn:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    pmdmxvbf16gerx2pn dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r6)
+; CHECK-NEXT:    stxvp vsp36, 64(r6)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r6)
+; CHECK-NEXT:    stxvp vsp36, 0(r6)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_pmdmxvbf16gerx2pn:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvp vsp34, 96(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 64(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-BE-NEXT:    lxvp vsp34, 32(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 0(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r5)
+; CHECK-BE-NEXT:    lxv v3, 16(r4)
+; CHECK-BE-NEXT:    pmdmxvbf16gerx2pn dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r6)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r6)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2pn(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2, i32 33, i32 5, i32 2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2np(<1024 x i1>, <256 x i1>, <16 x i8>, i32, i32, i32)
+
+define void @test_pmdmxvbf16gerx2np(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_pmdmxvbf16gerx2np:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    pmdmxvbf16gerx2np dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r6)
+; CHECK-NEXT:    stxvp vsp36, 64(r6)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r6)
+; CHECK-NEXT:    stxvp vsp36, 0(r6)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_pmdmxvbf16gerx2np:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvp vsp34, 96(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 64(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-BE-NEXT:    lxvp vsp34, 32(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 0(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r5)
+; CHECK-BE-NEXT:    lxv v3, 16(r4)
+; CHECK-BE-NEXT:    pmdmxvbf16gerx2np dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r6)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r6)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2np(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2, i32 33, i32 5, i32 2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
+
+declare <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2nn(<1024 x i1>, <256 x i1>, <16 x i8>, i32, i32, i32)
+
+define void @test_pmdmxvbf16gerx2nn(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) {
+; CHECK-LABEL: test_pmdmxvbf16gerx2nn:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    pmdmxvbf16gerx2nn dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r6)
+; CHECK-NEXT:    stxvp vsp36, 64(r6)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r6)
+; CHECK-NEXT:    stxvp vsp36, 0(r6)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_pmdmxvbf16gerx2nn:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvp vsp34, 96(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 64(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-BE-NEXT:    lxvp vsp34, 32(r3)
+; CHECK-BE-NEXT:    lxvp vsp36, 0(r3)
+; CHECK-BE-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-BE-NEXT:    lxv v2, 0(r4)
+; CHECK-BE-NEXT:    lxv vs0, 0(r5)
+; CHECK-BE-NEXT:    lxv v3, 16(r4)
+; CHECK-BE-NEXT:    pmdmxvbf16gerx2nn dmr0, vsp34, vs0, 33, 5, 2
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-BE-NEXT:    stxvp vsp36, 96(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 64(r6)
+; CHECK-BE-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-BE-NEXT:    stxvp vsp36, 32(r6)
+; CHECK-BE-NEXT:    stxvp vsp34, 0(r6)
+; CHECK-BE-NEXT:    blr
+entry:
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.pmdmxvbf16gerx2nn(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2, i32 33, i32 5, i32 2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
diff --git a/llvm/test/MC/Disassembler/PowerPC/ppc-encoding-ISAFuture.txt b/llvm/test/MC/Disassembler/PowerPC/ppc-encoding-ISAFuture.txt
index 06fdb3bd25641..b265b36c91277 100644
--- a/llvm/test/MC/Disassembler/PowerPC/ppc-encoding-ISAFuture.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc-encoding-ISAFuture.txt
@@ -92,4 +92,34 @@
 0xec,0x82,0x23,0x10
 
 #CHECK: pmdmxvi8gerx4spp 0, 2, 4, 8, 4, 4
-[0x07,0x90,0x40,0x84,0xec,0x02,0x23,0x10]
+0x07,0x90,0x40,0x84,0xec,0x02,0x23,0x10
+
+#CHECK: dmxvbf16gerx2 1, 2, 4
+0xec,0x82,0x22,0xd8
+
+#CHECK: dmxvbf16gerx2pp 1, 2, 4
+0xec,0x82,0x22,0x50
+
+#CHECK: dmxvbf16gerx2pn 1, 2, 4
+0xec,0x82,0x25,0x98
+
+#CHECK: dmxvbf16gerx2np 1, 2, 4
+0xec,0x82,0x23,0x98
+
+#CHECK: dmxvbf16gerx2nn 1, 2, 4
+0xec,0x82,0x27,0x50
+
+#CHECK: pmdmxvbf16gerx2 1, 2, 4, 8, 4, 2
+0x07,0x90,0x80,0x84,0xec,0x82,0x22,0xd8
+
+#CHECK: pmdmxvbf16gerx2pp 1, 2, 4, 8, 4, 2
+0x07,0x90,0x80,0x84,0xec,0x82,0x22,0x50
+
+#CHECK: pmdmxvbf16gerx2pn 1, 2, 4, 8, 4, 2
+0x07,0x90,0x80,0x84,0xec,0x82,0x25,0x98
+
+#CHECK: pmdmxvbf16gerx2np 1, 2, 4, 8, 4, 2
+0x07,0x90,0x80,0x84,0xec,0x82,0x23,0x98
+
+#CHECK: pmdmxvbf16gerx2nn 1, 2, 4, 8, 4, 2
+0x07,0x90,0x80,0x84,0xec,0x82,0x27,0x50
diff --git a/llvm/test/MC/Disassembler/PowerPC/ppc64le-encoding-ISAFuture.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64le-encoding-ISAFuture.txt
index c2e32a792d39d..d89588845c138 100644
--- a/llvm/test/MC/Disassembler/PowerPC/ppc64le-encoding-ISAFuture.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64le-encoding-ISAFuture.txt
@@ -87,3 +87,33 @@
 
 #CHECK: pmdmxvi8gerx4spp 0, 2, 4, 8, 4, 4
 0x84,0x40,0x90,0x07,0x10,0x23,0x02,0xec
+
+#CHECK: dmxvbf16gerx2 1, 2, 4
+0xd8,0x22,0x82,0xec
+
+#CHECK: dmxvbf16gerx2pp 1, 2, 4
+0x50,0x22,0x82,0xec
+
+#CHECK: dmxvbf16gerx2pn 1, 2, 4
+0x98,0x25,0x82,0xec
+
+#CHECK: dmxvbf16gerx2np 1, 2, 4
+0x98,0x23,0x82,0xec
+
+#CHECK: dmxvbf16gerx2nn 1, 2, 4
+0x50,0x27,0x82,0xec
+
+#CHECK: pmdmxvbf16gerx2 1, 2, 4, 8, 4, 2
+0x84,0x80,0x90,0x07,0xd8,0x22,0x82,0xec
+
+#CHECK: pmdmxvbf16gerx2pp 1, 2, 4, 8, 4, 2
+0x84,0x80,0x90,0x07,0x50,0x22,0x82,0xec
+
+#CHECK: pmdmxvbf16gerx2pn 1, 2, 4, 8, 4, 2
+0x84,0x80,0x90,0x07,0x98,0x25,0x82,0xec
+
+#CHECK: pmdmxvbf16gerx2np 1, 2, 4, 8, 4, 2
+0x84,0x80,0x90,0x07,0x98,0x23,0x82,0xec
+
+#CHECK: pmdmxvbf16gerx2nn 1, 2, 4, 8, 4, 2
+0x84,0x80,0x90,0x07,0x50,0x27,0x82,0xec
diff --git a/llvm/test/MC/PowerPC/ppc-encoding-ISAFuture.s b/llvm/test/MC/PowerPC/ppc-encoding-ISAFuture.s
index bd7cfc4dfebfb..1160bff24a49a 100644
--- a/llvm/test/MC/PowerPC/ppc-encoding-ISAFuture.s
+++ b/llvm/test/MC/PowerPC/ppc-encoding-ISAFuture.s
@@ -126,3 +126,53 @@
 #CHECK-BE-SAME:                                                  0xec,0x02,0x23,0x10]
 #CHECK-LE:  pmdmxvi8gerx4spp 0, 2, 4, 8, 4, 4       # encoding: [0x84,0x40,0x90,0x07,
 #CHECK-LE-SAME:                                                  0x10,0x23,0x02,0xec]
+
+            dmxvbf16gerx2 1, 2, 4
+#CHECK-BE:  dmxvbf16gerx2 1, 2, 4                   # encoding: [0xec,0x82,0x22,0xd8]
+#CHECK-LE:  dmxvbf16gerx2 1, 2, 4                   # encoding: [0xd8,0x22,0x82,0xec]
+
+            dmxvbf16gerx2pp 1, 2, 4
+#CHECK-BE:  dmxvbf16gerx2pp 1, 2, 4                 # encoding: [0xec,0x82,0x22,0x50]
+#CHECK-LE:  dmxvbf16gerx2pp 1, 2, 4                 # encoding: [0x50,0x22,0x82,0xec]
+
+            dmxvbf16gerx2pn 1, 2, 4
+#CHECK-BE:  dmxvbf16gerx2pn 1, 2, 4                 # encoding: [0xec,0x82,0x25,0x98]
+#CHECK-LE:  dmxvbf16gerx2pn 1, 2, 4                 # encoding: [0x98,0x25,0x82,0xec]
+
+            dmxvbf16gerx2np 1, 2, 4
+#CHECK-BE:  dmxvbf16gerx2np 1, 2, 4                 # encoding: [0xec,0x82,0x23,0x98]
+#CHECK-LE:  dmxvbf16gerx2np 1, 2, 4                 # encoding: [0x98,0x23,0x82,0xec]
+
+            dmxvbf16gerx2nn 1, 2, 4
+#CHECK-BE:  dmxvbf16gerx2nn 1, 2, 4                 # encoding: [0xec,0x82,0x27,0x50]
+#CHECK-LE:  dmxvbf16gerx2nn 1, 2, 4                 # encoding: [0x50,0x27,0x82,0xec]
+
+            pmdmxvbf16gerx2 1, 2, 4, 8, 4, 2
+#CHECK-BE:  pmdmxvbf16gerx2 1, 2, 4, 8, 4, 2        # encoding: [0x07,0x90,0x80,0x84,
+#CHECK-BE-SAME:                                                  0xec,0x82,0x22,0xd8]
+#CHECK-LE:  pmdmxvbf16gerx2 1, 2, 4, 8, 4, 2        # encoding: [0x84,0x80,0x90,0x07,
+#CHECK-LE-SAME:                                                  0xd8,0x22,0x82,0xec]
+
+            pmdmxvbf16gerx2pp 1, 2, 4, 8, 4, 2
+#CHECK-BE:  pmdmxvbf16gerx2pp 1, 2, 4, 8, 4, 2      # encoding: [0x07,0x90,0x80,0x84,
+#CHECK-BE-SAME:                                                  0xec,0x82,0x22,0x50]
+#CHECK-LE:  pmdmxvbf16gerx2pp 1, 2, 4, 8, 4, 2      # encoding: [0x84,0x80,0x90,0x07,
+#CHECK-LE-SAME:                                                  0x50,0x22,0x82,0xec]
+
+            pmdmxvbf16gerx2pn 1, 2, 4, 8, 4, 2
+#CHECK-BE:  pmdmxvbf16gerx2pn 1, 2, 4, 8, 4, 2      # encoding: [0x07,0x90,0x80,0x84,
+#CHECK-BE-SAME:                                                  0xec,0x82,0x25,0x98]
+#CHECK-LE:  pmdmxvbf16gerx2pn 1, 2, 4, 8, 4, 2      # encoding: [0x84,0x80,0x90,0x07,
+#CHECK-LE-SAME:                                                  0x98,0x25,0x82,0xec]
+
+            pmdmxvbf16gerx2np 1, 2, 4, 8, 4, 2
+#CHECK-BE:  pmdmxvbf16gerx2np 1, 2, 4, 8, 4, 2      # encoding: [0x07,0x90,0x80,0x84,
+#CHECK-BE-SAME:                                                  0xec,0x82,0x23,0x98]
+#CHECK-LE:  pmdmxvbf16gerx2np 1, 2, 4, 8, 4, 2      # encoding: [0x84,0x80,0x90,0x07,
+#CHECK-LE-SAME:                                                  0x98,0x23,0x82,0xec]
+
+            pmdmxvbf16gerx2nn 1, 2, 4, 8, 4, 2
+#CHECK-BE:  pmdmxvbf16gerx2nn 1, 2, 4, 8, 4, 2      # encoding: [0x07,0x90,0x80,0x84,
+#CHECK-BE-SAME:                                                  0xec,0x82,0x27,0x50]
+#CHECK-LE:  pmdmxvbf16gerx2nn 1, 2, 4, 8, 4, 2      # encoding: [0x84,0x80,0x90,0x07,
+#CHECK-LE-SAME:                                                  0x50,0x27,0x82,0xec]

From 56910a8b1b302ebf37e9d30bd200091fd23dc232 Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Mon, 21 Apr 2025 15:55:06 -0700
Subject: [PATCH 666/710] [NVPTX] Improve kernel byval parameter lowering
 (#136008)

This change introduces a new pattern for lowering kernel byval
parameters in `NVPTXLowerArgs`. Each byval argument is wrapped in a call
to a new intrinsic, `@llvm.nvvm.internal.addrspace.wrap`. This intrinsic
explicitly equates to no instructions and is removed during operation
legalization in SDAG. However, it allows us to change the addrspace of
the arguments to 101 to reflect the fact that they will occupy this
space when lowered by `LowerFormalArgs` in `NVPTXISelLowering`.
Optionally, if a generic pointer to a param is needed, a standard
`addrspacecast` is used. This approach offers several advantages:

- Exposes addrspace optimizations: By using a standard `addrspacecast`
back to generic space we allow InferAS to optimize this instruction,
potentially sinking it through control flow or in other ways unsupported
by `NVPTXLowerArgs`. This is demonstrated in several existing tests.

- Clearer, more consistent semantics: Previously an `addrspacecast` from
generic to param space was implicitly a no-op. This is problematic
because it's not reciprocal with the inverse cast, violating LLVM
semantics. Further it is very confusing given the existence of
`cvta.to.param`. After this change the cast equates to this instruction.

- Allow for the removal of all nvvm.ptr.* intrinsics: In a follow-up
change the nvvm.ptr.gen.to.param and nvvm.ptr.param.to.gen intrinsics
may be removed.
---
 llvm/include/llvm/IR/IntrinsicsNVVM.td        |  16 +
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp   |   5 +-
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp   |  24 ++
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td      |  16 +-
 llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp      |  89 ++--
 llvm/lib/Target/NVPTX/NVPTXUtilities.cpp      |  33 +-
 llvm/lib/Target/NVPTX/NVPTXUtilities.h        |   2 +-
 llvm/test/CodeGen/NVPTX/bug21465.ll           |   2 +-
 llvm/test/CodeGen/NVPTX/forward-ld-param.ll   |   2 +-
 .../CodeGen/NVPTX/lower-args-gridconstant.ll  | 201 ++++-----
 llvm/test/CodeGen/NVPTX/lower-args.ll         |   4 +-
 llvm/test/CodeGen/NVPTX/lower-byval-args.ll   | 382 ++++++++----------
 .../Inputs/nvptx-basic.ll.expected            |  12 +-
 13 files changed, 400 insertions(+), 388 deletions(-)

diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 93895048faaf3..d09e1da457249 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1909,6 +1909,22 @@ def int_nvvm_ptr_param_to_gen: DefaultAttrsIntrinsic<[llvm_anyptr_ty],
                                    [IntrNoMem, IntrSpeculatable, IntrNoCallback],
                                    "llvm.nvvm.ptr.param.to.gen">;
 
+// Represents an explicit hole in the LLVM IR type system. It may be inserted by
+// the compiler in cases where a pointer is of the wrong type. In the backend
+// this intrinsic will be folded away and not equate to any instruction. It
+// should not be used by any frontend and should only be considered well defined
+// when added in the following cases:
+//
+//  - NVPTXLowerArgs: When wrapping a byval pointer argument to a kernel
+//    function to convert the address space from generic (0) to param (101).
+//    This accounts for the fact that the parameter symbols will occupy this
+//    space when lowered during ISel.
+//
+def int_nvvm_internal_addrspace_wrap :
+  DefaultAttrsIntrinsic<[llvm_anyptr_ty], [llvm_anyptr_ty],
+                        [IntrNoMem, IntrSpeculatable, NoUndef<ArgIndex<0>>,
+                         NoUndef<RetIndex>]>;
+
 // Move intrinsics, used in nvvm internally
 
 def int_nvvm_move_i16 : Intrinsic<[llvm_i16_ty], [llvm_i16_ty], [IntrNoMem],
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index ec1f969494cd1..486c7c815435a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -985,6 +985,9 @@ void NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
     case ADDRESS_SPACE_LOCAL:
       Opc = TM.is64Bit() ? NVPTX::cvta_local_64 : NVPTX::cvta_local;
       break;
+    case ADDRESS_SPACE_PARAM:
+      Opc = TM.is64Bit() ? NVPTX::cvta_param_64 : NVPTX::cvta_param;
+      break;
     }
     ReplaceNode(N, CurDAG->getMachineNode(Opc, DL, N->getValueType(0), Src));
     return;
@@ -1008,7 +1011,7 @@ void NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
       Opc = TM.is64Bit() ? NVPTX::cvta_to_local_64 : NVPTX::cvta_to_local;
       break;
     case ADDRESS_SPACE_PARAM:
-      Opc = TM.is64Bit() ? NVPTX::IMOV64r : NVPTX::IMOV32r;
+      Opc = TM.is64Bit() ? NVPTX::cvta_to_param_64 : NVPTX::cvta_to_param;
       break;
     }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 277a34173e7b8..49f4f30096f00 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -1014,6 +1014,8 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
                      {MVT::v2i32, MVT::v4i32, MVT::v8i32, MVT::v16i32,
                       MVT::v32i32, MVT::v64i32, MVT::v128i32},
                      Custom);
+
+  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 }
 
 const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
@@ -1426,6 +1428,17 @@ static MachinePointerInfo refinePtrAS(SDValue &Ptr, SelectionDAG &DAG,
 
     return MachinePointerInfo(ADDRESS_SPACE_LOCAL);
   }
+
+  // Peel of an addrspacecast to generic and load directly from the specific
+  // address space.
+  if (Ptr->getOpcode() == ISD::ADDRSPACECAST) {
+    const auto *ASC = cast<AddrSpaceCastSDNode>(Ptr);
+    if (ASC->getDestAddressSpace() == ADDRESS_SPACE_GENERIC) {
+      Ptr = ASC->getOperand(0);
+      return MachinePointerInfo(ASC->getSrcAddressSpace());
+    }
+  }
+
   return MachinePointerInfo();
 }
 
@@ -2746,6 +2759,15 @@ static SDValue LowerIntrinsicVoid(SDValue Op, SelectionDAG &DAG) {
   return Op;
 }
 
+static SDValue lowerIntrinsicWOChain(SDValue Op, SelectionDAG &DAG) {
+  switch (Op->getConstantOperandVal(0)) {
+  default:
+    return Op;
+  case Intrinsic::nvvm_internal_addrspace_wrap:
+    return Op.getOperand(1);
+  }
+}
+
 // In PTX 64-bit CTLZ and CTPOP are supported, but they return a 32-bit value.
 // Lower these into a node returning the correct type which is zero-extended
 // back to the correct size.
@@ -2889,6 +2911,8 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
     return LowerGlobalAddress(Op, DAG);
   case ISD::INTRINSIC_W_CHAIN:
     return Op;
+  case ISD::INTRINSIC_WO_CHAIN:
+    return lowerIntrinsicWOChain(Op, DAG);
   case ISD::INTRINSIC_VOID:
     return LowerIntrinsicVoid(Op, DAG);
   case ISD::BUILD_VECTOR:
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 9634ad8eb8da2..4ba3e6f06bb5f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2395,18 +2395,10 @@ multiclass G_TO_NG<string Str> {
           "cvta.to." # Str # ".u64 \t$result, $src;", []>;
 }
 
-defm cvta_local  : NG_TO_G<"local">;
-defm cvta_shared : NG_TO_G<"shared">;
-defm cvta_global : NG_TO_G<"global">;
-defm cvta_const  : NG_TO_G<"const">;
-
-defm cvta_to_local  : G_TO_NG<"local">;
-defm cvta_to_shared : G_TO_NG<"shared">;
-defm cvta_to_global : G_TO_NG<"global">;
-defm cvta_to_const  : G_TO_NG<"const">;
-
-// nvvm.ptr.param.to.gen
-defm cvta_param : NG_TO_G<"param">;
+foreach space = ["local", "shared", "global", "const", "param"] in {
+  defm cvta_#space : NG_TO_G<space>;
+  defm cvta_to_#space : G_TO_NG<space>;
+}
 
 def : Pat<(int_nvvm_ptr_param_to_gen i32:$src),
           (cvta_param $src)>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index ee835dba9b020..bd396cd6a40b5 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -265,18 +265,9 @@ static void convertToParamAS(Use *OldUse, Value *Param, bool HasCvtaParam,
     if (HasCvtaParam) {
       auto GetParamAddrCastToGeneric =
           [](Value *Addr, Instruction *OriginalUser) -> Value * {
-        PointerType *ReturnTy =
-            PointerType::get(OriginalUser->getContext(), ADDRESS_SPACE_GENERIC);
-        Function *CvtToGen = Intrinsic::getOrInsertDeclaration(
-            OriginalUser->getModule(), Intrinsic::nvvm_ptr_param_to_gen,
-            {ReturnTy, PointerType::get(OriginalUser->getContext(),
-                                        ADDRESS_SPACE_PARAM)});
-
-        // Cast param address to generic address space
-        Value *CvtToGenCall =
-            CallInst::Create(CvtToGen, Addr, Addr->getName() + ".gen",
-                             OriginalUser->getIterator());
-        return CvtToGenCall;
+        IRBuilder<> IRB(OriginalUser);
+        Type *GenTy = IRB.getPtrTy(ADDRESS_SPACE_GENERIC);
+        return IRB.CreateAddrSpaceCast(Addr, GenTy, Addr->getName() + ".gen");
       };
       auto *ParamInGenericAS =
           GetParamAddrCastToGeneric(I.NewParam, I.OldInstruction);
@@ -515,23 +506,24 @@ void copyByValParam(Function &F, Argument &Arg) {
   BasicBlock::iterator FirstInst = F.getEntryBlock().begin();
   Type *StructType = Arg.getParamByValType();
   const DataLayout &DL = F.getDataLayout();
-  AllocaInst *AllocA = new AllocaInst(StructType, DL.getAllocaAddrSpace(),
-                                      Arg.getName(), FirstInst);
+  IRBuilder<> IRB(&*FirstInst);
+  AllocaInst *AllocA = IRB.CreateAlloca(StructType, nullptr, Arg.getName());
   // Set the alignment to alignment of the byval parameter. This is because,
   // later load/stores assume that alignment, and we are going to replace
   // the use of the byval parameter with this alloca instruction.
-  AllocA->setAlignment(F.getParamAlign(Arg.getArgNo())
-                           .value_or(DL.getPrefTypeAlign(StructType)));
+  AllocA->setAlignment(
+      Arg.getParamAlign().value_or(DL.getPrefTypeAlign(StructType)));
   Arg.replaceAllUsesWith(AllocA);
 
-  Value *ArgInParam = new AddrSpaceCastInst(
-      &Arg, PointerType::get(Arg.getContext(), ADDRESS_SPACE_PARAM),
-      Arg.getName(), FirstInst);
+  Value *ArgInParam =
+      IRB.CreateIntrinsic(Intrinsic::nvvm_internal_addrspace_wrap,
+                          {IRB.getPtrTy(ADDRESS_SPACE_PARAM), Arg.getType()},
+                          &Arg, {}, Arg.getName());
+
   // Be sure to propagate alignment to this load; LLVM doesn't know that NVPTX
   // addrspacecast preserves alignment.  Since params are constant, this load
   // is definitely not volatile.
   const auto ArgSize = *AllocA->getAllocationSize(DL);
-  IRBuilder<> IRB(&*FirstInst);
   IRB.CreateMemCpy(AllocA, AllocA->getAlign(), ArgInParam, AllocA->getAlign(),
                    ArgSize);
 }
@@ -539,9 +531,9 @@ void copyByValParam(Function &F, Argument &Arg) {
 
 static void handleByValParam(const NVPTXTargetMachine &TM, Argument *Arg) {
   Function *Func = Arg->getParent();
-  bool HasCvtaParam =
-      TM.getSubtargetImpl(*Func)->hasCvtaParam() && isKernelFunction(*Func);
-  bool IsGridConstant = HasCvtaParam && isParamGridConstant(*Arg);
+  assert(isKernelFunction(*Func));
+  const bool HasCvtaParam = TM.getSubtargetImpl(*Func)->hasCvtaParam();
+  const bool IsGridConstant = HasCvtaParam && isParamGridConstant(*Arg);
   const DataLayout &DL = Func->getDataLayout();
   BasicBlock::iterator FirstInst = Func->getEntryBlock().begin();
   Type *StructType = Arg->getParamByValType();
@@ -556,9 +548,11 @@ static void handleByValParam(const NVPTXTargetMachine &TM, Argument *Arg) {
     // skip creation of a local copy of the argument.
     SmallVector<Use *, 16> UsesToUpdate(llvm::make_pointer_range(Arg->uses()));
 
-    Value *ArgInParamAS = new AddrSpaceCastInst(
-        Arg, PointerType::get(StructType->getContext(), ADDRESS_SPACE_PARAM),
-        Arg->getName(), FirstInst);
+    IRBuilder<> IRB(&*FirstInst);
+    Value *ArgInParamAS = IRB.CreateIntrinsic(
+        Intrinsic::nvvm_internal_addrspace_wrap,
+        {IRB.getPtrTy(ADDRESS_SPACE_PARAM), Arg->getType()}, {Arg});
+
     for (Use *U : UsesToUpdate)
       convertToParamAS(U, ArgInParamAS, HasCvtaParam, IsGridConstant);
     LLVM_DEBUG(dbgs() << "No need to copy or cast " << *Arg << "\n");
@@ -576,30 +570,31 @@ static void handleByValParam(const NVPTXTargetMachine &TM, Argument *Arg) {
   // However, we're still not allowed to write to it. If the user specified
   // `__grid_constant__` for the argument, we'll consider escaped pointer as
   // read-only.
-  if (HasCvtaParam && (ArgUseIsReadOnly || IsGridConstant)) {
+  if (IsGridConstant || (HasCvtaParam && ArgUseIsReadOnly)) {
     LLVM_DEBUG(dbgs() << "Using non-copy pointer to " << *Arg << "\n");
     // Replace all argument pointer uses (which might include a device function
     // call) with a cast to the generic address space using cvta.param
     // instruction, which avoids a local copy.
     IRBuilder<> IRB(&Func->getEntryBlock().front());
 
-    // Cast argument to param address space
-    auto *CastToParam = cast<AddrSpaceCastInst>(IRB.CreateAddrSpaceCast(
-        Arg, IRB.getPtrTy(ADDRESS_SPACE_PARAM), Arg->getName() + ".param"));
+    // Cast argument to param address space. Because the backend will emit the
+    // argument already in the param address space, we need to use the noop
+    // intrinsic, this had the added benefit of preventing other optimizations
+    // from folding away this pair of addrspacecasts.
+    auto *ParamSpaceArg =
+        IRB.CreateIntrinsic(Intrinsic::nvvm_internal_addrspace_wrap,
+                            {IRB.getPtrTy(ADDRESS_SPACE_PARAM), Arg->getType()},
+                            Arg, {}, Arg->getName() + ".param");
 
-    // Cast param address to generic address space. We do not use an
-    // addrspacecast to generic here, because, LLVM considers `Arg` to be in the
-    // generic address space, and a `generic -> param` cast followed by a `param
-    // -> generic` cast will be folded away. The `param -> generic` intrinsic
-    // will be correctly lowered to `cvta.param`.
-    Value *CvtToGenCall = IRB.CreateIntrinsic(
-        IRB.getPtrTy(ADDRESS_SPACE_GENERIC), Intrinsic::nvvm_ptr_param_to_gen,
-        CastToParam, nullptr, CastToParam->getName() + ".gen");
+    // Cast param address to generic address space.
+    Value *GenericArg = IRB.CreateAddrSpaceCast(
+        ParamSpaceArg, IRB.getPtrTy(ADDRESS_SPACE_GENERIC),
+        Arg->getName() + ".gen");
 
-    Arg->replaceAllUsesWith(CvtToGenCall);
+    Arg->replaceAllUsesWith(GenericArg);
 
     // Do not replace Arg in the cast to param space
-    CastToParam->setOperand(0, Arg);
+    ParamSpaceArg->setOperand(0, Arg);
   } else
     copyByValParam(*Func, *Arg);
 }
@@ -713,12 +708,14 @@ static bool copyFunctionByValArgs(Function &F) {
   LLVM_DEBUG(dbgs() << "Creating a copy of byval args of " << F.getName()
                     << "\n");
   bool Changed = false;
-  for (Argument &Arg : F.args())
-    if (Arg.getType()->isPointerTy() && Arg.hasByValAttr() &&
-        !(isParamGridConstant(Arg) && isKernelFunction(F))) {
-      copyByValParam(F, Arg);
-      Changed = true;
-    }
+  if (isKernelFunction(F)) {
+    for (Argument &Arg : F.args())
+      if (Arg.getType()->isPointerTy() && Arg.hasByValAttr() &&
+          !isParamGridConstant(Arg)) {
+        copyByValParam(F, Arg);
+        Changed = true;
+      }
+  }
   return Changed;
 }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index b800445a3b19c..61b50b69b4e86 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -16,11 +16,13 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Argument.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/ModRef.h"
 #include "llvm/Support/Mutex.h"
 #include <cstdint>
 #include <cstring>
@@ -228,17 +230,30 @@ static std::optional<uint64_t> getVectorProduct(ArrayRef<unsigned> V) {
   return std::accumulate(V.begin(), V.end(), 1, std::multiplies<uint64_t>{});
 }
 
-bool isParamGridConstant(const Value &V) {
-  if (const Argument *Arg = dyn_cast<Argument>(&V)) {
-    // "grid_constant" counts argument indices starting from 1
-    if (Arg->hasByValAttr() &&
-        argHasNVVMAnnotation(*Arg, "grid_constant",
-                             /*StartArgIndexAtOne*/ true)) {
-      assert(isKernelFunction(*Arg->getParent()) &&
-             "only kernel arguments can be grid_constant");
+bool isParamGridConstant(const Argument &Arg) {
+  assert(isKernelFunction(*Arg.getParent()) &&
+         "only kernel arguments can be grid_constant");
+
+  if (!Arg.hasByValAttr())
+    return false;
+
+  // Lowering an argument as a grid_constant violates the byval semantics (and
+  // the C++ API) by reusing the same memory location for the argument across
+  // multiple threads. If an argument doesn't read memory and its address is not
+  // captured (its address is not compared with any value), then the tweak of
+  // the C++ API and byval semantics is unobservable by the program and we can
+  // lower the arg as a grid_constant.
+  if (Arg.onlyReadsMemory()) {
+    const auto CI = Arg.getAttributes().getCaptureInfo();
+    if (!capturesAddress(CI) && !capturesFullProvenance(CI))
       return true;
-    }
   }
+
+  // "grid_constant" counts argument indices starting from 1
+  if (argHasNVVMAnnotation(Arg, "grid_constant",
+                           /*StartArgIndexAtOne*/ true))
+    return true;
+
   return false;
 }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
index 2288241ec0178..70bf02035fd48 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
@@ -63,7 +63,7 @@ inline bool isKernelFunction(const Function &F) {
   return F.getCallingConv() == CallingConv::PTX_Kernel;
 }
 
-bool isParamGridConstant(const Value &);
+bool isParamGridConstant(const Argument &);
 
 inline MaybeAlign getAlign(const Function &F, unsigned Index) {
   return F.getAttributes().getAttributes(Index).getStackAlignment();
diff --git a/llvm/test/CodeGen/NVPTX/bug21465.ll b/llvm/test/CodeGen/NVPTX/bug21465.ll
index 76300e3cfdc5b..33c6dbddd5297 100644
--- a/llvm/test/CodeGen/NVPTX/bug21465.ll
+++ b/llvm/test/CodeGen/NVPTX/bug21465.ll
@@ -12,7 +12,7 @@ define ptx_kernel void @_Z11TakesStruct1SPi(ptr byval(%struct.S) nocapture reado
 entry:
 ; CHECK-LABEL: @_Z11TakesStruct1SPi
 ; PTX-LABEL: .visible .entry _Z11TakesStruct1SPi(
-; CHECK: addrspacecast ptr %input to ptr addrspace(101)
+; CHECK: call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr %input)
   %b = getelementptr inbounds %struct.S, ptr %input, i64 0, i32 1
   %0 = load i32, ptr %b, align 4
 ; PTX-NOT: ld.param.u32 {{%r[0-9]+}}, [{{%rd[0-9]+}}]
diff --git a/llvm/test/CodeGen/NVPTX/forward-ld-param.ll b/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
index 6d9710e6d2272..80ae8aac39115 100644
--- a/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
+++ b/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
@@ -65,7 +65,7 @@ define void @test_ld_param_byval(ptr byval(i32) %a) {
 ; CHECK-LABEL: test_ld_param_byval(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_ld_param_byval_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
index 836a7d78a0cc5..dd172cf685380 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
@@ -72,7 +72,7 @@ define ptx_kernel void @grid_const_int(ptr byval(i32) align 4 %input1, i32 %inpu
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_int(
 ; OPT-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], i32 [[INPUT2:%.*]], ptr [[OUT:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT11:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; OPT-NEXT:    [[INPUT11:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; OPT-NEXT:    [[TMP:%.*]] = load i32, ptr addrspace(101) [[INPUT11]], align 4
 ; OPT-NEXT:    [[ADD:%.*]] = add i32 [[TMP]], [[INPUT2]]
 ; OPT-NEXT:    store i32 [[ADD]], ptr [[OUT]], align 4
@@ -101,7 +101,7 @@ define ptx_kernel void @grid_const_struct(ptr byval(%struct.s) align 4 %input, p
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_struct(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT1:%.*]] = addrspacecast ptr [[INPUT]] to ptr addrspace(101)
+; OPT-NEXT:    [[INPUT1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; OPT-NEXT:    [[GEP13:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr addrspace(101) [[INPUT1]], i32 0, i32 0
 ; OPT-NEXT:    [[GEP22:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr addrspace(101) [[INPUT1]], i32 0, i32 1
 ; OPT-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(101) [[GEP13]], align 4
@@ -122,16 +122,15 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 %input) {
 ; PTX-LABEL: grid_const_escape(
 ; PTX:       {
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<5>;
+; PTX-NEXT:    .reg .b64 %rd<4>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd2, grid_const_escape_param_0;
-; PTX-NEXT:    mov.b64 %rd3, %rd2;
-; PTX-NEXT:    cvta.param.u64 %rd4, %rd3;
+; PTX-NEXT:    cvta.param.u64 %rd3, %rd2;
 ; PTX-NEXT:    mov.b64 %rd1, escape;
 ; PTX-NEXT:    { // callseq 0, 0
 ; PTX-NEXT:    .param .b64 param0;
-; PTX-NEXT:    st.param.b64 [param0], %rd4;
+; PTX-NEXT:    st.param.b64 [param0], %rd3;
 ; PTX-NEXT:    .param .b32 retval0;
 ; PTX-NEXT:    prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _);
 ; PTX-NEXT:    call (retval0),
@@ -145,8 +144,8 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 %input) {
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_escape(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT_PARAM:%.*]] = addrspacecast ptr [[INPUT]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT_PARAM]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    [[CALL:%.*]] = call i32 @escape(ptr [[INPUT_PARAM_GEN]])
 ; OPT-NEXT:    ret void
   %call = call i32 @escape(ptr %input)
@@ -160,29 +159,27 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<4>;
-; PTX-NEXT:    .reg .b64 %rd<10>;
+; PTX-NEXT:    .reg .b64 %rd<8>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %SPL, __local_depot4;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    mov.b64 %rd2, multiple_grid_const_escape_param_0;
-; PTX-NEXT:    mov.b64 %rd3, multiple_grid_const_escape_param_2;
-; PTX-NEXT:    mov.b64 %rd4, %rd3;
 ; PTX-NEXT:    ld.param.u32 %r1, [multiple_grid_const_escape_param_1];
-; PTX-NEXT:    cvta.param.u64 %rd5, %rd4;
-; PTX-NEXT:    mov.b64 %rd6, %rd2;
-; PTX-NEXT:    cvta.param.u64 %rd7, %rd6;
-; PTX-NEXT:    add.u64 %rd8, %SP, 0;
-; PTX-NEXT:    add.u64 %rd9, %SPL, 0;
-; PTX-NEXT:    st.local.u32 [%rd9], %r1;
+; PTX-NEXT:    mov.b64 %rd3, multiple_grid_const_escape_param_2;
+; PTX-NEXT:    cvta.param.u64 %rd4, %rd3;
+; PTX-NEXT:    cvta.param.u64 %rd5, %rd2;
+; PTX-NEXT:    add.u64 %rd6, %SP, 0;
+; PTX-NEXT:    add.u64 %rd7, %SPL, 0;
+; PTX-NEXT:    st.local.u32 [%rd7], %r1;
 ; PTX-NEXT:    mov.b64 %rd1, escape3;
 ; PTX-NEXT:    { // callseq 1, 0
 ; PTX-NEXT:    .param .b64 param0;
-; PTX-NEXT:    st.param.b64 [param0], %rd7;
+; PTX-NEXT:    st.param.b64 [param0], %rd5;
 ; PTX-NEXT:    .param .b64 param1;
-; PTX-NEXT:    st.param.b64 [param1], %rd8;
+; PTX-NEXT:    st.param.b64 [param1], %rd6;
 ; PTX-NEXT:    .param .b64 param2;
-; PTX-NEXT:    st.param.b64 [param2], %rd5;
+; PTX-NEXT:    st.param.b64 [param2], %rd4;
 ; PTX-NEXT:    .param .b32 retval0;
 ; PTX-NEXT:    prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _);
 ; PTX-NEXT:    call (retval0),
@@ -198,10 +195,10 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @multiple_grid_const_escape(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]], i32 [[A:%.*]], ptr byval(i32) align 4 [[B:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[B_PARAM:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(101)
-; OPT-NEXT:    [[B_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[B_PARAM]])
-; OPT-NEXT:    [[INPUT_PARAM:%.*]] = addrspacecast ptr [[INPUT]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT_PARAM]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[B]])
+; OPT-NEXT:    [[B_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
+; OPT-NEXT:    [[TMP2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP2]] to ptr
 ; OPT-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 ; OPT-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 ; OPT-NEXT:    [[CALL:%.*]] = call i32 @escape3(ptr [[INPUT_PARAM_GEN]], ptr [[A_ADDR]], ptr [[B_PARAM_GEN]])
@@ -215,20 +212,19 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4
 define ptx_kernel void @grid_const_memory_escape(ptr byval(%struct.s) align 4 %input, ptr %addr) {
 ; PTX-LABEL: grid_const_memory_escape(
 ; PTX:       {
-; PTX-NEXT:    .reg .b64 %rd<6>;
+; PTX-NEXT:    .reg .b64 %rd<5>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd1, grid_const_memory_escape_param_0;
 ; PTX-NEXT:    ld.param.u64 %rd2, [grid_const_memory_escape_param_1];
 ; PTX-NEXT:    cvta.to.global.u64 %rd3, %rd2;
-; PTX-NEXT:    mov.b64 %rd4, %rd1;
-; PTX-NEXT:    cvta.param.u64 %rd5, %rd4;
-; PTX-NEXT:    st.global.u64 [%rd3], %rd5;
+; PTX-NEXT:    cvta.param.u64 %rd4, %rd1;
+; PTX-NEXT:    st.global.u64 [%rd3], %rd4;
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_memory_escape(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT_PARAM:%.*]] = addrspacecast ptr [[INPUT]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT1:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT_PARAM]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    store ptr [[INPUT1]], ptr [[ADDR]], align 8
 ; OPT-NEXT:    ret void
   store ptr %input, ptr %addr, align 8
@@ -238,14 +234,13 @@ define ptx_kernel void @grid_const_memory_escape(ptr byval(%struct.s) align 4 %i
 define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4 %input, ptr %result) {
 ; PTX-LABEL: grid_const_inlineasm_escape(
 ; PTX:       {
-; PTX-NEXT:    .reg .b64 %rd<8>;
+; PTX-NEXT:    .reg .b64 %rd<7>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd4, grid_const_inlineasm_escape_param_0;
 ; PTX-NEXT:    ld.param.u64 %rd5, [grid_const_inlineasm_escape_param_1];
 ; PTX-NEXT:    cvta.to.global.u64 %rd6, %rd5;
-; PTX-NEXT:    mov.b64 %rd7, %rd4;
-; PTX-NEXT:    cvta.param.u64 %rd2, %rd7;
+; PTX-NEXT:    cvta.param.u64 %rd2, %rd4;
 ; PTX-NEXT:    add.s64 %rd3, %rd2, 4;
 ; PTX-NEXT:    // begin inline asm
 ; PTX-NEXT:    add.s64 %rd1, %rd2, %rd3;
@@ -255,8 +250,8 @@ define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4
 ; PTX-NOT      .local
 ; OPT-LABEL: define ptx_kernel void @grid_const_inlineasm_escape(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]], ptr [[RESULT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT_PARAM:%.*]] = addrspacecast ptr [[INPUT]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT1:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT_PARAM]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    [[TMPPTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT1]], i32 0, i32 0
 ; OPT-NEXT:    [[TMPPTR2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT1]], i32 0, i32 1
 ; OPT-NEXT:    [[TMP2:%.*]] = call i64 asm "add.s64 $0, $1, $2
@@ -273,21 +268,20 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) %input, ptr %ou
 ; PTX-LABEL: grid_const_partial_escape(
 ; PTX:       {
 ; PTX-NEXT:    .reg .b32 %r<5>;
-; PTX-NEXT:    .reg .b64 %rd<7>;
+; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd2, grid_const_partial_escape_param_0;
 ; PTX-NEXT:    ld.param.u64 %rd3, [grid_const_partial_escape_param_1];
 ; PTX-NEXT:    cvta.to.global.u64 %rd4, %rd3;
-; PTX-NEXT:    mov.b64 %rd5, %rd2;
-; PTX-NEXT:    cvta.param.u64 %rd6, %rd5;
-; PTX-NEXT:    ld.u32 %r1, [%rd6];
+; PTX-NEXT:    cvta.param.u64 %rd5, %rd2;
+; PTX-NEXT:    ld.param.u32 %r1, [grid_const_partial_escape_param_0];
 ; PTX-NEXT:    add.s32 %r2, %r1, %r1;
 ; PTX-NEXT:    st.global.u32 [%rd4], %r2;
 ; PTX-NEXT:    mov.b64 %rd1, escape;
 ; PTX-NEXT:    { // callseq 2, 0
 ; PTX-NEXT:    .param .b64 param0;
-; PTX-NEXT:    st.param.b64 [param0], %rd6;
+; PTX-NEXT:    st.param.b64 [param0], %rd5;
 ; PTX-NEXT:    .param .b32 retval0;
 ; PTX-NEXT:    prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _);
 ; PTX-NEXT:    call (retval0),
@@ -301,8 +295,8 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) %input, ptr %ou
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_partial_escape(
 ; OPT-SAME: ptr byval(i32) [[INPUT:%.*]], ptr [[OUTPUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT1:%.*]] = addrspacecast ptr [[INPUT]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT1_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT1]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT1_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    [[VAL1:%.*]] = load i32, ptr [[INPUT1_GEN]], align 4
 ; OPT-NEXT:    [[TWICE:%.*]] = add i32 [[VAL1]], [[VAL1]]
 ; OPT-NEXT:    store i32 [[TWICE]], ptr [[OUTPUT]], align 4
@@ -319,22 +313,21 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) %input,
 ; PTX-LABEL: grid_const_partial_escapemem(
 ; PTX:       {
 ; PTX-NEXT:    .reg .b32 %r<6>;
-; PTX-NEXT:    .reg .b64 %rd<7>;
+; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd2, grid_const_partial_escapemem_param_0;
 ; PTX-NEXT:    ld.param.u64 %rd3, [grid_const_partial_escapemem_param_1];
 ; PTX-NEXT:    cvta.to.global.u64 %rd4, %rd3;
-; PTX-NEXT:    mov.b64 %rd5, %rd2;
-; PTX-NEXT:    cvta.param.u64 %rd6, %rd5;
-; PTX-NEXT:    ld.u32 %r1, [%rd6];
-; PTX-NEXT:    ld.u32 %r2, [%rd6+4];
-; PTX-NEXT:    st.global.u64 [%rd4], %rd6;
+; PTX-NEXT:    cvta.param.u64 %rd5, %rd2;
+; PTX-NEXT:    ld.param.u32 %r1, [grid_const_partial_escapemem_param_0];
+; PTX-NEXT:    ld.param.u32 %r2, [grid_const_partial_escapemem_param_0+4];
+; PTX-NEXT:    st.global.u64 [%rd4], %rd5;
 ; PTX-NEXT:    add.s32 %r3, %r1, %r2;
 ; PTX-NEXT:    mov.b64 %rd1, escape;
 ; PTX-NEXT:    { // callseq 3, 0
 ; PTX-NEXT:    .param .b64 param0;
-; PTX-NEXT:    st.param.b64 [param0], %rd6;
+; PTX-NEXT:    st.param.b64 [param0], %rd5;
 ; PTX-NEXT:    .param .b32 retval0;
 ; PTX-NEXT:    prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _);
 ; PTX-NEXT:    call (retval0),
@@ -349,8 +342,8 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) %input,
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel i32 @grid_const_partial_escapemem(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) [[INPUT:%.*]], ptr [[OUTPUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT2:%.*]] = addrspacecast ptr [[INPUT]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT1:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT2]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT1]], i32 0, i32 0
 ; OPT-NEXT:    [[VAL1:%.*]] = load i32, ptr [[PTR1]], align 4
 ; OPT-NEXT:    [[PTR2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT1]], i32 0, i32 1
@@ -374,27 +367,25 @@ define ptx_kernel void @grid_const_phi(ptr byval(%struct.s) align 4 %input1, ptr
 ; PTX:       {
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<9>;
+; PTX-NEXT:    .reg .b64 %rd<7>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
-; PTX-NEXT:    mov.b64 %rd5, grid_const_phi_param_0;
-; PTX-NEXT:    ld.param.u64 %rd6, [grid_const_phi_param_1];
-; PTX-NEXT:    cvta.to.global.u64 %rd1, %rd6;
-; PTX-NEXT:    mov.b64 %rd7, %rd5;
-; PTX-NEXT:    cvta.param.u64 %rd8, %rd7;
+; PTX-NEXT:    mov.b64 %rd6, grid_const_phi_param_0;
+; PTX-NEXT:    ld.param.u64 %rd5, [grid_const_phi_param_1];
+; PTX-NEXT:    cvta.to.global.u64 %rd1, %rd5;
 ; PTX-NEXT:    ld.global.u32 %r1, [%rd1];
 ; PTX-NEXT:    setp.lt.s32 %p1, %r1, 0;
 ; PTX-NEXT:    @%p1 bra $L__BB9_2;
 ; PTX-NEXT:  // %bb.1: // %second
-; PTX-NEXT:    add.s64 %rd8, %rd8, 4;
+; PTX-NEXT:    add.s64 %rd6, %rd6, 4;
 ; PTX-NEXT:  $L__BB9_2: // %merge
-; PTX-NEXT:    ld.u32 %r2, [%rd8];
+; PTX-NEXT:    ld.param.u32 %r2, [%rd6];
 ; PTX-NEXT:    st.global.u32 [%rd1], %r2;
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_phi(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT1_PARAM:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT1_PARAM]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    [[VAL:%.*]] = load i32, ptr [[INOUT]], align 4
 ; OPT-NEXT:    [[LESS:%.*]] = icmp slt i32 [[VAL]], 0
 ; OPT-NEXT:    br i1 [[LESS]], label %[[FIRST:.*]], label %[[SECOND:.*]]
@@ -432,32 +423,28 @@ define ptx_kernel void @grid_const_phi_ngc(ptr byval(%struct.s) align 4 %input1,
 ; PTX:       {
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<12>;
+; PTX-NEXT:    .reg .b64 %rd<8>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
-; PTX-NEXT:    mov.b64 %rd6, grid_const_phi_ngc_param_0;
-; PTX-NEXT:    ld.param.u64 %rd7, [grid_const_phi_ngc_param_2];
-; PTX-NEXT:    cvta.to.global.u64 %rd1, %rd7;
-; PTX-NEXT:    mov.b64 %rd10, %rd6;
-; PTX-NEXT:    cvta.param.u64 %rd11, %rd10;
+; PTX-NEXT:    mov.b64 %rd7, grid_const_phi_ngc_param_0;
+; PTX-NEXT:    ld.param.u64 %rd6, [grid_const_phi_ngc_param_2];
+; PTX-NEXT:    cvta.to.global.u64 %rd1, %rd6;
 ; PTX-NEXT:    ld.global.u32 %r1, [%rd1];
 ; PTX-NEXT:    setp.lt.s32 %p1, %r1, 0;
 ; PTX-NEXT:    @%p1 bra $L__BB10_2;
 ; PTX-NEXT:  // %bb.1: // %second
-; PTX-NEXT:    mov.b64 %rd8, grid_const_phi_ngc_param_1;
-; PTX-NEXT:    mov.b64 %rd9, %rd8;
-; PTX-NEXT:    cvta.param.u64 %rd2, %rd9;
-; PTX-NEXT:    add.s64 %rd11, %rd2, 4;
+; PTX-NEXT:    mov.b64 %rd2, grid_const_phi_ngc_param_1;
+; PTX-NEXT:    add.s64 %rd7, %rd2, 4;
 ; PTX-NEXT:  $L__BB10_2: // %merge
-; PTX-NEXT:    ld.u32 %r2, [%rd11];
+; PTX-NEXT:    ld.param.u32 %r2, [%rd7];
 ; PTX-NEXT:    st.global.u32 [%rd1], %r2;
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_phi_ngc(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT2_PARAM:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT2_PARAM]])
-; OPT-NEXT:    [[INPUT1_PARAM:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT1_PARAM]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
+; OPT-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
+; OPT-NEXT:    [[TMP2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP2]] to ptr
 ; OPT-NEXT:    [[VAL:%.*]] = load i32, ptr [[INOUT]], align 4
 ; OPT-NEXT:    [[LESS:%.*]] = icmp slt i32 [[VAL]], 0
 ; OPT-NEXT:    br i1 [[LESS]], label %[[FIRST:.*]], label %[[SECOND:.*]]
@@ -494,29 +481,25 @@ define ptx_kernel void @grid_const_select(ptr byval(i32) align 4 %input1, ptr by
 ; PTX:       {
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<10>;
+; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd1, grid_const_select_param_0;
 ; PTX-NEXT:    ld.param.u64 %rd2, [grid_const_select_param_2];
 ; PTX-NEXT:    cvta.to.global.u64 %rd3, %rd2;
 ; PTX-NEXT:    mov.b64 %rd4, grid_const_select_param_1;
-; PTX-NEXT:    mov.b64 %rd5, %rd4;
-; PTX-NEXT:    cvta.param.u64 %rd6, %rd5;
-; PTX-NEXT:    mov.b64 %rd7, %rd1;
-; PTX-NEXT:    cvta.param.u64 %rd8, %rd7;
 ; PTX-NEXT:    ld.global.u32 %r1, [%rd3];
 ; PTX-NEXT:    setp.lt.s32 %p1, %r1, 0;
-; PTX-NEXT:    selp.b64 %rd9, %rd8, %rd6, %p1;
-; PTX-NEXT:    ld.u32 %r2, [%rd9];
+; PTX-NEXT:    selp.b64 %rd5, %rd1, %rd4, %p1;
+; PTX-NEXT:    ld.param.u32 %r2, [%rd5];
 ; PTX-NEXT:    st.global.u32 [%rd3], %r2;
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_select(
 ; OPT-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT2_PARAM:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT2_PARAM]])
-; OPT-NEXT:    [[INPUT1_PARAM:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT1_PARAM]])
+; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
+; OPT-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
+; OPT-NEXT:    [[TMP2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP2]] to ptr
 ; OPT-NEXT:    [[VAL:%.*]] = load i32, ptr [[INOUT]], align 4
 ; OPT-NEXT:    [[LESS:%.*]] = icmp slt i32 [[VAL]], 0
 ; OPT-NEXT:    [[PTRNEW:%.*]] = select i1 [[LESS]], ptr [[INPUT1_PARAM_GEN]], ptr [[INPUT2_PARAM_GEN]]
@@ -535,22 +518,21 @@ define ptx_kernel i32 @grid_const_ptrtoint(ptr byval(i32) %input) {
 ; PTX-LABEL: grid_const_ptrtoint(
 ; PTX:       {
 ; PTX-NEXT:    .reg .b32 %r<4>;
-; PTX-NEXT:    .reg .b64 %rd<4>;
+; PTX-NEXT:    .reg .b64 %rd<3>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd1, grid_const_ptrtoint_param_0;
-; PTX-NEXT:    mov.b64 %rd2, %rd1;
 ; PTX-NEXT:    ld.param.u32 %r1, [grid_const_ptrtoint_param_0];
-; PTX-NEXT:    cvta.param.u64 %rd3, %rd2;
-; PTX-NEXT:    cvt.u32.u64 %r2, %rd3;
+; PTX-NEXT:    cvta.param.u64 %rd2, %rd1;
+; PTX-NEXT:    cvt.u32.u64 %r2, %rd2;
 ; PTX-NEXT:    add.s32 %r3, %r1, %r2;
 ; PTX-NEXT:    st.param.b32 [func_retval0], %r3;
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel i32 @grid_const_ptrtoint(
 ; OPT-SAME: ptr byval(i32) align 4 [[INPUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT2:%.*]] = addrspacecast ptr [[INPUT]] to ptr addrspace(101)
+; OPT-NEXT:    [[INPUT2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; OPT-NEXT:    [[INPUT3:%.*]] = load i32, ptr addrspace(101) [[INPUT2]], align 4
-; OPT-NEXT:    [[INPUT1:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT2]])
+; OPT-NEXT:    [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[INPUT2]] to ptr
 ; OPT-NEXT:    [[PTRVAL:%.*]] = ptrtoint ptr [[INPUT1]] to i32
 ; OPT-NEXT:    [[KEEPALIVE:%.*]] = add i32 [[INPUT3]], [[PTRVAL]]
 ; OPT-NEXT:    ret i32 [[KEEPALIVE]]
@@ -560,13 +542,42 @@ define ptx_kernel i32 @grid_const_ptrtoint(ptr byval(i32) %input) {
   ret i32 %keepalive
 }
 
+declare void @device_func(ptr byval(i32) align 4)
+
+define ptx_kernel void @test_forward_byval_arg(ptr byval(i32) align 4 %input) {
+; OPT-LABEL: define ptx_kernel void @test_forward_byval_arg(
+; OPT-SAME: ptr byval(i32) align 4 [[INPUT:%.*]]) #[[ATTR0]] {
+; OPT-NEXT:    [[INPUT_PARAM:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[INPUT_PARAM]] to ptr
+; OPT-NEXT:    call void @device_func(ptr byval(i32) align 4 [[INPUT_PARAM_GEN]])
+; OPT-NEXT:    ret void
+;
+; PTX-LABEL: test_forward_byval_arg(
+; PTX:       {
+; PTX-NEXT:    .reg .b32 %r<2>;
+; PTX-EMPTY:
+; PTX-NEXT:  // %bb.0:
+; PTX-NEXT:    ld.param.u32 %r1, [test_forward_byval_arg_param_0];
+; PTX-NEXT:    { // callseq 4, 0
+; PTX-NEXT:    .param .align 4 .b8 param0[4];
+; PTX-NEXT:    st.param.b32 [param0], %r1;
+; PTX-NEXT:    call.uni
+; PTX-NEXT:    device_func,
+; PTX-NEXT:    (
+; PTX-NEXT:    param0
+; PTX-NEXT:    );
+; PTX-NEXT:    } // callseq 4
+; PTX-NEXT:    ret;
+  call void @device_func(ptr byval(i32) align 4 %input)
+  ret void
+}
 
 
 declare dso_local void @dummy() local_unnamed_addr
 declare dso_local ptr @escape(ptr) local_unnamed_addr
 declare dso_local ptr @escape3(ptr, ptr, ptr) local_unnamed_addr
 
-!nvvm.annotations = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23}
+!nvvm.annotations = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24}
 
 !0 = !{ptr @grid_const_int, !"grid_constant", !1}
 !1 = !{i32 1}
@@ -604,4 +615,6 @@ declare dso_local ptr @escape3(ptr, ptr, ptr) local_unnamed_addr
 !22 = !{ptr @grid_const_ptrtoint, !"grid_constant", !23}
 !23 = !{i32 1}
 
+!24 = !{ptr @test_forward_byval_arg, !"grid_constant", !25}
+!25 = !{i32 1}
 
diff --git a/llvm/test/CodeGen/NVPTX/lower-args.ll b/llvm/test/CodeGen/NVPTX/lower-args.ll
index 8fa7d5c3e0cbc..8e879871e295b 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args.ll
@@ -210,7 +210,7 @@ define ptx_kernel void @ptr_as_int(i64 noundef %i, i32 noundef %v) {
 define ptx_kernel void @ptr_as_int_aggr(ptr nocapture noundef readonly byval(%struct.S) align 8 %s, i32 noundef %v) {
 ; IRC-LABEL: define ptx_kernel void @ptr_as_int_aggr(
 ; IRC-SAME: ptr noundef readonly byval([[STRUCT_S:%.*]]) align 8 captures(none) [[S:%.*]], i32 noundef [[V:%.*]]) {
-; IRC-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; IRC-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; IRC-NEXT:    [[I:%.*]] = load i64, ptr addrspace(101) [[S3]], align 8
 ; IRC-NEXT:    [[P:%.*]] = inttoptr i64 [[I]] to ptr
 ; IRC-NEXT:    [[P1:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(1)
@@ -220,7 +220,7 @@ define ptx_kernel void @ptr_as_int_aggr(ptr nocapture noundef readonly byval(%st
 ;
 ; IRO-LABEL: define ptx_kernel void @ptr_as_int_aggr(
 ; IRO-SAME: ptr noundef readonly byval([[STRUCT_S:%.*]]) align 8 captures(none) [[S:%.*]], i32 noundef [[V:%.*]]) {
-; IRO-NEXT:    [[S1:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; IRO-NEXT:    [[S1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; IRO-NEXT:    [[I:%.*]] = load i64, ptr addrspace(101) [[S1]], align 8
 ; IRO-NEXT:    [[P:%.*]] = inttoptr i64 [[I]] to ptr
 ; IRO-NEXT:    store i32 [[V]], ptr [[P]], align 4
diff --git a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
index 2d8684c7cab48..1304ffe42c7b5 100644
--- a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
@@ -32,7 +32,7 @@ define dso_local ptx_kernel void @read_only(ptr nocapture noundef writeonly %out
 ; LOWER-ARGS-LABEL: define dso_local ptx_kernel void @read_only(
 ; LOWER-ARGS-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; LOWER-ARGS-NEXT:  [[ENTRY:.*:]]
-; LOWER-ARGS-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; LOWER-ARGS-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; LOWER-ARGS-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[S3]], align 4
 ; LOWER-ARGS-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
 ; LOWER-ARGS-NEXT:    ret void
@@ -40,10 +40,7 @@ define dso_local ptx_kernel void @read_only(ptr nocapture noundef writeonly %out
 ; COPY-LABEL: define dso_local ptx_kernel void @read_only(
 ; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
-; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; COPY-NEXT:    [[I:%.*]] = load i32, ptr [[S1]], align 4
+; COPY-NEXT:    [[I:%.*]] = load i32, ptr [[S]], align 4
 ; COPY-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
 ; COPY-NEXT:    ret void
 ;
@@ -69,7 +66,7 @@ define dso_local ptx_kernel void @read_only_gep(ptr nocapture noundef writeonly
 ; LOWER-ARGS-LABEL: define dso_local ptx_kernel void @read_only_gep(
 ; LOWER-ARGS-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; LOWER-ARGS-NEXT:  [[ENTRY:.*:]]
-; LOWER-ARGS-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; LOWER-ARGS-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; LOWER-ARGS-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
 ; LOWER-ARGS-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
 ; LOWER-ARGS-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
@@ -78,10 +75,7 @@ define dso_local ptx_kernel void @read_only_gep(ptr nocapture noundef writeonly
 ; COPY-LABEL: define dso_local ptx_kernel void @read_only_gep(
 ; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
-; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
+; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S]], i64 4
 ; COPY-NEXT:    [[I:%.*]] = load i32, ptr [[B]], align 4
 ; COPY-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
 ; COPY-NEXT:    ret void
@@ -104,122 +98,51 @@ entry:
   ret void
 }
 
-; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
-define dso_local ptx_kernel void @read_only_gep_asc(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; LOWER-ARGS-LABEL: define dso_local ptx_kernel void @read_only_gep_asc(
-; LOWER-ARGS-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; LOWER-ARGS-NEXT:  [[ENTRY:.*:]]
-; LOWER-ARGS-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; LOWER-ARGS-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
-; LOWER-ARGS-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
-; LOWER-ARGS-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
-; LOWER-ARGS-NEXT:    ret void
-;
-; COPY-LABEL: define dso_local ptx_kernel void @read_only_gep_asc(
-; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COPY-NEXT:  [[ENTRY:.*:]]
-; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
-; COPY-NEXT:    [[ASC:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(101)
-; COPY-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[ASC]], align 4
-; COPY-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
-; COPY-NEXT:    ret void
-;
-; PTX-LABEL: read_only_gep_asc(
-; PTX:       {
-; PTX-NEXT:    .reg .b32 %r<2>;
-; PTX-NEXT:    .reg .b64 %rd<3>;
-; PTX-EMPTY:
-; PTX-NEXT:  // %bb.0: // %entry
-; PTX-NEXT:    ld.param.u64 %rd1, [read_only_gep_asc_param_0];
-; PTX-NEXT:    cvta.to.global.u64 %rd2, %rd1;
-; PTX-NEXT:    ld.param.u32 %r1, [read_only_gep_asc_param_1+4];
-; PTX-NEXT:    st.global.u32 [%rd2], %r1;
-; PTX-NEXT:    ret;
-entry:
-  %b = getelementptr inbounds nuw i8, ptr %s, i64 4
-  %asc = addrspacecast ptr %b to ptr addrspace(101)
-  %i = load i32, ptr addrspace(101) %asc, align 4
-  store i32 %i, ptr %out, align 4
-  ret void
-}
+;; TODO: This test has been disabled because the addrspacecast is not legal on
+;; sm_60, and not well supported within nvptx-lower-args. We should determine
+;; in what cases it is safe to make assumptions about the address of a byval
+;; parameter and improve our handling of addrspacecast in nvptx-lower-args.
+
+; define dso_local ptx_kernel void @read_only_gep_asc(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
+; entry:
+;   %b = getelementptr inbounds nuw i8, ptr %s, i64 4
+;   %asc = addrspacecast ptr %b to ptr addrspace(101)
+;   %i = load i32, ptr addrspace(101) %asc, align 4
+;   store i32 %i, ptr %out, align 4
+;   ret void
+; }
+;
+; define dso_local ptx_kernel void @read_only_gep_asc0(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
+; entry:
+;   %b = getelementptr inbounds nuw i8, ptr %s, i64 4
+;   %asc = addrspacecast ptr %b to ptr addrspace(101)
+;   %asc0 = addrspacecast ptr addrspace(101) %asc to ptr
+;   %i = load i32, ptr %asc0, align 4
+;   store i32 %i, ptr %out, align 4
+;   ret void
+; }
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
-define dso_local ptx_kernel void @read_only_gep_asc0(ptr nocapture noundef writeonly %out, ptr nocapture noundef readonly byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; COMMON-LABEL: define dso_local ptx_kernel void @read_only_gep_asc0(
-; COMMON-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+define dso_local ptx_kernel void @escape_ptr(ptr nocapture noundef readnone %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
+; COMMON-LABEL: define dso_local ptx_kernel void @escape_ptr(
+; COMMON-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
-; COMMON-NEXT:    [[ASC:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(101)
-; COMMON-NEXT:    [[ASC0:%.*]] = addrspacecast ptr addrspace(101) [[ASC]] to ptr
-; COMMON-NEXT:    [[I:%.*]] = load i32, ptr [[ASC0]], align 4
-; COMMON-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
+; COMMON-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S1]]) #[[ATTR6:[0-9]+]]
 ; COMMON-NEXT:    ret void
 ;
-; PTX-LABEL: read_only_gep_asc0(
-; PTX:       {
-; PTX-NEXT:    .reg .b32 %r<2>;
-; PTX-NEXT:    .reg .b64 %rd<3>;
-; PTX-EMPTY:
-; PTX-NEXT:  // %bb.0: // %entry
-; PTX-NEXT:    ld.param.u64 %rd1, [read_only_gep_asc0_param_0];
-; PTX-NEXT:    cvta.to.global.u64 %rd2, %rd1;
-; PTX-NEXT:    ld.param.u32 %r1, [read_only_gep_asc0_param_1+4];
-; PTX-NEXT:    st.global.u32 [%rd2], %r1;
-; PTX-NEXT:    ret;
-entry:
-  %b = getelementptr inbounds nuw i8, ptr %s, i64 4
-  %asc = addrspacecast ptr %b to ptr addrspace(101)
-  %asc0 = addrspacecast ptr addrspace(101) %asc to ptr
-  %i = load i32, ptr %asc0, align 4
-  store i32 %i, ptr %out, align 4
-  ret void
-}
-
-; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
-define dso_local ptx_kernel void @escape_ptr(ptr nocapture noundef readnone %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; SM_60-LABEL: define dso_local ptx_kernel void @escape_ptr(
-; SM_60-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; SM_60-NEXT:  [[ENTRY:.*:]]
-; SM_60-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; SM_60-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; SM_60-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S1]]) #[[ATTR5:[0-9]+]]
-; SM_60-NEXT:    ret void
-;
-; SM_70-LABEL: define dso_local ptx_kernel void @escape_ptr(
-; SM_70-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; SM_70-NEXT:  [[ENTRY:.*:]]
-; SM_70-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; SM_70-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; SM_70-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S1]]) #[[ATTR6:[0-9]+]]
-; SM_70-NEXT:    ret void
-;
-; COPY-LABEL: define dso_local ptx_kernel void @escape_ptr(
-; COPY-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COPY-NEXT:  [[ENTRY:.*:]]
-; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; COPY-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S1]]) #[[ATTR5:[0-9]+]]
-; COPY-NEXT:    ret void
-;
 ; PTX-LABEL: escape_ptr(
 ; PTX:       {
-; PTX-NEXT:    .local .align 4 .b8 __local_depot4[8];
+; PTX-NEXT:    .local .align 4 .b8 __local_depot2[8];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<3>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
-; PTX-NEXT:    mov.b64 %SPL, __local_depot4;
+; PTX-NEXT:    mov.b64 %SPL, __local_depot2;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    add.u64 %rd1, %SP, 0;
 ; PTX-NEXT:    add.u64 %rd2, %SPL, 0;
@@ -244,46 +167,26 @@ entry:
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
 define dso_local ptx_kernel void @escape_ptr_gep(ptr nocapture noundef readnone %out, ptr noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
-; SM_60-LABEL: define dso_local ptx_kernel void @escape_ptr_gep(
-; SM_60-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; SM_60-NEXT:  [[ENTRY:.*:]]
-; SM_60-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; SM_60-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; SM_60-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
-; SM_60-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]]) #[[ATTR5]]
-; SM_60-NEXT:    ret void
-;
-; SM_70-LABEL: define dso_local ptx_kernel void @escape_ptr_gep(
-; SM_70-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; SM_70-NEXT:  [[ENTRY:.*:]]
-; SM_70-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; SM_70-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; SM_70-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; SM_70-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
-; SM_70-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]]) #[[ATTR6]]
-; SM_70-NEXT:    ret void
-;
-; COPY-LABEL: define dso_local ptx_kernel void @escape_ptr_gep(
-; COPY-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; COPY-NEXT:  [[ENTRY:.*:]]
-; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; COPY-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
-; COPY-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]]) #[[ATTR5]]
-; COPY-NEXT:    ret void
+; COMMON-LABEL: define dso_local ptx_kernel void @escape_ptr_gep(
+; COMMON-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COMMON-NEXT:  [[ENTRY:.*:]]
+; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
+; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
+; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
+; COMMON-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]]) #[[ATTR6]]
+; COMMON-NEXT:    ret void
 ;
 ; PTX-LABEL: escape_ptr_gep(
 ; PTX:       {
-; PTX-NEXT:    .local .align 4 .b8 __local_depot5[8];
+; PTX-NEXT:    .local .align 4 .b8 __local_depot3[8];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<4>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
-; PTX-NEXT:    mov.b64 %SPL, __local_depot5;
+; PTX-NEXT:    mov.b64 %SPL, __local_depot3;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    add.u64 %rd1, %SP, 0;
 ; PTX-NEXT:    add.u64 %rd2, %SPL, 0;
@@ -314,21 +217,21 @@ define dso_local ptx_kernel void @escape_ptr_store(ptr nocapture noundef writeon
 ; COMMON-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    store ptr [[S1]], ptr [[OUT]], align 8
 ; COMMON-NEXT:    ret void
 ;
 ; PTX-LABEL: escape_ptr_store(
 ; PTX:       {
-; PTX-NEXT:    .local .align 4 .b8 __local_depot6[8];
+; PTX-NEXT:    .local .align 4 .b8 __local_depot4[8];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<5>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
-; PTX-NEXT:    mov.b64 %SPL, __local_depot6;
+; PTX-NEXT:    mov.b64 %SPL, __local_depot4;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    ld.param.u64 %rd1, [escape_ptr_store_param_0];
 ; PTX-NEXT:    cvta.to.global.u64 %rd2, %rd1;
@@ -351,7 +254,7 @@ define dso_local ptx_kernel void @escape_ptr_gep_store(ptr nocapture noundef wri
 ; COMMON-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
 ; COMMON-NEXT:    store ptr [[B]], ptr [[OUT]], align 8
@@ -359,14 +262,14 @@ define dso_local ptx_kernel void @escape_ptr_gep_store(ptr nocapture noundef wri
 ;
 ; PTX-LABEL: escape_ptr_gep_store(
 ; PTX:       {
-; PTX-NEXT:    .local .align 4 .b8 __local_depot7[8];
+; PTX-NEXT:    .local .align 4 .b8 __local_depot5[8];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
-; PTX-NEXT:    mov.b64 %SPL, __local_depot7;
+; PTX-NEXT:    mov.b64 %SPL, __local_depot5;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    ld.param.u64 %rd1, [escape_ptr_gep_store_param_0];
 ; PTX-NEXT:    cvta.to.global.u64 %rd2, %rd1;
@@ -391,7 +294,7 @@ define dso_local ptx_kernel void @escape_ptrtoint(ptr nocapture noundef writeonl
 ; COMMON-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    [[I:%.*]] = ptrtoint ptr [[S1]] to i64
 ; COMMON-NEXT:    store i64 [[I]], ptr [[OUT]], align 8
@@ -399,14 +302,14 @@ define dso_local ptx_kernel void @escape_ptrtoint(ptr nocapture noundef writeonl
 ;
 ; PTX-LABEL: escape_ptrtoint(
 ; PTX:       {
-; PTX-NEXT:    .local .align 4 .b8 __local_depot8[8];
+; PTX-NEXT:    .local .align 4 .b8 __local_depot6[8];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<5>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
-; PTX-NEXT:    mov.b64 %SPL, __local_depot8;
+; PTX-NEXT:    mov.b64 %SPL, __local_depot6;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    ld.param.u64 %rd1, [escape_ptrtoint_param_0];
 ; PTX-NEXT:    cvta.to.global.u64 %rd2, %rd1;
@@ -429,17 +332,14 @@ define dso_local ptx_kernel void @memcpy_from_param(ptr nocapture noundef writeo
 ; LOWER-ARGS-LABEL: define dso_local ptx_kernel void @memcpy_from_param(
 ; LOWER-ARGS-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; LOWER-ARGS-NEXT:  [[ENTRY:.*:]]
-; LOWER-ARGS-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; LOWER-ARGS-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; LOWER-ARGS-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT]], ptr addrspace(101) [[S3]], i64 16, i1 true)
 ; LOWER-ARGS-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @memcpy_from_param(
 ; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
-; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
-; COPY-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[OUT]], ptr [[S1]], i64 16, i1 true)
+; COPY-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[OUT]], ptr [[S]], i64 16, i1 true)
 ; COPY-NEXT:    ret void
 ;
 ; PTX-LABEL: memcpy_from_param(
@@ -492,17 +392,14 @@ define dso_local ptx_kernel void @memcpy_from_param_noalign (ptr nocapture nound
 ; LOWER-ARGS-LABEL: define dso_local ptx_kernel void @memcpy_from_param_noalign(
 ; LOWER-ARGS-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; LOWER-ARGS-NEXT:  [[ENTRY:.*:]]
-; LOWER-ARGS-NEXT:    [[S3:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; LOWER-ARGS-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; LOWER-ARGS-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT]], ptr addrspace(101) [[S3]], i64 16, i1 true)
 ; LOWER-ARGS-NEXT:    ret void
 ;
 ; COPY-LABEL: define dso_local ptx_kernel void @memcpy_from_param_noalign(
 ; COPY-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COPY-NEXT:  [[ENTRY:.*:]]
-; COPY-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 8
-; COPY-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[S1]], ptr addrspace(101) align 8 [[S2]], i64 8, i1 false)
-; COPY-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[OUT]], ptr [[S1]], i64 16, i1 true)
+; COPY-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[OUT]], ptr [[S]], i64 16, i1 true)
 ; COPY-NEXT:    ret void
 ;
 ; PTX-LABEL: memcpy_from_param_noalign(
@@ -551,26 +448,26 @@ entry:
 }
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
-define dso_local ptx_kernel void @memcpy_to_param(ptr nocapture noundef readonly %in, ptr nocapture noundef readnone byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
+define dso_local ptx_kernel void @memcpy_to_param(ptr nocapture noundef readonly %in, ptr nocapture noundef byval(%struct.S) align 4 %s) local_unnamed_addr #0 {
 ; COMMON-LABEL: define dso_local ptx_kernel void @memcpy_to_param(
-; COMMON-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef readnone byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; COMMON-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[S1]], ptr [[IN]], i64 16, i1 true)
 ; COMMON-NEXT:    ret void
 ;
 ; PTX-LABEL: memcpy_to_param(
 ; PTX:       {
-; PTX-NEXT:    .local .align 8 .b8 __local_depot11[8];
+; PTX-NEXT:    .local .align 8 .b8 __local_depot9[8];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<48>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
-; PTX-NEXT:    mov.b64 %SPL, __local_depot11;
+; PTX-NEXT:    mov.b64 %SPL, __local_depot9;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    ld.param.u64 %rd1, [memcpy_to_param_param_0];
 ; PTX-NEXT:    add.u64 %rd3, %SPL, 0;
@@ -636,7 +533,7 @@ define dso_local ptx_kernel void @copy_on_store(ptr nocapture noundef readonly %
 ; COMMON-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[BB:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
+; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    [[I:%.*]] = load i32, ptr [[IN]], align 4
 ; COMMON-NEXT:    store i32 [[I]], ptr [[S1]], align 4
@@ -659,10 +556,10 @@ define ptx_kernel void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i3
 ; SM_60-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3:[0-9]+]] {
 ; SM_60-NEXT:  [[BB:.*:]]
 ; SM_60-NEXT:    [[INPUT24:%.*]] = alloca i32, align 4
-; SM_60-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; SM_60-NEXT:    [[INPUT25:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT24]], ptr addrspace(101) align 4 [[INPUT25]], i64 4, i1 false)
 ; SM_60-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
-; SM_60-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; SM_60-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
 ; SM_60-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT24]]
 ; SM_60-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
@@ -672,10 +569,10 @@ define ptx_kernel void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i3
 ; SM_70-LABEL: define ptx_kernel void @test_select(
 ; SM_70-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3:[0-9]+]] {
 ; SM_70-NEXT:  [[BB:.*:]]
-; SM_70-NEXT:    [[INPUT2_PARAM:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; SM_70-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT2_PARAM]])
-; SM_70-NEXT:    [[INPUT1_PARAM:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; SM_70-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT1_PARAM]])
+; SM_70-NEXT:    [[TMP0:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
+; SM_70-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP0]] to ptr
+; SM_70-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; SM_70-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; SM_70-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT1_PARAM_GEN]], ptr [[INPUT2_PARAM_GEN]]
 ; SM_70-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
 ; SM_70-NEXT:    store i32 [[VALLOADED]], ptr [[OUT]], align 4
@@ -685,10 +582,10 @@ define ptx_kernel void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i3
 ; COPY-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3:[0-9]+]] {
 ; COPY-NEXT:  [[BB:.*:]]
 ; COPY-NEXT:    [[INPUT23:%.*]] = alloca i32, align 4
-; COPY-NEXT:    [[INPUT24:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; COPY-NEXT:    [[INPUT24:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT23]], ptr addrspace(101) align 4 [[INPUT24]], i64 4, i1 false)
 ; COPY-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
-; COPY-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; COPY-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
 ; COPY-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT23]]
 ; COPY-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
@@ -719,7 +616,7 @@ define ptx_kernel void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i3
 ; PTX_70-NEXT:    .reg .pred %p<2>;
 ; PTX_70-NEXT:    .reg .b16 %rs<3>;
 ; PTX_70-NEXT:    .reg .b32 %r<2>;
-; PTX_70-NEXT:    .reg .b64 %rd<10>;
+; PTX_70-NEXT:    .reg .b64 %rd<6>;
 ; PTX_70-EMPTY:
 ; PTX_70-NEXT:  // %bb.0: // %bb
 ; PTX_70-NEXT:    ld.param.u8 %rs1, [test_select_param_3];
@@ -729,12 +626,8 @@ define ptx_kernel void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i3
 ; PTX_70-NEXT:    ld.param.u64 %rd2, [test_select_param_2];
 ; PTX_70-NEXT:    cvta.to.global.u64 %rd3, %rd2;
 ; PTX_70-NEXT:    mov.b64 %rd4, test_select_param_1;
-; PTX_70-NEXT:    mov.b64 %rd5, %rd4;
-; PTX_70-NEXT:    cvta.param.u64 %rd6, %rd5;
-; PTX_70-NEXT:    mov.b64 %rd7, %rd1;
-; PTX_70-NEXT:    cvta.param.u64 %rd8, %rd7;
-; PTX_70-NEXT:    selp.b64 %rd9, %rd8, %rd6, %p1;
-; PTX_70-NEXT:    ld.u32 %r1, [%rd9];
+; PTX_70-NEXT:    selp.b64 %rd5, %rd1, %rd4, %p1;
+; PTX_70-NEXT:    ld.param.u32 %r1, [%rd5];
 ; PTX_70-NEXT:    st.global.u32 [%rd3], %r1;
 ; PTX_70-NEXT:    ret;
 bb:
@@ -749,10 +642,10 @@ define ptx_kernel void @test_select_write(ptr byval(i32) align 4 %input1, ptr by
 ; COMMON-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3:[0-9]+]] {
 ; COMMON-NEXT:  [[BB:.*:]]
 ; COMMON-NEXT:    [[INPUT23:%.*]] = alloca i32, align 4
-; COMMON-NEXT:    [[INPUT24:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; COMMON-NEXT:    [[INPUT24:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT23]], ptr addrspace(101) align 4 [[INPUT24]], i64 4, i1 false)
 ; COMMON-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
-; COMMON-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; COMMON-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
 ; COMMON-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT23]]
 ; COMMON-NEXT:    store i32 1, ptr [[PTRNEW]], align 4
@@ -760,7 +653,7 @@ define ptx_kernel void @test_select_write(ptr byval(i32) align 4 %input1, ptr by
 ;
 ; PTX-LABEL: test_select_write(
 ; PTX:       {
-; PTX-NEXT:    .local .align 4 .b8 __local_depot14[8];
+; PTX-NEXT:    .local .align 4 .b8 __local_depot12[8];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .pred %p<2>;
@@ -769,7 +662,7 @@ define ptx_kernel void @test_select_write(ptr byval(i32) align 4 %input1, ptr by
 ; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %bb
-; PTX-NEXT:    mov.b64 %SPL, __local_depot14;
+; PTX-NEXT:    mov.b64 %SPL, __local_depot12;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    ld.param.u8 %rs1, [test_select_write_param_3];
 ; PTX-NEXT:    and.b16 %rs2, %rs1, 1;
@@ -795,10 +688,10 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; SM_60-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], ptr [[INOUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
 ; SM_60-NEXT:  [[BB:.*:]]
 ; SM_60-NEXT:    [[INPUT24:%.*]] = alloca [[STRUCT_S]], align 8
-; SM_60-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; SM_60-NEXT:    [[INPUT25:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT24]], ptr addrspace(101) align 8 [[INPUT25]], i64 8, i1 false)
 ; SM_60-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
-; SM_60-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; SM_60-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
 ; SM_60-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; SM_60:       [[FIRST]]:
@@ -816,10 +709,10 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; SM_70-LABEL: define ptx_kernel void @test_phi(
 ; SM_70-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], ptr [[INOUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
 ; SM_70-NEXT:  [[BB:.*:]]
-; SM_70-NEXT:    [[INPUT2_PARAM:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; SM_70-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT2_PARAM]])
-; SM_70-NEXT:    [[INPUT1_PARAM:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; SM_70-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = call ptr @llvm.nvvm.ptr.param.to.gen.p0.p101(ptr addrspace(101) [[INPUT1_PARAM]])
+; SM_70-NEXT:    [[TMP0:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
+; SM_70-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP0]] to ptr
+; SM_70-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; SM_70-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; SM_70-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; SM_70:       [[FIRST]]:
 ; SM_70-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT1_PARAM_GEN]], i32 0, i32 0
@@ -837,10 +730,10 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; COPY-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], ptr [[INOUT:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
 ; COPY-NEXT:  [[BB:.*:]]
 ; COPY-NEXT:    [[INPUT23:%.*]] = alloca [[STRUCT_S]], align 8
-; COPY-NEXT:    [[INPUT24:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; COPY-NEXT:    [[INPUT24:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT23]], ptr addrspace(101) align 8 [[INPUT24]], i64 8, i1 false)
 ; COPY-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
-; COPY-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; COPY-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
 ; COPY-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; COPY:       [[FIRST]]:
@@ -869,10 +762,10 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; PTX_60-NEXT:    ld.param.u64 %rd2, [test_phi_param_2];
 ; PTX_60-NEXT:    cvta.to.global.u64 %rd1, %rd2;
 ; PTX_60-NEXT:    ld.param.u32 %r4, [test_phi_param_0];
-; PTX_60-NEXT:    @%p1 bra $L__BB15_2;
+; PTX_60-NEXT:    @%p1 bra $L__BB13_2;
 ; PTX_60-NEXT:  // %bb.1: // %second
 ; PTX_60-NEXT:    ld.param.u32 %r4, [test_phi_param_1+4];
-; PTX_60-NEXT:  $L__BB15_2: // %merge
+; PTX_60-NEXT:  $L__BB13_2: // %merge
 ; PTX_60-NEXT:    st.global.u32 [%rd1], %r4;
 ; PTX_60-NEXT:    ret;
 ;
@@ -881,25 +774,21 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; PTX_70-NEXT:    .reg .pred %p<2>;
 ; PTX_70-NEXT:    .reg .b16 %rs<3>;
 ; PTX_70-NEXT:    .reg .b32 %r<2>;
-; PTX_70-NEXT:    .reg .b64 %rd<12>;
+; PTX_70-NEXT:    .reg .b64 %rd<8>;
 ; PTX_70-EMPTY:
 ; PTX_70-NEXT:  // %bb.0: // %bb
 ; PTX_70-NEXT:    ld.param.u8 %rs1, [test_phi_param_3];
 ; PTX_70-NEXT:    and.b16 %rs2, %rs1, 1;
 ; PTX_70-NEXT:    setp.ne.b16 %p1, %rs2, 0;
-; PTX_70-NEXT:    mov.b64 %rd6, test_phi_param_0;
-; PTX_70-NEXT:    ld.param.u64 %rd7, [test_phi_param_2];
-; PTX_70-NEXT:    cvta.to.global.u64 %rd1, %rd7;
-; PTX_70-NEXT:    mov.b64 %rd10, %rd6;
-; PTX_70-NEXT:    cvta.param.u64 %rd11, %rd10;
-; PTX_70-NEXT:    @%p1 bra $L__BB15_2;
+; PTX_70-NEXT:    mov.b64 %rd7, test_phi_param_0;
+; PTX_70-NEXT:    ld.param.u64 %rd6, [test_phi_param_2];
+; PTX_70-NEXT:    cvta.to.global.u64 %rd1, %rd6;
+; PTX_70-NEXT:    @%p1 bra $L__BB13_2;
 ; PTX_70-NEXT:  // %bb.1: // %second
-; PTX_70-NEXT:    mov.b64 %rd8, test_phi_param_1;
-; PTX_70-NEXT:    mov.b64 %rd9, %rd8;
-; PTX_70-NEXT:    cvta.param.u64 %rd2, %rd9;
-; PTX_70-NEXT:    add.s64 %rd11, %rd2, 4;
-; PTX_70-NEXT:  $L__BB15_2: // %merge
-; PTX_70-NEXT:    ld.u32 %r1, [%rd11];
+; PTX_70-NEXT:    mov.b64 %rd2, test_phi_param_1;
+; PTX_70-NEXT:    add.s64 %rd7, %rd2, 4;
+; PTX_70-NEXT:  $L__BB13_2: // %merge
+; PTX_70-NEXT:    ld.param.u32 %r1, [%rd7];
 ; PTX_70-NEXT:    st.global.u32 [%rd1], %r1;
 ; PTX_70-NEXT:    ret;
 bb:
@@ -925,10 +814,10 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ; COMMON-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], i1 [[COND:%.*]]) #[[ATTR3]] {
 ; COMMON-NEXT:  [[BB:.*:]]
 ; COMMON-NEXT:    [[INPUT24:%.*]] = alloca [[STRUCT_S]], align 8
-; COMMON-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
+; COMMON-NEXT:    [[INPUT25:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT24]], ptr addrspace(101) align 8 [[INPUT25]], i64 8, i1 false)
 ; COMMON-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
+; COMMON-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
 ; COMMON-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; COMMON:       [[FIRST]]:
@@ -944,7 +833,7 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ;
 ; PTX-LABEL: test_phi_write(
 ; PTX:       {
-; PTX-NEXT:    .local .align 4 .b8 __local_depot16[8];
+; PTX-NEXT:    .local .align 4 .b8 __local_depot14[8];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .pred %p<2>;
@@ -953,7 +842,7 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ; PTX-NEXT:    .reg .b64 %rd<7>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %bb
-; PTX-NEXT:    mov.b64 %SPL, __local_depot16;
+; PTX-NEXT:    mov.b64 %SPL, __local_depot14;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    ld.param.u8 %rs1, [test_phi_write_param_2];
 ; PTX-NEXT:    and.b16 %rs2, %rs1, 1;
@@ -964,10 +853,10 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ; PTX-NEXT:    add.u64 %rd6, %SPL, 4;
 ; PTX-NEXT:    ld.param.u32 %r2, [test_phi_write_param_0];
 ; PTX-NEXT:    st.u32 [%SP+4], %r2;
-; PTX-NEXT:    @%p1 bra $L__BB16_2;
+; PTX-NEXT:    @%p1 bra $L__BB14_2;
 ; PTX-NEXT:  // %bb.1: // %second
 ; PTX-NEXT:    mov.b64 %rd6, %rd1;
-; PTX-NEXT:  $L__BB16_2: // %merge
+; PTX-NEXT:  $L__BB14_2: // %merge
 ; PTX-NEXT:    mov.b32 %r3, 1;
 ; PTX-NEXT:    st.local.u32 [%rd6], %r3;
 ; PTX-NEXT:    ret;
@@ -988,6 +877,69 @@ merge:                                            ; preds = %second, %first
   ret void
 }
 
+define ptx_kernel void @test_forward_byval_arg(ptr byval(i32) align 4 %input) {
+; COMMON-LABEL: define ptx_kernel void @test_forward_byval_arg(
+; COMMON-SAME: ptr byval(i32) align 4 [[INPUT:%.*]]) #[[ATTR3]] {
+; COMMON-NEXT:    [[INPUT1:%.*]] = alloca i32, align 4
+; COMMON-NEXT:    [[INPUT2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT1]], ptr addrspace(101) align 4 [[INPUT2]], i64 4, i1 false)
+; COMMON-NEXT:    call void @device_func(ptr byval(i32) align 4 [[INPUT1]])
+; COMMON-NEXT:    ret void
+;
+; PTX-LABEL: test_forward_byval_arg(
+; PTX:       {
+; PTX-NEXT:    .local .align 4 .b8 __local_depot15[4];
+; PTX-NEXT:    .reg .b64 %SP;
+; PTX-NEXT:    .reg .b64 %SPL;
+; PTX-NEXT:    .reg .b32 %r<2>;
+; PTX-NEXT:    .reg .b64 %rd<3>;
+; PTX-EMPTY:
+; PTX-NEXT:  // %bb.0:
+; PTX-NEXT:    mov.b64 %SPL, __local_depot15;
+; PTX-NEXT:    add.u64 %rd2, %SPL, 0;
+; PTX-NEXT:    ld.param.u32 %r1, [test_forward_byval_arg_param_0];
+; PTX-NEXT:    st.local.u32 [%rd2], %r1;
+; PTX-NEXT:    { // callseq 2, 0
+; PTX-NEXT:    .param .align 4 .b8 param0[4];
+; PTX-NEXT:    st.param.b32 [param0], %r1;
+; PTX-NEXT:    call.uni
+; PTX-NEXT:    device_func,
+; PTX-NEXT:    (
+; PTX-NEXT:    param0
+; PTX-NEXT:    );
+; PTX-NEXT:    } // callseq 2
+; PTX-NEXT:    ret;
+  call void @device_func(ptr byval(i32) align 4 %input)
+  ret void
+}
+
+define void @device_func(ptr byval(i32) align 4 %input) {
+; COMMON-LABEL: define void @device_func(
+; COMMON-SAME: ptr byval(i32) align 4 [[INPUT:%.*]]) #[[ATTR3]] {
+; COMMON-NEXT:    call void @device_func(ptr byval(i32) align 4 [[INPUT]])
+; COMMON-NEXT:    ret void
+;
+; PTX-LABEL: device_func(
+; PTX:       {
+; PTX-NEXT:    .reg .b32 %r<2>;
+; PTX-NEXT:    .reg .b64 %rd<2>;
+; PTX-EMPTY:
+; PTX-NEXT:  // %bb.0:
+; PTX-NEXT:    ld.param.u32 %r1, [device_func_param_0];
+; PTX-NEXT:    { // callseq 3, 0
+; PTX-NEXT:    .param .align 4 .b8 param0[4];
+; PTX-NEXT:    st.param.b32 [param0], %r1;
+; PTX-NEXT:    call.uni
+; PTX-NEXT:    device_func,
+; PTX-NEXT:    (
+; PTX-NEXT:    param0
+; PTX-NEXT:    );
+; PTX-NEXT:    } // callseq 3
+; PTX-NEXT:    ret;
+  call void @device_func(ptr byval(i32) align 4 %input)
+  ret void
+}
+
 attributes #0 = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) "no-trapping-math"="true" "target-cpu"="sm_60" "target-features"="+ptx78,+sm_60" "uniform-work-group-size"="true" }
 attributes #1 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
 attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: write) }
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
index e470569bfae19..ad0b11ed6a806 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
@@ -6,7 +6,7 @@
 define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct.St8x4) align 8 %in, ptr nocapture noundef writeonly %ret) {
 ; CHECK-LABEL: caller_St8x4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<13>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
@@ -27,11 +27,11 @@ define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [retval0];
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd7, %rd8}, [retval0+16];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    ld.param.u32 %r3, [caller_St8x4_param_1];
-; CHECK-NEXT:    st.u64 [%r3], %rd5;
-; CHECK-NEXT:    st.u64 [%r3+8], %rd6;
-; CHECK-NEXT:    st.u64 [%r3+16], %rd7;
-; CHECK-NEXT:    st.u64 [%r3+24], %rd8;
+; CHECK-NEXT:    ld.param.u32 %r2, [caller_St8x4_param_1];
+; CHECK-NEXT:    st.u64 [%r2], %rd5;
+; CHECK-NEXT:    st.u64 [%r2+8], %rd6;
+; CHECK-NEXT:    st.u64 [%r2+16], %rd7;
+; CHECK-NEXT:    st.u64 [%r2+24], %rd8;
 ; CHECK-NEXT:    ret;
   %call = tail call fastcc [4 x i64] @callee_St8x4(ptr noundef nonnull byval(%struct.St8x4) align 8 %in) #2
   %.fca.0.extract = extractvalue [4 x i64] %call, 0

From e79d8f6892aa138bfef65334817c24d3d9881855 Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <razvan.lupusoru@gmail.com>
Date: Mon, 21 Apr 2025 16:03:12 -0700
Subject: [PATCH 667/710] [flang][acc] Update stride calculation to include
 inner-dimensions (#136613)

The acc.bounds operation allows specifying stride - but it did not
clarify what it meant. The dialect was updated to specifically note that
stride must capture inner dimension sizes when specified for outer
dimensions.

Flang lowering was also updated for OpenACC to adhere to this. This was
already the case for descriptor-based arrays - but now this is also
being done for all arrays.
---
 flang/include/flang/Lower/DirectivesCommon.h  | 24 +++++++++++++++----
 .../Optimizer/Builder/DirectivesCommon.h      | 20 +++++++++++-----
 flang/lib/Lower/OpenACC.cpp                   | 18 ++++++++++----
 flang/test/Lower/OpenACC/acc-bounds.f90       | 15 ++++++++++++
 .../acc-enter-data-unwrap-defaultbounds.f90   | 12 +++++-----
 flang/test/Lower/OpenACC/acc-enter-data.f90   |  6 ++---
 .../mlir/Dialect/OpenACC/OpenACCOps.td        | 11 +++++++--
 7 files changed, 80 insertions(+), 26 deletions(-)

diff --git a/flang/include/flang/Lower/DirectivesCommon.h b/flang/include/flang/Lower/DirectivesCommon.h
index 6e24343cebd3a..5849727bc365b 100644
--- a/flang/include/flang/Lower/DirectivesCommon.h
+++ b/flang/include/flang/Lower/DirectivesCommon.h
@@ -670,7 +670,8 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
              const std::vector<Fortran::evaluate::Subscript> &subscripts,
              std::stringstream &asFortran, fir::ExtendedValue &dataExv,
              bool dataExvIsAssumedSize, fir::factory::AddrAndBoundsInfo &info,
-             bool treatIndexAsSection = false) {
+             bool treatIndexAsSection = false,
+             bool strideIncludeLowerExtent = false) {
   int dimension = 0;
   mlir::Type idxTy = builder.getIndexType();
   mlir::Type boundTy = builder.getType<BoundsType>();
@@ -679,6 +680,7 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
   mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
   const int dataExvRank = static_cast<int>(dataExv.rank());
+  mlir::Value cumulativeExtent = one;
   for (const auto &subscript : subscripts) {
     const auto *triplet{std::get_if<Fortran::evaluate::Triplet>(&subscript.u)};
     if (triplet || treatIndexAsSection) {
@@ -847,6 +849,15 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
           ubound = builder.create<mlir::arith::SubIOp>(loc, extent, one);
         }
       }
+
+      // When the strideInBytes is true, it means the stride is from descriptor
+      // and this already includes the lower extents.
+      if (strideIncludeLowerExtent && !strideInBytes) {
+        stride = cumulativeExtent;
+        cumulativeExtent = builder.createOrFold<mlir::arith::MulIOp>(
+            loc, cumulativeExtent, extent);
+      }
+
       mlir::Value bound = builder.create<BoundsOp>(
           loc, boundTy, lbound, ubound, extent, stride, strideInBytes, baseLb);
       bounds.push_back(bound);
@@ -882,7 +893,8 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
     const Fortran::semantics::MaybeExpr &maybeDesignator,
     mlir::Location operandLocation, std::stringstream &asFortran,
     llvm::SmallVector<mlir::Value> &bounds, bool treatIndexAsSection = false,
-    bool unwrapFirBox = true, bool genDefaultBounds = true) {
+    bool unwrapFirBox = true, bool genDefaultBounds = true,
+    bool strideIncludeLowerExtent = false) {
   using namespace Fortran;
 
   fir::factory::AddrAndBoundsInfo info;
@@ -943,7 +955,8 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
       asFortran << '(';
       bounds = genBoundsOps<BoundsOp, BoundsType>(
           builder, operandLocation, converter, stmtCtx, arrayRef->subscript(),
-          asFortran, dataExv, dataExvIsAssumedSize, info, treatIndexAsSection);
+          asFortran, dataExv, dataExvIsAssumedSize, info, treatIndexAsSection,
+          strideIncludeLowerExtent);
     }
     asFortran << ')';
   } else if (auto compRef = detail::getRef<evaluate::Component>(designator)) {
@@ -955,7 +968,7 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
         mlir::isa<fir::SequenceType>(fir::unwrapRefType(info.addr.getType())))
       bounds = fir::factory::genBaseBoundsOps<BoundsOp, BoundsType>(
           builder, operandLocation, compExv,
-          /*isAssumedSize=*/false);
+          /*isAssumedSize=*/false, strideIncludeLowerExtent);
     asFortran << designator.AsFortran();
 
     if (semantics::IsOptional(compRef->GetLastSymbol())) {
@@ -1012,7 +1025,8 @@ fir::factory::AddrAndBoundsInfo gatherDataOperandAddrAndBounds(
       if (genDefaultBounds &&
           mlir::isa<fir::SequenceType>(fir::unwrapRefType(info.addr.getType())))
         bounds = fir::factory::genBaseBoundsOps<BoundsOp, BoundsType>(
-            builder, operandLocation, dataExv, dataExvIsAssumedSize);
+            builder, operandLocation, dataExv, dataExvIsAssumedSize,
+            strideIncludeLowerExtent);
       asFortran << symRef->get().name().ToString();
     } else { // Unsupported
       llvm::report_fatal_error("Unsupported type of OpenACC operand");
diff --git a/flang/include/flang/Optimizer/Builder/DirectivesCommon.h b/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
index c0ab557d26970..8684299ab6792 100644
--- a/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
+++ b/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
@@ -203,7 +203,8 @@ genBoundsOpsFromBox(fir::FirOpBuilder &builder, mlir::Location loc,
 template <typename BoundsOp, typename BoundsType>
 llvm::SmallVector<mlir::Value>
 genBaseBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
-                 fir::ExtendedValue dataExv, bool isAssumedSize) {
+                 fir::ExtendedValue dataExv, bool isAssumedSize,
+                 bool strideIncludeLowerExtent = false) {
   mlir::Type idxTy = builder.getIndexType();
   mlir::Type boundTy = builder.getType<BoundsType>();
   llvm::SmallVector<mlir::Value> bounds;
@@ -213,23 +214,30 @@ genBaseBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
 
   mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
   const unsigned rank = dataExv.rank();
+  mlir::Value cumulativeExtent = one;
   for (unsigned dim = 0; dim < rank; ++dim) {
     mlir::Value baseLb =
         fir::factory::readLowerBound(builder, loc, dataExv, dim, one);
     mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
     mlir::Value ub;
     mlir::Value lb = zero;
-    mlir::Value ext = fir::factory::readExtent(builder, loc, dataExv, dim);
+    mlir::Value extent = fir::factory::readExtent(builder, loc, dataExv, dim);
     if (isAssumedSize && dim + 1 == rank) {
-      ext = zero;
+      extent = zero;
       ub = lb;
     } else {
       // ub = extent - 1
-      ub = builder.create<mlir::arith::SubIOp>(loc, ext, one);
+      ub = builder.create<mlir::arith::SubIOp>(loc, extent, one);
+    }
+    mlir::Value stride = one;
+    if (strideIncludeLowerExtent) {
+      stride = cumulativeExtent;
+      cumulativeExtent = builder.createOrFold<mlir::arith::MulIOp>(
+          loc, cumulativeExtent, extent);
     }
 
-    mlir::Value bound =
-        builder.create<BoundsOp>(loc, boundTy, lb, ub, ext, one, false, baseLb);
+    mlir::Value bound = builder.create<BoundsOp>(loc, boundTy, lb, ub, extent,
+                                                 stride, false, baseLb);
     bounds.push_back(bound);
   }
   return bounds;
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index c4f2e27f69c3b..418bf4ee3d15f 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -52,6 +52,12 @@ static llvm::cl::opt<bool> generateDefaultBounds(
     llvm::cl::desc("Whether to generate default bounds for arrays."),
     llvm::cl::init(false));
 
+static llvm::cl::opt<bool> strideIncludeLowerExtent(
+    "openacc-stride-include-lower-extent",
+    llvm::cl::desc(
+        "Whether to include the lower dimensions extents in the stride."),
+    llvm::cl::init(true));
+
 // Special value for * passed in device_type or gang clauses.
 static constexpr std::int64_t starCst = -1;
 
@@ -396,7 +402,8 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
             converter, builder, semanticsContext, stmtCtx, symbol, designator,
             operandLocation, asFortran, bounds,
             /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
-            /*genDefaultBounds=*/generateDefaultBounds);
+            /*genDefaultBounds=*/generateDefaultBounds,
+            /*strideIncludeLowerExtent=*/strideIncludeLowerExtent);
     LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
 
     // If the input value is optional and is not a descriptor, we use the
@@ -437,7 +444,8 @@ static void genDeclareDataOperandOperations(
             converter, builder, semanticsContext, stmtCtx, symbol, designator,
             operandLocation, asFortran, bounds,
             /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
-            /*genDefaultBounds=*/generateDefaultBounds);
+            /*genDefaultBounds=*/generateDefaultBounds,
+            /*strideIncludeLowerExtent=*/strideIncludeLowerExtent);
     LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
     EntryOp op = createDataEntryOp<EntryOp>(
         builder, operandLocation, info.addr, asFortran, bounds, structured,
@@ -914,7 +922,8 @@ genPrivatizations(const Fortran::parser::AccObjectList &objectList,
             converter, builder, semanticsContext, stmtCtx, symbol, designator,
             operandLocation, asFortran, bounds,
             /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
-            /*genDefaultBounds=*/generateDefaultBounds);
+            /*genDefaultBounds=*/generateDefaultBounds,
+            /*strideIncludeLowerExtent=*/strideIncludeLowerExtent);
     LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
 
     RecipeOp recipe;
@@ -1545,7 +1554,8 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList,
             converter, builder, semanticsContext, stmtCtx, symbol, designator,
             operandLocation, asFortran, bounds,
             /*treatIndexAsSection=*/true, /*unwrapFirBox=*/unwrapFirBox,
-            /*genDefaultBounds=*/generateDefaultBounds);
+            /*genDefaultBounds=*/generateDefaultBounds,
+            /*strideIncludeLowerExtent=*/strideIncludeLowerExtent);
     LLVM_DEBUG(llvm::dbgs() << __func__ << "\n"; info.dump(llvm::dbgs()));
 
     mlir::Type reductionTy = fir::unwrapRefType(info.addr.getType());
diff --git a/flang/test/Lower/OpenACC/acc-bounds.f90 b/flang/test/Lower/OpenACC/acc-bounds.f90
index 8fea357f116a2..cff53a2bfd122 100644
--- a/flang/test/Lower/OpenACC/acc-bounds.f90
+++ b/flang/test/Lower/OpenACC/acc-bounds.f90
@@ -182,4 +182,19 @@ subroutine acc_optional_data3(a, n)
 ! CHECK: %[[NOCREATE:.*]] = acc.nocreate varPtr(%[[DECL_A]]#1 : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUNDS]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(1:n)"}
 ! CHECK: acc.data dataOperands(%[[NOCREATE]] : !fir.ref<!fir.array<?xf32>>) {
 
+  subroutine acc_explicit_shape_3d(arr)
+    real :: arr(1000,100,10)
+    !$acc data copyin(arr(:1000,:100,:10))
+    !$acc end data
+  end subroutine
+
+! Test that the stride is cummulative of the lower extents
+! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_explicit_shape_3d(
+! CHECK-SAME: %[[ARR:.*]]: !fir.ref<!fir.array<1000x100x10xf32>> {fir.bindc_name = "arr"}) {
+! CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%c0 : index) upperbound(%c999 : index) extent(%c1000 : index) stride(%c1{{.*}} : index) startIdx(%c1 : index)
+! CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%c0 : index) upperbound(%c99 : index) extent(%c100 : index) stride(%c1000{{.*}} : index) startIdx(%c1 : index)
+! CHECK: %[[BOUND3:.*]] = acc.bounds lowerbound(%c0 : index) upperbound(%c9 : index) extent(%c10 : index) stride(%c100000{{.*}} : index) startIdx(%c1 : index)
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr({{.*}} : !fir.ref<!fir.array<1000x100x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]], %[[BOUND3]]) -> !fir.ref<!fir.array<1000x100x10xf32>> {name = "arr(:1000,:100,:10)"}
+! CHECK: acc.data dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<1000x100x10xf32>>) {
+
 end module
diff --git a/flang/test/Lower/OpenACC/acc-enter-data-unwrap-defaultbounds.f90 b/flang/test/Lower/OpenACC/acc-enter-data-unwrap-defaultbounds.f90
index 3de00ebb9eb05..c42350a07c498 100644
--- a/flang/test/Lower/OpenACC/acc-enter-data-unwrap-defaultbounds.f90
+++ b/flang/test/Lower/OpenACC/acc-enter-data-unwrap-defaultbounds.f90
@@ -26,7 +26,7 @@ subroutine acc_enter_data
 !CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[LB:.*]] = arith.constant 0 : index
 !CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
 
@@ -37,7 +37,7 @@ subroutine acc_enter_data
 !CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[LB:.*]] = arith.constant 0 : index
 !CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: [[IF1:%.*]] = arith.constant true
 !CHECK: acc.enter_data if([[IF1]]) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
@@ -49,7 +49,7 @@ subroutine acc_enter_data
 !CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[LB:.*]] = arith.constant 0 : index
 !CHECK: %[[UB:.*]] = arith.subi %[[EXTENT_C10]], %[[ONE]] : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 !CHECK: [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
@@ -161,7 +161,7 @@ subroutine acc_enter_data
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB1]] : index) upperbound(%[[UB1]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
 !CHECK: %[[LB2:.*]] = arith.constant 0 : index
 !CHECK: %[[UB2:.*]] = arith.constant 4 : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%c1{{.*}} : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(1:,1:5)", structured = false}
 !CHECK: acc.enter_data   dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
@@ -172,7 +172,7 @@ subroutine acc_enter_data
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB1]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[LB:.*]] = arith.constant 0 : index
 !CHECK: %[[UB2:.*]] = arith.constant 4 : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(:10,1:5)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
@@ -182,7 +182,7 @@ subroutine acc_enter_data
 !CHECK: %[[UB:.*]] = arith.subi %c10{{.*}}, %[[ONE]] : index
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[UB:.*]] = arith.subi %c10{{.*}}, %[[ONE]] : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(:,:)", structured = false}
 end subroutine acc_enter_data
 
diff --git a/flang/test/Lower/OpenACC/acc-enter-data.f90 b/flang/test/Lower/OpenACC/acc-enter-data.f90
index d2cd097388828..3e49259c360eb 100644
--- a/flang/test/Lower/OpenACC/acc-enter-data.f90
+++ b/flang/test/Lower/OpenACC/acc-enter-data.f90
@@ -105,7 +105,7 @@ subroutine acc_enter_data
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB1]] : index) upperbound(%[[UB1]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
 !CHECK: %[[LB2:.*]] = arith.constant 0 : index
 !CHECK: %[[UB2:.*]] = arith.constant 4 : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%c1{{.*}} : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(1:,1:5)", structured = false}
 !CHECK: acc.enter_data   dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
@@ -116,7 +116,7 @@ subroutine acc_enter_data
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB1]] : index) extent(%[[C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[LB:.*]] = arith.constant 0 : index
 !CHECK: %[[UB2:.*]] = arith.constant 4 : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB2]] : index) extent(%[[EXTENT_C10]] : index) stride(%c1{{.*}} : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(:10,1:5)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
@@ -126,7 +126,7 @@ subroutine acc_enter_data
 !CHECK: %[[UB:.*]] = arith.subi %c10{{.*}}, %[[ONE]] : index
 !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[UB:.*]] = arith.subi %c10{{.*}}, %[[ONE]] : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%c10{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%c1{{.*}} : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(:,:)", structured = false}
 end subroutine acc_enter_data
 
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 70a2ba0919952..275472bc5edd9 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -377,6 +377,13 @@ def OpenACC_DataBoundsOp : OpenACC_Op<"bounds",
     but not checked for consistency). When the source language's arrays are
     not zero-based, the `startIdx` must specify the zero-position index.
 
+    The `stride` represents the distance between consecutive elements. For
+    multi-dimensional arrays, the `stride` for each outer dimension must account
+    for the complete size of all inner dimensions.
+
+    The `strideInBytes` flag indicates that the `stride` is specified in bytes
+    rather than the number of elements.
+
     Examples below show copying a slice of 10-element array except first element.
     Note that the examples use extent in data clause for C++ and upperbound
     for Fortran (as per 2.7.1). To simplify examples, the constants are used
@@ -389,7 +396,7 @@ def OpenACC_DataBoundsOp : OpenACC_Op<"bounds",
     ```
     =>
     ```mlir
-    acc.bounds lb(1) ub(9) extent(9) startIdx(0)
+    acc.bounds lb(1) ub(9) extent(9) startIdx(0) stride(1)
     ```
 
     Fortran:
@@ -399,7 +406,7 @@ def OpenACC_DataBoundsOp : OpenACC_Op<"bounds",
     ```
     =>
     ```mlir
-    acc.bounds lb(1) ub(9) extent(9) startIdx(1)
+    acc.bounds lb(1) ub(9) extent(9) startIdx(1) stride(1)
     ```
   }];
 

From 53e62c654a3fe181686f1ccd960d2e2b981e4313 Mon Sep 17 00:00:00 2001
From: Hubert Tong <hubert.reinterpretcast@gmail.com>
Date: Mon, 21 Apr 2025 19:34:33 -0400
Subject: [PATCH 668/710] [compiler-rt][profile][tests][NFC] Avoid using a.out
 from PATH (#136465)

Fix use of `a.out` from the PATH by specifying `./a.out`.
---
 .../AIX/pgo-lto-bcdtor-function-section.test  | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/compiler-rt/test/profile/AIX/pgo-lto-bcdtor-function-section.test b/compiler-rt/test/profile/AIX/pgo-lto-bcdtor-function-section.test
index 36170716b56c2..cc351ef4e0acd 100755
--- a/compiler-rt/test/profile/AIX/pgo-lto-bcdtor-function-section.test
+++ b/compiler-rt/test/profile/AIX/pgo-lto-bcdtor-function-section.test
@@ -15,22 +15,22 @@ int main() { return foo() - 3; }
 // RUN: %clang_pgogen -O2 -c -fno-function-sections main.c
 //
 // RUN: %clang_pgogen -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=BOTH
 //
 // RUN: %clang_pgogen -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=BOTH
 
 // RUN: %clang_pgogen -O2 -c -ffunction-sections foo.c
 // RUN: %clang_pgogen -O2 -c -ffunction-sections main.c
 //
 // RUN: %clang_pgogen -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=BOTH
 //
 // RUN: %clang_pgogen -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=BOTH
 
 // ## no PGO at compile step, but PGO at link step.
@@ -38,11 +38,11 @@ int main() { return foo() - 3; }
 // RUN: %clang -O2 -c  main.c
 //
 // RUN: %clang_pgogen -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=NONE
 //
 // RUN: %clang_pgogen -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=NONE
 
 // # LTO, with and without function-sections, and all permutations of -bcdtors
@@ -51,44 +51,44 @@ int main() { return foo() - 3; }
 // RUN: %clang -O2 -c -flto main.c
 //
 // RUN: %clang_pgogen -flto -fno-function-sections -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=NONE
 //
 // RUN: %clang_pgogen -flto -fno-function-sections -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=NONE
 //
 // RUN: %clang_pgogen -flto -ffunction-sections -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=NONE
 //
 // RUN: %clang_pgogen -flto -ffunction-sections -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=NONE
 
 // ## LTO one file, PGO at compile and link
 // RUN: %clang -O2 -c -fno-function-sections foo.c
 // RUN: %clang_pgogen -O2 -c -flto main.c
 // RUN: %clang_pgogen -flto -fno-function-sections -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=MAIN
 
 // RUN: %clang -O2 -c -flto foo.c
 // RUN: %clang_pgogen -O2 -c -fno-function-sections main.c
 // RUN: %clang_pgogen -flto -fno-function-sections -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=MAIN
 
 // RUN: %clang -O2 -c -flto foo.c
 // RUN: %clang_pgogen -O2 -c -ffunction-sections main.c
 // RUN: %clang_pgogen -flto -ffunction-sections -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=MAIN
 
 // RUN: %clang -O2 -c -ffunction-sections foo.c
 // RUN: %clang_pgogen -O2 -c -flto main.c
 // RUN: %clang_pgogen -flto -ffunction-sections -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=MAIN
 
 // ## LTO and PGO both files
@@ -96,19 +96,19 @@ int main() { return foo() - 3; }
 // RUN: %clang_pgogen -O2 -c -flto main.c
 //
 // RUN: %clang_pgogen -flto -fno-function-sections -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=BOTH
 //
 // RUN: %clang_pgogen -flto -fno-function-sections -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=BOTH
 //
 // RUN: %clang_pgogen -flto -ffunction-sections -Wl,-bcdtors:all foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=BOTH
 //
 // RUN: %clang_pgogen -flto -ffunction-sections -Wl,-bcdtors:mbr foo.o main.o
-// RUN: rm -f default* && %run a.out
+// RUN: rm -f default* && %run ./a.out
 // RUN: llvm-profdata show --all-functions default* | FileCheck  %s -check-prefix=BOTH
 
 // BOTH-DAG: foo:

From d2fedd67debf011cec7465e0c7ac330dfb548b72 Mon Sep 17 00:00:00 2001
From: Augie Fackler <augie@google.com>
Date: Mon, 21 Apr 2025 19:37:31 -0400
Subject: [PATCH 669/710] [NVPTX] mark unused variable introduced by
 56910a8b1b302ebf37e9d30bd200091fd23dc232 [NFC]

---
 llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index bd396cd6a40b5..f4362fe8d9056 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -536,7 +536,7 @@ static void handleByValParam(const NVPTXTargetMachine &TM, Argument *Arg) {
   const bool IsGridConstant = HasCvtaParam && isParamGridConstant(*Arg);
   const DataLayout &DL = Func->getDataLayout();
   BasicBlock::iterator FirstInst = Func->getEntryBlock().begin();
-  Type *StructType = Arg->getParamByValType();
+  [[maybe_unused]] Type *StructType = Arg->getParamByValType();
   assert(StructType && "Missing byval type");
 
   ArgUseChecker AUC(DL, IsGridConstant);

From d1f4f52aa6f2276da8880b5003f220ce6211ad44 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Mon, 21 Apr 2025 16:45:19 -0700
Subject: [PATCH 670/710] [msan] Handle
 x86.avx512fp16.{add,sub.mul,div,min,max}.ph.512 (#136619)

These are handled similarly to x86_avx512_(min|max)_p[sd]_512 intrinsics
(https://github.com/llvm/llvm-project/pull/124421) i.e., using
maybeHandleSimpleNomemIntrinsic, with the last parameter being the
rounding method.

Updates the test from https://github.com/llvm/llvm-project/pull/136260
---
 .../Instrumentation/MemorySanitizer.cpp       |   6 +
 .../X86/avx512fp16-arith-intrinsics.ll        | 382 +++++-------------
 2 files changed, 98 insertions(+), 290 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 506a880e3f67d..8e31e8d2a4fbd 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -5022,6 +5022,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     }
 
     // Packed
+    case Intrinsic::x86_avx512fp16_add_ph_512:
+    case Intrinsic::x86_avx512fp16_sub_ph_512:
+    case Intrinsic::x86_avx512fp16_mul_ph_512:
+    case Intrinsic::x86_avx512fp16_div_ph_512:
+    case Intrinsic::x86_avx512fp16_max_ph_512:
+    case Intrinsic::x86_avx512fp16_min_ph_512:
     case Intrinsic::x86_avx512_min_ps_512:
     case Intrinsic::x86_avx512_min_pd_512:
     case Intrinsic::x86_avx512_max_ps_512:
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll
index 121a17894fe9e..69d49008e1b78 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll
@@ -4,8 +4,6 @@
 ; Forked from llvm/test/CodeGen/X86/avx512fp16-arith-intrinsics.ll
 ;
 ; Handled suboptimally (visitInstruction):
-; - llvm.x86.avx512fp16.add.ph.512
-; - llvm.x86.avx512fp16.div.ph.512
 ; - llvm.x86.avx512fp16.mask.vcvtpd2ph.512
 ; - llvm.x86.avx512fp16.mask.vcvtph2pd.512
 ; - llvm.x86.avx512fp16.mask.vcvtph2psx.512
@@ -14,10 +12,6 @@
 ; - llvm.x86.avx512fp16.mask.vcvtsh2sd.round
 ; - llvm.x86.avx512fp16.mask.vcvtsh2ss.round
 ; - llvm.x86.avx512fp16.mask.vcvtss2sh.round
-; - llvm.x86.avx512fp16.max.ph.512
-; - llvm.x86.avx512fp16.min.ph.512
-; - llvm.x86.avx512fp16.mul.ph.512
-; - llvm.x86.avx512fp16.sub.ph.512
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -30,18 +24,10 @@ define <32 x half> @test_int_x86_avx512fp16_add_ph_512(<32 x half> %x1, <32 x ha
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       [[BB5]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x half> [[RES]]
 ;
   %res = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
@@ -58,22 +44,14 @@ define <32 x half> @test_int_x86_avx512fp16_mask_add_ph_512(<32 x half> %src, <3
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
 ; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> [[SRC]]
@@ -97,9 +75,9 @@ define <32 x half> @test_int_x86_avx512fp16_maskz_add_ph_512(<32 x half> %src, <
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1:![0-9]+]]
 ; CHECK:       [[BB6]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       [[BB7]]:
 ; CHECK-NEXT:    [[VAL:%.*]] = load <32 x half>, ptr [[PTR]], align 64
@@ -107,39 +85,23 @@ define <32 x half> @test_int_x86_avx512fp16_maskz_add_ph_512(<32 x half> %src, <
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP10]], align 64
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
-; CHECK:       [[BB13]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <32 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP4]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
-; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP19]], <32 x i16> [[TMP15]]
 ; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP20]], 0
-; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
-; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP21]], 0
-; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
-; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
-; CHECK:       [[BB22]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <32 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <32 x i16> [[_MSPROP2]], zeroinitializer
 ; CHECK-NEXT:    [[T2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[VAL]], i32 4)
-; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP3]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x half> [[T2]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP25]], zeroinitializer
-; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], [[_MSPROP3]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP27]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP28]], <32 x i16> [[TMP24]]
 ; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T2]], <32 x half> zeroinitializer
@@ -168,22 +130,14 @@ define <32 x half> @test_int_x86_avx512fp16_add_ph_512_round(<32 x half> %x1, <3
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[T1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.add.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 10)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[T1]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
 ; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T1]], <32 x half> [[SRC]]
@@ -204,18 +158,10 @@ define <32 x half> @test_int_x86_avx512fp16_sub_ph_512(<32 x half> %x1, <32 x ha
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
-; CHECK:       [[BB5]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x half> [[RES]]
 ;
   %res = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
@@ -232,22 +178,14 @@ define <32 x half> @test_int_x86_avx512fp16_mask_sub_ph_512(<32 x half> %src, <3
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
 ; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> [[SRC]]
@@ -281,39 +219,23 @@ define <32 x half> @test_int_x86_avx512fp16_maskz_sub_ph_512(<32 x half> %src, <
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP10]], align 64
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
-; CHECK:       [[BB13]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <32 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP4]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
-; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP19]], <32 x i16> [[TMP15]]
 ; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP20]], 0
-; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
-; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP21]], 0
-; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
-; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
-; CHECK:       [[BB22]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <32 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <32 x i16> [[_MSPROP2]], zeroinitializer
 ; CHECK-NEXT:    [[T2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[VAL]], i32 4)
-; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP3]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x half> [[T2]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP25]], zeroinitializer
-; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], [[_MSPROP3]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP27]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP28]], <32 x i16> [[TMP24]]
 ; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T2]], <32 x half> zeroinitializer
@@ -342,22 +264,14 @@ define <32 x half> @test_int_x86_avx512fp16_sub_ph_512_round(<32 x half> %x1, <3
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[T1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.sub.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 10)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[T1]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
 ; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T1]], <32 x half> [[SRC]]
@@ -378,18 +292,10 @@ define <32 x half> @test_int_x86_avx512fp16_mul_ph_512(<32 x half> %x1, <32 x ha
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
-; CHECK:       [[BB5]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x half> [[RES]]
 ;
   %res = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
@@ -406,22 +312,14 @@ define <32 x half> @test_int_x86_avx512fp16_mask_mul_ph_512(<32 x half> %src, <3
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
 ; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> [[SRC]]
@@ -455,39 +353,23 @@ define <32 x half> @test_int_x86_avx512fp16_maskz_mul_ph_512(<32 x half> %src, <
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP10]], align 64
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
-; CHECK:       [[BB13]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <32 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP4]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
-; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP19]], <32 x i16> [[TMP15]]
 ; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP20]], 0
-; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
-; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP21]], 0
-; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
-; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
-; CHECK:       [[BB22]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <32 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <32 x i16> [[_MSPROP2]], zeroinitializer
 ; CHECK-NEXT:    [[T2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[VAL]], i32 4)
-; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP3]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x half> [[T2]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP25]], zeroinitializer
-; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], [[_MSPROP3]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP27]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP28]], <32 x i16> [[TMP24]]
 ; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T2]], <32 x half> zeroinitializer
@@ -516,22 +398,14 @@ define <32 x half> @test_int_x86_avx512fp16_mul_ph_512_round(<32 x half> %x1, <3
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[T1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.mul.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 10)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[T1]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
 ; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T1]], <32 x half> [[SRC]]
@@ -552,18 +426,10 @@ define <32 x half> @test_int_x86_avx512fp16_div_ph_512(<32 x half> %x1, <32 x ha
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
-; CHECK:       [[BB5]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x half> [[RES]]
 ;
   %res = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> %x1, <32 x half> %x2, i32 4)
@@ -580,22 +446,14 @@ define <32 x half> @test_int_x86_avx512fp16_mask_div_ph_512(<32 x half> %src, <3
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
 ; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> [[SRC]]
@@ -629,39 +487,23 @@ define <32 x half> @test_int_x86_avx512fp16_maskz_div_ph_512(<32 x half> %src, <
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP10]], align 64
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x i16> [[TMP4]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
-; CHECK:       [[BB13]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <32 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP4]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 4)
-; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
-; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP19]], <32 x i16> [[TMP15]]
 ; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP20]], 0
-; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
-; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP21]], 0
-; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
-; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
-; CHECK:       [[BB22]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <32 x i16> [[TMP3]], [[_MSLD]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <32 x i16> [[_MSPROP2]], zeroinitializer
 ; CHECK-NEXT:    [[T2:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[VAL]], i32 4)
-; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP3]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x half> [[T2]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP25]], zeroinitializer
-; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], [[_MSPROP3]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP27]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP28]], <32 x i16> [[TMP24]]
 ; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T2]], <32 x half> zeroinitializer
@@ -690,22 +532,14 @@ define <32 x half> @test_int_x86_avx512fp16_div_ph_512_round(<32 x half> %x1, <3
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[T1:%.*]] = call <32 x half> @llvm.x86.avx512fp16.div.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 10)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x half> [[T1]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <32 x half> [[SRC]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP15]], <32 x i16> [[TMP10]]
 ; CHECK-NEXT:    [[RES:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[T1]], <32 x half> [[SRC]]
@@ -751,18 +585,10 @@ define <32 x half> @test_int_x86_avx512fp16_min_ph_512_sae(<32 x half> %x1, <32
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
-; CHECK:       [[BB5]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.min.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 8)
-; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x half> [[RES0]]
 ;
   %res0 = call <32 x half> @llvm.x86.avx512fp16.min.ph.512(<32 x half> %x1, <32 x half> %x2, i32 8)
@@ -778,21 +604,13 @@ define <32 x half> @test_int_x86_avx512fp16_maskz_min_ph_512_sae(<32 x half> %x1
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.min.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 8)
-; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor <32 x i16> [[TMP10]], zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i16> [[TMP13]], <32 x i16> [[TMP9]]
 ; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer
@@ -838,18 +656,10 @@ define <32 x half> @test_int_x86_avx512fp16_max_ph_512_sae(<32 x half> %x1, <32
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
-; CHECK:       [[BB5]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.max.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 8)
-; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x half> [[RES0]]
 ;
   %res0 = call <32 x half> @llvm.x86.avx512fp16.max.ph.512(<32 x half> %x1, <32 x half> %x2, i32 8)
@@ -865,21 +675,13 @@ define <32 x half> @test_int_x86_avx512fp16_maskz_max_ph_512_sae(<32 x half> %x1
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
 ; CHECK-NEXT:    [[MASK:%.*]] = bitcast i32 [[MSK]] to <32 x i1>
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], zeroinitializer
 ; CHECK-NEXT:    [[RES0:%.*]] = call <32 x half> @llvm.x86.avx512fp16.max.ph.512(<32 x half> [[X1]], <32 x half> [[X2]], i32 8)
-; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[MASK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[MASK]], <32 x i16> [[_MSPROP1]], <32 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x half> [[RES0]] to <32 x i16>
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor <32 x i16> [[TMP10]], zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], [[_MSPROP1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i16> [[TMP13]], <32 x i16> [[TMP9]]
 ; CHECK-NEXT:    [[RES1:%.*]] = select <32 x i1> [[MASK]], <32 x half> [[RES0]], <32 x half> zeroinitializer

From d5b40c71f6be972f677de5d9886f91866df007b5 Mon Sep 17 00:00:00 2001
From: royitaqi <royitaqi@users.noreply.github.com>
Date: Mon, 21 Apr 2025 16:53:14 -0700
Subject: [PATCH 671/710] [lldb] Avoid force loading symbols in statistics
 collection (#136236)

Currently, `DebuggerStats::ReportStatistics()` calls
`Module::GetSymtab(/*can_create=*/false)`, but then the latter calls
`SymbolFile::GetSymtab()`. This will load symbols if haven't yet. See
stacktrace below.

The problem is that `DebuggerStats::ReportStatistics` should be
read-only. This is especially important because it reports stats for
symtab parsing/indexing time, which could be affected by the reporting
itself if it's not read-only.

This patch fixes this problem by adding an optional parameter
`SymbolFile::GetSymtab(bool can_create = true)` and receive the `false`
value passed down from `Module::GetSymtab(/*can_create=*/false)` when
the call was initiated from `DebuggerStats::ReportStatistics()`.
---
 lldb/include/lldb/Symbol/ObjectFile.h         |  2 +-
 lldb/include/lldb/Symbol/SymbolFile.h         |  4 +-
 lldb/include/lldb/Symbol/SymbolFileOnDemand.h |  4 +-
 lldb/source/Core/Module.cpp                   |  2 +-
 lldb/source/Symbol/ObjectFile.cpp             |  5 +-
 lldb/source/Symbol/SymbolFile.cpp             |  4 +-
 .../commands/statistics/basic/TestStats.py    | 23 ++++++
 lldb/unittests/Symbol/LineTableTest.cpp       |  2 +-
 lldb/unittests/Symbol/SymtabTest.cpp          | 78 ++++++++++++++++++-
 9 files changed, 111 insertions(+), 13 deletions(-)

diff --git a/lldb/include/lldb/Symbol/ObjectFile.h b/lldb/include/lldb/Symbol/ObjectFile.h
index cfcca04a76de8..7fca6383fa9f3 100644
--- a/lldb/include/lldb/Symbol/ObjectFile.h
+++ b/lldb/include/lldb/Symbol/ObjectFile.h
@@ -319,7 +319,7 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
   ///
   /// \return
   ///     The symbol table for this object file.
-  Symtab *GetSymtab();
+  Symtab *GetSymtab(bool can_create = true);
 
   /// Parse the symbol table into the provides symbol table object.
   ///
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
index f35d3ee9f22ae..df2f263b18e17 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -144,7 +144,7 @@ class SymbolFile : public PluginInterface {
   virtual uint32_t GetNumCompileUnits() = 0;
   virtual lldb::CompUnitSP GetCompileUnitAtIndex(uint32_t idx) = 0;
 
-  virtual Symtab *GetSymtab() = 0;
+  virtual Symtab *GetSymtab(bool can_create = true) = 0;
 
   virtual lldb::LanguageType ParseLanguage(CompileUnit &comp_unit) = 0;
   /// Return the Xcode SDK comp_unit was compiled against.
@@ -533,7 +533,7 @@ class SymbolFileCommon : public SymbolFile {
     return m_abilities;
   }
 
-  Symtab *GetSymtab() override;
+  Symtab *GetSymtab(bool can_create = true) override;
 
   ObjectFile *GetObjectFile() override { return m_objfile_sp.get(); }
   const ObjectFile *GetObjectFile() const override {
diff --git a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
index 7a366bfabec86..6ed389a48880c 100644
--- a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
+++ b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
@@ -186,7 +186,9 @@ class SymbolFileOnDemand : public lldb_private::SymbolFile {
 
   uint32_t GetAbilities() override;
 
-  Symtab *GetSymtab() override { return m_sym_file_impl->GetSymtab(); }
+  Symtab *GetSymtab(bool can_create = true) override {
+    return m_sym_file_impl->GetSymtab(can_create);
+  }
 
   ObjectFile *GetObjectFile() override {
     return m_sym_file_impl->GetObjectFile();
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index ad7046e596278..625c14e4a2153 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -997,7 +997,7 @@ SymbolFile *Module::GetSymbolFile(bool can_create, Stream *feedback_strm) {
 
 Symtab *Module::GetSymtab(bool can_create) {
   if (SymbolFile *symbols = GetSymbolFile(can_create))
-    return symbols->GetSymtab();
+    return symbols->GetSymtab(can_create);
   return nullptr;
 }
 
diff --git a/lldb/source/Symbol/ObjectFile.cpp b/lldb/source/Symbol/ObjectFile.cpp
index 2f2c59d6af620..afd0d298e675e 100644
--- a/lldb/source/Symbol/ObjectFile.cpp
+++ b/lldb/source/Symbol/ObjectFile.cpp
@@ -734,10 +734,9 @@ void llvm::format_provider<ObjectFile::Strata>::format(
   }
 }
 
-
-Symtab *ObjectFile::GetSymtab() {
+Symtab *ObjectFile::GetSymtab(bool can_create) {
   ModuleSP module_sp(GetModule());
-  if (module_sp) {
+  if (module_sp && can_create) {
     // We can't take the module lock in ObjectFile::GetSymtab() or we can
     // deadlock in DWARF indexing when any file asks for the symbol table from
     // an object file. This currently happens in the preloading of symbols in
diff --git a/lldb/source/Symbol/SymbolFile.cpp b/lldb/source/Symbol/SymbolFile.cpp
index 94e32b55572dd..870d778dca740 100644
--- a/lldb/source/Symbol/SymbolFile.cpp
+++ b/lldb/source/Symbol/SymbolFile.cpp
@@ -152,10 +152,10 @@ void SymbolFile::AssertModuleLock() {
 
 SymbolFile::RegisterInfoResolver::~RegisterInfoResolver() = default;
 
-Symtab *SymbolFileCommon::GetSymtab() {
+Symtab *SymbolFileCommon::GetSymtab(bool can_create) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
   // Fetch the symtab from the main object file.
-  auto *symtab = GetMainObjectFile()->GetSymtab();
+  auto *symtab = GetMainObjectFile()->GetSymtab(can_create);
   if (m_symtab != symtab) {
     m_symtab = symtab;
 
diff --git a/lldb/test/API/commands/statistics/basic/TestStats.py b/lldb/test/API/commands/statistics/basic/TestStats.py
index 0265a0d7c9948..e2c539bfaeddb 100644
--- a/lldb/test/API/commands/statistics/basic/TestStats.py
+++ b/lldb/test/API/commands/statistics/basic/TestStats.py
@@ -209,6 +209,29 @@ def test_default_no_run(self):
             )
             self.assertGreater(module_stats["symbolsLoaded"], 0)
 
+    def test_default_no_run_no_preload_symbols(self):
+        """Test "statistics dump" without running the target and without
+        preloading symbols.
+
+        Checks that symbol count are zero.
+        """
+        # Make sure symbols will not be preloaded.
+        self.runCmd("settings set target.preload-symbols false")
+
+        # Build and load the target
+        self.build()
+        target = self.createTestTarget()
+
+        # Get statistics
+        debug_stats = self.get_stats()
+
+        # No symbols should be loaded
+        self.assertEqual(debug_stats["totalSymbolsLoaded"], 0)
+
+        # No symbols should be loaded in each module
+        for module_stats in debug_stats["modules"]:
+            self.assertEqual(module_stats["symbolsLoaded"], 0)
+
     def test_default_with_run(self):
         """Test "statistics dump" when running the target to a breakpoint.
 
diff --git a/lldb/unittests/Symbol/LineTableTest.cpp b/lldb/unittests/Symbol/LineTableTest.cpp
index f67ac46c738e7..eadab40a37fac 100644
--- a/lldb/unittests/Symbol/LineTableTest.cpp
+++ b/lldb/unittests/Symbol/LineTableTest.cpp
@@ -58,7 +58,7 @@ class FakeSymbolFile : public SymbolFile {
   uint32_t CalculateAbilities() override { return UINT32_MAX; }
   uint32_t GetNumCompileUnits() override { return 1; }
   CompUnitSP GetCompileUnitAtIndex(uint32_t) override { return m_cu_sp; }
-  Symtab *GetSymtab() override { return nullptr; }
+  Symtab *GetSymtab(bool can_create = true) override { return nullptr; }
   LanguageType ParseLanguage(CompileUnit &) override { return eLanguageTypeC; }
   size_t ParseFunctions(CompileUnit &) override { return 0; }
   bool ParseLineTable(CompileUnit &) override { return true; }
diff --git a/lldb/unittests/Symbol/SymtabTest.cpp b/lldb/unittests/Symbol/SymtabTest.cpp
index e431c31e1eac1..d3f462fab86a1 100644
--- a/lldb/unittests/Symbol/SymtabTest.cpp
+++ b/lldb/unittests/Symbol/SymtabTest.cpp
@@ -721,7 +721,7 @@ TEST_F(SymtabTest, TestDecodeCStringMaps) {
   ASSERT_NE(symbol, nullptr);
 }
 
-TEST_F(SymtabTest, TestSymtabCreatedOnDemand) {
+TEST_F(SymtabTest, TestSymbolFileCreatedOnDemand) {
   auto ExpectedFile = TestFile::fromYaml(R"(
 --- !ELF
 FileHeader:
@@ -749,7 +749,7 @@ TEST_F(SymtabTest, TestSymtabCreatedOnDemand) {
   ASSERT_THAT_EXPECTED(ExpectedFile, llvm::Succeeded());
   auto module_sp = std::make_shared<Module>(ExpectedFile->moduleSpec());
 
-  // The symbol table should not be loaded by default.
+  // The symbol file should not be created by default.
   Symtab *module_symtab = module_sp->GetSymtab(/*can_create=*/false);
   ASSERT_EQ(module_symtab, nullptr);
 
@@ -761,3 +761,77 @@ TEST_F(SymtabTest, TestSymtabCreatedOnDemand) {
   Symtab *cached_module_symtab = module_sp->GetSymtab(/*can_create=*/false);
   ASSERT_EQ(module_symtab, cached_module_symtab);
 }
+
+TEST_F(SymtabTest, TestSymbolTableCreatedOnDemand) {
+  const char *yamldata = R"(
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_386
+DWARF:
+  debug_abbrev:
+    - Table:
+        - Code:            0x00000001
+          Tag:             DW_TAG_compile_unit
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_addr_base
+              Form:            DW_FORM_sec_offset
+  debug_info:
+    - Version:         5
+      AddrSize:        4
+      UnitType:        DW_UT_compile
+      Entries:
+        - AbbrCode:        0x00000001
+          Values:
+            - Value:           0x8 # Offset of the first Address past the header
+        - AbbrCode:        0x0
+  debug_addr:
+    - Version: 5
+      AddressSize: 4
+      Entries:
+        - Address: 0x1234
+        - Address: 0x5678
+  debug_line:
+    - Length:          42
+      Version:         2
+      PrologueLength:  36
+      MinInstLength:   1
+      DefaultIsStmt:   1
+      LineBase:        251
+      LineRange:       14
+      OpcodeBase:      13
+      StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
+      IncludeDirs:
+        - '/tmp'
+      Files:
+        - Name:            main.cpp
+          DirIdx:          1
+          ModTime:         0
+          Length:          0
+)";
+  llvm::Expected<TestFile> file = TestFile::fromYaml(yamldata);
+  EXPECT_THAT_EXPECTED(file, llvm::Succeeded());
+  auto module_sp = std::make_shared<Module>(file->moduleSpec());
+
+  SymbolFile *symbol_file = module_sp->GetSymbolFile();
+  // At this point, the symbol table is not created. This is because the above
+  // yaml data contains the necessary sections in order for
+  // SymbolFileDWARF::CalculateAbilities() to identify all abilities, saving it
+  // from calling SymbolFileDWARFDebugMap::CalculateAbilities(), which
+  // eventually loads the symbol table, which we don't want.
+
+  // The symbol table should not be created if asked not to.
+  Symtab *symtab = symbol_file->GetSymtab(/*can_create=*/false);
+  ASSERT_EQ(symtab, nullptr);
+
+  // But it should be created on demand.
+  symtab = symbol_file->GetSymtab(/*can_create=*/true);
+  ASSERT_NE(symtab, nullptr);
+
+  // And we should be able to get it again once it has been created.
+  Symtab *cached_symtab = symbol_file->GetSymtab(/*can_create=*/false);
+  ASSERT_EQ(symtab, cached_symtab);
+}

From a1db2c64918efa3d74dfdb402b8cff30bc555521 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 21 Apr 2025 16:53:15 -0700
Subject: [PATCH 672/710] [RISCV] Remove duplicate call to MFI.getStackID(FI).
 NFC

Reuse existing local variable.
---
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 89a9f7d871391..1fd1048bcebf8 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -1272,7 +1272,7 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
     // |--------------------------|
     // | VarSize objects          |
     // |--------------------------| <-- SP
-    if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
+    if (StackID == TargetStackID::ScalableVector) {
       assert(!RI->hasStackRealignment(MF) &&
              "Can't index across variable sized realign");
       // We don't expect any extra RVV alignment padding, as the stack size

From 2cdf474f12090919e5b14c6ef922e5d5df407418 Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Mon, 21 Apr 2025 17:05:34 -0700
Subject: [PATCH 673/710] [MemProf] Merge callee clones as needed before
 assigning functions (#135702)

We perform cloning for each allocation node separately. However, this
sometimes results in a situation where the same node calls multiple
clones of the same callee, created for different allocations. This
causes issues when assigning functions to these clones, as each node can
in reality only call a single callee clone.

To address this, before assigning functions, merge callee clone nodes as
needed using a post order traversal from the allocations. We attempt to
use existing clones as the merge node when legal, and to share them
among callers with the same properties (callers calling the same set of
callee clone nodes for the same allocations).

Without this fix, in some cases incorrect function assignment will lead
to calling the wrong allocation clone. In fact, this showed up in an
existing test, that I didn't notice as it existed to test earlier parts
of the cloning process.
---
 .../IPO/MemProfContextDisambiguation.cpp      | 364 ++++++++++++++
 .../mergenodes.ll                             | 404 +++++++++++++++
 .../mergenodes2.ll                            | 474 ++++++++++++++++++
 .../overlapping-contexts.ll                   |  29 +-
 4 files changed, 1259 insertions(+), 12 deletions(-)
 create mode 100644 llvm/test/Transforms/MemProfContextDisambiguation/mergenodes.ll
 create mode 100644 llvm/test/Transforms/MemProfContextDisambiguation/mergenodes2.ll

diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 257a7de6935d7..207e86b22db1d 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -90,6 +90,10 @@ STATISTIC(FoundProfiledCalleeMaxDepth,
 STATISTIC(FoundProfiledCalleeNonUniquelyCount,
           "Number of profiled callees found via multiple tail call chains");
 STATISTIC(DeferredBackedges, "Number of backedges with deferred cloning");
+STATISTIC(NewMergedNodes, "Number of new nodes created during merging");
+STATISTIC(NonNewMergedNodes, "Number of non new nodes used during merging");
+STATISTIC(MissingAllocForContextId,
+          "Number of missing alloc nodes for context ids");
 
 static cl::opt<std::string> DotFilePathPrefix(
     "memprof-dot-file-path-prefix", cl::init(""), cl::Hidden,
@@ -160,6 +164,13 @@ static cl::opt<bool> CloneRecursiveContexts(
     "memprof-clone-recursive-contexts", cl::init(true), cl::Hidden,
     cl::desc("Allow cloning of contexts through recursive cycles"));
 
+// Generally this is needed for correct assignment of allocation clones to
+// function clones, however, allow it to be disabled for debugging while the
+// functionality is new and being tested more widely.
+static cl::opt<bool>
+    MergeClones("memprof-merge-clones", cl::init(true), cl::Hidden,
+                cl::desc("Merge clones before assigning functions"));
+
 // When disabled, try to detect and prevent cloning of recursive contexts.
 // This is only necessary until we support cloning through recursive cycles.
 // Leave on by default for now, as disabling requires a little bit of compile
@@ -560,6 +571,10 @@ class CallsiteContextGraph {
   /// Mark backedges via the standard DFS based backedge algorithm.
   void markBackedges();
 
+  /// Merge clones generated during cloning for different allocations but that
+  /// are called by the same caller node, to ensure proper function assignment.
+  void mergeClones();
+
   // Try to partition calls on the given node (already placed into the AllCalls
   // array) by callee function, creating new copies of Node as needed to hold
   // calls with different callees, and moving the callee edges appropriately.
@@ -778,6 +793,21 @@ class CallsiteContextGraph {
   void markBackedges(ContextNode *Node, DenseSet<const ContextNode *> &Visited,
                      DenseSet<const ContextNode *> &CurrentStack);
 
+  /// Recursive helper for merging clones.
+  void
+  mergeClones(ContextNode *Node, DenseSet<const ContextNode *> &Visited,
+              DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
+  /// Main worker for merging callee clones for a given node.
+  void mergeNodeCalleeClones(
+      ContextNode *Node, DenseSet<const ContextNode *> &Visited,
+      DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
+  /// Helper to find other callers of the given set of callee edges that can
+  /// share the same callee merge node.
+  void findOtherCallersToShareMerge(
+      ContextNode *Node, std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
+      DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
+      DenseSet<ContextNode *> &OtherCallersToShareMerge);
+
   /// Recursively perform cloning on the graph for the given Node and its
   /// callers, in order to uniquely identify the allocation behavior of an
   /// allocation given its context. The context ids of the allocation being
@@ -4018,6 +4048,338 @@ IndexCallsiteContextGraph::cloneFunctionForCallsite(
   return {Func.func(), CloneNo};
 }
 
+// We perform cloning for each allocation node separately. However, this
+// sometimes results in a situation where the same node calls multiple
+// clones of the same callee, created for different allocations. This
+// causes issues when assigning functions to these clones, as each node can
+// in reality only call a single callee clone.
+//
+// To address this, before assigning functions, merge callee clone nodes as
+// needed using a post order traversal from the allocations. We attempt to
+// use existing clones as the merge node when legal, and to share them
+// among callers with the same properties (callers calling the same set of
+// callee clone nodes for the same allocations).
+//
+// Without this fix, in some cases incorrect function assignment will lead
+// to calling the wrong allocation clone.
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones() {
+  if (!MergeClones)
+    return;
+
+  // Generate a map from context id to the associated allocation node for use
+  // when merging clones.
+  DenseMap<uint32_t, ContextNode *> ContextIdToAllocationNode;
+  for (auto &Entry : AllocationCallToContextNodeMap) {
+    auto *Node = Entry.second;
+    for (auto Id : Node->getContextIds())
+      ContextIdToAllocationNode[Id] = Node->getOrigNode();
+    for (auto *Clone : Node->Clones) {
+      for (auto Id : Clone->getContextIds())
+        ContextIdToAllocationNode[Id] = Clone->getOrigNode();
+    }
+  }
+
+  // Post order traversal starting from allocations to ensure each callsite
+  // calls a single clone of its callee. Callee nodes that are clones of each
+  // other are merged (via new merge nodes if needed) to achieve this.
+  DenseSet<const ContextNode *> Visited;
+  for (auto &Entry : AllocationCallToContextNodeMap) {
+    auto *Node = Entry.second;
+
+    mergeClones(Node, Visited, ContextIdToAllocationNode);
+
+    // Make a copy so the recursive post order traversal that may create new
+    // clones doesn't mess up iteration. Note that the recursive traversal
+    // itself does not call mergeClones on any of these nodes, which are all
+    // (clones of) allocations.
+    auto Clones = Node->Clones;
+    for (auto *Clone : Clones)
+      mergeClones(Clone, Visited, ContextIdToAllocationNode);
+  }
+
+  if (DumpCCG) {
+    dbgs() << "CCG after merging:\n";
+    dbgs() << *this;
+  }
+  if (ExportToDot)
+    exportToDot("aftermerge");
+
+  if (VerifyCCG) {
+    check();
+  }
+}
+
+// Recursive helper for above mergeClones method.
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones(
+    ContextNode *Node, DenseSet<const ContextNode *> &Visited,
+    DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
+  auto Inserted = Visited.insert(Node);
+  if (!Inserted.second)
+    return;
+
+  // Make a copy since the recursive call may move a caller edge to a new
+  // callee, messing up the iterator.
+  auto CallerEdges = Node->CallerEdges;
+  for (auto CallerEdge : CallerEdges) {
+    // Skip any caller edge moved onto a different callee during recursion.
+    if (CallerEdge->Callee != Node)
+      continue;
+    mergeClones(CallerEdge->Caller, Visited, ContextIdToAllocationNode);
+  }
+
+  // Merge for this node after we handle its callers.
+  mergeNodeCalleeClones(Node, Visited, ContextIdToAllocationNode);
+}
+
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeNodeCalleeClones(
+    ContextNode *Node, DenseSet<const ContextNode *> &Visited,
+    DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
+  // Ignore Node if we moved all of its contexts to clones.
+  if (Node->emptyContextIds())
+    return;
+
+  // First identify groups of clones among Node's callee edges, by building
+  // a map from each callee base node to the associated callee edges from Node.
+  MapVector<ContextNode *, std::vector<std::shared_ptr<ContextEdge>>>
+      OrigNodeToCloneEdges;
+  for (const auto &E : Node->CalleeEdges) {
+    auto *Callee = E->Callee;
+    if (!Callee->CloneOf && Callee->Clones.empty())
+      continue;
+    ContextNode *Base = Callee->getOrigNode();
+    OrigNodeToCloneEdges[Base].push_back(E);
+  }
+
+  // Helper for callee edge sorting below. Return true if A's callee has fewer
+  // caller edges than B, or if A is a clone and B is not, or if A's first
+  // context id is smaller than B's.
+  auto CalleeCallerEdgeLessThan = [](const std::shared_ptr<ContextEdge> &A,
+                                     const std::shared_ptr<ContextEdge> &B) {
+    if (A->Callee->CallerEdges.size() != B->Callee->CallerEdges.size())
+      return A->Callee->CallerEdges.size() < B->Callee->CallerEdges.size();
+    if (A->Callee->CloneOf && !B->Callee->CloneOf)
+      return true;
+    else if (!A->Callee->CloneOf && B->Callee->CloneOf)
+      return false;
+    // Use the first context id for each edge as a
+    // tie-breaker.
+    return *A->ContextIds.begin() < *B->ContextIds.begin();
+  };
+
+  // Process each set of callee clones called by Node, performing the needed
+  // merging.
+  for (auto Entry : OrigNodeToCloneEdges) {
+    // CalleeEdges is the set of edges from Node reaching callees that are
+    // mutual clones of each other.
+    auto &CalleeEdges = Entry.second;
+    auto NumCalleeClones = CalleeEdges.size();
+    // A single edge means there is no merging needed.
+    if (NumCalleeClones == 1)
+      continue;
+    // Sort the CalleeEdges calling this group of clones in ascending order of
+    // their caller edge counts, putting the original non-clone node first in
+    // cases of a tie. This simplifies finding an existing node to use as the
+    // merge node.
+    std::stable_sort(CalleeEdges.begin(), CalleeEdges.end(),
+                     CalleeCallerEdgeLessThan);
+
+    /// Find other callers of the given set of callee edges that can
+    /// share the same callee merge node. See the comments at this method
+    /// definition for details.
+    DenseSet<ContextNode *> OtherCallersToShareMerge;
+    findOtherCallersToShareMerge(Node, CalleeEdges, ContextIdToAllocationNode,
+                                 OtherCallersToShareMerge);
+
+    // Now do the actual merging. Identify existing or create a new MergeNode
+    // during the first iteration. Move each callee over, along with edges from
+    // other callers we've determined above can share the same merge node.
+    ContextNode *MergeNode = nullptr;
+    DenseMap<ContextNode *, unsigned> CallerToMoveCount;
+    for (auto CalleeEdge : CalleeEdges) {
+      auto *OrigCallee = CalleeEdge->Callee;
+      // If we don't have a MergeNode yet (only happens on the first iteration,
+      // as a new one will be created when we go to move the first callee edge
+      // over as needed), see if we can use this callee.
+      if (!MergeNode) {
+        // If there are no other callers, simply use this callee.
+        if (CalleeEdge->Callee->CallerEdges.size() == 1) {
+          MergeNode = OrigCallee;
+          NonNewMergedNodes++;
+          continue;
+        }
+        // Otherwise, if we have identified other caller nodes that can share
+        // the merge node with Node, see if all of OrigCallee's callers are
+        // going to share the same merge node. In that case we can use callee
+        // (since all of its callers would move to the new merge node).
+        if (!OtherCallersToShareMerge.empty()) {
+          bool MoveAllCallerEdges = true;
+          for (auto CalleeCallerE : OrigCallee->CallerEdges) {
+            if (CalleeCallerE == CalleeEdge)
+              continue;
+            if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller)) {
+              MoveAllCallerEdges = false;
+              break;
+            }
+          }
+          // If we are going to move all callers over, we can use this callee as
+          // the MergeNode.
+          if (MoveAllCallerEdges) {
+            MergeNode = OrigCallee;
+            NonNewMergedNodes++;
+            continue;
+          }
+        }
+      }
+      // Move this callee edge, creating a new merge node if necessary.
+      if (MergeNode) {
+        assert(MergeNode != OrigCallee);
+        moveEdgeToExistingCalleeClone(CalleeEdge, MergeNode,
+                                      /*NewClone*/ false);
+      } else {
+        MergeNode = moveEdgeToNewCalleeClone(CalleeEdge);
+        NewMergedNodes++;
+      }
+      // Now move all identified edges from other callers over to the merge node
+      // as well.
+      if (!OtherCallersToShareMerge.empty()) {
+        // Make and iterate over a copy of OrigCallee's caller edges because
+        // some of these will be moved off of the OrigCallee and that would mess
+        // up the iteration from OrigCallee.
+        auto OrigCalleeCallerEdges = OrigCallee->CallerEdges;
+        for (auto &CalleeCallerE : OrigCalleeCallerEdges) {
+          if (CalleeCallerE == CalleeEdge)
+            continue;
+          if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller))
+            continue;
+          CallerToMoveCount[CalleeCallerE->Caller]++;
+          moveEdgeToExistingCalleeClone(CalleeCallerE, MergeNode,
+                                        /*NewClone*/ false);
+        }
+      }
+      removeNoneTypeCalleeEdges(OrigCallee);
+      removeNoneTypeCalleeEdges(MergeNode);
+    }
+  }
+}
+
+// Look for other nodes that have edges to the same set of callee
+// clones as the current Node. Those can share the eventual merge node
+// (reducing cloning and binary size overhead) iff:
+// - they have edges to the same set of callee clones
+// - each callee edge reaches a subset of the same allocations as Node's
+//   corresponding edge to the same callee clone.
+// The second requirement is to ensure that we don't undo any of the
+// necessary cloning to distinguish contexts with different allocation
+// behavior.
+// FIXME: This is somewhat conservative, as we really just need to ensure
+// that they don't reach the same allocations as contexts on edges from Node
+// going to any of the *other* callee clones being merged. However, that
+// requires more tracking and checking to get right.
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
+    findOtherCallersToShareMerge(
+        ContextNode *Node,
+        std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
+        DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
+        DenseSet<ContextNode *> &OtherCallersToShareMerge) {
+  auto NumCalleeClones = CalleeEdges.size();
+  // This map counts how many edges to the same callee clone exist for other
+  // caller nodes of each callee clone.
+  DenseMap<ContextNode *, unsigned> OtherCallersToSharedCalleeEdgeCount;
+  // Counts the number of other caller nodes that have edges to all callee
+  // clones that don't violate the allocation context checking.
+  unsigned PossibleOtherCallerNodes = 0;
+
+  // We only need to look at other Caller nodes if the first callee edge has
+  // multiple callers (recall they are sorted in ascending order above).
+  if (CalleeEdges[0]->Callee->CallerEdges.size() < 2)
+    return;
+
+  // For each callee edge:
+  // - Collect the count of other caller nodes calling the same callees.
+  // - Collect the alloc nodes reached by contexts on each callee edge.
+  DenseMap<ContextEdge *, DenseSet<ContextNode *>> CalleeEdgeToAllocNodes;
+  for (auto CalleeEdge : CalleeEdges) {
+    assert(CalleeEdge->Callee->CallerEdges.size() > 1);
+    // For each other caller of the same callee, increment the count of
+    // edges reaching the same callee clone.
+    for (auto CalleeCallerEdges : CalleeEdge->Callee->CallerEdges) {
+      if (CalleeCallerEdges->Caller == Node) {
+        assert(CalleeCallerEdges == CalleeEdge);
+        continue;
+      }
+      OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller]++;
+      // If this caller edge now reaches all of the same callee clones,
+      // increment the count of candidate other caller nodes.
+      if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller] ==
+          NumCalleeClones)
+        PossibleOtherCallerNodes++;
+    }
+    // Collect the alloc nodes reached by contexts on each callee edge, for
+    // later analysis.
+    for (auto Id : CalleeEdge->getContextIds()) {
+      auto *Alloc = ContextIdToAllocationNode.lookup(Id);
+      if (!Alloc) {
+        // FIXME: unclear why this happens occasionally, presumably
+        // imperfect graph updates possibly with recursion.
+        MissingAllocForContextId++;
+        continue;
+      }
+      CalleeEdgeToAllocNodes[CalleeEdge.get()].insert(Alloc);
+    }
+  }
+
+  // Now walk the callee edges again, and make sure that for each candidate
+  // caller node all of its edges to the callees reach the same allocs (or
+  // a subset) as those along the corresponding callee edge from Node.
+  for (auto CalleeEdge : CalleeEdges) {
+    assert(CalleeEdge->Callee->CallerEdges.size() > 1);
+    // Stop if we do not have any (more) candidate other caller nodes.
+    if (!PossibleOtherCallerNodes)
+      break;
+    auto &CurCalleeAllocNodes = CalleeEdgeToAllocNodes[CalleeEdge.get()];
+    // Check each other caller of this callee clone.
+    for (auto &CalleeCallerE : CalleeEdge->Callee->CallerEdges) {
+      // Not interested in the callee edge from Node itself.
+      if (CalleeCallerE == CalleeEdge)
+        continue;
+      // Skip any callers that didn't have callee edges to all the same
+      // callee clones.
+      if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] !=
+          NumCalleeClones)
+        continue;
+      // Make sure that each context along edge from candidate caller node
+      // reaches an allocation also reached by this callee edge from Node.
+      for (auto Id : CalleeCallerE->getContextIds()) {
+        auto *Alloc = ContextIdToAllocationNode.lookup(Id);
+        if (!Alloc)
+          continue;
+        // If not, simply reset the map entry to 0 so caller is ignored, and
+        // reduce the count of candidate other caller nodes.
+        if (!CurCalleeAllocNodes.contains(Alloc)) {
+          OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] = 0;
+          PossibleOtherCallerNodes--;
+          break;
+        }
+      }
+    }
+  }
+
+  if (!PossibleOtherCallerNodes)
+    return;
+
+  // Build the set of other caller nodes that can use the same callee merge
+  // node.
+  for (auto &[OtherCaller, Count] : OtherCallersToSharedCalleeEdgeCount) {
+    if (Count != NumCalleeClones)
+      continue;
+    OtherCallersToShareMerge.insert(OtherCaller);
+  }
+}
+
 // This method assigns cloned callsites to functions, cloning the functions as
 // needed. The assignment is greedy and proceeds roughly as follows:
 //
@@ -4053,6 +4415,8 @@ template <typename DerivedCCG, typename FuncTy, typename CallTy>
 bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
   bool Changed = false;
 
+  mergeClones();
+
   // Keep track of the assignment of nodes (callsites) to function clones they
   // call.
   DenseMap<ContextNode *, FuncInfo> CallsiteToCalleeFuncCloneMap;
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/mergenodes.ll b/llvm/test/Transforms/MemProfContextDisambiguation/mergenodes.ll
new file mode 100644
index 0000000000000..990a4a4e4d064
--- /dev/null
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/mergenodes.ll
@@ -0,0 +1,404 @@
+;; Test that correct clones are generated and reached when we need to
+;; re-merge clone nodes before function assignment.
+;;
+;; The code is similar to that of basic.ll, but with a second allocation.
+
+; RUN: opt -passes=memprof-context-disambiguation -supports-hot-cold-new \
+; RUN:	-memprof-verify-ccg -memprof-dump-ccg %s -S 2>&1 | FileCheck %s \
+; RUN:  --check-prefix=IR --check-prefix=DUMP
+
+;; Make sure the option to disable merging causes the expected regression.
+; RUN: opt -passes=memprof-context-disambiguation -supports-hot-cold-new \
+; RUN:  -memprof-merge-clones=false %s -S 2>&1 | FileCheck %s --check-prefix=NOMERGE
+;; main should incorrectly call the same clone of foo.
+; NOMERGE: define {{.*}} @main
+; NOMERGE-NEXT: entry:
+; NOMERGE-NEXT: call {{.*}} @_Z3foov.memprof.1()
+; NOMERGE-NEXT: call {{.*}} @_Z3foov.memprof.1()
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main() {
+entry:
+  ;; Ultimately calls bar and allocates notcold memory from first call to new
+  ;; and cold memory from second call to new.
+  %call = call noundef ptr @_Z3foov(), !callsite !0
+  ;; Ultimately calls bar and allocates cold memory from first call to new
+  ;; and notcold memory from second call to new.
+  %call1 = call noundef ptr @_Z3foov(), !callsite !1
+  ret i32 0
+}
+
+define internal ptr @_Z3barv() {
+entry:
+  ;; notcold when called from first call to foo from main, cold when called from second.
+  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0, !memprof !2, !callsite !7
+  ;; cold when called from first call to foo from main, notcold when called from second.
+  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0, !memprof !13, !callsite !18
+  ret ptr null
+}
+
+declare ptr @_Znam(i64)
+
+define internal ptr @_Z3bazv() {
+entry:
+  %call = call noundef ptr @_Z3barv(), !callsite !8
+  ret ptr null
+}
+
+; Function Attrs: noinline
+define internal ptr @_Z3foov() {
+entry:
+  %call = call noundef ptr @_Z3bazv(), !callsite !9
+  ret ptr null
+}
+
+attributes #0 = { builtin }
+
+!0 = !{i64 8632435727821051414}
+!1 = !{i64 -3421689549917153178}
+!2 = !{!3, !5}
+!3 = !{!4, !"notcold"}
+!4 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 8632435727821051414}
+!5 = !{!6, !"cold"}
+!6 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 -3421689549917153178}
+!7 = !{i64 9086428284934609951}
+!8 = !{i64 -5964873800580613432}
+!9 = !{i64 2732490490862098848}
+!13 = !{!14, !16}
+!14 = !{!15, !"cold"}
+!15 = !{i64 123, i64 -5964873800580613432, i64 2732490490862098848, i64 8632435727821051414}
+!16 = !{!17, !"notcold"}
+!17 = !{i64 123, i64 -5964873800580613432, i64 2732490490862098848, i64 -3421689549917153178}
+!18 = !{i64 123}
+
+;; After cloning, each callsite in main calls different clones of foo with
+;; different allocaton types, and ditto all the way through the leaf
+;; allocation callsites. The single allocation-type clones are shared between
+;; the two callsites in main. This would lead to incorrect assignment of
+;; the leaf allocations to function clones as is, since we have lost the
+;; information that each callsite in main ultimately reaches two allocation
+;; callsites with *different* allocation types.
+; DUMP: CCG after cloning:
+; DUMP: Callsite Context Graph:
+; DUMP: Node [[BAR1ALLOC1:0x[a-f0-9]+]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ1:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 	Clones: [[BAR2ALLOC1:0x[a-f0-9]+]]
+
+; DUMP: Node [[BAZ1]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ1]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 		Edge from Callee [[BAR1ALLOC2:0x[a-f0-9]+]] to Caller: [[BAZ1]] AllocTypes: NotCold ContextIds: 4
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ1]] to Caller: [[FOO1:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 1 4
+; DUMP: 	Clones: [[BAZ2:0x[a-f0-9]+]]
+
+; DUMP: Node [[FOO1]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ1]] to Caller: [[FOO1]] AllocTypes: NotCold ContextIds: 1 4
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN1:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN2:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 4
+; DUMP: 	Clones: [[FOO2:0x[a-f0-9]+]]
+
+; DUMP: Node [[MAIN1]]
+; DUMP: 	  %call = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 		Edge from Callee [[FOO2:0x[a-f0-9]+]] to Caller: [[MAIN1]] AllocTypes: Cold ContextIds: 3
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN2]]
+; DUMP: 	  %call1 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN2]] AllocTypes: NotCold ContextIds: 4
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[BAR1ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 4
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ1]] AllocTypes: NotCold ContextIds: 4
+; DUMP: 	Clones: [[BAR2ALLOC2:0x[a-f0-9]+]]
+
+; DUMP: Node [[FOO2]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO2]] AllocTypes: Cold ContextIds: 2 3
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN1]] AllocTypes: Cold ContextIds: 3
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ2]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 2
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 3
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO2]] AllocTypes: Cold ContextIds: 2 3
+; DUMP: 	Clone of [[BAZ1]]
+
+; DUMP: Node [[BAR2ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 2
+; DUMP: 	Clone of [[BAR1ALLOC1]]
+
+; DUMP: Node [[BAR2ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 3
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 3
+; DUMP: 	Clone of [[BAR1ALLOC2]]
+
+;; After merging, each callsite in main calls a different single clone of foo
+;; with both cold and not cold allocation types, but ultimately reaches two
+;; single allocation type allocation callsite clones of the correct
+;; combination. The graph after assigning function clones is the same, but
+;; with function calls updated to the new function clones.
+; DUMP: CCG after merging:
+; DUMP: Callsite Context Graph:
+; DUMP: Node [[BAR1ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ2]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 	Clones: [[BAR2ALLOC1]]
+
+; DUMP: Node [[MAIN1]]
+; DUMP: 	  %call = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO3:0x[a-f0-9]+]] to Caller: [[MAIN1]] AllocTypes: NotColdCold ContextIds: 1 3
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN2]]
+; DUMP: 	  %call1 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[BAR1ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 4
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ3:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 4
+; DUMP: 	Clones: [[BAR2ALLOC2]]
+
+; DUMP: Node [[FOO2]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ3]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ2]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 3
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ2]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO3]] AllocTypes: NotColdCold ContextIds: 1 3
+; DUMP: 	Clone of [[BAZ1]]
+
+; DUMP: Node [[BAR2ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ3]] AllocTypes: Cold ContextIds: 2
+; DUMP: 	Clone of [[BAR1ALLOC1]]
+
+; DUMP: Node [[BAR2ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 3
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 3
+; DUMP: 	Clone of [[BAR1ALLOC2]]
+
+; DUMP: Node [[FOO3]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO3]] AllocTypes: NotColdCold ContextIds: 1 3
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN1]] AllocTypes: NotColdCold ContextIds: 1 3
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ3]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ3]] AllocTypes: Cold ContextIds: 2
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ3]] AllocTypes: NotCold ContextIds: 4
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ3]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	Clone of [[BAZ1]]
+
+; DUMP: CCG after assigning function clones:
+; DUMP: Callsite Context Graph:
+; DUMP: Node [[BAR1ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ2]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 	Clones: [[BAR2ALLOC1]]
+
+; DUMP: Node [[MAIN1]]
+; DUMP: 	  %call = call noundef ptr @_Z3foov.memprof.1()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN1]] AllocTypes: NotColdCold ContextIds: 1 3
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN2]]
+; DUMP: 	  %call1 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[BAR1ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #1	(clone 1)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 4
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ3]] AllocTypes: NotCold ContextIds: 4
+; DUMP: 	Clones: [[BAR2ALLOC2]]
+
+; DUMP: Node [[FOO2]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv.memprof.1()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ3]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ2]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 3
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ2]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO3]] AllocTypes: NotColdCold ContextIds: 1 3
+; DUMP: 	Clone of [[BAZ1]]
+
+; DUMP: Node [[BAR2ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 1)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ3]] AllocTypes: Cold ContextIds: 2
+; DUMP: 	Clone of [[BAR1ALLOC1]]
+
+; DUMP: Node [[BAR2ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #1	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 3
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 3
+; DUMP: 	Clone of [[BAR1ALLOC2]]
+
+; DUMP: Node [[FOO3]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 1)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO3]] AllocTypes: NotColdCold ContextIds: 1 3
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN1]] AllocTypes: NotColdCold ContextIds: 1 3
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ3]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv.memprof.1()	(clone 1)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ3]] AllocTypes: Cold ContextIds: 2
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ3]] AllocTypes: NotCold ContextIds: 4
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ3]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	Clone of [[BAZ1]]
+
+; IR: define {{.*}} @main
+;; The first call to foo should now call foo.memprof.1 that ultimately
+;; calls bar with the first allocation hinted notcold and the second cold
+;; (via call chain foo.memprof.1 -> baz -> bar).
+; IR:   call {{.*}} @_Z3foov.memprof.1()
+;; The second call to foo still calls the original foo, but ultimately
+;; reaches a clone of bar with the first allocation hinted cold and the
+;; second notcold.
+; IR:   call {{.*}} @_Z3foov()
+; IR: define internal {{.*}} @_Z3barv()
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[NOTCOLD:[0-9]+]]
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[COLD:[0-9]+]]
+; IR: define internal {{.*}} @_Z3bazv()
+; IR:   call {{.*}} @_Z3barv()
+; IR: define internal {{.*}} @_Z3foov()
+; IR:   call {{.*}} @_Z3bazv.memprof.1()
+; IR: define internal {{.*}} @_Z3barv.memprof.1()
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[COLD]]
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[NOTCOLD]]
+; IR: define internal {{.*}} @_Z3bazv.memprof.1()
+; IR:   call {{.*}} @_Z3barv.memprof.1()
+; IR: define internal {{.*}} @_Z3foov.memprof.1()
+; IR:   call {{.*}} @_Z3bazv()
+; IR: attributes #[[NOTCOLD]] = { builtin "memprof"="notcold" }
+; IR: attributes #[[COLD]] = { builtin "memprof"="cold" }
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/mergenodes2.ll b/llvm/test/Transforms/MemProfContextDisambiguation/mergenodes2.ll
new file mode 100644
index 0000000000000..80f6bc7d8ddaf
--- /dev/null
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/mergenodes2.ll
@@ -0,0 +1,474 @@
+;; Test that correct clones are generated and reached when we need to
+;; re-merge clone nodes before function assignment.
+;;
+;; The code is similar to that of mergenodes.ll, but with two additional
+;; contexts sharing the same two allocations. This tests that we correctly
+;; share the merged nodes when possible.
+
+; RUN: opt -passes=memprof-context-disambiguation -supports-hot-cold-new \
+; RUN:	-memprof-verify-ccg -memprof-dump-ccg %s -S 2>&1 | FileCheck %s \
+; RUN:  --check-prefix=IR --check-prefix=DUMP
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main() {
+entry:
+  ;; Ultimately calls bar and allocates notcold memory from first call to new
+  ;; and cold memory from second call to new.
+  %call = call noundef ptr @_Z3foov(), !callsite !0
+  ;; Ultimately calls bar and allocates cold memory from first call to new
+  ;; and notcold memory from second call to new.
+  %call1 = call noundef ptr @_Z3foov(), !callsite !1
+  ;; Same allocation pattern as the first call to foo above, should end up
+  ;; sharing the same merge nodes.
+  %call2 = call noundef ptr @_Z3foov(), !callsite !19
+  ;; Same allocation pattern as the second call to foo above, should end up
+  ;; sharing the same merge nodes.
+  %call3 = call noundef ptr @_Z3foov(), !callsite !20
+  ret i32 0
+}
+
+define internal ptr @_Z3barv() {
+entry:
+  ;; notcold when called from first and third calls to foo from main, cold when
+  ;; called from second and fourth.
+  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0, !memprof !2, !callsite !7
+  ;; cold when called from first and third calls to foo from main, notcold when
+  ;; called from second and fourth.
+  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0, !memprof !13, !callsite !18
+  ret ptr null
+}
+
+declare ptr @_Znam(i64)
+
+define internal ptr @_Z3bazv() {
+entry:
+  %call = call noundef ptr @_Z3barv(), !callsite !8
+  ret ptr null
+}
+
+; Function Attrs: noinline
+define internal ptr @_Z3foov() {
+entry:
+  %call = call noundef ptr @_Z3bazv(), !callsite !9
+  ret ptr null
+}
+
+attributes #0 = { builtin }
+
+!0 = !{i64 8632435727821051414}
+!1 = !{i64 -3421689549917153178}
+!19 = !{i64 8910}
+!20 = !{i64 91011}
+!2 = !{!3, !5, !21, !23}
+!3 = !{!4, !"notcold"}
+!4 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 8632435727821051414}
+!5 = !{!6, !"cold"}
+!6 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 -3421689549917153178}
+!21 = !{!22, !"notcold"}
+!22 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 8910}
+!23 = !{!24, !"cold"}
+!24 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 91011}
+!7 = !{i64 9086428284934609951}
+!8 = !{i64 -5964873800580613432}
+!9 = !{i64 2732490490862098848}
+!13 = !{!14, !16, !25, !27}
+!14 = !{!15, !"cold"}
+!15 = !{i64 123, i64 -5964873800580613432, i64 2732490490862098848, i64 8632435727821051414}
+!16 = !{!17, !"notcold"}
+!17 = !{i64 123, i64 -5964873800580613432, i64 2732490490862098848, i64 -3421689549917153178}
+!25 = !{!26, !"cold"}
+!26 = !{i64 123, i64 -5964873800580613432, i64 2732490490862098848, i64 8910}
+!27 = !{!28, !"notcold"}
+!28 = !{i64 123, i64 -5964873800580613432, i64 2732490490862098848, i64 91011}
+!18 = !{i64 123}
+
+;; After cloning, each callsite in main calls different clones of foo with
+;; different allocaton types, and ditto all the way through the leaf
+;; allocation callsites. The single allocation-type clones are shared between
+;; the four callsites in main. This would lead to incorrect assignment of
+;; the leaf allocations to function clones as is, since we have lost the
+;; information that each callsite in main ultimately reaches two allocation
+;; callsites with *different* allocation types.
+; DUMP: CCG after cloning:
+; DUMP: Callsite Context Graph:
+; DUMP: Node [[BAR1ALLOC1:0x[a-f0-9]+]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ1:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 1 3
+; DUMP: 	Clones: [[BAR2ALLOC1:0x[a-f0-9]+]]
+
+; DUMP: Node [[BAZ1]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1 3 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ1]] AllocTypes: NotCold ContextIds: 1 3
+; DUMP: 		Edge from Callee [[BAR1ALLOC2:0x[a-f0-9]+]] to Caller: [[BAZ1]] AllocTypes: NotCold ContextIds: 6 8
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ1]] to Caller: [[FOO1:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 1 3 6 8
+; DUMP: 	Clones: [[BAZ2:0x[a-f0-9]+]]
+
+; DUMP: Node [[FOO1]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1 3 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ1]] to Caller: [[FOO1]] AllocTypes: NotCold ContextIds: 1 3 6 8
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN1:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN3:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 3
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN2:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 6
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN4:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 8
+; DUMP: 	Clones: [[FOO2:0x[a-f0-9]+]]
+
+; DUMP: Node [[MAIN1]]
+; DUMP: 	  %call = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 5
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 1
+; DUMP: 		Edge from Callee [[FOO2:0x[a-f0-9]+]] to Caller: [[MAIN1]] AllocTypes: Cold ContextIds: 5
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN2]]
+; DUMP: 	  %call1 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 6
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN2]] AllocTypes: NotCold ContextIds: 6
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN3]]
+; DUMP: 	  %call2 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 3 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN3]] AllocTypes: NotCold ContextIds: 3
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN3]] AllocTypes: Cold ContextIds: 7
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN4]]
+; DUMP: 	  %call3 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 4 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO1]] to Caller: [[MAIN4]] AllocTypes: NotCold ContextIds: 8
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN4]] AllocTypes: Cold ContextIds: 4
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[BAR1ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ1]] AllocTypes: NotCold ContextIds: 6 8
+; DUMP: 	Clones: [[BAR2ALLOC2:0x[a-f0-9]+]]
+
+; DUMP: Node [[FOO2]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2 4 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO2]] AllocTypes: Cold ContextIds: 2 4 5 7
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN4]] AllocTypes: Cold ContextIds: 4
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN1]] AllocTypes: Cold ContextIds: 5
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN3]] AllocTypes: Cold ContextIds: 7
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ2]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2 4 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 2 4
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 5 7
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO2]] AllocTypes: Cold ContextIds: 2 4 5 7
+; DUMP: 	Clone of [[BAZ1]]
+
+; DUMP: Node [[BAR2ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 2 4
+; DUMP: 	Clone of [[BAR1ALLOC1]]
+
+; DUMP: Node [[BAR2ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 5 7
+; DUMP: 	Clone of [[BAR1ALLOC2]]
+
+;; After merging, each pair of callsites in main with the same allocation
+;; pattern call a different single clone of foo with both cold and not cold
+;; allocation types, but ultimately reach two single allocation type
+;; allocation callsite clones of the correct combination. The graph after
+;; assigning function clones is the same, but with function calls updated to
+;; the new function clones.
+; DUMP: CCG after merging:
+; DUMP: Callsite Context Graph:
+; DUMP: Node [[BAR1ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ2]] AllocTypes: NotCold ContextIds: 1 3
+; DUMP: 	Clones: [[BAR2ALLOC1]]
+
+; DUMP: Node [[MAIN1]]
+; DUMP: 	  %call = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 5
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO3:0x[a-f0-9]+]] to Caller: [[MAIN1]] AllocTypes: NotColdCold ContextIds: 1 5
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN2]]
+; DUMP: 	  %call1 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 6
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: NotColdCold ContextIds: 2 6
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN3]]
+; DUMP: 	  %call2 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 3 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN3]] AllocTypes: NotColdCold ContextIds: 3 7
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN4]]
+; DUMP: 	  %call3 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 4 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN4]] AllocTypes: NotColdCold ContextIds: 4 8
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[BAR1ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ3:0x[a-f0-9]+]] AllocTypes: NotCold ContextIds: 6 8
+; DUMP: 	Clones: [[BAR2ALLOC2]]
+
+; DUMP: Node [[FOO2]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ3]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: NotColdCold ContextIds: 2 6
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN4]] AllocTypes: NotColdCold ContextIds: 4 8
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ2]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 5 7
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ2]] AllocTypes: NotCold ContextIds: 1 3
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO3]] AllocTypes: NotColdCold ContextIds: 1 3 5 7
+; DUMP: 	Clone of [[BAZ1]]
+
+; DUMP: Node [[BAR2ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ3]] AllocTypes: Cold ContextIds: 2 4
+; DUMP: 	Clone of [[BAR1ALLOC1]]
+
+; DUMP: Node [[BAR2ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 5 7
+; DUMP: 	Clone of [[BAR1ALLOC2]]
+
+; DUMP: Node [[FOO3]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO3]] AllocTypes: NotColdCold ContextIds: 1 3 5 7
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN1]] AllocTypes: NotColdCold ContextIds: 1 5
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN3]] AllocTypes: NotColdCold ContextIds: 3 7
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ3]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ3]] AllocTypes: Cold ContextIds: 2 4
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ3]] AllocTypes: NotCold ContextIds: 6 8
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ3]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 2 4 6 8
+; DUMP: 	Clone of [[BAZ1]]
+
+; DUMP: CCG after assigning function clones:
+; DUMP: Callsite Context Graph:
+; DUMP: Node [[BAR1ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 0)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 1 3
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ2]] AllocTypes: NotCold ContextIds: 1 3
+; DUMP: 	Clones: [[BAR2ALLOC1]]
+
+; DUMP: Node [[MAIN1]]
+; DUMP: 	  %call = call noundef ptr @_Z3foov.memprof.1()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 5
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN1]] AllocTypes: NotColdCold ContextIds: 1 5
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN2]]
+; DUMP: 	  %call1 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 6
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: NotColdCold ContextIds: 2 6
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN3]]
+; DUMP: 	  %call2 = call noundef ptr @_Z3foov.memprof.1()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 3 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN3]] AllocTypes: NotColdCold ContextIds: 3 7
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[MAIN4]]
+; DUMP: 	  %call3 = call noundef ptr @_Z3foov()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 4 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN4]] AllocTypes: NotColdCold ContextIds: 4 8
+; DUMP: 	CallerEdges:
+
+; DUMP: Node [[BAR1ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #1	(clone 1)
+; DUMP: 	AllocTypes: NotCold
+; DUMP: 	ContextIds: 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ3]] AllocTypes: NotCold ContextIds: 6 8
+; DUMP: 	Clones: [[BAR2ALLOC2]]
+
+; DUMP: Node [[FOO2]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv.memprof.1()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ3]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 2 4
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: NotColdCold ContextIds: 2 6
+; DUMP: 		Edge from Callee [[FOO2]] to Caller: [[MAIN4]] AllocTypes: NotColdCold ContextIds: 4 8
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ2]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv()	(clone 0)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 5 7
+; DUMP: 		Edge from Callee [[BAR1ALLOC1]] to Caller: [[BAZ2]] AllocTypes: NotCold ContextIds: 1 3
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO3]] AllocTypes: NotColdCold ContextIds: 1 3 5 7
+; DUMP: 	Clone of [[BAZ1]]
+
+; DUMP: Node [[BAR2ALLOC1]]
+; DUMP: 	  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #0	(clone 1)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 2 4
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ3]] AllocTypes: Cold ContextIds: 2 4
+; DUMP: 	Clone of [[BAR1ALLOC1]]
+
+; DUMP: Node [[BAR2ALLOC2]]
+; DUMP: 	  %call2 = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #1	(clone 0)
+; DUMP: 	AllocTypes: Cold
+; DUMP: 	ContextIds: 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 5 7
+; DUMP: 	Clone of [[BAR1ALLOC2]]
+
+; DUMP: Node [[FOO3]]
+; DUMP: 	  %call = call noundef ptr @_Z3bazv()	(clone 1)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 1 3 5 7
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAZ2]] to Caller: [[FOO3]] AllocTypes: NotColdCold ContextIds: 1 3 5 7
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN1]] AllocTypes: NotColdCold ContextIds: 1 5
+; DUMP: 		Edge from Callee [[FOO3]] to Caller: [[MAIN3]] AllocTypes: NotColdCold ContextIds: 3 7
+; DUMP: 	Clone of [[FOO1]]
+
+; DUMP: Node [[BAZ3]]
+; DUMP: 	  %call = call noundef ptr @_Z3barv.memprof.1()	(clone 1)
+; DUMP: 	AllocTypes: NotColdCold
+; DUMP: 	ContextIds: 2 4 6 8
+; DUMP: 	CalleeEdges:
+; DUMP: 		Edge from Callee [[BAR2ALLOC1]] to Caller: [[BAZ3]] AllocTypes: Cold ContextIds: 2 4
+; DUMP: 		Edge from Callee [[BAR1ALLOC2]] to Caller: [[BAZ3]] AllocTypes: NotCold ContextIds: 6 8
+; DUMP: 	CallerEdges:
+; DUMP: 		Edge from Callee [[BAZ3]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 2 4 6 8
+; DUMP: 	Clone of [[BAZ1]]
+
+; IR: define {{.*}} @main
+;; The first call to foo should now call foo.memprof.1 that ultimately
+;; calls bar with the first allocation hinted notcold and the second cold
+;; (via call chain foo.memprof.1 -> baz -> bar).
+; IR:   call {{.*}} @_Z3foov.memprof.1()
+;; The second call to foo still calls the original foo, but ultimately
+;; reaches a clone of bar with the first allocation hinted cold and the
+;; second notcold.
+; IR:   call {{.*}} @_Z3foov()
+; IR: define internal {{.*}} @_Z3barv()
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[NOTCOLD:[0-9]+]]
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[COLD:[0-9]+]]
+; IR: define internal {{.*}} @_Z3bazv()
+; IR:   call {{.*}} @_Z3barv()
+; IR: define internal {{.*}} @_Z3foov()
+; IR:   call {{.*}} @_Z3bazv.memprof.1()
+; IR: define internal {{.*}} @_Z3barv.memprof.1()
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[COLD]]
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[NOTCOLD]]
+; IR: define internal {{.*}} @_Z3bazv.memprof.1()
+; IR:   call {{.*}} @_Z3barv.memprof.1()
+; IR: define internal {{.*}} @_Z3foov.memprof.1()
+; IR:   call {{.*}} @_Z3bazv()
+; IR: attributes #[[NOTCOLD]] = { builtin "memprof"="notcold" }
+; IR: attributes #[[COLD]] = { builtin "memprof"="cold" }
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/overlapping-contexts.ll b/llvm/test/Transforms/MemProfContextDisambiguation/overlapping-contexts.ll
index 7fe9dc96921c6..af93d56c5f33a 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/overlapping-contexts.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/overlapping-contexts.ll
@@ -67,7 +67,9 @@
 
 ; REMARKS: created clone _Z1Ab.memprof.1
 ; REMARKS: created clone _Z1Xb.memprof.1
+; REMARKS: created clone _Z1Xb.memprof.2
 ; REMARKS: created clone _Z1Bb.memprof.1
+; REMARKS: created clone _Z1Bb.memprof.2
 ; REMARKS: created clone _Z1Cb.memprof.1
 ; REMARKS: created clone _Z1Eb.memprof.1
 ; REMARKS: call in clone _Z1Gv assigned to call function clone _Z1Eb.memprof.1
@@ -78,17 +80,18 @@
 ; REMARKS: call in clone _Z1Cb.memprof.1 assigned to call function clone _Z1Bb.memprof.1
 ; REMARKS: call in clone _Z1Fv assigned to call function clone _Z1Eb
 ; REMARKS: call in clone _Z1Eb assigned to call function clone _Z1Cb
-; REMARKS: call in clone _Z1Cb assigned to call function clone _Z1Bb.memprof.1
-; REMARKS: call in clone _Z1Bb.memprof.1 assigned to call function clone _Z1Xb.memprof.1
-; REMARKS: call in clone _Z1Xb.memprof.1 assigned to call function clone _Z1Ab.memprof.1
+; REMARKS: call in clone _Z1Cb assigned to call function clone _Z1Bb.memprof.2
+; REMARKS: call in clone _Z1Bb.memprof.2 assigned to call function clone _Z1Xb.memprof.2
+; REMARKS: call in clone _Z1Xb.memprof.2 assigned to call function clone _Z1Ab.memprof.1
 ; REMARKS: call in clone _Z1Ab.memprof.1 marked with memprof allocation attribute cold
-; REMARKS: call in clone _Z1Bb.memprof.1 assigned to call function clone _Z1Xb
+; REMARKS: call in clone _Z1Bb.memprof.1 assigned to call function clone _Z1Xb.memprof.1
+; REMARKS: call in clone _Z1Xb.memprof.1 assigned to call function clone _Z1Ab
 ; REMARKS: call in clone _Z1Dv assigned to call function clone _Z1Bb
 ; REMARKS: call in clone _Z1Bb assigned to call function clone _Z1Xb
 ; REMARKS: call in clone _Z1Xb assigned to call function clone _Z1Ab
 ; REMARKS: call in clone _Z1Ab marked with memprof allocation attribute notcold
-; REMARKS: call in clone _Z1Ab.memprof.1 marked with memprof allocation attribute cold
-; REMARKS: call in clone _Z1Ab marked with memprof allocation attribute notcold
+; REMARKS: call in clone _Z1Ab marked with memprof allocation attribute cold
+; REMARKS: call in clone _Z1Ab.memprof.1 marked with memprof allocation attribute notcold
 
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -204,7 +207,7 @@ attributes #7 = { builtin }
 
 ; IR: define {{.*}} @_Z1Cb(i1 noundef zeroext %b)
 ; IR-NEXT: entry:
-; IR-NEXT:   call {{.*}} @_Z1Bb.memprof.1(i1 noundef zeroext %b)
+; IR-NEXT:   call {{.*}} @_Z1Bb.memprof.2(i1 noundef zeroext %b)
 
 ; IR: define {{.*}} @_Z1Ab.memprof.1(i1 noundef zeroext %b)
 ; IR-NEXT: entry:
@@ -215,18 +218,20 @@ attributes #7 = { builtin }
 ; IR-NEXT:   br label %if.end
 ; IR-EMPTY:
 ; IR-NEXT: if.else:
-; IR-NEXT:   call {{.*}} @_Znam(i64 noundef 10) #[[COLD]]
+; IR-NEXT:   call {{.*}} @_Znam(i64 noundef 10) #[[NOTCOLD:[0-9]+]]
 
-; IR: define {{.*}} @_Z1Xb.memprof.1(i1 noundef zeroext %b)
+; IR: define {{.*}} @_Z1Xb.memprof.2(i1 noundef zeroext %b)
 ; IR-NEXT: entry:
 ; IR-NEXT:   call {{.*}} @_Z1Ab.memprof.1(i1 noundef zeroext %b)
 
-; IR: define {{.*}} @_Z1Bb.memprof.1(i1 noundef zeroext %b)
+; IR: define {{.*}} @_Z1Bb.memprof.2(i1 noundef zeroext %b)
 ; IR-NEXT: entry:
-; IR-NEXT:   call {{.*}} @_Z1Xb.memprof.1(i1 noundef zeroext %b)
+; IR-NEXT:   call {{.*}} @_Z1Xb.memprof.2(i1 noundef zeroext %b)
 
+; IR: attributes #[[NOTCOLD]] = { builtin "memprof"="notcold" }
 ; IR: attributes #[[COLD]] = { builtin "memprof"="cold" }
 
 ; STATS: 2 memprof-context-disambiguation - Number of cold static allocations (possibly cloned)
 ; STATS: 2 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned)
-; STATS: 5 memprof-context-disambiguation - Number of function clones created during whole program analysis
+; STATS: 7 memprof-context-disambiguation - Number of function clones created during whole program analysis
+; STATS: 2 memprof-context-disambiguation - Number of new nodes created during merging

From 08b4c52540727455194b0cf0f6310f391e87c2a5 Mon Sep 17 00:00:00 2001
From: Shubham Sandeep Rastogi <srastogi22@apple.com>
Date: Mon, 21 Apr 2025 17:19:54 -0700
Subject: [PATCH 674/710] Revert "[lldb] Avoid force loading symbols in
 statistics collection (#136236)"

This reverts commit d5b40c71f6be972f677de5d9886f91866df007b5.

This change broke greendragon lldb test:

lldb-api :: commands/statistics/basic/TestStats.py

And is therefore being reverted.
---
 lldb/include/lldb/Symbol/ObjectFile.h         |  2 +-
 lldb/include/lldb/Symbol/SymbolFile.h         |  4 +-
 lldb/include/lldb/Symbol/SymbolFileOnDemand.h |  4 +-
 lldb/source/Core/Module.cpp                   |  2 +-
 lldb/source/Symbol/ObjectFile.cpp             |  5 +-
 lldb/source/Symbol/SymbolFile.cpp             |  4 +-
 .../commands/statistics/basic/TestStats.py    | 23 ------
 lldb/unittests/Symbol/LineTableTest.cpp       |  2 +-
 lldb/unittests/Symbol/SymtabTest.cpp          | 78 +------------------
 9 files changed, 13 insertions(+), 111 deletions(-)

diff --git a/lldb/include/lldb/Symbol/ObjectFile.h b/lldb/include/lldb/Symbol/ObjectFile.h
index 7fca6383fa9f3..cfcca04a76de8 100644
--- a/lldb/include/lldb/Symbol/ObjectFile.h
+++ b/lldb/include/lldb/Symbol/ObjectFile.h
@@ -319,7 +319,7 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
   ///
   /// \return
   ///     The symbol table for this object file.
-  Symtab *GetSymtab(bool can_create = true);
+  Symtab *GetSymtab();
 
   /// Parse the symbol table into the provides symbol table object.
   ///
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
index df2f263b18e17..f35d3ee9f22ae 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -144,7 +144,7 @@ class SymbolFile : public PluginInterface {
   virtual uint32_t GetNumCompileUnits() = 0;
   virtual lldb::CompUnitSP GetCompileUnitAtIndex(uint32_t idx) = 0;
 
-  virtual Symtab *GetSymtab(bool can_create = true) = 0;
+  virtual Symtab *GetSymtab() = 0;
 
   virtual lldb::LanguageType ParseLanguage(CompileUnit &comp_unit) = 0;
   /// Return the Xcode SDK comp_unit was compiled against.
@@ -533,7 +533,7 @@ class SymbolFileCommon : public SymbolFile {
     return m_abilities;
   }
 
-  Symtab *GetSymtab(bool can_create = true) override;
+  Symtab *GetSymtab() override;
 
   ObjectFile *GetObjectFile() override { return m_objfile_sp.get(); }
   const ObjectFile *GetObjectFile() const override {
diff --git a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
index 6ed389a48880c..7a366bfabec86 100644
--- a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
+++ b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
@@ -186,9 +186,7 @@ class SymbolFileOnDemand : public lldb_private::SymbolFile {
 
   uint32_t GetAbilities() override;
 
-  Symtab *GetSymtab(bool can_create = true) override {
-    return m_sym_file_impl->GetSymtab(can_create);
-  }
+  Symtab *GetSymtab() override { return m_sym_file_impl->GetSymtab(); }
 
   ObjectFile *GetObjectFile() override {
     return m_sym_file_impl->GetObjectFile();
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index 625c14e4a2153..ad7046e596278 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -997,7 +997,7 @@ SymbolFile *Module::GetSymbolFile(bool can_create, Stream *feedback_strm) {
 
 Symtab *Module::GetSymtab(bool can_create) {
   if (SymbolFile *symbols = GetSymbolFile(can_create))
-    return symbols->GetSymtab(can_create);
+    return symbols->GetSymtab();
   return nullptr;
 }
 
diff --git a/lldb/source/Symbol/ObjectFile.cpp b/lldb/source/Symbol/ObjectFile.cpp
index afd0d298e675e..2f2c59d6af620 100644
--- a/lldb/source/Symbol/ObjectFile.cpp
+++ b/lldb/source/Symbol/ObjectFile.cpp
@@ -734,9 +734,10 @@ void llvm::format_provider<ObjectFile::Strata>::format(
   }
 }
 
-Symtab *ObjectFile::GetSymtab(bool can_create) {
+
+Symtab *ObjectFile::GetSymtab() {
   ModuleSP module_sp(GetModule());
-  if (module_sp && can_create) {
+  if (module_sp) {
     // We can't take the module lock in ObjectFile::GetSymtab() or we can
     // deadlock in DWARF indexing when any file asks for the symbol table from
     // an object file. This currently happens in the preloading of symbols in
diff --git a/lldb/source/Symbol/SymbolFile.cpp b/lldb/source/Symbol/SymbolFile.cpp
index 870d778dca740..94e32b55572dd 100644
--- a/lldb/source/Symbol/SymbolFile.cpp
+++ b/lldb/source/Symbol/SymbolFile.cpp
@@ -152,10 +152,10 @@ void SymbolFile::AssertModuleLock() {
 
 SymbolFile::RegisterInfoResolver::~RegisterInfoResolver() = default;
 
-Symtab *SymbolFileCommon::GetSymtab(bool can_create) {
+Symtab *SymbolFileCommon::GetSymtab() {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
   // Fetch the symtab from the main object file.
-  auto *symtab = GetMainObjectFile()->GetSymtab(can_create);
+  auto *symtab = GetMainObjectFile()->GetSymtab();
   if (m_symtab != symtab) {
     m_symtab = symtab;
 
diff --git a/lldb/test/API/commands/statistics/basic/TestStats.py b/lldb/test/API/commands/statistics/basic/TestStats.py
index e2c539bfaeddb..0265a0d7c9948 100644
--- a/lldb/test/API/commands/statistics/basic/TestStats.py
+++ b/lldb/test/API/commands/statistics/basic/TestStats.py
@@ -209,29 +209,6 @@ def test_default_no_run(self):
             )
             self.assertGreater(module_stats["symbolsLoaded"], 0)
 
-    def test_default_no_run_no_preload_symbols(self):
-        """Test "statistics dump" without running the target and without
-        preloading symbols.
-
-        Checks that symbol count are zero.
-        """
-        # Make sure symbols will not be preloaded.
-        self.runCmd("settings set target.preload-symbols false")
-
-        # Build and load the target
-        self.build()
-        target = self.createTestTarget()
-
-        # Get statistics
-        debug_stats = self.get_stats()
-
-        # No symbols should be loaded
-        self.assertEqual(debug_stats["totalSymbolsLoaded"], 0)
-
-        # No symbols should be loaded in each module
-        for module_stats in debug_stats["modules"]:
-            self.assertEqual(module_stats["symbolsLoaded"], 0)
-
     def test_default_with_run(self):
         """Test "statistics dump" when running the target to a breakpoint.
 
diff --git a/lldb/unittests/Symbol/LineTableTest.cpp b/lldb/unittests/Symbol/LineTableTest.cpp
index eadab40a37fac..f67ac46c738e7 100644
--- a/lldb/unittests/Symbol/LineTableTest.cpp
+++ b/lldb/unittests/Symbol/LineTableTest.cpp
@@ -58,7 +58,7 @@ class FakeSymbolFile : public SymbolFile {
   uint32_t CalculateAbilities() override { return UINT32_MAX; }
   uint32_t GetNumCompileUnits() override { return 1; }
   CompUnitSP GetCompileUnitAtIndex(uint32_t) override { return m_cu_sp; }
-  Symtab *GetSymtab(bool can_create = true) override { return nullptr; }
+  Symtab *GetSymtab() override { return nullptr; }
   LanguageType ParseLanguage(CompileUnit &) override { return eLanguageTypeC; }
   size_t ParseFunctions(CompileUnit &) override { return 0; }
   bool ParseLineTable(CompileUnit &) override { return true; }
diff --git a/lldb/unittests/Symbol/SymtabTest.cpp b/lldb/unittests/Symbol/SymtabTest.cpp
index d3f462fab86a1..e431c31e1eac1 100644
--- a/lldb/unittests/Symbol/SymtabTest.cpp
+++ b/lldb/unittests/Symbol/SymtabTest.cpp
@@ -721,7 +721,7 @@ TEST_F(SymtabTest, TestDecodeCStringMaps) {
   ASSERT_NE(symbol, nullptr);
 }
 
-TEST_F(SymtabTest, TestSymbolFileCreatedOnDemand) {
+TEST_F(SymtabTest, TestSymtabCreatedOnDemand) {
   auto ExpectedFile = TestFile::fromYaml(R"(
 --- !ELF
 FileHeader:
@@ -749,7 +749,7 @@ TEST_F(SymtabTest, TestSymbolFileCreatedOnDemand) {
   ASSERT_THAT_EXPECTED(ExpectedFile, llvm::Succeeded());
   auto module_sp = std::make_shared<Module>(ExpectedFile->moduleSpec());
 
-  // The symbol file should not be created by default.
+  // The symbol table should not be loaded by default.
   Symtab *module_symtab = module_sp->GetSymtab(/*can_create=*/false);
   ASSERT_EQ(module_symtab, nullptr);
 
@@ -761,77 +761,3 @@ TEST_F(SymtabTest, TestSymbolFileCreatedOnDemand) {
   Symtab *cached_module_symtab = module_sp->GetSymtab(/*can_create=*/false);
   ASSERT_EQ(module_symtab, cached_module_symtab);
 }
-
-TEST_F(SymtabTest, TestSymbolTableCreatedOnDemand) {
-  const char *yamldata = R"(
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_EXEC
-  Machine: EM_386
-DWARF:
-  debug_abbrev:
-    - Table:
-        - Code:            0x00000001
-          Tag:             DW_TAG_compile_unit
-          Children:        DW_CHILDREN_no
-          Attributes:
-            - Attribute:       DW_AT_addr_base
-              Form:            DW_FORM_sec_offset
-  debug_info:
-    - Version:         5
-      AddrSize:        4
-      UnitType:        DW_UT_compile
-      Entries:
-        - AbbrCode:        0x00000001
-          Values:
-            - Value:           0x8 # Offset of the first Address past the header
-        - AbbrCode:        0x0
-  debug_addr:
-    - Version: 5
-      AddressSize: 4
-      Entries:
-        - Address: 0x1234
-        - Address: 0x5678
-  debug_line:
-    - Length:          42
-      Version:         2
-      PrologueLength:  36
-      MinInstLength:   1
-      DefaultIsStmt:   1
-      LineBase:        251
-      LineRange:       14
-      OpcodeBase:      13
-      StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
-      IncludeDirs:
-        - '/tmp'
-      Files:
-        - Name:            main.cpp
-          DirIdx:          1
-          ModTime:         0
-          Length:          0
-)";
-  llvm::Expected<TestFile> file = TestFile::fromYaml(yamldata);
-  EXPECT_THAT_EXPECTED(file, llvm::Succeeded());
-  auto module_sp = std::make_shared<Module>(file->moduleSpec());
-
-  SymbolFile *symbol_file = module_sp->GetSymbolFile();
-  // At this point, the symbol table is not created. This is because the above
-  // yaml data contains the necessary sections in order for
-  // SymbolFileDWARF::CalculateAbilities() to identify all abilities, saving it
-  // from calling SymbolFileDWARFDebugMap::CalculateAbilities(), which
-  // eventually loads the symbol table, which we don't want.
-
-  // The symbol table should not be created if asked not to.
-  Symtab *symtab = symbol_file->GetSymtab(/*can_create=*/false);
-  ASSERT_EQ(symtab, nullptr);
-
-  // But it should be created on demand.
-  symtab = symbol_file->GetSymtab(/*can_create=*/true);
-  ASSERT_NE(symtab, nullptr);
-
-  // And we should be able to get it again once it has been created.
-  Symtab *cached_symtab = symbol_file->GetSymtab(/*can_create=*/false);
-  ASSERT_EQ(symtab, cached_symtab);
-}

From 2b44eb9f56a51a25391319541de750bb9b5c1fc9 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 21 Apr 2025 17:46:24 -0700
Subject: [PATCH 675/710] [LLVM][TableGen] Fix VarlenDecoder.td to not used
 fixed opcode values (#136632)

---
 llvm/test/TableGen/VarLenDecoder.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/test/TableGen/VarLenDecoder.td b/llvm/test/TableGen/VarLenDecoder.td
index 0ca5c6ce53506..db7a520533e85 100644
--- a/llvm/test/TableGen/VarLenDecoder.td
+++ b/llvm/test/TableGen/VarLenDecoder.td
@@ -56,9 +56,9 @@ def FOO32 : MyVarInst<MemOp32> {
 
 // CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
 // CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 8, 4, 0, 0, // Skip to: 12
-// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_Decode, 178, 2, 0, // Opcode: FOO16
+// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
 // CHECK-LARGE-NEXT: /* 12 */      MCD::OPC_FilterValue, 9, 4, 0, 0, // Skip to: 21
-// CHECK-LARGE-NEXT: /* 17 */      MCD::OPC_Decode, 179, 2, 1, // Opcode: FOO32
+// CHECK-LARGE-NEXT: /* 17 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
 // CHECK-LARGE-NEXT: /* 21 */      MCD::OPC_Fail,
 
 // Instruction length table

From c7ea01b8a0382c21da1f64a9bf077c951096db8e Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <peter@pcc.me.uk>
Date: Mon, 21 Apr 2025 17:49:48 -0700
Subject: [PATCH 676/710] LowerTypeTests: Switch to emitting one inline asm
 call per jump table entry.

With the previous approach of emitting one inline asm call for all jump
table entries we would encounter SelectionDAG's limit on the number
of operands per node (65536) when the number of jump table entries
exceeded that number. Fix the problem by switching to one inline asm
per jump table entry so that each DAG node only needs one operand.

Reviewers: fmayer, vitalybuka

Reviewed By: fmayer

Pull Request: https://github.com/llvm/llvm-project/pull/136265
---
 llvm/lib/Transforms/IPO/LowerTypeTests.cpp    | 97 ++++++++-----------
 .../LowerTypeTests/aarch64-jumptable.ll       |  3 +-
 .../LowerTypeTests/cfi-direct-call1.ll        |  5 +-
 .../LowerTypeTests/function-arm-thumb.ll      |  7 +-
 .../LowerTypeTests/function-thumb-bti.ll      |  6 +-
 .../Transforms/LowerTypeTests/function.ll     | 36 ++++---
 .../LowerTypeTests/x86-jumptable.ll           |  6 +-
 .../Transforms/MergeFunc/cfi-thunk-merging.ll |  3 +-
 8 files changed, 83 insertions(+), 80 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 7c01f8b560fea..8f62ff10b1523 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -441,10 +441,6 @@ class LowerTypeTestsModule {
   // Cache variable used by hasBranchTargetEnforcement().
   int HasBranchTargetEnforcement = -1;
 
-  // The jump table type we ended up deciding on. (Usually the same as
-  // Arch, except that 'arm' and 'thumb' are often interchangeable.)
-  Triple::ArchType JumpTableArch = Triple::UnknownArch;
-
   IntegerType *Int1Ty = Type::getInt1Ty(M.getContext());
   IntegerType *Int8Ty = Type::getInt8Ty(M.getContext());
   PointerType *PtrTy = PointerType::getUnqual(M.getContext());
@@ -525,11 +521,8 @@ class LowerTypeTestsModule {
   Triple::ArchType
   selectJumpTableArmEncoding(ArrayRef<GlobalTypeMember *> Functions);
   bool hasBranchTargetEnforcement();
-  unsigned getJumpTableEntrySize();
-  Type *getJumpTableEntryType();
-  void createJumpTableEntry(raw_ostream &AsmOS, raw_ostream &ConstraintOS,
-                            Triple::ArchType JumpTableArch,
-                            SmallVectorImpl<Value *> &AsmArgs, Function *Dest);
+  unsigned getJumpTableEntrySize(Triple::ArchType JumpTableArch);
+  InlineAsm *createJumpTableEntryAsm(Triple::ArchType JumpTableArch);
   void verifyTypeMDNode(GlobalObject *GO, MDNode *Type);
   void buildBitSetsFromFunctions(ArrayRef<Metadata *> TypeIds,
                                  ArrayRef<GlobalTypeMember *> Functions);
@@ -548,7 +541,8 @@ class LowerTypeTestsModule {
   void findGlobalVariableUsersOf(Constant *C,
                                  SmallSetVector<GlobalVariable *, 8> &Out);
 
-  void createJumpTable(Function *F, ArrayRef<GlobalTypeMember *> Functions);
+  void createJumpTable(Function *F, ArrayRef<GlobalTypeMember *> Functions,
+                       Triple::ArchType JumpTableArch);
 
   /// replaceCfiUses - Go through the uses list for this definition
   /// and make each use point to "V" instead of "this" when the use is outside
@@ -1245,7 +1239,8 @@ bool LowerTypeTestsModule::hasBranchTargetEnforcement() {
   return HasBranchTargetEnforcement;
 }
 
-unsigned LowerTypeTestsModule::getJumpTableEntrySize() {
+unsigned
+LowerTypeTestsModule::getJumpTableEntrySize(Triple::ArchType JumpTableArch) {
   switch (JumpTableArch) {
   case Triple::x86:
   case Triple::x86_64:
@@ -1278,33 +1273,32 @@ unsigned LowerTypeTestsModule::getJumpTableEntrySize() {
   }
 }
 
-// Create a jump table entry for the target. This consists of an instruction
-// sequence containing a relative branch to Dest. Appends inline asm text,
-// constraints and arguments to AsmOS, ConstraintOS and AsmArgs.
-void LowerTypeTestsModule::createJumpTableEntry(
-    raw_ostream &AsmOS, raw_ostream &ConstraintOS,
-    Triple::ArchType JumpTableArch, SmallVectorImpl<Value *> &AsmArgs,
-    Function *Dest) {
-  unsigned ArgIndex = AsmArgs.size();
+// Create an inline asm constant representing a jump table entry for the target.
+// This consists of an instruction sequence containing a relative branch to
+// Dest.
+InlineAsm *
+LowerTypeTestsModule::createJumpTableEntryAsm(Triple::ArchType JumpTableArch) {
+  std::string Asm;
+  raw_string_ostream AsmOS(Asm);
 
   if (JumpTableArch == Triple::x86 || JumpTableArch == Triple::x86_64) {
     bool Endbr = false;
     if (const auto *MD = mdconst::extract_or_null<ConstantInt>(
-          Dest->getParent()->getModuleFlag("cf-protection-branch")))
+            M.getModuleFlag("cf-protection-branch")))
       Endbr = !MD->isZero();
     if (Endbr)
       AsmOS << (JumpTableArch == Triple::x86 ? "endbr32\n" : "endbr64\n");
-    AsmOS << "jmp ${" << ArgIndex << ":c}@plt\n";
+    AsmOS << "jmp ${0:c}@plt\n";
     if (Endbr)
       AsmOS << ".balign 16, 0xcc\n";
     else
       AsmOS << "int3\nint3\nint3\n";
   } else if (JumpTableArch == Triple::arm) {
-    AsmOS << "b $" << ArgIndex << "\n";
+    AsmOS << "b $0\n";
   } else if (JumpTableArch == Triple::aarch64) {
     if (hasBranchTargetEnforcement())
       AsmOS << "bti c\n";
-    AsmOS << "b $" << ArgIndex << "\n";
+    AsmOS << "b $0\n";
   } else if (JumpTableArch == Triple::thumb) {
     if (!CanUseThumbBWJumpTable) {
       // In Armv6-M, this sequence will generate a branch without corrupting
@@ -1328,28 +1322,26 @@ void LowerTypeTestsModule::createJumpTableEntry(
             << "str r0, [sp, #4]\n"
             << "pop {r0,pc}\n"
             << ".balign 4\n"
-            << "1: .word $" << ArgIndex << " - (0b + 4)\n";
+            << "1: .word $0 - (0b + 4)\n";
     } else {
       if (hasBranchTargetEnforcement())
         AsmOS << "bti\n";
-      AsmOS << "b.w $" << ArgIndex << "\n";
+      AsmOS << "b.w $0\n";
     }
   } else if (JumpTableArch == Triple::riscv32 ||
              JumpTableArch == Triple::riscv64) {
-    AsmOS << "tail $" << ArgIndex << "@plt\n";
+    AsmOS << "tail $0@plt\n";
   } else if (JumpTableArch == Triple::loongarch64) {
-    AsmOS << "pcalau12i $$t0, %pc_hi20($" << ArgIndex << ")\n"
-          << "jirl $$r0, $$t0, %pc_lo12($" << ArgIndex << ")\n";
+    AsmOS << "pcalau12i $$t0, %pc_hi20($0)\n"
+          << "jirl $$r0, $$t0, %pc_lo12($0)\n";
   } else {
     report_fatal_error("Unsupported architecture for jump tables");
   }
 
-  ConstraintOS << (ArgIndex > 0 ? ",s" : "s");
-  AsmArgs.push_back(Dest);
-}
-
-Type *LowerTypeTestsModule::getJumpTableEntryType() {
-  return ArrayType::get(Int8Ty, getJumpTableEntrySize());
+  return InlineAsm::get(
+      FunctionType::get(Type::getVoidTy(M.getContext()), PtrTy, false),
+      AsmOS.str(), "s",
+      /*hasSideEffects=*/true);
 }
 
 /// Given a disjoint set of type identifiers and functions, build the bit sets
@@ -1498,12 +1490,18 @@ Triple::ArchType LowerTypeTestsModule::selectJumpTableArmEncoding(
 }
 
 void LowerTypeTestsModule::createJumpTable(
-    Function *F, ArrayRef<GlobalTypeMember *> Functions) {
+    Function *F, ArrayRef<GlobalTypeMember *> Functions,
+    Triple::ArchType JumpTableArch) {
   std::string AsmStr, ConstraintStr;
   raw_string_ostream AsmOS(AsmStr), ConstraintOS(ConstraintStr);
   SmallVector<Value *, 16> AsmArgs;
   AsmArgs.reserve(Functions.size() * 2);
 
+  BasicBlock *BB = BasicBlock::Create(M.getContext(), "entry", F);
+  IRBuilder<> IRB(BB);
+
+  InlineAsm *JumpTableAsm = createJumpTableEntryAsm(JumpTableArch);
+
   // Check if all entries have the NoUnwind attribute.
   // If all entries have it, we can safely mark the
   // cfi.jumptable as NoUnwind, otherwise, direct calls
@@ -1514,12 +1512,12 @@ void LowerTypeTestsModule::createJumpTable(
              ->hasFnAttribute(llvm::Attribute::NoUnwind)) {
       areAllEntriesNounwind = false;
     }
-    createJumpTableEntry(AsmOS, ConstraintOS, JumpTableArch, AsmArgs,
-                         cast<Function>(GTM->getGlobal()));
+    IRB.CreateCall(JumpTableAsm, GTM->getGlobal());
   }
+  IRB.CreateUnreachable();
 
   // Align the whole table by entry size.
-  F->setAlignment(Align(getJumpTableEntrySize()));
+  F->setAlignment(Align(getJumpTableEntrySize(JumpTableArch)));
   // Skip prologue.
   // Disabled on win32 due to https://llvm.org/bugs/show_bug.cgi?id=28641#c3.
   // Luckily, this function does not get any prologue even without the
@@ -1568,21 +1566,6 @@ void LowerTypeTestsModule::createJumpTable(
 
   // Make sure we do not inline any calls to the cfi.jumptable.
   F->addFnAttr(Attribute::NoInline);
-
-  BasicBlock *BB = BasicBlock::Create(M.getContext(), "entry", F);
-  IRBuilder<> IRB(BB);
-
-  SmallVector<Type *, 16> ArgTypes;
-  ArgTypes.reserve(AsmArgs.size());
-  for (const auto &Arg : AsmArgs)
-    ArgTypes.push_back(Arg->getType());
-  InlineAsm *JumpTableAsm =
-      InlineAsm::get(FunctionType::get(IRB.getVoidTy(), ArgTypes, false),
-                     AsmOS.str(), ConstraintOS.str(),
-                     /*hasSideEffects=*/true);
-
-  IRB.CreateCall(JumpTableAsm, AsmArgs);
-  IRB.CreateUnreachable();
 }
 
 /// Given a disjoint set of type identifiers and functions, build a jump table
@@ -1669,11 +1652,11 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
 
   // Decide on the jump table encoding, so that we know how big the
   // entries will be.
-  JumpTableArch = selectJumpTableArmEncoding(Functions);
+  Triple::ArchType JumpTableArch = selectJumpTableArmEncoding(Functions);
 
   // Build a simple layout based on the regular layout of jump tables.
   DenseMap<GlobalTypeMember *, uint64_t> GlobalLayout;
-  unsigned EntrySize = getJumpTableEntrySize();
+  unsigned EntrySize = getJumpTableEntrySize(JumpTableArch);
   for (unsigned I = 0; I != Functions.size(); ++I)
     GlobalLayout[Functions[I]] = I * EntrySize;
 
@@ -1684,7 +1667,7 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
                        M.getDataLayout().getProgramAddressSpace(),
                        ".cfi.jumptable", &M);
   ArrayType *JumpTableType =
-      ArrayType::get(getJumpTableEntryType(), Functions.size());
+      ArrayType::get(ArrayType::get(Int8Ty, EntrySize), Functions.size());
   auto JumpTable = ConstantExpr::getPointerCast(
       JumpTableFn, PointerType::getUnqual(M.getContext()));
 
@@ -1742,7 +1725,7 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
     }
   }
 
-  createJumpTable(JumpTableFn, Functions);
+  createJumpTable(JumpTableFn, Functions, JumpTableArch);
 }
 
 /// Assign a dummy layout using an incrementing counter, tag each function
diff --git a/llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll b/llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll
index 3464a748778b6..5ac6d00d9afd1 100644
--- a/llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll
+++ b/llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll
@@ -53,6 +53,7 @@ define i1 @foo(ptr %p) {
 ; AARCH64-LABEL: define private void @.cfi.jumptable
 ; AARCH64-SAME: () #[[ATTR1:[0-9]+]] align 8 {
 ; AARCH64-NEXT:  entry:
-; AARCH64-NEXT:    call void asm sideeffect "bti c\0Ab $0\0Abti c\0Ab $1\0A", "s,s"(ptr @f.cfi, ptr @g.cfi)
+; AARCH64-NEXT:    call void asm sideeffect "bti c\0Ab $0\0A", "s"(ptr @f.cfi)
+; AARCH64-NEXT:    call void asm sideeffect "bti c\0Ab $0\0A", "s"(ptr @g.cfi)
 ; AARCH64-NEXT:    unreachable
 ;
diff --git a/llvm/test/Transforms/LowerTypeTests/cfi-direct-call1.ll b/llvm/test/Transforms/LowerTypeTests/cfi-direct-call1.ll
index 3afb4875ca288..16e8dcc1d6f4c 100644
--- a/llvm/test/Transforms/LowerTypeTests/cfi-direct-call1.ll
+++ b/llvm/test/Transforms/LowerTypeTests/cfi-direct-call1.ll
@@ -73,7 +73,10 @@ entry:
 ; Check which jump table entries are created
 ; FULL: define private void @.cfi.jumptable(){{.*}}
 ; FULL-NEXT: entry:
-; FULL-NEXT: call void asm{{.*}}local_func1.cfi{{.*}}local_func2.cfi{{.*}}extern_weak{{.*}}extern_decl
+; FULL-NEXT: call void asm{{.*}}local_func1.cfi
+; FULL-NEXT: call void asm{{.*}}local_func2.cfi
+; FULL-NEXT: call void asm{{.*}}extern_weak
+; FULL-NEXT: call void asm{{.*}}extern_decl
 
 ; Make sure all local functions have been renamed to <name>.cfi
 ; THIN: define hidden i32 @local_func1.cfi()
diff --git a/llvm/test/Transforms/LowerTypeTests/function-arm-thumb.ll b/llvm/test/Transforms/LowerTypeTests/function-arm-thumb.ll
index 736a06c98003a..3c6c4280a0ca2 100644
--- a/llvm/test/Transforms/LowerTypeTests/function-arm-thumb.ll
+++ b/llvm/test/Transforms/LowerTypeTests/function-arm-thumb.ll
@@ -35,13 +35,16 @@ define void @addrtaken() {
 
 ; CHECK: define private void {{.*}} #[[AT:.*]] align 4 {
 ; CHECK-NEXT: entry:
-; CHECK-NEXT:  call void asm sideeffect "b.w $0\0Ab.w $1\0A", "s,s"(ptr @f1.cfi, ptr @g1.cfi)
+; CHECK-NEXT:  call void asm sideeffect "b.w $0\0A", "s"(ptr @f1.cfi)
+; CHECK-NEXT:  call void asm sideeffect "b.w $0\0A", "s"(ptr @g1.cfi)
 ; CHECK-NEXT:  unreachable
 ; CHECK-NEXT: }
 
 ; CHECK: define private void {{.*}} #[[AA:.*]] align 4 {
 ; CHECK-NEXT: entry:
-; CHECK-NEXT:  call void asm sideeffect "b $0\0Ab $1\0Ab $2\0A", "s,s,s"(ptr @f2.cfi, ptr @g2.cfi, ptr @h2.cfi)
+; CHECK-NEXT:  call void asm sideeffect "b $0\0A", "s"(ptr @f2.cfi)
+; CHECK-NEXT:  call void asm sideeffect "b $0\0A", "s"(ptr @g2.cfi)
+; CHECK-NEXT:  call void asm sideeffect "b $0\0A", "s"(ptr @h2.cfi)
 ; CHECK-NEXT:  unreachable
 ; CHECK-NEXT: }
 
diff --git a/llvm/test/Transforms/LowerTypeTests/function-thumb-bti.ll b/llvm/test/Transforms/LowerTypeTests/function-thumb-bti.ll
index d8f96cabcde75..7f55a15466399 100644
--- a/llvm/test/Transforms/LowerTypeTests/function-thumb-bti.ll
+++ b/llvm/test/Transforms/LowerTypeTests/function-thumb-bti.ll
@@ -40,8 +40,10 @@ define i1 @foo(ptr %p) {
 
 ; And check the actual jump table asm string:
 
-; BTI:   call void asm sideeffect "bti\0Ab.w $0\0Abti\0Ab.w $1\0A", "s,s"(ptr @f.cfi, ptr @g.cfi)
-; NOBTI: call void asm sideeffect "b.w $0\0Ab.w $1\0A", "s,s"(ptr @f.cfi, ptr @g.cfi)
+; BTI:      call void asm sideeffect "bti\0Ab.w $0\0A", "s"(ptr @f.cfi)
+; BTI-NEXT: call void asm sideeffect "bti\0Ab.w $0\0A", "s"(ptr @g.cfi)
+; NOBTI:      call void asm sideeffect "b.w $0\0A", "s"(ptr @f.cfi)
+; NOBTI-NEXT: call void asm sideeffect "b.w $0\0A", "s"(ptr @g.cfi)
 
 ; BTI: attributes [[ATTRS]] = { naked noinline "target-features"="+thumb-mode,+pacbti" }
 ; NOBTI: attributes [[ATTRS]] = { naked noinline "target-cpu"="cortex-a8" "target-features"="+thumb-mode" }
diff --git a/llvm/test/Transforms/LowerTypeTests/function.ll b/llvm/test/Transforms/LowerTypeTests/function.ll
index 1504d6a47847e..f80e99ebfba2c 100644
--- a/llvm/test/Transforms/LowerTypeTests/function.ll
+++ b/llvm/test/Transforms/LowerTypeTests/function.ll
@@ -73,16 +73,10 @@ define i1 @foo(ptr %p) {
 ; X86-SAME: int3
 ; X86-SAME: int3
 ; X86-SAME: int3
-; X86-SAME: jmp ${1:c}@plt
-; X86-SAME: int3
-; X86-SAME: int3
-; X86-SAME: int3
 
 ; ARM:      b $0
-; ARM-SAME: b $1
 
 ; THUMB:      b.w $0
-; THUMB-SAME: b.w $1
 
 ; THUMBV6M:      push {r0,r1}
 ; THUMBV6M-SAME: ldr r0, 1f
@@ -91,23 +85,37 @@ define i1 @foo(ptr %p) {
 ; THUMBV6M-SAME: pop {r0,pc}
 ; THUMBV6M-SAME: .balign 4
 ; THUMBV6M-SAME: 1: .word $0 - (0b + 4)
-; THUMBV6M-SAME: push {r0,r1}
+
+; RISCV:      tail $0@plt
+
+; LOONGARCH64:      pcalau12i $$t0, %pc_hi20($0)
+; LOONGARCH64-SAME: jirl $$r0, $$t0, %pc_lo12($0)
+
+; NATIVE-SAME: "s"(ptr @f.cfi)
+
+; X86-NEXT: jmp ${0:c}@plt
+; X86-SAME: int3
+; X86-SAME: int3
+; X86-SAME: int3
+
+; ARM-NEXT: b $0
+
+; THUMB-NEXT: b.w $0
+
+; THUMBV6M-NEXT: push {r0,r1}
 ; THUMBV6M-SAME: ldr r0, 1f
 ; THUMBV6M-SAME: 0: add r0, r0, pc
 ; THUMBV6M-SAME: str r0, [sp, #4]
 ; THUMBV6M-SAME: pop {r0,pc}
 ; THUMBV6M-SAME: .balign 4
-; THUMBV6M-SAME: 1: .word $1 - (0b + 4)
+; THUMBV6M-SAME: 1: .word $0 - (0b + 4)
 
-; RISCV:      tail $0@plt
-; RISCV-SAME: tail $1@plt
+; RISCV-NEXT: tail $0@plt
 
-; LOONGARCH64:      pcalau12i $$t0, %pc_hi20($0)
+; LOONGARCH64-NEXT: pcalau12i $$t0, %pc_hi20($0)
 ; LOONGARCH64-SAME: jirl $$r0, $$t0, %pc_lo12($0)
-; LOONGARCH64-SAME: pcalau12i $$t0, %pc_hi20($1)
-; LOONGARCH64-SAME: jirl $$r0, $$t0, %pc_lo12($1)
 
-; NATIVE-SAME: "s,s"(ptr @f.cfi, ptr @g.cfi)
+; NATIVE-SAME: "s"(ptr @g.cfi)
 
 ; X86-LINUX: attributes #[[ATTR]] = { naked nocf_check noinline }
 ; X86-WIN32: attributes #[[ATTR]] = { nocf_check noinline }
diff --git a/llvm/test/Transforms/LowerTypeTests/x86-jumptable.ll b/llvm/test/Transforms/LowerTypeTests/x86-jumptable.ll
index f56d30be37959..76acf9469785d 100644
--- a/llvm/test/Transforms/LowerTypeTests/x86-jumptable.ll
+++ b/llvm/test/Transforms/LowerTypeTests/x86-jumptable.ll
@@ -25,7 +25,9 @@ define i1 @foo(ptr %p) {
 
 ; X86:         define private void @.cfi.jumptable() #[[#ATTR:]] align 16 {
 ; X86-NEXT:    entry:
-; X86_32-NEXT:   call void asm sideeffect "endbr32\0Ajmp ${0:c}@plt\0A.balign 16, 0xcc\0Aendbr32\0Ajmp ${1:c}@plt\0A.balign 16, 0xcc\0A", "s,s"(ptr @f.cfi, ptr @g.cfi)
-; X86_64-NEXT:   call void asm sideeffect "endbr64\0Ajmp ${0:c}@plt\0A.balign 16, 0xcc\0Aendbr64\0Ajmp ${1:c}@plt\0A.balign 16, 0xcc\0A", "s,s"(ptr @f.cfi, ptr @g.cfi)
+; X86_32-NEXT:   call void asm sideeffect "endbr32\0Ajmp ${0:c}@plt\0A.balign 16, 0xcc\0A", "s"(ptr @f.cfi)
+; X86_32-NEXT:   call void asm sideeffect "endbr32\0Ajmp ${0:c}@plt\0A.balign 16, 0xcc\0A", "s"(ptr @g.cfi)
+; X86_64-NEXT:   call void asm sideeffect "endbr64\0Ajmp ${0:c}@plt\0A.balign 16, 0xcc\0A", "s"(ptr @f.cfi)
+; X86_64-NEXT:   call void asm sideeffect "endbr64\0Ajmp ${0:c}@plt\0A.balign 16, 0xcc\0A", "s"(ptr @g.cfi)
 
 ; X86_64: attributes #[[#ATTR]] = { naked nocf_check noinline }
diff --git a/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll b/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
index 562cc1a973d81..f4225f95538a0 100644
--- a/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
+++ b/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
@@ -205,6 +205,7 @@ attributes #3 = { noreturn nounwind }
 ; LOWERTYPETESTS-LABEL: define private void @.cfi.jumptable
 ; LOWERTYPETESTS-SAME: () #[[ATTR3:[0-9]+]] align 8 {
 ; LOWERTYPETESTS-NEXT:  entry:
-; LOWERTYPETESTS-NEXT:    call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0Ajmp ${1:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s,s"(ptr @f, ptr @f_thunk)
+; LOWERTYPETESTS-NEXT:    call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s"(ptr @f)
+; LOWERTYPETESTS-NEXT:    call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s"(ptr @f_thunk)
 ; LOWERTYPETESTS-NEXT:    unreachable
 ;

From 698cd48db9c2058442e5e6355a77e91e3456ec2c Mon Sep 17 00:00:00 2001
From: Jim Lin <jim@andestech.com>
Date: Tue, 22 Apr 2025 09:17:37 +0800
Subject: [PATCH 677/710] [RISCV] Fix Lsb > Msb case in (sra (sext_inreg X, _),
 C) for th.ext (#136287)

According the
[spec](https://github.com/XUANTIE-RV/thead-extension-spec/releases/tag/2.3.0),
the operation of `th.ext rd, rs1, msb, lsb` is

  reg[rd] := sign_extend(reg[rs1][msb:lsb])

The spec doesn't specify if lsb is greater than msb.

I don't think lsb can be greater than msb. So that If the shift-right
amount is greater than msb, we can set lsb equal to msb to extract the
bit rs1[msb] and sign-extend it.
---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp |  4 ++-
 llvm/test/CodeGen/RISCV/rv32xtheadbb.ll     | 32 +++++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 093e8b5966fee..4de93d5d5abde 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -653,7 +653,9 @@ bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
       return false;
 
     const unsigned Msb = ExtSize - 1;
-    const unsigned Lsb = RightShAmt;
+    // If the shift-right amount is greater than Msb, it means that extracts
+    // the X[Msb] bit and sign-extend it.
+    const unsigned Lsb = RightShAmt > Msb ? Msb : RightShAmt;
 
     SDNode *TH_EXT = BitfieldExtract(N0, Msb, Lsb, DL, VT);
     ReplaceNode(Node, TH_EXT);
diff --git a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
index 4bb8d6c248caa..04a2f67c4942b 100644
--- a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
@@ -401,6 +401,38 @@ define i64 @no_sexth_i64(i64 %a) nounwind {
   ret i64 %shr
 }
 
+define i32 @sext_sextinreg_sra(i16 %a) nounwind {
+; RV32I-LABEL: sext_sextinreg_sra:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srai a0, a0, 26
+; RV32I-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: sext_sextinreg_sra:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    th.ext a0, a0, 15, 10
+; RV32XTHEADBB-NEXT:    ret
+  %sext = sext i16 %a to i32
+  %shr = ashr exact i32 %sext, 10
+  ret i32 %shr
+}
+
+define i32 @sext_sextinreg_sra_2(i16 %a) nounwind {
+; RV32I-LABEL: sext_sextinreg_sra_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srai a0, a0, 31
+; RV32I-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: sext_sextinreg_sra_2:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    th.ext a0, a0, 15, 15
+; RV32XTHEADBB-NEXT:    ret
+  %sext = sext i16 %a to i32
+  %shr = ashr exact i32 %sext, 24
+  ret i32 %shr
+}
+
 define i32 @zexth_i32(i32 %a) nounwind {
 ; RV32I-LABEL: zexth_i32:
 ; RV32I:       # %bb.0:

From ddb8870a5f396f2f17462f878fb7c0cffc879ce4 Mon Sep 17 00:00:00 2001
From: Jim Lin <jim@andestech.com>
Date: Tue, 22 Apr 2025 09:06:56 +0800
Subject: [PATCH 678/710] [RISCV] Remove the FIXME for using sraiw+and. NFC.

https://github.com/llvm/llvm-project/pull/102034 has implemented it.
---
 llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll b/llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll
index 0d96fbfa81279..3fbaefffac2ef 100644
--- a/llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll
+++ b/llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll
@@ -263,7 +263,6 @@ define i64 @sel_shift_bool_i64(i1 %t) {
   ret i64 %shl
 }
 
-; FIXME: This should use sraiw+and
 define i64 @sraiw_andi(i32 signext %0, i32 signext %1) nounwind {
 ; RV32-LABEL: sraiw_andi:
 ; RV32:       # %bb.0: # %entry

From 46e734746db7176f6e32b3c98beacf1e94fced37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Mon, 21 Apr 2025 19:22:07 -0700
Subject: [PATCH 679/710] [flang][cuda] Update stream type for cuf kernel op
 (#136627)

Update the type of the stream operand to be similar to KernelLaunchOp.
---
 .../flang/Optimizer/Dialect/CUF/CUFOps.td     | 19 +++++++------------
 flang/lib/Lower/Bridge.cpp                    | 10 ++++------
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    |  2 +-
 .../Lower/CUDA/cuda-kernel-loop-directive.cuf |  4 +---
 4 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
index 926983d364ed1..46cc59cda1612 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
@@ -254,24 +254,19 @@ def cuf_KernelOp : cuf_Op<"kernel", [AttrSizedOperandSegments,
     represented by a 0 constant value.
   }];
 
-  let arguments = (ins
-    Variadic<I32>:$grid, // empty means `*`
-    Variadic<I32>:$block, // empty means `*`
-    Optional<I32>:$stream,
-    Variadic<Index>:$lowerbound,
-    Variadic<Index>:$upperbound,
-    Variadic<Index>:$step,
-    OptionalAttr<I64Attr>:$n,
-    Variadic<AnyType>:$reduceOperands,
-    OptionalAttr<ArrayAttr>:$reduceAttrs
-  );
+  let arguments = (ins Variadic<I32>:$grid, // empty means `*`
+      Variadic<I32>:$block,                 // empty means `*`
+      Optional<fir_ReferenceType>:$stream, Variadic<Index>:$lowerbound,
+      Variadic<Index>:$upperbound, Variadic<Index>:$step,
+      OptionalAttr<I64Attr>:$n, Variadic<AnyType>:$reduceOperands,
+      OptionalAttr<ArrayAttr>:$reduceAttrs);
 
   let regions = (region AnyRegion:$region);
 
   let assemblyFormat = [{
     `<` `<` `<` custom<CUFKernelValues>($grid, type($grid)) `,` 
                 custom<CUFKernelValues>($block, type($block))
-        ( `,` `stream` `=` $stream^ )? `>` `>` `>`
+        ( `,` `stream` `=` $stream^ `:` qualified(type($stream)))? `>` `>` `>`
         ( `reduce` `(` $reduceOperands^ `:` type($reduceOperands) `:` $reduceAttrs `)` )?
         custom<CUFKernelLoopControl>($region, $lowerbound, type($lowerbound),
             $upperbound, type($upperbound), $step, type($step))
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 1652a86ed7e63..7b76845b5af05 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -3097,7 +3097,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     llvm::SmallVector<mlir::Value> gridValues;
     llvm::SmallVector<mlir::Value> blockValues;
-    mlir::Value streamValue;
+    mlir::Value streamAddr;
 
     if (launchConfig) {
       const std::list<Fortran::parser::CUFKernelDoConstruct::StarOrExpr> &grid =
@@ -3130,10 +3130,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       }
 
       if (stream)
-        streamValue = builder->createConvert(
-            loc, builder->getI32Type(),
-            fir::getBase(
-                genExprValue(*Fortran::semantics::GetExpr(*stream), stmtCtx)));
+        streamAddr = fir::getBase(
+            genExprAddr(*Fortran::semantics::GetExpr(*stream), stmtCtx));
     }
 
     const auto &outerDoConstruct =
@@ -3267,7 +3265,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     }
 
     auto op = builder->create<cuf::KernelOp>(
-        loc, gridValues, blockValues, streamValue, lbs, ubs, steps, n,
+        loc, gridValues, blockValues, streamAddr, lbs, ubs, steps, n,
         mlir::ValueRange(reduceOperands), builder->getArrayAttr(reduceAttrs));
     builder->createBlock(&op.getRegion(), op.getRegion().end(), ivTypes,
                          ivLocs);
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index a86f12c2c4a55..24033bc15b8eb 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -271,7 +271,7 @@ llvm::LogicalResult cuf::KernelOp::verify() {
         return emitOpError("expect reduce attributes to be ReduceAttr");
     }
   }
-  return mlir::success();
+  return checkStreamType(*this);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf b/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
index 0fceb292f10d2..10f0b9e3d1215 100644
--- a/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
+++ b/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
@@ -75,9 +75,7 @@ subroutine sub1()
   end do
 end
 
-! CHECK: %[[STREAM_LOAD:.*]] = fir.load %[[STREAM]]#0 : !fir.ref<i64>
-! CHECK: %[[STREAM_I32:.*]] = fir.convert %[[STREAM_LOAD]] : (i64) -> i32
-! CHECK: cuf.kernel<<<*, *, stream = %[[STREAM_I32]]>>>
+! CHECK: cuf.kernel<<<*, *, stream = %[[STREAM]]#0 : !fir.ref<i64>>>>
 
 
 ! Test lowering with unstructured construct inside.

From b144258b0c0cc63dffba00a911d6539f00ed07bb Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Tue, 22 Apr 2025 10:41:16 +0800
Subject: [PATCH 680/710] [Clang] Improve error recovery for invalid calls
 (#136295)

It doesn't make sense that we only build a RecoveryExpr for expressions
with invalid trailing commas. This patch extends it so that we now
always build up a RecoveryExpr whenever the call contains anything
invalid. As a result, we can back out HasTrailingComma.

There is only one diagnostic change as to concepts, where a RecoveryExpr
than an ExprError is now used to model an invalid requires clause, for
which we suggest adding parentheses around it. (This looks like what GCC
diagnoses)
---
 clang/docs/ReleaseNotes.rst                   |  4 ++-
 clang/include/clang/Parse/Parser.h            |  3 +-
 clang/lib/Parse/ParseExpr.cpp                 | 31 +++++++------------
 clang/test/AST/ast-dump-recovery.cpp          | 11 ++++---
 clang/test/AST/new-unknown-type.cpp           |  8 ++++-
 .../Parser/cxx-concepts-requires-clause.cpp   |  3 +-
 6 files changed, 31 insertions(+), 29 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 36b528d9e20f8..86d37f5616356 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -398,6 +398,9 @@ Improvements to Clang's diagnostics
   constructors to initialize their non-modifiable members. The diagnostic is
   not new; being controlled via a warning group is what's new. Fixes #GH41104
 
+
+- Improved Clang's error recovery for invalid function calls.
+
 - Improved bit-field diagnostics to consider the type specified by the
   ``preferred_type`` attribute. These diagnostics are controlled by the flags
   ``-Wpreferred-type-bitfield-enum-conversion`` and
@@ -405,7 +408,6 @@ Improvements to Clang's diagnostics
   they're only triggered if the authors are already making the choice to use
   ``preferred_type`` attribute.
 
-
 Improvements to Clang's time-trace
 ----------------------------------
 
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 7ffe23c73eda9..645df4aabc374 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -1942,8 +1942,7 @@ class Parser : public CodeCompletionHandler {
                            llvm::function_ref<void()> ExpressionStarts =
                                llvm::function_ref<void()>(),
                            bool FailImmediatelyOnInvalidExpr = false,
-                           bool EarlyTypoCorrection = false,
-                           bool *HasTrailingComma = nullptr);
+                           bool EarlyTypoCorrection = false);
 
   /// ParseSimpleExpressionList - A simple comma-separated list of expressions,
   /// used for misc language extensions.
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 3e3e7cbcd68b2..59ce8a998969d 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -2218,19 +2218,13 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         CalledSignatureHelp = true;
         return PreferredType;
       };
+      bool ExpressionListIsInvalid = false;
       if (OpKind == tok::l_paren || !LHS.isInvalid()) {
         if (Tok.isNot(tok::r_paren)) {
-          bool HasTrailingComma = false;
-          bool HasError = ParseExpressionList(
-              ArgExprs,
-              [&] {
-                PreferredType.enterFunctionArgument(Tok.getLocation(),
-                                                    RunSignatureHelp);
-              },
-              /*FailImmediatelyOnInvalidExpr*/ false,
-              /*EarlyTypoCorrection*/ false, &HasTrailingComma);
-
-          if (HasError && !HasTrailingComma) {
+          if ((ExpressionListIsInvalid = ParseExpressionList(ArgExprs, [&] {
+                 PreferredType.enterFunctionArgument(Tok.getLocation(),
+                                                     RunSignatureHelp);
+               }))) {
             (void)Actions.CorrectDelayedTyposInExpr(LHS);
             // If we got an error when parsing expression list, we don't call
             // the CodeCompleteCall handler inside the parser. So call it here
@@ -2238,7 +2232,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
             // middle of a parameter.
             if (PP.isCodeCompletionReached() && !CalledSignatureHelp)
               RunSignatureHelp();
-            LHS = ExprError();
           } else if (LHS.isInvalid()) {
             for (auto &E : ArgExprs)
               Actions.CorrectDelayedTyposInExpr(E);
@@ -2249,6 +2242,12 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       // Match the ')'.
       if (LHS.isInvalid()) {
         SkipUntil(tok::r_paren, StopAtSemi);
+      } else if (ExpressionListIsInvalid) {
+        Expr *Fn = LHS.get();
+        ArgExprs.insert(ArgExprs.begin(), Fn);
+        LHS = Actions.CreateRecoveryExpr(Fn->getBeginLoc(), Tok.getLocation(),
+                                         ArgExprs);
+        SkipUntil(tok::r_paren, StopAtSemi);
       } else if (Tok.isNot(tok::r_paren)) {
         bool HadDelayedTypo = false;
         if (Actions.CorrectDelayedTyposInExpr(LHS).get() != LHS.get())
@@ -3700,8 +3699,7 @@ void Parser::injectEmbedTokens() {
 bool Parser::ParseExpressionList(SmallVectorImpl<Expr *> &Exprs,
                                  llvm::function_ref<void()> ExpressionStarts,
                                  bool FailImmediatelyOnInvalidExpr,
-                                 bool EarlyTypoCorrection,
-                                 bool *HasTrailingComma) {
+                                 bool EarlyTypoCorrection) {
   bool SawError = false;
   while (true) {
     if (ExpressionStarts)
@@ -3744,11 +3742,6 @@ bool Parser::ParseExpressionList(SmallVectorImpl<Expr *> &Exprs,
     Token Comma = Tok;
     ConsumeToken();
     checkPotentialAngleBracketDelimiter(Comma);
-
-    if (Tok.is(tok::r_paren)) {
-      if (HasTrailingComma)
-        *HasTrailingComma = true;
-    }
   }
   if (SawError) {
     // Ensure typos get diagnosed when errors were encountered while parsing the
diff --git a/clang/test/AST/ast-dump-recovery.cpp b/clang/test/AST/ast-dump-recovery.cpp
index b59fa3778192f..b8195950f2fa1 100644
--- a/clang/test/AST/ast-dump-recovery.cpp
+++ b/clang/test/AST/ast-dump-recovery.cpp
@@ -34,21 +34,22 @@ void test_invalid_call_1(int s) {
 
 int some_func2(int a, int b);
 void test_invalid_call_2() {
-  // CHECK:   -RecoveryExpr {{.*}} 'int' contains-errors
+  // CHECK:   -RecoveryExpr {{.*}} '<dependent type>' contains-errors
   // CHECK-NEXT: `-UnresolvedLookupExpr {{.*}} '<overloaded function type>' lvalue (ADL) = 'some_func2'
   some_func2(,);
 
-  // CHECK:   -RecoveryExpr {{.*}} 'int' contains-errors
+  // CHECK:   -RecoveryExpr {{.*}} '<dependent type>' contains-errors
   // CHECK-NEXT: `-UnresolvedLookupExpr {{.*}} '<overloaded function type>' lvalue (ADL) = 'some_func2'
   some_func2(,,);
 
-  // CHECK:   `-RecoveryExpr {{.*}} 'int' contains-errors
+  // CHECK:   -RecoveryExpr {{.*}} '<dependent type>' contains-errors
   // CHECK-NEXT: |-UnresolvedLookupExpr {{.*}} '<overloaded function type>' lvalue (ADL) = 'some_func2'
   // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 1
   some_func2(1,);
 
-  // FIXME: Handle invalid argument with recovery
-  // CHECK-NOT: `-RecoveryExpr
+  // CHECK:   -RecoveryExpr {{.*}} '<dependent type>' contains-errors
+  // CHECK-NEXT: |-UnresolvedLookupExpr {{.*}} '<overloaded function type>' lvalue (ADL) = 'some_func2'
+  // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 1
   some_func2(,1);
 }
 
diff --git a/clang/test/AST/new-unknown-type.cpp b/clang/test/AST/new-unknown-type.cpp
index e7dd3c90145d2..9d6b19acba739 100644
--- a/clang/test/AST/new-unknown-type.cpp
+++ b/clang/test/AST/new-unknown-type.cpp
@@ -17,9 +17,15 @@ namespace b {
 // CHECK:      |-NamespaceDecl 0x{{[^ ]*}} <line:5:1, line:11:1> line:5:11 a
 // CHECK-NEXT: | `-FunctionDecl 0x{{[^ ]*}} <line:6:3, line:10:3> line:6:8 computeSomething 'void ()'
 // CHECK-NEXT: |   `-CompoundStmt 0x{{[^ ]*}} <col:27, line:10:3>
+// CHECK-NEXT: |   |-RecoveryExpr {{.*}} '<dependent type>' contains-errors
+// CHECK-NEXT: |   | `-UnresolvedLookupExpr {{.*}} '<overloaded function type>' lvalue (ADL) = 'foo'
+// CHECK-NEXT: |   |-RecoveryExpr {{.*}} '<dependent type>' contains-errors
+// CHECK-NEXT: |   | `-UnresolvedLookupExpr {{.*}} '<overloaded function type>' lvalue (ADL) = 'foo'
+// CHECK-NEXT: |   `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
+// CHECK-NEXT: |    `-UnresolvedLookupExpr {{.*}} '<overloaded function type>' lvalue (ADL) = 'foo'
 // CHECK-NEXT: |-NamespaceDecl 0x{{[^ ]*}} <line:13:1, line:15:1> line:13:11 b
 // CHECK-NEXT: | `-CXXRecordDecl 0x{{[^ ]*}} <line:14:3, col:14> col:10 referenced struct Bar definition
 
 static b::Bar bar;
-// CHECK:      `-VarDecl 0x{{[^ ]*}} <line:23:1, col:15> col:15 bar 'b::Bar' static callinit
+// CHECK:      `-VarDecl 0x{{[^ ]*}} <line:29:1, col:15> col:15 bar 'b::Bar' static callinit
 // CHECK-NEXT:   `-CXXConstructExpr 0x{{[^ ]*}} <col:15> 'b::Bar' 'void () noexcept'
diff --git a/clang/test/Parser/cxx-concepts-requires-clause.cpp b/clang/test/Parser/cxx-concepts-requires-clause.cpp
index 13b4e8439882d..b921be5dbfd6d 100644
--- a/clang/test/Parser/cxx-concepts-requires-clause.cpp
+++ b/clang/test/Parser/cxx-concepts-requires-clause.cpp
@@ -119,7 +119,8 @@ template<typename T> requires 0
 
 template<typename T> requires foo<T>
 (int) bar() { };
-// expected-error@-1{{expected '(' for function-style cast or type construction}}
+// expected-error@-1{{expected '(' for function-style cast or type construction}} \
+// expected-error@-2{{parentheses are required around this expression in a requires clause}}
 
 template<typename T>
 void bar() requires foo<T>();

From 0014b49482c0862c140149c650d653b4e41fa9b4 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Tue, 22 Apr 2025 06:27:29 +0300
Subject: [PATCH 681/710] [TTI] Make all interface methods const (NFCI)
 (#136598)

Making `TargetTransformInfo::Model::Impl` `const` makes sure all
interface methods are `const`, in `BasicTTIImpl`, its bases, and in all
derived classes.

Pull Request: https://github.com/llvm/llvm-project/pull/136598
---
 .../llvm/Analysis/TargetTransformInfo.h       |  2 +-
 .../llvm/Analysis/TargetTransformInfoImpl.h   | 28 +++----
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      | 83 +++++++++----------
 .../AArch64/AArch64TargetTransformInfo.cpp    | 61 +++++++-------
 .../AArch64/AArch64TargetTransformInfo.h      | 64 +++++++-------
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      | 14 ++--
 .../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 12 +--
 .../Target/AMDGPU/R600TargetTransformInfo.cpp | 10 +--
 .../Target/AMDGPU/R600TargetTransformInfo.h   |  6 +-
 .../lib/Target/ARM/ARMTargetTransformInfo.cpp | 27 +++---
 llvm/lib/Target/ARM/ARMTargetTransformInfo.h  | 41 +++++----
 llvm/lib/Target/BPF/BPFTargetTransformInfo.h  |  3 +-
 .../DirectX/DirectXTargetTransformInfo.cpp    |  6 +-
 .../DirectX/DirectXTargetTransformInfo.h      |  5 +-
 .../Hexagon/HexagonTargetTransformInfo.cpp    | 20 ++---
 .../Hexagon/HexagonTargetTransformInfo.h      | 30 +++----
 .../Target/Lanai/LanaiTargetTransformInfo.h   |  9 +-
 .../LoongArchTargetTransformInfo.cpp          |  4 +-
 .../LoongArch/LoongArchTargetTransformInfo.h  |  4 +-
 .../Target/Mips/MipsTargetTransformInfo.cpp   |  4 +-
 .../lib/Target/Mips/MipsTargetTransformInfo.h |  4 +-
 .../Target/NVPTX/NVPTXTargetTransformInfo.cpp | 12 +--
 .../Target/NVPTX/NVPTXTargetTransformInfo.h   | 12 +--
 .../Target/PowerPC/PPCTargetTransformInfo.cpp | 48 +++++------
 .../Target/PowerPC/PPCTargetTransformInfo.h   | 37 ++++-----
 .../Target/RISCV/RISCVTargetTransformInfo.cpp | 31 +++----
 .../Target/RISCV/RISCVTargetTransformInfo.h   | 42 +++++-----
 .../Target/SPIRV/SPIRVTargetTransformInfo.h   |  2 +-
 .../SystemZ/SystemZTargetTransformInfo.cpp    | 28 ++++---
 .../SystemZ/SystemZTargetTransformInfo.h      | 26 +++---
 llvm/lib/Target/VE/VETargetTransformInfo.h    |  8 +-
 .../WebAssemblyTargetTransformInfo.h          |  2 +-
 .../lib/Target/X86/X86TargetTransformInfo.cpp | 68 ++++++++-------
 llvm/lib/Target/X86/X86TargetTransformInfo.h  | 40 ++++-----
 34 files changed, 397 insertions(+), 396 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 2efca0d1d754f..b5d766c34d09d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -2367,7 +2367,7 @@ class TargetTransformInfo::Concept {
 
 template <typename T>
 class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
-  T Impl;
+  const T Impl;
 
 public:
   Model(T Impl) : Impl(std::move(Impl)) {}
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index ea481baddc5c3..d28803cf164a3 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -341,7 +341,7 @@ class TargetTransformInfoImplBase {
   }
 
   bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
-                                    Align Alignment, unsigned AddrSpace) {
+                                    Align Alignment, unsigned AddrSpace) const {
     return false;
   }
 
@@ -440,7 +440,7 @@ class TargetTransformInfoImplBase {
 
   bool enableSelectOptimize() const { return true; }
 
-  bool shouldTreatInstructionLikeSelect(const Instruction *I) {
+  bool shouldTreatInstructionLikeSelect(const Instruction *I) const {
     // A select with two constant operands will usually be better left as a
     // select.
     using namespace llvm::PatternMatch;
@@ -747,7 +747,7 @@ class TargetTransformInfoImplBase {
 
   unsigned getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
                                      const APInt &DemandedDstElts,
-                                     TTI::TargetCostKind CostKind) {
+                                     TTI::TargetCostKind CostKind) const {
     return 1;
   }
 
@@ -1170,7 +1170,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
 
   InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr,
                              ArrayRef<const Value *> Operands, Type *AccessType,
-                             TTI::TargetCostKind CostKind) {
+                             TTI::TargetCostKind CostKind) const {
     assert(PointeeType && Ptr && "can't get GEPCost of nullptr");
     auto *BaseGV = dyn_cast<GlobalValue>(Ptr->stripPointerCasts());
     bool HasBaseReg = (BaseGV == nullptr);
@@ -1234,7 +1234,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
 
     // If the final address of the GEP is a legal addressing mode for the given
     // access type, then we can fold it into its users.
-    if (static_cast<T *>(this)->isLegalAddressingMode(
+    if (static_cast<const T *>(this)->isLegalAddressingMode(
             AccessType, const_cast<GlobalValue *>(BaseGV),
             BaseOffset.sextOrTrunc(64).getSExtValue(), HasBaseReg, Scale,
             Ptr->getType()->getPointerAddressSpace()))
@@ -1250,7 +1250,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
                                        const Value *Base,
                                        const TTI::PointersChainInfo &Info,
                                        Type *AccessTy,
-                                       TTI::TargetCostKind CostKind) {
+                                       TTI::TargetCostKind CostKind) const {
     InstructionCost Cost = TTI::TCC_Free;
     // In the basic model we take into account GEP instructions only
     // (although here can come alloca instruction, a value, constants and/or
@@ -1269,15 +1269,15 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
       if (Info.isSameBase() && V != Base) {
         if (GEP->hasAllConstantIndices())
           continue;
-        Cost += static_cast<T *>(this)->getArithmeticInstrCost(
+        Cost += static_cast<const T *>(this)->getArithmeticInstrCost(
             Instruction::Add, GEP->getType(), CostKind,
             {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
             {});
       } else {
         SmallVector<const Value *> Indices(GEP->indices());
-        Cost += static_cast<T *>(this)->getGEPCost(GEP->getSourceElementType(),
-                                                   GEP->getPointerOperand(),
-                                                   Indices, AccessTy, CostKind);
+        Cost += static_cast<const T *>(this)->getGEPCost(
+            GEP->getSourceElementType(), GEP->getPointerOperand(), Indices,
+            AccessTy, CostKind);
       }
     }
     return Cost;
@@ -1285,10 +1285,10 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
 
   InstructionCost getInstructionCost(const User *U,
                                      ArrayRef<const Value *> Operands,
-                                     TTI::TargetCostKind CostKind) {
+                                     TTI::TargetCostKind CostKind) const {
     using namespace llvm::PatternMatch;
 
-    auto *TargetTTI = static_cast<T *>(this);
+    auto *TargetTTI = static_cast<const T *>(this);
     // Handle non-intrinsic calls, invokes, and callbr.
     // FIXME: Unlikely to be true for anything but CodeSize.
     auto *CB = dyn_cast<CallBase>(U);
@@ -1585,8 +1585,8 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
     return CostKind == TTI::TCK_RecipThroughput ? -1 : TTI::TCC_Basic;
   }
 
-  bool isExpensiveToSpeculativelyExecute(const Instruction *I) {
-    auto *TargetTTI = static_cast<T *>(this);
+  bool isExpensiveToSpeculativelyExecute(const Instruction *I) const {
+    auto *TargetTTI = static_cast<const T *>(this);
     SmallVector<const Value *, 4> Ops(I->operand_values());
     InstructionCost Cost = TargetTTI->getInstructionCost(
         I, Ops, TargetTransformInfo::TCK_SizeAndLatency);
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index b42223eda9922..ca32d36297beb 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -379,11 +379,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return (CallerBits & CalleeBits) == CalleeBits;
   }
 
-  bool hasBranchDivergence(const Function *F = nullptr) { return false; }
+  bool hasBranchDivergence(const Function *F = nullptr) const { return false; }
 
-  bool isSourceOfDivergence(const Value *V) { return false; }
+  bool isSourceOfDivergence(const Value *V) const { return false; }
 
-  bool isAlwaysUniform(const Value *V) { return false; }
+  bool isAlwaysUniform(const Value *V) const { return false; }
 
   bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const {
     return false;
@@ -393,7 +393,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return true;
   }
 
-  unsigned getFlatAddressSpace() {
+  unsigned getFlatAddressSpace() const {
     // Return an invalid address space.
     return -1;
   }
@@ -426,22 +426,22 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return nullptr;
   }
 
-  bool isLegalAddImmediate(int64_t imm) {
+  bool isLegalAddImmediate(int64_t imm) const {
     return getTLI()->isLegalAddImmediate(imm);
   }
 
-  bool isLegalAddScalableImmediate(int64_t Imm) {
+  bool isLegalAddScalableImmediate(int64_t Imm) const {
     return getTLI()->isLegalAddScalableImmediate(Imm);
   }
 
-  bool isLegalICmpImmediate(int64_t imm) {
+  bool isLegalICmpImmediate(int64_t imm) const {
     return getTLI()->isLegalICmpImmediate(imm);
   }
 
   bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
                              bool HasBaseReg, int64_t Scale, unsigned AddrSpace,
                              Instruction *I = nullptr,
-                             int64_t ScalableOffset = 0) {
+                             int64_t ScalableOffset = 0) const {
     TargetLoweringBase::AddrMode AM;
     AM.BaseGV = BaseGV;
     AM.BaseOffs = BaseOffset;
@@ -487,11 +487,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return getTLI()->isIndexedStoreLegal(getISDIndexedMode(M), VT);
   }
 
-  bool isLSRCostLess(TTI::LSRCost C1, TTI::LSRCost C2) {
+  bool isLSRCostLess(TTI::LSRCost C1, TTI::LSRCost C2) const {
     return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
   }
 
-  bool isNumRegsMajorCostOfLSR() {
+  bool isNumRegsMajorCostOfLSR() const {
     return TargetTransformInfoImplBase::isNumRegsMajorCostOfLSR();
   }
 
@@ -499,13 +499,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return TargetTransformInfoImplBase::shouldDropLSRSolutionIfLessProfitable();
   }
 
-  bool isProfitableLSRChainElement(Instruction *I) {
+  bool isProfitableLSRChainElement(Instruction *I) const {
     return TargetTransformInfoImplBase::isProfitableLSRChainElement(I);
   }
 
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
                                        StackOffset BaseOffset, bool HasBaseReg,
-                                       int64_t Scale, unsigned AddrSpace) {
+                                       int64_t Scale,
+                                       unsigned AddrSpace) const {
     TargetLoweringBase::AddrMode AM;
     AM.BaseGV = BaseGV;
     AM.BaseOffs = BaseOffset.getFixed();
@@ -517,11 +518,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return InstructionCost::getInvalid();
   }
 
-  bool isTruncateFree(Type *Ty1, Type *Ty2) {
+  bool isTruncateFree(Type *Ty1, Type *Ty2) const {
     return getTLI()->isTruncateFree(Ty1, Ty2);
   }
 
-  bool isProfitableToHoist(Instruction *I) {
+  bool isProfitableToHoist(Instruction *I) const {
     return getTLI()->isProfitableToHoist(I);
   }
 
@@ -539,14 +540,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr,
                              ArrayRef<const Value *> Operands, Type *AccessType,
-                             TTI::TargetCostKind CostKind) {
+                             TTI::TargetCostKind CostKind) const {
     return BaseT::getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind);
   }
 
   unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
                                             unsigned &JumpTableSize,
                                             ProfileSummaryInfo *PSI,
-                                            BlockFrequencyInfo *BFI) {
+                                            BlockFrequencyInfo *BFI) const {
     /// Try to find the estimated number of clusters. Note that the number of
     /// clusters identified in this function could be different from the actual
     /// numbers found in lowering. This function ignore switches that are
@@ -602,7 +603,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return N;
   }
 
-  bool shouldBuildLookupTables() {
+  bool shouldBuildLookupTables() const {
     const TargetLoweringBase *TLI = getTLI();
     return TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
            TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
@@ -633,18 +634,16 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return true;
   }
 
-  bool haveFastSqrt(Type *Ty) {
+  bool haveFastSqrt(Type *Ty) const {
     const TargetLoweringBase *TLI = getTLI();
     EVT VT = TLI->getValueType(DL, Ty);
     return TLI->isTypeLegal(VT) &&
            TLI->isOperationLegalOrCustom(ISD::FSQRT, VT);
   }
 
-  bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) {
-    return true;
-  }
+  bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { return true; }
 
-  InstructionCost getFPOpCost(Type *Ty) {
+  InstructionCost getFPOpCost(Type *Ty) const {
     // Check whether FADD is available, as a proxy for floating-point in
     // general.
     const TargetLoweringBase *TLI = getTLI();
@@ -674,7 +673,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   unsigned getInliningThresholdMultiplier() const { return 1; }
-  unsigned adjustInliningThreshold(const CallBase *CB) { return 0; }
+  unsigned adjustInliningThreshold(const CallBase *CB) const { return 0; }
   unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const {
     return 0;
   }
@@ -683,7 +682,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE) {
+                               OptimizationRemarkEmitter *ORE) const {
     // This unrolling functionality is target independent, but to provide some
     // motivation for its intended use, for x86:
 
@@ -754,7 +753,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP) {
+                             TTI::PeelingPreferences &PP) const {
     PP.PeelCount = 0;
     PP.AllowPeeling = true;
     PP.AllowLoopNestsPeeling = false;
@@ -762,34 +761,33 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
-                                AssumptionCache &AC,
-                                TargetLibraryInfo *LibInfo,
-                                HardwareLoopInfo &HWLoopInfo) {
+                                AssumptionCache &AC, TargetLibraryInfo *LibInfo,
+                                HardwareLoopInfo &HWLoopInfo) const {
     return BaseT::isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
   }
 
-  unsigned getEpilogueVectorizationMinVF() {
+  unsigned getEpilogueVectorizationMinVF() const {
     return BaseT::getEpilogueVectorizationMinVF();
   }
 
-  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) {
+  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
     return BaseT::preferPredicateOverEpilogue(TFI);
   }
 
   TailFoldingStyle
-  getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) {
+  getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
     return BaseT::getPreferredTailFoldingStyle(IVUpdateMayOverflow);
   }
 
   std::optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
-                                               IntrinsicInst &II) {
+                                                    IntrinsicInst &II) const {
     return BaseT::instCombineIntrinsic(IC, II);
   }
 
   std::optional<Value *>
   simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
                                    APInt DemandedMask, KnownBits &Known,
-                                   bool &KnownBitsComputed) {
+                                   bool &KnownBitsComputed) const {
     return BaseT::simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
                                                    KnownBitsComputed);
   }
@@ -798,7 +796,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
       APInt &UndefElts2, APInt &UndefElts3,
       std::function<void(Instruction *, unsigned, APInt, APInt &)>
-          SimplifyAndSetOp) {
+          SimplifyAndSetOp) const {
     return BaseT::simplifyDemandedVectorEltsIntrinsic(
         IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
         SimplifyAndSetOp);
@@ -1015,7 +1013,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     }
   }
 
-  unsigned getMaxInterleaveFactor(ElementCount VF) { return 1; }
+  unsigned getMaxInterleaveFactor(ElementCount VF) const { return 1; }
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
@@ -1337,7 +1335,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst,
-                                           VectorType *VecTy, unsigned Index) {
+                                           VectorType *VecTy,
+                                           unsigned Index) const {
     TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
     return thisT()->getVectorInstrCost(Instruction::ExtractElement, VecTy,
                                        CostKind, Index, nullptr, nullptr) +
@@ -1417,14 +1416,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost getVectorInstrCost(
       unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
       Value *Scalar,
-      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) {
+      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {
     return thisT()->getVectorInstrCost(Opcode, Val, CostKind, Index, nullptr,
                                        nullptr);
   }
 
   InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index) {
+                                     unsigned Index) const {
     Value *Op0 = nullptr;
     Value *Op1 = nullptr;
     if (auto *IE = dyn_cast<InsertElementInst>(&I)) {
@@ -1554,7 +1553,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false) {
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const {
 
     // We cannot scalarize scalable vectors, so return Invalid.
     if (isa<ScalableVectorType>(VecTy))
@@ -2886,7 +2885,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *,
-                                            const SCEV *) {
+                                            const SCEV *) const {
     return 0;
   }
 
@@ -3067,7 +3066,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned,
                                            Type *ResTy, VectorType *Ty,
                                            std::optional<FastMathFlags> FMF,
-                                           TTI::TargetCostKind CostKind) {
+                                           TTI::TargetCostKind CostKind) const {
     if (auto *FTy = dyn_cast<FixedVectorType>(Ty);
         FTy && IsUnsigned && Opcode == Instruction::Add &&
         FTy->getElementType() == IntegerType::getInt1Ty(Ty->getContext())) {
@@ -3095,7 +3094,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   InstructionCost getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
                                          VectorType *Ty,
-                                         TTI::TargetCostKind CostKind) {
+                                         TTI::TargetCostKind CostKind) const {
     // Without any native support, this is equivalent to the cost of
     // vecreduce.add(mul(ext(Ty A), ext(Ty B))) or
     // vecreduce.add(mul(A, B)).
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 51fa5237fcc50..720daa384968c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -415,7 +415,7 @@ AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
 InstructionCost AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                                   const APInt &Imm, Type *Ty,
                                                   TTI::TargetCostKind CostKind,
-                                                  Instruction *Inst) {
+                                                  Instruction *Inst) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -483,7 +483,7 @@ InstructionCost AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
 InstructionCost
 AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
-                                    TTI::TargetCostKind CostKind) {
+                                    TTI::TargetCostKind CostKind) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -533,7 +533,7 @@ AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
 }
 
 TargetTransformInfo::PopcntSupportKind
-AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) {
+AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) const {
   assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
   if (TyWidth == 32 || TyWidth == 64)
     return TTI::PSK_FastHardware;
@@ -3560,7 +3560,7 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
 InstructionCost AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode,
                                                          Type *Dst,
                                                          VectorType *VecTy,
-                                                         unsigned Index) {
+                                                         unsigned Index) const {
 
   // Make sure we were given a valid extend opcode.
   assert((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
@@ -3812,7 +3812,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 InstructionCost AArch64TTIImpl::getVectorInstrCost(
     unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
     Value *Scalar,
-    ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) {
+    ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {
   return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, false, nullptr,
                                   Scalar, ScalarUserAndIdx);
 }
@@ -3820,7 +3820,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCost(
 InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I,
                                                    Type *Val,
                                                    TTI::TargetCostKind CostKind,
-                                                   unsigned Index) {
+                                                   unsigned Index) const {
   return getVectorInstrCostHelper(I.getOpcode(), Val, CostKind, Index,
                                   true /* HasRealUse */, &I);
 }
@@ -4149,9 +4149,9 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
   }
 }
 
-InstructionCost AArch64TTIImpl::getAddressComputationCost(Type *Ty,
-                                                          ScalarEvolution *SE,
-                                                          const SCEV *Ptr) {
+InstructionCost
+AArch64TTIImpl::getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
+                                          const SCEV *Ptr) const {
   // Address computations in vectorized code with non-consecutive addresses will
   // likely result in more instructions compared to scalar code where the
   // computation can more often be merged into the index mode. The resulting
@@ -4466,7 +4466,7 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
 InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-    bool UseMaskForCond, bool UseMaskForGaps) {
+    bool UseMaskForCond, bool UseMaskForGaps) const {
   assert(Factor >= 2 && "Invalid interleave factor");
   auto *VecVTy = cast<VectorType>(VecTy);
 
@@ -4499,7 +4499,7 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
 }
 
 InstructionCost
-AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) {
+AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
   InstructionCost Cost = 0;
   TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   for (auto *I : Tys) {
@@ -4513,7 +4513,7 @@ AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) {
   return Cost;
 }
 
-unsigned AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF) {
+unsigned AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
   return ST->getMaxInterleaveFactor();
 }
 
@@ -4573,7 +4573,7 @@ getFalkorUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 //  1. Has a valid cost, and
 //  2. Has a cost within the supplied budget.
 // Otherwise it returns false.
-static bool isLoopSizeWithinBudget(Loop *L, AArch64TTIImpl &TTI,
+static bool isLoopSizeWithinBudget(Loop *L, const AArch64TTIImpl &TTI,
                                    InstructionCost Budget,
                                    unsigned *FinalSize) {
   // Estimate the size of the loop.
@@ -4601,7 +4601,7 @@ static bool isLoopSizeWithinBudget(Loop *L, AArch64TTIImpl &TTI,
 }
 
 static bool shouldUnrollMultiExitLoop(Loop *L, ScalarEvolution &SE,
-                                      AArch64TTIImpl &TTI) {
+                                      const AArch64TTIImpl &TTI) {
   // Only consider loops with unknown trip counts for which we can determine
   // a symbolic expression. Multi-exit loops with small known trip counts will
   // likely be unrolled anyway.
@@ -4639,7 +4639,7 @@ static bool shouldUnrollMultiExitLoop(Loop *L, ScalarEvolution &SE,
 static void
 getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
                                  TargetTransformInfo::UnrollingPreferences &UP,
-                                 AArch64TTIImpl &TTI) {
+                                 const AArch64TTIImpl &TTI) {
   // Limit loops with structure that is highly likely to benefit from runtime
   // unrolling; that is we exclude outer loops and loops with many blocks (i.e.
   // likely with complex control flow). Note that the heuristics here may be
@@ -4753,9 +4753,9 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
   }
 }
 
-void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
-                                             TTI::UnrollingPreferences &UP,
-                                             OptimizationRemarkEmitter *ORE) {
+void AArch64TTIImpl::getUnrollingPreferences(
+    Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
+    OptimizationRemarkEmitter *ORE) const {
   // Enable partial unrolling and runtime unrolling.
   BaseT::getUnrollingPreferences(L, SE, UP, ORE);
 
@@ -4835,12 +4835,13 @@ void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 }
 
 void AArch64TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                           TTI::PeelingPreferences &PP) {
+                                           TTI::PeelingPreferences &PP) const {
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 
-Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
-                                                         Type *ExpectedType) {
+Value *
+AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
+                                                  Type *ExpectedType) const {
   switch (Inst->getIntrinsicID()) {
   default:
     return nullptr;
@@ -4876,7 +4877,7 @@ Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
 }
 
 bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
-                                        MemIntrinsicInfo &Info) {
+                                        MemIntrinsicInfo &Info) const {
   switch (Inst->getIntrinsicID()) {
   default:
     break;
@@ -4921,7 +4922,7 @@ bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
 /// sext instructions that sign extended the same initial value. A getelementptr
 /// is considered as "complex" if it has more than 2 operands.
 bool AArch64TTIImpl::shouldConsiderAddressTypePromotion(
-    const Instruction &I, bool &AllowPromotionWithoutCommonHeader) {
+    const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
   bool Considerable = false;
   AllowPromotionWithoutCommonHeader = false;
   if (!isa<SExtInst>(&I))
@@ -5161,7 +5162,7 @@ AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
 
 InstructionCost AArch64TTIImpl::getExtendedReductionCost(
     unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *VecTy,
-    std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) {
+    std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
   EVT VecVT = TLI->getValueType(DL, VecTy);
   EVT ResVT = TLI->getValueType(DL, ResTy);
 
@@ -5189,7 +5190,7 @@ InstructionCost AArch64TTIImpl::getExtendedReductionCost(
 InstructionCost
 AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
                                        VectorType *VecTy,
-                                       TTI::TargetCostKind CostKind) {
+                                       TTI::TargetCostKind CostKind) const {
   EVT VecVT = TLI->getValueType(DL, VecTy);
   EVT ResVT = TLI->getValueType(DL, ResTy);
 
@@ -5669,7 +5670,7 @@ unsigned AArch64TTIImpl::getEpilogueVectorizationMinVF() const {
   return ST->getEpilogueVectorizationMinVF();
 }
 
-bool AArch64TTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) {
+bool AArch64TTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
   if (!ST->hasSVE())
     return false;
 
@@ -5733,7 +5734,8 @@ AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
   return InstructionCost::getInvalid();
 }
 
-bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(const Instruction *I) {
+bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(
+    const Instruction *I) const {
   if (EnableOrLikeSelectOpt) {
     // For the binary operators (e.g. or) we need to be more careful than
     // selects, here we only transform them if they are already at a natural
@@ -5751,8 +5753,9 @@ bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(const Instruction *I) {
   return BaseT::shouldTreatInstructionLikeSelect(I);
 }
 
-bool AArch64TTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                                   const TargetTransformInfo::LSRCost &C2) {
+bool AArch64TTIImpl::isLSRCostLess(
+    const TargetTransformInfo::LSRCost &C1,
+    const TargetTransformInfo::LSRCost &C2) const {
   // AArch64 specific here is adding the number of instructions to the
   // comparison (though not as the first consideration, as some targets do)
   // along with changing the priority of the base additions.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 46d2dc57b012d..44b076b9a1c75 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -105,20 +105,22 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,
-                                    Instruction *Inst = nullptr);
+                                    Instruction *Inst = nullptr) const;
   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                       const APInt &Imm, Type *Ty,
-                                      TTI::TargetCostKind CostKind);
-  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
+                                      TTI::TargetCostKind CostKind) const;
+  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const;
 
   /// @}
 
   /// \name Vector TTI Implementations
   /// @{
 
-  bool enableInterleavedAccessVectorization() { return true; }
+  bool enableInterleavedAccessVectorization() const { return true; }
 
-  bool enableMaskedInterleavedAccessVectorization() { return ST->hasSVE(); }
+  bool enableMaskedInterleavedAccessVectorization() const {
+    return ST->hasSVE();
+  }
 
   unsigned getNumberOfRegisters(unsigned ClassID) const {
     bool Vector = (ClassID == 1);
@@ -167,7 +169,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     return VF.getKnownMinValue() * ST->getVScaleForTuning();
   }
 
-  unsigned getMaxInterleaveFactor(ElementCount VF);
+  unsigned getMaxInterleaveFactor(ElementCount VF) const;
 
   bool prefersVectorizedAddressing() const;
 
@@ -190,7 +192,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
                                    const Instruction *I = nullptr) const;
 
   InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst,
-                                           VectorType *VecTy, unsigned Index);
+                                           VectorType *VecTy,
+                                           unsigned Index) const;
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
                                  const Instruction *I = nullptr) const;
@@ -207,11 +210,11 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   InstructionCost getVectorInstrCost(
       unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
       Value *Scalar,
-      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx);
+      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const;
 
   InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index);
+                                     unsigned Index) const;
 
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
@@ -232,7 +235,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
       const Instruction *CxtI = nullptr) const;
 
   InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
-                                            const SCEV *Ptr);
+                                            const SCEV *Ptr) const;
 
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
@@ -251,19 +254,19 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;
 
-  InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys);
+  InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const;
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
 
   Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
-                                           Type *ExpectedType);
+                                           Type *ExpectedType) const;
 
-  bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+  bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
 
   bool isElementTypeLegalForScalableVector(Type *Ty) const {
     if (Ty->isPointerTy())
@@ -282,7 +285,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     return false;
   }
 
-  bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) {
+  bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) const {
     if (!ST->hasSVE())
       return false;
 
@@ -295,12 +298,12 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   }
 
   bool isLegalMaskedLoad(Type *DataType, Align Alignment,
-                         unsigned /*AddressSpace*/) {
+                         unsigned /*AddressSpace*/) const {
     return isLegalMaskedLoadStore(DataType, Alignment);
   }
 
   bool isLegalMaskedStore(Type *DataType, Align Alignment,
-                          unsigned /*AddressSpace*/) {
+                          unsigned /*AddressSpace*/) const {
     return isLegalMaskedLoadStore(DataType, Alignment);
   }
 
@@ -342,7 +345,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     return false;
   }
 
-  bool isLegalNTStoreLoad(Type *DataType, Align Alignment) {
+  bool isLegalNTStoreLoad(Type *DataType, Align Alignment) const {
     // NOTE: The logic below is mostly geared towards LV, which calls it with
     //       vectors with 2 elements. We might want to improve that, if other
     //       users show up.
@@ -359,11 +362,11 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
     return BaseT::isLegalNTStore(DataType, Alignment);
   }
 
-  bool isLegalNTStore(Type *DataType, Align Alignment) {
+  bool isLegalNTStore(Type *DataType, Align Alignment) const {
     return isLegalNTStoreLoad(DataType, Alignment);
   }
 
-  bool isLegalNTLoad(Type *DataType, Align Alignment) {
+  bool isLegalNTLoad(Type *DataType, Align Alignment) const {
     // Only supports little-endian targets.
     if (ST->isLittleEndian())
       return isLegalNTStoreLoad(DataType, Alignment);
@@ -382,11 +385,10 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
 
-  bool
-  shouldConsiderAddressTypePromotion(const Instruction &I,
-                                     bool &AllowPromotionWithoutCommonHeader);
+  bool shouldConsiderAddressTypePromotion(
+      const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const;
 
   bool shouldExpandReduction(const IntrinsicInst *II) const { return false; }
 
@@ -411,7 +413,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
 
   unsigned getEpilogueVectorizationMinVF() const;
 
-  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI);
+  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const;
 
   bool supportsScalableVectors() const {
     return ST->isSVEorStreamingSVEAvailable();
@@ -434,11 +436,11 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned,
                                            Type *ResTy, VectorType *ValTy,
                                            std::optional<FastMathFlags> FMF,
-                                           TTI::TargetCostKind CostKind);
+                                           TTI::TargetCostKind CostKind) const;
 
   InstructionCost getMulAccReductionCost(
       bool IsUnsigned, Type *ResTy, VectorType *Ty,
-      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput);
+      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
 
   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
@@ -462,9 +464,9 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
                                        StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale, unsigned AddrSpace) const;
 
-  bool enableSelectOptimize() { return ST->enableSelectOptimize(); }
+  bool enableSelectOptimize() const { return ST->enableSelectOptimize(); }
 
-  bool shouldTreatInstructionLikeSelect(const Instruction *I);
+  bool shouldTreatInstructionLikeSelect(const Instruction *I) const;
 
   unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
                              Type *ScalarValTy) const {
@@ -478,7 +480,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   std::optional<unsigned> getMinPageSize() const { return 4096; }
 
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                     const TargetTransformInfo::LSRCost &C2);
+                     const TargetTransformInfo::LSRCost &C2) const;
 
   bool isProfitableToSinkOperands(Instruction *I,
                                   SmallVectorImpl<Use *> &Ops) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 3f2e13f8cfc5b..919e73a2c534d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -107,9 +107,9 @@ AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
       ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
       TLI(ST->getTargetLowering()) {}
 
-void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
-                                            TTI::UnrollingPreferences &UP,
-                                            OptimizationRemarkEmitter *ORE) {
+void AMDGPUTTIImpl::getUnrollingPreferences(
+    Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
+    OptimizationRemarkEmitter *ORE) const {
   const Function &F = *L->getHeader()->getParent();
   UP.Threshold =
       F.getFnAttributeAsParsedInteger("amdgpu-unroll-threshold", 300);
@@ -270,7 +270,7 @@ void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 }
 
 void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                          TTI::PeelingPreferences &PP) {
+                                          TTI::PeelingPreferences &PP) const {
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 
@@ -485,7 +485,7 @@ void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
   }
 }
 
-unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
+unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
   // Disable unrolling if the loop is not vectorized.
   // TODO: Enable this again.
   if (VF.isScalar())
@@ -1369,12 +1369,12 @@ unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB,
 
 void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                          TTI::UnrollingPreferences &UP,
-                                         OptimizationRemarkEmitter *ORE) {
+                                         OptimizationRemarkEmitter *ORE) const {
   CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);
 }
 
 void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                       TTI::PeelingPreferences &PP) {
+                                       TTI::PeelingPreferences &PP) const {
   CommonTTI.getPeelingPreferences(L, SE, PP);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 165ed3d59c35d..1a575c92cce61 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -52,10 +52,10 @@ class AMDGPUTTIImpl final : public BasicTTIImplBase<AMDGPUTTIImpl> {
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
 
   int64_t getMaxMemIntrinsicInlineSizeThreshold() const;
 };
@@ -108,12 +108,12 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
 
-  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) {
+  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const {
     assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
     return TTI::PSK_FastHardware;
   }
@@ -149,7 +149,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
       unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
       Align SrcAlign, Align DestAlign,
       std::optional<uint32_t> AtomicCpySize) const;
-  unsigned getMaxInterleaveFactor(ElementCount VF);
+  unsigned getMaxInterleaveFactor(ElementCount VF) const;
 
   bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
 
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
index 3f7274bad374c..6cc2611a8d1a8 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
@@ -82,7 +82,7 @@ bool R600TTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
   return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
 }
 
-unsigned R600TTIImpl::getMaxInterleaveFactor(ElementCount VF) {
+unsigned R600TTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
   // Disable unrolling if the loop is not vectorized.
   // TODO: Enable this again.
   if (VF.isScalar())
@@ -133,13 +133,13 @@ InstructionCost R600TTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
   }
 }
 
-void R600TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
-                                          TTI::UnrollingPreferences &UP,
-                                          OptimizationRemarkEmitter *ORE) {
+void R600TTIImpl::getUnrollingPreferences(
+    Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
+    OptimizationRemarkEmitter *ORE) const {
   CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);
 }
 
 void R600TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                        TTI::PeelingPreferences &PP) {
+                                        TTI::PeelingPreferences &PP) const {
   CommonTTI.getPeelingPreferences(L, SE, PP);
 }
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
index 4adece65a2863..527b225c4eed3 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
@@ -43,9 +43,9 @@ class R600TTIImpl final : public BasicTTIImplBase<R600TTIImpl> {
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
   unsigned getHardwareNumberOfRegisters(bool Vec) const;
   unsigned getNumberOfRegisters(bool Vec) const;
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const;
@@ -57,7 +57,7 @@ class R600TTIImpl final : public BasicTTIImplBase<R600TTIImpl> {
                                    unsigned AddrSpace) const;
   bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
                                     unsigned AddrSpace) const;
-  unsigned getMaxInterleaveFactor(ElementCount VF);
+  unsigned getMaxInterleaveFactor(ElementCount VF) const;
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
                                  const Instruction *I = nullptr) const;
   using BaseT::getVectorInstrCost;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 9c12e4bf62b67..89740a3d7f477 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -353,7 +353,8 @@ InstructionCost ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
 // Constants smaller than 256 fit in the immediate field of
 // Thumb1 instructions so we return a zero cost and 1 otherwise.
 InstructionCost ARMTTIImpl::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
-                                                  const APInt &Imm, Type *Ty) {
+                                                  const APInt &Imm,
+                                                  Type *Ty) const {
   if (Imm.isNonNegative() && Imm.getLimitedValue() < 256)
     return 0;
 
@@ -412,7 +413,7 @@ static bool isFPSatMinMaxPattern(Instruction *Inst, const APInt &Imm) {
 InstructionCost ARMTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                               const APInt &Imm, Type *Ty,
                                               TTI::TargetCostKind CostKind,
-                                              Instruction *Inst) {
+                                              Instruction *Inst) const {
   // Division by a constant can be turned into multiplication, but only if we
   // know it's constant. So it's not so much that the immediate is cheap (it's
   // not), but that the alternative is worse.
@@ -1085,7 +1086,7 @@ InstructionCost ARMTTIImpl::getCmpSelInstrCost(
 
 InstructionCost ARMTTIImpl::getAddressComputationCost(Type *Ty,
                                                       ScalarEvolution *SE,
-                                                      const SCEV *Ptr) {
+                                                      const SCEV *Ptr) const {
   // Address computations in vectorized code with non-consecutive addresses will
   // likely result in more instructions compared to scalar code where the
   // computation can more often be merged into the index mode. The resulting
@@ -1105,7 +1106,7 @@ InstructionCost ARMTTIImpl::getAddressComputationCost(Type *Ty,
   return BaseT::getAddressComputationCost(Ty, SE, Ptr);
 }
 
-bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) {
+bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) const {
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
     // If a VCTP is part of a chain, it's already profitable and shouldn't be
     // optimized, else LSR may block tail-predication.
@@ -1143,7 +1144,7 @@ bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
          (EltWidth == 16 && Alignment >= 2) || (EltWidth == 8);
 }
 
-bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) {
+bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) const {
   if (!EnableMaskedGatherScatters || !ST->hasMVEIntegerOps())
     return false;
 
@@ -1613,7 +1614,7 @@ ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
 InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-    bool UseMaskForCond, bool UseMaskForGaps) {
+    bool UseMaskForCond, bool UseMaskForGaps) const {
   assert(Factor >= 2 && "Invalid interleave factor");
   assert(isa<VectorType>(VecTy) && "Expect a vector type");
 
@@ -1843,7 +1844,7 @@ ARMTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
 
 InstructionCost ARMTTIImpl::getExtendedReductionCost(
     unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
-    std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) {
+    std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
   EVT ValVT = TLI->getValueType(DL, ValTy);
   EVT ResVT = TLI->getValueType(DL, ResTy);
 
@@ -1878,7 +1879,7 @@ InstructionCost ARMTTIImpl::getExtendedReductionCost(
 InstructionCost
 ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
                                    VectorType *ValTy,
-                                   TTI::TargetCostKind CostKind) {
+                                   TTI::TargetCostKind CostKind) const {
   EVT ValVT = TLI->getValueType(DL, ValTy);
   EVT ResVT = TLI->getValueType(DL, ResTy);
 
@@ -2170,7 +2171,7 @@ bool ARMTTIImpl::isLoweredToCall(const Function *F) const {
   return BaseT::isLoweredToCall(F);
 }
 
-bool ARMTTIImpl::maybeLoweredToCall(Instruction &I) {
+bool ARMTTIImpl::maybeLoweredToCall(Instruction &I) const {
   unsigned ISD = TLI->InstructionOpcodeToISD(I.getOpcode());
   EVT VT = TLI->getValueType(DL, I.getType(), true);
   if (TLI->getOperationAction(ISD, VT) == TargetLowering::LibCall)
@@ -2260,7 +2261,7 @@ bool ARMTTIImpl::maybeLoweredToCall(Instruction &I) {
 bool ARMTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                           AssumptionCache &AC,
                                           TargetLibraryInfo *LibInfo,
-                                          HardwareLoopInfo &HWLoopInfo) {
+                                          HardwareLoopInfo &HWLoopInfo) const {
   // Low-overhead branches are only supported in the 'low-overhead branch'
   // extension of v8.1-m.
   if (!ST->hasLOB() || DisableLowOverheadLoops) {
@@ -2493,7 +2494,7 @@ static bool canTailPredicateLoop(Loop *L, LoopInfo *LI, ScalarEvolution &SE,
   return true;
 }
 
-bool ARMTTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) {
+bool ARMTTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
   if (!EnableTailPredication) {
     LLVM_DEBUG(dbgs() << "Tail-predication not enabled.\n");
     return false;
@@ -2559,7 +2560,7 @@ ARMTTIImpl::getPreferredTailFoldingStyle(bool IVUpdateMayOverflow) const {
 }
 void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                          TTI::UnrollingPreferences &UP,
-                                         OptimizationRemarkEmitter *ORE) {
+                                         OptimizationRemarkEmitter *ORE) const {
   // Enable Upper bound unrolling universally, providing that we do not see an
   // active lane mask, which will be better kept as a loop to become tail
   // predicated than to be conditionally unrolled.
@@ -2688,7 +2689,7 @@ void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 }
 
 void ARMTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                       TTI::PeelingPreferences &PP) {
+                                       TTI::PeelingPreferences &PP) const {
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 307a0270fc0a9..b594f07e6a5a9 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -107,7 +107,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
 
-  bool enableInterleavedAccessVectorization() { return true; }
+  bool enableInterleavedAccessVectorization() const { return true; }
 
   TTI::AddressingModeKind
     getPreferredAddressingMode(const Loop *L, ScalarEvolution *SE) const;
@@ -115,7 +115,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   /// Floating-point computation using ARMv8 AArch32 Advanced
   /// SIMD instructions remains unchanged from ARMv7. Only AArch64 SIMD
   /// and Arm MVE are IEEE-754 compliant.
-  bool isFPVectorizationPotentiallyUnsafe() {
+  bool isFPVectorizationPotentiallyUnsafe() const {
     return !ST->isTargetDarwin() && !ST->hasMVEFloatOps();
   }
 
@@ -131,7 +131,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   /// @{
 
   InstructionCost getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
-                                        const APInt &Imm, Type *Ty);
+                                        const APInt &Imm, Type *Ty) const;
 
   using BaseT::getIntImmCost;
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
@@ -140,7 +140,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,
-                                    Instruction *Inst = nullptr);
+                                    Instruction *Inst = nullptr) const;
 
   /// @}
 
@@ -178,11 +178,11 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
     llvm_unreachable("Unsupported register kind");
   }
 
-  unsigned getMaxInterleaveFactor(ElementCount VF) {
+  unsigned getMaxInterleaveFactor(ElementCount VF) const {
     return ST->getMaxInterleaveFactor();
   }
 
-  bool isProfitableLSRChainElement(Instruction *I);
+  bool isProfitableLSRChainElement(Instruction *I) const;
 
   bool isLegalMaskedLoad(Type *DataTy, Align Alignment,
                          unsigned AddressSpace) const;
@@ -192,7 +192,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
     return isLegalMaskedLoad(DataTy, Alignment, AddressSpace);
   }
 
-  bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) {
+  bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) const {
     // For MVE, we have a custom lowering pass that will already have custom
     // legalised any gathers that we can lower to MVE intrinsics, and want to
     // expand all the rest. The pass runs before the masked intrinsic lowering
@@ -200,13 +200,13 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
     return true;
   }
 
-  bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) {
+  bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) const {
     return forceScalarizeMaskedGather(VTy, Alignment);
   }
 
-  bool isLegalMaskedGather(Type *Ty, Align Alignment);
+  bool isLegalMaskedGather(Type *Ty, Align Alignment) const;
 
-  bool isLegalMaskedScatter(Type *Ty, Align Alignment) {
+  bool isLegalMaskedScatter(Type *Ty, Align Alignment) const {
     return isLegalMaskedGather(Ty, Alignment);
   }
 
@@ -253,7 +253,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
                                      Value *Op1) const;
 
   InstructionCost getAddressComputationCost(Type *Val, ScalarEvolution *SE,
-                                            const SCEV *Ptr);
+                                            const SCEV *Ptr) const;
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
@@ -275,7 +275,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
 
   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
@@ -290,10 +290,10 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned,
                                            Type *ResTy, VectorType *ValTy,
                                            std::optional<FastMathFlags> FMF,
-                                           TTI::TargetCostKind CostKind);
+                                           TTI::TargetCostKind CostKind) const;
   InstructionCost getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
                                          VectorType *ValTy,
-                                         TTI::TargetCostKind CostKind);
+                                         TTI::TargetCostKind CostKind) const;
 
   InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                          FastMathFlags FMF,
@@ -310,22 +310,21 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
                                        StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale, unsigned AddrSpace) const;
 
-  bool maybeLoweredToCall(Instruction &I);
+  bool maybeLoweredToCall(Instruction &I) const;
   bool isLoweredToCall(const Function *F) const;
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
-                                AssumptionCache &AC,
-                                TargetLibraryInfo *LibInfo,
-                                HardwareLoopInfo &HWLoopInfo);
-  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI);
+                                AssumptionCache &AC, TargetLibraryInfo *LibInfo,
+                                HardwareLoopInfo &HWLoopInfo) const;
+  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const;
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
 
   TailFoldingStyle
   getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) const;
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
   bool shouldBuildLookupTablesForConstant(Constant *C) const {
     // In the ROPI and RWPI relocation models we can't have pointers to global
     // variables or functions in constant data, so don't convert switches to
diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index 47d11bf6805c9..05a53c8bab824 100644
--- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -37,7 +37,8 @@ class BPFTTIImpl : public BasicTTIImplBase<BPFTTIImpl> {
       : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
-  int getIntImmCost(const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind) {
+  int getIntImmCost(const APInt &Imm, Type *Ty,
+                    TTI::TargetCostKind CostKind) const {
     if (Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue()))
       return TTI::TCC_Free;
 
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
index 4cf3282b108f1..94ebe07d3783b 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -15,8 +15,8 @@
 
 using namespace llvm;
 
-bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
-                                                        unsigned ScalarOpdIdx) {
+bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg(
+    Intrinsic::ID ID, unsigned ScalarOpdIdx) const {
   switch (ID) {
   case Intrinsic::dx_wave_readlane:
     return ScalarOpdIdx == 1;
@@ -26,7 +26,7 @@ bool DirectXTTIImpl::isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
 }
 
 bool DirectXTTIImpl::isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
-                                                            int OpdIdx) {
+                                                            int OpdIdx) const {
   switch (ID) {
   case Intrinsic::dx_asdouble:
     return OpdIdx == 0;
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
index 992d0483de93e..170274c7aa32d 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
@@ -36,8 +36,9 @@ class DirectXTTIImpl : public BasicTTIImplBase<DirectXTTIImpl> {
   unsigned getMinVectorRegisterBitWidth() const { return 32; }
   bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) const;
   bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
-                                          unsigned ScalarOpdIdx);
-  bool isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx);
+                                          unsigned ScalarOpdIdx) const;
+  bool isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
+                                              int OpdIdx) const;
 };
 } // namespace llvm
 
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 0c2deec0a0dab..7825024f9ac0f 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -78,14 +78,14 @@ HexagonTTIImpl::getPopcntSupport(unsigned IntTyWidthInBit) const {
 }
 
 // The Hexagon target can unroll loops with run-time trip counts.
-void HexagonTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
-                                             TTI::UnrollingPreferences &UP,
-                                             OptimizationRemarkEmitter *ORE) {
+void HexagonTTIImpl::getUnrollingPreferences(
+    Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
+    OptimizationRemarkEmitter *ORE) const {
   UP.Runtime = UP.Partial = true;
 }
 
 void HexagonTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                           TTI::PeelingPreferences &PP) {
+                                           TTI::PeelingPreferences &PP) const {
   BaseT::getPeelingPreferences(L, SE, PP);
   // Only try to peel innermost loops with small runtime trip counts.
   if (L && L->isInnermost() && canPeel(L) &&
@@ -110,7 +110,7 @@ unsigned HexagonTTIImpl::getNumberOfRegisters(bool Vector) const {
   return 32;
 }
 
-unsigned HexagonTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
+unsigned HexagonTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
   return useHVX() ? 2 : 1;
 }
 
@@ -157,7 +157,7 @@ HexagonTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
 
 InstructionCost HexagonTTIImpl::getAddressComputationCost(Type *Tp,
                                                           ScalarEvolution *SE,
-                                                          const SCEV *S) {
+                                                          const SCEV *S) const {
   return 0;
 }
 
@@ -245,7 +245,7 @@ InstructionCost HexagonTTIImpl::getGatherScatterOpCost(
 InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-    bool UseMaskForCond, bool UseMaskForGaps) {
+    bool UseMaskForCond, bool UseMaskForGaps) const {
   if (Indices.size() != Factor || UseMaskForCond || UseMaskForGaps)
     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
                                              Alignment, AddressSpace,
@@ -340,14 +340,14 @@ InstructionCost HexagonTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 }
 
 bool HexagonTTIImpl::isLegalMaskedStore(Type *DataType, Align /*Alignment*/,
-                                        unsigned /*AddressSpace*/) {
+                                        unsigned /*AddressSpace*/) const {
   // This function is called from scalarize-masked-mem-intrin, which runs
   // in pre-isel. Use ST directly instead of calling isHVXVectorType.
   return HexagonMaskedVMem && ST.isTypeForHVX(DataType);
 }
 
 bool HexagonTTIImpl::isLegalMaskedLoad(Type *DataType, Align /*Alignment*/,
-                                       unsigned /*AddressSpace*/) {
+                                       unsigned /*AddressSpace*/) const {
   // This function is called from scalarize-masked-mem-intrin, which runs
   // in pre-isel. Use ST directly instead of calling isHVXVectorType.
   return HexagonMaskedVMem && ST.isTypeForHVX(DataType);
@@ -366,7 +366,7 @@ unsigned HexagonTTIImpl::getCacheLineSize() const {
 InstructionCost
 HexagonTTIImpl::getInstructionCost(const User *U,
                                    ArrayRef<const Value *> Operands,
-                                   TTI::TargetCostKind CostKind) {
+                                   TTI::TargetCostKind CostKind) const {
   auto isCastFoldedIntoLoad = [this](const CastInst *CI) -> bool {
     if (!CI->isIntegerCast())
       return false;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index a3820b71b1e4c..9d0c3843e2a7a 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -63,10 +63,10 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   // The Hexagon target can unroll loops with run-time trip counts.
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
 
   /// Bias LSR towards creating post-increment opportunities.
   TTI::AddressingModeKind
@@ -82,7 +82,7 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   /// @{
 
   unsigned getNumberOfRegisters(bool vector) const;
-  unsigned getMaxInterleaveFactor(ElementCount VF);
+  unsigned getMaxInterleaveFactor(ElementCount VF) const;
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getMinVectorRegisterBitWidth() const;
   ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const;
@@ -91,17 +91,13 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   shouldMaximizeVectorBandwidth(TargetTransformInfo::RegisterKind K) const {
     return true;
   }
-  bool supportsEfficientVectorElementLoadStore() { return false; }
-  bool hasBranchDivergence(const Function *F = nullptr) { return false; }
-  bool enableAggressiveInterleaving(bool LoopHasReductions) {
+  bool supportsEfficientVectorElementLoadStore() const { return false; }
+  bool hasBranchDivergence(const Function *F = nullptr) const { return false; }
+  bool enableAggressiveInterleaving(bool LoopHasReductions) const {
     return false;
   }
-  bool prefersVectorizedAddressing() {
-    return false;
-  }
-  bool enableInterleavedAccessVectorization() {
-    return true;
-  }
+  bool prefersVectorizedAddressing() const { return false; }
+  bool enableInterleavedAccessVectorization() const { return true; }
 
   InstructionCost getCallInstrCost(Function *F, Type *RetTy,
                                    ArrayRef<Type *> Tys,
@@ -109,7 +105,7 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind) const;
   InstructionCost getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
-                                            const SCEV *S);
+                                            const SCEV *S) const;
   InstructionCost getMemoryOpCost(
       unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
@@ -131,7 +127,7 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
       TTI::TargetCostKind CostKind,
@@ -160,15 +156,15 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   }
 
   bool isLegalMaskedStore(Type *DataType, Align Alignment,
-                          unsigned AddressSpace);
+                          unsigned AddressSpace) const;
   bool isLegalMaskedLoad(Type *DataType, Align Alignment,
-                         unsigned AddressSpace);
+                         unsigned AddressSpace) const;
 
   /// @}
 
   InstructionCost getInstructionCost(const User *U,
                                      ArrayRef<const Value *> Operands,
-                                     TTI::TargetCostKind CostKind);
+                                     TTI::TargetCostKind CostKind) const;
 
   // Hexagon specific decision to generate a lookup table.
   bool shouldBuildLookupTables() const;
diff --git a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
index 14759e7ca8f4a..ad7e561426395 100644
--- a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
@@ -43,14 +43,15 @@ class LanaiTTIImpl : public BasicTTIImplBase<LanaiTTIImpl> {
 
   bool shouldBuildLookupTables() const { return false; }
 
-  TargetTransformInfo::PopcntSupportKind getPopcntSupport(unsigned TyWidth) {
+  TargetTransformInfo::PopcntSupportKind
+  getPopcntSupport(unsigned TyWidth) const {
     if (TyWidth == 32)
       return TTI::PSK_FastHardware;
     return TTI::PSK_Software;
   }
 
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                TTI::TargetCostKind CostKind) {
+                                TTI::TargetCostKind CostKind) const {
     assert(Ty->isIntegerTy());
     unsigned BitSize = Ty->getPrimitiveSizeInBits();
     // There is no cost model for constants with a bit size of 0. Return
@@ -80,13 +81,13 @@ class LanaiTTIImpl : public BasicTTIImplBase<LanaiTTIImpl> {
   InstructionCost getIntImmCostInst(unsigned Opc, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,
-                                    Instruction *Inst = nullptr) {
+                                    Instruction *Inst = nullptr) const {
     return getIntImmCost(Imm, Ty, CostKind);
   }
 
   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                       const APInt &Imm, Type *Ty,
-                                      TTI::TargetCostKind CostKind) {
+                                      TTI::TargetCostKind CostKind) const {
     return getIntImmCost(Imm, Ty, CostKind);
   }
 
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
index cbc9c3f3beca0..ede5477f04bda 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
@@ -67,7 +67,7 @@ unsigned LoongArchTTIImpl::getRegisterClassForType(bool Vector,
   return LoongArchRegisterClass::GPRRC;
 }
 
-unsigned LoongArchTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
+unsigned LoongArchTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
   return ST->getMaxInterleaveFactor();
 }
 
@@ -84,7 +84,7 @@ const char *LoongArchTTIImpl::getRegisterClassName(unsigned ClassID) const {
 }
 
 TargetTransformInfo::PopcntSupportKind
-LoongArchTTIImpl::getPopcntSupport(unsigned TyWidth) {
+LoongArchTTIImpl::getPopcntSupport(unsigned TyWidth) const {
   assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
   return ST->hasExtLSX() ? TTI::PSK_FastHardware : TTI::PSK_Software;
 }
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
index b3edf131c584c..26d59c3e53a3e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
@@ -43,9 +43,9 @@ class LoongArchTTIImpl : public BasicTTIImplBase<LoongArchTTIImpl> {
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getNumberOfRegisters(unsigned ClassID) const;
   unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const;
-  unsigned getMaxInterleaveFactor(ElementCount VF);
+  unsigned getMaxInterleaveFactor(ElementCount VF) const;
   const char *getRegisterClassName(unsigned ClassID) const;
-  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
+  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const;
 
   unsigned getCacheLineSize() const override;
   unsigned getPrefetchDistance() const override;
diff --git a/llvm/lib/Target/Mips/MipsTargetTransformInfo.cpp b/llvm/lib/Target/Mips/MipsTargetTransformInfo.cpp
index 00dee75e70663..e861e43c56c75 100644
--- a/llvm/lib/Target/Mips/MipsTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsTargetTransformInfo.cpp
@@ -10,14 +10,14 @@
 
 using namespace llvm;
 
-bool MipsTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
+bool MipsTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) const {
   EVT VT = TLI->getValueType(DL, DataType);
   return TLI->isOperationLegalOrCustom(IsSigned ? ISD::SDIVREM : ISD::UDIVREM,
                                        VT);
 }
 
 bool MipsTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                                const TargetTransformInfo::LSRCost &C2) {
+                                const TargetTransformInfo::LSRCost &C2) const {
   // MIPS specific here are "instruction number 1st priority".
   // If we need to emit adds inside the loop to add up base registers, then
   // we need at least one extra temporary register.
diff --git a/llvm/lib/Target/Mips/MipsTargetTransformInfo.h b/llvm/lib/Target/Mips/MipsTargetTransformInfo.h
index f817a386e600e..cf3f873fd0cb0 100644
--- a/llvm/lib/Target/Mips/MipsTargetTransformInfo.h
+++ b/llvm/lib/Target/Mips/MipsTargetTransformInfo.h
@@ -32,10 +32,10 @@ class MipsTTIImpl : public BasicTTIImplBase<MipsTTIImpl> {
       : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
-  bool hasDivRemOp(Type *DataType, bool IsSigned);
+  bool hasDivRemOp(Type *DataType, bool IsSigned) const;
 
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                     const TargetTransformInfo::LSRCost &C2);
+                     const TargetTransformInfo::LSRCost &C2) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index b5f7d90cd29d9..5e5362b0461d0 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -74,7 +74,7 @@ static bool isNVVMAtomic(const IntrinsicInst *II) {
   }
 }
 
-bool NVPTXTTIImpl::isSourceOfDivergence(const Value *V) {
+bool NVPTXTTIImpl::isSourceOfDivergence(const Value *V) const {
   // Without inter-procedural analysis, we conservatively assume that arguments
   // to __device__ functions are divergent.
   if (const Argument *Arg = dyn_cast<Argument>(V))
@@ -483,7 +483,7 @@ NVPTXTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 InstructionCost
 NVPTXTTIImpl::getInstructionCost(const User *U,
                                  ArrayRef<const Value *> Operands,
-                                 TTI::TargetCostKind CostKind) {
+                                 TTI::TargetCostKind CostKind) const {
   if (const auto *CI = dyn_cast<CallInst>(U))
     if (const auto *IA = dyn_cast<InlineAsm>(CI->getCalledOperand())) {
       // Without this implementation getCallCost() would return the number
@@ -538,9 +538,9 @@ InstructionCost NVPTXTTIImpl::getArithmeticInstrCost(
   }
 }
 
-void NVPTXTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
-                                           TTI::UnrollingPreferences &UP,
-                                           OptimizationRemarkEmitter *ORE) {
+void NVPTXTTIImpl::getUnrollingPreferences(
+    Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
+    OptimizationRemarkEmitter *ORE) const {
   BaseT::getUnrollingPreferences(L, SE, UP, ORE);
 
   // Enable partial unrolling and runtime unrolling, but reduce the
@@ -552,7 +552,7 @@ void NVPTXTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 }
 
 void NVPTXTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                         TTI::PeelingPreferences &PP) {
+                                         TTI::PeelingPreferences &PP) const {
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index 9fd5c17f58959..ecd45a01ffbf2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -42,9 +42,9 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
       : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl()),
         TLI(ST->getTargetLowering()) {}
 
-  bool hasBranchDivergence(const Function *F = nullptr) { return true; }
+  bool hasBranchDivergence(const Function *F = nullptr) const { return true; }
 
-  bool isSourceOfDivergence(const Value *V);
+  bool isSourceOfDivergence(const Value *V) const;
 
   unsigned getFlatAddressSpace() const {
     return AddressSpace::ADDRESS_SPACE_GENERIC;
@@ -97,7 +97,7 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
 
   InstructionCost getInstructionCost(const User *U,
                                      ArrayRef<const Value *> Operands,
-                                     TTI::TargetCostKind CostKind);
+                                     TTI::TargetCostKind CostKind) const;
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
@@ -144,12 +144,12 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
 
-  bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) {
+  bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) const {
     // Volatile loads/stores are only supported for shared and global address
     // spaces, or for generic AS that maps to them.
     if (!(AddrSpace == llvm::ADDRESS_SPACE_GENERIC ||
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 77149045d6b72..762e2d4d28d27 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -53,7 +53,7 @@ SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(4), cl::Hidden,
 //===----------------------------------------------------------------------===//
 
 TargetTransformInfo::PopcntSupportKind
-PPCTTIImpl::getPopcntSupport(unsigned TyWidth) {
+PPCTTIImpl::getPopcntSupport(unsigned TyWidth) const {
   assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
   if (ST->hasPOPCNTD() != PPCSubtarget::POPCNTD_Unavailable && TyWidth <= 64)
     return ST->hasPOPCNTD() == PPCSubtarget::POPCNTD_Slow ?
@@ -190,9 +190,10 @@ InstructionCost PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
   return 4 * TTI::TCC_Basic;
 }
 
-InstructionCost PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
-                                                const APInt &Imm, Type *Ty,
-                                                TTI::TargetCostKind CostKind) {
+InstructionCost
+PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
+                                const APInt &Imm, Type *Ty,
+                                TTI::TargetCostKind CostKind) const {
   if (DisablePPCConstHoist)
     return BaseT::getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
 
@@ -228,7 +229,7 @@ InstructionCost PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
 InstructionCost PPCTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                               const APInt &Imm, Type *Ty,
                                               TTI::TargetCostKind CostKind,
-                                              Instruction *Inst) {
+                                              Instruction *Inst) const {
   if (DisablePPCConstHoist)
     return BaseT::getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst);
 
@@ -318,9 +319,9 @@ static bool isMMAType(Type *Ty) {
          (Ty->getPrimitiveSizeInBits() > 128);
 }
 
-InstructionCost PPCTTIImpl::getInstructionCost(const User *U,
-                                               ArrayRef<const Value *> Operands,
-                                               TTI::TargetCostKind CostKind) {
+InstructionCost
+PPCTTIImpl::getInstructionCost(const User *U, ArrayRef<const Value *> Operands,
+                               TTI::TargetCostKind CostKind) const {
   // We already implement getCastInstrCost and getMemoryOpCost where we perform
   // the vector adjustment there.
   if (isa<CastInst>(U) || isa<LoadInst>(U) || isa<StoreInst>(U))
@@ -338,7 +339,7 @@ InstructionCost PPCTTIImpl::getInstructionCost(const User *U,
 bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                           AssumptionCache &AC,
                                           TargetLibraryInfo *LibInfo,
-                                          HardwareLoopInfo &HWLoopInfo) {
+                                          HardwareLoopInfo &HWLoopInfo) const {
   const PPCTargetMachine &TM = ST->getTargetMachine();
   TargetSchedModel SchedModel;
   SchedModel.init(ST);
@@ -397,7 +398,7 @@ bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
 
 void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                          TTI::UnrollingPreferences &UP,
-                                         OptimizationRemarkEmitter *ORE) {
+                                         OptimizationRemarkEmitter *ORE) const {
   if (ST->getCPUDirective() == PPC::DIR_A2) {
     // The A2 is in-order with a deep pipeline, and concatenation unrolling
     // helps expose latency-hiding opportunities to the instruction scheduler.
@@ -412,18 +413,18 @@ void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 }
 
 void PPCTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                       TTI::PeelingPreferences &PP) {
+                                       TTI::PeelingPreferences &PP) const {
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 // This function returns true to allow using coldcc calling convention.
 // Returning true results in coldcc being used for functions which are cold at
 // all call sites when the callers of the functions are not calling any other
 // non coldcc functions.
-bool PPCTTIImpl::useColdCCForColdCall(Function &F) {
+bool PPCTTIImpl::useColdCCForColdCall(Function &F) const {
   return EnablePPCColdCC;
 }
 
-bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) {
+bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) const {
   // On the A2, always unroll aggressively.
   if (ST->getCPUDirective() == PPC::DIR_A2)
     return true;
@@ -439,9 +440,7 @@ PPCTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
   return Options;
 }
 
-bool PPCTTIImpl::enableInterleavedAccessVectorization() {
-  return true;
-}
+bool PPCTTIImpl::enableInterleavedAccessVectorization() const { return true; }
 
 unsigned PPCTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
   assert(ClassID == GPRRC || ClassID == FPRRC ||
@@ -512,7 +511,7 @@ unsigned PPCTTIImpl::getPrefetchDistance() const {
   return 300;
 }
 
-unsigned PPCTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
+unsigned PPCTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
   unsigned Directive = ST->getCPUDirective();
   // The 440 has no SIMD support, but floating-point instructions
   // have a 5-cycle latency, so unroll by 5x for latency hiding.
@@ -857,7 +856,7 @@ InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
 InstructionCost PPCTTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-    bool UseMaskForCond, bool UseMaskForGaps) {
+    bool UseMaskForCond, bool UseMaskForGaps) const {
   InstructionCost CostFactor =
       vectorCostAdjustmentFactor(Opcode, VecTy, nullptr);
   if (!CostFactor.isValid())
@@ -929,7 +928,8 @@ bool PPCTTIImpl::areTypesABICompatible(const Function *Caller,
 
 bool PPCTTIImpl::canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE,
                             LoopInfo *LI, DominatorTree *DT,
-                            AssumptionCache *AC, TargetLibraryInfo *LibInfo) {
+                            AssumptionCache *AC,
+                            TargetLibraryInfo *LibInfo) const {
   // Process nested loops first.
   for (Loop *I : *L)
     if (canSaveCmp(I, BI, SE, LI, DT, AC, LibInfo))
@@ -951,7 +951,7 @@ bool PPCTTIImpl::canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE,
 }
 
 bool PPCTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                               const TargetTransformInfo::LSRCost &C2) {
+                               const TargetTransformInfo::LSRCost &C2) const {
   // PowerPC default behaviour here is "instruction number 1st priority".
   // If LsrNoInsnsCost is set, call default implementation.
   if (!LsrNoInsnsCost)
@@ -962,9 +962,7 @@ bool PPCTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
   return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
 }
 
-bool PPCTTIImpl::isNumRegsMajorCostOfLSR() {
-  return false;
-}
+bool PPCTTIImpl::isNumRegsMajorCostOfLSR() const { return false; }
 
 bool PPCTTIImpl::shouldBuildRelLookupTables() const {
   const PPCTargetMachine &TM = ST->getTargetMachine();
@@ -975,7 +973,7 @@ bool PPCTTIImpl::shouldBuildRelLookupTables() const {
 }
 
 bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
-                                    MemIntrinsicInfo &Info) {
+                                    MemIntrinsicInfo &Info) const {
   switch (Inst->getIntrinsicID()) {
   case Intrinsic::ppc_altivec_lvx:
   case Intrinsic::ppc_altivec_lvxl:
@@ -1059,7 +1057,7 @@ InstructionCost PPCTTIImpl::getVPMemoryOpCost(unsigned Opcode, Type *Src,
                                               Align Alignment,
                                               unsigned AddressSpace,
                                               TTI::TargetCostKind CostKind,
-                                              const Instruction *I) {
+                                              const Instruction *I) const {
   InstructionCost Cost = BaseT::getVPMemoryOpCost(Opcode, Src, Alignment,
                                                   AddressSpace, CostKind, I);
   if (TLI->getValueType(DL, Src, true) == MVT::Other)
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index f81082d2d7029..9e4c40bb39295 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -53,42 +53,41 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,
-                                    Instruction *Inst = nullptr);
+                                    Instruction *Inst = nullptr) const;
   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                       const APInt &Imm, Type *Ty,
-                                      TTI::TargetCostKind CostKind);
+                                      TTI::TargetCostKind CostKind) const;
 
   InstructionCost getInstructionCost(const User *U,
                                      ArrayRef<const Value *> Operands,
-                                     TTI::TargetCostKind CostKind);
+                                     TTI::TargetCostKind CostKind) const;
 
-  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
+  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const;
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
-                                AssumptionCache &AC,
-                                TargetLibraryInfo *LibInfo,
-                                HardwareLoopInfo &HWLoopInfo);
+                                AssumptionCache &AC, TargetLibraryInfo *LibInfo,
+                                HardwareLoopInfo &HWLoopInfo) const;
   bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI,
                   DominatorTree *DT, AssumptionCache *AC,
-                  TargetLibraryInfo *LibInfo);
-  bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+                  TargetLibraryInfo *LibInfo) const;
+  bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                     const TargetTransformInfo::LSRCost &C2);
-  bool isNumRegsMajorCostOfLSR();
+                     const TargetTransformInfo::LSRCost &C2) const;
+  bool isNumRegsMajorCostOfLSR() const;
   bool shouldBuildRelLookupTables() const;
   /// @}
 
   /// \name Vector TTI Implementations
   /// @{
-  bool useColdCCForColdCall(Function &F);
-  bool enableAggressiveInterleaving(bool LoopHasReductions);
+  bool useColdCCForColdCall(Function &F) const;
+  bool enableAggressiveInterleaving(bool LoopHasReductions) const;
   TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize,
                                                     bool IsZeroCmp) const;
-  bool enableInterleavedAccessVectorization();
+  bool enableInterleavedAccessVectorization() const;
 
   enum PPCRegisterClass {
     GPRRC, FPRRC, VRRC, VSXRC
@@ -99,7 +98,7 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getCacheLineSize() const override;
   unsigned getPrefetchDistance() const override;
-  unsigned getMaxInterleaveFactor(ElementCount VF);
+  unsigned getMaxInterleaveFactor(ElementCount VF) const;
   InstructionCost vectorCostAdjustmentFactor(unsigned Opcode, Type *Ty1,
                                              Type *Ty2) const;
   InstructionCost getArithmeticInstrCost(
@@ -138,7 +137,7 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind) const;
   bool areInlineCompatible(const Function *Caller,
@@ -150,7 +149,7 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
   InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                     unsigned AddressSpace,
                                     TTI::TargetCostKind CostKind,
-                                    const Instruction *I = nullptr);
+                                    const Instruction *I = nullptr) const;
   bool supportsTailCallFor(const CallBase *CB) const;
 
 private:
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 6bedc43c92c0f..a6a14837bf473 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -133,8 +133,9 @@ static InstructionCost getIntImmCostImpl(const DataLayout &DL,
                                     /*CompressionCost=*/false, FreeZeroes);
 }
 
-InstructionCost RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
-                                            TTI::TargetCostKind CostKind) {
+InstructionCost
+RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
+                            TTI::TargetCostKind CostKind) const {
   return getIntImmCostImpl(getDataLayout(), getST(), Imm, Ty, CostKind, false);
 }
 
@@ -168,7 +169,7 @@ static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) {
 InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                                 const APInt &Imm, Type *Ty,
                                                 TTI::TargetCostKind CostKind,
-                                                Instruction *Inst) {
+                                                Instruction *Inst) const {
   assert(Ty->isIntegerTy() &&
          "getIntImmCost can only estimate cost of materialising integers");
 
@@ -276,7 +277,7 @@ InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
 InstructionCost
 RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                   const APInt &Imm, Type *Ty,
-                                  TTI::TargetCostKind CostKind) {
+                                  TTI::TargetCostKind CostKind) const {
   // Prevent hoisting in unknown cases.
   return TTI::TCC_Free;
 }
@@ -286,7 +287,7 @@ bool RISCVTTIImpl::hasActiveVectorLength(unsigned, Type *DataTy, Align) const {
 }
 
 TargetTransformInfo::PopcntSupportKind
-RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) {
+RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {
   assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
   return ST->hasStdExtZbb() || (ST->hasVendorXCVbitmanip() && !ST->is64Bit())
              ? TTI::PSK_FastHardware
@@ -888,7 +889,7 @@ RISCVTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
 InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-    bool UseMaskForCond, bool UseMaskForGaps) {
+    bool UseMaskForCond, bool UseMaskForGaps) const {
 
   // The interleaved memory access pass will lower interleaved memory ops (i.e
   // a load and store followed by a specific shuffle) to vlseg/vsseg
@@ -1069,7 +1070,7 @@ InstructionCost RISCVTTIImpl::getStridedMemoryOpCost(
 }
 
 InstructionCost
-RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) {
+RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
   // FIXME: This is a property of the default vector convention, not
   // all possible calling conventions.  Fixing that will require
   // some TTI API and SLP rework.
@@ -1899,7 +1900,7 @@ RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
 
 InstructionCost RISCVTTIImpl::getExtendedReductionCost(
     unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
-    std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) {
+    std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
   if (isa<FixedVectorType>(ValTy) && !ST->useRVVForFixedLengthVectors())
     return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,
                                            FMF, CostKind);
@@ -2444,7 +2445,7 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
 InstructionCost RISCVTTIImpl::getPointersChainCost(
     ArrayRef<const Value *> Ptrs, const Value *Base,
     const TTI::PointersChainInfo &Info, Type *AccessTy,
-    TTI::TargetCostKind CostKind) {
+    TTI::TargetCostKind CostKind) const {
   InstructionCost Cost = TTI::TCC_Free;
   // In the basic model we take into account GEP instructions only
   // (although here can come alloca instruction, a value, constants and/or
@@ -2488,9 +2489,9 @@ InstructionCost RISCVTTIImpl::getPointersChainCost(
   return Cost;
 }
 
-void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
-                                           TTI::UnrollingPreferences &UP,
-                                           OptimizationRemarkEmitter *ORE) {
+void RISCVTTIImpl::getUnrollingPreferences(
+    Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
+    OptimizationRemarkEmitter *ORE) const {
   // TODO: More tuning on benchmarks and metrics with changes as needed
   //       would apply to all settings below to enable performance.
 
@@ -2566,7 +2567,7 @@ void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 }
 
 void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                         TTI::PeelingPreferences &PP) {
+                                         TTI::PeelingPreferences &PP) const {
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 
@@ -2620,7 +2621,7 @@ RISCVTTIImpl::getPreferredAddressingMode(const Loop *L,
 }
 
 bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                                 const TargetTransformInfo::LSRCost &C2) {
+                                 const TargetTransformInfo::LSRCost &C2) const {
   // RISC-V specific here are "instruction number 1st priority".
   // If we need to emit adds inside the loop to add up base registers, then
   // we need at least one extra temporary register.
@@ -2669,7 +2670,7 @@ bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,
 /// sext instructions that sign extended the same initial value. A getelementptr
 /// is considered as "complex" if it has more than 2 operands.
 bool RISCVTTIImpl::shouldConsiderAddressTypePromotion(
-    const Instruction &I, bool &AllowPromotionWithoutCommonHeader) {
+    const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
   bool Considerable = false;
   AllowPromotionWithoutCommonHeader = false;
   if (!isa<SExtInst>(&I))
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index b72ffc2448313..21f00e80608b1 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -79,14 +79,14 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
                                   TTI::TargetCostKind CostKind) const;
 
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                TTI::TargetCostKind CostKind);
+                                TTI::TargetCostKind CostKind) const;
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,
-                                    Instruction *Inst = nullptr);
+                                    Instruction *Inst = nullptr) const;
   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                       const APInt &Imm, Type *Ty,
-                                      TTI::TargetCostKind CostKind);
+                                      TTI::TargetCostKind CostKind) const;
 
   /// \name EVL Support for predicated vectorization.
   /// Whether the target supports the %evl parameter of VP intrinsic efficiently
@@ -104,7 +104,8 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
                              Align Alignment) const;
 
-  TargetTransformInfo::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
+  TargetTransformInfo::PopcntSupportKind
+  getPopcntSupport(unsigned TyWidth) const;
 
   bool shouldExpandReduction(const IntrinsicInst *II) const;
   bool supportsScalableVectors() const { return ST->hasVInstructions(); }
@@ -141,14 +142,14 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
                                        const Value *Base,
                                        const TTI::PointersChainInfo &Info,
                                        Type *AccessTy,
-                                       TTI::TargetCostKind CostKind);
+                                       TTI::TargetCostKind CostKind) const;
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
 
   unsigned getMinVectorRegisterBitWidth() const {
     return ST->useRVVForFixedLengthVectors() ? 16 : 0;
@@ -173,7 +174,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
 
   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
@@ -192,7 +193,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
                                          TTI::TargetCostKind CostKind,
                                          const Instruction *I) const;
 
-  InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys);
+  InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const;
 
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
@@ -211,7 +212,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   InstructionCost getExtendedReductionCost(unsigned Opcode, bool IsUnsigned,
                                            Type *ResTy, VectorType *ValTy,
                                            std::optional<FastMathFlags> FMF,
-                                           TTI::TargetCostKind CostKind);
+                                           TTI::TargetCostKind CostKind) const;
 
   InstructionCost getMemoryOpCost(
       unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
@@ -264,11 +265,11 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   }
 
   bool isLegalMaskedLoad(Type *DataType, Align Alignment,
-                         unsigned /*AddressSpace*/) {
+                         unsigned /*AddressSpace*/) const {
     return isLegalMaskedLoadStore(DataType, Alignment);
   }
   bool isLegalMaskedStore(Type *DataType, Align Alignment,
-                          unsigned /*AddressSpace*/) {
+                          unsigned /*AddressSpace*/) const {
     return isLegalMaskedLoadStore(DataType, Alignment);
   }
 
@@ -302,12 +303,12 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
     return isLegalMaskedGatherScatter(DataType, Alignment);
   }
 
-  bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) {
+  bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) const {
     // Scalarize masked gather for RV64 if EEW=64 indices aren't supported.
     return ST->is64Bit() && !ST->hasVInstructionsI64();
   }
 
-  bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) {
+  bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) const {
     // Scalarize masked scatter for RV64 if EEW=64 indices aren't supported.
     return ST->is64Bit() && !ST->hasVInstructionsI64();
   }
@@ -318,7 +319,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   }
 
   bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
-                                    Align Alignment, unsigned AddrSpace) {
+                                    Align Alignment, unsigned AddrSpace) const {
     return TLI->isLegalInterleavedAccessType(VTy, Factor, Alignment, AddrSpace,
                                              DL);
   }
@@ -380,7 +381,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
     }
   }
 
-  unsigned getMaxInterleaveFactor(ElementCount VF) {
+  unsigned getMaxInterleaveFactor(ElementCount VF) const {
     // Don't interleave if the loop has been vectorized with scalable vectors.
     if (VF.isScalable())
       return 1;
@@ -389,7 +390,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
     return VF.isScalar() ? 1 : ST->getMaxInterleaveFactor();
   }
 
-  bool enableInterleavedAccessVectorization() { return true; }
+  bool enableInterleavedAccessVectorization() const { return true; }
 
   unsigned getMinTripCountTailFoldingThreshold() const;
 
@@ -446,11 +447,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   }
 
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                     const TargetTransformInfo::LSRCost &C2);
+                     const TargetTransformInfo::LSRCost &C2) const;
 
-  bool
-  shouldConsiderAddressTypePromotion(const Instruction &I,
-                                     bool &AllowPromotionWithoutCommonHeader);
+  bool shouldConsiderAddressTypePromotion(
+      const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const;
   std::optional<unsigned> getMinPageSize() const { return 4096; }
   /// Return true if the (vector) instruction I will be lowered to an
   /// instruction with a scalar splat operand for the given Operand number.
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h b/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
index 24047f31fab29..5c919c2dd4024 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
@@ -39,7 +39,7 @@ class SPIRVTTIImpl : public BasicTTIImplBase<SPIRVTTIImpl> {
       : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
-  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) {
+  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const {
     // SPIR-V natively supports OpBitcount, per 3.53.14 in the spec, as such it
     // is reasonable to assume the Op is fast / preferable to the expanded loop.
     // Furthermore, this prevents information being lost if transforms are
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 3811657aed83e..bfc50a02558cf 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -136,8 +136,9 @@ unsigned SystemZTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
   return Bonus;
 }
 
-InstructionCost SystemZTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
-                                              TTI::TargetCostKind CostKind) {
+InstructionCost
+SystemZTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
+                              TTI::TargetCostKind CostKind) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -173,7 +174,7 @@ InstructionCost SystemZTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
 InstructionCost SystemZTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                                   const APInt &Imm, Type *Ty,
                                                   TTI::TargetCostKind CostKind,
-                                                  Instruction *Inst) {
+                                                  Instruction *Inst) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -293,7 +294,7 @@ InstructionCost SystemZTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
 InstructionCost
 SystemZTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
-                                    TTI::TargetCostKind CostKind) {
+                                    TTI::TargetCostKind CostKind) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -342,16 +343,16 @@ SystemZTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
 }
 
 TargetTransformInfo::PopcntSupportKind
-SystemZTTIImpl::getPopcntSupport(unsigned TyWidth) {
+SystemZTTIImpl::getPopcntSupport(unsigned TyWidth) const {
   assert(isPowerOf2_32(TyWidth) && "Type width must be power of 2");
   if (ST->hasPopulationCount() && TyWidth <= 64)
     return TTI::PSK_FastHardware;
   return TTI::PSK_Software;
 }
 
-void SystemZTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
-                                             TTI::UnrollingPreferences &UP,
-                                             OptimizationRemarkEmitter *ORE) {
+void SystemZTTIImpl::getUnrollingPreferences(
+    Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
+    OptimizationRemarkEmitter *ORE) const {
   // Find out if L contains a call, what the machine instruction count
   // estimate is, and how many stores there are.
   bool HasCall = false;
@@ -406,12 +407,13 @@ void SystemZTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 }
 
 void SystemZTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                           TTI::PeelingPreferences &PP) {
+                                           TTI::PeelingPreferences &PP) const {
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 
-bool SystemZTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                                   const TargetTransformInfo::LSRCost &C2) {
+bool SystemZTTIImpl::isLSRCostLess(
+    const TargetTransformInfo::LSRCost &C1,
+    const TargetTransformInfo::LSRCost &C2) const {
   // SystemZ specific: check instruction count (first), and don't care about
   // ImmCost, since offsets are checked explicitly.
   return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost,
@@ -478,7 +480,7 @@ unsigned SystemZTTIImpl::getMinPrefetchStride(unsigned NumMemAccesses,
   return ST->hasMiscellaneousExtensions3() ? 8192 : 2048;
 }
 
-bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
+bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) const {
   EVT VT = TLI->getValueType(DL, DataType);
   return (VT.isScalarInteger() && TLI->isTypeLegal(VT));
 }
@@ -1375,7 +1377,7 @@ InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
 InstructionCost SystemZTTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-    bool UseMaskForCond, bool UseMaskForGaps) {
+    bool UseMaskForCond, bool UseMaskForGaps) const {
   if (UseMaskForCond || UseMaskForGaps)
     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
                                              Alignment, AddressSpace, CostKind,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 16a750d561f56..99faab7194892 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -43,27 +43,27 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   unsigned adjustInliningThreshold(const CallBase *CB) const;
 
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                TTI::TargetCostKind CostKind);
+                                TTI::TargetCostKind CostKind) const;
 
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,
-                                    Instruction *Inst = nullptr);
+                                    Instruction *Inst = nullptr) const;
   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                       const APInt &Imm, Type *Ty,
-                                      TTI::TargetCostKind CostKind);
+                                      TTI::TargetCostKind CostKind) const;
 
-  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
+  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const;
 
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                TTI::UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE);
+                               OptimizationRemarkEmitter *ORE) const;
 
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             TTI::PeelingPreferences &PP);
+                             TTI::PeelingPreferences &PP) const;
 
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                     const TargetTransformInfo::LSRCost &C2);
+                     const TargetTransformInfo::LSRCost &C2) const;
 
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
@@ -84,16 +84,16 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
                                 bool HasCall) const override;
   bool enableWritePrefetching() const override { return true; }
 
-  bool hasDivRemOp(Type *DataType, bool IsSigned);
-  bool prefersVectorizedAddressing() { return false; }
-  bool LSRWithInstrQueries() { return true; }
+  bool hasDivRemOp(Type *DataType, bool IsSigned) const;
+  bool prefersVectorizedAddressing() const { return false; }
+  bool LSRWithInstrQueries() const { return true; }
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
                                            TTI::TargetCostKind CostKind,
                                            ArrayRef<Value *> VL = {}) const;
-  bool supportsEfficientVectorElementLoadStore() { return true; }
-  bool enableInterleavedAccessVectorization() { return true; }
+  bool supportsEfficientVectorElementLoadStore() const { return true; }
+  bool enableInterleavedAccessVectorization() const { return true; }
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
@@ -137,7 +137,7 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
 
   InstructionCost
   getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
diff --git a/llvm/lib/Target/VE/VETargetTransformInfo.h b/llvm/lib/Target/VE/VETargetTransformInfo.h
index f0fa01ef22912..88b46ff128383 100644
--- a/llvm/lib/Target/VE/VETargetTransformInfo.h
+++ b/llvm/lib/Target/VE/VETargetTransformInfo.h
@@ -134,17 +134,17 @@ class VETTIImpl : public BasicTTIImplBase<VETTIImpl> {
 
   // Load & Store {
   bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment,
-                         unsigned /*AddressSpace*/) {
+                         unsigned /*AddressSpace*/) const {
     return isVectorLaneType(*getLaneType(DataType));
   }
   bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment,
-                          unsigned /*AddressSpace*/) {
+                          unsigned /*AddressSpace*/) const {
     return isVectorLaneType(*getLaneType(DataType));
   }
-  bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) {
+  bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) const {
     return isVectorLaneType(*getLaneType(DataType));
   };
-  bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) {
+  bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) const {
     return isVectorLaneType(*getLaneType(DataType));
   }
   // } Load & Store
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index dd0e672a2a21c..b8b6051ec5f28 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -57,7 +57,7 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
   /// \name Vector TTI Implementations
   /// @{
 
-  bool enableInterleavedAccessVectorization() { return true; }
+  bool enableInterleavedAccessVectorization() const { return true; }
 
   unsigned getNumberOfRegisters(unsigned ClassID) const;
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 59bcd3b921799..363019986d76c 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -102,7 +102,7 @@ using CostKindTblEntry = CostTblEntryT<CostKindCosts>;
 using TypeConversionCostKindTblEntry = TypeConversionCostTblEntryT<CostKindCosts>;
 
 TargetTransformInfo::PopcntSupportKind
-X86TTIImpl::getPopcntSupport(unsigned TyWidth) {
+X86TTIImpl::getPopcntSupport(unsigned TyWidth) const {
   assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
   // TODO: Currently the __builtin_popcount() implementation using SSE3
   //   instructions is inefficient. Once the problem is fixed, we should
@@ -225,7 +225,7 @@ unsigned X86TTIImpl::getLoadStoreVecRegBitWidth(unsigned) const {
       .getFixedValue();
 }
 
-unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF) {
+unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
   // If the loop will not be vectorized, don't interleave the loop.
   // Let regular unroll to unroll the loop, which saves the overflow
   // check and memory check cost.
@@ -5420,11 +5420,10 @@ X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy, Align Alignment,
   return Cost + LT.first;
 }
 
-InstructionCost
-X86TTIImpl::getPointersChainCost(ArrayRef<const Value *> Ptrs,
-                                 const Value *Base,
-                                 const TTI::PointersChainInfo &Info,
-                                 Type *AccessTy, TTI::TargetCostKind CostKind) {
+InstructionCost X86TTIImpl::getPointersChainCost(
+    ArrayRef<const Value *> Ptrs, const Value *Base,
+    const TTI::PointersChainInfo &Info, Type *AccessTy,
+    TTI::TargetCostKind CostKind) const {
   if (Info.isSameBase() && Info.isKnownStride()) {
     // If all the pointers have known stride all the differences are translated
     // into constants. X86 memory addressing allows encoding it into
@@ -5442,7 +5441,7 @@ X86TTIImpl::getPointersChainCost(ArrayRef<const Value *> Ptrs,
 
 InstructionCost X86TTIImpl::getAddressComputationCost(Type *Ty,
                                                       ScalarEvolution *SE,
-                                                      const SCEV *Ptr) {
+                                                      const SCEV *Ptr) const {
   // Address computations in vectorized code with non-consecutive addresses will
   // likely result in more instructions compared to scalar code where the
   // computation can more often be merged into the index mode. The resulting
@@ -5900,7 +5899,7 @@ X86TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *ValTy,
 /// Calculate the cost of materializing a 64-bit value. This helper
 /// method might only calculate a fraction of a larger immediate. Therefore it
 /// is valid to return a cost of ZERO.
-InstructionCost X86TTIImpl::getIntImmCost(int64_t Val) {
+InstructionCost X86TTIImpl::getIntImmCost(int64_t Val) const {
   if (Val == 0)
     return TTI::TCC_Free;
 
@@ -5911,7 +5910,7 @@ InstructionCost X86TTIImpl::getIntImmCost(int64_t Val) {
 }
 
 InstructionCost X86TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
-                                          TTI::TargetCostKind CostKind) {
+                                          TTI::TargetCostKind CostKind) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -5948,7 +5947,7 @@ InstructionCost X86TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
 InstructionCost X86TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                               const APInt &Imm, Type *Ty,
                                               TTI::TargetCostKind CostKind,
-                                              Instruction *Inst) {
+                                              Instruction *Inst) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -6050,9 +6049,10 @@ InstructionCost X86TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
   return X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
 }
 
-InstructionCost X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
-                                                const APInt &Imm, Type *Ty,
-                                                TTI::TargetCostKind CostKind) {
+InstructionCost
+X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
+                                const APInt &Imm, Type *Ty,
+                                TTI::TargetCostKind CostKind) const {
   assert(Ty->isIntegerTy());
 
   unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -6215,17 +6215,15 @@ InstructionCost X86TTIImpl::getGatherScatterOpCost(
 }
 
 bool X86TTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                               const TargetTransformInfo::LSRCost &C2) {
-    // X86 specific here are "instruction number 1st priority".
-    return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost,
-                    C1.NumIVMuls, C1.NumBaseAdds,
-                    C1.ScaleCost, C1.ImmCost, C1.SetupCost) <
-           std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost,
-                    C2.NumIVMuls, C2.NumBaseAdds,
-                    C2.ScaleCost, C2.ImmCost, C2.SetupCost);
+                               const TargetTransformInfo::LSRCost &C2) const {
+  // X86 specific here are "instruction number 1st priority".
+  return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost, C1.NumIVMuls,
+                  C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) <
+         std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost, C2.NumIVMuls,
+                  C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost);
 }
 
-bool X86TTIImpl::canMacroFuseCmp() {
+bool X86TTIImpl::canMacroFuseCmp() const {
   return ST->hasMacroFusion() || ST->hasBranchFusion();
 }
 
@@ -6279,7 +6277,7 @@ bool X86TTIImpl::isLegalMaskedStore(Type *DataTy, Align Alignment,
   return isLegalMaskedLoadStore(ScalarTy, ST);
 }
 
-bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) {
+bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) const {
   unsigned DataSize = DL.getTypeStoreSize(DataType);
   // The only supported nontemporal loads are for aligned vectors of 16 or 32
   // bytes.  Note that 32-byte nontemporal vector loads are supported by AVX2
@@ -6290,7 +6288,7 @@ bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) {
   return false;
 }
 
-bool X86TTIImpl::isLegalNTStore(Type *DataType, Align Alignment) {
+bool X86TTIImpl::isLegalNTStore(Type *DataType, Align Alignment) const {
   unsigned DataSize = DL.getTypeStoreSize(DataType);
 
   // SSE4A supports nontemporal stores of float and double at arbitrary
@@ -6322,7 +6320,7 @@ bool X86TTIImpl::isLegalBroadcastLoad(Type *ElementTy,
          ElementTy == Type::getDoubleTy(ElementTy->getContext());
 }
 
-bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) {
+bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) const {
   if (!isa<VectorType>(DataTy))
     return false;
 
@@ -6346,7 +6344,8 @@ bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) {
          ((IntWidth == 8 || IntWidth == 16) && ST->hasVBMI2());
 }
 
-bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy, Align Alignment) {
+bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy,
+                                            Align Alignment) const {
   return isLegalMaskedExpandLoad(DataTy, Alignment);
 }
 
@@ -6432,12 +6431,12 @@ bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) const {
   return isLegalMaskedGatherScatter(DataType, Alignment);
 }
 
-bool X86TTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
+bool X86TTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) const {
   EVT VT = TLI->getValueType(DL, DataType);
   return TLI->isOperationLegal(IsSigned ? ISD::SDIVREM : ISD::UDIVREM, VT);
 }
 
-bool X86TTIImpl::isExpensiveToSpeculativelyExecute(const Instruction* I) {
+bool X86TTIImpl::isExpensiveToSpeculativelyExecute(const Instruction *I) const {
   // FDIV is always expensive, even if it has a very low uop count.
   // TODO: Still necessary for recent CPUs with low latency/throughput fdiv?
   if (I->getOpcode() == Instruction::FDiv)
@@ -6446,9 +6445,7 @@ bool X86TTIImpl::isExpensiveToSpeculativelyExecute(const Instruction* I) {
   return BaseT::isExpensiveToSpeculativelyExecute(I);
 }
 
-bool X86TTIImpl::isFCmpOrdCheaperThanFCmpZero(Type *Ty) {
-  return false;
-}
+bool X86TTIImpl::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { return false; }
 
 bool X86TTIImpl::areInlineCompatible(const Function *Caller,
                                      const Function *Callee) const {
@@ -6563,7 +6560,7 @@ bool X86TTIImpl::supportsEfficientVectorElementLoadStore() const {
   return false;
 }
 
-bool X86TTIImpl::enableInterleavedAccessVectorization() {
+bool X86TTIImpl::enableInterleavedAccessVectorization() const {
   // TODO: We expect this to be beneficial regardless of arch,
   // but there are currently some unexplained performance artifacts on Atom.
   // As a temporary solution, disable on Atom.
@@ -6577,7 +6574,8 @@ bool X86TTIImpl::enableInterleavedAccessVectorization() {
 InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX512(
     unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
     ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
-    TTI::TargetCostKind CostKind, bool UseMaskForCond, bool UseMaskForGaps) {
+    TTI::TargetCostKind CostKind, bool UseMaskForCond,
+    bool UseMaskForGaps) const {
   // VecTy for interleave memop is <VF*Factor x Elt>.
   // So, for VF=4, Interleave Factor = 3, Element type = i32 we have
   // VecTy = <12 x i32>.
@@ -6731,7 +6729,7 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX512(
 InstructionCost X86TTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *BaseTy, unsigned Factor, ArrayRef<unsigned> Indices,
     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-    bool UseMaskForCond, bool UseMaskForGaps) {
+    bool UseMaskForCond, bool UseMaskForGaps) const {
   auto *VecTy = cast<FixedVectorType>(BaseTy);
 
   auto isSupportedOnAVX512 = [&](Type *VecTy) {
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index f93b97e94550f..1534301667ff1 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -116,7 +116,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
 
   /// \name Scalar TTI Implementations
   /// @{
-  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
+  TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const;
 
   /// @}
 
@@ -135,7 +135,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   bool hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const;
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AS) const;
-  unsigned getMaxInterleaveFactor(ElementCount VF);
+  unsigned getMaxInterleaveFactor(ElementCount VF) const;
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
@@ -194,9 +194,9 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                                        const Value *Base,
                                        const TTI::PointersChainInfo &Info,
                                        Type *AccessTy,
-                                       TTI::TargetCostKind CostKind);
+                                       TTI::TargetCostKind CostKind) const;
   InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
-                                            const SCEV *Ptr);
+                                            const SCEV *Ptr) const;
 
   std::optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
                                                     IntrinsicInst &II) const;
@@ -231,17 +231,17 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false);
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
   InstructionCost getInterleavedMemoryOpCostAVX512(
       unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
       ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind, bool UseMaskForCond = false,
-      bool UseMaskForGaps = false);
+      bool UseMaskForGaps = false) const;
 
-  InstructionCost getIntImmCost(int64_t);
+  InstructionCost getIntImmCost(int64_t) const;
 
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                TTI::TargetCostKind CostKind);
+                                TTI::TargetCostKind CostKind) const;
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
                                  const Instruction *I = nullptr) const;
@@ -249,10 +249,10 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
                                     const APInt &Imm, Type *Ty,
                                     TTI::TargetCostKind CostKind,
-                                    Instruction *Inst = nullptr);
+                                    Instruction *Inst = nullptr) const;
   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                       const APInt &Imm, Type *Ty,
-                                      TTI::TargetCostKind CostKind);
+                                      TTI::TargetCostKind CostKind) const;
   /// Return the cost of the scaling factor used in the addressing
   /// mode represented by AM for this target, for a load/store
   /// of the specified type.
@@ -263,14 +263,14 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                                        int64_t Scale, unsigned AddrSpace) const;
 
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                     const TargetTransformInfo::LSRCost &C2);
-  bool canMacroFuseCmp();
+                     const TargetTransformInfo::LSRCost &C2) const;
+  bool canMacroFuseCmp() const;
   bool isLegalMaskedLoad(Type *DataType, Align Alignment,
                          unsigned AddressSpace) const;
   bool isLegalMaskedStore(Type *DataType, Align Alignment,
                           unsigned AddressSpace) const;
-  bool isLegalNTLoad(Type *DataType, Align Alignment);
-  bool isLegalNTStore(Type *DataType, Align Alignment);
+  bool isLegalNTLoad(Type *DataType, Align Alignment) const;
+  bool isLegalNTStore(Type *DataType, Align Alignment) const;
   bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const;
   bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) const;
   bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) const {
@@ -279,13 +279,13 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   bool isLegalMaskedGatherScatter(Type *DataType, Align Alignment) const;
   bool isLegalMaskedGather(Type *DataType, Align Alignment) const;
   bool isLegalMaskedScatter(Type *DataType, Align Alignment) const;
-  bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment);
-  bool isLegalMaskedCompressStore(Type *DataType, Align Alignment);
+  bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) const;
+  bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) const;
   bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
                        const SmallBitVector &OpcodeMask) const;
-  bool hasDivRemOp(Type *DataType, bool IsSigned);
-  bool isExpensiveToSpeculativelyExecute(const Instruction *I);
-  bool isFCmpOrdCheaperThanFCmpZero(Type *Ty);
+  bool hasDivRemOp(Type *DataType, bool IsSigned) const;
+  bool isExpensiveToSpeculativelyExecute(const Instruction *I) const;
+  bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const;
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
   bool areTypesABICompatible(const Function *Caller, const Function *Callee,
@@ -300,7 +300,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   bool preferAlternateOpcodeVectorization() const { return false; }
   bool prefersVectorizedAddressing() const;
   bool supportsEfficientVectorElementLoadStore() const;
-  bool enableInterleavedAccessVectorization();
+  bool enableInterleavedAccessVectorization() const;
 
   InstructionCost getBranchMispredictPenalty() const;
 

From 9e26c797ae9699fba8bd62af99ab3ad6f9b16462 Mon Sep 17 00:00:00 2001
From: Liao Chunyu <chunyu@iscas.ac.cn>
Date: Tue, 22 Apr 2025 12:34:39 +0800
Subject: [PATCH 682/710] [RISCV] Add smcntrpmf extension (#136556)

spec: https://github.com/riscvarchive/riscv-smcntrpmf
---
 .../Driver/print-supported-extensions-riscv.c |  1 +
 .../test/Preprocessor/riscv-target-features.c |  9 +++++++
 llvm/docs/RISCVUsage.rst                      |  1 +
 llvm/lib/Target/RISCV/RISCVFeatures.td        |  3 +++
 llvm/lib/Target/RISCV/RISCVSystemOperands.td  | 10 ++++++++
 llvm/test/CodeGen/RISCV/attributes.ll         |  4 ++++
 llvm/test/CodeGen/RISCV/features-info.ll      |  1 +
 llvm/test/MC/RISCV/attribute-arch.s           |  3 +++
 llvm/test/MC/RISCV/machine-csr-names.s        | 24 +++++++++++++++++++
 llvm/test/MC/RISCV/rv32-machine-csr-names.s   | 24 +++++++++++++++++++
 .../TargetParser/RISCVISAInfoTest.cpp         |  1 +
 11 files changed, 81 insertions(+)

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 39002d7b4780a..1f83910969221 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -123,6 +123,7 @@
 // CHECK-NEXT:     shvstvecd            1.0       'Shvstvecd' (vstvec supports Direct mode)
 // CHECK-NEXT:     smaia                1.0       'Smaia' (Advanced Interrupt Architecture Machine Level)
 // CHECK-NEXT:     smcdeleg             1.0       'Smcdeleg' (Counter Delegation Machine Level)
+// CHECK-NEXT:     smcntrpmf            1.0       'Smcntrpmf' (Cycle and Instret Privilege Mode Filtering)
 // CHECK-NEXT:     smcsrind             1.0       'Smcsrind' (Indirect CSR Access Machine Level)
 // CHECK-NEXT:     smdbltrp             1.0       'Smdbltrp' (Double Trap Machine Level)
 // CHECK-NEXT:     smepmp               1.0       'Smepmp' (Enhanced Physical Memory Protection)
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 253e42419f453..03c291afe19bd 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -29,6 +29,7 @@
 // CHECK-NOT: __riscv_shvstvecd {{.*$}}
 // CHECK-NOT: __riscv_smaia {{.*$}}
 // CHECK-NOT: __riscv_smcdeleg {{.*$}}
+// CHECK-NOT: __riscv_smcntrpmf {{.*$}}
 // CHECK-NOT: __riscv_smcsrind {{.*$}}
 // CHECK-NOT: __riscv_smdbltrp {{.*$}}
 // CHECK-NOT: __riscv_smepmp {{.*$}}
@@ -1445,6 +1446,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SSAIA-EXT %s
 // CHECK-SSAIA-EXT: __riscv_ssaia  1000000{{$}}
 
+// RUN: %clang --target=riscv32 \
+// RUN:   -march=rv32ismcntrpmf1p0 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-SMCNTRPMF-EXT %s
+// RUN: %clang --target=riscv64 \
+// RUN:   -march=rv64ismcntrpmf1p0 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-SMCNTRPMF-EXT %s
+// CHECK-SMCNTRPMF-EXT: __riscv_smcntrpmf  1000000{{$}}
+
 // RUN: %clang --target=riscv32 \
 // RUN:   -march=rv32ismcsrind1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-SMCSRIND-EXT %s
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 137b537f00ea0..1ebe7b57abd7d 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -128,6 +128,7 @@ on support follow.
      ``Shvstvecd``     Assembly Support (`See note <#riscv-profiles-extensions-note>`__)
      ``Smaia``         Supported
      ``Smcdeleg``      Supported
+     ``Smcntrpmf``     Supported
      ``Smcsrind``      Supported
      ``Smdbltrp``      Supported
      ``Smepmp``        Supported
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index f51fcf82077f4..87cab1dafc75b 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -899,6 +899,9 @@ def FeatureStdExtSsaia
     : RISCVExtension<1, 0,
                      "Advanced Interrupt Architecture Supervisor Level">;
 
+def FeatureStdExtSmcntrpmf
+    : RISCVExtension<1, 0, "Cycle and Instret Privilege Mode Filtering">;
+
 def FeatureStdExtSmcsrind
     : RISCVExtension<1, 0, "Indirect CSR Access Machine Level">;
 def FeatureStdExtSscsrind
diff --git a/llvm/lib/Target/RISCV/RISCVSystemOperands.td b/llvm/lib/Target/RISCV/RISCVSystemOperands.td
index 23388c7575e86..79ec8134733e8 100644
--- a/llvm/lib/Target/RISCV/RISCVSystemOperands.td
+++ b/llvm/lib/Target/RISCV/RISCVSystemOperands.td
@@ -482,6 +482,16 @@ def : SysReg<"sctrdepth", 0x15f>;
 def : SysReg<"vsctrctl", 0x24e>;
 def : SysReg<"mctrctl", 0x34e>;
 
+//===-----------------------------------------------
+// Cycle and Instret Privilege Mode Filtering (Smcntrpmf)
+//===-----------------------------------------------
+def : SysReg<"mcyclecfg", 0x321>;
+def : SysReg<"minstretcfg", 0x322>;
+let isRV32Only = 1 in {
+def : SysReg<"mcyclecfgh", 0x721>;
+def : SysReg<"minstretcfgh", 0x722>;
+} // isRV32Only
+
 //===-----------------------------------------------
 // Vendor CSRs
 //===-----------------------------------------------
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 4bb3eb81f3dfb..b0dc65839559a 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -147,6 +147,7 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+ssdbltrp %s -o - | FileCheck --check-prefixes=CHECK,RV32SSDBLTRP %s
 ; RUN: llc -mtriple=riscv32 -mattr=+ssqosid %s -o - | FileCheck --check-prefix=RV32SSQOSID %s
 ; RUN: llc -mtriple=riscv32 -mattr=+smcdeleg %s -o - | FileCheck --check-prefixes=CHECK,RV32SMCDELEG %s
+; RUN: llc -mtriple=riscv32 -mattr=+smcntrpmf %s -o - | FileCheck --check-prefixes=CHECK,RV32SMCNTRPMF %s
 ; RUN: llc -mtriple=riscv32 -mattr=+smepmp %s -o - | FileCheck --check-prefixes=CHECK,RV32SMEPMP %s
 ; RUN: llc -mtriple=riscv32 -mattr=+smrnmi %s -o - | FileCheck --check-prefixes=CHECK,RV32SMRNMI %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zfbfmin %s -o - | FileCheck --check-prefixes=CHECK,RV32ZFBFMIN %s
@@ -301,6 +302,7 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+ssdbltrp %s -o - | FileCheck --check-prefixes=CHECK,RV64SSDBLTRP %s
 ; RUN: llc -mtriple=riscv64 -mattr=+ssqosid %s -o - | FileCheck --check-prefix=RV64SSQOSID %s
 ; RUN: llc -mtriple=riscv64 -mattr=+smcdeleg %s -o - | FileCheck --check-prefixes=CHECK,RV64SMCDELEG %s
+; RUN: llc -mtriple=riscv64 -mattr=+smcntrpmf %s -o - | FileCheck --check-prefixes=CHECK,RV64SMCNTRPMF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+smepmp %s -o - | FileCheck --check-prefixes=CHECK,RV64SMEPMP %s
 ; RUN: llc -mtriple=riscv64 -mattr=+smrnmi %s -o - | FileCheck --check-prefixes=CHECK,RV64SMRNMI %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zfbfmin %s -o - | FileCheck --check-prefixes=CHECK,RV64ZFBFMIN %s
@@ -485,6 +487,7 @@
 ; RV32SSDBLTRP: .attribute 5, "rv32i2p1_ssdbltrp1p0"
 ; RV32SSQOSID: .attribute 5, "rv32i2p1_ssqosid1p0"
 ; RV32SMCDELEG: .attribute 5, "rv32i2p1_smcdeleg1p0"
+; RV32SMCNTRPMF: .attribute 5, "rv32i2p1_smcntrpmf1p0"
 ; RV32SMEPMP: .attribute 5, "rv32i2p1_smepmp1p0"
 ; RV32SMRNMI: .attribute 5, "rv32i2p1_smrnmi1p0"
 ; RV32ZFBFMIN: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfbfmin1p0"
@@ -636,6 +639,7 @@
 ; RV64SSDBLTRP: .attribute 5, "rv64i2p1_ssdbltrp1p0"
 ; RV64SSQOSID: .attribute 5, "rv64i2p1_ssqosid1p0"
 ; RV64SMCDELEG: .attribute 5, "rv64i2p1_smcdeleg1p0"
+; RV64SMCNTRPMF: .attribute 5, "rv64i2p1_smcntrpmf1p0"
 ; RV64SMEPMP: .attribute 5, "rv64i2p1_smepmp1p0"
 ; RV64SMRNMI: .attribute 5, "rv64i2p1_smrnmi1p0"
 ; RV64ZFBFMIN: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfbfmin1p0"
diff --git a/llvm/test/CodeGen/RISCV/features-info.ll b/llvm/test/CodeGen/RISCV/features-info.ll
index d377bda059d33..f7824ec10834c 100644
--- a/llvm/test/CodeGen/RISCV/features-info.ll
+++ b/llvm/test/CodeGen/RISCV/features-info.ll
@@ -129,6 +129,7 @@
 ; CHECK-NEXT:   sifive7                          - SiFive 7-Series processors.
 ; CHECK-NEXT:   smaia                            - 'Smaia' (Advanced Interrupt Architecture Machine Level).
 ; CHECK-NEXT:   smcdeleg                         - 'Smcdeleg' (Counter Delegation Machine Level).
+; CHECK-NEXT:   smcntrpmf                        - 'Smcntrpmf' (Cycle and Instret Privilege Mode Filtering).
 ; CHECK-NEXT:   smcsrind                         - 'Smcsrind' (Indirect CSR Access Machine Level).
 ; CHECK-NEXT:   smdbltrp                         - 'Smdbltrp' (Double Trap Machine Level).
 ; CHECK-NEXT:   smepmp                           - 'Smepmp' (Enhanced Physical Memory Protection).
diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s
index 8edd6800a125f..16acd403eb9af 100644
--- a/llvm/test/MC/RISCV/attribute-arch.s
+++ b/llvm/test/MC/RISCV/attribute-arch.s
@@ -336,6 +336,9 @@
 .attribute arch, "rv32i_smcdeleg1p0"
 # CHECK: attribute      5, "rv32i2p1_smcdeleg1p0"
 
+.attribute arch, "rv32i_smcntrpmf1p0"
+# CHECK: attribute      5, "rv32i2p1_smcntrpmf1p0"
+
 .attribute arch, "rv32i_smepmp1p0"
 # CHECK: attribute      5, "rv32i2p1_smepmp1p0"
 
diff --git a/llvm/test/MC/RISCV/machine-csr-names.s b/llvm/test/MC/RISCV/machine-csr-names.s
index 0a76636f060e6..016c14d6f840b 100644
--- a/llvm/test/MC/RISCV/machine-csr-names.s
+++ b/llvm/test/MC/RISCV/machine-csr-names.s
@@ -2733,3 +2733,27 @@ csrrs t2, 0x742, zero
 csrrs t1, mnstatus, zero
 # uimm12
 csrrs t2, 0x744, zero
+
+# mcyclecfg
+# name
+# CHECK-INST: csrrs t1, mcyclecfg, zero
+# CHECK-ENC:  encoding: [0x73,0x23,0x10,0x32]
+# CHECK-INST-ALIAS: csrr t1, mcyclecfg
+csrrs t1, mcyclecfg, zero
+# uimm12
+# CHECK-INST: csrrs t2, mcyclecfg, zero
+# CHECK-ENC:  encoding: [0xf3,0x23,0x10,0x32]
+# CHECK-INST-ALIAS: csrr t2, mcyclecfg
+csrrs t2, 0x321, zero
+
+# minstretcfg
+# name
+# CHECK-INST: csrrs t1, minstretcfg, zero
+# CHECK-ENC:  encoding: [0x73,0x23,0x20,0x32]
+# CHECK-INST-ALIAS: csrr t1, minstretcfg
+csrrs t1, minstretcfg, zero
+# uimm12
+# CHECK-INST: csrrs t2, minstretcfg, zero
+# CHECK-ENC:  encoding: [0xf3,0x23,0x20,0x32]
+# CHECK-INST-ALIAS: csrr t2, minstretcfg
+csrrs t2, 0x322, zero
diff --git a/llvm/test/MC/RISCV/rv32-machine-csr-names.s b/llvm/test/MC/RISCV/rv32-machine-csr-names.s
index 9e929b7eddeed..9fd19c36d5b93 100644
--- a/llvm/test/MC/RISCV/rv32-machine-csr-names.s
+++ b/llvm/test/MC/RISCV/rv32-machine-csr-names.s
@@ -1163,3 +1163,27 @@ csrrs t2, 0x319, zero
 csrrs t1, miph, zero
 # uimm12
 csrrs t2, 0x354, zero
+
+# mcyclecfgh
+# name
+# CHECK-INST: csrrs t1, mcyclecfgh, zero
+# CHECK-ENC:  encoding: [0x73,0x23,0x10,0x72]
+# CHECK-INST-ALIAS: csrr t1, mcyclecfgh
+csrrs t1, mcyclecfgh, zero
+# uimm12
+# CHECK-INST: csrrs t2, mcyclecfgh, zero
+# CHECK-ENC:  encoding: [0xf3,0x23,0x10,0x72]
+# CHECK-INST-ALIAS: csrr t2, mcyclecfgh
+csrrs t2, 0x721, zero
+
+# minstretcfgh
+# name
+# CHECK-INST: csrrs t1, minstretcfgh, zero
+# CHECK-ENC:  encoding: [0x73,0x23,0x20,0x72]
+# CHECK-INST-ALIAS: csrr t1, minstretcfgh
+csrrs t1, minstretcfgh, zero
+# uimm12
+# CHECK-INST: csrrs t2, minstretcfgh, zero
+# CHECK-ENC:  encoding: [0xf3,0x23,0x20,0x72]
+# CHECK-INST-ALIAS: csrr t2, minstretcfgh
+csrrs t2, 0x722, zero
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 6e190ad3e7969..43896fede57d8 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1086,6 +1086,7 @@ R"(All available -march extensions for RISC-V
     shvstvecd            1.0
     smaia                1.0
     smcdeleg             1.0
+    smcntrpmf            1.0
     smcsrind             1.0
     smdbltrp             1.0
     smepmp               1.0

From 497382ee07100f3698621fc48b66a0bd50a1ca2a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 21 Apr 2025 22:39:31 -0700
Subject: [PATCH 683/710] [SelectionDAG] Make the FoldingSet profile in
 getAtomic match AddNodeIDCustom. (#136651)

In theory, the mismatch would have made CSE of AtomicSDNodes not work,
but I don't know how to test it.
---
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index ba6c5d884d381..5269962ea2062 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8994,8 +8994,10 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
                                 SDVTList VTList, ArrayRef<SDValue> Ops,
                                 MachineMemOperand *MMO) {
   FoldingSetNodeID ID;
-  ID.AddInteger(MemVT.getRawBits());
   AddNodeIDNode(ID, Opcode, VTList, Ops);
+  ID.AddInteger(MemVT.getRawBits());
+  ID.AddInteger(getSyntheticNodeSubclassData<AtomicSDNode>(
+      Opcode, dl.getIROrder(), VTList, MemVT, MMO));
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   ID.AddInteger(MMO->getFlags());
   void* IP = nullptr;

From 784dc16088885ebb28437ae2dbb90dcfd5df0457 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter@amd.com>
Date: Tue, 22 Apr 2025 08:32:03 +0200
Subject: [PATCH 684/710] [LangRef][IR] Fix default AS documentation for
 allocas without explicit AS (#135942)

So far, the Language Reference said that the alloca address space from
the datalayout is used if no explicit address space is provided, which
is not what the LLParser and the AsmWriter implement. This patch adjusts
the documentation to match the implementation: The default AS 0 is used
if none is explicitly specified.

This is an alternative to PR #135786, which would change the parser's
behavior to match the Language Reference instead.
---
 llvm/docs/LangRef.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index a5df82895d839..9e78f317bd9ea 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -11104,9 +11104,8 @@ Overview:
 
 The '``alloca``' instruction allocates memory on the stack frame of the
 currently executing function, to be automatically released when this
-function returns to its caller.  If the address space is not explicitly
-specified, the object is allocated in the alloca address space from the
-:ref:`datalayout string<langref_datalayout>`.
+function returns to its caller. If the address space is not explicitly
+specified, the default address space 0 is used.
 
 Arguments:
 """"""""""
@@ -11148,7 +11147,10 @@ which way the stack grows) is not specified.
 
 Note that '``alloca``' outside of the alloca address space from the
 :ref:`datalayout string<langref_datalayout>` is meaningful only if the
-target has assigned it a semantics.
+target has assigned it a semantics. For targets that specify a non-zero alloca
+address space in the :ref:`datalayout string<langref_datalayout>`, the alloca
+address space needs to be explicitly specified in the instruction if it is to be
+used.
 
 If the returned pointer is used by :ref:`llvm.lifetime.start <int_lifestart>`,
 the returned object is initially dead.

From 53927ab726e6d0dfb3255498b6d0b130adeafb8c Mon Sep 17 00:00:00 2001
From: Henrich Lauko <xlauko@mail.muni.cz>
Date: Tue, 22 Apr 2025 08:59:58 +0200
Subject: [PATCH 685/710] [CIR] Make ZeroAttr use
 AttrBuilderWithInferredContext (#136604)

This mirrors incubator changes from https://github.com/llvm/clangir/pull/1576
---
 clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h | 8 ++------
 clang/include/clang/CIR/Dialect/IR/CIRAttrs.td           | 7 +++++++
 clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp             | 2 +-
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index ee8af62ede0da..b303aa07838ee 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -83,21 +83,17 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     return getConstPtrAttr(t, 0);
   }
 
-  mlir::TypedAttr getZeroAttr(mlir::Type t) {
-    return cir::ZeroAttr::get(getContext(), t);
-  }
-
   mlir::TypedAttr getZeroInitAttr(mlir::Type ty) {
     if (mlir::isa<cir::IntType>(ty))
       return cir::IntAttr::get(ty, 0);
     if (cir::isAnyFloatingPointType(ty))
       return cir::FPAttr::getZero(ty);
     if (auto arrTy = mlir::dyn_cast<cir::ArrayType>(ty))
-      return getZeroAttr(arrTy);
+      return cir::ZeroAttr::get(arrTy);
     if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(ty))
       return getConstNullPtrAttr(ptrTy);
     if (auto recordTy = mlir::dyn_cast<cir::RecordType>(ty))
-      return getZeroAttr(recordTy);
+      return cir::ZeroAttr::get(recordTy);
     if (mlir::isa<cir::BoolType>(ty)) {
       return getFalseAttr();
     }
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index 25ceded7e8a5b..214db1b1caeeb 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -71,6 +71,13 @@ def ZeroAttr : CIR_Attr<"Zero", "zero", [TypedAttrInterface]> {
   }];
 
   let parameters = (ins AttributeSelfTypeParameter<"">:$type);
+
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "mlir::Type":$type), [{
+      return $_get(type.getContext(), type);
+    }]>
+  ];
+
   let assemblyFormat = [{}];
 }
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index 0caa8961ed0a6..b9a74e90a5960 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -183,7 +183,7 @@ emitArrayConstant(CIRGenModule &cgm, mlir::Type desiredType,
   }
 
   if (nonzeroLength == 0)
-    return cir::ZeroAttr::get(builder.getContext(), desiredType);
+    return cir::ZeroAttr::get(desiredType);
 
   const unsigned trailingZeroes = arrayBound - nonzeroLength;
 

From 0ca2d4d10451874b1d107e89c32eb7fd95e1a608 Mon Sep 17 00:00:00 2001
From: Kirill Chibisov <contact@kchibisov.com>
Date: Tue, 22 Apr 2025 16:03:42 +0900
Subject: [PATCH 686/710] [mlir][emitc] mark `emitc.load` with `CExpression`
 (#130802)

Follow the `call` and `call_opaque` operations, as well as `apply`,
which already are marked as `CExpression` even though they have side
effects.

Even though `emitc.load` can be included inside the `emitc.expression`,
the inlining and `--form-expression` pass won't actually inline them
inside other expression due to it having a side effect, thus unless the
user manually writes the `emitc.load` inside the `emitc.expression` it
won't appear there.

--

It was brought
https://github.com/llvm/llvm-project/pull/91475#issuecomment-2302529428
and while there was some opposition due to `load` having a side effect,
`emitc` already allows all the rest operations that have it, so for
consistency reasons, enabling it doesn't really hurt from my point of
view. Especially given that `--form-expression` doesn't allow
it to really inline inside other expressions, which makes sense, since
if the users want such behavior, they should explicitly opt-in.
---
 mlir/include/mlir/Dialect/EmitC/IR/EmitC.td | 11 ++--
 mlir/lib/Target/Cpp/TranslateToCpp.cpp      |  1 +
 mlir/test/Dialect/EmitC/transforms.mlir     | 34 ++++++++++++
 mlir/test/Target/Cpp/expressions.mlir       | 57 +++++++++++++++++++++
 4 files changed, 96 insertions(+), 7 deletions(-)

diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index a518e7425956c..d4aea52a0d485 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -489,12 +489,9 @@ def EmitC_ExpressionOp : EmitC_Op<"expression",
         auto applyOp = dyn_cast<emitc::ApplyOp>(op);
         if (applyOp)
           return applyOp.getApplicableOperator() == "*";
-        // Any operation using variables is assumed to have a side effect of
-        // reading memory mutable by emitc::assign ops.
-        return llvm::any_of(op.getOperands(), [](Value operand) {
-          Operation *def = operand.getDefiningOp();
-          return def && isa<emitc::VariableOp>(def);
-        });
+        // Any load operation is assumed to read from memory and thus perform
+        // a side effect.
+        return isa<emitc::LoadOp>(op);
       };
       return llvm::any_of(getRegion().front().without_terminator(), predicate);
     };
@@ -927,7 +924,7 @@ def EmitC_LogicalOrOp : EmitC_BinaryOp<"logical_or", [CExpression]> {
   let assemblyFormat = "operands attr-dict `:` type(operands)";
 }
 
-def EmitC_LoadOp : EmitC_Op<"load", [
+def EmitC_LoadOp : EmitC_Op<"load", [CExpression,
   TypesMatchWith<"result type matches value type of 'operand'",
                   "operand", "result",
                   "::llvm::cast<LValueType>($_self).getValueType()">
diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
index 81016ed2e9a8d..0c4975a13d301 100644
--- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp
+++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
@@ -100,6 +100,7 @@ static FailureOr<int> getOperatorPrecedence(Operation *operation) {
       })
       .Case<emitc::ConditionalOp>([&](auto op) { return 2; })
       .Case<emitc::DivOp>([&](auto op) { return 13; })
+      .Case<emitc::LoadOp>([&](auto op) { return 16; })
       .Case<emitc::LogicalAndOp>([&](auto op) { return 4; })
       .Case<emitc::LogicalNotOp>([&](auto op) { return 15; })
       .Case<emitc::LogicalOrOp>([&](auto op) { return 3; })
diff --git a/mlir/test/Dialect/EmitC/transforms.mlir b/mlir/test/Dialect/EmitC/transforms.mlir
index d204dec70d449..a38f396dad953 100644
--- a/mlir/test/Dialect/EmitC/transforms.mlir
+++ b/mlir/test/Dialect/EmitC/transforms.mlir
@@ -129,3 +129,37 @@ func.func @single_result_requirement() -> (i32, i32) {
   %0:2 = emitc.call_opaque "foo" () : () -> (i32, i32)
   return %0#0, %0#1 : i32, i32
 }
+
+// CHECK-LABEL:   func.func @expression_with_load(
+// CHECK-SAME:                                    %[[VAL_0:.*]]: i32,
+// CHECK-SAME:                                    %[[VAL_1:.*]]: !emitc.ptr<i32>) -> i1 {
+// CHECK:           %[[VAL_2:.*]] = "emitc.constant"() <{value = 0 : i64}> : () -> i64
+// CHECK:           %[[VAL_3:.*]] = "emitc.variable"() <{value = #emitc.opaque<"42">}> : () -> !emitc.lvalue<i32>
+// CHECK:           %[[VAL_4:.*]] = emitc.expression : i32 {
+// CHECK:             %[[VAL_5:.*]] = load %[[VAL_3]] : <i32>
+// CHECK:             yield %[[VAL_5]] : i32
+// CHECK:           }
+// CHECK:           %[[VAL_6:.*]] = emitc.subscript %[[VAL_1]]{{\[}}%[[VAL_2]]] : (!emitc.ptr<i32>, i64) -> !emitc.lvalue<i32>
+// CHECK:           %[[VAL_7:.*]] = emitc.expression : i32 {
+// CHECK:             %[[VAL_8:.*]] = load %[[VAL_6]] : <i32>
+// CHECK:             yield %[[VAL_8]] : i32
+// CHECK:           }
+// CHECK:           %[[VAL_9:.*]] = emitc.expression : i1 {
+// CHECK:             %[[VAL_10:.*]] = add %[[VAL_4]], %[[VAL_7]] : (i32, i32) -> i32
+// CHECK:             %[[VAL_11:.*]] = cmp lt, %[[VAL_10]], %[[VAL_0]] : (i32, i32) -> i1
+// CHECK:             yield %[[VAL_11]] : i1
+// CHECK:           }
+// CHECK:           return %[[VAL_9]] : i1
+// CHECK:         }
+
+
+func.func @expression_with_load(%arg0: i32, %arg1: !emitc.ptr<i32>) -> i1 {
+  %c0 = "emitc.constant"() {value = 0 : i64} : () -> i64
+  %0 = "emitc.variable"() <{value = #emitc.opaque<"42">}> : () -> !emitc.lvalue<i32>
+  %a = emitc.load %0 : !emitc.lvalue<i32>
+  %ptr = emitc.subscript %arg1[%c0] : (!emitc.ptr<i32>, i64) -> !emitc.lvalue<i32>
+  %ptr_load = emitc.load %ptr : !emitc.lvalue<i32>
+  %b = emitc.add %a, %ptr_load : (i32, i32) -> i32
+  %c = emitc.cmp lt, %b, %arg0 :(i32, i32) -> i1
+  return %c : i1
+}
diff --git a/mlir/test/Target/Cpp/expressions.mlir b/mlir/test/Target/Cpp/expressions.mlir
index 3a1694e7d15dc..9316d7b77619b 100644
--- a/mlir/test/Target/Cpp/expressions.mlir
+++ b/mlir/test/Target/Cpp/expressions.mlir
@@ -342,3 +342,60 @@ func.func @expression_with_subscript_user(%arg0: !emitc.ptr<!emitc.opaque<"void"
   %res_load = emitc.load %res : !emitc.lvalue<i32>
   return %res_load : i32
 }
+
+// CPP-DEFAULT: bool expression_with_load(int32_t [[VAL_1:v.+]], int32_t [[VAL_2:v.+]], int32_t* [[VAL_3:v.+]]) {
+// CPP-DEFAULT-NEXT:   int64_t [[VAL_4:v.+]] = 0;
+// CPP-DEFAULT-NEXT:   int32_t [[VAL_5:v.+]] = 42;
+// CPP-DEFAULT-NEXT:   bool [[VAL_6:v.+]] = [[VAL_5]] + [[VAL_2]] < [[VAL_3]][[[VAL_4]]] + [[VAL_1]];
+// CPP-DEFAULT-NEXT:   return [[VAL_6]];
+
+// CPP-DECLTOP: bool expression_with_load(int32_t [[VAL_1:v.+]], int32_t [[VAL_2:v.+]], int32_t* [[VAL_3:v.+]]) {
+// CPP-DECLTOP-NEXT:   int64_t [[VAL_4:v.+]];
+// CPP-DECLTOP-NEXT:   int32_t [[VAL_5:v.+]];
+// CPP-DECLTOP-NEXT:   bool [[VAL_6:v.+]];
+// CPP-DECLTOP-NEXT:   [[VAL_4]] = 0;
+// CPP-DECLTOP-NEXT:   [[VAL_5]] = 42;
+// CPP-DECLTOP-NEXT:   [[VAL_6]] = [[VAL_5]] + [[VAL_2]] < [[VAL_3]][[[VAL_4]]] + [[VAL_1]];
+// CPP-DECLTOP-NEXT:   return [[VAL_6]];
+
+func.func @expression_with_load(%arg0: i32, %arg1: i32, %arg2: !emitc.ptr<i32>) -> i1 {
+  %c0 = "emitc.constant"() {value = 0 : i64} : () -> i64
+  %0 = "emitc.variable"() <{value = #emitc.opaque<"42">}> : () -> !emitc.lvalue<i32>
+  %ptr = emitc.subscript %arg2[%c0] : (!emitc.ptr<i32>, i64) -> !emitc.lvalue<i32>
+  %result = emitc.expression : i1 {
+    %a = emitc.load %0 : !emitc.lvalue<i32>
+    %b = emitc.add %a, %arg1 : (i32, i32) -> i32
+    %c = emitc.load %ptr : !emitc.lvalue<i32>
+    %d = emitc.add %c, %arg0 : (i32, i32) -> i32
+    %e = emitc.cmp lt, %b, %d :(i32, i32) -> i1
+    yield %e : i1
+  }
+  return %result : i1
+}
+
+// CPP-DEFAULT: bool expression_with_load_and_call(int32_t* [[VAL_1:v.+]]) {
+// CPP-DEFAULT-NEXT:   int64_t [[VAL_2:v.+]] = 0;
+// CPP-DEFAULT-NEXT:   bool [[VAL_3:v.+]] = [[VAL_1]][[[VAL_2]]] + bar([[VAL_1]][[[VAL_2]]]) < [[VAL_1]][[[VAL_2]]];
+// CPP-DEFAULT-NEXT:   return [[VAL_3]];
+
+// CPP-DECLTOP: bool expression_with_load_and_call(int32_t* [[VAL_1:v.+]]) {
+// CPP-DECLTOP-NEXT:   int64_t [[VAL_2:v.+]];
+// CPP-DECLTOP-NEXT:   bool [[VAL_3:v.+]];
+// CPP-DECLTOP-NEXT:   [[VAL_2]] = 0;
+// CPP-DECLTOP-NEXT:   [[VAL_3]] = [[VAL_1]][[[VAL_2]]] + bar([[VAL_1]][[[VAL_2]]]) < [[VAL_1]][[[VAL_2]]];
+// CPP-DECLTOP-NEXT:   return [[VAL_3]];
+
+func.func @expression_with_load_and_call(%arg0: !emitc.ptr<i32>) -> i1 {
+  %c0 = "emitc.constant"() {value = 0 : i64} : () -> i64
+  %ptr = emitc.subscript %arg0[%c0] : (!emitc.ptr<i32>, i64) -> !emitc.lvalue<i32>
+  %result = emitc.expression : i1 {
+    %a = emitc.load %ptr : !emitc.lvalue<i32>
+    %b = emitc.load %ptr : !emitc.lvalue<i32>
+    %c = emitc.load %ptr : !emitc.lvalue<i32>
+    %d = emitc.call_opaque "bar" (%a) : (i32) -> (i32)
+    %e = add %c, %d : (i32, i32) -> i32
+    %f = emitc.cmp lt, %e, %b :(i32, i32) -> i1
+    yield %f : i1
+  }
+  return %result : i1
+}

From 8639b365a5988932973a82ffe5e620a9c8ef9039 Mon Sep 17 00:00:00 2001
From: Henrich Lauko <xlauko@mail.muni.cz>
Date: Tue, 22 Apr 2025 09:08:49 +0200
Subject: [PATCH 687/710] [CIR] Make UndefAttr use
 AttrBuilderWithInferredContext (#136605)

This mirrors incubator changes from https://github.com/llvm/clangir/pull/1577
---
 clang/include/clang/CIR/Dialect/IR/CIRAttrs.td | 7 +++++++
 clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp     | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index 214db1b1caeeb..dfe15a10fa54e 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -93,6 +93,13 @@ def UndefAttr : CIR_Attr<"Undef", "undef", [TypedAttrInterface]> {
   }];
 
   let parameters = (ins AttributeSelfTypeParameter<"">:$type);
+
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "mlir::Type":$type), [{
+      return $_get(type.getContext(), type);
+    }]>
+  ];
+
   let assemblyFormat = [{}];
 }
 
diff --git a/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp b/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
index 9cd5c54e6c19e..fe8a5e7428a81 100644
--- a/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
@@ -34,8 +34,8 @@ llvm::SmallVector<MemorySlot> cir::AllocaOp::getPromotableSlots() {
 
 Value cir::AllocaOp::getDefaultValue(const MemorySlot &slot,
                                      OpBuilder &builder) {
-  return builder.create<cir::ConstantOp>(
-      getLoc(), slot.elemType, builder.getAttr<cir::UndefAttr>(slot.elemType));
+  return builder.create<cir::ConstantOp>(getLoc(), slot.elemType,
+                                         cir::UndefAttr::get(slot.elemType));
 }
 
 void cir::AllocaOp::handleBlockArgument(const MemorySlot &slot,

From 2d3bbb6aafbc74ef6fc51286f09def0f0e35fe14 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 22 Apr 2025 09:12:00 +0200
Subject: [PATCH 688/710] [mlir][Transforms] Dialect conversion: Erase
 materialized constants instead of rollback (#136489)

When illegal (and not legalizable) constant operations are materialized
during a dialect conversion as part of op folding, these operations must
be deleted again. This used to be implemented via the rollback
mechanism. This commit switches the implementation to regular rewriter
API usage: simply delete the materialized constants with `eraseOp`.

This commit is in preparation of the One-Shot Dialect Conversion
refactoring, which will disallow IR rollbacks.

This commit also adds a new optional parameter to `OpBuilder::tryFold`
to get hold of the materialized constant ops.
---
 mlir/include/mlir/IR/Builders.h               |  8 +++++--
 mlir/lib/IR/Builders.cpp                      |  9 ++++++--
 .../Transforms/Utils/DialectConversion.cpp    | 21 ++++++++-----------
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/mlir/include/mlir/IR/Builders.h b/mlir/include/mlir/IR/Builders.h
index cd8d3ee0af72b..96dd14f142328 100644
--- a/mlir/include/mlir/IR/Builders.h
+++ b/mlir/include/mlir/IR/Builders.h
@@ -564,9 +564,13 @@ class OpBuilder : public Builder {
 
   /// Attempts to fold the given operation and places new results within
   /// `results`. Returns success if the operation was folded, failure otherwise.
-  /// If the fold was in-place, `results` will not be filled.
+  /// If the fold was in-place, `results` will not be filled. Optionally, newly
+  /// materialized constant operations can be returned to the caller.
+  ///
   /// Note: This function does not erase the operation on a successful fold.
-  LogicalResult tryFold(Operation *op, SmallVectorImpl<Value> &results);
+  LogicalResult
+  tryFold(Operation *op, SmallVectorImpl<Value> &results,
+          SmallVectorImpl<Operation *> *materializedConstants = nullptr);
 
   /// Creates a deep copy of the specified operation, remapping any operands
   /// that use values outside of the operation using the map that is provided
diff --git a/mlir/lib/IR/Builders.cpp b/mlir/lib/IR/Builders.cpp
index 16bd8201ad50a..89102115cdc40 100644
--- a/mlir/lib/IR/Builders.cpp
+++ b/mlir/lib/IR/Builders.cpp
@@ -465,8 +465,9 @@ Operation *OpBuilder::create(Location loc, StringAttr opName,
   return create(state);
 }
 
-LogicalResult OpBuilder::tryFold(Operation *op,
-                                 SmallVectorImpl<Value> &results) {
+LogicalResult
+OpBuilder::tryFold(Operation *op, SmallVectorImpl<Value> &results,
+                   SmallVectorImpl<Operation *> *materializedConstants) {
   assert(results.empty() && "expected empty results");
   ResultRange opResults = op->getResults();
 
@@ -528,6 +529,10 @@ LogicalResult OpBuilder::tryFold(Operation *op,
   for (Operation *cst : generatedConstants)
     insert(cst);
 
+  // Return materialized constant operations.
+  if (materializedConstants)
+    *materializedConstants = std::move(generatedConstants);
+
   return success();
 }
 
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 962207059c8aa..4d250329c6f45 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2090,8 +2090,6 @@ LogicalResult
 OperationLegalizer::legalizeWithFold(Operation *op,
                                      ConversionPatternRewriter &rewriter) {
   auto &rewriterImpl = rewriter.getImpl();
-  RewriterState curState = rewriterImpl.getCurrentState();
-
   LLVM_DEBUG({
     rewriterImpl.logger.startLine() << "* Fold {\n";
     rewriterImpl.logger.indent();
@@ -2099,28 +2097,27 @@ OperationLegalizer::legalizeWithFold(Operation *op,
 
   // Try to fold the operation.
   SmallVector<Value, 2> replacementValues;
+  SmallVector<Operation *, 2> newOps;
   rewriter.setInsertionPoint(op);
-  if (failed(rewriter.tryFold(op, replacementValues))) {
+  if (failed(rewriter.tryFold(op, replacementValues, &newOps))) {
     LLVM_DEBUG(logFailure(rewriterImpl.logger, "unable to fold"));
     return failure();
   }
+
   // An empty list of replacement values indicates that the fold was in-place.
   // As the operation changed, a new legalization needs to be attempted.
   if (replacementValues.empty())
     return legalize(op, rewriter);
 
   // Recursively legalize any new constant operations.
-  for (unsigned i = curState.numRewrites, e = rewriterImpl.rewrites.size();
-       i != e; ++i) {
-    auto *createOp =
-        dyn_cast<CreateOperationRewrite>(rewriterImpl.rewrites[i].get());
-    if (!createOp)
-      continue;
-    if (failed(legalize(createOp->getOperation(), rewriter))) {
+  for (Operation *newOp : newOps) {
+    if (failed(legalize(newOp, rewriter))) {
       LLVM_DEBUG(logFailure(rewriterImpl.logger,
                             "failed to legalize generated constant '{0}'",
-                            createOp->getOperation()->getName()));
-      rewriterImpl.resetState(curState);
+                            newOp->getName()));
+      // Legalization failed: erase all materialized constants.
+      for (Operation *op : newOps)
+        rewriter.eraseOp(op);
       return failure();
     }
   }

From 56f5bcb0350b8af44aa5f65ccf865a464df2cc4a Mon Sep 17 00:00:00 2001
From: Henrich Lauko <xlauko@mail.muni.cz>
Date: Tue, 22 Apr 2025 09:15:59 +0200
Subject: [PATCH 689/710] [CIR] Let ConstantOp builder infer its type
 automatically (#136606)

This mirrors incubator changes from https://github.com/llvm/clangir/pull/1578
---
 clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h | 4 ++--
 clang/include/clang/CIR/Dialect/IR/CIROps.td             | 9 ---------
 clang/lib/CIR/CodeGen/CIRGenBuilder.h                    | 2 +-
 clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp               | 4 ++--
 clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp               | 2 +-
 5 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index b303aa07838ee..0385b4f476c3b 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -61,11 +61,11 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value getConstAPInt(mlir::Location loc, mlir::Type typ,
                             const llvm::APInt &val) {
-    return create<cir::ConstantOp>(loc, typ, getAttr<cir::IntAttr>(typ, val));
+    return create<cir::ConstantOp>(loc, getAttr<cir::IntAttr>(typ, val));
   }
 
   cir::ConstantOp getConstant(mlir::Location loc, mlir::TypedAttr attr) {
-    return create<cir::ConstantOp>(loc, attr.getType(), attr);
+    return create<cir::ConstantOp>(loc, attr);
   }
 
   cir::ConstantOp getConstantInt(mlir::Location loc, mlir::Type ty,
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index b526d077a910c..577cb8db41f57 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -288,18 +288,9 @@ def ConstantOp : CIR_Op<"const",
     ```
   }];
 
-  // The constant operation takes an attribute as the only input.
   let arguments = (ins TypedAttrInterface:$value);
-
-  // The constant operation returns a single value of CIR_AnyType.
   let results = (outs CIR_AnyType:$res);
 
-  let builders = [
-    OpBuilder<(ins "cir::BoolAttr":$value), [{
-      build($_builder, $_state, value.getType(), value);
-    }]>
-  ];
-
   let assemblyFormat = "attr-dict $value";
 
   let hasVerifier = 1;
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
index 4f7ff5128d914..7d9988cc52c5a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
@@ -185,7 +185,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   // Creates constant nullptr for pointer type ty.
   cir::ConstantOp getNullPtr(mlir::Type ty, mlir::Location loc) {
     assert(!cir::MissingFeatures::targetCodeGenInfoGetNullPointer());
-    return create<cir::ConstantOp>(loc, ty, getConstPtrAttr(ty, 0));
+    return create<cir::ConstantOp>(loc, getConstPtrAttr(ty, 0));
   }
 
   mlir::Value createNeg(mlir::Value value) {
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index f1561d1b26fc0..1e69ecae831e9 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -138,7 +138,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
   mlir::Value VisitIntegerLiteral(const IntegerLiteral *e) {
     mlir::Type type = cgf.convertType(e->getType());
     return builder.create<cir::ConstantOp>(
-        cgf.getLoc(e->getExprLoc()), type,
+        cgf.getLoc(e->getExprLoc()),
         builder.getAttr<cir::IntAttr>(type, e->getValue()));
   }
 
@@ -147,7 +147,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     assert(mlir::isa<cir::CIRFPTypeInterface>(type) &&
            "expect floating-point type");
     return builder.create<cir::ConstantOp>(
-        cgf.getLoc(e->getExprLoc()), type,
+        cgf.getLoc(e->getExprLoc()),
         builder.getAttr<cir::FPAttr>(type, e->getValue()));
   }
 
diff --git a/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp b/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
index fe8a5e7428a81..20b086ffdd850 100644
--- a/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
@@ -34,7 +34,7 @@ llvm::SmallVector<MemorySlot> cir::AllocaOp::getPromotableSlots() {
 
 Value cir::AllocaOp::getDefaultValue(const MemorySlot &slot,
                                      OpBuilder &builder) {
-  return builder.create<cir::ConstantOp>(getLoc(), slot.elemType,
+  return builder.create<cir::ConstantOp>(getLoc(),
                                          cir::UndefAttr::get(slot.elemType));
 }
 

From 8bc0d4d1ccb708d20413876258885a48d3ca6b3d Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 22 Apr 2025 09:45:11 +0200
Subject: [PATCH 690/710] [mlir][Transforms] Dialect conversion: Add flag to
 disable rollback (#136490)

This commit adds a new flag to `ConversionConfig` to disallow the
rollback of IR modification. This commit is in preparation of the
One-Shot Dialect Conversion refactoring, which will remove the ability
to roll back IR modifications from the conversion driver.

RFC:
https://discourse.llvm.org/t/rfc-a-new-one-shot-dialect-conversion-driver/79083/46

By default, this flag is set to "true". I.e., the rollback of IR
modifications is allowed. When set to "false", the conversion driver
will report a fatal LLVM error when an IR rollback is requested. The
name of the rolled back pattern is included in the error message.
Moreover, the original IR is no longer restored after a failed
conversion.

Example:
```
within split at llvm-project/mlir/test/Conversion/ArithToSPIRV/fast-math.mlir:1 offset :11:8: error: pattern '(anonymous namespace)::CmpFOpNanKernelPattern' produced IR that could not be legalized
  %0 = arith.cmpf ord, %arg0, %arg1 fastmath<fast> : f32
       ^
within split at llvm-project/mlir/test/Conversion/ArithToSPIRV/fast-math.mlir:1 offset :11:8: note: see current operation: %1 = "arith.cmpf"(%arg0, %arg1) <{fastmath = #arith.fastmath<fast>, predicate = 7 : i64}> : (f32, f32) -> i1
pattern '(anonymous namespace)::CmpFOpNanKernelPattern' rollback of IR modifications requested
UNREACHABLE executed at llvm-project/mlir/lib/Transforms/Utils/DialectConversion.cpp:1231!
```

The majority of patterns in MLIR have already been updated such that
they do not trigger any rollbacks, but a few SPIRV patterns remain. More
information in the RFC.
---
 .../mlir/Transforms/DialectConversion.h       | 20 +++++++
 .../Transforms/Utils/DialectConversion.cpp    | 56 ++++++++++++++-----
 2 files changed, 62 insertions(+), 14 deletions(-)

diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index b6ab252456e70..b65b3ea971f91 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -1219,6 +1219,26 @@ struct ConversionConfig {
   /// materializations and instead inserts "builtin.unrealized_conversion_cast"
   /// ops to ensure that the resulting IR is valid.
   bool buildMaterializations = true;
+
+  /// If set to "true", pattern rollback is allowed. The conversion driver
+  /// rolls back IR modifications in the following situations.
+  ///
+  /// 1. Pattern implementation returns "failure" after modifying IR.
+  /// 2. Pattern produces IR (in-place modification or new IR) that is illegal
+  ///    and cannot be legalized by subsequent foldings / pattern applications.
+  ///
+  /// If set to "false", the conversion driver will produce an LLVM fatal error
+  /// instead of rolling back IR modifications. Moreover, in case of a failed
+  /// conversion, the original IR is not restored. The resulting IR may be a
+  /// mix of original and rewritten IR. (Same as a failed greedy pattern
+  /// rewrite.)
+  ///
+  /// Note: This flag was added in preparation of the One-Shot Dialect
+  /// Conversion refactoring, which will remove the ability to roll back IR
+  /// modifications from the conversion driver. Use this flag to ensure that
+  /// your patterns do not trigger any IR rollbacks. For details, see
+  /// https://discourse.llvm.org/t/rfc-a-new-one-shot-dialect-conversion-driver/79083.
+  bool allowPatternRollback = true;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 4d250329c6f45..eb3ca01a462a6 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -861,8 +861,10 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   /// conversion process succeeds.
   void applyRewrites();
 
-  /// Reset the state of the rewriter to a previously saved point.
-  void resetState(RewriterState state);
+  /// Reset the state of the rewriter to a previously saved point. Optionally,
+  /// the name of the pattern that triggered the rollback can specified for
+  /// debugging purposes.
+  void resetState(RewriterState state, StringRef patternName = "");
 
   /// Append a rewrite. Rewrites are committed upon success and rolled back upon
   /// failure.
@@ -873,8 +875,9 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   }
 
   /// Undo the rewrites (motions, splits) one by one in reverse order until
-  /// "numRewritesToKeep" rewrites remains.
-  void undoRewrites(unsigned numRewritesToKeep = 0);
+  /// "numRewritesToKeep" rewrites remains. Optionally, the name of the pattern
+  /// that triggered the rollback can specified for debugging purposes.
+  void undoRewrites(unsigned numRewritesToKeep = 0, StringRef patternName = "");
 
   /// Remap the given values to those with potentially different types. Returns
   /// success if the values could be remapped, failure otherwise. `valueDiagTag`
@@ -1204,9 +1207,10 @@ RewriterState ConversionPatternRewriterImpl::getCurrentState() {
   return RewriterState(rewrites.size(), ignoredOps.size(), replacedOps.size());
 }
 
-void ConversionPatternRewriterImpl::resetState(RewriterState state) {
+void ConversionPatternRewriterImpl::resetState(RewriterState state,
+                                               StringRef patternName) {
   // Undo any rewrites.
-  undoRewrites(state.numRewrites);
+  undoRewrites(state.numRewrites, patternName);
 
   // Pop all of the recorded ignored operations that are no longer valid.
   while (ignoredOps.size() != state.numIgnoredOperations)
@@ -1216,10 +1220,18 @@ void ConversionPatternRewriterImpl::resetState(RewriterState state) {
     replacedOps.pop_back();
 }
 
-void ConversionPatternRewriterImpl::undoRewrites(unsigned numRewritesToKeep) {
+void ConversionPatternRewriterImpl::undoRewrites(unsigned numRewritesToKeep,
+                                                 StringRef patternName) {
   for (auto &rewrite :
-       llvm::reverse(llvm::drop_begin(rewrites, numRewritesToKeep)))
+       llvm::reverse(llvm::drop_begin(rewrites, numRewritesToKeep))) {
+    if (!config.allowPatternRollback &&
+        !isa<UnresolvedMaterializationRewrite>(rewrite)) {
+      // Unresolved materializations can always be rolled back (erased).
+      llvm::report_fatal_error("pattern '" + patternName +
+                               "' rollback of IR modifications requested");
+    }
     rewrite->rollback();
+  }
   rewrites.resize(numRewritesToKeep);
 }
 
@@ -2158,7 +2170,7 @@ OperationLegalizer::legalizeWithPattern(Operation *op,
     });
     if (config.listener)
       config.listener->notifyPatternEnd(pattern, failure());
-    rewriterImpl.resetState(curState);
+    rewriterImpl.resetState(curState, pattern.getDebugName());
     appliedPatterns.erase(&pattern);
   };
 
@@ -2168,8 +2180,13 @@ OperationLegalizer::legalizeWithPattern(Operation *op,
     assert(rewriterImpl.pendingRootUpdates.empty() && "dangling root updates");
     auto result = legalizePatternResult(op, pattern, rewriter, curState);
     appliedPatterns.erase(&pattern);
-    if (failed(result))
-      rewriterImpl.resetState(curState);
+    if (failed(result)) {
+      if (!rewriterImpl.config.allowPatternRollback)
+        op->emitError("pattern '")
+            << pattern.getDebugName()
+            << "' produced IR that could not be legalized";
+      rewriterImpl.resetState(curState, pattern.getDebugName());
+    }
     if (config.listener)
       config.listener->notifyPatternEnd(pattern, result);
     return result;
@@ -2674,9 +2691,20 @@ LogicalResult OperationConverter::convertOperations(ArrayRef<Operation *> ops) {
   ConversionPatternRewriter rewriter(ops.front()->getContext(), config);
   ConversionPatternRewriterImpl &rewriterImpl = rewriter.getImpl();
 
-  for (auto *op : toConvert)
-    if (failed(convert(rewriter, op)))
-      return rewriterImpl.undoRewrites(), failure();
+  for (auto *op : toConvert) {
+    if (failed(convert(rewriter, op))) {
+      // Dialect conversion failed.
+      if (rewriterImpl.config.allowPatternRollback) {
+        // Rollback is allowed: restore the original IR.
+        rewriterImpl.undoRewrites();
+      } else {
+        // Rollback is not allowed: apply all modifications that have been
+        // performed so far.
+        rewriterImpl.applyRewrites();
+      }
+      return failure();
+    }
+  }
 
   // After a successful conversion, apply rewrites.
   rewriterImpl.applyRewrites();

From a22ad659cd0665669d89fae174f9e6a83d1a446d Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Tue, 22 Apr 2025 10:14:03 +0200
Subject: [PATCH 691/710] [lldb/cmake] Normalize use of HAVE_LIBCOMPRESSION
 (#135528)

I *think* this was the reason behind the failures in
2fd860c1f559c0b0be66cc000e38270a04d0a1a3: the clang include tool showed
the Config.h headers as unused, and because the macro was referenced
through an `#ifdef`, its removal didn't cause build failures. Switching
to `#cmakedefine01` + `#if` should make sure this does not happen again.

According to D48977, the `#ifndef`+`#cmakedefine` patterns is due to
some files redefining the macro themselves. I no longer see any such
files in the source tree (there also were no files like that in the
source tree at the revision mentioned, but the macro *was* defined in
the hand-maintained XCode project we had at the time).
---
 lldb/include/lldb/Host/Config.h.cmake                |  4 +---
 .../Process/gdb-remote/GDBRemoteCommunication.cpp    |  6 +++---
 .../Process/gdb-remote/GDBRemoteCommunication.h      |  2 +-
 .../gdb-remote/GDBRemoteCommunicationClient.cpp      | 12 +++---------
 4 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/lldb/include/lldb/Host/Config.h.cmake b/lldb/include/lldb/Host/Config.h.cmake
index 9e538534086a2..46e9a95781c32 100644
--- a/lldb/include/lldb/Host/Config.h.cmake
+++ b/lldb/include/lldb/Host/Config.h.cmake
@@ -23,9 +23,7 @@
 
 #cmakedefine01 HAVE_NR_PROCESS_VM_READV
 
-#ifndef HAVE_LIBCOMPRESSION
-#cmakedefine HAVE_LIBCOMPRESSION
-#endif
+#cmakedefine01 HAVE_LIBCOMPRESSION
 
 #cmakedefine01 LLDB_ENABLE_POSIX
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index abdc3da047dc7..7fe5dd4827cfe 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -41,7 +41,7 @@
 #define DEBUGSERVER_BASENAME "lldb-server"
 #endif
 
-#if defined(HAVE_LIBCOMPRESSION)
+#if HAVE_LIBCOMPRESSION
 #include <compression.h>
 #endif
 
@@ -72,7 +72,7 @@ GDBRemoteCommunication::~GDBRemoteCommunication() {
     Disconnect();
   }
 
-#if defined(HAVE_LIBCOMPRESSION)
+#if HAVE_LIBCOMPRESSION
   if (m_decompression_scratch)
     free (m_decompression_scratch);
 #endif
@@ -509,7 +509,7 @@ bool GDBRemoteCommunication::DecompressPacket() {
     }
   }
 
-#if defined(HAVE_LIBCOMPRESSION)
+#if HAVE_LIBCOMPRESSION
   if (m_compression_type == CompressionType::ZlibDeflate ||
       m_compression_type == CompressionType::LZFSE ||
       m_compression_type == CompressionType::LZ4 ||
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
index 3565a73f19586..387ad0f65bdea 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
@@ -214,7 +214,7 @@ class GDBRemoteCommunication : public Communication {
   HostThread m_listen_thread;
   std::string m_listen_url;
 
-#if defined(HAVE_LIBCOMPRESSION)
+#if HAVE_LIBCOMPRESSION
   CompressionType m_decompression_scratch_type = CompressionType::None;
   void *m_decompression_scratch = nullptr;
 #endif
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index 501670d62e75b..748f95c61cd3c 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -41,7 +41,7 @@
 #include "llvm/Config/llvm-config.h" // for LLVM_ENABLE_ZLIB
 #include "llvm/Support/JSON.h"
 
-#if defined(HAVE_LIBCOMPRESSION)
+#if HAVE_LIBCOMPRESSION
 #include <compression.h>
 #endif
 
@@ -1104,7 +1104,7 @@ void GDBRemoteCommunicationClient::MaybeEnableCompression(
   CompressionType avail_type = CompressionType::None;
   llvm::StringRef avail_name;
 
-#if defined(HAVE_LIBCOMPRESSION)
+#if HAVE_LIBCOMPRESSION
   if (avail_type == CompressionType::None) {
     for (auto compression : supported_compressions) {
       if (compression == "lzfse") {
@@ -1114,9 +1114,6 @@ void GDBRemoteCommunicationClient::MaybeEnableCompression(
       }
     }
   }
-#endif
-
-#if defined(HAVE_LIBCOMPRESSION)
   if (avail_type == CompressionType::None) {
     for (auto compression : supported_compressions) {
       if (compression == "zlib-deflate") {
@@ -1140,7 +1137,7 @@ void GDBRemoteCommunicationClient::MaybeEnableCompression(
   }
 #endif
 
-#if defined(HAVE_LIBCOMPRESSION)
+#if HAVE_LIBCOMPRESSION
   if (avail_type == CompressionType::None) {
     for (auto compression : supported_compressions) {
       if (compression == "lz4") {
@@ -1150,9 +1147,6 @@ void GDBRemoteCommunicationClient::MaybeEnableCompression(
       }
     }
   }
-#endif
-
-#if defined(HAVE_LIBCOMPRESSION)
   if (avail_type == CompressionType::None) {
     for (auto compression : supported_compressions) {
       if (compression == "lzma") {

From f541a3aad8e70b613ea411bf2d3e64a220f98e82 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <frederik.harwath@amd.com>
Date: Tue, 22 Apr 2025 10:17:35 +0200
Subject: [PATCH 692/710] [AMDGPU] SIInstrInfo: Fix resultDependsOnExec for
 VOPC instructions (#134629)

SIInstrInfo::resultDependsOnExec assumes that operand 0 of a comparison
is always the destination of the instruction. This is not true for
instructions in VOPC form where it is "src0". This led to a crash in
machine-cse.

---------

Co-authored-by: Matt Arsenault <arsenm2@gmail.com>
---
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        | 11 +++++---
 llvm/lib/Target/AMDGPU/SIInstrInfo.h          |  2 ++
 .../AMDGPU/si-instr-info-vopc-exec.mir        | 25 +++++++++++++++++++
 3 files changed, 35 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/si-instr-info-vopc-exec.mir

diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ecfffa5067a39..e6d54860df221 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -147,14 +147,19 @@ bool SIInstrInfo::isReallyTriviallyReMaterializable(
 }
 
 // Returns true if the scalar result of a VALU instruction depends on exec.
-static bool resultDependsOnExec(const MachineInstr &MI) {
+bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
   // Ignore comparisons which are only used masked with exec.
   // This allows some hoisting/sinking of VALU comparisons.
   if (MI.isCompare()) {
-    const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
-    Register DstReg = MI.getOperand(0).getReg();
+    const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::sdst);
+    if (!Dst)
+      return true;
+
+    Register DstReg = Dst->getReg();
     if (!DstReg.isVirtual())
       return true;
+
+    const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
     for (MachineInstr &Use : MRI.use_nodbg_instructions(DstReg)) {
       switch (Use.getOpcode()) {
       case AMDGPU::S_AND_SAVEEXEC_B32:
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 9eb9444d0fe96..a3a54659d299a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -183,6 +183,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
                   StringRef &ErrInfo) const;
 
+  bool resultDependsOnExec(const MachineInstr &MI) const;
+
 protected:
   /// If the specific machine instruction is a instruction that moves/copies
   /// value from one register to another register return destination and source
diff --git a/llvm/test/CodeGen/AMDGPU/si-instr-info-vopc-exec.mir b/llvm/test/CodeGen/AMDGPU/si-instr-info-vopc-exec.mir
new file mode 100644
index 0000000000000..52a11ee3cdedf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-instr-info-vopc-exec.mir
@@ -0,0 +1,25 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=machine-cse %s -o - | FileCheck %s
+
+# Check only that V_CMP_EQ_U32_e32 does not lead to a crash.
+
+---
+name: depends_on_exec_check_can_handle_vopc
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: depends_on_exec_check_can_handle_vopc
+    ; CHECK: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; CHECK-NEXT: V_CMP_EQ_U32_e32 1, killed undef [[DEF]], implicit-def $vcc_lo, implicit $exec
+    ; CHECK-NEXT: [[V_CNDMASK_B32_e32_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e32 16256, undef [[DEF]], implicit $vcc_lo, implicit $exec
+    ; CHECK-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, undef [[V_CNDMASK_B32_e32_]], implicit $exec
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = IMPLICIT_DEF
+    V_CMP_EQ_U32_e32 1, killed undef %0, implicit-def $vcc, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e32 16256, undef %0, implicit $vcc, implicit $exec
+    V_CMP_EQ_U32_e32 1, killed undef %0, implicit-def $vcc, implicit $exec
+    %2:vgpr_32 = V_CNDMASK_B32_e32 16256, undef %0, implicit $vcc, implicit $exec
+    %3:vgpr_32 = V_LSHLREV_B32_e64 16, killed undef %2, implicit $exec
+    %4:vgpr_32 = V_LSHLREV_B32_e64 16, killed undef %1, implicit $exec
+    SI_RETURN
+...

From ef72b936262bc712d65756df1ad3ee095c794e0e Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Tue, 22 Apr 2025 09:33:52 +0100
Subject: [PATCH 693/710] [LV] Use requested calling convention for vector math
 routines (#136122)

Some vector math routines, e.g. ArmPL, specify a particular
calling convention on the routines which can help improve
performance by specifying what registers have to be preserved
across the call.
---
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |   4 +-
 .../AArch64/veclib-function-calls.ll          | 160 +++++++++---------
 .../AArch64/veclib-intrinsic-calls.ll         |  68 ++++----
 3 files changed, 116 insertions(+), 116 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ce64ba7cd2714..651f44a0ec4fa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1207,6 +1207,7 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
 
 void VPWidenCallRecipe::execute(VPTransformState &State) {
   assert(State.VF.isVector() && "not widening");
+  assert(Variant != nullptr && "Can't create vector function.");
 
   FunctionType *VFTy = Variant->getFunctionType();
   // Add return type if intrinsic is overloaded on it.
@@ -1223,8 +1224,6 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
     Args.push_back(Arg);
   }
 
-  assert(Variant != nullptr && "Can't create vector function.");
-
   auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
   SmallVector<OperandBundleDef, 1> OpBundles;
   if (CI)
@@ -1232,6 +1231,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
 
   CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);
   applyFlags(*V);
+  V->setCallingConv(Variant->getCallingConv());
 
   if (!V->getType()->isVoidTy())
     State.set(this, V);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
index 12de7ac0471b8..c6ea44bb85f11 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
@@ -34,7 +34,7 @@ define void @acos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
@@ -79,7 +79,7 @@ define void @acos_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -127,7 +127,7 @@ define void @acosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @acosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vacoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vacoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -172,7 +172,7 @@ define void @acosh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @acosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vacoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vacoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -220,7 +220,7 @@ define void @asin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -265,7 +265,7 @@ define void @asin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -313,7 +313,7 @@ define void @asinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @asinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vasinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vasinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -358,7 +358,7 @@ define void @asinh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @asinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vasinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vasinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -406,7 +406,7 @@ define void @atan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -451,7 +451,7 @@ define void @atan_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -499,7 +499,7 @@ define void @atan2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -544,7 +544,7 @@ define void @atan2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -592,7 +592,7 @@ define void @atanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vatanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -637,7 +637,7 @@ define void @atanh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vatanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -685,7 +685,7 @@ define void @cbrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cbrt_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcbrtq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vcbrtq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cbrt_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -730,7 +730,7 @@ define void @cbrt_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cbrt_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcbrtq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vcbrtq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cbrt_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -778,7 +778,7 @@ define void @copysign_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcopysignq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vcopysignq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -823,7 +823,7 @@ define void @copysign_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcopysignq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vcopysignq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -871,7 +871,7 @@ define void @cos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vcosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -916,7 +916,7 @@ define void @cos_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vcosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -964,7 +964,7 @@ define void @cosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1009,7 +1009,7 @@ define void @cosh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1057,7 +1057,7 @@ define void @cospi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cospi_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcospiq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vcospiq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cospi_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1102,7 +1102,7 @@ define void @cospi_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cospi_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcospiq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vcospiq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cospi_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1150,7 +1150,7 @@ define void @erf_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @erf_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_verfq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_verfq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @erf_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1195,7 +1195,7 @@ define void @erf_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @erf_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_verfq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_verfq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @erf_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1243,7 +1243,7 @@ define void @erfc_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @erfc_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_verfcq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_verfcq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @erfc_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1288,7 +1288,7 @@ define void @erfc_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @erfc_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_verfcq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_verfcq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @erfc_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1336,7 +1336,7 @@ define void @exp_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexpq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vexpq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1381,7 +1381,7 @@ define void @exp_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexpq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vexpq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1429,7 +1429,7 @@ define void @exp10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexp10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vexp10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1474,7 +1474,7 @@ define void @exp10_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexp10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vexp10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1522,7 +1522,7 @@ define void @exp2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexp2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vexp2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1567,7 +1567,7 @@ define void @exp2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexp2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vexp2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1615,7 +1615,7 @@ define void @expm1_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @expm1_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexpm1q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vexpm1q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @expm1_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1660,7 +1660,7 @@ define void @expm1_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @expm1_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexpm1q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vexpm1q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @expm1_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1708,7 +1708,7 @@ define void @fdim_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fdim_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfdimq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vfdimq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fdim_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1753,7 +1753,7 @@ define void @fdim_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fdim_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfdimq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vfdimq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fdim_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1801,7 +1801,7 @@ define void @fma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfmaq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vfmaq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1846,7 +1846,7 @@ define void @fma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfmaq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vfmaq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1894,7 +1894,7 @@ define void @fmax_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fmax_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfmaxq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vfmaxq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmax_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1939,7 +1939,7 @@ define void @fmax_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fmax_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfmaxq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vfmaxq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmax_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1987,7 +1987,7 @@ define void @fmin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fmin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfminq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vfminq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2032,7 +2032,7 @@ define void @fmin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fmin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfminq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vfminq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2080,7 +2080,7 @@ define void @fmod_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fmod_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vfmodq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vfmodq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmod_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2125,7 +2125,7 @@ define void @fmod_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @fmod_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vfmodq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vfmodq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @fmod_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2173,7 +2173,7 @@ define void @hypot_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @hypot_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vhypotq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vhypotq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @hypot_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2218,7 +2218,7 @@ define void @hypot_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @hypot_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vhypotq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vhypotq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @hypot_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2266,7 +2266,7 @@ define void @ilogb_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @ilogb_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x i32> @armpl_vilogbq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x i32> @armpl_vilogbq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ilogb_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2311,7 +2311,7 @@ define void @ilogb_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @ilogb_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x i32> @armpl_vilogbq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x i32> @armpl_vilogbq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ilogb_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2359,7 +2359,7 @@ define void @ldexp_f64(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %
 ;
 ; ARMPL-NEON-LABEL: define void @ldexp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP4:%.*]] = call <2 x double> @armpl_vldexpq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x i32> [[WIDE_LOAD1:%.*]])
+; ARMPL-NEON:    [[TMP4:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vldexpq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x i32> [[WIDE_LOAD1:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ldexp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2406,7 +2406,7 @@ define void @ldexp_f32(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %
 ;
 ; ARMPL-NEON-LABEL: define void @ldexp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP4:%.*]] = call <4 x float> @armpl_vldexpq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x i32> [[WIDE_LOAD1:%.*]])
+; ARMPL-NEON:    [[TMP4:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vldexpq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x i32> [[WIDE_LOAD1:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @ldexp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2456,7 +2456,7 @@ define void @lgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @lgamma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlgammaq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vlgammaq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @lgamma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2501,7 +2501,7 @@ define void @lgamma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @lgamma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlgammaq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vlgammaq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @lgamma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2549,7 +2549,7 @@ define void @log_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlogq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vlogq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2594,7 +2594,7 @@ define void @log_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlogq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vlogq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2642,7 +2642,7 @@ define void @log10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vlog10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2687,7 +2687,7 @@ define void @log10_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vlog10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2735,7 +2735,7 @@ define void @log1p_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log1p_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog1pq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vlog1pq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log1p_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2780,7 +2780,7 @@ define void @log1p_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log1p_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog1pq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vlog1pq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log1p_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2828,7 +2828,7 @@ define void @log2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vlog2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2873,7 +2873,7 @@ define void @log2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vlog2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3015,7 +3015,7 @@ define void @nextafter_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @nextafter_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vnextafterq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vnextafterq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @nextafter_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3060,7 +3060,7 @@ define void @nextafter_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @nextafter_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vnextafterq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vnextafterq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @nextafter_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3108,7 +3108,7 @@ define void @pow_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vpowq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vpowq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3153,7 +3153,7 @@ define void @pow_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vpowq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vpowq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3201,7 +3201,7 @@ define void @sin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vsinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3246,7 +3246,7 @@ define void @sin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vsinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3478,7 +3478,7 @@ define void @sinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3523,7 +3523,7 @@ define void @sinh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3571,7 +3571,7 @@ define void @sinpi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sinpi_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinpiq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vsinpiq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinpi_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3616,7 +3616,7 @@ define void @sinpi_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sinpi_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinpiq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vsinpiq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinpi_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3664,7 +3664,7 @@ define void @sqrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsqrtq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vsqrtq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3709,7 +3709,7 @@ define void @sqrt_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsqrtq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vsqrtq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3757,7 +3757,7 @@ define void @tan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3802,7 +3802,7 @@ define void @tan_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3850,7 +3850,7 @@ define void @tanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3895,7 +3895,7 @@ define void @tanh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3943,7 +3943,7 @@ define void @tgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tgamma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtgammaq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vtgammaq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tgamma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3988,7 +3988,7 @@ define void @tgamma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tgamma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtgammaq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vtgammaq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tgamma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
index d0a98bddf08ab..f753df32d9ebc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
@@ -26,7 +26,7 @@ define void @acos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vacosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
@@ -61,7 +61,7 @@ define void @acos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vacosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -99,7 +99,7 @@ define void @asin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vasinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -134,7 +134,7 @@ define void @asin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vasinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -172,7 +172,7 @@ define void @atan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vatanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -207,7 +207,7 @@ define void @atan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vatanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -245,7 +245,7 @@ define void @atan2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vatan2q_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -280,7 +280,7 @@ define void @atan2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vatan2q_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -464,7 +464,7 @@ define void @cos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vcosq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -499,7 +499,7 @@ define void @cos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vcosq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -537,7 +537,7 @@ define void @cosh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vcoshq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -572,7 +572,7 @@ define void @cosh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vcoshq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -610,7 +610,7 @@ define void @exp_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexpq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vexpq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -645,7 +645,7 @@ define void @exp_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexpq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vexpq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -683,7 +683,7 @@ define void @exp10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexp10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vexp10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -718,7 +718,7 @@ define void @exp10_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexp10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vexp10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -756,7 +756,7 @@ define void @exp2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vexp2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vexp2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -791,7 +791,7 @@ define void @exp2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vexp2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vexp2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1048,7 +1048,7 @@ define void @log_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlogq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vlogq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1083,7 +1083,7 @@ define void @log_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlogq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vlogq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1121,7 +1121,7 @@ define void @log10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vlog10q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1156,7 +1156,7 @@ define void @log10_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vlog10q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1194,7 +1194,7 @@ define void @log2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vlog2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vlog2q_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1229,7 +1229,7 @@ define void @log2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vlog2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vlog2q_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1486,7 +1486,7 @@ define void @pow_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vpowq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vpowq_f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1521,7 +1521,7 @@ define void @pow_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vpowq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vpowq_f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]])
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1705,7 +1705,7 @@ define void @sin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vsinq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1740,7 +1740,7 @@ define void @sin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vsinq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1778,7 +1778,7 @@ define void @sinh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vsinhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1813,7 +1813,7 @@ define void @sinh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vsinhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1924,7 +1924,7 @@ define void @tan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vtanq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1959,7 +1959,7 @@ define void @tan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vtanq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1997,7 +1997,7 @@ define void @tanh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <2 x double> @armpl_vtanhq_f64(<2 x double> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -2032,7 +2032,7 @@ define void @tanh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-NEON:    [[TMP2:%.*]] = call <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
+; ARMPL-NEON:    [[TMP2:%.*]] = call aarch64_vector_pcs <4 x float> @armpl_vtanhq_f32(<4 x float> [[WIDE_LOAD:%.*]])
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {

From 3334c3597dd51f5a102e5005738e3bf4ef7530e2 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Tue, 22 Apr 2025 11:40:12 +0300
Subject: [PATCH 694/710] [TTI] Fix discrepancies in prototypes between
 interface and implementations (NFCI) (#136655)

These are not diagnosed because implementations hide the methods of the base class rather than overriding them.
This works as long as a hiding function is callable with the same arguments as the same function from the base class.

Pull Request: https://github.com/llvm/llvm-project/pull/136655
---
 .../llvm/Analysis/TargetTransformInfo.h       |  4 +--
 .../llvm/Analysis/TargetTransformInfoImpl.h   | 19 +++++++-------
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      | 10 +++-----
 .../AArch64/AArch64TargetTransformInfo.cpp    |  7 +++---
 .../AArch64/AArch64TargetTransformInfo.h      |  2 +-
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      |  4 +--
 .../Target/AMDGPU/AMDGPUTargetTransformInfo.h |  4 +--
 .../Target/AMDGPU/R600TargetTransformInfo.cpp |  3 ++-
 .../Target/AMDGPU/R600TargetTransformInfo.h   |  2 +-
 .../lib/Target/ARM/ARMTargetTransformInfo.cpp |  5 ++--
 llvm/lib/Target/ARM/ARMTargetTransformInfo.h  |  2 +-
 llvm/lib/Target/BPF/BPFTargetTransformInfo.h  |  4 +--
 .../Hexagon/HexagonTargetTransformInfo.cpp    | 25 ++++++++-----------
 .../Hexagon/HexagonTargetTransformInfo.h      |  9 ++++---
 .../Target/NVPTX/NVPTXTargetTransformInfo.h   |  2 +-
 .../Target/PowerPC/PPCTargetTransformInfo.cpp | 22 ++++++++--------
 .../Target/PowerPC/PPCTargetTransformInfo.h   |  7 +++---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  2 +-
 .../Target/RISCV/RISCVTargetTransformInfo.h   |  2 +-
 .../SystemZ/SystemZTargetTransformInfo.cpp    |  6 ++---
 .../SystemZ/SystemZTargetTransformInfo.h      |  2 +-
 llvm/lib/Target/VE/VETargetTransformInfo.h    |  8 +++---
 .../WebAssemblyTargetTransformInfo.cpp        |  2 +-
 .../WebAssemblyTargetTransformInfo.h          |  2 +-
 .../lib/Target/X86/X86TargetTransformInfo.cpp | 18 ++++++-------
 llvm/lib/Target/X86/X86TargetTransformInfo.h  |  2 +-
 26 files changed, 84 insertions(+), 91 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index b5d766c34d09d..918a02fb48e70 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -3101,10 +3101,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.areTypesABICompatible(Caller, Callee, Types);
   }
   bool isIndexedLoadLegal(MemIndexedMode Mode, Type *Ty) const override {
-    return Impl.isIndexedLoadLegal(Mode, Ty, getDataLayout());
+    return Impl.isIndexedLoadLegal(Mode, Ty);
   }
   bool isIndexedStoreLegal(MemIndexedMode Mode, Type *Ty) const override {
-    return Impl.isIndexedStoreLegal(Mode, Ty, getDataLayout());
+    return Impl.isIndexedStoreLegal(Mode, Ty);
   }
   unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const override {
     return Impl.getLoadStoreVecRegBitWidth(AddrSpace);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index d28803cf164a3..b46eb349c2249 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -471,7 +471,9 @@ class TargetTransformInfoImplBase {
 
   bool haveFastSqrt(Type *Ty) const { return false; }
 
-  bool isExpensiveToSpeculativelyExecute(const Instruction *I) { return true; }
+  bool isExpensiveToSpeculativelyExecute(const Instruction *I) const {
+    return true;
+  }
 
   bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { return true; }
 
@@ -745,9 +747,10 @@ class TargetTransformInfoImplBase {
     return 1;
   }
 
-  unsigned getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
-                                     const APInt &DemandedDstElts,
-                                     TTI::TargetCostKind CostKind) const {
+  InstructionCost
+  getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
+                            const APInt &DemandedDstElts,
+                            TTI::TargetCostKind CostKind) const {
     return 1;
   }
 
@@ -805,7 +808,7 @@ class TargetTransformInfoImplBase {
     return InstructionCost::getInvalid();
   }
 
-  unsigned getInterleavedMemoryOpCost(
+  InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
       bool UseMaskForCond, bool UseMaskForGaps) const {
@@ -965,13 +968,11 @@ class TargetTransformInfoImplBase {
             Callee->getFnAttribute("target-features"));
   }
 
-  bool isIndexedLoadLegal(TTI::MemIndexedMode Mode, Type *Ty,
-                          const DataLayout &DL) const {
+  bool isIndexedLoadLegal(TTI::MemIndexedMode Mode, Type *Ty) const {
     return false;
   }
 
-  bool isIndexedStoreLegal(TTI::MemIndexedMode Mode, Type *Ty,
-                           const DataLayout &DL) const {
+  bool isIndexedStoreLegal(TTI::MemIndexedMode Mode, Type *Ty) const {
     return false;
   }
 
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index ca32d36297beb..db5fb2f7f1a54 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -475,19 +475,17 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return VF;
   }
 
-  bool isIndexedLoadLegal(TTI::MemIndexedMode M, Type *Ty,
-                          const DataLayout &DL) const {
+  bool isIndexedLoadLegal(TTI::MemIndexedMode M, Type *Ty) const {
     EVT VT = getTLI()->getValueType(DL, Ty);
     return getTLI()->isIndexedLoadLegal(getISDIndexedMode(M), VT);
   }
 
-  bool isIndexedStoreLegal(TTI::MemIndexedMode M, Type *Ty,
-                           const DataLayout &DL) const {
+  bool isIndexedStoreLegal(TTI::MemIndexedMode M, Type *Ty) const {
     EVT VT = getTLI()->getValueType(DL, Ty);
     return getTLI()->isIndexedStoreLegal(getISDIndexedMode(M), VT);
   }
 
-  bool isLSRCostLess(TTI::LSRCost C1, TTI::LSRCost C2) const {
+  bool isLSRCostLess(const TTI::LSRCost &C1, const TTI::LSRCost &C2) const {
     return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
   }
 
@@ -1468,7 +1466,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 720daa384968c..594f1bff5c458 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4367,7 +4367,7 @@ bool AArch64TTIImpl::useNeonVector(const Type *Ty) const {
 }
 
 InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
-                                                MaybeAlign Alignment,
+                                                Align Alignment,
                                                 unsigned AddressSpace,
                                                 TTI::TargetCostKind CostKind,
                                                 TTI::OperandValueInfo OpInfo,
@@ -4402,7 +4402,7 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
     return 1;
 
   if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store &&
-      LT.second.is128BitVector() && (!Alignment || *Alignment < Align(16))) {
+      LT.second.is128BitVector() && Alignment < Align(16)) {
     // Unaligned stores are extremely inefficient. We don't split all
     // unaligned 128-bit stores because the negative impact that has shown in
     // practice on inlined block copy code.
@@ -4429,8 +4429,7 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
     EVT EltVT = VT.getVectorElementType();
     unsigned EltSize = EltVT.getScalarSizeInBits();
     if (!isPowerOf2_32(EltSize) || EltSize < 8 || EltSize > 64 ||
-        VT.getVectorNumElements() >= (128 / EltSize) || !Alignment ||
-        *Alignment != Align(1))
+        VT.getVectorNumElements() >= (128 / EltSize) || Alignment != Align(1))
       return LT.first;
     // FIXME: v3i8 lowering currently is very inefficient, due to automatic
     // widening to v4i8, which produces suboptimal results.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 44b076b9a1c75..7da2820bee323 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -249,7 +249,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   bool useNeonVector(const Type *Ty) const;
 
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 919e73a2c534d..cc2ca77ffb792 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -274,7 +274,7 @@ void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 
-int64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
+uint64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
   return 1024;
 }
 
@@ -412,7 +412,7 @@ bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
   return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
 }
 
-int64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
+uint64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
   return 1024;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 1a575c92cce61..57682ff8ed76f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -57,7 +57,7 @@ class AMDGPUTTIImpl final : public BasicTTIImplBase<AMDGPUTTIImpl> {
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
                              TTI::PeelingPreferences &PP) const;
 
-  int64_t getMaxMemIntrinsicInlineSizeThreshold() const;
+  uint64_t getMaxMemIntrinsicInlineSizeThreshold() const;
 };
 
 class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
@@ -137,7 +137,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
                                     unsigned AddrSpace) const;
 
-  int64_t getMaxMemIntrinsicInlineSizeThreshold() const;
+  uint64_t getMaxMemIntrinsicInlineSizeThreshold() const;
   Type *
   getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
                             unsigned SrcAddrSpace, unsigned DestAddrSpace,
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
index 6cc2611a8d1a8..bbfd189d08a76 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
@@ -32,7 +32,8 @@ unsigned R600TTIImpl::getHardwareNumberOfRegisters(bool Vec) const {
   return 4 * 128; // XXX - 4 channels. Should these count as vector instead?
 }
 
-unsigned R600TTIImpl::getNumberOfRegisters(bool Vec) const {
+unsigned R600TTIImpl::getNumberOfRegisters(unsigned ClassID) const {
+  bool Vec = ClassID == 1;
   return getHardwareNumberOfRegisters(Vec);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
index 527b225c4eed3..163d01516efa7 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
@@ -47,7 +47,7 @@ class R600TTIImpl final : public BasicTTIImplBase<R600TTIImpl> {
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
                              TTI::PeelingPreferences &PP) const;
   unsigned getHardwareNumberOfRegisters(bool Vec) const;
-  unsigned getNumberOfRegisters(bool Vec) const;
+  unsigned getNumberOfRegisters(unsigned ClassID) const;
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const;
   unsigned getMinVectorRegisterBitWidth() const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 89740a3d7f477..76eb01033c3cb 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1545,7 +1545,7 @@ InstructionCost ARMTTIImpl::getArithmeticInstrCost(
 }
 
 InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
-                                            MaybeAlign Alignment,
+                                            Align Alignment,
                                             unsigned AddressSpace,
                                             TTI::TargetCostKind CostKind,
                                             TTI::OperandValueInfo OpInfo,
@@ -1559,8 +1559,7 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
     return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                   CostKind);
 
-  if (ST->hasNEON() && Src->isVectorTy() &&
-      (Alignment && *Alignment != Align(16)) &&
+  if (ST->hasNEON() && Src->isVectorTy() && Alignment != Align(16) &&
       cast<VectorType>(Src)->getElementType()->isDoubleTy()) {
     // Unaligned loads/stores are extremely inefficient.
     // We need 4 uops for vst.1/vld.1 vs 1uop for vldr/vstr.
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index b594f07e6a5a9..2a3ad431cc169 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -263,7 +263,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
       const Instruction *CxtI = nullptr) const;
 
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;
diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index 05a53c8bab824..9667f7e8f9e64 100644
--- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -37,8 +37,8 @@ class BPFTTIImpl : public BasicTTIImplBase<BPFTTIImpl> {
       : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
-  int getIntImmCost(const APInt &Imm, Type *Ty,
-                    TTI::TargetCostKind CostKind) const {
+  InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
+                                TTI::TargetCostKind CostKind) const {
     if (Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue()))
       return TTI::TCC_Free;
 
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 7825024f9ac0f..d6c1750e862c2 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -104,7 +104,8 @@ HexagonTTIImpl::getPreferredAddressingMode(const Loop *L,
 
 /// --- Vector TTI begin ---
 
-unsigned HexagonTTIImpl::getNumberOfRegisters(bool Vector) const {
+unsigned HexagonTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
+  bool Vector = ClassID == 1;
   if (Vector)
     return useHVX() ? 32 : 0;
   return 32;
@@ -162,7 +163,7 @@ InstructionCost HexagonTTIImpl::getAddressComputationCost(Type *Tp,
 }
 
 InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
-                                                MaybeAlign Alignment,
+                                                Align Alignment,
                                                 unsigned AddressSpace,
                                                 TTI::TargetCostKind CostKind,
                                                 TTI::OperandValueInfo OpInfo,
@@ -189,10 +190,9 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
         return VecWidth / RegWidth;
       // Cost of constructing HVX vector from scalar loads
       const Align RegAlign(RegWidth / 8);
-      if (!Alignment || *Alignment > RegAlign)
+      if (Alignment > RegAlign)
         Alignment = RegAlign;
-      assert(Alignment);
-      unsigned AlignWidth = 8 * Alignment->value();
+      unsigned AlignWidth = 8 * Alignment.value();
       unsigned NumLoads = alignTo(VecWidth, AlignWidth) / AlignWidth;
       return 3 * NumLoads;
     }
@@ -203,7 +203,7 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
         VecTy->getElementType()->isFloatingPointTy() ? FloatFactor : 1;
 
     // At this point unspecified alignment is considered as Align(1).
-    const Align BoundAlignment = std::min(Alignment.valueOrOne(), Align(8));
+    const Align BoundAlignment = std::min(Alignment, Align(8));
     unsigned AlignWidth = 8 * BoundAlignment.value();
     unsigned NumLoads = alignTo(VecWidth, AlignWidth) / AlignWidth;
     if (Alignment == Align(4) || Alignment == Align(8))
@@ -226,12 +226,10 @@ HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                       CostKind);
 }
 
-InstructionCost HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
-                                               ArrayRef<int> Mask,
-                                               TTI::TargetCostKind CostKind,
-                                               int Index, Type *SubTp,
-                                               ArrayRef<const Value *> Args,
-                                               const Instruction *CxtI) const {
+InstructionCost HexagonTTIImpl::getShuffleCost(
+    TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
+    TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
+    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
   return 1;
 }
 
@@ -251,8 +249,7 @@ InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost(
                                              Alignment, AddressSpace,
                                              CostKind,
                                              UseMaskForCond, UseMaskForGaps);
-  return getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace,
-                         CostKind);
+  return getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind);
 }
 
 InstructionCost HexagonTTIImpl::getCmpSelInstrCost(
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 9d0c3843e2a7a..b9dc41b6f4fe1 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -81,7 +81,7 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   /// \name Vector TTI Implementations
   /// @{
 
-  unsigned getNumberOfRegisters(bool vector) const;
+  unsigned getNumberOfRegisters(unsigned ClassID) const;
   unsigned getMaxInterleaveFactor(ElementCount VF) const;
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getMinVectorRegisterBitWidth() const;
@@ -107,17 +107,18 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
   InstructionCost getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
                                             const SCEV *S) const;
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
                                         TTI::TargetCostKind CostKind) const;
-  InstructionCost getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
+  InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
-                                 Type *SubTp, ArrayRef<const Value *> Args = {},
+                                 VectorType *SubTp,
+                                 ArrayRef<const Value *> Args = {},
                                  const Instruction *CxtI = nullptr) const;
   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
                                          const Value *Ptr, bool VariableMask,
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index ecd45a01ffbf2..b57efe8360d4e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -74,7 +74,7 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
   // vectorizers but disables heuristics based on the number of registers.
   // FIXME: Return a more reasonable number, while keeping an eye on
   // LoopVectorizer's unrolling heuristics.
-  unsigned getNumberOfRegisters(bool Vector) const { return 1; }
+  unsigned getNumberOfRegisters(unsigned ClassID) const { return 1; }
 
   // Only <2 x half> should be vectorized, so always return 32 for the vector
   // register size.
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 762e2d4d28d27..eb487bdaa88b9 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -599,10 +599,10 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(
   return Cost * CostFactor;
 }
 
-InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
-                                           ArrayRef<int> Mask,
+InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
+                                           VectorType *Tp, ArrayRef<int> Mask,
                                            TTI::TargetCostKind CostKind,
-                                           int Index, Type *SubTp,
+                                           int Index, VectorType *SubTp,
                                            ArrayRef<const Value *> Args,
                                            const Instruction *CxtI) const {
 
@@ -759,7 +759,7 @@ InstructionCost PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 }
 
 InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
-                                            MaybeAlign Alignment,
+                                            Align Alignment,
                                             unsigned AddressSpace,
                                             TTI::TargetCostKind CostKind,
                                             TTI::OperandValueInfo OpInfo,
@@ -803,13 +803,12 @@ InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
       return 1;
 
     // Use lfiwax/xxspltw
-    Align AlignBytes = Alignment ? *Alignment : Align(1);
-    if (Opcode == Instruction::Load && MemBits == 32 && AlignBytes < SrcBytes)
+    if (Opcode == Instruction::Load && MemBits == 32 && Alignment < SrcBytes)
       return 2;
   }
 
   // Aligned loads and stores are easy.
-  if (!SrcBytes || !Alignment || *Alignment >= SrcBytes)
+  if (!SrcBytes || Alignment >= SrcBytes)
     return Cost;
 
   // If we can use the permutation-based load sequence, then this is also
@@ -820,7 +819,7 @@ InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   // than using the permutation-based load sequence. On the P8, that's no
   // longer true.
   if (Opcode == Instruction::Load && (!ST->hasP8Vector() && IsAltivecType) &&
-      *Alignment >= LT.second.getScalarType().getStoreSize())
+      Alignment >= LT.second.getScalarType().getStoreSize())
     return Cost + LT.first; // Add the cost of the permutations.
 
   // For VSX, we can do unaligned loads and stores on Altivec/VSX types. On the
@@ -838,8 +837,7 @@ InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   // to be decomposed based on the alignment factor.
 
   // Add the cost of each scalar load or store.
-  assert(Alignment);
-  Cost += LT.first * ((SrcBytes / Alignment->value()) - 1);
+  Cost += LT.first * ((SrcBytes / Alignment.value()) - 1);
 
   // For a vector type, there is also scalarization overhead (only for
   // stores, loads are expanded using the vector-load + permutation sequence,
@@ -874,8 +872,8 @@ InstructionCost PPCTTIImpl::getInterleavedMemoryOpCost(
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VecTy);
 
   // Firstly, the cost of load/store operation.
-  InstructionCost Cost = getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment),
-                                         AddressSpace, CostKind);
+  InstructionCost Cost =
+      getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind);
 
   // PPC, for both Altivec/VSX, support cheap arbitrary permutations
   // (at least in the sense that there need only be one non-loop-invariant
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 9e4c40bb39295..d5428c7e4e24c 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -107,10 +107,11 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {},
       const Instruction *CxtI = nullptr) const;
-  InstructionCost getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
+  InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
-                                 Type *SubTp, ArrayRef<const Value *> Args = {},
+                                 VectorType *SubTp,
+                                 ArrayRef<const Value *> Args = {},
                                  const Instruction *CxtI = nullptr) const;
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    TTI::CastContextHint CCH,
@@ -130,7 +131,7 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
                                      unsigned Index, Value *Op0,
                                      Value *Op1) const;
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index a6a14837bf473..b7700a57fa10c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1952,7 +1952,7 @@ RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,
 }
 
 InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
-                                              MaybeAlign Alignment,
+                                              Align Alignment,
                                               unsigned AddressSpace,
                                               TTI::TargetCostKind CostKind,
                                               TTI::OperandValueInfo OpInfo,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 21f00e80608b1..43a367b3d3e5a 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -215,7 +215,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
                                            TTI::TargetCostKind CostKind) const;
 
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpdInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index bfc50a02558cf..53270ac096bb0 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -372,8 +372,8 @@ void SystemZTTIImpl::getUnrollingPreferences(
       }
       if (isa<StoreInst>(&I)) {
         Type *MemAccessTy = I.getOperand(0)->getType();
-        NumStores += getMemoryOpCost(Instruction::Store, MemAccessTy,
-                                     std::nullopt, 0, TTI::TCK_RecipThroughput);
+        NumStores += getMemoryOpCost(Instruction::Store, MemAccessTy, Align(),
+                                     0, TTI::TCK_RecipThroughput);
       }
     }
 
@@ -1298,7 +1298,7 @@ static bool isBswapIntrinsicCall(const Value *V) {
 }
 
 InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
-                                                MaybeAlign Alignment,
+                                                Align Alignment,
                                                 unsigned AddressSpace,
                                                 TTI::TargetCostKind CostKind,
                                                 TTI::OperandValueInfo OpInfo,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 99faab7194892..cb993b7b2df11 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -129,7 +129,7 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   bool isFoldableLoad(const LoadInst *Ld,
                       const Instruction *&FoldedValue) const;
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;
diff --git a/llvm/lib/Target/VE/VETargetTransformInfo.h b/llvm/lib/Target/VE/VETargetTransformInfo.h
index 88b46ff128383..93092ae7221b9 100644
--- a/llvm/lib/Target/VE/VETargetTransformInfo.h
+++ b/llvm/lib/Target/VE/VETargetTransformInfo.h
@@ -133,18 +133,18 @@ class VETTIImpl : public BasicTTIImplBase<VETTIImpl> {
   }
 
   // Load & Store {
-  bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment,
+  bool isLegalMaskedLoad(Type *DataType, Align Alignment,
                          unsigned /*AddressSpace*/) const {
     return isVectorLaneType(*getLaneType(DataType));
   }
-  bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment,
+  bool isLegalMaskedStore(Type *DataType, Align Alignment,
                           unsigned /*AddressSpace*/) const {
     return isVectorLaneType(*getLaneType(DataType));
   }
-  bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) const {
+  bool isLegalMaskedGather(Type *DataType, Align Alignment) const {
     return isVectorLaneType(*getLaneType(DataType));
   };
-  bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) const {
+  bool isLegalMaskedScatter(Type *DataType, Align Alignment) const {
     return isVectorLaneType(*getLaneType(DataType));
   }
   // } Load & Store
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 406daa8a6f9b8..49aa94956bda6 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -142,7 +142,7 @@ InstructionCost WebAssemblyTTIImpl::getCastInstrCost(
 }
 
 InstructionCost WebAssemblyTTIImpl::getMemoryOpCost(
-    unsigned Opcode, Type *Ty, MaybeAlign Alignment, unsigned AddressSpace,
+    unsigned Opcode, Type *Ty, Align Alignment, unsigned AddressSpace,
     TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,
     const Instruction *I) const {
   if (!ST->hasSIMD128() || !isa<FixedVectorType>(Ty)) {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index b8b6051ec5f28..cb9adf7d1d5e9 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -73,7 +73,7 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
                                    TTI::TargetCostKind CostKind,
                                    const Instruction *I = nullptr) const;
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 363019986d76c..2375a8f992aa5 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5184,7 +5184,7 @@ X86TTIImpl::getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
 }
 
 InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
-                                            MaybeAlign Alignment,
+                                            Align Alignment,
                                             unsigned AddressSpace,
                                             TTI::TargetCostKind CostKind,
                                             TTI::OperandValueInfo OpInfo,
@@ -5293,8 +5293,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
       // Can we use this vector size, as per the remaining element count?
       // Iff the vector is naturally aligned, we can do a wide load regardless.
       if (NumEltRemaining < CurrNumEltPerOp &&
-          (!IsLoad || Alignment.valueOrOne() < CurrOpSizeBytes) &&
-          CurrOpSizeBytes != 1)
+          (!IsLoad || Alignment < CurrOpSizeBytes) && CurrOpSizeBytes != 1)
         break; // Try smalled vector size.
 
       // This isn't exactly right. We're using slow unaligned 32-byte accesses
@@ -5344,7 +5343,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
 
       SubVecEltsLeft -= CurrNumEltPerOp;
       NumEltRemaining -= CurrNumEltPerOp;
-      Alignment = commonAlignment(Alignment.valueOrOne(), CurrOpSizeBytes);
+      Alignment = commonAlignment(Alignment, CurrOpSizeBytes);
     }
   }
 
@@ -6183,8 +6182,7 @@ InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
   const int GSOverhead = (Opcode == Instruction::Load) ? getGatherOverhead()
                                                        : getScatterOverhead();
   return GSOverhead + VF * getMemoryOpCost(Opcode, SrcVTy->getScalarType(),
-                                           MaybeAlign(Alignment), AddressSpace,
-                                           CostKind);
+                                           Alignment, AddressSpace, CostKind);
 }
 
 /// Calculate the cost of Gather / Scatter operation
@@ -6596,8 +6594,8 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX512(
     MemOpCost = getMaskedMemoryOpCost(Opcode, SingleMemOpTy, Alignment,
                                       AddressSpace, CostKind);
   else
-    MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy, MaybeAlign(Alignment),
-                                AddressSpace, CostKind);
+    MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy, Alignment, AddressSpace,
+                                CostKind);
 
   unsigned VF = VecTy->getNumElements() / Factor;
   MVT VT =
@@ -6780,8 +6778,8 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCost(
 
   // Get the cost of all the memory operations.
   // FIXME: discount dead loads.
-  InstructionCost MemOpCosts = getMemoryOpCost(
-      Opcode, VecTy, MaybeAlign(Alignment), AddressSpace, CostKind);
+  InstructionCost MemOpCosts =
+      getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind);
 
   auto *VT = FixedVectorType::get(ScalarTy, VF);
   EVT ETy = TLI->getValueType(DL, VT);
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 1534301667ff1..ef7b7e90eeb5a 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -178,7 +178,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
                                             const APInt &DemandedDstElts,
                                             TTI::TargetCostKind CostKind) const;
   InstructionCost getMemoryOpCost(
-      unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace,
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
       const Instruction *I = nullptr) const;

From a35f940b876a09211f3e68dd25d00271b7195145 Mon Sep 17 00:00:00 2001
From: Christian Kandeler <christian.kandeler@qt.io>
Date: Tue, 22 Apr 2025 10:42:56 +0200
Subject: [PATCH 695/710] [clangd] Support operators new and delete in
 textDocument/references (#135620)

---
 clang-tools-extra/clangd/unittests/XRefsTests.cpp | 10 +++++++++-
 clang/lib/Index/IndexBody.cpp                     | 14 ++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index 693e965e78a96..1892f87c8e82a 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -2303,7 +2303,15 @@ TEST(FindReferences, WithinAST) {
         bool $decl[[operator]]"" _u^dl(unsigned long long value);
         bool x = $(x)[[1_udl]];
       )cpp",
-  };
+      R"cpp(
+        struct S {
+        public:
+          static void $decl(S)[[operator]] delete(void *);
+          static void deleteObject(S *S) {
+            $(S::deleteObject)[[de^lete]] S;
+          }
+        };
+      )cpp"};
   for (const char *Test : Tests)
     checkFindRefs(Test);
 }
diff --git a/clang/lib/Index/IndexBody.cpp b/clang/lib/Index/IndexBody.cpp
index 5e69987820730..2ed20df22bda0 100644
--- a/clang/lib/Index/IndexBody.cpp
+++ b/clang/lib/Index/IndexBody.cpp
@@ -153,6 +153,20 @@ class BodyIndexer : public RecursiveASTVisitor<BodyIndexer> {
                                     ParentDC);
   }
 
+  bool VisitCXXNewExpr(CXXNewExpr *E) {
+    if (E->isGlobalNew() || !E->getOperatorNew())
+      return true;
+    return IndexCtx.handleReference(E->getOperatorNew(), E->getBeginLoc(),
+                                    Parent, ParentDC);
+  }
+
+  bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
+    if (E->isGlobalDelete() || !E->getOperatorDelete())
+      return true;
+    return IndexCtx.handleReference(E->getOperatorDelete(), E->getBeginLoc(),
+                                    Parent, ParentDC);
+  }
+
   bool VisitLabelStmt(LabelStmt *S) {
     if (IndexCtx.shouldIndexFunctionLocalSymbols())
       return IndexCtx.handleDecl(S->getDecl());

From c3f815ba82defc84244a9688fd2578da513340fb Mon Sep 17 00:00:00 2001
From: anjenner <161845516+anjenner@users.noreply.github.com>
Date: Tue, 22 Apr 2025 09:45:15 +0100
Subject: [PATCH 696/710] =?UTF-8?q?Modify=20the=20localCache=20API=20to=20?=
 =?UTF-8?q?require=20an=20explicit=20commit=20on=20CachedFile=E2=80=A6=20(?=
 =?UTF-8?q?#136121)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…Stream.

CachedFileStream has previously performed the commit step in its
destructor, but this means its only recourse for error handling is
report_fatal_error. Modify this to add an explicit commit() method, and
call this in the appropriate places with appropriate error handling for
the location.

Currently the destructor of CacheStream gives an assert failure in Debug
builds if commit() was not called. This will help track down any
remaining uses of the API that assume the old destructior behaviour. In
Release builds we fall back to the previous behaviour and call
report_fatal_error if the commit fails.

This is version 2 of this PR, superseding reverted PR
https://github.com/llvm/llvm-project/pull/115331 . I have incorporated a
change to the testcase to make it more reliable on Windows, as well as
two follow-up changes
(https://github.com/llvm/llvm-project/commit/df79000896101acc9b8d7435e59f767b36c00ac8
and
https://github.com/llvm/llvm-project/commit/b0baa1d8bd68a2ce2f7c5f2b62333e410e9122a1)
that were also reverted when 115331 was reverted.

---------

Co-authored-by: Augie Fackler <augie@google.com>
Co-authored-by: Vitaly Buka <vitalybuka@google.com>
---
 lldb/source/Core/DataFileCache.cpp    |   5 +
 llvm/include/llvm/Support/Caching.h   |  21 +++-
 llvm/lib/CGData/CodeGenData.cpp       |   3 +
 llvm/lib/Debuginfod/Debuginfod.cpp    |  13 +++
 llvm/lib/LTO/LTOBackend.cpp           |   3 +
 llvm/lib/Support/Caching.cpp          |  29 +++--
 llvm/tools/gold/gold-plugin.cpp       |   6 +-
 llvm/tools/llvm-lto2/llvm-lto2.cpp    |   4 +-
 llvm/unittests/Support/CMakeLists.txt |   1 +
 llvm/unittests/Support/Caching.cpp    | 157 ++++++++++++++++++++++++++
 10 files changed, 225 insertions(+), 17 deletions(-)
 create mode 100644 llvm/unittests/Support/Caching.cpp

diff --git a/lldb/source/Core/DataFileCache.cpp b/lldb/source/Core/DataFileCache.cpp
index ef0e07a8b0342..9109269711231 100644
--- a/lldb/source/Core/DataFileCache.cpp
+++ b/lldb/source/Core/DataFileCache.cpp
@@ -132,6 +132,11 @@ bool DataFileCache::SetCachedData(llvm::StringRef key,
       if (file_or_err) {
         llvm::CachedFileStream *cfs = file_or_err->get();
         cfs->OS->write((const char *)data.data(), data.size());
+        if (llvm::Error err = cfs->commit()) {
+          Log *log = GetLog(LLDBLog::Modules);
+          LLDB_LOG_ERROR(log, std::move(err),
+                         "failed to commit to the cache for key: {0}");
+        }
         return true;
       } else {
         Log *log = GetLog(LLDBLog::Modules);
diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h
index cf45145619d95..9a82921e6ffc7 100644
--- a/llvm/include/llvm/Support/Caching.h
+++ b/llvm/include/llvm/Support/Caching.h
@@ -24,15 +24,32 @@ class MemoryBuffer;
 /// This class wraps an output stream for a file. Most clients should just be
 /// able to return an instance of this base class from the stream callback, but
 /// if a client needs to perform some action after the stream is written to,
-/// that can be done by deriving from this class and overriding the destructor.
+/// that can be done by deriving from this class and overriding the destructor
+/// or the commit() method.
 class CachedFileStream {
 public:
   CachedFileStream(std::unique_ptr<raw_pwrite_stream> OS,
                    std::string OSPath = "")
       : OS(std::move(OS)), ObjectPathName(OSPath) {}
+
+  /// Must be called exactly once after the writes to OS have been completed
+  /// but before the CachedFileStream object is destroyed.
+  virtual Error commit() {
+    if (Committed)
+      return createStringError(make_error_code(std::errc::invalid_argument),
+                               Twine("CacheStream already committed."));
+    Committed = true;
+
+    return Error::success();
+  }
+
+  bool Committed = false;
   std::unique_ptr<raw_pwrite_stream> OS;
   std::string ObjectPathName;
-  virtual ~CachedFileStream() = default;
+  virtual ~CachedFileStream() {
+    if (!Committed)
+      report_fatal_error("CachedFileStream was not committed.\n");
+  }
 };
 
 /// This type defines the callback to add a file that is generated on the fly.
diff --git a/llvm/lib/CGData/CodeGenData.cpp b/llvm/lib/CGData/CodeGenData.cpp
index 02de528c4d007..7b9c584d64867 100644
--- a/llvm/lib/CGData/CodeGenData.cpp
+++ b/llvm/lib/CGData/CodeGenData.cpp
@@ -233,6 +233,9 @@ void saveModuleForTwoRounds(const Module &TheModule, unsigned Task,
 
   WriteBitcodeToFile(TheModule, *Stream->OS,
                      /*ShouldPreserveUseListOrder=*/true);
+
+  if (Error Err = Stream->commit())
+    report_fatal_error(std::move(Err));
 }
 
 std::unique_ptr<Module> loadModuleForTwoRounds(BitcodeModule &OrigModule,
diff --git a/llvm/lib/Debuginfod/Debuginfod.cpp b/llvm/lib/Debuginfod/Debuginfod.cpp
index 4c785117ae8ef..db316a1b37b3b 100644
--- a/llvm/lib/Debuginfod/Debuginfod.cpp
+++ b/llvm/lib/Debuginfod/Debuginfod.cpp
@@ -188,6 +188,11 @@ class StreamedHTTPResponseHandler : public HTTPResponseHandler {
 public:
   StreamedHTTPResponseHandler(CreateStreamFn CreateStream, HTTPClient &Client)
       : CreateStream(CreateStream), Client(Client) {}
+
+  /// Must be called exactly once after the writes have been completed
+  /// but before the StreamedHTTPResponseHandler object is destroyed.
+  Error commit();
+
   virtual ~StreamedHTTPResponseHandler() = default;
 
   Error handleBodyChunk(StringRef BodyChunk) override;
@@ -210,6 +215,12 @@ Error StreamedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) {
   return Error::success();
 }
 
+Error StreamedHTTPResponseHandler::commit() {
+  if (FileStream)
+    return FileStream->commit();
+  return Error::success();
+}
+
 // An over-accepting simplification of the HTTP RFC 7230 spec.
 static bool isHeader(StringRef S) {
   StringRef Name;
@@ -298,6 +309,8 @@ Expected<std::string> getCachedOrDownloadArtifact(
       Error Err = Client.perform(Request, Handler);
       if (Err)
         return std::move(Err);
+      if ((Err = Handler.commit()))
+        return std::move(Err);
 
       unsigned Code = Client.responseCode();
       if (Code && Code != 200)
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 1c764a0188eda..dd9197efa7718 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -460,6 +460,9 @@ static void codegen(const Config &Conf, TargetMachine *TM,
 
   if (DwoOut)
     DwoOut->keep();
+
+  if (Error Err = Stream->commit())
+    report_fatal_error(std::move(Err));
 }
 
 static void splitCodeGen(const Config &C, TargetMachine *TM,
diff --git a/llvm/lib/Support/Caching.cpp b/llvm/lib/Support/Caching.cpp
index 66e540efaca97..40a5c44771b65 100644
--- a/llvm/lib/Support/Caching.cpp
+++ b/llvm/lib/Support/Caching.cpp
@@ -88,9 +88,10 @@ Expected<FileCache> llvm::localCache(const Twine &CacheNameRef,
             AddBuffer(std::move(AddBuffer)), TempFile(std::move(TempFile)),
             ModuleName(ModuleName), Task(Task) {}
 
-      ~CacheStream() {
-        // TODO: Manually commit rather than using non-trivial destructor,
-        // allowing to replace report_fatal_errors with a return Error.
+      Error commit() override {
+        Error E = CachedFileStream::commit();
+        if (E)
+          return E;
 
         // Make sure the stream is closed before committing it.
         OS.reset();
@@ -100,10 +101,12 @@ Expected<FileCache> llvm::localCache(const Twine &CacheNameRef,
             MemoryBuffer::getOpenFile(
                 sys::fs::convertFDToNativeFile(TempFile.FD), ObjectPathName,
                 /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
-        if (!MBOrErr)
-          report_fatal_error(Twine("Failed to open new cache file ") +
-                             TempFile.TmpName + ": " +
-                             MBOrErr.getError().message() + "\n");
+        if (!MBOrErr) {
+          std::error_code EC = MBOrErr.getError();
+          return createStringError(EC, Twine("Failed to open new cache file ") +
+                                           TempFile.TmpName + ": " +
+                                           EC.message() + "\n");
+        }
 
         // On POSIX systems, this will atomically replace the destination if
         // it already exists. We try to emulate this on Windows, but this may
@@ -114,11 +117,14 @@ Expected<FileCache> llvm::localCache(const Twine &CacheNameRef,
         // AddBuffer a copy of the bytes we wrote in that case. We do this
         // instead of just using the existing file, because the pruner might
         // delete the file before we get a chance to use it.
-        Error E = TempFile.keep(ObjectPathName);
+        E = TempFile.keep(ObjectPathName);
         E = handleErrors(std::move(E), [&](const ECError &E) -> Error {
           std::error_code EC = E.convertToErrorCode();
           if (EC != errc::permission_denied)
-            return errorCodeToError(EC);
+            return createStringError(
+                EC, Twine("Failed to rename temporary file ") +
+                        TempFile.TmpName + " to " + ObjectPathName + ": " +
+                        EC.message() + "\n");
 
           auto MBCopy = MemoryBuffer::getMemBufferCopy((*MBOrErr)->getBuffer(),
                                                        ObjectPathName);
@@ -131,11 +137,10 @@ Expected<FileCache> llvm::localCache(const Twine &CacheNameRef,
         });
 
         if (E)
-          report_fatal_error(Twine("Failed to rename temporary file ") +
-                             TempFile.TmpName + " to " + ObjectPathName + ": " +
-                             toString(std::move(E)) + "\n");
+          return E;
 
         AddBuffer(Task, ModuleName, std::move(*MBOrErr));
+        return Error::success();
       }
     };
 
diff --git a/llvm/tools/gold/gold-plugin.cpp b/llvm/tools/gold/gold-plugin.cpp
index ae965e6f486aa..256933d3f53f9 100644
--- a/llvm/tools/gold/gold-plugin.cpp
+++ b/llvm/tools/gold/gold-plugin.cpp
@@ -1117,9 +1117,11 @@ static std::vector<std::pair<SmallString<128>, bool>> runLTO() {
         std::make_unique<llvm::raw_fd_ostream>(FD, true));
   };
 
-  auto AddBuffer = [&](size_t Task, const Twine &moduleName,
+  auto AddBuffer = [&](size_t Task, const Twine &ModuleName,
                        std::unique_ptr<MemoryBuffer> MB) {
-    *AddStream(Task, moduleName)->OS << MB->getBuffer();
+    auto Stream = AddStream(Task, ModuleName);
+    *Stream->OS << MB->getBuffer();
+    check(Stream->commit(), "Failed to commit cache");
   };
 
   FileCache Cache;
diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp
index 33626c1fd9433..01cbbaae32a5c 100644
--- a/llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -443,7 +443,9 @@ static int run(int argc, char **argv) {
 
   auto AddBuffer = [&](size_t Task, const Twine &ModuleName,
                        std::unique_ptr<MemoryBuffer> MB) {
-    *AddStream(Task, ModuleName)->OS << MB->getBuffer();
+    auto Stream = AddStream(Task, ModuleName);
+    *Stream->OS << MB->getBuffer();
+    check(Stream->commit(), "Failed to commit cache");
   };
 
   FileCache Cache;
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 4a12a928af119..14c1915f4e2ee 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -18,6 +18,7 @@ add_llvm_unittest(SupportTests
   BranchProbabilityTest.cpp
   CachePruningTest.cpp
   CrashRecoveryTest.cpp
+  Caching.cpp
   Casting.cpp
   CheckedArithmeticTest.cpp
   Chrono.cpp
diff --git a/llvm/unittests/Support/Caching.cpp b/llvm/unittests/Support/Caching.cpp
new file mode 100644
index 0000000000000..581675deac066
--- /dev/null
+++ b/llvm/unittests/Support/Caching.cpp
@@ -0,0 +1,157 @@
+//===- Caching.cpp --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Caching.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+#define ASSERT_NO_ERROR(x)                                                     \
+  if (std::error_code ASSERT_NO_ERROR_ec = x) {                                \
+    SmallString<128> MessageStorage;                                           \
+    raw_svector_ostream Message(MessageStorage);                               \
+    Message << #x ": did not return errc::success.\n"                          \
+            << "error number: " << ASSERT_NO_ERROR_ec.value() << "\n"          \
+            << "error message: " << ASSERT_NO_ERROR_ec.message() << "\n";      \
+    GTEST_FATAL_FAILURE_(MessageStorage.c_str());                              \
+  } else {                                                                     \
+  }
+
+char data[] = "some data";
+
+TEST(Caching, Normal) {
+  SmallString<256> CacheDir;
+  sys::fs::createUniquePath("llvm_test_cache-%%%%%%", CacheDir, true);
+
+  sys::fs::remove_directories(CacheDir.str());
+
+  std::unique_ptr<MemoryBuffer> CachedBuffer;
+  auto AddBuffer = [&CachedBuffer](unsigned Task, const Twine &ModuleName,
+                                   std::unique_ptr<MemoryBuffer> M) {
+    CachedBuffer = std::move(M);
+  };
+  auto CacheOrErr =
+      localCache("LLVMTestCache", "LLVMTest", CacheDir, AddBuffer);
+  ASSERT_TRUE(bool(CacheOrErr));
+
+  FileCache &Cache = *CacheOrErr;
+
+  {
+    auto AddStreamOrErr = Cache(1, "foo", "");
+    ASSERT_TRUE(bool(AddStreamOrErr));
+
+    AddStreamFn &AddStream = *AddStreamOrErr;
+    ASSERT_TRUE(AddStream);
+
+    auto FileOrErr = AddStream(1, "");
+    ASSERT_TRUE(bool(FileOrErr));
+
+    CachedFileStream *CFS = FileOrErr->get();
+    (*CFS->OS).write(data, sizeof(data));
+    ASSERT_THAT_ERROR(CFS->commit(), Succeeded());
+  }
+
+  {
+    auto AddStreamOrErr = Cache(1, "foo", "");
+    ASSERT_TRUE(bool(AddStreamOrErr));
+
+    AddStreamFn &AddStream = *AddStreamOrErr;
+    ASSERT_FALSE(AddStream);
+
+    ASSERT_TRUE(CachedBuffer->getBufferSize() == sizeof(data));
+    StringRef readData = CachedBuffer->getBuffer();
+    ASSERT_TRUE(memcmp(data, readData.data(), sizeof(data)) == 0);
+  }
+
+  ASSERT_NO_ERROR(sys::fs::remove_directories(CacheDir.str()));
+}
+
+TEST(Caching, WriteAfterCommit) {
+  SmallString<256> CacheDir;
+  sys::fs::createUniquePath("llvm_test_cache-%%%%%%", CacheDir, true);
+
+  sys::fs::remove_directories(CacheDir.str());
+
+  std::unique_ptr<MemoryBuffer> CachedBuffer;
+  auto AddBuffer = [&CachedBuffer](unsigned Task, const Twine &ModuleName,
+                                   std::unique_ptr<MemoryBuffer> M) {
+    CachedBuffer = std::move(M);
+  };
+  auto CacheOrErr =
+      localCache("LLVMTestCache", "LLVMTest", CacheDir, AddBuffer);
+  ASSERT_TRUE(bool(CacheOrErr));
+
+  FileCache &Cache = *CacheOrErr;
+
+  auto AddStreamOrErr = Cache(1, "foo", "");
+  ASSERT_TRUE(bool(AddStreamOrErr));
+
+  AddStreamFn &AddStream = *AddStreamOrErr;
+  ASSERT_TRUE(AddStream);
+
+  auto FileOrErr = AddStream(1, "");
+  ASSERT_TRUE(bool(FileOrErr));
+
+  CachedFileStream *CFS = FileOrErr->get();
+  (*CFS->OS).write(data, sizeof(data));
+  ASSERT_THAT_ERROR(CFS->commit(), Succeeded());
+
+  EXPECT_DEATH(
+      { (*CFS->OS).write(data, sizeof(data)); }, "")
+      << "Write after commit did not cause abort";
+
+  ASSERT_NO_ERROR(sys::fs::remove_directories(CacheDir.str()));
+}
+
+TEST(Caching, NoCommit) {
+  SmallString<256> CacheDir;
+  sys::fs::createUniquePath("llvm_test_cache-%%%%%%", CacheDir, true);
+
+  sys::fs::remove_directories(CacheDir.str());
+
+  std::unique_ptr<MemoryBuffer> CachedBuffer;
+  auto AddBuffer = [&CachedBuffer](unsigned Task, const Twine &ModuleName,
+                                   std::unique_ptr<MemoryBuffer> M) {
+    CachedBuffer = std::move(M);
+  };
+  auto CacheOrErr =
+      localCache("LLVMTestCache", "LLVMTest", CacheDir, AddBuffer);
+  ASSERT_TRUE(bool(CacheOrErr));
+
+  FileCache &Cache = *CacheOrErr;
+
+  auto AddStreamOrErr = Cache(1, "foo", "");
+  ASSERT_TRUE(bool(AddStreamOrErr));
+
+  AddStreamFn &AddStream = *AddStreamOrErr;
+  ASSERT_TRUE(AddStream);
+
+  auto FileOrErr = AddStream(1, "");
+  ASSERT_TRUE(bool(FileOrErr));
+
+  CachedFileStream *CFS = FileOrErr->get();
+  (*CFS->OS).write(data, sizeof(data));
+  ASSERT_THAT_ERROR(CFS->commit(), Succeeded());
+
+  EXPECT_DEATH(
+      {
+        auto FileOrErr = AddStream(1, "");
+        ASSERT_TRUE(bool(FileOrErr));
+
+        CachedFileStream *CFS = FileOrErr->get();
+        (*CFS->OS).write(data, sizeof(data));
+      },
+      "")
+      << "destruction without commit did not cause error";
+
+  ASSERT_NO_ERROR(sys::fs::remove_directories(CacheDir.str()));
+}

From 382263376fcfed967f5bc17400d9e4542b37801c Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Tue, 22 Apr 2025 08:53:01 +0000
Subject: [PATCH 697/710] [lldb][docs] Repeat required Python version number

We do say it in the table below but if you didn't
want any optional stuff you'd miss it.
---
 lldb/docs/resources/build.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/docs/resources/build.rst b/lldb/docs/resources/build.rst
index 9ba2405fc9628..e59dcc1972418 100644
--- a/lldb/docs/resources/build.rst
+++ b/lldb/docs/resources/build.rst
@@ -30,7 +30,7 @@ The following requirements are shared on all platforms.
 If you want to run the test suite, you'll need to build LLDB with Python
 scripting support.
 
-* `Python <http://www.python.org/>`_
+* `Python <http://www.python.org/>`_ 3.8 or later.
 * `SWIG <http://swig.org/>`_ 4 or later.
 
 If you are on FreeBSD or NetBSD, you will need to install ``gmake`` for building

From 278c429d11e63bc709ea8c537b23c4e350ce2a07 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 22 Apr 2025 10:53:30 +0200
Subject: [PATCH 698/710] [ValueTracking] Drop ucmp/scmp from
 getIntrinsicRange() (NFCI)

After #135642 we have a range attribute on the intrinsic declaration,
so we should not need the special handling here.
---
 llvm/lib/Analysis/ValueTracking.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index cdf7f052943c8..1d3f8b7207a63 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -10146,10 +10146,6 @@ static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II,
     if (!II.getParent() || !II.getFunction())
       break;
     return getVScaleRange(II.getFunction(), Width);
-  case Intrinsic::scmp:
-  case Intrinsic::ucmp:
-    return ConstantRange::getNonEmpty(APInt::getAllOnes(Width),
-                                      APInt(Width, 2));
   default:
     break;
   }

From 4bcc414af3782c333f3d535c0e0a92e6120868f5 Mon Sep 17 00:00:00 2001
From: Robert Konicar <rkonicar@mail.muni.cz>
Date: Tue, 22 Apr 2025 11:00:56 +0200
Subject: [PATCH 699/710] [MLIR][TableGen] Error on APInt parameter without
 custom comparator (#135970)

The error is triggered when an attribute or type uses an APInt typed
parameter with the generated equality operator. If the compared APInts
have different bit widths the equality operator triggers an assert. This
is dangerous, since `StorageUniquer` for types and attributes uses the
equality operator when a hash collision appears. As such, it is
necessary to use custom provided comarator or `APIntParameter` that
already has it.
This commit also replaces uses of the raw `APInt` parameter with the
`APIntParameter` and removes the no longer necessary custom StorageClass
for the `BitVectorAttr` from the SMT dialect that was a workaround for
the described issue.

---------

Co-authored-by: Tobias Gysi <tobias.gysi@nextsilicon.com>
---
 .../mlir/Dialect/SMT/IR/SMTAttributes.td      | 12 +------
 mlir/include/mlir/IR/BuiltinAttributes.td     |  2 +-
 mlir/include/mlir/TableGen/AttrOrTypeDef.h    |  6 +++-
 mlir/lib/Dialect/SMT/IR/SMTAttributes.cpp     | 36 -------------------
 mlir/lib/TableGen/AttrOrTypeDef.cpp           |  4 +++
 mlir/test/mlir-tblgen/apint-param-error.td    | 17 +++++++++
 mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp   | 12 ++++++-
 7 files changed, 39 insertions(+), 50 deletions(-)
 create mode 100644 mlir/test/mlir-tblgen/apint-param-error.td

diff --git a/mlir/include/mlir/Dialect/SMT/IR/SMTAttributes.td b/mlir/include/mlir/Dialect/SMT/IR/SMTAttributes.td
index 0909c9abad951..c1c5bfc76e055 100644
--- a/mlir/include/mlir/Dialect/SMT/IR/SMTAttributes.td
+++ b/mlir/include/mlir/Dialect/SMT/IR/SMTAttributes.td
@@ -45,21 +45,11 @@ def BitVectorAttr : AttrDef<SMTDialect, "BitVector", [
     present).
   }];
 
-  let parameters = (ins "llvm::APInt":$value);
+  let parameters = (ins APIntParameter<"">:$value);
 
   let hasCustomAssemblyFormat = true;
   let genVerifyDecl = true;
 
-  // We need to manually define the storage class because the generated one is
-  // buggy (because the APInt asserts matching bitwidth in the `==` operator and
-  // the generated storage uses that directly.
-  // Alternatively: add a type parameter to redundantly store the bitwidth of
-  // of the attribute type, it it's in the order before the 'value' it will be
-  // checked before the APInt equality (this is the reason it works for the
-  // builtin integer attribute), but would be more fragile (and we'd store
-  // duplicate data).
-  let genStorageClass = false;
-
   let builders = [
     AttrBuilder<(ins "llvm::StringRef":$value)>,
     AttrBuilder<(ins "uint64_t":$value, "unsigned":$width)>,
diff --git a/mlir/include/mlir/IR/BuiltinAttributes.td b/mlir/include/mlir/IR/BuiltinAttributes.td
index 6826d1a437775..50dcb8de1f7e7 100644
--- a/mlir/include/mlir/IR/BuiltinAttributes.td
+++ b/mlir/include/mlir/IR/BuiltinAttributes.td
@@ -700,7 +700,7 @@ def Builtin_IntegerAttr : Builtin_Attr<"Integer", "integer",
     false // A bool, i.e. i1, value.
     ```
   }];
-  let parameters = (ins AttributeSelfTypeParameter<"">:$type, "APInt":$value);
+  let parameters = (ins AttributeSelfTypeParameter<"">:$type, APIntParameter<"">:$value);
   let builders = [
     AttrBuilderWithInferredContext<(ins "Type":$type,
                                         "const APInt &":$value), [{
diff --git a/mlir/include/mlir/TableGen/AttrOrTypeDef.h b/mlir/include/mlir/TableGen/AttrOrTypeDef.h
index c3d730e42ef70..8c1d399b39e0b 100644
--- a/mlir/include/mlir/TableGen/AttrOrTypeDef.h
+++ b/mlir/include/mlir/TableGen/AttrOrTypeDef.h
@@ -68,7 +68,11 @@ class AttrOrTypeParameter {
   /// If specified, get the custom allocator code for this parameter.
   std::optional<StringRef> getAllocator() const;
 
-  /// If specified, get the custom comparator code for this parameter.
+  /// Return true if user defined comparator is specified.
+  bool hasCustomComparator() const;
+
+  /// Get the custom comparator code for this parameter or fallback to the
+  /// default.
   StringRef getComparator() const;
 
   /// Get the C++ type of this parameter.
diff --git a/mlir/lib/Dialect/SMT/IR/SMTAttributes.cpp b/mlir/lib/Dialect/SMT/IR/SMTAttributes.cpp
index c28f3558a02d2..3f40d6a42eafd 100644
--- a/mlir/lib/Dialect/SMT/IR/SMTAttributes.cpp
+++ b/mlir/lib/Dialect/SMT/IR/SMTAttributes.cpp
@@ -21,42 +21,6 @@ using namespace mlir::smt;
 // BitVectorAttr
 //===----------------------------------------------------------------------===//
 
-namespace mlir {
-namespace smt {
-namespace detail {
-struct BitVectorAttrStorage : public mlir::AttributeStorage {
-  using KeyTy = APInt;
-  BitVectorAttrStorage(APInt value) : value(std::move(value)) {}
-
-  KeyTy getAsKey() const { return value; }
-
-  // NOTE: the implementation of this operator is the reason we need to define
-  // the storage manually. The auto-generated version would just do the direct
-  // equality check of the APInt, but that asserts the bitwidth of both to be
-  // the same, leading to a crash. This implementation, therefore, checks for
-  // matching bit-width beforehand.
-  bool operator==(const KeyTy &key) const {
-    return (value.getBitWidth() == key.getBitWidth() && value == key);
-  }
-
-  static llvm::hash_code hashKey(const KeyTy &key) {
-    return llvm::hash_value(key);
-  }
-
-  static BitVectorAttrStorage *
-  construct(mlir::AttributeStorageAllocator &allocator, KeyTy &&key) {
-    return new (allocator.allocate<BitVectorAttrStorage>())
-        BitVectorAttrStorage(std::move(key));
-  }
-
-  APInt value;
-};
-} // namespace detail
-} // namespace smt
-} // namespace mlir
-
-APInt BitVectorAttr::getValue() const { return getImpl()->value; }
-
 LogicalResult BitVectorAttr::verify(
     function_ref<InFlightDiagnostic()> emitError,
     APInt value) { // NOLINT(performance-unnecessary-value-param)
diff --git a/mlir/lib/TableGen/AttrOrTypeDef.cpp b/mlir/lib/TableGen/AttrOrTypeDef.cpp
index 9e8f789d71b5e..ccb0a6c6c261e 100644
--- a/mlir/lib/TableGen/AttrOrTypeDef.cpp
+++ b/mlir/lib/TableGen/AttrOrTypeDef.cpp
@@ -278,6 +278,10 @@ std::optional<StringRef> AttrOrTypeParameter::getAllocator() const {
   return getDefValue<StringInit>("allocator");
 }
 
+bool AttrOrTypeParameter::hasCustomComparator() const {
+  return getDefValue<StringInit>("comparator").has_value();
+}
+
 StringRef AttrOrTypeParameter::getComparator() const {
   return getDefValue<StringInit>("comparator").value_or("$_lhs == $_rhs");
 }
diff --git a/mlir/test/mlir-tblgen/apint-param-error.td b/mlir/test/mlir-tblgen/apint-param-error.td
new file mode 100644
index 0000000000000..602180790bbff
--- /dev/null
+++ b/mlir/test/mlir-tblgen/apint-param-error.td
@@ -0,0 +1,17 @@
+// RUN: not mlir-tblgen -gen-attrdef-decls -I %S/../../include %s 2>&1 | FileCheck %s
+
+include "mlir/IR/AttrTypeBase.td"
+include "mlir/IR/OpBase.td"
+
+def Test_Dialect: Dialect {
+  let name = "TestDialect";
+  let cppNamespace = "::test";
+}
+
+def RawAPIntAttr : AttrDef<Test_Dialect, "RawAPInt"> {
+  let mnemonic = "raw_ap_int";
+  let parameters = (ins "APInt":$value);
+  let hasCustomAssemblyFormat = 1;
+}
+
+// CHECK: apint-param-error.td:11:5: error: Using a raw APInt parameter
diff --git a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
index cf0d827942949..cef859bc24abb 100644
--- a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
+++ b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
@@ -678,8 +678,18 @@ void DefGen::emitStorageClass() {
   emitConstruct();
   // Emit the storage class members as public, at the very end of the struct.
   storageCls->finalize();
-  for (auto &param : params)
+  for (auto &param : params) {
+    if (param.getCppType().contains("APInt") && !param.hasCustomComparator()) {
+      PrintFatalError(
+          def.getLoc(),
+          "Using a raw APInt parameter without a custom comparator is "
+          "not supported because an assert in the equality operator is "
+          "triggered when the two APInts have different bit widths. This can "
+          "lead to unexpected crashes. Use an `APIntParameter` or "
+          "provide a custom comparator.");
+    }
     storageCls->declare<Field>(param.getCppType(), param.getName());
+  }
 }
 
 //===----------------------------------------------------------------------===//

From d8b0e616b746a6c7bdd0492f2f349105028684b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche@google.com>
Date: Tue, 22 Apr 2025 09:07:02 +0000
Subject: [PATCH 700/710] [SPIR-V] Fix OpVectorShuffle operands on load
 (#135954)

The generated OpVectorShuffle was wrong, as the indices we pass are not
to select the vector to sample from, but the position in the vector.
---
 llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp        | 5 +++--
 .../SPIRV/pointers/getelementptr-downcast-vector.ll       | 8 ++++----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
index 5ba4fbb02560d..69051132cacff 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
@@ -81,8 +81,9 @@ class SPIRVLegalizePointerCast : public FunctionPass {
     LoadInst *NewLoad = B.CreateLoad(SourceType, Source);
     buildAssignType(B, SourceType, NewLoad);
 
-    SmallVector<int> Mask(/* Size= */ TargetType->getNumElements(),
-                          /* Value= */ 0);
+    SmallVector<int> Mask(/* Size= */ TargetType->getNumElements());
+    for (unsigned I = 0; I < TargetType->getNumElements(); ++I)
+      Mask[I] = I;
     Value *Output = B.CreateShuffleVector(NewLoad, NewLoad, Mask);
     buildAssignType(B, TargetType, Output);
     return Output;
diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll
index d4131fa8a2658..1287a512ac96d 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll
@@ -19,7 +19,7 @@ define internal spir_func <3 x i32> @foo(ptr addrspace(10) %a) {
   ; partial loading of a vector: v4 -> v3.
   %2 = load <3 x i32>, ptr addrspace(10) %1, align 16
 ; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16
-; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0
+; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 1 2
 
   ret <3 x i32> %2
 ; CHECK: OpReturnValue %[[#val]]
@@ -33,7 +33,7 @@ define internal spir_func <3 x i32> @fooDefault(ptr %a) {
   ; partial loading of a vector: v4 -> v3.
   %2 = load <3 x i32>, ptr %1, align 16
 ; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16
-; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0
+; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 1 2
 
   ret <3 x i32> %2
 ; CHECK: OpReturnValue %[[#val]]
@@ -47,7 +47,7 @@ define internal spir_func <3 x i32> @fooBounds(ptr %a) {
   ; partial loading of a vector: v4 -> v3.
   %2 = load <3 x i32>, ptr %1, align 16
 ; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16
-; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0
+; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 1 2
 
   ret <3 x i32> %2
 ; CHECK: OpReturnValue %[[#val]]
@@ -61,7 +61,7 @@ define internal spir_func <2 x i32> @bar(ptr addrspace(10) %a) {
   ; partial loading of a vector: v4 -> v2.
   %2 = load <2 x i32>, ptr addrspace(10) %1, align 16
 ; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16
-; CHECK: %[[#val:]] = OpVectorShuffle %[[#v2]] %[[#load]] %[[#load]] 0 0
+; CHECK: %[[#val:]] = OpVectorShuffle %[[#v2]] %[[#load]] %[[#load]] 0 1
 
   ret <2 x i32> %2
 ; CHECK: OpReturnValue %[[#val]]

From e12681ae733e8341c161534d4d4de2611573b616 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Tue, 22 Apr 2025 11:08:39 +0200
Subject: [PATCH 701/710] [clang-cl] Parse the /dynamicdeopt option

which was mentioned in
https://devblogs.microsoft.com/cppblog/cpp-dynamic-debugging-full-debuggability-for-optimized-builds/

(That post also mentions /d2DDTrimInlines, which we already parse via
a /d2 catch-all.)
---
 clang/include/clang/Driver/Options.td | 1 +
 clang/test/Driver/cl-options.c        | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 8224e734a5a5c..a9f4083920663 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -9079,6 +9079,7 @@ def _SLASH_clr : CLJoined<"clr">;
 def _SLASH_d1 : CLJoined<"d1">;
 def _SLASH_d2 : CLJoined<"d2">;
 def _SLASH_doc : CLJoined<"doc">;
+def _SLASH_dynamicdeopt : CLJoined<"dynamicdeopt">;
 def _SLASH_experimental : CLJoined<"experimental:">;
 def _SLASH_exportHeader : CLFlag<"exportHeader">;
 def _SLASH_external : CLJoined<"external:">;
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index eb4c74378f9d7..beaef09c8a8d9 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -437,7 +437,9 @@
 // RUN:     /d1import_no_registry \
 // RUN:     /d1nodatetime \
 // RUN:     /d2FH4 \
+// RUN:     /d2TrimInlines \
 // RUN:     /docname \
+// RUN:     /dynamicdeopt \
 // RUN:     /experimental:external \
 // RUN:     /experimental:module \
 // RUN:     /experimental:preprocessor \

From adb671ea23af72c0fa1acd42103a5e9ca413d729 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Tue, 22 Apr 2025 11:38:55 +0200
Subject: [PATCH 702/710] [lldb] Clean up StartDebugserverProcess before I
 start refactoring it (#135342)

- use early exits where possible
- avoid the listen thread by using Socket APIs which allow separate
"listen" and "accept" steps
- use formatv-like log statements

There "should" be no functional changes from this patch.
---
 .../gdb-remote/GDBRemoteCommunication.cpp     | 481 ++++++++----------
 .../gdb-remote/GDBRemoteCommunication.h       |  15 -
 2 files changed, 199 insertions(+), 297 deletions(-)

diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index 7fe5dd4827cfe..adedfa8ece917 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -15,7 +15,6 @@
 #include "lldb/Host/Pipe.h"
 #include "lldb/Host/ProcessLaunchInfo.h"
 #include "lldb/Host/Socket.h"
-#include "lldb/Host/ThreadLauncher.h"
 #include "lldb/Host/common/TCPSocket.h"
 #include "lldb/Host/posix/ConnectionFileDescriptorPosix.h"
 #include "lldb/Target/Platform.h"
@@ -27,10 +26,10 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/llvm-config.h" // for LLVM_ENABLE_ZLIB
+#include "llvm/Support/Error.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include <climits>
 #include <cstring>
-#include <future>
 #include <sys/stat.h>
 
 #if defined(__APPLE__)
@@ -63,7 +62,7 @@ GDBRemoteCommunication::GDBRemoteCommunication()
 #endif
       m_echo_number(0), m_supports_qEcho(eLazyBoolCalculate), m_history(512),
       m_send_acks(true), m_is_platform(false),
-      m_compression_type(CompressionType::None), m_listen_url() {
+      m_compression_type(CompressionType::None) {
 }
 
 // Destructor
@@ -835,53 +834,6 @@ GDBRemoteCommunication::CheckForPacket(const uint8_t *src, size_t src_len,
   return GDBRemoteCommunication::PacketType::Invalid;
 }
 
-Status GDBRemoteCommunication::StartListenThread(const char *hostname,
-                                                 uint16_t port) {
-  if (m_listen_thread.IsJoinable())
-    return Status::FromErrorString("listen thread already running");
-
-  char listen_url[512];
-  if (hostname && hostname[0])
-    snprintf(listen_url, sizeof(listen_url), "listen://%s:%i", hostname, port);
-  else
-    snprintf(listen_url, sizeof(listen_url), "listen://%i", port);
-  m_listen_url = listen_url;
-  SetConnection(std::make_unique<ConnectionFileDescriptor>());
-  llvm::Expected<HostThread> listen_thread = ThreadLauncher::LaunchThread(
-      listen_url, [this] { return GDBRemoteCommunication::ListenThread(); });
-  if (!listen_thread)
-    return Status::FromError(listen_thread.takeError());
-  m_listen_thread = *listen_thread;
-
-  return Status();
-}
-
-bool GDBRemoteCommunication::JoinListenThread() {
-  if (m_listen_thread.IsJoinable())
-    m_listen_thread.Join(nullptr);
-  return true;
-}
-
-lldb::thread_result_t GDBRemoteCommunication::ListenThread() {
-  Status error;
-  ConnectionFileDescriptor *connection =
-      (ConnectionFileDescriptor *)GetConnection();
-
-  if (connection) {
-    // Do the listen on another thread so we can continue on...
-    if (connection->Connect(
-            m_listen_url.c_str(),
-            [this](llvm::StringRef port_str) {
-              uint16_t port = 0;
-              llvm::to_integer(port_str, port, 10);
-              m_port_promise.set_value(port);
-            },
-            &error) != eConnectionStatusSuccess)
-      SetConnection(nullptr);
-  }
-  return {};
-}
-
 FileSpec GDBRemoteCommunication::GetDebugserverPath(Platform *platform) {
   Log *log = GetLog(GDBRLog::Process);
   // If we locate debugserver, keep that located version around
@@ -945,277 +897,242 @@ Status GDBRemoteCommunication::StartDebugserverProcess(
     const char *url, Platform *platform, ProcessLaunchInfo &launch_info,
     uint16_t *port, const Args *inferior_args, shared_fd_t pass_comm_fd) {
   Log *log = GetLog(GDBRLog::Process);
-  LLDB_LOGF(log, "GDBRemoteCommunication::%s(url=%s, port=%" PRIu16 ")",
-            __FUNCTION__, url ? url : "<empty>", port ? *port : uint16_t(0));
+  LLDB_LOG(log, "Starting debug server: url={0}, port={1}",
+           url ? url : "<empty>", port ? *port : uint16_t(0));
 
-  Status error;
-  FileSpec &debugserver_file_spec = launch_info.GetExecutableFile();
-  if ((debugserver_file_spec = GetDebugserverPath(platform))) {
-    std::string debugserver_path = debugserver_file_spec.GetPath();
+  FileSpec debugserver_file_spec = GetDebugserverPath(platform);
+  if (!debugserver_file_spec)
+    return Status::FromErrorString("unable to locate " DEBUGSERVER_BASENAME);
 
-    Args &debugserver_args = launch_info.GetArguments();
-    debugserver_args.Clear();
+  std::string debugserver_path = debugserver_file_spec.GetPath();
+  launch_info.SetExecutableFile(debugserver_file_spec,
+                                /*add_exe_file_as_first_arg=*/true);
 
-    // Start args with "debugserver /file/path -r --"
-    debugserver_args.AppendArgument(llvm::StringRef(debugserver_path));
+  Args &debugserver_args = launch_info.GetArguments();
 
 #if !defined(__APPLE__)
-    // First argument to lldb-server must be mode in which to run.
-    debugserver_args.AppendArgument(llvm::StringRef("gdbserver"));
+  // First argument to lldb-server must be mode in which to run.
+  debugserver_args.AppendArgument(llvm::StringRef("gdbserver"));
 #endif
 
-    // If a url is supplied then use it
-    if (url && url[0])
-      debugserver_args.AppendArgument(llvm::StringRef(url));
+  // If a url is supplied then use it
+  if (url && url[0])
+    debugserver_args.AppendArgument(llvm::StringRef(url));
 
-    if (pass_comm_fd != SharedSocket::kInvalidFD) {
-      StreamString fd_arg;
-      fd_arg.Printf("--fd=%" PRIi64, (int64_t)pass_comm_fd);
-      debugserver_args.AppendArgument(fd_arg.GetString());
-      // Send "pass_comm_fd" down to the inferior so it can use it to
-      // communicate back with this process. Ignored on Windows.
+  if (pass_comm_fd != SharedSocket::kInvalidFD) {
+    StreamString fd_arg;
+    fd_arg.Printf("--fd=%" PRIi64, (int64_t)pass_comm_fd);
+    debugserver_args.AppendArgument(fd_arg.GetString());
+    // Send "pass_comm_fd" down to the inferior so it can use it to
+    // communicate back with this process. Ignored on Windows.
 #ifndef _WIN32
-      launch_info.AppendDuplicateFileAction((int)pass_comm_fd,
-                                            (int)pass_comm_fd);
+    launch_info.AppendDuplicateFileAction((int)pass_comm_fd, (int)pass_comm_fd);
 #endif
-    }
+  }
 
-    // use native registers, not the GDB registers
-    debugserver_args.AppendArgument(llvm::StringRef("--native-regs"));
+  // use native registers, not the GDB registers
+  debugserver_args.AppendArgument(llvm::StringRef("--native-regs"));
 
-    if (launch_info.GetLaunchInSeparateProcessGroup()) {
-      debugserver_args.AppendArgument(llvm::StringRef("--setsid"));
-    }
+  if (launch_info.GetLaunchInSeparateProcessGroup())
+    debugserver_args.AppendArgument(llvm::StringRef("--setsid"));
 
-    llvm::SmallString<128> named_pipe_path;
-    // socket_pipe is used by debug server to communicate back either
-    // TCP port or domain socket name which it listens on.
-    // The second purpose of the pipe to serve as a synchronization point -
-    // once data is written to the pipe, debug server is up and running.
-    Pipe socket_pipe;
-
-    // port is null when debug server should listen on domain socket - we're
-    // not interested in port value but rather waiting for debug server to
-    // become available.
-    if (pass_comm_fd == SharedSocket::kInvalidFD) {
-      if (url) {
+  llvm::SmallString<128> named_pipe_path;
+  // socket_pipe is used by debug server to communicate back either
+  // TCP port or domain socket name which it listens on.
+  // The second purpose of the pipe to serve as a synchronization point -
+  // once data is written to the pipe, debug server is up and running.
+  Pipe socket_pipe;
+
+  std::unique_ptr<TCPSocket> sock_up;
+
+  // port is null when debug server should listen on domain socket - we're
+  // not interested in port value but rather waiting for debug server to
+  // become available.
+  if (pass_comm_fd == SharedSocket::kInvalidFD) {
+    if (url) {
 // Create a temporary file to get the stdout/stderr and redirect the output of
 // the command into this file. We will later read this file if all goes well
 // and fill the data into "command_output_ptr"
 #if defined(__APPLE__)
-        // Binding to port zero, we need to figure out what port it ends up
-        // using using a named pipe...
-        error = socket_pipe.CreateWithUniqueName("debugserver-named-pipe",
-                                                 false, named_pipe_path);
-        if (error.Fail()) {
-          LLDB_LOGF(log,
-                    "GDBRemoteCommunication::%s() "
-                    "named pipe creation failed: %s",
-                    __FUNCTION__, error.AsCString());
-          return error;
-        }
-        debugserver_args.AppendArgument(llvm::StringRef("--named-pipe"));
-        debugserver_args.AppendArgument(named_pipe_path);
+      // Binding to port zero, we need to figure out what port it ends up
+      // using using a named pipe...
+      Status error = socket_pipe.CreateWithUniqueName("debugserver-named-pipe",
+                                                      false, named_pipe_path);
+      if (error.Fail()) {
+        LLDB_LOG(log, "named pipe creation failed: {0}", error);
+        return error;
+      }
+      debugserver_args.AppendArgument(llvm::StringRef("--named-pipe"));
+      debugserver_args.AppendArgument(named_pipe_path);
 #else
-        // Binding to port zero, we need to figure out what port it ends up
-        // using using an unnamed pipe...
-        error = socket_pipe.CreateNew(true);
-        if (error.Fail()) {
-          LLDB_LOGF(log,
-                    "GDBRemoteCommunication::%s() "
-                    "unnamed pipe creation failed: %s",
-                    __FUNCTION__, error.AsCString());
-          return error;
-        }
-        pipe_t write = socket_pipe.GetWritePipe();
-        debugserver_args.AppendArgument(llvm::StringRef("--pipe"));
-        debugserver_args.AppendArgument(llvm::to_string(write));
-        launch_info.AppendCloseFileAction(socket_pipe.GetReadFileDescriptor());
-#endif
-      } else {
-        // No host and port given, so lets listen on our end and make the
-        // debugserver connect to us..
-        error = StartListenThread("127.0.0.1", 0);
-        if (error.Fail()) {
-          LLDB_LOGF(log,
-                    "GDBRemoteCommunication::%s() unable to start listen "
-                    "thread: %s",
-                    __FUNCTION__, error.AsCString());
-          return error;
-        }
-
-        // Wait for 10 seconds to resolve the bound port
-        std::future<uint16_t> port_future = m_port_promise.get_future();
-        uint16_t port_ = port_future.wait_for(std::chrono::seconds(10)) ==
-                                 std::future_status::ready
-                             ? port_future.get()
-                             : 0;
-        if (port_ > 0) {
-          char port_cstr[32];
-          snprintf(port_cstr, sizeof(port_cstr), "127.0.0.1:%i", port_);
-          // Send the host and port down that debugserver and specify an option
-          // so that it connects back to the port we are listening to in this
-          // process
-          debugserver_args.AppendArgument(llvm::StringRef("--reverse-connect"));
-          debugserver_args.AppendArgument(llvm::StringRef(port_cstr));
-          if (port)
-            *port = port_;
-        } else {
-          LLDB_LOGF(log, "GDBRemoteCommunication::%s() failed: %s",
-                    __FUNCTION__, error.AsCString());
-          return Status::FromErrorString(
-              "failed to bind to port 0 on 127.0.0.1");
-        }
+      // Binding to port zero, we need to figure out what port it ends up
+      // using using an unnamed pipe...
+      Status error = socket_pipe.CreateNew(true);
+      if (error.Fail()) {
+        LLDB_LOG(log, "unnamed pipe creation failed: {0}", error);
+        return error;
       }
-    }
+      pipe_t write = socket_pipe.GetWritePipe();
+      debugserver_args.AppendArgument(llvm::StringRef("--pipe"));
+      debugserver_args.AppendArgument(llvm::to_string(write));
+      launch_info.AppendCloseFileAction(socket_pipe.GetReadFileDescriptor());
+#endif
+    } else {
+      // No host and port given, so lets listen on our end and make the
+      // debugserver connect to us..
+      if (llvm::Expected<std::unique_ptr<TCPSocket>> expected_sock =
+              Socket::TcpListen("127.0.0.1:0"))
+        sock_up = std::move(*expected_sock);
+      else
+        return Status::FromError(expected_sock.takeError());
 
-    Environment host_env = Host::GetEnvironment();
-    std::string env_debugserver_log_file =
-        host_env.lookup("LLDB_DEBUGSERVER_LOG_FILE");
-    if (!env_debugserver_log_file.empty()) {
+      uint16_t port_ = sock_up->GetLocalPortNumber();
+      // Send the host and port down that debugserver and specify an option
+      // so that it connects back to the port we are listening to in this
+      // process
+      debugserver_args.AppendArgument(llvm::StringRef("--reverse-connect"));
       debugserver_args.AppendArgument(
-          llvm::formatv("--log-file={0}", env_debugserver_log_file).str());
+          llvm::formatv("127.0.0.1:{0}", port_).str());
+      if (port)
+        *port = port_;
     }
+  }
+
+  Environment host_env = Host::GetEnvironment();
+  std::string env_debugserver_log_file =
+      host_env.lookup("LLDB_DEBUGSERVER_LOG_FILE");
+  if (!env_debugserver_log_file.empty()) {
+    debugserver_args.AppendArgument(
+        llvm::formatv("--log-file={0}", env_debugserver_log_file).str());
+  }
 
 #if defined(__APPLE__)
-    const char *env_debugserver_log_flags =
-        getenv("LLDB_DEBUGSERVER_LOG_FLAGS");
-    if (env_debugserver_log_flags) {
-      debugserver_args.AppendArgument(
-          llvm::formatv("--log-flags={0}", env_debugserver_log_flags).str());
-    }
+  const char *env_debugserver_log_flags = getenv("LLDB_DEBUGSERVER_LOG_FLAGS");
+  if (env_debugserver_log_flags) {
+    debugserver_args.AppendArgument(
+        llvm::formatv("--log-flags={0}", env_debugserver_log_flags).str());
+  }
 #else
-    std::string env_debugserver_log_channels =
-        host_env.lookup("LLDB_SERVER_LOG_CHANNELS");
-    if (!env_debugserver_log_channels.empty()) {
-      debugserver_args.AppendArgument(
-          llvm::formatv("--log-channels={0}", env_debugserver_log_channels)
-              .str());
-    }
+  std::string env_debugserver_log_channels =
+      host_env.lookup("LLDB_SERVER_LOG_CHANNELS");
+  if (!env_debugserver_log_channels.empty()) {
+    debugserver_args.AppendArgument(
+        llvm::formatv("--log-channels={0}", env_debugserver_log_channels)
+            .str());
+  }
 #endif
 
-    // Add additional args, starting with LLDB_DEBUGSERVER_EXTRA_ARG_1 until an
-    // env var doesn't come back.
-    uint32_t env_var_index = 1;
-    bool has_env_var;
-    do {
-      char env_var_name[64];
-      snprintf(env_var_name, sizeof(env_var_name),
-               "LLDB_DEBUGSERVER_EXTRA_ARG_%" PRIu32, env_var_index++);
-      std::string extra_arg = host_env.lookup(env_var_name);
-      has_env_var = !extra_arg.empty();
-
-      if (has_env_var) {
-        debugserver_args.AppendArgument(llvm::StringRef(extra_arg));
-        LLDB_LOGF(log,
-                  "GDBRemoteCommunication::%s adding env var %s contents "
-                  "to stub command line (%s)",
-                  __FUNCTION__, env_var_name, extra_arg.c_str());
-      }
-    } while (has_env_var);
-
-    if (inferior_args && inferior_args->GetArgumentCount() > 0) {
-      debugserver_args.AppendArgument(llvm::StringRef("--"));
-      debugserver_args.AppendArguments(*inferior_args);
+  // Add additional args, starting with LLDB_DEBUGSERVER_EXTRA_ARG_1 until an
+  // env var doesn't come back.
+  uint32_t env_var_index = 1;
+  bool has_env_var;
+  do {
+    char env_var_name[64];
+    snprintf(env_var_name, sizeof(env_var_name),
+             "LLDB_DEBUGSERVER_EXTRA_ARG_%" PRIu32, env_var_index++);
+    std::string extra_arg = host_env.lookup(env_var_name);
+    has_env_var = !extra_arg.empty();
+
+    if (has_env_var) {
+      debugserver_args.AppendArgument(llvm::StringRef(extra_arg));
+      LLDB_LOGF(log,
+                "GDBRemoteCommunication::%s adding env var %s contents "
+                "to stub command line (%s)",
+                __FUNCTION__, env_var_name, extra_arg.c_str());
     }
+  } while (has_env_var);
 
-    // Copy the current environment to the gdbserver/debugserver instance
-    launch_info.GetEnvironment() = host_env;
+  if (inferior_args && inferior_args->GetArgumentCount() > 0) {
+    debugserver_args.AppendArgument(llvm::StringRef("--"));
+    debugserver_args.AppendArguments(*inferior_args);
+  }
 
-    // Close STDIN, STDOUT and STDERR.
-    launch_info.AppendCloseFileAction(STDIN_FILENO);
-    launch_info.AppendCloseFileAction(STDOUT_FILENO);
-    launch_info.AppendCloseFileAction(STDERR_FILENO);
+  // Copy the current environment to the gdbserver/debugserver instance
+  launch_info.GetEnvironment() = host_env;
+
+  // Close STDIN, STDOUT and STDERR.
+  launch_info.AppendCloseFileAction(STDIN_FILENO);
+  launch_info.AppendCloseFileAction(STDOUT_FILENO);
+  launch_info.AppendCloseFileAction(STDERR_FILENO);
+
+  // Redirect STDIN, STDOUT and STDERR to "/dev/null".
+  launch_info.AppendSuppressFileAction(STDIN_FILENO, true, false);
+  launch_info.AppendSuppressFileAction(STDOUT_FILENO, false, true);
+  launch_info.AppendSuppressFileAction(STDERR_FILENO, false, true);
+
+  if (log) {
+    StreamString string_stream;
+    Platform *const platform = nullptr;
+    launch_info.Dump(string_stream, platform);
+    LLDB_LOG(log, "launch info for gdb-remote stub:\n{0}",
+             string_stream.GetData());
+  }
+  if (Status error = Host::LaunchProcess(launch_info); error.Fail()) {
+    LLDB_LOG(log, "launch failed: {0}", error);
+    return error;
+  }
 
-    // Redirect STDIN, STDOUT and STDERR to "/dev/null".
-    launch_info.AppendSuppressFileAction(STDIN_FILENO, true, false);
-    launch_info.AppendSuppressFileAction(STDOUT_FILENO, false, true);
-    launch_info.AppendSuppressFileAction(STDERR_FILENO, false, true);
+  if (pass_comm_fd != SharedSocket::kInvalidFD)
+    return Status();
 
-    if (log) {
-      StreamString string_stream;
-      Platform *const platform = nullptr;
-      launch_info.Dump(string_stream, platform);
-      LLDB_LOGF(log, "launch info for gdb-remote stub:\n%s",
-                string_stream.GetData());
+  Status error;
+  if (named_pipe_path.size() > 0) {
+    error = socket_pipe.OpenAsReader(named_pipe_path, false);
+    if (error.Fail()) {
+      LLDB_LOG(log, "failed to open named pipe {0} for reading: {1}",
+               named_pipe_path, error);
     }
-    error = Host::LaunchProcess(launch_info);
-
-    if (error.Success() &&
-        (launch_info.GetProcessID() != LLDB_INVALID_PROCESS_ID) &&
-        pass_comm_fd == SharedSocket::kInvalidFD) {
-      if (named_pipe_path.size() > 0) {
-        error = socket_pipe.OpenAsReader(named_pipe_path, false);
-        if (error.Fail())
-          LLDB_LOGF(log,
-                    "GDBRemoteCommunication::%s() "
-                    "failed to open named pipe %s for reading: %s",
-                    __FUNCTION__, named_pipe_path.c_str(), error.AsCString());
-      }
+  }
 
-      if (socket_pipe.CanWrite())
-        socket_pipe.CloseWriteFileDescriptor();
-      if (socket_pipe.CanRead()) {
-        // Read port from pipe with 10 second timeout.
-        std::string port_str;
-        while (error.Success()) {
-          char buf[10];
-          if (llvm::Expected<size_t> num_bytes = socket_pipe.Read(
-                  buf, std::size(buf), std::chrono::seconds(10))) {
-            if (*num_bytes == 0)
-              break;
-            port_str.append(buf, *num_bytes);
-          } else {
-            error = Status::FromError(num_bytes.takeError());
-          }
-        }
-        if (error.Success() && (port != nullptr)) {
-          // NB: Deliberately using .c_str() to stop at embedded '\0's
-          llvm::StringRef port_ref = port_str.c_str();
-          uint16_t child_port = 0;
-          // FIXME: improve error handling
-          llvm::to_integer(port_ref, child_port);
-          if (*port == 0 || *port == child_port) {
-            *port = child_port;
-            LLDB_LOGF(log,
-                      "GDBRemoteCommunication::%s() "
-                      "debugserver listens %u port",
-                      __FUNCTION__, *port);
-          } else {
-            LLDB_LOGF(log,
-                      "GDBRemoteCommunication::%s() "
-                      "debugserver listening on port "
-                      "%d but requested port was %d",
-                      __FUNCTION__, (uint32_t)child_port, (uint32_t)(*port));
-          }
-        } else {
-          LLDB_LOGF(log,
-                    "GDBRemoteCommunication::%s() "
-                    "failed to read a port value from pipe %s: %s",
-                    __FUNCTION__, named_pipe_path.c_str(), error.AsCString());
-        }
-        socket_pipe.Close();
+  if (socket_pipe.CanWrite())
+    socket_pipe.CloseWriteFileDescriptor();
+  if (socket_pipe.CanRead()) {
+    // Read port from pipe with 10 second timeout.
+    std::string port_str;
+    while (error.Success()) {
+      char buf[10];
+      if (llvm::Expected<size_t> num_bytes =
+              socket_pipe.Read(buf, std::size(buf), std::chrono::seconds(10))) {
+        if (*num_bytes == 0)
+          break;
+        port_str.append(buf, *num_bytes);
+      } else {
+        error = Status::FromError(num_bytes.takeError());
       }
-
-      if (named_pipe_path.size() > 0) {
-        const auto err = socket_pipe.Delete(named_pipe_path);
-        if (err.Fail()) {
-          LLDB_LOGF(log,
-                    "GDBRemoteCommunication::%s failed to delete pipe %s: %s",
-                    __FUNCTION__, named_pipe_path.c_str(), err.AsCString());
-        }
+    }
+    if (error.Success() && (port != nullptr)) {
+      // NB: Deliberately using .c_str() to stop at embedded '\0's
+      llvm::StringRef port_ref = port_str.c_str();
+      uint16_t child_port = 0;
+      // FIXME: improve error handling
+      llvm::to_integer(port_ref, child_port);
+      if (*port == 0 || *port == child_port) {
+        *port = child_port;
+        LLDB_LOG(log, "debugserver listens on port {0}", *port);
+      } else {
+        LLDB_LOG(log,
+                 "debugserver listening on port {0} but requested port was {1}",
+                 child_port, (*port));
       }
-
-      // Make sure we actually connect with the debugserver...
-      JoinListenThread();
+    } else {
+      LLDB_LOG(log, "failed to read a port value from pipe {0}: {1}",
+               named_pipe_path, error);
     }
-  } else {
-    error = Status::FromErrorString("unable to locate " DEBUGSERVER_BASENAME);
+    socket_pipe.Close();
+  }
+
+  if (named_pipe_path.size() > 0) {
+    if (Status err = socket_pipe.Delete(named_pipe_path); err.Fail())
+      LLDB_LOG(log, "failed to delete pipe {0}: {1}", named_pipe_path, err);
   }
 
-  if (error.Fail()) {
-    LLDB_LOGF(log, "GDBRemoteCommunication::%s() failed: %s", __FUNCTION__,
-              error.AsCString());
+  if (error.Success() && sock_up) {
+    Socket *accepted_socket = nullptr;
+    error = sock_up->Accept(/*timeout=*/std::nullopt, accepted_socket);
+    if (accepted_socket) {
+      SetConnection(
+          std::make_unique<ConnectionFileDescriptor>(accepted_socket));
+    }
   }
 
   return error;
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
index 387ad0f65bdea..ee87629d9077b 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
@@ -12,11 +12,9 @@
 #include "GDBRemoteCommunicationHistory.h"
 #include "lldb/Core/Communication.h"
 #include "lldb/Host/Config.h"
-#include "lldb/Host/HostThread.h"
 #include "lldb/Host/Socket.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/StringExtractorGDBRemote.h"
-#include <future>
 #include <mutex>
 #include <string>
 
@@ -200,20 +198,7 @@ class GDBRemoteCommunication : public Communication {
   // on m_bytes.  The checksum was for the compressed packet.
   bool DecompressPacket();
 
-  Status StartListenThread(const char *hostname = "127.0.0.1",
-                           uint16_t port = 0);
-
-  bool JoinListenThread();
-
-  lldb::thread_result_t ListenThread();
-
 private:
-  // Promise used to grab the port number from listening thread
-  std::promise<uint16_t> m_port_promise;
-
-  HostThread m_listen_thread;
-  std::string m_listen_url;
-
 #if HAVE_LIBCOMPRESSION
   CompressionType m_decompression_scratch_type = CompressionType::None;
   void *m_decompression_scratch = nullptr;

From a86f4ee774e6d2eb9f38502ddda65842179a246a Mon Sep 17 00:00:00 2001
From: Yuval Deutscher <yuvald@sweet.security>
Date: Tue, 22 Apr 2025 12:43:23 +0300
Subject: [PATCH 703/710] [lldb] Use correct path for debugserver (#131609)

This solves an issue that arises when running lldb-server through a
symlink which is not named exactly `lldb-server`. For example, in many
distros lldb-server is packaged as e.g.
`/usr/lib/llvm-19/bin/lldb-server` which is then accessed through a
symlink such as `/usr/bin/lldb-server-19`.

It turns out that there is a cascade of bugs here:
* `GetShlibDir` attempts to locate the LLVM library directory by calling
`GetModuleFileSpecForHostAddress` on the address of the function
`ComputeSharedLibraryDirectory`, assuming that it is inside
`liblldb.so`. However, in every packaging I've seen of lldb-server the
function `ComputeSharedLibraryDirectory` is statically linked into the
`lldb-server` binary and is not in `liblldb.so`.
* When run through a symlink, `GetModuleFileSpecForHostAddress` on an
address that is in `lldb-server` returns the path of the symlink, not
the path of the binary itself. So we get e.g. `/usr/bin/` and not
`/usr/lib/llvm-19/bin/`.
* `GetDebugserverPath` attempts to concat `"lldb-server"` to the
directory we obtained, and thus fails when the symlink is not named
exactly `lldb-server`.
* Ironically, the reason that this works in the first place is precisely
because `GetModuleFileSpecForHostAddress` returns an incorrect path -
when the server is run as `lldb-server-19 ...` it returns
`"lldb-server-19"` which then causes `ComputePathRelativeToLibrary` to
fail and then `ComputeSupportExeDirectory` falls back to just using
`GetProgramFileSpec` instead (which is the only option that actually
yields a correct path).
---
 .../TestPlatformLaunchGDBServer.py            | 39 +++++++++++++++++++
 .../tools/lldb-server/SystemInitializerLLGS.h |  9 ++++-
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py b/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
index c365bc993e338..ea846149e4983 100644
--- a/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
+++ b/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
@@ -58,3 +58,42 @@ def test_platform_process_launch_gdb_server(self):
 
         self.runCmd("target create {}".format(self.getBuildArtifact("a.out")))
         self.expect("run", substrs=["unable to launch a GDB server on"], error=True)
+
+    @skipIfRemote
+    @skipUnlessPlatform(["linux"])
+    @add_test_categories(["lldb-server"])
+    def test_lldb_server_weird_symlinks(self):
+        self.build()
+
+        hostname = socket.getaddrinfo("localhost", 0, proto=socket.IPPROTO_TCP)[0][4][0]
+        listen_url = "[%s]:0" % hostname
+
+        port_file = self.getBuildArtifact("port")
+        commandline_args = [
+            "platform",
+            "--listen",
+            listen_url,
+            "--socket-file",
+            port_file,
+        ]
+
+        # Run lldb-server from a symlink without any binary called "lldb-server" in the directory.
+        new_lldb_server = self.getBuildArtifact(
+            "lldb-server-with-an-unconventional-name"
+        )
+        os.symlink(lldbgdbserverutils.get_lldb_server_exe(), new_lldb_server)
+
+        proc = self.spawnSubprocess(new_lldb_server, commandline_args)
+        socket_id = lldbutil.wait_for_file_on_target(self, port_file)
+
+        new_platform = lldb.SBPlatform("remote-" + self.getPlatform())
+        self.dbg.SetSelectedPlatform(new_platform)
+
+        connect_url = "connect://[%s]:%s" % (hostname, socket_id)
+        self.runCmd("platform connect %s" % connect_url)
+        self.runCmd("target create {}".format(self.getBuildArtifact("a.out")))
+        self.runCmd("run")
+        self.expect(
+            "process status",
+            patterns=["Process .* exited with status = 0"],
+        )
diff --git a/lldb/tools/lldb-server/SystemInitializerLLGS.h b/lldb/tools/lldb-server/SystemInitializerLLGS.h
index 4469a8ba5f60a..c6020b0dd37da 100644
--- a/lldb/tools/lldb-server/SystemInitializerLLGS.h
+++ b/lldb/tools/lldb-server/SystemInitializerLLGS.h
@@ -11,10 +11,17 @@
 
 #include "lldb/Initialization/SystemInitializer.h"
 #include "lldb/Initialization/SystemInitializerCommon.h"
+#include "lldb/Utility/FileSpec.h"
 
 class SystemInitializerLLGS : public lldb_private::SystemInitializerCommon {
 public:
-  SystemInitializerLLGS() : SystemInitializerCommon(nullptr) {}
+  SystemInitializerLLGS()
+      : SystemInitializerCommon(
+            // Finding the shared libraries directory on lldb-server is broken
+            // since lldb-server isn't dynamically linked with liblldb.so.
+            // Clearing the filespec here causes GetShlibDir to fail and
+            // GetSupportExeDir to fall-back to using the binary path instead.
+            [](lldb_private::FileSpec &file) { file.Clear(); }) {}
 
   llvm::Error Initialize() override;
   void Terminate() override;

From 9efd798a278a7ddda3b88365558ceb655c329d11 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Tue, 22 Apr 2025 12:47:34 +0300
Subject: [PATCH 704/710] [TTI] Make the rest of TTI::Concept/TTI::Model
 methods const (NFC) (#136668)

This will simplify future changes.
---
 .../llvm/Analysis/TargetTransformInfo.h       | 591 ++++++++++--------
 1 file changed, 313 insertions(+), 278 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 918a02fb48e70..ab5306b7b614e 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1923,7 +1923,7 @@ class TargetTransformInfo {
   /// implementation in a type erased interface.
   template <typename T> class Model;
 
-  std::unique_ptr<Concept> TTIImpl;
+  std::unique_ptr<const Concept> TTIImpl;
 };
 
 class TargetTransformInfo::Concept {
@@ -1933,37 +1933,37 @@ class TargetTransformInfo::Concept {
   virtual InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr,
                                      ArrayRef<const Value *> Operands,
                                      Type *AccessType,
-                                     TTI::TargetCostKind CostKind) = 0;
+                                     TTI::TargetCostKind CostKind) const = 0;
   virtual InstructionCost
   getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
                        const TTI::PointersChainInfo &Info, Type *AccessTy,
-                       TTI::TargetCostKind CostKind) = 0;
+                       TTI::TargetCostKind CostKind) const = 0;
   virtual unsigned getInliningThresholdMultiplier() const = 0;
   virtual unsigned getInliningCostBenefitAnalysisSavingsMultiplier() const = 0;
   virtual unsigned
   getInliningCostBenefitAnalysisProfitableMultiplier() const = 0;
   virtual int getInliningLastCallToStaticBonus() const = 0;
-  virtual unsigned adjustInliningThreshold(const CallBase *CB) = 0;
+  virtual unsigned adjustInliningThreshold(const CallBase *CB) const = 0;
   virtual int getInlinerVectorBonusPercent() const = 0;
   virtual unsigned getCallerAllocaCost(const CallBase *CB,
                                        const AllocaInst *AI) const = 0;
-  virtual InstructionCost getMemcpyCost(const Instruction *I) = 0;
+  virtual InstructionCost getMemcpyCost(const Instruction *I) const = 0;
   virtual uint64_t getMaxMemIntrinsicInlineSizeThreshold() const = 0;
   virtual unsigned
   getEstimatedNumberOfCaseClusters(const SwitchInst &SI, unsigned &JTSize,
                                    ProfileSummaryInfo *PSI,
-                                   BlockFrequencyInfo *BFI) = 0;
+                                   BlockFrequencyInfo *BFI) const = 0;
   virtual InstructionCost getInstructionCost(const User *U,
                                              ArrayRef<const Value *> Operands,
-                                             TargetCostKind CostKind) = 0;
-  virtual BranchProbability getPredictableBranchThreshold() = 0;
-  virtual InstructionCost getBranchMispredictPenalty() = 0;
-  virtual bool hasBranchDivergence(const Function *F = nullptr) = 0;
-  virtual bool isSourceOfDivergence(const Value *V) = 0;
-  virtual bool isAlwaysUniform(const Value *V) = 0;
+                                             TargetCostKind CostKind) const = 0;
+  virtual BranchProbability getPredictableBranchThreshold() const = 0;
+  virtual InstructionCost getBranchMispredictPenalty() const = 0;
+  virtual bool hasBranchDivergence(const Function *F = nullptr) const = 0;
+  virtual bool isSourceOfDivergence(const Value *V) const = 0;
+  virtual bool isAlwaysUniform(const Value *V) const = 0;
   virtual bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const = 0;
   virtual bool addrspacesMayAlias(unsigned AS0, unsigned AS1) const = 0;
-  virtual unsigned getFlatAddressSpace() = 0;
+  virtual unsigned getFlatAddressSpace() const = 0;
   virtual bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
                                           Intrinsic::ID IID) const = 0;
   virtual bool isNoopAddrSpaceCast(unsigned FromAS, unsigned ToAS) const = 0;
@@ -1976,140 +1976,150 @@ class TargetTransformInfo::Concept {
   virtual Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
                                                   Value *OldV,
                                                   Value *NewV) const = 0;
-  virtual bool isLoweredToCall(const Function *F) = 0;
-  virtual void getUnrollingPreferences(Loop *L, ScalarEvolution &,
-                                       UnrollingPreferences &UP,
-                                       OptimizationRemarkEmitter *ORE) = 0;
+  virtual bool isLoweredToCall(const Function *F) const = 0;
+  virtual void
+  getUnrollingPreferences(Loop *L, ScalarEvolution &, UnrollingPreferences &UP,
+                          OptimizationRemarkEmitter *ORE) const = 0;
   virtual void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                                     PeelingPreferences &PP) = 0;
+                                     PeelingPreferences &PP) const = 0;
   virtual bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                         AssumptionCache &AC,
                                         TargetLibraryInfo *LibInfo,
-                                        HardwareLoopInfo &HWLoopInfo) = 0;
-  virtual unsigned getEpilogueVectorizationMinVF() = 0;
-  virtual bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) = 0;
+                                        HardwareLoopInfo &HWLoopInfo) const = 0;
+  virtual unsigned getEpilogueVectorizationMinVF() const = 0;
+  virtual bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const = 0;
   virtual TailFoldingStyle
-  getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) = 0;
-  virtual std::optional<Instruction *> instCombineIntrinsic(
-      InstCombiner &IC, IntrinsicInst &II) = 0;
-  virtual std::optional<Value *> simplifyDemandedUseBitsIntrinsic(
-      InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask,
-      KnownBits & Known, bool &KnownBitsComputed) = 0;
+  getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) const = 0;
+  virtual std::optional<Instruction *>
+  instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const = 0;
+  virtual std::optional<Value *>
+  simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
+                                   APInt DemandedMask, KnownBits &Known,
+                                   bool &KnownBitsComputed) const = 0;
   virtual std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
-      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts,
-      APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
       std::function<void(Instruction *, unsigned, APInt, APInt &)>
-          SimplifyAndSetOp) = 0;
-  virtual bool isLegalAddImmediate(int64_t Imm) = 0;
-  virtual bool isLegalAddScalableImmediate(int64_t Imm) = 0;
-  virtual bool isLegalICmpImmediate(int64_t Imm) = 0;
+          SimplifyAndSetOp) const = 0;
+  virtual bool isLegalAddImmediate(int64_t Imm) const = 0;
+  virtual bool isLegalAddScalableImmediate(int64_t Imm) const = 0;
+  virtual bool isLegalICmpImmediate(int64_t Imm) const = 0;
   virtual bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
                                      int64_t BaseOffset, bool HasBaseReg,
                                      int64_t Scale, unsigned AddrSpace,
                                      Instruction *I,
-                                     int64_t ScalableOffset) = 0;
+                                     int64_t ScalableOffset) const = 0;
   virtual bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                             const TargetTransformInfo::LSRCost &C2) = 0;
-  virtual bool isNumRegsMajorCostOfLSR() = 0;
+                             const TargetTransformInfo::LSRCost &C2) const = 0;
+  virtual bool isNumRegsMajorCostOfLSR() const = 0;
   virtual bool shouldDropLSRSolutionIfLessProfitable() const = 0;
-  virtual bool isProfitableLSRChainElement(Instruction *I) = 0;
-  virtual bool canMacroFuseCmp() = 0;
+  virtual bool isProfitableLSRChainElement(Instruction *I) const = 0;
+  virtual bool canMacroFuseCmp() const = 0;
   virtual bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE,
                           LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC,
-                          TargetLibraryInfo *LibInfo) = 0;
+                          TargetLibraryInfo *LibInfo) const = 0;
   virtual AddressingModeKind
     getPreferredAddressingMode(const Loop *L, ScalarEvolution *SE) const = 0;
   virtual bool isLegalMaskedStore(Type *DataType, Align Alignment,
-                                  unsigned AddressSpace) = 0;
+                                  unsigned AddressSpace) const = 0;
   virtual bool isLegalMaskedLoad(Type *DataType, Align Alignment,
-                                 unsigned AddressSpace) = 0;
-  virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
-  virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
+                                 unsigned AddressSpace) const = 0;
+  virtual bool isLegalNTStore(Type *DataType, Align Alignment) const = 0;
+  virtual bool isLegalNTLoad(Type *DataType, Align Alignment) const = 0;
   virtual bool isLegalBroadcastLoad(Type *ElementTy,
                                     ElementCount NumElements) const = 0;
-  virtual bool isLegalMaskedScatter(Type *DataType, Align Alignment) = 0;
-  virtual bool isLegalMaskedGather(Type *DataType, Align Alignment) = 0;
+  virtual bool isLegalMaskedScatter(Type *DataType, Align Alignment) const = 0;
+  virtual bool isLegalMaskedGather(Type *DataType, Align Alignment) const = 0;
   virtual bool forceScalarizeMaskedGather(VectorType *DataType,
-                                          Align Alignment) = 0;
+                                          Align Alignment) const = 0;
   virtual bool forceScalarizeMaskedScatter(VectorType *DataType,
-                                           Align Alignment) = 0;
-  virtual bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) = 0;
-  virtual bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) = 0;
-  virtual bool isLegalStridedLoadStore(Type *DataType, Align Alignment) = 0;
+                                           Align Alignment) const = 0;
+  virtual bool isLegalMaskedCompressStore(Type *DataType,
+                                          Align Alignment) const = 0;
+  virtual bool isLegalMaskedExpandLoad(Type *DataType,
+                                       Align Alignment) const = 0;
+  virtual bool isLegalStridedLoadStore(Type *DataType,
+                                       Align Alignment) const = 0;
   virtual bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
                                             Align Alignment,
-                                            unsigned AddrSpace) = 0;
+                                            unsigned AddrSpace) const = 0;
 
-  virtual bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) = 0;
+  virtual bool isLegalMaskedVectorHistogram(Type *AddrType,
+                                            Type *DataType) const = 0;
   virtual bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
                                unsigned Opcode1,
                                const SmallBitVector &OpcodeMask) const = 0;
-  virtual bool enableOrderedReductions() = 0;
-  virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0;
-  virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) = 0;
-  virtual bool prefersVectorizedAddressing() = 0;
+  virtual bool enableOrderedReductions() const = 0;
+  virtual bool hasDivRemOp(Type *DataType, bool IsSigned) const = 0;
+  virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) const = 0;
+  virtual bool prefersVectorizedAddressing() const = 0;
   virtual InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
                                                StackOffset BaseOffset,
                                                bool HasBaseReg, int64_t Scale,
-                                               unsigned AddrSpace) = 0;
-  virtual bool LSRWithInstrQueries() = 0;
-  virtual bool isTruncateFree(Type *Ty1, Type *Ty2) = 0;
-  virtual bool isProfitableToHoist(Instruction *I) = 0;
-  virtual bool useAA() = 0;
+                                               unsigned AddrSpace) const = 0;
+  virtual bool LSRWithInstrQueries() const = 0;
+  virtual bool isTruncateFree(Type *Ty1, Type *Ty2) const = 0;
+  virtual bool isProfitableToHoist(Instruction *I) const = 0;
+  virtual bool useAA() const = 0;
   virtual bool isTypeLegal(Type *Ty) const = 0;
   virtual unsigned getRegUsageForType(Type *Ty) const = 0;
-  virtual bool shouldBuildLookupTables() = 0;
-  virtual bool shouldBuildLookupTablesForConstant(Constant *C) = 0;
-  virtual bool shouldBuildRelLookupTables() = 0;
-  virtual bool useColdCCForColdCall(Function &F) = 0;
-  virtual bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) = 0;
-  virtual bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
-                                                  unsigned ScalarOpdIdx) = 0;
+  virtual bool shouldBuildLookupTables() const = 0;
+  virtual bool shouldBuildLookupTablesForConstant(Constant *C) const = 0;
+  virtual bool shouldBuildRelLookupTables() const = 0;
+  virtual bool useColdCCForColdCall(Function &F) const = 0;
+  virtual bool
+  isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) const = 0;
+  virtual bool
+  isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
+                                     unsigned ScalarOpdIdx) const = 0;
   virtual bool isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
-                                                      int OpdIdx) = 0;
+                                                      int OpdIdx) const = 0;
   virtual bool
   isTargetIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID,
-                                                   int RetIdx) = 0;
+                                                   int RetIdx) const = 0;
   virtual InstructionCost
   getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
                            bool Insert, bool Extract, TargetCostKind CostKind,
-                           ArrayRef<Value *> VL = {}) = 0;
+                           ArrayRef<Value *> VL = {}) const = 0;
   virtual InstructionCost
   getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
                                    ArrayRef<Type *> Tys,
-                                   TargetCostKind CostKind) = 0;
-  virtual bool supportsEfficientVectorElementLoadStore() = 0;
-  virtual bool supportsTailCalls() = 0;
-  virtual bool supportsTailCallFor(const CallBase *CB) = 0;
-  virtual bool enableAggressiveInterleaving(bool LoopHasReductions) = 0;
+                                   TargetCostKind CostKind) const = 0;
+  virtual bool supportsEfficientVectorElementLoadStore() const = 0;
+  virtual bool supportsTailCalls() const = 0;
+  virtual bool supportsTailCallFor(const CallBase *CB) const = 0;
+  virtual bool enableAggressiveInterleaving(bool LoopHasReductions) const = 0;
   virtual MemCmpExpansionOptions
   enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const = 0;
-  virtual bool enableSelectOptimize() = 0;
-  virtual bool shouldTreatInstructionLikeSelect(const Instruction *I) = 0;
-  virtual bool enableInterleavedAccessVectorization() = 0;
-  virtual bool enableMaskedInterleavedAccessVectorization() = 0;
-  virtual bool isFPVectorizationPotentiallyUnsafe() = 0;
+  virtual bool enableSelectOptimize() const = 0;
+  virtual bool shouldTreatInstructionLikeSelect(const Instruction *I) const = 0;
+  virtual bool enableInterleavedAccessVectorization() const = 0;
+  virtual bool enableMaskedInterleavedAccessVectorization() const = 0;
+  virtual bool isFPVectorizationPotentiallyUnsafe() const = 0;
   virtual bool allowsMisalignedMemoryAccesses(LLVMContext &Context,
                                               unsigned BitWidth,
                                               unsigned AddressSpace,
                                               Align Alignment,
-                                              unsigned *Fast) = 0;
-  virtual PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) = 0;
-  virtual bool haveFastSqrt(Type *Ty) = 0;
-  virtual bool isExpensiveToSpeculativelyExecute(const Instruction *I) = 0;
-  virtual bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) = 0;
-  virtual InstructionCost getFPOpCost(Type *Ty) = 0;
+                                              unsigned *Fast) const = 0;
+  virtual PopcntSupportKind
+  getPopcntSupport(unsigned IntTyWidthInBit) const = 0;
+  virtual bool haveFastSqrt(Type *Ty) const = 0;
+  virtual bool
+  isExpensiveToSpeculativelyExecute(const Instruction *I) const = 0;
+  virtual bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const = 0;
+  virtual InstructionCost getFPOpCost(Type *Ty) const = 0;
   virtual InstructionCost getIntImmCodeSizeCost(unsigned Opc, unsigned Idx,
-                                                const APInt &Imm, Type *Ty) = 0;
+                                                const APInt &Imm,
+                                                Type *Ty) const = 0;
   virtual InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                        TargetCostKind CostKind) = 0;
-  virtual InstructionCost getIntImmCostInst(unsigned Opc, unsigned Idx,
-                                            const APInt &Imm, Type *Ty,
-                                            TargetCostKind CostKind,
-                                            Instruction *Inst = nullptr) = 0;
-  virtual InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
-                                              const APInt &Imm, Type *Ty,
-                                              TargetCostKind CostKind) = 0;
+                                        TargetCostKind CostKind) const = 0;
+  virtual InstructionCost
+  getIntImmCostInst(unsigned Opc, unsigned Idx, const APInt &Imm, Type *Ty,
+                    TargetCostKind CostKind,
+                    Instruction *Inst = nullptr) const = 0;
+  virtual InstructionCost
+  getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm,
+                      Type *Ty, TargetCostKind CostKind) const = 0;
   virtual bool preferToKeepConstantsAttached(const Instruction &Inst,
                                              const Function &Fn) const = 0;
   virtual unsigned getNumberOfRegisters(unsigned ClassID) const = 0;
@@ -2130,7 +2140,7 @@ class TargetTransformInfo::Concept {
   virtual unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
                                      Type *ScalarValTy) const = 0;
   virtual bool shouldConsiderAddressTypePromotion(
-      const Instruction &I, bool &AllowPromotionWithoutCommonHeader) = 0;
+      const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const = 0;
   virtual unsigned getCacheLineSize() const = 0;
   virtual std::optional<unsigned> getCacheSize(CacheLevel Level) const = 0;
   virtual std::optional<unsigned> getCacheAssociativity(CacheLevel Level)
@@ -2178,39 +2188,43 @@ class TargetTransformInfo::Concept {
                           PartialReductionExtendKind OpBExtend,
                           std::optional<unsigned> BinOp) const = 0;
 
-  virtual unsigned getMaxInterleaveFactor(ElementCount VF) = 0;
-  virtual InstructionCost getArithmeticInstrCost(
-      unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
-      OperandValueInfo Opd1Info, OperandValueInfo Opd2Info,
-      ArrayRef<const Value *> Args, const Instruction *CxtI = nullptr) = 0;
+  virtual unsigned getMaxInterleaveFactor(ElementCount VF) const = 0;
+  virtual InstructionCost
+  getArithmeticInstrCost(unsigned Opcode, Type *Ty,
+                         TTI::TargetCostKind CostKind,
+                         OperandValueInfo Opd1Info, OperandValueInfo Opd2Info,
+                         ArrayRef<const Value *> Args,
+                         const Instruction *CxtI = nullptr) const = 0;
   virtual InstructionCost getAltInstrCost(
       VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
       const SmallBitVector &OpcodeMask,
       TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const = 0;
 
-  virtual InstructionCost
-  getShuffleCost(ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
-                 TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
-                 ArrayRef<const Value *> Args, const Instruction *CxtI) = 0;
+  virtual InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp,
+                                         ArrayRef<int> Mask,
+                                         TTI::TargetCostKind CostKind,
+                                         int Index, VectorType *SubTp,
+                                         ArrayRef<const Value *> Args,
+                                         const Instruction *CxtI) const = 0;
   virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst,
                                            Type *Src, CastContextHint CCH,
                                            TTI::TargetCostKind CostKind,
-                                           const Instruction *I) = 0;
+                                           const Instruction *I) const = 0;
   virtual InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst,
                                                    VectorType *VecTy,
-                                                   unsigned Index) = 0;
-  virtual InstructionCost getCFInstrCost(unsigned Opcode,
-                                         TTI::TargetCostKind CostKind,
-                                         const Instruction *I = nullptr) = 0;
+                                                   unsigned Index) const = 0;
+  virtual InstructionCost
+  getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
+                 const Instruction *I = nullptr) const = 0;
   virtual InstructionCost
   getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
                      CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind,
                      OperandValueInfo Op1Info, OperandValueInfo Op2Info,
-                     const Instruction *I) = 0;
+                     const Instruction *I) const = 0;
   virtual InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                              TTI::TargetCostKind CostKind,
                                              unsigned Index, Value *Op0,
-                                             Value *Op1) = 0;
+                                             Value *Op1) const = 0;
 
   /// \param ScalarUserAndIdx encodes the information about extracts from a
   /// vector with 'Scalar' being the value being extracted,'User' being the user
@@ -2219,81 +2233,84 @@ class TargetTransformInfo::Concept {
   virtual InstructionCost getVectorInstrCost(
       unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
       Value *Scalar,
-      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) = 0;
+      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const = 0;
 
   virtual InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
                                              TTI::TargetCostKind CostKind,
-                                             unsigned Index) = 0;
+                                             unsigned Index) const = 0;
 
   virtual InstructionCost
   getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
                             const APInt &DemandedDstElts,
-                            TTI::TargetCostKind CostKind) = 0;
+                            TTI::TargetCostKind CostKind) const = 0;
 
   virtual InstructionCost
-  getInsertExtractValueCost(unsigned Opcode, TTI::TargetCostKind CostKind) = 0;
+  getInsertExtractValueCost(unsigned Opcode,
+                            TTI::TargetCostKind CostKind) const = 0;
 
   virtual InstructionCost
   getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                   unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                  OperandValueInfo OpInfo, const Instruction *I) = 0;
+                  OperandValueInfo OpInfo, const Instruction *I) const = 0;
   virtual InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src,
                                             Align Alignment,
                                             unsigned AddressSpace,
                                             TTI::TargetCostKind CostKind,
-                                            const Instruction *I) = 0;
+                                            const Instruction *I) const = 0;
   virtual InstructionCost
   getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                         unsigned AddressSpace,
-                        TTI::TargetCostKind CostKind) = 0;
+                        TTI::TargetCostKind CostKind) const = 0;
   virtual InstructionCost
   getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
                          bool VariableMask, Align Alignment,
                          TTI::TargetCostKind CostKind,
-                         const Instruction *I = nullptr) = 0;
+                         const Instruction *I = nullptr) const = 0;
   virtual InstructionCost getExpandCompressMemoryOpCost(
       unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
-      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) = 0;
+      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) const = 0;
   virtual InstructionCost
   getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
                          bool VariableMask, Align Alignment,
                          TTI::TargetCostKind CostKind,
-                         const Instruction *I = nullptr) = 0;
+                         const Instruction *I = nullptr) const = 0;
 
   virtual InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond = false, bool UseMaskForGaps = false) = 0;
+      bool UseMaskForCond = false, bool UseMaskForGaps = false) const = 0;
   virtual InstructionCost
   getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                              std::optional<FastMathFlags> FMF,
-                             TTI::TargetCostKind CostKind) = 0;
+                             TTI::TargetCostKind CostKind) const = 0;
   virtual InstructionCost
   getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
-                         TTI::TargetCostKind CostKind) = 0;
+                         TTI::TargetCostKind CostKind) const = 0;
   virtual InstructionCost getExtendedReductionCost(
       unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty,
       std::optional<FastMathFlags> FMF,
-      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) = 0;
+      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const = 0;
   virtual InstructionCost getMulAccReductionCost(
       bool IsUnsigned, Type *ResTy, VectorType *Ty,
-      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) = 0;
+      TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const = 0;
   virtual InstructionCost
   getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                        TTI::TargetCostKind CostKind) = 0;
-  virtual InstructionCost getCallInstrCost(Function *F, Type *RetTy,
-                                           ArrayRef<Type *> Tys,
-                                           TTI::TargetCostKind CostKind) = 0;
-  virtual unsigned getNumberOfParts(Type *Tp) = 0;
+                        TTI::TargetCostKind CostKind) const = 0;
   virtual InstructionCost
-  getAddressComputationCost(Type *Ty, ScalarEvolution *SE, const SCEV *Ptr) = 0;
+  getCallInstrCost(Function *F, Type *RetTy, ArrayRef<Type *> Tys,
+                   TTI::TargetCostKind CostKind) const = 0;
+  virtual unsigned getNumberOfParts(Type *Tp) const = 0;
+  virtual InstructionCost getAddressComputationCost(Type *Ty,
+                                                    ScalarEvolution *SE,
+                                                    const SCEV *Ptr) const = 0;
   virtual InstructionCost
-  getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) = 0;
+  getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const = 0;
   virtual bool getTgtMemIntrinsic(IntrinsicInst *Inst,
-                                  MemIntrinsicInfo &Info) = 0;
+                                  MemIntrinsicInfo &Info) const = 0;
   virtual unsigned getAtomicMemIntrinsicMaxElementSize() const = 0;
-  virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
-                                                   Type *ExpectedType) = 0;
+  virtual Value *
+  getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
+                                    Type *ExpectedType) const = 0;
   virtual Type *getMemcpyLoopLoweringType(
       LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
       unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
@@ -2380,20 +2397,20 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   InstructionCost
   getGEPCost(Type *PointeeType, const Value *Ptr,
              ArrayRef<const Value *> Operands, Type *AccessType,
-             TargetTransformInfo::TargetCostKind CostKind) override {
+             TargetTransformInfo::TargetCostKind CostKind) const override {
     return Impl.getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind);
   }
   InstructionCost getPointersChainCost(ArrayRef<const Value *> Ptrs,
                                        const Value *Base,
                                        const PointersChainInfo &Info,
                                        Type *AccessTy,
-                                       TargetCostKind CostKind) override {
+                                       TargetCostKind CostKind) const override {
     return Impl.getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
   }
   unsigned getInliningThresholdMultiplier() const override {
     return Impl.getInliningThresholdMultiplier();
   }
-  unsigned adjustInliningThreshold(const CallBase *CB) override {
+  unsigned adjustInliningThreshold(const CallBase *CB) const override {
     return Impl.adjustInliningThreshold(CB);
   }
   unsigned getInliningCostBenefitAnalysisSavingsMultiplier() const override {
@@ -2412,7 +2429,7 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                const AllocaInst *AI) const override {
     return Impl.getCallerAllocaCost(CB, AI);
   }
-  InstructionCost getMemcpyCost(const Instruction *I) override {
+  InstructionCost getMemcpyCost(const Instruction *I) const override {
     return Impl.getMemcpyCost(I);
   }
 
@@ -2422,23 +2439,23 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
 
   InstructionCost getInstructionCost(const User *U,
                                      ArrayRef<const Value *> Operands,
-                                     TargetCostKind CostKind) override {
+                                     TargetCostKind CostKind) const override {
     return Impl.getInstructionCost(U, Operands, CostKind);
   }
-  BranchProbability getPredictableBranchThreshold() override {
+  BranchProbability getPredictableBranchThreshold() const override {
     return Impl.getPredictableBranchThreshold();
   }
-  InstructionCost getBranchMispredictPenalty() override {
+  InstructionCost getBranchMispredictPenalty() const override {
     return Impl.getBranchMispredictPenalty();
   }
-  bool hasBranchDivergence(const Function *F = nullptr) override {
+  bool hasBranchDivergence(const Function *F = nullptr) const override {
     return Impl.hasBranchDivergence(F);
   }
-  bool isSourceOfDivergence(const Value *V) override {
+  bool isSourceOfDivergence(const Value *V) const override {
     return Impl.isSourceOfDivergence(V);
   }
 
-  bool isAlwaysUniform(const Value *V) override {
+  bool isAlwaysUniform(const Value *V) const override {
     return Impl.isAlwaysUniform(V);
   }
 
@@ -2450,7 +2467,9 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.addrspacesMayAlias(AS0, AS1);
   }
 
-  unsigned getFlatAddressSpace() override { return Impl.getFlatAddressSpace(); }
+  unsigned getFlatAddressSpace() const override {
+    return Impl.getFlatAddressSpace();
+  }
 
   bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
                                   Intrinsic::ID IID) const override {
@@ -2482,41 +2501,41 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.rewriteIntrinsicWithAddressSpace(II, OldV, NewV);
   }
 
-  bool isLoweredToCall(const Function *F) override {
+  bool isLoweredToCall(const Function *F) const override {
     return Impl.isLoweredToCall(F);
   }
   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                UnrollingPreferences &UP,
-                               OptimizationRemarkEmitter *ORE) override {
+                               OptimizationRemarkEmitter *ORE) const override {
     return Impl.getUnrollingPreferences(L, SE, UP, ORE);
   }
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                             PeelingPreferences &PP) override {
+                             PeelingPreferences &PP) const override {
     return Impl.getPeelingPreferences(L, SE, PP);
   }
   bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
                                 AssumptionCache &AC, TargetLibraryInfo *LibInfo,
-                                HardwareLoopInfo &HWLoopInfo) override {
+                                HardwareLoopInfo &HWLoopInfo) const override {
     return Impl.isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
   }
-  unsigned getEpilogueVectorizationMinVF() override {
+  unsigned getEpilogueVectorizationMinVF() const override {
     return Impl.getEpilogueVectorizationMinVF();
   }
-  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) override {
+  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const override {
     return Impl.preferPredicateOverEpilogue(TFI);
   }
   TailFoldingStyle
-  getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) override {
+  getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) const override {
     return Impl.getPreferredTailFoldingStyle(IVUpdateMayOverflow);
   }
   std::optional<Instruction *>
-  instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) override {
+  instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override {
     return Impl.instCombineIntrinsic(IC, II);
   }
   std::optional<Value *>
   simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
                                    APInt DemandedMask, KnownBits &Known,
-                                   bool &KnownBitsComputed) override {
+                                   bool &KnownBitsComputed) const override {
     return Impl.simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
                                                  KnownBitsComputed);
   }
@@ -2524,43 +2543,44 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
       InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
       APInt &UndefElts2, APInt &UndefElts3,
       std::function<void(Instruction *, unsigned, APInt, APInt &)>
-          SimplifyAndSetOp) override {
+          SimplifyAndSetOp) const override {
     return Impl.simplifyDemandedVectorEltsIntrinsic(
         IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
         SimplifyAndSetOp);
   }
-  bool isLegalAddImmediate(int64_t Imm) override {
+  bool isLegalAddImmediate(int64_t Imm) const override {
     return Impl.isLegalAddImmediate(Imm);
   }
-  bool isLegalAddScalableImmediate(int64_t Imm) override {
+  bool isLegalAddScalableImmediate(int64_t Imm) const override {
     return Impl.isLegalAddScalableImmediate(Imm);
   }
-  bool isLegalICmpImmediate(int64_t Imm) override {
+  bool isLegalICmpImmediate(int64_t Imm) const override {
     return Impl.isLegalICmpImmediate(Imm);
   }
   bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
                              bool HasBaseReg, int64_t Scale, unsigned AddrSpace,
-                             Instruction *I, int64_t ScalableOffset) override {
+                             Instruction *I,
+                             int64_t ScalableOffset) const override {
     return Impl.isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, Scale,
                                       AddrSpace, I, ScalableOffset);
   }
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
-                     const TargetTransformInfo::LSRCost &C2) override {
+                     const TargetTransformInfo::LSRCost &C2) const override {
     return Impl.isLSRCostLess(C1, C2);
   }
-  bool isNumRegsMajorCostOfLSR() override {
+  bool isNumRegsMajorCostOfLSR() const override {
     return Impl.isNumRegsMajorCostOfLSR();
   }
   bool shouldDropLSRSolutionIfLessProfitable() const override {
     return Impl.shouldDropLSRSolutionIfLessProfitable();
   }
-  bool isProfitableLSRChainElement(Instruction *I) override {
+  bool isProfitableLSRChainElement(Instruction *I) const override {
     return Impl.isProfitableLSRChainElement(I);
   }
-  bool canMacroFuseCmp() override { return Impl.canMacroFuseCmp(); }
+  bool canMacroFuseCmp() const override { return Impl.canMacroFuseCmp(); }
   bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI,
                   DominatorTree *DT, AssumptionCache *AC,
-                  TargetLibraryInfo *LibInfo) override {
+                  TargetLibraryInfo *LibInfo) const override {
     return Impl.canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo);
   }
   AddressingModeKind
@@ -2569,206 +2589,212 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.getPreferredAddressingMode(L, SE);
   }
   bool isLegalMaskedStore(Type *DataType, Align Alignment,
-                          unsigned AddressSpace) override {
+                          unsigned AddressSpace) const override {
     return Impl.isLegalMaskedStore(DataType, Alignment, AddressSpace);
   }
   bool isLegalMaskedLoad(Type *DataType, Align Alignment,
-                         unsigned AddressSpace) override {
+                         unsigned AddressSpace) const override {
     return Impl.isLegalMaskedLoad(DataType, Alignment, AddressSpace);
   }
-  bool isLegalNTStore(Type *DataType, Align Alignment) override {
+  bool isLegalNTStore(Type *DataType, Align Alignment) const override {
     return Impl.isLegalNTStore(DataType, Alignment);
   }
-  bool isLegalNTLoad(Type *DataType, Align Alignment) override {
+  bool isLegalNTLoad(Type *DataType, Align Alignment) const override {
     return Impl.isLegalNTLoad(DataType, Alignment);
   }
   bool isLegalBroadcastLoad(Type *ElementTy,
                             ElementCount NumElements) const override {
     return Impl.isLegalBroadcastLoad(ElementTy, NumElements);
   }
-  bool isLegalMaskedScatter(Type *DataType, Align Alignment) override {
+  bool isLegalMaskedScatter(Type *DataType, Align Alignment) const override {
     return Impl.isLegalMaskedScatter(DataType, Alignment);
   }
-  bool isLegalMaskedGather(Type *DataType, Align Alignment) override {
+  bool isLegalMaskedGather(Type *DataType, Align Alignment) const override {
     return Impl.isLegalMaskedGather(DataType, Alignment);
   }
   bool forceScalarizeMaskedGather(VectorType *DataType,
-                                  Align Alignment) override {
+                                  Align Alignment) const override {
     return Impl.forceScalarizeMaskedGather(DataType, Alignment);
   }
   bool forceScalarizeMaskedScatter(VectorType *DataType,
-                                   Align Alignment) override {
+                                   Align Alignment) const override {
     return Impl.forceScalarizeMaskedScatter(DataType, Alignment);
   }
-  bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) override {
+  bool isLegalMaskedCompressStore(Type *DataType,
+                                  Align Alignment) const override {
     return Impl.isLegalMaskedCompressStore(DataType, Alignment);
   }
-  bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) override {
+  bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) const override {
     return Impl.isLegalMaskedExpandLoad(DataType, Alignment);
   }
-  bool isLegalStridedLoadStore(Type *DataType, Align Alignment) override {
+  bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const override {
     return Impl.isLegalStridedLoadStore(DataType, Alignment);
   }
   bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
                                     Align Alignment,
-                                    unsigned AddrSpace) override {
+                                    unsigned AddrSpace) const override {
     return Impl.isLegalInterleavedAccessType(VTy, Factor, Alignment, AddrSpace);
   }
-  bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) override {
+  bool isLegalMaskedVectorHistogram(Type *AddrType,
+                                    Type *DataType) const override {
     return Impl.isLegalMaskedVectorHistogram(AddrType, DataType);
   }
   bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
                        const SmallBitVector &OpcodeMask) const override {
     return Impl.isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask);
   }
-  bool enableOrderedReductions() override {
+  bool enableOrderedReductions() const override {
     return Impl.enableOrderedReductions();
   }
-  bool hasDivRemOp(Type *DataType, bool IsSigned) override {
+  bool hasDivRemOp(Type *DataType, bool IsSigned) const override {
     return Impl.hasDivRemOp(DataType, IsSigned);
   }
-  bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) override {
+  bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) const override {
     return Impl.hasVolatileVariant(I, AddrSpace);
   }
-  bool prefersVectorizedAddressing() override {
+  bool prefersVectorizedAddressing() const override {
     return Impl.prefersVectorizedAddressing();
   }
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
                                        StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale,
-                                       unsigned AddrSpace) override {
+                                       unsigned AddrSpace) const override {
     return Impl.getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale,
                                      AddrSpace);
   }
-  bool LSRWithInstrQueries() override { return Impl.LSRWithInstrQueries(); }
-  bool isTruncateFree(Type *Ty1, Type *Ty2) override {
+  bool LSRWithInstrQueries() const override {
+    return Impl.LSRWithInstrQueries();
+  }
+  bool isTruncateFree(Type *Ty1, Type *Ty2) const override {
     return Impl.isTruncateFree(Ty1, Ty2);
   }
-  bool isProfitableToHoist(Instruction *I) override {
+  bool isProfitableToHoist(Instruction *I) const override {
     return Impl.isProfitableToHoist(I);
   }
-  bool useAA() override { return Impl.useAA(); }
+  bool useAA() const override { return Impl.useAA(); }
   bool isTypeLegal(Type *Ty) const override { return Impl.isTypeLegal(Ty); }
   unsigned getRegUsageForType(Type *Ty) const override {
     return Impl.getRegUsageForType(Ty);
   }
-  bool shouldBuildLookupTables() override {
+  bool shouldBuildLookupTables() const override {
     return Impl.shouldBuildLookupTables();
   }
-  bool shouldBuildLookupTablesForConstant(Constant *C) override {
+  bool shouldBuildLookupTablesForConstant(Constant *C) const override {
     return Impl.shouldBuildLookupTablesForConstant(C);
   }
-  bool shouldBuildRelLookupTables() override {
+  bool shouldBuildRelLookupTables() const override {
     return Impl.shouldBuildRelLookupTables();
   }
-  bool useColdCCForColdCall(Function &F) override {
+  bool useColdCCForColdCall(Function &F) const override {
     return Impl.useColdCCForColdCall(F);
   }
-  bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) override {
+  bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) const override {
     return Impl.isTargetIntrinsicTriviallyScalarizable(ID);
   }
 
-  bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
-                                          unsigned ScalarOpdIdx) override {
+  bool
+  isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
+                                     unsigned ScalarOpdIdx) const override {
     return Impl.isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
   }
 
   bool isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
-                                              int OpdIdx) override {
+                                              int OpdIdx) const override {
     return Impl.isTargetIntrinsicWithOverloadTypeAtArg(ID, OpdIdx);
   }
 
-  bool isTargetIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID,
-                                                        int RetIdx) override {
+  bool
+  isTargetIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID,
+                                                   int RetIdx) const override {
     return Impl.isTargetIntrinsicWithStructReturnOverloadAtField(ID, RetIdx);
   }
 
-  InstructionCost getScalarizationOverhead(VectorType *Ty,
-                                           const APInt &DemandedElts,
-                                           bool Insert, bool Extract,
-                                           TargetCostKind CostKind,
-                                           ArrayRef<Value *> VL = {}) override {
+  InstructionCost
+  getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
+                           bool Insert, bool Extract, TargetCostKind CostKind,
+                           ArrayRef<Value *> VL = {}) const override {
     return Impl.getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
                                          CostKind, VL);
   }
   InstructionCost
   getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
                                    ArrayRef<Type *> Tys,
-                                   TargetCostKind CostKind) override {
+                                   TargetCostKind CostKind) const override {
     return Impl.getOperandsScalarizationOverhead(Args, Tys, CostKind);
   }
 
-  bool supportsEfficientVectorElementLoadStore() override {
+  bool supportsEfficientVectorElementLoadStore() const override {
     return Impl.supportsEfficientVectorElementLoadStore();
   }
 
-  bool supportsTailCalls() override { return Impl.supportsTailCalls(); }
-  bool supportsTailCallFor(const CallBase *CB) override {
+  bool supportsTailCalls() const override { return Impl.supportsTailCalls(); }
+  bool supportsTailCallFor(const CallBase *CB) const override {
     return Impl.supportsTailCallFor(CB);
   }
 
-  bool enableAggressiveInterleaving(bool LoopHasReductions) override {
+  bool enableAggressiveInterleaving(bool LoopHasReductions) const override {
     return Impl.enableAggressiveInterleaving(LoopHasReductions);
   }
   MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize,
                                                bool IsZeroCmp) const override {
     return Impl.enableMemCmpExpansion(OptSize, IsZeroCmp);
   }
-  bool enableSelectOptimize() override {
+  bool enableSelectOptimize() const override {
     return Impl.enableSelectOptimize();
   }
-  bool shouldTreatInstructionLikeSelect(const Instruction *I) override {
+  bool shouldTreatInstructionLikeSelect(const Instruction *I) const override {
     return Impl.shouldTreatInstructionLikeSelect(I);
   }
-  bool enableInterleavedAccessVectorization() override {
+  bool enableInterleavedAccessVectorization() const override {
     return Impl.enableInterleavedAccessVectorization();
   }
-  bool enableMaskedInterleavedAccessVectorization() override {
+  bool enableMaskedInterleavedAccessVectorization() const override {
     return Impl.enableMaskedInterleavedAccessVectorization();
   }
-  bool isFPVectorizationPotentiallyUnsafe() override {
+  bool isFPVectorizationPotentiallyUnsafe() const override {
     return Impl.isFPVectorizationPotentiallyUnsafe();
   }
   bool allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned BitWidth,
                                       unsigned AddressSpace, Align Alignment,
-                                      unsigned *Fast) override {
+                                      unsigned *Fast) const override {
     return Impl.allowsMisalignedMemoryAccesses(Context, BitWidth, AddressSpace,
                                                Alignment, Fast);
   }
-  PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) override {
+  PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) const override {
     return Impl.getPopcntSupport(IntTyWidthInBit);
   }
-  bool haveFastSqrt(Type *Ty) override { return Impl.haveFastSqrt(Ty); }
+  bool haveFastSqrt(Type *Ty) const override { return Impl.haveFastSqrt(Ty); }
 
-  bool isExpensiveToSpeculativelyExecute(const Instruction* I) override {
+  bool isExpensiveToSpeculativelyExecute(const Instruction *I) const override {
     return Impl.isExpensiveToSpeculativelyExecute(I);
   }
 
-  bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) override {
+  bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const override {
     return Impl.isFCmpOrdCheaperThanFCmpZero(Ty);
   }
 
-  InstructionCost getFPOpCost(Type *Ty) override {
+  InstructionCost getFPOpCost(Type *Ty) const override {
     return Impl.getFPOpCost(Ty);
   }
 
   InstructionCost getIntImmCodeSizeCost(unsigned Opc, unsigned Idx,
-                                        const APInt &Imm, Type *Ty) override {
+                                        const APInt &Imm,
+                                        Type *Ty) const override {
     return Impl.getIntImmCodeSizeCost(Opc, Idx, Imm, Ty);
   }
   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
-                                TargetCostKind CostKind) override {
+                                TargetCostKind CostKind) const override {
     return Impl.getIntImmCost(Imm, Ty, CostKind);
   }
-  InstructionCost getIntImmCostInst(unsigned Opc, unsigned Idx,
-                                    const APInt &Imm, Type *Ty,
-                                    TargetCostKind CostKind,
-                                    Instruction *Inst = nullptr) override {
+  InstructionCost
+  getIntImmCostInst(unsigned Opc, unsigned Idx, const APInt &Imm, Type *Ty,
+                    TargetCostKind CostKind,
+                    Instruction *Inst = nullptr) const override {
     return Impl.getIntImmCostInst(Opc, Idx, Imm, Ty, CostKind, Inst);
   }
   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                       const APInt &Imm, Type *Ty,
-                                      TargetCostKind CostKind) override {
+                                      TargetCostKind CostKind) const override {
     return Impl.getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
   }
   bool preferToKeepConstantsAttached(const Instruction &Inst,
@@ -2819,7 +2845,8 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy);
   }
   bool shouldConsiderAddressTypePromotion(
-      const Instruction &I, bool &AllowPromotionWithoutCommonHeader) override {
+      const Instruction &I,
+      bool &AllowPromotionWithoutCommonHeader) const override {
     return Impl.shouldConsiderAddressTypePromotion(
         I, AllowPromotionWithoutCommonHeader);
   }
@@ -2880,20 +2907,21 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                         BinOp);
   }
 
-  unsigned getMaxInterleaveFactor(ElementCount VF) override {
+  unsigned getMaxInterleaveFactor(ElementCount VF) const override {
     return Impl.getMaxInterleaveFactor(VF);
   }
-  unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
-                                            unsigned &JTSize,
-                                            ProfileSummaryInfo *PSI,
-                                            BlockFrequencyInfo *BFI) override {
+  unsigned
+  getEstimatedNumberOfCaseClusters(const SwitchInst &SI, unsigned &JTSize,
+                                   ProfileSummaryInfo *PSI,
+                                   BlockFrequencyInfo *BFI) const override {
     return Impl.getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI);
   }
-  InstructionCost getArithmeticInstrCost(
-      unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
-      OperandValueInfo Opd1Info, OperandValueInfo Opd2Info,
-      ArrayRef<const Value *> Args,
-      const Instruction *CxtI = nullptr) override {
+  InstructionCost
+  getArithmeticInstrCost(unsigned Opcode, Type *Ty,
+                         TTI::TargetCostKind CostKind,
+                         OperandValueInfo Opd1Info, OperandValueInfo Opd2Info,
+                         ArrayRef<const Value *> Args,
+                         const Instruction *CxtI = nullptr) const override {
     return Impl.getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, Opd2Info,
                                        Args, CxtI);
   }
@@ -2909,23 +2937,24 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,
                                  ArrayRef<const Value *> Args,
-                                 const Instruction *CxtI) override {
+                                 const Instruction *CxtI) const override {
     return Impl.getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp, Args,
                                CxtI);
   }
   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                    CastContextHint CCH,
                                    TTI::TargetCostKind CostKind,
-                                   const Instruction *I) override {
+                                   const Instruction *I) const override {
     return Impl.getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
   }
   InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst,
                                            VectorType *VecTy,
-                                           unsigned Index) override {
+                                           unsigned Index) const override {
     return Impl.getExtractWithExtendCost(Opcode, Dst, VecTy, Index);
   }
-  InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                 const Instruction *I = nullptr) override {
+  InstructionCost
+  getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
+                 const Instruction *I = nullptr) const override {
     return Impl.getCFInstrCost(Opcode, CostKind, I);
   }
   InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
@@ -2933,58 +2962,60 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                      TTI::TargetCostKind CostKind,
                                      OperandValueInfo Op1Info,
                                      OperandValueInfo Op2Info,
-                                     const Instruction *I) override {
+                                     const Instruction *I) const override {
     return Impl.getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
                                    Op1Info, Op2Info, I);
   }
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
                                      TTI::TargetCostKind CostKind,
                                      unsigned Index, Value *Op0,
-                                     Value *Op1) override {
+                                     Value *Op1) const override {
     return Impl.getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);
   }
-  InstructionCost getVectorInstrCost(
-      unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
-      Value *Scalar,
-      ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) override {
+  InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
+                                     TTI::TargetCostKind CostKind,
+                                     unsigned Index, Value *Scalar,
+                                     ArrayRef<std::tuple<Value *, User *, int>>
+                                         ScalarUserAndIdx) const override {
     return Impl.getVectorInstrCost(Opcode, Val, CostKind, Index, Scalar,
                                    ScalarUserAndIdx);
   }
   InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
                                      TTI::TargetCostKind CostKind,
-                                     unsigned Index) override {
+                                     unsigned Index) const override {
     return Impl.getVectorInstrCost(I, Val, CostKind, Index);
   }
   InstructionCost
   getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
                             const APInt &DemandedDstElts,
-                            TTI::TargetCostKind CostKind) override {
+                            TTI::TargetCostKind CostKind) const override {
     return Impl.getReplicationShuffleCost(EltTy, ReplicationFactor, VF,
                                           DemandedDstElts, CostKind);
   }
   InstructionCost
   getInsertExtractValueCost(unsigned Opcode,
-                            TTI::TargetCostKind CostKind) override {
+                            TTI::TargetCostKind CostKind) const override {
     return Impl.getInsertExtractValueCost(Opcode, CostKind);
   }
   InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                   unsigned AddressSpace,
                                   TTI::TargetCostKind CostKind,
                                   OperandValueInfo OpInfo,
-                                  const Instruction *I) override {
+                                  const Instruction *I) const override {
     return Impl.getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
                                 OpInfo, I);
   }
   InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                     unsigned AddressSpace,
                                     TTI::TargetCostKind CostKind,
-                                    const Instruction *I) override {
+                                    const Instruction *I) const override {
     return Impl.getVPMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                   CostKind, I);
   }
-  InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
-                                        Align Alignment, unsigned AddressSpace,
-                                        TTI::TargetCostKind CostKind) override {
+  InstructionCost
+  getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
+                        unsigned AddressSpace,
+                        TTI::TargetCostKind CostKind) const override {
     return Impl.getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                       CostKind);
   }
@@ -2992,13 +3023,15 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
                          bool VariableMask, Align Alignment,
                          TTI::TargetCostKind CostKind,
-                         const Instruction *I = nullptr) override {
+                         const Instruction *I = nullptr) const override {
     return Impl.getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
                                        Alignment, CostKind, I);
   }
-  InstructionCost getExpandCompressMemoryOpCost(
-      unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
-      TTI::TargetCostKind CostKind, const Instruction *I = nullptr) override {
+  InstructionCost
+  getExpandCompressMemoryOpCost(unsigned Opcode, Type *DataTy,
+                                bool VariableMask, Align Alignment,
+                                TTI::TargetCostKind CostKind,
+                                const Instruction *I = nullptr) const override {
     return Impl.getExpandCompressMemoryOpCost(Opcode, DataTy, VariableMask,
                                               Alignment, CostKind, I);
   }
@@ -3006,14 +3039,14 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
                          bool VariableMask, Align Alignment,
                          TTI::TargetCostKind CostKind,
-                         const Instruction *I = nullptr) override {
+                         const Instruction *I = nullptr) const override {
     return Impl.getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
                                        Alignment, CostKind, I);
   }
   InstructionCost getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
-      bool UseMaskForCond, bool UseMaskForGaps) override {
+      bool UseMaskForCond, bool UseMaskForGaps) const override {
     return Impl.getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
                                            Alignment, AddressSpace, CostKind,
                                            UseMaskForCond, UseMaskForGaps);
@@ -3021,54 +3054,56 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   InstructionCost
   getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                              std::optional<FastMathFlags> FMF,
-                             TTI::TargetCostKind CostKind) override {
+                             TTI::TargetCostKind CostKind) const override {
     return Impl.getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
   }
   InstructionCost
   getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
-                         TTI::TargetCostKind CostKind) override {
+                         TTI::TargetCostKind CostKind) const override {
     return Impl.getMinMaxReductionCost(IID, Ty, FMF, CostKind);
   }
   InstructionCost
   getExtendedReductionCost(unsigned Opcode, bool IsUnsigned, Type *ResTy,
                            VectorType *Ty, std::optional<FastMathFlags> FMF,
-                           TTI::TargetCostKind CostKind) override {
+                           TTI::TargetCostKind CostKind) const override {
     return Impl.getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF,
                                          CostKind);
   }
   InstructionCost
   getMulAccReductionCost(bool IsUnsigned, Type *ResTy, VectorType *Ty,
-                         TTI::TargetCostKind CostKind) override {
+                         TTI::TargetCostKind CostKind) const override {
     return Impl.getMulAccReductionCost(IsUnsigned, ResTy, Ty, CostKind);
   }
-  InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
-                                        TTI::TargetCostKind CostKind) override {
+  InstructionCost
+  getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
+                        TTI::TargetCostKind CostKind) const override {
     return Impl.getIntrinsicInstrCost(ICA, CostKind);
   }
-  InstructionCost getCallInstrCost(Function *F, Type *RetTy,
-                                   ArrayRef<Type *> Tys,
-                                   TTI::TargetCostKind CostKind) override {
+  InstructionCost
+  getCallInstrCost(Function *F, Type *RetTy, ArrayRef<Type *> Tys,
+                   TTI::TargetCostKind CostKind) const override {
     return Impl.getCallInstrCost(F, RetTy, Tys, CostKind);
   }
-  unsigned getNumberOfParts(Type *Tp) override {
+  unsigned getNumberOfParts(Type *Tp) const override {
     return Impl.getNumberOfParts(Tp);
   }
   InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
-                                            const SCEV *Ptr) override {
+                                            const SCEV *Ptr) const override {
     return Impl.getAddressComputationCost(Ty, SE, Ptr);
   }
-  InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) override {
+  InstructionCost
+  getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const override {
     return Impl.getCostOfKeepingLiveOverCall(Tys);
   }
   bool getTgtMemIntrinsic(IntrinsicInst *Inst,
-                          MemIntrinsicInfo &Info) override {
+                          MemIntrinsicInfo &Info) const override {
     return Impl.getTgtMemIntrinsic(Inst, Info);
   }
   unsigned getAtomicMemIntrinsicMaxElementSize() const override {
     return Impl.getAtomicMemIntrinsicMaxElementSize();
   }
   Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
-                                           Type *ExpectedType) override {
+                                           Type *ExpectedType) const override {
     return Impl.getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
   }
   Type *getMemcpyLoopLoweringType(

From 112ffe7c621e8bf145c5c5926e36f3af9e0fd045 Mon Sep 17 00:00:00 2001
From: Sylvestre Ledru <sylvestre@debian.org>
Date: Tue, 22 Apr 2025 11:50:58 +0200
Subject: [PATCH 705/710] Add support of the next Ubuntu (Ubuntu 25.10 -
 Questing Quokka)

---
 clang/include/clang/Driver/Distro.h | 3 ++-
 clang/lib/Driver/Distro.cpp         | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/clang/include/clang/Driver/Distro.h b/clang/include/clang/Driver/Distro.h
index b4d485dac8a26..9f27c2baaeb47 100644
--- a/clang/include/clang/Driver/Distro.h
+++ b/clang/include/clang/Driver/Distro.h
@@ -81,6 +81,7 @@ class Distro {
     UbuntuNoble,
     UbuntuOracular,
     UbuntuPlucky,
+    UbuntuQuesting,
     UnknownDistro
   };
 
@@ -132,7 +133,7 @@ class Distro {
   }
 
   bool IsUbuntu() const {
-    return DistroVal >= UbuntuHardy && DistroVal <= UbuntuPlucky;
+    return DistroVal >= UbuntuHardy && DistroVal <= UbuntuQuesting;
   }
 
   bool IsAlpineLinux() const { return DistroVal == AlpineLinux; }
diff --git a/clang/lib/Driver/Distro.cpp b/clang/lib/Driver/Distro.cpp
index 3cc79535de8da..82c627819d9fc 100644
--- a/clang/lib/Driver/Distro.cpp
+++ b/clang/lib/Driver/Distro.cpp
@@ -96,6 +96,7 @@ static Distro::DistroType DetectLsbRelease(llvm::vfs::FileSystem &VFS) {
                     .Case("noble", Distro::UbuntuNoble)
                     .Case("oracular", Distro::UbuntuOracular)
                     .Case("plucky", Distro::UbuntuPlucky)
+                    .Case("questing", Distro::UbuntuQuesting)
                     .Default(Distro::UnknownDistro);
   return Version;
 }

From 97eb416c65863cdf25ec3fa19ed056aac38d1013 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Tue, 22 Apr 2025 10:55:50 +0100
Subject: [PATCH 706/710] [flang][Parser][OpenMP] Fix unparser for
 cancellation_construct_type (#136001)

Previously the unparser would print like
```
!$OMP CANCEL CANCELLATION_CONSTRUCT_TYPE(SECTIONS)
```

This is not valid Fortran. I have fixed it to print without the clause
name.
---
 flang/lib/Parser/unparse.cpp                  |  4 ++
 flang/test/Parser/OpenMP/cancel.f90           | 53 +++++++++++++++++++
 .../llvm/Frontend/Directive/DirectiveBase.td  |  3 ++
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 llvm/include/llvm/TableGen/DirectiveEmitter.h |  4 ++
 .../utils/TableGen/Basic/DirectiveEmitter.cpp |  2 +
 6 files changed, 67 insertions(+)
 create mode 100644 flang/test/Parser/OpenMP/cancel.f90

diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 47dae0ae753d2..35fb0790b633d 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2871,6 +2871,10 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
+  // Clause unparsers are usually generated by tablegen in the form
+  // CLAUSE(VALUE). Here we only want to print VALUE so a custom unparser is
+  // needed.
+  void Unparse(const OmpClause::CancellationConstructType &x) { Walk(x.v); }
   void Unparse(const OpenMPCancellationPointConstruct &x) {
     BeginOpenMP();
     Word("!$OMP ");
diff --git a/flang/test/Parser/OpenMP/cancel.f90 b/flang/test/Parser/OpenMP/cancel.f90
new file mode 100644
index 0000000000000..f5c1309e0d8c1
--- /dev/null
+++ b/flang/test/Parser/OpenMP/cancel.f90
@@ -0,0 +1,53 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck %s
+
+! CHECK: SUBROUTINE parallel
+subroutine parallel
+! CHECK: !$OMP PARALLEL
+  !$omp parallel
+! CHECK: !$OMP CANCEL PARALLEL
+    !$omp cancel parallel
+! CHECK: !$OMP END PARALLEL
+  !$omp end parallel
+! CHECK: END SUBROUTINE
+end subroutine
+
+! CHECK: SUBROUTINE sections
+subroutine sections
+! CHECK: !$OMP PARALLEL SECTIONS
+  !$omp parallel sections
+! CHECK: !$OMP CANCEL SECTIONS
+    !$omp cancel sections
+! CHECK: !$OMP END PARALLEL SECTIONS
+  !$omp end parallel sections
+! CHECK: END SUBROUTINE
+end subroutine
+
+! CHECK: SUBROUTINE loop
+subroutine loop
+! CHECK: !$OMP PARALLEL DO
+  !$omp parallel do
+! CHECK: DO i=1_4,10_4
+    do i=1,10
+! CHECK: !$OMP CANCEL DO
+      !$omp cancel do
+! CHECK: END DO
+    enddo
+! CHECK: !$OMP END PARALLEL DO
+  !$omp end parallel do
+! CHECK: END SUBROUTINE
+end subroutine
+
+! CHECK: SUBROUTINE taskgroup
+subroutine taskgroup
+! CHECK: !$OMP TASKGROUP
+  !$omp taskgroup
+! CHECK: !$OMP TASK
+    !$omp task
+! CHECK: !$OMP CANCEL TASKGROUP
+      !$omp cancel taskgroup
+! CHECK: !$OMP END TASK
+    !$omp end task
+! CHECK: !$OMP END TASKGROUP
+  !$omp end taskgroup
+! CHECK: END SUBROUTINE
+end subroutine
diff --git a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
index 87ce80846dd93..912ba15a6aa5a 100644
--- a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
+++ b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
@@ -83,6 +83,9 @@ class Clause<string c> {
   // Optional class holding value of the clause in flang AST.
   string flangClass = "";
 
+  // If set to true, don't emit flang Unparser.
+  bit skipFlangUnparser = false;
+
   // If set to true, value is optional. Not optional by default.
   bit isValueOptional = false;
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index e2a1449d8cc76..1f286abaa9679 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -106,6 +106,7 @@ def OMPC_CancellationConstructType : Clause<"cancellation_construct_type"> {
     OMP_CANCELLATION_CONSTRUCT_None
   ];
   let flangClass = "OmpCancellationConstructTypeClause";
+  let skipFlangUnparser = true;
 }
 def OMPC_Contains : Clause<"contains"> {
   let clangClass = "OMPContainsClause";
diff --git a/llvm/include/llvm/TableGen/DirectiveEmitter.h b/llvm/include/llvm/TableGen/DirectiveEmitter.h
index 28373271ef1e8..22a0ae583e84d 100644
--- a/llvm/include/llvm/TableGen/DirectiveEmitter.h
+++ b/llvm/include/llvm/TableGen/DirectiveEmitter.h
@@ -257,6 +257,10 @@ class Clause : public BaseRecord {
     return Def->getValueAsListOfDefs("allowedClauseValues");
   }
 
+  bool skipFlangUnparser() const {
+    return Def->getValueAsBit("skipFlangUnparser");
+  }
+
   bool isValueOptional() const { return Def->getValueAsBit("isValueOptional"); }
 
   bool isValueList() const { return Def->getValueAsBit("isValueList"); }
diff --git a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
index b5276d695f1bc..96ba89d0e78cd 100644
--- a/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
@@ -960,6 +960,8 @@ static void generateFlangClauseUnparse(const DirectiveLanguage &DirLang,
   OS << "\n";
 
   for (const Clause Clause : DirLang.getClauses()) {
+    if (Clause.skipFlangUnparser())
+      continue;
     if (!Clause.getFlangClass().empty()) {
       if (Clause.isValueOptional() && Clause.getDefaultValue().empty()) {
         OS << "void Unparse(const " << DirLang.getFlangClauseBaseClass()

From c1940cd0deef84c9cbbf6966d956b8fef4519efa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche@google.com>
Date: Tue, 22 Apr 2025 09:58:55 +0000
Subject: [PATCH 707/710] [SPIR-V] Add store legalization for ptrcast (#135369)

This commits adds handling for spv.ptrcast result being used in a store
instruction, modifying the store to operate on the source type.
---
 .../Target/SPIRV/SPIRVLegalizePointerCast.cpp | 133 ++++++++++++++++++
 .../pointers/getelementptr-downcast-struct.ll |  20 +++
 .../pointers/getelementptr-downcast-vector.ll | 110 +++++++++++++++
 .../CodeGen/SPIRV/pointers/store-struct.ll    | 114 +++++++++++++++
 4 files changed, 377 insertions(+)
 create mode 100644 llvm/test/CodeGen/SPIRV/pointers/store-struct.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
index 69051132cacff..5cda6a07352d5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
@@ -151,6 +151,124 @@ class SPIRVLegalizePointerCast : public FunctionPass {
     DeadInstructions.push_back(LI);
   }
 
+  // Creates an spv_insertelt instruction (equivalent to llvm's insertelement).
+  Value *makeInsertElement(IRBuilder<> &B, Value *Vector, Value *Element,
+                           unsigned Index) {
+    Type *Int32Ty = Type::getInt32Ty(B.getContext());
+    SmallVector<Type *, 4> Types = {Vector->getType(), Vector->getType(),
+                                    Element->getType(), Int32Ty};
+    SmallVector<Value *> Args = {Vector, Element, B.getInt32(Index)};
+    Instruction *NewI =
+        B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
+    buildAssignType(B, Vector->getType(), NewI);
+    return NewI;
+  }
+
+  // Creates an spv_extractelt instruction (equivalent to llvm's
+  // extractelement).
+  Value *makeExtractElement(IRBuilder<> &B, Type *ElementType, Value *Vector,
+                            unsigned Index) {
+    Type *Int32Ty = Type::getInt32Ty(B.getContext());
+    SmallVector<Type *, 3> Types = {ElementType, Vector->getType(), Int32Ty};
+    SmallVector<Value *> Args = {Vector, B.getInt32(Index)};
+    Instruction *NewI =
+        B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
+    buildAssignType(B, ElementType, NewI);
+    return NewI;
+  }
+
+  // Stores the given Src vector operand into the Dst vector, adjusting the size
+  // if required.
+  Value *storeVectorFromVector(IRBuilder<> &B, Value *Src, Value *Dst,
+                               Align Alignment) {
+    FixedVectorType *SrcType = cast<FixedVectorType>(Src->getType());
+    FixedVectorType *DstType =
+        cast<FixedVectorType>(GR->findDeducedElementType(Dst));
+    assert(DstType->getNumElements() >= SrcType->getNumElements());
+
+    LoadInst *LI = B.CreateLoad(DstType, Dst);
+    LI->setAlignment(Alignment);
+    Value *OldValues = LI;
+    buildAssignType(B, OldValues->getType(), OldValues);
+    Value *NewValues = Src;
+
+    for (unsigned I = 0; I < SrcType->getNumElements(); ++I) {
+      Value *Element =
+          makeExtractElement(B, SrcType->getElementType(), NewValues, I);
+      OldValues = makeInsertElement(B, OldValues, Element, I);
+    }
+
+    StoreInst *SI = B.CreateStore(OldValues, Dst);
+    SI->setAlignment(Alignment);
+    return SI;
+  }
+
+  void buildGEPIndexChain(IRBuilder<> &B, Type *Search, Type *Aggregate,
+                          SmallVectorImpl<Value *> &Indices) {
+    Indices.push_back(B.getInt32(0));
+
+    if (Search == Aggregate)
+      return;
+
+    if (auto *ST = dyn_cast<StructType>(Aggregate))
+      buildGEPIndexChain(B, Search, ST->getTypeAtIndex(0u), Indices);
+    else if (auto *AT = dyn_cast<ArrayType>(Aggregate))
+      buildGEPIndexChain(B, Search, AT->getElementType(), Indices);
+    else if (auto *VT = dyn_cast<FixedVectorType>(Aggregate))
+      buildGEPIndexChain(B, Search, VT->getElementType(), Indices);
+    else
+      llvm_unreachable("Bad access chain?");
+  }
+
+  // Stores the given Src value into the first entry of the Dst aggregate.
+  Value *storeToFirstValueAggregate(IRBuilder<> &B, Value *Src, Value *Dst,
+                                    Type *DstPointeeType, Align Alignment) {
+    SmallVector<Type *, 2> Types = {Dst->getType(), Dst->getType()};
+    SmallVector<Value *, 3> Args{/* isInBounds= */ B.getInt1(true), Dst};
+    buildGEPIndexChain(B, Src->getType(), DstPointeeType, Args);
+    auto *GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
+    GR->buildAssignPtr(B, Src->getType(), GEP);
+    StoreInst *SI = B.CreateStore(Src, GEP);
+    SI->setAlignment(Alignment);
+    return SI;
+  }
+
+  bool isTypeFirstElementAggregate(Type *Search, Type *Aggregate) {
+    if (Search == Aggregate)
+      return true;
+    if (auto *ST = dyn_cast<StructType>(Aggregate))
+      return isTypeFirstElementAggregate(Search, ST->getTypeAtIndex(0u));
+    if (auto *VT = dyn_cast<FixedVectorType>(Aggregate))
+      return isTypeFirstElementAggregate(Search, VT->getElementType());
+    if (auto *AT = dyn_cast<ArrayType>(Aggregate))
+      return isTypeFirstElementAggregate(Search, AT->getElementType());
+    return false;
+  }
+
+  // Transforms a store instruction (or SPV intrinsic) using a ptrcast as
+  // operand into a valid logical SPIR-V store with no ptrcast.
+  void transformStore(IRBuilder<> &B, Instruction *BadStore, Value *Src,
+                      Value *Dst, Align Alignment) {
+    Type *ToTy = GR->findDeducedElementType(Dst);
+    Type *FromTy = Src->getType();
+
+    auto *S_VT = dyn_cast<FixedVectorType>(FromTy);
+    auto *D_ST = dyn_cast<StructType>(ToTy);
+    auto *D_VT = dyn_cast<FixedVectorType>(ToTy);
+
+    B.SetInsertPoint(BadStore);
+    if (D_ST && isTypeFirstElementAggregate(FromTy, D_ST))
+      storeToFirstValueAggregate(B, Src, Dst, D_ST, Alignment);
+    else if (D_VT && S_VT)
+      storeVectorFromVector(B, Src, Dst, Alignment);
+    else if (D_VT && !S_VT && FromTy == D_VT->getElementType())
+      storeToFirstValueAggregate(B, Src, Dst, D_VT, Alignment);
+    else
+      llvm_unreachable("Unsupported ptrcast use in store. Please fix.");
+
+    DeadInstructions.push_back(BadStore);
+  }
+
   void legalizePointerCast(IntrinsicInst *II) {
     Value *CastedOperand = II;
     Value *OriginalOperand = II->getOperand(0);
@@ -166,6 +284,12 @@ class SPIRVLegalizePointerCast : public FunctionPass {
         continue;
       }
 
+      if (StoreInst *SI = dyn_cast<StoreInst>(User)) {
+        transformStore(B, SI, SI->getValueOperand(), OriginalOperand,
+                       SI->getAlign());
+        continue;
+      }
+
       if (IntrinsicInst *Intrin = dyn_cast<IntrinsicInst>(User)) {
         if (Intrin->getIntrinsicID() == Intrinsic::spv_assign_ptr_type) {
           DeadInstructions.push_back(Intrin);
@@ -177,6 +301,15 @@ class SPIRVLegalizePointerCast : public FunctionPass {
                                  /* DeleteOld= */ false);
           continue;
         }
+
+        if (Intrin->getIntrinsicID() == Intrinsic::spv_store) {
+          Align Alignment;
+          if (ConstantInt *C = dyn_cast<ConstantInt>(Intrin->getOperand(3)))
+            Alignment = Align(C->getZExtValue());
+          transformStore(B, Intrin, Intrin->getArgOperand(0), OriginalOperand,
+                         Alignment);
+          continue;
+        }
       }
 
       llvm_unreachable("Unsupported ptrcast user. Please fix.");
diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-struct.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-struct.ll
index b0a68a30e29be..35e5880881e5c 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-struct.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-struct.ll
@@ -45,3 +45,23 @@ entry:
   %val = load i32, ptr addrspace(10) %ptr
   ret i32 %val
 }
+
+define spir_func void @foos(i64 noundef %index) local_unnamed_addr {
+; CHECK: %[[#index:]] = OpFunctionParameter %[[#uint64]]
+entry:
+; CHECK: %[[#ptr:]] = OpInBoundsAccessChain %[[#uint_pp]] %[[#global1]] %[[#uint_0]] %[[#index]]
+  %ptr = getelementptr inbounds %S1, ptr addrspace(10) @global1, i64 0, i32 0, i64 %index
+; CHECK: OpStore %[[#ptr]] %[[#uint_0]] Aligned 4
+  store i32 0, ptr addrspace(10) %ptr
+  ret void
+}
+
+define spir_func void @bars(i64 noundef %index) local_unnamed_addr {
+; CHECK: %[[#index:]] = OpFunctionParameter %[[#uint64]]
+entry:
+; CHECK: %[[#ptr:]] = OpInBoundsAccessChain %[[#uint_pp]] %[[#global2]] %[[#uint_0]] %[[#uint_0]] %[[#index]] %[[#uint_1]]
+  %ptr = getelementptr inbounds %S2, ptr addrspace(10) @global2, i64 0, i32 0, i32 0, i64 %index, i32 1
+; CHECK: OpStore %[[#ptr]] %[[#uint_0]] Aligned 4
+  store i32 0, ptr addrspace(10) %ptr
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll
index 1287a512ac96d..e20aba80015be 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll
@@ -5,9 +5,13 @@
 ; CHECK-DAG: %[[#uint_pp:]] = OpTypePointer Private %[[#uint]]
 ; CHECK-DAG: %[[#uint_fp:]] = OpTypePointer Function %[[#uint]]
 ; CHECK-DAG:  %[[#uint_0:]] = OpConstant %[[#uint]] 0
+; CHECK-DAG:  %[[#uint_1:]] = OpConstant %[[#uint]] 1
+; CHECK-DAG:  %[[#uint_2:]] = OpConstant %[[#uint]] 2
 ; CHECK-DAG:      %[[#v2:]] = OpTypeVector %[[#uint]] 2
 ; CHECK-DAG:      %[[#v3:]] = OpTypeVector %[[#uint]] 3
 ; CHECK-DAG:      %[[#v4:]] = OpTypeVector %[[#uint]] 4
+; CHECK-DAG:   %[[#v2_01:]] = OpConstantComposite %[[#v2]] %[[#uint_0]] %[[#uint_1]]
+; CHECK-DAG:  %[[#v3_012:]] = OpConstantComposite %[[#v3]] %[[#uint_0]] %[[#uint_1]] %[[#uint_2]]
 ; CHECK-DAG:   %[[#v4_pp:]] = OpTypePointer Private %[[#v4]]
 ; CHECK-DAG:   %[[#v4_fp:]] = OpTypePointer Function %[[#v4]]
 
@@ -108,3 +112,109 @@ define internal spir_func i32 @bazBounds(ptr %a) {
   ret i32 %2
 ; CHECK: OpReturnValue %[[#val]]
 }
+
+define internal spir_func void @foos(ptr addrspace(10) %a) {
+
+  %1 = getelementptr inbounds <4 x i32>, ptr addrspace(10) %a, i64 0
+; CHECK: %[[#ptr:]]  = OpInBoundsAccessChain %[[#v4_pp]] %[[#]]
+
+  store <3 x i32> <i32 0, i32 1, i32 2>, ptr addrspace(10) %1, align 16
+; CHECK: %[[#out0:]] = OpLoad %[[#v4]] %[[#ptr]] Aligned 16
+; CHECK:    %[[#A:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 0
+; CHECK: %[[#out1:]] = OpCompositeInsert %[[#v4]] %[[#A]] %[[#out0]] 0
+; CHECK:    %[[#B:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 1
+; CHECK: %[[#out2:]] = OpCompositeInsert %[[#v4]] %[[#B]] %[[#out1]] 1
+; CHECK:    %[[#C:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 2
+; CHECK: %[[#out3:]] = OpCompositeInsert %[[#v4]] %[[#C]] %[[#out2]] 2
+; CHECK: OpStore %[[#ptr]] %[[#out3]] Aligned 16
+
+  ret void
+}
+
+define internal spir_func void @foosDefault(ptr %a) {
+
+  %1 = getelementptr inbounds <4 x i32>, ptr %a, i64 0
+; CHECK: %[[#ptr:]]  = OpInBoundsAccessChain %[[#v4_fp]] %[[#]]
+
+  store <3 x i32> <i32 0, i32 1, i32 2>, ptr %1, align 16
+; CHECK: %[[#out0:]] = OpLoad %[[#v4]] %[[#ptr]] Aligned 16
+; CHECK:    %[[#A:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 0
+; CHECK: %[[#out1:]] = OpCompositeInsert %[[#v4]] %[[#A]] %[[#out0]] 0
+; CHECK:    %[[#B:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 1
+; CHECK: %[[#out2:]] = OpCompositeInsert %[[#v4]] %[[#B]] %[[#out1]] 1
+; CHECK:    %[[#C:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 2
+; CHECK: %[[#out3:]] = OpCompositeInsert %[[#v4]] %[[#C]] %[[#out2]] 2
+; CHECK: OpStore %[[#ptr]] %[[#out3]] Aligned 16
+
+  ret void
+}
+
+define internal spir_func void @foosBounds(ptr %a) {
+
+  %1 = getelementptr <4 x i32>, ptr %a, i64 0
+; CHECK: %[[#ptr:]]  = OpAccessChain %[[#v4_fp]] %[[#]]
+
+  store <3 x i32> <i32 0, i32 1, i32 2>, ptr %1, align 64
+; CHECK: %[[#out0:]] = OpLoad %[[#v4]] %[[#ptr]] Aligned 64
+; CHECK:    %[[#A:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 0
+; CHECK: %[[#out1:]] = OpCompositeInsert %[[#v4]] %[[#A]] %[[#out0]] 0
+; CHECK:    %[[#B:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 1
+; CHECK: %[[#out2:]] = OpCompositeInsert %[[#v4]] %[[#B]] %[[#out1]] 1
+; CHECK:    %[[#C:]] = OpCompositeExtract %[[#uint]] %[[#v3_012]] 2
+; CHECK: %[[#out3:]] = OpCompositeInsert %[[#v4]] %[[#C]] %[[#out2]] 2
+; CHECK: OpStore %[[#ptr]] %[[#out3]] Aligned 64
+
+  ret void
+}
+
+define internal spir_func void @bars(ptr addrspace(10) %a) {
+
+  %1 = getelementptr <4 x i32>, ptr addrspace(10) %a, i64 0
+; CHECK: %[[#ptr:]]  = OpAccessChain %[[#v4_pp]] %[[#]]
+
+  store <2 x i32> <i32 0, i32 1>, ptr addrspace(10) %1, align 16
+; CHECK: %[[#out0:]] = OpLoad %[[#v4]] %[[#ptr]] Aligned 16
+; CHECK:    %[[#A:]] = OpCompositeExtract %[[#uint]] %[[#v2_01]] 0
+; CHECK: %[[#out1:]] = OpCompositeInsert %[[#v4]] %[[#A]] %[[#out0]] 0
+; CHECK:    %[[#B:]] = OpCompositeExtract %[[#uint]] %[[#v2_01]] 1
+; CHECK: %[[#out2:]] = OpCompositeInsert %[[#v4]] %[[#B]] %[[#out1]] 1
+; CHECK: OpStore %[[#ptr]] %[[#out2]] Aligned 1
+
+  ret void
+}
+
+define internal spir_func void @bazs(ptr addrspace(10) %a) {
+
+  %1 = getelementptr <4 x i32>, ptr addrspace(10) %a, i64 0
+; CHECK: %[[#ptr:]]  = OpAccessChain %[[#v4_pp]] %[[#]]
+
+  store i32 0, ptr addrspace(10) %1, align 32
+; CHECK:  %[[#tmp:]] = OpInBoundsAccessChain %[[#uint_pp]] %[[#ptr]] %[[#uint_0]]
+; CHECK: OpStore %[[#tmp]] %[[#uint_0]] Aligned 32
+
+  ret void
+}
+
+define internal spir_func void @bazsDefault(ptr %a) {
+
+  %1 = getelementptr inbounds <4 x i32>, ptr %a, i64 0
+; CHECK: %[[#ptr:]]  = OpInBoundsAccessChain %[[#v4_fp]] %[[#]]
+
+  store i32 0, ptr %1, align 16
+; CHECK:  %[[#tmp:]] = OpInBoundsAccessChain %[[#uint_fp]] %[[#ptr]] %[[#uint_0]]
+; CHECK: OpStore %[[#tmp]] %[[#uint_0]] Aligned 16
+
+  ret void
+}
+
+define internal spir_func void @bazsBounds(ptr %a) {
+
+  %1 = getelementptr <4 x i32>, ptr %a, i64 0
+; CHECK: %[[#ptr:]]  = OpAccessChain %[[#v4_fp]] %[[#]]
+
+  store i32 0, ptr %1, align 16
+; CHECK:  %[[#tmp:]] = OpInBoundsAccessChain %[[#uint_fp]] %[[#ptr]] %[[#uint_0]]
+; CHECK: OpStore %[[#tmp]] %[[#uint_0]] Aligned 16
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/store-struct.ll b/llvm/test/CodeGen/SPIRV/pointers/store-struct.ll
new file mode 100644
index 0000000000000..1ff53614990d5
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/store-struct.ll
@@ -0,0 +1,114 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG:      %[[#uint:]] = OpTypeInt 32 0
+; CHECK-DAG:     %[[#float:]] = OpTypeFloat 32
+; CHECK-DAG:  %[[#float_fp:]] = OpTypePointer Function %[[#float]]
+; CHECK-DAG:  %[[#float_pp:]] = OpTypePointer Private %[[#float]]
+; CHECK-DAG:   %[[#uint_fp:]] = OpTypePointer Function %[[#uint]]
+; CHECK-DAG:    %[[#uint_0:]] = OpConstant %[[#uint]] 0
+; CHECK-DAG:    %[[#uint_4:]] = OpConstant %[[#uint]] 4
+; CHECK-DAG:    %[[#float_0:]] = OpConstant %[[#float]] 0
+; CHECK-DAG:        %[[#sf:]] = OpTypeStruct %[[#float]]
+; CHECK-DAG:        %[[#su:]] = OpTypeStruct %[[#uint]]
+; CHECK-DAG:       %[[#ssu:]] = OpTypeStruct %[[#su]]
+; CHECK-DAG:        %[[#sfuf:]] = OpTypeStruct %[[#float]] %[[#uint]] %[[#float]]
+; CHECK-DAG:        %[[#uint4:]] = OpTypeVector %[[#uint]] 4
+; CHECK-DAG:        %[[#sv:]] = OpTypeStruct %[[#uint4]]
+; CHECK-DAG:        %[[#ssv:]] = OpTypeStruct %[[#sv]]
+; CHECK-DAG:        %[[#assv:]] = OpTypeArray %[[#ssv]] %[[#uint_4]]
+; CHECK-DAG:        %[[#sassv:]] = OpTypeStruct %[[#assv]]
+; CHECK-DAG:        %[[#ssassv:]] = OpTypeStruct %[[#sassv]]
+; CHECK-DAG:     %[[#sf_fp:]] = OpTypePointer Function %[[#sf]]
+; CHECK-DAG:     %[[#su_fp:]] = OpTypePointer Function %[[#su]]
+; CHECK-DAG:    %[[#ssu_fp:]] = OpTypePointer Function %[[#ssu]]
+; CHECK-DAG:    %[[#ssv_fp:]] = OpTypePointer Function %[[#ssv]]
+; CHECK-DAG: %[[#ssassv_fp:]] = OpTypePointer Function %[[#ssassv]]
+; CHECK-DAG:   %[[#sfuf_fp:]] = OpTypePointer Function %[[#sfuf]]
+; CHECK-DAG:   %[[#sfuf_pp:]] = OpTypePointer Private %[[#sfuf]]
+
+%struct.SF = type { float }
+%struct.SU = type { i32 }
+%struct.SFUF = type { float, i32, float }
+%struct.SSU = type { %struct.SU }
+%struct.SV = type { <4 x i32> }
+%struct.SSV = type { %struct.SV }
+%struct.SASSV = type { [4 x %struct.SSV] }
+%struct.SSASSV = type { %struct.SASSV }
+
+@gsfuf = external addrspace(10) global %struct.SFUF
+; CHECK-DAG: %[[#gsfuf:]] = OpVariable %[[#sfuf_pp]] Private
+
+define internal spir_func void @foo() {
+  %1 = alloca %struct.SF, align 4
+; CHECK: %[[#var:]]  = OpVariable %[[#sf_fp]] Function
+
+  store float 0.0, ptr %1, align 4
+; CHECK: %[[#tmp:]]  = OpInBoundsAccessChain %[[#float_fp]] %[[#var]] %[[#uint_0]]
+; CHECK:               OpStore %[[#tmp]] %[[#float_0]] Aligned 4
+
+  ret void
+}
+
+define internal spir_func void @bar() {
+  %1 = alloca %struct.SU, align 4
+; CHECK: %[[#var:]]  = OpVariable %[[#su_fp]] Function
+
+  store i32 0, ptr %1, align 4
+; CHECK: %[[#tmp:]]  = OpInBoundsAccessChain %[[#uint_fp]] %[[#var]] %[[#uint_0]]
+; CHECK:               OpStore %[[#tmp]] %[[#uint_0]] Aligned 4
+
+  ret void
+}
+
+define internal spir_func void @baz() {
+  %1 = alloca %struct.SFUF, align 4
+; CHECK: %[[#var:]]  = OpVariable %[[#sfuf_fp]] Function
+
+  store float 0.0, ptr %1, align 4
+; CHECK: %[[#tmp:]]  = OpInBoundsAccessChain %[[#float_fp]] %[[#var]] %[[#uint_0]]
+; CHECK:               OpStore %[[#tmp]] %[[#float_0]] Aligned 4
+
+  ret void
+}
+
+define internal spir_func void @biz() {
+  store float 0.0, ptr addrspace(10) @gsfuf, align 4
+; CHECK: %[[#tmp:]]  = OpInBoundsAccessChain %[[#float_pp]] %[[#gsfuf]] %[[#uint_0]]
+; CHECK:               OpStore %[[#tmp]] %[[#float_0]] Aligned 4
+
+  ret void
+}
+
+define internal spir_func void @nested_store() {
+  %1 = alloca %struct.SSU, align 4
+; CHECK: %[[#var:]]  = OpVariable %[[#ssu_fp]] Function
+
+  store i32 0, ptr %1, align 4
+; CHECK: %[[#tmp:]]  = OpInBoundsAccessChain %[[#uint_fp]] %[[#var]] %[[#uint_0]] %[[#uint_0]]
+; CHECK:               OpStore %[[#tmp]] %[[#uint_0]] Aligned 4
+
+  ret void
+}
+
+define internal spir_func void @nested_store_vector() {
+  %1 = alloca %struct.SSV, align 4
+; CHECK: %[[#var:]]  = OpVariable %[[#ssv_fp]] Function
+
+  store i32 0, ptr %1, align 4
+; CHECK: %[[#tmp:]]  = OpInBoundsAccessChain %[[#uint_fp]] %[[#var]] %[[#uint_0]] %[[#uint_0]] %[[#uint_0]]
+; CHECK:               OpStore %[[#tmp]] %[[#uint_0]] Aligned 4
+
+  ret void
+}
+
+define internal spir_func void @nested_array_vector() {
+  %1 = alloca %struct.SSASSV, align 4
+; CHECK: %[[#var:]]  = OpVariable %[[#ssassv_fp]] Function
+
+  store i32 0, ptr %1, align 4
+; CHECK: %[[#tmp:]]  = OpInBoundsAccessChain %[[#uint_fp]] %[[#var]] %[[#uint_0]] %[[#uint_0]] %[[#uint_0]] %[[#uint_0]] %[[#uint_0]] %[[#uint_0]]
+; CHECK:               OpStore %[[#tmp]] %[[#uint_0]] Aligned 4
+
+  ret void
+}

From 8f8853a5743fe204b390f0276cb6d0266e4b9a9b Mon Sep 17 00:00:00 2001
From: quic_hchandel <quic_hchandel@quicinc.com>
Date: Tue, 22 Apr 2025 15:40:03 +0530
Subject: [PATCH 708/710] [RISCV] Add ISel patterns for Xqcia instructions
 (#136548)

This patch adds instruction selection patterns for generating the
integer arithmetic instructions.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp |   9 ++
 llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td |  13 ++
 llvm/test/CodeGen/RISCV/xqcia.ll            | 128 ++++++++++++++++++++
 3 files changed, 150 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/xqcia.ll

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 98fba9e86e88a..850ea647ea049 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -422,6 +422,15 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   else if (!Subtarget.hasVendorXTHeadCondMov())
     setOperationAction(ISD::SELECT, XLenVT, Custom);
 
+  if (Subtarget.hasVendorXqcia() && !Subtarget.is64Bit()) {
+    setOperationAction(ISD::UADDSAT, MVT::i32, Legal);
+    setOperationAction(ISD::SADDSAT, MVT::i32, Legal);
+    setOperationAction(ISD::USUBSAT, MVT::i32, Legal);
+    setOperationAction(ISD::SSUBSAT, MVT::i32, Legal);
+    setOperationAction(ISD::SSHLSAT, MVT::i32, Legal);
+    setOperationAction(ISD::USHLSAT, MVT::i32, Legal);
+  }
+
   static const unsigned FPLegalNodeTypes[] = {
       ISD::FMINNUM,       ISD::FMAXNUM,        ISD::FMINIMUMNUM,
       ISD::FMAXIMUMNUM,   ISD::LRINT,          ISD::LLRINT,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index bb0818b001d38..741c4ce8f5d1e 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -1247,6 +1247,10 @@ class PatGprNoX0Simm32NoSimm26<SDPatternOperator OpNode, RVInst48 Inst>
     : Pat<(i32 (OpNode (i32 GPRNoX0:$rs1), simm32_nosimm26:$imm)),
           (Inst GPRNoX0:$rs1, simm32_nosimm26:$imm)>;
 
+class PatGprNoX0GprNoX0<SDPatternOperator OpNode, RVInstR Inst>
+    : Pat<(i32 (OpNode (i32 GPRNoX0:$rs1), (i32 GPRNoX0:$rs2))),
+          (Inst GPRNoX0:$rs1, GPRNoX0:$rs2)>;
+
 class QC48LdPat<PatFrag LoadOp, RVInst48 Inst>
     : Pat<(i32 (LoadOp (AddLike (i32 GPR:$rs1), simm26_nosimm12:$imm26))),
           (Inst GPR:$rs1, simm26_nosimm12:$imm26)>;
@@ -1308,5 +1312,14 @@ let Predicates = [HasVendorXqcisls, IsRV32], AddedComplexity = 1 in {
   def : QCScaledStPat<store, QC_SRW>;
 } // Predicates = [HasVendorXqcisls, IsRV32], AddedComplexity = 1
 
+let Predicates = [HasVendorXqcia, IsRV32] in {
+def : PatGprNoX0GprNoX0<saddsat, QC_ADDSAT>;
+def : PatGprNoX0GprNoX0<uaddsat, QC_ADDUSAT>;
+def : PatGprNoX0GprNoX0<ssubsat, QC_SUBSAT>;
+def : PatGprNoX0GprNoX0<usubsat, QC_SUBUSAT>;
+def : PatGprNoX0GprNoX0<ushlsat, QC_SHLUSAT>;
+def : PatGprNoX0GprNoX0<sshlsat, QC_SHLSAT>;
+} // Predicates = [HasVendorXqcia, IsRV32]
+
 let Predicates = [HasVendorXqciint, IsRV32] in
 def : Pat<(riscv_mileaveret_glue), (QC_C_MILEAVERET)>;
diff --git a/llvm/test/CodeGen/RISCV/xqcia.ll b/llvm/test/CodeGen/RISCV/xqcia.ll
new file mode 100644
index 0000000000000..c75bb9daefcf2
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xqcia.ll
@@ -0,0 +1,128 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test that we are able to generate the Xqcia instructions
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV32I
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcia -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV32IXQCIA
+
+define i32 @addsat(i32 %a, i32 %b) {
+; RV32I-LABEL: addsat:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    slt a2, a0, a2
+; RV32I-NEXT:    slti a1, a1, 0
+; RV32I-NEXT:    beq a1, a2, .LBB0_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srai a0, a0, 31
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIA-LABEL: addsat:
+; RV32IXQCIA:       # %bb.0:
+; RV32IXQCIA-NEXT:    qc.addsat a0, a0, a1
+; RV32IXQCIA-NEXT:    ret
+ %saddsat = tail call i32 @llvm.sadd.sat.i32(i32 %a,i32 %b)
+ ret i32 %saddsat
+}
+
+define i32 @addusat(i32 %a, i32 %b) {
+; RV32I-LABEL: addusat:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a1, a0, a1
+; RV32I-NEXT:    sltu a0, a1, a0
+; RV32I-NEXT:    neg a0, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIA-LABEL: addusat:
+; RV32IXQCIA:       # %bb.0:
+; RV32IXQCIA-NEXT:    qc.addusat a0, a0, a1
+; RV32IXQCIA-NEXT:    ret
+ %uaddsat = tail call i32 @llvm.uadd.sat.i32(i32 %a,i32 %b)
+ ret i32 %uaddsat
+}
+
+define i32 @subsat(i32 %a, i32 %b) {
+; RV32I-LABEL: subsat:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    sgtz a3, a1
+; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    slt a1, a0, a2
+; RV32I-NEXT:    beq a3, a1, .LBB2_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srai a0, a0, 31
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:  .LBB2_2:
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIA-LABEL: subsat:
+; RV32IXQCIA:       # %bb.0:
+; RV32IXQCIA-NEXT:    qc.subsat a0, a0, a1
+; RV32IXQCIA-NEXT:    ret
+ %ssubsat = tail call i32 @llvm.ssub.sat.i32(i32 %a,i32 %b)
+ ret i32 %ssubsat
+}
+
+define i32 @subusat(i32 %a, i32 %b) {
+; RV32I-LABEL: subusat:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sub a1, a0, a1
+; RV32I-NEXT:    sltu a0, a0, a1
+; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIA-LABEL: subusat:
+; RV32IXQCIA:       # %bb.0:
+; RV32IXQCIA-NEXT:    qc.subusat a0, a0, a1
+; RV32IXQCIA-NEXT:    ret
+ %usubsat = tail call i32 @llvm.usub.sat.i32(i32 %a,i32 %b)
+ ret i32 %usubsat
+}
+
+define i32 @shlusat(i32 %a, i32 %b) {
+; RV32I-LABEL: shlusat:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sll a2, a0, a1
+; RV32I-NEXT:    srl a1, a2, a1
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIA-LABEL: shlusat:
+; RV32IXQCIA:       # %bb.0:
+; RV32IXQCIA-NEXT:    qc.shlusat a0, a0, a1
+; RV32IXQCIA-NEXT:    ret
+ %ushlsat = tail call i32 @llvm.ushl.sat.i32(i32 %a,i32 %b)
+ ret i32 %ushlsat
+}
+
+define i32 @shlsat(i32 %a, i32 %b) {
+; RV32I-LABEL: shlsat:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a2, a0
+; RV32I-NEXT:    sll a0, a0, a1
+; RV32I-NEXT:    sra a1, a0, a1
+; RV32I-NEXT:    beq a2, a1, .LBB5_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srai a2, a2, 31
+; RV32I-NEXT:    lui a0, 524288
+; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    xor a0, a2, a0
+; RV32I-NEXT:  .LBB5_2:
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIA-LABEL: shlsat:
+; RV32IXQCIA:       # %bb.0:
+; RV32IXQCIA-NEXT:    qc.shlsat a0, a0, a1
+; RV32IXQCIA-NEXT:    ret
+ %sshlsat = tail call i32 @llvm.sshl.sat.i32(i32 %a,i32 %b)
+ ret i32 %sshlsat
+}

From a095ebc58c9f7cd0d39796ed0aabc63b23ed88ce Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Tue, 22 Apr 2025 11:20:17 +0100
Subject: [PATCH 709/710] [LLVM][CostModel][AArch64] Remove magic numbers from
 f16 vector compares. (#135795)

The PR also extends the code to cover bfloat vector compares that are
also promoted to float.

NOTE: There is a bail out for the compares that are scalarised that will
be removed by https://github.com/llvm/llvm-project/pull/135398.
---
 .../AArch64/AArch64TargetTransformInfo.cpp    | 32 ++++++++++++++++---
 .../CostModel/AArch64/vector-select.ll        | 16 +++++-----
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 594f1bff5c458..59291c02e6555 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4236,10 +4236,34 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
   }
 
   if (isa<FixedVectorType>(ValTy) && ISD == ISD::SETCC) {
-    auto LT = getTypeLegalizationCost(ValTy);
-    // Cost v4f16 FCmp without FP16 support via converting to v4f32 and back.
-    if (LT.second == MVT::v4f16 && !ST->hasFullFP16())
-      return LT.first * 4; // fcvtl + fcvtl + fcmp + xtn
+    Type *ValScalarTy = ValTy->getScalarType();
+    if ((ValScalarTy->isHalfTy() && !ST->hasFullFP16()) ||
+        ValScalarTy->isBFloatTy()) {
+      auto *ValVTy = cast<FixedVectorType>(ValTy);
+
+      // FIXME: We currently scalarise these.
+      if (ValVTy->getNumElements() > 4)
+        return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred,
+                                         CostKind, Op1Info, Op2Info, I);
+
+      // Without dedicated instructions we promote [b]f16 compares to f32.
+      auto *PromotedTy =
+          VectorType::get(Type::getFloatTy(ValTy->getContext()), ValVTy);
+
+      InstructionCost Cost = 0;
+      // Promote operands to float vectors.
+      Cost += 2 * getCastInstrCost(Instruction::FPExt, PromotedTy, ValTy,
+                                   TTI::CastContextHint::None, CostKind);
+      // Compare float vectors.
+      Cost += getCmpSelInstrCost(Opcode, PromotedTy, CondTy, VecPred, CostKind,
+                                 Op1Info, Op2Info);
+      // During codegen we'll truncate the vector result from i32 to i16.
+      Cost +=
+          getCastInstrCost(Instruction::Trunc, VectorType::getInteger(ValVTy),
+                           VectorType::getInteger(PromotedTy),
+                           TTI::CastContextHint::None, CostKind);
+      return Cost;
+    }
   }
 
   // Treat the icmp in icmp(and, 0) as free, as we can make use of ands.
diff --git a/llvm/test/Analysis/CostModel/AArch64/vector-select.ll b/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
index c2256159a8ee2..e66f94dd54f21 100644
--- a/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
@@ -168,7 +168,7 @@ define <2 x double> @v2f64_select_ogt(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x bfloat> @v4bf16_select_ogt(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
 ; COST-LABEL: 'v4bf16_select_ogt'
-; COST-NEXT:  Cost Model: Found costs of 1 for: %cmp.1 = fcmp ogt <4 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ogt <4 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <4 x i1> %cmp.1, <4 x bfloat> %a, <4 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x bfloat> %s.1
 ;
@@ -255,7 +255,7 @@ define <2 x double> @v2f64_select_oge(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x bfloat> @v4bf16_select_oge(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
 ; COST-LABEL: 'v4bf16_select_oge'
-; COST-NEXT:  Cost Model: Found costs of 1 for: %cmp.1 = fcmp oge <4 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oge <4 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <4 x i1> %cmp.1, <4 x bfloat> %a, <4 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x bfloat> %s.1
 ;
@@ -342,7 +342,7 @@ define <2 x double> @v2f64_select_olt(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x bfloat> @v4bf16_select_olt(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
 ; COST-LABEL: 'v4bf16_select_olt'
-; COST-NEXT:  Cost Model: Found costs of 1 for: %cmp.1 = fcmp olt <4 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp olt <4 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <4 x i1> %cmp.1, <4 x bfloat> %a, <4 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x bfloat> %s.1
 ;
@@ -429,7 +429,7 @@ define <2 x double> @v2f64_select_ole(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x bfloat> @v4bf16_select_ole(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
 ; COST-LABEL: 'v4bf16_select_ole'
-; COST-NEXT:  Cost Model: Found costs of 1 for: %cmp.1 = fcmp ole <4 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ole <4 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <4 x i1> %cmp.1, <4 x bfloat> %a, <4 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x bfloat> %s.1
 ;
@@ -516,7 +516,7 @@ define <2 x double> @v2f64_select_oeq(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x bfloat> @v4bf16_select_oeq(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
 ; COST-LABEL: 'v4bf16_select_oeq'
-; COST-NEXT:  Cost Model: Found costs of 1 for: %cmp.1 = fcmp oeq <4 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oeq <4 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <4 x i1> %cmp.1, <4 x bfloat> %a, <4 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x bfloat> %s.1
 ;
@@ -603,7 +603,7 @@ define <2 x double> @v2f64_select_one(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x bfloat> @v4bf16_select_one(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
 ; COST-LABEL: 'v4bf16_select_one'
-; COST-NEXT:  Cost Model: Found costs of 1 for: %cmp.1 = fcmp one <4 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp one <4 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <4 x i1> %cmp.1, <4 x bfloat> %a, <4 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x bfloat> %s.1
 ;
@@ -690,7 +690,7 @@ define <2 x double> @v2f64_select_une(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x bfloat> @v4bf16_select_une(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
 ; COST-LABEL: 'v4bf16_select_une'
-; COST-NEXT:  Cost Model: Found costs of 1 for: %cmp.1 = fcmp une <4 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp une <4 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <4 x i1> %cmp.1, <4 x bfloat> %a, <4 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x bfloat> %s.1
 ;
@@ -777,7 +777,7 @@ define <2 x double> @v2f64_select_ord(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x bfloat> @v4bf16_select_ord(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
 ; COST-LABEL: 'v4bf16_select_ord'
-; COST-NEXT:  Cost Model: Found costs of 1 for: %cmp.1 = fcmp ord <4 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ord <4 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <4 x i1> %cmp.1, <4 x bfloat> %a, <4 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x bfloat> %s.1
 ;

From 59a25721f7f7c27e7e746450aa663e1ddb10c8bb Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Fri, 11 Apr 2025 16:17:55 +0100
Subject: [PATCH 710/710] [LLVM][CodeGen][AArch64] Don't scalarise v8{f16,bf16}
 vsetcc operations.

I have also removed custom promotion code for the v4{f16,bf16} cases
because the same common code can be used.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  36 +-
 .../AArch64/AArch64TargetTransformInfo.cpp    |   5 -
 llvm/test/Analysis/CostModel/AArch64/cmp.ll   |   4 +-
 .../CostModel/AArch64/vector-select.ll        |  32 +-
 .../CodeGen/AArch64/bf16-v8-instructions.ll   | 901 +++--------------
 llvm/test/CodeGen/AArch64/fcmp.ll             | 582 ++---------
 .../CodeGen/AArch64/fp16-v8-instructions.ll   | 949 +++---------------
 7 files changed, 442 insertions(+), 2067 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 771eee1b3fecf..5201e1a3c386c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -843,11 +843,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationPromotedToType(ISD::FRINT,      V4Narrow, MVT::v4f32);
     setOperationPromotedToType(ISD::FNEARBYINT, V4Narrow, MVT::v4f32);
     setOperationPromotedToType(ISD::FCANONICALIZE, V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::SETCC,         V4Narrow, MVT::v4f32);
 
     setOperationAction(ISD::FABS,        V4Narrow, Legal);
-    setOperationAction(ISD::FNEG, 	 V4Narrow, Legal);
+    setOperationAction(ISD::FNEG,        V4Narrow, Legal);
     setOperationAction(ISD::FMA,         V4Narrow, Expand);
-    setOperationAction(ISD::SETCC,       V4Narrow, Custom);
     setOperationAction(ISD::BR_CC,       V4Narrow, Expand);
     setOperationAction(ISD::SELECT,      V4Narrow, Expand);
     setOperationAction(ISD::SELECT_CC,   V4Narrow, Expand);
@@ -855,6 +855,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FSQRT,       V4Narrow, Expand);
 
     auto V8Narrow = MVT::getVectorVT(ScalarVT, 8);
+    setOperationPromotedToType(ISD::FCANONICALIZE, V8Narrow, MVT::v8f32);
+    setOperationPromotedToType(ISD::SETCC,         V8Narrow, MVT::v8f32);
+
     setOperationAction(ISD::FABS,        V8Narrow, Legal);
     setOperationAction(ISD::FADD,        V8Narrow, Legal);
     setOperationAction(ISD::FCEIL,       V8Narrow, Legal);
@@ -864,19 +867,17 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FMA,         V8Narrow, Expand);
     setOperationAction(ISD::FMUL,        V8Narrow, Legal);
     setOperationAction(ISD::FNEARBYINT,  V8Narrow, Legal);
-    setOperationAction(ISD::FNEG, 	 V8Narrow, Legal);
+    setOperationAction(ISD::FNEG,        V8Narrow, Legal);
     setOperationAction(ISD::FROUND,      V8Narrow, Legal);
     setOperationAction(ISD::FROUNDEVEN,  V8Narrow, Legal);
     setOperationAction(ISD::FRINT,       V8Narrow, Legal);
     setOperationAction(ISD::FSQRT,       V8Narrow, Expand);
     setOperationAction(ISD::FSUB,        V8Narrow, Legal);
     setOperationAction(ISD::FTRUNC,      V8Narrow, Legal);
-    setOperationAction(ISD::SETCC,       V8Narrow, Expand);
     setOperationAction(ISD::BR_CC,       V8Narrow, Expand);
     setOperationAction(ISD::SELECT,      V8Narrow, Expand);
     setOperationAction(ISD::SELECT_CC,   V8Narrow, Expand);
     setOperationAction(ISD::FP_EXTEND,   V8Narrow, Expand);
-    setOperationPromotedToType(ISD::FCANONICALIZE, V8Narrow, MVT::v8f32);
   };
 
   if (!Subtarget->hasFullFP16()) {
@@ -15904,6 +15905,11 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
   if (LHS.getValueType().getVectorElementType().isInteger())
     return Op;
 
+  assert((!Subtarget->hasFullFP16() &&
+          LHS.getValueType().getVectorElementType() != MVT::f16) ||
+         LHS.getValueType().getVectorElementType() != MVT::bf16 ||
+         LHS.getValueType().getVectorElementType() != MVT::f128);
+
   // Lower isnan(x) | isnan(never-nan) to x != x.
   // Lower !isnan(x) & !isnan(never-nan) to x == x.
   if (CC == ISD::SETUO || CC == ISD::SETO) {
@@ -15922,26 +15928,6 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
     }
   }
 
-  const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
-
-  // Make v4f16 (only) fcmp operations utilise vector instructions
-  // v8f16 support will be a litle more complicated
-  if ((!FullFP16 && LHS.getValueType().getVectorElementType() == MVT::f16) ||
-      LHS.getValueType().getVectorElementType() == MVT::bf16) {
-    if (LHS.getValueType().getVectorNumElements() == 4) {
-      LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, LHS);
-      RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, RHS);
-      SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC);
-      DAG.ReplaceAllUsesWith(Op, NewSetcc);
-      CmpVT = MVT::v4i32;
-    } else
-      return SDValue();
-  }
-
-  assert((!FullFP16 && LHS.getValueType().getVectorElementType() != MVT::f16) ||
-         LHS.getValueType().getVectorElementType() != MVT::bf16 ||
-         LHS.getValueType().getVectorElementType() != MVT::f128);
-
   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
   // clean.  Some of them require two branches to implement.
   AArch64CC::CondCode CC1, CC2;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 59291c02e6555..cb377aa007f31 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4241,11 +4241,6 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
         ValScalarTy->isBFloatTy()) {
       auto *ValVTy = cast<FixedVectorType>(ValTy);
 
-      // FIXME: We currently scalarise these.
-      if (ValVTy->getNumElements() > 4)
-        return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred,
-                                         CostKind, Op1Info, Op2Info, I);
-
       // Without dedicated instructions we promote [b]f16 compares to f32.
       auto *PromotedTy =
           VectorType::get(Type::getFloatTy(ValTy->getContext()), ValVTy);
diff --git a/llvm/test/Analysis/CostModel/AArch64/cmp.ll b/llvm/test/Analysis/CostModel/AArch64/cmp.ll
index f49d9be6b16b7..aba113865af10 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cmp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cmp.ll
@@ -17,10 +17,10 @@ define void @cmps() {
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %cf32 = fcmp ogt float undef, undef
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %cf64 = fcmp ogt double undef, undef
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %cbf64 = fcmp ogt bfloat undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cfv816 = fcmp olt <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cfv816 = fcmp olt <8 x half> undef, undef
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %cfv432 = fcmp oge <4 x float> undef, undef
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %cfv264 = fcmp oge <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cbfv816 = fcmp olt <8 x bfloat> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cbfv816 = fcmp olt <8 x bfloat> undef, undef
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c8 = icmp slt i8 undef, undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/vector-select.ll b/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
index e66f94dd54f21..d568f4b2c0b5c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
@@ -119,7 +119,7 @@ define <4 x half> @v4f16_select_ogt(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 define <8 x half> @v8f16_select_ogt(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-NOFP16-LABEL: 'v8f16_select_ogt'
-; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ogt <8 x half> %a, %b
+; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ogt <8 x half> %a, %b
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %s.1
 ;
@@ -179,7 +179,7 @@ define <4 x bfloat> @v4bf16_select_ogt(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bf
 
 define <8 x bfloat> @v8bf16_select_ogt(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) {
 ; COST-LABEL: 'v8bf16_select_ogt'
-; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ogt <8 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ogt <8 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x bfloat> %a, <8 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x bfloat> %s.1
 ;
@@ -206,7 +206,7 @@ define <4 x half> @v4f16_select_oge(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 define <8 x half> @v8f16_select_oge(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-NOFP16-LABEL: 'v8f16_select_oge'
-; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oge <8 x half> %a, %b
+; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oge <8 x half> %a, %b
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %s.1
 ;
@@ -266,7 +266,7 @@ define <4 x bfloat> @v4bf16_select_oge(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bf
 
 define <8 x bfloat> @v8bf16_select_oge(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) {
 ; COST-LABEL: 'v8bf16_select_oge'
-; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oge <8 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oge <8 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x bfloat> %a, <8 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x bfloat> %s.1
 ;
@@ -293,7 +293,7 @@ define <4 x half> @v4f16_select_olt(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 define <8 x half> @v8f16_select_olt(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-NOFP16-LABEL: 'v8f16_select_olt'
-; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp olt <8 x half> %a, %b
+; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp olt <8 x half> %a, %b
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %s.1
 ;
@@ -353,7 +353,7 @@ define <4 x bfloat> @v4bf16_select_olt(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bf
 
 define <8 x bfloat> @v8bf16_select_olt(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) {
 ; COST-LABEL: 'v8bf16_select_olt'
-; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp olt <8 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp olt <8 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x bfloat> %a, <8 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x bfloat> %s.1
 ;
@@ -380,7 +380,7 @@ define <4 x half> @v4f16_select_ole(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 define <8 x half> @v8f16_select_ole(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-NOFP16-LABEL: 'v8f16_select_ole'
-; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ole <8 x half> %a, %b
+; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ole <8 x half> %a, %b
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %s.1
 ;
@@ -440,7 +440,7 @@ define <4 x bfloat> @v4bf16_select_ole(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bf
 
 define <8 x bfloat> @v8bf16_select_ole(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) {
 ; COST-LABEL: 'v8bf16_select_ole'
-; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ole <8 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ole <8 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x bfloat> %a, <8 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x bfloat> %s.1
 ;
@@ -467,7 +467,7 @@ define <4 x half> @v4f16_select_oeq(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 define <8 x half> @v8f16_select_oeq(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-NOFP16-LABEL: 'v8f16_select_oeq'
-; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oeq <8 x half> %a, %b
+; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oeq <8 x half> %a, %b
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %s.1
 ;
@@ -527,7 +527,7 @@ define <4 x bfloat> @v4bf16_select_oeq(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bf
 
 define <8 x bfloat> @v8bf16_select_oeq(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) {
 ; COST-LABEL: 'v8bf16_select_oeq'
-; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oeq <8 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp oeq <8 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x bfloat> %a, <8 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x bfloat> %s.1
 ;
@@ -554,7 +554,7 @@ define <4 x half> @v4f16_select_one(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 define <8 x half> @v8f16_select_one(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-NOFP16-LABEL: 'v8f16_select_one'
-; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp one <8 x half> %a, %b
+; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp one <8 x half> %a, %b
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %s.1
 ;
@@ -614,7 +614,7 @@ define <4 x bfloat> @v4bf16_select_one(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bf
 
 define <8 x bfloat> @v8bf16_select_one(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) {
 ; COST-LABEL: 'v8bf16_select_one'
-; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp one <8 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp one <8 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x bfloat> %a, <8 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x bfloat> %s.1
 ;
@@ -641,7 +641,7 @@ define <4 x half> @v4f16_select_une(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 define <8 x half> @v8f16_select_une(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-NOFP16-LABEL: 'v8f16_select_une'
-; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp une <8 x half> %a, %b
+; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp une <8 x half> %a, %b
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %s.1
 ;
@@ -701,7 +701,7 @@ define <4 x bfloat> @v4bf16_select_une(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bf
 
 define <8 x bfloat> @v8bf16_select_une(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) {
 ; COST-LABEL: 'v8bf16_select_une'
-; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp une <8 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp une <8 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x bfloat> %a, <8 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x bfloat> %s.1
 ;
@@ -728,7 +728,7 @@ define <4 x half> @v4f16_select_ord(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 
 define <8 x half> @v8f16_select_ord(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-NOFP16-LABEL: 'v8f16_select_ord'
-; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ord <8 x half> %a, %b
+; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ord <8 x half> %a, %b
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %s.1
 ;
@@ -788,7 +788,7 @@ define <4 x bfloat> @v4bf16_select_ord(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bf
 
 define <8 x bfloat> @v8bf16_select_ord(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) {
 ; COST-LABEL: 'v8bf16_select_ord'
-; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ord <8 x bfloat> %a, %b
+; COST-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %cmp.1 = fcmp ord <8 x bfloat> %a, %b
 ; COST-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:1 Lat:1 SizeLat:1 for: %s.1 = select <8 x i1> %cmp.1, <8 x bfloat> %a, <8 x bfloat> %c
 ; COST-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x bfloat> %s.1
 ;
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index 3a55b68f2d1a3..715693dd6ed07 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -882,61 +882,17 @@ define <8 x i16> @fptoui_i16(<8 x bfloat> %a) #0 {
 define <8 x i1> @test_fcmp_une(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_une:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, ne
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, ne
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, ne
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, ne
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, ne
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, ne
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, ne
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, ne
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmeq v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp une <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -945,69 +901,21 @@ define <8 x i1> @test_fcmp_une(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_ueq(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_ueq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, eq
-; CHECK-NEXT:    csinv w9, w9, wzr, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmgt v4.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmgt v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    fcmgt v3.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp ueq <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1016,61 +924,17 @@ define <8 x i1> @test_fcmp_ueq(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_ugt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_ugt:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, hi
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, hi
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, hi
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, hi
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, hi
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v1.8h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, hi
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, hi
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, hi
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcmge v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp ugt <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1079,61 +943,17 @@ define <8 x i1> @test_fcmp_ugt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_uge(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_uge:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, pl
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, pl
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, pl
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, pl
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, pl
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v1.8h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, pl
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, pl
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, pl
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp uge <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1142,61 +962,17 @@ define <8 x i1> @test_fcmp_uge(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_ult(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_ult:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, lt
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, lt
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, lt
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, lt
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, lt
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, lt
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, lt
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, lt
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmge v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp ult <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1205,61 +981,17 @@ define <8 x i1> @test_fcmp_ult(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_ule(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_ule:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, le
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, le
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, le
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, le
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp ule <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1268,61 +1000,21 @@ define <8 x i1> @test_fcmp_ule(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_uno(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_uno:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, vs
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, vs
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, vs
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, vs
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, vs
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, vs
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, vs
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, vs
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmge v4.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmgt v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    fcmge v3.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp uno <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1331,69 +1023,20 @@ define <8 x i1> @test_fcmp_uno(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_one(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_one:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    csinv w8, w8, wzr, le
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, mi
-; CHECK-NEXT:    csinv w9, w9, wzr, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    csinv w8, w8, wzr, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    csinv w8, w8, wzr, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    csinv w8, w8, wzr, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    csinv w8, w8, wzr, le
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    csinv w8, w8, wzr, le
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    csinv w8, w8, wzr, le
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmgt v4.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmgt v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    fcmgt v3.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp one <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1402,61 +1045,16 @@ define <8 x i1> @test_fcmp_one(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_oeq(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_oeq:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, eq
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, eq
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmeq v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp oeq <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1465,61 +1063,16 @@ define <8 x i1> @test_fcmp_oeq(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_ogt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_ogt:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, gt
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, gt
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, gt
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, gt
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, gt
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, gt
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, gt
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, gt
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp ogt <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1528,61 +1081,16 @@ define <8 x i1> @test_fcmp_ogt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_oge(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_oge:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, ge
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, ge
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, ge
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, ge
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, ge
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, ge
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, ge
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, ge
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmge v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp oge <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1591,61 +1099,16 @@ define <8 x i1> @test_fcmp_oge(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_olt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_olt:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, mi
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v1.8h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, mi
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp olt <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1654,61 +1117,16 @@ define <8 x i1> @test_fcmp_olt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_ole(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_ole:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, ls
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, ls
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, ls
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, ls
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, ls
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v1.8h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, ls
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, ls
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, ls
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-NEXT:    fcmge v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp ole <8 x bfloat> %a, %b
   ret <8 x i1> %1
@@ -1717,61 +1135,20 @@ define <8 x i1> @test_fcmp_ole(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 define <8 x i1> @test_fcmp_ord(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fcmp_ord:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v2.4h, v1.h[1]
-; CHECK-NEXT:    dup v3.4h, v0.h[1]
-; CHECK-NEXT:    dup v4.4h, v1.h[2]
-; CHECK-NEXT:    dup v5.4h, v0.h[2]
-; CHECK-NEXT:    dup v6.4h, v0.h[3]
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v2.4s, v1.4h, #16
-; CHECK-NEXT:    shll v3.4s, v0.4h, #16
-; CHECK-NEXT:    csetm w8, vc
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    shll v3.4s, v4.4h, #16
-; CHECK-NEXT:    shll v4.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.4h, v1.h[3]
-; CHECK-NEXT:    csetm w9, vc
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[4]
-; CHECK-NEXT:    dup v6.8h, v0.h[4]
-; CHECK-NEXT:    mov v2.h[1], w8
-; CHECK-NEXT:    csetm w8, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[5]
-; CHECK-NEXT:    dup v6.8h, v0.h[5]
-; CHECK-NEXT:    mov v2.h[2], w8
-; CHECK-NEXT:    csetm w8, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
-; CHECK-NEXT:    dup v5.8h, v1.h[6]
-; CHECK-NEXT:    dup v6.8h, v0.h[6]
-; CHECK-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-NEXT:    dup v0.8h, v0.h[7]
-; CHECK-NEXT:    mov v2.h[3], w8
-; CHECK-NEXT:    csetm w8, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    shll v3.4s, v5.4h, #16
-; CHECK-NEXT:    shll v4.4s, v6.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v0.8h, #16
 ; CHECK-NEXT:    shll v1.4s, v1.4h, #16
 ; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov v2.h[4], w8
-; CHECK-NEXT:    csetm w8, vc
-; CHECK-NEXT:    fcmp s4, s3
-; CHECK-NEXT:    mov v2.h[5], w8
-; CHECK-NEXT:    csetm w8, vc
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    mov v2.h[6], w8
-; CHECK-NEXT:    csetm w8, vc
-; CHECK-NEXT:    mov v2.h[7], w8
-; CHECK-NEXT:    xtn v0.8b, v2.8h
+; CHECK-NEXT:    fcmge v4.4s, v3.4s, v2.4s
+; CHECK-NEXT:    fcmgt v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    fcmge v3.4s, v0.4s, v1.4s
+; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    ret
   %1 = fcmp ord <8 x bfloat> %a, %b
   ret <8 x i1> %1
diff --git a/llvm/test/CodeGen/AArch64/fcmp.ll b/llvm/test/CodeGen/AArch64/fcmp.ll
index 66f26fc9d8597..fa0cb46e16bb3 100644
--- a/llvm/test/CodeGen/AArch64/fcmp.ll
+++ b/llvm/test/CodeGen/AArch64/fcmp.ll
@@ -1138,61 +1138,15 @@ entry:
 define <7 x half> @v7f16_half(<7 x half> %a, <7 x half> %b, <7 x half> %d, <7 x half> %e) {
 ; CHECK-SD-NOFP16-LABEL: v7f16_half:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h0
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcmp s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s7, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    csetm w9, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s4, s7
-; CHECK-SD-NOFP16-NEXT:    fmov s4, w9
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcmp s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s6, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[4], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[5], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[6], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[7], w8
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v4.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcmgt v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
+; CHECK-SD-NOFP16-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-SD-NOFP16-NEXT:    cmlt v0.8h, v0.8h, #0
 ; CHECK-SD-NOFP16-NEXT:    bsl v0.16b, v2.16b, v3.16b
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -1314,61 +1268,15 @@ entry:
 define <8 x half> @v8f16_half(<8 x half> %a, <8 x half> %b, <8 x half> %d, <8 x half> %e) {
 ; CHECK-SD-NOFP16-LABEL: v8f16_half:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h0
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcmp s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s7, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    csetm w9, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s4, s7
-; CHECK-SD-NOFP16-NEXT:    fmov s4, w9
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcmp s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s6, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[4], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[5], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[6], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[7], w8
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v4.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcmgt v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
+; CHECK-SD-NOFP16-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-SD-NOFP16-NEXT:    cmlt v0.8h, v0.8h, #0
 ; CHECK-SD-NOFP16-NEXT:    bsl v0.16b, v2.16b, v3.16b
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -1406,116 +1314,25 @@ entry:
 define <16 x half> @v16f16_half(<16 x half> %a, <16 x half> %b, <16 x half> %d, <16 x half> %e) {
 ; CHECK-SD-NOFP16-LABEL: v16f16_half:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s21, s20
-; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    csetm w14, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    csetm w13, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    csetm w11, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    csetm w12, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    csetm w10, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    csetm w9, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s1, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    csetm w15, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    csetm w16, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s3, s1
-; CHECK-SD-NOFP16-NEXT:    fmov s1, w14
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    csetm w14, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmov s3, w14
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], w16
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], w13
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], w11
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], w12
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], w10
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], w9
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], w15
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], w8
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v16.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v17.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v18.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v19.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcmgt v16.4s, v17.4s, v16.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v1.4s, v3.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v3.4s, v19.4s, v18.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v0.4s, v2.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v16.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-SD-NOFP16-NEXT:    shl v1.8h, v1.8h, #15
+; CHECK-SD-NOFP16-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-SD-NOFP16-NEXT:    cmlt v1.8h, v1.8h, #0
+; CHECK-SD-NOFP16-NEXT:    cmlt v0.8h, v0.8h, #0
 ; CHECK-SD-NOFP16-NEXT:    bsl v1.16b, v5.16b, v7.16b
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
 ; CHECK-SD-NOFP16-NEXT:    bsl v0.16b, v4.16b, v6.16b
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -1567,89 +1384,44 @@ entry:
 define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32> %e) {
 ; CHECK-SD-NOFP16-LABEL: v7f16_i32:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcmp s3, s2
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    csetm w9, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    csetm w10, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    csetm w11, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s7, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    csetm w12, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h5
-; CHECK-SD-NOFP16-NEXT:    fmov s4, w9
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    add x9, sp, #8
-; CHECK-SD-NOFP16-NEXT:    csetm w13, mi
-; CHECK-SD-NOFP16-NEXT:    fmov s5, w13
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], w8
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v1.8h
 ; CHECK-SD-NOFP16-NEXT:    mov x8, sp
-; CHECK-SD-NOFP16-NEXT:    fcmp s3, s2
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    ldr s4, [sp, #24]
+; CHECK-SD-NOFP16-NEXT:    add x9, sp, #32
+; CHECK-SD-NOFP16-NEXT:    ld1 { v4.s }[1], [x9]
+; CHECK-SD-NOFP16-NEXT:    add x9, sp, #16
+; CHECK-SD-NOFP16-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmov s3, w4
+; CHECK-SD-NOFP16-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fmov s1, w0
+; CHECK-SD-NOFP16-NEXT:    mov v3.s[1], w5
+; CHECK-SD-NOFP16-NEXT:    mov v1.s[1], w1
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
 ; CHECK-SD-NOFP16-NEXT:    fmov s2, w7
-; CHECK-SD-NOFP16-NEXT:    fmov s3, w0
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[1], w12
+; CHECK-SD-NOFP16-NEXT:    mov v3.s[2], w6
 ; CHECK-SD-NOFP16-NEXT:    ld1 { v2.s }[1], [x8]
-; CHECK-SD-NOFP16-NEXT:    mov v3.s[1], w1
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], w10
-; CHECK-SD-NOFP16-NEXT:    fcmp s0, s1
-; CHECK-SD-NOFP16-NEXT:    fmov s1, w4
-; CHECK-SD-NOFP16-NEXT:    ldr s0, [sp, #24]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[2], w8
-; CHECK-SD-NOFP16-NEXT:    ld1 { v2.s }[2], [x9]
-; CHECK-SD-NOFP16-NEXT:    add x9, sp, #32
-; CHECK-SD-NOFP16-NEXT:    mov v3.s[2], w2
-; CHECK-SD-NOFP16-NEXT:    mov v1.s[1], w5
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], w11
-; CHECK-SD-NOFP16-NEXT:    ld1 { v0.s }[1], [x9]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[3], w8
-; CHECK-SD-NOFP16-NEXT:    add x8, sp, #16
-; CHECK-SD-NOFP16-NEXT:    ld1 { v2.s }[3], [x8]
-; CHECK-SD-NOFP16-NEXT:    mov v3.s[3], w3
+; CHECK-SD-NOFP16-NEXT:    mov v1.s[2], w2
+; CHECK-SD-NOFP16-NEXT:    add x8, sp, #8
+; CHECK-SD-NOFP16-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-SD-NOFP16-NEXT:    ld1 { v2.s }[2], [x8]
 ; CHECK-SD-NOFP16-NEXT:    add x8, sp, #40
-; CHECK-SD-NOFP16-NEXT:    mov v1.s[2], w6
-; CHECK-SD-NOFP16-NEXT:    sshll v4.4s, v4.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ld1 { v0.s }[2], [x8]
-; CHECK-SD-NOFP16-NEXT:    sshll v5.4s, v5.4h, #0
-; CHECK-SD-NOFP16-NEXT:    bit v2.16b, v3.16b, v4.16b
-; CHECK-SD-NOFP16-NEXT:    bit v0.16b, v1.16b, v5.16b
-; CHECK-SD-NOFP16-NEXT:    mov w1, v2.s[1]
-; CHECK-SD-NOFP16-NEXT:    mov w2, v2.s[2]
-; CHECK-SD-NOFP16-NEXT:    mov w3, v2.s[3]
-; CHECK-SD-NOFP16-NEXT:    fmov w0, s2
+; CHECK-SD-NOFP16-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-SD-NOFP16-NEXT:    mov v1.s[3], w3
+; CHECK-SD-NOFP16-NEXT:    ld1 { v4.s }[2], [x8]
+; CHECK-SD-NOFP16-NEXT:    ld1 { v2.s }[3], [x9]
+; CHECK-SD-NOFP16-NEXT:    sshll v5.4s, v0.4h, #0
+; CHECK-SD-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-SD-NOFP16-NEXT:    bif v1.16b, v2.16b, v5.16b
+; CHECK-SD-NOFP16-NEXT:    bsl v0.16b, v3.16b, v4.16b
+; CHECK-SD-NOFP16-NEXT:    mov w1, v1.s[1]
+; CHECK-SD-NOFP16-NEXT:    mov w2, v1.s[2]
+; CHECK-SD-NOFP16-NEXT:    mov w3, v1.s[3]
 ; CHECK-SD-NOFP16-NEXT:    mov w5, v0.s[1]
 ; CHECK-SD-NOFP16-NEXT:    mov w6, v0.s[2]
+; CHECK-SD-NOFP16-NEXT:    fmov w0, s1
 ; CHECK-SD-NOFP16-NEXT:    fmov w4, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -1858,62 +1630,19 @@ entry:
 define <8 x i32> @v8f16_i32(<8 x half> %a, <8 x half> %b, <8 x i32> %d, <8 x i32> %e) {
 ; CHECK-SD-NOFP16-LABEL: v8f16_i32:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcmp s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    csetm w9, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    csetm w10, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s7, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h0
-; CHECK-SD-NOFP16-NEXT:    csetm w11, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    csetm w12, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s7, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fmov s16, w9
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    csetm w13, mi
-; CHECK-SD-NOFP16-NEXT:    fmov s17, w13
-; CHECK-SD-NOFP16-NEXT:    mov v16.h[1], w8
-; CHECK-SD-NOFP16-NEXT:    fcmp s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov v17.h[1], w12
-; CHECK-SD-NOFP16-NEXT:    mov v16.h[2], w10
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v17.h[2], w8
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    mov v16.h[3], w11
-; CHECK-SD-NOFP16-NEXT:    mov v17.h[3], w8
-; CHECK-SD-NOFP16-NEXT:    sshll v1.4s, v16.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v0.4s, v17.4h, #0
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcmgt v6.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v6.8h
+; CHECK-SD-NOFP16-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-SD-NOFP16-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-SD-NOFP16-NEXT:    sshll v6.4s, v0.4h, #0
+; CHECK-SD-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v6.16b
 ; CHECK-SD-NOFP16-NEXT:    bsl v1.16b, v3.16b, v5.16b
 ; CHECK-SD-NOFP16-NEXT:    bsl v0.16b, v2.16b, v4.16b
 ; CHECK-SD-NOFP16-NEXT:    ret
@@ -1965,126 +1694,37 @@ entry:
 define <16 x i32> @v16f16_i32(<16 x half> %a, <16 x half> %b, <16 x i32> %d, <16 x i32> %e) {
 ; CHECK-SD-NOFP16-LABEL: v16f16_i32:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    csetm w10, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    csetm w9, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    csetm w11, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    csetm w12, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    csetm w14, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    csetm w13, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s1, s3
-; CHECK-SD-NOFP16-NEXT:    mov h1, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    csetm w15, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    csetm w16, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    csetm w17, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s3, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h0
-; CHECK-SD-NOFP16-NEXT:    csetm w18, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s19, s18
-; CHECK-SD-NOFP16-NEXT:    fmov s18, w14
-; CHECK-SD-NOFP16-NEXT:    fmov s19, w17
-; CHECK-SD-NOFP16-NEXT:    csetm w0, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s3, s1
-; CHECK-SD-NOFP16-NEXT:    mov h1, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov v18.h[1], w12
-; CHECK-SD-NOFP16-NEXT:    mov v19.h[1], w16
-; CHECK-SD-NOFP16-NEXT:    csetm w1, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s17, s16
-; CHECK-SD-NOFP16-NEXT:    fmov s16, w10
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    csetm w2, mi
-; CHECK-SD-NOFP16-NEXT:    mov v16.h[1], w8
-; CHECK-SD-NOFP16-NEXT:    mov v18.h[2], w13
-; CHECK-SD-NOFP16-NEXT:    fmov s17, w2
-; CHECK-SD-NOFP16-NEXT:    mov v19.h[2], w18
-; CHECK-SD-NOFP16-NEXT:    fcmp s3, s1
-; CHECK-SD-NOFP16-NEXT:    mov v17.h[1], w1
-; CHECK-SD-NOFP16-NEXT:    mov v16.h[2], w9
-; CHECK-SD-NOFP16-NEXT:    mov v18.h[3], w15
-; CHECK-SD-NOFP16-NEXT:    mov v19.h[3], w0
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    fcmp s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov v17.h[2], w8
-; CHECK-SD-NOFP16-NEXT:    mov v16.h[3], w11
-; CHECK-SD-NOFP16-NEXT:    csetm w8, mi
-; CHECK-SD-NOFP16-NEXT:    mov v17.h[3], w8
-; CHECK-SD-NOFP16-NEXT:    sshll v2.4s, v16.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v16.4s, v18.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ldp q0, q18, [sp]
-; CHECK-SD-NOFP16-NEXT:    sshll v1.4s, v17.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v17.4s, v19.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ldp q19, q3, [sp, #32]
-; CHECK-SD-NOFP16-NEXT:    bit v0.16b, v4.16b, v1.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v17.16b
-; CHECK-SD-NOFP16-NEXT:    bit v3.16b, v7.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v17.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v18.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v16.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v19.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcmgt v17.4s, v18.4s, v17.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v0.4s, v2.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v2.4s, v19.4s, v16.4s
+; CHECK-SD-NOFP16-NEXT:    fcmgt v1.4s, v3.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v17.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NOFP16-NEXT:    ldp q2, q20, [sp]
+; CHECK-SD-NOFP16-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-SD-NOFP16-NEXT:    shl v1.8h, v1.8h, #15
+; CHECK-SD-NOFP16-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-SD-NOFP16-NEXT:    cmlt v1.8h, v1.8h, #0
+; CHECK-SD-NOFP16-NEXT:    sshll v3.4s, v0.4h, #0
+; CHECK-SD-NOFP16-NEXT:    sshll v16.4s, v1.4h, #0
+; CHECK-SD-NOFP16-NEXT:    sshll2 v17.4s, v1.8h, #0
+; CHECK-SD-NOFP16-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    mov v3.16b, v17.16b
+; CHECK-SD-NOFP16-NEXT:    bsl v1.16b, v5.16b, v20.16b
+; CHECK-SD-NOFP16-NEXT:    bsl v0.16b, v4.16b, v2.16b
 ; CHECK-SD-NOFP16-NEXT:    mov v2.16b, v16.16b
-; CHECK-SD-NOFP16-NEXT:    bsl v1.16b, v5.16b, v18.16b
-; CHECK-SD-NOFP16-NEXT:    bsl v2.16b, v6.16b, v19.16b
+; CHECK-SD-NOFP16-NEXT:    bsl v3.16b, v7.16b, v19.16b
+; CHECK-SD-NOFP16-NEXT:    bsl v2.16b, v6.16b, v18.16b
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: v16f16_i32:
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index d4130e7a848b1..34788827d3075 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -514,61 +514,17 @@ define <8 x i16> @fptoui_i16(<8 x half> %a) #0 {
 define <8 x i1> @test_fcmp_une(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_une:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, ne
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, ne
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, ne
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, ne
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, ne
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, ne
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, ne
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, ne
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmeq v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    mvn v0.16b, v0.16b
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_une:
@@ -584,69 +540,21 @@ define <8 x i1> @test_fcmp_une(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_ueq(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_ueq:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s6, h0
-; CHECK-CVT-NEXT:    mov h5, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    mov h5, v0.h[3]
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-CVT-NEXT:    fcmp s6, s4
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v1.h[3]
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    csetm w9, eq
-; CHECK-CVT-NEXT:    csinv w9, w9, wzr, vc
-; CHECK-CVT-NEXT:    fcmp s2, s3
-; CHECK-CVT-NEXT:    mov h2, v0.h[4]
-; CHECK-CVT-NEXT:    fcvt s3, h4
-; CHECK-CVT-NEXT:    fcvt s4, h5
-; CHECK-CVT-NEXT:    fmov s5, w9
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v5.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    mov v5.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-CVT-NEXT:    fcmp s2, s6
-; CHECK-CVT-NEXT:    fcvt s2, h3
-; CHECK-CVT-NEXT:    fcvt s3, h4
-; CHECK-CVT-NEXT:    mov h4, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v5.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    fcvt s2, h4
-; CHECK-CVT-NEXT:    fcvt s3, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v5.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov v5.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v5.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, vc
-; CHECK-CVT-NEXT:    mov v5.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v5.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmgt v4.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmgt v2.4s, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    fcmgt v3.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-CVT-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-CVT-NEXT:    mvn v0.16b, v0.16b
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_ueq:
@@ -664,61 +572,17 @@ define <8 x i1> @test_fcmp_ueq(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_ugt(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_ugt:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, hi
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, hi
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, hi
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, hi
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, hi
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, hi
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, hi
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, hi
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcmge v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    mvn v0.16b, v0.16b
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_ugt:
@@ -734,61 +598,17 @@ define <8 x i1> @test_fcmp_ugt(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_uge(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_uge:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, pl
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, pl
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, pl
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, pl
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, pl
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, pl
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, pl
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, pl
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    mvn v0.16b, v0.16b
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_uge:
@@ -804,61 +624,17 @@ define <8 x i1> @test_fcmp_uge(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_ult(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_ult:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, lt
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, lt
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, lt
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, lt
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, lt
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, lt
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, lt
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, lt
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmge v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    mvn v0.16b, v0.16b
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_ult:
@@ -874,61 +650,17 @@ define <8 x i1> @test_fcmp_ult(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_ule(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_ule:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, le
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, le
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, le
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, le
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, le
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, le
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, le
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, le
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    mvn v0.16b, v0.16b
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_ule:
@@ -944,61 +676,21 @@ define <8 x i1> @test_fcmp_ule(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_uno(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_uno:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, vs
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, vs
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, vs
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, vs
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, vs
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, vs
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, vs
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, vs
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmge v4.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmgt v2.4s, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    fcmge v3.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-CVT-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-CVT-NEXT:    mvn v0.16b, v0.16b
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_uno:
@@ -1016,69 +708,20 @@ define <8 x i1> @test_fcmp_uno(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_one(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_one:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s6, h0
-; CHECK-CVT-NEXT:    mov h5, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    mov h5, v0.h[3]
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, le
-; CHECK-CVT-NEXT:    fcmp s6, s4
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v1.h[3]
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    csetm w9, mi
-; CHECK-CVT-NEXT:    csinv w9, w9, wzr, le
-; CHECK-CVT-NEXT:    fcmp s2, s3
-; CHECK-CVT-NEXT:    mov h2, v0.h[4]
-; CHECK-CVT-NEXT:    fcvt s3, h4
-; CHECK-CVT-NEXT:    fcvt s4, h5
-; CHECK-CVT-NEXT:    fmov s5, w9
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v5.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, le
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    mov v5.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, le
-; CHECK-CVT-NEXT:    fcmp s2, s6
-; CHECK-CVT-NEXT:    fcvt s2, h3
-; CHECK-CVT-NEXT:    fcvt s3, h4
-; CHECK-CVT-NEXT:    mov h4, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v5.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, le
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    fcvt s2, h4
-; CHECK-CVT-NEXT:    fcvt s3, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v5.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, le
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov v5.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, le
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v5.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    csinv w8, w8, wzr, le
-; CHECK-CVT-NEXT:    mov v5.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v5.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmgt v4.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmgt v2.4s, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    fcmgt v3.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-CVT-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_one:
@@ -1095,61 +738,16 @@ define <8 x i1> @test_fcmp_one(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_oeq(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_oeq:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, eq
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, eq
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmeq v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_oeq:
@@ -1164,61 +762,16 @@ define <8 x i1> @test_fcmp_oeq(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_ogt(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_ogt:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, gt
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, gt
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, gt
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, gt
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, gt
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, gt
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, gt
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, gt
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_ogt:
@@ -1233,61 +786,16 @@ define <8 x i1> @test_fcmp_ogt(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_oge(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_oge:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, ge
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, ge
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, ge
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, ge
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, ge
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, ge
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, ge
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, ge
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmge v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_oge:
@@ -1302,61 +810,16 @@ define <8 x i1> @test_fcmp_oge(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_olt(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_olt:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, mi
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, mi
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_olt:
@@ -1371,61 +834,16 @@ define <8 x i1> @test_fcmp_olt(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_ole(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_ole:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, ls
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, ls
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, ls
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, ls
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, ls
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, ls
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, ls
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, ls
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcmge v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_ole:
@@ -1440,61 +858,20 @@ define <8 x i1> @test_fcmp_ole(<8 x half> %a, <8 x half> %b) #0 {
 define <8 x i1> @test_fcmp_ord(<8 x half> %a, <8 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_ord:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcmp s3, s2
-; CHECK-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-CVT-NEXT:    mov h3, v1.h[3]
-; CHECK-CVT-NEXT:    csetm w8, vc
-; CHECK-CVT-NEXT:    fcmp s5, s4
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    mov h4, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    mov h6, v0.h[4]
-; CHECK-CVT-NEXT:    csetm w9, vc
-; CHECK-CVT-NEXT:    fcmp s2, s5
-; CHECK-CVT-NEXT:    fmov s2, w9
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    mov h5, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[1], w8
-; CHECK-CVT-NEXT:    csetm w8, vc
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov h3, v1.h[5]
-; CHECK-CVT-NEXT:    mov h4, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[2], w8
-; CHECK-CVT-NEXT:    csetm w8, vc
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcmp s6, s5
-; CHECK-CVT-NEXT:    mov h5, v1.h[6]
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], w8
-; CHECK-CVT-NEXT:    csetm w8, vc
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    fcvt s3, h5
-; CHECK-CVT-NEXT:    fcvt s4, h6
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], w8
-; CHECK-CVT-NEXT:    csetm w8, vc
-; CHECK-CVT-NEXT:    fcmp s4, s3
-; CHECK-CVT-NEXT:    mov v2.h[5], w8
-; CHECK-CVT-NEXT:    csetm w8, vc
-; CHECK-CVT-NEXT:    fcmp s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[6], w8
-; CHECK-CVT-NEXT:    csetm w8, vc
-; CHECK-CVT-NEXT:    mov v2.h[7], w8
-; CHECK-CVT-NEXT:    xtn v0.8b, v2.8h
+; CHECK-CVT-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcmge v4.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fcmgt v2.4s, v2.4s, v3.4s
+; CHECK-CVT-NEXT:    fcmge v3.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-CVT-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-CVT-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-CVT-NEXT:    shl v0.8h, v0.8h, #15
+; CHECK-CVT-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: test_fcmp_ord: